From f6f2e89faf1a03356b628cbff7d6fb0a47274b8e Mon Sep 17 00:00:00 2001
From: bz <bz@FreeBSD.org>
Date: Thu, 13 Aug 2009 15:02:02 +0000
Subject: MFC r196185:   Rather than replicating the maths from the kernel, use
 the   value the kernel calculated directly as we already read it   with
 struct vnet.  This will make kvm_vnet.c more resilent   in case of possible
 kernel changes.

  Reviewed by:  rwatson

Approved by:	re (kib)
---
 lib/libkvm/kvm_vnet.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libkvm/kvm_vnet.c b/lib/libkvm/kvm_vnet.c
index 1a02cad..d192c67 100644
--- a/lib/libkvm/kvm_vnet.c
+++ b/lib/libkvm/kvm_vnet.c
@@ -195,7 +195,7 @@ _kvm_vnet_selectpid(kvm_t *kd, pid_t pid)
 	kd->vnet_start = nl[NLIST_START_VNET].n_value;
 	kd->vnet_stop = nl[NLIST_STOP_VNET].n_value;
 	kd->vnet_current = (uintptr_t)prison.pr_vnet;
-	kd->vnet_base = (uintptr_t)vnet.vnet_data_mem - kd->vnet_start;
+	kd->vnet_base = vnet.vnet_data_base;
 	return (0);
 }
 
-- 
cgit v1.1


From e84e2e363118a59d3501ec21bf59d8e2776cb88a Mon Sep 17 00:00:00 2001
From: rwatson <rwatson@FreeBSD.org>
Date: Thu, 13 Aug 2009 15:08:05 +0000
Subject: Merge r196123 from head to stable/8:

  Update posix1e-related man pages, especially as relates to MAC, to more
  accurately reflect the last ten years of work.

Approved by:	re (kib)
---
 lib/libc/posix1e/Makefile.inc |   4 +-
 lib/libc/posix1e/mac.3        | 153 ++++++++++++++++++++----------------------
 lib/libc/posix1e/mac_free.3   |   9 +--
 lib/libc/posix1e/mac_get.3    |   1 +
 lib/libc/posix1e/mac_set.3    |   1 +
 lib/libc/posix1e/mac_text.3   |   1 +
 lib/libc/posix1e/posix1e.3    | 116 +++++++++++++-------------------
 7 files changed, 129 insertions(+), 156 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/posix1e/Makefile.inc b/lib/libc/posix1e/Makefile.inc
index de0302b..c9e78bd 100644
--- a/lib/libc/posix1e/Makefile.inc
+++ b/lib/libc/posix1e/Makefile.inc
@@ -106,14 +106,16 @@ MLINKS+=acl_create_entry.3 acl_create_entry_np.3\
 	mac_get.3 mac_get_fd.3			\
 	mac_get.3 mac_get_file.3		\
 	mac_get.3 mac_get_link.3		\
+	mac_get.3 mac_get_peer.3		\
 	mac_get.3 mac_get_pid.3			\
 	mac_get.3 mac_get_proc.3		\
 	mac_prepare.3 mac_prepare_file_label.3	\
 	mac_prepare.3 mac_prepare_ifnet_label.3	\
 	mac_prepare.3 mac_prepare_process_label.3	\
-	mac_set.3 mac_set_link.3		\
+	mac_prepare.3 mac_prepare_type.3	\
 	mac_set.3 mac_set_fd.3			\
 	mac_set.3 mac_set_file.3		\
+	mac_set.3 mac_set_link.3		\
 	mac_set.3 mac_set_proc.3		\
 	mac_text.3 mac_from_text.3		\
 	mac_text.3 mac_to_text.3
diff --git a/lib/libc/posix1e/mac.3 b/lib/libc/posix1e/mac.3
index ac6affd..c570998 100644
--- a/lib/libc/posix1e/mac.3
+++ b/lib/libc/posix1e/mac.3
@@ -1,4 +1,5 @@
 .\" Copyright (c) 2001, 2003 Networks Associates Technology, Inc.
+.\" Copyright (c) 2009 Robert N. M. Watson
 .\" All rights reserved.
 .\"
 .\" This software was developed for the FreeBSD Project by Chris
@@ -30,7 +31,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd April 19, 2003
+.Dd August 7, 2009
 .Dt MAC 3
 .Os
 .Sh NAME
@@ -44,81 +45,82 @@
 In the kernel configuration file:
 .Cd "options MAC"
 .Sh DESCRIPTION
-.Fx
-permits administrators to define Mandatory Access Control labels
-defining levels for the privacy and integrity of data,
-overriding discretionary policies
-for those objects.
-Not all objects currently provide support for MAC labels,
-and MAC support must be explicitly enabled by the administrator.
-The library calls include routines to retrieve, duplicate,
-and set MAC labels associated with files and processes.
+Mandatory Access Control labels describe confidentiality, integrity, and
+other security attributes of operating system objects, overriding
+discretionary access control.
+Not all system objects support MAC labeling, and MAC policies must be
+explicitly enabled by the administrator.
+This API, based on POSIX.1e, includes routines to retrieve, manipulate, set,
+and convert to and from text the MAC labels on files and processes.
 .Pp
-POSIX.1e describes a set of MAC manipulation routines
-to manage the contents of MAC labels,
-as well as their relationships with
-files and processes;
-almost all of these support routines
-are implemented in
-.Fx .
+MAC labels consist of a set of (name, value) tuples, representing security
+attributes from MAC policies.
+For example, this label contains security labels defined by two policies,
+.Xr mac_biba 4
+and
+.Xr mac_mls 4 :
+.Bd -literal -offset indent
+biba/low,mls/low
+.Ed
+.Pp
+Further syntax and semantics of MAC labels may be found in
+.Xr maclabel 7 .
 .Pp
-Available functions, sorted by behavior, include:
+Applications operate on labels stored in
+.Vt mac_t ,
+but can convert between this internal format and a text format for the
+purposes of presentation to uses or external storage.
+When querying a label on an object, a
+.Vt mac_t
+must first be prepared using the interfaces described in
+.Xr mac_prepare 3 ,
+allowing the application to declare which policies it wishes to interogate.
+The application writer can also rely on default label names declared in
+.Xr mac.conf 5 .
+.Pp
+When finished with a
+.Vt mac_t ,
+the application must call
+.Xr mac_free 3
+to release its storage.
+.Pp
+The following functions are defined:
 .Bl -tag -width indent
-.It Fn mac_get_fd
-This function is described in
-.Xr mac_get 3 ,
-and may be used to retrieve the
-MAC label associated with
-a specific file descriptor.
-.It Fn mac_get_file
-This function is described in
+.It Fn mac_is_present
+This function, described in
+.Xr mac_is_present 3 ,
+allows applications to test whether MAC is configured, as well as whether
+specific policies are configured.
+.It Fn mac_get_fd , Fn mac_get_file , Fn mac_get_link , Fn mac_get_peer
+These functions, described in
 .Xr mac_get 3 ,
-and may be used to retrieve the
-MAC label associated with
-a named file.
-.It Fn mac_get_proc
-This function is described in
+retrieve the MAC labels associated with file descriptors, files, and socket
+peers.
+.It Fn mac_get_pid , Fn mac_get_proc
+These functions, described in
 .Xr mac_get 3 ,
-and may be used to retrieve the
-MAC label associated with
-the calling process.
-.It Fn mac_set_fd
-This function is described in
-.Xr mac_set 3 ,
-and may be used to set the
-MAC label associated with
-a specific file descriptor.
-.It Fn mac_set_file
-This function is described in
+retrieve the MAC labels associated with processes.
+.It Fn mac_set_fd , Fn mac_set_file , Fn mac_set_link
+These functions, described in
 .Xr mac_set 3 ,
-and may be used to set the
-MAC label associated with
-a named file.
+set the MAC labels associated with file descriptors and files.
 .It Fn mac_set_proc
-This function is described in
+This function, described in
 .Xr mac_set 3 ,
-and may be used to set the
-MAC label associated with
-the calling process.
+sets the MAC label associated with the current process.
 .It Fn mac_free
-This function is described in
+This function, desribed in
 .Xr mac_free 3 ,
-and may be used to free
-userland working MAC label storage.
+frees working MAC label storage.
 .It Fn mac_from_text
-This function is described in
+This function, described in
 .Xr mac_text 3 ,
-and may be used to convert
-a text-form MAC label
-into a working
+converts a text-form MAC label into working MAC label storage,
 .Vt mac_t .
-.It Fn mac_prepare
-.It Fn mac_prepare_file_label
-.It Fn mac_prepare_ifnet_label
-.It Fn mac_prepare_process_label
-These functions are described in
+.It Fn mac_prepare , Fn mac_prepare_file_label , Fn mac_prepare_ifnet_label , Fn mac_prepare_process_label , Fn mac_prepare_type
+These functions,  described in
 .Xr mac_prepare 3 ,
-and may be used to preallocate storage for MAC label retrieval.
+allocate working storage for MAC label operations.
 .Xr mac_prepare 3
 prepares a label based on caller-specified label names; the other calls
 rely on the default configuration specified in
@@ -130,15 +132,6 @@ and may be used to convert a
 .Vt mac_t
 into a text-form MAC label.
 .El
-The behavior of some of these calls is influenced by the configuration
-settings found in
-.Xr mac.conf 5 ,
-the MAC library run-time configuration file.
-.Sh IMPLEMENTATION NOTES
-.Fx Ns 's
-support for POSIX.1e interfaces and features
-is
-.Ud .
 .Sh FILES
 .Bl -tag -width ".Pa /etc/mac.conf" -compact
 .It Pa /etc/mac.conf
@@ -150,24 +143,20 @@ system objects, but without policy-specific knowledge.
 .Sh SEE ALSO
 .Xr mac_free 3 ,
 .Xr mac_get 3 ,
+.Xr mac_is_present 3 ,
 .Xr mac_prepare 3 ,
 .Xr mac_set 3 ,
 .Xr mac_text 3 ,
+.Xr posix1e 3 ,
 .Xr mac 4 ,
 .Xr mac.conf 5 ,
 .Xr mac 9
 .Sh STANDARDS
-These APIs are loosely based on the APIs described in POSIX.1e.
-POSIX.1e is described in IEEE POSIX.1e draft 17.
-Discussion of the draft
-continues on the cross-platform POSIX.1e implementation mailing list.
-To join this list, see the
-.Fx
-POSIX.1e implementation page
-for more information.
-However, the resemblance of these APIs to the POSIX APIs is only loose,
-as the POSIX APIs were unable to express many notions required for
-flexible and extensible access control.
+These APIs are loosely based on the APIs described in POSIX.1e, as described
+in IEEE POSIX.1e draft 17.
+However, the resemblence of these APIS to the POSIX APIs is loose, as the
+PSOXI APIS were unable to express some notinos required for flexible and
+extensible access control.
 .Sh HISTORY
 Support for Mandatory Access Control was introduced in
 .Fx 5.0
diff --git a/lib/libc/posix1e/mac_free.3 b/lib/libc/posix1e/mac_free.3
index 33ef9ec..78118a1 100644
--- a/lib/libc/posix1e/mac_free.3
+++ b/lib/libc/posix1e/mac_free.3
@@ -64,6 +64,7 @@ function.
 .Xr mac_prepare 3 ,
 .Xr mac_set 3 ,
 .Xr mac_text 3 ,
+.Xr posix1e 3 ,
 .Xr mac 4 ,
 .Xr mac 9
 .Sh STANDARDS
@@ -91,8 +92,8 @@ is a complex structure in the
 .Tn TrustedBSD
 implementation,
 .Fn mac_free
-is specific to that type, and must not be used to free the character
-strings returned from
+is specific to
+.Vt mac_3 ,
+and must not be used to free the character strings returned from
 .Fn mac_to_text .
-Doing so may result in undefined behavior,
-including application failure.
+Doing so may result in undefined behavior.
diff --git a/lib/libc/posix1e/mac_get.3 b/lib/libc/posix1e/mac_get.3
index cd498d0..35fa72b 100644
--- a/lib/libc/posix1e/mac_get.3
+++ b/lib/libc/posix1e/mac_get.3
@@ -133,6 +133,7 @@ is not a directory.
 .Xr mac_prepare 3 ,
 .Xr mac_set 3 ,
 .Xr mac_text 3 ,
+.Xr posix1e 3 ,
 .Xr mac 4 ,
 .Xr mac 9
 .Sh STANDARDS
diff --git a/lib/libc/posix1e/mac_set.3 b/lib/libc/posix1e/mac_set.3
index 84a1800..0b245a0 100644
--- a/lib/libc/posix1e/mac_set.3
+++ b/lib/libc/posix1e/mac_set.3
@@ -138,6 +138,7 @@ is not a directory.
 .Xr mac_is_present 3 ,
 .Xr mac_prepare 3 ,
 .Xr mac_text 3 ,
+.Xr posix1e 3 ,
 .Xr mac 4 ,
 .Xr mac 9
 .Sh HISTORY
diff --git a/lib/libc/posix1e/mac_text.3 b/lib/libc/posix1e/mac_text.3
index 66c1e21..dde6ccf 100644
--- a/lib/libc/posix1e/mac_text.3
+++ b/lib/libc/posix1e/mac_text.3
@@ -98,6 +98,7 @@ to allocate internal storage.
 .Xr mac_is_present 3 ,
 .Xr mac_prepare 3 ,
 .Xr mac_set 3 ,
+.Xr posix1e 3 ,
 .Xr mac 4 ,
 .Xr maclabel 7
 .Sh STANDARDS
diff --git a/lib/libc/posix1e/posix1e.3 b/lib/libc/posix1e/posix1e.3
index 2065523..84ce2ec 100644
--- a/lib/libc/posix1e/posix1e.3
+++ b/lib/libc/posix1e/posix1e.3
@@ -1,5 +1,5 @@
 .\"-
-.\" Copyright (c) 2000 Robert N. M. Watson
+.\" Copyright (c) 2000, 2009 Robert N. M. Watson
 .\" All rights reserved.
 .\"
 .\" Redistribution and use in source and binary forms, with or without
@@ -25,7 +25,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd May 20, 2009
+.Dd August 7, 2009
 .Dt POSIX1E 3
 .Os
 .Sh NAME
@@ -36,99 +36,77 @@
 .Sh SYNOPSIS
 .In sys/types.h
 .In sys/acl.h
-.\" .In sys/capability.h
 .In sys/mac.h
 .Sh DESCRIPTION
-The IEEE POSIX.1e specification never left draft form, but the interfaces
-it describes are now widely used despite inherent limitations.
-Currently, only a few of the interfaces and features are implemented in
-.Fx ,
-although efforts are underway to complete the integration at this time.
+POSIX.1e describes five security extensions to the POSIX.1 API: Access
+Control Lists (ACLs), Auditing, Capabilities, Mandatory Access Control, and
+Information Flow Labels.
+While IEEE POSIX.1e D17 specification has not been standardized, several of
+its interfaces are widely used.
 .Pp
-POSIX.1e describes five security extensions to the base POSIX.1 API:
-Access Control Lists (ACLs), Auditing, Capabilities, Mandatory Access
-Control, and Information Flow Labels.
 .Fx
-supports POSIX.1e ACL interfaces, as well as POSIX.1e-like MAC
-interfaces.
-The TrustedBSD Project has produced but not integrated an implementation
-of POSIX.1e Capabilities.
-.Pp
-POSIX.1e defines both syntax and semantics for these features, but fairly
-substantial changes are required to implement these features in the
-operating system.
-.Pp
-As shipped,
-.Fx 4.0
-provides API and VFS support for ACLs, but not an implementation on any
-native file system.
-.Fx 5.0
-includes support for ACLs as part of UFS1 and UFS2, as well as necessary
-VFS support for additional file systems to export ACLs as appropriate.
-Available API calls relating to ACLs are described in detail in
-.Xr acl 3 .
-.Pp
-As shipped,
-.Fx 5.0
-includes support for Mandatory Access Control as well as POSIX.1e-like
-APIs for label management.
-More information on API calls relating to MAC is available in
-.Xr mac 3 .
+implements POSIX.1e interface for access control lists, described in
+.Xr acl 3 ,
+and supports ACLs on the
+.Xr ffs 7
+file system; ACLs must be administratively enabled using
+.Xr tunefs 8 .
 .Pp
-Additional patches supporting POSIX.1e features are provided by the
-TrustedBSD project:
+.Fx
+implements a POSIX.1e-like mandatory access control interface, described in
+.Xr mac 3 ,
+although with a number of extensions and important semantic differences.
 .Pp
-http://www.TrustedBSD.org/
-.Sh IMPLEMENTATION NOTES
-.Fx Ns 's
-support for POSIX.1e interfaces and features is still under
-development at this time, and many of these features are considered new
-or experimental.
+.Fx
+does not implement the POSIX.1e audit, privilege (capability), or information
+flow label APIs.
+However,
+.Fx
+does implement the
+.Xr libbsm
+audit API.
 .Sh ENVIRONMENT
-POSIX.1e assigns security labels to all objects, extending the security
+POSIX.1e assigns security attributes to all objects, extending the security
 functionality described in POSIX.1.
-These additional labels provide
-fine-grained discretionary access control, fine-grained capabilities,
-and labels necessary for mandatory access control.
-POSIX.2c describes
-a set of userland utilities for manipulating these labels.
+These additional attributes store fine-grained discretionary access control
+information and mandatory access control labels; for files, they are stored
+in extended attributes, described in
+.Xr extattr 3 .
 .Pp
-Many of these services are supported by extended attributes, documented
-in
-.Xr extattr 2
+POSIX.2c describes
+a set of userland utilities for manipulating these attributes, including
+.Xr getfacl 1
+and
+.Xr setfacl 1
+for access control lists, and
+.Xr getfmac 8
 and
-.Xr extattr 9 .
-While these APIs are not documented in POSIX.1e, they are similar in
-structure.
+.Xr setfmac 8
+for mandatory access control labels.
 .Sh SEE ALSO
+.Xr getfacl 1 ,
+.Xr setfacl 1 ,
 .Xr extattr 2 ,
 .Xr acl 3 ,
+.Xr extattr 3 ,
 .Xr libbsm 3 ,
 .Xr mac 3 ,
+.Xr ffs 7 ,
+.Xr getfmac 8 ,
+.Xr setfmac 8 ,
+.Xr tunefs 8 ,
 .Xr acl 9 ,
 .Xr extattr 9 ,
 .Xr mac 9
 .Sh STANDARDS
 POSIX.1e is described in IEEE POSIX.1e draft 17.
-Discussion of the draft continues
-on the cross-platform POSIX.1e implementation
-mailing list.
-To join this list, see the
-.Fx
-POSIX.1e implementation
-page for more information.
 .Sh HISTORY
 POSIX.1e support was introduced in
 .Fx 4.0 ;
-most of the features are available as of
+most features were available as of
 .Fx 5.0 .
-Development continues.
 .Sh AUTHORS
 .An Robert N M Watson
 .An Chris D. Faulhaber
 .An Thomas Moestl
 .An Ilmar S Habibulin
-.Sh BUGS
-Many of these features are considered new or experimental in
-.Fx 5.0
-and should be deployed with appropriate caution.
-- 
cgit v1.1


From 4bc31c3c24ad7325984412b3e7b0358a5ed595fa Mon Sep 17 00:00:00 2001
From: ed <ed@FreeBSD.org>
Date: Fri, 28 Aug 2009 10:25:26 +0000
Subject: MFC r196508:   Our implementation of granpt(3) could be valid in the
 future.

  When I wrote the pseudo-terminal driver for the MPSAFE TTY code, Robert
  Watson and I agreed the best way to implement this, would be to let
  posix_openpt() create a pseudo-terminal with proper permissions in place
  and let grantpt() and unlockpt() be no-ops.

  This isn't valid behaviour when looking at the spec. Because I thought
  it was an elegant solution, I filed a bug report at the Austin Group
  about this. In their last teleconference, they agreed on this subject.
  This means that future revisions of POSIX may allow grantpt() and
  unlockpt() to be no-ops if an open() on /dev/ptmx (if the implementation
  has such a device) and posix_openpt() already do the right thing.

  I'd rather put this in the manpage, because simply mentioning we don't
  comply to any standard makes it look worse than it is. Right now we
  don't, but at least we took care of it.

  Approved by:  re (kib)
---
 lib/libc/stdlib/ptsname.3 | 26 +++++++++++---------------
 1 file changed, 11 insertions(+), 15 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/stdlib/ptsname.3 b/lib/libc/stdlib/ptsname.3
index bfa3647..f674a54 100644
--- a/lib/libc/stdlib/ptsname.3
+++ b/lib/libc/stdlib/ptsname.3
@@ -134,14 +134,22 @@ The slave pseudo-terminal device could not be accessed.
 The
 .Fn ptsname
 function conforms to
-.St -p1003.1-2001 .
+.St -p1003.1-2008 .
 .Pp
 This implementation of
 .Fn grantpt
 and
 .Fn unlockpt
-does not comply with any standard, because these functions assume the
-pseudo-terminal has the correct attributes upon creation.
+does not conform to
+.St -p1003.1-2008 ,
+because it depends on
+.Xr posix_openpt 2
+to create the pseudo-terminal device with proper permissions in place.
+It only validates whether
+.Fa fildes
+is a valid pseudo-terminal master device.
+Future revisions of the specification will likely allow this behaviour,
+as stated by the Austin Group.
 .Sh HISTORY
 The
 .Fn grantpt ,
@@ -150,15 +158,3 @@ and
 .Fn unlockpt
 functions appeared in
 .Fx 5.0 .
-.Sh NOTES
-The purpose of the
-.Fn grantpt
-and
-.Fn unlockpt
-functions has no meaning in
-.Fx ,
-because pseudo-terminals obtained by
-.Xr posix_openpt 2
-are created on demand.
-Because these devices are created with proper permissions in place, they
-are guaranteed to be unused by unprivileged processes.
-- 
cgit v1.1


From bc1de82641d5a9be69feee477bff716e1f245aa8 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Fri, 28 Aug 2009 19:48:06 +0000
Subject: MFC r196525:

Consider flag == 0 as the same of flag == R_NEXT.  This change will restore
a historical behavior that has been changed by revision 190491, and has seen
to break exim.

Approved by:	re (kib)
---
 lib/libc/db/hash/hash.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/db/hash/hash.c b/lib/libc/db/hash/hash.c
index b58b36f..83d2657 100644
--- a/lib/libc/db/hash/hash.c
+++ b/lib/libc/db/hash/hash.c
@@ -711,7 +711,7 @@ hash_seq(const DB *dbp, DBT *key, DBT *data, u_int32_t flag)
 		hashp->cndx = 1;
 		hashp->cpage = NULL;
 	}
- next_bucket:
+next_bucket:
 	for (bp = NULL; !bp || !bp[0]; ) {
 		if (!(bufp = hashp->cpage)) {
 			for (bucket = hashp->cbucket;
@@ -732,7 +732,7 @@ hash_seq(const DB *dbp, DBT *key, DBT *data, u_int32_t flag)
 			}
 		} else {
 			bp = (u_int16_t *)hashp->cpage->page;
-			if (flag == R_NEXT) {
+			if (flag == R_NEXT || flag == 0) {
 				hashp->cndx += 2;
 				if (hashp->cndx > bp[0]) {
 					hashp->cpage = NULL;
-- 
cgit v1.1


From d40d30c7397aca89b9085b67c126fce64ab0c0bf Mon Sep 17 00:00:00 2001
From: trasz <trasz@FreeBSD.org>
Date: Sat, 5 Sep 2009 15:01:56 +0000
Subject: MFC r196740:

Fix regression introduced with NFSv4 ACL support - make acl_to_text(3)
and acl_calc_mask(3) return error instead of crashing when acl passed
to them is NULL.

Submitted by:	markus
Reviewed by:	rwatson
Approved by:	re (kib)
---
 lib/libc/posix1e/acl_calc_mask.c | 13 +++++++------
 lib/libc/posix1e/acl_to_text.c   | 10 +++++-----
 2 files changed, 12 insertions(+), 11 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/posix1e/acl_calc_mask.c b/lib/libc/posix1e/acl_calc_mask.c
index 95fd260..a2d1527 100644
--- a/lib/libc/posix1e/acl_calc_mask.c
+++ b/lib/libc/posix1e/acl_calc_mask.c
@@ -50,12 +50,6 @@ acl_calc_mask(acl_t *acl_p)
 	acl_t		acl_new;
 	int		i, mask_mode, mask_num;
 
-	if (!_acl_brand_may_be(*acl_p, ACL_BRAND_POSIX)) {
-		errno = EINVAL;
-		return (-1);
-	}
-	_acl_brand_as(*acl_p, ACL_BRAND_POSIX);
-
 	/*
 	 * (23.4.2.4) requires acl_p to point to a pointer to a valid ACL.
 	 * Since one of the primary reasons to use this function would be
@@ -67,6 +61,13 @@ acl_calc_mask(acl_t *acl_p)
 		errno = EINVAL;
 		return (-1);
 	}
+
+	if (!_acl_brand_may_be(*acl_p, ACL_BRAND_POSIX)) {
+		errno = EINVAL;
+		return (-1);
+	}
+	_acl_brand_as(*acl_p, ACL_BRAND_POSIX);
+
 	acl_int = &(*acl_p)->ats_acl;
 	if ((acl_int->acl_cnt < 3) || (acl_int->acl_cnt > ACL_MAX_ENTRIES)) {
 		errno = EINVAL;
diff --git a/lib/libc/posix1e/acl_to_text.c b/lib/libc/posix1e/acl_to_text.c
index 3485b1e..79a950a 100644
--- a/lib/libc/posix1e/acl_to_text.c
+++ b/lib/libc/posix1e/acl_to_text.c
@@ -70,11 +70,6 @@ _posix1e_acl_to_text(acl_t acl, ssize_t *len_p, int flags)
 	if (buf == NULL)
 		return(NULL);
 
-	if (acl == NULL) {
-		errno = EINVAL;
-		return(NULL);
-	}
-
 	acl_int = &acl->ats_acl;
 
 	mask_perm = ACL_PERM_BITS;	/* effective is regular if no mask */
@@ -243,6 +238,11 @@ char *
 acl_to_text_np(acl_t acl, ssize_t *len_p, int flags)
 {
 
+	if (acl == NULL) {
+		errno = EINVAL;
+		return(NULL);
+	}
+
 	switch (_acl_brand(acl)) {
 	case ACL_BRAND_POSIX:
 		return (_posix1e_acl_to_text(acl, len_p, flags));
-- 
cgit v1.1


From 90a09c13dfdde3ffcf990b535aa7fe8eeb253dbf Mon Sep 17 00:00:00 2001
From: kensmith <kensmith@FreeBSD.org>
Date: Thu, 10 Sep 2009 14:04:00 +0000
Subject: Remove extra debugging support that is turned on for head but turned
 off for stable branches:

	- shift to MALLOC_PRODUCTION
	- turn off automatic crash dumps
	- Remove kernel debuggers, INVARIANTS*[1], WITNESS* from
	  GENERIC kernel config files[2]

[1] INVARIANTS* left on for ia64 by request marcel
[2] sun4v was left as-is

Reviewed by:	marcel, kib
Approved by:	re (implicit)
---
 lib/libc/stdlib/malloc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libc/stdlib/malloc.c b/lib/libc/stdlib/malloc.c
index 270d641..e60746a 100644
--- a/lib/libc/stdlib/malloc.c
+++ b/lib/libc/stdlib/malloc.c
@@ -114,7 +114,7 @@
  * defaults the A and J runtime options to off.  These settings are appropriate
  * for production systems.
  */
-/* #define	MALLOC_PRODUCTION */
+#define	MALLOC_PRODUCTION
 
 #ifndef MALLOC_PRODUCTION
    /*
-- 
cgit v1.1


From 59162ba6c318eb5246931eab97b9653eac040b7d Mon Sep 17 00:00:00 2001
From: kib <kib@FreeBSD.org>
Date: Sat, 12 Sep 2009 18:16:46 +0000
Subject: MFC r196861: Handle zero size for posix_memalign. Return NULL or
 unique address according to the 'V' option.

Approved by:	re (kensmith)
---
 lib/libc/stdlib/malloc.c | 9 +++++++++
 1 file changed, 9 insertions(+)

(limited to 'lib')

diff --git a/lib/libc/stdlib/malloc.c b/lib/libc/stdlib/malloc.c
index e60746a..8b13d12 100644
--- a/lib/libc/stdlib/malloc.c
+++ b/lib/libc/stdlib/malloc.c
@@ -5320,6 +5320,15 @@ posix_memalign(void **memptr, size_t alignment, size_t size)
 			goto RETURN;
 		}
 
+		if (size == 0) {
+			if (opt_sysv == false)
+				size = 1;
+			else {
+				result = NULL;
+				ret = 0;
+				goto RETURN;
+			}
+		}
 		result = ipalloc(alignment, size);
 	}
 
-- 
cgit v1.1


From 51bfbbd15e84d8be305cd3cd3f7507c7399a66a7 Mon Sep 17 00:00:00 2001
From: attilio <attilio@FreeBSD.org>
Date: Tue, 15 Sep 2009 19:24:18 +0000
Subject: MFC r197224: Use explicit int values for the device states in order
 to allow, if necessary, in the future, adds of new states without breaking
 ABI between revisions.

Please note that this is a special condition as we want this fix in
before RC1 as we assume it is critical and so it has been handled
as an instant-merge.

Approved by:	re (kib)
---
 lib/libdevinfo/devinfo.h | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/libdevinfo/devinfo.h b/lib/libdevinfo/devinfo.h
index 91930a4..3e580f2 100644
--- a/lib/libdevinfo/devinfo.h
+++ b/lib/libdevinfo/devinfo.h
@@ -41,10 +41,10 @@ typedef __uintptr_t	devinfo_handle_t;
  */
 /* XXX not sure if I want a copy here, or expose sys/bus.h */
 typedef enum devinfo_state {
-	DIS_NOTPRESENT,			/* not probed or probe failed */
-	DIS_ALIVE,			/* probe succeeded */
-	DIS_ATTACHED,			/* attach method called */
-	DIS_BUSY			/* device is open */
+	DIS_NOTPRESENT = 10,		/* not probed or probe failed */
+	DIS_ALIVE = 20,			/* probe succeeded */
+	DIS_ATTACHED = 30,		/* attach method called */
+	DIS_BUSY = 40			/* device is open */
 } devinfo_state_t;
 
 struct devinfo_dev {
-- 
cgit v1.1


From 3134bec5442476cc5b10fc86356716e5e8d24853 Mon Sep 17 00:00:00 2001
From: kientzle <kientzle@FreeBSD.org>
Date: Thu, 17 Sep 2009 06:31:59 +0000
Subject: MFC r196962: Fix /usr/bin/unzip: A bug deep in libarchive's
 read-ahead logic (incorrect handling of zero-length reads before the copy
 buffer is allocated) is masked by the iso9660 taster.  Tar and cpio both
 enable that taster so were protected from the bug; unzip is susceptible.

This both fixes the bug and updates the test harness to exercise
this case.

Submitted by: Ed Schouten diagnosed the bug and drafted a patch
Approved by:	re (kib)
---
 lib/libarchive/archive_read.c         | 7 +++++--
 lib/libarchive/test/test_compat_zip.c | 2 +-
 2 files changed, 6 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/libarchive/archive_read.c b/lib/libarchive/archive_read.c
index 9274d66..fd4f888 100644
--- a/lib/libarchive/archive_read.c
+++ b/lib/libarchive/archive_read.c
@@ -928,9 +928,12 @@ __archive_read_filter_ahead(struct archive_read_filter *filter,
 	for (;;) {
 
 		/*
-		 * If we can satisfy from the copy buffer, we're done.
+		 * If we can satisfy from the copy buffer (and the
+		 * copy buffer isn't empty), we're done.  In particular,
+		 * note that min == 0 is a perfectly well-defined
+		 * request.
 		 */
-		if (filter->avail >= min) {
+		if (filter->avail >= min && filter->avail > 0) {
 			if (avail != NULL)
 				*avail = filter->avail;
 			return (filter->next);
diff --git a/lib/libarchive/test/test_compat_zip.c b/lib/libarchive/test/test_compat_zip.c
index f60bc4f..fefc6fe 100644
--- a/lib/libarchive/test/test_compat_zip.c
+++ b/lib/libarchive/test/test_compat_zip.c
@@ -36,7 +36,7 @@ test_compat_zip_1(void)
 
 	assert((a = archive_read_new()) != NULL);
 	assertEqualIntA(a, ARCHIVE_OK, archive_read_support_compression_all(a));
-	assertEqualIntA(a, ARCHIVE_OK, archive_read_support_format_all(a));
+	assertEqualIntA(a, ARCHIVE_OK, archive_read_support_format_zip(a));
 	extract_reference_file(name);
 	assertEqualIntA(a, ARCHIVE_OK, archive_read_open_filename(a, name, 10240));
 
-- 
cgit v1.1


From 5333c9baf74c842291a18ce7d0a5216c2353e411 Mon Sep 17 00:00:00 2001
From: attilio <attilio@FreeBSD.org>
Date: Thu, 24 Sep 2009 08:35:17 +0000
Subject: MFC r197445: Let fall down in the hard path (thus handling shared
 waiters wakeup correctly) for the shared waiters also in the rwlock held in
 shared mode as well, fixing possible deadlocks.

Please note that this is a special condition as we want this fix in
before RC2 as we assume it is critical and so it has been handled
as an instant-merge.  For the STABLE_7 branch, 1 week before the MFC
is assumed.

Approved by:	re (kib)
---
 lib/libthr/thread/thr_umtx.h | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/libthr/thread/thr_umtx.h b/lib/libthr/thread/thr_umtx.h
index 41b5f96..a6e462e 100644
--- a/lib/libthr/thread/thr_umtx.h
+++ b/lib/libthr/thread/thr_umtx.h
@@ -171,8 +171,11 @@ _thr_rwlock_unlock(struct urwlock *rwlock)
 		for (;;) {
 			if (__predict_false(URWLOCK_READER_COUNT(state) == 0))
 				return (EPERM);
-			if (!((state & URWLOCK_WRITE_WAITERS) && URWLOCK_READER_COUNT(state) == 1)) {
-				if (atomic_cmpset_rel_32(&rwlock->rw_state, state, state-1))
+			if (!((state & (URWLOCK_WRITE_WAITERS |
+			    URWLOCK_READ_WAITERS)) &&
+			    URWLOCK_READER_COUNT(state) == 1)) {
+				if (atomic_cmpset_rel_32(&rwlock->rw_state,
+				    state, state-1))
 					return (0);
 				state = rwlock->rw_state;
 			} else {
-- 
cgit v1.1


From 0ba688a4932f6bcc507979e42c93fb154118c991 Mon Sep 17 00:00:00 2001
From: nyan <nyan@FreeBSD.org>
Date: Thu, 24 Sep 2009 15:34:18 +0000
Subject: MFC: r197322 and r197374

  Revision: 197322
  Log:
    Correct BIOS header sanitizing on pc98.

  Revision: 197374
  Log:
    Disable a check on a disk size because it's too strict.  This change is
    to avoid using incorrect geometry.

   It seems that this is the same problem in g_part_bsd_read()@g_part_bsd.c.

   Reviewed by: rink

Approved by:	re (kib)
---
 lib/libdisk/change.c | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/libdisk/change.c b/lib/libdisk/change.c
index 297ac11..5f0c9c6 100644
--- a/lib/libdisk/change.c
+++ b/lib/libdisk/change.c
@@ -36,17 +36,22 @@ Sanitize_Bios_Geom(struct disk *disk)
 
 	if (disk->bios_cyl >= 65536)
 		sane = 0;
-	if (disk->bios_hd > 256)
-		sane = 0;
 #ifdef PC98
+	if (disk->bios_hd >= 256)
+		sane = 0;
 	if (disk->bios_sect >= 256)
+		sane = 0;
 #else
+	if (disk->bios_hd > 256)
+		sane = 0;
 	if (disk->bios_sect > 63)
-#endif
 		sane = 0;
+#endif
+#if 0	/* Disable a check on a disk size.  It's too strict. */
 	if (disk->bios_cyl * disk->bios_hd * disk->bios_sect !=
 	    disk->chunks->size)
 		sane = 0;
+#endif
 	if (sane)
 		return;
 
-- 
cgit v1.1


From d40bf286b68e59da94f2deb986fc8ab232f45a20 Mon Sep 17 00:00:00 2001
From: brooks <brooks@FreeBSD.org>
Date: Thu, 24 Sep 2009 21:35:13 +0000
Subject: MFC r196990: cr_groups is no longer embedded in struct ucred and is
 instead stored in a seperate array.  As such we need to use kvm_read rather
 than bcopy to populate the ki_groups field.

This fixes a crash when running ps -ax on a coredump.

Reported by:	brucec
Tested by:	brucec
MFC after:	3 days
Approved by:	re@ (kib)
---
 lib/libkvm/kvm_proc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libkvm/kvm_proc.c b/lib/libkvm/kvm_proc.c
index 7fb85a4..071f859 100644
--- a/lib/libkvm/kvm_proc.c
+++ b/lib/libkvm/kvm_proc.c
@@ -151,7 +151,7 @@ kvm_proclist(kd, what, arg, p, bp, maxcnt)
 				kp->ki_cr_flags |= KI_CRF_GRP_OVERFLOW;
 			}
 				kp->ki_ngroups = ucred.cr_ngroups;
-			bcopy(ucred.cr_groups, kp->ki_groups,
+			kvm_read(kd, (u_long)ucred.cr_groups, kp->ki_groups,
 			    kp->ki_ngroups * sizeof(gid_t));
 			kp->ki_uid = ucred.cr_uid;
 			if (ucred.cr_prison != NULL) {
-- 
cgit v1.1


From 56fcdb9e459db70ae77a457a5f5f78776457ca42 Mon Sep 17 00:00:00 2001
From: marcus <marcus@FreeBSD.org>
Date: Sun, 4 Oct 2009 21:46:43 +0000
Subject: MFC: rev. 197681

Correct the pthread stub prototype for pthread_mutexattr_settype to allow for
the type argument.  This is known to fix some pthread_mutexattr_settype()
invocations, especially when it comes to pulseaudio.

Approved by:	re (kib)
---
 lib/libc/gen/_pthread_stubs.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libc/gen/_pthread_stubs.c b/lib/libc/gen/_pthread_stubs.c
index a3e55b0..147235e 100644
--- a/lib/libc/gen/_pthread_stubs.c
+++ b/lib/libc/gen/_pthread_stubs.c
@@ -222,7 +222,7 @@ STUB_FUNC1(pthread_mutex_trylock, PJT_MUTEX_TRYLOCK, int, void *)
 STUB_FUNC1(pthread_mutex_unlock, PJT_MUTEX_UNLOCK, int, void *)
 STUB_FUNC1(pthread_mutexattr_destroy, PJT_MUTEXATTR_DESTROY, int, void *)
 STUB_FUNC1(pthread_mutexattr_init, PJT_MUTEXATTR_INIT, int, void *)
-STUB_FUNC1(pthread_mutexattr_settype, PJT_MUTEXATTR_SETTYPE, int, void *)
+STUB_FUNC2(pthread_mutexattr_settype, PJT_MUTEXATTR_SETTYPE, int, void *, int)
 STUB_FUNC2(pthread_once, 	PJT_ONCE, int, void *, void *)
 STUB_FUNC1(pthread_rwlock_destroy, PJT_RWLOCK_DESTROY, int, void *)
 STUB_FUNC2(pthread_rwlock_init,	PJT_RWLOCK_INIT, int, void *, void *)
-- 
cgit v1.1


From 6a88d4c4d6bd818cbd5a84db8e6a10744d7d8eb2 Mon Sep 17 00:00:00 2001
From: brueffer <brueffer@FreeBSD.org>
Date: Fri, 9 Oct 2009 13:52:49 +0000
Subject: MFC: r197312

Fix setfib(1) section number.

Approved by:	re (kib)
---
 lib/libc/sys/setfib.2 | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/sys/setfib.2 b/lib/libc/sys/setfib.2
index ca08202..a65a064 100644
--- a/lib/libc/sys/setfib.2
+++ b/lib/libc/sys/setfib.2
@@ -67,8 +67,8 @@ if the
 .Fa fib
 argument is greater than the current system maximum.
 .Sh SEE ALSO
-.Xr setsockopt 2 ,
-.Xr setfib 8 
+.Xr setfib 1 ,
+.Xr setsockopt 2
 .Sh STANDARDS
 The
 .Fn setfib
-- 
cgit v1.1


From 77a2c065511fd56f5abcb85492a40a871e89a6a8 Mon Sep 17 00:00:00 2001
From: kib <kib@FreeBSD.org>
Date: Wed, 14 Oct 2009 14:26:19 +0000
Subject: MFC r197958: In nanosleep(2), note that the calling thread is put to
 sleep, not the whole process. Also explicitely name the parameter that
 specifies sleep interval.

Approved by:	re (kensmith)
---
 lib/libc/sys/nanosleep.2 | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libc/sys/nanosleep.2 b/lib/libc/sys/nanosleep.2
index 18e4c88..f50544b 100644
--- a/lib/libc/sys/nanosleep.2
+++ b/lib/libc/sys/nanosleep.2
@@ -47,7 +47,9 @@
 The
 .Fn nanosleep
 system call
-causes the process to sleep for the specified time.
+causes the calling thread to sleep until the time interval specified by
+.Fa rqtp
+has elapsed.
 An unmasked signal will
 cause it to terminate the sleep early, regardless of the
 .Dv SA_RESTART
-- 
cgit v1.1


From dae6c247dba58450adb88a4f80252645b1ab6d83 Mon Sep 17 00:00:00 2001
From: kan <kan@FreeBSD.org>
Date: Tue, 20 Oct 2009 19:05:43 +0000
Subject: MFC Revision 197277:

Make libc.a provide __stack_chk_fail_local weak alias. This is
needed to satisfy static libraries that are compiled with -fpic
and linked into static binary afterwards. Several libraries in
gcc are examples of such static libs.

Approved by: re (kib)
---
 lib/libc/sys/stack_protector.c | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'lib')

diff --git a/lib/libc/sys/stack_protector.c b/lib/libc/sys/stack_protector.c
index 63beebc..14c20eb 100644
--- a/lib/libc/sys/stack_protector.c
+++ b/lib/libc/sys/stack_protector.c
@@ -108,4 +108,8 @@ __chk_fail(void)
 	__fail("buffer overflow detected; terminated");
 }
 
+#ifdef PIC
 __sym_compat(__stack_chk_fail_local, __stack_chk_fail, FBSD_1.0);
+#else
+__weak_reference(__stack_chk_fail, __stack_chk_fail_local);
+#endif
-- 
cgit v1.1


From ad58b4ba938908e46296c7a6c7314da0d0a19ca4 Mon Sep 17 00:00:00 2001
From: thompsa <thompsa@FreeBSD.org>
Date: Fri, 23 Oct 2009 12:02:01 +0000
Subject: MFC r198376

 Prevent wraparound of the timeout variable.

Submitted by:	HPS
Approved by:	re (kib)
---
 lib/libusb/libusb20_ugen20.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libusb/libusb20_ugen20.c b/lib/libusb/libusb20_ugen20.c
index 0dee793..f9f3689 100644
--- a/lib/libusb/libusb20_ugen20.c
+++ b/lib/libusb/libusb20_ugen20.c
@@ -800,7 +800,11 @@ ugen20_tr_submit(struct libusb20_transfer *xfer)
 	if (xfer->flags & LIBUSB20_TRANSFER_DO_CLEAR_STALL) {
 		fsep->flags |= USB_FS_FLAG_CLEAR_STALL;
 	}
-	fsep->timeout = xfer->timeout;
+	/* NOTE: The "fsep->timeout" variable is 16-bit. */
+	if (xfer->timeout > 65535)
+		fsep->timeout = 65535;
+	else
+		fsep->timeout = xfer->timeout;
 
 	temp.ep_index = xfer->trIndex;
 
-- 
cgit v1.1


From 88c99df1f745f2a16b992001673bdc2cb203451e Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Fri, 30 Oct 2009 12:57:28 +0000
Subject: MFC rev. 197086: Add simple embedded RADIUS server support to
 libradius, by extending existing API, keeping backward compatibility.

First consumer for this functionality is going to become forthcoming MPD-5.4,
supporting CoA and DR of RFC 3576: Dynamic Authorization Extensions to RADIUS.
---
 lib/libradius/libradius.3      |  51 +++++-
 lib/libradius/radlib.c         | 346 ++++++++++++++++++++++++++++++++---------
 lib/libradius/radlib.h         |  12 ++
 lib/libradius/radlib_private.h |  15 +-
 4 files changed, 334 insertions(+), 90 deletions(-)

(limited to 'lib')

diff --git a/lib/libradius/libradius.3 b/lib/libradius/libradius.3
index 095d6e9..9a71521 100644
--- a/lib/libradius/libradius.3
+++ b/lib/libradius/libradius.3
@@ -1,4 +1,5 @@
 .\" Copyright 1998 Juniper Networks, Inc.
+.\" Copyright 2009 Alexander Motin <mav@FreeBSD.org>.
 .\" All rights reserved.
 .\"
 .\" Redistribution and use in source and binary forms, with or without
@@ -24,12 +25,12 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd April 27, 2004
+.Dd August 5, 2009
 .Dt LIBRADIUS 3
 .Os
 .Sh NAME
 .Nm libradius
-.Nd RADIUS client library
+.Nd RADIUS client/server library
 .Sh SYNOPSIS
 .In radlib.h
 .Ft "struct rad_handle *"
@@ -46,6 +47,8 @@
 .Fn rad_continue_send_request "struct rad_handle *h" "int selected" "int *fd" "struct timeval *tv"
 .Ft int
 .Fn rad_create_request "struct rad_handle *h" "int code"
+.Ft int
+.Fn rad_create_response "struct rad_handle *h" "int code"
 .Ft "struct in_addr"
 .Fn rad_cvt_addr "const void *data"
 .Ft u_int32_t
@@ -79,7 +82,13 @@
 .Ft ssize_t
 .Fn rad_request_authenticator "struct rad_handle *h" "char *buf" "size_t len"
 .Ft int
+.Fn rad_receive_request "struct rad_handle *h"
+.Ft int
 .Fn rad_send_request "struct rad_handle *h"
+.Ft int
+.Fn rad_send_response "struct rad_handle *h"
+.Ft "struct rad_handle *"
+.Fn rad_server_open "int fd"
 .Ft "const char *"
 .Fn rad_server_secret "struct rad_handle *h"
 .Ft u_char *
@@ -91,16 +100,17 @@
 .Sh DESCRIPTION
 The
 .Nm
-library implements the client side of the Remote Authentication Dial
-In User Service (RADIUS).
+library implements the Remote Authentication Dial In User Service (RADIUS).
 RADIUS, defined in RFCs 2865 and 2866,
 allows clients to perform authentication and accounting by means of
 network requests to remote servers.
 .Ss Initialization
 To use the library, an application must first call
 .Fn rad_auth_open
-or
+,
 .Fn rad_acct_open
+or
+.Fn rad_server_open
 to obtain a
 .Vt "struct rad_handle *" ,
 which provides the context for subsequent operations.
@@ -108,8 +118,10 @@ The former function is used for RADIUS authentication and the
 latter is used for RADIUS accounting.
 Calls to
 .Fn rad_auth_open
-and
+,
 .Fn rad_acct_open
+and
+.Fn rad_server_open
 always succeed unless insufficient virtual memory is available.
 If
 the necessary memory cannot be allocated, the functions return
@@ -451,6 +463,25 @@ subsequent library calls using the same handle.
 .Ss Cleanup
 To free the resources used by the RADIUS library, call
 .Fn rad_close .
+.Ss Server operation
+Server mode operates much alike to client mode, except packet send and receieve
+steps are swapped. To operate as server you should obtain server context with
+.Fn rad_server_open
+function, passing opened and bound UDP socket file descriptor as argument.
+You should define allowed clients and their secrets using
+.Fn rad_add_server
+function. port, timeout and max_tries arguments are ignored in server mode.
+You should call
+.Fn rad_receive_request
+function to receive request from client. If you do not want to block on socket
+read, you are free to use any poll(), select() or non-blocking sockets for
+the socket.
+Received request can be parsed with same parsing functions as for client.
+To respond to the request you should call
+.Fn rad_create_response
+and fill response content with same packet writing functions as for client.
+When packet is ready, it should be sent with
+.Fn rad_send_response
 .Sh RETURN VALUES
 The following functions return a non-negative value on success.
 If
@@ -466,6 +497,8 @@ which can be retrieved using
 .It
 .Fn rad_create_request
 .It
+.Fn rad_create_response
+.It
 .Fn rad_get_attr
 .It
 .Fn rad_put_addr
@@ -483,6 +516,8 @@ which can be retrieved using
 .Fn rad_continue_send_request
 .It
 .Fn rad_send_request
+.It
+.Fn rad_send_response
 .El
 .Pp
 The following functions return a
@@ -499,6 +534,8 @@ without recording an error message.
 .It
 .Fn rad_auth_open
 .It
+.Fn rad_server_open
+.It
 .Fn rad_cvt_string
 .El
 .Pp
@@ -553,3 +590,5 @@ subsequently added the ability to perform RADIUS
 accounting.
 Later additions and changes by
 .An Michael Bretterklieber .
+Server mode support was added by
+.An Alexander Motin .
diff --git a/lib/libradius/radlib.c b/lib/libradius/radlib.c
index b21447e..8faaf7e 100644
--- a/lib/libradius/radlib.c
+++ b/lib/libradius/radlib.c
@@ -103,7 +103,7 @@ insert_scrambled_password(struct rad_handle *h, int srv)
 	srvp = &h->servers[srv];
 	padded_len = h->pass_len == 0 ? 16 : (h->pass_len+15) & ~0xf;
 
-	memcpy(md5, &h->request[POS_AUTH], LEN_AUTH);
+	memcpy(md5, &h->out[POS_AUTH], LEN_AUTH);
 	for (pos = 0;  pos < padded_len;  pos += 16) {
 		int i;
 
@@ -120,49 +120,55 @@ insert_scrambled_password(struct rad_handle *h, int srv)
 		 * in calculating the scrambler for next time.
 		 */
 		for (i = 0;  i < 16;  i++)
-			h->request[h->pass_pos + pos + i] =
+			h->out[h->pass_pos + pos + i] =
 			    md5[i] ^= h->pass[pos + i];
 	}
 }
 
 static void
-insert_request_authenticator(struct rad_handle *h, int srv)
+insert_request_authenticator(struct rad_handle *h, int resp)
 {
 	MD5_CTX ctx;
 	const struct rad_server *srvp;
 
-	srvp = &h->servers[srv];
+	srvp = &h->servers[h->srv];
 
 	/* Create the request authenticator */
 	MD5Init(&ctx);
-	MD5Update(&ctx, &h->request[POS_CODE], POS_AUTH - POS_CODE);
-	MD5Update(&ctx, memset(&h->request[POS_AUTH], 0, LEN_AUTH), LEN_AUTH);
-	MD5Update(&ctx, &h->request[POS_ATTRS], h->req_len - POS_ATTRS);
+	MD5Update(&ctx, &h->out[POS_CODE], POS_AUTH - POS_CODE);
+	if (resp)
+	    MD5Update(&ctx, &h->in[POS_AUTH], LEN_AUTH);
+	else
+	    MD5Update(&ctx, &h->out[POS_AUTH], LEN_AUTH);
+	MD5Update(&ctx, &h->out[POS_ATTRS], h->out_len - POS_ATTRS);
 	MD5Update(&ctx, srvp->secret, strlen(srvp->secret));
-	MD5Final(&h->request[POS_AUTH], &ctx);
+	MD5Final(&h->out[POS_AUTH], &ctx);
 }
 
 static void
-insert_message_authenticator(struct rad_handle *h, int srv)
+insert_message_authenticator(struct rad_handle *h, int resp)
 {
 #ifdef WITH_SSL
 	u_char md[EVP_MAX_MD_SIZE];
 	u_int md_len;
 	const struct rad_server *srvp;
 	HMAC_CTX ctx;
-	srvp = &h->servers[srv];
+	srvp = &h->servers[h->srv];
 
 	if (h->authentic_pos != 0) {
 		HMAC_CTX_init(&ctx);
 		HMAC_Init(&ctx, srvp->secret, strlen(srvp->secret), EVP_md5());
-		HMAC_Update(&ctx, &h->request[POS_CODE], POS_AUTH - POS_CODE);
-		HMAC_Update(&ctx, &h->request[POS_AUTH], LEN_AUTH);
-		HMAC_Update(&ctx, &h->request[POS_ATTRS],
-		    h->req_len - POS_ATTRS);
+		HMAC_Update(&ctx, &h->out[POS_CODE], POS_AUTH - POS_CODE);
+		if (resp)
+		    HMAC_Update(&ctx, &h->in[POS_AUTH], LEN_AUTH);
+		else
+		    HMAC_Update(&ctx, &h->out[POS_AUTH], LEN_AUTH);
+		HMAC_Update(&ctx, &h->out[POS_ATTRS],
+		    h->out_len - POS_ATTRS);
 		HMAC_Final(&ctx, md, &md_len);
 		HMAC_CTX_cleanup(&ctx);
 		HMAC_cleanup(&ctx);
-		memcpy(&h->request[h->authentic_pos + 2], md, md_len);
+		memcpy(&h->out[h->authentic_pos + 2], md, md_len);
 	}
 #endif
 }
@@ -195,20 +201,20 @@ is_valid_response(struct rad_handle *h, int srv,
 		return 0;
 
 	/* Check the message length */
-	if (h->resp_len < POS_ATTRS)
+	if (h->in_len < POS_ATTRS)
 		return 0;
-	len = h->response[POS_LENGTH] << 8 | h->response[POS_LENGTH+1];
-	if (len > h->resp_len)
+	len = h->in[POS_LENGTH] << 8 | h->in[POS_LENGTH+1];
+	if (len > h->in_len)
 		return 0;
 
 	/* Check the response authenticator */
 	MD5Init(&ctx);
-	MD5Update(&ctx, &h->response[POS_CODE], POS_AUTH - POS_CODE);
-	MD5Update(&ctx, &h->request[POS_AUTH], LEN_AUTH);
-	MD5Update(&ctx, &h->response[POS_ATTRS], len - POS_ATTRS);
+	MD5Update(&ctx, &h->in[POS_CODE], POS_AUTH - POS_CODE);
+	MD5Update(&ctx, &h->out[POS_AUTH], LEN_AUTH);
+	MD5Update(&ctx, &h->in[POS_ATTRS], len - POS_ATTRS);
 	MD5Update(&ctx, srvp->secret, strlen(srvp->secret));
 	MD5Final(md5, &ctx);
-	if (memcmp(&h->response[POS_AUTH], md5, sizeof md5) != 0)
+	if (memcmp(&h->in[POS_AUTH], md5, sizeof md5) != 0)
 		return 0;
 
 #ifdef WITH_SSL
@@ -216,42 +222,111 @@ is_valid_response(struct rad_handle *h, int srv,
 	 * For non accounting responses check the message authenticator,
 	 * if any.
 	 */
-	if (h->response[POS_CODE] != RAD_ACCOUNTING_RESPONSE) {
+	if (h->in[POS_CODE] != RAD_ACCOUNTING_RESPONSE) {
 
-		memcpy(resp, h->response, MSGSIZE);
+		memcpy(resp, h->in, MSGSIZE);
 		pos = POS_ATTRS;
 
 		/* Search and verify the Message-Authenticator */
 		while (pos < len - 2) {
 
-			if (h->response[pos] == RAD_MESSAGE_AUTHENTIC) {
+			if (h->in[pos] == RAD_MESSAGE_AUTHENTIC) {
 				/* zero fill the Message-Authenticator */
 				memset(&resp[pos + 2], 0, MD5_DIGEST_LENGTH);
 
 				HMAC_CTX_init(&hctx);
 				HMAC_Init(&hctx, srvp->secret,
 				    strlen(srvp->secret), EVP_md5());
-				HMAC_Update(&hctx, &h->response[POS_CODE],
+				HMAC_Update(&hctx, &h->in[POS_CODE],
 				    POS_AUTH - POS_CODE);
-				HMAC_Update(&hctx, &h->request[POS_AUTH],
+				HMAC_Update(&hctx, &h->out[POS_AUTH],
 				    LEN_AUTH);
 				HMAC_Update(&hctx, &resp[POS_ATTRS],
-				    h->resp_len - POS_ATTRS);
+				    h->in_len - POS_ATTRS);
 				HMAC_Final(&hctx, md, &md_len);
 				HMAC_CTX_cleanup(&hctx);
 				HMAC_cleanup(&hctx);
-				if (memcmp(md, &h->response[pos + 2],
+				if (memcmp(md, &h->in[pos + 2],
 				    MD5_DIGEST_LENGTH) != 0)
 					return 0;
 				break;
 			}
-			pos += h->response[pos + 1];
+			pos += h->in[pos + 1];
 		}
 	}
 #endif
 	return 1;
 }
 
+/*
+ * Return true if the current request is valid for the specified server.
+ */
+static int
+is_valid_request(struct rad_handle *h)
+{
+	MD5_CTX ctx;
+	unsigned char md5[MD5_DIGEST_LENGTH];
+	const struct rad_server *srvp;
+	int len;
+#ifdef WITH_SSL
+	HMAC_CTX hctx;
+	u_char resp[MSGSIZE], md[EVP_MAX_MD_SIZE];
+	u_int md_len;
+	int pos;
+#endif
+
+	srvp = &h->servers[h->srv];
+
+	/* Check the message length */
+	if (h->in_len < POS_ATTRS)
+		return (0);
+	len = h->in[POS_LENGTH] << 8 | h->in[POS_LENGTH+1];
+	if (len > h->in_len)
+		return (0);
+
+	if (h->in[POS_CODE] != RAD_ACCESS_REQUEST) {
+		uint32_t zeroes[4] = { 0, 0, 0, 0 };
+		/* Check the request authenticator */
+		MD5Init(&ctx);
+		MD5Update(&ctx, &h->in[POS_CODE], POS_AUTH - POS_CODE);
+		MD5Update(&ctx, zeroes, LEN_AUTH);
+		MD5Update(&ctx, &h->in[POS_ATTRS], len - POS_ATTRS);
+		MD5Update(&ctx, srvp->secret, strlen(srvp->secret));
+		MD5Final(md5, &ctx);
+		if (memcmp(&h->in[POS_AUTH], md5, sizeof md5) != 0)
+			return (0);
+	}
+
+#ifdef WITH_SSL
+	/* Search and verify the Message-Authenticator */
+	pos = POS_ATTRS;
+	while (pos < len - 2) {
+		if (h->in[pos] == RAD_MESSAGE_AUTHENTIC) {
+			memcpy(resp, h->in, MSGSIZE);
+			/* zero fill the Request-Authenticator */
+			if (h->in[POS_CODE] != RAD_ACCESS_REQUEST)
+				memset(&resp[POS_AUTH], 0, LEN_AUTH);
+			/* zero fill the Message-Authenticator */
+			memset(&resp[pos + 2], 0, MD5_DIGEST_LENGTH);
+
+			HMAC_CTX_init(&hctx);
+			HMAC_Init(&hctx, srvp->secret,
+			    strlen(srvp->secret), EVP_md5());
+			HMAC_Update(&hctx, resp, h->in_len);
+			HMAC_Final(&hctx, md, &md_len);
+			HMAC_CTX_cleanup(&hctx);
+			HMAC_cleanup(&hctx);
+			if (memcmp(md, &h->in[pos + 2],
+			    MD5_DIGEST_LENGTH) != 0)
+				return (0);
+			break;
+		}
+		pos += h->in[pos + 1];
+	}
+#endif
+	return (1);
+}
+
 static int
 put_password_attr(struct rad_handle *h, int type, const void *value, size_t len)
 {
@@ -273,7 +348,7 @@ put_password_attr(struct rad_handle *h, int type, const void *value, size_t len)
 	 */
 	clear_password(h);
 	put_raw_attr(h, type, h->pass, padded_len);
-	h->pass_pos = h->req_len - padded_len;
+	h->pass_pos = h->out_len - padded_len;
 
 	/* Save the cleartext password, padded as necessary */
 	memcpy(h->pass, value, len);
@@ -289,14 +364,14 @@ put_raw_attr(struct rad_handle *h, int type, const void *value, size_t len)
 		generr(h, "Attribute too long");
 		return -1;
 	}
-	if (h->req_len + 2 + len > MSGSIZE) {
+	if (h->out_len + 2 + len > MSGSIZE) {
 		generr(h, "Maximum message length exceeded");
 		return -1;
 	}
-	h->request[h->req_len++] = type;
-	h->request[h->req_len++] = len + 2;
-	memcpy(&h->request[h->req_len], value, len);
-	h->req_len += len;
+	h->out[h->out_len++] = type;
+	h->out[h->out_len++] = len + 2;
+	memcpy(&h->out[h->out_len], value, len);
+	h->out_len += len;
 	return 0;
 }
 
@@ -523,22 +598,26 @@ rad_continue_send_request(struct rad_handle *h, int selected, int *fd,
 {
 	int n;
 
+	if (h->type == RADIUS_SERVER) {
+		generr(h, "denied function call");
+		return (-1);
+	}
 	if (selected) {
 		struct sockaddr_in from;
 		socklen_t fromlen;
 
 		fromlen = sizeof from;
-		h->resp_len = recvfrom(h->fd, h->response,
+		h->in_len = recvfrom(h->fd, h->in,
 		    MSGSIZE, MSG_WAITALL, (struct sockaddr *)&from, &fromlen);
-		if (h->resp_len == -1) {
+		if (h->in_len == -1) {
 			generr(h, "recvfrom: %s", strerror(errno));
 			return -1;
 		}
 		if (is_valid_response(h, h->srv, &from)) {
-			h->resp_len = h->response[POS_LENGTH] << 8 |
-			    h->response[POS_LENGTH+1];
-			h->resp_pos = POS_ATTRS;
-			return h->response[POS_CODE];
+			h->in_len = h->in[POS_LENGTH] << 8 |
+			    h->in[POS_LENGTH+1];
+			h->in_pos = POS_ATTRS;
+			return h->in[POS_CODE];
 		}
 	}
 
@@ -556,21 +635,22 @@ rad_continue_send_request(struct rad_handle *h, int selected, int *fd,
 		if (++h->srv >= h->num_servers)
 			h->srv = 0;
 
-	if (h->request[POS_CODE] == RAD_ACCOUNTING_REQUEST)
-		/* Insert the request authenticator into the request */
-		insert_request_authenticator(h, h->srv);
-	else
+	if (h->out[POS_CODE] == RAD_ACCESS_REQUEST) {
 		/* Insert the scrambled password into the request */
 		if (h->pass_pos != 0)
 			insert_scrambled_password(h, h->srv);
-
-	insert_message_authenticator(h, h->srv);
+	}
+	insert_message_authenticator(h, 0);
+	if (h->out[POS_CODE] != RAD_ACCESS_REQUEST) {
+		/* Insert the request authenticator into the request */
+		insert_request_authenticator(h, h->srv);
+	}
 
 	/* Send the request */
-	n = sendto(h->fd, h->request, h->req_len, 0,
+	n = sendto(h->fd, h->out, h->out_len, 0,
 	    (const struct sockaddr *)&h->servers[h->srv].addr,
 	    sizeof h->servers[h->srv].addr);
-	if (n != h->req_len) {
+	if (n != h->out_len) {
 		if (n == -1)
 			generr(h, "sendto: %s", strerror(errno));
 		else
@@ -588,22 +668,117 @@ rad_continue_send_request(struct rad_handle *h, int selected, int *fd,
 }
 
 int
+rad_receive_request(struct rad_handle *h)
+{
+	struct sockaddr_in from;
+	socklen_t fromlen;
+	int n;
+
+	if (h->type != RADIUS_SERVER) {
+		generr(h, "denied function call");
+		return (-1);
+	}
+	h->srv = -1;
+	fromlen = sizeof(from);
+	h->in_len = recvfrom(h->fd, h->in,
+	    MSGSIZE, MSG_WAITALL, (struct sockaddr *)&from, &fromlen);
+	if (h->in_len == -1) {
+		generr(h, "recvfrom: %s", strerror(errno));
+		return (-1);
+	}
+	for (n = 0; n < h->num_servers; n++) {
+		if (h->servers[n].addr.sin_addr.s_addr == from.sin_addr.s_addr) {
+			h->servers[n].addr.sin_port = from.sin_port;
+			h->srv = n;
+			break;
+		}
+	}
+	if (h->srv == -1)
+		return (-2);
+	if (is_valid_request(h)) {
+		h->in_len = h->in[POS_LENGTH] << 8 |
+		    h->in[POS_LENGTH+1];
+		h->in_pos = POS_ATTRS;
+		return (h->in[POS_CODE]);
+	}
+	return (-3);
+}
+
+int
+rad_send_response(struct rad_handle *h)
+{
+	int n;
+
+	if (h->type != RADIUS_SERVER) {
+		generr(h, "denied function call");
+		return (-1);
+	}
+	/* Fill in the length field in the message */
+	h->out[POS_LENGTH] = h->out_len >> 8;
+	h->out[POS_LENGTH+1] = h->out_len;
+
+	insert_message_authenticator(h,
+	    (h->in[POS_CODE] == RAD_ACCESS_REQUEST) ? 1 : 0);
+	insert_request_authenticator(h, 1);
+
+	/* Send the request */
+	n = sendto(h->fd, h->out, h->out_len, 0,
+	    (const struct sockaddr *)&h->servers[h->srv].addr,
+	    sizeof h->servers[h->srv].addr);
+	if (n != h->out_len) {
+		if (n == -1)
+			generr(h, "sendto: %s", strerror(errno));
+		else
+			generr(h, "sendto: short write");
+		return -1;
+	}
+
+	return 0;
+}
+
+int
 rad_create_request(struct rad_handle *h, int code)
 {
 	int i;
 
-	h->request[POS_CODE] = code;
-	h->request[POS_IDENT] = ++h->ident;
-	/* Create a random authenticator */
-	for (i = 0;  i < LEN_AUTH;  i += 2) {
-		long r;
-		r = random();
-		h->request[POS_AUTH+i] = (u_char)r;
-		h->request[POS_AUTH+i+1] = (u_char)(r >> 8);
+	if (h->type == RADIUS_SERVER) {
+		generr(h, "denied function call");
+		return (-1);
+	}
+	h->out[POS_CODE] = code;
+	h->out[POS_IDENT] = ++h->ident;
+	if (code == RAD_ACCESS_REQUEST) {
+		/* Create a random authenticator */
+		for (i = 0;  i < LEN_AUTH;  i += 2) {
+			long r;
+			r = random();
+			h->out[POS_AUTH+i] = (u_char)r;
+			h->out[POS_AUTH+i+1] = (u_char)(r >> 8);
+		}
+	} else
+		memset(&h->out[POS_AUTH], 0, LEN_AUTH);
+	h->out_len = POS_ATTRS;
+	clear_password(h);
+	h->authentic_pos = 0;
+	h->out_created = 1;
+	return 0;
+}
+
+int
+rad_create_response(struct rad_handle *h, int code)
+{
+
+	if (h->type != RADIUS_SERVER) {
+		generr(h, "denied function call");
+		return (-1);
 	}
-	h->req_len = POS_ATTRS;
+	h->out[POS_CODE] = code;
+	h->out[POS_IDENT] = h->in[POS_IDENT];
+	memset(&h->out[POS_AUTH], 0, LEN_AUTH);
+	h->out_len = POS_ATTRS;
 	clear_password(h);
-	h->request_created = 1;
+	h->authentic_pos = 0;
+	h->out_created = 1;
 	return 0;
 }
 
@@ -647,20 +822,20 @@ rad_get_attr(struct rad_handle *h, const void **value, size_t *len)
 {
 	int type;
 
-	if (h->resp_pos >= h->resp_len)
+	if (h->in_pos >= h->in_len)
 		return 0;
-	if (h->resp_pos + 2 > h->resp_len) {
+	if (h->in_pos + 2 > h->in_len) {
 		generr(h, "Malformed attribute in response");
 		return -1;
 	}
-	type = h->response[h->resp_pos++];
-	*len = h->response[h->resp_pos++] - 2;
-	if (h->resp_pos + (int)*len > h->resp_len) {
+	type = h->in[h->in_pos++];
+	*len = h->in[h->in_pos++] - 2;
+	if (h->in_pos + (int)*len > h->in_len) {
 		generr(h, "Malformed attribute in response");
 		return -1;
 	}
-	*value = &h->response[h->resp_pos];
-	h->resp_pos += *len;
+	*value = &h->in[h->in_pos];
+	h->in_pos += *len;
 	return type;
 }
 
@@ -672,6 +847,10 @@ rad_init_send_request(struct rad_handle *h, int *fd, struct timeval *tv)
 {
 	int srv;
 
+	if (h->type == RADIUS_SERVER) {
+		generr(h, "denied function call");
+		return (-1);
+	}
 	/* Make sure we have a socket to use */
 	if (h->fd == -1) {
 		struct sockaddr_in sin;
@@ -694,7 +873,7 @@ rad_init_send_request(struct rad_handle *h, int *fd, struct timeval *tv)
 		}
 	}
 
-	if (h->request[POS_CODE] == RAD_ACCOUNTING_REQUEST) {
+	if (h->out[POS_CODE] != RAD_ACCESS_REQUEST) {
 		/* Make sure no password given */
 		if (h->pass_pos || h->chap_pass) {
 			generr(h, "User or Chap Password"
@@ -718,8 +897,8 @@ rad_init_send_request(struct rad_handle *h, int *fd, struct timeval *tv)
 	}
 
 	/* Fill in the length field in the message */
-	h->request[POS_LENGTH] = h->req_len >> 8;
-	h->request[POS_LENGTH+1] = h->req_len;
+	h->out[POS_LENGTH] = h->out_len >> 8;
+	h->out[POS_LENGTH+1] = h->out_len;
 
 	/*
 	 * Count the total number of tries we will make, and zero the
@@ -763,7 +942,7 @@ rad_auth_open(void)
 		h->chap_pass = 0;
 		h->authentic_pos = 0;
 		h->type = RADIUS_AUTH;
-		h->request_created = 0;
+		h->out_created = 0;
 		h->eap_msg = 0;
 	}
 	return h;
@@ -781,6 +960,19 @@ rad_acct_open(void)
 }
 
 struct rad_handle *
+rad_server_open(int fd)
+{
+	struct rad_handle *h;
+
+	h = rad_open();
+	if (h != NULL) {
+	        h->type = RADIUS_SERVER;
+	        h->fd = fd;
+	}
+	return h;
+}
+
+struct rad_handle *
 rad_open(void)
 {
     return rad_auth_open();
@@ -797,13 +989,13 @@ rad_put_attr(struct rad_handle *h, int type, const void *value, size_t len)
 {
 	int result;
 
-	if (!h->request_created) {
+	if (!h->out_created) {
 		generr(h, "Please call rad_create_request()"
 		    " before putting attributes");
 		return -1;
 	}
 
-	if (h->request[POS_CODE] == RAD_ACCOUNTING_REQUEST) {
+	if (h->out[POS_CODE] == RAD_ACCOUNTING_REQUEST) {
 		if (type == RAD_EAP_MESSAGE) {
 			generr(h, "EAP-Message attribute is not valid"
 			    " in accounting requests");
@@ -858,14 +1050,14 @@ rad_put_message_authentic(struct rad_handle *h)
 #ifdef WITH_SSL
 	u_char md_zero[MD5_DIGEST_LENGTH];
 
-	if (h->request[POS_CODE] == RAD_ACCOUNTING_REQUEST) {
+	if (h->out[POS_CODE] == RAD_ACCOUNTING_REQUEST) {
 		generr(h, "Message-Authenticator is not valid"
 		    " in accounting requests");
 		return -1;
 	}
 
 	if (h->authentic_pos == 0) {
-		h->authentic_pos = h->req_len;
+		h->authentic_pos = h->out_len;
 		memset(md_zero, 0, sizeof(md_zero));
 		return (put_raw_attr(h, RAD_MESSAGE_AUTHENTIC, md_zero,
 		    sizeof(md_zero)));
@@ -1041,7 +1233,7 @@ rad_put_vendor_attr(struct rad_handle *h, int vendor, int type,
 	struct vendor_attribute *attr;
 	int res;
 
-	if (!h->request_created) {
+	if (!h->out_created) {
 		generr(h, "Please call rad_create_request()"
 		    " before putting attributes");
 		return -1;
@@ -1088,7 +1280,7 @@ rad_request_authenticator(struct rad_handle *h, char *buf, size_t len)
 {
 	if (len < LEN_AUTH)
 		return (-1);
-	memcpy(buf, h->request + POS_AUTH, LEN_AUTH);
+	memcpy(buf, h->out + POS_AUTH, LEN_AUTH);
 	if (len > LEN_AUTH)
 		buf[LEN_AUTH] = '\0';
 	return (LEN_AUTH);
diff --git a/lib/libradius/radlib.h b/lib/libradius/radlib.h
index 2c42c3a..b26be41 100644
--- a/lib/libradius/radlib.h
+++ b/lib/libradius/radlib.h
@@ -42,6 +42,12 @@
 #define RAD_ACCOUNTING_REQUEST		4
 #define RAD_ACCOUNTING_RESPONSE		5
 #define RAD_ACCESS_CHALLENGE		11
+#define RAD_DISCONNECT_REQUEST		40
+#define RAD_DISCONNECT_ACK		41
+#define RAD_DISCONNECT_NAK		42
+#define RAD_COA_REQUEST			43
+#define RAD_COA_ACK			44
+#define RAD_COA_NAK			45
 
 /* Attribute types and values */
 #define RAD_USER_NAME			1	/* String */
@@ -179,6 +185,8 @@
 #define	RAD_ACCT_MULTI_SESSION_ID	50	/* String */
 #define	RAD_ACCT_LINK_COUNT		51	/* Integer */
 
+#define	RAD_ERROR_CAUSE			101	/* Integer */
+
 struct rad_handle;
 struct timeval;
 
@@ -192,6 +200,7 @@ int			 rad_config(struct rad_handle *, const char *);
 int			 rad_continue_send_request(struct rad_handle *, int,
 			    int *, struct timeval *);
 int			 rad_create_request(struct rad_handle *, int);
+int			 rad_create_response(struct rad_handle *, int);
 struct in_addr		 rad_cvt_addr(const void *);
 u_int32_t		 rad_cvt_int(const void *);
 char			*rad_cvt_string(const void *, size_t);
@@ -209,7 +218,10 @@ int			 rad_put_string(struct rad_handle *, int,
 int			 rad_put_message_authentic(struct rad_handle *);
 ssize_t			 rad_request_authenticator(struct rad_handle *, char *,
 			    size_t);
+int			 rad_receive_request(struct rad_handle *);
 int			 rad_send_request(struct rad_handle *);
+int			 rad_send_response(struct rad_handle *);
+struct rad_handle	*rad_server_open(int fd);
 const char		*rad_server_secret(struct rad_handle *);
 const char		*rad_strerror(struct rad_handle *);
 u_char			*rad_demangle(struct rad_handle *, const void *,
diff --git a/lib/libradius/radlib_private.h b/lib/libradius/radlib_private.h
index d323cbd..d886c74 100644
--- a/lib/libradius/radlib_private.h
+++ b/lib/libradius/radlib_private.h
@@ -38,6 +38,7 @@
 /* Handle types */
 #define RADIUS_AUTH		0   /* RADIUS authentication, default */
 #define RADIUS_ACCT		1   /* RADIUS accounting */
+#define RADIUS_SERVER		2   /* RADIUS server */
 
 /* Defaults */
 #define MAXTRIES		3
@@ -75,18 +76,18 @@ struct rad_handle {
 	int		 num_servers;	/* Number of valid server entries */
 	int		 ident;		/* Current identifier value */
 	char		 errmsg[ERRSIZE];	/* Most recent error message */
-	unsigned char	 request[MSGSIZE];	/* Request to send */
-	char	 	 request_created; /* rad_create_request() called? */
-	int		 req_len;	/* Length of request */
+	unsigned char	 out[MSGSIZE];	/* Request to send */
+	char		 out_created;	/* rad_create_request() called? */
+	int		 out_len;	/* Length of request */
 	char		 pass[PASSSIZE];	/* Cleartext password */
 	int		 pass_len;	/* Length of cleartext password */
 	int		 pass_pos;	/* Position of scrambled password */
-	char	 	 chap_pass;	/* Have we got a CHAP_PASSWORD ? */
+	char		 chap_pass;	/* Have we got a CHAP_PASSWORD ? */
 	int		 authentic_pos;	/* Position of message authenticator */
 	char		 eap_msg;	/* Are we an EAP Proxy? */
-	unsigned char	 response[MSGSIZE];	/* Response received */
-	int		 resp_len;	/* Length of response */
-	int		 resp_pos;	/* Current position scanning attrs */
+	unsigned char	 in[MSGSIZE];	/* Response received */
+	int		 in_len;	/* Length of response */
+	int		 in_pos;	/* Current position scanning attrs */
 	int		 total_tries;	/* How many requests we'll send */
 	int		 try;		/* How many requests we've sent */
 	int		 srv;		/* Server number we did last */
-- 
cgit v1.1


From f8c10fe31db266a7800e82230f6a563e1b5435ea Mon Sep 17 00:00:00 2001
From: mav <mav@FreeBSD.org>
Date: Fri, 30 Oct 2009 13:02:08 +0000
Subject: MFC rev. 197621: Fix bug, when RADIUS client gave up after single
 sendto() error, do not trying backup servers.

PR:		kern/103764, misc/139214
---
 lib/libradius/radlib.c | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

(limited to 'lib')

diff --git a/lib/libradius/radlib.c b/lib/libradius/radlib.c
index 8faaf7e..e4e4a94 100644
--- a/lib/libradius/radlib.c
+++ b/lib/libradius/radlib.c
@@ -650,17 +650,12 @@ rad_continue_send_request(struct rad_handle *h, int selected, int *fd,
 	n = sendto(h->fd, h->out, h->out_len, 0,
 	    (const struct sockaddr *)&h->servers[h->srv].addr,
 	    sizeof h->servers[h->srv].addr);
-	if (n != h->out_len) {
-		if (n == -1)
-			generr(h, "sendto: %s", strerror(errno));
-		else
-			generr(h, "sendto: short write");
-		return -1;
-	}
-
+	if (n != h->out_len)
+		tv->tv_sec = 1; /* Do not wait full timeout if send failed. */
+	else
+		tv->tv_sec = h->servers[h->srv].timeout;
 	h->try++;
 	h->servers[h->srv].num_tries++;
-	tv->tv_sec = h->servers[h->srv].timeout;
 	tv->tv_usec = 0;
 	*fd = h->fd;
 
-- 
cgit v1.1


From 250abcae59abf84b718b892517af472e077c3a03 Mon Sep 17 00:00:00 2001
From: edwin <edwin@FreeBSD.org>
Date: Sat, 31 Oct 2009 06:35:40 +0000
Subject: MFCs of r197764, r197765, r197766, r197847:

Modified locale(1) to be able to show the altmon_X fields and the
[cxX]_fmt's.  Also modify the "-k list" option to display only
fields with a certain prefix.

Add the comment "(FreeBSD only)" to the altmonth_x keywords
---
 lib/libc/locale/nl_langinfo.c |  6 ++++++
 lib/libc/stdtime/localtime.c  | 11 +++++++++--
 2 files changed, 15 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/locale/nl_langinfo.c b/lib/libc/locale/nl_langinfo.c
index 9bd5082..26ca025 100644
--- a/lib/libc/locale/nl_langinfo.c
+++ b/lib/libc/locale/nl_langinfo.c
@@ -93,6 +93,12 @@ nl_langinfo(nl_item item)
 	case ABMON_9: case ABMON_10: case ABMON_11: case ABMON_12:
 		ret = (char*) __get_current_time_locale()->mon[_REL(ABMON_1)];
 		break;
+	case ALTMON_1: case ALTMON_2: case ALTMON_3: case ALTMON_4:
+	case ALTMON_5: case ALTMON_6: case ALTMON_7: case ALTMON_8:
+	case ALTMON_9: case ALTMON_10: case ALTMON_11: case ALTMON_12:
+		ret = (char*)
+		    __get_current_time_locale()->alt_month[_REL(ALTMON_1)];
+		break;
 	case ERA:
 		/* XXX: need to be implemented  */
 		ret = "";
diff --git a/lib/libc/stdtime/localtime.c b/lib/libc/stdtime/localtime.c
index e0ed73f..9fc5f3e 100644
--- a/lib/libc/stdtime/localtime.c
+++ b/lib/libc/stdtime/localtime.c
@@ -21,6 +21,7 @@ __FBSDID("$FreeBSD$");
 #include "namespace.h"
 #include <sys/types.h>
 #include <sys/stat.h>
+#include <errno.h>
 #include <fcntl.h>
 #include <pthread.h>
 #include "private.h"
@@ -1413,13 +1414,16 @@ const time_t * const	timep;
 	static pthread_mutex_t localtime_mutex = PTHREAD_MUTEX_INITIALIZER;
 	static pthread_key_t localtime_key = -1;
 	struct tm *p_tm;
+	int r;
 
 	if (__isthreaded != 0) {
 		if (localtime_key < 0) {
 			_pthread_mutex_lock(&localtime_mutex);
 			if (localtime_key < 0) {
-				if (_pthread_key_create(&localtime_key, free) < 0) {
+				if ((r = _pthread_key_create(&localtime_key,
+				    free)) != 0) {
 					_pthread_mutex_unlock(&localtime_mutex);
+					errno = r;
 					return(NULL);
 				}
 			}
@@ -1512,13 +1516,16 @@ const time_t * const	timep;
 	static pthread_mutex_t gmtime_mutex = PTHREAD_MUTEX_INITIALIZER;
 	static pthread_key_t gmtime_key = -1;
 	struct tm *p_tm;
+	int r;
 
 	if (__isthreaded != 0) {
 		if (gmtime_key < 0) {
 			_pthread_mutex_lock(&gmtime_mutex);
 			if (gmtime_key < 0) {
-				if (_pthread_key_create(&gmtime_key, free) < 0) {
+				if ((r = _pthread_key_create(&gmtime_key,
+				    free)) != 0) {
 					_pthread_mutex_unlock(&gmtime_mutex);
+					errno = r;
 					return(NULL);
 				}
 			}
-- 
cgit v1.1


From 8aa1396550009dbbb910c8802b9f340d933c5a29 Mon Sep 17 00:00:00 2001
From: sson <sson@FreeBSD.org>
Date: Sat, 31 Oct 2009 21:22:18 +0000
Subject: MFC 197240,197241,197242,197243,197293,197294,197407:

Add EVFILT_USER filter and EV_DISPATCH/EV_RECEIPT flags to kevent(2).

Approved by: rwatson (mentor)
---
 lib/libc/sys/kqueue.2 | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 53 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libc/sys/kqueue.2 b/lib/libc/sys/kqueue.2
index 119ff79..e899a1b 100644
--- a/lib/libc/sys/kqueue.2
+++ b/lib/libc/sys/kqueue.2
@@ -24,7 +24,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd September 6, 2007
+.Dd September 15, 2009
 .Dt KQUEUE 2
 .Os
 .Sh NAME
@@ -201,11 +201,25 @@ Disable the event so
 .Fn kevent
 will not return it.
 The filter itself is not disabled.
+.It EV_DISPATCH
+Disable the event source immediately after delivery of an event.
+See 
+.Dv EV_DISABLE
+above.
 .It EV_DELETE
 Removes the event from the kqueue.
 Events which are attached to
 file descriptors are automatically deleted on the last close of
 the descriptor.
+.It EV_RECEIPT
+This flag is useful for making bulk changes to a kqueue without draining
+any pending events.
+When passed as input, it forces
+.Dv EV_ERROR
+to always be returned.
+When a filter is successfully added the 
+.Va data
+field will be zero.
 .It EV_ONESHOT
 Causes the event to return only the first occurrence of the filter
 being triggered.
@@ -441,6 +455,44 @@ The link state is invalid.
 On return,
 .Va fflags
 contains the events which triggered the filter.
+.It Dv EVFILT_USER
+Establishes a user event identified by
+.Va ident
+which is not assosicated with any kernel mechanism but is triggered by
+user level code.
+The lower 24 bits of the 
+.Va fflags
+may be used for user defined flags and manipulated using the following:
+.Bl -tag -width XXNOTE_FFLAGSMASK 
+.It Dv NOTE_FFNOP
+Ignore the input
+.Va fflags .
+.It Dv NOTE_FFAND
+Bitwise AND
+.Va fflags .
+.It Dv NOTE_FFOR
+Bitwise OR
+.Va fflags .
+.It Dv NOTE_COPY
+Copy
+.Va fflags .
+.It Dv NOTE_FFCTRLMASK
+Control mask for
+.Va fflags .
+.It Dv NOTE_FFLAGSMASK
+User defined flag mask for
+.Va fflags .
+.El
+.Pp
+A user event is triggered for output with the following:
+.Bl -tag -width XXNOTE_FFLAGSMASK
+.It Dv NOTE_TRIGGER
+Cause the event to be triggered.
+.El
+.Pp
+On return,
+.Va fflags
+contains the users defined flags in the lower 24 bits.
 .El
 .Sh RETURN VALUES
 The
-- 
cgit v1.1


From 3f9761f9fd2e480b15ee8885d6684e9ef6d6ab63 Mon Sep 17 00:00:00 2001
From: alc <alc@FreeBSD.org>
Date: Sun, 1 Nov 2009 19:22:07 +0000
Subject: MFC r197331, r197394   Add getpagesizes(3).

---
 lib/libc/gen/Makefile.inc   |  6 +--
 lib/libc/gen/Symbol.map     |  4 ++
 lib/libc/gen/getpagesizes.3 | 99 +++++++++++++++++++++++++++++++++++++++++++++
 lib/libc/gen/getpagesizes.c | 78 +++++++++++++++++++++++++++++++++++
 4 files changed, 184 insertions(+), 3 deletions(-)
 create mode 100644 lib/libc/gen/getpagesizes.3
 create mode 100644 lib/libc/gen/getpagesizes.c

(limited to 'lib')

diff --git a/lib/libc/gen/Makefile.inc b/lib/libc/gen/Makefile.inc
index 8837038..b06f846 100644
--- a/lib/libc/gen/Makefile.inc
+++ b/lib/libc/gen/Makefile.inc
@@ -15,7 +15,7 @@ SRCS+=  __getosreldate.c __xuname.c \
 	getbootfile.c getbsize.c \
 	getcap.c getcwd.c getdomainname.c getgrent.c getgrouplist.c \
 	gethostname.c getloadavg.c getlogin.c getmntinfo.c getnetgrent.c \
-	getosreldate.c getpagesize.c \
+	getosreldate.c getpagesize.c getpagesizes.c \
 	getpeereid.c getprogname.c getpwent.c getttyent.c \
 	getusershell.c getvfsbyname.c glob.c \
 	initgroups.c isatty.c isinf.c isnan.c jrand48.c lcong48.c \
@@ -51,8 +51,8 @@ MAN+=	alarm.3 arc4random.3 \
 	getbootfile.3 getbsize.3 getcap.3 getcontext.3 getcwd.3 \
 	getdiskbyname.3 getdomainname.3 getfsent.3 \
 	getgrent.3 getgrouplist.3 gethostname.3 getloadavg.3 \
-	getmntinfo.3 getnetgrent.3 getosreldate.3 \
-	getpagesize.3 getpass.3 getpeereid.3 getprogname.3 getpwent.3 \
+	getmntinfo.3 getnetgrent.3 getosreldate.3 getpagesize.3 \
+	getpagesizes.3 getpass.3 getpeereid.3 getprogname.3 getpwent.3 \
 	getttyent.3 getusershell.3 getvfsbyname.3 \
 	glob.3 initgroups.3 isgreater.3 ldexp.3 lockf.3 makecontext.3 \
 	modf.3 \
diff --git a/lib/libc/gen/Symbol.map b/lib/libc/gen/Symbol.map
index 4f1efaf..197430a 100644
--- a/lib/libc/gen/Symbol.map
+++ b/lib/libc/gen/Symbol.map
@@ -366,6 +366,10 @@ FBSD_1.1 {
 	tcsetsid;
 };
 
+FBSD_1.2 {
+	getpagesizes;
+};
+
 FBSDprivate_1.0 {
 	/* needed by thread libraries */
 	__thr_jtable;
diff --git a/lib/libc/gen/getpagesizes.3 b/lib/libc/gen/getpagesizes.3
new file mode 100644
index 0000000..959df81
--- /dev/null
+++ b/lib/libc/gen/getpagesizes.3
@@ -0,0 +1,99 @@
+.\" Copyright (c) 2009 Alan L. Cox <alc@cs.rice.edu>
+.\" All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd September 21, 2009
+.Dt GETPAGESIZES 3
+.Os
+.Sh NAME
+.Nm getpagesizes
+.Nd "get system page sizes"
+.Sh LIBRARY
+.Lb libc
+.Sh SYNOPSIS
+.In sys/mman.h
+.Ft int
+.Fn getpagesizes "size_t pagesize[]" "int nelem"
+.Sh DESCRIPTION
+The
+.Fn getpagesizes
+function retrieves page size information from the system.
+When it is called with
+.Fa pagesize
+specified as
+.Dv NULL
+and
+.Fa nelem
+specified as 0, it returns the number of distinct page sizes that are
+supported by the system.
+Otherwise, it assigns up to
+.Fa nelem
+of the system-supported page sizes to consecutive elements of the
+array referenced by
+.Fa pagesize .
+These page sizes are expressed in bytes.
+In this case,
+.Fn getpagesizes
+returns the number of such page sizes that it assigned to the array. 
+.Sh RETURN VALUES
+If successful, the
+.Fn getpagesizes
+function returns either the number of page sizes that are supported by
+the system or the number of supported page sizes that it assigned to
+the array referenced by
+.Fa pagesize .
+Otherwise, it returns the value\~\-1 and sets
+.Va errno
+to indicate the error.
+.Sh ERRORS
+The
+.Fn getpagesizes
+function will succeed unless:
+.Bl -tag -width Er
+.It Bq Er EINVAL
+The
+.Fa pagesize
+argument is
+.Dv NULL
+and the
+.Fa nelem
+argument is non-zero.
+.It Bq Er EINVAL
+The
+.Fa nelem
+argument is less than zero.
+.El
+.Sh SEE ALSO
+.Xr getpagesize 3
+.Sh HISTORY
+The
+.Fn getpagesizes
+function first appeared in Solaris 9.
+This manual page was written in conjunction with a new but compatible
+implementation that was first released in
+.Fx 7.3 .
+.Sh AUTHORS
+This manual page was written by
+.An Alan L. Cox Aq alc@cs.rice.edu .
diff --git a/lib/libc/gen/getpagesizes.c b/lib/libc/gen/getpagesizes.c
new file mode 100644
index 0000000..b0de939
--- /dev/null
+++ b/lib/libc/gen/getpagesizes.c
@@ -0,0 +1,78 @@
+/*-
+ * Copyright (c) 2009 Alan L. Cox <alc@cs.rice.edu>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/mman.h>
+#include <sys/sysctl.h>
+
+#include <errno.h>
+
+/*
+ * Retrieves page size information from the system.  Specifically, returns the
+ * number of distinct page sizes that are supported by the system, if
+ * "pagesize" is NULL and "nelem" is 0.  Otherwise, assigns up to "nelem" of
+ * the system-supported page sizes to consecutive elements of the array
+ * referenced by "pagesize", and returns the number of such page sizes that it
+ * assigned to the array.  These page sizes are expressed in bytes.
+ *
+ * The implementation of this function does not directly or indirectly call
+ * malloc(3) or any other dynamic memory allocator that may itself call this
+ * function.
+ */
+int
+getpagesizes(size_t pagesize[], int nelem)
+{
+	static u_long ps[MAXPAGESIZES];
+	static int nops;
+	size_t size;
+	int i; 
+
+	if (nelem < 0 || (nelem > 0 && pagesize == NULL)) {
+		errno = EINVAL;
+		return (-1);
+	}
+	/* Cache the result of the sysctl(2). */
+	if (nops == 0) {
+		size = sizeof(ps);
+		if (sysctlbyname("hw.pagesizes", ps, &size, NULL, 0) == -1)
+			return (-1);
+		/* Count the number of page sizes that are supported. */
+		nops = size / sizeof(ps[0]);
+		while (nops > 0 && ps[nops - 1] == 0)
+			nops--;
+	}
+	if (pagesize == NULL)
+		return (nops);
+	/* Return up to "nelem" page sizes from the cached result. */
+	if (nelem > nops)
+		nelem = nops;
+	for (i = 0; i < nelem; i++)
+		pagesize[i] = ps[i];
+	return (nelem);
+}
-- 
cgit v1.1


From ae23acf799f18392a73bb2f885e0a8e47e358720 Mon Sep 17 00:00:00 2001
From: alc <alc@FreeBSD.org>
Date: Sun, 1 Nov 2009 20:24:17 +0000
Subject: MFC r197163   Add the FBSD_1.2 namespace.

Reminded by:	kib
---
 lib/libc/Versions.def | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libc/Versions.def b/lib/libc/Versions.def
index a17e0a5..da0ca6f 100644
--- a/lib/libc/Versions.def
+++ b/lib/libc/Versions.def
@@ -1,5 +1,11 @@
 # $FreeBSD$
 
+#
+# Note: Whenever bumping the FBSD version, always make
+#       FBSDprivate_1.0 depend on the new FBSD version.
+#       This will keep it at the end of the dependency chain.
+#
+
 # This is our first version; it depends on no other.
 # This version was first added to 7.0-current.
 FBSD_1.0 {
@@ -9,6 +15,10 @@ FBSD_1.0 {
 FBSD_1.1 {
 } FBSD_1.0;
 
+# This version was first added to 9.0-current.
+FBSD_1.2 {
+} FBSD_1.1;
+
 # This is our private namespace.  Any global interfaces that are
 # strictly for use only by other FreeBSD applications and libraries
 # are listed here.  We use a separate namespace so we can write
@@ -16,4 +26,4 @@ FBSD_1.1 {
 #
 # Please do NOT increment the version of this namespace.
 FBSDprivate_1.0 {
-} FBSD_1.1;
+} FBSD_1.2;
-- 
cgit v1.1


From 29a76c7eb2b81324d3a4f6987a1aaaa32f8732ab Mon Sep 17 00:00:00 2001
From: alc <alc@FreeBSD.org>
Date: Mon, 2 Nov 2009 18:44:01 +0000
Subject: MFC r197524   Make malloc(3) superpage aware.

---
 lib/libc/stdlib/malloc.3 |  6 ++++--
 lib/libc/stdlib/malloc.c | 15 +++++++++++++++
 2 files changed, 19 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/stdlib/malloc.3 b/lib/libc/stdlib/malloc.3
index a621a24..308ba7b 100644
--- a/lib/libc/stdlib/malloc.3
+++ b/lib/libc/stdlib/malloc.3
@@ -32,7 +32,7 @@
 .\"     @(#)malloc.3	8.1 (Berkeley) 6/4/93
 .\" $FreeBSD$
 .\"
-.Dd August 26, 2008
+.Dd September 26, 2009
 .Dt MALLOC 3
 .Os
 .Sh NAME
@@ -245,7 +245,8 @@ will be initialized to 0x5a.
 This is intended for debugging and will impact performance negatively.
 .It K
 Double/halve the virtual memory chunk size.
-The default chunk size is 1 MB.
+The default chunk size is the maximum of 1 MB and the largest
+page size that is less than or equal to 4 MB.
 .It M
 Use
 .Xr mmap 2
@@ -561,6 +562,7 @@ _malloc_options = "X";
 .Xr alloca 3 ,
 .Xr atexit 3 ,
 .Xr getpagesize 3 ,
+.Xr getpagesizes 3 ,
 .Xr memory 3 ,
 .Xr posix_memalign 3
 .Sh STANDARDS
diff --git a/lib/libc/stdlib/malloc.c b/lib/libc/stdlib/malloc.c
index 8b13d12..80e26e6 100644
--- a/lib/libc/stdlib/malloc.c
+++ b/lib/libc/stdlib/malloc.c
@@ -4795,6 +4795,21 @@ malloc_init_hard(void)
 		}
 	}
 
+	/*
+	 * Increase the chunk size to the largest page size that is greater
+	 * than the default chunk size and less than or equal to 4MB.
+	 */
+	{
+		size_t pagesizes[MAXPAGESIZES];
+		int k, nsizes;
+
+		nsizes = getpagesizes(pagesizes, MAXPAGESIZES);
+		for (k = 0; k < nsizes; k++)
+			if (pagesizes[k] <= (1LU << 22))
+				while ((1LU << opt_chunk_2pow) < pagesizes[k])
+					opt_chunk_2pow++;
+	}
+
 	for (i = 0; i < 3; i++) {
 		unsigned j;
 
-- 
cgit v1.1


From 4d34fdb382f7bfd134efd42ee3fd2344edd5391f Mon Sep 17 00:00:00 2001
From: brueffer <brueffer@FreeBSD.org>
Date: Wed, 4 Nov 2009 13:40:04 +0000
Subject: MFC: r198542

Initialize f_rabuf in the raw device case. A subsequent close()
later on would try to free it, leading to a crash.
---
 lib/libstand/open.c | 1 +
 1 file changed, 1 insertion(+)

(limited to 'lib')

diff --git a/lib/libstand/open.c b/lib/libstand/open.c
index acbda1c..49dc660 100644
--- a/lib/libstand/open.c
+++ b/lib/libstand/open.c
@@ -113,6 +113,7 @@ open(const char *fname, int mode)
     /* see if we opened a raw device; otherwise, 'file' is the file name. */
     if (file == (char *)0 || *file == '\0') {
 	f->f_flags |= F_RAW;
+	f->f_rabuf = NULL;
 	return (fd);
     }
 
-- 
cgit v1.1


From 42a4c580efb286c53601701f114a993d65880356 Mon Sep 17 00:00:00 2001
From: thompsa <thompsa@FreeBSD.org>
Date: Wed, 11 Nov 2009 01:27:58 +0000
Subject: MFC r199055

 - fix refcounting error during data transfer
 - fix a memory leak on the USB backend
 - fix invalid pointer computations (in one case memory outside the allocated
   area was written in LibUSB v1.0)
 - make sure memory is always initialised, also in failing cases
 - add missing functions from v1.0.4

PR:		usb/140325
---
 lib/libusb/libusb.h          |  37 +++++++---
 lib/libusb/libusb10.c        |  26 ++++++-
 lib/libusb/libusb10_desc.c   |  34 ++++++---
 lib/libusb/libusb10_io.c     | 172 +++++++++++++++++++++++++++++++++++++++++--
 lib/libusb/libusb20.c        |   8 +-
 lib/libusb/libusb20_desc.c   |   3 +
 lib/libusb/libusb20_ugen20.c |   6 ++
 7 files changed, 251 insertions(+), 35 deletions(-)

(limited to 'lib')

diff --git a/lib/libusb/libusb.h b/lib/libusb/libusb.h
index f65b57a..bdf2812 100644
--- a/lib/libusb/libusb.h
+++ b/lib/libusb/libusb.h
@@ -271,9 +271,11 @@ typedef struct libusb_control_setup {
 	uint16_t wLength;
 }	libusb_control_setup;
 
+#define	LIBUSB_CONTROL_SETUP_SIZE	8	/* bytes */
+
 typedef struct libusb_iso_packet_descriptor {
-	unsigned int length;
-	unsigned int actual_length;
+	uint32_t length;
+	uint32_t actual_length;
 	enum libusb_transfer_status status;
 }	libusb_iso_packet_descriptor __aligned(sizeof(void *));
 
@@ -282,9 +284,9 @@ typedef void (*libusb_transfer_cb_fn) (struct libusb_transfer *transfer);
 typedef struct libusb_transfer {
 	libusb_device_handle *dev_handle;
 	uint8_t	flags;
-	unsigned int endpoint;
+	uint32_t endpoint;
 	uint8_t type;
-	unsigned int timeout;
+	uint32_t timeout;
 	enum libusb_transfer_status status;
 	int	length;
 	int	actual_length;
@@ -320,7 +322,7 @@ int	libusb_get_configuration(libusb_device_handle * devh, int *config);
 int	libusb_set_configuration(libusb_device_handle * devh, int configuration);
 int	libusb_claim_interface(libusb_device_handle * devh, int interface_number);
 int	libusb_release_interface(libusb_device_handle * devh, int interface_number);
-int	libusb_reset_device(libusb_device_handle * dev);
+int	libusb_reset_device(libusb_device_handle * devh);
 int 	libusb_kernel_driver_active(libusb_device_handle * devh, int interface);
 int 	libusb_detach_kernel_driver(libusb_device_handle * devh, int interface);
 int 	libusb_attach_kernel_driver(libusb_device_handle * devh, int interface);
@@ -333,7 +335,8 @@ int	libusb_get_active_config_descriptor(libusb_device * dev, struct libusb_confi
 int	libusb_get_config_descriptor(libusb_device * dev, uint8_t config_index, struct libusb_config_descriptor **config);
 int	libusb_get_config_descriptor_by_value(libusb_device * dev, uint8_t bConfigurationValue, struct libusb_config_descriptor **config);
 void	libusb_free_config_descriptor(struct libusb_config_descriptor *config);
-int	libusb_get_string_descriptor_ascii(libusb_device_handle * dev, uint8_t desc_index, uint8_t *data, int length);
+int	libusb_get_string_descriptor_ascii(libusb_device_handle * devh, uint8_t desc_index, uint8_t *data, int length);
+int	libusb_get_descriptor(libusb_device_handle * devh, uint8_t desc_type, uint8_t desc_index, uint8_t *data, int length);
 
 /* Asynchronous device I/O */
 
@@ -341,7 +344,16 @@ struct libusb_transfer *libusb_alloc_transfer(int iso_packets);
 void	libusb_free_transfer(struct libusb_transfer *transfer);
 int	libusb_submit_transfer(struct libusb_transfer *transfer);
 int	libusb_cancel_transfer(struct libusb_transfer *transfer);
-uint8_t *libusb_get_iso_packet_buffer_simple(struct libusb_transfer *transfer, unsigned int packet);
+uint8_t *libusb_get_iso_packet_buffer(struct libusb_transfer *transfer, uint32_t index);
+uint8_t *libusb_get_iso_packet_buffer_simple(struct libusb_transfer *transfer, uint32_t index);
+void	libusb_set_iso_packet_lengths(struct libusb_transfer *transfer, uint32_t length);
+uint8_t *libusb_control_transfer_get_data(struct libusb_transfer *transfer);
+struct libusb_control_setup *libusb_control_transfer_get_setup(struct libusb_transfer *transfer);
+void	libusb_fill_control_setup(uint8_t *buf, uint8_t bmRequestType, uint8_t bRequest, uint16_t wValue, uint16_t wIndex, uint16_t wLength);
+void	libusb_fill_control_transfer(struct libusb_transfer *transfer, libusb_device_handle *devh, uint8_t *buf, libusb_transfer_cb_fn callback, void *user_data, uint32_t timeout);
+void	libusb_fill_bulk_transfer(struct libusb_transfer *transfer, libusb_device_handle *devh, uint8_t endpoint, uint8_t *buf, int length, libusb_transfer_cb_fn callback, void *user_data, uint32_t timeout);
+void	libusb_fill_interrupt_transfer(struct libusb_transfer *transfer, libusb_device_handle *devh, uint8_t endpoint, uint8_t *buf, int length, libusb_transfer_cb_fn callback, void *user_data, uint32_t timeout);
+void	libusb_fill_iso_transfer(struct libusb_transfer *transfer, libusb_device_handle *devh, uint8_t endpoint, uint8_t *buf, int length, int npacket, libusb_transfer_cb_fn callback, void *user_data, uint32_t timeout);
 
 /* Polling and timing */
 
@@ -362,9 +374,14 @@ struct libusb_pollfd **libusb_get_pollfds(libusb_context * ctx);
 
 /* Synchronous device I/O */
 
-int	libusb_control_transfer(libusb_device_handle * devh, uint8_t bmRequestType, uint8_t bRequest, uint16_t wValue, uint16_t wIndex, uint8_t *data, uint16_t wLength, unsigned int timeout);
-int	libusb_bulk_transfer(libusb_device_handle *devh, uint8_t endpoint, uint8_t *data, int length, int *transferred, unsigned int timeout);
-int	libusb_interrupt_transfer(libusb_device_handle *devh, uint8_t endpoint, uint8_t *data, int length, int *transferred, unsigned int timeout);
+int	libusb_control_transfer(libusb_device_handle * devh, uint8_t bmRequestType, uint8_t bRequest, uint16_t wValue, uint16_t wIndex, uint8_t *data, uint16_t wLength, uint32_t timeout);
+int	libusb_bulk_transfer(libusb_device_handle * devh, uint8_t endpoint, uint8_t *data, int length, int *transferred, uint32_t timeout);
+int	libusb_interrupt_transfer(libusb_device_handle * devh, uint8_t endpoint, uint8_t *data, int length, int *transferred, uint32_t timeout);
+
+/* Byte-order */
+
+uint16_t libusb_cpu_to_le16(uint16_t x);
+uint16_t libusb_le16_to_cpu(uint16_t x);
 
 #if 0
 {					/* indent fix */
diff --git a/lib/libusb/libusb10.c b/lib/libusb/libusb10.c
index fa9130d..55ee1c5 100644
--- a/lib/libusb/libusb10.c
+++ b/lib/libusb/libusb10.c
@@ -35,6 +35,7 @@
 #include <sys/ioctl.h>
 #include <sys/filio.h>
 #include <sys/queue.h>
+#include <sys/endian.h>
 
 #include "libusb20.h"
 #include "libusb20_desc.h"
@@ -185,8 +186,6 @@ libusb_get_device_list(libusb_context *ctx, libusb_device ***list)
 	/* create libusb v1.0 compliant devices */
 	i = 0;
 	while ((pdev = libusb20_be_device_foreach(usb_backend, NULL))) {
-		/* get device into libUSB v1.0 list */
-		libusb20_be_dequeue_device(usb_backend, pdev);
 
 		dev = malloc(sizeof(*dev));
 		if (dev == NULL) {
@@ -199,6 +198,10 @@ libusb_get_device_list(libusb_context *ctx, libusb_device ***list)
 			libusb20_be_free(usb_backend);
 			return (LIBUSB_ERROR_NO_MEM);
 		}
+
+		/* get device into libUSB v1.0 list */
+		libusb20_be_dequeue_device(usb_backend, pdev);
+
 		memset(dev, 0, sizeof(*dev));
 
 		/* init transfer queues */
@@ -416,6 +419,8 @@ libusb_close(struct libusb20_device *pdev)
 	libusb10_remove_pollfd(ctx, &dev->dev_poll);
 
 	libusb20_dev_close(pdev);
+
+	/* unref will free the "pdev" when the refcount reaches zero */
 	libusb_unref_device(dev);
 
 	/* make sure our event loop detects the closed device */
@@ -1195,7 +1200,7 @@ libusb_submit_transfer(struct libusb_transfer *uxfer)
 	struct libusb20_transfer *pxfer1;
 	struct libusb_super_transfer *sxfer;
 	struct libusb_device *dev;
-	unsigned int endpoint;
+	uint32_t endpoint;
 	int err;
 
 	if (uxfer == NULL)
@@ -1252,7 +1257,7 @@ libusb_cancel_transfer(struct libusb_transfer *uxfer)
 	struct libusb20_transfer *pxfer1;
 	struct libusb_super_transfer *sxfer;
 	struct libusb_device *dev;
-	unsigned int endpoint;
+	uint32_t endpoint;
 
 	if (uxfer == NULL)
 		return (LIBUSB_ERROR_INVALID_PARAM);
@@ -1312,3 +1317,16 @@ libusb10_cancel_all_transfer(libusb_device *dev)
 {
 	/* TODO */
 }
+
+uint16_t
+libusb_cpu_to_le16(uint16_t x)
+{
+	return (htole16(x));
+}
+
+uint16_t
+libusb_le16_to_cpu(uint16_t x)
+{
+	return (le16toh(x));
+}
+
diff --git a/lib/libusb/libusb10_desc.c b/lib/libusb/libusb10_desc.c
index c43443a..0215bce 100644
--- a/lib/libusb/libusb10_desc.c
+++ b/lib/libusb/libusb10_desc.c
@@ -35,6 +35,8 @@
 #include "libusb.h"
 #include "libusb10.h"
 
+#define	N_ALIGN(n) (-((-(n)) & (-8UL)))
+
 /* USB descriptors */
 
 int
@@ -114,17 +116,17 @@ libusb_get_config_descriptor(libusb_device *dev, uint8_t config_index,
 
 	nalt = nif = pconf->num_interface;
 	nep = 0;
-	nextra = pconf->extra.len;
+	nextra = N_ALIGN(pconf->extra.len);
 
 	for (i = 0; i < nif; i++) {
 
 		pinf = pconf->interface + i;
-		nextra += pinf->extra.len;
+		nextra += N_ALIGN(pinf->extra.len);
 		nep += pinf->num_endpoints;
 		k = pinf->num_endpoints;
 		pend = pinf->endpoints;
 		while (k--) {
-			nextra += pend->extra.len;
+			nextra += N_ALIGN(pend->extra.len);
 			pend++;
 		}
 
@@ -132,12 +134,12 @@ libusb_get_config_descriptor(libusb_device *dev, uint8_t config_index,
 		nalt += pinf->num_altsetting;
 		pinf = pinf->altsetting;
 		while (j--) {
-			nextra += pinf->extra.len;
+			nextra += N_ALIGN(pinf->extra.len);
 			nep += pinf->num_endpoints;
 			k = pinf->num_endpoints;
 			pend = pinf->endpoints;
 			while (k--) {
-				nextra += pend->extra.len;
+				nextra += N_ALIGN(pend->extra.len);
 				pend++;
 			}
 			pinf++;
@@ -150,17 +152,18 @@ libusb_get_config_descriptor(libusb_device *dev, uint8_t config_index,
 	    (nalt * sizeof(libusb_interface_descriptor)) +
 	    (nep * sizeof(libusb_endpoint_descriptor));
 
+	nextra = N_ALIGN(nextra);
+
 	pconfd = malloc(nextra);
 
 	if (pconfd == NULL) {
 		free(pconf);
 		return (LIBUSB_ERROR_NO_MEM);
 	}
-	/* make sure memory is clean */
+	/* make sure memory is initialised */
 	memset(pconfd, 0, nextra);
 
-	pconfd->interface = (libusb_interface *) (pconfd +
-	    sizeof(libusb_config_descriptor));
+	pconfd->interface = (libusb_interface *) (pconfd + 1);
 
 	ifd = (libusb_interface_descriptor *) (pconfd->interface + nif);
 	endd = (libusb_endpoint_descriptor *) (ifd + nalt);
@@ -181,7 +184,7 @@ libusb_get_config_descriptor(libusb_device *dev, uint8_t config_index,
 		pconfd->extra_length = pconf->extra.len;
 		pconfd->extra = pextra;
 		memcpy(pextra, pconf->extra.ptr, pconfd->extra_length);
-		pextra += pconfd->extra_length;
+		pextra += N_ALIGN(pconfd->extra_length);
 	}
 	/* setup all interface and endpoint pointers */
 
@@ -221,7 +224,7 @@ libusb_get_config_descriptor(libusb_device *dev, uint8_t config_index,
 				ifd->extra_length = pinf->extra.len;
 				ifd->extra = pextra;
 				memcpy(pextra, pinf->extra.ptr, pinf->extra.len);
-				pextra += pinf->extra.len;
+				pextra += N_ALIGN(pinf->extra.len);
 			}
 			for (k = 0; k < pinf->num_endpoints; k++) {
 				pend = &pinf->endpoints[k];
@@ -238,7 +241,7 @@ libusb_get_config_descriptor(libusb_device *dev, uint8_t config_index,
 					endd->extra_length = pend->extra.len;
 					endd->extra = pextra;
 					memcpy(pextra, pend->extra.ptr, pend->extra.len);
-					pextra += pend->extra.len;
+					pextra += N_ALIGN(pend->extra.len);
 				}
 			}
 		}
@@ -304,3 +307,12 @@ libusb_get_string_descriptor_ascii(libusb_device_handle *pdev,
 
 	return (LIBUSB_ERROR_OTHER);
 }
+
+int
+libusb_get_descriptor(libusb_device_handle * devh, uint8_t desc_type, 
+    uint8_t desc_index, uint8_t *data, int length)
+{
+	return (libusb_control_transfer(devh, LIBUSB_ENDPOINT_IN,
+	    LIBUSB_REQUEST_GET_DESCRIPTOR, (desc_type << 8) | desc_index, 0, data,
+	    length, 1000));
+}
diff --git a/lib/libusb/libusb10_io.c b/lib/libusb/libusb10_io.c
index a5bb85f..3d6daef 100644
--- a/lib/libusb/libusb10_io.c
+++ b/lib/libusb/libusb10_io.c
@@ -32,6 +32,7 @@
 #include <time.h>
 #include <errno.h>
 #include <sys/queue.h>
+#include <sys/endian.h>
 
 #include "libusb20.h"
 #include "libusb20_desc.h"
@@ -148,19 +149,19 @@ libusb10_handle_events_sub(struct libusb_context *ctx, struct timeval *tv)
 		goto do_done;
 	}
 	for (i = 0; i != nfds; i++) {
-		if (fds[i].revents == 0)
-			continue;
 		if (ppdev[i] != NULL) {
 			dev = libusb_get_device(ppdev[i]);
 
-			err = libusb20_dev_process(ppdev[i]);
+			if (fds[i].revents == 0)
+				err = 0;	/* nothing to do */
+			else
+				err = libusb20_dev_process(ppdev[i]);
+
 			if (err) {
 				/* cancel all transfers - device is gone */
 				libusb10_cancel_all_transfer(dev);
-				/*
-				 * make sure we don't go into an infinite
-				 * loop
-				 */
+
+				/* remove USB device from polling loop */
 				libusb10_remove_pollfd(dev->ctx, &dev->dev_poll);
 			}
 			CTX_UNLOCK(ctx);
@@ -573,3 +574,160 @@ libusb_interrupt_transfer(libusb_device_handle *devh,
 	DPRINTF(ctx, LIBUSB_DEBUG_FUNCTION, "libusb_interrupt_transfer leave");
 	return (ret);
 }
+
+uint8_t *
+libusb_get_iso_packet_buffer(struct libusb_transfer *transfer, uint32_t index)
+{
+	uint8_t *ptr;
+	uint32_t n;
+
+	if (transfer->num_iso_packets < 0)
+		return (NULL);
+
+	if (index >= (uint32_t)transfer->num_iso_packets)
+		return (NULL);
+
+	ptr = transfer->buffer;
+	if (ptr == NULL)
+		return (NULL);
+
+	for (n = 0; n != index; n++) {
+		ptr += transfer->iso_packet_desc[n].length;
+	}
+	return (ptr);
+}
+
+uint8_t *
+libusb_get_iso_packet_buffer_simple(struct libusb_transfer *transfer, uint32_t index)
+{
+	uint8_t *ptr;
+
+	if (transfer->num_iso_packets < 0)
+		return (NULL);
+
+	if (index >= (uint32_t)transfer->num_iso_packets)
+		return (NULL);
+
+	ptr = transfer->buffer;
+	if (ptr == NULL)
+		return (NULL);
+
+	ptr += transfer->iso_packet_desc[0].length * index;
+
+	return (ptr);
+}
+
+void
+libusb_set_iso_packet_lengths(struct libusb_transfer *transfer, uint32_t length)
+{
+	int n;
+
+	if (transfer->num_iso_packets < 0)
+		return;
+
+	for (n = 0; n != transfer->num_iso_packets; n++)
+		transfer->iso_packet_desc[n].length = length;
+}
+
+uint8_t *
+libusb_control_transfer_get_data(struct libusb_transfer *transfer)
+{
+	if (transfer->buffer == NULL)
+		return (NULL);
+
+	return (transfer->buffer + LIBUSB_CONTROL_SETUP_SIZE);
+}
+
+struct libusb_control_setup *
+libusb_control_transfer_get_setup(struct libusb_transfer *transfer)
+{
+	return ((struct libusb_control_setup *)transfer->buffer);
+}
+
+void
+libusb_fill_control_setup(uint8_t *buf, uint8_t bmRequestType,
+    uint8_t bRequest, uint16_t wValue,
+    uint16_t wIndex, uint16_t wLength)
+{
+	struct libusb_control_setup *req = (struct libusb_control_setup *)buf;
+
+	/* The alignment is OK for all fields below. */
+	req->bmRequestType = bmRequestType;
+	req->bRequest = bRequest;
+	req->wValue = htole16(wValue);
+	req->wIndex = htole16(wIndex);
+	req->wLength = htole16(wLength);
+}
+
+void
+libusb_fill_control_transfer(struct libusb_transfer *transfer, 
+    libusb_device_handle *devh, uint8_t *buf,
+    libusb_transfer_cb_fn callback, void *user_data,
+    uint32_t timeout)
+{
+	struct libusb_control_setup *setup = (struct libusb_control_setup *)buf;
+
+	transfer->dev_handle = devh;
+	transfer->endpoint = 0;
+	transfer->type = LIBUSB_TRANSFER_TYPE_CONTROL;
+	transfer->timeout = timeout;
+	transfer->buffer = buf;
+	if (setup != NULL)
+		transfer->length = LIBUSB_CONTROL_SETUP_SIZE
+			+ le16toh(setup->wLength);
+	else
+		transfer->length = 0;
+	transfer->user_data = user_data;
+	transfer->callback = callback;
+
+}
+
+void
+libusb_fill_bulk_transfer(struct libusb_transfer *transfer, 
+    libusb_device_handle *devh, uint8_t endpoint, uint8_t *buf, 
+    int length, libusb_transfer_cb_fn callback, void *user_data,
+    uint32_t timeout)
+{
+	transfer->dev_handle = devh;
+	transfer->endpoint = endpoint;
+	transfer->type = LIBUSB_TRANSFER_TYPE_BULK;
+	transfer->timeout = timeout;
+	transfer->buffer = buf;
+	transfer->length = length;
+	transfer->user_data = user_data;
+	transfer->callback = callback;
+}
+
+void
+libusb_fill_interrupt_transfer(struct libusb_transfer *transfer,
+    libusb_device_handle *devh, uint8_t endpoint, uint8_t *buf,
+    int length, libusb_transfer_cb_fn callback, void *user_data,
+    uint32_t timeout)
+{
+	transfer->dev_handle = devh;
+	transfer->endpoint = endpoint;
+	transfer->type = LIBUSB_TRANSFER_TYPE_INTERRUPT;
+	transfer->timeout = timeout;
+	transfer->buffer = buf;
+	transfer->length = length;
+	transfer->user_data = user_data;
+	transfer->callback = callback;
+}
+
+void
+libusb_fill_iso_transfer(struct libusb_transfer *transfer, 
+    libusb_device_handle *devh, uint8_t endpoint, uint8_t *buf,
+    int length, int npacket, libusb_transfer_cb_fn callback,
+    void *user_data, uint32_t timeout)
+{
+	transfer->dev_handle = devh;
+	transfer->endpoint = endpoint;
+	transfer->type = LIBUSB_TRANSFER_TYPE_ISOCHRONOUS;
+	transfer->timeout = timeout;
+	transfer->buffer = buf;
+	transfer->length = length;
+	transfer->num_iso_packets = npacket;
+	transfer->user_data = user_data;
+	transfer->callback = callback;
+}
+
diff --git a/lib/libusb/libusb20.c b/lib/libusb/libusb20.c
index 6e0ec47..1a33805 100644
--- a/lib/libusb/libusb20.c
+++ b/lib/libusb/libusb20.c
@@ -630,6 +630,9 @@ libusb20_dev_req_string_sync(struct libusb20_device *pdev,
 	struct LIBUSB20_CONTROL_SETUP_DECODED req;
 	int error;
 
+	/* make sure memory is initialised */
+	memset(ptr, 0, len);
+
 	if (len < 4) {
 		/* invalid length */
 		return (LIBUSB20_ERROR_INVALID_PARAM);
@@ -1093,7 +1096,8 @@ libusb20_be_free(struct libusb20_backend *pbe)
 	if (pbe->methods->exit_backend) {
 		pbe->methods->exit_backend(pbe);
 	}
-	return;
+	/* free backend */
+	free(pbe);
 }
 
 void
@@ -1101,7 +1105,6 @@ libusb20_be_enqueue_device(struct libusb20_backend *pbe, struct libusb20_device
 {
 	pdev->beMethods = pbe->methods;	/* copy backend methods */
 	TAILQ_INSERT_TAIL(&(pbe->usb_devs), pdev, dev_entry);
-	return;
 }
 
 void
@@ -1109,5 +1112,4 @@ libusb20_be_dequeue_device(struct libusb20_backend *pbe,
     struct libusb20_device *pdev)
 {
 	TAILQ_REMOVE(&(pbe->usb_devs), pdev, dev_entry);
-	return;
 }
diff --git a/lib/libusb/libusb20_desc.c b/lib/libusb/libusb20_desc.c
index e0d2c54..5206cf4 100644
--- a/lib/libusb/libusb20_desc.c
+++ b/lib/libusb/libusb20_desc.c
@@ -118,6 +118,9 @@ libusb20_parse_config_desc(const void *config_desc)
 	if (lub_config == NULL) {
 		return (NULL);		/* out of memory */
 	}
+	/* make sure memory is initialised */
+	memset(lub_config, 0, size);
+
 	lub_interface = (void *)(lub_config + 1);
 	lub_alt_interface = (void *)(lub_interface + niface_no_alt);
 	lub_endpoint = (void *)(lub_interface + niface);
diff --git a/lib/libusb/libusb20_ugen20.c b/lib/libusb/libusb20_ugen20.c
index f9f3689..efcca63 100644
--- a/lib/libusb/libusb20_ugen20.c
+++ b/lib/libusb/libusb20_ugen20.c
@@ -449,6 +449,8 @@ ugen20_get_config_desc_full(struct libusb20_device *pdev,
 	uint16_t len;
 	int error;
 
+	/* make sure memory is initialised */
+	memset(&cdesc, 0, sizeof(cdesc));
 	memset(&gen_desc, 0, sizeof(gen_desc));
 
 	gen_desc.ugd_data = &cdesc;
@@ -468,6 +470,10 @@ ugen20_get_config_desc_full(struct libusb20_device *pdev,
 	if (!ptr) {
 		return (LIBUSB20_ERROR_NO_MEM);
 	}
+
+	/* make sure memory is initialised */
+	memset(ptr, 0, len);
+
 	gen_desc.ugd_data = ptr;
 	gen_desc.ugd_maxlen = len;
 
-- 
cgit v1.1


From bdd190d3246e1347a952c97b02eb18cd69e8e0a7 Mon Sep 17 00:00:00 2001
From: ume <ume@FreeBSD.org>
Date: Mon, 16 Nov 2009 03:52:18 +0000
Subject: MFC r199080, r199081, r199082: Add ja_JP.UTF-8 and ja_JP.eucJP
 catalogs.

---
 lib/libc/nls/Makefile.inc    |   2 +
 lib/libc/nls/ja_JP.UTF-8.msg | 249 +++++++++++++++++++++++++++++++++++++++++++
 lib/libc/nls/ja_JP.eucJP.msg | 249 +++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 500 insertions(+)
 create mode 100644 lib/libc/nls/ja_JP.UTF-8.msg
 create mode 100644 lib/libc/nls/ja_JP.eucJP.msg

(limited to 'lib')

diff --git a/lib/libc/nls/Makefile.inc b/lib/libc/nls/Makefile.inc
index 9e19e85..8e46470 100644
--- a/lib/libc/nls/Makefile.inc
+++ b/lib/libc/nls/Makefile.inc
@@ -22,6 +22,8 @@ NLS+=	fi_FI.ISO8859-1
 NLS+=	fr_FR.ISO8859-1
 NLS+=	hu_HU.ISO8859-2
 NLS+=	it_IT.ISO8859-15
+NLS+=	ja_JP.UTF-8
+NLS+=	ja_JP.eucJP
 NLS+=	ko_KR.UTF-8
 NLS+=	ko_KR.eucKR
 NLS+=	mn_MN.UTF-8
diff --git a/lib/libc/nls/ja_JP.UTF-8.msg b/lib/libc/nls/ja_JP.UTF-8.msg
new file mode 100644
index 0000000..9ee98d1
--- /dev/null
+++ b/lib/libc/nls/ja_JP.UTF-8.msg
@@ -0,0 +1,249 @@
+$ $FreeBSD$
+$
+$ Message catalog for ja_JP.UTF-8 locale
+$
+$ strerror() support catalog
+$
+$set 1
+$ EPERM
+1 許��れ�����作��
+$ ENOENT
+2 ��よ��ファイル���ディレクトリ��り��ん
+$ ESRCH
+3 ��よ��プロセス��り��ん
+$ EINTR
+4 システムコール�中断�れ���
+$ EIO
+5 入出力エラー��
+$ ENXIO
+6 デ�イス�準備�れ����ん
+$ E2BIG
+7 引数�リスト�長����
+$ ENOEXEC
+8 無効�実行形���
+$ EBADF
+9 無効�ファイル記述���
+$ ECHILD
+10 �プロセス��り��ん
+$ EDEADLK
+11 リソースデッドロックを回�����
+$ ENOMEM
+12 メモリ�割り当������ん
+$ EACCES
+13 パーミッション�拒絶�れ���
+$ EFAULT
+14 無効�アドレス��
+$ ENOTBLK
+15 ブロックデ�イス��求�れ����
+$ EBUSY
+16 デ�イス�ビジー状態��
+$ EEXIST
+17 ファイル�存在���
+$ EXDEV
+18 デ�イスを���リンク��
+$ ENODEV
+19 デ�イス�対応�����作��
+$ ENOTDIR
+20 ディレクトリ���り��ん
+$ EISDIR
+21 ディレクトリ��
+$ EINVAL
+22 無効�引数��
+$ ENFILE
+23 システム内�オープン�れ��るファイル�多����
+$ EMFILE
+24 オープン���るファイル�多����
+$ ENOTTY
+25 デ�イス�対応����� ioctl ��
+$ ETXTBSY
+26 テキストファイル�ビジー状態��
+$ EFBIG
+27 ファイル�大�����
+$ ENOSPC
+28 デ�イス�空�領域�足��
+$ ESPIPE
+29 無効�シーク��
+$ EROFS
+30 読�込�専用ファイルシステム��
+$ EMLINK
+31 リンク数�多����
+$ EPIPE
+32 パイプ�破壊�れ����
+$ EDOM
+33 数値引数�範囲外��
+$ ERANGE
+34 �果�大�����
+$ EAGAIN, EWOULDBLOCK
+35 リソース�一時的�利用����ん
+$ EINPROGRESS
+36 �作��在進行中��
+$ EALREADY
+37 �作�既�進行中��
+$ ENOTSOCK
+38 ソケット���も�����ソケット�作を行����
+$ EDESTADDRREQ
+39 宛先アドレス��求�れ����
+$ EMSGSIZE
+40 メッセージ�長����
+$ EPROTOTYPE
+41 ソケット�対応�����プロトコルタイプ��
+$ ENOPROTOOPT
+42 利用����プロトコル��
+$ EPROTONOSUPPORT
+43 対応�����プロトコル��
+$ ESOCKTNOSUPPORT
+44 対応�����ソケットタイプ��
+$ EOPNOTSUPP
+45 対応������作��
+$ EPFNOSUPPORT
+46 対応�����プロトコルファミリ��
+$ EAFNOSUPPORT
+47 プロトコルファミリ�対応�����アドレスファミリ�指定�れ���
+$ EADDRINUSE
+48 アドレス�既�使用中��
+$ EADDRNOTAVAIL
+49 �求�れ�アドレスを割り当�����ん
+$ ENETDOWN
+50 �ットワーク�ダウン�����
+$ ENETUNREACH
+51 �ットワーク�到�����ん
+$ ENETRESET
+52 リセット�より�ットワーク�接続�失�れ���
+$ ECONNABORTED
+53 ソフトウェア�よ��接続�切断�れ���
+$ ECONNRESET
+54 接続�通信相手�よ��リセット�れ���
+$ ENOBUFS
+55 �ッファ�容��足��
+$ EISCONN
+56 ソケット�既�接続�れ����
+$ ENOTCONN
+57 ソケット�接続�れ����ん
+$ ESHUTDOWN
+58 ソケット�シャットダウン�後��信�����ん
+$ ETOOMANYREFS
+59 処��界を超�る多��照��
+$ ETIMEDOUT
+60 �作�タイムアウト����
+$ ECONNREFUSED
+61 接続�拒絶�れ���
+$ ELOOP
+62 処��界を超�るシンボリックリンクレベル��
+$ ENAMETOOLONG
+63 ファイル��長����
+$ EHOSTDOWN
+64 ホスト�ダウン�����
+$ EHOSTUNREACH
+65 ホスト��経路��り��ん
+$ ENOTEMPTY
+66 ディレクトリ�空���り��ん
+$ EPROCLIM
+67 プロセス�多����
+$ EUSERS
+68 ユーザ�多����
+$ EDQUOT
+69 ディスククォータ�超�����
+$ ESTALE
+70 失効�� NFS ファイル�ンドル��
+$ EREMOTE
+71 パス中�リモート�レベル�多����
+$ EBADRPC
+72 無効� RPC 構造体��
+$ ERPCMISMATCH
+73 RPC �ージョン�間������
+$ EPROGUNAVAIL
+74 RPC プログラム�利用����ん
+$ EPROGMISMATCH
+75 プログラム��ージョン�������ん
+$ EPROCUNAVAIL
+76 プログラム��利用���� procedure ��
+$ ENOLCK
+77 ロック�利用����ん
+$ ENOSYS
+78 関数�実装�れ����ん
+$ EFTYPE
+79 ファイル�型���形����切��
+$ EAUTH
+80 �証エラー��
+$ ENEEDAUTH
+81 �証物�必���
+$ EIDRM
+82 識別��削除�れ���
+$ ENOMSG
+83 �求�れ�型�メッセージ��り��ん
+$ EOVERFLOW
+84 データ型�格��る��大���る値��
+$ ECANCELED
+85 処��キャンセル�れ���
+$ EILSEQ
+86 �正��イト列��
+$ ENOATTR
+87 ��よ��属性��り��ん
+$ EDOOFUS
+88 プログラミングエラー��
+$
+$ strsignal() support catalog
+$
+$set 2
+$ SIGHUP
+1 �ングアップ
+$ SIGINT
+2 割り込�
+$ SIGQUIT
+3 中断
+$ SIGILL
+4 �正命令
+$ SIGTRAP
+5 トレース/BPT トラップ
+$ SIGABRT
+6 アボートトラップ
+$ SIGEMT
+7 EMT トラップ
+$ SIGFPE
+8 浮動�数点例外
+$ SIGKILL
+9 Kill �れ�
+$ SIGBUS
+10 �スエラー
+$ SIGSEGV
+11 セグメンテーション��
+$ SIGSYS
+12 存在���システムコール
+$ SIGPIPE
+13 パイプ破壊
+$ SIGALRM
+14 アラームクロック
+$ SIGTERM
+15 終了
+$ SIGURG
+16 緊急入出力状�
+$ SIGSTOP
+17 一時�止 (シグナル)
+$ SIGTSTP
+18 一時�止
+$ SIGCONT
+19 継続
+$ SIGCHLD
+20 �プロセス�終了
+$ SIGTTIN
+21 一時�止 (tty 入力)
+$ SIGTTOU
+22 一時�止 (tty 出力)
+$ SIGIO
+23 入出力�能
+$ SIGXCPU
+24 CPU 時間�制�超�
+$ SIGXFSZ
+25 ファイルサイズ�制�超�
+$ SIGVTALRM
+26 仮想タイマ�期�超�
+$ SIGPROF
+27 プロファイルタイマ�期�超�
+$ SIGWINCH
+28 ウィンドウサイズ�変化
+$ SIGINFO
+29 情報�求
+$ SIGUSR1
+30 ユーザ定義シグナル 1
+$ SIGUSR2
+31 ユーザ定義シグナル 2
diff --git a/lib/libc/nls/ja_JP.eucJP.msg b/lib/libc/nls/ja_JP.eucJP.msg
new file mode 100644
index 0000000..cba0157
--- /dev/null
+++ b/lib/libc/nls/ja_JP.eucJP.msg
@@ -0,0 +1,249 @@
+$ $FreeBSD$
+$
+$ Message catalog for ja_JP.eucJP locale
+$
+$ strerror() support catalog
+$
+$set 1
+$ EPERM
+1 µö²Ä¤µ¤ì¤Æ¤¤¤Ê¤¤Áàºî¤Ç¤¹
+$ ENOENT
+2 ¤½¤Î¤è¤¦¤Ê¥Õ¥¡¥¤¥ë¤Þ¤¿¤Ï¥Ç¥£¥ì¥¯¥È¥ê¤Ï¤¢¤ê¤Þ¤»¤ó
+$ ESRCH
+3 ¤½¤Î¤è¤¦¤Ê¥×¥í¥»¥¹¤Ï¤¢¤ê¤Þ¤»¤ó
+$ EINTR
+4 ¥·¥¹¥Æ¥à¥³¡¼¥ë¤¬ÃæÃǤµ¤ì¤Þ¤·¤¿
+$ EIO
+5 Æþ½ÐÎÏ¥¨¥é¡¼¤Ç¤¹
+$ ENXIO
+6 ¥Ç¥Ð¥¤¥¹¤¬½àÈ÷¤µ¤ì¤Æ¤¤¤Þ¤»¤ó
+$ E2BIG
+7 °ú¿ô¤Î¥ê¥¹¥È¤¬Ä¹¤¹¤®¤Þ¤¹
+$ ENOEXEC
+8 ̵¸ú¤Ê¼Â¹Ô·Á¼°¤Ç¤¹
+$ EBADF
+9 ̵¸ú¤Ê¥Õ¥¡¥¤¥ëµ­½Ò»Ò¤Ç¤¹
+$ ECHILD
+10 »Ò¥×¥í¥»¥¹¤¬¤¢¤ê¤Þ¤»¤ó
+$ EDEADLK
+11 ¥ê¥½¡¼¥¹¥Ç¥Ã¥É¥í¥Ã¥¯¤ò²óÈò¤·¤Þ¤·¤¿
+$ ENOMEM
+12 ¥á¥â¥ê¤Î³ä¤êÅö¤Æ¤¬¤Ç¤­¤Þ¤»¤ó
+$ EACCES
+13 ¥Ñ¡¼¥ß¥Ã¥·¥ç¥ó¤¬µñÀ䤵¤ì¤Þ¤·¤¿
+$ EFAULT
+14 ̵¸ú¤Ê¥¢¥É¥ì¥¹¤Ç¤¹
+$ ENOTBLK
+15 ¥Ö¥í¥Ã¥¯¥Ç¥Ð¥¤¥¹¤¬Í׵ᤵ¤ì¤Æ¤¤¤Þ¤¹
+$ EBUSY
+16 ¥Ç¥Ð¥¤¥¹¤¬¥Ó¥¸¡¼¾õÂ֤Ǥ¹
+$ EEXIST
+17 ¥Õ¥¡¥¤¥ë¤¬Â¸ºß¤·¤Þ¤¹
+$ EXDEV
+18 ¥Ç¥Ð¥¤¥¹¤ò¤Þ¤¿¤°¥ê¥ó¥¯¤Ç¤¹
+$ ENODEV
+19 ¥Ç¥Ð¥¤¥¹¤¬Âбþ¤·¤Æ¤Ê¤¤Áàºî¤Ç¤¹
+$ ENOTDIR
+20 ¥Ç¥£¥ì¥¯¥È¥ê¤Ç¤Ï¤¢¤ê¤Þ¤»¤ó
+$ EISDIR
+21 ¥Ç¥£¥ì¥¯¥È¥ê¤Ç¤¹
+$ EINVAL
+22 ̵¸ú¤Ê°ú¿ô¤Ç¤¹
+$ ENFILE
+23 ¥·¥¹¥Æ¥àÆâ¤Ç¥ª¡¼¥×¥ó¤µ¤ì¤Æ¤¤¤ë¥Õ¥¡¥¤¥ë¤¬Â¿¤¹¤®¤Þ¤¹
+$ EMFILE
+24 ¥ª¡¼¥×¥ó¤·¤Æ¤¤¤ë¥Õ¥¡¥¤¥ë¤¬Â¿¤¹¤®¤Þ¤¹
+$ ENOTTY
+25 ¥Ç¥Ð¥¤¥¹¤¬Âбþ¤·¤Æ¤¤¤Ê¤¤ ioctl ¤Ç¤¹
+$ ETXTBSY
+26 ¥Æ¥­¥¹¥È¥Õ¥¡¥¤¥ë¤¬¥Ó¥¸¡¼¾õÂ֤Ǥ¹
+$ EFBIG
+27 ¥Õ¥¡¥¤¥ë¤¬Â礭¤¹¤®¤Þ¤¹
+$ ENOSPC
+28 ¥Ç¥Ð¥¤¥¹¤Î¶õ¤­ÎΰèÉÔ­¤Ç¤¹
+$ ESPIPE
+29 ̵¸ú¤Ê¥·¡¼¥¯¤Ç¤¹
+$ EROFS
+30 Æɤ߹þ¤ßÀìÍÑ¥Õ¥¡¥¤¥ë¥·¥¹¥Æ¥à¤Ç¤¹
+$ EMLINK
+31 ¥ê¥ó¥¯¿ô¤¬Â¿¤¹¤®¤Þ¤¹
+$ EPIPE
+32 ¥Ñ¥¤¥×¤¬Ç˲õ¤µ¤ì¤Æ¤Þ¤·¤¿
+$ EDOM
+33 ¿ôÃÍ°ú¿ô¤¬Èϰϳ°¤Ç¤¹
+$ ERANGE
+34 ·ë²Ì¤¬Â礭²á¤®¤Þ¤¹
+$ EAGAIN, EWOULDBLOCK
+35 ¥ê¥½¡¼¥¹¤¬°ì»þŪ¤ËÍøÍѤǤ­¤Þ¤»¤ó
+$ EINPROGRESS
+36 Áàºî¤¬¸½ºß¿Ê¹ÔÃæ¤Ç¤¹
+$ EALREADY
+37 Áàºî¤Ï´û¤Ë¿Ê¹ÔÃæ¤Ç¤¹
+$ ENOTSOCK
+38 ¥½¥±¥Ã¥È¤Ç¤Ê¤¤¤â¤Î¤Ë¤Ä¤¤¤Æ¥½¥±¥Ã¥ÈÁàºî¤ò¹Ô¤¤¤Þ¤·¤¿
+$ EDESTADDRREQ
+39 °¸À襢¥É¥ì¥¹¤¬Í׵ᤵ¤ì¤Æ¤¤¤Þ¤¹
+$ EMSGSIZE
+40 ¥á¥Ã¥»¡¼¥¸¤¬Ä¹¤¹¤®¤Þ¤¹
+$ EPROTOTYPE
+41 ¥½¥±¥Ã¥È¤¬Âбþ¤·¤Æ¤¤¤Ê¤¤¥×¥í¥È¥³¥ë¥¿¥¤¥×¤Ç¤¹
+$ ENOPROTOOPT
+42 ÍøÍѤǤ­¤Ê¤¤¥×¥í¥È¥³¥ë¤Ç¤¹
+$ EPROTONOSUPPORT
+43 Âбþ¤·¤Æ¤¤¤Ê¤¤¥×¥í¥È¥³¥ë¤Ç¤¹
+$ ESOCKTNOSUPPORT
+44 Âбþ¤·¤Æ¤¤¤Ê¤¤¥½¥±¥Ã¥È¥¿¥¤¥×¤Ç¤¹
+$ EOPNOTSUPP
+45 Âбþ¤·¤Æ¤¤¤Ê¤¤Áàºî¤Ç¤¹
+$ EPFNOSUPPORT
+46 Âбþ¤·¤Æ¤¤¤Ê¤¤¥×¥í¥È¥³¥ë¥Õ¥¡¥ß¥ê¤Ç¤¹
+$ EAFNOSUPPORT
+47 ¥×¥í¥È¥³¥ë¥Õ¥¡¥ß¥ê¤¬Âбþ¤·¤Æ¤¤¤Ê¤¤¥¢¥É¥ì¥¹¥Õ¥¡¥ß¥ê¤¬»ØÄꤵ¤ì¤Þ¤·¤¿
+$ EADDRINUSE
+48 ¥¢¥É¥ì¥¹¤¬´û¤Ë»ÈÍÑÃæ¤Ç¤¹
+$ EADDRNOTAVAIL
+49 Í׵ᤵ¤ì¤¿¥¢¥É¥ì¥¹¤ò³ä¤êÅö¤Æ¤Ç¤­¤Þ¤»¤ó
+$ ENETDOWN
+50 ¥Í¥Ã¥È¥ï¡¼¥¯¤¬¥À¥¦¥ó¤·¤Æ¤¤¤Þ¤¹
+$ ENETUNREACH
+51 ¥Í¥Ã¥È¥ï¡¼¥¯¤ËÅþã¤Ç¤­¤Þ¤»¤ó
+$ ENETRESET
+52 ¥ê¥»¥Ã¥È¤Ë¤è¤ê¥Í¥Ã¥È¥ï¡¼¥¯¤ÎÀܳ¤¬¼º¤ï¤ì¤Þ¤·¤¿
+$ ECONNABORTED
+53 ¥½¥Õ¥È¥¦¥§¥¢¤Ë¤è¤Ã¤ÆÀܳ¤¬ÀÚÃǤµ¤ì¤Þ¤·¤¿
+$ ECONNRESET
+54 Àܳ¤¬ÄÌ¿®Áê¼ê¤Ë¤è¤Ã¤Æ¥ê¥»¥Ã¥È¤µ¤ì¤Þ¤·¤¿
+$ ENOBUFS
+55 ¥Ð¥Ã¥Õ¥¡¤ÎÍÆÎÌÉÔ­¤Ç¤¹
+$ EISCONN
+56 ¥½¥±¥Ã¥È¤Ï´û¤ËÀܳ¤µ¤ì¤Æ¤¤¤Þ¤¹
+$ ENOTCONN
+57 ¥½¥±¥Ã¥È¤ÏÀܳ¤µ¤ì¤Æ¤¤¤Þ¤»¤ó
+$ ESHUTDOWN
+58 ¥½¥±¥Ã¥È¤Î¥·¥ã¥Ã¥È¥À¥¦¥ó¤Î¸å¤ÇÁ÷¿®¤¬¤Ç¤­¤Þ¤»¤ó
+$ ETOOMANYREFS
+59 ½èÍý¸Â³¦¤òĶ¤¨¤ë¿½Å»²¾È¤Ç¤¹
+$ ETIMEDOUT
+60 Áàºî¤¬¥¿¥¤¥à¥¢¥¦¥È¤·¤Þ¤·¤¿
+$ ECONNREFUSED
+61 Àܳ¤¬µñÀ䤵¤ì¤Þ¤·¤¿
+$ ELOOP
+62 ½èÍý¸Â³¦¤òĶ¤¨¤ë¥·¥ó¥Ü¥ê¥Ã¥¯¥ê¥ó¥¯¥ì¥Ù¥ë¤Ç¤¹
+$ ENAMETOOLONG
+63 ¥Õ¥¡¥¤¥ë̾¤¬Ä¹¤¹¤®¤Þ¤¹
+$ EHOSTDOWN
+64 ¥Û¥¹¥È¤¬¥À¥¦¥ó¤·¤Æ¤¤¤Þ¤¹
+$ EHOSTUNREACH
+65 ¥Û¥¹¥È¤Ø¤Î·ÐÏ©¤¬¤¢¤ê¤Þ¤»¤ó
+$ ENOTEMPTY
+66 ¥Ç¥£¥ì¥¯¥È¥ê¤¬¶õ¤Ç¤Ï¤¢¤ê¤Þ¤»¤ó
+$ EPROCLIM
+67 ¥×¥í¥»¥¹¤¬Â¿¤¹¤®¤Þ¤¹
+$ EUSERS
+68 ¥æ¡¼¥¶¤¬Â¿¤¹¤®¤Þ¤¹
+$ EDQUOT
+69 ¥Ç¥£¥¹¥¯¥¯¥©¡¼¥¿¤¬Ä¶²á¤·¤Þ¤·¤¿
+$ ESTALE
+70 ¼º¸ú¤·¤¿ NFS ¥Õ¥¡¥¤¥ë¥Ï¥ó¥É¥ë¤Ç¤¹
+$ EREMOTE
+71 ¥Ñ¥¹Ãæ¤Î¥ê¥â¡¼¥È¤Î¥ì¥Ù¥ë¤¬Â¿¤¹¤®¤Þ¤¹
+$ EBADRPC
+72 ̵¸ú¤Ê RPC ¹½Â¤ÂΤǤ¹
+$ ERPCMISMATCH
+73 RPC ¥Ð¡¼¥¸¥ç¥ó¤¬´Ö°ã¤Ã¤Æ¤¤¤Þ¤¹
+$ EPROGUNAVAIL
+74 RPC ¥×¥í¥°¥é¥à¤¬ÍøÍѤǤ­¤Þ¤»¤ó
+$ EPROGMISMATCH
+75 ¥×¥í¥°¥é¥à¤Î¥Ð¡¼¥¸¥ç¥ó¤¬¹ç¤Ã¤Æ¤¤¤Þ¤»¤ó
+$ EPROCUNAVAIL
+76 ¥×¥í¥°¥é¥à¤Ç¤ÏÍøÍѤǤ­¤Ê¤¤ procedure ¤Ç¤¹
+$ ENOLCK
+77 ¥í¥Ã¥¯¤¬ÍøÍѤǤ­¤Þ¤»¤ó
+$ ENOSYS
+78 ´Ø¿ô¤¬¼ÂÁõ¤µ¤ì¤Æ¤¤¤Þ¤»¤ó
+$ EFTYPE
+79 ¥Õ¥¡¥¤¥ë¤Î·¿¤Þ¤¿¤Ï·Á¼°¤¬ÉÔŬÀڤǤ¹
+$ EAUTH
+80 ǧ¾Ú¥¨¥é¡¼¤Ç¤¹
+$ ENEEDAUTH
+81 ǧ¾Úʪ¤¬É¬ÍפǤ¹
+$ EIDRM
+82 ¼±Ê̻ҤϺï½ü¤µ¤ì¤Þ¤·¤¿
+$ ENOMSG
+83 Í׵ᤵ¤ì¤¿·¿¤Î¥á¥Ã¥»¡¼¥¸¤¬¤¢¤ê¤Þ¤»¤ó
+$ EOVERFLOW
+84 ¥Ç¡¼¥¿·¿¤Ë³ÊǼ¤¹¤ë¤Ë¤ÏÂ礭¤¹¤®¤ëÃͤǤ¹
+$ ECANCELED
+85 ½èÍý¤¬¥­¥ã¥ó¥»¥ë¤µ¤ì¤Þ¤·¤¿
+$ EILSEQ
+86 ÉÔÀµ¤Ê¥Ð¥¤¥ÈÎó¤Ç¤¹
+$ ENOATTR
+87 ¤½¤Î¤è¤¦¤Ê°À­¤Ï¤¢¤ê¤Þ¤»¤ó
+$ EDOOFUS
+88 ¥×¥í¥°¥é¥ß¥ó¥°¥¨¥é¡¼¤Ç¤¹
+$
+$ strsignal() support catalog
+$
+$set 2
+$ SIGHUP
+1 ¥Ï¥ó¥°¥¢¥Ã¥×
+$ SIGINT
+2 ³ä¤ê¹þ¤ß
+$ SIGQUIT
+3 ̾̂
+$ SIGILL
+4 ÉÔÀµÌ¿Îá
+$ SIGTRAP
+5 ¥È¥ì¡¼¥¹/BPT ¥È¥é¥Ã¥×
+$ SIGABRT
+6 ¥¢¥Ü¡¼¥È¥È¥é¥Ã¥×
+$ SIGEMT
+7 EMT ¥È¥é¥Ã¥×
+$ SIGFPE
+8 ÉâÆ°¾®¿ôÅÀÎã³°
+$ SIGKILL
+9 Kill ¤µ¤ì¤¿
+$ SIGBUS
+10 ¥Ð¥¹¥¨¥é¡¼
+$ SIGSEGV
+11 ¥»¥°¥á¥ó¥Æ¡¼¥·¥ç¥ó°ãÈ¿
+$ SIGSYS
+12 ¸ºß¤·¤Ê¤¤¥·¥¹¥Æ¥à¥³¡¼¥ë
+$ SIGPIPE
+13 ¥Ñ¥¤¥×Ç˲õ
+$ SIGALRM
+14 ¥¢¥é¡¼¥à¥¯¥í¥Ã¥¯
+$ SIGTERM
+15 ½ªÎ»
+$ SIGURG
+16 ¶ÛµÞÆþ½ÐÎϾõ¶·
+$ SIGSTOP
+17 °ì»þÄä»ß (¥·¥°¥Ê¥ë)
+$ SIGTSTP
+18 °ì»þÄä»ß
+$ SIGCONT
+19 ·Ñ³
+$ SIGCHLD
+20 »Ò¥×¥í¥»¥¹¤Î½ªÎ»
+$ SIGTTIN
+21 °ì»þÄä»ß (tty ÆþÎÏ)
+$ SIGTTOU
+22 °ì»þÄä»ß (tty ½ÐÎÏ)
+$ SIGIO
+23 Æþ½ÐÎϲÄǽ
+$ SIGXCPU
+24 CPU »þ´Ö¤ÎÀ©¸ÂĶ²á
+$ SIGXFSZ
+25 ¥Õ¥¡¥¤¥ë¥µ¥¤¥º¤ÎÀ©¸ÂĶ²á
+$ SIGVTALRM
+26 ²¾ÁÛ¥¿¥¤¥Þ¤Î´ü¸ÂĶ²á
+$ SIGPROF
+27 ¥×¥í¥Õ¥¡¥¤¥ë¥¿¥¤¥Þ¤Î´ü¸ÂĶ²á
+$ SIGWINCH
+28 ¥¦¥£¥ó¥É¥¦¥µ¥¤¥º¤ÎÊѲ½
+$ SIGINFO
+29 ¾ðÊóÍ×µá
+$ SIGUSR1
+30 ¥æ¡¼¥¶ÄêµÁ¥·¥°¥Ê¥ë 1
+$ SIGUSR2
+31 ¥æ¡¼¥¶ÄêµÁ¥·¥°¥Ê¥ë 2
-- 
cgit v1.1


From a638ddfac313cd2604b87003d8112542018502e2 Mon Sep 17 00:00:00 2001
From: brueffer <brueffer@FreeBSD.org>
Date: Mon, 16 Nov 2009 08:26:56 +0000
Subject: MFC: r199046

Fix a copy+paste error by checking the correct variable against MM_NULLACT.
---
 lib/libc/gen/fmtmsg.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libc/gen/fmtmsg.c b/lib/libc/gen/fmtmsg.c
index 6caabbb..e6b1b96 100644
--- a/lib/libc/gen/fmtmsg.c
+++ b/lib/libc/gen/fmtmsg.c
@@ -128,7 +128,7 @@ printfmt(char *msgverb, long class, const char *label, int sev,
 		size += strlen(sevname);
 	if (text != MM_NULLTXT)
 		size += strlen(text);
-	if (text != MM_NULLACT)
+	if (act != MM_NULLACT)
 		size += strlen(act);
 	if (tag != MM_NULLTAG)
 		size += strlen(tag);
-- 
cgit v1.1


From e3c9717703321ed32dfe4b6db831f7ad0d719e81 Mon Sep 17 00:00:00 2001
From: jhb <jhb@FreeBSD.org>
Date: Tue, 17 Nov 2009 16:50:57 +0000
Subject: MFC 198986: Fix a copy-paste bug when reading data from the last 3 (7
 for PAE) bytes of a page mapped by a large page in the kernel.

---
 lib/libkvm/kvm_i386.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

(limited to 'lib')

diff --git a/lib/libkvm/kvm_i386.c b/lib/libkvm/kvm_i386.c
index 303b4af..ac86ba6 100644
--- a/lib/libkvm/kvm_i386.c
+++ b/lib/libkvm/kvm_i386.c
@@ -295,9 +295,9 @@ _kvm_vatop(kvm_t *kd, u_long va, off_t *pa)
 #define	PG_FRAME4M	(~PAGE4M_MASK)
 		pde_pa = ((u_long)pde & PG_FRAME4M) + (va & PAGE4M_MASK);
 		s = _kvm_pa2off(kd, pde_pa, &ofs);
-		if (s < sizeof pde) {
-			_kvm_syserr(kd, kd->program,
-			    "_kvm_vatop: pde_pa not found");
+		if (s == 0) {
+			_kvm_err(kd, kd->program,
+			    "_kvm_vatop: 4MB page address not in dump");
 			goto invalid;
 		}
 		*pa = ofs;
@@ -391,9 +391,9 @@ _kvm_vatop_pae(kvm_t *kd, u_long va, off_t *pa)
 #define	PG_FRAME2M	(~PAGE2M_MASK)
 		pde_pa = ((u_long)pde & PG_FRAME2M) + (va & PAGE2M_MASK);
 		s = _kvm_pa2off(kd, pde_pa, &ofs);
-		if (s < sizeof pde) {
-			_kvm_syserr(kd, kd->program,
-			    "_kvm_vatop_pae: pde_pa not found");
+		if (s == 0) {
+			_kvm_err(kd, kd->program,
+			    "_kvm_vatop: 2MB page address not in dump");
 			goto invalid;
 		}
 		*pa = ofs;
-- 
cgit v1.1


From 7fec43a93f7ad0b6a048296d2081ce2ec81a228e Mon Sep 17 00:00:00 2001
From: ume <ume@FreeBSD.org>
Date: Wed, 18 Nov 2009 14:40:00 +0000
Subject: MFC r199188: ANSIfy.

---
 lib/libc/net/ip6opt.c | 33 ++++++++-------------------------
 1 file changed, 8 insertions(+), 25 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/net/ip6opt.c b/lib/libc/net/ip6opt.c
index 7b65d06..d999fd4 100644
--- a/lib/libc/net/ip6opt.c
+++ b/lib/libc/net/ip6opt.c
@@ -55,8 +55,7 @@ static void inet6_insert_padopt(u_char *p, int len);
  * byte, the length byte, and the option data.
  */
 int
-inet6_option_space(nbytes)
-	int nbytes;
+inet6_option_space(int nbytes)
 {
 	nbytes += 2;	/* we need space for nxt-hdr and length fields */
 	return(CMSG_SPACE((nbytes + 7) & ~7));
@@ -68,10 +67,7 @@ inet6_option_space(nbytes)
  * success or -1 on an error.
  */
 int
-inet6_option_init(bp, cmsgp, type)
-	void *bp;
-	struct cmsghdr **cmsgp;
-	int type;
+inet6_option_init(void *bp, struct cmsghdr **cmsgp, int type)
 {
 	struct cmsghdr *ch = (struct cmsghdr *)bp;
 
@@ -98,11 +94,8 @@ inet6_option_init(bp, cmsgp, type)
  * earlier.  It must have a value between 0 and 7, inclusive.
  */
 int
-inet6_option_append(cmsg, typep, multx, plusy)
-	struct cmsghdr *cmsg;
-	const u_int8_t *typep;
-	int multx;
-	int plusy;
+inet6_option_append(struct cmsghdr *cmsg, const u_int8_t *typep, int multx,
+    int plusy)
 {
 	int padlen, optlen, off;
 	u_char *bp = (u_char *)cmsg + cmsg->cmsg_len;
@@ -171,11 +164,7 @@ inet6_option_append(cmsg, typep, multx, plusy)
  * 
  */
 u_int8_t *
-inet6_option_alloc(cmsg, datalen, multx, plusy)
-	struct cmsghdr *cmsg;
-	int datalen;
-	int multx;
-	int plusy;
+inet6_option_alloc(struct cmsghdr *cmsg, int datalen, int multx, int plusy)
 {
 	int padlen, off;
 	u_int8_t *bp = (u_char *)cmsg + cmsg->cmsg_len;
@@ -238,9 +227,7 @@ inet6_option_alloc(cmsg, datalen, multx, plusy)
  * (RFC 2292, 6.3.5)
  */
 int
-inet6_option_next(cmsg, tptrp)
-	const struct cmsghdr *cmsg;
-	u_int8_t **tptrp;
+inet6_option_next(const struct cmsghdr *cmsg, u_int8_t **tptrp)
 {
 	struct ip6_ext *ip6e;
 	int hdrlen, optlen;
@@ -296,10 +283,7 @@ inet6_option_next(cmsg, tptrp)
  *       it's a typo. The variable should be type of u_int8_t **.
  */
 int
-inet6_option_find(cmsg, tptrp, type)
-	const struct cmsghdr *cmsg;
-	u_int8_t **tptrp;
-	int type;
+inet6_option_find(const struct cmsghdr *cmsg, u_int8_t **tptrp, int type)
 {
 	struct ip6_ext *ip6e;
 	int hdrlen, optlen;
@@ -352,8 +336,7 @@ inet6_option_find(cmsg, tptrp, type)
  * calculated length and the limitation of the buffer.
  */
 static int
-ip6optlen(opt, lim)
-	u_int8_t *opt, *lim;
+ip6optlen(u_int8_t *opt, u_int8_t *lim)
 {
 	int optlen;
 
-- 
cgit v1.1


From 1b5e05542f3a6672d37c7d6c078750aebfaaefbc Mon Sep 17 00:00:00 2001
From: brueffer <brueffer@FreeBSD.org>
Date: Fri, 20 Nov 2009 06:55:35 +0000
Subject: MFC: r199255, r199257

Improved the manpage description.  The committed wording
was provided by jhb.

Remove a note about vfork(4) going to be eliminated, it's here to stay.
---
 lib/libc/sys/vfork.2 | 12 ++----------
 1 file changed, 2 insertions(+), 10 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/sys/vfork.2 b/lib/libc/sys/vfork.2
index 928130b..3cd3368 100644
--- a/lib/libc/sys/vfork.2
+++ b/lib/libc/sys/vfork.2
@@ -28,12 +28,12 @@
 .\"     @(#)vfork.2	8.1 (Berkeley) 6/4/93
 .\" $FreeBSD$
 .\"
-.Dd June 4, 1993
+.Dd November 13, 2009
 .Dt VFORK 2
 .Os
 .Sh NAME
 .Nm vfork
-.Nd spawn new process in a virtual memory efficient way
+.Nd create a new process without copying the address space
 .Sh LIBRARY
 .Lb libc
 .Sh SYNOPSIS
@@ -113,14 +113,6 @@ The
 system call appeared in
 .Bx 2.9 .
 .Sh BUGS
-This system call will be eliminated when proper system sharing
-mechanisms are implemented.
-Users should not depend on the memory
-sharing semantics of
-.Fn vfork
-as it will, in that case, be made synonymous to
-.Xr fork 2 .
-.Pp
 To avoid a possible deadlock situation,
 processes that are children in the middle
 of a
-- 
cgit v1.1


From 7f43a11af011e3802f49582d1176cbb79e9b4e00 Mon Sep 17 00:00:00 2001
From: ume <ume@FreeBSD.org>
Date: Sun, 22 Nov 2009 17:05:46 +0000
Subject: MFC r199083: Add NLS catalogs support to gai_strerror(3). Controlled
 by NLS define.

---
 lib/libc/net/gai_strerror.c | 58 +++++++++++++++++++++++++++++++++++++++++++++
 lib/libc/nls/C.msg          | 36 ++++++++++++++++++++++++++++
 2 files changed, 94 insertions(+)

(limited to 'lib')

diff --git a/lib/libc/net/gai_strerror.c b/lib/libc/net/gai_strerror.c
index 5611559..4f60f2d 100644
--- a/lib/libc/net/gai_strerror.c
+++ b/lib/libc/net/gai_strerror.c
@@ -30,7 +30,17 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include "namespace.h"
 #include <netdb.h>
+#if defined(NLS)
+#include <nl_types.h>
+#include <errno.h>
+#include <limits.h>
+#include <stdlib.h>
+#include <string.h>
+#include "reentrant.h"
+#endif
+#include "un-namespace.h"
 
 /* Entries EAI_ADDRFAMILY (1) and EAI_NODATA (7) are obsoleted, but left */
 /* for backward compatibility with userland code prior to 2553bis-02 */
@@ -52,9 +62,57 @@ static const char *ai_errlist[] = {
 	"Argument buffer overflow"			/* EAI_OVERFLOW */
 };
 
+#if defined(NLS)
+static char		gai_buf[NL_TEXTMAX];
+static once_t		gai_init_once = ONCE_INITIALIZER;
+static thread_key_t	gai_key;
+static int		gai_keycreated = 0;
+
+static void
+gai_keycreate(void)
+{
+	gai_keycreated = (thr_keycreate(&gai_key, free) == 0);
+}
+#endif
+
 const char *
 gai_strerror(int ecode)
 {
+#if defined(NLS)
+	nl_catd catd;
+	char *buf;
+
+	if (thr_main() != 0)
+		buf = gai_buf;
+	else {
+		if (thr_once(&gai_init_once, gai_keycreate) != 0 ||
+		    !gai_keycreated)
+			goto thr_err;
+		if ((buf = thr_getspecific(gai_key)) == NULL) {
+			if ((buf = malloc(sizeof(gai_buf))) == NULL)
+				goto thr_err;
+			if (thr_setspecific(gai_key, buf) != 0) {
+				free(buf);
+				goto thr_err;
+			}
+		}
+	}
+
+	catd = catopen("libc", NL_CAT_LOCALE);
+	if (ecode > 0 && ecode < EAI_MAX)
+		strlcpy(buf, catgets(catd, 3, ecode, ai_errlist[ecode]),
+		    sizeof(gai_buf));
+	else if (ecode == 0)
+		strlcpy(buf, catgets(catd, 3, NL_MSGMAX - 1, "Success"),
+		    sizeof(gai_buf));
+	else
+		strlcpy(buf, catgets(catd, 3, NL_MSGMAX, "Unknown error"),
+		    sizeof(gai_buf));
+	catclose(catd);
+	return buf;
+
+thr_err:
+#endif
 	if (ecode >= 0 && ecode < EAI_MAX)
 		return ai_errlist[ecode];
 	return "Unknown error";
diff --git a/lib/libc/nls/C.msg b/lib/libc/nls/C.msg
index aa2c4cc..a8286f1 100644
--- a/lib/libc/nls/C.msg
+++ b/lib/libc/nls/C.msg
@@ -247,3 +247,39 @@ $ SIGUSR1
 30 User defined signal 1
 $ SIGUSR2
 31 User defined signal 2
+$
+$ gai_strerror() support catalog
+$
+$set 3
+$ 1 (obsolete)
+1 Address family for hostname not supported
+$ EAI_AGAIN
+2 Temporary failure in name resolution
+$ EAI_BADFLAGS
+3 Invalid value for ai_flags
+$ EAI_FAIL
+4 Non-recoverable failure in name resolution
+$ EAI_FAMILY
+5 ai_family not supported
+$ EAI_MEMORY
+6 Memory allocation failure
+$ 7 (obsolete)
+7 No address associated with hostname
+$ EAI_NONAME
+8 hostname nor servname provided, or not known
+$ EAI_SERVICE
+9 servname not supported for ai_socktype
+$ EAI_SOCKTYPE
+10 ai_socktype not supported
+$ EAI_SYSTEM
+11 System error returned in errno
+$ EAI_BADHINTS
+12 Invalid value for hints
+$ EAI_PROTOCOL
+13 Resolved protocol is unknown
+$ EAI_OVERFLOW
+14 Argument buffer overflow
+$ 0
+32766 Success
+$ NL_MSGMAX
+32767 Unknown error
-- 
cgit v1.1


From 449af2da885b66a5779083760a82df7a3fd17c60 Mon Sep 17 00:00:00 2001
From: ume <ume@FreeBSD.org>
Date: Sun, 22 Nov 2009 17:08:52 +0000
Subject: MFC r199092: Add gai_strerror() catalog for ja_JP.UTF-8 and
 ja_JP.eucJP.

---
 lib/libc/nls/ja_JP.UTF-8.msg | 36 ++++++++++++++++++++++++++++++++++++
 lib/libc/nls/ja_JP.eucJP.msg | 36 ++++++++++++++++++++++++++++++++++++
 2 files changed, 72 insertions(+)

(limited to 'lib')

diff --git a/lib/libc/nls/ja_JP.UTF-8.msg b/lib/libc/nls/ja_JP.UTF-8.msg
index 9ee98d1..b4d2144 100644
--- a/lib/libc/nls/ja_JP.UTF-8.msg
+++ b/lib/libc/nls/ja_JP.UTF-8.msg
@@ -247,3 +247,39 @@ $ SIGUSR1
 30 ユーザ定義シグナル 1
 $ SIGUSR2
 31 ユーザ定義シグナル 2
+$
+$ gai_strerror() support catalog
+$
+$set 3
+$ 1 (obsolete)
+1 ホスト��アドレスファミリー�サ�ート�れ��ん
+$ EAI_AGAIN
+2 ��解決��一時的�失敗
+$ EAI_BADFLAGS
+3 ai_flags �値�無効
+$ EAI_FAIL
+4 ��解決��回復�能�失敗
+$ EAI_FAMILY
+5 ai_family �サ�ート�れ��ん
+$ EAI_MEMORY
+6 メモリ割り当�失敗
+$ 7 (obsolete)
+7 ホスト��対応�るアドレス��り��ん
+$ EAI_NONAME
+8 ホスト��サービス��指定�れ�������明
+$ EAI_SERVICE
+9 サービス�� ai_socktype �対��サ�ート�れ��ん
+$ EAI_SOCKTYPE
+10 ai_socktype �サ�ート�れ��ん
+$ EAI_SYSTEM
+11 システムエラー�errno �照
+$ EAI_BADHINTS
+12 hints �値�無効
+$ EAI_PROTOCOL
+13 解決�れ�プロトコル��明��
+$ EAI_OVERFLOW
+14 引数�ッファオー�フロー
+$ 0
+32766 �功
+$ NL_MSGMAX
+32767 �明�エラー
diff --git a/lib/libc/nls/ja_JP.eucJP.msg b/lib/libc/nls/ja_JP.eucJP.msg
index cba0157..461a39b 100644
--- a/lib/libc/nls/ja_JP.eucJP.msg
+++ b/lib/libc/nls/ja_JP.eucJP.msg
@@ -247,3 +247,39 @@ $ SIGUSR1
 30 ¥æ¡¼¥¶ÄêµÁ¥·¥°¥Ê¥ë 1
 $ SIGUSR2
 31 ¥æ¡¼¥¶ÄêµÁ¥·¥°¥Ê¥ë 2
+$
+$ gai_strerror() support catalog
+$
+$set 3
+$ 1 (obsolete)
+1 ¥Û¥¹¥È̾¤Î¥¢¥É¥ì¥¹¥Õ¥¡¥ß¥ê¡¼¤Ï¥µ¥Ý¡¼¥È¤µ¤ì¤Þ¤»¤ó
+$ EAI_AGAIN
+2 ̾Á°²ò·è¤Ç¤Î°ì»þŪ¤Ê¼ºÇÔ
+$ EAI_BADFLAGS
+3 ai_flags ¤ÎÃͤ¬Ìµ¸ú
+$ EAI_FAIL
+4 ̾Á°²ò·è¤Ç¤Î²óÉüÉÔǽ¤Ê¼ºÇÔ
+$ EAI_FAMILY
+5 ai_family ¤Ï¥µ¥Ý¡¼¥È¤µ¤ì¤Þ¤»¤ó
+$ EAI_MEMORY
+6 ¥á¥â¥ê³ä¤êÅö¤Æ¼ºÇÔ
+$ 7 (obsolete)
+7 ¥Û¥¹¥È̾¤ËÂбþ¤¹¤ë¥¢¥É¥ì¥¹¤Ï¤¢¤ê¤Þ¤»¤ó
+$ EAI_NONAME
+8 ¥Û¥¹¥È̾¤«¥µ¡¼¥Ó¥¹Ì¾¤¬»ØÄꤵ¤ì¤Ê¤¤¡¢¤Þ¤¿¤ÏÉÔÌÀ
+$ EAI_SERVICE
+9 ¥µ¡¼¥Ó¥¹Ì¾¤Ï ai_socktype ¤ËÂФ·¤Æ¥µ¥Ý¡¼¥È¤µ¤ì¤Þ¤»¤ó
+$ EAI_SOCKTYPE
+10 ai_socktype ¤Ï¥µ¥Ý¡¼¥È¤µ¤ì¤Þ¤»¤ó
+$ EAI_SYSTEM
+11 ¥·¥¹¥Æ¥à¥¨¥é¡¼¡¢errno »²¾È
+$ EAI_BADHINTS
+12 hints ¤ÎÃͤ¬Ìµ¸ú
+$ EAI_PROTOCOL
+13 ²ò·è¤µ¤ì¤¿¥×¥í¥È¥³¥ë¤ÏÉÔÌÀ¤Ç¤¹
+$ EAI_OVERFLOW
+14 °ú¿ô¥Ð¥Ã¥Õ¥¡¥ª¡¼¥Ð¥Õ¥í¡¼
+$ 0
+32766 À®¸ù
+$ NL_MSGMAX
+32767 ÉÔÌÀ¤Ê¥¨¥é¡¼
-- 
cgit v1.1


From 27cff457a53b959e6782708f0e229d181d6e740e Mon Sep 17 00:00:00 2001
From: brueffer <brueffer@FreeBSD.org>
Date: Mon, 23 Nov 2009 09:10:08 +0000
Subject: MFC: r199317

Fix a memory leak in acl_from_text() in case the conversion succeeded.
---
 lib/libc/posix1e/acl_from_text.c | 1 +
 1 file changed, 1 insertion(+)

(limited to 'lib')

diff --git a/lib/libc/posix1e/acl_from_text.c b/lib/libc/posix1e/acl_from_text.c
index 98c5426..c600987 100644
--- a/lib/libc/posix1e/acl_from_text.c
+++ b/lib/libc/posix1e/acl_from_text.c
@@ -257,6 +257,7 @@ acl_from_text(const char *buf_p)
 	}
 #endif
 
+	free(mybuf_p);
 	return(acl);
 
 error_label:
-- 
cgit v1.1


From 5bb9e329e2ecafc5d1a5e581a7b4c5f57afa443e Mon Sep 17 00:00:00 2001
From: brueffer <brueffer@FreeBSD.org>
Date: Mon, 23 Nov 2009 09:44:08 +0000
Subject: MFC: r199320

Fix grammar.
---
 lib/libc/locale/nl_langinfo.3 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libc/locale/nl_langinfo.3 b/lib/libc/locale/nl_langinfo.3
index 8de1682..789cac2 100644
--- a/lib/libc/locale/nl_langinfo.3
+++ b/lib/libc/locale/nl_langinfo.3
@@ -53,7 +53,7 @@ with a category corresponding to the category of
 or to the
 category
 .Dv LC_ALL ,
-may overwrite buffer pointed by the return value.
+may overwrite the buffer pointed to by the return value.
 .Sh RETURN VALUES
 In a locale where langinfo data is not defined,
 .Fn nl_langinfo
-- 
cgit v1.1


From 9c731efbb9ec1f6227530c53d4a2682dc0621d22 Mon Sep 17 00:00:00 2001
From: marcel <marcel@FreeBSD.org>
Date: Tue, 24 Nov 2009 03:23:40 +0000
Subject: MFC r198450: Implement _umtx_op_err() for ia64.

---
 lib/libthr/arch/ia64/Makefile.inc         |  2 +-
 lib/libthr/arch/ia64/ia64/_umtx_op_err.S  | 35 +++++++++++++++++++++++++++++++
 lib/libthr/arch/ia64/include/pthread_md.h |  2 ++
 3 files changed, 38 insertions(+), 1 deletion(-)
 create mode 100644 lib/libthr/arch/ia64/ia64/_umtx_op_err.S

(limited to 'lib')

diff --git a/lib/libthr/arch/ia64/Makefile.inc b/lib/libthr/arch/ia64/Makefile.inc
index 1c0ce94..160cdf6 100644
--- a/lib/libthr/arch/ia64/Makefile.inc
+++ b/lib/libthr/arch/ia64/Makefile.inc
@@ -2,4 +2,4 @@
 
 .PATH: ${.CURDIR}/arch/${MACHINE_ARCH}/${MACHINE_ARCH}
 
-SRCS+= pthread_md.c
+SRCS+= _umtx_op_err.S pthread_md.c
diff --git a/lib/libthr/arch/ia64/ia64/_umtx_op_err.S b/lib/libthr/arch/ia64/ia64/_umtx_op_err.S
new file mode 100644
index 0000000..a712210
--- /dev/null
+++ b/lib/libthr/arch/ia64/ia64/_umtx_op_err.S
@@ -0,0 +1,35 @@
+/*-
+ * Copyright (c) 2009 Marcel Moolenaar
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <machine/asm.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/syscall.h>
+
+ENTRY(_umtx_op_err, 5)
+	CALLSYS_NOERROR(_umtx_op)
+	br.ret.sptk.few rp
+END(_umtx_op_err)
diff --git a/lib/libthr/arch/ia64/include/pthread_md.h b/lib/libthr/arch/ia64/include/pthread_md.h
index 0cff7c9..69b3eec 100644
--- a/lib/libthr/arch/ia64/include/pthread_md.h
+++ b/lib/libthr/arch/ia64/include/pthread_md.h
@@ -33,6 +33,8 @@
 
 #define	CPU_SPINWAIT
 
+#define	HAS__UMTX_OP_ERR	1
+
 #define	DTV_OFFSET		offsetof(struct tcb, tcb_dtv)
 
 /*
-- 
cgit v1.1


From 152a6d5bd58ed4ad0ba006ad52ee3a13702c4fb0 Mon Sep 17 00:00:00 2001
From: roam <roam@FreeBSD.org>
Date: Wed, 25 Nov 2009 10:52:07 +0000
Subject: Merge rev. 199180 to 8.x-STABLE, the change to isblank(3): Fix the
 grammar as in the PR, and then some.

PR:             140454
Submitted by:   Jeremy Huddleston <jeremyhu@apple.com>
---
 lib/libc/locale/isblank.3 | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/locale/isblank.3 b/lib/libc/locale/isblank.3
index 4cc6bbb..6734dcc 100644
--- a/lib/libc/locale/isblank.3
+++ b/lib/libc/locale/isblank.3
@@ -50,9 +50,9 @@ For any locale, this includes the following standard characters:
 .It "\&``\et''\t`` ''"
 .El
 .Pp
-In the "C" locale
+In the "C" locale, a successful
 .Fn isblank
-successful test is limited to this characters only.
+test is limited to these characters only.
 The value of the argument must be representable as an
 .Vt "unsigned char"
 or the value of
-- 
cgit v1.1


From 1f98e49ef9b9bbf5080150bb8c76d67a354231f5 Mon Sep 17 00:00:00 2001
From: fabient <fabient@FreeBSD.org>
Date: Tue, 1 Dec 2009 23:23:52 +0000
Subject: MFC 198339: Fix the NO_PROXY handling.

PR: 139751
---
 lib/libfetch/common.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libfetch/common.c b/lib/libfetch/common.c
index d49336a..c54804b 100644
--- a/lib/libfetch/common.c
+++ b/lib/libfetch/common.c
@@ -772,7 +772,7 @@ fetch_no_proxy_match(const char *host)
 				break;
 
 		d_len = q - p;
-		if (d_len > 0 && h_len > d_len &&
+		if (d_len > 0 && h_len >= d_len &&
 		    strncasecmp(host + h_len - d_len,
 			p, d_len) == 0) {
 			/* domain name matches */
-- 
cgit v1.1


From 42056712b21a97e733139da7d0667caa7cd02c54 Mon Sep 17 00:00:00 2001
From: wollman <wollman@FreeBSD.org>
Date: Wed, 2 Dec 2009 02:47:29 +0000
Subject: MFC revs 199781,199782,199784,199785,199786:

  Eliminate dead stores.

  In __mbsconv(), if prec was zero, nconv could have been used
  uninitialized.  Initialize it to a safe value so that there's no
  chance of returning an error if stack garbage happens to be equal to
  (size_t)-1 or (size_t)-2.

  In svc_raw_reply(), don't leave stat uninitialized if the MSG_ACCEPTED
  && SUCCESS case succeeds.  The stack garbage might be zero.

  In clnt_raw_create(), avoid minor race condition initializing the
  file-scope variable clntraw_private.

Found by:		Clang static analyzer
---
 lib/libc/gen/getcap.c       |  5 ++---
 lib/libc/gen/getusershell.c |  2 +-
 lib/libc/gen/wordexp.c      |  2 +-
 lib/libc/rpc/clnt_raw.c     | 11 +++++++----
 lib/libc/rpc/getnetconfig.c |  4 ++--
 lib/libc/rpc/key_call.c     |  2 +-
 lib/libc/rpc/svc_raw.c      |  5 ++---
 lib/libc/stdio/fgetws.c     |  2 +-
 lib/libc/stdio/fvwrite.c    |  2 +-
 lib/libc/stdio/vfwprintf.c  |  2 +-
 lib/libc/yp/yplib.c         |  2 +-
 11 files changed, 20 insertions(+), 19 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/gen/getcap.c b/lib/libc/gen/getcap.c
index 32d66d48..2700d3d 100644
--- a/lib/libc/gen/getcap.c
+++ b/lib/libc/gen/getcap.c
@@ -647,7 +647,7 @@ int
 cgetnext(char **bp, char **db_array)
 {
 	size_t len;
-	int done, hadreaderr, i, savederrno, status;
+	int done, hadreaderr, savederrno, status;
 	char *cp, *line, *rp, *np, buf[BSIZE], nbuf[BSIZE];
 	u_int dummy;
 
@@ -658,7 +658,7 @@ cgetnext(char **bp, char **db_array)
 		(void)cgetclose();
 		return (-1);
 	}
-	for(;;) {
+	for (;;) {
 		if (toprec && !gottoprec) {
 			gottoprec = 1;
 			line = toprec;
@@ -709,7 +709,6 @@ cgetnext(char **bp, char **db_array)
 		/*
 		 * Line points to a name line.
 		 */
-		i = 0;
 		done = 0;
 		np = nbuf;
 		for (;;) {
diff --git a/lib/libc/gen/getusershell.c b/lib/libc/gen/getusershell.c
index b0da2a1..d09b50c 100644
--- a/lib/libc/gen/getusershell.c
+++ b/lib/libc/gen/getusershell.c
@@ -124,7 +124,7 @@ _local_initshells(rv, cb_data, ap)
 	if ((fp = fopen(_PATH_SHELLS, "r")) == NULL)
 		return NS_UNAVAIL;
 
-	sp = cp = line;
+	cp = line;
 	while (fgets(cp, MAXPATHLEN + 1, fp) != NULL) {
 		while (*cp != '#' && *cp != '/' && *cp != '\0')
 			cp++;
diff --git a/lib/libc/gen/wordexp.c b/lib/libc/gen/wordexp.c
index a437543..11fb697 100644
--- a/lib/libc/gen/wordexp.c
+++ b/lib/libc/gen/wordexp.c
@@ -282,7 +282,7 @@ we_check(const char *words, int flags)
 				if (c == '\0' || level != 0)
 					return (WRDE_SYNTAX);
 			} else
-				c = *--words;
+				--words;
 			break;
 		default:
 			break;
diff --git a/lib/libc/rpc/clnt_raw.c b/lib/libc/rpc/clnt_raw.c
index 9d34a3d..cd3a384 100644
--- a/lib/libc/rpc/clnt_raw.c
+++ b/lib/libc/rpc/clnt_raw.c
@@ -92,13 +92,13 @@ clnt_raw_create(prog, vers)
 	rpcprog_t prog;
 	rpcvers_t vers;
 {
-	struct clntraw_private *clp = clntraw_private;
+	struct clntraw_private *clp;
 	struct rpc_msg call_msg;
-	XDR *xdrs = &clp->xdr_stream;
-	CLIENT	*client = &clp->client_object;
+	XDR *xdrs;
+	CLIENT	*client;
 
 	mutex_lock(&clntraw_lock);
-	if (clp == NULL) {
+	if ((clp = clntraw_private) == NULL) {
 		clp = (struct clntraw_private *)calloc(1, sizeof (*clp));
 		if (clp == NULL) {
 			mutex_unlock(&clntraw_lock);
@@ -110,6 +110,9 @@ clnt_raw_create(prog, vers)
 		clp->_raw_buf = __rpc_rawcombuf;
 		clntraw_private = clp;
 	}
+	xdrs = &clp->xdr_stream;
+	client = &clp->client_object;
+
 	/*
 	 * pre-serialize the static part of the call msg and stash it away
 	 */
diff --git a/lib/libc/rpc/getnetconfig.c b/lib/libc/rpc/getnetconfig.c
index 9baa224..e5db51a 100644
--- a/lib/libc/rpc/getnetconfig.c
+++ b/lib/libc/rpc/getnetconfig.c
@@ -412,13 +412,13 @@ void *handlep;
      * Noone needs these entries anymore, then frees them.
      * Make sure all info in netconfig_info structure has been reinitialized.
      */
-    q = p = ni.head;
+    q = ni.head;
     ni.eof = ni.ref = 0;
     ni.head = NULL;
     ni.tail = NULL;
     mutex_unlock(&ni_lock);
 
-    while (q) {
+    while (q != NULL) {
 	p = q->next;
 	if (q->ncp->nc_lookups != NULL) free(q->ncp->nc_lookups);
 	free(q->ncp);
diff --git a/lib/libc/rpc/key_call.c b/lib/libc/rpc/key_call.c
index 615f24d..6cc1139 100644
--- a/lib/libc/rpc/key_call.c
+++ b/lib/libc/rpc/key_call.c
@@ -302,7 +302,7 @@ int	vers;
 	void *localhandle;
 	struct netconfig *nconf;
 	struct netconfig *tpconf;
-	struct key_call_private *kcp = key_call_private_main;
+	struct key_call_private *kcp;
 	struct timeval wait_time;
 	struct utsname u;
 	int main_thread;
diff --git a/lib/libc/rpc/svc_raw.c b/lib/libc/rpc/svc_raw.c
index 7492046..67bcba1 100644
--- a/lib/libc/rpc/svc_raw.c
+++ b/lib/libc/rpc/svc_raw.c
@@ -176,9 +176,8 @@ svc_raw_reply(xprt, msg)
 		msg->acpted_rply.ar_results.proc = (xdrproc_t) xdr_void;
 		msg->acpted_rply.ar_results.where = NULL;
 
-		if (!xdr_replymsg(xdrs, msg) ||
-		    !SVCAUTH_WRAP(&SVC_AUTH(xprt), xdrs, xdr_proc, xdr_where))
-			stat = FALSE;
+		stat = xdr_replymsg(xdrs, msg) &&
+		    SVCAUTH_WRAP(&SVC_AUTH(xprt), xdrs, xdr_proc, xdr_where);
 	} else {
 		stat = xdr_replymsg(xdrs, msg);
 	}
diff --git a/lib/libc/stdio/fgetws.c b/lib/libc/stdio/fgetws.c
index bbc0299..550843d 100644
--- a/lib/libc/stdio/fgetws.c
+++ b/lib/libc/stdio/fgetws.c
@@ -89,7 +89,7 @@ fgetws(wchar_t * __restrict ws, int n, FILE * __restrict fp)
 	if (!__mbsinit(&fp->_mbstate))
 		/* Incomplete character */
 		goto error;
-	*wsp++ = L'\0';
+	*wsp = L'\0';
 	FUNLOCKFILE(fp);
 
 	return (ws);
diff --git a/lib/libc/stdio/fvwrite.c b/lib/libc/stdio/fvwrite.c
index fd69eb9..7206676 100644
--- a/lib/libc/stdio/fvwrite.c
+++ b/lib/libc/stdio/fvwrite.c
@@ -60,7 +60,7 @@ __sfvwrite(fp, uio)
 	char *nl;
 	int nlknown, nldist;
 
-	if ((len = uio->uio_resid) == 0)
+	if (uio->uio_resid == 0)
 		return (0);
 	/* make sure we can write */
 	if (prepwrite(fp) != 0)
diff --git a/lib/libc/stdio/vfwprintf.c b/lib/libc/stdio/vfwprintf.c
index f3768de..d34f559 100644
--- a/lib/libc/stdio/vfwprintf.c
+++ b/lib/libc/stdio/vfwprintf.c
@@ -293,7 +293,7 @@ __mbsconv(char *mbsarg, int prec)
 		 * number of characters to print.
 		 */
 		p = mbsarg;
-		insize = nchars = 0;
+		insize = nchars = nconv = 0;
 		mbs = initial_mbs;
 		while (nchars != (size_t)prec) {
 			nconv = mbrlen(p, MB_CUR_MAX, &mbs);
diff --git a/lib/libc/yp/yplib.c b/lib/libc/yp/yplib.c
index eb5c34c..87d16dd 100644
--- a/lib/libc/yp/yplib.c
+++ b/lib/libc/yp/yplib.c
@@ -241,7 +241,7 @@ static bool_t
 ypmatch_cache_lookup(struct dom_binding *ypdb, char *map, keydat *key,
     valdat *val)
 {
-	struct ypmatch_ent	*c = ypdb->cache;
+	struct ypmatch_ent	*c;
 
 	ypmatch_cache_expire(ypdb);
 
-- 
cgit v1.1


From c28b64e0d9f31cb4da220ab09958fd4adc90e69d Mon Sep 17 00:00:00 2001
From: brueffer <brueffer@FreeBSD.org>
Date: Fri, 4 Dec 2009 07:08:15 +0000
Subject: MFC: r199988

Add an .Nm for strncat.
---
 lib/libc/string/strcat.3 | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/string/strcat.3 b/lib/libc/string/strcat.3
index 0290994..4c9fec9 100644
--- a/lib/libc/string/strcat.3
+++ b/lib/libc/string/strcat.3
@@ -32,11 +32,12 @@
 .\"     @(#)strcat.3	8.1 (Berkeley) 6/4/93
 .\" $FreeBSD$
 .\"
-.Dd June 4, 1993
+.Dd December 1, 2009
 .Dt STRCAT 3
 .Os
 .Sh NAME
-.Nm strcat
+.Nm strcat ,
+.Nm strncat
 .Nd concatenate strings
 .Sh LIBRARY
 .Lb libc
-- 
cgit v1.1


From 38a74188d15dc3ef17034d3ed9f7b3fbe217852f Mon Sep 17 00:00:00 2001
From: jh <jh@FreeBSD.org>
Date: Fri, 4 Dec 2009 11:26:52 +0000
Subject: MFC r199843:

Clarify that the value of the fts_info field is different in post-order.

Approved by:	trasz (mentor)
---
 lib/libc/gen/fts.3 | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/gen/fts.3 b/lib/libc/gen/fts.3
index 8e1a1e4..d2473be 100644
--- a/lib/libc/gen/fts.3
+++ b/lib/libc/gen/fts.3
@@ -28,7 +28,7 @@
 .\"     @(#)fts.3	8.5 (Berkeley) 4/16/94
 .\" $FreeBSD$
 .\"
-.Dd January 26, 2008
+.Dd November 25, 2009
 .Dt FTS 3
 .Os
 .Sh NAME
@@ -198,10 +198,9 @@ A directory being visited in post-order.
 The contents of the
 .Vt FTSENT
 structure will be unchanged from when
-it was returned in pre-order, i.e., with the
+the directory was visited in pre-order, except for the
 .Fa fts_info
-field set to
-.Dv FTS_D .
+field.
 .It Dv FTS_ERR
 This is an error return, and the
 .Fa fts_errno
-- 
cgit v1.1


From 474d50676beda16b58d9b04cf350f9d2e3daf0bd Mon Sep 17 00:00:00 2001
From: jilles <jilles@FreeBSD.org>
Date: Sun, 6 Dec 2009 22:14:58 +0000
Subject: MFC r198406: wordexp(3): fix some bugs with signals and long outputs

* retry various system calls on EINTR
* retry the rest after a short read (common if there is more than about 1K
  of output)
* block SIGCHLD like system(3) does (note that this does not and cannot
  work fully in threaded programs, they will need to be careful with wait
  functions)

PR:		90580
---
 lib/libc/gen/wordexp.c | 76 +++++++++++++++++++++++++++++++++++++-------------
 1 file changed, 57 insertions(+), 19 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/gen/wordexp.c b/lib/libc/gen/wordexp.c
index 11fb697..bcab1f5 100644
--- a/lib/libc/gen/wordexp.c
+++ b/lib/libc/gen/wordexp.c
@@ -28,8 +28,10 @@
 #include <sys/cdefs.h>
 #include <sys/types.h>
 #include <sys/wait.h>
+#include <errno.h>
 #include <fcntl.h>
 #include <paths.h>
+#include <signal.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -73,6 +75,24 @@ wordexp(const char * __restrict words, wordexp_t * __restrict we, int flags)
 	return (0);
 }
 
+static size_t
+we_read_fully(int fd, char *buffer, size_t len)
+{
+	size_t done;
+	ssize_t nread;
+
+	done = 0;
+	do {
+		nread = _read(fd, buffer + done, len - done);
+		if (nread == -1 && errno == EINTR)
+			continue;
+		if (nread <= 0)
+			break;
+		done += nread;
+	} while (done != len);
+	return done;
+}
+
 /*
  * we_askshell --
  *	Use the `wordexp' /bin/sh builtin function to do most of the work
@@ -90,20 +110,31 @@ we_askshell(const char *words, wordexp_t *we, int flags)
 	size_t sofs;			/* Offset into we->we_strings */
 	size_t vofs;			/* Offset into we->we_wordv */
 	pid_t pid;			/* Process ID of child */
+	pid_t wpid;			/* waitpid return value */
 	int status;			/* Child exit status */
+	int error;			/* Our return value */
+	int serrno;			/* errno to return */
 	char *ifs;			/* IFS env. var. */
 	char *np, *p;			/* Handy pointers */
 	char *nstrings;			/* Temporary for realloc() */
 	char **nwv;			/* Temporary for realloc() */
+	sigset_t newsigblock, oldsigblock;
 
+	serrno = errno;
 	if ((ifs = getenv("IFS")) == NULL)
 		ifs = " \t\n";
 
 	if (pipe(pdes) < 0)
 		return (WRDE_NOSPACE);	/* XXX */
+	(void)sigemptyset(&newsigblock);
+	(void)sigaddset(&newsigblock, SIGCHLD);
+	(void)_sigprocmask(SIG_BLOCK, &newsigblock, &oldsigblock);
 	if ((pid = fork()) < 0) {
+		serrno = errno;
 		_close(pdes[0]);
 		_close(pdes[1]);
+		(void)_sigprocmask(SIG_SETMASK, &oldsigblock, NULL);
+		errno = serrno;
 		return (WRDE_NOSPACE);	/* XXX */
 	}
 	else if (pid == 0) {
@@ -114,6 +145,7 @@ we_askshell(const char *words, wordexp_t *we, int flags)
 		int devnull;
 		char *cmd;
 
+		(void)_sigprocmask(SIG_SETMASK, &oldsigblock, NULL);
 		_close(pdes[0]);
 		if (_dup2(pdes[1], STDOUT_FILENO) < 0)
 			_exit(1);
@@ -139,10 +171,11 @@ we_askshell(const char *words, wordexp_t *we, int flags)
 	 * the expanded words separated by nulls.
 	 */
 	_close(pdes[1]);
-	if (_read(pdes[0], wbuf, 8) != 8 || _read(pdes[0], bbuf, 8) != 8) {
-		_close(pdes[0]);
-		_waitpid(pid, &status, 0);
-		return (flags & WRDE_UNDEF ? WRDE_BADVAL : WRDE_SYNTAX);
+	if (we_read_fully(pdes[0], wbuf, 8) != 8 ||
+			we_read_fully(pdes[0], bbuf, 8) != 8) {
+		error = flags & WRDE_UNDEF ? WRDE_BADVAL : WRDE_SYNTAX;
+		serrno = errno;
+		goto cleanup;
 	}
 	wbuf[8] = bbuf[8] = '\0';
 	nwords = strtol(wbuf, NULL, 16);
@@ -162,33 +195,38 @@ we_askshell(const char *words, wordexp_t *we, int flags)
 	if ((nwv = realloc(we->we_wordv, (we->we_wordc + 1 +
 	    (flags & WRDE_DOOFFS ?  we->we_offs : 0)) *
 	    sizeof(char *))) == NULL) {
-		_close(pdes[0]);
-		_waitpid(pid, &status, 0);
-		return (WRDE_NOSPACE);
+		error = WRDE_NOSPACE;
+		goto cleanup;
 	}
 	we->we_wordv = nwv;
 	if ((nstrings = realloc(we->we_strings, we->we_nbytes)) == NULL) {
-		_close(pdes[0]);
-		_waitpid(pid, &status, 0);
-		return (WRDE_NOSPACE);
+		error = WRDE_NOSPACE;
+		goto cleanup;
 	}
 	for (i = 0; i < vofs; i++)
 		if (we->we_wordv[i] != NULL)
 			we->we_wordv[i] += nstrings - we->we_strings;
 	we->we_strings = nstrings;
 
-	if (_read(pdes[0], we->we_strings + sofs, nbytes) != nbytes) {
-		_close(pdes[0]);
-		_waitpid(pid, &status, 0);
-		return (flags & WRDE_UNDEF ? WRDE_BADVAL : WRDE_SYNTAX);
+	if (we_read_fully(pdes[0], we->we_strings + sofs, nbytes) != nbytes) {
+		error = flags & WRDE_UNDEF ? WRDE_BADVAL : WRDE_SYNTAX;
+		serrno = errno;
+		goto cleanup;
 	}
 
-	if (_waitpid(pid, &status, 0) < 0 || !WIFEXITED(status) ||
-	    WEXITSTATUS(status) != 0) {
-		_close(pdes[0]);
-		return (flags & WRDE_UNDEF ? WRDE_BADVAL : WRDE_SYNTAX);
-	}
+	error = 0;
+cleanup:
 	_close(pdes[0]);
+	do
+		wpid = _waitpid(pid, &status, 0);
+	while (wpid < 0 && errno == EINTR);
+	(void)_sigprocmask(SIG_SETMASK, &oldsigblock, NULL);
+	if (error != 0) {
+		errno = serrno;
+		return (error);
+	}
+	if (wpid < 0 || !WIFEXITED(status) || WEXITSTATUS(status) != 0)
+		return (flags & WRDE_UNDEF ? WRDE_BADVAL : WRDE_SYNTAX);
 
 	/*
 	 * Break the null-terminated expanded word strings out into
-- 
cgit v1.1


From 6b90f6c201275c1edfa6b986407d0f94475c1e68 Mon Sep 17 00:00:00 2001
From: brueffer <brueffer@FreeBSD.org>
Date: Mon, 7 Dec 2009 08:38:44 +0000
Subject: MFC: r200019

Fix the dprintf() prototype.
---
 lib/libc/stdio/printf.3 | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/stdio/printf.3 b/lib/libc/stdio/printf.3
index 27d5bf0..2587aa2 100644
--- a/lib/libc/stdio/printf.3
+++ b/lib/libc/stdio/printf.3
@@ -32,7 +32,7 @@
 .\"     @(#)printf.3	8.1 (Berkeley) 6/4/93
 .\" $FreeBSD$
 .\"
-.Dd March 3, 2009
+.Dd December 2, 2009
 .Dt PRINTF 3
 .Os
 .Sh NAME
@@ -55,7 +55,7 @@
 .Ft int
 .Fn asprintf "char **ret" "const char *format" ...
 .Ft int
-.Fn dprintf "int" "const char * restrict format" ...
+.Fn dprintf "int fd" "const char * restrict format" ...
 .In stdarg.h
 .Ft int
 .Fn vprintf "const char * restrict format" "va_list ap"
-- 
cgit v1.1


From f95fa4d4d01b6911513c52f88083a311657bda24 Mon Sep 17 00:00:00 2001
From: jhb <jhb@FreeBSD.org>
Date: Mon, 7 Dec 2009 19:07:45 +0000
Subject: MFC 200061: The fd_mask type is an unsigned long, not an int, so
 treat the mask as a long instead of an int when examining the results of
 select() to look for RPC requests.  Previously this routine would ignore RPC
 requests to sockets whose file descriptor mod 64 was greater than 31 on a
 64-bit platform.

---
 lib/libc/rpc/svc.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/rpc/svc.c b/lib/libc/rpc/svc.c
index d205121..282c2be 100644
--- a/lib/libc/rpc/svc.c
+++ b/lib/libc/rpc/svc.c
@@ -627,8 +627,8 @@ svc_getreqset(readfds)
 
 	maskp = readfds->fds_bits;
 	for (sock = 0; sock < FD_SETSIZE; sock += NFDBITS) {
-	    for (mask = *maskp++; (bit = ffs(mask)) != 0;
-		mask ^= (1 << (bit - 1))) {
+	    for (mask = *maskp++; (bit = ffsl(mask)) != 0;
+		mask ^= (1ul << (bit - 1))) {
 		/* sock has input waiting */
 		fd = sock + bit - 1;
 		svc_getreq_common(fd);
-- 
cgit v1.1


From 3c2bafe65e8458fc5abe508bf9e336768eda01bb Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Mon, 7 Dec 2009 19:59:28 +0000
Subject: MFC revision 197579 and 199617:

Add two new fcntls to enable/disable read-ahead:

 - F_READAHEAD: specify the amount for sequential access.  The amount is
   specified in bytes and is rounded up to nearest block size.
 - F_RDAHEAD: Darwin compatible version that use 128KB as the sequential
   access size.

A third argument of zero disables the read-ahead behavior.

Please note that the read-ahead amount is also constrainted by sysctl
variable, vfs.read_max, which may need to be raised in order to better
utilize this feature.

Thanks Igor Sysoev for proposing the feature and submitting the original
version, and kib@ for his valuable comments.
---
 lib/libc/sys/fcntl.2 | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libc/sys/fcntl.2 b/lib/libc/sys/fcntl.2
index a16724c..250cef4 100644
--- a/lib/libc/sys/fcntl.2
+++ b/lib/libc/sys/fcntl.2
@@ -28,7 +28,7 @@
 .\"     @(#)fcntl.2	8.2 (Berkeley) 1/12/94
 .\" $FreeBSD$
 .\"
-.Dd March 8, 2008
+.Dd September 28, 2009
 .Dt FCNTL 2
 .Os
 .Sh NAME
@@ -241,6 +241,22 @@ will be interrupted if the signal handler has not specified the
 .Dv SA_RESTART
 (see
 .Xr sigaction 2 ) .
+.It Dv F_READAHEAD
+Set or clear the read ahead amount for sequential access to the third
+argument,
+.Fa arg ,
+which is rounded up to the nearest block size.
+A zero value in
+.Fa arg
+turns off read ahead.
+.It Dv F_RDAHEAD
+Equivalent to Darwin counterpart which sets read ahead amount of 128KB
+when the third argument,
+.Fa arg
+is non-zero.
+A zero value in
+.Fa arg
+turns off read ahead.
 .El
 .Pp
 When a shared lock has been set on a segment of a file,
-- 
cgit v1.1


From 14c9615c2ae5920e4cc43af1d15d6cbee9ab7349 Mon Sep 17 00:00:00 2001
From: fabient <fabient@FreeBSD.org>
Date: Mon, 7 Dec 2009 20:44:43 +0000
Subject: MFC 198433: Not all Intel Core (TM) CPUs implement PMC_CLASS_IAF
 fixed-function counters.  For such CPUs, use an alternate mapping of
 convenience names to events supported by PMC_CLASS_IAP programmable counters.

---
 lib/libpmc/libpmc.c | 59 ++++++++++++++++++++++++++++++++++++++++++-----------
 1 file changed, 47 insertions(+), 12 deletions(-)

(limited to 'lib')

diff --git a/lib/libpmc/libpmc.c b/lib/libpmc/libpmc.c
index 76bc89d..ecc697d 100644
--- a/lib/libpmc/libpmc.c
+++ b/lib/libpmc/libpmc.c
@@ -442,6 +442,10 @@ static struct pmc_event_alias core_aliases[] = {
 /*
  * Intel Core2 (Family 6, Model F), Core2Extreme (Family 6, Model 17H)
  * and Atom (Family 6, model 1CH) PMCs.
+ *
+ * We map aliases to events on the fixed-function counters if these
+ * are present.  Note that not all CPUs in this family contain fixed-function
+ * counters.
  */
 
 static struct pmc_event_alias core2_aliases[] = {
@@ -454,8 +458,22 @@ static struct pmc_event_alias core2_aliases[] = {
 	EV_ALIAS("unhalted-cycles",	"iaf-cpu-clk-unhalted.core"),
 	EV_ALIAS(NULL, NULL)
 };
-#define	atom_aliases	core2_aliases
-#define corei7_aliases	core2_aliases
+
+static struct pmc_event_alias core2_aliases_without_iaf[] = {
+	EV_ALIAS("branches",		"iap-br-inst-retired.any"),
+	EV_ALIAS("branch-mispredicts",	"iap-br-inst-retired.mispred"),
+	EV_ALIAS("cycles",		"tsc-tsc"),
+	EV_ALIAS("ic-misses",		"iap-l1i-misses"),
+	EV_ALIAS("instructions",	"iap-inst-retired.any_p"),
+	EV_ALIAS("interrupts",		"iap-hw-int-rcv"),
+	EV_ALIAS("unhalted-cycles",	"iap-cpu-clk-unhalted.core_p"),
+	EV_ALIAS(NULL, NULL)
+};
+
+#define	atom_aliases			core2_aliases
+#define	atom_aliases_without_iaf	core2_aliases_without_iaf
+#define corei7_aliases			core2_aliases
+#define corei7_aliases_without_iaf	core2_aliases_without_iaf
 
 #define	IAF_KW_OS		"os"
 #define	IAF_KW_USR		"usr"
@@ -2379,6 +2397,10 @@ pmc_init(void)
 	uint32_t abi_version;
 	struct module_stat pmc_modstat;
 	struct pmc_op_getcpuinfo op_cpu_info;
+#if defined(__amd64__) || defined(__i386__)
+	int cpu_has_iaf_counters;
+	unsigned int t;
+#endif
 
 	if (pmc_syscall != -1) /* already inited */
 		return (0);
@@ -2420,6 +2442,8 @@ pmc_init(void)
 	if (pmc_class_table == NULL)
 		return (-1);
 
+	for (n = 0; n < PMC_CLASS_TABLE_SIZE; n++)
+		pmc_class_table[n] = NULL;
 
 	/*
 	 * Fill in the class table.
@@ -2427,6 +2451,14 @@ pmc_init(void)
 	n = 0;
 #if defined(__amd64__) || defined(__i386__)
 	pmc_class_table[n++] = &tsc_class_table_descr;
+
+	/*
+ 	 * Check if this CPU has fixed function counters.
+	 */
+	cpu_has_iaf_counters = 0;
+	for (t = 0; t < cpu_info.pm_nclass; t++)
+		if (cpu_info.pm_classes[t].pm_class == PMC_CLASS_IAF)
+			cpu_has_iaf_counters = 1;
 #endif
 
 #define	PMC_MDEP_INIT(C) do {					\
@@ -2436,6 +2468,16 @@ pmc_init(void)
 		    PMC_TABLE_SIZE(C##_pmc_classes);		\
 	} while (0)
 
+#define	PMC_MDEP_INIT_INTEL_V2(C) do {					\
+		PMC_MDEP_INIT(C);					\
+		if (cpu_has_iaf_counters) 				\
+			pmc_class_table[n++] = &iaf_class_table_descr;	\
+		else							\
+			pmc_mdep_event_aliases =			\
+				C##_aliases_without_iaf;		\
+		pmc_class_table[n] = &C##_class_table_descr;		\
+	} while (0)
+
 	/* Configure the event name parser. */
 	switch (cpu_info.pm_cputype) {
 #if defined(__i386__)
@@ -2461,24 +2503,17 @@ pmc_init(void)
 		pmc_class_table[n] = &k8_class_table_descr;
 		break;
 	case PMC_CPU_INTEL_ATOM:
-		PMC_MDEP_INIT(atom);
-		pmc_class_table[n++] = &iaf_class_table_descr;
-		pmc_class_table[n]   = &atom_class_table_descr;
+		PMC_MDEP_INIT_INTEL_V2(atom);
 		break;
 	case PMC_CPU_INTEL_CORE:
 		PMC_MDEP_INIT(core);
-		pmc_class_table[n] = &core_class_table_descr;
 		break;
 	case PMC_CPU_INTEL_CORE2:
 	case PMC_CPU_INTEL_CORE2EXTREME:
-		PMC_MDEP_INIT(core2);
-		pmc_class_table[n++] = &iaf_class_table_descr;
-		pmc_class_table[n]   = &core2_class_table_descr;
+		PMC_MDEP_INIT_INTEL_V2(core2);
 		break;
 	case PMC_CPU_INTEL_COREI7:
-		PMC_MDEP_INIT(corei7);
-		pmc_class_table[n++] = &iaf_class_table_descr;
-		pmc_class_table[n]   = &corei7_class_table_descr;
+		PMC_MDEP_INIT_INTEL_V2(corei7);
 		break;
 	case PMC_CPU_INTEL_PIV:
 		PMC_MDEP_INIT(p4);
-- 
cgit v1.1


From c57856a7abab4b7e75e856abab338a796b2a141b Mon Sep 17 00:00:00 2001
From: thompsa <thompsa@FreeBSD.org>
Date: Wed, 9 Dec 2009 21:34:38 +0000
Subject: MFC r199575

  - fix a transfer cancelling bug/segfault [1]
  - correct a return code in the transfer cancel function.
  - add new API function, libusb20_tr_bulk_intr_sync().

 Submitted by:	HPS
 Reported by:	Robert Jenssen [1]
---
 lib/libusb/libusb10.c | 41 +++++++++++++++++++----
 lib/libusb/libusb10.h |  4 ++-
 lib/libusb/libusb20.3 | 27 ++++++++++++++-
 lib/libusb/libusb20.c | 91 ++++++++++++++++++++++++++++++++++++++++++++++++++-
 lib/libusb/libusb20.h |  3 +-
 5 files changed, 155 insertions(+), 11 deletions(-)

(limited to 'lib')

diff --git a/lib/libusb/libusb10.c b/lib/libusb/libusb10.c
index 55ee1c5..9a5154e 100644
--- a/lib/libusb/libusb10.c
+++ b/lib/libusb/libusb10.c
@@ -833,8 +833,12 @@ libusb10_complete_transfer(struct libusb20_transfer *pxfer,
 	if (pxfer != NULL)
 		libusb20_tr_set_priv_sc1(pxfer, NULL);
 
+	/* set transfer status */
 	uxfer->status = status;
 
+	/* update super transfer state */
+	sxfer->state = LIBUSB_SUPER_XFER_ST_NONE;
+
 	dev = libusb_get_device(uxfer->dev_handle);
 
 	TAILQ_INSERT_TAIL(&dev->ctx->tr_done, sxfer, entry);
@@ -1111,6 +1115,8 @@ libusb10_submit_transfer_sub(struct libusb20_device *pdev, uint8_t endpoint)
 		return;
 	case 2:
 		sxfer = libusb20_tr_get_priv_sc1(pxfer1);
+		if (sxfer == NULL)
+			return;		/* cancelling */
 		if (sxfer->rem_len)
 			return;		/* cannot queue another one */
 		/* swap transfers */
@@ -1118,6 +1124,8 @@ libusb10_submit_transfer_sub(struct libusb20_device *pdev, uint8_t endpoint)
 		break;
 	case 1:
 		sxfer = libusb20_tr_get_priv_sc1(pxfer0);
+		if (sxfer == NULL)
+			return;		/* cancelling */
 		if (sxfer->rem_len)
 			return;		/* cannot queue another one */
 		/* swap transfers */
@@ -1229,12 +1237,18 @@ libusb_submit_transfer(struct libusb_transfer *uxfer)
 	if (pxfer0 == NULL || pxfer1 == NULL) {
 		err = LIBUSB_ERROR_OTHER;
 	} else if ((sxfer->entry.tqe_prev != NULL) ||
-		    (libusb20_tr_get_priv_sc1(pxfer0) == sxfer) ||
+	    (libusb20_tr_get_priv_sc1(pxfer0) == sxfer) ||
 	    (libusb20_tr_get_priv_sc1(pxfer1) == sxfer)) {
 		err = LIBUSB_ERROR_BUSY;
 	} else {
+
+		/* set pending state */
+		sxfer->state = LIBUSB_SUPER_XFER_ST_PEND;
+
+		/* insert transfer into transfer head list */
 		TAILQ_INSERT_TAIL(&dev->tr_head, sxfer, entry);
 
+		/* start work transfers */
 		libusb10_submit_transfer_sub(
 		    uxfer->dev_handle, endpoint);
 
@@ -1258,12 +1272,14 @@ libusb_cancel_transfer(struct libusb_transfer *uxfer)
 	struct libusb_super_transfer *sxfer;
 	struct libusb_device *dev;
 	uint32_t endpoint;
+	int retval;
 
 	if (uxfer == NULL)
 		return (LIBUSB_ERROR_INVALID_PARAM);
 
+	/* check if not initialised */
 	if (uxfer->dev_handle == NULL)
-		return (LIBUSB_ERROR_INVALID_PARAM);
+		return (LIBUSB_ERROR_NOT_FOUND);
 
 	endpoint = uxfer->endpoint;
 
@@ -1277,39 +1293,50 @@ libusb_cancel_transfer(struct libusb_transfer *uxfer)
 	sxfer = (struct libusb_super_transfer *)(
 	    (uint8_t *)uxfer - sizeof(*sxfer));
 
+	retval = 0;
+
 	CTX_LOCK(dev->ctx);
 
 	pxfer0 = libusb10_get_transfer(uxfer->dev_handle, endpoint, 0);
 	pxfer1 = libusb10_get_transfer(uxfer->dev_handle, endpoint, 1);
 
-	if (sxfer->entry.tqe_prev != NULL) {
+	if (sxfer->state != LIBUSB_SUPER_XFER_ST_PEND) {
+		/* only update the transfer status */
+		uxfer->status = LIBUSB_TRANSFER_CANCELLED;
+		retval = LIBUSB_ERROR_NOT_FOUND;
+	} else if (sxfer->entry.tqe_prev != NULL) {
 		/* we are lucky - transfer is on a queue */
 		TAILQ_REMOVE(&dev->tr_head, sxfer, entry);
 		sxfer->entry.tqe_prev = NULL;
-		libusb10_complete_transfer(NULL, sxfer, LIBUSB_TRANSFER_CANCELLED);
+		libusb10_complete_transfer(NULL,
+		    sxfer, LIBUSB_TRANSFER_CANCELLED);
 	} else if (pxfer0 == NULL || pxfer1 == NULL) {
 		/* not started */
+		retval = LIBUSB_ERROR_NOT_FOUND;
 	} else if (libusb20_tr_get_priv_sc1(pxfer0) == sxfer) {
-		libusb10_complete_transfer(pxfer0, sxfer, LIBUSB_TRANSFER_CANCELLED);
+		libusb10_complete_transfer(pxfer0,
+		    sxfer, LIBUSB_TRANSFER_CANCELLED);
 		libusb20_tr_stop(pxfer0);
 		/* make sure the queue doesn't stall */
 		libusb10_submit_transfer_sub(
 		    uxfer->dev_handle, endpoint);
 	} else if (libusb20_tr_get_priv_sc1(pxfer1) == sxfer) {
-		libusb10_complete_transfer(pxfer1, sxfer, LIBUSB_TRANSFER_CANCELLED);
+		libusb10_complete_transfer(pxfer1,
+		    sxfer, LIBUSB_TRANSFER_CANCELLED);
 		libusb20_tr_stop(pxfer1);
 		/* make sure the queue doesn't stall */
 		libusb10_submit_transfer_sub(
 		    uxfer->dev_handle, endpoint);
 	} else {
 		/* not started */
+		retval = LIBUSB_ERROR_NOT_FOUND;
 	}
 
 	CTX_UNLOCK(dev->ctx);
 
 	DPRINTF(dev->ctx, LIBUSB_DEBUG_FUNCTION, "libusb_cancel_transfer leave");
 
-	return (0);
+	return (retval);
 }
 
 UNEXPORTED void
diff --git a/lib/libusb/libusb10.h b/lib/libusb/libusb10.h
index 338c2aa..d2a2bd7 100644
--- a/lib/libusb/libusb10.h
+++ b/lib/libusb/libusb10.h
@@ -65,7 +65,9 @@ struct libusb_super_transfer {
 	uint8_t *curr_data;
 	uint32_t rem_len;
 	uint32_t last_len;
-	uint8_t	flags;
+	uint8_t	state;
+#define	LIBUSB_SUPER_XFER_ST_NONE 0
+#define	LIBUSB_SUPER_XFER_ST_PEND 1
 };
 
 struct libusb_context {
diff --git a/lib/libusb/libusb20.3 b/lib/libusb/libusb20.3
index f902883..5404013 100644
--- a/lib/libusb/libusb20.3
+++ b/lib/libusb/libusb20.3
@@ -26,7 +26,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd June 22, 2009
+.Dd November 18, 2009
 .Dt LIBUSB20 3
 .Os
 .Sh NAME
@@ -98,6 +98,8 @@ USB access library (libusb -lusb)
 .Fn libusb20_tr_setup_intr "struct libusb20_transfer *xfer" "void *pbuf" "uint32_t length" "uint32_t timeout"
 .Ft void
 .Fn libusb20_tr_setup_isoc "struct libusb20_transfer *xfer" "void *pbuf" "uint32_t length" "uint61_t fr_index"
+.Ft uint8_t
+.Fn libusb20_tr_bulk_intr_sync "struct libusb20_transfer *xfer" "void *pbuf" "uint32_t length" "uint32_t *pactlen" "uint32_t timeout"
 .Ft void
 .Fn libusb20_tr_start "struct libusb20_transfer *xfer"
 .Ft void
@@ -451,6 +453,29 @@ is a helper function for setting up a multi frame USB ISOCHRONOUS transfer.
 .
 .Pp
 .
+.Fn libusb20_tr_bulk_intr_sync
+will perform a synchronous BULK or INTERRUPT transfer having length given by the
+.Fa length
+argument and buffer pointer given by the
+.Fa pbuf
+argument on the USB transfer given by the
+.Fa xfer
+argument.
+.
+If the
+.Fa pactlen
+argument is non-NULL the actual transfer length will be stored at the given pointer destination.
+.
+If the
+.Fa timeout
+argument is non-zero the transfer will timeout after the given value in milliseconds.
+.
+This function does not change the transfer flags, like short packet not ok.
+.
+This function returns zero on success else a LIBUSB20_TRANSFER_XXX value is returned.
+.
+.Pp
+.
 .Fn libusb20_tr_start
 will get the USB transfer started, if not already
 started.
diff --git a/lib/libusb/libusb20.c b/lib/libusb/libusb20.c
index 1a33805..41f9ac7 100644
--- a/lib/libusb/libusb20.c
+++ b/lib/libusb/libusb20.c
@@ -1,6 +1,6 @@
 /* $FreeBSD$ */
 /*-
- * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
+ * Copyright (c) 2008-2009 Hans Petter Selasky. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -263,6 +263,10 @@ libusb20_tr_get_priv_sc1(struct libusb20_transfer *xfer)
 void
 libusb20_tr_stop(struct libusb20_transfer *xfer)
 {
+	if (!xfer->is_opened) {
+		/* transfer is not opened */
+		return;
+	}
 	if (!xfer->is_pending) {
 		/* transfer not pending */
 		return;
@@ -280,6 +284,10 @@ libusb20_tr_stop(struct libusb20_transfer *xfer)
 void
 libusb20_tr_drain(struct libusb20_transfer *xfer)
 {
+	if (!xfer->is_opened) {
+		/* transfer is not opened */
+		return;
+	}
 	/* make sure that we are cancelling */
 	libusb20_tr_stop(xfer);
 
@@ -415,9 +423,79 @@ libusb20_tr_setup_isoc(struct libusb20_transfer *xfer, void *pBuf, uint32_t leng
 	return;
 }
 
+uint8_t
+libusb20_tr_bulk_intr_sync(struct libusb20_transfer *xfer,
+    void *pbuf, uint32_t length, uint32_t *pactlen,
+    uint32_t timeout)
+{
+	struct libusb20_device *pdev = xfer->pdev;
+	uint32_t transfer_max;
+	uint32_t transfer_act;
+	uint8_t retval;
+
+	/* set some sensible default value */
+	if (pactlen != NULL)
+		*pactlen = 0;
+
+	/* check for error condition */
+	if (libusb20_tr_pending(xfer))
+		return (LIBUSB20_ERROR_OTHER);
+
+	do {
+		/* compute maximum transfer length */
+		transfer_max = 
+		    libusb20_tr_get_max_total_length(xfer);
+
+		if (transfer_max > length)
+			transfer_max = length;
+
+		/* setup bulk or interrupt transfer */
+		libusb20_tr_setup_bulk(xfer, pbuf, 
+		    transfer_max, timeout);
+
+		/* start the transfer */
+		libusb20_tr_start(xfer);
+
+		/* wait for transfer completion */
+		while (libusb20_dev_process(pdev) == 0) {
+
+			if (libusb20_tr_pending(xfer) == 0)
+				break;
+
+			libusb20_dev_wait_process(pdev, -1);
+		}
+
+		transfer_act = libusb20_tr_get_actual_length(xfer);
+
+		/* update actual length, if any */
+		if (pactlen != NULL)
+			pactlen[0] += transfer_act;
+
+		/* check transfer status */
+		retval = libusb20_tr_get_status(xfer);
+		if (retval)
+			break;
+
+		/* check for short transfer */
+		if (transfer_act != transfer_max)
+			break;
+
+		/* update buffer pointer and length */
+		pbuf = ((uint8_t *)pbuf) + transfer_max;
+		length = length - transfer_max;
+
+	} while (length != 0);
+
+	return (retval);
+}
+
 void
 libusb20_tr_submit(struct libusb20_transfer *xfer)
 {
+	if (!xfer->is_opened) {
+		/* transfer is not opened */
+		return;
+	}
 	if (xfer->is_pending) {
 		/* should not happen */
 		return;
@@ -433,6 +511,10 @@ libusb20_tr_submit(struct libusb20_transfer *xfer)
 void
 libusb20_tr_start(struct libusb20_transfer *xfer)
 {
+	if (!xfer->is_opened) {
+		/* transfer is not opened */
+		return;
+	}
 	if (xfer->is_pending) {
 		if (xfer->is_cancel) {
 			/* cancelling - restart */
@@ -461,7 +543,14 @@ libusb20_dev_close(struct libusb20_device *pdev)
 	for (x = 0; x != pdev->nTransfer; x++) {
 		xfer = pdev->pTransfer + x;
 
+		if (!xfer->is_opened) {
+			/* transfer is not opened */
+			continue;
+		}
+
 		libusb20_tr_drain(xfer);
+
+		libusb20_tr_close(xfer);
 	}
 
 	if (pdev->pTransfer != NULL) {
diff --git a/lib/libusb/libusb20.h b/lib/libusb/libusb20.h
index 9f45861..bb7d8a4 100644
--- a/lib/libusb/libusb20.h
+++ b/lib/libusb/libusb20.h
@@ -1,6 +1,6 @@
 /* $FreeBSD$ */
 /*-
- * Copyright (c) 2008 Hans Petter Selasky. All rights reserved.
+ * Copyright (c) 2008-2009 Hans Petter Selasky. All rights reserved.
  * Copyright (c) 2007-2008 Daniel Drake.  All rights reserved.
  * Copyright (c) 2001 Johannes Erdfelt.  All rights reserved.
  *
@@ -226,6 +226,7 @@ void	libusb20_tr_setup_bulk(struct libusb20_transfer *xfer, void *pbuf, uint32_t
 void	libusb20_tr_setup_control(struct libusb20_transfer *xfer, void *psetup, void *pbuf, uint32_t timeout);
 void	libusb20_tr_setup_intr(struct libusb20_transfer *xfer, void *pbuf, uint32_t length, uint32_t timeout);
 void	libusb20_tr_setup_isoc(struct libusb20_transfer *xfer, void *pbuf, uint32_t length, uint16_t fr_index);
+uint8_t	libusb20_tr_bulk_intr_sync(struct libusb20_transfer *xfer, void *pbuf, uint32_t length, uint32_t *pactlen, uint32_t timeout);
 void	libusb20_tr_start(struct libusb20_transfer *xfer);
 void	libusb20_tr_stop(struct libusb20_transfer *xfer);
 void	libusb20_tr_submit(struct libusb20_transfer *xfer);
-- 
cgit v1.1


From fe35bb371035da80542ae632759a31caac101e0a Mon Sep 17 00:00:00 2001
From: dougb <dougb@FreeBSD.org>
Date: Fri, 11 Dec 2009 01:23:58 +0000
Subject: MFC r199958:

Update to BIND 9.6.1-P2. The vulnerability this is designed to fix is
related to DNSSEC validation on a resolving name server that allows
access to untrusted users. If your system does not fall into all 3 of
these categories you do not need to update immediately.
---
 lib/bind/config.h | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'lib')

diff --git a/lib/bind/config.h b/lib/bind/config.h
index 8e52426..1d68450 100644
--- a/lib/bind/config.h
+++ b/lib/bind/config.h
@@ -277,6 +277,10 @@ int sigwait(const unsigned int *set, int *sig);
 /* Define to 1 if you have the <unistd.h> header file. */
 #define HAVE_UNISTD_H 1
 
+/* Define to the sub-directory in which libtool stores uninstalled libraries.
+   */
+#define LT_OBJDIR ".libs/"
+
 /* Defined if extern char *optarg is not declared. */
 /* #undef NEED_OPTARG */
 
-- 
cgit v1.1


From 64dfd453c6caa9e0e21d382f621b8b6a17d25ae8 Mon Sep 17 00:00:00 2001
From: rwatson <rwatson@FreeBSD.org>
Date: Mon, 14 Dec 2009 13:13:43 +0000
Subject: Merge r197841 from head to stable/8:

  Add a new errno, ENOTCAPABLE, to be returned when a process requests an
  operation on a file descriptor that is not authorized by the descriptor's
  capability flags.

  Sponsored by:	Google
---
 lib/libc/gen/errlst.c | 1 +
 lib/libc/sys/intro.2  | 3 +++
 2 files changed, 4 insertions(+)

(limited to 'lib')

diff --git a/lib/libc/gen/errlst.c b/lib/libc/gen/errlst.c
index bc3e0c9..db6bfa2 100644
--- a/lib/libc/gen/errlst.c
+++ b/lib/libc/gen/errlst.c
@@ -150,5 +150,6 @@ const char *const sys_errlist[] = {
 	"Multihop attempted",			/* 90 - EMULTIHOP */
 	"Link has been severed",		/* 91 - ENOLINK */
 	"Protocol error",			/* 92 - EPROTO */
+	"Capabilities insufficient",		/* 93 - ENOTCAPABLE */
 };
 const int sys_nerr = sizeof(sys_errlist) / sizeof(sys_errlist[0]);
diff --git a/lib/libc/sys/intro.2 b/lib/libc/sys/intro.2
index 7338f8b..1138483 100644
--- a/lib/libc/sys/intro.2
+++ b/lib/libc/sys/intro.2
@@ -456,6 +456,9 @@ The specified extended attribute does not exist.
 .It Er 88 EDOOFUS Em "Programming error" .
 A function or API is being abused in a way which could only be detected
 at run-time.
+.It Er 93 ENOTCAPABLE Em "Capabilities insufficient" .
+An operation on a capability file descriptor requires greater privilege than
+the capability allows.
 .El
 .Sh DEFINITIONS
 .Bl -tag -width Ds
-- 
cgit v1.1


From 0e440163ad12fb2b7cce130c8d5ed6eb4b566c3c Mon Sep 17 00:00:00 2001
From: marcel <marcel@FreeBSD.org>
Date: Thu, 17 Dec 2009 02:06:07 +0000
Subject: MFC rev 200498: Work-around a race condition on ia64 while unlocking
 a contested lock.

---
 lib/libthr/thread/thr_umtx.c | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'lib')

diff --git a/lib/libthr/thread/thr_umtx.c b/lib/libthr/thread/thr_umtx.c
index 7c59bb7..5af923d 100644
--- a/lib/libthr/thread/thr_umtx.c
+++ b/lib/libthr/thread/thr_umtx.c
@@ -112,10 +112,13 @@ __thr_umutex_timedlock(struct umutex *mtx, uint32_t id,
 int
 __thr_umutex_unlock(struct umutex *mtx, uint32_t id)
 {
+#ifndef __ia64__
+	/* XXX this logic has a race-condition on ia64. */
 	if ((mtx->m_flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) == 0) {
 		atomic_cmpset_rel_32(&mtx->m_owner, id | UMUTEX_CONTESTED, UMUTEX_CONTESTED);
 		return _umtx_op_err(mtx, UMTX_OP_MUTEX_WAKE, 0, 0, 0);
 	}
+#endif /* __ia64__ */
 	return _umtx_op_err(mtx, UMTX_OP_MUTEX_UNLOCK, 0, 0, 0);
 }
 
-- 
cgit v1.1


From bde46f0c425a67a0bcb9d266dfff41b9b315ee5d Mon Sep 17 00:00:00 2001
From: kib <kib@FreeBSD.org>
Date: Thu, 17 Dec 2009 18:56:52 +0000
Subject: MFC r199826: sigset() is the name of function specified by SUSv4.
 Replace it to avoid conflict.

---
 lib/libutil/pw_util.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/libutil/pw_util.c b/lib/libutil/pw_util.c
index 69232fb..75459e3 100644
--- a/lib/libutil/pw_util.c
+++ b/lib/libutil/pw_util.c
@@ -289,7 +289,7 @@ int
 pw_edit(int notsetuid)
 {
 	struct sigaction sa, sa_int, sa_quit;
-	sigset_t oldsigset, sigset;
+	sigset_t oldsigset, nsigset;
 	struct stat st1, st2;
 	const char *editor;
 	int pstat;
@@ -303,9 +303,9 @@ pw_edit(int notsetuid)
 	sa.sa_flags = 0;
 	sigaction(SIGINT, &sa, &sa_int);
 	sigaction(SIGQUIT, &sa, &sa_quit);
-	sigemptyset(&sigset);
-	sigaddset(&sigset, SIGCHLD);
-	sigprocmask(SIG_BLOCK, &sigset, &oldsigset);
+	sigemptyset(&nsigset);
+	sigaddset(&nsigset, SIGCHLD);
+	sigprocmask(SIG_BLOCK, &nsigset, &oldsigset);
 	switch ((editpid = fork())) {
 	case -1:
 		return (-1);
-- 
cgit v1.1


From a840b236b48e169ae967870732dcf792adb9acb5 Mon Sep 17 00:00:00 2001
From: jhb <jhb@FreeBSD.org>
Date: Thu, 17 Dec 2009 20:41:27 +0000
Subject: MFC 199606, 199614: Add an internal _once() method.  This works
 identical to pthread_once(3) with the additional property that it is safe for
 routines in libc to use in both single-threaded and multi-threaded processes.
  Multi-threaded processes use the pthread_once() implementation from the
 threading library while single-threaded processes use a simplified "stub"
 version internal to libc.

---
 lib/libc/gen/Makefile.inc       |  3 +-
 lib/libc/gen/_once_stub.c       | 64 +++++++++++++++++++++++++++++++++++++++++
 lib/libc/include/libc_private.h |  7 +++++
 3 files changed, 73 insertions(+), 1 deletion(-)
 create mode 100644 lib/libc/gen/_once_stub.c

(limited to 'lib')

diff --git a/lib/libc/gen/Makefile.inc b/lib/libc/gen/Makefile.inc
index b06f846..903fa22 100644
--- a/lib/libc/gen/Makefile.inc
+++ b/lib/libc/gen/Makefile.inc
@@ -5,7 +5,8 @@
 .PATH: ${.CURDIR}/${MACHINE_ARCH}/gen ${.CURDIR}/gen
 
 SRCS+=  __getosreldate.c __xuname.c \
-	_pthread_stubs.c _rand48.c _spinlock_stub.c _thread_init.c \
+	_once_stub.c _pthread_stubs.c _rand48.c _spinlock_stub.c \
+	_thread_init.c \
 	alarm.c arc4random.c assert.c basename.c check_utility_compat.c \
 	clock.c closedir.c confstr.c \
 	crypt.c ctermid.c daemon.c devname.c dirname.c disklabel.c \
diff --git a/lib/libc/gen/_once_stub.c b/lib/libc/gen/_once_stub.c
new file mode 100644
index 0000000..d2acc29
--- /dev/null
+++ b/lib/libc/gen/_once_stub.c
@@ -0,0 +1,64 @@
+/*-
+ * Copyright (c) 2009 Advanced Computing Technologies LLC
+ * Written by: John H. Baldwin <jhb@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "namespace.h"
+#include <pthread.h>
+#include "un-namespace.h"
+#include "libc_private.h"
+
+/* This implements pthread_once() for the single-threaded case. */
+static int
+_libc_once(pthread_once_t *once_control, void (*init_routine)(void))
+{
+
+	if (once_control->state == PTHREAD_DONE_INIT)
+		return (0);
+	init_routine();
+	once_control->state = PTHREAD_DONE_INIT;
+	return (0);
+}
+
+/*
+ * This is the internal interface provided to libc.  It will use
+ * pthread_once() from the threading library in a multi-threaded
+ * process and _libc_once() for a single-threaded library.  Because
+ * _libc_once() uses the same ABI for the values in the pthread_once_t
+ * structure as the threading library, it is safe for a process to
+ * switch from _libc_once() to pthread_once() when threading is
+ * enabled.
+ */
+int
+_once(pthread_once_t *once_control, void (*init_routine)(void))
+{
+
+	if (__isthreaded)
+		return (_pthread_once(once_control, init_routine));
+	return (_libc_once(once_control, init_routine));
+}
diff --git a/lib/libc/include/libc_private.h b/lib/libc/include/libc_private.h
index 50903f3..052eb13 100644
--- a/lib/libc/include/libc_private.h
+++ b/lib/libc/include/libc_private.h
@@ -34,6 +34,7 @@
 
 #ifndef _LIBC_PRIVATE_H_
 #define _LIBC_PRIVATE_H_
+#include <sys/_pthreadtypes.h>
 
 /*
  * This global flag is non-zero when a process has created one
@@ -147,6 +148,12 @@ int _yp_check(char **);
 void _init_tls(void);
 
 /*
+ * Provides pthread_once()-like functionality for both single-threaded
+ * and multi-threaded applications.
+ */
+int _once(pthread_once_t *, void (*)(void));
+
+/*
  * Set the TLS thread pointer
  */
 void _set_tp(void *tp);
-- 
cgit v1.1


From c55271bb4d1df908a79cd7e9a0f7f8839eac2d45 Mon Sep 17 00:00:00 2001
From: brooks <brooks@FreeBSD.org>
Date: Fri, 18 Dec 2009 06:09:43 +0000
Subject: MFC r200366

Add a missing else that negated the truncation of ki_ngroups to NGROUPS.

Submitted by:   Dmitry Pryanishnikov <lynx dot ripe at gmail dot com>
---
 lib/libkvm/kvm_proc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libkvm/kvm_proc.c b/lib/libkvm/kvm_proc.c
index 071f859..f8f8483 100644
--- a/lib/libkvm/kvm_proc.c
+++ b/lib/libkvm/kvm_proc.c
@@ -149,7 +149,7 @@ kvm_proclist(kd, what, arg, p, bp, maxcnt)
 			if (ucred.cr_ngroups > KI_NGROUPS) {
 				kp->ki_ngroups = KI_NGROUPS;
 				kp->ki_cr_flags |= KI_CRF_GRP_OVERFLOW;
-			}
+			} else
 				kp->ki_ngroups = ucred.cr_ngroups;
 			kvm_read(kd, (u_long)ucred.cr_groups, kp->ki_groups,
 			    kp->ki_ngroups * sizeof(gid_t));
-- 
cgit v1.1


From 099d1bfda5e6c17840e60f4ec0be795da67d3585 Mon Sep 17 00:00:00 2001
From: scf <scf@FreeBSD.org>
Date: Fri, 18 Dec 2009 20:05:10 +0000
Subject: Merge from head to stable/8:

r200423:
Remove a dead store.
---
 lib/libutil/gr_util.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/libutil/gr_util.c b/lib/libutil/gr_util.c
index 77e0653..633f435 100644
--- a/lib/libutil/gr_util.c
+++ b/lib/libutil/gr_util.c
@@ -117,8 +117,8 @@ gr_make(const struct group *gr)
 	/* Create the group line and fill it. */
 	if ((line = malloc(line_size)) == NULL)
 		return (NULL);
-	line_size = snprintf(line, line_size, group_line_format, gr->gr_name,
-	    gr->gr_passwd, (uintmax_t)gr->gr_gid);
+	snprintf(line, line_size, group_line_format, gr->gr_name, gr->gr_passwd,
+	    (uintmax_t)gr->gr_gid);
 	if (gr->gr_mem != NULL)
 		for (ndx = 0; gr->gr_mem[ndx] != NULL; ndx++) {
 			strcat(line, gr->gr_mem[ndx]);
-- 
cgit v1.1


From 176e26b324e1868e4bf61c5bb3a7a64eb7ad048b Mon Sep 17 00:00:00 2001
From: kib <kib@FreeBSD.org>
Date: Sat, 19 Dec 2009 11:47:00 +0000
Subject: MFC r198508, r198509: Reimplement pselect() in kernel, making change
 of sigmask and sleep atomic.

MFC r198538:
Move pselect(3) man page to section 2.
---
 lib/libc/gen/Makefile.inc        |   4 +-
 lib/libc/gen/Symbol.map          |   2 -
 lib/libc/gen/pselect.3           | 127 ---------------------------------------
 lib/libc/gen/pselect.c           |  78 ------------------------
 lib/libc/sys/Makefile.inc        |   3 +-
 lib/libc/sys/Symbol.map          |   3 +
 lib/libc/sys/pselect.2           | 122 +++++++++++++++++++++++++++++++++++++
 lib/libthr/thread/thr_syscalls.c |   4 +-
 8 files changed, 132 insertions(+), 211 deletions(-)
 delete mode 100644 lib/libc/gen/pselect.3
 delete mode 100644 lib/libc/gen/pselect.c
 create mode 100644 lib/libc/sys/pselect.2

(limited to 'lib')

diff --git a/lib/libc/gen/Makefile.inc b/lib/libc/gen/Makefile.inc
index 903fa22..d04dd75 100644
--- a/lib/libc/gen/Makefile.inc
+++ b/lib/libc/gen/Makefile.inc
@@ -22,7 +22,7 @@ SRCS+=  __getosreldate.c __xuname.c \
 	initgroups.c isatty.c isinf.c isnan.c jrand48.c lcong48.c \
 	lockf.c lrand48.c mrand48.c nftw.c nice.c \
 	nlist.c nrand48.c opendir.c \
-	pause.c pmadvise.c popen.c posix_spawn.c pselect.c \
+	pause.c pmadvise.c popen.c posix_spawn.c \
 	psignal.c pw_scan.c pwcache.c \
 	raise.c readdir.c readpassphrase.c rewinddir.c \
 	scandir.c seed48.c seekdir.c sem.c semctl.c \
@@ -63,7 +63,7 @@ MAN+=	alarm.3 arc4random.3 \
 	posix_spawnattr_getpgroup.3 posix_spawnattr_getschedparam.3 \
 	posix_spawnattr_getschedpolicy.3 posix_spawnattr_init.3 \
 	posix_spawnattr_getsigdefault.3 posix_spawnattr_getsigmask.3 \
-	pselect.3 psignal.3 pwcache.3 \
+	psignal.3 pwcache.3 \
 	raise.3 rand48.3 readpassphrase.3 rfork_thread.3 \
 	scandir.3 sem_destroy.3 sem_getvalue.3 sem_init.3 \
 	sem_open.3 sem_post.3 sem_timedwait.3 sem_wait.3 \
diff --git a/lib/libc/gen/Symbol.map b/lib/libc/gen/Symbol.map
index 197430a..4c19a3d 100644
--- a/lib/libc/gen/Symbol.map
+++ b/lib/libc/gen/Symbol.map
@@ -223,7 +223,6 @@ FBSD_1.0 {
 	posix_madvise;
 	popen;
 	pclose;
-	pselect;
 	psignal;
 	raise;
 	readdir;
@@ -453,7 +452,6 @@ FBSDprivate_1.0 {
 	__opendir2;
 	__pause;
 	_pause;
-	__pselect;
 	__pw_scan;	/* Used by (at least) libutil */
 	__raise;
 	_raise;
diff --git a/lib/libc/gen/pselect.3 b/lib/libc/gen/pselect.3
deleted file mode 100644
index a56a0a8..0000000
--- a/lib/libc/gen/pselect.3
+++ /dev/null
@@ -1,127 +0,0 @@
-.\"
-.\" Copyright 2002 Massachusetts Institute of Technology
-.\"
-.\" Permission to use, copy, modify, and distribute this software and
-.\" its documentation for any purpose and without fee is hereby
-.\" granted, provided that both the above copyright notice and this
-.\" permission notice appear in all copies, that both the above
-.\" copyright notice and this permission notice appear in all
-.\" supporting documentation, and that the name of M.I.T. not be used
-.\" in advertising or publicity pertaining to distribution of the
-.\" software without specific, written prior permission.  M.I.T. makes
-.\" no representations about the suitability of this software for any
-.\" purpose.  It is provided "as is" without express or implied
-.\" warranty.
-.\"
-.\" THIS SOFTWARE IS PROVIDED BY M.I.T. ``AS IS''.  M.I.T. DISCLAIMS
-.\" ALL EXPRESS OR IMPLIED WARRANTIES WITH REGARD TO THIS SOFTWARE,
-.\" INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-.\" MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. IN NO EVENT
-.\" SHALL M.I.T. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-.\" SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-.\" LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
-.\" USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-.\" ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-.\" OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
-.\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-.\" SUCH DAMAGE.
-.\"
-.\" $FreeBSD$
-.\"
-.Dd June 16, 2002
-.Dt PSELECT 3
-.Os
-.Sh NAME
-.Nm pselect
-.Nd synchronous I/O multiplexing a la POSIX.1g
-.Sh LIBRARY
-.Lb libc
-.Sh SYNOPSIS
-.In sys/select.h
-.Ft int
-.Fo pselect
-.Fa "int nfds"
-.Fa "fd_set * restrict readfds"
-.Fa "fd_set * restrict writefds"
-.Fa "fd_set * restrict exceptfds"
-.Fa "const struct timespec * restrict timeout"
-.Fa "const sigset_t * restrict newsigmask"
-.Fc
-.Sh DESCRIPTION
-The
-.Fn pselect
-function was introduced by
-.St -p1003.1g-2000
-as a slightly stronger version of
-.Xr select 2 .
-The
-.Fa nfds , readfds , writefds ,
-and
-.Fa exceptfds
-arguments are all identical to the analogous arguments of
-.Fn select .
-The
-.Fa timeout
-argument in
-.Fn pselect
-points to a
-.Vt "const struct timespec"
-rather than the (modifiable)
-.Vt "struct timeval"
-used by
-.Fn select ;
-as in
-.Fn select ,
-a null pointer may be passed to indicate that
-.Fn pselect
-should wait indefinitely.
-Finally,
-.Fa newsigmask
-specifies a signal mask which is set while waiting for input.
-When
-.Fn pselect
-returns, the original signal mask is restored.
-.Pp
-See
-.Xr select 2
-for a more detailed discussion of the semantics of this interface, and
-for macros used to manipulate the
-.Vt "fd_set"
-data type.
-.Sh IMPLEMENTATION NOTES
-The
-.Fn pselect
-function is implemented in the C library as a wrapper around
-.Fn select .
-.Sh RETURN VALUES
-The
-.Fn pselect
-function returns the same values and under the same conditions as
-.Fn select .
-.Sh ERRORS
-The
-.Fn pselect
-function may fail for any of the reasons documented for
-.Xr select 2
-and (if a signal mask is provided)
-.Xr sigprocmask 2 .
-.Sh SEE ALSO
-.Xr kqueue 2 ,
-.Xr poll 2 ,
-.Xr select 2 ,
-.Xr sigprocmask 2
-.Sh STANDARDS
-The
-.Fn pselect
-function conforms to
-.St -p1003.1-2001 .
-.Sh HISTORY
-The
-.Fn pselect
-function first appeared in
-.Fx 5.0 .
-.Sh AUTHORS
-The
-.Fn pselect
-function and this manual page were written by
-.An Garrett Wollman Aq wollman@FreeBSD.org .
diff --git a/lib/libc/gen/pselect.c b/lib/libc/gen/pselect.c
deleted file mode 100644
index 28066a2..0000000
--- a/lib/libc/gen/pselect.c
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Copyright 2000 Massachusetts Institute of Technology
- *
- * Permission to use, copy, modify, and distribute this software and
- * its documentation for any purpose and without fee is hereby
- * granted, provided that both the above copyright notice and this
- * permission notice appear in all copies, that both the above
- * copyright notice and this permission notice appear in all
- * supporting documentation, and that the name of M.I.T. not be used
- * in advertising or publicity pertaining to distribution of the
- * software without specific, written prior permission.  M.I.T. makes
- * no representations about the suitability of this software for any
- * purpose.  It is provided "as is" without express or implied
- * warranty.
- *
- * THIS SOFTWARE IS PROVIDED BY M.I.T. ``AS IS''.  M.I.T. DISCLAIMS
- * ALL EXPRESS OR IMPLIED WARRANTIES WITH REGARD TO THIS SOFTWARE,
- * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. IN NO EVENT
- * SHALL M.I.T. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
- * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include "namespace.h"
-#include <sys/select.h>
-#include <sys/time.h>
-
-#include <errno.h>
-#include <signal.h>
-#include "un-namespace.h"
-
-__weak_reference(__pselect, pselect);
-
-/*
- * Emulate the POSIX 1003.1g-2000 `pselect' interface.  This is the
- * same as the traditional BSD `select' function, except that it uses
- * a timespec rather than a timeval, doesn't modify the timeout argument,
- * and allows the user to specify a signal mask to apply during the select.
- */
-int
-__pselect(int count, fd_set * __restrict rfds, fd_set * __restrict wfds, 
-	fd_set * __restrict efds, const struct timespec * __restrict timo, 
-	const sigset_t * __restrict mask)
-{
-	sigset_t omask;
-	struct timeval tvtimo, *tvp;
-	int rv, sverrno;
-
-	if (timo) {
-		TIMESPEC_TO_TIMEVAL(&tvtimo, timo);
-		tvp = &tvtimo;
-	} else
-		tvp = 0;
-
-	if (mask != 0) {
-		rv = _sigprocmask(SIG_SETMASK, mask, &omask);
-		if (rv != 0)
-			return rv;
-	}
-
-	rv = _select(count, rfds, wfds, efds, tvp);
-	if (mask != 0) {
-		sverrno = errno;
-		_sigprocmask(SIG_SETMASK, &omask, (sigset_t *)0);
-		errno = sverrno;
-	}
-
-	return rv;
-}
diff --git a/lib/libc/sys/Makefile.inc b/lib/libc/sys/Makefile.inc
index 1e6059d..1915c55 100644
--- a/lib/libc/sys/Makefile.inc
+++ b/lib/libc/sys/Makefile.inc
@@ -83,7 +83,8 @@ MAN+=	abort2.2 accept.2 access.2 acct.2 adjtime.2 \
 	mq_setattr.2 \
 	msgctl.2 msgget.2 msgrcv.2 msgsnd.2 \
 	msync.2 munmap.2 nanosleep.2 nfssvc.2 ntp_adjtime.2 open.2 \
-	pathconf.2 pipe.2 poll.2 posix_openpt.2 profil.2 ptrace.2 quotactl.2 \
+	pathconf.2 pipe.2 poll.2 posix_openpt.2 profil.2 \
+	pselect.2 ptrace.2 quotactl.2 \
 	read.2 readlink.2 reboot.2 recv.2 rename.2 revoke.2 rfork.2 rmdir.2 \
 	rtprio.2
 .if !defined(NO_P1003_1B)
diff --git a/lib/libc/sys/Symbol.map b/lib/libc/sys/Symbol.map
index 56d8aaa..7ff42da 100644
--- a/lib/libc/sys/Symbol.map
+++ b/lib/libc/sys/Symbol.map
@@ -211,6 +211,7 @@ FBSD_1.0 {
 	posix_openpt;
 	preadv;
 	profil;
+	pselect;
 	ptrace;
 	pwritev;
 	quotactl;
@@ -779,6 +780,8 @@ FBSDprivate_1.0 {
 	__sys_preadv;
 	_profil;
 	__sys_profil;
+	_pselect;
+	__sys_pselect;
 	_ptrace;
 	__sys_ptrace;
 	_pwritev;
diff --git a/lib/libc/sys/pselect.2 b/lib/libc/sys/pselect.2
new file mode 100644
index 0000000..cf784a5
--- /dev/null
+++ b/lib/libc/sys/pselect.2
@@ -0,0 +1,122 @@
+.\"
+.\" Copyright 2002 Massachusetts Institute of Technology
+.\"
+.\" Permission to use, copy, modify, and distribute this software and
+.\" its documentation for any purpose and without fee is hereby
+.\" granted, provided that both the above copyright notice and this
+.\" permission notice appear in all copies, that both the above
+.\" copyright notice and this permission notice appear in all
+.\" supporting documentation, and that the name of M.I.T. not be used
+.\" in advertising or publicity pertaining to distribution of the
+.\" software without specific, written prior permission.  M.I.T. makes
+.\" no representations about the suitability of this software for any
+.\" purpose.  It is provided "as is" without express or implied
+.\" warranty.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY M.I.T. ``AS IS''.  M.I.T. DISCLAIMS
+.\" ALL EXPRESS OR IMPLIED WARRANTIES WITH REGARD TO THIS SOFTWARE,
+.\" INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+.\" MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. IN NO EVENT
+.\" SHALL M.I.T. BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+.\" SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+.\" LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+.\" USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+.\" ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+.\" OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+.\" OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd October 27, 2009
+.Dt PSELECT 2
+.Os
+.Sh NAME
+.Nm pselect
+.Nd synchronous I/O multiplexing a la POSIX.1g
+.Sh LIBRARY
+.Lb libc
+.Sh SYNOPSIS
+.In sys/select.h
+.Ft int
+.Fo pselect
+.Fa "int nfds"
+.Fa "fd_set * restrict readfds"
+.Fa "fd_set * restrict writefds"
+.Fa "fd_set * restrict exceptfds"
+.Fa "const struct timespec * restrict timeout"
+.Fa "const sigset_t * restrict newsigmask"
+.Fc
+.Sh DESCRIPTION
+The
+.Fn pselect
+function was introduced by
+.St -p1003.1g-2000
+as a slightly stronger version of
+.Xr select 2 .
+The
+.Fa nfds , readfds , writefds ,
+and
+.Fa exceptfds
+arguments are all identical to the analogous arguments of
+.Fn select .
+The
+.Fa timeout
+argument in
+.Fn pselect
+points to a
+.Vt "const struct timespec"
+rather than the (modifiable)
+.Vt "struct timeval"
+used by
+.Fn select ;
+as in
+.Fn select ,
+a null pointer may be passed to indicate that
+.Fn pselect
+should wait indefinitely.
+Finally,
+.Fa newsigmask
+specifies a signal mask which is set while waiting for input.
+When
+.Fn pselect
+returns, the original signal mask is restored.
+.Pp
+See
+.Xr select 2
+for a more detailed discussion of the semantics of this interface, and
+for macros used to manipulate the
+.Vt "fd_set"
+data type.
+.Sh RETURN VALUES
+The
+.Fn pselect
+function returns the same values and under the same conditions as
+.Fn select .
+.Sh ERRORS
+The
+.Fn pselect
+function may fail for any of the reasons documented for
+.Xr select 2
+and (if a signal mask is provided)
+.Xr sigprocmask 2 .
+.Sh SEE ALSO
+.Xr kqueue 2 ,
+.Xr poll 2 ,
+.Xr select 2 ,
+.Xr sigprocmask 2
+.Sh STANDARDS
+The
+.Fn pselect
+function conforms to
+.St -p1003.1-2001 .
+.Sh HISTORY
+The
+.Fn pselect
+function first appeared in
+.Fx 5.0 .
+.Sh AUTHORS
+The first implementation of
+.Fn pselect
+function and this manual page were written by
+.An Garrett Wollman Aq wollman@FreeBSD.org .
diff --git a/lib/libthr/thread/thr_syscalls.c b/lib/libthr/thread/thr_syscalls.c
index 698c0c3..31af6fa 100644
--- a/lib/libthr/thread/thr_syscalls.c
+++ b/lib/libthr/thread/thr_syscalls.c
@@ -104,6 +104,8 @@ extern int	__sys_accept(int, struct sockaddr *, socklen_t *);
 extern int	__sys_connect(int, const struct sockaddr *, socklen_t);
 extern int	__sys_fsync(int);
 extern int	__sys_msync(void *, size_t, int);
+extern int	__sys_pselect(int, fd_set *, fd_set *, fd_set *,
+			const struct timespec *, const sigset_t *);
 extern int	__sys_poll(struct pollfd *, unsigned, int);
 extern ssize_t	__sys_recv(int, void *, size_t, int);
 extern ssize_t	__sys_recvfrom(int, void *, size_t, int, struct sockaddr *, socklen_t *);
@@ -366,7 +368,7 @@ ___pselect(int count, fd_set *rfds, fd_set *wfds, fd_set *efds,
 	int ret;
 
 	_thr_cancel_enter(curthread);
-	ret = __pselect(count, rfds, wfds, efds, timo, mask);
+	ret = __sys_pselect(count, rfds, wfds, efds, timo, mask);
 	_thr_cancel_leave(curthread);
 
 	return (ret);
-- 
cgit v1.1


From 18b2bd4c02f62a2b46ccb3eb6aa402d7f2368f40 Mon Sep 17 00:00:00 2001
From: attilio <attilio@FreeBSD.org>
Date: Sat, 19 Dec 2009 19:22:09 +0000
Subject: MFC r199801: Fix a socket leak.

Sponsored by:	Sandvine Incorporated
---
 lib/libfetch/ftp.c | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/libfetch/ftp.c b/lib/libfetch/ftp.c
index d0d597f..a0ba510 100644
--- a/lib/libfetch/ftp.c
+++ b/lib/libfetch/ftp.c
@@ -1122,17 +1122,19 @@ ftp_request(struct url *url, const char *op, struct url_stat *us,
 
 	/* change directory */
 	if (ftp_cwd(conn, url->doc) == -1)
-		return (NULL);
+		goto errsock;
 
 	/* stat file */
 	if (us && ftp_stat(conn, url->doc, us) == -1
 	    && fetchLastErrCode != FETCH_PROTO
 	    && fetchLastErrCode != FETCH_UNAVAIL)
-		return (NULL);
+		goto errsock;
 
 	/* just a stat */
-	if (strcmp(op, "STAT") == 0)
+	if (strcmp(op, "STAT") == 0) {
+		ftp_disconnect(conn);
 		return (FILE *)1; /* bogus return value */
+	}
 	if (strcmp(op, "STOR") == 0 || strcmp(op, "APPE") == 0)
 		oflag = O_WRONLY;
 	else
@@ -1140,6 +1142,10 @@ ftp_request(struct url *url, const char *op, struct url_stat *us,
 
 	/* initiate the transfer */
 	return (ftp_transfer(conn, op, url->doc, oflag, url->offset, flags));
+
+errsock:
+	ftp_disconnect(conn);
+	return (NULL);
 }
 
 /*
-- 
cgit v1.1


From ed4791ae105cf5fe3f9807963e17806048d19686 Mon Sep 17 00:00:00 2001
From: attilio <attilio@FreeBSD.org>
Date: Sat, 19 Dec 2009 19:23:25 +0000
Subject: MFC r199802: Fix handling of empty attributes.

Sponsored by:	Sandvine Incorporated
---
 lib/libtacplus/taclib.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libtacplus/taclib.c b/lib/libtacplus/taclib.c
index 6ac3c72..a44e834 100644
--- a/lib/libtacplus/taclib.c
+++ b/lib/libtacplus/taclib.c
@@ -1263,8 +1263,13 @@ tac_get_av_value(struct tac_handle *h, const char *attribute)
 			 *     h->srvr_avs[0] = "foobie=var1"
 			 *     h->srvr_avs[1] = "foo=var2"
 			 * is handled.
+			 *
+			 * Note that for empty string attribute values a
+			 * 0-length string is returned in order to distinguish
+			 * against unset values.
+			 * dump_str() will handle srvr.len == 0 correctly.
 			 */
-			if (found_seperator == 1 && ch != end) {
+			if (found_seperator == 1) {
 				srvr.len = end - ch;
 				srvr.data = ch;
 				return dup_str(h, &srvr, NULL);
-- 
cgit v1.1


From 436b85c22382d16f0163e9e7bbc2ba2faa79e532 Mon Sep 17 00:00:00 2001
From: jamie <jamie@FreeBSD.org>
Date: Sun, 20 Dec 2009 04:49:29 +0000
Subject: MFC r200623:

  Add a null pointer check so "name" can be used as a key parameter in
  jailparam_get.

PR:		bin/141692
Submitted by:	delphij
---
 lib/libjail/jail.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libjail/jail.c b/lib/libjail/jail.c
index 9411b88..39c88d0 100644
--- a/lib/libjail/jail.c
+++ b/lib/libjail/jail.c
@@ -532,7 +532,7 @@ jailparam_get(struct jailparam *jp, unsigned njp, int flags)
 	}
 	jp_key = jp_lastjid ? jp_lastjid :
 	    jp_jid && jp_jid->jp_valuelen == sizeof(int) &&
-	    *(int *)jp_jid->jp_value ? jp_jid : jp_name;
+	    jp_jid->jp_value && *(int *)jp_jid->jp_value ? jp_jid : jp_name;
 	if (jp_key == NULL || jp_key->jp_value == NULL) {
 		strlcpy(jail_errmsg, "no jail specified", JAIL_ERRMSGLEN);
 		errno = ENOENT;
-- 
cgit v1.1


From defe73f9cee2bf25885efe6f31614f4947dcdeca Mon Sep 17 00:00:00 2001
From: scf <scf@FreeBSD.org>
Date: Sun, 20 Dec 2009 18:57:43 +0000
Subject: Merge from head to stable/8:

r200424:
Fix libusb_open_device_with_vid_pid() to return a NULL if no device is
found instead of the last device in its search list.

Reviewed by:	thompsa
---
 lib/libusb/libusb10.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/libusb/libusb10.c b/lib/libusb/libusb10.c
index 9a5154e..25520d2 100644
--- a/lib/libusb/libusb10.c
+++ b/lib/libusb/libusb10.c
@@ -379,8 +379,6 @@ libusb_open_device_with_vid_pid(libusb_context *ctx, uint16_t vendor_id,
 	if ((i = libusb_get_device_list(ctx, &devs)) < 0)
 		return (NULL);
 
-	pdev = NULL;
-
 	for (j = 0; j < i; j++) {
 		pdev = devs[j]->os_priv;
 		pdesc = libusb20_dev_get_device_desc(pdev);
@@ -396,6 +394,8 @@ libusb_open_device_with_vid_pid(libusb_context *ctx, uint16_t vendor_id,
 			break;
 		}
 	}
+	if (j == i)
+		pdev = NULL;
 
 	libusb_free_device_list(devs, 1);
 	DPRINTF(ctx, LIBUSB_DEBUG_FUNCTION, "libusb_open_device_width_vid_pid leave");
-- 
cgit v1.1


From 745ca4f2c407635e1681257b0f599c231a47b48b Mon Sep 17 00:00:00 2001
From: jilles <jilles@FreeBSD.org>
Date: Tue, 22 Dec 2009 13:45:29 +0000
Subject: MFC r200589: cpuset(2): fix a typo and a markup error in the man page

---
 lib/libc/sys/cpuset.2 | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/sys/cpuset.2 b/lib/libc/sys/cpuset.2
index c07f8d9..fb9a770 100644
--- a/lib/libc/sys/cpuset.2
+++ b/lib/libc/sys/cpuset.2
@@ -96,7 +96,7 @@ The
 .Fa which
 argument may have the following values:
 .Bl -column CPU_WHICH_CPUSET -offset indent
-.It Dv CPU_WHICH_TID Ta "id is lpwid_t (thread id)"
+.It Dv CPU_WHICH_TID Ta "id is lwpid_t (thread id)"
 .It Dv CPU_WHICH_PID Ta "id is pid_t (process id)"
 .It Dv CPU_WHICH_CPUSET Ta "id is a cpusetid_t (cpuset id)"
 .It Dv CPU_WHICH_IRQ Ta "id is an irq number"
@@ -209,7 +209,7 @@ The calling process did not have the credentials required to complete the
 operation.
 .It Bq Er ENFILE
 There was no free
-.Fn cpusetid_t
+.Ft cpusetid_t
 for allocation.
 .El
 .Sh SEE ALSO
-- 
cgit v1.1


From d74cd5eae6166cb23de1cf9489b847350cd30f7e Mon Sep 17 00:00:00 2001
From: roam <roam@FreeBSD.org>
Date: Thu, 24 Dec 2009 20:35:01 +0000
Subject: MFC r199244:   Fix the grammar in the isgraph(3) description, almost
 as per the PR.

  PR:           140455
  Submitted by: Jeremy Huddleston <jeremyhu@apple.com>
---
 lib/libc/locale/isgraph.3 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libc/locale/isgraph.3 b/lib/libc/locale/isgraph.3
index 4ba78e3..5e06e2e 100644
--- a/lib/libc/locale/isgraph.3
+++ b/lib/libc/locale/isgraph.3
@@ -50,7 +50,7 @@ The
 function tests for any printing character except space
 .Pq Ql "\ "
 and other
-locale specific space-like characters.
+locale-specific space-like characters.
 The value of the argument must be representable as an
 .Vt "unsigned char"
 or the value of
-- 
cgit v1.1


From 7ae7e0a261d5953e772cc84a9c78e469c005ba50 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Sun, 27 Dec 2009 07:04:27 +0000
Subject: Explicitly say that this is an internal library which is intended to
 be used within FreeBSD base system only, and discourage user applications
 from using it.  User applications should use the expat version from the
 ports/package collection.

Reviewed by:	simon (earlier version)
---
 lib/libexpat/libbsdxml.3 | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libexpat/libbsdxml.3 b/lib/libexpat/libbsdxml.3
index 7d7b0e7..f687518 100644
--- a/lib/libexpat/libbsdxml.3
+++ b/lib/libexpat/libbsdxml.3
@@ -25,7 +25,7 @@
 .\"
 .\" $FreeBSD$
 .\"/
-.Dd May 5, 2008
+.Dd December 12, 2009
 .Dt LIBBSDXML 3
 .Os
 .Sh NAME
@@ -38,6 +38,15 @@ The
 .Nm
 library is a verbatim copy of the eXpat XML library version 2.0.1.
 .Pp
+The
+.Nm
+library is intended to use within the
+.Fx
+base system only.
+Use of the
+.Nm
+library for other purposes is not supported and discouraged.
+.Pp
 To avoid version and autoconfiguration issues, the library has been
 renamed to
 .Nm
-- 
cgit v1.1


From 983a441bd6cee6cbccd5c4982ba22e6982f24e2c Mon Sep 17 00:00:00 2001
From: kib <kib@FreeBSD.org>
Date: Sun, 27 Dec 2009 20:39:58 +0000
Subject: MFC r200038: Properly support -fPIE by linking PIE binaries with
 specially-built Scrt1.o instead of crt1.o, since the later is built as
 non-PIC.

Separate i386-elf crt1.c into the pure assembler part and C code,
supplying all data extracted by assembler stub as explicit parameters.
Hide and localize _start1 symbol used as an interface between asm and
C code.
---
 lib/csu/amd64/Makefile    |   5 +-
 lib/csu/arm/Makefile      |   5 +-
 lib/csu/i386-elf/Makefile |  24 ++++++++--
 lib/csu/i386-elf/crt1.c   | 113 ----------------------------------------------
 lib/csu/i386-elf/crt1_c.c |  95 ++++++++++++++++++++++++++++++++++++++
 lib/csu/i386-elf/crt1_s.S |  44 ++++++++++++++++++
 lib/csu/ia64/Makefile     |   5 +-
 lib/csu/mips/Makefile     |   5 +-
 lib/csu/powerpc/Makefile  |   5 +-
 lib/csu/sparc64/Makefile  |   5 +-
 10 files changed, 182 insertions(+), 124 deletions(-)
 delete mode 100644 lib/csu/i386-elf/crt1.c
 create mode 100644 lib/csu/i386-elf/crt1_c.c
 create mode 100644 lib/csu/i386-elf/crt1_s.S

(limited to 'lib')

diff --git a/lib/csu/amd64/Makefile b/lib/csu/amd64/Makefile
index 71ccd67..1a74efc 100644
--- a/lib/csu/amd64/Makefile
+++ b/lib/csu/amd64/Makefile
@@ -4,7 +4,7 @@
 
 SRCS=		crt1.c crti.S crtn.S
 OBJS=		${SRCS:N*.h:R:S/$/.o/g}
-OBJS+=		gcrt1.o
+OBJS+=		Scrt1.o gcrt1.o
 CFLAGS+=	-I${.CURDIR}/../common \
 		-I${.CURDIR}/../../libc/include
 CFLAGS+=	-fno-omit-frame-pointer
@@ -16,6 +16,9 @@ CLEANFILES=	${OBJS}
 gcrt1.o: crt1.c
 	${CC} ${CFLAGS} -DGCRT -c -o gcrt1.o ${.CURDIR}/crt1.c
 
+Scrt1.o: crt1.c
+	${CC} ${CFLAGS} -fPIC -DPIC -c -o Scrt1.o ${.CURDIR}/crt1.c
+
 realinstall:
 	${INSTALL} -o ${LIBOWN} -g ${LIBGRP} -m ${LIBMODE} \
 	    ${OBJS} ${DESTDIR}${LIBDIR}
diff --git a/lib/csu/arm/Makefile b/lib/csu/arm/Makefile
index 23954e8..097f82d 100644
--- a/lib/csu/arm/Makefile
+++ b/lib/csu/arm/Makefile
@@ -4,7 +4,7 @@
 
 SRCS=		crt1.c crti.S crtn.S
 OBJS=		${SRCS:N*.h:R:S/$/.o/g}
-OBJS+=		gcrt1.o
+OBJS+=		Scrt1.o gcrt1.o
 CFLAGS+=	-Wall -Wno-unused \
 		-I${.CURDIR}/../common \
 		-I${.CURDIR}/../../libc/include
@@ -16,6 +16,9 @@ CLEANFILES=	${OBJS}
 gcrt1.o: crt1.c
 	${CC} ${CFLAGS} -DGCRT -c -o gcrt1.o ${.ALLSRC}
 
+Scrt1.o: crt1.c
+	${CC} ${CFLAGS} -fPIC -DPIC -c -o Scrt1.o ${.ALLSRC}
+
 realinstall:
 	${INSTALL} -o ${LIBOWN} -g ${LIBGRP} -m ${LIBMODE} \
 	    ${OBJS} ${DESTDIR}${LIBDIR}
diff --git a/lib/csu/i386-elf/Makefile b/lib/csu/i386-elf/Makefile
index 8598ce8..c2af118 100644
--- a/lib/csu/i386-elf/Makefile
+++ b/lib/csu/i386-elf/Makefile
@@ -2,8 +2,8 @@
 
 .PATH: ${.CURDIR}/../common
 
-SRCS=		crt1.c crti.S crtn.S
-FILES=		${SRCS:N*.h:R:S/$/.o/g} gcrt1.o
+SRCS=		crti.S crtn.S
+FILES=		${SRCS:N*.h:R:S/$/.o/g} gcrt1.o crt1.o Scrt1.o
 FILESOWN=	${LIBOWN}
 FILESGRP=	${LIBGRP}
 FILESMODE=	${LIBMODE}
@@ -11,9 +11,23 @@ FILESDIR=	${LIBDIR}
 WARNS?=		6
 CFLAGS+=	-I${.CURDIR}/../common \
 		-I${.CURDIR}/../../libc/include
-CLEANFILES=	${FILES}
+CLEANFILES=	${FILES} crt1_c.o crt1_s.o gcrt1_c.o Scrt1_c.o
 
-gcrt1.o: crt1.c
-	${CC} ${CFLAGS} -DGCRT -c -o gcrt1.o ${.CURDIR}/crt1.c
+gcrt1_c.o: crt1_c.c
+	${CC} ${CFLAGS} -DGCRT -c -o gcrt1_c.o ${.CURDIR}/crt1_c.c
+
+gcrt1.o: gcrt1_c.o crt1_s.o
+	${LD} ${LDFLAGS} -o gcrt1.o -r crt1_s.o gcrt1_c.o
+
+crt1.o:	crt1_c.o crt1_s.o
+	${LD} ${LDFLAGS} -o crt1.o -r crt1_s.o crt1_c.o
+	objcopy --localize-symbol _start1 crt1.o
+
+Scrt1_c.o:	crt1_c.c
+	${CC} ${CFLAGS} -DGCRT -fPIC -DPIC -c -o Scrt1_c.o ${.CURDIR}/crt1_c.c
+
+Scrt1.o: Scrt1_c.o crt1_s.o
+	${LD} ${LDFLAGS} -o Scrt1.o -r crt1_s.o Scrt1_c.o
+	objcopy --localize-symbol _start1 Scrt1.o
 
 .include <bsd.prog.mk>
diff --git a/lib/csu/i386-elf/crt1.c b/lib/csu/i386-elf/crt1.c
deleted file mode 100644
index 0934333..0000000
--- a/lib/csu/i386-elf/crt1.c
+++ /dev/null
@@ -1,113 +0,0 @@
-/* LINTLIBRARY */
-/*-
- * Copyright 1996-1998 John D. Polstra.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
- * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
- * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef lint
-#ifndef __GNUC__
-#error "GCC is needed to compile this file"
-#endif
-#endif /* lint */
-
-#include <stdlib.h>
-
-#include "libc_private.h"
-#include "crtbrand.c"
-
-extern int _DYNAMIC;
-#pragma weak _DYNAMIC
-
-typedef void (*fptr)(void);
-
-extern void _fini(void);
-extern void _init(void);
-extern int main(int, char **, char **);
-extern void _start(char *, ...);
-
-#ifdef GCRT
-extern void _mcleanup(void);
-extern void monstartup(void *, void *);
-extern int eprol;
-extern int etext;
-#endif
-
-char **environ;
-const char *__progname = "";
-
-static __inline fptr
-get_rtld_cleanup(void)
-{
-	fptr retval;
-
-#ifdef	__GNUC__
-	__asm__("movl %%edx,%0" : "=rm"(retval));
-#else
-	retval = (fptr)0; /* XXXX Fix this for other compilers */
-#endif
-	return(retval);
-}
-
-/* The entry function. */
-void
-_start(char *ap, ...)
-{
-	fptr cleanup;
-	int argc;
-	char **argv;
-	char **env;
-	const char *s;
-
-#ifdef __GNUC__
-	__asm__("and $0xfffffff0,%esp");
-#endif
-	cleanup = get_rtld_cleanup();
-	argv = &ap;
-	argc = *(long *)(void *)(argv - 1);
-	env = argv + argc + 1;
-	environ = env;
-	if (argc > 0 && argv[0] != NULL) {
-		__progname = argv[0];
-		for (s = __progname; *s != '\0'; s++)
-			if (*s == '/')
-				__progname = s + 1;
-	}
-
-	if (&_DYNAMIC != NULL)
-		atexit(cleanup);
-	else
-		_init_tls();
-
-#ifdef GCRT
-	atexit(_mcleanup);
-#endif
-	atexit(_fini);
-#ifdef GCRT
-	monstartup(&eprol, &etext);
-__asm__("eprol:");
-#endif
-	_init();
-	exit( main(argc, argv, env) );
-}
-
-__asm__(".ident\t\"$FreeBSD$\"");
diff --git a/lib/csu/i386-elf/crt1_c.c b/lib/csu/i386-elf/crt1_c.c
new file mode 100644
index 0000000..c38f267
--- /dev/null
+++ b/lib/csu/i386-elf/crt1_c.c
@@ -0,0 +1,95 @@
+/* LINTLIBRARY */
+/*-
+ * Copyright 1996-1998 John D. Polstra.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef lint
+#ifndef __GNUC__
+#error "GCC is needed to compile this file"
+#endif
+#endif /* lint */
+
+#include <stdlib.h>
+
+#include "libc_private.h"
+#include "crtbrand.c"
+
+extern int _DYNAMIC;
+#pragma weak _DYNAMIC
+
+typedef void (*fptr)(void);
+
+extern void _fini(void);
+extern void _init(void);
+extern int main(int, char **, char **);
+extern void _start(char *, ...);
+
+#ifdef GCRT
+extern void _mcleanup(void);
+extern void monstartup(void *, void *);
+extern int eprol;
+extern int etext;
+#endif
+
+char **environ;
+const char *__progname = "";
+
+void _start1(fptr, int, char *[]) __dead2;
+
+/* The entry function, C part. */
+void
+_start1(fptr cleanup, int argc, char *argv[])
+{
+	char **env;
+	const char *s;
+
+	env = argv + argc + 1;
+	environ = env;
+	if (argc > 0 && argv[0] != NULL) {
+		__progname = argv[0];
+		for (s = __progname; *s != '\0'; s++)
+			if (*s == '/')
+				__progname = s + 1;
+	}
+
+	if (&_DYNAMIC != NULL)
+		atexit(cleanup);
+	else
+		_init_tls();
+
+#ifdef GCRT
+	atexit(_mcleanup);
+#endif
+	atexit(_fini);
+#ifdef GCRT
+	monstartup(&eprol, &etext);
+__asm__("eprol:");
+#endif
+	_init();
+	exit( main(argc, argv, env) );
+}
+
+__asm(".hidden	_start1");
diff --git a/lib/csu/i386-elf/crt1_s.S b/lib/csu/i386-elf/crt1_s.S
new file mode 100644
index 0000000..2af8a1b
--- /dev/null
+++ b/lib/csu/i386-elf/crt1_s.S
@@ -0,0 +1,44 @@
+/*-
+ * Copyright 2009 Konstantin Belousov.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+
+	.text
+	.align	4
+	.globl	_start
+	.type	_start, @function
+_start:	xorl	%ebp,%ebp
+	pushl	%ebp
+	movl	%esp,%ebp
+	andl	$0xfffffff0,%esp # align stack
+	leal	8(%ebp),%eax
+	pushl	%eax		# argv
+	pushl	4(%ebp)		# argc
+	pushl	%edx		# rtld cleanup
+	call	_start1
+	.size	_start, . - _start
+
+	.ident	"$FreeBSD$"
diff --git a/lib/csu/ia64/Makefile b/lib/csu/ia64/Makefile
index c906c09..d795103 100644
--- a/lib/csu/ia64/Makefile
+++ b/lib/csu/ia64/Makefile
@@ -4,7 +4,7 @@
 
 SRCS=		crt1.S crti.S crtn.S
 OBJS=		${SRCS:N*.h:R:S/$/.o/g}
-OBJS+=		gcrt1.o
+OBJS+=		Scrt1.o gcrt1.o
 CFLAGS+=	-Wall -Wno-unused \
 		-I${.CURDIR}/../common \
 		-I${.CURDIR}/../../libc/include
@@ -16,6 +16,9 @@ CLEANFILES=	${OBJS}
 gcrt1.o: crt1.S
 	${CC} ${CFLAGS} -DGCRT -c -o gcrt1.o ${.ALLSRC}
 
+Scrt1.o: crt1.S
+	${CC} ${CFLAGS} -fPIC -DPIC -c -o Scrt1.o ${.ALLSRC}
+
 realinstall:
 	${INSTALL} -o ${LIBOWN} -g ${LIBGRP} -m ${LIBMODE} \
 	    ${OBJS} ${DESTDIR}${LIBDIR}
diff --git a/lib/csu/mips/Makefile b/lib/csu/mips/Makefile
index 23954e8..097f82d 100644
--- a/lib/csu/mips/Makefile
+++ b/lib/csu/mips/Makefile
@@ -4,7 +4,7 @@
 
 SRCS=		crt1.c crti.S crtn.S
 OBJS=		${SRCS:N*.h:R:S/$/.o/g}
-OBJS+=		gcrt1.o
+OBJS+=		Scrt1.o gcrt1.o
 CFLAGS+=	-Wall -Wno-unused \
 		-I${.CURDIR}/../common \
 		-I${.CURDIR}/../../libc/include
@@ -16,6 +16,9 @@ CLEANFILES=	${OBJS}
 gcrt1.o: crt1.c
 	${CC} ${CFLAGS} -DGCRT -c -o gcrt1.o ${.ALLSRC}
 
+Scrt1.o: crt1.c
+	${CC} ${CFLAGS} -fPIC -DPIC -c -o Scrt1.o ${.ALLSRC}
+
 realinstall:
 	${INSTALL} -o ${LIBOWN} -g ${LIBGRP} -m ${LIBMODE} \
 	    ${OBJS} ${DESTDIR}${LIBDIR}
diff --git a/lib/csu/powerpc/Makefile b/lib/csu/powerpc/Makefile
index 23954e8..097f82d 100644
--- a/lib/csu/powerpc/Makefile
+++ b/lib/csu/powerpc/Makefile
@@ -4,7 +4,7 @@
 
 SRCS=		crt1.c crti.S crtn.S
 OBJS=		${SRCS:N*.h:R:S/$/.o/g}
-OBJS+=		gcrt1.o
+OBJS+=		Scrt1.o gcrt1.o
 CFLAGS+=	-Wall -Wno-unused \
 		-I${.CURDIR}/../common \
 		-I${.CURDIR}/../../libc/include
@@ -16,6 +16,9 @@ CLEANFILES=	${OBJS}
 gcrt1.o: crt1.c
 	${CC} ${CFLAGS} -DGCRT -c -o gcrt1.o ${.ALLSRC}
 
+Scrt1.o: crt1.c
+	${CC} ${CFLAGS} -fPIC -DPIC -c -o Scrt1.o ${.ALLSRC}
+
 realinstall:
 	${INSTALL} -o ${LIBOWN} -g ${LIBGRP} -m ${LIBMODE} \
 	    ${OBJS} ${DESTDIR}${LIBDIR}
diff --git a/lib/csu/sparc64/Makefile b/lib/csu/sparc64/Makefile
index 2e1d03f..7f8dd7a 100644
--- a/lib/csu/sparc64/Makefile
+++ b/lib/csu/sparc64/Makefile
@@ -4,7 +4,7 @@
 
 SRCS=		crt1.c crti.S crtn.S
 OBJS=		${SRCS:N*.h:R:S/$/.o/g}
-OBJS+=		gcrt1.o
+OBJS+=		Scrt1.o gcrt1.o
 CFLAGS+=	-I${.CURDIR}/../common -I${.CURDIR}/../../libc/include
 
 all: ${OBJS}
@@ -14,6 +14,9 @@ CLEANFILES=	${OBJS}
 gcrt1.o: crt1.c
 	${CC} ${CFLAGS} -DGCRT -c -o gcrt1.o ${.ALLSRC}
 
+Scrt1.o: crt1.c
+	${CC} ${CFLAGS} -fPIC -DPIC -c -o Scrt1.o ${.ALLSRC}
+
 realinstall:
 	${INSTALL} -o ${LIBOWN} -g ${LIBGRP} -m ${LIBMODE} \
 	    ${OBJS} ${DESTDIR}${LIBDIR}
-- 
cgit v1.1


From 145c1109517ba8110c826af4a7bfcc913d13e413 Mon Sep 17 00:00:00 2001
From: edwin <edwin@FreeBSD.org>
Date: Tue, 29 Dec 2009 10:05:20 +0000
Subject: MFC of r200832, tzdata2009t

zic:
- Fix URL / reference to Calendrical Calculations: Third Edition

libc/stdtime:
- Fix typo in tzfile.5 (no changes in our part)
---
 lib/libc/stdtime/tzfile.5 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libc/stdtime/tzfile.5 b/lib/libc/stdtime/tzfile.5
index 15625d2..1606b5a 100644
--- a/lib/libc/stdtime/tzfile.5
+++ b/lib/libc/stdtime/tzfile.5
@@ -147,6 +147,6 @@ such instants).
 .Xr ctime 3 ,
 .Xr time2posix 3 ,
 .Xr zic 8
-.\" @(#)tzfile.5	8.2
+.\" @(#)tzfile.5	8.3
 .\" This file is in the public domain, so clarified as of
 .\" 1996-06-05 by Arthur David Olson.
-- 
cgit v1.1


From 367048d38ba2f1d8fe8787015b16a496acc2b8e6 Mon Sep 17 00:00:00 2001
From: markus <markus@FreeBSD.org>
Date: Wed, 30 Dec 2009 16:37:58 +0000
Subject: MFC r200992:

Use a local copy of entry_d for finding matches. Otherwise, if entry_d pointed
to an entry of 'acl', all ACL entries starting with entry_d would be deleted.

Approved by:	emax (mentor)
---
 lib/libc/posix1e/acl_delete_entry.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libc/posix1e/acl_delete_entry.c b/lib/libc/posix1e/acl_delete_entry.c
index 7dd60b8..09b4507 100644
--- a/lib/libc/posix1e/acl_delete_entry.c
+++ b/lib/libc/posix1e/acl_delete_entry.c
@@ -75,6 +75,7 @@ int
 acl_delete_entry(acl_t acl, acl_entry_t entry_d)
 {
 	struct acl *acl_int;
+	struct acl_entry entry_int;
 	int i, j, found = 0;
 
 	if (acl == NULL || entry_d == NULL) {
@@ -94,8 +95,12 @@ acl_delete_entry(acl_t acl, acl_entry_t entry_d)
 		errno = EINVAL;
 		return (-1);
 	}
+
+	/* Use a local copy to prevent deletion of more than this entry */
+	entry_int = *entry_d;
+
 	for (i = 0; i < acl->ats_acl.acl_cnt;) {
-		if (_entry_matches(&(acl->ats_acl.acl_entry[i]), entry_d)) {
+		if (_entry_matches(&(acl->ats_acl.acl_entry[i]), &entry_int)) {
 			/* ...shift the remaining entries... */
 			for (j = i; j < acl->ats_acl.acl_cnt - 1; ++j)
 				acl->ats_acl.acl_entry[j] =
-- 
cgit v1.1


From 10dd6a21bc57f7d7d159a278e7724d882e3edcb5 Mon Sep 17 00:00:00 2001
From: jh <jh@FreeBSD.org>
Date: Wed, 30 Dec 2009 17:22:00 +0000
Subject: MFC r199844:

Reset path name back to original correctly in fts_build() when
FTS_NOCHDIR option is used. fts_build() could strip a trailing slash
from path name in post-order visit if a path pointing to an empty
directory was given for fts_open().

PR:		bin/133907, kern/134513
Approved by:	trasz (mentor)
---
 lib/libc/gen/fts.c | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/gen/fts.c b/lib/libc/gen/fts.c
index 41443c5..88f1a23 100644
--- a/lib/libc/gen/fts.c
+++ b/lib/libc/gen/fts.c
@@ -836,11 +836,8 @@ mem1:				saved_errno = errno;
 	 * If not changing directories, reset the path back to original
 	 * state.
 	 */
-	if (ISSET(FTS_NOCHDIR)) {
-		if (len == sp->fts_pathlen || nitems == 0)
-			--cp;
-		*cp = '\0';
-	}
+	if (ISSET(FTS_NOCHDIR))
+		sp->fts_path[cur->fts_pathlen] = '\0';
 
 	/*
 	 * If descended after called from fts_children or after called from
-- 
cgit v1.1


From 0bf920218abf0c03d6f39a336eca5ff879d5c093 Mon Sep 17 00:00:00 2001
From: jhb <jhb@FreeBSD.org>
Date: Wed, 30 Dec 2009 17:53:07 +0000
Subject: MFC 200919: Fix a bug in gzipfs that prevented lseek() from working
 and add lseek() support to bzip2fs.  This fixes problems with loading
 compressed amd64 kernel modules containing debug symbols.

---
 lib/libstand/bzipfs.c | 67 +++++++++++++++++++++++++++++++++++++++++++++------
 lib/libstand/gzipfs.c | 16 ++++++------
 2 files changed, 67 insertions(+), 16 deletions(-)

(limited to 'lib')

diff --git a/lib/libstand/bzipfs.c b/lib/libstand/bzipfs.c
index 47c799f..46a151b 100644
--- a/lib/libstand/bzipfs.c
+++ b/lib/libstand/bzipfs.c
@@ -31,6 +31,7 @@ __FBSDID("$FreeBSD$");
 #ifndef REGRESSION
 #include "stand.h"
 #else
+#include <stdlib.h>
 #include <sys/errno.h>
 #include <sys/fcntl.h>
 #include <sys/types.h>
@@ -42,7 +43,7 @@ struct open_file {
 };
 #define F_READ          0x0001  /* file opened for reading */
 #define EOFFSET (ELAST+8)       /* relative seek not supported */
-static inline u_int min(u_int a, u_int b) { return (a < b ? a : b); }
+static inline u_int min(u_int a, u_int b) { return(a < b ? a : b); }
 #define panic(x, y) abort()
 #endif
 
@@ -174,6 +175,8 @@ bzf_open(const char *fname, struct open_file *f)
 
     /* Construct new name */
     bzfname = malloc(strlen(fname) + 5);
+    if (bzfname == NULL)
+	return(ENOMEM);
     sprintf(bzfname, "%s.bz2", fname);
 
     /* Try to open the compressed datafile */
@@ -195,13 +198,14 @@ bzf_open(const char *fname, struct open_file *f)
 
     /* Allocate a bz_file structure, populate it */
     bzf = malloc(sizeof(struct bz_file));
+    if (bzf == NULL)
+	return(ENOMEM);
     bzero(bzf, sizeof(struct bz_file));
     bzf->bzf_rawfd = rawfd;
 
-    /* Verify that the file is bzipped (XXX why do this afterwards?) */
+    /* Verify that the file is bzipped */
     if (check_header(bzf)) {
 	close(bzf->bzf_rawfd);
-	BZ2_bzDecompressEnd(&(bzf->bzf_bzstream));
 	free(bzf);
 	return(EFTYPE);
     }
@@ -247,7 +251,7 @@ bzf_read(struct open_file *f, void *buf, size_t size, size_t *resid)
 	if (bzf->bzf_bzstream.avail_in == 0) {		/* oops, unexpected EOF */
 	    printf("bzf_read: unexpected EOF\n");
 	    if (bzf->bzf_bzstream.avail_out == size)
-		return (EIO);
+		return(EIO);
 	    break;
 	}
 
@@ -266,6 +270,50 @@ bzf_read(struct open_file *f, void *buf, size_t size, size_t *resid)
     return(0);
 }
 
+static int
+bzf_rewind(struct open_file *f)
+{
+    struct bz_file	*bzf = (struct bz_file *)f->f_fsdata;
+    struct bz_file	*bzf_tmp;
+
+    /*
+     * Since bzip2 does not have an equivalent inflateReset function a crude
+     * one needs to be provided.  The functions all called in such a way that
+     * at any time an error occurs a role back can be done (effectively making
+     * this rewind 'atomic', either the reset occurs successfully or not at all,
+     * with no 'undefined' state happening).
+     */
+
+    /* Allocate a bz_file structure, populate it */
+    bzf_tmp = malloc(sizeof(struct bz_file));
+    if (bzf_tmp == NULL)
+	return(-1);
+    bzero(bzf_tmp, sizeof(struct bz_file));
+    bzf_tmp->bzf_rawfd = bzf->bzf_rawfd;
+
+    /* Initialise the inflation engine */
+    if (BZ2_bzDecompressInit(&(bzf_tmp->bzf_bzstream), 0, 1) != BZ_OK) {
+	free(bzf_tmp);
+	return(-1);
+    }
+
+    /* Seek back to the beginning of the file */
+    if (lseek(bzf->bzf_rawfd, 0, SEEK_SET) == -1) {
+	BZ2_bzDecompressEnd(&(bzf_tmp->bzf_bzstream));
+	free(bzf_tmp);
+	return(-1);
+    }
+
+    /* Free old bz_file data */
+    BZ2_bzDecompressEnd(&(bzf->bzf_bzstream));
+    free(bzf);
+
+    /* Use the new bz_file data */
+    f->f_fsdata = bzf_tmp;
+
+    return(0);
+}
+
 static off_t
 bzf_seek(struct open_file *f, off_t offset, int where)
 {
@@ -284,14 +332,17 @@ bzf_seek(struct open_file *f, off_t offset, int where)
 	target = -1;
     default:
 	errno = EINVAL;
-	return (-1);
+	return(-1);
     }
 
     /* Can we get there from here? */
-    if (target < bzf->bzf_bzstream.total_out_lo32) {
+    if (target < bzf->bzf_bzstream.total_out_lo32 && bzf_rewind(f) != 0) {
 	errno = EOFFSET;
 	return -1;
-    } 
+    }
+
+    /* if bzf_rewind was called then bzf has changed */
+    bzf = (struct bz_file *)f->f_fsdata;
 
     /* skip forwards if required */
     while (target > bzf->bzf_bzstream.total_out_lo32) {
@@ -301,7 +352,7 @@ bzf_seek(struct open_file *f, off_t offset, int where)
 	    return(-1);
     }
     /* This is where we are (be honest if we overshot) */
-    return (bzf->bzf_bzstream.total_out_lo32);
+    return(bzf->bzf_bzstream.total_out_lo32);
 }
 
 static int
diff --git a/lib/libstand/gzipfs.c b/lib/libstand/gzipfs.c
index 4f40a4a..9ff7985 100644
--- a/lib/libstand/gzipfs.c
+++ b/lib/libstand/gzipfs.c
@@ -212,10 +212,9 @@ zf_open(const char *fname, struct open_file *f)
     bzero(zf, sizeof(struct z_file));
     zf->zf_rawfd = rawfd;
 
-    /* Verify that the file is gzipped (XXX why do this afterwards?) */
+    /* Verify that the file is gzipped */
     if (check_header(zf)) {
 	close(zf->zf_rawfd);
-	inflateEnd(&(zf->zf_zstream));
 	free(zf);
 	return(EFTYPE);
     }
@@ -261,7 +260,7 @@ zf_read(struct open_file *f, void *buf, size_t size, size_t *resid)
 	if (zf->zf_zstream.avail_in == 0) {		/* oops, unexpected EOF */
 	    printf("zf_read: unexpected EOF\n");
 	    if (zf->zf_zstream.avail_out == size)
-		return (EIO);
+		return(EIO);
 	    break;
 	}
 
@@ -286,12 +285,13 @@ zf_rewind(struct open_file *f)
     struct z_file	*zf = (struct z_file *)f->f_fsdata;
 
     if (lseek(zf->zf_rawfd, zf->zf_dataoffset, SEEK_SET) == -1)
-	return -1;
+	return(-1);
     zf->zf_zstream.avail_in = 0;
     zf->zf_zstream.next_in = NULL;
+    zf->zf_endseen = 0;
     (void)inflateReset(&zf->zf_zstream);
 
-    return 0;
+    return(0);
 }
 
 static off_t
@@ -312,12 +312,12 @@ zf_seek(struct open_file *f, off_t offset, int where)
 	target = -1;
     default:
 	errno = EINVAL;
-	return (-1);
+	return(-1);
     }
 
     /* rewind if required */
     if (target < zf->zf_zstream.total_out && zf_rewind(f) != 0)
-	return -1;
+	return(-1);
 
     /* skip forwards if required */
     while (target > zf->zf_zstream.total_out) {
@@ -327,7 +327,7 @@ zf_seek(struct open_file *f, off_t offset, int where)
 	    return(-1);
     }
     /* This is where we are (be honest if we overshot) */
-    return (zf->zf_zstream.total_out);
+    return(zf->zf_zstream.total_out);
 }
 
 
-- 
cgit v1.1


From b587b2f652c48043cb99be49dcdeadfac5852d06 Mon Sep 17 00:00:00 2001
From: kib <kib@FreeBSD.org>
Date: Fri, 1 Jan 2010 11:43:09 +0000
Subject: MFC r201201: Document CLOCK_SECOND, add cross-reference from time(3)
 to clock_gettime(2).

MFC r201204:
Document _FAST and _PRECISE clocks.
---
 lib/libc/gen/time.3          |  1 +
 lib/libc/sys/clock_gettime.2 | 55 +++++++++++++++++++++++++++++++++++---------
 2 files changed, 45 insertions(+), 11 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/gen/time.3 b/lib/libc/gen/time.3
index 2770e49..d022e06 100644
--- a/lib/libc/gen/time.3
+++ b/lib/libc/gen/time.3
@@ -66,6 +66,7 @@ The
 function may fail for any of the reasons described in
 .Xr gettimeofday 2 .
 .Sh SEE ALSO
+.Xr clock_gettime 2 ,
 .Xr gettimeofday 2 ,
 .Xr ctime 3
 .Sh STANDARDS
diff --git a/lib/libc/sys/clock_gettime.2 b/lib/libc/sys/clock_gettime.2
index 242af1c..a2fa624 100644
--- a/lib/libc/sys/clock_gettime.2
+++ b/lib/libc/sys/clock_gettime.2
@@ -29,7 +29,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd November 4, 2006
+.Dd December 29, 2009
 .Dt CLOCK_GETTIME 2
 .Os
 .Sh NAME
@@ -59,21 +59,43 @@ used by a clock which is specified by
 The
 .Fa clock_id
 argument
-can be one of five values:
-.Dv CLOCK_REALTIME
+can be one of the following values:
+.Dv CLOCK_REALTIME ,
+.Dv CLOCK_REALTIME_PRECISE ,
+.Dv CLOCK_REALTIME_FAST
 for time that increments as
-a wall clock should,
-.Dv CLOCK_MONOTONIC
-which increments in SI seconds,
-.Dv CLOCK_UPTIME
+a wall clock should;
+.Dv CLOCK_MONOTONIC ,
+.Dv CLOCK_MONOTONIC_PRECISE ,
+.Dv CLOCK_MONOTONIC_FAST
+which increments in SI seconds;
+.Dv CLOCK_UPTIME ,
+.Dv CLOCK_UPTIME_PRECISE ,
+.Dv CLOCK_UPTIME_FAST
 which starts at zero when the kernel boots and increments
-monotonically in SI seconds while the machine is running,
+monotonically in SI seconds while the machine is running;
 .Dv CLOCK_VIRTUAL
 for time that increments only when
-the CPU is running in user mode on behalf of the calling process, or
+the CPU is running in user mode on behalf of the calling process;
 .Dv CLOCK_PROF
 for time that increments when the CPU is running in user or
-kernel mode.
+kernel mode; or
+.Dv CLOCK_SECOND
+which returns the current second without performing a full time counter
+query, using in-kernel cached value of current second.
+.Pp
+The clock IDs
+.Fa CLOCK_REALTIME_FAST ,
+.Fa CLOCK_MONOTONIC_FAST ,
+.Fa CLOCK_UPTIME_FAST
+are analogs of corresponding IDs without _FAST suffix but do not perform
+a full time counter query, so their accuracy is one timer tick.
+Similarly,
+.Fa CLOCK_REALTIME_PRECISE ,
+.Fa CLOCK_MONOTONIC_PRECISE ,
+.Fa CLOCK_UPTIME_PRECISE
+are used to get the most exact value as possible, at the expense of
+execution time.
 .Pp
 The structure pointed to by
 .Fa tp
@@ -88,7 +110,8 @@ struct timespec {
 };
 .Ed
 .Pp
-Only the super-user may set the time of day.
+Only the super-user may set the time of day, using only
+.Fa CLOCK_REALTIME .
 If the system securelevel is greater than 1 (see
 .Xr init 8 ) ,
 the time may only be advanced.
@@ -134,3 +157,13 @@ and
 .Fn clock_getres
 system calls conform to
 .St -p1003.1b-93 .
+The clock IDs
+.Fa CLOCK_REALTIME_FAST ,
+.Fa CLOCK_REALTIME_PRECISE ,
+.Fa CLOCK_MONOTONIC_FAST ,
+.Fa CLOCK_MONOTONIC_PRECISE ,
+.Fa CLOCK_UPTIME ,
+.Fa CLOCK_UPTIME_FAST ,
+.Fa CLOCK_UPTIME_PRECISE ,
+.Fa CLOCK_SECOND
+are FreeBSD extensions to the POSIX interface.
-- 
cgit v1.1


From 32f7e2fc4c57b31b1f38f481f98d31ced3c1f521 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Mon, 4 Jan 2010 01:09:59 +0000
Subject: MFC r201137:

Grammar fix.

Submitted by:	Kenyon Ralph <kenyon kenyonralph com>
---
 lib/libexpat/libbsdxml.3 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libexpat/libbsdxml.3 b/lib/libexpat/libbsdxml.3
index f687518..555bc9a 100644
--- a/lib/libexpat/libbsdxml.3
+++ b/lib/libexpat/libbsdxml.3
@@ -40,7 +40,7 @@ library is a verbatim copy of the eXpat XML library version 2.0.1.
 .Pp
 The
 .Nm
-library is intended to use within the
+library is intended to be used within the
 .Fx
 base system only.
 Use of the
-- 
cgit v1.1


From 194840d7a1fa5739749e87b5bb40df6b69dfc5d9 Mon Sep 17 00:00:00 2001
From: kib <kib@FreeBSD.org>
Date: Tue, 5 Jan 2010 12:32:09 +0000
Subject: MFC r201194: Use clock_gettime(CLOCK_SECOND) instead of
 gettimeofday(2) for implementation of time(3). CLOCK_SECOND is much faster.

---
 lib/libc/gen/time.c | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/gen/time.c b/lib/libc/gen/time.c
index 840f9b6..214dfce 100644
--- a/lib/libc/gen/time.c
+++ b/lib/libc/gen/time.c
@@ -37,13 +37,12 @@ __FBSDID("$FreeBSD$");
 #include <sys/time.h>
 
 time_t
-time(t)
-	time_t *t;
+time(time_t *t)
 {
-	struct timeval tt;
+	struct timespec tt;
 	time_t retval;
 
-	if (gettimeofday(&tt, (struct timezone *)0) < 0)
+	if (clock_gettime(CLOCK_SECOND, &tt) < 0)
 		retval = -1;
 	else
 		retval = tt.tv_sec;
-- 
cgit v1.1


From 03e26beaeb74ba217b53b89fa85857ab89184303 Mon Sep 17 00:00:00 2001
From: kib <kib@FreeBSD.org>
Date: Sun, 10 Jan 2010 11:25:34 +0000
Subject: MFC r201743: Give some information on SF_MNOWAIT flag.

MFC r201759 (by brueffer):
Fix a typo and bump date for the previous commit.

MFC r201760:
Further fix grammar.
---
 lib/libc/sys/sendfile.2 | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/sys/sendfile.2 b/lib/libc/sys/sendfile.2
index 322971f..d9f8cab 100644
--- a/lib/libc/sys/sendfile.2
+++ b/lib/libc/sys/sendfile.2
@@ -25,7 +25,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd November 24, 2006
+.Dd January 7, 2010
 .Dt SENDFILE 2
 .Os
 .Sh NAME
@@ -116,9 +116,17 @@ Busy servers may benefit by transferring requests that would
 block to a separate I/O worker thread.
 .It
 .Dv SF_MNOWAIT .
-(description missing)
+Do not wait for some kernel resource to become available,
+in particular,
+.Vt mbuf
+and
+.Vt sf_buf .
+The flag does not make the
+.Fn sendfile
+syscall truly non-blocking, since other resources are still allocated
+in a blocking fashion.
 .It
-.Dv SF_SYNC ,
+.Dv SF_SYNC .
 .Nm
 sleeps until the network stack no longer references the VM pages
 of the file, making subsequent modifications to it safe.
-- 
cgit v1.1


From 50f4dd01c42bf5858416f8e7e567e08596d79846 Mon Sep 17 00:00:00 2001
From: jhb <jhb@FreeBSD.org>
Date: Wed, 13 Jan 2010 18:12:21 +0000
Subject: MFC 199607, 200797, 201270, 201669: Use pthread_once() to initialize
 the thread-local storage for localtime() and gmtime() and _once() to
 initialize gmt state rather than home-rolled versions using pthread mutex
 locks.

---
 lib/libc/stdtime/localtime.c | 81 ++++++++++++++++++++++----------------------
 1 file changed, 41 insertions(+), 40 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/stdtime/localtime.c b/lib/libc/stdtime/localtime.c
index 9fc5f3e..bee916b 100644
--- a/lib/libc/stdtime/localtime.c
+++ b/lib/libc/stdtime/localtime.c
@@ -235,9 +235,14 @@ static struct state	gmtmem;
 
 static char		lcl_TZname[TZ_STRLEN_MAX + 1];
 static int		lcl_is_set;
-static int		gmt_is_set;
+static pthread_once_t	gmt_once = PTHREAD_ONCE_INIT;
 static pthread_rwlock_t	lcl_rwlock = PTHREAD_RWLOCK_INITIALIZER;
-static pthread_mutex_t	gmt_mutex = PTHREAD_MUTEX_INITIALIZER;
+static pthread_once_t	gmtime_once = PTHREAD_ONCE_INIT;
+static pthread_key_t	gmtime_key;
+static int		gmtime_key_error;
+static pthread_once_t	localtime_once = PTHREAD_ONCE_INIT;
+static pthread_key_t	localtime_key;
+static int		localtime_key_error;
 
 char *			tzname[2] = {
 	wildabbr,
@@ -1407,27 +1412,24 @@ struct tm * const	tmp;
 	return result;
 }
 
+static void
+localtime_key_init(void)
+{
+
+	localtime_key_error = _pthread_key_create(&localtime_key, free);
+}
+
 struct tm *
 localtime(timep)
 const time_t * const	timep;
 {
-	static pthread_mutex_t localtime_mutex = PTHREAD_MUTEX_INITIALIZER;
-	static pthread_key_t localtime_key = -1;
 	struct tm *p_tm;
-	int r;
 
 	if (__isthreaded != 0) {
-		if (localtime_key < 0) {
-			_pthread_mutex_lock(&localtime_mutex);
-			if (localtime_key < 0) {
-				if ((r = _pthread_key_create(&localtime_key,
-				    free)) != 0) {
-					_pthread_mutex_unlock(&localtime_mutex);
-					errno = r;
-					return(NULL);
-				}
-			}
-			_pthread_mutex_unlock(&localtime_mutex);
+		_pthread_once(&localtime_once, localtime_key_init);
+		if (localtime_key_error != 0) {
+			errno = localtime_key_error;
+			return(NULL);
 		}
 		p_tm = _pthread_getspecific(localtime_key);
 		if (p_tm == NULL) {
@@ -1464,6 +1466,17 @@ struct tm *		tmp;
 	return tmp;
 }
 
+static void
+gmt_init(void)
+{
+
+#ifdef ALL_STATE
+	gmtptr = (struct state *) malloc(sizeof *gmtptr);
+	if (gmtptr != NULL)
+#endif /* defined ALL_STATE */
+		gmtload(gmtptr);
+}
+
 /*
 ** gmtsub is to gmtime as localsub is to localtime.
 */
@@ -1476,16 +1489,7 @@ struct tm * const	tmp;
 {
 	register struct tm *	result;
 
-	_MUTEX_LOCK(&gmt_mutex);
-	if (!gmt_is_set) {
-#ifdef ALL_STATE
-		gmtptr = (struct state *) malloc(sizeof *gmtptr);
-		if (gmtptr != NULL)
-#endif /* defined ALL_STATE */
-			gmtload(gmtptr);
-		gmt_is_set = TRUE;
-	}
-	_MUTEX_UNLOCK(&gmt_mutex);
+	_once(&gmt_once, gmt_init);
 	result = timesub(timep, offset, gmtptr, tmp);
 #ifdef TM_ZONE
 	/*
@@ -1509,27 +1513,24 @@ struct tm * const	tmp;
 	return result;
 }
 
+static void
+gmtime_key_init(void)
+{
+
+	gmtime_key_error = _pthread_key_create(&gmtime_key, free);
+}
+
 struct tm *
 gmtime(timep)
 const time_t * const	timep;
 {
-	static pthread_mutex_t gmtime_mutex = PTHREAD_MUTEX_INITIALIZER;
-	static pthread_key_t gmtime_key = -1;
 	struct tm *p_tm;
-	int r;
 
 	if (__isthreaded != 0) {
-		if (gmtime_key < 0) {
-			_pthread_mutex_lock(&gmtime_mutex);
-			if (gmtime_key < 0) {
-				if ((r = _pthread_key_create(&gmtime_key,
-				    free)) != 0) {
-					_pthread_mutex_unlock(&gmtime_mutex);
-					errno = r;
-					return(NULL);
-				}
-			}
-			_pthread_mutex_unlock(&gmtime_mutex);
+		_pthread_once(&gmtime_once, gmtime_key_init);
+		if (gmtime_key_error != 0) {
+			errno = gmtime_key_error;
+			return(NULL);
 		}
 		/*
 		 * Changed to follow POSIX.1 threads standard, which
-- 
cgit v1.1


From 46ef0903053fbd1c63f5b29c8a492494b88c6fe3 Mon Sep 17 00:00:00 2001
From: brueffer <brueffer@FreeBSD.org>
Date: Fri, 15 Jan 2010 19:42:09 +0000
Subject: MFC: r201836

Remove unnecessary quoting and markup, add missing punctuation.
---
 lib/libc/stdio/getc.3 | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/stdio/getc.3 b/lib/libc/stdio/getc.3
index c8b9386..d0f3c15 100644
--- a/lib/libc/stdio/getc.3
+++ b/lib/libc/stdio/getc.3
@@ -56,7 +56,7 @@
 .Ft int
 .Fn getchar void
 .Ft int
-.Fn getchar_unlocked "void"
+.Fn getchar_unlocked void
 .Ft int
 .Fn getw "FILE *stream"
 .Sh DESCRIPTION
@@ -141,7 +141,7 @@ until the condition is cleared with
 .Sh STANDARDS
 The
 .Fn fgetc ,
-.Fn getc
+.Fn getc ,
 and
 .Fn getchar
 functions
@@ -167,4 +167,3 @@ The size and byte order of an
 varies from one machine to another, and
 .Fn getw
 is not recommended for portable applications.
-.Pp
-- 
cgit v1.1


From 3cbec0168670df0e5b4a149992b2289266d546f1 Mon Sep 17 00:00:00 2001
From: brueffer <brueffer@FreeBSD.org>
Date: Fri, 15 Jan 2010 19:53:36 +0000
Subject: MFC: r202159

Remove useless .TE groff macro.
---
 lib/libelf/elf.3 | 1 -
 1 file changed, 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libelf/elf.3 b/lib/libelf/elf.3
index 0727ba8..854e1af 100644
--- a/lib/libelf/elf.3
+++ b/lib/libelf/elf.3
@@ -379,7 +379,6 @@ See
 .It Dv SHT_SUNW_move Ta Dv ELF_T_MOVE Ta ELF move records.
 .It Dv SHT_SUNW_syminfo Ta Dv ELF_T_SYMINFO Ta Additional symbol flags.
 .El
-.TE
 .Ss Functional Grouping
 This section contains a brief overview of the available functionality
 in the ELF library.
-- 
cgit v1.1


From 4780cb8a56a40e9f01006416041e995fcedfcf1e Mon Sep 17 00:00:00 2001
From: marcel <marcel@FreeBSD.org>
Date: Sat, 16 Jan 2010 04:24:10 +0000
Subject: MFC rev 201937: Implement the fo_readdir method.

---
 lib/libstand/dosfs.c | 69 +++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 68 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libstand/dosfs.c b/lib/libstand/dosfs.c
index 161d330..5ba10c1 100644
--- a/lib/libstand/dosfs.c
+++ b/lib/libstand/dosfs.c
@@ -47,6 +47,7 @@ static int	dos_close(struct open_file *fd);
 static int	dos_read(struct open_file *fd, void *buf, size_t size, size_t *resid);
 static off_t	dos_seek(struct open_file *fd, off_t offset, int whence);
 static int	dos_stat(struct open_file *fd, struct stat *sb);
+static int	dos_readdir(struct open_file *fd, struct dirent *d);
 
 struct fs_ops dosfs_fsops = {
 	"dosfs",
@@ -56,7 +57,7 @@ struct fs_ops dosfs_fsops = {
 	null_write,
 	dos_seek,
 	dos_stat,
-	null_readdir
+	dos_readdir
 };
 
 #define SECSIZ  512             /* sector size */
@@ -354,6 +355,72 @@ dos_stat(struct open_file *fd, struct stat *sb)
     return (0);
 }
 
+static int
+dos_readdir(struct open_file *fd, struct dirent *d)
+{
+    DOS_FILE *f = (DOS_FILE *)fd->f_fsdata;
+    u_char fn[261];
+    DOS_DIR dd;
+    size_t res;
+    u_int chk, i, x, xdn;
+    int err;
+
+    x = chk = 0;
+    while (1) {
+	xdn = x;
+	x = 0;
+	err = dos_read(fd, &dd, sizeof(dd), &res);
+	if (err)
+	    return (err);
+	if (res == sizeof(dd))
+	    return (ENOENT);
+	if (dd.de.name[0] == 0)
+	    return (ENOENT);
+
+	/* Skip deleted entries */
+	if (dd.de.name[0] == 0xe5)
+	    continue;
+
+	/* Skip volume labels */
+	if (dd.de.attr & FA_LABEL)
+	    continue;
+
+	if ((dd.de.attr & FA_MASK) == FA_XDE) {
+	    if (dd.xde.seq & 0x40)
+		chk = dd.xde.chk;
+	    else if (dd.xde.seq != xdn - 1 || dd.xde.chk != chk)
+		continue;
+	    x = dd.xde.seq & ~0x40;
+	    if (x < 1 || x > 20) {
+		x = 0;
+		continue;
+	    }
+	    cp_xdnm(fn, &dd.xde);
+	} else {
+	    if (xdn == 1) {
+		x = 0;
+		for (i = 0; i < 11; i++) {
+		    x = ((x & 1) << 7) | (x >> 1);
+		    x += dd.de.name[i];
+		    x &= 0xff;
+		}
+		if (x == chk)
+		    break;
+	    } else {
+		cp_sfn(fn, &dd.de);
+		break;
+	    }
+	    x = 0;
+	}
+    }
+
+    d->d_fileno = dd.de.clus[1] << 8 + dd.de.clus[0];
+    d->d_reclen = sizeof(*d);
+    d->d_type = (dd.de.attr & FA_DIR) ? DT_DIR : DT_REG;
+    memcpy(d->d_name, fn, sizeof(d->d_name));
+    return(0);
+}
+
 /*
  * Parse DOS boot sector
  */
-- 
cgit v1.1


From f803d28ff49724436bacd3932eec21253761c255 Mon Sep 17 00:00:00 2001
From: thompsa <thompsa@FreeBSD.org>
Date: Sun, 17 Jan 2010 18:24:40 +0000
Subject: MFC r202025

 Reset variable fields in case the transfer is opened again
---
 lib/libusb/libusb20.c | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libusb/libusb20.c b/lib/libusb/libusb20.c
index 41f9ac7..1c75fc1 100644
--- a/lib/libusb/libusb20.c
+++ b/lib/libusb/libusb20.c
@@ -130,8 +130,19 @@ libusb20_tr_close(struct libusb20_transfer *xfer)
 	if (xfer->ppBuffer) {
 		free(xfer->ppBuffer);
 	}
-	/* clear some fields */
+	/* reset variable fields in case the transfer is opened again */
+	xfer->priv_sc0 = 0;
+	xfer->priv_sc1 = 0;
 	xfer->is_opened = 0;
+	xfer->is_pending = 0;
+	xfer->is_cancel = 0;
+	xfer->is_draining = 0;
+	xfer->is_restart = 0;
+	xfer->status = 0;
+	xfer->flags = 0;
+	xfer->nFrames = 0;
+	xfer->aFrames = 0;
+	xfer->timeout = 0;
 	xfer->maxFrames = 0;
 	xfer->maxTotalLength = 0;
 	xfer->maxPacketLen = 0;
-- 
cgit v1.1


From 8dd66f6ac2932f88ea784062f7de0a2fee3d688e Mon Sep 17 00:00:00 2001
From: brueffer <brueffer@FreeBSD.org>
Date: Tue, 19 Jan 2010 16:51:51 +0000
Subject: MFC: r201603

Fix a double free().
---
 lib/libc/rpc/getnetpath.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libc/rpc/getnetpath.c b/lib/libc/rpc/getnetpath.c
index 0563544..d1ea554 100644
--- a/lib/libc/rpc/getnetpath.c
+++ b/lib/libc/rpc/getnetpath.c
@@ -101,7 +101,7 @@ setnetpath()
     if ((np_sessionp->nc_handlep = setnetconfig()) == NULL) {
 	free(np_sessionp);
 	syslog (LOG_ERR, "rpc: failed to open " NETCONFIG);
-	goto failed;
+    	return (NULL);
     }
     np_sessionp->valid = NP_VALID;
     np_sessionp->ncp_list = NULL;
-- 
cgit v1.1


From 0c4f66b0f2f83fc97fdd618b79ad08ac49bec9cd Mon Sep 17 00:00:00 2001
From: brueffer <brueffer@FreeBSD.org>
Date: Tue, 19 Jan 2010 17:09:18 +0000
Subject: MFC: r202176

Miscellaneous mdoc, spelling and inconsistency fixes.
---
 lib/libc/net/sctp_bindx.3           |  2 +-
 lib/libc/net/sctp_connectx.3        |  6 +++---
 lib/libc/net/sctp_getaddrlen.3      |  2 +-
 lib/libc/net/sctp_getassocid.3      |  2 +-
 lib/libc/net/sctp_getpaddrs.3       |  6 +++---
 lib/libc/net/sctp_opt_info.3        |  4 ++--
 lib/libc/net/sctp_recvmsg.3         | 19 ++++++++++---------
 lib/libc/net/sctp_send.3            |  2 +-
 lib/libc/net/sctp_sendmsg.3         |  4 ++--
 lib/libc/sys/sctp_generic_recvmsg.2 | 24 ++++++++++++++----------
 lib/libc/sys/sctp_generic_sendmsg.2 | 24 ++++++++++++++----------
 lib/libc/sys/sctp_peeloff.2         |  4 ++--
 12 files changed, 54 insertions(+), 45 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/net/sctp_bindx.3 b/lib/libc/net/sctp_bindx.3
index a2a2600..d047126 100644
--- a/lib/libc/net/sctp_bindx.3
+++ b/lib/libc/net/sctp_bindx.3
@@ -90,7 +90,7 @@ The call returns 0 on success and -1 upon failure.
 .Sh ERRORS
 The
 .Fn sctp_bindx
-can return the following errors.
+function can return the following errors:
 .Bl -tag -width Er
 .It Bq Er EINVAL
 This value is returned if the
diff --git a/lib/libc/net/sctp_connectx.3 b/lib/libc/net/sctp_connectx.3
index 7b75f4a..d454bb9 100644
--- a/lib/libc/net/sctp_connectx.3
+++ b/lib/libc/net/sctp_connectx.3
@@ -44,7 +44,7 @@
 .In sys/socket.h
 .In netinet/sctp.h
 .Ft int
-.Fn sctp_connectx "int s" "struct sockaddr *" "int addrcnt" "sctp_assoc_t *"
+.Fn sctp_connectx "int sd" "struct sockaddr *addrs" "int addrcnt" "sctp_assoc_t *id"
 .Sh DESCRIPTION
 The
 .Fn sctp_connectx
@@ -75,7 +75,7 @@ the extra addresses sent in the
 call will be silently discarded from the association.
 On
 successful completion the provided
-.Fa "sctp_assoc_t *"
+.Fa id
 will be
 filled in with the association identification of the newly
 forming association.
@@ -84,7 +84,7 @@ The call returns 0 on success and -1 upon failure.
 .Sh ERRORS
 The
 .Fn sctp_connectx
-can return the following errors.
+function can return the following errors:
 .Bl -tag -width Er
 .It Bq Er EINVAL
 An address listed has an invalid family or no
diff --git a/lib/libc/net/sctp_getaddrlen.3 b/lib/libc/net/sctp_getaddrlen.3
index 325e2b0..ac7bfb3 100644
--- a/lib/libc/net/sctp_getaddrlen.3
+++ b/lib/libc/net/sctp_getaddrlen.3
@@ -76,7 +76,7 @@ system expects for the specific address family or -1.
 .Sh ERRORS
 The
 .Fn sctp_getaddrlen
-function can return the following errors.
+function can return the following errors:
 .Bl -tag -width Er
 .It Bq Er EINVAL
 The address family specified does NOT exist.
diff --git a/lib/libc/net/sctp_getassocid.3 b/lib/libc/net/sctp_getassocid.3
index ee0926a..909b9c7 100644
--- a/lib/libc/net/sctp_getassocid.3
+++ b/lib/libc/net/sctp_getassocid.3
@@ -58,7 +58,7 @@ The call returns the association id upon success and
 .Sh ERRORS
 The
 .Fn sctp_getassocid
-function can return the following errors.
+function can return the following errors:
 .Bl -tag -width Er
 .It Bq Er ENOENT
 The address does not have an association setup to it.
diff --git a/lib/libc/net/sctp_getpaddrs.3 b/lib/libc/net/sctp_getpaddrs.3
index 9f6d632..6ad5ea0 100644
--- a/lib/libc/net/sctp_getpaddrs.3
+++ b/lib/libc/net/sctp_getpaddrs.3
@@ -33,7 +33,7 @@
 .\" $FreeBSD$
 .\"
 .Dd December 15, 2006
-.Dt SCTP_GETPADDR 3
+.Dt SCTP_GETPADDRS 3
 .Os
 .Sh NAME
 .Nm sctp_getpaddrs ,
@@ -64,7 +64,7 @@ array of socket addresses returned in the argument
 .Fa addrs
 upon success.
 .Pp
-After the caller is through the function
+After the caller is finished, the function
 .Fn sctp_freepaddrs
 or
 .Fn sctp_freeladdrs
@@ -76,7 +76,7 @@ the number of addresses returned in
 .Fa addrs
 upon success.
 .Sh ERRORS
-The functions can return the following errors.
+The functions can return the following errors:
 .Bl -tag -width Er
 .It Bq Er EINVAL
 An address listed has an invalid family or no
diff --git a/lib/libc/net/sctp_opt_info.3 b/lib/libc/net/sctp_opt_info.3
index 560b3e4..581a220 100644
--- a/lib/libc/net/sctp_opt_info.3
+++ b/lib/libc/net/sctp_opt_info.3
@@ -45,7 +45,7 @@
 .In sys/socket.h
 .In netinet/sctp.h
 .Ft int
-.Fn sctp_opt_info "int s" "sctp_assoc_t" "int opt" "void *arg" "socklen_t *size"
+.Fn sctp_opt_info "int sd" "sctp_assoc_t id" "int opt" "void *arg" "socklen_t *size"
 .Sh DESCRIPTION
 The
 .Fn sctp_opt_info
@@ -90,7 +90,7 @@ socket options.
 .Sh ERRORS
 The
 .Fn sctp_opt_info
-function can return the following errors.
+function can return the following errors:
 .Bl -tag -width Er
 .It Bq Er EINVAL
 The argument
diff --git a/lib/libc/net/sctp_recvmsg.3 b/lib/libc/net/sctp_recvmsg.3
index b9ccfb8..f874880 100644
--- a/lib/libc/net/sctp_recvmsg.3
+++ b/lib/libc/net/sctp_recvmsg.3
@@ -160,7 +160,7 @@ struct sctp_sndrcvinfo {
 .Pp
 The
 .Fa sinfo->sinfo_ppid
-is an opaque 32 bit value that is passed transparently
+field is an opaque 32 bit value that is passed transparently
 through the stack from the peer endpoint. 
 Note that the stack passes this value without regard to byte
 order.
@@ -180,12 +180,13 @@ as soon as possible.
 When this flag is absent the message
 was delivered in order within the stream it was received.
 .Pp
+The
 .Fa sinfo->sinfo_stream
-is the SCTP stream that the message was received on. 
+field is the SCTP stream that the message was received on. 
 Streams in SCTP are reliable (or partially reliable) flows of ordered
 messages.
 .Pp
-The 
+The
 .Fa sinfo->sinfo_context
 field is used only if the local application set an association level
 context with the
@@ -197,7 +198,7 @@ association.
 .Pp
 The
 .Fa sinfo->sinfo_ssn
-will hold the stream sequence number assigned
+field will hold the stream sequence number assigned
 by the peer endpoint if the message is
 .Em not
 unordered.
@@ -205,7 +206,7 @@ For unordered messages this field holds an undefined value.
 .Pp
 The
 .Fa sinfo->sinfo_tsn
-holds a transport sequence number (TSN) that was assigned
+field holds a transport sequence number (TSN) that was assigned
 to this message by the peer endpoint.
 For messages that fit in or less
 than the path MTU this will be the only TSN assigned.
@@ -215,12 +216,12 @@ message.
 .Pp
 The
 .Fa sinfo->sinfo_cumtsn
-holds the current cumulative acknowledgment point of
+field holds the current cumulative acknowledgment point of
 the transport association.
 Note that this may be larger
 or smaller than the TSN assigned to the message itself.
 .Pp
-The 
+The
 .Fa sinfo->sinfo_assoc_id
 is the unique association identification that was assigned
 to the association.
@@ -232,10 +233,10 @@ setting various socket options on the specific association
 (see 
 .Xr sctp 4 ) .
 .Pp
-The 
+The
 .Fa sinfo->info_timetolive
 field is not used by 
-.Fa sctp_recvmsg .
+.Fn sctp_recvmsg .
 .Sh RETURN VALUES
 The call returns the number of characters sent, or -1
 if an error occurred.
diff --git a/lib/libc/net/sctp_send.3 b/lib/libc/net/sctp_send.3
index 2301730..f6f0c95 100644
--- a/lib/libc/net/sctp_send.3
+++ b/lib/libc/net/sctp_send.3
@@ -294,7 +294,7 @@ if an error occurred.
 The
 .Fn sctp_send
 system call
-fail if:
+fails if:
 .Bl -tag -width Er
 .It Bq Er EBADF
 An invalid descriptor was specified.
diff --git a/lib/libc/net/sctp_sendmsg.3 b/lib/libc/net/sctp_sendmsg.3
index 61eec22..0c5fa02 100644
--- a/lib/libc/net/sctp_sendmsg.3
+++ b/lib/libc/net/sctp_sendmsg.3
@@ -271,7 +271,7 @@ if an error occurred.
 The
 .Fn sctp_sendmsg
 system call
-fail if:
+fails if:
 .Bl -tag -width Er
 .It Bq Er EBADF
 An invalid descriptor was specified.
@@ -324,7 +324,7 @@ is not connected and is a one-to-one style socket.
 .Xr sendmsg 3 ,
 .Xr sctp 4
 .Sh BUGS
-Because in the one-to-many style socket the
+Because in the one-to-many style socket
 .Fn sctp_sendmsg 
 or
 .Fn sctp_sendmsgx
diff --git a/lib/libc/sys/sctp_generic_recvmsg.2 b/lib/libc/sys/sctp_generic_recvmsg.2
index ccdb9db..d6fd1b5 100644
--- a/lib/libc/sys/sctp_generic_recvmsg.2
+++ b/lib/libc/sys/sctp_generic_recvmsg.2
@@ -36,7 +36,7 @@
 .Os
 .Sh NAME
 .Nm sctp_generic_recvmsg
-.Nd receive data from a peer.
+.Nd receive data from a peer
 .Sh LIBRARY
 .Lb libc
 .Sh SYNOPSIS
@@ -46,16 +46,20 @@
 .Ft int
 .Fn sctp_generic_recvmsg "int s" "struct iovec *iov" "int iovlen" "struct sockaddr *from" "socklen_t *fromlen" "struct sctp_sndrcvinfo *sinfo" "int *msgflags"
 .Sh DESCRIPTION
-The
 .Fn sctp_generic_recvmsg
-is the true system calls used by the
-.Fn sctp_recvmsg
-function call. This call is more efficient since it is a
-true system calls but it is specific to FreeBSD and
-can be expected NOT to be present on any other Operating
-System. For detailed useage please see either the
-.Fn sctp_recvmsg 
-function call. 
+is the true system call used by the
+.Xr sctp_recvmsg 3
+function call.
+This call is more efficient since it is a
+true system call but it is specific to
+.Fx
+and can be expected
+.Em not
+to be present on any other operating
+system.
+For detailed usage please see the
+.Xr sctp_recvmsg 3
+function call.
 .Sh RETURN VALUES
 The call returns the number of bytes read on success and -1 upon failure.
 .Sh ERRORS
diff --git a/lib/libc/sys/sctp_generic_sendmsg.2 b/lib/libc/sys/sctp_generic_sendmsg.2
index d7050f2..fee4211 100644
--- a/lib/libc/sys/sctp_generic_sendmsg.2
+++ b/lib/libc/sys/sctp_generic_sendmsg.2
@@ -37,7 +37,7 @@
 .Sh NAME
 .Nm sctp_generic_sendmsg
 .Nm sctp_generic_sendmsg_iov
-.Nd send data to a peer.
+.Nd send data to a peer
 .Sh LIBRARY
 .Lb libc
 .Sh SYNOPSIS
@@ -49,21 +49,25 @@
 .Ft int
 .Fn sctp_generic_sendmsg_iov "int s" "struct iovec *iov" "int iovlen" "struct sockaddr *to" "struct sctp_sndrcvinfo *sinfo" "int flags"
 .Sh DESCRIPTION
-The
 .Fn sctp_generic_sendmsg
 and
 .Fn sctp_generic_sendmsg_iov
 are the true system calls used by the
-.Fn sctp_sendmsg
+.Xr sctp_sendmsg 3
 and 
-.Fn sctp_send
-function calls. These are more efficient since they are
-true system calls but they are specific to FreeBSD and
-can be expected NOT to be present on any other Operating
-System. For detailed useage please see either the
-.Fn sctp_send
+.Xr sctp_send 3
+function calls.
+These are more efficient since they are
+true system calls but they are specific to
+.Fx
+and can be expected
+.Em not
+to be present on any other operating
+system.
+For detailed usage please see either the
+.Xr sctp_send 3
 or
-.Fn sctp_sendmsg
+.Xr sctp_sendmsg 3
 function calls. 
 .Sh RETURN VALUES
 The call returns the number of bytes written on success and -1 upon failure.
diff --git a/lib/libc/sys/sctp_peeloff.2 b/lib/libc/sys/sctp_peeloff.2
index 0ac5be6..c513afa 100644
--- a/lib/libc/sys/sctp_peeloff.2
+++ b/lib/libc/sys/sctp_peeloff.2
@@ -36,7 +36,7 @@
 .Os
 .Sh NAME
 .Nm sctp_peeloff
-.Nd detach an association from a one-to-many socket to its on fd
+.Nd detach an association from a one-to-many socket to its own fd
 .Sh LIBRARY
 .Lb libc
 .Sh SYNOPSIS
@@ -58,7 +58,7 @@ upon success.
 .Sh ERRORS
 The
 .Fn sctp_peeloff
-can return the following errors.
+system call can return the following errors:
 .Bl -tag -width Er
 .It Bq Er ENOTCONN
 The 
-- 
cgit v1.1


From 2f8777a272e2636da3cab02155819ce00d3be75a Mon Sep 17 00:00:00 2001
From: fabient <fabient@FreeBSD.org>
Date: Tue, 19 Jan 2010 20:55:57 +0000
Subject: MFC 202157:  Bug fix: add a missing initializer.

---
 lib/libpmc/libpmc.c | 1 +
 1 file changed, 1 insertion(+)

(limited to 'lib')

diff --git a/lib/libpmc/libpmc.c b/lib/libpmc/libpmc.c
index ecc697d..af716da 100644
--- a/lib/libpmc/libpmc.c
+++ b/lib/libpmc/libpmc.c
@@ -2507,6 +2507,7 @@ pmc_init(void)
 		break;
 	case PMC_CPU_INTEL_CORE:
 		PMC_MDEP_INIT(core);
+		pmc_class_table[n] = &core_class_table_descr;
 		break;
 	case PMC_CPU_INTEL_CORE2:
 	case PMC_CPU_INTEL_CORE2EXTREME:
-- 
cgit v1.1


From 611984fd9c7f0d6f95aeb5cacf35228e2db05d9e Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Wed, 20 Jan 2010 00:52:24 +0000
Subject: MFC r200799:

K&R -> ANSI prototype.
---
 lib/libc/stdio/vsscanf.c | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/stdio/vsscanf.c b/lib/libc/stdio/vsscanf.c
index e5e9691..22b5d2b 100644
--- a/lib/libc/stdio/vsscanf.c
+++ b/lib/libc/stdio/vsscanf.c
@@ -45,20 +45,15 @@ eofread(void *, char *, int);
 
 /* ARGSUSED */
 static int
-eofread(cookie, buf, len)
-	void *cookie;
-	char *buf;
-	int len;
+eofread(void *cookie, char *buf, int len)
 {
 
 	return (0);
 }
 
 int
-vsscanf(str, fmt, ap)
-	const char * __restrict str;
-	const char * __restrict fmt;
-	__va_list ap;
+vsscanf(const char * __restrict str, const char * __restrict fmt,
+	__va_list ap)
 {
 	FILE f;
 
-- 
cgit v1.1


From b823cbbcbac890bc9443bc1ab5ad5df5a68ad952 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Wed, 20 Jan 2010 00:53:03 +0000
Subject: MFC r200800:

Use vsscanf instead of rolling our own.

PR:		bin/140530
Submitted by:	Jeremy Huddleston <jeremyhu apple.com>
---
 lib/libc/stdio/sscanf.c | 25 +------------------------
 1 file changed, 1 insertion(+), 24 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/stdio/sscanf.c b/lib/libc/stdio/sscanf.c
index 3c792e0..c793b86 100644
--- a/lib/libc/stdio/sscanf.c
+++ b/lib/libc/stdio/sscanf.c
@@ -41,37 +41,14 @@ __FBSDID("$FreeBSD$");
 #include <stdarg.h>
 #include "local.h"
 
-static int eofread(void *, char *, int);
-
-/* ARGSUSED */
-static int
-eofread(cookie, buf, len)
-	void *cookie;
-	char *buf;
-	int len;
-{
-
-	return (0);
-}
-
 int
 sscanf(const char * __restrict str, char const * __restrict fmt, ...)
 {
 	int ret;
 	va_list ap;
-	FILE f;
 
-	f._file = -1;
-	f._flags = __SRD;
-	f._bf._base = f._p = (unsigned char *)str;
-	f._bf._size = f._r = strlen(str);
-	f._read = eofread;
-	f._ub._base = NULL;
-	f._lb._base = NULL;
-	f._orientation = 0;
-	memset(&f._mbstate, 0, sizeof(mbstate_t));
 	va_start(ap, fmt);
-	ret = __svfscanf(&f, fmt, ap);
+	ret = vsscanf(str, fmt, ap);
 	va_end(ap);
 	return (ret);
 }
-- 
cgit v1.1


From 831edd40677c273e864cffd01740ac8286bd04ba Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Wed, 20 Jan 2010 00:53:44 +0000
Subject: MFC r200802:

Use vsprintf instead of rolling our own.

PR:		bin/140496
Submitted by:	Jeremy Huddleston <jeremyhu apple.com>
---
 lib/libc/stdio/sprintf.c | 10 +---------
 1 file changed, 1 insertion(+), 9 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/stdio/sprintf.c b/lib/libc/stdio/sprintf.c
index aaaae55..b55bd6c 100644
--- a/lib/libc/stdio/sprintf.c
+++ b/lib/libc/stdio/sprintf.c
@@ -46,17 +46,9 @@ sprintf(char * __restrict str, char const * __restrict fmt, ...)
 {
 	int ret;
 	va_list ap;
-	FILE f;
 
-	f._file = -1;
-	f._flags = __SWR | __SSTR;
-	f._bf._base = f._p = (unsigned char *)str;
-	f._bf._size = f._w = INT_MAX;
-	f._orientation = 0;
-	memset(&f._mbstate, 0, sizeof(mbstate_t));
 	va_start(ap, fmt);
-	ret = __vfprintf(&f, fmt, ap);
+	ret = vsprintf(str, fmt, ap);
 	va_end(ap);
-	*f._p = 0;
 	return (ret);
 }
-- 
cgit v1.1


From 7670355465ac38d5365cd3c0afe593b2f90d3f62 Mon Sep 17 00:00:00 2001
From: avg <avg@FreeBSD.org>
Date: Fri, 22 Jan 2010 09:27:31 +0000
Subject: MFC r202585: fix a comment typo

---
 lib/libstand/bzipfs.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libstand/bzipfs.c b/lib/libstand/bzipfs.c
index 46a151b..1b2e9eb 100644
--- a/lib/libstand/bzipfs.c
+++ b/lib/libstand/bzipfs.c
@@ -279,7 +279,7 @@ bzf_rewind(struct open_file *f)
     /*
      * Since bzip2 does not have an equivalent inflateReset function a crude
      * one needs to be provided.  The functions all called in such a way that
-     * at any time an error occurs a role back can be done (effectively making
+     * at any time an error occurs a roll back can be done (effectively making
      * this rewind 'atomic', either the reset occurs successfully or not at all,
      * with no 'undefined' state happening).
      */
-- 
cgit v1.1


From d13809bdba98ca6d5e15d5838c916191af7b09c5 Mon Sep 17 00:00:00 2001
From: brooks <brooks@FreeBSD.org>
Date: Fri, 22 Jan 2010 19:51:34 +0000
Subject: MFC r201350:

  The devices that supported EVFILT_NETDEV kqueue filters were removed in
  r195175.  Remove all definitions, documentation, and usage.

The change of function signature for vlan_link_state() was not merged to
maintain the ABI.
---
 lib/libc/sys/kqueue.2 | 20 --------------------
 1 file changed, 20 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/sys/kqueue.2 b/lib/libc/sys/kqueue.2
index e899a1b..326632d 100644
--- a/lib/libc/sys/kqueue.2
+++ b/lib/libc/sys/kqueue.2
@@ -438,19 +438,6 @@ There is a system wide limit on the number of timers
 which is controlled by the
 .Va kern.kq_calloutmax
 sysctl.
-.It Dv EVFILT_NETDEV
-Takes a descriptor to a network interface as the identifier, and the events to watch for in
-.Va fflags .
-It returns, when one or more of the requested events occur on the descriptor.
-The events to monitor are:
-.Bl -tag -width XXNOTE_LINKDOWN
-.It Dv NOTE_LINKUP
-The link is up.
-.It Dv NOTE_LINKDOWN
-The link is down.
-.It Dv NOTE_LINKINV
-The link state is invalid.
-.El
 .Pp
 On return,
 .Va fflags
@@ -595,13 +582,6 @@ system and this manual page were written by
 .An Jonathan Lemon Aq jlemon@FreeBSD.org .
 .Sh BUGS
 The
-.Dv EVFILT_NETDEV
-filter is currently only implemented for devices that use the
-.Xr miibus 4
-driver for LINKUP and LINKDOWN operations.
-Therefore, it will not work with many non-ethernet devices.
-.Pp
-The
 .Fa timeout
 value is limited to 24 hours; longer timeouts will be silently
 reinterpreted as 24 hours.
-- 
cgit v1.1


From 4e79f22935c2afb11c50e718e8e97ea35e256c7c Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Sat, 23 Jan 2010 00:43:44 +0000
Subject: MFC r201892:

Add a set of manual pages for pthread[_attr]_[sg]etaffinity(3).

Reviewed by:	davidxu
---
 lib/libc/sys/cpuset.2             | 6 ++++--
 lib/libc/sys/cpuset_getaffinity.2 | 6 ++++--
 2 files changed, 8 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/sys/cpuset.2 b/lib/libc/sys/cpuset.2
index fb9a770..1cdff68 100644
--- a/lib/libc/sys/cpuset.2
+++ b/lib/libc/sys/cpuset.2
@@ -25,7 +25,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd March 29, 2008
+.Dd January 8, 2010
 .Dt CPUSET 2
 .Os
 .Sh NAME
@@ -216,7 +216,9 @@ for allocation.
 .Xr cpuset 1 ,
 .Xr cpuset_getaffinity 2 ,
 .Xr cpuset_setaffinity 2 ,
-.Xr CPU_SET 3
+.Xr CPU_SET 3 ,
+.Xr pthread_affinity_np 3 ,
+.Xr pthread_attr_affinity_np 3
 .Sh HISTORY
 The
 .Nm
diff --git a/lib/libc/sys/cpuset_getaffinity.2 b/lib/libc/sys/cpuset_getaffinity.2
index b065ac3..c8b272f 100644
--- a/lib/libc/sys/cpuset_getaffinity.2
+++ b/lib/libc/sys/cpuset_getaffinity.2
@@ -25,7 +25,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd November 29, 2008
+.Dd January 8, 2010
 .Dt CPUSET 2
 .Os
 .Sh NAME
@@ -147,7 +147,9 @@ operation.
 .Xr cpuset 2 ,
 .Xr cpuset_getid 2 ,
 .Xr cpuset_setid 2 ,
-.Xr CPU_SET 3
+.Xr CPU_SET 3 ,
+.Xr pthread_affinity_np 3 ,
+.Xr pthread_attr_affinity_np 3
 .Sh HISTORY
 The
 .Nm
-- 
cgit v1.1


From 9fefaa1da5ac95af3adaeb90fa4952d512222d64 Mon Sep 17 00:00:00 2001
From: kib <kib@FreeBSD.org>
Date: Sun, 24 Jan 2010 12:35:36 +0000
Subject: Merge scandir(3) interface update to stable/8.

MFC r201512:
Modernize scandir(3) and alphasort(3) interfaces according to the IEEE
Std 1003.1-2008.

MFC r201602:
Move scandir(3) and alphasort(3) into XSI namespace.

MFC r201604:
Use thunks to adapt alphasort-like interface to the comparision function
required by qsort() and qsort_r().

MFC r202556 (by ache):
Use strcoll() in opendir() and alphasort(). Remove some comments.

MFC r202572 (by ache):
Revert to using strcmp() for opendir().

MFC r202677 (by ache):
Style.

MFC r202679 (by ache):
Style: rename internal function to opendir_compar().

MFC r202691 (by ache):
For alphasort(3) add reference to strcoll(3).

MFC r202693 (by ache):
Style: reword comment.
---
 lib/libc/gen/opendir.c | 11 ++++++++++-
 lib/libc/gen/scandir.3 | 13 +++++++------
 lib/libc/gen/scandir.c | 35 ++++++++++++++++++++++-------------
 3 files changed, 39 insertions(+), 20 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/gen/opendir.c b/lib/libc/gen/opendir.c
index 9625ec6..b312c89 100644
--- a/lib/libc/gen/opendir.c
+++ b/lib/libc/gen/opendir.c
@@ -92,6 +92,14 @@ __opendir2(const char *name, int flags)
 	return __opendir_common(fd, name, flags);
 }
 
+static int
+opendir_compar(const void *p1, const void *p2)
+{
+
+	return (strcmp((*(const struct dirent **)p1)->d_name,
+	    (*(const struct dirent **)p2)->d_name));
+}
+
 /*
  * Common routine for opendir(3), __opendir2(3) and fdopendir(3).
  */
@@ -240,7 +248,8 @@ __opendir_common(int fd, const char *name, int flags)
 				/*
 				 * This sort must be stable.
 				 */
-				mergesort(dpv, n, sizeof(*dpv), alphasort);
+				mergesort(dpv, n, sizeof(*dpv),
+				    opendir_compar);
 
 				dpv[n] = NULL;
 				xp = NULL;
diff --git a/lib/libc/gen/scandir.3 b/lib/libc/gen/scandir.3
index 42ccac2..e32b0d8 100644
--- a/lib/libc/gen/scandir.3
+++ b/lib/libc/gen/scandir.3
@@ -28,7 +28,7 @@
 .\"     @(#)scandir.3	8.1 (Berkeley) 6/4/93
 .\" $FreeBSD$
 .\"
-.Dd June 4, 1993
+.Dd January 3, 2010
 .Dt SCANDIR 3
 .Os
 .Sh NAME
@@ -38,12 +38,11 @@
 .Sh LIBRARY
 .Lb libc
 .Sh SYNOPSIS
-.In sys/types.h
 .In dirent.h
 .Ft int
-.Fn scandir "const char *dirname" "struct dirent ***namelist" "int \\*(lp*select\\*(rp\\*(lpstruct dirent *\\*(rp" "int \\*(lp*compar\\*(rp\\*(lpconst void *, const void *\\*(rp"
+.Fn scandir "const char *dirname" "struct dirent ***namelist" "int \\*(lp*select\\*(rp\\*(lpconst struct dirent *\\*(rp" "int \\*(lp*compar\\*(rp\\*(lpconst struct dirent **, const struct dirent **\\*(rp"
 .Ft int
-.Fn alphasort "const void *d1" "const void *d2"
+.Fn alphasort "const struct dirent **d1" "const struct dirent **d2"
 .Sh DESCRIPTION
 The
 .Fn scandir
@@ -82,7 +81,8 @@ The
 function
 is a routine which can be used for the
 .Fa compar
-argument to sort the array alphabetically.
+argument to sort the array alphabetically using
+.Xr strcoll 3 .
 .Pp
 The memory allocated for the array can be deallocated with
 .Xr free 3 ,
@@ -95,7 +95,8 @@ cannot allocate enough memory to hold all the data structures.
 .Xr directory 3 ,
 .Xr malloc 3 ,
 .Xr qsort 3 ,
-.Xr dir 5
+.Xr dir 5 ,
+.Xr strcoll 3
 .Sh HISTORY
 The
 .Fn scandir
diff --git a/lib/libc/gen/scandir.c b/lib/libc/gen/scandir.c
index 1dae85d..93bc852 100644
--- a/lib/libc/gen/scandir.c
+++ b/lib/libc/gen/scandir.c
@@ -46,6 +46,8 @@ __FBSDID("$FreeBSD$");
 #include <string.h>
 #include "un-namespace.h"
 
+static int alphasort_thunk(void *thunk, const void *p1, const void *p2);
+
 /*
  * The DIRSIZ macro is the minimum record length which will hold the directory
  * entry.  This requires the amount of space in struct dirent without the
@@ -58,11 +60,9 @@ __FBSDID("$FreeBSD$");
 	    (((dp)->d_namlen + 1 + 3) &~ 3))
 
 int
-scandir(dirname, namelist, select, dcomp)
-	const char *dirname;
-	struct dirent ***namelist;
-	int (*select)(struct dirent *);
-	int (*dcomp)(const void *, const void *);
+scandir(const char *dirname, struct dirent ***namelist,
+    int (*select)(const struct dirent *), int (*dcomp)(const struct dirent **,
+	const struct dirent **))
 {
 	struct dirent *d, *p, **names = NULL;
 	size_t nitems = 0;
@@ -111,26 +111,35 @@ scandir(dirname, namelist, select, dcomp)
 	}
 	closedir(dirp);
 	if (nitems && dcomp != NULL)
-		qsort(names, nitems, sizeof(struct dirent *), dcomp);
+		qsort_r(names, nitems, sizeof(struct dirent *),
+		    &dcomp, alphasort_thunk);
 	*namelist = names;
-	return(nitems);
+	return (nitems);
 
 fail:
 	while (nitems > 0)
 		free(names[--nitems]);
 	free(names);
 	closedir(dirp);
-	return -1;
+	return (-1);
 }
 
 /*
  * Alphabetic order comparison routine for those who want it.
+ * POSIX 2008 requires that alphasort() uses strcoll().
  */
 int
-alphasort(d1, d2)
-	const void *d1;
-	const void *d2;
+alphasort(const struct dirent **d1, const struct dirent **d2)
+{
+
+	return (strcoll((*d1)->d_name, (*d2)->d_name));
+}
+
+static int
+alphasort_thunk(void *thunk, const void *p1, const void *p2)
 {
-	return(strcmp((*(struct dirent **)d1)->d_name,
-	    (*(struct dirent **)d2)->d_name));
+	int (*dc)(const struct dirent **, const struct dirent **);
+
+	dc = *(int (**)(const struct dirent **, const struct dirent **))thunk;
+	return (dc((const struct dirent **)p1, (const struct dirent **)p2));
 }
-- 
cgit v1.1


From 30e602e5e05f10f1ec2803b6f491152d8fdf8597 Mon Sep 17 00:00:00 2001
From: ed <ed@FreeBSD.org>
Date: Sun, 24 Jan 2010 14:30:57 +0000
Subject: MFC r202500:

  Fix a regression that was introduced in r191882.

  I changed login_tty() to only work when the application is not a session
  leader yet. This works fine for applications in the base system, but it
  turns out various applications call this function after daemonizing,
  which means they already use their own session.

  If setsid() fails, just call tcsetsid() on the current session.
  tcsetsid() will already perform proper security checks.

Reported by:	Oliver Lehmann
---
 lib/libutil/login_tty.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libutil/login_tty.c b/lib/libutil/login_tty.c
index a14e244..92dc87f 100644
--- a/lib/libutil/login_tty.c
+++ b/lib/libutil/login_tty.c
@@ -50,7 +50,7 @@ login_tty(int fd)
 
 	s = setsid();
 	if (s == -1)
-		return (-1);
+		s = getsid(0);
 	if (tcsetsid(fd, s) == -1)
 		return (-1);
 	(void) dup2(fd, 0);
-- 
cgit v1.1


From ae4685a7b06726fc32aff3db278a4a5b1b2c0c1a Mon Sep 17 00:00:00 2001
From: das <das@FreeBSD.org>
Date: Sun, 24 Jan 2010 20:15:59 +0000
Subject: MFC r197752:

  Better glibc compatibility for getline/getdelim:

  - Tolerate applications that pass a NULL pointer for the buffer and
    claim that the capacity of the buffer is nonzero.

  - If an application passes in a non-NULL buffer pointer and claims the
    buffer has zero capacity, we should free (well, realloc) it
    anyway. It could have been obtained from malloc(0), so failing to
    free it would be a small memory leak.
---
 lib/libc/stdio/getdelim.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/stdio/getdelim.c b/lib/libc/stdio/getdelim.c
index 7af154f..d7d5627 100644
--- a/lib/libc/stdio/getdelim.c
+++ b/lib/libc/stdio/getdelim.c
@@ -120,8 +120,8 @@ getdelim(char ** __restrict linep, size_t * __restrict linecapp, int delim,
 		goto error;
 	}
 
-	if (*linecapp == 0)
-		*linep = NULL;
+	if (*linep == NULL)
+		*linecapp = 0;
 
 	if (fp->_r <= 0 && __srefill(fp)) {
 		/* If fp is at EOF already, we just need space for the NUL. */
-- 
cgit v1.1


From 430383e8e29efa3f53ea43b953bfbbce6afd1334 Mon Sep 17 00:00:00 2001
From: emax <emax@FreeBSD.org>
Date: Tue, 26 Jan 2010 00:38:56 +0000
Subject: MFC SVN rev 198492

Fix typo in bluetooth.3
Do not use reserved C++ keyword "new"
---
 lib/libbluetooth/bluetooth.3 | 2 +-
 lib/libbluetooth/bluetooth.h | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/libbluetooth/bluetooth.3 b/lib/libbluetooth/bluetooth.3
index c4eab90..40ea20c 100644
--- a/lib/libbluetooth/bluetooth.3
+++ b/lib/libbluetooth/bluetooth.3
@@ -272,7 +272,7 @@ otherwise 0.
 .Pp
 The
 .Fn bt_devinfo
-function populates prodivded
+function populates provided
 .Vt bt_devinfo
 structure with the information about given Bluetooth device.
 The caller is expected to pass Bluetooth device name in the
diff --git a/lib/libbluetooth/bluetooth.h b/lib/libbluetooth/bluetooth.h
index 5ad3c3c..a73dbe9 100644
--- a/lib/libbluetooth/bluetooth.h
+++ b/lib/libbluetooth/bluetooth.h
@@ -163,8 +163,8 @@ int		bt_devclose(int s);
 int		bt_devsend (int s, uint16_t opcode, void *param, size_t plen);
 ssize_t		bt_devrecv (int s, void *buf, size_t size, time_t to);
 int		bt_devreq  (int s, struct bt_devreq *r, time_t to);
-int		bt_devfilter(int s, struct bt_devfilter const *new,
-			     struct bt_devfilter *old);
+int		bt_devfilter(int s, struct bt_devfilter const *newp,
+			     struct bt_devfilter *oldp);
 void		bt_devfilter_pkt_set(struct bt_devfilter *filter, uint8_t type);
 void		bt_devfilter_pkt_clr(struct bt_devfilter *filter, uint8_t type);
 int		bt_devfilter_pkt_tst(struct bt_devfilter const *filter, uint8_t type);
-- 
cgit v1.1


From 55f26a2e9c5ce78a54d723f154a436053f46cd3e Mon Sep 17 00:00:00 2001
From: des <des@FreeBSD.org>
Date: Tue, 26 Jan 2010 14:15:12 +0000
Subject: MFH (r202613, r202623): HTTP digest authentication support.

---
 lib/libfetch/Makefile |   7 +-
 lib/libfetch/http.c   | 882 ++++++++++++++++++++++++++++++++++++++++++++++----
 2 files changed, 821 insertions(+), 68 deletions(-)

(limited to 'lib')

diff --git a/lib/libfetch/Makefile b/lib/libfetch/Makefile
index 3576db7..5091adf 100644
--- a/lib/libfetch/Makefile
+++ b/lib/libfetch/Makefile
@@ -16,8 +16,11 @@ CFLAGS+=	-DINET6
 
 .if ${MK_OPENSSL} != "no"
 CFLAGS+=	-DWITH_SSL
-DPADD=		${LIBSSL} ${LIBCRYPTO}
-LDADD=		-lssl -lcrypto
+DPADD=		${LIBSSL} ${LIBCRYPTO} ${LIBMD}
+LDADD=		-lssl -lcrypto -lmd
+.else
+DPADD=		${LIBMD}
+LDADD=		-lmd
 .endif
 
 CFLAGS+=	-DFTP_COMBINE_CWDS
diff --git a/lib/libfetch/http.c b/lib/libfetch/http.c
index dd98349..f44366e 100644
--- a/lib/libfetch/http.c
+++ b/lib/libfetch/http.c
@@ -76,6 +76,7 @@ __FBSDID("$FreeBSD$");
 #include <string.h>
 #include <time.h>
 #include <unistd.h>
+#include <md5.h>
 
 #include <netinet/in.h>
 #include <netinet/tcp.h>
@@ -343,7 +344,8 @@ typedef enum {
 	hdr_last_modified,
 	hdr_location,
 	hdr_transfer_encoding,
-	hdr_www_authenticate
+	hdr_www_authenticate,
+	hdr_proxy_authenticate,
 } hdr_t;
 
 /* Names of interesting headers */
@@ -357,6 +359,7 @@ static struct {
 	{ hdr_location,			"Location" },
 	{ hdr_transfer_encoding,	"Transfer-Encoding" },
 	{ hdr_www_authenticate,		"WWW-Authenticate" },
+	{ hdr_proxy_authenticate,	"Proxy-Authenticate" },
 	{ hdr_unknown,			NULL },
 };
 
@@ -446,21 +449,114 @@ http_match(const char *str, const char *hdr)
 	return (hdr);
 }
 
+
 /*
- * Get the next header and return the appropriate symbolic code.
+ * Get the next header and return the appropriate symbolic code.  We
+ * need to read one line ahead for checking for a continuation line
+ * belonging to the current header (continuation lines start with
+ * white space). 
+ *
+ * We get called with a fresh line already in the conn buffer, either
+ * from the previous http_next_header() invocation, or, the first
+ * time, from a fetch_getln() performed by our caller.
+ *
+ * This stops when we encounter an empty line (we dont read beyond the header
+ * area).
+ * 
+ * Note that the "headerbuf" is just a place to return the result. Its
+ * contents are not used for the next call. This means that no cleanup
+ * is needed when ie doing another connection, just call the cleanup when
+ * fully done to deallocate memory.
  */
-static hdr_t
-http_next_header(conn_t *conn, const char **p)
+
+/* Limit the max number of continuation lines to some reasonable value */
+#define HTTP_MAX_CONT_LINES 10
+
+/* Place into which to build a header from one or several lines */
+typedef struct {
+	char	*buf;		/* buffer */
+	size_t	 bufsize;	/* buffer size */
+	size_t	 buflen;	/* length of buffer contents */
+} http_headerbuf_t;
+
+static void
+init_http_headerbuf(http_headerbuf_t *buf)
 {
-	int i;
+	buf->buf = NULL;
+	buf->bufsize = 0;
+	buf->buflen = 0;
+}
 
-	if (fetch_getln(conn) == -1)
-		return (hdr_syserror);
-	while (conn->buflen && isspace((unsigned char)conn->buf[conn->buflen - 1]))
+static void 
+clean_http_headerbuf(http_headerbuf_t *buf)
+{
+	if (buf->buf)
+		free(buf->buf);
+	init_http_headerbuf(buf);
+}
+
+/* Remove whitespace at the end of the buffer */
+static void 
+http_conn_trimright(conn_t *conn)
+{
+	while (conn->buflen && 
+	       isspace((unsigned char)conn->buf[conn->buflen - 1]))
 		conn->buflen--;
 	conn->buf[conn->buflen] = '\0';
+}
+
+static hdr_t
+http_next_header(conn_t *conn, http_headerbuf_t *hbuf, const char **p)
+{
+	int i, len;
+
+	/* 
+	 * Have to do the stripping here because of the first line. So
+	 * it's done twice for the subsequent lines. No big deal 
+	 */
+	http_conn_trimright(conn);
 	if (conn->buflen == 0)
 		return (hdr_end);
+
+	/* Copy the line to the headerbuf */
+	if (hbuf->bufsize < conn->buflen + 1) {
+		if ((hbuf->buf = realloc(hbuf->buf, conn->buflen + 1)) == NULL)
+			return (hdr_syserror);
+		hbuf->bufsize = conn->buflen + 1;
+	}
+	strcpy(hbuf->buf, conn->buf);
+	hbuf->buflen = conn->buflen;
+
+	/* 
+	 * Fetch possible continuation lines. Stop at 1st non-continuation
+	 * and leave it in the conn buffer 
+         */
+	for (i = 0; i < HTTP_MAX_CONT_LINES; i++) {
+		if (fetch_getln(conn) == -1)
+			return (hdr_syserror);
+
+		/* 
+		 * Note: we carry on the idea from the previous version
+		 * that a pure whitespace line is equivalent to an empty
+		 * one (so it's not continuation and will be handled when
+		 * we are called next) 
+		 */
+		http_conn_trimright(conn);
+		if (conn->buf[0] != ' ' && conn->buf[0] != "\t"[0])
+			break;
+
+		/* Got a continuation line. Concatenate to previous */
+		len = hbuf->buflen + conn->buflen;
+		if (hbuf->bufsize < len + 1) {
+			len *= 2;
+			if ((hbuf->buf = realloc(hbuf->buf, len + 1)) == NULL)
+				return (hdr_syserror);
+			hbuf->bufsize = len + 1;
+		}
+		strcpy(hbuf->buf + hbuf->buflen, conn->buf);
+		hbuf->buflen += conn->buflen;
+	} 
+
 	/*
 	 * We could check for malformed headers but we don't really care.
 	 * A valid header starts with a token immediately followed by a
@@ -468,11 +564,290 @@ http_next_header(conn_t *conn, const char **p)
 	 * characters except "()<>@,;:\\\"{}".
 	 */
 	for (i = 0; hdr_names[i].num != hdr_unknown; i++)
-		if ((*p = http_match(hdr_names[i].name, conn->buf)) != NULL)
+		if ((*p = http_match(hdr_names[i].name, hbuf->buf)) != NULL)
 			return (hdr_names[i].num);
+
 	return (hdr_unknown);
 }
 
+/**************************
+ * [Proxy-]Authenticate header parsing
+ */
+
+/* 
+ * Read doublequote-delimited string into output buffer obuf (allocated 
+ * by caller, whose responsibility it is to ensure that it's big enough)
+ * cp points to the first char after the initial '"'
+ * Handles \ quoting 
+ * Returns pointer to the first char after the terminating double quote, or 
+ * NULL for error.
+ */
+static const char *
+http_parse_headerstring(const char *cp, char *obuf)
+{
+	for (;;) {
+		switch (*cp) {
+		case 0: /* Unterminated string */
+			*obuf = 0;
+			return (NULL);
+		case '"': /* Ending quote */
+			*obuf = 0;
+			return (++cp);
+		case '\\':
+			if (*++cp == 0) {
+				*obuf = 0;
+				return (NULL);
+			}
+			/* FALLTHROUGH */
+		default:
+			*obuf++ = *cp++;
+		}
+	}
+}
+
+/* Http auth challenge schemes */
+typedef enum {HTTPAS_UNKNOWN, HTTPAS_BASIC,HTTPAS_DIGEST} http_auth_schemes_t;
+
+/* Data holder for a Basic or Digest challenge. */
+typedef struct {
+	http_auth_schemes_t scheme;
+	char	*realm;
+	char	*qop;
+	char	*nonce;
+	char	*opaque;
+	char	*algo;
+	int	 stale;
+	int	 nc; /* Nonce count */
+} http_auth_challenge_t;
+
+static void 
+init_http_auth_challenge(http_auth_challenge_t *b)
+{
+	b->scheme = HTTPAS_UNKNOWN;
+	b->realm = b->qop = b->nonce = b->opaque = b->algo = NULL;
+	b->stale = b->nc = 0;
+}
+
+static void 
+clean_http_auth_challenge(http_auth_challenge_t *b)
+{
+	if (b->realm) 
+		free(b->realm);
+	if (b->qop) 
+		free(b->qop);
+	if (b->nonce) 
+		free(b->nonce);
+	if (b->opaque) 
+		free(b->opaque);
+	if (b->algo) 
+		free(b->algo);
+	init_http_auth_challenge(b);
+}
+
+/* Data holder for an array of challenges offered in an http response. */
+#define MAX_CHALLENGES 10
+typedef struct {
+	http_auth_challenge_t *challenges[MAX_CHALLENGES];
+	int	count; /* Number of parsed challenges in the array */
+	int	valid; /* We did parse an authenticate header */
+} http_auth_challenges_t;
+
+static void 
+init_http_auth_challenges(http_auth_challenges_t *cs)
+{
+	int i;
+	for (i = 0; i < MAX_CHALLENGES; i++)
+		cs->challenges[i] = NULL;
+	cs->count = cs->valid = 0;
+}
+
+static void 
+clean_http_auth_challenges(http_auth_challenges_t *cs)
+{
+	int i;
+	/* We rely on non-zero pointers being allocated, not on the count */
+	for (i = 0; i < MAX_CHALLENGES; i++) {
+		if (cs->challenges[i] != NULL) {
+			clean_http_auth_challenge(cs->challenges[i]);
+			free(cs->challenges[i]);
+		}
+	}
+	init_http_auth_challenges(cs);
+}
+
+/* 
+ * Enumeration for lexical elements. Separators will be returned as their own
+ * ascii value
+ */
+typedef enum {HTTPHL_WORD=256, HTTPHL_STRING=257, HTTPHL_END=258,
+	      HTTPHL_ERROR = 259} http_header_lex_t;
+
+/* 
+ * Determine what kind of token comes next and return possible value
+ * in buf, which is supposed to have been allocated big enough by
+ * caller. Advance input pointer and return element type. 
+ */
+static int 
+http_header_lex(const char **cpp, char *buf)
+{
+	size_t l;
+	/* Eat initial whitespace */
+	*cpp += strspn(*cpp, " \t");
+	if (**cpp == 0)
+		return (HTTPHL_END);
+
+	/* Separator ? */
+	if (**cpp == ',' || **cpp == '=')
+		return (*((*cpp)++));
+
+	/* String ? */
+	if (**cpp == '"') {
+		*cpp = http_parse_headerstring(++*cpp, buf);
+		if (*cpp == NULL)
+			return (HTTPHL_ERROR);
+		return (HTTPHL_STRING);
+	}
+
+	/* Read other token, until separator or whitespace */
+	l = strcspn(*cpp, " \t,=");
+	memcpy(buf, *cpp, l);
+	buf[l] = 0;
+	*cpp += l;
+	return (HTTPHL_WORD);
+}
+
+/* 
+ * Read challenges from http xxx-authenticate header and accumulate them
+ * in the challenges list structure.
+ *
+ * Headers with multiple challenges are specified by rfc2617, but
+ * servers (ie: squid) often send them in separate headers instead,
+ * which in turn is forbidden by the http spec (multiple headers with
+ * the same name are only allowed for pure comma-separated lists, see
+ * rfc2616 sec 4.2).
+ *
+ * We support both approaches anyway
+ */
+static int 
+http_parse_authenticate(const char *cp, http_auth_challenges_t *cs)
+{
+	int ret = -1;
+	http_header_lex_t lex;
+	char *key = malloc(strlen(cp) + 1);
+	char *value = malloc(strlen(cp) + 1);
+	char *buf = malloc(strlen(cp) + 1);
+
+	if (key == NULL || value == NULL || buf == NULL) {
+		fetch_syserr();
+		goto out;
+	}
+
+	/* In any case we've seen the header and we set the valid bit */
+	cs->valid = 1;
+
+	/* Need word first */
+	lex = http_header_lex(&cp, key);
+	if (lex != HTTPHL_WORD)
+		goto out;
+
+	/* Loop on challenges */
+	for (; cs->count < MAX_CHALLENGES; cs->count++) {
+		cs->challenges[cs->count] = 
+			malloc(sizeof(http_auth_challenge_t));
+		if (cs->challenges[cs->count] == NULL) {
+			fetch_syserr();
+			goto out;
+		}
+		init_http_auth_challenge(cs->challenges[cs->count]);
+		if (!strcasecmp(key, "basic")) {
+			cs->challenges[cs->count]->scheme = HTTPAS_BASIC;
+		} else if (!strcasecmp(key, "digest")) {
+			cs->challenges[cs->count]->scheme = HTTPAS_DIGEST;
+		} else {
+			cs->challenges[cs->count]->scheme = HTTPAS_UNKNOWN;
+			/* 
+                         * Continue parsing as basic or digest may
+			 * follow, and the syntax is the same for
+			 * all. We'll just ignore this one when
+			 * looking at the list
+			 */
+		}
+	
+		/* Loop on attributes */
+		for (;;) {
+			/* Key */
+			lex = http_header_lex(&cp, key);
+			if (lex != HTTPHL_WORD)
+				goto out;
+
+			/* Equal sign */
+			lex = http_header_lex(&cp, buf);
+			if (lex != '=')
+				goto out;
+
+			/* Value */
+			lex = http_header_lex(&cp, value);
+			if (lex != HTTPHL_WORD && lex != HTTPHL_STRING)
+				goto out;
+
+			if (!strcasecmp(key, "realm"))
+				cs->challenges[cs->count]->realm = 
+					strdup(value);
+			else if (!strcasecmp(key, "qop"))
+				cs->challenges[cs->count]->qop = 
+					strdup(value);
+			else if (!strcasecmp(key, "nonce"))
+				cs->challenges[cs->count]->nonce = 
+					strdup(value);
+			else if (!strcasecmp(key, "opaque"))
+				cs->challenges[cs->count]->opaque = 
+					strdup(value);
+			else if (!strcasecmp(key, "algorithm"))
+				cs->challenges[cs->count]->algo = 
+					strdup(value);
+			else if (!strcasecmp(key, "stale"))
+				cs->challenges[cs->count]->stale = 
+					strcasecmp(value, "no");
+			/* Else ignore unknown attributes */
+
+			/* Comma or Next challenge or End */
+			lex = http_header_lex(&cp, key);
+			/* 
+                         * If we get a word here, this is the beginning of the
+			 * next challenge. Break the attributes loop 
+                         */
+			if (lex == HTTPHL_WORD)
+				break;
+
+			if (lex == HTTPHL_END) {
+				/* End while looking for ',' is normal exit */
+				cs->count++;
+				ret = 0;
+				goto out;
+			}
+			/* Anything else is an error */
+			if (lex != ',')
+				goto out;
+
+		} /* End attributes loop */
+	} /* End challenge loop */
+
+	/* 
+         * Challenges max count exceeded. This really can't happen
+	 * with normal data, something's fishy -> error 
+	 */ 
+
+out:
+	if (key)
+		free(key);
+	if (value)
+		free(value);
+	if (buf)
+		free(buf);
+	return (ret);
+}
+
+
 /*
  * Parse a last-modified header
  */
@@ -618,6 +993,291 @@ http_base64(const char *src)
 	return (str);
 }
 
+
+/*
+ * Extract authorization parameters from environment value.
+ * The value is like scheme:realm:user:pass
+ */
+typedef struct {
+	char	*scheme;
+	char	*realm;
+	char	*user;
+	char	*password;
+} http_auth_params_t;
+
+static void
+init_http_auth_params(http_auth_params_t *s)
+{
+	s->scheme = s->realm = s->user = s->password = 0;
+}
+
+static void 
+clean_http_auth_params(http_auth_params_t *s)
+{
+	if (s->scheme) 
+		free(s->scheme);
+	if (s->realm) 
+		free(s->realm);
+	if (s->user) 
+		free(s->user);
+	if (s->password) 
+		free(s->password);
+	init_http_auth_params(s);
+}
+
+static int
+http_authfromenv(const char *p, http_auth_params_t *parms)
+{
+	int ret = -1;
+	char *v, *ve;
+	char *str = strdup(p);
+
+	if (str == NULL) {
+		fetch_syserr();
+		return (-1);
+	}
+	v = str;
+
+	if ((ve = strchr(v, ':')) == NULL)
+		goto out;
+
+	*ve = 0;
+	if ((parms->scheme = strdup(v)) == NULL) {
+		fetch_syserr();
+		goto out;
+	}
+	v = ve + 1;
+
+	if ((ve = strchr(v, ':')) == NULL)
+		goto out;
+
+	*ve = 0;
+	if ((parms->realm = strdup(v)) == NULL) {
+		fetch_syserr();
+		goto out;
+	}
+	v = ve + 1;
+
+	if ((ve = strchr(v, ':')) == NULL)
+		goto out;
+
+	*ve = 0;
+	if ((parms->user = strdup(v)) == NULL) {
+		fetch_syserr();
+		goto out;
+	}
+	v = ve + 1;
+
+
+	if ((parms->password = strdup(v)) == NULL) {
+		fetch_syserr();
+		goto out;
+	}
+	ret = 0;
+out:
+	if (ret == -1) 
+		clean_http_auth_params(parms);
+	if (str)
+		free(str);
+	return (ret);
+}
+
+
+/* 
+ * Digest response: the code to compute the digest is taken from the
+ * sample implementation in RFC2616 
+ */
+#define IN
+#define OUT
+
+#define HASHLEN 16
+typedef char HASH[HASHLEN];
+#define HASHHEXLEN 32
+typedef char HASHHEX[HASHHEXLEN+1];
+
+static const char *hexchars = "0123456789abcdef";
+static void 
+CvtHex(IN HASH Bin, OUT HASHHEX Hex)
+{
+	unsigned short i;
+	unsigned char j;
+
+	for (i = 0; i < HASHLEN; i++) {
+		j = (Bin[i] >> 4) & 0xf;
+		Hex[i*2] = hexchars[j];
+		j = Bin[i] & 0xf;
+		Hex[i*2+1] = hexchars[j];
+	};
+	Hex[HASHHEXLEN] = '\0';
+};
+
+/* calculate H(A1) as per spec */
+static void 
+DigestCalcHA1(
+	IN char * pszAlg,
+	IN char * pszUserName,
+	IN char * pszRealm,
+	IN char * pszPassword,
+	IN char * pszNonce,
+	IN char * pszCNonce,
+	OUT HASHHEX SessionKey
+	)
+{
+	MD5_CTX Md5Ctx;
+	HASH HA1;
+
+	MD5Init(&Md5Ctx);
+	MD5Update(&Md5Ctx, pszUserName, strlen(pszUserName));
+	MD5Update(&Md5Ctx, ":", 1);
+	MD5Update(&Md5Ctx, pszRealm, strlen(pszRealm));
+	MD5Update(&Md5Ctx, ":", 1);
+	MD5Update(&Md5Ctx, pszPassword, strlen(pszPassword));
+	MD5Final(HA1, &Md5Ctx);
+	if (strcasecmp(pszAlg, "md5-sess") == 0) {
+
+		MD5Init(&Md5Ctx);
+		MD5Update(&Md5Ctx, HA1, HASHLEN);
+		MD5Update(&Md5Ctx, ":", 1);
+		MD5Update(&Md5Ctx, pszNonce, strlen(pszNonce));
+		MD5Update(&Md5Ctx, ":", 1);
+		MD5Update(&Md5Ctx, pszCNonce, strlen(pszCNonce));
+		MD5Final(HA1, &Md5Ctx);
+	};
+	CvtHex(HA1, SessionKey);
+}
+
+/* calculate request-digest/response-digest as per HTTP Digest spec */
+static void 
+DigestCalcResponse(
+	IN HASHHEX HA1,           /* H(A1) */
+	IN char * pszNonce,       /* nonce from server */
+	IN char * pszNonceCount,  /* 8 hex digits */
+	IN char * pszCNonce,      /* client nonce */
+	IN char * pszQop,         /* qop-value: "", "auth", "auth-int" */
+	IN char * pszMethod,      /* method from the request */
+	IN char * pszDigestUri,   /* requested URL */
+	IN HASHHEX HEntity,       /* H(entity body) if qop="auth-int" */
+	OUT HASHHEX Response      /* request-digest or response-digest */
+	)
+{
+/*	DEBUG(fprintf(stderr, 
+		      "Calc: HA1[%s] Nonce[%s] qop[%s] method[%s] URI[%s]\n",
+		      HA1, pszNonce, pszQop, pszMethod, pszDigestUri));*/
+	MD5_CTX Md5Ctx;
+	HASH HA2;
+	HASH RespHash;
+	HASHHEX HA2Hex;
+
+	// calculate H(A2)
+	MD5Init(&Md5Ctx);
+	MD5Update(&Md5Ctx, pszMethod, strlen(pszMethod));
+	MD5Update(&Md5Ctx, ":", 1);
+	MD5Update(&Md5Ctx, pszDigestUri, strlen(pszDigestUri));
+	if (strcasecmp(pszQop, "auth-int") == 0) {
+		MD5Update(&Md5Ctx, ":", 1);
+		MD5Update(&Md5Ctx, HEntity, HASHHEXLEN);
+	};
+	MD5Final(HA2, &Md5Ctx);
+	CvtHex(HA2, HA2Hex);
+
+	// calculate response
+	MD5Init(&Md5Ctx);
+	MD5Update(&Md5Ctx, HA1, HASHHEXLEN);
+	MD5Update(&Md5Ctx, ":", 1);
+	MD5Update(&Md5Ctx, pszNonce, strlen(pszNonce));
+	MD5Update(&Md5Ctx, ":", 1);
+	if (*pszQop) {
+		MD5Update(&Md5Ctx, pszNonceCount, strlen(pszNonceCount));
+		MD5Update(&Md5Ctx, ":", 1);
+		MD5Update(&Md5Ctx, pszCNonce, strlen(pszCNonce));
+		MD5Update(&Md5Ctx, ":", 1);
+		MD5Update(&Md5Ctx, pszQop, strlen(pszQop));
+		MD5Update(&Md5Ctx, ":", 1);
+	};
+	MD5Update(&Md5Ctx, HA2Hex, HASHHEXLEN);
+	MD5Final(RespHash, &Md5Ctx);
+	CvtHex(RespHash, Response);
+}
+
+/* 
+ * Generate/Send a Digest authorization header 
+ * This looks like: [Proxy-]Authorization: credentials
+ *
+ *  credentials      = "Digest" digest-response
+ *  digest-response  = 1#( username | realm | nonce | digest-uri
+ *                      | response | [ algorithm ] | [cnonce] |
+ *                      [opaque] | [message-qop] |
+ *                          [nonce-count]  | [auth-param] )
+ *  username         = "username" "=" username-value
+ *  username-value   = quoted-string
+ *  digest-uri       = "uri" "=" digest-uri-value
+ *  digest-uri-value = request-uri   ; As specified by HTTP/1.1
+ *  message-qop      = "qop" "=" qop-value
+ *  cnonce           = "cnonce" "=" cnonce-value
+ *  cnonce-value     = nonce-value
+ *  nonce-count      = "nc" "=" nc-value
+ *  nc-value         = 8LHEX
+ *  response         = "response" "=" request-digest
+ *  request-digest = <"> 32LHEX <">
+ */
+static int
+http_digest_auth(conn_t *conn, const char *hdr, http_auth_challenge_t *c,
+		 http_auth_params_t *parms, struct url *url)
+{
+	int r;
+	char noncecount[10];
+	char cnonce[40];
+	char *options = 0;
+
+	if (!c->realm || !c->nonce) {
+		DEBUG(fprintf(stderr, "realm/nonce not set in challenge\n"));
+		return(-1);
+	}
+	if (!c->algo) 
+		c->algo = strdup("");
+
+	if (asprintf(&options, "%s%s%s%s", 
+		     *c->algo? ",algorithm=" : "", c->algo,
+		     c->opaque? ",opaque=" : "", c->opaque?c->opaque:"")== -1)
+		return (-1);
+
+	if (!c->qop) {
+		c->qop = strdup("");
+		*noncecount = 0;
+		*cnonce = 0;
+	} else {
+		c->nc++;
+		sprintf(noncecount, "%08x", c->nc);
+		/* We don't try very hard with the cnonce ... */
+		sprintf(cnonce, "%x%lx", getpid(), (unsigned long)time(0));
+	}
+
+	HASHHEX HA1;
+	DigestCalcHA1(c->algo, parms->user, c->realm,
+		      parms->password, c->nonce, cnonce, HA1);
+	DEBUG(fprintf(stderr, "HA1: [%s]\n", HA1));
+	HASHHEX digest;
+	DigestCalcResponse(HA1, c->nonce, noncecount, cnonce, c->qop,
+			   "GET", url->doc, "", digest);
+
+	if (c->qop[0]) {
+		r = http_cmd(conn, "%s: Digest username=\"%s\",realm=\"%s\","
+			     "nonce=\"%s\",uri=\"%s\",response=\"%s\","
+			     "qop=\"auth\", cnonce=\"%s\", nc=%s%s",
+			     hdr, parms->user, c->realm, 
+			     c->nonce, url->doc, digest,
+			     cnonce, noncecount, options);
+	} else {
+		r = http_cmd(conn, "%s: Digest username=\"%s\",realm=\"%s\","
+			     "nonce=\"%s\",uri=\"%s\",response=\"%s\"%s",
+			     hdr, parms->user, c->realm, 
+			     c->nonce, url->doc, digest, options);
+	}
+	if (options)
+		free(options);
+	return (r);
+}
+
 /*
  * Encode username and password
  */
@@ -627,8 +1287,8 @@ http_basic_auth(conn_t *conn, const char *hdr, const char *usr, const char *pwd)
 	char *upw, *auth;
 	int r;
 
-	DEBUG(fprintf(stderr, "usr: [%s]\n", usr));
-	DEBUG(fprintf(stderr, "pwd: [%s]\n", pwd));
+	DEBUG(fprintf(stderr, "basic: usr: [%s]\n", usr));
+	DEBUG(fprintf(stderr, "basic: pwd: [%s]\n", pwd));
 	if (asprintf(&upw, "%s:%s", usr, pwd) == -1)
 		return (-1);
 	auth = http_base64(upw);
@@ -641,33 +1301,49 @@ http_basic_auth(conn_t *conn, const char *hdr, const char *usr, const char *pwd)
 }
 
 /*
- * Send an authorization header
+ * Chose the challenge to answer and call the appropriate routine to 
+ * produce the header.
  */
 static int
-http_authorize(conn_t *conn, const char *hdr, const char *p)
+http_authorize(conn_t *conn, const char *hdr, http_auth_challenges_t *cs,
+	       http_auth_params_t *parms, struct url *url)
 {
-	/* basic authorization */
-	if (strncasecmp(p, "basic:", 6) == 0) {
-		char *user, *pwd, *str;
-		int r;
-
-		/* skip realm */
-		for (p += 6; *p && *p != ':'; ++p)
-			/* nothing */ ;
-		if (!*p || strchr(++p, ':') == NULL)
-			return (-1);
-		if ((str = strdup(p)) == NULL)
-			return (-1); /* XXX */
-		user = str;
-		pwd = strchr(str, ':');
-		*pwd++ = '\0';
-		r = http_basic_auth(conn, hdr, user, pwd);
-		free(str);
-		return (r);
+	http_auth_challenge_t *basic = NULL;
+	http_auth_challenge_t *digest = NULL;
+	int i;
+
+	/* If user or pass are null we're not happy */
+	if (!parms->user || !parms->password) {
+		DEBUG(fprintf(stderr, "NULL usr or pass\n"));
+		return (-1);
+	}
+
+	/* Look for a Digest and a Basic challenge */
+	for (i = 0; i < cs->count; i++) {
+		if (cs->challenges[i]->scheme == HTTPAS_BASIC)
+			basic = cs->challenges[i];
+		if (cs->challenges[i]->scheme == HTTPAS_DIGEST)
+			digest = cs->challenges[i];
 	}
-	return (-1);
-}
 
+	/* Error if "Digest" was specified and there is no Digest challenge */
+	if (!digest && (parms->scheme && 
+			!strcasecmp(parms->scheme, "digest"))) {
+		DEBUG(fprintf(stderr, 
+			      "Digest auth in env, not supported by peer\n"));
+		return (-1);
+	}
+	/* 
+         * If "basic" was specified in the environment, or there is no Digest
+	 * challenge, do the basic thing. Don't need a challenge for this,
+	 * so no need to check basic!=NULL 
+	 */
+	if (!digest || (parms->scheme && !strcasecmp(parms->scheme,"basic")))
+		return (http_basic_auth(conn,hdr,parms->user,parms->password));
+
+	/* Else, prefer digest. We just checked that it's not NULL */
+	return (http_digest_auth(conn, hdr, digest, parms, url));
+}
 
 /*****************************************************************************
  * Helper functions for connecting to a server or proxy
@@ -797,13 +1473,13 @@ http_print_html(FILE *out, FILE *in)
  */
 FILE *
 http_request(struct url *URL, const char *op, struct url_stat *us,
-    struct url *purl, const char *flags)
+	struct url *purl, const char *flags)
 {
 	char timebuf[80];
 	char hbuf[MAXHOSTNAMELEN + 7], *host;
 	conn_t *conn;
 	struct url *url, *new;
-	int chunked, direct, ims, need_auth, noredirect, verbose;
+	int chunked, direct, ims, noredirect, verbose;
 	int e, i, n, val;
 	off_t offset, clength, length, size;
 	time_t mtime;
@@ -811,6 +1487,14 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
 	FILE *f;
 	hdr_t h;
 	struct tm *timestruct;
+	http_headerbuf_t headerbuf;
+	http_auth_challenges_t server_challenges;
+	http_auth_challenges_t proxy_challenges;
+
+	/* The following calls don't allocate anything */
+	init_http_headerbuf(&headerbuf); 
+	init_http_auth_challenges(&server_challenges);
+	init_http_auth_challenges(&proxy_challenges);
 
 	direct = CHECK_FLAG('d');
 	noredirect = CHECK_FLAG('A');
@@ -830,7 +1514,6 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
 	i = 0;
 
 	e = HTTP_PROTOCOL_ERROR;
-	need_auth = 0;
 	do {
 		new = NULL;
 		chunked = 0;
@@ -895,27 +1578,67 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
 		/* virtual host */
 		http_cmd(conn, "Host: %s", host);
 
-		/* proxy authorization */
-		if (purl) {
-			if (*purl->user || *purl->pwd)
-				http_basic_auth(conn, "Proxy-Authorization",
-				    purl->user, purl->pwd);
-			else if ((p = getenv("HTTP_PROXY_AUTH")) != NULL && *p != '\0')
-				http_authorize(conn, "Proxy-Authorization", p);
+		/* 
+                 * Proxy authorization: we only send auth after we received
+		 * a 407 error. We do not first try basic anyway (changed 
+                 * when support was added for digest-auth)
+                 */
+		if (purl && proxy_challenges.valid) {
+			http_auth_params_t aparams;
+			init_http_auth_params(&aparams);
+			if (*purl->user || *purl->pwd) {
+				aparams.user = purl->user ? 
+					strdup(purl->user) : strdup("");
+				aparams.password = purl->pwd?
+					strdup(purl->pwd) : strdup("");
+			} else if ((p = getenv("HTTP_PROXY_AUTH")) != NULL && 
+				   *p != '\0') {
+				if (http_authfromenv(p, &aparams) < 0) {
+					http_seterr(HTTP_NEED_PROXY_AUTH);
+					goto ouch;
+				}
+			}
+			http_authorize(conn, "Proxy-Authorization", 
+				       &proxy_challenges, &aparams, url);
+			clean_http_auth_params(&aparams);
 		}
 
-		/* server authorization */
-		if (need_auth || *url->user || *url->pwd) {
-			if (*url->user || *url->pwd)
-				http_basic_auth(conn, "Authorization", url->user, url->pwd);
-			else if ((p = getenv("HTTP_AUTH")) != NULL && *p != '\0')
-				http_authorize(conn, "Authorization", p);
-			else if (fetchAuthMethod && fetchAuthMethod(url) == 0) {
-				http_basic_auth(conn, "Authorization", url->user, url->pwd);
+		/* 
+                 * Server authorization: we never send "a priori"
+		 * Basic auth, which used to be done if user/pass were
+		 * set in the url. This would be weird because we'd send the
+		 * password in the clear even if Digest is finally to be 
+		 * used (it would have made more sense for the
+		 * pre-digest version to do this when Basic was specified 
+                 * in the environment) 
+                 */
+		if (server_challenges.valid) {
+			http_auth_params_t aparams;
+			init_http_auth_params(&aparams);
+			if (*url->user || *url->pwd) {
+				aparams.user = url->user ? 
+					strdup(url->user) : strdup("");
+				aparams.password = url->pwd ? 
+					strdup(url->pwd) : strdup("");
+			} else if ((p = getenv("HTTP_AUTH")) != NULL && 
+				   *p != '\0') {
+				if (http_authfromenv(p, &aparams) < 0) {
+					http_seterr(HTTP_NEED_AUTH);
+					goto ouch;
+				}
+			} else if (fetchAuthMethod && 
+				   fetchAuthMethod(url) == 0) {
+				aparams.user = url->user ? 
+					strdup(url->user) : strdup("");
+				aparams.password = url->pwd ? 
+					strdup(url->pwd) : strdup("");
 			} else {
 				http_seterr(HTTP_NEED_AUTH);
 				goto ouch;
 			}
+			http_authorize(conn, "Authorization", 
+				       &server_challenges, &aparams, url);
+			clean_http_auth_params(&aparams);
 		}
 
 		/* other headers */
@@ -965,7 +1688,7 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
 			 */
 			break;
 		case HTTP_NEED_AUTH:
-			if (need_auth) {
+			if (server_challenges.valid) {
 				/*
 				 * We already sent out authorization code,
 				 * so there's nothing more we can do.
@@ -978,13 +1701,18 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
 				fetch_info("server requires authorization");
 			break;
 		case HTTP_NEED_PROXY_AUTH:
-			/*
-			 * If we're talking to a proxy, we already sent
-			 * our proxy authorization code, so there's
-			 * nothing more we can do.
-			 */
-			http_seterr(conn->err);
-			goto ouch;
+			if (proxy_challenges.valid) {
+				/*
+				 * We already sent our proxy
+				 * authorization code, so there's
+				 * nothing more we can do. */
+				http_seterr(conn->err);
+				goto ouch;
+			}
+			/* try again, but send the password this time */
+			if (verbose)
+				fetch_info("proxy requires authorization");
+			break;
 		case HTTP_BAD_RANGE:
 			/*
 			 * This can happen if we ask for 0 bytes because
@@ -1004,9 +1732,13 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
 			/* fall through so we can get the full error message */
 		}
 
-		/* get headers */
+		/* get headers. http_next_header expects one line readahead */
+		if (fetch_getln(conn) == -1) {
+		    fetch_syserr();
+		    goto ouch;
+		}
 		do {
-			switch ((h = http_next_header(conn, &p))) {
+		    switch ((h = http_next_header(conn, &headerbuf, &p))) {
 			case hdr_syserror:
 				fetch_syserr();
 				goto ouch;
@@ -1054,7 +1786,12 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
 			case hdr_www_authenticate:
 				if (conn->err != HTTP_NEED_AUTH)
 					break;
-				/* if we were smarter, we'd check the method and realm */
+				http_parse_authenticate(p, &server_challenges);
+				break;
+			case hdr_proxy_authenticate:
+				if (conn->err != HTTP_NEED_PROXY_AUTH)
+					break;
+				http_parse_authenticate(p, &proxy_challenges);
 				break;
 			case hdr_end:
 				/* fall through */
@@ -1065,9 +1802,17 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
 		} while (h > hdr_end);
 
 		/* we need to provide authentication */
-		if (conn->err == HTTP_NEED_AUTH) {
+		if (conn->err == HTTP_NEED_AUTH || 
+		    conn->err == HTTP_NEED_PROXY_AUTH) {
 			e = conn->err;
-			need_auth = 1;
+			if ((conn->err == HTTP_NEED_AUTH && 
+			     !server_challenges.valid) || 
+			    (conn->err == HTTP_NEED_PROXY_AUTH && 
+			     !proxy_challenges.valid)) {
+				/* 401/7 but no www/proxy-authenticate ?? */
+				DEBUG(fprintf(stderr, "401/7 and no auth header\n"));
+				goto ouch;
+			}
 			fetch_close(conn);
 			conn = NULL;
 			continue;
@@ -1096,7 +1841,7 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
 
 		/* all other cases: we got a redirect */
 		e = conn->err;
-		need_auth = 0;
+		clean_http_auth_challenges(&server_challenges);
 		fetch_close(conn);
 		conn = NULL;
 		if (!new) {
@@ -1172,7 +1917,9 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
 		fclose(f);
 		f = NULL;
 	}
-
+	clean_http_headerbuf(&headerbuf);
+	clean_http_auth_challenges(&server_challenges);
+	clean_http_auth_challenges(&proxy_challenges);
 	return (f);
 
 ouch:
@@ -1182,6 +1929,9 @@ ouch:
 		fetchFreeURL(purl);
 	if (conn != NULL)
 		fetch_close(conn);
+	clean_http_headerbuf(&headerbuf);
+	clean_http_auth_challenges(&server_challenges);
+	clean_http_auth_challenges(&proxy_challenges);
 	return (NULL);
 }
 
-- 
cgit v1.1


From a1b044b56513afcbb4f8cad16d16aa448177373c Mon Sep 17 00:00:00 2001
From: des <des@FreeBSD.org>
Date: Tue, 26 Jan 2010 15:07:47 +0000
Subject: insta-mfh r203028 (doc update)

---
 lib/libfetch/fetch.3 | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

(limited to 'lib')

diff --git a/lib/libfetch/fetch.3 b/lib/libfetch/fetch.3
index 9b312b0..aaf1a8d 100644
--- a/lib/libfetch/fetch.3
+++ b/lib/libfetch/fetch.3
@@ -25,7 +25,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd December 14, 2008
+.Dd January 26, 2010
 .Dt FETCH 3
 .Os
 .Sh NAME
@@ -509,9 +509,13 @@ Specifies HTTP authorization parameters as a colon-separated list of
 items.
 The first and second item are the authorization scheme and realm
 respectively; further items are scheme-dependent.
-Currently, only basic authorization is supported.
+Currently, the
+.Dq basic
+and
+.Dq digest
+authorization methods are supported.
 .Pp
-Basic authorization requires two parameters: the user name and
+Both methods require two parameters: the user name and
 password, in that order.
 .Pp
 This variable is only used if the server requires authorization and
@@ -656,12 +660,14 @@ The
 .Nm fetch
 library was mostly written by
 .An Dag-Erling Sm\(/orgrav Aq des@FreeBSD.org
-with numerous suggestions from
+with numerous suggestions and contributions from
 .An Jordan K. Hubbard Aq jkh@FreeBSD.org ,
-.An Eugene Skepner Aq eu@qub.com
-and other
-.Fx
-developers.
+.An Eugene Skepner Aq eu@qub.com ,
+.An Hajimu Umemoto Aq ume@FreeBSD.org ,
+.An Henry Whincup Aq henry@techiebod.com ,
+.An Jukka A. Ukkonen Aq jau@iki.fi ,
+.An Jean-Fran\(,cois Dockes Aq jf@dockes.org
+and others.
 It replaces the older
 .Nm ftpio
 library written by
-- 
cgit v1.1


From ade3e629421f12a6799785d918a77632dfae157a Mon Sep 17 00:00:00 2001
From: antoine <antoine@FreeBSD.org>
Date: Sat, 30 Jan 2010 12:11:21 +0000
Subject: MFC r201145 to stable/8:   (S)LIST_HEAD_INITIALIZER takes a
 (S)LIST_HEAD as an argument.   Fix some wrong usages.   Note: this does not
 affect generated binaries as this argument is not used.

  PR:		137213
  Submitted by:	Eygene Ryabinkin (initial version)
---
 lib/libc/gen/sem.c                  | 2 +-
 lib/libgssapi/gss_mech_switch.c     | 2 +-
 lib/librpcsec_gss/rpcsec_gss_conf.c | 4 ++--
 lib/librpcsec_gss/svc_rpcsec_gss.c  | 4 ++--
 4 files changed, 6 insertions(+), 6 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/gen/sem.c b/lib/libc/gen/sem.c
index 439f0fe..cd31d31 100644
--- a/lib/libc/gen/sem.c
+++ b/lib/libc/gen/sem.c
@@ -73,7 +73,7 @@
 static sem_t sem_alloc(unsigned int value, semid_t semid, int system_sem);
 static void  sem_free(sem_t sem);
 
-static LIST_HEAD(, sem) named_sems = LIST_HEAD_INITIALIZER(&named_sems);
+static LIST_HEAD(, sem) named_sems = LIST_HEAD_INITIALIZER(named_sems);
 static pthread_mutex_t named_sems_mtx = PTHREAD_MUTEX_INITIALIZER;
 
 __weak_reference(__sem_init, sem_init);
diff --git a/lib/libgssapi/gss_mech_switch.c b/lib/libgssapi/gss_mech_switch.c
index feb88f1..d07db88 100644
--- a/lib/libgssapi/gss_mech_switch.c
+++ b/lib/libgssapi/gss_mech_switch.c
@@ -42,7 +42,7 @@
 #endif
 
 struct _gss_mech_switch_list _gss_mechs =
-	SLIST_HEAD_INITIALIZER(&_gss_mechs);
+	SLIST_HEAD_INITIALIZER(_gss_mechs);
 gss_OID_set _gss_mech_oids;
 
 /*
diff --git a/lib/librpcsec_gss/rpcsec_gss_conf.c b/lib/librpcsec_gss/rpcsec_gss_conf.c
index 4a0349b..14e063c 100644
--- a/lib/librpcsec_gss/rpcsec_gss_conf.c
+++ b/lib/librpcsec_gss/rpcsec_gss_conf.c
@@ -55,7 +55,7 @@ struct mech_info {
 };
 SLIST_HEAD(mech_info_list, mech_info);
 
-static struct mech_info_list mechs = SLIST_HEAD_INITIALIZER(&mechs);
+static struct mech_info_list mechs = SLIST_HEAD_INITIALIZER(mechs);
 static const char **mech_names;
 
 struct qop_info {
@@ -66,7 +66,7 @@ struct qop_info {
 };
 SLIST_HEAD(qop_info_list, qop_info);
 
-static struct qop_info_list qops = SLIST_HEAD_INITIALIZER(&qops);
+static struct qop_info_list qops = SLIST_HEAD_INITIALIZER(qops);
 
 static int
 _rpc_gss_string_to_oid(const char* s, gss_OID oid)
diff --git a/lib/librpcsec_gss/svc_rpcsec_gss.c b/lib/librpcsec_gss/svc_rpcsec_gss.c
index 276126b..d0599be 100644
--- a/lib/librpcsec_gss/svc_rpcsec_gss.c
+++ b/lib/librpcsec_gss/svc_rpcsec_gss.c
@@ -90,7 +90,7 @@ struct svc_rpc_gss_callback {
 	rpc_gss_callback_t	cb_callback;
 };
 static SLIST_HEAD(svc_rpc_gss_callback_list, svc_rpc_gss_callback)
-	svc_rpc_gss_callbacks = SLIST_HEAD_INITIALIZER(&svc_rpc_gss_callbacks);
+	svc_rpc_gss_callbacks = SLIST_HEAD_INITIALIZER(svc_rpc_gss_callbacks);
 
 struct svc_rpc_gss_svc_name {
 	SLIST_ENTRY(svc_rpc_gss_svc_name) sn_link;
@@ -102,7 +102,7 @@ struct svc_rpc_gss_svc_name {
 	u_int			sn_version;
 };
 static SLIST_HEAD(svc_rpc_gss_svc_name_list, svc_rpc_gss_svc_name)
-	svc_rpc_gss_svc_names = SLIST_HEAD_INITIALIZER(&svc_rpc_gss_svc_names);
+	svc_rpc_gss_svc_names = SLIST_HEAD_INITIALIZER(svc_rpc_gss_svc_names);
 
 enum svc_rpc_gss_client_state {
 	CLIENT_NEW,				/* still authenticating */
-- 
cgit v1.1


From 5860340b9e706a6aa77a83f830e2e748f9d518c4 Mon Sep 17 00:00:00 2001
From: kib <kib@FreeBSD.org>
Date: Sun, 31 Jan 2010 18:38:03 +0000
Subject: MFC r199827: Implement sighold, sigignore, sigpause, sigrelse, sigset
 functions.

MFC r200881 (by cognet):
Don't name parameters.
---
 lib/libc/compat-43/Makefile.inc |   5 ++
 lib/libc/compat-43/Symbol.map   |   8 ++
 lib/libc/compat-43/sigcompat.c  |  85 ++++++++++++++++++++-
 lib/libc/compat-43/sigpause.2   | 160 ++++++++++++++++++++++++++++++++++++++--
 4 files changed, 250 insertions(+), 8 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/compat-43/Makefile.inc b/lib/libc/compat-43/Makefile.inc
index 836f0a6..8505ff2 100644
--- a/lib/libc/compat-43/Makefile.inc
+++ b/lib/libc/compat-43/Makefile.inc
@@ -13,6 +13,11 @@ MAN+=	creat.2 killpg.2 sigpause.2 sigsetmask.2 sigvec.2
 MAN+=	gethostid.3 setruid.3
 
 MLINKS+=gethostid.3 sethostid.3
+MLINKS+=sigpause.2 sighold.2
+MLINKS+=sigpause.2 sigignore.2
+MLINKS+=sigpause.2 sigrelse.2
+MLINKS+=sigpause.2 sigset.2
+MLINKS+=sigpause.2 xsi_sigpause.2
 MLINKS+=setruid.3 setrgid.3
 
 MLINKS+=sigsetmask.2 sigblock.2
diff --git a/lib/libc/compat-43/Symbol.map b/lib/libc/compat-43/Symbol.map
index e859a31..bd49f99 100644
--- a/lib/libc/compat-43/Symbol.map
+++ b/lib/libc/compat-43/Symbol.map
@@ -17,6 +17,14 @@ FBSD_1.0 {
 	sigvec;
 };
 
+FBSD_1.2 {
+	sighold;
+	sigignore;
+	sigrelse;
+	sigset;
+	xsi_sigpause;
+};
+
 FBSDprivate_1.0 {
 	__creat;
 	_creat;
diff --git a/lib/libc/compat-43/sigcompat.c b/lib/libc/compat-43/sigcompat.c
index 6280183..c3ba30a 100644
--- a/lib/libc/compat-43/sigcompat.c
+++ b/lib/libc/compat-43/sigcompat.c
@@ -36,6 +36,7 @@ __FBSDID("$FreeBSD$");
 #include "namespace.h"
 #include <sys/param.h>
 #include <signal.h>
+#include <string.h>
 #include "un-namespace.h"
 #include "libc_private.h"
 
@@ -97,8 +98,7 @@ sigblock(mask)
 }
 
 int
-sigpause(mask)
-	int mask;
+sigpause(int mask)
 {
 	sigset_t set;
 
@@ -106,3 +106,84 @@ sigpause(mask)
 	set.__bits[0] = mask;
 	return (_sigsuspend(&set));
 }
+
+int
+xsi_sigpause(int sig)
+{
+	sigset_t set;
+
+	sigemptyset(&set);
+	sigaddset(&set, sig);
+	return (_sigsuspend(&set));
+}
+
+int
+sighold(int sig)
+{
+	sigset_t set;
+
+	sigemptyset(&set);
+	sigaddset(&set, sig);
+	return (_sigprocmask(SIG_BLOCK, &set, NULL));
+}
+
+int
+sigignore(int sig)
+{
+	struct sigaction sa;
+
+	bzero(&sa, sizeof(sa));
+	sa.sa_handler = SIG_IGN;
+	return (_sigaction(sig, &sa, NULL));
+}
+
+int
+sigrelse(int sig)
+{
+	sigset_t set;
+
+	sigemptyset(&set);
+	sigaddset(&set, sig);
+	return (_sigprocmask(SIG_UNBLOCK, &set, NULL));
+}
+
+void
+(*sigset(int sig, void (*disp)(int)))(int)
+{
+	sigset_t set, pset;
+	struct sigaction sa, psa;
+	int error;
+
+	sigemptyset(&set);
+	sigaddset(&set, sig);
+	error = _sigprocmask(SIG_BLOCK, NULL, &pset);
+	if (error == -1)
+		return (SIG_ERR);
+	if ((__sighandler_t *)disp == SIG_HOLD) {
+		error = _sigprocmask(SIG_BLOCK, &set, &pset);
+		if (error == -1)
+			return (SIG_ERR);
+		if (sigismember(&pset, sig))
+			return (SIG_HOLD);
+		else {
+			error = _sigaction(sig, NULL, &psa);
+			if (error == -1)
+				return (SIG_ERR);
+			return (psa.sa_handler);
+		}
+	} else {
+		error = _sigprocmask(SIG_UNBLOCK, &set, &pset);
+		if (error == -1)
+			return (SIG_ERR);
+	}
+
+	bzero(&sa, sizeof(sa));
+	sa.sa_handler = disp;
+	error = _sigaction(sig, &sa, &psa);
+	if (error == -1)
+		return (SIG_ERR);
+	if (sigismember(&pset, sig))
+		return (SIG_HOLD);
+	else
+		return (psa.sa_handler);
+}
diff --git a/lib/libc/compat-43/sigpause.2 b/lib/libc/compat-43/sigpause.2
index 39edb0b..bf3cf0b 100644
--- a/lib/libc/compat-43/sigpause.2
+++ b/lib/libc/compat-43/sigpause.2
@@ -28,21 +28,118 @@
 .\"     @(#)sigpause.2	8.1 (Berkeley) 6/2/93
 .\" $FreeBSD$
 .\"
+.\" Part of the content of the man page was derived from
+.\" The Open Group Base Specifications Issue 7
+.\" IEEE Std 1003.1-2008
+.\"
 .Dd June 2, 1993
 .Dt SIGPAUSE 2
 .Os
 .Sh NAME
-.Nm sigpause
-.Nd atomically release blocked signals and wait for interrupt
+.Nm sighold ,
+.Nm sigignore ,
+.Nm sigpause ,
+.Nm sigrelse ,
+.Nm sigset
+.Nd legacy interface for signal management
 .Sh LIBRARY
 .Lb libc
 .Sh SYNOPSIS
 .In signal.h
 .Ft int
+.Fn sighold "int sig"
+.Ft int
+.Fn sigignore "int sig"
+.Ft int
+.Fn xsi_sigpause "int sigmask"
+.Ft int
+.Fn sigrelse "int sig"
+.Ft void (*)(int)
+.Fn sigset "int" "void (*disp)(int)"
+.Ft int
 .Fn sigpause "int sigmask"
 .Sh DESCRIPTION
 .Sy This interface is made obsolete by
-.Xr sigsuspend 2 .
+.Xr sigsuspend 2
+.Sy and
+.Xr sigaction 2
+.Pp
+The
+.Fn sigset
+function modifies signal dispositions.
+The
+.Fa sig
+argument specifies the signal, which may be any signal except
+.Dv SIGKILL
+and
+.Dv SIGSTOP .
+The
+.Fa disp
+argument specifies the signal's disposition,
+which may be
+.Dv SIG_DFL ,
+.Dv SIG_IGN ,
+or the address of a signal handler.
+If
+.Fn sigset
+is used, and
+.Fa disp
+is the address of a signal handler, the
+system adds
+.Fa sig
+to the signal mask of the calling process before executing the signal
+handler; when the signal handler returns, the system restores the
+signal mask of the calling process to its state prior to the delivery
+of the signal.
+In addition, if
+.Fn sigset
+is used, and
+.Fa disp
+is equal to
+.Dv SIG_HOLD ,
+.Fa sig
+is added to the signal
+mask of the calling process and
+.Fa sig 's
+disposition remains unchanged.
+If
+.Fn sigset
+is used, and
+.Fa disp
+is not equal to
+.Dv SIG_HOLD ,
+.Fa sig
+is removed from the signal mask of the calling process.
+.Pp
+The
+.Fn sighold
+function adds
+.Fa sig
+to the signal mask of the calling process.
+.Pp
+The
+.Fn sigrelse
+function removes
+.Fa sig
+from the signal mask of the calling process.
+.Pp
+The
+.Fn sigignore
+function sets the disposition of
+.Fa sig
+to
+.Dv SIG_IGN .
+.Pp
+The
+.Fn xsi_sigpause
+function removes
+.Fa sig
+from the signal mask of the calling process and suspend the calling process
+until a signal is received.
+The
+.Fn xsi_sigpause
+function restores the signal mask of the process to its original state before
+returning.
 .Pp
 The
 .Fn sigpause
@@ -57,13 +154,47 @@ The
 argument
 is usually 0 to indicate that no
 signals are to be blocked.
+.Sh RETURN VALUES
 The
 .Fn sigpause
-function
-always terminates by being interrupted, returning -1 with
+and
+.Fn xsi_sigpause
+functions
+always terminate by being interrupted, returning -1 with
 .Va errno
 set to
-.Er EINTR
+.Er EINTR .
+.Pp
+Upon successful completion,
+.Fn sigset
+returns
+.Dv SIG_HOLD
+if the signal had been blocked and the signal's previous disposition if
+it had not been blocked.
+Otherwise,
+.Dv SIG_ERR is returned and
+.Va errno
+set to indicate the error.
+.Pp
+For all other functions, upon successful completion, 0 is returned.
+Otherwise, -1 is returned and
+.Va errno
+is set to indicate the error:
+.Bl -tag -width Er
+.It Bq Er EINVAL
+The
+.Fa sig
+argument
+is not a valid signal number.
+.It Bq Er EINVAL
+For
+.Fn sigset
+and
+.Fn sigignore
+functions, an attempt was made to catch or ignore
+.Dv SIGKILL
+or
+.Dv SIGSTOP .
 .Sh SEE ALSO
 .Xr kill 2 ,
 .Xr sigaction 2 ,
@@ -85,9 +216,26 @@ and was copied from there into the
 .Pq Tn XSI
 option of
 .St -p1003.1-2001 .
+.Fx
+implements it under the name
+.Fn xsi_sigpause .
+The
+.Fn sighold ,
+.Fn sigignore ,
+.Fn sigrelse
+and
+.Fn sigset
+functions are implemented for compatibility with
+.Sy System V
+and
+.Sy XSI
+interfaces.
 .Sh HISTORY
 The
 .Fn sigpause
 function appeared in
 .Bx 4.2
 and has been deprecated.
+All other functions appeared in
+.Fx 9.0
+and were deprecated before being implemented.
-- 
cgit v1.1


From 9a944ce09eb54108162372ef2d8e6bf920413311 Mon Sep 17 00:00:00 2001
From: gavin <gavin@FreeBSD.org>
Date: Tue, 2 Feb 2010 19:37:26 +0000
Subject: Merge r203025,r203026 from head:

  Correct the HISTORY section of these man pages to show when the function,
  not the "manual page example" was introduced.

Approved by:	ed (mentor, implicit)
---
 lib/librpcsec_gss/rpc_gss_get_error.3           | 4 ++--
 lib/librpcsec_gss/rpc_gss_get_mech_info.3       | 4 ++--
 lib/librpcsec_gss/rpc_gss_get_mechanisms.3      | 4 ++--
 lib/librpcsec_gss/rpc_gss_get_principal_name.3  | 4 ++--
 lib/librpcsec_gss/rpc_gss_get_versions.3        | 4 ++--
 lib/librpcsec_gss/rpc_gss_getcred.3             | 4 ++--
 lib/librpcsec_gss/rpc_gss_is_installed.3        | 4 ++--
 lib/librpcsec_gss/rpc_gss_max_data_length.3     | 4 ++--
 lib/librpcsec_gss/rpc_gss_mech_to_oid.3         | 4 ++--
 lib/librpcsec_gss/rpc_gss_oid_to_mech.3         | 4 ++--
 lib/librpcsec_gss/rpc_gss_qop_to_num.3          | 4 ++--
 lib/librpcsec_gss/rpc_gss_seccreate.3           | 4 ++--
 lib/librpcsec_gss/rpc_gss_set_callback.3        | 4 ++--
 lib/librpcsec_gss/rpc_gss_set_defaults.3        | 4 ++--
 lib/librpcsec_gss/rpc_gss_set_svc_name.3        | 4 ++--
 lib/librpcsec_gss/rpc_gss_svc_max_data_length.3 | 4 ++--
 lib/librpcsec_gss/rpcsec_gss.3                  | 4 ++--
 17 files changed, 34 insertions(+), 34 deletions(-)

(limited to 'lib')

diff --git a/lib/librpcsec_gss/rpc_gss_get_error.3 b/lib/librpcsec_gss/rpc_gss_get_error.3
index e108766..74d2fa0 100644
--- a/lib/librpcsec_gss/rpc_gss_get_error.3
+++ b/lib/librpcsec_gss/rpc_gss_get_error.3
@@ -24,7 +24,7 @@
 .\" SUCH DAMAGE.
 .\"
 .\" $FreeBSD$
-.Dd July 4, 2008
+.Dd January 26, 2010
 .Dt RPC_GSS_GET_ERROR 3
 .Os
 .Sh NAME
@@ -50,7 +50,7 @@ A pointer to a structure where the error details will be returned
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 8.0 .
 .Sh AUTHORS
 This
diff --git a/lib/librpcsec_gss/rpc_gss_get_mech_info.3 b/lib/librpcsec_gss/rpc_gss_get_mech_info.3
index a7f7d6b..945b00b 100644
--- a/lib/librpcsec_gss/rpc_gss_get_mech_info.3
+++ b/lib/librpcsec_gss/rpc_gss_get_mech_info.3
@@ -24,7 +24,7 @@
 .\" SUCH DAMAGE.
 .\"
 .\" $FreeBSD$
-.Dd July 4, 2008
+.Dd January 26, 2010
 .Dt RPC_GSS_GET_MECH_INFO 3
 .Os
 .Sh NAME
@@ -60,7 +60,7 @@ otherwise
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 8.0 .
 .Sh AUTHORS
 This
diff --git a/lib/librpcsec_gss/rpc_gss_get_mechanisms.3 b/lib/librpcsec_gss/rpc_gss_get_mechanisms.3
index 4b57ac6..e18a648 100644
--- a/lib/librpcsec_gss/rpc_gss_get_mechanisms.3
+++ b/lib/librpcsec_gss/rpc_gss_get_mechanisms.3
@@ -24,7 +24,7 @@
 .\" SUCH DAMAGE.
 .\"
 .\" $FreeBSD$
-.Dd July 4, 2008
+.Dd January 26, 2010
 .Dt RPC_GSS_GET_MECHANISMS 3
 .Os
 .Sh NAME
@@ -47,7 +47,7 @@ terminated list of installed security mechanisms.
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 8.0 .
 .Sh AUTHORS
 This
diff --git a/lib/librpcsec_gss/rpc_gss_get_principal_name.3 b/lib/librpcsec_gss/rpc_gss_get_principal_name.3
index 6f57212..19297f6 100644
--- a/lib/librpcsec_gss/rpc_gss_get_principal_name.3
+++ b/lib/librpcsec_gss/rpc_gss_get_principal_name.3
@@ -24,7 +24,7 @@
 .\" SUCH DAMAGE.
 .\"
 .\" $FreeBSD$
-.Dd July 4, 2008
+.Dd January 26, 2010
 .Dt RPC_GSS_GET_PRINCIPAL_NAME 3
 .Os
 .Sh NAME
@@ -74,7 +74,7 @@ otherwise
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 8.0 .
 .Sh AUTHORS
 This
diff --git a/lib/librpcsec_gss/rpc_gss_get_versions.3 b/lib/librpcsec_gss/rpc_gss_get_versions.3
index f824066..78ebe73 100644
--- a/lib/librpcsec_gss/rpc_gss_get_versions.3
+++ b/lib/librpcsec_gss/rpc_gss_get_versions.3
@@ -24,7 +24,7 @@
 .\" SUCH DAMAGE.
 .\"
 .\" $FreeBSD$
-.Dd July 4, 2008
+.Dd January 26, 2010
 .Dt RPC_GSS_GET_VERSIONS 3
 .Os
 .Sh NAME
@@ -56,7 +56,7 @@ is set to the lowest suppored protocol version
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 8.0 .
 .Sh AUTHORS
 This
diff --git a/lib/librpcsec_gss/rpc_gss_getcred.3 b/lib/librpcsec_gss/rpc_gss_getcred.3
index 2ede33e..844e61e 100644
--- a/lib/librpcsec_gss/rpc_gss_getcred.3
+++ b/lib/librpcsec_gss/rpc_gss_getcred.3
@@ -24,7 +24,7 @@
 .\" SUCH DAMAGE.
 .\"
 .\" $FreeBSD$
-.Dd July 4, 2008
+.Dd January 26, 2010
 .Dt RPC_GSS_GETCRED 3
 .Os
 .Sh NAME
@@ -77,7 +77,7 @@ otherwise.
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 8.0 .
 .Sh AUTHORS
 This
diff --git a/lib/librpcsec_gss/rpc_gss_is_installed.3 b/lib/librpcsec_gss/rpc_gss_is_installed.3
index 2859ed2..a043c45 100644
--- a/lib/librpcsec_gss/rpc_gss_is_installed.3
+++ b/lib/librpcsec_gss/rpc_gss_is_installed.3
@@ -24,7 +24,7 @@
 .\" SUCH DAMAGE.
 .\"
 .\" $FreeBSD$
-.Dd July 4, 2008
+.Dd January 26, 2010
 .Dt RPC_GSS_IS_INSTALLED 3
 .Os
 .Sh NAME
@@ -57,7 +57,7 @@ otherwise.
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 8.0 .
 .Sh AUTHORS
 This
diff --git a/lib/librpcsec_gss/rpc_gss_max_data_length.3 b/lib/librpcsec_gss/rpc_gss_max_data_length.3
index 8957119..a600192 100644
--- a/lib/librpcsec_gss/rpc_gss_max_data_length.3
+++ b/lib/librpcsec_gss/rpc_gss_max_data_length.3
@@ -24,7 +24,7 @@
 .\" SUCH DAMAGE.
 .\"
 .\" $FreeBSD$
-.Dd July 4, 2008
+.Dd January 26, 2010
 .Dt RPC_GSS_MAX_DATA_LENGTH 3
 .Os
 .Sh NAME
@@ -56,7 +56,7 @@ The maximum message size that can be encoded
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 8.0 .
 .Sh AUTHORS
 This
diff --git a/lib/librpcsec_gss/rpc_gss_mech_to_oid.3 b/lib/librpcsec_gss/rpc_gss_mech_to_oid.3
index c1f9f2a..95d0cd7 100644
--- a/lib/librpcsec_gss/rpc_gss_mech_to_oid.3
+++ b/lib/librpcsec_gss/rpc_gss_mech_to_oid.3
@@ -24,7 +24,7 @@
 .\" SUCH DAMAGE.
 .\"
 .\" $FreeBSD$
-.Dd July 4, 2008
+.Dd January 26, 2010
 .Dt RPC_GSS_MECH_TO_OID 3
 .Os
 .Sh NAME
@@ -60,7 +60,7 @@ is returned, otherwise
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 8.0 .
 .Sh AUTHORS
 This
diff --git a/lib/librpcsec_gss/rpc_gss_oid_to_mech.3 b/lib/librpcsec_gss/rpc_gss_oid_to_mech.3
index 6183f26..c91a2f5 100644
--- a/lib/librpcsec_gss/rpc_gss_oid_to_mech.3
+++ b/lib/librpcsec_gss/rpc_gss_oid_to_mech.3
@@ -24,7 +24,7 @@
 .\" SUCH DAMAGE.
 .\"
 .\" $FreeBSD$
-.Dd July 4, 2008
+.Dd January 26, 2010
 .Dt RPC_GSS_OID_TO_MECH 3
 .Os
 .Sh NAME
@@ -60,7 +60,7 @@ is returned, otherwise
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 8.0 .
 .Sh AUTHORS
 This
diff --git a/lib/librpcsec_gss/rpc_gss_qop_to_num.3 b/lib/librpcsec_gss/rpc_gss_qop_to_num.3
index e58d553..38d2725 100644
--- a/lib/librpcsec_gss/rpc_gss_qop_to_num.3
+++ b/lib/librpcsec_gss/rpc_gss_qop_to_num.3
@@ -24,7 +24,7 @@
 .\" SUCH DAMAGE.
 .\"
 .\" $FreeBSD$
-.Dd July 4, 2008
+.Dd January 26, 2010
 .Dt RPC_GSS_QOP_TO_NUM 3
 .Os
 .Sh NAME
@@ -62,7 +62,7 @@ is returned, otherwise
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 8.0 .
 .Sh AUTHORS
 This
diff --git a/lib/librpcsec_gss/rpc_gss_seccreate.3 b/lib/librpcsec_gss/rpc_gss_seccreate.3
index 0f7dfab..07204cb 100644
--- a/lib/librpcsec_gss/rpc_gss_seccreate.3
+++ b/lib/librpcsec_gss/rpc_gss_seccreate.3
@@ -24,7 +24,7 @@
 .\" SUCH DAMAGE.
 .\"
 .\" $FreeBSD$
-.Dd July 4, 2008
+.Dd January 26, 2010
 .Dt RPC_GSS_SECCREATE 3
 .Os
 .Sh NAME
@@ -104,7 +104,7 @@ to this value.
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 8.0 .
 .Sh AUTHORS
 This
diff --git a/lib/librpcsec_gss/rpc_gss_set_callback.3 b/lib/librpcsec_gss/rpc_gss_set_callback.3
index 0ad861f..f701cf1 100644
--- a/lib/librpcsec_gss/rpc_gss_set_callback.3
+++ b/lib/librpcsec_gss/rpc_gss_set_callback.3
@@ -24,7 +24,7 @@
 .\" SUCH DAMAGE.
 .\"
 .\" $FreeBSD$
-.Dd July 4, 2008
+.Dd January 26, 2010
 .Dt RPC_GSS_SET_CALLBACK 3
 .Os
 .Sh NAME
@@ -100,7 +100,7 @@ otherwise
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 8.0 .
 .Sh AUTHORS
 This
diff --git a/lib/librpcsec_gss/rpc_gss_set_defaults.3 b/lib/librpcsec_gss/rpc_gss_set_defaults.3
index 241cf9e..c4a24cf 100644
--- a/lib/librpcsec_gss/rpc_gss_set_defaults.3
+++ b/lib/librpcsec_gss/rpc_gss_set_defaults.3
@@ -24,7 +24,7 @@
 .\" SUCH DAMAGE.
 .\"
 .\" $FreeBSD$
-.Dd July 4, 2008
+.Dd January 26, 2010
 .Dt RPC_GSS_SET_DEFAULTS 3
 .Os
 .Sh NAME
@@ -62,7 +62,7 @@ if the values were set
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 8.0 .
 .Sh AUTHORS
 This
diff --git a/lib/librpcsec_gss/rpc_gss_set_svc_name.3 b/lib/librpcsec_gss/rpc_gss_set_svc_name.3
index 9be8fdf..79a9c68 100644
--- a/lib/librpcsec_gss/rpc_gss_set_svc_name.3
+++ b/lib/librpcsec_gss/rpc_gss_set_svc_name.3
@@ -24,7 +24,7 @@
 .\" SUCH DAMAGE.
 .\"
 .\" $FreeBSD$
-.Dd July 4, 2008
+.Dd January 26, 2010
 .Dt RPC_GSS_SET_SVC_NAME 3
 .Os
 .Sh NAME
@@ -79,7 +79,7 @@ otherwise.
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 8.0 .
 .Sh AUTHORS
 This
diff --git a/lib/librpcsec_gss/rpc_gss_svc_max_data_length.3 b/lib/librpcsec_gss/rpc_gss_svc_max_data_length.3
index 2445f75..ef4871f 100644
--- a/lib/librpcsec_gss/rpc_gss_svc_max_data_length.3
+++ b/lib/librpcsec_gss/rpc_gss_svc_max_data_length.3
@@ -24,7 +24,7 @@
 .\" SUCH DAMAGE.
 .\"
 .\" $FreeBSD$
-.Dd July 4, 2008
+.Dd January 26, 2010
 .Dt RPC_GSS_SVC_MAX_DATA_LENGTH 3
 .Os
 .Sh NAME
@@ -56,7 +56,7 @@ The maximum message size that can be encoded
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 8.0 .
 .Sh AUTHORS
 This
diff --git a/lib/librpcsec_gss/rpcsec_gss.3 b/lib/librpcsec_gss/rpcsec_gss.3
index 793f012..45bd82f 100644
--- a/lib/librpcsec_gss/rpcsec_gss.3
+++ b/lib/librpcsec_gss/rpcsec_gss.3
@@ -24,7 +24,7 @@
 .\" SUCH DAMAGE.
 .\"
 .\" $FreeBSD$
-.Dd July 4, 2008
+.Dd January 26, 2010
 .Dt RPC_GSS_SECCREATE 3
 .Os
 .Sh NAME
@@ -222,7 +222,7 @@ Calculate maximum server message sizes.
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+library first appeared in
 .Fx 8.0 .
 .Sh AUTHORS
 This
-- 
cgit v1.1


From 9ce8997aab4144c0a5f5fb94bd7c3a1cb77bcbba Mon Sep 17 00:00:00 2001
From: gavin <gavin@FreeBSD.org>
Date: Tue, 2 Feb 2010 19:44:52 +0000
Subject: Merge r203027 from head:

  Correct the HISTORY section of these man pages to show when the function,
  not the "manual page example" was introduced.

Approved by:	ed (mentor, implicit)
---
 lib/libgssapi/gss_accept_sec_context.3     | 4 ++--
 lib/libgssapi/gss_acquire_cred.3           | 4 ++--
 lib/libgssapi/gss_add_cred.3               | 4 ++--
 lib/libgssapi/gss_add_oid_set_member.3     | 4 ++--
 lib/libgssapi/gss_canonicalize_name.3      | 4 ++--
 lib/libgssapi/gss_compare_name.3           | 4 ++--
 lib/libgssapi/gss_context_time.3           | 4 ++--
 lib/libgssapi/gss_create_empty_oid_set.3   | 4 ++--
 lib/libgssapi/gss_delete_sec_context.3     | 4 ++--
 lib/libgssapi/gss_display_name.3           | 4 ++--
 lib/libgssapi/gss_display_status.3         | 4 ++--
 lib/libgssapi/gss_duplicate_name.3         | 4 ++--
 lib/libgssapi/gss_export_name.3            | 4 ++--
 lib/libgssapi/gss_export_sec_context.3     | 4 ++--
 lib/libgssapi/gss_get_mic.3                | 4 ++--
 lib/libgssapi/gss_import_name.3            | 4 ++--
 lib/libgssapi/gss_import_sec_context.3     | 4 ++--
 lib/libgssapi/gss_indicate_mechs.3         | 4 ++--
 lib/libgssapi/gss_init_sec_context.3       | 4 ++--
 lib/libgssapi/gss_inquire_context.3        | 4 ++--
 lib/libgssapi/gss_inquire_cred.3           | 4 ++--
 lib/libgssapi/gss_inquire_cred_by_mech.3   | 4 ++--
 lib/libgssapi/gss_inquire_mechs_for_name.3 | 4 ++--
 lib/libgssapi/gss_inquire_names_for_mech.3 | 4 ++--
 lib/libgssapi/gss_process_context_token.3  | 4 ++--
 lib/libgssapi/gss_release_buffer.3         | 4 ++--
 lib/libgssapi/gss_release_cred.3           | 4 ++--
 lib/libgssapi/gss_release_name.3           | 4 ++--
 lib/libgssapi/gss_release_oid_set.3        | 4 ++--
 lib/libgssapi/gss_test_oid_set_member.3    | 4 ++--
 lib/libgssapi/gss_unwrap.3                 | 4 ++--
 lib/libgssapi/gss_verify_mic.3             | 4 ++--
 lib/libgssapi/gss_wrap.3                   | 4 ++--
 lib/libgssapi/gss_wrap_size_limit.3        | 4 ++--
 lib/libgssapi/gssapi.3                     | 4 ++--
 lib/libgssapi/mech.5                       | 4 ++--
 36 files changed, 72 insertions(+), 72 deletions(-)

(limited to 'lib')

diff --git a/lib/libgssapi/gss_accept_sec_context.3 b/lib/libgssapi/gss_accept_sec_context.3
index 6b57518..8957831 100644
--- a/lib/libgssapi/gss_accept_sec_context.3
+++ b/lib/libgssapi/gss_accept_sec_context.3
@@ -27,7 +27,7 @@
 .\"	$FreeBSD$
 .\"
 .\" The following commands are required for all man pages.
-.Dd October 30, 2007
+.Dd January 26, 2010
 .Os
 .Dt GSS_ACCEPT_SEC_CONTEXT 3 PRM
 .Sh NAME
@@ -451,7 +451,7 @@ Generic Security Service API Version 2 : C-bindings
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 7.0 .
 .Sh AUTHORS
 John Wray, Iris Associates
diff --git a/lib/libgssapi/gss_acquire_cred.3 b/lib/libgssapi/gss_acquire_cred.3
index b43185c..c48a468 100644
--- a/lib/libgssapi/gss_acquire_cred.3
+++ b/lib/libgssapi/gss_acquire_cred.3
@@ -27,7 +27,7 @@
 .\"	$FreeBSD$
 .\"
 .\" The following commands are required for all man pages.
-.Dd October 30, 2007
+.Dd January 26, 2010
 .Os
 .Dt GSS_ACQUIRE_CRED 3 PRM
 .Sh NAME
@@ -205,7 +205,7 @@ Generic Security Service API Version 2 : C-bindings
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 7.0 .
 .Sh AUTHORS
 John Wray, Iris Associates
diff --git a/lib/libgssapi/gss_add_cred.3 b/lib/libgssapi/gss_add_cred.3
index 72f2ab0..53df140 100644
--- a/lib/libgssapi/gss_add_cred.3
+++ b/lib/libgssapi/gss_add_cred.3
@@ -27,7 +27,7 @@
 .\"	$FreeBSD$
 .\"
 .\" The following commands are required for all man pages.
-.Dd October 30, 2007
+.Dd January 26, 2010
 .Os
 .Dt GSS_ADD_CRED 3 PRM
 .Sh NAME
@@ -305,7 +305,7 @@ Generic Security Service API Version 2 : C-bindings
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 7.0 .
 .Sh AUTHORS
 John Wray, Iris Associates
diff --git a/lib/libgssapi/gss_add_oid_set_member.3 b/lib/libgssapi/gss_add_oid_set_member.3
index 255050b..fb2119c 100644
--- a/lib/libgssapi/gss_add_oid_set_member.3
+++ b/lib/libgssapi/gss_add_oid_set_member.3
@@ -27,7 +27,7 @@
 .\"	$FreeBSD$
 .\"
 .\" The following commands are required for all man pages.
-.Dd October 30, 2007
+.Dd January 26, 2010
 .Os
 .Dt GSS_ADD_OID_SET_MEMBER 3 PRM
 .Sh NAME
@@ -97,7 +97,7 @@ Generic Security Service API Version 2 : C-bindings
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 7.0 .
 .Sh AUTHORS
 John Wray, Iris Associates
diff --git a/lib/libgssapi/gss_canonicalize_name.3 b/lib/libgssapi/gss_canonicalize_name.3
index b38ca93..86957dc 100644
--- a/lib/libgssapi/gss_canonicalize_name.3
+++ b/lib/libgssapi/gss_canonicalize_name.3
@@ -27,7 +27,7 @@
 .\"	$FreeBSD$
 .\"
 .\" The following commands are required for all man pages.
-.Dd October 30, 2007
+.Dd January 26, 2010
 .Os
 .Dt GSS_CANONICALIZE_NAME 3 PRM
 .Sh NAME
@@ -104,7 +104,7 @@ Generic Security Service API Version 2 : C-bindings
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 7.0 .
 .Sh AUTHORS
 John Wray, Iris Associates
diff --git a/lib/libgssapi/gss_compare_name.3 b/lib/libgssapi/gss_compare_name.3
index 561c994..7f69d43 100644
--- a/lib/libgssapi/gss_compare_name.3
+++ b/lib/libgssapi/gss_compare_name.3
@@ -27,7 +27,7 @@
 .\"	$FreeBSD$
 .\"
 .\" The following commands are required for all man pages.
-.Dd October 30, 2007
+.Dd January 26, 2010
 .Os
 .Dt GSS_COMPARE_NAME PRM
 .Sh NAME
@@ -89,7 +89,7 @@ Generic Security Service API Version 2 : C-bindings
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 7.0 .
 .Sh AUTHORS
 John Wray, Iris Associates
diff --git a/lib/libgssapi/gss_context_time.3 b/lib/libgssapi/gss_context_time.3
index 96f61a9..9fb43ea 100644
--- a/lib/libgssapi/gss_context_time.3
+++ b/lib/libgssapi/gss_context_time.3
@@ -27,7 +27,7 @@
 .\"	$FreeBSD$
 .\"
 .\" The following commands are required for all man pages.
-.Dd October 30, 2007
+.Dd January 26, 2010
 .Os
 .Dt GSS_CONTEXT_TIME 3 PRM
 .Sh NAME
@@ -75,7 +75,7 @@ Generic Security Service API Version 2 : C-bindings
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 7.0 .
 .Sh AUTHORS
 John Wray, Iris Associates
diff --git a/lib/libgssapi/gss_create_empty_oid_set.3 b/lib/libgssapi/gss_create_empty_oid_set.3
index fd6cca6..08d82f6 100644
--- a/lib/libgssapi/gss_create_empty_oid_set.3
+++ b/lib/libgssapi/gss_create_empty_oid_set.3
@@ -27,7 +27,7 @@
 .\"	$FreeBSD$
 .\"
 .\" The following commands are required for all man pages.
-.Dd October 30, 2007
+.Dd January 26, 2010
 .Os
 .Dt GSS_CREATE_EMPTY_OID_SET 3 PRM
 .Sh NAME
@@ -78,7 +78,7 @@ Generic Security Service API Version 2 : C-bindings
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 7.0 .
 .Sh AUTHORS
 John Wray, Iris Associates
diff --git a/lib/libgssapi/gss_delete_sec_context.3 b/lib/libgssapi/gss_delete_sec_context.3
index d24644a..a0c26df 100644
--- a/lib/libgssapi/gss_delete_sec_context.3
+++ b/lib/libgssapi/gss_delete_sec_context.3
@@ -27,7 +27,7 @@
 .\"	$FreeBSD$
 .\"
 .\" The following commands are required for all man pages.
-.Dd October 30, 2007
+.Dd January 26, 2010
 .Os
 .Dt GSS_DELETE_SEC_CONTEXT 3 PRM
 .Sh NAME
@@ -130,7 +130,7 @@ Generic Security Service API Version 2 : C-bindings
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 7.0 .
 .Sh AUTHORS
 John Wray, Iris Associates
diff --git a/lib/libgssapi/gss_display_name.3 b/lib/libgssapi/gss_display_name.3
index 4f50870..2441eef 100644
--- a/lib/libgssapi/gss_display_name.3
+++ b/lib/libgssapi/gss_display_name.3
@@ -27,7 +27,7 @@
 .\"	$FreeBSD$
 .\"
 .\" The following commands are required for all man pages.
-.Dd October 30, 2007
+.Dd January 26, 2010
 .Os
 .Dt GSS_DISPLAY_NAME 3 PRM
 .Sh NAME
@@ -118,7 +118,7 @@ Generic Security Service API Version 2 : C-bindings
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 7.0 .
 .Sh AUTHORS
 John Wray, Iris Associates
diff --git a/lib/libgssapi/gss_display_status.3 b/lib/libgssapi/gss_display_status.3
index 3b3c323..2b1affd 100644
--- a/lib/libgssapi/gss_display_status.3
+++ b/lib/libgssapi/gss_display_status.3
@@ -27,7 +27,7 @@
 .\"	$FreeBSD$
 .\"
 .\" The following commands are required for all man pages.
-.Dd October 30, 2007
+.Dd January 26, 2010
 .Os
 .Dt GSS_DISPLAY_STATUS 3 PRM
 .Sh NAME
@@ -177,7 +177,7 @@ Generic Security Service API Version 2 : C-bindings
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 7.0 .
 .Sh AUTHORS
 John Wray, Iris Associates
diff --git a/lib/libgssapi/gss_duplicate_name.3 b/lib/libgssapi/gss_duplicate_name.3
index af885bf..334a126 100644
--- a/lib/libgssapi/gss_duplicate_name.3
+++ b/lib/libgssapi/gss_duplicate_name.3
@@ -27,7 +27,7 @@
 .\"	$FreeBSD$
 .\"
 .\" The following commands are required for all man pages.
-.Dd October 30, 2007
+.Dd January 26, 2010
 .Os
 .Dt GSS_DUPLICATE_NAME 3 PRM
 .Sh NAME
@@ -90,7 +90,7 @@ Generic Security Service API Version 2 : C-bindings
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 7.0 .
 .Sh AUTHORS
 John Wray, Iris Associates
diff --git a/lib/libgssapi/gss_export_name.3 b/lib/libgssapi/gss_export_name.3
index c10f472..e1feee2 100644
--- a/lib/libgssapi/gss_export_name.3
+++ b/lib/libgssapi/gss_export_name.3
@@ -27,7 +27,7 @@
 .\"	$FreeBSD$
 .\"
 .\" The following commands are required for all man pages.
-.Dd October 30, 2007
+.Dd January 26, 2010
 .Os
 .Dt GSS_EXPORT_NAME 3 PRM
 .Sh NAME
@@ -95,7 +95,7 @@ Generic Security Service API Version 2 : C-bindings
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 7.0 .
 .Sh AUTHORS
 John Wray, Iris Associates
diff --git a/lib/libgssapi/gss_export_sec_context.3 b/lib/libgssapi/gss_export_sec_context.3
index d8359c8..1f7f01d 100644
--- a/lib/libgssapi/gss_export_sec_context.3
+++ b/lib/libgssapi/gss_export_sec_context.3
@@ -27,7 +27,7 @@
 .\"	$FreeBSD$
 .\"
 .\" The following commands are required for all man pages.
-.Dd October 30, 2007
+.Dd January 26, 2010
 .Os
 .Dt GSS_EXPORT_SEC_CONTEXT 3 PRM
 .Sh NAME
@@ -135,7 +135,7 @@ Generic Security Service API Version 2 : C-bindings
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 7.0 .
 .Sh AUTHORS
 John Wray, Iris Associates
diff --git a/lib/libgssapi/gss_get_mic.3 b/lib/libgssapi/gss_get_mic.3
index 8f435d6..8892808 100644
--- a/lib/libgssapi/gss_get_mic.3
+++ b/lib/libgssapi/gss_get_mic.3
@@ -27,7 +27,7 @@
 .\"	$FreeBSD$
 .\"
 .\" The following commands are required for all man pages.
-.Dd October 30, 2007
+.Dd January 26, 2010
 .Os
 .Dt GSS_GET_MIC 3 PRM
 .Sh NAME
@@ -132,7 +132,7 @@ Generic Security Service API Version 2 : C-bindings
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 7.0 .
 .Sh AUTHORS
 John Wray, Iris Associates
diff --git a/lib/libgssapi/gss_import_name.3 b/lib/libgssapi/gss_import_name.3
index ed16278..3119f04 100644
--- a/lib/libgssapi/gss_import_name.3
+++ b/lib/libgssapi/gss_import_name.3
@@ -27,7 +27,7 @@
 .\"	$FreeBSD$
 .\"
 .\" The following commands are required for all man pages.
-.Dd October 30, 2007
+.Dd January 26, 2010
 .Os
 .Dt GSS_IMPORT_NAME 3 PRM
 .Sh NAME
@@ -106,7 +106,7 @@ Generic Security Service API Version 2 : C-bindings
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 7.0 .
 .Sh AUTHORS
 John Wray, Iris Associates
diff --git a/lib/libgssapi/gss_import_sec_context.3 b/lib/libgssapi/gss_import_sec_context.3
index f4dda84..a889e2b 100644
--- a/lib/libgssapi/gss_import_sec_context.3
+++ b/lib/libgssapi/gss_import_sec_context.3
@@ -27,7 +27,7 @@
 .\"	$FreeBSD$
 .\"
 .\" The following commands are required for all man pages.
-.Dd October 30, 2007
+.Dd January 26, 2010
 .Os
 .Dt GSS_IMPORT_SEC_CONTEXT 3 PRM
 .Sh NAME
@@ -87,7 +87,7 @@ Generic Security Service API Version 2 : C-bindings
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 7.0 .
 .Sh AUTHORS
 John Wray, Iris Associates
diff --git a/lib/libgssapi/gss_indicate_mechs.3 b/lib/libgssapi/gss_indicate_mechs.3
index 469ad1d..26ffaa6 100644
--- a/lib/libgssapi/gss_indicate_mechs.3
+++ b/lib/libgssapi/gss_indicate_mechs.3
@@ -27,7 +27,7 @@
 .\"	$FreeBSD$
 .\"
 .\" The following commands are required for all man pages.
-.Dd October 30, 2007
+.Dd January 26, 2010
 .Os
 .Dt GSS_INDICATE_MECHS 3 PRM
 .Sh NAME
@@ -74,7 +74,7 @@ Generic Security Service API Version 2 : C-bindings
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 7.0 .
 .Sh AUTHORS
 John Wray, Iris Associates
diff --git a/lib/libgssapi/gss_init_sec_context.3 b/lib/libgssapi/gss_init_sec_context.3
index 5c39de4..787aff3 100644
--- a/lib/libgssapi/gss_init_sec_context.3
+++ b/lib/libgssapi/gss_init_sec_context.3
@@ -27,7 +27,7 @@
 .\"	$FreeBSD$
 .\"
 .\" The following commands are required for all man pages.
-.Dd October 30, 2007
+.Dd January 26, 2010
 .Os
 .Dt GSS_INIT_SEC_CONTEXT 3 PRM
 .Sh NAME
@@ -538,7 +538,7 @@ Generic Security Service API Version 2 : C-bindings
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 7.0 .
 .Sh AUTHORS
 John Wray, Iris Associates
diff --git a/lib/libgssapi/gss_inquire_context.3 b/lib/libgssapi/gss_inquire_context.3
index 2ecd4d6..51fa18d 100644
--- a/lib/libgssapi/gss_inquire_context.3
+++ b/lib/libgssapi/gss_inquire_context.3
@@ -27,7 +27,7 @@
 .\"	$FreeBSD$
 .\"
 .\" The following commands are required for all man pages.
-.Dd October 30, 2007
+.Dd January 26, 2010
 .Os
 .Dt GSS_INQUIRE_CONTEXT 3 PRM
 .Sh NAME
@@ -251,7 +251,7 @@ Generic Security Service API Version 2 : C-bindings
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 7.0 .
 .Sh AUTHORS
 John Wray, Iris Associates
diff --git a/lib/libgssapi/gss_inquire_cred.3 b/lib/libgssapi/gss_inquire_cred.3
index 979f96e..99c20ce 100644
--- a/lib/libgssapi/gss_inquire_cred.3
+++ b/lib/libgssapi/gss_inquire_cred.3
@@ -27,7 +27,7 @@
 .\"	$FreeBSD$
 .\"
 .\" The following commands are required for all man pages.
-.Dd October 30, 2007
+.Dd January 26, 2010
 .Os
 .Dt GSS_INQUIRE_CRED 3 PRM
 .Sh NAME
@@ -125,7 +125,7 @@ Generic Security Service API Version 2 : C-bindings
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 7.0 .
 .Sh AUTHORS
 John Wray, Iris Associates
diff --git a/lib/libgssapi/gss_inquire_cred_by_mech.3 b/lib/libgssapi/gss_inquire_cred_by_mech.3
index 85b5ecc..22319bd 100644
--- a/lib/libgssapi/gss_inquire_cred_by_mech.3
+++ b/lib/libgssapi/gss_inquire_cred_by_mech.3
@@ -27,7 +27,7 @@
 .\"	$FreeBSD$
 .\"
 .\" The following commands are required for all man pages.
-.Dd October 30, 2007
+.Dd January 26, 2010
 .Os
 .Dt GSS_INQUIRE_CRED_BY_MECH 3 PRM
 .Sh NAME
@@ -139,7 +139,7 @@ Generic Security Service API Version 2 : C-bindings
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 7.0 .
 .Sh AUTHORS
 John Wray, Iris Associates
diff --git a/lib/libgssapi/gss_inquire_mechs_for_name.3 b/lib/libgssapi/gss_inquire_mechs_for_name.3
index 91d1c81..b56e663 100644
--- a/lib/libgssapi/gss_inquire_mechs_for_name.3
+++ b/lib/libgssapi/gss_inquire_mechs_for_name.3
@@ -27,7 +27,7 @@
 .\"	$FreeBSD$
 .\"
 .\" The following commands are required for all man pages.
-.Dd October 30, 2007
+.Dd January 26, 2010
 .Os
 .Dt GSS_INQUIRE_MECHS_FOR_NAME 3 PRM
 .Sh NAME
@@ -100,7 +100,7 @@ Generic Security Service API Version 2 : C-bindings
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 7.0 .
 .Sh AUTHORS
 John Wray, Iris Associates
diff --git a/lib/libgssapi/gss_inquire_names_for_mech.3 b/lib/libgssapi/gss_inquire_names_for_mech.3
index 64d8c9c..8b7f069 100644
--- a/lib/libgssapi/gss_inquire_names_for_mech.3
+++ b/lib/libgssapi/gss_inquire_names_for_mech.3
@@ -27,7 +27,7 @@
 .\"	$FreeBSD$
 .\"
 .\" The following commands are required for all man pages.
-.Dd October 30, 2007
+.Dd January 26, 2010
 .Os
 .Dt GSS_INQUIRE_NAMES_FOR_MECH 3 PRM
 .Sh NAME
@@ -74,7 +74,7 @@ Generic Security Service API Version 2 : C-bindings
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 7.0 .
 .Sh AUTHORS
 John Wray, Iris Associates
diff --git a/lib/libgssapi/gss_process_context_token.3 b/lib/libgssapi/gss_process_context_token.3
index 884a556..85e1029 100644
--- a/lib/libgssapi/gss_process_context_token.3
+++ b/lib/libgssapi/gss_process_context_token.3
@@ -27,7 +27,7 @@
 .\"	$FreeBSD$
 .\"
 .\" The following commands are required for all man pages.
-.Dd October 30, 2007
+.Dd January 26, 2010
 .Os
 .Dt GSS_PROCESS_CONTEXT_TOKEN 3 PRM
 .Sh NAME
@@ -103,7 +103,7 @@ Generic Security Service API Version 2 : C-bindings
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 7.0 .
 .Sh AUTHORS
 John Wray, Iris Associates
diff --git a/lib/libgssapi/gss_release_buffer.3 b/lib/libgssapi/gss_release_buffer.3
index 4cc449f..432d422 100644
--- a/lib/libgssapi/gss_release_buffer.3
+++ b/lib/libgssapi/gss_release_buffer.3
@@ -27,7 +27,7 @@
 .\"	$FreeBSD$
 .\"
 .\" The following commands are required for all man pages.
-.Dd October 30, 2007
+.Dd January 26, 2010
 .Os
 .Dt GSS_RELEASE_BUFFER 3 PRM
 .Sh NAME
@@ -78,7 +78,7 @@ Generic Security Service API Version 2 : C-bindings
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 7.0 .
 .Sh AUTHORS
 John Wray, Iris Associates
diff --git a/lib/libgssapi/gss_release_cred.3 b/lib/libgssapi/gss_release_cred.3
index 2f9fca6..ca99fc4 100644
--- a/lib/libgssapi/gss_release_cred.3
+++ b/lib/libgssapi/gss_release_cred.3
@@ -27,7 +27,7 @@
 .\"	$FreeBSD$
 .\"
 .\" The following commands are required for all man pages.
-.Dd October 30, 2007
+.Dd January 26, 2010
 .Os
 .Dt GSS_RELEASE_CRED 3 PRM
 .Sh NAME
@@ -75,7 +75,7 @@ Generic Security Service API Version 2 : C-bindings
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 7.0 .
 .Sh AUTHORS
 John Wray, Iris Associates
diff --git a/lib/libgssapi/gss_release_name.3 b/lib/libgssapi/gss_release_name.3
index a6d858c..7bf1181 100644
--- a/lib/libgssapi/gss_release_name.3
+++ b/lib/libgssapi/gss_release_name.3
@@ -27,7 +27,7 @@
 .\"	$FreeBSD$
 .\"
 .\" The following commands are required for all man pages.
-.Dd October 30, 2007
+.Dd January 26, 2010
 .Os
 .Dt GSS_RELEASE_NAME 3 PRM
 .Sh NAME
@@ -71,7 +71,7 @@ Generic Security Service API Version 2 : C-bindings
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 7.0 .
 .Sh AUTHORS
 John Wray, Iris Associates
diff --git a/lib/libgssapi/gss_release_oid_set.3 b/lib/libgssapi/gss_release_oid_set.3
index 1e11f44..0142e2f 100644
--- a/lib/libgssapi/gss_release_oid_set.3
+++ b/lib/libgssapi/gss_release_oid_set.3
@@ -27,7 +27,7 @@
 .\"	$FreeBSD$
 .\"
 .\" The following commands are required for all man pages.
-.Dd October 30, 2007
+.Dd January 26, 2010
 .Os
 .Dt GSS_RELEASE_OID_SET 3 PRM
 .Sh NAME
@@ -76,7 +76,7 @@ Generic Security Service API Version 2 : C-bindings
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 7.0 .
 .Sh AUTHORS
 John Wray, Iris Associates
diff --git a/lib/libgssapi/gss_test_oid_set_member.3 b/lib/libgssapi/gss_test_oid_set_member.3
index 38e5f3f..86e06b3 100644
--- a/lib/libgssapi/gss_test_oid_set_member.3
+++ b/lib/libgssapi/gss_test_oid_set_member.3
@@ -27,7 +27,7 @@
 .\"	$FreeBSD$
 .\"
 .\" The following commands are required for all man pages.
-.Dd October 30, 2007
+.Dd January 26, 2010
 .Os
 .Dt GSS_TEST_OID_SET_MEMBER 3 PRM
 .Sh NAME
@@ -83,7 +83,7 @@ Generic Security Service API Version 2 : C-bindings
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 7.0 .
 .Sh AUTHORS
 John Wray, Iris Associates
diff --git a/lib/libgssapi/gss_unwrap.3 b/lib/libgssapi/gss_unwrap.3
index 770d9c9..4cf3c6d 100644
--- a/lib/libgssapi/gss_unwrap.3
+++ b/lib/libgssapi/gss_unwrap.3
@@ -27,7 +27,7 @@
 .\"	$FreeBSD$
 .\"
 .\" The following commands are required for all man pages.
-.Dd October 30, 2007
+.Dd January 26, 2010
 .Os
 .Dt GSS_UNWRAP 3 PRM
 .Sh NAME
@@ -158,7 +158,7 @@ Generic Security Service API Version 2 : C-bindings
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 7.0 .
 .Sh AUTHORS
 John Wray, Iris Associates
diff --git a/lib/libgssapi/gss_verify_mic.3 b/lib/libgssapi/gss_verify_mic.3
index e482dbf..b72f468 100644
--- a/lib/libgssapi/gss_verify_mic.3
+++ b/lib/libgssapi/gss_verify_mic.3
@@ -27,7 +27,7 @@
 .\"	$FreeBSD$
 .\"
 .\" The following commands are required for all man pages.
-.Dd October 30, 2007
+.Dd January 26, 2010
 .Os
 .Dt GSS_VERIFY_MIC 3 PRM
 .Sh NAME
@@ -139,7 +139,7 @@ Generic Security Service API Version 2 : C-bindings
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 7.0 .
 .Sh AUTHORS
 John Wray, Iris Associates
diff --git a/lib/libgssapi/gss_wrap.3 b/lib/libgssapi/gss_wrap.3
index a80cf46..c432b48 100644
--- a/lib/libgssapi/gss_wrap.3
+++ b/lib/libgssapi/gss_wrap.3
@@ -27,7 +27,7 @@
 .\"	$FreeBSD$
 .\"
 .\" The following commands are required for all man pages.
-.Dd October 30, 2007
+.Dd January 26, 2010
 .Os
 .Dt GSS_WRAP 3 PRM
 .Sh NAME
@@ -146,7 +146,7 @@ Generic Security Service API Version 2 : C-bindings
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 7.0 .
 .Sh AUTHORS
 John Wray, Iris Associates
diff --git a/lib/libgssapi/gss_wrap_size_limit.3 b/lib/libgssapi/gss_wrap_size_limit.3
index a3e5a9d..c1d40da 100644
--- a/lib/libgssapi/gss_wrap_size_limit.3
+++ b/lib/libgssapi/gss_wrap_size_limit.3
@@ -27,7 +27,7 @@
 .\"	$FreeBSD$
 .\"
 .\" The following commands are required for all man pages.
-.Dd October 30, 2007
+.Dd January 26, 2010
 .Os
 .Dt GSS_WRAP_SIZE_LIMIT 3 PRM
 .Sh NAME
@@ -131,7 +131,7 @@ Generic Security Service API Version 2 : C-bindings
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+function first appeared in
 .Fx 7.0 .
 .Sh AUTHORS
 John Wray, Iris Associates
diff --git a/lib/libgssapi/gssapi.3 b/lib/libgssapi/gssapi.3
index ce26e7e..2ed54a9 100644
--- a/lib/libgssapi/gssapi.3
+++ b/lib/libgssapi/gssapi.3
@@ -26,7 +26,7 @@
 .\"
 .\"	$FreeBSD$
 .\"
-.Dd November 30, 2005
+.Dd January 26, 2010
 .Dt GSSAPI 3
 .Os
 .Sh NAME
@@ -229,7 +229,7 @@ Generic Security Service API Version 2 : C-bindings
 .Sh HISTORY
 The
 .Nm
-manual page first appeared in
+library first appeared in
 .Fx 7.0 .
 .Sh AUTHORS
 John Wray, Iris Associates
diff --git a/lib/libgssapi/mech.5 b/lib/libgssapi/mech.5
index fd17e3a..6078828 100644
--- a/lib/libgssapi/mech.5
+++ b/lib/libgssapi/mech.5
@@ -23,7 +23,7 @@
 .\" SUCH DAMAGE.
 .\"
 .\" $FreeBSD$
-.Dd November 14, 2005
+.Dd January 26, 2010
 .Dt MECH 5
 .Os
 .Sh NAME
@@ -93,7 +93,7 @@ GSS_KRB5_CONF_C_QOP_DES		0x0100	kerberosv5
 .Sh HISTORY
 The
 .Nm
-manual page example first appeared in
+manual page first appeared in
 .Fx 7.0 .
 .Sh AUTHORS
 This
-- 
cgit v1.1


From 29b12428b8ad7caf2654fa46475ed1587a474fb0 Mon Sep 17 00:00:00 2001
From: joerg <joerg@FreeBSD.org>
Date: Thu, 4 Feb 2010 19:49:07 +0000
Subject: MFC r203356: add a manpage for gpib(3).

---
 lib/libgpib/Makefile |  19 ++
 lib/libgpib/gpib.3   | 738 +++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 757 insertions(+)
 create mode 100644 lib/libgpib/gpib.3

(limited to 'lib')

diff --git a/lib/libgpib/Makefile b/lib/libgpib/Makefile
index 5465d8b..0ead93f 100644
--- a/lib/libgpib/Makefile
+++ b/lib/libgpib/Makefile
@@ -7,4 +7,23 @@ INCSDIR=	${INCLUDEDIR}/gpib
 SRCS=		ibfoo.c
 WARNS?=		6
 
+MAN=		gpib.3
+
+# MLINKS are only provided for functions that are actually
+# implemented; update this if missing pieces have been filled in.
+MLINKS+=	gpib.3 ibclr.3
+MLINKS+=	gpib.3 ibdev.3
+MLINKS+=	gpib.3 ibdma.3
+MLINKS+=	gpib.3 ibeos.3
+MLINKS+=	gpib.3 ibeot.3
+MLINKS+=	gpib.3 ibloc.3
+MLINKS+=	gpib.3 ibonl.3
+MLINKS+=	gpib.3 ibpad.3
+MLINKS+=	gpib.3 ibrd.3
+MLINKS+=	gpib.3 ibsad.3
+MLINKS+=	gpib.3 ibsic.3
+MLINKS+=	gpib.3 ibtmo.3
+MLINKS+=	gpib.3 ibtrg.3
+MLINKS+=	gpib.3 ibwrt.3
+
 .include <bsd.lib.mk>
diff --git a/lib/libgpib/gpib.3 b/lib/libgpib/gpib.3
new file mode 100644
index 0000000..a57d3a4
--- /dev/null
+++ b/lib/libgpib/gpib.3
@@ -0,0 +1,738 @@
+.\" Copyright (c) 2010, Joerg Wunsch
+.\" All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd February 1, 2010
+.Dt GPIB 3
+.Os
+.Sh NAME
+.\" .Nm ibask ,
+.\" .Nm ibbna ,
+.\" .Nm ibcac ,
+.Nm ibclr ,
+.\" .Nm ibcmd ,
+.\" .Nm ibcmda ,
+.\" .Nm ibconfig ,
+.Nm ibdev ,
+.\" .Nm ibdiag ,
+.Nm ibdma ,
+.Nm ibeos ,
+.Nm ibeot ,
+.\" .Nm ibevent ,
+.\" .Nm ibfind ,
+.\" .Nm ibgts ,
+.\" .Nm ibist ,
+.\" .Nm iblines ,
+.\" .Nm ibllo ,
+.\" .Nm ibln ,
+.Nm ibloc ,
+.Nm ibonl ,
+.Nm ibpad ,
+.\" .Nm ibpct ,
+.\" .Nm ibpoke ,
+.\" .Nm ibppc ,
+.Nm ibrd ,
+.\" .Nm ibrda ,
+.\" .Nm ibrdf ,
+.\" .Nm ibrdkey ,
+.\" .Nm ibrpp ,
+.\" .Nm ibrsc ,
+.\" .Nm ibrsp ,
+.\" .Nm ibrsv ,
+.Nm ibsad ,
+.\" .Nm ibsgnl ,
+.Nm ibsic ,
+.\" .Nm ibsre ,
+.\" .Nm ibsrq ,
+.\" .Nm ibstop ,
+.Nm ibtmo ,
+.\" .Nm ibtrap ,
+.Nm ibtrg ,
+.\" .Nm ibwait ,
+.Nm ibwrt
+.\" .Nm ibwrta ,
+.\" .Nm ibwrtf ,
+.\" .Nm ibwrtkey ,
+.\" .Nm ibxtrc
+.Nd "GPIB library"
+.Sh LIBRARY
+.Lb libgpib
+.Sh SYNOPSIS
+.In gpib.h
+.Pp
+.Dv extern int ibcnt ,
+.Dv iberr ,
+.Dv ibsta ;
+.Pp
+.Ft int
+.Fn ibask "int handle" "int option" "int *retval"
+.Ft int
+.Fn ibbna "int handle" "char *bdname"
+.Ft int
+.Fn ibcac "int handle" "int v"
+.Ft int
+.Fn ibclr "int handle"
+.Ft int
+.Fn ibcmd "int handle" "void *buffer" "long cnt"
+.Ft int
+.Fn ibcmda "int handle" "void *buffer" "long cnt"
+.Ft int
+.Fn ibconfig "int handle" "int option" "int value"
+.Ft int
+.Fn ibdev "int boardID" "int pad" "int sad" "int tmo" "int eot" "int eos"
+.Ft int
+.Fn ibdiag "int handle" "void *buffer" "long cnt"
+.Ft int
+.Fn ibdma "int handle" "int v"
+.Ft int
+.Fn ibeos "int handle" "int eos"
+.Ft int
+.Fn ibeot "int handle" "int eot"
+.Ft int
+.Fn ibevent "int handle" "short *event"
+.Ft int
+.Fn ibfind "char *bdname"
+.Ft int
+.Fn ibgts "int handle" "int v"
+.Ft int
+.Fn ibist "int handle" "int v"
+.Ft int
+.Fn iblines "int handle" "short *lines"
+.Ft int
+.Fn ibllo "int handle"
+.Ft int
+.Fn ibln "int handle" "int padval" "int sadval" "short *listenflag"
+.Ft int
+.Fn ibloc "int handle"
+.Ft int
+.Fn ibonl "int handle" "int v"
+.Ft int
+.Fn ibpad "int handle" "int pad"
+.Ft int
+.Fn ibpct "int handle"
+.Ft int
+.Fn ibpoke "int handle" "int option" "int value"
+.Ft int
+.Fn ibppc "int handle" "int v"
+.Ft int
+.Fn ibrd "int handle" "void *buffer" "long cnt"
+.Ft int
+.Fn ibrda "int handle" "void *buffer" "long cnt"
+.Ft int
+.Fn ibrdf "int handle" "char *flname"
+.Ft int
+.Fn ibrdkey "int handle" "void *buffer" "int cnt"
+.Ft int
+.Fn ibrpp "int handle" "char *ppr"
+.Ft int
+.Fn ibrsc "int handle" "int v"
+.Ft int
+.Fn ibrsp "int handle" "char *spr"
+.Ft int
+.Fn ibrsv "int handle" "int v"
+.Ft int
+.Fn ibsad "int handle" "int sad"
+.Ft int
+.Fn ibsgnl "int handle" "int v"
+.Ft int
+.Fn ibsic "int handle"
+.Ft int
+.Fn ibsre "int handle" "int v"
+.Ft int
+.Fn ibsrq "(*func) void)"
+.Ft int
+.Fn ibstop "int handle"
+.Ft int
+.Fn ibtmo "int handle" "int tmo"
+.Ft int
+.Fn ibtrap "int  mask" "int mode"
+.Ft int
+.Fn ibtrg "int handle"
+.Ft int
+.Fn ibwait "int handle" "int mask"
+.Ft int
+.Fn ibwrt "int handle" "const void *buffer" "long cnt"
+.Ft int
+.Fn ibwrta "int handle" "const void *buffer" "long cnt"
+.Ft int
+.Fn ibwrtf "int handle" "const char *flname"
+.Ft int
+.Fn ibwrtkey "int handle" "const void *buffer" "int cnt"
+.Ft int
+.Fn ibxtrc "int handle" "void *buffer" "long cnt"
+.Sh DESCRIPTION
+The
+.Nm
+library provides access to the
+.Xr gpib 4
+kernel devices.
+.Ss Variable Description
+The variable
+.Dv ibcnt
+contains the number of bytes transferred in the most recent call to
+.Fn ibcmd ,
+.Fn ibrd ,
+or
+.Fn ibwrt .
+.Pp
+The name
+.Dv ibcntl
+is an alias for
+.Dv ibcnt ,
+provided for backwards compatibility.
+.Pp
+The variable
+.Dv iberr
+provides an error code for the most recent library call.
+The possible error codes are:
+.Bl -tag -offset indent -compact
+.It EDVR
+System error
+.It ECIC
+Not Active Controller
+.It ENOL
+Nobody listening
+.It EADR
+Controller not addressed
+.It EARG
+Invalid argument
+.It ESAC
+Not System Controller
+.It EABO
+I/O Aborted/Time out
+.It ENEB
+No such controller
+.It EOIP
+Async I/O in progress
+.It ECAP
+No such capability
+.It EFSO
+File system error
+.It EBUS
+Command byte xfer error
+.It ESTB
+Serial poll status byte lost
+.It ESRQ
+SRQ line stuck
+.It ETAB
+Table problem
+.El
+.Pp
+The variable
+.Dv ibsta
+contains the controller status.
+This is an ORed status value, with the following individual bit names:
+.Bl -tag -offset indent -compact
+.It ERR
+Error
+.It TIMO
+Timeout
+.It END
+EOI/EOS
+.It SRQI
+SRQ
+.It RQS
+Device requests service
+.It SPOLL
+Serial Poll
+.It EVENT
+Event occured
+.It CMPL
+I/O complete
+.It LOK
+Lockout
+.It REM
+Remote
+.It CIC
+CIC
+.It ATN
+ATN
+.It TACS
+Talker
+.It LACS
+Listener
+.It DTAS
+Device trigger status
+.It DCAS
+Device clear state
+.El
+.Ss Function Description
+.Pp
+The function
+.Fn ibdev
+is used to open the GPIB device, and establish the parameters to
+communicate with a particular bus device. The device is selected
+by its primary address
+.Fa pad ,
+a numerical value between 0 and 30, possibly additionally by its
+secondary address
+.Fa sad ,
+a numerical value between 96 and 126, or 0 to not use secondary
+addressing.
+The
+.Fa tmo
+value specifies the timeout to use when communicating with the device.
+This can be any of the constants
+.Dv TNONE ,
+.Dv T10us ,
+.Dv T30us ,
+.Dv T100us ,
+.Dv T300us ,
+.Dv T1ms ,
+.Dv T3ms ,
+.Dv T10ms ,
+.Dv T30ms ,
+.Dv T100ms ,
+.Dv T300ms ,
+.Dv T1s ,
+.Dv T3s ,
+.Dv T10s ,
+.Dv T30s ,
+.Dv T100s ,
+.Dv T300s ,
+or
+.Dv T1000s .
+The boolean parameter
+.Fa eot
+specifies whether the bus signal
+.Li EOI
+(end-or-identify) should be asserted when sending the last byte of a
+message to the device.
+Finally, the
+.Fa eos
+parameter determines whether any special character should be used to
+identify the end of a device message when transferring messages on the
+bus.
+The lower 8 bits of
+.Fa eos
+are interpreted as an end-of-string character,
+.Li EOS .
+This character can be ORed with the following values:
+.Bl -tag -compact -offset indent
+.It Dv REOS
+When receiving a message byte on the bus that matches the
+.Li EOS
+character, treat it as if the
+.Li EOI
+signal were asserted, and stop receiving.
+.It Dv XEOS
+When transmitting a message byte on the bus that matches the
+.Li EOS
+character, assert the
+.Li EOI
+bus signal by the same time, and stop sending.
+.It Dv BIN
+If set, include all 8 bits of the
+.Li EOS
+character in the comparison; if unset, compare only 7 bit ASCII
+values.
+.El
+Passing 0 as
+.Fa eos
+will turn off any special character treatment, allowing for a fully
+8-bit transparent communications channel to the device.
+.Pp
+The function
+.Fn ibfind
+is meant to find the
+.Em board index
+of a board identified by the name
+.Fa bdname .
+.Em This function is currently not implemented.
+.Pp
+All remaining functions take the handle returned by calling
+.Fn ibdev
+as their first argument
+.Fa handle .
+.Pp
+The function
+.Fn ibask
+is used to query configuration values that have been set with
+.Fn ibconfig .
+.Em This function is currently not implemented.
+.Pp
+The function
+.Fn ibbna
+is meant to change the access board for the given device to
+a new one, named
+.Fa bdname .
+.Em This function is currently not implemented.
+.Pp
+The function
+.Fn ibcac
+is used to become the active controller on the bus, by asserting the
+.Li ATN
+signal line.
+.Em This function is currently not implemented.
+.Pp
+The function
+.Fn ibclr
+is used to transmit a
+.Em Selected Device Clear
+command to the device.
+.Pp
+The function
+.Fn ibcmd
+is used to directly write
+.Fa cnt
+GPIB command bytes from a buffer starting at
+.Fa buffer
+to the device.
+.Em This function is currently not implemented.
+.Pp
+The function
+.Fn ibcmda
+does the same as
+.Fn ibcmd
+except it operates asynchronously, so it returns to the caller
+immediately.
+.Em This function is currently not implemented.
+.Pp
+The function
+.Fn ibconfig
+is used to set certain configuration parameters.
+.Em This function is currently not implemented.
+.Pp
+The function
+.Fn ibdiag
+is obsolete, and not implemented.
+.Pp
+The function
+.Fn ibdma
+is used to enable or disable DMA transfers.
+Parameter
+.Fa v
+is a boolean parameter indicating DMA transfers are to be used.
+Depending on the hardware and operating system configuration, DMA
+transfers might not be available for a particular access board.
+.Pp
+The function
+.Fn ibeos
+configures the end-of-string character.
+See
+.Fn ibdev
+for an explanation.
+.Pp
+The function
+.Fn ibeot
+configures the assertion of the
+.Li EOI
+signal line when transmitting the last byte of a message; see
+.Fn ibdev
+for an explanation.
+.Pp
+The function
+.Fn ibevent
+is used to obtain an event from the board's event queue.
+.Em This function is currently not implemented.
+.Pp
+The function
+.Fn ibgts
+makes the current controller the standby controller, by deasserting
+the
+.Li ATN
+signal line.
+.Em This function is currently not implemented.
+.Pp
+The function
+.Fn ibist
+sets the individual status bits of the controller to the value
+.Fa v .
+.Em This function is currently not implemented.
+.Pp
+The function
+.Fn iblines
+returns the status of the control and handshake bus lines into the
+area pointed to by
+.Fa lines .
+.Em This function is currently not implemented.
+.Pp
+The function
+.Fn ibllo
+is obsolete, and not implemented.
+.Pp
+The function
+.Fn ibln
+checks for a listener at the primary address
+.Fa  padval
+and the optional secondary address
+.Fa sadval .
+If a listener was found, the value pointed to by
+.Fa listenflag
+will be set to a non-zero value.
+.Em This function is currently not implemented.
+.Pp
+The function
+.Fn ibloc
+turns the device into local mode.
+.Pp
+The function
+.Fn ibonl
+is used to close or reinitialize a device handle.
+If parameter
+.Fa v
+is passed as zero, the handle will be closed, and cannot be used
+again.
+If it is passed as a non-zero value, all parameters of the handle
+will be returned to their defaults;
+.Em this functionality is currently unsupported.
+.Pp
+The function
+.Fn ibpad
+is used to change the primary address of the device being communicated
+with to
+.Fa pad .
+See
+.Fn ibdev
+for an explanation.
+.Pp
+The function
+.Fn ibpct
+is used to make the device associated with the handle the
+controller-in-charge.
+.Em This function is currently not implemented.
+.Pp
+The function
+.Fn ibpoke
+is obsolete, and not implemented.
+.Pp
+The function
+.Fn ibppc
+is used to configure the parallel poll response to
+.Fa v .
+.Em This function is currently not implemented.
+.Pp
+The fucntion
+.Fn ibrd
+is used to receive
+.Fa cnt
+bytes from the device, and store it to the address passed as
+.Fa buffer .
+.Pp
+The function
+.Fn ibrda
+behaves similar to
+.Fn ibrd
+except it operates asynchronously, and returns immediately to the
+caller.
+.Em This function is currently not implemented.
+.Pp
+The function
+.Fn ibrdf
+read data from the device, and appends it to the file with the name
+.Fa flname .
+.Em This function is currently not implemented.
+.Pp
+The function
+.Fn ibrdkey
+is obsolete, and not implemented.
+.Pp
+The function
+.Fn ibrpp
+performs a parallel poll, and stores the result at the location
+pointed to by
+.Fa ppr .
+.Em This function is currently not implemented.
+.Pp
+The function
+.Fn ibrsc
+makes the board specified by the handle the
+.Em system controller
+if the argument
+.Fa v
+is non-zero.
+.Em This function is currently not implemented.
+.Pp
+The function
+.Fn ibrsp
+conducts a serial poll, and stores the result in the byte pointed
+to by
+.Fa spr .
+.Em This function is currently not implemented.
+.Pp
+The function
+.Fn ibrsv
+sets the serial poll response of the board to
+.Fa v ,
+possibly requesting service from the controller if the SRQ bit (0x40)
+is set.
+.Em This function is currently not implemented.
+.Pp
+The function
+.Fn ibsad
+changes the secondary address of the device being communicated with to
+.Fa sad .
+See
+.Fn ibdev
+for an explanation.
+.Pp
+The function
+.Fn ibsgnl
+is obsolete, and not implemented.
+.Pp
+The function
+.Fn ibsic
+asserts the
+.Em Interface Clear (IFC)
+signal line on the bus for at least 100 microseconds.
+This will make all devices attached to the bus to unlisten and untalk.
+This function should only be executed on the system controller.
+.Pp
+The function
+.Fn ibsre
+asserts the
+.Em Remote Enable (REN)
+signal line on the bus if argument
+.Fa v
+is non-zero, or deasserts it otherwise.
+.Em This function is currently not implemented.
+.Pp
+The function
+.Fn ibsrq
+is obsolete, and not implemented.
+.Pp
+The function
+.Fn ibstop
+stops or aborts any asynchronous I/O operation.
+.Em This function is currently not implemented.
+.Pp
+The function
+.Fn ibtmo
+reconfigures the communication timeout.
+See
+.Fn ibdev
+for an explanation.
+.Pp
+The function
+.Fn ibtrap
+is obsolete, and not implemented.
+.Pp
+The function
+.Fn ibtrg
+sends a
+.Em Group Execute Trigger (GET)
+command to the device.
+.Pp
+The function
+.Fn ibwait
+waits for a status condition as specified by
+.Fa mask .
+If
+.Fa mask
+is given as zero, it returns immediately.
+.Em This function is currently not implemented.
+.Pp
+The function
+.Fn ibwrt
+is used to send
+.Fa cnt
+bytes to the device, starting at the address pointed to by
+.Fa buffer .
+.Pp
+The function
+.Fn ibwrta
+performs the same operation as
+.Fn ibwrt
+in an asynchronous way, returning immediately to the caller.
+.Em This function is currently not implemented.
+.Pp
+The function
+.Fn ibwrtf
+opens the file named by
+.Fa flname ,
+and sends its contents to the device.
+.Em This function is currently not implemented.
+.Pp
+The function
+.Fn ibwrtkey
+is obsolete, and not implemented
+.Pp
+The function
+.Fn ibxtrc
+is obsolete, and not implemented.
+.Sh RETURN VALUES
+The function
+.Fn ibdev
+returns a handle to be used for the remaining functions.
+Upon failure, -1 is returned.
+.Pp
+All other functions return the value of the variable
+.Dv ibsta .
+.Sh DIAGNOSTICS
+None.
+.Sh COMPATIBILITY
+The
+.Nm
+library tries to be compatible with the Linux GPIB library,
+which in turn appears to be compatible with the GPIB library
+shipped by National Instruments.
+.Sh ERRORS
+Errors in the functions above might set
+.Dv errno
+to one of these values:
+.Bl -tag -width Er
+.It Bq Er ENOENT
+No such file or directory.
+.It Bq Er EIO
+Input/output error.
+.It Bq Er ENXIO
+Device not configured.
+.It Bq Er E2BIG
+Argument list too long.
+.It Bq Er ENOMEM
+Cannot allocate memory.
+.It Bq Er EACCES
+Permission denied.
+.It Bq Er EFAULT
+Bad address.
+.It Bq Er EBUSY
+Device busy.
+.It Bq Er EINVAL
+Invalid argument.
+.It Bq Er ENFILE
+Too many open files in system.
+.It Bq Er EMFILE
+Too many open files.
+.It Bq Er EOPNOTSUPP
+Operation not supported.
+.El
+.Sh SEE ALSO
+.Xr gpib 4
+.Sh HISTORY
+The
+.Nm
+library was written by
+.An Poul-Henning Kamp
+and first appeared in
+.Fx 5.4 .
+.Sh AUTHORS
+This manual page was written by
+.An J\(:org Wunsch .
+.Sh BUGS
+Currently, the library can only handle a single
+.Xr gpib 4
+device with instance number 0.
+.Pp
+Many functions are currently not implemented, see above for details.
-- 
cgit v1.1


From 6815fe2938e067978f7c3fc931e213b3a5ae7f05 Mon Sep 17 00:00:00 2001
From: ume <ume@FreeBSD.org>
Date: Sun, 7 Feb 2010 04:27:18 +0000
Subject: MFC r202916: Make strsignal(3) thread-safe.

---
 lib/libc/string/strsignal.c | 52 +++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 48 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/string/strsignal.c b/lib/libc/string/strsignal.c
index e4267a3..c51f34d 100644
--- a/lib/libc/string/strsignal.c
+++ b/lib/libc/string/strsignal.c
@@ -33,22 +33,64 @@ static char sccsid[] = "@(#)strerror.c	8.1 (Berkeley) 6/4/93";
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include "namespace.h"
 #if defined(NLS)
 #include <nl_types.h>
 #endif
-
 #include <limits.h>
 #include <errno.h>
+#include <stdlib.h>
 #include <string.h>
 #include <signal.h>
+#include "reentrant.h"
+#include "un-namespace.h"
 
 #define	UPREFIX		"Unknown signal"
 
+static char		sig_ebuf[NL_TEXTMAX];
+static char		sig_ebuf_err[NL_TEXTMAX];
+static once_t		sig_init_once = ONCE_INITIALIZER;
+static thread_key_t	sig_key;
+static int		sig_keycreated = 0;
+
+static void
+sig_keycreate(void)
+{
+	sig_keycreated = (thr_keycreate(&sig_key, free) == 0);
+}
+
+static char *
+sig_tlsalloc(void)
+{
+	char *ebuf = NULL;
+
+	if (thr_main() != 0)
+		ebuf = sig_ebuf;
+	else {
+		if (thr_once(&sig_init_once, sig_keycreate) != 0 ||
+		    !sig_keycreated)
+			goto thr_err;
+		if ((ebuf = thr_getspecific(sig_key)) == NULL) {
+			if ((ebuf = malloc(sizeof(sig_ebuf))) == NULL)
+				goto thr_err;
+			if (thr_setspecific(sig_key, ebuf) != 0) {
+				free(ebuf);
+				ebuf = NULL;
+				goto thr_err;
+			}
+		}
+	}
+thr_err:
+	if (ebuf == NULL)
+		ebuf = sig_ebuf_err;
+	return (ebuf);
+}
+
 /* XXX: negative 'num' ? (REGR) */
 char *
 strsignal(int num)
 {
-	static char ebuf[NL_TEXTMAX];
+	char *ebuf;
 	char tmp[20];
 	size_t n;
 	int signum;
@@ -60,6 +102,8 @@ strsignal(int num)
 	catd = catopen("libc", NL_CAT_LOCALE);
 #endif
 
+	ebuf = sig_tlsalloc();
+
 	if (num > 0 && num < sys_nsig) {
 		n = strlcpy(ebuf,
 #if defined(NLS)
@@ -67,7 +111,7 @@ strsignal(int num)
 #else
 			sys_siglist[num],
 #endif
-			sizeof(ebuf));
+			sizeof(sig_ebuf));
 	} else {
 		n = strlcpy(ebuf,
 #if defined(NLS)
@@ -75,7 +119,7 @@ strsignal(int num)
 #else
 			UPREFIX,
 #endif
-			sizeof(ebuf));
+			sizeof(sig_ebuf));
 	}
 
 	signum = num;
-- 
cgit v1.1


From 4d2b1b10d9a30417eb60d614c3dab6805613a24d Mon Sep 17 00:00:00 2001
From: dougb <dougb@FreeBSD.org>
Date: Sun, 7 Feb 2010 20:26:45 +0000
Subject: MFC 202960:

Copyright-only changes to generated files as part of the 9.6.1-P3 update
---
 lib/bind/dns/code.h            | 2 +-
 lib/bind/dns/dns/enumclass.h   | 2 +-
 lib/bind/dns/dns/enumtype.h    | 2 +-
 lib/bind/dns/dns/rdatastruct.h | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/bind/dns/code.h b/lib/bind/dns/code.h
index db8d8c5..264fc11 100644
--- a/lib/bind/dns/code.h
+++ b/lib/bind/dns/code.h
@@ -1,7 +1,7 @@
 /* $FreeBSD$ */
 
 /*
- * Copyright (C) 2004-2009 Internet Systems Consortium, Inc. ("ISC")
+ * Copyright (C) 2004-2010 Internet Systems Consortium, Inc. ("ISC")
  * Copyright (C) 1998-2003 Internet Software Consortium.
  *
  * Permission to use, copy, modify, and distribute this software for any
diff --git a/lib/bind/dns/dns/enumclass.h b/lib/bind/dns/dns/enumclass.h
index b1e363f..e33d4c4 100644
--- a/lib/bind/dns/dns/enumclass.h
+++ b/lib/bind/dns/dns/enumclass.h
@@ -1,7 +1,7 @@
 /* $FreeBSD$ */
 
 /*
- * Copyright (C) 2004-2009 Internet Systems Consortium, Inc. ("ISC")
+ * Copyright (C) 2004-2010 Internet Systems Consortium, Inc. ("ISC")
  * Copyright (C) 1998-2003 Internet Software Consortium.
  *
  * Permission to use, copy, modify, and distribute this software for any
diff --git a/lib/bind/dns/dns/enumtype.h b/lib/bind/dns/dns/enumtype.h
index 74a6db7..4ad3358 100644
--- a/lib/bind/dns/dns/enumtype.h
+++ b/lib/bind/dns/dns/enumtype.h
@@ -1,7 +1,7 @@
 /* $FreeBSD$ */
 
 /*
- * Copyright (C) 2004-2009 Internet Systems Consortium, Inc. ("ISC")
+ * Copyright (C) 2004-2010 Internet Systems Consortium, Inc. ("ISC")
  * Copyright (C) 1998-2003 Internet Software Consortium.
  *
  * Permission to use, copy, modify, and distribute this software for any
diff --git a/lib/bind/dns/dns/rdatastruct.h b/lib/bind/dns/dns/rdatastruct.h
index d723941..94f57b8 100644
--- a/lib/bind/dns/dns/rdatastruct.h
+++ b/lib/bind/dns/dns/rdatastruct.h
@@ -1,7 +1,7 @@
 /* $FreeBSD$ */
 
 /*
- * Copyright (C) 2004-2009 Internet Systems Consortium, Inc. ("ISC")
+ * Copyright (C) 2004-2010 Internet Systems Consortium, Inc. ("ISC")
  * Copyright (C) 1998-2003 Internet Software Consortium.
  *
  * Permission to use, copy, modify, and distribute this software for any
-- 
cgit v1.1


From c39123cbe1bc4b4f762bd1e2b62779a329c24505 Mon Sep 17 00:00:00 2001
From: emaste <emaste@FreeBSD.org>
Date: Tue, 9 Feb 2010 13:07:32 +0000
Subject: MFC r203077:

  Add missing return, in a rare case where we can't allocate memory in
  deallocate.

Submitted by: Ryan Stone (rysto32 at gmail dot com)
Approved by:  jasone
---
 lib/libc/stdlib/malloc.c | 1 +
 1 file changed, 1 insertion(+)

(limited to 'lib')

diff --git a/lib/libc/stdlib/malloc.c b/lib/libc/stdlib/malloc.c
index 80e26e6..8bc2c85 100644
--- a/lib/libc/stdlib/malloc.c
+++ b/lib/libc/stdlib/malloc.c
@@ -3861,6 +3861,7 @@ arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr)
 					arena_dalloc_small(arena, chunk, ptr,
 					    mapelm);
 					malloc_spin_unlock(&arena->lock);
+					return;
 				}
 				mag_rack = rack;
 			}
-- 
cgit v1.1


From 6d70c92b6e47fbf677df3874a9d4a40f1a8e9ced Mon Sep 17 00:00:00 2001
From: gavin <gavin@FreeBSD.org>
Date: Sat, 13 Feb 2010 10:26:00 +0000
Subject: Merge r203393,r203395 from head:

  The multiplicand a = 0x5deece66d = 25214903917, not 0xfdeece66d.
  This bug in the man page has gone unnoticed for over 15 years!

PR:		docs/143461
Submitted by:	Jeremy Huddleston  jeremyhu apple.com
---
 lib/libc/gen/rand48.3 | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/gen/rand48.3 b/lib/libc/gen/rand48.3
index a703ba0..8d325a1 100644
--- a/lib/libc/gen/rand48.3
+++ b/lib/libc/gen/rand48.3
@@ -12,7 +12,7 @@
 .\"     @(#)rand48.3 V1.0 MB 8 Oct 1993
 .\" $FreeBSD$
 .\"
-.Dd October 8, 1993
+.Dd February 2, 2010
 .Dt RAND48 3
 .Os
 .Sh NAME
@@ -57,7 +57,7 @@ The
 particular formula employed is
 r(n+1) = (a * r(n) + c) mod m
 where the default values are
-for the multiplicand a = 0xfdeece66d = 25214903917 and
+for the multiplicand a = 0x5deece66d = 25214903917 and
 the addend c = 0xb = 11.
 The modulo is always fixed at m = 2 ** 48.
 r(n) is called the seed of the random number generator.
-- 
cgit v1.1


From 2107ded34e22813b72aeb97cc608029162d93758 Mon Sep 17 00:00:00 2001
From: kib <kib@FreeBSD.org>
Date: Wed, 17 Feb 2010 09:09:12 +0000
Subject: MFC r203441: Placate new binutils, by using 16-bit %ax instead of
 32-bit %eax as an argument for fnstsw. Explicitely specify sizes for the XMM
 control and status word and X87 control and status words.

---
 lib/msun/amd64/fenv.c | 11 +++++++----
 lib/msun/amd64/fenv.h | 13 ++++++++-----
 lib/msun/i387/fenv.c  | 15 +++++++++------
 lib/msun/i387/fenv.h  | 17 ++++++++++-------
 4 files changed, 34 insertions(+), 22 deletions(-)

(limited to 'lib')

diff --git a/lib/msun/amd64/fenv.c b/lib/msun/amd64/fenv.c
index cf07c8b..e126070 100644
--- a/lib/msun/amd64/fenv.c
+++ b/lib/msun/amd64/fenv.c
@@ -86,7 +86,7 @@ fegetenv(fenv_t *envp)
 int
 feholdexcept(fenv_t *envp)
 {
-	int mxcsr;
+	__uint32_t mxcsr;
 
 	__stmxcsr(&mxcsr);
 	__fnstenv(&envp->__x87);
@@ -101,7 +101,8 @@ feholdexcept(fenv_t *envp)
 int
 feupdateenv(const fenv_t *envp)
 {
-	int mxcsr, status;
+	__uint32_t mxcsr;
+	__uint16_t status;
 
 	__fnstsw(&status);
 	__stmxcsr(&mxcsr);
@@ -113,7 +114,8 @@ feupdateenv(const fenv_t *envp)
 int
 __feenableexcept(int mask)
 {
-	int mxcsr, control, omask;
+	__uint32_t mxcsr, omask;
+	__uint16_t control;
 
 	mask &= FE_ALL_EXCEPT;
 	__fnstcw(&control);
@@ -129,7 +131,8 @@ __feenableexcept(int mask)
 int
 __fedisableexcept(int mask)
 {
-	int mxcsr, control, omask;
+	__uint32_t mxcsr, omask;
+	__uint16_t control;
 
 	mask &= FE_ALL_EXCEPT;
 	__fnstcw(&control);
diff --git a/lib/msun/amd64/fenv.h b/lib/msun/amd64/fenv.h
index b4d7379..875664a 100644
--- a/lib/msun/amd64/fenv.h
+++ b/lib/msun/amd64/fenv.h
@@ -110,7 +110,8 @@ feclearexcept(int __excepts)
 static __inline int
 fegetexceptflag(fexcept_t *__flagp, int __excepts)
 {
-	int __mxcsr, __status;
+	__uint32_t __mxcsr;
+	__uint16_t __status;
 
 	__stmxcsr(&__mxcsr);
 	__fnstsw(&__status);
@@ -124,7 +125,8 @@ int feraiseexcept(int __excepts);
 static __inline int
 fetestexcept(int __excepts)
 {
-	int __mxcsr, __status;
+	__uint32_t __mxcsr;
+	__uint16_t __status;
 
 	__stmxcsr(&__mxcsr);
 	__fnstsw(&__status);
@@ -134,7 +136,7 @@ fetestexcept(int __excepts)
 static __inline int
 fegetround(void)
 {
-	int __control;
+	__uint16_t __control;
 
 	/*
 	 * We assume that the x87 and the SSE unit agree on the
@@ -149,7 +151,8 @@ fegetround(void)
 static __inline int
 fesetround(int __round)
 {
-	int __mxcsr, __control;
+	__uint32_t __mxcsr;
+	__uint16_t __control;
 
 	if (__round & ~_ROUND_MASK)
 		return (-1);
@@ -197,7 +200,7 @@ int fedisableexcept(int __mask);
 static __inline int
 fegetexcept(void)
 {
-	int __control;
+	__uint16_t __control;
 
 	/*
 	 * We assume that the masks for the x87 and the SSE unit are
diff --git a/lib/msun/i387/fenv.c b/lib/msun/i387/fenv.c
index ace2d32..5c996b3 100644
--- a/lib/msun/i387/fenv.c
+++ b/lib/msun/i387/fenv.c
@@ -87,7 +87,7 @@ int
 fesetexceptflag(const fexcept_t *flagp, int excepts)
 {
 	fenv_t env;
-	int mxcsr;
+	__uint32_t mxcsr;
 
 	__fnstenv(&env);
 	env.__status &= ~excepts;
@@ -117,7 +117,7 @@ feraiseexcept(int excepts)
 int
 fegetenv(fenv_t *envp)
 {
-	int mxcsr;
+	__uint32_t mxcsr;
 
 	__fnstenv(envp);
 	/*
@@ -135,7 +135,7 @@ fegetenv(fenv_t *envp)
 int
 feholdexcept(fenv_t *envp)
 {
-	int mxcsr;
+	__uint32_t mxcsr;
 
 	__fnstenv(envp);
 	__fnclex();
@@ -152,7 +152,8 @@ feholdexcept(fenv_t *envp)
 int
 feupdateenv(const fenv_t *envp)
 {
-	int mxcsr, status;
+	__uint32_t mxcsr;
+	__uint16_t status;
 
 	__fnstsw(&status);
 	if (__HAS_SSE())
@@ -167,7 +168,8 @@ feupdateenv(const fenv_t *envp)
 int
 __feenableexcept(int mask)
 {
-	int mxcsr, control, omask;
+	__uint32_t mxcsr, omask;
+	__uint16_t control;
 
 	mask &= FE_ALL_EXCEPT;
 	__fnstcw(&control);
@@ -188,7 +190,8 @@ __feenableexcept(int mask)
 int
 __fedisableexcept(int mask)
 {
-	int mxcsr, control, omask;
+	__uint32_t mxcsr, omask;
+	__uint16_t control;
 
 	mask &= FE_ALL_EXCEPT;
 	__fnstcw(&control);
diff --git a/lib/msun/i387/fenv.h b/lib/msun/i387/fenv.h
index c4eab02..cc58a92 100644
--- a/lib/msun/i387/fenv.h
+++ b/lib/msun/i387/fenv.h
@@ -114,7 +114,7 @@ static __inline int
 feclearexcept(int __excepts)
 {
 	fenv_t __env;
-	int __mxcsr;
+	__uint32_t __mxcsr;
 
 	if (__excepts == FE_ALL_EXCEPT) {
 		__fnclex();
@@ -134,7 +134,8 @@ feclearexcept(int __excepts)
 static __inline int
 fegetexceptflag(fexcept_t *__flagp, int __excepts)
 {
-	int __mxcsr, __status;
+	__uint32_t __mxcsr;
+	__uint16_t __status;
 
 	__fnstsw(&__status);
 	if (__HAS_SSE())
@@ -151,7 +152,8 @@ int feraiseexcept(int __excepts);
 static __inline int
 fetestexcept(int __excepts)
 {
-	int __mxcsr, __status;
+	__uint32_t __mxcsr;
+	__uint16_t __status;
 
 	__fnstsw(&__status);
 	if (__HAS_SSE())
@@ -164,7 +166,7 @@ fetestexcept(int __excepts)
 static __inline int
 fegetround(void)
 {
-	int __control;
+	__uint16_t __control;
 
 	/*
 	 * We assume that the x87 and the SSE unit agree on the
@@ -179,7 +181,8 @@ fegetround(void)
 static __inline int
 fesetround(int __round)
 {
-	int __mxcsr, __control;
+	__uint32_t __mxcsr;
+	__uint16_t __control;
 
 	if (__round & ~_ROUND_MASK)
 		return (-1);
@@ -206,7 +209,7 @@ static __inline int
 fesetenv(const fenv_t *__envp)
 {
 	fenv_t __env = *__envp;
-	int __mxcsr;
+	__uint32_t __mxcsr;
 
 	__mxcsr = __get_mxcsr(__env);
 	__set_mxcsr(__env, 0xffffffff);
@@ -234,7 +237,7 @@ int fedisableexcept(int __mask);
 static __inline int
 fegetexcept(void)
 {
-	int __control;
+	__uint16_t __control;
 
 	/*
 	 * We assume that the masks for the x87 and the SSE unit are
-- 
cgit v1.1


From 7d0f076c54c2fecb825ddadf2950088886baf938 Mon Sep 17 00:00:00 2001
From: brucec <brucec@FreeBSD.org>
Date: Thu, 18 Feb 2010 10:51:31 +0000
Subject: MFC r203322:

Fix typo of ENOTCONN.
Add missing RETURN VALUES section in sctp_opt_info(3).

Approved by:	rrs (mentor)
---
 lib/libc/net/sctp_opt_info.3 | 2 ++
 lib/libc/net/sctp_recvmsg.3  | 2 +-
 lib/libc/net/sctp_send.3     | 2 +-
 lib/libc/net/sctp_sendmsg.3  | 2 +-
 4 files changed, 5 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/net/sctp_opt_info.3 b/lib/libc/net/sctp_opt_info.3
index 581a220..f010613 100644
--- a/lib/libc/net/sctp_opt_info.3
+++ b/lib/libc/net/sctp_opt_info.3
@@ -87,6 +87,8 @@ socket options.
 .Dv SCTP_PEER_AUTH_CHUNKS
 .Pp
 .Dv SCTP_LOCAL_AUTH_CHUNKS
+.Sh RETURN VALUES
+The call returns 0 on success and -1 upon error.
 .Sh ERRORS
 The
 .Fn sctp_opt_info
diff --git a/lib/libc/net/sctp_recvmsg.3 b/lib/libc/net/sctp_recvmsg.3
index f874880..a5926bf 100644
--- a/lib/libc/net/sctp_recvmsg.3
+++ b/lib/libc/net/sctp_recvmsg.3
@@ -269,7 +269,7 @@ This generally indicates that the interface has stopped sending,
 but may be caused by transient congestion.
 .It Bq Er EHOSTUNREACH
 The remote host was unreachable.
-.It Bq Er ENOTCON
+.It Bq Er ENOTCONN
 On a one-to-one style socket no association exists.
 .It Bq Er ECONNRESET
 An abort was received by the stack while the user was
diff --git a/lib/libc/net/sctp_send.3 b/lib/libc/net/sctp_send.3
index f6f0c95..010430a 100644
--- a/lib/libc/net/sctp_send.3
+++ b/lib/libc/net/sctp_send.3
@@ -319,7 +319,7 @@ This generally indicates that the interface has stopped sending,
 but may be caused by transient congestion.
 .It Bq Er EHOSTUNREACH
 The remote host was unreachable.
-.It Bq Er ENOTCON
+.It Bq Er ENOTCONN
 On a one-to-one style socket no association exists.
 .It Bq Er ECONNRESET
 An abort was received by the stack while the user was
diff --git a/lib/libc/net/sctp_sendmsg.3 b/lib/libc/net/sctp_sendmsg.3
index 0c5fa02..52faa31 100644
--- a/lib/libc/net/sctp_sendmsg.3
+++ b/lib/libc/net/sctp_sendmsg.3
@@ -296,7 +296,7 @@ This generally indicates that the interface has stopped sending,
 but may be caused by transient congestion.
 .It Bq Er EHOSTUNREACH
 The remote host was unreachable.
-.It Bq Er ENOTCON
+.It Bq Er ENOTCONN
 On a one-to-one style socket no association exists.
 .It Bq Er ECONNRESET
 An abort was received by the stack while the user was
-- 
cgit v1.1


From 927a538c83b04beaf071c9350ee88de781a66a46 Mon Sep 17 00:00:00 2001
From: brucec <brucec@FreeBSD.org>
Date: Thu, 18 Feb 2010 10:55:42 +0000
Subject: MFC r203323:

Remove extra semicolon.

Approved by:	rrs (mentor)
---
 lib/libc/net/sctp_sys_calls.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libc/net/sctp_sys_calls.c b/lib/libc/net/sctp_sys_calls.c
index 715fcbd..d876293 100644
--- a/lib/libc/net/sctp_sys_calls.c
+++ b/lib/libc/net/sctp_sys_calls.c
@@ -784,7 +784,7 @@ sctp_sendx(int sd, const void *msg, size_t msg_len,
 	free(buf);
 	if (ret != 0) {
 		if (errno == EALREADY) {
-			no_end_cx = 1;;
+			no_end_cx = 1;
 			goto continue_send;
 		}
 		return (ret);
-- 
cgit v1.1


From 55df83ef97851d7b62a966a05e14171862779651 Mon Sep 17 00:00:00 2001
From: ru <ru@FreeBSD.org>
Date: Thu, 18 Feb 2010 16:52:49 +0000
Subject: MFC: r203918: Unbreak makefiles by removing mentions of
 DEFAULT_THREAD_LIB.

---
 lib/libc_r/Makefile | 16 ----------------
 lib/libkse/Makefile | 20 --------------------
 2 files changed, 36 deletions(-)

(limited to 'lib')

diff --git a/lib/libc_r/Makefile b/lib/libc_r/Makefile
index 0b286f2..a424876 100644
--- a/lib/libc_r/Makefile
+++ b/lib/libc_r/Makefile
@@ -10,10 +10,6 @@
 
 .include <bsd.own.mk>
 
-.if ${DEFAULT_THREAD_LIB} == "libc_r" && ${SHLIBDIR} == "/usr/lib"
-SHLIBDIR= /lib
-.endif
-
 LIB=c_r
 SHLIB_MAJOR= 7
 CFLAGS+=-DPTHREAD_KERNEL 
@@ -32,16 +28,4 @@ PRECIOUSLIB=
 .include "${.CURDIR}/uthread/Makefile.inc"
 .include "${.CURDIR}/sys/Makefile.inc"
 
-.if ${DEFAULT_THREAD_LIB} == "libc_r"
-.if ${MK_INSTALLLIB} != "no"
-SYMLINKS+=lib${LIB}.a ${LIBDIR}/libpthread.a
-.endif
-.if !defined(NO_PIC)
-SYMLINKS+=lib${LIB}.so ${LIBDIR}/libpthread.so
-.endif
-.if ${MK_PROFILE} != "no"
-SYMLINKS+=lib${LIB}_p.a ${LIBDIR}/libpthread_p.a
-.endif
-.endif
-
 .include <bsd.lib.mk>
diff --git a/lib/libkse/Makefile b/lib/libkse/Makefile
index 9fe09c5..c4e2b5a 100644
--- a/lib/libkse/Makefile
+++ b/lib/libkse/Makefile
@@ -10,15 +10,7 @@
 
 .include <bsd.own.mk>
 
-.if ${DEFAULT_THREAD_LIB} == "libkse" || ${MK_LIBTHR} == "no"
-LIB=kse
-.if ${SHLIBDIR} == "/usr/lib"
-SHLIBDIR= /lib
-.endif
-.else
 SHLIB=kse
-.endif
-
 SHLIB_MAJOR= 4
 CFLAGS+=-DPTHREAD_KERNEL
 CFLAGS+=-I${.CURDIR}/../libc/include -I${.CURDIR}/thread \
@@ -51,16 +43,4 @@ PRECIOUSLIB=
 .include "${.CURDIR}/sys/Makefile.inc"
 .include "${.CURDIR}/thread/Makefile.inc"
 
-.if ${DEFAULT_THREAD_LIB} == "libkse" || ${MK_LIBTHR} == "no"
-.if ${MK_INSTALLLIB} != "no"
-SYMLINKS+=lib${LIB}.a ${LIBDIR}/libpthread.a
-.endif
-.if !defined(NO_PIC)
-SYMLINKS+=lib${LIB}.so ${LIBDIR}/libpthread.so
-.endif
-.if ${MK_PROFILE} != "no"
-SYMLINKS+=lib${LIB}_p.a ${LIBDIR}/libpthread_p.a
-.endif
-.endif
-
 .include <bsd.lib.mk>
-- 
cgit v1.1


From e345edec0f48f330d3c572936a468e2a5552e256 Mon Sep 17 00:00:00 2001
From: ru <ru@FreeBSD.org>
Date: Mon, 22 Feb 2010 15:58:10 +0000
Subject: MFC: r204008: realloc() with a proper amount of memory.

---
 lib/libjail/jail.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libjail/jail.c b/lib/libjail/jail.c
index 39c88d0..c82cb11 100644
--- a/lib/libjail/jail.c
+++ b/lib/libjail/jail.c
@@ -191,7 +191,7 @@ jailparam_all(struct jailparam **jpp)
 		/* Add the parameter to the list */
 		if (njp >= nlist) {
 			nlist *= 2;
-			jp = realloc(jp, nlist * sizeof(jp));
+			jp = realloc(jp, nlist * sizeof(*jp));
 			if (jp == NULL) {
 				jailparam_free(jp, njp);
 				return (-1);
-- 
cgit v1.1


From ddfa9b895b20fc04a73aee44c52f2db7b47d48b6 Mon Sep 17 00:00:00 2001
From: mckusick <mckusick@FreeBSD.org>
Date: Fri, 26 Feb 2010 21:49:11 +0000
Subject: MFC of 203763, 203764, 203768, 203769, 203770, 203782, and 203784.

These fixes correct a problem in the file system that treats large
inode numbers as negative rather than unsigned. For a default
(16K block) file system, this bug began to show up at a file system
size above about 16Tb.

These fixes also update newfs to ensure that it will never create a
filesystem with more than 2^32 inodes.

They also update libufs, tunefs, and growfs so that they properly
handle inode numbers as unsigned.

Reported by: Scott Burns, John Kilburg, and Bruce Evans
Followup by: Jeff Roberson
PR:          133980
---
 lib/libufs/cgroup.c | 2 +-
 lib/libufs/sblock.c | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/libufs/cgroup.c b/lib/libufs/cgroup.c
index 2185682..28e5ea8 100644
--- a/lib/libufs/cgroup.c
+++ b/lib/libufs/cgroup.c
@@ -59,7 +59,7 @@ cgread1(struct uufsd *disk, int c)
 
 	fs = &disk->d_fs;
 
-	if (c >= fs->fs_ncg) {
+	if ((unsigned)c >= fs->fs_ncg) {
 		return (0);
 	}
 	ccg = fsbtodb(fs, cgtod(fs, c)) * disk->d_bsize;
diff --git a/lib/libufs/sblock.c b/lib/libufs/sblock.c
index c9125f2..8986290 100644
--- a/lib/libufs/sblock.c
+++ b/lib/libufs/sblock.c
@@ -93,7 +93,7 @@ int
 sbwrite(struct uufsd *disk, int all)
 {
 	struct fs *fs;
-	int i;
+	unsigned i;
 
 	ERROR(disk, NULL);
 
-- 
cgit v1.1


From 05b666175c3574ab196b72aa01279fbccde0db29 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Tue, 2 Mar 2010 01:56:55 +0000
Subject: MFC x86emu/x86bios emulator and make previously i386 only dpms and
 vesa framebuffer driver, etc. work on FreeBSD/amd64.

A significant amount of improvements were done by jkim@ during the recent
months to make vesa(4) work better, over the initial code import.  This
work is based on OpenBSD's x86emu implementation and contributed by
paradox <ddkprog yahoo com> and swell.k at gmail com.

Hopefully I have stolen all their work to 8-STABLE :)

All bugs in this commit are mine, as usual.
---
 lib/Makefile | 1 +
 1 file changed, 1 insertion(+)

(limited to 'lib')

diff --git a/lib/Makefile b/lib/Makefile
index 7cacb70..00b1c60 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -112,6 +112,7 @@ _libsmb=	libsmb
 _libncp=	libncp
 .endif
 _libsmb=	libsmb
+_libvgl=	libvgl
 .endif
 
 .if ${MACHINE_ARCH} == "powerpc"
-- 
cgit v1.1


From 4d425cbc0262bb55ea0f0cf425f77b655b5cb044 Mon Sep 17 00:00:00 2001
From: brooks <brooks@FreeBSD.org>
Date: Wed, 3 Mar 2010 21:47:25 +0000
Subject: MFC r201959

Use the correct types to store uids and gids in the credential cache and
eliminate an inappropriate use of NGROUPS.
---
 lib/libc/rpc/svc_auth_des.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/rpc/svc_auth_des.c b/lib/libc/rpc/svc_auth_des.c
index 84f1e19..de4d1b4 100644
--- a/lib/libc/rpc/svc_auth_des.c
+++ b/lib/libc/rpc/svc_auth_des.c
@@ -449,10 +449,10 @@ cache_spot(key, name, timestamp)
 #define INVALID		-1 	/* grouplen, if cache entry is invalid */
 
 struct bsdcred {
-	short uid;		/* cached uid */
-	short gid;		/* cached gid */
-	short grouplen;	/* length of cached groups */
-	short groups[NGROUPS];	/* cached groups */
+	uid_t uid;		/* cached uid */
+	gid_t gid;		/* cached gid */
+	int grouplen;	/* length of cached groups */
+	gid_t groups[NGRPS];	/* cached groups */
 };
 
 /*
-- 
cgit v1.1


From 0a48f6a9f77f976420b17a5d1d10ff4e4ac409c8 Mon Sep 17 00:00:00 2001
From: kib <kib@FreeBSD.org>
Date: Thu, 4 Mar 2010 07:12:44 +0000
Subject: MFC r204307: Make pause(3) implementation not depended on the legacy
 sigcompat.c interfaces. Do not block rt signals during and after pause(3)
 calls. Use private libc namespace to call proper methods.

---
 lib/libc/gen/pause.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libc/gen/pause.c b/lib/libc/gen/pause.c
index 86d3643..cb0fa97 100644
--- a/lib/libc/gen/pause.c
+++ b/lib/libc/gen/pause.c
@@ -33,8 +33,10 @@ static char sccsid[] = "@(#)pause.c	8.1 (Berkeley) 6/4/93";
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include "namespace.h"
 #include <signal.h>
 #include <unistd.h>
+#include "un-namespace.h"
 
 /*
  * Backwards compatible pause.
@@ -42,7 +44,11 @@ __FBSDID("$FreeBSD$");
 int
 __pause()
 {
-	return sigpause(sigblock(0L));
+	sigset_t oset;
+
+	if (_sigprocmask(SIG_BLOCK, NULL, &oset) == -1)
+		return (-1);
+	return (_sigsuspend(&oset));
 }
 __weak_reference(__pause, pause);
 __weak_reference(__pause, _pause);
-- 
cgit v1.1


From e8bb8a0692a40f30fbf7cca11e49f09130b585c5 Mon Sep 17 00:00:00 2001
From: marcel <marcel@FreeBSD.org>
Date: Sun, 7 Mar 2010 00:07:00 +0000
Subject: MFC revs 203696, 203783: Add PT_VM_TIMESTAMP and PT_VM_ENTRY so that
 the tracing process can obtain the memory map of the traced process.

Requested by: kib@
---
 lib/libc/sys/ptrace.2 | 85 ++++++++++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 84 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libc/sys/ptrace.2 b/lib/libc/sys/ptrace.2
index 9d8f550..8265fb6 100644
--- a/lib/libc/sys/ptrace.2
+++ b/lib/libc/sys/ptrace.2
@@ -2,7 +2,7 @@
 .\"	$NetBSD: ptrace.2,v 1.2 1995/02/27 12:35:37 cgd Exp $
 .\"
 .\" This file is in the public domain.
-.Dd March 27, 2009
+.Dd February 11, 2010
 .Dt PTRACE 2
 .Os
 .Sh NAME
@@ -327,6 +327,68 @@ This request will trace the specified process on each system call exit.
 .It PT_SYSCALL
 This request will trace the specified process
 on each system call entry and exit.
+.It PT_VM_TIMESTAMP
+This request returns the generation number or timestamp of the memory map of
+the traced process as the return value from
+.Fn ptrace .
+This provides a low-cost way for the tracing process to determine if the
+VM map changed since the last time this request was made.
+.It PT_VM_ENTRY
+This request is used to iterate over the entries of the VM map of the traced
+process.
+The
+.Fa addr
+argument specifies a pointer to a 
+.Vt "struct ptrace_vm_entry" ,
+which is defined as follows:
+.Bd -literal
+struct ptrace_vm_entry {
+	int		pve_entry;
+	int		pve_timestamp;
+	u_long		pve_start;
+	u_long		pve_end;
+	u_long		pve_offset;
+	u_int		pve_prot;
+	u_int		pve_pathlen;
+	long		pve_fileid;
+	uint32_t	pve_fsid;
+	char		*pve_path;
+};
+.Ed
+.Pp
+The first entry is returned by setting
+.Va pve_entry
+to zero.
+Subsequent entries are returned by leaving
+.Va pve_entry
+unmodified from the value returned by previous requests.
+The
+.Va pve_timestamp
+field can be used to detect changes to the VM map while iterating over the
+entries.
+The tracing process can then take appropriate action, such as restarting.
+By setting
+.Va pve_pathlen
+to a non-zero value on entry, the pathname of the backing object is returned
+in the buffer pointed to by
+.Va pve_path ,
+provided the entry is backed by a vnode.
+The
+.Va pve_pathlen
+field is updated with the actual length of the pathname (including the
+terminating null character).
+The
+.Va pve_offset
+field is the offset within the backing object at which the range starts.
+The range is located in the VM space at
+.Va pve_start
+and extends up to
+.Va pve_end
+(inclusive).
+.Pp
+The
+.Fa data
+argument is ignored.
 .El
 .Pp
 Additionally, machine-specific requests can exist.
@@ -376,6 +438,11 @@ or
 .Dv PT_SETDBREGS
 was attempted on a process with no valid register set.
 (This is normally true only of system processes.)
+.It
+.Dv PT_VM_ENTRY
+was given an invalid value for
+.Fa pve_entry .
+This can also be caused by changes to the VM map of the process.
 .El
 .It Bq Er EBUSY
 .Bl -bullet -compact
@@ -405,6 +472,22 @@ on a process in violation of the requirements listed under
 .Dv PT_ATTACH
 above.
 .El
+.It Bq Er ENOENT
+.Bl -bullet -compact
+.It
+.Dv PT_VM_ENTRY
+previously returned the last entry of the memory map.
+No more entries exist.
+.El
+.It Bq Er ENAMETOOLONG
+.Bl -bullet -compact
+.It
+.Dv PT_VM_ENTRY
+cannot return the pathname of the backing object because the buffer is not big
+enough.
+.Fa pve_pathlen
+holds the minimum buffer size required on return.
+.El
 .El
 .Sh SEE ALSO
 .Xr execve 2 ,
-- 
cgit v1.1


From dc61101b9e948e19b6506d55ef3c0f7eaa40a0be Mon Sep 17 00:00:00 2001
From: gshapiro <gshapiro@FreeBSD.org>
Date: Sun, 7 Mar 2010 02:02:07 +0000
Subject: MFC: Enable the use of nanosleep() instead of using pause() and
 signals.      This Makefile change can be removed when the next version of
 sendmail      is imported as it will have this built in to the FreeBSD conf.h
 section.

---
 lib/libsm/Makefile | 1 +
 1 file changed, 1 insertion(+)

(limited to 'lib')

diff --git a/lib/libsm/Makefile b/lib/libsm/Makefile
index 38c7806..0fdcaa1 100644
--- a/lib/libsm/Makefile
+++ b/lib/libsm/Makefile
@@ -7,6 +7,7 @@ SENDMAIL_DIR=${.CURDIR}/../../contrib/sendmail
 
 CFLAGS+=-I${SENDMAIL_DIR}/src -I${SENDMAIL_DIR}/include -I.
 CFLAGS+=-DNEWDB -DNIS -DMAP_REGEX -DNOT_SENDMAIL
+CFLAGS+=-DHAVE_NANOSLEEP
 
 .if ${MK_INET6_SUPPORT} != "no"
 CFLAGS+=-DNETINET6
-- 
cgit v1.1


From c4a13ea37fbc64339be45f9da5746f4c1faae737 Mon Sep 17 00:00:00 2001
From: jh <jh@FreeBSD.org>
Date: Fri, 12 Mar 2010 06:56:51 +0000
Subject: MFC r204447:

In _gettemp(), check that the length of the path doesn't exceed
MAXPATHLEN. Otherwise the path name (or part of it) may not fit to
carrybuf causing a buffer overflow.

PR:		bin/140228
---
 lib/libc/stdio/mktemp.c | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'lib')

diff --git a/lib/libc/stdio/mktemp.c b/lib/libc/stdio/mktemp.c
index 3f1e699..a30b930 100644
--- a/lib/libc/stdio/mktemp.c
+++ b/lib/libc/stdio/mktemp.c
@@ -116,6 +116,10 @@ _gettemp(path, doopen, domkdir, slen)
 
 	for (trv = path; *trv != '\0'; ++trv)
 		;
+	if (trv - path >= MAXPATHLEN) {
+		errno = ENAMETOOLONG;
+		return (0);
+	}
 	trv -= slen;
 	suffp = trv;
 	--trv;
-- 
cgit v1.1


From 883fd46760684a12a08dee7a5e8c39ff4b1555b8 Mon Sep 17 00:00:00 2001
From: marius <marius@FreeBSD.org>
Date: Mon, 15 Mar 2010 18:32:57 +0000
Subject: MFC: r204974, r205002

- The OPSZ macro actually only does the right thing for int32 and int64
  operands but not for double and extended double ones. Instead of trying
  to fix the macro just nuke it and unroll the loops in the correct way
  though as extended double operands turn out to be the only special case.
- For FxTO{s,d,q} the source operand is int64 so rs2 has to be re-decoded
  after setting type accordingly as it's generally decoded using the low
  2 bits as the type, which are 0 for these three instructions.
- Similarly, in case of F{s,d,q}TOx the target is int64 so rd has to be
  re-decoded using not only the operand mask appropriate for int64 but
  also the correct register number encoding.
- Use const where appropriate.
- Wrap long lines.

Submitted by:	Peter Jeremy (partly)
---
 lib/libc/sparc64/fpu/fpu.c     | 48 ++++++++++++++++++------------------------
 lib/libc/sparc64/fpu/fpu_emu.h |  2 +-
 2 files changed, 21 insertions(+), 29 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/sparc64/fpu/fpu.c b/lib/libc/sparc64/fpu/fpu.c
index 74f07ec..7c8a3a9 100644
--- a/lib/libc/sparc64/fpu/fpu.c
+++ b/lib/libc/sparc64/fpu/fpu.c
@@ -97,7 +97,7 @@ __FBSDID("$FreeBSD$");
 #define	X8(x) X4(x),X4(x)
 #define	X16(x) X8(x),X8(x)
 
-static char cx_to_trapx[] = {
+static const char cx_to_trapx[] = {
 	X1(FSR_NX),
 	X2(FSR_DZ),
 	X4(FSR_UF),
@@ -113,7 +113,8 @@ int __fpe_debug = 0;
 #endif
 #endif	/* FPU_DEBUG */
 
-static int __fpu_execute(struct utrapframe *, struct fpemu *, u_int32_t, u_long);
+static int __fpu_execute(struct utrapframe *, struct fpemu *, u_int32_t,
+    u_long);
 
 /*
  * Need to use an fpstate on the stack; we could switch, so we cannot safely
@@ -169,7 +170,7 @@ __fpu_exception(struct utrapframe *uf)
 void
 __fpu_dumpfpn(struct fpn *fp)
 {
-	static char *class[] = {
+	static const char *const class[] = {
 		"SNAN", "QNAN", "ZERO", "NUM", "INF"
 	};
 
@@ -181,15 +182,11 @@ __fpu_dumpfpn(struct fpn *fp)
 }
 #endif
 
-static int opmask[] = {0, 0, 1, 3};
+static const int opmask[] = {0, 0, 1, 3, 1};
 
 /* Decode 5 bit register field depending on the type. */
 #define	RN_DECODE(tp, rn) \
-	((tp == FTYPE_DBL || tp == FTYPE_EXT ? INSFPdq_RN((rn)) : (rn)) & \
-	    ~opmask[tp])
-
-/* Operand size in 32-bit registers. */
-#define	OPSZ(tp)	((tp) == FTYPE_LNG ? 2 : (1 << (tp)))
+	((tp) >= FTYPE_DBL ? INSFPdq_RN(rn) & ~opmask[tp] : (rn))
 
 /*
  * Helper for forming the below case statements. Build only the op3 and opf
@@ -209,8 +206,6 @@ static void
 __fpu_mov(struct fpemu *fe, int type, int rd, int rs2, u_int32_t nand,
     u_int32_t xor)
 {
-	u_int64_t tmp64;
-	int i;
 
 	if (type == FTYPE_INT || type == FTYPE_SNG)
 		__fpu_setreg(rd, (__fpu_getreg(rs2) & ~nand) ^ xor);
@@ -219,13 +214,10 @@ __fpu_mov(struct fpemu *fe, int type, int rd, int rs2, u_int32_t nand,
 		 * Need to use the double versions to be able to access
 		 * the upper 32 fp registers.
 		 */
-		for (i = 0; i < OPSZ(type); i += 2, rd += 2, rs2 += 2) {
-			tmp64 = __fpu_getreg64(rs2);
-			if (i == 0)
-				tmp64 = (tmp64 & ~((u_int64_t)nand << 32)) ^
-				    ((u_int64_t)xor << 32);
-			__fpu_setreg64(rd, tmp64);
-		}
+		__fpu_setreg64(rd, (__fpu_getreg64(rs2) &
+		    ~((u_int64_t)nand << 32)) ^ ((u_int64_t)xor << 32));
+		if (type == FTYPE_EXT)
+			__fpu_setreg64(rd + 2, __fpu_getreg64(rs2 + 2));
 	}
 }
 
@@ -271,17 +263,17 @@ __fpu_cmpck(struct fpemu *fe)
  * multiply two integers this way.
  */
 static int
-__fpu_execute(struct utrapframe *uf, struct fpemu *fe, u_int32_t insn, u_long tstate)
+__fpu_execute(struct utrapframe *uf, struct fpemu *fe, u_int32_t insn,
+    u_long tstate)
 {
 	struct fpn *fp;
 	int opf, rs1, rs2, rd, type, mask, cx, cond;
 	u_long reg, fsr;
 	u_int space[4];
-	int i;
 
 	/*
 	 * `Decode' and execute instruction.  Start with no exceptions.
-	 * The type of any opf opcode is in the bottom two bits, so we
+	 * The type of almost any OPF opcode is in the bottom two bits, so we
 	 * squish them out here.
 	 */
 	opf = insn & (IF_MASK(IF_F3_OP3_SHIFT, IF_F3_OP3_BITS) |
@@ -359,7 +351,7 @@ __fpu_execute(struct utrapframe *uf, struct fpemu *fe, u_int32_t insn, u_long ts
 		__fpu_explode(fe, &fe->fe_f2, type, rs2);
 		__fpu_compare(fe, 1, IF_F3_CC(insn));
 		return (__fpu_cmpck(fe));
-	case FOP(INS2_FPop1, INSFP1_FMOV):	/* these should all be pretty obvious */
+	case FOP(INS2_FPop1, INSFP1_FMOV):
 		__fpu_mov(fe, type, rd, rs2, 0, 0);
 		return (0);
 	case FOP(INS2_FPop1, INSFP1_FNEG):
@@ -410,6 +402,7 @@ __fpu_execute(struct utrapframe *uf, struct fpemu *fe, u_int32_t insn, u_long ts
 	case FOP(INS2_FPop1, INSFP1_FxTOd):
 	case FOP(INS2_FPop1, INSFP1_FxTOq):
 		type = FTYPE_LNG;
+		rs2 = RN_DECODE(type, IF_F3_RS2(insn));
 		__fpu_explode(fe, fp = &fe->fe_f1, type, rs2);
 		/* sneaky; depends on instruction encoding */
 		type = (IF_F3_OPF(insn) >> 2) & 3;
@@ -418,8 +411,7 @@ __fpu_execute(struct utrapframe *uf, struct fpemu *fe, u_int32_t insn, u_long ts
 	case FOP(INS2_FPop1, INSFP1_FTOx):
 		__fpu_explode(fe, fp = &fe->fe_f1, type, rs2);
 		type = FTYPE_LNG;
-		mask = 1;	/* needs 2 registers */
-		rd = IF_F3_RD(insn) & ~mask;
+		rd = RN_DECODE(type, IF_F3_RD(insn));
 		break;
 	case FOP(INS2_FPop1, INSFP1_FTOs):
 	case FOP(INS2_FPop1, INSFP1_FTOd):
@@ -457,10 +449,10 @@ __fpu_execute(struct utrapframe *uf, struct fpemu *fe, u_int32_t insn, u_long ts
 	if (type == FTYPE_INT || type == FTYPE_SNG)
 		__fpu_setreg(rd, space[0]);
 	else {
-		for (i = 0; i < OPSZ(type); i += 2) {
-			__fpu_setreg64(rd + i, ((u_int64_t)space[i] << 32) |
-			    space[i + 1]);
-		}
+		__fpu_setreg64(rd, ((u_int64_t)space[0] << 32) | space[1]);
+		if (type == FTYPE_EXT)
+			__fpu_setreg64(rd + 2,
+			    ((u_int64_t)space[2] << 32) | space[3]);
 	}
 	return (0);	/* success */
 }
diff --git a/lib/libc/sparc64/fpu/fpu_emu.h b/lib/libc/sparc64/fpu/fpu_emu.h
index de156e2..0d1d16d 100644
--- a/lib/libc/sparc64/fpu/fpu_emu.h
+++ b/lib/libc/sparc64/fpu/fpu_emu.h
@@ -140,7 +140,7 @@ struct fpn {
 #define	FTYPE_SNG	INSFP_s
 #define	FTYPE_DBL	INSFP_d
 #define	FTYPE_EXT	INSFP_q
-#define	FTYPE_LNG	-1
+#define	FTYPE_LNG	4
 
 /*
  * Emulator state.
-- 
cgit v1.1


From 9d42867c19941d56932e8cea0e02a9538e58cd55 Mon Sep 17 00:00:00 2001
From: jhb <jhb@FreeBSD.org>
Date: Mon, 22 Mar 2010 15:52:26 +0000
Subject: MFC 204950,205020: Use thr_once() with once_t controls to initialize
 various thread_key_t objects used to provide per-thread storage in the RPC
 code.  Almost all of these used double-checking with a dedicated mutex
 (tsd_lock) to do this before.  However, that is not always safe with more
 relaxed memory orders. There were also other bugs, such as one in
 __rpc_createrr() that caused a new key to be allocated each time
 __rpc_createrr() was invoked.

---
 lib/libc/rpc/Symbol.map     |  4 ----
 lib/libc/rpc/clnt_simple.c  | 22 ++++++++++++++++------
 lib/libc/rpc/getnetconfig.c | 29 +++++++++++++++--------------
 lib/libc/rpc/key_call.c     | 20 +++++++++++++-------
 lib/libc/rpc/mt_misc.c      | 27 +++++++++++++--------------
 lib/libc/rpc/mt_misc.h      |  1 -
 lib/libc/rpc/rpc_generic.c  | 28 +++++++++++++++-------------
 lib/libc/rpc/rpc_soc.c      | 15 +++++++++------
 8 files changed, 81 insertions(+), 65 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/rpc/Symbol.map b/lib/libc/rpc/Symbol.map
index ccf0bfa..4f356de 100644
--- a/lib/libc/rpc/Symbol.map
+++ b/lib/libc/rpc/Symbol.map
@@ -239,10 +239,6 @@ FBSDprivate_1.0 {
 	__key_encryptsession_pk_LOCAL;
 	__key_decryptsession_pk_LOCAL;
 	__key_gendes_LOCAL;
-	__tsd_lock;	/*
-			 * Why does usr.bin/rpcinfo/Makefile need rpc_generic.c?
-			 * Remove this hack if rpcinfo stops building with it.
-			 */
 	__svc_clean_idle;
 	__rpc_gss_unwrap;
 	__rpc_gss_unwrap_stub;
diff --git a/lib/libc/rpc/clnt_simple.c b/lib/libc/rpc/clnt_simple.c
index 12b6679..ba21d2d 100644
--- a/lib/libc/rpc/clnt_simple.c
+++ b/lib/libc/rpc/clnt_simple.c
@@ -76,7 +76,11 @@ struct rpc_call_private {
 	char	nettype[NETIDLEN];	/* Network type */
 };
 static struct rpc_call_private *rpc_call_private_main;
+static thread_key_t rpc_call_key;
+static once_t rpc_call_once = ONCE_INITIALIZER;
+static int rpc_call_key_error;
 
+static void rpc_call_key_init(void);
 static void rpc_call_destroy(void *);
 
 static void
@@ -91,6 +95,13 @@ rpc_call_destroy(void *vp)
 	}
 }
 
+static void
+rpc_call_key_init(void)
+{
+
+	rpc_call_key_error = thr_keycreate(&rpc_call_key, rpc_call_destroy);
+}
+
 /*
  * This is the simplified interface to the client rpc layer.
  * The client handle is not destroyed here and is reused for
@@ -112,17 +123,16 @@ rpc_call(host, prognum, versnum, procnum, inproc, in, outproc, out, nettype)
 	struct rpc_call_private *rcp = (struct rpc_call_private *) 0;
 	enum clnt_stat clnt_stat;
 	struct timeval timeout, tottimeout;
-	static thread_key_t rpc_call_key;
 	int main_thread = 1;
 
 	if ((main_thread = thr_main())) {
 		rcp = rpc_call_private_main;
 	} else {
-		if (rpc_call_key == 0) {
-			mutex_lock(&tsd_lock);
-			if (rpc_call_key == 0)
-				thr_keycreate(&rpc_call_key, rpc_call_destroy);
-			mutex_unlock(&tsd_lock);
+		if (thr_once(&rpc_call_once, rpc_call_key_init) != 0 ||
+		    rpc_call_key_error != 0) {
+			rpc_createerr.cf_stat = RPC_SYSTEMERROR;
+			rpc_createerr.cf_error.re_errno = rpc_call_key_error;
+			return (rpc_createerr.cf_stat);
 		}
 		rcp = (struct rpc_call_private *)thr_getspecific(rpc_call_key);
 	}
diff --git a/lib/libc/rpc/getnetconfig.c b/lib/libc/rpc/getnetconfig.c
index e5db51a..1310e36 100644
--- a/lib/libc/rpc/getnetconfig.c
+++ b/lib/libc/rpc/getnetconfig.c
@@ -130,21 +130,29 @@ static struct netconfig *dup_ncp(struct netconfig *);
 
 
 static FILE *nc_file;		/* for netconfig db */
-static pthread_mutex_t nc_file_lock = PTHREAD_MUTEX_INITIALIZER;
+static mutex_t nc_file_lock = MUTEX_INITIALIZER;
 
 static struct netconfig_info	ni = { 0, 0, NULL, NULL};
-static pthread_mutex_t ni_lock = PTHREAD_MUTEX_INITIALIZER;
+static mutex_t ni_lock = MUTEX_INITIALIZER;
 
+static thread_key_t nc_key;
+static once_t nc_once = ONCE_INITIALIZER;
+static int nc_key_error;
+
+static void
+nc_key_init(void)
+{
+
+	nc_key_error = thr_keycreate(&nc_key, free);
+}
 
 #define MAXNETCONFIGLINE    1000
 
 static int *
 __nc_error()
 {
-	static pthread_mutex_t nc_lock = PTHREAD_MUTEX_INITIALIZER;
-	static thread_key_t nc_key = 0;
 	static int nc_error = 0;
-	int error, *nc_addr;
+	int *nc_addr;
 
 	/*
 	 * Use the static `nc_error' if we are the main thread
@@ -153,15 +161,8 @@ __nc_error()
 	 */
 	if (thr_main())
 		return (&nc_error);
-	if (nc_key == 0) {
-		error = 0;
-		mutex_lock(&nc_lock);
-		if (nc_key == 0)
-			error = thr_keycreate(&nc_key, free);
-		mutex_unlock(&nc_lock);
-		if (error)
-			return (&nc_error);
-	}
+	if (thr_once(&nc_once, nc_key_init) != 0 || nc_key_error != 0)
+		return (&nc_error);
 	if ((nc_addr = (int *)thr_getspecific(nc_key)) == NULL) {
 		nc_addr = (int *)malloc(sizeof (int));
 		if (thr_setspecific(nc_key, (void *) nc_addr) != 0) {
diff --git a/lib/libc/rpc/key_call.c b/lib/libc/rpc/key_call.c
index 6cc1139..afb11c8 100644
--- a/lib/libc/rpc/key_call.c
+++ b/lib/libc/rpc/key_call.c
@@ -279,6 +279,9 @@ struct  key_call_private {
 	uid_t	uid;		/* user-id at last authorization */
 };
 static struct key_call_private *key_call_private_main = NULL;
+static thread_key_t key_call_key;
+static once_t key_call_once = ONCE_INITIALIZER;
+static int key_call_key_error;
 
 static void
 key_call_destroy(void *vp)
@@ -292,6 +295,13 @@ key_call_destroy(void *vp)
 	}
 }
 
+static void
+key_call_init(void)
+{
+
+	key_call_key_error = thr_keycreate(&key_call_key, key_call_destroy);
+}
+
 /*
  * Keep the handle cached.  This call may be made quite often.
  */
@@ -307,7 +317,6 @@ int	vers;
 	struct utsname u;
 	int main_thread;
 	int fd;
-	static thread_key_t key_call_key;
 
 #define	TOTAL_TIMEOUT	30	/* total timeout talking to keyserver */
 #define	TOTAL_TRIES	5	/* Number of tries */
@@ -315,12 +324,9 @@ int	vers;
 	if ((main_thread = thr_main())) {
 		kcp = key_call_private_main;
 	} else {
-		if (key_call_key == 0) {
-			mutex_lock(&tsd_lock);
-			if (key_call_key == 0)
-				thr_keycreate(&key_call_key, key_call_destroy);
-			mutex_unlock(&tsd_lock);
-		}
+		if (thr_once(&key_call_once, key_call_init) != 0 ||
+		    key_call_key_error != 0)
+			return ((CLIENT *) NULL);
 		kcp = (struct key_call_private *)thr_getspecific(key_call_key);
 	}	
 	if (kcp == (struct key_call_private *)NULL) {
diff --git a/lib/libc/rpc/mt_misc.c b/lib/libc/rpc/mt_misc.c
index 6241611..7abcaed 100644
--- a/lib/libc/rpc/mt_misc.c
+++ b/lib/libc/rpc/mt_misc.c
@@ -28,7 +28,6 @@ __FBSDID("$FreeBSD$");
 #define	proglst_lock		__proglst_lock
 #define	rpcsoc_lock		__rpcsoc_lock
 #define	svcraw_lock		__svcraw_lock
-#define	tsd_lock		__tsd_lock
 #define	xprtlist_lock		__xprtlist_lock
 
 /* protects the services list (svc.c) */
@@ -76,33 +75,33 @@ pthread_mutex_t	rpcsoc_lock = PTHREAD_MUTEX_INITIALIZER;
 /* svc_raw.c serialization */
 pthread_mutex_t	svcraw_lock = PTHREAD_MUTEX_INITIALIZER;
 
-/* protects TSD key creation */
-pthread_mutex_t	tsd_lock = PTHREAD_MUTEX_INITIALIZER;
-
 /* xprtlist (svc_generic.c) */
 pthread_mutex_t	xprtlist_lock = PTHREAD_MUTEX_INITIALIZER;
 
 #undef	rpc_createerr
 
 struct rpc_createerr rpc_createerr;
+static thread_key_t rce_key;
+static once_t rce_once = ONCE_INITIALIZER;
+static int rce_key_error;
+
+static void
+rce_key_init(void)
+{
+
+	rce_key_error = thr_keycreate(&rce_key, free);
+}
 
 struct rpc_createerr *
 __rpc_createerr()
 {
-	static thread_key_t rce_key = 0;
 	struct rpc_createerr *rce_addr = 0;
 
 	if (thr_main())
 		return (&rpc_createerr);
-	if ((rce_addr =
-	    (struct rpc_createerr *)thr_getspecific(rce_key)) != 0) {
-		mutex_lock(&tsd_lock);
-		if (thr_keycreate(&rce_key, free) != 0) {
-			mutex_unlock(&tsd_lock);
-			return (&rpc_createerr);
-		}
-		mutex_unlock(&tsd_lock);
-	}
+	if (thr_once(&rce_once, rce_key_init) != 0 || rce_key_error != 0)
+		return (&rpc_createerr);
+	rce_addr = (struct rpc_createerr *)thr_getspecific(rce_key);
 	if (!rce_addr) {
 		rce_addr = (struct rpc_createerr *)
 			malloc(sizeof (struct rpc_createerr));
diff --git a/lib/libc/rpc/mt_misc.h b/lib/libc/rpc/mt_misc.h
index e785c30..9cc2349 100644
--- a/lib/libc/rpc/mt_misc.h
+++ b/lib/libc/rpc/mt_misc.h
@@ -42,7 +42,6 @@
 #define	proglst_lock		__proglst_lock
 #define	rpcsoc_lock		__rpcsoc_lock
 #define	svcraw_lock		__svcraw_lock
-#define	tsd_lock		__tsd_lock
 #define	xprtlist_lock		__xprtlist_lock
 
 extern pthread_rwlock_t	svc_lock;
diff --git a/lib/libc/rpc/rpc_generic.c b/lib/libc/rpc/rpc_generic.c
index 81bd92b..ab259d5 100644
--- a/lib/libc/rpc/rpc_generic.c
+++ b/lib/libc/rpc/rpc_generic.c
@@ -221,6 +221,18 @@ getnettype(nettype)
 	return (_rpctypelist[i].type);
 }
 
+static thread_key_t tcp_key, udp_key;
+static once_t keys_once = ONCE_INITIALIZER;
+static int tcp_key_error, udp_key_error;
+
+static void
+keys_init(void)
+{
+
+	tcp_key_error = thr_keycreate(&tcp_key, free);
+	udp_key_error = thr_keycreate(&udp_key, free);
+}
+
 /*
  * For the given nettype (tcp or udp only), return the first structure found.
  * This should be freed by calling freenetconfigent()
@@ -236,25 +248,15 @@ __rpc_getconfip(nettype)
 	static char *netid_udp_main;
 	struct netconfig *dummy;
 	int main_thread;
-	static thread_key_t tcp_key, udp_key;
 
 	if ((main_thread = thr_main())) {
 		netid_udp = netid_udp_main;
 		netid_tcp = netid_tcp_main;
 	} else {
-		if (tcp_key == 0) {
-			mutex_lock(&tsd_lock);
-			if (tcp_key == 0)
-				thr_keycreate(&tcp_key, free);
-			mutex_unlock(&tsd_lock);
-		}
+		if (thr_once(&keys_once, keys_init) != 0 ||
+		    tcp_key_error != 0 || udp_key_error != 0)
+			return (NULL);
 		netid_tcp = (char *)thr_getspecific(tcp_key);
-		if (udp_key == 0) {
-			mutex_lock(&tsd_lock);
-			if (udp_key == 0)
-				thr_keycreate(&udp_key, free);
-			mutex_unlock(&tsd_lock);
-		}
 		netid_udp = (char *)thr_getspecific(udp_key);
 	}
 	if (!netid_udp && !netid_tcp) {
diff --git a/lib/libc/rpc/rpc_soc.c b/lib/libc/rpc/rpc_soc.c
index 5922063..2568d43 100644
--- a/lib/libc/rpc/rpc_soc.c
+++ b/lib/libc/rpc/rpc_soc.c
@@ -360,6 +360,14 @@ registerrpc(prognum, versnum, procnum, progname, inproc, outproc)
  */
 static thread_key_t	clnt_broadcast_key;
 static resultproc_t	clnt_broadcast_result_main;
+static once_t		clnt_broadcast_once = ONCE_INITIALIZER;
+
+static void
+clnt_broadcast_key_init(void)
+{
+
+	thr_keycreate(&clnt_broadcast_key, free);
+}
 
 /*
  * Need to translate the netbuf address into sockaddr_in address.
@@ -402,12 +410,7 @@ clnt_broadcast(prog, vers, proc, xargs, argsp, xresults, resultsp, eachresult)
 	if (thr_main())
 		clnt_broadcast_result_main = eachresult;
 	else {
-		if (clnt_broadcast_key == 0) {
-			mutex_lock(&tsd_lock);
-			if (clnt_broadcast_key == 0)
-				thr_keycreate(&clnt_broadcast_key, free);
-			mutex_unlock(&tsd_lock);
-		}
+		thr_once(&clnt_broadcast_once, clnt_broadcast_key_init);
 		thr_setspecific(clnt_broadcast_key, (void *) eachresult);
 	}
 	return rpc_broadcast((rpcprog_t)prog, (rpcvers_t)vers,
-- 
cgit v1.1


From dde080c859dfbc0b12fd706108b5debe53529c9f Mon Sep 17 00:00:00 2001
From: jilles <jilles@FreeBSD.org>
Date: Tue, 23 Mar 2010 23:25:17 +0000
Subject: MFC r205398: Do not create *.gmon files for PIE executables on i386.

Scrt1_c.o was accidentally compiled with -DGCRT (profiling), like gcrt1_c.o.
This problem is i386-specific, the other architectures are OK.

If you have problems with PIE executables such as samba and cups leaving
behind gmon files, rebuild them after installing this change.

PR:		ports/143924
---
 lib/csu/i386-elf/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/csu/i386-elf/Makefile b/lib/csu/i386-elf/Makefile
index c2af118..45c162f 100644
--- a/lib/csu/i386-elf/Makefile
+++ b/lib/csu/i386-elf/Makefile
@@ -24,7 +24,7 @@ crt1.o:	crt1_c.o crt1_s.o
 	objcopy --localize-symbol _start1 crt1.o
 
 Scrt1_c.o:	crt1_c.c
-	${CC} ${CFLAGS} -DGCRT -fPIC -DPIC -c -o Scrt1_c.o ${.CURDIR}/crt1_c.c
+	${CC} ${CFLAGS} -fPIC -DPIC -c -o Scrt1_c.o ${.CURDIR}/crt1_c.c
 
 Scrt1.o: Scrt1_c.o crt1_s.o
 	${LD} ${LDFLAGS} -o Scrt1.o -r crt1_s.o Scrt1_c.o
-- 
cgit v1.1


From db13dfe8cdd4ec67909d7149a2214030034b5349 Mon Sep 17 00:00:00 2001
From: dougb <dougb@FreeBSD.org>
Date: Mon, 29 Mar 2010 06:31:58 +0000
Subject: Update to 9.6.2-P1, the latest patchfix release which deals with the
 problems related to the handling of broken DNSSEC trust chains.

This fix is only relevant for those who have DNSSEC validation
enabled and configure trust anchors from third parties, either
manually, or through a system like DLV.
---
 lib/bind/config.h | 6 ++++++
 1 file changed, 6 insertions(+)

(limited to 'lib')

diff --git a/lib/bind/config.h b/lib/bind/config.h
index 1d68450..3038b5b 100644
--- a/lib/bind/config.h
+++ b/lib/bind/config.h
@@ -166,6 +166,12 @@ int sigwait(const unsigned int *set, int *sig);
 /* Define to 1 if you have the <dlfcn.h> header file. */
 #define HAVE_DLFCN_H 1
 
+/* Define to 1 if you have the `EVP_sha256' function. */
+#define HAVE_EVP_SHA256 1
+
+/* Define to 1 if you have the `EVP_sha512' function. */
+#define HAVE_EVP_SHA512 1
+
 /* Define to 1 if you have the <fcntl.h> header file. */
 #define HAVE_FCNTL_H 1
 
-- 
cgit v1.1


From eff331f048ccbce2e947b38fb71422c6faf0bc06 Mon Sep 17 00:00:00 2001
From: marius <marius@FreeBSD.org>
Date: Tue, 30 Mar 2010 18:58:07 +0000
Subject: MFC: r205393

- Remove a bogus forward declaration.
- Fix whitespace.
---
 lib/libc/sparc64/fpu/fpu_extern.h  | 7 +++----
 lib/libc/sparc64/fpu/fpu_implode.c | 3 ---
 2 files changed, 3 insertions(+), 7 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/sparc64/fpu/fpu_extern.h b/lib/libc/sparc64/fpu/fpu_extern.h
index f22c463..0de30fc 100644
--- a/lib/libc/sparc64/fpu/fpu_extern.h
+++ b/lib/libc/sparc64/fpu/fpu_extern.h
@@ -41,7 +41,6 @@
 #define _SPARC64_FPU_FPU_EXTERN_H_
 
 struct utrapframe;
-union instr;
 struct fpemu;
 struct fpn;
 
@@ -61,9 +60,9 @@ struct fpn *__fpu_div(struct fpemu *);
 int __fpu_itof(struct fpn *, u_int);
 int __fpu_xtof(struct fpn *, u_int64_t);
 int __fpu_stof(struct fpn *, u_int);
-int __fpu_dtof(struct fpn *, u_int, u_int );
-int __fpu_qtof(struct fpn *, u_int, u_int , u_int , u_int );
-void __fpu_explode(struct fpemu *, struct fpn *, int, int );
+int __fpu_dtof(struct fpn *, u_int, u_int);
+int __fpu_qtof(struct fpn *, u_int, u_int, u_int, u_int);
+void __fpu_explode(struct fpemu *, struct fpn *, int, int);
 
 /* fpu_implode.c */
 u_int __fpu_ftoi(struct fpemu *, struct fpn *);
diff --git a/lib/libc/sparc64/fpu/fpu_implode.c b/lib/libc/sparc64/fpu/fpu_implode.c
index 5287d2b..32a0e7c 100644
--- a/lib/libc/sparc64/fpu/fpu_implode.c
+++ b/lib/libc/sparc64/fpu/fpu_implode.c
@@ -198,7 +198,6 @@ __fpu_ftoi(fe, fp)
 
 	sign = fp->fp_sign;
 	switch (fp->fp_class) {
-
 	case FPC_ZERO:
 		return (0);
 
@@ -248,7 +247,6 @@ __fpu_ftox(fe, fp, res)
 
 	sign = fp->fp_sign;
 	switch (fp->fp_class) {
-
 	case FPC_ZERO:
 		res[1] = 0;
 		return (0);
@@ -504,7 +502,6 @@ __fpu_implode(fe, fp, type, space)
 {
 
 	switch (type) {
-
 	case FTYPE_LNG:
 		space[0] = __fpu_ftox(fe, fp, space);
 		break;
-- 
cgit v1.1


From ca721ce8830393b7024be0906c4c518d5090e2dc Mon Sep 17 00:00:00 2001
From: marius <marius@FreeBSD.org>
Date: Tue, 30 Mar 2010 19:03:26 +0000
Subject: MFC: r205394

Ensure that __fpu_ftox() both returns the high bits and res[1] contains
the low bits also in the default case.

PR:		144900
Obtained from:	OpenBSD
---
 lib/libc/sparc64/fpu/fpu_implode.c | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/sparc64/fpu/fpu_implode.c b/lib/libc/sparc64/fpu/fpu_implode.c
index 32a0e7c..91e40c1 100644
--- a/lib/libc/sparc64/fpu/fpu_implode.c
+++ b/lib/libc/sparc64/fpu/fpu_implode.c
@@ -248,8 +248,8 @@ __fpu_ftox(fe, fp, res)
 	sign = fp->fp_sign;
 	switch (fp->fp_class) {
 	case FPC_ZERO:
-		res[1] = 0;
-		return (0);
+		i = 0;
+		goto done;
 
 	case FPC_NUM:
 		/*
@@ -273,15 +273,17 @@ __fpu_ftox(fe, fp, res)
 			break;
 		if (sign)
 			i = -i;
-		res[1] = (int)i;
-		return (i >> 32);
+		goto done;
 
 	default:		/* Inf, qNaN, sNaN */
 		break;
 	}
 	/* overflow: replace any inexact exception with invalid */
 	fe->fe_cx = (fe->fe_cx & ~FSR_NX) | FSR_NV;
-	return (0x7fffffffffffffffLL + sign);
+	i = 0x7fffffffffffffffLL + sign;
+done:
+	res[1] = i & 0xffffffff;
+	return (i >> 32);
 }
 
 /*
-- 
cgit v1.1


From 09058e4e259d9d79ef54eff5d335d02ce2c18668 Mon Sep 17 00:00:00 2001
From: marius <marius@FreeBSD.org>
Date: Tue, 30 Mar 2010 19:05:08 +0000
Subject: MFC: r205395

FPU_DEBUG requires <stdio.h>.

PR:		144900
Submitted by:	Peter Jeremy
---
 lib/libc/sparc64/fpu/fpu.c         | 5 ++++-
 lib/libc/sparc64/fpu/fpu_explode.c | 4 ++++
 lib/libc/sparc64/fpu/fpu_implode.c | 4 ++++
 3 files changed, 12 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libc/sparc64/fpu/fpu.c b/lib/libc/sparc64/fpu/fpu.c
index 7c8a3a9..4e92788 100644
--- a/lib/libc/sparc64/fpu/fpu.c
+++ b/lib/libc/sparc64/fpu/fpu.c
@@ -69,9 +69,12 @@ __FBSDID("$FreeBSD$");
 
 #include "namespace.h"
 #include <errno.h>
-#include <unistd.h>
 #include <signal.h>
+#ifdef FPU_DEBUG
+#include <stdio.h>
+#endif
 #include <stdlib.h>
+#include <unistd.h>
 #include "un-namespace.h"
 #include "libc_private.h"
 
diff --git a/lib/libc/sparc64/fpu/fpu_explode.c b/lib/libc/sparc64/fpu/fpu_explode.c
index 09cfd5a..12c8802 100644
--- a/lib/libc/sparc64/fpu/fpu_explode.c
+++ b/lib/libc/sparc64/fpu/fpu_explode.c
@@ -49,6 +49,10 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 
+#ifdef FPU_DEBUG
+#include <stdio.h>
+#endif
+
 #include <machine/frame.h>
 #include <machine/fp.h>
 #include <machine/fsr.h>
diff --git a/lib/libc/sparc64/fpu/fpu_implode.c b/lib/libc/sparc64/fpu/fpu_implode.c
index 91e40c1..82a8460 100644
--- a/lib/libc/sparc64/fpu/fpu_implode.c
+++ b/lib/libc/sparc64/fpu/fpu_implode.c
@@ -49,6 +49,10 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 
+#ifdef FPU_DEBUG
+#include <stdio.h>
+#endif
+
 #include <machine/frame.h>
 #include <machine/fp.h>
 #include <machine/fsr.h>
-- 
cgit v1.1


From 5c0d723c8f041700174535e11644e42e7b2cd5eb Mon Sep 17 00:00:00 2001
From: marius <marius@FreeBSD.org>
Date: Tue, 30 Mar 2010 19:06:31 +0000
Subject: MFC: r205396

Division should take both arguments' signs into account when the
the dividend is infinity or zero and the divisor is not the same.

PR:		144900
Submitted by:	Peter Jeremy
---
 lib/libc/sparc64/fpu/fpu_div.c | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/sparc64/fpu/fpu_div.c b/lib/libc/sparc64/fpu/fpu_div.c
index c4b8ef6..4748daf 100644
--- a/lib/libc/sparc64/fpu/fpu_div.c
+++ b/lib/libc/sparc64/fpu/fpu_div.c
@@ -167,14 +167,16 @@ __fpu_div(fe)
 	 * return it.  Otherwise we have the following cases:
 	 *
 	 *	Inf / Inf = NaN, plus NV exception
-	 *	Inf / num = Inf [i.e., return x]
-	 *	Inf / 0   = Inf [i.e., return x]
-	 *	0 / Inf = 0 [i.e., return x]
-	 *	0 / num = 0 [i.e., return x]
+	 *	Inf / num = Inf [i.e., return x #]
+	 *	Inf / 0   = Inf [i.e., return x #]
+	 *	0 / Inf = 0 [i.e., return x #]
+	 *	0 / num = 0 [i.e., return x #]
 	 *	0 / 0   = NaN, plus NV exception
-	 *	num / Inf = 0
+	 *	num / Inf = 0 #
 	 *	num / num = num (do the divide)
-	 *	num / 0   = Inf, plus DZ exception
+	 *	num / 0   = Inf #, plus DZ exception
+	 *
+	 * # Sign of result is XOR of operand signs.
 	 */
 	if (ISNAN(x) || ISNAN(y)) {
 		ORDER(x, y);
@@ -183,10 +185,10 @@ __fpu_div(fe)
 	if (ISINF(x) || ISZERO(x)) {
 		if (x->fp_class == y->fp_class)
 			return (__fpu_newnan(fe));
+		x->fp_sign ^= y->fp_sign;
 		return (x);
 	}
 
-	/* all results at this point use XOR of operand signs */
 	x->fp_sign ^= y->fp_sign;
 	if (ISINF(y)) {
 		x->fp_class = FPC_ZERO;
-- 
cgit v1.1


From a9048188435e001c00b98ed0536b49501140bd77 Mon Sep 17 00:00:00 2001
From: marius <marius@FreeBSD.org>
Date: Tue, 30 Mar 2010 19:08:02 +0000
Subject: MFC: r205397

- While SPARC V9 allows tininess to be detected either before or after
  rounding (impl. dep. #55), the SPARC JPS1 responsible for SPARC64 and
  UltraSPARC processors defines that in all cases tinyness is detected
  before rounding, therefore rounding up to the smallest normalised
  number should set the underflow flag.
- If an infinite result is rounded down, the result should have an
  exponent 1 less than the value for infinity.

PR:		144900
Submitted by:	Peter Jeremy
---
 lib/libc/sparc64/fpu/fpu_implode.c | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/sparc64/fpu/fpu_implode.c b/lib/libc/sparc64/fpu/fpu_implode.c
index 82a8460..2a2a9d0 100644
--- a/lib/libc/sparc64/fpu/fpu_implode.c
+++ b/lib/libc/sparc64/fpu/fpu_implode.c
@@ -329,8 +329,9 @@ __fpu_ftos(fe, fp)
 	 * right to introduce leading zeroes.  Rounding then acts
 	 * differently for normals and subnormals: the largest subnormal
 	 * may round to the smallest normal (1.0 x 2^minexp), or may
-	 * remain subnormal.  In the latter case, signal an underflow
-	 * if the result was inexact or if underflow traps are enabled.
+	 * remain subnormal.  A number that is subnormal before rounding
+	 * will signal an underflow if the result is inexact or if underflow
+	 * traps are enabled.
 	 *
 	 * Rounding a normal, on the other hand, always produces another
 	 * normal (although either way the result might be too big for
@@ -345,8 +346,10 @@ __fpu_ftos(fe, fp)
 	if ((exp = fp->fp_exp + SNG_EXP_BIAS) <= 0) {	/* subnormal */
 		/* -NG for g,r; -SNG_FRACBITS-exp for fraction */
 		(void) __fpu_shr(fp, FP_NMANT - FP_NG - SNG_FRACBITS - exp);
-		if (fpround(fe, fp) && fp->fp_mant[3] == SNG_EXP(1))
+		if (fpround(fe, fp) && fp->fp_mant[3] == SNG_EXP(1)) {
+			fe->fe_cx |= FSR_UF;
 			return (sign | SNG_EXP(1) | 0);
+		}
 		if ((fe->fe_cx & FSR_NX) ||
 		    (fe->fe_fsr & (FSR_UF << FSR_TEM_SHIFT)))
 			fe->fe_cx |= FSR_UF;
@@ -407,6 +410,7 @@ zero:		res[1] = 0;
 	if ((exp = fp->fp_exp + DBL_EXP_BIAS) <= 0) {
 		(void) __fpu_shr(fp, FP_NMANT - FP_NG - DBL_FRACBITS - exp);
 		if (fpround(fe, fp) && fp->fp_mant[2] == DBL_EXP(1)) {
+			fe->fe_cx |= FSR_UF;
 			res[1] = 0;
 			return (sign | DBL_EXP(1) | 0);
 		}
@@ -426,7 +430,7 @@ zero:		res[1] = 0;
 			return (sign | DBL_EXP(DBL_EXP_INFNAN) | 0);
 		}
 		res[1] = ~0;
-		return (sign | DBL_EXP(DBL_EXP_INFNAN) | DBL_MASK);
+		return (sign | DBL_EXP(DBL_EXP_INFNAN - 1) | DBL_MASK);
 	}
 done:
 	res[1] = fp->fp_mant[3];
@@ -468,6 +472,7 @@ zero:		res[1] = res[2] = res[3] = 0;
 	if ((exp = fp->fp_exp + EXT_EXP_BIAS) <= 0) {
 		(void) __fpu_shr(fp, FP_NMANT - FP_NG - EXT_FRACBITS - exp);
 		if (fpround(fe, fp) && fp->fp_mant[0] == EXT_EXP(1)) {
+			fe->fe_cx |= FSR_UF;
 			res[1] = res[2] = res[3] = 0;
 			return (sign | EXT_EXP(1) | 0);
 		}
@@ -487,7 +492,7 @@ zero:		res[1] = res[2] = res[3] = 0;
 			return (sign | EXT_EXP(EXT_EXP_INFNAN) | 0);
 		}
 		res[1] = res[2] = res[3] = ~0;
-		return (sign | EXT_EXP(EXT_EXP_INFNAN) | EXT_MASK);
+		return (sign | EXT_EXP(EXT_EXP_INFNAN - 1) | EXT_MASK);
 	}
 done:
 	res[1] = fp->fp_mant[1];
-- 
cgit v1.1


From c54f8e795a1ebd5c6fb6729a0d1e70819cc139ba Mon Sep 17 00:00:00 2001
From: marius <marius@FreeBSD.org>
Date: Tue, 30 Mar 2010 19:13:37 +0000
Subject: MFC: r205410

Avoid aliasing which leads to incorrect results when compiling with the
default strict aliasing rules.

PR:		144900
Submitted by:	Peter Jeremy
---
 lib/libc/sparc64/fpu/fpu_explode.c | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/sparc64/fpu/fpu_explode.c b/lib/libc/sparc64/fpu/fpu_explode.c
index 12c8802..474e0df 100644
--- a/lib/libc/sparc64/fpu/fpu_explode.c
+++ b/lib/libc/sparc64/fpu/fpu_explode.c
@@ -139,9 +139,9 @@ __fpu_xtof(fp, i)
 	 * a signed or unsigned entity.
 	 */
 	if (fp->fp_sign && (int64_t)i < 0)
-		*((int64_t*)fp->fp_mant) = -i;
+		*((int64_t *)fp->fp_mant) = -i;
 	else
-		*((int64_t*)fp->fp_mant) = i;
+		*((int64_t *)fp->fp_mant) = i;
 	fp->fp_mant[2] = 0;
 	fp->fp_mant[3] = 0;
 	__fpu_norm(fp);
@@ -262,14 +262,12 @@ __fpu_explode(fe, fp, type, reg)
 	struct fpn *fp;
 	int type, reg;
 {
-	u_int32_t s, *sp;
-	u_int64_t l[2];
-	void *vl = l;
+	u_int64_t l0, l1;
+	u_int32_t s;
 
 	if (type == FTYPE_LNG || type == FTYPE_DBL || type == FTYPE_EXT) {
-		l[0] = __fpu_getreg64(reg & ~1);
-		sp = vl;
-		fp->fp_sign = sp[0] >> 31;
+		l0 = __fpu_getreg64(reg & ~1);
+		fp->fp_sign = l0 >> 63;
 	} else {
 		s = __fpu_getreg(reg);
 		fp->fp_sign = s >> 31;
@@ -277,7 +275,7 @@ __fpu_explode(fe, fp, type, reg)
 	fp->fp_sticky = 0;
 	switch (type) {
 	case FTYPE_LNG:
-		s = __fpu_xtof(fp, l[0]);
+		s = __fpu_xtof(fp, l0);
 		break;
 
 	case FTYPE_INT:
@@ -289,12 +287,13 @@ __fpu_explode(fe, fp, type, reg)
 		break;
 
 	case FTYPE_DBL:
-		s = __fpu_dtof(fp, sp[0], sp[1]);
+		s = __fpu_dtof(fp, l0 >> 32, l0 & 0xffffffff);
 		break;
 
 	case FTYPE_EXT:
-		l[1] = __fpu_getreg64((reg & ~1) + 2);
-		s = __fpu_qtof(fp, sp[0], sp[1], sp[2], sp[3]);
+		l1 = __fpu_getreg64((reg & ~1) + 2);
+		s = __fpu_qtof(fp, l0 >> 32, l0 & 0xffffffff, l1 >> 32,
+		    l1 & 0xffffffff);
 		break;
 
 	default:
-- 
cgit v1.1


From 5c3b044c3171a07b537efc08d6c9c5bab7c12e62 Mon Sep 17 00:00:00 2001
From: gahr <gahr@FreeBSD.org>
Date: Wed, 31 Mar 2010 13:51:31 +0000
Subject: MFC r205606 Remove const'ness from dlerror(3) prototype, for
 consistency with POSIX.

Approved by:	cognet
---
 lib/libc/gen/dlfcn.c  | 4 ++--
 lib/libc/gen/dlopen.3 | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/gen/dlfcn.c b/lib/libc/gen/dlfcn.c
index 4be8847..a1ca29d 100644
--- a/lib/libc/gen/dlfcn.c
+++ b/lib/libc/gen/dlfcn.c
@@ -34,7 +34,7 @@ __FBSDID("$FreeBSD$");
 #include <link.h>
 #include <stddef.h>
 
-static const char sorry[] = "Service unavailable";
+static char sorry[] = "Service unavailable";
 
 /*
  * For ELF, the dynamic linker directly resolves references to its
@@ -69,7 +69,7 @@ dlclose(void *handle)
 }
 
 #pragma weak dlerror
-const char *
+char *
 dlerror(void)
 {
 	return sorry;
diff --git a/lib/libc/gen/dlopen.3 b/lib/libc/gen/dlopen.3
index 6488bef..d4b111a 100644
--- a/lib/libc/gen/dlopen.3
+++ b/lib/libc/gen/dlopen.3
@@ -52,7 +52,7 @@
 .Fn dlsym "void * restrict handle" "const char * restrict symbol"
 .Ft dlfunc_t
 .Fn dlfunc "void * restrict handle" "const char * restrict symbol"
-.Ft const char *
+.Ft char *
 .Fn dlerror "void"
 .Ft int
 .Fn dlclose "void *handle"
-- 
cgit v1.1


From e57fc48ad00a5d6274166cf7517775ba1d2301bd Mon Sep 17 00:00:00 2001
From: thompsa <thompsa@FreeBSD.org>
Date: Wed, 7 Apr 2010 00:34:05 +0000
Subject: MFC r203147

 Add a function to check if the usb devices is still connected.

Submitted by:	Hans Petter Selasky
---
 lib/libusb/libusb20.3        | 11 +++++++++++
 lib/libusb/libusb20.c        | 10 ++++++++++
 lib/libusb/libusb20.h        |  1 +
 lib/libusb/libusb20_int.h    |  2 ++
 lib/libusb/libusb20_ugen20.c | 20 ++++++++++++++++++++
 5 files changed, 44 insertions(+)

(limited to 'lib')

diff --git a/lib/libusb/libusb20.3 b/lib/libusb/libusb20.3
index 5404013..7896e8f0 100644
--- a/lib/libusb/libusb20.3
+++ b/lib/libusb/libusb20.3
@@ -143,6 +143,8 @@ USB access library (libusb -lusb)
 .Ft int
 .Fn libusb20_dev_reset "struct libusb20_device *pdev"
 .Ft int
+.Fn libusb20_dev_check_connected "struct libusb20_device *pdev"
+.Ft int
 .Fn libusb20_dev_set_power_mode "struct libusb20_device *pdev" "uint8_t power_mode"
 .Ft uint8_t
 .Fn libusb20_dev_get_power_mode "struct libusb20_device *pdev"
@@ -677,6 +679,15 @@ the last set USB configuration.
 This function returns zero on success else a LIBUSB20_ERROR value is
 returned.
 .
+.
+.Pp
+.
+.Fn libusb20_dev_check_connected
+will check if an opened USB device is still connected.
+.
+This function returns zero if the device is still connected else a LIBUSB20_ERROR value is returned.
+.
+.
 .Pp
 .
 .Fn libusb20_dev_set_power_mode
diff --git a/lib/libusb/libusb20.c b/lib/libusb/libusb20.c
index 1c75fc1..e181b49 100644
--- a/lib/libusb/libusb20.c
+++ b/lib/libusb/libusb20.c
@@ -67,6 +67,7 @@ dummy_callback(struct libusb20_transfer *xfer)
 #define	dummy_set_config_index (void *)dummy_int
 #define	dummy_set_alt_index (void *)dummy_int
 #define	dummy_reset_device (void *)dummy_int
+#define	dummy_check_connected (void *)dummy_int
 #define	dummy_set_power_mode (void *)dummy_int
 #define	dummy_get_power_mode (void *)dummy_int
 #define	dummy_kernel_driver_active (void *)dummy_int
@@ -673,6 +674,15 @@ libusb20_dev_reset(struct libusb20_device *pdev)
 }
 
 int
+libusb20_dev_check_connected(struct libusb20_device *pdev)
+{
+	int error;
+
+	error = pdev->methods->check_connected(pdev);
+	return (error);
+}
+
+int
 libusb20_dev_set_power_mode(struct libusb20_device *pdev, uint8_t power_mode)
 {
 	int error;
diff --git a/lib/libusb/libusb20.h b/lib/libusb/libusb20.h
index bb7d8a4..6b253aa 100644
--- a/lib/libusb/libusb20.h
+++ b/lib/libusb/libusb20.h
@@ -250,6 +250,7 @@ int	libusb20_dev_request_sync(struct libusb20_device *pdev, struct LIBUSB20_CONT
 int	libusb20_dev_req_string_sync(struct libusb20_device *pdev, uint8_t index, uint16_t langid, void *ptr, uint16_t len);
 int	libusb20_dev_req_string_simple_sync(struct libusb20_device *pdev, uint8_t index, void *ptr, uint16_t len);
 int	libusb20_dev_reset(struct libusb20_device *pdev);
+int	libusb20_dev_check_connected(struct libusb20_device *pdev);
 int	libusb20_dev_set_power_mode(struct libusb20_device *pdev, uint8_t power_mode);
 uint8_t	libusb20_dev_get_power_mode(struct libusb20_device *pdev);
 int	libusb20_dev_set_alt_index(struct libusb20_device *pdev, uint8_t iface_index, uint8_t alt_index);
diff --git a/lib/libusb/libusb20_int.h b/lib/libusb/libusb20_int.h
index 004b0cf..b0e0e2d 100644
--- a/lib/libusb/libusb20_int.h
+++ b/lib/libusb/libusb20_int.h
@@ -101,6 +101,7 @@ typedef int (libusb20_set_power_mode_t)(struct libusb20_device *pdev, uint8_t po
 typedef int (libusb20_get_power_mode_t)(struct libusb20_device *pdev, uint8_t *power_mode);
 typedef int (libusb20_set_alt_index_t)(struct libusb20_device *pdev, uint8_t iface_index, uint8_t alt_index);
 typedef int (libusb20_set_config_index_t)(struct libusb20_device *pdev, uint8_t index);
+typedef int (libusb20_check_connected_t)(struct libusb20_device *pdev);
 
 /* USB transfer specific */
 typedef int (libusb20_tr_open_t)(struct libusb20_transfer *xfer, uint32_t MaxBufSize, uint32_t MaxFrameCount, uint8_t ep_no);
@@ -117,6 +118,7 @@ typedef void (libusb20_tr_cancel_async_t)(struct libusb20_transfer *xfer);
   m(n, kernel_driver_active) \
   m(n, process) \
   m(n, reset_device) \
+  m(n, check_connected) \
   m(n, set_power_mode) \
   m(n, get_power_mode) \
   m(n, set_alt_index) \
diff --git a/lib/libusb/libusb20_ugen20.c b/lib/libusb/libusb20_ugen20.c
index efcca63..892ba0e 100644
--- a/lib/libusb/libusb20_ugen20.c
+++ b/lib/libusb/libusb20_ugen20.c
@@ -67,6 +67,7 @@ static libusb20_get_config_index_t ugen20_get_config_index;
 static libusb20_set_config_index_t ugen20_set_config_index;
 static libusb20_set_alt_index_t ugen20_set_alt_index;
 static libusb20_reset_device_t ugen20_reset_device;
+static libusb20_check_connected_t ugen20_check_connected;
 static libusb20_set_power_mode_t ugen20_set_power_mode;
 static libusb20_get_power_mode_t ugen20_get_power_mode;
 static libusb20_kernel_driver_active_t ugen20_kernel_driver_active;
@@ -553,6 +554,25 @@ ugen20_reset_device(struct libusb20_device *pdev)
 }
 
 static int
+ugen20_check_connected(struct libusb20_device *pdev)
+{
+	uint32_t plugtime;
+	int error = 0;
+
+	if (ioctl(pdev->file_ctrl, USB_GET_PLUGTIME, &plugtime)) {
+		error = LIBUSB20_ERROR_NO_DEVICE;
+		goto done;
+	}
+
+	if (pdev->session_data.plugtime != plugtime) {
+		error = LIBUSB20_ERROR_NO_DEVICE;
+		goto done;
+	}
+done:
+	return (error);
+}
+
+static int
 ugen20_set_power_mode(struct libusb20_device *pdev, uint8_t power_mode)
 {
 	int temp;
-- 
cgit v1.1


From 522064f59658ae18f0477f0981fbc7ca1f8e06a3 Mon Sep 17 00:00:00 2001
From: avg <avg@FreeBSD.org>
Date: Thu, 8 Apr 2010 07:43:15 +0000
Subject: MFC r206177: hash.3: fix a factual mistake in the man page

---
 lib/libc/db/man/hash.3 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libc/db/man/hash.3 b/lib/libc/db/man/hash.3
index 6724042..133885d 100644
--- a/lib/libc/db/man/hash.3
+++ b/lib/libc/db/man/hash.3
@@ -78,7 +78,7 @@ The
 element
 defines the
 .Nm
-table bucket size, and is, by default, 256 bytes.
+table bucket size, and is, by default, 4096 bytes.
 It may be preferable to increase the page size for disk-resident tables
 and tables with large data items.
 .It Va ffactor
-- 
cgit v1.1


From 467d5479951a967aa4ed31d07540c64b7f2500d8 Mon Sep 17 00:00:00 2001
From: avg <avg@FreeBSD.org>
Date: Mon, 12 Apr 2010 21:12:03 +0000
Subject: MFC r206178: ibc/db/hash: cap auto-tuned block size

---
 lib/libc/db/hash/hash.c | 2 ++
 lib/libc/db/hash/hash.h | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libc/db/hash/hash.c b/lib/libc/db/hash/hash.c
index 83d2657..638814c 100644
--- a/lib/libc/db/hash/hash.c
+++ b/lib/libc/db/hash/hash.c
@@ -293,6 +293,8 @@ init_hash(HTAB *hashp, const char *file, const HASHINFO *info)
 		if (stat(file, &statbuf))
 			return (NULL);
 		hashp->BSIZE = statbuf.st_blksize;
+		if (hashp->BSIZE > MAX_BSIZE)
+			hashp->BSIZE = MAX_BSIZE;
 		hashp->BSHIFT = __log2(hashp->BSIZE);
 	}
 
diff --git a/lib/libc/db/hash/hash.h b/lib/libc/db/hash/hash.h
index 8329413..cd11a3a 100644
--- a/lib/libc/db/hash/hash.h
+++ b/lib/libc/db/hash/hash.h
@@ -118,7 +118,7 @@ typedef struct htab	 {		/* Memory resident data structure */
 /*
  * Constants
  */
-#define	MAX_BSIZE		65536		/* 2^16 */
+#define	MAX_BSIZE		32768		/* 2^15 but should be 65536 */
 #define MIN_BUFFERS		6
 #define MINHDRSIZE		512
 #define DEF_BUFSIZE		65536		/* 64 K */
-- 
cgit v1.1


From c79cf0b1c9291cf43f8fbdbe73cd46a4c0579063 Mon Sep 17 00:00:00 2001
From: trasz <trasz@FreeBSD.org>
Date: Tue, 13 Apr 2010 06:01:24 +0000
Subject: MFC r205796:

Make acl_to_text_np(3) not crash on long group or user names in NFSv4 ACLs.

PR:		amd64/145091
---
 lib/libc/posix1e/acl_to_text_nfs4.c | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/posix1e/acl_to_text_nfs4.c b/lib/libc/posix1e/acl_to_text_nfs4.c
index 3adbfb4..5d0aa31 100644
--- a/lib/libc/posix1e/acl_to_text_nfs4.c
+++ b/lib/libc/posix1e/acl_to_text_nfs4.c
@@ -167,7 +167,7 @@ format_additional_id(char *str, size_t size, const acl_entry_t entry)
 static int
 format_entry(char *str, size_t size, const acl_entry_t entry, int flags)
 {
-	size_t off = 0, padding_length, maximum_who_field_length = 18;
+	size_t off = 0, min_who_field_length = 18;
 	acl_permset_t permset;
 	acl_flagset_t flagset;
 	int error, len;
@@ -188,12 +188,9 @@ format_entry(char *str, size_t size, const acl_entry_t entry, int flags)
 	if (error)
 		return (error);
 	len = strlen(buf);
-	padding_length = maximum_who_field_length - len;
-	if (padding_length > 0) {
-		memset(str, ' ', padding_length);
-		off += padding_length;
-	}
-	off += snprintf(str + off, size - off, "%s:", buf);
+	if (len < min_who_field_length)
+		len = min_who_field_length;
+	off += snprintf(str + off, size - off, "%*s:", len, buf);
 
 	error = _nfs4_format_access_mask(buf, sizeof(buf), *permset,
 	    flags & ACL_TEXT_VERBOSE);
-- 
cgit v1.1


From 89a00e020b2c8a517462b74b3c52eaf1d11f7d6f Mon Sep 17 00:00:00 2001
From: jhb <jhb@FreeBSD.org>
Date: Wed, 14 Apr 2010 15:22:58 +0000
Subject: MFC 205536: Reject attempts to create a MAP_ANON mapping with a
 non-zero offset.

---
 lib/libc/sys/mmap.2 | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libc/sys/mmap.2 b/lib/libc/sys/mmap.2
index e632748..85019b2 100644
--- a/lib/libc/sys/mmap.2
+++ b/lib/libc/sys/mmap.2
@@ -105,7 +105,7 @@ The file descriptor used for creating
 must be \-1.
 The
 .Fa offset
-argument is ignored.
+argument must be 0.
 .\".It Dv MAP_FILE
 .\"Mapped from a regular file or character-special device memory.
 .It Dv MAP_FIXED
@@ -312,6 +312,11 @@ was equal to zero.
 was specified and the
 .Fa fd
 argument was not -1.
+.It Bq Er EINVAL
+.Dv MAP_ANON
+was specified and the
+.Fa offset
+argument was not 0.
 .It Bq Er ENODEV
 .Dv MAP_ANON
 has not been specified and
-- 
cgit v1.1


From 9323e2ba3f24089d879edac55fbd2d36dde42947 Mon Sep 17 00:00:00 2001
From: jhb <jhb@FreeBSD.org>
Date: Wed, 14 Apr 2010 17:17:06 +0000
Subject: MFC 205900: Use panic() (which the environment is required to provide
 to libstand) to implement assert() instead of relying on a non-required
 exit().  The exit() invocation also did not match the semantics of the exit()
 routine that current boot environments happen to require.

---
 lib/libstand/assert.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/libstand/assert.c b/lib/libstand/assert.c
index b424358..8eec63a 100644
--- a/lib/libstand/assert.c
+++ b/lib/libstand/assert.c
@@ -35,10 +35,10 @@ void
 __assert(const char *func, const char *file, int line, const char *expression)
 {
 	if (func == NULL)
-		printf("Assertion failed: (%s), file %s, line %d.\n",
+		panic("Assertion failed: (%s), file %s, line %d.\n",
 		    expression, file, line);
 	else
-		printf("Assertion failed: (%s), function %s, file %s, line "
-		    "%d.\n", expression, func, file, line);
-	exit();
+		panic(
+		    "Assertion failed: (%s), function %s, file %s, line %d.\n",
+		    expression, func, file, line);
 }
-- 
cgit v1.1


From 3a63a51857c2a0f86bb2d3b2638ec357a8500db8 Mon Sep 17 00:00:00 2001
From: fabient <fabient@FreeBSD.org>
Date: Fri, 16 Apr 2010 15:45:09 +0000
Subject: MFC r206089, r206684:

- Support for uncore counting events: one fixed PMC with the uncore
   domain clock, 8 programmable PMC.
- Westmere based CPU (Xeon 5600, Corei7 980X) support.
- New man pages with events list for core and uncore.
- Updated Corei7 events with Intel 253669-033US December 2009 doc.
  There is some removed events in the documentation, they have been
  kept in the code but documented in the man page as obsolete.
- Offcore response events can be setup with rsp token.

Sponsored by: NETASQ
---
 lib/libpmc/Makefile         |    5 +
 lib/libpmc/libpmc.c         |  181 ++++-
 lib/libpmc/pmc.corei7.3     | 1581 +++++++++++++++++++++++++++++++++++++++++++
 lib/libpmc/pmc.corei7uc.3   |  880 ++++++++++++++++++++++++
 lib/libpmc/pmc.ucf.3        |  115 ++++
 lib/libpmc/pmc.westmere.3   | 1329 ++++++++++++++++++++++++++++++++++++
 lib/libpmc/pmc.westmereuc.3 | 1083 +++++++++++++++++++++++++++++
 7 files changed, 5167 insertions(+), 7 deletions(-)
 create mode 100644 lib/libpmc/pmc.corei7.3
 create mode 100644 lib/libpmc/pmc.corei7uc.3
 create mode 100644 lib/libpmc/pmc.ucf.3
 create mode 100644 lib/libpmc/pmc.westmere.3
 create mode 100644 lib/libpmc/pmc.westmereuc.3

(limited to 'lib')

diff --git a/lib/libpmc/Makefile b/lib/libpmc/Makefile
index 7d97c96..3ad1c4d 100644
--- a/lib/libpmc/Makefile
+++ b/lib/libpmc/Makefile
@@ -28,11 +28,16 @@ MAN+=	pmc.atom.3
 MAN+=	pmc.core.3
 MAN+=	pmc.core2.3
 MAN+=	pmc.iaf.3
+MAN+=	pmc.ucf.3
 MAN+=	pmc.k7.3
 MAN+=	pmc.k8.3
 MAN+=	pmc.p4.3
 MAN+=	pmc.p5.3
 MAN+=	pmc.p6.3
+MAN+=	pmc.corei7.3
+MAN+=	pmc.corei7uc.3
+MAN+=	pmc.westmere.3
+MAN+=	pmc.westmereuc.3
 MAN+=	pmc.tsc.3
 
 MLINKS+= \
diff --git a/lib/libpmc/libpmc.c b/lib/libpmc/libpmc.c
index af716da..929ca73 100644
--- a/lib/libpmc/libpmc.c
+++ b/lib/libpmc/libpmc.c
@@ -54,6 +54,10 @@ static int iaf_allocate_pmc(enum pmc_event _pe, char *_ctrspec,
     struct pmc_op_pmcallocate *_pmc_config);
 static int iap_allocate_pmc(enum pmc_event _pe, char *_ctrspec,
     struct pmc_op_pmcallocate *_pmc_config);
+static int ucf_allocate_pmc(enum pmc_event _pe, char *_ctrspec,
+    struct pmc_op_pmcallocate *_pmc_config);
+static int ucp_allocate_pmc(enum pmc_event _pe, char *_ctrspec,
+    struct pmc_op_pmcallocate *_pmc_config);
 static int k8_allocate_pmc(enum pmc_event _pe, char *_ctrspec,
     struct pmc_op_pmcallocate *_pmc_config);
 static int p4_allocate_pmc(enum pmc_event _pe, char *_ctrspec,
@@ -132,6 +136,7 @@ PMC_CLASSDEP_TABLE(k8, K8);
 PMC_CLASSDEP_TABLE(p4, P4);
 PMC_CLASSDEP_TABLE(p5, P5);
 PMC_CLASSDEP_TABLE(p6, P6);
+PMC_CLASSDEP_TABLE(ucf, UCF);
 
 #undef	__PMC_EV_ALIAS
 #define	__PMC_EV_ALIAS(N,CODE) 	{ N, PMC_EV_##CODE },
@@ -157,6 +162,21 @@ static const struct pmc_event_descr corei7_event_table[] =
 	__PMC_EV_ALIAS_COREI7()
 };
 
+static const struct pmc_event_descr westmere_event_table[] =
+{
+	__PMC_EV_ALIAS_WESTMERE()
+};
+
+static const struct pmc_event_descr corei7uc_event_table[] =
+{
+	__PMC_EV_ALIAS_COREI7UC()
+};
+
+static const struct pmc_event_descr westmereuc_event_table[] =
+{
+	__PMC_EV_ALIAS_WESTMEREUC()
+};
+
 /*
  * PMC_MDEP_TABLE(NAME, PRIMARYCLASS, ADDITIONAL_CLASSES...)
  *
@@ -170,7 +190,8 @@ static const struct pmc_event_descr corei7_event_table[] =
 PMC_MDEP_TABLE(atom, IAP, PMC_CLASS_IAF, PMC_CLASS_TSC);
 PMC_MDEP_TABLE(core, IAP, PMC_CLASS_TSC);
 PMC_MDEP_TABLE(core2, IAP, PMC_CLASS_IAF, PMC_CLASS_TSC);
-PMC_MDEP_TABLE(corei7, IAP, PMC_CLASS_IAF, PMC_CLASS_TSC);
+PMC_MDEP_TABLE(corei7, IAP, PMC_CLASS_IAF, PMC_CLASS_TSC, PMC_CLASS_UCF, PMC_CLASS_UCP);
+PMC_MDEP_TABLE(westmere, IAP, PMC_CLASS_IAF, PMC_CLASS_TSC, PMC_CLASS_UCF, PMC_CLASS_UCP);
 PMC_MDEP_TABLE(k7, K7, PMC_CLASS_TSC);
 PMC_MDEP_TABLE(k8, K8, PMC_CLASS_TSC);
 PMC_MDEP_TABLE(p4, P4, PMC_CLASS_TSC);
@@ -201,6 +222,10 @@ PMC_CLASS_TABLE_DESC(atom, IAP, atom, iap);
 PMC_CLASS_TABLE_DESC(core, IAP, core, iap);
 PMC_CLASS_TABLE_DESC(core2, IAP, core2, iap);
 PMC_CLASS_TABLE_DESC(corei7, IAP, corei7, iap);
+PMC_CLASS_TABLE_DESC(westmere, IAP, westmere, iap);
+PMC_CLASS_TABLE_DESC(ucf, UCF, ucf, ucf);
+PMC_CLASS_TABLE_DESC(corei7uc, UCP, corei7uc, ucp);
+PMC_CLASS_TABLE_DESC(westmereuc, UCP, westmereuc, ucp);
 #endif
 #if	defined(__i386__)
 PMC_CLASS_TABLE_DESC(k7, K7, k7, k7);
@@ -281,7 +306,7 @@ struct pmc_masks {
 	const uint32_t	pm_value;
 };
 #define	PMCMASK(N,V)	{ .pm_name = #N, .pm_value = (V) }
-#define	NULLMASK	PMCMASK(NULL,0)
+#define	NULLMASK	{ .pm_name = NULL }
 
 #if defined(__amd64__) || defined(__i386__)
 static int
@@ -474,6 +499,8 @@ static struct pmc_event_alias core2_aliases_without_iaf[] = {
 #define	atom_aliases_without_iaf	core2_aliases_without_iaf
 #define corei7_aliases			core2_aliases
 #define corei7_aliases_without_iaf	core2_aliases_without_iaf
+#define westmere_aliases		core2_aliases
+#define westmere_aliases_without_iaf	core2_aliases_without_iaf
 
 #define	IAF_KW_OS		"os"
 #define	IAF_KW_USR		"usr"
@@ -524,6 +551,7 @@ iaf_allocate_pmc(enum pmc_event pe, char *ctrspec,
 #define	IAP_KW_SNOOPTYPE	"snooptype"
 #define	IAP_KW_TRANSITION	"trans"
 #define	IAP_KW_USR		"usr"
+#define	IAP_KW_RSP		"rsp"
 
 static struct pmc_masks iap_core_mask[] = {
 	PMCMASK(all,	(0x3 << 14)),
@@ -571,19 +599,38 @@ static struct pmc_masks iap_transition_mask[] = {
 	NULLMASK
 };
 
+static struct pmc_masks iap_rsp_mask[] = {
+	PMCMASK(DMND_DATA_RD,		(1 <<  0)),
+	PMCMASK(DMND_RFO,		(1 <<  1)),
+	PMCMASK(DMND_IFETCH,		(1 <<  2)),
+	PMCMASK(WB,			(1 <<  3)),
+	PMCMASK(PF_DATA_RD,		(1 <<  4)),
+	PMCMASK(PF_RFO,			(1 <<  5)),
+	PMCMASK(PF_IFETCH,		(1 <<  6)),
+	PMCMASK(OTHER,			(1 <<  7)),
+	PMCMASK(UNCORE_HIT,		(1 <<  8)),
+	PMCMASK(OTHER_CORE_HIT_SNP,	(1 <<  9)),
+	PMCMASK(OTHER_CORE_HITM,	(1 << 10)),
+	PMCMASK(REMOTE_CACHE_FWD,	(1 << 12)),
+	PMCMASK(REMOTE_DRAM,		(1 << 13)),
+	PMCMASK(LOCAL_DRAM,		(1 << 14)),
+	PMCMASK(NON_DRAM,		(1 << 15)),
+	NULLMASK
+};
+
 static int
 iap_allocate_pmc(enum pmc_event pe, char *ctrspec,
     struct pmc_op_pmcallocate *pmc_config)
 {
 	char *e, *p, *q;
-	uint32_t cachestate, evmask;
+	uint32_t cachestate, evmask, rsp;
 	int count, n;
 
 	pmc_config->pm_caps |= (PMC_CAP_READ | PMC_CAP_WRITE |
 	    PMC_CAP_QUALIFIER);
 	pmc_config->pm_md.pm_iap.pm_iap_config = 0;
 
-	cachestate = evmask = 0;
+	cachestate = evmask = rsp = 0;
 
 	/* Parse additional modifiers if present */
 	while ((p = strsep(&ctrspec, ",")) != NULL) {
@@ -630,8 +677,7 @@ iap_allocate_pmc(enum pmc_event pe, char *ctrspec,
 				return (-1);
 		} else if (cpu_info.pm_cputype == PMC_CPU_INTEL_ATOM ||
 		    cpu_info.pm_cputype == PMC_CPU_INTEL_CORE2 ||
-		    cpu_info.pm_cputype == PMC_CPU_INTEL_CORE2EXTREME ||
-		    cpu_info.pm_cputype == PMC_CPU_INTEL_COREI7) {
+		    cpu_info.pm_cputype == PMC_CPU_INTEL_CORE2EXTREME) {
 			if (KWPREFIXMATCH(p, IAP_KW_SNOOPRESPONSE "=")) {
 				n = pmc_parse_mask(iap_snoopresponse_mask, p,
 				    &evmask);
@@ -640,6 +686,12 @@ iap_allocate_pmc(enum pmc_event pe, char *ctrspec,
 				    &evmask);
 			} else
 				return (-1);
+		} else if (cpu_info.pm_cputype == PMC_CPU_INTEL_COREI7 ||
+		    cpu_info.pm_cputype == PMC_CPU_INTEL_WESTMERE) {
+			if (KWPREFIXMATCH(p, IAP_KW_RSP "=")) {
+				n = pmc_parse_mask(iap_rsp_mask, p, &rsp);
+			} else
+				return (-1);
 		} else
 			return (-1);
 
@@ -672,6 +724,69 @@ iap_allocate_pmc(enum pmc_event pe, char *ctrspec,
 	}
 
 	pmc_config->pm_md.pm_iap.pm_iap_config |= cachestate;
+	pmc_config->pm_md.pm_iap.pm_iap_rsp = rsp;
+
+	return (0);
+}
+
+/*
+ * Intel Uncore.
+ */
+
+static int
+ucf_allocate_pmc(enum pmc_event pe, char *ctrspec,
+    struct pmc_op_pmcallocate *pmc_config)
+{
+	(void) pe;
+	(void) ctrspec;
+
+	pmc_config->pm_caps |= (PMC_CAP_READ | PMC_CAP_WRITE);
+	pmc_config->pm_md.pm_ucf.pm_ucf_flags = 0;
+
+	return (0);
+}
+
+#define	UCP_KW_CMASK		"cmask"
+#define	UCP_KW_EDGE		"edge"
+#define	UCP_KW_INV		"inv"
+
+static int
+ucp_allocate_pmc(enum pmc_event pe, char *ctrspec,
+    struct pmc_op_pmcallocate *pmc_config)
+{
+	char *e, *p, *q;
+	int count, n;
+
+	(void) pe;
+
+	pmc_config->pm_caps |= (PMC_CAP_READ | PMC_CAP_WRITE |
+	    PMC_CAP_QUALIFIER);
+	pmc_config->pm_md.pm_ucp.pm_ucp_config = 0;
+
+	/* Parse additional modifiers if present */
+	while ((p = strsep(&ctrspec, ",")) != NULL) {
+
+		n = 0;
+		if (KWPREFIXMATCH(p, UCP_KW_CMASK "=")) {
+			q = strchr(p, '=');
+			if (*++q == '\0') /* skip '=' */
+				return (-1);
+			count = strtol(q, &e, 0);
+			if (e == q || *e != '\0')
+				return (-1);
+			pmc_config->pm_caps |= PMC_CAP_THRESHOLD;
+			pmc_config->pm_md.pm_ucp.pm_ucp_config |=
+			    UCP_CMASK(count);
+		} else if (KWMATCH(p, UCP_KW_EDGE)) {
+			pmc_config->pm_caps |= PMC_CAP_EDGE;
+		} else if (KWMATCH(p, UCP_KW_INV)) {
+			pmc_config->pm_caps |= PMC_CAP_INVERT;
+		} else
+			return (-1);
+
+		if (n < 0)	/* Parsing failed. */
+			return (-1);
+	}
 
 	return (0);
 }
@@ -2309,6 +2424,31 @@ pmc_event_names_of_class(enum pmc_class cl, const char ***eventnames,
 			ev = corei7_event_table;
 			count = PMC_EVENT_TABLE_SIZE(corei7);
 			break;
+		case PMC_CPU_INTEL_WESTMERE:
+			ev = westmere_event_table;
+			count = PMC_EVENT_TABLE_SIZE(westmere);
+			break;
+		}
+		break;
+	case PMC_CLASS_UCF:
+		ev = ucf_event_table;
+		count = PMC_EVENT_TABLE_SIZE(ucf);
+		break;
+	case PMC_CLASS_UCP:
+		/*
+		 * Return the most appropriate set of event name
+		 * spellings for the current CPU.
+		 */
+		switch (cpu_info.pm_cputype) {
+		default:
+		case PMC_CPU_INTEL_COREI7:
+			ev = corei7uc_event_table;
+			count = PMC_EVENT_TABLE_SIZE(corei7uc);
+			break;
+		case PMC_CPU_INTEL_WESTMERE:
+			ev = westmereuc_event_table;
+			count = PMC_EVENT_TABLE_SIZE(westmereuc);
+			break;
 		}
 		break;
 	case PMC_CLASS_TSC:
@@ -2514,8 +2654,15 @@ pmc_init(void)
 		PMC_MDEP_INIT_INTEL_V2(core2);
 		break;
 	case PMC_CPU_INTEL_COREI7:
+		pmc_class_table[n++] = &ucf_class_table_descr;
+		pmc_class_table[n++] = &corei7uc_class_table_descr;
 		PMC_MDEP_INIT_INTEL_V2(corei7);
 		break;
+	case PMC_CPU_INTEL_WESTMERE:
+		pmc_class_table[n++] = &ucf_class_table_descr;
+		pmc_class_table[n++] = &westmereuc_class_table_descr;
+		PMC_MDEP_INIT_INTEL_V2(westmere);
+		break;
 	case PMC_CPU_INTEL_PIV:
 		PMC_MDEP_INIT(p4);
 		pmc_class_table[n] = &p4_class_table_descr;
@@ -2618,10 +2765,30 @@ _pmc_name_of_event(enum pmc_event pe, enum pmc_cputype cpu)
 			ev = corei7_event_table;
 			evfence = corei7_event_table + PMC_EVENT_TABLE_SIZE(corei7);
 			break;
+		case PMC_CPU_INTEL_WESTMERE:
+			ev = westmere_event_table;
+			evfence = westmere_event_table + PMC_EVENT_TABLE_SIZE(westmere);
+			break;
+		default:	/* Unknown CPU type. */
+			break;
+		}
+	} else if (pe >= PMC_EV_UCF_FIRST && pe <= PMC_EV_UCF_LAST) {
+		ev = ucf_event_table;
+		evfence = ucf_event_table + PMC_EVENT_TABLE_SIZE(ucf);
+	} else if (pe >= PMC_EV_UCP_FIRST && pe <= PMC_EV_UCP_LAST) {
+		switch (cpu) {
+		case PMC_CPU_INTEL_COREI7:
+			ev = corei7uc_event_table;
+			evfence = corei7uc_event_table + PMC_EVENT_TABLE_SIZE(corei7uc);
+			break;
+		case PMC_CPU_INTEL_WESTMERE:
+			ev = westmereuc_event_table;
+			evfence = westmereuc_event_table + PMC_EVENT_TABLE_SIZE(westmereuc);
+			break;
 		default:	/* Unknown CPU type. */
 			break;
 		}
-	} if (pe >= PMC_EV_K7_FIRST && pe <= PMC_EV_K7_LAST) {
+	} else if (pe >= PMC_EV_K7_FIRST && pe <= PMC_EV_K7_LAST) {
 		ev = k7_event_table;
 		evfence = k7_event_table + PMC_EVENT_TABLE_SIZE(k7);
 	} else if (pe >= PMC_EV_K8_FIRST && pe <= PMC_EV_K8_LAST) {
diff --git a/lib/libpmc/pmc.corei7.3 b/lib/libpmc/pmc.corei7.3
new file mode 100644
index 0000000..1a8b9d9
--- /dev/null
+++ b/lib/libpmc/pmc.corei7.3
@@ -0,0 +1,1581 @@
+.\" Copyright (c) 2010 Fabien Thomas.  All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" This software is provided by Joseph Koshy ``as is'' and
+.\" any express or implied warranties, including, but not limited to, the
+.\" implied warranties of merchantability and fitness for a particular purpose
+.\" are disclaimed.  in no event shall Joseph Koshy be liable
+.\" for any direct, indirect, incidental, special, exemplary, or consequential
+.\" damages (including, but not limited to, procurement of substitute goods
+.\" or services; loss of use, data, or profits; or business interruption)
+.\" however caused and on any theory of liability, whether in contract, strict
+.\" liability, or tort (including negligence or otherwise) arising in any way
+.\" out of the use of this software, even if advised of the possibility of
+.\" such damage.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd March 24, 2010
+.Os
+.Dt PMC.COREI7 3
+.Sh NAME
+.Nm pmc.corei7
+.Nd measurement events for
+.Tn Intel
+.Tn Core i7 and Xeon 5500
+family CPUs
+.Sh LIBRARY
+.Lb libpmc
+.Sh SYNOPSIS
+.In pmc.h
+.Sh DESCRIPTION
+.Tn Intel
+.Tn "Core i7"
+CPUs contain PMCs conforming to version 2 of the
+.Tn Intel
+performance measurement architecture.
+These CPUs may contain up to three classes of PMCs:
+.Bl -tag -width "Li PMC_CLASS_IAP"
+.It Li PMC_CLASS_IAF
+Fixed-function counters that count only one hardware event per counter.
+.It Li PMC_CLASS_IAP
+Programmable counters that may be configured to count one of a defined
+set of hardware events.
+.El
+.Pp
+The number of PMCs available in each class and their widths need to be
+determined at run time by calling
+.Xr pmc_cpuinfo 3 .
+.Pp
+Intel Core i7 and Xeon 5500 PMCs are documented in
+.Rs
+.%B "Intel(R) 64 and IA-32 Architectures Software Developes Manual"
+.%T "Volume 3B: System Programming Guide, Part 2"
+.%N "Order Number: 253669-033US"
+.%D December 2009
+.%Q "Intel Corporation"
+.Re
+.Ss COREI7 AND XEON 5500 FIXED FUNCTION PMCS
+These PMCs and their supported events are documented in
+.Xr pmc.iaf 3 .
+Not all CPUs in this family implement fixed-function counters.
+.Ss COREI7 AND XEON 5500 PROGRAMMABLE PMCS
+The programmable PMCs support the following capabilities:
+.Bl -column "PMC_CAP_INTERRUPT" "Support"
+.It Em Capability Ta Em Support
+.It PMC_CAP_CASCADE Ta \&No
+.It PMC_CAP_EDGE Ta Yes
+.It PMC_CAP_INTERRUPT Ta Yes
+.It PMC_CAP_INVERT Ta Yes
+.It PMC_CAP_READ Ta Yes
+.It PMC_CAP_PRECISE Ta \&No
+.It PMC_CAP_SYSTEM Ta Yes
+.It PMC_CAP_TAGGING Ta \&No
+.It PMC_CAP_THRESHOLD Ta Yes
+.It PMC_CAP_USER Ta Yes
+.It PMC_CAP_WRITE Ta Yes
+.El
+.Ss Event Qualifiers
+Event specifiers for these PMCs support the following common
+qualifiers:
+.Bl -tag -width indent
+.It Li rsp= Ns Ar value
+Configure the Off-core Response bits.
+.Bl -tag -width indent
+.It Li DMND_DATA_RD
+Counts the number of demand and DCU prefetch data reads of full
+and partial cachelines as well as demand data page table entry
+cacheline reads. Does not count L2 data read prefetches or
+instruction fetches.
+.It Li DMND_RFO
+Counts the number of demand and DCU prefetch reads for ownership
+(RFO) requests generated by a write to data cacheline. Does not
+count L2 RFO.
+.It Li DMND_IFETCH
+Counts the number of demand and DCU prefetch instruction cacheline
+reads. Does not count L2 code read prefetches.
+WB
+Counts the number of writeback (modified to exclusive) transactions.
+.It Li PF_DATA_RD
+Counts the number of data cacheline reads generated by L2 prefetchers.
+.It Li PF_RFO
+Counts the number of RFO requests generated by L2 prefetchers.
+.It Li PF_IFETCH
+Counts the number of code reads generated by L2 prefetchers.
+.It Li OTHER
+Counts one of the following transaction types, including L3 invalidate,
+I/O, full or partial writes, WC or non-temporal stores, CLFLUSH, Fences,
+lock, unlock, split lock.
+.It Li UNCORE_HIT
+L3 Hit: local or remote home requests that hit L3 cache in the uncore
+with no coherency actions required (snooping).
+.It Li OTHER_CORE_HIT_SNP
+L3 Hit: local or remote home requests that hit L3 cache in the uncore
+and was serviced by another core with a cross core snoop where no modified
+copies were found (clean).
+.It Li OTHER_CORE_HITM
+L3 Hit: local or remote home requests that hit L3 cache in the uncore
+and was serviced by another core with a cross core snoop where modified
+copies were found (HITM).
+.It Li REMOTE_CACHE_FWD
+L3 Miss: local homed requests that missed the L3 cache and was serviced
+by forwarded data following a cross package snoop where no modified
+copies found. (Remote home requests are not counted)
+.It Li REMOTE_DRAM
+L3 Miss: remote home requests that missed the L3 cache and were serviced
+by remote DRAM.
+.It Li LOCAL_DRAM
+L3 Miss: local home requests that missed the L3 cache and were serviced
+by local DRAM.
+.It Li NON_DRAM
+Non-DRAM requests that were serviced by IOH.
+.El
+.It Li cmask= Ns Ar value
+Configure the PMC to increment only if the number of configured
+events measured in a cycle is greater than or equal to
+.Ar value .
+.It Li edge
+Configure the PMC to count the number of de-asserted to asserted
+transitions of the conditions expressed by the other qualifiers.
+If specified, the counter will increment only once whenever a
+condition becomes true, irrespective of the number of clocks during
+which the condition remains true.
+.It Li inv
+Invert the sense of comparison when the
+.Dq Li cmask
+qualifier is present, making the counter increment when the number of
+events per cycle is less than the value specified by the
+.Dq Li cmask
+qualifier.
+.It Li os
+Configure the PMC to count events happening at processor privilege
+level 0.
+.It Li usr
+Configure the PMC to count events occurring at privilege levels 1, 2
+or 3.
+.El
+.Pp
+If neither of the
+.Dq Li os
+or
+.Dq Li usr
+qualifiers are specified, the default is to enable both.
+.Ss Event Specifiers (Programmable PMCs)
+Core i7 and Xeon 5500 programmable PMCs support the following events:
+.Bl -tag -width indent
+.It Li SB_DRAIN.ANY
+.Pq Event 04H , Umask 07H
+Counts the number of store buffer drains.
+.It Li STORE_BLOCKS.AT_RET
+.Pq Event 06H , Umask 04H
+Counts number of loads delayed with at-Retirement block code. The following
+loads need to be executed at retirement and wait for all senior stores on
+the same thread to be drained: load splitting across 4K boundary (page
+split), load accessing uncacheable (UC or USWC) memory, load lock, and load
+with page table in UC or USWC memory region.
+.It Li STORE_BLOCKS.L1D_BLOCK
+.Pq Event 06H , Umask 08H
+Cacheable loads delayed with L1D block code
+.It Li PARTIAL_ADDRESS_ALIAS
+.Pq Event 07H , Umask 01H
+Counts false dependency due to partial address aliasing
+.It Li DTLB_LOAD_MISSES.ANY
+.Pq Event 08H , Umask 01H
+Counts all load misses that cause a page walk
+.It Li DTLB_LOAD_MISSES.WALK_COMPLETED
+.Pq Event 08H , Umask 02H
+Counts number of completed page walks due to load miss in the STLB.
+.It Li DTLB_LOAD_MISSES.STLB_HIT
+.Pq Event 08H , Umask 10H
+Number of cache load STLB hits
+.It Li DTLB_LOAD_MISSES.PDE_MISS
+.Pq Event 08H , Umask 20H
+Number of DTLB cache load misses where the low part of the linear to
+physical address translation was missed.
+.It Li DTLB_LOAD_MISSES.PDP_MISS
+.Pq Event 08H , Umask 40H
+Number of DTLB cache load misses where the high part of the linear to
+physical address translation was missed.
+.It Li DTLB_LOAD_MISSES.LARGE_WALK_COMPLETED
+.Pq Event 08H , Umask 80H
+Counts number of completed large page walks due to load miss in the STLB.
+.It Li MEM_INST_RETIRED.LOADS
+.Pq Event 0BH , Umask 01H
+Counts the number of instructions with an architecturally-visible store
+retired on the architected path.
+In conjunction with ld_lat facility
+.It Li MEM_INST_RETIRED.STORES
+.Pq Event 0BH , Umask 02H
+Counts the number of instructions with an architecturally-visible store
+retired on the architected path.
+In conjunction with ld_lat facility
+.It Li MEM_INST_RETIRED.LATENCY_ABOVE_THRESHOLD
+.Pq Event 0BH , Umask 10H
+Counts the number of instructions exceeding the latency specified with
+ld_lat facility.
+In conjunction with ld_lat facility
+.It Li MEM_STORE_RETIRED.DTLB_MISS
+.Pq Event 0CH , Umask 01H
+The event counts the number of retired stores that missed the DTLB. The DTLB
+miss is not counted if the store operation causes a fault. Does not counter
+prefetches. Counts both primary and secondary misses to the TLB
+.It Li UOPS_ISSUED.ANY
+.Pq Event 0EH , Umask 01H
+Counts the number of Uops issued by the Register Allocation Table to the
+Reservation Station, i.e. the UOPs issued from the front end to the back
+end.
+.It Li UOPS_ISSUED.STALLED_CYCLES
+.Pq Event 0EH , Umask 01H
+Counts the number of cycles no Uops issued by the Register Allocation Table
+to the Reservation Station, i.e. the UOPs issued from the front end to the
+back end.
+set invert=1, cmask = 1
+.It Li UOPS_ISSUED.FUSED
+.Pq Event 0EH , Umask 02H
+Counts the number of fused Uops that were issued from the Register
+Allocation Table to the Reservation Station.
+.It Li MEM_UNCORE_RETIRED.L3_DATA_MISS_UNKNOWN
+.Pq Event 0FH , Umask 01H
+Counts number of memory load instructions retired where the memory reference
+missed L3 and data source is unknown.
+Available only for CPUID signature 06_2EH
+.It Li MEM_UNCORE_RETIRED.OTHER_CORE_L2_HITM
+.Pq Event 0FH , Umask 02H
+Counts number of memory load instructions retired where the memory reference
+hit modified data in a sibling core residing on the same socket.
+.It Li MEM_UNCORE_RETIRED.REMOTE_CACHE_LOCAL_HOME_HIT
+.Pq Event 0FH , Umask 08H
+Counts number of memory load instructions retired where the memory reference
+missed the L1, L2 and L3 caches and HIT in a remote socket's cache. Only
+counts locally homed lines.
+.It Li MEM_UNCORE_RETIRED.REMOTE_DRAM
+.Pq Event 0FH , Umask 10H
+Counts number of memory load instructions retired where the memory reference
+missed the L1, L2 and L3 caches and was remotely homed. This includes both
+DRAM access and HITM in a remote socket's cache for remotely homed lines.
+.It Li MEM_UNCORE_RETIRED.LOCAL_DRAM
+.Pq Event 0FH , Umask 20H
+Counts number of memory load instructions retired where the memory reference
+missed the L1, L2 and L3 caches and required a local socket memory
+reference. This includes locally homed cachelines that were in a modified
+state in another socket.
+.It Li MEM_UNCORE_RETIRED.UNCACHEABLE
+.Pq Event 0FH , Umask 80H
+Counts number of memory load instructions retired where the memory reference
+missed the L1, L2 and L3 caches and to perform I/O.
+Available only for CPUID signature 06_2EH
+.It Li FP_COMP_OPS_EXE.X87
+.Pq Event 10H , Umask 01H
+Counts the number of FP Computational Uops Executed. The number of FADD,
+FSUB, FCOM, FMULs, integer MULsand IMULs, FDIVs, FPREMs, FSQRTS, integer
+DIVs, and IDIVs. This event does not distinguish an FADD used in the middle
+of a transcendental flow from a separate FADD instruction.
+.It Li FP_COMP_OPS_EXE.MMX
+.Pq Event 10H , Umask 02H
+Counts number of MMX Uops executed.
+.It Li FP_COMP_OPS_EXE.SSE_FP
+.Pq Event 10H , Umask 04H
+Counts number of SSE and SSE2 FP uops executed.
+.It Li FP_COMP_OPS_EXE.SSE2_INTEGER
+.Pq Event 10H , Umask 08H
+Counts number of SSE2 integer uops executed.
+.It Li FP_COMP_OPS_EXE.SSE_FP_PACKED
+.Pq Event 10H , Umask 10H
+Counts number of SSE FP packed uops executed.
+.It Li FP_COMP_OPS_EXE.SSE_FP_SCALAR
+.Pq Event 10H , Umask 20H
+Counts number of SSE FP scalar uops executed.
+.It Li FP_COMP_OPS_EXE.SSE_SINGLE_PRECISION
+.Pq Event 10H , Umask 40H
+Counts number of SSE* FP single precision uops executed.
+.It Li FP_COMP_OPS_EXE.SSE_DOUBLE_PRECISION
+.Pq Event 10H , Umask 80H
+Counts number of SSE* FP double precision uops executed.
+.It Li SIMD_INT_128.PACKED_MPY
+.Pq Event 12H , Umask 01H
+Counts number of 128 bit SIMD integer multiply operations.
+.It Li SIMD_INT_128.PACKED_SHIFT
+.Pq Event 12H , Umask 02H
+Counts number of 128 bit SIMD integer shift operations.
+.It Li SIMD_INT_128.PACK
+.Pq Event 12H , Umask 04H
+Counts number of 128 bit SIMD integer pack operations.
+.It Li SIMD_INT_128.UNPACK
+.Pq Event 12H , Umask 08H
+Counts number of 128 bit SIMD integer unpack operations.
+.It Li SIMD_INT_128.PACKED_LOGICAL
+.Pq Event 12H , Umask 10H
+Counts number of 128 bit SIMD integer logical operations.
+.It Li SIMD_INT_128.PACKED_ARITH
+.Pq Event 12H , Umask 20H
+Counts number of 128 bit SIMD integer arithmetic operations.
+.It Li SIMD_INT_128.SHUFFLE_MOVE
+.Pq Event 12H , Umask 40H
+Counts number of 128 bit SIMD integer shuffle and move operations.
+.It Li LOAD_DISPATCH.RS
+.Pq Event 13H , Umask 01H
+Counts number of loads dispatched from the Reservation Station that bypass
+the Memory Order Buffer.
+.It Li LOAD_DISPATCH.RS_DELAYED
+.Pq Event 13H , Umask 02H
+Counts the number of delayed RS dispatches at the stage latch. If an RS
+dispatch can not bypass to LB, it has another chance to dispatch from the
+one-cycle delayed staging latch before it is written into the LB.
+.It Li LOAD_DISPATCH.MOB
+.Pq Event 13H , Umask 04H
+Counts the number of loads dispatched from the Reservation Station to the
+Memory Order Buffer.
+.It Li LOAD_DISPATCH.ANY
+.Pq Event 13H , Umask 07H
+Counts all loads dispatched from the Reservation Station.
+.It Li ARITH.CYCLES_DIV_BUSY
+.Pq Event 14H , Umask 01H
+Counts the number of cycles the divider is busy executing divide or square
+root operations. The divide can be integer, X87 or Streaming SIMD Extensions
+(SSE). The square root operation can be either X87 or SSE.
+Set 'edge =1, invert=1, cmask=1' to count the number of divides.
+Count may be incorrect When SMT is on.
+.It Li ARITH.MUL
+.Pq Event 14H , Umask 02H
+Counts the number of multiply operations executed. This includes integer as
+well as floating point multiply operations but excludes DPPS mul and MPSAD.
+Count may be incorrect When SMT is on
+.It Li INST_QUEUE_WRITES
+.Pq Event 17H , Umask 01H
+Counts the number of instructions written into the instruction queue every
+cycle.
+.It Li INST_DECODED.DEC0
+.Pq Event 18H , Umask 01H
+Counts number of instructions that require decoder 0 to be decoded. Usually,
+this means that the instruction maps to more than 1 uop
+.It Li TWO_UOP_INSTS_DECODED
+.Pq Event 19H , Umask 01H
+An instruction that generates two uops was decoded
+.It Li INST_QUEUE_WRITE_CYCLES
+.Pq Event 1EH , Umask 01H
+This event counts the number of cycles during which instructions are written
+to the instruction queue. Dividing this counter by the number of
+instructions written to the instruction queue (INST_QUEUE_WRITES) yields the
+average number of instructions decoded each cycle. If this number is less
+than four and the pipe stalls, this indicates that the decoder is failing to
+decode enough instructions per cycle to sustain the 4-wide pipeline.
+If SSE* instructions that are 6 bytes or longer arrive one after another,
+then front end throughput may limit execution speed. In such case,
+.It Li LSD_OVERFLOW
+.Pq Event 20H , Umask 01H
+Counts number of loops that cant stream from the instruction queue.
+.It Li L2_RQSTS.LD_HIT
+.Pq Event 24H , Umask 01H
+Counts number of loads that hit the L2 cache. L2 loads include both L1D
+demand misses as well as L1D prefetches. L2 loads can be rejected for
+various reasons. Only non rejected loads are counted.
+.It Li L2_RQSTS.LD_MISS
+.Pq Event 24H , Umask 02H
+Counts the number of loads that miss the L2 cache. L2 loads include both L1D
+demand misses as well as L1D prefetches.
+.It Li L2_RQSTS.LOADS
+.Pq Event 24H , Umask 03H
+Counts all L2 load requests. L2 loads include both L1D demand misses as well
+as L1D prefetches.
+.It Li L2_RQSTS.RFO_HIT
+.Pq Event 24H , Umask 04H
+Counts the number of store RFO requests that hit the L2 cache. L2 RFO
+requests include both L1D demand RFO misses as well as L1D RFO prefetches.
+Count includes WC memory requests, where the data is not fetched but the
+permission to write the line is required.
+.It Li L2_RQSTS.RFO_MISS
+.Pq Event 24H , Umask 08H
+Counts the number of store RFO requests that miss the L2 cache. L2 RFO
+requests include both L1D demand RFO misses as well as L1D RFO prefetches.
+.It Li L2_RQSTS.RFOS
+.Pq Event 24H , Umask 0CH
+Counts all L2 store RFO requests. L2 RFO requests include both L1D demand
+RFO misses as well as L1D RFO prefetches.
+.It Li L2_RQSTS.IFETCH_HIT
+.Pq Event 24H , Umask 10H
+Counts number of instruction fetches that hit the L2 cache. L2 instruction
+fetches include both L1I demand misses as well as L1I instruction
+prefetches.
+.It Li L2_RQSTS.IFETCH_MISS
+.Pq Event 24H , Umask 20H
+Counts number of instruction fetches that miss the L2 cache. L2 instruction
+fetches include both L1I demand misses as well as L1I instruction
+prefetches.
+.It Li L2_RQSTS.IFETCHES
+.Pq Event 24H , Umask 30H
+Counts all instruction fetches. L2 instruction fetches include both L1I
+demand misses as well as L1I instruction prefetches.
+.It Li L2_RQSTS.PREFETCH_HIT
+.Pq Event 24H , Umask 40H
+Counts L2 prefetch hits for both code and data.
+.It Li L2_RQSTS.PREFETCH_MISS
+.Pq Event 24H , Umask 80H
+Counts L2 prefetch misses for both code and data.
+.It Li L2_RQSTS.PREFETCHES
+.Pq Event 24H , Umask C0H
+Counts all L2 prefetches for both code and data.
+.It Li L2_RQSTS.MISS
+.Pq Event 24H , Umask AAH
+Counts all L2 misses for both code and data.
+.It Li L2_RQSTS.REFERENCES
+.Pq Event 24H , Umask FFH
+Counts all L2 requests for both code and data.
+.It Li L2_DATA_RQSTS.DEMAND.I_STATE
+.Pq Event 26H , Umask 01H
+Counts number of L2 data demand loads where the cache line to be loaded is
+in the I (invalid) state, i.e. a cache miss. L2 demand loads are both L1D
+demand misses and L1D prefetches.
+.It Li L2_DATA_RQSTS.DEMAND.S_STATE
+.Pq Event 26H , Umask 02H
+Counts number of L2 data demand loads where the cache line to be loaded is
+in the S (shared) state. L2 demand loads are both L1D demand misses and L1D
+prefetches.
+.It Li L2_DATA_RQSTS.DEMAND.E_STATE
+.Pq Event 26H , Umask 04H
+Counts number of L2 data demand loads where the cache line to be loaded is
+in the E (exclusive) state. L2 demand loads are both L1D demand misses and
+L1D prefetches.
+.It Li L2_DATA_RQSTS.DEMAND.M_STATE
+.Pq Event 26H , Umask 08H
+Counts number of L2 data demand loads where the cache line to be loaded is
+in the M (modified) state. L2 demand loads are both L1D demand misses and
+L1D prefetches.
+.It Li L2_DATA_RQSTS.DEMAND.MESI
+.Pq Event 26H , Umask 0FH
+Counts all L2 data demand requests. L2 demand loads are both L1D demand
+misses and L1D prefetches.
+.It Li L2_DATA_RQSTS.PREFETCH.I_STATE
+.Pq Event 26H , Umask 10H
+Counts number of L2 prefetch data loads where the cache line to be loaded is
+in the I (invalid) state, i.e. a cache miss.
+.It Li L2_DATA_RQSTS.PREFETCH.S_STATE
+.Pq Event 26H , Umask 20H
+Counts number of L2 prefetch data loads where the cache line to be loaded is
+in the S (shared) state. A prefetch RFO will miss on an S state line, while
+a prefetch read will hit on an S state line.
+.It Li L2_DATA_RQSTS.PREFETCH.E_STATE
+.Pq Event 26H , Umask 40H
+Counts number of L2 prefetch data loads where the cache line to be loaded is
+in the E (exclusive) state.
+.It Li L2_DATA_RQSTS.PREFETCH.M_STATE
+.Pq Event 26H , Umask 80H
+Counts number of L2 prefetch data loads where the cache line to be loaded is
+in the M (modified) state.
+.It Li L2_DATA_RQSTS.PREFETCH.MESI
+.Pq Event 26H , Umask F0H
+Counts all L2 prefetch requests.
+.It Li L2_DATA_RQSTS.ANY
+.Pq Event 26H , Umask FFH
+Counts all L2 data requests.
+.It Li L2_WRITE.RFO.I_STATE
+.Pq Event 27H , Umask 01H
+Counts number of L2 demand store RFO requests where the cache line to be
+loaded is in the I (invalid) state, i.e, a cache miss. The L1D prefetcher
+does not issue a RFO prefetch.
+This is a demand RFO request
+.It Li L2_WRITE.RFO.S_STATE
+.Pq Event 27H , Umask 02H
+Counts number of L2 store RFO requests where the cache line to be loaded is
+in the S (shared) state. The L1D prefetcher does not issue a RFO prefetch,.
+This is a demand RFO request
+.It Li L2_WRITE.RFO.M_STATE
+.Pq Event 27H , Umask 08H
+Counts number of L2 store RFO requests where the cache line to be loaded is
+in the M (modified) state. The L1D prefetcher does not issue a RFO prefetch.
+This is a demand RFO request
+.It Li L2_WRITE.RFO.HIT
+.Pq Event 27H , Umask 0EH
+Counts number of L2 store RFO requests where the cache line to be loaded is
+in either the S, E or M states. The L1D prefetcher does not issue a RFO
+prefetch.
+This is a demand RFO request
+.It Li L2_WRITE.RFO.MESI
+.Pq Event 27H , Umask 0FH
+Counts all L2 store RFO requests.The L1D prefetcher does not issue a RFO
+prefetch.
+This is a demand RFO request
+.It Li L2_WRITE.LOCK.I_STATE
+.Pq Event 27H , Umask 10H
+Counts number of L2 demand lock RFO requests where the cache line to be
+loaded is in the I (invalid) state, i.e. a cache miss.
+.It Li L2_WRITE.LOCK.S_STATE
+.Pq Event 27H , Umask 20H
+Counts number of L2 lock RFO requests where the cache line to be loaded is
+in the S (shared) state.
+.It Li L2_WRITE.LOCK.E_STATE
+.Pq Event 27H , Umask 40H
+Counts number of L2 demand lock RFO requests where the cache line to be
+loaded is in the E (exclusive) state.
+.It Li L2_WRITE.LOCK.M_STATE
+.Pq Event 27H , Umask 80H
+Counts number of L2 demand lock RFO requests where the cache line to be
+loaded is in the M (modified) state.
+.It Li L2_WRITE.LOCK.HIT
+.Pq Event 27H , Umask E0H
+Counts number of L2 demand lock RFO requests where the cache line to be
+loaded is in either the S, E, or M state.
+.It Li L2_WRITE.LOCK.MESI
+.Pq Event 27H , Umask F0H
+Counts all L2 demand lock RFO requests.
+.It Li L1D_WB_L2.I_STATE
+.Pq Event 28H , Umask 01H
+Counts number of L1 writebacks to the L2 where the cache line to be written
+is in the I (invalid) state, i.e. a cache miss.
+.It Li L1D_WB_L2.S_STATE
+.Pq Event 28H , Umask 02H
+Counts number of L1 writebacks to the L2 where the cache line to be written
+is in the S state.
+.It Li L1D_WB_L2.E_STATE
+.Pq Event 28H , Umask 04H
+Counts number of L1 writebacks to the L2 where the cache line to be written
+is in the E (exclusive) state.
+.It Li L1D_WB_L2.M_STATE
+.Pq Event 28H , Umask 08H
+Counts number of L1 writebacks to the L2 where the cache line to be written
+is in the M (modified) state.
+.It Li L1D_WB_L2.MESI
+.Pq Event 28H , Umask 0FH
+Counts all L1 writebacks to the L2.
+.It Li L3_LAT_CACHE.REFERENCE
+.Pq Event 2EH , Umask 4FH
+This event counts requests originating from the core that reference a cache
+line in the last level cache. The event count includes speculative traffic
+but excludes cache line fills due to a L2 hardware-prefetch. Because cache
+hierarchy, cache sizes and other implementation-specific characteristics;
+value comparison to estimate performance differences is not recommended.
+see Table A-1
+.It Li L3_LAT_CACHE.MISS
+.Pq Event 2EH , Umask 41H
+This event counts each cache miss condition for references to the last level
+cache. The event count may include speculative traffic but excludes cache
+line fills due to L2 hardware-prefetches. Because cache hierarchy, cache
+sizes and other implementation-specific characteristics; value comparison to
+estimate performance differences is not recommended.
+see Table A-1
+.It Li CPU_CLK_UNHALTED.THREAD_P
+.Pq Event 3CH , Umask 00H
+Counts the number of thread cycles while the thread is not in a halt state.
+The thread enters the halt state when it is running the HLT instruction. The
+core frequency may change from time to time due to power or thermal
+throttling.
+see Table A-1
+.It Li CPU_CLK_UNHALTED.REF_P
+.Pq Event 3CH , Umask 01H
+Increments at the frequency of TSC when not halted.
+see Table A-1
+.It Li L1D_CACHE_LD.I_STATE
+.Pq Event 40H , Umask 01H
+Counts L1 data cache read requests where the cache line to be loaded is in
+the I (invalid) state, i.e. the read request missed the cache.
+Counter 0, 1 only
+.It Li L1D_CACHE_LD.S_STATE
+.Pq Event 40H , Umask 02H
+Counts L1 data cache read requests where the cache line to be loaded is in
+the S (shared) state.
+Counter 0, 1 only
+.It Li L1D_CACHE_LD.E_STATE
+.Pq Event 40H , Umask 04H
+Counts L1 data cache read requests where the cache line to be loaded is in
+the E (exclusive) state.
+Counter 0, 1 only
+.It Li L1D_CACHE_LD.M_STATE
+.Pq Event 40H , Umask 08H
+Counts L1 data cache read requests where the cache line to be loaded is in
+the M (modified) state.
+Counter 0, 1 only
+.It Li L1D_CACHE_LD.MESI
+.Pq Event 40H , Umask 0FH
+Counts L1 data cache read requests.
+Counter 0, 1 only
+.It Li L1D_CACHE_ST.S_STATE
+.Pq Event 41H , Umask 02H
+Counts L1 data cache store RFO requests where the cache line to be loaded is
+in the S (shared) state.
+Counter 0, 1 only
+.It Li L1D_CACHE_ST.E_STATE
+.Pq Event 41H , Umask 04H
+Counts L1 data cache store RFO requests where the cache line to be loaded is
+in the E (exclusive) state.
+Counter 0, 1 only
+.It Li L1D_CACHE_ST.M_STATE
+.Pq Event 41H , Umask 08H
+Counts L1 data cache store RFO requests where cache line to be loaded is in
+the M (modified) state.
+Counter 0, 1 only
+.It Li L1D_CACHE_LOCK.HIT
+.Pq Event 42H , Umask 01H
+Counts retired load locks that hit in the L1 data cache or hit in an already
+allocated fill buffer.	The lock portion of the load lock transaction must
+hit in the L1D.
+The initial load will pull the lock into the L1 data cache. Counter 0, 1
+only
+.It Li L1D_CACHE_LOCK.S_STATE
+.Pq Event 42H , Umask 02H
+Counts L1 data cache retired load locks that hit the target cache line in
+the shared state.
+Counter 0, 1 only
+.It Li L1D_CACHE_LOCK.E_STATE
+.Pq Event 42H , Umask 04H
+Counts L1 data cache retired load locks that hit the target cache line in
+the exclusive state.
+Counter 0, 1 only
+.It Li L1D_CACHE_LOCK.M_STATE
+.Pq Event 42H , Umask 08H
+Counts L1 data cache retired load locks that hit the target cache line in
+the modified state.
+Counter 0, 1 only
+.It Li L1D_ALL_REF.ANY
+.Pq Event 43H , Umask 01H
+Counts all references (uncached, speculated and retired) to the L1 data
+cache, including all loads and stores with any memory types. The event
+counts memory accesses only when they are actually performed. For example, a
+load blocked by unknown store address and later performed is only counted
+once.
+The event does not include non- memory accesses, such as I/O accesses.
+Counter 0, 1 only
+.It Li L1D_ALL_REF.CACHEABLE
+.Pq Event 43H , Umask 02H
+Counts all data reads and writes (speculated and retired) from cacheable
+memory, including locked operations.
+Counter 0, 1 only
+.It Li L1D_PEND_MISS.LOAD_BUFFERS_FULL
+.Pq Event 48H , Umask 02H
+Counts cycles of L1 data cache load fill buffers full.
+Counter 0, 1 only
+.It Li DTLB_MISSES.ANY
+.Pq Event 49H , Umask 01H
+Counts the number of misses in the STLB which causes a page walk.
+.It Li DTLB_MISSES.WALK_COMPLETED
+.Pq Event 49H , Umask 02H
+Counts number of misses in the STLB which resulted in a completed page walk.
+.It Li DTLB_MISSES.STLB_HIT
+.Pq Event 49H , Umask 10H
+Counts the number of DTLB first level misses that hit in the second level
+TLB. This event is only relevant if the core contains multiple DTLB levels.
+.It Li LOAD_HIT_PRE
+.Pq Event 4CH , Umask 01H
+Counts load operations sent to the L1 data cache while a previous SSE
+prefetch instruction to the same cache line has started prefetching but has
+not yet finished.
+.It Li L1D_PREFETCH.REQUESTS
+.Pq Event 4EH , Umask 01H
+Counts number of hardware prefetch requests dispatched out of the prefetch
+FIFO.
+.It Li L1D_PREFETCH.MISS
+.Pq Event 4EH , Umask 02H
+Counts number of hardware prefetch requests that miss the L1D. There are two
+prefetchers in the L1D. A streamer, which predicts lines sequentially after
+this one should be fetched, and the IP prefetcher that remembers access
+patterns for the current instruction. The streamer prefetcher stops on an
+L1D hit, while the IP prefetcher does not.
+.It Li L1D_PREFETCH.TRIGGERS
+.Pq Event 4EH , Umask 04H
+Counts number of prefetch requests triggered by the Finite State Machine and
+pushed into the prefetch FIFO. Some of the prefetch requests are dropped due
+to overwrites or competition between the IP index prefetcher and streamer
+prefetcher. The prefetch FIFO contains 4 entries.
+.It Li L1D.REPL
+.Pq Event 51H , Umask 01H
+Counts the number of lines brought into the L1 data cache.
+Counter 0, 1 only
+.It Li L1D.M_REPL
+.Pq Event 51H , Umask 02H
+Counts the number of modified lines brought into the L1 data cache.
+Counter 0, 1 only
+.It Li L1D.M_EVICT
+.Pq Event 51H , Umask 04H
+Counts the number of modified lines evicted from the L1 data cache due to
+replacement.
+Counter 0, 1 only
+.It Li L1D.M_SNOOP_EVICT
+.Pq Event 51H , Umask 08H
+Counts the number of modified lines evicted from the L1 data cache due to
+snoop HITM intervention.
+Counter 0, 1 only
+.It Li L1D_CACHE_PREFETCH_LOCK_FB_HIT
+.Pq Event 52H , Umask 01H
+Counts the number of cacheable load lock speculated instructions accepted
+into the fill buffer.
+.It Li L1D_CACHE_LOCK_FB_HIT
+.Pq Event 53H , Umask 01H
+Counts the number of cacheable load lock speculated or retired instructions
+accepted into the fill buffer.
+.It Li CACHE_LOCK_CYCLES.L1D_L2
+.Pq Event 63H , Umask 01H
+Cycle count during which the L1D and L2 are locked. A lock is asserted when
+there is a locked memory access, due to uncacheable memory, a locked
+operation that spans two cache lines, or a page walk from an uncacheable
+page table.
+Counter 0, 1 only. L1D and L2 locks have a very high performance penalty and
+it is highly recommended to avoid such accesses.
+.It Li CACHE_LOCK_CYCLES.L1D
+.Pq Event 63H , Umask 02H
+Counts the number of cycles that cacheline in the L1 data cache unit is
+locked.
+Counter 0, 1 only.
+.It Li IO_TRANSACTIONS
+.Pq Event 6CH , Umask 01H
+Counts the number of completed I/O transactions.
+.It Li L1I.HITS
+.Pq Event 80H , Umask 01H
+Counts all instruction fetches that hit the L1 instruction cache.
+.It Li L1I.MISSES
+.Pq Event 80H , Umask 02H
+Counts all instruction fetches that miss the L1I cache. This includes
+instruction cache misses, streaming buffer misses, victim cache misses and
+uncacheable fetches. An instruction fetch miss is counted only once and not
+once for every cycle it is outstanding.
+.It Li L1I.READS
+.Pq Event 80H , Umask 03H
+Counts all instruction fetches, including uncacheable fetches that bypass
+the L1I.
+.It Li L1I.CYCLES_STALLED
+.Pq Event 80H , Umask 04H
+Cycle counts for which an instruction fetch stalls due to a L1I cache miss,
+ITLB miss or ITLB fault.
+.It Li LARGE_ITLB.HIT
+.Pq Event 82H , Umask 01H
+Counts number of large ITLB hits.
+.It Li ITLB_MISSES.ANY
+.Pq Event 85H , Umask 01H
+Counts the number of misses in all levels of the ITLB which causes a page
+walk.
+.It Li ITLB_MISSES.WALK_COMPLETED
+.Pq Event 85H , Umask 02H
+Counts number of misses in all levels of the ITLB which resulted in a
+completed page walk.
+.It Li ILD_STALL.LCP
+.Pq Event 87H , Umask 01H
+Cycles Instruction Length Decoder stalls due to length changing prefixes:
+66, 67 or REX.W (for EM64T) instructions which change the length of the
+decoded instruction.
+.It Li ILD_STALL.MRU
+.Pq Event 87H , Umask 02H
+Instruction Length Decoder stall cycles due to Brand Prediction Unit (PBU)
+Most Recently Used (MRU) bypass.
+.It Li ILD_STALL.IQ_FULL
+.Pq Event 87H , Umask 04H
+Stall cycles due to a full instruction queue.
+.It Li ILD_STALL.REGEN
+.Pq Event 87H , Umask 08H
+Counts the number of regen stalls.
+.It Li ILD_STALL.ANY
+.Pq Event 87H , Umask 0FH
+Counts any cycles the Instruction Length Decoder is stalled.
+.It Li BR_INST_EXEC.COND
+.Pq Event 88H , Umask 01H
+Counts the number of conditional near branch instructions executed, but not
+necessarily retired.
+.It Li BR_INST_EXEC.DIRECT
+.Pq Event 88H , Umask 02H
+Counts all unconditional near branch instructions excluding calls and
+indirect branches.
+.It Li BR_INST_EXEC.INDIRECT_NON_CALL
+.Pq Event 88H , Umask 04H
+Counts the number of executed indirect near branch instructions that are not
+calls.
+.It Li BR_INST_EXEC.NON_CALLS
+.Pq Event 88H , Umask 07H
+Counts all non call near branch instructions executed, but not necessarily
+retired.
+.It Li BR_INST_EXEC.RETURN_NEAR
+.Pq Event 88H , Umask 08H
+Counts indirect near branches that have a return mnemonic.
+.It Li BR_INST_EXEC.DIRECT_NEAR_CALL
+.Pq Event 88H , Umask 10H
+Counts unconditional near call branch instructions, excluding non call
+branch, executed.
+.It Li BR_INST_EXEC.INDIRECT_NEAR_CALL
+.Pq Event 88H , Umask 20H
+Counts indirect near calls, including both register and memory indirect,
+executed.
+.It Li BR_INST_EXEC.NEAR_CALLS
+.Pq Event 88H , Umask 30H
+Counts all near call branches executed, but not necessarily retired.
+.It Li BR_INST_EXEC.TAKEN
+.Pq Event 88H , Umask 40H
+Counts taken near branches executed, but not necessarily retired.
+.It Li BR_INST_EXEC.ANY
+.Pq Event 88H , Umask 7FH
+Counts all near executed branches (not necessarily retired). This includes
+only instructions and not micro-op branches. Frequent branching is not
+necessarily a major performance issue. However frequent branch
+mispredictions may be a problem.
+.It Li BR_MISP_EXEC.COND
+.Pq Event 89H , Umask 01H
+Counts the number of mispredicted conditional near branch instructions
+executed, but not necessarily retired.
+.It Li BR_MISP_EXEC.DIRECT
+.Pq Event 89H , Umask 02H
+Counts mispredicted macro unconditional near branch instructions, excluding
+calls and indirect branches (should always be 0).
+.It Li BR_MISP_EXEC.INDIRECT_NON_CALL
+.Pq Event 89H , Umask 04H
+Counts the number of executed mispredicted indirect near branch instructions
+that are not calls.
+.It Li BR_MISP_EXEC.NON_CALLS
+.Pq Event 89H , Umask 07H
+Counts mispredicted non call near branches executed, but not necessarily
+retired.
+.It Li BR_MISP_EXEC.RETURN_NEAR
+.Pq Event 89H , Umask 08H
+Counts mispredicted indirect branches that have a rear return mnemonic.
+.It Li BR_MISP_EXEC.DIRECT_NEAR_CALL
+.Pq Event 89H , Umask 10H
+Counts mispredicted non-indirect near calls executed, (should always be 0).
+.It Li BR_MISP_EXEC.INDIRECT_NEAR_CALL
+.Pq Event 89H , Umask 20H
+Counts mispredicted indirect near calls exeucted, including both register
+and memory indirect.
+.It Li BR_MISP_EXEC.NEAR_CALLS
+.Pq Event 89H , Umask 30H
+Counts all mispredicted near call branches executed, but not necessarily
+retired.
+.It Li BR_MISP_EXEC.TAKEN
+.Pq Event 89H , Umask 40H
+Counts executed mispredicted near branches that are taken, but not
+necessarily retired.
+.It Li BR_MISP_EXEC.ANY
+.Pq Event 89H , Umask 7FH
+Counts the number of mispredicted near branch instructions that were
+executed, but not necessarily retired.
+.It Li RESOURCE_STALLS.ANY
+.Pq Event A2H , Umask 01H
+Counts the number of Allocator resource related stalls. Includes register
+renaming buffer entries, memory buffer entries. In addition to resource
+related stalls, this event counts some other events. Includes stalls arising
+during branch misprediction recovery, such as if retirement of the
+mispredicted branch is delayed and stalls arising while store buffer is
+draining from synchronizing operations.
+Does not include stalls due to SuperQ (off core) queue full, too many cache
+misses, etc.
+.It Li RESOURCE_STALLS.LOAD
+.Pq Event A2H , Umask 02H
+Counts the cycles of stall due to lack of load buffer for load operation.
+.It Li RESOURCE_STALLS.RS_FULL
+.Pq Event A2H , Umask 04H
+This event counts the number of cycles when the number of instructions in
+the pipeline waiting for execution reaches the limit the processor can
+handle. A high count of this event indicates that there are long latency
+operations in the pipe (possibly load and store operations that miss the L2
+cache, or instructions dependent upon instructions further down the pipeline
+that have yet to retire.
+When RS is full, new instructions can not enter the reservation station and
+start execution.
+.It Li RESOURCE_STALLS.STORE
+.Pq Event A2H , Umask 08H
+This event counts the number of cycles that a resource related stall will
+occur due to the number of store instructions reaching the limit of the
+pipeline, (i.e. all store buffers are used). The stall ends when a store
+instruction commits its data to the cache or memory.
+.It Li RESOURCE_STALLS.ROB_FULL
+.Pq Event A2H , Umask 10H
+Counts the cycles of stall due to re- order buffer full.
+.It Li RESOURCE_STALLS.FPCW
+.Pq Event A2H , Umask 20H
+Counts the number of cycles while execution was stalled due to writing the
+floating-point unit (FPU) control word.
+.It Li RESOURCE_STALLS.MXCSR
+.Pq Event A2H , Umask 40H
+Stalls due to the MXCSR register rename occurring to close to a previous
+MXCSR rename. The MXCSR provides control and status for the MMX registers.
+.It Li RESOURCE_STALLS.OTHER
+.Pq Event A2H , Umask 80H
+Counts the number of cycles while execution was stalled due to other
+resource issues.
+.It Li MACRO_INSTS.FUSIONS_DECODED
+.Pq Event A6H , Umask 01H
+Counts the number of instructions decoded that are macro-fused but not
+necessarily executed or retired.
+.It Li BACLEAR_FORCE_IQ
+.Pq Event A7H , Umask 01H
+Counts number of times a BACLEAR was forced by the Instruction Queue. The IQ
+is also responsible for providing conditional branch prediciton direction
+based on a static scheme and dynamic data provided by the L2 Branch
+Prediction Unit. If the conditional branch target is not found in the Target
+Array and the IQ predicts that the branch is taken, then the IQ will force
+the Branch Address Calculator to issue a BACLEAR. Each BACLEAR asserted by
+the BAC generates approximately an 8 cycle bubble in the instruction fetch
+pipeline.
+.It Li LSD.UOPS
+.Pq Event A8H , Umask 01H
+Counts the number of micro-ops delivered by loop stream detector
+Use cmask=1 and invert to count cycles
+.It Li ITLB_FLUSH
+.Pq Event AEH , Umask 01H
+Counts the number of ITLB flushes
+.It Li OFFCORE_REQUESTS.L1D_WRITEBACK
+.Pq Event B0H , Umask 40H
+Counts number of L1D writebacks to the uncore.
+.It Li UOPS_EXECUTED.PORT0
+.Pq Event B1H , Umask 01H
+Counts number of Uops executed that were issued on port 0. Port 0 handles
+integer arithmetic, SIMD and FP add Uops.
+.It Li UOPS_EXECUTED.PORT1
+.Pq Event B1H , Umask 02H
+Counts number of Uops executed that were issued on port 1. Port 1 handles
+integer arithmetic, SIMD, integer shift, FP multiply and FP divide Uops.
+.It Li UOPS_EXECUTED.PORT2_CORE
+.Pq Event B1H , Umask 04H
+Counts number of Uops executed that were issued on port 2. Port 2 handles
+the load Uops. This is a core count only and can not be collected per
+thread.
+.It Li UOPS_EXECUTED.PORT3_CORE
+.Pq Event B1H , Umask 08H
+Counts number of Uops executed that were issued on port 3. Port 3 handles
+store Uops. This is a core count only and can not be collected per thread.
+.It Li UOPS_EXECUTED.PORT4_CORE
+.Pq Event B1H , Umask 10H
+Counts number of Uops executed that where issued on port 4. Port 4 handles
+the value to be stored for the store Uops issued on port 3. This is a core
+count only and can not be collected per thread.
+.It Li UOPS_EXECUTED.CORE_ACTIVE_CYCLES_NO_PORT5
+.Pq Event B1H , Umask 1FH
+Counts cycles when the Uops executed were issued from any ports except port
+5. Use Cmask=1 for active cycles; Cmask=0 for weighted cycles; Use CMask=1,
+Invert=1 to count P0-4 stalled cycles Use Cmask=1, Edge=1, Invert=1 to count
+P0-4 stalls.
+.It Li UOPS_EXECUTED.PORT5
+.Pq Event B1H , Umask 20H
+Counts number of Uops executed that where issued on port 5.
+.It Li UOPS_EXECUTED.CORE_ACTIVE_CYCLES
+.Pq Event B1H , Umask 3FH
+Counts cycles when the Uops are executing. Use Cmask=1 for active cycles;
+Cmask=0 for weighted cycles; Use CMask=1, Invert=1 to count P0-4 stalled
+cycles Use Cmask=1, Edge=1, Invert=1 to count P0-4 stalls.
+.It Li UOPS_EXECUTED.PORT015
+.Pq Event B1H , Umask 40H
+Counts number of Uops executed that where issued on port 0, 1, or 5.
+use cmask=1, invert=1 to count stall cycles
+.It Li UOPS_EXECUTED.PORT234
+.Pq Event B1H , Umask 80H
+Counts number of Uops executed that where issued on port 2, 3, or 4.
+.It Li OFFCORE_REQUESTS_SQ_FULL
+.Pq Event B2H , Umask 01H
+Counts number of cycles the SQ is full to handle off-core requests.
+.It Li OFF_CORE_RESPONSE_0
+.Pq Event B7H , Umask 01H
+see Section 30.6.1.3, Off-core Response Performance Monitoring in the
+Processor Core
+Requires programming MSR 01A6H
+.It Li SNOOP_RESPONSE.HIT
+.Pq Event B8H , Umask 01H
+Counts HIT snoop response sent by this thread in response to a snoop
+request.
+.It Li SNOOP_RESPONSE.HITE
+.Pq Event B8H , Umask 02H
+Counts HIT E snoop response sent by this thread in response to a snoop
+request.
+.It Li SNOOP_RESPONSE.HITM
+.Pq Event B8H , Umask 04H
+Counts HIT M snoop response sent by this thread in response to a snoop
+request.
+.It Li OFF_CORE_RESPONSE_1
+.Pq Event BBH , Umask 01H
+see Section 30.6.1.3, Off-core Response Performance Monitoring in the
+Processor Core
+Requires programming MSR 01A7H
+.It Li INST_RETIRED.ANY_P
+.Pq Event C0H , Umask 01H
+See Table A-1
+Notes: INST_RETIRED.ANY is counted by a designated fixed counter.
+INST_RETIRED.ANY_P is counted by a programmable counter and is an
+architectural performance event. Event is supported if CPUID.A.EBX[1] = 0.
+Counting: Faulting executions of GETSEC/VM entry/VM Exit/MWait will not
+count as retired instructions.
+.It Li INST_RETIRED.X87
+.Pq Event C0H , Umask 02H
+Counts the number of MMX instructions retired:.
+.It Li INST_RETIRED.MMX
+.Pq Event C0H , Umask 04H
+Counts the number of floating point computational operations retired:
+floating point computational operations executed by the assist handler and
+sub-operations of complex floating point instructions like transcendental
+instructions.
+.It Li UOPS_RETIRED.ANY
+.Pq Event C2H , Umask 01H
+Counts the number of micro-ops retired, (macro-fused=1, micro- fused=2,
+others=1; maximum count of 8 per cycle). Most instructions are composed of
+one or two micro-ops. Some instructions are decoded into longer sequences
+such as repeat instructions, floating point transcendental instructions, and
+assists.
+Use cmask=1 and invert to count active cycles or stalled cycles
+.It Li UOPS_RETIRED.RETIRE_SLOTS
+.Pq Event C2H , Umask 02H
+Counts the number of retirement slots used each cycle
+.It Li UOPS_RETIRED.MACRO_FUSED
+.Pq Event C2H , Umask 04H
+Counts number of macro-fused uops retired.
+.It Li MACHINE_CLEARS.CYCLES
+.Pq Event C3H , Umask 01H
+Counts the cycles machine clear is asserted.
+.It Li MACHINE_CLEARS.MEM_ORDER
+.Pq Event C3H , Umask 02H
+Counts the number of machine clears due to memory order conflicts.
+.It Li MACHINE_CLEARS.SMC
+.Pq Event C3H , Umask 04H
+Counts the number of times that a program writes to a code section.
+Self-modifying code causes a sever penalty in all Intel 64 and IA-32
+processors. The modified cache line is written back to the L2 and L3caches.
+.It Li BR_INST_RETIRED.ALL_BRANCHES
+.Pq Event C4H , Umask 00H
+See Table A-1
+.It Li BR_INST_RETIRED.CONDITIONAL
+.Pq Event C4H , Umask 01H
+Counts the number of conditional branch instructions retired.
+.It Li BR_INST_RETIRED.NEAR_CALL
+.Pq Event C4H , Umask 02H
+Counts the number of direct & indirect near unconditional calls retired
+.It Li BR_INST_RETIRED.ALL_BRANCHES
+.Pq Event C4H , Umask 04H
+Counts the number of branch instructions retired
+.It Li BR_MISP_RETIRED.ALL_BRANCHES
+.Pq Event C5H , Umask 00H
+See Table A-1
+.It Li BR_MISP_RETIRED.NEAR_CALL
+.Pq Event C5H , Umask 02H
+Counts mispredicted direct & indirect near unconditional retired calls.
+.It Li SSEX_UOPS_RETIRED.PACKED_SINGLE
+.Pq Event C7H , Umask 01H
+Counts SIMD packed single-precision floating point Uops retired.
+.It Li SSEX_UOPS_RETIRED.SCALAR_SINGLE
+.Pq Event C7H , Umask 02H
+Counts SIMD calar single-precision floating point Uops retired.
+.It Li SSEX_UOPS_RETIRED.PACKED_DOUBLE
+.Pq Event C7H , Umask 04H
+Counts SIMD packed double- precision floating point Uops retired.
+.It Li SSEX_UOPS_RETIRED.SCALAR_DOUBLE
+.Pq Event C7H , Umask 08H
+Counts SIMD scalar double-precision floating point Uops retired.
+.It Li SSEX_UOPS_RETIRED.VECTOR_INTEGER
+.Pq Event C7H , Umask 10H
+Counts 128-bit SIMD vector integer Uops retired.
+.It Li ITLB_MISS_RETIRED
+.Pq Event C8H , Umask 20H
+Counts the number of retired instructions that missed the ITLB when the
+instruction was fetched.
+.It Li MEM_LOAD_RETIRED.L1D_HIT
+.Pq Event CBH , Umask 01H
+Counts number of retired loads that hit the L1 data cache.
+.It Li MEM_LOAD_RETIRED.L2_HIT
+.Pq Event CBH , Umask 02H
+Counts number of retired loads that hit the L2 data cache.
+.It Li MEM_LOAD_RETIRED.L3_UNSHARED_HIT
+.Pq Event CBH , Umask 04H
+Counts number of retired loads that hit their own, unshared lines in the L3
+cache.
+.It Li MEM_LOAD_RETIRED.OTHER_CORE_L2_HIT_HITM
+.Pq Event CBH , Umask 08H
+Counts number of retired loads that hit in a sibling core's L2 (on die
+core). Since the L3 is inclusive of all cores on the package, this is an L3
+hit. This counts both clean or modified hits.
+.It Li MEM_LOAD_RETIRED.L3_MISS
+.Pq Event CBH , Umask 10H
+Counts number of retired loads that miss the L3 cache. The load was
+satisfied by a remote socket, local memory or an IOH.
+.It Li MEM_LOAD_RETIRED.HIT_LFB
+.Pq Event CBH , Umask 40H
+Counts number of retired loads that miss the L1D and the address is located
+in an allocated line fill buffer and will soon be committed to cache. This
+is counting secondary L1D misses.
+.It Li MEM_LOAD_RETIRED.DTLB_MISS
+.Pq Event CBH , Umask 80H
+Counts the number of retired loads that missed the DTLB. The DTLB miss is
+not counted if the load operation causes a fault. This event counts loads
+from cacheable memory only. The event does not count loads by software
+prefetches. Counts both primary and secondary misses to the TLB.
+.It Li FP_MMX_TRANS.TO_FP
+.Pq Event CCH , Umask 01H
+Counts the first floating-point instruction following any MMX instruction.
+You can use this event to estimate the penalties for the transitions between
+floating-point and MMX technology states.
+.It Li FP_MMX_TRANS.TO_MMX
+.Pq Event CCH , Umask 02H
+Counts the first MMX instruction following a floating-point instruction. You
+can use this event to estimate the penalties for the transitions between
+floating-point and MMX technology states.
+.It Li FP_MMX_TRANS.ANY
+.Pq Event CCH , Umask 03H
+Counts all transitions from floating point to MMX instructions and from MMX
+instructions to floating point instructions. You can use this event to
+estimate the penalties for the transitions between floating-point and MMX
+technology states.
+.It Li MACRO_INSTS.DECODED
+.Pq Event D0H , Umask 01H
+Counts the number of instructions decoded, (but not necessarily executed or
+retired).
+.It Li UOPS_DECODED.MS
+.Pq Event D1H , Umask 02H
+Counts the number of Uops decoded by the Microcode Sequencer, MS. The MS
+delivers uops when the instruction is more than 4 uops long or a microcode
+assist is occurring.
+.It Li UOPS_DECODED.ESP_FOLDING
+.Pq Event D1H , Umask 04H
+Counts number of stack pointer (ESP) instructions decoded: push , pop , call
+, ret, etc. ESP instructions do not generate a Uop to increment or decrement
+ESP. Instead, they update an ESP_Offset register that keeps track of the
+delta to the current value of the ESP register.
+.It Li UOPS_DECODED.ESP_SYNC
+.Pq Event D1H , Umask 08H
+Counts number of stack pointer (ESP) sync operations where an ESP
+instruction is corrected by adding the ESP offset register to the current
+value of the ESP register.
+.It Li RAT_STALLS.FLAGS
+.Pq Event D2H , Umask 01H
+Counts the number of cycles during which execution stalled due to several
+reasons, one of which is a partial flag register stall. A partial register
+stall may occur when two conditions are met: 1) an instruction modifies
+some, but not all, of the flags in the flag register and 2) the next
+instruction, which depends on flags, depends on flags that were not modified
+by this instruction.
+.It Li RAT_STALLS.REGISTERS
+.Pq Event D2H , Umask 02H
+This event counts the number of cycles instruction execution latency became
+longer than the defined latency because the instruction used a register that
+was partially written by previous instruction.
+.It Li RAT_STALLS.ROB_READ_PORT
+.Pq Event D2H , Umask 04H
+Counts the number of cycles when ROB read port stalls occurred, which did
+not allow new micro-ops to enter the out-of-order pipeline. Note that, at
+this stage in the pipeline, additional stalls may occur at the same cycle
+and prevent the stalled micro-ops from entering the pipe. In such a case,
+micro-ops retry entering the execution pipe in the next cycle and the
+ROB-read port stall is counted again.
+.It Li RAT_STALLS.SCOREBOARD
+.Pq Event D2H , Umask 08H
+Counts the cycles where we stall due to microarchitecturally required
+serialization. Microcode scoreboarding stalls.
+.It Li RAT_STALLS.ANY
+.Pq Event D2H , Umask 0FH
+Counts all Register Allocation Table stall cycles due to: Cycles when ROB
+read port stalls occurred, which did not allow new micro-ops to enter the
+execution pipe. Cycles when partial register stalls occurred Cycles when
+flag stalls occurred Cycles floating-point unit (FPU) status word stalls
+occurred. To count each of these conditions separately use the events:
+RAT_STALLS.ROB_READ_PORT, RAT_STALLS.PARTIAL, RAT_STALLS.FLAGS, and
+RAT_STALLS.FPSW.
+.It Li SEG_RENAME_STALLS
+.Pq Event D4H , Umask 01H
+Counts the number of stall cycles due to the lack of renaming resources for
+the ES, DS, FS, and GS segment registers. If a segment is renamed but not
+retired and a second update to the same segment occurs, a stall occurs in
+the front-end of the pipeline until the renamed segment retires.
+.It Li ES_REG_RENAMES
+.Pq Event D5H , Umask 01H
+Counts the number of times the ES segment register is renamed.
+.It Li UOP_UNFUSION
+.Pq Event DBH , Umask 01H
+Counts unfusion events due to floating point exception to a fused uop.
+.It Li BR_INST_DECODED
+.Pq Event E0H , Umask 01H
+Counts the number of branch instructions decoded.
+.It Li BPU_MISSED_CALL_RET
+.Pq Event E5H , Umask 01H
+Counts number of times the Branch Prediciton Unit missed predicting a call
+or return branch.
+.It Li BACLEAR.CLEAR
+.Pq Event E6H , Umask 01H
+Counts the number of times the front end is resteered, mainly when the
+Branch Prediction Unit cannot provide a correct prediction and this is
+corrected by the Branch Address Calculator at the front end. This can occur
+if the code has many branches such that they cannot be consumed by the BPU.
+Each BACLEAR asserted by the BAC generates approximately an 8 cycle bubble
+in the instruction fetch pipeline. The effect on total execution time
+depends on the surrounding code.
+.It Li BACLEAR.BAD_TARGET
+.Pq Event E6H , Umask 02H
+Counts number of Branch Address Calculator clears (BACLEAR) asserted due to
+conditional branch instructions in which there was a target hit but the
+direction was wrong. Each BACLEAR asserted by the BAC generates
+approximately an 8 cycle bubble in the instruction fetch pipeline.
+.It Li BPU_CLEARS.EARLY
+.Pq Event E8H , Umask 01H
+Counts early (normal) Branch Prediction Unit clears: BPU predicted a taken
+branch after incorrectly assuming that it was not taken.
+The BPU clear leads to 2 cycle bubble in the Front End.
+.It Li BPU_CLEARS.LATE
+.Pq Event E8H , Umask 02H
+Counts late Branch Prediction Unit clears due to Most Recently Used
+conflicts. The PBU clear leads to a 3 cycle bubble in the Front End.
+.It Li BPU_CLEARS.ANY
+.Pq Event E8H , Umask 03H
+Counts all BPU clears.
+.It Li L2_TRANSACTIONS.LOAD
+.Pq Event F0H , Umask 01H
+Counts L2 load operations due to HW prefetch or demand loads.
+.It Li L2_TRANSACTIONS.RFO
+.Pq Event F0H , Umask 02H
+Counts L2 RFO operations due to HW prefetch or demand RFOs.
+.It Li L2_TRANSACTIONS.IFETCH
+.Pq Event F0H , Umask 04H
+Counts L2 instruction fetch operations due to HW prefetch or demand ifetch.
+.It Li L2_TRANSACTIONS.PREFETCH
+.Pq Event F0H , Umask 08H
+Counts L2 prefetch operations.
+.It Li L2_TRANSACTIONS.L1D_WB
+.Pq Event F0H , Umask 10H
+Counts L1D writeback operations to the L2.
+.It Li L2_TRANSACTIONS.FILL
+.Pq Event F0H , Umask 20H
+Counts L2 cache line fill operations due to load, RFO, L1D writeback or
+prefetch.
+.It Li L2_TRANSACTIONS.WB
+.Pq Event F0H , Umask 40H
+Counts L2 writeback operations to the L3.
+.It Li L2_TRANSACTIONS.ANY
+.Pq Event F0H , Umask 80H
+Counts all L2 cache operations.
+.It Li L2_LINES_IN.S_STATE
+.Pq Event F1H , Umask 02H
+Counts the number of cache lines allocated in the L2 cache in the S (shared)
+state.
+.It Li L2_LINES_IN.E_STATE
+.Pq Event F1H , Umask 04H
+Counts the number of cache lines allocated in the L2 cache in the E
+(exclusive) state.
+.It Li L2_LINES_IN.ANY
+.Pq Event F1H , Umask 07H
+Counts the number of cache lines allocated in the L2 cache.
+.It Li L2_LINES_OUT.DEMAND_CLEAN
+.Pq Event F2H , Umask 01H
+Counts L2 clean cache lines evicted by a demand request.
+.It Li L2_LINES_OUT.DEMAND_DIRTY
+.Pq Event F2H , Umask 02H
+Counts L2 dirty (modified) cache lines evicted by a demand request.
+.It Li L2_LINES_OUT.PREFETCH_CLEAN
+.Pq Event F2H , Umask 04H
+Counts L2 clean cache line evicted by a prefetch request.
+.It Li L2_LINES_OUT.PREFETCH_DIRTY
+.Pq Event F2H , Umask 08H
+Counts L2 modified cache line evicted by a prefetch request.
+.It Li L2_LINES_OUT.ANY
+.Pq Event F2H , Umask 0FH
+Counts all L2 cache lines evicted for any reason.
+.It Li SQ_MISC.SPLIT_LOCK
+.Pq Event F4H , Umask 10H
+Counts the number of SQ lock splits across a cache line.
+.It Li SQ_FULL_STALL_CYCLES
+.Pq Event F6H , Umask 01H
+Counts cycles the Super Queue is full. Neither of the threads on this core
+will be able to access the uncore.
+.It Li FP_ASSIST.ALL
+.Pq Event F7H , Umask 01H
+Counts the number of floating point operations executed that required
+micro-code assist intervention. Assists are required in the following cases:
+SSE instructions, (Denormal input when the DAZ flag is off or Underflow
+result when the FTZ flag is off): x87 instructions, (NaN or denormal are
+loaded to a register or used as input from memory, Division by 0 or
+Underflow output).
+.It Li FP_ASSIST.OUTPUT
+.Pq Event F7H , Umask 02H
+Counts number of floating point micro-code assist when the output value
+(destination register) is invalid.
+.It Li FP_ASSIST.INPUT
+.Pq Event F7H , Umask 04H
+Counts number of floating point micro-code assist when the input value (one
+of the source operands to an FP instruction) is invalid.
+.It Li SIMD_INT_64.PACKED_MPY
+.Pq Event FDH , Umask 01H
+Counts number of SID integer 64 bit packed multiply operations.
+.It Li SIMD_INT_64.PACKED_SHIFT
+.Pq Event FDH , Umask 02H
+Counts number of SID integer 64 bit packed shift operations.
+.It Li SIMD_INT_64.PACK
+.Pq Event FDH , Umask 04H
+Counts number of SID integer 64 bit pack operations.
+.It Li SIMD_INT_64.UNPACK
+.Pq Event FDH , Umask 08H
+Counts number of SID integer 64 bit unpack operations.
+.It Li SIMD_INT_64.PACKED_LOGICAL
+.Pq Event FDH , Umask 10H
+Counts number of SID integer 64 bit logical operations.
+.It Li SIMD_INT_64.PACKED_ARITH
+.Pq Event FDH , Umask 20H
+Counts number of SID integer 64 bit arithmetic operations.
+.It Li SIMD_INT_64.SHUFFLE_MOVE
+.Pq Event FDH , Umask 40H
+Counts number of SID integer 64 bit shift or move operations.
+.El
+.Ss Event Specifiers (Programmable PMCs)
+Core i7 and Xeon 5500 programmable PMCs support the following events as
+June 2009 document (removed in December 2009):
+.Bl -tag -width indent
+.It Li SB_FORWARD.ANY
+.Pq Event 02H , Umask 01H
+Counts the number of store forwards.
+.It Li LOAD_BLOCK.STD
+.Pq Event 03H , Umask 01H
+Counts the number of loads blocked by a preceding store with unknown data.
+.It Li LOAD_BLOCK.ADDRESS_OFFSET
+.Pq Event 03H , Umask 04H
+Counts the number of loads blocked by a preceding store address.
+.It Li LOAD_BLOCK.ADDRESS_OFFSET
+.Pq Event 01H , Umask 04H
+Counts the cycles of store buffer drains.
+.It Li MISALIGN_MEM_REF.LOAD
+.Pq Event 05H , Umask 01H
+Counts the number of misaligned load references
+.It Li MISALIGN_MEM_REF.STORE
+.Pq Event 05H , Umask 02H
+Counts the number of misaligned store references
+.It Li MISALIGN_MEM_REF.ANY
+.Pq Event 05H , Umask 03H
+Counts the number of misaligned memory references
+.It Li STORE_BLOCKS.NOT_STA
+.Pq Event 06H , Umask 01H
+This event counts the number of load operations delayed caused by preceding
+stores whose addresses are known but whose data is unknown, and preceding
+stores that conflict with the load but which incompletely overlap the load.
+.It Li STORE_BLOCKS.STA
+.Pq Event 06H , Umask 02H
+This event counts load operations delayed caused by preceding stores whose
+addresses are unknown (STA block).
+.It Li STORE_BLOCKS.ANY
+.Pq Event 06H , Umask 0FH
+All loads delayed due to store blocks
+.It Li MEMORY_DISAMBIGURATION.RESET
+.Pq Event 09H , Umask 01H
+Counts memory disambiguration reset cycles
+.It Li MEMORY_DISAMBIGURATION.SUCCESS
+.Pq Event 09H , Umask 02H
+Counts the number of loads that memory disambiguration succeeded
+.It Li MEMORY_DISAMBIGURATION.WATCHDOG
+.Pq Event 09H , Umask 04H
+Counts the number of times the memory disambiguration watchdog kicked in.
+.It Li MEMORY_DISAMBIGURATION.WATCH_CYCLES
+.Pq Event 09H , Umask 08H
+Counts the cycles that the memory disambiguration watchdog is active.
+set invert=1, cmask = 1
+.It Li HW_INT.RCV
+.Pq Event 1DH , Umask 01H
+Number of interrupt received
+.It Li HW_INT.CYCLES_MASKED
+.Pq Event 1DH , Umask 02H
+Number of cycles interrupt are masked
+.It Li HW_INT.CYCLES_PENDING_AND_MASKED
+.Pq Event 1DH , Umask 04H
+Number of cycles interrupts are pending and masked
+.It Li HW_INT.CYCLES_PENDING_AND_MASKED
+.Pq Event 04H , Umask 04H
+Counts number of L2 store RFO requests where the cache line to be loaded is
+in the E (exclusive) state. The L1D prefetcher does not issue a RFO
+prefetch.
+This is a demand RFO request
+.It Li HW_INT.CYCLES_PENDING_AND_MASKED
+.Pq Event 27H , Umask 04H
+LONGEST_LAT_CACH E.MISS
+.It Li UOPS_DECODED.DEC0
+.Pq Event 3DH , Umask 01H
+Counts micro-ops decoded by decoder 0.
+.It Li UOPS_DECODED.DEC0
+.Pq Event 01H , Umask 01H
+Counts L1 data cache store RFO requests where the cache line to be loaded is
+in the I state.
+Counter 0, 1 only
+.It Li 0FH
+.Pq Event 41H , Umask 41H
+L1D_CACHE_ST.MESI
+Counts L1 data cache store RFO requests.
+Counter 0, 1 only
+.It Li DTLB_MISSES.PDE_MISS
+.Pq Event 49H , Umask 20H
+Number of DTLB cache misses where the low part of the linear to physical
+address translation was missed.
+.It Li DTLB_MISSES.PDP_MISS
+.Pq Event 49H , Umask 40H
+Number of DTLB misses where the high part of the linear to physical address
+translation was missed.
+.It Li DTLB_MISSES.LARGE_WALK_COMPLETED
+.Pq Event 49H , Umask 80H
+Counts number of completed large page walks due to misses in the STLB.
+.It Li SSE_MEM_EXEC.NTA
+.Pq Event 4BH , Umask 01H
+Counts number of SSE NTA prefetch/weakly-ordered instructions which missed
+the L1 data cache.
+.It Li SSE_MEM_EXEC.STREAMING_STORES
+.Pq Event 4BH , Umask 08H
+Counts number of SSE non temporal stores
+.It Li SFENCE_CYCLES
+.Pq Event 4DH , Umask 01H
+Counts store fence cycles
+.It Li EPT.EPDE_MISS
+.Pq Event 4FH , Umask 02H
+Counts Extended Page Directory Entry misses. The Extended Page Directory
+cache is used by Virtual Machine operating systems while the guest operating
+systems use the standard TLB caches.
+.It Li EPT.EPDPE_HIT
+.Pq Event 4FH , Umask 04H
+Counts Extended Page Directory Pointer Entry hits.
+.It Li EPT.EPDPE_MISS
+.Pq Event 4FH , Umask 08H
+Counts Extended Page Directory Pointer Entry misses. T
+.It Li OFFCORE_REQUESTS_OUTSTANDING.DEMAND.READ_DATA
+.Pq Event 60H , Umask 01H
+Counts weighted cycles of offcore demand data read requests. Does not
+include L2 prefetch requests.
+counter 0
+.It Li OFFCORE_REQUESTS_OUTSTANDING.DEMAND.READ_CODE
+.Pq Event 60H , Umask 02H
+Counts weighted cycles of offcore demand code read requests. Does not
+include L2 prefetch requests.
+counter 0
+.It Li OFFCORE_REQUESTS_OUTSTANDING.DEMAND.RFO
+.Pq Event 60H , Umask 04H
+Counts weighted cycles of offcore demand RFO requests. Does not include L2
+prefetch requests.
+counter 0
+.It Li OFFCORE_REQUESTS_OUTSTANDING.ANY.READ
+.Pq Event 60H , Umask 08H
+Counts weighted cycles of offcore read requests of any kind. Include L2
+prefetch requests.
+counter 0
+.It Li IFU_IVC.FULL
+.Pq Event 81H , Umask 01H
+Instruction Fetche unit victim cache full.
+.It Li IFU_IVC.L1I_EVICTION
+.Pq Event 81H , Umask 02H
+L1 Instruction cache evictions.
+.It Li L1I_OPPORTUNISTIC_HITS
+.Pq Event 83H , Umask 01H
+Opportunistic hits in streaming.
+.It Li ITLB_MISSES.WALK_CYCLES
+.Pq Event 85H , Umask 04H
+Counts ITLB miss page walk cycles.
+.It Li ITLB_MISSES.PMH_BUSY_CYCLES
+.Pq Event 85H , Umask 04H
+Counts PMH busy cycles.
+.It Li ITLB_MISSES.STLB_HIT
+.Pq Event 85H , Umask 10H
+Counts the number of ITLB misses that hit in the second level TLB.
+.It Li ITLB_MISSES.PDE_MISS
+.Pq Event 85H , Umask 20H
+Number of ITLB misses where the low part of the linear to physical address
+translation was missed.
+.It Li ITLB_MISSES.PDP_MISS
+.Pq Event 85H , Umask 40H
+Number of ITLB misses where the high part of the linear to physical address
+translation was missed.
+.It Li ITLB_MISSES.LARGE_WALK_COMPLETED
+.Pq Event 85H , Umask 80H
+Counts number of completed large page walks due to misses in the STLB.
+.It Li ITLB_MISSES.LARGE_WALK_COMPLETED
+.Pq Event 01H , Umask 80H
+Counts number of offcore demand data read requests. Does not count L2
+prefetch requests.
+.It Li OFFCORE_REQUESTS.DEMAND.READ_CODE
+.Pq Event B0H , Umask 02H
+Counts number of offcore demand code read requests. Does not count L2
+prefetch requests.
+.It Li OFFCORE_REQUESTS.DEMAND.RFO
+.Pq Event B0H , Umask 04H
+Counts number of offcore demand RFO requests. Does not count L2 prefetch
+requests.
+.It Li OFFCORE_REQUESTS.ANY.READ
+.Pq Event B0H , Umask 08H
+Counts number of offcore read requests. Includes L2 prefetch requests.
+.It Li OFFCORE_REQUESTS.ANY.RFO
+.Pq Event B0H , Umask 10H
+Counts number of offcore RFO requests. Includes L2 prefetch requests.
+.It Li OFFCORE_REQUESTS.UNCACHED_MEM
+.Pq Event B0H , Umask 20H
+Counts number of offcore uncached memory requests.
+.It Li OFFCORE_REQUESTS.ANY
+.Pq Event B0H , Umask 80H
+Counts all offcore requests.
+.It Li SNOOPQ_REQUESTS_OUTSTANDING.DATA
+.Pq Event B3H , Umask 01H
+Counts weighted cycles of snoopq requests for data. Counter 0 only
+Use cmask=1 to count cycles not empty.
+.It Li SNOOPQ_REQUESTS_OUTSTANDING.INVALIDATE
+.Pq Event B3H , Umask 02H
+Counts weighted cycles of snoopq invalidate requests. Counter 0 only
+Use cmask=1 to count cycles not empty.
+.It Li SNOOPQ_REQUESTS_OUTSTANDING.CODE
+.Pq Event B3H , Umask 04H
+Counts weighted cycles of snoopq requests for code. Counter 0 only
+Use cmask=1 to count cycles not empty.
+.It Li SNOOPQ_REQUESTS_OUTSTANDING.CODE
+.Pq Event BAH , Umask 04H
+Counts number of TPR reads
+.It Li PIC_ACCESSES.TPR_WRITES
+.Pq Event BAH , Umask 02H
+Counts number of TPR writes
+one or two micro-ops. Some instructions are decoded into longer sequences
+.It Li MACHINE_CLEARS.FUSION_ASSIST
+.Pq Event C3H , Umask 10H
+Counts the number of macro-fusion assists
+Counts SIMD packed single- precision floating point Uops retired.
+.It Li BOGUS_BR
+.Pq Event E4H , Umask 01H
+Counts the number of bogus branches.
+.It Li L2_HW_PREFETCH.HIT
+.Pq Event F3H , Umask 01H
+Count L2 HW prefetcher detector hits
+.It Li L2_HW_PREFETCH.ALLOC
+.Pq Event F3H , Umask 02H
+Count L2 HW prefetcher allocations
+.It Li L2_HW_PREFETCH.DATA_TRIGGER
+.Pq Event F3H , Umask 04H
+Count L2 HW data prefetcher triggered
+.It Li L2_HW_PREFETCH.CODE_TRIGGER
+.Pq Event F3H , Umask 08H
+Count L2 HW code prefetcher triggered
+.It Li L2_HW_PREFETCH.DCA_TRIGGER
+.Pq Event F3H , Umask 10H
+Count L2 HW DCA prefetcher triggered
+.It Li L2_HW_PREFETCH.KICK_START
+.Pq Event F3H , Umask 20H
+Count L2 HW prefetcher kick started
+.It Li SQ_MISC.PROMOTION
+.Pq Event F4H , Umask 01H
+Counts the number of L2 secondary misses that hit the Super Queue.
+.It Li SQ_MISC.PROMOTION_POST_GO
+.Pq Event F4H , Umask 02H
+Counts the number of L2 secondary misses during the Super Queue filling L2.
+.It Li SQ_MISC.LRU_HINTS
+.Pq Event F4H , Umask 04H
+Counts number of Super Queue LRU hints sent to L3.
+.It Li SQ_MISC.FILL_DROPPED
+.Pq Event F4H , Umask 08H
+Counts the number of SQ L2 fills dropped due to L2 busy.
+.It Li SEGMENT_REG_LOADS
+.Pq Event F8H , Umask 01H
+Counts number of segment register loads.
+.El
+.Sh SEE ALSO
+.Xr pmc 3 ,
+.Xr pmc.atom 3 ,
+.Xr pmc.core 3 ,
+.Xr pmc.iaf 3 ,
+.Xr pmc.ucf 3 ,
+.Xr pmc.k7 3 ,
+.Xr pmc.k8 3 ,
+.Xr pmc.p4 3 ,
+.Xr pmc.p5 3 ,
+.Xr pmc.p6 3 ,
+.Xr pmc.corei7uc 3 ,
+.Xr pmc.westmere 3 ,
+.Xr pmc.westmereuc 3 ,
+.Xr pmc.tsc 3 ,
+.Xr pmc_cpuinfo 3 ,
+.Xr pmclog 3 ,
+.Xr hwpmc 4
+.Sh HISTORY
+The
+.Nm pmc
+library first appeared in
+.Fx 6.0 .
+.Sh AUTHORS
+The
+.Lb libpmc
+library was written by
+.An "Joseph Koshy"
+.Aq jkoshy@FreeBSD.org .
diff --git a/lib/libpmc/pmc.corei7uc.3 b/lib/libpmc/pmc.corei7uc.3
new file mode 100644
index 0000000..2c1b3dd
--- /dev/null
+++ b/lib/libpmc/pmc.corei7uc.3
@@ -0,0 +1,880 @@
+.\" Copyright (c) 2010 Fabien Thomas.  All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" This software is provided by Joseph Koshy ``as is'' and
+.\" any express or implied warranties, including, but not limited to, the
+.\" implied warranties of merchantability and fitness for a particular purpose
+.\" are disclaimed.  in no event shall Joseph Koshy be liable
+.\" for any direct, indirect, incidental, special, exemplary, or consequential
+.\" damages (including, but not limited to, procurement of substitute goods
+.\" or services; loss of use, data, or profits; or business interruption)
+.\" however caused and on any theory of liability, whether in contract, strict
+.\" liability, or tort (including negligence or otherwise) arising in any way
+.\" out of the use of this software, even if advised of the possibility of
+.\" such damage.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd March 24, 2010
+.Os
+.Dt PMC.COREI7UC 3
+.Sh NAME
+.Nm pmc.corei7uc
+.Nd uncore measurement events for
+.Tn Intel
+.Tn Core i7 and Xeon 5500
+family CPUs
+.Sh LIBRARY
+.Lb libpmc
+.Sh SYNOPSIS
+.In pmc.h
+.Sh DESCRIPTION
+.Tn Intel
+.Tn "Core i7"
+CPUs contain PMCs conforming to version 2 of the
+.Tn Intel
+performance measurement architecture.
+These CPUs contain 2 classes of PMCs:
+.Bl -tag -width "Li PMC_CLASS_UCP"
+.It Li PMC_CLASS_UCF
+Fixed-function counters that count only one hardware event per counter.
+.It Li PMC_CLASS_UCP
+Programmable counters that may be configured to count one of a defined
+set of hardware events.
+.El
+.Pp
+The number of PMCs available in each class and their widths need to be
+determined at run time by calling
+.Xr pmc_cpuinfo 3 .
+.Pp
+Intel Core i7 and Xeon 5500 PMCs are documented in
+.Rs
+.%B "Intel(R) 64 and IA-32 Architectures Software Developes Manual"
+.%T "Volume 3B: System Programming Guide, Part 2"
+.%N "Order Number: 253669-033US"
+.%D December 2009
+.%Q "Intel Corporation"
+.Re
+.Ss COREI7 AND XEON 5500 UNCORE FIXED FUNCTION PMCS
+These PMCs and their supported events are documented in
+.Xr pmc.ucf 3 .
+.Ss COREI7 AND XEON 5500 UNCORE PROGRAMMABLE PMCS
+The programmable PMCs support the following capabilities:
+.Bl -column "PMC_CAP_INTERRUPT" "Support"
+.It Em Capability Ta Em Support
+.It PMC_CAP_CASCADE Ta \&No
+.It PMC_CAP_EDGE Ta Yes
+.It PMC_CAP_INTERRUPT Ta \&No
+.It PMC_CAP_INVERT Ta Yes
+.It PMC_CAP_READ Ta Yes
+.It PMC_CAP_PRECISE Ta \&No
+.It PMC_CAP_SYSTEM Ta \&No
+.It PMC_CAP_TAGGING Ta \&No
+.It PMC_CAP_THRESHOLD Ta Yes
+.It PMC_CAP_USER Ta \&No
+.It PMC_CAP_WRITE Ta Yes
+.El
+.Ss Event Qualifiers
+Event specifiers for these PMCs support the following common
+qualifiers:
+.Bl -tag -width indent
+.It Li cmask= Ns Ar value
+Configure the PMC to increment only if the number of configured
+events measured in a cycle is greater than or equal to
+.Ar value .
+.It Li edge
+Configure the PMC to count the number of de-asserted to asserted
+transitions of the conditions expressed by the other qualifiers.
+If specified, the counter will increment only once whenever a
+condition becomes true, irrespective of the number of clocks during
+which the condition remains true.
+.It Li inv
+Invert the sense of comparison when the
+.Dq Li cmask
+qualifier is present, making the counter increment when the number of
+events per cycle is less than the value specified by the
+.Dq Li cmask
+qualifier.
+.El
+.Ss Event Specifiers (Programmable PMCs)
+Core i7 and Xeon 5500 uncore programmable PMCs support the following events:
+.Bl -tag -width indent
+.It Li GQ_CYCLES_FULL.READ_TRACKER
+.Pq Event 00H , Umask 01H
+Uncore cycles Global Queue read tracker is full.
+.It Li GQ_CYCLES_FULL.WRITE_TRACKER
+.Pq Event 00H , Umask 02H
+Uncore cycles Global Queue write tracker is full.
+.It Li GQ_CYCLES_FULL.PEER_PROBE_TRACKER
+.Pq Event 00H , Umask 04H
+Uncore cycles Global Queue peer probe tracker is full. The peer probe
+tracker queue tracks snoops from the IOH and remote sockets.
+.It Li GQ_CYCLES_NOT_EMPTY.READ_TRACKER
+.Pq Event 01H , Umask 01H
+Uncore cycles were Global Queue read tracker has at least one valid entry.
+.It Li GQ_CYCLES_NOT_EMPTY.WRITE_TRACKER
+.Pq Event 01H , Umask 02H
+Uncore cycles were Global Queue write tracker has at least one valid entry.
+.It Li GQ_CYCLES_NOT_EMPTY.PEER_PROBE_TRACKER
+.Pq Event 01H , Umask 04H
+Uncore cycles were Global Queue peer probe tracker has at least one valid
+entry. The peer probe tracker queue tracks IOH and remote socket snoops.
+.It Li GQ_ALLOC.READ_TRACKER
+.Pq Event 03H , Umask 01H
+Counts the number of tread tracker allocate to deallocate entries. The GQ
+read tracker allocate to deallocate occupancy count is divided by the count
+to obtain the average read tracker latency.
+.It Li GQ_ALLOC.RT_L3_MISS
+.Pq Event 03H , Umask 02H
+Counts the number GQ read tracker entries for which a full cache line read
+has missed the L3. The GQ read tracker L3 miss to fill occupancy count is
+divided by this count to obtain the average cache line read L3 miss latency.
+The latency represents the time after which the L3 has determined that the
+cache line has missed. The time between a GQ read tracker allocation and the
+L3 determining that the cache line has missed is the average L3 hit latency.
+The total L3 cache line read miss latency is the hit latency + L3 miss
+latency.
+.It Li GQ_ALLOC.RT_TO_L3_RESP
+.Pq Event 03H , Umask 04H
+Counts the number of GQ read tracker entries that are allocated in the read
+tracker queue that hit or miss the L3. The GQ read tracker L3 hit occupancy
+count is divided by this count to obtain the average L3 hit latency.
+.It Li GQ_ALLOC.RT_TO_RTID_ACQUIRED
+.Pq Event 03H , Umask 08H
+Counts the number of GQ read tracker entries that are allocated in the read
+tracker, have missed in the L3 and have not acquired a Request Transaction
+ID.	The GQ read tracker L3 miss to RTID acquired occupancy count is
+divided by this count to obtain the average latency for a read L3 miss to
+acquire an RTID.
+.It Li GQ_ALLOC.WT_TO_RTID_ACQUIRED
+.Pq Event 03H , Umask 10H
+Counts the number of GQ write tracker entries that are allocated in the
+write tracker, have missed in the L3 and have not acquired a Request
+Transaction ID.	The GQ write tracker L3 miss to RTID occupancy count is
+divided by this count to obtain the average latency for a write L3 miss to
+acquire an RTID.
+.It Li GQ_ALLOC.WRITE_TRACKER
+.Pq Event 03H , Umask 20H
+Counts the number of GQ write tracker entries that are allocated in the
+write tracker queue that miss the L3. The GQ write tracker occupancy count
+is divided by the this count to obtain the average L3 write miss latency.
+.It Li GQ_ALLOC.PEER_PROBE_TRACKER
+.Pq Event 03H , Umask 40H
+Counts the number of GQ peer probe tracker (snoop) entries that are
+allocated in the peer probe tracker queue that miss the L3. The GQ peer
+probe occupancy count is divided by this count to obtain the average L3 peer
+probe miss latency.
+.It Li GQ_DATA.FROM_QPI
+.Pq Event 04H , Umask 01H
+Cycles Global Queue Quickpath Interface input data port is busy importing
+data from the Quickpath Interface. Each cycle the input port can transfer 8
+or 16 bytes of data.
+.It Li GQ_DATA.FROM_QMC
+.Pq Event 04H , Umask 02H
+Cycles Global Queue Quickpath Memory Interface input data port is busy
+importing data from the Quickpath Memory Interface. Each cycle the input
+port can transfer 8 or 16 bytes of data.
+.It Li GQ_DATA.FROM_L3
+.Pq Event 04H , Umask 04H
+Cycles GQ L3 input data port is busy importing data from the Last Level
+Cache. Each cycle the input port can transfer 32 bytes of data.
+.It Li GQ_DATA.FROM_CORES_02
+.Pq Event 04H , Umask 08H
+Cycles GQ Core 0 and 2 input data port is busy importing data from processor
+cores 0 and 2. Each cycle the input port can transfer 32 bytes of data.
+.It Li GQ_DATA.FROM_CORES_13
+.Pq Event 04H , Umask 10H
+Cycles GQ Core 1 and 3 input data port is busy importing data from processor
+cores 1 and 3. Each cycle the input port can transfer 32 bytes of data.
+.It Li GQ_DATA.TO_QPI_QMC
+.Pq Event 05H , Umask 01H
+Cycles GQ QPI and QMC output data port is busy sending data to the Quickpath
+Interface or Quickpath Memory Interface. Each cycle the output port can
+transfer 32 bytes of data.
+.It Li GQ_DATA.TO_L3
+.Pq Event 05H , Umask 02H
+Cycles GQ L3 output data port is busy sending data to the Last Level Cache.
+Each cycle the output port can transfer 32 bytes of data.
+.It Li GQ_DATA.TO_CORES
+.Pq Event 05H , Umask 04H
+Cycles GQ Core output data port is busy sending data to the Cores. Each
+cycle the output port can transfer 32 bytes of data.
+.It Li SNP_RESP_TO_LOCAL_HOME.I_STATE
+.Pq Event 06H , Umask 01H
+Number of snoop responses to the local home that L3 does not have the
+referenced cache line.
+.It Li SNP_RESP_TO_LOCAL_HOME.S_STATE
+.Pq Event 06H , Umask 02H
+Number of snoop responses to the local home that L3 has the referenced line
+cached in the S state.
+.It Li SNP_RESP_TO_LOCAL_HOME.FWD_S_STATE
+.Pq Event 06H , Umask 04H
+Number of responses to code or data read snoops to the local home that the
+L3 has the referenced cache line in the E state. The L3 cache line state is
+changed to the S state and the line is forwarded to the local home in the S
+state.
+.It Li SNP_RESP_TO_LOCAL_HOME.FWD_I_STATE
+.Pq Event 06H , Umask 08H
+Number of responses to read invalidate snoops to the local home that the L3
+has the referenced cache line in the M state. The L3 cache line state is
+invalidated and the line is forwarded to the local home in the M state.
+.It Li SNP_RESP_TO_LOCAL_HOME.CONFLICT
+.Pq Event 06H , Umask 10H
+Number of conflict snoop responses sent to the local home.
+.It Li SNP_RESP_TO_LOCAL_HOME.WB
+.Pq Event 06H , Umask 20H
+Number of responses to code or data read snoops to the local home that the
+L3 has the referenced line cached in the M state.
+.It Li SNP_RESP_TO_REMOTE_HOME.I_STATE
+.Pq Event 07H , Umask 01H
+Number of snoop responses to a remote home that L3 does not have the
+referenced cache line.
+.It Li SNP_RESP_TO_REMOTE_HOME.S_STATE
+.Pq Event 07H , Umask 02H
+Number of snoop responses to a remote home that L3 has the referenced line
+cached in the S state.
+.It Li SNP_RESP_TO_REMOTE_HOME.FWD_S_STATE
+.Pq Event 07H , Umask 04H
+Number of responses to code or data read snoops to a remote home that the L3
+has the referenced cache line in the E state. The L3 cache line state is
+changed to the S state and the line is forwarded to the remote home in the S
+state.
+.It Li SNP_RESP_TO_REMOTE_HOME.FWD_I_STATE
+.Pq Event 07H , Umask 08H
+Number of responses to read invalidate snoops to a remote home that the L3
+has the referenced cache line in the M state. The L3 cache line state is
+invalidated and the line is forwarded to the remote home in the M state.
+.It Li SNP_RESP_TO_REMOTE_HOME.CONFLICT
+.Pq Event 07H , Umask 10H
+Number of conflict snoop responses sent to the local home.
+.It Li SNP_RESP_TO_REMOTE_HOME.WB
+.Pq Event 07H , Umask 20H
+Number of responses to code or data read snoops to a remote home that the L3
+has the referenced line cached in the M state.
+.It Li SNP_RESP_TO_REMOTE_HOME.HITM
+.Pq Event 07H , Umask 24H
+Number of HITM snoop responses to a remote home
+.It Li L3_HITS.READ
+.Pq Event 08H , Umask 01H
+Number of code read, data read and RFO requests that hit in the L3
+.It Li L3_HITS.WRITE
+.Pq Event 08H , Umask 02H
+Number of writeback requests that hit in the L3. Writebacks from the cores
+will always result in L3 hits due to the inclusive property of the L3.
+.It Li L3_HITS.PROBE
+.Pq Event 08H , Umask 04H
+Number of snoops from IOH or remote sockets that hit in the L3.
+.It Li L3_HITS.ANY
+.Pq Event 08H , Umask 03H
+Number of reads and writes that hit the L3.
+.It Li L3_MISS.READ
+.Pq Event 09H , Umask 01H
+Number of code read, data read and RFO requests that miss the L3.
+.It Li L3_MISS.WRITE
+.Pq Event 09H , Umask 02H
+Number of writeback requests that miss the L3. Should always be zero as
+writebacks from the cores will always result in L3 hits due to the inclusive
+property of the L3.
+.It Li L3_MISS.PROBE
+.Pq Event 09H , Umask 04H
+Number of snoops from IOH or remote sockets that miss the L3.
+.It Li L3_MISS.ANY
+.Pq Event 09H , Umask 03H
+Number of reads and writes that miss the L3.
+.It Li L3_LINES_IN.M_STATE
+.Pq Event 0AH , Umask 01H
+Counts the number of L3 lines allocated in M state. The only time a cache
+line is allocated in the M state is when the line was forwarded in M state
+is forwarded due to a Snoop Read Invalidate Own request.
+.It Li L3_LINES_IN.E_STATE
+.Pq Event 0AH , Umask 02H
+Counts the number of L3 lines allocated in E state.
+.It Li L3_LINES_IN.S_STATE
+.Pq Event 0AH , Umask 04H
+Counts the number of L3 lines allocated in S state.
+.It Li L3_LINES_IN.F_STATE
+.Pq Event 0AH , Umask 08H
+Counts the number of L3 lines allocated in F state.
+.It Li L3_LINES_IN.ANY
+.Pq Event 0AH , Umask 0FH
+Counts the number of L3 lines allocated in any state.
+.It Li L3_LINES_OUT.M_STATE
+.Pq Event 0BH , Umask 01H
+Counts the number of L3 lines victimized that were in the M state. When the
+victim cache line is in M state, the line is written to its home cache agent
+which can be either local or remote.
+.It Li L3_LINES_OUT.E_STATE
+.Pq Event 0BH , Umask 02H
+Counts the number of L3 lines victimized that were in the E state.
+.It Li L3_LINES_OUT.S_STATE
+.Pq Event 0BH , Umask 04H
+Counts the number of L3 lines victimized that were in the S state.
+.It Li L3_LINES_OUT.I_STATE
+.Pq Event 0BH , Umask 08H
+Counts the number of L3 lines victimized that were in the I state.
+.It Li L3_LINES_OUT.F_STATE
+.Pq Event 0BH , Umask 10H
+Counts the number of L3 lines victimized that were in the F state.
+.It Li L3_LINES_OUT.ANY
+.Pq Event 0BH , Umask 1FH
+Counts the number of L3 lines victimized in any state.
+.It Li QHL_REQUESTS.IOH_READS
+.Pq Event 20H , Umask 01H
+Counts number of Quickpath Home Logic read requests from the IOH.
+.It Li QHL_REQUESTS.IOH_WRITES
+.Pq Event 20H , Umask 02H
+Counts number of Quickpath Home Logic write requests from the IOH.
+.It Li QHL_REQUESTS.REMOTE_READS
+.Pq Event 20H , Umask 04H
+Counts number of Quickpath Home Logic read requests from a remote socket.
+.It Li QHL_REQUESTS.REMOTE_WRITES
+.Pq Event 20H , Umask 08H
+Counts number of Quickpath Home Logic write requests from a remote socket.
+.It Li QHL_REQUESTS.LOCAL_READS
+.Pq Event 20H , Umask 10H
+Counts number of Quickpath Home Logic read requests from the local socket.
+.It Li QHL_REQUESTS.LOCAL_WRITES
+.Pq Event 20H , Umask 20H
+Counts number of Quickpath Home Logic write requests from the local socket.
+.It Li QHL_CYCLES_FULL.IOH
+.Pq Event 21H , Umask 01H
+Counts uclk cycles all entries in the Quickpath Home Logic IOH are full.
+.It Li QHL_CYCLES_FULL.REMOTE
+.Pq Event 21H , Umask 02H
+Counts uclk cycles all entries in the Quickpath Home Logic remote tracker
+are full.
+.It Li QHL_CYCLES_FULL.LOCAL
+.Pq Event 21H , Umask 04H
+Counts uclk cycles all entries in the Quickpath Home Logic local tracker are
+full.
+.It Li QHL_CYCLES_NOT_EMPTY.IOH
+.Pq Event 22H , Umask 01H
+Counts uclk cycles all entries in the Quickpath Home Logic IOH is busy.
+.It Li QHL_CYCLES_NOT_EMPTY.REMOTE
+.Pq Event 22H , Umask 02H
+Counts uclk cycles all entries in the Quickpath Home Logic remote tracker is
+busy.
+.It Li QHL_CYCLES_NOT_EMPTY.LOCAL
+.Pq Event 22H , Umask 04H
+Counts uclk cycles all entries in the Quickpath Home Logic local tracker is
+busy.
+.It Li QHL_OCCUPANCY.IOH
+.Pq Event 23H , Umask 01H
+QHL IOH tracker allocate to deallocate read occupancy.
+.It Li QHL_OCCUPANCY.REMOTE
+.Pq Event 23H , Umask 02H
+QHL remote tracker allocate to deallocate read occupancy.
+.It Li QHL_OCCUPANCY.LOCAL
+.Pq Event 23H , Umask 04H
+QHL local tracker allocate to deallocate read occupancy.
+.It Li QHL_ADDRESS_CONFLICTS.2WAY
+.Pq Event 24H , Umask 02H
+Counts number of QHL Active Address Table (AAT) entries that saw a max of 2
+conflicts. The AAT is a structure that tracks requests that are in conflict.
+The requests themselves are in the home tracker entries. The count is
+reported when an AAT entry deallocates.
+.It Li QHL_ADDRESS_CONFLICTS.3WAY
+.Pq Event 24H , Umask 04H
+Counts number of QHL Active Address Table (AAT) entries that saw a max of 3
+conflicts. The AAT is a structure that tracks requests that are in conflict.
+The requests themselves are in the home tracker entries. The count is
+reported when an AAT entry deallocates.
+.It Li QHL_CONFLICT_CYCLES.IOH
+.Pq Event 25H , Umask 01H
+Counts cycles the Quickpath Home Logic IOH Tracker contains two or more
+requests with an address conflict. A max of 3 requests can be in conflict.
+.It Li QHL_CONFLICT_CYCLES.REMOTE
+.Pq Event 25H , Umask 02H
+Counts cycles the Quickpath Home Logic Remote Tracker contains two or more
+requests with an address conflict. A max of 3 requests can be in conflict.
+.It Li QHL_CONFLICT_CYCLES.LOCAL
+.Pq Event 25H , Umask 04H
+Counts cycles the Quickpath Home Logic Local Tracker contains two or more
+requests with an address conflict. A max of 3 requests can be in conflict.
+.It Li QHL_TO_QMC_BYPASS
+.Pq Event 26H , Umask 01H
+Counts number or requests to the Quickpath Memory Controller that bypass the
+Quickpath Home Logic. All local accesses can be bypassed. For remote
+requests, only read requests can be bypassed.
+.It Li QMC_NORMAL_FULL.READ.CH0
+.Pq Event 27H , Umask 01H
+Uncore cycles all the entries in the DRAM channel 0 medium or low priority
+queue are occupied with read requests.
+.It Li QMC_NORMAL_FULL.READ.CH1
+.Pq Event 27H , Umask 02H
+Uncore cycles all the entries in the DRAM channel 1 medium or low priority
+queue are occupied with read requests.
+.It Li QMC_NORMAL_FULL.READ.CH2
+.Pq Event 27H , Umask 04H
+Uncore cycles all the entries in the DRAM channel 2 medium or low priority
+queue are occupied with read requests.
+.It Li QMC_NORMAL_FULL.WRITE.CH0
+.Pq Event 27H , Umask 08H
+Uncore cycles all the entries in the DRAM channel 0 medium or low priority
+queue are occupied with write requests.
+.It Li QMC_NORMAL_FULL.WRITE.CH1
+.Pq Event 27H , Umask 10H
+Counts cycles all the entries in the DRAM channel 1 medium or low priority
+queue are occupied with write requests.
+.It Li QMC_NORMAL_FULL.WRITE.CH2
+.Pq Event 27H , Umask 20H
+Uncore cycles all the entries in the DRAM channel 2 medium or low priority
+queue are occupied with write requests.
+.It Li QMC_ISOC_FULL.READ.CH0
+.Pq Event 28H , Umask 01H
+Counts cycles all the entries in the DRAM channel 0 high priority queue are
+occupied with isochronous read requests.
+.It Li QMC_ISOC_FULL.READ.CH1
+.Pq Event 28H , Umask 02H
+Counts cycles all the entries in the DRAM channel 1high priority queue are
+occupied with isochronous read requests.
+.It Li QMC_ISOC_FULL.READ.CH2
+.Pq Event 28H , Umask 04H
+Counts cycles all the entries in the DRAM channel 2 high priority queue are
+occupied with isochronous read requests.
+.It Li QMC_ISOC_FULL.WRITE.CH0
+.Pq Event 28H , Umask 08H
+Counts cycles all the entries in the DRAM channel 0 high priority queue are
+occupied with isochronous write requests.
+.It Li QMC_ISOC_FULL.WRITE.CH1
+.Pq Event 28H , Umask 10H
+Counts cycles all the entries in the DRAM channel 1 high priority queue are
+occupied with isochronous write requests.
+.It Li QMC_ISOC_FULL.WRITE.CH2
+.Pq Event 28H , Umask 20H
+Counts cycles all the entries in the DRAM channel 2 high priority queue are
+occupied with isochronous write requests.
+.It Li QMC_BUSY.READ.CH0
+.Pq Event 29H , Umask 01H
+Counts cycles where Quickpath Memory Controller has at least 1 outstanding
+read request to DRAM channel 0.
+.It Li QMC_BUSY.READ.CH1
+.Pq Event 29H , Umask 02H
+Counts cycles where Quickpath Memory Controller has at least 1 outstanding
+read request to DRAM channel 1.
+.It Li QMC_BUSY.READ.CH2
+.Pq Event 29H , Umask 04H
+Counts cycles where Quickpath Memory Controller has at least 1 outstanding
+read request to DRAM channel 2.
+.It Li QMC_BUSY.WRITE.CH0
+.Pq Event 29H , Umask 08H
+Counts cycles where Quickpath Memory Controller has at least 1 outstanding
+write request to DRAM channel 0.
+.It Li QMC_BUSY.WRITE.CH1
+.Pq Event 29H , Umask 10H
+Counts cycles where Quickpath Memory Controller has at least 1 outstanding
+write request to DRAM channel 1.
+.It Li QMC_BUSY.WRITE.CH2
+.Pq Event 29H , Umask 20H
+Counts cycles where Quickpath Memory Controller has at least 1 outstanding
+write request to DRAM channel 2.
+.It Li QMC_OCCUPANCY.CH0
+.Pq Event 2AH , Umask 01H
+IMC channel 0 normal read request occupancy.
+.It Li QMC_OCCUPANCY.CH1
+.Pq Event 2AH , Umask 02H
+IMC channel 1 normal read request occupancy.
+.It Li QMC_OCCUPANCY.CH2
+.Pq Event 2AH , Umask 04H
+IMC channel 2 normal read request occupancy.
+.It Li QMC_ISSOC_OCCUPANCY.CH0
+.Pq Event 2BH , Umask 01H
+IMC channel 0 issoc read request occupancy.
+.It Li QMC_ISSOC_OCCUPANCY.CH1
+.Pq Event 2BH , Umask 02H
+IMC channel 1 issoc read request occupancy.
+.It Li QMC_ISSOC_OCCUPANCY.CH2
+.Pq Event 2BH , Umask 04H
+IMC channel 2 issoc read request occupancy.
+.It Li QMC_ISSOC_READS.ANY
+.Pq Event 2BH , Umask 07H
+IMC issoc read request occupancy.
+.It Li QMC_NORMAL_READS.CH0
+.Pq Event 2CH , Umask 01H
+Counts the number of Quickpath Memory Controller channel 0 medium and low
+priority read requests. The QMC channel 0 normal read occupancy divided by
+this count provides the average QMC channel 0 read latency.
+.It Li QMC_NORMAL_READS.CH1
+.Pq Event 2CH , Umask 02H
+Counts the number of Quickpath Memory Controller channel 1 medium and low
+priority read requests. The QMC channel 1 normal read occupancy divided by
+this count provides the average QMC channel 1 read latency.
+.It Li QMC_NORMAL_READS.CH2
+.Pq Event 2CH , Umask 04H
+Counts the number of Quickpath Memory Controller channel 2 medium and low
+priority read requests. The QMC channel 2 normal read occupancy divided by
+this count provides the average QMC channel 2 read latency.
+.It Li QMC_NORMAL_READS.ANY
+.Pq Event 2CH , Umask 07H
+Counts the number of Quickpath Memory Controller medium and low priority
+read requests. The QMC normal read occupancy divided by this count provides
+the average QMC read latency.
+.It Li QMC_HIGH_PRIORITY_READS.CH0
+.Pq Event 2DH , Umask 01H
+Counts the number of Quickpath Memory Controller channel 0 high priority
+isochronous read requests.
+.It Li QMC_HIGH_PRIORITY_READS.CH1
+.Pq Event 2DH , Umask 02H
+Counts the number of Quickpath Memory Controller channel 1 high priority
+isochronous read requests.
+.It Li QMC_HIGH_PRIORITY_READS.CH2
+.Pq Event 2DH , Umask 04H
+Counts the number of Quickpath Memory Controller channel 2 high priority
+isochronous read requests.
+.It Li QMC_HIGH_PRIORITY_READS.ANY
+.Pq Event 2DH , Umask 07H
+Counts the number of Quickpath Memory Controller high priority isochronous
+read requests.
+.It Li QMC_CRITICAL_PRIORITY_READS.CH0
+.Pq Event 2EH , Umask 01H
+Counts the number of Quickpath Memory Controller channel 0 critical priority
+isochronous read requests.
+.It Li QMC_CRITICAL_PRIORITY_READS.CH1
+.Pq Event 2EH , Umask 02H
+Counts the number of Quickpath Memory Controller channel 1 critical priority
+isochronous read requests.
+.It Li QMC_CRITICAL_PRIORITY_READS.CH2
+.Pq Event 2EH , Umask 04H
+Counts the number of Quickpath Memory Controller channel 2 critical priority
+isochronous read requests.
+.It Li QMC_CRITICAL_PRIORITY_READS.ANY
+.Pq Event 2EH , Umask 07H
+Counts the number of Quickpath Memory Controller critical priority
+isochronous read requests.
+.It Li QMC_WRITES.FULL.CH0
+.Pq Event 2FH , Umask 01H
+Counts number of full cache line writes to DRAM channel 0.
+.It Li QMC_WRITES.FULL.CH1
+.Pq Event 2FH , Umask 02H
+Counts number of full cache line writes to DRAM channel 1.
+.It Li QMC_WRITES.FULL.CH2
+.Pq Event 2FH , Umask 04H
+Counts number of full cache line writes to DRAM channel 2.
+.It Li QMC_WRITES.FULL.ANY
+.Pq Event 2FH , Umask 07H
+Counts number of full cache line writes to DRAM.
+.It Li QMC_WRITES.PARTIAL.CH0
+.Pq Event 2FH , Umask 08H
+Counts number of partial cache line writes to DRAM channel 0.
+.It Li QMC_WRITES.PARTIAL.CH1
+.Pq Event 2FH , Umask 10H
+Counts number of partial cache line writes to DRAM channel 1.
+.It Li QMC_WRITES.PARTIAL.CH2
+.Pq Event 2FH , Umask 20H
+Counts number of partial cache line writes to DRAM channel 2.
+.It Li QMC_WRITES.PARTIAL.ANY
+.Pq Event 2FH , Umask 38H
+Counts number of partial cache line writes to DRAM.
+.It Li QMC_CANCEL.CH0
+.Pq Event 30H , Umask 01H
+Counts number of DRAM channel 0 cancel requests.
+.It Li QMC_CANCEL.CH1
+.Pq Event 30H , Umask 02H
+Counts number of DRAM channel 1 cancel requests.
+.It Li QMC_CANCEL.CH2
+.Pq Event 30H , Umask 04H
+Counts number of DRAM channel 2 cancel requests.
+.It Li QMC_CANCEL.ANY
+.Pq Event 30H , Umask 07H
+Counts number of DRAM cancel requests.
+.It Li QMC_PRIORITY_UPDATES.CH0
+.Pq Event 31H , Umask 01H
+Counts number of DRAM channel 0 priority updates. A priority update occurs
+when an ISOC high or critical request is received by the QHL and there is a
+matching request with normal priority that has already been issued to the
+QMC. In this instance, the QHL will send a priority update to QMC to
+expedite the request.
+.It Li QMC_PRIORITY_UPDATES.CH1
+.Pq Event 31H , Umask 02H
+Counts number of DRAM channel 1 priority updates. A priority update occurs
+when an ISOC high or critical request is received by the QHL and there is a
+matching request with normal priority that has already been issued to the
+QMC. In this instance, the QHL will send a priority update to QMC to
+expedite the request.
+.It Li QMC_PRIORITY_UPDATES.CH2
+.Pq Event 31H , Umask 04H
+Counts number of DRAM channel 2 priority updates. A priority update occurs
+when an ISOC high or critical request is received by the QHL and there is a
+matching request with normal priority that has already been issued to the
+QMC. In this instance, the QHL will send a priority update to QMC to
+expedite the request.
+.It Li QMC_PRIORITY_UPDATES.ANY
+.Pq Event 31H , Umask 07H
+Counts number of DRAM priority updates. A priority update occurs when an
+ISOC high or critical request is received by the QHL and there is a matching
+request with normal priority that has already been issued to the QMC. In
+this instance, the QHL will send a priority update to QMC to expedite the
+request.
+.It Li QHL_FRC_ACK_CNFLTS.LOCAL
+.Pq Event 33H , Umask 04H
+Counts number of Force Acknowledge Conflict messages sent by the Quickpath
+Home Logic to the local home.
+.It Li QPI_TX_STALLED_SINGLE_FLIT.HOME.LINK_0
+.Pq Event 40H , Umask 01H
+Counts cycles the Quickpath outbound link 0 HOME virtual channel is stalled
+due to lack of a VNA and VN0 credit. Note that this event does not filter
+out when a flit would not have been selected for arbitration because another
+virtual channel is getting arbitrated.
+.It Li QPI_TX_STALLED_SINGLE_FLIT.SNOOP.LINK_0
+.Pq Event 40H , Umask 02H
+Counts cycles the Quickpath outbound link 0 SNOOP virtual channel is stalled
+due to lack of a VNA and VN0 credit. Note that this event does not filter
+out when a flit would not have been selected for arbitration because another
+virtual channel is getting arbitrated.
+.It Li QPI_TX_STALLED_SINGLE_FLIT.NDR.LINK_0
+.Pq Event 40H , Umask 04H
+Counts cycles the Quickpath outbound link 0 non-data response virtual
+channel is stalled due to lack of a VNA and VN0 credit. Note that this event
+does not filter out when a flit would not have been selected for arbitration
+because another virtual channel is getting arbitrated.
+.It Li QPI_TX_STALLED_SINGLE_FLIT.HOME.LINK_1
+.Pq Event 40H , Umask 08H
+Counts cycles the Quickpath outbound link 1 HOME virtual channel is stalled
+due to lack of a VNA and VN0 credit. Note that this event does not filter
+out when a flit would not have been selected for arbitration because another
+virtual channel is getting arbitrated.
+.It Li QPI_TX_STALLED_SINGLE_FLIT.SNOOP.LINK_1
+.Pq Event 40H , Umask 10H
+Counts cycles the Quickpath outbound link 1 SNOOP virtual channel is stalled
+due to lack of a VNA and VN0 credit. Note that this event does not filter
+out when a flit would not have been selected for arbitration because another
+virtual channel is getting arbitrated.
+.It Li QPI_TX_STALLED_SINGLE_FLIT.NDR.LINK_1
+.Pq Event 40H , Umask 20H
+Counts cycles the Quickpath outbound link 1 non-data response virtual
+channel is stalled due to lack of a VNA and VN0 credit. Note that this event
+does not filter out when a flit would not have been selected for arbitration
+because another virtual channel is getting arbitrated.
+.It Li QPI_TX_STALLED_SINGLE_FLIT.LINK_0
+.Pq Event 40H , Umask 07H
+Counts cycles the Quickpath outbound link 0 virtual channels are stalled due
+to lack of a VNA and VN0 credit. Note that this event does not filter out
+when a flit would not have been selected for arbitration because another
+virtual channel is getting arbitrated.
+.It Li QPI_TX_STALLED_SINGLE_FLIT.LINK_1
+.Pq Event 40H , Umask 38H
+Counts cycles the Quickpath outbound link 1 virtual channels are stalled due
+to lack of a VNA and VN0 credit. Note that this event does not filter out
+when a flit would not have been selected for arbitration because another
+virtual channel is getting arbitrated.
+.It Li QPI_TX_STALLED_MULTI_FLIT.DRS.LINK_0
+.Pq Event 41H , Umask 01H
+Counts cycles the Quickpath outbound link 0 Data ResponSe virtual channel is
+stalled due to lack of VNA and VN0 credits. Note that this event does not
+filter out when a flit would not have been selected for arbitration because
+another virtual channel is getting arbitrated.
+.It Li QPI_TX_STALLED_MULTI_FLIT.NCB.LINK_0
+.Pq Event 41H , Umask 02H
+Counts cycles the Quickpath outbound link 0 Non-Coherent Bypass virtual
+channel is stalled due to lack of VNA and VN0 credits. Note that this event
+does not filter out when a flit would not have been selected for arbitration
+because another virtual channel is getting arbitrated.
+.It Li QPI_TX_STALLED_MULTI_FLIT.NCS.LINK_0
+.Pq Event 41H , Umask 04H
+Counts cycles the Quickpath outbound link 0 Non-Coherent Standard virtual
+channel is stalled due to lack of VNA and VN0 credits. Note that this event
+does not filter out when a flit would not have been selected for arbitration
+because another virtual channel is getting arbitrated.
+.It Li QPI_TX_STALLED_MULTI_FLIT.DRS.LINK_1
+.Pq Event 41H , Umask 08H
+Counts cycles the Quickpath outbound link 1 Data ResponSe virtual channel is
+stalled due to lack of VNA and VN0 credits. Note that this event does not
+filter out when a flit would not have been selected for arbitration because
+another virtual channel is getting arbitrated.
+.It Li QPI_TX_STALLED_MULTI_FLIT.NCB.LINK_1
+.Pq Event 41H , Umask 10H
+Counts cycles the Quickpath outbound link 1 Non-Coherent Bypass virtual
+channel is stalled due to lack of VNA and VN0 credits. Note that this event
+does not filter out when a flit would not have been selected for arbitration
+because another virtual channel is getting arbitrated.
+.It Li QPI_TX_STALLED_MULTI_FLIT.NCS.LINK_1
+.Pq Event 41H , Umask 20H
+Counts cycles the Quickpath outbound link 1 Non-Coherent Standard virtual
+channel is stalled due to lack of VNA and VN0 credits. Note that this event
+does not filter out when a flit would not have been selected for arbitration
+because another virtual channel is getting arbitrated.
+.It Li QPI_TX_STALLED_MULTI_FLIT.LINK_0
+.Pq Event 41H , Umask 07H
+Counts cycles the Quickpath outbound link 0 virtual channels are stalled due
+to lack of VNA and VN0 credits. Note that this event does not filter out
+when a flit would not have been selected for arbitration because another
+virtual channel is getting arbitrated.
+.It Li QPI_TX_STALLED_MULTI_FLIT.LINK_1
+.Pq Event 41H , Umask 38H
+Counts cycles the Quickpath outbound link 1 virtual channels are stalled due
+to lack of VNA and VN0 credits. Note that this event does not filter out
+when a flit would not have been selected for arbitration because another
+virtual channel is getting arbitrated.
+.It Li QPI_TX_HEADER.BUSY.LINK_0
+.Pq Event 42H , Umask 02H
+Number of cycles that the header buffer in the Quickpath Interface outbound
+link 0 is busy.
+.It Li QPI_TX_HEADER.BUSY.LINK_1
+.Pq Event 42H , Umask 08H
+Number of cycles that the header buffer in the Quickpath Interface outbound
+link 1 is busy.
+.It Li QPI_RX_NO_PPT_CREDIT.STALLS.LINK_0
+.Pq Event 43H , Umask 01H
+Number of cycles that snoop packets incoming to the Quickpath Interface link
+0 are stalled and not sent to the GQ because the GQ Peer Probe Tracker (PPT)
+does not have any available entries.
+.It Li QPI_RX_NO_PPT_CREDIT.STALLS.LINK_1
+.Pq Event 43H , Umask 02H
+Number of cycles that snoop packets incoming to the Quickpath Interface link
+1 are stalled and not sent to the GQ because the GQ Peer Probe Tracker (PPT)
+does not have any available entries.
+.It Li DRAM_OPEN.CH0
+.Pq Event 60H , Umask 01H
+Counts number of DRAM Channel 0 open commands issued either for read or
+write. To read or write data, the referenced DRAM page must first be opened.
+.It Li DRAM_OPEN.CH1
+.Pq Event 60H , Umask 02H
+Counts number of DRAM Channel 1 open commands issued either for read or
+write. To read or write data, the referenced DRAM page must first be opened.
+.It Li DRAM_OPEN.CH2
+.Pq Event 60H , Umask 04H
+Counts number of DRAM Channel 2 open commands issued either for read or
+write. To read or write data, the referenced DRAM page must first be opened.
+.It Li DRAM_PAGE_CLOSE.CH0
+.Pq Event 61H , Umask 01H
+DRAM channel 0 command issued to CLOSE a page due to page idle timer
+expiration. Closing a page is done by issuing a precharge.
+.It Li DRAM_PAGE_CLOSE.CH1
+.Pq Event 61H , Umask 02H
+DRAM channel 1 command issued to CLOSE a page due to page idle timer
+expiration. Closing a page is done by issuing a precharge.
+.It Li DRAM_PAGE_CLOSE.CH2
+.Pq Event 61H , Umask 04H
+DRAM channel 2 command issued to CLOSE a page due to page idle timer
+expiration. Closing a page is done by issuing a precharge.
+.It Li DRAM_PAGE_MISS.CH0
+.Pq Event 62H , Umask 01H
+Counts the number of precharges (PRE) that were issued to DRAM channel 0
+because there was a page miss. A page miss refers to a situation in which a
+page is currently open and another page from the same bank needs to be
+opened. The new page experiences a page miss. Closing of the old page is
+done by issuing a precharge.
+.It Li DRAM_PAGE_MISS.CH1
+.Pq Event 62H , Umask 02H
+Counts the number of precharges (PRE) that were issued to DRAM channel 1
+because there was a page miss. A page miss refers to a situation in which a
+page is currently open and another page from the same bank needs to be
+opened. The new page experiences a page miss. Closing of the old page is
+done by issuing a precharge.
+.It Li DRAM_PAGE_MISS.CH2
+.Pq Event 62H , Umask 04H
+Counts the number of precharges (PRE) that were issued to DRAM channel 2
+because there was a page miss. A page miss refers to a situation in which a
+page is currently open and another page from the same bank needs to be
+opened. The new page experiences a page miss. Closing of the old page is
+done by issuing a precharge.
+.It Li DRAM_READ_CAS.CH0
+.Pq Event 63H , Umask 01H
+Counts the number of times a read CAS command was issued on DRAM channel 0.
+.It Li DRAM_READ_CAS.AUTOPRE_CH0
+.Pq Event 63H , Umask 02H
+Counts the number of times a read CAS command was issued on DRAM channel 0
+where the command issued used the auto-precharge (auto page close) mode.
+.It Li DRAM_READ_CAS.CH1
+.Pq Event 63H , Umask 04H
+Counts the number of times a read CAS command was issued on DRAM channel 1.
+.It Li DRAM_READ_CAS.AUTOPRE_CH1
+.Pq Event 63H , Umask 08H
+Counts the number of times a read CAS command was issued on DRAM channel 1
+where the command issued used the auto-precharge (auto page close) mode.
+.It Li DRAM_READ_CAS.CH2
+.Pq Event 63H , Umask 10H
+Counts the number of times a read CAS command was issued on DRAM channel 2.
+.It Li DRAM_READ_CAS.AUTOPRE_CH2
+.Pq Event 63H , Umask 20H
+Counts the number of times a read CAS command was issued on DRAM channel 2
+where the command issued used the auto-precharge (auto page close) mode.
+.It Li DRAM_WRITE_CAS.CH0
+.Pq Event 64H , Umask 01H
+Counts the number of times a write CAS command was issued on DRAM channel 0.
+.It Li DRAM_WRITE_CAS.AUTOPRE_CH0
+.Pq Event 64H , Umask 02H
+Counts the number of times a write CAS command was issued on DRAM channel 0
+where the command issued used the auto-precharge (auto page close) mode.
+.It Li DRAM_WRITE_CAS.CH1
+.Pq Event 64H , Umask 04H
+Counts the number of times a write CAS command was issued on DRAM channel 1.
+.It Li DRAM_WRITE_CAS.AUTOPRE_CH1
+.Pq Event 64H , Umask 08H
+Counts the number of times a write CAS command was issued on DRAM channel 1
+where the command issued used the auto-precharge (auto page close) mode.
+.It Li DRAM_WRITE_CAS.CH2
+.Pq Event 64H , Umask 10H
+Counts the number of times a write CAS command was issued on DRAM channel 2.
+.It Li DRAM_WRITE_CAS.AUTOPRE_CH2
+.Pq Event 64H , Umask 20H
+Counts the number of times a write CAS command was issued on DRAM channel 2
+where the command issued used the auto-precharge (auto page close) mode.
+.It Li DRAM_REFRESH.CH0
+.Pq Event 65H , Umask 01H
+Counts number of DRAM channel 0 refresh commands. DRAM loses data content
+over time. In order to keep correct data content, the data values have to be
+refreshed periodically.
+.It Li DRAM_REFRESH.CH1
+.Pq Event 65H , Umask 02H
+Counts number of DRAM channel 1 refresh commands. DRAM loses data content
+over time. In order to keep correct data content, the data values have to be
+refreshed periodically.
+.It Li DRAM_REFRESH.CH2
+.Pq Event 65H , Umask 04H
+Counts number of DRAM channel 2 refresh commands. DRAM loses data content
+over time. In order to keep correct data content, the data values have to be
+refreshed periodically.
+.It Li DRAM_PRE_ALL.CH0
+.Pq Event 66H , Umask 01H
+Counts number of DRAM Channel 0 precharge-all (PREALL) commands that close
+all open pages in a rank. PREALL is issued when the DRAM needs to be
+refreshed or needs to go into a power down mode.
+.It Li DRAM_PRE_ALL.CH1
+.Pq Event 66H , Umask 02H
+Counts number of DRAM Channel 1 precharge-all (PREALL) commands that close
+all open pages in a rank. PREALL is issued when the DRAM needs to be
+refreshed or needs to go into a power down mode.
+.It Li DRAM_PRE_ALL.CH2
+.Pq Event 66H , Umask 04H
+Counts number of DRAM Channel 2 precharge-all (PREALL) commands that close
+all open pages in a rank. PREALL is issued when the DRAM needs to be
+refreshed or needs to go into a power down mode.
+.El
+.Sh SEE ALSO
+.Xr pmc 3 ,
+.Xr pmc.atom 3 ,
+.Xr pmc.core 3 ,
+.Xr pmc.iaf 3 ,
+.Xr pmc.ucf 3 ,
+.Xr pmc.k7 3 ,
+.Xr pmc.k8 3 ,
+.Xr pmc.p4 3 ,
+.Xr pmc.p5 3 ,
+.Xr pmc.p6 3 ,
+.Xr pmc.corei7 3 ,
+.Xr pmc.westmere 3 ,
+.Xr pmc.westmereuc 3 ,
+.Xr pmc.tsc 3 ,
+.Xr pmc_cpuinfo 3 ,
+.Xr pmclog 3 ,
+.Xr hwpmc 4
+.Sh HISTORY
+The
+.Nm pmc
+library first appeared in
+.Fx 6.0 .
+.Sh AUTHORS
+The
+.Lb libpmc
+library was written by
+.An "Joseph Koshy"
+.Aq jkoshy@FreeBSD.org .
diff --git a/lib/libpmc/pmc.ucf.3 b/lib/libpmc/pmc.ucf.3
new file mode 100644
index 0000000..c8f2468
--- /dev/null
+++ b/lib/libpmc/pmc.ucf.3
@@ -0,0 +1,115 @@
+.\" Copyright (c) 2010 Fabien Thomas.  All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" This software is provided by Joseph Koshy ``as is'' and
+.\" any express or implied warranties, including, but not limited to, the
+.\" implied warranties of merchantability and fitness for a particular purpose
+.\" are disclaimed.  in no event shall Joseph Koshy be liable
+.\" for any direct, indirect, incidental, special, exemplary, or consequential
+.\" damages (including, but not limited to, procurement of substitute goods
+.\" or services; loss of use, data, or profits; or business interruption)
+.\" however caused and on any theory of liability, whether in contract, strict
+.\" liability, or tort (including negligence or otherwise) arising in any way
+.\" out of the use of this software, even if advised of the possibility of
+.\" such damage.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd March 30, 2010
+.Os
+.Dt PMC.UCF 3
+.Sh NAME
+.Nm pmc.ucf
+.Nd measurement events for
+.Tn Intel
+uncore fixed function performance counters.
+.Sh LIBRARY
+.Lb libpmc
+.Sh SYNOPSIS
+.In pmc.h
+.Sh DESCRIPTION
+Each fixed-function PMC measures a specific hardware event.
+The number of fixed-function PMCs implemented in a CPU can vary.
+The number of fixed-function PMCs present can be determined at runtime
+by using function
+.Xr pmc_cpuinfo 3 .
+.Pp
+Intel uncore fixed-function PMCs are documented in
+.Rs
+.%B "Intel(R) 64 and IA-32 Architectures Software Developes Manual"
+.%T "Volume 3B: System Programming Guide, Part 2"
+.%N "Order Number: 253669-033US"
+.%D December 2009
+.%Q "Intel Corporation"
+.Re
+.Pp
+.Ss PMC Capabilities
+Fixed-function PMCs support the following capabilities:
+.Bl -column "PMC_CAP_INTERRUPT" "Support"
+.It Em Capability Ta Em Support
+.It PMC_CAP_CASCADE Ta \&No
+.It PMC_CAP_EDGE Ta \&No
+.It PMC_CAP_INTERRUPT Ta \&No
+.It PMC_CAP_INVERT Ta \&No
+.It PMC_CAP_READ Ta Yes
+.It PMC_CAP_PRECISE Ta \&No
+.It PMC_CAP_SYSTEM Ta \&No
+.It PMC_CAP_TAGGING Ta \&No
+.It PMC_CAP_THRESHOLD Ta \&No
+.It PMC_CAP_USER Ta \&No
+.It PMC_CAP_WRITE Ta Yes
+.El
+.Ss Class Name Prefix
+These PMCs are named using a class name prefix of
+.Dq Li ucf- .
+.Ss Event Specifiers (Fixed Function PMCs)
+The fixed function PMCs are selectable using the following
+event names:
+.Bl -tag -width indent
+.It Li UCLOCK
+.Pq Fixed Function Counter 0
+The fixed-function uncore counter increments at the rate of the U-clock.
+The frequency of the uncore clock domain can be determined from the uncore
+clock ratio which is available in the PCI configuration space register at
+offset C0H under device number 0 and Function 0.
+.El
+.Sh SEE ALSO
+.Xr pmc 3 ,
+.Xr pmc.atom 3 ,
+.Xr pmc.core 3 ,
+.Xr pmc.core2 3 ,
+.Xr pmc.iaf 3 ,
+.Xr pmc.k7 3 ,
+.Xr pmc.k8 3 ,
+.Xr pmc.p4 3 ,
+.Xr pmc.p5 3 ,
+.Xr pmc.p6 3 ,
+.Xr pmc.corei7 3 ,
+.Xr pmc.corei7uc 3 ,
+.Xr pmc.westmere 3 ,
+.Xr pmc.westmereuc 3 ,
+.Xr pmc.tsc 3 ,
+.Xr pmc_cpuinfo 3 ,
+.Xr pmclog 3 ,
+.Xr hwpmc 4
+.Sh HISTORY
+The
+.Nm pmc
+library first appeared in
+.Fx 6.0 .
+.Sh AUTHORS
+The
+.Lb libpmc
+library was written by
+.An "Joseph Koshy"
+.Aq jkoshy@FreeBSD.org .
+
+
diff --git a/lib/libpmc/pmc.westmere.3 b/lib/libpmc/pmc.westmere.3
new file mode 100644
index 0000000..0b87e50
--- /dev/null
+++ b/lib/libpmc/pmc.westmere.3
@@ -0,0 +1,1329 @@
+.\" Copyright (c) 2010 Fabien Thomas.  All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" This software is provided by Joseph Koshy ``as is'' and
+.\" any express or implied warranties, including, but not limited to, the
+.\" implied warranties of merchantability and fitness for a particular purpose
+.\" are disclaimed.  in no event shall Joseph Koshy be liable
+.\" for any direct, indirect, incidental, special, exemplary, or consequential
+.\" damages (including, but not limited to, procurement of substitute goods
+.\" or services; loss of use, data, or profits; or business interruption)
+.\" however caused and on any theory of liability, whether in contract, strict
+.\" liability, or tort (including negligence or otherwise) arising in any way
+.\" out of the use of this software, even if advised of the possibility of
+.\" such damage.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd March 24, 2010
+.Os
+.Dt PMC.WESTMERE 3
+.Sh NAME
+.Nm pmc.westmere
+.Nd measurement events for
+.Tn Intel
+.Tn Westmere
+family CPUs
+.Sh LIBRARY
+.Lb libpmc
+.Sh SYNOPSIS
+.In pmc.h
+.Sh DESCRIPTION
+.Tn Intel
+.Tn "Westmere"
+CPUs contain PMCs conforming to version 2 of the
+.Tn Intel
+performance measurement architecture.
+These CPUs may contain up to three classes of PMCs:
+.Bl -tag -width "Li PMC_CLASS_IAP"
+.It Li PMC_CLASS_IAF
+Fixed-function counters that count only one hardware event per counter.
+.It Li PMC_CLASS_IAP
+Programmable counters that may be configured to count one of a defined
+set of hardware events.
+.El
+.Pp
+The number of PMCs available in each class and their widths need to be
+determined at run time by calling
+.Xr pmc_cpuinfo 3 .
+.Pp
+Intel Westmere PMCs are documented in
+.Rs
+.%B "Intel(R) 64 and IA-32 Architectures Software Developes Manual"
+.%T "Volume 3B: System Programming Guide, Part 2"
+.%N "Order Number: 253669-033US"
+.%D December 2009
+.%Q "Intel Corporation"
+.Re
+.Ss WESTMERE FIXED FUNCTION PMCS
+These PMCs and their supported events are documented in
+.Xr pmc.iaf 3 .
+.Ss WESTMERE PROGRAMMABLE PMCS
+The programmable PMCs support the following capabilities:
+.Bl -column "PMC_CAP_INTERRUPT" "Support"
+.It Em Capability Ta Em Support
+.It PMC_CAP_CASCADE Ta \&No
+.It PMC_CAP_EDGE Ta Yes
+.It PMC_CAP_INTERRUPT Ta Yes
+.It PMC_CAP_INVERT Ta Yes
+.It PMC_CAP_READ Ta Yes
+.It PMC_CAP_PRECISE Ta \&No
+.It PMC_CAP_SYSTEM Ta Yes
+.It PMC_CAP_TAGGING Ta \&No
+.It PMC_CAP_THRESHOLD Ta Yes
+.It PMC_CAP_USER Ta Yes
+.It PMC_CAP_WRITE Ta Yes
+.El
+.Ss Event Qualifiers
+Event specifiers for these PMCs support the following common
+qualifiers:
+.Bl -tag -width indent
+.It Li rsp= Ns Ar value
+Configure the Off-core Response bits.
+.Bl -tag -width indent
+.It Li DMND_DATA_RD
+Counts the number of demand and DCU prefetch data reads of full
+and partial cachelines as well as demand data page table entry
+cacheline reads. Does not count L2 data read prefetches or
+instruction fetches.
+.It Li DMND_RFO
+Counts the number of demand and DCU prefetch reads for ownership
+(RFO) requests generated by a write to data cacheline. Does not
+count L2 RFO.
+.It Li DMND_IFETCH
+Counts the number of demand and DCU prefetch instruction cacheline
+reads. Does not count L2 code read prefetches.
+WB
+Counts the number of writeback (modified to exclusive) transactions.
+.It Li PF_DATA_RD
+Counts the number of data cacheline reads generated by L2 prefetchers.
+.It Li PF_RFO
+Counts the number of RFO requests generated by L2 prefetchers.
+.It Li PF_IFETCH
+Counts the number of code reads generated by L2 prefetchers.
+.It Li OTHER
+Counts one of the following transaction types, including L3 invalidate,
+I/O, full or partial writes, WC or non-temporal stores, CLFLUSH, Fences,
+lock, unlock, split lock.
+.It Li UNCORE_HIT
+L3 Hit: local or remote home requests that hit L3 cache in the uncore
+with no coherency actions required (snooping).
+.It Li OTHER_CORE_HIT_SNP
+L3 Hit: local or remote home requests that hit L3 cache in the uncore
+and was serviced by another core with a cross core snoop where no modified
+copies were found (clean).
+.It Li OTHER_CORE_HITM
+L3 Hit: local or remote home requests that hit L3 cache in the uncore
+and was serviced by another core with a cross core snoop where modified
+copies were found (HITM).
+.It Li REMOTE_CACHE_FWD
+L3 Miss: local homed requests that missed the L3 cache and was serviced
+by forwarded data following a cross package snoop where no modified
+copies found. (Remote home requests are not counted)
+.It Li REMOTE_DRAM
+L3 Miss: remote home requests that missed the L3 cache and were serviced
+by remote DRAM.
+.It Li LOCAL_DRAM
+L3 Miss: local home requests that missed the L3 cache and were serviced
+by local DRAM.
+.It Li NON_DRAM
+Non-DRAM requests that were serviced by IOH.
+.El
+.It Li cmask= Ns Ar value
+Configure the PMC to increment only if the number of configured
+events measured in a cycle is greater than or equal to
+.Ar value .
+.It Li edge
+Configure the PMC to count the number of de-asserted to asserted
+transitions of the conditions expressed by the other qualifiers.
+If specified, the counter will increment only once whenever a
+condition becomes true, irrespective of the number of clocks during
+which the condition remains true.
+.It Li inv
+Invert the sense of comparison when the
+.Dq Li cmask
+qualifier is present, making the counter increment when the number of
+events per cycle is less than the value specified by the
+.Dq Li cmask
+qualifier.
+.It Li os
+Configure the PMC to count events happening at processor privilege
+level 0.
+.It Li usr
+Configure the PMC to count events occurring at privilege levels 1, 2
+or 3.
+.El
+.Pp
+If neither of the
+.Dq Li os
+or
+.Dq Li usr
+qualifiers are specified, the default is to enable both.
+.Ss Event Specifiers (Programmable PMCs)
+Westmere programmable PMCs support the following events:
+.Bl -tag -width indent
+.It Li LOAD_BLOCK.OVERLAP_STORE
+.Pq Event 03H , Umask 02H
+Loads that partially overlap an earlier store
+.It Li SB_DRAIN.ANY
+.Pq Event 04H , Umask 07H
+All Store buffer stall cycles
+.It Li MISALIGN_MEMORY.STORE
+.Pq Event 05H , Umask 02H
+All store referenced with misaligned address
+.It Li STORE_BLOCKS.AT_RET
+.Pq Event 06H , Umask 04H
+Counts number of loads delayed with at-Retirement block code. The following
+loads need to be executed at retirement and wait for all senior stores on
+the same thread to be drained: load splitting across 4K boundary (page
+split), load accessing uncacheable (UC or USWC) memory, load lock, and load
+with page table in UC or USWC memory region.
+.It Li STORE_BLOCKS.L1D_BLOCK
+.Pq Event 06H , Umask 08H
+Cacheable loads delayed with L1D block code
+.It Li PARTIAL_ADDRESS_ALIAS
+.Pq Event 07H , Umask 01H
+Counts false dependency due to partial address aliasing
+.It Li DTLB_LOAD_MISSES.ANY
+.Pq Event 08H , Umask 01H
+Counts all load misses that cause a page walk
+.It Li DTLB_LOAD_MISSES.WALK_COMPLETED
+.Pq Event 08H , Umask 02H
+Counts number of completed page walks due to load miss in the STLB.
+.It Li DTLB_LOAD_MISSES.WALK_CYCLES
+.Pq Event 08H , Umask 04H
+Cycles PMH is busy with a page walk due to a load miss in the STLB.
+.It Li DTLB_LOAD_MISSES.STLB_HIT
+.Pq Event 08H , Umask 10H
+Number of cache load STLB hits
+.It Li DTLB_LOAD_MISSES.PDE_MISS
+.Pq Event 08H , Umask 20H
+Number of DTLB cache load misses where the low part of the linear to
+physical address translation was missed.
+.It Li MEM_INST_RETIRED.LOADS
+.Pq Event 0BH , Umask 01H
+Counts the number of instructions with an architecturally-visible store
+retired on the architected path.
+In conjunction with ld_lat facility
+.It Li MEM_INST_RETIRED.STORES
+.Pq Event 0BH , Umask 02H
+Counts the number of instructions with an architecturally-visible store
+retired on the architected path.
+In conjunction with ld_lat facility
+.It Li MEM_INST_RETIRED.LATENCY_ABOVE_THRESHOLD
+.Pq Event 0BH , Umask 10H
+Counts the number of instructions exceeding the latency specified with
+ld_lat facility.
+In conjunction with ld_lat facility
+.It Li MEM_STORE_RETIRED.DTLB_MISS
+.Pq Event 0CH , Umask 01H
+The event counts the number of retired stores that missed the DTLB. The DTLB
+miss is not counted if the store operation causes a fault. Does not counter
+prefetches. Counts both primary and secondary misses to the TLB
+.It Li UOPS_ISSUED.ANY
+.Pq Event 0EH , Umask 01H
+Counts the number of Uops issued by the Register Allocation Table to the
+Reservation Station, i.e. the UOPs issued from the front end to the back
+end.
+.It Li UOPS_ISSUED.STALLED_CYCLES
+.Pq Event 0EH , Umask 01H
+Counts the number of cycles no Uops issued by the Register Allocation Table
+to the Reservation Station, i.e. the UOPs issued from the front end to the
+back end.
+set invert=1, cmask = 1
+.It Li UOPS_ISSUED.FUSED
+.Pq Event 0EH , Umask 02H
+Counts the number of fused Uops that were issued from the Register
+Allocation Table to the Reservation Station.
+.It Li MEM_UNCORE_RETIRED.LOCAL_HITM
+.Pq Event 0FH , Umask 02H
+Load instructions retired that HIT modified data in sibling core (Precise
+Event)
+.It Li MEM_UNCORE_RETIRED.LOCAL_DRAM_AND_REMOTE_CACHE_HIT
+.Pq Event 0FH , Umask 08H
+Load instructions retired local dram and remote cache HIT data sources
+(Precise Event)
+.It Li MEM_UNCORE_RETIRED.LOCAL_DRAM
+.Pq Event 0FH , Umask 10H
+Load instructions retired with a data source of local DRAM or locally homed
+remote cache HITM (Precise Event)
+.It Li MEM_UNCORE_RETIRED.REMOTE_DRAM
+.Pq Event 0FH , Umask 20H
+Load instructions retired remote DRAM and remote home-remote cache HITM
+(Precise Event)
+.It Li MEM_UNCORE_RETIRED.UNCACHEABLE
+.Pq Event 0FH , Umask 80H
+Load instructions retired I/O (Precise Event)
+.It Li FP_COMP_OPS_EXE.X87
+.Pq Event 10H , Umask 01H
+Counts the number of FP Computational Uops Executed. The number of FADD,
+FSUB, FCOM, FMULs, integer MULsand IMULs, FDIVs, FPREMs, FSQRTS, integer
+DIVs, and IDIVs. This event does not distinguish an FADD used in the middle
+of a transcendental flow from a separate FADD instruction.
+.It Li FP_COMP_OPS_EXE.MMX
+.Pq Event 10H , Umask 02H
+Counts number of MMX Uops executed.
+.It Li FP_COMP_OPS_EXE.SSE_FP
+.Pq Event 10H , Umask 04H
+Counts number of SSE and SSE2 FP uops executed.
+.It Li FP_COMP_OPS_EXE.SSE2_INTEGER
+.Pq Event 10H , Umask 08H
+Counts number of SSE2 integer uops executed.
+.It Li FP_COMP_OPS_EXE.SSE_FP_PACKED
+.Pq Event 10H , Umask 10H
+Counts number of SSE FP packed uops executed.
+.It Li FP_COMP_OPS_EXE.SSE_FP_SCALAR
+.Pq Event 10H , Umask 20H
+Counts number of SSE FP scalar uops executed.
+.It Li FP_COMP_OPS_EXE.SSE_SINGLE_PRECISION
+.Pq Event 10H , Umask 40H
+Counts number of SSE* FP single precision uops executed.
+.It Li FP_COMP_OPS_EXE.SSE_DOUBLE_PRECISION
+.Pq Event 10H , Umask 80H
+Counts number of SSE* FP double precision uops executed.
+.It Li SIMD_INT_128.PACKED_MPY
+.Pq Event 12H , Umask 01H
+Counts number of 128 bit SIMD integer multiply operations.
+.It Li SIMD_INT_128.PACKED_SHIFT
+.Pq Event 12H , Umask 02H
+Counts number of 128 bit SIMD integer shift operations.
+.It Li SIMD_INT_128.PACK
+.Pq Event 12H , Umask 04H
+Counts number of 128 bit SIMD integer pack operations.
+.It Li SIMD_INT_128.UNPACK
+.Pq Event 12H , Umask 08H
+Counts number of 128 bit SIMD integer unpack operations.
+.It Li SIMD_INT_128.PACKED_LOGICAL
+.Pq Event 12H , Umask 10H
+Counts number of 128 bit SIMD integer logical operations.
+.It Li SIMD_INT_128.PACKED_ARITH
+.Pq Event 12H , Umask 20H
+Counts number of 128 bit SIMD integer arithmetic operations.
+.It Li SIMD_INT_128.SHUFFLE_MOVE
+.Pq Event 12H , Umask 40H
+Counts number of 128 bit SIMD integer shuffle and move operations.
+.It Li LOAD_DISPATCH.RS
+.Pq Event 13H , Umask 01H
+Counts number of loads dispatched from the Reservation Station that bypass
+the Memory Order Buffer.
+.It Li LOAD_DISPATCH.RS_DELAYED
+.Pq Event 13H , Umask 02H
+Counts the number of delayed RS dispatches at the stage latch. If an RS
+dispatch can not bypass to LB, it has another chance to dispatch from the
+one-cycle delayed staging latch before it is written into the LB.
+.It Li LOAD_DISPATCH.MOB
+.Pq Event 13H , Umask 04H
+Counts the number of loads dispatched from the Reservation Station to the
+Memory Order Buffer.
+.It Li LOAD_DISPATCH.ANY
+.Pq Event 13H , Umask 07H
+Counts all loads dispatched from the Reservation Station.
+.It Li ARITH.CYCLES_DIV_BUSY
+.Pq Event 14H , Umask 01H
+Counts the number of cycles the divider is busy executing divide or square
+root operations. The divide can be integer, X87 or Streaming SIMD Extensions
+(SSE). The square root operation can be either X87 or SSE.
+Set 'edge =1, invert=1, cmask=1' to count the number of divides.
+Count may be incorrect When SMT is on
+.It Li ARITH.MUL
+.Pq Event 14H , Umask 02H
+Counts the number of multiply operations executed. This includes integer as
+well as floating point multiply operations but excludes DPPS mul and MPSAD.
+Count may be incorrect When SMT is on
+.It Li INST_QUEUE_WRITES
+.Pq Event 17H , Umask 01H
+Counts the number of instructions written into the instruction queue every
+cycle.
+.It Li INST_DECODED.DEC0
+.Pq Event 18H , Umask 01H
+Counts number of instructions that require decoder 0 to be decoded. Usually,
+this means that the instruction maps to more than 1 uop
+.It Li TWO_UOP_INSTS_DECODED
+.Pq Event 19H , Umask 01H
+An instruction that generates two uops was decoded
+.It Li INST_QUEUE_WRITE_CYCLES
+.Pq Event 1EH , Umask 01H
+This event counts the number of cycles during which instructions are written
+to the instruction queue. Dividing this counter by the number of
+instructions written to the instruction queue (INST_QUEUE_WRITES) yields the
+average number of instructions decoded each cycle. If this number is less
+than four and the pipe stalls, this indicates that the decoder is failing to
+decode enough instructions per cycle to sustain the 4-wide pipeline.
+If SSE* instructions that are 6 bytes or longer arrive one after another,
+then front end throughput may limit execution speed. In such case,
+.It Li LSD_OVERFLOW
+.Pq Event 20H , Umask 01H
+Number of loops that can not stream from the instruction queue.
+.It Li L2_RQSTS.LD_HIT
+.Pq Event 24H , Umask 01H
+Counts number of loads that hit the L2 cache. L2 loads include both L1D
+demand misses as well as L1D prefetches. L2 loads can be rejected for
+various reasons. Only non rejected loads are counted.
+.It Li L2_RQSTS.LD_MISS
+.Pq Event 24H , Umask 02H
+Counts the number of loads that miss the L2 cache. L2 loads include both L1D
+demand misses as well as L1D prefetches.
+.It Li L2_RQSTS.LOADS
+.Pq Event 24H , Umask 03H
+Counts all L2 load requests. L2 loads include both L1D demand misses as well
+as L1D prefetches.
+.It Li L2_RQSTS.RFO_HIT
+.Pq Event 24H , Umask 04H
+Counts the number of store RFO requests that hit the L2 cache. L2 RFO
+requests include both L1D demand RFO misses as well as L1D RFO prefetches.
+Count includes WC memory requests, where the data is not fetched but the
+permission to write the line is required.
+.It Li L2_RQSTS.RFO_MISS
+.Pq Event 24H , Umask 08H
+Counts the number of store RFO requests that miss the L2 cache. L2 RFO
+requests include both L1D demand RFO misses as well as L1D RFO prefetches.
+.It Li L2_RQSTS.RFOS
+.Pq Event 24H , Umask 0CH
+Counts all L2 store RFO requests. L2 RFO requests include both L1D demand
+RFO misses as well as L1D RFO prefetches..
+.It Li L2_RQSTS.IFETCH_HIT
+.Pq Event 24H , Umask 10H
+Counts number of instruction fetches that hit the L2 cache. L2 instruction
+fetches include both L1I demand misses as well as L1I instruction
+prefetches.
+.It Li L2_RQSTS.IFETCH_MISS
+.Pq Event 24H , Umask 20H
+Counts number of instruction fetches that miss the L2 cache. L2 instruction
+fetches include both L1I demand misses as well as L1I instruction
+prefetches.
+.It Li L2_RQSTS.IFETCHES
+.Pq Event 24H , Umask 30H
+Counts all instruction fetches. L2 instruction fetches include both L1I
+demand misses as well as L1I instruction prefetches.
+.It Li L2_RQSTS.PREFETCH_HIT
+.Pq Event 24H , Umask 40H
+Counts L2 prefetch hits for both code and data.
+.It Li L2_RQSTS.PREFETCH_MISS
+.Pq Event 24H , Umask 80H
+Counts L2 prefetch misses for both code and data.
+.It Li L2_RQSTS.PREFETCHES
+.Pq Event 24H , Umask C0H
+Counts all L2 prefetches for both code and data.
+.It Li L2_RQSTS.MISS
+.Pq Event 24H , Umask AAH
+Counts all L2 misses for both code and data.
+.It Li L2_RQSTS.REFERENCES
+.Pq Event 24H , Umask FFH
+Counts all L2 requests for both code and data.
+.It Li L2_DATA_RQSTS.DEMAND.I_STATE
+.Pq Event 26H , Umask 01H
+Counts number of L2 data demand loads where the cache line to be loaded is
+in the I (invalid) state, i.e. a cache miss. L2 demand loads are both L1D
+demand misses and L1D prefetches.
+.It Li L2_DATA_RQSTS.DEMAND.S_STATE
+.Pq Event 26H , Umask 02H
+Counts number of L2 data demand loads where the cache line to be loaded is
+in the S (shared) state. L2 demand loads are both L1D demand misses and L1D
+prefetches.
+.It Li L2_DATA_RQSTS.DEMAND.E_STATE
+.Pq Event 26H , Umask 04H
+Counts number of L2 data demand loads where the cache line to be loaded is
+in the E (exclusive) state. L2 demand loads are both L1D demand misses and
+L1D prefetches.
+.It Li L2_DATA_RQSTS.DEMAND.M_STATE
+.Pq Event 26H , Umask 08H
+Counts number of L2 data demand loads where the cache line to be loaded is
+in the M (modified) state. L2 demand loads are both L1D demand misses and
+L1D prefetches.
+.It Li L2_DATA_RQSTS.DEMAND.MESI
+.Pq Event 26H , Umask 0FH
+Counts all L2 data demand requests. L2 demand loads are both L1D demand
+misses and L1D prefetches.
+.It Li L2_DATA_RQSTS.PREFETCH.I_STATE
+.Pq Event 26H , Umask 10H
+Counts number of L2 prefetch data loads where the cache line to be loaded is
+in the I (invalid) state, i.e. a cache miss.
+.It Li L2_DATA_RQSTS.PREFETCH.S_STATE
+.Pq Event 26H , Umask 20H
+Counts number of L2 prefetch data loads where the cache line to be loaded is
+in the S (shared) state. A prefetch RFO will miss on an S state line, while
+a prefetch read will hit on an S state line.
+.It Li L2_DATA_RQSTS.PREFETCH.E_STATE
+.Pq Event 26H , Umask 40H
+Counts number of L2 prefetch data loads where the cache line to be loaded is
+in the E (exclusive) state.
+.It Li L2_DATA_RQSTS.PREFETCH.M_STATE
+.Pq Event 26H , Umask 80H
+Counts number of L2 prefetch data loads where the cache line to be loaded is
+in the M (modified) state.
+.It Li L2_DATA_RQSTS.PREFETCH.MESI
+.Pq Event 26H , Umask F0H
+Counts all L2 prefetch requests.
+.It Li L2_DATA_RQSTS.ANY
+.Pq Event 26H , Umask FFH
+Counts all L2 data requests.
+.It Li L2_WRITE.RFO.I_STATE
+.Pq Event 27H , Umask 01H
+Counts number of L2 demand store RFO requests where the cache line to be
+loaded is in the I (invalid) state, i.e, a cache miss. The L1D prefetcher
+does not issue a RFO prefetch.
+This is a demand RFO request
+.It Li L2_WRITE.RFO.S_STATE
+.Pq Event 27H , Umask 02H
+Counts number of L2 store RFO requests where the cache line to be loaded is
+in the S (shared) state. The L1D prefetcher does not issue a RFO prefetch,.
+This is a demand RFO request
+.It Li L2_WRITE.RFO.M_STATE
+.Pq Event 27H , Umask 08H
+Counts number of L2 store RFO requests where the cache line to be loaded is
+in the M (modified) state. The L1D prefetcher does not issue a RFO prefetch.
+This is a demand RFO request
+.It Li L2_WRITE.RFO.HIT
+.Pq Event 27H , Umask 0EH
+Counts number of L2 store RFO requests where the cache line to be loaded is
+in either the S, E or M states. The L1D prefetcher does not issue a RFO
+prefetch.
+This is a demand RFO request
+.It Li L2_WRITE.RFO.MESI
+.Pq Event 27H , Umask 0FH
+Counts all L2 store RFO requests.The L1D prefetcher does not issue a RFO
+prefetch.
+This is a demand RFO request
+.It Li L2_WRITE.LOCK.I_STATE
+.Pq Event 27H , Umask 10H
+Counts number of L2 demand lock RFO requests where the cache line to be
+loaded is in the I (invalid) state, i.e. a cache miss.
+.It Li L2_WRITE.LOCK.S_STATE
+.Pq Event 27H , Umask 20H
+Counts number of L2 lock RFO requests where the cache line to be loaded is
+in the S (shared) state.
+.It Li L2_WRITE.LOCK.E_STATE
+.Pq Event 27H , Umask 40H
+Counts number of L2 demand lock RFO requests where the cache line to be
+loaded is in the E (exclusive) state.
+.It Li L2_WRITE.LOCK.M_STATE
+.Pq Event 27H , Umask 80H
+Counts number of L2 demand lock RFO requests where the cache line to be
+loaded is in the M (modified) state.
+.It Li L2_WRITE.LOCK.HIT
+.Pq Event 27H , Umask E0H
+Counts number of L2 demand lock RFO requests where the cache line to be
+loaded is in either the S, E, or M state.
+.It Li L2_WRITE.LOCK.MESI
+.Pq Event 27H , Umask F0H
+Counts all L2 demand lock RFO requests.
+.It Li L1D_WB_L2.I_STATE
+.Pq Event 28H , Umask 01H
+Counts number of L1 writebacks to the L2 where the cache line to be written
+is in the I (invalid) state, i.e. a cache miss.
+.It Li L1D_WB_L2.S_STATE
+.Pq Event 28H , Umask 02H
+Counts number of L1 writebacks to the L2 where the cache line to be written
+is in the S state.
+.It Li L1D_WB_L2.E_STATE
+.Pq Event 28H , Umask 04H
+Counts number of L1 writebacks to the L2 where the cache line to be written
+is in the E (exclusive) state.
+.It Li L1D_WB_L2.M_STATE
+.Pq Event 28H , Umask 08H
+Counts number of L1 writebacks to the L2 where the cache line to be written
+is in the M (modified) state.
+.It Li L1D_WB_L2.MESI
+.Pq Event 28H , Umask 0FH
+Counts all L1 writebacks to the L2.
+.It Li L3_LAT_CACHE.REFERENCE
+.Pq Event 2EH , Umask 02H
+Counts uncore Last Level Cache references. Because cache hierarchy, cache
+sizes and other implementation-specific characteristics; value comparison to
+estimate performance differences is not recommended.
+see Table A-1
+.It Li L3_LAT_CACHE.MISS
+.Pq Event 2EH , Umask 01H
+Counts uncore Last Level Cache misses. Because cache hierarchy, cache sizes
+and other implementation-specific characteristics; value comparison to
+estimate performance differences is not recommended.
+see Table A-1
+.It Li CPU_CLK_UNHALTED.THREAD_P
+.Pq Event 3CH , Umask 00H
+Counts the number of thread cycles while the thread is not in a halt state.
+The thread enters the halt state when it is running the HLT instruction. The
+core frequency may change from time to time due to power or thermal
+throttling.
+see Table A-1
+.It Li CPU_CLK_UNHALTED.REF_P
+.Pq Event 3CH , Umask 01H
+Increments at the frequency of TSC when not halted.
+see Table A-1
+.It Li DTLB_MISSES.ANY
+.Pq Event 49H , Umask 01H
+Counts the number of misses in the STLB which causes a page walk.
+.It Li DTLB_MISSES.WALK_COMPLETED
+.Pq Event 49H , Umask 02H
+Counts number of misses in the STLB which resulted in a completed page walk.
+.It Li DTLB_MISSES.WALK_CYCLES
+.Pq Event 49H , Umask 04H
+Counts cycles of page walk due to misses in the STLB.
+.It Li DTLB_MISSES.STLB_HIT
+.Pq Event 49H , Umask 10H
+Counts the number of DTLB first level misses that hit in the second level
+TLB. This event is only relevant if the core contains multiple DTLB levels.
+.It Li DTLB_MISSES.LARGE_WALK_COMPLETED
+.Pq Event 49H , Umask 80H
+Counts number of completed large page walks due to misses in the STLB.
+.It Li LOAD_HIT_PRE
+.Pq Event 4CH , Umask 01H
+Counts load operations sent to the L1 data cache while a previous SSE
+prefetch instruction to the same cache line has started prefetching but has
+not yet finished.
+.It Li L1D_PREFETCH.REQUESTS
+.Pq Event 4EH , Umask 01H
+Counts number of hardware prefetch requests dispatched out of the prefetch
+FIFO.
+.It Li L1D_PREFETCH.MISS
+.Pq Event 4EH , Umask 02H
+Counts number of hardware prefetch requests that miss the L1D. There are two
+prefetchers in the L1D. A streamer, which predicts lines sequentially after
+this one should be fetched, and the IP prefetcher that remembers access
+patterns for the current instruction. The streamer prefetcher stops on an
+L1D hit, while the IP prefetcher does not.
+.It Li L1D_PREFETCH.TRIGGERS
+.Pq Event 4EH , Umask 04H
+Counts number of prefetch requests triggered by the Finite State Machine and
+pushed into the prefetch FIFO. Some of the prefetch requests are dropped due
+to overwrites or competition between the IP index prefetcher and streamer
+prefetcher. The prefetch FIFO contains 4 entries.
+.It Li EPT.WALK_CYCLES
+.Pq Event 4FH , Umask 10H
+Counts Extended Page walk cycles.
+.It Li L1D.REPL
+.Pq Event 51H , Umask 01H
+Counts the number of lines brought into the L1 data cache.
+Counter 0, 1 only
+.It Li L1D.M_REPL
+.Pq Event 51H , Umask 02H
+Counts the number of modified lines brought into the L1 data cache.
+Counter 0, 1 only
+.It Li L1D.M_EVICT
+.Pq Event 51H , Umask 04H
+Counts the number of modified lines evicted from the L1 data cache due to
+replacement.
+Counter 0, 1 only
+.It Li L1D.M_SNOOP_EVICT
+.Pq Event 51H , Umask 08H
+Counts the number of modified lines evicted from the L1 data cache due to
+snoop HITM intervention.
+Counter 0, 1 only
+.It Li L1D_CACHE_PREFETCH_LOCK_FB_HIT
+.Pq Event 52H , Umask 01H
+Counts the number of cacheable load lock speculated instructions accepted
+into the fill buffer.
+.It Li L1D_CACHE_LOCK_FB_HIT
+.Pq Event 53H , Umask 01H
+Counts the number of cacheable load lock speculated or retired instructions
+accepted into the fill buffer.
+.It Li OFFCORE_REQUESTS_OUTSTANDING.DEMAND.READ_DATA
+.Pq Event 60H , Umask 01H
+Counts weighted cycles of offcore demand data read requests. Does not
+include L2 prefetch requests.
+counter 0
+.It Li OFFCORE_REQUESTS_OUTSTANDING.DEMAND.READ_CODE
+.Pq Event 60H , Umask 02H
+Counts weighted cycles of offcore demand code read requests. Does not
+include L2 prefetch requests.
+counter 0
+.It Li OFFCORE_REQUESTS_OUTSTANDING.DEMAND.RFO
+.Pq Event 60H , Umask 04H
+Counts weighted cycles of offcore demand RFO requests. Does not include L2
+prefetch requests.
+counter 0
+.It Li OFFCORE_REQUESTS_OUTSTANDING.ANY.READ
+.Pq Event 60H , Umask 08H
+Counts weighted cycles of offcore read requests of any kind. Include L2
+prefetch requests.
+counter 0
+.It Li CACHE_LOCK_CYCLES.L1D_L2
+.Pq Event 63H , Umask 01H
+Cycle count during which the L1D and L2 are locked. A lock is asserted when
+there is a locked memory access, due to uncacheable memory, a locked
+operation that spans two cache lines, or a page walk from an uncacheable
+page table.
+Counter 0, 1 only. L1D and L2 locks have a very high performance penalty and
+it is highly recommended to avoid such accesses.
+.It Li CACHE_LOCK_CYCLES.L1D
+.Pq Event 63H , Umask 02H
+Counts the number of cycles that cacheline in the L1 data cache unit is
+locked.
+Counter 0, 1 only.
+.It Li IO_TRANSACTIONS
+.Pq Event 6CH , Umask 01H
+Counts the number of completed I/O transactions.
+.It Li L1I.HITS
+.Pq Event 80H , Umask 01H
+Counts all instruction fetches that hit the L1 instruction cache.
+.It Li L1I.MISSES
+.Pq Event 80H , Umask 02H
+Counts all instruction fetches that miss the L1I cache. This includes
+instruction cache misses, streaming buffer misses, victim cache misses and
+uncacheable fetches. An instruction fetch miss is counted only once and not
+once for every cycle it is outstanding.
+.It Li L1I.READS
+.Pq Event 80H , Umask 03H
+Counts all instruction fetches, including uncacheable fetches that bypass
+the L1I.
+.It Li L1I.CYCLES_STALLED
+.Pq Event 80H , Umask 04H
+Cycle counts for which an instruction fetch stalls due to a L1I cache miss,
+ITLB miss or ITLB fault.
+.It Li LARGE_ITLB.HIT
+.Pq Event 82H , Umask 01H
+Counts number of large ITLB hits.
+.It Li ITLB_MISSES.ANY
+.Pq Event 85H , Umask 01H
+Counts the number of misses in all levels of the ITLB which causes a page
+walk.
+.It Li ITLB_MISSES.WALK_COMPLETED
+.Pq Event 85H , Umask 02H
+Counts number of misses in all levels of the ITLB which resulted in a
+completed page walk.
+.It Li ITLB_MISSES.WALK_CYCLES
+.Pq Event 85H , Umask 04H
+Counts ITLB miss page walk cycles.
+.It Li ITLB_MISSES.LARGE_WALK_COMPLETED
+.Pq Event 85H , Umask 80H
+Counts number of completed large page walks due to misses in the STLB.
+.It Li ILD_STALL.LCP
+.Pq Event 87H , Umask 01H
+Cycles Instruction Length Decoder stalls due to length changing prefixes:
+66, 67 or REX.W (for EM64T) instructions which change the length of the
+decoded instruction.
+.It Li ILD_STALL.MRU
+.Pq Event 87H , Umask 02H
+Instruction Length Decoder stall cycles due to Brand Prediction Unit (PBU)
+Most Recently Used (MRU) bypass.
+.It Li ILD_STALL.IQ_FULL
+.Pq Event 87H , Umask 04H
+Stall cycles due to a full instruction queue.
+.It Li ILD_STALL.REGEN
+.Pq Event 87H , Umask 08H
+Counts the number of regen stalls.
+.It Li ILD_STALL.ANY
+.Pq Event 87H , Umask 0FH
+Counts any cycles the Instruction Length Decoder is stalled.
+.It Li BR_INST_EXEC.COND
+.Pq Event 88H , Umask 01H
+Counts the number of conditional near branch instructions executed, but not
+necessarily retired.
+.It Li BR_INST_EXEC.DIRECT
+.Pq Event 88H , Umask 02H
+Counts all unconditional near branch instructions excluding calls and
+indirect branches.
+.It Li BR_INST_EXEC.INDIRECT_NON_CALL
+.Pq Event 88H , Umask 04H
+Counts the number of executed indirect near branch instructions that are not
+calls.
+.It Li BR_INST_EXEC.NON_CALLS
+.Pq Event 88H , Umask 07H
+Counts all non call near branch instructions executed, but not necessarily
+retired.
+.It Li BR_INST_EXEC.RETURN_NEAR
+.Pq Event 88H , Umask 08H
+Counts indirect near branches that have a return mnemonic.
+.It Li BR_INST_EXEC.DIRECT_NEAR_CALL
+.Pq Event 88H , Umask 10H
+Counts unconditional near call branch instructions, excluding non call
+branch, executed.
+.It Li BR_INST_EXEC.INDIRECT_NEAR_CALL
+.Pq Event 88H , Umask 20H
+Counts indirect near calls, including both register and memory indirect,
+executed.
+.It Li BR_INST_EXEC.NEAR_CALLS
+.Pq Event 88H , Umask 30H
+Counts all near call branches executed, but not necessarily retired.
+.It Li BR_INST_EXEC.TAKEN
+.Pq Event 88H , Umask 40H
+Counts taken near branches executed, but not necessarily retired.
+.It Li BR_INST_EXEC.ANY
+.Pq Event 88H , Umask 7FH
+Counts all near executed branches (not necessarily retired). This includes
+only instructions and not micro-op branches. Frequent branching is not
+necessarily a major performance issue. However frequent branch
+mispredictions may be a problem.
+.It Li BR_MISP_EXEC.COND
+.Pq Event 89H , Umask 01H
+Counts the number of mispredicted conditional near branch instructions
+executed, but not necessarily retired.
+.It Li BR_MISP_EXEC.DIRECT
+.Pq Event 89H , Umask 02H
+Counts mispredicted macro unconditional near branch instructions, excluding
+calls and indirect branches (should always be 0).
+.It Li BR_MISP_EXEC.INDIRECT_NON_CALL
+.Pq Event 89H , Umask 04H
+Counts the number of executed mispredicted indirect near branch instructions
+that are not calls.
+.It Li BR_MISP_EXEC.NON_CALLS
+.Pq Event 89H , Umask 07H
+Counts mispredicted non call near branches executed, but not necessarily
+retired.
+.It Li BR_MISP_EXEC.RETURN_NEAR
+.Pq Event 89H , Umask 08H
+Counts mispredicted indirect branches that have a rear return mnemonic.
+.It Li BR_MISP_EXEC.DIRECT_NEAR_CALL
+.Pq Event 89H , Umask 10H
+Counts mispredicted non-indirect near calls executed, (should always be 0).
+.It Li BR_MISP_EXEC.INDIRECT_NEAR_CALL
+.Pq Event 89H , Umask 20H
+Counts mispredicted indirect near calls exeucted, including both register
+and memory indirect.
+.It Li BR_MISP_EXEC.NEAR_CALLS
+.Pq Event 89H , Umask 30H
+Counts all mispredicted near call branches executed, but not necessarily
+retired.
+.It Li BR_MISP_EXEC.TAKEN
+.Pq Event 89H , Umask 40H
+Counts executed mispredicted near branches that are taken, but not
+necessarily retired.
+.It Li BR_MISP_EXEC.ANY
+.Pq Event 89H , Umask 7FH
+Counts the number of mispredicted near branch instructions that were
+executed, but not necessarily retired.
+.It Li RESOURCE_STALLS.ANY
+.Pq Event A2H , Umask 01H
+Counts the number of Allocator resource related stalls. Includes register
+renaming buffer entries, memory buffer entries. In addition to resource
+related stalls, this event counts some other events. Includes stalls arising
+during branch misprediction recovery, such as if retirement of the
+mispredicted branch is delayed and stalls arising while store buffer is
+draining from synchronizing operations.
+Does not include stalls due to SuperQ (off core) queue full, too many cache
+misses, etc.
+.It Li RESOURCE_STALLS.LOAD
+.Pq Event A2H , Umask 02H
+Counts the cycles of stall due to lack of load buffer for load operation.
+.It Li RESOURCE_STALLS.RS_FULL
+.Pq Event A2H , Umask 04H
+This event counts the number of cycles when the number of instructions in
+the pipeline waiting for execution reaches the limit the processor can
+handle. A high count of this event indicates that there are long latency
+operations in the pipe (possibly load and store operations that miss the L2
+cache, or instructions dependent upon instructions further down the pipeline
+that have yet to retire.
+When RS is full, new instructions can not enter the reservation station and
+start execution.
+.It Li RESOURCE_STALLS.STORE
+.Pq Event A2H , Umask 08H
+This event counts the number of cycles that a resource related stall will
+occur due to the number of store instructions reaching the limit of the
+pipeline, (i.e. all store buffers are used). The stall ends when a store
+instruction commits its data to the cache or memory.
+.It Li RESOURCE_STALLS.ROB_FULL
+.Pq Event A2H , Umask 10H
+Counts the cycles of stall due to re- order buffer full.
+.It Li RESOURCE_STALLS.FPCW
+.Pq Event A2H , Umask 20H
+Counts the number of cycles while execution was stalled due to writing the
+floating-point unit (FPU) control word.
+.It Li RESOURCE_STALLS.MXCSR
+.Pq Event A2H , Umask 40H
+Stalls due to the MXCSR register rename occurring to close to a previous
+MXCSR rename. The MXCSR provides control and status for the MMX registers.
+.It Li RESOURCE_STALLS.OTHER
+.Pq Event A2H , Umask 80H
+Counts the number of cycles while execution was stalled due to other
+resource issues.
+.It Li MACRO_INSTS.FUSIONS_DECODED
+.Pq Event A6H , Umask 01H
+Counts the number of instructions decoded that are macro-fused but not
+necessarily executed or retired.
+.It Li BACLEAR_FORCE_IQ
+.Pq Event A7H , Umask 01H
+Counts number of times a BACLEAR was forced by the Instruction Queue. The IQ
+is also responsible for providing conditional branch prediciton direction
+based on a static scheme and dynamic data provided by the L2 Branch
+Prediction Unit. If the conditional branch target is not found in the Target
+Array and the IQ predicts that the branch is taken, then the IQ will force
+the Branch Address Calculator to issue a BACLEAR. Each BACLEAR asserted by
+the BAC generates approximately an 8 cycle bubble in the instruction fetch
+pipeline.
+.It Li LSD.UOPS
+.Pq Event A8H , Umask 01H
+Counts the number of micro-ops delivered by loop stream detector
+Use cmask=1 and invert to count cycles
+.It Li ITLB_FLUSH
+.Pq Event AEH , Umask 01H
+Counts the number of ITLB flushes
+.It Li OFFCORE_REQUESTS.DEMAND.READ_DATA
+.Pq Event B0H , Umask 01H
+Counts number of offcore demand data read requests. Does not count L2
+prefetch requests.
+.It Li OFFCORE_REQUESTS.DEMAND.READ_CODE
+.Pq Event B0H , Umask 02H
+Counts number of offcore demand code read requests. Does not count L2
+prefetch requests.
+.It Li OFFCORE_REQUESTS.DEMAND.RFO
+.Pq Event B0H , Umask 04H
+Counts number of offcore demand RFO requests. Does not count L2 prefetch
+requests.
+.It Li OFFCORE_REQUESTS.ANY.READ
+.Pq Event B0H , Umask 08H
+Counts number of offcore read requests. Includes L2 prefetch requests.
+.It Li OFFCORE_REQUESTS.ANY.RFO
+.Pq Event 80H , Umask 10H
+Counts number of offcore RFO requests. Includes L2 prefetch requests.
+.It Li OFFCORE_REQUESTS.L1D_WRITEBACK
+.Pq Event B0H , Umask 40H
+Counts number of L1D writebacks to the uncore.
+.It Li OFFCORE_REQUESTS.ANY
+.Pq Event B0H , Umask 80H
+Counts all offcore requests.
+.It Li UOPS_EXECUTED.PORT0
+.Pq Event B1H , Umask 01H
+Counts number of Uops executed that were issued on port 0. Port 0 handles
+integer arithmetic, SIMD and FP add Uops.
+.It Li UOPS_EXECUTED.PORT1
+.Pq Event B1H , Umask 02H
+Counts number of Uops executed that were issued on port 1. Port 1 handles
+integer arithmetic, SIMD, integer shift, FP multiply and FP divide Uops.
+.It Li UOPS_EXECUTED.PORT2_CORE
+.Pq Event B1H , Umask 04H
+Counts number of Uops executed that were issued on port 2. Port 2 handles
+the load Uops. This is a core count only and can not be collected per
+thread.
+.It Li UOPS_EXECUTED.PORT3_CORE
+.Pq Event B1H , Umask 08H
+Counts number of Uops executed that were issued on port 3. Port 3 handles
+store Uops. This is a core count only and can not be collected per thread.
+.It Li UOPS_EXECUTED.PORT4_CORE
+.Pq Event B1H , Umask 10H
+Counts number of Uops executed that where issued on port 4. Port 4 handles
+the value to be stored for the store Uops issued on port 3. This is a core
+count only and can not be collected per thread.
+.It Li UOPS_EXECUTED.CORE_ACTIVE_CYCLES_NO_PORT5
+.Pq Event B1H , Umask 1FH
+Counts number of cycles there are one or more uops being executed and were
+issued on ports 0-4. This is a core count only and can not be collected per
+thread.
+.It Li UOPS_EXECUTED.PORT5
+.Pq Event B1H , Umask 20H
+Counts number of Uops executed that where issued on port 5.
+.It Li UOPS_EXECUTED.CORE_ACTIVE_CYCLES
+.Pq Event B1H , Umask 3FH
+Counts number of cycles there are one or more uops being executed on any
+ports. This is a core count only and can not be collected per thread.
+.It Li UOPS_EXECUTED.PORT015
+.Pq Event B1H , Umask 40H
+Counts number of Uops executed that where issued on port 0, 1, or 5.
+use cmask=1, invert=1 to count stall cycles
+.It Li UOPS_EXECUTED.PORT234
+.Pq Event B1H , Umask 80H
+Counts number of Uops executed that where issued on port 2, 3, or 4.
+.It Li OFFCORE_REQUESTS_SQ_FULL
+.Pq Event B2H , Umask 01H
+Counts number of cycles the SQ is full to handle off-core requests.
+.It Li SNOOPQ_REQUESTS_OUTSTANDING.DATA
+.Pq Event B3H , Umask 01H
+Counts weighted cycles of snoopq requests for data. Counter 0 only
+Use cmask=1 to count cycles not empty.
+.It Li SNOOPQ_REQUESTS_OUTSTANDING.INVALIDATE
+.Pq Event B3H , Umask 02H
+Counts weighted cycles of snoopq invalidate requests. Counter 0 only
+Use cmask=1 to count cycles not empty.
+.It Li SNOOPQ_REQUESTS_OUTSTANDING.CODE
+.Pq Event B3H , Umask 04H
+Counts weighted cycles of snoopq requests for code. Counter 0 only
+Use cmask=1 to count cycles not empty.
+.It Li SNOOPQ_REQUESTS.CODE
+.Pq Event B4H , Umask 01H
+Counts the number of snoop code requests
+.It Li SNOOPQ_REQUESTS.DATA
+.Pq Event B4H , Umask 02H
+Counts the number of snoop data requests
+.It Li SNOOPQ_REQUESTS.INVALIDATE
+.Pq Event B4H , Umask 04H
+Counts the number of snoop invalidate requests
+.It Li OFF_CORE_RESPONSE_0
+.Pq Event B7H , Umask 01H
+see Section 30.6.1.3, Off-core Response Performance Monitoring in the
+Processor Core.
+Requires programming MSR 01A6H
+.It Li SNOOP_RESPONSE.HIT
+.Pq Event B8H , Umask 01H
+Counts HIT snoop response sent by this thread in response to a snoop
+request.
+.It Li SNOOP_RESPONSE.HITE
+.Pq Event B8H , Umask 02H
+Counts HIT E snoop response sent by this thread in response to a snoop
+request.
+.It Li SNOOP_RESPONSE.HITM
+.Pq Event B8H , Umask 04H
+Counts HIT M snoop response sent by this thread in response to a snoop
+request.
+.It Li OFF_CORE_RESPONSE_1
+.Pq Event BBH , Umask 01H
+see Section 30.6.1.3, Off-core Response Performance Monitoring in the
+Processor Core
+Use MSR 01A7H
+.It Li INST_RETIRED.ANY_P
+.Pq Event C0H , Umask 01H
+See Table A-1
+Notes: INST_RETIRED.ANY is counted by a designated fixed counter.
+INST_RETIRED.ANY_P is counted by a programmable counter and is an
+architectural performance event. Event is supported if CPUID.A.EBX[1] = 0.
+Counting: Faulting executions of GETSEC/VM entry/VM Exit/MWait will not
+count as retired instructions.
+.It Li INST_RETIRED.X87
+.Pq Event C0H , Umask 02H
+Counts the number of floating point computational operations retired:
+floating point computational operations executed by the assist handler and
+sub-operations of complex floating point instructions like transcendental
+instructions.
+.It Li INST_RETIRED.MMX
+.Pq Event C0H , Umask 04H
+Counts the number of retired: MMX instructions.
+.It Li UOPS_RETIRED.ANY
+.Pq Event C2H , Umask 01H
+Counts the number of micro-ops retired, (macro-fused=1, micro- fused=2,
+others=1; maximum count of 8 per cycle). Most instructions are composed of
+one or two micro-ops. Some instructions are decoded into longer sequences
+such as repeat instructions, floating point transcendental instructions, and
+assists.
+Use cmask=1 and invert to count active cycles or stalled cycles
+.It Li UOPS_RETIRED.RETIRE_SLOTS
+.Pq Event C2H , Umask 02H
+Counts the number of retirement slots used each cycle
+.It Li UOPS_RETIRED.MACRO_FUSED
+.Pq Event C2H , Umask 04H
+Counts number of macro-fused uops retired.
+.It Li MACHINE_CLEARS.CYCLES
+.Pq Event C3H , Umask 01H
+Counts the cycles machine clear is asserted.
+.It Li MACHINE_CLEARS.MEM_ORDER
+.Pq Event C3H , Umask 02H
+Counts the number of machine clears due to memory order conflicts.
+.It Li MACHINE_CLEARS.SMC
+.Pq Event C3H , Umask 04H
+Counts the number of times that a program writes to a code section.
+Self-modifying code causes a sever penalty in all Intel 64 and IA-32
+processors. The modified cache line is written back to the L2 and L3caches.
+.It Li BR_INST_RETIRED.ALL_BRANCHES
+.Pq Event C4H , Umask 00H
+See Table A-1
+.It Li BR_INST_RETIRED.CONDITIONAL
+.Pq Event C4H , Umask 01H
+Counts the number of conditional branch instructions retired.
+.It Li BR_INST_RETIRED.NEAR_CALL
+.Pq Event C4H , Umask 02H
+Counts the number of direct & indirect near unconditional calls retired
+.It Li BR_INST_RETIRED.ALL_BRANCHES
+.Pq Event C4H , Umask 04H
+Counts the number of branch instructions retired
+.It Li BR_MISP_RETIRED.ALL_BRANCHES
+.Pq Event C5H , Umask 00H
+See Table A-1
+.It Li BR_MISP_RETIRED.CONDITIONAL
+.Pq Event C5H , Umask 01H
+Counts mispredicted conditional retired calls.
+.It Li BR_MISP_RETIRED.NEAR_CALL
+.Pq Event C5H , Umask 02H
+Counts mispredicted direct & indirect near unconditional retired calls.
+.It Li BR_MISP_RETIRED.ALL_BRANCHES
+.Pq Event C5H , Umask 04H
+Counts all mispredicted retired calls.
+.It Li SSEX_UOPS_RETIRED.PACKED_SINGLE
+.Pq Event C7H , Umask 01H
+Counts SIMD packed single-precision floating point Uops retired.
+.It Li SSEX_UOPS_RETIRED.SCALAR_SINGLE
+.Pq Event C7H , Umask 02H
+Counts SIMD calar single-precision floating point Uops retired.
+.It Li SSEX_UOPS_RETIRED.PACKED_DOUBLE
+.Pq Event C7H , Umask 04H
+Counts SIMD packed double- precision floating point Uops retired.
+.It Li SSEX_UOPS_RETIRED.SCALAR_DOUBLE
+.Pq Event C7H , Umask 08H
+Counts SIMD scalar double-precision floating point Uops retired.
+.It Li SSEX_UOPS_RETIRED.VECTOR_INTEGER
+.Pq Event C7H , Umask 10H
+Counts 128-bit SIMD vector integer Uops retired.
+.It Li ITLB_MISS_RETIRED
+.Pq Event C8H , Umask 20H
+Counts the number of retired instructions that missed the ITLB when the
+instruction was fetched.
+.It Li MEM_LOAD_RETIRED.L1D_HIT
+.Pq Event CBH , Umask 01H
+Counts number of retired loads that hit the L1 data cache.
+.It Li MEM_LOAD_RETIRED.L2_HIT
+.Pq Event CBH , Umask 02H
+Counts number of retired loads that hit the L2 data cache.
+.It Li MEM_LOAD_RETIRED.L3_UNSHARED_HIT
+.Pq Event CBH , Umask 04H
+Counts number of retired loads that hit their own, unshared lines in the L3
+cache.
+.It Li MEM_LOAD_RETIRED.OTHER_CORE_L2_HIT_HITM
+.Pq Event CBH , Umask 08H
+Counts number of retired loads that hit in a sibling core's L2 (on die
+core). Since the L3 is inclusive of all cores on the package, this is an L3
+hit. This counts both clean or modified hits.
+.It Li MEM_LOAD_RETIRED.L3_MISS
+.Pq Event CBH , Umask 10H
+Counts number of retired loads that miss the L3 cache. The load was
+satisfied by a remote socket, local memory or an IOH.
+.It Li MEM_LOAD_RETIRED.HIT_LFB
+.Pq Event CBH , Umask 40H
+Counts number of retired loads that miss the L1D and the address is located
+in an allocated line fill buffer and will soon be committed to cache. This
+is counting secondary L1D misses.
+.It Li MEM_LOAD_RETIRED.DTLB_MISS
+.Pq Event CBH , Umask 80H
+Counts the number of retired loads that missed the DTLB. The DTLB miss is
+not counted if the load operation causes a fault. This event counts loads
+from cacheable memory only. The event does not count loads by software
+prefetches. Counts both primary and secondary misses to the TLB.
+.It Li FP_MMX_TRANS.TO_FP
+.Pq Event CCH , Umask 01H
+Counts the first floating-point instruction following any MMX instruction.
+You can use this event to estimate the penalties for the transitions between
+floating-point and MMX technology states.
+.It Li FP_MMX_TRANS.TO_MMX
+.Pq Event CCH , Umask 02H
+Counts the first MMX instruction following a floating-point instruction. You
+can use this event to estimate the penalties for the transitions between
+floating-point and MMX technology states.
+.It Li FP_MMX_TRANS.ANY
+.Pq Event CCH , Umask 03H
+Counts all transitions from floating point to MMX instructions and from MMX
+instructions to floating point instructions. You can use this event to
+estimate the penalties for the transitions between floating-point and MMX
+technology states.
+.It Li MACRO_INSTS.DECODED
+.Pq Event D0H , Umask 01H
+Counts the number of instructions decoded, (but not necessarily executed or
+retired).
+.It Li UOPS_DECODED.STALL_CYCLES
+.Pq Event D1H , Umask 01H
+Counts the cycles of decoder stalls.
+.It Li UOPS_DECODED.MS
+.Pq Event D1H , Umask 02H
+Counts the number of Uops decoded by the Microcode Sequencer, MS. The MS
+delivers uops when the instruction is more than 4 uops long or a microcode
+assist is occurring.
+.It Li UOPS_DECODED.ESP_FOLDING
+.Pq Event D1H , Umask 04H
+Counts number of stack pointer (ESP) instructions decoded: push , pop , call
+, ret, etc. ESP instructions do not generate a Uop to increment or decrement
+ESP. Instead, they update an ESP_Offset register that keeps track of the
+delta to the current value of the ESP register.
+.It Li UOPS_DECODED.ESP_SYNC
+.Pq Event D1H , Umask 08H
+Counts number of stack pointer (ESP) sync operations where an ESP
+instruction is corrected by adding the ESP offset register to the current
+value of the ESP register.
+.It Li RAT_STALLS.FLAGS
+.Pq Event D2H , Umask 01H
+Counts the number of cycles during which execution stalled due to several
+reasons, one of which is a partial flag register stall. A partial register
+stall may occur when two conditions are met: 1) an instruction modifies
+some, but not all, of the flags in the flag register and 2) the next
+instruction, which depends on flags, depends on flags that were not modified
+by this instruction.
+.It Li RAT_STALLS.REGISTERS
+.Pq Event D2H , Umask 02H
+This event counts the number of cycles instruction execution latency became
+longer than the defined latency because the instruction used a register that
+was partially written by previous instruction.
+.It Li RAT_STALLS.ROB_READ_PORT
+.Pq Event D2H , Umask 04H
+Counts the number of cycles when ROB read port stalls occurred, which did
+not allow new micro-ops to enter the out-of-order pipeline. Note that, at
+this stage in the pipeline, additional stalls may occur at the same cycle
+and prevent the stalled micro-ops from entering the pipe. In such a case,
+micro-ops retry entering the execution pipe in the next cycle and the
+ROB-read port stall is counted again.
+.It Li RAT_STALLS.SCOREBOARD
+.Pq Event D2H , Umask 08H
+Counts the cycles where we stall due to microarchitecturally required
+serialization. Microcode scoreboarding stalls.
+.It Li RAT_STALLS.ANY
+.Pq Event D2H , Umask 0FH
+Counts all Register Allocation Table stall cycles due to: Cycles when ROB
+read port stalls occurred, which did not allow new micro-ops to enter the
+execution pipe. Cycles when partial register stalls occurred Cycles when
+flag stalls occurred Cycles floating-point unit (FPU) status word stalls
+occurred. To count each of these conditions separately use the events:
+RAT_STALLS.ROB_READ_PORT, RAT_STALLS.PARTIAL, RAT_STALLS.FLAGS, and
+RAT_STALLS.FPSW.
+.It Li SEG_RENAME_STALLS
+.Pq Event D4H , Umask 01H
+Counts the number of stall cycles due to the lack of renaming resources for
+the ES, DS, FS, and GS segment registers. If a segment is renamed but not
+retired and a second update to the same segment occurs, a stall occurs in
+the front- end of the pipeline until the renamed segment retires.
+.It Li ES_REG_RENAMES
+.Pq Event D5H , Umask 01H
+Counts the number of times the ES segment register is renamed.
+.It Li UOP_UNFUSION
+.Pq Event DBH , Umask 01H
+Counts unfusion events due to floating point exception to a fused uop.
+.It Li BR_INST_DECODED
+.Pq Event E0H , Umask 01H
+Counts the number of branch instructions decoded.
+.It Li BPU_MISSED_CALL_RET
+.Pq Event E5H , Umask 01H
+Counts number of times the Branch Prediciton Unit missed predicting a call
+or return branch.
+.It Li BACLEAR.CLEAR
+.Pq Event E6H , Umask 01H
+Counts the number of times the front end is resteered, mainly when the
+Branch Prediction Unit cannot provide a correct prediction and this is
+corrected by the Branch Address Calculator at the front end. This can occur
+if the code has many branches such that they cannot be consumed by the BPU.
+Each BACLEAR asserted by the BAC generates approximately an 8 cycle bubble
+in the instruction fetch pipeline. The effect on total execution time
+depends on the surrounding code.
+.It Li BACLEAR.BAD_TARGET
+.Pq Event E6H , Umask 02H
+Counts number of Branch Address Calculator clears (BACLEAR) asserted due to
+conditional branch instructions in which there was a target hit but the
+direction was wrong. Each BACLEAR asserted by the BAC generates
+approximately an 8 cycle bubble in the instruction fetch pipeline.
+.It Li BPU_CLEARS.EARLY
+.Pq Event E8H , Umask 01H
+Counts early (normal) Branch Prediction Unit clears: BPU predicted a taken
+branch after incorrectly assuming that it was not taken.
+The BPU clear leads to 2 cycle bubble in the Front End.
+.It Li BPU_CLEARS.LATE
+.Pq Event E8H , Umask 02H
+Counts late Branch Prediction Unit clears due to Most Recently Used
+conflicts. The PBU clear leads to a 3 cycle bubble in the Front End.
+.It Li THREAD_ACTIVE
+.Pq Event ECH , Umask 01H
+Counts cycles threads are active.
+.It Li L2_TRANSACTIONS.LOAD
+.Pq Event F0H , Umask 01H
+Counts L2 load operations due to HW prefetch or demand loads.
+.It Li L2_TRANSACTIONS.RFO
+.Pq Event F0H , Umask 02H
+Counts L2 RFO operations due to HW prefetch or demand RFOs.
+.It Li L2_TRANSACTIONS.IFETCH
+.Pq Event F0H , Umask 04H
+Counts L2 instruction fetch operations due to HW prefetch or demand ifetch.
+.It Li L2_TRANSACTIONS.PREFETCH
+.Pq Event F0H , Umask 08H
+Counts L2 prefetch operations.
+.It Li L2_TRANSACTIONS.L1D_WB
+.Pq Event F0H , Umask 10H
+Counts L1D writeback operations to the L2.
+.It Li L2_TRANSACTIONS.FILL
+.Pq Event F0H , Umask 20H
+Counts L2 cache line fill operations due to load, RFO, L1D writeback or
+prefetch.
+.It Li L2_TRANSACTIONS.WB
+.Pq Event F0H , Umask 40H
+Counts L2 writeback operations to the L3.
+.It Li L2_TRANSACTIONS.ANY
+.Pq Event F0H , Umask 80H
+Counts all L2 cache operations.
+.It Li L2_LINES_IN.S_STATE
+.Pq Event F1H , Umask 02H
+Counts the number of cache lines allocated in the L2 cache in the S (shared)
+state.
+.It Li L2_LINES_IN.E_STATE
+.Pq Event F1H , Umask 04H
+Counts the number of cache lines allocated in the L2 cache in the E
+(exclusive) state.
+.It Li L2_LINES_IN.ANY
+.Pq Event F1H , Umask 07H
+Counts the number of cache lines allocated in the L2 cache.
+.It Li L2_LINES_OUT.DEMAND_CLEAN
+.Pq Event F2H , Umask 01H
+Counts L2 clean cache lines evicted by a demand request.
+.It Li L2_LINES_OUT.DEMAND_DIRTY
+.Pq Event F2H , Umask 02H
+Counts L2 dirty (modified) cache lines evicted by a demand request.
+.It Li L2_LINES_OUT.PREFETCH_CLEAN
+.Pq Event F2H , Umask 04H
+Counts L2 clean cache line evicted by a prefetch request.
+.It Li L2_LINES_OUT.PREFETCH_DIRTY
+.Pq Event F2H , Umask 08H
+Counts L2 modified cache line evicted by a prefetch request.
+.It Li L2_LINES_OUT.ANY
+.Pq Event F2H , Umask 0FH
+Counts all L2 cache lines evicted for any reason.
+.It Li SQ_MISC.LRU_HINTS
+.Pq Event F4H , Umask 04H
+Counts number of Super Queue LRU hints sent to L3.
+.It Li SQ_MISC.SPLIT_LOCK
+.Pq Event F4H , Umask 10H
+Counts the number of SQ lock splits across a cache line.
+.It Li SQ_FULL_STALL_CYCLES
+.Pq Event F6H , Umask 01H
+Counts cycles the Super Queue is full. Neither of the threads on this core
+will be able to access the uncore.
+.It Li FP_ASSIST.ALL
+.Pq Event F7H , Umask 01H
+Counts the number of floating point operations executed that required
+micro-code assist intervention. Assists are required in the following cases:
+SSE instructions, (Denormal input when the DAZ flag is off or Underflow
+result when the FTZ flag is off): x87 instructions, (NaN or denormal are
+loaded to a register or used as input from memory, Division by 0 or
+Underflow output).
+.It Li FP_ASSIST.OUTPUT
+.Pq Event F7H , Umask 02H
+Counts number of floating point micro-code assist when the output value
+(destination register) is invalid.
+.It Li FP_ASSIST.INPUT
+.Pq Event F7H , Umask 04H
+Counts number of floating point micro-code assist when the input value (one
+of the source operands to an FP instruction) is invalid.
+.It Li SIMD_INT_64.PACKED_MPY
+.Pq Event FDH , Umask 01H
+Counts number of SID integer 64 bit packed multiply operations.
+.It Li SIMD_INT_64.PACKED_SHIFT
+.Pq Event FDH , Umask 02H
+Counts number of SID integer 64 bit packed shift operations.
+.It Li SIMD_INT_64.PACK
+.Pq Event FDH , Umask 04H
+Counts number of SID integer 64 bit pack operations.
+.It Li SIMD_INT_64.UNPACK
+.Pq Event FDH , Umask 08H
+Counts number of SID integer 64 bit unpack operations.
+.It Li SIMD_INT_64.PACKED_LOGICAL
+.Pq Event FDH , Umask 10H
+Counts number of SID integer 64 bit logical operations.
+.It Li SIMD_INT_64.PACKED_ARITH
+.Pq Event FDH , Umask 20H
+Counts number of SID integer 64 bit arithmetic operations.
+.It Li SIMD_INT_64.SHUFFLE_MOVE
+.Pq Event FDH , Umask 40H
+Counts number of SID integer 64 bit shift or move operations.
+.El
+.Sh SEE ALSO
+.Xr pmc 3 ,
+.Xr pmc.atom 3 ,
+.Xr pmc.core 3 ,
+.Xr pmc.iaf 3 ,
+.Xr pmc.ucf 3 ,
+.Xr pmc.k7 3 ,
+.Xr pmc.k8 3 ,
+.Xr pmc.p4 3 ,
+.Xr pmc.p5 3 ,
+.Xr pmc.p6 3 ,
+.Xr pmc.corei7 3 ,
+.Xr pmc.corei7uc 3 ,
+.Xr pmc.westmereuc 3 ,
+.Xr pmc.tsc 3 ,
+.Xr pmc_cpuinfo 3 ,
+.Xr pmclog 3 ,
+.Xr hwpmc 4
+.Sh HISTORY
+The
+.Nm pmc
+library first appeared in
+.Fx 6.0 .
+.Sh AUTHORS
+The
+.Lb libpmc
+library was written by
+.An "Joseph Koshy"
+.Aq jkoshy@FreeBSD.org .
diff --git a/lib/libpmc/pmc.westmereuc.3 b/lib/libpmc/pmc.westmereuc.3
new file mode 100644
index 0000000..c5fd12d
--- /dev/null
+++ b/lib/libpmc/pmc.westmereuc.3
@@ -0,0 +1,1083 @@
+.\" Copyright (c) 2010 Fabien Thomas.  All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" This software is provided by Joseph Koshy ``as is'' and
+.\" any express or implied warranties, including, but not limited to, the
+.\" implied warranties of merchantability and fitness for a particular purpose
+.\" are disclaimed.  in no event shall Joseph Koshy be liable
+.\" for any direct, indirect, incidental, special, exemplary, or consequential
+.\" damages (including, but not limited to, procurement of substitute goods
+.\" or services; loss of use, data, or profits; or business interruption)
+.\" however caused and on any theory of liability, whether in contract, strict
+.\" liability, or tort (including negligence or otherwise) arising in any way
+.\" out of the use of this software, even if advised of the possibility of
+.\" such damage.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd March 24, 2010
+.Os
+.Dt PMC.WESTMEREUC 3
+.Sh NAME
+.Nm pmc.westmere
+.Nd uncore measurement events for
+.Tn Intel
+.Tn Westmere
+family CPUs
+.Sh LIBRARY
+.Lb libpmc
+.Sh SYNOPSIS
+.In pmc.h
+.Sh DESCRIPTION
+.Tn Intel
+.Tn "Westmere"
+CPUs contain PMCs conforming to version 2 of the
+.Tn Intel
+performance measurement architecture.
+These CPUs contain two classes of PMCs:
+.Bl -tag -width "Li PMC_CLASS_UCP"
+.It Li PMC_CLASS_UCF
+Fixed-function counters that count only one hardware event per counter.
+.It Li PMC_CLASS_UCP
+Programmable counters that may be configured to count one of a defined
+set of hardware events.
+.El
+.Pp
+The number of PMCs available in each class and their widths need to be
+determined at run time by calling
+.Xr pmc_cpuinfo 3 .
+.Pp
+Intel Westmere PMCs are documented in
+.Rs
+.%B "Intel(R) 64 and IA-32 Architectures Software Developes Manual"
+.%T "Volume 3B: System Programming Guide, Part 2"
+.%N "Order Number: 253669-033US"
+.%D December 2009
+.%Q "Intel Corporation"
+.Re
+.Ss WESTMERE UNCORE FIXED FUNCTION PMCS
+These PMCs and their supported events are documented in
+.Xr pmc.ucf 3 .
+Not all CPUs in this family implement fixed-function counters.
+.Ss WESTMERE UNCORE PROGRAMMABLE PMCS
+The programmable PMCs support the following capabilities:
+.Bl -column "PMC_CAP_INTERRUPT" "Support"
+.It Em Capability Ta Em Support
+.It PMC_CAP_CASCADE Ta \&No
+.It PMC_CAP_EDGE Ta Yes
+.It PMC_CAP_INTERRUPT Ta \&No
+.It PMC_CAP_INVERT Ta Yes
+.It PMC_CAP_READ Ta Yes
+.It PMC_CAP_PRECISE Ta \&No
+.It PMC_CAP_SYSTEM Ta \&No
+.It PMC_CAP_TAGGING Ta \&No
+.It PMC_CAP_THRESHOLD Ta Yes
+.It PMC_CAP_USER Ta \&No
+.It PMC_CAP_WRITE Ta Yes
+.El
+.Ss Event Qualifiers
+Event specifiers for these PMCs support the following common
+qualifiers:
+.Bl -tag -width indent
+.It Li cmask= Ns Ar value
+Configure the PMC to increment only if the number of configured
+events measured in a cycle is greater than or equal to
+.Ar value .
+.It Li edge
+Configure the PMC to count the number of de-asserted to asserted
+transitions of the conditions expressed by the other qualifiers.
+If specified, the counter will increment only once whenever a
+condition becomes true, irrespective of the number of clocks during
+which the condition remains true.
+.It Li inv
+Invert the sense of comparison when the
+.Dq Li cmask
+qualifier is present, making the counter increment when the number of
+events per cycle is less than the value specified by the
+.Dq Li cmask
+qualifier.
+.El
+.Ss Event Specifiers (Programmable PMCs)
+Westmere uncore programmable PMCs support the following events:
+.Bl -tag -width indent
+.It Li GQ_CYCLES_FULL.READ_TRACKER
+.Pq Event 00H , Umask 01H
+Uncore cycles Global Queue read tracker is full.
+.It Li GQ_CYCLES_FULL.WRITE_TRACKER
+.Pq Event 00H , Umask 02H
+Uncore cycles Global Queue write tracker is full.
+.It Li GQ_CYCLES_FULL.PEER_PROBE_TRACKER
+.Pq Event 00H , Umask 04H
+Uncore cycles Global Queue peer probe tracker is full. The peer probe
+tracker queue tracks snoops from the IOH and remote sockets.
+.It Li GQ_CYCLES_NOT_EMPTY.READ_TRACKER
+.Pq Event 01H , Umask 01H
+Uncore cycles were Global Queue read tracker has at least one valid entry.
+.It Li GQ_CYCLES_NOT_EMPTY.WRITE_TRACKER
+.Pq Event 01H , Umask 02H
+Uncore cycles were Global Queue write tracker has at least one valid entry.
+.It Li GQ_CYCLES_NOT_EMPTY.PEER_PROBE_TRACKER
+.Pq Event 01H , Umask 04H
+Uncore cycles were Global Queue peer probe tracker has at least one valid
+entry. The peer probe tracker queue tracks IOH and remote socket snoops.
+.It Li GQ_OCCUPANCY.READ_TRACKER
+.Pq Event 02H , Umask 01H
+Increments the number of queue entries (code read, data read, and RFOs) in
+the tread tracker. The GQ read tracker allocate to deallocate occupancy
+count is divided by the count to obtain the average read tracker latency.
+.It Li GQ_ALLOC.READ_TRACKER
+.Pq Event 03H , Umask 01H
+Counts the number of tread tracker allocate to deallocate entries. The GQ
+read tracker allocate to deallocate occupancy count is divided by the count
+to obtain the average read tracker latency.
+.It Li GQ_ALLOC.RT_L3_MISS
+.Pq Event 03H , Umask 02H
+Counts the number GQ read tracker entries for which a full cache line read
+has missed the L3. The GQ read tracker L3 miss to fill occupancy count is
+divided by this count to obtain the average cache line read L3 miss latency.
+The latency represents the time after which the L3 has determined that the
+cache line has missed. The time between a GQ read tracker allocation and the
+L3 determining that the cache line has missed is the average L3 hit latency.
+The total L3 cache line read miss latency is the hit latency + L3 miss
+latency.
+.It Li GQ_ALLOC.RT_TO_L3_RESP
+.Pq Event 03H , Umask 04H
+Counts the number of GQ read tracker entries that are allocated in the read
+tracker queue that hit or miss the L3. The GQ read tracker L3 hit occupancy
+count is divided by this count to obtain the average L3 hit latency.
+.It Li GQ_ALLOC.RT_TO_RTID_ACQUIRED
+.Pq Event 03H , Umask 08H
+Counts the number of GQ read tracker entries that are allocated in the read
+tracker, have missed in the L3 and have not acquired a Request Transaction
+ID.	The GQ read tracker L3 miss to RTID acquired occupancy count is
+divided by this count to obtain the average latency for a read L3 miss to
+acquire an RTID.
+.It Li GQ_ALLOC.WT_TO_RTID_ACQUIRED
+.Pq Event 03H , Umask 10H
+Counts the number of GQ write tracker entries that are allocated in the
+write tracker, have missed in the L3 and have not acquired a Request
+Transaction ID.	The GQ write tracker L3 miss to RTID occupancy count is
+divided by this count to obtain the average latency for a write L3 miss to
+acquire an RTID.
+.It Li GQ_ALLOC.WRITE_TRACKER
+.Pq Event 03H , Umask 20H
+Counts the number of GQ write tracker entries that are allocated in the
+write tracker queue that miss the L3. The GQ write tracker occupancy count
+is divided by the this count to obtain the average L3 write miss latency.
+.It Li GQ_ALLOC.PEER_PROBE_TRACKER
+.Pq Event 03H , Umask 40H
+Counts the number of GQ peer probe tracker (snoop) entries that are
+allocated in the peer probe tracker queue that miss the L3. The GQ peer
+probe occupancy count is divided by this count to obtain the average L3 peer
+probe miss latency.
+.It Li GQ_DATA.FROM_QPI
+.Pq Event 04H , Umask 01H
+Cycles Global Queue Quickpath Interface input data port is busy importing
+data from the Quickpath Interface. Each cycle the input port can transfer 8
+or 16 bytes of data.
+.It Li GQ_DATA.FROM_QMC
+.Pq Event 04H , Umask 02H
+Cycles Global Queue Quickpath Memory Interface input data port is busy
+importing data from the Quickpath Memory Interface. Each cycle the input
+port can transfer 8 or 16 bytes of data.
+.It Li GQ_DATA.FROM_L3
+.Pq Event 04H , Umask 04H
+Cycles GQ L3 input data port is busy importing data from the Last Level
+Cache. Each cycle the input port can transfer 32 bytes of data.
+.It Li GQ_DATA.FROM_CORES_02
+.Pq Event 04H , Umask 08H
+Cycles GQ Core 0 and 2 input data port is busy importing data from processor
+cores 0 and 2. Each cycle the input port can transfer 32 bytes of data.
+.It Li GQ_DATA.FROM_CORES_13
+.Pq Event 04H , Umask 10H
+Cycles GQ Core 1 and 3 input data port is busy importing data from processor
+cores 1 and 3. Each cycle the input port can transfer 32 bytes of data.
+.It Li GQ_DATA.TO_QPI_QMC
+.Pq Event 05H , Umask 01H
+Cycles GQ QPI and QMC output data port is busy sending data to the Quickpath
+Interface or Quickpath Memory Interface. Each cycle the output port can
+transfer 32 bytes of data.
+.It Li GQ_DATA.TO_L3
+.Pq Event 05H , Umask 02H
+Cycles GQ L3 output data port is busy sending data to the Last Level Cache.
+Each cycle the output port can transfer 32 bytes of data.
+.It Li GQ_DATA.TO_CORES
+.Pq Event 05H , Umask 04H
+Cycles GQ Core output data port is busy sending data to the Cores. Each
+cycle the output port can transfer 32 bytes of data.
+.It Li SNP_RESP_TO_LOCAL_HOME.I_STATE
+.Pq Event 06H , Umask 01H
+Number of snoop responses to the local home that L3 does not have the
+referenced cache line.
+.It Li SNP_RESP_TO_LOCAL_HOME.S_STATE
+.Pq Event 06H , Umask 02H
+Number of snoop responses to the local home that L3 has the referenced line
+cached in the S state.
+.It Li SNP_RESP_TO_LOCAL_HOME.FWD_S_STATE
+.Pq Event 06H , Umask 04H
+Number of responses to code or data read snoops to the local home that the
+L3 has the referenced cache line in the E state. The L3 cache line state is
+changed to the S state and the line is forwarded to the local home in the S
+state.
+.It Li SNP_RESP_TO_LOCAL_HOME.FWD_I_STATE
+.Pq Event 06H , Umask 08H
+Number of responses to read invalidate snoops to the local home that the L3
+has the referenced cache line in the M state. The L3 cache line state is
+invalidated and the line is forwarded to the local home in the M state.
+.It Li SNP_RESP_TO_LOCAL_HOME.CONFLICT
+.Pq Event 06H , Umask 10H
+Number of conflict snoop responses sent to the local home.
+.It Li SNP_RESP_TO_LOCAL_HOME.WB
+.Pq Event 06H , Umask 20H
+Number of responses to code or data read snoops to the local home that the
+L3 has the referenced line cached in the M state.
+.It Li SNP_RESP_TO_REMOTE_HOME.I_STATE
+.Pq Event 07H , Umask 01H
+Number of snoop responses to a remote home that L3 does not have the
+referenced cache line.
+.It Li SNP_RESP_TO_REMOTE_HOME.S_STATE
+.Pq Event 07H , Umask 02H
+Number of snoop responses to a remote home that L3 has the referenced line
+cached in the S state.
+.It Li SNP_RESP_TO_REMOTE_HOME.FWD_S_STATE
+.Pq Event 07H , Umask 04H
+Number of responses to code or data read snoops to a remote home that the L3
+has the referenced cache line in the E state. The L3 cache line state is
+changed to the S state and the line is forwarded to the remote home in the S
+state.
+.It Li SNP_RESP_TO_REMOTE_HOME.FWD_I_STATE
+.Pq Event 07H , Umask 08H
+Number of responses to read invalidate snoops to a remote home that the L3
+has the referenced cache line in the M state. The L3 cache line state is
+invalidated and the line is forwarded to the remote home in the M state.
+.It Li SNP_RESP_TO_REMOTE_HOME.CONFLICT
+.Pq Event 07H , Umask 10H
+Number of conflict snoop responses sent to the local home.
+.It Li SNP_RESP_TO_REMOTE_HOME.WB
+.Pq Event 07H , Umask 20H
+Number of responses to code or data read snoops to a remote home that the L3
+has the referenced line cached in the M state.
+.It Li SNP_RESP_TO_REMOTE_HOME.HITM
+.Pq Event 07H , Umask 24H
+Number of HITM snoop responses to a remote home
+.It Li L3_HITS.READ
+.Pq Event 08H , Umask 01H
+Number of code read, data read and RFO requests that hit in the L3
+.It Li L3_HITS.WRITE
+.Pq Event 08H , Umask 02H
+Number of writeback requests that hit in the L3. Writebacks from the cores
+will always result in L3 hits due to the inclusive property of the L3.
+.It Li L3_HITS.PROBE
+.Pq Event 08H , Umask 04H
+Number of snoops from IOH or remote sockets that hit in the L3.
+.It Li L3_HITS.ANY
+.Pq Event 08H , Umask 03H
+Number of reads and writes that hit the L3.
+.It Li L3_MISS.READ
+.Pq Event 09H , Umask 01H
+Number of code read, data read and RFO requests that miss the L3.
+.It Li L3_MISS.WRITE
+.Pq Event 09H , Umask 02H
+Number of writeback requests that miss the L3. Should always be zero as
+writebacks from the cores will always result in L3 hits due to the inclusive
+property of the L3.
+.It Li L3_MISS.PROBE
+.Pq Event 09H , Umask 04H
+Number of snoops from IOH or remote sockets that miss the L3.
+.It Li L3_MISS.ANY
+.Pq Event 09H , Umask 03H
+Number of reads and writes that miss the L3.
+.It Li L3_LINES_IN.M_STATE
+.Pq Event 0AH , Umask 01H
+Counts the number of L3 lines allocated in M state. The only time a cache
+line is allocated in the M state is when the line was forwarded in M state
+is forwarded due to a Snoop Read Invalidate Own request.
+.It Li L3_LINES_IN.E_STATE
+.Pq Event 0AH , Umask 02H
+Counts the number of L3 lines allocated in E state.
+.It Li L3_LINES_IN.S_STATE
+.Pq Event 0AH , Umask 04H
+Counts the number of L3 lines allocated in S state.
+.It Li L3_LINES_IN.F_STATE
+.Pq Event 0AH , Umask 08H
+Counts the number of L3 lines allocated in F state.
+.It Li L3_LINES_IN.ANY
+.Pq Event 0AH , Umask 0FH
+Counts the number of L3 lines allocated in any state.
+.It Li L3_LINES_OUT.M_STATE
+.Pq Event 0BH , Umask 01H
+Counts the number of L3 lines victimized that were in the M state. When the
+victim cache line is in M state, the line is written to its home cache agent
+which can be either local or remote.
+.It Li L3_LINES_OUT.E_STATE
+.Pq Event 0BH , Umask 02H
+Counts the number of L3 lines victimized that were in the E state.
+.It Li L3_LINES_OUT.S_STATE
+.Pq Event 0BH , Umask 04H
+Counts the number of L3 lines victimized that were in the S state.
+.It Li L3_LINES_OUT.I_STATE
+.Pq Event 0BH , Umask 08H
+Counts the number of L3 lines victimized that were in the I state.
+.It Li L3_LINES_OUT.F_STATE
+.Pq Event 0BH , Umask 10H
+Counts the number of L3 lines victimized that were in the F state.
+.It Li L3_LINES_OUT.ANY
+.Pq Event 0BH , Umask 1FH
+Counts the number of L3 lines victimized in any state.
+.It Li GQ_SNOOP.GOTO_S
+.Pq Event 0CH , Umask 01H
+Counts the number of remote snoops that have requested a cache line be set
+to the S state.
+.It Li GQ_SNOOP.GOTO_I
+.Pq Event 0CH , Umask 02H
+Counts the number of remote snoops that have requested a cache line be set
+to the I state.
+.It Li GQ_SNOOP.GOTO_S_HIT_E
+.Pq Event 0CH , Umask 04H
+Counts the number of remote snoops that have requested a cache line be set
+to the S state from E state.
+Requires writing MSR 301H with mask = 2H
+.It Li GQ_SNOOP.GOTO_S_HIT_F
+.Pq Event 0CH , Umask 04H
+Counts the number of remote snoops that have requested a cache line be set
+to the S state from F (forward) state.
+Requires writing MSR 301H with mask = 8H
+.It Li GQ_SNOOP.GOTO_S_HIT_M
+.Pq Event 0CH , Umask 04H
+Counts the number of remote snoops that have requested a cache line be set
+to the S state from M state.
+Requires writing MSR 301H with mask = 1H
+.It Li GQ_SNOOP.GOTO_S_HIT_S
+.Pq Event 0CH , Umask 04H
+Counts the number of remote snoops that have requested a cache line be set
+to the S state from S state.
+Requires writing MSR 301H with mask = 4H
+.It Li GQ_SNOOP.GOTO_I_HIT_E
+.Pq Event 0CH , Umask 08H
+Counts the number of remote snoops that have requested a cache line be set
+to the I state from E state.
+Requires writing MSR 301H with mask = 2H
+.It Li GQ_SNOOP.GOTO_I_HIT_F
+.Pq Event 0CH , Umask 08H
+Counts the number of remote snoops that have requested a cache line be set
+to the I state from F (forward) state.
+Requires writing MSR 301H with mask = 8H
+.It Li GQ_SNOOP.GOTO_I_HIT_M
+.Pq Event 0CH , Umask 08H
+Counts the number of remote snoops that have requested a cache line be set
+to the I state from M state.
+Requires writing MSR 301H with mask = 1H
+.It Li GQ_SNOOP.GOTO_I_HIT_S
+.Pq Event 0CH , Umask 08H
+Counts the number of remote snoops that have requested a cache line be set
+to the I state from S state.
+Requires writing MSR 301H with mask = 4H
+.It Li QHL_REQUESTS.IOH_READS
+.Pq Event 20H , Umask 01H
+Counts number of Quickpath Home Logic read requests from the IOH.
+.It Li QHL_REQUESTS.IOH_WRITES
+.Pq Event 20H , Umask 02H
+Counts number of Quickpath Home Logic write requests from the IOH.
+.It Li QHL_REQUESTS.REMOTE_READS
+.Pq Event 20H , Umask 04H
+Counts number of Quickpath Home Logic read requests from a remote socket.
+.It Li QHL_REQUESTS.REMOTE_WRITES
+.Pq Event 20H , Umask 08H
+Counts number of Quickpath Home Logic write requests from a remote socket.
+.It Li QHL_REQUESTS.LOCAL_READS
+.Pq Event 20H , Umask 10H
+Counts number of Quickpath Home Logic read requests from the local socket.
+.It Li QHL_REQUESTS.LOCAL_WRITES
+.Pq Event 20H , Umask 20H
+Counts number of Quickpath Home Logic write requests from the local socket.
+.It Li QHL_CYCLES_FULL.IOH
+.Pq Event 21H , Umask 01H
+Counts uclk cycles all entries in the Quickpath Home Logic IOH are full.
+.It Li QHL_CYCLES_FULL.REMOTE
+.Pq Event 21H , Umask 02H
+Counts uclk cycles all entries in the Quickpath Home Logic remote tracker
+are full.
+.It Li QHL_CYCLES_FULL.LOCAL
+.Pq Event 21H , Umask 04H
+Counts uclk cycles all entries in the Quickpath Home Logic local tracker are
+full.
+.It Li QHL_CYCLES_NOT_EMPTY.IOH
+.Pq Event 22H , Umask 01H
+Counts uclk cycles all entries in the Quickpath Home Logic IOH is busy.
+.It Li QHL_CYCLES_NOT_EMPTY.REMOTE
+.Pq Event 22H , Umask 02H
+Counts uclk cycles all entries in the Quickpath Home Logic remote tracker is
+busy.
+.It Li QHL_CYCLES_NOT_EMPTY.LOCAL
+.Pq Event 22H , Umask 04H
+Counts uclk cycles all entries in the Quickpath Home Logic local tracker is
+busy.
+.It Li QHL_OCCUPANCY.IOH
+.Pq Event 23H , Umask 01H
+QHL IOH tracker allocate to deallocate read occupancy.
+.It Li QHL_OCCUPANCY.REMOTE
+.Pq Event 23H , Umask 02H
+QHL remote tracker allocate to deallocate read occupancy.
+.It Li QHL_OCCUPANCY.LOCAL
+.Pq Event 23H , Umask 04H
+QHL local tracker allocate to deallocate read occupancy.
+.It Li QHL_ADDRESS_CONFLICTS.2WAY
+.Pq Event 24H , Umask 02H
+Counts number of QHL Active Address Table (AAT) entries that saw a max of 2
+conflicts. The AAT is a structure that tracks requests that are in conflict.
+The requests themselves are in the home tracker entries. The count is
+reported when an AAT entry deallocates.
+.It Li QHL_ADDRESS_CONFLICTS.3WAY
+.Pq Event 24H , Umask 04H
+Counts number of QHL Active Address Table (AAT) entries that saw a max of 3
+conflicts. The AAT is a structure that tracks requests that are in conflict.
+The requests themselves are in the home tracker entries. The count is
+reported when an AAT entry deallocates.
+.It Li QHL_CONFLICT_CYCLES.IOH
+.Pq Event 25H , Umask 01H
+Counts cycles the Quickpath Home Logic IOH Tracker contains two or more
+requests with an address conflict. A max of 3 requests can be in conflict.
+.It Li QHL_CONFLICT_CYCLES.REMOTE
+.Pq Event 25H , Umask 02H
+Counts cycles the Quickpath Home Logic Remote Tracker contains two or more
+requests with an address conflict. A max of 3 requests can be in conflict.
+.It Li QHL_CONFLICT_CYCLES.LOCAL
+.Pq Event 25H , Umask 04H
+Counts cycles the Quickpath Home Logic Local Tracker contains two or more
+requests with an address conflict. A max of 3 requests can be in conflict.
+.It Li QHL_TO_QMC_BYPASS
+.Pq Event 26H , Umask 01H
+Counts number or requests to the Quickpath Memory Controller that bypass the
+Quickpath Home Logic. All local accesses can be bypassed. For remote
+requests, only read requests can be bypassed.
+.It Li QMC_ISOC_FULL.READ.CH0
+.Pq Event 28H , Umask 01H
+Counts cycles all the entries in the DRAM channel 0 high priority queue are
+occupied with isochronous read requests.
+.It Li QMC_ISOC_FULL.READ.CH1
+.Pq Event 28H , Umask 02H
+Counts cycles all the entries in the DRAM channel 1high priority queue are
+occupied with isochronous read requests.
+.It Li QMC_ISOC_FULL.READ.CH2
+.Pq Event 28H , Umask 04H
+Counts cycles all the entries in the DRAM channel 2 high priority queue are
+occupied with isochronous read requests.
+.It Li QMC_ISOC_FULL.WRITE.CH0
+.Pq Event 28H , Umask 08H
+Counts cycles all the entries in the DRAM channel 0 high priority queue are
+occupied with isochronous write requests.
+.It Li QMC_ISOC_FULL.WRITE.CH1
+.Pq Event 28H , Umask 10H
+Counts cycles all the entries in the DRAM channel 1 high priority queue are
+occupied with isochronous write requests.
+.It Li QMC_ISOC_FULL.WRITE.CH2
+.Pq Event 28H , Umask 20H
+Counts cycles all the entries in the DRAM channel 2 high priority queue are
+occupied with isochronous write requests.
+.It Li QMC_BUSY.READ.CH0
+.Pq Event 29H , Umask 01H
+Counts cycles where Quickpath Memory Controller has at least 1 outstanding
+read request to DRAM channel 0.
+.It Li QMC_BUSY.READ.CH1
+.Pq Event 29H , Umask 02H
+Counts cycles where Quickpath Memory Controller has at least 1 outstanding
+read request to DRAM channel 1.
+.It Li QMC_BUSY.READ.CH2
+.Pq Event 29H , Umask 04H
+Counts cycles where Quickpath Memory Controller has at least 1 outstanding
+read request to DRAM channel 2.
+.It Li QMC_BUSY.WRITE.CH0
+.Pq Event 29H , Umask 08H
+Counts cycles where Quickpath Memory Controller has at least 1 outstanding
+write request to DRAM channel 0.
+.It Li QMC_BUSY.WRITE.CH1
+.Pq Event 29H , Umask 10H
+Counts cycles where Quickpath Memory Controller has at least 1 outstanding
+write request to DRAM channel 1.
+.It Li QMC_BUSY.WRITE.CH2
+.Pq Event 29H , Umask 20H
+Counts cycles where Quickpath Memory Controller has at least 1 outstanding
+write request to DRAM channel 2.
+.It Li QMC_OCCUPANCY.CH0
+.Pq Event 2AH , Umask 01H
+IMC channel 0 normal read request occupancy.
+.It Li QMC_OCCUPANCY.CH1
+.Pq Event 2AH , Umask 02H
+IMC channel 1 normal read request occupancy.
+.It Li QMC_OCCUPANCY.CH2
+.Pq Event 2AH , Umask 04H
+IMC channel 2 normal read request occupancy.
+.It Li QMC_OCCUPANCY.ANY
+.Pq Event 2AH , Umask 07H
+Normal read request occupancy for any channel.
+.It Li QMC_ISSOC_OCCUPANCY.CH0
+.Pq Event 2BH , Umask 01H
+IMC channel 0 issoc read request occupancy.
+.It Li QMC_ISSOC_OCCUPANCY.CH1
+.Pq Event 2BH , Umask 02H
+IMC channel 1 issoc read request occupancy.
+.It Li QMC_ISSOC_OCCUPANCY.CH2
+.Pq Event 2BH , Umask 04H
+IMC channel 2 issoc read request occupancy.
+.It Li QMC_ISSOC_READS.ANY
+.Pq Event 2BH , Umask 07H
+IMC issoc read request occupancy.
+.It Li QMC_NORMAL_READS.CH0
+.Pq Event 2CH , Umask 01H
+Counts the number of Quickpath Memory Controller channel 0 medium and low
+priority read requests. The QMC channel 0 normal read occupancy divided by
+this count provides the average QMC channel 0 read latency.
+.It Li QMC_NORMAL_READS.CH1
+.Pq Event 2CH , Umask 02H
+Counts the number of Quickpath Memory Controller channel 1 medium and low
+priority read requests. The QMC channel 1 normal read occupancy divided by
+this count provides the average QMC channel 1 read latency.
+.It Li QMC_NORMAL_READS.CH2
+.Pq Event 2CH , Umask 04H
+Counts the number of Quickpath Memory Controller channel 2 medium and low
+priority read requests. The QMC channel 2 normal read occupancy divided by
+this count provides the average QMC channel 2 read latency.
+.It Li QMC_NORMAL_READS.ANY
+.Pq Event 2CH , Umask 07H
+Counts the number of Quickpath Memory Controller medium and low priority
+read requests. The QMC normal read occupancy divided by this count provides
+the average QMC read latency.
+.It Li QMC_HIGH_PRIORITY_READS.CH0
+.Pq Event 2DH , Umask 01H
+Counts the number of Quickpath Memory Controller channel 0 high priority
+isochronous read requests.
+.It Li QMC_HIGH_PRIORITY_READS.CH1
+.Pq Event 2DH , Umask 02H
+Counts the number of Quickpath Memory Controller channel 1 high priority
+isochronous read requests.
+.It Li QMC_HIGH_PRIORITY_READS.CH2
+.Pq Event 2DH , Umask 04H
+Counts the number of Quickpath Memory Controller channel 2 high priority
+isochronous read requests.
+.It Li QMC_HIGH_PRIORITY_READS.ANY
+.Pq Event 2DH , Umask 07H
+Counts the number of Quickpath Memory Controller high priority isochronous
+read requests.
+.It Li QMC_CRITICAL_PRIORITY_READS.CH0
+.Pq Event 2EH , Umask 01H
+Counts the number of Quickpath Memory Controller channel 0 critical priority
+isochronous read requests.
+.It Li QMC_CRITICAL_PRIORITY_READS.CH1
+.Pq Event 2EH , Umask 02H
+Counts the number of Quickpath Memory Controller channel 1 critical priority
+isochronous read requests.
+.It Li QMC_CRITICAL_PRIORITY_READS.CH2
+.Pq Event 2EH , Umask 04H
+Counts the number of Quickpath Memory Controller channel 2 critical priority
+isochronous read requests.
+.It Li QMC_CRITICAL_PRIORITY_READS.ANY
+.Pq Event 2EH , Umask 07H
+Counts the number of Quickpath Memory Controller critical priority
+isochronous read requests.
+.It Li QMC_WRITES.FULL.CH0
+.Pq Event 2FH , Umask 01H
+Counts number of full cache line writes to DRAM channel 0.
+.It Li QMC_WRITES.FULL.CH1
+.Pq Event 2FH , Umask 02H
+Counts number of full cache line writes to DRAM channel 1.
+.It Li QMC_WRITES.FULL.CH2
+.Pq Event 2FH , Umask 04H
+Counts number of full cache line writes to DRAM channel 2.
+.It Li QMC_WRITES.FULL.ANY
+.Pq Event 2FH , Umask 07H
+Counts number of full cache line writes to DRAM.
+.It Li QMC_WRITES.PARTIAL.CH0
+.Pq Event 2FH , Umask 08H
+Counts number of partial cache line writes to DRAM channel 0.
+.It Li QMC_WRITES.PARTIAL.CH1
+.Pq Event 2FH , Umask 10H
+Counts number of partial cache line writes to DRAM channel 1.
+.It Li QMC_WRITES.PARTIAL.CH2
+.Pq Event 2FH , Umask 20H
+Counts number of partial cache line writes to DRAM channel 2.
+.It Li QMC_WRITES.PARTIAL.ANY
+.Pq Event 2FH , Umask 38H
+Counts number of partial cache line writes to DRAM.
+.It Li QMC_CANCEL.CH0
+.Pq Event 30H , Umask 01H
+Counts number of DRAM channel 0 cancel requests.
+.It Li QMC_CANCEL.CH1
+.Pq Event 30H , Umask 02H
+Counts number of DRAM channel 1 cancel requests.
+.It Li QMC_CANCEL.CH2
+.Pq Event 30H , Umask 04H
+Counts number of DRAM channel 2 cancel requests.
+.It Li QMC_CANCEL.ANY
+.Pq Event 30H , Umask 07H
+Counts number of DRAM cancel requests.
+.It Li QMC_PRIORITY_UPDATES.CH0
+.Pq Event 31H , Umask 01H
+Counts number of DRAM channel 0 priority updates. A priority update occurs
+when an ISOC high or critical request is received by the QHL and there is a
+matching request with normal priority that has already been issued to the
+QMC. In this instance, the QHL will send a priority update to QMC to
+expedite the request.
+.It Li QMC_PRIORITY_UPDATES.CH1
+.Pq Event 31H , Umask 02H
+Counts number of DRAM channel 1 priority updates. A priority update occurs
+when an ISOC high or critical request is received by the QHL and there is a
+matching request with normal priority that has already been issued to the
+QMC. In this instance, the QHL will send a priority update to QMC to
+expedite the request.
+.It Li QMC_PRIORITY_UPDATES.CH2
+.Pq Event 31H , Umask 04H
+Counts number of DRAM channel 2 priority updates. A priority update occurs
+when an ISOC high or critical request is received by the QHL and there is a
+matching request with normal priority that has already been issued to the
+QMC. In this instance, the QHL will send a priority update to QMC to
+expedite the request.
+.It Li QMC_PRIORITY_UPDATES.ANY
+.Pq Event 31H , Umask 07H
+Counts number of DRAM priority updates. A priority update occurs when an
+ISOC high or critical request is received by the QHL and there is a matching
+request with normal priority that has already been issued to the QMC. In
+this instance, the QHL will send a priority update to QMC to expedite the
+request.
+.It Li IMC_RETRY.CH0
+.Pq Event 32H , Umask 01H
+Counts number of IMC DRAM channel 0 retries. DRAM retry only occurs when
+configured in RAS mode.
+.It Li IMC_RETRY.CH1
+.Pq Event 32H , Umask 02H
+Counts number of IMC DRAM channel 1 retries. DRAM retry only occurs when
+configured in RAS mode.
+.It Li IMC_RETRY.CH2
+.Pq Event 32H , Umask 04H
+Counts number of IMC DRAM channel 2 retries. DRAM retry only occurs when
+configured in RAS mode.
+.It Li IMC_RETRY.ANY
+.Pq Event 32H , Umask 07H
+Counts number of IMC DRAM retries from any channel. DRAM retry only occurs
+when configured in RAS mode.
+.It Li QHL_FRC_ACK_CNFLTS.IOH
+.Pq Event 33H , Umask 01H
+Counts number of Force Acknowledge Conflict messages sent by the Quickpath
+Home Logic to the IOH.
+.It Li QHL_FRC_ACK_CNFLTS.REMOTE
+.Pq Event 33H , Umask 02H
+Counts number of Force Acknowledge Conflict messages sent by the Quickpath
+Home Logic to the remote home.
+.It Li QHL_FRC_ACK_CNFLTS.LOCAL
+.Pq Event 33H , Umask 04H
+Counts number of Force Acknowledge Conflict messages sent by the Quickpath
+Home Logic to the local home.
+.It Li QHL_FRC_ACK_CNFLTS.ANY
+.Pq Event 33H , Umask 07H
+Counts number of Force Acknowledge Conflict messages sent by the Quickpath
+Home Logic.
+.It Li QHL_SLEEPS.IOH_ORDER
+.Pq Event 34H , Umask 01H
+Counts number of occurrences a request was put to sleep due to IOH ordering
+(write after read) conflicts. While in the sleep state, the request is not
+eligible to be scheduled to the QMC.
+.It Li QHL_SLEEPS.REMOTE_ORDER
+.Pq Event 34H , Umask 02H
+Counts number of occurrences a request was put to sleep due to remote socket
+ordering (write after read) conflicts. While in the sleep state, the request
+is not eligible to be scheduled to the QMC.
+.It Li QHL_SLEEPS.LOCAL_ORDER
+.Pq Event 34H , Umask 04H
+Counts number of occurrences a request was put to sleep due to local socket
+ordering (write after read) conflicts. While in the sleep state, the request
+is not eligible to be scheduled to the QMC.
+.It Li QHL_SLEEPS.IOH_CONFLICT
+.Pq Event 34H , Umask 08H
+Counts number of occurrences a request was put to sleep due to IOH address
+conflicts. While in the sleep state, the request is not eligible to be
+scheduled to the QMC.
+.It Li QHL_SLEEPS.REMOTE_CONFLICT
+.Pq Event 34H , Umask 10H
+Counts number of occurrences a request was put to sleep due to remote socket
+address conflicts. While in the sleep state, the request is not eligible to
+be scheduled to the QMC.
+.It Li QHL_SLEEPS.LOCAL_CONFLICT
+.Pq Event 34H , Umask 20H
+Counts number of occurrences a request was put to sleep due to local socket
+address conflicts. While in the sleep state, the request is not eligible to
+be scheduled to the QMC.
+.It Li ADDR_OPCODE_MATCH.IOH
+.Pq Event 35H , Umask 01H
+Counts number of requests from the IOH, address/opcode of request is
+qualified by mask value written to MSR 396H. The following mask values are
+supported:
+0: NONE 40000000_00000000H:RSPFWDI 40001A00_00000000H:RSPFWDS
+40001D00_00000000H:RSPIWB
+Match opcode/addres s by writing MSR 396H with mask supported mask value
+.It Li ADDR_OPCODE_MATCH.REMOTE
+.Pq Event 35H , Umask 02H
+Counts number of requests from the remote socket, address/opcode of request
+is qualified by mask value written to MSR 396H. The following mask values
+are supported:
+0: NONE 40000000_00000000H:RSPFWDI 40001A00_00000000H:RSPFWDS
+40001D00_00000000H:RSPIWB
+Match opcode/addres s by writing MSR 396H with mask supported mask value
+.It Li ADDR_OPCODE_MATCH.LOCAL
+.Pq Event 35H , Umask 04H
+Counts number of requests from the local socket, address/opcode of request
+is qualified by mask value written to MSR 396H. The following mask values
+are supported:
+0: NONE 40000000_00000000H:RSPFWDI 40001A00_00000000H:RSPFWDS
+40001D00_00000000H:RSPIWB
+Match opcode/addres s by writing MSR 396H with mask supported mask value
+.It Li QPI_TX_STALLED_SINGLE_FLIT.HOME.LINK_0
+.Pq Event 40H , Umask 01H
+Counts cycles the Quickpath outbound link 0 HOME virtual channel is stalled
+due to lack of a VNA and VN0 credit. Note that this event does not filter
+out when a flit would not have been selected for arbitration because another
+virtual channel is getting arbitrated.
+.It Li QPI_TX_STALLED_SINGLE_FLIT.SNOOP.LINK_0
+.Pq Event 40H , Umask 02H
+Counts cycles the Quickpath outbound link 0 SNOOP virtual channel is stalled
+due to lack of a VNA and VN0 credit. Note that this event does not filter
+out when a flit would not have been selected for arbitration because another
+virtual channel is getting arbitrated.
+.It Li QPI_TX_STALLED_SINGLE_FLIT.NDR.LINK_0
+.Pq Event 40H , Umask 04H
+Counts cycles the Quickpath outbound link 0 non-data response virtual
+channel is stalled due to lack of a VNA and VN0 credit. Note that this event
+does not filter out when a flit would not have been selected for arbitration
+because another virtual channel is getting arbitrated.
+.It Li QPI_TX_STALLED_SINGLE_FLIT.HOME.LINK_1
+.Pq Event 40H , Umask 08H
+Counts cycles the Quickpath outbound link 1 HOME virtual channel is stalled
+due to lack of a VNA and VN0 credit. Note that this event does not filter
+out when a flit would not have been selected for arbitration because another
+virtual channel is getting arbitrated.
+.It Li QPI_TX_STALLED_SINGLE_FLIT.SNOOP.LINK_1
+.Pq Event 40H , Umask 10H
+Counts cycles the Quickpath outbound link 1 SNOOP virtual channel is stalled
+due to lack of a VNA and VN0 credit. Note that this event does not filter
+out when a flit would not have been selected for arbitration because another
+virtual channel is getting arbitrated.
+.It Li QPI_TX_STALLED_SINGLE_FLIT.NDR.LINK_1
+.Pq Event 40H , Umask 20H
+Counts cycles the Quickpath outbound link 1 non-data response virtual
+channel is stalled due to lack of a VNA and VN0 credit. Note that this event
+does not filter out when a flit would not have been selected for arbitration
+because another virtual channel is getting arbitrated.
+.It Li QPI_TX_STALLED_SINGLE_FLIT.LINK_0
+.Pq Event 40H , Umask 07H
+Counts cycles the Quickpath outbound link 0 virtual channels are stalled due
+to lack of a VNA and VN0 credit. Note that this event does not filter out
+when a flit would not have been selected for arbitration because another
+virtual channel is getting arbitrated.
+.It Li QPI_TX_STALLED_SINGLE_FLIT.LINK_1
+.Pq Event 40H , Umask 38H
+Counts cycles the Quickpath outbound link 1 virtual channels are stalled due
+to lack of a VNA and VN0 credit. Note that this event does not filter out
+when a flit would not have been selected for arbitration because another
+virtual channel is getting arbitrated.
+.It Li QPI_TX_STALLED_MULTI_FLIT.DRS.LINK_0
+.Pq Event 41H , Umask 01H
+Counts cycles the Quickpath outbound link 0 Data ResponSe virtual channel is
+stalled due to lack of VNA and VN0 credits. Note that this event does not
+filter out when a flit would not have been selected for arbitration because
+another virtual channel is getting arbitrated.
+.It Li QPI_TX_STALLED_MULTI_FLIT.NCB.LINK_0
+.Pq Event 41H , Umask 02H
+Counts cycles the Quickpath outbound link 0 Non-Coherent Bypass virtual
+channel is stalled due to lack of VNA and VN0 credits. Note that this event
+does not filter out when a flit would not have been selected for arbitration
+because another virtual channel is getting arbitrated.
+.It Li QPI_TX_STALLED_MULTI_FLIT.NCS.LINK_0
+.Pq Event 41H , Umask 04H
+Counts cycles the Quickpath outbound link 0 Non-Coherent Standard virtual
+channel is stalled due to lack of VNA and VN0 credits. Note that this event
+does not filter out when a flit would not have been selected for arbitration
+because another virtual channel is getting arbitrated.
+.It Li QPI_TX_STALLED_MULTI_FLIT.DRS.LINK_1
+.Pq Event 41H , Umask 08H
+Counts cycles the Quickpath outbound link 1 Data ResponSe virtual channel is
+stalled due to lack of VNA and VN0 credits. Note that this event does not
+filter out when a flit would not have been selected for arbitration because
+another virtual channel is getting arbitrated.
+.It Li QPI_TX_STALLED_MULTI_FLIT.NCB.LINK_1
+.Pq Event 41H , Umask 10H
+Counts cycles the Quickpath outbound link 1 Non-Coherent Bypass virtual
+channel is stalled due to lack of VNA and VN0 credits. Note that this event
+does not filter out when a flit would not have been selected for arbitration
+because another virtual channel is getting arbitrated.
+.It Li QPI_TX_STALLED_MULTI_FLIT.NCS.LINK_1
+.Pq Event 41H , Umask 20H
+Counts cycles the Quickpath outbound link 1 Non-Coherent Standard virtual
+channel is stalled due to lack of VNA and VN0 credits. Note that this event
+does not filter out when a flit would not have been selected for arbitration
+because another virtual channel is getting arbitrated.
+.It Li QPI_TX_STALLED_MULTI_FLIT.LINK_0
+.Pq Event 41H , Umask 07H
+Counts cycles the Quickpath outbound link 0 virtual channels are stalled due
+to lack of VNA and VN0 credits. Note that this event does not filter out
+when a flit would not have been selected for arbitration because another
+virtual channel is getting arbitrated.
+.It Li QPI_TX_STALLED_MULTI_FLIT.LINK_1
+.Pq Event 41H , Umask 38H
+Counts cycles the Quickpath outbound link 1 virtual channels are stalled due
+to lack of VNA and VN0 credits. Note that this event does not filter out
+when a flit would not have been selected for arbitration because another
+virtual channel is getting arbitrated.
+.It Li QPI_TX_HEADER.FULL.LINK_0
+.Pq Event 42H , Umask 01H
+Number of cycles that the header buffer in the Quickpath Interface outbound
+link 0 is full.
+.It Li QPI_TX_HEADER.BUSY.LINK_0
+.Pq Event 42H , Umask 02H
+Number of cycles that the header buffer in the Quickpath Interface outbound
+link 0 is busy.
+.It Li QPI_TX_HEADER.FULL.LINK_1
+.Pq Event 42H , Umask 04H
+Number of cycles that the header buffer in the Quickpath Interface outbound
+link 1 is full.
+.It Li QPI_TX_HEADER.BUSY.LINK_1
+.Pq Event 42H , Umask 08H
+Number of cycles that the header buffer in the Quickpath Interface outbound
+link 1 is busy.
+.It Li QPI_RX_NO_PPT_CREDIT.STALLS.LINK_0
+.Pq Event 43H , Umask 01H
+Number of cycles that snoop packets incoming to the Quickpath Interface link
+0 are stalled and not sent to the GQ because the GQ Peer Probe Tracker (PPT)
+does not have any available entries.
+.It Li QPI_RX_NO_PPT_CREDIT.STALLS.LINK_1
+.Pq Event 43H , Umask 02H
+Number of cycles that snoop packets incoming to the Quickpath Interface link
+1 are stalled and not sent to the GQ because the GQ Peer Probe Tracker (PPT)
+does not have any available entries.
+.It Li DRAM_OPEN.CH0
+.Pq Event 60H , Umask 01H
+Counts number of DRAM Channel 0 open commands issued either for read or
+write. To read or write data, the referenced DRAM page must first be opened.
+.It Li DRAM_OPEN.CH1
+.Pq Event 60H , Umask 02H
+Counts number of DRAM Channel 1 open commands issued either for read or
+write. To read or write data, the referenced DRAM page must first be opened.
+.It Li DRAM_OPEN.CH2
+.Pq Event 60H , Umask 04H
+Counts number of DRAM Channel 2 open commands issued either for read or
+write. To read or write data, the referenced DRAM page must first be opened.
+.It Li DRAM_PAGE_CLOSE.CH0
+.Pq Event 61H , Umask 01H
+DRAM channel 0 command issued to CLOSE a page due to page idle timer
+expiration. Closing a page is done by issuing a precharge.
+.It Li DRAM_PAGE_CLOSE.CH1
+.Pq Event 61H , Umask 02H
+DRAM channel 1 command issued to CLOSE a page due to page idle timer
+expiration. Closing a page is done by issuing a precharge.
+.It Li DRAM_PAGE_CLOSE.CH2
+.Pq Event 61H , Umask 04H
+DRAM channel 2 command issued to CLOSE a page due to page idle timer
+expiration. Closing a page is done by issuing a precharge.
+.It Li DRAM_PAGE_MISS.CH0
+.Pq Event 62H , Umask 01H
+Counts the number of precharges (PRE) that were issued to DRAM channel 0
+because there was a page miss. A page miss refers to a situation in which a
+page is currently open and another page from the same bank needs to be
+opened. The new page experiences a page miss. Closing of the old page is
+done by issuing a precharge.
+.It Li DRAM_PAGE_MISS.CH1
+.Pq Event 62H , Umask 02H
+Counts the number of precharges (PRE) that were issued to DRAM channel 1
+because there was a page miss. A page miss refers to a situation in which a
+page is currently open and another page from the same bank needs to be
+opened. The new page experiences a page miss. Closing of the old page is
+done by issuing a precharge.
+.It Li DRAM_PAGE_MISS.CH2
+.Pq Event 62H , Umask 04H
+Counts the number of precharges (PRE) that were issued to DRAM channel 2
+because there was a page miss. A page miss refers to a situation in which a
+page is currently open and another page from the same bank needs to be
+opened. The new page experiences a page miss. Closing of the old page is
+done by issuing a precharge.
+.It Li DRAM_READ_CAS.CH0
+.Pq Event 63H , Umask 01H
+Counts the number of times a read CAS command was issued on DRAM channel 0.
+.It Li DRAM_READ_CAS.AUTOPRE_CH0
+.Pq Event 63H , Umask 02H
+Counts the number of times a read CAS command was issued on DRAM channel 0
+where the command issued used the auto-precharge (auto page close) mode.
+.It Li DRAM_READ_CAS.CH1
+.Pq Event 63H , Umask 04H
+Counts the number of times a read CAS command was issued on DRAM channel 1.
+.It Li DRAM_READ_CAS.AUTOPRE_CH1
+.Pq Event 63H , Umask 08H
+Counts the number of times a read CAS command was issued on DRAM channel 1
+where the command issued used the auto-precharge (auto page close) mode.
+.It Li DRAM_READ_CAS.CH2
+.Pq Event 63H , Umask 10H
+Counts the number of times a read CAS command was issued on DRAM channel 2.
+.It Li DRAM_READ_CAS.AUTOPRE_CH2
+.Pq Event 63H , Umask 20H
+Counts the number of times a read CAS command was issued on DRAM channel 2
+where the command issued used the auto-precharge (auto page close) mode.
+.It Li DRAM_WRITE_CAS.CH0
+.Pq Event 64H , Umask 01H
+Counts the number of times a write CAS command was issued on DRAM channel 0.
+.It Li DRAM_WRITE_CAS.AUTOPRE_CH0
+.Pq Event 64H , Umask 02H
+Counts the number of times a write CAS command was issued on DRAM channel 0
+where the command issued used the auto-precharge (auto page close) mode.
+.It Li DRAM_WRITE_CAS.CH1
+.Pq Event 64H , Umask 04H
+Counts the number of times a write CAS command was issued on DRAM channel 1.
+.It Li DRAM_WRITE_CAS.AUTOPRE_CH1
+.Pq Event 64H , Umask 08H
+Counts the number of times a write CAS command was issued on DRAM channel 1
+where the command issued used the auto-precharge (auto page close) mode.
+.It Li DRAM_WRITE_CAS.CH2
+.Pq Event 64H , Umask 10H
+Counts the number of times a write CAS command was issued on DRAM channel 2.
+.It Li DRAM_WRITE_CAS.AUTOPRE_CH2
+.Pq Event 64H , Umask 20H
+Counts the number of times a write CAS command was issued on DRAM channel 2
+where the command issued used the auto-precharge (auto page close) mode.
+.It Li DRAM_REFRESH.CH0
+.Pq Event 65H , Umask 01H
+Counts number of DRAM channel 0 refresh commands. DRAM loses data content
+over time. In order to keep correct data content, the data values have to be
+refreshed periodically.
+.It Li DRAM_REFRESH.CH1
+.Pq Event 65H , Umask 02H
+Counts number of DRAM channel 1 refresh commands. DRAM loses data content
+over time. In order to keep correct data content, the data values have to be
+refreshed periodically.
+.It Li DRAM_REFRESH.CH2
+.Pq Event 65H , Umask 04H
+Counts number of DRAM channel 2 refresh commands. DRAM loses data content
+over time. In order to keep correct data content, the data values have to be
+refreshed periodically.
+.It Li DRAM_PRE_ALL.CH0
+.Pq Event 66H , Umask 01H
+Counts number of DRAM Channel 0 precharge-all (PREALL) commands that close
+all open pages in a rank. PREALL is issued when the DRAM needs to be
+refreshed or needs to go into a power down mode.
+.It Li DRAM_PRE_ALL.CH1
+.Pq Event 66H , Umask 02H
+Counts number of DRAM Channel 1 precharge-all (PREALL) commands that close
+all open pages in a rank. PREALL is issued when the DRAM needs to be
+refreshed or needs to go into a power down mode.
+.It Li DRAM_PRE_ALL.CH2
+.Pq Event 66H , Umask 04H
+Counts number of DRAM Channel 2 precharge-all (PREALL) commands that close
+all open pages in a rank. PREALL is issued when the DRAM needs to be
+refreshed or needs to go into a power down mode.
+.It Li DRAM_THERMAL_THROTTLED
+.Pq Event 67H , Umask 01H
+Uncore cycles DRAM was throttled due to its temperature being above the
+thermal throttling threshold.
+.It Li THERMAL_THROTTLING_TEMP.CORE_0
+.Pq Event 80H , Umask 01H
+Cycles that the PCU records that core 0 is above the thermal throttling
+threshold temperature.
+.It Li THERMAL_THROTTLING_TEMP.CORE_1
+.Pq Event 80H , Umask 02H
+Cycles that the PCU records that core 1 is above the thermal throttling
+threshold temperature.
+.It Li THERMAL_THROTTLING_TEMP.CORE_2
+.Pq Event 80H , Umask 04H
+Cycles that the PCU records that core 2 is above the thermal throttling
+threshold temperature.
+.It Li THERMAL_THROTTLING_TEMP.CORE_3
+.Pq Event 80H , Umask 08H
+Cycles that the PCU records that core 3 is above the thermal throttling
+threshold temperature.
+.It Li THERMAL_THROTTLED_TEMP.CORE_0
+.Pq Event 81H , Umask 01H
+Cycles that the PCU records that core 0 is in the power throttled state due
+to cores temperature being above the thermal throttling threshold.
+.It Li THERMAL_THROTTLED_TEMP.CORE_1
+.Pq Event 81H , Umask 02H
+Cycles that the PCU records that core 1 is in the power throttled state due
+to cores temperature being above the thermal throttling threshold.
+.It Li THERMAL_THROTTLED_TEMP.CORE_2
+.Pq Event 81H , Umask 04H
+Cycles that the PCU records that core 2 is in the power throttled state due
+to cores temperature being above the thermal throttling threshold.
+.It Li THERMAL_THROTTLED_TEMP.CORE_3
+.Pq Event 81H , Umask 08H
+Cycles that the PCU records that core 3 is in the power throttled state due
+to cores temperature being above the thermal throttling threshold.
+.It Li PROCHOT_ASSERTION
+.Pq Event 82H , Umask 01H
+Number of system assertions of PROCHOT indicating the entire processor has
+exceeded the thermal limit.
+.It Li THERMAL_THROTTLING_PROCHOT.CORE_0
+.Pq Event 83H , Umask 01H
+Cycles that the PCU records that core 0 is a low power state due to the
+system asserting PROCHOT the entire processor has exceeded the thermal
+limit.
+.It Li THERMAL_THROTTLING_PROCHOT.CORE_1
+.Pq Event 83H , Umask 02H
+Cycles that the PCU records that core 1 is a low power state due to the
+system asserting PROCHOT the entire processor has exceeded the thermal
+limit.
+.It Li THERMAL_THROTTLING_PROCHOT.CORE_2
+.Pq Event 83H , Umask 04H
+Cycles that the PCU records that core 2 is a low power state due to the
+system asserting PROCHOT the entire processor has exceeded the thermal
+limit.
+.It Li THERMAL_THROTTLING_PROCHOT.CORE_3
+.Pq Event 83H , Umask 08H
+Cycles that the PCU records that core 3 is a low power state due to the
+system asserting PROCHOT the entire processor has exceeded the thermal
+limit.
+.It Li TURBO_MODE.CORE_0
+.Pq Event 84H , Umask 01H
+Uncore cycles that core 0 is operating in turbo mode.
+.It Li TURBO_MODE.CORE_1
+.Pq Event 84H , Umask 02H
+Uncore cycles that core 1 is operating in turbo mode.
+.It Li TURBO_MODE.CORE_2
+.Pq Event 84H , Umask 04H
+Uncore cycles that core 2 is operating in turbo mode.
+.It Li TURBO_MODE.CORE_3
+.Pq Event 84H , Umask 08H
+Uncore cycles that core 3 is operating in turbo mode.
+.It Li CYCLES_UNHALTED_L3_FLL_ENABLE
+.Pq Event 85H , Umask 02H
+Uncore cycles that at least one core is unhalted and all L3 ways are
+enabled.
+.It Li CYCLES_UNHALTED_L3_FLL_DISABLE
+.Pq Event 86H , Umask 01H
+Uncore cycles that at least one core is unhalted and all L3 ways are
+disabled.
+.El
+.Sh SEE ALSO
+.Xr pmc 3 ,
+.Xr pmc.atom 3 ,
+.Xr pmc.core 3 ,
+.Xr pmc.iaf 3 ,
+.Xr pmc.ucf 3 ,
+.Xr pmc.k7 3 ,
+.Xr pmc.k8 3 ,
+.Xr pmc.p4 3 ,
+.Xr pmc.p5 3 ,
+.Xr pmc.p6 3 ,
+.Xr pmc.corei7 3 ,
+.Xr pmc.corei7uc 3 ,
+.Xr pmc.westmere 3 ,
+.Xr pmc.tsc 3 ,
+.Xr pmc_cpuinfo 3 ,
+.Xr pmclog 3 ,
+.Xr hwpmc 4
+.Sh HISTORY
+The
+.Nm pmc
+library first appeared in
+.Fx 6.0 .
+.Sh AUTHORS
+The
+.Lb libpmc
+library was written by
+.An "Joseph Koshy"
+.Aq jkoshy@FreeBSD.org .
-- 
cgit v1.1


From 79412ab1a15b9e5939e48ea214459a68d9ec5193 Mon Sep 17 00:00:00 2001
From: ume <ume@FreeBSD.org>
Date: Sun, 18 Apr 2010 03:52:41 +0000
Subject: MFC ir206152, r206153, r206154: - Stop adding trailing '\n'.  The
 servent_unpack() doesn't expect   lines terminated with '\n'. - Treat '+' as
 special only when in compat mode, and simplify   the logic bit. - Reduce
 duplicate code.

---
 lib/libc/net/getservent.c | 106 +++++++++++++++-------------------------------
 1 file changed, 35 insertions(+), 71 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/net/getservent.c b/lib/libc/net/getservent.c
index ebb7a7d..3dcad3f 100644
--- a/lib/libc/net/getservent.c
+++ b/lib/libc/net/getservent.c
@@ -207,6 +207,32 @@ servent_unpack(char *p, struct servent *serv, char **aliases,
 	return 0;
 }
 
+static int
+parse_result(struct servent *serv, char *buffer, size_t bufsize,
+    char *resultbuf, size_t resultbuflen, int *errnop)
+{
+	char **aliases;
+	int aliases_size;
+
+	if (bufsize <= resultbuflen + _ALIGNBYTES + sizeof(char *)) {
+		*errnop = ERANGE;
+		return (NS_RETURN);
+	}
+	aliases = (char **)_ALIGN(&buffer[resultbuflen + 1]);
+	aliases_size = (buffer + bufsize - (char *)aliases) / sizeof(char *);
+	if (aliases_size < 1) {
+		*errnop = ERANGE;
+		return (NS_RETURN);
+	}
+
+	memcpy(buffer, resultbuf, resultbuflen);
+	buffer[resultbuflen] = '\0';
+
+	if (servent_unpack(buffer, serv, aliases, aliases_size, errnop) != 0)
+		return ((*errnop == 0) ? NS_NOTFOUND : NS_RETURN);
+	return (NS_SUCCESS);
+}
+
 /* files backend implementation */
 static	void
 files_endstate(void *p)
@@ -258,8 +284,6 @@ files_servent(void *retval, void *mdata, va_list ap)
 	size_t bufsize;
 	int *errnop;
 
-	char **aliases;
-	int aliases_size;
 	size_t linesize;
 	char *line;
 	char **cp;
@@ -315,28 +339,8 @@ files_servent(void *retval, void *mdata, va_list ap)
 				break;
 			}
 
-			if (*line=='+') {
-				if (serv_mdata->compat_mode != 0)
-					st->compat_mode_active = 1;
-			} else {
-				if (bufsize <= linesize + _ALIGNBYTES +
-				    sizeof(char *)) {
-					*errnop = ERANGE;
-					rv = NS_RETURN;
-					break;
-				}
-				aliases = (char **)_ALIGN(&buffer[linesize+1]);
-				aliases_size = (buffer + bufsize -
-				    (char *)aliases) / sizeof(char *);
-				if (aliases_size < 1) {
-					*errnop = ERANGE;
-					rv = NS_RETURN;
-					break;
-				}
-
-				memcpy(buffer, line, linesize);
-				buffer[linesize] = '\0';
-			}
+			if (*line=='+' && serv_mdata->compat_mode != 0)
+				st->compat_mode_active = 1;
 		}
 
 		if (st->compat_mode_active != 0) {
@@ -367,18 +371,12 @@ files_servent(void *retval, void *mdata, va_list ap)
 			continue;
 		}
 
-		rv = servent_unpack(buffer, serv, aliases, aliases_size,
+		rv = parse_result(serv, buffer, bufsize, line, linesize,
 		    errnop);
-		if (rv !=0 ) {
-			if (*errnop == 0) {
-				rv = NS_NOTFOUND;
-				continue;
-			}
-			else {
-				rv = NS_RETURN;
-				break;
-			}
-		}
+		if (rv == NS_NOTFOUND)
+			continue;
+		if (rv == NS_RETURN)
+			break;
 
 		rv = NS_NOTFOUND;
 		switch (serv_mdata->how) {
@@ -486,9 +484,6 @@ nis_servent(void *retval, void *mdata, va_list ap)
 	size_t bufsize;
 	int *errnop;
 
-	char **aliases;
-	int aliases_size;
-
 	name = NULL;
 	proto = NULL;
 	how = (enum nss_lookup_type)mdata;
@@ -594,39 +589,8 @@ nis_servent(void *retval, void *mdata, va_list ap)
 			break;
 		};
 
-		/* we need a room for additional \n symbol */
-		if (bufsize <=
-		    resultbuflen + 1 + _ALIGNBYTES + sizeof(char *)) {
-			*errnop = ERANGE;
-			rv = NS_RETURN;
-			break;
-		}
-
-		aliases = (char **)_ALIGN(&buffer[resultbuflen + 2]);
-		aliases_size =
-		    (buffer + bufsize - (char *)aliases) / sizeof(char *);
-		if (aliases_size < 1) {
-			*errnop = ERANGE;
-			rv = NS_RETURN;
-			break;
-		}
-
-		/*
-		 * servent_unpack expects lines terminated with \n --
-		 * make it happy
-		 */
-		memcpy(buffer, resultbuf, resultbuflen);
-		buffer[resultbuflen] = '\n';
-		buffer[resultbuflen + 1] = '\0';
-
-		if (servent_unpack(buffer, serv, aliases, aliases_size,
-		    errnop) != 0) {
-			if (*errnop == 0)
-				rv = NS_NOTFOUND;
-			else
-				rv = NS_RETURN;
-		} else
-			rv = NS_SUCCESS;
+		rv = parse_result(serv, buffer, bufsize, resultbuf,
+		    resultbuflen, errnop);
 		free(resultbuf);
 
 	} while (!(rv & NS_TERMINATE) && how == nss_lt_all);
-- 
cgit v1.1


From 50edecc45441eac7385beca569f1da14f1b9a490 Mon Sep 17 00:00:00 2001
From: ume <ume@FreeBSD.org>
Date: Sun, 18 Apr 2010 04:07:32 +0000
Subject: MFC r206155, r206267: Add capability to use a db version of services.
  It is enabled by specifying `db' as source of services in
 /etc/nsswitch.conf.

---
 lib/libc/net/getservent.c | 200 ++++++++++++++++++++++++++++++++++++++++++++++
 lib/libc/net/nsdispatch.3 |   3 +-
 2 files changed, 202 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libc/net/getservent.c b/lib/libc/net/getservent.c
index 3dcad3f..3228bdc 100644
--- a/lib/libc/net/getservent.c
+++ b/lib/libc/net/getservent.c
@@ -37,7 +37,9 @@ __FBSDID("$FreeBSD$");
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <arpa/inet.h>
+#include <db.h>
 #include <errno.h>
+#include <fcntl.h>
 #include <limits.h>
 #include <netdb.h>
 #include <nsswitch.h>
@@ -94,6 +96,19 @@ NSS_TLS_HANDLING(files);
 static int files_servent(void *, void *, va_list);
 static int files_setservent(void *, void *, va_list);
 
+/* db backend declarations */
+struct db_state
+{
+        DB *db;
+	int stayopen;
+	int keynum;
+};
+static void db_endstate(void *);
+NSS_TLS_HANDLING(db);
+
+static int db_servent(void *, void *, va_list);
+static int db_setservent(void *, void *, va_list);
+
 #ifdef YP
 /* nis backend declarations */
 static 	int 	nis_servent(void *, void *, va_list);
@@ -263,6 +278,8 @@ files_servent(void *retval, void *mdata, va_list ap)
 		{ NULL, 0 }
 	};
 	ns_dtab compat_dtab[] = {
+		{ NSSRC_DB, db_servent,
+			(void *)((struct servent_mdata *)mdata)->how },
 #ifdef YP
 		{ NSSRC_NIS, nis_servent,
 			(void *)((struct servent_mdata *)mdata)->how },
@@ -452,6 +469,183 @@ files_setservent(void *retval, void *mdata, va_list ap)
 	return (NS_UNAVAIL);
 }
 
+/* db backend implementation */
+static	void
+db_endstate(void *p)
+{
+	DB *db;
+
+	if (p == NULL)
+		return;
+
+	db = ((struct db_state *)p)->db;
+	if (db != NULL)
+		db->close(db);
+
+	free(p);
+}
+
+static int
+db_servent(void *retval, void *mdata, va_list ap)
+{
+	char buf[BUFSIZ];
+	DBT key, data, *result;
+	DB *db;
+
+	struct db_state *st;
+	int rv;
+	int stayopen;
+
+	enum nss_lookup_type how;
+	char *name;
+	char *proto;
+	int port;
+
+	struct servent *serv;
+	char *buffer;
+	size_t bufsize;
+	int *errnop;
+
+	name = NULL;
+	proto = NULL;
+	how = (enum nss_lookup_type)mdata;
+	switch (how) {
+	case nss_lt_name:
+		name = va_arg(ap, char *);
+		proto = va_arg(ap, char *);
+		break;
+	case nss_lt_id:
+		port = va_arg(ap, int);
+		proto = va_arg(ap, char *);
+		break;
+	case nss_lt_all:
+		break;
+	default:
+		return NS_NOTFOUND;
+	};
+
+	serv = va_arg(ap, struct servent *);
+	buffer  = va_arg(ap, char *);
+	bufsize = va_arg(ap, size_t);
+	errnop = va_arg(ap,int *);
+
+	*errnop = db_getstate(&st);
+	if (*errnop != 0)
+		return (NS_UNAVAIL);
+
+	if (how == nss_lt_all && st->keynum < 0)
+		return (NS_NOTFOUND);
+
+	if (st->db == NULL) {
+		st->db = dbopen(_PATH_SERVICES_DB, O_RDONLY, 0, DB_HASH, NULL);
+		if (st->db == NULL) {
+			*errnop = errno;
+			return (NS_UNAVAIL);
+		}
+	}
+
+	stayopen = (how == nss_lt_all) ? 1 : st->stayopen;
+	db = st->db;
+
+	do {
+		switch (how) {
+		case nss_lt_name:
+			key.data = buf;
+			if (proto == NULL)
+				key.size = snprintf(buf, sizeof(buf),
+				    "\376%s", name);
+			else
+				key.size = snprintf(buf, sizeof(buf),
+				    "\376%s/%s", name, proto);
+			key.size++;
+			if (db->get(db, &key, &data, 0) != 0 ||
+			    db->get(db, &data, &key, 0) != 0) {
+				rv = NS_NOTFOUND;
+				goto db_fin;
+			}
+			result = &key;
+			break;
+		case nss_lt_id:
+			key.data = buf;
+			port = htons(port);
+			if (proto == NULL)
+				key.size = snprintf(buf, sizeof(buf),
+				    "\377%d", port);
+			else
+				key.size = snprintf(buf, sizeof(buf),
+				    "\377%d/%s", port, proto);
+			key.size++;
+			if (db->get(db, &key, &data, 0) != 0 ||
+			    db->get(db, &data, &key, 0) != 0) {
+				rv = NS_NOTFOUND;
+				goto db_fin;
+			}
+			result = &key;
+			break;
+		case nss_lt_all:
+			key.data = buf;
+			key.size = snprintf(buf, sizeof(buf), "%d",
+			    st->keynum++);
+			key.size++;
+			if (db->get(db, &key, &data, 0) != 0) {
+				st->keynum = -1;
+				rv = NS_NOTFOUND;
+				goto db_fin;
+			}
+			result = &data;
+			break;
+		}
+
+		rv = parse_result(serv, buffer, bufsize, result->data,
+		    result->size - 1, errnop);
+
+	} while (!(rv & NS_TERMINATE) && how == nss_lt_all);
+
+db_fin:
+	if (!stayopen && st->db != NULL) {
+		db->close(db);
+		st->db = NULL;
+	}
+
+	if (rv == NS_SUCCESS && retval != NULL)
+		*(struct servent **)retval = serv;
+
+	return (rv);
+}
+
+static int
+db_setservent(void *retval, void *mdata, va_list ap)
+{
+	DB *db;
+	struct db_state *st;
+	int rv;
+	int f;
+
+	rv = db_getstate(&st);
+	if (rv != 0)
+		return (NS_UNAVAIL);
+
+	switch ((enum constants)mdata) {
+	case SETSERVENT:
+		f = va_arg(ap, int);
+		st->stayopen |= f;
+		st->keynum = 0;
+		break;
+	case ENDSERVENT:
+		db = st->db;
+		if (db != NULL) {
+			db->close(db);
+			st->db = NULL;
+		}
+		st->stayopen = 0;
+		break;
+	default:
+		break;
+	};
+
+	return (NS_UNAVAIL);
+}
+
 /* nis backend implementation */
 #ifdef YP
 static 	void
@@ -638,6 +832,7 @@ compat_setservent(void *retval, void *mdata, va_list ap)
 		{ NULL, 0 }
 	};
 	ns_dtab compat_dtab[] = {
+		{ NSSRC_DB, db_setservent, mdata },
 #ifdef YP
 		{ NSSRC_NIS, nis_setservent, mdata },
 #endif
@@ -924,6 +1119,7 @@ getservbyname_r(const char *name, const char *proto, struct servent *serv,
 #endif /* NS_CACHING */
 	static const ns_dtab dtab[] = {
 		{ NSSRC_FILES, files_servent, (void *)&mdata },
+		{ NSSRC_DB, db_servent, (void *)nss_lt_name },
 #ifdef YP
 		{ NSSRC_NIS, nis_servent, (void *)nss_lt_name },
 #endif
@@ -960,6 +1156,7 @@ getservbyport_r(int port, const char *proto, struct servent *serv,
 #endif
 	static const ns_dtab dtab[] = {
 		{ NSSRC_FILES, files_servent, (void *)&mdata },
+		{ NSSRC_DB, db_servent, (void *)nss_lt_id },
 #ifdef YP
 		{ NSSRC_NIS, nis_servent, (void *)nss_lt_id },
 #endif
@@ -995,6 +1192,7 @@ getservent_r(struct servent *serv, char *buffer, size_t bufsize,
 #endif
 	static const ns_dtab dtab[] = {
 		{ NSSRC_FILES, files_servent, (void *)&mdata },
+		{ NSSRC_DB, db_servent, (void *)nss_lt_all },
 #ifdef YP
 		{ NSSRC_NIS, nis_servent, (void *)nss_lt_all },
 #endif
@@ -1027,6 +1225,7 @@ setservent(int stayopen)
 #endif
 	static const ns_dtab dtab[] = {
 		{ NSSRC_FILES, files_setservent, (void *)SETSERVENT },
+		{ NSSRC_DB, db_setservent, (void *)SETSERVENT },
 #ifdef YP
 		{ NSSRC_NIS, nis_setservent, (void *)SETSERVENT },
 #endif
@@ -1051,6 +1250,7 @@ endservent()
 #endif
 	static const ns_dtab dtab[] = {
 		{ NSSRC_FILES, files_setservent, (void *)ENDSERVENT },
+		{ NSSRC_DB, db_setservent, (void *)ENDSERVENT },
 #ifdef YP
 		{ NSSRC_NIS, nis_setservent, (void *)ENDSERVENT },
 #endif
diff --git a/lib/libc/net/nsdispatch.3 b/lib/libc/net/nsdispatch.3
index cf15002..8ae7540 100644
--- a/lib/libc/net/nsdispatch.3
+++ b/lib/libc/net/nsdispatch.3
@@ -32,7 +32,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd January 22, 2007
+.Dd April 4, 2010
 .Dt NSDISPATCH 3
 .Os
 .Sh NAME
@@ -176,6 +176,7 @@ While there is support for arbitrary sources, the following
 .Bl -column NSSRC_COMPAT compat -offset indent
 .It Sy "#define	value"
 .It Dv NSSRC_FILES Ta """files""
+.It Dv NSSRC_DB Ta """db""
 .It Dv NSSRC_DNS Ta """dns""
 .It Dv NSSRC_NIS Ta """nis""
 .It Dv NSSRC_COMPAT Ta """compat""
-- 
cgit v1.1


From 32d0c03b86b06a7797753e8d40aaea46fae6ce29 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Tue, 20 Apr 2010 22:57:05 +0000
Subject: MFC assembler version of match functions for amd64 and i386(*).

This gives approximately 15% improvement on compression case.

(*) i386 assembler version is enabled ONLY when MACHINE_CPU have
'i686' which is not default on FreeBSD/i386.  One can specify
for instance CPUTYPE=pentium4 in /etc/make.conf to get this
feature.
---
 lib/libz/Makefile                      |  12 +
 lib/libz/contrib/README.contrib        |  77 +++++
 lib/libz/contrib/asm686/README.686     |  51 +++
 lib/libz/contrib/asm686/match.S        | 343 ++++++++++++++++++++
 lib/libz/contrib/gcc_gvmat64/gvmat64.S | 574 +++++++++++++++++++++++++++++++++
 5 files changed, 1057 insertions(+)
 create mode 100644 lib/libz/contrib/README.contrib
 create mode 100644 lib/libz/contrib/asm686/README.686
 create mode 100644 lib/libz/contrib/asm686/match.S
 create mode 100644 lib/libz/contrib/gcc_gvmat64/gvmat64.S

(limited to 'lib')

diff --git a/lib/libz/Makefile b/lib/libz/Makefile
index 63383b4..1390d08 100644
--- a/lib/libz/Makefile
+++ b/lib/libz/Makefile
@@ -19,6 +19,18 @@ SRCS = adler32.c compress.c crc32.c gzio.c uncompr.c deflate.c trees.c \
        zutil.c inflate.c inftrees.c inffast.c zopen.c infback.c
 INCS=		zconf.h zlib.h
 
+.if ${MACHINE_ARCH} == "i386" && ${MACHINE_CPU:M*i686*}
+.PATH:		${.CURDIR}/contrib/asm686
+SRCS+=		match.S
+CFLAGS+=	-DASMV -DNO_UNDERLINE
+.endif
+
+.if ${MACHINE_ARCH} == "amd64"
+.PATH:		${.CURDIR}/contrib/gcc_gvmat64
+SRCS+=		gvmat64.S
+CFLAGS+=	-DASMV -DNO_UNDERLINE
+.endif
+
 minigzip:	all minigzip.o
 	$(CC) -o minigzip minigzip.o -L. -lz
 
diff --git a/lib/libz/contrib/README.contrib b/lib/libz/contrib/README.contrib
new file mode 100644
index 0000000..dd2285d
--- /dev/null
+++ b/lib/libz/contrib/README.contrib
@@ -0,0 +1,77 @@
+All files under this contrib directory are UNSUPPORTED. There were
+provided by users of zlib and were not tested by the authors of zlib.
+Use at your own risk. Please contact the authors of the contributions
+for help about these, not the zlib authors. Thanks.
+
+
+ada/        by Dmitriy Anisimkov <anisimkov@yahoo.com>
+        Support for Ada
+        See http://zlib-ada.sourceforge.net/
+
+amd64/      by Mikhail Teterin <mi@ALDAN.algebra.com>
+        asm code for AMD64
+        See patch at http://www.freebsd.org/cgi/query-pr.cgi?pr=bin/96393
+
+asm686/     by Brian Raiter <breadbox@muppetlabs.com>
+        asm code for Pentium and PPro/PII, using the AT&T (GNU as) syntax
+        See http://www.muppetlabs.com/~breadbox/software/assembly.html
+
+blast/      by Mark Adler <madler@alumni.caltech.edu>
+        Decompressor for output of PKWare Data Compression Library (DCL)
+
+delphi/     by Cosmin Truta <cosmint@cs.ubbcluj.ro>
+        Support for Delphi and C++ Builder
+
+dotzlib/    by Henrik Ravn <henrik@ravn.com>
+        Support for Microsoft .Net and Visual C++ .Net
+
+gcc_gvmat64/by Gilles Vollant <info@winimage.com>
+        GCC Version of x86 64-bit (AMD64 and Intel EM64t) code for x64
+        assembler to replace longest_match() and inflate_fast()
+
+infback9/   by Mark Adler <madler@alumni.caltech.edu>
+        Unsupported diffs to infback to decode the deflate64 format
+
+inflate86/  by Chris Anderson <christop@charm.net>
+        Tuned x86 gcc asm code to replace inflate_fast()
+
+iostream/   by Kevin Ruland <kevin@rodin.wustl.edu>
+        A C++ I/O streams interface to the zlib gz* functions
+
+iostream2/  by Tyge Løvset <Tyge.Lovset@cmr.no>
+        Another C++ I/O streams interface
+
+iostream3/  by Ludwig Schwardt <schwardt@sun.ac.za>
+            and Kevin Ruland <kevin@rodin.wustl.edu>
+        Yet another C++ I/O streams interface
+
+masmx64/    by Gilles Vollant <info@winimage.com>
+        x86 64-bit (AMD64 and Intel EM64t) code for x64 assembler to
+        replace longest_match() and inflate_fast(),  also masm x86
+        64-bits translation of Chris Anderson inflate_fast()
+
+masmx86/    by Gilles Vollant <info@winimage.com>
+        x86 asm code to replace longest_match() and inflate_fast(),
+        for Visual C++ and MASM (32 bits).
+        Based on Brian Raiter (asm686) and Chris Anderson (inflate86)
+
+minizip/    by Gilles Vollant <info@winimage.com>
+        Mini zip and unzip based on zlib
+        Includes Zip64 support by Mathias Svensson <mathias@result42.com>
+        See http://www.winimage.com/zLibDll/unzip.html
+
+pascal/     by Bob Dellaca <bobdl@xtra.co.nz> et al.
+        Support for Pascal
+
+puff/       by Mark Adler <madler@alumni.caltech.edu>
+        Small, low memory usage inflate.  Also serves to provide an
+        unambiguous description of the deflate format.
+
+testzlib/   by Gilles Vollant <info@winimage.com>
+        Example of the use of zlib
+
+untgz/      by Pedro A. Aranda Gutierrez <paag@tid.es>
+        A very simple tar.gz file extractor using zlib
+
+vstudio/    by Gilles Vollant <info@winimage.com>
+        Building a minizip-enhanced zlib with Microsoft Visual Studio
diff --git a/lib/libz/contrib/asm686/README.686 b/lib/libz/contrib/asm686/README.686
new file mode 100644
index 0000000..a0bf3be
--- /dev/null
+++ b/lib/libz/contrib/asm686/README.686
@@ -0,0 +1,51 @@
+This is a patched version of zlib, modified to use
+Pentium-Pro-optimized assembly code in the deflation algorithm. The
+files changed/added by this patch are:
+
+README.686
+match.S
+
+The speedup that this patch provides varies, depending on whether the
+compiler used to build the original version of zlib falls afoul of the
+PPro's speed traps. My own tests show a speedup of around 10-20% at
+the default compression level, and 20-30% using -9, against a version
+compiled using gcc 2.7.2.3. Your mileage may vary.
+
+Note that this code has been tailored for the PPro/PII in particular,
+and will not perform particuarly well on a Pentium.
+
+If you are using an assembler other than GNU as, you will have to
+translate match.S to use your assembler's syntax. (Have fun.)
+
+Brian Raiter
+breadbox@muppetlabs.com
+April, 1998
+
+
+Added for zlib 1.1.3:
+
+The patches come from
+http://www.muppetlabs.com/~breadbox/software/assembly.html
+
+To compile zlib with this asm file, copy match.S to the zlib directory
+then do:
+
+CFLAGS="-O3 -DASMV" ./configure
+make OBJA=match.o
+
+
+Update:
+
+I've been ignoring these assembly routines for years, believing that
+gcc's generated code had caught up with it sometime around gcc 2.95
+and the major rearchitecting of the Pentium 4. However, I recently
+learned that, despite what I believed, this code still has some life
+in it. On the Pentium 4 and AMD64 chips, it continues to run about 8%
+faster than the code produced by gcc 4.1.
+
+In acknowledgement of its continuing usefulness, I've altered the
+license to match that of the rest of zlib. Share and Enjoy!
+
+Brian Raiter
+breadbox@muppetlabs.com
+April, 2007
diff --git a/lib/libz/contrib/asm686/match.S b/lib/libz/contrib/asm686/match.S
new file mode 100644
index 0000000..06817e1
--- /dev/null
+++ b/lib/libz/contrib/asm686/match.S
@@ -0,0 +1,343 @@
+/* match.S -- x86 assembly version of the zlib longest_match() function.
+ * Optimized for the Intel 686 chips (PPro and later).
+ *
+ * Copyright (C) 1998, 2007 Brian Raiter <breadbox@muppetlabs.com>
+ *
+ * This software is provided 'as-is', without any express or implied
+ * warranty.  In no event will the author be held liable for any damages
+ * arising from the use of this software.
+ *
+ * Permission is granted to anyone to use this software for any purpose,
+ * including commercial applications, and to alter it and redistribute it
+ * freely, subject to the following restrictions:
+ *
+ * 1. The origin of this software must not be misrepresented; you must not
+ *    claim that you wrote the original software. If you use this software
+ *    in a product, an acknowledgment in the product documentation would be
+ *    appreciated but is not required.
+ * 2. Altered source versions must be plainly marked as such, and must not be
+ *    misrepresented as being the original software.
+ * 3. This notice may not be removed or altered from any source distribution.
+ */
+
+#ifndef NO_UNDERLINE
+#define	match_init	_match_init
+#define	longest_match	_longest_match
+#endif
+
+#define	MAX_MATCH	(258)
+#define	MIN_MATCH	(3)
+#define	MIN_LOOKAHEAD	(MAX_MATCH + MIN_MATCH + 1)
+#define	MAX_MATCH_8	((MAX_MATCH + 7) & ~7)
+
+/* stack frame offsets */
+
+#define	chainlenwmask		0	/* high word: current chain len	*/
+					/* low word: s->wmask		*/
+#define	window			4	/* local copy of s->window	*/
+#define	windowbestlen		8	/* s->window + bestlen		*/
+#define	scanstart		16	/* first two bytes of string	*/
+#define	scanend			12	/* last two bytes of string	*/
+#define	scanalign		20	/* dword-misalignment of string	*/
+#define	nicematch		24	/* a good enough match size	*/
+#define	bestlen			28	/* size of best match so far	*/
+#define	scan			32	/* ptr to string wanting match	*/
+
+#define	LocalVarsSize		(36)
+/*	saved ebx		36 */
+/*	saved edi		40 */
+/*	saved esi		44 */
+/*	saved ebp		48 */
+/*	return address		52 */
+#define	deflatestate		56	/* the function arguments	*/
+#define	curmatch		60
+
+/* All the +zlib1222add offsets are due to the addition of fields
+ *  in zlib in the deflate_state structure since the asm code was first written
+ * (if you compile with zlib 1.0.4 or older, use "zlib1222add equ (-4)").
+ * (if you compile with zlib between 1.0.5 and 1.2.2.1, use "zlib1222add equ 0").
+ * if you compile with zlib 1.2.2.2 or later , use "zlib1222add equ 8").
+ */
+
+#define zlib1222add		(8)
+
+#define	dsWSize			(36+zlib1222add)
+#define	dsWMask			(44+zlib1222add)
+#define	dsWindow		(48+zlib1222add)
+#define	dsPrev			(56+zlib1222add)
+#define	dsMatchLen		(88+zlib1222add)
+#define	dsPrevMatch		(92+zlib1222add)
+#define	dsStrStart		(100+zlib1222add)
+#define	dsMatchStart		(104+zlib1222add)
+#define	dsLookahead		(108+zlib1222add)
+#define	dsPrevLen		(112+zlib1222add)
+#define	dsMaxChainLen		(116+zlib1222add)
+#define	dsGoodMatch		(132+zlib1222add)
+#define	dsNiceMatch		(136+zlib1222add)
+
+
+.file "match.S"
+
+.globl	match_init, longest_match
+
+.text
+
+/* uInt longest_match(deflate_state *deflatestate, IPos curmatch) */
+
+longest_match:
+
+/* Save registers that the compiler may be using, and adjust %esp to	*/
+/* make room for our stack frame.					*/
+
+		pushl	%ebp
+		pushl	%edi
+		pushl	%esi
+		pushl	%ebx
+		subl	$LocalVarsSize, %esp
+
+/* Retrieve the function arguments. %ecx will hold cur_match		*/
+/* throughout the entire function. %edx will hold the pointer to the	*/
+/* deflate_state structure during the function's setup (before		*/
+/* entering the main loop).						*/
+
+		movl	deflatestate(%esp), %edx
+		movl	curmatch(%esp), %ecx
+
+/* uInt wmask = s->w_mask;						*/
+/* unsigned chain_length = s->max_chain_length;				*/
+/* if (s->prev_length >= s->good_match) {				*/
+/*     chain_length >>= 2;						*/
+/* }									*/
+
+		movl	dsPrevLen(%edx), %eax
+		movl	dsGoodMatch(%edx), %ebx
+		cmpl	%ebx, %eax
+		movl	dsWMask(%edx), %eax
+		movl	dsMaxChainLen(%edx), %ebx
+		jl	LastMatchGood
+		shrl	$2, %ebx
+LastMatchGood:
+
+/* chainlen is decremented once beforehand so that the function can	*/
+/* use the sign flag instead of the zero flag for the exit test.	*/
+/* It is then shifted into the high word, to make room for the wmask	*/
+/* value, which it will always accompany.				*/
+
+		decl	%ebx
+		shll	$16, %ebx
+		orl	%eax, %ebx
+		movl	%ebx, chainlenwmask(%esp)
+
+/* if ((uInt)nice_match > s->lookahead) nice_match = s->lookahead;	*/
+
+		movl	dsNiceMatch(%edx), %eax
+		movl	dsLookahead(%edx), %ebx
+		cmpl	%eax, %ebx
+		jl	LookaheadLess
+		movl	%eax, %ebx
+LookaheadLess:	movl	%ebx, nicematch(%esp)
+
+/* register Bytef *scan = s->window + s->strstart;			*/
+
+		movl	dsWindow(%edx), %esi
+		movl	%esi, window(%esp)
+		movl	dsStrStart(%edx), %ebp
+		lea	(%esi,%ebp), %edi
+		movl	%edi, scan(%esp)
+
+/* Determine how many bytes the scan ptr is off from being		*/
+/* dword-aligned.							*/
+
+		movl	%edi, %eax
+		negl	%eax
+		andl	$3, %eax
+		movl	%eax, scanalign(%esp)
+
+/* IPos limit = s->strstart > (IPos)MAX_DIST(s) ?			*/
+/*     s->strstart - (IPos)MAX_DIST(s) : NIL;				*/
+
+		movl	dsWSize(%edx), %eax
+		subl	$MIN_LOOKAHEAD, %eax
+		subl	%eax, %ebp
+		jg	LimitPositive
+		xorl	%ebp, %ebp
+LimitPositive:
+
+/* int best_len = s->prev_length;					*/
+
+		movl	dsPrevLen(%edx), %eax
+		movl	%eax, bestlen(%esp)
+
+/* Store the sum of s->window + best_len in %esi locally, and in %esi.	*/
+
+		addl	%eax, %esi
+		movl	%esi, windowbestlen(%esp)
+
+/* register ush scan_start = *(ushf*)scan;				*/
+/* register ush scan_end   = *(ushf*)(scan+best_len-1);			*/
+/* Posf *prev = s->prev;						*/
+
+		movzwl	(%edi), %ebx
+		movl	%ebx, scanstart(%esp)
+		movzwl	-1(%edi,%eax), %ebx
+		movl	%ebx, scanend(%esp)
+		movl	dsPrev(%edx), %edi
+
+/* Jump into the main loop.						*/
+
+		movl	chainlenwmask(%esp), %edx
+		jmp	LoopEntry
+
+.balign 16
+
+/* do {
+ *     match = s->window + cur_match;
+ *     if (*(ushf*)(match+best_len-1) != scan_end ||
+ *         *(ushf*)match != scan_start) continue;
+ *     [...]
+ * } while ((cur_match = prev[cur_match & wmask]) > limit
+ *          && --chain_length != 0);
+ *
+ * Here is the inner loop of the function. The function will spend the
+ * majority of its time in this loop, and majority of that time will
+ * be spent in the first ten instructions.
+ *
+ * Within this loop:
+ * %ebx = scanend
+ * %ecx = curmatch
+ * %edx = chainlenwmask - i.e., ((chainlen << 16) | wmask)
+ * %esi = windowbestlen - i.e., (window + bestlen)
+ * %edi = prev
+ * %ebp = limit
+ */
+LookupLoop:
+		andl	%edx, %ecx
+		movzwl	(%edi,%ecx,2), %ecx
+		cmpl	%ebp, %ecx
+		jbe	LeaveNow
+		subl	$0x00010000, %edx
+		js	LeaveNow
+LoopEntry:	movzwl	-1(%esi,%ecx), %eax
+		cmpl	%ebx, %eax
+		jnz	LookupLoop
+		movl	window(%esp), %eax
+		movzwl	(%eax,%ecx), %eax
+		cmpl	scanstart(%esp), %eax
+		jnz	LookupLoop
+
+/* Store the current value of chainlen.					*/
+
+		movl	%edx, chainlenwmask(%esp)
+
+/* Point %edi to the string under scrutiny, and %esi to the string we	*/
+/* are hoping to match it up with. In actuality, %esi and %edi are	*/
+/* both pointed (MAX_MATCH_8 - scanalign) bytes ahead, and %edx is	*/
+/* initialized to -(MAX_MATCH_8 - scanalign).				*/
+
+		movl	window(%esp), %esi
+		movl	scan(%esp), %edi
+		addl	%ecx, %esi
+		movl	scanalign(%esp), %eax
+		movl	$(-MAX_MATCH_8), %edx
+		lea	MAX_MATCH_8(%edi,%eax), %edi
+		lea	MAX_MATCH_8(%esi,%eax), %esi
+
+/* Test the strings for equality, 8 bytes at a time. At the end,
+ * adjust %edx so that it is offset to the exact byte that mismatched.
+ *
+ * We already know at this point that the first three bytes of the
+ * strings match each other, and they can be safely passed over before
+ * starting the compare loop. So what this code does is skip over 0-3
+ * bytes, as much as necessary in order to dword-align the %edi
+ * pointer. (%esi will still be misaligned three times out of four.)
+ *
+ * It should be confessed that this loop usually does not represent
+ * much of the total running time. Replacing it with a more
+ * straightforward "rep cmpsb" would not drastically degrade
+ * performance.
+ */
+LoopCmps:
+		movl	(%esi,%edx), %eax
+		xorl	(%edi,%edx), %eax
+		jnz	LeaveLoopCmps
+		movl	4(%esi,%edx), %eax
+		xorl	4(%edi,%edx), %eax
+		jnz	LeaveLoopCmps4
+		addl	$8, %edx
+		jnz	LoopCmps
+		jmp	LenMaximum
+LeaveLoopCmps4:	addl	$4, %edx
+LeaveLoopCmps:	testl	$0x0000FFFF, %eax
+		jnz	LenLower
+		addl	$2, %edx
+		shrl	$16, %eax
+LenLower:	subb	$1, %al
+		adcl	$0, %edx
+
+/* Calculate the length of the match. If it is longer than MAX_MATCH,	*/
+/* then automatically accept it as the best possible match and leave.	*/
+
+		lea	(%edi,%edx), %eax
+		movl	scan(%esp), %edi
+		subl	%edi, %eax
+		cmpl	$MAX_MATCH, %eax
+		jge	LenMaximum
+
+/* If the length of the match is not longer than the best match we	*/
+/* have so far, then forget it and return to the lookup loop.		*/
+
+		movl	deflatestate(%esp), %edx
+		movl	bestlen(%esp), %ebx
+		cmpl	%ebx, %eax
+		jg	LongerMatch
+		movl	windowbestlen(%esp), %esi
+		movl	dsPrev(%edx), %edi
+		movl	scanend(%esp), %ebx
+		movl	chainlenwmask(%esp), %edx
+		jmp	LookupLoop
+
+/*         s->match_start = cur_match;					*/
+/*         best_len = len;						*/
+/*         if (len >= nice_match) break;				*/
+/*         scan_end = *(ushf*)(scan+best_len-1);			*/
+
+LongerMatch:	movl	nicematch(%esp), %ebx
+		movl	%eax, bestlen(%esp)
+		movl	%ecx, dsMatchStart(%edx)
+		cmpl	%ebx, %eax
+		jge	LeaveNow
+		movl	window(%esp), %esi
+		addl	%eax, %esi
+		movl	%esi, windowbestlen(%esp)
+		movzwl	-1(%edi,%eax), %ebx
+		movl	dsPrev(%edx), %edi
+		movl	%ebx, scanend(%esp)
+		movl	chainlenwmask(%esp), %edx
+		jmp	LookupLoop
+
+/* Accept the current string, with the maximum possible length.		*/
+
+LenMaximum:	movl	deflatestate(%esp), %edx
+		movl	$MAX_MATCH, bestlen(%esp)
+		movl	%ecx, dsMatchStart(%edx)
+
+/* if ((uInt)best_len <= s->lookahead) return (uInt)best_len;		*/
+/* return s->lookahead;							*/
+
+LeaveNow:
+		movl	deflatestate(%esp), %edx
+		movl	bestlen(%esp), %ebx
+		movl	dsLookahead(%edx), %eax
+		cmpl	%eax, %ebx
+		jg	LookaheadRet
+		movl	%ebx, %eax
+LookaheadRet:
+
+/* Restore the stack and return from whence we came.			*/
+
+		addl	$LocalVarsSize, %esp
+		popl	%ebx
+		popl	%esi
+		popl	%edi
+		popl	%ebp
+match_init:	ret
diff --git a/lib/libz/contrib/gcc_gvmat64/gvmat64.S b/lib/libz/contrib/gcc_gvmat64/gvmat64.S
new file mode 100644
index 0000000..23309fa
--- /dev/null
+++ b/lib/libz/contrib/gcc_gvmat64/gvmat64.S
@@ -0,0 +1,574 @@
+/*
+;uInt longest_match_x64(
+;    deflate_state *s,
+;    IPos cur_match);                             // current match 
+
+; gvmat64.S -- Asm portion of the optimized longest_match for 32 bits x86_64
+;  (AMD64 on Athlon 64, Opteron, Phenom
+;     and Intel EM64T on Pentium 4 with EM64T, Pentium D, Core 2 Duo, Core I5/I7)
+; this file is translation from gvmat64.asm to GCC 4.x (for Linux, Mac XCode)
+; Copyright (C) 1995-2010 Jean-loup Gailly, Brian Raiter and Gilles Vollant.
+;
+; File written by Gilles Vollant, by converting to assembly the longest_match
+;  from Jean-loup Gailly in deflate.c of zLib and infoZip zip.
+;  and by taking inspiration on asm686 with masm, optimised assembly code
+;        from Brian Raiter, written 1998
+;
+;  This software is provided 'as-is', without any express or implied
+;  warranty.  In no event will the authors be held liable for any damages
+;  arising from the use of this software.
+;
+;  Permission is granted to anyone to use this software for any purpose,
+;  including commercial applications, and to alter it and redistribute it
+;  freely, subject to the following restrictions:
+;
+;  1. The origin of this software must not be misrepresented; you must not
+;     claim that you wrote the original software. If you use this software
+;     in a product, an acknowledgment in the product documentation would be
+;     appreciated but is not required.
+;  2. Altered source versions must be plainly marked as such, and must not be
+;     misrepresented as being the original software
+;  3. This notice may not be removed or altered from any source distribution.
+;
+;         http://www.zlib.net
+;         http://www.winimage.com/zLibDll
+;         http://www.muppetlabs.com/~breadbox/software/assembly.html
+;
+; to compile this file for zLib, I use option:
+;   gcc -c -arch x86_64 gvmat64.S
+
+
+;uInt longest_match(s, cur_match)
+;    deflate_state *s;
+;    IPos cur_match;                             // current match /
+;
+; with XCode for Mac, I had strange error with some jump on intel syntax
+; this is why BEFORE_JMP and AFTER_JMP are used
+ */
+
+
+#define BEFORE_JMP .att_syntax
+#define AFTER_JMP .intel_syntax noprefix
+
+#ifndef NO_UNDERLINE
+#	define	match_init	_match_init
+#	define	longest_match	_longest_match
+#endif
+
+.intel_syntax noprefix
+
+.globl	match_init, longest_match
+.text
+longest_match:
+
+
+
+#define LocalVarsSize 96
+/*
+; register used : rax,rbx,rcx,rdx,rsi,rdi,r8,r9,r10,r11,r12
+; free register :  r14,r15
+; register can be saved : rsp
+*/
+
+#define chainlenwmask     (rsp + 8 - LocalVarsSize)
+#define nicematch         (rsp + 16 - LocalVarsSize)
+
+#define save_rdi        (rsp + 24 - LocalVarsSize)
+#define save_rsi        (rsp + 32 - LocalVarsSize)
+#define save_rbx        (rsp + 40 - LocalVarsSize)
+#define save_rbp        (rsp + 48 - LocalVarsSize)
+#define save_r12        (rsp + 56 - LocalVarsSize)
+#define save_r13        (rsp + 64 - LocalVarsSize)
+#define save_r14        (rsp + 72 - LocalVarsSize)
+#define save_r15        (rsp + 80 - LocalVarsSize)
+
+
+/*
+;  all the +4 offsets are due to the addition of pending_buf_size (in zlib
+;  in the deflate_state structure since the asm code was first written
+;  (if you compile with zlib 1.0.4 or older, remove the +4).
+;  Note : these value are good with a 8 bytes boundary pack structure
+*/
+
+#define    MAX_MATCH              258
+#define    MIN_MATCH              3
+#define    MIN_LOOKAHEAD          (MAX_MATCH+MIN_MATCH+1)
+
+/*
+;;; Offsets for fields in the deflate_state structure. These numbers
+;;; are calculated from the definition of deflate_state, with the
+;;; assumption that the compiler will dword-align the fields. (Thus,
+;;; changing the definition of deflate_state could easily cause this
+;;; program to crash horribly, without so much as a warning at
+;;; compile time. Sigh.)
+
+;  all the +zlib1222add offsets are due to the addition of fields
+;  in zlib in the deflate_state structure since the asm code was first written
+;  (if you compile with zlib 1.0.4 or older, use "zlib1222add equ (-4)").
+;  (if you compile with zlib between 1.0.5 and 1.2.2.1, use "zlib1222add equ 0").
+;  if you compile with zlib 1.2.2.2 or later , use "zlib1222add equ 8").
+*/
+
+
+
+/* you can check the structure offset by running
+
+#include <stdlib.h>
+#include <stdio.h>
+#include "deflate.h"
+
+void print_depl()
+{
+deflate_state ds;
+deflate_state *s=&ds;
+printf("size pointer=%u\n",(int)sizeof(void*));
+
+printf("#define dsWSize         %u\n",(int)(((char*)&(s->w_size))-((char*)s)));
+printf("#define dsWMask         %u\n",(int)(((char*)&(s->w_mask))-((char*)s)));
+printf("#define dsWindow        %u\n",(int)(((char*)&(s->window))-((char*)s)));
+printf("#define dsPrev          %u\n",(int)(((char*)&(s->prev))-((char*)s)));
+printf("#define dsMatchLen      %u\n",(int)(((char*)&(s->match_length))-((char*)s)));
+printf("#define dsPrevMatch     %u\n",(int)(((char*)&(s->prev_match))-((char*)s)));
+printf("#define dsStrStart      %u\n",(int)(((char*)&(s->strstart))-((char*)s)));
+printf("#define dsMatchStart    %u\n",(int)(((char*)&(s->match_start))-((char*)s)));
+printf("#define dsLookahead     %u\n",(int)(((char*)&(s->lookahead))-((char*)s)));
+printf("#define dsPrevLen       %u\n",(int)(((char*)&(s->prev_length))-((char*)s)));
+printf("#define dsMaxChainLen   %u\n",(int)(((char*)&(s->max_chain_length))-((char*)s)));
+printf("#define dsGoodMatch     %u\n",(int)(((char*)&(s->good_match))-((char*)s)));
+printf("#define dsNiceMatch     %u\n",(int)(((char*)&(s->nice_match))-((char*)s)));
+}
+*/
+
+#define dsWSize          68
+#define dsWMask          76
+#define dsWindow         80
+#define dsPrev           96
+#define dsMatchLen       144
+#define dsPrevMatch      148
+#define dsStrStart       156
+#define dsMatchStart     160
+#define dsLookahead      164
+#define dsPrevLen        168
+#define dsMaxChainLen    172
+#define dsGoodMatch      188
+#define dsNiceMatch      192
+
+#define window_size      [ rcx + dsWSize]
+#define WMask            [ rcx + dsWMask]
+#define window_ad        [ rcx + dsWindow]
+#define prev_ad          [ rcx + dsPrev]
+#define strstart         [ rcx + dsStrStart]
+#define match_start      [ rcx + dsMatchStart]
+#define Lookahead        [ rcx + dsLookahead] //; 0ffffffffh on infozip
+#define prev_length      [ rcx + dsPrevLen]
+#define max_chain_length [ rcx + dsMaxChainLen]
+#define good_match       [ rcx + dsGoodMatch]
+#define nice_match       [ rcx + dsNiceMatch]
+
+/*
+; windows:
+; parameter 1 in rcx(deflate state s), param 2 in rdx (cur match)
+
+; see http://weblogs.asp.net/oldnewthing/archive/2004/01/14/58579.aspx and
+; http://msdn.microsoft.com/library/en-us/kmarch/hh/kmarch/64bitAMD_8e951dd2-ee77-4728-8702-55ce4b5dd24a.xml.asp
+;
+; All registers must be preserved across the call, except for
+;   rax, rcx, rdx, r8, r9, r10, and r11, which are scratch.
+
+;
+; gcc on macosx-linux:
+; see http://www.x86-64.org/documentation/abi-0.99.pdf
+; param 1 in rdi, param 2 in rsi
+; rbx, rsp, rbp, r12 to r15 must be preserved
+
+;;; Save registers that the compiler may be using, and adjust esp to
+;;; make room for our stack frame.
+
+
+;;; Retrieve the function arguments. r8d will hold cur_match
+;;; throughout the entire function. edx will hold the pointer to the
+;;; deflate_state structure during the function's setup (before
+;;; entering the main loop.
+
+; ms: parameter 1 in rcx (deflate_state* s), param 2 in edx -> r8 (cur match)
+; mac: param 1 in rdi, param 2 rsi
+; this clear high 32 bits of r8, which can be garbage in both r8 and rdx
+*/
+        mov [save_rbx],rbx
+        mov [save_rbp],rbp
+
+
+        mov rcx,rdi
+
+        mov r8d,esi
+
+
+        mov [save_r12],r12
+        mov [save_r13],r13
+        mov [save_r14],r14
+        mov [save_r15],r15
+
+
+//;;; uInt wmask = s->w_mask;
+//;;; unsigned chain_length = s->max_chain_length;
+//;;; if (s->prev_length >= s->good_match) {
+//;;;     chain_length >>= 2;
+//;;; }
+
+
+        mov edi, prev_length
+        mov esi, good_match
+        mov eax, WMask
+        mov ebx, max_chain_length
+        cmp edi, esi
+        jl  LastMatchGood
+        shr ebx, 2
+LastMatchGood:
+
+//;;; chainlen is decremented once beforehand so that the function can
+//;;; use the sign flag instead of the zero flag for the exit test.
+//;;; It is then shifted into the high word, to make room for the wmask
+//;;; value, which it will always accompany.
+
+        dec ebx
+        shl ebx, 16
+        or  ebx, eax
+
+//;;; on zlib only
+//;;; if ((uInt)nice_match > s->lookahead) nice_match = s->lookahead;
+
+
+
+        mov eax, nice_match
+        mov [chainlenwmask], ebx
+        mov r10d, Lookahead
+        cmp r10d, eax
+        cmovnl r10d, eax
+        mov [nicematch],r10d
+
+
+
+//;;; register Bytef *scan = s->window + s->strstart;
+        mov r10, window_ad
+        mov ebp, strstart
+        lea r13, [r10 + rbp]
+
+//;;; Determine how many bytes the scan ptr is off from being
+//;;; dword-aligned.
+
+         mov r9,r13
+         neg r13
+         and r13,3
+
+//;;; IPos limit = s->strstart > (IPos)MAX_DIST(s) ?
+//;;;     s->strstart - (IPos)MAX_DIST(s) : NIL;
+
+
+        mov eax, window_size
+        sub eax, MIN_LOOKAHEAD
+
+
+        xor edi,edi
+        sub ebp, eax
+
+        mov r11d, prev_length
+
+        cmovng ebp,edi
+
+//;;; int best_len = s->prev_length;
+
+
+//;;; Store the sum of s->window + best_len in esi locally, and in esi.
+
+       lea  rsi,[r10+r11]
+
+//;;; register ush scan_start = *(ushf*)scan;
+//;;; register ush scan_end   = *(ushf*)(scan+best_len-1);
+//;;; Posf *prev = s->prev;
+
+        movzx r12d,word ptr [r9]
+        movzx ebx, word ptr [r9 + r11 - 1]
+
+        mov rdi, prev_ad
+
+//;;; Jump into the main loop.
+
+        mov edx, [chainlenwmask]
+
+        cmp bx,word ptr [rsi + r8 - 1]
+        jz  LookupLoopIsZero
+				
+						
+						
+LookupLoop1:
+        and r8d, edx
+
+        movzx   r8d, word ptr [rdi + r8*2]
+        cmp r8d, ebp
+        jbe LeaveNow
+		
+		
+		
+        sub edx, 0x00010000
+		BEFORE_JMP
+        js  LeaveNow
+		AFTER_JMP
+
+LoopEntry1:
+        cmp bx,word ptr [rsi + r8 - 1]
+		BEFORE_JMP
+        jz  LookupLoopIsZero
+		AFTER_JMP
+
+LookupLoop2:
+        and r8d, edx
+
+        movzx   r8d, word ptr [rdi + r8*2]
+        cmp r8d, ebp
+		BEFORE_JMP
+        jbe LeaveNow
+		AFTER_JMP
+        sub edx, 0x00010000
+		BEFORE_JMP
+        js  LeaveNow
+		AFTER_JMP
+
+LoopEntry2:
+        cmp bx,word ptr [rsi + r8 - 1]
+		BEFORE_JMP
+        jz  LookupLoopIsZero
+		AFTER_JMP
+
+LookupLoop4:
+        and r8d, edx
+
+        movzx   r8d, word ptr [rdi + r8*2]
+        cmp r8d, ebp
+		BEFORE_JMP
+        jbe LeaveNow
+		AFTER_JMP
+        sub edx, 0x00010000
+		BEFORE_JMP
+        js  LeaveNow
+		AFTER_JMP
+
+LoopEntry4:
+
+        cmp bx,word ptr [rsi + r8 - 1]
+		BEFORE_JMP
+        jnz LookupLoop1
+        jmp LookupLoopIsZero
+		AFTER_JMP
+/*
+;;; do {
+;;;     match = s->window + cur_match;
+;;;     if (*(ushf*)(match+best_len-1) != scan_end ||
+;;;         *(ushf*)match != scan_start) continue;
+;;;     [...]
+;;; } while ((cur_match = prev[cur_match & wmask]) > limit
+;;;          && --chain_length != 0);
+;;;
+;;; Here is the inner loop of the function. The function will spend the
+;;; majority of its time in this loop, and majority of that time will
+;;; be spent in the first ten instructions.
+;;;
+;;; Within this loop:
+;;; ebx = scanend
+;;; r8d = curmatch
+;;; edx = chainlenwmask - i.e., ((chainlen << 16) | wmask)
+;;; esi = windowbestlen - i.e., (window + bestlen)
+;;; edi = prev
+;;; ebp = limit
+*/
+.balign 16
+LookupLoop:
+        and r8d, edx
+
+        movzx   r8d, word ptr [rdi + r8*2]
+        cmp r8d, ebp
+		BEFORE_JMP
+        jbe LeaveNow
+		AFTER_JMP
+        sub edx, 0x00010000
+		BEFORE_JMP
+        js  LeaveNow
+		AFTER_JMP
+
+LoopEntry:
+
+        cmp bx,word ptr [rsi + r8 - 1]
+		BEFORE_JMP
+        jnz LookupLoop1
+		AFTER_JMP
+LookupLoopIsZero:
+        cmp     r12w, word ptr [r10 + r8]
+		BEFORE_JMP
+        jnz LookupLoop1
+		AFTER_JMP
+
+
+//;;; Store the current value of chainlen.
+        mov [chainlenwmask], edx
+/*
+;;; Point edi to the string under scrutiny, and esi to the string we
+;;; are hoping to match it up with. In actuality, esi and edi are
+;;; both pointed (MAX_MATCH_8 - scanalign) bytes ahead, and edx is
+;;; initialized to -(MAX_MATCH_8 - scanalign).
+*/
+        lea rsi,[r8+r10]
+        mov rdx, 0xfffffffffffffef8 //; -(MAX_MATCH_8)
+        lea rsi, [rsi + r13 + 0x0108] //;MAX_MATCH_8]
+        lea rdi, [r9 + r13 + 0x0108] //;MAX_MATCH_8]
+
+        prefetcht1 [rsi+rdx]
+        prefetcht1 [rdi+rdx]
+
+/*
+;;; Test the strings for equality, 8 bytes at a time. At the end,
+;;; adjust rdx so that it is offset to the exact byte that mismatched.
+;;;
+;;; We already know at this point that the first three bytes of the
+;;; strings match each other, and they can be safely passed over before
+;;; starting the compare loop. So what this code does is skip over 0-3
+;;; bytes, as much as necessary in order to dword-align the edi
+;;; pointer. (rsi will still be misaligned three times out of four.)
+;;;
+;;; It should be confessed that this loop usually does not represent
+;;; much of the total running time. Replacing it with a more
+;;; straightforward "rep cmpsb" would not drastically degrade
+;;; performance.
+*/
+
+LoopCmps:
+        mov rax, [rsi + rdx]
+        xor rax, [rdi + rdx]
+        jnz LeaveLoopCmps
+
+        mov rax, [rsi + rdx + 8]
+        xor rax, [rdi + rdx + 8]
+        jnz LeaveLoopCmps8
+
+
+        mov rax, [rsi + rdx + 8+8]
+        xor rax, [rdi + rdx + 8+8]
+        jnz LeaveLoopCmps16
+
+        add rdx,8+8+8
+
+		BEFORE_JMP
+        jnz  LoopCmps
+        jmp  LenMaximum
+		AFTER_JMP
+		
+LeaveLoopCmps16: add rdx,8
+LeaveLoopCmps8: add rdx,8
+LeaveLoopCmps:
+
+        test    eax, 0x0000FFFF
+        jnz LenLower
+
+        test eax,0xffffffff
+
+        jnz LenLower32
+
+        add rdx,4
+        shr rax,32
+        or ax,ax
+		BEFORE_JMP
+        jnz LenLower
+		AFTER_JMP
+
+LenLower32:
+        shr eax,16
+        add rdx,2
+		
+LenLower:		
+        sub al, 1
+        adc rdx, 0
+//;;; Calculate the length of the match. If it is longer than MAX_MATCH,
+//;;; then automatically accept it as the best possible match and leave.
+
+        lea rax, [rdi + rdx]
+        sub rax, r9
+        cmp eax, MAX_MATCH
+		BEFORE_JMP
+        jge LenMaximum
+		AFTER_JMP
+/*
+;;; If the length of the match is not longer than the best match we
+;;; have so far, then forget it and return to the lookup loop.
+;///////////////////////////////////
+*/
+        cmp eax, r11d
+        jg  LongerMatch
+
+        lea rsi,[r10+r11]
+
+        mov rdi, prev_ad
+        mov edx, [chainlenwmask]
+		BEFORE_JMP
+        jmp LookupLoop
+		AFTER_JMP
+/*
+;;;         s->match_start = cur_match;
+;;;         best_len = len;
+;;;         if (len >= nice_match) break;
+;;;         scan_end = *(ushf*)(scan+best_len-1);
+*/
+LongerMatch:
+        mov r11d, eax
+        mov match_start, r8d
+        cmp eax, [nicematch]
+		BEFORE_JMP
+        jge LeaveNow
+		AFTER_JMP
+
+        lea rsi,[r10+rax]
+
+        movzx   ebx, word ptr [r9 + rax - 1]
+        mov rdi, prev_ad
+        mov edx, [chainlenwmask]
+		BEFORE_JMP
+        jmp LookupLoop
+		AFTER_JMP
+
+//;;; Accept the current string, with the maximum possible length.
+
+LenMaximum:
+        mov r11d,MAX_MATCH
+        mov match_start, r8d
+
+//;;; if ((uInt)best_len <= s->lookahead) return (uInt)best_len;
+//;;; return s->lookahead;
+
+LeaveNow:
+        mov eax, Lookahead
+        cmp r11d, eax
+        cmovng eax, r11d
+
+
+
+//;;; Restore the stack and return from whence we came.
+
+
+//        mov rsi,[save_rsi]
+//        mov rdi,[save_rdi]
+        mov rbx,[save_rbx]
+        mov rbp,[save_rbp]
+        mov r12,[save_r12]
+        mov r13,[save_r13]
+        mov r14,[save_r14]
+        mov r15,[save_r15]
+
+
+        ret 0
+//; please don't remove this string !
+//; Your can freely use gvmat64 in any free or commercial app
+//; but it is far better don't remove the string in the binary!
+ //   db     0dh,0ah,"asm686 with masm, optimised assembly code from Brian Raiter, written 1998, converted to amd 64 by Gilles Vollant 2005",0dh,0ah,0
+
+
+match_init:
+  ret 0
+
+
-- 
cgit v1.1


From 8b86799ca0b1a9b2f49e8d4d736adf1c101baac0 Mon Sep 17 00:00:00 2001
From: des <des@FreeBSD.org>
Date: Wed, 21 Apr 2010 06:33:10 +0000
Subject: MFH OpenSSH 5.4p1

---
 lib/libpam/modules/pam_ssh/Makefile  | 3 +++
 lib/libpam/modules/pam_ssh/pam_ssh.c | 3 +++
 2 files changed, 6 insertions(+)

(limited to 'lib')

diff --git a/lib/libpam/modules/pam_ssh/Makefile b/lib/libpam/modules/pam_ssh/Makefile
index f7dcd0c..b638d8b 100644
--- a/lib/libpam/modules/pam_ssh/Makefile
+++ b/lib/libpam/modules/pam_ssh/Makefile
@@ -7,6 +7,9 @@ LIB=	pam_ssh
 MAN=	pam_ssh.8
 SRCS=	pam_ssh.c
 
+# required when linking with a dynamic libssh
+SRCS+=	roaming_dummy.c
+
 WARNS?=	0
 CFLAGS+= -I${SSHDIR} -include ssh_namespace.h
 
diff --git a/lib/libpam/modules/pam_ssh/pam_ssh.c b/lib/libpam/modules/pam_ssh/pam_ssh.c
index 25c63ca..9d89045 100644
--- a/lib/libpam/modules/pam_ssh/pam_ssh.c
+++ b/lib/libpam/modules/pam_ssh/pam_ssh.c
@@ -61,6 +61,9 @@ __FBSDID("$FreeBSD$");
 #include "authfd.h"
 #include "authfile.h"
 
+#define ssh_add_identity(auth, key, comment) \
+	ssh_add_identity_constrained(auth, key, comment, 0, 0)
+
 extern char **environ;
 
 struct pam_ssh_key {
-- 
cgit v1.1


From d17c958e27ca3e5193ad3c1013bec5ef876c7b18 Mon Sep 17 00:00:00 2001
From: kib <kib@FreeBSD.org>
Date: Wed, 21 Apr 2010 11:09:13 +0000
Subject: MFC r206549: Align the declaration for sa_sigaction with POSIX.

MFC r206649:
Still reference struct __sigaction with clarification when this form
of argument declaration is needed.

MFC r206802:
Revert r206649. Simplify the presented declaration of struct sigaction.
---
 lib/libc/sys/sigaction.2 | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/sys/sigaction.2 b/lib/libc/sys/sigaction.2
index 4c4f18f..3b4f2f0 100644
--- a/lib/libc/sys/sigaction.2
+++ b/lib/libc/sys/sigaction.2
@@ -28,7 +28,7 @@
 .\"	From: @(#)sigaction.2	8.2 (Berkeley) 4/3/94
 .\" $FreeBSD$
 .\"
-.Dd June 7, 2004
+.Dd April 18, 2010
 .Dt SIGACTION 2
 .Os
 .Sh NAME
@@ -40,16 +40,11 @@
 .In signal.h
 .Bd -literal
 struct  sigaction {
-        union {
-                void    (*__sa_handler)(int);
-                void    (*__sa_sigaction)(int, struct __siginfo *, void *);
-        } __sigaction_u;                /* signal handler */
+        void    (*sa_handler)(int);
+        void    (*sa_sigaction)(int, siginfo_t *, void *);
         int     sa_flags;               /* see signal options below */
         sigset_t sa_mask;               /* signal mask to apply */
 };
-
-#define	sa_handler	__sigaction_u.__sa_handler
-#define	sa_sigaction	__sigaction_u.__sa_sigaction
 .Ed
 .Ft int
 .Fo sigaction
@@ -148,6 +143,16 @@ If
 is non-zero, the previous handling information for the signal
 is returned to the user.
 .Pp
+The above declaration of
+.Vt "struct sigaction"
+is not literal.
+It is provided only to list the accessible members.
+See
+.In sys/signal.h
+for the actual definition.
+In particular, the storage occupied by sa_handler and sa_sigaction overlaps,
+and an application can not use both simultaneously.
+.Pp
 Once a signal handler is installed, it normally remains installed
 until another
 .Fn sigaction
-- 
cgit v1.1


From 62840af138e26bcfd4e5b81e04a67f22560448d2 Mon Sep 17 00:00:00 2001
From: jilles <jilles@FreeBSD.org>
Date: Fri, 23 Apr 2010 18:01:19 +0000
Subject: MFC r206711: fnmatch: Fix bad FNM_PERIOD disabling if an asterisk has
 been seen.

Example: fnmatch("a*b/*", "abbb/.x", FNM_PATHNAME | FNM_PERIOD)

PR:		116074
---
 lib/libc/gen/fnmatch.c | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/gen/fnmatch.c b/lib/libc/gen/fnmatch.c
index e697bdc..3acbc3b 100644
--- a/lib/libc/gen/fnmatch.c
+++ b/lib/libc/gen/fnmatch.c
@@ -67,7 +67,8 @@ __FBSDID("$FreeBSD$");
 #define RANGE_ERROR     (-1)
 
 static int rangematch(const char *, wchar_t, int, char **, mbstate_t *);
-static int fnmatch1(const char *, const char *, int, mbstate_t, mbstate_t);
+static int fnmatch1(const char *, const char *, const char *, int, mbstate_t,
+		mbstate_t);
 
 int
 fnmatch(pattern, string, flags)
@@ -76,22 +77,21 @@ fnmatch(pattern, string, flags)
 {
 	static const mbstate_t initial;
 
-	return (fnmatch1(pattern, string, flags, initial, initial));
+	return (fnmatch1(pattern, string, string, flags, initial, initial));
 }
 
 static int
-fnmatch1(pattern, string, flags, patmbs, strmbs)
-	const char *pattern, *string;
+fnmatch1(pattern, string, stringstart, flags, patmbs, strmbs)
+	const char *pattern, *string, *stringstart;
 	int flags;
 	mbstate_t patmbs, strmbs;
 {
-	const char *stringstart;
 	char *newp;
 	char c;
 	wchar_t pc, sc;
 	size_t pclen, sclen;
 
-	for (stringstart = string;;) {
+	for (;;) {
 		pclen = mbrtowc(&pc, pattern, MB_LEN_MAX, &patmbs);
 		if (pclen == (size_t)-1 || pclen == (size_t)-2)
 			return (FNM_NOMATCH);
@@ -145,8 +145,8 @@ fnmatch1(pattern, string, flags, patmbs, strmbs)
 
 			/* General case, use recursion. */
 			while (sc != EOS) {
-				if (!fnmatch1(pattern, string,
-				    flags & ~FNM_PERIOD, patmbs, strmbs))
+				if (!fnmatch1(pattern, string, stringstart,
+				    flags, patmbs, strmbs))
 					return (0);
 				sclen = mbrtowc(&sc, string, MB_LEN_MAX,
 				    &strmbs);
-- 
cgit v1.1


From 49c4a03fe87df8c3c772e1ab83bfeafc541041b9 Mon Sep 17 00:00:00 2001
From: jilles <jilles@FreeBSD.org>
Date: Sun, 2 May 2010 12:38:59 +0000
Subject: MFC r207187: kvm(3): Mention that some of the functions use sysctl(3)
 instead of kmem.

Additionally, because of sysctl(3) use (which is generally good), behaviour
for crash dumps differs slightly from behaviour for live kernels and this
will probably never be fixed entirely, so weaken that claim.
---
 lib/libkvm/kvm.3 | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/libkvm/kvm.3 b/lib/libkvm/kvm.3
index f694fd8..9dcd772 100644
--- a/lib/libkvm/kvm.3
+++ b/lib/libkvm/kvm.3
@@ -32,7 +32,7 @@
 .\"     @(#)kvm.3	8.1 (Berkeley) 6/4/93
 .\" $FreeBSD$
 .\"
-.Dd January 29, 2004
+.Dd April 25, 2010
 .Dt KVM 3
 .Os
 .Sh NAME
@@ -46,12 +46,15 @@ The
 library provides a uniform interface for accessing kernel virtual memory
 images, including live systems and crash dumps.
 Access to live systems is via
+.Xr sysctl 3
+for some functions, and
 .Xr mem 4
 and
 .Xr kmem 4
+for other functions,
 while crash dumps can be examined via the core file generated by
 .Xr savecore 8 .
-The interface behaves identically in both cases.
+The interface behaves similarly in both cases.
 Memory can be read and written, kernel symbol addresses can be
 looked up efficiently, and information about user processes can
 be gathered.
@@ -112,5 +115,6 @@ given descriptor.
 .Xr kvm_openfiles 3 ,
 .Xr kvm_read 3 ,
 .Xr kvm_write 3 ,
+.Xr sysctl 3 ,
 .Xr kmem 4 ,
 .Xr mem 4
-- 
cgit v1.1


From 3b1de8da0b13e8f66e54d5f7aecaf1b9322dd000 Mon Sep 17 00:00:00 2001
From: marius <marius@FreeBSD.org>
Date: Sun, 2 May 2010 16:52:23 +0000
Subject: MFC: r206490, r206492

While SPARC V9 allows tininess to be detected either before or after
rounding (impl. dep. #55), the SPARC JPS1 responsible for SPARC64 and
UltraSPARC processors defines that in all cases tininess is detected
before rounding therefore rounding up to the smallest normalized number
should set the underflow flag. This change is needed for using SoftFloat
on sparc64 for reference purposes.

PR:		144900
Submitted by:	Peter Jeremy
---
 lib/libc/softfloat/softfloat-specialize | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'lib')

diff --git a/lib/libc/softfloat/softfloat-specialize b/lib/libc/softfloat/softfloat-specialize
index c8c8028..e8585ce 100644
--- a/lib/libc/softfloat/softfloat-specialize
+++ b/lib/libc/softfloat/softfloat-specialize
@@ -44,7 +44,11 @@ Underflow tininess-detection mode, statically initialized to default value.
 #ifdef SOFTFLOAT_FOR_GCC
 static
 #endif
+#ifdef __sparc64__
+int8 float_detect_tininess = float_tininess_before_rounding;
+#else
 int8 float_detect_tininess = float_tininess_after_rounding;
+#endif
 
 /*
 -------------------------------------------------------------------------------
-- 
cgit v1.1


From 45003aa79a1fb655b26b0f19031c16082a25c4bc Mon Sep 17 00:00:00 2001
From: kib <kib@FreeBSD.org>
Date: Tue, 4 May 2010 05:34:18 +0000
Subject: MFC r206893: Slightly modernize realpath(3).

SUSv4 requires that implementation returns EINVAL if supplied path is NULL,
and ENOENT if path is empty string [1].
Bring prototype in conformance with SUSv4, adding restrict keywords.
Allow the resolved path buffer pointer be NULL, in which case realpath(3)
allocates storage with malloc().

MFC r206898:
Free() is not allowed to modify errno, remove safety brackets around it.
Add small optimization, do not copy a string to the buffer that is
to be freed immediately after.

MFC r206997:
Move realpath(3) prototype to a POSIX section.

MFC r206998:
Add standards section, improve wording, taking into account the handling
of NULL and changed type in declaration.
---
 lib/libc/stdlib/realpath.3 | 25 +++++++++++++++++++-----
 lib/libc/stdlib/realpath.c | 48 ++++++++++++++++++++++++++++++++++++++++------
 2 files changed, 62 insertions(+), 11 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/stdlib/realpath.3 b/lib/libc/stdlib/realpath.3
index 4205a3d..aee65f7 100644
--- a/lib/libc/stdlib/realpath.3
+++ b/lib/libc/stdlib/realpath.3
@@ -31,7 +31,7 @@
 .\"     @(#)realpath.3	8.2 (Berkeley) 2/16/94
 .\" $FreeBSD$
 .\"
-.Dd February 16, 1994
+.Dd April 19, 2010
 .Dt REALPATH 3
 .Os
 .Sh NAME
@@ -43,7 +43,7 @@
 .In sys/param.h
 .In stdlib.h
 .Ft "char *"
-.Fn realpath "const char *pathname" "char resolved_path[PATH_MAX]"
+.Fn realpath "const char *pathname" "char *resolved_path"
 .Sh DESCRIPTION
 The
 .Fn realpath
@@ -64,7 +64,8 @@ argument
 .Em must
 refer to a buffer capable of storing at least
 .Dv PATH_MAX
-characters.
+characters, or be
+.Dv NULL .
 .Pp
 The
 .Fn realpath
@@ -82,13 +83,22 @@ The
 function returns
 .Fa resolved_path
 on success.
+If the function was supplied
+.Dv NULL
+as
+.Fa resolved_path ,
+and operation did not cause errors, the returned value is
+a null-terminated string in a buffer allocated by a call to
+.Fn malloc 3 .
 If an error occurs,
 .Fn realpath
 returns
 .Dv NULL ,
-and
+and if
 .Fa resolved_path
-contains the pathname which caused the problem.
+is not
+.Dv NULL ,
+the array that it points to contains the pathname which caused the problem.
 .Sh ERRORS
 The function
 .Fn realpath
@@ -113,6 +123,11 @@ when given a relative
 .Fa pathname .
 .Sh "SEE ALSO"
 .Xr getcwd 3
+.Sh STANDARDS
+The
+.Fn realpath
+function conforms to
+.St -p1003.1-2001 .
 .Sh HISTORY
 The
 .Fn realpath
diff --git a/lib/libc/stdlib/realpath.c b/lib/libc/stdlib/realpath.c
index 3082f5f..e75ee4a 100644
--- a/lib/libc/stdlib/realpath.c
+++ b/lib/libc/stdlib/realpath.c
@@ -43,23 +43,37 @@ __FBSDID("$FreeBSD$");
 #include "un-namespace.h"
 
 /*
- * char *realpath(const char *path, char resolved[PATH_MAX]);
- *
  * Find the real name of path, by removing all ".", ".." and symlink
  * components.  Returns (resolved) on success, or (NULL) on failure,
  * in which case the path which caused trouble is left in (resolved).
  */
 char *
-realpath(const char *path, char resolved[PATH_MAX])
+realpath(const char * __restrict path, char * __restrict resolved)
 {
 	struct stat sb;
 	char *p, *q, *s;
 	size_t left_len, resolved_len;
 	unsigned symlinks;
-	int serrno, slen;
+	int serrno, slen, m;
 	char left[PATH_MAX], next_token[PATH_MAX], symlink[PATH_MAX];
 
+	if (path == NULL) {
+		errno = EINVAL;
+		return (NULL);
+	}
+	if (path[0] == '\0') {
+		errno = ENOENT;
+		return (NULL);
+	}
 	serrno = errno;
+	if (resolved == NULL) {
+		resolved = malloc(PATH_MAX);
+		if (resolved == NULL)
+			return (NULL);
+		m = 1;
+	} else
+		m = 0;
+
 	symlinks = 0;
 	if (path[0] == '/') {
 		resolved[0] = '/';
@@ -70,13 +84,18 @@ realpath(const char *path, char resolved[PATH_MAX])
 		left_len = strlcpy(left, path + 1, sizeof(left));
 	} else {
 		if (getcwd(resolved, PATH_MAX) == NULL) {
-			strlcpy(resolved, ".", PATH_MAX);
+			if (m)
+				free(resolved);
+			else
+				strlcpy(resolved, ".", PATH_MAX);
 			return (NULL);
 		}
 		resolved_len = strlen(resolved);
 		left_len = strlcpy(left, path, sizeof(left));
 	}
 	if (left_len >= sizeof(left) || resolved_len >= PATH_MAX) {
+		if (m)
+			free(resolved);
 		errno = ENAMETOOLONG;
 		return (NULL);
 	}
@@ -92,6 +111,8 @@ realpath(const char *path, char resolved[PATH_MAX])
 		p = strchr(left, '/');
 		s = p ? p : left + left_len;
 		if (s - left >= sizeof(next_token)) {
+			if (m)
+				free(resolved);
 			errno = ENAMETOOLONG;
 			return (NULL);
 		}
@@ -102,6 +123,8 @@ realpath(const char *path, char resolved[PATH_MAX])
 			memmove(left, s + 1, left_len + 1);
 		if (resolved[resolved_len - 1] != '/') {
 			if (resolved_len + 1 >= PATH_MAX) {
+				if (m)
+					free(resolved);
 				errno = ENAMETOOLONG;
 				return (NULL);
 			}
@@ -133,6 +156,8 @@ realpath(const char *path, char resolved[PATH_MAX])
 		 */
 		resolved_len = strlcat(resolved, next_token, PATH_MAX);
 		if (resolved_len >= PATH_MAX) {
+			if (m)
+				free(resolved);
 			errno = ENAMETOOLONG;
 			return (NULL);
 		}
@@ -141,16 +166,23 @@ realpath(const char *path, char resolved[PATH_MAX])
 				errno = serrno;
 				return (resolved);
 			}
+			if (m)
+				free(resolved);
 			return (NULL);
 		}
 		if (S_ISLNK(sb.st_mode)) {
 			if (symlinks++ > MAXSYMLINKS) {
+				if (m)
+					free(resolved);
 				errno = ELOOP;
 				return (NULL);
 			}
 			slen = readlink(resolved, symlink, sizeof(symlink) - 1);
-			if (slen < 0)
+			if (slen < 0) {
+				if (m)
+					free(resolved);
 				return (NULL);
+			}
 			symlink[slen] = '\0';
 			if (symlink[0] == '/') {
 				resolved[1] = 0;
@@ -171,6 +203,8 @@ realpath(const char *path, char resolved[PATH_MAX])
 			if (p != NULL) {
 				if (symlink[slen - 1] != '/') {
 					if (slen + 1 >= sizeof(symlink)) {
+						if (m)
+							free(resolved);
 						errno = ENAMETOOLONG;
 						return (NULL);
 					}
@@ -179,6 +213,8 @@ realpath(const char *path, char resolved[PATH_MAX])
 				}
 				left_len = strlcat(symlink, left, sizeof(left));
 				if (left_len >= sizeof(left)) {
+					if (m)
+						free(resolved);
 					errno = ENAMETOOLONG;
 					return (NULL);
 				}
-- 
cgit v1.1


From 4925aa8c65bf2657c27d8d1086b5c67b8637c949 Mon Sep 17 00:00:00 2001
From: kib <kib@FreeBSD.org>
Date: Wed, 5 May 2010 09:29:34 +0000
Subject: MFC r207009: C language does not has references, it provides
 pointers.

---
 lib/libc/stdlib/realpath.3 | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/stdlib/realpath.3 b/lib/libc/stdlib/realpath.3
index aee65f7..166bb12 100644
--- a/lib/libc/stdlib/realpath.3
+++ b/lib/libc/stdlib/realpath.3
@@ -56,13 +56,13 @@ and
 in
 .Fa pathname ,
 and copies the resulting absolute pathname into
-the memory referenced by
+the memory pointed to by
 .Fa resolved_path .
 The
 .Fa resolved_path
 argument
 .Em must
-refer to a buffer capable of storing at least
+point to a buffer capable of storing at least
 .Dv PATH_MAX
 characters, or be
 .Dv NULL .
-- 
cgit v1.1


From 91722e5e54cdcd3b5c4ee49118ba83663b9bd144 Mon Sep 17 00:00:00 2001
From: jilles <jilles@FreeBSD.org>
Date: Wed, 5 May 2010 22:00:57 +0000
Subject: MFC r207186: sysctl(3): Update description of various kern.*
 variables. Also add xrefs for confstr(3) (as sysconf(3) but for strings) and
 kvm(3) (which is a more convenient way to access some of the variables).

PR:		116480
---
 lib/libc/gen/sysctl.3 | 22 ++++++++++------------
 1 file changed, 10 insertions(+), 12 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/gen/sysctl.3 b/lib/libc/gen/sysctl.3
index 75f8668..77e7d92 100644
--- a/lib/libc/gen/sysctl.3
+++ b/lib/libc/gen/sysctl.3
@@ -28,7 +28,7 @@
 .\"	@(#)sysctl.3	8.4 (Berkeley) 5/9/95
 .\" $FreeBSD$
 .\"
-.Dd January 28, 2009
+.Dd April 25, 2010
 .Dt SYSCTL 3
 .Os
 .Sh NAME
@@ -326,7 +326,7 @@ information.
 .It "KERN_BOOTFILE	string	yes"
 .It "KERN_BOOTTIME	struct timeval	no"
 .It "KERN_CLOCKRATE	struct clockinfo	no"
-.It "KERN_FILE	struct file	no"
+.It "KERN_FILE	struct xfile	no"
 .It "KERN_HOSTID	integer	yes"
 .It "KERN_HOSTUUID	string	yes"
 .It "KERN_HOSTNAME	string	yes"
@@ -343,14 +343,14 @@ information.
 .It "KERN_OSREV	integer	no"
 .It "KERN_OSTYPE	string	no"
 .It "KERN_POSIX1	integer	no"
-.It "KERN_PROC	struct proc	no"
+.It "KERN_PROC	node	not applicable"
 .It "KERN_PROF	node	not applicable"
 .It "KERN_QUANTUM	integer	yes"
 .It "KERN_SAVED_IDS	integer	no"
 .It "KERN_SECURELVL	integer	raise only"
 .It "KERN_UPDATEINTERVAL	integer	no"
 .It "KERN_VERSION	string	no"
-.It "KERN_VNODE	struct vnode	no"
+.It "KERN_VNODE	struct xvnode	no"
 .El
 .Pp
 .Bl -tag -width 6n
@@ -372,10 +372,8 @@ This structure contains the clock, statistics clock and profiling clock
 frequencies, the number of micro-seconds per hz tick and the skew rate.
 .It Li KERN_FILE
 Return the entire file table.
-The returned data consists of a single
-.Va struct filehead
-followed by an array of
-.Va struct file ,
+The returned data consists of an array of
+.Va struct xfile ,
 whose size depends on the current number of such objects in the system.
 .It Li KERN_HOSTID
 Get or set the host ID.
@@ -527,10 +525,8 @@ Note, the vnode table is not necessarily a consistent snapshot of
 the system.
 The returned data consists of an array whose size depends on the
 current number of such objects in the system.
-Each element of the array contains the kernel address of a vnode
-.Va struct vnode *
-followed by the vnode itself
-.Va struct vnode .
+Each element of the array consists of a
+.Va struct xvnode .
 .El
 .Ss CTL_NET
 The string and integer information available for the CTL_NET level
@@ -859,6 +855,8 @@ An attempt is made to set a read-only value.
 A process without appropriate privilege attempts to set a value.
 .El
 .Sh SEE ALSO
+.Xr confstr 3 ,
+.Xr kvm 3 ,
 .Xr sysconf 3 ,
 .Xr sysctl 8
 .Sh HISTORY
-- 
cgit v1.1


From dced090d8c1e52f4b2beeccd22da467066a3f03b Mon Sep 17 00:00:00 2001
From: jilles <jilles@FreeBSD.org>
Date: Wed, 5 May 2010 22:12:56 +0000
Subject: MFC r207190: unlinkat(2): unlinkat(AT_REMOVEDIR) fails with ENOTEMPTY
 like rmdir() for non-empty directories.

POSIX permits both ENOTEMPTY and EEXIST, but we use the clearer ENOTEMPTY,
following BSD tradition.
---
 lib/libc/sys/unlink.2 | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/sys/unlink.2 b/lib/libc/sys/unlink.2
index 41a2288..ac18971 100644
--- a/lib/libc/sys/unlink.2
+++ b/lib/libc/sys/unlink.2
@@ -28,7 +28,7 @@
 .\"     @(#)unlink.2	8.1 (Berkeley) 6/4/93
 .\" $FreeBSD$
 .\"
-.Dd April 10, 2008
+.Dd April 25, 2010
 .Dt UNLINK 2
 .Os
 .Sh NAME
@@ -165,7 +165,7 @@ argument does not specify an absolute path and the
 argument is neither
 .Dv AT_FDCWD
 nor a valid file descriptor open for searching.
-.It Bq Er EEXIST
+.It Bq Er ENOTEMPTY
 The
 .Fa flag
 parameter has the
-- 
cgit v1.1


From f127752dd2cf20c1672e4b541d92b10ca9a73fb8 Mon Sep 17 00:00:00 2001
From: jilles <jilles@FreeBSD.org>
Date: Wed, 5 May 2010 22:17:17 +0000
Subject: MFC r206760: getcwd(3): Clarify that EACCES may or may not be
 checked.

POSIX permits but does not require checking access on the current and parent
directories.

Because various programs do not like it if getcwd(3) fails, it seems best
to avoid checking access as much as possible. There are various reports in
GNATS about this (search for getcwd).

Our getcwd(3) implementation first queries the kernel for the pathname
directly, which does not check any permissions but sometimes fails, and then
falls back to reading all parent directories for the names.

PR:		standards/44425
---
 lib/libc/gen/getcwd.3 | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/gen/getcwd.3 b/lib/libc/gen/getcwd.3
index 12a2bd6..88291c3 100644
--- a/lib/libc/gen/getcwd.3
+++ b/lib/libc/gen/getcwd.3
@@ -28,7 +28,7 @@
 .\"     @(#)getcwd.3	8.2 (Berkeley) 12/11/93
 .\" $FreeBSD$
 .\"
-.Dd November 24, 1997
+.Dd April 17, 2010
 .Dt GETCWD 3
 .Os
 .Sh NAME
@@ -108,8 +108,6 @@ The
 function
 will fail if:
 .Bl -tag -width Er
-.It Bq Er EACCES
-Read or search permission was denied for a component of the pathname.
 .It Bq Er EINVAL
 The
 .Fa size
@@ -124,6 +122,16 @@ The
 argument is greater than zero but smaller than the length of the pathname
 plus 1.
 .El
+.Pp
+The
+.Fn getcwd
+function
+may fail if:
+.Bl -tag -width Er
+.It Bq Er EACCES
+Read or search permission was denied for a component of the pathname.
+This is only checked in limited cases, depending on implementation details.
+.El
 .Sh SEE ALSO
 .Xr chdir 2 ,
 .Xr fchdir 2 ,
-- 
cgit v1.1


From 57be3b3deba6ed645b3d61ad5899c5db876d271f Mon Sep 17 00:00:00 2001
From: kib <kib@FreeBSD.org>
Date: Tue, 11 May 2010 13:39:37 +0000
Subject: MFC r207604: Document RUSAGE_THREAD.

---
 lib/libc/sys/getrusage.2 | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/sys/getrusage.2 b/lib/libc/sys/getrusage.2
index bdf5d45..423503f 100644
--- a/lib/libc/sys/getrusage.2
+++ b/lib/libc/sys/getrusage.2
@@ -28,7 +28,7 @@
 .\"     @(#)getrusage.2	8.1 (Berkeley) 6/4/93
 .\" $FreeBSD$
 .\"
-.Dd June 4, 1993
+.Dd May 1, 2010
 .Dt GETRUSAGE 2
 .Os
 .Sh NAME
@@ -42,6 +42,7 @@
 .In sys/resource.h
 .Fd "#define	RUSAGE_SELF	 0"
 .Fd "#define	RUSAGE_CHILDREN	-1"
+.Fd "#define	RUSAGE_THREAD	1"
 .Ft int
 .Fn getrusage "int who" "struct rusage *rusage"
 .Sh DESCRIPTION
@@ -49,11 +50,12 @@ The
 .Fn getrusage
 system call
 returns information describing the resources utilized by the current
-process, or all its terminated child processes.
+thread, the current process, or all its terminated child processes.
 The
 .Fa who
 argument is either
-.Dv RUSAGE_SELF
+.Dv RUSAGE_THREAD ,
+.Dv RUSAGE_SELF ,
 or
 .Dv RUSAGE_CHILDREN .
 The buffer to which
@@ -175,6 +177,10 @@ The
 .Fn getrusage
 system call appeared in
 .Bx 4.2 .
+The
+.Dv RUSAGE_THREAD
+facility first appeared in
+.Fx 8.1 .
 .Sh BUGS
 There is no way to obtain information about a child process
 that has not yet terminated.
-- 
cgit v1.1


From a324d90d0a18d25c754b7f0c8621be0aca0ace6f Mon Sep 17 00:00:00 2001
From: jilles <jilles@FreeBSD.org>
Date: Thu, 13 May 2010 15:44:49 +0000
Subject: MFC r207734: sigaltstack(2): document some modernizations: *
 un-document 'struct sigaltstack' tag for stack_t as this is BSD-specific;  
 this doesn't seem useful enough to document as such * alternate stacks are
 per thread, not per process * update error codes to what the kernel does and
 POSIX requires

---
 lib/libc/sys/sigaltstack.2 | 32 +++++++++++++++-----------------
 1 file changed, 15 insertions(+), 17 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/sys/sigaltstack.2 b/lib/libc/sys/sigaltstack.2
index da6877e..6119c6a 100644
--- a/lib/libc/sys/sigaltstack.2
+++ b/lib/libc/sys/sigaltstack.2
@@ -28,7 +28,7 @@
 .\"     @(#)sigaltstack.2	8.2 (Berkeley) 5/1/95
 .\" $FreeBSD$
 .\"
-.Dd May 1, 1995
+.Dd May 6, 2010
 .Dt SIGALTSTACK 2
 .Os
 .Sh NAME
@@ -39,7 +39,7 @@
 .Sh SYNOPSIS
 .In signal.h
 .Bd -literal
-typedef struct sigaltstack {
+typedef struct {
         char    *ss_sp;
         size_t  ss_size;
         int     ss_flags;
@@ -51,25 +51,25 @@ typedef struct sigaltstack {
 The
 .Fn sigaltstack
 system call
-allows users to define an alternate stack on which signals
-are to be processed.
+allows defining an alternate stack on which signals
+are to be processed for the current thread.
 If
 .Fa ss
 is non-zero,
 it specifies a pointer to and the size of a
 .Em "signal stack"
-on which to deliver signals,
-and tells the system if the process is currently executing
-on that stack.
+on which to deliver signals.
 When a signal's action indicates its handler
 should execute on the signal stack (specified with a
 .Xr sigaction 2
 system call), the system checks to see
-if the process is currently executing on that stack.
-If the process is not currently executing on the signal stack,
+if the thread is currently executing on that stack.
+If the thread is not currently executing on the signal stack,
 the system arranges a switch to the signal stack for the
 duration of the signal handler's execution.
 .Pp
+An active stack cannot be modified.
+.Pp
 If
 .Dv SS_DISABLE
 is set in
@@ -78,12 +78,6 @@ is set in
 and
 .Fa ss_size
 are ignored and the signal stack will be disabled.
-Trying to disable an active stack will cause
-.Fn sigaltstack
-to return -1 with
-.Va errno
-set to
-.Er EINVAL .
 A disabled stack will cause all signals to be
 taken on the regular user stack.
 If the stack is later re-enabled then all signals that were specified
@@ -96,7 +90,7 @@ The
 .Fa ss_flags
 field will contain the value
 .Dv SS_ONSTACK
-if the process is currently on a signal stack and
+if the thread is currently on a signal stack and
 .Dv SS_DISABLE
 if the signal stack is currently disabled.
 .Sh NOTES
@@ -146,8 +140,12 @@ or
 .Fa oss
 points to memory that is not a valid part of the process
 address space.
+.It Bq Er EPERM
+An attempt was made to modify an active stack.
 .It Bq Er EINVAL
-An attempt was made to disable an active stack.
+The
+.Fa ss_flags
+field was invalid.
 .It Bq Er ENOMEM
 Size of alternate stack area is less than or equal to
 .Dv MINSIGSTKSZ .
-- 
cgit v1.1


From 05725ac18b7ac599ca6a906125209db0e3c779c6 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Thu, 13 May 2010 23:28:20 +0000
Subject: MFC r205099-205100,205108:

Two optimizations to MI strlen(3) inspired by David S. Miller's
blog posting [1].

 - Use word-sized test for unaligned pointer before working
   the hard way.

   Memory page boundary is always integral multiple of a word
   alignment boundary.  Therefore, if we can access memory
   referenced by pointer p, then (p & ~word mask) must be also
   accessible.

 - Better utilization of multi-issue processor's ability of
   concurrency.

   The previous implementation utilized a formular that must be
   executed sequentially.  However, the ~, & and - operations can
   actually be caculated at the same time when the operand were
   different and unrelated.

   The original Hacker's Delight formular also offered consistent
   performance regardless whether the input would contain
   characters with their highest-bit set, as it catches real
   nul characters only.

These two optimizations has shown further improvements over the
previous implementation on microbenchmarks on i386 and amd64 CPU
including Pentium 4, Core Duo 2 and i7.

[1] http://vger.kernel.org/~davem/cgi-bin/blog.cgi/2010/03/08#strlen_1
---
 lib/libc/string/strlen.c | 70 +++++++++++++++++++++++++++++++-----------------
 1 file changed, 45 insertions(+), 25 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/string/strlen.c b/lib/libc/string/strlen.c
index 860a988..2bc1f2b 100644
--- a/lib/libc/string/strlen.c
+++ b/lib/libc/string/strlen.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2009 Xin LI <delphij@FreeBSD.org>
+ * Copyright (c) 2009, 2010 Xin LI <delphij@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -43,15 +43,17 @@ __FBSDID("$FreeBSD$");
  *	((x - 0x01....01) & ~x & 0x80....80)
  *
  * would evaluate to a non-zero value iff any of the bytes in the
- * original word is zero.  However, we can further reduce ~1/3 of
- * time if we consider that strlen() usually operate on 7-bit ASCII
- * by employing the following expression, which allows false positive
- * when high bit of 1 and use the tail case to catch these case:
+ * original word is zero.
  *
- *	((x - 0x01....01) & 0x80....80)
+ * On multi-issue processors, we can divide the above expression into:
+ *	a)  (x - 0x01....01)
+ *	b) (~x & 0x80....80)
+ *	c) a & b
  *
- * This is more than 5.2 times as fast as the raw implementation on
- * Intel T7300 under long mode for strings longer than word length.
+ * Where, a) and b) can be partially computed in parallel.
+ *
+ * The algorithm above is found on "Hacker's Delight" by
+ * Henry S. Warren, Jr.
  */
 
 /* Magic numbers for the algorithm */
@@ -82,29 +84,47 @@ strlen(const char *str)
 {
 	const char *p;
 	const unsigned long *lp;
+	long va, vb;
 
-	/* Skip the first few bytes until we have an aligned p */
-	for (p = str; (uintptr_t)p & LONGPTR_MASK; p++)
-	    if (*p == '\0')
-		return (p - str);
+	/*
+	 * Before trying the hard (unaligned byte-by-byte access) way
+	 * to figure out whether there is a nul character, try to see
+	 * if there is a nul character is within this accessible word
+	 * first.
+	 *
+	 * p and (p & ~LONGPTR_MASK) must be equally accessible since
+	 * they always fall in the same memory page, as long as page
+	 * boundaries is integral multiple of word size.
+	 */
+	lp = (const unsigned long *)((uintptr_t)str & ~LONGPTR_MASK);
+	va = (*lp - mask01);
+	vb = ((~*lp) & mask80);
+	lp++;
+	if (va & vb)
+		/* Check if we have \0 in the first part */
+		for (p = str; p < (const char *)lp; p++)
+			if (*p == '\0')
+				return (p - str);
 
 	/* Scan the rest of the string using word sized operation */
-	for (lp = (const unsigned long *)p; ; lp++)
-	    if ((*lp - mask01) & mask80) {
-		p = (const char *)(lp);
-		testbyte(0);
-		testbyte(1);
-		testbyte(2);
-		testbyte(3);
+	for (; ; lp++) {
+		va = (*lp - mask01);
+		vb = ((~*lp) & mask80);
+		if (va & vb) {
+			p = (const char *)(lp);
+			testbyte(0);
+			testbyte(1);
+			testbyte(2);
+			testbyte(3);
 #if (LONG_BIT >= 64)
-		testbyte(4);
-		testbyte(5);
-		testbyte(6);
-		testbyte(7);
+			testbyte(4);
+			testbyte(5);
+			testbyte(6);
+			testbyte(7);
 #endif
-	    }
+		}
+	}
 
 	/* NOTREACHED */
 	return (0);
 }
-
-- 
cgit v1.1


From e058c73bc5fcf603d4ea8f776c416ac1b7b0bf13 Mon Sep 17 00:00:00 2001
From: mm <mm@FreeBSD.org>
Date: Mon, 17 May 2010 14:27:40 +0000
Subject: MFC r207553, r207555, r207651:

Implement the no_user_check option to pam_krb5.

This option is available in the Linux implementation of pam_krb5
and allows to authorize a user not known to the local system.

Ccache is not used as we don't have a secure uid/gid for the cache file.

Usable for authentication of external kerberos users (e.g Active Directory)
via PAM from applications like Cyrus saslauthd, PHP or perl.

PR:		bin/146186
Approved by:	deplhij (mentor)
---
 lib/libpam/modules/pam_krb5/pam_krb5.8 |  6 +++-
 lib/libpam/modules/pam_krb5/pam_krb5.c | 53 +++++++++++++++++++---------------
 2 files changed, 35 insertions(+), 24 deletions(-)

(limited to 'lib')

diff --git a/lib/libpam/modules/pam_krb5/pam_krb5.8 b/lib/libpam/modules/pam_krb5/pam_krb5.8
index 3e0db91..bd7ac5b 100644
--- a/lib/libpam/modules/pam_krb5/pam_krb5.8
+++ b/lib/libpam/modules/pam_krb5/pam_krb5.8
@@ -1,7 +1,7 @@
 .\"
 .\" $Id: pam_krb5.5,v 1.5 2000/01/05 00:59:56 fcusack Exp $
 .\" $FreeBSD$
-.Dd January 15, 1999
+.Dd May 3, 2010
 .Dt PAM_KRB5 8
 .Os
 .Sh NAME
@@ -108,6 +108,10 @@ and
 .Ql %p ,
 to designate the current process ID; can be used in
 .Ar name .
+.It Cm no_user_check
+Do not verify if a user exists on the local system. This option implies the
+.Cm no_ccache
+option because there is no secure local uid/gid for the cache file.
 .El
 .Ss Kerberos 5 Account Management Module
 The Kerberos 5 account management component
diff --git a/lib/libpam/modules/pam_krb5/pam_krb5.c b/lib/libpam/modules/pam_krb5/pam_krb5.c
index b56e0a3..439fcf9 100644
--- a/lib/libpam/modules/pam_krb5/pam_krb5.c
+++ b/lib/libpam/modules/pam_krb5/pam_krb5.c
@@ -89,6 +89,7 @@ static void	compat_free_data_contents(krb5_context, krb5_data *);
 #define PAM_OPT_DEBUG		"debug"
 #define PAM_OPT_FORWARDABLE	"forwardable"
 #define PAM_OPT_NO_CCACHE	"no_ccache"
+#define PAM_OPT_NO_USER_CHECK	"no_user_check"
 #define PAM_OPT_REUSE_CCACHE	"reuse_ccache"
 
 /*
@@ -194,34 +195,39 @@ pam_sm_authenticate(pam_handle_t *pamh, int flags __unused,
 
 	PAM_LOG("Got password");
 
-	/* Verify the local user exists (AFTER getting the password) */
-	if (strchr(user, '@')) {
-		/* get a local account name for this principal */
-		krbret = krb5_aname_to_localname(pam_context, princ,
-		    sizeof(luser), luser);
-		if (krbret != 0) {
-			PAM_VERBOSE_ERROR("Kerberos 5 error");
-			PAM_LOG("Error krb5_aname_to_localname(): %s",
-			    krb5_get_err_text(pam_context, krbret));
-			retval = PAM_USER_UNKNOWN;
-			goto cleanup2;
+	if (openpam_get_option(pamh, PAM_OPT_NO_USER_CHECK))
+		PAM_LOG("Skipping local user check");
+	else {
+
+		/* Verify the local user exists (AFTER getting the password) */
+		if (strchr(user, '@')) {
+			/* get a local account name for this principal */
+			krbret = krb5_aname_to_localname(pam_context, princ,
+			    sizeof(luser), luser);
+			if (krbret != 0) {
+				PAM_VERBOSE_ERROR("Kerberos 5 error");
+				PAM_LOG("Error krb5_aname_to_localname(): %s",
+				    krb5_get_err_text(pam_context, krbret));
+				retval = PAM_USER_UNKNOWN;
+				goto cleanup2;
+			}
+
+			retval = pam_set_item(pamh, PAM_USER, luser);
+			if (retval != PAM_SUCCESS)
+				goto cleanup2;
+
+			PAM_LOG("PAM_USER Redone");
 		}
 
-		retval = pam_set_item(pamh, PAM_USER, luser);
-		if (retval != PAM_SUCCESS)
+		pwd = getpwnam(user);
+		if (pwd == NULL) {
+			retval = PAM_USER_UNKNOWN;
 			goto cleanup2;
+		}
 
-		PAM_LOG("PAM_USER Redone");
-	}
-
-	pwd = getpwnam(user);
-	if (pwd == NULL) {
-		retval = PAM_USER_UNKNOWN;
-		goto cleanup2;
+		PAM_LOG("Done getpwnam()");
 	}
 
-	PAM_LOG("Done getpwnam()");
-
 	/* Get a TGT */
 	memset(&creds, 0, sizeof(krb5_creds));
 	krbret = krb5_get_init_creds_password(pam_context, &creds, princ,
@@ -366,7 +372,8 @@ pam_sm_setcred(pam_handle_t *pamh, int flags,
 		return (PAM_SERVICE_ERR);
 
 	/* If a persistent cache isn't desired, stop now. */
-	if (openpam_get_option(pamh, PAM_OPT_NO_CCACHE))
+	if (openpam_get_option(pamh, PAM_OPT_NO_CCACHE) ||
+		openpam_get_option(pamh, PAM_OPT_NO_USER_CHECK))
 		return (PAM_SUCCESS);
 
 	PAM_LOG("Establishing credentials");
-- 
cgit v1.1


From 40310299b8ffb63ea75c19d7352572e3f5bea061 Mon Sep 17 00:00:00 2001
From: thompsa <thompsa@FreeBSD.org>
Date: Mon, 17 May 2010 23:55:23 +0000
Subject: MFC r208020

 Fix header file compliancy with libusb 1.0 from sourceforge.
---
 lib/libusb/libusb.h        | 4 ++--
 lib/libusb/libusb10.c      | 2 ++
 lib/libusb/libusb10_desc.c | 2 ++
 lib/libusb/libusb10_io.c   | 2 ++
 4 files changed, 8 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/libusb/libusb.h b/lib/libusb/libusb.h
index bdf2812..fb18c6e 100644
--- a/lib/libusb/libusb.h
+++ b/lib/libusb/libusb.h
@@ -185,7 +185,7 @@ enum libusb_debug_level {
 struct libusb_context;
 struct libusb_device;
 struct libusb_transfer;
-struct libusb20_device;
+struct libusb_device_handle;
 
 struct libusb_pollfd {
 	int	fd;
@@ -194,7 +194,7 @@ struct libusb_pollfd {
 
 typedef struct libusb_context libusb_context;
 typedef struct libusb_device libusb_device;
-typedef struct libusb20_device libusb_device_handle;
+typedef struct libusb_device_handle libusb_device_handle;
 typedef struct libusb_pollfd libusb_pollfd;
 typedef void (*libusb_pollfd_added_cb) (int fd, short events, void *user_data);
 typedef void (*libusb_pollfd_removed_cb) (int fd, void *user_data);
diff --git a/lib/libusb/libusb10.c b/lib/libusb/libusb10.c
index 25520d2..598e78c 100644
--- a/lib/libusb/libusb10.c
+++ b/lib/libusb/libusb10.c
@@ -37,6 +37,8 @@
 #include <sys/queue.h>
 #include <sys/endian.h>
 
+#define	libusb_device_handle libusb20_device
+
 #include "libusb20.h"
 #include "libusb20_desc.h"
 #include "libusb20_int.h"
diff --git a/lib/libusb/libusb10_desc.c b/lib/libusb/libusb10_desc.c
index 0215bce..b5b06bc 100644
--- a/lib/libusb/libusb10_desc.c
+++ b/lib/libusb/libusb10_desc.c
@@ -29,6 +29,8 @@
 #include <pthread.h>
 #include <sys/queue.h>
 
+#define	libusb_device_handle libusb20_device
+
 #include "libusb20.h"
 #include "libusb20_desc.h"
 #include "libusb20_int.h"
diff --git a/lib/libusb/libusb10_io.c b/lib/libusb/libusb10_io.c
index 3d6daef..ce79723 100644
--- a/lib/libusb/libusb10_io.c
+++ b/lib/libusb/libusb10_io.c
@@ -34,6 +34,8 @@
 #include <sys/queue.h>
 #include <sys/endian.h>
 
+#define	libusb_device_handle libusb20_device
+
 #include "libusb20.h"
 #include "libusb20_desc.h"
 #include "libusb20_int.h"
-- 
cgit v1.1


From 4c0081bfc8f7b68e52b1a5d8c3e5d5427ad3ef59 Mon Sep 17 00:00:00 2001
From: thompsa <thompsa@FreeBSD.org>
Date: Mon, 17 May 2010 23:56:17 +0000
Subject: MFC r208021

 Fix return values for usb_find_busses() and usb_find_devices(). We should try
 to return the actual number of busses and devices.
---
 lib/libusb/libusb20_compat01.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/libusb/libusb20_compat01.c b/lib/libusb/libusb20_compat01.c
index 48a7dd5..e53770c 100644
--- a/lib/libusb/libusb20_compat01.c
+++ b/lib/libusb/libusb20_compat01.c
@@ -820,7 +820,7 @@ int
 usb_find_busses(void)
 {
 	usb_busses = &usb_global_bus;
-	return (0);
+	return (1);
 }
 
 int
@@ -904,7 +904,7 @@ usb_find_devices(void)
 		LIST_ADD(usb_global_bus.devices, udev);
 	}
 
-	return (0);			/* success */
+	return (devnum - 1);			/* success */
 }
 
 struct usb_device *
-- 
cgit v1.1


From fe4b80709b0c6a420232cfe54bcb668771f04347 Mon Sep 17 00:00:00 2001
From: thompsa <thompsa@FreeBSD.org>
Date: Mon, 17 May 2010 23:57:34 +0000
Subject: MFC r208012

 Support getting signed and unsigned HID data.
---
 lib/libusbhid/data.c | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

(limited to 'lib')

diff --git a/lib/libusbhid/data.c b/lib/libusbhid/data.c
index d278ddb..087989c 100644
--- a/lib/libusbhid/data.c
+++ b/lib/libusbhid/data.c
@@ -53,13 +53,17 @@ hid_get_data(const void *p, const hid_item_t *h)
 	data = 0;
 	for (i = 0; i <= end; i++)
 		data |= buf[offs + i] << (i*8);
+
+	/* Correctly shift down data */
 	data >>= hpos % 8;
-	data &= (1 << hsize) - 1;
-	if (h->logical_minimum < 0) {
-		/* Need to sign extend */
-		hsize = sizeof data * 8 - hsize;
-		data = (data << hsize) >> hsize;
-	}
+	hsize = 32 - hsize;
+
+	/* Mask and sign extend in one */
+	if ((h->logical_minimum < 0) || (h->logical_maximum < 0))
+		data = (int32_t)((int32_t)data << hsize) >> hsize;
+	else
+		data = (uint32_t)((uint32_t)data << hsize) >> hsize;
+
 	return (data);
 }
 
-- 
cgit v1.1


From 90fb8f94bd2930565982e30ae0b9f6d1ee1bda6a Mon Sep 17 00:00:00 2001
From: thompsa <thompsa@FreeBSD.org>
Date: Mon, 17 May 2010 23:59:14 +0000
Subject: MFC r203773

 Within libusb 0.1 API, bus number is always faked to 0. Device numbers,
 however, are possitive and seem to be reverse sorted in the list.

 Conform device numbering and bring a result that is consistent with the
 libusb 0.1 API.  It is now possible to distinguish a device based on its
 (bus, dev) numbers.
---
 lib/libusb/libusb20_compat01.c | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'lib')

diff --git a/lib/libusb/libusb20_compat01.c b/lib/libusb/libusb20_compat01.c
index e53770c..43e3344 100644
--- a/lib/libusb/libusb20_compat01.c
+++ b/lib/libusb/libusb20_compat01.c
@@ -829,6 +829,7 @@ usb_find_devices(void)
 	struct libusb20_device *pdev;
 	struct usb_device *udev;
 	struct LIBUSB20_DEVICE_DESC_DECODED *ddesc;
+	int devnum;
 	int err;
 
 	/* cleanup after last device search */
@@ -855,6 +856,7 @@ usb_find_devices(void)
 	}
 	/* iterate all devices */
 
+	devnum = 1;
 	pdev = NULL;
 	while ((pdev = libusb20_be_device_foreach(usb_backend, pdev))) {
 		udev = malloc(sizeof(*udev));
@@ -891,6 +893,7 @@ usb_find_devices(void)
 			/* truncate number of configurations */
 			udev->descriptor.bNumConfigurations = USB_MAXCONFIG;
 		}
+		udev->devnum = devnum++;
 		/* link together the two structures */
 		udev->dev = pdev;
 		pdev->privLuData = udev;
-- 
cgit v1.1


From aeccfd7aa1f603dae9baccbaf18fe0dbcb27ac5e Mon Sep 17 00:00:00 2001
From: mm <mm@FreeBSD.org>
Date: Tue, 18 May 2010 00:46:15 +0000
Subject: MFC r204738, r205113 (imp):

MFC r204738:
Remove stale references to libkrb5.
Rejigger the SUBDIR setting a smidge: we now set all the libraries that depend
on something else, and then SUBDIR+= the rest.

MFC r205113:
Make this conform to the other top-level Makefile subdir listings with
one file per line.

Approved by:	delphij (mentor)
---
 lib/Makefile | 93 ++++++++++++++++++++++++++++++++++++++++++++++++++----------
 1 file changed, 78 insertions(+), 15 deletions(-)

(limited to 'lib')

diff --git a/lib/Makefile b/lib/Makefile
index 00b1c60..694b606 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -9,8 +9,8 @@
 # csu must be built before all shared libaries for ELF.
 # libc must be built before all other shared libraries.
 # libbsm must be built before ibauditd.
-# libcom_err must be built before libkrb5 and libpam.
-# libcrypt must be built before libkrb5 and libpam.
+# libcom_err must be built before libpam.
+# libcrypt must be built before libpam.
 # libkvm must be built before libdevstat.
 # msun must be built before libg++ and libstdc++.
 # libmd must be built before libatm, libopie, libradius, and libtacplus.
@@ -26,21 +26,84 @@
 # libgssapi must be built before librpcsec_gss
 #
 # Otherwise, the SUBDIR list should be in alphabetical order.
-
-SUBDIR=	${_csu} libc libbsm libauditd libcom_err libcrypt libelf libkvm msun \
+#
+# Except it appears bind needs to be compiled last
+
+SUBDIR_ORDERED=	${_csu} \
+	libc \
+	libbsm \
+	libauditd \
+	libcom_err \
+	libcrypt \
+	libelf \
+	libkvm \
+	msun \
 	libmd \
-	ncurses ${_libnetgraph} libradius librpcsvc libsbuf \
-	libtacplus libutil ${_libypclnt} libalias libarchive ${_libatm} \
-	libbegemot ${_libbluetooth} ${_libbsnmp} libbz2 \
-	libcalendar libcam libcompat libdevinfo libdevstat libdisk \
-	libdwarf libedit libexpat libfetch libftpio libgeom ${_libgpib} \
-	${_libgssapi} ${_librpcsec_gss} libipsec \
-	${_libipx} libjail libkiconv libmagic libmemstat ${_libmilter} \
-	${_libmp} ${_libncp} ${_libngatm} libopie libpam libpcap \
-	${_libpmc} libproc librt ${_libsdp} ${_libsm} ${_libsmb} \
+	ncurses \
+	${_libnetgraph} \
+	libradius \
+	librpcsvc \
+	libsbuf \
+	libtacplus \
+	libutil \
+	${_libypclnt}
+
+SUBDIR= ${SUBDIR_ORDERED} \
+	libalias \
+	libarchive \
+	${_libatm} \
+	libbegemot \
+	${_libbluetooth} \
+	${_libbsnmp} \
+	libbz2 \
+	libcalendar \
+	libcam \
+	libcompat \
+	libdevinfo \
+	libdevstat \
+	libdisk \
+	libdwarf \
+	libedit \
+	libexpat \
+	libfetch \
+	libftpio \
+	libgeom \
+	${_libgpib} \
+	${_libgssapi} \
+	${_librpcsec_gss} \
+	libipsec \
+	${_libipx} \
+	libjail \
+	libkiconv \
+	libmagic \
+	libmemstat \
+	${_libmilter} \
+	${_libmp} \
+	${_libncp} \
+	${_libngatm} \
+	libopie \
+	libpam \
+	libpcap \
+	${_libpmc} \
+	libproc \
+	librt \
+	${_libsdp} \
+	${_libsm} \
+	${_libsmb} \
 	${_libsmdb} \
-	${_libsmutil} libstand ${_libtelnet} ${_libthr} libthread_db libufs \
-	libugidfw ${_libusbhid} ${_libusb} ${_libvgl} libwrap liby libz \
+	${_libsmutil} \
+	libstand \
+	${_libtelnet} \
+	${_libthr} \
+	libthread_db \
+	libufs \
+	libugidfw \
+	${_libusbhid} \
+	${_libusb} \
+	${_libvgl} \
+	libwrap \
+	liby \
+	libz \
 	${_bind}
 
 .if exists(${.CURDIR}/csu/${MACHINE_ARCH}-elf)
-- 
cgit v1.1


From 684fea40b494a86f147beb6d274924a54e0106a7 Mon Sep 17 00:00:00 2001
From: thompsa <thompsa@FreeBSD.org>
Date: Tue, 18 May 2010 03:16:02 +0000
Subject: MFC r208023

 Use fixed width integer types for parsing the binary hid data.
---
 lib/libusbhid/data.c   | 9 ++++++---
 lib/libusbhid/usbhid.h | 5 +++--
 2 files changed, 9 insertions(+), 5 deletions(-)

(limited to 'lib')

diff --git a/lib/libusbhid/data.c b/lib/libusbhid/data.c
index 087989c..eb42422 100644
--- a/lib/libusbhid/data.c
+++ b/lib/libusbhid/data.c
@@ -33,7 +33,7 @@ __FBSDID("$FreeBSD$");
 #include <stdlib.h>
 #include "usbhid.h"
 
-int
+int32_t
 hid_get_data(const void *p, const hid_item_t *h)
 {
 	const unsigned char *buf;
@@ -68,12 +68,15 @@ hid_get_data(const void *p, const hid_item_t *h)
 }
 
 void
-hid_set_data(void *p, const hid_item_t *h, int data)
+hid_set_data(void *p, const hid_item_t *h, int32_t data)
 {
 	unsigned char *buf;
 	unsigned int hpos;
 	unsigned int hsize;
-	int i, end, offs, mask;
+	uint32_t mask;
+	int i;
+	int end;
+	int offs;
 
 	buf = p;
 	hpos = h->pos;			/* bit position of data */
diff --git a/lib/libusbhid/usbhid.h b/lib/libusbhid/usbhid.h
index b8751cd..90f0f03 100644
--- a/lib/libusbhid/usbhid.h
+++ b/lib/libusbhid/usbhid.h
@@ -30,6 +30,7 @@
  */
 
 #include <sys/cdefs.h>
+#include <sys/types.h>
 
 typedef struct report_desc *report_desc_t;
 
@@ -105,7 +106,7 @@ int hid_parse_usage_in_page(const char *name);
 int hid_parse_usage_page(const char *name);
 
 /* Extracting/insertion of data, data.c: */
-int hid_get_data(const void *p, const hid_item_t *h);
-void hid_set_data(void *p, const hid_item_t *h, int data);
+int32_t hid_get_data(const void *p, const hid_item_t *h);
+void hid_set_data(void *p, const hid_item_t *h, int32_t data);
 
 __END_DECLS
-- 
cgit v1.1


From be8f6d151def6056cf827b4e13ffb4ea51328c7d Mon Sep 17 00:00:00 2001
From: mm <mm@FreeBSD.org>
Date: Tue, 18 May 2010 09:59:09 +0000
Subject: MFC r207842, r207844, r208099:

MFC r207842:
Import of liblzma, xz, xzdec, lzmainfo from vendor branch
Add support for xz and lzma to lesspipe.sh (xzless, lzless)

MFC r207844:
Add two public headers missing in r207842
Adjust CFLAGS for lzmainfo, xz, xzdec

MFC r208099:
Add versioned symbols to liblzma
Use default SHLIB_MAJOR.

Approved by:	delphij (mentor)
---
 lib/Makefile             |   1 +
 lib/liblzma/Makefile     | 141 +++++++++++++++++++++++++++++++++++
 lib/liblzma/Symbol.map   | 189 +++++++++++++++++++++++++++++++++++++++++++++++
 lib/liblzma/Versions.def |   9 +++
 lib/liblzma/config.h     |  95 ++++++++++++++++++++++++
 5 files changed, 435 insertions(+)
 create mode 100644 lib/liblzma/Makefile
 create mode 100644 lib/liblzma/Symbol.map
 create mode 100644 lib/liblzma/Versions.def
 create mode 100644 lib/liblzma/config.h

(limited to 'lib')

diff --git a/lib/Makefile b/lib/Makefile
index 694b606..e9f4ec3 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -75,6 +75,7 @@ SUBDIR= ${SUBDIR_ORDERED} \
 	${_libipx} \
 	libjail \
 	libkiconv \
+	liblzma \
 	libmagic \
 	libmemstat \
 	${_libmilter} \
diff --git a/lib/liblzma/Makefile b/lib/liblzma/Makefile
new file mode 100644
index 0000000..f84ac93
--- /dev/null
+++ b/lib/liblzma/Makefile
@@ -0,0 +1,141 @@
+# $FreeBSD$
+
+LIB=		lzma
+LZMADIR=	${.CURDIR}/../../contrib/xz/src/liblzma
+
+.PATH: ${LZMADIR}/../common
+SRCS+=	tuklib_physmem.c
+
+.PATH: ${LZMADIR}/api/lzma
+
+MAININCS=	../lzma.h
+MAININCSDIR=	${INCLUDEDIR}
+
+LZMAINCS+=	base.h \
+		bcj.h \
+		block.h \
+		check.h \
+		container.h \
+		delta.h \
+		filter.h \
+		hardware.h \
+		index.h \
+		index_hash.h \
+		lzma.h \
+		stream_flags.h \
+		subblock.h \
+		version.h \
+		vli.h
+
+LZMAINCSDIR=	${INCLUDEDIR}/lzma
+
+INCSGROUPS=	MAININCS LZMAINCS
+
+.PATH: ${LZMADIR}/common
+SRCS+=	common.c \
+	block_util.c \
+	easy_preset.c \
+	filter_common.c \
+	hardware_physmem.c \
+	index.c \
+	stream_flags_common.c \
+	vli_size.c \
+	alone_encoder.c \
+	block_buffer_encoder.c \
+	block_encoder.c \
+	block_header_encoder.c \
+	easy_buffer_encoder.c \
+	easy_encoder.c \
+	easy_encoder_memusage.c \
+	filter_buffer_encoder.c \
+	filter_encoder.c \
+	filter_flags_encoder.c \
+	index_encoder.c \
+	stream_buffer_encoder.c \
+	stream_encoder.c \
+	stream_flags_encoder.c \
+	vli_encoder.c \
+	alone_decoder.c \
+	auto_decoder.c \
+	block_buffer_decoder.c \
+	block_decoder.c \
+	block_header_decoder.c \
+	easy_decoder_memusage.c \
+	filter_buffer_decoder.c \
+	filter_decoder.c \
+	filter_flags_decoder.c \
+	index_decoder.c \
+	index_hash.c \
+	stream_buffer_decoder.c \
+	stream_decoder.c \
+	stream_flags_decoder.c \
+	vli_decoder.c
+
+.PATH: ${LZMADIR}/check
+SRCS+=	check.c \
+	crc32_table.c \
+	crc64_table.c \
+	sha256.c
+.if defined(MACHINE_ARCH) && ${MACHINE_ARCH} == "i386"
+SRCS+=	crc32_x86.S \
+	crc64_x86.S
+.else
+SRCS+=	crc32_fast.c \
+	crc64_fast.c
+.endif
+
+.PATH: ${LZMADIR}/lz
+SRCS+=	lz_encoder.c \
+	lz_encoder_mf.c \
+	lz_decoder.c
+
+.PATH: ${LZMADIR}/lzma
+SRCS+=	lzma_encoder.c \
+	lzma_encoder_presets.c \
+	lzma_encoder_optimum_fast.c \
+	lzma_encoder_optimum_normal.c \
+	fastpos_table.c \
+	lzma_decoder.c \
+	lzma2_encoder.c \
+	lzma2_decoder.c
+
+.PATH: ${LZMADIR}/rangecoder
+SRCS+=	price_table.c
+
+.PATH: ${LZMADIR}/delta
+SRCS+=	delta_common.c \
+	delta_encoder.c \
+	delta_decoder.c
+
+.PATH: ${LZMADIR}/simple
+SRCS+=	simple_coder.c \
+	simple_encoder.c \
+	simple_decoder.c \
+	x86.c \
+	powerpc.c \
+	ia64.c \
+	arm.c \
+	armthumb.c \
+	sparc.c
+		
+WARNS?=	3
+
+CFLAGS+=	-DHAVE_CONFIG_H \
+		-DTUKLIB_SYMBOL_PREFIX=lzma_ \
+		-I${.CURDIR} \
+		-I${LZMADIR}/api \
+		-I${LZMADIR}/common \
+		-I${LZMADIR}/check \
+		-I${LZMADIR}/lz \
+		-I${LZMADIR}/rangecoder \
+		-I${LZMADIR}/lzma \
+		-I${LZMADIR}/subblock \
+		-I${LZMADIR}/delta \
+		-I${LZMADIR}/simple \
+		-I${LZMADIR}/../common
+
+VERSION_DEF=	${.CURDIR}/Versions.def
+SYMBOL_MAPS=	${.CURDIR}/Symbol.map
+CFLAGS+=	-DSYMBOL_VERSIONING
+
+.include <bsd.lib.mk>
diff --git a/lib/liblzma/Symbol.map b/lib/liblzma/Symbol.map
new file mode 100644
index 0000000..efefffd
--- /dev/null
+++ b/lib/liblzma/Symbol.map
@@ -0,0 +1,189 @@
+/*
+ * $FreeBSD$
+ */
+
+XZ_5.0 {
+	lzma_alone_decoder;
+	lzma_alone_encoder;
+	lzma_auto_decoder;
+	lzma_block_buffer_bound;
+	lzma_block_buffer_decode;
+	lzma_block_buffer_encode;
+	lzma_block_compressed_size;
+	lzma_block_decoder;
+	lzma_block_encoder;
+	lzma_block_header_decode;
+	lzma_block_header_encode;
+	lzma_block_header_size;
+	lzma_block_total_size;
+	lzma_block_unpadded_size;
+	lzma_check_is_supported;
+	lzma_check_size;
+	lzma_code;
+	lzma_crc32;
+	lzma_crc64;
+	lzma_easy_buffer_encode;
+	lzma_easy_decoder_memusage;
+	lzma_easy_encoder;
+	lzma_easy_encoder_memusage;
+	lzma_end;
+	lzma_filter_decoder_is_supported;
+	lzma_filter_encoder_is_supported;
+	lzma_filter_flags_decode;
+	lzma_filter_flags_encode;
+	lzma_filter_flags_size;
+	lzma_filters_copy;
+	lzma_filters_update;
+	lzma_get_check;
+	lzma_index_append;
+	lzma_index_block_count;
+	lzma_index_buffer_decode;
+	lzma_index_buffer_encode;
+	lzma_index_cat;
+	lzma_index_checks;
+	lzma_index_decoder;
+	lzma_index_dup;
+	lzma_index_encoder;
+	lzma_index_end;
+	lzma_index_file_size;
+	lzma_index_hash_append;
+	lzma_index_hash_decode;
+	lzma_index_hash_end;
+	lzma_index_hash_init;
+	lzma_index_hash_size;
+	lzma_index_init;
+	lzma_index_iter_init;
+	lzma_index_iter_locate;
+	lzma_index_iter_next;
+	lzma_index_iter_rewind;
+	lzma_index_memusage;
+	lzma_index_memused;
+	lzma_index_size;
+	lzma_index_stream_count;
+	lzma_index_stream_flags;
+	lzma_index_stream_padding;
+	lzma_index_stream_size;
+	lzma_index_total_size;
+	lzma_index_uncompressed_size;
+	lzma_lzma_preset;
+	lzma_memlimit_get;
+	lzma_memlimit_set;
+	lzma_memusage;
+	lzma_mf_is_supported;
+	lzma_mode_is_supported;
+	lzma_physmem;
+	lzma_properties_decode;
+	lzma_properties_encode;
+	lzma_properties_size;
+	lzma_raw_buffer_decode;
+	lzma_raw_buffer_encode;
+	lzma_raw_decoder;
+	lzma_raw_decoder_memusage;
+	lzma_raw_encoder;
+	lzma_raw_encoder_memusage;
+	lzma_stream_buffer_bound;
+	lzma_stream_buffer_decode;
+	lzma_stream_buffer_encode;
+	lzma_stream_decoder;
+	lzma_stream_encoder;
+	lzma_stream_flags_compare;
+	lzma_stream_footer_decode;
+	lzma_stream_footer_encode;
+	lzma_stream_header_decode;
+	lzma_stream_header_encode;
+	lzma_version_number;
+	lzma_version_string;
+	lzma_vli_decode;
+	lzma_vli_encode;
+	lzma_vli_size;
+};
+
+XZprivate_1.0 {
+	lzma_alloc;
+	lzma_alone_decoder_init;
+	lzma_block_decoder_init;
+	lzma_block_encoder_init;
+	lzma_bufcpy;
+	lzma_check_finish;
+	lzma_check_init;
+	lzma_check_update;
+	lzma_chunk_size;
+	lzma_delta_coder_init;
+	lzma_delta_coder_memusage;
+	lzma_delta_decoder_init;
+	lzma_delta_encoder_init;
+	lzma_delta_props_decode;
+	lzma_delta_props_encode;
+	lzma_easy_preset;
+	lzma_free;
+	lzma_index_encoder_init;
+	lzma_index_padding_size;
+	lzma_index_prealloc;
+	lzma_lz_decoder_init;
+	lzma_lz_decoder_memusage;
+	lzma_lz_decoder_uncompressed;
+	lzma_lz_encoder_init;
+	lzma_lz_encoder_memusage;
+	lzma_lzma2_decoder_init;
+	lzma_lzma2_decoder_memusage;
+	lzma_lzma2_encoder_init;
+	lzma_lzma2_encoder_memusage;
+	lzma_lzma2_props_decode;
+	lzma_lzma2_props_encode;
+	lzma_lzma_decoder_create;
+	lzma_lzma_decoder_init;
+	lzma_lzma_decoder_memusage;
+	lzma_lzma_decoder_memusage_nocheck;
+	lzma_lzma_encode;
+	lzma_lzma_encoder_create;
+	lzma_lzma_encoder_init;
+	lzma_lzma_encoder_memusage;
+	lzma_lzma_encoder_reset;
+	lzma_lzma_lclppb_decode;
+	lzma_lzma_lclppb_encode;
+	lzma_lzma_optimum_fast;
+	lzma_lzma_optimum_normal;
+	lzma_lzma_props_decode;
+	lzma_lzma_props_encode;
+	lzma_mf_bt2_find;
+	lzma_mf_bt2_skip;
+	lzma_mf_bt3_find;
+	lzma_mf_bt3_skip;
+	lzma_mf_bt4_find;
+	lzma_mf_bt4_skip;
+	lzma_mf_find;
+	lzma_mf_hc3_find;
+	lzma_mf_hc3_skip;
+	lzma_mf_hc4_find;
+	lzma_mf_hc4_skip;
+	lzma_next_end;
+	lzma_next_filter_init;
+	lzma_next_filter_update;
+	lzma_raw_coder_init;
+	lzma_raw_coder_memusage;
+	lzma_raw_decoder_init;
+	lzma_raw_encoder_init;
+	lzma_sha256_finish;
+	lzma_sha256_init;
+	lzma_sha256_update;
+	lzma_simple_arm_decoder_init;
+	lzma_simple_arm_encoder_init;
+	lzma_simple_armthumb_decoder_init;
+	lzma_simple_armthumb_encoder_init;
+	lzma_simple_coder_init;
+	lzma_simple_ia64_decoder_init;
+	lzma_simple_ia64_encoder_init;
+	lzma_simple_powerpc_decoder_init;
+	lzma_simple_powerpc_encoder_init;
+	lzma_simple_props_decode;
+	lzma_simple_props_encode;
+	lzma_simple_props_size;
+	lzma_simple_sparc_decoder_init;
+	lzma_simple_sparc_encoder_init;
+	lzma_simple_x86_decoder_init;
+	lzma_simple_x86_encoder_init;
+	lzma_stream_decoder_init;
+	lzma_stream_encoder_init;
+	lzma_strm_init;
+	lzma_tuklib_physmem;
+};
diff --git a/lib/liblzma/Versions.def b/lib/liblzma/Versions.def
new file mode 100644
index 0000000..1f65670
--- /dev/null
+++ b/lib/liblzma/Versions.def
@@ -0,0 +1,9 @@
+# $FreeBSD$
+
+XZ_5.0 {
+};
+
+
+XZprivate_1.0 {
+} XZ_5.0;
+
diff --git a/lib/liblzma/config.h b/lib/liblzma/config.h
new file mode 100644
index 0000000..223cb95
--- /dev/null
+++ b/lib/liblzma/config.h
@@ -0,0 +1,95 @@
+// $FreeBSD$
+#define ASSUME_RAM 128
+#define HAVE_CHECK_CRC32 1
+#define HAVE_CHECK_CRC64 1
+#define HAVE_CHECK_SHA256 1
+#define HAVE_DECL_PROGRAM_INVOCATION_NAME 0
+#define HAVE_DECODER 1
+#define HAVE_DECODER_ARM 1
+#define HAVE_DECODER_ARMTHUMB 1
+#define HAVE_DECODER_DELTA 1
+#define HAVE_DECODER_IA64 1
+#define HAVE_DECODER_LZMA1 1
+#define HAVE_DECODER_LZMA2 1
+#define HAVE_DECODER_POWERPC 1
+#define HAVE_DECODER_SPARC 1
+#define HAVE_DECODER_X86 1
+#define HAVE_DLFCN_H 1
+#define HAVE_ENCODER 1
+#define HAVE_ENCODER_ARM 1
+#define HAVE_ENCODER_ARMTHUMB 1
+#define HAVE_ENCODER_DELTA 1
+#define HAVE_ENCODER_IA64 1
+#define HAVE_ENCODER_LZMA1 1
+#define HAVE_ENCODER_LZMA2 1
+#define HAVE_ENCODER_POWERPC 1
+#define HAVE_ENCODER_SPARC 1
+#define HAVE_ENCODER_X86 1
+#define HAVE_FCNTL_H 1
+#define HAVE_FUTIMES 1
+#define HAVE_GETOPT_H 1
+#define HAVE_GETOPT_LONG 1
+#define HAVE_INTTYPES_H 1
+#define HAVE_LIMITS_H 1
+#define HAVE_MEMORY_H 1
+#define HAVE_MF_BT2 1
+#define HAVE_MF_BT3 1
+#define HAVE_MF_BT4 1
+#define HAVE_MF_HC3 1
+#define HAVE_MF_HC4 1
+#define HAVE_OPTRESET 1
+#define HAVE_PTHREAD 1
+#define HAVE_STDBOOL_H 1
+#define HAVE_STDINT_H 1
+#define HAVE_STDLIB_H 1
+#define HAVE_STRINGS_H 1
+#define HAVE_STRING_H 1
+#define HAVE_STRUCT_STAT_ST_ATIMESPEC_TV_NSEC 1
+#define HAVE_SYS_ENDIAN_H 1
+#define HAVE_SYS_PARAM_H 1
+#define HAVE_SYS_STAT_H 1
+#define HAVE_SYS_TIME_H 1
+#define HAVE_SYS_TYPES_H 1
+#define HAVE_UINTPTR_T 1
+#define HAVE_UNISTD_H 1
+#define HAVE_VISIBILITY 1
+#define HAVE__BOOL 1
+#define LT_OBJDIR ".libs/"
+#define NDEBUG 1
+#define PACKAGE "xz"
+#define PACKAGE_BUGREPORT "lasse.collin@tukaani.org"
+#define PACKAGE_NAME "XZ Utils"
+#define PACKAGE_STRING "XZ Utils 4.999.9beta"
+#define PACKAGE_TARNAME "xz"
+#define PACKAGE_URL "http://tukaani.org/xz/"
+#define PACKAGE_VERSION "4.999.9beta"
+#define SIZEOF_SIZE_T 8
+#define STDC_HEADERS 1
+#define TUKLIB_CPUCORES_SYSCONF 1
+#define TUKLIB_FAST_UNALIGNED_ACCESS 1
+#define TUKLIB_PHYSMEM_SYSCONF 1
+#ifndef _ALL_SOURCE
+# define _ALL_SOURCE 1
+#endif
+#ifndef _GNU_SOURCE
+# define _GNU_SOURCE 1
+#endif
+#ifndef _POSIX_PTHREAD_SEMANTICS
+# define _POSIX_PTHREAD_SEMANTICS 1
+#endif
+#ifndef _TANDEM_SOURCE
+# define _TANDEM_SOURCE 1
+#endif
+#ifndef __EXTENSIONS__
+# define __EXTENSIONS__ 1
+#endif
+#define VERSION "4.999.9beta"
+#if defined AC_APPLE_UNIVERSAL_BUILD
+# if defined __BIG_ENDIAN__
+#  define WORDS_BIGENDIAN 1
+# endif
+#else
+# ifndef WORDS_BIGENDIAN
+/* #  undef WORDS_BIGENDIAN */
+# endif
+#endif
-- 
cgit v1.1


From c00023af29c999b7ae1d2d84a75d96141fb83b1f Mon Sep 17 00:00:00 2001
From: kaiw <kaiw@FreeBSD.org>
Date: Tue, 18 May 2010 10:32:20 +0000
Subject: MFC r205728

  Merge improvements from kernel HID parser to the userland usbhid(3)
  parser.  This merge does not change any API and should not break any
  native or thirdparty applications.

  Changes include:

  * Merge multiple report ID support and other improvements from kernel
    HID parser.
  * Ignore rid argument in hid_start_parser, parse all the report items since
    we now support multiple report ID.
  * Skip report ID byte in hid_get_data() and set report ID byte in
    hid_set_data(), if report ID is non-zero.
  * Reimplement hid_get_report_id: instead get report id from uhid device
    (which is always 0), try parsing the report descriptor and return the
    first report ID encountered.

MFC r207812

  hid_get_data() now expects that the hid data passed in always contains
  the report ID byte. Thus we should not skip the the report ID byte in
  hid_interrupt().  Also, if HUP_KEYBOARD usage is an array, do not try
  to modify the 'data' pointer, instead, increase the hid_item_t field
  'pos' by 'report_size' before calling hid_get_data() during each
  iteration.
---
 lib/libusbhid/data.c   |  31 ++-
 lib/libusbhid/descr.c  |  22 +-
 lib/libusbhid/parse.c  | 536 ++++++++++++++++++++++++++++++-------------------
 lib/libusbhid/usage.c  |   1 +
 lib/libusbhid/usbhid.h |  63 +++---
 lib/libusbhid/usbvar.h |   4 +-
 6 files changed, 406 insertions(+), 251 deletions(-)

(limited to 'lib')

diff --git a/lib/libusbhid/data.c b/lib/libusbhid/data.c
index eb42422..3b90ac6 100644
--- a/lib/libusbhid/data.c
+++ b/lib/libusbhid/data.c
@@ -29,6 +29,7 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include <sys/param.h>
 #include <assert.h>
 #include <stdlib.h>
 #include "usbhid.h"
@@ -36,18 +37,27 @@ __FBSDID("$FreeBSD$");
 int32_t
 hid_get_data(const void *p, const hid_item_t *h)
 {
-	const unsigned char *buf;
-	unsigned int hpos;
-	unsigned int hsize;
-	int data;
+	const uint8_t *buf;
+	uint32_t hpos;
+	uint32_t hsize;
+	uint32_t data;
 	int i, end, offs;
 
 	buf = p;
+
+	/* Skip report ID byte. */
+	if (h->report_ID > 0)
+		buf++;
+
 	hpos = h->pos;			/* bit position of data */
 	hsize = h->report_size;		/* bit length of data */
 
+	/* Range check and limit */
 	if (hsize == 0)
 		return (0);
+	if (hsize > 32)
+		hsize = 32;
+
 	offs = hpos / 8;
 	end = (hpos + hsize) / 8 - offs;
 	data = 0;
@@ -70,15 +80,20 @@ hid_get_data(const void *p, const hid_item_t *h)
 void
 hid_set_data(void *p, const hid_item_t *h, int32_t data)
 {
-	unsigned char *buf;
-	unsigned int hpos;
-	unsigned int hsize;
+	uint8_t *buf;
+	uint32_t hpos;
+	uint32_t hsize;
 	uint32_t mask;
 	int i;
 	int end;
 	int offs;
 
 	buf = p;
+
+	/* Set report ID byte. */
+	if (h->report_ID > 0)
+		*buf++ = h->report_ID & 0xff;
+
 	hpos = h->pos;			/* bit position of data */
 	hsize = h->report_size;		/* bit length of data */
 
@@ -97,5 +112,5 @@ hid_set_data(void *p, const hid_item_t *h, int32_t data)
 
 	for (i = 0; i <= end; i++)
 		buf[offs + i] = (buf[offs + i] & (mask >> (i*8))) |
-			((data >> (i*8)) & 0xff);
+		    ((data >> (i*8)) & 0xff);
 }
diff --git a/lib/libusbhid/descr.c b/lib/libusbhid/descr.c
index 2ce1bf0..a5c033a 100644
--- a/lib/libusbhid/descr.c
+++ b/lib/libusbhid/descr.c
@@ -38,7 +38,6 @@ __FBSDID("$FreeBSD$");
 #include <unistd.h>
 #include <sys/time.h>
 #include <sys/ioctl.h>
-
 #include <dev/usb/usb_ioctl.h>
 
 #include "usbhid.h"
@@ -59,9 +58,30 @@ hid_set_immed(int fd, int enable)
 int
 hid_get_report_id(int fd)
 {
+	report_desc_t rep;
+	hid_data_t d;
+	hid_item_t h;
+	int kindset;
 	int temp = -1;
 	int ret;
 
+	if ((rep = hid_get_report_desc(fd)) == NULL)
+		goto use_ioctl;
+	kindset = 1 << hid_input | 1 << hid_output | 1 << hid_feature;
+	for (d = hid_start_parse(rep, kindset, 0); hid_get_item(d, &h); ) {
+		/* Return the first report ID we met. */
+		if (h.report_ID != 0) {
+			temp = h.report_ID;
+			break;
+		}
+	}
+	hid_end_parse(d);
+	hid_dispose_report_desc(rep);
+
+	if (temp > 0)
+		return (temp);
+
+use_ioctl:
 	ret = ioctl(fd, USB_GET_REPORT_ID, &temp);
 #ifdef HID_COMPAT7
 	if (ret < 0)
diff --git a/lib/libusbhid/parse.c b/lib/libusbhid/parse.c
index 3abc036..ce034b1 100644
--- a/lib/libusbhid/parse.c
+++ b/lib/libusbhid/parse.c
@@ -40,42 +40,43 @@ __FBSDID("$FreeBSD$");
 #include "usbhid.h"
 #include "usbvar.h"
 
-#define MAXUSAGE 100
-struct hid_data {
-	u_char *start;
-	u_char *end;
-	u_char *p;
-	hid_item_t cur;
-	unsigned int usages[MAXUSAGE];
-	int nusage;
-	int minset;
-	int logminsize;
-	int multi;
-	int multimax;
-	int kindset;
-	int reportid;
-
-	/*
-	 * The start of collection item has no report ID set, so save
-	 * it until we know the ID.
-	 */
-	hid_item_t savedcoll;
-	u_char hassavedcoll;
-	/*
-	 * Absolute data position (bits) for input/output/feature.
-	 *  Assumes that hid_input, hid_output and hid_feature have
-	 *  values 0, 1 and 2.
-	 */
-	unsigned int kindpos[3];
-};
+#define	MAXUSAGE 100
+#define	MAXPUSH 4
+#define	MAXID 64
 
-static int min(int x, int y) { return x < y ? x : y; }
+struct hid_pos_data {
+	int32_t rid;
+	uint32_t pos;
+};
 
-static int hid_get_item_raw(hid_data_t s, hid_item_t *h);
+struct hid_data {
+	const uint8_t *start;
+	const uint8_t *end;
+	const uint8_t *p;
+	struct hid_item cur[MAXPUSH];
+	struct hid_pos_data last_pos[MAXID];
+	int32_t usages_min[MAXUSAGE];
+	int32_t usages_max[MAXUSAGE];
+	int32_t usage_last;	/* last seen usage */
+	uint32_t loc_size;	/* last seen size */
+	uint32_t loc_count;	/* last seen count */
+	uint8_t	kindset;	/* we have 5 kinds so 8 bits are enough */
+	uint8_t	pushlevel;	/* current pushlevel */
+	uint8_t	ncount;		/* end usage item count */
+	uint8_t icount;		/* current usage item count */
+	uint8_t	nusage;		/* end "usages_min/max" index */
+	uint8_t	iusage;		/* current "usages_min/max" index */
+	uint8_t ousage;		/* current "usages_min/max" offset */
+	uint8_t	susage;		/* usage set flags */
+};
 
+/*------------------------------------------------------------------------*
+ *	hid_clear_local
+ *------------------------------------------------------------------------*/
 static void
 hid_clear_local(hid_item_t *c)
 {
+
 	c->usage = 0;
 	c->usage_minimum = 0;
 	c->usage_maximum = 0;
@@ -88,8 +89,61 @@ hid_clear_local(hid_item_t *c)
 	c->set_delimiter = 0;
 }
 
+static void
+hid_switch_rid(struct hid_data *s, struct hid_item *c, int32_t next_rID)
+{
+	uint8_t i;
+
+	/* check for same report ID - optimise */
+
+	if (c->report_ID == next_rID)
+		return;
+
+	/* save current position for current rID */
+
+	if (c->report_ID == 0) {
+		i = 0;
+	} else {
+		for (i = 1; i != MAXID; i++) {
+			if (s->last_pos[i].rid == c->report_ID)
+				break;
+			if (s->last_pos[i].rid == 0)
+				break;
+		}
+	}
+	if (i != MAXID) {
+		s->last_pos[i].rid = c->report_ID;
+		s->last_pos[i].pos = c->pos;
+	}
+
+	/* store next report ID */
+
+	c->report_ID = next_rID;
+
+	/* lookup last position for next rID */
+
+	if (next_rID == 0) {
+		i = 0;
+	} else {
+		for (i = 1; i != MAXID; i++) {
+			if (s->last_pos[i].rid == next_rID)
+				break;
+			if (s->last_pos[i].rid == 0)
+				break;
+		}
+	}
+	if (i != MAXID) {
+		s->last_pos[i].rid = next_rID;
+		c->pos = s->last_pos[i].pos;
+	} else
+		c->pos = 0;	/* Out of RID entries. */
+}
+
+/*------------------------------------------------------------------------*
+ *	hid_start_parse
+ *------------------------------------------------------------------------*/
 hid_data_t
-hid_start_parse(report_desc_t d, int kindset, int id)
+hid_start_parse(report_desc_t d, int kindset, int id __unused)
 {
 	struct hid_data *s;
 
@@ -98,213 +152,207 @@ hid_start_parse(report_desc_t d, int kindset, int id)
 	s->start = s->p = d->data;
 	s->end = d->data + d->size;
 	s->kindset = kindset;
-	s->reportid = id;
-	s->hassavedcoll = 0;
 	return (s);
 }
 
+/*------------------------------------------------------------------------*
+ *	hid_end_parse
+ *------------------------------------------------------------------------*/
 void
 hid_end_parse(hid_data_t s)
 {
-	while (s->cur.next) {
-		hid_item_t *hi = s->cur.next->next;
-		free(s->cur.next);
-		s->cur.next = hi;
-	}
+
+	if (s == NULL)
+		return;
+
 	free(s);
 }
 
-int
-hid_get_item(hid_data_t s, hid_item_t *h)
+/*------------------------------------------------------------------------*
+ *	get byte from HID descriptor
+ *------------------------------------------------------------------------*/
+static uint8_t
+hid_get_byte(struct hid_data *s, const uint16_t wSize)
 {
-	int r;
+	const uint8_t *ptr;
+	uint8_t retval;
 
-	for (;;) {
-		r = hid_get_item_raw(s, h);
-		if (r <= 0)
-			break;
-		if (h->report_ID == s->reportid || s->reportid == -1)
-			break;
-	}
-	return (r);
+	ptr = s->p;
+
+	/* check if end is reached */
+	if (ptr == s->end)
+		return (0);
+
+	/* read out a byte */
+	retval = *ptr;
+
+	/* check if data pointer can be advanced by "wSize" bytes */
+	if ((s->end - ptr) < wSize)
+		ptr = s->end;
+	else
+		ptr += wSize;
+
+	/* update pointer */
+	s->p = ptr;
+
+	return (retval);
 }
 
-#define REPORT_SAVED_COLL \
-	do { \
-		if (s->hassavedcoll) { \
-			*h = s->savedcoll; \
-			h->report_ID = c->report_ID; \
-			s->hassavedcoll = 0; \
-			return (1); \
-		} \
-	} while(/*LINTED*/ 0)
-
-static int
-hid_get_item_raw(hid_data_t s, hid_item_t *h)
+/*------------------------------------------------------------------------*
+ *	hid_get_item
+ *------------------------------------------------------------------------*/
+int
+hid_get_item(hid_data_t s, hid_item_t *h)
 {
 	hid_item_t *c;
-	unsigned int bTag = 0, bType = 0, bSize;
-	unsigned char *data;
-	int dval;
-	unsigned char *p;
-	hid_item_t *hi;
-	hid_item_t nc;
-	int i;
-	hid_kind_t retkind;
+	unsigned int bTag, bType, bSize;
+	uint32_t oldpos;
+	int32_t mask;
+	int32_t dval;
+
+	if (s == NULL)
+		return (0);
 
-	c = &s->cur;
+	c = &s->cur[s->pushlevel];
 
  top:
-	if (s->multimax) {
-		REPORT_SAVED_COLL;
-		if (c->logical_minimum >= c->logical_maximum) {
-			if (s->logminsize == 1)
-				c->logical_minimum =(int8_t)c->logical_minimum;
-			else if (s->logminsize == 2)
-				c->logical_minimum =(int16_t)c->logical_minimum;
+	/* check if there is an array of items */
+	if (s->icount < s->ncount) {
+		/* get current usage */
+		if (s->iusage < s->nusage) {
+			dval = s->usages_min[s->iusage] + s->ousage;
+			c->usage = dval;
+			s->usage_last = dval;
+			if (dval == s->usages_max[s->iusage]) {
+				s->iusage ++;
+				s->ousage = 0;
+			} else {
+				s->ousage ++;
+			}
+		} else {
+			/* Using last usage */
+			dval = s->usage_last;
 		}
-		if (s->multi < s->multimax) {
-			c->usage = s->usages[min(s->multi, s->nusage-1)];
-			s->multi++;
+		s->icount ++;
+		/* 
+		 * Only copy HID item, increment position and return
+		 * if correct kindset!
+		 */
+		if (s->kindset & (1 << c->kind)) {
 			*h = *c;
-			/*
-			 * 'multimax' is only non-zero if the current
-                         *  item kind is input/output/feature
-			 */
-			h->pos = s->kindpos[c->kind];
-			s->kindpos[c->kind] += c->report_size;
-			h->next = 0;
+			c->pos += c->report_size * c->report_count;
 			return (1);
-		} else {
-			c->report_count = s->multimax;
-			s->multimax = 0;
-			s->nusage = 0;
-			hid_clear_local(c);
 		}
 	}
-	for (;;) {
-		p = s->p;
-		if (p >= s->end)
-			return (0);
 
-		bSize = *p++;
+	/* reset state variables */
+	s->icount = 0;
+	s->ncount = 0;
+	s->iusage = 0;
+	s->nusage = 0;
+	s->susage = 0;
+	s->ousage = 0;
+	hid_clear_local(c);
+
+	/* get next item */
+	while (s->p != s->end) {
+
+		bSize = hid_get_byte(s, 1);
 		if (bSize == 0xfe) {
 			/* long item */
-			bSize = *p++;
-			bSize |= *p++ << 8;
-			bTag = *p++;
-			data = p;
-			p += bSize;
+			bSize = hid_get_byte(s, 1);
+			bSize |= hid_get_byte(s, 1) << 8;
+			bTag = hid_get_byte(s, 1);
+			bType = 0xff;	/* XXX what should it be */
 		} else {
 			/* short item */
 			bTag = bSize >> 4;
 			bType = (bSize >> 2) & 3;
 			bSize &= 3;
-			if (bSize == 3) bSize = 4;
-			data = p;
-			p += bSize;
+			if (bSize == 3)
+				bSize = 4;
 		}
-		s->p = p;
-		/*
-		 * The spec is unclear if the data is signed or unsigned.
-		 */
+
 		switch(bSize) {
 		case 0:
 			dval = 0;
+			mask = 0;
 			break;
 		case 1:
-			dval = *data++;
+			dval = (int8_t)hid_get_byte(s, 1);
+			mask = 0xFF;
 			break;
 		case 2:
-			dval = *data++;
-			dval |= *data++ << 8;
+			dval = hid_get_byte(s, 1);
+			dval |= hid_get_byte(s, 1) << 8;
+			dval = (int16_t)dval;
+			mask = 0xFFFF;
 			break;
 		case 4:
-			dval = *data++;
-			dval |= *data++ << 8;
-			dval |= *data++ << 16;
-			dval |= *data++ << 24;
+			dval = hid_get_byte(s, 1);
+			dval |= hid_get_byte(s, 1) << 8;
+			dval |= hid_get_byte(s, 1) << 16;
+			dval |= hid_get_byte(s, 1) << 24;
+			mask = 0xFFFFFFFF;
 			break;
 		default:
-			return (-1);
+			dval = hid_get_byte(s, bSize);
+			continue;
 		}
 
 		switch (bType) {
-		case 0:			/* Main */
+		case 0:		/* Main */
 			switch (bTag) {
-			case 8:		/* Input */
-				retkind = hid_input;
-			ret:
-				if (!(s->kindset & (1 << retkind))) {
-					/* Drop the items of this kind */
-					s->nusage = 0;
-					continue;
-				}
-				c->kind = retkind;
+			case 8:	/* Input */
+				c->kind = hid_input;
 				c->flags = dval;
+		ret:
+				c->report_count = s->loc_count;
+				c->report_size = s->loc_size;
+
 				if (c->flags & HIO_VARIABLE) {
-					s->multimax = c->report_count;
-					s->multi = 0;
+					/* range check usage count */
+					if (c->report_count > 255) {
+						s->ncount = 255;
+					} else
+						s->ncount = c->report_count;
+
+					/* 
+					 * The "top" loop will return
+					 * one and one item:
+					 */
 					c->report_count = 1;
-					if (s->minset) {
-						for (i = c->usage_minimum;
-						     i <= c->usage_maximum;
-						     i++) {
-							s->usages[s->nusage] = i;
-							if (s->nusage < MAXUSAGE-1)
-								s->nusage++;
-						}
-						c->usage_minimum = 0;
-						c->usage_maximum = 0;
-						s->minset = 0;
-					}
-					goto top;
 				} else {
-					if (s->minset)
-						c->usage = c->usage_minimum;
-					*h = *c;
-					h->next = 0;
-					h->pos = s->kindpos[c->kind];
-					s->kindpos[c->kind] +=
-					    c->report_size * c->report_count;
-					hid_clear_local(c);
-					s->minset = 0;
-					return (1);
+					s->ncount = 1;
 				}
-			case 9:		/* Output */
-				retkind = hid_output;
+				goto top;
+
+			case 9:	/* Output */
+				c->kind = hid_output;
+				c->flags = dval;
 				goto ret;
 			case 10:	/* Collection */
 				c->kind = hid_collection;
 				c->collection = dval;
 				c->collevel++;
-				nc = *c;
-				hid_clear_local(c);
-				/*c->report_ID = NO_REPORT_ID;*/
-				s->nusage = 0;
-				if (s->hassavedcoll) {
-					*h = s->savedcoll;
-					h->report_ID = nc.report_ID;
-					s->savedcoll = nc;
-					return (1);
-				} else {
-					s->hassavedcoll = 1;
-					s->savedcoll = nc;
-				}
-				break;
+				c->usage = s->usage_last;
+				*h = *c;
+				return (1);
 			case 11:	/* Feature */
-				retkind = hid_feature;
+				c->kind = hid_feature;
+				c->flags = dval;
 				goto ret;
 			case 12:	/* End collection */
-				REPORT_SAVED_COLL;
 				c->kind = hid_endcollection;
+				if (c->collevel == 0) {
+					/* Invalid end collection. */
+					return (0);
+				}
 				c->collevel--;
 				*h = *c;
-				/*hid_clear_local(c);*/
-				s->nusage = 0;
 				return (1);
 			default:
-				return (-2);
+				break;
 			}
 			break;
 
@@ -315,13 +363,12 @@ hid_get_item_raw(hid_data_t s, hid_item_t *h)
 				break;
 			case 1:
 				c->logical_minimum = dval;
-				s->logminsize = bSize;
 				break;
 			case 2:
 				c->logical_maximum = dval;
 				break;
 			case 3:
-				c->physical_maximum = dval;
+				c->physical_minimum = dval;
 				break;
 			case 4:
 				c->physical_maximum = dval;
@@ -333,45 +380,97 @@ hid_get_item_raw(hid_data_t s, hid_item_t *h)
 				c->unit = dval;
 				break;
 			case 7:
-				c->report_size = dval;
+				/* mask because value is unsigned */
+				s->loc_size = dval & mask;
 				break;
 			case 8:
-				c->report_ID = dval;
-				s->kindpos[hid_input] =
-				    s->kindpos[hid_output] =
-				    s->kindpos[hid_feature] = 0;
+				hid_switch_rid(s, c, dval);
 				break;
 			case 9:
-				c->report_count = dval;
+				/* mask because value is unsigned */
+				s->loc_count = dval & mask;
 				break;
-			case 10: /* Push */
-				hi = malloc(sizeof *hi);
-				*hi = s->cur;
-				c->next = hi;
+			case 10:	/* Push */
+				s->pushlevel ++;
+				if (s->pushlevel < MAXPUSH) {
+					s->cur[s->pushlevel] = *c;
+					/* store size and count */
+					c->report_size = s->loc_size;
+					c->report_count = s->loc_count;
+					/* update current item pointer */
+					c = &s->cur[s->pushlevel];
+				}
 				break;
-			case 11: /* Pop */
-				hi = c->next;
-				s->cur = *hi;
-				free(hi);
+			case 11:	/* Pop */
+				s->pushlevel --;
+				if (s->pushlevel < MAXPUSH) {
+					/* preserve position */
+					oldpos = c->pos;
+					c = &s->cur[s->pushlevel];
+					/* restore size and count */
+					s->loc_size = c->report_size;
+					s->loc_count = c->report_count;
+					/* set default item location */
+					c->pos = oldpos;
+					c->report_size = 0;
+					c->report_count = 0;
+				}
 				break;
 			default:
-				return (-3);
+				break;
 			}
 			break;
 		case 2:		/* Local */
 			switch (bTag) {
 			case 0:
-				c->usage = c->_usage_page | dval;
-				if (s->nusage < MAXUSAGE)
-					s->usages[s->nusage++] = c->usage;
+				if (bSize != 4)
+					dval = (dval & mask) | c->_usage_page;
+
+				/* set last usage, in case of a collection */
+				s->usage_last = dval;
+
+				if (s->nusage < MAXUSAGE) {
+					s->usages_min[s->nusage] = dval;
+					s->usages_max[s->nusage] = dval;
+					s->nusage ++;
+				}
 				/* else XXX */
+
+				/* clear any pending usage sets */
+				s->susage = 0;
 				break;
 			case 1:
-				s->minset = 1;
-				c->usage_minimum = c->_usage_page | dval;
-				break;
+				s->susage |= 1;
+
+				if (bSize != 4)
+					dval = (dval & mask) | c->_usage_page;
+				c->usage_minimum = dval;
+
+				goto check_set;
 			case 2:
-				c->usage_maximum = c->_usage_page | dval;
+				s->susage |= 2;
+
+				if (bSize != 4)
+					dval = (dval & mask) | c->_usage_page;
+				c->usage_maximum = dval;
+
+			check_set:
+				if (s->susage != 3)
+					break;
+
+				/* sanity check */
+				if ((s->nusage < MAXUSAGE) &&
+				    (c->usage_minimum <= c->usage_maximum)) {
+					/* add usage range */
+					s->usages_min[s->nusage] = 
+					    c->usage_minimum;
+					s->usages_max[s->nusage] = 
+					    c->usage_maximum;
+					s->nusage ++;
+				}
+				/* else XXX */
+
+				s->susage = 0;
 				break;
 			case 3:
 				c->designator_index = dval;
@@ -395,40 +494,63 @@ hid_get_item_raw(hid_data_t s, hid_item_t *h)
 				c->set_delimiter = dval;
 				break;
 			default:
-				return (-4);
+				break;
 			}
 			break;
 		default:
-			return (-5);
+			break;
 		}
 	}
+	return (0);
 }
 
 int
 hid_report_size(report_desc_t r, enum hid_kind k, int id)
 {
 	struct hid_data *d;
-	hid_item_t h;
-	int size;
+	struct hid_item h;
+	uint32_t temp;
+	uint32_t hpos;
+	uint32_t lpos;
+
+	hpos = 0;
+	lpos = 0xFFFFFFFF;
 
 	memset(&h, 0, sizeof h);
-	size = 0;
-	for (d = hid_start_parse(r, 1<<k, id); hid_get_item(d, &h); ) {
+	for (d = hid_start_parse(r, 1 << k, id); hid_get_item(d, &h); ) {
 		if (h.report_ID == id && h.kind == k) {
-			size = d->kindpos[k];
+			/* compute minimum */
+			if (lpos > h.pos)
+				lpos = h.pos;
+			/* compute end position */
+			temp = h.pos + (h.report_size * h.report_count);
+			/* compute maximum */
+			if (hpos < temp)
+				hpos = temp;
 		}
 	}
 	hid_end_parse(d);
-	return ((size + 7) / 8);
+
+	/* safety check - can happen in case of currupt descriptors */
+	if (lpos > hpos)
+		temp = 0;
+	else
+		temp = hpos - lpos;
+
+	if (id)
+		temp += 8;
+
+	/* return length in bytes rounded up */
+	return ((temp + 7) / 8);
 }
 
 int
 hid_locate(report_desc_t desc, unsigned int u, enum hid_kind k,
 	   hid_item_t *h, int id)
 {
-	hid_data_t d;
+	struct hid_data *d;
 
-	for (d = hid_start_parse(desc, 1<<k, id); hid_get_item(d, h); ) {
+	for (d = hid_start_parse(desc, 1 << k, id); hid_get_item(d, h); ) {
 		if (h->kind == k && !(h->flags & HIO_CONST) && h->usage == u) {
 			hid_end_parse(d);
 			return (1);
diff --git a/lib/libusbhid/usage.c b/lib/libusbhid/usage.c
index 280ef69..eeff818 100644
--- a/lib/libusbhid/usage.c
+++ b/lib/libusbhid/usage.c
@@ -29,6 +29,7 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include <sys/param.h>
 #include <assert.h>
 #include <ctype.h>
 #include <err.h>
diff --git a/lib/libusbhid/usbhid.h b/lib/libusbhid/usbhid.h
index 90f0f03..df5f7c1 100644
--- a/lib/libusbhid/usbhid.h
+++ b/lib/libusbhid/usbhid.h
@@ -29,7 +29,7 @@
  *
  */
 
-#include <sys/cdefs.h>
+#include <sys/types.h>
 #include <sys/types.h>
 
 typedef struct report_desc *report_desc_t;
@@ -37,45 +37,41 @@ typedef struct report_desc *report_desc_t;
 typedef struct hid_data *hid_data_t;
 
 typedef enum hid_kind {
-	hid_input = 0,
-	hid_output = 1,
-	hid_feature = 2,
-	hid_collection,
-	hid_endcollection
+	hid_input, hid_output, hid_feature, hid_collection, hid_endcollection
 } hid_kind_t;
 
 typedef struct hid_item {
 	/* Global */
-	unsigned int _usage_page;
-	int logical_minimum;
-	int logical_maximum;
-	int physical_minimum;
-	int physical_maximum;
-	int unit_exponent;
-	int unit;
-	int report_size;
-	int report_ID;
+	uint32_t _usage_page;
+	int32_t logical_minimum;
+	int32_t logical_maximum;
+	int32_t physical_minimum;
+	int32_t physical_maximum;
+	int32_t unit_exponent;
+	int32_t unit;
+	int32_t report_size;
+	int32_t report_ID;
 #define NO_REPORT_ID 0
-	int report_count;
+	int32_t report_count;
 	/* Local */
-	unsigned int usage;
-	int usage_minimum;
-	int usage_maximum;
-	int designator_index;
-	int designator_minimum;
-	int designator_maximum;
-	int string_index;
-	int string_minimum;
-	int string_maximum;
-	int set_delimiter;
+	uint32_t usage;
+	int32_t usage_minimum;
+	int32_t usage_maximum;
+	int32_t designator_index;
+	int32_t designator_minimum;
+	int32_t designator_maximum;
+	int32_t string_index;
+	int32_t string_minimum;
+	int32_t string_maximum;
+	int32_t set_delimiter;
 	/* Misc */
-	int collection;
-	int collevel;
+	int32_t collection;
+	int	collevel;
 	enum hid_kind kind;
-	unsigned int flags;
-	/* Absolute data position (bits) */
-	unsigned int pos;
-	/* */
+	uint32_t flags;
+	/* Location */
+	uint32_t pos;
+	/* unused */
 	struct hid_item *next;
 } hid_item_t;
 
@@ -96,7 +92,8 @@ hid_data_t hid_start_parse(report_desc_t d, int kindset, int id);
 void hid_end_parse(hid_data_t s);
 int hid_get_item(hid_data_t s, hid_item_t *h);
 int hid_report_size(report_desc_t d, enum hid_kind k, int id);
-int hid_locate(report_desc_t d, unsigned int usage, enum hid_kind k, hid_item_t *h, int id);
+int hid_locate(report_desc_t d, unsigned int usage, enum hid_kind k,
+    hid_item_t *h, int id);
 
 /* Conversion to/from usage names, usage.c: */
 const char *hid_usage_page(int i);
diff --git a/lib/libusbhid/usbvar.h b/lib/libusbhid/usbvar.h
index d437005..9605106 100644
--- a/lib/libusbhid/usbvar.h
+++ b/lib/libusbhid/usbvar.h
@@ -30,8 +30,8 @@
  */
 
 struct report_desc {
-	unsigned int size;
-	unsigned char data[1];
+	uint32_t size;
+	uint8_t data[1];
 };
 
 /* internal backwards compatibility functions */
-- 
cgit v1.1


From 66f9b83110b17ba68a936b1076cc39796bdbad2d Mon Sep 17 00:00:00 2001
From: gordon <gordon@FreeBSD.org>
Date: Wed, 19 May 2010 22:03:45 +0000
Subject: MFC r207981:  Fix a bug due to a type conversion from 64 to 32 bits.
 The side effect of  this type conversion is the high bits which were used to
 indicate if a  special character was a literal or special were dropped. As a
 result, all  special character were treated as special, even if they were
 supposed to  be literals.

Approved by:	mentor (wes@)
---
 lib/libc/gen/glob.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/gen/glob.c b/lib/libc/gen/glob.c
index 32af5db..c3d9f08 100644
--- a/lib/libc/gen/glob.c
+++ b/lib/libc/gen/glob.c
@@ -433,9 +433,9 @@ static int
 glob0(const Char *pattern, glob_t *pglob, size_t *limit)
 {
 	const Char *qpatnext;
-	int c, err;
+	int err;
 	size_t oldpathc;
-	Char *bufnext, patbuf[MAXPATHLEN];
+	Char *bufnext, c, patbuf[MAXPATHLEN];
 
 	qpatnext = globtilde(pattern, patbuf, MAXPATHLEN, pglob);
 	oldpathc = pglob->gl_pathc;
-- 
cgit v1.1


From c6cf476a04a39801e182bd55572c1f23bdc6860f Mon Sep 17 00:00:00 2001
From: trasz <trasz@FreeBSD.org>
Date: Thu, 20 May 2010 18:45:07 +0000
Subject: MFC r208033:

Make it possible to actually use NFSv4 permission bits with acl_set_perm(3)
and acl_delete_perm(3).  It went undetected, because neither setfacl(1)
nor Samba use this routines.  D'oh.
---
 lib/libc/posix1e/acl_perm.c | 52 ++++++++++++++++++++++++++++-----------------
 1 file changed, 32 insertions(+), 20 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/posix1e/acl_perm.c b/lib/libc/posix1e/acl_perm.c
index 37d29d6..b5108ca 100644
--- a/lib/libc/posix1e/acl_perm.c
+++ b/lib/libc/posix1e/acl_perm.c
@@ -35,6 +35,20 @@ __FBSDID("$FreeBSD$");
 #include <errno.h>
 #include <string.h>
 
+static int
+_perm_is_invalid(acl_perm_t perm)
+{
+
+	/* Check if more than a single bit is set. */
+	if ((perm & -perm) == perm &&
+	    (perm & (ACL_POSIX1E_BITS | ACL_NFS4_PERM_BITS)) == perm)
+		return (0);
+
+	errno = EINVAL;
+
+	return (1);
+}
+
 /*
  * acl_add_perm() (23.4.1): add the permission contained in perm to the
  * permission set permset_d
@@ -43,18 +57,17 @@ int
 acl_add_perm(acl_permset_t permset_d, acl_perm_t perm)
 {
 
-	if (permset_d) {
-		switch(perm) {
-		case ACL_READ:
-		case ACL_WRITE:
-		case ACL_EXECUTE:
-			*permset_d |= perm;
-			return (0);
-		}
+	if (permset_d == NULL) {
+		errno = EINVAL;
+		return (-1);
 	}
 
-	errno = EINVAL;
-	return (-1);
+	if (_perm_is_invalid(perm))
+		return (-1);
+
+	*permset_d |= perm;
+
+	return (0);
 }
 
 /*
@@ -83,16 +96,15 @@ int
 acl_delete_perm(acl_permset_t permset_d, acl_perm_t perm)
 {
 
-	if (permset_d) {
-		switch(perm) {
-		case ACL_READ:
-		case ACL_WRITE:
-		case ACL_EXECUTE:
-			*permset_d &= ~(perm & ACL_PERM_BITS);
-			return (0);
-		}
+	if (permset_d == NULL) {
+		errno = EINVAL;
+		return (-1);
 	}
 
-	errno = EINVAL;
-	return (-1);
+	if (_perm_is_invalid(perm))
+		return (-1);
+
+	*permset_d &= ~perm;
+
+	return (0);
 }
-- 
cgit v1.1


From 98afbc1457cefd1e8de545be6151baacadbf37dc Mon Sep 17 00:00:00 2001
From: trasz <trasz@FreeBSD.org>
Date: Thu, 20 May 2010 18:46:11 +0000
Subject: MFC r208034:

Make branding less intrusive - in acl_set(3), in case ACL brand
is ACL_BRAND_UNKNOWN, do what the programmer says instead of failing.
---
 lib/libc/posix1e/acl_branding.c | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'lib')

diff --git a/lib/libc/posix1e/acl_branding.c b/lib/libc/posix1e/acl_branding.c
index 9839477..84b0063 100644
--- a/lib/libc/posix1e/acl_branding.c
+++ b/lib/libc/posix1e/acl_branding.c
@@ -129,6 +129,9 @@ _acl_type_not_valid_for_acl(const acl_t acl, acl_type_t type)
 		if (type == ACL_TYPE_ACCESS || type == ACL_TYPE_DEFAULT)
 			return (0);
 		break;
+
+	case ACL_BRAND_UNKNOWN:
+		return (0);
 	}
 
 	return (-1);
-- 
cgit v1.1


From f70ec033b6d3a079e307f66ef62c8b1274671dbf Mon Sep 17 00:00:00 2001
From: marius <marius@FreeBSD.org>
Date: Sun, 23 May 2010 11:18:05 +0000
Subject: MFC: r208341

Update to a config.h created by a file 5.03 configure script. This causes
file.1 to contain the correct version number and SIZEOF_LONG_LONG to be
defined as appropriate, which is crucial for 64-bit big-endian ELF files
to be handled correctly on big-endian systems.

PR:		146387
Reviewed by:	delphij
---
 lib/libmagic/config.h | 95 ++++++++++++++++++++++++++++++---------------------
 1 file changed, 57 insertions(+), 38 deletions(-)

(limited to 'lib')

diff --git a/lib/libmagic/config.h b/lib/libmagic/config.h
index 1e97361..8574ddc 100644
--- a/lib/libmagic/config.h
+++ b/lib/libmagic/config.h
@@ -1,18 +1,18 @@
 /* $FreeBSD$ */
 
 /* config.h.  Generated from config.h.in by configure.  */
-/* config.h.in.  Generated from configure.in by autoheader.  */
+/* config.h.in.  Generated from configure.ac by autoheader.  */
 
-/* Use the builtin ELF recognition code */
+/* Define in built-in ELF support is used */
 #define BUILTIN_ELF 1
 
-/* Recognize ELF core files */
+/* Define for ELF core file support */
 #define ELFCORE 1
 
 /* Define to 1 if you have the `asprintf' function. */
 #define HAVE_ASPRINTF 1
 
-/* */
+/* HAVE_DAYLIGHT */
 /* #undef HAVE_DAYLIGHT */
 
 /* Define to 1 if you have the <dlfcn.h> header file. */
@@ -51,9 +51,6 @@
 /* Define to 1 if you have the <locale.h> header file. */
 #define HAVE_LOCALE_H 1
 
-/* */
-#define HAVE_LONG_LONG 1
-
 /* Define to 1 if you have the `mbrtowc' function. */
 #define HAVE_MBRTOWC 1
 
@@ -69,9 +66,6 @@
 /* Define to 1 if you have the `mmap' function. */
 #define HAVE_MMAP 1
 
-/* Define to 1 if you have the `snprintf' function. */
-#define HAVE_SNPRINTF 1
-
 /* Define to 1 if you have the <stdint.h> header file. */
 #define HAVE_STDINT_H 1
 
@@ -102,13 +96,12 @@
 /* Define to 1 if you have the `strtoul' function. */
 #define HAVE_STRTOUL 1
 
+/* HAVE_STRUCT_OPTION */
+#define HAVE_STRUCT_OPTION 1
+
 /* Define to 1 if `st_rdev' is member of `struct stat'. */
 #define HAVE_STRUCT_STAT_ST_RDEV 1
 
-/* Define to 1 if your `struct stat' has `st_rdev'. Deprecated, use
-   `HAVE_STRUCT_STAT_ST_RDEV' instead. */
-#define HAVE_ST_RDEV 1
-
 /* Define to 1 if `tm_gmtoff' is member of `struct tm'. */
 #define HAVE_STRUCT_TM_TM_GMTOFF 1
 
@@ -133,7 +126,7 @@
 /* Define to 1 if you have <sys/wait.h> that is POSIX.1 compatible. */
 #define HAVE_SYS_WAIT_H 1
 
-/* */
+/* HAVE_TM_ISDST */
 #define HAVE_TM_ISDST 1
 
 /* HAVE_TM_ZONE */
@@ -169,9 +162,6 @@
 /* Define to 1 if you have the `vasprintf' function. */
 #define HAVE_VASPRINTF 1
 
-/* Define to 1 if you have the `vsnprintf' function. */
-#define HAVE_VSNPRINTF 1
-
 /* Define to 1 if you have the <wchar.h> header file. */
 #define HAVE_WCHAR_H 1
 
@@ -199,34 +189,22 @@
 #define PACKAGE "file"
 
 /* Define to the address where bug reports for this package should be sent. */
-#define PACKAGE_BUGREPORT ""
+#define PACKAGE_BUGREPORT "christos@astron.com"
 
 /* Define to the full name of this package. */
-#define PACKAGE_NAME ""
+#define PACKAGE_NAME "file"
 
 /* Define to the full name and version of this package. */
-#define PACKAGE_STRING ""
+#define PACKAGE_STRING "file 5.03"
 
 /* Define to the one symbol short name of this package. */
-#define PACKAGE_TARNAME ""
+#define PACKAGE_TARNAME "file"
 
 /* Define to the version of this package. */
-#define PACKAGE_VERSION VERSION
-
-/* */
-#define SIZEOF_INT64_T 8
+#define PACKAGE_VERSION "5.03"
 
-/* */
-#define SIZEOF_UINT16_T 2
-
-/* */
-#define SIZEOF_UINT32_T 4
-
-/* */
-#define SIZEOF_UINT64_T 8
-
-/* */
-#define SIZEOF_UINT8_T 1
+/* The size of `long long', as computed by sizeof. */
+#define SIZEOF_LONG_LONG 8
 
 /* Define to 1 if you have the ANSI C header files. */
 #define STDC_HEADERS 1
@@ -235,7 +213,7 @@
 /* #undef TM_IN_SYS_TIME */
 
 /* Version number of package */
-#define VERSION "5.00"
+#define VERSION "5.03"
 
 /* Number of bits in a file offset, on hosts where this is settable. */
 /* #undef _FILE_OFFSET_BITS */
@@ -247,8 +225,49 @@
 #endif
 #endif
 
+/* Define to 1 to make fseeko visible on some hosts (e.g. glibc 2.2). */
+/* #undef _LARGEFILE_SOURCE */
+
 /* Define for large files, on AIX-style hosts. */
 /* #undef _LARGE_FILES */
 
 /* Define to empty if `const' does not conform to ANSI C. */
 /* #undef const */
+
+/* Define to a type if <wchar.h> does not define. */
+/* #undef mbstate_t */
+
+/* Define to `long int' if <sys/types.h> does not define. */
+/* #undef off_t */
+
+/* Define to `unsigned int' if <sys/types.h> does not define. */
+/* #undef size_t */
+
+
+#ifndef HAVE_UINT8_T
+typedef unsigned char uint8_t;
+#endif
+#ifndef HAVE_UINT16_T
+typedef unsigned short uint16_t;
+#endif
+#ifndef HAVE_UINT32_T
+typedef unsigned int uint32_t;
+#endif
+#ifndef HAVE_INT32_T
+typedef int int32_t;
+#endif
+#ifndef HAVE_UINT64_T
+#if SIZEOF_LONG_LONG == 8
+typedef unsigned long long uint64_t;
+#else
+typedef unsigned long uint64_t;
+#endif
+#endif
+#ifndef HAVE_INT64_T
+#if SIZEOF_LONG_LONG == 8
+typedef long long int64_t;
+#else
+typedef long int64_t;
+#endif
+#endif
+
-- 
cgit v1.1


From 53d0fa35580ecc3df5f178617ddf8d12849a0690 Mon Sep 17 00:00:00 2001
From: rstone <rstone@FreeBSD.org>
Date: Mon, 24 May 2010 19:42:27 +0000
Subject: MFC r207482

 When configuring hwpmc to use the EXT_SNOOP event, only send a default
 cachestate qualifier on the Atom processor.  Other Intel processors do not
 accept a cachestate qualifier and currently hwpmc will return EINVAL if you
 try to use the EXT_SNOOP event on those processors

Approved by:	emaste (mentor)
---
 lib/libpmc/libpmc.c | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libpmc/libpmc.c b/lib/libpmc/libpmc.c
index 929ca73..551fe12 100644
--- a/lib/libpmc/libpmc.c
+++ b/lib/libpmc/libpmc.c
@@ -716,9 +716,16 @@ iap_allocate_pmc(enum pmc_event pe, char *ctrspec,
 	case PMC_EV_IAP_EVENT_40H: /* Core */
 	case PMC_EV_IAP_EVENT_41H: /* Core */
 	case PMC_EV_IAP_EVENT_42H: /* Core, Core2, Atom */
-	case PMC_EV_IAP_EVENT_77H: /* Core */
 		if (cachestate == 0)
 			cachestate = (0xF << 8);
+		break;
+	case PMC_EV_IAP_EVENT_77H: /* Atom */
+		/* IAP_EVENT_77H only accepts a cachestate qualifier on the
+		 * Atom processor
+		 */
+		if(cpu_info.pm_cputype == PMC_CPU_INTEL_ATOM && cachestate == 0)
+			cachestate = (0xF << 8);
+	    break;
 	default:
 		break;
 	}
-- 
cgit v1.1


From cecdfcbeea0822ed4b1ce847da15efb757dfa5d5 Mon Sep 17 00:00:00 2001
From: cperciva <cperciva@FreeBSD.org>
Date: Thu, 27 May 2010 03:15:04 +0000
Subject: Change the current working directory to be inside the jail created by
 the jail(8) command. [10:04]

Fix a one-NUL-byte buffer overflow in libopie. [10:05]

Correctly sanity-check a buffer length in nfs mount. [10:06]

Approved by:	so (cperciva)
Approved by:	re (kensmith)
Security:	FreeBSD-SA-10:04.jail
Security:	FreeBSD-SA-10:05.opie
Security:	FreeBSD-SA-10:06.nfsclient
---
 lib/libc/sys/mount.2 | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libc/sys/mount.2 b/lib/libc/sys/mount.2
index 6ce2d4d..3d48f41 100644
--- a/lib/libc/sys/mount.2
+++ b/lib/libc/sys/mount.2
@@ -107,7 +107,7 @@ This restriction can be removed by setting the
 .Va vfs.usermount
 .Xr sysctl 8
 variable
-to a non-zero value.
+to a non-zero value; see the BUGS section for more information.
 .Pp
 The following
 .Fa flags
@@ -370,3 +370,10 @@ functions appeared in
 .At v6 .
 .Sh BUGS
 Some of the error codes need translation to more obvious messages.
+.Pp
+Allowing untrusted users to mount arbitrary media, e.g. by enabling
+.Va vfs.usermount ,
+should not be considered safe.
+Most file systems in
+.Fx
+were not built to safeguard against malicious devices.
-- 
cgit v1.1


From 3c153e0b018436cca9346857ea8a99ab379b01ea Mon Sep 17 00:00:00 2001
From: uqs <uqs@FreeBSD.org>
Date: Sun, 30 May 2010 08:38:41 +0000
Subject: MFC r208493:    These features will first appear in 8.1, soon to be
 released

Approved by:	re (kib)
---
 lib/libc/compat-43/sigpause.2 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libc/compat-43/sigpause.2 b/lib/libc/compat-43/sigpause.2
index bf3cf0b..0bea04e 100644
--- a/lib/libc/compat-43/sigpause.2
+++ b/lib/libc/compat-43/sigpause.2
@@ -237,5 +237,5 @@ function appeared in
 .Bx 4.2
 and has been deprecated.
 All other functions appeared in
-.Fx 9.0
+.Fx 8.1
 and were deprecated before being implemented.
-- 
cgit v1.1


From ae44b4322ea606485db5239ff83cdf784e472941 Mon Sep 17 00:00:00 2001
From: rwatson <rwatson@FreeBSD.org>
Date: Tue, 1 Jun 2010 14:09:35 +0000
Subject: Merge r204435 from head to stable/8:

  Fix typo in comment.

Approved by:	re (bz)
---
 lib/libkvm/kvm_vnet.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/libkvm/kvm_vnet.c b/lib/libkvm/kvm_vnet.c
index d192c67..a5c8aad 100644
--- a/lib/libkvm/kvm_vnet.c
+++ b/lib/libkvm/kvm_vnet.c
@@ -117,8 +117,8 @@ _kvm_vnet_selectpid(kvm_t *kd, pid_t pid)
 	}
 
 	/*
-	 * First, find the process for this pid.  If we are workig on a dump,
-	 * either locate the thread dumptid is refering to or proc0.
+	 * First, find the process for this pid.  If we are working on a
+	 * dump, either locate the thread dumptid is refering to or proc0.
 	 * Based on either, take the address of the ucred.
 	 */
 	credp = 0;
-- 
cgit v1.1


From 2d5cd857113bfc9af9145ac6ad2e8cf7923dc8a9 Mon Sep 17 00:00:00 2001
From: trasz <trasz@FreeBSD.org>
Date: Wed, 2 Jun 2010 20:35:56 +0000
Subject: MFC r208437:

Make acl_get_perm_np(3) work with NFSv4 ACLs.

Reviewed by:	kientzle@
Approved by:	re (kib)
---
 lib/libc/posix1e/acl_get.c  | 24 ------------------------
 lib/libc/posix1e/acl_perm.c | 18 ++++++++++++++++++
 2 files changed, 18 insertions(+), 24 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/posix1e/acl_get.c b/lib/libc/posix1e/acl_get.c
index 8a77ab1..b323501 100644
--- a/lib/libc/posix1e/acl_get.c
+++ b/lib/libc/posix1e/acl_get.c
@@ -132,30 +132,6 @@ acl_get_fd_np(int fd, acl_type_t type)
 	return (aclp);
 }
 
-int
-acl_get_perm_np(acl_permset_t permset_d, acl_perm_t perm)
-{
-
-	if (permset_d == NULL) {
-		errno = EINVAL;
-		return (-1);
-	}
-
-	switch(perm) {
-	case ACL_READ:
-	case ACL_WRITE:
-	case ACL_EXECUTE:
-		if (*permset_d & perm)
-			return (1);
-		break;
-	default:
-		errno = EINVAL;
-		return (-1);
-	}
-
-	return (0);
-}
-
 /*
  * acl_get_permset() (23.4.17): return via permset_p a descriptor to
  * the permission set in the ACL entry entry_d.
diff --git a/lib/libc/posix1e/acl_perm.c b/lib/libc/posix1e/acl_perm.c
index b5108ca..ad2f47e 100644
--- a/lib/libc/posix1e/acl_perm.c
+++ b/lib/libc/posix1e/acl_perm.c
@@ -108,3 +108,21 @@ acl_delete_perm(acl_permset_t permset_d, acl_perm_t perm)
 
 	return (0);
 }
+
+int
+acl_get_perm_np(acl_permset_t permset_d, acl_perm_t perm)
+{
+
+	if (permset_d == NULL) {
+		errno = EINVAL;
+		return (-1);
+	}
+
+	if (_perm_is_invalid(perm))
+		return (-1);
+
+	if (*permset_d & perm)
+		return (1);
+
+	return (0);
+}
-- 
cgit v1.1


From 7ebb302abcbc0ab56797f28e24a3769665e41f3c Mon Sep 17 00:00:00 2001
From: brian <brian@FreeBSD.org>
Date: Wed, 9 Jun 2010 07:31:41 +0000
Subject: MFC r197477 - clean up keys deleted via pthread_key_delete()

PR:		135462
Approved by:	re (kib)
Nod timeout:	davidxu
---
 lib/libthr/thread/thr_spec.c | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libthr/thread/thr_spec.c b/lib/libthr/thread/thr_spec.c
index 06d1287..0740d6b 100644
--- a/lib/libthr/thread/thr_spec.c
+++ b/lib/libthr/thread/thr_spec.c
@@ -131,9 +131,19 @@ _thread_cleanupspecific(void)
 				curthread->specific[key].data = NULL;
 				curthread->specific_data_count--;
 			}
+			else if (curthread->specific[key].data != NULL) {
+				/* 
+				 * This can happen if the key is deleted via
+				 * pthread_key_delete without first setting the value
+				 * to NULL in all threads.  POSIX says that the
+				 * destructor is not invoked in this case.
+				 */
+				curthread->specific[key].data = NULL;
+				curthread->specific_data_count--;
+			}
 
 			/*
-			 * If there is a destructore, call it
+			 * If there is a destructor, call it
 			 * with the key table entry unlocked:
 			 */
 			if (destructor != NULL) {
-- 
cgit v1.1


From 14ae56e45249ff7263e0a4b2d5c298e8c6e38e23 Mon Sep 17 00:00:00 2001
From: trasz <trasz@FreeBSD.org>
Date: Fri, 11 Jun 2010 15:21:12 +0000
Subject: Fix usage of uninitialized variable.

Found with:	Coverity Prevent
CID:		7517
Approved by:	re (kib)
---
 lib/libc/posix1e/acl_strip.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libc/posix1e/acl_strip.c b/lib/libc/posix1e/acl_strip.c
index 1dcdfcd..8aa850f 100644
--- a/lib/libc/posix1e/acl_strip.c
+++ b/lib/libc/posix1e/acl_strip.c
@@ -46,7 +46,7 @@ static acl_t
 _nfs4_acl_strip_np(const acl_t aclp, int recalculate_mask)
 {
 	acl_t newacl;
-	mode_t mode;
+	mode_t mode = 0;
 
 	newacl = acl_init(ACL_MAX_ENTRIES);
 	if (newacl == NULL) {
-- 
cgit v1.1


From 693ced2a4639e406f17a09ace2b38edb38d0f194 Mon Sep 17 00:00:00 2001
From: trasz <trasz@FreeBSD.org>
Date: Fri, 11 Jun 2010 15:26:15 +0000
Subject: MFC r208811:

Don't use pointer to 64 bit value (id_t) to point to 32 bit value (uid_t).

Found with:	Coverity Prevent
CID:		7466, 7467
Approved by:	re (kib)
---
 lib/libc/posix1e/acl_to_text_nfs4.c | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/posix1e/acl_to_text_nfs4.c b/lib/libc/posix1e/acl_to_text_nfs4.c
index 5d0aa31..2ea92d9 100644
--- a/lib/libc/posix1e/acl_to_text_nfs4.c
+++ b/lib/libc/posix1e/acl_to_text_nfs4.c
@@ -50,7 +50,7 @@ format_who(char *str, size_t size, const acl_entry_t entry, int numeric)
 	acl_tag_t tag;
 	struct passwd *pwd;
 	struct group *grp;
-	id_t *id;
+	uid_t *id;
 
 	error = acl_get_tag_type(entry, &tag);
 	if (error)
@@ -62,7 +62,7 @@ format_who(char *str, size_t size, const acl_entry_t entry, int numeric)
 		break;
 
 	case ACL_USER:
-		id = (id_t *)acl_get_qualifier(entry);
+		id = (uid_t *)acl_get_qualifier(entry);
 		if (id == NULL)
 			return (-1);
 		/* XXX: Thread-unsafe. */
@@ -81,7 +81,7 @@ format_who(char *str, size_t size, const acl_entry_t entry, int numeric)
 		break;
 
 	case ACL_GROUP:
-		id = (id_t *)acl_get_qualifier(entry);
+		id = (uid_t *)acl_get_qualifier(entry);
 		if (id == NULL)
 			return (-1);
 		/* XXX: Thread-unsafe. */
@@ -141,7 +141,7 @@ format_additional_id(char *str, size_t size, const acl_entry_t entry)
 {
 	int error;
 	acl_tag_t tag;
-	id_t *id;
+	uid_t *id;
 
 	error = acl_get_tag_type(entry, &tag);
 	if (error)
@@ -155,7 +155,7 @@ format_additional_id(char *str, size_t size, const acl_entry_t entry)
 		break;
 
 	default:
-		id = (id_t *)acl_get_qualifier(entry);
+		id = (uid_t *)acl_get_qualifier(entry);
 		if (id == NULL)
 			return (-1);
 		snprintf(str, size, ":%d", (unsigned int)*id);
-- 
cgit v1.1


From 2b4920d05f55f6e6d79794be2b9d00c3001823e5 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Sat, 12 Jun 2010 05:21:29 +0000
Subject: MFC r209078:

Detect bit endianness through machine/endian.h.
This fixes xz on big-endian systems.

Tested on:	sparc64 (kindly provided by linimon), amd64
Approved by:	re (kensmith)
---
 lib/liblzma/config.h | 7 +++++++
 1 file changed, 7 insertions(+)

(limited to 'lib')

diff --git a/lib/liblzma/config.h b/lib/liblzma/config.h
index 223cb95..e7b61bb 100644
--- a/lib/liblzma/config.h
+++ b/lib/liblzma/config.h
@@ -84,6 +84,12 @@
 # define __EXTENSIONS__ 1
 #endif
 #define VERSION "4.999.9beta"
+#if defined(__FreeBSD__)
+#include <machine/endian.h>
+#if _BYTE_ORDER == _BIG_ENDIAN
+# define WORDS_BIGENDIAN 1
+#endif
+#else
 #if defined AC_APPLE_UNIVERSAL_BUILD
 # if defined __BIG_ENDIAN__
 #  define WORDS_BIGENDIAN 1
@@ -93,3 +99,4 @@
 /* #  undef WORDS_BIGENDIAN */
 # endif
 #endif
+#endif
-- 
cgit v1.1


From 1cceb68541d136b076fa2956a754b7c258258b2e Mon Sep 17 00:00:00 2001
From: des <des@FreeBSD.org>
Date: Tue, 22 Jun 2010 19:56:07 +0000
Subject: merge r196650 from head (via stable/8): tty might be NULL

Approved by:	re (kib@)
---
 lib/libpam/modules/pam_lastlog/pam_lastlog.c | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'lib')

diff --git a/lib/libpam/modules/pam_lastlog/pam_lastlog.c b/lib/libpam/modules/pam_lastlog/pam_lastlog.c
index 817d1f3..9899f8d 100644
--- a/lib/libpam/modules/pam_lastlog/pam_lastlog.c
+++ b/lib/libpam/modules/pam_lastlog/pam_lastlog.c
@@ -183,6 +183,11 @@ pam_sm_close_session(pam_handle_t *pamh __unused, int flags __unused,
 	pam_err = pam_get_item(pamh, PAM_TTY, (const void **)&tty);
 	if (pam_err != PAM_SUCCESS)
 		goto err;
+	if (tty == NULL) {
+		PAM_LOG("No PAM_TTY");
+		pam_err = PAM_SERVICE_ERR;
+		goto err;
+	}
 	if (strncmp(tty, _PATH_DEV, strlen(_PATH_DEV)) == 0)
 		tty = (const char *)tty + strlen(_PATH_DEV);
 	if (*(const char *)tty == '\0')
-- 
cgit v1.1


From d45b7f14ae6fa78882fa9ec3be976733ca4767b4 Mon Sep 17 00:00:00 2001
From: grehan <grehan@FreeBSD.org>
Date: Fri, 13 May 2011 04:54:01 +0000
Subject: Import of bhyve hypervisor and utilities, part 1.   vmm.ko - kernel
 module for VT-x, VT-d and hypervisor control   bhyve  - user-space sequencer
 and i/o emulation   vmmctl - dump of hypervisor register state   libvmm -
 front-end to vmm.ko chardev interface

bhyve was designed and implemented by Neel Natu.

Thanks to the following folk from NetApp who helped to make this available:
	Joe CaraDonna
	Peter Snyder
	Jeff Heller
	Sandeep Mann
	Steve Miller
	Brian Pawlowski
---
 lib/Makefile                   |   2 +
 lib/libvmmapi/Makefile         |   9 +
 lib/libvmmapi/mptable.c        | 336 +++++++++++++++++++++
 lib/libvmmapi/mptable.h        | 171 +++++++++++
 lib/libvmmapi/vmmapi.c         | 647 +++++++++++++++++++++++++++++++++++++++++
 lib/libvmmapi/vmmapi.h         |  98 +++++++
 lib/libvmmapi/vmmapi_freebsd.c | 187 ++++++++++++
 7 files changed, 1450 insertions(+)
 create mode 100644 lib/libvmmapi/Makefile
 create mode 100644 lib/libvmmapi/mptable.c
 create mode 100644 lib/libvmmapi/mptable.h
 create mode 100644 lib/libvmmapi/vmmapi.c
 create mode 100644 lib/libvmmapi/vmmapi.h
 create mode 100644 lib/libvmmapi/vmmapi_freebsd.c

(limited to 'lib')

diff --git a/lib/Makefile b/lib/Makefile
index e9f4ec3..8f33206 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -102,6 +102,7 @@ SUBDIR= ${SUBDIR_ORDERED} \
 	${_libusbhid} \
 	${_libusb} \
 	${_libvgl} \
+	${_libvmmapi} \
 	libwrap \
 	liby \
 	libz \
@@ -177,6 +178,7 @@ _libncp=	libncp
 .endif
 _libsmb=	libsmb
 _libvgl=	libvgl
+_libvmmapi=	libvmmapi
 .endif
 
 .if ${MACHINE_ARCH} == "powerpc"
diff --git a/lib/libvmmapi/Makefile b/lib/libvmmapi/Makefile
new file mode 100644
index 0000000..492391f
--- /dev/null
+++ b/lib/libvmmapi/Makefile
@@ -0,0 +1,9 @@
+# $FreeBSD$
+
+LIB=	vmmapi
+SRCS=	vmmapi.c vmmapi_freebsd.c mptable.c
+INCS=	vmmapi.h
+
+CFLAGS+= -I${.CURDIR}
+
+.include <bsd.lib.mk>
diff --git a/lib/libvmmapi/mptable.c b/lib/libvmmapi/mptable.c
new file mode 100644
index 0000000..1aea61a
--- /dev/null
+++ b/lib/libvmmapi/mptable.c
@@ -0,0 +1,336 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/types.h>
+#include <sys/mman.h>
+
+#include <stdio.h>
+#include <string.h>
+#include <machine/vmm.h>
+#include <machine/vmm_dev.h>
+
+#include "vmmapi.h"
+#include "mptable.h"
+
+#define LAPIC_PADDR		(0xFEE00000)
+#define LAPIC_VERSION 		(16)
+
+#define IOAPIC_PADDR		(0xFEC00000)
+#define IOAPIC_VERSION		(0x11)
+
+extern int errno;
+
+static uint8_t
+mp_compute_checksum(void *base, size_t len)
+{
+	uint8_t	*bytes = base;
+	uint8_t	 sum = 0;
+	for(; len > 0; len--) {
+		sum += *bytes++;
+	}
+	return 256 - sum;
+}
+
+static void
+mp_build_mpfp(struct mp_floating_pointer *mpfp, vm_paddr_t mpfp_gpa)
+{
+	memset(mpfp, 0, sizeof(*mpfp));
+	memcpy(mpfp->signature, MPFP_SIGNATURE, MPFP_SIGNATURE_LEN);
+	mpfp->mptable_paddr = mpfp_gpa + sizeof(*mpfp);
+	mpfp->specrev = MP_SPECREV;
+	mpfp->feature2 = 0;
+	mpfp->checksum = mp_compute_checksum(mpfp, sizeof(*mpfp));
+}
+
+static void
+mp_build_mpch(struct mp_config_hdr *mpch)
+{
+	memset(mpch, 0, sizeof(*mpch));
+	mpch->specrev = MP_SPECREV;
+	memcpy(mpch->signature, MPCH_SIGNATURE, MPCH_SIGNATURE_LEN);
+	memcpy(mpch->oemid, MPCH_OEMID, MPCH_OEMID_LEN);
+	memcpy(mpch->prodid, MPCH_PRODID, MPCH_PRODID_LEN);
+	mpch->lapic_paddr = LAPIC_PADDR;
+
+
+}
+
+static void
+mp_build_proc_entries(struct mpe_proc *mpep, int num_proc)
+{
+	int i;
+
+	for (i = 0; i < num_proc; i++) {
+		memset(mpep, 0, sizeof(*mpep));
+		mpep->entry_type = MP_ENTRY_PROC;
+		mpep->lapic_id = i; // XXX
+		mpep->lapic_version = LAPIC_VERSION;
+		mpep->proc_flags = (i == 0)?MPEP_FLAGS_BSP:0;
+		mpep->proc_flags |= MPEP_FLAGS_EN;
+		mpep->proc_signature = MPEP_SIGNATURE;
+		mpep->feature_flags = MPEP_FEATURES;
+		mpep++;
+	}
+	
+}
+
+static void
+mp_build_bus_entries(struct mpe_bus *mpeb)
+{
+	memset(mpeb, 0, sizeof(*mpeb));
+	mpeb->entry_type = MP_ENTRY_BUS;
+	mpeb->busid = MPE_BUSID_ISA;
+	memcpy(mpeb->busname, MPE_BUSNAME_ISA, MPE_BUSNAME_LEN);
+	mpeb++;
+
+	memset(mpeb, 0, sizeof(*mpeb));
+	mpeb->entry_type = MP_ENTRY_BUS;
+	mpeb->busid = MPE_BUSID_PCI;
+	memcpy(mpeb->busname, MPE_BUSNAME_PCI, MPE_BUSNAME_LEN);
+
+}
+
+static void
+mp_build_ioapic_entries(struct mpe_ioapic *mpei)
+{
+	memset(mpei, 0, sizeof(*mpei));
+	mpei->entry_type = MP_ENTRY_IOAPIC;
+	mpei->ioapic_id = MPE_IOAPIC_ID;
+	mpei->ioapic_version = IOAPIC_VERSION;
+	mpei->ioapic_flags = MPE_IOAPIC_FLAG_EN;
+	mpei->ioapic_paddr = IOAPIC_PADDR;
+}
+
+static void
+mp_build_ioint_entries(struct mpe_ioint *mpeii, int num_pins)
+{
+	int pin;
+
+	/*
+	 * The following config is taken from kernel mptable.c
+	 * mptable_parse_default_config_ints(...), for now 
+	 * just use the default config, tweek later if needed.
+	 */
+
+
+	/* Run through all 16 pins. */
+	for (pin = 0; pin < num_pins; pin++) {
+		memset(mpeii, 0, sizeof(*mpeii));
+		mpeii->entry_type = MP_ENTRY_IOINT;
+		mpeii->src_bus_id = MPE_BUSID_ISA;
+		mpeii->dst_apic_id = MPE_IOAPIC_ID;
+
+		/*
+		 * All default configs route IRQs from bus 0 to the first 16 pins
+		 * of the first I/O APIC with an APIC ID of 2.
+		 */
+		mpeii->dst_apic_intin = pin;
+		switch (pin) {
+		case 0:
+			/* Pin 0 is an ExtINT pin. */
+			mpeii->intr_type = MPEII_INTR_EXTINT;
+			break;
+		case 2:
+			/* IRQ 0 is routed to pin 2. */
+			mpeii->intr_type = MPEII_INTR_INT;
+			mpeii->src_bus_irq = 0;
+			break;
+		case 5:
+		case 10:
+		case 11:
+			/*
+			 * PCI Irqs set to level triggered.
+			 */
+			mpeii->intr_flags = MPEII_FLAGS_TRIGMODE_LEVEL;
+			mpeii->src_bus_id = MPE_BUSID_PCI;
+		default:
+			/* All other pins are identity mapped. */
+			mpeii->intr_type = MPEII_INTR_INT;
+			mpeii->src_bus_irq = pin;
+			break;
+		}
+		mpeii++;
+	}
+
+}
+
+#define COPYSTR(dest, src, bytes)		\
+	memcpy(dest, src, bytes); 		\
+	str[bytes] = 0;
+
+
+static void
+mptable_dump(struct mp_floating_pointer *mpfp, struct mp_config_hdr *mpch)
+{
+	static char 	 str[16];
+	int 		 i;
+	char 		*cur;
+
+	union mpe {
+		struct mpe_proc 	*proc;
+		struct mpe_bus  	*bus;
+		struct mpe_ioapic 	*ioapic;
+		struct mpe_ioint 	*ioint;
+		struct mpe_lint 	*lnit;
+		char   			*p;
+	};
+
+	union mpe mpe;
+
+	printf(" MP Floating Pointer :\n");
+	COPYSTR(str, mpfp->signature, 4);
+	printf("    signature: 	%s\n", str);
+	printf("    mpch paddr: %x\n", mpfp->mptable_paddr);
+	printf("    length: 	%x\n", mpfp->length);
+	printf("    specrec: 	%x\n", mpfp->specrev);
+	printf("    checksum: 	%x\n", mpfp->checksum);
+	printf("    feature1: 	%x\n", mpfp->feature1);
+	printf("    feature2: 	%x\n", mpfp->feature2);
+	printf("    feature3: 	%x\n", mpfp->feature3);
+	printf("    feature4: 	%x\n", mpfp->feature4);
+
+	printf(" MP Configuration Header :\n");
+	COPYSTR(str, mpch->signature, 4);
+	printf("    signature: 		%s\n", str);
+	printf("    length: 		%x\n", mpch->length);
+	printf("    specrec: 		%x\n", mpch->specrev);
+	printf("    checksum: 		%x\n", mpch->checksum);
+	COPYSTR(str, mpch->oemid, MPCH_OEMID_LEN);
+	printf("    oemid: 		%s\n", str);
+	COPYSTR(str, mpch->prodid, MPCH_PRODID_LEN);
+	printf("    prodid: 		%s\n", str);
+	printf("    oem_ptr: 		%x\n", mpch->oem_ptr);
+	printf("    oem_sz: 		%x\n", mpch->oem_sz);
+	printf("    nr_entries: 	%x\n", mpch->nr_entries);
+	printf("    apic paddr: 	%x\n", mpch->lapic_paddr);
+	printf("    ext_length: 	%x\n", mpch->ext_length);
+	printf("    ext_checksum: 	%x\n", mpch->ext_checksum);
+
+	cur = (char *)mpch + sizeof(*mpch);
+	for (i = 0; i < mpch->nr_entries; i++) {
+		mpe.p = cur;
+		switch(*mpe.p) {		
+			case MP_ENTRY_PROC:
+				printf(" MP Processor Entry :\n");
+				printf("	lapic_id: 	%x\n", mpe.proc->lapic_id);
+				printf("	lapic_version:	%x\n", mpe.proc->lapic_version);
+				printf("	proc_flags: 	%x\n", mpe.proc->proc_flags);
+				printf("	proc_signature: %x\n", mpe.proc->proc_signature);
+				printf("	feature_flags: 	%x\n", mpe.proc->feature_flags);
+				cur += sizeof(struct mpe_proc);
+				break;
+			case MP_ENTRY_BUS:
+				printf(" MP Bus Entry :\n");
+				printf("	busid: 		%x\n", mpe.bus->busid);
+				COPYSTR(str, mpe.bus->busname, MPE_BUSNAME_LEN);
+				printf("	busname: 	%s\n", str);
+				cur += sizeof(struct mpe_bus);
+				break;
+			case MP_ENTRY_IOAPIC:
+				printf(" MP IOAPIC Entry :\n");
+				printf("	ioapi_id: 		%x\n", mpe.ioapic->ioapic_id);
+				printf("	ioapi_version: 		%x\n", mpe.ioapic->ioapic_version);
+				printf("	ioapi_flags: 		%x\n", mpe.ioapic->ioapic_flags);
+				printf("	ioapi_paddr: 		%x\n", mpe.ioapic->ioapic_paddr);
+				cur += sizeof(struct mpe_ioapic);
+				break;
+			case MP_ENTRY_IOINT:
+				printf(" MP IO Interrupt Entry :\n");
+				printf("	intr_type: 		%x\n", mpe.ioint->intr_type);
+				printf("	intr_flags: 		%x\n", mpe.ioint->intr_flags);
+				printf("	src_bus_id: 		%x\n", mpe.ioint->src_bus_id);
+				printf("	src_bus_irq: 		%x\n", mpe.ioint->src_bus_irq);
+				printf("	dst_apic_id: 		%x\n", mpe.ioint->dst_apic_id);
+				printf("	dst_apic_intin:		%x\n", mpe.ioint->dst_apic_intin);
+				cur += sizeof(struct mpe_ioint);
+				break;
+			case MP_ENTRY_LINT:
+				printf(" MP Local Interrupt Entry :\n");
+				cur += sizeof(struct mpe_lint);
+				break;
+		}
+
+	}
+}
+
+int
+vm_build_mptable(struct vmctx *ctx, vm_paddr_t gpa, int len, int ncpu,
+		 void *oemp, int oemsz)
+{
+	struct mp_config_hdr	*mpch;
+	char 			*mapaddr;
+	char 			*startaddr;
+	int 	 		 error;
+
+	mapaddr = vm_map_memory(ctx, gpa, len);
+	if (mapaddr == MAP_FAILED) {
+		printf("%s\n", strerror(errno));
+		goto err;
+	}
+	startaddr = mapaddr;
+
+	mp_build_mpfp((struct mp_floating_pointer*) mapaddr, gpa);
+	mapaddr += sizeof(struct mp_floating_pointer);
+
+	mpch = (struct mp_config_hdr*)mapaddr;
+	mp_build_mpch(mpch);
+	mapaddr += sizeof(struct mp_config_hdr);
+
+	mp_build_proc_entries((struct mpe_proc*) mapaddr, ncpu);
+	mapaddr += (sizeof(struct mpe_proc)*ncpu);
+	mpch->nr_entries += ncpu;
+
+	mp_build_bus_entries((struct mpe_bus*)mapaddr);
+	mapaddr += (sizeof(struct mpe_bus)*MPE_NUM_BUSES);
+	mpch->nr_entries += MPE_NUM_BUSES;
+#if 0
+	mp_build_ioapic_entries((struct mpe_ioapic*)mapaddr);
+	mapaddr += sizeof(struct mpe_ioapic);
+	mpch->nr_entries++;
+
+	mp_build_ioint_entries((struct mpe_ioint*)mapaddr, MPEII_MAX_IRQ);
+	mapaddr += sizeof(struct mpe_ioint)*MPEII_MAX_IRQ;
+	mpch->nr_entries += MPEII_MAX_IRQ;
+
+#endif
+	if (oemp) {
+		mpch->oem_ptr = mapaddr - startaddr + gpa;
+		mpch->oem_sz = oemsz;
+		memcpy(mapaddr, oemp, oemsz);
+	}
+	mpch->length = (mapaddr) - ((char*) mpch);
+	mpch->checksum = mp_compute_checksum(mpch, sizeof(*mpch));
+
+
+	// mptable_dump((struct mp_floating_pointer*)startaddr, mpch);
+err:
+	return (error);
+}
diff --git a/lib/libvmmapi/mptable.h b/lib/libvmmapi/mptable.h
new file mode 100644
index 0000000..cad8834
--- /dev/null
+++ b/lib/libvmmapi/mptable.h
@@ -0,0 +1,171 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _MPTABLE_h_
+#define	_MPTABLE_h_
+
+#define MP_SPECREV		(4)		// MP spec revision 1.1
+
+/*
+ * MP Floating Pointer Structure
+ */ 
+#define MPFP_SIGNATURE		"_MP_"
+#define MPFP_SIGNATURE_LEN	(4)
+#define MPFP_FEATURE2		(0x80)	// IMCR is present
+struct mp_floating_pointer {
+	uint8_t 	signature[MPFP_SIGNATURE_LEN];
+	uint32_t 	mptable_paddr;
+	uint8_t 	length;
+	uint8_t 	specrev;
+	uint8_t 	checksum;
+	uint8_t 	feature1;
+	uint8_t 	feature2;
+	uint8_t 	feature3;
+	uint8_t 	feature4;
+	uint8_t 	feature5;
+};
+
+
+/*
+ * MP Configuration Table Header
+ */
+#define MPCH_SIGNATURE		"PCMP"
+#define MPCH_SIGNATURE_LEN	(4)
+
+#define MPCH_OEMID		"NETAPP  "
+#define MPCH_OEMID_LEN		(8)
+#define MPCH_PRODID		"vFiler      "
+#define MPCH_PRODID_LEN		(12)
+
+struct mp_config_hdr {
+	uint8_t 	signature[MPCH_SIGNATURE_LEN];
+	uint16_t 	length;
+	uint8_t 	specrev;
+	uint8_t 	checksum;
+	uint8_t 	oemid[MPCH_OEMID_LEN];
+	uint8_t 	prodid[MPCH_PRODID_LEN];
+	uint32_t 	oem_ptr;
+	uint16_t 	oem_sz;
+	uint16_t 	nr_entries;
+	uint32_t 	lapic_paddr;
+	uint16_t 	ext_length;
+	uint8_t 	ext_checksum;
+	uint8_t 	reserved;
+};
+
+#define MP_ENTRY_PROC	(0)
+#define MP_ENTRY_BUS	(1)
+#define MP_ENTRY_IOAPIC	(2)
+#define MP_ENTRY_IOINT	(3)
+#define MP_ENTRY_LINT	(4)
+
+/*
+ * MP Processor Entry
+ */
+
+#define MPEP_FLAGS_EN		(0x1)
+#define MPEP_FLAGS_BSP		(0x2)
+
+#define MPEP_SIG_FAMILY         (6)
+#define MPEP_SIG_MODEL          (26)
+#define MPEP_SIG_STEPPING       (5)
+#define MPEP_SIGNATURE  	((MPEP_SIG_FAMILY << 8) | (MPEP_SIG_MODEL << 4)  \
+				| (MPEP_SIG_STEPPING))
+
+#define MPEP_FEATURES		(0xBFEBFBFF) // Value from Intel i7 CPUID
+
+struct mpe_proc {
+	uint8_t 	entry_type;
+	uint8_t 	lapic_id;
+	uint8_t 	lapic_version;
+	uint8_t 	proc_flags;
+	uint32_t 	proc_signature;
+	uint32_t 	feature_flags;
+	uint8_t 	reserved[8];
+};
+
+/*
+ * MP Bus Entry
+ */
+
+#define MPE_NUM_BUSES	(2)
+#define MPE_BUSNAME_LEN	(6)
+#define MPE_BUSID_ISA	(0)
+#define MPE_BUSID_PCI	(1)
+#define MPE_BUSNAME_ISA	"ISA   "
+#define MPE_BUSNAME_PCI	"PCI   "
+struct mpe_bus {
+	uint8_t 	entry_type;
+	uint8_t		busid;
+	uint8_t		busname[MPE_BUSNAME_LEN];
+};
+
+/*
+ * MP IO APIC Entry
+ */
+#define MPE_IOAPIC_ID		(2)
+#define MPE_IOAPIC_FLAG_EN	(1)
+struct mpe_ioapic {
+	uint8_t 	entry_type;
+	uint8_t		ioapic_id;
+	uint8_t		ioapic_version;
+	uint8_t		ioapic_flags;
+	uint32_t	ioapic_paddr;
+
+};
+
+/*
+ * MP IO Interrupt Assignment Entry
+ */
+#define MPEII_INTR_INT		(0)
+#define MPEII_INTR_NMI		(1)
+#define MPEII_INTR_SMI		(2)
+#define MPEII_INTR_EXTINT	(3)
+#define MPEII_PCI_IRQ_MASK    	(0x0c20U) /* IRQ 5,10,11 are PCI connected */
+#define MPEII_MAX_IRQ		(16)
+#define MPEII_FLAGS_TRIGMODE_LEVEL	(0x3)
+struct mpe_ioint {
+	uint8_t 	entry_type;
+	uint8_t		intr_type;
+	uint16_t	intr_flags;
+	uint8_t		src_bus_id;
+	uint8_t		src_bus_irq;
+	uint8_t		dst_apic_id;
+	uint8_t		dst_apic_intin;
+};
+
+/*
+ * MP Local Interrupt Assignment Entry
+ */
+struct mpe_lint {
+	uint8_t 	entry_type;
+};
+
+int	vm_build_mptable(struct vmctx *ctxt, vm_paddr_t gpa, int len,
+			 int ncpu, void *oemp, int oemsz);
+#endif	/* _MPTABLE_h_ */
diff --git a/lib/libvmmapi/vmmapi.c b/lib/libvmmapi/vmmapi.c
new file mode 100644
index 0000000..7f95fea
--- /dev/null
+++ b/lib/libvmmapi/vmmapi.c
@@ -0,0 +1,647 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/types.h>
+#include <sys/sysctl.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+
+#include <machine/specialreg.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <string.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+#include <machine/vmm.h>
+#include <machine/vmm_dev.h>
+
+#include "vmmapi.h"
+#include "mptable.h"
+
+#ifndef	CR4_VMXE
+#define	CR4_VMXE	(1UL << 13)
+#endif
+
+#define BIOS_ROM_BASE		(0xf0000)
+#define BIOS_ROM_SIZE		(0x10000)
+
+struct vmctx {
+	int	fd;
+	char	*name;
+};
+
+#define	CREATE(x)  sysctlbyname("hw.vmm.create", NULL, NULL, (x), strlen((x)))
+#define	DESTROY(x) sysctlbyname("hw.vmm.destroy", NULL, NULL, (x), strlen((x)))
+
+static int
+vm_device_open(const char *name)
+{
+        int fd, len;
+        char *vmfile;
+
+	len = strlen("/dev/vmm/") + strlen(name) + 1;
+	vmfile = malloc(len);
+	assert(vmfile != NULL);
+	snprintf(vmfile, len, "/dev/vmm/%s", name);
+
+        /* Open the device file */
+        fd = open(vmfile, O_RDWR, 0);
+
+	free(vmfile);
+        return (fd);
+}
+
+int
+vm_create(const char *name)
+{
+
+	return (CREATE((char *)name));
+}
+
+struct vmctx *
+vm_open(const char *name)
+{
+	struct vmctx *vm;
+
+	vm = malloc(sizeof(struct vmctx) + strlen(name) + 1);
+	assert(vm != NULL);
+
+	vm->fd = -1;
+	vm->name = (char *)(vm + 1);
+	strcpy(vm->name, name);
+
+	if ((vm->fd = vm_device_open(vm->name)) < 0)
+		goto err;
+
+	return (vm);
+err:
+	vm_destroy(vm);
+	return (NULL);
+}
+
+void
+vm_destroy(struct vmctx *vm)
+{
+	assert(vm != NULL);
+
+	DESTROY(vm->name);
+	if (vm->fd >= 0)
+		close(vm->fd);
+	free(vm);
+}
+
+int
+vm_get_memory_seg(struct vmctx *ctx, vm_paddr_t gpa,
+		  vm_paddr_t *ret_hpa, size_t *ret_len)
+{
+	int error;
+	struct vm_memory_segment seg;
+
+	bzero(&seg, sizeof(seg));
+	seg.gpa = gpa;
+	error = ioctl(ctx->fd, VM_GET_MEMORY_SEG, &seg);
+	*ret_hpa = seg.hpa;
+	*ret_len = seg.len;
+	return (error);
+}
+
+int
+vm_setup_memory(struct vmctx *ctx, vm_paddr_t gpa, size_t len, char **mapaddr)
+{
+	int error;
+	struct vm_memory_segment seg;
+
+	/*
+	 * Create and optionally map 'len' bytes of memory at guest
+	 * physical address 'gpa'
+	 */
+	bzero(&seg, sizeof(seg));
+	seg.gpa = gpa;
+	seg.len = len;
+	error = ioctl(ctx->fd, VM_MAP_MEMORY, &seg);
+	if (error == 0 && mapaddr != NULL) {
+		*mapaddr = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED,
+				ctx->fd, gpa);
+	}
+	return (error);
+}
+
+char *
+vm_map_memory(struct vmctx *ctx, vm_paddr_t gpa, size_t len)
+{
+
+	/* Map 'len' bytes of memory at guest physical address 'gpa' */
+	return ((char *)mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED,
+		     ctx->fd, gpa));
+}
+
+int
+vm_set_desc(struct vmctx *ctx, int vcpu, int reg,
+	    uint64_t base, uint32_t limit, uint32_t access)
+{
+	int error;
+	struct vm_seg_desc vmsegdesc;
+
+	bzero(&vmsegdesc, sizeof(vmsegdesc));
+	vmsegdesc.cpuid = vcpu;
+	vmsegdesc.regnum = reg;
+	vmsegdesc.desc.base = base;
+	vmsegdesc.desc.limit = limit;
+	vmsegdesc.desc.access = access;
+
+	error = ioctl(ctx->fd, VM_SET_SEGMENT_DESCRIPTOR, &vmsegdesc);
+	return (error);
+}
+
+int
+vm_get_desc(struct vmctx *ctx, int vcpu, int reg,
+	    uint64_t *base, uint32_t *limit, uint32_t *access)
+{
+	int error;
+	struct vm_seg_desc vmsegdesc;
+
+	bzero(&vmsegdesc, sizeof(vmsegdesc));
+	vmsegdesc.cpuid = vcpu;
+	vmsegdesc.regnum = reg;
+
+	error = ioctl(ctx->fd, VM_GET_SEGMENT_DESCRIPTOR, &vmsegdesc);
+	if (error == 0) {
+		*base = vmsegdesc.desc.base;
+		*limit = vmsegdesc.desc.limit;
+		*access = vmsegdesc.desc.access;
+	}
+	return (error);
+}
+
+int
+vm_set_register(struct vmctx *ctx, int vcpu, int reg, uint64_t val)
+{
+	int error;
+	struct vm_register vmreg;
+
+	bzero(&vmreg, sizeof(vmreg));
+	vmreg.cpuid = vcpu;
+	vmreg.regnum = reg;
+	vmreg.regval = val;
+
+	error = ioctl(ctx->fd, VM_SET_REGISTER, &vmreg);
+	return (error);
+}
+
+int
+vm_get_register(struct vmctx *ctx, int vcpu, int reg, uint64_t *ret_val)
+{
+	int error;
+	struct vm_register vmreg;
+
+	bzero(&vmreg, sizeof(vmreg));
+	vmreg.cpuid = vcpu;
+	vmreg.regnum = reg;
+
+	error = ioctl(ctx->fd, VM_GET_REGISTER, &vmreg);
+	*ret_val = vmreg.regval;
+	return (error);
+}
+
+int
+vm_get_pinning(struct vmctx *ctx, int vcpu, int *host_cpuid)
+{
+	int error;
+	struct vm_pin vmpin;
+
+	bzero(&vmpin, sizeof(vmpin));
+	vmpin.vm_cpuid = vcpu;
+
+	error = ioctl(ctx->fd, VM_GET_PINNING, &vmpin);
+	*host_cpuid = vmpin.host_cpuid;
+	return (error);
+}
+
+int
+vm_set_pinning(struct vmctx *ctx, int vcpu, int host_cpuid)
+{
+	int error;
+	struct vm_pin vmpin;
+
+	bzero(&vmpin, sizeof(vmpin));
+	vmpin.vm_cpuid = vcpu;
+	vmpin.host_cpuid = host_cpuid;
+
+	error = ioctl(ctx->fd, VM_SET_PINNING, &vmpin);
+	return (error);
+}
+
+int
+vm_run(struct vmctx *ctx, int vcpu, uint64_t rip, struct vm_exit *vmexit)
+{
+	int error;
+	struct vm_run vmrun;
+
+	bzero(&vmrun, sizeof(vmrun));
+	vmrun.cpuid = vcpu;
+	vmrun.rip = rip;
+
+	error = ioctl(ctx->fd, VM_RUN, &vmrun);
+	bcopy(&vmrun.vm_exit, vmexit, sizeof(struct vm_exit));
+	return (error);
+}
+
+static int
+vm_inject_event_real(struct vmctx *ctx, int vcpu, enum vm_event_type type,
+		     int vector, int error_code, int error_code_valid)
+{
+	struct vm_event ev;
+
+	bzero(&ev, sizeof(ev));
+	ev.cpuid = vcpu;
+	ev.type = type;
+	ev.vector = vector;
+	ev.error_code = error_code;
+	ev.error_code_valid = error_code_valid;
+
+	return (ioctl(ctx->fd, VM_INJECT_EVENT, &ev));
+}
+
+int
+vm_inject_event(struct vmctx *ctx, int vcpu, enum vm_event_type type,
+		int vector)
+{
+
+	return (vm_inject_event_real(ctx, vcpu, type, vector, 0, 0));
+}
+
+int
+vm_inject_event2(struct vmctx *ctx, int vcpu, enum vm_event_type type,
+		 int vector, int error_code)
+{
+
+	return (vm_inject_event_real(ctx, vcpu, type, vector, error_code, 1));
+}
+
+int
+vm_build_tables(struct vmctx *ctxt, int ncpu, void *oemtbl, int oemtblsz)
+{
+
+	return (vm_build_mptable(ctxt, BIOS_ROM_BASE, BIOS_ROM_SIZE, ncpu,
+				 oemtbl, oemtblsz));
+}
+
+int
+vm_lapic_irq(struct vmctx *ctx, int vcpu, int vector)
+{
+	struct vm_lapic_irq vmirq;
+
+	bzero(&vmirq, sizeof(vmirq));
+	vmirq.cpuid = vcpu;
+	vmirq.vector = vector;
+
+	return (ioctl(ctx->fd, VM_LAPIC_IRQ, &vmirq));
+}
+
+int
+vm_inject_nmi(struct vmctx *ctx, int vcpu)
+{
+	struct vm_nmi vmnmi;
+
+	bzero(&vmnmi, sizeof(vmnmi));
+	vmnmi.cpuid = vcpu;
+
+	return (ioctl(ctx->fd, VM_INJECT_NMI, &vmnmi));
+}
+
+int
+vm_capability_name2type(const char *capname)
+{
+	int i;
+
+	static struct {
+		const char	*name;
+		int		type;
+	} capstrmap[] = {
+		{ "hlt_exit",		VM_CAP_HALT_EXIT },
+		{ "mtrap_exit",		VM_CAP_MTRAP_EXIT },
+		{ "pause_exit",		VM_CAP_PAUSE_EXIT },
+		{ "unrestricted_guest",	VM_CAP_UNRESTRICTED_GUEST },
+		{ 0 }
+	};
+
+	for (i = 0; capstrmap[i].name != NULL && capname != NULL; i++) {
+		if (strcmp(capstrmap[i].name, capname) == 0)
+			return (capstrmap[i].type);
+	}
+
+	return (-1);
+}
+
+int
+vm_get_capability(struct vmctx *ctx, int vcpu, enum vm_cap_type cap,
+		  int *retval)
+{
+	int error;
+	struct vm_capability vmcap;
+
+	bzero(&vmcap, sizeof(vmcap));
+	vmcap.cpuid = vcpu;
+	vmcap.captype = cap;
+
+	error = ioctl(ctx->fd, VM_GET_CAPABILITY, &vmcap);
+	*retval = vmcap.capval;
+	return (error);
+}
+
+int
+vm_set_capability(struct vmctx *ctx, int vcpu, enum vm_cap_type cap, int val)
+{
+	struct vm_capability vmcap;
+
+	bzero(&vmcap, sizeof(vmcap));
+	vmcap.cpuid = vcpu;
+	vmcap.captype = cap;
+	vmcap.capval = val;
+	
+	return (ioctl(ctx->fd, VM_SET_CAPABILITY, &vmcap));
+}
+
+int
+vm_assign_pptdev(struct vmctx *ctx, int bus, int slot, int func)
+{
+	struct vm_pptdev pptdev;
+
+	bzero(&pptdev, sizeof(pptdev));
+	pptdev.bus = bus;
+	pptdev.slot = slot;
+	pptdev.func = func;
+
+	return (ioctl(ctx->fd, VM_BIND_PPTDEV, &pptdev));
+}
+
+int
+vm_unassign_pptdev(struct vmctx *ctx, int bus, int slot, int func)
+{
+	struct vm_pptdev pptdev;
+
+	bzero(&pptdev, sizeof(pptdev));
+	pptdev.bus = bus;
+	pptdev.slot = slot;
+	pptdev.func = func;
+
+	return (ioctl(ctx->fd, VM_UNBIND_PPTDEV, &pptdev));
+}
+
+int
+vm_map_pptdev_mmio(struct vmctx *ctx, int bus, int slot, int func,
+		   vm_paddr_t gpa, size_t len, vm_paddr_t hpa)
+{
+	struct vm_pptdev_mmio pptmmio;
+
+	bzero(&pptmmio, sizeof(pptmmio));
+	pptmmio.bus = bus;
+	pptmmio.slot = slot;
+	pptmmio.func = func;
+	pptmmio.gpa = gpa;
+	pptmmio.len = len;
+	pptmmio.hpa = hpa;
+
+	return (ioctl(ctx->fd, VM_MAP_PPTDEV_MMIO, &pptmmio));
+}
+
+int
+vm_setup_msi(struct vmctx *ctx, int vcpu, int bus, int slot, int func,
+	     int destcpu, int vector, int numvec)
+{
+	struct vm_pptdev_msi pptmsi;
+
+	bzero(&pptmsi, sizeof(pptmsi));
+	pptmsi.vcpu = vcpu;
+	pptmsi.bus = bus;
+	pptmsi.slot = slot;
+	pptmsi.func = func;
+	pptmsi.destcpu = destcpu;
+	pptmsi.vector = vector;
+	pptmsi.numvec = numvec;
+
+	return (ioctl(ctx->fd, VM_PPTDEV_MSI, &pptmsi));
+}
+
+uint64_t *
+vm_get_stats(struct vmctx *ctx, int vcpu, struct timeval *ret_tv,
+	     int *ret_entries)
+{
+	int error;
+
+	static struct vm_stats vmstats;
+
+	vmstats.cpuid = vcpu;
+
+	error = ioctl(ctx->fd, VM_STATS, &vmstats);
+	if (error == 0) {
+		if (ret_entries)
+			*ret_entries = vmstats.num_entries;
+		if (ret_tv)
+			*ret_tv = vmstats.tv;
+		return (vmstats.statbuf);
+	} else
+		return (NULL);
+}
+
+const char *
+vm_get_stat_desc(struct vmctx *ctx, int index)
+{
+	int error;
+
+	static struct vm_stat_desc statdesc;
+
+	statdesc.index = index;
+	if (ioctl(ctx->fd, VM_STAT_DESC, &statdesc) == 0)
+		return (statdesc.desc);
+	else
+		return (NULL);
+}
+
+/*
+ * From Intel Vol 3a:
+ * Table 9-1. IA-32 Processor States Following Power-up, Reset or INIT
+ */
+int
+vcpu_reset(struct vmctx *vmctx, int vcpu)
+{
+	int error;
+	uint64_t rflags, rip, cr0, cr4, zero, desc_base, rdx;
+	uint32_t desc_access, desc_limit;
+	uint16_t sel;
+
+	zero = 0;
+
+	rflags = 0x2;
+	error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_RFLAGS, rflags);
+	if (error)
+		goto done;
+
+	rip = 0xfff0;
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_RIP, rip)) != 0)
+		goto done;
+
+	cr0 = CR0_NE;
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_CR0, cr0)) != 0)
+		goto done;
+
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_CR3, zero)) != 0)
+		goto done;
+	
+	cr4 = CR4_VMXE;
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_CR4, cr4)) != 0)
+		goto done;
+
+	/*
+	 * CS: present, r/w, accessed, 16-bit, byte granularity, usable
+	 */
+	desc_base = 0xffff0000;
+	desc_limit = 0xffff;
+	desc_access = 0x0093;
+	error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_CS,
+			    desc_base, desc_limit, desc_access);
+	if (error)
+		goto done;
+
+	sel = 0xf000;
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_CS, sel)) != 0)
+		goto done;
+
+	/*
+	 * SS,DS,ES,FS,GS: present, r/w, accessed, 16-bit, byte granularity
+	 */
+	desc_base = 0;
+	desc_limit = 0xffff;
+	desc_access = 0x0093;
+	error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_SS,
+			    desc_base, desc_limit, desc_access);
+	if (error)
+		goto done;
+
+	error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_DS,
+			    desc_base, desc_limit, desc_access);
+	if (error)
+		goto done;
+
+	error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_ES,
+			    desc_base, desc_limit, desc_access);
+	if (error)
+		goto done;
+
+	error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_FS,
+			    desc_base, desc_limit, desc_access);
+	if (error)
+		goto done;
+
+	error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_GS,
+			    desc_base, desc_limit, desc_access);
+	if (error)
+		goto done;
+
+	sel = 0;
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_SS, sel)) != 0)
+		goto done;
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_DS, sel)) != 0)
+		goto done;
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_ES, sel)) != 0)
+		goto done;
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_FS, sel)) != 0)
+		goto done;
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_GS, sel)) != 0)
+		goto done;
+
+	/* General purpose registers */
+	rdx = 0xf00;
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_RAX, zero)) != 0)
+		goto done;
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_RBX, zero)) != 0)
+		goto done;
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_RCX, zero)) != 0)
+		goto done;
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_RDX, rdx)) != 0)
+		goto done;
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_RSI, zero)) != 0)
+		goto done;
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_RDI, zero)) != 0)
+		goto done;
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_RBP, zero)) != 0)
+		goto done;
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_RSP, zero)) != 0)
+		goto done;
+
+	/* GDTR, IDTR */
+	desc_base = 0;
+	desc_limit = 0xffff;
+	desc_access = 0;
+	error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_GDTR,
+			    desc_base, desc_limit, desc_access);
+	if (error != 0)
+		goto done;
+
+	error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_IDTR,
+			    desc_base, desc_limit, desc_access);
+	if (error != 0)
+		goto done;
+
+	/* TR */
+	desc_base = 0;
+	desc_limit = 0xffff;
+	desc_access = 0x0000008b;
+	error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_TR, 0, 0, desc_access);
+	if (error)
+		goto done;
+
+	sel = 0;
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_TR, sel)) != 0)
+		goto done;
+
+	/* LDTR */
+	desc_base = 0;
+	desc_limit = 0xffff;
+	desc_access = 0x00000082;
+	error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_LDTR, desc_base,
+			    desc_limit, desc_access);
+	if (error)
+		goto done;
+
+	sel = 0;
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_LDTR, 0)) != 0)
+		goto done;
+
+	/* XXX cr2, debug registers */
+
+	error = 0;
+done:
+	return (error);
+}
diff --git a/lib/libvmmapi/vmmapi.h b/lib/libvmmapi/vmmapi.h
new file mode 100644
index 0000000..38533a8
--- /dev/null
+++ b/lib/libvmmapi/vmmapi.h
@@ -0,0 +1,98 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _VMMAPI_H_
+#define	_VMMAPI_H_
+
+struct vmctx;
+
+int	vm_create(const char *name);
+struct vmctx *vm_open(const char *name);
+void	vm_destroy(struct vmctx *ctx);
+int	vm_get_memory_seg(struct vmctx *ctx, vm_paddr_t gpa,
+			  vm_paddr_t *ret_hpa, size_t *ret_len);
+/*
+ * Create a memory segment of 'len' bytes in the guest physical address space
+ * at offset 'gpa'.
+ *
+ * If 'mapaddr' is not NULL then this region is mmap'ed into the address
+ * space of the calling process. If there is an mmap error then *mapaddr
+ * will be set to MAP_FAILED.
+ */
+
+int	vm_setup_memory(struct vmctx *ctx, vm_paddr_t gpa, size_t len,
+		      char **mapaddr);
+char *  vm_map_memory(struct vmctx *ctx, vm_paddr_t gpa, size_t len);
+int	vm_set_desc(struct vmctx *ctx, int vcpu, int reg,
+		    uint64_t base, uint32_t limit, uint32_t access);
+int	vm_get_desc(struct vmctx *ctx, int vcpu, int reg,
+		    uint64_t *base, uint32_t *limit, uint32_t *access);
+int	vm_set_register(struct vmctx *ctx, int vcpu, int reg, uint64_t val);
+int	vm_get_register(struct vmctx *ctx, int vcpu, int reg, uint64_t *retval);
+int	vm_get_pinning(struct vmctx *ctx, int vcpu, int *host_cpuid);
+int	vm_set_pinning(struct vmctx *ctx, int vcpu, int host_cpuid);
+int	vm_run(struct vmctx *ctx, int vcpu, uint64_t rip,
+	       struct vm_exit *ret_vmexit);
+int	vm_build_tables(struct vmctx *ctxt, int ncpus, void *oemtbl,
+			int oemtblsz);
+int	vm_inject_event(struct vmctx *ctx, int vcpu, enum vm_event_type type,
+			int vector);
+int	vm_inject_event2(struct vmctx *ctx, int vcpu, enum vm_event_type type,
+			 int vector, int error_code);
+int	vm_lapic_irq(struct vmctx *ctx, int vcpu, int vector);
+int	vm_inject_nmi(struct vmctx *ctx, int vcpu);
+int	vm_capability_name2type(const char *capname);
+int	vm_get_capability(struct vmctx *ctx, int vcpu, enum vm_cap_type cap,
+			  int *retval);
+int	vm_set_capability(struct vmctx *ctx, int vcpu, enum vm_cap_type cap,
+			  int val);
+int	vm_assign_pptdev(struct vmctx *ctx, int bus, int slot, int func);
+int	vm_unassign_pptdev(struct vmctx *ctx, int bus, int slot, int func);
+int	vm_map_pptdev_mmio(struct vmctx *ctx, int bus, int slot, int func,
+			   vm_paddr_t gpa, size_t len, vm_paddr_t hpa);
+int	vm_setup_msi(struct vmctx *ctx, int vcpu, int bus, int slot, int func,
+		     int dest, int vector, int numvec);
+
+/*
+ * Return a pointer to the statistics buffer. Note that this is not MT-safe.
+ */
+uint64_t *vm_get_stats(struct vmctx *ctx, int vcpu, struct timeval *ret_tv,
+		       int *ret_entries);
+const char *vm_get_stat_desc(struct vmctx *ctx, int index);
+
+/* Reset vcpu register state */
+int	vcpu_reset(struct vmctx *ctx, int vcpu);
+
+/*
+ * FreeBSD specific APIs
+ */
+int	vm_setup_freebsd_registers(struct vmctx *ctx, int vcpu,
+				uint64_t rip, uint64_t cr3, uint64_t gdtbase,
+				uint64_t rsp);
+void	vm_setup_freebsd_gdt(uint64_t *gdtr);
+#endif	/* _VMMAPI_H_ */
diff --git a/lib/libvmmapi/vmmapi_freebsd.c b/lib/libvmmapi/vmmapi_freebsd.c
new file mode 100644
index 0000000..c4ad989
--- /dev/null
+++ b/lib/libvmmapi/vmmapi_freebsd.c
@@ -0,0 +1,187 @@
+/*-
+ * Copyright (c) 2011 NetApp, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/types.h>
+
+#include <machine/specialreg.h>
+#include <machine/segments.h>
+#include <machine/vmm.h>
+
+#include "vmmapi.h"
+
+#ifndef	CR4_VMXE
+#define	CR4_VMXE	(1UL << 13)
+#endif
+
+#define	DESC_UNUSABLE		0x00010000
+
+#define	GUEST_NULL_SEL		0
+#define	GUEST_CODE_SEL		1
+#define	GUEST_DATA_SEL		2
+#define	GUEST_GDTR_LIMIT	(3 * 8 - 1)
+
+void     
+vm_setup_freebsd_gdt(uint64_t *gdtr)
+{       
+	gdtr[GUEST_NULL_SEL] = 0;
+	gdtr[GUEST_CODE_SEL] = 0x0020980000000000;
+	gdtr[GUEST_DATA_SEL] = 0x0000900000000000;
+}
+
+/*
+ * Setup the 'vcpu' register set such that it will begin execution at
+ * 'rip' in long mode.
+ */
+int
+vm_setup_freebsd_registers(struct vmctx *vmctx, int vcpu,
+			   uint64_t rip, uint64_t cr3, uint64_t gdtbase,
+			   uint64_t rsp)
+{
+	int error;
+	uint64_t cr0, cr4, efer, rflags, desc_base;
+	uint32_t desc_access, desc_limit;
+	uint16_t gsel;
+
+	cr0 = CR0_PE | CR0_PG | CR0_NE;
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_CR0, cr0)) != 0)
+		goto done;
+
+	cr4 = CR4_PAE | CR4_VMXE;
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_CR4, cr4)) != 0)
+		goto done;
+
+	efer = EFER_LME | EFER_LMA;
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_EFER, efer)))
+		goto done;
+
+	rflags = 0x2;
+	error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_RFLAGS, rflags);
+	if (error)
+		goto done;
+
+	desc_base = 0;
+	desc_limit = 0;
+	desc_access = 0x0000209B;
+	error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_CS,
+			    desc_base, desc_limit, desc_access);
+	if (error)
+		goto done;
+
+	desc_access = 0x00000093;
+	error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_DS,
+			    desc_base, desc_limit, desc_access);
+	if (error)
+		goto done;
+
+	error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_ES,
+			    desc_base, desc_limit, desc_access);
+	if (error)
+		goto done;
+
+	error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_FS,
+			    desc_base, desc_limit, desc_access);
+	if (error)
+		goto done;
+
+	error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_GS,
+			    desc_base, desc_limit, desc_access);
+	if (error)
+		goto done;
+
+	error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_SS,
+			    desc_base, desc_limit, desc_access);
+	if (error)
+		goto done;
+
+	/*
+	 * XXX TR is pointing to null selector even though we set the
+	 * TSS segment to be usable with a base address and limit of 0.
+	 */
+	desc_access = 0x0000008b;
+	error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_TR, 0, 0, desc_access);
+	if (error)
+		goto done;
+
+	error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_LDTR, 0, 0,
+			    DESC_UNUSABLE);
+	if (error)
+		goto done;
+
+	gsel = GSEL(GUEST_CODE_SEL, SEL_KPL);
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_CS, gsel)) != 0)
+		goto done;
+	
+	gsel = GSEL(GUEST_DATA_SEL, SEL_KPL);
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_DS, gsel)) != 0)
+		goto done;
+	
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_ES, gsel)) != 0)
+		goto done;
+
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_FS, gsel)) != 0)
+		goto done;
+	
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_GS, gsel)) != 0)
+		goto done;
+	
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_SS, gsel)) != 0)
+		goto done;
+
+	/* XXX TR is pointing to the null selector */
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_TR, 0)) != 0)
+		goto done;
+
+	/* LDTR is pointing to the null selector */
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_LDTR, 0)) != 0)
+		goto done;
+
+	/* entry point */
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_RIP, rip)) != 0)
+		goto done;
+
+	/* page table base */
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_CR3, cr3)) != 0)
+		goto done;
+
+	desc_base = gdtbase;
+	desc_limit = GUEST_GDTR_LIMIT;
+	error = vm_set_desc(vmctx, vcpu, VM_REG_GUEST_GDTR,
+			    desc_base, desc_limit, 0);
+	if (error != 0)
+		goto done;
+
+	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_RSP, rsp)) != 0)
+		goto done;
+
+	error = 0;
+done:
+	return (error);
+}
-- 
cgit v1.1


From 9f0c999f8126597eb572b80056df88335dbd0070 Mon Sep 17 00:00:00 2001
From: grehan <grehan@FreeBSD.org>
Date: Sat, 28 Apr 2012 16:28:00 +0000
Subject: MSI-x interrupt support for PCI pass-thru devices.

Includes instruction emulation for memory r/w access. This
opens the door for io-apic, local apic, hpet timer, and
legacy device emulation.

Submitted by:	ryan dot berryhill at sandvine dot com
Reviewed by:	grehan
Obtained from:	Sandvine
---
 lib/libvmmapi/vmmapi.c | 19 +++++++++++++++++++
 lib/libvmmapi/vmmapi.h |  2 ++
 2 files changed, 21 insertions(+)

(limited to 'lib')

diff --git a/lib/libvmmapi/vmmapi.c b/lib/libvmmapi/vmmapi.c
index 17f2063..a33a94e 100644
--- a/lib/libvmmapi/vmmapi.c
+++ b/lib/libvmmapi/vmmapi.c
@@ -454,6 +454,25 @@ vm_setup_msi(struct vmctx *ctx, int vcpu, int bus, int slot, int func,
 	return (ioctl(ctx->fd, VM_PPTDEV_MSI, &pptmsi));
 }
 
+int	
+vm_setup_msix(struct vmctx *ctx, int vcpu, int bus, int slot, int func,
+	      int idx, uint32_t msg, uint32_t vector_control, uint64_t addr)
+{
+	struct vm_pptdev_msix pptmsix;
+
+	bzero(&pptmsix, sizeof(pptmsix));
+	pptmsix.vcpu = vcpu;
+	pptmsix.bus = bus;
+	pptmsix.slot = slot;
+	pptmsix.func = func;
+	pptmsix.idx = idx;
+	pptmsix.msg = msg;
+	pptmsix.addr = addr;
+	pptmsix.vector_control = vector_control;
+
+	return ioctl(ctx->fd, VM_PPTDEV_MSIX, &pptmsix);
+}
+
 uint64_t *
 vm_get_stats(struct vmctx *ctx, int vcpu, struct timeval *ret_tv,
 	     int *ret_entries)
diff --git a/lib/libvmmapi/vmmapi.h b/lib/libvmmapi/vmmapi.h
index 38533a8..7b66421 100644
--- a/lib/libvmmapi/vmmapi.h
+++ b/lib/libvmmapi/vmmapi.h
@@ -77,6 +77,8 @@ int	vm_map_pptdev_mmio(struct vmctx *ctx, int bus, int slot, int func,
 			   vm_paddr_t gpa, size_t len, vm_paddr_t hpa);
 int	vm_setup_msi(struct vmctx *ctx, int vcpu, int bus, int slot, int func,
 		     int dest, int vector, int numvec);
+int	vm_setup_msix(struct vmctx *ctx, int vcpu, int bus, int slot, int func,
+		      int idx, uint32_t msg, uint32_t vector_control, uint64_t addr);
 
 /*
  * Return a pointer to the statistics buffer. Note that this is not MT-safe.
-- 
cgit v1.1


From ea90afcf927db1d8e7cf9775414b82ca13dff8b3 Mon Sep 17 00:00:00 2001
From: dougb <dougb@FreeBSD.org>
Date: Mon, 4 Jun 2012 22:06:29 +0000
Subject: Vendor import of BIND 9.8.3-P1

---
 lib/dns/rdata.c     |  8 ++++----
 lib/dns/rdataslab.c | 11 ++++++++---
 2 files changed, 12 insertions(+), 7 deletions(-)

(limited to 'lib')

diff --git a/lib/dns/rdata.c b/lib/dns/rdata.c
index 8773145..f6807d1 100644
--- a/lib/dns/rdata.c
+++ b/lib/dns/rdata.c
@@ -329,8 +329,8 @@ dns_rdata_compare(const dns_rdata_t *rdata1, const dns_rdata_t *rdata2) {
 
 	REQUIRE(rdata1 != NULL);
 	REQUIRE(rdata2 != NULL);
-	REQUIRE(rdata1->data != NULL);
-	REQUIRE(rdata2->data != NULL);
+	REQUIRE(rdata1->length == 0 || rdata1->data != NULL);
+	REQUIRE(rdata2->length == 0 || rdata2->data != NULL);
 	REQUIRE(DNS_RDATA_VALIDFLAGS(rdata1));
 	REQUIRE(DNS_RDATA_VALIDFLAGS(rdata2));
 
@@ -360,8 +360,8 @@ dns_rdata_casecompare(const dns_rdata_t *rdata1, const dns_rdata_t *rdata2) {
 
 	REQUIRE(rdata1 != NULL);
 	REQUIRE(rdata2 != NULL);
-	REQUIRE(rdata1->data != NULL);
-	REQUIRE(rdata2->data != NULL);
+	REQUIRE(rdata1->length == 0 || rdata1->data != NULL);
+	REQUIRE(rdata2->length == 0 || rdata2->data != NULL);
 	REQUIRE(DNS_RDATA_VALIDFLAGS(rdata1));
 	REQUIRE(DNS_RDATA_VALIDFLAGS(rdata2));
 
diff --git a/lib/dns/rdataslab.c b/lib/dns/rdataslab.c
index 150d9b8..6fbfdd7 100644
--- a/lib/dns/rdataslab.c
+++ b/lib/dns/rdataslab.c
@@ -126,6 +126,11 @@ isc_result_t
 dns_rdataslab_fromrdataset(dns_rdataset_t *rdataset, isc_mem_t *mctx,
 			   isc_region_t *region, unsigned int reservelen)
 {
+	/*
+	 * Use &removed as a sentinal pointer for duplicate
+	 * rdata as rdata.data == NULL is valid.
+	 */
+	static unsigned char removed;
 	struct xrdata  *x;
 	unsigned char  *rawbuf;
 #if DNS_RDATASET_FIXED
@@ -169,6 +174,7 @@ dns_rdataslab_fromrdataset(dns_rdataset_t *rdataset, isc_mem_t *mctx,
 		INSIST(result == ISC_R_SUCCESS);
 		dns_rdata_init(&x[i].rdata);
 		dns_rdataset_current(rdataset, &x[i].rdata);
+		INSIST(x[i].rdata.data != &removed);
 #if DNS_RDATASET_FIXED
 		x[i].order = i;
 #endif
@@ -201,8 +207,7 @@ dns_rdataslab_fromrdataset(dns_rdataset_t *rdataset, isc_mem_t *mctx,
 	 */
 	for (i = 1; i < nalloc; i++) {
 		if (compare_rdata(&x[i-1].rdata, &x[i].rdata) == 0) {
-			x[i-1].rdata.data = NULL;
-			x[i-1].rdata.length = 0;
+			x[i-1].rdata.data = &removed;
 #if DNS_RDATASET_FIXED
 			/*
 			 * Preserve the least order so A, B, A -> A, B
@@ -292,7 +297,7 @@ dns_rdataslab_fromrdataset(dns_rdataset_t *rdataset, isc_mem_t *mctx,
 #endif
 
 	for (i = 0; i < nalloc; i++) {
-		if (x[i].rdata.data == NULL)
+		if (x[i].rdata.data == &removed)
 			continue;
 #if DNS_RDATASET_FIXED
 		offsettable[x[i].order] = rawbuf - offsetbase;
-- 
cgit v1.1


From bf4ca397d7b9fc67cd4254cd07e7ce9b42b22250 Mon Sep 17 00:00:00 2001
From: dougb <dougb@FreeBSD.org>
Date: Tue, 24 Jul 2012 18:48:17 +0000
Subject: Vendor import of BIND 9.8.3-P2

---
 lib/dns/resolver.c |  5 +++--
 lib/dns/zone.c     | 26 ++++++++++++++++++++------
 2 files changed, 23 insertions(+), 8 deletions(-)

(limited to 'lib')

diff --git a/lib/dns/resolver.c b/lib/dns/resolver.c
index 1ae2f16..4c8b144 100644
--- a/lib/dns/resolver.c
+++ b/lib/dns/resolver.c
@@ -8448,6 +8448,7 @@ dns_resolver_addbadcache(dns_resolver_t *resolver, dns_name_t *name,
 			goto cleanup;
 		bad->type = type;
 		bad->hashval = hashval;
+		bad->expire = *expire;
 		isc_buffer_init(&buffer, bad + 1, name->length);
 		dns_name_init(&bad->name, NULL);
 		dns_name_copy(name, &bad->name, &buffer);
@@ -8459,8 +8460,8 @@ dns_resolver_addbadcache(dns_resolver_t *resolver, dns_name_t *name,
 		if (resolver->badcount < resolver->badhash * 2 &&
 		    resolver->badhash > DNS_BADCACHE_SIZE)
 			resizehash(resolver, &now, ISC_FALSE);
-	}
-	bad->expire = *expire;
+	} else
+		bad->expire = *expire;
  cleanup:
 	UNLOCK(&resolver->lock);
 }
diff --git a/lib/dns/zone.c b/lib/dns/zone.c
index 22870dc..efe31eb 100644
--- a/lib/dns/zone.c
+++ b/lib/dns/zone.c
@@ -8027,13 +8027,14 @@ zone_maintenance(dns_zone_t *zone) {
 	case dns_zone_master:
 	case dns_zone_slave:
 	case dns_zone_key:
+	case dns_zone_stub:
 		LOCK_ZONE(zone);
 		if (zone->masterfile != NULL &&
 		    isc_time_compare(&now, &zone->dumptime) >= 0 &&
 		    DNS_ZONE_FLAG(zone, DNS_ZONEFLG_LOADED) &&
 		    DNS_ZONE_FLAG(zone, DNS_ZONEFLG_NEEDDUMP)) {
 			dumping = was_dumping(zone);
-		} else
+		} else 
 			dumping = ISC_TRUE;
 		UNLOCK_ZONE(zone);
 		if (!dumping) {
@@ -8386,7 +8387,7 @@ zone_dump(dns_zone_t *zone, isc_boolean_t compact) {
 		goto fail;
 	}
 
-	if (compact) {
+	if (compact && zone->type != dns_zone_stub) {
 		dns_zone_t *dummy = NULL;
 		LOCK_ZONE(zone);
 		zone_iattach(zone, &dummy);
@@ -9242,7 +9243,7 @@ stub_callback(isc_task_t *task, isc_event_t *event) {
 	dns_zone_t *zone = NULL;
 	char master[ISC_SOCKADDR_FORMATSIZE];
 	char source[ISC_SOCKADDR_FORMATSIZE];
-	isc_uint32_t nscnt, cnamecnt;
+	isc_uint32_t nscnt, cnamecnt, refresh, retry, expire;
 	isc_result_t result;
 	isc_time_t now;
 	isc_boolean_t exiting = ISC_FALSE;
@@ -9390,19 +9391,32 @@ stub_callback(isc_task_t *task, isc_event_t *event) {
 	ZONEDB_LOCK(&zone->dblock, isc_rwlocktype_write);
 	if (zone->db == NULL)
 		zone_attachdb(zone, stub->db);
+	result = zone_get_from_db(zone, zone->db, NULL, NULL, NULL, &refresh,
+				  &retry, &expire, NULL, NULL);
+	if (result == ISC_R_SUCCESS) {
+		zone->refresh = RANGE(refresh, zone->minrefresh,
+				      zone->maxrefresh);
+		zone->retry = RANGE(retry, zone->minretry, zone->maxretry);
+		zone->expire = RANGE(expire, zone->refresh + zone->retry,
+				     DNS_MAX_EXPIRE);
+		DNS_ZONE_SETFLAG(zone, DNS_ZONEFLG_HAVETIMERS);
+	}
 	ZONEDB_UNLOCK(&zone->dblock, isc_rwlocktype_write);
 	dns_db_detach(&stub->db);
 
-	if (zone->masterfile != NULL)
-		zone_needdump(zone, 0);
-
 	dns_message_destroy(&msg);
 	isc_event_free(&event);
 	dns_request_destroy(&zone->request);
+
 	DNS_ZONE_CLRFLAG(zone, DNS_ZONEFLG_REFRESH);
+	DNS_ZONE_SETFLAG(zone, DNS_ZONEFLG_LOADED);
 	DNS_ZONE_JITTER_ADD(&now, zone->refresh, &zone->refreshtime);
 	isc_interval_set(&i, zone->expire, 0);
 	DNS_ZONE_TIME_ADD(&now, zone->expire, &zone->expiretime);
+
+	if (zone->masterfile != NULL)
+		zone_needdump(zone, 0);
+
 	zone_settimer(zone, &now);
 	goto free_stub;
 
-- 
cgit v1.1


From 19e3bb399c65527fcc13a4ae345b71ee6c5fee84 Mon Sep 17 00:00:00 2001
From: neel <neel@FreeBSD.org>
Date: Sat, 4 Aug 2012 02:14:27 +0000
Subject: There is no need to explicitly specify the CR4_VMXE bit when writing
 to guest CR4. This bit is specific to the Intel VTX and removing it makes the
 library more portable to AMD/SVM.

In the Intel VTX implementation, the hypervisor will ensure that this bit is
always set. See vmx_fix_cr4() for details.

Suggested by: grehan
---
 lib/libvmmapi/vmmapi.c         | 6 +-----
 lib/libvmmapi/vmmapi_freebsd.c | 6 +-----
 2 files changed, 2 insertions(+), 10 deletions(-)

(limited to 'lib')

diff --git a/lib/libvmmapi/vmmapi.c b/lib/libvmmapi/vmmapi.c
index a33a94e..077a46b 100644
--- a/lib/libvmmapi/vmmapi.c
+++ b/lib/libvmmapi/vmmapi.c
@@ -49,10 +49,6 @@ __FBSDID("$FreeBSD$");
 #include "vmmapi.h"
 #include "mptable.h"
 
-#ifndef	CR4_VMXE
-#define	CR4_VMXE	(1UL << 13)
-#endif
-
 #define BIOS_ROM_BASE		(0xf0000)
 #define BIOS_ROM_SIZE		(0x10000)
 
@@ -536,7 +532,7 @@ vcpu_reset(struct vmctx *vmctx, int vcpu)
 	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_CR3, zero)) != 0)
 		goto done;
 	
-	cr4 = CR4_VMXE;
+	cr4 = 0;
 	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_CR4, cr4)) != 0)
 		goto done;
 
diff --git a/lib/libvmmapi/vmmapi_freebsd.c b/lib/libvmmapi/vmmapi_freebsd.c
index c4ad989..9bd2988 100644
--- a/lib/libvmmapi/vmmapi_freebsd.c
+++ b/lib/libvmmapi/vmmapi_freebsd.c
@@ -37,10 +37,6 @@ __FBSDID("$FreeBSD$");
 
 #include "vmmapi.h"
 
-#ifndef	CR4_VMXE
-#define	CR4_VMXE	(1UL << 13)
-#endif
-
 #define	DESC_UNUSABLE		0x00010000
 
 #define	GUEST_NULL_SEL		0
@@ -74,7 +70,7 @@ vm_setup_freebsd_registers(struct vmctx *vmctx, int vcpu,
 	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_CR0, cr0)) != 0)
 		goto done;
 
-	cr4 = CR4_PAE | CR4_VMXE;
+	cr4 = CR4_PAE;
 	if ((error = vm_set_register(vmctx, vcpu, VM_REG_GUEST_CR4, cr4)) != 0)
 		goto done;
 
-- 
cgit v1.1


From f07527ef6066c09742e5ca0ff4631f9c2c68e82d Mon Sep 17 00:00:00 2001
From: neel <neel@FreeBSD.org>
Date: Sat, 4 Aug 2012 02:38:05 +0000
Subject: API to map an apic id to the vcpu.

At the moment this is a simple mapping because the numerical values are
identical.
---
 lib/libvmmapi/vmmapi.c | 10 ++++++++++
 lib/libvmmapi/vmmapi.h |  1 +
 2 files changed, 11 insertions(+)

(limited to 'lib')

diff --git a/lib/libvmmapi/vmmapi.c b/lib/libvmmapi/vmmapi.c
index 077a46b..d88b694 100644
--- a/lib/libvmmapi/vmmapi.c
+++ b/lib/libvmmapi/vmmapi.c
@@ -314,6 +314,16 @@ vm_build_tables(struct vmctx *ctxt, int ncpu, void *oemtbl, int oemtblsz)
 }
 
 int
+vm_apicid2vcpu(struct vmctx *ctx, int apicid)
+{
+	/*
+	 * The apic id associated with the 'vcpu' has the same numerical value
+	 * as the 'vcpu' itself.
+	 */
+	return (apicid);
+}
+
+int
 vm_lapic_irq(struct vmctx *ctx, int vcpu, int vector)
 {
 	struct vm_lapic_irq vmirq;
diff --git a/lib/libvmmapi/vmmapi.h b/lib/libvmmapi/vmmapi.h
index 7b66421..a216319 100644
--- a/lib/libvmmapi/vmmapi.h
+++ b/lib/libvmmapi/vmmapi.h
@@ -60,6 +60,7 @@ int	vm_run(struct vmctx *ctx, int vcpu, uint64_t rip,
 	       struct vm_exit *ret_vmexit);
 int	vm_build_tables(struct vmctx *ctxt, int ncpus, void *oemtbl,
 			int oemtblsz);
+int	vm_apicid2vcpu(struct vmctx *ctx, int apicid);
 int	vm_inject_event(struct vmctx *ctx, int vcpu, enum vm_event_type type,
 			int vector);
 int	vm_inject_event2(struct vmctx *ctx, int vcpu, enum vm_event_type type,
-- 
cgit v1.1


From ac88464d24af550cd19d2c9aefd2f431e6ae89ed Mon Sep 17 00:00:00 2001
From: neel <neel@FreeBSD.org>
Date: Sat, 4 Aug 2012 22:46:29 +0000
Subject: Allow the 'bhyve' process to control whether or not the virtual
 machine sees an ioapic.

Obtained from: NetApp
---
 lib/libvmmapi/mptable.c | 26 +++++++++++++++-----------
 lib/libvmmapi/mptable.h |  3 +--
 lib/libvmmapi/vmmapi.c  |  5 +++--
 lib/libvmmapi/vmmapi.h  |  4 ++--
 4 files changed, 21 insertions(+), 17 deletions(-)

(limited to 'lib')

diff --git a/lib/libvmmapi/mptable.c b/lib/libvmmapi/mptable.c
index fe84f35..048ea2d 100644
--- a/lib/libvmmapi/mptable.c
+++ b/lib/libvmmapi/mptable.c
@@ -118,20 +118,20 @@ mp_build_bus_entries(struct mpe_bus *mpeb)
 
 }
 
-#ifdef notyet
 static void
-mp_build_ioapic_entries(struct mpe_ioapic *mpei)
+mp_build_ioapic_entries(struct mpe_ioapic *mpei, int id)
 {
 	memset(mpei, 0, sizeof(*mpei));
 	mpei->entry_type = MP_ENTRY_IOAPIC;
-	mpei->ioapic_id = MPE_IOAPIC_ID;
+	mpei->ioapic_id = id;
 	mpei->ioapic_version = IOAPIC_VERSION;
 	mpei->ioapic_flags = MPE_IOAPIC_FLAG_EN;
 	mpei->ioapic_paddr = IOAPIC_PADDR;
 }
 
+#ifdef notyet
 static void
-mp_build_ioint_entries(struct mpe_ioint *mpeii, int num_pins)
+mp_build_ioint_entries(struct mpe_ioint *mpeii, int num_pins, int id)
 {
 	int pin;
 
@@ -147,7 +147,7 @@ mp_build_ioint_entries(struct mpe_ioint *mpeii, int num_pins)
 		memset(mpeii, 0, sizeof(*mpeii));
 		mpeii->entry_type = MP_ENTRY_IOINT;
 		mpeii->src_bus_id = MPE_BUSID_ISA;
-		mpeii->dst_apic_id = MPE_IOAPIC_ID;
+		mpeii->dst_apic_id = id;
 
 		/*
 		 * All default configs route IRQs from bus 0 to the first 16 pins
@@ -285,7 +285,7 @@ mptable_dump(struct mp_floating_pointer *mpfp, struct mp_config_hdr *mpch)
 
 int
 vm_build_mptable(struct vmctx *ctx, vm_paddr_t gpa, int len, int ncpu,
-		 void *oemp, int oemsz)
+		 int ioapic, void *oemp, int oemsz)
 {
 	struct mp_config_hdr	*mpch;
 	char 			*mapaddr;
@@ -313,12 +313,16 @@ vm_build_mptable(struct vmctx *ctx, vm_paddr_t gpa, int len, int ncpu,
 	mp_build_bus_entries((struct mpe_bus*)mapaddr);
 	mapaddr += (sizeof(struct mpe_bus)*MPE_NUM_BUSES);
 	mpch->nr_entries += MPE_NUM_BUSES;
-#if 0
-	mp_build_ioapic_entries((struct mpe_ioapic*)mapaddr);
-	mapaddr += sizeof(struct mpe_ioapic);
-	mpch->nr_entries++;
 
-	mp_build_ioint_entries((struct mpe_ioint*)mapaddr, MPEII_MAX_IRQ);
+	if (ioapic) {
+		mp_build_ioapic_entries((struct mpe_ioapic*)mapaddr, ncpu + 1);
+		mapaddr += sizeof(struct mpe_ioapic);
+		mpch->nr_entries++;
+	}
+
+#ifdef notyet
+	mp_build_ioint_entries((struct mpe_ioint*)mapaddr, MPEII_MAX_IRQ,
+				ncpu + 1);
 	mapaddr += sizeof(struct mpe_ioint)*MPEII_MAX_IRQ;
 	mpch->nr_entries += MPEII_MAX_IRQ;
 
diff --git a/lib/libvmmapi/mptable.h b/lib/libvmmapi/mptable.h
index cad8834..51c7bc1 100644
--- a/lib/libvmmapi/mptable.h
+++ b/lib/libvmmapi/mptable.h
@@ -128,7 +128,6 @@ struct mpe_bus {
 /*
  * MP IO APIC Entry
  */
-#define MPE_IOAPIC_ID		(2)
 #define MPE_IOAPIC_FLAG_EN	(1)
 struct mpe_ioapic {
 	uint8_t 	entry_type;
@@ -167,5 +166,5 @@ struct mpe_lint {
 };
 
 int	vm_build_mptable(struct vmctx *ctxt, vm_paddr_t gpa, int len,
-			 int ncpu, void *oemp, int oemsz);
+			 int ncpu, int ioapic, void *oemp, int oemsz);
 #endif	/* _MPTABLE_h_ */
diff --git a/lib/libvmmapi/vmmapi.c b/lib/libvmmapi/vmmapi.c
index d88b694..5882bd2 100644
--- a/lib/libvmmapi/vmmapi.c
+++ b/lib/libvmmapi/vmmapi.c
@@ -306,11 +306,12 @@ vm_inject_event2(struct vmctx *ctx, int vcpu, enum vm_event_type type,
 }
 
 int
-vm_build_tables(struct vmctx *ctxt, int ncpu, void *oemtbl, int oemtblsz)
+vm_build_tables(struct vmctx *ctxt, int ncpu, int ioapic,
+		void *oemtbl, int oemtblsz)
 {
 
 	return (vm_build_mptable(ctxt, BIOS_ROM_BASE, BIOS_ROM_SIZE, ncpu,
-				 oemtbl, oemtblsz));
+				 ioapic, oemtbl, oemtblsz));
 }
 
 int
diff --git a/lib/libvmmapi/vmmapi.h b/lib/libvmmapi/vmmapi.h
index a216319..de3d4b7 100644
--- a/lib/libvmmapi/vmmapi.h
+++ b/lib/libvmmapi/vmmapi.h
@@ -58,8 +58,8 @@ int	vm_get_pinning(struct vmctx *ctx, int vcpu, int *host_cpuid);
 int	vm_set_pinning(struct vmctx *ctx, int vcpu, int host_cpuid);
 int	vm_run(struct vmctx *ctx, int vcpu, uint64_t rip,
 	       struct vm_exit *ret_vmexit);
-int	vm_build_tables(struct vmctx *ctxt, int ncpus, void *oemtbl,
-			int oemtblsz);
+int	vm_build_tables(struct vmctx *ctxt, int ncpus, int ioapic,
+			void *oemtbl, int oemtblsz);
 int	vm_apicid2vcpu(struct vmctx *ctx, int apicid);
 int	vm_inject_event(struct vmctx *ctx, int vcpu, enum vm_event_type type,
 			int vector);
-- 
cgit v1.1


From cc73504950eb7b5dff2dded9bedd67bc36d64641 Mon Sep 17 00:00:00 2001
From: dim <dim@FreeBSD.org>
Date: Sun, 19 Aug 2012 10:33:04 +0000
Subject: Vendor import of clang trunk r162107:
 http://llvm.org/svn/llvm-project/cfe/trunk@162107

---
 lib/AST/APValue.cpp                                |   5 +-
 lib/AST/ASTContext.cpp                             |  68 +++-
 lib/AST/ASTDiagnostic.cpp                          |   2 +-
 lib/AST/CommentCommandTraits.cpp                   |  10 +-
 lib/AST/DeclCXX.cpp                                |  13 +-
 lib/AST/DeclPrinter.cpp                            |  56 +--
 lib/AST/DeclTemplate.cpp                           |   3 +-
 lib/AST/DumpXML.cpp                                |   5 +
 lib/AST/NestedNameSpecifier.cpp                    |   9 +-
 lib/AST/RawCommentList.cpp                         |  21 +-
 lib/AST/Stmt.cpp                                   |  23 +-
 lib/AST/StmtPrinter.cpp                            |  32 +-
 lib/AST/TemplateBase.cpp                           |   6 +-
 lib/Basic/Diagnostic.cpp                           |   6 +-
 lib/Basic/Targets.cpp                              |   3 +-
 lib/CodeGen/CGBuiltin.cpp                          |  69 +++-
 lib/CodeGen/CGDebugInfo.cpp                        |  41 +-
 lib/CodeGen/CGExpr.cpp                             |  80 ++++
 lib/CodeGen/CGExprCXX.cpp                          |   9 +-
 lib/CodeGen/CGStmt.cpp                             |  24 +-
 lib/CodeGen/CGValue.h                              |   2 +-
 lib/Driver/Tools.cpp                               |   3 +
 lib/Frontend/ASTConsumers.cpp                      |   2 +-
 lib/Frontend/CacheTokens.cpp                       |   2 +-
 lib/Lex/PTHLexer.cpp                               |   6 +-
 lib/Parse/ParseDecl.cpp                            |  73 +++-
 lib/Parse/ParseStmt.cpp                            | 181 ++++-----
 lib/Rewrite/RewriteModernObjC.cpp                  |   9 +-
 lib/Rewrite/RewriteObjC.cpp                        |  10 +-
 lib/Sema/AttributeList.cpp                         |   2 +
 lib/Sema/SemaCast.cpp                              |  20 +-
 lib/Sema/SemaChecking.cpp                          | 420 ++++++++++++++++++++-
 lib/Sema/SemaCodeComplete.cpp                      |   1 -
 lib/Sema/SemaDecl.cpp                              |  72 +++-
 lib/Sema/SemaDeclAttr.cpp                          | 163 ++++++--
 lib/Sema/SemaDeclCXX.cpp                           |   2 +-
 lib/Sema/SemaExceptionSpec.cpp                     |   3 +-
 lib/Sema/SemaExpr.cpp                              |   3 +-
 lib/Sema/SemaExprMember.cpp                        |   2 +-
 lib/Sema/SemaOverload.cpp                          |   2 +-
 lib/Sema/SemaStmt.cpp                              | 329 ++++++++++------
 lib/Sema/SemaTemplate.cpp                          |   7 +
 lib/Sema/SemaTemplateInstantiate.cpp               |   3 +-
 lib/Sema/SemaType.cpp                              |  46 ++-
 lib/Sema/TreeTransform.h                           |  19 +-
 lib/Serialization/ASTReader.cpp                    |   1 -
 lib/Serialization/ASTWriter.cpp                    |   1 -
 .../Checkers/CallAndMessageChecker.cpp             |  12 +-
 .../Checkers/DynamicTypePropagation.cpp            |  87 ++++-
 lib/StaticAnalyzer/Checkers/RetainCountChecker.cpp | 151 +-------
 lib/StaticAnalyzer/Core/AnalysisManager.cpp        |  45 +--
 lib/StaticAnalyzer/Core/BugReporter.cpp            | 189 ++++------
 lib/StaticAnalyzer/Core/BugReporterVisitors.cpp    |   2 +-
 lib/StaticAnalyzer/Core/CallEvent.cpp              |  86 +++--
 .../Core/ExprEngineCallAndReturn.cpp               |  42 ++-
 lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp        |  48 +--
 lib/StaticAnalyzer/Core/PathDiagnostic.cpp         |  18 +-
 lib/StaticAnalyzer/Core/PlistDiagnostics.cpp       |  92 +++--
 lib/StaticAnalyzer/Core/ProgramState.cpp           |   7 +-
 lib/StaticAnalyzer/Core/TextPathDiagnostics.cpp    |  12 +-
 lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp   |  74 +++-
 61 files changed, 1869 insertions(+), 865 deletions(-)

(limited to 'lib')

diff --git a/lib/AST/APValue.cpp b/lib/AST/APValue.cpp
index a74ef14..2d7c9bd 100644
--- a/lib/AST/APValue.cpp
+++ b/lib/AST/APValue.cpp
@@ -367,8 +367,7 @@ void APValue::printPretty(raw_ostream &Out, ASTContext &Ctx, QualType Ty) const{
       if (const ValueDecl *VD = Base.dyn_cast<const ValueDecl*>())
         Out << *VD;
       else
-        Base.get<const Expr*>()->printPretty(Out, Ctx, 0,
-                                             Ctx.getPrintingPolicy());
+        Base.get<const Expr*>()->printPretty(Out, 0, Ctx.getPrintingPolicy());
       if (!O.isZero()) {
         Out << " + " << (O / S);
         if (IsReference)
@@ -389,7 +388,7 @@ void APValue::printPretty(raw_ostream &Out, ASTContext &Ctx, QualType Ty) const{
       ElemTy = VD->getType();
     } else {
       const Expr *E = Base.get<const Expr*>();
-      E->printPretty(Out, Ctx, 0,Ctx.getPrintingPolicy());
+      E->printPretty(Out, 0, Ctx.getPrintingPolicy());
       ElemTy = E->getType();
     }
 
diff --git a/lib/AST/ASTContext.cpp b/lib/AST/ASTContext.cpp
index ad48dff..c021323 100644
--- a/lib/AST/ASTContext.cpp
+++ b/lib/AST/ASTContext.cpp
@@ -66,6 +66,12 @@ RawComment *ASTContext::getRawCommentForDeclNoCache(const Decl *D) const {
   if (D->isImplicit())
     return NULL;
 
+  // User can not attach documentation to implicit instantiations.
+  if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
+    if (FD->getTemplateSpecializationKind() == TSK_ImplicitInstantiation)
+      return NULL;
+  }
+
   // TODO: handle comments for function parameters properly.
   if (isa<ParmVarDecl>(D))
     return NULL;
@@ -145,7 +151,6 @@ RawComment *ASTContext::getRawCommentForDeclNoCache(const Decl *D) const {
         SourceMgr.getLineNumber(DeclLocDecomp.first, DeclLocDecomp.second)
           == SourceMgr.getLineNumber(CommentBeginDecomp.first,
                                      CommentBeginDecomp.second)) {
-      (*Comment)->setDecl(D);
       return *Comment;
     }
   }
@@ -185,13 +190,13 @@ RawComment *ASTContext::getRawCommentForDeclNoCache(const Decl *D) const {
   if (Text.find_first_of(",;{}#@") != StringRef::npos)
     return NULL;
 
-   (*Comment)->setDecl(D);
   return *Comment;
 }
 
-const RawComment *ASTContext::getRawCommentForAnyRedecl(const Decl *D) const {
-  // If we have a 'templated' declaration for a template, adjust 'D' to
-  // refer to the actual template.
+namespace {
+/// If we have a 'templated' declaration for a template, adjust 'D' to
+/// refer to the actual template.
+const Decl *adjustDeclToTemplate(const Decl *D) {
   if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
     if (const FunctionTemplateDecl *FTD = FD->getDescribedFunctionTemplate())
       D = FTD;
@@ -200,6 +205,14 @@ const RawComment *ASTContext::getRawCommentForAnyRedecl(const Decl *D) const {
       D = CTD;
   }
   // FIXME: Alias templates?
+  return D;
+}
+} // unnamed namespace
+
+const RawComment *ASTContext::getRawCommentForAnyRedecl(
+                                                const Decl *D,
+                                                const Decl **OriginalDecl) const {
+  D = adjustDeclToTemplate(D);
 
   // Check whether we have cached a comment for this declaration already.
   {
@@ -207,13 +220,17 @@ const RawComment *ASTContext::getRawCommentForAnyRedecl(const Decl *D) const {
         RedeclComments.find(D);
     if (Pos != RedeclComments.end()) {
       const RawCommentAndCacheFlags &Raw = Pos->second;
-      if (Raw.getKind() != RawCommentAndCacheFlags::NoCommentInDecl)
+      if (Raw.getKind() != RawCommentAndCacheFlags::NoCommentInDecl) {
+        if (OriginalDecl)
+          *OriginalDecl = Raw.getOriginalDecl();
         return Raw.getRaw();
+      }
     }
   }
 
   // Search for comments attached to declarations in the redeclaration chain.
   const RawComment *RC = NULL;
+  const Decl *OriginalDeclForRC = NULL;
   for (Decl::redecl_iterator I = D->redecls_begin(),
                              E = D->redecls_end();
        I != E; ++I) {
@@ -223,16 +240,19 @@ const RawComment *ASTContext::getRawCommentForAnyRedecl(const Decl *D) const {
       const RawCommentAndCacheFlags &Raw = Pos->second;
       if (Raw.getKind() != RawCommentAndCacheFlags::NoCommentInDecl) {
         RC = Raw.getRaw();
+        OriginalDeclForRC = Raw.getOriginalDecl();
         break;
       }
     } else {
       RC = getRawCommentForDeclNoCache(*I);
+      OriginalDeclForRC = *I;
       RawCommentAndCacheFlags Raw;
       if (RC) {
         Raw.setRaw(RC);
         Raw.setKind(RawCommentAndCacheFlags::FromDecl);
       } else
         Raw.setKind(RawCommentAndCacheFlags::NoCommentInDecl);
+      Raw.setOriginalDecl(*I);
       RedeclComments[*I] = Raw;
       if (RC)
         break;
@@ -242,10 +262,14 @@ const RawComment *ASTContext::getRawCommentForAnyRedecl(const Decl *D) const {
   // If we found a comment, it should be a documentation comment.
   assert(!RC || RC->isDocumentation());
 
+  if (OriginalDecl)
+    *OriginalDecl = OriginalDeclForRC;
+
   // Update cache for every declaration in the redeclaration chain.
   RawCommentAndCacheFlags Raw;
   Raw.setRaw(RC);
   Raw.setKind(RawCommentAndCacheFlags::FromRedecl);
+  Raw.setOriginalDecl(OriginalDeclForRC);
 
   for (Decl::redecl_iterator I = D->redecls_begin(),
                              E = D->redecls_end();
@@ -259,11 +283,24 @@ const RawComment *ASTContext::getRawCommentForAnyRedecl(const Decl *D) const {
 }
 
 comments::FullComment *ASTContext::getCommentForDecl(const Decl *D) const {
-  const RawComment *RC = getRawCommentForAnyRedecl(D);
+  D = adjustDeclToTemplate(D);
+  const Decl *Canonical = D->getCanonicalDecl();
+  llvm::DenseMap<const Decl *, comments::FullComment *>::iterator Pos =
+      ParsedComments.find(Canonical);
+  if (Pos != ParsedComments.end())
+    return Pos->second;
+
+  const Decl *OriginalDecl;
+  const RawComment *RC = getRawCommentForAnyRedecl(D, &OriginalDecl);
   if (!RC)
     return NULL;
 
-  return RC->getParsed(*this);
+  if (D != OriginalDecl)
+    return getCommentForDecl(OriginalDecl);
+
+  comments::FullComment *FC = RC->parse(*this, D);
+  ParsedComments[Canonical] = FC;
+  return FC;
 }
 
 void 
@@ -5437,7 +5474,8 @@ ASTContext::getQualifiedTemplateName(NestedNameSpecifier *NNS,
   QualifiedTemplateName *QTN =
     QualifiedTemplateNames.FindNodeOrInsertPos(ID, InsertPos);
   if (!QTN) {
-    QTN = new (*this,4) QualifiedTemplateName(NNS, TemplateKeyword, Template);
+    QTN = new (*this, llvm::alignOf<QualifiedTemplateName>())
+        QualifiedTemplateName(NNS, TemplateKeyword, Template);
     QualifiedTemplateNames.InsertNode(QTN, InsertPos);
   }
 
@@ -5464,10 +5502,12 @@ ASTContext::getDependentTemplateName(NestedNameSpecifier *NNS,
 
   NestedNameSpecifier *CanonNNS = getCanonicalNestedNameSpecifier(NNS);
   if (CanonNNS == NNS) {
-    QTN = new (*this,4) DependentTemplateName(NNS, Name);
+    QTN = new (*this, llvm::alignOf<DependentTemplateName>())
+        DependentTemplateName(NNS, Name);
   } else {
     TemplateName Canon = getDependentTemplateName(CanonNNS, Name);
-    QTN = new (*this,4) DependentTemplateName(NNS, Name, Canon);
+    QTN = new (*this, llvm::alignOf<DependentTemplateName>())
+        DependentTemplateName(NNS, Name, Canon);
     DependentTemplateName *CheckQTN =
       DependentTemplateNames.FindNodeOrInsertPos(ID, InsertPos);
     assert(!CheckQTN && "Dependent type name canonicalization broken");
@@ -5498,10 +5538,12 @@ ASTContext::getDependentTemplateName(NestedNameSpecifier *NNS,
   
   NestedNameSpecifier *CanonNNS = getCanonicalNestedNameSpecifier(NNS);
   if (CanonNNS == NNS) {
-    QTN = new (*this,4) DependentTemplateName(NNS, Operator);
+    QTN = new (*this, llvm::alignOf<DependentTemplateName>())
+        DependentTemplateName(NNS, Operator);
   } else {
     TemplateName Canon = getDependentTemplateName(CanonNNS, Operator);
-    QTN = new (*this,4) DependentTemplateName(NNS, Operator, Canon);
+    QTN = new (*this, llvm::alignOf<DependentTemplateName>())
+        DependentTemplateName(NNS, Operator, Canon);
     
     DependentTemplateName *CheckQTN
       = DependentTemplateNames.FindNodeOrInsertPos(ID, InsertPos);
diff --git a/lib/AST/ASTDiagnostic.cpp b/lib/AST/ASTDiagnostic.cpp
index 35fcd41..a605f1a 100644
--- a/lib/AST/ASTDiagnostic.cpp
+++ b/lib/AST/ASTDiagnostic.cpp
@@ -1149,7 +1149,7 @@ class TemplateDiff {
     if (!E)
       OS << "(no argument)";
     else
-      E->printPretty(OS, Context, 0, Policy); return;
+      E->printPretty(OS, 0, Policy); return;
   }
 
   /// PrintTemplateTemplate - Handles printing of template template arguments,
diff --git a/lib/AST/CommentCommandTraits.cpp b/lib/AST/CommentCommandTraits.cpp
index d8ce1f3..dc7a0bd 100644
--- a/lib/AST/CommentCommandTraits.cpp
+++ b/lib/AST/CommentCommandTraits.cpp
@@ -15,9 +15,9 @@ namespace comments {
 
 // TODO: tablegen
 
-bool CommandTraits::isVerbatimBlockCommand(StringRef BeginName,
+bool CommandTraits::isVerbatimBlockCommand(StringRef StartName,
                                            StringRef &EndName) const {
-  const char *Result = llvm::StringSwitch<const char *>(BeginName)
+  const char *Result = llvm::StringSwitch<const char *>(StartName)
     .Case("code", "endcode")
     .Case("verbatim", "endverbatim")
     .Case("htmlonly", "endhtmlonly")
@@ -44,7 +44,7 @@ bool CommandTraits::isVerbatimBlockCommand(StringRef BeginName,
            I = VerbatimBlockCommands.begin(),
            E = VerbatimBlockCommands.end();
        I != E; ++I)
-    if (I->BeginName == BeginName) {
+    if (I->StartName == StartName) {
       EndName = I->EndName;
       return true;
     }
@@ -115,10 +115,10 @@ bool CommandTraits::isDeclarationCommand(StringRef Name) const {
       .Default(false);
 }
 
-void CommandTraits::addVerbatimBlockCommand(StringRef BeginName,
+void CommandTraits::addVerbatimBlockCommand(StringRef StartName,
                                             StringRef EndName) {
   VerbatimBlockCommand VBC;
-  VBC.BeginName = BeginName;
+  VBC.StartName = StartName;
   VBC.EndName = EndName;
   VerbatimBlockCommands.push_back(VBC);
 }
diff --git a/lib/AST/DeclCXX.cpp b/lib/AST/DeclCXX.cpp
index eec2e9d..2f21e4c 100644
--- a/lib/AST/DeclCXX.cpp
+++ b/lib/AST/DeclCXX.cpp
@@ -1294,15 +1294,20 @@ static bool recursivelyOverrides(const CXXMethodDecl *DerivedMD,
 }
 
 CXXMethodDecl *
-CXXMethodDecl::getCorrespondingMethodInClass(const CXXRecordDecl *RD) {
+CXXMethodDecl::getCorrespondingMethodInClass(const CXXRecordDecl *RD,
+                                             bool MayBeBase) {
   if (this->getParent()->getCanonicalDecl() == RD->getCanonicalDecl())
     return this;
 
   // Lookup doesn't work for destructors, so handle them separately.
   if (isa<CXXDestructorDecl>(this)) {
     CXXMethodDecl *MD = RD->getDestructor();
-    if (MD && recursivelyOverrides(MD, this))
-      return MD;
+    if (MD) {
+      if (recursivelyOverrides(MD, this))
+        return MD;
+      if (MayBeBase && recursivelyOverrides(this, MD))
+        return MD;
+    }
     return NULL;
   }
 
@@ -1313,6 +1318,8 @@ CXXMethodDecl::getCorrespondingMethodInClass(const CXXRecordDecl *RD) {
       continue;
     if (recursivelyOverrides(MD, this))
       return MD;
+    if (MayBeBase && recursivelyOverrides(this, MD))
+      return MD;
   }
 
   for (CXXRecordDecl::base_class_const_iterator I = RD->bases_begin(),
diff --git a/lib/AST/DeclPrinter.cpp b/lib/AST/DeclPrinter.cpp
index aad0ca1..7f47604 100644
--- a/lib/AST/DeclPrinter.cpp
+++ b/lib/AST/DeclPrinter.cpp
@@ -26,7 +26,6 @@ using namespace clang;
 namespace {
   class DeclPrinter : public DeclVisitor<DeclPrinter> {
     raw_ostream &Out;
-    ASTContext &Context;
     PrintingPolicy Policy;
     unsigned Indentation;
     bool PrintInstantiation;
@@ -38,11 +37,9 @@ namespace {
     void Print(AccessSpecifier AS);
 
   public:
-    DeclPrinter(raw_ostream &Out, ASTContext &Context,
-                const PrintingPolicy &Policy,
-                unsigned Indentation = 0,
-                bool PrintInstantiation = false)
-      : Out(Out), Context(Context), Policy(Policy), Indentation(Indentation),
+    DeclPrinter(raw_ostream &Out, const PrintingPolicy &Policy,
+                unsigned Indentation = 0, bool PrintInstantiation = false)
+      : Out(Out), Policy(Policy), Indentation(Indentation),
         PrintInstantiation(PrintInstantiation) { }
 
     void VisitDeclContext(DeclContext *DC, bool Indent = true);
@@ -96,7 +93,7 @@ void Decl::print(raw_ostream &Out, unsigned Indentation,
 
 void Decl::print(raw_ostream &Out, const PrintingPolicy &Policy,
                  unsigned Indentation, bool PrintInstantiation) const {
-  DeclPrinter Printer(Out, getASTContext(), Policy, Indentation, PrintInstantiation);
+  DeclPrinter Printer(Out, Policy, Indentation, PrintInstantiation);
   Printer.Visit(const_cast<Decl*>(this));
 }
 
@@ -171,13 +168,17 @@ void DeclContext::dumpDeclContext() const {
     DC = DC->getParent();
   
   ASTContext &Ctx = cast<TranslationUnitDecl>(DC)->getASTContext();
-  DeclPrinter Printer(llvm::errs(), Ctx, Ctx.getPrintingPolicy(), 0);
+  DeclPrinter Printer(llvm::errs(), Ctx.getPrintingPolicy(), 0);
   Printer.VisitDeclContext(const_cast<DeclContext *>(this), /*Indent=*/false);
 }
 
+void Decl::dump() const {
+  dump(llvm::errs());
+}
+
 void Decl::dump(raw_ostream &Out) const {
   PrintingPolicy Policy = getASTContext().getPrintingPolicy();
-  Policy.Dump = true;
+  Policy.DumpSourceManager = &getASTContext().getSourceManager();
   print(Out, Policy, /*Indentation*/ 0, /*PrintInstantiation*/ true);
 }
 
@@ -191,8 +192,8 @@ void DeclPrinter::prettyPrintAttributes(Decl *D) {
   if (D->hasAttrs()) {
     AttrVec &Attrs = D->getAttrs();
     for (AttrVec::const_iterator i=Attrs.begin(), e=Attrs.end(); i!=e; ++i) {
-        Attr *A = *i;
-        A->printPretty(Out, Context);
+      Attr *A = *i;
+      A->printPretty(Out, Policy);
     }
   }
 }
@@ -231,7 +232,7 @@ void DeclPrinter::VisitDeclContext(DeclContext *DC, bool Indent) {
     if (isa<ObjCIvarDecl>(*D))
       continue;
 
-    if (!Policy.Dump) {
+    if (!Policy.DumpSourceManager) {
       // Skip over implicit declarations in pretty-printing mode.
       if (D->isImplicit()) continue;
       // FIXME: Ugly hack so we don't pretty-print the builtin declaration
@@ -381,7 +382,7 @@ void DeclPrinter::VisitEnumConstantDecl(EnumConstantDecl *D) {
   Out << *D;
   if (Expr *Init = D->getInitExpr()) {
     Out << " = ";
-    Init->printPretty(Out, Context, 0, Policy, Indentation);
+    Init->printPretty(Out, 0, Policy, Indentation);
   }
 }
 
@@ -420,7 +421,7 @@ void DeclPrinter::VisitFunctionDecl(FunctionDecl *D) {
     Proto += "(";
     if (FT) {
       llvm::raw_string_ostream POut(Proto);
-      DeclPrinter ParamPrinter(POut, Context, SubPolicy, Indentation);
+      DeclPrinter ParamPrinter(POut, SubPolicy, Indentation);
       for (unsigned i = 0, e = D->getNumParams(); i != e; ++i) {
         if (i) POut << ", ";
         ParamPrinter.VisitParmVarDecl(D->getParamDecl(i));
@@ -466,7 +467,7 @@ void DeclPrinter::VisitFunctionDecl(FunctionDecl *D) {
       if (FT->getExceptionSpecType() == EST_ComputedNoexcept) {
         Proto += "(";
         llvm::raw_string_ostream EOut(Proto);
-        FT->getNoexceptExpr()->printPretty(EOut, Context, 0, SubPolicy,
+        FT->getNoexceptExpr()->printPretty(EOut, 0, SubPolicy,
                                            Indentation);
         EOut.flush();
         Proto += EOut.str();
@@ -522,7 +523,7 @@ void DeclPrinter::VisitFunctionDecl(FunctionDecl *D) {
             SimpleInit = Init;
           
           if (SimpleInit)
-            SimpleInit->printPretty(Out, Context, 0, Policy, Indentation);
+            SimpleInit->printPretty(Out, 0, Policy, Indentation);
           else {
             for (unsigned I = 0; I != NumArgs; ++I) {
               if (isa<CXXDefaultArgExpr>(Args[I]))
@@ -530,7 +531,7 @@ void DeclPrinter::VisitFunctionDecl(FunctionDecl *D) {
               
               if (I)
                 Out << ", ";
-              Args[I]->printPretty(Out, Context, 0, Policy, Indentation);
+              Args[I]->printPretty(Out, 0, Policy, Indentation);
             }
           }
         }
@@ -554,7 +555,7 @@ void DeclPrinter::VisitFunctionDecl(FunctionDecl *D) {
       // This is a K&R function definition, so we need to print the
       // parameters.
       Out << '\n';
-      DeclPrinter ParamPrinter(Out, Context, SubPolicy, Indentation);
+      DeclPrinter ParamPrinter(Out, SubPolicy, Indentation);
       Indentation += Policy.Indentation;
       for (unsigned i = 0, e = D->getNumParams(); i != e; ++i) {
         Indent();
@@ -565,7 +566,7 @@ void DeclPrinter::VisitFunctionDecl(FunctionDecl *D) {
     } else
       Out << ' ';
 
-    D->getBody()->printPretty(Out, Context, 0, SubPolicy, Indentation);
+    D->getBody()->printPretty(Out, 0, SubPolicy, Indentation);
     Out << '\n';
   }
 }
@@ -580,7 +581,7 @@ void DeclPrinter::VisitFieldDecl(FieldDecl *D) {
 
   if (D->isBitField()) {
     Out << " : ";
-    D->getBitWidth()->printPretty(Out, Context, 0, Policy, Indentation);
+    D->getBitWidth()->printPretty(Out, 0, Policy, Indentation);
   }
 
   Expr *Init = D->getInClassInitializer();
@@ -589,7 +590,7 @@ void DeclPrinter::VisitFieldDecl(FieldDecl *D) {
       Out << " ";
     else
       Out << " = ";
-    Init->printPretty(Out, Context, 0, Policy, Indentation);
+    Init->printPretty(Out, 0, Policy, Indentation);
   }
   prettyPrintAttributes(D);
 }
@@ -625,7 +626,7 @@ void DeclPrinter::VisitVarDecl(VarDecl *D) {
       else if (D->getInitStyle() == VarDecl::CInit) {
         Out << " = ";
       }
-      Init->printPretty(Out, Context, 0, Policy, Indentation);
+      Init->printPretty(Out, 0, Policy, Indentation);
       if (D->getInitStyle() == VarDecl::CallInit)
         Out << ")";
     }
@@ -639,7 +640,7 @@ void DeclPrinter::VisitParmVarDecl(ParmVarDecl *D) {
 
 void DeclPrinter::VisitFileScopeAsmDecl(FileScopeAsmDecl *D) {
   Out << "__asm (";
-  D->getAsmString()->printPretty(Out, Context, 0, Policy, Indentation);
+  D->getAsmString()->printPretty(Out, 0, Policy, Indentation);
   Out << ")";
 }
 
@@ -650,9 +651,9 @@ void DeclPrinter::VisitImportDecl(ImportDecl *D) {
 
 void DeclPrinter::VisitStaticAssertDecl(StaticAssertDecl *D) {
   Out << "static_assert(";
-  D->getAssertExpr()->printPretty(Out, Context, 0, Policy, Indentation);
+  D->getAssertExpr()->printPretty(Out, 0, Policy, Indentation);
   Out << ", ";
-  D->getMessage()->printPretty(Out, Context, 0, Policy, Indentation);
+  D->getMessage()->printPretty(Out, 0, Policy, Indentation);
   Out << ")";
 }
 
@@ -786,8 +787,7 @@ void DeclPrinter::PrintTemplateParameters(
         Args->get(i).print(Policy, Out);
       } else if (NTTP->hasDefaultArgument()) {
         Out << " = ";
-        NTTP->getDefaultArgument()->printPretty(Out, Context, 0, Policy,
-                                                Indentation);
+        NTTP->getDefaultArgument()->printPretty(Out, 0, Policy, Indentation);
       }
     } else if (const TemplateTemplateParmDecl *TTPD =
                  dyn_cast<TemplateTemplateParmDecl>(Param)) {
@@ -871,7 +871,7 @@ void DeclPrinter::VisitObjCMethodDecl(ObjCMethodDecl *OMD) {
 
   if (OMD->getBody()) {
     Out << ' ';
-    OMD->getBody()->printPretty(Out, Context, 0, Policy);
+    OMD->getBody()->printPretty(Out, 0, Policy);
     Out << '\n';
   }
 }
diff --git a/lib/AST/DeclTemplate.cpp b/lib/AST/DeclTemplate.cpp
index 5aebc2b..a7e8999 100644
--- a/lib/AST/DeclTemplate.cpp
+++ b/lib/AST/DeclTemplate.cpp
@@ -43,7 +43,8 @@ TemplateParameterList::Create(const ASTContext &C, SourceLocation TemplateLoc,
                               unsigned NumParams, SourceLocation RAngleLoc) {
   unsigned Size = sizeof(TemplateParameterList) 
                 + sizeof(NamedDecl *) * NumParams;
-  unsigned Align = llvm::AlignOf<TemplateParameterList>::Alignment;
+  unsigned Align = std::max(llvm::alignOf<TemplateParameterList>(),
+                            llvm::alignOf<NamedDecl*>());
   void *Mem = C.Allocate(Size, Align);
   return new (Mem) TemplateParameterList(TemplateLoc, LAngleLoc, Params,
                                          NumParams, RAngleLoc);
diff --git a/lib/AST/DumpXML.cpp b/lib/AST/DumpXML.cpp
index c1432b5..84f3fc4 100644
--- a/lib/AST/DumpXML.cpp
+++ b/lib/AST/DumpXML.cpp
@@ -1022,12 +1022,17 @@ struct XMLDumper : public XMLDeclVisitor<XMLDumper>,
 };
 }
 
+void Decl::dumpXML() const {
+  dump(llvm::errs());
+}
+
 void Decl::dumpXML(raw_ostream &out) const {
   XMLDumper(out, getASTContext()).dispatch(const_cast<Decl*>(this));
 }
 
 #else /* ifndef NDEBUG */
 
+void Decl::dumpXML() const {}
 void Decl::dumpXML(raw_ostream &out) const {}
 
 #endif
diff --git a/lib/AST/NestedNameSpecifier.cpp b/lib/AST/NestedNameSpecifier.cpp
index dbf267b..49b119b 100644
--- a/lib/AST/NestedNameSpecifier.cpp
+++ b/lib/AST/NestedNameSpecifier.cpp
@@ -18,6 +18,7 @@
 #include "clang/AST/PrettyPrinter.h"
 #include "clang/AST/Type.h"
 #include "clang/AST/TypeLoc.h"
+#include "llvm/Support/AlignOf.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cassert>
 
@@ -33,7 +34,8 @@ NestedNameSpecifier::FindOrInsert(const ASTContext &Context,
   NestedNameSpecifier *NNS
     = Context.NestedNameSpecifiers.FindNodeOrInsertPos(ID, InsertPos);
   if (!NNS) {
-    NNS = new (Context, 4) NestedNameSpecifier(Mockup);
+    NNS = new (Context, llvm::alignOf<NestedNameSpecifier>())
+        NestedNameSpecifier(Mockup);
     Context.NestedNameSpecifiers.InsertNode(NNS, InsertPos);
   }
 
@@ -107,7 +109,9 @@ NestedNameSpecifier::Create(const ASTContext &Context, IdentifierInfo *II) {
 NestedNameSpecifier *
 NestedNameSpecifier::GlobalSpecifier(const ASTContext &Context) {
   if (!Context.GlobalNestedNameSpecifier)
-    Context.GlobalNestedNameSpecifier = new (Context, 4) NestedNameSpecifier();
+    Context.GlobalNestedNameSpecifier =
+        new (Context, llvm::alignOf<NestedNameSpecifier>())
+            NestedNameSpecifier();
   return Context.GlobalNestedNameSpecifier;
 }
 
@@ -630,4 +634,3 @@ NestedNameSpecifierLocBuilder::getWithLocInContext(ASTContext &Context) const {
   memcpy(Mem, Buffer, BufferSize);
   return NestedNameSpecifierLoc(Representation, Mem);
 }
-
diff --git a/lib/AST/RawCommentList.cpp b/lib/AST/RawCommentList.cpp
index c704cab..a5a3287 100644
--- a/lib/AST/RawCommentList.cpp
+++ b/lib/AST/RawCommentList.cpp
@@ -65,7 +65,7 @@ bool mergedCommentIsTrailingComment(StringRef Comment) {
 RawComment::RawComment(const SourceManager &SourceMgr, SourceRange SR,
                        bool Merged) :
     Range(SR), RawTextValid(false), BriefTextValid(false),
-    IsAlmostTrailingComment(false),
+    IsAttached(false), IsAlmostTrailingComment(false),
     BeginLineValid(false), EndLineValid(false) {
   // Extract raw comment text, if possible.
   if (SR.getBegin() == SR.getEnd() || getRawText(SourceMgr).empty()) {
@@ -87,16 +87,6 @@ RawComment::RawComment(const SourceManager &SourceMgr, SourceRange SR,
   }
 }
 
-const Decl *RawComment::getDecl() const {
-  if (DeclOrParsedComment.isNull())
-    return NULL;
-
-  if (const Decl *D = DeclOrParsedComment.dyn_cast<const Decl *>())
-    return D;
-
-  return DeclOrParsedComment.get<comments::FullComment *>()->getDecl();
-}
-
 unsigned RawComment::getBeginLine(const SourceManager &SM) const {
   if (BeginLineValid)
     return BeginLine;
@@ -169,7 +159,8 @@ const char *RawComment::extractBriefText(const ASTContext &Context) const {
   return BriefTextPtr;
 }
 
-comments::FullComment *RawComment::parse(const ASTContext &Context) const {
+comments::FullComment *RawComment::parse(const ASTContext &Context,
+                                         const Decl *D) const {
   // Make sure that RawText is valid.
   getRawText(Context.getSourceManager());
 
@@ -179,13 +170,11 @@ comments::FullComment *RawComment::parse(const ASTContext &Context) const {
                     RawText.begin(), RawText.end());
   comments::Sema S(Context.getAllocator(), Context.getSourceManager(),
                    Context.getDiagnostics(), Traits);
-  S.setDecl(getDecl());
+  S.setDecl(D);
   comments::Parser P(L, S, Context.getAllocator(), Context.getSourceManager(),
                      Context.getDiagnostics(), Traits);
 
-  comments::FullComment *FC = P.parseFullComment();
-  DeclOrParsedComment = FC;
-  return FC;
+  return P.parseFullComment();
 }
 
 namespace {
diff --git a/lib/AST/Stmt.cpp b/lib/AST/Stmt.cpp
index d877c3f..77452c9 100644
--- a/lib/AST/Stmt.cpp
+++ b/lib/AST/Stmt.cpp
@@ -584,22 +584,27 @@ AsmStmt::AsmStmt(ASTContext &C, SourceLocation asmloc, bool issimple,
 }
 
 MSAsmStmt::MSAsmStmt(ASTContext &C, SourceLocation asmloc,
-                     bool issimple, bool isvolatile, ArrayRef<Token> asmtoks,
-                     ArrayRef<unsigned> lineends, StringRef asmstr,
+                     SourceLocation lbraceloc, bool issimple, bool isvolatile,
+                     ArrayRef<Token> asmtoks, ArrayRef<IdentifierInfo*> inputs,
+                     ArrayRef<IdentifierInfo*> outputs, StringRef asmstr,
                      ArrayRef<StringRef> clobbers, SourceLocation endloc)
-  : Stmt(MSAsmStmtClass), AsmLoc(asmloc), EndLoc(endloc),
+  : Stmt(MSAsmStmtClass), AsmLoc(asmloc), LBraceLoc(lbraceloc), EndLoc(endloc),
     AsmStr(asmstr.str()), IsSimple(issimple), IsVolatile(isvolatile),
-    NumAsmToks(asmtoks.size()), NumLineEnds(lineends.size()),
-    NumClobbers(clobbers.size()) {
+    NumAsmToks(asmtoks.size()), NumInputs(inputs.size()),
+    NumOutputs(outputs.size()), NumClobbers(clobbers.size()) {
+
+  unsigned NumExprs = NumOutputs + NumInputs;
+
+  Names = new (C) IdentifierInfo*[NumExprs];
+  for (unsigned i = 0, e = NumOutputs; i != e; ++i)
+    Names[i] = outputs[i];
+  for (unsigned i = NumOutputs, e = NumExprs; i != e; ++i)
+    Names[i] = inputs[i];
 
   AsmToks = new (C) Token[NumAsmToks];
   for (unsigned i = 0, e = NumAsmToks; i != e; ++i)
     AsmToks[i] = asmtoks[i];
 
-  LineEnds = new (C) unsigned[NumLineEnds];
-  for (unsigned i = 0, e = NumLineEnds; i != e; ++i)
-    LineEnds[i] = lineends[i];
-
   Clobbers = new (C) StringRef[NumClobbers];
   for (unsigned i = 0, e = NumClobbers; i != e; ++i) {
     // FIXME: Avoid the allocation/copy if at all possible.
diff --git a/lib/AST/StmtPrinter.cpp b/lib/AST/StmtPrinter.cpp
index 2f7cb55..c0960ce 100644
--- a/lib/AST/StmtPrinter.cpp
+++ b/lib/AST/StmtPrinter.cpp
@@ -30,17 +30,15 @@ using namespace clang;
 namespace  {
   class StmtPrinter : public StmtVisitor<StmtPrinter> {
     raw_ostream &OS;
-    ASTContext &Context;
     unsigned IndentLevel;
     clang::PrinterHelper* Helper;
     PrintingPolicy Policy;
 
   public:
-    StmtPrinter(raw_ostream &os, ASTContext &C, PrinterHelper* helper,
+    StmtPrinter(raw_ostream &os, PrinterHelper* helper,
                 const PrintingPolicy &Policy,
                 unsigned Indentation = 0)
-      : OS(os), Context(C), IndentLevel(Indentation), Helper(helper),
-        Policy(Policy) {}
+      : OS(os), IndentLevel(Indentation), Helper(helper), Policy(Policy) {}
 
     void PrintStmt(Stmt *S) {
       PrintStmt(S, Policy.Indentation);
@@ -181,7 +179,7 @@ void StmtPrinter::VisitAttributedStmt(AttributedStmt *Node) {
       first = false;
     }
     // TODO: check this
-    (*it)->printPretty(OS, Context);
+    (*it)->printPretty(OS, Policy);
   }
   OS << "]] ";
   PrintStmt(Node->getSubStmt(), 0);
@@ -432,7 +430,12 @@ void StmtPrinter::VisitAsmStmt(AsmStmt *Node) {
 
 void StmtPrinter::VisitMSAsmStmt(MSAsmStmt *Node) {
   // FIXME: Implement MS style inline asm statement printer.
-  Indent() << "asm ()";
+  Indent() << "__asm ";
+  if (Node->hasBraces())
+    OS << "{\n";
+  OS << *(Node->getAsmString()) << "\n";
+  if (Node->hasBraces())
+    Indent() << "}\n";
 }
 
 void StmtPrinter::VisitObjCAtTryStmt(ObjCAtTryStmt *Node) {
@@ -1390,7 +1393,7 @@ void StmtPrinter::VisitCXXNewExpr(CXXNewExpr *E) {
   std::string TypeS;
   if (Expr *Size = E->getArraySize()) {
     llvm::raw_string_ostream s(TypeS);
-    Size->printPretty(s, Context, Helper, Policy);
+    Size->printPretty(s, Helper, Policy);
     s.flush();
     TypeS = "[" + TypeS + "]";
   }
@@ -1799,13 +1802,12 @@ void StmtPrinter::VisitAsTypeExpr(AsTypeExpr *Node) {
 // Stmt method implementations
 //===----------------------------------------------------------------------===//
 
-void Stmt::dumpPretty(ASTContext& Context) const {
-  printPretty(llvm::errs(), Context, 0,
-              PrintingPolicy(Context.getLangOpts()));
+void Stmt::dumpPretty(ASTContext &Context) const {
+  printPretty(llvm::errs(), 0, PrintingPolicy(Context.getLangOpts()));
 }
 
-void Stmt::printPretty(raw_ostream &OS, ASTContext& Context,
-                       PrinterHelper* Helper,
+void Stmt::printPretty(raw_ostream &OS,
+                       PrinterHelper *Helper,
                        const PrintingPolicy &Policy,
                        unsigned Indentation) const {
   if (this == 0) {
@@ -1813,12 +1815,12 @@ void Stmt::printPretty(raw_ostream &OS, ASTContext& Context,
     return;
   }
 
-  if (Policy.Dump && &Context) {
-    dump(OS, Context.getSourceManager());
+  if (Policy.DumpSourceManager) {
+    dump(OS, *Policy.DumpSourceManager);
     return;
   }
 
-  StmtPrinter P(OS, Context, Helper, Policy, Indentation);
+  StmtPrinter P(OS, Helper, Policy, Indentation);
   P.Visit(const_cast<Stmt*>(this));
 }
 
diff --git a/lib/AST/TemplateBase.cpp b/lib/AST/TemplateBase.cpp
index f8dd396..95ff4ed 100644
--- a/lib/AST/TemplateBase.cpp
+++ b/lib/AST/TemplateBase.cpp
@@ -556,8 +556,7 @@ const DiagnosticBuilder &clang::operator<<(const DiagnosticBuilder &DB,
 const ASTTemplateArgumentListInfo *
 ASTTemplateArgumentListInfo::Create(ASTContext &C,
                                     const TemplateArgumentListInfo &List) {
-  std::size_t size = sizeof(CXXDependentScopeMemberExpr) +
-                     ASTTemplateArgumentListInfo::sizeFor(List.size());
+  std::size_t size = ASTTemplateArgumentListInfo::sizeFor(List.size());
   void *Mem = C.Allocate(size, llvm::alignOf<ASTTemplateArgumentListInfo>());
   ASTTemplateArgumentListInfo *TAI = new (Mem) ASTTemplateArgumentListInfo();
   TAI->initializeFrom(List);
@@ -642,6 +641,7 @@ ASTTemplateKWAndArgsInfo::initializeFrom(SourceLocation TemplateKWLoc) {
 std::size_t
 ASTTemplateKWAndArgsInfo::sizeFor(unsigned NumTemplateArgs) {
   // Add space for the template keyword location.
+  // FIXME: There's room for this in the padding before the template args in
+  //        64-bit builds.
   return Base::sizeFor(NumTemplateArgs) + sizeof(SourceLocation);
 }
-
diff --git a/lib/Basic/Diagnostic.cpp b/lib/Basic/Diagnostic.cpp
index e689502..8065b2d 100644
--- a/lib/Basic/Diagnostic.cpp
+++ b/lib/Basic/Diagnostic.cpp
@@ -145,6 +145,9 @@ DiagnosticsEngine::GetDiagStatePointForLoc(SourceLocation L) const {
   assert(DiagStatePoints.front().Loc.isInvalid() &&
          "Should have created a DiagStatePoint for command-line");
 
+  if (!SourceMgr)
+    return DiagStatePoints.end() - 1;
+
   FullSourceLoc Loc(L, *SourceMgr);
   if (Loc.isInvalid())
     return DiagStatePoints.end() - 1;
@@ -167,8 +170,9 @@ void DiagnosticsEngine::setDiagnosticMapping(diag::kind Diag, diag::Mapping Map,
           (Map == diag::MAP_FATAL || Map == diag::MAP_ERROR)) &&
          "Cannot map errors into warnings!");
   assert(!DiagStatePoints.empty());
+  assert((L.isInvalid() || SourceMgr) && "No SourceMgr for valid location");
 
-  FullSourceLoc Loc(L, *SourceMgr);
+  FullSourceLoc Loc = SourceMgr? FullSourceLoc(L, *SourceMgr) : FullSourceLoc();
   FullSourceLoc LastStateChangePos = DiagStatePoints.back().Loc;
   // Don't allow a mapping to a warning override an error/fatal mapping.
   if (Map == diag::MAP_WARNING) {
diff --git a/lib/Basic/Targets.cpp b/lib/Basic/Targets.cpp
index 883864f..1d495f1 100644
--- a/lib/Basic/Targets.cpp
+++ b/lib/Basic/Targets.cpp
@@ -3083,9 +3083,8 @@ public:
                                 unsigned &NumAliases) const;
   virtual bool validateAsmConstraint(const char *&Name,
                                      TargetInfo::ConstraintInfo &Info) const {
-    // FIXME: Check if this is complete
     switch (*Name) {
-    default:
+    default: break;
     case 'l': // r0-r7
     case 'h': // r8-r15
     case 'w': // VFP Floating point register single precision
diff --git a/lib/CodeGen/CGBuiltin.cpp b/lib/CodeGen/CGBuiltin.cpp
index 65c782e..59ed313 100644
--- a/lib/CodeGen/CGBuiltin.cpp
+++ b/lib/CodeGen/CGBuiltin.cpp
@@ -229,6 +229,35 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
 
     return RValue::get(Result);
   }
+  
+  case Builtin::BI__builtin_conj:
+  case Builtin::BI__builtin_conjf:
+  case Builtin::BI__builtin_conjl: {
+    ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
+    Value *Real = ComplexVal.first;
+    Value *Imag = ComplexVal.second;
+    Value *Zero = 
+      Imag->getType()->isFPOrFPVectorTy() 
+        ? llvm::ConstantFP::getZeroValueForNegation(Imag->getType())
+        : llvm::Constant::getNullValue(Imag->getType());
+    
+    Imag = Builder.CreateFSub(Zero, Imag, "sub");
+    return RValue::getComplex(std::make_pair(Real, Imag));
+  }
+  case Builtin::BI__builtin_creal:
+  case Builtin::BI__builtin_crealf:
+  case Builtin::BI__builtin_creall: {
+    ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
+    return RValue::get(ComplexVal.first);
+  }
+      
+  case Builtin::BI__builtin_cimag:
+  case Builtin::BI__builtin_cimagf:
+  case Builtin::BI__builtin_cimagl: {
+    ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
+    return RValue::get(ComplexVal.second);
+  }
+      
   case Builtin::BI__builtin_ctzs:
   case Builtin::BI__builtin_ctz:
   case Builtin::BI__builtin_ctzl:
@@ -1720,8 +1749,29 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     Ops.push_back(GetPointeeAlignmentValue(E->getArg(0)));
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vld1, Ty),
                         Ops, "vld1");
-  case ARM::BI__builtin_neon_vld1_lane_v:
-  case ARM::BI__builtin_neon_vld1q_lane_v: {
+  case ARM::BI__builtin_neon_vld1q_lane_v:
+    // Handle 64-bit integer elements as a special case.  Use shuffles of
+    // one-element vectors to avoid poor code for i64 in the backend.
+    if (VTy->getElementType()->isIntegerTy(64)) {
+      // Extract the other lane.
+      Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
+      int Lane = cast<ConstantInt>(Ops[2])->getZExtValue();
+      Value *SV = llvm::ConstantVector::get(ConstantInt::get(Int32Ty, 1-Lane));
+      Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
+      // Load the value as a one-element vector.
+      Ty = llvm::VectorType::get(VTy->getElementType(), 1);
+      Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld1, Ty);
+      Value *Ld = Builder.CreateCall2(F, Ops[0],
+                                      GetPointeeAlignmentValue(E->getArg(0)));
+      // Combine them.
+      SmallVector<Constant*, 2> Indices;
+      Indices.push_back(ConstantInt::get(Int32Ty, 1-Lane));
+      Indices.push_back(ConstantInt::get(Int32Ty, Lane));
+      SV = llvm::ConstantVector::get(Indices);
+      return Builder.CreateShuffleVector(Ops[1], Ld, SV, "vld1q_lane");
+    }
+    // fall through
+  case ARM::BI__builtin_neon_vld1_lane_v: {
     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
     Ty = llvm::PointerType::getUnqual(VTy->getElementType());
     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
@@ -2086,8 +2136,19 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     Ops.push_back(GetPointeeAlignmentValue(E->getArg(0)));
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst1, Ty),
                         Ops, "");
-  case ARM::BI__builtin_neon_vst1_lane_v:
-  case ARM::BI__builtin_neon_vst1q_lane_v: {
+  case ARM::BI__builtin_neon_vst1q_lane_v:
+    // Handle 64-bit integer elements as a special case.  Use a shuffle to get
+    // a one-element vector and avoid poor code for i64 in the backend.
+    if (VTy->getElementType()->isIntegerTy(64)) {
+      Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
+      Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2]));
+      Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
+      Ops[2] = GetPointeeAlignmentValue(E->getArg(0));
+      return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst1,
+                                                 Ops[1]->getType()), Ops);
+    }
+    // fall through
+  case ARM::BI__builtin_neon_vst1_lane_v: {
     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
     Ops[1] = Builder.CreateExtractElement(Ops[1], Ops[2]);
     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
diff --git a/lib/CodeGen/CGDebugInfo.cpp b/lib/CodeGen/CGDebugInfo.cpp
index 00127ac..fd1c7a3 100644
--- a/lib/CodeGen/CGDebugInfo.cpp
+++ b/lib/CodeGen/CGDebugInfo.cpp
@@ -94,8 +94,10 @@ llvm::DIDescriptor CGDebugInfo::getContextDescriptor(const Decl *Context) {
 
   llvm::DenseMap<const Decl *, llvm::WeakVH>::iterator
     I = RegionMap.find(Context);
-  if (I != RegionMap.end())
-    return llvm::DIDescriptor(dyn_cast_or_null<llvm::MDNode>(&*I->second));
+  if (I != RegionMap.end()) {
+    llvm::Value *V = I->second;
+    return llvm::DIDescriptor(dyn_cast_or_null<llvm::MDNode>(V));
+  }
 
   // Check namespace.
   if (const NamespaceDecl *NSDecl = dyn_cast<NamespaceDecl>(Context))
@@ -227,8 +229,8 @@ llvm::DIFile CGDebugInfo::getOrCreateFile(SourceLocation Loc) {
 
   if (it != DIFileCache.end()) {
     // Verify that the information still exists.
-    if (&*it->second)
-      return llvm::DIFile(cast<llvm::MDNode>(it->second));
+    if (llvm::Value *V = it->second)
+      return llvm::DIFile(cast<llvm::MDNode>(V));
   }
 
   llvm::DIFile F = DBuilder.createFile(PLoc.getFilename(), getCurrentDirname());
@@ -525,8 +527,10 @@ llvm::DIDescriptor CGDebugInfo::createContextChain(const Decl *Context) {
   // See if we already have the parent.
   llvm::DenseMap<const Decl *, llvm::WeakVH>::iterator
     I = RegionMap.find(Context);
-  if (I != RegionMap.end())
-    return llvm::DIDescriptor(dyn_cast_or_null<llvm::MDNode>(&*I->second));
+  if (I != RegionMap.end()) {
+    llvm::Value *V = I->second;
+    return llvm::DIDescriptor(dyn_cast_or_null<llvm::MDNode>(V));
+  }
   
   // Check namespace.
   if (const NamespaceDecl *NSDecl = dyn_cast<NamespaceDecl>(Context))
@@ -1660,8 +1664,8 @@ llvm::DIType CGDebugInfo::getTypeOrNull(QualType Ty) {
     TypeCache.find(Ty.getAsOpaquePtr());
   if (it != TypeCache.end()) {
     // Verify that the debug info still exists.
-    if (&*it->second)
-      return llvm::DIType(cast<llvm::MDNode>(it->second));
+    if (llvm::Value *V = it->second)
+      return llvm::DIType(cast<llvm::MDNode>(V));
   }
 
   return llvm::DIType();
@@ -1679,8 +1683,8 @@ llvm::DIType CGDebugInfo::getCompletedTypeOrNull(QualType Ty) {
     CompletedTypeCache.find(Ty.getAsOpaquePtr());
   if (it != CompletedTypeCache.end()) {
     // Verify that the debug info still exists.
-    if (&*it->second)
-      return llvm::DIType(cast<llvm::MDNode>(it->second));
+    if (llvm::Value *V = it->second)
+      return llvm::DIType(cast<llvm::MDNode>(V));
   }
 
   return llvm::DIType();
@@ -1942,7 +1946,8 @@ llvm::DISubprogram CGDebugInfo::getFunctionDeclaration(const Decl *D) {
   llvm::DenseMap<const FunctionDecl *, llvm::WeakVH>::iterator
     MI = SPCache.find(FD->getCanonicalDecl());
   if (MI != SPCache.end()) {
-    llvm::DISubprogram SP(dyn_cast_or_null<llvm::MDNode>(&*MI->second));
+    llvm::Value *V = MI->second;
+    llvm::DISubprogram SP(dyn_cast_or_null<llvm::MDNode>(V));
     if (SP.isSubprogram() && !llvm::DISubprogram(SP).isDefinition())
       return SP;
   }
@@ -1953,7 +1958,8 @@ llvm::DISubprogram CGDebugInfo::getFunctionDeclaration(const Decl *D) {
     llvm::DenseMap<const FunctionDecl *, llvm::WeakVH>::iterator
       MI = SPCache.find(NextFD->getCanonicalDecl());
     if (MI != SPCache.end()) {
-      llvm::DISubprogram SP(dyn_cast_or_null<llvm::MDNode>(&*MI->second));
+      llvm::Value *V = MI->second;
+      llvm::DISubprogram SP(dyn_cast_or_null<llvm::MDNode>(V));
       if (SP.isSubprogram() && !llvm::DISubprogram(SP).isDefinition())
         return SP;
     }
@@ -2013,7 +2019,8 @@ void CGDebugInfo::EmitFunctionStart(GlobalDecl GD, QualType FnType,
     llvm::DenseMap<const FunctionDecl *, llvm::WeakVH>::iterator
       FI = SPCache.find(FD->getCanonicalDecl());
     if (FI != SPCache.end()) {
-      llvm::DIDescriptor SP(dyn_cast_or_null<llvm::MDNode>(&*FI->second));
+      llvm::Value *V = FI->second;
+      llvm::DIDescriptor SP(dyn_cast_or_null<llvm::MDNode>(V));
       if (SP.isSubprogram() && llvm::DISubprogram(SP).isDefinition()) {
         llvm::MDNode *SPN = SP;
         LexicalBlockStack.push_back(SPN);
@@ -2701,15 +2708,15 @@ void CGDebugInfo::finalize(void) {
          = ReplaceMap.begin(), VE = ReplaceMap.end(); VI != VE; ++VI) {
     llvm::DIType Ty, RepTy;
     // Verify that the debug info still exists.
-    if (&*VI->second)
-      Ty = llvm::DIType(cast<llvm::MDNode>(VI->second));
+    if (llvm::Value *V = VI->second)
+      Ty = llvm::DIType(cast<llvm::MDNode>(V));
     
     llvm::DenseMap<void *, llvm::WeakVH>::iterator it =
       TypeCache.find(VI->first);
     if (it != TypeCache.end()) {
       // Verify that the debug info still exists.
-      if (&*it->second)
-        RepTy = llvm::DIType(cast<llvm::MDNode>(it->second));
+      if (llvm::Value *V = it->second)
+        RepTy = llvm::DIType(cast<llvm::MDNode>(V));
     }
     
     if (Ty.Verify() && Ty.isForwardDecl() && RepTy.Verify()) {
diff --git a/lib/CodeGen/CGExpr.cpp b/lib/CodeGen/CGExpr.cpp
index ecee7b4..1fe4c18 100644
--- a/lib/CodeGen/CGExpr.cpp
+++ b/lib/CodeGen/CGExpr.cpp
@@ -938,6 +938,50 @@ llvm::MDNode *CodeGenFunction::getRangeForLoadFromType(QualType Ty) {
 llvm::Value *CodeGenFunction::EmitLoadOfScalar(llvm::Value *Addr, bool Volatile,
                                               unsigned Alignment, QualType Ty,
                                               llvm::MDNode *TBAAInfo) {
+  
+  // For better performance, handle vector loads differently.
+  if (Ty->isVectorType()) {
+    llvm::Value *V;
+    const llvm::Type *EltTy =
+    cast<llvm::PointerType>(Addr->getType())->getElementType();
+    
+    const llvm::VectorType *VTy = cast<llvm::VectorType>(EltTy);
+      
+    // Handle vectors of size 3, like size 4 for better performance.
+    if (VTy->getNumElements() == 3) {
+        
+      // Bitcast to vec4 type.
+      llvm::VectorType *vec4Ty = llvm::VectorType::get(VTy->getElementType(),
+                                                         4);
+      llvm::PointerType *ptVec4Ty =
+      llvm::PointerType::get(vec4Ty,
+                             (cast<llvm::PointerType>(
+                                      Addr->getType()))->getAddressSpace());
+      llvm::Value *Cast = Builder.CreateBitCast(Addr, ptVec4Ty,
+                                                "castToVec4");
+      // Now load value.
+      llvm::Value *LoadVal = Builder.CreateLoad(Cast, Volatile, "loadVec4");
+        
+      // Shuffle vector to get vec3.
+      llvm::SmallVector<llvm::Constant*, 3> Mask;
+      Mask.push_back(llvm::ConstantInt::get(
+                                    llvm::Type::getInt32Ty(getLLVMContext()),
+                                            0));
+      Mask.push_back(llvm::ConstantInt::get(
+                                    llvm::Type::getInt32Ty(getLLVMContext()),
+                                            1));
+      Mask.push_back(llvm::ConstantInt::get(
+                                     llvm::Type::getInt32Ty(getLLVMContext()),
+                                            2));
+        
+      llvm::Value *MaskV = llvm::ConstantVector::get(Mask);
+      V = Builder.CreateShuffleVector(LoadVal,
+                                      llvm::UndefValue::get(vec4Ty),
+                                      MaskV, "extractVec");
+      return EmitFromMemory(V, Ty);
+    }
+  }
+  
   llvm::LoadInst *Load = Builder.CreateLoad(Addr);
   if (Volatile)
     Load->setVolatile(true);
@@ -984,6 +1028,42 @@ void CodeGenFunction::EmitStoreOfScalar(llvm::Value *Value, llvm::Value *Addr,
                                         QualType Ty,
                                         llvm::MDNode *TBAAInfo,
                                         bool isInit) {
+  
+  // Handle vectors differently to get better performance.
+  if (Ty->isVectorType()) {
+    llvm::Type *SrcTy = Value->getType();
+    llvm::VectorType *VecTy = cast<llvm::VectorType>(SrcTy);
+    // Handle vec3 special.
+    if (VecTy->getNumElements() == 3) {
+      llvm::LLVMContext &VMContext = getLLVMContext();
+      
+      // Our source is a vec3, do a shuffle vector to make it a vec4.
+      llvm::SmallVector<llvm::Constant*, 4> Mask;
+      Mask.push_back(llvm::ConstantInt::get(
+                                            llvm::Type::getInt32Ty(VMContext),
+                                            0));
+      Mask.push_back(llvm::ConstantInt::get(
+                                            llvm::Type::getInt32Ty(VMContext),
+                                            1));
+      Mask.push_back(llvm::ConstantInt::get(
+                                            llvm::Type::getInt32Ty(VMContext),
+                                            2));
+      Mask.push_back(llvm::UndefValue::get(llvm::Type::getInt32Ty(VMContext)));
+      
+      llvm::Value *MaskV = llvm::ConstantVector::get(Mask);
+      Value = Builder.CreateShuffleVector(Value,
+                                          llvm::UndefValue::get(VecTy),
+                                          MaskV, "extractVec");
+      SrcTy = llvm::VectorType::get(VecTy->getElementType(), 4);
+    }
+    llvm::PointerType *DstPtr = cast<llvm::PointerType>(Addr->getType());
+    if (DstPtr->getElementType() != SrcTy) {
+      llvm::Type *MemTy =
+      llvm::PointerType::get(SrcTy, DstPtr->getAddressSpace());
+      Addr = Builder.CreateBitCast(Addr, MemTy, "storetmp");
+    }
+  }
+  
   Value = EmitToMemory(Value, Ty);
   
   llvm::StoreInst *Store = Builder.CreateStore(Value, Addr, Volatile);
diff --git a/lib/CodeGen/CGExprCXX.cpp b/lib/CodeGen/CGExprCXX.cpp
index 7c2c9f1..31ea1b5 100644
--- a/lib/CodeGen/CGExprCXX.cpp
+++ b/lib/CodeGen/CGExprCXX.cpp
@@ -123,7 +123,14 @@ static bool canDevirtualizeMemberFunctionCalls(ASTContext &Context,
     
     return false;
   }
-  
+
+  // We can devirtualize calls on an object accessed by a class member access
+  // expression, since by C++11 [basic.life]p6 we know that it can't refer to
+  // a derived class object constructed in the same location.
+  if (const MemberExpr *ME = dyn_cast<MemberExpr>(Base))
+    if (const ValueDecl *VD = dyn_cast<ValueDecl>(ME->getMemberDecl()))
+      return VD->getType()->isRecordType();
+
   // We can always devirtualize calls on temporary object expressions.
   if (isa<CXXConstructExpr>(Base))
     return true;
diff --git a/lib/CodeGen/CGStmt.cpp b/lib/CodeGen/CGStmt.cpp
index 467c779..d78908d 100644
--- a/lib/CodeGen/CGStmt.cpp
+++ b/lib/CodeGen/CGStmt.cpp
@@ -1691,14 +1691,36 @@ void CodeGenFunction::EmitMSAsmStmt(const MSAsmStmt &S) {
 
   std::vector<llvm::Value*> Args;
   std::vector<llvm::Type *> ArgTypes;
+  std::string Constraints;
+
+  // Clobbers
+  for (unsigned i = 0, e = S.getNumClobbers(); i != e; ++i) {
+    StringRef Clobber = S.getClobber(i);
+
+    if (Clobber != "memory" && Clobber != "cc")
+      Clobber = Target.getNormalizedGCCRegisterName(Clobber);
+
+    if (i != 0)
+      Constraints += ',';
+
+    Constraints += "~{";
+    Constraints += Clobber;
+    Constraints += '}';
+  }
 
+  // Add machine specific clobbers
   std::string MachineClobbers = Target.getClobbers();
+  if (!MachineClobbers.empty()) {
+    if (!Constraints.empty())
+      Constraints += ',';
+    Constraints += MachineClobbers;
+  }
 
   llvm::FunctionType *FTy =
     llvm::FunctionType::get(VoidTy, ArgTypes, false);
 
   llvm::InlineAsm *IA =
-    llvm::InlineAsm::get(FTy, *S.getAsmString(), MachineClobbers, true);
+    llvm::InlineAsm::get(FTy, *S.getAsmString(), Constraints, true);
   llvm::CallInst *Result = Builder.CreateCall(IA, Args);
   Result->addAttribute(~0, llvm::Attribute::NoUnwind);
   Result->addAttribute(~0, llvm::Attribute::IANSDialect);
diff --git a/lib/CodeGen/CGValue.h b/lib/CodeGen/CGValue.h
index a46f313..c2b8e4d 100644
--- a/lib/CodeGen/CGValue.h
+++ b/lib/CodeGen/CGValue.h
@@ -128,7 +128,7 @@ class LValue {
 
   // The alignment to use when accessing this lvalue.  (For vector elements,
   // this is the alignment of the whole vector.)
-  unsigned short Alignment;
+  int64_t Alignment;
 
   // objective-c's ivar
   bool Ivar:1;
diff --git a/lib/Driver/Tools.cpp b/lib/Driver/Tools.cpp
index b4234cf..ed67f7b 100644
--- a/lib/Driver/Tools.cpp
+++ b/lib/Driver/Tools.cpp
@@ -5723,6 +5723,9 @@ void linuxtools::Link::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back(Args.MakeArgString(Plugin));
   }
 
+  if (Args.hasArg(options::OPT_Z_Xlinker__no_demangle))
+    CmdArgs.push_back("--no-demangle");
+
   AddLinkerInputs(ToolChain, Inputs, Args, CmdArgs);
 
   if (D.CCCIsCXX &&
diff --git a/lib/Frontend/ASTConsumers.cpp b/lib/Frontend/ASTConsumers.cpp
index bb1a4e6..0f0d835 100644
--- a/lib/Frontend/ASTConsumers.cpp
+++ b/lib/Frontend/ASTConsumers.cpp
@@ -89,7 +89,7 @@ namespace {
 
   class ASTDeclNodeLister : public ASTConsumer,
                      public RecursiveASTVisitor<ASTDeclNodeLister> {
-    typedef RecursiveASTVisitor<ASTPrinter> base;
+    typedef RecursiveASTVisitor<ASTDeclNodeLister> base;
 
   public:
     ASTDeclNodeLister(raw_ostream *Out = NULL)
diff --git a/lib/Frontend/CacheTokens.cpp b/lib/Frontend/CacheTokens.cpp
index 58a6b8d..3e66613 100644
--- a/lib/Frontend/CacheTokens.cpp
+++ b/lib/Frontend/CacheTokens.cpp
@@ -447,7 +447,7 @@ Offset PTHWriter::EmitCachedSpellings() {
 
 void PTHWriter::GeneratePTH(const std::string &MainFile) {
   // Generate the prologue.
-  Out << "cfe-pth";
+  Out << "cfe-pth" << '\0';
   Emit32(PTHManager::Version);
 
   // Leave 4 words for the prologue.
diff --git a/lib/Lex/PTHLexer.cpp b/lib/Lex/PTHLexer.cpp
index f104f96..67738e9 100644
--- a/lib/Lex/PTHLexer.cpp
+++ b/lib/Lex/PTHLexer.cpp
@@ -452,14 +452,14 @@ PTHManager *PTHManager::Create(const std::string &file,
   const unsigned char *BufEnd = (unsigned char*)File->getBufferEnd();
 
   // Check the prologue of the file.
-  if ((BufEnd - BufBeg) < (signed)(sizeof("cfe-pth") + 3 + 4) ||
-      memcmp(BufBeg, "cfe-pth", sizeof("cfe-pth") - 1) != 0) {
+  if ((BufEnd - BufBeg) < (signed)(sizeof("cfe-pth") + 4 + 4) ||
+      memcmp(BufBeg, "cfe-pth", sizeof("cfe-pth")) != 0) {
     Diags.Report(diag::err_invalid_pth_file) << file;
     return 0;
   }
 
   // Read the PTH version.
-  const unsigned char *p = BufBeg + (sizeof("cfe-pth") - 1);
+  const unsigned char *p = BufBeg + (sizeof("cfe-pth"));
   unsigned Version = ReadLE32(p);
 
   if (Version < PTHManager::Version) {
diff --git a/lib/Parse/ParseDecl.cpp b/lib/Parse/ParseDecl.cpp
index b830d9c..cb865cc 100644
--- a/lib/Parse/ParseDecl.cpp
+++ b/lib/Parse/ParseDecl.cpp
@@ -68,7 +68,6 @@ static bool isAttributeLateParsed(const IdentifierInfo &II) {
         .Default(false);
 }
 
-
 /// ParseGNUAttributes - Parse a non-empty attributes list.
 ///
 /// [GNU] attributes:
@@ -193,6 +192,11 @@ void Parser::ParseGNUAttributeArgs(IdentifierInfo *AttrName,
     ParseThreadSafetyAttribute(*AttrName, AttrNameLoc, Attrs, EndLoc);
     return;
   }
+  // Type safety attributes have their own grammar.
+  if (AttrName->isStr("type_tag_for_datatype")) {
+    ParseTypeTagForDatatypeAttribute(*AttrName, AttrNameLoc, Attrs, EndLoc);
+    return;
+  }
 
   ConsumeParen(); // ignore the left paren loc for now
 
@@ -866,7 +870,8 @@ void Parser::ParseLexedAttributes(ParsingClass &Class) {
 void Parser::ParseLexedAttributeList(LateParsedAttrList &LAs, Decl *D,
                                      bool EnterScope, bool OnDefinition) {
   for (unsigned i = 0, ni = LAs.size(); i < ni; ++i) {
-    LAs[i]->addDecl(D);
+    if (D)
+      LAs[i]->addDecl(D);
     ParseLexedAttribute(*LAs[i], EnterScope, OnDefinition);
     delete LAs[i];
   }
@@ -1019,6 +1024,70 @@ void Parser::ParseThreadSafetyAttribute(IdentifierInfo &AttrName,
     *EndLoc = T.getCloseLocation();
 }
 
+void Parser::ParseTypeTagForDatatypeAttribute(IdentifierInfo &AttrName,
+                                              SourceLocation AttrNameLoc,
+                                              ParsedAttributes &Attrs,
+                                              SourceLocation *EndLoc) {
+  assert(Tok.is(tok::l_paren) && "Attribute arg list not starting with '('");
+
+  BalancedDelimiterTracker T(*this, tok::l_paren);
+  T.consumeOpen();
+
+  if (Tok.isNot(tok::identifier)) {
+    Diag(Tok, diag::err_expected_ident);
+    T.skipToEnd();
+    return;
+  }
+  IdentifierInfo *ArgumentKind = Tok.getIdentifierInfo();
+  SourceLocation ArgumentKindLoc = ConsumeToken();
+
+  if (Tok.isNot(tok::comma)) {
+    Diag(Tok, diag::err_expected_comma);
+    T.skipToEnd();
+    return;
+  }
+  ConsumeToken();
+
+  SourceRange MatchingCTypeRange;
+  TypeResult MatchingCType = ParseTypeName(&MatchingCTypeRange);
+  if (MatchingCType.isInvalid()) {
+    T.skipToEnd();
+    return;
+  }
+
+  bool LayoutCompatible = false;
+  bool MustBeNull = false;
+  while (Tok.is(tok::comma)) {
+    ConsumeToken();
+    if (Tok.isNot(tok::identifier)) {
+      Diag(Tok, diag::err_expected_ident);
+      T.skipToEnd();
+      return;
+    }
+    IdentifierInfo *Flag = Tok.getIdentifierInfo();
+    if (Flag->isStr("layout_compatible"))
+      LayoutCompatible = true;
+    else if (Flag->isStr("must_be_null"))
+      MustBeNull = true;
+    else {
+      Diag(Tok, diag::err_type_safety_unknown_flag) << Flag;
+      T.skipToEnd();
+      return;
+    }
+    ConsumeToken(); // consume flag
+  }
+
+  if (!T.consumeClose()) {
+    Attrs.addNewTypeTagForDatatype(&AttrName, AttrNameLoc, 0, AttrNameLoc,
+                                   ArgumentKind, ArgumentKindLoc,
+                                   MatchingCType.release(), LayoutCompatible,
+                                   MustBeNull, AttributeList::AS_GNU);
+  }
+
+  if (EndLoc)
+    *EndLoc = T.getCloseLocation();
+}
+
 /// DiagnoseProhibitedCXX11Attribute - We have found the opening square brackets
 /// of a C++11 attribute-specifier in a location where an attribute is not
 /// permitted. By C++11 [dcl.attr.grammar]p6, this is ill-formed. Diagnose this
diff --git a/lib/Parse/ParseStmt.cpp b/lib/Parse/ParseStmt.cpp
index d2e4309..df9b996 100644
--- a/lib/Parse/ParseStmt.cpp
+++ b/lib/Parse/ParseStmt.cpp
@@ -1495,8 +1495,7 @@ StmtResult Parser::ParseForStatement(SourceLocation *TrailingElseLoc) {
   StmtResult ForEachStmt;
 
   if (ForRange) {
-    ForRangeStmt = Actions.ActOnCXXForRangeStmt(ForLoc, T.getOpenLocation(),
-                                                FirstPart.take(),
+    ForRangeStmt = Actions.ActOnCXXForRangeStmt(ForLoc, FirstPart.take(),
                                                 ForRangeInit.ColonLoc,
                                                 ForRangeInit.RangeExpr.get(),
                                                 T.getCloseLocation());
@@ -1505,7 +1504,7 @@ StmtResult Parser::ParseForStatement(SourceLocation *TrailingElseLoc) {
   // Similarly, we need to do the semantic analysis for a for-range
   // statement immediately in order to close over temporaries correctly.
   } else if (ForEach) {
-    ForEachStmt = Actions.ActOnObjCForCollectionStmt(ForLoc, T.getOpenLocation(),
+    ForEachStmt = Actions.ActOnObjCForCollectionStmt(ForLoc,
                                                      FirstPart.take(),
                                                      Collection.take(),
                                                      T.getCloseLocation());
@@ -1657,112 +1656,98 @@ StmtResult Parser::ParseMicrosoftAsmStatement(SourceLocation AsmLoc) {
   SourceManager &SrcMgr = PP.getSourceManager();
   SourceLocation EndLoc = AsmLoc;
   SmallVector<Token, 4> AsmToks;
-  SmallVector<unsigned, 4> LineEnds;
-  do {
-    bool InBraces = false;
-    unsigned short savedBraceCount = 0;
-    bool InAsmComment = false;
-    FileID FID;
-    unsigned LineNo = 0;
-    unsigned NumTokensRead = 0;
-    SourceLocation LBraceLoc;
-
-    if (Tok.is(tok::l_brace)) {
-      // Braced inline asm: consume the opening brace.
-      InBraces = true;
-      savedBraceCount = BraceCount;
-      EndLoc = LBraceLoc = ConsumeBrace();
-      ++NumTokensRead;
-    } else {
-      // Single-line inline asm; compute which line it is on.
-      std::pair<FileID, unsigned> ExpAsmLoc =
-          SrcMgr.getDecomposedExpansionLoc(EndLoc);
-      FID = ExpAsmLoc.first;
-      LineNo = SrcMgr.getLineNumber(FID, ExpAsmLoc.second);
-    }
 
-    SourceLocation TokLoc = Tok.getLocation();
-    do {
-      // If we hit EOF, we're done, period.
-      if (Tok.is(tok::eof))
-        break;
+  bool InBraces = false;
+  unsigned short savedBraceCount = 0;
+  bool InAsmComment = false;
+  FileID FID;
+  unsigned LineNo = 0;
+  unsigned NumTokensRead = 0;
+  SourceLocation LBraceLoc;
+
+  if (Tok.is(tok::l_brace)) {
+    // Braced inline asm: consume the opening brace.
+    InBraces = true;
+    savedBraceCount = BraceCount;
+    EndLoc = LBraceLoc = ConsumeBrace();
+    ++NumTokensRead;
+  } else {
+    // Single-line inline asm; compute which line it is on.
+    std::pair<FileID, unsigned> ExpAsmLoc =
+      SrcMgr.getDecomposedExpansionLoc(EndLoc);
+    FID = ExpAsmLoc.first;
+    LineNo = SrcMgr.getLineNumber(FID, ExpAsmLoc.second);
+  }
 
-      // The asm keyword is a statement separator, so multiple asm statements
-      // are allowed.
-      if (!InAsmComment && Tok.is(tok::kw_asm))
-        break;
+  SourceLocation TokLoc = Tok.getLocation();
+  do {
+    // If we hit EOF, we're done, period.
+    if (Tok.is(tok::eof))
+      break;
 
-      if (!InAsmComment && Tok.is(tok::semi)) {
-        // A semicolon in an asm is the start of a comment.
-        InAsmComment = true;
-        if (InBraces) {
-          // Compute which line the comment is on.
-          std::pair<FileID, unsigned> ExpSemiLoc =
-              SrcMgr.getDecomposedExpansionLoc(TokLoc);
-          FID = ExpSemiLoc.first;
-          LineNo = SrcMgr.getLineNumber(FID, ExpSemiLoc.second);
-        }
-      } else if (!InBraces || InAsmComment) {
-        // If end-of-line is significant, check whether this token is on a
-        // new line.
-        std::pair<FileID, unsigned> ExpLoc =
-            SrcMgr.getDecomposedExpansionLoc(TokLoc);
-        if (ExpLoc.first != FID ||
-            SrcMgr.getLineNumber(ExpLoc.first, ExpLoc.second) != LineNo) {
-          // If this is a single-line __asm, we're done.
-          if (!InBraces)
-            break;
-          // We're no longer in a comment.
-          InAsmComment = false;
-        } else if (!InAsmComment && Tok.is(tok::r_brace)) {
-          // Single-line asm always ends when a closing brace is seen.
-          // FIXME: This is compatible with Apple gcc's -fasm-blocks; what
-          // does MSVC do here?
-          break;
-        }
+    if (!InAsmComment && Tok.is(tok::semi)) {
+      // A semicolon in an asm is the start of a comment.
+      InAsmComment = true;
+      if (InBraces) {
+        // Compute which line the comment is on.
+        std::pair<FileID, unsigned> ExpSemiLoc =
+          SrcMgr.getDecomposedExpansionLoc(TokLoc);
+        FID = ExpSemiLoc.first;
+        LineNo = SrcMgr.getLineNumber(FID, ExpSemiLoc.second);
       }
-      if (!InAsmComment && InBraces && Tok.is(tok::r_brace) &&
-          BraceCount == (savedBraceCount + 1)) {
-        // Consume the closing brace, and finish
-        EndLoc = ConsumeBrace();
+    } else if (!InBraces || InAsmComment) {
+      // If end-of-line is significant, check whether this token is on a
+      // new line.
+      std::pair<FileID, unsigned> ExpLoc =
+        SrcMgr.getDecomposedExpansionLoc(TokLoc);
+      if (ExpLoc.first != FID ||
+          SrcMgr.getLineNumber(ExpLoc.first, ExpLoc.second) != LineNo) {
+        // If this is a single-line __asm, we're done.
+        if (!InBraces)
+          break;
+        // We're no longer in a comment.
+        InAsmComment = false;
+      } else if (!InAsmComment && Tok.is(tok::r_brace)) {
+        // Single-line asm always ends when a closing brace is seen.
+        // FIXME: This is compatible with Apple gcc's -fasm-blocks; what
+        // does MSVC do here?
         break;
       }
-
-      // Consume the next token; make sure we don't modify the brace count etc.
-      // if we are in a comment.
-      EndLoc = TokLoc;
-      if (InAsmComment)
-        PP.Lex(Tok);
-      else {
-        AsmToks.push_back(Tok);
-        ConsumeAnyToken();
-      }
-      TokLoc = Tok.getLocation();
-      ++NumTokensRead;
-    } while (1);
-
-    LineEnds.push_back(AsmToks.size());
-
-    if (InBraces && BraceCount != savedBraceCount) {
-      // __asm without closing brace (this can happen at EOF).
-      Diag(Tok, diag::err_expected_rbrace);
-      Diag(LBraceLoc, diag::note_matching) << "{";
-      return StmtError();
-    } else if (NumTokensRead == 0) {
-      // Empty __asm.
-      Diag(Tok, diag::err_expected_lbrace);
-      return StmtError();
     }
-    // Multiple adjacent asm's form together into a single asm statement
-    // in the AST.
-    if (!Tok.is(tok::kw_asm))
+    if (!InAsmComment && InBraces && Tok.is(tok::r_brace) &&
+        BraceCount == (savedBraceCount + 1)) {
+      // Consume the closing brace, and finish
+      EndLoc = ConsumeBrace();
       break;
-    EndLoc = ConsumeToken();
+    }
+
+    // Consume the next token; make sure we don't modify the brace count etc.
+    // if we are in a comment.
+    EndLoc = TokLoc;
+    if (InAsmComment)
+      PP.Lex(Tok);
+    else {
+      AsmToks.push_back(Tok);
+      ConsumeAnyToken();
+    }
+    TokLoc = Tok.getLocation();
+    ++NumTokensRead;
   } while (1);
 
+  if (InBraces && BraceCount != savedBraceCount) {
+    // __asm without closing brace (this can happen at EOF).
+    Diag(Tok, diag::err_expected_rbrace);
+    Diag(LBraceLoc, diag::note_matching) << "{";
+    return StmtError();
+  } else if (NumTokensRead == 0) {
+    // Empty __asm.
+    Diag(Tok, diag::err_expected_lbrace);
+    return StmtError();
+  }
+
   // FIXME: We should be passing source locations for better diagnostics.
-  return Actions.ActOnMSAsmStmt(AsmLoc, llvm::makeArrayRef(AsmToks),
-                                llvm::makeArrayRef(LineEnds), EndLoc);
+  return Actions.ActOnMSAsmStmt(AsmLoc, LBraceLoc,
+                                llvm::makeArrayRef(AsmToks), EndLoc);
 }
 
 /// ParseAsmStatement - Parse a GNU extended asm statement.
diff --git a/lib/Rewrite/RewriteModernObjC.cpp b/lib/Rewrite/RewriteModernObjC.cpp
index 9f42fca..dcd003f 100644
--- a/lib/Rewrite/RewriteModernObjC.cpp
+++ b/lib/Rewrite/RewriteModernObjC.cpp
@@ -241,7 +241,7 @@ namespace {
       // Get the new text.
       std::string SStr;
       llvm::raw_string_ostream S(SStr);
-      New->printPretty(S, *Context, 0, PrintingPolicy(LangOpts));
+      New->printPretty(S, 0, PrintingPolicy(LangOpts));
       const std::string &Str = S.str();
 
       // If replacement succeeded or warning disabled return with no warning.
@@ -2549,8 +2549,7 @@ Stmt *RewriteModernObjC::RewriteObjCStringLiteral(ObjCStringLiteral *Exp) {
   // The pretty printer for StringLiteral handles escape characters properly.
   std::string prettyBufS;
   llvm::raw_string_ostream prettyBuf(prettyBufS);
-  Exp->getString()->printPretty(prettyBuf, *Context, 0,
-                                PrintingPolicy(LangOpts));
+  Exp->getString()->printPretty(prettyBuf, 0, PrintingPolicy(LangOpts));
   Preamble += prettyBuf.str();
   Preamble += ",";
   Preamble += utostr(Exp->getString()->getByteLength()) + "};\n";
@@ -4341,7 +4340,7 @@ void RewriteModernObjC::SynthesizeBlockLiterals(SourceLocation FunLocStart,
     std::string SStr;
   
     llvm::raw_string_ostream constructorExprBuf(SStr);
-    GlobalConstructionExp->printPretty(constructorExprBuf, *Context, 0,
+    GlobalConstructionExp->printPretty(constructorExprBuf, 0,
                                          PrintingPolicy(LangOpts));
     globalBuf += constructorExprBuf.str();
     globalBuf += ";\n";
@@ -5610,7 +5609,7 @@ Stmt *RewriteModernObjC::RewriteFunctionBodyOrGlobalInitializer(Stmt *S) {
     // Get the new text.
     std::string SStr;
     llvm::raw_string_ostream Buf(SStr);
-    Replacement->printPretty(Buf, *Context);
+    Replacement->printPretty(Buf);
     const std::string &Str = Buf.str();
 
     printf("CAST = %s\n", &Str[0]);
diff --git a/lib/Rewrite/RewriteObjC.cpp b/lib/Rewrite/RewriteObjC.cpp
index 425cd77..37c17e6 100644
--- a/lib/Rewrite/RewriteObjC.cpp
+++ b/lib/Rewrite/RewriteObjC.cpp
@@ -227,7 +227,7 @@ namespace {
       // Get the new text.
       std::string SStr;
       llvm::raw_string_ostream S(SStr);
-      New->printPretty(S, *Context, 0, PrintingPolicy(LangOpts));
+      New->printPretty(S, 0, PrintingPolicy(LangOpts));
       const std::string &Str = S.str();
 
       // If replacement succeeded or warning disabled return with no warning.
@@ -1720,8 +1720,7 @@ Stmt *RewriteObjC::RewriteObjCSynchronizedStmt(ObjCAtSynchronizedStmt *S) {
                                       CK, syncExpr);
   std::string syncExprBufS;
   llvm::raw_string_ostream syncExprBuf(syncExprBufS);
-  syncExpr->printPretty(syncExprBuf, *Context, 0,
-                        PrintingPolicy(LangOpts));
+  syncExpr->printPretty(syncExprBuf, 0, PrintingPolicy(LangOpts));
   syncBuf += syncExprBuf.str();
   syncBuf += ");";
   
@@ -2553,8 +2552,7 @@ Stmt *RewriteObjC::RewriteObjCStringLiteral(ObjCStringLiteral *Exp) {
   // The pretty printer for StringLiteral handles escape characters properly.
   std::string prettyBufS;
   llvm::raw_string_ostream prettyBuf(prettyBufS);
-  Exp->getString()->printPretty(prettyBuf, *Context, 0,
-                                PrintingPolicy(LangOpts));
+  Exp->getString()->printPretty(prettyBuf, 0, PrintingPolicy(LangOpts));
   Preamble += prettyBuf.str();
   Preamble += ",";
   Preamble += utostr(Exp->getString()->getByteLength()) + "};\n";
@@ -4885,7 +4883,7 @@ Stmt *RewriteObjC::RewriteFunctionBodyOrGlobalInitializer(Stmt *S) {
     // Get the new text.
     std::string SStr;
     llvm::raw_string_ostream Buf(SStr);
-    Replacement->printPretty(Buf, *Context);
+    Replacement->printPretty(Buf);
     const std::string &Str = Buf.str();
 
     printf("CAST = %s\n", &Str[0]);
diff --git a/lib/Sema/AttributeList.cpp b/lib/Sema/AttributeList.cpp
index 0f209fd..7c79879 100644
--- a/lib/Sema/AttributeList.cpp
+++ b/lib/Sema/AttributeList.cpp
@@ -21,6 +21,8 @@ using namespace clang;
 
 size_t AttributeList::allocated_size() const {
   if (IsAvailability) return AttributeFactory::AvailabilityAllocSize;
+  else if (IsTypeTagForDatatype)
+    return AttributeFactory::TypeTagForDatatypeAllocSize;
   return (sizeof(AttributeList) + NumArgs * sizeof(Expr*));
 }
 
diff --git a/lib/Sema/SemaCast.cpp b/lib/Sema/SemaCast.cpp
index 8199751..d8d51e7 100644
--- a/lib/Sema/SemaCast.cpp
+++ b/lib/Sema/SemaCast.cpp
@@ -1477,6 +1477,21 @@ void Sema::CheckCompatibleReinterpretCast(QualType SrcType, QualType DestType,
   Diag(Range.getBegin(), DiagID) << SrcType << DestType << Range;
 }
 
+static void DiagnoseCastOfObjCSEL(Sema &Self, const ExprResult &SrcExpr,
+                                  QualType DestType) {
+  QualType SrcType = SrcExpr.get()->getType();
+  if (const PointerType *SrcPtrTy = SrcType->getAs<PointerType>())
+    if (SrcPtrTy->isObjCSelType()) {
+      QualType DT = DestType;
+      if (isa<PointerType>(DestType))
+        DT = DestType->getPointeeType();
+      if (!DT.getUnqualifiedType()->isVoidType())
+        Self.Diag(SrcExpr.get()->getExprLoc(),
+                  diag::warn_cast_pointer_from_sel)
+        << SrcType << DestType << SrcExpr.get()->getSourceRange();
+    }
+}
+
 static TryCastResult TryReinterpretCast(Sema &Self, ExprResult &SrcExpr,
                                         QualType DestType, bool CStyle,
                                         const SourceRange &OpRange,
@@ -1721,7 +1736,9 @@ static TryCastResult TryReinterpretCast(Sema &Self, ExprResult &SrcExpr,
   if (CStyle && DestType->isObjCObjectPointerType()) {
     return TC_Success;
   }
-    
+  if (CStyle)
+    DiagnoseCastOfObjCSEL(Self, SrcExpr, DestType);
+  
   // Not casting away constness, so the only remaining check is for compatible
   // pointer categories.
 
@@ -2058,6 +2075,7 @@ void CastOperation::CheckCStyleCast() {
       return;
     }
   }
+  DiagnoseCastOfObjCSEL(Self, SrcExpr, DestType);
   
   Kind = Self.PrepareScalarCast(SrcExpr, DestType);
   if (SrcExpr.isInvalid())
diff --git a/lib/Sema/SemaChecking.cpp b/lib/Sema/SemaChecking.cpp
index 2594648..2559f00 100644
--- a/lib/Sema/SemaChecking.cpp
+++ b/lib/Sema/SemaChecking.cpp
@@ -513,6 +513,13 @@ void Sema::checkCall(NamedDecl *FDecl, Expr **Args,
          I = FDecl->specific_attr_begin<NonNullAttr>(),
          E = FDecl->specific_attr_end<NonNullAttr>(); I != E; ++I)
     CheckNonNullArguments(*I, Args, Loc);
+
+  // Type safety checking.
+  for (specific_attr_iterator<ArgumentWithTypeTagAttr>
+         i = FDecl->specific_attr_begin<ArgumentWithTypeTagAttr>(),
+         e = FDecl->specific_attr_end<ArgumentWithTypeTagAttr>(); i != e; ++i) {
+    CheckArgumentWithTypeTag(*i, Args);
+  }
 }
 
 /// CheckConstructorCall - Check a constructor call for correctness and safety
@@ -3170,7 +3177,7 @@ void Sema::CheckStrlcpycatArguments(const CallExpr *Call,
   SmallString<128> sizeString;
   llvm::raw_svector_ostream OS(sizeString);
   OS << "sizeof(";
-  DstArg->printPretty(OS, Context, 0, getPrintingPolicy());
+  DstArg->printPretty(OS, 0, getPrintingPolicy());
   OS << ")";
   
   Diag(OriginalSizeArg->getLocStart(), diag::note_strlcpycat_wrong_size)
@@ -3267,10 +3274,10 @@ void Sema::CheckStrncatArguments(const CallExpr *CE,
   SmallString<128> sizeString;
   llvm::raw_svector_ostream OS(sizeString);
   OS << "sizeof(";
-  DstArg->printPretty(OS, Context, 0, getPrintingPolicy());
+  DstArg->printPretty(OS, 0, getPrintingPolicy());
   OS << ") - ";
   OS << "strlen(";
-  DstArg->printPretty(OS, Context, 0, getPrintingPolicy());
+  DstArg->printPretty(OS, 0, getPrintingPolicy());
   OS << ") - 1";
 
   Diag(SL, diag::note_strncat_wrong_size)
@@ -5468,3 +5475,410 @@ void Sema::DiagnoseEmptyLoopBody(const Stmt *S,
     Diag(NBody->getSemiLoc(), diag::note_empty_body_on_separate_line);
   }
 }
+
+//===--- Layout compatibility ----------------------------------------------//
+
+namespace {
+
+bool isLayoutCompatible(ASTContext &C, QualType T1, QualType T2);
+
+/// \brief Check if two enumeration types are layout-compatible.
+bool isLayoutCompatible(ASTContext &C, EnumDecl *ED1, EnumDecl *ED2) {
+  // C++11 [dcl.enum] p8:
+  // Two enumeration types are layout-compatible if they have the same
+  // underlying type.
+  return ED1->isComplete() && ED2->isComplete() &&
+         C.hasSameType(ED1->getIntegerType(), ED2->getIntegerType());
+}
+
+/// \brief Check if two fields are layout-compatible.
+bool isLayoutCompatible(ASTContext &C, FieldDecl *Field1, FieldDecl *Field2) {
+  if (!isLayoutCompatible(C, Field1->getType(), Field2->getType()))
+    return false;
+
+  if (Field1->isBitField() != Field2->isBitField())
+    return false;
+
+  if (Field1->isBitField()) {
+    // Make sure that the bit-fields are the same length.
+    unsigned Bits1 = Field1->getBitWidthValue(C);
+    unsigned Bits2 = Field2->getBitWidthValue(C);
+
+    if (Bits1 != Bits2)
+      return false;
+  }
+
+  return true;
+}
+
+/// \brief Check if two standard-layout structs are layout-compatible.
+/// (C++11 [class.mem] p17)
+bool isLayoutCompatibleStruct(ASTContext &C,
+                              RecordDecl *RD1,
+                              RecordDecl *RD2) {
+  // If both records are C++ classes, check that base classes match.
+  if (const CXXRecordDecl *D1CXX = dyn_cast<CXXRecordDecl>(RD1)) {
+    // If one of records is a CXXRecordDecl we are in C++ mode,
+    // thus the other one is a CXXRecordDecl, too.
+    const CXXRecordDecl *D2CXX = cast<CXXRecordDecl>(RD2);
+    // Check number of base classes.
+    if (D1CXX->getNumBases() != D2CXX->getNumBases())
+      return false;
+
+    // Check the base classes.
+    for (CXXRecordDecl::base_class_const_iterator
+               Base1 = D1CXX->bases_begin(),
+           BaseEnd1 = D1CXX->bases_end(),
+              Base2 = D2CXX->bases_begin();
+         Base1 != BaseEnd1;
+         ++Base1, ++Base2) {
+      if (!isLayoutCompatible(C, Base1->getType(), Base2->getType()))
+        return false;
+    }
+  } else if (const CXXRecordDecl *D2CXX = dyn_cast<CXXRecordDecl>(RD2)) {
+    // If only RD2 is a C++ class, it should have zero base classes.
+    if (D2CXX->getNumBases() > 0)
+      return false;
+  }
+
+  // Check the fields.
+  RecordDecl::field_iterator Field2 = RD2->field_begin(),
+                             Field2End = RD2->field_end(),
+                             Field1 = RD1->field_begin(),
+                             Field1End = RD1->field_end();
+  for ( ; Field1 != Field1End && Field2 != Field2End; ++Field1, ++Field2) {
+    if (!isLayoutCompatible(C, *Field1, *Field2))
+      return false;
+  }
+  if (Field1 != Field1End || Field2 != Field2End)
+    return false;
+
+  return true;
+}
+
+/// \brief Check if two standard-layout unions are layout-compatible.
+/// (C++11 [class.mem] p18)
+bool isLayoutCompatibleUnion(ASTContext &C,
+                             RecordDecl *RD1,
+                             RecordDecl *RD2) {
+  llvm::SmallPtrSet<FieldDecl *, 8> UnmatchedFields;
+  for (RecordDecl::field_iterator Field2 = RD2->field_begin(),
+                                  Field2End = RD2->field_end();
+       Field2 != Field2End; ++Field2) {
+    UnmatchedFields.insert(*Field2);
+  }
+
+  for (RecordDecl::field_iterator Field1 = RD1->field_begin(),
+                                  Field1End = RD1->field_end();
+       Field1 != Field1End; ++Field1) {
+    llvm::SmallPtrSet<FieldDecl *, 8>::iterator
+        I = UnmatchedFields.begin(),
+        E = UnmatchedFields.end();
+
+    for ( ; I != E; ++I) {
+      if (isLayoutCompatible(C, *Field1, *I)) {
+        bool Result = UnmatchedFields.erase(*I);
+        (void) Result;
+        assert(Result);
+        break;
+      }
+    }
+    if (I == E)
+      return false;
+  }
+
+  return UnmatchedFields.empty();
+}
+
+bool isLayoutCompatible(ASTContext &C, RecordDecl *RD1, RecordDecl *RD2) {
+  if (RD1->isUnion() != RD2->isUnion())
+    return false;
+
+  if (RD1->isUnion())
+    return isLayoutCompatibleUnion(C, RD1, RD2);
+  else
+    return isLayoutCompatibleStruct(C, RD1, RD2);
+}
+
+/// \brief Check if two types are layout-compatible in C++11 sense.
+bool isLayoutCompatible(ASTContext &C, QualType T1, QualType T2) {
+  if (T1.isNull() || T2.isNull())
+    return false;
+
+  // C++11 [basic.types] p11:
+  // If two types T1 and T2 are the same type, then T1 and T2 are
+  // layout-compatible types.
+  if (C.hasSameType(T1, T2))
+    return true;
+
+  T1 = T1.getCanonicalType().getUnqualifiedType();
+  T2 = T2.getCanonicalType().getUnqualifiedType();
+
+  const Type::TypeClass TC1 = T1->getTypeClass();
+  const Type::TypeClass TC2 = T2->getTypeClass();
+
+  if (TC1 != TC2)
+    return false;
+
+  if (TC1 == Type::Enum) {
+    return isLayoutCompatible(C,
+                              cast<EnumType>(T1)->getDecl(),
+                              cast<EnumType>(T2)->getDecl());
+  } else if (TC1 == Type::Record) {
+    if (!T1->isStandardLayoutType() || !T2->isStandardLayoutType())
+      return false;
+
+    return isLayoutCompatible(C,
+                              cast<RecordType>(T1)->getDecl(),
+                              cast<RecordType>(T2)->getDecl());
+  }
+
+  return false;
+}
+}
+
+//===--- CHECK: pointer_with_type_tag attribute: datatypes should match ----//
+
+namespace {
+/// \brief Given a type tag expression find the type tag itself.
+///
+/// \param TypeExpr Type tag expression, as it appears in user's code.
+///
+/// \param VD Declaration of an identifier that appears in a type tag.
+///
+/// \param MagicValue Type tag magic value.
+bool FindTypeTagExpr(const Expr *TypeExpr, const ASTContext &Ctx,
+                     const ValueDecl **VD, uint64_t *MagicValue) {
+  while(true) {
+    if (!TypeExpr)
+      return false;
+
+    TypeExpr = TypeExpr->IgnoreParenImpCasts()->IgnoreParenCasts();
+
+    switch (TypeExpr->getStmtClass()) {
+    case Stmt::UnaryOperatorClass: {
+      const UnaryOperator *UO = cast<UnaryOperator>(TypeExpr);
+      if (UO->getOpcode() == UO_AddrOf || UO->getOpcode() == UO_Deref) {
+        TypeExpr = UO->getSubExpr();
+        continue;
+      }
+      return false;
+    }
+
+    case Stmt::DeclRefExprClass: {
+      const DeclRefExpr *DRE = cast<DeclRefExpr>(TypeExpr);
+      *VD = DRE->getDecl();
+      return true;
+    }
+
+    case Stmt::IntegerLiteralClass: {
+      const IntegerLiteral *IL = cast<IntegerLiteral>(TypeExpr);
+      llvm::APInt MagicValueAPInt = IL->getValue();
+      if (MagicValueAPInt.getActiveBits() <= 64) {
+        *MagicValue = MagicValueAPInt.getZExtValue();
+        return true;
+      } else
+        return false;
+    }
+
+    case Stmt::BinaryConditionalOperatorClass:
+    case Stmt::ConditionalOperatorClass: {
+      const AbstractConditionalOperator *ACO =
+          cast<AbstractConditionalOperator>(TypeExpr);
+      bool Result;
+      if (ACO->getCond()->EvaluateAsBooleanCondition(Result, Ctx)) {
+        if (Result)
+          TypeExpr = ACO->getTrueExpr();
+        else
+          TypeExpr = ACO->getFalseExpr();
+        continue;
+      }
+      return false;
+    }
+
+    case Stmt::BinaryOperatorClass: {
+      const BinaryOperator *BO = cast<BinaryOperator>(TypeExpr);
+      if (BO->getOpcode() == BO_Comma) {
+        TypeExpr = BO->getRHS();
+        continue;
+      }
+      return false;
+    }
+
+    default:
+      return false;
+    }
+  }
+}
+
+/// \brief Retrieve the C type corresponding to type tag TypeExpr.
+///
+/// \param TypeExpr Expression that specifies a type tag.
+///
+/// \param MagicValues Registered magic values.
+///
+/// \param FoundWrongKind Set to true if a type tag was found, but of a wrong
+///        kind.
+///
+/// \param TypeInfo Information about the corresponding C type.
+///
+/// \returns true if the corresponding C type was found.
+bool GetMatchingCType(
+        const IdentifierInfo *ArgumentKind,
+        const Expr *TypeExpr, const ASTContext &Ctx,
+        const llvm::DenseMap<Sema::TypeTagMagicValue,
+                             Sema::TypeTagData> *MagicValues,
+        bool &FoundWrongKind,
+        Sema::TypeTagData &TypeInfo) {
+  FoundWrongKind = false;
+
+  // Variable declaration that has type_tag_for_datatype attribute.
+  const ValueDecl *VD = NULL;
+
+  uint64_t MagicValue;
+
+  if (!FindTypeTagExpr(TypeExpr, Ctx, &VD, &MagicValue))
+    return false;
+
+  if (VD) {
+    for (specific_attr_iterator<TypeTagForDatatypeAttr>
+             I = VD->specific_attr_begin<TypeTagForDatatypeAttr>(),
+             E = VD->specific_attr_end<TypeTagForDatatypeAttr>();
+         I != E; ++I) {
+      if (I->getArgumentKind() != ArgumentKind) {
+        FoundWrongKind = true;
+        return false;
+      }
+      TypeInfo.Type = I->getMatchingCType();
+      TypeInfo.LayoutCompatible = I->getLayoutCompatible();
+      TypeInfo.MustBeNull = I->getMustBeNull();
+      return true;
+    }
+    return false;
+  }
+
+  if (!MagicValues)
+    return false;
+
+  llvm::DenseMap<Sema::TypeTagMagicValue,
+                 Sema::TypeTagData>::const_iterator I =
+      MagicValues->find(std::make_pair(ArgumentKind, MagicValue));
+  if (I == MagicValues->end())
+    return false;
+
+  TypeInfo = I->second;
+  return true;
+}
+} // unnamed namespace
+
+void Sema::RegisterTypeTagForDatatype(const IdentifierInfo *ArgumentKind,
+                                      uint64_t MagicValue, QualType Type,
+                                      bool LayoutCompatible,
+                                      bool MustBeNull) {
+  if (!TypeTagForDatatypeMagicValues)
+    TypeTagForDatatypeMagicValues.reset(
+        new llvm::DenseMap<TypeTagMagicValue, TypeTagData>);
+
+  TypeTagMagicValue Magic(ArgumentKind, MagicValue);
+  (*TypeTagForDatatypeMagicValues)[Magic] =
+      TypeTagData(Type, LayoutCompatible, MustBeNull);
+}
+
+namespace {
+bool IsSameCharType(QualType T1, QualType T2) {
+  const BuiltinType *BT1 = T1->getAs<BuiltinType>();
+  if (!BT1)
+    return false;
+
+  const BuiltinType *BT2 = T2->getAs<BuiltinType>();
+  if (!BT2)
+    return false;
+
+  BuiltinType::Kind T1Kind = BT1->getKind();
+  BuiltinType::Kind T2Kind = BT2->getKind();
+
+  return (T1Kind == BuiltinType::SChar  && T2Kind == BuiltinType::Char_S) ||
+         (T1Kind == BuiltinType::UChar  && T2Kind == BuiltinType::Char_U) ||
+         (T1Kind == BuiltinType::Char_U && T2Kind == BuiltinType::UChar) ||
+         (T1Kind == BuiltinType::Char_S && T2Kind == BuiltinType::SChar);
+}
+} // unnamed namespace
+
+void Sema::CheckArgumentWithTypeTag(const ArgumentWithTypeTagAttr *Attr,
+                                    const Expr * const *ExprArgs) {
+  const IdentifierInfo *ArgumentKind = Attr->getArgumentKind();
+  bool IsPointerAttr = Attr->getIsPointer();
+
+  const Expr *TypeTagExpr = ExprArgs[Attr->getTypeTagIdx()];
+  bool FoundWrongKind;
+  TypeTagData TypeInfo;
+  if (!GetMatchingCType(ArgumentKind, TypeTagExpr, Context,
+                        TypeTagForDatatypeMagicValues.get(),
+                        FoundWrongKind, TypeInfo)) {
+    if (FoundWrongKind)
+      Diag(TypeTagExpr->getExprLoc(),
+           diag::warn_type_tag_for_datatype_wrong_kind)
+        << TypeTagExpr->getSourceRange();
+    return;
+  }
+
+  const Expr *ArgumentExpr = ExprArgs[Attr->getArgumentIdx()];
+  if (IsPointerAttr) {
+    // Skip implicit cast of pointer to `void *' (as a function argument).
+    if (const ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(ArgumentExpr))
+      if (ICE->getType()->isVoidPointerType())
+        ArgumentExpr = ICE->getSubExpr();
+  }
+  QualType ArgumentType = ArgumentExpr->getType();
+
+  // Passing a `void*' pointer shouldn't trigger a warning.
+  if (IsPointerAttr && ArgumentType->isVoidPointerType())
+    return;
+
+  if (TypeInfo.MustBeNull) {
+    // Type tag with matching void type requires a null pointer.
+    if (!ArgumentExpr->isNullPointerConstant(Context,
+                                             Expr::NPC_ValueDependentIsNotNull)) {
+      Diag(ArgumentExpr->getExprLoc(),
+           diag::warn_type_safety_null_pointer_required)
+          << ArgumentKind->getName()
+          << ArgumentExpr->getSourceRange()
+          << TypeTagExpr->getSourceRange();
+    }
+    return;
+  }
+
+  QualType RequiredType = TypeInfo.Type;
+  if (IsPointerAttr)
+    RequiredType = Context.getPointerType(RequiredType);
+
+  bool mismatch = false;
+  if (!TypeInfo.LayoutCompatible) {
+    mismatch = !Context.hasSameType(ArgumentType, RequiredType);
+
+    // C++11 [basic.fundamental] p1:
+    // Plain char, signed char, and unsigned char are three distinct types.
+    //
+    // But we treat plain `char' as equivalent to `signed char' or `unsigned
+    // char' depending on the current char signedness mode.
+    if (mismatch)
+      if ((IsPointerAttr && IsSameCharType(ArgumentType->getPointeeType(),
+                                           RequiredType->getPointeeType())) ||
+          (!IsPointerAttr && IsSameCharType(ArgumentType, RequiredType)))
+        mismatch = false;
+  } else
+    if (IsPointerAttr)
+      mismatch = !isLayoutCompatible(Context,
+                                     ArgumentType->getPointeeType(),
+                                     RequiredType->getPointeeType());
+    else
+      mismatch = !isLayoutCompatible(Context, ArgumentType, RequiredType);
+
+  if (mismatch)
+    Diag(ArgumentExpr->getExprLoc(), diag::warn_type_safety_type_mismatch)
+        << ArgumentType << ArgumentKind->getName()
+        << TypeInfo.LayoutCompatible << RequiredType
+        << ArgumentExpr->getSourceRange()
+        << TypeTagExpr->getSourceRange();
+}
+
diff --git a/lib/Sema/SemaCodeComplete.cpp b/lib/Sema/SemaCodeComplete.cpp
index 9fa757d..adf1327 100644
--- a/lib/Sema/SemaCodeComplete.cpp
+++ b/lib/Sema/SemaCodeComplete.cpp
@@ -4476,7 +4476,6 @@ static void AddObjCExpressionResults(ResultBuilder &Results, bool NeedAt) {
   Builder.AddResultTypeChunk("NSDictionary *");
   Builder.AddTypedTextChunk(OBJC_AT_KEYWORD_NAME(NeedAt,"{"));
   Builder.AddPlaceholderChunk("key");
-  Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
   Builder.AddChunk(CodeCompletionString::CK_Colon);
   Builder.AddChunk(CodeCompletionString::CK_HorizontalSpace);
   Builder.AddPlaceholderChunk("object, ...");
diff --git a/lib/Sema/SemaDecl.cpp b/lib/Sema/SemaDecl.cpp
index 3aae99a..ea181de 100644
--- a/lib/Sema/SemaDecl.cpp
+++ b/lib/Sema/SemaDecl.cpp
@@ -6174,6 +6174,7 @@ namespace {
     Decl *OrigDecl;
     bool isRecordType;
     bool isPODType;
+    bool isReferenceType;
 
   public:
     typedef EvaluatedExprVisitor<SelfReferenceChecker> Inherited;
@@ -6182,9 +6183,11 @@ namespace {
                                                     S(S), OrigDecl(OrigDecl) {
       isPODType = false;
       isRecordType = false;
+      isReferenceType = false;
       if (ValueDecl *VD = dyn_cast<ValueDecl>(OrigDecl)) {
         isPODType = VD->getType().isPODType(S.Context);
         isRecordType = VD->getType()->isRecordType();
+        isReferenceType = VD->getType()->isReferenceType();
       }
     }
 
@@ -6192,9 +6195,9 @@ namespace {
     // to determine which DeclRefExpr's to check.  Assume that the casts
     // are present and continue visiting the expression.
     void HandleExpr(Expr *E) {
-      // Skip checking T a = a where T is not a record type.  Doing so is a
-      // way to silence uninitialized warnings.
-      if (isRecordType)
+      // Skip checking T a = a where T is not a record or reference type.
+      // Doing so is a way to silence uninitialized warnings.
+      if (isRecordType || isReferenceType)
         if (DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(E))
           HandleDeclRefExpr(DRE);
 
@@ -6309,11 +6312,11 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init,
   }
 
   // Check for self-references within variable initializers.
-  // Variables declared within a function/method body are handled
-  // by a dataflow analysis.
+  // Variables declared within a function/method body (except for references)
+  // are handled by a dataflow analysis.
   // Record types initialized by initializer list are handled here.
   // Initialization by constructors are handled in TryConstructorInitialization.
-  if (!VDecl->hasLocalStorage() && !VDecl->isStaticLocal() &&
+  if ((!VDecl->hasLocalStorage() || VDecl->getType()->isReferenceType()) &&
       (isa<InitListExpr>(Init) || !VDecl->getType()->isRecordType()))
     CheckSelfReference(RealDecl, Init);
 
@@ -6754,6 +6757,10 @@ void Sema::ActOnUninitializedDecl(Decl *RealDecl,
                                  diag::err_abstract_type_in_decl,
                                  AbstractVariableType))
         Var->setInvalidDecl();
+      if (!Type->isDependentType() && !Var->isInvalidDecl() &&
+          Var->getStorageClass() == SC_PrivateExtern)
+        Diag(Var->getLocation(), diag::warn_private_extern);
+        
       return;
 
     case VarDecl::TentativeDefinition:
@@ -7027,6 +7034,42 @@ void
 Sema::FinalizeDeclaration(Decl *ThisDecl) {
   // Note that we are no longer parsing the initializer for this declaration.
   ParsingInitForAutoVars.erase(ThisDecl);
+
+  // Now we have parsed the initializer and can update the table of magic
+  // tag values.
+  if (ThisDecl && ThisDecl->hasAttr<TypeTagForDatatypeAttr>()) {
+    const VarDecl *VD = dyn_cast<VarDecl>(ThisDecl);
+    if (VD && VD->getType()->isIntegralOrEnumerationType()) {
+      for (specific_attr_iterator<TypeTagForDatatypeAttr>
+               I = ThisDecl->specific_attr_begin<TypeTagForDatatypeAttr>(),
+               E = ThisDecl->specific_attr_end<TypeTagForDatatypeAttr>();
+           I != E; ++I) {
+        const Expr *MagicValueExpr = VD->getInit();
+        if (!MagicValueExpr) {
+          continue;
+        }
+        llvm::APSInt MagicValueInt;
+        if (!MagicValueExpr->isIntegerConstantExpr(MagicValueInt, Context)) {
+          Diag(I->getRange().getBegin(),
+               diag::err_type_tag_for_datatype_not_ice)
+            << LangOpts.CPlusPlus << MagicValueExpr->getSourceRange();
+          continue;
+        }
+        if (MagicValueInt.getActiveBits() > 64) {
+          Diag(I->getRange().getBegin(),
+               diag::err_type_tag_for_datatype_too_large)
+            << LangOpts.CPlusPlus << MagicValueExpr->getSourceRange();
+          continue;
+        }
+        uint64_t MagicValue = MagicValueInt.getZExtValue();
+        RegisterTypeTagForDatatype(I->getArgumentKind(),
+                                   MagicValue,
+                                   I->getMatchingCType(),
+                                   I->getLayoutCompatible(),
+                                   I->getMustBeNull());
+      }
+    }
+  }
 }
 
 Sema::DeclGroupPtrTy
@@ -7623,7 +7666,9 @@ Decl *Sema::ActOnStartOfFunctionDef(Scope *FnBodyScope, Decl *D) {
         << FD->getName() << "dllimport";
     }
   }
-  ActOnDocumentableDecl(FD);
+  // We want to attach documentation to original Decl (which might be
+  // a function template).
+  ActOnDocumentableDecl(D);
   return FD;
 }
 
@@ -7750,7 +7795,8 @@ Decl *Sema::ActOnFinishFunctionBody(Decl *dcl, Stmt *Body,
     // Verify that gotos and switch cases don't jump into scopes illegally.
     if (getCurFunction()->NeedsScopeChecking() &&
         !dcl->isInvalidDecl() &&
-        !hasAnyUnrecoverableErrorsInThisFunction())
+        !hasAnyUnrecoverableErrorsInThisFunction() &&
+        !PP.isCodeCompletionEnabled())
       DiagnoseInvalidJumps(Body);
 
     if (CXXDestructorDecl *Destructor = dyn_cast<CXXDestructorDecl>(dcl)) {
@@ -9907,6 +9953,13 @@ void Sema::ActOnFields(Scope* S,
           }
         }
       }
+      if (isa<ObjCContainerDecl>(EnclosingDecl) &&
+          RequireNonAbstractType(FD->getLocation(), FD->getType(),
+                                 diag::err_abstract_type_in_decl,
+                                 AbstractIvarType)) {
+        // Ivars can not have abstract class types
+        FD->setInvalidDecl();
+      }
       if (Record && FDTTy->getDecl()->hasObjectMember())
         Record->setHasObjectMember(true);
     } else if (FDTy->isObjCObjectType()) {
@@ -9915,8 +9968,7 @@ void Sema::ActOnFields(Scope* S,
         << FixItHint::CreateInsertion(FD->getLocation(), "*");
       QualType T = Context.getObjCObjectPointerType(FD->getType());
       FD->setType(T);
-    } 
-    else if (!getLangOpts().CPlusPlus) {
+    } else if (!getLangOpts().CPlusPlus) {
       if (getLangOpts().ObjCAutoRefCount && Record && !ARCErrReported) {
         // It's an error in ARC if a field has lifetime.
         // We don't want to report this in a system header, though,
diff --git a/lib/Sema/SemaDeclAttr.cpp b/lib/Sema/SemaDeclAttr.cpp
index 22bff86..caa7b2f 100644
--- a/lib/Sema/SemaDeclAttr.cpp
+++ b/lib/Sema/SemaDeclAttr.cpp
@@ -221,6 +221,53 @@ static bool checkAttributeAtLeastNumArgs(Sema &S, const AttributeList &Attr,
   return true;
 }
 
+/// \brief Check if IdxExpr is a valid argument index for a function or
+/// instance method D.  May output an error.
+///
+/// \returns true if IdxExpr is a valid index.
+static bool checkFunctionOrMethodArgumentIndex(Sema &S, const Decl *D,
+                                               StringRef AttrName,
+                                               SourceLocation AttrLoc,
+                                               unsigned AttrArgNum,
+                                               const Expr *IdxExpr,
+                                               uint64_t &Idx)
+{
+  assert(isFunctionOrMethod(D) && hasFunctionProto(D));
+
+  // In C++ the implicit 'this' function parameter also counts.
+  // Parameters are counted from one.
+  const bool HasImplicitThisParam = isInstanceMethod(D);
+  const unsigned NumArgs = getFunctionOrMethodNumArgs(D) + HasImplicitThisParam;
+  const unsigned FirstIdx = 1;
+
+  llvm::APSInt IdxInt;
+  if (IdxExpr->isTypeDependent() || IdxExpr->isValueDependent() ||
+      !IdxExpr->isIntegerConstantExpr(IdxInt, S.Context)) {
+    S.Diag(AttrLoc, diag::err_attribute_argument_n_not_int)
+      << AttrName << AttrArgNum << IdxExpr->getSourceRange();
+    return false;
+  }
+
+  Idx = IdxInt.getLimitedValue();
+  if (Idx < FirstIdx || (!isFunctionOrMethodVariadic(D) && Idx > NumArgs)) {
+    S.Diag(AttrLoc, diag::err_attribute_argument_out_of_bounds)
+      << AttrName << AttrArgNum << IdxExpr->getSourceRange();
+    return false;
+  }
+  Idx--; // Convert to zero-based.
+  if (HasImplicitThisParam) {
+    if (Idx == 0) {
+      S.Diag(AttrLoc,
+             diag::err_attribute_invalid_implicit_this_argument)
+        << AttrName << IdxExpr->getSourceRange();
+      return false;
+    }
+    --Idx;
+  }
+
+  return true;
+}
+
 ///
 /// \brief Check if passed in Decl is a field or potentially shared global var
 /// \return true if the Decl is a field or potentially shared global variable
@@ -3523,25 +3570,16 @@ static void handleCallConvAttr(Sema &S, Decl *D, const AttributeList &Attr) {
     D->addAttr(::new (S.Context) PascalAttr(Attr.getRange(), S.Context));
     return;
   case AttributeList::AT_Pcs: {
-    Expr *Arg = Attr.getArg(0);
-    StringLiteral *Str = dyn_cast<StringLiteral>(Arg);
-    if (!Str || !Str->isAscii()) {
-      S.Diag(Attr.getLoc(), diag::err_attribute_argument_n_not_string)
-        << "pcs" << 1;
-      Attr.setInvalid();
-      return;
-    }
-
-    StringRef StrRef = Str->getString();
     PcsAttr::PCSType PCS;
-    if (StrRef == "aapcs")
+    switch (CC) {
+    case CC_AAPCS:
       PCS = PcsAttr::AAPCS;
-    else if (StrRef == "aapcs-vfp")
+      break;
+    case CC_AAPCS_VFP:
       PCS = PcsAttr::AAPCS_VFP;
-    else {
-      S.Diag(Attr.getLoc(), diag::err_invalid_pcs);
-      Attr.setInvalid();
-      return;
+      break;
+    default:
+      llvm_unreachable("unexpected calling convention in pcs attribute");
     }
 
     D->addAttr(::new (S.Context) PcsAttr(Attr.getRange(), S.Context, PCS));
@@ -3560,10 +3598,9 @@ bool Sema::CheckCallingConvAttr(const AttributeList &attr, CallingConv &CC) {
   if (attr.isInvalid())
     return true;
 
-  if ((attr.getNumArgs() != 0 &&
-      !(attr.getKind() == AttributeList::AT_Pcs && attr.getNumArgs() == 1)) ||
-      attr.getParameterName()) {
-    Diag(attr.getLoc(), diag::err_attribute_wrong_number_arguments) << 0;
+  unsigned ReqArgs = attr.getKind() == AttributeList::AT_Pcs ? 1 : 0;
+  if (attr.getNumArgs() != ReqArgs || attr.getParameterName()) {
+    Diag(attr.getLoc(), diag::err_attribute_wrong_number_arguments) << ReqArgs;
     attr.setInvalid();
     return true;
   }
@@ -3594,7 +3631,10 @@ bool Sema::CheckCallingConvAttr(const AttributeList &attr, CallingConv &CC) {
       CC = CC_AAPCS_VFP;
       break;
     }
-    // FALLS THROUGH
+
+    attr.setInvalid();
+    Diag(attr.getLoc(), diag::err_invalid_pcs);
+    return true;
   }
   default: llvm_unreachable("unexpected attribute kind");
   }
@@ -3703,6 +3743,79 @@ static void handleLaunchBoundsAttr(Sema &S, Decl *D, const AttributeList &Attr){
   }
 }
 
+static void handleArgumentWithTypeTagAttr(Sema &S, Decl *D,
+                                          const AttributeList &Attr) {
+  StringRef AttrName = Attr.getName()->getName();
+  if (!Attr.getParameterName()) {
+    S.Diag(Attr.getLoc(), diag::err_attribute_argument_n_not_identifier)
+      << Attr.getName() << /* arg num = */ 1;
+    return;
+  }
+
+  if (Attr.getNumArgs() != 2) {
+    S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments)
+      << /* required args = */ 3;
+    return;
+  }
+
+  IdentifierInfo *ArgumentKind = Attr.getParameterName();
+
+  if (!isFunctionOrMethod(D) || !hasFunctionProto(D)) {
+    S.Diag(Attr.getLoc(), diag::err_attribute_wrong_decl_type)
+      << Attr.getName() << ExpectedFunctionOrMethod;
+    return;
+  }
+
+  uint64_t ArgumentIdx;
+  if (!checkFunctionOrMethodArgumentIndex(S, D, AttrName,
+                                          Attr.getLoc(), 2,
+                                          Attr.getArg(0), ArgumentIdx))
+    return;
+
+  uint64_t TypeTagIdx;
+  if (!checkFunctionOrMethodArgumentIndex(S, D, AttrName,
+                                          Attr.getLoc(), 3,
+                                          Attr.getArg(1), TypeTagIdx))
+    return;
+
+  bool IsPointer = (AttrName == "pointer_with_type_tag");
+  if (IsPointer) {
+    // Ensure that buffer has a pointer type.
+    QualType BufferTy = getFunctionOrMethodArgType(D, ArgumentIdx);
+    if (!BufferTy->isPointerType()) {
+      S.Diag(Attr.getLoc(), diag::err_attribute_pointers_only)
+        << AttrName;
+    }
+  }
+
+  D->addAttr(::new (S.Context) ArgumentWithTypeTagAttr(Attr.getRange(),
+                                                       S.Context,
+                                                       ArgumentKind,
+                                                       ArgumentIdx,
+                                                       TypeTagIdx,
+                                                       IsPointer));
+}
+
+static void handleTypeTagForDatatypeAttr(Sema &S, Decl *D,
+                                         const AttributeList &Attr) {
+  IdentifierInfo *PointerKind = Attr.getParameterName();
+  if (!PointerKind) {
+    S.Diag(Attr.getLoc(), diag::err_attribute_argument_n_not_identifier)
+      << "type_tag_for_datatype" << 1;
+    return;
+  }
+
+  QualType MatchingCType = S.GetTypeFromParser(Attr.getMatchingCType(), NULL);
+
+  D->addAttr(::new (S.Context) TypeTagForDatatypeAttr(
+                                  Attr.getRange(),
+                                  S.Context,
+                                  PointerKind,
+                                  MatchingCType,
+                                  Attr.getLayoutCompatible(),
+                                  Attr.getMustBeNull()));
+}
+
 //===----------------------------------------------------------------------===//
 // Checker-specific attribute handlers.
 //===----------------------------------------------------------------------===//
@@ -4333,6 +4446,14 @@ static void ProcessInheritableDeclAttr(Sema &S, Scope *scope, Decl *D,
     handleAcquiredAfterAttr(S, D, Attr);
     break;
 
+  // Type safety attributes.
+  case AttributeList::AT_ArgumentWithTypeTag:
+    handleArgumentWithTypeTagAttr(S, D, Attr);
+    break;
+  case AttributeList::AT_TypeTagForDatatype:
+    handleTypeTagForDatatypeAttr(S, D, Attr);
+    break;
+
   default:
     // Ask target about the attribute.
     const TargetAttributesSema &TargetAttrs = S.getTargetAttributesSema();
diff --git a/lib/Sema/SemaDeclCXX.cpp b/lib/Sema/SemaDeclCXX.cpp
index 1d45a68..eeac9b8 100644
--- a/lib/Sema/SemaDeclCXX.cpp
+++ b/lib/Sema/SemaDeclCXX.cpp
@@ -9807,7 +9807,7 @@ Decl *Sema::BuildStaticAssertDeclaration(SourceLocation StaticAssertLoc,
     if (!Failed && !Cond) {
       llvm::SmallString<256> MsgBuffer;
       llvm::raw_svector_ostream Msg(MsgBuffer);
-      AssertMessage->printPretty(Msg, Context, 0, getPrintingPolicy());
+      AssertMessage->printPretty(Msg, 0, getPrintingPolicy());
       Diag(StaticAssertLoc, diag::err_static_assert_failed)
         << Msg.str() << AssertExpr->getSourceRange();
       Failed = true;
diff --git a/lib/Sema/SemaExceptionSpec.cpp b/lib/Sema/SemaExceptionSpec.cpp
index 63bfa9d..e6266fb 100644
--- a/lib/Sema/SemaExceptionSpec.cpp
+++ b/lib/Sema/SemaExceptionSpec.cpp
@@ -242,8 +242,7 @@ bool Sema::CheckEquivalentExceptionSpec(FunctionDecl *Old, FunctionDecl *New) {
 
     case EST_ComputedNoexcept:
       OS << "noexcept(";
-      OldProto->getNoexceptExpr()->printPretty(OS, Context, 0, 
-                                               getPrintingPolicy());
+      OldProto->getNoexceptExpr()->printPretty(OS, 0, getPrintingPolicy());
       OS << ")";
       break;
 
diff --git a/lib/Sema/SemaExpr.cpp b/lib/Sema/SemaExpr.cpp
index 6a503ee..3875ba1 100644
--- a/lib/Sema/SemaExpr.cpp
+++ b/lib/Sema/SemaExpr.cpp
@@ -9461,7 +9461,8 @@ ExprResult Sema::ActOnBlockStmtExpr(SourceLocation CaretLoc,
 
   // If needed, diagnose invalid gotos and switches in the block.
   if (getCurFunction()->NeedsScopeChecking() &&
-      !hasAnyUnrecoverableErrorsInThisFunction())
+      !hasAnyUnrecoverableErrorsInThisFunction() &&
+      !PP.isCodeCompletionEnabled())
     DiagnoseInvalidJumps(cast<CompoundStmt>(Body));
 
   BSI->TheDecl->setBody(cast<CompoundStmt>(Body));
diff --git a/lib/Sema/SemaExprMember.cpp b/lib/Sema/SemaExprMember.cpp
index 53f22f6..8f445e2 100644
--- a/lib/Sema/SemaExprMember.cpp
+++ b/lib/Sema/SemaExprMember.cpp
@@ -1137,7 +1137,7 @@ Sema::LookupMemberExpr(LookupResult &R, ExprResult &BaseExpr,
         goto fail;
       // There's an implicit 'isa' ivar on all objects.
       // But we only actually find it this way on objects of type 'id',
-      // apparently.ghjg
+      // apparently.
       if (OTy->isObjCId() && Member->isStr("isa")) {
         Diag(MemberLoc, diag::warn_objc_isa_use);
         return Owned(new (Context) ObjCIsaExpr(BaseExpr.take(), IsArrow, MemberLoc,
diff --git a/lib/Sema/SemaOverload.cpp b/lib/Sema/SemaOverload.cpp
index a874489..9382f7d 100644
--- a/lib/Sema/SemaOverload.cpp
+++ b/lib/Sema/SemaOverload.cpp
@@ -57,7 +57,7 @@ static bool IsStandardConversion(Sema &S, Expr* From, QualType ToType,
                                  StandardConversionSequence &SCS,
                                  bool CStyle,
                                  bool AllowObjCWritebackConversion);
-  
+
 static bool IsTransparentUnionStandardConversion(Sema &S, Expr* From, 
                                                  QualType &ToType,
                                                  bool InOverloadResolution,
diff --git a/lib/Sema/SemaStmt.cpp b/lib/Sema/SemaStmt.cpp
index d22deb2..86884b7 100644
--- a/lib/Sema/SemaStmt.cpp
+++ b/lib/Sema/SemaStmt.cpp
@@ -28,6 +28,7 @@
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Basic/TargetInfo.h"
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallString.h"
@@ -35,11 +36,15 @@
 #include "llvm/ADT/Triple.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCInstPrinter.h"
+#include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCObjectFileInfo.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCTargetAsmParser.h"
+#include "llvm/MC/MCParser/MCAsmLexer.h"
 #include "llvm/MC/MCParser/MCAsmParser.h"
 #include "llvm/Support/SourceMgr.h"
 #include "llvm/Support/TargetRegistry.h"
@@ -1516,7 +1521,6 @@ Sema::CheckObjCForCollectionOperand(SourceLocation forLoc, Expr *collection) {
 
 StmtResult
 Sema::ActOnObjCForCollectionStmt(SourceLocation ForLoc,
-                                 SourceLocation LParenLoc,
                                  Stmt *First, Expr *collection,
                                  SourceLocation RParenLoc) {
 
@@ -1698,9 +1702,9 @@ static bool ObjCEnumerationCollection(Expr *Collection) {
           && Collection->getType()->getAs<ObjCObjectPointerType>() != 0;
 }
 
-/// ActOnCXXForRangeStmt - Check and build a C++0x for-range statement.
+/// ActOnCXXForRangeStmt - Check and build a C++11 for-range statement.
 ///
-/// C++0x [stmt.ranged]:
+/// C++11 [stmt.ranged]:
 ///   A range-based for statement is equivalent to
 ///
 ///   {
@@ -1717,15 +1721,14 @@ static bool ObjCEnumerationCollection(Expr *Collection) {
 /// The body of the loop is not available yet, since it cannot be analysed until
 /// we have determined the type of the for-range-declaration.
 StmtResult
-Sema::ActOnCXXForRangeStmt(SourceLocation ForLoc, SourceLocation LParenLoc,
+Sema::ActOnCXXForRangeStmt(SourceLocation ForLoc,
                            Stmt *First, SourceLocation ColonLoc, Expr *Range,
                            SourceLocation RParenLoc) {
   if (!First || !Range)
     return StmtError();
 
   if (ObjCEnumerationCollection(Range))
-    return ActOnObjCForCollectionStmt(ForLoc, LParenLoc, First, Range,
-                                      RParenLoc);
+    return ActOnObjCForCollectionStmt(ForLoc, First, Range, RParenLoc);
 
   DeclStmt *DS = dyn_cast<DeclStmt>(First);
   assert(DS && "first part of for range not a decl stmt");
@@ -2759,165 +2762,225 @@ StmtResult Sema::ActOnAsmStmt(SourceLocation AsmLoc, bool IsSimple,
   return Owned(NS);
 }
 
-// needSpaceAsmToken - This function handles whitespace around asm punctuation.
-// Returns true if a space should be emitted.
-static inline bool needSpaceAsmToken(Token currTok) {
-  static Token prevTok;
-
-  // No need for space after prevToken.
-  switch(prevTok.getKind()) {
-  default:
-    break;
-  case tok::l_square:
-  case tok::r_square:
-  case tok::l_brace:
-  case tok::r_brace:
-  case tok::colon:
-    prevTok = currTok;
-    return false;
-  }
+// isMSAsmKeyword - Return true if this is an MS-style inline asm keyword. These
+// require special handling.
+static bool isMSAsmKeyword(StringRef Name) {
+  bool Ret = llvm::StringSwitch<bool>(Name)
+    .Cases("EVEN", "ALIGN", true) // Alignment directives.
+    .Cases("LENGTH", "SIZE", "TYPE", true) // Type and variable sizes.
+    .Case("_emit", true) // _emit Pseudoinstruction.
+    .Default(false);
+  return Ret;
+}
 
-  // No need for a space before currToken.
-  switch(currTok.getKind()) {
-  default:
-    break;
-  case tok::l_square:
-  case tok::r_square:
-  case tok::l_brace:
-  case tok::r_brace:
-  case tok::comma:
-  case tok::colon:
-    prevTok = currTok;
-    return false;
-  }
-  prevTok = currTok;
-  return true;
+static StringRef getSpelling(Sema &SemaRef, Token AsmTok) {
+  StringRef Asm;
+  SmallString<512> TokenBuf;
+  TokenBuf.resize(512);
+  bool StringInvalid = false;
+  Asm = SemaRef.PP.getSpelling(AsmTok, TokenBuf, &StringInvalid);
+  assert (!StringInvalid && "Expected valid string!");
+  return Asm;
 }
 
 static void patchMSAsmStrings(Sema &SemaRef, bool &IsSimple,
                               SourceLocation AsmLoc,
                               ArrayRef<Token> AsmToks,
-                              ArrayRef<unsigned> LineEnds,
                               const TargetInfo &TI,
+                              std::vector<llvm::BitVector> &AsmRegs,
+                              std::vector<llvm::BitVector> &AsmNames,
                               std::vector<std::string> &AsmStrings) {
   assert (!AsmToks.empty() && "Didn't expect an empty AsmToks!");
 
-  // Assume simple asm stmt until we parse a non-register identifer.
+  // Assume simple asm stmt until we parse a non-register identifer (or we just
+  // need to bail gracefully).
   IsSimple = true;
 
-  for (unsigned i = 0, e = LineEnds.size(); i != e; ++i) {
-    SmallString<512> Asm;
+  SmallString<512> Asm;
+  unsigned NumAsmStrings = 0;
+  for (unsigned i = 0, e = AsmToks.size(); i != e; ++i) {
+
+    // Determine if this should be considered a new asm.
+    bool isNewAsm = i == 0 || AsmToks[i].isAtStartOfLine() ||
+      AsmToks[i].is(tok::kw_asm);
+
+    // Emit the previous asm string.
+    if (i && isNewAsm) {
+      AsmStrings[NumAsmStrings++] = Asm.c_str();
+      if (AsmToks[i].is(tok::kw_asm)) {
+        ++i; // Skip __asm
+        assert (i != e && "Expected another token.");
+      }
+    }
 
-    // Check the operands.
-    for (unsigned j = (i == 0) ? 0 : LineEnds[i-1], e = LineEnds[i]; j != e; ++j) {
+    // Start a new asm string with the opcode.
+    if (isNewAsm) {
+      AsmRegs[NumAsmStrings].resize(AsmToks.size());
+      AsmNames[NumAsmStrings].resize(AsmToks.size());
 
-      IdentifierInfo *II;
-      if (j == 0 || (i > 0 && j == LineEnds[i-1])) {
-        II = AsmToks[j].getIdentifierInfo();
-        Asm = II->getName().str();
-        continue;
+      StringRef Piece = AsmToks[i].getIdentifierInfo()->getName();
+      // MS-style inline asm keywords require special handling.
+      if (isMSAsmKeyword(Piece))
+        IsSimple = false;
+
+      // TODO: Verify this is a valid opcode.
+      Asm = Piece;
+      continue;
+    }
+
+    if (i && AsmToks[i].hasLeadingSpace())
+      Asm += ' ';
+
+    // Check the operand(s).
+    switch (AsmToks[i].getKind()) {
+    default:
+      IsSimple = false;
+      Asm += getSpelling(SemaRef, AsmToks[i]);
+      break;
+    case tok::comma: Asm += ","; break;
+    case tok::colon: Asm += ":"; break;
+    case tok::l_square: Asm += "["; break;
+    case tok::r_square: Asm += "]"; break;
+    case tok::l_brace: Asm += "{"; break;
+    case tok::r_brace: Asm += "}"; break;
+    case tok::numeric_constant:
+      Asm += getSpelling(SemaRef, AsmToks[i]);
+      break;
+    case tok::identifier: {
+      IdentifierInfo *II = AsmToks[i].getIdentifierInfo();
+      StringRef Name = II->getName();
+
+      // Valid register?
+      if (TI.isValidGCCRegisterName(Name)) {
+        AsmRegs[NumAsmStrings].set(i);
+        Asm += Name;
+        break;
       }
 
-      if (needSpaceAsmToken(AsmToks[j]))
-        Asm += " ";
+      IsSimple = false;
 
-      switch (AsmToks[j].getKind()) {
-      default:
-        //llvm_unreachable("Unknown token.");
+      // MS-style inline asm keywords require special handling.
+      if (isMSAsmKeyword(Name)) {
+        IsSimple = false;
+        Asm += Name;
         break;
-      case tok::comma: Asm += ","; break;
-      case tok::colon: Asm += ":"; break;
-      case tok::l_square: Asm += "["; break;
-      case tok::r_square: Asm += "]"; break;
-      case tok::l_brace: Asm += "{"; break;
-      case tok::r_brace: Asm += "}"; break;
-      case tok::numeric_constant: {
-        SmallString<32> TokenBuf;
-        TokenBuf.resize(32);
-        bool StringInvalid = false;
-        Asm += SemaRef.PP.getSpelling(AsmToks[j], TokenBuf, &StringInvalid);
-        assert (!StringInvalid && "Expected valid string!");
+      }
+
+      // FIXME: Why are we missing this segment register?
+      if (Name == "fs") {
+        Asm += Name;
         break;
       }
-      case tok::identifier: {
-        II = AsmToks[j].getIdentifierInfo();
-        StringRef Name = II->getName();
 
-        // Valid registers don't need modification.
-        if (TI.isValidGCCRegisterName(Name)) {
-          Asm += Name;
-          break;
-        }
+      // Lookup the identifier.
+      // TODO: Someone with more experience with clang should verify this the
+      // proper way of doing a symbol lookup.
+      DeclarationName DeclName(II);
+      Scope *CurScope = SemaRef.getCurScope();
+      LookupResult R(SemaRef, DeclName, AsmLoc, Sema::LookupOrdinaryName);
+      if (!SemaRef.LookupName(R, CurScope, false/*AllowBuiltinCreation*/))
+        break;
 
-        // TODO: Lookup the identifier.
-        IsSimple = false;
+      assert (R.isSingleResult() && "Expected a single result?!");
+      NamedDecl *Decl = R.getFoundDecl();
+      switch (Decl->getKind()) {
+      default:
+        assert(0 && "Unknown decl kind.");
+        break;
+      case Decl::Var: {
+      case Decl::ParmVar:
+        AsmNames[NumAsmStrings].set(i);
+
+        VarDecl *Var = cast<VarDecl>(Decl);
+        QualType Ty = Var->getType();
+        (void)Ty; // Avoid warning.
+        // TODO: Patch identifier with valid operand.  One potential idea is to
+        // probe the backend with type information to guess the possible
+        // operand.
+        break;
+      }
       }
-      } // AsmToks[i].getKind()
+      break;
+    }
     }
-    AsmStrings[i] = Asm.c_str();
   }
+
+  // Emit the final (and possibly only) asm string.
+  AsmStrings[NumAsmStrings] = Asm.c_str();
 }
 
 // Build the unmodified MSAsmString.
 static std::string buildMSAsmString(Sema &SemaRef,
                                     ArrayRef<Token> AsmToks,
-                                    ArrayRef<unsigned> LineEnds) {
+                                    unsigned &NumAsmStrings) {
   assert (!AsmToks.empty() && "Didn't expect an empty AsmToks!");
+  NumAsmStrings = 0;
+
   SmallString<512> Asm;
-  SmallString<512> TokenBuf;
-  TokenBuf.resize(512);
-  unsigned AsmLineNum = 0;
   for (unsigned i = 0, e = AsmToks.size(); i < e; ++i) {
-    const char *ThisTokBuf = &TokenBuf[0];
-    bool StringInvalid = false;
-    unsigned ThisTokLen =
-      Lexer::getSpelling(AsmToks[i], ThisTokBuf, SemaRef.getSourceManager(),
-                         SemaRef.getLangOpts(), &StringInvalid);
-    if (i && (!AsmLineNum || i != LineEnds[AsmLineNum-1]) &&
-        needSpaceAsmToken(AsmToks[i]))
-      Asm += ' ';
-    Asm += StringRef(ThisTokBuf, ThisTokLen);
-    if (i + 1 == LineEnds[AsmLineNum] && i + 1 != AsmToks.size()) {
-      Asm += '\n';
-      ++AsmLineNum;
+    bool isNewAsm = i == 0 || AsmToks[i].isAtStartOfLine() ||
+      AsmToks[i].is(tok::kw_asm);
+
+    if (isNewAsm) {
+      ++NumAsmStrings;
+      if (i)
+        Asm += '\n';
+      if (AsmToks[i].is(tok::kw_asm)) {
+        i++; // Skip __asm
+        assert (i != e && "Expected another token");
+      }
     }
+
+    if (i && AsmToks[i].hasLeadingSpace() && !isNewAsm)
+      Asm += ' ';
+
+    Asm += getSpelling(SemaRef, AsmToks[i]);
   }
   return Asm.c_str();
 }
 
 StmtResult Sema::ActOnMSAsmStmt(SourceLocation AsmLoc,
+                                SourceLocation LBraceLoc,
                                 ArrayRef<Token> AsmToks,
-                                ArrayRef<unsigned> LineEnds,
                                 SourceLocation EndLoc) {
   // MS-style inline assembly is not fully supported, so emit a warning.
   Diag(AsmLoc, diag::warn_unsupported_msasm);
   SmallVector<StringRef,4> Clobbers;
+  std::set<std::string> ClobberRegs;
+  SmallVector<IdentifierInfo*, 4> Inputs;
+  SmallVector<IdentifierInfo*, 4> Outputs;
 
   // Empty asm statements don't need to instantiate the AsmParser, etc.
   if (AsmToks.empty()) {
     StringRef AsmString;
     MSAsmStmt *NS =
-      new (Context) MSAsmStmt(Context, AsmLoc, /* IsSimple */ true,
-                              /* IsVolatile */ true, AsmToks, LineEnds,
+      new (Context) MSAsmStmt(Context, AsmLoc, LBraceLoc, /*IsSimple*/ true,
+                              /*IsVolatile*/ true, AsmToks, Inputs, Outputs,
                               AsmString, Clobbers, EndLoc);
     return Owned(NS);
   }
 
-  std::string AsmString = buildMSAsmString(*this, AsmToks, LineEnds);
+  unsigned NumAsmStrings;
+  std::string AsmString = buildMSAsmString(*this, AsmToks, NumAsmStrings);
 
   bool IsSimple;
+  std::vector<llvm::BitVector> Regs;
+  std::vector<llvm::BitVector> Names;
   std::vector<std::string> PatchedAsmStrings;
-  PatchedAsmStrings.resize(LineEnds.size());
+
+  Regs.resize(NumAsmStrings);
+  Names.resize(NumAsmStrings);
+  PatchedAsmStrings.resize(NumAsmStrings);
 
   // Rewrite operands to appease the AsmParser.
-  patchMSAsmStrings(*this, IsSimple, AsmLoc, AsmToks, LineEnds, 
-                   Context.getTargetInfo(), PatchedAsmStrings);
+  patchMSAsmStrings(*this, IsSimple, AsmLoc, AsmToks,
+                    Context.getTargetInfo(), Regs, Names, PatchedAsmStrings);
 
   // patchMSAsmStrings doesn't correctly patch non-simple asm statements.
   if (!IsSimple) {
     MSAsmStmt *NS =
-      new (Context) MSAsmStmt(Context, AsmLoc, /* IsSimple */ true,
-                              /* IsVolatile */ true, AsmToks, LineEnds,
+      new (Context) MSAsmStmt(Context, AsmLoc, LBraceLoc, /*IsSimple*/ true,
+                              /*IsVolatile*/ true, AsmToks, Inputs, Outputs,
                               AsmString, Clobbers, EndLoc);
     return Owned(NS);
   }
@@ -2947,7 +3010,7 @@ StmtResult Sema::ActOnMSAsmStmt(SourceLocation AsmLoc,
     // Tell SrcMgr about this buffer, which is what the parser will pick up.
     SrcMgr.AddNewSourceBuffer(Buffer, llvm::SMLoc());
 
-    OwningPtr<llvm::MCStreamer> Str;
+    OwningPtr<llvm::MCStreamer> Str(createNullStreamer(Ctx));
     OwningPtr<llvm::MCAsmParser>
       Parser(createMCAsmParser(SrcMgr, Ctx, *Str.get(), *MAI));
     OwningPtr<llvm::MCTargetAsmParser>
@@ -2956,13 +3019,63 @@ StmtResult Sema::ActOnMSAsmStmt(SourceLocation AsmLoc,
     Parser->setAssemblerDialect(1);
     Parser->setTargetParser(*TargetParser.get());
 
-    // TODO: Start parsing.
+    // Prime the lexer.
+    Parser->Lex();
+
+    // Parse the opcode.
+    StringRef IDVal;
+    Parser->ParseIdentifier(IDVal);
+
+    // Canonicalize the opcode to lower case.
+    SmallString<128> Opcode;
+    for (unsigned i = 0, e = IDVal.size(); i != e; ++i)
+      Opcode.push_back(tolower(IDVal[i]));
+
+    // Parse the operands.
+    llvm::SMLoc IDLoc;
+    SmallVector<llvm::MCParsedAsmOperand*, 8> Operands;
+    bool HadError = TargetParser->ParseInstruction(Opcode.str(), IDLoc,
+                                                   Operands);
+    assert (!HadError && "Unexpected error parsing instruction");
+
+    // Match the MCInstr.
+    SmallVector<llvm::MCInst, 2> Instrs;
+    HadError = TargetParser->MatchInstruction(IDLoc, Operands, Instrs);
+    assert (!HadError && "Unexpected error matching instruction");
+    assert ((Instrs.size() == 1) && "Expected only a single instruction.");
+
+    // Get the instruction descriptor.
+    llvm::MCInst Inst = Instrs[0];
+    const llvm::MCInstrInfo *MII = TheTarget->createMCInstrInfo();
+    const llvm::MCInstrDesc &Desc = MII->get(Inst.getOpcode());
+    llvm::MCInstPrinter *IP =
+      TheTarget->createMCInstPrinter(1, *MAI, *MII, *MRI, *STI);
+
+    // Build the list of clobbers.
+    for (unsigned i = 0, e = Desc.getNumDefs(); i != e; ++i) {
+      const llvm::MCOperand &Op = Inst.getOperand(i);
+      if (!Op.isReg())
+        continue;
+
+      std::string Reg;
+      llvm::raw_string_ostream OS(Reg);
+      IP->printRegName(OS, Op.getReg());
+
+      StringRef Clobber(OS.str());
+      if (!Context.getTargetInfo().isValidClobber(Clobber))
+        return StmtError(Diag(AsmLoc, diag::err_asm_unknown_register_name) <<
+                         Clobber);
+      ClobberRegs.insert(Reg);
+    }
   }
+  for (std::set<std::string>::iterator I = ClobberRegs.begin(),
+         E = ClobberRegs.end(); I != E; ++I)
+    Clobbers.push_back(*I);
 
   MSAsmStmt *NS =
-    new (Context) MSAsmStmt(Context, AsmLoc, IsSimple, /* IsVolatile */ true,
-                            AsmToks, LineEnds, AsmString, Clobbers, EndLoc);
-
+    new (Context) MSAsmStmt(Context, AsmLoc, LBraceLoc, IsSimple,
+                            /*IsVolatile*/ true, AsmToks, Inputs, Outputs,
+                            AsmString, Clobbers, EndLoc);
   return Owned(NS);
 }
 
diff --git a/lib/Sema/SemaTemplate.cpp b/lib/Sema/SemaTemplate.cpp
index c8e4501..4dbf3e4 100644
--- a/lib/Sema/SemaTemplate.cpp
+++ b/lib/Sema/SemaTemplate.cpp
@@ -5518,6 +5518,13 @@ Sema::ActOnClassTemplateSpecialization(Scope *S, unsigned TagSpec,
   if (Attr)
     ProcessDeclAttributeList(S, Specialization, Attr);
 
+  // Add alignment attributes if necessary; these attributes are checked when
+  // the ASTContext lays out the structure.
+  if (TUK == TUK_Definition) {
+    AddAlignmentAttributesForRecord(Specialization);
+    AddMsStructLayoutForRecord(Specialization);
+  }
+
   if (ModulePrivateLoc.isValid())
     Diag(Specialization->getLocation(), diag::err_module_private_specialization)
       << (isPartialSpecialization? 1 : 0)
diff --git a/lib/Sema/SemaTemplateInstantiate.cpp b/lib/Sema/SemaTemplateInstantiate.cpp
index c7cbc41..20e755f 100644
--- a/lib/Sema/SemaTemplateInstantiate.cpp
+++ b/lib/Sema/SemaTemplateInstantiate.cpp
@@ -988,12 +988,11 @@ TemplateInstantiator::RebuildElaboratedType(SourceLocation KeywordLoc,
 
     SourceLocation TagLocation = KeywordLoc;
 
-    // FIXME: type might be anonymous.
     IdentifierInfo *Id = TD->getIdentifier();
 
     // TODO: should we even warn on struct/class mismatches for this?  Seems
     // like it's likely to produce a lot of spurious errors.
-    if (Keyword != ETK_None && Keyword != ETK_Typename) {
+    if (Id && Keyword != ETK_None && Keyword != ETK_Typename) {
       TagTypeKind Kind = TypeWithKeyword::getTagTypeKindForKeyword(Keyword);
       if (!SemaRef.isAcceptableTagRedeclaration(TD, Kind, /*isDefinition*/false,
                                                 TagLocation, *Id)) {
diff --git a/lib/Sema/SemaType.cpp b/lib/Sema/SemaType.cpp
index 20fe036..54f8dba 100644
--- a/lib/Sema/SemaType.cpp
+++ b/lib/Sema/SemaType.cpp
@@ -2258,6 +2258,51 @@ static TypeSourceInfo *GetFullTypeForDeclarator(TypeProcessingState &state,
         ASM = ArrayType::Normal;
         D.setInvalidType(true);
       }
+
+      // C99 6.7.5.2p1: The optional type qualifiers and the keyword static
+      // shall appear only in a declaration of a function parameter with an
+      // array type, ...
+      if (ASM == ArrayType::Static || ATI.TypeQuals) {
+        if (!(D.isPrototypeContext() ||
+              D.getContext() == Declarator::KNRTypeListContext)) {
+          S.Diag(DeclType.Loc, diag::err_array_static_outside_prototype) <<
+              (ASM == ArrayType::Static ? "'static'" : "type qualifier");
+          // Remove the 'static' and the type qualifiers.
+          if (ASM == ArrayType::Static)
+            ASM = ArrayType::Normal;
+          ATI.TypeQuals = 0;
+          D.setInvalidType(true);
+        }
+
+        // C99 6.7.5.2p1: ... and then only in the outermost array type
+        // derivation.
+        unsigned x = chunkIndex;
+        while (x != 0) {
+          // Walk outwards along the declarator chunks.
+          x--;
+          const DeclaratorChunk &DC = D.getTypeObject(x);
+          switch (DC.Kind) {
+          case DeclaratorChunk::Paren:
+            continue;
+          case DeclaratorChunk::Array:
+          case DeclaratorChunk::Pointer:
+          case DeclaratorChunk::Reference:
+          case DeclaratorChunk::MemberPointer:
+            S.Diag(DeclType.Loc, diag::err_array_static_not_outermost) <<
+              (ASM == ArrayType::Static ? "'static'" : "type qualifier");
+            if (ASM == ArrayType::Static)
+              ASM = ArrayType::Normal;
+            ATI.TypeQuals = 0;
+            D.setInvalidType(true);
+            break;
+          case DeclaratorChunk::Function:
+          case DeclaratorChunk::BlockPointer:
+            // These are invalid anyway, so just ignore.
+            break;
+          }
+        }
+      }
+
       T = S.BuildArrayType(T, ASM, ArraySize, ATI.TypeQuals,
                            SourceRange(DeclType.Loc, DeclType.EndLoc), Name);
       break;
@@ -3224,7 +3269,6 @@ namespace {
       assert(Chunk.Kind == DeclaratorChunk::Function);
       TL.setLocalRangeBegin(Chunk.Loc);
       TL.setLocalRangeEnd(Chunk.EndLoc);
-      TL.setTrailingReturn(Chunk.Fun.hasTrailingReturnType());
 
       const DeclaratorChunk::FunctionTypeInfo &FTI = Chunk.Fun;
       for (unsigned i = 0, e = TL.getNumArgs(), tpi = 0; i != e; ++i) {
diff --git a/lib/Sema/TreeTransform.h b/lib/Sema/TreeTransform.h
index 90d5840..619ad33 100644
--- a/lib/Sema/TreeTransform.h
+++ b/lib/Sema/TreeTransform.h
@@ -1185,10 +1185,10 @@ public:
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
   StmtResult RebuildMSAsmStmt(SourceLocation AsmLoc,
+                              SourceLocation LBraceLoc,
                               ArrayRef<Token> AsmToks,
-                              ArrayRef<unsigned> LineEnds,
                               SourceLocation EndLoc) {
-    return getSema().ActOnMSAsmStmt(AsmLoc, AsmToks, LineEnds, EndLoc);
+    return getSema().ActOnMSAsmStmt(AsmLoc, LBraceLoc, AsmToks, EndLoc);
   }
 
   /// \brief Build a new Objective-C \@try statement.
@@ -1277,12 +1277,11 @@ public:
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
   StmtResult RebuildObjCForCollectionStmt(SourceLocation ForLoc,
-                                          SourceLocation LParenLoc,
                                           Stmt *Element,
                                           Expr *Collection,
                                           SourceLocation RParenLoc,
                                           Stmt *Body) {
-    StmtResult ForEachStmt = getSema().ActOnObjCForCollectionStmt(ForLoc, LParenLoc,
+    StmtResult ForEachStmt = getSema().ActOnObjCForCollectionStmt(ForLoc,
                                                 Element,
                                                 Collection,
                                                 RParenLoc);
@@ -4205,7 +4204,7 @@ TreeTransform<Derived>::TransformFunctionProtoType(TypeLocBuilder &TLB,
 
   QualType ResultType;
 
-  if (TL.getTrailingReturn()) {
+  if (T->hasTrailingReturn()) {
     if (getDerived().TransformFunctionTypeParams(TL.getBeginLoc(),
                                                  TL.getParmArray(),
                                                  TL.getNumArgs(),
@@ -4262,7 +4261,6 @@ TreeTransform<Derived>::TransformFunctionProtoType(TypeLocBuilder &TLB,
   FunctionProtoTypeLoc NewTL = TLB.push<FunctionProtoTypeLoc>(Result);
   NewTL.setLocalRangeBegin(TL.getLocalRangeBegin());
   NewTL.setLocalRangeEnd(TL.getLocalRangeEnd());
-  NewTL.setTrailingReturn(TL.getTrailingReturn());
   for (unsigned i = 0, e = NewTL.getNumArgs(); i != e; ++i)
     NewTL.setArg(i, ParamDecls[i]);
 
@@ -4286,7 +4284,6 @@ QualType TreeTransform<Derived>::TransformFunctionNoProtoType(
   FunctionNoProtoTypeLoc NewTL = TLB.push<FunctionNoProtoTypeLoc>(Result);
   NewTL.setLocalRangeBegin(TL.getLocalRangeBegin());
   NewTL.setLocalRangeEnd(TL.getLocalRangeEnd());
-  NewTL.setTrailingReturn(false);
 
   return Result;
 }
@@ -5612,12 +5609,9 @@ StmtResult
 TreeTransform<Derived>::TransformMSAsmStmt(MSAsmStmt *S) {
   ArrayRef<Token> AsmToks =
     llvm::makeArrayRef(S->getAsmToks(), S->getNumAsmToks());
-  ArrayRef<unsigned> LineEnds =
-    llvm::makeArrayRef(S->getLineEnds(), S->getNumLineEnds());
 
-  // No need to transform the asm string literal.
-  return getDerived().RebuildMSAsmStmt(S->getAsmLoc(), AsmToks, LineEnds,
-                                       S->getEndLoc());
+  return getDerived().RebuildMSAsmStmt(S->getAsmLoc(), S->getLBraceLoc(),
+                                       AsmToks, S->getEndLoc());
 }
 
 template<typename Derived>
@@ -5808,7 +5802,6 @@ TreeTransform<Derived>::TransformObjCForCollectionStmt(
 
   // Build a new statement.
   return getDerived().RebuildObjCForCollectionStmt(S->getForLoc(),
-                                                   /*FIXME:*/S->getForLoc(),
                                                    Element.get(),
                                                    Collection.get(),
                                                    S->getRParenLoc(),
diff --git a/lib/Serialization/ASTReader.cpp b/lib/Serialization/ASTReader.cpp
index beef338..3adbc57 100644
--- a/lib/Serialization/ASTReader.cpp
+++ b/lib/Serialization/ASTReader.cpp
@@ -4259,7 +4259,6 @@ void TypeLocReader::VisitExtVectorTypeLoc(ExtVectorTypeLoc TL) {
 void TypeLocReader::VisitFunctionTypeLoc(FunctionTypeLoc TL) {
   TL.setLocalRangeBegin(ReadSourceLocation(Record, Idx));
   TL.setLocalRangeEnd(ReadSourceLocation(Record, Idx));
-  TL.setTrailingReturn(Record[Idx++]);
   for (unsigned i = 0, e = TL.getNumArgs(); i != e; ++i) {
     TL.setArg(i, ReadDeclAs<ParmVarDecl>(Record, Idx));
   }
diff --git a/lib/Serialization/ASTWriter.cpp b/lib/Serialization/ASTWriter.cpp
index b7718c4..425d2e3 100644
--- a/lib/Serialization/ASTWriter.cpp
+++ b/lib/Serialization/ASTWriter.cpp
@@ -486,7 +486,6 @@ void TypeLocWriter::VisitExtVectorTypeLoc(ExtVectorTypeLoc TL) {
 void TypeLocWriter::VisitFunctionTypeLoc(FunctionTypeLoc TL) {
   Writer.AddSourceLocation(TL.getLocalRangeBegin(), Record);
   Writer.AddSourceLocation(TL.getLocalRangeEnd(), Record);
-  Record.push_back(TL.getTrailingReturn());
   for (unsigned i = 0, e = TL.getNumArgs(); i != e; ++i)
     Writer.AddDeclRef(TL.getArg(i), Record);
 }
diff --git a/lib/StaticAnalyzer/Checkers/CallAndMessageChecker.cpp b/lib/StaticAnalyzer/Checkers/CallAndMessageChecker.cpp
index 30f45c7..5edcf09 100644
--- a/lib/StaticAnalyzer/Checkers/CallAndMessageChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/CallAndMessageChecker.cpp
@@ -235,17 +235,20 @@ void CallAndMessageChecker::checkPreStmt(const CallExpr *CE,
   ProgramStateRef StNonNull, StNull;
   llvm::tie(StNonNull, StNull) = State->assume(cast<DefinedOrUnknownSVal>(L));
 
-  // FIXME: Do we want to record the non-null assumption here?
   if (StNull && !StNonNull) {
     if (!BT_call_null)
       BT_call_null.reset(
         new BuiltinBug("Called function pointer is null (null dereference)"));
     emitBadCall(BT_call_null.get(), C, Callee);
   }
+
+  C.addTransition(StNonNull);
 }
 
 void CallAndMessageChecker::checkPreCall(const CallEvent &Call,
                                          CheckerContext &C) const {
+  ProgramStateRef State = C.getState();
+
   // If this is a call to a C++ method, check if the callee is null or
   // undefined.
   if (const CXXInstanceCall *CC = dyn_cast<CXXInstanceCall>(&Call)) {
@@ -258,11 +261,9 @@ void CallAndMessageChecker::checkPreCall(const CallEvent &Call,
       return;
     }
 
-    ProgramStateRef State = C.getState();
     ProgramStateRef StNonNull, StNull;
     llvm::tie(StNonNull, StNull) = State->assume(cast<DefinedOrUnknownSVal>(V));
 
-    // FIXME: Do we want to record the non-null assumption here?
     if (StNull && !StNonNull) {
       if (!BT_cxx_call_null)
         BT_cxx_call_null.reset(new BuiltinBug("Called C++ object pointer "
@@ -270,6 +271,8 @@ void CallAndMessageChecker::checkPreCall(const CallEvent &Call,
       emitBadCall(BT_cxx_call_null.get(), C, CC->getCXXThisExpr());
       return;
     }
+
+    State = StNonNull;
   }
 
   // Don't check for uninitialized field values in arguments if the
@@ -291,6 +294,9 @@ void CallAndMessageChecker::checkPreCall(const CallEvent &Call,
                            Call.getArgExpr(i), /*IsFirstArgument=*/i == 0,
                            checkUninitFields, Call, *BT))
       return;
+
+  // If we make it here, record our assumptions about the callee.
+  C.addTransition(State);
 }
 
 void CallAndMessageChecker::checkPreObjCMessage(const ObjCMethodCall &msg,
diff --git a/lib/StaticAnalyzer/Checkers/DynamicTypePropagation.cpp b/lib/StaticAnalyzer/Checkers/DynamicTypePropagation.cpp
index fea5733..b636efb 100644
--- a/lib/StaticAnalyzer/Checkers/DynamicTypePropagation.cpp
+++ b/lib/StaticAnalyzer/Checkers/DynamicTypePropagation.cpp
@@ -25,7 +25,8 @@ using namespace ento;
 
 namespace {
 class DynamicTypePropagation:
-    public Checker< check::PostCall,
+    public Checker< check::PreCall,
+                    check::PostCall,
                     check::PostStmt<ImplicitCastExpr> > {
   const ObjCObjectType *getObjectTypeForAllocAndNew(const ObjCMessageExpr *MsgE,
                                                     CheckerContext &C) const;
@@ -34,11 +35,70 @@ class DynamicTypePropagation:
   const ObjCObjectPointerType *getBetterObjCType(const Expr *CastE,
                                                  CheckerContext &C) const;
 public:
+  void checkPreCall(const CallEvent &Call, CheckerContext &C) const;
   void checkPostCall(const CallEvent &Call, CheckerContext &C) const;
   void checkPostStmt(const ImplicitCastExpr *CastE, CheckerContext &C) const;
 };
 }
 
+static void recordFixedType(const MemRegion *Region, const CXXMethodDecl *MD,
+                            CheckerContext &C) {
+  assert(Region);
+  assert(MD);
+
+  ASTContext &Ctx = C.getASTContext();
+  QualType Ty = Ctx.getPointerType(Ctx.getRecordType(MD->getParent()));
+
+  ProgramStateRef State = C.getState();
+  State = State->setDynamicTypeInfo(Region, Ty, /*CanBeSubclass=*/false);
+  C.addTransition(State);
+  return;
+}
+
+void DynamicTypePropagation::checkPreCall(const CallEvent &Call,
+                                          CheckerContext &C) const {
+  if (const CXXConstructorCall *Ctor = dyn_cast<CXXConstructorCall>(&Call)) {
+    // C++11 [class.cdtor]p4: When a virtual function is called directly or
+    //   indirectly from a constructor or from a destructor, including during
+    //   the construction or destruction of the class’s non-static data members,
+    //   and the object to which the call applies is the object under
+    //   construction or destruction, the function called is the final overrider
+    //   in the constructor's or destructor's class and not one overriding it in
+    //   a more-derived class.
+
+    switch (Ctor->getOriginExpr()->getConstructionKind()) {
+    case CXXConstructExpr::CK_Complete:
+    case CXXConstructExpr::CK_Delegating:
+      // No additional type info necessary.
+      return;
+    case CXXConstructExpr::CK_NonVirtualBase:
+    case CXXConstructExpr::CK_VirtualBase:
+      if (const MemRegion *Target = Ctor->getCXXThisVal().getAsRegion())
+        recordFixedType(Target, Ctor->getDecl(), C);
+      return;
+    }
+
+    return;
+  }
+
+  if (const CXXDestructorCall *Dtor = dyn_cast<CXXDestructorCall>(&Call)) {
+    // C++11 [class.cdtor]p4 (see above)
+
+    const MemRegion *Target = Dtor->getCXXThisVal().getAsRegion();
+    if (!Target)
+      return;
+
+    // FIXME: getRuntimeDefinition() can be expensive. It would be better to do
+    // this when we are entering the stack frame for the destructor.
+    const Decl *D = Dtor->getRuntimeDefinition().getDecl();
+    if (!D)
+      return;
+
+    recordFixedType(Target, cast<CXXDestructorDecl>(D), C);
+    return;
+  }
+}
+
 void DynamicTypePropagation::checkPostCall(const CallEvent &Call,
                                            CheckerContext &C) const {
   // We can obtain perfect type info for return values from some calls.
@@ -82,6 +142,31 @@ void DynamicTypePropagation::checkPostCall(const CallEvent &Call,
       break;
     }
     }
+
+    return;
+  }
+
+  if (const CXXConstructorCall *Ctor = dyn_cast<CXXConstructorCall>(&Call)) {
+    // We may need to undo the effects of our pre-call check.
+    switch (Ctor->getOriginExpr()->getConstructionKind()) {
+    case CXXConstructExpr::CK_Complete:
+    case CXXConstructExpr::CK_Delegating:
+      // No additional work necessary.
+      // Note: This will leave behind the actual type of the object for
+      // complete constructors, but arguably that's a good thing, since it
+      // means the dynamic type info will be correct even for objects
+      // constructed with operator new.
+      return;
+    case CXXConstructExpr::CK_NonVirtualBase:
+    case CXXConstructExpr::CK_VirtualBase:
+      if (const MemRegion *Target = Ctor->getCXXThisVal().getAsRegion()) {
+        // We just finished a base constructor. Now we can use the subclass's
+        // type when resolving virtual calls.
+        const Decl *D = C.getLocationContext()->getDecl();
+        recordFixedType(Target, cast<CXXConstructorDecl>(D), C);
+      }
+      return;
+    }
   }
 }
 
diff --git a/lib/StaticAnalyzer/Checkers/RetainCountChecker.cpp b/lib/StaticAnalyzer/Checkers/RetainCountChecker.cpp
index 5503b23..3c00d99 100644
--- a/lib/StaticAnalyzer/Checkers/RetainCountChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/RetainCountChecker.cpp
@@ -1107,18 +1107,6 @@ RetainSummaryManager::getFunctionSummary(const FunctionDecl *FD) {
     if (S)
       break;
 
-    // Enable this code once the semantics of NSDeallocateObject are resolved
-    // for GC.  <rdar://problem/6619988>
-#if 0
-    // Handle: NSDeallocateObject(id anObject);
-    // This method does allow 'nil' (although we don't check it now).
-    if (strcmp(FName, "NSDeallocateObject") == 0) {
-      return RetTy == Ctx.VoidTy
-        ? getPersistentSummary(RetEffect::MakeNoRet(), DoNothing, Dealloc)
-        : getPersistentStopSummary();
-    }
-#endif
-
     if (RetTy->isPointerType()) {
       // For CoreFoundation ('CF') types.
       if (cocoa::isRefType(RetTy, "CF", FName)) {
@@ -1591,28 +1579,12 @@ void RetainSummaryManager::InitializeMethodSummaries() {
 
   addClassMethSummary("NSWindow", "alloc", NoTrackYet);
 
-#if 0
-  addInstMethSummary("NSWindow", NoTrackYet, "initWithContentRect",
-                     "styleMask", "backing", "defer", NULL);
-
-  addInstMethSummary("NSWindow", NoTrackYet, "initWithContentRect",
-                     "styleMask", "backing", "defer", "screen", NULL);
-#endif
-
   // For NSPanel (which subclasses NSWindow), allocated objects are not
   //  self-owned.
   // FIXME: For now we don't track NSPanels. object for the same reason
   //   as for NSWindow objects.
   addClassMethSummary("NSPanel", "alloc", NoTrackYet);
 
-#if 0
-  addInstMethSummary("NSPanel", NoTrackYet, "initWithContentRect",
-                     "styleMask", "backing", "defer", NULL);
-
-  addInstMethSummary("NSPanel", NoTrackYet, "initWithContentRect",
-                     "styleMask", "backing", "defer", "screen", NULL);
-#endif
-
   // Don't track allocated autorelease pools yet, as it is okay to prematurely
   // exit a method.
   addClassMethSummary("NSAutoreleasePool", "alloc", NoTrackYet);
@@ -1636,62 +1608,6 @@ void RetainSummaryManager::InitializeMethodSummaries() {
 }
 
 //===----------------------------------------------------------------------===//
-// AutoreleaseBindings - State used to track objects in autorelease pools.
-//===----------------------------------------------------------------------===//
-#define AUTORELEASE_POOL_MODELING (0)
-// We do not currently have complete modeling of autorelease pools.
-
-#if AUTORELEASE_POOL_MODELING
-
-typedef llvm::ImmutableMap<SymbolRef, unsigned> ARCounts;
-typedef llvm::ImmutableMap<SymbolRef, ARCounts> ARPoolContents;
-typedef llvm::ImmutableList<SymbolRef> ARStack;
-
-static int AutoRCIndex = 0;
-static int AutoRBIndex = 0;
-
-namespace { class AutoreleasePoolContents {}; }
-namespace { class AutoreleaseStack {}; }
-
-namespace clang {
-namespace ento {
-template<> struct ProgramStateTrait<AutoreleaseStack>
-  : public ProgramStatePartialTrait<ARStack> {
-  static inline void *GDMIndex() { return &AutoRBIndex; }
-};
-
-template<> struct ProgramStateTrait<AutoreleasePoolContents>
-  : public ProgramStatePartialTrait<ARPoolContents> {
-  static inline void *GDMIndex() { return &AutoRCIndex; }
-};
-} // end GR namespace
-} // end clang namespace
-
-static SymbolRef GetCurrentAutoreleasePool(ProgramStateRef state) {
-  ARStack stack = state->get<AutoreleaseStack>();
-  return stack.isEmpty() ? SymbolRef() : stack.getHead();
-}
-
-static ProgramStateRef 
-SendAutorelease(ProgramStateRef state,
-                ARCounts::Factory &F,
-                SymbolRef sym) {
-  SymbolRef pool = GetCurrentAutoreleasePool(state);
-  const ARCounts *cnts = state->get<AutoreleasePoolContents>(pool);
-  ARCounts newCnts(0);
-
-  if (cnts) {
-    const unsigned *cnt = (*cnts).lookup(sym);
-    newCnts = F.add(*cnts, sym, cnt ? *cnt  + 1 : 1);
-  }
-  else
-    newCnts = F.add(F.getEmptyMap(), sym, 1);
-
-  return state->set<AutoreleasePoolContents>(pool, newCnts);
-}
-#endif
-
-//===----------------------------------------------------------------------===//
 // Error reporting.
 //===----------------------------------------------------------------------===//
 namespace {
@@ -2431,11 +2347,6 @@ class RetainCountChecker
 
   mutable OwningPtr<RetainSummaryManager> Summaries;
   mutable OwningPtr<RetainSummaryManager> SummariesGC;
-
-#if AUTORELEASE_POOL_MODELING
-  mutable ARCounts::Factory ARCountFactory;
-#endif
-
   mutable SummaryLogTy SummaryLog;
   mutable bool ShouldResetSummaryLog;
 
@@ -2892,15 +2803,7 @@ void RetainCountChecker::checkSummary(const RetainSummary &Summ,
                                                           ResultTy));
 
       // FIXME: Add a flag to the checker where allocations are assumed to
-      // *not* fail. (The code below is out-of-date, though.)
-#if 0
-      if (RE.getKind() == RetEffect::OwnedAllocatedSymbol) {
-        bool isFeasible;
-        state = state.assume(loc::SymbolVal(Sym), true, isFeasible);
-        assert(isFeasible && "Cannot assume fresh symbol is non-null.");
-      }
-#endif
-
+      // *not* fail.
       break;
     }
 
@@ -3011,9 +2914,6 @@ RetainCountChecker::updateSymbol(ProgramStateRef state, SymbolRef sym,
 
     case NewAutoreleasePool:
       assert(!C.isObjCGCEnabled());
-#if AUTORELEASE_POOL_MODELING
-      state = state->add<AutoreleaseStack>(sym);
-#endif
       return state;
 
     case MayEscape:
@@ -3030,13 +2930,7 @@ RetainCountChecker::updateSymbol(ProgramStateRef state, SymbolRef sym,
     case Autorelease:
       if (C.isObjCGCEnabled())
         return state;
-
       // Update the autorelease counts.
-      // TODO: AutoreleasePoolContents are not currently used. We will need to
-      // call SendAutorelease after it's wired up.
-#if AUTORELEASE_POOL_MODELING
-      state = SendAutorelease(state, ARCountFactory, sym);
-#endif
       V = V.autorelease();
       break;
 
@@ -3719,35 +3613,6 @@ void RetainCountChecker::checkDeadSymbols(SymbolReaper &SymReaper,
   C.addTransition(state, Pred);
 }
 
-//===----------------------------------------------------------------------===//
-// Debug printing of refcount bindings and autorelease pools.
-//===----------------------------------------------------------------------===//
-
-#if AUTORELEASE_POOL_MODELING
-static void PrintPool(raw_ostream &Out, SymbolRef Sym,
-                      ProgramStateRef State) {
-  Out << ' ';
-  if (Sym)
-    Sym->dumpToStream(Out);
-  else
-    Out << "<pool>";
-  Out << ":{";
-
-  // Get the contents of the pool.
-  if (const ARCounts *Cnts = State->get<AutoreleasePoolContents>(Sym))
-    for (ARCounts::iterator I = Cnts->begin(), E = Cnts->end(); I != E; ++I)
-      Out << '(' << I.getKey() << ',' << I.getData() << ')';
-  Out << '}';
-}
-
-static bool UsesAutorelease(ProgramStateRef state) {
-  // A state uses autorelease if it allocated an autorelease pool or if it has
-  // objects in the caller's autorelease pool.
-  return !state->get<AutoreleaseStack>().isEmpty() ||
-          state->get<AutoreleasePoolContents>(SymbolRef());
-}
-#endif
-
 void RetainCountChecker::printState(raw_ostream &Out, ProgramStateRef State,
                                     const char *NL, const char *Sep) const {
 
@@ -3761,20 +3626,6 @@ void RetainCountChecker::printState(raw_ostream &Out, ProgramStateRef State,
     I->second.print(Out);
     Out << NL;
   }
-
-#if AUTORELEASE_POOL_MODELING
-  // Print the autorelease stack.
-  if (UsesAutorelease(State)) {
-    Out << Sep << NL << "AR pool stack:";
-    ARStack Stack = State->get<AutoreleaseStack>();
-
-    PrintPool(Out, SymbolRef(), State);  // Print the caller's pool.
-    for (ARStack::iterator I = Stack.begin(), E = Stack.end(); I != E; ++I)
-      PrintPool(Out, *I, State);
-
-    Out << NL;
-  }
-#endif
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/lib/StaticAnalyzer/Core/AnalysisManager.cpp b/lib/StaticAnalyzer/Core/AnalysisManager.cpp
index 5aac640..efeba17 100644
--- a/lib/StaticAnalyzer/Core/AnalysisManager.cpp
+++ b/lib/StaticAnalyzer/Core/AnalysisManager.cpp
@@ -16,7 +16,7 @@ void AnalysisManager::anchor() { }
 
 AnalysisManager::AnalysisManager(ASTContext &ctx, DiagnosticsEngine &diags,
                                  const LangOptions &lang,
-                                 PathDiagnosticConsumer *pd,
+                                 const PathDiagnosticConsumers &PDC,
                                  StoreManagerCreator storemgr,
                                  ConstraintManagerCreator constraintmgr, 
                                  CheckerManager *checkerMgr,
@@ -33,7 +33,8 @@ AnalysisManager::AnalysisManager(ASTContext &ctx, DiagnosticsEngine &diags,
                                  AnalysisInliningMode IMode,
                                  bool NoRetry)
   : AnaCtxMgr(useUnoptimizedCFG, addImplicitDtors, /*addInitializers=*/true),
-    Ctx(ctx), Diags(diags), LangOpts(lang), PD(pd),
+    Ctx(ctx), Diags(diags), LangOpts(lang),
+    PathConsumers(PDC),
     CreateStoreMgr(storemgr), CreateConstraintMgr(constraintmgr),
     CheckerMgr(checkerMgr), 
     MaxNodes(maxnodes), MaxVisit(maxvisit),
@@ -49,29 +50,19 @@ AnalysisManager::AnalysisManager(ASTContext &ctx, DiagnosticsEngine &diags,
   AnaCtxMgr.getCFGBuildOptions().setAllAlwaysAdd();
 }
 
-AnalysisManager::AnalysisManager(ASTContext &ctx, DiagnosticsEngine &diags,
-                                 AnalysisManager &ParentAM)
-  : AnaCtxMgr(ParentAM.AnaCtxMgr.getUseUnoptimizedCFG(),
-              ParentAM.AnaCtxMgr.getCFGBuildOptions().AddImplicitDtors,
-              ParentAM.AnaCtxMgr.getCFGBuildOptions().AddInitializers),
-    Ctx(ctx), Diags(diags),
-    LangOpts(ParentAM.LangOpts), PD(ParentAM.getPathDiagnosticConsumer()),
-    CreateStoreMgr(ParentAM.CreateStoreMgr),
-    CreateConstraintMgr(ParentAM.CreateConstraintMgr),
-    CheckerMgr(ParentAM.CheckerMgr),
-    MaxNodes(ParentAM.MaxNodes),
-    MaxVisit(ParentAM.MaxVisit),
-    VisualizeEGDot(ParentAM.VisualizeEGDot),
-    VisualizeEGUbi(ParentAM.VisualizeEGUbi),
-    PurgeDead(ParentAM.PurgeDead),
-    EagerlyAssume(ParentAM.EagerlyAssume),
-    TrimGraph(ParentAM.TrimGraph),
-    EagerlyTrimEGraph(ParentAM.EagerlyTrimEGraph),
-    IPAMode(ParentAM.IPAMode),
-    InlineMaxStackDepth(ParentAM.InlineMaxStackDepth),
-    InlineMaxFunctionSize(ParentAM.InlineMaxFunctionSize),
-    InliningMode(ParentAM.InliningMode),
-    NoRetryExhausted(ParentAM.NoRetryExhausted)
-{
-  AnaCtxMgr.getCFGBuildOptions().setAllAlwaysAdd();
+AnalysisManager::~AnalysisManager() {
+  FlushDiagnostics();
+  for (PathDiagnosticConsumers::iterator I = PathConsumers.begin(),
+       E = PathConsumers.end(); I != E; ++I) {
+    delete *I;
+  }
+}
+
+void AnalysisManager::FlushDiagnostics() {
+  PathDiagnosticConsumer::FilesMade filesMade;
+  for (PathDiagnosticConsumers::iterator I = PathConsumers.begin(),
+       E = PathConsumers.end();
+       I != E; ++I) {
+    (*I)->FlushDiagnostics(&filesMade);
+  }
 }
diff --git a/lib/StaticAnalyzer/Core/BugReporter.cpp b/lib/StaticAnalyzer/Core/BugReporter.cpp
index 7ba2fa7..571baec 100644
--- a/lib/StaticAnalyzer/Core/BugReporter.cpp
+++ b/lib/StaticAnalyzer/Core/BugReporter.cpp
@@ -1345,6 +1345,9 @@ BugReport::~BugReport() {
   for (visitor_iterator I = visitor_begin(), E = visitor_end(); I != E; ++I) {
     delete *I;
   }
+  while (!interestingSymbols.empty()) {
+    popInterestingSymbolsAndRegions();
+  }
 }
 
 const Decl *BugReport::getDeclWithIssue() const {
@@ -1386,11 +1389,11 @@ void BugReport::markInteresting(SymbolRef sym) {
     return;
 
   // If the symbol wasn't already in our set, note a configuration change.
-  if (interestingSymbols.insert(sym).second)
+  if (getInterestingSymbols().insert(sym).second)
     ++ConfigurationChangeToken;
 
   if (const SymbolMetadata *meta = dyn_cast<SymbolMetadata>(sym))
-    interestingRegions.insert(meta->getRegion());
+    getInterestingRegions().insert(meta->getRegion());
 }
 
 void BugReport::markInteresting(const MemRegion *R) {
@@ -1399,11 +1402,11 @@ void BugReport::markInteresting(const MemRegion *R) {
 
   // If the base region wasn't already in our set, note a configuration change.
   R = R->getBaseRegion();
-  if (interestingRegions.insert(R).second)
+  if (getInterestingRegions().insert(R).second)
     ++ConfigurationChangeToken;
 
   if (const SymbolicRegion *SR = dyn_cast<SymbolicRegion>(R))
-    interestingSymbols.insert(SR->getSymbol());
+    getInterestingSymbols().insert(SR->getSymbol());
 }
 
 void BugReport::markInteresting(SVal V) {
@@ -1411,30 +1414,58 @@ void BugReport::markInteresting(SVal V) {
   markInteresting(V.getAsSymbol());
 }
 
-bool BugReport::isInteresting(SVal V) const {
+bool BugReport::isInteresting(SVal V) {
   return isInteresting(V.getAsRegion()) || isInteresting(V.getAsSymbol());
 }
 
-bool BugReport::isInteresting(SymbolRef sym) const {
+bool BugReport::isInteresting(SymbolRef sym) {
   if (!sym)
     return false;
   // We don't currently consider metadata symbols to be interesting
   // even if we know their region is interesting. Is that correct behavior?
-  return interestingSymbols.count(sym);
+  return getInterestingSymbols().count(sym);
 }
 
-bool BugReport::isInteresting(const MemRegion *R) const {
+bool BugReport::isInteresting(const MemRegion *R) {
   if (!R)
     return false;
   R = R->getBaseRegion();
-  bool b = interestingRegions.count(R);
+  bool b = getInterestingRegions().count(R);
   if (b)
     return true;
   if (const SymbolicRegion *SR = dyn_cast<SymbolicRegion>(R))
-    return interestingSymbols.count(SR->getSymbol());
+    return getInterestingSymbols().count(SR->getSymbol());
   return false;
 }
-  
+
+void BugReport::lazyInitializeInterestingSets() {
+  if (interestingSymbols.empty()) {
+    interestingSymbols.push_back(new Symbols());
+    interestingRegions.push_back(new Regions());
+  }
+}
+
+BugReport::Symbols &BugReport::getInterestingSymbols() {
+  lazyInitializeInterestingSets();
+  return *interestingSymbols.back();
+}
+
+BugReport::Regions &BugReport::getInterestingRegions() {
+  lazyInitializeInterestingSets();
+  return *interestingRegions.back();
+}
+
+void BugReport::pushInterestingSymbolsAndRegions() {
+  interestingSymbols.push_back(new Symbols(getInterestingSymbols()));
+  interestingRegions.push_back(new Regions(getInterestingRegions()));
+}
+
+void BugReport::popInterestingSymbolsAndRegions() {
+  delete interestingSymbols.back();
+  interestingSymbols.pop_back();
+  delete interestingRegions.back();
+  interestingRegions.pop_back();
+}
 
 const Stmt *BugReport::getStmt() const {
   if (!ErrorNode)
@@ -1793,12 +1824,13 @@ static void CompactPathDiagnostic(PathPieces &path, const SourceManager& SM) {
 }
 
 void GRBugReporter::GeneratePathDiagnostic(PathDiagnostic& PD,
-                        SmallVectorImpl<BugReport *> &bugReports) {
+                                           PathDiagnosticConsumer &PC,
+                                           ArrayRef<BugReport *> &bugReports) {
 
   assert(!bugReports.empty());
   SmallVector<const ExplodedNode *, 10> errorNodes;
-  for (SmallVectorImpl<BugReport*>::iterator I = bugReports.begin(),
-    E = bugReports.end(); I != E; ++I) {
+  for (ArrayRef<BugReport*>::iterator I = bugReports.begin(),
+                                      E = bugReports.end(); I != E; ++I) {
       errorNodes.push_back((*I)->getErrorNode());
   }
 
@@ -1818,8 +1850,7 @@ void GRBugReporter::GeneratePathDiagnostic(PathDiagnostic& PD,
   const ExplodedNode *N = GPair.second.first;
 
   // Start building the path diagnostic...
-  PathDiagnosticBuilder PDB(*this, R, BackMap.get(),
-                            getPathDiagnosticConsumer());
+  PathDiagnosticBuilder PDB(*this, R, BackMap.get(), &PC);
 
   // Register additional node visitors.
   R->addVisitor(new NilReceiverBRVisitor());
@@ -1867,6 +1898,8 @@ void GRBugReporter::GeneratePathDiagnostic(PathDiagnostic& PD,
     case PathDiagnosticConsumer::Minimal:
       GenerateMinimalPathDiagnostic(PD, PDB, N, visitors);
       break;
+    case PathDiagnosticConsumer::None:
+      llvm_unreachable("PathDiagnosticConsumer::None should never appear here");
     }
 
     // Clean up the visitors we used.
@@ -2022,53 +2055,21 @@ FindReportInEquivalenceClass(BugReportEquivClass& EQ,
   return exampleReport;
 }
 
-//===----------------------------------------------------------------------===//
-// DiagnosticCache.  This is a hack to cache analyzer diagnostics.  It
-// uses global state, which eventually should go elsewhere.
-//===----------------------------------------------------------------------===//
-namespace {
-class DiagCacheItem : public llvm::FoldingSetNode {
-  llvm::FoldingSetNodeID ID;
-public:
-  DiagCacheItem(BugReport *R, PathDiagnostic *PD) {
-    R->Profile(ID);
-    PD->Profile(ID);
-  }
-  
-  void Profile(llvm::FoldingSetNodeID &id) {
-    id = ID;
-  }
-  
-  llvm::FoldingSetNodeID &getID() { return ID; }
-};
-}
-
-static bool IsCachedDiagnostic(BugReport *R, PathDiagnostic *PD) {
-  // FIXME: Eventually this diagnostic cache should reside in something
-  // like AnalysisManager instead of being a static variable.  This is
-  // really unsafe in the long term.
-  typedef llvm::FoldingSet<DiagCacheItem> DiagnosticCache;
-  static DiagnosticCache DC;
-  
-  void *InsertPos;
-  DiagCacheItem *Item = new DiagCacheItem(R, PD);
-  
-  if (DC.FindNodeOrInsertPos(Item->getID(), InsertPos)) {
-    delete Item;
-    return true;
-  }
-  
-  DC.InsertNode(Item, InsertPos);
-  return false;
-}
-
 void BugReporter::FlushReport(BugReportEquivClass& EQ) {
   SmallVector<BugReport*, 10> bugReports;
   BugReport *exampleReport = FindReportInEquivalenceClass(EQ, bugReports);
-  if (!exampleReport)
-    return;
-  
-  PathDiagnosticConsumer* PD = getPathDiagnosticConsumer();
+  if (exampleReport) {
+    const PathDiagnosticConsumers &C = getPathDiagnosticConsumers();
+    for (PathDiagnosticConsumers::const_iterator I=C.begin(),
+                                                 E=C.end(); I != E; ++I) {
+      FlushReport(exampleReport, **I, bugReports);
+    }
+  }
+}
+
+void BugReporter::FlushReport(BugReport *exampleReport,
+                              PathDiagnosticConsumer &PD,
+                              ArrayRef<BugReport*> bugReports) {
 
   // FIXME: Make sure we use the 'R' for the path that was actually used.
   // Probably doesn't make a difference in practice.
@@ -2077,65 +2078,39 @@ void BugReporter::FlushReport(BugReportEquivClass& EQ) {
   OwningPtr<PathDiagnostic>
     D(new PathDiagnostic(exampleReport->getDeclWithIssue(),
                          exampleReport->getBugType().getName(),
-                         !PD || PD->useVerboseDescription()
+                         PD.useVerboseDescription()
                          ? exampleReport->getDescription() 
                          : exampleReport->getShortDescription(),
                          BT.getCategory()));
 
-  if (!bugReports.empty())
-    GeneratePathDiagnostic(*D.get(), bugReports);
-  
-  // Get the meta data.
-  const BugReport::ExtraTextList &Meta =
-                                  exampleReport->getExtraText();
-  for (BugReport::ExtraTextList::const_iterator i = Meta.begin(),
-                                                e = Meta.end(); i != e; ++i) {
-    D->addMeta(*i);
-  }
-
-  // Emit a summary diagnostic to the regular Diagnostics engine.
-  BugReport::ranges_iterator Beg, End;
-  llvm::tie(Beg, End) = exampleReport->getRanges();
-  DiagnosticsEngine &Diag = getDiagnostic();
-  
-  if (!IsCachedDiagnostic(exampleReport, D.get())) {
-    // Search the description for '%', as that will be interpretted as a
-    // format character by FormatDiagnostics.
-    StringRef desc = exampleReport->getShortDescription();
-
-    SmallString<512> TmpStr;
-    llvm::raw_svector_ostream Out(TmpStr);
-    for (StringRef::iterator I=desc.begin(), E=desc.end(); I!=E; ++I) {
-      if (*I == '%')
-        Out << "%%";
-      else
-        Out << *I;
-    }
-    
-    Out.flush();
-    unsigned ErrorDiag = Diag.getCustomDiagID(DiagnosticsEngine::Warning, TmpStr);
-
-    DiagnosticBuilder diagBuilder = Diag.Report(
-      exampleReport->getLocation(getSourceManager()).asLocation(), ErrorDiag);
-    for (BugReport::ranges_iterator I = Beg; I != End; ++I)
-      diagBuilder << *I;
+  // Generate the full path diagnostic, using the generation scheme
+  // specified by the PathDiagnosticConsumer.
+  if (PD.getGenerationScheme() != PathDiagnosticConsumer::None) {
+    if (!bugReports.empty())
+      GeneratePathDiagnostic(*D.get(), PD, bugReports);
   }
 
-  // Emit a full diagnostic for the path if we have a PathDiagnosticConsumer.
-  if (!PD)
-    return;
-
+  // If the path is empty, generate a single step path with the location
+  // of the issue.
   if (D->path.empty()) {
-    PathDiagnosticPiece *piece = new PathDiagnosticEventPiece(
-                                 exampleReport->getLocation(getSourceManager()),
-                                 exampleReport->getDescription());
+    PathDiagnosticLocation L = exampleReport->getLocation(getSourceManager());
+    PathDiagnosticPiece *piece =
+      new PathDiagnosticEventPiece(L, exampleReport->getDescription());
+    BugReport::ranges_iterator Beg, End;
+    llvm::tie(Beg, End) = exampleReport->getRanges();
     for ( ; Beg != End; ++Beg)
       piece->addRange(*Beg);
-
     D->getActivePath().push_back(piece);
   }
 
-  PD->HandlePathDiagnostic(D.take());
+  // Get the meta data.
+  const BugReport::ExtraTextList &Meta = exampleReport->getExtraText();
+  for (BugReport::ExtraTextList::const_iterator i = Meta.begin(),
+                                                e = Meta.end(); i != e; ++i) {
+    D->addMeta(*i);
+  }
+
+  PD.HandlePathDiagnostic(D.take());
 }
 
 void BugReporter::EmitBasicReport(const Decl *DeclWithIssue,
diff --git a/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp b/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
index 46aa9e2..e729587 100644
--- a/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
+++ b/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
@@ -323,7 +323,7 @@ void bugreporter::addTrackNullOrUndefValueVisitor(const ExplodedNode *N,
   // Walk through lvalue-to-rvalue conversions.
   const Expr *Ex = dyn_cast<Expr>(S);
   if (Ex) {
-    Ex = Ex->IgnoreParenLValueCasts();
+    Ex = Ex->IgnoreParenCasts();
     if (const DeclRefExpr *DR = dyn_cast<DeclRefExpr>(Ex)) {
       if (const VarDecl *VD = dyn_cast<VarDecl>(DR->getDecl())) {
         const VarRegion *R =
diff --git a/lib/StaticAnalyzer/Core/CallEvent.cpp b/lib/StaticAnalyzer/Core/CallEvent.cpp
index e3f4c61..5345bd5 100644
--- a/lib/StaticAnalyzer/Core/CallEvent.cpp
+++ b/lib/StaticAnalyzer/Core/CallEvent.cpp
@@ -207,10 +207,14 @@ SourceRange CallEvent::getArgSourceRange(unsigned Index) const {
   return ArgE->getSourceRange();
 }
 
+void CallEvent::dump() const {
+  dump(llvm::errs());
+}
+
 void CallEvent::dump(raw_ostream &Out) const {
   ASTContext &Ctx = getState()->getStateManager().getContext();
   if (const Expr *E = getOriginExpr()) {
-    E->printPretty(Out, Ctx, 0, Ctx.getPrintingPolicy());
+    E->printPretty(Out, 0, Ctx.getPrintingPolicy());
     Out << "\n";
     return;
   }
@@ -372,47 +376,49 @@ void CXXInstanceCall::getExtraInvalidatedRegions(RegionList &Regions) const {
     Regions.push_back(R);
 }
 
-static const CXXMethodDecl *devirtualize(const CXXMethodDecl *MD, SVal ThisVal){
-  const MemRegion *R = ThisVal.getAsRegion();
-  if (!R)
-    return 0;
-
-  const TypedValueRegion *TR = dyn_cast<TypedValueRegion>(R->StripCasts());
-  if (!TR)
-    return 0;
-
-  const CXXRecordDecl *RD = TR->getValueType()->getAsCXXRecordDecl();
-  if (!RD)
-    return 0;
-
-  const CXXMethodDecl *Result = MD->getCorrespondingMethodInClass(RD);
-  const FunctionDecl *Definition;
-  if (!Result->hasBody(Definition))
-    return 0;
-
-  return cast<CXXMethodDecl>(Definition);
-}
-
 
 RuntimeDefinition CXXInstanceCall::getRuntimeDefinition() const {
+  // Do we have a decl at all?
   const Decl *D = getDecl();
   if (!D)
     return RuntimeDefinition();
 
+  // If the method is non-virtual, we know we can inline it.
   const CXXMethodDecl *MD = cast<CXXMethodDecl>(D);
   if (!MD->isVirtual())
     return AnyFunctionCall::getRuntimeDefinition();
 
-  // If the method is virtual, see if we can find the actual implementation
-  // based on context-sensitivity.
-  // FIXME: Virtual method calls behave differently when an object is being
-  // constructed or destructed. It's not as simple as "no devirtualization"
-  // because a /partially/ constructed object can be referred to through a
-  // base pointer. We'll eventually want to use DynamicTypeInfo here.
-  if (const CXXMethodDecl *Devirtualized = devirtualize(MD, getCXXThisVal()))
-    return RuntimeDefinition(Devirtualized);
+  // Do we know the implicit 'this' object being called?
+  const MemRegion *R = getCXXThisVal().getAsRegion();
+  if (!R)
+    return RuntimeDefinition();
 
-  return RuntimeDefinition();
+  // Do we know anything about the type of 'this'?
+  DynamicTypeInfo DynType = getState()->getDynamicTypeInfo(R);
+  if (!DynType.isValid())
+    return RuntimeDefinition();
+
+  // Is the type a C++ class? (This is mostly a defensive check.)
+  QualType RegionType = DynType.getType()->getPointeeType();
+  const CXXRecordDecl *RD = RegionType->getAsCXXRecordDecl();
+  if (!RD || !RD->hasDefinition())
+    return RuntimeDefinition();
+
+  // Find the decl for this method in that class.
+  const CXXMethodDecl *Result = MD->getCorrespondingMethodInClass(RD, true);
+  assert(Result && "At the very least the static decl should show up.");
+
+  // Does the decl that we found have an implementation?
+  const FunctionDecl *Definition;
+  if (!Result->hasBody(Definition))
+    return RuntimeDefinition();
+
+  // We found a definition. If we're not sure that this devirtualization is
+  // actually what will happen at runtime, make sure to provide the region so
+  // that ExprEngine can decide what to do with it.
+  if (DynType.canBeASubClass())
+    return RuntimeDefinition(Definition, R->StripCasts());
+  return RuntimeDefinition(Definition, /*DispatchRegion=*/0);
 }
 
 void CXXInstanceCall::getInitialStackFrameContents(
@@ -421,16 +427,17 @@ void CXXInstanceCall::getInitialStackFrameContents(
   AnyFunctionCall::getInitialStackFrameContents(CalleeCtx, Bindings);
 
   // Handle the binding of 'this' in the new stack frame.
-  // We need to make sure we have the proper layering of CXXBaseObjectRegions.
   SVal ThisVal = getCXXThisVal();
   if (!ThisVal.isUnknown()) {
     ProgramStateManager &StateMgr = getState()->getStateManager();
     SValBuilder &SVB = StateMgr.getSValBuilder();
-    
+
     const CXXMethodDecl *MD = cast<CXXMethodDecl>(CalleeCtx->getDecl());
     Loc ThisLoc = SVB.getCXXThis(MD, CalleeCtx);
 
-    if (const MemRegion *ThisReg = ThisVal.getAsRegion()) {
+    // If we devirtualized to a different member function, we need to make sure
+    // we have the proper layering of CXXBaseObjectRegions.
+    if (MD->getCanonicalDecl() != getDecl()->getCanonicalDecl()) {
       ASTContext &Ctx = SVB.getContext();
       const CXXRecordDecl *Class = MD->getParent();
       QualType Ty = Ctx.getPointerType(Ctx.getRecordType(Class));
@@ -439,13 +446,10 @@ void CXXInstanceCall::getInitialStackFrameContents(
       bool Failed;
       ThisVal = StateMgr.getStoreManager().evalDynamicCast(ThisVal, Ty, Failed);
       assert(!Failed && "Calling an incorrectly devirtualized method");
-
-      // If we couldn't build the correct cast, just strip off all casts.
-      if (ThisVal.isUnknown())
-        ThisVal = loc::MemRegionVal(ThisReg->StripCasts());
     }
 
-    Bindings.push_back(std::make_pair(ThisLoc, ThisVal));
+    if (!ThisVal.isUnknown())
+      Bindings.push_back(std::make_pair(ThisLoc, ThisVal));
   }
 }
 
@@ -666,6 +670,9 @@ bool ObjCMethodCall::canBeOverridenInSubclass(ObjCInterfaceDecl *IDecl,
   if (InterfLoc.isValid() && SM.isFromMainFile(InterfLoc))
     return false;
 
+  // Assume that property accessors are not overridden.
+  if (getMessageKind() == OCM_PropertyAccess)
+    return false;
 
   // We assume that if the method is public (declared outside of main file) or
   // has a parent which publicly declares the method, the method could be
@@ -853,4 +860,3 @@ CallEventManager::getCaller(const StackFrameContext *CalleeCtx,
   return getCXXDestructorCall(Dtor, Trigger, ThisVal.getAsRegion(),
                               State, CallerCtx);
 }
-
diff --git a/lib/StaticAnalyzer/Core/ExprEngineCallAndReturn.cpp b/lib/StaticAnalyzer/Core/ExprEngineCallAndReturn.cpp
index 8ee6723..3b2e4ec 100644
--- a/lib/StaticAnalyzer/Core/ExprEngineCallAndReturn.cpp
+++ b/lib/StaticAnalyzer/Core/ExprEngineCallAndReturn.cpp
@@ -154,6 +154,11 @@ void ExprEngine::processCallExit(ExplodedNode *CEBNode) {
     }
   }
 
+  // Generate a CallEvent /before/ cleaning the state, so that we can get the
+  // correct value for 'this' (if necessary).
+  CallEventManager &CEMgr = getStateManager().getCallEventManager();
+  CallEventRef<> Call = CEMgr.getCaller(calleeCtx, state);
+
   // Step 3: BindedRetNode -> CleanedNodes
   // If we can find a statement and a block in the inlined function, run remove
   // dead bindings before returning from the call. This is important to ensure
@@ -203,21 +208,21 @@ void ExprEngine::processCallExit(ExplodedNode *CEBNode) {
         &Ctx);
     SaveAndRestore<unsigned> CBISave(currentStmtIdx, calleeCtx->getIndex());
 
-    CallEventManager &CEMgr = getStateManager().getCallEventManager();
-    CallEventRef<> Call = CEMgr.getCaller(calleeCtx, CEEState);
+    CallEventRef<> UpdatedCall = Call.cloneWithState(CEEState);
 
     ExplodedNodeSet DstPostCall;
-    getCheckerManager().runCheckersForPostCall(DstPostCall, CEENode, *Call,
-                                               *this, true);
+    getCheckerManager().runCheckersForPostCall(DstPostCall, CEENode,
+                                               *UpdatedCall, *this,
+                                               /*WasInlined=*/true);
 
     ExplodedNodeSet Dst;
-    if (isa<ObjCMethodCall>(Call)) {
-      getCheckerManager().runCheckersForPostObjCMessage(Dst, DstPostCall,
-                                                    cast<ObjCMethodCall>(*Call),
-                                                        *this, true);
+    if (const ObjCMethodCall *Msg = dyn_cast<ObjCMethodCall>(Call)) {
+      getCheckerManager().runCheckersForPostObjCMessage(Dst, DstPostCall, *Msg,
+                                                        *this,
+                                                        /*WasInlined=*/true);
     } else if (CE) {
       getCheckerManager().runCheckersForPostStmt(Dst, DstPostCall, CE,
-                                                 *this, true);
+                                                 *this, /*WasInlined=*/true);
     } else {
       Dst.insert(DstPostCall);
     }
@@ -555,12 +560,20 @@ void ExprEngine::defaultEvalCall(NodeBuilder &Bldr, ExplodedNode *Pred,
     RuntimeDefinition RD = Call->getRuntimeDefinition();
     const Decl *D = RD.getDecl();
     if (D) {
-      // Explore with and without inlining the call.
-      if (RD.mayHaveOtherDefinitions() &&
-          getAnalysisManager().IPAMode == DynamicDispatchBifurcate) {
-        BifurcateCall(RD.getDispatchRegion(), *Call, D, Bldr, Pred);
-        return;
+      if (RD.mayHaveOtherDefinitions()) {
+        // Explore with and without inlining the call.
+        if (getAnalysisManager().IPAMode == DynamicDispatchBifurcate) {
+          BifurcateCall(RD.getDispatchRegion(), *Call, D, Bldr, Pred);
+          return;
+        }
+
+        // Don't inline if we're not in any dynamic dispatch mode.
+        if (getAnalysisManager().IPAMode != DynamicDispatch) {
+          conservativeEvalCall(*Call, Bldr, Pred, State);
+          return;
+        }
       }
+
       // We are not bifurcating and we do have a Decl, so just inline.
       if (inlineCall(*Call, D, Bldr, Pred, State))
         return;
@@ -575,6 +588,7 @@ void ExprEngine::BifurcateCall(const MemRegion *BifurReg,
                                const CallEvent &Call, const Decl *D,
                                NodeBuilder &Bldr, ExplodedNode *Pred) {
   assert(BifurReg);
+  BifurReg = BifurReg->StripCasts();
 
   // Check if we've performed the split already - note, we only want
   // to split the path once per memory region.
diff --git a/lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp b/lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp
index 0152e32..982bcbf 100644
--- a/lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp
+++ b/lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp
@@ -45,7 +45,7 @@ public:
   virtual ~HTMLDiagnostics() { FlushDiagnostics(NULL); }
 
   virtual void FlushDiagnosticsImpl(std::vector<const PathDiagnostic *> &Diags,
-                                    SmallVectorImpl<std::string> *FilesMade);
+                                    FilesMade *filesMade);
 
   virtual StringRef getName() const {
     return "HTMLDiagnostics";
@@ -63,7 +63,7 @@ public:
                       const char *HighlightEnd = "</span>");
 
   void ReportDiag(const PathDiagnostic& D,
-                  SmallVectorImpl<std::string> *FilesMade);
+                  FilesMade *filesMade);
 };
 
 } // end anonymous namespace
@@ -76,10 +76,10 @@ HTMLDiagnostics::HTMLDiagnostics(const std::string& prefix,
   FilePrefix.appendComponent("report");
 }
 
-PathDiagnosticConsumer*
-ento::createHTMLDiagnosticConsumer(const std::string& prefix,
-                                 const Preprocessor &PP) {
-  return new HTMLDiagnostics(prefix, PP);
+void ento::createHTMLDiagnosticConsumer(PathDiagnosticConsumers &C,
+                                        const std::string& prefix,
+                                        const Preprocessor &PP) {
+  C.push_back(new HTMLDiagnostics(prefix, PP));
 }
 
 //===----------------------------------------------------------------------===//
@@ -88,15 +88,15 @@ ento::createHTMLDiagnosticConsumer(const std::string& prefix,
 
 void HTMLDiagnostics::FlushDiagnosticsImpl(
   std::vector<const PathDiagnostic *> &Diags,
-  SmallVectorImpl<std::string> *FilesMade) {
+  FilesMade *filesMade) {
   for (std::vector<const PathDiagnostic *>::iterator it = Diags.begin(),
        et = Diags.end(); it != et; ++it) {
-    ReportDiag(**it, FilesMade);
+    ReportDiag(**it, filesMade);
   }
 }
 
 void HTMLDiagnostics::ReportDiag(const PathDiagnostic& D,
-                                 SmallVectorImpl<std::string> *FilesMade) {
+                                 FilesMade *filesMade) {
     
   // Create the HTML directory if it is missing.
   if (!createdDir) {
@@ -266,8 +266,10 @@ void HTMLDiagnostics::ReportDiag(const PathDiagnostic& D,
     return;
   }
 
-  if (FilesMade)
-    FilesMade->push_back(llvm::sys::path::filename(H.str()));
+  if (filesMade) {
+    filesMade->push_back(std::make_pair(StringRef(getName()),
+                                        llvm::sys::path::filename(H.str())));
+  }
 
   // Emit the HTML to disk.
   for (RewriteBuffer::iterator I = Buf->begin(), E = Buf->end(); I!=E; ++I)
@@ -480,29 +482,11 @@ void HTMLDiagnostics::HandlePiece(Rewriter& R, FileID BugFileID,
   R.InsertTextBefore(Loc, os.str());
 
   // Now highlight the ranges.
-  for (const SourceRange *I = P.ranges_begin(), *E = P.ranges_end();
-        I != E; ++I)
+  ArrayRef<SourceRange> Ranges = P.getRanges();
+  for (ArrayRef<SourceRange>::iterator I = Ranges.begin(),
+                                       E = Ranges.end(); I != E; ++I) {
     HighlightRange(R, LPosInfo.first, *I);
-
-#if 0
-  // If there is a code insertion hint, insert that code.
-  // FIXME: This code is disabled because it seems to mangle the HTML
-  // output. I'm leaving it here because it's generally the right idea,
-  // but needs some help from someone more familiar with the rewriter.
-  for (const FixItHint *Hint = P.fixit_begin(), *HintEnd = P.fixit_end();
-       Hint != HintEnd; ++Hint) {
-    if (Hint->RemoveRange.isValid()) {
-      HighlightRange(R, LPosInfo.first, Hint->RemoveRange,
-                     "<span class=\"CodeRemovalHint\">", "</span>");
-    }
-    if (Hint->InsertionLoc.isValid()) {
-      std::string EscapedCode = html::EscapeText(Hint->CodeToInsert, true);
-      EscapedCode = "<span class=\"CodeInsertionHint\">" + EscapedCode
-        + "</span>";
-      R.InsertTextBefore(Hint->InsertionLoc, EscapedCode);
-    }
   }
-#endif
 }
 
 static void EmitAlphaCounter(raw_ostream &os, unsigned n) {
diff --git a/lib/StaticAnalyzer/Core/PathDiagnostic.cpp b/lib/StaticAnalyzer/Core/PathDiagnostic.cpp
index 7d52aac..c849778 100644
--- a/lib/StaticAnalyzer/Core/PathDiagnostic.cpp
+++ b/lib/StaticAnalyzer/Core/PathDiagnostic.cpp
@@ -157,13 +157,13 @@ void PathDiagnosticConsumer::HandlePathDiagnostic(PathDiagnostic *D) {
           return; // FIXME: Emit a warning?
       
         // Check the source ranges.
-        for (PathDiagnosticPiece::range_iterator RI = piece->ranges_begin(),
-             RE = piece->ranges_end();
-             RI != RE; ++RI) {
-          SourceLocation L = SMgr.getExpansionLoc(RI->getBegin());
+        ArrayRef<SourceRange> Ranges = piece->getRanges();
+        for (ArrayRef<SourceRange>::iterator I = Ranges.begin(),
+                                             E = Ranges.end(); I != E; ++I) {
+          SourceLocation L = SMgr.getExpansionLoc(I->getBegin());
           if (!L.isFileID() || SMgr.getFileID(L) != FID)
             return; // FIXME: Emit a warning?
-          L = SMgr.getExpansionLoc(RI->getEnd());
+          L = SMgr.getExpansionLoc(I->getEnd());
           if (!L.isFileID() || SMgr.getFileID(L) != FID)
             return; // FIXME: Emit a warning?
         }
@@ -240,8 +240,8 @@ struct CompareDiagnostics {
 };  
 }
 
-void
-PathDiagnosticConsumer::FlushDiagnostics(SmallVectorImpl<std::string> *Files) {
+void PathDiagnosticConsumer::FlushDiagnostics(
+                                     PathDiagnosticConsumer::FilesMade *Files) {
   if (flushed)
     return;
   
@@ -718,7 +718,9 @@ void PathDiagnosticPiece::Profile(llvm::FoldingSetNodeID &ID) const {
   ID.AddString(str);
   // FIXME: Add profiling support for code hints.
   ID.AddInteger((unsigned) getDisplayHint());
-  for (range_iterator I = ranges_begin(), E = ranges_end(); I != E; ++I) {
+  ArrayRef<SourceRange> Ranges = getRanges();
+  for (ArrayRef<SourceRange>::iterator I = Ranges.begin(), E = Ranges.end();
+                                        I != E; ++I) {
     ID.AddInteger(I->getBegin().getRawEncoding());
     ID.AddInteger(I->getEnd().getRawEncoding());
   }  
diff --git a/lib/StaticAnalyzer/Core/PlistDiagnostics.cpp b/lib/StaticAnalyzer/Core/PlistDiagnostics.cpp
index 58a4bba..d5fdd9d 100644
--- a/lib/StaticAnalyzer/Core/PlistDiagnostics.cpp
+++ b/lib/StaticAnalyzer/Core/PlistDiagnostics.cpp
@@ -30,23 +30,21 @@ namespace {
   class PlistDiagnostics : public PathDiagnosticConsumer {
     const std::string OutputFile;
     const LangOptions &LangOpts;
-    OwningPtr<PathDiagnosticConsumer> SubPD;
     const bool SupportsCrossFileDiagnostics;
   public:
     PlistDiagnostics(const std::string& prefix, const LangOptions &LangOpts,
-                     bool supportsMultipleFiles,
-                     PathDiagnosticConsumer *subPD);
+                     bool supportsMultipleFiles);
 
     virtual ~PlistDiagnostics() {}
 
     void FlushDiagnosticsImpl(std::vector<const PathDiagnostic *> &Diags,
-                              SmallVectorImpl<std::string> *FilesMade);
+                              FilesMade *filesMade);
     
     virtual StringRef getName() const {
       return "PlistDiagnostics";
     }
 
-    PathGenerationScheme getGenerationScheme() const;
+    PathGenerationScheme getGenerationScheme() const { return Extensive; }
     bool supportsLogicalOpControlFlow() const { return true; }
     bool supportsAllBlockEdges() const { return true; }
     virtual bool useVerboseDescription() const { return false; }
@@ -58,29 +56,20 @@ namespace {
 
 PlistDiagnostics::PlistDiagnostics(const std::string& output,
                                    const LangOptions &LO,
-                                   bool supportsMultipleFiles,
-                                   PathDiagnosticConsumer *subPD)
-  : OutputFile(output), LangOpts(LO), SubPD(subPD),
+                                   bool supportsMultipleFiles)
+  : OutputFile(output), LangOpts(LO),
     SupportsCrossFileDiagnostics(supportsMultipleFiles) {}
 
-PathDiagnosticConsumer*
-ento::createPlistDiagnosticConsumer(const std::string& s, const Preprocessor &PP,
-                                  PathDiagnosticConsumer *subPD) {
-  return new PlistDiagnostics(s, PP.getLangOpts(), false, subPD);
+void ento::createPlistDiagnosticConsumer(PathDiagnosticConsumers &C,
+                                         const std::string& s,
+                                         const Preprocessor &PP) {
+  C.push_back(new PlistDiagnostics(s, PP.getLangOpts(), false));
 }
 
-PathDiagnosticConsumer*
-ento::createPlistMultiFileDiagnosticConsumer(const std::string &s,
-                                              const Preprocessor &PP) {
-  return new PlistDiagnostics(s, PP.getLangOpts(), true, 0);
-}
-
-PathDiagnosticConsumer::PathGenerationScheme
-PlistDiagnostics::getGenerationScheme() const {
-  if (const PathDiagnosticConsumer *PD = SubPD.get())
-    return PD->getGenerationScheme();
-
-  return Extensive;
+void ento::createPlistMultiFileDiagnosticConsumer(PathDiagnosticConsumers &C,
+                                                  const std::string &s,
+                                                  const Preprocessor &PP) {
+  C.push_back(new PlistDiagnostics(s, PP.getLangOpts(), true));
 }
 
 static void AddFID(FIDMap &FIDs, SmallVectorImpl<FileID> &V,
@@ -231,15 +220,16 @@ static void ReportEvent(raw_ostream &o, const PathDiagnosticPiece& P,
   EmitLocation(o, SM, LangOpts, L, FM, indent);
 
   // Output the ranges (if any).
-  PathDiagnosticPiece::range_iterator RI = P.ranges_begin(),
-  RE = P.ranges_end();
+  ArrayRef<SourceRange> Ranges = P.getRanges();
 
-  if (RI != RE) {
+  if (!Ranges.empty()) {
     Indent(o, indent) << "<key>ranges</key>\n";
     Indent(o, indent) << "<array>\n";
     ++indent;
-    for (; RI != RE; ++RI)
-      EmitRange(o, SM, LangOpts, *RI, FM, indent+1);
+    for (ArrayRef<SourceRange>::iterator I = Ranges.begin(), E = Ranges.end();
+         I != E; ++I) {
+      EmitRange(o, SM, LangOpts, *I, FM, indent+1);
+    }
     --indent;
     Indent(o, indent) << "</array>\n";
   }
@@ -353,7 +343,7 @@ static void ReportPiece(raw_ostream &o,
 
 void PlistDiagnostics::FlushDiagnosticsImpl(
                                     std::vector<const PathDiagnostic *> &Diags,
-                                    SmallVectorImpl<std::string> *FilesMade) {
+                                    FilesMade *filesMade) {
   // Build up a set of FIDs that we use by scanning the locations and
   // ranges of the diagnostics.
   FIDMap FM;
@@ -380,11 +370,11 @@ void PlistDiagnostics::FlushDiagnosticsImpl(
            I!=E; ++I) {
         const PathDiagnosticPiece *piece = I->getPtr();
         AddFID(FM, Fids, SM, piece->getLocation().asLocation());
-
-        for (PathDiagnosticPiece::range_iterator RI = piece->ranges_begin(),
-             RE= piece->ranges_end(); RI != RE; ++RI) {
-          AddFID(FM, Fids, SM, RI->getBegin());
-          AddFID(FM, Fids, SM, RI->getEnd());
+        ArrayRef<SourceRange> Ranges = piece->getRanges();
+        for (ArrayRef<SourceRange>::iterator I = Ranges.begin(),
+                                             E = Ranges.end(); I != E; ++I) {
+          AddFID(FM, Fids, SM, I->getBegin());
+          AddFID(FM, Fids, SM, I->getEnd());
         }
 
         if (const PathDiagnosticCallPiece *call =
@@ -507,19 +497,21 @@ void PlistDiagnostics::FlushDiagnosticsImpl(
     EmitLocation(o, *SM, LangOpts, D->getLocation(), FM, 2);
 
     // Output the diagnostic to the sub-diagnostic client, if any.
-    if (SubPD) {
-      std::vector<const PathDiagnostic *> SubDiags;
-      SubDiags.push_back(D);
-      SmallVector<std::string, 1> SubFilesMade;
-      SubPD->FlushDiagnosticsImpl(SubDiags, &SubFilesMade);
-
-      if (!SubFilesMade.empty()) {
-        o << "  <key>" << SubPD->getName() << "_files</key>\n";
-        o << "  <array>\n";
-        for (size_t i = 0, n = SubFilesMade.size(); i < n ; ++i)
-          o << "   <string>" << SubFilesMade[i] << "</string>\n";
-        o << "  </array>\n";
+    if (!filesMade->empty()) {
+      StringRef lastName;
+      for (FilesMade::iterator I = filesMade->begin(), E = filesMade->end();
+           I != E; ++I) {
+        StringRef newName = I->first;
+        if (newName != lastName) {
+          if (!lastName.empty())
+            o << "  </array>\n";
+          lastName = newName;
+          o <<  "  <key>" << lastName << "_files</key>\n";
+          o << "  <array>\n";
+        }
+        o << "   <string>" << I->second << "</string>\n";
       }
+      o << "  </array>\n";
     }
 
     // Close up the entry.
@@ -531,6 +523,8 @@ void PlistDiagnostics::FlushDiagnosticsImpl(
   // Finish.
   o << "</dict>\n</plist>";
   
-  if (FilesMade)
-    FilesMade->push_back(OutputFile);
+  if (filesMade) {
+    StringRef Name(getName());
+    filesMade->push_back(std::make_pair(Name, OutputFile));
+  }
 }
diff --git a/lib/StaticAnalyzer/Core/ProgramState.cpp b/lib/StaticAnalyzer/Core/ProgramState.cpp
index dc988cc..2000338 100644
--- a/lib/StaticAnalyzer/Core/ProgramState.cpp
+++ b/lib/StaticAnalyzer/Core/ProgramState.cpp
@@ -745,14 +745,16 @@ template<> struct ProgramStateTrait<DynamicTypeMap>
 }}
 
 DynamicTypeInfo ProgramState::getDynamicTypeInfo(const MemRegion *Reg) const {
+  Reg = Reg->StripCasts();
+
   // Look up the dynamic type in the GDM.
   const DynamicTypeInfo *GDMType = get<DynamicTypeMap>(Reg);
   if (GDMType)
     return *GDMType;
 
   // Otherwise, fall back to what we know about the region.
-  if (const TypedValueRegion *TR = dyn_cast<TypedValueRegion>(Reg))
-    return DynamicTypeInfo(TR->getValueType());
+  if (const TypedRegion *TR = dyn_cast<TypedRegion>(Reg))
+    return DynamicTypeInfo(TR->getLocationType(), /*CanBeSubclass=*/false);
 
   if (const SymbolicRegion *SR = dyn_cast<SymbolicRegion>(Reg)) {
     SymbolRef Sym = SR->getSymbol();
@@ -764,6 +766,7 @@ DynamicTypeInfo ProgramState::getDynamicTypeInfo(const MemRegion *Reg) const {
 
 ProgramStateRef ProgramState::setDynamicTypeInfo(const MemRegion *Reg,
                                                  DynamicTypeInfo NewTy) const {
+  Reg = Reg->StripCasts();
   ProgramStateRef NewState = set<DynamicTypeMap>(Reg, NewTy);
   assert(NewState);
   return NewState;
diff --git a/lib/StaticAnalyzer/Core/TextPathDiagnostics.cpp b/lib/StaticAnalyzer/Core/TextPathDiagnostics.cpp
index e5b8553..66bf4bb 100644
--- a/lib/StaticAnalyzer/Core/TextPathDiagnostics.cpp
+++ b/lib/StaticAnalyzer/Core/TextPathDiagnostics.cpp
@@ -32,7 +32,7 @@ public:
     : OutputFile(output), Diag(diag) {}
 
   void FlushDiagnosticsImpl(std::vector<const PathDiagnostic *> &Diags,
-                            SmallVectorImpl<std::string> *FilesMade);
+                            FilesMade *filesMade);
   
   virtual StringRef getName() const {
     return "TextPathDiagnostics";
@@ -47,15 +47,15 @@ public:
 
 } // end anonymous namespace
 
-PathDiagnosticConsumer*
-ento::createTextPathDiagnosticConsumer(const std::string& out,
-                                     const Preprocessor &PP) {
-  return new TextPathDiagnostics(out, PP.getDiagnostics());
+void ento::createTextPathDiagnosticConsumer(PathDiagnosticConsumers &C,
+                                            const std::string& out,
+                                            const Preprocessor &PP) {
+  C.push_back(new TextPathDiagnostics(out, PP.getDiagnostics()));
 }
 
 void TextPathDiagnostics::FlushDiagnosticsImpl(
                               std::vector<const PathDiagnostic *> &Diags,
-                              SmallVectorImpl<std::string> *FilesMade) {
+                              FilesMade *) {
   for (std::vector<const PathDiagnostic *>::iterator it = Diags.begin(),
        et = Diags.end(); it != et; ++it) {
     const PathDiagnostic *D = *it;
diff --git a/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp b/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp
index fcdaaea..34b5266 100644
--- a/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp
+++ b/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp
@@ -64,14 +64,55 @@ STATISTIC(MaxCFGSize, "The maximum number of basic blocks in a function.");
 // Special PathDiagnosticConsumers.
 //===----------------------------------------------------------------------===//
 
-static PathDiagnosticConsumer*
-createPlistHTMLDiagnosticConsumer(const std::string& prefix,
-                                const Preprocessor &PP) {
-  PathDiagnosticConsumer *PD =
-    createHTMLDiagnosticConsumer(llvm::sys::path::parent_path(prefix), PP);
-  return createPlistDiagnosticConsumer(prefix, PP, PD);
+static void createPlistHTMLDiagnosticConsumer(PathDiagnosticConsumers &C,
+                                              const std::string &prefix,
+                                              const Preprocessor &PP) {
+  createHTMLDiagnosticConsumer(C, llvm::sys::path::parent_path(prefix), PP);
+  createPlistDiagnosticConsumer(C, prefix, PP);
 }
 
+namespace {
+class ClangDiagPathDiagConsumer : public PathDiagnosticConsumer {
+  DiagnosticsEngine &Diag;
+public:
+  ClangDiagPathDiagConsumer(DiagnosticsEngine &Diag) : Diag(Diag) {}
+  virtual ~ClangDiagPathDiagConsumer() {}
+  virtual StringRef getName() const { return "ClangDiags"; }
+  virtual bool useVerboseDescription() const { return false; }
+  virtual PathGenerationScheme getGenerationScheme() const { return None; }
+
+  void FlushDiagnosticsImpl(std::vector<const PathDiagnostic *> &Diags,
+                            FilesMade *filesMade) {
+    for (std::vector<const PathDiagnostic*>::iterator I = Diags.begin(),
+         E = Diags.end(); I != E; ++I) {
+      const PathDiagnostic *PD = *I;
+      StringRef desc = PD->getDescription();
+      SmallString<512> TmpStr;
+      llvm::raw_svector_ostream Out(TmpStr);
+      for (StringRef::iterator I=desc.begin(), E=desc.end(); I!=E; ++I) {
+        if (*I == '%')
+          Out << "%%";
+        else
+          Out << *I;
+      }
+      Out.flush();
+      unsigned ErrorDiag = Diag.getCustomDiagID(DiagnosticsEngine::Warning,
+                                                TmpStr);
+      SourceLocation L = PD->getLocation().asLocation();
+      DiagnosticBuilder diagBuilder = Diag.Report(L, ErrorDiag);
+
+      // Get the ranges from the last point in the path.
+      ArrayRef<SourceRange> Ranges = PD->path.back()->getRanges();
+
+      for (ArrayRef<SourceRange>::iterator I = Ranges.begin(),
+                                           E = Ranges.end(); I != E; ++I) {
+        diagBuilder << *I;
+      }
+    }
+  }
+};
+} // end anonymous namespace
+
 //===----------------------------------------------------------------------===//
 // AnalysisConsumer declaration.
 //===----------------------------------------------------------------------===//
@@ -105,8 +146,8 @@ public:
   /// working with a PCH file.
   SetOfDecls LocalTUDecls;
                            
-  // PD is owned by AnalysisManager.
-  PathDiagnosticConsumer *PD;
+  // Set of PathDiagnosticConsumers.  Owned by AnalysisManager.
+  PathDiagnosticConsumers PathConsumers;
 
   StoreManagerCreator CreateStoreMgr;
   ConstraintManagerCreator CreateConstraintMgr;
@@ -126,7 +167,7 @@ public:
                    const AnalyzerOptions& opts,
                    ArrayRef<std::string> plugins)
     : RecVisitorMode(ANALYSIS_ALL), RecVisitorBR(0),
-      Ctx(0), PP(pp), OutDir(outdir), Opts(opts), Plugins(plugins), PD(0) {
+      Ctx(0), PP(pp), OutDir(outdir), Opts(opts), Plugins(plugins) {
     DigestAnalyzerOptions();
     if (Opts.PrintStats) {
       llvm::EnableStatistics();
@@ -141,17 +182,19 @@ public:
 
   void DigestAnalyzerOptions() {
     // Create the PathDiagnosticConsumer.
+    PathConsumers.push_back(new ClangDiagPathDiagConsumer(PP.getDiagnostics()));
+
     if (!OutDir.empty()) {
       switch (Opts.AnalysisDiagOpt) {
       default:
 #define ANALYSIS_DIAGNOSTICS(NAME, CMDFLAG, DESC, CREATEFN, AUTOCREATE) \
-        case PD_##NAME: PD = CREATEFN(OutDir, PP); break;
+        case PD_##NAME: CREATEFN(PathConsumers, OutDir, PP); break;
 #include "clang/Frontend/Analyses.def"
       }
     } else if (Opts.AnalysisDiagOpt == PD_TEXT) {
       // Create the text client even without a specified output file since
       // it just uses diagnostic notes.
-      PD = createTextPathDiagnosticConsumer("", PP);
+      createTextPathDiagnosticConsumer(PathConsumers, "", PP);
     }
 
     // Create the analyzer component creators.
@@ -205,9 +248,12 @@ public:
     Ctx = &Context;
     checkerMgr.reset(createCheckerManager(Opts, PP.getLangOpts(), Plugins,
                                           PP.getDiagnostics()));
-    Mgr.reset(new AnalysisManager(*Ctx, PP.getDiagnostics(),
-                                  PP.getLangOpts(), PD,
-                                  CreateStoreMgr, CreateConstraintMgr,
+    Mgr.reset(new AnalysisManager(*Ctx,
+                                  PP.getDiagnostics(),
+                                  PP.getLangOpts(),
+                                  PathConsumers,
+                                  CreateStoreMgr,
+                                  CreateConstraintMgr,
                                   checkerMgr.get(),
                                   Opts.MaxNodes, Opts.MaxLoop,
                                   Opts.VisualizeEGDot, Opts.VisualizeEGUbi,
-- 
cgit v1.1


From 6c5ad005bed33e80c94460b6694d199348dac472 Mon Sep 17 00:00:00 2001
From: grehan <grehan@FreeBSD.org>
Date: Sun, 26 Aug 2012 01:41:41 +0000
Subject: Add sysctls to display the total and free amount of hard-wired mem
 for VMs    # sysctl hw.vmm    hw.vmm.mem_free: 2145386496   
 hw.vmm.mem_total: 2145386496

Submitted by:	Takeshi HASEGAWA hasegaw at gmail com
---
 lib/libvmmapi/vmmapi.c | 24 ++++++++++++++++++++++++
 lib/libvmmapi/vmmapi.h |  2 ++
 2 files changed, 26 insertions(+)

(limited to 'lib')

diff --git a/lib/libvmmapi/vmmapi.c b/lib/libvmmapi/vmmapi.c
index 5882bd2..d7e6143 100644
--- a/lib/libvmmapi/vmmapi.c
+++ b/lib/libvmmapi/vmmapi.c
@@ -117,6 +117,30 @@ vm_destroy(struct vmctx *vm)
 	free(vm);
 }
 
+size_t
+vmm_get_mem_total(void)
+{
+	size_t mem_total = 0;
+	size_t oldlen = sizeof(mem_total);
+	int error;
+	error = sysctlbyname("hw.vmm.mem_total", &mem_total, &oldlen, NULL, 0);
+	if (error)
+		return -1;
+	return mem_total;
+}
+
+size_t
+vmm_get_mem_free(void)
+{
+	size_t mem_free = 0;
+	size_t oldlen = sizeof(mem_free);
+	int error;
+	error = sysctlbyname("hw.vmm.mem_free", &mem_free, &oldlen, NULL, 0);
+	if (error)
+		return -1;
+	return mem_free;
+}
+
 int
 vm_get_memory_seg(struct vmctx *ctx, vm_paddr_t gpa,
 		  vm_paddr_t *ret_hpa, size_t *ret_len)
diff --git a/lib/libvmmapi/vmmapi.h b/lib/libvmmapi/vmmapi.h
index de3d4b7..516bbc3 100644
--- a/lib/libvmmapi/vmmapi.h
+++ b/lib/libvmmapi/vmmapi.h
@@ -34,6 +34,8 @@ struct vmctx;
 int	vm_create(const char *name);
 struct vmctx *vm_open(const char *name);
 void	vm_destroy(struct vmctx *ctx);
+size_t	vmm_get_mem_total(void);
+size_t	vmm_get_mem_free(void);
 int	vm_get_memory_seg(struct vmctx *ctx, vm_paddr_t gpa,
 			  vm_paddr_t *ret_hpa, size_t *ret_len);
 /*
-- 
cgit v1.1


From be47cf2d63c5d79e95cc63a17f7dcefd3035e5da Mon Sep 17 00:00:00 2001
From: dougb <dougb@FreeBSD.org>
Date: Thu, 20 Sep 2012 03:38:28 +0000
Subject: Vendor import of BIND 9.8.3-P3

---
 lib/dns/include/dns/rdata.h | 11 +++++++++++
 lib/dns/master.c            |  2 +-
 lib/dns/rdata.c             | 28 ++++++++++++++++++++++------
 lib/dns/rdataslab.c         |  1 +
 4 files changed, 35 insertions(+), 7 deletions(-)

(limited to 'lib')

diff --git a/lib/dns/include/dns/rdata.h b/lib/dns/include/dns/rdata.h
index e3183c0..c3e7db6 100644
--- a/lib/dns/include/dns/rdata.h
+++ b/lib/dns/include/dns/rdata.h
@@ -147,6 +147,17 @@ struct dns_rdata {
 	(((rdata)->flags & ~(DNS_RDATA_UPDATE|DNS_RDATA_OFFLINE)) == 0)
 
 /*
+ * The maximum length of a RDATA that can be sent on the wire.
+ * Max packet size (65535) less header (12), less name (1), type (2),
+ * class (2), ttl(4), length (2).
+ *
+ * None of the defined types that support name compression can exceed
+ * this and all new types are to be sent uncompressed.
+ */
+
+#define DNS_RDATA_MAXLENGTH	65512U
+
+/*
  * Flags affecting rdata formatting style.  Flags 0xFFFF0000
  * are used by masterfile-level formatting and defined elsewhere.
  * See additional comments at dns_rdata_tofmttext().
diff --git a/lib/dns/master.c b/lib/dns/master.c
index ae07e55..7f6cf58 100644
--- a/lib/dns/master.c
+++ b/lib/dns/master.c
@@ -75,7 +75,7 @@
 /*%
  * max message size - header - root - type - class - ttl - rdlen
  */
-#define MINTSIZ (65535 - 12 - 1 - 2 - 2 - 4 - 2)
+#define MINTSIZ DNS_RDATA_MAXLENGTH
 /*%
  * Size for tokens in the presentation format,
  * The largest tokens are the base64 blocks in KEY and CERT records,
diff --git a/lib/dns/rdata.c b/lib/dns/rdata.c
index f6807d1..d200f1b 100644
--- a/lib/dns/rdata.c
+++ b/lib/dns/rdata.c
@@ -429,6 +429,7 @@ dns_rdata_fromwire(dns_rdata_t *rdata, dns_rdataclass_t rdclass,
 	isc_buffer_t st;
 	isc_boolean_t use_default = ISC_FALSE;
 	isc_uint32_t activelength;
+	size_t length;
 
 	REQUIRE(dctx != NULL);
 	if (rdata != NULL) {
@@ -459,6 +460,14 @@ dns_rdata_fromwire(dns_rdata_t *rdata, dns_rdataclass_t rdclass,
 	}
 
 	/*
+	 * Reject any rdata that expands out to more than DNS_RDATA_MAXLENGTH
+	 * as we cannot transmit it.
+	 */
+	length = isc_buffer_usedlength(target) - isc_buffer_usedlength(&st);
+	if (result == ISC_R_SUCCESS && length > DNS_RDATA_MAXLENGTH)
+		result = DNS_R_FORMERR;
+
+	/*
 	 * We should have consumed all of our buffer.
 	 */
 	if (result == ISC_R_SUCCESS && !buffer_empty(source))
@@ -466,8 +475,7 @@ dns_rdata_fromwire(dns_rdata_t *rdata, dns_rdataclass_t rdclass,
 
 	if (rdata != NULL && result == ISC_R_SUCCESS) {
 		region.base = isc_buffer_used(&st);
-		region.length = isc_buffer_usedlength(target) -
-				isc_buffer_usedlength(&st);
+		region.length = length;
 		dns_rdata_fromregion(rdata, rdclass, type, &region);
 	}
 
@@ -602,6 +610,7 @@ dns_rdata_fromtext(dns_rdata_t *rdata, dns_rdataclass_t rdclass,
 	unsigned long line;
 	void (*callback)(dns_rdatacallbacks_t *, const char *, ...);
 	isc_result_t tresult;
+	size_t length;
 
 	REQUIRE(origin == NULL || dns_name_isabsolute(origin) == ISC_TRUE);
 	if (rdata != NULL) {
@@ -673,10 +682,13 @@ dns_rdata_fromtext(dns_rdata_t *rdata, dns_rdataclass_t rdclass,
 		}
 	} while (1);
 
+	length = isc_buffer_usedlength(target) - isc_buffer_usedlength(&st);
+	if (result == ISC_R_SUCCESS && length > DNS_RDATA_MAXLENGTH)
+		result = ISC_R_NOSPACE;
+
 	if (rdata != NULL && result == ISC_R_SUCCESS) {
 		region.base = isc_buffer_used(&st);
-		region.length = isc_buffer_usedlength(target) -
-				isc_buffer_usedlength(&st);
+		region.length = length;
 		dns_rdata_fromregion(rdata, rdclass, type, &region);
 	}
 	if (result != ISC_R_SUCCESS) {
@@ -804,6 +816,7 @@ dns_rdata_fromstruct(dns_rdata_t *rdata, dns_rdataclass_t rdclass,
 	isc_buffer_t st;
 	isc_region_t region;
 	isc_boolean_t use_default = ISC_FALSE;
+	size_t length;
 
 	REQUIRE(source != NULL);
 	if (rdata != NULL) {
@@ -818,10 +831,13 @@ dns_rdata_fromstruct(dns_rdata_t *rdata, dns_rdataclass_t rdclass,
 	if (use_default)
 		(void)NULL;
 
+	length = isc_buffer_usedlength(target) - isc_buffer_usedlength(&st);
+	if (result == ISC_R_SUCCESS && length > DNS_RDATA_MAXLENGTH)
+		result = ISC_R_NOSPACE;
+
 	if (rdata != NULL && result == ISC_R_SUCCESS) {
 		region.base = isc_buffer_used(&st);
-		region.length = isc_buffer_usedlength(target) -
-				isc_buffer_usedlength(&st);
+		region.length = length;
 		dns_rdata_fromregion(rdata, rdclass, type, &region);
 	}
 	if (result != ISC_R_SUCCESS)
diff --git a/lib/dns/rdataslab.c b/lib/dns/rdataslab.c
index 6fbfdd7..cb9ae54 100644
--- a/lib/dns/rdataslab.c
+++ b/lib/dns/rdataslab.c
@@ -305,6 +305,7 @@ dns_rdataslab_fromrdataset(dns_rdataset_t *rdataset, isc_mem_t *mctx,
 		length = x[i].rdata.length;
 		if (rdataset->type == dns_rdatatype_rrsig)
 			length++;
+		INSIST(length <= 0xffff);
 		*rawbuf++ = (length & 0xff00) >> 8;
 		*rawbuf++ = (length & 0x00ff);
 #if DNS_RDATASET_FIXED
-- 
cgit v1.1


From ebdd69568d7fa97153aa47a86afe367476a0a1de Mon Sep 17 00:00:00 2001
From: neel <neel@FreeBSD.org>
Date: Tue, 25 Sep 2012 19:08:51 +0000
Subject: Add ioctls to control the X2APIC capability exposed by the virtual
 machine to the guest.

At the moment this simply sets the state in the 'vcpu' instance but there is
no code that acts upon these settings.
---
 lib/libvmmapi/vmmapi.c | 29 +++++++++++++++++++++++++++++
 lib/libvmmapi/vmmapi.h |  4 ++++
 2 files changed, 33 insertions(+)

(limited to 'lib')

diff --git a/lib/libvmmapi/vmmapi.c b/lib/libvmmapi/vmmapi.c
index d7e6143..78ed368 100644
--- a/lib/libvmmapi/vmmapi.c
+++ b/lib/libvmmapi/vmmapi.c
@@ -537,6 +537,35 @@ vm_get_stat_desc(struct vmctx *ctx, int index)
 		return (NULL);
 }
 
+int
+vm_get_x2apic_state(struct vmctx *ctx, int vcpu, enum x2apic_state *state)
+{
+	int error;
+	struct vm_x2apic x2apic;
+
+	bzero(&x2apic, sizeof(x2apic));
+	x2apic.cpuid = vcpu;
+
+	error = ioctl(ctx->fd, VM_GET_X2APIC_STATE, &x2apic);
+	*state = x2apic.state;
+	return (error);
+}
+
+int
+vm_set_x2apic_state(struct vmctx *ctx, int vcpu, enum x2apic_state state)
+{
+	int error;
+	struct vm_x2apic x2apic;
+
+	bzero(&x2apic, sizeof(x2apic));
+	x2apic.cpuid = vcpu;
+	x2apic.state = state;
+
+	error = ioctl(ctx->fd, VM_SET_X2APIC_STATE, &x2apic);
+
+	return (error);
+}
+
 /*
  * From Intel Vol 3a:
  * Table 9-1. IA-32 Processor States Following Power-up, Reset or INIT
diff --git a/lib/libvmmapi/vmmapi.h b/lib/libvmmapi/vmmapi.h
index 516bbc3..b918409 100644
--- a/lib/libvmmapi/vmmapi.h
+++ b/lib/libvmmapi/vmmapi.h
@@ -30,6 +30,7 @@
 #define	_VMMAPI_H_
 
 struct vmctx;
+enum x2apic_state;
 
 int	vm_create(const char *name);
 struct vmctx *vm_open(const char *name);
@@ -90,6 +91,9 @@ uint64_t *vm_get_stats(struct vmctx *ctx, int vcpu, struct timeval *ret_tv,
 		       int *ret_entries);
 const char *vm_get_stat_desc(struct vmctx *ctx, int index);
 
+int	vm_get_x2apic_state(struct vmctx *ctx, int vcpu, enum x2apic_state *s);
+int	vm_set_x2apic_state(struct vmctx *ctx, int vcpu, enum x2apic_state s);
+
 /* Reset vcpu register state */
 int	vcpu_reset(struct vmctx *ctx, int vcpu);
 
-- 
cgit v1.1


From 18dd2c0d511c600e708ac8f756e8e51151b43656 Mon Sep 17 00:00:00 2001
From: neel <neel@FreeBSD.org>
Date: Thu, 4 Oct 2012 02:27:14 +0000
Subject: Change vm_malloc() to map pages in the guest physical address space
 in 4KB chunks. This breaks the assumption that the entire memory segment is
 contiguously allocated in the host physical address space.

This also paves the way to satisfy the 4KB page allocations by requesting
free pages from the VM subsystem as opposed to hard-partitioning host memory
at boot time.
---
 lib/libvmmapi/vmmapi.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/libvmmapi/vmmapi.c b/lib/libvmmapi/vmmapi.c
index 78ed368..8364dd5 100644
--- a/lib/libvmmapi/vmmapi.c
+++ b/lib/libvmmapi/vmmapi.c
@@ -111,9 +111,10 @@ vm_destroy(struct vmctx *vm)
 {
 	assert(vm != NULL);
 
-	DESTROY(vm->name);
 	if (vm->fd >= 0)
 		close(vm->fd);
+	DESTROY(vm->name);
+
 	free(vm);
 }
 
@@ -151,7 +152,6 @@ vm_get_memory_seg(struct vmctx *ctx, vm_paddr_t gpa,
 	bzero(&seg, sizeof(seg));
 	seg.gpa = gpa;
 	error = ioctl(ctx->fd, VM_GET_MEMORY_SEG, &seg);
-	*ret_hpa = seg.hpa;
 	*ret_len = seg.len;
 	return (error);
 }
-- 
cgit v1.1


From 09939583a708e1199d76573009937c49606d19e5 Mon Sep 17 00:00:00 2001
From: neel <neel@FreeBSD.org>
Date: Thu, 4 Oct 2012 03:07:05 +0000
Subject: The ioctl VM_GET_MEMORY_SEG is no longer able to return the host
 physical address associated with the guest memory segment. This is because
 there is no longer a 1:1 mapping between GPA and HPA.

As a result 'vmmctl' can only display the guest physical address and the
length of the lowmem and highmem segments.
---
 lib/libvmmapi/vmmapi.c | 3 +--
 lib/libvmmapi/vmmapi.h | 3 +--
 2 files changed, 2 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/libvmmapi/vmmapi.c b/lib/libvmmapi/vmmapi.c
index 8364dd5..e083fde 100644
--- a/lib/libvmmapi/vmmapi.c
+++ b/lib/libvmmapi/vmmapi.c
@@ -143,8 +143,7 @@ vmm_get_mem_free(void)
 }
 
 int
-vm_get_memory_seg(struct vmctx *ctx, vm_paddr_t gpa,
-		  vm_paddr_t *ret_hpa, size_t *ret_len)
+vm_get_memory_seg(struct vmctx *ctx, vm_paddr_t gpa, size_t *ret_len)
 {
 	int error;
 	struct vm_memory_segment seg;
diff --git a/lib/libvmmapi/vmmapi.h b/lib/libvmmapi/vmmapi.h
index b918409..3f434ab 100644
--- a/lib/libvmmapi/vmmapi.h
+++ b/lib/libvmmapi/vmmapi.h
@@ -37,8 +37,7 @@ struct vmctx *vm_open(const char *name);
 void	vm_destroy(struct vmctx *ctx);
 size_t	vmm_get_mem_total(void);
 size_t	vmm_get_mem_free(void);
-int	vm_get_memory_seg(struct vmctx *ctx, vm_paddr_t gpa,
-			  vm_paddr_t *ret_hpa, size_t *ret_len);
+int	vm_get_memory_seg(struct vmctx *ctx, vm_paddr_t gpa, size_t *ret_len);
 /*
  * Create a memory segment of 'len' bytes in the guest physical address space
  * at offset 'gpa'.
-- 
cgit v1.1


From a62f9562ca8f5ed0f5099962cb864c87a2a2865c Mon Sep 17 00:00:00 2001
From: neel <neel@FreeBSD.org>
Date: Fri, 12 Oct 2012 17:39:28 +0000
Subject: Add an api to map a vm capability type into a string to be used for
 display purposes.

---
 lib/libvmmapi/vmmapi.c | 35 ++++++++++++++++++++++++-----------
 lib/libvmmapi/vmmapi.h |  1 +
 2 files changed, 25 insertions(+), 11 deletions(-)

(limited to 'lib')

diff --git a/lib/libvmmapi/vmmapi.c b/lib/libvmmapi/vmmapi.c
index e083fde..fdbbbcb 100644
--- a/lib/libvmmapi/vmmapi.c
+++ b/lib/libvmmapi/vmmapi.c
@@ -370,22 +370,22 @@ vm_inject_nmi(struct vmctx *ctx, int vcpu)
 	return (ioctl(ctx->fd, VM_INJECT_NMI, &vmnmi));
 }
 
+static struct {
+	const char	*name;
+	int		type;
+} capstrmap[] = {
+	{ "hlt_exit",		VM_CAP_HALT_EXIT },
+	{ "mtrap_exit",		VM_CAP_MTRAP_EXIT },
+	{ "pause_exit",		VM_CAP_PAUSE_EXIT },
+	{ "unrestricted_guest",	VM_CAP_UNRESTRICTED_GUEST },
+	{ 0 }
+};
+
 int
 vm_capability_name2type(const char *capname)
 {
 	int i;
 
-	static struct {
-		const char	*name;
-		int		type;
-	} capstrmap[] = {
-		{ "hlt_exit",		VM_CAP_HALT_EXIT },
-		{ "mtrap_exit",		VM_CAP_MTRAP_EXIT },
-		{ "pause_exit",		VM_CAP_PAUSE_EXIT },
-		{ "unrestricted_guest",	VM_CAP_UNRESTRICTED_GUEST },
-		{ 0 }
-	};
-
 	for (i = 0; capstrmap[i].name != NULL && capname != NULL; i++) {
 		if (strcmp(capstrmap[i].name, capname) == 0)
 			return (capstrmap[i].type);
@@ -394,6 +394,19 @@ vm_capability_name2type(const char *capname)
 	return (-1);
 }
 
+const char *
+vm_capability_type2name(int type)
+{
+	int i;
+
+	for (i = 0; capstrmap[i].name != NULL; i++) {
+		if (capstrmap[i].type == type)
+			return (capstrmap[i].name);
+	}
+
+	return (NULL);
+}
+
 int
 vm_get_capability(struct vmctx *ctx, int vcpu, enum vm_cap_type cap,
 		  int *retval)
diff --git a/lib/libvmmapi/vmmapi.h b/lib/libvmmapi/vmmapi.h
index 3f434ab..fe5116e 100644
--- a/lib/libvmmapi/vmmapi.h
+++ b/lib/libvmmapi/vmmapi.h
@@ -70,6 +70,7 @@ int	vm_inject_event2(struct vmctx *ctx, int vcpu, enum vm_event_type type,
 int	vm_lapic_irq(struct vmctx *ctx, int vcpu, int vector);
 int	vm_inject_nmi(struct vmctx *ctx, int vcpu);
 int	vm_capability_name2type(const char *capname);
+const char *vm_capability_type2name(int type);
 int	vm_get_capability(struct vmctx *ctx, int vcpu, enum vm_cap_type cap,
 			  int *retval);
 int	vm_set_capability(struct vmctx *ctx, int vcpu, enum vm_cap_type cap,
-- 
cgit v1.1


From 1372a368e0c75317b5fcf37426faaabc995735c9 Mon Sep 17 00:00:00 2001
From: grehan <grehan@FreeBSD.org>
Date: Fri, 26 Oct 2012 13:40:12 +0000
Subject: Remove mptable generation code from libvmmapi and move it to bhyve.
 Firmware tables require too much knowledge of system configuration, and it's
 difficult to pass that information in general terms to a library. The
 upcoming ACPI work exposed this - it will also livein bhyve.

Also, remove code specific to NetApp from the mptable name, and remove
the -n option from bhyve.

Reviewed by:	neel
Obtained from:	NetApp
---
 lib/libvmmapi/Makefile  |   2 +-
 lib/libvmmapi/mptable.c | 342 ------------------------------------------------
 lib/libvmmapi/mptable.h | 170 ------------------------
 lib/libvmmapi/vmmapi.c  |  13 --
 lib/libvmmapi/vmmapi.h  |   2 -
 5 files changed, 1 insertion(+), 528 deletions(-)
 delete mode 100644 lib/libvmmapi/mptable.c
 delete mode 100644 lib/libvmmapi/mptable.h

(limited to 'lib')

diff --git a/lib/libvmmapi/Makefile b/lib/libvmmapi/Makefile
index 82dde08..93d3c85 100644
--- a/lib/libvmmapi/Makefile
+++ b/lib/libvmmapi/Makefile
@@ -1,7 +1,7 @@
 # $FreeBSD$
 
 LIB=	vmmapi
-SRCS=	vmmapi.c vmmapi_freebsd.c mptable.c
+SRCS=	vmmapi.c vmmapi_freebsd.c
 INCS=	vmmapi.h
 
 WARNS?=	2
diff --git a/lib/libvmmapi/mptable.c b/lib/libvmmapi/mptable.c
deleted file mode 100644
index 048ea2d..0000000
--- a/lib/libvmmapi/mptable.c
+++ /dev/null
@@ -1,342 +0,0 @@
-/*-
- * Copyright (c) 2011 NetApp, Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * $FreeBSD$
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/types.h>
-#include <sys/mman.h>
-
-#include <stdio.h>
-#include <string.h>
-#include <machine/vmm.h>
-#include <machine/vmm_dev.h>
-
-#include "vmmapi.h"
-#include "mptable.h"
-
-#define LAPIC_PADDR		(0xFEE00000)
-#define LAPIC_VERSION 		(16)
-
-#define IOAPIC_PADDR		(0xFEC00000)
-#define IOAPIC_VERSION		(0x11)
-
-extern int errno;
-
-static uint8_t
-mp_compute_checksum(void *base, size_t len)
-{
-	uint8_t	*bytes = base;
-	uint8_t	 sum = 0;
-	for(; len > 0; len--) {
-		sum += *bytes++;
-	}
-	return 256 - sum;
-}
-
-static void
-mp_build_mpfp(struct mp_floating_pointer *mpfp, vm_paddr_t mpfp_gpa)
-{
-	memset(mpfp, 0, sizeof(*mpfp));
-	memcpy(mpfp->signature, MPFP_SIGNATURE, MPFP_SIGNATURE_LEN);
-	mpfp->mptable_paddr = mpfp_gpa + sizeof(*mpfp);
-	mpfp->specrev = MP_SPECREV;
-	mpfp->feature2 = 0;
-	mpfp->checksum = mp_compute_checksum(mpfp, sizeof(*mpfp));
-}
-
-static void
-mp_build_mpch(struct mp_config_hdr *mpch)
-{
-	memset(mpch, 0, sizeof(*mpch));
-	mpch->specrev = MP_SPECREV;
-	memcpy(mpch->signature, MPCH_SIGNATURE, MPCH_SIGNATURE_LEN);
-	memcpy(mpch->oemid, MPCH_OEMID, MPCH_OEMID_LEN);
-	memcpy(mpch->prodid, MPCH_PRODID, MPCH_PRODID_LEN);
-	mpch->lapic_paddr = LAPIC_PADDR;
-
-
-}
-
-static void
-mp_build_proc_entries(struct mpe_proc *mpep, int num_proc)
-{
-	int i;
-
-	for (i = 0; i < num_proc; i++) {
-		memset(mpep, 0, sizeof(*mpep));
-		mpep->entry_type = MP_ENTRY_PROC;
-		mpep->lapic_id = i; // XXX
-		mpep->lapic_version = LAPIC_VERSION;
-		mpep->proc_flags = (i == 0)?MPEP_FLAGS_BSP:0;
-		mpep->proc_flags |= MPEP_FLAGS_EN;
-		mpep->proc_signature = MPEP_SIGNATURE;
-		mpep->feature_flags = MPEP_FEATURES;
-		mpep++;
-	}
-	
-}
-
-static void
-mp_build_bus_entries(struct mpe_bus *mpeb)
-{
-	memset(mpeb, 0, sizeof(*mpeb));
-	mpeb->entry_type = MP_ENTRY_BUS;
-	mpeb->busid = MPE_BUSID_ISA;
-	memcpy(mpeb->busname, MPE_BUSNAME_ISA, MPE_BUSNAME_LEN);
-	mpeb++;
-
-	memset(mpeb, 0, sizeof(*mpeb));
-	mpeb->entry_type = MP_ENTRY_BUS;
-	mpeb->busid = MPE_BUSID_PCI;
-	memcpy(mpeb->busname, MPE_BUSNAME_PCI, MPE_BUSNAME_LEN);
-
-}
-
-static void
-mp_build_ioapic_entries(struct mpe_ioapic *mpei, int id)
-{
-	memset(mpei, 0, sizeof(*mpei));
-	mpei->entry_type = MP_ENTRY_IOAPIC;
-	mpei->ioapic_id = id;
-	mpei->ioapic_version = IOAPIC_VERSION;
-	mpei->ioapic_flags = MPE_IOAPIC_FLAG_EN;
-	mpei->ioapic_paddr = IOAPIC_PADDR;
-}
-
-#ifdef notyet
-static void
-mp_build_ioint_entries(struct mpe_ioint *mpeii, int num_pins, int id)
-{
-	int pin;
-
-	/*
-	 * The following config is taken from kernel mptable.c
-	 * mptable_parse_default_config_ints(...), for now 
-	 * just use the default config, tweek later if needed.
-	 */
-
-
-	/* Run through all 16 pins. */
-	for (pin = 0; pin < num_pins; pin++) {
-		memset(mpeii, 0, sizeof(*mpeii));
-		mpeii->entry_type = MP_ENTRY_IOINT;
-		mpeii->src_bus_id = MPE_BUSID_ISA;
-		mpeii->dst_apic_id = id;
-
-		/*
-		 * All default configs route IRQs from bus 0 to the first 16 pins
-		 * of the first I/O APIC with an APIC ID of 2.
-		 */
-		mpeii->dst_apic_intin = pin;
-		switch (pin) {
-		case 0:
-			/* Pin 0 is an ExtINT pin. */
-			mpeii->intr_type = MPEII_INTR_EXTINT;
-			break;
-		case 2:
-			/* IRQ 0 is routed to pin 2. */
-			mpeii->intr_type = MPEII_INTR_INT;
-			mpeii->src_bus_irq = 0;
-			break;
-		case 5:
-		case 10:
-		case 11:
-			/*
-			 * PCI Irqs set to level triggered.
-			 */
-			mpeii->intr_flags = MPEII_FLAGS_TRIGMODE_LEVEL;
-			mpeii->src_bus_id = MPE_BUSID_PCI;
-		default:
-			/* All other pins are identity mapped. */
-			mpeii->intr_type = MPEII_INTR_INT;
-			mpeii->src_bus_irq = pin;
-			break;
-		}
-		mpeii++;
-	}
-
-}
-
-#define COPYSTR(dest, src, bytes)		\
-	memcpy(dest, src, bytes); 		\
-	str[bytes] = 0;
-
-
-static void
-mptable_dump(struct mp_floating_pointer *mpfp, struct mp_config_hdr *mpch)
-{
-	static char 	 str[16];
-	int 		 i;
-	char 		*cur;
-
-	union mpe {
-		struct mpe_proc 	*proc;
-		struct mpe_bus  	*bus;
-		struct mpe_ioapic 	*ioapic;
-		struct mpe_ioint 	*ioint;
-		struct mpe_lint 	*lnit;
-		char   			*p;
-	};
-
-	union mpe mpe;
-
-	printf(" MP Floating Pointer :\n");
-	COPYSTR(str, mpfp->signature, 4);
-	printf("    signature: 	%s\n", str);
-	printf("    mpch paddr: %x\n", mpfp->mptable_paddr);
-	printf("    length: 	%x\n", mpfp->length);
-	printf("    specrec: 	%x\n", mpfp->specrev);
-	printf("    checksum: 	%x\n", mpfp->checksum);
-	printf("    feature1: 	%x\n", mpfp->feature1);
-	printf("    feature2: 	%x\n", mpfp->feature2);
-	printf("    feature3: 	%x\n", mpfp->feature3);
-	printf("    feature4: 	%x\n", mpfp->feature4);
-
-	printf(" MP Configuration Header :\n");
-	COPYSTR(str, mpch->signature, 4);
-	printf("    signature: 		%s\n", str);
-	printf("    length: 		%x\n", mpch->length);
-	printf("    specrec: 		%x\n", mpch->specrev);
-	printf("    checksum: 		%x\n", mpch->checksum);
-	COPYSTR(str, mpch->oemid, MPCH_OEMID_LEN);
-	printf("    oemid: 		%s\n", str);
-	COPYSTR(str, mpch->prodid, MPCH_PRODID_LEN);
-	printf("    prodid: 		%s\n", str);
-	printf("    oem_ptr: 		%x\n", mpch->oem_ptr);
-	printf("    oem_sz: 		%x\n", mpch->oem_sz);
-	printf("    nr_entries: 	%x\n", mpch->nr_entries);
-	printf("    apic paddr: 	%x\n", mpch->lapic_paddr);
-	printf("    ext_length: 	%x\n", mpch->ext_length);
-	printf("    ext_checksum: 	%x\n", mpch->ext_checksum);
-
-	cur = (char *)mpch + sizeof(*mpch);
-	for (i = 0; i < mpch->nr_entries; i++) {
-		mpe.p = cur;
-		switch(*mpe.p) {		
-			case MP_ENTRY_PROC:
-				printf(" MP Processor Entry :\n");
-				printf("	lapic_id: 	%x\n", mpe.proc->lapic_id);
-				printf("	lapic_version:	%x\n", mpe.proc->lapic_version);
-				printf("	proc_flags: 	%x\n", mpe.proc->proc_flags);
-				printf("	proc_signature: %x\n", mpe.proc->proc_signature);
-				printf("	feature_flags: 	%x\n", mpe.proc->feature_flags);
-				cur += sizeof(struct mpe_proc);
-				break;
-			case MP_ENTRY_BUS:
-				printf(" MP Bus Entry :\n");
-				printf("	busid: 		%x\n", mpe.bus->busid);
-				COPYSTR(str, mpe.bus->busname, MPE_BUSNAME_LEN);
-				printf("	busname: 	%s\n", str);
-				cur += sizeof(struct mpe_bus);
-				break;
-			case MP_ENTRY_IOAPIC:
-				printf(" MP IOAPIC Entry :\n");
-				printf("	ioapi_id: 		%x\n", mpe.ioapic->ioapic_id);
-				printf("	ioapi_version: 		%x\n", mpe.ioapic->ioapic_version);
-				printf("	ioapi_flags: 		%x\n", mpe.ioapic->ioapic_flags);
-				printf("	ioapi_paddr: 		%x\n", mpe.ioapic->ioapic_paddr);
-				cur += sizeof(struct mpe_ioapic);
-				break;
-			case MP_ENTRY_IOINT:
-				printf(" MP IO Interrupt Entry :\n");
-				printf("	intr_type: 		%x\n", mpe.ioint->intr_type);
-				printf("	intr_flags: 		%x\n", mpe.ioint->intr_flags);
-				printf("	src_bus_id: 		%x\n", mpe.ioint->src_bus_id);
-				printf("	src_bus_irq: 		%x\n", mpe.ioint->src_bus_irq);
-				printf("	dst_apic_id: 		%x\n", mpe.ioint->dst_apic_id);
-				printf("	dst_apic_intin:		%x\n", mpe.ioint->dst_apic_intin);
-				cur += sizeof(struct mpe_ioint);
-				break;
-			case MP_ENTRY_LINT:
-				printf(" MP Local Interrupt Entry :\n");
-				cur += sizeof(struct mpe_lint);
-				break;
-		}
-
-	}
-}
-#endif
-
-int
-vm_build_mptable(struct vmctx *ctx, vm_paddr_t gpa, int len, int ncpu,
-		 int ioapic, void *oemp, int oemsz)
-{
-	struct mp_config_hdr	*mpch;
-	char 			*mapaddr;
-	char 			*startaddr;
-	int 	 		 error;
-
-	mapaddr = vm_map_memory(ctx, gpa, len);
-	if (mapaddr == MAP_FAILED) {
-		printf("%s\n", strerror(errno));
-		goto err;
-	}
-	startaddr = mapaddr;
-
-	mp_build_mpfp((struct mp_floating_pointer*) mapaddr, gpa);
-	mapaddr += sizeof(struct mp_floating_pointer);
-
-	mpch = (struct mp_config_hdr*)mapaddr;
-	mp_build_mpch(mpch);
-	mapaddr += sizeof(struct mp_config_hdr);
-
-	mp_build_proc_entries((struct mpe_proc*) mapaddr, ncpu);
-	mapaddr += (sizeof(struct mpe_proc)*ncpu);
-	mpch->nr_entries += ncpu;
-
-	mp_build_bus_entries((struct mpe_bus*)mapaddr);
-	mapaddr += (sizeof(struct mpe_bus)*MPE_NUM_BUSES);
-	mpch->nr_entries += MPE_NUM_BUSES;
-
-	if (ioapic) {
-		mp_build_ioapic_entries((struct mpe_ioapic*)mapaddr, ncpu + 1);
-		mapaddr += sizeof(struct mpe_ioapic);
-		mpch->nr_entries++;
-	}
-
-#ifdef notyet
-	mp_build_ioint_entries((struct mpe_ioint*)mapaddr, MPEII_MAX_IRQ,
-				ncpu + 1);
-	mapaddr += sizeof(struct mpe_ioint)*MPEII_MAX_IRQ;
-	mpch->nr_entries += MPEII_MAX_IRQ;
-
-#endif
-	if (oemp) {
-		mpch->oem_ptr = mapaddr - startaddr + gpa;
-		mpch->oem_sz = oemsz;
-		memcpy(mapaddr, oemp, oemsz);
-	}
-	mpch->length = (mapaddr) - ((char*) mpch);
-	mpch->checksum = mp_compute_checksum(mpch, sizeof(*mpch));
-
-
-	// mptable_dump((struct mp_floating_pointer*)startaddr, mpch);
-err:
-	return (error);
-}
diff --git a/lib/libvmmapi/mptable.h b/lib/libvmmapi/mptable.h
deleted file mode 100644
index 51c7bc1..0000000
--- a/lib/libvmmapi/mptable.h
+++ /dev/null
@@ -1,170 +0,0 @@
-/*-
- * Copyright (c) 2011 NetApp, Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * $FreeBSD$
- */
-
-#ifndef _MPTABLE_h_
-#define	_MPTABLE_h_
-
-#define MP_SPECREV		(4)		// MP spec revision 1.1
-
-/*
- * MP Floating Pointer Structure
- */ 
-#define MPFP_SIGNATURE		"_MP_"
-#define MPFP_SIGNATURE_LEN	(4)
-#define MPFP_FEATURE2		(0x80)	// IMCR is present
-struct mp_floating_pointer {
-	uint8_t 	signature[MPFP_SIGNATURE_LEN];
-	uint32_t 	mptable_paddr;
-	uint8_t 	length;
-	uint8_t 	specrev;
-	uint8_t 	checksum;
-	uint8_t 	feature1;
-	uint8_t 	feature2;
-	uint8_t 	feature3;
-	uint8_t 	feature4;
-	uint8_t 	feature5;
-};
-
-
-/*
- * MP Configuration Table Header
- */
-#define MPCH_SIGNATURE		"PCMP"
-#define MPCH_SIGNATURE_LEN	(4)
-
-#define MPCH_OEMID		"NETAPP  "
-#define MPCH_OEMID_LEN		(8)
-#define MPCH_PRODID		"vFiler      "
-#define MPCH_PRODID_LEN		(12)
-
-struct mp_config_hdr {
-	uint8_t 	signature[MPCH_SIGNATURE_LEN];
-	uint16_t 	length;
-	uint8_t 	specrev;
-	uint8_t 	checksum;
-	uint8_t 	oemid[MPCH_OEMID_LEN];
-	uint8_t 	prodid[MPCH_PRODID_LEN];
-	uint32_t 	oem_ptr;
-	uint16_t 	oem_sz;
-	uint16_t 	nr_entries;
-	uint32_t 	lapic_paddr;
-	uint16_t 	ext_length;
-	uint8_t 	ext_checksum;
-	uint8_t 	reserved;
-};
-
-#define MP_ENTRY_PROC	(0)
-#define MP_ENTRY_BUS	(1)
-#define MP_ENTRY_IOAPIC	(2)
-#define MP_ENTRY_IOINT	(3)
-#define MP_ENTRY_LINT	(4)
-
-/*
- * MP Processor Entry
- */
-
-#define MPEP_FLAGS_EN		(0x1)
-#define MPEP_FLAGS_BSP		(0x2)
-
-#define MPEP_SIG_FAMILY         (6)
-#define MPEP_SIG_MODEL          (26)
-#define MPEP_SIG_STEPPING       (5)
-#define MPEP_SIGNATURE  	((MPEP_SIG_FAMILY << 8) | (MPEP_SIG_MODEL << 4)  \
-				| (MPEP_SIG_STEPPING))
-
-#define MPEP_FEATURES		(0xBFEBFBFF) // Value from Intel i7 CPUID
-
-struct mpe_proc {
-	uint8_t 	entry_type;
-	uint8_t 	lapic_id;
-	uint8_t 	lapic_version;
-	uint8_t 	proc_flags;
-	uint32_t 	proc_signature;
-	uint32_t 	feature_flags;
-	uint8_t 	reserved[8];
-};
-
-/*
- * MP Bus Entry
- */
-
-#define MPE_NUM_BUSES	(2)
-#define MPE_BUSNAME_LEN	(6)
-#define MPE_BUSID_ISA	(0)
-#define MPE_BUSID_PCI	(1)
-#define MPE_BUSNAME_ISA	"ISA   "
-#define MPE_BUSNAME_PCI	"PCI   "
-struct mpe_bus {
-	uint8_t 	entry_type;
-	uint8_t		busid;
-	uint8_t		busname[MPE_BUSNAME_LEN];
-};
-
-/*
- * MP IO APIC Entry
- */
-#define MPE_IOAPIC_FLAG_EN	(1)
-struct mpe_ioapic {
-	uint8_t 	entry_type;
-	uint8_t		ioapic_id;
-	uint8_t		ioapic_version;
-	uint8_t		ioapic_flags;
-	uint32_t	ioapic_paddr;
-
-};
-
-/*
- * MP IO Interrupt Assignment Entry
- */
-#define MPEII_INTR_INT		(0)
-#define MPEII_INTR_NMI		(1)
-#define MPEII_INTR_SMI		(2)
-#define MPEII_INTR_EXTINT	(3)
-#define MPEII_PCI_IRQ_MASK    	(0x0c20U) /* IRQ 5,10,11 are PCI connected */
-#define MPEII_MAX_IRQ		(16)
-#define MPEII_FLAGS_TRIGMODE_LEVEL	(0x3)
-struct mpe_ioint {
-	uint8_t 	entry_type;
-	uint8_t		intr_type;
-	uint16_t	intr_flags;
-	uint8_t		src_bus_id;
-	uint8_t		src_bus_irq;
-	uint8_t		dst_apic_id;
-	uint8_t		dst_apic_intin;
-};
-
-/*
- * MP Local Interrupt Assignment Entry
- */
-struct mpe_lint {
-	uint8_t 	entry_type;
-};
-
-int	vm_build_mptable(struct vmctx *ctxt, vm_paddr_t gpa, int len,
-			 int ncpu, int ioapic, void *oemp, int oemsz);
-#endif	/* _MPTABLE_h_ */
diff --git a/lib/libvmmapi/vmmapi.c b/lib/libvmmapi/vmmapi.c
index fdbbbcb..cfb42d0 100644
--- a/lib/libvmmapi/vmmapi.c
+++ b/lib/libvmmapi/vmmapi.c
@@ -47,10 +47,6 @@ __FBSDID("$FreeBSD$");
 #include <machine/vmm_dev.h>
 
 #include "vmmapi.h"
-#include "mptable.h"
-
-#define BIOS_ROM_BASE		(0xf0000)
-#define BIOS_ROM_SIZE		(0x10000)
 
 struct vmctx {
 	int	fd;
@@ -329,15 +325,6 @@ vm_inject_event2(struct vmctx *ctx, int vcpu, enum vm_event_type type,
 }
 
 int
-vm_build_tables(struct vmctx *ctxt, int ncpu, int ioapic,
-		void *oemtbl, int oemtblsz)
-{
-
-	return (vm_build_mptable(ctxt, BIOS_ROM_BASE, BIOS_ROM_SIZE, ncpu,
-				 ioapic, oemtbl, oemtblsz));
-}
-
-int
 vm_apicid2vcpu(struct vmctx *ctx, int apicid)
 {
 	/*
diff --git a/lib/libvmmapi/vmmapi.h b/lib/libvmmapi/vmmapi.h
index fe5116e..de04252 100644
--- a/lib/libvmmapi/vmmapi.h
+++ b/lib/libvmmapi/vmmapi.h
@@ -60,8 +60,6 @@ int	vm_get_pinning(struct vmctx *ctx, int vcpu, int *host_cpuid);
 int	vm_set_pinning(struct vmctx *ctx, int vcpu, int host_cpuid);
 int	vm_run(struct vmctx *ctx, int vcpu, uint64_t rip,
 	       struct vm_exit *ret_vmexit);
-int	vm_build_tables(struct vmctx *ctxt, int ncpus, int ioapic,
-			void *oemtbl, int oemtblsz);
 int	vm_apicid2vcpu(struct vmctx *ctx, int apicid);
 int	vm_inject_event(struct vmctx *ctx, int vcpu, enum vm_event_type type,
 			int vector);
-- 
cgit v1.1


From 9413d826123906120b1317ad76bcc03cada2f98a Mon Sep 17 00:00:00 2001
From: dim <dim@FreeBSD.org>
Date: Mon, 5 Nov 2012 18:49:21 +0000
Subject: Fix a few warnings from newer clang 3.2 in libpmc, about comparing
 enum pmc_event values against integer constants which fall outside the enum
 range.

Reviewed by:	fabient, sbruno
MFC after:	3 days
---
 lib/libpmc/libpmc.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/libpmc/libpmc.c b/lib/libpmc/libpmc.c
index 85c460d..4fb2572 100644
--- a/lib/libpmc/libpmc.c
+++ b/lib/libpmc/libpmc.c
@@ -2264,7 +2264,7 @@ soft_allocate_pmc(enum pmc_event pe, char *ctrspec,
 	(void)ctrspec;
 	(void)pmc_config;
 
-	if (pe < PMC_EV_SOFT_FIRST || pe > PMC_EV_SOFT_LAST)
+	if ((int)pe < PMC_EV_SOFT_FIRST || (int)pe > PMC_EV_SOFT_LAST)
 		return (-1);
 
 	pmc_config->pm_caps |= (PMC_CAP_READ | PMC_CAP_WRITE);
@@ -3190,7 +3190,7 @@ _pmc_name_of_event(enum pmc_event pe, enum pmc_cputype cpu)
 	} else if (pe == PMC_EV_TSC_TSC) {
 		ev = tsc_event_table;
 		evfence = tsc_event_table + PMC_EVENT_TABLE_SIZE(tsc);
-	} else if (pe >= PMC_EV_SOFT_FIRST && pe <= PMC_EV_SOFT_LAST) {
+	} else if ((int)pe >= PMC_EV_SOFT_FIRST && (int)pe <= PMC_EV_SOFT_LAST) {
 		ev = soft_event_table;
 		evfence = soft_event_table + soft_event_info.pm_nevent;
 	}
-- 
cgit v1.1


From de6ea8b20e870490db809a8d8a965bd784981d81 Mon Sep 17 00:00:00 2001
From: marcel <marcel@FreeBSD.org>
Date: Wed, 7 Nov 2012 00:23:51 +0000
Subject: Remove -L<path> from LDADD, it doesn't belong there. Add it to
 LDFLAGS instead.

Submitted by:   Garrett Cooper <yanegomi@gmail.com>
---
 lib/atf/libatf-c++/Makefile | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/atf/libatf-c++/Makefile b/lib/atf/libatf-c++/Makefile
index 71369e2..37d6073 100644
--- a/lib/atf/libatf-c++/Makefile
+++ b/lib/atf/libatf-c++/Makefile
@@ -32,7 +32,9 @@ SHLIB_MAJOR=	1
 
 # libatf-c++ depends on the C version of the ATF library to build.
 DPADD=		${LIBATFC}
-LDADD=		-L${.OBJDIR}/../libatf-c -latf-c
+LDADD=		-latf-c
+
+LDFLAGS+=	-L${.OBJDIR}/../libatf-c
 
 .PATH:		${ATF}
 .PATH:		${ATF}/atf-c++
-- 
cgit v1.1


From 25611f9cf9abb970322ecc6e9eb957d8f47b4819 Mon Sep 17 00:00:00 2001
From: kevlo <kevlo@FreeBSD.org>
Date: Wed, 7 Nov 2012 07:00:59 +0000
Subject: Fix typo; s/ouput/output

---
 lib/msun/src/k_rem_pio2.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/msun/src/k_rem_pio2.c b/lib/msun/src/k_rem_pio2.c
index a2ffca6..3942441 100644
--- a/lib/msun/src/k_rem_pio2.c
+++ b/lib/msun/src/k_rem_pio2.c
@@ -45,7 +45,7 @@ __FBSDID("$FreeBSD$");
  *			z    = (z-x[i])*2**24
  *
  *
- *	y[]	ouput result in an array of double precision numbers.
+ *	y[]	output result in an array of double precision numbers.
  *		The dimension of y[] is:
  *			24-bit  precision	1
  *			53-bit  precision	2
-- 
cgit v1.1


From 79b7e755b41a4f7e2a36e4a7c6122f35fba258eb Mon Sep 17 00:00:00 2001
From: sjg <sjg@FreeBSD.org>
Date: Wed, 7 Nov 2012 22:03:59 +0000
Subject: Simple unit-tests for libcrypt, to show how easy it is.

Approved by:	marcel (mentor)
---
 lib/libcrypt/tests/Makefile      | 10 ++++++++
 lib/libcrypt/tests/crypt_tests.c | 54 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 64 insertions(+)
 create mode 100644 lib/libcrypt/tests/Makefile
 create mode 100644 lib/libcrypt/tests/crypt_tests.c

(limited to 'lib')

diff --git a/lib/libcrypt/tests/Makefile b/lib/libcrypt/tests/Makefile
new file mode 100644
index 0000000..3190dbe
--- /dev/null
+++ b/lib/libcrypt/tests/Makefile
@@ -0,0 +1,10 @@
+# $FreeBSD$
+
+# exercise libcrypt
+
+TESTS_C= crypt_tests
+
+CFLAGS+= -I${.CURDIR:H}
+LDADD+= -L${.OBJDIR:H} -lcrypt
+
+.include <atf.test.mk>
diff --git a/lib/libcrypt/tests/crypt_tests.c b/lib/libcrypt/tests/crypt_tests.c
new file mode 100644
index 0000000..3331d12
--- /dev/null
+++ b/lib/libcrypt/tests/crypt_tests.c
@@ -0,0 +1,54 @@
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/types.h>
+#include <crypt.h>
+#include <unistd.h>
+
+#include <atf-c.h>
+
+#define	LEET "0.s0.l33t"
+
+ATF_TC(md5);
+ATF_TC_HEAD(md5, tc)
+{
+
+	atf_tc_set_md_var(tc, "descr", "Tests the MD5 based password hash");
+}
+
+ATF_TC_BODY(md5, tc)
+{
+	const char want[] = "$1$deadbeef$0Huu6KHrKLVWfqa4WljDE0";
+	char *pw;
+
+	pw = crypt(LEET, want);
+	ATF_CHECK_STREQ(pw, want);
+}
+
+ATF_TC(invalid);
+ATF_TC_HEAD(invalid, tc)
+{
+
+	atf_tc_set_md_var(tc, "descr", "Tests that invalid password fails");
+}
+
+ATF_TC_BODY(invalid, tc)
+{
+	const char want[] = "$1$cafebabe$0Huu6KHrKLVWfqa4WljDE0";
+	char *pw;
+
+	pw = crypt(LEET, want);
+	ATF_CHECK(strcmp(pw, want) != 0);
+}
+
+/*
+ * This function must not do anything except enumerate
+ * the test cases, else atf-run is likely to be upset.
+ */
+ATF_TP_ADD_TCS(tp)
+{
+
+	ATF_TP_ADD_TC(tp, md5);
+	ATF_TP_ADD_TC(tp, invalid);
+	return atf_no_error();
+}
-- 
cgit v1.1


From 99d6a5644c78b3f3504ecbf7be003b7c3e99a62a Mon Sep 17 00:00:00 2001
From: jhibbits <jhibbits@FreeBSD.org>
Date: Wed, 7 Nov 2012 23:45:09 +0000
Subject: Implement DTrace for PowerPC.  This includes both 32-bit and 64-bit.

There is one known issue:  Some probes will display an error message along the
lines of:  "Invalid address (0)"

I tested this with both a simple dtrace probe and dtruss on a few different
binaries on 32-bit.  I only compiled 64-bit, did not run it, but I don't expect
problems without the modules loaded.  Volunteers are welcome.

MFC after:	1 month
---
 lib/Makefile            | 5 +++++
 lib/libproc/proc_bkpt.c | 3 +++
 lib/libproc/proc_regs.c | 8 ++++++++
 3 files changed, 16 insertions(+)

(limited to 'lib')

diff --git a/lib/Makefile b/lib/Makefile
index 199e08b..7c3ccbd 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -207,6 +207,11 @@ _libproc=	libproc
 _librtld_db=	librtld_db
 .endif
 
+.if ${MACHINE_CPUARCH} == "powerpc"
+_libproc=	libproc
+_librtld_db=	librtld_db
+.endif
+
 .if ${MK_OPENSSL} != "no"
 _libmp=		libmp
 .endif
diff --git a/lib/libproc/proc_bkpt.c b/lib/libproc/proc_bkpt.c
index e16b0fc..c15e53c 100644
--- a/lib/libproc/proc_bkpt.c
+++ b/lib/libproc/proc_bkpt.c
@@ -47,6 +47,9 @@ __FBSDID("$FreeBSD$");
 #elif defined(__mips__)
 #define BREAKPOINT_INSTR	0xd	/* break */
 #define	BREAKPOINT_INSTR_SZ	4
+#elif defined(__powerpc__)
+#define BREAKPOINT_INSTR	0x7fe00008	/* trap */
+#define BREAKPOINT_INSTR_SZ 4
 #else
 #error "Add support for your architecture"
 #endif
diff --git a/lib/libproc/proc_regs.c b/lib/libproc/proc_regs.c
index c299b9b..aac0125 100644
--- a/lib/libproc/proc_regs.c
+++ b/lib/libproc/proc_regs.c
@@ -60,6 +60,8 @@ proc_regget(struct proc_handle *phdl, proc_reg_t reg, unsigned long *regvalue)
 		*regvalue = regs.r_eip;
 #elif defined(__mips__)
 		*regvalue = regs.r_regs[PC];
+#elif defined(__powerpc__)
+		*regvalue = regs.pc;
 #endif
 		break;
 	case REG_SP:
@@ -69,6 +71,8 @@ proc_regget(struct proc_handle *phdl, proc_reg_t reg, unsigned long *regvalue)
 		*regvalue = regs.r_esp;
 #elif defined(__mips__)
 		*regvalue = regs.r_regs[SP];
+#elif defined(__powerpc__)
+		*regvalue = regs.fixreg[1];
 #endif
 		break;
 	default:
@@ -99,6 +103,8 @@ proc_regset(struct proc_handle *phdl, proc_reg_t reg, unsigned long regvalue)
 		regs.r_eip = regvalue;
 #elif defined(__mips__)
 		regs.r_regs[PC] = regvalue;
+#elif defined(__powerpc__)
+		regs.pc = regvalue;
 #endif
 		break;
 	case REG_SP:
@@ -108,6 +114,8 @@ proc_regset(struct proc_handle *phdl, proc_reg_t reg, unsigned long regvalue)
 		regs.r_esp = regvalue;
 #elif defined(__mips__)
 		regs.r_regs[PC] = regvalue;
+#elif defined(__powerpc__)
+		regs.fixreg[1] = regvalue;
 #endif
 		break;
 	default:
-- 
cgit v1.1


From 8af663c65db48078492379d8dc6a7d55934ea502 Mon Sep 17 00:00:00 2001
From: grog <grog@FreeBSD.org>
Date: Thu, 8 Nov 2012 02:01:04 +0000
Subject: Clarify that the ' flag is an apostrophe.

MFC after:	2 weeks
---
 lib/libc/stdio/printf.3 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libc/stdio/printf.3 b/lib/libc/stdio/printf.3
index 0d9339e..05c30dc 100644
--- a/lib/libc/stdio/printf.3
+++ b/lib/libc/stdio/printf.3
@@ -267,7 +267,7 @@ number produced by a signed conversion.
 A
 .Cm +
 overrides a space if both are used.
-.It Sq Cm '
+.It So "'" Sc (apostrophe)
 Decimal conversions
 .Cm ( d , u ,
 or
-- 
cgit v1.1


From 464808b6f100795216849a2948e345eeaac23fec Mon Sep 17 00:00:00 2001
From: dim <dim@FreeBSD.org>
Date: Sat, 10 Nov 2012 21:22:10 +0000
Subject: Only define isnan, isnanf, __isnan and __isnanf in libc.so, not in
 libc.a and libc_p.a.  In addition, define isnan in libm.a and libm_p.a, but
 not in libm.so.

This makes it possible to statically link executables using both isnan
and isnanf with libc and libm.

Tested by:	kargl
MFC after:	1 week
---
 lib/libc/gen/isnan.c   | 2 ++
 lib/msun/src/s_isnan.c | 7 ++++---
 2 files changed, 6 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/gen/isnan.c b/lib/libc/gen/isnan.c
index ec81362..64b0ac1 100644
--- a/lib/libc/gen/isnan.c
+++ b/lib/libc/gen/isnan.c
@@ -35,6 +35,7 @@
  *     binary compat until we can bump libm's major version number.
  */
 
+#ifdef PIC
 __weak_reference(__isnan, isnan);
 __weak_reference(__isnanf, isnanf);
 
@@ -55,3 +56,4 @@ __isnanf(float f)
 	u.f = f;
 	return (u.bits.exp == 255 && u.bits.man != 0);
 }
+#endif /* PIC */
diff --git a/lib/msun/src/s_isnan.c b/lib/msun/src/s_isnan.c
index 0f544db..a54ded3 100644
--- a/lib/msun/src/s_isnan.c
+++ b/lib/msun/src/s_isnan.c
@@ -30,8 +30,9 @@
 
 #include "fpmath.h"
 
-/* Provided by libc */
-#if 0
+/* Provided by libc.so */
+#ifndef PIC
+#undef isnan
 int
 isnan(double d)
 {
@@ -40,7 +41,7 @@ isnan(double d)
 	u.d = d;
 	return (u.bits.exp == 2047 && (u.bits.manl != 0 || u.bits.manh != 0));
 }
-#endif
+#endif /* !PIC */
 
 int
 __isnanf(float f)
-- 
cgit v1.1


From 551d372a1bddcd79984b86718d69d54076349901 Mon Sep 17 00:00:00 2001
From: dim <dim@FreeBSD.org>
Date: Sun, 11 Nov 2012 13:28:04 +0000
Subject: Add an explanatory comment to lib/libc/gen/isnan.c about the fix to
 make static linking with libc and libm work.

Requested by:	jilles
MFC after:	1 week
X-MFC-With:	242879
---
 lib/libc/gen/isnan.c | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'lib')

diff --git a/lib/libc/gen/isnan.c b/lib/libc/gen/isnan.c
index 64b0ac1..72c2868 100644
--- a/lib/libc/gen/isnan.c
+++ b/lib/libc/gen/isnan.c
@@ -33,6 +33,11 @@
 /*
  * XXX These routines belong in libm, but they must remain in libc for
  *     binary compat until we can bump libm's major version number.
+ *
+ * Note this only applies to the dynamic versions of libm and libc, so
+ * for the static and profiled versions we stub out the definitions.
+ * Otherwise you cannot link statically to libm and libc at the same
+ * time, when calling both functions.
  */
 
 #ifdef PIC
-- 
cgit v1.1


From 37c97ba01ba0cd42b9334d786dcadbf99cbb2456 Mon Sep 17 00:00:00 2001
From: kib <kib@FreeBSD.org>
Date: Tue, 13 Nov 2012 12:55:52 +0000
Subject: Implement the waitid() SUSv4 function using wait6() system call.

PR:	standards/170346
Submitted by:	"Jukka A. Ukkonen" <jau@iki.fi>
MFC after:	1 month
---
 lib/libc/gen/Makefile.inc       |  2 +-
 lib/libc/gen/Symbol.map         |  1 +
 lib/libc/gen/waitid.c           | 65 +++++++++++++++++++++++++++++++++++++++++
 lib/libc/include/namespace.h    |  1 +
 lib/libc/include/un-namespace.h |  1 +
 lib/libc/sys/Symbol.map         |  3 ++
 6 files changed, 72 insertions(+), 1 deletion(-)
 create mode 100644 lib/libc/gen/waitid.c

(limited to 'lib')

diff --git a/lib/libc/gen/Makefile.inc b/lib/libc/gen/Makefile.inc
index 2481f28..906f4ce 100644
--- a/lib/libc/gen/Makefile.inc
+++ b/lib/libc/gen/Makefile.inc
@@ -34,7 +34,7 @@ SRCS+=  __getosreldate.c __xuname.c \
 	syslog.c telldir.c termios.c time.c times.c timezone.c tls.c \
 	ttyname.c ttyslot.c ualarm.c ulimit.c uname.c unvis.c \
 	usleep.c utime.c utxdb.c valloc.c vis.c wait.c wait3.c waitpid.c \
-	wordexp.c
+	waitid.c wordexp.c
 
 .PATH: ${.CURDIR}/../../contrib/libc-pwcache
 SRCS+=	pwcache.c pwcache.h
diff --git a/lib/libc/gen/Symbol.map b/lib/libc/gen/Symbol.map
index b4f88ae..f5d4f71 100644
--- a/lib/libc/gen/Symbol.map
+++ b/lib/libc/gen/Symbol.map
@@ -391,6 +391,7 @@ FBSD_1.3 {
 	pwcache_userdb;
 	pwcache_groupdb;
 	uid_from_user;
+	waitid;
 };
 
 FBSDprivate_1.0 {
diff --git a/lib/libc/gen/waitid.c b/lib/libc/gen/waitid.c
new file mode 100644
index 0000000..795b208
--- /dev/null
+++ b/lib/libc/gen/waitid.c
@@ -0,0 +1,65 @@
+/*-
+ * Copyright (c) 2012 Jukka A. Ukkonen
+ * All rights reserved.
+ *
+ * This software was developed by Jukka Ukkonen for FreeBSD.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "namespace.h"
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <stddef.h>
+#include <string.h>
+#include <signal.h>
+#include <errno.h>
+#include "un-namespace.h"
+
+int
+__waitid(idtype_t idtype, id_t id, siginfo_t *info, int flags)
+{
+	int status;
+	pid_t ret;
+
+	ret = _wait6(idtype, id, &status, flags, NULL, info);
+
+	/*
+	 * According to SUSv4, waitid() shall not return a PID when a
+	 * process is found, but only 0.  If a process was actually
+	 * found, siginfo_t fields si_signo and si_pid will be
+	 * non-zero.  In case WNOHANG was set in the flags and no
+	 * process was found those fields are set to zero using
+	 * memset() below.
+	 */
+	if (ret == 0 && info != NULL)
+		memset(info, 0, sizeof(*info));
+	else if (ret > 0)
+		ret = 0;
+	return (ret);
+}
+
+__weak_reference(__waitid, waitid);
+__weak_reference(__waitid, _waitid);
diff --git a/lib/libc/include/namespace.h b/lib/libc/include/namespace.h
index 1e00030..739d7b1 100644
--- a/lib/libc/include/namespace.h
+++ b/lib/libc/include/namespace.h
@@ -229,6 +229,7 @@
 #define		socketpair			_socketpair
 #define		usleep				_usleep
 #define		wait4				_wait4
+#define		wait6				_wait6
 #define		waitpid				_waitpid
 #define		write				_write
 #define		writev				_writev
diff --git a/lib/libc/include/un-namespace.h b/lib/libc/include/un-namespace.h
index 9c9ce97..f31fa7a 100644
--- a/lib/libc/include/un-namespace.h
+++ b/lib/libc/include/un-namespace.h
@@ -210,6 +210,7 @@
 #undef		socketpair
 #undef		usleep
 #undef		wait4
+#undef		wait6
 #undef		waitpid
 #undef		write
 #undef		writev
diff --git a/lib/libc/sys/Symbol.map b/lib/libc/sys/Symbol.map
index 008b8da..babae30 100644
--- a/lib/libc/sys/Symbol.map
+++ b/lib/libc/sys/Symbol.map
@@ -384,6 +384,7 @@ FBSD_1.3 {
 	ffclock_getestimate;
 	ffclock_setestimate;
 	posix_fadvise;
+	wait6;
 };
 
 FBSDprivate_1.0 {
@@ -1019,6 +1020,8 @@ FBSDprivate_1.0 {
 	__sys_vadvise;
 	_wait4;
 	__sys_wait4;
+	_wait6;
+	__sys_wait6;
 	_write;
 	__sys_write;
 	_writev;
-- 
cgit v1.1


From 39bfcc5eed2a3f48c8f7e92b61173fa2f7edb3a9 Mon Sep 17 00:00:00 2001
From: kib <kib@FreeBSD.org>
Date: Tue, 13 Nov 2012 12:56:42 +0000
Subject: Document wait6() and waitid().

PR:	standards/170346
Submitted by:	"Jukka A. Ukkonen" <jau@iki.fi>
MFC after:	1 month
---
 lib/libc/sys/Makefile.inc |   3 +-
 lib/libc/sys/wait.2       | 311 +++++++++++++++++++++++++++++++++++++++++-----
 2 files changed, 282 insertions(+), 32 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/sys/Makefile.inc b/lib/libc/sys/Makefile.inc
index df4ef42..a35ed5d 100644
--- a/lib/libc/sys/Makefile.inc
+++ b/lib/libc/sys/Makefile.inc
@@ -217,5 +217,6 @@ MLINKS+=timer_settime.2 timer_getoverrun.2 timer_settime.2 timer_gettime.2
 MLINKS+=truncate.2 ftruncate.2
 MLINKS+=unlink.2 unlinkat.2
 MLINKS+=utimes.2 futimes.2 utimes.2 futimesat.2 utimes.2 lutimes.2
-MLINKS+=wait.2 wait3.2 wait.2 wait4.2 wait.2 waitpid.2
+MLINKS+=wait.2 wait3.2 wait.2 wait4.2 wait.2 waitpid.2 \
+	wait.2 waitid.2 wait.2 wait6.2
 MLINKS+=write.2 pwrite.2 write.2 pwritev.2 write.2 writev.2
diff --git a/lib/libc/sys/wait.2 b/lib/libc/sys/wait.2
index 71bd529..0c494bb 100644
--- a/lib/libc/sys/wait.2
+++ b/lib/libc/sys/wait.2
@@ -28,15 +28,17 @@
 .\"     @(#)wait.2	8.2 (Berkeley) 4/19/94
 .\" $FreeBSD$
 .\"
-.Dd November 12, 2005
+.Dd November 10, 2012
 .Dt WAIT 2
 .Os
 .Sh NAME
 .Nm wait ,
+.Nm waitid ,
 .Nm waitpid ,
+.Nm wait3 ,
 .Nm wait4 ,
-.Nm wait3
-.Nd wait for process termination
+.Nm wait6
+.Nd wait for processes to change status
 .Sh LIBRARY
 .Lb libc
 .Sh SYNOPSIS
@@ -46,12 +48,17 @@
 .Fn wait "int *status"
 .Ft pid_t
 .Fn waitpid "pid_t wpid" "int *status" "int options"
+.In sys/signal.h
+.Ft int
+.Fn waitid "idtype_t idtype" "id_t id" "siginfo_t *info" "int options"
 .In sys/time.h
 .In sys/resource.h
 .Ft pid_t
 .Fn wait3 "int *status" "int options" "struct rusage *rusage"
 .Ft pid_t
 .Fn wait4 "pid_t wpid" "int *status" "int options" "struct rusage *rusage"
+.Ft pid_t
+.Fn wait6 "idtype_t idtype" "id_t id" "int *status" "int options" "struct __wrusage *wrusage" "siginfo_t *infop"
 .Sh DESCRIPTION
 The
 .Fn wait
@@ -86,28 +93,172 @@ system call provides a more general interface for programs
 that need to wait for certain child processes,
 that need resource utilization statistics accumulated by child processes,
 or that require options.
-The other wait functions are implemented using
-.Fn wait4 .
 .Pp
+The broadest interface of all functions in this family is
+.Fn wait6
+which is otherwise very much like
+.Fn wait4
+but with a few very important distinctions.
+To wait for exited processes, the option flag
+.Dv WEXITED
+need to be explicitly specified.
+This allows for waiting for processes which have experienced other
+status changes without having to handle also the exit status from
+the terminated processes.
+Instead of the traditional
+.Dv rusage
+argument, a pointer to a new structure
+.Bd -literal
+struct __wrusage {
+	struct rusage   wru_self;
+	struct rusage   wru_children;
+};
+.Ed
+can be passed.
+This allows the calling process to collect resource usage statistics
+from both its own child process as well as from its grand children.
+When no resource usage statistics are needed this pointer can be
+.Dv NULL .
+The last argument
+.Fa infop
+must be either
+.Dv NULL
+or a pointer to a
+.Fa siginfo_t
+structure.
+When specified, the structure is filled the same as for
+.Dv SIGNCHLD
+signal, delivered at the process state change.
+.br
+The process, which state is queried, is specified by two arguments
+.Fa idtype
+and
+.Fa id .
+The separate
+.Fa idtype
+and
+.Fa id
+arguments allows to support many other types of
+IDs  as well in addition to PID and PGID.
+.Bl -bullet -offset indent
+.It
+If
+.Fa idtype
+is
+.Dv P_PID ,
+.Fn waitid
+and
+.Fn wait6
+wait for the child process with a process ID equal to
+.Dv (pid_t)id .
+.It
+If
+.Fa idtype
+is
+.Dv P_PGID ,
+.Fn waitid
+and
+.Fn wait6
+wait for the child process with a process group ID equal to
+.Dv (pid_t)id .
+.It
+If
+.Fa idtype
+is
+.Dv P_ALL ,
+.Fn waitid
+and
+.Fn wait6
+wait for any child process and the
+.Dv id
+is ignored.
+.It
+If
+.Fa idtype
+is
+.Dv P_PID
+or
+.Dv P_PGID
+and the
+.Dv id
+is zero,
+.Fn waitid
+and
+.Fn wait6
+wait for any child process in the same process group as the caller.
+.El
+.Pp
+Non-standard specifiers for the process to wait for, supported by this
+implementation of
+.Fn waitid
+and
+.Fn wait6 ,
+are:
+.Bl -bullet -offset indent
+.It
 The
+.Fa idtype
+value
+.Dv P_UID
+waits for processes which effective UID is equal to
+.Dv (uid_t)id .
+.It
+The
+.Fa idtype
+value
+.Dv P_GID
+waits for processes which effective GID is equal to
+.Dv (gid_t)id .
+.It
+The
+.Fa idtype
+value
+.Dv P_SID
+waits for processes which session ID is equal to
+.Dv id .
+In case the child process started its own new session,
+SID will be the same as its own PID.
+Otherwise the SID of a child process will match the caller's SID.
+.It
+The
+.Fa idtype
+value
+.Dv P_JAILID
+waits for processes within a jail which jail identifier is equal
+to
+.Dv id .
+.El
+.Pp
+For
+.Fn wait ,
+.Fn wait3 ,
+and
+.Fn wait4
+functions, the single
 .Fa wpid
 argument specifies the set of child processes for which to wait.
+.Bl -bullet -offset indent
+.It
 If
 .Fa wpid
 is -1, the call waits for any child process.
+.It
 If
 .Fa wpid
 is 0,
 the call waits for any child process in the process group of the caller.
+.It
 If
 .Fa wpid
 is greater than zero, the call waits for the process with process id
 .Fa wpid .
+.It
 If
 .Fa wpid
 is less than -1, the call waits for any process whose process group id
 equals the absolute value of
 .Fa wpid .
+.El
 .Pp
 The
 .Fa status
@@ -116,41 +267,102 @@ argument is defined below.
 The
 .Fa options
 argument contains the bitwise OR of any of the following options.
-The
-.Dv WCONTINUED
-option indicates that children of the current process that
+.Bl -tag -width Ds
+.It Dv WCONTINUED
+indicates that children of the current process that
 have continued from a job control stop, by receiving a
 .Dv SIGCONT
 signal, should also have their status reported.
-The
-.Dv WNOHANG
-option
-is used to indicate that the call should not block if
-there are no processes that wish to report status.
-If the
-.Dv WUNTRACED
-option is set,
-children of the current process that are stopped
+.It Dv WNOHANG
+is used to indicate that the call should not block when
+there are no processes wishing to report status.
+.It Dv WUNTRACED
+indicates that children of the current process which are stopped
 due to a
 .Dv SIGTTIN , SIGTTOU , SIGTSTP ,
 or
 .Dv SIGSTOP
-signal also have their status reported.
-The
-.Dv WSTOPPED
-option is an alias for
+signal shall have their status reported.
+.It Dv WSTOPPED
+is an alias for
 .Dv WUNTRACED .
-The
-.Dv WNOWAIT
-option keeps the process whose status is returned in a waitable state.
+.It Dv WTRAPPED
+allows waiting for processes which have trapped or reached a breakpoint.
+.It Dv WEXITED
+indicates that the caller is wants to receive status reports from
+terminated processes.
+This flag is implicitly set for the functions
+.Fn wait ,
+.Fn waitpid ,
+.Fn wait3 ,
+and
+.Fn wait4 .
+.br
+For the
+.Fn waitid
+and
+.Fn wait6
+functions, the flag has to be explicitly included in the
+.Fa options ,
+if status reports from terminated processes are expected.
+.It Dv WNOWAIT
+keeps the process whose status is returned in a waitable state.
 The process may be waited for again after this call completes.
+.El
+.sp
+For the
+.Fn waitid
+and
+.Fn wait6
+functions, at least one of the options
+.Dv WEXITED ,
+.Dv WUNTRACED ,
+.Dv WSTOPPED ,
+.Dv WTRAPPED ,
+or
+.Dv WCONTINUED
+must be specified.
+Otherwise there will be no events for the call to report.
+To avoid hanging indefinitely in such a case these functions
+return -1 with
+.Dv errno
+set to
+.Dv EINVAL .
 .Pp
 If
 .Fa rusage
-is non-zero, a summary of the resources used by the terminated
-process and all its
-children is returned (this information is currently not available
-for stopped or continued processes).
+is non-NULL, a summary of the resources used by the terminated
+process and all its children is returned.
+.Pp
+If
+.Fa wrusage
+argument is non-NULL, a resource usage statistics
+from both its own child process as well as from its grand children
+is returned.
+.Pp
+If
+.Fa infop
+is non-NULL, it must point to a
+.Dv siginfo_t
+structure which is filled on return such that the
+.Dv si_signo
+field is always
+.Dv SIGCHLD
+and the field
+.Dv si_pid
+if be non-zero, if there is a status change to report.
+If there are no status changes to report and WNOHANG is applied,
+both of these fields are returned zero.
+When using the
+.Fn waitid
+function with the
+.Dv WNOHANG
+option set, checking these fields is the only way to know whether
+there were any status changes to report, because the return value
+from
+.Fn waitid
+is be zero as it is for any successful return from
+.Fn waitid .
 .Pp
 When the
 .Dv WNOHANG
@@ -175,6 +387,20 @@ call is the same as
 with a
 .Fa wpid
 value of -1.
+The
+.Fn wait6
+call, with the bits
+.Dv WEXITED
+and
+.Dv WTRAPPED
+set in the
+.Fa options
+and with
+.Fa infop
+set to
+.Dv NULL ,
+is similar to
+.Fn wait4 .
 .Pp
 The following macros may be used to test the manner of exit of the process.
 One of the first four macros will evaluate to a non-zero (true) value:
@@ -284,6 +510,7 @@ is returned and
 is set to indicate the error.
 .Pp
 If
+.Fn wait6 ,
 .Fn wait4 ,
 .Fn wait3 ,
 or
@@ -306,6 +533,18 @@ a value of -1
 is returned and
 .Va errno
 is set to indicate the error.
+.Pp
+If
+.Fn waitid
+returns because one or more processes have a state change to report,
+0 is returned.
+To indicate an error, -1 will be returned and
+.Dv errno
+set to an appropriate value.
+If
+.Dv WNOHANG
+was used, 0 can be returned indicating no error, but no processes
+may have changed state either, if si_signo and/or si_pid are zero.
 .Sh ERRORS
 The
 .Fn wait
@@ -335,6 +574,14 @@ The call was interrupted by a caught signal,
 or the signal did not have the
 .Dv SA_RESTART
 flag set.
+.It Bq Er EINVAL
+An invalid value was specified for
+.Fa options ,
+or
+.Fa idtype
+and
+.Fa id
+do not specify a valid set of processes.
 .El
 .Sh SEE ALSO
 .Xr _exit 2 ,
@@ -344,11 +591,13 @@ flag set.
 .Xr siginfo 3
 .Sh STANDARDS
 The
-.Fn wait
+.Fn wait ,
+.Fn waitpid ,
 and
-.Fn waitpid
+.Fn waitid
 functions are defined by POSIX;
-.Fn wait4
+.Fn wait6 ,
+.Fn wait4 ,
 and
 .Fn wait3
 are not specified by POSIX.
-- 
cgit v1.1


From 0ebb6520078c6cde55a20b445228f085b52b4857 Mon Sep 17 00:00:00 2001
From: kevlo <kevlo@FreeBSD.org>
Date: Wed, 14 Nov 2012 01:45:10 +0000
Subject: Document that sendfile(2) can fail with ENOBUFS.

Reviewed by:	glebius
---
 lib/libc/sys/sendfile.2 | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'lib')

diff --git a/lib/libc/sys/sendfile.2 b/lib/libc/sys/sendfile.2
index 9d1fba5..0e7dbe3 100644
--- a/lib/libc/sys/sendfile.2
+++ b/lib/libc/sys/sendfile.2
@@ -265,6 +265,8 @@ is negative.
 .It Bq Er EIO
 An error occurred while reading from
 .Fa fd .
+.It Bq Er ENOBUFS
+The system was unable to allocate an internal buffer.
 .It Bq Er ENOTCONN
 The
 .Fa s
-- 
cgit v1.1


From ff16940b26eab3cfb42489dc6e1c176415175b38 Mon Sep 17 00:00:00 2001
From: grog <grog@FreeBSD.org>
Date: Fri, 16 Nov 2012 01:41:42 +0000
Subject: Complete man page.

MFC after:	2 weeks
---
 lib/libc/gen/getbsize.3 | 38 +++++++++++++++++++++++++++-----------
 1 file changed, 27 insertions(+), 11 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/gen/getbsize.3 b/lib/libc/gen/getbsize.3
index 1407051..5a759da 100644
--- a/lib/libc/gen/getbsize.3
+++ b/lib/libc/gen/getbsize.3
@@ -28,12 +28,12 @@
 .\"     @(#)getbsize.3	8.1 (Berkeley) 6/4/93
 .\" $FreeBSD$
 .\"
-.Dd June 4, 1993
+.Dd November 16, 2012
 .Dt GETBSIZE 3
 .Os
 .Sh NAME
 .Nm getbsize
-.Nd get user block size
+.Nd get preferred block size
 .Sh LIBRARY
 .Lb libc
 .Sh SYNOPSIS
@@ -43,11 +43,31 @@
 .Sh DESCRIPTION
 The
 .Fn getbsize
-function determines the user's preferred block size based on the value of the
-.Dq BLOCKSIZE
-environment variable; see
-.Xr environ 7
-for details on its use and format.
+function returns a preferred block size for reporting by system utilities
+.Xr df 1 ,
+.Xr du 1 ,
+.Xr ls 1
+and
+.Xr systat 1 ,
+based on the value of the
+.En BLOCKSIZE
+environment variable.
+.En BLOCKSIZE
+may be specified directly in bytes, or in multiples of a kilobyte by
+specifying a number followed by ``K'' or ``k'', in multiples of a
+megabyte by specifying a number followed by ``M'' or ``m'' or in
+multiples of a gigabyte by specifying a number followed by ``G'' or
+``g''.
+Multiples must be integers.
+.Pp
+Valid values of
+.En BLOCKSIZE
+are 512 bytes to 1 gigabyte.
+Sizes less than 512 bytes are rounded up to 512 bytes, and sizes
+greater than 1 GB are rounded down to 1 GB.
+In each case
+.Fn getbsize
+produces a warning message.
 .Pp
 The
 .Fn getbsize
@@ -61,10 +81,6 @@ terminating null).
 The memory referenced by
 .Fa blocksizep
 is filled in with block size, in bytes.
-.Pp
-If the user's block size is unreasonable, a warning message is
-written to standard error and the returned information reflects
-a block size of 512 bytes.
 .Sh SEE ALSO
 .Xr df 1 ,
 .Xr du 1 ,
-- 
cgit v1.1


From 5c97761dcf450dbea3ccd3e3706c3f843d1ac27f Mon Sep 17 00:00:00 2001
From: kevlo <kevlo@FreeBSD.org>
Date: Fri, 16 Nov 2012 09:56:25 +0000
Subject: Document that rtprio(2) and rtprio_thread(2) can fail with EFAULT due
 to the invoked copyout(9).

Reviewed by:	davidxu
---
 lib/libc/sys/rtprio.2 | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/sys/rtprio.2 b/lib/libc/sys/rtprio.2
index ac78b50..f30ed22 100644
--- a/lib/libc/sys/rtprio.2
+++ b/lib/libc/sys/rtprio.2
@@ -145,13 +145,21 @@ Higher real/idle priority threads
 preempt lower real/idle priority threads.
 Threads of equal real/idle priority are run round-robin.
 .Sh RETURN VALUES
-.Rv -std rtprio
+.Rv -std rtprio rtprio_thread
 .Sh ERRORS
 The
 .Fn rtprio
-system call
+and
+.Fn rtprio_thread
+system calls
 will fail if:
 .Bl -tag -width Er
+.It Bq Er EFAULT
+The rtp pointer passed to
+.Fn rtprio
+or
+.Fn rtprio_thread
+was invalid.
 .It Bq Er EINVAL
 The specified
 .Fa prio
-- 
cgit v1.1


From 3421e21beeae4f238f92e3d9e59b6e63fe8bf971 Mon Sep 17 00:00:00 2001
From: joel <joel@FreeBSD.org>
Date: Fri, 16 Nov 2012 12:03:50 +0000
Subject: mdoc: Use the Ev macro for environmental variables.

---
 lib/libc/gen/getbsize.3 | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/gen/getbsize.3 b/lib/libc/gen/getbsize.3
index 5a759da..2caf5fe 100644
--- a/lib/libc/gen/getbsize.3
+++ b/lib/libc/gen/getbsize.3
@@ -50,9 +50,9 @@ function returns a preferred block size for reporting by system utilities
 and
 .Xr systat 1 ,
 based on the value of the
-.En BLOCKSIZE
+.Ev BLOCKSIZE
 environment variable.
-.En BLOCKSIZE
+.Ev BLOCKSIZE
 may be specified directly in bytes, or in multiples of a kilobyte by
 specifying a number followed by ``K'' or ``k'', in multiples of a
 megabyte by specifying a number followed by ``M'' or ``m'' or in
@@ -61,7 +61,7 @@ multiples of a gigabyte by specifying a number followed by ``G'' or
 Multiples must be integers.
 .Pp
 Valid values of
-.En BLOCKSIZE
+.Ev BLOCKSIZE
 are 512 bytes to 1 gigabyte.
 Sizes less than 512 bytes are rounded up to 512 bytes, and sizes
 greater than 1 GB are rounded down to 1 GB.
-- 
cgit v1.1


From f81cc6662509ddec4578b29f294375340c8ae404 Mon Sep 17 00:00:00 2001
From: des <des@FreeBSD.org>
Date: Fri, 16 Nov 2012 12:31:43 +0000
Subject: Fix weird indentation.

---
 lib/libfetch/http.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/libfetch/http.c b/lib/libfetch/http.c
index 00dd887..abf79a3 100644
--- a/lib/libfetch/http.c
+++ b/lib/libfetch/http.c
@@ -1752,11 +1752,11 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
 
 		/* get headers. http_next_header expects one line readahead */
 		if (fetch_getln(conn) == -1) {
-		    fetch_syserr();
-		    goto ouch;
+			fetch_syserr();
+			goto ouch;
 		}
 		do {
-		    switch ((h = http_next_header(conn, &headerbuf, &p))) {
+			switch ((h = http_next_header(conn, &headerbuf, &p))) {
 			case hdr_syserror:
 				fetch_syserr();
 				goto ouch;
@@ -1785,7 +1785,7 @@ http_request(struct url *URL, const char *op, struct url_stat *us,
 				    conn->err != HTTP_USE_PROXY) {
 					n = 1;
 					break;
-                                }
+				}
 				if (new)
 					free(new);
 				if (verbose)
-- 
cgit v1.1


From f2c2ff1baf52aee84f7261dd1c3b8687b02c1a66 Mon Sep 17 00:00:00 2001
From: kevlo <kevlo@FreeBSD.org>
Date: Fri, 16 Nov 2012 15:02:35 +0000
Subject: - the preferred way to write a NULL pointer constant is with NULL -
 whitespace nit

Reviewed by:	glebius
---
 lib/libc/net/getnetent.3   | 5 ++---
 lib/libc/net/getprotoent.3 | 5 ++---
 lib/libc/net/getservent.3  | 5 ++---
 3 files changed, 6 insertions(+), 9 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/net/getnetent.3 b/lib/libc/net/getnetent.3
index 2e9cd33..ec08b6b 100644
--- a/lib/libc/net/getnetent.3
+++ b/lib/libc/net/getnetent.3
@@ -73,7 +73,7 @@ The order of the lookups is controlled by the
 `networks' entry in
 .Xr nsswitch.conf 5 .
 .Bd -literal -offset indent
-struct	netent {
+struct netent {
 	char		*n_name;	/* official name of net */
 	char		**n_aliases;	/* alias list */
 	int		n_addrtype;	/* net number type */
@@ -142,8 +142,7 @@ Network numbers are supplied in host order.
 .It Pa /etc/resolv.conf
 .El
 .Sh DIAGNOSTICS
-Null pointer
-(0) returned on
+Null pointer returned on
 .Dv EOF
 or error.
 .Sh SEE ALSO
diff --git a/lib/libc/net/getprotoent.3 b/lib/libc/net/getprotoent.3
index 9f9e00d..565e038 100644
--- a/lib/libc/net/getprotoent.3
+++ b/lib/libc/net/getprotoent.3
@@ -65,7 +65,7 @@ containing the broken-out
 fields of a line in the network protocol data base,
 .Pa /etc/protocols .
 .Bd -literal -offset indent
-struct	protoent {
+struct protoent {
 	char	*p_name;	/* official name of protocol */
 	char	**p_aliases;	/* alias list */
 	int	p_proto;	/* protocol number */
@@ -117,8 +117,7 @@ or until
 .Dv EOF
 is encountered.
 .Sh RETURN VALUES
-Null pointer
-(0) returned on
+Null pointer returned on
 .Dv EOF
 or error.
 .Sh FILES
diff --git a/lib/libc/net/getservent.3 b/lib/libc/net/getservent.3
index 65d40bb..5f5a452 100644
--- a/lib/libc/net/getservent.3
+++ b/lib/libc/net/getservent.3
@@ -65,7 +65,7 @@ containing the broken-out
 fields of a line in the network services data base,
 .Pa /etc/services .
 .Bd -literal -offset indent
-struct	servent {
+struct servent {
 	char	*s_name;	/* official name of service */
 	char	**s_aliases;	/* alias list */
 	int	s_port;		/* port service resides at */
@@ -130,8 +130,7 @@ searches must also match the protocol.
 .It Pa /etc/services
 .El
 .Sh DIAGNOSTICS
-Null pointer
-(0) returned on
+Null pointer returned on
 .Dv EOF
 or error.
 .Sh SEE ALSO
-- 
cgit v1.1


From 55b42c566aa34fe926dd40004d1a8f3ad15ed7dd Mon Sep 17 00:00:00 2001
From: tuexen <tuexen@FreeBSD.org>
Date: Mon, 19 Nov 2012 19:19:04 +0000
Subject: Fix the handling of mapped IPv6 addresses in sctp_connectx().

MFC after: 3 days
---
 lib/libc/net/sctp_sys_calls.c | 1 -
 1 file changed, 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libc/net/sctp_sys_calls.c b/lib/libc/net/sctp_sys_calls.c
index 1e06f44..031cfd3 100644
--- a/lib/libc/net/sctp_sys_calls.c
+++ b/lib/libc/net/sctp_sys_calls.c
@@ -202,7 +202,6 @@ sctp_connectx(int sd, const struct sockaddr *addrs, int addrcnt,
 				return (-1);
 			}
 			if (IN6_IS_ADDR_V4MAPPED(&((struct sockaddr_in6 *)at)->sin6_addr)) {
-				len += sizeof(struct sockaddr_in);
 				in6_sin6_2_sin((struct sockaddr_in *)cpto, (struct sockaddr_in6 *)at);
 				cpto = ((caddr_t)cpto + sizeof(struct sockaddr_in));
 				len += sizeof(struct sockaddr_in);
-- 
cgit v1.1


From debae4f153b89e64c1376cb9fdb14cb6fb70fefc Mon Sep 17 00:00:00 2001
From: tuexen <tuexen@FreeBSD.org>
Date: Mon, 19 Nov 2012 19:26:19 +0000
Subject: Cleanup the code a bit, which improves the portability. MFC after: 1
 week

---
 lib/libc/net/sctp_sys_calls.c | 85 +++++++++++++++++++++----------------------
 1 file changed, 42 insertions(+), 43 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/net/sctp_sys_calls.c b/lib/libc/net/sctp_sys_calls.c
index 031cfd3..beedf94 100644
--- a/lib/libc/net/sctp_sys_calls.c
+++ b/lib/libc/net/sctp_sys_calls.c
@@ -188,15 +188,18 @@ sctp_connectx(int sd, const struct sockaddr *addrs, int addrcnt,
 	cpto = ((caddr_t)buf + sizeof(int));
 	/* validate all the addresses and get the size */
 	for (i = 0; i < addrcnt; i++) {
-		if (at->sa_family == AF_INET) {
+		switch (at->sa_family) {
+		case AF_INET:
 			if (at->sa_len != sizeof(struct sockaddr_in)) {
 				errno = EINVAL;
 				return (-1);
 			}
-			memcpy(cpto, at, at->sa_len);
-			cpto = ((caddr_t)cpto + at->sa_len);
-			len += at->sa_len;
-		} else if (at->sa_family == AF_INET6) {
+			memcpy(cpto, at, sizeof(struct sockaddr_in));
+			cpto = ((caddr_t)cpto + sizeof(struct sockaddr_in));
+			len += sizeof(struct sockaddr_in);
+			at = (struct sockaddr *)((caddr_t)at + sizeof(struct sockaddr_in));
+			break;
+		case AF_INET6:
 			if (at->sa_len != sizeof(struct sockaddr_in6)) {
 				errno = EINVAL;
 				return (-1);
@@ -206,11 +209,13 @@ sctp_connectx(int sd, const struct sockaddr *addrs, int addrcnt,
 				cpto = ((caddr_t)cpto + sizeof(struct sockaddr_in));
 				len += sizeof(struct sockaddr_in);
 			} else {
-				memcpy(cpto, at, at->sa_len);
-				cpto = ((caddr_t)cpto + at->sa_len);
-				len += at->sa_len;
+				memcpy(cpto, at, sizeof(struct sockaddr_in6));
+				cpto = ((caddr_t)cpto + sizeof(struct sockaddr_in6));
+				len += sizeof(struct sockaddr_in6);
 			}
-		} else {
+			at = (struct sockaddr *)((caddr_t)at + sizeof(struct sockaddr_in6));
+			break;
+		default:
 			errno = EINVAL;
 			return (-1);
 		}
@@ -219,7 +224,6 @@ sctp_connectx(int sd, const struct sockaddr *addrs, int addrcnt,
 			errno = E2BIG;
 			return (-1);
 		}
-		at = (struct sockaddr *)((caddr_t)at + at->sa_len);
 		cnt++;
 	}
 	/* do we have any? */
@@ -260,56 +264,57 @@ sctp_bindx(int sd, struct sockaddr *addrs, int addrcnt, int flags)
 		errno = EINVAL;
 		return (-1);
 	}
-	argsz = (sizeof(struct sockaddr_storage) +
-	    sizeof(struct sctp_getaddresses));
-	gaddrs = (struct sctp_getaddresses *)calloc(1, argsz);
-	if (gaddrs == NULL) {
-		errno = ENOMEM;
-		return (-1);
-	}
 	/* First pre-screen the addresses */
 	sa = addrs;
 	for (i = 0; i < addrcnt; i++) {
-		if (sa->sa_family == AF_INET) {
-			if (sa->sa_len != sizeof(struct sockaddr_in))
-				goto out_error;
+		switch (sa->sa_family) {
+		case AF_INET:
+			if (sa->sa_len != sizeof(struct sockaddr_in)) {
+				errno = EINVAL;
+				return (-1);
+			}
 			sin = (struct sockaddr_in *)sa;
 			if (sin->sin_port) {
 				/* non-zero port, check or save */
 				if (sport) {
 					/* Check against our port */
 					if (sport != sin->sin_port) {
-						goto out_error;
+						errno = EINVAL;
+						return (-1);
 					}
 				} else {
 					/* save off the port */
 					sport = sin->sin_port;
 				}
 			}
-		} else if (sa->sa_family == AF_INET6) {
-			if (sa->sa_len != sizeof(struct sockaddr_in6))
-				goto out_error;
+			break;
+		case AF_INET6:
+			if (sa->sa_len != sizeof(struct sockaddr_in6)) {
+				errno = EINVAL;
+				return (-1);
+			}
 			sin6 = (struct sockaddr_in6 *)sa;
 			if (sin6->sin6_port) {
 				/* non-zero port, check or save */
 				if (sport) {
 					/* Check against our port */
 					if (sport != sin6->sin6_port) {
-						goto out_error;
+						errno = EINVAL;
+						return (-1);
 					}
 				} else {
 					/* save off the port */
 					sport = sin6->sin6_port;
 				}
 			}
-		} else {
-			/* invalid address family specified */
-			goto out_error;
+			break;
+		default:
+			/* Invalid address family specified. */
+			errno = EINVAL;
+			return (-1);
 		}
-
 		sa = (struct sockaddr *)((caddr_t)sa + sa->sa_len);
 	}
-	sa = addrs;
 	/*
 	 * Now if there was a port mentioned, assure that the first address
 	 * has that port to make sure it fails or succeeds correctly.
@@ -318,20 +323,14 @@ sctp_bindx(int sd, struct sockaddr *addrs, int addrcnt, int flags)
 		sin = (struct sockaddr_in *)sa;
 		sin->sin_port = sport;
 	}
+	argsz = sizeof(struct sctp_getaddresses) +
+	    sizeof(struct sockaddr_storage);
+	if ((gaddrs = (struct sctp_getaddresses *)malloc(argsz)) == NULL) {
+		errno = ENOMEM;
+		return (-1);
+	}
+	sa = addrs;
 	for (i = 0; i < addrcnt; i++) {
-		if (sa->sa_family == AF_INET) {
-			if (sa->sa_len != sizeof(struct sockaddr_in))
-				goto out_error;
-		} else if (sa->sa_family == AF_INET6) {
-			if (sa->sa_len != sizeof(struct sockaddr_in6))
-				goto out_error;
-		} else {
-			/* invalid address family specified */
-	out_error:
-			free(gaddrs);
-			errno = EINVAL;
-			return (-1);
-		}
 		memset(gaddrs, 0, argsz);
 		gaddrs->sget_assoc_id = 0;
 		memcpy(gaddrs->addr, sa, sa->sa_len);
-- 
cgit v1.1


From 9acb3e40c2b5badc282bac6ad0196dbbb3f443c7 Mon Sep 17 00:00:00 2001
From: bapt <bapt@FreeBSD.org>
Date: Tue, 20 Nov 2012 07:22:07 +0000
Subject: change mode the group file to 0644 after a successfull rename(2)

---
 lib/libutil/gr_util.c | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libutil/gr_util.c b/lib/libutil/gr_util.c
index 6d96d5e..8d0490b 100644
--- a/lib/libutil/gr_util.c
+++ b/lib/libutil/gr_util.c
@@ -318,7 +318,14 @@ gr_copy(int ffd, int tfd, const struct group *gr, struct group *old_gr)
 int
 gr_mkdb(void)
 {
-	return (rename(tempname, group_file));
+	int ret;
+
+	ret = rename(tempname, group_file);
+
+	if (ret == 0)
+		chmod(group_file, 0644);
+
+	return (ret);
 }
 
 /*
-- 
cgit v1.1


From 2ed31d29a136b086906cadaedceedd82398b9e7b Mon Sep 17 00:00:00 2001
From: bapt <bapt@FreeBSD.org>
Date: Tue, 20 Nov 2012 14:03:09 +0000
Subject: only rename(2) after chmod(2) has succeed report error if chmod(2)
 fails

Reported by:	jh
---
 lib/libutil/gr_util.c | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

(limited to 'lib')

diff --git a/lib/libutil/gr_util.c b/lib/libutil/gr_util.c
index 8d0490b..be34395 100644
--- a/lib/libutil/gr_util.c
+++ b/lib/libutil/gr_util.c
@@ -318,14 +318,10 @@ gr_copy(int ffd, int tfd, const struct group *gr, struct group *old_gr)
 int
 gr_mkdb(void)
 {
-	int ret;
-
-	ret = rename(tempname, group_file);
-
-	if (ret == 0)
-		chmod(group_file, 0644);
+	if (chmod(tempname, 0644) != 0)
+		return (-1);
 
-	return (ret);
+	return (rename(tempname, group_file));
 }
 
 /*
-- 
cgit v1.1


From ec5aeeffd06a344918e75a6621a4dd36f5b1ccc5 Mon Sep 17 00:00:00 2001
From: kevlo <kevlo@FreeBSD.org>
Date: Fri, 23 Nov 2012 10:14:54 +0000
Subject: Document that getpeername(2) and getsockname(2) can fail with EINVAL.

Reviewed by:	glebius
---
 lib/libc/sys/getpeername.2 | 4 ++++
 lib/libc/sys/getsockname.2 | 4 ++++
 2 files changed, 8 insertions(+)

(limited to 'lib')

diff --git a/lib/libc/sys/getpeername.2 b/lib/libc/sys/getpeername.2
index 9037119..255c7a0 100644
--- a/lib/libc/sys/getpeername.2
+++ b/lib/libc/sys/getpeername.2
@@ -67,6 +67,10 @@ The argument
 is not a valid descriptor.
 .It Bq Er ECONNRESET
 The connection has been reset by the peer.
+.It Bq Er EINVAL
+The value of the
+.Fa namelen
+argument is not valid.
 .It Bq Er ENOTSOCK
 The argument
 .Fa s
diff --git a/lib/libc/sys/getsockname.2 b/lib/libc/sys/getsockname.2
index f1537c7..1de0257 100644
--- a/lib/libc/sys/getsockname.2
+++ b/lib/libc/sys/getsockname.2
@@ -66,6 +66,10 @@ The argument
 is not a valid descriptor.
 .It Bq Er ECONNRESET
 The connection has been reset by the peer.
+.It Bq Er EINVAL
+The value of the
+.Fa namelen
+argument is not valid.
 .It Bq Er ENOTSOCK
 The argument
 .Fa s
-- 
cgit v1.1


From 04619555a48eeacc9a13b826b57fa7f2932aae7c Mon Sep 17 00:00:00 2001
From: jilles <jilles@FreeBSD.org>
Date: Fri, 30 Nov 2012 23:51:33 +0000
Subject: libc: Allow setting close-on-exec in fopen/freopen/fdopen.

This commit adds a new mode option 'e' that must follow any 'b', '+' and/or
'x' options. C11 is clear about the 'x' needing to follow 'b' and/or '+' and
that is what we implement; therefore, require a strict position for 'e' as
well.

For freopen() with a non-NULL path argument and fopen(), the close-on-exec
flag is set iff the 'e' mode option is specified. For freopen() with a NULL
path argument and fdopen(), the close-on-exec flag is turned on if the 'e'
mode option is specified and remains unchanged otherwise.

Although the same behaviour for fopen() can be obtained by open(O_CLOEXEC)
and fdopen(), this needlessly complicates the calling code.

Apart from the ordering requirement, the new option matches glibc.

PR:		kern/169320
---
 lib/libc/stdio/fdopen.c  |  6 ++++++
 lib/libc/stdio/flags.c   |  5 +++++
 lib/libc/stdio/fopen.3   | 19 ++++++++++++++++++-
 lib/libc/stdio/freopen.c |  5 ++++-
 4 files changed, 33 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/stdio/fdopen.c b/lib/libc/stdio/fdopen.c
index 26e2cd7..8fc90a4 100644
--- a/lib/libc/stdio/fdopen.c
+++ b/lib/libc/stdio/fdopen.c
@@ -80,6 +80,12 @@ fdopen(fd, mode)
 
 	if ((fp = __sfp()) == NULL)
 		return (NULL);
+
+	if ((oflags & O_CLOEXEC) && _fcntl(fd, F_SETFD, FD_CLOEXEC) == -1) {
+		fp->_flags = 0;
+		return (NULL);
+	}
+
 	fp->_flags = flags;
 	/*
 	 * If opened for appending, but underlying descriptor does not have
diff --git a/lib/libc/stdio/flags.c b/lib/libc/stdio/flags.c
index 0b6b075..e445fed 100644
--- a/lib/libc/stdio/flags.c
+++ b/lib/libc/stdio/flags.c
@@ -97,6 +97,7 @@ __sflags(mode, optr)
 
 	/* 'x' means exclusive (fail if the file exists) */
 	if (*mode == 'x') {
+		mode++;
 		if (m == O_RDONLY) {
 			errno = EINVAL;
 			return (0);
@@ -104,6 +105,10 @@ __sflags(mode, optr)
 		o |= O_EXCL;
 	}
 
+	/* set close-on-exec */
+	if (*mode == 'e')
+		o |= O_CLOEXEC;
+
 	*optr = m | o;
 	return (ret);
 }
diff --git a/lib/libc/stdio/fopen.3 b/lib/libc/stdio/fopen.3
index 08438e7..da89ed4 100644
--- a/lib/libc/stdio/fopen.3
+++ b/lib/libc/stdio/fopen.3
@@ -32,7 +32,7 @@
 .\"     @(#)fopen.3	8.1 (Berkeley) 6/4/93
 .\" $FreeBSD$
 .\"
-.Dd October 17, 2011
+.Dd November 30, 2012
 .Dt FOPEN 3
 .Os
 .Sh NAME
@@ -97,6 +97,14 @@ or
 causes the
 .Fn fopen
 call to fail if the file already exists.
+An optional
+.Dq Li e
+following the above
+causes the
+.Fn fopen
+call to set the
+.Dv FD_CLOEXEC
+flag on the underlying file descriptor.
 .Pp
 The
 .Fa mode
@@ -144,6 +152,11 @@ of the stream must be compatible with the mode of the file descriptor.
 The
 .Dq Li x
 mode option is ignored.
+If the
+.Dq Li e
+mode option is present, the
+.Dv FD_CLOEXEC
+flag is set, otherwise it remains unchanged.
 When the stream is closed via
 .Xr fclose 3 ,
 .Fa fildes
@@ -277,3 +290,7 @@ The
 function
 conforms to
 .St -p1003.1-88 .
+The
+.Dq Li e
+mode option does not conform to any standard
+but is also supported by glibc.
diff --git a/lib/libc/stdio/freopen.c b/lib/libc/stdio/freopen.c
index be7bc8a..8d1ec91 100644
--- a/lib/libc/stdio/freopen.c
+++ b/lib/libc/stdio/freopen.c
@@ -118,6 +118,8 @@ freopen(file, mode, fp)
 			(void) ftruncate(fp->_file, (off_t)0);
 		if (!(oflags & O_APPEND))
 			(void) _sseek(fp, (fpos_t)0, SEEK_SET);
+		if (oflags & O_CLOEXEC)
+			(void) _fcntl(fp->_file, F_SETFD, FD_CLOEXEC);
 		f = fp->_file;
 		isopen = 0;
 		wantfd = -1;
@@ -194,7 +196,8 @@ finish:
 	 * assume stderr is always fd STDERR_FILENO, even if being freopen'd.
 	 */
 	if (wantfd >= 0) {
-		if (_dup2(f, wantfd) >= 0) {
+		if ((oflags & O_CLOEXEC ? _fcntl(f, F_DUP2FD_CLOEXEC, wantfd) :
+		    _dup2(f, wantfd)) >= 0) {
 			(void)_close(f);
 			f = wantfd;
 		} else
-- 
cgit v1.1


From 242dabb510e96151dfa1c14d7e2ffd1db8fc882a Mon Sep 17 00:00:00 2001
From: eadler <eadler@FreeBSD.org>
Date: Sat, 1 Dec 2012 15:25:41 +0000
Subject: The getline function returns the number of characters read, not
 written. Use clearer text for this.

PR:		docs/174023
Submitted by:	Paul Procacci <pprocacci@gmail.com>
Approved by:	bcr (mentor)
MFC after:	1 week
---
 lib/libc/stdio/getline.3 | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/stdio/getline.3 b/lib/libc/stdio/getline.3
index 6061bfc..2161999 100644
--- a/lib/libc/stdio/getline.3
+++ b/lib/libc/stdio/getline.3
@@ -24,7 +24,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd November 30, 2010
+.Dd November 30, 2012
 .Dt GETLINE 3
 .Os
 .Sh NAME
@@ -76,7 +76,7 @@ The
 .Fn getdelim
 and
 .Fn getline
-functions return the number of characters written, excluding the
+functions return the number of characters stored in the buffer, excluding the
 terminating
 .Dv NUL
 character.
-- 
cgit v1.1


From 5192abbac636f4d423299ad2010b603072124621 Mon Sep 17 00:00:00 2001
From: marcel <marcel@FreeBSD.org>
Date: Sat, 1 Dec 2012 17:44:06 +0000
Subject: In globextend() when the pathv vector cannot be (re-)allocated, don't
 free and clear the gl_pathv pointer in the glob_t structure. Such breaks the
 invariant of the glob_t structure, as stated in the comment right in front of
 the globextend() function. If gl_pathv was non-NULL, then gl_pathc was > 0.
 Making gl_pathv a NULL pointer without also setting gl_pathc to 0 is wrong.

Since we otherwise don't free the memory associated with a glob_t in
error cases, it's unlikely that this change will cause a memory leak
that wasn't already there to begin with. Callers of glob(3) must
call globfree(3) irrespective of whether glob(3) returned an error
or not.
---
 lib/libc/gen/glob.c | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/gen/glob.c b/lib/libc/gen/glob.c
index 211b535..07eeaa8 100644
--- a/lib/libc/gen/glob.c
+++ b/lib/libc/gen/glob.c
@@ -718,13 +718,8 @@ globextend(const Char *path, glob_t *pglob, size_t *limit)
 	pathv = pglob->gl_pathv ?
 		    realloc((char *)pglob->gl_pathv, newsize) :
 		    malloc(newsize);
-	if (pathv == NULL) {
-		if (pglob->gl_pathv) {
-			free(pglob->gl_pathv);
-			pglob->gl_pathv = NULL;
-		}
+	if (pathv == NULL)
 		return (GLOB_NOSPACE);
-	}
 
 	if (pglob->gl_pathv == NULL && pglob->gl_offs > 0) {
 		/* first time around -- clear initial gl_offs items */
-- 
cgit v1.1


From e0cbca43aafc47c5c47db73820ede29110da0d0d Mon Sep 17 00:00:00 2001
From: marcel <marcel@FreeBSD.org>
Date: Sat, 1 Dec 2012 17:50:39 +0000
Subject: In globextend(), take advantage of the fact that realloc(NULL, size)
 is equivalent to malloc(size). This eliminates the conditional expression
 used for calling either realloc() or malloc() when realloc() will do all the
 time.

---
 lib/libc/gen/glob.c | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/gen/glob.c b/lib/libc/gen/glob.c
index 07eeaa8..3c5020c 100644
--- a/lib/libc/gen/glob.c
+++ b/lib/libc/gen/glob.c
@@ -715,9 +715,8 @@ globextend(const Char *path, glob_t *pglob, size_t *limit)
 	}
 
 	newsize = sizeof(*pathv) * (2 + pglob->gl_pathc + pglob->gl_offs);
-	pathv = pglob->gl_pathv ?
-		    realloc((char *)pglob->gl_pathv, newsize) :
-		    malloc(newsize);
+	/* realloc(NULL, newsize) is equivalent to malloc(newsize). */
+	pathv = realloc((void *)pglob->gl_pathv, newsize);
 	if (pathv == NULL)
 		return (GLOB_NOSPACE);
 
-- 
cgit v1.1


From aee35ecd72729f8d859eba45444833f5bb23d581 Mon Sep 17 00:00:00 2001
From: marcel <marcel@FreeBSD.org>
Date: Sat, 1 Dec 2012 21:26:46 +0000
Subject: Protect against DoS attacks, such as being described in
 CVE-2010-2632. The changes were derived from what has been committed to
 NetBSD, with modifications. These are: 1.  Preserve the existsing GLOB_LIMIT
 behaviour by including the number     of matches to the set of parameters to
 limit. 2.  Change some of the limits to avoid impacting normal use cases:    
 GLOB_LIMIT_STRING - change from 65536 to ARG_MAX so that glob(3) 	can
 still provide a full command line of expanded names.     GLOB_LIMIT_STAT -
 change from 128 to 1024 for no other reason than 	that 128 feels too low
 (it's not a limit that impacts the 	behaviour of the test program listed
 in CVE-2010-2632).     GLOB_LIMIT_PATH - change from 1024 to 65536 so that
 glob(3) can 	still provide a fill command line of expanded names. 3. 
 Protect against buffer overruns when we hit the GLOB_LIMIT_STAT or    
 GLOB_LIMIT_READDIR limits. We append SEP and EOS to pathend in     those
 cases. Return GLOB_ABORTED instead of GLOB_NOSPACE when we     would
 otherwise overrun the buffer.

This change also modifies the existing behaviour of glob(3) in case
GLOB_LIMIT is specifies by limiting the *new* matches and not all
matches. This is an important distinction when GLOB_APPEND is set or
when the caller uses a non-zero gl_offs. Previously pre-existing
matches or the value of gl_offs would be counted in the number of
matches even though the man page states that glob(3) would return
GLOB_NOSPACE when gl_matchc or more matches were found.

The limits that cannot be circumvented are GLOB_LIMIT_STRING and
GLOB_LIMIT_PATH all others can be crossed by simply calling glob(3)
again and with GLOB_APPEND set.

The entire description above applies only when GLOB_LIMIT has been
specified of course. No limits apply when this flag isn't set!

Obtained from: Juniper Networks, Inc
---
 lib/libc/gen/glob.c | 101 ++++++++++++++++++++++++++++++++++++++++------------
 1 file changed, 79 insertions(+), 22 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/gen/glob.c b/lib/libc/gen/glob.c
index 3c5020c..832dc8d 100644
--- a/lib/libc/gen/glob.c
+++ b/lib/libc/gen/glob.c
@@ -94,6 +94,25 @@ __FBSDID("$FreeBSD$");
 
 #include "collate.h"
 
+/*
+ * glob(3) expansion limits. Stop the expansion if any of these limits
+ * is reached. This caps the runtime in the face of DoS attacks. See
+ * also CVE-2010-2632
+ */
+#define	GLOB_LIMIT_BRACE	128	/* number of brace calls */
+#define	GLOB_LIMIT_PATH		65536	/* number of path elements */
+#define	GLOB_LIMIT_READDIR	16384	/* number of readdirs */
+#define	GLOB_LIMIT_STAT		1024	/* number of stat system calls */
+#define	GLOB_LIMIT_STRING	ARG_MAX	/* maximum total size for paths */
+
+struct glob_limit {
+	size_t	l_brace_cnt;
+	size_t	l_path_lim;
+	size_t	l_readdir_cnt;	
+	size_t	l_stat_cnt;	
+	size_t	l_string_cnt;
+};
+
 #define	DOLLAR		'$'
 #define	DOT		'.'
 #define	EOS		'\0'
@@ -153,15 +172,18 @@ static const Char *g_strchr(const Char *, wchar_t);
 static Char	*g_strcat(Char *, const Char *);
 #endif
 static int	 g_stat(Char *, struct stat *, glob_t *);
-static int	 glob0(const Char *, glob_t *, size_t *);
-static int	 glob1(Char *, glob_t *, size_t *);
-static int	 glob2(Char *, Char *, Char *, Char *, glob_t *, size_t *);
-static int	 glob3(Char *, Char *, Char *, Char *, Char *, glob_t *, size_t *);
-static int	 globextend(const Char *, glob_t *, size_t *);
-static const Char *	
+static int	 glob0(const Char *, glob_t *, struct glob_limit *);
+static int	 glob1(Char *, glob_t *, struct glob_limit *);
+static int	 glob2(Char *, Char *, Char *, Char *, glob_t *,
+    struct glob_limit *);
+static int	 glob3(Char *, Char *, Char *, Char *, Char *, glob_t *,
+    struct glob_limit *);
+static int	 globextend(const Char *, glob_t *, struct glob_limit *);
+static const Char *
 		 globtilde(const Char *, Char *, size_t, glob_t *);
-static int	 globexp1(const Char *, glob_t *, size_t *);
-static int	 globexp2(const Char *, const Char *, glob_t *, int *, size_t *);
+static int	 globexp1(const Char *, glob_t *, struct glob_limit *);
+static int	 globexp2(const Char *, const Char *, glob_t *, int *,
+    struct glob_limit *);
 static int	 match(Char *, Char *, Char *);
 #ifdef DEBUG
 static void	 qprintf(const char *, Char *);
@@ -171,8 +193,8 @@ int
 glob(const char * __restrict pattern, int flags,
 	 int (*errfunc)(const char *, int), glob_t * __restrict pglob)
 {
+	struct glob_limit limit = { 0, 0, 0, 0, 0 };
 	const char *patnext;
-	size_t limit;
 	Char *bufnext, *bufend, patbuf[MAXPATHLEN], prot;
 	mbstate_t mbs;
 	wchar_t wc;
@@ -186,11 +208,10 @@ glob(const char * __restrict pattern, int flags,
 			pglob->gl_offs = 0;
 	}
 	if (flags & GLOB_LIMIT) {
-		limit = pglob->gl_matchc;
-		if (limit == 0)
-			limit = ARG_MAX;
-	} else
-		limit = 0;
+		limit.l_path_lim = pglob->gl_matchc;
+		if (limit.l_path_lim == 0)
+			limit.l_path_lim = GLOB_LIMIT_PATH;
+	}
 	pglob->gl_flags = flags & ~GLOB_MAGCHAR;
 	pglob->gl_errfunc = errfunc;
 	pglob->gl_matchc = 0;
@@ -243,11 +264,17 @@ glob(const char * __restrict pattern, int flags,
  * characters
  */
 static int
-globexp1(const Char *pattern, glob_t *pglob, size_t *limit)
+globexp1(const Char *pattern, glob_t *pglob, struct glob_limit *limit)
 {
 	const Char* ptr = pattern;
 	int rv;
 
+	if ((pglob->gl_flags & GLOB_LIMIT) &&
+	    limit->l_brace_cnt++ >= GLOB_LIMIT_BRACE) {
+		errno = 0;
+		return (GLOB_NOSPACE);
+	}
+
 	/* Protect a single {}, for find(1), like csh */
 	if (pattern[0] == LBRACE && pattern[1] == RBRACE && pattern[2] == EOS)
 		return glob0(pattern, pglob, limit);
@@ -266,7 +293,8 @@ globexp1(const Char *pattern, glob_t *pglob, size_t *limit)
  * If it fails then it tries to glob the rest of the pattern and returns.
  */
 static int
-globexp2(const Char *ptr, const Char *pattern, glob_t *pglob, int *rv, size_t *limit)
+globexp2(const Char *ptr, const Char *pattern, glob_t *pglob, int *rv,
+    struct glob_limit *limit)
 {
 	int     i;
 	Char   *lm, *ls;
@@ -436,7 +464,7 @@ globtilde(const Char *pattern, Char *patbuf, size_t patbuf_len, glob_t *pglob)
  * if things went well, nonzero if errors occurred.
  */
 static int
-glob0(const Char *pattern, glob_t *pglob, size_t *limit)
+glob0(const Char *pattern, glob_t *pglob, struct glob_limit *limit)
 {
 	const Char *qpatnext;
 	int err;
@@ -529,7 +557,7 @@ compare(const void *p, const void *q)
 }
 
 static int
-glob1(Char *pattern, glob_t *pglob, size_t *limit)
+glob1(Char *pattern, glob_t *pglob, struct glob_limit *limit)
 {
 	Char pathbuf[MAXPATHLEN];
 
@@ -547,7 +575,7 @@ glob1(Char *pattern, glob_t *pglob, size_t *limit)
  */
 static int
 glob2(Char *pathbuf, Char *pathend, Char *pathend_last, Char *pattern,
-      glob_t *pglob, size_t *limit)
+      glob_t *pglob, struct glob_limit *limit)
 {
 	struct stat sb;
 	Char *p, *q;
@@ -563,6 +591,15 @@ glob2(Char *pathbuf, Char *pathend, Char *pathend_last, Char *pattern,
 			if (g_lstat(pathbuf, &sb, pglob))
 				return (0);
 
+			if ((pglob->gl_flags & GLOB_LIMIT) &&
+			    limit->l_stat_cnt++ >= GLOB_LIMIT_STAT) {
+				errno = 0;
+				if (pathend + 1 > pathend_last)
+					return (GLOB_ABORTED);
+				*pathend++ = SEP;
+				*pathend = EOS;
+				return (GLOB_NOSPACE);
+			}
 			if (((pglob->gl_flags & GLOB_MARK) &&
 			    pathend[-1] != SEP) && (S_ISDIR(sb.st_mode)
 			    || (S_ISLNK(sb.st_mode) &&
@@ -606,7 +643,7 @@ glob2(Char *pathbuf, Char *pathend, Char *pathend_last, Char *pattern,
 static int
 glob3(Char *pathbuf, Char *pathend, Char *pathend_last,
       Char *pattern, Char *restpattern,
-      glob_t *pglob, size_t *limit)
+      glob_t *pglob, struct glob_limit *limit)
 {
 	struct dirent *dp;
 	DIR *dirp;
@@ -652,6 +689,19 @@ glob3(Char *pathbuf, Char *pathend, Char *pathend_last,
 		size_t clen;
 		mbstate_t mbs;
 
+		if ((pglob->gl_flags & GLOB_LIMIT) &&
+		    limit->l_readdir_cnt++ >= GLOB_LIMIT_READDIR) {
+			errno = 0;
+			if (pathend + 1 > pathend_last)
+				err = GLOB_ABORTED;
+			else {
+				*pathend++ = SEP;
+				*pathend = EOS;
+				err = GLOB_NOSPACE;
+			}
+			break;
+		}
+
 		/* Initial DOT must be matched literally. */
 		if (dp->d_name[0] == DOT && *pattern != DOT)
 			continue;
@@ -702,14 +752,15 @@ glob3(Char *pathbuf, Char *pathend, Char *pathend_last,
  *	gl_pathv points to (gl_offs + gl_pathc + 1) items.
  */
 static int
-globextend(const Char *path, glob_t *pglob, size_t *limit)
+globextend(const Char *path, glob_t *pglob, struct glob_limit *limit)
 {
 	char **pathv;
 	size_t i, newsize, len;
 	char *copy;
 	const Char *p;
 
-	if (*limit && pglob->gl_pathc > *limit) {
+	if ((pglob->gl_flags & GLOB_LIMIT) &&
+	    pglob->gl_matchc > limit->l_path_lim) {
 		errno = 0;
 		return (GLOB_NOSPACE);
 	}
@@ -731,6 +782,12 @@ globextend(const Char *path, glob_t *pglob, size_t *limit)
 	for (p = path; *p++;)
 		continue;
 	len = MB_CUR_MAX * (size_t)(p - path);	/* XXX overallocation */
+	limit->l_string_cnt += len;
+	if ((pglob->gl_flags & GLOB_LIMIT) &&
+	    limit->l_string_cnt >= GLOB_LIMIT_STRING) {
+		errno = 0;
+		return (GLOB_NOSPACE);
+	}
 	if ((copy = malloc(len)) != NULL) {
 		if (g_Ctoc(path, copy, len)) {
 			free(copy);
-- 
cgit v1.1


From 6de2c08bc400b4aca9fb46684e8bdb56eed9b09f Mon Sep 17 00:00:00 2001
From: dim <dim@FreeBSD.org>
Date: Sun, 2 Dec 2012 13:10:19 +0000
Subject: Vendor import of llvm release_32 branch r168974 (effectively, 3.2
 RC2): http://llvm.org/svn/llvm-project/llvm/branches/release_32@168974

---
 lib/Analysis/AliasAnalysis.cpp                     |   10 +-
 lib/Analysis/AliasSetTracker.cpp                   |    6 +-
 lib/Analysis/Analysis.cpp                          |    4 +-
 lib/Analysis/BasicAliasAnalysis.cpp                |  170 +-
 lib/Analysis/BranchProbabilityInfo.cpp             |  134 +-
 lib/Analysis/CMakeLists.txt                        |    5 +-
 lib/Analysis/CaptureTracking.cpp                   |    4 +-
 lib/Analysis/CodeMetrics.cpp                       |   10 +-
 lib/Analysis/ConstantFolding.cpp                   |  265 +-
 lib/Analysis/CostModel.cpp                         |  193 +
 lib/Analysis/DependenceAnalysis.cpp                | 3786 ++++++++++++++++++++
 lib/Analysis/DominanceFrontier.cpp                 |    2 +
 lib/Analysis/IPA/CallGraph.cpp                     |   13 +-
 lib/Analysis/IPA/GlobalsModRef.cpp                 |    6 +-
 lib/Analysis/IVUsers.cpp                           |    6 +-
 lib/Analysis/InlineCost.cpp                        |   95 +-
 lib/Analysis/InstructionSimplify.cpp               |   89 +-
 lib/Analysis/LazyValueInfo.cpp                     |   52 +-
 lib/Analysis/Lint.cpp                              |   56 +-
 lib/Analysis/Loads.cpp                             |    8 +-
 lib/Analysis/LoopDependenceAnalysis.cpp            |  362 --
 lib/Analysis/LoopInfo.cpp                          |   11 +-
 lib/Analysis/MemoryBuiltins.cpp                    |  184 +-
 lib/Analysis/MemoryDependenceAnalysis.cpp          |   30 +-
 lib/Analysis/NoAliasAnalysis.cpp                   |    4 +-
 lib/Analysis/PHITransAddr.cpp                      |    2 +
 lib/Analysis/ProfileDataLoader.cpp                 |  155 +
 lib/Analysis/ProfileDataLoaderPass.cpp             |  188 +
 lib/Analysis/ProfileEstimatorPass.cpp              |    2 +-
 lib/Analysis/ProfileInfo.cpp                       |   26 -
 lib/Analysis/RegionInfo.cpp                        |   26 +-
 lib/Analysis/RegionPass.cpp                        |    5 +-
 lib/Analysis/ScalarEvolution.cpp                   |  116 +-
 lib/Analysis/ScalarEvolutionExpander.cpp           |   23 +-
 lib/Analysis/Trace.cpp                             |    2 +
 lib/Analysis/ValueTracking.cpp                     |   61 +-
 lib/Archive/ArchiveInternals.h                     |    2 +-
 lib/Archive/ArchiveReader.cpp                      |    4 +-
 lib/AsmParser/LLLexer.cpp                          |    9 +-
 lib/AsmParser/LLParser.cpp                         |  257 +-
 lib/AsmParser/LLParser.h                           |    2 +-
 lib/AsmParser/LLToken.h                            |   11 +-
 lib/Bitcode/Reader/BitcodeReader.cpp               |  159 +-
 lib/Bitcode/Reader/BitcodeReader.h                 |   67 +-
 lib/Bitcode/Writer/BitcodeWriter.cpp               |  130 +-
 lib/Bitcode/Writer/ValueEnumerator.h               |    6 +-
 lib/CodeGen/AggressiveAntiDepBreaker.cpp           |    4 +-
 lib/CodeGen/AllocationOrder.cpp                    |    5 +-
 lib/CodeGen/Analysis.cpp                           |   18 +-
 lib/CodeGen/AsmPrinter/ARMException.cpp            |    2 +-
 lib/CodeGen/AsmPrinter/AsmPrinter.cpp              |  171 +-
 lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp         |    4 +-
 lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp     |  234 +-
 lib/CodeGen/AsmPrinter/DIE.cpp                     |   18 +-
 lib/CodeGen/AsmPrinter/DIE.h                       |    8 -
 lib/CodeGen/AsmPrinter/DwarfAccelTable.cpp         |    4 +-
 lib/CodeGen/AsmPrinter/DwarfAccelTable.h           |    4 +-
 lib/CodeGen/AsmPrinter/DwarfCFIException.cpp       |    2 +-
 lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp        |   67 +-
 lib/CodeGen/AsmPrinter/DwarfCompileUnit.h          |    7 +-
 lib/CodeGen/AsmPrinter/DwarfDebug.cpp              |  203 +-
 lib/CodeGen/AsmPrinter/DwarfDebug.h                |   27 +-
 lib/CodeGen/AsmPrinter/DwarfException.cpp          |    4 +-
 lib/CodeGen/AsmPrinter/DwarfException.h            |   40 +-
 lib/CodeGen/AsmPrinter/OcamlGCPrinter.cpp          |    4 +-
 lib/CodeGen/AsmPrinter/Win64Exception.cpp          |    2 +-
 lib/CodeGen/BranchFolding.cpp                      |   25 +-
 lib/CodeGen/CMakeLists.txt                         |    3 +
 lib/CodeGen/CalcSpillWeights.cpp                   |    6 +-
 lib/CodeGen/CallingConvLower.cpp                   |    4 +-
 lib/CodeGen/CodeGen.cpp                            |    2 +
 lib/CodeGen/CodePlacementOpt.cpp                   |    2 +-
 lib/CodeGen/CriticalAntiDepBreaker.cpp             |    2 +-
 lib/CodeGen/DeadMachineInstructionElim.cpp         |    8 +-
 lib/CodeGen/EarlyIfConversion.cpp                  |   10 +-
 lib/CodeGen/ExecutionDepsFix.cpp                   |   11 +-
 lib/CodeGen/ExpandPostRAPseudos.cpp                |    4 +-
 lib/CodeGen/GCStrategy.cpp                         |   14 +-
 lib/CodeGen/IfConversion.cpp                       |    6 +-
 lib/CodeGen/InlineSpiller.cpp                      |    6 +-
 lib/CodeGen/IntrinsicLowering.cpp                  |    8 +-
 lib/CodeGen/LLVMTargetMachine.cpp                  |    6 +-
 lib/CodeGen/LiveDebugVariables.cpp                 |    3 +-
 lib/CodeGen/LiveInterval.cpp                       |   97 +-
 lib/CodeGen/LiveIntervalAnalysis.cpp               |  831 ++---
 lib/CodeGen/LiveIntervalUnion.h                    |    4 +-
 lib/CodeGen/LiveRangeCalc.cpp                      |    6 +-
 lib/CodeGen/LiveRangeEdit.cpp                      |   11 +-
 lib/CodeGen/LiveRegMatrix.cpp                      |    4 +-
 lib/CodeGen/LiveRegMatrix.h                        |    2 +-
 lib/CodeGen/LiveStackAnalysis.cpp                  |    5 +-
 lib/CodeGen/LiveVariables.cpp                      |   42 +-
 lib/CodeGen/MachineBasicBlock.cpp                  |   90 +-
 lib/CodeGen/MachineBlockPlacement.cpp              |    6 +-
 lib/CodeGen/MachineBranchProbabilityInfo.cpp       |   20 +-
 lib/CodeGen/MachineCSE.cpp                         |   70 +-
 lib/CodeGen/MachineCopyPropagation.cpp             |   13 +-
 lib/CodeGen/MachineFunction.cpp                    |   49 +-
 lib/CodeGen/MachineFunctionPrinterPass.cpp         |    2 +-
 lib/CodeGen/MachineInstr.cpp                       |  337 +-
 lib/CodeGen/MachineInstrBundle.cpp                 |   62 +-
 lib/CodeGen/MachineLICM.cpp                        |    2 +-
 lib/CodeGen/MachineLoopInfo.cpp                    |    2 +
 lib/CodeGen/MachineModuleInfo.cpp                  |    2 +-
 lib/CodeGen/MachineModuleInfoImpls.cpp             |    4 +-
 lib/CodeGen/MachinePostDominators.cpp              |   55 +
 lib/CodeGen/MachineRegisterInfo.cpp                |   18 +-
 lib/CodeGen/MachineScheduler.cpp                   | 1458 +++++---
 lib/CodeGen/MachineSink.cpp                        |    2 -
 lib/CodeGen/MachineTraceMetrics.cpp                |   74 +-
 lib/CodeGen/MachineTraceMetrics.h                  |   13 +-
 lib/CodeGen/MachineVerifier.cpp                    |  157 +-
 lib/CodeGen/Passes.cpp                             |   17 +-
 lib/CodeGen/PeepholeOptimizer.cpp                  |    5 +
 lib/CodeGen/PostRASchedulerList.cpp                |    9 +-
 lib/CodeGen/ProcessImplicitDefs.cpp                |    3 +-
 lib/CodeGen/PrologEpilogInserter.cpp               |    8 +-
 lib/CodeGen/RegAllocBasic.cpp                      |    3 +-
 lib/CodeGen/RegAllocFast.cpp                       |   73 +-
 lib/CodeGen/RegAllocGreedy.cpp                     |    8 +-
 lib/CodeGen/RegAllocPBQP.cpp                       |   17 +-
 lib/CodeGen/RegisterClassInfo.cpp                  |   10 +-
 lib/CodeGen/RegisterCoalescer.cpp                  | 1077 ++++--
 lib/CodeGen/RegisterCoalescer.h                    |    7 +
 lib/CodeGen/RegisterPressure.cpp                   |   35 +-
 lib/CodeGen/RegisterScavenging.cpp                 |    7 +-
 lib/CodeGen/ScheduleDAG.cpp                        |    2 +
 lib/CodeGen/ScheduleDAGInstrs.cpp                  |  346 +-
 lib/CodeGen/ScheduleDAGPrinter.cpp                 |    3 +-
 lib/CodeGen/ScoreboardHazardRecognizer.cpp         |    2 +
 lib/CodeGen/SelectionDAG/DAGCombiner.cpp           | 1190 +++++-
 lib/CodeGen/SelectionDAG/FastISel.cpp              |    4 +-
 lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp  |    8 +-
 lib/CodeGen/SelectionDAG/InstrEmitter.cpp          |   55 +-
 lib/CodeGen/SelectionDAG/LegalizeDAG.cpp           |  147 +-
 lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp    |   50 +-
 lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp  |   43 +-
 lib/CodeGen/SelectionDAG/LegalizeTypes.cpp         |    2 +-
 lib/CodeGen/SelectionDAG/LegalizeTypes.h           |    3 +-
 lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp  |   48 +-
 lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp     |   73 +-
 lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp   |   27 +-
 lib/CodeGen/SelectionDAG/SDNodeOrdering.h          |    4 +-
 lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp       |  182 +-
 lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp     |   44 +-
 lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp    |   18 +-
 lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h      |    9 +-
 lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp       |    2 +-
 lib/CodeGen/SelectionDAG/SelectionDAG.cpp          |  284 +-
 lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp   |  372 +-
 lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h     |   19 +-
 lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp    |   11 +-
 lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp      |   31 +-
 lib/CodeGen/SelectionDAG/SelectionDAGPrinter.cpp   |    7 +-
 lib/CodeGen/SelectionDAG/TargetLowering.cpp        |   43 +-
 .../SelectionDAG/TargetSelectionDAGInfo.cpp        |    2 +-
 lib/CodeGen/ShrinkWrapping.cpp                     |   22 +-
 lib/CodeGen/SjLjEHPrepare.cpp                      |   86 +-
 lib/CodeGen/SlotIndexes.cpp                        |    4 +
 lib/CodeGen/SplitKit.cpp                           |    2 +
 lib/CodeGen/StackColoring.cpp                      |  783 ++++
 lib/CodeGen/StackProtector.cpp                     |   69 +-
 lib/CodeGen/StackSlotColoring.cpp                  |    6 +-
 lib/CodeGen/StrongPHIElimination.cpp               |   11 +-
 lib/CodeGen/TailDuplication.cpp                    |    3 +-
 lib/CodeGen/TargetInstrInfoImpl.cpp                |  120 +-
 lib/CodeGen/TargetLoweringObjectFileImpl.cpp       |   12 +-
 lib/CodeGen/TargetSchedule.cpp                     |  306 ++
 lib/CodeGen/TwoAddressInstructionPass.cpp          |  499 +--
 lib/CodeGen/VirtRegMap.cpp                         |   16 +-
 lib/CodeGen/VirtRegMap.h                           |    4 +-
 lib/DebugInfo/CMakeLists.txt                       |    1 +
 lib/DebugInfo/DIContext.cpp                        |    7 +-
 lib/DebugInfo/DWARFCompileUnit.cpp                 |   48 +-
 lib/DebugInfo/DWARFCompileUnit.h                   |   16 +-
 lib/DebugInfo/DWARFContext.cpp                     |  220 +-
 lib/DebugInfo/DWARFContext.h                       |   40 +-
 lib/DebugInfo/DWARFDebugAranges.cpp                |    1 -
 lib/DebugInfo/DWARFDebugInfoEntry.cpp              |  184 +-
 lib/DebugInfo/DWARFDebugInfoEntry.h                |   54 +-
 lib/DebugInfo/DWARFDebugLine.cpp                   |   27 +
 lib/DebugInfo/DWARFDebugLine.h                     |    8 +
 lib/DebugInfo/DWARFDebugRangeList.cpp              |   67 +
 lib/DebugInfo/DWARFDebugRangeList.h                |   78 +
 lib/DebugInfo/DWARFFormValue.cpp                   |   76 +-
 lib/DebugInfo/DWARFFormValue.h                     |    2 +-
 lib/ExecutionEngine/ExecutionEngine.cpp            |   43 +-
 lib/ExecutionEngine/ExecutionEngineBindings.cpp    |    2 +-
 lib/ExecutionEngine/IntelJITEvents/CMakeLists.txt  |    9 +-
 .../IntelJITEvents/IntelJITEventListener.cpp       |   32 +-
 .../IntelJITEvents/IntelJITEventsWrapper.h         |  102 +
 lib/ExecutionEngine/IntelJITEvents/Makefile        |    5 +-
 .../IntelJITEvents/ittnotify_config.h              |  454 +++
 .../IntelJITEvents/ittnotify_types.h               |   70 +
 lib/ExecutionEngine/IntelJITEvents/jitprofiling.c  |  481 +++
 lib/ExecutionEngine/IntelJITEvents/jitprofiling.h  |  259 ++
 .../Interpreter/ExternalFunctions.cpp              |    8 +-
 lib/ExecutionEngine/Interpreter/Interpreter.cpp    |    2 +-
 lib/ExecutionEngine/Interpreter/Interpreter.h      |    4 +-
 lib/ExecutionEngine/JIT/JIT.cpp                    |   14 +-
 lib/ExecutionEngine/JIT/JITDwarfEmitter.cpp        |    4 +-
 lib/ExecutionEngine/JIT/JITDwarfEmitter.h          |    4 +-
 lib/ExecutionEngine/JIT/JITEmitter.cpp             |   36 +-
 lib/ExecutionEngine/MCJIT/CMakeLists.txt           |    1 -
 lib/ExecutionEngine/MCJIT/MCJIT.cpp                |  113 +-
 lib/ExecutionEngine/MCJIT/MCJIT.h                  |   26 +-
 lib/ExecutionEngine/MCJIT/MCJITMemoryManager.cpp   |   14 -
 lib/ExecutionEngine/MCJIT/MCJITMemoryManager.h     |   50 -
 lib/ExecutionEngine/RuntimeDyld/GDBRegistrar.cpp   |    8 +-
 lib/ExecutionEngine/RuntimeDyld/JITRegistrar.h     |    6 +-
 lib/ExecutionEngine/RuntimeDyld/ObjectImage.h      |   59 -
 .../RuntimeDyld/ObjectImageCommon.h                |   76 +
 lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp    |  145 +-
 lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp |  555 ++-
 lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h   |   56 +-
 lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h  |   96 +-
 .../RuntimeDyld/RuntimeDyldMachO.cpp               |   28 +-
 lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h |    6 +-
 lib/ExecutionEngine/TargetSelect.cpp               |    9 +-
 lib/MC/ELFObjectWriter.cpp                         |   28 +-
 lib/MC/MCAsmBackend.cpp                            |    7 +-
 lib/MC/MCAsmInfo.cpp                               |    2 +-
 lib/MC/MCAsmInfoCOFF.cpp                           |    4 +-
 lib/MC/MCAsmInfoDarwin.cpp                         |    1 +
 lib/MC/MCAsmStreamer.cpp                           |   27 +-
 lib/MC/MCAssembler.cpp                             |   15 +-
 lib/MC/MCContext.cpp                               |    6 +
 lib/MC/MCDisassembler/Disassembler.cpp             |   14 +
 lib/MC/MCDisassembler/EDDisassembler.cpp           |    3 +-
 lib/MC/MCDwarf.cpp                                 |    2 +
 lib/MC/MCELFObjectTargetWriter.cpp                 |    8 +
 lib/MC/MCELFStreamer.cpp                           |   52 +-
 lib/MC/MCExpr.cpp                                  |   11 +-
 lib/MC/MCInst.cpp                                  |    4 +
 lib/MC/MCInstPrinter.cpp                           |   14 +
 lib/MC/MCLabel.cpp                                 |    2 +
 lib/MC/MCMachOStreamer.cpp                         |   59 +-
 lib/MC/MCObjectFileInfo.cpp                        |   33 +-
 lib/MC/MCObjectStreamer.cpp                        |   45 +-
 lib/MC/MCParser/AsmLexer.cpp                       |   13 +-
 lib/MC/MCParser/AsmParser.cpp                      |  578 ++-
 lib/MC/MCParser/ELFAsmParser.cpp                   |    2 +-
 lib/MC/MCParser/MCAsmLexer.cpp                     |    3 +-
 lib/MC/MCParser/MCAsmParser.cpp                    |    2 +
 lib/MC/MCParser/MCTargetAsmParser.cpp              |    2 +-
 lib/MC/MCRegisterInfo.cpp                          |    3 +
 lib/MC/MCStreamer.cpp                              |    4 +
 lib/MC/MCSubtargetInfo.cpp                         |   57 +-
 lib/MC/MCSymbol.cpp                                |    4 +-
 lib/MC/MCValue.cpp                                 |    2 +
 lib/MC/MachObjectWriter.cpp                        |   53 +-
 lib/MC/SubtargetFeature.cpp                        |   35 +-
 lib/MC/WinCOFFStreamer.cpp                         |   42 -
 lib/Object/COFFObjectFile.cpp                      |   14 +-
 lib/Object/MachOObjectFile.cpp                     |   21 +-
 lib/Support/APFloat.cpp                            |  230 +-
 lib/Support/Atomic.cpp                             |   14 +-
 lib/Support/CMakeLists.txt                         |    6 -
 lib/Support/CommandLine.cpp                        |   26 +-
 lib/Support/DAGDeltaAlgorithm.cpp                  |   10 +-
 lib/Support/DataExtractor.cpp                      |    6 +-
 lib/Support/DataStream.cpp                         |    2 +-
 lib/Support/DynamicLibrary.cpp                     |    2 +-
 lib/Support/Errno.cpp                              |   15 +-
 lib/Support/FoldingSet.cpp                         |   18 +
 lib/Support/Host.cpp                               |    4 +
 lib/Support/LockFileManager.cpp                    |    2 +-
 lib/Support/Makefile                               |    3 -
 lib/Support/Memory.cpp                             |   56 -
 lib/Support/MemoryBuffer.cpp                       |   58 +-
 lib/Support/SmallVector.cpp                        |    6 +-
 lib/Support/StreamableMemoryObject.cpp             |   24 +-
 lib/Support/StringMap.cpp                          |    9 +-
 lib/Support/StringRef.cpp                          |    4 +-
 lib/Support/Triple.cpp                             |   59 +-
 lib/Support/Unix/Memory.inc                        |  206 +-
 lib/Support/Unix/Path.inc                          |   17 +-
 lib/Support/Unix/Signals.inc                       |   38 +-
 lib/Support/Windows/Memory.inc                     |  165 +-
 lib/Support/Windows/PathV2.inc                     |    2 +-
 lib/Support/YAMLParser.cpp                         |    7 +
 lib/Support/raw_ostream.cpp                        |   12 +-
 lib/Support/regexec.c                              |    2 +-
 lib/Support/system_error.cpp                       |   10 +-
 lib/TableGen/CMakeLists.txt                        |    5 -
 lib/TableGen/Error.cpp                             |   35 +-
 lib/TableGen/Main.cpp                              |  138 +-
 lib/TableGen/Makefile                              |    4 -
 lib/TableGen/Record.cpp                            |  615 ++--
 lib/TableGen/TGParser.cpp                          |   94 +-
 lib/TableGen/TGParser.h                            |   23 +-
 lib/TableGen/TableGenAction.cpp                    |   15 -
 lib/Target/ARM/ARM.h                               |    1 +
 lib/Target/ARM/ARM.td                              |   31 +-
 lib/Target/ARM/ARMAsmPrinter.cpp                   |   98 +-
 lib/Target/ARM/ARMAsmPrinter.h                     |   40 +-
 lib/Target/ARM/ARMBaseInstrInfo.cpp                |  943 ++++-
 lib/Target/ARM/ARMBaseInstrInfo.h                  |   16 +-
 lib/Target/ARM/ARMBaseRegisterInfo.cpp             |  173 +-
 lib/Target/ARM/ARMBaseRegisterInfo.h               |   13 +-
 lib/Target/ARM/ARMCallingConv.td                   |    2 +
 lib/Target/ARM/ARMCodeEmitter.cpp                  |    8 +-
 lib/Target/ARM/ARMConstantIslandPass.cpp           |    9 +-
 lib/Target/ARM/ARMConstantPoolValue.h              |    5 -
 lib/Target/ARM/ARMELFWriterInfo.cpp                |   78 -
 lib/Target/ARM/ARMELFWriterInfo.h                  |   59 -
 lib/Target/ARM/ARMExpandPseudoInsts.cpp            |   16 +-
 lib/Target/ARM/ARMFastISel.cpp                     |  233 +-
 lib/Target/ARM/ARMFrameLowering.cpp                |   41 +-
 lib/Target/ARM/ARMHazardRecognizer.cpp             |    2 +-
 lib/Target/ARM/ARMISelDAGToDAG.cpp                 |  174 +-
 lib/Target/ARM/ARMISelLowering.cpp                 |  679 +++-
 lib/Target/ARM/ARMISelLowering.h                   |   19 +-
 lib/Target/ARM/ARMInstrFormats.td                  |   21 +
 lib/Target/ARM/ARMInstrInfo.cpp                    |   62 +
 lib/Target/ARM/ARMInstrInfo.td                     |  212 +-
 lib/Target/ARM/ARMInstrNEON.td                     |  125 +-
 lib/Target/ARM/ARMInstrThumb.td                    |    6 +-
 lib/Target/ARM/ARMInstrThumb2.td                   |  138 +-
 lib/Target/ARM/ARMInstrVFP.td                      |   12 +-
 lib/Target/ARM/ARMJITInfo.cpp                      |    2 +-
 lib/Target/ARM/ARMLoadStoreOptimizer.cpp           |    6 +-
 lib/Target/ARM/ARMMachineFunctionInfo.h            |   12 +-
 lib/Target/ARM/ARMRegisterInfo.td                  |   23 +-
 lib/Target/ARM/ARMSchedule.td                      |    2 +
 lib/Target/ARM/ARMScheduleA9.td                    |  597 ++-
 lib/Target/ARM/ARMScheduleSwift.td                 | 1085 ++++++
 lib/Target/ARM/ARMSelectionDAGInfo.cpp             |    2 +-
 lib/Target/ARM/ARMSubtarget.cpp                    |    9 +-
 lib/Target/ARM/ARMSubtarget.h                      |   14 +-
 lib/Target/ARM/ARMTargetMachine.cpp                |   19 +-
 lib/Target/ARM/ARMTargetMachine.h                  |   35 +-
 lib/Target/ARM/AsmParser/ARMAsmParser.cpp          |  280 +-
 lib/Target/ARM/CMakeLists.txt                      |    1 -
 lib/Target/ARM/Disassembler/ARMDisassembler.cpp    |   92 +-
 lib/Target/ARM/InstPrinter/ARMInstPrinter.cpp      |  584 +--
 lib/Target/ARM/InstPrinter/ARMInstPrinter.h        |    3 +-
 lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp      |   15 +-
 lib/Target/ARM/MCTargetDesc/ARMELFObjectWriter.cpp |    7 +
 lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.cpp       |    1 -
 lib/Target/ARM/MCTargetDesc/ARMMCCodeEmitter.cpp   |   10 +-
 lib/Target/ARM/MCTargetDesc/ARMMCExpr.h            |    5 +-
 lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.cpp    |    8 +
 lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.h      |    2 +-
 .../ARM/MCTargetDesc/ARMMachObjectWriter.cpp       |   51 +-
 lib/Target/ARM/MLxExpansionPass.cpp                |   74 +-
 lib/Target/CMakeLists.txt                          |    3 +-
 lib/Target/CellSPU/SPUAsmPrinter.cpp               |    6 +-
 lib/Target/CellSPU/SPUFrameLowering.cpp            |    2 +-
 lib/Target/CellSPU/SPUISelDAGToDAG.cpp             |   43 +-
 lib/Target/CellSPU/SPUSubtarget.h                  |    4 +-
 lib/Target/CellSPU/SPUTargetMachine.cpp            |    5 +-
 lib/Target/CellSPU/SPUTargetMachine.h              |   17 +-
 lib/Target/CppBackend/CPPBackend.cpp               |   35 +-
 lib/Target/CppBackend/CPPTargetMachine.h           |    4 +-
 lib/Target/Hexagon/CMakeLists.txt                  |    1 +
 lib/Target/Hexagon/HexagonAsmPrinter.cpp           |    2 +-
 lib/Target/Hexagon/HexagonCallingConvLower.cpp     |    2 +-
 lib/Target/Hexagon/HexagonISelLowering.cpp         |    2 +
 lib/Target/Hexagon/HexagonInstrFormats.td          |   10 +
 lib/Target/Hexagon/HexagonInstrInfo.cpp            |   28 +-
 lib/Target/Hexagon/HexagonInstrInfo.td             |  312 +-
 lib/Target/Hexagon/HexagonMachineScheduler.cpp     |  681 ++++
 lib/Target/Hexagon/HexagonMachineScheduler.h       |  244 ++
 lib/Target/Hexagon/HexagonNewValueJump.cpp         |    2 +-
 lib/Target/Hexagon/HexagonPeephole.cpp             |   35 +
 lib/Target/Hexagon/HexagonRegisterInfo.cpp         |   52 +
 lib/Target/Hexagon/HexagonRegisterInfo.h           |    5 +
 lib/Target/Hexagon/HexagonRemoveSZExtArgs.cpp      |    2 +-
 lib/Target/Hexagon/HexagonSchedule.td              |    1 +
 lib/Target/Hexagon/HexagonScheduleV4.td            |    1 +
 lib/Target/Hexagon/HexagonSubtarget.cpp            |   33 +-
 lib/Target/Hexagon/HexagonTargetMachine.cpp        |   28 +-
 lib/Target/Hexagon/HexagonTargetMachine.h          |   17 +-
 lib/Target/Hexagon/HexagonTargetObjectFile.cpp     |    4 +-
 lib/Target/Hexagon/HexagonVLIWPacketizer.cpp       |    4 +-
 .../Hexagon/HexagonVarargsCallingConvention.h      |    8 +-
 .../Hexagon/MCTargetDesc/HexagonMCAsmInfo.cpp      |    2 +-
 lib/Target/MBlaze/AsmParser/MBlazeAsmParser.cpp    |   29 +-
 lib/Target/MBlaze/CMakeLists.txt                   |    1 -
 lib/Target/MBlaze/MBlazeAsmPrinter.cpp             |    2 +-
 lib/Target/MBlaze/MBlazeELFWriterInfo.cpp          |  107 -
 lib/Target/MBlaze/MBlazeELFWriterInfo.h            |   59 -
 lib/Target/MBlaze/MBlazeFrameLowering.cpp          |    2 +-
 lib/Target/MBlaze/MBlazeIntrinsicInfo.cpp          |    5 +-
 lib/Target/MBlaze/MBlazeRegisterInfo.cpp           |    2 +-
 lib/Target/MBlaze/MBlazeTargetMachine.cpp          |    7 +-
 lib/Target/MBlaze/MBlazeTargetMachine.h            |   20 +-
 lib/Target/MBlaze/MBlazeTargetObjectFile.cpp       |    4 +-
 .../MBlaze/MCTargetDesc/MBlazeAsmBackend.cpp       |    3 +-
 .../MBlaze/MCTargetDesc/MBlazeMCCodeEmitter.cpp    |    4 +-
 .../MBlaze/MCTargetDesc/MBlazeMCTargetDesc.h       |    3 +-
 lib/Target/MSP430/MSP430FrameLowering.cpp          |   16 +-
 lib/Target/MSP430/MSP430FrameLowering.h            |    1 +
 lib/Target/MSP430/MSP430ISelDAGToDAG.cpp           |    4 +-
 lib/Target/MSP430/MSP430ISelLowering.cpp           |    4 +-
 lib/Target/MSP430/MSP430ISelLowering.h             |    2 +-
 lib/Target/MSP430/MSP430RegisterInfo.cpp           |   14 -
 lib/Target/MSP430/MSP430RegisterInfo.h             |    2 -
 lib/Target/MSP430/MSP430TargetMachine.cpp          |    6 +-
 lib/Target/MSP430/MSP430TargetMachine.h            |   16 +-
 lib/Target/Mangler.cpp                             |    9 +-
 lib/Target/Mips/AsmParser/CMakeLists.txt           |    1 +
 lib/Target/Mips/AsmParser/MipsAsmParser.cpp        | 1289 ++++++-
 lib/Target/Mips/CMakeLists.txt                     |    2 +
 lib/Target/Mips/Disassembler/MipsDisassembler.cpp  |   29 +
 lib/Target/Mips/MCTargetDesc/CMakeLists.txt        |    1 +
 lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp    |   16 +-
 lib/Target/Mips/MCTargetDesc/MipsBaseInfo.h        |    6 +-
 .../Mips/MCTargetDesc/MipsDirectObjLower.cpp       |   81 +
 lib/Target/Mips/MCTargetDesc/MipsDirectObjLower.h  |   28 +
 .../Mips/MCTargetDesc/MipsELFObjectWriter.cpp      |   10 +-
 lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.cpp |   60 +-
 lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.h    |   12 +-
 lib/Target/Mips/Makefile                           |    4 +-
 lib/Target/Mips/Mips.td                            |   19 +-
 lib/Target/Mips/Mips16FrameLowering.cpp            |   56 +-
 lib/Target/Mips/Mips16FrameLowering.h              |    5 +
 lib/Target/Mips/Mips16InstrInfo.cpp                |   93 +-
 lib/Target/Mips/Mips16InstrInfo.h                  |    4 +
 lib/Target/Mips/Mips16InstrInfo.td                 | 1225 ++++++-
 lib/Target/Mips/Mips16RegisterInfo.cpp             |  122 +-
 lib/Target/Mips/Mips16RegisterInfo.h               |    5 +-
 lib/Target/Mips/Mips64InstrInfo.td                 |   95 +-
 lib/Target/Mips/MipsAnalyzeImmediate.cpp           |    2 +-
 lib/Target/Mips/MipsAsmPrinter.cpp                 |   42 +-
 lib/Target/Mips/MipsAsmPrinter.h                   |    8 +
 lib/Target/Mips/MipsCallingConv.td                 |   12 -
 lib/Target/Mips/MipsCodeEmitter.cpp                |   46 +-
 lib/Target/Mips/MipsDSPInstrFormats.td             |  309 ++
 lib/Target/Mips/MipsDSPInstrInfo.td                | 1319 +++++++
 lib/Target/Mips/MipsDelaySlotFiller.cpp            |   11 +-
 lib/Target/Mips/MipsFrameLowering.cpp              |   36 +-
 lib/Target/Mips/MipsFrameLowering.h                |    3 +
 lib/Target/Mips/MipsISelDAGToDAG.cpp               |  210 +-
 lib/Target/Mips/MipsISelLowering.cpp               | 1204 ++++---
 lib/Target/Mips/MipsISelLowering.h                 |  164 +
 lib/Target/Mips/MipsInstrFPU.td                    |   41 +-
 lib/Target/Mips/MipsInstrFormats.td                |   29 +
 lib/Target/Mips/MipsInstrInfo.cpp                  |   55 +-
 lib/Target/Mips/MipsInstrInfo.h                    |   12 -
 lib/Target/Mips/MipsInstrInfo.td                   |  208 +-
 lib/Target/Mips/MipsLongBranch.cpp                 |  232 +-
 lib/Target/Mips/MipsMCInstLower.cpp                |   29 -
 lib/Target/Mips/MipsMCInstLower.h                  |    3 +-
 lib/Target/Mips/MipsMachineFunction.cpp            |   13 +
 lib/Target/Mips/MipsMachineFunction.h              |   50 +-
 lib/Target/Mips/MipsRegisterInfo.cpp               |   50 +-
 lib/Target/Mips/MipsRegisterInfo.h                 |    4 +-
 lib/Target/Mips/MipsRegisterInfo.td                |   27 +-
 lib/Target/Mips/MipsSEFrameLowering.cpp            |   16 +-
 lib/Target/Mips/MipsSEInstrInfo.cpp                |   51 +-
 lib/Target/Mips/MipsSEInstrInfo.h                  |    8 +-
 lib/Target/Mips/MipsSERegisterInfo.cpp             |   45 +-
 lib/Target/Mips/MipsSERegisterInfo.h               |    9 +-
 lib/Target/Mips/MipsSubtarget.cpp                  |    9 +-
 lib/Target/Mips/MipsSubtarget.h                    |   11 +-
 lib/Target/Mips/MipsTargetMachine.cpp              |    7 +-
 lib/Target/Mips/MipsTargetMachine.h                |   18 +-
 lib/Target/Mips/MipsTargetObjectFile.cpp           |   10 +-
 lib/Target/NVPTX/NVPTX.td                          |   34 +-
 lib/Target/NVPTX/NVPTXAllocaHoisting.h             |    4 +-
 lib/Target/NVPTX/NVPTXAsmPrinter.cpp               |  105 +-
 lib/Target/NVPTX/NVPTXISelLowering.cpp             |   68 +-
 lib/Target/NVPTX/NVPTXISelLowering.h               |    3 +
 lib/Target/NVPTX/NVPTXLowerAggrCopies.cpp          |    4 +-
 lib/Target/NVPTX/NVPTXLowerAggrCopies.h            |    4 +-
 lib/Target/NVPTX/NVPTXSubtarget.cpp                |   20 +-
 lib/Target/NVPTX/NVPTXSubtarget.h                  |   12 +-
 lib/Target/NVPTX/NVPTXTargetMachine.cpp            |    7 +-
 lib/Target/NVPTX/NVPTXTargetMachine.h              |   16 +-
 lib/Target/PowerPC/InstPrinter/PPCInstPrinter.cpp  |    8 +-
 lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp  |   15 +-
 .../PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp    |   92 +-
 lib/Target/PowerPC/MCTargetDesc/PPCFixupKinds.h    |   10 +
 lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.cpp   |    4 +-
 .../PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp      |   56 +-
 .../PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp       |    2 +-
 lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.h  |    2 +-
 lib/Target/PowerPC/PPC.td                          |   10 +
 lib/Target/PowerPC/PPCAsmPrinter.cpp               |  101 +-
 lib/Target/PowerPC/PPCCallingConv.td               |    7 +
 lib/Target/PowerPC/PPCFrameLowering.cpp            |  266 +-
 lib/Target/PowerPC/PPCFrameLowering.h              |   71 +-
 lib/Target/PowerPC/PPCISelDAGToDAG.cpp             |  156 +-
 lib/Target/PowerPC/PPCISelLowering.cpp             | 1217 ++++++-
 lib/Target/PowerPC/PPCISelLowering.h               |   60 +-
 lib/Target/PowerPC/PPCInstr64Bit.td                |  115 +-
 lib/Target/PowerPC/PPCInstrAltivec.td              |   32 +
 lib/Target/PowerPC/PPCInstrFormats.td              |   41 +-
 lib/Target/PowerPC/PPCInstrInfo.cpp                |   24 +-
 lib/Target/PowerPC/PPCInstrInfo.td                 |  200 +-
 lib/Target/PowerPC/PPCRegisterInfo.cpp             |   36 +-
 lib/Target/PowerPC/PPCRegisterInfo.h               |    5 +-
 lib/Target/PowerPC/PPCSchedule.td                  |   88 +-
 lib/Target/PowerPC/PPCSchedule440.td               |   60 +-
 lib/Target/PowerPC/PPCScheduleA2.td                |   81 +-
 lib/Target/PowerPC/PPCScheduleE500mc.td            |  265 ++
 lib/Target/PowerPC/PPCScheduleE5500.td             |  309 ++
 lib/Target/PowerPC/PPCScheduleG3.td                |    7 +-
 lib/Target/PowerPC/PPCScheduleG4.td                |    7 +-
 lib/Target/PowerPC/PPCScheduleG4Plus.td            |    8 +-
 lib/Target/PowerPC/PPCScheduleG5.td                |   10 +-
 lib/Target/PowerPC/PPCSubtarget.cpp                |   15 +-
 lib/Target/PowerPC/PPCSubtarget.h                  |   57 +-
 lib/Target/PowerPC/PPCTargetMachine.cpp            |    5 +-
 lib/Target/PowerPC/PPCTargetMachine.h              |   15 +-
 lib/Target/README.txt                              |    7 +-
 lib/Target/Sparc/SparcFrameLowering.cpp            |    2 +-
 lib/Target/Sparc/SparcISelLowering.cpp             |    2 +-
 lib/Target/Sparc/SparcInstrInfo.td                 |    2 +-
 lib/Target/Sparc/SparcTargetMachine.cpp            |    4 +-
 lib/Target/Sparc/SparcTargetMachine.h              |   15 +-
 lib/Target/Target.cpp                              |   19 +-
 lib/Target/TargetData.cpp                          |  663 ----
 lib/Target/TargetELFWriterInfo.cpp                 |   25 -
 lib/Target/TargetLibraryInfo.cpp                   |   75 +-
 lib/Target/TargetLoweringObjectFile.cpp            |    4 +-
 lib/Target/TargetMachineC.cpp                      |   12 +-
 lib/Target/TargetRegisterInfo.cpp                  |    6 +-
 lib/Target/TargetTransformImpl.cpp                 |  353 ++
 lib/Target/X86/AsmParser/X86AsmLexer.cpp           |   43 +-
 lib/Target/X86/AsmParser/X86AsmParser.cpp          |  445 ++-
 lib/Target/X86/CMakeLists.txt                      |    1 -
 lib/Target/X86/Disassembler/X86Disassembler.cpp    |   12 +-
 lib/Target/X86/Disassembler/X86Disassembler.h      |    2 +-
 .../X86/Disassembler/X86DisassemblerDecoder.c      |   16 +-
 .../X86/Disassembler/X86DisassemblerDecoder.h      |   12 +-
 .../Disassembler/X86DisassemblerDecoderCommon.h    |    5 +
 lib/Target/X86/InstPrinter/X86ATTInstPrinter.cpp   |   66 +-
 lib/Target/X86/InstPrinter/X86ATTInstPrinter.h     |    3 +-
 lib/Target/X86/InstPrinter/X86IntelInstPrinter.cpp |   48 +-
 lib/Target/X86/InstPrinter/X86IntelInstPrinter.h   |    5 +-
 lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp      |   70 +-
 lib/Target/X86/MCTargetDesc/X86BaseInfo.h          |   16 +-
 lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp |   25 +-
 lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp       |    5 +
 lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp   |   37 +-
 lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp    |  115 +-
 lib/Target/X86/MCTargetDesc/X86MCTargetDesc.h      |   11 +-
 .../X86/MCTargetDesc/X86MachObjectWriter.cpp       |   48 +-
 lib/Target/X86/README-SSE.txt                      |   12 +
 lib/Target/X86/X86.td                              |   14 +-
 lib/Target/X86/X86AsmPrinter.cpp                   |  113 +-
 lib/Target/X86/X86AsmPrinter.h                     |   37 +-
 lib/Target/X86/X86COFFMachineModuleInfo.h          |    2 +-
 lib/Target/X86/X86CallingConv.td                   |   59 +-
 lib/Target/X86/X86CodeEmitter.cpp                  |   61 +-
 lib/Target/X86/X86ELFWriterInfo.cpp                |  147 -
 lib/Target/X86/X86ELFWriterInfo.h                  |   59 -
 lib/Target/X86/X86FastISel.cpp                     |   56 +-
 lib/Target/X86/X86FloatingPoint.cpp                |    6 +-
 lib/Target/X86/X86FrameLowering.cpp                |   13 +-
 lib/Target/X86/X86ISelDAGToDAG.cpp                 |  609 ++--
 lib/Target/X86/X86ISelLowering.cpp                 | 3120 +++++++++++-----
 lib/Target/X86/X86ISelLowering.h                   |  154 +-
 lib/Target/X86/X86InstrCompiler.td                 |  475 ++-
 lib/Target/X86/X86InstrControl.td                  |    7 +-
 lib/Target/X86/X86InstrFMA.td                      |  432 ++-
 lib/Target/X86/X86InstrFormats.td                  |   91 +-
 lib/Target/X86/X86InstrFragmentsSIMD.td            |   31 +-
 lib/Target/X86/X86InstrInfo.cpp                    |  536 +--
 lib/Target/X86/X86InstrInfo.h                      |    3 +
 lib/Target/X86/X86InstrInfo.td                     |   83 +-
 lib/Target/X86/X86InstrMMX.td                      |   75 +-
 lib/Target/X86/X86InstrSSE.td                      | 1488 +++++---
 lib/Target/X86/X86InstrShiftRotate.td              |   78 +
 lib/Target/X86/X86InstrTSX.td                      |   32 +
 lib/Target/X86/X86InstrXOP.td                      |   17 +-
 lib/Target/X86/X86JITInfo.cpp                      |   17 +-
 lib/Target/X86/X86MCInstLower.cpp                  |   54 +-
 lib/Target/X86/X86MCInstLower.h                    |   52 -
 lib/Target/X86/X86RegisterInfo.cpp                 |   66 +-
 lib/Target/X86/X86RegisterInfo.h                   |    9 +-
 lib/Target/X86/X86RegisterInfo.td                  |  496 +--
 lib/Target/X86/X86SelectionDAGInfo.cpp             |    2 +-
 lib/Target/X86/X86Subtarget.cpp                    |   27 +-
 lib/Target/X86/X86Subtarget.h                      |   31 +-
 lib/Target/X86/X86TargetMachine.cpp                |   19 +-
 lib/Target/X86/X86TargetMachine.h                  |   32 +-
 lib/Target/X86/X86VZeroUpper.cpp                   |   12 +-
 lib/Target/XCore/XCoreAsmPrinter.cpp               |    4 +-
 lib/Target/XCore/XCoreFrameLowering.cpp            |   13 +-
 lib/Target/XCore/XCoreISelLowering.cpp             |   16 +-
 lib/Target/XCore/XCoreInstrInfo.td                 |    4 +-
 lib/Target/XCore/XCoreRegisterInfo.cpp             |    2 +-
 lib/Target/XCore/XCoreTargetMachine.cpp            |    4 +-
 lib/Target/XCore/XCoreTargetMachine.h              |   15 +-
 lib/Transforms/IPO/ArgumentPromotion.cpp           |   64 +-
 lib/Transforms/IPO/BarrierNoopPass.cpp             |   47 +
 lib/Transforms/IPO/CMakeLists.txt                  |    1 +
 lib/Transforms/IPO/ConstantMerge.cpp               |    8 +-
 lib/Transforms/IPO/DeadArgumentElimination.cpp     |  107 +-
 lib/Transforms/IPO/ExtractGV.cpp                   |   63 +-
 lib/Transforms/IPO/FunctionAttrs.cpp               |   34 +-
 lib/Transforms/IPO/GlobalOpt.cpp                   |   93 +-
 lib/Transforms/IPO/IPO.cpp                         |    7 +-
 lib/Transforms/IPO/InlineAlways.cpp                |    6 +-
 lib/Transforms/IPO/InlineSimple.cpp                |    4 +-
 lib/Transforms/IPO/Inliner.cpp                     |   28 +-
 lib/Transforms/IPO/Internalize.cpp                 |   38 +-
 lib/Transforms/IPO/MergeFunctions.cpp              |   26 +-
 lib/Transforms/IPO/PassManagerBuilder.cpp          |   56 +-
 lib/Transforms/IPO/PruneEH.cpp                     |   10 +-
 lib/Transforms/InstCombine/InstCombine.h           |   14 +-
 lib/Transforms/InstCombine/InstCombineAddSub.cpp   |    2 +-
 lib/Transforms/InstCombine/InstCombineCalls.cpp    |  172 +-
 lib/Transforms/InstCombine/InstCombineCasts.cpp    |   25 +-
 lib/Transforms/InstCombine/InstCombineCompares.cpp |   70 +-
 .../InstCombine/InstCombineLoadStoreAlloca.cpp     |  201 +-
 .../InstCombine/InstCombineMulDivRem.cpp           |   17 +-
 lib/Transforms/InstCombine/InstCombinePHI.cpp      |    2 +-
 lib/Transforms/InstCombine/InstCombineSelect.cpp   |   30 +-
 lib/Transforms/InstCombine/InstCombineShifts.cpp   |    2 +-
 .../InstCombine/InstCombineSimplifyDemanded.cpp    |    2 +-
 .../InstCombine/InstCombineVectorOps.cpp           |    7 +-
 lib/Transforms/InstCombine/InstCombineWorklist.h   |    4 +-
 .../InstCombine/InstructionCombining.cpp           |  401 ++-
 .../Instrumentation/AddressSanitizer.cpp           |  437 ++-
 lib/Transforms/Instrumentation/BlackList.cpp       |  105 +
 lib/Transforms/Instrumentation/BlackList.h         |   57 +
 lib/Transforms/Instrumentation/BoundsChecking.cpp  |   16 +-
 lib/Transforms/Instrumentation/CMakeLists.txt      |    2 +-
 .../Instrumentation/FunctionBlackList.cpp          |   79 -
 lib/Transforms/Instrumentation/FunctionBlackList.h |   37 -
 lib/Transforms/Instrumentation/GCOVProfiling.cpp   |   73 +-
 .../Instrumentation/MaximumSpanningTree.h          |   53 +-
 lib/Transforms/Instrumentation/ThreadSanitizer.cpp |  137 +-
 lib/Transforms/Scalar/CMakeLists.txt               |    1 +
 lib/Transforms/Scalar/CodeGenPrepare.cpp           |   80 +-
 lib/Transforms/Scalar/ConstantProp.cpp             |    4 +-
 .../Scalar/CorrelatedValuePropagation.cpp          |    5 +
 lib/Transforms/Scalar/DCE.cpp                      |   17 +-
 lib/Transforms/Scalar/DeadStoreElimination.cpp     |  203 +-
 lib/Transforms/Scalar/EarlyCSE.cpp                 |  120 +-
 lib/Transforms/Scalar/GVN.cpp                      |   99 +-
 lib/Transforms/Scalar/GlobalMerge.cpp              |   10 +-
 lib/Transforms/Scalar/IndVarSimplify.cpp           |   42 +-
 lib/Transforms/Scalar/JumpThreading.cpp            |    8 +-
 lib/Transforms/Scalar/LICM.cpp                     |   45 +-
 lib/Transforms/Scalar/LoopIdiomRecognize.cpp       |   37 +-
 lib/Transforms/Scalar/LoopInstSimplify.cpp         |    6 +-
 lib/Transforms/Scalar/LoopRotation.cpp             |   65 +-
 lib/Transforms/Scalar/LoopStrengthReduce.cpp       |   58 +-
 lib/Transforms/Scalar/LoopUnrollPass.cpp           |    9 +-
 lib/Transforms/Scalar/LoopUnswitch.cpp             |   13 +-
 lib/Transforms/Scalar/MemCpyOptimizer.cpp          |   72 +-
 lib/Transforms/Scalar/ObjCARC.cpp                  |  150 +-
 lib/Transforms/Scalar/Reassociate.cpp              |  105 +-
 lib/Transforms/Scalar/SCCP.cpp                     |   10 +-
 lib/Transforms/Scalar/SROA.cpp                     | 3697 +++++++++++++++++++
 lib/Transforms/Scalar/Scalar.cpp                   |    3 +-
 lib/Transforms/Scalar/ScalarReplAggregates.cpp     |  198 +-
 lib/Transforms/Scalar/SimplifyCFGPass.cpp          |   62 +-
 lib/Transforms/Scalar/SimplifyLibCalls.cpp         |  980 +----
 lib/Transforms/Utils/AddrModeMatcher.cpp           |    6 +-
 lib/Transforms/Utils/BasicBlockUtils.cpp           |   45 +-
 lib/Transforms/Utils/BuildLibCalls.cpp             |  154 +-
 lib/Transforms/Utils/BypassSlowDivision.cpp        |  262 ++
 lib/Transforms/Utils/CMakeLists.txt                |    4 +
 lib/Transforms/Utils/CloneFunction.cpp             |   14 +-
 lib/Transforms/Utils/CodeExtractor.cpp             |    2 +-
 lib/Transforms/Utils/InlineFunction.cpp            |    4 +-
 lib/Transforms/Utils/IntegerDivision.cpp           |  420 +++
 lib/Transforms/Utils/LCSSA.cpp                     |   15 +-
 lib/Transforms/Utils/Local.cpp                     |   82 +-
 lib/Transforms/Utils/LoopSimplify.cpp              |    7 +
 lib/Transforms/Utils/MetaRenamer.cpp               |  132 +
 lib/Transforms/Utils/PromoteMemoryToRegister.cpp   |   18 +-
 lib/Transforms/Utils/SSAUpdater.cpp                |    2 +-
 lib/Transforms/Utils/SimplifyCFG.cpp               | 1507 ++++++--
 lib/Transforms/Utils/SimplifyIndVar.cpp            |    6 +-
 lib/Transforms/Utils/SimplifyInstructions.cpp      |    6 +-
 lib/Transforms/Utils/SimplifyLibCalls.cpp          | 1149 ++++++
 lib/Transforms/Utils/Utils.cpp                     |    1 +
 lib/Transforms/Utils/ValueMapper.cpp               |    2 +-
 lib/Transforms/Vectorize/BBVectorize.cpp           | 1012 ++++--
 lib/Transforms/Vectorize/CMakeLists.txt            |    1 +
 lib/Transforms/Vectorize/LoopVectorize.cpp         | 1941 ++++++++++
 lib/Transforms/Vectorize/Vectorize.cpp             |    8 +-
 lib/VMCore/AsmWriter.cpp                           |  129 +-
 lib/VMCore/Attributes.cpp                          |  518 ++-
 lib/VMCore/AttributesImpl.h                        |   71 +
 lib/VMCore/AutoUpgrade.cpp                         |    3 +-
 lib/VMCore/CMakeLists.txt                          |    6 +-
 lib/VMCore/ConstantFold.cpp                        |   28 +-
 lib/VMCore/Constants.cpp                           |   27 +
 lib/VMCore/ConstantsContext.h                      |   35 +-
 lib/VMCore/Core.cpp                                |   62 +-
 lib/VMCore/DIBuilder.cpp                           |   40 +-
 lib/VMCore/DataLayout.cpp                          |  749 ++++
 lib/VMCore/DebugInfo.cpp                           |   10 +
 lib/VMCore/Dominators.cpp                          |   10 +
 lib/VMCore/Function.cpp                            |   21 +-
 lib/VMCore/GCOV.cpp                                |   30 +-
 lib/VMCore/IRBuilder.cpp                           |    6 +-
 lib/VMCore/InlineAsm.cpp                           |   13 +-
 lib/VMCore/Instructions.cpp                        |   63 +-
 lib/VMCore/LLVMContext.cpp                         |    5 +
 lib/VMCore/LLVMContextImpl.cpp                     |   18 +-
 lib/VMCore/LLVMContextImpl.h                       |    9 +-
 lib/VMCore/Makefile                                |    1 -
 lib/VMCore/PassManager.cpp                         |    2 +-
 lib/VMCore/TargetTransformInfo.cpp                 |   31 +
 lib/VMCore/Type.cpp                                |   41 +-
 lib/VMCore/User.cpp                                |    9 +
 lib/VMCore/Value.cpp                               |    2 +-
 lib/VMCore/ValueTypes.cpp                          |   30 +-
 lib/VMCore/Verifier.cpp                            |  181 +-
 709 files changed, 53708 insertions(+), 18126 deletions(-)
 create mode 100644 lib/Analysis/CostModel.cpp
 create mode 100644 lib/Analysis/DependenceAnalysis.cpp
 delete mode 100644 lib/Analysis/LoopDependenceAnalysis.cpp
 create mode 100644 lib/Analysis/ProfileDataLoader.cpp
 create mode 100644 lib/Analysis/ProfileDataLoaderPass.cpp
 create mode 100644 lib/CodeGen/MachinePostDominators.cpp
 create mode 100644 lib/CodeGen/StackColoring.cpp
 create mode 100644 lib/CodeGen/TargetSchedule.cpp
 create mode 100644 lib/DebugInfo/DWARFDebugRangeList.cpp
 create mode 100644 lib/DebugInfo/DWARFDebugRangeList.h
 create mode 100644 lib/ExecutionEngine/IntelJITEvents/IntelJITEventsWrapper.h
 create mode 100644 lib/ExecutionEngine/IntelJITEvents/ittnotify_config.h
 create mode 100644 lib/ExecutionEngine/IntelJITEvents/ittnotify_types.h
 create mode 100644 lib/ExecutionEngine/IntelJITEvents/jitprofiling.c
 create mode 100644 lib/ExecutionEngine/IntelJITEvents/jitprofiling.h
 delete mode 100644 lib/ExecutionEngine/MCJIT/MCJITMemoryManager.cpp
 delete mode 100644 lib/ExecutionEngine/MCJIT/MCJITMemoryManager.h
 delete mode 100644 lib/ExecutionEngine/RuntimeDyld/ObjectImage.h
 create mode 100644 lib/ExecutionEngine/RuntimeDyld/ObjectImageCommon.h
 delete mode 100644 lib/TableGen/TableGenAction.cpp
 delete mode 100644 lib/Target/ARM/ARMELFWriterInfo.cpp
 delete mode 100644 lib/Target/ARM/ARMELFWriterInfo.h
 create mode 100644 lib/Target/ARM/ARMScheduleSwift.td
 create mode 100644 lib/Target/Hexagon/HexagonMachineScheduler.cpp
 create mode 100644 lib/Target/Hexagon/HexagonMachineScheduler.h
 delete mode 100644 lib/Target/MBlaze/MBlazeELFWriterInfo.cpp
 delete mode 100644 lib/Target/MBlaze/MBlazeELFWriterInfo.h
 create mode 100644 lib/Target/Mips/MCTargetDesc/MipsDirectObjLower.cpp
 create mode 100644 lib/Target/Mips/MCTargetDesc/MipsDirectObjLower.h
 create mode 100644 lib/Target/Mips/MipsDSPInstrFormats.td
 create mode 100644 lib/Target/Mips/MipsDSPInstrInfo.td
 create mode 100644 lib/Target/PowerPC/PPCScheduleE500mc.td
 create mode 100644 lib/Target/PowerPC/PPCScheduleE5500.td
 delete mode 100644 lib/Target/TargetData.cpp
 delete mode 100644 lib/Target/TargetELFWriterInfo.cpp
 create mode 100644 lib/Target/TargetTransformImpl.cpp
 delete mode 100644 lib/Target/X86/X86ELFWriterInfo.cpp
 delete mode 100644 lib/Target/X86/X86ELFWriterInfo.h
 create mode 100644 lib/Target/X86/X86InstrTSX.td
 delete mode 100644 lib/Target/X86/X86MCInstLower.h
 create mode 100644 lib/Transforms/IPO/BarrierNoopPass.cpp
 create mode 100644 lib/Transforms/Instrumentation/BlackList.cpp
 create mode 100644 lib/Transforms/Instrumentation/BlackList.h
 delete mode 100644 lib/Transforms/Instrumentation/FunctionBlackList.cpp
 delete mode 100644 lib/Transforms/Instrumentation/FunctionBlackList.h
 create mode 100644 lib/Transforms/Scalar/SROA.cpp
 create mode 100644 lib/Transforms/Utils/BypassSlowDivision.cpp
 create mode 100644 lib/Transforms/Utils/IntegerDivision.cpp
 create mode 100644 lib/Transforms/Utils/MetaRenamer.cpp
 create mode 100644 lib/Transforms/Utils/SimplifyLibCalls.cpp
 create mode 100644 lib/Transforms/Vectorize/LoopVectorize.cpp
 create mode 100644 lib/VMCore/AttributesImpl.h
 create mode 100644 lib/VMCore/DataLayout.cpp
 create mode 100644 lib/VMCore/TargetTransformInfo.cpp

(limited to 'lib')

diff --git a/lib/Analysis/AliasAnalysis.cpp b/lib/Analysis/AliasAnalysis.cpp
index 3b6aab1..752edd5 100644
--- a/lib/Analysis/AliasAnalysis.cpp
+++ b/lib/Analysis/AliasAnalysis.cpp
@@ -35,7 +35,8 @@
 #include "llvm/Instructions.h"
 #include "llvm/LLVMContext.h"
 #include "llvm/Type.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
+#include "llvm/Target/TargetLibraryInfo.h"
 using namespace llvm;
 
 // Register the AliasAnalysis interface, providing a nice name to refer to.
@@ -451,7 +452,8 @@ AliasAnalysis::~AliasAnalysis() {}
 /// AliasAnalysis interface before any other methods are called.
 ///
 void AliasAnalysis::InitializeAliasAnalysis(Pass *P) {
-  TD = P->getAnalysisIfAvailable<TargetData>();
+  TD = P->getAnalysisIfAvailable<DataLayout>();
+  TLI = P->getAnalysisIfAvailable<TargetLibraryInfo>();
   AA = &P->getAnalysis<AliasAnalysis>();
 }
 
@@ -461,7 +463,7 @@ void AliasAnalysis::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.addRequired<AliasAnalysis>();         // All AA's chain
 }
 
-/// getTypeStoreSize - Return the TargetData store size for the given type,
+/// getTypeStoreSize - Return the DataLayout store size for the given type,
 /// if known, or a conservative value otherwise.
 ///
 uint64_t AliasAnalysis::getTypeStoreSize(Type *Ty) {
@@ -501,7 +503,7 @@ bool AliasAnalysis::canInstructionRangeModify(const Instruction &I1,
 bool llvm::isNoAliasCall(const Value *V) {
   if (isa<CallInst>(V) || isa<InvokeInst>(V))
     return ImmutableCallSite(cast<Instruction>(V))
-      .paramHasAttr(0, Attribute::NoAlias);
+      .paramHasAttr(0, Attributes::NoAlias);
   return false;
 }
 
diff --git a/lib/Analysis/AliasSetTracker.cpp b/lib/Analysis/AliasSetTracker.cpp
index 92e8906..388c755 100644
--- a/lib/Analysis/AliasSetTracker.cpp
+++ b/lib/Analysis/AliasSetTracker.cpp
@@ -18,7 +18,7 @@
 #include "llvm/LLVMContext.h"
 #include "llvm/Pass.h"
 #include "llvm/Type.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Assembly/Writer.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -550,7 +550,7 @@ void AliasSetTracker::copyValue(Value *From, Value *To) {
 //===----------------------------------------------------------------------===//
 
 void AliasSet::print(raw_ostream &OS) const {
-  OS << "  AliasSet[" << (void*)this << ", " << RefCount << "] ";
+  OS << "  AliasSet[" << (const void*)this << ", " << RefCount << "] ";
   OS << (AliasTy == MustAlias ? "must" : "may") << " alias, ";
   switch (AccessTy) {
   case NoModRef: OS << "No access "; break;
@@ -590,8 +590,10 @@ void AliasSetTracker::print(raw_ostream &OS) const {
   OS << "\n";
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void AliasSet::dump() const { print(dbgs()); }
 void AliasSetTracker::dump() const { print(dbgs()); }
+#endif
 
 //===----------------------------------------------------------------------===//
 //                     ASTCallbackVH Class Implementation
diff --git a/lib/Analysis/Analysis.cpp b/lib/Analysis/Analysis.cpp
index 0ba6af9..9dc81a6 100644
--- a/lib/Analysis/Analysis.cpp
+++ b/lib/Analysis/Analysis.cpp
@@ -26,11 +26,13 @@ void llvm::initializeAnalysis(PassRegistry &Registry) {
   initializeBasicAliasAnalysisPass(Registry);
   initializeBlockFrequencyInfoPass(Registry);
   initializeBranchProbabilityInfoPass(Registry);
+  initializeCostModelAnalysisPass(Registry);
   initializeCFGViewerPass(Registry);
   initializeCFGPrinterPass(Registry);
   initializeCFGOnlyViewerPass(Registry);
   initializeCFGOnlyPrinterPass(Registry);
   initializePrintDbgInfoPass(Registry);
+  initializeDependenceAnalysisPass(Registry);
   initializeDominanceFrontierPass(Registry);
   initializeDomViewerPass(Registry);
   initializeDomPrinterPass(Registry);
@@ -46,7 +48,6 @@ void llvm::initializeAnalysis(PassRegistry &Registry) {
   initializeLazyValueInfoPass(Registry);
   initializeLibCallAliasAnalysisPass(Registry);
   initializeLintPass(Registry);
-  initializeLoopDependenceAnalysisPass(Registry);
   initializeLoopInfoPass(Registry);
   initializeMemDepPrinterPass(Registry);
   initializeMemoryDependenceAnalysisPass(Registry);
@@ -61,6 +62,7 @@ void llvm::initializeAnalysis(PassRegistry &Registry) {
   initializePathProfileLoaderPassPass(Registry);
   initializeProfileVerifierPassPass(Registry);
   initializePathProfileVerifierPass(Registry);
+  initializeProfileMetadataLoaderPassPass(Registry);
   initializeRegionInfoPass(Registry);
   initializeRegionViewerPass(Registry);
   initializeRegionPrinterPass(Registry);
diff --git a/lib/Analysis/BasicAliasAnalysis.cpp b/lib/Analysis/BasicAliasAnalysis.cpp
index 1d028c2..4bb93ee 100644
--- a/lib/Analysis/BasicAliasAnalysis.cpp
+++ b/lib/Analysis/BasicAliasAnalysis.cpp
@@ -29,7 +29,7 @@
 #include "llvm/Analysis/MemoryBuiltins.h"
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/ValueTracking.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetLibraryInfo.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallVector.h"
@@ -58,12 +58,12 @@ static bool isNonEscapingLocalObject(const Value *V) {
   // then it has not escaped before entering the function.  Check if it escapes
   // inside the function.
   if (const Argument *A = dyn_cast<Argument>(V))
-    if (A->hasByValAttr() || A->hasNoAliasAttr()) {
-      // Don't bother analyzing arguments already known not to escape.
-      if (A->hasNoCaptureAttr())
-        return true;
+    if (A->hasByValAttr() || A->hasNoAliasAttr())
+      // Note even if the argument is marked nocapture we still need to check
+      // for copies made inside the function. The nocapture attribute only
+      // specifies that there are no copies made that outlive the function.
       return !PointerMayBeCaptured(V, false, /*StoreCaptures=*/true);
-    }
+
   return false;
 }
 
@@ -84,10 +84,11 @@ static bool isEscapeSource(const Value *V) {
 
 /// getObjectSize - Return the size of the object specified by V, or
 /// UnknownSize if unknown.
-static uint64_t getObjectSize(const Value *V, const TargetData &TD,
+static uint64_t getObjectSize(const Value *V, const DataLayout &TD,
+                              const TargetLibraryInfo &TLI,
                               bool RoundToAlign = false) {
   uint64_t Size;
-  if (getObjectSize(V, Size, &TD, RoundToAlign))
+  if (getObjectSize(V, Size, &TD, &TLI, RoundToAlign))
     return Size;
   return AliasAnalysis::UnknownSize;
 }
@@ -95,10 +96,11 @@ static uint64_t getObjectSize(const Value *V, const TargetData &TD,
 /// isObjectSmallerThan - Return true if we can prove that the object specified
 /// by V is smaller than Size.
 static bool isObjectSmallerThan(const Value *V, uint64_t Size,
-                                const TargetData &TD) {
+                                const DataLayout &TD,
+                                const TargetLibraryInfo &TLI) {
   // This function needs to use the aligned object size because we allow
   // reads a bit past the end given sufficient alignment.
-  uint64_t ObjectSize = getObjectSize(V, TD, /*RoundToAlign*/true);
+  uint64_t ObjectSize = getObjectSize(V, TD, TLI, /*RoundToAlign*/true);
   
   return ObjectSize != AliasAnalysis::UnknownSize && ObjectSize < Size;
 }
@@ -106,8 +108,8 @@ static bool isObjectSmallerThan(const Value *V, uint64_t Size,
 /// isObjectSize - Return true if we can prove that the object specified
 /// by V has size Size.
 static bool isObjectSize(const Value *V, uint64_t Size,
-                         const TargetData &TD) {
-  uint64_t ObjectSize = getObjectSize(V, TD);
+                         const DataLayout &TD, const TargetLibraryInfo &TLI) {
+  uint64_t ObjectSize = getObjectSize(V, TD, TLI);
   return ObjectSize != AliasAnalysis::UnknownSize && ObjectSize == Size;
 }
 
@@ -126,6 +128,15 @@ namespace {
     const Value *V;
     ExtensionKind Extension;
     int64_t Scale;
+
+    bool operator==(const VariableGEPIndex &Other) const {
+      return V == Other.V && Extension == Other.Extension &&
+        Scale == Other.Scale;
+    }
+
+    bool operator!=(const VariableGEPIndex &Other) const {
+      return !operator==(Other);
+    }
   };
 }
 
@@ -140,7 +151,7 @@ namespace {
 /// represented in the result.
 static Value *GetLinearExpression(Value *V, APInt &Scale, APInt &Offset,
                                   ExtensionKind &Extension,
-                                  const TargetData &TD, unsigned Depth) {
+                                  const DataLayout &TD, unsigned Depth) {
   assert(V->getType()->isIntegerTy() && "Not an integer value");
 
   // Limit our recursion depth.
@@ -215,14 +226,14 @@ static Value *GetLinearExpression(Value *V, APInt &Scale, APInt &Offset,
 /// specified amount, but which may have other unrepresented high bits. As such,
 /// the gep cannot necessarily be reconstructed from its decomposed form.
 ///
-/// When TargetData is around, this function is capable of analyzing everything
+/// When DataLayout is around, this function is capable of analyzing everything
 /// that GetUnderlyingObject can look through.  When not, it just looks
 /// through pointer casts.
 ///
 static const Value *
 DecomposeGEPExpression(const Value *V, int64_t &BaseOffs,
                        SmallVectorImpl<VariableGEPIndex> &VarIndices,
-                       const TargetData *TD) {
+                       const DataLayout *TD) {
   // Limit recursion depth to limit compile time in crazy cases.
   unsigned MaxLookup = 6;
   
@@ -266,7 +277,7 @@ DecomposeGEPExpression(const Value *V, int64_t &BaseOffs,
         ->getElementType()->isSized())
       return V;
     
-    // If we are lacking TargetData information, we can't compute the offets of
+    // If we are lacking DataLayout information, we can't compute the offets of
     // elements computed by GEPs.  However, we can handle bitcast equivalent
     // GEPs.
     if (TD == 0) {
@@ -417,13 +428,7 @@ namespace {
   /// BasicAliasAnalysis - This is the primary alias analysis implementation.
   struct BasicAliasAnalysis : public ImmutablePass, public AliasAnalysis {
     static char ID; // Class identification, replacement for typeinfo
-    BasicAliasAnalysis() : ImmutablePass(ID),
-                           // AliasCache rarely has more than 1 or 2 elements,
-                           // so start it off fairly small so that clear()
-                           // doesn't have to tromp through 64 (the default)
-                           // elements on each alias query. This really wants
-                           // something like a SmallDenseMap.
-                           AliasCache(8) {
+    BasicAliasAnalysis() : ImmutablePass(ID) {
       initializeBasicAliasAnalysisPass(*PassRegistry::getPassRegistry());
     }
 
@@ -443,7 +448,11 @@ namespace {
              "BasicAliasAnalysis doesn't support interprocedural queries.");
       AliasResult Alias = aliasCheck(LocA.Ptr, LocA.Size, LocA.TBAATag,
                                      LocB.Ptr, LocB.Size, LocB.TBAATag);
-      AliasCache.clear();
+      // AliasCache rarely has more than 1 or 2 elements, always use
+      // shrink_and_clear so it quickly returns to the inline capacity of the
+      // SmallDenseMap if it ever grows larger.
+      // FIXME: This should really be shrink_to_inline_capacity_and_clear().
+      AliasCache.shrink_and_clear();
       return Alias;
     }
 
@@ -481,7 +490,7 @@ namespace {
   private:
     // AliasCache - Track alias queries to guard against recursion.
     typedef std::pair<Location, Location> LocPair;
-    typedef DenseMap<LocPair, AliasResult> AliasCacheTy;
+    typedef SmallDenseMap<LocPair, AliasResult, 8> AliasCacheTy;
     AliasCacheTy AliasCache;
 
     // Visited - Track instructions visited by pointsToConstantMemory.
@@ -490,6 +499,7 @@ namespace {
     // aliasGEP - Provide a bunch of ad-hoc rules to disambiguate a GEP
     // instruction against another.
     AliasResult aliasGEP(const GEPOperator *V1, uint64_t V1Size,
+                         const MDNode *V1TBAAInfo,
                          const Value *V2, uint64_t V2Size,
                          const MDNode *V2TBAAInfo,
                          const Value *UnderlyingV1, const Value *UnderlyingV2);
@@ -807,6 +817,21 @@ BasicAliasAnalysis::getModRefInfo(ImmutableCallSite CS,
   return ModRefResult(AliasAnalysis::getModRefInfo(CS, Loc) & Min);
 }
 
+static bool areVarIndicesEqual(SmallVector<VariableGEPIndex, 4> &Indices1,
+                               SmallVector<VariableGEPIndex, 4> &Indices2) {
+  unsigned Size1 = Indices1.size();
+  unsigned Size2 = Indices2.size();
+
+  if (Size1 != Size2)
+    return false;
+
+  for (unsigned I = 0; I != Size1; ++I)
+    if (Indices1[I] != Indices2[I])
+      return false;
+
+  return true;
+}
+
 /// aliasGEP - Provide a bunch of ad-hoc rules to disambiguate a GEP instruction
 /// against another pointer.  We know that V1 is a GEP, but we don't know
 /// anything about V2.  UnderlyingV1 is GetUnderlyingObject(GEP1, TD),
@@ -814,6 +839,7 @@ BasicAliasAnalysis::getModRefInfo(ImmutableCallSite CS,
 ///
 AliasAnalysis::AliasResult
 BasicAliasAnalysis::aliasGEP(const GEPOperator *GEP1, uint64_t V1Size,
+                             const MDNode *V1TBAAInfo,
                              const Value *V2, uint64_t V2Size,
                              const MDNode *V2TBAAInfo,
                              const Value *UnderlyingV1,
@@ -821,9 +847,41 @@ BasicAliasAnalysis::aliasGEP(const GEPOperator *GEP1, uint64_t V1Size,
   int64_t GEP1BaseOffset;
   SmallVector<VariableGEPIndex, 4> GEP1VariableIndices;
 
-  // If we have two gep instructions with must-alias'ing base pointers, figure
-  // out if the indexes to the GEP tell us anything about the derived pointer.
+  // If we have two gep instructions with must-alias or not-alias'ing base
+  // pointers, figure out if the indexes to the GEP tell us anything about the
+  // derived pointer.
   if (const GEPOperator *GEP2 = dyn_cast<GEPOperator>(V2)) {
+    // Check for geps of non-aliasing underlying pointers where the offsets are
+    // identical.
+    if (V1Size == V2Size) {
+      // Do the base pointers alias assuming type and size.
+      AliasResult PreciseBaseAlias = aliasCheck(UnderlyingV1, V1Size,
+                                                V1TBAAInfo, UnderlyingV2,
+                                                V2Size, V2TBAAInfo);
+      if (PreciseBaseAlias == NoAlias) {
+        // See if the computed offset from the common pointer tells us about the
+        // relation of the resulting pointer.
+        int64_t GEP2BaseOffset;
+        SmallVector<VariableGEPIndex, 4> GEP2VariableIndices;
+        const Value *GEP2BasePtr =
+          DecomposeGEPExpression(GEP2, GEP2BaseOffset, GEP2VariableIndices, TD);
+        const Value *GEP1BasePtr =
+          DecomposeGEPExpression(GEP1, GEP1BaseOffset, GEP1VariableIndices, TD);
+        // DecomposeGEPExpression and GetUnderlyingObject should return the
+        // same result except when DecomposeGEPExpression has no DataLayout.
+        if (GEP1BasePtr != UnderlyingV1 || GEP2BasePtr != UnderlyingV2) {
+          assert(TD == 0 &&
+             "DecomposeGEPExpression and GetUnderlyingObject disagree!");
+          return MayAlias;
+        }
+        // Same offsets.
+        if (GEP1BaseOffset == GEP2BaseOffset &&
+            areVarIndicesEqual(GEP1VariableIndices, GEP2VariableIndices))
+          return NoAlias;
+        GEP1VariableIndices.clear();
+      }
+    }
+
     // Do the base pointers alias?
     AliasResult BaseAlias = aliasCheck(UnderlyingV1, UnknownSize, 0,
                                        UnderlyingV2, UnknownSize, 0);
@@ -843,9 +901,8 @@ BasicAliasAnalysis::aliasGEP(const GEPOperator *GEP1, uint64_t V1Size,
     const Value *GEP2BasePtr =
       DecomposeGEPExpression(GEP2, GEP2BaseOffset, GEP2VariableIndices, TD);
     
-    // If DecomposeGEPExpression isn't able to look all the way through the
-    // addressing operation, we must not have TD and this is too complex for us
-    // to handle without it.
+    // DecomposeGEPExpression and GetUnderlyingObject should return the
+    // same result except when DecomposeGEPExpression has no DataLayout.
     if (GEP1BasePtr != UnderlyingV1 || GEP2BasePtr != UnderlyingV2) {
       assert(TD == 0 &&
              "DecomposeGEPExpression and GetUnderlyingObject disagree!");
@@ -879,9 +936,8 @@ BasicAliasAnalysis::aliasGEP(const GEPOperator *GEP1, uint64_t V1Size,
     const Value *GEP1BasePtr =
       DecomposeGEPExpression(GEP1, GEP1BaseOffset, GEP1VariableIndices, TD);
     
-    // If DecomposeGEPExpression isn't able to look all the way through the
-    // addressing operation, we must not have TD and this is too complex for us
-    // to handle without it.
+    // DecomposeGEPExpression and GetUnderlyingObject should return the
+    // same result except when DecomposeGEPExpression has no DataLayout.
     if (GEP1BasePtr != UnderlyingV1) {
       assert(TD == 0 &&
              "DecomposeGEPExpression and GetUnderlyingObject disagree!");
@@ -1004,12 +1060,42 @@ BasicAliasAnalysis::aliasPHI(const PHINode *PN, uint64_t PNSize,
   // on corresponding edges.
   if (const PHINode *PN2 = dyn_cast<PHINode>(V2))
     if (PN2->getParent() == PN->getParent()) {
+      LocPair Locs(Location(PN, PNSize, PNTBAAInfo),
+                   Location(V2, V2Size, V2TBAAInfo));
+      if (PN > V2)
+        std::swap(Locs.first, Locs.second);
+
       AliasResult Alias =
         aliasCheck(PN->getIncomingValue(0), PNSize, PNTBAAInfo,
                    PN2->getIncomingValueForBlock(PN->getIncomingBlock(0)),
                    V2Size, V2TBAAInfo);
       if (Alias == MayAlias)
         return MayAlias;
+
+      // If the first source of the PHI nodes NoAlias and the other inputs are
+      // the PHI node itself through some amount of recursion this does not add
+      // any new information so just return NoAlias.
+      // bb:
+      //    ptr = ptr2 + 1
+      // loop:
+      //    ptr_phi = phi [bb, ptr], [loop, ptr_plus_one]
+      //    ptr2_phi = phi [bb, ptr2], [loop, ptr2_plus_one]
+      //    ...
+      //    ptr_plus_one = gep ptr_phi, 1
+      //    ptr2_plus_one = gep ptr2_phi, 1
+      // We assume for the recursion that the the phis (ptr_phi, ptr2_phi) do
+      // not alias each other.
+      bool ArePhisAssumedNoAlias = false;
+      AliasResult OrigAliasResult = NoAlias;
+      if (Alias == NoAlias) {
+        // Pretend the phis do not alias.
+        assert(AliasCache.count(Locs) &&
+               "There must exist an entry for the phi node");
+        OrigAliasResult = AliasCache[Locs];
+        AliasCache[Locs] = NoAlias;
+        ArePhisAssumedNoAlias = true;
+      }
+
       for (unsigned i = 1, e = PN->getNumIncomingValues(); i != e; ++i) {
         AliasResult ThisAlias =
           aliasCheck(PN->getIncomingValue(i), PNSize, PNTBAAInfo,
@@ -1019,6 +1105,11 @@ BasicAliasAnalysis::aliasPHI(const PHINode *PN, uint64_t PNSize,
         if (Alias == MayAlias)
           break;
       }
+
+      // Reset if speculation failed.
+      if (ArePhisAssumedNoAlias && Alias != NoAlias)
+        AliasCache[Locs] = OrigAliasResult;
+
       return Alias;
     }
 
@@ -1133,8 +1224,8 @@ BasicAliasAnalysis::aliasCheck(const Value *V1, uint64_t V1Size,
   // If the size of one access is larger than the entire object on the other
   // side, then we know such behavior is undefined and can assume no alias.
   if (TD)
-    if ((V1Size != UnknownSize && isObjectSmallerThan(O2, V1Size, *TD)) ||
-        (V2Size != UnknownSize && isObjectSmallerThan(O1, V2Size, *TD)))
+    if ((V1Size != UnknownSize && isObjectSmallerThan(O2, V1Size, *TD, *TLI)) ||
+        (V2Size != UnknownSize && isObjectSmallerThan(O1, V2Size, *TD, *TLI)))
       return NoAlias;
   
   // Check the cache before climbing up use-def chains. This also terminates
@@ -1154,15 +1245,17 @@ BasicAliasAnalysis::aliasCheck(const Value *V1, uint64_t V1Size,
     std::swap(V1, V2);
     std::swap(V1Size, V2Size);
     std::swap(O1, O2);
+    std::swap(V1TBAAInfo, V2TBAAInfo);
   }
   if (const GEPOperator *GV1 = dyn_cast<GEPOperator>(V1)) {
-    AliasResult Result = aliasGEP(GV1, V1Size, V2, V2Size, V2TBAAInfo, O1, O2);
+    AliasResult Result = aliasGEP(GV1, V1Size, V1TBAAInfo, V2, V2Size, V2TBAAInfo, O1, O2);
     if (Result != MayAlias) return AliasCache[Locs] = Result;
   }
 
   if (isa<PHINode>(V2) && !isa<PHINode>(V1)) {
     std::swap(V1, V2);
     std::swap(V1Size, V2Size);
+    std::swap(V1TBAAInfo, V2TBAAInfo);
   }
   if (const PHINode *PN = dyn_cast<PHINode>(V1)) {
     AliasResult Result = aliasPHI(PN, V1Size, V1TBAAInfo,
@@ -1173,6 +1266,7 @@ BasicAliasAnalysis::aliasCheck(const Value *V1, uint64_t V1Size,
   if (isa<SelectInst>(V2) && !isa<SelectInst>(V1)) {
     std::swap(V1, V2);
     std::swap(V1Size, V2Size);
+    std::swap(V1TBAAInfo, V2TBAAInfo);
   }
   if (const SelectInst *S1 = dyn_cast<SelectInst>(V1)) {
     AliasResult Result = aliasSelect(S1, V1Size, V1TBAAInfo,
@@ -1184,8 +1278,8 @@ BasicAliasAnalysis::aliasCheck(const Value *V1, uint64_t V1Size,
   // accesses is accessing the entire object, then the accesses must
   // overlap in some way.
   if (TD && O1 == O2)
-    if ((V1Size != UnknownSize && isObjectSize(O1, V1Size, *TD)) ||
-        (V2Size != UnknownSize && isObjectSize(O2, V2Size, *TD)))
+    if ((V1Size != UnknownSize && isObjectSize(O1, V1Size, *TD, *TLI)) ||
+        (V2Size != UnknownSize && isObjectSize(O2, V2Size, *TD, *TLI)))
       return AliasCache[Locs] = PartialAlias;
 
   AliasResult Result =
diff --git a/lib/Analysis/BranchProbabilityInfo.cpp b/lib/Analysis/BranchProbabilityInfo.cpp
index b255ce6..04a6560 100644
--- a/lib/Analysis/BranchProbabilityInfo.cpp
+++ b/lib/Analysis/BranchProbabilityInfo.cpp
@@ -115,14 +115,14 @@ bool BranchProbabilityInfo::calcUnreachableHeuristics(BasicBlock *BB) {
     return false;
   }
 
-  SmallPtrSet<BasicBlock *, 4> UnreachableEdges;
-  SmallPtrSet<BasicBlock *, 4> ReachableEdges;
+  SmallVector<unsigned, 4> UnreachableEdges;
+  SmallVector<unsigned, 4> ReachableEdges;
 
   for (succ_iterator I = succ_begin(BB), E = succ_end(BB); I != E; ++I) {
     if (PostDominatedByUnreachable.count(*I))
-      UnreachableEdges.insert(*I);
+      UnreachableEdges.push_back(I.getSuccessorIndex());
     else
-      ReachableEdges.insert(*I);
+      ReachableEdges.push_back(I.getSuccessorIndex());
   }
 
   // If all successors are in the set of blocks post-dominated by unreachable,
@@ -136,18 +136,19 @@ bool BranchProbabilityInfo::calcUnreachableHeuristics(BasicBlock *BB) {
     return false;
 
   uint32_t UnreachableWeight =
-    std::max(UR_TAKEN_WEIGHT / UnreachableEdges.size(), MIN_WEIGHT);
-  for (SmallPtrSet<BasicBlock *, 4>::iterator I = UnreachableEdges.begin(),
-                                              E = UnreachableEdges.end();
+    std::max(UR_TAKEN_WEIGHT / (unsigned)UnreachableEdges.size(), MIN_WEIGHT);
+  for (SmallVector<unsigned, 4>::iterator I = UnreachableEdges.begin(),
+                                          E = UnreachableEdges.end();
        I != E; ++I)
     setEdgeWeight(BB, *I, UnreachableWeight);
 
   if (ReachableEdges.empty())
     return true;
   uint32_t ReachableWeight =
-    std::max(UR_NONTAKEN_WEIGHT / ReachableEdges.size(), NORMAL_WEIGHT);
-  for (SmallPtrSet<BasicBlock *, 4>::iterator I = ReachableEdges.begin(),
-                                              E = ReachableEdges.end();
+    std::max(UR_NONTAKEN_WEIGHT / (unsigned)ReachableEdges.size(),
+             NORMAL_WEIGHT);
+  for (SmallVector<unsigned, 4>::iterator I = ReachableEdges.begin(),
+                                          E = ReachableEdges.end();
        I != E; ++I)
     setEdgeWeight(BB, *I, ReachableWeight);
 
@@ -187,7 +188,7 @@ bool BranchProbabilityInfo::calcMetadataWeights(BasicBlock *BB) {
   }
   assert(Weights.size() == TI->getNumSuccessors() && "Checked above");
   for (unsigned i = 0, e = TI->getNumSuccessors(); i != e; ++i)
-    setEdgeWeight(BB, TI->getSuccessor(i), Weights[i]);
+    setEdgeWeight(BB, i, Weights[i]);
 
   return true;
 }
@@ -211,19 +212,17 @@ bool BranchProbabilityInfo::calcPointerHeuristics(BasicBlock *BB) {
 
   assert(CI->getOperand(1)->getType()->isPointerTy());
 
-  BasicBlock *Taken = BI->getSuccessor(0);
-  BasicBlock *NonTaken = BI->getSuccessor(1);
-
   // p != 0   ->   isProb = true
   // p == 0   ->   isProb = false
   // p != q   ->   isProb = true
   // p == q   ->   isProb = false;
+  unsigned TakenIdx = 0, NonTakenIdx = 1;
   bool isProb = CI->getPredicate() == ICmpInst::ICMP_NE;
   if (!isProb)
-    std::swap(Taken, NonTaken);
+    std::swap(TakenIdx, NonTakenIdx);
 
-  setEdgeWeight(BB, Taken, PH_TAKEN_WEIGHT);
-  setEdgeWeight(BB, NonTaken, PH_NONTAKEN_WEIGHT);
+  setEdgeWeight(BB, TakenIdx, PH_TAKEN_WEIGHT);
+  setEdgeWeight(BB, NonTakenIdx, PH_NONTAKEN_WEIGHT);
   return true;
 }
 
@@ -234,17 +233,17 @@ bool BranchProbabilityInfo::calcLoopBranchHeuristics(BasicBlock *BB) {
   if (!L)
     return false;
 
-  SmallPtrSet<BasicBlock *, 8> BackEdges;
-  SmallPtrSet<BasicBlock *, 8> ExitingEdges;
-  SmallPtrSet<BasicBlock *, 8> InEdges; // Edges from header to the loop.
+  SmallVector<unsigned, 8> BackEdges;
+  SmallVector<unsigned, 8> ExitingEdges;
+  SmallVector<unsigned, 8> InEdges; // Edges from header to the loop.
 
   for (succ_iterator I = succ_begin(BB), E = succ_end(BB); I != E; ++I) {
     if (!L->contains(*I))
-      ExitingEdges.insert(*I);
+      ExitingEdges.push_back(I.getSuccessorIndex());
     else if (L->getHeader() == *I)
-      BackEdges.insert(*I);
+      BackEdges.push_back(I.getSuccessorIndex());
     else
-      InEdges.insert(*I);
+      InEdges.push_back(I.getSuccessorIndex());
   }
 
   if (uint32_t numBackEdges = BackEdges.size()) {
@@ -252,10 +251,9 @@ bool BranchProbabilityInfo::calcLoopBranchHeuristics(BasicBlock *BB) {
     if (backWeight < NORMAL_WEIGHT)
       backWeight = NORMAL_WEIGHT;
 
-    for (SmallPtrSet<BasicBlock *, 8>::iterator EI = BackEdges.begin(),
+    for (SmallVector<unsigned, 8>::iterator EI = BackEdges.begin(),
          EE = BackEdges.end(); EI != EE; ++EI) {
-      BasicBlock *Back = *EI;
-      setEdgeWeight(BB, Back, backWeight);
+      setEdgeWeight(BB, *EI, backWeight);
     }
   }
 
@@ -264,10 +262,9 @@ bool BranchProbabilityInfo::calcLoopBranchHeuristics(BasicBlock *BB) {
     if (inWeight < NORMAL_WEIGHT)
       inWeight = NORMAL_WEIGHT;
 
-    for (SmallPtrSet<BasicBlock *, 8>::iterator EI = InEdges.begin(),
+    for (SmallVector<unsigned, 8>::iterator EI = InEdges.begin(),
          EE = InEdges.end(); EI != EE; ++EI) {
-      BasicBlock *Back = *EI;
-      setEdgeWeight(BB, Back, inWeight);
+      setEdgeWeight(BB, *EI, inWeight);
     }
   }
 
@@ -276,10 +273,9 @@ bool BranchProbabilityInfo::calcLoopBranchHeuristics(BasicBlock *BB) {
     if (exitWeight < MIN_WEIGHT)
       exitWeight = MIN_WEIGHT;
 
-    for (SmallPtrSet<BasicBlock *, 8>::iterator EI = ExitingEdges.begin(),
+    for (SmallVector<unsigned, 8>::iterator EI = ExitingEdges.begin(),
          EE = ExitingEdges.end(); EI != EE; ++EI) {
-      BasicBlock *Exiting = *EI;
-      setEdgeWeight(BB, Exiting, exitWeight);
+      setEdgeWeight(BB, *EI, exitWeight);
     }
   }
 
@@ -335,14 +331,13 @@ bool BranchProbabilityInfo::calcZeroHeuristics(BasicBlock *BB) {
     return false;
   }
 
-  BasicBlock *Taken = BI->getSuccessor(0);
-  BasicBlock *NonTaken = BI->getSuccessor(1);
+  unsigned TakenIdx = 0, NonTakenIdx = 1;
 
   if (!isProb)
-    std::swap(Taken, NonTaken);
+    std::swap(TakenIdx, NonTakenIdx);
 
-  setEdgeWeight(BB, Taken, ZH_TAKEN_WEIGHT);
-  setEdgeWeight(BB, NonTaken, ZH_NONTAKEN_WEIGHT);
+  setEdgeWeight(BB, TakenIdx, ZH_TAKEN_WEIGHT);
+  setEdgeWeight(BB, NonTakenIdx, ZH_NONTAKEN_WEIGHT);
 
   return true;
 }
@@ -372,14 +367,13 @@ bool BranchProbabilityInfo::calcFloatingPointHeuristics(BasicBlock *BB) {
     return false;
   }
 
-  BasicBlock *Taken = BI->getSuccessor(0);
-  BasicBlock *NonTaken = BI->getSuccessor(1);
+  unsigned TakenIdx = 0, NonTakenIdx = 1;
 
   if (!isProb)
-    std::swap(Taken, NonTaken);
+    std::swap(TakenIdx, NonTakenIdx);
 
-  setEdgeWeight(BB, Taken, FPH_TAKEN_WEIGHT);
-  setEdgeWeight(BB, NonTaken, FPH_NONTAKEN_WEIGHT);
+  setEdgeWeight(BB, TakenIdx, FPH_TAKEN_WEIGHT);
+  setEdgeWeight(BB, NonTakenIdx, FPH_NONTAKEN_WEIGHT);
 
   return true;
 }
@@ -389,11 +383,8 @@ bool BranchProbabilityInfo::calcInvokeHeuristics(BasicBlock *BB) {
   if (!II)
     return false;
 
-  BasicBlock *Normal = II->getNormalDest();
-  BasicBlock *Unwind = II->getUnwindDest();
-
-  setEdgeWeight(BB, Normal, IH_TAKEN_WEIGHT);
-  setEdgeWeight(BB, Unwind, IH_NONTAKEN_WEIGHT);
+  setEdgeWeight(BB, 0/*Index for Normal*/, IH_TAKEN_WEIGHT);
+  setEdgeWeight(BB, 1/*Index for Unwind*/, IH_NONTAKEN_WEIGHT);
   return true;
 }
 
@@ -450,8 +441,7 @@ uint32_t BranchProbabilityInfo::getSumForBlock(const BasicBlock *BB) const {
   uint32_t Sum = 0;
 
   for (succ_const_iterator I = succ_begin(BB), E = succ_end(BB); I != E; ++I) {
-    const BasicBlock *Succ = *I;
-    uint32_t Weight = getEdgeWeight(BB, Succ);
+    uint32_t Weight = getEdgeWeight(BB, I.getSuccessorIndex());
     uint32_t PrevSum = Sum;
 
     Sum += Weight;
@@ -494,11 +484,13 @@ BasicBlock *BranchProbabilityInfo::getHotSucc(BasicBlock *BB) const {
   return 0;
 }
 
-// Return edge's weight. If can't find it, return DEFAULT_WEIGHT value.
+/// Get the raw edge weight for the edge. If can't find it, return
+/// DEFAULT_WEIGHT value. Here an edge is specified using PredBlock and an index
+/// to the successors.
 uint32_t BranchProbabilityInfo::
-getEdgeWeight(const BasicBlock *Src, const BasicBlock *Dst) const {
-  Edge E(Src, Dst);
-  DenseMap<Edge, uint32_t>::const_iterator I = Weights.find(E);
+getEdgeWeight(const BasicBlock *Src, unsigned IndexInSuccessors) const {
+  DenseMap<Edge, uint32_t>::const_iterator I =
+      Weights.find(std::make_pair(Src, IndexInSuccessors));
 
   if (I != Weights.end())
     return I->second;
@@ -506,15 +498,43 @@ getEdgeWeight(const BasicBlock *Src, const BasicBlock *Dst) const {
   return DEFAULT_WEIGHT;
 }
 
+/// Get the raw edge weight calculated for the block pair. This returns the sum
+/// of all raw edge weights from Src to Dst.
+uint32_t BranchProbabilityInfo::
+getEdgeWeight(const BasicBlock *Src, const BasicBlock *Dst) const {
+  uint32_t Weight = 0;
+  DenseMap<Edge, uint32_t>::const_iterator MapI;
+  for (succ_const_iterator I = succ_begin(Src), E = succ_end(Src); I != E; ++I)
+    if (*I == Dst) {
+      MapI = Weights.find(std::make_pair(Src, I.getSuccessorIndex()));
+      if (MapI != Weights.end())
+        Weight += MapI->second;
+    }
+  return (Weight == 0) ? DEFAULT_WEIGHT : Weight;
+}
+
+/// Set the edge weight for a given edge specified by PredBlock and an index
+/// to the successors.
 void BranchProbabilityInfo::
-setEdgeWeight(const BasicBlock *Src, const BasicBlock *Dst, uint32_t Weight) {
-  Weights[std::make_pair(Src, Dst)] = Weight;
+setEdgeWeight(const BasicBlock *Src, unsigned IndexInSuccessors,
+              uint32_t Weight) {
+  Weights[std::make_pair(Src, IndexInSuccessors)] = Weight;
   DEBUG(dbgs() << "set edge " << Src->getName() << " -> "
-               << Dst->getName() << " weight to " << Weight
-               << (isEdgeHot(Src, Dst) ? " [is HOT now]\n" : "\n"));
+               << IndexInSuccessors << " successor weight to "
+               << Weight << "\n");
 }
 
+/// Get an edge's probability, relative to other out-edges from Src.
+BranchProbability BranchProbabilityInfo::
+getEdgeProbability(const BasicBlock *Src, unsigned IndexInSuccessors) const {
+  uint32_t N = getEdgeWeight(Src, IndexInSuccessors);
+  uint32_t D = getSumForBlock(Src);
+
+  return BranchProbability(N, D);
+}
 
+/// Get the probability of going from Src to Dst. It returns the sum of all
+/// probabilities for edges from Src to Dst.
 BranchProbability BranchProbabilityInfo::
 getEdgeProbability(const BasicBlock *Src, const BasicBlock *Dst) const {
 
diff --git a/lib/Analysis/CMakeLists.txt b/lib/Analysis/CMakeLists.txt
index 96e68b4..b3a40be 100644
--- a/lib/Analysis/CMakeLists.txt
+++ b/lib/Analysis/CMakeLists.txt
@@ -10,9 +10,11 @@ add_llvm_library(LLVMAnalysis
   BranchProbabilityInfo.cpp
   CFGPrinter.cpp
   CaptureTracking.cpp
+  CostModel.cpp
   CodeMetrics.cpp
   ConstantFolding.cpp
   DbgInfoPrinter.cpp
+  DependenceAnalysis.cpp
   DomPrinter.cpp
   DominanceFrontier.cpp
   IVUsers.cpp
@@ -26,7 +28,6 @@ add_llvm_library(LLVMAnalysis
   LibCallSemantics.cpp
   Lint.cpp
   Loads.cpp
-  LoopDependenceAnalysis.cpp
   LoopInfo.cpp
   LoopPass.cpp
   MemDepPrinter.cpp
@@ -44,6 +45,8 @@ add_llvm_library(LLVMAnalysis
   ProfileInfoLoader.cpp
   ProfileInfoLoaderPass.cpp
   ProfileVerifierPass.cpp
+  ProfileDataLoader.cpp
+  ProfileDataLoaderPass.cpp
   RegionInfo.cpp
   RegionPass.cpp
   RegionPrinter.cpp
diff --git a/lib/Analysis/CaptureTracking.cpp b/lib/Analysis/CaptureTracking.cpp
index 974b906..d9c0299 100644
--- a/lib/Analysis/CaptureTracking.cpp
+++ b/lib/Analysis/CaptureTracking.cpp
@@ -23,6 +23,8 @@ using namespace llvm;
 
 CaptureTracker::~CaptureTracker() {}
 
+bool CaptureTracker::shouldExplore(Use *U) { return true; }
+
 namespace {
   struct SimpleCaptureTracker : public CaptureTracker {
     explicit SimpleCaptureTracker(bool ReturnCaptures)
@@ -30,8 +32,6 @@ namespace {
 
     void tooManyUses() { Captured = true; }
 
-    bool shouldExplore(Use *U) { return true; }
-
     bool captured(Use *U) {
       if (isa<ReturnInst>(U->getUser()) && !ReturnCaptures)
         return false;
diff --git a/lib/Analysis/CodeMetrics.cpp b/lib/Analysis/CodeMetrics.cpp
index acda34b..651a54b 100644
--- a/lib/Analysis/CodeMetrics.cpp
+++ b/lib/Analysis/CodeMetrics.cpp
@@ -15,7 +15,7 @@
 #include "llvm/Function.h"
 #include "llvm/Support/CallSite.h"
 #include "llvm/IntrinsicInst.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 
 using namespace llvm;
 
@@ -54,7 +54,7 @@ bool llvm::callIsSmall(ImmutableCallSite CS) {
   return false;
 }
 
-bool llvm::isInstructionFree(const Instruction *I, const TargetData *TD) {
+bool llvm::isInstructionFree(const Instruction *I, const DataLayout *TD) {
   if (isa<PHINode>(I))
     return true;
 
@@ -119,7 +119,7 @@ bool llvm::isInstructionFree(const Instruction *I, const TargetData *TD) {
 /// analyzeBasicBlock - Fill in the current structure with information gleaned
 /// from the specified block.
 void CodeMetrics::analyzeBasicBlock(const BasicBlock *BB,
-                                    const TargetData *TD) {
+                                    const DataLayout *TD) {
   ++NumBlocks;
   unsigned NumInstsBeforeThisBB = NumInsts;
   for (BasicBlock::const_iterator II = BB->begin(), E = BB->end();
@@ -189,14 +189,14 @@ void CodeMetrics::analyzeBasicBlock(const BasicBlock *BB,
   NumBBInsts[BB] = NumInsts - NumInstsBeforeThisBB;
 }
 
-void CodeMetrics::analyzeFunction(Function *F, const TargetData *TD) {
+void CodeMetrics::analyzeFunction(Function *F, const DataLayout *TD) {
   // If this function contains a call that "returns twice" (e.g., setjmp or
   // _setjmp) and it isn't marked with "returns twice" itself, never inline it.
   // This is a hack because we depend on the user marking their local variables
   // as volatile if they are live across a setjmp call, and they probably
   // won't do this in callers.
   exposesReturnsTwice = F->callsFunctionThatReturnsTwice() &&
-    !F->hasFnAttr(Attribute::ReturnsTwice);
+    !F->getFnAttributes().hasAttribute(Attributes::ReturnsTwice);
 
   // Look at the size of the callee.
   for (Function::const_iterator BB = F->begin(), E = F->end(); BB != E; ++BB)
diff --git a/lib/Analysis/ConstantFolding.cpp b/lib/Analysis/ConstantFolding.cpp
index f5e619c..91a5b84 100644
--- a/lib/Analysis/ConstantFolding.cpp
+++ b/lib/Analysis/ConstantFolding.cpp
@@ -11,7 +11,7 @@
 //
 // Also, to supplement the basic VMCore ConstantExpr simplifications,
 // this file defines some additional folding routines that can make use of
-// TargetData information. These functions cannot go in VMCore due to library
+// DataLayout information. These functions cannot go in VMCore due to library
 // dependency issues.
 //
 //===----------------------------------------------------------------------===//
@@ -25,7 +25,7 @@
 #include "llvm/Intrinsics.h"
 #include "llvm/Operator.h"
 #include "llvm/Analysis/ValueTracking.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetLibraryInfo.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringMap.h"
@@ -41,11 +41,11 @@ using namespace llvm;
 // Constant Folding internal helper functions
 //===----------------------------------------------------------------------===//
 
-/// FoldBitCast - Constant fold bitcast, symbolically evaluating it with 
-/// TargetData.  This always returns a non-null constant, but it may be a
+/// FoldBitCast - Constant fold bitcast, symbolically evaluating it with
+/// DataLayout.  This always returns a non-null constant, but it may be a
 /// ConstantExpr if unfoldable.
 static Constant *FoldBitCast(Constant *C, Type *DestTy,
-                             const TargetData &TD) {
+                             const DataLayout &TD) {
   // Catch the obvious splat cases.
   if (C->isNullValue() && !DestTy->isX86_MMXTy())
     return Constant::getNullValue(DestTy);
@@ -59,9 +59,9 @@ static Constant *FoldBitCast(Constant *C, Type *DestTy,
       return ConstantExpr::getBitCast(C, DestTy);
 
     unsigned NumSrcElts = CDV->getType()->getNumElements();
-    
+
     Type *SrcEltTy = CDV->getType()->getElementType();
-    
+
     // If the vector is a vector of floating point, convert it to vector of int
     // to simplify things.
     if (SrcEltTy->isFloatingPointTy()) {
@@ -72,7 +72,7 @@ static Constant *FoldBitCast(Constant *C, Type *DestTy,
       C = ConstantExpr::getBitCast(C, SrcIVTy);
       CDV = cast<ConstantDataVector>(C);
     }
-    
+
     // Now that we know that the input value is a vector of integers, just shift
     // and insert them into our result.
     unsigned BitShift = TD.getTypeAllocSizeInBits(SrcEltTy);
@@ -84,43 +84,43 @@ static Constant *FoldBitCast(Constant *C, Type *DestTy,
       else
         Result |= CDV->getElementAsInteger(i);
     }
-   
+
     return ConstantInt::get(IT, Result);
   }
-  
+
   // The code below only handles casts to vectors currently.
   VectorType *DestVTy = dyn_cast<VectorType>(DestTy);
   if (DestVTy == 0)
     return ConstantExpr::getBitCast(C, DestTy);
-  
+
   // If this is a scalar -> vector cast, convert the input into a <1 x scalar>
   // vector so the code below can handle it uniformly.
   if (isa<ConstantFP>(C) || isa<ConstantInt>(C)) {
     Constant *Ops = C; // don't take the address of C!
     return FoldBitCast(ConstantVector::get(Ops), DestTy, TD);
   }
-  
+
   // If this is a bitcast from constant vector -> vector, fold it.
   if (!isa<ConstantDataVector>(C) && !isa<ConstantVector>(C))
     return ConstantExpr::getBitCast(C, DestTy);
-  
+
   // If the element types match, VMCore can fold it.
   unsigned NumDstElt = DestVTy->getNumElements();
   unsigned NumSrcElt = C->getType()->getVectorNumElements();
   if (NumDstElt == NumSrcElt)
     return ConstantExpr::getBitCast(C, DestTy);
-  
+
   Type *SrcEltTy = C->getType()->getVectorElementType();
   Type *DstEltTy = DestVTy->getElementType();
-  
-  // Otherwise, we're changing the number of elements in a vector, which 
+
+  // Otherwise, we're changing the number of elements in a vector, which
   // requires endianness information to do the right thing.  For example,
   //    bitcast (<2 x i64> <i64 0, i64 1> to <4 x i32>)
   // folds to (little endian):
   //    <4 x i32> <i32 0, i32 0, i32 1, i32 0>
   // and to (big endian):
   //    <4 x i32> <i32 0, i32 0, i32 0, i32 1>
-  
+
   // First thing is first.  We only want to think about integer here, so if
   // we have something in FP form, recast it as integer.
   if (DstEltTy->isFloatingPointTy()) {
@@ -130,11 +130,11 @@ static Constant *FoldBitCast(Constant *C, Type *DestTy,
       VectorType::get(IntegerType::get(C->getContext(), FPWidth), NumDstElt);
     // Recursively handle this integer conversion, if possible.
     C = FoldBitCast(C, DestIVTy, TD);
-    
+
     // Finally, VMCore can handle this now that #elts line up.
     return ConstantExpr::getBitCast(C, DestTy);
   }
-  
+
   // Okay, we know the destination is integer, if the input is FP, convert
   // it to integer first.
   if (SrcEltTy->isFloatingPointTy()) {
@@ -148,13 +148,13 @@ static Constant *FoldBitCast(Constant *C, Type *DestTy,
         !isa<ConstantDataVector>(C))
       return C;
   }
-  
+
   // Now we know that the input and output vectors are both integer vectors
   // of the same size, and that their #elements is not the same.  Do the
   // conversion here, which depends on whether the input or output has
   // more elements.
   bool isLittleEndian = TD.isLittleEndian();
-  
+
   SmallVector<Constant*, 32> Result;
   if (NumDstElt < NumSrcElt) {
     // Handle: bitcast (<4 x i32> <i32 0, i32 1, i32 2, i32 3> to <2 x i64>)
@@ -170,15 +170,15 @@ static Constant *FoldBitCast(Constant *C, Type *DestTy,
         Constant *Src =dyn_cast<ConstantInt>(C->getAggregateElement(SrcElt++));
         if (!Src)  // Reject constantexpr elements.
           return ConstantExpr::getBitCast(C, DestTy);
-        
+
         // Zero extend the element to the right size.
         Src = ConstantExpr::getZExt(Src, Elt->getType());
-        
+
         // Shift it to the right place, depending on endianness.
-        Src = ConstantExpr::getShl(Src, 
+        Src = ConstantExpr::getShl(Src,
                                    ConstantInt::get(Src->getType(), ShiftAmt));
         ShiftAmt += isLittleEndian ? SrcBitSize : -SrcBitSize;
-        
+
         // Mix it in.
         Elt = ConstantExpr::getOr(Elt, Src);
       }
@@ -186,30 +186,30 @@ static Constant *FoldBitCast(Constant *C, Type *DestTy,
     }
     return ConstantVector::get(Result);
   }
-  
+
   // Handle: bitcast (<2 x i64> <i64 0, i64 1> to <4 x i32>)
   unsigned Ratio = NumDstElt/NumSrcElt;
   unsigned DstBitSize = DstEltTy->getPrimitiveSizeInBits();
-  
+
   // Loop over each source value, expanding into multiple results.
   for (unsigned i = 0; i != NumSrcElt; ++i) {
     Constant *Src = dyn_cast<ConstantInt>(C->getAggregateElement(i));
     if (!Src)  // Reject constantexpr elements.
       return ConstantExpr::getBitCast(C, DestTy);
-    
+
     unsigned ShiftAmt = isLittleEndian ? 0 : DstBitSize*(Ratio-1);
     for (unsigned j = 0; j != Ratio; ++j) {
       // Shift the piece of the value into the right place, depending on
       // endianness.
-      Constant *Elt = ConstantExpr::getLShr(Src, 
+      Constant *Elt = ConstantExpr::getLShr(Src,
                                   ConstantInt::get(Src->getType(), ShiftAmt));
       ShiftAmt += isLittleEndian ? DstBitSize : -DstBitSize;
-      
+
       // Truncate and remember this piece.
       Result.push_back(ConstantExpr::getTrunc(Elt, DstEltTy));
     }
   }
-  
+
   return ConstantVector::get(Result);
 }
 
@@ -218,34 +218,34 @@ static Constant *FoldBitCast(Constant *C, Type *DestTy,
 /// from a global, return the global and the constant.  Because of
 /// constantexprs, this function is recursive.
 static bool IsConstantOffsetFromGlobal(Constant *C, GlobalValue *&GV,
-                                       int64_t &Offset, const TargetData &TD) {
+                                       int64_t &Offset, const DataLayout &TD) {
   // Trivial case, constant is the global.
   if ((GV = dyn_cast<GlobalValue>(C))) {
     Offset = 0;
     return true;
   }
-  
+
   // Otherwise, if this isn't a constant expr, bail out.
   ConstantExpr *CE = dyn_cast<ConstantExpr>(C);
   if (!CE) return false;
-  
+
   // Look through ptr->int and ptr->ptr casts.
   if (CE->getOpcode() == Instruction::PtrToInt ||
       CE->getOpcode() == Instruction::BitCast)
     return IsConstantOffsetFromGlobal(CE->getOperand(0), GV, Offset, TD);
-  
-  // i32* getelementptr ([5 x i32]* @a, i32 0, i32 5)    
+
+  // i32* getelementptr ([5 x i32]* @a, i32 0, i32 5)
   if (CE->getOpcode() == Instruction::GetElementPtr) {
     // Cannot compute this if the element type of the pointer is missing size
     // info.
     if (!cast<PointerType>(CE->getOperand(0)->getType())
                  ->getElementType()->isSized())
       return false;
-    
+
     // If the base isn't a global+constant, we aren't either.
     if (!IsConstantOffsetFromGlobal(CE->getOperand(0), GV, Offset, TD))
       return false;
-    
+
     // Otherwise, add any offset that our operands provide.
     gep_type_iterator GTI = gep_type_begin(CE);
     for (User::const_op_iterator i = CE->op_begin() + 1, e = CE->op_end();
@@ -253,7 +253,7 @@ static bool IsConstantOffsetFromGlobal(Constant *C, GlobalValue *&GV,
       ConstantInt *CI = dyn_cast<ConstantInt>(*i);
       if (!CI) return false;  // Index isn't a simple constant?
       if (CI->isZero()) continue;  // Not adding anything.
-      
+
       if (StructType *ST = dyn_cast<StructType>(*GTI)) {
         // N = N + Offset
         Offset += TD.getStructLayout(ST)->getElementOffset(CI->getZExtValue());
@@ -264,7 +264,7 @@ static bool IsConstantOffsetFromGlobal(Constant *C, GlobalValue *&GV,
     }
     return true;
   }
-  
+
   return false;
 }
 
@@ -274,30 +274,33 @@ static bool IsConstantOffsetFromGlobal(Constant *C, GlobalValue *&GV,
 /// the CurPtr buffer.  TD is the target data.
 static bool ReadDataFromGlobal(Constant *C, uint64_t ByteOffset,
                                unsigned char *CurPtr, unsigned BytesLeft,
-                               const TargetData &TD) {
+                               const DataLayout &TD) {
   assert(ByteOffset <= TD.getTypeAllocSize(C->getType()) &&
          "Out of range access");
-  
+
   // If this element is zero or undefined, we can just return since *CurPtr is
   // zero initialized.
   if (isa<ConstantAggregateZero>(C) || isa<UndefValue>(C))
     return true;
-  
+
   if (ConstantInt *CI = dyn_cast<ConstantInt>(C)) {
     if (CI->getBitWidth() > 64 ||
         (CI->getBitWidth() & 7) != 0)
       return false;
-    
+
     uint64_t Val = CI->getZExtValue();
     unsigned IntBytes = unsigned(CI->getBitWidth()/8);
-    
+
     for (unsigned i = 0; i != BytesLeft && ByteOffset != IntBytes; ++i) {
-      CurPtr[i] = (unsigned char)(Val >> (ByteOffset * 8));
+      int n = ByteOffset;
+      if (!TD.isLittleEndian())
+        n = IntBytes - n - 1;
+      CurPtr[i] = (unsigned char)(Val >> (n * 8));
       ++ByteOffset;
     }
     return true;
   }
-  
+
   if (ConstantFP *CFP = dyn_cast<ConstantFP>(C)) {
     if (CFP->getType()->isDoubleTy()) {
       C = FoldBitCast(C, Type::getInt64Ty(C->getContext()), TD);
@@ -309,13 +312,13 @@ static bool ReadDataFromGlobal(Constant *C, uint64_t ByteOffset,
     }
     return false;
   }
-  
+
   if (ConstantStruct *CS = dyn_cast<ConstantStruct>(C)) {
     const StructLayout *SL = TD.getStructLayout(CS->getType());
     unsigned Index = SL->getElementContainingOffset(ByteOffset);
     uint64_t CurEltOffset = SL->getElementOffset(Index);
     ByteOffset -= CurEltOffset;
-    
+
     while (1) {
       // If the element access is to the element itself and not to tail padding,
       // read the bytes from the element.
@@ -325,9 +328,9 @@ static bool ReadDataFromGlobal(Constant *C, uint64_t ByteOffset,
           !ReadDataFromGlobal(CS->getOperand(Index), ByteOffset, CurPtr,
                               BytesLeft, TD))
         return false;
-      
+
       ++Index;
-      
+
       // Check to see if we read from the last struct element, if so we're done.
       if (Index == CS->getType()->getNumElements())
         return true;
@@ -375,11 +378,11 @@ static bool ReadDataFromGlobal(Constant *C, uint64_t ByteOffset,
     }
     return true;
   }
-      
+
   if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C)) {
     if (CE->getOpcode() == Instruction::IntToPtr &&
-        CE->getOperand(0)->getType() == TD.getIntPtrType(CE->getContext())) 
-      return ReadDataFromGlobal(CE->getOperand(0), ByteOffset, CurPtr, 
+        CE->getOperand(0)->getType() == TD.getIntPtrType(CE->getContext()))
+      return ReadDataFromGlobal(CE->getOperand(0), ByteOffset, CurPtr,
                                 BytesLeft, TD);
   }
 
@@ -388,10 +391,10 @@ static bool ReadDataFromGlobal(Constant *C, uint64_t ByteOffset,
 }
 
 static Constant *FoldReinterpretLoadFromConstPtr(Constant *C,
-                                                 const TargetData &TD) {
+                                                 const DataLayout &TD) {
   Type *LoadTy = cast<PointerType>(C->getType())->getElementType();
   IntegerType *IntType = dyn_cast<IntegerType>(LoadTy);
-  
+
   // If this isn't an integer load we can't fold it directly.
   if (!IntType) {
     // If this is a float/double load, we can try folding it as an int32/64 load
@@ -415,15 +418,15 @@ static Constant *FoldReinterpretLoadFromConstPtr(Constant *C,
       return FoldBitCast(Res, LoadTy, TD);
     return 0;
   }
-  
+
   unsigned BytesLoaded = (IntType->getBitWidth() + 7) / 8;
   if (BytesLoaded > 32 || BytesLoaded == 0) return 0;
-  
+
   GlobalValue *GVal;
   int64_t Offset;
   if (!IsConstantOffsetFromGlobal(C, GVal, Offset, TD))
     return 0;
-  
+
   GlobalVariable *GV = dyn_cast<GlobalVariable>(GVal);
   if (!GV || !GV->isConstant() || !GV->hasDefinitiveInitializer() ||
       !GV->getInitializer()->getType()->isSized())
@@ -432,20 +435,29 @@ static Constant *FoldReinterpretLoadFromConstPtr(Constant *C,
   // If we're loading off the beginning of the global, some bytes may be valid,
   // but we don't try to handle this.
   if (Offset < 0) return 0;
-  
+
   // If we're not accessing anything in this constant, the result is undefined.
   if (uint64_t(Offset) >= TD.getTypeAllocSize(GV->getInitializer()->getType()))
     return UndefValue::get(IntType);
-  
+
   unsigned char RawBytes[32] = {0};
   if (!ReadDataFromGlobal(GV->getInitializer(), Offset, RawBytes,
                           BytesLoaded, TD))
     return 0;
 
-  APInt ResultVal = APInt(IntType->getBitWidth(), RawBytes[BytesLoaded-1]);
-  for (unsigned i = 1; i != BytesLoaded; ++i) {
-    ResultVal <<= 8;
-    ResultVal |= RawBytes[BytesLoaded-1-i];
+  APInt ResultVal = APInt(IntType->getBitWidth(), 0);
+  if (TD.isLittleEndian()) {
+    ResultVal = RawBytes[BytesLoaded - 1];
+    for (unsigned i = 1; i != BytesLoaded; ++i) {
+      ResultVal <<= 8;
+      ResultVal |= RawBytes[BytesLoaded-1-i];
+    }
+  } else {
+    ResultVal = RawBytes[0];
+    for (unsigned i = 1; i != BytesLoaded; ++i) {
+      ResultVal <<= 8;
+      ResultVal |= RawBytes[i];
+    }
   }
 
   return ConstantInt::get(IntType->getContext(), ResultVal);
@@ -455,7 +467,7 @@ static Constant *FoldReinterpretLoadFromConstPtr(Constant *C,
 /// produce if it is constant and determinable.  If this is not determinable,
 /// return null.
 Constant *llvm::ConstantFoldLoadFromConstPtr(Constant *C,
-                                             const TargetData *TD) {
+                                             const DataLayout *TD) {
   // First, try the easy cases:
   if (GlobalVariable *GV = dyn_cast<GlobalVariable>(C))
     if (GV->isConstant() && GV->hasDefinitiveInitializer())
@@ -464,15 +476,15 @@ Constant *llvm::ConstantFoldLoadFromConstPtr(Constant *C,
   // If the loaded value isn't a constant expr, we can't handle it.
   ConstantExpr *CE = dyn_cast<ConstantExpr>(C);
   if (!CE) return 0;
-  
+
   if (CE->getOpcode() == Instruction::GetElementPtr) {
     if (GlobalVariable *GV = dyn_cast<GlobalVariable>(CE->getOperand(0)))
       if (GV->isConstant() && GV->hasDefinitiveInitializer())
-        if (Constant *V = 
+        if (Constant *V =
              ConstantFoldLoadThroughGEPConstantExpr(GV->getInitializer(), CE))
           return V;
   }
-  
+
   // Instead of loading constant c string, use corresponding integer value
   // directly if string length is small enough.
   StringRef Str;
@@ -500,14 +512,14 @@ Constant *llvm::ConstantFoldLoadFromConstPtr(Constant *C,
         SingleChar = 0;
         StrVal = (StrVal << 8) | SingleChar;
       }
-      
+
       Constant *Res = ConstantInt::get(CE->getContext(), StrVal);
       if (Ty->isFloatingPointTy())
         Res = ConstantExpr::getBitCast(Res, Ty);
       return Res;
     }
   }
-  
+
   // If this load comes from anywhere in a constant global, and if the global
   // is all undef or zero, we know what it loads.
   if (GlobalVariable *GV =
@@ -520,18 +532,16 @@ Constant *llvm::ConstantFoldLoadFromConstPtr(Constant *C,
         return UndefValue::get(ResTy);
     }
   }
-  
-  // Try hard to fold loads from bitcasted strange and non-type-safe things.  We
-  // currently don't do any of this for big endian systems.  It can be
-  // generalized in the future if someone is interested.
-  if (TD && TD->isLittleEndian())
+
+  // Try hard to fold loads from bitcasted strange and non-type-safe things.
+  if (TD)
     return FoldReinterpretLoadFromConstPtr(CE, *TD);
   return 0;
 }
 
-static Constant *ConstantFoldLoadInst(const LoadInst *LI, const TargetData *TD){
+static Constant *ConstantFoldLoadInst(const LoadInst *LI, const DataLayout *TD){
   if (LI->isVolatile()) return 0;
-  
+
   if (Constant *C = dyn_cast<Constant>(LI->getOperand(0)))
     return ConstantFoldLoadFromConstPtr(C, TD);
 
@@ -540,23 +550,23 @@ static Constant *ConstantFoldLoadInst(const LoadInst *LI, const TargetData *TD){
 
 /// SymbolicallyEvaluateBinop - One of Op0/Op1 is a constant expression.
 /// Attempt to symbolically evaluate the result of a binary operator merging
-/// these together.  If target data info is available, it is provided as TD, 
+/// these together.  If target data info is available, it is provided as TD,
 /// otherwise TD is null.
 static Constant *SymbolicallyEvaluateBinop(unsigned Opc, Constant *Op0,
-                                           Constant *Op1, const TargetData *TD){
+                                           Constant *Op1, const DataLayout *TD){
   // SROA
-  
+
   // Fold (and 0xffffffff00000000, (shl x, 32)) -> shl.
   // Fold (lshr (or X, Y), 32) -> (lshr [X/Y], 32) if one doesn't contribute
   // bits.
-  
-  
+
+
   // If the constant expr is something like &A[123] - &A[4].f, fold this into a
   // constant.  This happens frequently when iterating over a global array.
   if (Opc == Instruction::Sub && TD) {
     GlobalValue *GV1, *GV2;
     int64_t Offs1, Offs2;
-    
+
     if (IsConstantOffsetFromGlobal(Op0, GV1, Offs1, *TD))
       if (IsConstantOffsetFromGlobal(Op1, GV2, Offs2, *TD) &&
           GV1 == GV2) {
@@ -564,7 +574,7 @@ static Constant *SymbolicallyEvaluateBinop(unsigned Opc, Constant *Op0,
         return ConstantInt::get(Op0->getType(), Offs1-Offs2);
       }
   }
-    
+
   return 0;
 }
 
@@ -572,7 +582,7 @@ static Constant *SymbolicallyEvaluateBinop(unsigned Opc, Constant *Op0,
 /// explicitly cast them so that they aren't implicitly casted by the
 /// getelementptr.
 static Constant *CastGEPIndices(ArrayRef<Constant *> Ops,
-                                Type *ResultTy, const TargetData *TD,
+                                Type *ResultTy, const DataLayout *TD,
                                 const TargetLibraryInfo *TLI) {
   if (!TD) return 0;
   Type *IntPtrTy = TD->getIntPtrType(ResultTy->getContext());
@@ -622,20 +632,20 @@ static Constant* StripPtrCastKeepAS(Constant* Ptr) {
 /// SymbolicallyEvaluateGEP - If we can symbolically evaluate the specified GEP
 /// constant expression, do so.
 static Constant *SymbolicallyEvaluateGEP(ArrayRef<Constant *> Ops,
-                                         Type *ResultTy, const TargetData *TD,
+                                         Type *ResultTy, const DataLayout *TD,
                                          const TargetLibraryInfo *TLI) {
   Constant *Ptr = Ops[0];
   if (!TD || !cast<PointerType>(Ptr->getType())->getElementType()->isSized() ||
       !Ptr->getType()->isPointerTy())
     return 0;
-  
+
   Type *IntPtrTy = TD->getIntPtrType(Ptr->getContext());
 
   // If this is a constant expr gep that is effectively computing an
   // "offsetof", fold it into 'cast int Size to T*' instead of 'gep 0, 0, 12'
   for (unsigned i = 1, e = Ops.size(); i != e; ++i)
     if (!isa<ConstantInt>(Ops[i])) {
-      
+
       // If this is "gep i8* Ptr, (sub 0, V)", fold this as:
       // "inttoptr (sub (ptrtoint Ptr), V)"
       if (Ops.size() == 2 &&
@@ -659,7 +669,8 @@ static Constant *SymbolicallyEvaluateGEP(ArrayRef<Constant *> Ops,
   unsigned BitWidth = TD->getTypeSizeInBits(IntPtrTy);
   APInt Offset =
     APInt(BitWidth, TD->getIndexedOffset(Ptr->getType(),
-                                         makeArrayRef((Value **)Ops.data() + 1,
+                                         makeArrayRef((Value *const*)
+                                                        Ops.data() + 1,
                                                       Ops.size() - 1)));
   Ptr = StripPtrCastKeepAS(Ptr);
 
@@ -708,12 +719,12 @@ static Constant *SymbolicallyEvaluateGEP(ArrayRef<Constant *> Ops,
         // The only pointer indexing we'll do is on the first index of the GEP.
         if (!NewIdxs.empty())
           break;
-       
+
         // Only handle pointers to sized types, not pointers to functions.
         if (!ATy->getElementType()->isSized())
           return 0;
       }
-        
+
       // Determine which element of the array the offset points into.
       APInt ElemSize(BitWidth, TD->getTypeAllocSize(ATy->getElementType()));
       IntegerType *IntPtrTy = TD->getIntPtrType(Ty->getContext());
@@ -785,7 +796,7 @@ static Constant *SymbolicallyEvaluateGEP(ArrayRef<Constant *> Ops,
 /// this function can only fail when attempting to fold instructions like loads
 /// and stores, which have no constant expression form.
 Constant *llvm::ConstantFoldInstruction(Instruction *I,
-                                        const TargetData *TD,
+                                        const DataLayout *TD,
                                         const TargetLibraryInfo *TLI) {
   // Handle PHI nodes quickly here...
   if (PHINode *PN = dyn_cast<PHINode>(I)) {
@@ -836,7 +847,7 @@ Constant *llvm::ConstantFoldInstruction(Instruction *I,
   if (const CmpInst *CI = dyn_cast<CmpInst>(I))
     return ConstantFoldCompareInstOperands(CI->getPredicate(), Ops[0], Ops[1],
                                            TD, TLI);
-  
+
   if (const LoadInst *LI = dyn_cast<LoadInst>(I))
     return ConstantFoldLoadInst(LI, TD);
 
@@ -855,10 +866,10 @@ Constant *llvm::ConstantFoldInstruction(Instruction *I,
 }
 
 /// ConstantFoldConstantExpression - Attempt to fold the constant expression
-/// using the specified TargetData.  If successful, the constant result is
+/// using the specified DataLayout.  If successful, the constant result is
 /// result is returned, if not, null is returned.
 Constant *llvm::ConstantFoldConstantExpression(const ConstantExpr *CE,
-                                               const TargetData *TD,
+                                               const DataLayout *TD,
                                                const TargetLibraryInfo *TLI) {
   SmallVector<Constant*, 8> Ops;
   for (User::const_op_iterator i = CE->op_begin(), e = CE->op_end();
@@ -886,19 +897,19 @@ Constant *llvm::ConstantFoldConstantExpression(const ConstantExpr *CE,
 /// information, due to only being passed an opcode and operands. Constant
 /// folding using this function strips this information.
 ///
-Constant *llvm::ConstantFoldInstOperands(unsigned Opcode, Type *DestTy, 
+Constant *llvm::ConstantFoldInstOperands(unsigned Opcode, Type *DestTy,
                                          ArrayRef<Constant *> Ops,
-                                         const TargetData *TD,
-                                         const TargetLibraryInfo *TLI) {                                         
+                                         const DataLayout *TD,
+                                         const TargetLibraryInfo *TLI) {
   // Handle easy binops first.
   if (Instruction::isBinaryOp(Opcode)) {
     if (isa<ConstantExpr>(Ops[0]) || isa<ConstantExpr>(Ops[1]))
       if (Constant *C = SymbolicallyEvaluateBinop(Opcode, Ops[0], Ops[1], TD))
         return C;
-    
+
     return ConstantExpr::get(Opcode, Ops[0], Ops[1]);
   }
-  
+
   switch (Opcode) {
   default: return 0;
   case Instruction::ICmp:
@@ -916,7 +927,7 @@ Constant *llvm::ConstantFoldInstOperands(unsigned Opcode, Type *DestTy,
         Constant *Input = CE->getOperand(0);
         unsigned InWidth = Input->getType()->getScalarSizeInBits();
         if (TD->getPointerSizeInBits() < InWidth) {
-          Constant *Mask = 
+          Constant *Mask =
             ConstantInt::get(CE->getContext(), APInt::getLowBitsSet(InWidth,
                                                   TD->getPointerSizeInBits()));
           Input = ConstantExpr::getAnd(Input, Mask);
@@ -964,7 +975,7 @@ Constant *llvm::ConstantFoldInstOperands(unsigned Opcode, Type *DestTy,
       return C;
     if (Constant *C = SymbolicallyEvaluateGEP(Ops, DestTy, TD, TLI))
       return C;
-    
+
     return ConstantExpr::getGetElementPtr(Ops[0], Ops.slice(1));
   }
 }
@@ -974,8 +985,8 @@ Constant *llvm::ConstantFoldInstOperands(unsigned Opcode, Type *DestTy,
 /// returns a constant expression of the specified operands.
 ///
 Constant *llvm::ConstantFoldCompareInstOperands(unsigned Predicate,
-                                                Constant *Ops0, Constant *Ops1, 
-                                                const TargetData *TD,
+                                                Constant *Ops0, Constant *Ops1,
+                                                const DataLayout *TD,
                                                 const TargetLibraryInfo *TLI) {
   // fold: icmp (inttoptr x), null         -> icmp x, 0
   // fold: icmp (ptrtoint x), 0            -> icmp x, null
@@ -995,17 +1006,17 @@ Constant *llvm::ConstantFoldCompareInstOperands(unsigned Predicate,
         Constant *Null = Constant::getNullValue(C->getType());
         return ConstantFoldCompareInstOperands(Predicate, C, Null, TD, TLI);
       }
-      
+
       // Only do this transformation if the int is intptrty in size, otherwise
       // there is a truncation or extension that we aren't modeling.
-      if (CE0->getOpcode() == Instruction::PtrToInt && 
+      if (CE0->getOpcode() == Instruction::PtrToInt &&
           CE0->getType() == IntPtrTy) {
         Constant *C = CE0->getOperand(0);
         Constant *Null = Constant::getNullValue(C->getType());
         return ConstantFoldCompareInstOperands(Predicate, C, Null, TD, TLI);
       }
     }
-    
+
     if (ConstantExpr *CE1 = dyn_cast<ConstantExpr>(Ops1)) {
       if (TD && CE0->getOpcode() == CE1->getOpcode()) {
         Type *IntPtrTy = TD->getIntPtrType(CE0->getContext());
@@ -1029,24 +1040,24 @@ Constant *llvm::ConstantFoldCompareInstOperands(unsigned Predicate,
                                                  CE1->getOperand(0), TD, TLI);
       }
     }
-    
+
     // icmp eq (or x, y), 0 -> (icmp eq x, 0) & (icmp eq y, 0)
     // icmp ne (or x, y), 0 -> (icmp ne x, 0) | (icmp ne y, 0)
     if ((Predicate == ICmpInst::ICMP_EQ || Predicate == ICmpInst::ICMP_NE) &&
         CE0->getOpcode() == Instruction::Or && Ops1->isNullValue()) {
-      Constant *LHS = 
+      Constant *LHS =
         ConstantFoldCompareInstOperands(Predicate, CE0->getOperand(0), Ops1,
                                         TD, TLI);
-      Constant *RHS = 
+      Constant *RHS =
         ConstantFoldCompareInstOperands(Predicate, CE0->getOperand(1), Ops1,
                                         TD, TLI);
-      unsigned OpC = 
+      unsigned OpC =
         Predicate == ICmpInst::ICMP_EQ ? Instruction::And : Instruction::Or;
       Constant *Ops[] = { LHS, RHS };
       return ConstantFoldInstOperands(OpC, LHS->getType(), Ops, TD, TLI);
     }
   }
-  
+
   return ConstantExpr::getCompare(Predicate, Ops0, Ops1);
 }
 
@@ -1054,7 +1065,7 @@ Constant *llvm::ConstantFoldCompareInstOperands(unsigned Predicate,
 /// ConstantFoldLoadThroughGEPConstantExpr - Given a constant and a
 /// getelementptr constantexpr, return the constant value being addressed by the
 /// constant expression, or null if something is funny and we can't decide.
-Constant *llvm::ConstantFoldLoadThroughGEPConstantExpr(Constant *C, 
+Constant *llvm::ConstantFoldLoadThroughGEPConstantExpr(Constant *C,
                                                        ConstantExpr *CE) {
   if (!CE->getOperand(1)->isNullValue())
     return 0;  // Do not allow stepping over the value!
@@ -1124,14 +1135,14 @@ llvm::canConstantFoldCallTo(const Function *F) {
 
   if (!F->hasName()) return false;
   StringRef Name = F->getName();
-  
+
   // In these cases, the check of the length is required.  We don't want to
   // return true for a name like "cos\0blah" which strcmp would return equal to
   // "cos", but has length 8.
   switch (Name[0]) {
   default: return false;
   case 'a':
-    return Name == "acos" || Name == "asin" || 
+    return Name == "acos" || Name == "asin" ||
       Name == "atan" || Name == "atan2";
   case 'c':
     return Name == "cos" || Name == "ceil" || Name == "cosf" || Name == "cosh";
@@ -1151,7 +1162,7 @@ llvm::canConstantFoldCallTo(const Function *F) {
   }
 }
 
-static Constant *ConstantFoldFP(double (*NativeFP)(double), double V, 
+static Constant *ConstantFoldFP(double (*NativeFP)(double), double V,
                                 Type *Ty) {
   sys::llvm_fenv_clearexcept();
   V = NativeFP(V);
@@ -1159,7 +1170,7 @@ static Constant *ConstantFoldFP(double (*NativeFP)(double), double V,
     sys::llvm_fenv_clearexcept();
     return 0;
   }
-  
+
   if (Ty->isFloatTy())
     return ConstantFP::get(Ty->getContext(), APFloat((float)V));
   if (Ty->isDoubleTy())
@@ -1175,7 +1186,7 @@ static Constant *ConstantFoldBinaryFP(double (*NativeFP)(double, double),
     sys::llvm_fenv_clearexcept();
     return 0;
   }
-  
+
   if (Ty->isFloatTy())
     return ConstantFP::get(Ty->getContext(), APFloat((float)V));
   if (Ty->isDoubleTy())
@@ -1269,7 +1280,7 @@ llvm::ConstantFoldCall(Function *F, ArrayRef<Constant *> Operands,
       case 'e':
         if (Name == "exp" && TLI->has(LibFunc::exp))
           return ConstantFoldFP(exp, V, Ty);
-  
+
         if (Name == "exp2" && TLI->has(LibFunc::exp2)) {
           // Constant fold exp2(x) as pow(2,x) in case the host doesn't have a
           // C99 library.
@@ -1345,7 +1356,7 @@ llvm::ConstantFoldCall(Function *F, ArrayRef<Constant *> Operands,
     }
 
     // Support ConstantVector in case we have an Undef in the top.
-    if (isa<ConstantVector>(Operands[0]) || 
+    if (isa<ConstantVector>(Operands[0]) ||
         isa<ConstantDataVector>(Operands[0])) {
       Constant *Op = cast<Constant>(Operands[0]);
       switch (F->getIntrinsicID()) {
@@ -1364,11 +1375,11 @@ llvm::ConstantFoldCall(Function *F, ArrayRef<Constant *> Operands,
       case Intrinsic::x86_sse2_cvttsd2si64:
         if (ConstantFP *FPOp =
               dyn_cast_or_null<ConstantFP>(Op->getAggregateElement(0U)))
-          return ConstantFoldConvertToInt(FPOp->getValueAPF(), 
+          return ConstantFoldConvertToInt(FPOp->getValueAPF(),
                                           /*roundTowardZero=*/true, Ty);
       }
     }
-  
+
     if (isa<UndefValue>(Operands[0])) {
       if (F->getIntrinsicID() == Intrinsic::bswap)
         return Operands[0];
@@ -1382,14 +1393,14 @@ llvm::ConstantFoldCall(Function *F, ArrayRef<Constant *> Operands,
     if (ConstantFP *Op1 = dyn_cast<ConstantFP>(Operands[0])) {
       if (!Ty->isFloatTy() && !Ty->isDoubleTy())
         return 0;
-      double Op1V = Ty->isFloatTy() ? 
+      double Op1V = Ty->isFloatTy() ?
                       (double)Op1->getValueAPF().convertToFloat() :
                       Op1->getValueAPF().convertToDouble();
       if (ConstantFP *Op2 = dyn_cast<ConstantFP>(Operands[1])) {
         if (Op2->getType() != Op1->getType())
           return 0;
 
-        double Op2V = Ty->isFloatTy() ? 
+        double Op2V = Ty->isFloatTy() ?
                       (double)Op2->getValueAPF().convertToFloat():
                       Op2->getValueAPF().convertToDouble();
 
@@ -1416,7 +1427,7 @@ llvm::ConstantFoldCall(Function *F, ArrayRef<Constant *> Operands,
       }
       return 0;
     }
-    
+
     if (ConstantInt *Op1 = dyn_cast<ConstantInt>(Operands[0])) {
       if (ConstantInt *Op2 = dyn_cast<ConstantInt>(Operands[1])) {
         switch (F->getIntrinsicID()) {
@@ -1466,7 +1477,7 @@ llvm::ConstantFoldCall(Function *F, ArrayRef<Constant *> Operands,
           return ConstantInt::get(Ty, Op1->getValue().countLeadingZeros());
         }
       }
-      
+
       return 0;
     }
     return 0;
diff --git a/lib/Analysis/CostModel.cpp b/lib/Analysis/CostModel.cpp
new file mode 100644
index 0000000..5adbf45
--- /dev/null
+++ b/lib/Analysis/CostModel.cpp
@@ -0,0 +1,193 @@
+//===- CostModel.cpp ------ Cost Model Analysis ---------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the cost model analysis. It provides a very basic cost
+// estimation for LLVM-IR. The cost result can be thought of as cycles, but it
+// is really unit-less. The estimated cost is ment to be used for comparing
+// alternatives.
+//
+//===----------------------------------------------------------------------===//
+
+#define CM_NAME "cost-model"
+#define DEBUG_TYPE CM_NAME
+#include "llvm/Analysis/Passes.h"
+#include "llvm/Function.h"
+#include "llvm/Instructions.h"
+#include "llvm/Pass.h"
+#include "llvm/TargetTransformInfo.h"
+#include "llvm/Value.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
+using namespace llvm;
+
+namespace {
+  class CostModelAnalysis : public FunctionPass {
+
+  public:
+    static char ID; // Class identification, replacement for typeinfo
+    CostModelAnalysis() : FunctionPass(ID), F(0), VTTI(0) {
+      initializeCostModelAnalysisPass(
+        *PassRegistry::getPassRegistry());
+    }
+
+    /// Returns the expected cost of the instruction.
+    /// Returns -1 if the cost is unknown.
+    /// Note, this method does not cache the cost calculation and it
+    /// can be expensive in some cases.
+    unsigned getInstructionCost(Instruction *I) const;
+
+  private:
+    virtual void getAnalysisUsage(AnalysisUsage &AU) const;
+    virtual bool runOnFunction(Function &F);
+    virtual void print(raw_ostream &OS, const Module*) const;
+
+    /// The function that we analyze.
+    Function *F;
+    /// Vector target information.
+    const VectorTargetTransformInfo *VTTI;
+  };
+}  // End of anonymous namespace
+
+// Register this pass.
+char CostModelAnalysis::ID = 0;
+static const char cm_name[] = "Cost Model Analysis";
+INITIALIZE_PASS_BEGIN(CostModelAnalysis, CM_NAME, cm_name, false, true)
+INITIALIZE_PASS_END  (CostModelAnalysis, CM_NAME, cm_name, false, true)
+
+FunctionPass *llvm::createCostModelAnalysisPass() {
+  return new CostModelAnalysis();
+}
+
+void
+CostModelAnalysis::getAnalysisUsage(AnalysisUsage &AU) const {
+  AU.setPreservesAll();
+}
+
+bool
+CostModelAnalysis::runOnFunction(Function &F) {
+ this->F = &F;
+
+ // Target information.
+ TargetTransformInfo *TTI;
+ TTI = getAnalysisIfAvailable<TargetTransformInfo>();
+ if (TTI)
+   VTTI = TTI->getVectorTargetTransformInfo();
+
+ return false;
+}
+
+unsigned CostModelAnalysis::getInstructionCost(Instruction *I) const {
+  if (!VTTI)
+    return -1;
+
+  switch (I->getOpcode()) {
+  case Instruction::Ret:
+  case Instruction::PHI:
+  case Instruction::Br: {
+    return VTTI->getCFInstrCost(I->getOpcode());
+  }
+  case Instruction::Add:
+  case Instruction::FAdd:
+  case Instruction::Sub:
+  case Instruction::FSub:
+  case Instruction::Mul:
+  case Instruction::FMul:
+  case Instruction::UDiv:
+  case Instruction::SDiv:
+  case Instruction::FDiv:
+  case Instruction::URem:
+  case Instruction::SRem:
+  case Instruction::FRem:
+  case Instruction::Shl:
+  case Instruction::LShr:
+  case Instruction::AShr:
+  case Instruction::And:
+  case Instruction::Or:
+  case Instruction::Xor: {
+    return VTTI->getArithmeticInstrCost(I->getOpcode(), I->getType());
+  }
+  case Instruction::Select: {
+    SelectInst *SI = cast<SelectInst>(I);
+    Type *CondTy = SI->getCondition()->getType();
+    return VTTI->getCmpSelInstrCost(I->getOpcode(), I->getType(), CondTy);
+  }
+  case Instruction::ICmp:
+  case Instruction::FCmp: {
+    Type *ValTy = I->getOperand(0)->getType();
+    return VTTI->getCmpSelInstrCost(I->getOpcode(), ValTy);
+  }
+  case Instruction::Store: {
+    StoreInst *SI = cast<StoreInst>(I);
+    Type *ValTy = SI->getValueOperand()->getType();
+    return VTTI->getMemoryOpCost(I->getOpcode(), ValTy,
+                                 SI->getAlignment(),
+                                 SI->getPointerAddressSpace());
+  }
+  case Instruction::Load: {
+    LoadInst *LI = cast<LoadInst>(I);
+    return VTTI->getMemoryOpCost(I->getOpcode(), I->getType(),
+                                 LI->getAlignment(),
+                                 LI->getPointerAddressSpace());
+  }
+  case Instruction::ZExt:
+  case Instruction::SExt:
+  case Instruction::FPToUI:
+  case Instruction::FPToSI:
+  case Instruction::FPExt:
+  case Instruction::PtrToInt:
+  case Instruction::IntToPtr:
+  case Instruction::SIToFP:
+  case Instruction::UIToFP:
+  case Instruction::Trunc:
+  case Instruction::FPTrunc:
+  case Instruction::BitCast: {
+    Type *SrcTy = I->getOperand(0)->getType();
+    return VTTI->getCastInstrCost(I->getOpcode(), I->getType(), SrcTy);
+  }
+  case Instruction::ExtractElement: {
+    ExtractElementInst * EEI = cast<ExtractElementInst>(I);
+    ConstantInt *CI = dyn_cast<ConstantInt>(I->getOperand(1));
+    unsigned Idx = -1;
+    if (CI)
+      Idx = CI->getZExtValue();
+    return VTTI->getVectorInstrCost(I->getOpcode(),
+                                    EEI->getOperand(0)->getType(), Idx);
+  }
+  case Instruction::InsertElement: {
+      InsertElementInst * IE = cast<InsertElementInst>(I);
+      ConstantInt *CI = dyn_cast<ConstantInt>(IE->getOperand(2));
+      unsigned Idx = -1;
+      if (CI)
+        Idx = CI->getZExtValue();
+      return VTTI->getVectorInstrCost(I->getOpcode(),
+                                      IE->getType(), Idx);
+    }
+  default:
+    // We don't have any information on this instruction.
+    return -1;
+  }
+}
+
+void CostModelAnalysis::print(raw_ostream &OS, const Module*) const {
+  if (!F)
+    return;
+
+  for (Function::iterator B = F->begin(), BE = F->end(); B != BE; ++B) {
+    for (BasicBlock::iterator it = B->begin(), e = B->end(); it != e; ++it) {
+      Instruction *Inst = it;
+      unsigned Cost = getInstructionCost(Inst);
+      if (Cost != (unsigned)-1)
+        OS << "Cost Model: Found an estimated cost of " << Cost;
+      else
+        OS << "Cost Model: Unknown cost";
+
+      OS << " for instruction: "<< *Inst << "\n";
+    }
+  }
+}
diff --git a/lib/Analysis/DependenceAnalysis.cpp b/lib/Analysis/DependenceAnalysis.cpp
new file mode 100644
index 0000000..95ac5ea
--- /dev/null
+++ b/lib/Analysis/DependenceAnalysis.cpp
@@ -0,0 +1,3786 @@
+//===-- DependenceAnalysis.cpp - DA Implementation --------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// DependenceAnalysis is an LLVM pass that analyses dependences between memory
+// accesses. Currently, it is an (incomplete) implementation of the approach
+// described in
+//
+//            Practical Dependence Testing
+//            Goff, Kennedy, Tseng
+//            PLDI 1991
+//
+// There's a single entry point that analyzes the dependence between a pair
+// of memory references in a function, returning either NULL, for no dependence,
+// or a more-or-less detailed description of the dependence between them.
+//
+// Currently, the implementation cannot propagate constraints between
+// coupled RDIV subscripts and lacks a multi-subscript MIV test.
+// Both of these are conservative weaknesses;
+// that is, not a source of correctness problems.
+//
+// The implementation depends on the GEP instruction to
+// differentiate subscripts. Since Clang linearizes subscripts
+// for most arrays, we give up some precision (though the existing MIV tests
+// will help). We trust that the GEP instruction will eventually be extended.
+// In the meantime, we should explore Maslov's ideas about delinearization.
+//
+// We should pay some careful attention to the possibility of integer overflow
+// in the implementation of the various tests. This could happen with Add,
+// Subtract, or Multiply, with both APInt's and SCEV's.
+//
+// Some non-linear subscript pairs can be handled by the GCD test
+// (and perhaps other tests).
+// Should explore how often these things occur.
+//
+// Finally, it seems like certain test cases expose weaknesses in the SCEV
+// simplification, especially in the handling of sign and zero extensions.
+// It could be useful to spend time exploring these.
+//
+// Please note that this is work in progress and the interface is subject to
+// change.
+//
+//===----------------------------------------------------------------------===//
+//                                                                            //
+//                   In memory of Ken Kennedy, 1945 - 2007                    //
+//                                                                            //
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "da"
+
+#include "llvm/Analysis/DependenceAnalysis.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/Operator.h"
+#include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/ValueTracking.h"
+#include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/Analysis/ScalarEvolutionExpressions.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/InstIterator.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+
+//===----------------------------------------------------------------------===//
+// statistics
+
+STATISTIC(TotalArrayPairs, "Array pairs tested");
+STATISTIC(SeparableSubscriptPairs, "Separable subscript pairs");
+STATISTIC(CoupledSubscriptPairs, "Coupled subscript pairs");
+STATISTIC(NonlinearSubscriptPairs, "Nonlinear subscript pairs");
+STATISTIC(ZIVapplications, "ZIV applications");
+STATISTIC(ZIVindependence, "ZIV independence");
+STATISTIC(StrongSIVapplications, "Strong SIV applications");
+STATISTIC(StrongSIVsuccesses, "Strong SIV successes");
+STATISTIC(StrongSIVindependence, "Strong SIV independence");
+STATISTIC(WeakCrossingSIVapplications, "Weak-Crossing SIV applications");
+STATISTIC(WeakCrossingSIVsuccesses, "Weak-Crossing SIV successes");
+STATISTIC(WeakCrossingSIVindependence, "Weak-Crossing SIV independence");
+STATISTIC(ExactSIVapplications, "Exact SIV applications");
+STATISTIC(ExactSIVsuccesses, "Exact SIV successes");
+STATISTIC(ExactSIVindependence, "Exact SIV independence");
+STATISTIC(WeakZeroSIVapplications, "Weak-Zero SIV applications");
+STATISTIC(WeakZeroSIVsuccesses, "Weak-Zero SIV successes");
+STATISTIC(WeakZeroSIVindependence, "Weak-Zero SIV independence");
+STATISTIC(ExactRDIVapplications, "Exact RDIV applications");
+STATISTIC(ExactRDIVindependence, "Exact RDIV independence");
+STATISTIC(SymbolicRDIVapplications, "Symbolic RDIV applications");
+STATISTIC(SymbolicRDIVindependence, "Symbolic RDIV independence");
+STATISTIC(DeltaApplications, "Delta applications");
+STATISTIC(DeltaSuccesses, "Delta successes");
+STATISTIC(DeltaIndependence, "Delta independence");
+STATISTIC(DeltaPropagations, "Delta propagations");
+STATISTIC(GCDapplications, "GCD applications");
+STATISTIC(GCDsuccesses, "GCD successes");
+STATISTIC(GCDindependence, "GCD independence");
+STATISTIC(BanerjeeApplications, "Banerjee applications");
+STATISTIC(BanerjeeIndependence, "Banerjee independence");
+STATISTIC(BanerjeeSuccesses, "Banerjee successes");
+
+//===----------------------------------------------------------------------===//
+// basics
+
+INITIALIZE_PASS_BEGIN(DependenceAnalysis, "da",
+                      "Dependence Analysis", true, true)
+INITIALIZE_PASS_DEPENDENCY(LoopInfo)
+INITIALIZE_PASS_DEPENDENCY(ScalarEvolution)
+INITIALIZE_AG_DEPENDENCY(AliasAnalysis)
+INITIALIZE_PASS_END(DependenceAnalysis, "da",
+                    "Dependence Analysis", true, true)
+
+char DependenceAnalysis::ID = 0;
+
+
+FunctionPass *llvm::createDependenceAnalysisPass() {
+  return new DependenceAnalysis();
+}
+
+
+bool DependenceAnalysis::runOnFunction(Function &F) {
+  this->F = &F;
+  AA = &getAnalysis<AliasAnalysis>();
+  SE = &getAnalysis<ScalarEvolution>();
+  LI = &getAnalysis<LoopInfo>();
+  return false;
+}
+
+
+void DependenceAnalysis::releaseMemory() {
+}
+
+
+void DependenceAnalysis::getAnalysisUsage(AnalysisUsage &AU) const {
+  AU.setPreservesAll();
+  AU.addRequiredTransitive<AliasAnalysis>();
+  AU.addRequiredTransitive<ScalarEvolution>();
+  AU.addRequiredTransitive<LoopInfo>();
+}
+
+
+// Used to test the dependence analyzer.
+// Looks through the function, noting the first store instruction
+// and the first load instruction
+// (which always follows the first load in our tests).
+// Calls depends() and prints out the result.
+// Ignores all other instructions.
+static
+void dumpExampleDependence(raw_ostream &OS, Function *F,
+                           DependenceAnalysis *DA) {
+  for (inst_iterator SrcI = inst_begin(F), SrcE = inst_end(F);
+       SrcI != SrcE; ++SrcI) {
+    if (const StoreInst *Src = dyn_cast<StoreInst>(&*SrcI)) {
+      for (inst_iterator DstI = SrcI, DstE = inst_end(F);
+           DstI != DstE; ++DstI) {
+        if (const LoadInst *Dst = dyn_cast<LoadInst>(&*DstI)) {
+          OS << "da analyze - ";
+          if (Dependence *D = DA->depends(Src, Dst, true)) {
+            D->dump(OS);
+            for (unsigned Level = 1; Level <= D->getLevels(); Level++) {
+              if (D->isSplitable(Level)) {
+                OS << "da analyze - split level = " << Level;
+                OS << ", iteration = " << *DA->getSplitIteration(D, Level);
+                OS << "!\n";
+              }
+            }
+            delete D;
+          }
+          else
+            OS << "none!\n";
+          return;
+        }
+      }
+    }
+  }
+}
+
+
+void DependenceAnalysis::print(raw_ostream &OS, const Module*) const {
+  dumpExampleDependence(OS, F, const_cast<DependenceAnalysis *>(this));
+}
+
+//===----------------------------------------------------------------------===//
+// Dependence methods
+
+// Returns true if this is an input dependence.
+bool Dependence::isInput() const {
+  return Src->mayReadFromMemory() && Dst->mayReadFromMemory();
+}
+
+
+// Returns true if this is an output dependence.
+bool Dependence::isOutput() const {
+  return Src->mayWriteToMemory() && Dst->mayWriteToMemory();
+}
+
+
+// Returns true if this is an flow (aka true)  dependence.
+bool Dependence::isFlow() const {
+  return Src->mayWriteToMemory() && Dst->mayReadFromMemory();
+}
+
+
+// Returns true if this is an anti dependence.
+bool Dependence::isAnti() const {
+  return Src->mayReadFromMemory() && Dst->mayWriteToMemory();
+}
+
+
+// Returns true if a particular level is scalar; that is,
+// if no subscript in the source or destination mention the induction
+// variable associated with the loop at this level.
+// Leave this out of line, so it will serve as a virtual method anchor
+bool Dependence::isScalar(unsigned level) const {
+  return false;
+}
+
+
+//===----------------------------------------------------------------------===//
+// FullDependence methods
+
+FullDependence::FullDependence(const Instruction *Source,
+                               const Instruction *Destination,
+                               bool PossiblyLoopIndependent,
+                               unsigned CommonLevels) :
+  Dependence(Source, Destination),
+  Levels(CommonLevels),
+  LoopIndependent(PossiblyLoopIndependent) {
+  Consistent = true;
+  DV = CommonLevels ? new DVEntry[CommonLevels] : NULL;
+}
+
+// The rest are simple getters that hide the implementation.
+
+// getDirection - Returns the direction associated with a particular level.
+unsigned FullDependence::getDirection(unsigned Level) const {
+  assert(0 < Level && Level <= Levels && "Level out of range");
+  return DV[Level - 1].Direction;
+}
+
+
+// Returns the distance (or NULL) associated with a particular level.
+const SCEV *FullDependence::getDistance(unsigned Level) const {
+  assert(0 < Level && Level <= Levels && "Level out of range");
+  return DV[Level - 1].Distance;
+}
+
+
+// Returns true if a particular level is scalar; that is,
+// if no subscript in the source or destination mention the induction
+// variable associated with the loop at this level.
+bool FullDependence::isScalar(unsigned Level) const {
+  assert(0 < Level && Level <= Levels && "Level out of range");
+  return DV[Level - 1].Scalar;
+}
+
+
+// Returns true if peeling the first iteration from this loop
+// will break this dependence.
+bool FullDependence::isPeelFirst(unsigned Level) const {
+  assert(0 < Level && Level <= Levels && "Level out of range");
+  return DV[Level - 1].PeelFirst;
+}
+
+
+// Returns true if peeling the last iteration from this loop
+// will break this dependence.
+bool FullDependence::isPeelLast(unsigned Level) const {
+  assert(0 < Level && Level <= Levels && "Level out of range");
+  return DV[Level - 1].PeelLast;
+}
+
+
+// Returns true if splitting this loop will break the dependence.
+bool FullDependence::isSplitable(unsigned Level) const {
+  assert(0 < Level && Level <= Levels && "Level out of range");
+  return DV[Level - 1].Splitable;
+}
+
+
+//===----------------------------------------------------------------------===//
+// DependenceAnalysis::Constraint methods
+
+// If constraint is a point <X, Y>, returns X.
+// Otherwise assert.
+const SCEV *DependenceAnalysis::Constraint::getX() const {
+  assert(Kind == Point && "Kind should be Point");
+  return A;
+}
+
+
+// If constraint is a point <X, Y>, returns Y.
+// Otherwise assert.
+const SCEV *DependenceAnalysis::Constraint::getY() const {
+  assert(Kind == Point && "Kind should be Point");
+  return B;
+}
+
+
+// If constraint is a line AX + BY = C, returns A.
+// Otherwise assert.
+const SCEV *DependenceAnalysis::Constraint::getA() const {
+  assert((Kind == Line || Kind == Distance) &&
+         "Kind should be Line (or Distance)");
+  return A;
+}
+
+
+// If constraint is a line AX + BY = C, returns B.
+// Otherwise assert.
+const SCEV *DependenceAnalysis::Constraint::getB() const {
+  assert((Kind == Line || Kind == Distance) &&
+         "Kind should be Line (or Distance)");
+  return B;
+}
+
+
+// If constraint is a line AX + BY = C, returns C.
+// Otherwise assert.
+const SCEV *DependenceAnalysis::Constraint::getC() const {
+  assert((Kind == Line || Kind == Distance) &&
+         "Kind should be Line (or Distance)");
+  return C;
+}
+
+
+// If constraint is a distance, returns D.
+// Otherwise assert.
+const SCEV *DependenceAnalysis::Constraint::getD() const {
+  assert(Kind == Distance && "Kind should be Distance");
+  return SE->getNegativeSCEV(C);
+}
+
+
+// Returns the loop associated with this constraint.
+const Loop *DependenceAnalysis::Constraint::getAssociatedLoop() const {
+  assert((Kind == Distance || Kind == Line || Kind == Point) &&
+         "Kind should be Distance, Line, or Point");
+  return AssociatedLoop;
+}
+
+
+void DependenceAnalysis::Constraint::setPoint(const SCEV *X,
+                                              const SCEV *Y,
+                                              const Loop *CurLoop) {
+  Kind = Point;
+  A = X;
+  B = Y;
+  AssociatedLoop = CurLoop;
+}
+
+
+void DependenceAnalysis::Constraint::setLine(const SCEV *AA,
+                                             const SCEV *BB,
+                                             const SCEV *CC,
+                                             const Loop *CurLoop) {
+  Kind = Line;
+  A = AA;
+  B = BB;
+  C = CC;
+  AssociatedLoop = CurLoop;
+}
+
+
+void DependenceAnalysis::Constraint::setDistance(const SCEV *D,
+                                                 const Loop *CurLoop) {
+  Kind = Distance;
+  A = SE->getConstant(D->getType(), 1);
+  B = SE->getNegativeSCEV(A);
+  C = SE->getNegativeSCEV(D);
+  AssociatedLoop = CurLoop;
+}
+
+
+void DependenceAnalysis::Constraint::setEmpty() {
+  Kind = Empty;
+}
+
+
+void DependenceAnalysis::Constraint::setAny(ScalarEvolution *NewSE) {
+  SE = NewSE;
+  Kind = Any;
+}
+
+
+// For debugging purposes. Dumps the constraint out to OS.
+void DependenceAnalysis::Constraint::dump(raw_ostream &OS) const {
+  if (isEmpty())
+    OS << " Empty\n";
+  else if (isAny())
+    OS << " Any\n";
+  else if (isPoint())
+    OS << " Point is <" << *getX() << ", " << *getY() << ">\n";
+  else if (isDistance())
+    OS << " Distance is " << *getD() <<
+      " (" << *getA() << "*X + " << *getB() << "*Y = " << *getC() << ")\n";
+  else if (isLine())
+    OS << " Line is " << *getA() << "*X + " <<
+      *getB() << "*Y = " << *getC() << "\n";
+  else
+    llvm_unreachable("unknown constraint type in Constraint::dump");
+}
+
+
+// Updates X with the intersection
+// of the Constraints X and Y. Returns true if X has changed.
+// Corresponds to Figure 4 from the paper
+//
+//            Practical Dependence Testing
+//            Goff, Kennedy, Tseng
+//            PLDI 1991
+bool DependenceAnalysis::intersectConstraints(Constraint *X,
+                                              const Constraint *Y) {
+  ++DeltaApplications;
+  DEBUG(dbgs() << "\tintersect constraints\n");
+  DEBUG(dbgs() << "\t    X ="; X->dump(dbgs()));
+  DEBUG(dbgs() << "\t    Y ="; Y->dump(dbgs()));
+  assert(!Y->isPoint() && "Y must not be a Point");
+  if (X->isAny()) {
+    if (Y->isAny())
+      return false;
+    *X = *Y;
+    return true;
+  }
+  if (X->isEmpty())
+    return false;
+  if (Y->isEmpty()) {
+    X->setEmpty();
+    return true;
+  }
+
+  if (X->isDistance() && Y->isDistance()) {
+    DEBUG(dbgs() << "\t    intersect 2 distances\n");
+    if (isKnownPredicate(CmpInst::ICMP_EQ, X->getD(), Y->getD()))
+      return false;
+    if (isKnownPredicate(CmpInst::ICMP_NE, X->getD(), Y->getD())) {
+      X->setEmpty();
+      ++DeltaSuccesses;
+      return true;
+    }
+    // Hmmm, interesting situation.
+    // I guess if either is constant, keep it and ignore the other.
+    if (isa<SCEVConstant>(Y->getD())) {
+      *X = *Y;
+      return true;
+    }
+    return false;
+  }
+
+  // At this point, the pseudo-code in Figure 4 of the paper
+  // checks if (X->isPoint() && Y->isPoint()).
+  // This case can't occur in our implementation,
+  // since a Point can only arise as the result of intersecting
+  // two Line constraints, and the right-hand value, Y, is never
+  // the result of an intersection.
+  assert(!(X->isPoint() && Y->isPoint()) &&
+         "We shouldn't ever see X->isPoint() && Y->isPoint()");
+
+  if (X->isLine() && Y->isLine()) {
+    DEBUG(dbgs() << "\t    intersect 2 lines\n");
+    const SCEV *Prod1 = SE->getMulExpr(X->getA(), Y->getB());
+    const SCEV *Prod2 = SE->getMulExpr(X->getB(), Y->getA());
+    if (isKnownPredicate(CmpInst::ICMP_EQ, Prod1, Prod2)) {
+      // slopes are equal, so lines are parallel
+      DEBUG(dbgs() << "\t\tsame slope\n");
+      Prod1 = SE->getMulExpr(X->getC(), Y->getB());
+      Prod2 = SE->getMulExpr(X->getB(), Y->getC());
+      if (isKnownPredicate(CmpInst::ICMP_EQ, Prod1, Prod2))
+        return false;
+      if (isKnownPredicate(CmpInst::ICMP_NE, Prod1, Prod2)) {
+        X->setEmpty();
+        ++DeltaSuccesses;
+        return true;
+      }
+      return false;
+    }
+    if (isKnownPredicate(CmpInst::ICMP_NE, Prod1, Prod2)) {
+      // slopes differ, so lines intersect
+      DEBUG(dbgs() << "\t\tdifferent slopes\n");
+      const SCEV *C1B2 = SE->getMulExpr(X->getC(), Y->getB());
+      const SCEV *C1A2 = SE->getMulExpr(X->getC(), Y->getA());
+      const SCEV *C2B1 = SE->getMulExpr(Y->getC(), X->getB());
+      const SCEV *C2A1 = SE->getMulExpr(Y->getC(), X->getA());
+      const SCEV *A1B2 = SE->getMulExpr(X->getA(), Y->getB());
+      const SCEV *A2B1 = SE->getMulExpr(Y->getA(), X->getB());
+      const SCEVConstant *C1A2_C2A1 =
+        dyn_cast<SCEVConstant>(SE->getMinusSCEV(C1A2, C2A1));
+      const SCEVConstant *C1B2_C2B1 =
+        dyn_cast<SCEVConstant>(SE->getMinusSCEV(C1B2, C2B1));
+      const SCEVConstant *A1B2_A2B1 =
+        dyn_cast<SCEVConstant>(SE->getMinusSCEV(A1B2, A2B1));
+      const SCEVConstant *A2B1_A1B2 =
+        dyn_cast<SCEVConstant>(SE->getMinusSCEV(A2B1, A1B2));
+      if (!C1B2_C2B1 || !C1A2_C2A1 ||
+          !A1B2_A2B1 || !A2B1_A1B2)
+        return false;
+      APInt Xtop = C1B2_C2B1->getValue()->getValue();
+      APInt Xbot = A1B2_A2B1->getValue()->getValue();
+      APInt Ytop = C1A2_C2A1->getValue()->getValue();
+      APInt Ybot = A2B1_A1B2->getValue()->getValue();
+      DEBUG(dbgs() << "\t\tXtop = " << Xtop << "\n");
+      DEBUG(dbgs() << "\t\tXbot = " << Xbot << "\n");
+      DEBUG(dbgs() << "\t\tYtop = " << Ytop << "\n");
+      DEBUG(dbgs() << "\t\tYbot = " << Ybot << "\n");
+      APInt Xq = Xtop; // these need to be initialized, even
+      APInt Xr = Xtop; // though they're just going to be overwritten
+      APInt::sdivrem(Xtop, Xbot, Xq, Xr);
+      APInt Yq = Ytop;
+      APInt Yr = Ytop;;
+      APInt::sdivrem(Ytop, Ybot, Yq, Yr);
+      if (Xr != 0 || Yr != 0) {
+        X->setEmpty();
+        ++DeltaSuccesses;
+        return true;
+      }
+      DEBUG(dbgs() << "\t\tX = " << Xq << ", Y = " << Yq << "\n");
+      if (Xq.slt(0) || Yq.slt(0)) {
+        X->setEmpty();
+        ++DeltaSuccesses;
+        return true;
+      }
+      if (const SCEVConstant *CUB =
+          collectConstantUpperBound(X->getAssociatedLoop(), Prod1->getType())) {
+        APInt UpperBound = CUB->getValue()->getValue();
+        DEBUG(dbgs() << "\t\tupper bound = " << UpperBound << "\n");
+        if (Xq.sgt(UpperBound) || Yq.sgt(UpperBound)) {
+          X->setEmpty();
+          ++DeltaSuccesses;
+          return true;
+        }
+      }
+      X->setPoint(SE->getConstant(Xq),
+                  SE->getConstant(Yq),
+                  X->getAssociatedLoop());
+      ++DeltaSuccesses;
+      return true;
+    }
+    return false;
+  }
+
+  // if (X->isLine() && Y->isPoint()) This case can't occur.
+  assert(!(X->isLine() && Y->isPoint()) && "This case should never occur");
+
+  if (X->isPoint() && Y->isLine()) {
+    DEBUG(dbgs() << "\t    intersect Point and Line\n");
+    const SCEV *A1X1 = SE->getMulExpr(Y->getA(), X->getX());
+    const SCEV *B1Y1 = SE->getMulExpr(Y->getB(), X->getY());
+    const SCEV *Sum = SE->getAddExpr(A1X1, B1Y1);
+    if (isKnownPredicate(CmpInst::ICMP_EQ, Sum, Y->getC()))
+      return false;
+    if (isKnownPredicate(CmpInst::ICMP_NE, Sum, Y->getC())) {
+      X->setEmpty();
+      ++DeltaSuccesses;
+      return true;
+    }
+    return false;
+  }
+
+  llvm_unreachable("shouldn't reach the end of Constraint intersection");
+  return false;
+}
+
+
+//===----------------------------------------------------------------------===//
+// DependenceAnalysis methods
+
+// For debugging purposes. Dumps a dependence to OS.
+void Dependence::dump(raw_ostream &OS) const {
+  bool Splitable = false;
+  if (isConfused())
+    OS << "confused";
+  else {
+    if (isConsistent())
+      OS << "consistent ";
+    if (isFlow())
+      OS << "flow";
+    else if (isOutput())
+      OS << "output";
+    else if (isAnti())
+      OS << "anti";
+    else if (isInput())
+      OS << "input";
+    unsigned Levels = getLevels();
+    if (Levels) {
+      OS << " [";
+      for (unsigned II = 1; II <= Levels; ++II) {
+        if (isSplitable(II))
+          Splitable = true;
+        if (isPeelFirst(II))
+          OS << 'p';
+        const SCEV *Distance = getDistance(II);
+        if (Distance)
+          OS << *Distance;
+        else if (isScalar(II))
+          OS << "S";
+        else {
+          unsigned Direction = getDirection(II);
+          if (Direction == DVEntry::ALL)
+            OS << "*";
+          else {
+            if (Direction & DVEntry::LT)
+              OS << "<";
+            if (Direction & DVEntry::EQ)
+              OS << "=";
+            if (Direction & DVEntry::GT)
+              OS << ">";
+          }
+        }
+        if (isPeelLast(II))
+          OS << 'p';
+        if (II < Levels)
+          OS << " ";
+      }
+      if (isLoopIndependent())
+        OS << "|<";
+      OS << "]";
+      if (Splitable)
+        OS << " splitable";
+    }
+  }
+  OS << "!\n";
+}
+
+
+
+static
+AliasAnalysis::AliasResult underlyingObjectsAlias(AliasAnalysis *AA,
+                                                  const Value *A,
+                                                  const Value *B) {
+  const Value *AObj = GetUnderlyingObject(A);
+  const Value *BObj = GetUnderlyingObject(B);
+  return AA->alias(AObj, AA->getTypeStoreSize(AObj->getType()),
+                   BObj, AA->getTypeStoreSize(BObj->getType()));
+}
+
+
+// Returns true if the load or store can be analyzed. Atomic and volatile
+// operations have properties which this analysis does not understand.
+static
+bool isLoadOrStore(const Instruction *I) {
+  if (const LoadInst *LI = dyn_cast<LoadInst>(I))
+    return LI->isUnordered();
+  else if (const StoreInst *SI = dyn_cast<StoreInst>(I))
+    return SI->isUnordered();
+  return false;
+}
+
+
+static
+const Value *getPointerOperand(const Instruction *I) {
+  if (const LoadInst *LI = dyn_cast<LoadInst>(I))
+    return LI->getPointerOperand();
+  if (const StoreInst *SI = dyn_cast<StoreInst>(I))
+    return SI->getPointerOperand();
+  llvm_unreachable("Value is not load or store instruction");
+  return 0;
+}
+
+
+// Examines the loop nesting of the Src and Dst
+// instructions and establishes their shared loops. Sets the variables
+// CommonLevels, SrcLevels, and MaxLevels.
+// The source and destination instructions needn't be contained in the same
+// loop. The routine establishNestingLevels finds the level of most deeply
+// nested loop that contains them both, CommonLevels. An instruction that's
+// not contained in a loop is at level = 0. MaxLevels is equal to the level
+// of the source plus the level of the destination, minus CommonLevels.
+// This lets us allocate vectors MaxLevels in length, with room for every
+// distinct loop referenced in both the source and destination subscripts.
+// The variable SrcLevels is the nesting depth of the source instruction.
+// It's used to help calculate distinct loops referenced by the destination.
+// Here's the map from loops to levels:
+//            0 - unused
+//            1 - outermost common loop
+//          ... - other common loops
+// CommonLevels - innermost common loop
+//          ... - loops containing Src but not Dst
+//    SrcLevels - innermost loop containing Src but not Dst
+//          ... - loops containing Dst but not Src
+//    MaxLevels - innermost loops containing Dst but not Src
+// Consider the follow code fragment:
+//   for (a = ...) {
+//     for (b = ...) {
+//       for (c = ...) {
+//         for (d = ...) {
+//           A[] = ...;
+//         }
+//       }
+//       for (e = ...) {
+//         for (f = ...) {
+//           for (g = ...) {
+//             ... = A[];
+//           }
+//         }
+//       }
+//     }
+//   }
+// If we're looking at the possibility of a dependence between the store
+// to A (the Src) and the load from A (the Dst), we'll note that they
+// have 2 loops in common, so CommonLevels will equal 2 and the direction
+// vector for Result will have 2 entries. SrcLevels = 4 and MaxLevels = 7.
+// A map from loop names to loop numbers would look like
+//     a - 1
+//     b - 2 = CommonLevels
+//     c - 3
+//     d - 4 = SrcLevels
+//     e - 5
+//     f - 6
+//     g - 7 = MaxLevels
+void DependenceAnalysis::establishNestingLevels(const Instruction *Src,
+                                                const Instruction *Dst) {
+  const BasicBlock *SrcBlock = Src->getParent();
+  const BasicBlock *DstBlock = Dst->getParent();
+  unsigned SrcLevel = LI->getLoopDepth(SrcBlock);
+  unsigned DstLevel = LI->getLoopDepth(DstBlock);
+  const Loop *SrcLoop = LI->getLoopFor(SrcBlock);
+  const Loop *DstLoop = LI->getLoopFor(DstBlock);
+  SrcLevels = SrcLevel;
+  MaxLevels = SrcLevel + DstLevel;
+  while (SrcLevel > DstLevel) {
+    SrcLoop = SrcLoop->getParentLoop();
+    SrcLevel--;
+  }
+  while (DstLevel > SrcLevel) {
+    DstLoop = DstLoop->getParentLoop();
+    DstLevel--;
+  }
+  while (SrcLoop != DstLoop) {
+    SrcLoop = SrcLoop->getParentLoop();
+    DstLoop = DstLoop->getParentLoop();
+    SrcLevel--;
+  }
+  CommonLevels = SrcLevel;
+  MaxLevels -= CommonLevels;
+}
+
+
+// Given one of the loops containing the source, return
+// its level index in our numbering scheme.
+unsigned DependenceAnalysis::mapSrcLoop(const Loop *SrcLoop) const {
+  return SrcLoop->getLoopDepth();
+}
+
+
+// Given one of the loops containing the destination,
+// return its level index in our numbering scheme.
+unsigned DependenceAnalysis::mapDstLoop(const Loop *DstLoop) const {
+  unsigned D = DstLoop->getLoopDepth();
+  if (D > CommonLevels)
+    return D - CommonLevels + SrcLevels;
+  else
+    return D;
+}
+
+
+// Returns true if Expression is loop invariant in LoopNest.
+bool DependenceAnalysis::isLoopInvariant(const SCEV *Expression,
+                                         const Loop *LoopNest) const {
+  if (!LoopNest)
+    return true;
+  return SE->isLoopInvariant(Expression, LoopNest) &&
+    isLoopInvariant(Expression, LoopNest->getParentLoop());
+}
+
+
+
+// Finds the set of loops from the LoopNest that
+// have a level <= CommonLevels and are referred to by the SCEV Expression.
+void DependenceAnalysis::collectCommonLoops(const SCEV *Expression,
+                                            const Loop *LoopNest,
+                                            SmallBitVector &Loops) const {
+  while (LoopNest) {
+    unsigned Level = LoopNest->getLoopDepth();
+    if (Level <= CommonLevels && !SE->isLoopInvariant(Expression, LoopNest))
+      Loops.set(Level);
+    LoopNest = LoopNest->getParentLoop();
+  }
+}
+
+
+// removeMatchingExtensions - Examines a subscript pair.
+// If the source and destination are identically sign (or zero)
+// extended, it strips off the extension in an effect to simplify
+// the actual analysis.
+void DependenceAnalysis::removeMatchingExtensions(Subscript *Pair) {
+  const SCEV *Src = Pair->Src;
+  const SCEV *Dst = Pair->Dst;
+  if ((isa<SCEVZeroExtendExpr>(Src) && isa<SCEVZeroExtendExpr>(Dst)) ||
+      (isa<SCEVSignExtendExpr>(Src) && isa<SCEVSignExtendExpr>(Dst))) {
+    const SCEVCastExpr *SrcCast = cast<SCEVCastExpr>(Src);
+    const SCEVCastExpr *DstCast = cast<SCEVCastExpr>(Dst);
+    if (SrcCast->getType() == DstCast->getType()) {
+      Pair->Src = SrcCast->getOperand();
+      Pair->Dst = DstCast->getOperand();
+    }
+  }
+}
+
+
+// Examine the scev and return true iff it's linear.
+// Collect any loops mentioned in the set of "Loops".
+bool DependenceAnalysis::checkSrcSubscript(const SCEV *Src,
+                                           const Loop *LoopNest,
+                                           SmallBitVector &Loops) {
+  const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Src);
+  if (!AddRec)
+    return isLoopInvariant(Src, LoopNest);
+  const SCEV *Start = AddRec->getStart();
+  const SCEV *Step = AddRec->getStepRecurrence(*SE);
+  if (!isLoopInvariant(Step, LoopNest))
+    return false;
+  Loops.set(mapSrcLoop(AddRec->getLoop()));
+  return checkSrcSubscript(Start, LoopNest, Loops);
+}
+
+
+
+// Examine the scev and return true iff it's linear.
+// Collect any loops mentioned in the set of "Loops".
+bool DependenceAnalysis::checkDstSubscript(const SCEV *Dst,
+                                           const Loop *LoopNest,
+                                           SmallBitVector &Loops) {
+  const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Dst);
+  if (!AddRec)
+    return isLoopInvariant(Dst, LoopNest);
+  const SCEV *Start = AddRec->getStart();
+  const SCEV *Step = AddRec->getStepRecurrence(*SE);
+  if (!isLoopInvariant(Step, LoopNest))
+    return false;
+  Loops.set(mapDstLoop(AddRec->getLoop()));
+  return checkDstSubscript(Start, LoopNest, Loops);
+}
+
+
+// Examines the subscript pair (the Src and Dst SCEVs)
+// and classifies it as either ZIV, SIV, RDIV, MIV, or Nonlinear.
+// Collects the associated loops in a set.
+DependenceAnalysis::Subscript::ClassificationKind
+DependenceAnalysis::classifyPair(const SCEV *Src, const Loop *SrcLoopNest,
+                                 const SCEV *Dst, const Loop *DstLoopNest,
+                                 SmallBitVector &Loops) {
+  SmallBitVector SrcLoops(MaxLevels + 1);
+  SmallBitVector DstLoops(MaxLevels + 1);
+  if (!checkSrcSubscript(Src, SrcLoopNest, SrcLoops))
+    return Subscript::NonLinear;
+  if (!checkDstSubscript(Dst, DstLoopNest, DstLoops))
+    return Subscript::NonLinear;
+  Loops = SrcLoops;
+  Loops |= DstLoops;
+  unsigned N = Loops.count();
+  if (N == 0)
+    return Subscript::ZIV;
+  if (N == 1)
+    return Subscript::SIV;
+  if (N == 2 && (SrcLoops.count() == 0 ||
+                 DstLoops.count() == 0 ||
+                 (SrcLoops.count() == 1 && DstLoops.count() == 1)))
+    return Subscript::RDIV;
+  return Subscript::MIV;
+}
+
+
+// A wrapper around SCEV::isKnownPredicate.
+// Looks for cases where we're interested in comparing for equality.
+// If both X and Y have been identically sign or zero extended,
+// it strips off the (confusing) extensions before invoking
+// SCEV::isKnownPredicate. Perhaps, someday, the ScalarEvolution package
+// will be similarly updated.
+//
+// If SCEV::isKnownPredicate can't prove the predicate,
+// we try simple subtraction, which seems to help in some cases
+// involving symbolics.
+bool DependenceAnalysis::isKnownPredicate(ICmpInst::Predicate Pred,
+                                          const SCEV *X,
+                                          const SCEV *Y) const {
+  if (Pred == CmpInst::ICMP_EQ ||
+      Pred == CmpInst::ICMP_NE) {
+    if ((isa<SCEVSignExtendExpr>(X) &&
+         isa<SCEVSignExtendExpr>(Y)) ||
+        (isa<SCEVZeroExtendExpr>(X) &&
+         isa<SCEVZeroExtendExpr>(Y))) {
+      const SCEVCastExpr *CX = cast<SCEVCastExpr>(X);
+      const SCEVCastExpr *CY = cast<SCEVCastExpr>(Y);
+      const SCEV *Xop = CX->getOperand();
+      const SCEV *Yop = CY->getOperand();
+      if (Xop->getType() == Yop->getType()) {
+        X = Xop;
+        Y = Yop;
+      }
+    }
+  }
+  if (SE->isKnownPredicate(Pred, X, Y))
+    return true;
+  // If SE->isKnownPredicate can't prove the condition,
+  // we try the brute-force approach of subtracting
+  // and testing the difference.
+  // By testing with SE->isKnownPredicate first, we avoid
+  // the possibility of overflow when the arguments are constants.
+  const SCEV *Delta = SE->getMinusSCEV(X, Y);
+  switch (Pred) {
+  case CmpInst::ICMP_EQ:
+    return Delta->isZero();
+  case CmpInst::ICMP_NE:
+    return SE->isKnownNonZero(Delta);
+  case CmpInst::ICMP_SGE:
+    return SE->isKnownNonNegative(Delta);
+  case CmpInst::ICMP_SLE:
+    return SE->isKnownNonPositive(Delta);
+  case CmpInst::ICMP_SGT:
+    return SE->isKnownPositive(Delta);
+  case CmpInst::ICMP_SLT:
+    return SE->isKnownNegative(Delta);
+  default:
+    llvm_unreachable("unexpected predicate in isKnownPredicate");
+  }
+}
+
+
+// All subscripts are all the same type.
+// Loop bound may be smaller (e.g., a char).
+// Should zero extend loop bound, since it's always >= 0.
+// This routine collects upper bound and extends if needed.
+// Return null if no bound available.
+const SCEV *DependenceAnalysis::collectUpperBound(const Loop *L,
+                                                  Type *T) const {
+  if (SE->hasLoopInvariantBackedgeTakenCount(L)) {
+    const SCEV *UB = SE->getBackedgeTakenCount(L);
+    return SE->getNoopOrZeroExtend(UB, T);
+  }
+  return NULL;
+}
+
+
+// Calls collectUpperBound(), then attempts to cast it to SCEVConstant.
+// If the cast fails, returns NULL.
+const SCEVConstant *DependenceAnalysis::collectConstantUpperBound(const Loop *L,
+                                                                  Type *T
+                                                                  ) const {
+  if (const SCEV *UB = collectUpperBound(L, T))
+    return dyn_cast<SCEVConstant>(UB);
+  return NULL;
+}
+
+
+// testZIV -
+// When we have a pair of subscripts of the form [c1] and [c2],
+// where c1 and c2 are both loop invariant, we attack it using
+// the ZIV test. Basically, we test by comparing the two values,
+// but there are actually three possible results:
+// 1) the values are equal, so there's a dependence
+// 2) the values are different, so there's no dependence
+// 3) the values might be equal, so we have to assume a dependence.
+//
+// Return true if dependence disproved.
+bool DependenceAnalysis::testZIV(const SCEV *Src,
+                                 const SCEV *Dst,
+                                 FullDependence &Result) const {
+  DEBUG(dbgs() << "    src = " << *Src << "\n");
+  DEBUG(dbgs() << "    dst = " << *Dst << "\n");
+  ++ZIVapplications;
+  if (isKnownPredicate(CmpInst::ICMP_EQ, Src, Dst)) {
+    DEBUG(dbgs() << "    provably dependent\n");
+    return false; // provably dependent
+  }
+  if (isKnownPredicate(CmpInst::ICMP_NE, Src, Dst)) {
+    DEBUG(dbgs() << "    provably independent\n");
+    ++ZIVindependence;
+    return true; // provably independent
+  }
+  DEBUG(dbgs() << "    possibly dependent\n");
+  Result.Consistent = false;
+  return false; // possibly dependent
+}
+
+
+// strongSIVtest -
+// From the paper, Practical Dependence Testing, Section 4.2.1
+//
+// When we have a pair of subscripts of the form [c1 + a*i] and [c2 + a*i],
+// where i is an induction variable, c1 and c2 are loop invariant,
+//  and a is a constant, we can solve it exactly using the Strong SIV test.
+//
+// Can prove independence. Failing that, can compute distance (and direction).
+// In the presence of symbolic terms, we can sometimes make progress.
+//
+// If there's a dependence,
+//
+//    c1 + a*i = c2 + a*i'
+//
+// The dependence distance is
+//
+//    d = i' - i = (c1 - c2)/a
+//
+// A dependence only exists if d is an integer and abs(d) <= U, where U is the
+// loop's upper bound. If a dependence exists, the dependence direction is
+// defined as
+//
+//                { < if d > 0
+//    direction = { = if d = 0
+//                { > if d < 0
+//
+// Return true if dependence disproved.
+bool DependenceAnalysis::strongSIVtest(const SCEV *Coeff,
+                                       const SCEV *SrcConst,
+                                       const SCEV *DstConst,
+                                       const Loop *CurLoop,
+                                       unsigned Level,
+                                       FullDependence &Result,
+                                       Constraint &NewConstraint) const {
+  DEBUG(dbgs() << "\tStrong SIV test\n");
+  DEBUG(dbgs() << "\t    Coeff = " << *Coeff);
+  DEBUG(dbgs() << ", " << *Coeff->getType() << "\n");
+  DEBUG(dbgs() << "\t    SrcConst = " << *SrcConst);
+  DEBUG(dbgs() << ", " << *SrcConst->getType() << "\n");
+  DEBUG(dbgs() << "\t    DstConst = " << *DstConst);
+  DEBUG(dbgs() << ", " << *DstConst->getType() << "\n");
+  ++StrongSIVapplications;
+  assert(0 < Level && Level <= CommonLevels && "level out of range");
+  Level--;
+
+  const SCEV *Delta = SE->getMinusSCEV(SrcConst, DstConst);
+  DEBUG(dbgs() << "\t    Delta = " << *Delta);
+  DEBUG(dbgs() << ", " << *Delta->getType() << "\n");
+
+  // check that |Delta| < iteration count
+  if (const SCEV *UpperBound = collectUpperBound(CurLoop, Delta->getType())) {
+    DEBUG(dbgs() << "\t    UpperBound = " << *UpperBound);
+    DEBUG(dbgs() << ", " << *UpperBound->getType() << "\n");
+    const SCEV *AbsDelta =
+      SE->isKnownNonNegative(Delta) ? Delta : SE->getNegativeSCEV(Delta);
+    const SCEV *AbsCoeff =
+      SE->isKnownNonNegative(Coeff) ? Coeff : SE->getNegativeSCEV(Coeff);
+    const SCEV *Product = SE->getMulExpr(UpperBound, AbsCoeff);
+    if (isKnownPredicate(CmpInst::ICMP_SGT, AbsDelta, Product)) {
+      // Distance greater than trip count - no dependence
+      ++StrongSIVindependence;
+      ++StrongSIVsuccesses;
+      return true;
+    }
+  }
+
+  // Can we compute distance?
+  if (isa<SCEVConstant>(Delta) && isa<SCEVConstant>(Coeff)) {
+    APInt ConstDelta = cast<SCEVConstant>(Delta)->getValue()->getValue();
+    APInt ConstCoeff = cast<SCEVConstant>(Coeff)->getValue()->getValue();
+    APInt Distance  = ConstDelta; // these need to be initialized
+    APInt Remainder = ConstDelta;
+    APInt::sdivrem(ConstDelta, ConstCoeff, Distance, Remainder);
+    DEBUG(dbgs() << "\t    Distance = " << Distance << "\n");
+    DEBUG(dbgs() << "\t    Remainder = " << Remainder << "\n");
+    // Make sure Coeff divides Delta exactly
+    if (Remainder != 0) {
+      // Coeff doesn't divide Distance, no dependence
+      ++StrongSIVindependence;
+      ++StrongSIVsuccesses;
+      return true;
+    }
+    Result.DV[Level].Distance = SE->getConstant(Distance);
+    NewConstraint.setDistance(SE->getConstant(Distance), CurLoop);
+    if (Distance.sgt(0))
+      Result.DV[Level].Direction &= Dependence::DVEntry::LT;
+    else if (Distance.slt(0))
+      Result.DV[Level].Direction &= Dependence::DVEntry::GT;
+    else
+      Result.DV[Level].Direction &= Dependence::DVEntry::EQ;
+    ++StrongSIVsuccesses;
+  }
+  else if (Delta->isZero()) {
+    // since 0/X == 0
+    Result.DV[Level].Distance = Delta;
+    NewConstraint.setDistance(Delta, CurLoop);
+    Result.DV[Level].Direction &= Dependence::DVEntry::EQ;
+    ++StrongSIVsuccesses;
+  }
+  else {
+    if (Coeff->isOne()) {
+      DEBUG(dbgs() << "\t    Distance = " << *Delta << "\n");
+      Result.DV[Level].Distance = Delta; // since X/1 == X
+      NewConstraint.setDistance(Delta, CurLoop);
+    }
+    else {
+      Result.Consistent = false;
+      NewConstraint.setLine(Coeff,
+                            SE->getNegativeSCEV(Coeff),
+                            SE->getNegativeSCEV(Delta), CurLoop);
+    }
+
+    // maybe we can get a useful direction
+    bool DeltaMaybeZero     = !SE->isKnownNonZero(Delta);
+    bool DeltaMaybePositive = !SE->isKnownNonPositive(Delta);
+    bool DeltaMaybeNegative = !SE->isKnownNonNegative(Delta);
+    bool CoeffMaybePositive = !SE->isKnownNonPositive(Coeff);
+    bool CoeffMaybeNegative = !SE->isKnownNonNegative(Coeff);
+    // The double negatives above are confusing.
+    // It helps to read !SE->isKnownNonZero(Delta)
+    // as "Delta might be Zero"
+    unsigned NewDirection = Dependence::DVEntry::NONE;
+    if ((DeltaMaybePositive && CoeffMaybePositive) ||
+        (DeltaMaybeNegative && CoeffMaybeNegative))
+      NewDirection = Dependence::DVEntry::LT;
+    if (DeltaMaybeZero)
+      NewDirection |= Dependence::DVEntry::EQ;
+    if ((DeltaMaybeNegative && CoeffMaybePositive) ||
+        (DeltaMaybePositive && CoeffMaybeNegative))
+      NewDirection |= Dependence::DVEntry::GT;
+    if (NewDirection < Result.DV[Level].Direction)
+      ++StrongSIVsuccesses;
+    Result.DV[Level].Direction &= NewDirection;
+  }
+  return false;
+}
+
+
+// weakCrossingSIVtest -
+// From the paper, Practical Dependence Testing, Section 4.2.2
+//
+// When we have a pair of subscripts of the form [c1 + a*i] and [c2 - a*i],
+// where i is an induction variable, c1 and c2 are loop invariant,
+// and a is a constant, we can solve it exactly using the
+// Weak-Crossing SIV test.
+//
+// Given c1 + a*i = c2 - a*i', we can look for the intersection of
+// the two lines, where i = i', yielding
+//
+//    c1 + a*i = c2 - a*i
+//    2a*i = c2 - c1
+//    i = (c2 - c1)/2a
+//
+// If i < 0, there is no dependence.
+// If i > upperbound, there is no dependence.
+// If i = 0 (i.e., if c1 = c2), there's a dependence with distance = 0.
+// If i = upperbound, there's a dependence with distance = 0.
+// If i is integral, there's a dependence (all directions).
+// If the non-integer part = 1/2, there's a dependence (<> directions).
+// Otherwise, there's no dependence.
+//
+// Can prove independence. Failing that,
+// can sometimes refine the directions.
+// Can determine iteration for splitting.
+//
+// Return true if dependence disproved.
+bool DependenceAnalysis::weakCrossingSIVtest(const SCEV *Coeff,
+                                             const SCEV *SrcConst,
+                                             const SCEV *DstConst,
+                                             const Loop *CurLoop,
+                                             unsigned Level,
+                                             FullDependence &Result,
+                                             Constraint &NewConstraint,
+                                             const SCEV *&SplitIter) const {
+  DEBUG(dbgs() << "\tWeak-Crossing SIV test\n");
+  DEBUG(dbgs() << "\t    Coeff = " << *Coeff << "\n");
+  DEBUG(dbgs() << "\t    SrcConst = " << *SrcConst << "\n");
+  DEBUG(dbgs() << "\t    DstConst = " << *DstConst << "\n");
+  ++WeakCrossingSIVapplications;
+  assert(0 < Level && Level <= CommonLevels && "Level out of range");
+  Level--;
+  Result.Consistent = false;
+  const SCEV *Delta = SE->getMinusSCEV(DstConst, SrcConst);
+  DEBUG(dbgs() << "\t    Delta = " << *Delta << "\n");
+  NewConstraint.setLine(Coeff, Coeff, Delta, CurLoop);
+  if (Delta->isZero()) {
+    Result.DV[Level].Direction &= unsigned(~Dependence::DVEntry::LT);
+    Result.DV[Level].Direction &= unsigned(~Dependence::DVEntry::GT);
+    ++WeakCrossingSIVsuccesses;
+    if (!Result.DV[Level].Direction) {
+      ++WeakCrossingSIVindependence;
+      return true;
+    }
+    Result.DV[Level].Distance = Delta; // = 0
+    return false;
+  }
+  const SCEVConstant *ConstCoeff = dyn_cast<SCEVConstant>(Coeff);
+  if (!ConstCoeff)
+    return false;
+
+  Result.DV[Level].Splitable = true;
+  if (SE->isKnownNegative(ConstCoeff)) {
+    ConstCoeff = dyn_cast<SCEVConstant>(SE->getNegativeSCEV(ConstCoeff));
+    assert(ConstCoeff &&
+           "dynamic cast of negative of ConstCoeff should yield constant");
+    Delta = SE->getNegativeSCEV(Delta);
+  }
+  assert(SE->isKnownPositive(ConstCoeff) && "ConstCoeff should be positive");
+
+  // compute SplitIter for use by DependenceAnalysis::getSplitIteration()
+  SplitIter =
+    SE->getUDivExpr(SE->getSMaxExpr(SE->getConstant(Delta->getType(), 0),
+                                    Delta),
+                    SE->getMulExpr(SE->getConstant(Delta->getType(), 2),
+                                   ConstCoeff));
+  DEBUG(dbgs() << "\t    Split iter = " << *SplitIter << "\n");
+
+  const SCEVConstant *ConstDelta = dyn_cast<SCEVConstant>(Delta);
+  if (!ConstDelta)
+    return false;
+
+  // We're certain that ConstCoeff > 0; therefore,
+  // if Delta < 0, then no dependence.
+  DEBUG(dbgs() << "\t    Delta = " << *Delta << "\n");
+  DEBUG(dbgs() << "\t    ConstCoeff = " << *ConstCoeff << "\n");
+  if (SE->isKnownNegative(Delta)) {
+    // No dependence, Delta < 0
+    ++WeakCrossingSIVindependence;
+    ++WeakCrossingSIVsuccesses;
+    return true;
+  }
+
+  // We're certain that Delta > 0 and ConstCoeff > 0.
+  // Check Delta/(2*ConstCoeff) against upper loop bound
+  if (const SCEV *UpperBound = collectUpperBound(CurLoop, Delta->getType())) {
+    DEBUG(dbgs() << "\t    UpperBound = " << *UpperBound << "\n");
+    const SCEV *ConstantTwo = SE->getConstant(UpperBound->getType(), 2);
+    const SCEV *ML = SE->getMulExpr(SE->getMulExpr(ConstCoeff, UpperBound),
+                                    ConstantTwo);
+    DEBUG(dbgs() << "\t    ML = " << *ML << "\n");
+    if (isKnownPredicate(CmpInst::ICMP_SGT, Delta, ML)) {
+      // Delta too big, no dependence
+      ++WeakCrossingSIVindependence;
+      ++WeakCrossingSIVsuccesses;
+      return true;
+    }
+    if (isKnownPredicate(CmpInst::ICMP_EQ, Delta, ML)) {
+      // i = i' = UB
+      Result.DV[Level].Direction &= unsigned(~Dependence::DVEntry::LT);
+      Result.DV[Level].Direction &= unsigned(~Dependence::DVEntry::GT);
+      ++WeakCrossingSIVsuccesses;
+      if (!Result.DV[Level].Direction) {
+        ++WeakCrossingSIVindependence;
+        return true;
+      }
+      Result.DV[Level].Splitable = false;
+      Result.DV[Level].Distance = SE->getConstant(Delta->getType(), 0);
+      return false;
+    }
+  }
+
+  // check that Coeff divides Delta
+  APInt APDelta = ConstDelta->getValue()->getValue();
+  APInt APCoeff = ConstCoeff->getValue()->getValue();
+  APInt Distance = APDelta; // these need to be initialzed
+  APInt Remainder = APDelta;
+  APInt::sdivrem(APDelta, APCoeff, Distance, Remainder);
+  DEBUG(dbgs() << "\t    Remainder = " << Remainder << "\n");
+  if (Remainder != 0) {
+    // Coeff doesn't divide Delta, no dependence
+    ++WeakCrossingSIVindependence;
+    ++WeakCrossingSIVsuccesses;
+    return true;
+  }
+  DEBUG(dbgs() << "\t    Distance = " << Distance << "\n");
+
+  // if 2*Coeff doesn't divide Delta, then the equal direction isn't possible
+  APInt Two = APInt(Distance.getBitWidth(), 2, true);
+  Remainder = Distance.srem(Two);
+  DEBUG(dbgs() << "\t    Remainder = " << Remainder << "\n");
+  if (Remainder != 0) {
+    // Equal direction isn't possible
+    Result.DV[Level].Direction &= unsigned(~Dependence::DVEntry::EQ);
+    ++WeakCrossingSIVsuccesses;
+  }
+  return false;
+}
+
+
+// Kirch's algorithm, from
+//
+//        Optimizing Supercompilers for Supercomputers
+//        Michael Wolfe
+//        MIT Press, 1989
+//
+// Program 2.1, page 29.
+// Computes the GCD of AM and BM.
+// Also finds a solution to the equation ax - by = gdc(a, b).
+// Returns true iff the gcd divides Delta.
+static
+bool findGCD(unsigned Bits, APInt AM, APInt BM, APInt Delta,
+             APInt &G, APInt &X, APInt &Y) {
+  APInt A0(Bits, 1, true), A1(Bits, 0, true);
+  APInt B0(Bits, 0, true), B1(Bits, 1, true);
+  APInt G0 = AM.abs();
+  APInt G1 = BM.abs();
+  APInt Q = G0; // these need to be initialized
+  APInt R = G0;
+  APInt::sdivrem(G0, G1, Q, R);
+  while (R != 0) {
+    APInt A2 = A0 - Q*A1; A0 = A1; A1 = A2;
+    APInt B2 = B0 - Q*B1; B0 = B1; B1 = B2;
+    G0 = G1; G1 = R;
+    APInt::sdivrem(G0, G1, Q, R);
+  }
+  G = G1;
+  DEBUG(dbgs() << "\t    GCD = " << G << "\n");
+  X = AM.slt(0) ? -A1 : A1;
+  Y = BM.slt(0) ? B1 : -B1;
+
+  // make sure gcd divides Delta
+  R = Delta.srem(G);
+  if (R != 0)
+    return true; // gcd doesn't divide Delta, no dependence
+  Q = Delta.sdiv(G);
+  X *= Q;
+  Y *= Q;
+  return false;
+}
+
+
+static
+APInt floorOfQuotient(APInt A, APInt B) {
+  APInt Q = A; // these need to be initialized
+  APInt R = A;
+  APInt::sdivrem(A, B, Q, R);
+  if (R == 0)
+    return Q;
+  if ((A.sgt(0) && B.sgt(0)) ||
+      (A.slt(0) && B.slt(0)))
+    return Q;
+  else
+    return Q - 1;
+}
+
+
+static
+APInt ceilingOfQuotient(APInt A, APInt B) {
+  APInt Q = A; // these need to be initialized
+  APInt R = A;
+  APInt::sdivrem(A, B, Q, R);
+  if (R == 0)
+    return Q;
+  if ((A.sgt(0) && B.sgt(0)) ||
+      (A.slt(0) && B.slt(0)))
+    return Q + 1;
+  else
+    return Q;
+}
+
+
+static
+APInt maxAPInt(APInt A, APInt B) {
+  return A.sgt(B) ? A : B;
+}
+
+
+static
+APInt minAPInt(APInt A, APInt B) {
+  return A.slt(B) ? A : B;
+}
+
+
+// exactSIVtest -
+// When we have a pair of subscripts of the form [c1 + a1*i] and [c2 + a2*i],
+// where i is an induction variable, c1 and c2 are loop invariant, and a1
+// and a2 are constant, we can solve it exactly using an algorithm developed
+// by Banerjee and Wolfe. See Section 2.5.3 in
+//
+//        Optimizing Supercompilers for Supercomputers
+//        Michael Wolfe
+//        MIT Press, 1989
+//
+// It's slower than the specialized tests (strong SIV, weak-zero SIV, etc),
+// so use them if possible. They're also a bit better with symbolics and,
+// in the case of the strong SIV test, can compute Distances.
+//
+// Return true if dependence disproved.
+bool DependenceAnalysis::exactSIVtest(const SCEV *SrcCoeff,
+                                      const SCEV *DstCoeff,
+                                      const SCEV *SrcConst,
+                                      const SCEV *DstConst,
+                                      const Loop *CurLoop,
+                                      unsigned Level,
+                                      FullDependence &Result,
+                                      Constraint &NewConstraint) const {
+  DEBUG(dbgs() << "\tExact SIV test\n");
+  DEBUG(dbgs() << "\t    SrcCoeff = " << *SrcCoeff << " = AM\n");
+  DEBUG(dbgs() << "\t    DstCoeff = " << *DstCoeff << " = BM\n");
+  DEBUG(dbgs() << "\t    SrcConst = " << *SrcConst << "\n");
+  DEBUG(dbgs() << "\t    DstConst = " << *DstConst << "\n");
+  ++ExactSIVapplications;
+  assert(0 < Level && Level <= CommonLevels && "Level out of range");
+  Level--;
+  Result.Consistent = false;
+  const SCEV *Delta = SE->getMinusSCEV(DstConst, SrcConst);
+  DEBUG(dbgs() << "\t    Delta = " << *Delta << "\n");
+  NewConstraint.setLine(SrcCoeff, SE->getNegativeSCEV(DstCoeff),
+                        Delta, CurLoop);
+  const SCEVConstant *ConstDelta = dyn_cast<SCEVConstant>(Delta);
+  const SCEVConstant *ConstSrcCoeff = dyn_cast<SCEVConstant>(SrcCoeff);
+  const SCEVConstant *ConstDstCoeff = dyn_cast<SCEVConstant>(DstCoeff);
+  if (!ConstDelta || !ConstSrcCoeff || !ConstDstCoeff)
+    return false;
+
+  // find gcd
+  APInt G, X, Y;
+  APInt AM = ConstSrcCoeff->getValue()->getValue();
+  APInt BM = ConstDstCoeff->getValue()->getValue();
+  unsigned Bits = AM.getBitWidth();
+  if (findGCD(Bits, AM, BM, ConstDelta->getValue()->getValue(), G, X, Y)) {
+    // gcd doesn't divide Delta, no dependence
+    ++ExactSIVindependence;
+    ++ExactSIVsuccesses;
+    return true;
+  }
+
+  DEBUG(dbgs() << "\t    X = " << X << ", Y = " << Y << "\n");
+
+  // since SCEV construction normalizes, LM = 0
+  APInt UM(Bits, 1, true);
+  bool UMvalid = false;
+  // UM is perhaps unavailable, let's check
+  if (const SCEVConstant *CUB =
+      collectConstantUpperBound(CurLoop, Delta->getType())) {
+    UM = CUB->getValue()->getValue();
+    DEBUG(dbgs() << "\t    UM = " << UM << "\n");
+    UMvalid = true;
+  }
+
+  APInt TU(APInt::getSignedMaxValue(Bits));
+  APInt TL(APInt::getSignedMinValue(Bits));
+
+  // test(BM/G, LM-X) and test(-BM/G, X-UM)
+  APInt TMUL = BM.sdiv(G);
+  if (TMUL.sgt(0)) {
+    TL = maxAPInt(TL, ceilingOfQuotient(-X, TMUL));
+    DEBUG(dbgs() << "\t    TL = " << TL << "\n");
+    if (UMvalid) {
+      TU = minAPInt(TU, floorOfQuotient(UM - X, TMUL));
+      DEBUG(dbgs() << "\t    TU = " << TU << "\n");
+    }
+  }
+  else {
+    TU = minAPInt(TU, floorOfQuotient(-X, TMUL));
+    DEBUG(dbgs() << "\t    TU = " << TU << "\n");
+    if (UMvalid) {
+      TL = maxAPInt(TL, ceilingOfQuotient(UM - X, TMUL));
+      DEBUG(dbgs() << "\t    TL = " << TL << "\n");
+    }
+  }
+
+  // test(AM/G, LM-Y) and test(-AM/G, Y-UM)
+  TMUL = AM.sdiv(G);
+  if (TMUL.sgt(0)) {
+    TL = maxAPInt(TL, ceilingOfQuotient(-Y, TMUL));
+    DEBUG(dbgs() << "\t    TL = " << TL << "\n");
+    if (UMvalid) {
+      TU = minAPInt(TU, floorOfQuotient(UM - Y, TMUL));
+      DEBUG(dbgs() << "\t    TU = " << TU << "\n");
+    }
+  }
+  else {
+    TU = minAPInt(TU, floorOfQuotient(-Y, TMUL));
+    DEBUG(dbgs() << "\t    TU = " << TU << "\n");
+    if (UMvalid) {
+      TL = maxAPInt(TL, ceilingOfQuotient(UM - Y, TMUL));
+      DEBUG(dbgs() << "\t    TL = " << TL << "\n");
+    }
+  }
+  if (TL.sgt(TU)) {
+    ++ExactSIVindependence;
+    ++ExactSIVsuccesses;
+    return true;
+  }
+
+  // explore directions
+  unsigned NewDirection = Dependence::DVEntry::NONE;
+
+  // less than
+  APInt SaveTU(TU); // save these
+  APInt SaveTL(TL);
+  DEBUG(dbgs() << "\t    exploring LT direction\n");
+  TMUL = AM - BM;
+  if (TMUL.sgt(0)) {
+    TL = maxAPInt(TL, ceilingOfQuotient(X - Y + 1, TMUL));
+    DEBUG(dbgs() << "\t\t    TL = " << TL << "\n");
+  }
+  else {
+    TU = minAPInt(TU, floorOfQuotient(X - Y + 1, TMUL));
+    DEBUG(dbgs() << "\t\t    TU = " << TU << "\n");
+  }
+  if (TL.sle(TU)) {
+    NewDirection |= Dependence::DVEntry::LT;
+    ++ExactSIVsuccesses;
+  }
+
+  // equal
+  TU = SaveTU; // restore
+  TL = SaveTL;
+  DEBUG(dbgs() << "\t    exploring EQ direction\n");
+  if (TMUL.sgt(0)) {
+    TL = maxAPInt(TL, ceilingOfQuotient(X - Y, TMUL));
+    DEBUG(dbgs() << "\t\t    TL = " << TL << "\n");
+  }
+  else {
+    TU = minAPInt(TU, floorOfQuotient(X - Y, TMUL));
+    DEBUG(dbgs() << "\t\t    TU = " << TU << "\n");
+  }
+  TMUL = BM - AM;
+  if (TMUL.sgt(0)) {
+    TL = maxAPInt(TL, ceilingOfQuotient(Y - X, TMUL));
+    DEBUG(dbgs() << "\t\t    TL = " << TL << "\n");
+  }
+  else {
+    TU = minAPInt(TU, floorOfQuotient(Y - X, TMUL));
+    DEBUG(dbgs() << "\t\t    TU = " << TU << "\n");
+  }
+  if (TL.sle(TU)) {
+    NewDirection |= Dependence::DVEntry::EQ;
+    ++ExactSIVsuccesses;
+  }
+
+  // greater than
+  TU = SaveTU; // restore
+  TL = SaveTL;
+  DEBUG(dbgs() << "\t    exploring GT direction\n");
+  if (TMUL.sgt(0)) {
+    TL = maxAPInt(TL, ceilingOfQuotient(Y - X + 1, TMUL));
+    DEBUG(dbgs() << "\t\t    TL = " << TL << "\n");
+  }
+  else {
+    TU = minAPInt(TU, floorOfQuotient(Y - X + 1, TMUL));
+    DEBUG(dbgs() << "\t\t    TU = " << TU << "\n");
+  }
+  if (TL.sle(TU)) {
+    NewDirection |= Dependence::DVEntry::GT;
+    ++ExactSIVsuccesses;
+  }
+
+  // finished
+  Result.DV[Level].Direction &= NewDirection;
+  if (Result.DV[Level].Direction == Dependence::DVEntry::NONE)
+    ++ExactSIVindependence;
+  return Result.DV[Level].Direction == Dependence::DVEntry::NONE;
+}
+
+
+
+// Return true if the divisor evenly divides the dividend.
+static
+bool isRemainderZero(const SCEVConstant *Dividend,
+                     const SCEVConstant *Divisor) {
+  APInt ConstDividend = Dividend->getValue()->getValue();
+  APInt ConstDivisor = Divisor->getValue()->getValue();
+  return ConstDividend.srem(ConstDivisor) == 0;
+}
+
+
+// weakZeroSrcSIVtest -
+// From the paper, Practical Dependence Testing, Section 4.2.2
+//
+// When we have a pair of subscripts of the form [c1] and [c2 + a*i],
+// where i is an induction variable, c1 and c2 are loop invariant,
+// and a is a constant, we can solve it exactly using the
+// Weak-Zero SIV test.
+//
+// Given
+//
+//    c1 = c2 + a*i
+//
+// we get
+//
+//    (c1 - c2)/a = i
+//
+// If i is not an integer, there's no dependence.
+// If i < 0 or > UB, there's no dependence.
+// If i = 0, the direction is <= and peeling the
+// 1st iteration will break the dependence.
+// If i = UB, the direction is >= and peeling the
+// last iteration will break the dependence.
+// Otherwise, the direction is *.
+//
+// Can prove independence. Failing that, we can sometimes refine
+// the directions. Can sometimes show that first or last
+// iteration carries all the dependences (so worth peeling).
+//
+// (see also weakZeroDstSIVtest)
+//
+// Return true if dependence disproved.
+bool DependenceAnalysis::weakZeroSrcSIVtest(const SCEV *DstCoeff,
+                                            const SCEV *SrcConst,
+                                            const SCEV *DstConst,
+                                            const Loop *CurLoop,
+                                            unsigned Level,
+                                            FullDependence &Result,
+                                            Constraint &NewConstraint) const {
+  // For the WeakSIV test, it's possible the loop isn't common to
+  // the Src and Dst loops. If it isn't, then there's no need to
+  // record a direction.
+  DEBUG(dbgs() << "\tWeak-Zero (src) SIV test\n");
+  DEBUG(dbgs() << "\t    DstCoeff = " << *DstCoeff << "\n");
+  DEBUG(dbgs() << "\t    SrcConst = " << *SrcConst << "\n");
+  DEBUG(dbgs() << "\t    DstConst = " << *DstConst << "\n");
+  ++WeakZeroSIVapplications;
+  assert(0 < Level && Level <= MaxLevels && "Level out of range");
+  Level--;
+  Result.Consistent = false;
+  const SCEV *Delta = SE->getMinusSCEV(SrcConst, DstConst);
+  NewConstraint.setLine(SE->getConstant(Delta->getType(), 0),
+                        DstCoeff, Delta, CurLoop);
+  DEBUG(dbgs() << "\t    Delta = " << *Delta << "\n");
+  if (isKnownPredicate(CmpInst::ICMP_EQ, SrcConst, DstConst)) {
+    if (Level < CommonLevels) {
+      Result.DV[Level].Direction &= Dependence::DVEntry::LE;
+      Result.DV[Level].PeelFirst = true;
+      ++WeakZeroSIVsuccesses;
+    }
+    return false; // dependences caused by first iteration
+  }
+  const SCEVConstant *ConstCoeff = dyn_cast<SCEVConstant>(DstCoeff);
+  if (!ConstCoeff)
+    return false;
+  const SCEV *AbsCoeff =
+    SE->isKnownNegative(ConstCoeff) ?
+    SE->getNegativeSCEV(ConstCoeff) : ConstCoeff;
+  const SCEV *NewDelta =
+    SE->isKnownNegative(ConstCoeff) ? SE->getNegativeSCEV(Delta) : Delta;
+
+  // check that Delta/SrcCoeff < iteration count
+  // really check NewDelta < count*AbsCoeff
+  if (const SCEV *UpperBound = collectUpperBound(CurLoop, Delta->getType())) {
+    DEBUG(dbgs() << "\t    UpperBound = " << *UpperBound << "\n");
+    const SCEV *Product = SE->getMulExpr(AbsCoeff, UpperBound);
+    if (isKnownPredicate(CmpInst::ICMP_SGT, NewDelta, Product)) {
+      ++WeakZeroSIVindependence;
+      ++WeakZeroSIVsuccesses;
+      return true;
+    }
+    if (isKnownPredicate(CmpInst::ICMP_EQ, NewDelta, Product)) {
+      // dependences caused by last iteration
+      if (Level < CommonLevels) {
+        Result.DV[Level].Direction &= Dependence::DVEntry::GE;
+        Result.DV[Level].PeelLast = true;
+        ++WeakZeroSIVsuccesses;
+      }
+      return false;
+    }
+  }
+
+  // check that Delta/SrcCoeff >= 0
+  // really check that NewDelta >= 0
+  if (SE->isKnownNegative(NewDelta)) {
+    // No dependence, newDelta < 0
+    ++WeakZeroSIVindependence;
+    ++WeakZeroSIVsuccesses;
+    return true;
+  }
+
+  // if SrcCoeff doesn't divide Delta, then no dependence
+  if (isa<SCEVConstant>(Delta) &&
+      !isRemainderZero(cast<SCEVConstant>(Delta), ConstCoeff)) {
+    ++WeakZeroSIVindependence;
+    ++WeakZeroSIVsuccesses;
+    return true;
+  }
+  return false;
+}
+
+
+// weakZeroDstSIVtest -
+// From the paper, Practical Dependence Testing, Section 4.2.2
+//
+// When we have a pair of subscripts of the form [c1 + a*i] and [c2],
+// where i is an induction variable, c1 and c2 are loop invariant,
+// and a is a constant, we can solve it exactly using the
+// Weak-Zero SIV test.
+//
+// Given
+//
+//    c1 + a*i = c2
+//
+// we get
+//
+//    i = (c2 - c1)/a
+//
+// If i is not an integer, there's no dependence.
+// If i < 0 or > UB, there's no dependence.
+// If i = 0, the direction is <= and peeling the
+// 1st iteration will break the dependence.
+// If i = UB, the direction is >= and peeling the
+// last iteration will break the dependence.
+// Otherwise, the direction is *.
+//
+// Can prove independence. Failing that, we can sometimes refine
+// the directions. Can sometimes show that first or last
+// iteration carries all the dependences (so worth peeling).
+//
+// (see also weakZeroSrcSIVtest)
+//
+// Return true if dependence disproved.
+bool DependenceAnalysis::weakZeroDstSIVtest(const SCEV *SrcCoeff,
+                                            const SCEV *SrcConst,
+                                            const SCEV *DstConst,
+                                            const Loop *CurLoop,
+                                            unsigned Level,
+                                            FullDependence &Result,
+                                            Constraint &NewConstraint) const {
+  // For the WeakSIV test, it's possible the loop isn't common to the
+  // Src and Dst loops. If it isn't, then there's no need to record a direction.
+  DEBUG(dbgs() << "\tWeak-Zero (dst) SIV test\n");
+  DEBUG(dbgs() << "\t    SrcCoeff = " << *SrcCoeff << "\n");
+  DEBUG(dbgs() << "\t    SrcConst = " << *SrcConst << "\n");
+  DEBUG(dbgs() << "\t    DstConst = " << *DstConst << "\n");
+  ++WeakZeroSIVapplications;
+  assert(0 < Level && Level <= SrcLevels && "Level out of range");
+  Level--;
+  Result.Consistent = false;
+  const SCEV *Delta = SE->getMinusSCEV(DstConst, SrcConst);
+  NewConstraint.setLine(SrcCoeff, SE->getConstant(Delta->getType(), 0),
+                        Delta, CurLoop);
+  DEBUG(dbgs() << "\t    Delta = " << *Delta << "\n");
+  if (isKnownPredicate(CmpInst::ICMP_EQ, DstConst, SrcConst)) {
+    if (Level < CommonLevels) {
+      Result.DV[Level].Direction &= Dependence::DVEntry::LE;
+      Result.DV[Level].PeelFirst = true;
+      ++WeakZeroSIVsuccesses;
+    }
+    return false; // dependences caused by first iteration
+  }
+  const SCEVConstant *ConstCoeff = dyn_cast<SCEVConstant>(SrcCoeff);
+  if (!ConstCoeff)
+    return false;
+  const SCEV *AbsCoeff =
+    SE->isKnownNegative(ConstCoeff) ?
+    SE->getNegativeSCEV(ConstCoeff) : ConstCoeff;
+  const SCEV *NewDelta =
+    SE->isKnownNegative(ConstCoeff) ? SE->getNegativeSCEV(Delta) : Delta;
+
+  // check that Delta/SrcCoeff < iteration count
+  // really check NewDelta < count*AbsCoeff
+  if (const SCEV *UpperBound = collectUpperBound(CurLoop, Delta->getType())) {
+    DEBUG(dbgs() << "\t    UpperBound = " << *UpperBound << "\n");
+    const SCEV *Product = SE->getMulExpr(AbsCoeff, UpperBound);
+    if (isKnownPredicate(CmpInst::ICMP_SGT, NewDelta, Product)) {
+      ++WeakZeroSIVindependence;
+      ++WeakZeroSIVsuccesses;
+      return true;
+    }
+    if (isKnownPredicate(CmpInst::ICMP_EQ, NewDelta, Product)) {
+      // dependences caused by last iteration
+      if (Level < CommonLevels) {
+        Result.DV[Level].Direction &= Dependence::DVEntry::GE;
+        Result.DV[Level].PeelLast = true;
+        ++WeakZeroSIVsuccesses;
+      }
+      return false;
+    }
+  }
+
+  // check that Delta/SrcCoeff >= 0
+  // really check that NewDelta >= 0
+  if (SE->isKnownNegative(NewDelta)) {
+    // No dependence, newDelta < 0
+    ++WeakZeroSIVindependence;
+    ++WeakZeroSIVsuccesses;
+    return true;
+  }
+
+  // if SrcCoeff doesn't divide Delta, then no dependence
+  if (isa<SCEVConstant>(Delta) &&
+      !isRemainderZero(cast<SCEVConstant>(Delta), ConstCoeff)) {
+    ++WeakZeroSIVindependence;
+    ++WeakZeroSIVsuccesses;
+    return true;
+  }
+  return false;
+}
+
+
+// exactRDIVtest - Tests the RDIV subscript pair for dependence.
+// Things of the form [c1 + a*i] and [c2 + b*j],
+// where i and j are induction variable, c1 and c2 are loop invariant,
+// and a and b are constants.
+// Returns true if any possible dependence is disproved.
+// Marks the result as inconsistent.
+// Works in some cases that symbolicRDIVtest doesn't, and vice versa.
+bool DependenceAnalysis::exactRDIVtest(const SCEV *SrcCoeff,
+                                       const SCEV *DstCoeff,
+                                       const SCEV *SrcConst,
+                                       const SCEV *DstConst,
+                                       const Loop *SrcLoop,
+                                       const Loop *DstLoop,
+                                       FullDependence &Result) const {
+  DEBUG(dbgs() << "\tExact RDIV test\n");
+  DEBUG(dbgs() << "\t    SrcCoeff = " << *SrcCoeff << " = AM\n");
+  DEBUG(dbgs() << "\t    DstCoeff = " << *DstCoeff << " = BM\n");
+  DEBUG(dbgs() << "\t    SrcConst = " << *SrcConst << "\n");
+  DEBUG(dbgs() << "\t    DstConst = " << *DstConst << "\n");
+  ++ExactRDIVapplications;
+  Result.Consistent = false;
+  const SCEV *Delta = SE->getMinusSCEV(DstConst, SrcConst);
+  DEBUG(dbgs() << "\t    Delta = " << *Delta << "\n");
+  const SCEVConstant *ConstDelta = dyn_cast<SCEVConstant>(Delta);
+  const SCEVConstant *ConstSrcCoeff = dyn_cast<SCEVConstant>(SrcCoeff);
+  const SCEVConstant *ConstDstCoeff = dyn_cast<SCEVConstant>(DstCoeff);
+  if (!ConstDelta || !ConstSrcCoeff || !ConstDstCoeff)
+    return false;
+
+  // find gcd
+  APInt G, X, Y;
+  APInt AM = ConstSrcCoeff->getValue()->getValue();
+  APInt BM = ConstDstCoeff->getValue()->getValue();
+  unsigned Bits = AM.getBitWidth();
+  if (findGCD(Bits, AM, BM, ConstDelta->getValue()->getValue(), G, X, Y)) {
+    // gcd doesn't divide Delta, no dependence
+    ++ExactRDIVindependence;
+    return true;
+  }
+
+  DEBUG(dbgs() << "\t    X = " << X << ", Y = " << Y << "\n");
+
+  // since SCEV construction seems to normalize, LM = 0
+  APInt SrcUM(Bits, 1, true);
+  bool SrcUMvalid = false;
+  // SrcUM is perhaps unavailable, let's check
+  if (const SCEVConstant *UpperBound =
+      collectConstantUpperBound(SrcLoop, Delta->getType())) {
+    SrcUM = UpperBound->getValue()->getValue();
+    DEBUG(dbgs() << "\t    SrcUM = " << SrcUM << "\n");
+    SrcUMvalid = true;
+  }
+
+  APInt DstUM(Bits, 1, true);
+  bool DstUMvalid = false;
+  // UM is perhaps unavailable, let's check
+  if (const SCEVConstant *UpperBound =
+      collectConstantUpperBound(DstLoop, Delta->getType())) {
+    DstUM = UpperBound->getValue()->getValue();
+    DEBUG(dbgs() << "\t    DstUM = " << DstUM << "\n");
+    DstUMvalid = true;
+  }
+
+  APInt TU(APInt::getSignedMaxValue(Bits));
+  APInt TL(APInt::getSignedMinValue(Bits));
+
+  // test(BM/G, LM-X) and test(-BM/G, X-UM)
+  APInt TMUL = BM.sdiv(G);
+  if (TMUL.sgt(0)) {
+    TL = maxAPInt(TL, ceilingOfQuotient(-X, TMUL));
+    DEBUG(dbgs() << "\t    TL = " << TL << "\n");
+    if (SrcUMvalid) {
+      TU = minAPInt(TU, floorOfQuotient(SrcUM - X, TMUL));
+      DEBUG(dbgs() << "\t    TU = " << TU << "\n");
+    }
+  }
+  else {
+    TU = minAPInt(TU, floorOfQuotient(-X, TMUL));
+    DEBUG(dbgs() << "\t    TU = " << TU << "\n");
+    if (SrcUMvalid) {
+      TL = maxAPInt(TL, ceilingOfQuotient(SrcUM - X, TMUL));
+      DEBUG(dbgs() << "\t    TL = " << TL << "\n");
+    }
+  }
+
+  // test(AM/G, LM-Y) and test(-AM/G, Y-UM)
+  TMUL = AM.sdiv(G);
+  if (TMUL.sgt(0)) {
+    TL = maxAPInt(TL, ceilingOfQuotient(-Y, TMUL));
+    DEBUG(dbgs() << "\t    TL = " << TL << "\n");
+    if (DstUMvalid) {
+      TU = minAPInt(TU, floorOfQuotient(DstUM - Y, TMUL));
+      DEBUG(dbgs() << "\t    TU = " << TU << "\n");
+    }
+  }
+  else {
+    TU = minAPInt(TU, floorOfQuotient(-Y, TMUL));
+    DEBUG(dbgs() << "\t    TU = " << TU << "\n");
+    if (DstUMvalid) {
+      TL = maxAPInt(TL, ceilingOfQuotient(DstUM - Y, TMUL));
+      DEBUG(dbgs() << "\t    TL = " << TL << "\n");
+    }
+  }
+  if (TL.sgt(TU))
+    ++ExactRDIVindependence;
+  return TL.sgt(TU);
+}
+
+
+// symbolicRDIVtest -
+// In Section 4.5 of the Practical Dependence Testing paper,the authors
+// introduce a special case of Banerjee's Inequalities (also called the
+// Extreme-Value Test) that can handle some of the SIV and RDIV cases,
+// particularly cases with symbolics. Since it's only able to disprove
+// dependence (not compute distances or directions), we'll use it as a
+// fall back for the other tests.
+//
+// When we have a pair of subscripts of the form [c1 + a1*i] and [c2 + a2*j]
+// where i and j are induction variables and c1 and c2 are loop invariants,
+// we can use the symbolic tests to disprove some dependences, serving as a
+// backup for the RDIV test. Note that i and j can be the same variable,
+// letting this test serve as a backup for the various SIV tests.
+//
+// For a dependence to exist, c1 + a1*i must equal c2 + a2*j for some
+//  0 <= i <= N1 and some 0 <= j <= N2, where N1 and N2 are the (normalized)
+// loop bounds for the i and j loops, respectively. So, ...
+//
+// c1 + a1*i = c2 + a2*j
+// a1*i - a2*j = c2 - c1
+//
+// To test for a dependence, we compute c2 - c1 and make sure it's in the
+// range of the maximum and minimum possible values of a1*i - a2*j.
+// Considering the signs of a1 and a2, we have 4 possible cases:
+//
+// 1) If a1 >= 0 and a2 >= 0, then
+//        a1*0 - a2*N2 <= c2 - c1 <= a1*N1 - a2*0
+//              -a2*N2 <= c2 - c1 <= a1*N1
+//
+// 2) If a1 >= 0 and a2 <= 0, then
+//        a1*0 - a2*0 <= c2 - c1 <= a1*N1 - a2*N2
+//                  0 <= c2 - c1 <= a1*N1 - a2*N2
+//
+// 3) If a1 <= 0 and a2 >= 0, then
+//        a1*N1 - a2*N2 <= c2 - c1 <= a1*0 - a2*0
+//        a1*N1 - a2*N2 <= c2 - c1 <= 0
+//
+// 4) If a1 <= 0 and a2 <= 0, then
+//        a1*N1 - a2*0  <= c2 - c1 <= a1*0 - a2*N2
+//        a1*N1         <= c2 - c1 <=       -a2*N2
+//
+// return true if dependence disproved
+bool DependenceAnalysis::symbolicRDIVtest(const SCEV *A1,
+                                          const SCEV *A2,
+                                          const SCEV *C1,
+                                          const SCEV *C2,
+                                          const Loop *Loop1,
+                                          const Loop *Loop2) const {
+  ++SymbolicRDIVapplications;
+  DEBUG(dbgs() << "\ttry symbolic RDIV test\n");
+  DEBUG(dbgs() << "\t    A1 = " << *A1);
+  DEBUG(dbgs() << ", type = " << *A1->getType() << "\n");
+  DEBUG(dbgs() << "\t    A2 = " << *A2 << "\n");
+  DEBUG(dbgs() << "\t    C1 = " << *C1 << "\n");
+  DEBUG(dbgs() << "\t    C2 = " << *C2 << "\n");
+  const SCEV *N1 = collectUpperBound(Loop1, A1->getType());
+  const SCEV *N2 = collectUpperBound(Loop2, A1->getType());
+  DEBUG(if (N1) dbgs() << "\t    N1 = " << *N1 << "\n");
+  DEBUG(if (N2) dbgs() << "\t    N2 = " << *N2 << "\n");
+  const SCEV *C2_C1 = SE->getMinusSCEV(C2, C1);
+  const SCEV *C1_C2 = SE->getMinusSCEV(C1, C2);
+  DEBUG(dbgs() << "\t    C2 - C1 = " << *C2_C1 << "\n");
+  DEBUG(dbgs() << "\t    C1 - C2 = " << *C1_C2 << "\n");
+  if (SE->isKnownNonNegative(A1)) {
+    if (SE->isKnownNonNegative(A2)) {
+      // A1 >= 0 && A2 >= 0
+      if (N1) {
+        // make sure that c2 - c1 <= a1*N1
+        const SCEV *A1N1 = SE->getMulExpr(A1, N1);
+        DEBUG(dbgs() << "\t    A1*N1 = " << *A1N1 << "\n");
+        if (isKnownPredicate(CmpInst::ICMP_SGT, C2_C1, A1N1)) {
+          ++SymbolicRDIVindependence;
+          return true;
+        }
+      }
+      if (N2) {
+        // make sure that -a2*N2 <= c2 - c1, or a2*N2 >= c1 - c2
+        const SCEV *A2N2 = SE->getMulExpr(A2, N2);
+        DEBUG(dbgs() << "\t    A2*N2 = " << *A2N2 << "\n");
+        if (isKnownPredicate(CmpInst::ICMP_SLT, A2N2, C1_C2)) {
+          ++SymbolicRDIVindependence;
+          return true;
+        }
+      }
+    }
+    else if (SE->isKnownNonPositive(A2)) {
+      // a1 >= 0 && a2 <= 0
+      if (N1 && N2) {
+        // make sure that c2 - c1 <= a1*N1 - a2*N2
+        const SCEV *A1N1 = SE->getMulExpr(A1, N1);
+        const SCEV *A2N2 = SE->getMulExpr(A2, N2);
+        const SCEV *A1N1_A2N2 = SE->getMinusSCEV(A1N1, A2N2);
+        DEBUG(dbgs() << "\t    A1*N1 - A2*N2 = " << *A1N1_A2N2 << "\n");
+        if (isKnownPredicate(CmpInst::ICMP_SGT, C2_C1, A1N1_A2N2)) {
+          ++SymbolicRDIVindependence;
+          return true;
+        }
+      }
+      // make sure that 0 <= c2 - c1
+      if (SE->isKnownNegative(C2_C1)) {
+        ++SymbolicRDIVindependence;
+        return true;
+      }
+    }
+  }
+  else if (SE->isKnownNonPositive(A1)) {
+    if (SE->isKnownNonNegative(A2)) {
+      // a1 <= 0 && a2 >= 0
+      if (N1 && N2) {
+        // make sure that a1*N1 - a2*N2 <= c2 - c1
+        const SCEV *A1N1 = SE->getMulExpr(A1, N1);
+        const SCEV *A2N2 = SE->getMulExpr(A2, N2);
+        const SCEV *A1N1_A2N2 = SE->getMinusSCEV(A1N1, A2N2);
+        DEBUG(dbgs() << "\t    A1*N1 - A2*N2 = " << *A1N1_A2N2 << "\n");
+        if (isKnownPredicate(CmpInst::ICMP_SGT, A1N1_A2N2, C2_C1)) {
+          ++SymbolicRDIVindependence;
+          return true;
+        }
+      }
+      // make sure that c2 - c1 <= 0
+      if (SE->isKnownPositive(C2_C1)) {
+        ++SymbolicRDIVindependence;
+        return true;
+      }
+    }
+    else if (SE->isKnownNonPositive(A2)) {
+      // a1 <= 0 && a2 <= 0
+      if (N1) {
+        // make sure that a1*N1 <= c2 - c1
+        const SCEV *A1N1 = SE->getMulExpr(A1, N1);
+        DEBUG(dbgs() << "\t    A1*N1 = " << *A1N1 << "\n");
+        if (isKnownPredicate(CmpInst::ICMP_SGT, A1N1, C2_C1)) {
+          ++SymbolicRDIVindependence;
+          return true;
+        }
+      }
+      if (N2) {
+        // make sure that c2 - c1 <= -a2*N2, or c1 - c2 >= a2*N2
+        const SCEV *A2N2 = SE->getMulExpr(A2, N2);
+        DEBUG(dbgs() << "\t    A2*N2 = " << *A2N2 << "\n");
+        if (isKnownPredicate(CmpInst::ICMP_SLT, C1_C2, A2N2)) {
+          ++SymbolicRDIVindependence;
+          return true;
+        }
+      }
+    }
+  }
+  return false;
+}
+
+
+// testSIV -
+// When we have a pair of subscripts of the form [c1 + a1*i] and [c2 - a2*i]
+// where i is an induction variable, c1 and c2 are loop invariant, and a1 and
+// a2 are constant, we attack it with an SIV test. While they can all be
+// solved with the Exact SIV test, it's worthwhile to use simpler tests when
+// they apply; they're cheaper and sometimes more precise.
+//
+// Return true if dependence disproved.
+bool DependenceAnalysis::testSIV(const SCEV *Src,
+                                 const SCEV *Dst,
+                                 unsigned &Level,
+                                 FullDependence &Result,
+                                 Constraint &NewConstraint,
+                                 const SCEV *&SplitIter) const {
+  DEBUG(dbgs() << "    src = " << *Src << "\n");
+  DEBUG(dbgs() << "    dst = " << *Dst << "\n");
+  const SCEVAddRecExpr *SrcAddRec = dyn_cast<SCEVAddRecExpr>(Src);
+  const SCEVAddRecExpr *DstAddRec = dyn_cast<SCEVAddRecExpr>(Dst);
+  if (SrcAddRec && DstAddRec) {
+    const SCEV *SrcConst = SrcAddRec->getStart();
+    const SCEV *DstConst = DstAddRec->getStart();
+    const SCEV *SrcCoeff = SrcAddRec->getStepRecurrence(*SE);
+    const SCEV *DstCoeff = DstAddRec->getStepRecurrence(*SE);
+    const Loop *CurLoop = SrcAddRec->getLoop();
+    assert(CurLoop == DstAddRec->getLoop() &&
+           "both loops in SIV should be same");
+    Level = mapSrcLoop(CurLoop);
+    bool disproven;
+    if (SrcCoeff == DstCoeff)
+      disproven = strongSIVtest(SrcCoeff, SrcConst, DstConst, CurLoop,
+                                Level, Result, NewConstraint);
+    else if (SrcCoeff == SE->getNegativeSCEV(DstCoeff))
+      disproven = weakCrossingSIVtest(SrcCoeff, SrcConst, DstConst, CurLoop,
+                                      Level, Result, NewConstraint, SplitIter);
+    else
+      disproven = exactSIVtest(SrcCoeff, DstCoeff, SrcConst, DstConst, CurLoop,
+                               Level, Result, NewConstraint);
+    return disproven ||
+      gcdMIVtest(Src, Dst, Result) ||
+      symbolicRDIVtest(SrcCoeff, DstCoeff, SrcConst, DstConst, CurLoop, CurLoop);
+  }
+  if (SrcAddRec) {
+    const SCEV *SrcConst = SrcAddRec->getStart();
+    const SCEV *SrcCoeff = SrcAddRec->getStepRecurrence(*SE);
+    const SCEV *DstConst = Dst;
+    const Loop *CurLoop = SrcAddRec->getLoop();
+    Level = mapSrcLoop(CurLoop);
+    return weakZeroDstSIVtest(SrcCoeff, SrcConst, DstConst, CurLoop,
+                              Level, Result, NewConstraint) ||
+      gcdMIVtest(Src, Dst, Result);
+  }
+  if (DstAddRec) {
+    const SCEV *DstConst = DstAddRec->getStart();
+    const SCEV *DstCoeff = DstAddRec->getStepRecurrence(*SE);
+    const SCEV *SrcConst = Src;
+    const Loop *CurLoop = DstAddRec->getLoop();
+    Level = mapDstLoop(CurLoop);
+    return weakZeroSrcSIVtest(DstCoeff, SrcConst, DstConst,
+                              CurLoop, Level, Result, NewConstraint) ||
+      gcdMIVtest(Src, Dst, Result);
+  }
+  llvm_unreachable("SIV test expected at least one AddRec");
+  return false;
+}
+
+
+// testRDIV -
+// When we have a pair of subscripts of the form [c1 + a1*i] and [c2 + a2*j]
+// where i and j are induction variables, c1 and c2 are loop invariant,
+// and a1 and a2 are constant, we can solve it exactly with an easy adaptation
+// of the Exact SIV test, the Restricted Double Index Variable (RDIV) test.
+// It doesn't make sense to talk about distance or direction in this case,
+// so there's no point in making special versions of the Strong SIV test or
+// the Weak-crossing SIV test.
+//
+// With minor algebra, this test can also be used for things like
+// [c1 + a1*i + a2*j][c2].
+//
+// Return true if dependence disproved.
+bool DependenceAnalysis::testRDIV(const SCEV *Src,
+                                  const SCEV *Dst,
+                                  FullDependence &Result) const {
+  // we have 3 possible situations here:
+  //   1) [a*i + b] and [c*j + d]
+  //   2) [a*i + c*j + b] and [d]
+  //   3) [b] and [a*i + c*j + d]
+  // We need to find what we've got and get organized
+
+  const SCEV *SrcConst, *DstConst;
+  const SCEV *SrcCoeff, *DstCoeff;
+  const Loop *SrcLoop, *DstLoop;
+
+  DEBUG(dbgs() << "    src = " << *Src << "\n");
+  DEBUG(dbgs() << "    dst = " << *Dst << "\n");
+  const SCEVAddRecExpr *SrcAddRec = dyn_cast<SCEVAddRecExpr>(Src);
+  const SCEVAddRecExpr *DstAddRec = dyn_cast<SCEVAddRecExpr>(Dst);
+  if (SrcAddRec && DstAddRec) {
+    SrcConst = SrcAddRec->getStart();
+    SrcCoeff = SrcAddRec->getStepRecurrence(*SE);
+    SrcLoop = SrcAddRec->getLoop();
+    DstConst = DstAddRec->getStart();
+    DstCoeff = DstAddRec->getStepRecurrence(*SE);
+    DstLoop = DstAddRec->getLoop();
+  }
+  else if (SrcAddRec) {
+    if (const SCEVAddRecExpr *tmpAddRec =
+        dyn_cast<SCEVAddRecExpr>(SrcAddRec->getStart())) {
+      SrcConst = tmpAddRec->getStart();
+      SrcCoeff = tmpAddRec->getStepRecurrence(*SE);
+      SrcLoop = tmpAddRec->getLoop();
+      DstConst = Dst;
+      DstCoeff = SE->getNegativeSCEV(SrcAddRec->getStepRecurrence(*SE));
+      DstLoop = SrcAddRec->getLoop();
+    }
+    else
+      llvm_unreachable("RDIV reached by surprising SCEVs");
+  }
+  else if (DstAddRec) {
+    if (const SCEVAddRecExpr *tmpAddRec =
+        dyn_cast<SCEVAddRecExpr>(DstAddRec->getStart())) {
+      DstConst = tmpAddRec->getStart();
+      DstCoeff = tmpAddRec->getStepRecurrence(*SE);
+      DstLoop = tmpAddRec->getLoop();
+      SrcConst = Src;
+      SrcCoeff = SE->getNegativeSCEV(DstAddRec->getStepRecurrence(*SE));
+      SrcLoop = DstAddRec->getLoop();
+    }
+    else
+      llvm_unreachable("RDIV reached by surprising SCEVs");
+  }
+  else
+    llvm_unreachable("RDIV expected at least one AddRec");
+  return exactRDIVtest(SrcCoeff, DstCoeff,
+                       SrcConst, DstConst,
+                       SrcLoop, DstLoop,
+                       Result) ||
+    gcdMIVtest(Src, Dst, Result) ||
+    symbolicRDIVtest(SrcCoeff, DstCoeff,
+                     SrcConst, DstConst,
+                     SrcLoop, DstLoop);
+}
+
+
+// Tests the single-subscript MIV pair (Src and Dst) for dependence.
+// Return true if dependence disproved.
+// Can sometimes refine direction vectors.
+bool DependenceAnalysis::testMIV(const SCEV *Src,
+                                 const SCEV *Dst,
+                                 const SmallBitVector &Loops,
+                                 FullDependence &Result) const {
+  DEBUG(dbgs() << "    src = " << *Src << "\n");
+  DEBUG(dbgs() << "    dst = " << *Dst << "\n");
+  Result.Consistent = false;
+  return gcdMIVtest(Src, Dst, Result) ||
+    banerjeeMIVtest(Src, Dst, Loops, Result);
+}
+
+
+// Given a product, e.g., 10*X*Y, returns the first constant operand,
+// in this case 10. If there is no constant part, returns NULL.
+static
+const SCEVConstant *getConstantPart(const SCEVMulExpr *Product) {
+  for (unsigned Op = 0, Ops = Product->getNumOperands(); Op < Ops; Op++) {
+    if (const SCEVConstant *Constant = dyn_cast<SCEVConstant>(Product->getOperand(Op)))
+      return Constant;
+  }
+  return NULL;
+}
+
+
+//===----------------------------------------------------------------------===//
+// gcdMIVtest -
+// Tests an MIV subscript pair for dependence.
+// Returns true if any possible dependence is disproved.
+// Marks the result as inconsistent.
+// Can sometimes disprove the equal direction for 1 or more loops,
+// as discussed in Michael Wolfe's book,
+// High Performance Compilers for Parallel Computing, page 235.
+//
+// We spend some effort (code!) to handle cases like
+// [10*i + 5*N*j + 15*M + 6], where i and j are induction variables,
+// but M and N are just loop-invariant variables.
+// This should help us handle linearized subscripts;
+// also makes this test a useful backup to the various SIV tests.
+//
+// It occurs to me that the presence of loop-invariant variables
+// changes the nature of the test from "greatest common divisor"
+// to "a common divisor!"
+bool DependenceAnalysis::gcdMIVtest(const SCEV *Src,
+                                    const SCEV *Dst,
+                                    FullDependence &Result) const {
+  DEBUG(dbgs() << "starting gcd\n");
+  ++GCDapplications;
+  unsigned BitWidth = Src->getType()->getIntegerBitWidth();
+  APInt RunningGCD = APInt::getNullValue(BitWidth);
+
+  // Examine Src coefficients.
+  // Compute running GCD and record source constant.
+  // Because we're looking for the constant at the end of the chain,
+  // we can't quit the loop just because the GCD == 1.
+  const SCEV *Coefficients = Src;
+  while (const SCEVAddRecExpr *AddRec =
+         dyn_cast<SCEVAddRecExpr>(Coefficients)) {
+    const SCEV *Coeff = AddRec->getStepRecurrence(*SE);
+    const SCEVConstant *Constant = dyn_cast<SCEVConstant>(Coeff);
+    if (const SCEVMulExpr *Product = dyn_cast<SCEVMulExpr>(Coeff))
+      // If the coefficient is the product of a constant and other stuff,
+      // we can use the constant in the GCD computation.
+      Constant = getConstantPart(Product);
+    if (!Constant)
+      return false;
+    APInt ConstCoeff = Constant->getValue()->getValue();
+    RunningGCD = APIntOps::GreatestCommonDivisor(RunningGCD, ConstCoeff.abs());
+    Coefficients = AddRec->getStart();
+  }
+  const SCEV *SrcConst = Coefficients;
+
+  // Examine Dst coefficients.
+  // Compute running GCD and record destination constant.
+  // Because we're looking for the constant at the end of the chain,
+  // we can't quit the loop just because the GCD == 1.
+  Coefficients = Dst;
+  while (const SCEVAddRecExpr *AddRec =
+         dyn_cast<SCEVAddRecExpr>(Coefficients)) {
+    const SCEV *Coeff = AddRec->getStepRecurrence(*SE);
+    const SCEVConstant *Constant = dyn_cast<SCEVConstant>(Coeff);
+    if (const SCEVMulExpr *Product = dyn_cast<SCEVMulExpr>(Coeff))
+      // If the coefficient is the product of a constant and other stuff,
+      // we can use the constant in the GCD computation.
+      Constant = getConstantPart(Product);
+    if (!Constant)
+      return false;
+    APInt ConstCoeff = Constant->getValue()->getValue();
+    RunningGCD = APIntOps::GreatestCommonDivisor(RunningGCD, ConstCoeff.abs());
+    Coefficients = AddRec->getStart();
+  }
+  const SCEV *DstConst = Coefficients;
+
+  APInt ExtraGCD = APInt::getNullValue(BitWidth);
+  const SCEV *Delta = SE->getMinusSCEV(DstConst, SrcConst);
+  DEBUG(dbgs() << "    Delta = " << *Delta << "\n");
+  const SCEVConstant *Constant = dyn_cast<SCEVConstant>(Delta);
+  if (const SCEVAddExpr *Sum = dyn_cast<SCEVAddExpr>(Delta)) {
+    // If Delta is a sum of products, we may be able to make further progress.
+    for (unsigned Op = 0, Ops = Sum->getNumOperands(); Op < Ops; Op++) {
+      const SCEV *Operand = Sum->getOperand(Op);
+      if (isa<SCEVConstant>(Operand)) {
+        assert(!Constant && "Surprised to find multiple constants");
+        Constant = cast<SCEVConstant>(Operand);
+      }
+      else if (const SCEVMulExpr *Product = dyn_cast<SCEVMulExpr>(Operand)) {
+        // Search for constant operand to participate in GCD;
+        // If none found; return false.
+        const SCEVConstant *ConstOp = getConstantPart(Product);
+        if (!ConstOp)
+          return false;
+        APInt ConstOpValue = ConstOp->getValue()->getValue();
+        ExtraGCD = APIntOps::GreatestCommonDivisor(ExtraGCD,
+                                                   ConstOpValue.abs());
+      }
+      else
+        return false;
+    }
+  }
+  if (!Constant)
+    return false;
+  APInt ConstDelta = cast<SCEVConstant>(Constant)->getValue()->getValue();
+  DEBUG(dbgs() << "    ConstDelta = " << ConstDelta << "\n");
+  if (ConstDelta == 0)
+    return false;
+  RunningGCD = APIntOps::GreatestCommonDivisor(RunningGCD, ExtraGCD);
+  DEBUG(dbgs() << "    RunningGCD = " << RunningGCD << "\n");
+  APInt Remainder = ConstDelta.srem(RunningGCD);
+  if (Remainder != 0) {
+    ++GCDindependence;
+    return true;
+  }
+
+  // Try to disprove equal directions.
+  // For example, given a subscript pair [3*i + 2*j] and [i' + 2*j' - 1],
+  // the code above can't disprove the dependence because the GCD = 1.
+  // So we consider what happen if i = i' and what happens if j = j'.
+  // If i = i', we can simplify the subscript to [2*i + 2*j] and [2*j' - 1],
+  // which is infeasible, so we can disallow the = direction for the i level.
+  // Setting j = j' doesn't help matters, so we end up with a direction vector
+  // of [<>, *]
+  //
+  // Given A[5*i + 10*j*M + 9*M*N] and A[15*i + 20*j*M - 21*N*M + 5],
+  // we need to remember that the constant part is 5 and the RunningGCD should
+  // be initialized to ExtraGCD = 30.
+  DEBUG(dbgs() << "    ExtraGCD = " << ExtraGCD << '\n');
+
+  bool Improved = false;
+  Coefficients = Src;
+  while (const SCEVAddRecExpr *AddRec =
+         dyn_cast<SCEVAddRecExpr>(Coefficients)) {
+    Coefficients = AddRec->getStart();
+    const Loop *CurLoop = AddRec->getLoop();
+    RunningGCD = ExtraGCD;
+    const SCEV *SrcCoeff = AddRec->getStepRecurrence(*SE);
+    const SCEV *DstCoeff = SE->getMinusSCEV(SrcCoeff, SrcCoeff);
+    const SCEV *Inner = Src;
+    while (RunningGCD != 1 && isa<SCEVAddRecExpr>(Inner)) {
+      AddRec = cast<SCEVAddRecExpr>(Inner);
+      const SCEV *Coeff = AddRec->getStepRecurrence(*SE);
+      if (CurLoop == AddRec->getLoop())
+        ; // SrcCoeff == Coeff
+      else {
+        if (const SCEVMulExpr *Product = dyn_cast<SCEVMulExpr>(Coeff))
+          // If the coefficient is the product of a constant and other stuff,
+          // we can use the constant in the GCD computation.
+          Constant = getConstantPart(Product);
+        else
+          Constant = cast<SCEVConstant>(Coeff);
+        APInt ConstCoeff = Constant->getValue()->getValue();
+        RunningGCD = APIntOps::GreatestCommonDivisor(RunningGCD, ConstCoeff.abs());
+      }
+      Inner = AddRec->getStart();
+    }
+    Inner = Dst;
+    while (RunningGCD != 1 && isa<SCEVAddRecExpr>(Inner)) {
+      AddRec = cast<SCEVAddRecExpr>(Inner);
+      const SCEV *Coeff = AddRec->getStepRecurrence(*SE);
+      if (CurLoop == AddRec->getLoop())
+        DstCoeff = Coeff;
+      else {
+        if (const SCEVMulExpr *Product = dyn_cast<SCEVMulExpr>(Coeff))
+          // If the coefficient is the product of a constant and other stuff,
+          // we can use the constant in the GCD computation.
+          Constant = getConstantPart(Product);
+        else
+          Constant = cast<SCEVConstant>(Coeff);
+        APInt ConstCoeff = Constant->getValue()->getValue();
+        RunningGCD = APIntOps::GreatestCommonDivisor(RunningGCD, ConstCoeff.abs());
+      }
+      Inner = AddRec->getStart();
+    }
+    Delta = SE->getMinusSCEV(SrcCoeff, DstCoeff);
+    if (const SCEVMulExpr *Product = dyn_cast<SCEVMulExpr>(Delta))
+      // If the coefficient is the product of a constant and other stuff,
+      // we can use the constant in the GCD computation.
+      Constant = getConstantPart(Product);
+    else if (isa<SCEVConstant>(Delta))
+      Constant = cast<SCEVConstant>(Delta);
+    else {
+      // The difference of the two coefficients might not be a product
+      // or constant, in which case we give up on this direction.
+      continue;
+    }
+    APInt ConstCoeff = Constant->getValue()->getValue();
+    RunningGCD = APIntOps::GreatestCommonDivisor(RunningGCD, ConstCoeff.abs());
+    DEBUG(dbgs() << "\tRunningGCD = " << RunningGCD << "\n");
+    if (RunningGCD != 0) {
+      Remainder = ConstDelta.srem(RunningGCD);
+      DEBUG(dbgs() << "\tRemainder = " << Remainder << "\n");
+      if (Remainder != 0) {
+        unsigned Level = mapSrcLoop(CurLoop);
+        Result.DV[Level - 1].Direction &= unsigned(~Dependence::DVEntry::EQ);
+        Improved = true;
+      }
+    }
+  }
+  if (Improved)
+    ++GCDsuccesses;
+  DEBUG(dbgs() << "all done\n");
+  return false;
+}
+
+
+//===----------------------------------------------------------------------===//
+// banerjeeMIVtest -
+// Use Banerjee's Inequalities to test an MIV subscript pair.
+// (Wolfe, in the race-car book, calls this the Extreme Value Test.)
+// Generally follows the discussion in Section 2.5.2 of
+//
+//    Optimizing Supercompilers for Supercomputers
+//    Michael Wolfe
+//
+// The inequalities given on page 25 are simplified in that loops are
+// normalized so that the lower bound is always 0 and the stride is always 1.
+// For example, Wolfe gives
+//
+//     LB^<_k = (A^-_k - B_k)^- (U_k - L_k - N_k) + (A_k - B_k)L_k - B_k N_k
+//
+// where A_k is the coefficient of the kth index in the source subscript,
+// B_k is the coefficient of the kth index in the destination subscript,
+// U_k is the upper bound of the kth index, L_k is the lower bound of the Kth
+// index, and N_k is the stride of the kth index. Since all loops are normalized
+// by the SCEV package, N_k = 1 and L_k = 0, allowing us to simplify the
+// equation to
+//
+//     LB^<_k = (A^-_k - B_k)^- (U_k - 0 - 1) + (A_k - B_k)0 - B_k 1
+//            = (A^-_k - B_k)^- (U_k - 1)  - B_k
+//
+// Similar simplifications are possible for the other equations.
+//
+// When we can't determine the number of iterations for a loop,
+// we use NULL as an indicator for the worst case, infinity.
+// When computing the upper bound, NULL denotes +inf;
+// for the lower bound, NULL denotes -inf.
+//
+// Return true if dependence disproved.
+bool DependenceAnalysis::banerjeeMIVtest(const SCEV *Src,
+                                         const SCEV *Dst,
+                                         const SmallBitVector &Loops,
+                                         FullDependence &Result) const {
+  DEBUG(dbgs() << "starting Banerjee\n");
+  ++BanerjeeApplications;
+  DEBUG(dbgs() << "    Src = " << *Src << '\n');
+  const SCEV *A0;
+  CoefficientInfo *A = collectCoeffInfo(Src, true, A0);
+  DEBUG(dbgs() << "    Dst = " << *Dst << '\n');
+  const SCEV *B0;
+  CoefficientInfo *B = collectCoeffInfo(Dst, false, B0);
+  BoundInfo *Bound = new BoundInfo[MaxLevels + 1];
+  const SCEV *Delta = SE->getMinusSCEV(B0, A0);
+  DEBUG(dbgs() << "\tDelta = " << *Delta << '\n');
+
+  // Compute bounds for all the * directions.
+  DEBUG(dbgs() << "\tBounds[*]\n");
+  for (unsigned K = 1; K <= MaxLevels; ++K) {
+    Bound[K].Iterations = A[K].Iterations ? A[K].Iterations : B[K].Iterations;
+    Bound[K].Direction = Dependence::DVEntry::ALL;
+    Bound[K].DirSet = Dependence::DVEntry::NONE;
+    findBoundsALL(A, B, Bound, K);
+#ifndef NDEBUG
+    DEBUG(dbgs() << "\t    " << K << '\t');
+    if (Bound[K].Lower[Dependence::DVEntry::ALL])
+      DEBUG(dbgs() << *Bound[K].Lower[Dependence::DVEntry::ALL] << '\t');
+    else
+      DEBUG(dbgs() << "-inf\t");
+    if (Bound[K].Upper[Dependence::DVEntry::ALL])
+      DEBUG(dbgs() << *Bound[K].Upper[Dependence::DVEntry::ALL] << '\n');
+    else
+      DEBUG(dbgs() << "+inf\n");
+#endif
+  }
+
+  // Test the *, *, *, ... case.
+  bool Disproved = false;
+  if (testBounds(Dependence::DVEntry::ALL, 0, Bound, Delta)) {
+    // Explore the direction vector hierarchy.
+    unsigned DepthExpanded = 0;
+    unsigned NewDeps = exploreDirections(1, A, B, Bound,
+                                         Loops, DepthExpanded, Delta);
+    if (NewDeps > 0) {
+      bool Improved = false;
+      for (unsigned K = 1; K <= CommonLevels; ++K) {
+        if (Loops[K]) {
+          unsigned Old = Result.DV[K - 1].Direction;
+          Result.DV[K - 1].Direction = Old & Bound[K].DirSet;
+          Improved |= Old != Result.DV[K - 1].Direction;
+          if (!Result.DV[K - 1].Direction) {
+            Improved = false;
+            Disproved = true;
+            break;
+          }
+        }
+      }
+      if (Improved)
+        ++BanerjeeSuccesses;
+    }
+    else {
+      ++BanerjeeIndependence;
+      Disproved = true;
+    }
+  }
+  else {
+    ++BanerjeeIndependence;
+    Disproved = true;
+  }
+  delete [] Bound;
+  delete [] A;
+  delete [] B;
+  return Disproved;
+}
+
+
+// Hierarchically expands the direction vector
+// search space, combining the directions of discovered dependences
+// in the DirSet field of Bound. Returns the number of distinct
+// dependences discovered. If the dependence is disproved,
+// it will return 0.
+unsigned DependenceAnalysis::exploreDirections(unsigned Level,
+                                               CoefficientInfo *A,
+                                               CoefficientInfo *B,
+                                               BoundInfo *Bound,
+                                               const SmallBitVector &Loops,
+                                               unsigned &DepthExpanded,
+                                               const SCEV *Delta) const {
+  if (Level > CommonLevels) {
+    // record result
+    DEBUG(dbgs() << "\t[");
+    for (unsigned K = 1; K <= CommonLevels; ++K) {
+      if (Loops[K]) {
+        Bound[K].DirSet |= Bound[K].Direction;
+#ifndef NDEBUG
+        switch (Bound[K].Direction) {
+        case Dependence::DVEntry::LT:
+          DEBUG(dbgs() << " <");
+          break;
+        case Dependence::DVEntry::EQ:
+          DEBUG(dbgs() << " =");
+          break;
+        case Dependence::DVEntry::GT:
+          DEBUG(dbgs() << " >");
+          break;
+        case Dependence::DVEntry::ALL:
+          DEBUG(dbgs() << " *");
+          break;
+        default:
+          llvm_unreachable("unexpected Bound[K].Direction");
+        }
+#endif
+      }
+    }
+    DEBUG(dbgs() << " ]\n");
+    return 1;
+  }
+  if (Loops[Level]) {
+    if (Level > DepthExpanded) {
+      DepthExpanded = Level;
+      // compute bounds for <, =, > at current level
+      findBoundsLT(A, B, Bound, Level);
+      findBoundsGT(A, B, Bound, Level);
+      findBoundsEQ(A, B, Bound, Level);
+#ifndef NDEBUG
+      DEBUG(dbgs() << "\tBound for level = " << Level << '\n');
+      DEBUG(dbgs() << "\t    <\t");
+      if (Bound[Level].Lower[Dependence::DVEntry::LT])
+        DEBUG(dbgs() << *Bound[Level].Lower[Dependence::DVEntry::LT] << '\t');
+      else
+        DEBUG(dbgs() << "-inf\t");
+      if (Bound[Level].Upper[Dependence::DVEntry::LT])
+        DEBUG(dbgs() << *Bound[Level].Upper[Dependence::DVEntry::LT] << '\n');
+      else
+        DEBUG(dbgs() << "+inf\n");
+      DEBUG(dbgs() << "\t    =\t");
+      if (Bound[Level].Lower[Dependence::DVEntry::EQ])
+        DEBUG(dbgs() << *Bound[Level].Lower[Dependence::DVEntry::EQ] << '\t');
+      else
+        DEBUG(dbgs() << "-inf\t");
+      if (Bound[Level].Upper[Dependence::DVEntry::EQ])
+        DEBUG(dbgs() << *Bound[Level].Upper[Dependence::DVEntry::EQ] << '\n');
+      else
+        DEBUG(dbgs() << "+inf\n");
+      DEBUG(dbgs() << "\t    >\t");
+      if (Bound[Level].Lower[Dependence::DVEntry::GT])
+        DEBUG(dbgs() << *Bound[Level].Lower[Dependence::DVEntry::GT] << '\t');
+      else
+        DEBUG(dbgs() << "-inf\t");
+      if (Bound[Level].Upper[Dependence::DVEntry::GT])
+        DEBUG(dbgs() << *Bound[Level].Upper[Dependence::DVEntry::GT] << '\n');
+      else
+        DEBUG(dbgs() << "+inf\n");
+#endif
+    }
+
+    unsigned NewDeps = 0;
+
+    // test bounds for <, *, *, ...
+    if (testBounds(Dependence::DVEntry::LT, Level, Bound, Delta))
+      NewDeps += exploreDirections(Level + 1, A, B, Bound,
+                                   Loops, DepthExpanded, Delta);
+
+    // Test bounds for =, *, *, ...
+    if (testBounds(Dependence::DVEntry::EQ, Level, Bound, Delta))
+      NewDeps += exploreDirections(Level + 1, A, B, Bound,
+                                   Loops, DepthExpanded, Delta);
+
+    // test bounds for >, *, *, ...
+    if (testBounds(Dependence::DVEntry::GT, Level, Bound, Delta))
+      NewDeps += exploreDirections(Level + 1, A, B, Bound,
+                                   Loops, DepthExpanded, Delta);
+
+    Bound[Level].Direction = Dependence::DVEntry::ALL;
+    return NewDeps;
+  }
+  else
+    return exploreDirections(Level + 1, A, B, Bound, Loops, DepthExpanded, Delta);
+}
+
+
+// Returns true iff the current bounds are plausible.
+bool DependenceAnalysis::testBounds(unsigned char DirKind,
+                                    unsigned Level,
+                                    BoundInfo *Bound,
+                                    const SCEV *Delta) const {
+  Bound[Level].Direction = DirKind;
+  if (const SCEV *LowerBound = getLowerBound(Bound))
+    if (isKnownPredicate(CmpInst::ICMP_SGT, LowerBound, Delta))
+      return false;
+  if (const SCEV *UpperBound = getUpperBound(Bound))
+    if (isKnownPredicate(CmpInst::ICMP_SGT, Delta, UpperBound))
+      return false;
+  return true;
+}
+
+
+// Computes the upper and lower bounds for level K
+// using the * direction. Records them in Bound.
+// Wolfe gives the equations
+//
+//    LB^*_k = (A^-_k - B^+_k)(U_k - L_k) + (A_k - B_k)L_k
+//    UB^*_k = (A^+_k - B^-_k)(U_k - L_k) + (A_k - B_k)L_k
+//
+// Since we normalize loops, we can simplify these equations to
+//
+//    LB^*_k = (A^-_k - B^+_k)U_k
+//    UB^*_k = (A^+_k - B^-_k)U_k
+//
+// We must be careful to handle the case where the upper bound is unknown.
+// Note that the lower bound is always <= 0
+// and the upper bound is always >= 0.
+void DependenceAnalysis::findBoundsALL(CoefficientInfo *A,
+                                       CoefficientInfo *B,
+                                       BoundInfo *Bound,
+                                       unsigned K) const {
+  Bound[K].Lower[Dependence::DVEntry::ALL] = NULL; // Default value = -infinity.
+  Bound[K].Upper[Dependence::DVEntry::ALL] = NULL; // Default value = +infinity.
+  if (Bound[K].Iterations) {
+    Bound[K].Lower[Dependence::DVEntry::ALL] =
+      SE->getMulExpr(SE->getMinusSCEV(A[K].NegPart, B[K].PosPart),
+                     Bound[K].Iterations);
+    Bound[K].Upper[Dependence::DVEntry::ALL] =
+      SE->getMulExpr(SE->getMinusSCEV(A[K].PosPart, B[K].NegPart),
+                     Bound[K].Iterations);
+  }
+  else {
+    // If the difference is 0, we won't need to know the number of iterations.
+    if (isKnownPredicate(CmpInst::ICMP_EQ, A[K].NegPart, B[K].PosPart))
+      Bound[K].Lower[Dependence::DVEntry::ALL] =
+        SE->getConstant(A[K].Coeff->getType(), 0);
+    if (isKnownPredicate(CmpInst::ICMP_EQ, A[K].PosPart, B[K].NegPart))
+      Bound[K].Upper[Dependence::DVEntry::ALL] =
+        SE->getConstant(A[K].Coeff->getType(), 0);
+  }
+}
+
+
+// Computes the upper and lower bounds for level K
+// using the = direction. Records them in Bound.
+// Wolfe gives the equations
+//
+//    LB^=_k = (A_k - B_k)^- (U_k - L_k) + (A_k - B_k)L_k
+//    UB^=_k = (A_k - B_k)^+ (U_k - L_k) + (A_k - B_k)L_k
+//
+// Since we normalize loops, we can simplify these equations to
+//
+//    LB^=_k = (A_k - B_k)^- U_k
+//    UB^=_k = (A_k - B_k)^+ U_k
+//
+// We must be careful to handle the case where the upper bound is unknown.
+// Note that the lower bound is always <= 0
+// and the upper bound is always >= 0.
+void DependenceAnalysis::findBoundsEQ(CoefficientInfo *A,
+                                      CoefficientInfo *B,
+                                      BoundInfo *Bound,
+                                      unsigned K) const {
+  Bound[K].Lower[Dependence::DVEntry::EQ] = NULL; // Default value = -infinity.
+  Bound[K].Upper[Dependence::DVEntry::EQ] = NULL; // Default value = +infinity.
+  if (Bound[K].Iterations) {
+    const SCEV *Delta = SE->getMinusSCEV(A[K].Coeff, B[K].Coeff);
+    const SCEV *NegativePart = getNegativePart(Delta);
+    Bound[K].Lower[Dependence::DVEntry::EQ] =
+      SE->getMulExpr(NegativePart, Bound[K].Iterations);
+    const SCEV *PositivePart = getPositivePart(Delta);
+    Bound[K].Upper[Dependence::DVEntry::EQ] =
+      SE->getMulExpr(PositivePart, Bound[K].Iterations);
+  }
+  else {
+    // If the positive/negative part of the difference is 0,
+    // we won't need to know the number of iterations.
+    const SCEV *Delta = SE->getMinusSCEV(A[K].Coeff, B[K].Coeff);
+    const SCEV *NegativePart = getNegativePart(Delta);
+    if (NegativePart->isZero())
+      Bound[K].Lower[Dependence::DVEntry::EQ] = NegativePart; // Zero
+    const SCEV *PositivePart = getPositivePart(Delta);
+    if (PositivePart->isZero())
+      Bound[K].Upper[Dependence::DVEntry::EQ] = PositivePart; // Zero
+  }
+}
+
+
+// Computes the upper and lower bounds for level K
+// using the < direction. Records them in Bound.
+// Wolfe gives the equations
+//
+//    LB^<_k = (A^-_k - B_k)^- (U_k - L_k - N_k) + (A_k - B_k)L_k - B_k N_k
+//    UB^<_k = (A^+_k - B_k)^+ (U_k - L_k - N_k) + (A_k - B_k)L_k - B_k N_k
+//
+// Since we normalize loops, we can simplify these equations to
+//
+//    LB^<_k = (A^-_k - B_k)^- (U_k - 1) - B_k
+//    UB^<_k = (A^+_k - B_k)^+ (U_k - 1) - B_k
+//
+// We must be careful to handle the case where the upper bound is unknown.
+void DependenceAnalysis::findBoundsLT(CoefficientInfo *A,
+                                      CoefficientInfo *B,
+                                      BoundInfo *Bound,
+                                      unsigned K) const {
+  Bound[K].Lower[Dependence::DVEntry::LT] = NULL; // Default value = -infinity.
+  Bound[K].Upper[Dependence::DVEntry::LT] = NULL; // Default value = +infinity.
+  if (Bound[K].Iterations) {
+    const SCEV *Iter_1 =
+      SE->getMinusSCEV(Bound[K].Iterations,
+                       SE->getConstant(Bound[K].Iterations->getType(), 1));
+    const SCEV *NegPart =
+      getNegativePart(SE->getMinusSCEV(A[K].NegPart, B[K].Coeff));
+    Bound[K].Lower[Dependence::DVEntry::LT] =
+      SE->getMinusSCEV(SE->getMulExpr(NegPart, Iter_1), B[K].Coeff);
+    const SCEV *PosPart =
+      getPositivePart(SE->getMinusSCEV(A[K].PosPart, B[K].Coeff));
+    Bound[K].Upper[Dependence::DVEntry::LT] =
+      SE->getMinusSCEV(SE->getMulExpr(PosPart, Iter_1), B[K].Coeff);
+  }
+  else {
+    // If the positive/negative part of the difference is 0,
+    // we won't need to know the number of iterations.
+    const SCEV *NegPart =
+      getNegativePart(SE->getMinusSCEV(A[K].NegPart, B[K].Coeff));
+    if (NegPart->isZero())
+      Bound[K].Lower[Dependence::DVEntry::LT] = SE->getNegativeSCEV(B[K].Coeff);
+    const SCEV *PosPart =
+      getPositivePart(SE->getMinusSCEV(A[K].PosPart, B[K].Coeff));
+    if (PosPart->isZero())
+      Bound[K].Upper[Dependence::DVEntry::LT] = SE->getNegativeSCEV(B[K].Coeff);
+  }
+}
+
+
+// Computes the upper and lower bounds for level K
+// using the > direction. Records them in Bound.
+// Wolfe gives the equations
+//
+//    LB^>_k = (A_k - B^+_k)^- (U_k - L_k - N_k) + (A_k - B_k)L_k + A_k N_k
+//    UB^>_k = (A_k - B^-_k)^+ (U_k - L_k - N_k) + (A_k - B_k)L_k + A_k N_k
+//
+// Since we normalize loops, we can simplify these equations to
+//
+//    LB^>_k = (A_k - B^+_k)^- (U_k - 1) + A_k
+//    UB^>_k = (A_k - B^-_k)^+ (U_k - 1) + A_k
+//
+// We must be careful to handle the case where the upper bound is unknown.
+void DependenceAnalysis::findBoundsGT(CoefficientInfo *A,
+                                      CoefficientInfo *B,
+                                      BoundInfo *Bound,
+                                      unsigned K) const {
+  Bound[K].Lower[Dependence::DVEntry::GT] = NULL; // Default value = -infinity.
+  Bound[K].Upper[Dependence::DVEntry::GT] = NULL; // Default value = +infinity.
+  if (Bound[K].Iterations) {
+    const SCEV *Iter_1 =
+      SE->getMinusSCEV(Bound[K].Iterations,
+                       SE->getConstant(Bound[K].Iterations->getType(), 1));
+    const SCEV *NegPart =
+      getNegativePart(SE->getMinusSCEV(A[K].Coeff, B[K].PosPart));
+    Bound[K].Lower[Dependence::DVEntry::GT] =
+      SE->getAddExpr(SE->getMulExpr(NegPart, Iter_1), A[K].Coeff);
+    const SCEV *PosPart =
+      getPositivePart(SE->getMinusSCEV(A[K].Coeff, B[K].NegPart));
+    Bound[K].Upper[Dependence::DVEntry::GT] =
+      SE->getAddExpr(SE->getMulExpr(PosPart, Iter_1), A[K].Coeff);
+  }
+  else {
+    // If the positive/negative part of the difference is 0,
+    // we won't need to know the number of iterations.
+    const SCEV *NegPart = getNegativePart(SE->getMinusSCEV(A[K].Coeff, B[K].PosPart));
+    if (NegPart->isZero())
+      Bound[K].Lower[Dependence::DVEntry::GT] = A[K].Coeff;
+    const SCEV *PosPart = getPositivePart(SE->getMinusSCEV(A[K].Coeff, B[K].NegPart));
+    if (PosPart->isZero())
+      Bound[K].Upper[Dependence::DVEntry::GT] = A[K].Coeff;
+  }
+}
+
+
+// X^+ = max(X, 0)
+const SCEV *DependenceAnalysis::getPositivePart(const SCEV *X) const {
+  return SE->getSMaxExpr(X, SE->getConstant(X->getType(), 0));
+}
+
+
+// X^- = min(X, 0)
+const SCEV *DependenceAnalysis::getNegativePart(const SCEV *X) const {
+  return SE->getSMinExpr(X, SE->getConstant(X->getType(), 0));
+}
+
+
+// Walks through the subscript,
+// collecting each coefficient, the associated loop bounds,
+// and recording its positive and negative parts for later use.
+DependenceAnalysis::CoefficientInfo *
+DependenceAnalysis::collectCoeffInfo(const SCEV *Subscript,
+                                     bool SrcFlag,
+                                     const SCEV *&Constant) const {
+  const SCEV *Zero = SE->getConstant(Subscript->getType(), 0);
+  CoefficientInfo *CI = new CoefficientInfo[MaxLevels + 1];
+  for (unsigned K = 1; K <= MaxLevels; ++K) {
+    CI[K].Coeff = Zero;
+    CI[K].PosPart = Zero;
+    CI[K].NegPart = Zero;
+    CI[K].Iterations = NULL;
+  }
+  while (const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Subscript)) {
+    const Loop *L = AddRec->getLoop();
+    unsigned K = SrcFlag ? mapSrcLoop(L) : mapDstLoop(L);
+    CI[K].Coeff = AddRec->getStepRecurrence(*SE);
+    CI[K].PosPart = getPositivePart(CI[K].Coeff);
+    CI[K].NegPart = getNegativePart(CI[K].Coeff);
+    CI[K].Iterations = collectUpperBound(L, Subscript->getType());
+    Subscript = AddRec->getStart();
+  }
+  Constant = Subscript;
+#ifndef NDEBUG
+  DEBUG(dbgs() << "\tCoefficient Info\n");
+  for (unsigned K = 1; K <= MaxLevels; ++K) {
+    DEBUG(dbgs() << "\t    " << K << "\t" << *CI[K].Coeff);
+    DEBUG(dbgs() << "\tPos Part = ");
+    DEBUG(dbgs() << *CI[K].PosPart);
+    DEBUG(dbgs() << "\tNeg Part = ");
+    DEBUG(dbgs() << *CI[K].NegPart);
+    DEBUG(dbgs() << "\tUpper Bound = ");
+    if (CI[K].Iterations)
+      DEBUG(dbgs() << *CI[K].Iterations);
+    else
+      DEBUG(dbgs() << "+inf");
+    DEBUG(dbgs() << '\n');
+  }
+  DEBUG(dbgs() << "\t    Constant = " << *Subscript << '\n');
+#endif
+  return CI;
+}
+
+
+// Looks through all the bounds info and
+// computes the lower bound given the current direction settings
+// at each level. If the lower bound for any level is -inf,
+// the result is -inf.
+const SCEV *DependenceAnalysis::getLowerBound(BoundInfo *Bound) const {
+  const SCEV *Sum = Bound[1].Lower[Bound[1].Direction];
+  for (unsigned K = 2; Sum && K <= MaxLevels; ++K) {
+    if (Bound[K].Lower[Bound[K].Direction])
+      Sum = SE->getAddExpr(Sum, Bound[K].Lower[Bound[K].Direction]);
+    else
+      Sum = NULL;
+  }
+  return Sum;
+}
+
+
+// Looks through all the bounds info and
+// computes the upper bound given the current direction settings
+// at each level. If the upper bound at any level is +inf,
+// the result is +inf.
+const SCEV *DependenceAnalysis::getUpperBound(BoundInfo *Bound) const {
+  const SCEV *Sum = Bound[1].Upper[Bound[1].Direction];
+  for (unsigned K = 2; Sum && K <= MaxLevels; ++K) {
+    if (Bound[K].Upper[Bound[K].Direction])
+      Sum = SE->getAddExpr(Sum, Bound[K].Upper[Bound[K].Direction]);
+    else
+      Sum = NULL;
+  }
+  return Sum;
+}
+
+
+//===----------------------------------------------------------------------===//
+// Constraint manipulation for Delta test.
+
+// Given a linear SCEV,
+// return the coefficient (the step)
+// corresponding to the specified loop.
+// If there isn't one, return 0.
+// For example, given a*i + b*j + c*k, zeroing the coefficient
+// corresponding to the j loop would yield b.
+const SCEV *DependenceAnalysis::findCoefficient(const SCEV *Expr,
+                                                const Loop *TargetLoop)  const {
+  const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Expr);
+  if (!AddRec)
+    return SE->getConstant(Expr->getType(), 0);
+  if (AddRec->getLoop() == TargetLoop)
+    return AddRec->getStepRecurrence(*SE);
+  return findCoefficient(AddRec->getStart(), TargetLoop);
+}
+
+
+// Given a linear SCEV,
+// return the SCEV given by zeroing out the coefficient
+// corresponding to the specified loop.
+// For example, given a*i + b*j + c*k, zeroing the coefficient
+// corresponding to the j loop would yield a*i + c*k.
+const SCEV *DependenceAnalysis::zeroCoefficient(const SCEV *Expr,
+                                                const Loop *TargetLoop)  const {
+  const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Expr);
+  if (!AddRec)
+    return Expr; // ignore
+  if (AddRec->getLoop() == TargetLoop)
+    return AddRec->getStart();
+  return SE->getAddRecExpr(zeroCoefficient(AddRec->getStart(), TargetLoop),
+                           AddRec->getStepRecurrence(*SE),
+                           AddRec->getLoop(),
+                           AddRec->getNoWrapFlags());
+}
+
+
+// Given a linear SCEV Expr,
+// return the SCEV given by adding some Value to the
+// coefficient corresponding to the specified TargetLoop.
+// For example, given a*i + b*j + c*k, adding 1 to the coefficient
+// corresponding to the j loop would yield a*i + (b+1)*j + c*k.
+const SCEV *DependenceAnalysis::addToCoefficient(const SCEV *Expr,
+                                                 const Loop *TargetLoop,
+                                                 const SCEV *Value)  const {
+  const SCEVAddRecExpr *AddRec = dyn_cast<SCEVAddRecExpr>(Expr);
+  if (!AddRec) // create a new addRec
+    return SE->getAddRecExpr(Expr,
+                             Value,
+                             TargetLoop,
+                             SCEV::FlagAnyWrap); // Worst case, with no info.
+  if (AddRec->getLoop() == TargetLoop) {
+    const SCEV *Sum = SE->getAddExpr(AddRec->getStepRecurrence(*SE), Value);
+    if (Sum->isZero())
+      return AddRec->getStart();
+    return SE->getAddRecExpr(AddRec->getStart(),
+                             Sum,
+                             AddRec->getLoop(),
+                             AddRec->getNoWrapFlags());
+  }
+  return SE->getAddRecExpr(addToCoefficient(AddRec->getStart(),
+                                            TargetLoop, Value),
+                           AddRec->getStepRecurrence(*SE),
+                           AddRec->getLoop(),
+                           AddRec->getNoWrapFlags());
+}
+
+
+// Review the constraints, looking for opportunities
+// to simplify a subscript pair (Src and Dst).
+// Return true if some simplification occurs.
+// If the simplification isn't exact (that is, if it is conservative
+// in terms of dependence), set consistent to false.
+// Corresponds to Figure 5 from the paper
+//
+//            Practical Dependence Testing
+//            Goff, Kennedy, Tseng
+//            PLDI 1991
+bool DependenceAnalysis::propagate(const SCEV *&Src,
+                                   const SCEV *&Dst,
+                                   SmallBitVector &Loops,
+                                   SmallVector<Constraint, 4> &Constraints,
+                                   bool &Consistent) {
+  bool Result = false;
+  for (int LI = Loops.find_first(); LI >= 0; LI = Loops.find_next(LI)) {
+    DEBUG(dbgs() << "\t    Constraint[" << LI << "] is");
+    DEBUG(Constraints[LI].dump(dbgs()));
+    if (Constraints[LI].isDistance())
+      Result |= propagateDistance(Src, Dst, Constraints[LI], Consistent);
+    else if (Constraints[LI].isLine())
+      Result |= propagateLine(Src, Dst, Constraints[LI], Consistent);
+    else if (Constraints[LI].isPoint())
+      Result |= propagatePoint(Src, Dst, Constraints[LI]);
+  }
+  return Result;
+}
+
+
+// Attempt to propagate a distance
+// constraint into a subscript pair (Src and Dst).
+// Return true if some simplification occurs.
+// If the simplification isn't exact (that is, if it is conservative
+// in terms of dependence), set consistent to false.
+bool DependenceAnalysis::propagateDistance(const SCEV *&Src,
+                                           const SCEV *&Dst,
+                                           Constraint &CurConstraint,
+                                           bool &Consistent) {
+  const Loop *CurLoop = CurConstraint.getAssociatedLoop();
+  DEBUG(dbgs() << "\t\tSrc is " << *Src << "\n");
+  const SCEV *A_K = findCoefficient(Src, CurLoop);
+  if (A_K->isZero())
+    return false;
+  const SCEV *DA_K = SE->getMulExpr(A_K, CurConstraint.getD());
+  Src = SE->getMinusSCEV(Src, DA_K);
+  Src = zeroCoefficient(Src, CurLoop);
+  DEBUG(dbgs() << "\t\tnew Src is " << *Src << "\n");
+  DEBUG(dbgs() << "\t\tDst is " << *Dst << "\n");
+  Dst = addToCoefficient(Dst, CurLoop, SE->getNegativeSCEV(A_K));
+  DEBUG(dbgs() << "\t\tnew Dst is " << *Dst << "\n");
+  if (!findCoefficient(Dst, CurLoop)->isZero())
+    Consistent = false;
+  return true;
+}
+
+
+// Attempt to propagate a line
+// constraint into a subscript pair (Src and Dst).
+// Return true if some simplification occurs.
+// If the simplification isn't exact (that is, if it is conservative
+// in terms of dependence), set consistent to false.
+bool DependenceAnalysis::propagateLine(const SCEV *&Src,
+                                       const SCEV *&Dst,
+                                       Constraint &CurConstraint,
+                                       bool &Consistent) {
+  const Loop *CurLoop = CurConstraint.getAssociatedLoop();
+  const SCEV *A = CurConstraint.getA();
+  const SCEV *B = CurConstraint.getB();
+  const SCEV *C = CurConstraint.getC();
+  DEBUG(dbgs() << "\t\tA = " << *A << ", B = " << *B << ", C = " << *C << "\n");
+  DEBUG(dbgs() << "\t\tSrc = " << *Src << "\n");
+  DEBUG(dbgs() << "\t\tDst = " << *Dst << "\n");
+  if (A->isZero()) {
+    const SCEVConstant *Bconst = dyn_cast<SCEVConstant>(B);
+    const SCEVConstant *Cconst = dyn_cast<SCEVConstant>(C);
+    if (!Bconst || !Cconst) return false;
+    APInt Beta = Bconst->getValue()->getValue();
+    APInt Charlie = Cconst->getValue()->getValue();
+    APInt CdivB = Charlie.sdiv(Beta);
+    assert(Charlie.srem(Beta) == 0 && "C should be evenly divisible by B");
+    const SCEV *AP_K = findCoefficient(Dst, CurLoop);
+    //    Src = SE->getAddExpr(Src, SE->getMulExpr(AP_K, SE->getConstant(CdivB)));
+    Src = SE->getMinusSCEV(Src, SE->getMulExpr(AP_K, SE->getConstant(CdivB)));
+    Dst = zeroCoefficient(Dst, CurLoop);
+    if (!findCoefficient(Src, CurLoop)->isZero())
+      Consistent = false;
+  }
+  else if (B->isZero()) {
+    const SCEVConstant *Aconst = dyn_cast<SCEVConstant>(A);
+    const SCEVConstant *Cconst = dyn_cast<SCEVConstant>(C);
+    if (!Aconst || !Cconst) return false;
+    APInt Alpha = Aconst->getValue()->getValue();
+    APInt Charlie = Cconst->getValue()->getValue();
+    APInt CdivA = Charlie.sdiv(Alpha);
+    assert(Charlie.srem(Alpha) == 0 && "C should be evenly divisible by A");
+    const SCEV *A_K = findCoefficient(Src, CurLoop);
+    Src = SE->getAddExpr(Src, SE->getMulExpr(A_K, SE->getConstant(CdivA)));
+    Src = zeroCoefficient(Src, CurLoop);
+    if (!findCoefficient(Dst, CurLoop)->isZero())
+      Consistent = false;
+  }
+  else if (isKnownPredicate(CmpInst::ICMP_EQ, A, B)) {
+    const SCEVConstant *Aconst = dyn_cast<SCEVConstant>(A);
+    const SCEVConstant *Cconst = dyn_cast<SCEVConstant>(C);
+    if (!Aconst || !Cconst) return false;
+    APInt Alpha = Aconst->getValue()->getValue();
+    APInt Charlie = Cconst->getValue()->getValue();
+    APInt CdivA = Charlie.sdiv(Alpha);
+    assert(Charlie.srem(Alpha) == 0 && "C should be evenly divisible by A");
+    const SCEV *A_K = findCoefficient(Src, CurLoop);
+    Src = SE->getAddExpr(Src, SE->getMulExpr(A_K, SE->getConstant(CdivA)));
+    Src = zeroCoefficient(Src, CurLoop);
+    Dst = addToCoefficient(Dst, CurLoop, A_K);
+    if (!findCoefficient(Dst, CurLoop)->isZero())
+      Consistent = false;
+  }
+  else {
+    // paper is incorrect here, or perhaps just misleading
+    const SCEV *A_K = findCoefficient(Src, CurLoop);
+    Src = SE->getMulExpr(Src, A);
+    Dst = SE->getMulExpr(Dst, A);
+    Src = SE->getAddExpr(Src, SE->getMulExpr(A_K, C));
+    Src = zeroCoefficient(Src, CurLoop);
+    Dst = addToCoefficient(Dst, CurLoop, SE->getMulExpr(A_K, B));
+    if (!findCoefficient(Dst, CurLoop)->isZero())
+      Consistent = false;
+  }
+  DEBUG(dbgs() << "\t\tnew Src = " << *Src << "\n");
+  DEBUG(dbgs() << "\t\tnew Dst = " << *Dst << "\n");
+  return true;
+}
+
+
+// Attempt to propagate a point
+// constraint into a subscript pair (Src and Dst).
+// Return true if some simplification occurs.
+bool DependenceAnalysis::propagatePoint(const SCEV *&Src,
+                                        const SCEV *&Dst,
+                                        Constraint &CurConstraint) {
+  const Loop *CurLoop = CurConstraint.getAssociatedLoop();
+  const SCEV *A_K = findCoefficient(Src, CurLoop);
+  const SCEV *AP_K = findCoefficient(Dst, CurLoop);
+  const SCEV *XA_K = SE->getMulExpr(A_K, CurConstraint.getX());
+  const SCEV *YAP_K = SE->getMulExpr(AP_K, CurConstraint.getY());
+  DEBUG(dbgs() << "\t\tSrc is " << *Src << "\n");
+  Src = SE->getAddExpr(Src, SE->getMinusSCEV(XA_K, YAP_K));
+  Src = zeroCoefficient(Src, CurLoop);
+  DEBUG(dbgs() << "\t\tnew Src is " << *Src << "\n");
+  DEBUG(dbgs() << "\t\tDst is " << *Dst << "\n");
+  Dst = zeroCoefficient(Dst, CurLoop);
+  DEBUG(dbgs() << "\t\tnew Dst is " << *Dst << "\n");
+  return true;
+}
+
+
+// Update direction vector entry based on the current constraint.
+void DependenceAnalysis::updateDirection(Dependence::DVEntry &Level,
+                                         const Constraint &CurConstraint
+                                         ) const {
+  DEBUG(dbgs() << "\tUpdate direction, constraint =");
+  DEBUG(CurConstraint.dump(dbgs()));
+  if (CurConstraint.isAny())
+    ; // use defaults
+  else if (CurConstraint.isDistance()) {
+    // this one is consistent, the others aren't
+    Level.Scalar = false;
+    Level.Distance = CurConstraint.getD();
+    unsigned NewDirection = Dependence::DVEntry::NONE;
+    if (!SE->isKnownNonZero(Level.Distance)) // if may be zero
+      NewDirection = Dependence::DVEntry::EQ;
+    if (!SE->isKnownNonPositive(Level.Distance)) // if may be positive
+      NewDirection |= Dependence::DVEntry::LT;
+    if (!SE->isKnownNonNegative(Level.Distance)) // if may be negative
+      NewDirection |= Dependence::DVEntry::GT;
+    Level.Direction &= NewDirection;
+  }
+  else if (CurConstraint.isLine()) {
+    Level.Scalar = false;
+    Level.Distance = NULL;
+    // direction should be accurate
+  }
+  else if (CurConstraint.isPoint()) {
+    Level.Scalar = false;
+    Level.Distance = NULL;
+    unsigned NewDirection = Dependence::DVEntry::NONE;
+    if (!isKnownPredicate(CmpInst::ICMP_NE,
+                          CurConstraint.getY(),
+                          CurConstraint.getX()))
+      // if X may be = Y
+      NewDirection |= Dependence::DVEntry::EQ;
+    if (!isKnownPredicate(CmpInst::ICMP_SLE,
+                          CurConstraint.getY(),
+                          CurConstraint.getX()))
+      // if Y may be > X
+      NewDirection |= Dependence::DVEntry::LT;
+    if (!isKnownPredicate(CmpInst::ICMP_SGE,
+                          CurConstraint.getY(),
+                          CurConstraint.getX()))
+      // if Y may be < X
+      NewDirection |= Dependence::DVEntry::GT;
+    Level.Direction &= NewDirection;
+  }
+  else
+    llvm_unreachable("constraint has unexpected kind");
+}
+
+
+//===----------------------------------------------------------------------===//
+
+#ifndef NDEBUG
+// For debugging purposes, dump a small bit vector to dbgs().
+static void dumpSmallBitVector(SmallBitVector &BV) {
+  dbgs() << "{";
+  for (int VI = BV.find_first(); VI >= 0; VI = BV.find_next(VI)) {
+    dbgs() << VI;
+    if (BV.find_next(VI) >= 0)
+      dbgs() << ' ';
+  }
+  dbgs() << "}\n";
+}
+#endif
+
+
+// depends -
+// Returns NULL if there is no dependence.
+// Otherwise, return a Dependence with as many details as possible.
+// Corresponds to Section 3.1 in the paper
+//
+//            Practical Dependence Testing
+//            Goff, Kennedy, Tseng
+//            PLDI 1991
+//
+// Care is required to keep the code below up to date w.r.t. this routine.
+Dependence *DependenceAnalysis::depends(const Instruction *Src,
+                                        const Instruction *Dst,
+                                        bool PossiblyLoopIndependent) {
+  if ((!Src->mayReadFromMemory() && !Src->mayWriteToMemory()) ||
+      (!Dst->mayReadFromMemory() && !Dst->mayWriteToMemory()))
+    // if both instructions don't reference memory, there's no dependence
+    return NULL;
+
+  if (!isLoadOrStore(Src) || !isLoadOrStore(Dst))
+    // can only analyze simple loads and stores, i.e., no calls, invokes, etc.
+    return new Dependence(Src, Dst);
+
+  const Value *SrcPtr = getPointerOperand(Src);
+  const Value *DstPtr = getPointerOperand(Dst);
+
+  switch (underlyingObjectsAlias(AA, DstPtr, SrcPtr)) {
+  case AliasAnalysis::MayAlias:
+  case AliasAnalysis::PartialAlias:
+    // cannot analyse objects if we don't understand their aliasing.
+    return new Dependence(Src, Dst);
+  case AliasAnalysis::NoAlias:
+    // If the objects noalias, they are distinct, accesses are independent.
+    return NULL;
+  case AliasAnalysis::MustAlias:
+    break; // The underlying objects alias; test accesses for dependence.
+  }
+
+  const GEPOperator *SrcGEP = dyn_cast<GEPOperator>(SrcPtr);
+  const GEPOperator *DstGEP = dyn_cast<GEPOperator>(DstPtr);
+  if (!SrcGEP || !DstGEP)
+    return new Dependence(Src, Dst); // missing GEP, assume dependence
+
+  if (SrcGEP->getPointerOperandType() != DstGEP->getPointerOperandType())
+    return new Dependence(Src, Dst); // different types, assume dependence
+
+  // establish loop nesting levels
+  establishNestingLevels(Src, Dst);
+  DEBUG(dbgs() << "    common nesting levels = " << CommonLevels << "\n");
+  DEBUG(dbgs() << "    maximum nesting levels = " << MaxLevels << "\n");
+
+  FullDependence Result(Src, Dst, PossiblyLoopIndependent, CommonLevels);
+  ++TotalArrayPairs;
+
+  // classify subscript pairs
+  unsigned Pairs = SrcGEP->idx_end() - SrcGEP->idx_begin();
+  SmallVector<Subscript, 4> Pair(Pairs);
+  for (unsigned SI = 0; SI < Pairs; ++SI) {
+    Pair[SI].Loops.resize(MaxLevels + 1);
+    Pair[SI].GroupLoops.resize(MaxLevels + 1);
+    Pair[SI].Group.resize(Pairs);
+  }
+  Pairs = 0;
+  for (GEPOperator::const_op_iterator SrcIdx = SrcGEP->idx_begin(),
+         SrcEnd = SrcGEP->idx_end(),
+         DstIdx = DstGEP->idx_begin(),
+         DstEnd = DstGEP->idx_end();
+       SrcIdx != SrcEnd && DstIdx != DstEnd;
+       ++SrcIdx, ++DstIdx, ++Pairs) {
+    Pair[Pairs].Src = SE->getSCEV(*SrcIdx);
+    Pair[Pairs].Dst = SE->getSCEV(*DstIdx);
+    removeMatchingExtensions(&Pair[Pairs]);
+    Pair[Pairs].Classification =
+      classifyPair(Pair[Pairs].Src, LI->getLoopFor(Src->getParent()),
+                   Pair[Pairs].Dst, LI->getLoopFor(Dst->getParent()),
+                   Pair[Pairs].Loops);
+    Pair[Pairs].GroupLoops = Pair[Pairs].Loops;
+    Pair[Pairs].Group.set(Pairs);
+    DEBUG(dbgs() << "    subscript " << Pairs << "\n");
+    DEBUG(dbgs() << "\tsrc = " << *Pair[Pairs].Src << "\n");
+    DEBUG(dbgs() << "\tdst = " << *Pair[Pairs].Dst << "\n");
+    DEBUG(dbgs() << "\tclass = " << Pair[Pairs].Classification << "\n");
+    DEBUG(dbgs() << "\tloops = ");
+    DEBUG(dumpSmallBitVector(Pair[Pairs].Loops));
+  }
+
+  SmallBitVector Separable(Pairs);
+  SmallBitVector Coupled(Pairs);
+
+  // Partition subscripts into separable and minimally-coupled groups
+  // Algorithm in paper is algorithmically better;
+  // this may be faster in practice. Check someday.
+  //
+  // Here's an example of how it works. Consider this code:
+  //
+  //   for (i = ...) {
+  //     for (j = ...) {
+  //       for (k = ...) {
+  //         for (l = ...) {
+  //           for (m = ...) {
+  //             A[i][j][k][m] = ...;
+  //             ... = A[0][j][l][i + j];
+  //           }
+  //         }
+  //       }
+  //     }
+  //   }
+  //
+  // There are 4 subscripts here:
+  //    0 [i] and [0]
+  //    1 [j] and [j]
+  //    2 [k] and [l]
+  //    3 [m] and [i + j]
+  //
+  // We've already classified each subscript pair as ZIV, SIV, etc.,
+  // and collected all the loops mentioned by pair P in Pair[P].Loops.
+  // In addition, we've initialized Pair[P].GroupLoops to Pair[P].Loops
+  // and set Pair[P].Group = {P}.
+  //
+  //      Src Dst    Classification Loops  GroupLoops Group
+  //    0 [i] [0]         SIV       {1}      {1}        {0}
+  //    1 [j] [j]         SIV       {2}      {2}        {1}
+  //    2 [k] [l]         RDIV      {3,4}    {3,4}      {2}
+  //    3 [m] [i + j]     MIV       {1,2,5}  {1,2,5}    {3}
+  //
+  // For each subscript SI 0 .. 3, we consider each remaining subscript, SJ.
+  // So, 0 is compared against 1, 2, and 3; 1 is compared against 2 and 3, etc.
+  //
+  // We begin by comparing 0 and 1. The intersection of the GroupLoops is empty.
+  // Next, 0 and 2. Again, the intersection of their GroupLoops is empty.
+  // Next 0 and 3. The intersection of their GroupLoop = {1}, not empty,
+  // so Pair[3].Group = {0,3} and Done = false (that is, 0 will not be added
+  // to either Separable or Coupled).
+  //
+  // Next, we consider 1 and 2. The intersection of the GroupLoops is empty.
+  // Next, 1 and 3. The intersectionof their GroupLoops = {2}, not empty,
+  // so Pair[3].Group = {0, 1, 3} and Done = false.
+  //
+  // Next, we compare 2 against 3. The intersection of the GroupLoops is empty.
+  // Since Done remains true, we add 2 to the set of Separable pairs.
+  //
+  // Finally, we consider 3. There's nothing to compare it with,
+  // so Done remains true and we add it to the Coupled set.
+  // Pair[3].Group = {0, 1, 3} and GroupLoops = {1, 2, 5}.
+  //
+  // In the end, we've got 1 separable subscript and 1 coupled group.
+  for (unsigned SI = 0; SI < Pairs; ++SI) {
+    if (Pair[SI].Classification == Subscript::NonLinear) {
+      // ignore these, but collect loops for later
+      ++NonlinearSubscriptPairs;
+      collectCommonLoops(Pair[SI].Src,
+                         LI->getLoopFor(Src->getParent()),
+                         Pair[SI].Loops);
+      collectCommonLoops(Pair[SI].Dst,
+                         LI->getLoopFor(Dst->getParent()),
+                         Pair[SI].Loops);
+      Result.Consistent = false;
+    }
+    else if (Pair[SI].Classification == Subscript::ZIV) {
+      // always separable
+      Separable.set(SI);
+    }
+    else {
+      // SIV, RDIV, or MIV, so check for coupled group
+      bool Done = true;
+      for (unsigned SJ = SI + 1; SJ < Pairs; ++SJ) {
+        SmallBitVector Intersection = Pair[SI].GroupLoops;
+        Intersection &= Pair[SJ].GroupLoops;
+        if (Intersection.any()) {
+          // accumulate set of all the loops in group
+          Pair[SJ].GroupLoops |= Pair[SI].GroupLoops;
+          // accumulate set of all subscripts in group
+          Pair[SJ].Group |= Pair[SI].Group;
+          Done = false;
+        }
+      }
+      if (Done) {
+        if (Pair[SI].Group.count() == 1) {
+          Separable.set(SI);
+          ++SeparableSubscriptPairs;
+        }
+        else {
+          Coupled.set(SI);
+          ++CoupledSubscriptPairs;
+        }
+      }
+    }
+  }
+
+  DEBUG(dbgs() << "    Separable = ");
+  DEBUG(dumpSmallBitVector(Separable));
+  DEBUG(dbgs() << "    Coupled = ");
+  DEBUG(dumpSmallBitVector(Coupled));
+
+  Constraint NewConstraint;
+  NewConstraint.setAny(SE);
+
+  // test separable subscripts
+  for (int SI = Separable.find_first(); SI >= 0; SI = Separable.find_next(SI)) {
+    DEBUG(dbgs() << "testing subscript " << SI);
+    switch (Pair[SI].Classification) {
+    case Subscript::ZIV:
+      DEBUG(dbgs() << ", ZIV\n");
+      if (testZIV(Pair[SI].Src, Pair[SI].Dst, Result))
+        return NULL;
+      break;
+    case Subscript::SIV: {
+      DEBUG(dbgs() << ", SIV\n");
+      unsigned Level;
+      const SCEV *SplitIter = NULL;
+      if (testSIV(Pair[SI].Src, Pair[SI].Dst, Level,
+                  Result, NewConstraint, SplitIter))
+        return NULL;
+      break;
+    }
+    case Subscript::RDIV:
+      DEBUG(dbgs() << ", RDIV\n");
+      if (testRDIV(Pair[SI].Src, Pair[SI].Dst, Result))
+        return NULL;
+      break;
+    case Subscript::MIV:
+      DEBUG(dbgs() << ", MIV\n");
+      if (testMIV(Pair[SI].Src, Pair[SI].Dst, Pair[SI].Loops, Result))
+        return NULL;
+      break;
+    default:
+      llvm_unreachable("subscript has unexpected classification");
+    }
+  }
+
+  if (Coupled.count()) {
+    // test coupled subscript groups
+    DEBUG(dbgs() << "starting on coupled subscripts\n");
+    DEBUG(dbgs() << "MaxLevels + 1 = " << MaxLevels + 1 << "\n");
+    SmallVector<Constraint, 4> Constraints(MaxLevels + 1);
+    for (unsigned II = 0; II <= MaxLevels; ++II)
+      Constraints[II].setAny(SE);
+    for (int SI = Coupled.find_first(); SI >= 0; SI = Coupled.find_next(SI)) {
+      DEBUG(dbgs() << "testing subscript group " << SI << " { ");
+      SmallBitVector Group(Pair[SI].Group);
+      SmallBitVector Sivs(Pairs);
+      SmallBitVector Mivs(Pairs);
+      SmallBitVector ConstrainedLevels(MaxLevels + 1);
+      for (int SJ = Group.find_first(); SJ >= 0; SJ = Group.find_next(SJ)) {
+        DEBUG(dbgs() << SJ << " ");
+        if (Pair[SJ].Classification == Subscript::SIV)
+          Sivs.set(SJ);
+        else
+          Mivs.set(SJ);
+      }
+      DEBUG(dbgs() << "}\n");
+      while (Sivs.any()) {
+        bool Changed = false;
+        for (int SJ = Sivs.find_first(); SJ >= 0; SJ = Sivs.find_next(SJ)) {
+          DEBUG(dbgs() << "testing subscript " << SJ << ", SIV\n");
+          // SJ is an SIV subscript that's part of the current coupled group
+          unsigned Level;
+          const SCEV *SplitIter = NULL;
+          DEBUG(dbgs() << "SIV\n");
+          if (testSIV(Pair[SJ].Src, Pair[SJ].Dst, Level,
+                      Result, NewConstraint, SplitIter))
+            return NULL;
+          ConstrainedLevels.set(Level);
+          if (intersectConstraints(&Constraints[Level], &NewConstraint)) {
+            if (Constraints[Level].isEmpty()) {
+              ++DeltaIndependence;
+              return NULL;
+            }
+            Changed = true;
+          }
+          Sivs.reset(SJ);
+        }
+        if (Changed) {
+          // propagate, possibly creating new SIVs and ZIVs
+          DEBUG(dbgs() << "    propagating\n");
+          DEBUG(dbgs() << "\tMivs = ");
+          DEBUG(dumpSmallBitVector(Mivs));
+          for (int SJ = Mivs.find_first(); SJ >= 0; SJ = Mivs.find_next(SJ)) {
+            // SJ is an MIV subscript that's part of the current coupled group
+            DEBUG(dbgs() << "\tSJ = " << SJ << "\n");
+            if (propagate(Pair[SJ].Src, Pair[SJ].Dst, Pair[SJ].Loops,
+                          Constraints, Result.Consistent)) {
+              DEBUG(dbgs() << "\t    Changed\n");
+              ++DeltaPropagations;
+              Pair[SJ].Classification =
+                classifyPair(Pair[SJ].Src, LI->getLoopFor(Src->getParent()),
+                             Pair[SJ].Dst, LI->getLoopFor(Dst->getParent()),
+                             Pair[SJ].Loops);
+              switch (Pair[SJ].Classification) {
+              case Subscript::ZIV:
+                DEBUG(dbgs() << "ZIV\n");
+                if (testZIV(Pair[SJ].Src, Pair[SJ].Dst, Result))
+                  return NULL;
+                Mivs.reset(SJ);
+                break;
+              case Subscript::SIV:
+                Sivs.set(SJ);
+                Mivs.reset(SJ);
+                break;
+              case Subscript::RDIV:
+              case Subscript::MIV:
+                break;
+              default:
+                llvm_unreachable("bad subscript classification");
+              }
+            }
+          }
+        }
+      }
+
+      // test & propagate remaining RDIVs
+      for (int SJ = Mivs.find_first(); SJ >= 0; SJ = Mivs.find_next(SJ)) {
+        if (Pair[SJ].Classification == Subscript::RDIV) {
+          DEBUG(dbgs() << "RDIV test\n");
+          if (testRDIV(Pair[SJ].Src, Pair[SJ].Dst, Result))
+            return NULL;
+          // I don't yet understand how to propagate RDIV results
+          Mivs.reset(SJ);
+        }
+      }
+
+      // test remaining MIVs
+      // This code is temporary.
+      // Better to somehow test all remaining subscripts simultaneously.
+      for (int SJ = Mivs.find_first(); SJ >= 0; SJ = Mivs.find_next(SJ)) {
+        if (Pair[SJ].Classification == Subscript::MIV) {
+          DEBUG(dbgs() << "MIV test\n");
+          if (testMIV(Pair[SJ].Src, Pair[SJ].Dst, Pair[SJ].Loops, Result))
+            return NULL;
+        }
+        else
+          llvm_unreachable("expected only MIV subscripts at this point");
+      }
+
+      // update Result.DV from constraint vector
+      DEBUG(dbgs() << "    updating\n");
+      for (int SJ = ConstrainedLevels.find_first();
+           SJ >= 0; SJ = ConstrainedLevels.find_next(SJ)) {
+        updateDirection(Result.DV[SJ - 1], Constraints[SJ]);
+        if (Result.DV[SJ - 1].Direction == Dependence::DVEntry::NONE)
+          return NULL;
+      }
+    }
+  }
+
+  // make sure Scalar flags are set correctly
+  SmallBitVector CompleteLoops(MaxLevels + 1);
+  for (unsigned SI = 0; SI < Pairs; ++SI)
+    CompleteLoops |= Pair[SI].Loops;
+  for (unsigned II = 1; II <= CommonLevels; ++II)
+    if (CompleteLoops[II])
+      Result.DV[II - 1].Scalar = false;
+
+  // make sure loopIndepent flag is set correctly
+  if (PossiblyLoopIndependent) {
+    for (unsigned II = 1; II <= CommonLevels; ++II) {
+      if (!(Result.getDirection(II) & Dependence::DVEntry::EQ)) {
+        Result.LoopIndependent = false;
+        break;
+      }
+    }
+  }
+
+  FullDependence *Final = new FullDependence(Result);
+  Result.DV = NULL;
+  return Final;
+}
+
+
+
+//===----------------------------------------------------------------------===//
+// getSplitIteration -
+// Rather than spend rarely-used space recording the splitting iteration
+// during the Weak-Crossing SIV test, we re-compute it on demand.
+// The re-computation is basically a repeat of the entire dependence test,
+// though simplified since we know that the dependence exists.
+// It's tedious, since we must go through all propagations, etc.
+//
+// Care is required to keep this code up to date w.r.t. the code above.
+//
+// Generally, the dependence analyzer will be used to build
+// a dependence graph for a function (basically a map from instructions
+// to dependences). Looking for cycles in the graph shows us loops
+// that cannot be trivially vectorized/parallelized.
+//
+// We can try to improve the situation by examining all the dependences
+// that make up the cycle, looking for ones we can break.
+// Sometimes, peeling the first or last iteration of a loop will break
+// dependences, and we've got flags for those possibilities.
+// Sometimes, splitting a loop at some other iteration will do the trick,
+// and we've got a flag for that case. Rather than waste the space to
+// record the exact iteration (since we rarely know), we provide
+// a method that calculates the iteration. It's a drag that it must work
+// from scratch, but wonderful in that it's possible.
+//
+// Here's an example:
+//
+//    for (i = 0; i < 10; i++)
+//        A[i] = ...
+//        ... = A[11 - i]
+//
+// There's a loop-carried flow dependence from the store to the load,
+// found by the weak-crossing SIV test. The dependence will have a flag,
+// indicating that the dependence can be broken by splitting the loop.
+// Calling getSplitIteration will return 5.
+// Splitting the loop breaks the dependence, like so:
+//
+//    for (i = 0; i <= 5; i++)
+//        A[i] = ...
+//        ... = A[11 - i]
+//    for (i = 6; i < 10; i++)
+//        A[i] = ...
+//        ... = A[11 - i]
+//
+// breaks the dependence and allows us to vectorize/parallelize
+// both loops.
+const  SCEV *DependenceAnalysis::getSplitIteration(const Dependence *Dep,
+                                                   unsigned SplitLevel) {
+  assert(Dep && "expected a pointer to a Dependence");
+  assert(Dep->isSplitable(SplitLevel) &&
+         "Dep should be splitable at SplitLevel");
+  const Instruction *Src = Dep->getSrc();
+  const Instruction *Dst = Dep->getDst();
+  assert(Src->mayReadFromMemory() || Src->mayWriteToMemory());
+  assert(Dst->mayReadFromMemory() || Dst->mayWriteToMemory());
+  assert(isLoadOrStore(Src));
+  assert(isLoadOrStore(Dst));
+  const Value *SrcPtr = getPointerOperand(Src);
+  const Value *DstPtr = getPointerOperand(Dst);
+  assert(underlyingObjectsAlias(AA, DstPtr, SrcPtr) ==
+         AliasAnalysis::MustAlias);
+  const GEPOperator *SrcGEP = dyn_cast<GEPOperator>(SrcPtr);
+  const GEPOperator *DstGEP = dyn_cast<GEPOperator>(DstPtr);
+  assert(SrcGEP);
+  assert(DstGEP);
+  assert(SrcGEP->getPointerOperandType() == DstGEP->getPointerOperandType());
+
+  // establish loop nesting levels
+  establishNestingLevels(Src, Dst);
+
+  FullDependence Result(Src, Dst, false, CommonLevels);
+
+  // classify subscript pairs
+  unsigned Pairs = SrcGEP->idx_end() - SrcGEP->idx_begin();
+  SmallVector<Subscript, 4> Pair(Pairs);
+  for (unsigned SI = 0; SI < Pairs; ++SI) {
+    Pair[SI].Loops.resize(MaxLevels + 1);
+    Pair[SI].GroupLoops.resize(MaxLevels + 1);
+    Pair[SI].Group.resize(Pairs);
+  }
+  Pairs = 0;
+  for (GEPOperator::const_op_iterator SrcIdx = SrcGEP->idx_begin(),
+         SrcEnd = SrcGEP->idx_end(),
+         DstIdx = DstGEP->idx_begin(),
+         DstEnd = DstGEP->idx_end();
+       SrcIdx != SrcEnd && DstIdx != DstEnd;
+       ++SrcIdx, ++DstIdx, ++Pairs) {
+    Pair[Pairs].Src = SE->getSCEV(*SrcIdx);
+    Pair[Pairs].Dst = SE->getSCEV(*DstIdx);
+    Pair[Pairs].Classification =
+      classifyPair(Pair[Pairs].Src, LI->getLoopFor(Src->getParent()),
+                   Pair[Pairs].Dst, LI->getLoopFor(Dst->getParent()),
+                   Pair[Pairs].Loops);
+    Pair[Pairs].GroupLoops = Pair[Pairs].Loops;
+    Pair[Pairs].Group.set(Pairs);
+  }
+
+  SmallBitVector Separable(Pairs);
+  SmallBitVector Coupled(Pairs);
+
+  // partition subscripts into separable and minimally-coupled groups
+  for (unsigned SI = 0; SI < Pairs; ++SI) {
+    if (Pair[SI].Classification == Subscript::NonLinear) {
+      // ignore these, but collect loops for later
+      collectCommonLoops(Pair[SI].Src,
+                         LI->getLoopFor(Src->getParent()),
+                         Pair[SI].Loops);
+      collectCommonLoops(Pair[SI].Dst,
+                         LI->getLoopFor(Dst->getParent()),
+                         Pair[SI].Loops);
+      Result.Consistent = false;
+    }
+    else if (Pair[SI].Classification == Subscript::ZIV)
+      Separable.set(SI);
+    else {
+      // SIV, RDIV, or MIV, so check for coupled group
+      bool Done = true;
+      for (unsigned SJ = SI + 1; SJ < Pairs; ++SJ) {
+        SmallBitVector Intersection = Pair[SI].GroupLoops;
+        Intersection &= Pair[SJ].GroupLoops;
+        if (Intersection.any()) {
+          // accumulate set of all the loops in group
+          Pair[SJ].GroupLoops |= Pair[SI].GroupLoops;
+          // accumulate set of all subscripts in group
+          Pair[SJ].Group |= Pair[SI].Group;
+          Done = false;
+        }
+      }
+      if (Done) {
+        if (Pair[SI].Group.count() == 1)
+          Separable.set(SI);
+        else
+          Coupled.set(SI);
+      }
+    }
+  }
+
+  Constraint NewConstraint;
+  NewConstraint.setAny(SE);
+
+  // test separable subscripts
+  for (int SI = Separable.find_first(); SI >= 0; SI = Separable.find_next(SI)) {
+    switch (Pair[SI].Classification) {
+    case Subscript::SIV: {
+      unsigned Level;
+      const SCEV *SplitIter = NULL;
+      (void) testSIV(Pair[SI].Src, Pair[SI].Dst, Level,
+                     Result, NewConstraint, SplitIter);
+      if (Level == SplitLevel) {
+        assert(SplitIter != NULL);
+        return SplitIter;
+      }
+      break;
+    }
+    case Subscript::ZIV:
+    case Subscript::RDIV:
+    case Subscript::MIV:
+      break;
+    default:
+      llvm_unreachable("subscript has unexpected classification");
+    }
+  }
+
+  if (Coupled.count()) {
+    // test coupled subscript groups
+    SmallVector<Constraint, 4> Constraints(MaxLevels + 1);
+    for (unsigned II = 0; II <= MaxLevels; ++II)
+      Constraints[II].setAny(SE);
+    for (int SI = Coupled.find_first(); SI >= 0; SI = Coupled.find_next(SI)) {
+      SmallBitVector Group(Pair[SI].Group);
+      SmallBitVector Sivs(Pairs);
+      SmallBitVector Mivs(Pairs);
+      SmallBitVector ConstrainedLevels(MaxLevels + 1);
+      for (int SJ = Group.find_first(); SJ >= 0; SJ = Group.find_next(SJ)) {
+        if (Pair[SJ].Classification == Subscript::SIV)
+          Sivs.set(SJ);
+        else
+          Mivs.set(SJ);
+      }
+      while (Sivs.any()) {
+        bool Changed = false;
+        for (int SJ = Sivs.find_first(); SJ >= 0; SJ = Sivs.find_next(SJ)) {
+          // SJ is an SIV subscript that's part of the current coupled group
+          unsigned Level;
+          const SCEV *SplitIter = NULL;
+          (void) testSIV(Pair[SJ].Src, Pair[SJ].Dst, Level,
+                         Result, NewConstraint, SplitIter);
+          if (Level == SplitLevel && SplitIter)
+            return SplitIter;
+          ConstrainedLevels.set(Level);
+          if (intersectConstraints(&Constraints[Level], &NewConstraint))
+            Changed = true;
+          Sivs.reset(SJ);
+        }
+        if (Changed) {
+          // propagate, possibly creating new SIVs and ZIVs
+          for (int SJ = Mivs.find_first(); SJ >= 0; SJ = Mivs.find_next(SJ)) {
+            // SJ is an MIV subscript that's part of the current coupled group
+            if (propagate(Pair[SJ].Src, Pair[SJ].Dst,
+                          Pair[SJ].Loops, Constraints, Result.Consistent)) {
+              Pair[SJ].Classification =
+                classifyPair(Pair[SJ].Src, LI->getLoopFor(Src->getParent()),
+                             Pair[SJ].Dst, LI->getLoopFor(Dst->getParent()),
+                             Pair[SJ].Loops);
+              switch (Pair[SJ].Classification) {
+              case Subscript::ZIV:
+                Mivs.reset(SJ);
+                break;
+              case Subscript::SIV:
+                Sivs.set(SJ);
+                Mivs.reset(SJ);
+                break;
+              case Subscript::RDIV:
+              case Subscript::MIV:
+                break;
+              default:
+                llvm_unreachable("bad subscript classification");
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+  llvm_unreachable("somehow reached end of routine");
+  return NULL;
+}
diff --git a/lib/Analysis/DominanceFrontier.cpp b/lib/Analysis/DominanceFrontier.cpp
index 1604576..3e537e9 100644
--- a/lib/Analysis/DominanceFrontier.cpp
+++ b/lib/Analysis/DominanceFrontier.cpp
@@ -133,7 +133,9 @@ void DominanceFrontierBase::print(raw_ostream &OS, const Module* ) const {
   }
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void DominanceFrontierBase::dump() const {
   print(dbgs());
 }
+#endif
 
diff --git a/lib/Analysis/IPA/CallGraph.cpp b/lib/Analysis/IPA/CallGraph.cpp
index 0df3e8a..dec0ece 100644
--- a/lib/Analysis/IPA/CallGraph.cpp
+++ b/lib/Analysis/IPA/CallGraph.cpp
@@ -141,12 +141,13 @@ private:
       for (BasicBlock::iterator II = BB->begin(), IE = BB->end();
            II != IE; ++II) {
         CallSite CS(cast<Value>(II));
-        if (CS && !isa<IntrinsicInst>(II)) {
+        if (CS) {
           const Function *Callee = CS.getCalledFunction();
-          if (Callee)
-            Node->addCalledFunction(CS, getOrInsertFunction(Callee));
-          else
+          if (!Callee)
+            // Indirect calls of intrinsics are not allowed so no need to check.
             Node->addCalledFunction(CS, CallsExternalNode);
+          else if (!Callee->isIntrinsic())
+            Node->addCalledFunction(CS, getOrInsertFunction(Callee));
         }
       }
   }
@@ -198,9 +199,11 @@ void CallGraph::print(raw_ostream &OS, Module*) const {
   for (CallGraph::const_iterator I = begin(), E = end(); I != E; ++I)
     I->second->print(OS);
 }
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void CallGraph::dump() const {
   print(dbgs(), 0);
 }
+#endif
 
 //===----------------------------------------------------------------------===//
 // Implementations of public modification methods
@@ -267,7 +270,9 @@ void CallGraphNode::print(raw_ostream &OS) const {
   OS << '\n';
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void CallGraphNode::dump() const { print(dbgs()); }
+#endif
 
 /// removeCallEdgeFor - This method removes the edge in the node for the
 /// specified call site.  Note that this method takes linear time, so it
diff --git a/lib/Analysis/IPA/GlobalsModRef.cpp b/lib/Analysis/IPA/GlobalsModRef.cpp
index 22f6e96..990caa8 100644
--- a/lib/Analysis/IPA/GlobalsModRef.cpp
+++ b/lib/Analysis/IPA/GlobalsModRef.cpp
@@ -263,7 +263,7 @@ bool GlobalsModRef::AnalyzeUsesOfPointer(Value *V,
     } else if (BitCastInst *BCI = dyn_cast<BitCastInst>(U)) {
       if (AnalyzeUsesOfPointer(BCI, Readers, Writers, OkayStoreDest))
         return true;
-    } else if (isFreeCall(U)) {
+    } else if (isFreeCall(U, TLI)) {
       Writers.push_back(cast<Instruction>(U)->getParent()->getParent());
     } else if (CallInst *CI = dyn_cast<CallInst>(U)) {
       // Make sure that this is just the function being called, not that it is
@@ -329,7 +329,7 @@ bool GlobalsModRef::AnalyzeIndirectGlobalMemory(GlobalValue *GV) {
       // Check the value being stored.
       Value *Ptr = GetUnderlyingObject(SI->getOperand(0));
 
-      if (!isAllocLikeFn(Ptr))
+      if (!isAllocLikeFn(Ptr, TLI))
         return false;  // Too hard to analyze.
 
       // Analyze all uses of the allocation.  If any of them are used in a
@@ -458,7 +458,7 @@ void GlobalsModRef::AnalyzeCallGraph(CallGraph &CG, Module &M) {
           if (SI->isVolatile())
             // Treat volatile stores as reading memory somewhere.
             FunctionEffect |= Ref;
-        } else if (isAllocationFn(&*II) || isFreeCall(&*II)) {
+        } else if (isAllocationFn(&*II, TLI) || isFreeCall(&*II, TLI)) {
           FunctionEffect |= ModRef;
         } else if (IntrinsicInst *Intrinsic = dyn_cast<IntrinsicInst>(&*II)) {
           // The callgraph doesn't include intrinsic calls.
diff --git a/lib/Analysis/IVUsers.cpp b/lib/Analysis/IVUsers.cpp
index 0a6682a..d4221b8 100644
--- a/lib/Analysis/IVUsers.cpp
+++ b/lib/Analysis/IVUsers.cpp
@@ -22,7 +22,7 @@
 #include "llvm/Analysis/LoopPass.h"
 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
 #include "llvm/Analysis/ValueTracking.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Assembly/Writer.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/Debug.h"
@@ -235,7 +235,7 @@ bool IVUsers::runOnLoop(Loop *l, LPPassManager &LPM) {
   LI = &getAnalysis<LoopInfo>();
   DT = &getAnalysis<DominatorTree>();
   SE = &getAnalysis<ScalarEvolution>();
-  TD = getAnalysisIfAvailable<TargetData>();
+  TD = getAnalysisIfAvailable<DataLayout>();
 
   // Find all uses of induction variables in this loop, and categorize
   // them by stride.  Start by finding all of the PHI nodes in the header for
@@ -273,9 +273,11 @@ void IVUsers::print(raw_ostream &OS, const Module *M) const {
   }
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void IVUsers::dump() const {
   print(dbgs());
 }
+#endif
 
 void IVUsers::releaseMemory() {
   Processed.clear();
diff --git a/lib/Analysis/InlineCost.cpp b/lib/Analysis/InlineCost.cpp
index bc1ecd2..5f51f77 100644
--- a/lib/Analysis/InlineCost.cpp
+++ b/lib/Analysis/InlineCost.cpp
@@ -24,7 +24,7 @@
 #include "llvm/IntrinsicInst.h"
 #include "llvm/Operator.h"
 #include "llvm/GlobalAlias.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallVector.h"
@@ -41,8 +41,8 @@ class CallAnalyzer : public InstVisitor<CallAnalyzer, bool> {
   typedef InstVisitor<CallAnalyzer, bool> Base;
   friend class InstVisitor<CallAnalyzer, bool>;
 
-  // TargetData if available, or null.
-  const TargetData *const TD;
+  // DataLayout if available, or null.
+  const DataLayout *const TD;
 
   // The called function.
   Function &F;
@@ -51,9 +51,12 @@ class CallAnalyzer : public InstVisitor<CallAnalyzer, bool> {
   int Cost;
   const bool AlwaysInline;
 
-  bool IsRecursive;
+  bool IsCallerRecursive;
+  bool IsRecursiveCall;
   bool ExposesReturnsTwice;
   bool HasDynamicAlloca;
+  /// Number of bytes allocated statically by the callee.
+  uint64_t AllocatedSize;
   unsigned NumInstructions, NumVectorInstructions;
   int FiftyPercentVectorBonus, TenPercentVectorBonus;
   int VectorBonus;
@@ -123,10 +126,11 @@ class CallAnalyzer : public InstVisitor<CallAnalyzer, bool> {
   bool visitCallSite(CallSite CS);
 
 public:
-  CallAnalyzer(const TargetData *TD, Function &Callee, int Threshold)
+  CallAnalyzer(const DataLayout *TD, Function &Callee, int Threshold)
     : TD(TD), F(Callee), Threshold(Threshold), Cost(0),
-      AlwaysInline(F.hasFnAttr(Attribute::AlwaysInline)),
-      IsRecursive(false), ExposesReturnsTwice(false), HasDynamicAlloca(false),
+      AlwaysInline(F.getFnAttributes().hasAttribute(Attributes::AlwaysInline)),
+      IsCallerRecursive(false), IsRecursiveCall(false),
+      ExposesReturnsTwice(false), HasDynamicAlloca(false), AllocatedSize(0),
       NumInstructions(0), NumVectorInstructions(0),
       FiftyPercentVectorBonus(0), TenPercentVectorBonus(0), VectorBonus(0),
       NumConstantArgs(0), NumConstantOffsetPtrArgs(0), NumAllocaArgs(0),
@@ -138,6 +142,7 @@ public:
 
   int getThreshold() { return Threshold; }
   int getCost() { return Cost; }
+  bool isAlwaysInline() { return AlwaysInline; }
 
   // Keep a bunch of stats about the cost savings found so we can print them
   // out when debugging.
@@ -269,6 +274,13 @@ bool CallAnalyzer::visitAlloca(AllocaInst &I) {
   // FIXME: Check whether inlining will turn a dynamic alloca into a static
   // alloca, and handle that case.
 
+  // Accumulate the allocated size.
+  if (I.isStaticAlloca()) {
+    Type *Ty = I.getAllocatedType();
+    AllocatedSize += (TD ? TD->getTypeAllocSize(Ty) :
+                      Ty->getPrimitiveSizeInBits());
+  }
+
   // We will happily inline static alloca instructions or dynamic alloca
   // instructions in always-inline situations.
   if (AlwaysInline || I.isStaticAlloca())
@@ -602,7 +614,7 @@ bool CallAnalyzer::visitStore(StoreInst &I) {
 
 bool CallAnalyzer::visitCallSite(CallSite CS) {
   if (CS.isCall() && cast<CallInst>(CS.getInstruction())->canReturnTwice() &&
-      !F.hasFnAttr(Attribute::ReturnsTwice)) {
+      !F.getFnAttributes().hasAttribute(Attributes::ReturnsTwice)) {
     // This aborts the entire analysis.
     ExposesReturnsTwice = true;
     return false;
@@ -625,7 +637,7 @@ bool CallAnalyzer::visitCallSite(CallSite CS) {
     if (F == CS.getInstruction()->getParent()->getParent()) {
       // This flag will fully abort the analysis, so don't bother with anything
       // else.
-      IsRecursive = true;
+      IsRecursiveCall = true;
       return false;
     }
 
@@ -712,7 +724,14 @@ bool CallAnalyzer::analyzeBlock(BasicBlock *BB) {
       Cost += InlineConstants::InstrCost;
 
     // If the visit this instruction detected an uninlinable pattern, abort.
-    if (IsRecursive || ExposesReturnsTwice || HasDynamicAlloca)
+    if (IsRecursiveCall || ExposesReturnsTwice || HasDynamicAlloca)
+      return false;
+
+    // If the caller is a recursive function then we don't want to inline
+    // functions which allocate a lot of stack space because it would increase
+    // the caller stack usage dramatically.
+    if (IsCallerRecursive &&
+        AllocatedSize > InlineConstants::TotalAllocaSizeRecursiveCaller)
       return false;
 
     if (NumVectorInstructions > NumInstructions/2)
@@ -814,7 +833,7 @@ bool CallAnalyzer::analyzeCall(CallSite CS) {
         // one load and one store per word copied.
         // FIXME: The maxStoresPerMemcpy setting from the target should be used
         // here instead of a magic number of 8, but it's not available via
-        // TargetData.
+        // DataLayout.
         NumStores = std::min(NumStores, 8U);
 
         Cost -= 2 * NumStores * InlineConstants::InstrCost;
@@ -831,12 +850,14 @@ bool CallAnalyzer::analyzeCall(CallSite CS) {
       Cost += InlineConstants::LastCallToStaticBonus;
 
     // If the instruction after the call, or if the normal destination of the
-    // invoke is an unreachable instruction, the function is noreturn.  As such,
-    // there is little point in inlining this unless there is literally zero cost.
-    if (InvokeInst *II = dyn_cast<InvokeInst>(CS.getInstruction())) {
+    // invoke is an unreachable instruction, the function is noreturn. As such,
+    // there is little point in inlining this unless there is literally zero
+    // cost.
+    Instruction *Instr = CS.getInstruction();
+    if (InvokeInst *II = dyn_cast<InvokeInst>(Instr)) {
       if (isa<UnreachableInst>(II->getNormalDest()->begin()))
         Threshold = 1;
-    } else if (isa<UnreachableInst>(++BasicBlock::iterator(CS.getInstruction())))
+    } else if (isa<UnreachableInst>(++BasicBlock::iterator(Instr)))
       Threshold = 1;
 
     // If this function uses the coldcc calling convention, prefer not to inline
@@ -852,6 +873,20 @@ bool CallAnalyzer::analyzeCall(CallSite CS) {
   if (F.empty())
     return true;
 
+  Function *Caller = CS.getInstruction()->getParent()->getParent();
+  // Check if the caller function is recursive itself.
+  for (Value::use_iterator U = Caller->use_begin(), E = Caller->use_end();
+       U != E; ++U) {
+    CallSite Site(cast<Value>(*U));
+    if (!Site)
+      continue;
+    Instruction *I = Site.getInstruction();
+    if (I->getParent()->getParent() == Caller) {
+      IsCallerRecursive = true;
+      break;
+    }
+  }
+
   // Track whether we've seen a return instruction. The first return
   // instruction is free, as at least one will usually disappear in inlining.
   bool HasReturn = false;
@@ -908,9 +943,9 @@ bool CallAnalyzer::analyzeCall(CallSite CS) {
 
     // We never want to inline functions that contain an indirectbr.  This is
     // incorrect because all the blockaddress's (in static global initializers
-    // for example) would be referring to the original function, and this indirect
-    // jump would jump from the inlined copy of the function into the original
-    // function which is extremely undefined behavior.
+    // for example) would be referring to the original function, and this
+    // indirect jump would jump from the inlined copy of the function into the 
+    // original function which is extremely undefined behavior.
     // FIXME: This logic isn't really right; we can safely inline functions
     // with indirectbr's as long as no other function or global references the
     // blockaddress of a block within the current function.  And as a QOI issue,
@@ -928,8 +963,16 @@ bool CallAnalyzer::analyzeCall(CallSite CS) {
     // Analyze the cost of this block. If we blow through the threshold, this
     // returns false, and we can bail on out.
     if (!analyzeBlock(BB)) {
-      if (IsRecursive || ExposesReturnsTwice || HasDynamicAlloca)
+      if (IsRecursiveCall || ExposesReturnsTwice || HasDynamicAlloca)
         return false;
+
+      // If the caller is a recursive function then we don't want to inline
+      // functions which allocate a lot of stack space because it would increase
+      // the caller stack usage dramatically.
+      if (IsCallerRecursive &&
+          AllocatedSize > InlineConstants::TotalAllocaSizeRecursiveCaller)
+        return false;
+
       break;
     }
 
@@ -955,7 +998,8 @@ bool CallAnalyzer::analyzeCall(CallSite CS) {
 
     // If we're unable to select a particular successor, just count all of
     // them.
-    for (unsigned TIdx = 0, TSize = TI->getNumSuccessors(); TIdx != TSize; ++TIdx)
+    for (unsigned TIdx = 0, TSize = TI->getNumSuccessors(); TIdx != TSize;
+         ++TIdx)
       BBWorklist.insert(TI->getSuccessor(TIdx));
 
     // If we had any successors at this point, than post-inlining is likely to
@@ -974,6 +1018,7 @@ bool CallAnalyzer::analyzeCall(CallSite CS) {
   return AlwaysInline || Cost < Threshold;
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 /// \brief Dump stats about this call's analysis.
 void CallAnalyzer::dump() {
 #define DEBUG_PRINT_STAT(x) llvm::dbgs() << "      " #x ": " << x << "\n"
@@ -987,6 +1032,7 @@ void CallAnalyzer::dump() {
   DEBUG_PRINT_STAT(SROACostSavingsLost);
 #undef DEBUG_PRINT_STAT
 }
+#endif
 
 InlineCost InlineCostAnalyzer::getInlineCost(CallSite CS, int Threshold) {
   return getInlineCost(CS, CS.getCalledFunction(), Threshold);
@@ -998,10 +1044,12 @@ InlineCost InlineCostAnalyzer::getInlineCost(CallSite CS, Function *Callee,
   // something else.  Don't inline functions marked noinline or call sites
   // marked noinline.
   if (!Callee || Callee->mayBeOverridden() ||
-      Callee->hasFnAttr(Attribute::NoInline) || CS.isNoInline())
+      Callee->getFnAttributes().hasAttribute(Attributes::NoInline) ||
+      CS.isNoInline())
     return llvm::InlineCost::getNever();
 
-  DEBUG(llvm::dbgs() << "      Analyzing call of " << Callee->getName() << "...\n");
+  DEBUG(llvm::dbgs() << "      Analyzing call of " << Callee->getName()
+        << "...\n");
 
   CallAnalyzer CA(TD, *Callee, Threshold);
   bool ShouldInline = CA.analyzeCall(CS);
@@ -1011,7 +1059,8 @@ InlineCost InlineCostAnalyzer::getInlineCost(CallSite CS, Function *Callee,
   // Check if there was a reason to force inlining or no inlining.
   if (!ShouldInline && CA.getCost() < CA.getThreshold())
     return InlineCost::getNever();
-  if (ShouldInline && CA.getCost() >= CA.getThreshold())
+  if (ShouldInline && (CA.isAlwaysInline() ||
+                       CA.getCost() >= CA.getThreshold()))
     return InlineCost::getAlways();
 
   return llvm::InlineCost::get(CA.getCost(), CA.getThreshold());
diff --git a/lib/Analysis/InstructionSimplify.cpp b/lib/Analysis/InstructionSimplify.cpp
index 379a35a..a76e5ad 100644
--- a/lib/Analysis/InstructionSimplify.cpp
+++ b/lib/Analysis/InstructionSimplify.cpp
@@ -31,7 +31,7 @@
 #include "llvm/Support/GetElementPtrTypeIterator.h"
 #include "llvm/Support/PatternMatch.h"
 #include "llvm/Support/ValueHandle.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 using namespace llvm;
 using namespace llvm::PatternMatch;
 
@@ -42,11 +42,11 @@ STATISTIC(NumFactor , "Number of factorizations");
 STATISTIC(NumReassoc, "Number of reassociations");
 
 struct Query {
-  const TargetData *TD;
+  const DataLayout *TD;
   const TargetLibraryInfo *TLI;
   const DominatorTree *DT;
 
-  Query(const TargetData *td, const TargetLibraryInfo *tli,
+  Query(const DataLayout *td, const TargetLibraryInfo *tli,
         const DominatorTree *dt) : TD(td), TLI(tli), DT(dt) {}
 };
 
@@ -651,7 +651,7 @@ static Value *SimplifyAddInst(Value *Op0, Value *Op1, bool isNSW, bool isNUW,
 }
 
 Value *llvm::SimplifyAddInst(Value *Op0, Value *Op1, bool isNSW, bool isNUW,
-                             const TargetData *TD, const TargetLibraryInfo *TLI,
+                             const DataLayout *TD, const TargetLibraryInfo *TLI,
                              const DominatorTree *DT) {
   return ::SimplifyAddInst(Op0, Op1, isNSW, isNUW, Query (TD, TLI, DT),
                            RecursionLimit);
@@ -664,7 +664,7 @@ Value *llvm::SimplifyAddInst(Value *Op0, Value *Op1, bool isNSW, bool isNUW,
 /// if the GEP has all-constant indices. Returns false if any non-constant
 /// index is encountered leaving the 'Offset' in an undefined state. The
 /// 'Offset' APInt must be the bitwidth of the target's pointer size.
-static bool accumulateGEPOffset(const TargetData &TD, GEPOperator *GEP,
+static bool accumulateGEPOffset(const DataLayout &TD, GEPOperator *GEP,
                                 APInt &Offset) {
   unsigned IntPtrWidth = TD.getPointerSizeInBits();
   assert(IntPtrWidth == Offset.getBitWidth());
@@ -696,7 +696,7 @@ static bool accumulateGEPOffset(const TargetData &TD, GEPOperator *GEP,
 /// accumulates the total constant offset applied in the returned constant. It
 /// returns 0 if V is not a pointer, and returns the constant '0' if there are
 /// no constant offsets applied.
-static Constant *stripAndComputeConstantOffsets(const TargetData &TD,
+static Constant *stripAndComputeConstantOffsets(const DataLayout &TD,
                                                 Value *&V) {
   if (!V->getType()->isPointerTy())
     return 0;
@@ -731,7 +731,7 @@ static Constant *stripAndComputeConstantOffsets(const TargetData &TD,
 
 /// \brief Compute the constant difference between two pointer values.
 /// If the difference is not a constant, returns zero.
-static Constant *computePointerDifference(const TargetData &TD,
+static Constant *computePointerDifference(const DataLayout &TD,
                                           Value *LHS, Value *RHS) {
   Constant *LHSOffset = stripAndComputeConstantOffsets(TD, LHS);
   if (!LHSOffset)
@@ -880,7 +880,7 @@ static Value *SimplifySubInst(Value *Op0, Value *Op1, bool isNSW, bool isNUW,
 }
 
 Value *llvm::SimplifySubInst(Value *Op0, Value *Op1, bool isNSW, bool isNUW,
-                             const TargetData *TD, const TargetLibraryInfo *TLI,
+                             const DataLayout *TD, const TargetLibraryInfo *TLI,
                              const DominatorTree *DT) {
   return ::SimplifySubInst(Op0, Op1, isNSW, isNUW, Query (TD, TLI, DT),
                            RecursionLimit);
@@ -951,7 +951,7 @@ static Value *SimplifyMulInst(Value *Op0, Value *Op1, const Query &Q,
   return 0;
 }
 
-Value *llvm::SimplifyMulInst(Value *Op0, Value *Op1, const TargetData *TD,
+Value *llvm::SimplifyMulInst(Value *Op0, Value *Op1, const DataLayout *TD,
                              const TargetLibraryInfo *TLI,
                              const DominatorTree *DT) {
   return ::SimplifyMulInst(Op0, Op1, Query (TD, TLI, DT), RecursionLimit);
@@ -1039,7 +1039,7 @@ static Value *SimplifySDivInst(Value *Op0, Value *Op1, const Query &Q,
   return 0;
 }
 
-Value *llvm::SimplifySDivInst(Value *Op0, Value *Op1, const TargetData *TD,
+Value *llvm::SimplifySDivInst(Value *Op0, Value *Op1, const DataLayout *TD,
                               const TargetLibraryInfo *TLI,
                               const DominatorTree *DT) {
   return ::SimplifySDivInst(Op0, Op1, Query (TD, TLI, DT), RecursionLimit);
@@ -1055,7 +1055,7 @@ static Value *SimplifyUDivInst(Value *Op0, Value *Op1, const Query &Q,
   return 0;
 }
 
-Value *llvm::SimplifyUDivInst(Value *Op0, Value *Op1, const TargetData *TD,
+Value *llvm::SimplifyUDivInst(Value *Op0, Value *Op1, const DataLayout *TD,
                               const TargetLibraryInfo *TLI,
                               const DominatorTree *DT) {
   return ::SimplifyUDivInst(Op0, Op1, Query (TD, TLI, DT), RecursionLimit);
@@ -1074,7 +1074,7 @@ static Value *SimplifyFDivInst(Value *Op0, Value *Op1, const Query &Q,
   return 0;
 }
 
-Value *llvm::SimplifyFDivInst(Value *Op0, Value *Op1, const TargetData *TD,
+Value *llvm::SimplifyFDivInst(Value *Op0, Value *Op1, const DataLayout *TD,
                               const TargetLibraryInfo *TLI,
                               const DominatorTree *DT) {
   return ::SimplifyFDivInst(Op0, Op1, Query (TD, TLI, DT), RecursionLimit);
@@ -1144,7 +1144,7 @@ static Value *SimplifySRemInst(Value *Op0, Value *Op1, const Query &Q,
   return 0;
 }
 
-Value *llvm::SimplifySRemInst(Value *Op0, Value *Op1, const TargetData *TD,
+Value *llvm::SimplifySRemInst(Value *Op0, Value *Op1, const DataLayout *TD,
                               const TargetLibraryInfo *TLI,
                               const DominatorTree *DT) {
   return ::SimplifySRemInst(Op0, Op1, Query (TD, TLI, DT), RecursionLimit);
@@ -1160,7 +1160,7 @@ static Value *SimplifyURemInst(Value *Op0, Value *Op1, const Query &Q,
   return 0;
 }
 
-Value *llvm::SimplifyURemInst(Value *Op0, Value *Op1, const TargetData *TD,
+Value *llvm::SimplifyURemInst(Value *Op0, Value *Op1, const DataLayout *TD,
                               const TargetLibraryInfo *TLI,
                               const DominatorTree *DT) {
   return ::SimplifyURemInst(Op0, Op1, Query (TD, TLI, DT), RecursionLimit);
@@ -1179,7 +1179,7 @@ static Value *SimplifyFRemInst(Value *Op0, Value *Op1, const Query &,
   return 0;
 }
 
-Value *llvm::SimplifyFRemInst(Value *Op0, Value *Op1, const TargetData *TD,
+Value *llvm::SimplifyFRemInst(Value *Op0, Value *Op1, const DataLayout *TD,
                               const TargetLibraryInfo *TLI,
                               const DominatorTree *DT) {
   return ::SimplifyFRemInst(Op0, Op1, Query (TD, TLI, DT), RecursionLimit);
@@ -1248,7 +1248,7 @@ static Value *SimplifyShlInst(Value *Op0, Value *Op1, bool isNSW, bool isNUW,
 }
 
 Value *llvm::SimplifyShlInst(Value *Op0, Value *Op1, bool isNSW, bool isNUW,
-                             const TargetData *TD, const TargetLibraryInfo *TLI,
+                             const DataLayout *TD, const TargetLibraryInfo *TLI,
                              const DominatorTree *DT) {
   return ::SimplifyShlInst(Op0, Op1, isNSW, isNUW, Query (TD, TLI, DT),
                            RecursionLimit);
@@ -1275,7 +1275,7 @@ static Value *SimplifyLShrInst(Value *Op0, Value *Op1, bool isExact,
 }
 
 Value *llvm::SimplifyLShrInst(Value *Op0, Value *Op1, bool isExact,
-                              const TargetData *TD,
+                              const DataLayout *TD,
                               const TargetLibraryInfo *TLI,
                               const DominatorTree *DT) {
   return ::SimplifyLShrInst(Op0, Op1, isExact, Query (TD, TLI, DT),
@@ -1307,7 +1307,7 @@ static Value *SimplifyAShrInst(Value *Op0, Value *Op1, bool isExact,
 }
 
 Value *llvm::SimplifyAShrInst(Value *Op0, Value *Op1, bool isExact,
-                              const TargetData *TD,
+                              const DataLayout *TD,
                               const TargetLibraryInfo *TLI,
                               const DominatorTree *DT) {
   return ::SimplifyAShrInst(Op0, Op1, isExact, Query (TD, TLI, DT),
@@ -1407,7 +1407,7 @@ static Value *SimplifyAndInst(Value *Op0, Value *Op1, const Query &Q,
   return 0;
 }
 
-Value *llvm::SimplifyAndInst(Value *Op0, Value *Op1, const TargetData *TD,
+Value *llvm::SimplifyAndInst(Value *Op0, Value *Op1, const DataLayout *TD,
                              const TargetLibraryInfo *TLI,
                              const DominatorTree *DT) {
   return ::SimplifyAndInst(Op0, Op1, Query (TD, TLI, DT), RecursionLimit);
@@ -1501,7 +1501,7 @@ static Value *SimplifyOrInst(Value *Op0, Value *Op1, const Query &Q,
   return 0;
 }
 
-Value *llvm::SimplifyOrInst(Value *Op0, Value *Op1, const TargetData *TD,
+Value *llvm::SimplifyOrInst(Value *Op0, Value *Op1, const DataLayout *TD,
                             const TargetLibraryInfo *TLI,
                             const DominatorTree *DT) {
   return ::SimplifyOrInst(Op0, Op1, Query (TD, TLI, DT), RecursionLimit);
@@ -1561,7 +1561,7 @@ static Value *SimplifyXorInst(Value *Op0, Value *Op1, const Query &Q,
   return 0;
 }
 
-Value *llvm::SimplifyXorInst(Value *Op0, Value *Op1, const TargetData *TD,
+Value *llvm::SimplifyXorInst(Value *Op0, Value *Op1, const DataLayout *TD,
                              const TargetLibraryInfo *TLI,
                              const DominatorTree *DT) {
   return ::SimplifyXorInst(Op0, Op1, Query (TD, TLI, DT), RecursionLimit);
@@ -1591,7 +1591,7 @@ static Value *ExtractEquivalentCondition(Value *V, CmpInst::Predicate Pred,
   return 0;
 }
 
-static Constant *computePointerICmp(const TargetData &TD,
+static Constant *computePointerICmp(const DataLayout &TD,
                                     CmpInst::Predicate Pred,
                                     Value *LHS, Value *RHS) {
   // We can only fold certain predicates on pointer comparisons.
@@ -2065,8 +2065,25 @@ static Value *SimplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS,
     if (A && C && (A == C || A == D || B == C || B == D) &&
         NoLHSWrapProblem && NoRHSWrapProblem) {
       // Determine Y and Z in the form icmp (X+Y), (X+Z).
-      Value *Y = (A == C || A == D) ? B : A;
-      Value *Z = (C == A || C == B) ? D : C;
+      Value *Y, *Z;
+      if (A == C) {
+        // C + B == C + D  ->  B == D
+        Y = B;
+        Z = D;
+      } else if (A == D) {
+        // D + B == C + D  ->  B == C
+        Y = B;
+        Z = C;
+      } else if (B == C) {
+        // A + C == C + D  ->  A == D
+        Y = A;
+        Z = D;
+      } else {
+        assert(B == D);
+        // A + D == C + D  ->  A == C
+        Y = A;
+        Z = C;
+      }
       if (Value *V = SimplifyICmpInst(Pred, Y, Z, Q, MaxRecurse-1))
         return V;
     }
@@ -2399,7 +2416,7 @@ static Value *SimplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS,
 }
 
 Value *llvm::SimplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS,
-                              const TargetData *TD,
+                              const DataLayout *TD,
                               const TargetLibraryInfo *TLI,
                               const DominatorTree *DT) {
   return ::SimplifyICmpInst(Predicate, LHS, RHS, Query (TD, TLI, DT),
@@ -2496,7 +2513,7 @@ static Value *SimplifyFCmpInst(unsigned Predicate, Value *LHS, Value *RHS,
 }
 
 Value *llvm::SimplifyFCmpInst(unsigned Predicate, Value *LHS, Value *RHS,
-                              const TargetData *TD,
+                              const DataLayout *TD,
                               const TargetLibraryInfo *TLI,
                               const DominatorTree *DT) {
   return ::SimplifyFCmpInst(Predicate, LHS, RHS, Query (TD, TLI, DT),
@@ -2531,7 +2548,7 @@ static Value *SimplifySelectInst(Value *CondVal, Value *TrueVal,
 }
 
 Value *llvm::SimplifySelectInst(Value *Cond, Value *TrueVal, Value *FalseVal,
-                                const TargetData *TD,
+                                const DataLayout *TD,
                                 const TargetLibraryInfo *TLI,
                                 const DominatorTree *DT) {
   return ::SimplifySelectInst(Cond, TrueVal, FalseVal, Query (TD, TLI, DT),
@@ -2579,7 +2596,7 @@ static Value *SimplifyGEPInst(ArrayRef<Value *> Ops, const Query &Q, unsigned) {
   return ConstantExpr::getGetElementPtr(cast<Constant>(Ops[0]), Ops.slice(1));
 }
 
-Value *llvm::SimplifyGEPInst(ArrayRef<Value *> Ops, const TargetData *TD,
+Value *llvm::SimplifyGEPInst(ArrayRef<Value *> Ops, const DataLayout *TD,
                              const TargetLibraryInfo *TLI,
                              const DominatorTree *DT) {
   return ::SimplifyGEPInst(Ops, Query (TD, TLI, DT), RecursionLimit);
@@ -2616,7 +2633,7 @@ static Value *SimplifyInsertValueInst(Value *Agg, Value *Val,
 
 Value *llvm::SimplifyInsertValueInst(Value *Agg, Value *Val,
                                      ArrayRef<unsigned> Idxs,
-                                     const TargetData *TD,
+                                     const DataLayout *TD,
                                      const TargetLibraryInfo *TLI,
                                      const DominatorTree *DT) {
   return ::SimplifyInsertValueInst(Agg, Val, Idxs, Query (TD, TLI, DT),
@@ -2664,7 +2681,7 @@ static Value *SimplifyTruncInst(Value *Op, Type *Ty, const Query &Q, unsigned) {
   return 0;
 }
 
-Value *llvm::SimplifyTruncInst(Value *Op, Type *Ty, const TargetData *TD,
+Value *llvm::SimplifyTruncInst(Value *Op, Type *Ty, const DataLayout *TD,
                                const TargetLibraryInfo *TLI,
                                const DominatorTree *DT) {
   return ::SimplifyTruncInst(Op, Ty, Query (TD, TLI, DT), RecursionLimit);
@@ -2730,7 +2747,7 @@ static Value *SimplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS,
 }
 
 Value *llvm::SimplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS,
-                           const TargetData *TD, const TargetLibraryInfo *TLI,
+                           const DataLayout *TD, const TargetLibraryInfo *TLI,
                            const DominatorTree *DT) {
   return ::SimplifyBinOp(Opcode, LHS, RHS, Query (TD, TLI, DT), RecursionLimit);
 }
@@ -2745,7 +2762,7 @@ static Value *SimplifyCmpInst(unsigned Predicate, Value *LHS, Value *RHS,
 }
 
 Value *llvm::SimplifyCmpInst(unsigned Predicate, Value *LHS, Value *RHS,
-                             const TargetData *TD, const TargetLibraryInfo *TLI,
+                             const DataLayout *TD, const TargetLibraryInfo *TLI,
                              const DominatorTree *DT) {
   return ::SimplifyCmpInst(Predicate, LHS, RHS, Query (TD, TLI, DT),
                            RecursionLimit);
@@ -2761,7 +2778,7 @@ static Value *SimplifyCallInst(CallInst *CI, const Query &) {
 
 /// SimplifyInstruction - See if we can compute a simplified version of this
 /// instruction.  If not, this returns null.
-Value *llvm::SimplifyInstruction(Instruction *I, const TargetData *TD,
+Value *llvm::SimplifyInstruction(Instruction *I, const DataLayout *TD,
                                  const TargetLibraryInfo *TLI,
                                  const DominatorTree *DT) {
   Value *Result;
@@ -2881,7 +2898,7 @@ Value *llvm::SimplifyInstruction(Instruction *I, const TargetData *TD,
 /// This routine returns 'true' only when *it* simplifies something. The passed
 /// in simplified value does not count toward this.
 static bool replaceAndRecursivelySimplifyImpl(Instruction *I, Value *SimpleV,
-                                              const TargetData *TD,
+                                              const DataLayout *TD,
                                               const TargetLibraryInfo *TLI,
                                               const DominatorTree *DT) {
   bool Simplified = false;
@@ -2936,14 +2953,14 @@ static bool replaceAndRecursivelySimplifyImpl(Instruction *I, Value *SimpleV,
 }
 
 bool llvm::recursivelySimplifyInstruction(Instruction *I,
-                                          const TargetData *TD,
+                                          const DataLayout *TD,
                                           const TargetLibraryInfo *TLI,
                                           const DominatorTree *DT) {
   return replaceAndRecursivelySimplifyImpl(I, 0, TD, TLI, DT);
 }
 
 bool llvm::replaceAndRecursivelySimplify(Instruction *I, Value *SimpleV,
-                                         const TargetData *TD,
+                                         const DataLayout *TD,
                                          const TargetLibraryInfo *TLI,
                                          const DominatorTree *DT) {
   assert(I != SimpleV && "replaceAndRecursivelySimplify(X,X) is not valid!");
diff --git a/lib/Analysis/LazyValueInfo.cpp b/lib/Analysis/LazyValueInfo.cpp
index 9140786..2b87d80 100644
--- a/lib/Analysis/LazyValueInfo.cpp
+++ b/lib/Analysis/LazyValueInfo.cpp
@@ -13,13 +13,14 @@
 //===----------------------------------------------------------------------===//
 
 #define DEBUG_TYPE "lazy-value-info"
+#include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/LazyValueInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/Constants.h"
 #include "llvm/Instructions.h"
 #include "llvm/IntrinsicInst.h"
 #include "llvm/Analysis/ConstantFolding.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetLibraryInfo.h"
 #include "llvm/Support/CFG.h"
 #include "llvm/Support/ConstantRange.h"
@@ -212,7 +213,7 @@ public:
 
         // Unless we can prove that the two Constants are different, we must
         // move to overdefined.
-        // FIXME: use TargetData/TargetLibraryInfo for smarter constant folding.
+        // FIXME: use DataLayout/TargetLibraryInfo for smarter constant folding.
         if (ConstantInt *Res = dyn_cast<ConstantInt>(
                 ConstantFoldCompareInstOperands(CmpInst::ICMP_NE,
                                                 getConstant(),
@@ -238,7 +239,7 @@ public:
 
         // Unless we can prove that the two Constants are different, we must
         // move to overdefined.
-        // FIXME: use TargetData/TargetLibraryInfo for smarter constant folding.
+        // FIXME: use DataLayout/TargetLibraryInfo for smarter constant folding.
         if (ConstantInt *Res = dyn_cast<ConstantInt>(
                 ConstantFoldCompareInstOperands(CmpInst::ICMP_NE,
                                                 getNotConstant(),
@@ -294,7 +295,7 @@ raw_ostream &operator<<(raw_ostream &OS, const LVILatticeVal &Val) {
 //===----------------------------------------------------------------------===//
 
 namespace {
-  /// LVIValueHandle - A callback value handle update the cache when
+  /// LVIValueHandle - A callback value handle updates the cache when
   /// values are erased.
   class LazyValueInfoCache;
   struct LVIValueHandle : public CallbackVH {
@@ -470,8 +471,10 @@ bool LazyValueInfoCache::hasBlockValue(Value *Val, BasicBlock *BB) {
     return true;
 
   LVIValueHandle ValHandle(Val, this);
-  if (!ValueCache.count(ValHandle)) return false;
-  return ValueCache[ValHandle].count(BB);
+  std::map<LVIValueHandle, ValueCacheEntryTy>::iterator I =
+    ValueCache.find(ValHandle);
+  if (I == ValueCache.end()) return false;
+  return I->second.count(BB);
 }
 
 LVILatticeVal LazyValueInfoCache::getBlockValue(Value *Val, BasicBlock *BB) {
@@ -555,13 +558,11 @@ bool LazyValueInfoCache::solveBlockValue(Value *Val, BasicBlock *BB) {
 static bool InstructionDereferencesPointer(Instruction *I, Value *Ptr) {
   if (LoadInst *L = dyn_cast<LoadInst>(I)) {
     return L->getPointerAddressSpace() == 0 &&
-        GetUnderlyingObject(L->getPointerOperand()) ==
-        GetUnderlyingObject(Ptr);
+        GetUnderlyingObject(L->getPointerOperand()) == Ptr;
   }
   if (StoreInst *S = dyn_cast<StoreInst>(I)) {
     return S->getPointerAddressSpace() == 0 &&
-        GetUnderlyingObject(S->getPointerOperand()) ==
-        GetUnderlyingObject(Ptr);
+        GetUnderlyingObject(S->getPointerOperand()) == Ptr;
   }
   if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(I)) {
     if (MI->isVolatile()) return false;
@@ -571,11 +572,11 @@ static bool InstructionDereferencesPointer(Instruction *I, Value *Ptr) {
     if (!Len || Len->isZero()) return false;
 
     if (MI->getDestAddressSpace() == 0)
-      if (MI->getRawDest() == Ptr || MI->getDest() == Ptr)
+      if (GetUnderlyingObject(MI->getRawDest()) == Ptr)
         return true;
     if (MemTransferInst *MTI = dyn_cast<MemTransferInst>(MI))
       if (MTI->getSourceAddressSpace() == 0)
-        if (MTI->getRawSource() == Ptr || MTI->getSource() == Ptr)
+        if (GetUnderlyingObject(MTI->getRawSource()) == Ptr)
           return true;
   }
   return false;
@@ -589,13 +590,19 @@ bool LazyValueInfoCache::solveBlockValueNonLocal(LVILatticeVal &BBLV,
   // then we know that the pointer can't be NULL.
   bool NotNull = false;
   if (Val->getType()->isPointerTy()) {
-    if (isa<AllocaInst>(Val)) {
+    if (isKnownNonNull(Val)) {
       NotNull = true;
     } else {
-      for (BasicBlock::iterator BI = BB->begin(), BE = BB->end();BI != BE;++BI){
-        if (InstructionDereferencesPointer(BI, Val)) {
-          NotNull = true;
-          break;
+      Value *UnderlyingVal = GetUnderlyingObject(Val);
+      // If 'GetUnderlyingObject' didn't converge, skip it. It won't converge
+      // inside InstructionDereferencesPointer either.
+      if (UnderlyingVal == GetUnderlyingObject(UnderlyingVal, NULL, 1)) {
+        for (BasicBlock::iterator BI = BB->begin(), BE = BB->end();
+             BI != BE; ++BI) {
+          if (InstructionDereferencesPointer(BI, UnderlyingVal)) {
+            NotNull = true;
+            break;
+          }
         }
       }
     }
@@ -845,9 +852,12 @@ static bool getEdgeValueLocal(Value *Val, BasicBlock *BBFrom,
     for (SwitchInst::CaseIt i = SI->case_begin(), e = SI->case_end();
          i != e; ++i) {
       ConstantRange EdgeVal(i.getCaseValue()->getValue());
-      if (DefaultCase)
-        EdgesVals = EdgesVals.difference(EdgeVal);
-      else if (i.getCaseSuccessor() == BBTo)
+      if (DefaultCase) {
+        // It is possible that the default destination is the destination of
+        // some cases. There is no need to perform difference for those cases.
+        if (i.getCaseSuccessor() != BBTo)
+          EdgesVals = EdgesVals.difference(EdgeVal);
+      } else if (i.getCaseSuccessor() == BBTo)
         EdgesVals = EdgesVals.unionWith(EdgeVal);
     }
     Result = LVILatticeVal::getRange(EdgesVals);
@@ -1004,7 +1014,7 @@ bool LazyValueInfo::runOnFunction(Function &F) {
   if (PImpl)
     getCache(PImpl).clear();
 
-  TD = getAnalysisIfAvailable<TargetData>();
+  TD = getAnalysisIfAvailable<DataLayout>();
   TLI = &getAnalysis<TargetLibraryInfo>();
 
   // Fully lazy.
diff --git a/lib/Analysis/Lint.cpp b/lib/Analysis/Lint.cpp
index 83bdf52..6d6d580 100644
--- a/lib/Analysis/Lint.cpp
+++ b/lib/Analysis/Lint.cpp
@@ -43,7 +43,7 @@
 #include "llvm/Analysis/Loads.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/Assembly/Writer.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetLibraryInfo.h"
 #include "llvm/Pass.h"
 #include "llvm/PassManager.h"
@@ -103,7 +103,7 @@ namespace {
     Module *Mod;
     AliasAnalysis *AA;
     DominatorTree *DT;
-    TargetData *TD;
+    DataLayout *TD;
     TargetLibraryInfo *TLI;
 
     std::string Messages;
@@ -177,7 +177,7 @@ bool Lint::runOnFunction(Function &F) {
   Mod = F.getParent();
   AA = &getAnalysis<AliasAnalysis>();
   DT = &getAnalysis<DominatorTree>();
-  TD = getAnalysisIfAvailable<TargetData>();
+  TD = getAnalysisIfAvailable<DataLayout>();
   TLI = &getAnalysis<TargetLibraryInfo>();
   visit(F);
   dbgs() << MessagesStr.str();
@@ -411,14 +411,50 @@ void Lint::visitMemoryReference(Instruction &I,
             "Undefined behavior: Branch to non-blockaddress", &I);
   }
 
+  // Check for buffer overflows and misalignment.
   if (TD) {
-    if (Align == 0 && Ty) Align = TD->getABITypeAlignment(Ty);
+    // Only handles memory references that read/write something simple like an
+    // alloca instruction or a global variable.
+    int64_t Offset = 0;
+    if (Value *Base = GetPointerBaseWithConstantOffset(Ptr, Offset, *TD)) {
+      // OK, so the access is to a constant offset from Ptr.  Check that Ptr is
+      // something we can handle and if so extract the size of this base object
+      // along with its alignment.
+      uint64_t BaseSize = AliasAnalysis::UnknownSize;
+      unsigned BaseAlign = 0;
+
+      if (AllocaInst *AI = dyn_cast<AllocaInst>(Base)) {
+        Type *ATy = AI->getAllocatedType();
+        if (!AI->isArrayAllocation() && ATy->isSized())
+          BaseSize = TD->getTypeAllocSize(ATy);
+        BaseAlign = AI->getAlignment();
+        if (BaseAlign == 0 && ATy->isSized())
+          BaseAlign = TD->getABITypeAlignment(ATy);
+      } else if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Base)) {
+        // If the global may be defined differently in another compilation unit
+        // then don't warn about funky memory accesses.
+        if (GV->hasDefinitiveInitializer()) {
+          Type *GTy = GV->getType()->getElementType();
+          if (GTy->isSized())
+            BaseSize = TD->getTypeAllocSize(GTy);
+          BaseAlign = GV->getAlignment();
+          if (BaseAlign == 0 && GTy->isSized())
+            BaseAlign = TD->getABITypeAlignment(GTy);
+        }
+      }
 
-    if (Align != 0) {
-      unsigned BitWidth = TD->getTypeSizeInBits(Ptr->getType());
-      APInt KnownZero(BitWidth, 0), KnownOne(BitWidth, 0);
-      ComputeMaskedBits(Ptr, KnownZero, KnownOne, TD);
-      Assert1(!(KnownOne & APInt::getLowBitsSet(BitWidth, Log2_32(Align))),
+      // Accesses from before the start or after the end of the object are not
+      // defined.
+      Assert1(Size == AliasAnalysis::UnknownSize ||
+              BaseSize == AliasAnalysis::UnknownSize ||
+              (Offset >= 0 && Offset + Size <= BaseSize),
+              "Undefined behavior: Buffer overflow", &I);
+
+      // Accesses that say that the memory is more aligned than it is are not
+      // defined.
+      if (Align == 0 && Ty && Ty->isSized())
+        Align = TD->getABITypeAlignment(Ty);
+      Assert1(!BaseAlign || Align <= MinAlign(BaseAlign, Offset),
               "Undefined behavior: Memory reference address is misaligned", &I);
     }
   }
@@ -470,7 +506,7 @@ void Lint::visitShl(BinaryOperator &I) {
             "Undefined result: Shift count out of range", &I);
 }
 
-static bool isZero(Value *V, TargetData *TD) {
+static bool isZero(Value *V, DataLayout *TD) {
   // Assume undef could be zero.
   if (isa<UndefValue>(V)) return true;
 
diff --git a/lib/Analysis/Loads.cpp b/lib/Analysis/Loads.cpp
index 873a275..73aa8b4 100644
--- a/lib/Analysis/Loads.cpp
+++ b/lib/Analysis/Loads.cpp
@@ -13,7 +13,7 @@
 
 #include "llvm/Analysis/Loads.h"
 #include "llvm/Analysis/AliasAnalysis.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/GlobalAlias.h"
 #include "llvm/GlobalVariable.h"
 #include "llvm/IntrinsicInst.h"
@@ -52,8 +52,8 @@ static bool AreEquivalentAddressValues(const Value *A, const Value *B) {
 /// bitcasts to get back to the underlying object being addressed, keeping
 /// track of the offset in bytes from the GEPs relative to the result.
 /// This is closely related to GetUnderlyingObject but is located
-/// here to avoid making VMCore depend on TargetData.
-static Value *getUnderlyingObjectWithOffset(Value *V, const TargetData *TD,
+/// here to avoid making VMCore depend on DataLayout.
+static Value *getUnderlyingObjectWithOffset(Value *V, const DataLayout *TD,
                                             uint64_t &ByteOffset,
                                             unsigned MaxLookup = 6) {
   if (!V->getType()->isPointerTy())
@@ -85,7 +85,7 @@ static Value *getUnderlyingObjectWithOffset(Value *V, const TargetData *TD,
 /// specified pointer, we do a quick local scan of the basic block containing
 /// ScanFrom, to determine if the address is already accessed.
 bool llvm::isSafeToLoadUnconditionally(Value *V, Instruction *ScanFrom,
-                                       unsigned Align, const TargetData *TD) {
+                                       unsigned Align, const DataLayout *TD) {
   uint64_t ByteOffset = 0;
   Value *Base = V;
   if (TD)
diff --git a/lib/Analysis/LoopDependenceAnalysis.cpp b/lib/Analysis/LoopDependenceAnalysis.cpp
deleted file mode 100644
index 463269d..0000000
--- a/lib/Analysis/LoopDependenceAnalysis.cpp
+++ /dev/null
@@ -1,362 +0,0 @@
-//===- LoopDependenceAnalysis.cpp - LDA Implementation ----------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This is the (beginning) of an implementation of a loop dependence analysis
-// framework, which is used to detect dependences in memory accesses in loops.
-//
-// Please note that this is work in progress and the interface is subject to
-// change.
-//
-// TODO: adapt as implementation progresses.
-//
-// TODO: document lingo (pair, subscript, index)
-//
-//===----------------------------------------------------------------------===//
-
-#define DEBUG_TYPE "lda"
-#include "llvm/ADT/DenseSet.h"
-#include "llvm/ADT/Statistic.h"
-#include "llvm/Analysis/AliasAnalysis.h"
-#include "llvm/Analysis/LoopDependenceAnalysis.h"
-#include "llvm/Analysis/LoopPass.h"
-#include "llvm/Analysis/ScalarEvolution.h"
-#include "llvm/Analysis/ScalarEvolutionExpressions.h"
-#include "llvm/Analysis/ValueTracking.h"
-#include "llvm/Assembly/Writer.h"
-#include "llvm/Instructions.h"
-#include "llvm/Operator.h"
-#include "llvm/Support/Allocator.h"
-#include "llvm/Support/Debug.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/raw_ostream.h"
-#include "llvm/Target/TargetData.h"
-using namespace llvm;
-
-STATISTIC(NumAnswered,    "Number of dependence queries answered");
-STATISTIC(NumAnalysed,    "Number of distinct dependence pairs analysed");
-STATISTIC(NumDependent,   "Number of pairs with dependent accesses");
-STATISTIC(NumIndependent, "Number of pairs with independent accesses");
-STATISTIC(NumUnknown,     "Number of pairs with unknown accesses");
-
-LoopPass *llvm::createLoopDependenceAnalysisPass() {
-  return new LoopDependenceAnalysis();
-}
-
-INITIALIZE_PASS_BEGIN(LoopDependenceAnalysis, "lda",
-                "Loop Dependence Analysis", false, true)
-INITIALIZE_PASS_DEPENDENCY(ScalarEvolution)
-INITIALIZE_AG_DEPENDENCY(AliasAnalysis)
-INITIALIZE_PASS_END(LoopDependenceAnalysis, "lda",
-                "Loop Dependence Analysis", false, true)
-char LoopDependenceAnalysis::ID = 0;
-
-//===----------------------------------------------------------------------===//
-//                             Utility Functions
-//===----------------------------------------------------------------------===//
-
-static inline bool IsMemRefInstr(const Value *V) {
-  const Instruction *I = dyn_cast<const Instruction>(V);
-  return I && (I->mayReadFromMemory() || I->mayWriteToMemory());
-}
-
-static void GetMemRefInstrs(const Loop *L,
-                            SmallVectorImpl<Instruction*> &Memrefs) {
-  for (Loop::block_iterator b = L->block_begin(), be = L->block_end();
-       b != be; ++b)
-    for (BasicBlock::iterator i = (*b)->begin(), ie = (*b)->end();
-         i != ie; ++i)
-      if (IsMemRefInstr(i))
-        Memrefs.push_back(i);
-}
-
-static bool IsLoadOrStoreInst(Value *I) {
-  // Returns true if the load or store can be analyzed. Atomic and volatile
-  // operations have properties which this analysis does not understand.
-  if (LoadInst *LI = dyn_cast<LoadInst>(I))
-    return LI->isUnordered();
-  else if (StoreInst *SI = dyn_cast<StoreInst>(I))
-    return SI->isUnordered();
-  return false;
-}
-
-static Value *GetPointerOperand(Value *I) {
-  if (LoadInst *i = dyn_cast<LoadInst>(I))
-    return i->getPointerOperand();
-  if (StoreInst *i = dyn_cast<StoreInst>(I))
-    return i->getPointerOperand();
-  llvm_unreachable("Value is no load or store instruction!");
-}
-
-static AliasAnalysis::AliasResult UnderlyingObjectsAlias(AliasAnalysis *AA,
-                                                         const Value *A,
-                                                         const Value *B) {
-  const Value *aObj = GetUnderlyingObject(A);
-  const Value *bObj = GetUnderlyingObject(B);
-  return AA->alias(aObj, AA->getTypeStoreSize(aObj->getType()),
-                   bObj, AA->getTypeStoreSize(bObj->getType()));
-}
-
-static inline const SCEV *GetZeroSCEV(ScalarEvolution *SE) {
-  return SE->getConstant(Type::getInt32Ty(SE->getContext()), 0L);
-}
-
-//===----------------------------------------------------------------------===//
-//                             Dependence Testing
-//===----------------------------------------------------------------------===//
-
-bool LoopDependenceAnalysis::isDependencePair(const Value *A,
-                                              const Value *B) const {
-  return IsMemRefInstr(A) &&
-         IsMemRefInstr(B) &&
-         (cast<const Instruction>(A)->mayWriteToMemory() ||
-          cast<const Instruction>(B)->mayWriteToMemory());
-}
-
-bool LoopDependenceAnalysis::findOrInsertDependencePair(Value *A,
-                                                        Value *B,
-                                                        DependencePair *&P) {
-  void *insertPos = 0;
-  FoldingSetNodeID id;
-  id.AddPointer(A);
-  id.AddPointer(B);
-
-  P = Pairs.FindNodeOrInsertPos(id, insertPos);
-  if (P) return true;
-
-  P = new (PairAllocator) DependencePair(id, A, B);
-  Pairs.InsertNode(P, insertPos);
-  return false;
-}
-
-void LoopDependenceAnalysis::getLoops(const SCEV *S,
-                                      DenseSet<const Loop*>* Loops) const {
-  // Refactor this into an SCEVVisitor, if efficiency becomes a concern.
-  for (const Loop *L = this->L; L != 0; L = L->getParentLoop())
-    if (!SE->isLoopInvariant(S, L))
-      Loops->insert(L);
-}
-
-bool LoopDependenceAnalysis::isLoopInvariant(const SCEV *S) const {
-  DenseSet<const Loop*> loops;
-  getLoops(S, &loops);
-  return loops.empty();
-}
-
-bool LoopDependenceAnalysis::isAffine(const SCEV *S) const {
-  const SCEVAddRecExpr *rec = dyn_cast<SCEVAddRecExpr>(S);
-  return isLoopInvariant(S) || (rec && rec->isAffine());
-}
-
-bool LoopDependenceAnalysis::isZIVPair(const SCEV *A, const SCEV *B) const {
-  return isLoopInvariant(A) && isLoopInvariant(B);
-}
-
-bool LoopDependenceAnalysis::isSIVPair(const SCEV *A, const SCEV *B) const {
-  DenseSet<const Loop*> loops;
-  getLoops(A, &loops);
-  getLoops(B, &loops);
-  return loops.size() == 1;
-}
-
-LoopDependenceAnalysis::DependenceResult
-LoopDependenceAnalysis::analyseZIV(const SCEV *A,
-                                   const SCEV *B,
-                                   Subscript *S) const {
-  assert(isZIVPair(A, B) && "Attempted to ZIV-test non-ZIV SCEVs!");
-  return A == B ? Dependent : Independent;
-}
-
-LoopDependenceAnalysis::DependenceResult
-LoopDependenceAnalysis::analyseSIV(const SCEV *A,
-                                   const SCEV *B,
-                                   Subscript *S) const {
-  return Unknown; // TODO: Implement.
-}
-
-LoopDependenceAnalysis::DependenceResult
-LoopDependenceAnalysis::analyseMIV(const SCEV *A,
-                                   const SCEV *B,
-                                   Subscript *S) const {
-  return Unknown; // TODO: Implement.
-}
-
-LoopDependenceAnalysis::DependenceResult
-LoopDependenceAnalysis::analyseSubscript(const SCEV *A,
-                                         const SCEV *B,
-                                         Subscript *S) const {
-  DEBUG(dbgs() << "  Testing subscript: " << *A << ", " << *B << "\n");
-
-  if (A == B) {
-    DEBUG(dbgs() << "  -> [D] same SCEV\n");
-    return Dependent;
-  }
-
-  if (!isAffine(A) || !isAffine(B)) {
-    DEBUG(dbgs() << "  -> [?] not affine\n");
-    return Unknown;
-  }
-
-  if (isZIVPair(A, B))
-    return analyseZIV(A, B, S);
-
-  if (isSIVPair(A, B))
-    return analyseSIV(A, B, S);
-
-  return analyseMIV(A, B, S);
-}
-
-LoopDependenceAnalysis::DependenceResult
-LoopDependenceAnalysis::analysePair(DependencePair *P) const {
-  DEBUG(dbgs() << "Analysing:\n" << *P->A << "\n" << *P->B << "\n");
-
-  // We only analyse loads and stores but no possible memory accesses by e.g.
-  // free, call, or invoke instructions.
-  if (!IsLoadOrStoreInst(P->A) || !IsLoadOrStoreInst(P->B)) {
-    DEBUG(dbgs() << "--> [?] no load/store\n");
-    return Unknown;
-  }
-
-  Value *aPtr = GetPointerOperand(P->A);
-  Value *bPtr = GetPointerOperand(P->B);
-
-  switch (UnderlyingObjectsAlias(AA, aPtr, bPtr)) {
-  case AliasAnalysis::MayAlias:
-  case AliasAnalysis::PartialAlias:
-    // We can not analyse objects if we do not know about their aliasing.
-    DEBUG(dbgs() << "---> [?] may alias\n");
-    return Unknown;
-
-  case AliasAnalysis::NoAlias:
-    // If the objects noalias, they are distinct, accesses are independent.
-    DEBUG(dbgs() << "---> [I] no alias\n");
-    return Independent;
-
-  case AliasAnalysis::MustAlias:
-    break; // The underlying objects alias, test accesses for dependence.
-  }
-
-  const GEPOperator *aGEP = dyn_cast<GEPOperator>(aPtr);
-  const GEPOperator *bGEP = dyn_cast<GEPOperator>(bPtr);
-
-  if (!aGEP || !bGEP)
-    return Unknown;
-
-  // FIXME: Is filtering coupled subscripts necessary?
-
-  // Collect GEP operand pairs (FIXME: use GetGEPOperands from BasicAA), adding
-  // trailing zeroes to the smaller GEP, if needed.
-  typedef SmallVector<std::pair<const SCEV*, const SCEV*>, 4> GEPOpdPairsTy;
-  GEPOpdPairsTy opds;
-  for(GEPOperator::const_op_iterator aIdx = aGEP->idx_begin(),
-                                     aEnd = aGEP->idx_end(),
-                                     bIdx = bGEP->idx_begin(),
-                                     bEnd = bGEP->idx_end();
-      aIdx != aEnd && bIdx != bEnd;
-      aIdx += (aIdx != aEnd), bIdx += (bIdx != bEnd)) {
-    const SCEV* aSCEV = (aIdx != aEnd) ? SE->getSCEV(*aIdx) : GetZeroSCEV(SE);
-    const SCEV* bSCEV = (bIdx != bEnd) ? SE->getSCEV(*bIdx) : GetZeroSCEV(SE);
-    opds.push_back(std::make_pair(aSCEV, bSCEV));
-  }
-
-  if (!opds.empty() && opds[0].first != opds[0].second) {
-    // We cannot (yet) handle arbitrary GEP pointer offsets. By limiting
-    //
-    // TODO: this could be relaxed by adding the size of the underlying object
-    // to the first subscript. If we have e.g. (GEP x,0,i; GEP x,2,-i) and we
-    // know that x is a [100 x i8]*, we could modify the first subscript to be
-    // (i, 200-i) instead of (i, -i).
-    return Unknown;
-  }
-
-  // Now analyse the collected operand pairs (skipping the GEP ptr offsets).
-  for (GEPOpdPairsTy::const_iterator i = opds.begin() + 1, end = opds.end();
-       i != end; ++i) {
-    Subscript subscript;
-    DependenceResult result = analyseSubscript(i->first, i->second, &subscript);
-    if (result != Dependent) {
-      // We either proved independence or failed to analyse this subscript.
-      // Further subscripts will not improve the situation, so abort early.
-      return result;
-    }
-    P->Subscripts.push_back(subscript);
-  }
-  // We successfully analysed all subscripts but failed to prove independence.
-  return Dependent;
-}
-
-bool LoopDependenceAnalysis::depends(Value *A, Value *B) {
-  assert(isDependencePair(A, B) && "Values form no dependence pair!");
-  ++NumAnswered;
-
-  DependencePair *p;
-  if (!findOrInsertDependencePair(A, B, p)) {
-    // The pair is not cached, so analyse it.
-    ++NumAnalysed;
-    switch (p->Result = analysePair(p)) {
-    case Dependent:   ++NumDependent;   break;
-    case Independent: ++NumIndependent; break;
-    case Unknown:     ++NumUnknown;     break;
-    }
-  }
-  return p->Result != Independent;
-}
-
-//===----------------------------------------------------------------------===//
-//                   LoopDependenceAnalysis Implementation
-//===----------------------------------------------------------------------===//
-
-bool LoopDependenceAnalysis::runOnLoop(Loop *L, LPPassManager &) {
-  this->L = L;
-  AA = &getAnalysis<AliasAnalysis>();
-  SE = &getAnalysis<ScalarEvolution>();
-  return false;
-}
-
-void LoopDependenceAnalysis::releaseMemory() {
-  Pairs.clear();
-  PairAllocator.Reset();
-}
-
-void LoopDependenceAnalysis::getAnalysisUsage(AnalysisUsage &AU) const {
-  AU.setPreservesAll();
-  AU.addRequiredTransitive<AliasAnalysis>();
-  AU.addRequiredTransitive<ScalarEvolution>();
-}
-
-static void PrintLoopInfo(raw_ostream &OS,
-                          LoopDependenceAnalysis *LDA, const Loop *L) {
-  if (!L->empty()) return; // ignore non-innermost loops
-
-  SmallVector<Instruction*, 8> memrefs;
-  GetMemRefInstrs(L, memrefs);
-
-  OS << "Loop at depth " << L->getLoopDepth() << ", header block: ";
-  WriteAsOperand(OS, L->getHeader(), false);
-  OS << "\n";
-
-  OS << "  Load/store instructions: " << memrefs.size() << "\n";
-  for (SmallVector<Instruction*, 8>::const_iterator x = memrefs.begin(),
-       end = memrefs.end(); x != end; ++x)
-    OS << "\t" << (x - memrefs.begin()) << ": " << **x << "\n";
-
-  OS << "  Pairwise dependence results:\n";
-  for (SmallVector<Instruction*, 8>::const_iterator x = memrefs.begin(),
-       end = memrefs.end(); x != end; ++x)
-    for (SmallVector<Instruction*, 8>::const_iterator y = x + 1;
-         y != end; ++y)
-      if (LDA->isDependencePair(*x, *y))
-        OS << "\t" << (x - memrefs.begin()) << "," << (y - memrefs.begin())
-           << ": " << (LDA->depends(*x, *y) ? "dependent" : "independent")
-           << "\n";
-}
-
-void LoopDependenceAnalysis::print(raw_ostream &OS, const Module*) const {
-  // TODO: doc why const_cast is safe
-  PrintLoopInfo(OS, const_cast<LoopDependenceAnalysis*>(this), this->L);
-}
diff --git a/lib/Analysis/LoopInfo.cpp b/lib/Analysis/LoopInfo.cpp
index 20c33a3..8341f9d 100644
--- a/lib/Analysis/LoopInfo.cpp
+++ b/lib/Analysis/LoopInfo.cpp
@@ -306,9 +306,11 @@ BasicBlock *Loop::getUniqueExitBlock() const {
   return 0;
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void Loop::dump() const {
   print(dbgs());
 }
+#endif
 
 //===----------------------------------------------------------------------===//
 // UnloopUpdater implementation
@@ -429,8 +431,8 @@ void UnloopUpdater::updateSubloopParents() {
     Unloop->removeChildLoop(llvm::prior(Unloop->end()));
 
     assert(SubloopParents.count(Subloop) && "DFS failed to visit subloop");
-    if (SubloopParents[Subloop])
-      SubloopParents[Subloop]->addChildLoop(Subloop);
+    if (Loop *Parent = SubloopParents[Subloop])
+      Parent->addChildLoop(Subloop);
     else
       LI->addTopLevelLoop(Subloop);
   }
@@ -456,9 +458,8 @@ Loop *UnloopUpdater::getNearestLoop(BasicBlock *BB, Loop *BBLoop) {
       assert(Subloop && "subloop is not an ancestor of the original loop");
     }
     // Get the current nearest parent of the Subloop exits, initially Unloop.
-    if (!SubloopParents.count(Subloop))
-      SubloopParents[Subloop] = Unloop;
-    NearLoop = SubloopParents[Subloop];
+    NearLoop =
+      SubloopParents.insert(std::make_pair(Subloop, Unloop)).first->second;
   }
 
   succ_iterator I = succ_begin(BB), E = succ_end(BB);
diff --git a/lib/Analysis/MemoryBuiltins.cpp b/lib/Analysis/MemoryBuiltins.cpp
index e77d2ff..0a539fe 100644
--- a/lib/Analysis/MemoryBuiltins.cpp
+++ b/lib/Analysis/MemoryBuiltins.cpp
@@ -25,7 +25,8 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
+#include "llvm/Target/TargetLibraryInfo.h"
 #include "llvm/Transforms/Utils/Local.h"
 using namespace llvm;
 
@@ -39,7 +40,7 @@ enum AllocType {
 };
 
 struct AllocFnsTy {
-  const char *Name;
+  LibFunc::Func Func;
   AllocType AllocTy;
   unsigned char NumParams;
   // First and Second size parameters (or -1 if unused)
@@ -49,22 +50,22 @@ struct AllocFnsTy {
 // FIXME: certain users need more information. E.g., SimplifyLibCalls needs to
 // know which functions are nounwind, noalias, nocapture parameters, etc.
 static const AllocFnsTy AllocationFnData[] = {
-  {"malloc",              MallocLike,  1, 0,  -1},
-  {"valloc",              MallocLike,  1, 0,  -1},
-  {"_Znwj",               MallocLike,  1, 0,  -1}, // new(unsigned int)
-  {"_ZnwjRKSt9nothrow_t", MallocLike,  2, 0,  -1}, // new(unsigned int, nothrow)
-  {"_Znwm",               MallocLike,  1, 0,  -1}, // new(unsigned long)
-  {"_ZnwmRKSt9nothrow_t", MallocLike,  2, 0,  -1}, // new(unsigned long, nothrow)
-  {"_Znaj",               MallocLike,  1, 0,  -1}, // new[](unsigned int)
-  {"_ZnajRKSt9nothrow_t", MallocLike,  2, 0,  -1}, // new[](unsigned int, nothrow)
-  {"_Znam",               MallocLike,  1, 0,  -1}, // new[](unsigned long)
-  {"_ZnamRKSt9nothrow_t", MallocLike,  2, 0,  -1}, // new[](unsigned long, nothrow)
-  {"posix_memalign",      MallocLike,  3, 2,  -1},
-  {"calloc",              CallocLike,  2, 0,  1},
-  {"realloc",             ReallocLike, 2, 1,  -1},
-  {"reallocf",            ReallocLike, 2, 1,  -1},
-  {"strdup",              StrDupLike,  1, -1, -1},
-  {"strndup",             StrDupLike,  2, 1,  -1}
+  {LibFunc::malloc,              MallocLike,  1, 0,  -1},
+  {LibFunc::valloc,              MallocLike,  1, 0,  -1},
+  {LibFunc::Znwj,                MallocLike,  1, 0,  -1}, // new(unsigned int)
+  {LibFunc::ZnwjRKSt9nothrow_t,  MallocLike,  2, 0,  -1}, // new(unsigned int, nothrow)
+  {LibFunc::Znwm,                MallocLike,  1, 0,  -1}, // new(unsigned long)
+  {LibFunc::ZnwmRKSt9nothrow_t,  MallocLike,  2, 0,  -1}, // new(unsigned long, nothrow)
+  {LibFunc::Znaj,                MallocLike,  1, 0,  -1}, // new[](unsigned int)
+  {LibFunc::ZnajRKSt9nothrow_t,  MallocLike,  2, 0,  -1}, // new[](unsigned int, nothrow)
+  {LibFunc::Znam,                MallocLike,  1, 0,  -1}, // new[](unsigned long)
+  {LibFunc::ZnamRKSt9nothrow_t,  MallocLike,  2, 0,  -1}, // new[](unsigned long, nothrow)
+  {LibFunc::posix_memalign,      MallocLike,  3, 2,  -1},
+  {LibFunc::calloc,              CallocLike,  2, 0,   1},
+  {LibFunc::realloc,             ReallocLike, 2, 1,  -1},
+  {LibFunc::reallocf,            ReallocLike, 2, 1,  -1},
+  {LibFunc::strdup,              StrDupLike,  1, -1, -1},
+  {LibFunc::strndup,             StrDupLike,  2, 1,  -1}
 };
 
 
@@ -85,15 +86,22 @@ static Function *getCalledFunction(const Value *V, bool LookThroughBitCast) {
 /// \brief Returns the allocation data for the given value if it is a call to a
 /// known allocation function, and NULL otherwise.
 static const AllocFnsTy *getAllocationData(const Value *V, AllocType AllocTy,
+                                           const TargetLibraryInfo *TLI,
                                            bool LookThroughBitCast = false) {
   Function *Callee = getCalledFunction(V, LookThroughBitCast);
   if (!Callee)
     return 0;
 
+  // Make sure that the function is available.
+  StringRef FnName = Callee->getName();
+  LibFunc::Func TLIFn;
+  if (!TLI || !TLI->getLibFunc(FnName, TLIFn) || !TLI->has(TLIFn))
+    return 0;
+
   unsigned i = 0;
   bool found = false;
   for ( ; i < array_lengthof(AllocationFnData); ++i) {
-    if (Callee->getName() == AllocationFnData[i].Name) {
+    if (AllocationFnData[i].Func == TLIFn) {
       found = true;
       break;
     }
@@ -106,7 +114,6 @@ static const AllocFnsTy *getAllocationData(const Value *V, AllocType AllocTy,
     return 0;
 
   // Check function prototype.
-  // FIXME: Check the nobuiltin metadata?? (PR5130)
   int FstParam = FnData->FstParam;
   int SndParam = FnData->SndParam;
   FunctionType *FTy = Callee->getFunctionType();
@@ -125,64 +132,72 @@ static const AllocFnsTy *getAllocationData(const Value *V, AllocType AllocTy,
 
 static bool hasNoAliasAttr(const Value *V, bool LookThroughBitCast) {
   ImmutableCallSite CS(LookThroughBitCast ? V->stripPointerCasts() : V);
-  return CS && CS.hasFnAttr(Attribute::NoAlias);
+  return CS && CS.hasFnAttr(Attributes::NoAlias);
 }
 
 
 /// \brief Tests if a value is a call or invoke to a library function that
 /// allocates or reallocates memory (either malloc, calloc, realloc, or strdup
 /// like).
-bool llvm::isAllocationFn(const Value *V, bool LookThroughBitCast) {
-  return getAllocationData(V, AnyAlloc, LookThroughBitCast);
+bool llvm::isAllocationFn(const Value *V, const TargetLibraryInfo *TLI,
+                          bool LookThroughBitCast) {
+  return getAllocationData(V, AnyAlloc, TLI, LookThroughBitCast);
 }
 
 /// \brief Tests if a value is a call or invoke to a function that returns a
 /// NoAlias pointer (including malloc/calloc/realloc/strdup-like functions).
-bool llvm::isNoAliasFn(const Value *V, bool LookThroughBitCast) {
+bool llvm::isNoAliasFn(const Value *V, const TargetLibraryInfo *TLI,
+                       bool LookThroughBitCast) {
   // it's safe to consider realloc as noalias since accessing the original
   // pointer is undefined behavior
-  return isAllocationFn(V, LookThroughBitCast) ||
+  return isAllocationFn(V, TLI, LookThroughBitCast) ||
          hasNoAliasAttr(V, LookThroughBitCast);
 }
 
 /// \brief Tests if a value is a call or invoke to a library function that
 /// allocates uninitialized memory (such as malloc).
-bool llvm::isMallocLikeFn(const Value *V, bool LookThroughBitCast) {
-  return getAllocationData(V, MallocLike, LookThroughBitCast);
+bool llvm::isMallocLikeFn(const Value *V, const TargetLibraryInfo *TLI,
+                          bool LookThroughBitCast) {
+  return getAllocationData(V, MallocLike, TLI, LookThroughBitCast);
 }
 
 /// \brief Tests if a value is a call or invoke to a library function that
 /// allocates zero-filled memory (such as calloc).
-bool llvm::isCallocLikeFn(const Value *V, bool LookThroughBitCast) {
-  return getAllocationData(V, CallocLike, LookThroughBitCast);
+bool llvm::isCallocLikeFn(const Value *V, const TargetLibraryInfo *TLI,
+                          bool LookThroughBitCast) {
+  return getAllocationData(V, CallocLike, TLI, LookThroughBitCast);
 }
 
 /// \brief Tests if a value is a call or invoke to a library function that
 /// allocates memory (either malloc, calloc, or strdup like).
-bool llvm::isAllocLikeFn(const Value *V, bool LookThroughBitCast) {
-  return getAllocationData(V, AllocLike, LookThroughBitCast);
+bool llvm::isAllocLikeFn(const Value *V, const TargetLibraryInfo *TLI,
+                         bool LookThroughBitCast) {
+  return getAllocationData(V, AllocLike, TLI, LookThroughBitCast);
 }
 
 /// \brief Tests if a value is a call or invoke to a library function that
 /// reallocates memory (such as realloc).
-bool llvm::isReallocLikeFn(const Value *V, bool LookThroughBitCast) {
-  return getAllocationData(V, ReallocLike, LookThroughBitCast);
+bool llvm::isReallocLikeFn(const Value *V, const TargetLibraryInfo *TLI,
+                           bool LookThroughBitCast) {
+  return getAllocationData(V, ReallocLike, TLI, LookThroughBitCast);
 }
 
 /// extractMallocCall - Returns the corresponding CallInst if the instruction
 /// is a malloc call.  Since CallInst::CreateMalloc() only creates calls, we
 /// ignore InvokeInst here.
-const CallInst *llvm::extractMallocCall(const Value *I) {
-  return isMallocLikeFn(I) ? dyn_cast<CallInst>(I) : 0;
+const CallInst *llvm::extractMallocCall(const Value *I,
+                                        const TargetLibraryInfo *TLI) {
+  return isMallocLikeFn(I, TLI) ? dyn_cast<CallInst>(I) : 0;
 }
 
-static Value *computeArraySize(const CallInst *CI, const TargetData *TD,
+static Value *computeArraySize(const CallInst *CI, const DataLayout *TD,
+                               const TargetLibraryInfo *TLI,
                                bool LookThroughSExt = false) {
   if (!CI)
     return NULL;
 
   // The size of the malloc's result type must be known to determine array size.
-  Type *T = getMallocAllocatedType(CI);
+  Type *T = getMallocAllocatedType(CI, TLI);
   if (!T || !T->isSized() || !TD)
     return NULL;
 
@@ -204,9 +219,11 @@ static Value *computeArraySize(const CallInst *CI, const TargetData *TD,
 /// isArrayMalloc - Returns the corresponding CallInst if the instruction 
 /// is a call to malloc whose array size can be determined and the array size
 /// is not constant 1.  Otherwise, return NULL.
-const CallInst *llvm::isArrayMalloc(const Value *I, const TargetData *TD) {
-  const CallInst *CI = extractMallocCall(I);
-  Value *ArraySize = computeArraySize(CI, TD);
+const CallInst *llvm::isArrayMalloc(const Value *I,
+                                    const DataLayout *TD,
+                                    const TargetLibraryInfo *TLI) {
+  const CallInst *CI = extractMallocCall(I, TLI);
+  Value *ArraySize = computeArraySize(CI, TD, TLI);
 
   if (ArraySize &&
       ArraySize != ConstantInt::get(CI->getArgOperand(0)->getType(), 1))
@@ -221,8 +238,9 @@ const CallInst *llvm::isArrayMalloc(const Value *I, const TargetData *TD) {
 ///   0: PointerType is the calls' return type.
 ///   1: PointerType is the bitcast's result type.
 ///  >1: Unique PointerType cannot be determined, return NULL.
-PointerType *llvm::getMallocType(const CallInst *CI) {
-  assert(isMallocLikeFn(CI) && "getMallocType and not malloc call");
+PointerType *llvm::getMallocType(const CallInst *CI,
+                                 const TargetLibraryInfo *TLI) {
+  assert(isMallocLikeFn(CI, TLI) && "getMallocType and not malloc call");
   
   PointerType *MallocType = NULL;
   unsigned NumOfBitCastUses = 0;
@@ -252,8 +270,9 @@ PointerType *llvm::getMallocType(const CallInst *CI) {
 ///   0: PointerType is the malloc calls' return type.
 ///   1: PointerType is the bitcast's result type.
 ///  >1: Unique PointerType cannot be determined, return NULL.
-Type *llvm::getMallocAllocatedType(const CallInst *CI) {
-  PointerType *PT = getMallocType(CI);
+Type *llvm::getMallocAllocatedType(const CallInst *CI,
+                                   const TargetLibraryInfo *TLI) {
+  PointerType *PT = getMallocType(CI, TLI);
   return PT ? PT->getElementType() : NULL;
 }
 
@@ -262,22 +281,24 @@ Type *llvm::getMallocAllocatedType(const CallInst *CI) {
 /// then return that multiple.  For non-array mallocs, the multiple is
 /// constant 1.  Otherwise, return NULL for mallocs whose array size cannot be
 /// determined.
-Value *llvm::getMallocArraySize(CallInst *CI, const TargetData *TD,
+Value *llvm::getMallocArraySize(CallInst *CI, const DataLayout *TD,
+                                const TargetLibraryInfo *TLI,
                                 bool LookThroughSExt) {
-  assert(isMallocLikeFn(CI) && "getMallocArraySize and not malloc call");
-  return computeArraySize(CI, TD, LookThroughSExt);
+  assert(isMallocLikeFn(CI, TLI) && "getMallocArraySize and not malloc call");
+  return computeArraySize(CI, TD, TLI, LookThroughSExt);
 }
 
 
 /// extractCallocCall - Returns the corresponding CallInst if the instruction
 /// is a calloc call.
-const CallInst *llvm::extractCallocCall(const Value *I) {
-  return isCallocLikeFn(I) ? cast<CallInst>(I) : 0;
+const CallInst *llvm::extractCallocCall(const Value *I,
+                                        const TargetLibraryInfo *TLI) {
+  return isCallocLikeFn(I, TLI) ? cast<CallInst>(I) : 0;
 }
 
 
 /// isFreeCall - Returns non-null if the value is a call to the builtin free()
-const CallInst *llvm::isFreeCall(const Value *I) {
+const CallInst *llvm::isFreeCall(const Value *I, const TargetLibraryInfo *TLI) {
   const CallInst *CI = dyn_cast<CallInst>(I);
   if (!CI)
     return 0;
@@ -285,9 +306,14 @@ const CallInst *llvm::isFreeCall(const Value *I) {
   if (Callee == 0 || !Callee->isDeclaration())
     return 0;
 
-  if (Callee->getName() != "free" &&
-      Callee->getName() != "_ZdlPv" && // operator delete(void*)
-      Callee->getName() != "_ZdaPv")   // operator delete[](void*)
+  StringRef FnName = Callee->getName();
+  LibFunc::Func TLIFn;
+  if (!TLI || !TLI->getLibFunc(FnName, TLIFn) || !TLI->has(TLIFn))
+    return 0;
+
+  if (TLIFn != LibFunc::free &&
+      TLIFn != LibFunc::ZdlPv && // operator delete(void*)
+      TLIFn != LibFunc::ZdaPv)   // operator delete[](void*)
     return 0;
 
   // Check free prototype.
@@ -315,12 +341,12 @@ const CallInst *llvm::isFreeCall(const Value *I) {
 /// object size in Size if successful, and false otherwise.
 /// If RoundToAlign is true, then Size is rounded up to the aligment of allocas,
 /// byval arguments, and global variables.
-bool llvm::getObjectSize(const Value *Ptr, uint64_t &Size, const TargetData *TD,
-                         bool RoundToAlign) {
+bool llvm::getObjectSize(const Value *Ptr, uint64_t &Size, const DataLayout *TD,
+                         const TargetLibraryInfo *TLI, bool RoundToAlign) {
   if (!TD)
     return false;
 
-  ObjectSizeOffsetVisitor Visitor(TD, Ptr->getContext(), RoundToAlign);
+  ObjectSizeOffsetVisitor Visitor(TD, TLI, Ptr->getContext(), RoundToAlign);
   SizeOffsetType Data = Visitor.compute(const_cast<Value*>(Ptr));
   if (!Visitor.bothKnown(Data))
     return false;
@@ -347,10 +373,11 @@ APInt ObjectSizeOffsetVisitor::align(APInt Size, uint64_t Align) {
   return Size;
 }
 
-ObjectSizeOffsetVisitor::ObjectSizeOffsetVisitor(const TargetData *TD,
+ObjectSizeOffsetVisitor::ObjectSizeOffsetVisitor(const DataLayout *TD,
+                                                 const TargetLibraryInfo *TLI,
                                                  LLVMContext &Context,
                                                  bool RoundToAlign)
-: TD(TD), RoundToAlign(RoundToAlign) {
+: TD(TD), TLI(TLI), RoundToAlign(RoundToAlign) {
   IntegerType *IntTy = TD->getIntPtrType(Context);
   IntTyBits = IntTy->getBitWidth();
   Zero = APInt::getNullValue(IntTyBits);
@@ -358,11 +385,16 @@ ObjectSizeOffsetVisitor::ObjectSizeOffsetVisitor(const TargetData *TD,
 
 SizeOffsetType ObjectSizeOffsetVisitor::compute(Value *V) {
   V = V->stripPointerCasts();
+  if (Instruction *I = dyn_cast<Instruction>(V)) {
+    // If we have already seen this instruction, bail out. Cycles can happen in
+    // unreachable code after constant propagation.
+    if (!SeenInsts.insert(I))
+      return unknown();
 
-  if (GEPOperator *GEP = dyn_cast<GEPOperator>(V))
-    return visitGEPOperator(*GEP);
-  if (Instruction *I = dyn_cast<Instruction>(V))
+    if (GEPOperator *GEP = dyn_cast<GEPOperator>(V))
+      return visitGEPOperator(*GEP);
     return visit(*I);
+  }
   if (Argument *A = dyn_cast<Argument>(V))
     return visitArgument(*A);
   if (ConstantPointerNull *P = dyn_cast<ConstantPointerNull>(V))
@@ -371,9 +403,12 @@ SizeOffsetType ObjectSizeOffsetVisitor::compute(Value *V) {
     return visitGlobalVariable(*GV);
   if (UndefValue *UV = dyn_cast<UndefValue>(V))
     return visitUndefValue(*UV);
-  if (ConstantExpr *CE = dyn_cast<ConstantExpr>(V))
+  if (ConstantExpr *CE = dyn_cast<ConstantExpr>(V)) {
     if (CE->getOpcode() == Instruction::IntToPtr)
       return unknown(); // clueless
+    if (CE->getOpcode() == Instruction::GetElementPtr)
+      return visitGEPOperator(cast<GEPOperator>(*CE));
+  }
 
   DEBUG(dbgs() << "ObjectSizeOffsetVisitor::compute() unhandled value: " << *V
         << '\n');
@@ -408,7 +443,8 @@ SizeOffsetType ObjectSizeOffsetVisitor::visitArgument(Argument &A) {
 }
 
 SizeOffsetType ObjectSizeOffsetVisitor::visitCallSite(CallSite CS) {
-  const AllocFnsTy *FnData = getAllocationData(CS.getInstruction(), AnyAlloc);
+  const AllocFnsTy *FnData = getAllocationData(CS.getInstruction(), AnyAlloc,
+                                               TLI);
   if (!FnData)
     return unknown();
 
@@ -473,10 +509,6 @@ ObjectSizeOffsetVisitor::visitExtractValueInst(ExtractValueInst&) {
 }
 
 SizeOffsetType ObjectSizeOffsetVisitor::visitGEPOperator(GEPOperator &GEP) {
-  // Ignore self-referencing GEPs, they can occur in unreachable code.
-  if (&GEP == GEP.getPointerOperand())
-    return unknown();
-
   SizeOffsetType PtrData = compute(GEP.getPointerOperand());
   if (!bothKnown(PtrData) || !GEP.hasAllConstantIndices())
     return unknown();
@@ -510,10 +542,6 @@ SizeOffsetType ObjectSizeOffsetVisitor::visitPHINode(PHINode&) {
 }
 
 SizeOffsetType ObjectSizeOffsetVisitor::visitSelectInst(SelectInst &I) {
-  // ignore malformed self-looping selects
-  if (I.getTrueValue() == &I || I.getFalseValue() == &I)
-    return unknown();
-
   SizeOffsetType TrueSide  = compute(I.getTrueValue());
   SizeOffsetType FalseSide = compute(I.getFalseValue());
   if (bothKnown(TrueSide) && bothKnown(FalseSide) && TrueSide == FalseSide)
@@ -531,10 +559,10 @@ SizeOffsetType ObjectSizeOffsetVisitor::visitInstruction(Instruction &I) {
 }
 
 
-ObjectSizeOffsetEvaluator::ObjectSizeOffsetEvaluator(const TargetData *TD,
+ObjectSizeOffsetEvaluator::ObjectSizeOffsetEvaluator(const DataLayout *TD,
+                                                   const TargetLibraryInfo *TLI,
                                                      LLVMContext &Context)
-: TD(TD), Context(Context), Builder(Context, TargetFolder(TD)),
-Visitor(TD, Context) {
+: TD(TD), TLI(TLI), Context(Context), Builder(Context, TargetFolder(TD)) {
   IntTy = TD->getIntPtrType(Context);
   Zero = ConstantInt::get(IntTy, 0);
 }
@@ -559,6 +587,7 @@ SizeOffsetEvalType ObjectSizeOffsetEvaluator::compute(Value *V) {
 }
 
 SizeOffsetEvalType ObjectSizeOffsetEvaluator::compute_(Value *V) {
+  ObjectSizeOffsetVisitor Visitor(TD, TLI, Context);
   SizeOffsetType Const = Visitor.compute(V);
   if (Visitor.bothKnown(Const))
     return std::make_pair(ConstantInt::get(Context, Const.first),
@@ -621,7 +650,8 @@ SizeOffsetEvalType ObjectSizeOffsetEvaluator::visitAllocaInst(AllocaInst &I) {
 }
 
 SizeOffsetEvalType ObjectSizeOffsetEvaluator::visitCallSite(CallSite CS) {
-  const AllocFnsTy *FnData = getAllocationData(CS.getInstruction(), AnyAlloc);
+  const AllocFnsTy *FnData = getAllocationData(CS.getInstruction(), AnyAlloc,
+                                               TLI);
   if (!FnData)
     return unknown();
 
@@ -719,10 +749,6 @@ SizeOffsetEvalType ObjectSizeOffsetEvaluator::visitPHINode(PHINode &PHI) {
 }
 
 SizeOffsetEvalType ObjectSizeOffsetEvaluator::visitSelectInst(SelectInst &I) {
-  // ignore malformed self-looping selects
-  if (I.getTrueValue() == &I || I.getFalseValue() == &I)
-    return unknown();
-
   SizeOffsetEvalType TrueSide  = compute_(I.getTrueValue());
   SizeOffsetEvalType FalseSide = compute_(I.getFalseValue());
 
diff --git a/lib/Analysis/MemoryDependenceAnalysis.cpp b/lib/Analysis/MemoryDependenceAnalysis.cpp
index 059e574..9872890 100644
--- a/lib/Analysis/MemoryDependenceAnalysis.cpp
+++ b/lib/Analysis/MemoryDependenceAnalysis.cpp
@@ -30,7 +30,7 @@
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/PredIteratorCache.h"
 #include "llvm/Support/Debug.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 using namespace llvm;
 
 STATISTIC(NumCacheNonLocal, "Number of fully cached non-local responses");
@@ -89,7 +89,7 @@ void MemoryDependenceAnalysis::getAnalysisUsage(AnalysisUsage &AU) const {
 
 bool MemoryDependenceAnalysis::runOnFunction(Function &) {
   AA = &getAnalysis<AliasAnalysis>();
-  TD = getAnalysisIfAvailable<TargetData>();
+  TD = getAnalysisIfAvailable<DataLayout>();
   DT = getAnalysisIfAvailable<DominatorTree>();
   if (PredCache == 0)
     PredCache.reset(new PredIteratorCache());
@@ -148,7 +148,7 @@ AliasAnalysis::ModRefResult GetLocation(const Instruction *Inst,
     return AliasAnalysis::ModRef;
   }
 
-  if (const CallInst *CI = isFreeCall(Inst)) {
+  if (const CallInst *CI = isFreeCall(Inst, AA->getTargetLibraryInfo())) {
     // calls to free() deallocate the entire structure
     Loc = AliasAnalysis::Location(CI->getArgOperand(0));
     return AliasAnalysis::Mod;
@@ -256,7 +256,7 @@ isLoadLoadClobberIfExtendedToFullWidth(const AliasAnalysis::Location &MemLoc,
                                        const Value *&MemLocBase,
                                        int64_t &MemLocOffs,
                                        const LoadInst *LI,
-                                       const TargetData *TD) {
+                                       const DataLayout *TD) {
   // If we have no target data, we can't do this.
   if (TD == 0) return false;
 
@@ -280,7 +280,7 @@ isLoadLoadClobberIfExtendedToFullWidth(const AliasAnalysis::Location &MemLoc,
 unsigned MemoryDependenceAnalysis::
 getLoadLoadClobberFullWidthSize(const Value *MemLocBase, int64_t MemLocOffs,
                                 unsigned MemLocSize, const LoadInst *LI,
-                                const TargetData &TD) {
+                                const DataLayout &TD) {
   // We can only extend simple integer loads.
   if (!isa<IntegerType>(LI->getType()) || !LI->isSimple()) return 0;
   
@@ -327,12 +327,12 @@ getLoadLoadClobberFullWidthSize(const Value *MemLocBase, int64_t MemLocOffs,
       return 0;
 
     if (LIOffs+NewLoadByteSize > MemLocEnd &&
-        LI->getParent()->getParent()->hasFnAttr(Attribute::AddressSafety)) {
+        LI->getParent()->getParent()->getFnAttributes().
+          hasAttribute(Attributes::AddressSafety))
       // We will be reading past the location accessed by the original program.
       // While this is safe in a regular build, Address Safety analysis tools
       // may start reporting false warnings. So, don't do widening.
       return 0;
-    }
 
     // If a load of this width would include all of MemLoc, then we succeed.
     if (LIOffs+NewLoadByteSize >= MemLocEnd)
@@ -479,12 +479,20 @@ getPointerDependencyFrom(const AliasAnalysis::Location &MemLoc, bool isLoad,
     // a subsequent bitcast of the malloc call result.  There can be stores to
     // the malloced memory between the malloc call and its bitcast uses, and we
     // need to continue scanning until the malloc call.
-    if (isa<AllocaInst>(Inst) || isNoAliasFn(Inst)) {
+    const TargetLibraryInfo *TLI = AA->getTargetLibraryInfo();
+    if (isa<AllocaInst>(Inst) || isNoAliasFn(Inst, TLI)) {
       const Value *AccessPtr = GetUnderlyingObject(MemLoc.Ptr, TD);
       
       if (AccessPtr == Inst || AA->isMustAlias(Inst, AccessPtr))
         return MemDepResult::getDef(Inst);
-      continue;
+      // Be conservative if the accessed pointer may alias the allocation.
+      if (AA->alias(Inst, AccessPtr) != AliasAnalysis::NoAlias)
+        return MemDepResult::getClobber(Inst);
+      // If the allocation is not aliased and does not read memory (like
+      // strdup), it is safe to ignore.
+      if (isa<AllocaInst>(Inst) ||
+          isMallocLikeFn(Inst, TLI) || isCallocLikeFn(Inst, TLI))
+        continue;
     }
 
     // See if this instruction (e.g. a call or vaarg) mod/ref's the pointer.
@@ -975,7 +983,7 @@ getNonLocalPointerDepFromBB(const PHITransAddr &Pointer,
     for (NonLocalDepInfo::iterator I = Cache->begin(), E = Cache->end();
          I != E; ++I) {
       Visited.insert(std::make_pair(I->getBB(), Addr));
-      if (!I->getResult().isNonLocal())
+      if (!I->getResult().isNonLocal() && DT->isReachableFromEntry(I->getBB()))
         Result.push_back(NonLocalDepResult(I->getBB(), I->getResult(), Addr));
     }
     ++NumCacheCompleteNonLocalPtr;
@@ -1021,7 +1029,7 @@ getNonLocalPointerDepFromBB(const PHITransAddr &Pointer,
                                                  NumSortedEntries);
       
       // If we got a Def or Clobber, add this to the list of results.
-      if (!Dep.isNonLocal()) {
+      if (!Dep.isNonLocal() && DT->isReachableFromEntry(BB)) {
         Result.push_back(NonLocalDepResult(BB, Dep, Pointer.getAddr()));
         continue;
       }
diff --git a/lib/Analysis/NoAliasAnalysis.cpp b/lib/Analysis/NoAliasAnalysis.cpp
index 101c2d5..2eb4137 100644
--- a/lib/Analysis/NoAliasAnalysis.cpp
+++ b/lib/Analysis/NoAliasAnalysis.cpp
@@ -15,7 +15,7 @@
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/Passes.h"
 #include "llvm/Pass.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 using namespace llvm;
 
 namespace {
@@ -36,7 +36,7 @@ namespace {
     virtual void initializePass() {
       // Note: NoAA does not call InitializeAliasAnalysis because it's
       // special and does not support chaining.
-      TD = getAnalysisIfAvailable<TargetData>();
+      TD = getAnalysisIfAvailable<DataLayout>();
     }
 
     virtual AliasResult alias(const Location &LocA, const Location &LocB) {
diff --git a/lib/Analysis/PHITransAddr.cpp b/lib/Analysis/PHITransAddr.cpp
index 38cb1c9..c35737e 100644
--- a/lib/Analysis/PHITransAddr.cpp
+++ b/lib/Analysis/PHITransAddr.cpp
@@ -41,6 +41,7 @@ static bool CanPHITrans(Instruction *Inst) {
   return false;
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void PHITransAddr::dump() const {
   if (Addr == 0) {
     dbgs() << "PHITransAddr: null\n";
@@ -50,6 +51,7 @@ void PHITransAddr::dump() const {
   for (unsigned i = 0, e = InstInputs.size(); i != e; ++i)
     dbgs() << "  Input #" << i << " is " << *InstInputs[i] << "\n";
 }
+#endif
 
 
 static bool VerifySubExpr(Value *Expr,
diff --git a/lib/Analysis/ProfileDataLoader.cpp b/lib/Analysis/ProfileDataLoader.cpp
new file mode 100644
index 0000000..a4f634a
--- /dev/null
+++ b/lib/Analysis/ProfileDataLoader.cpp
@@ -0,0 +1,155 @@
+//===- ProfileDataLoader.cpp - Load profile information from disk ---------===//
+//
+//                      The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// The ProfileDataLoader class is used to load raw profiling data from the dump
+// file.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/OwningPtr.h"
+#include "llvm/Module.h"
+#include "llvm/InstrTypes.h"
+#include "llvm/Analysis/ProfileDataLoader.h"
+#include "llvm/Analysis/ProfileDataTypes.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/system_error.h"
+#include <cstdio>
+#include <cstdlib>
+using namespace llvm;
+
+raw_ostream &llvm::operator<<(raw_ostream &O, std::pair<const BasicBlock *,
+                                                        const BasicBlock *> E) {
+  O << "(";
+
+  if (E.first)
+    O << E.first->getName();
+  else
+    O << "0";
+
+  O << ",";
+
+  if (E.second)
+    O << E.second->getName();
+  else
+    O << "0";
+
+  return O << ")";
+}
+
+/// AddCounts - Add 'A' and 'B', accounting for the fact that the value of one
+/// (or both) may not be defined.
+static unsigned AddCounts(unsigned A, unsigned B) {
+  // If either value is undefined, use the other.
+  // Undefined + undefined = undefined.
+  if (A == ProfileDataLoader::Uncounted) return B;
+  if (B == ProfileDataLoader::Uncounted) return A;
+
+  return A + B;
+}
+
+/// ReadProfilingData - Load 'NumEntries' items of type 'T' from file 'F'
+template <typename T>
+static void ReadProfilingData(const char *ToolName, FILE *F,
+                              T *Data, size_t NumEntries) {
+  // Read in the block of data...
+  if (fread(Data, sizeof(T), NumEntries, F) != NumEntries)
+    report_fatal_error(Twine(ToolName) + ": Profiling data truncated");
+}
+
+/// ReadProfilingNumEntries - Read how many entries are in this profiling data
+/// packet.
+static unsigned ReadProfilingNumEntries(const char *ToolName, FILE *F,
+                                        bool ShouldByteSwap) {
+  unsigned Entry;
+  ReadProfilingData<unsigned>(ToolName, F, &Entry, 1);
+  return ShouldByteSwap ? ByteSwap_32(Entry) : Entry;
+}
+
+/// ReadProfilingBlock - Read the number of entries in the next profiling data
+/// packet and then accumulate the entries into 'Data'.
+static void ReadProfilingBlock(const char *ToolName, FILE *F,
+                               bool ShouldByteSwap,
+                               SmallVector<unsigned, 32> &Data) {
+  // Read the number of entries...
+  unsigned NumEntries = ReadProfilingNumEntries(ToolName, F, ShouldByteSwap);
+
+  // Read in the data.
+  SmallVector<unsigned, 8> TempSpace(NumEntries);
+  ReadProfilingData<unsigned>(ToolName, F, TempSpace.data(), NumEntries);
+
+  // Make sure we have enough space ...
+  if (Data.size() < NumEntries)
+    Data.resize(NumEntries, ProfileDataLoader::Uncounted);
+
+  // Accumulate the data we just read into the existing data.
+  for (unsigned i = 0; i < NumEntries; ++i) {
+    unsigned Entry = ShouldByteSwap ? ByteSwap_32(TempSpace[i]) : TempSpace[i];
+    Data[i] = AddCounts(Entry, Data[i]);
+  }
+}
+
+/// ReadProfilingArgBlock - Read the command line arguments that the progam was
+/// run with when the current profiling data packet(s) were generated.
+static void ReadProfilingArgBlock(const char *ToolName, FILE *F,
+                                  bool ShouldByteSwap,
+                                  SmallVector<std::string, 1> &CommandLines) {
+  // Read the number of bytes ...
+  unsigned ArgLength = ReadProfilingNumEntries(ToolName, F, ShouldByteSwap);
+
+  // Read in the arguments (if there are any to read).  Round up the length to
+  // the nearest 4-byte multiple.
+  SmallVector<char, 8> Args(ArgLength+4);
+  if (ArgLength)
+    ReadProfilingData<char>(ToolName, F, Args.data(), (ArgLength+3) & ~3);
+
+  // Store the arguments.
+  CommandLines.push_back(std::string(&Args[0], &Args[ArgLength]));
+}
+
+const unsigned ProfileDataLoader::Uncounted = ~0U;
+
+/// ProfileDataLoader ctor - Read the specified profiling data file, reporting
+/// a fatal error if the file is invalid or broken.
+ProfileDataLoader::ProfileDataLoader(const char *ToolName,
+                                     const std::string &Filename)
+  : Filename(Filename) {
+  FILE *F = fopen(Filename.c_str(), "rb");
+  if (F == 0)
+    report_fatal_error(Twine(ToolName) + ": Error opening '" +
+                       Filename + "': ");
+
+  // Keep reading packets until we run out of them.
+  unsigned PacketType;
+  while (fread(&PacketType, sizeof(unsigned), 1, F) == 1) {
+    // If the low eight bits of the packet are zero, we must be dealing with an
+    // endianness mismatch.  Byteswap all words read from the profiling
+    // information.  This can happen when the compiler host and target have
+    // different endianness.
+    bool ShouldByteSwap = (char)PacketType == 0;
+    PacketType = ShouldByteSwap ? ByteSwap_32(PacketType) : PacketType;
+
+    switch (PacketType) {
+      case ArgumentInfo:
+        ReadProfilingArgBlock(ToolName, F, ShouldByteSwap, CommandLines);
+        break;
+
+      case EdgeInfo:
+        ReadProfilingBlock(ToolName, F, ShouldByteSwap, EdgeCounts);
+        break;
+
+      default:
+        report_fatal_error(std::string(ToolName)
+                           + ": Unknown profiling packet type");
+        break;
+    }
+  }
+
+  fclose(F);
+}
diff --git a/lib/Analysis/ProfileDataLoaderPass.cpp b/lib/Analysis/ProfileDataLoaderPass.cpp
new file mode 100644
index 0000000..c43cff0
--- /dev/null
+++ b/lib/Analysis/ProfileDataLoaderPass.cpp
@@ -0,0 +1,188 @@
+//===- ProfileDataLoaderPass.cpp - Set branch weight metadata from prof ---===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass loads profiling data from a dump file and sets branch weight
+// metadata.
+//
+// TODO: Replace all "profile-metadata-loader" strings with "profile-loader"
+// once ProfileInfo etc. has been removed.
+//
+//===----------------------------------------------------------------------===//
+#define DEBUG_TYPE "profile-metadata-loader"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/BasicBlock.h"
+#include "llvm/InstrTypes.h"
+#include "llvm/Module.h"
+#include "llvm/LLVMContext.h"
+#include "llvm/MDBuilder.h"
+#include "llvm/Metadata.h"
+#include "llvm/Pass.h"
+#include "llvm/Analysis/Passes.h"
+#include "llvm/Analysis/ProfileDataLoader.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/CFG.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/Format.h"
+#include "llvm/ADT/Statistic.h"
+using namespace llvm;
+
+STATISTIC(NumEdgesRead, "The # of edges read.");
+STATISTIC(NumTermsAnnotated, "The # of terminator instructions annotated.");
+
+static cl::opt<std::string>
+ProfileMetadataFilename("profile-file", cl::init("llvmprof.out"),
+                  cl::value_desc("filename"),
+                  cl::desc("Profile file loaded by -profile-metadata-loader"));
+
+namespace {
+  /// This pass loads profiling data from a dump file and sets branch weight
+  /// metadata.
+  class ProfileMetadataLoaderPass : public ModulePass {
+    std::string Filename;
+  public:
+    static char ID; // Class identification, replacement for typeinfo
+    explicit ProfileMetadataLoaderPass(const std::string &filename = "")
+        : ModulePass(ID), Filename(filename) {
+      initializeProfileMetadataLoaderPassPass(*PassRegistry::getPassRegistry());
+      if (filename.empty()) Filename = ProfileMetadataFilename;
+    }
+
+    virtual void getAnalysisUsage(AnalysisUsage &AU) const {
+      AU.setPreservesAll();
+    }
+
+    virtual const char *getPassName() const {
+      return "Profile loader";
+    }
+
+    virtual void readEdge(unsigned, ProfileData&, ProfileData::Edge,
+                          ArrayRef<unsigned>);
+    virtual unsigned matchEdges(Module&, ProfileData&, ArrayRef<unsigned>);
+    virtual void setBranchWeightMetadata(Module&, ProfileData&);
+
+    virtual bool runOnModule(Module &M);
+  };
+}  // End of anonymous namespace
+
+char ProfileMetadataLoaderPass::ID = 0;
+INITIALIZE_PASS_BEGIN(ProfileMetadataLoaderPass, "profile-metadata-loader",
+              "Load profile information from llvmprof.out", false, true)
+INITIALIZE_PASS_END(ProfileMetadataLoaderPass, "profile-metadata-loader",
+              "Load profile information from llvmprof.out", false, true)
+
+char &llvm::ProfileMetadataLoaderPassID = ProfileMetadataLoaderPass::ID;
+
+/// createProfileMetadataLoaderPass - This function returns a Pass that loads
+/// the profiling information for the module from the specified filename,
+/// making it available to the optimizers.
+ModulePass *llvm::createProfileMetadataLoaderPass() { 
+    return new ProfileMetadataLoaderPass();
+}
+ModulePass *llvm::createProfileMetadataLoaderPass(const std::string &Filename) {
+  return new ProfileMetadataLoaderPass(Filename);
+}
+
+/// readEdge - Take the value from a profile counter and assign it to an edge.
+void ProfileMetadataLoaderPass::readEdge(unsigned ReadCount,
+                                         ProfileData &PB, ProfileData::Edge e,
+                                         ArrayRef<unsigned> Counters) {
+  if (ReadCount >= Counters.size()) return;
+
+  unsigned weight = Counters[ReadCount];
+  assert(weight != ProfileDataLoader::Uncounted);
+  PB.addEdgeWeight(e, weight);
+
+  DEBUG(dbgs() << "-- Read Edge Counter for " << e
+               << " (# "<< (ReadCount) << "): "
+               << PB.getEdgeWeight(e) << "\n");
+}
+
+/// matchEdges - Link every profile counter with an edge.
+unsigned ProfileMetadataLoaderPass::matchEdges(Module &M, ProfileData &PB,
+                                               ArrayRef<unsigned> Counters) {
+  if (Counters.size() == 0) return 0;
+
+  unsigned ReadCount = 0;
+
+  for (Module::iterator F = M.begin(), E = M.end(); F != E; ++F) {
+    if (F->isDeclaration()) continue;
+    DEBUG(dbgs() << "Loading edges in '" << F->getName() << "'\n");
+    readEdge(ReadCount++, PB, PB.getEdge(0, &F->getEntryBlock()), Counters);
+    for (Function::iterator BB = F->begin(), E = F->end(); BB != E; ++BB) {
+      TerminatorInst *TI = BB->getTerminator();
+      for (unsigned s = 0, e = TI->getNumSuccessors(); s != e; ++s) {
+        readEdge(ReadCount++, PB, PB.getEdge(BB,TI->getSuccessor(s)),
+                 Counters);
+      }
+    }
+  }
+
+  return ReadCount;
+}
+
+/// setBranchWeightMetadata - Translate the counter values associated with each
+/// edge into branch weights for each conditional branch (a branch with 2 or
+/// more desinations).
+void ProfileMetadataLoaderPass::setBranchWeightMetadata(Module &M,
+                                                        ProfileData &PB) {
+  for (Module::iterator F = M.begin(), E = M.end(); F != E; ++F) {
+    if (F->isDeclaration()) continue;
+    DEBUG(dbgs() << "Setting branch metadata in '" << F->getName() << "'\n");
+
+    for (Function::iterator BB = F->begin(), E = F->end(); BB != E; ++BB) {
+      TerminatorInst *TI = BB->getTerminator();
+      unsigned NumSuccessors = TI->getNumSuccessors();
+
+      // If there is only one successor then we can not set a branch
+      // probability as the target is certain.
+      if (NumSuccessors < 2) continue;
+
+      // Load the weights of all edges leading from this terminator.
+      DEBUG(dbgs() << "-- Terminator with " << NumSuccessors
+                   << " successors:\n");
+      SmallVector<uint32_t, 4> Weights(NumSuccessors);
+      for (unsigned s = 0 ; s < NumSuccessors ; ++s) {
+          ProfileData::Edge edge = PB.getEdge(BB, TI->getSuccessor(s));
+          Weights[s] = (uint32_t)PB.getEdgeWeight(edge);
+          DEBUG(dbgs() << "---- Edge '" << edge << "' has weight "
+                       << Weights[s] << "\n");
+      }
+
+      // Set branch weight metadata.  This will set branch probabilities of
+      // 100%/0% if that is true of the dynamic execution.
+      // BranchProbabilityInfo can account for this when it loads this metadata
+      // (it gives the unexectuted branch a weight of 1 for the purposes of
+      // probability calculations).
+      MDBuilder MDB(TI->getContext());
+      MDNode *Node = MDB.createBranchWeights(Weights);
+      TI->setMetadata(LLVMContext::MD_prof, Node);
+      NumTermsAnnotated++;
+    }
+  }
+}
+
+bool ProfileMetadataLoaderPass::runOnModule(Module &M) {
+  ProfileDataLoader PDL("profile-data-loader", Filename);
+  ProfileData PB;
+
+  ArrayRef<unsigned> Counters = PDL.getRawEdgeCounts();
+
+  unsigned ReadCount = matchEdges(M, PB, Counters);
+
+  if (ReadCount != Counters.size()) {
+    errs() << "WARNING: profile information is inconsistent with "
+           << "the current program!\n";
+  }
+  NumEdgesRead = ReadCount;
+
+  setBranchWeightMetadata(M, PB);
+
+  return ReadCount > 0;
+}
diff --git a/lib/Analysis/ProfileEstimatorPass.cpp b/lib/Analysis/ProfileEstimatorPass.cpp
index 63468f8..12b59e0 100644
--- a/lib/Analysis/ProfileEstimatorPass.cpp
+++ b/lib/Analysis/ProfileEstimatorPass.cpp
@@ -286,7 +286,7 @@ void ProfileEstimatorPass::recurseBasicBlock(BasicBlock *BB) {
     }
   }
 
-  double fraction = floor(BBWeight/Edges.size());
+  double fraction = Edges.size() ? floor(BBWeight/Edges.size()) : 0.0;
   // Finally we know what flow is still not leaving the block, distribute this
   // flow onto the empty edges.
   for (SmallVector<Edge, 8>::iterator ei = Edges.begin(), ee = Edges.end();
diff --git a/lib/Analysis/ProfileInfo.cpp b/lib/Analysis/ProfileInfo.cpp
index 173de2c..b5b7ac1 100644
--- a/lib/Analysis/ProfileInfo.cpp
+++ b/lib/Analysis/ProfileInfo.cpp
@@ -1016,40 +1016,14 @@ void ProfileInfoT<Function,BasicBlock>::repair(const Function *F) {
   }
 }
 
-raw_ostream& operator<<(raw_ostream &O, const Function *F) {
-  return O << F->getName();
-}
-
 raw_ostream& operator<<(raw_ostream &O, const MachineFunction *MF) {
   return O << MF->getFunction()->getName() << "(MF)";
 }
 
-raw_ostream& operator<<(raw_ostream &O, const BasicBlock *BB) {
-  return O << BB->getName();
-}
-
 raw_ostream& operator<<(raw_ostream &O, const MachineBasicBlock *MBB) {
   return O << MBB->getBasicBlock()->getName() << "(MB)";
 }
 
-raw_ostream& operator<<(raw_ostream &O, std::pair<const BasicBlock *, const BasicBlock *> E) {
-  O << "(";
-
-  if (E.first)
-    O << E.first;
-  else
-    O << "0";
-
-  O << ",";
-
-  if (E.second)
-    O << E.second;
-  else
-    O << "0";
-
-  return O << ")";
-}
-
 raw_ostream& operator<<(raw_ostream &O, std::pair<const MachineBasicBlock *, const MachineBasicBlock *> E) {
   O << "(";
 
diff --git a/lib/Analysis/RegionInfo.cpp b/lib/Analysis/RegionInfo.cpp
index 868f483..30f0d2f 100644
--- a/lib/Analysis/RegionInfo.cpp
+++ b/lib/Analysis/RegionInfo.cpp
@@ -47,7 +47,7 @@ static cl::opt<enum Region::PrintStyle> printStyle("print-region-style",
   cl::values(
     clEnumValN(Region::PrintNone, "none",  "print no details"),
     clEnumValN(Region::PrintBB, "bb",
-               "print regions in detail with block_node_iterator"),
+               "print regions in detail with block_iterator"),
     clEnumValN(Region::PrintRN, "rn",
                "print regions in detail with element_iterator"),
     clEnumValEnd));
@@ -246,22 +246,6 @@ void Region::verifyRegionNest() const {
   verifyRegion();
 }
 
-Region::block_node_iterator Region::block_node_begin() {
-  return GraphTraits<FlatIt<Region*> >::nodes_begin(this);
-}
-
-Region::block_node_iterator Region::block_node_end() {
-  return GraphTraits<FlatIt<Region*> >::nodes_end(this);
-}
-
-Region::const_block_node_iterator Region::block_node_begin() const {
-  return GraphTraits<FlatIt<const Region*> >::nodes_begin(this);
-}
-
-Region::const_block_node_iterator Region::block_node_end() const {
-  return GraphTraits<FlatIt<const Region*> >::nodes_end(this);
-}
-
 Region::element_iterator Region::element_begin() {
   return GraphTraits<Region*>::nodes_begin(this);
 }
@@ -425,10 +409,8 @@ void Region::print(raw_ostream &OS, bool print_tree, unsigned level,
     OS.indent(level*2 + 2);
 
     if (Style == PrintBB) {
-      for (const_block_node_iterator I = block_node_begin(),
-                                     E = block_node_end();
-           I != E; ++I)
-        OS << **I << ", "; // TODO: remove the last ","
+      for (const_block_iterator I = block_begin(), E = block_end(); I != E; ++I)
+        OS << (*I)->getName() << ", "; // TODO: remove the last ","
     } else if (Style == PrintRN) {
       for (const_element_iterator I = element_begin(), E = element_end(); I!=E; ++I)
         OS << **I << ", "; // TODO: remove the last ",
@@ -445,9 +427,11 @@ void Region::print(raw_ostream &OS, bool print_tree, unsigned level,
     OS.indent(level*2) << "} \n";
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void Region::dump() const {
   print(dbgs(), true, getDepth(), printStyle.getValue());
 }
+#endif
 
 void Region::clearNodeCache() {
   // Free the cached nodes.
diff --git a/lib/Analysis/RegionPass.cpp b/lib/Analysis/RegionPass.cpp
index c97b5eb..9208fa2 100644
--- a/lib/Analysis/RegionPass.cpp
+++ b/lib/Analysis/RegionPass.cpp
@@ -195,10 +195,9 @@ public:
 
   virtual bool runOnRegion(Region *R, RGPassManager &RGM) {
     Out << Banner;
-    for (Region::block_node_iterator I = R->block_node_begin(),
-                                     E = R->block_node_end();
+    for (Region::block_iterator I = R->block_begin(), E = R->block_end();
          I != E; ++I)
-      (*I)->getEntry()->print(Out);
+      (*I)->print(Out);
 
     return false;
   }
diff --git a/lib/Analysis/ScalarEvolution.cpp b/lib/Analysis/ScalarEvolution.cpp
index a654648..e3189ec 100644
--- a/lib/Analysis/ScalarEvolution.cpp
+++ b/lib/Analysis/ScalarEvolution.cpp
@@ -73,7 +73,7 @@
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/Assembly/Writer.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetLibraryInfo.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ConstantRange.h"
@@ -105,6 +105,11 @@ MaxBruteForceIterations("scalar-evolution-max-iterations", cl::ReallyHidden,
                                  "derived loop"),
                         cl::init(100));
 
+// FIXME: Enable this with XDEBUG when the test suite is clean.
+static cl::opt<bool>
+VerifySCEV("verify-scev",
+           cl::desc("Verify ScalarEvolution's backedge taken counts (slow)"));
+
 INITIALIZE_PASS_BEGIN(ScalarEvolution, "scalar-evolution",
                 "Scalar Evolution Analysis", false, true)
 INITIALIZE_PASS_DEPENDENCY(LoopInfo)
@@ -122,10 +127,12 @@ char ScalarEvolution::ID = 0;
 // Implementation of the SCEV class.
 //
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void SCEV::dump() const {
   print(dbgs());
   dbgs() << '\n';
 }
+#endif
 
 void SCEV::print(raw_ostream &OS) const {
   switch (getSCEVType()) {
@@ -2580,7 +2587,7 @@ const SCEV *ScalarEvolution::getUMinExpr(const SCEV *LHS,
 }
 
 const SCEV *ScalarEvolution::getSizeOfExpr(Type *AllocTy) {
-  // If we have TargetData, we can bypass creating a target-independent
+  // If we have DataLayout, we can bypass creating a target-independent
   // constant expression and then folding it back into a ConstantInt.
   // This is just a compile-time optimization.
   if (TD)
@@ -2606,7 +2613,7 @@ const SCEV *ScalarEvolution::getAlignOfExpr(Type *AllocTy) {
 
 const SCEV *ScalarEvolution::getOffsetOfExpr(StructType *STy,
                                              unsigned FieldNo) {
-  // If we have TargetData, we can bypass creating a target-independent
+  // If we have DataLayout, we can bypass creating a target-independent
   // constant expression and then folding it back into a ConstantInt.
   // This is just a compile-time optimization.
   if (TD)
@@ -2671,7 +2678,7 @@ bool ScalarEvolution::isSCEVable(Type *Ty) const {
 uint64_t ScalarEvolution::getTypeSizeInBits(Type *Ty) const {
   assert(isSCEVable(Ty) && "Type is not SCEVable!");
 
-  // If we have a TargetData, use it!
+  // If we have a DataLayout, use it!
   if (TD)
     return TD->getTypeSizeInBits(Ty);
 
@@ -2679,7 +2686,7 @@ uint64_t ScalarEvolution::getTypeSizeInBits(Type *Ty) const {
   if (Ty->isIntegerTy())
     return Ty->getPrimitiveSizeInBits();
 
-  // The only other support type is pointer. Without TargetData, conservatively
+  // The only other support type is pointer. Without DataLayout, conservatively
   // assume pointers are 64-bit.
   assert(Ty->isPointerTy() && "isSCEVable permitted a non-SCEVable type!");
   return 64;
@@ -2699,7 +2706,7 @@ Type *ScalarEvolution::getEffectiveSCEVType(Type *Ty) const {
   assert(Ty->isPointerTy() && "Unexpected non-pointer non-integer type!");
   if (TD) return TD->getIntPtrType(getContext());
 
-  // Without TargetData, conservatively assume pointers are 64-bit.
+  // Without DataLayout, conservatively assume pointers are 64-bit.
   return Type::getInt64Ty(getContext());
 }
 
@@ -3978,8 +3985,11 @@ getSmallConstantTripMultiple(Loop *L, BasicBlock *ExitingBlock) {
 
   ConstantInt *Result = MulC->getValue();
 
-  // Guard against huge trip counts.
-  if (!Result || Result->getValue().getActiveBits() > 32)
+  // Guard against huge trip counts (this requires checking
+  // for zero to handle the case where the trip count == -1 and the
+  // addition wraps).
+  if (!Result || Result->getValue().getActiveBits() > 32 ||
+      Result->getValue().getActiveBits() == 0)
     return 1;
 
   return (unsigned)Result->getZExtValue();
@@ -4749,7 +4759,7 @@ static PHINode *getConstantEvolvingPHI(Value *V, const Loop *L) {
 /// reason, return null.
 static Constant *EvaluateExpression(Value *V, const Loop *L,
                                     DenseMap<Instruction *, Constant *> &Vals,
-                                    const TargetData *TD,
+                                    const DataLayout *TD,
                                     const TargetLibraryInfo *TLI) {
   // Convenient constant check, but redundant for recursive calls.
   if (Constant *C = dyn_cast<Constant>(V)) return C;
@@ -6141,7 +6151,7 @@ bool ScalarEvolution::isImpliedCond(ICmpInst::Predicate Pred,
       return CmpInst::isTrueWhenEqual(Pred);
   if (SimplifyICmpOperands(FoundPred, FoundLHS, FoundRHS))
     if (FoundLHS == FoundRHS)
-      return CmpInst::isFalseWhenEqual(Pred);
+      return CmpInst::isFalseWhenEqual(FoundPred);
 
   // Check to see if we can make the LHS or RHS match.
   if (LHS == FoundRHS || RHS == FoundLHS) {
@@ -6588,7 +6598,7 @@ ScalarEvolution::ScalarEvolution()
 bool ScalarEvolution::runOnFunction(Function &F) {
   this->F = &F;
   LI = &getAnalysis<LoopInfo>();
-  TD = getAnalysisIfAvailable<TargetData>();
+  TD = getAnalysisIfAvailable<DataLayout>();
   TLI = &getAnalysis<TargetLibraryInfo>();
   DT = &getAnalysis<DominatorTree>();
   return false;
@@ -6930,3 +6940,87 @@ void ScalarEvolution::forgetMemoizedResults(const SCEV *S) {
   UnsignedRanges.erase(S);
   SignedRanges.erase(S);
 }
+
+typedef DenseMap<const Loop *, std::string> VerifyMap;
+
+/// replaceSubString - Replaces all occurences of From in Str with To.
+static void replaceSubString(std::string &Str, StringRef From, StringRef To) {
+  size_t Pos = 0;
+  while ((Pos = Str.find(From, Pos)) != std::string::npos) {
+    Str.replace(Pos, From.size(), To.data(), To.size());
+    Pos += To.size();
+  }
+}
+
+/// getLoopBackedgeTakenCounts - Helper method for verifyAnalysis.
+static void
+getLoopBackedgeTakenCounts(Loop *L, VerifyMap &Map, ScalarEvolution &SE) {
+  for (Loop::reverse_iterator I = L->rbegin(), E = L->rend(); I != E; ++I) {
+    getLoopBackedgeTakenCounts(*I, Map, SE); // recurse.
+
+    std::string &S = Map[L];
+    if (S.empty()) {
+      raw_string_ostream OS(S);
+      SE.getBackedgeTakenCount(L)->print(OS);
+
+      // false and 0 are semantically equivalent. This can happen in dead loops.
+      replaceSubString(OS.str(), "false", "0");
+      // Remove wrap flags, their use in SCEV is highly fragile.
+      // FIXME: Remove this when SCEV gets smarter about them.
+      replaceSubString(OS.str(), "<nw>", "");
+      replaceSubString(OS.str(), "<nsw>", "");
+      replaceSubString(OS.str(), "<nuw>", "");
+    }
+  }
+}
+
+void ScalarEvolution::verifyAnalysis() const {
+  if (!VerifySCEV)
+    return;
+
+  ScalarEvolution &SE = *const_cast<ScalarEvolution *>(this);
+
+  // Gather stringified backedge taken counts for all loops using SCEV's caches.
+  // FIXME: It would be much better to store actual values instead of strings,
+  //        but SCEV pointers will change if we drop the caches.
+  VerifyMap BackedgeDumpsOld, BackedgeDumpsNew;
+  for (LoopInfo::reverse_iterator I = LI->rbegin(), E = LI->rend(); I != E; ++I)
+    getLoopBackedgeTakenCounts(*I, BackedgeDumpsOld, SE);
+
+  // Gather stringified backedge taken counts for all loops without using
+  // SCEV's caches.
+  SE.releaseMemory();
+  for (LoopInfo::reverse_iterator I = LI->rbegin(), E = LI->rend(); I != E; ++I)
+    getLoopBackedgeTakenCounts(*I, BackedgeDumpsNew, SE);
+
+  // Now compare whether they're the same with and without caches. This allows
+  // verifying that no pass changed the cache.
+  assert(BackedgeDumpsOld.size() == BackedgeDumpsNew.size() &&
+         "New loops suddenly appeared!");
+
+  for (VerifyMap::iterator OldI = BackedgeDumpsOld.begin(),
+                           OldE = BackedgeDumpsOld.end(),
+                           NewI = BackedgeDumpsNew.begin();
+       OldI != OldE; ++OldI, ++NewI) {
+    assert(OldI->first == NewI->first && "Loop order changed!");
+
+    // Compare the stringified SCEVs. We don't care if undef backedgetaken count
+    // changes.
+    // FIXME: We currently ignore SCEV changes from/to CouldNotCompute. This
+    // means that a pass is buggy or SCEV has to learn a new pattern but is
+    // usually not harmful.
+    if (OldI->second != NewI->second &&
+        OldI->second.find("undef") == std::string::npos &&
+        NewI->second.find("undef") == std::string::npos &&
+        OldI->second != "***COULDNOTCOMPUTE***" &&
+        NewI->second != "***COULDNOTCOMPUTE***") {
+      dbgs() << "SCEVValidator: SCEV for loop '"
+             << OldI->first->getHeader()->getName()
+             << "' changed from '" << OldI->second
+             << "' to '" << NewI->second << "'!\n";
+      std::abort();
+    }
+  }
+
+  // TODO: Verify more things.
+}
diff --git a/lib/Analysis/ScalarEvolutionExpander.cpp b/lib/Analysis/ScalarEvolutionExpander.cpp
index 62710c5..111bfb4 100644
--- a/lib/Analysis/ScalarEvolutionExpander.cpp
+++ b/lib/Analysis/ScalarEvolutionExpander.cpp
@@ -18,7 +18,7 @@
 #include "llvm/IntrinsicInst.h"
 #include "llvm/LLVMContext.h"
 #include "llvm/Support/Debug.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetLowering.h"
 #include "llvm/ADT/STLExtras.h"
 
@@ -212,7 +212,7 @@ static bool FactorOutConstant(const SCEV *&S,
                               const SCEV *&Remainder,
                               const SCEV *Factor,
                               ScalarEvolution &SE,
-                              const TargetData *TD) {
+                              const DataLayout *TD) {
   // Everything is divisible by one.
   if (Factor->isOne())
     return true;
@@ -253,7 +253,7 @@ static bool FactorOutConstant(const SCEV *&S,
   // of the given factor.
   if (const SCEVMulExpr *M = dyn_cast<SCEVMulExpr>(S)) {
     if (TD) {
-      // With TargetData, the size is known. Check if there is a constant
+      // With DataLayout, the size is known. Check if there is a constant
       // operand which is a multiple of the given factor. If so, we can
       // factor it.
       const SCEVConstant *FC = cast<SCEVConstant>(Factor);
@@ -267,7 +267,7 @@ static bool FactorOutConstant(const SCEV *&S,
           return true;
         }
     } else {
-      // Without TargetData, check if Factor can be factored out of any of the
+      // Without DataLayout, check if Factor can be factored out of any of the
       // Mul's operands. If so, we can just remove it.
       for (unsigned i = 0, e = M->getNumOperands(); i != e; ++i) {
         const SCEV *SOp = M->getOperand(i);
@@ -458,7 +458,7 @@ Value *SCEVExpander::expandAddToGEP(const SCEV *const *op_begin,
       // An empty struct has no fields.
       if (STy->getNumElements() == 0) break;
       if (SE.TD) {
-        // With TargetData, field offsets are known. See if a constant offset
+        // With DataLayout, field offsets are known. See if a constant offset
         // falls within any of the struct fields.
         if (Ops.empty()) break;
         if (const SCEVConstant *C = dyn_cast<SCEVConstant>(Ops[0]))
@@ -477,7 +477,7 @@ Value *SCEVExpander::expandAddToGEP(const SCEV *const *op_begin,
             }
           }
       } else {
-        // Without TargetData, just check for an offsetof expression of the
+        // Without DataLayout, just check for an offsetof expression of the
         // appropriate struct type.
         for (unsigned i = 0, e = Ops.size(); i != e; ++i)
           if (const SCEVUnknown *U = dyn_cast<SCEVUnknown>(Ops[i])) {
@@ -1618,6 +1618,17 @@ unsigned SCEVExpander::replaceCongruentIVs(Loop *L, const DominatorTree *DT,
          PEnd = Phis.end(); PIter != PEnd; ++PIter) {
     PHINode *Phi = *PIter;
 
+    // Fold constant phis. They may be congruent to other constant phis and
+    // would confuse the logic below that expects proper IVs.
+    if (Value *V = Phi->hasConstantValue()) {
+      Phi->replaceAllUsesWith(V);
+      DeadInsts.push_back(Phi);
+      ++NumElim;
+      DEBUG_WITH_TYPE(DebugType, dbgs()
+                      << "INDVARS: Eliminated constant iv: " << *Phi << '\n');
+      continue;
+    }
+
     if (!SE.isSCEVable(Phi->getType()))
       continue;
 
diff --git a/lib/Analysis/Trace.cpp b/lib/Analysis/Trace.cpp
index ff5010b..22da857 100644
--- a/lib/Analysis/Trace.cpp
+++ b/lib/Analysis/Trace.cpp
@@ -43,9 +43,11 @@ void Trace::print(raw_ostream &O) const {
   O << "; Trace parent function: \n" << *F;
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 /// dump - Debugger convenience method; writes trace to standard error
 /// output stream.
 ///
 void Trace::dump() const {
   print(dbgs());
 }
+#endif
diff --git a/lib/Analysis/ValueTracking.cpp b/lib/Analysis/ValueTracking.cpp
index cea34e1..3beb373 100644
--- a/lib/Analysis/ValueTracking.cpp
+++ b/lib/Analysis/ValueTracking.cpp
@@ -22,7 +22,7 @@
 #include "llvm/LLVMContext.h"
 #include "llvm/Metadata.h"
 #include "llvm/Operator.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Support/ConstantRange.h"
 #include "llvm/Support/GetElementPtrTypeIterator.h"
 #include "llvm/Support/MathExtras.h"
@@ -36,7 +36,7 @@ const unsigned MaxDepth = 6;
 
 /// getBitWidth - Returns the bitwidth of the given scalar or pointer type (if
 /// unknown returns 0).  For vector types, returns the element type's bitwidth.
-static unsigned getBitWidth(Type *Ty, const TargetData *TD) {
+static unsigned getBitWidth(Type *Ty, const DataLayout *TD) {
   if (unsigned BitWidth = Ty->getScalarSizeInBits())
     return BitWidth;
   assert(isa<PointerType>(Ty) && "Expected a pointer type!");
@@ -46,7 +46,7 @@ static unsigned getBitWidth(Type *Ty, const TargetData *TD) {
 static void ComputeMaskedBitsAddSub(bool Add, Value *Op0, Value *Op1, bool NSW,
                                     APInt &KnownZero, APInt &KnownOne,
                                     APInt &KnownZero2, APInt &KnownOne2,
-                                    const TargetData *TD, unsigned Depth) {
+                                    const DataLayout *TD, unsigned Depth) {
   if (!Add) {
     if (ConstantInt *CLHS = dyn_cast<ConstantInt>(Op0)) {
       // We know that the top bits of C-X are clear if X contains less bits
@@ -132,7 +132,7 @@ static void ComputeMaskedBitsAddSub(bool Add, Value *Op0, Value *Op1, bool NSW,
 static void ComputeMaskedBitsMul(Value *Op0, Value *Op1, bool NSW,
                                  APInt &KnownZero, APInt &KnownOne,
                                  APInt &KnownZero2, APInt &KnownOne2,
-                                 const TargetData *TD, unsigned Depth) {
+                                 const DataLayout *TD, unsigned Depth) {
   unsigned BitWidth = KnownZero.getBitWidth();
   ComputeMaskedBits(Op1, KnownZero, KnownOne, TD, Depth+1);
   ComputeMaskedBits(Op0, KnownZero2, KnownOne2, TD, Depth+1);
@@ -226,7 +226,7 @@ void llvm::computeMaskedBitsLoad(const MDNode &Ranges, APInt &KnownZero) {
 /// same width as the vector element, and the bit is set only if it is true
 /// for all of the elements in the vector.
 void llvm::ComputeMaskedBits(Value *V, APInt &KnownZero, APInt &KnownOne,
-                             const TargetData *TD, unsigned Depth) {
+                             const DataLayout *TD, unsigned Depth) {
   assert(V && "No Value?");
   assert(Depth <= MaxDepth && "Limit Search Depth");
   unsigned BitWidth = KnownZero.getBitWidth();
@@ -308,11 +308,20 @@ void llvm::ComputeMaskedBits(Value *V, APInt &KnownZero, APInt &KnownOne,
   }
   
   if (Argument *A = dyn_cast<Argument>(V)) {
-    // Get alignment information off byval arguments if specified in the IR.
-    if (A->hasByValAttr())
-      if (unsigned Align = A->getParamAlignment())
-        KnownZero = APInt::getLowBitsSet(BitWidth,
-                                         CountTrailingZeros_32(Align));
+    unsigned Align = 0;
+
+    if (A->hasByValAttr()) {
+      // Get alignment information off byval arguments if specified in the IR.
+      Align = A->getParamAlignment();
+    } else if (TD && A->hasStructRetAttr()) {
+      // An sret parameter has at least the ABI alignment of the return type.
+      Type *EltTy = cast<PointerType>(A->getType())->getElementType();
+      if (EltTy->isSized())
+        Align = TD->getABITypeAlignment(EltTy);
+    }
+
+    if (Align)
+      KnownZero = APInt::getLowBitsSet(BitWidth, CountTrailingZeros_32(Align));
     return;
   }
 
@@ -420,15 +429,13 @@ void llvm::ComputeMaskedBits(Value *V, APInt &KnownZero, APInt &KnownOne,
   case Instruction::ZExt:
   case Instruction::Trunc: {
     Type *SrcTy = I->getOperand(0)->getType();
-    
+
     unsigned SrcBitWidth;
     // Note that we handle pointer operands here because of inttoptr/ptrtoint
     // which fall through here.
-    if (SrcTy->isPointerTy())
-      SrcBitWidth = TD->getTypeSizeInBits(SrcTy);
-    else
-      SrcBitWidth = SrcTy->getScalarSizeInBits();
-    
+    SrcBitWidth = TD->getTypeSizeInBits(SrcTy->getScalarType());
+
+    assert(SrcBitWidth && "SrcBitWidth can't be zero");
     KnownZero = KnownZero.zextOrTrunc(SrcBitWidth);
     KnownOne = KnownOne.zextOrTrunc(SrcBitWidth);
     ComputeMaskedBits(I->getOperand(0), KnownZero, KnownOne, TD, Depth+1);
@@ -778,7 +785,7 @@ void llvm::ComputeMaskedBits(Value *V, APInt &KnownZero, APInt &KnownOne,
 /// ComputeSignBit - Determine whether the sign bit is known to be zero or
 /// one.  Convenience wrapper around ComputeMaskedBits.
 void llvm::ComputeSignBit(Value *V, bool &KnownZero, bool &KnownOne,
-                          const TargetData *TD, unsigned Depth) {
+                          const DataLayout *TD, unsigned Depth) {
   unsigned BitWidth = getBitWidth(V->getType(), TD);
   if (!BitWidth) {
     KnownZero = false;
@@ -796,7 +803,7 @@ void llvm::ComputeSignBit(Value *V, bool &KnownZero, bool &KnownOne,
 /// bit set when defined. For vectors return true if every element is known to
 /// be a power of two when defined.  Supports values with integer or pointer
 /// types and vectors of integers.
-bool llvm::isPowerOfTwo(Value *V, const TargetData *TD, bool OrZero,
+bool llvm::isPowerOfTwo(Value *V, const DataLayout *TD, bool OrZero,
                         unsigned Depth) {
   if (Constant *C = dyn_cast<Constant>(V)) {
     if (C->isNullValue())
@@ -859,7 +866,7 @@ bool llvm::isPowerOfTwo(Value *V, const TargetData *TD, bool OrZero,
 /// when defined.  For vectors return true if every element is known to be
 /// non-zero when defined.  Supports values with integer or pointer type and
 /// vectors of integers.
-bool llvm::isKnownNonZero(Value *V, const TargetData *TD, unsigned Depth) {
+bool llvm::isKnownNonZero(Value *V, const DataLayout *TD, unsigned Depth) {
   if (Constant *C = dyn_cast<Constant>(V)) {
     if (C->isNullValue())
       return false;
@@ -986,7 +993,7 @@ bool llvm::isKnownNonZero(Value *V, const TargetData *TD, unsigned Depth) {
 /// same width as the vector element, and the bit is set only if it is true
 /// for all of the elements in the vector.
 bool llvm::MaskedValueIsZero(Value *V, const APInt &Mask,
-                             const TargetData *TD, unsigned Depth) {
+                             const DataLayout *TD, unsigned Depth) {
   APInt KnownZero(Mask.getBitWidth(), 0), KnownOne(Mask.getBitWidth(), 0);
   ComputeMaskedBits(V, KnownZero, KnownOne, TD, Depth);
   assert((KnownZero & KnownOne) == 0 && "Bits known to be one AND zero?"); 
@@ -1003,10 +1010,10 @@ bool llvm::MaskedValueIsZero(Value *V, const APInt &Mask,
 ///
 /// 'Op' must have a scalar integer type.
 ///
-unsigned llvm::ComputeNumSignBits(Value *V, const TargetData *TD,
+unsigned llvm::ComputeNumSignBits(Value *V, const DataLayout *TD,
                                   unsigned Depth) {
   assert((TD || V->getType()->isIntOrIntVectorTy()) &&
-         "ComputeNumSignBits requires a TargetData object to operate "
+         "ComputeNumSignBits requires a DataLayout object to operate "
          "on non-integer values!");
   Type *Ty = V->getType();
   unsigned TyBits = TD ? TD->getTypeSizeInBits(V->getType()->getScalarType()) :
@@ -1582,7 +1589,7 @@ Value *llvm::FindInsertedValue(Value *V, ArrayRef<unsigned> idx_range,
 /// it can be expressed as a base pointer plus a constant offset.  Return the
 /// base and offset to the caller.
 Value *llvm::GetPointerBaseWithConstantOffset(Value *Ptr, int64_t &Offset,
-                                              const TargetData &TD) {
+                                              const DataLayout &TD) {
   Operator *PtrOp = dyn_cast<Operator>(Ptr);
   if (PtrOp == 0 || Ptr->getType()->isVectorTy())
     return Ptr;
@@ -1614,7 +1621,7 @@ Value *llvm::GetPointerBaseWithConstantOffset(Value *Ptr, int64_t &Offset,
   // right.
   unsigned PtrSize = TD.getPointerSizeInBits();
   if (PtrSize < 64)
-    Offset = (Offset << (64-PtrSize)) >> (64-PtrSize);
+    Offset = SignExtend64(Offset, PtrSize);
   
   return GetPointerBaseWithConstantOffset(GEP->getPointerOperand(), Offset, TD);
 }
@@ -1768,7 +1775,7 @@ uint64_t llvm::GetStringLength(Value *V) {
 }
 
 Value *
-llvm::GetUnderlyingObject(Value *V, const TargetData *TD, unsigned MaxLookup) {
+llvm::GetUnderlyingObject(Value *V, const DataLayout *TD, unsigned MaxLookup) {
   if (!V->getType()->isPointerTy())
     return V;
   for (unsigned Count = 0; MaxLookup == 0 || Count < MaxLookup; ++Count) {
@@ -1799,7 +1806,7 @@ llvm::GetUnderlyingObject(Value *V, const TargetData *TD, unsigned MaxLookup) {
 void
 llvm::GetUnderlyingObjects(Value *V,
                            SmallVectorImpl<Value *> &Objects,
-                           const TargetData *TD,
+                           const DataLayout *TD,
                            unsigned MaxLookup) {
   SmallPtrSet<Value *, 4> Visited;
   SmallVector<Value *, 4> Worklist;
@@ -1844,7 +1851,7 @@ bool llvm::onlyUsedByLifetimeMarkers(const Value *V) {
 }
 
 bool llvm::isSafeToSpeculativelyExecute(const Value *V,
-                                        const TargetData *TD) {
+                                        const DataLayout *TD) {
   const Operator *Inst = dyn_cast<Operator>(V);
   if (!Inst)
     return false;
diff --git a/lib/Archive/ArchiveInternals.h b/lib/Archive/ArchiveInternals.h
index 55684f7..639f5ac 100644
--- a/lib/Archive/ArchiveInternals.h
+++ b/lib/Archive/ArchiveInternals.h
@@ -66,7 +66,7 @@ namespace llvm {
       fmag[1] = '\n';
     }
 
-    bool checkSignature() {
+    bool checkSignature() const {
       return 0 == memcmp(fmag, ARFILE_MEMBER_MAGIC,2);
     }
   };
diff --git a/lib/Archive/ArchiveReader.cpp b/lib/Archive/ArchiveReader.cpp
index 5cfc810..5052495 100644
--- a/lib/Archive/ArchiveReader.cpp
+++ b/lib/Archive/ArchiveReader.cpp
@@ -79,7 +79,7 @@ Archive::parseMemberHeader(const char*& At, const char* End, std::string* error)
   }
 
   // Cast archive member header
-  ArchiveMemberHeader* Hdr = (ArchiveMemberHeader*)At;
+  const ArchiveMemberHeader* Hdr = (const ArchiveMemberHeader*)At;
   At += sizeof(ArchiveMemberHeader);
 
   int flags = 0;
@@ -196,7 +196,7 @@ Archive::parseMemberHeader(const char*& At, const char* End, std::string* error)
       /* FALL THROUGH */
 
     default:
-      char* slash = (char*) memchr(Hdr->name, '/', 16);
+      const char* slash = (const char*) memchr(Hdr->name, '/', 16);
       if (slash == 0)
         slash = Hdr->name + 16;
       pathname.assign(Hdr->name, slash - Hdr->name);
diff --git a/lib/AsmParser/LLLexer.cpp b/lib/AsmParser/LLLexer.cpp
index 481733d..a60e4aa 100644
--- a/lib/AsmParser/LLLexer.cpp
+++ b/lib/AsmParser/LLLexer.cpp
@@ -456,11 +456,12 @@ lltok::Kind LLLexer::LexIdentifier() {
   KEYWORD(private);
   KEYWORD(linker_private);
   KEYWORD(linker_private_weak);
-  KEYWORD(linker_private_weak_def_auto);
+  KEYWORD(linker_private_weak_def_auto); // FIXME: For backwards compatibility.
   KEYWORD(internal);
   KEYWORD(available_externally);
   KEYWORD(linkonce);
   KEYWORD(linkonce_odr);
+  KEYWORD(linkonce_odr_auto_hide);
   KEYWORD(weak);
   KEYWORD(weak_odr);
   KEYWORD(appending);
@@ -509,6 +510,7 @@ lltok::Kind LLLexer::LexIdentifier() {
   KEYWORD(asm);
   KEYWORD(sideeffect);
   KEYWORD(alignstack);
+  KEYWORD(inteldialect);
   KEYWORD(gc);
 
   KEYWORD(ccc);
@@ -523,6 +525,9 @@ lltok::Kind LLLexer::LexIdentifier() {
   KEYWORD(msp430_intrcc);
   KEYWORD(ptx_kernel);
   KEYWORD(ptx_device);
+  KEYWORD(spir_kernel);
+  KEYWORD(spir_func);
+  KEYWORD(intel_ocl_bicc);
 
   KEYWORD(cc);
   KEYWORD(c);
@@ -553,7 +558,7 @@ lltok::Kind LLLexer::LexIdentifier() {
   KEYWORD(naked);
   KEYWORD(nonlazybind);
   KEYWORD(address_safety);
-  KEYWORD(ia_nsdialect);
+  KEYWORD(minsize);
 
   KEYWORD(type);
   KEYWORD(opaque);
diff --git a/lib/AsmParser/LLParser.cpp b/lib/AsmParser/LLParser.cpp
index 0ff8edd..b24291f 100644
--- a/lib/AsmParser/LLParser.cpp
+++ b/lib/AsmParser/LLParser.cpp
@@ -184,12 +184,13 @@ bool LLParser::ParseTopLevelEntities() {
     case lltok::kw_private:             // OptionalLinkage
     case lltok::kw_linker_private:      // OptionalLinkage
     case lltok::kw_linker_private_weak: // OptionalLinkage
-    case lltok::kw_linker_private_weak_def_auto: // OptionalLinkage
+    case lltok::kw_linker_private_weak_def_auto: // FIXME: backwards compat.
     case lltok::kw_internal:            // OptionalLinkage
     case lltok::kw_weak:                // OptionalLinkage
     case lltok::kw_weak_odr:            // OptionalLinkage
     case lltok::kw_linkonce:            // OptionalLinkage
     case lltok::kw_linkonce_odr:        // OptionalLinkage
+    case lltok::kw_linkonce_odr_auto_hide: // OptionalLinkage
     case lltok::kw_appending:           // OptionalLinkage
     case lltok::kw_dllexport:           // OptionalLinkage
     case lltok::kw_common:              // OptionalLinkage
@@ -576,8 +577,7 @@ bool LLParser::ParseAlias(const std::string &Name, LocTy NameLoc,
       Linkage != GlobalValue::InternalLinkage &&
       Linkage != GlobalValue::PrivateLinkage &&
       Linkage != GlobalValue::LinkerPrivateLinkage &&
-      Linkage != GlobalValue::LinkerPrivateWeakLinkage &&
-      Linkage != GlobalValue::LinkerPrivateWeakDefAutoLinkage)
+      Linkage != GlobalValue::LinkerPrivateWeakLinkage)
     return Error(LinkageLoc, "invalid linkage type for alias");
 
   Constant *Aliasee;
@@ -779,7 +779,9 @@ GlobalValue *LLParser::GetGlobalVal(const std::string &Name, Type *Ty,
     FwdVal = Function::Create(FT, GlobalValue::ExternalWeakLinkage, Name, M);
   else
     FwdVal = new GlobalVariable(*M, PTy->getElementType(), false,
-                                GlobalValue::ExternalWeakLinkage, 0, Name);
+                                GlobalValue::ExternalWeakLinkage, 0, Name,
+                                0, GlobalVariable::NotThreadLocal,
+                                PTy->getAddressSpace());
 
   ForwardRefVals[Name] = std::make_pair(FwdVal, Loc);
   return FwdVal;
@@ -916,59 +918,50 @@ bool LLParser::ParseOptionalAddrSpace(unsigned &AddrSpace) {
 /// ParseOptionalAttrs - Parse a potentially empty attribute list.  AttrKind
 /// indicates what kind of attribute list this is: 0: function arg, 1: result,
 /// 2: function attr.
-bool LLParser::ParseOptionalAttrs(Attributes &Attrs, unsigned AttrKind) {
-  Attrs = Attribute::None;
+bool LLParser::ParseOptionalAttrs(AttrBuilder &B, unsigned AttrKind) {
   LocTy AttrLoc = Lex.getLoc();
+  bool HaveError = false;
+
+  B.clear();
 
   while (1) {
-    switch (Lex.getKind()) {
+    lltok::Kind Token = Lex.getKind();
+    switch (Token) {
     default:  // End of attributes.
-      if (AttrKind != 2 && (Attrs & Attribute::FunctionOnly))
-        return Error(AttrLoc, "invalid use of function-only attribute");
-
-      // As a hack, we allow "align 2" on functions as a synonym for
-      // "alignstack 2".
-      if (AttrKind == 2 &&
-          (Attrs & ~(Attribute::FunctionOnly | Attribute::Alignment)))
-        return Error(AttrLoc, "invalid use of attribute on a function");
-
-      if (AttrKind != 0 && (Attrs & Attribute::ParameterOnly))
-        return Error(AttrLoc, "invalid use of parameter-only attribute");
-
-      return false;
-    case lltok::kw_zeroext:         Attrs |= Attribute::ZExt; break;
-    case lltok::kw_signext:         Attrs |= Attribute::SExt; break;
-    case lltok::kw_inreg:           Attrs |= Attribute::InReg; break;
-    case lltok::kw_sret:            Attrs |= Attribute::StructRet; break;
-    case lltok::kw_noalias:         Attrs |= Attribute::NoAlias; break;
-    case lltok::kw_nocapture:       Attrs |= Attribute::NoCapture; break;
-    case lltok::kw_byval:           Attrs |= Attribute::ByVal; break;
-    case lltok::kw_nest:            Attrs |= Attribute::Nest; break;
-
-    case lltok::kw_noreturn:        Attrs |= Attribute::NoReturn; break;
-    case lltok::kw_nounwind:        Attrs |= Attribute::NoUnwind; break;
-    case lltok::kw_uwtable:         Attrs |= Attribute::UWTable; break;
-    case lltok::kw_returns_twice:   Attrs |= Attribute::ReturnsTwice; break;
-    case lltok::kw_noinline:        Attrs |= Attribute::NoInline; break;
-    case lltok::kw_readnone:        Attrs |= Attribute::ReadNone; break;
-    case lltok::kw_readonly:        Attrs |= Attribute::ReadOnly; break;
-    case lltok::kw_inlinehint:      Attrs |= Attribute::InlineHint; break;
-    case lltok::kw_alwaysinline:    Attrs |= Attribute::AlwaysInline; break;
-    case lltok::kw_optsize:         Attrs |= Attribute::OptimizeForSize; break;
-    case lltok::kw_ssp:             Attrs |= Attribute::StackProtect; break;
-    case lltok::kw_sspreq:          Attrs |= Attribute::StackProtectReq; break;
-    case lltok::kw_noredzone:       Attrs |= Attribute::NoRedZone; break;
-    case lltok::kw_noimplicitfloat: Attrs |= Attribute::NoImplicitFloat; break;
-    case lltok::kw_naked:           Attrs |= Attribute::Naked; break;
-    case lltok::kw_nonlazybind:     Attrs |= Attribute::NonLazyBind; break;
-    case lltok::kw_address_safety:  Attrs |= Attribute::AddressSafety; break;
-    case lltok::kw_ia_nsdialect:    Attrs |= Attribute::IANSDialect; break;
+      return HaveError;
+    case lltok::kw_zeroext:         B.addAttribute(Attributes::ZExt); break;
+    case lltok::kw_signext:         B.addAttribute(Attributes::SExt); break;
+    case lltok::kw_inreg:           B.addAttribute(Attributes::InReg); break;
+    case lltok::kw_sret:            B.addAttribute(Attributes::StructRet); break;
+    case lltok::kw_noalias:         B.addAttribute(Attributes::NoAlias); break;
+    case lltok::kw_nocapture:       B.addAttribute(Attributes::NoCapture); break;
+    case lltok::kw_byval:           B.addAttribute(Attributes::ByVal); break;
+    case lltok::kw_nest:            B.addAttribute(Attributes::Nest); break;
+
+    case lltok::kw_noreturn:        B.addAttribute(Attributes::NoReturn); break;
+    case lltok::kw_nounwind:        B.addAttribute(Attributes::NoUnwind); break;
+    case lltok::kw_uwtable:         B.addAttribute(Attributes::UWTable); break;
+    case lltok::kw_returns_twice:   B.addAttribute(Attributes::ReturnsTwice); break;
+    case lltok::kw_noinline:        B.addAttribute(Attributes::NoInline); break;
+    case lltok::kw_readnone:        B.addAttribute(Attributes::ReadNone); break;
+    case lltok::kw_readonly:        B.addAttribute(Attributes::ReadOnly); break;
+    case lltok::kw_inlinehint:      B.addAttribute(Attributes::InlineHint); break;
+    case lltok::kw_alwaysinline:    B.addAttribute(Attributes::AlwaysInline); break;
+    case lltok::kw_optsize:         B.addAttribute(Attributes::OptimizeForSize); break;
+    case lltok::kw_ssp:             B.addAttribute(Attributes::StackProtect); break;
+    case lltok::kw_sspreq:          B.addAttribute(Attributes::StackProtectReq); break;
+    case lltok::kw_noredzone:       B.addAttribute(Attributes::NoRedZone); break;
+    case lltok::kw_noimplicitfloat: B.addAttribute(Attributes::NoImplicitFloat); break;
+    case lltok::kw_naked:           B.addAttribute(Attributes::Naked); break;
+    case lltok::kw_nonlazybind:     B.addAttribute(Attributes::NonLazyBind); break;
+    case lltok::kw_address_safety:  B.addAttribute(Attributes::AddressSafety); break;
+    case lltok::kw_minsize:         B.addAttribute(Attributes::MinSize); break;
 
     case lltok::kw_alignstack: {
       unsigned Alignment;
       if (ParseOptionalStackAlignment(Alignment))
         return true;
-      Attrs |= Attribute::constructStackAlignmentFromInt(Alignment);
+      B.addStackAlignmentAttr(Alignment);
       continue;
     }
 
@@ -976,11 +969,57 @@ bool LLParser::ParseOptionalAttrs(Attributes &Attrs, unsigned AttrKind) {
       unsigned Alignment;
       if (ParseOptionalAlignment(Alignment))
         return true;
-      Attrs |= Attribute::constructAlignmentFromInt(Alignment);
+      B.addAlignmentAttr(Alignment);
       continue;
     }
 
     }
+
+    // Perform some error checking.
+    switch (Token) {
+    default:
+      if (AttrKind == 2)
+        HaveError |= Error(AttrLoc, "invalid use of attribute on a function");
+      break;
+    case lltok::kw_align:
+      // As a hack, we allow "align 2" on functions as a synonym for
+      // "alignstack 2".
+      break;
+
+    // Parameter Only:
+    case lltok::kw_sret:
+    case lltok::kw_nocapture:
+    case lltok::kw_byval:
+    case lltok::kw_nest:
+      if (AttrKind != 0)
+        HaveError |= Error(AttrLoc, "invalid use of parameter-only attribute");
+      break;
+
+    // Function Only:
+    case lltok::kw_noreturn:
+    case lltok::kw_nounwind:
+    case lltok::kw_readnone:
+    case lltok::kw_readonly:
+    case lltok::kw_noinline:
+    case lltok::kw_alwaysinline:
+    case lltok::kw_optsize:
+    case lltok::kw_ssp:
+    case lltok::kw_sspreq:
+    case lltok::kw_noredzone:
+    case lltok::kw_noimplicitfloat:
+    case lltok::kw_naked:
+    case lltok::kw_inlinehint:
+    case lltok::kw_alignstack:
+    case lltok::kw_uwtable:
+    case lltok::kw_nonlazybind:
+    case lltok::kw_returns_twice:
+    case lltok::kw_address_safety:
+    case lltok::kw_minsize:
+      if (AttrKind != 2)
+        HaveError |= Error(AttrLoc, "invalid use of function-only attribute");
+      break;
+    }
+
     Lex.Lex();
   }
 }
@@ -990,12 +1029,12 @@ bool LLParser::ParseOptionalAttrs(Attributes &Attrs, unsigned AttrKind) {
 ///   ::= 'private'
 ///   ::= 'linker_private'
 ///   ::= 'linker_private_weak'
-///   ::= 'linker_private_weak_def_auto'
 ///   ::= 'internal'
 ///   ::= 'weak'
 ///   ::= 'weak_odr'
 ///   ::= 'linkonce'
 ///   ::= 'linkonce_odr'
+///   ::= 'linkonce_odr_auto_hide'
 ///   ::= 'available_externally'
 ///   ::= 'appending'
 ///   ::= 'dllexport'
@@ -1012,14 +1051,15 @@ bool LLParser::ParseOptionalLinkage(unsigned &Res, bool &HasLinkage) {
   case lltok::kw_linker_private_weak:
     Res = GlobalValue::LinkerPrivateWeakLinkage;
     break;
-  case lltok::kw_linker_private_weak_def_auto:
-    Res = GlobalValue::LinkerPrivateWeakDefAutoLinkage;
-    break;
   case lltok::kw_internal:       Res = GlobalValue::InternalLinkage;      break;
   case lltok::kw_weak:           Res = GlobalValue::WeakAnyLinkage;       break;
   case lltok::kw_weak_odr:       Res = GlobalValue::WeakODRLinkage;       break;
   case lltok::kw_linkonce:       Res = GlobalValue::LinkOnceAnyLinkage;   break;
   case lltok::kw_linkonce_odr:   Res = GlobalValue::LinkOnceODRLinkage;   break;
+  case lltok::kw_linkonce_odr_auto_hide:
+  case lltok::kw_linker_private_weak_def_auto: // FIXME: For backwards compat.
+    Res = GlobalValue::LinkOnceODRAutoHideLinkage;
+    break;
   case lltok::kw_available_externally:
     Res = GlobalValue::AvailableExternallyLinkage;
     break;
@@ -1056,6 +1096,7 @@ bool LLParser::ParseOptionalVisibility(unsigned &Res) {
 ///   ::= /*empty*/
 ///   ::= 'ccc'
 ///   ::= 'fastcc'
+///   ::= 'kw_intel_ocl_bicc'
 ///   ::= 'coldcc'
 ///   ::= 'x86_stdcallcc'
 ///   ::= 'x86_fastcallcc'
@@ -1066,6 +1107,8 @@ bool LLParser::ParseOptionalVisibility(unsigned &Res) {
 ///   ::= 'msp430_intrcc'
 ///   ::= 'ptx_kernel'
 ///   ::= 'ptx_device'
+///   ::= 'spir_func'
+///   ::= 'spir_kernel'
 ///   ::= 'cc' UINT
 ///
 bool LLParser::ParseOptionalCallingConv(CallingConv::ID &CC) {
@@ -1083,6 +1126,9 @@ bool LLParser::ParseOptionalCallingConv(CallingConv::ID &CC) {
   case lltok::kw_msp430_intrcc:  CC = CallingConv::MSP430_INTR; break;
   case lltok::kw_ptx_kernel:     CC = CallingConv::PTX_Kernel; break;
   case lltok::kw_ptx_device:     CC = CallingConv::PTX_Device; break;
+  case lltok::kw_spir_kernel:    CC = CallingConv::SPIR_KERNEL; break;
+  case lltok::kw_spir_func:      CC = CallingConv::SPIR_FUNC; break;
+  case lltok::kw_intel_ocl_bicc: CC = CallingConv::Intel_OCL_BI; break;
   case lltok::kw_cc: {
       unsigned ArbitraryCC;
       Lex.Lex();
@@ -1395,16 +1441,16 @@ bool LLParser::ParseParameterList(SmallVectorImpl<ParamInfo> &ArgList,
     // Parse the argument.
     LocTy ArgLoc;
     Type *ArgTy = 0;
-    Attributes ArgAttrs1;
-    Attributes ArgAttrs2;
+    AttrBuilder ArgAttrs;
     Value *V;
     if (ParseType(ArgTy, ArgLoc))
       return true;
 
     // Otherwise, handle normal operands.
-    if (ParseOptionalAttrs(ArgAttrs1, 0) || ParseValue(ArgTy, V, PFS))
+    if (ParseOptionalAttrs(ArgAttrs, 0) || ParseValue(ArgTy, V, PFS))
       return true;
-    ArgList.push_back(ParamInfo(ArgLoc, V, ArgAttrs1|ArgAttrs2));
+    ArgList.push_back(ParamInfo(ArgLoc, V, Attributes::get(V->getContext(),
+                                                           ArgAttrs)));
   }
 
   Lex.Lex();  // Lex the ')'.
@@ -1436,7 +1482,7 @@ bool LLParser::ParseArgumentList(SmallVectorImpl<ArgInfo> &ArgList,
   } else {
     LocTy TypeLoc = Lex.getLoc();
     Type *ArgTy = 0;
-    Attributes Attrs;
+    AttrBuilder Attrs;
     std::string Name;
 
     if (ParseType(ArgTy) ||
@@ -1453,7 +1499,9 @@ bool LLParser::ParseArgumentList(SmallVectorImpl<ArgInfo> &ArgList,
     if (!FunctionType::isValidArgumentType(ArgTy))
       return Error(TypeLoc, "invalid type for function argument");
 
-    ArgList.push_back(ArgInfo(TypeLoc, ArgTy, Attrs, Name));
+    ArgList.push_back(ArgInfo(TypeLoc, ArgTy,
+                              Attributes::get(ArgTy->getContext(),
+                                              Attrs), Name));
 
     while (EatIfPresent(lltok::comma)) {
       // Handle ... at end of arg list.
@@ -1479,7 +1527,9 @@ bool LLParser::ParseArgumentList(SmallVectorImpl<ArgInfo> &ArgList,
       if (!ArgTy->isFirstClassType())
         return Error(TypeLoc, "invalid type for function argument");
 
-      ArgList.push_back(ArgInfo(TypeLoc, ArgTy, Attrs, Name));
+      ArgList.push_back(ArgInfo(TypeLoc, ArgTy,
+                                Attributes::get(ArgTy->getContext(), Attrs),
+                                Name));
     }
   }
 
@@ -1503,7 +1553,7 @@ bool LLParser::ParseFunctionType(Type *&Result) {
   for (unsigned i = 0, e = ArgList.size(); i != e; ++i) {
     if (!ArgList[i].Name.empty())
       return Error(ArgList[i].Loc, "argument name invalid in function type");
-    if (ArgList[i].Attrs)
+    if (ArgList[i].Attrs.hasAttributes())
       return Error(ArgList[i].Loc,
                    "argument attributes invalid in function type");
   }
@@ -2069,16 +2119,18 @@ bool LLParser::ParseValID(ValID &ID, PerFunctionState *PFS) {
 
   case lltok::kw_asm: {
     // ValID ::= 'asm' SideEffect? AlignStack? STRINGCONSTANT ',' STRINGCONSTANT
-    bool HasSideEffect, AlignStack;
+    bool HasSideEffect, AlignStack, AsmDialect;
     Lex.Lex();
     if (ParseOptionalToken(lltok::kw_sideeffect, HasSideEffect) ||
         ParseOptionalToken(lltok::kw_alignstack, AlignStack) ||
+        ParseOptionalToken(lltok::kw_inteldialect, AsmDialect) ||
         ParseStringConstant(ID.StrVal) ||
         ParseToken(lltok::comma, "expected comma in inline asm expression") ||
         ParseToken(lltok::StringConstant, "expected constraint string"))
       return true;
     ID.StrVal2 = Lex.getStrVal();
-    ID.UIntVal = unsigned(HasSideEffect) | (unsigned(AlignStack)<<1);
+    ID.UIntVal = unsigned(HasSideEffect) | (unsigned(AlignStack)<<1) |
+      (unsigned(AsmDialect)<<2);
     ID.Kind = ValID::t_InlineAsm;
     return false;
   }
@@ -2495,7 +2547,8 @@ bool LLParser::ConvertValIDToValue(Type *Ty, ValID &ID, Value *&V,
       PTy ? dyn_cast<FunctionType>(PTy->getElementType()) : 0;
     if (!FTy || !InlineAsm::Verify(FTy, ID.StrVal2))
       return Error(ID.Loc, "invalid type for inline asm constraint string");
-    V = InlineAsm::get(FTy, ID.StrVal, ID.StrVal2, ID.UIntVal&1, ID.UIntVal>>1);
+    V = InlineAsm::get(FTy, ID.StrVal, ID.StrVal2, ID.UIntVal&1,
+                       (ID.UIntVal>>1)&1, (InlineAsm::AsmDialect(ID.UIntVal>>2)));
     return false;
   }
   case ValID::t_MDNode:
@@ -2630,7 +2683,7 @@ bool LLParser::ParseFunctionHeader(Function *&Fn, bool isDefine) {
   unsigned Linkage;
 
   unsigned Visibility;
-  Attributes RetAttrs;
+  AttrBuilder RetAttrs;
   CallingConv::ID CC;
   Type *RetType = 0;
   LocTy RetTypeLoc = Lex.getLoc();
@@ -2653,11 +2706,11 @@ bool LLParser::ParseFunctionHeader(Function *&Fn, bool isDefine) {
   case GlobalValue::PrivateLinkage:
   case GlobalValue::LinkerPrivateLinkage:
   case GlobalValue::LinkerPrivateWeakLinkage:
-  case GlobalValue::LinkerPrivateWeakDefAutoLinkage:
   case GlobalValue::InternalLinkage:
   case GlobalValue::AvailableExternallyLinkage:
   case GlobalValue::LinkOnceAnyLinkage:
   case GlobalValue::LinkOnceODRLinkage:
+  case GlobalValue::LinkOnceODRAutoHideLinkage:
   case GlobalValue::WeakAnyLinkage:
   case GlobalValue::WeakODRLinkage:
   case GlobalValue::DLLExportLinkage:
@@ -2694,7 +2747,7 @@ bool LLParser::ParseFunctionHeader(Function *&Fn, bool isDefine) {
 
   SmallVector<ArgInfo, 8> ArgList;
   bool isVarArg;
-  Attributes FuncAttrs;
+  AttrBuilder FuncAttrs;
   std::string Section;
   unsigned Alignment;
   std::string GC;
@@ -2713,9 +2766,9 @@ bool LLParser::ParseFunctionHeader(Function *&Fn, bool isDefine) {
     return true;
 
   // If the alignment was parsed as an attribute, move to the alignment field.
-  if (FuncAttrs & Attribute::Alignment) {
-    Alignment = Attribute::getAlignmentFromAttrs(FuncAttrs);
-    FuncAttrs &= ~Attribute::Alignment;
+  if (FuncAttrs.hasAlignmentAttr()) {
+    Alignment = FuncAttrs.getAlignment();
+    FuncAttrs.removeAttribute(Attributes::Alignment);
   }
 
   // Okay, if we got here, the function is syntactically valid.  Convert types
@@ -2723,21 +2776,28 @@ bool LLParser::ParseFunctionHeader(Function *&Fn, bool isDefine) {
   std::vector<Type*> ParamTypeList;
   SmallVector<AttributeWithIndex, 8> Attrs;
 
-  if (RetAttrs != Attribute::None)
-    Attrs.push_back(AttributeWithIndex::get(0, RetAttrs));
+  if (RetAttrs.hasAttributes())
+    Attrs.push_back(
+      AttributeWithIndex::get(AttrListPtr::ReturnIndex,
+                              Attributes::get(RetType->getContext(),
+                                              RetAttrs)));
 
   for (unsigned i = 0, e = ArgList.size(); i != e; ++i) {
     ParamTypeList.push_back(ArgList[i].Ty);
-    if (ArgList[i].Attrs != Attribute::None)
+    if (ArgList[i].Attrs.hasAttributes())
       Attrs.push_back(AttributeWithIndex::get(i+1, ArgList[i].Attrs));
   }
 
-  if (FuncAttrs != Attribute::None)
-    Attrs.push_back(AttributeWithIndex::get(~0, FuncAttrs));
+  if (FuncAttrs.hasAttributes())
+    Attrs.push_back(
+      AttributeWithIndex::get(AttrListPtr::FunctionIndex,
+                              Attributes::get(RetType->getContext(),
+                                              FuncAttrs)));
 
-  AttrListPtr PAL = AttrListPtr::get(Attrs);
+  AttrListPtr PAL = AttrListPtr::get(Context, Attrs);
 
-  if (PAL.paramHasAttr(1, Attribute::StructRet) && !RetType->isVoidTy())
+  if (PAL.getParamAttributes(1).hasAttribute(Attributes::StructRet) &&
+      !RetType->isVoidTy())
     return Error(RetTypeLoc, "functions with 'sret' argument must return void");
 
   FunctionType *FT =
@@ -2752,6 +2812,9 @@ bool LLParser::ParseFunctionHeader(Function *&Fn, bool isDefine) {
       ForwardRefVals.find(FunctionName);
     if (FRVI != ForwardRefVals.end()) {
       Fn = M->getFunction(FunctionName);
+      if (!Fn)
+        return Error(FRVI->second.second, "invalid forward reference to "
+                     "function as global value!");
       if (Fn->getType() != PFT)
         return Error(FRVI->second.second, "invalid forward reference to "
                      "function '" + FunctionName + "' with wrong type!");
@@ -3205,7 +3268,7 @@ bool LLParser::ParseIndirectBr(Instruction *&Inst, PerFunctionState &PFS) {
 ///       OptionalAttrs 'to' TypeAndValue 'unwind' TypeAndValue
 bool LLParser::ParseInvoke(Instruction *&Inst, PerFunctionState &PFS) {
   LocTy CallLoc = Lex.getLoc();
-  Attributes RetAttrs, FnAttrs;
+  AttrBuilder RetAttrs, FnAttrs;
   CallingConv::ID CC;
   Type *RetType = 0;
   LocTy RetTypeLoc;
@@ -3250,8 +3313,11 @@ bool LLParser::ParseInvoke(Instruction *&Inst, PerFunctionState &PFS) {
 
   // Set up the Attributes for the function.
   SmallVector<AttributeWithIndex, 8> Attrs;
-  if (RetAttrs != Attribute::None)
-    Attrs.push_back(AttributeWithIndex::get(0, RetAttrs));
+  if (RetAttrs.hasAttributes())
+    Attrs.push_back(
+      AttributeWithIndex::get(AttrListPtr::ReturnIndex,
+                              Attributes::get(Callee->getContext(),
+                                              RetAttrs)));
 
   SmallVector<Value*, 8> Args;
 
@@ -3271,18 +3337,21 @@ bool LLParser::ParseInvoke(Instruction *&Inst, PerFunctionState &PFS) {
       return Error(ArgList[i].Loc, "argument is not of expected type '" +
                    getTypeString(ExpectedTy) + "'");
     Args.push_back(ArgList[i].V);
-    if (ArgList[i].Attrs != Attribute::None)
+    if (ArgList[i].Attrs.hasAttributes())
       Attrs.push_back(AttributeWithIndex::get(i+1, ArgList[i].Attrs));
   }
 
   if (I != E)
     return Error(CallLoc, "not enough parameters specified for call");
 
-  if (FnAttrs != Attribute::None)
-    Attrs.push_back(AttributeWithIndex::get(~0, FnAttrs));
+  if (FnAttrs.hasAttributes())
+    Attrs.push_back(
+      AttributeWithIndex::get(AttrListPtr::FunctionIndex,
+                              Attributes::get(Callee->getContext(),
+                                              FnAttrs)));
 
   // Finish off the Attributes and check them
-  AttrListPtr PAL = AttrListPtr::get(Attrs);
+  AttrListPtr PAL = AttrListPtr::get(Context, Attrs);
 
   InvokeInst *II = InvokeInst::Create(Callee, NormalBB, UnwindBB, Args);
   II->setCallingConv(CC);
@@ -3604,7 +3673,7 @@ bool LLParser::ParseLandingPad(Instruction *&Inst, PerFunctionState &PFS) {
 ///       ParameterList OptionalAttrs
 bool LLParser::ParseCall(Instruction *&Inst, PerFunctionState &PFS,
                          bool isTail) {
-  Attributes RetAttrs, FnAttrs;
+  AttrBuilder RetAttrs, FnAttrs;
   CallingConv::ID CC;
   Type *RetType = 0;
   LocTy RetTypeLoc;
@@ -3646,8 +3715,11 @@ bool LLParser::ParseCall(Instruction *&Inst, PerFunctionState &PFS,
 
   // Set up the Attributes for the function.
   SmallVector<AttributeWithIndex, 8> Attrs;
-  if (RetAttrs != Attribute::None)
-    Attrs.push_back(AttributeWithIndex::get(0, RetAttrs));
+  if (RetAttrs.hasAttributes())
+    Attrs.push_back(
+      AttributeWithIndex::get(AttrListPtr::ReturnIndex,
+                              Attributes::get(Callee->getContext(),
+                                              RetAttrs)));
 
   SmallVector<Value*, 8> Args;
 
@@ -3667,18 +3739,21 @@ bool LLParser::ParseCall(Instruction *&Inst, PerFunctionState &PFS,
       return Error(ArgList[i].Loc, "argument is not of expected type '" +
                    getTypeString(ExpectedTy) + "'");
     Args.push_back(ArgList[i].V);
-    if (ArgList[i].Attrs != Attribute::None)
+    if (ArgList[i].Attrs.hasAttributes())
       Attrs.push_back(AttributeWithIndex::get(i+1, ArgList[i].Attrs));
   }
 
   if (I != E)
     return Error(CallLoc, "not enough parameters specified for call");
 
-  if (FnAttrs != Attribute::None)
-    Attrs.push_back(AttributeWithIndex::get(~0, FnAttrs));
+  if (FnAttrs.hasAttributes())
+    Attrs.push_back(
+      AttributeWithIndex::get(AttrListPtr::FunctionIndex,
+                              Attributes::get(Callee->getContext(),
+                                              FnAttrs)));
 
   // Finish off the Attributes and check them
-  AttrListPtr PAL = AttrListPtr::get(Attrs);
+  AttrListPtr PAL = AttrListPtr::get(Context, Attrs);
 
   CallInst *CI = CallInst::Create(Callee, Args);
   CI->setTailCall(isTail);
diff --git a/lib/AsmParser/LLParser.h b/lib/AsmParser/LLParser.h
index 257c726..c6bbdb2 100644
--- a/lib/AsmParser/LLParser.h
+++ b/lib/AsmParser/LLParser.h
@@ -175,7 +175,7 @@ namespace llvm {
     bool ParseTLSModel(GlobalVariable::ThreadLocalMode &TLM);
     bool ParseOptionalThreadLocal(GlobalVariable::ThreadLocalMode &TLM);
     bool ParseOptionalAddrSpace(unsigned &AddrSpace);
-    bool ParseOptionalAttrs(Attributes &Attrs, unsigned AttrKind);
+    bool ParseOptionalAttrs(AttrBuilder &Attrs, unsigned AttrKind);
     bool ParseOptionalLinkage(unsigned &Linkage, bool &HasLinkage);
     bool ParseOptionalLinkage(unsigned &Linkage) {
       bool HasLinkage; return ParseOptionalLinkage(Linkage, HasLinkage);
diff --git a/lib/AsmParser/LLToken.h b/lib/AsmParser/LLToken.h
index 0b0b980..036686d 100644
--- a/lib/AsmParser/LLToken.h
+++ b/lib/AsmParser/LLToken.h
@@ -37,8 +37,10 @@ namespace lltok {
     kw_global,  kw_constant,
 
     kw_private, kw_linker_private, kw_linker_private_weak,
-    kw_linker_private_weak_def_auto, kw_internal,
-    kw_linkonce, kw_linkonce_odr, kw_weak, kw_weak_odr, kw_appending,
+    kw_linker_private_weak_def_auto, // FIXME: For backwards compatibility.
+    kw_internal,
+    kw_linkonce, kw_linkonce_odr, kw_linkonce_odr_auto_hide,
+    kw_weak, kw_weak_odr, kw_appending,
     kw_dllimport, kw_dllexport, kw_common, kw_available_externally,
     kw_default, kw_hidden, kw_protected,
     kw_unnamed_addr,
@@ -70,14 +72,17 @@ namespace lltok {
     kw_asm,
     kw_sideeffect,
     kw_alignstack,
+    kw_inteldialect,
     kw_gc,
     kw_c,
 
     kw_cc, kw_ccc, kw_fastcc, kw_coldcc,
+	  kw_intel_ocl_bicc,
     kw_x86_stdcallcc, kw_x86_fastcallcc, kw_x86_thiscallcc,
     kw_arm_apcscc, kw_arm_aapcscc, kw_arm_aapcs_vfpcc,
     kw_msp430_intrcc,
     kw_ptx_kernel, kw_ptx_device,
+    kw_spir_kernel, kw_spir_func,
 
     kw_signext,
     kw_zeroext,
@@ -105,7 +110,7 @@ namespace lltok {
     kw_naked,
     kw_nonlazybind,
     kw_address_safety,
-    kw_ia_nsdialect,
+    kw_minsize,
 
     kw_type,
     kw_opaque,
diff --git a/lib/Bitcode/Reader/BitcodeReader.cpp b/lib/Bitcode/Reader/BitcodeReader.cpp
index 4ffee38..4ec9da1 100644
--- a/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -52,6 +52,8 @@ void BitcodeReader::FreeState() {
   std::vector<Function*>().swap(FunctionsWithBodies);
   DeferredFunctionInfo.clear();
   MDKindMap.clear();
+
+  assert(BlockAddrFwdRefs.empty() && "Unresolved blockaddress fwd references");
 }
 
 //===----------------------------------------------------------------------===//
@@ -89,7 +91,7 @@ static GlobalValue::LinkageTypes GetDecodedLinkage(unsigned Val) {
   case 12: return GlobalValue::AvailableExternallyLinkage;
   case 13: return GlobalValue::LinkerPrivateLinkage;
   case 14: return GlobalValue::LinkerPrivateWeakLinkage;
-  case 15: return GlobalValue::LinkerPrivateWeakDefAutoLinkage;
+  case 15: return GlobalValue::LinkOnceODRAutoHideLinkage;
   }
 }
 
@@ -197,7 +199,7 @@ namespace {
   /// @brief A class for maintaining the slot number definition
   /// as a placeholder for the actual definition for forward constants defs.
   class ConstantPlaceHolder : public ConstantExpr {
-    void operator=(const ConstantPlaceHolder &); // DO NOT IMPLEMENT
+    void operator=(const ConstantPlaceHolder &) LLVM_DELETED_FUNCTION;
   public:
     // allocate space for exactly one operand
     void *operator new(size_t s) {
@@ -209,7 +211,6 @@ namespace {
     }
 
     /// @brief Methods to support type inquiry through isa, cast, and dyn_cast.
-    //static inline bool classof(const ConstantPlaceHolder *) { return true; }
     static bool classof(const Value *V) {
       return isa<ConstantExpr>(V) &&
              cast<ConstantExpr>(V)->getOpcode() == Instruction::UserOp1;
@@ -475,17 +476,18 @@ bool BitcodeReader::ParseAttributeBlock() {
 
       for (unsigned i = 0, e = Record.size(); i != e; i += 2) {
         Attributes ReconstitutedAttr =
-          Attribute::decodeLLVMAttributesForBitcode(Record[i+1]);
+          Attributes::decodeLLVMAttributesForBitcode(Context, Record[i+1]);
         Record[i+1] = ReconstitutedAttr.Raw();
       }
 
       for (unsigned i = 0, e = Record.size(); i != e; i += 2) {
-        if (Attributes(Record[i+1]) != Attribute::None)
+        AttrBuilder B(Record[i+1]);
+        if (B.hasAttributes())
           Attrs.push_back(AttributeWithIndex::get(Record[i],
-                                                  Attributes(Record[i+1])));
+                                                  Attributes::get(Context, B)));
       }
 
-      MAttributes.push_back(AttrListPtr::get(Attrs));
+      MAttributes.push_back(AttrListPtr::get(Context, Attrs));
       Attrs.clear();
       break;
     }
@@ -889,9 +891,9 @@ bool BitcodeReader::ParseMetadata() {
   }
 }
 
-/// DecodeSignRotatedValue - Decode a signed value stored with the sign bit in
+/// decodeSignRotatedValue - Decode a signed value stored with the sign bit in
 /// the LSB for dense VBR encoding.
-static uint64_t DecodeSignRotatedValue(uint64_t V) {
+uint64_t BitcodeReader::decodeSignRotatedValue(uint64_t V) {
   if ((V & 1) == 0)
     return V >> 1;
   if (V != 1)
@@ -941,7 +943,7 @@ bool BitcodeReader::ResolveGlobalAndAliasInits() {
 static APInt ReadWideAPInt(ArrayRef<uint64_t> Vals, unsigned TypeBits) {
   SmallVector<uint64_t, 8> Words(Vals.size());
   std::transform(Vals.begin(), Vals.end(), Words.begin(),
-                 DecodeSignRotatedValue);
+                 BitcodeReader::decodeSignRotatedValue);
 
   return APInt(TypeBits, Words);
 }
@@ -995,7 +997,7 @@ bool BitcodeReader::ParseConstants() {
     case bitc::CST_CODE_INTEGER:   // INTEGER: [intval]
       if (!CurTy->isIntegerTy() || Record.empty())
         return Error("Invalid CST_INTEGER record");
-      V = ConstantInt::get(CurTy, DecodeSignRotatedValue(Record[0]));
+      V = ConstantInt::get(CurTy, decodeSignRotatedValue(Record[0]));
       break;
     case bitc::CST_CODE_WIDE_INTEGER: {// WIDE_INTEGER: [n x intval]
       if (!CurTy->isIntegerTy() || Record.empty())
@@ -1245,7 +1247,9 @@ bool BitcodeReader::ParseConstants() {
         V = ConstantExpr::getICmp(Record[3], Op0, Op1);
       break;
     }
-    case bitc::CST_CODE_INLINEASM: {
+    // This maintains backward compatibility, pre-asm dialect keywords.
+    // FIXME: Remove with the 4.0 release.
+    case bitc::CST_CODE_INLINEASM_OLD: {
       if (Record.size() < 2) return Error("Invalid INLINEASM record");
       std::string AsmStr, ConstrStr;
       bool HasSideEffects = Record[0] & 1;
@@ -1266,6 +1270,31 @@ bool BitcodeReader::ParseConstants() {
                          AsmStr, ConstrStr, HasSideEffects, IsAlignStack);
       break;
     }
+    // This version adds support for the asm dialect keywords (e.g.,
+    // inteldialect).
+    case bitc::CST_CODE_INLINEASM: {
+      if (Record.size() < 2) return Error("Invalid INLINEASM record");
+      std::string AsmStr, ConstrStr;
+      bool HasSideEffects = Record[0] & 1;
+      bool IsAlignStack = (Record[0] >> 1) & 1;
+      unsigned AsmDialect = Record[0] >> 2;
+      unsigned AsmStrSize = Record[1];
+      if (2+AsmStrSize >= Record.size())
+        return Error("Invalid INLINEASM record");
+      unsigned ConstStrSize = Record[2+AsmStrSize];
+      if (3+AsmStrSize+ConstStrSize > Record.size())
+        return Error("Invalid INLINEASM record");
+
+      for (unsigned i = 0; i != AsmStrSize; ++i)
+        AsmStr += (char)Record[2+i];
+      for (unsigned i = 0; i != ConstStrSize; ++i)
+        ConstrStr += (char)Record[3+AsmStrSize+i];
+      PointerType *PTy = cast<PointerType>(CurTy);
+      V = InlineAsm::get(cast<FunctionType>(PTy->getElementType()),
+                         AsmStr, ConstrStr, HasSideEffects, IsAlignStack,
+                         InlineAsm::AsmDialect(AsmDialect));
+      break;
+    }
     case bitc::CST_CODE_BLOCKADDRESS:{
       if (Record.size() < 3) return Error("Invalid CE_BLOCKADDRESS record");
       Type *FnTy = getTypeByID(Record[0]);
@@ -1273,13 +1302,27 @@ bool BitcodeReader::ParseConstants() {
       Function *Fn =
         dyn_cast_or_null<Function>(ValueList.getConstantFwdRef(Record[1],FnTy));
       if (Fn == 0) return Error("Invalid CE_BLOCKADDRESS record");
-      
-      GlobalVariable *FwdRef = new GlobalVariable(*Fn->getParent(),
-                                                  Type::getInt8Ty(Context),
+
+      // If the function is already parsed we can insert the block address right
+      // away.
+      if (!Fn->empty()) {
+        Function::iterator BBI = Fn->begin(), BBE = Fn->end();
+        for (size_t I = 0, E = Record[2]; I != E; ++I) {
+          if (BBI == BBE)
+            return Error("Invalid blockaddress block #");
+          ++BBI;
+        }
+        V = BlockAddress::get(Fn, BBI);
+      } else {
+        // Otherwise insert a placeholder and remember it so it can be inserted
+        // when the function is parsed.
+        GlobalVariable *FwdRef = new GlobalVariable(*Fn->getParent(),
+                                                    Type::getInt8Ty(Context),
                                             false, GlobalValue::InternalLinkage,
-                                                  0, "");
-      BlockAddrFwdRefs[Fn].push_back(std::make_pair(Record[2], FwdRef));
-      V = FwdRef;
+                                                    0, "");
+        BlockAddrFwdRefs[Fn].push_back(std::make_pair(Record[2], FwdRef));
+        V = FwdRef;
+      }
       break;
     }  
     }
@@ -1481,13 +1524,22 @@ bool BitcodeReader::ParseModule(bool Resume) {
     // Read a record.
     switch (Stream.ReadRecord(Code, Record)) {
     default: break;  // Default behavior, ignore unknown content.
-    case bitc::MODULE_CODE_VERSION:  // VERSION: [version#]
+    case bitc::MODULE_CODE_VERSION: {  // VERSION: [version#]
       if (Record.size() < 1)
         return Error("Malformed MODULE_CODE_VERSION");
-      // Only version #0 is supported so far.
-      if (Record[0] != 0)
-        return Error("Unknown bitstream version!");
+      // Only version #0 and #1 are supported so far.
+      unsigned module_version = Record[0];
+      switch (module_version) {
+        default: return Error("Unknown bitstream version!");
+        case 0:
+          UseRelativeIDs = false;
+          break;
+        case 1:
+          UseRelativeIDs = true;
+          break;
+      }
       break;
+    }
     case bitc::MODULE_CODE_TRIPLE: {  // TRIPLE: [strchr x N]
       std::string S;
       if (ConvertToString(Record, 0, S))
@@ -1754,13 +1806,6 @@ bool BitcodeReader::ParseModuleTriple(std::string &Triple) {
     // Read a record.
     switch (Stream.ReadRecord(Code, Record)) {
     default: break;  // Default behavior, ignore unknown content.
-    case bitc::MODULE_CODE_VERSION:  // VERSION: [version#]
-      if (Record.size() < 1)
-        return Error("Malformed MODULE_CODE_VERSION");
-      // Only version #0 is supported so far.
-      if (Record[0] != 0)
-        return Error("Unknown bitstream version!");
-      break;
     case bitc::MODULE_CODE_TRIPLE: {  // TRIPLE: [strchr x N]
       std::string S;
       if (ConvertToString(Record, 0, S))
@@ -1973,7 +2018,7 @@ bool BitcodeReader::ParseFunctionBody(Function *F) {
       unsigned OpNum = 0;
       Value *LHS, *RHS;
       if (getValueTypePair(Record, OpNum, NextValueNo, LHS) ||
-          getValue(Record, OpNum, LHS->getType(), RHS) ||
+          popValue(Record, OpNum, NextValueNo, LHS->getType(), RHS) ||
           OpNum+1 > Record.size())
         return Error("Invalid BINOP record");
 
@@ -2088,8 +2133,8 @@ bool BitcodeReader::ParseFunctionBody(Function *F) {
       unsigned OpNum = 0;
       Value *TrueVal, *FalseVal, *Cond;
       if (getValueTypePair(Record, OpNum, NextValueNo, TrueVal) ||
-          getValue(Record, OpNum, TrueVal->getType(), FalseVal) ||
-          getValue(Record, OpNum, Type::getInt1Ty(Context), Cond))
+          popValue(Record, OpNum, NextValueNo, TrueVal->getType(), FalseVal) ||
+          popValue(Record, OpNum, NextValueNo, Type::getInt1Ty(Context), Cond))
         return Error("Invalid SELECT record");
 
       I = SelectInst::Create(Cond, TrueVal, FalseVal);
@@ -2103,7 +2148,7 @@ bool BitcodeReader::ParseFunctionBody(Function *F) {
       unsigned OpNum = 0;
       Value *TrueVal, *FalseVal, *Cond;
       if (getValueTypePair(Record, OpNum, NextValueNo, TrueVal) ||
-          getValue(Record, OpNum, TrueVal->getType(), FalseVal) ||
+          popValue(Record, OpNum, NextValueNo, TrueVal->getType(), FalseVal) ||
           getValueTypePair(Record, OpNum, NextValueNo, Cond))
         return Error("Invalid SELECT record");
 
@@ -2128,7 +2173,7 @@ bool BitcodeReader::ParseFunctionBody(Function *F) {
       unsigned OpNum = 0;
       Value *Vec, *Idx;
       if (getValueTypePair(Record, OpNum, NextValueNo, Vec) ||
-          getValue(Record, OpNum, Type::getInt32Ty(Context), Idx))
+          popValue(Record, OpNum, NextValueNo, Type::getInt32Ty(Context), Idx))
         return Error("Invalid EXTRACTELT record");
       I = ExtractElementInst::Create(Vec, Idx);
       InstructionList.push_back(I);
@@ -2139,9 +2184,9 @@ bool BitcodeReader::ParseFunctionBody(Function *F) {
       unsigned OpNum = 0;
       Value *Vec, *Elt, *Idx;
       if (getValueTypePair(Record, OpNum, NextValueNo, Vec) ||
-          getValue(Record, OpNum,
+          popValue(Record, OpNum, NextValueNo,
                    cast<VectorType>(Vec->getType())->getElementType(), Elt) ||
-          getValue(Record, OpNum, Type::getInt32Ty(Context), Idx))
+          popValue(Record, OpNum, NextValueNo, Type::getInt32Ty(Context), Idx))
         return Error("Invalid INSERTELT record");
       I = InsertElementInst::Create(Vec, Elt, Idx);
       InstructionList.push_back(I);
@@ -2152,7 +2197,7 @@ bool BitcodeReader::ParseFunctionBody(Function *F) {
       unsigned OpNum = 0;
       Value *Vec1, *Vec2, *Mask;
       if (getValueTypePair(Record, OpNum, NextValueNo, Vec1) ||
-          getValue(Record, OpNum, Vec1->getType(), Vec2))
+          popValue(Record, OpNum, NextValueNo, Vec1->getType(), Vec2))
         return Error("Invalid SHUFFLEVEC record");
 
       if (getValueTypePair(Record, OpNum, NextValueNo, Mask))
@@ -2172,7 +2217,7 @@ bool BitcodeReader::ParseFunctionBody(Function *F) {
       unsigned OpNum = 0;
       Value *LHS, *RHS;
       if (getValueTypePair(Record, OpNum, NextValueNo, LHS) ||
-          getValue(Record, OpNum, LHS->getType(), RHS) ||
+          popValue(Record, OpNum, NextValueNo, LHS->getType(), RHS) ||
           OpNum+1 != Record.size())
         return Error("Invalid CMP record");
 
@@ -2217,7 +2262,8 @@ bool BitcodeReader::ParseFunctionBody(Function *F) {
       }
       else {
         BasicBlock *FalseDest = getBasicBlock(Record[1]);
-        Value *Cond = getFnValueByID(Record[2], Type::getInt1Ty(Context));
+        Value *Cond = getValue(Record, 2, NextValueNo,
+                               Type::getInt1Ty(Context));
         if (FalseDest == 0 || Cond == 0)
           return Error("Invalid BR record");
         I = BranchInst::Create(TrueDest, FalseDest, Cond);
@@ -2233,7 +2279,7 @@ bool BitcodeReader::ParseFunctionBody(Function *F) {
         Type *OpTy = getTypeByID(Record[1]);
         unsigned ValueBitWidth = cast<IntegerType>(OpTy)->getBitWidth();
 
-        Value *Cond = getFnValueByID(Record[2], OpTy);
+        Value *Cond = getValue(Record, 2, NextValueNo, OpTy);
         BasicBlock *Default = getBasicBlock(Record[3]);
         if (OpTy == 0 || Cond == 0 || Default == 0)
           return Error("Invalid SWITCH record");
@@ -2288,7 +2334,7 @@ bool BitcodeReader::ParseFunctionBody(Function *F) {
       if (Record.size() < 3 || (Record.size() & 1) == 0)
         return Error("Invalid SWITCH record");
       Type *OpTy = getTypeByID(Record[0]);
-      Value *Cond = getFnValueByID(Record[1], OpTy);
+      Value *Cond = getValue(Record, 1, NextValueNo, OpTy);
       BasicBlock *Default = getBasicBlock(Record[2]);
       if (OpTy == 0 || Cond == 0 || Default == 0)
         return Error("Invalid SWITCH record");
@@ -2312,7 +2358,7 @@ bool BitcodeReader::ParseFunctionBody(Function *F) {
       if (Record.size() < 2)
         return Error("Invalid INDIRECTBR record");
       Type *OpTy = getTypeByID(Record[0]);
-      Value *Address = getFnValueByID(Record[1], OpTy);
+      Value *Address = getValue(Record, 1, NextValueNo, OpTy);
       if (OpTy == 0 || Address == 0)
         return Error("Invalid INDIRECTBR record");
       unsigned NumDests = Record.size()-2;
@@ -2354,7 +2400,8 @@ bool BitcodeReader::ParseFunctionBody(Function *F) {
 
       SmallVector<Value*, 16> Ops;
       for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i, ++OpNum) {
-        Ops.push_back(getFnValueByID(Record[OpNum], FTy->getParamType(i)));
+        Ops.push_back(getValue(Record, OpNum, NextValueNo,
+                               FTy->getParamType(i)));
         if (Ops.back() == 0) return Error("Invalid INVOKE record");
       }
 
@@ -2401,7 +2448,14 @@ bool BitcodeReader::ParseFunctionBody(Function *F) {
       InstructionList.push_back(PN);
 
       for (unsigned i = 0, e = Record.size()-1; i != e; i += 2) {
-        Value *V = getFnValueByID(Record[1+i], Ty);
+        Value *V;
+        // With the new function encoding, it is possible that operands have
+        // negative IDs (for forward references).  Use a signed VBR
+        // representation to keep the encoding small.
+        if (UseRelativeIDs)
+          V = getValueSigned(Record, 1+i, NextValueNo, Ty);
+        else
+          V = getValue(Record, 1+i, NextValueNo, Ty);
         BasicBlock *BB = getBasicBlock(Record[2+i]);
         if (!V || !BB) return Error("Invalid PHI record");
         PN->addIncoming(V, BB);
@@ -2499,7 +2553,7 @@ bool BitcodeReader::ParseFunctionBody(Function *F) {
       unsigned OpNum = 0;
       Value *Val, *Ptr;
       if (getValueTypePair(Record, OpNum, NextValueNo, Ptr) ||
-          getValue(Record, OpNum,
+          popValue(Record, OpNum, NextValueNo,
                     cast<PointerType>(Ptr->getType())->getElementType(), Val) ||
           OpNum+2 != Record.size())
         return Error("Invalid STORE record");
@@ -2513,7 +2567,7 @@ bool BitcodeReader::ParseFunctionBody(Function *F) {
       unsigned OpNum = 0;
       Value *Val, *Ptr;
       if (getValueTypePair(Record, OpNum, NextValueNo, Ptr) ||
-          getValue(Record, OpNum,
+          popValue(Record, OpNum, NextValueNo,
                     cast<PointerType>(Ptr->getType())->getElementType(), Val) ||
           OpNum+4 != Record.size())
         return Error("Invalid STOREATOMIC record");
@@ -2536,9 +2590,9 @@ bool BitcodeReader::ParseFunctionBody(Function *F) {
       unsigned OpNum = 0;
       Value *Ptr, *Cmp, *New;
       if (getValueTypePair(Record, OpNum, NextValueNo, Ptr) ||
-          getValue(Record, OpNum,
+          popValue(Record, OpNum, NextValueNo,
                     cast<PointerType>(Ptr->getType())->getElementType(), Cmp) ||
-          getValue(Record, OpNum,
+          popValue(Record, OpNum, NextValueNo,
                     cast<PointerType>(Ptr->getType())->getElementType(), New) ||
           OpNum+3 != Record.size())
         return Error("Invalid CMPXCHG record");
@@ -2556,7 +2610,7 @@ bool BitcodeReader::ParseFunctionBody(Function *F) {
       unsigned OpNum = 0;
       Value *Ptr, *Val;
       if (getValueTypePair(Record, OpNum, NextValueNo, Ptr) ||
-          getValue(Record, OpNum,
+          popValue(Record, OpNum, NextValueNo,
                     cast<PointerType>(Ptr->getType())->getElementType(), Val) ||
           OpNum+4 != Record.size())
         return Error("Invalid ATOMICRMW record");
@@ -2610,7 +2664,8 @@ bool BitcodeReader::ParseFunctionBody(Function *F) {
         if (FTy->getParamType(i)->isLabelTy())
           Args.push_back(getBasicBlock(Record[OpNum]));
         else
-          Args.push_back(getFnValueByID(Record[OpNum], FTy->getParamType(i)));
+          Args.push_back(getValue(Record, OpNum, NextValueNo,
+                                  FTy->getParamType(i)));
         if (Args.back() == 0) return Error("Invalid CALL record");
       }
 
@@ -2639,7 +2694,7 @@ bool BitcodeReader::ParseFunctionBody(Function *F) {
       if (Record.size() < 3)
         return Error("Invalid VAARG record");
       Type *OpTy = getTypeByID(Record[0]);
-      Value *Op = getFnValueByID(Record[1], OpTy);
+      Value *Op = getValue(Record, 1, NextValueNo, OpTy);
       Type *ResTy = getTypeByID(Record[2]);
       if (!OpTy || !Op || !ResTy)
         return Error("Invalid VAARG record");
@@ -2837,7 +2892,7 @@ bool BitcodeReader::InitStream() {
 }
 
 bool BitcodeReader::InitStreamFromBuffer() {
-  const unsigned char *BufPtr = (unsigned char *)Buffer->getBufferStart();
+  const unsigned char *BufPtr = (const unsigned char*)Buffer->getBufferStart();
   const unsigned char *BufEnd = BufPtr+Buffer->getBufferSize();
 
   if (Buffer->getBufferSize() & 3) {
diff --git a/lib/Bitcode/Reader/BitcodeReader.h b/lib/Bitcode/Reader/BitcodeReader.h
index e7c4e94..3d5c0eb 100644
--- a/lib/Bitcode/Reader/BitcodeReader.h
+++ b/lib/Bitcode/Reader/BitcodeReader.h
@@ -179,18 +179,27 @@ class BitcodeReader : public GVMaterializer {
   typedef std::pair<unsigned, GlobalVariable*> BlockAddrRefTy;
   DenseMap<Function*, std::vector<BlockAddrRefTy> > BlockAddrFwdRefs;
 
+  /// UseRelativeIDs - Indicates that we are using a new encoding for
+  /// instruction operands where most operands in the current
+  /// FUNCTION_BLOCK are encoded relative to the instruction number,
+  /// for a more compact encoding.  Some instruction operands are not
+  /// relative to the instruction ID: basic block numbers, and types.
+  /// Once the old style function blocks have been phased out, we would
+  /// not need this flag.
+  bool UseRelativeIDs;
+
 public:
   explicit BitcodeReader(MemoryBuffer *buffer, LLVMContext &C)
     : Context(C), TheModule(0), Buffer(buffer), BufferOwned(false),
       LazyStreamer(0), NextUnreadBit(0), SeenValueSymbolTable(false),
       ErrorString(0), ValueList(C), MDValueList(C),
-      SeenFirstFunctionBody(false) {
+      SeenFirstFunctionBody(false), UseRelativeIDs(false) {
   }
   explicit BitcodeReader(DataStreamer *streamer, LLVMContext &C)
     : Context(C), TheModule(0), Buffer(0), BufferOwned(false),
       LazyStreamer(streamer), NextUnreadBit(0), SeenValueSymbolTable(false),
       ErrorString(0), ValueList(C), MDValueList(C),
-      SeenFirstFunctionBody(false) {
+      SeenFirstFunctionBody(false), UseRelativeIDs(false) {
   }
   ~BitcodeReader() {
     FreeState();
@@ -223,6 +232,9 @@ public:
   /// @brief Cheap mechanism to just extract module triple
   /// @returns true if an error occurred.
   bool ParseTriple(std::string &Triple);
+
+  static uint64_t decodeSignRotatedValue(uint64_t V);
+
 private:
   Type *getTypeByID(unsigned ID);
   Value *getFnValueByID(unsigned ID, Type *Ty) {
@@ -247,6 +259,9 @@ private:
                         unsigned InstNum, Value *&ResVal) {
     if (Slot == Record.size()) return true;
     unsigned ValNo = (unsigned)Record[Slot++];
+    // Adjust the ValNo, if it was encoded relative to the InstNum.
+    if (UseRelativeIDs)
+      ValNo = InstNum - ValNo;
     if (ValNo < InstNum) {
       // If this is not a forward reference, just return the value we already
       // have.
@@ -255,20 +270,54 @@ private:
     } else if (Slot == Record.size()) {
       return true;
     }
-    
+
     unsigned TypeNo = (unsigned)Record[Slot++];
     ResVal = getFnValueByID(ValNo, getTypeByID(TypeNo));
     return ResVal == 0;
   }
-  bool getValue(SmallVector<uint64_t, 64> &Record, unsigned &Slot,
-                Type *Ty, Value *&ResVal) {
-    if (Slot == Record.size()) return true;
-    unsigned ValNo = (unsigned)Record[Slot++];
-    ResVal = getFnValueByID(ValNo, Ty);
+
+  /// popValue - Read a value out of the specified record from slot 'Slot'.
+  /// Increment Slot past the number of slots used by the value in the record.
+  /// Return true if there is an error.
+  bool popValue(SmallVector<uint64_t, 64> &Record, unsigned &Slot,
+                unsigned InstNum, Type *Ty, Value *&ResVal) {
+    if (getValue(Record, Slot, InstNum, Ty, ResVal))
+      return true;
+    // All values currently take a single record slot.
+    ++Slot;
+    return false;
+  }
+
+  /// getValue -- Like popValue, but does not increment the Slot number.
+  bool getValue(SmallVector<uint64_t, 64> &Record, unsigned Slot,
+                unsigned InstNum, Type *Ty, Value *&ResVal) {
+    ResVal = getValue(Record, Slot, InstNum, Ty);
     return ResVal == 0;
   }
 
-  
+  /// getValue -- Version of getValue that returns ResVal directly,
+  /// or 0 if there is an error.
+  Value *getValue(SmallVector<uint64_t, 64> &Record, unsigned Slot,
+                  unsigned InstNum, Type *Ty) {
+    if (Slot == Record.size()) return 0;
+    unsigned ValNo = (unsigned)Record[Slot];
+    // Adjust the ValNo, if it was encoded relative to the InstNum.
+    if (UseRelativeIDs)
+      ValNo = InstNum - ValNo;
+    return getFnValueByID(ValNo, Ty);
+  }
+
+  /// getValueSigned -- Like getValue, but decodes signed VBRs.
+  Value *getValueSigned(SmallVector<uint64_t, 64> &Record, unsigned Slot,
+                        unsigned InstNum, Type *Ty) {
+    if (Slot == Record.size()) return 0;
+    unsigned ValNo = (unsigned)decodeSignRotatedValue(Record[Slot]);
+    // Adjust the ValNo, if it was encoded relative to the InstNum.
+    if (UseRelativeIDs)
+      ValNo = InstNum - ValNo;
+    return getFnValueByID(ValNo, Ty);
+  }
+
   bool ParseModule(bool Resume);
   bool ParseAttributeBlock();
   bool ParseTypeTable();
diff --git a/lib/Bitcode/Writer/BitcodeWriter.cpp b/lib/Bitcode/Writer/BitcodeWriter.cpp
index 5b1725f..60c657a 100644
--- a/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -41,8 +41,6 @@ EnablePreserveUseListOrdering("enable-bc-uselist-preserve",
 /// These are manifest constants used by the bitcode writer. They do not need to
 /// be kept in sync with the reader, but need to be consistent within this file.
 enum {
-  CurVersion = 0,
-
   // VALUE_SYMTAB_BLOCK abbrev id's.
   VST_ENTRY_8_ABBREV = bitc::FIRST_APPLICATION_ABBREV,
   VST_ENTRY_7_ABBREV,
@@ -177,7 +175,7 @@ static void WriteAttributeTable(const ValueEnumerator &VE,
     for (unsigned i = 0, e = A.getNumSlots(); i != e; ++i) {
       const AttributeWithIndex &PAWI = A.getSlot(i);
       Record.push_back(PAWI.Index);
-      Record.push_back(Attribute::encodeLLVMAttributesForBitcode(PAWI.Attrs));
+      Record.push_back(Attributes::encodeLLVMAttributesForBitcode(PAWI.Attrs));
     }
 
     Stream.EmitRecord(bitc::PARAMATTR_CODE_ENTRY, Record);
@@ -365,7 +363,7 @@ static unsigned getEncodedLinkage(const GlobalValue *GV) {
   case GlobalValue::AvailableExternallyLinkage:      return 12;
   case GlobalValue::LinkerPrivateLinkage:            return 13;
   case GlobalValue::LinkerPrivateWeakLinkage:        return 14;
-  case GlobalValue::LinkerPrivateWeakDefAutoLinkage: return 15;
+  case GlobalValue::LinkOnceODRAutoHideLinkage:      return 15;
   }
   llvm_unreachable("Invalid linkage");
 }
@@ -722,16 +720,20 @@ static void WriteModuleMetadataStore(const Module *M, BitstreamWriter &Stream) {
   Stream.ExitBlock();
 }
 
+static void emitSignedInt64(SmallVectorImpl<uint64_t> &Vals, uint64_t V) {
+  if ((int64_t)V >= 0)
+    Vals.push_back(V << 1);
+  else
+    Vals.push_back((-V << 1) | 1);
+}
+
 static void EmitAPInt(SmallVectorImpl<uint64_t> &Vals,
                       unsigned &Code, unsigned &AbbrevToUse, const APInt &Val,
                       bool EmitSizeForWideNumbers = false
                       ) {
   if (Val.getBitWidth() <= 64) {
     uint64_t V = Val.getSExtValue();
-    if ((int64_t)V >= 0)
-      Vals.push_back(V << 1);
-    else
-      Vals.push_back((-V << 1) | 1);
+    emitSignedInt64(Vals, V);
     Code = bitc::CST_CODE_INTEGER;
     AbbrevToUse = CONSTANTS_INTEGER_ABBREV;
   } else {
@@ -747,11 +749,7 @@ static void EmitAPInt(SmallVectorImpl<uint64_t> &Vals,
     
     const uint64_t *RawWords = Val.getRawData();
     for (unsigned i = 0; i != NWords; ++i) {
-      int64_t V = RawWords[i];
-      if (V >= 0)
-        Vals.push_back(V << 1);
-      else
-        Vals.push_back((-V << 1) | 1);
+      emitSignedInt64(Vals, RawWords[i]);
     }
     Code = bitc::CST_CODE_WIDE_INTEGER;
   }
@@ -814,7 +812,8 @@ static void WriteConstants(unsigned FirstVal, unsigned LastVal,
 
     if (const InlineAsm *IA = dyn_cast<InlineAsm>(V)) {
       Record.push_back(unsigned(IA->hasSideEffects()) |
-                       unsigned(IA->isAlignStack()) << 1);
+                       unsigned(IA->isAlignStack()) << 1 |
+                       unsigned(IA->getDialect()&1) << 2);
 
       // Add the asm string.
       const std::string &AsmStr = IA->getAsmString();
@@ -1024,12 +1023,13 @@ static void WriteModuleConstants(const ValueEnumerator &VE,
 ///
 /// This function adds V's value ID to Vals.  If the value ID is higher than the
 /// instruction ID, then it is a forward reference, and it also includes the
-/// type ID.
+/// type ID.  The value ID that is written is encoded relative to the InstID.
 static bool PushValueAndType(const Value *V, unsigned InstID,
                              SmallVector<unsigned, 64> &Vals,
                              ValueEnumerator &VE) {
   unsigned ValID = VE.getValueID(V);
-  Vals.push_back(ValID);
+  // Make encoding relative to the InstID.
+  Vals.push_back(InstID - ValID);
   if (ValID >= InstID) {
     Vals.push_back(VE.getTypeID(V->getType()));
     return true;
@@ -1037,6 +1037,30 @@ static bool PushValueAndType(const Value *V, unsigned InstID,
   return false;
 }
 
+/// pushValue - Like PushValueAndType, but where the type of the value is
+/// omitted (perhaps it was already encoded in an earlier operand).
+static void pushValue(const Value *V, unsigned InstID,
+                      SmallVector<unsigned, 64> &Vals,
+                      ValueEnumerator &VE) {
+  unsigned ValID = VE.getValueID(V);
+  Vals.push_back(InstID - ValID);
+}
+
+static void pushValue64(const Value *V, unsigned InstID,
+                        SmallVector<uint64_t, 128> &Vals,
+                        ValueEnumerator &VE) {
+  uint64_t ValID = VE.getValueID(V);
+  Vals.push_back(InstID - ValID);
+}
+
+static void pushValueSigned(const Value *V, unsigned InstID,
+                            SmallVector<uint64_t, 128> &Vals,
+                            ValueEnumerator &VE) {
+  unsigned ValID = VE.getValueID(V);
+  int64_t diff = ((int32_t)InstID - (int32_t)ValID);
+  emitSignedInt64(Vals, diff);
+}
+
 /// WriteInstruction - Emit an instruction to the specified stream.
 static void WriteInstruction(const Instruction &I, unsigned InstID,
                              ValueEnumerator &VE, BitstreamWriter &Stream,
@@ -1057,7 +1081,7 @@ static void WriteInstruction(const Instruction &I, unsigned InstID,
       Code = bitc::FUNC_CODE_INST_BINOP;
       if (!PushValueAndType(I.getOperand(0), InstID, Vals, VE))
         AbbrevToUse = FUNCTION_INST_BINOP_ABBREV;
-      Vals.push_back(VE.getValueID(I.getOperand(1)));
+      pushValue(I.getOperand(1), InstID, Vals, VE);
       Vals.push_back(GetEncodedBinaryOpcode(I.getOpcode()));
       uint64_t Flags = GetOptimizationFlags(&I);
       if (Flags != 0) {
@@ -1095,32 +1119,32 @@ static void WriteInstruction(const Instruction &I, unsigned InstID,
   case Instruction::Select:
     Code = bitc::FUNC_CODE_INST_VSELECT;
     PushValueAndType(I.getOperand(1), InstID, Vals, VE);
-    Vals.push_back(VE.getValueID(I.getOperand(2)));
+    pushValue(I.getOperand(2), InstID, Vals, VE);
     PushValueAndType(I.getOperand(0), InstID, Vals, VE);
     break;
   case Instruction::ExtractElement:
     Code = bitc::FUNC_CODE_INST_EXTRACTELT;
     PushValueAndType(I.getOperand(0), InstID, Vals, VE);
-    Vals.push_back(VE.getValueID(I.getOperand(1)));
+    pushValue(I.getOperand(1), InstID, Vals, VE);
     break;
   case Instruction::InsertElement:
     Code = bitc::FUNC_CODE_INST_INSERTELT;
     PushValueAndType(I.getOperand(0), InstID, Vals, VE);
-    Vals.push_back(VE.getValueID(I.getOperand(1)));
-    Vals.push_back(VE.getValueID(I.getOperand(2)));
+    pushValue(I.getOperand(1), InstID, Vals, VE);
+    pushValue(I.getOperand(2), InstID, Vals, VE);
     break;
   case Instruction::ShuffleVector:
     Code = bitc::FUNC_CODE_INST_SHUFFLEVEC;
     PushValueAndType(I.getOperand(0), InstID, Vals, VE);
-    Vals.push_back(VE.getValueID(I.getOperand(1)));
-    Vals.push_back(VE.getValueID(I.getOperand(2)));
+    pushValue(I.getOperand(1), InstID, Vals, VE);
+    pushValue(I.getOperand(2), InstID, Vals, VE);
     break;
   case Instruction::ICmp:
   case Instruction::FCmp:
     // compare returning Int1Ty or vector of Int1Ty
     Code = bitc::FUNC_CODE_INST_CMP2;
     PushValueAndType(I.getOperand(0), InstID, Vals, VE);
-    Vals.push_back(VE.getValueID(I.getOperand(1)));
+    pushValue(I.getOperand(1), InstID, Vals, VE);
     Vals.push_back(cast<CmpInst>(I).getPredicate());
     break;
 
@@ -1146,7 +1170,7 @@ static void WriteInstruction(const Instruction &I, unsigned InstID,
       Vals.push_back(VE.getValueID(II.getSuccessor(0)));
       if (II.isConditional()) {
         Vals.push_back(VE.getValueID(II.getSuccessor(1)));
-        Vals.push_back(VE.getValueID(II.getCondition()));
+        pushValue(II.getCondition(), InstID, Vals, VE);
       }
     }
     break;
@@ -1163,7 +1187,7 @@ static void WriteInstruction(const Instruction &I, unsigned InstID,
       Vals64.push_back(SwitchRecordHeader);      
       
       Vals64.push_back(VE.getTypeID(SI.getCondition()->getType()));
-      Vals64.push_back(VE.getValueID(SI.getCondition()));
+      pushValue64(SI.getCondition(), InstID, Vals64, VE);
       Vals64.push_back(VE.getValueID(SI.getDefaultDest()));
       Vals64.push_back(SI.getNumCases());
       for (SwitchInst::CaseIt i = SI.case_begin(), e = SI.case_end();
@@ -1214,7 +1238,9 @@ static void WriteInstruction(const Instruction &I, unsigned InstID,
   case Instruction::IndirectBr:
     Code = bitc::FUNC_CODE_INST_INDIRECTBR;
     Vals.push_back(VE.getTypeID(I.getOperand(0)->getType()));
-    for (unsigned i = 0, e = I.getNumOperands(); i != e; ++i)
+    // Encode the address operand as relative, but not the basic blocks.
+    pushValue(I.getOperand(0), InstID, Vals, VE);
+    for (unsigned i = 1, e = I.getNumOperands(); i != e; ++i)
       Vals.push_back(VE.getValueID(I.getOperand(i)));
     break;
       
@@ -1233,7 +1259,7 @@ static void WriteInstruction(const Instruction &I, unsigned InstID,
 
     // Emit value #'s for the fixed parameters.
     for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i)
-      Vals.push_back(VE.getValueID(I.getOperand(i)));  // fixed param.
+      pushValue(I.getOperand(i), InstID, Vals, VE);  // fixed param.
 
     // Emit type/value pairs for varargs params.
     if (FTy->isVarArg()) {
@@ -1255,12 +1281,19 @@ static void WriteInstruction(const Instruction &I, unsigned InstID,
   case Instruction::PHI: {
     const PHINode &PN = cast<PHINode>(I);
     Code = bitc::FUNC_CODE_INST_PHI;
-    Vals.push_back(VE.getTypeID(PN.getType()));
+    // With the newer instruction encoding, forward references could give
+    // negative valued IDs.  This is most common for PHIs, so we use
+    // signed VBRs.
+    SmallVector<uint64_t, 128> Vals64;
+    Vals64.push_back(VE.getTypeID(PN.getType()));
     for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i) {
-      Vals.push_back(VE.getValueID(PN.getIncomingValue(i)));
-      Vals.push_back(VE.getValueID(PN.getIncomingBlock(i)));
+      pushValueSigned(PN.getIncomingValue(i), InstID, Vals64, VE);
+      Vals64.push_back(VE.getValueID(PN.getIncomingBlock(i)));
     }
-    break;
+    // Emit a Vals64 vector and exit.
+    Stream.EmitRecord(Code, Vals64, AbbrevToUse);
+    Vals64.clear();
+    return;
   }
 
   case Instruction::LandingPad: {
@@ -1310,7 +1343,7 @@ static void WriteInstruction(const Instruction &I, unsigned InstID,
     else
       Code = bitc::FUNC_CODE_INST_STORE;
     PushValueAndType(I.getOperand(1), InstID, Vals, VE);  // ptrty + ptr
-    Vals.push_back(VE.getValueID(I.getOperand(0)));       // val.
+    pushValue(I.getOperand(0), InstID, Vals, VE);         // val.
     Vals.push_back(Log2_32(cast<StoreInst>(I).getAlignment())+1);
     Vals.push_back(cast<StoreInst>(I).isVolatile());
     if (cast<StoreInst>(I).isAtomic()) {
@@ -1321,8 +1354,8 @@ static void WriteInstruction(const Instruction &I, unsigned InstID,
   case Instruction::AtomicCmpXchg:
     Code = bitc::FUNC_CODE_INST_CMPXCHG;
     PushValueAndType(I.getOperand(0), InstID, Vals, VE);  // ptrty + ptr
-    Vals.push_back(VE.getValueID(I.getOperand(1)));       // cmp.
-    Vals.push_back(VE.getValueID(I.getOperand(2)));       // newval.
+    pushValue(I.getOperand(1), InstID, Vals, VE);         // cmp.
+    pushValue(I.getOperand(2), InstID, Vals, VE);         // newval.
     Vals.push_back(cast<AtomicCmpXchgInst>(I).isVolatile());
     Vals.push_back(GetEncodedOrdering(
                      cast<AtomicCmpXchgInst>(I).getOrdering()));
@@ -1332,7 +1365,7 @@ static void WriteInstruction(const Instruction &I, unsigned InstID,
   case Instruction::AtomicRMW:
     Code = bitc::FUNC_CODE_INST_ATOMICRMW;
     PushValueAndType(I.getOperand(0), InstID, Vals, VE);  // ptrty + ptr
-    Vals.push_back(VE.getValueID(I.getOperand(1)));       // val.
+    pushValue(I.getOperand(1), InstID, Vals, VE);         // val.
     Vals.push_back(GetEncodedRMWOperation(
                      cast<AtomicRMWInst>(I).getOperation()));
     Vals.push_back(cast<AtomicRMWInst>(I).isVolatile());
@@ -1357,8 +1390,13 @@ static void WriteInstruction(const Instruction &I, unsigned InstID,
     PushValueAndType(CI.getCalledValue(), InstID, Vals, VE);  // Callee
 
     // Emit value #'s for the fixed parameters.
-    for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i)
-      Vals.push_back(VE.getValueID(CI.getArgOperand(i)));  // fixed param.
+    for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i) {
+      // Check for labels (can happen with asm labels).
+      if (FTy->getParamType(i)->isLabelTy())
+        Vals.push_back(VE.getValueID(CI.getArgOperand(i)));
+      else
+        pushValue(CI.getArgOperand(i), InstID, Vals, VE);  // fixed param.
+    }
 
     // Emit type/value pairs for varargs params.
     if (FTy->isVarArg()) {
@@ -1371,7 +1409,7 @@ static void WriteInstruction(const Instruction &I, unsigned InstID,
   case Instruction::VAArg:
     Code = bitc::FUNC_CODE_INST_VAARG;
     Vals.push_back(VE.getTypeID(I.getOperand(0)->getType()));   // valistty
-    Vals.push_back(VE.getValueID(I.getOperand(0))); // valist.
+    pushValue(I.getOperand(0), InstID, Vals, VE); // valist.
     Vals.push_back(VE.getTypeID(I.getType())); // restype.
     break;
   }
@@ -1513,8 +1551,8 @@ static void WriteFunction(const Function &F, ValueEnumerator &VE,
 // Emit blockinfo, which defines the standard abbreviations etc.
 static void WriteBlockInfo(const ValueEnumerator &VE, BitstreamWriter &Stream) {
   // We only want to emit block info records for blocks that have multiple
-  // instances: CONSTANTS_BLOCK, FUNCTION_BLOCK and VALUE_SYMTAB_BLOCK.  Other
-  // blocks can defined their abbrevs inline.
+  // instances: CONSTANTS_BLOCK, FUNCTION_BLOCK and VALUE_SYMTAB_BLOCK.
+  // Other blocks can define their abbrevs inline.
   Stream.EnterBlockInfoBlock(2);
 
   { // 8-bit fixed-width VST_ENTRY/VST_BBENTRY strings.
@@ -1772,12 +1810,10 @@ static void WriteModuleUseLists(const Module *M, ValueEnumerator &VE,
 static void WriteModule(const Module *M, BitstreamWriter &Stream) {
   Stream.EnterSubblock(bitc::MODULE_BLOCK_ID, 3);
 
-  // Emit the version number if it is non-zero.
-  if (CurVersion) {
-    SmallVector<unsigned, 1> Vals;
-    Vals.push_back(CurVersion);
-    Stream.EmitRecord(bitc::MODULE_CODE_VERSION, Vals);
-  }
+  SmallVector<unsigned, 1> Vals;
+  unsigned CurVersion = 1;
+  Vals.push_back(CurVersion);
+  Stream.EmitRecord(bitc::MODULE_CODE_VERSION, Vals);
 
   // Analyze the module, enumerating globals, functions, etc.
   ValueEnumerator VE(M);
diff --git a/lib/Bitcode/Writer/ValueEnumerator.h b/lib/Bitcode/Writer/ValueEnumerator.h
index a6ca536..75468e6 100644
--- a/lib/Bitcode/Writer/ValueEnumerator.h
+++ b/lib/Bitcode/Writer/ValueEnumerator.h
@@ -78,9 +78,9 @@ private:
 
   unsigned FirstFuncConstantID;
   unsigned FirstInstID;
-  
-  ValueEnumerator(const ValueEnumerator &);  // DO NOT IMPLEMENT
-  void operator=(const ValueEnumerator &);   // DO NOT IMPLEMENT
+
+  ValueEnumerator(const ValueEnumerator &) LLVM_DELETED_FUNCTION;
+  void operator=(const ValueEnumerator &) LLVM_DELETED_FUNCTION;
 public:
   ValueEnumerator(const Module *M);
 
diff --git a/lib/CodeGen/AggressiveAntiDepBreaker.cpp b/lib/CodeGen/AggressiveAntiDepBreaker.cpp
index 205480a..7a1c049 100644
--- a/lib/CodeGen/AggressiveAntiDepBreaker.cpp
+++ b/lib/CodeGen/AggressiveAntiDepBreaker.cpp
@@ -635,7 +635,7 @@ bool AggressiveAntiDepBreaker::FindSuitableFreeRegisters(
     --R;
     const unsigned NewSuperReg = Order[R];
     // Don't consider non-allocatable registers
-    if (!RegClassInfo.isAllocatable(NewSuperReg)) continue;
+    if (!MRI.isAllocatable(NewSuperReg)) continue;
     // Don't replace a register with itself.
     if (NewSuperReg == SuperReg) continue;
 
@@ -818,7 +818,7 @@ unsigned AggressiveAntiDepBreaker::BreakAntiDependencies(
         DEBUG(dbgs() << "\tAntidep reg: " << TRI->getName(AntiDepReg));
         assert(AntiDepReg != 0 && "Anti-dependence on reg0?");
 
-        if (!RegClassInfo.isAllocatable(AntiDepReg)) {
+        if (!MRI.isAllocatable(AntiDepReg)) {
           // Don't break anti-dependencies on non-allocatable registers.
           DEBUG(dbgs() << " (non-allocatable)\n");
           continue;
diff --git a/lib/CodeGen/AllocationOrder.cpp b/lib/CodeGen/AllocationOrder.cpp
index 32ad34a..7cde136 100644
--- a/lib/CodeGen/AllocationOrder.cpp
+++ b/lib/CodeGen/AllocationOrder.cpp
@@ -29,6 +29,7 @@ AllocationOrder::AllocationOrder(unsigned VirtReg,
   const TargetRegisterClass *RC = VRM.getRegInfo().getRegClass(VirtReg);
   std::pair<unsigned, unsigned> HintPair =
     VRM.getRegInfo().getRegAllocationHint(VirtReg);
+  const MachineRegisterInfo &MRI = VRM.getRegInfo();
 
   // HintPair.second is a register, phys or virt.
   Hint = HintPair.second;
@@ -52,7 +53,7 @@ AllocationOrder::AllocationOrder(unsigned VirtReg,
     unsigned *P = new unsigned[Order.size()];
     Begin = P;
     for (unsigned i = 0; i != Order.size(); ++i)
-      if (!RCI.isReserved(Order[i]))
+      if (!MRI.isReserved(Order[i]))
         *P++ = Order[i];
     End = P;
 
@@ -69,7 +70,7 @@ AllocationOrder::AllocationOrder(unsigned VirtReg,
 
   // The hint must be a valid physreg for allocation.
   if (Hint && (!TargetRegisterInfo::isPhysicalRegister(Hint) ||
-               !RC->contains(Hint) || RCI.isReserved(Hint)))
+               !RC->contains(Hint) || MRI.isReserved(Hint)))
     Hint = 0;
 }
 
diff --git a/lib/CodeGen/Analysis.cpp b/lib/CodeGen/Analysis.cpp
index 447f398..5162ad7 100644
--- a/lib/CodeGen/Analysis.cpp
+++ b/lib/CodeGen/Analysis.cpp
@@ -21,7 +21,7 @@
 #include "llvm/Module.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/SelectionDAG.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetLowering.h"
 #include "llvm/Target/TargetOptions.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -79,7 +79,7 @@ void llvm::ComputeValueVTs(const TargetLowering &TLI, Type *Ty,
                            uint64_t StartingOffset) {
   // Given a struct type, recursively traverse the elements.
   if (StructType *STy = dyn_cast<StructType>(Ty)) {
-    const StructLayout *SL = TLI.getTargetData()->getStructLayout(STy);
+    const StructLayout *SL = TLI.getDataLayout()->getStructLayout(STy);
     for (StructType::element_iterator EB = STy->element_begin(),
                                       EI = EB,
                                       EE = STy->element_end();
@@ -91,7 +91,7 @@ void llvm::ComputeValueVTs(const TargetLowering &TLI, Type *Ty,
   // Given an array type, recursively traverse the elements.
   if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
     Type *EltTy = ATy->getElementType();
-    uint64_t EltSize = TLI.getTargetData()->getTypeAllocSize(EltTy);
+    uint64_t EltSize = TLI.getDataLayout()->getTypeAllocSize(EltTy);
     for (unsigned i = 0, e = ATy->getNumElements(); i != e; ++i)
       ComputeValueVTs(TLI, EltTy, ValueVTs, Offsets,
                       StartingOffset + i * EltSize);
@@ -314,11 +314,13 @@ bool llvm::isInTailCallPosition(ImmutableCallSite CS, Attributes CalleeRetAttr,
   // the return. Ignore noalias because it doesn't affect the call sequence.
   const Function *F = ExitBB->getParent();
   Attributes CallerRetAttr = F->getAttributes().getRetAttributes();
-  if ((CalleeRetAttr ^ CallerRetAttr) & ~Attribute::NoAlias)
+  if (AttrBuilder(CalleeRetAttr).removeAttribute(Attributes::NoAlias) !=
+      AttrBuilder(CallerRetAttr).removeAttribute(Attributes::NoAlias))
     return false;
 
   // It's not safe to eliminate the sign / zero extension of the return value.
-  if ((CallerRetAttr & Attribute::ZExt) || (CallerRetAttr & Attribute::SExt))
+  if (CallerRetAttr.hasAttribute(Attributes::ZExt) ||
+      CallerRetAttr.hasAttribute(Attributes::SExt))
     return false;
 
   // Otherwise, make sure the unmodified return value of I is the return value.
@@ -354,11 +356,13 @@ bool llvm::isInTailCallPosition(SelectionDAG &DAG, SDNode *Node,
   // Conservatively require the attributes of the call to match those of
   // the return. Ignore noalias because it doesn't affect the call sequence.
   Attributes CallerRetAttr = F->getAttributes().getRetAttributes();
-  if (CallerRetAttr & ~Attribute::NoAlias)
+  if (AttrBuilder(CallerRetAttr)
+      .removeAttribute(Attributes::NoAlias).hasAttributes())
     return false;
 
   // It's not safe to eliminate the sign / zero extension of the return value.
-  if ((CallerRetAttr & Attribute::ZExt) || (CallerRetAttr & Attribute::SExt))
+  if (CallerRetAttr.hasAttribute(Attributes::ZExt) ||
+      CallerRetAttr.hasAttribute(Attributes::SExt))
     return false;
 
   // Check if the only use is a function return node.
diff --git a/lib/CodeGen/AsmPrinter/ARMException.cpp b/lib/CodeGen/AsmPrinter/ARMException.cpp
index bf5d8c4..b2ebf04 100644
--- a/lib/CodeGen/AsmPrinter/ARMException.cpp
+++ b/lib/CodeGen/AsmPrinter/ARMException.cpp
@@ -24,7 +24,7 @@
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/Target/Mangler.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetFrameLowering.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
diff --git a/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index d9be7a1..d74a703 100644
--- a/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -33,7 +33,7 @@
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/Target/Mangler.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetInstrInfo.h"
 #include "llvm/Target/TargetLowering.h"
 #include "llvm/Target/TargetLoweringObjectFile.h"
@@ -67,7 +67,7 @@ static gcp_map_type &getGCMap(void *&P) {
 /// getGVAlignmentLog2 - Return the alignment to use for the specified global
 /// value in log2 form.  This rounds up to the preferred alignment if possible
 /// and legal.
-static unsigned getGVAlignmentLog2(const GlobalValue *GV, const TargetData &TD,
+static unsigned getGVAlignmentLog2(const GlobalValue *GV, const DataLayout &TD,
                                    unsigned InBits = 0) {
   unsigned NumBits = 0;
   if (const GlobalVariable *GVar = dyn_cast<GlobalVariable>(GV))
@@ -131,9 +131,9 @@ const TargetLoweringObjectFile &AsmPrinter::getObjFileLowering() const {
 }
 
 
-/// getTargetData - Return information about data layout.
-const TargetData &AsmPrinter::getTargetData() const {
-  return *TM.getTargetData();
+/// getDataLayout - Return information about data layout.
+const DataLayout &AsmPrinter::getDataLayout() const {
+  return *TM.getDataLayout();
 }
 
 /// getCurrentSection() - Return the current section we are emitting to.
@@ -160,7 +160,7 @@ bool AsmPrinter::doInitialization(Module &M) {
   const_cast<TargetLoweringObjectFile&>(getObjFileLowering())
     .Initialize(OutContext, TM);
 
-  Mang = new Mangler(OutContext, *TM.getTargetData());
+  Mang = new Mangler(OutContext, *TM.getDataLayout());
 
   // Allow the target to emit any magic that it wants at the start of the file.
   EmitStartOfAsmFile(M);
@@ -213,16 +213,16 @@ void AsmPrinter::EmitLinkage(unsigned Linkage, MCSymbol *GVSym) const {
   case GlobalValue::CommonLinkage:
   case GlobalValue::LinkOnceAnyLinkage:
   case GlobalValue::LinkOnceODRLinkage:
+  case GlobalValue::LinkOnceODRAutoHideLinkage:
   case GlobalValue::WeakAnyLinkage:
   case GlobalValue::WeakODRLinkage:
   case GlobalValue::LinkerPrivateWeakLinkage:
-  case GlobalValue::LinkerPrivateWeakDefAutoLinkage:
     if (MAI->getWeakDefDirective() != 0) {
       // .globl _foo
       OutStreamer.EmitSymbolAttribute(GVSym, MCSA_Global);
 
       if ((GlobalValue::LinkageTypes)Linkage !=
-          GlobalValue::LinkerPrivateWeakDefAutoLinkage)
+          GlobalValue::LinkOnceODRAutoHideLinkage)
         // .weak_definition _foo
         OutStreamer.EmitSymbolAttribute(GVSym, MCSA_WeakDefinition);
       else
@@ -280,7 +280,7 @@ void AsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) {
 
   SectionKind GVKind = TargetLoweringObjectFile::getKindForGlobal(GV, TM);
 
-  const TargetData *TD = TM.getTargetData();
+  const DataLayout *TD = TM.getDataLayout();
   uint64_t Size = TD->getTypeAllocSize(GV->getType()->getElementType());
 
   // If the alignment is specified, we *must* obey it.  Overaligning a global
@@ -312,8 +312,8 @@ void AsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) {
       return;
     }
 
-    if (MAI->getLCOMMDirectiveType() != LCOMM::None &&
-        (MAI->getLCOMMDirectiveType() != LCOMM::NoAlignment || Align == 1)) {
+    if (Align == 1 ||
+        MAI->getLCOMMDirectiveAlignmentType() != LCOMM::NoAlignment) {
       // .lcomm _foo, 42
       OutStreamer.EmitLocalCommonSymbol(GVSym, Size, Align);
       return;
@@ -482,9 +482,8 @@ void AsmPrinter::EmitFunctionEntryLabel() {
                      "' label emitted multiple times to assembly file");
 }
 
-
-/// EmitComments - Pretty-print comments for instructions.
-static void EmitComments(const MachineInstr &MI, raw_ostream &CommentOS) {
+/// emitComments - Pretty-print comments for instructions.
+static void emitComments(const MachineInstr &MI, raw_ostream &CommentOS) {
   const MachineFunction *MF = MI.getParent()->getParent();
   const TargetMachine &TM = MF->getTarget();
 
@@ -519,16 +518,16 @@ static void EmitComments(const MachineInstr &MI, raw_ostream &CommentOS) {
     CommentOS << " Reload Reuse\n";
 }
 
-/// EmitImplicitDef - This method emits the specified machine instruction
+/// emitImplicitDef - This method emits the specified machine instruction
 /// that is an implicit def.
-static void EmitImplicitDef(const MachineInstr *MI, AsmPrinter &AP) {
+static void emitImplicitDef(const MachineInstr *MI, AsmPrinter &AP) {
   unsigned RegNo = MI->getOperand(0).getReg();
   AP.OutStreamer.AddComment(Twine("implicit-def: ") +
                             AP.TM.getRegisterInfo()->getName(RegNo));
   AP.OutStreamer.AddBlankLine();
 }
 
-static void EmitKill(const MachineInstr *MI, AsmPrinter &AP) {
+static void emitKill(const MachineInstr *MI, AsmPrinter &AP) {
   std::string Str = "kill:";
   for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
     const MachineOperand &Op = MI->getOperand(i);
@@ -541,10 +540,10 @@ static void EmitKill(const MachineInstr *MI, AsmPrinter &AP) {
   AP.OutStreamer.AddBlankLine();
 }
 
-/// EmitDebugValueComment - This method handles the target-independent form
+/// emitDebugValueComment - This method handles the target-independent form
 /// of DBG_VALUE, returning true if it was able to do so.  A false return
 /// means the target will need to handle MI in EmitInstruction.
-static bool EmitDebugValueComment(const MachineInstr *MI, AsmPrinter &AP) {
+static bool emitDebugValueComment(const MachineInstr *MI, AsmPrinter &AP) {
   // This code handles only the 3-operand target-independent form.
   if (MI->getNumOperands() != 3)
     return false;
@@ -674,7 +673,7 @@ void AsmPrinter::EmitFunctionBody() {
       }
 
       if (isVerbose())
-        EmitComments(*II, OutStreamer.GetCommentOS());
+        emitComments(*II, OutStreamer.GetCommentOS());
 
       switch (II->getOpcode()) {
       case TargetOpcode::PROLOG_LABEL:
@@ -690,15 +689,15 @@ void AsmPrinter::EmitFunctionBody() {
         break;
       case TargetOpcode::DBG_VALUE:
         if (isVerbose()) {
-          if (!EmitDebugValueComment(II, *this))
+          if (!emitDebugValueComment(II, *this))
             EmitInstruction(II);
         }
         break;
       case TargetOpcode::IMPLICIT_DEF:
-        if (isVerbose()) EmitImplicitDef(II, *this);
+        if (isVerbose()) emitImplicitDef(II, *this);
         break;
       case TargetOpcode::KILL:
-        if (isVerbose()) EmitKill(II, *this);
+        if (isVerbose()) emitKill(II, *this);
         break;
       default:
         if (!TM.hasMCUseLoc())
@@ -992,7 +991,7 @@ void AsmPrinter::EmitConstantPool() {
       Kind = SectionKind::getReadOnlyWithRelLocal();
       break;
     case 0:
-    switch (TM.getTargetData()->getTypeAllocSize(CPE.getType())) {
+    switch (TM.getDataLayout()->getTypeAllocSize(CPE.getType())) {
     case 4:  Kind = SectionKind::getMergeableConst4(); break;
     case 8:  Kind = SectionKind::getMergeableConst8(); break;
     case 16: Kind = SectionKind::getMergeableConst16();break;
@@ -1038,7 +1037,7 @@ void AsmPrinter::EmitConstantPool() {
       OutStreamer.EmitFill(NewOffset - Offset, 0/*fillval*/, 0/*addrspace*/);
 
       Type *Ty = CPE.getType();
-      Offset = NewOffset + TM.getTargetData()->getTypeAllocSize(Ty);
+      Offset = NewOffset + TM.getDataLayout()->getTypeAllocSize(Ty);
       OutStreamer.EmitLabel(GetCPISymbol(CPI));
 
       if (CPE.isMachineConstantPoolEntry())
@@ -1081,7 +1080,12 @@ void AsmPrinter::EmitJumpTableInfo() {
     JTInDiffSection = true;
   }
 
-  EmitAlignment(Log2_32(MJTI->getEntryAlignment(*TM.getTargetData())));
+  EmitAlignment(Log2_32(MJTI->getEntryAlignment(*TM.getDataLayout())));
+
+  // Jump tables in code sections are marked with a data_region directive
+  // where that's supported.
+  if (!JTInDiffSection)
+    OutStreamer.EmitDataRegion(MCDR_DataRegionJT32);
 
   for (unsigned JTI = 0, e = JT.size(); JTI != e; ++JTI) {
     const std::vector<MachineBasicBlock*> &JTBBs = JT[JTI].MBBs;
@@ -1123,6 +1127,8 @@ void AsmPrinter::EmitJumpTableInfo() {
     for (unsigned ii = 0, ee = JTBBs.size(); ii != ee; ++ii)
       EmitJumpTableEntry(MJTI, JTBBs[ii], JTI);
   }
+  if (!JTInDiffSection)
+    OutStreamer.EmitDataRegion(MCDR_DataRegionEnd);
 }
 
 /// EmitJumpTableEntry - Emit a jump table entry for the specified MBB to the
@@ -1190,7 +1196,7 @@ void AsmPrinter::EmitJumpTableEntry(const MachineJumpTableInfo *MJTI,
 
   assert(Value && "Unknown entry kind!");
 
-  unsigned EntrySize = MJTI->getEntrySize(*TM.getTargetData());
+  unsigned EntrySize = MJTI->getEntrySize(*TM.getDataLayout());
   OutStreamer.EmitValue(Value, EntrySize, /*addrspace*/0);
 }
 
@@ -1292,7 +1298,7 @@ void AsmPrinter::EmitXXStructorList(const Constant *List, bool isCtor) {
   }
 
   // Emit the function pointers in the target-specific order
-  const TargetData *TD = TM.getTargetData();
+  const DataLayout *TD = TM.getDataLayout();
   unsigned Align = Log2_32(TD->getPointerPrefAlignment());
   std::stable_sort(Structors.begin(), Structors.end(), priority_order);
   for (unsigned i = 0, e = Structors.size(); i != e; ++i) {
@@ -1408,7 +1414,7 @@ void AsmPrinter::EmitLabelPlusOffset(const MCSymbol *Label, uint64_t Offset,
 // if required for correctness.
 //
 void AsmPrinter::EmitAlignment(unsigned NumBits, const GlobalValue *GV) const {
-  if (GV) NumBits = getGVAlignmentLog2(GV, *TM.getTargetData(), NumBits);
+  if (GV) NumBits = getGVAlignmentLog2(GV, *TM.getDataLayout(), NumBits);
 
   if (NumBits == 0) return;   // 1-byte aligned: no need to emit alignment.
 
@@ -1422,9 +1428,9 @@ void AsmPrinter::EmitAlignment(unsigned NumBits, const GlobalValue *GV) const {
 // Constant emission.
 //===----------------------------------------------------------------------===//
 
-/// LowerConstant - Lower the specified LLVM Constant to an MCExpr.
+/// lowerConstant - Lower the specified LLVM Constant to an MCExpr.
 ///
-static const MCExpr *LowerConstant(const Constant *CV, AsmPrinter &AP) {
+static const MCExpr *lowerConstant(const Constant *CV, AsmPrinter &AP) {
   MCContext &Ctx = AP.OutContext;
 
   if (CV->isNullValue() || isa<UndefValue>(CV))
@@ -1447,12 +1453,12 @@ static const MCExpr *LowerConstant(const Constant *CV, AsmPrinter &AP) {
   switch (CE->getOpcode()) {
   default:
     // If the code isn't optimized, there may be outstanding folding
-    // opportunities. Attempt to fold the expression using TargetData as a
+    // opportunities. Attempt to fold the expression using DataLayout as a
     // last resort before giving up.
     if (Constant *C =
-          ConstantFoldConstantExpression(CE, AP.TM.getTargetData()))
+          ConstantFoldConstantExpression(CE, AP.TM.getDataLayout()))
       if (C != CE)
-        return LowerConstant(C, AP);
+        return lowerConstant(C, AP);
 
     // Otherwise report the problem to the user.
     {
@@ -1464,21 +1470,20 @@ static const MCExpr *LowerConstant(const Constant *CV, AsmPrinter &AP) {
       report_fatal_error(OS.str());
     }
   case Instruction::GetElementPtr: {
-    const TargetData &TD = *AP.TM.getTargetData();
+    const DataLayout &TD = *AP.TM.getDataLayout();
     // Generate a symbolic expression for the byte address
     const Constant *PtrVal = CE->getOperand(0);
     SmallVector<Value*, 8> IdxVec(CE->op_begin()+1, CE->op_end());
     int64_t Offset = TD.getIndexedOffset(PtrVal->getType(), IdxVec);
 
-    const MCExpr *Base = LowerConstant(CE->getOperand(0), AP);
+    const MCExpr *Base = lowerConstant(CE->getOperand(0), AP);
     if (Offset == 0)
       return Base;
 
     // Truncate/sext the offset to the pointer size.
-    if (TD.getPointerSizeInBits() != 64) {
-      int SExtAmount = 64-TD.getPointerSizeInBits();
-      Offset = (Offset << SExtAmount) >> SExtAmount;
-    }
+    unsigned Width = TD.getPointerSizeInBits();
+    if (Width < 64)
+      Offset = SignExtend64(Offset, Width);
 
     return MCBinaryExpr::CreateAdd(Base, MCConstantExpr::Create(Offset, Ctx),
                                    Ctx);
@@ -1491,26 +1496,26 @@ static const MCExpr *LowerConstant(const Constant *CV, AsmPrinter &AP) {
     // is reasonable to treat their delta as a 32-bit value.
     // FALL THROUGH.
   case Instruction::BitCast:
-    return LowerConstant(CE->getOperand(0), AP);
+    return lowerConstant(CE->getOperand(0), AP);
 
   case Instruction::IntToPtr: {
-    const TargetData &TD = *AP.TM.getTargetData();
+    const DataLayout &TD = *AP.TM.getDataLayout();
     // Handle casts to pointers by changing them into casts to the appropriate
     // integer type.  This promotes constant folding and simplifies this code.
     Constant *Op = CE->getOperand(0);
     Op = ConstantExpr::getIntegerCast(Op, TD.getIntPtrType(CV->getContext()),
                                       false/*ZExt*/);
-    return LowerConstant(Op, AP);
+    return lowerConstant(Op, AP);
   }
 
   case Instruction::PtrToInt: {
-    const TargetData &TD = *AP.TM.getTargetData();
+    const DataLayout &TD = *AP.TM.getDataLayout();
     // Support only foldable casts to/from pointers that can be eliminated by
     // changing the pointer to the appropriately sized integer type.
     Constant *Op = CE->getOperand(0);
     Type *Ty = CE->getType();
 
-    const MCExpr *OpExpr = LowerConstant(Op, AP);
+    const MCExpr *OpExpr = lowerConstant(Op, AP);
 
     // We can emit the pointer value into this slot if the slot is an
     // integer slot equal to the size of the pointer.
@@ -1536,8 +1541,8 @@ static const MCExpr *LowerConstant(const Constant *CV, AsmPrinter &AP) {
   case Instruction::And:
   case Instruction::Or:
   case Instruction::Xor: {
-    const MCExpr *LHS = LowerConstant(CE->getOperand(0), AP);
-    const MCExpr *RHS = LowerConstant(CE->getOperand(1), AP);
+    const MCExpr *LHS = lowerConstant(CE->getOperand(0), AP);
+    const MCExpr *RHS = lowerConstant(CE->getOperand(1), AP);
     switch (CE->getOpcode()) {
     default: llvm_unreachable("Unknown binary operator constant cast expr");
     case Instruction::Add: return MCBinaryExpr::CreateAdd(LHS, RHS, Ctx);
@@ -1554,7 +1559,7 @@ static const MCExpr *LowerConstant(const Constant *CV, AsmPrinter &AP) {
   }
 }
 
-static void EmitGlobalConstantImpl(const Constant *C, unsigned AddrSpace,
+static void emitGlobalConstantImpl(const Constant *C, unsigned AddrSpace,
                                    AsmPrinter &AP);
 
 /// isRepeatedByteSequence - Determine whether the given value is
@@ -1578,7 +1583,7 @@ static int isRepeatedByteSequence(const Value *V, TargetMachine &TM) {
   if (const ConstantInt *CI = dyn_cast<ConstantInt>(V)) {
     if (CI->getBitWidth() > 64) return -1;
 
-    uint64_t Size = TM.getTargetData()->getTypeAllocSize(V->getType());
+    uint64_t Size = TM.getDataLayout()->getTypeAllocSize(V->getType());
     uint64_t Value = CI->getZExtValue();
 
     // Make sure the constant is at least 8 bits long and has a power
@@ -1616,13 +1621,13 @@ static int isRepeatedByteSequence(const Value *V, TargetMachine &TM) {
   return -1;
 }
 
-static void EmitGlobalConstantDataSequential(const ConstantDataSequential *CDS,
+static void emitGlobalConstantDataSequential(const ConstantDataSequential *CDS,
                                              unsigned AddrSpace,AsmPrinter &AP){
   
   // See if we can aggregate this into a .fill, if so, emit it as such.
   int Value = isRepeatedByteSequence(CDS, AP.TM);
   if (Value != -1) {
-    uint64_t Bytes = AP.TM.getTargetData()->getTypeAllocSize(CDS->getType());
+    uint64_t Bytes = AP.TM.getDataLayout()->getTypeAllocSize(CDS->getType());
     // Don't emit a 1-byte object as a .fill.
     if (Bytes > 1)
       return AP.OutStreamer.EmitFill(Bytes, Value, AddrSpace);
@@ -1672,7 +1677,7 @@ static void EmitGlobalConstantDataSequential(const ConstantDataSequential *CDS,
     }
   }
 
-  const TargetData &TD = *AP.TM.getTargetData();
+  const DataLayout &TD = *AP.TM.getDataLayout();
   unsigned Size = TD.getTypeAllocSize(CDS->getType());
   unsigned EmittedSize = TD.getTypeAllocSize(CDS->getType()->getElementType()) *
                         CDS->getNumElements();
@@ -1681,28 +1686,28 @@ static void EmitGlobalConstantDataSequential(const ConstantDataSequential *CDS,
 
 }
 
-static void EmitGlobalConstantArray(const ConstantArray *CA, unsigned AddrSpace,
+static void emitGlobalConstantArray(const ConstantArray *CA, unsigned AddrSpace,
                                     AsmPrinter &AP) {
   // See if we can aggregate some values.  Make sure it can be
   // represented as a series of bytes of the constant value.
   int Value = isRepeatedByteSequence(CA, AP.TM);
 
   if (Value != -1) {
-    uint64_t Bytes = AP.TM.getTargetData()->getTypeAllocSize(CA->getType());
+    uint64_t Bytes = AP.TM.getDataLayout()->getTypeAllocSize(CA->getType());
     AP.OutStreamer.EmitFill(Bytes, Value, AddrSpace);
   }
   else {
     for (unsigned i = 0, e = CA->getNumOperands(); i != e; ++i)
-      EmitGlobalConstantImpl(CA->getOperand(i), AddrSpace, AP);
+      emitGlobalConstantImpl(CA->getOperand(i), AddrSpace, AP);
   }
 }
 
-static void EmitGlobalConstantVector(const ConstantVector *CV,
+static void emitGlobalConstantVector(const ConstantVector *CV,
                                      unsigned AddrSpace, AsmPrinter &AP) {
   for (unsigned i = 0, e = CV->getType()->getNumElements(); i != e; ++i)
-    EmitGlobalConstantImpl(CV->getOperand(i), AddrSpace, AP);
+    emitGlobalConstantImpl(CV->getOperand(i), AddrSpace, AP);
 
-  const TargetData &TD = *AP.TM.getTargetData();
+  const DataLayout &TD = *AP.TM.getDataLayout();
   unsigned Size = TD.getTypeAllocSize(CV->getType());
   unsigned EmittedSize = TD.getTypeAllocSize(CV->getType()->getElementType()) *
                          CV->getType()->getNumElements();
@@ -1710,10 +1715,10 @@ static void EmitGlobalConstantVector(const ConstantVector *CV,
     AP.OutStreamer.EmitZeros(Padding, AddrSpace);
 }
 
-static void EmitGlobalConstantStruct(const ConstantStruct *CS,
+static void emitGlobalConstantStruct(const ConstantStruct *CS,
                                      unsigned AddrSpace, AsmPrinter &AP) {
   // Print the fields in successive locations. Pad to align if needed!
-  const TargetData *TD = AP.TM.getTargetData();
+  const DataLayout *TD = AP.TM.getDataLayout();
   unsigned Size = TD->getTypeAllocSize(CS->getType());
   const StructLayout *Layout = TD->getStructLayout(CS->getType());
   uint64_t SizeSoFar = 0;
@@ -1727,7 +1732,7 @@ static void EmitGlobalConstantStruct(const ConstantStruct *CS,
     SizeSoFar += FieldSize + PadSize;
 
     // Now print the actual field value.
-    EmitGlobalConstantImpl(Field, AddrSpace, AP);
+    emitGlobalConstantImpl(Field, AddrSpace, AP);
 
     // Insert padding - this may include padding to increase the size of the
     // current field up to the ABI size (if the struct is not packed) as well
@@ -1738,7 +1743,7 @@ static void EmitGlobalConstantStruct(const ConstantStruct *CS,
          "Layout of constant struct may be incorrect!");
 }
 
-static void EmitGlobalConstantFP(const ConstantFP *CFP, unsigned AddrSpace,
+static void emitGlobalConstantFP(const ConstantFP *CFP, unsigned AddrSpace,
                                  AsmPrinter &AP) {
   if (CFP->getType()->isHalfTy()) {
     if (AP.isVerbose()) {
@@ -1793,7 +1798,7 @@ static void EmitGlobalConstantFP(const ConstantFP *CFP, unsigned AddrSpace,
         << DoubleVal.convertToDouble() << '\n';
     }
 
-    if (AP.TM.getTargetData()->isBigEndian()) {
+    if (AP.TM.getDataLayout()->isBigEndian()) {
       AP.OutStreamer.EmitIntValue(p[1], 2, AddrSpace);
       AP.OutStreamer.EmitIntValue(p[0], 8, AddrSpace);
     } else {
@@ -1802,7 +1807,7 @@ static void EmitGlobalConstantFP(const ConstantFP *CFP, unsigned AddrSpace,
     }
 
     // Emit the tail padding for the long double.
-    const TargetData &TD = *AP.TM.getTargetData();
+    const DataLayout &TD = *AP.TM.getDataLayout();
     AP.OutStreamer.EmitZeros(TD.getTypeAllocSize(CFP->getType()) -
                              TD.getTypeStoreSize(CFP->getType()), AddrSpace);
     return;
@@ -1814,7 +1819,7 @@ static void EmitGlobalConstantFP(const ConstantFP *CFP, unsigned AddrSpace,
   // API needed to prevent premature destruction.
   APInt API = CFP->getValueAPF().bitcastToAPInt();
   const uint64_t *p = API.getRawData();
-  if (AP.TM.getTargetData()->isBigEndian()) {
+  if (AP.TM.getDataLayout()->isBigEndian()) {
     AP.OutStreamer.EmitIntValue(p[0], 8, AddrSpace);
     AP.OutStreamer.EmitIntValue(p[1], 8, AddrSpace);
   } else {
@@ -1823,9 +1828,9 @@ static void EmitGlobalConstantFP(const ConstantFP *CFP, unsigned AddrSpace,
   }
 }
 
-static void EmitGlobalConstantLargeInt(const ConstantInt *CI,
+static void emitGlobalConstantLargeInt(const ConstantInt *CI,
                                        unsigned AddrSpace, AsmPrinter &AP) {
-  const TargetData *TD = AP.TM.getTargetData();
+  const DataLayout *TD = AP.TM.getDataLayout();
   unsigned BitWidth = CI->getBitWidth();
   assert((BitWidth & 63) == 0 && "only support multiples of 64-bits");
 
@@ -1839,9 +1844,9 @@ static void EmitGlobalConstantLargeInt(const ConstantInt *CI,
   }
 }
 
-static void EmitGlobalConstantImpl(const Constant *CV, unsigned AddrSpace,
+static void emitGlobalConstantImpl(const Constant *CV, unsigned AddrSpace,
                                    AsmPrinter &AP) {
-  const TargetData *TD = AP.TM.getTargetData();
+  const DataLayout *TD = AP.TM.getDataLayout();
   uint64_t Size = TD->getTypeAllocSize(CV->getType());
   if (isa<ConstantAggregateZero>(CV) || isa<UndefValue>(CV))
     return AP.OutStreamer.EmitZeros(Size, AddrSpace);
@@ -1858,13 +1863,13 @@ static void EmitGlobalConstantImpl(const Constant *CV, unsigned AddrSpace,
       AP.OutStreamer.EmitIntValue(CI->getZExtValue(), Size, AddrSpace);
       return;
     default:
-      EmitGlobalConstantLargeInt(CI, AddrSpace, AP);
+      emitGlobalConstantLargeInt(CI, AddrSpace, AP);
       return;
     }
   }
 
   if (const ConstantFP *CFP = dyn_cast<ConstantFP>(CV))
-    return EmitGlobalConstantFP(CFP, AddrSpace, AP);
+    return emitGlobalConstantFP(CFP, AddrSpace, AP);
 
   if (isa<ConstantPointerNull>(CV)) {
     AP.OutStreamer.EmitIntValue(0, Size, AddrSpace);
@@ -1872,19 +1877,19 @@ static void EmitGlobalConstantImpl(const Constant *CV, unsigned AddrSpace,
   }
 
   if (const ConstantDataSequential *CDS = dyn_cast<ConstantDataSequential>(CV))
-    return EmitGlobalConstantDataSequential(CDS, AddrSpace, AP);
+    return emitGlobalConstantDataSequential(CDS, AddrSpace, AP);
   
   if (const ConstantArray *CVA = dyn_cast<ConstantArray>(CV))
-    return EmitGlobalConstantArray(CVA, AddrSpace, AP);
+    return emitGlobalConstantArray(CVA, AddrSpace, AP);
 
   if (const ConstantStruct *CVS = dyn_cast<ConstantStruct>(CV))
-    return EmitGlobalConstantStruct(CVS, AddrSpace, AP);
+    return emitGlobalConstantStruct(CVS, AddrSpace, AP);
 
   if (const ConstantExpr *CE = dyn_cast<ConstantExpr>(CV)) {
     // Look through bitcasts, which might not be able to be MCExpr'ized (e.g. of
     // vectors).
     if (CE->getOpcode() == Instruction::BitCast)
-      return EmitGlobalConstantImpl(CE->getOperand(0), AddrSpace, AP);
+      return emitGlobalConstantImpl(CE->getOperand(0), AddrSpace, AP);
 
     if (Size > 8) {
       // If the constant expression's size is greater than 64-bits, then we have
@@ -1892,23 +1897,23 @@ static void EmitGlobalConstantImpl(const Constant *CV, unsigned AddrSpace,
       // that way.
       Constant *New = ConstantFoldConstantExpression(CE, TD);
       if (New && New != CE)
-        return EmitGlobalConstantImpl(New, AddrSpace, AP);
+        return emitGlobalConstantImpl(New, AddrSpace, AP);
     }
   }
   
   if (const ConstantVector *V = dyn_cast<ConstantVector>(CV))
-    return EmitGlobalConstantVector(V, AddrSpace, AP);
+    return emitGlobalConstantVector(V, AddrSpace, AP);
     
   // Otherwise, it must be a ConstantExpr.  Lower it to an MCExpr, then emit it
   // thread the streamer with EmitValue.
-  AP.OutStreamer.EmitValue(LowerConstant(CV, AP), Size, AddrSpace);
+  AP.OutStreamer.EmitValue(lowerConstant(CV, AP), Size, AddrSpace);
 }
 
 /// EmitGlobalConstant - Print a general LLVM constant to the .s file.
 void AsmPrinter::EmitGlobalConstant(const Constant *CV, unsigned AddrSpace) {
-  uint64_t Size = TM.getTargetData()->getTypeAllocSize(CV->getType());
+  uint64_t Size = TM.getDataLayout()->getTypeAllocSize(CV->getType());
   if (Size)
-    EmitGlobalConstantImpl(CV, AddrSpace, *this);
+    emitGlobalConstantImpl(CV, AddrSpace, *this);
   else if (MAI->hasSubsectionsViaSymbols()) {
     // If the global has zero size, emit a single byte so that two labels don't
     // look like they are at the same location.
@@ -2023,8 +2028,8 @@ static void PrintChildLoopComment(raw_ostream &OS, const MachineLoop *Loop,
   }
 }
 
-/// EmitBasicBlockLoopComments - Pretty-print comments for basic blocks.
-static void EmitBasicBlockLoopComments(const MachineBasicBlock &MBB,
+/// emitBasicBlockLoopComments - Pretty-print comments for basic blocks.
+static void emitBasicBlockLoopComments(const MachineBasicBlock &MBB,
                                        const MachineLoopInfo *LI,
                                        const AsmPrinter &AP) {
   // Add loop depth information
@@ -2090,7 +2095,7 @@ void AsmPrinter::EmitBasicBlockStart(const MachineBasicBlock *MBB) const {
     if (const BasicBlock *BB = MBB->getBasicBlock())
       if (BB->hasName())
         OutStreamer.AddComment("%" + BB->getName());
-    EmitBasicBlockLoopComments(*MBB, LI, *this);
+    emitBasicBlockLoopComments(*MBB, LI, *this);
   }
 
   // Print the main label for the block.
diff --git a/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp b/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp
index 90d511c..d94e1fe 100644
--- a/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp
+++ b/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp
@@ -18,7 +18,7 @@
 #include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSymbol.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetFrameLowering.h"
 #include "llvm/Target/TargetLoweringObjectFile.h"
 #include "llvm/Target/TargetMachine.h"
@@ -112,7 +112,7 @@ unsigned AsmPrinter::GetSizeOfEncodedValue(unsigned Encoding) const {
   
   switch (Encoding & 0x07) {
   default: llvm_unreachable("Invalid encoded value.");
-  case dwarf::DW_EH_PE_absptr: return TM.getTargetData()->getPointerSize();
+  case dwarf::DW_EH_PE_absptr: return TM.getDataLayout()->getPointerSize();
   case dwarf::DW_EH_PE_udata2: return 2;
   case dwarf::DW_EH_PE_udata4: return 4;
   case dwarf::DW_EH_PE_udata8: return 8;
diff --git a/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp b/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp
index db43b06..50f0fc3 100644
--- a/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp
+++ b/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp
@@ -43,10 +43,10 @@ namespace {
   };
 }
 
-/// SrcMgrDiagHandler - This callback is invoked when the SourceMgr for an
+/// srcMgrDiagHandler - This callback is invoked when the SourceMgr for an
 /// inline asm has an error in it.  diagInfo is a pointer to the SrcMgrDiagInfo
 /// struct above.
-static void SrcMgrDiagHandler(const SMDiagnostic &Diag, void *diagInfo) {
+static void srcMgrDiagHandler(const SMDiagnostic &Diag, void *diagInfo) {
   SrcMgrDiagInfo *DiagInfo = static_cast<SrcMgrDiagInfo *>(diagInfo);
   assert(DiagInfo && "Diagnostic context not passed down?");
 
@@ -68,7 +68,8 @@ static void SrcMgrDiagHandler(const SMDiagnostic &Diag, void *diagInfo) {
 }
 
 /// EmitInlineAsm - Emit a blob of inline asm to the output streamer.
-void AsmPrinter::EmitInlineAsm(StringRef Str, const MDNode *LocMDNode) const {
+void AsmPrinter::EmitInlineAsm(StringRef Str, const MDNode *LocMDNode,
+                               InlineAsm::AsmDialect Dialect) const {
   assert(!Str.empty() && "Can't emit empty inline asm block");
 
   // Remember if the buffer is nul terminated or not so we can avoid a copy.
@@ -91,12 +92,12 @@ void AsmPrinter::EmitInlineAsm(StringRef Str, const MDNode *LocMDNode) const {
   LLVMContext &LLVMCtx = MMI->getModule()->getContext();
   bool HasDiagHandler = false;
   if (LLVMCtx.getInlineAsmDiagnosticHandler() != 0) {
-    // If the source manager has an issue, we arrange for SrcMgrDiagHandler
+    // If the source manager has an issue, we arrange for srcMgrDiagHandler
     // to be invoked, getting DiagInfo passed into it.
     DiagInfo.LocInfo = LocMDNode;
     DiagInfo.DiagHandler = LLVMCtx.getInlineAsmDiagnosticHandler();
     DiagInfo.DiagContext = LLVMCtx.getInlineAsmDiagnosticContext();
-    SrcMgr.setDiagHandler(SrcMgrDiagHandler, &DiagInfo);
+    SrcMgr.setDiagHandler(srcMgrDiagHandler, &DiagInfo);
     HasDiagHandler = true;
   }
 
@@ -126,6 +127,7 @@ void AsmPrinter::EmitInlineAsm(StringRef Str, const MDNode *LocMDNode) const {
   if (!TAP)
     report_fatal_error("Inline asm not supported by this streamer because"
                        " we don't have an asm parser for this target\n");
+  Parser->setAssemblerDialect(Dialect);
   Parser->setTargetParser(*TAP.get());
 
   // Don't implicitly switch to the text section before the asm.
@@ -135,71 +137,113 @@ void AsmPrinter::EmitInlineAsm(StringRef Str, const MDNode *LocMDNode) const {
     report_fatal_error("Error parsing inline asm\n");
 }
 
+static void EmitMSInlineAsmStr(const char *AsmStr, const MachineInstr *MI,
+                               MachineModuleInfo *MMI, int InlineAsmVariant,
+                               AsmPrinter *AP, unsigned LocCookie,
+                               raw_ostream &OS) {
+  // Switch to the inline assembly variant.
+  OS << "\t.intel_syntax\n\t";
 
-/// EmitInlineAsm - This method formats and emits the specified machine
-/// instruction that is an inline asm.
-void AsmPrinter::EmitInlineAsm(const MachineInstr *MI) const {
-  assert(MI->isInlineAsm() && "printInlineAsm only works on inline asms");
-
+  const char *LastEmitted = AsmStr; // One past the last character emitted.
   unsigned NumOperands = MI->getNumOperands();
 
-  // Count the number of register definitions to find the asm string.
-  unsigned NumDefs = 0;
-  for (; MI->getOperand(NumDefs).isReg() && MI->getOperand(NumDefs).isDef();
-       ++NumDefs)
-    assert(NumDefs != NumOperands-2 && "No asm string?");
+  while (*LastEmitted) {
+    switch (*LastEmitted) {
+    default: {
+      // Not a special case, emit the string section literally.
+      const char *LiteralEnd = LastEmitted+1;
+      while (*LiteralEnd && *LiteralEnd != '{' && *LiteralEnd != '|' &&
+             *LiteralEnd != '}' && *LiteralEnd != '$' && *LiteralEnd != '\n')
+        ++LiteralEnd;
 
-  assert(MI->getOperand(NumDefs).isSymbol() && "No asm string?");
+      OS.write(LastEmitted, LiteralEnd-LastEmitted);
+      LastEmitted = LiteralEnd;
+      break;
+    }
+    case '\n':
+      ++LastEmitted;   // Consume newline character.
+      OS << '\n';      // Indent code with newline.
+      break;
+    case '$': {
+      ++LastEmitted;   // Consume '$' character.
+      bool Done = true;
 
-  // Disassemble the AsmStr, printing out the literal pieces, the operands, etc.
-  const char *AsmStr = MI->getOperand(NumDefs).getSymbolName();
+      // Handle escapes.
+      switch (*LastEmitted) {
+      default: Done = false; break;
+      case '$':
+        ++LastEmitted;  // Consume second '$' character.
+        break;
+      }
+      if (Done) break;
 
-  // If this asmstr is empty, just print the #APP/#NOAPP markers.
-  // These are useful to see where empty asm's wound up.
-  if (AsmStr[0] == 0) {
-    // Don't emit the comments if writing to a .o file.
-    if (!OutStreamer.hasRawTextSupport()) return;
+      const char *IDStart = LastEmitted;
+      const char *IDEnd = IDStart;
+      while (*IDEnd >= '0' && *IDEnd <= '9') ++IDEnd;
 
-    OutStreamer.EmitRawText(Twine("\t")+MAI->getCommentString()+
-                            MAI->getInlineAsmStart());
-    OutStreamer.EmitRawText(Twine("\t")+MAI->getCommentString()+
-                            MAI->getInlineAsmEnd());
-    return;
-  }
+      unsigned Val;
+      if (StringRef(IDStart, IDEnd-IDStart).getAsInteger(10, Val))
+        report_fatal_error("Bad $ operand number in inline asm string: '" +
+                           Twine(AsmStr) + "'");
+      LastEmitted = IDEnd;
 
-  // Emit the #APP start marker.  This has to happen even if verbose-asm isn't
-  // enabled, so we use EmitRawText.
-  if (OutStreamer.hasRawTextSupport())
-    OutStreamer.EmitRawText(Twine("\t")+MAI->getCommentString()+
-                            MAI->getInlineAsmStart());
+      if (Val >= NumOperands-1)
+        report_fatal_error("Invalid $ operand number in inline asm string: '" +
+                           Twine(AsmStr) + "'");
 
-  // Get the !srcloc metadata node if we have it, and decode the loc cookie from
-  // it.
-  unsigned LocCookie = 0;
-  const MDNode *LocMD = 0;
-  for (unsigned i = MI->getNumOperands(); i != 0; --i) {
-    if (MI->getOperand(i-1).isMetadata() &&
-        (LocMD = MI->getOperand(i-1).getMetadata()) &&
-        LocMD->getNumOperands() != 0) {
-      if (const ConstantInt *CI = dyn_cast<ConstantInt>(LocMD->getOperand(0))) {
-        LocCookie = CI->getZExtValue();
-        break;
-      }
-    }
-  }
+      // Okay, we finally have a value number.  Ask the target to print this
+      // operand!
+      unsigned OpNo = InlineAsm::MIOp_FirstOperand;
 
-  // Emit the inline asm to a temporary string so we can emit it through
-  // EmitInlineAsm.
-  SmallString<256> StringData;
-  raw_svector_ostream OS(StringData);
+      bool Error = false;
 
-  OS << '\t';
+      // Scan to find the machine operand number for the operand.
+      for (; Val; --Val) {
+        if (OpNo >= MI->getNumOperands()) break;
+        unsigned OpFlags = MI->getOperand(OpNo).getImm();
+        OpNo += InlineAsm::getNumOperandRegisters(OpFlags) + 1;
+      }
 
-  // The variant of the current asmprinter.
-  int AsmPrinterVariant = MAI->getAssemblerDialect();
+      // We may have a location metadata attached to the end of the
+      // instruction, and at no point should see metadata at any
+      // other point while processing. It's an error if so.
+      if (OpNo >= MI->getNumOperands() ||
+          MI->getOperand(OpNo).isMetadata()) {
+        Error = true;
+      } else {
+        unsigned OpFlags = MI->getOperand(OpNo).getImm();
+        ++OpNo;  // Skip over the ID number.
+        
+        if (InlineAsm::isMemKind(OpFlags)) {
+          Error = AP->PrintAsmMemoryOperand(MI, OpNo, InlineAsmVariant,
+                                            /*Modifier*/ 0, OS);
+        } else {
+          Error = AP->PrintAsmOperand(MI, OpNo, InlineAsmVariant,
+                                      /*Modifier*/ 0, OS);
+        }
+      }
+      if (Error) {
+        std::string msg;
+        raw_string_ostream Msg(msg);
+        Msg << "invalid operand in inline asm: '" << AsmStr << "'";
+        MMI->getModule()->getContext().emitError(LocCookie, Msg.str());
+      }
+      break;
+    }
+    }
+  }
+  OS << "\n\t.att_syntax\n" << (char)0;  // null terminate string.
+}
 
+static void EmitGCCInlineAsmStr(const char *AsmStr, const MachineInstr *MI,
+                                MachineModuleInfo *MMI, int InlineAsmVariant,
+                                int AsmPrinterVariant, AsmPrinter *AP,
+                                unsigned LocCookie, raw_ostream &OS) {
   int CurVariant = -1;            // The number of the {.|.|.} region we are in.
   const char *LastEmitted = AsmStr; // One past the last character emitted.
+  unsigned NumOperands = MI->getNumOperands();
+
+  OS << '\t';
 
   while (*LastEmitted) {
     switch (*LastEmitted) {
@@ -272,7 +316,7 @@ void AsmPrinter::EmitInlineAsm(const MachineInstr *MI) const {
                              " string: '" + Twine(AsmStr) + "'");
 
         std::string Val(StrStart, StrEnd);
-        PrintSpecial(MI, OS, Val.c_str());
+        AP->PrintSpecial(MI, OS, Val.c_str());
         LastEmitted = StrEnd+1;
         break;
       }
@@ -340,13 +384,12 @@ void AsmPrinter::EmitInlineAsm(const MachineInstr *MI) const {
             // FIXME: What if the operand isn't an MBB, report error?
             OS << *MI->getOperand(OpNo).getMBB()->getSymbol();
           else {
-            AsmPrinter *AP = const_cast<AsmPrinter*>(this);
             if (InlineAsm::isMemKind(OpFlags)) {
-              Error = AP->PrintAsmMemoryOperand(MI, OpNo, AsmPrinterVariant,
+              Error = AP->PrintAsmMemoryOperand(MI, OpNo, InlineAsmVariant,
                                                 Modifier[0] ? Modifier : 0,
                                                 OS);
             } else {
-              Error = AP->PrintAsmOperand(MI, OpNo, AsmPrinterVariant,
+              Error = AP->PrintAsmOperand(MI, OpNo, InlineAsmVariant,
                                           Modifier[0] ? Modifier : 0, OS);
             }
           }
@@ -363,7 +406,74 @@ void AsmPrinter::EmitInlineAsm(const MachineInstr *MI) const {
     }
   }
   OS << '\n' << (char)0;  // null terminate string.
-  EmitInlineAsm(OS.str(), LocMD);
+}
+
+/// EmitInlineAsm - This method formats and emits the specified machine
+/// instruction that is an inline asm.
+void AsmPrinter::EmitInlineAsm(const MachineInstr *MI) const {
+  assert(MI->isInlineAsm() && "printInlineAsm only works on inline asms");
+
+  // Count the number of register definitions to find the asm string.
+  unsigned NumDefs = 0;
+  for (; MI->getOperand(NumDefs).isReg() && MI->getOperand(NumDefs).isDef();
+       ++NumDefs)
+    assert(NumDefs != MI->getNumOperands()-2 && "No asm string?");
+
+  assert(MI->getOperand(NumDefs).isSymbol() && "No asm string?");
+
+  // Disassemble the AsmStr, printing out the literal pieces, the operands, etc.
+  const char *AsmStr = MI->getOperand(NumDefs).getSymbolName();
+
+  // If this asmstr is empty, just print the #APP/#NOAPP markers.
+  // These are useful to see where empty asm's wound up.
+  if (AsmStr[0] == 0) {
+    // Don't emit the comments if writing to a .o file.
+    if (!OutStreamer.hasRawTextSupport()) return;
+
+    OutStreamer.EmitRawText(Twine("\t")+MAI->getCommentString()+
+                            MAI->getInlineAsmStart());
+    OutStreamer.EmitRawText(Twine("\t")+MAI->getCommentString()+
+                            MAI->getInlineAsmEnd());
+    return;
+  }
+
+  // Emit the #APP start marker.  This has to happen even if verbose-asm isn't
+  // enabled, so we use EmitRawText.
+  if (OutStreamer.hasRawTextSupport())
+    OutStreamer.EmitRawText(Twine("\t")+MAI->getCommentString()+
+                            MAI->getInlineAsmStart());
+
+  // Get the !srcloc metadata node if we have it, and decode the loc cookie from
+  // it.
+  unsigned LocCookie = 0;
+  const MDNode *LocMD = 0;
+  for (unsigned i = MI->getNumOperands(); i != 0; --i) {
+    if (MI->getOperand(i-1).isMetadata() &&
+        (LocMD = MI->getOperand(i-1).getMetadata()) &&
+        LocMD->getNumOperands() != 0) {
+      if (const ConstantInt *CI = dyn_cast<ConstantInt>(LocMD->getOperand(0))) {
+        LocCookie = CI->getZExtValue();
+        break;
+      }
+    }
+  }
+
+  // Emit the inline asm to a temporary string so we can emit it through
+  // EmitInlineAsm.
+  SmallString<256> StringData;
+  raw_svector_ostream OS(StringData);
+
+  // The variant of the current asmprinter.
+  int AsmPrinterVariant = MAI->getAssemblerDialect();
+  InlineAsm::AsmDialect InlineAsmVariant = MI->getInlineAsmDialect();
+  AsmPrinter *AP = const_cast<AsmPrinter*>(this);
+  if (InlineAsmVariant == InlineAsm::AD_ATT)
+    EmitGCCInlineAsmStr(AsmStr, MI, MMI, InlineAsmVariant, AsmPrinterVariant,
+                        AP, LocCookie, OS);
+  else
+    EmitMSInlineAsmStr(AsmStr, MI, MMI, InlineAsmVariant, AP, LocCookie, OS);
+
+  EmitInlineAsm(OS.str(), LocMD, MI->getInlineAsmDialect());
 
   // Emit the #NOAPP end marker.  This has to happen even if verbose-asm isn't
   // enabled, so we use EmitRawText.
@@ -409,8 +519,8 @@ void AsmPrinter::PrintSpecial(const MachineInstr *MI, raw_ostream &OS,
 /// instruction, using the specified assembler variant.  Targets should
 /// override this to format as appropriate.
 bool AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
-    unsigned AsmVariant, const char *ExtraCode,
-    raw_ostream &O) {
+                                 unsigned AsmVariant, const char *ExtraCode,
+                                 raw_ostream &O) {
   // Does this asm operand have a single letter operand modifier?
   if (ExtraCode && ExtraCode[0]) {
     if (ExtraCode[1] != 0) return true; // Unknown modifier.
diff --git a/lib/CodeGen/AsmPrinter/DIE.cpp b/lib/CodeGen/AsmPrinter/DIE.cpp
index 3776848..4d73b3c 100644
--- a/lib/CodeGen/AsmPrinter/DIE.cpp
+++ b/lib/CodeGen/AsmPrinter/DIE.cpp
@@ -17,7 +17,7 @@
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSymbol.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -182,6 +182,12 @@ void DIEValue::dump() {
 void DIEInteger::EmitValue(AsmPrinter *Asm, unsigned Form) const {
   unsigned Size = ~0U;
   switch (Form) {
+  case dwarf::DW_FORM_flag_present:
+    // Emit something to keep the lines and comments in sync.
+    // FIXME: Is there a better way to do this?
+    if (Asm->OutStreamer.hasRawTextSupport())
+      Asm->OutStreamer.EmitRawText(StringRef(""));
+    return;
   case dwarf::DW_FORM_flag:  // Fall thru
   case dwarf::DW_FORM_ref1:  // Fall thru
   case dwarf::DW_FORM_data1: Size = 1; break;
@@ -193,7 +199,8 @@ void DIEInteger::EmitValue(AsmPrinter *Asm, unsigned Form) const {
   case dwarf::DW_FORM_data8: Size = 8; break;
   case dwarf::DW_FORM_udata: Asm->EmitULEB128(Integer); return;
   case dwarf::DW_FORM_sdata: Asm->EmitSLEB128(Integer); return;
-  case dwarf::DW_FORM_addr:  Size = Asm->getTargetData().getPointerSize(); break;
+  case dwarf::DW_FORM_addr:
+    Size = Asm->getDataLayout().getPointerSize(); break;
   default: llvm_unreachable("DIE Value form not supported yet");
   }
   Asm->OutStreamer.EmitIntValue(Integer, Size, 0/*addrspace*/);
@@ -203,6 +210,7 @@ void DIEInteger::EmitValue(AsmPrinter *Asm, unsigned Form) const {
 ///
 unsigned DIEInteger::SizeOf(AsmPrinter *AP, unsigned Form) const {
   switch (Form) {
+  case dwarf::DW_FORM_flag_present: return 0;
   case dwarf::DW_FORM_flag:  // Fall thru
   case dwarf::DW_FORM_ref1:  // Fall thru
   case dwarf::DW_FORM_data1: return sizeof(int8_t);
@@ -214,7 +222,7 @@ unsigned DIEInteger::SizeOf(AsmPrinter *AP, unsigned Form) const {
   case dwarf::DW_FORM_data8: return sizeof(int64_t);
   case dwarf::DW_FORM_udata: return MCAsmInfo::getULEB128Size(Integer);
   case dwarf::DW_FORM_sdata: return MCAsmInfo::getSLEB128Size(Integer);
-  case dwarf::DW_FORM_addr:  return AP->getTargetData().getPointerSize();
+  case dwarf::DW_FORM_addr:  return AP->getDataLayout().getPointerSize();
   default: llvm_unreachable("DIE Value form not supported yet");
   }
 }
@@ -241,7 +249,7 @@ void DIELabel::EmitValue(AsmPrinter *AP, unsigned Form) const {
 unsigned DIELabel::SizeOf(AsmPrinter *AP, unsigned Form) const {
   if (Form == dwarf::DW_FORM_data4) return 4;
   if (Form == dwarf::DW_FORM_strp) return 4;
-  return AP->getTargetData().getPointerSize();
+  return AP->getDataLayout().getPointerSize();
 }
 
 #ifndef NDEBUG
@@ -265,7 +273,7 @@ void DIEDelta::EmitValue(AsmPrinter *AP, unsigned Form) const {
 unsigned DIEDelta::SizeOf(AsmPrinter *AP, unsigned Form) const {
   if (Form == dwarf::DW_FORM_data4) return 4;
   if (Form == dwarf::DW_FORM_strp) return 4;
-  return AP->getTargetData().getPointerSize();
+  return AP->getDataLayout().getPointerSize();
 }
 
 #ifndef NDEBUG
diff --git a/lib/CodeGen/AsmPrinter/DIE.h b/lib/CodeGen/AsmPrinter/DIE.h
index f93ea1b..28a96f3 100644
--- a/lib/CodeGen/AsmPrinter/DIE.h
+++ b/lib/CodeGen/AsmPrinter/DIE.h
@@ -214,9 +214,6 @@ namespace llvm {
     ///
     virtual unsigned SizeOf(AsmPrinter *AP, unsigned Form) const = 0;
 
-    // Implement isa/cast/dyncast.
-    static bool classof(const DIEValue *) { return true; }
-
 #ifndef NDEBUG
     virtual void print(raw_ostream &O) = 0;
     void dump();
@@ -257,7 +254,6 @@ namespace llvm {
     virtual unsigned SizeOf(AsmPrinter *AP, unsigned Form) const;
 
     // Implement isa/cast/dyncast.
-    static bool classof(const DIEInteger *) { return true; }
     static bool classof(const DIEValue *I) { return I->getType() == isInteger; }
 
 #ifndef NDEBUG
@@ -286,7 +282,6 @@ namespace llvm {
     virtual unsigned SizeOf(AsmPrinter *AP, unsigned Form) const;
 
     // Implement isa/cast/dyncast.
-    static bool classof(const DIELabel *)  { return true; }
     static bool classof(const DIEValue *L) { return L->getType() == isLabel; }
 
 #ifndef NDEBUG
@@ -313,7 +308,6 @@ namespace llvm {
     virtual unsigned SizeOf(AsmPrinter *AP, unsigned Form) const;
 
     // Implement isa/cast/dyncast.
-    static bool classof(const DIEDelta *)  { return true; }
     static bool classof(const DIEValue *D) { return D->getType() == isDelta; }
 
 #ifndef NDEBUG
@@ -343,7 +337,6 @@ namespace llvm {
     }
 
     // Implement isa/cast/dyncast.
-    static bool classof(const DIEEntry *)  { return true; }
     static bool classof(const DIEValue *E) { return E->getType() == isEntry; }
 
 #ifndef NDEBUG
@@ -383,7 +376,6 @@ namespace llvm {
     virtual unsigned SizeOf(AsmPrinter *AP, unsigned Form) const;
 
     // Implement isa/cast/dyncast.
-    static bool classof(const DIEBlock *)  { return true; }
     static bool classof(const DIEValue *E) { return E->getType() == isBlock; }
 
 #ifndef NDEBUG
diff --git a/lib/CodeGen/AsmPrinter/DwarfAccelTable.cpp b/lib/CodeGen/AsmPrinter/DwarfAccelTable.cpp
index 454a923..05e0f2f 100644
--- a/lib/CodeGen/AsmPrinter/DwarfAccelTable.cpp
+++ b/lib/CodeGen/AsmPrinter/DwarfAccelTable.cpp
@@ -133,8 +133,8 @@ void DwarfAccelTable::EmitHeader(AsmPrinter *Asm) {
   }
 }
 
-// Walk through and emit the buckets for the table. This will look
-// like a list of numbers of how many elements are in each bucket.
+// Walk through and emit the buckets for the table. Each index is
+// an offset into the list of hashes.
 void DwarfAccelTable::EmitBuckets(AsmPrinter *Asm) {
   unsigned index = 0;
   for (size_t i = 0, e = Buckets.size(); i < e; ++i) {
diff --git a/lib/CodeGen/AsmPrinter/DwarfAccelTable.h b/lib/CodeGen/AsmPrinter/DwarfAccelTable.h
index 963b8cd..92d1bbe 100644
--- a/lib/CodeGen/AsmPrinter/DwarfAccelTable.h
+++ b/lib/CodeGen/AsmPrinter/DwarfAccelTable.h
@@ -237,8 +237,8 @@ private:
     #endif
   };
 
-  DwarfAccelTable(const DwarfAccelTable&); // DO NOT IMPLEMENT
-  void operator=(const DwarfAccelTable&);  // DO NOT IMPLEMENT
+  DwarfAccelTable(const DwarfAccelTable&) LLVM_DELETED_FUNCTION;
+  void operator=(const DwarfAccelTable&) LLVM_DELETED_FUNCTION;
 
   // Internal Functions
   void EmitHeader(AsmPrinter *);
diff --git a/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp b/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp
index d975f1f..4fdd5ca 100644
--- a/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp
+++ b/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp
@@ -25,7 +25,7 @@
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/Target/Mangler.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetFrameLowering.h"
 #include "llvm/Target/TargetLoweringObjectFile.h"
 #include "llvm/Target/TargetMachine.h"
diff --git a/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp b/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
index d30e5bb..2b07dda 100644
--- a/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
+++ b/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp
@@ -22,7 +22,7 @@
 #include "llvm/Instructions.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Target/Mangler.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetFrameLowering.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetRegisterInfo.h"
@@ -51,6 +51,15 @@ DIEEntry *CompileUnit::createDIEEntry(DIE *Entry) {
   return Value;
 }
 
+/// addFlag - Add a flag that is true.
+void CompileUnit::addFlag(DIE *Die, unsigned Attribute) {
+  if (!DD->useDarwinGDBCompat())
+    Die->addValue(Attribute, dwarf::DW_FORM_flag_present,
+                  DIEIntegerOne);
+  else
+    addUInt(Die, Attribute, dwarf::DW_FORM_flag, 1);
+}
+
 /// addUInt - Add an unsigned integer attribute data and value.
 ///
 void CompileUnit::addUInt(DIE *Die, unsigned Attribute,
@@ -501,7 +510,7 @@ bool CompileUnit::addConstantFPValue(DIE *Die, const MachineOperand &MO) {
   const char *FltPtr = (const char*)FltVal.getRawData();
 
   int NumBytes = FltVal.getBitWidth() / 8; // 8 bits per byte.
-  bool LittleEndian = Asm->getTargetData().isLittleEndian();
+  bool LittleEndian = Asm->getDataLayout().isLittleEndian();
   int Incr = (LittleEndian ? 1 : -1);
   int Start = (LittleEndian ? 0 : NumBytes - 1);
   int Stop = (LittleEndian ? NumBytes : -1);
@@ -543,7 +552,7 @@ bool CompileUnit::addConstantValue(DIE *Die, const ConstantInt *CI,
   const uint64_t *Ptr64 = Val.getRawData();
 
   int NumBytes = Val.getBitWidth() / 8; // 8 bits per byte.
-  bool LittleEndian = Asm->getTargetData().isLittleEndian();
+  bool LittleEndian = Asm->getDataLayout().isLittleEndian();
 
   // Output the constant to DWARF one byte at a time.
   for (int i = 0; i < NumBytes; i++) {
@@ -794,7 +803,7 @@ void CompileUnit::constructTypeDIE(DIE &Buffer, DICompositeType CTy) {
         (Language == dwarf::DW_LANG_C89 ||
          Language == dwarf::DW_LANG_C99 ||
          Language == dwarf::DW_LANG_ObjC))
-      addUInt(&Buffer, dwarf::DW_AT_prototyped, dwarf::DW_FORM_flag, 1);
+      addFlag(&Buffer, dwarf::DW_AT_prototyped);
   }
     break;
   case dwarf::DW_TAG_structure_type:
@@ -825,15 +834,15 @@ void CompileUnit::constructTypeDIE(DIE &Buffer, DICompositeType CTy) {
           addUInt(ElemDie, dwarf::DW_AT_accessibility, dwarf::DW_FORM_data1,
             dwarf::DW_ACCESS_public);
         if (SP.isExplicit())
-          addUInt(ElemDie, dwarf::DW_AT_explicit, dwarf::DW_FORM_flag, 1);
+          addFlag(ElemDie, dwarf::DW_AT_explicit);
       }
       else if (Element.isVariable()) {
         DIVariable DV(Element);
         ElemDie = new DIE(dwarf::DW_TAG_variable);
         addString(ElemDie, dwarf::DW_AT_name, DV.getName());
         addType(ElemDie, DV.getType());
-        addUInt(ElemDie, dwarf::DW_AT_declaration, dwarf::DW_FORM_flag, 1);
-        addUInt(ElemDie, dwarf::DW_AT_external, dwarf::DW_FORM_flag, 1);
+        addFlag(ElemDie, dwarf::DW_AT_declaration);
+        addFlag(ElemDie, dwarf::DW_AT_external);
         addSourceLine(ElemDie, DV);
       } else if (Element.isDerivedType()) {
         DIDerivedType DDTy(Element);
@@ -883,7 +892,7 @@ void CompileUnit::constructTypeDIE(DIE &Buffer, DICompositeType CTy) {
     }
 
     if (CTy.isAppleBlockExtension())
-      addUInt(&Buffer, dwarf::DW_AT_APPLE_block, dwarf::DW_FORM_flag, 1);
+      addFlag(&Buffer, dwarf::DW_AT_APPLE_block);
 
     DICompositeType ContainingType = CTy.getContainingType();
     if (DIDescriptor(ContainingType).isCompositeType())
@@ -895,8 +904,7 @@ void CompileUnit::constructTypeDIE(DIE &Buffer, DICompositeType CTy) {
     }
 
     if (CTy.isObjcClassComplete())
-      addUInt(&Buffer, dwarf::DW_AT_APPLE_objc_complete_type,
-              dwarf::DW_FORM_flag, 1);
+      addFlag(&Buffer, dwarf::DW_AT_APPLE_objc_complete_type);
 
     // Add template parameters to a class, structure or union types.
     // FIXME: The support isn't in the metadata for this yet.
@@ -929,7 +937,7 @@ void CompileUnit::constructTypeDIE(DIE &Buffer, DICompositeType CTy) {
 
     // If we're a forward decl, say so.
     if (CTy.isForwardDecl())
-      addUInt(&Buffer, dwarf::DW_AT_declaration, dwarf::DW_FORM_flag, 1);
+      addFlag(&Buffer, dwarf::DW_AT_declaration);
 
     // Add source line info if available.
     if (!CTy.isForwardDecl())
@@ -1028,8 +1036,10 @@ DIE *CompileUnit::getOrCreateSubprogramDIE(DISubprogram SP) {
   // AT_specification code in order to work around a bug in older
   // gdbs that requires the linkage name to resolve multiple template
   // functions.
+  // TODO: Remove this set of code when we get rid of the old gdb
+  // compatibility.
   StringRef LinkageName = SP.getLinkageName();
-  if (!LinkageName.empty())
+  if (!LinkageName.empty() && DD->useDarwinGDBCompat())
     addString(SPDie, dwarf::DW_AT_MIPS_linkage_name,
               getRealLinkageName(LinkageName));
 
@@ -1043,6 +1053,11 @@ DIE *CompileUnit::getOrCreateSubprogramDIE(DISubprogram SP) {
     return SPDie;
   }
 
+  // Add the linkage name if we have one.
+  if (!LinkageName.empty() && !DD->useDarwinGDBCompat())
+    addString(SPDie, dwarf::DW_AT_MIPS_linkage_name,
+              getRealLinkageName(LinkageName));
+
   // Constructors and operators for anonymous aggregates do not have names.
   if (!SP.getName().empty())
     addString(SPDie, dwarf::DW_AT_name, SP.getName());
@@ -1055,7 +1070,7 @@ DIE *CompileUnit::getOrCreateSubprogramDIE(DISubprogram SP) {
       (Language == dwarf::DW_LANG_C89 ||
        Language == dwarf::DW_LANG_C99 ||
        Language == dwarf::DW_LANG_ObjC))
-    addUInt(SPDie, dwarf::DW_AT_prototyped, dwarf::DW_FORM_flag, 1);
+    addFlag(SPDie, dwarf::DW_AT_prototyped);
 
   // Add Return Type.
   DICompositeType SPTy = SP.getType();
@@ -1079,7 +1094,7 @@ DIE *CompileUnit::getOrCreateSubprogramDIE(DISubprogram SP) {
   }
 
   if (!SP.isDefinition()) {
-    addUInt(SPDie, dwarf::DW_AT_declaration, dwarf::DW_FORM_flag, 1);
+    addFlag(SPDie, dwarf::DW_AT_declaration);
     
     // Add arguments. Do not add arguments for subprogram definition. They will
     // be handled while processing variables.
@@ -1090,22 +1105,22 @@ DIE *CompileUnit::getOrCreateSubprogramDIE(DISubprogram SP) {
     if (SPTag == dwarf::DW_TAG_subroutine_type)
       for (unsigned i = 1, N =  Args.getNumElements(); i < N; ++i) {
         DIE *Arg = new DIE(dwarf::DW_TAG_formal_parameter);
-        DIType ATy = DIType(DIType(Args.getElement(i)));
+        DIType ATy = DIType(Args.getElement(i));
         addType(Arg, ATy);
         if (ATy.isArtificial())
-          addUInt(Arg, dwarf::DW_AT_artificial, dwarf::DW_FORM_flag, 1);
+          addFlag(Arg, dwarf::DW_AT_artificial);
         SPDie->addChild(Arg);
       }
   }
 
   if (SP.isArtificial())
-    addUInt(SPDie, dwarf::DW_AT_artificial, dwarf::DW_FORM_flag, 1);
+    addFlag(SPDie, dwarf::DW_AT_artificial);
 
   if (!SP.isLocalToUnit())
-    addUInt(SPDie, dwarf::DW_AT_external, dwarf::DW_FORM_flag, 1);
+    addFlag(SPDie, dwarf::DW_AT_external);
 
   if (SP.isOptimized())
-    addUInt(SPDie, dwarf::DW_AT_APPLE_optimized, dwarf::DW_FORM_flag, 1);
+    addFlag(SPDie, dwarf::DW_AT_APPLE_optimized);
 
   if (unsigned isa = Asm->getISAEncoding()) {
     addUInt(SPDie, dwarf::DW_AT_APPLE_isa, dwarf::DW_FORM_flag, isa);
@@ -1168,7 +1183,7 @@ void CompileUnit::createGlobalVariableDIE(const MDNode *N) {
 
   // Add scoping info.
   if (!GV.isLocalToUnit())
-    addUInt(VariableDIE, dwarf::DW_AT_external, dwarf::DW_FORM_flag, 1);
+    addFlag(VariableDIE, dwarf::DW_AT_external);
 
   // Add line number info.
   addSourceLine(VariableDIE, GV);
@@ -1193,8 +1208,7 @@ void CompileUnit::createGlobalVariableDIE(const MDNode *N) {
       addDIEEntry(VariableSpecDIE, dwarf::DW_AT_specification,
                   dwarf::DW_FORM_ref4, VariableDIE);
       addBlock(VariableSpecDIE, dwarf::DW_AT_location, 0, Block);
-      addUInt(VariableDIE, dwarf::DW_AT_declaration, dwarf::DW_FORM_flag,
-                     1);
+      addFlag(VariableDIE, dwarf::DW_AT_declaration);
       addDie(VariableSpecDIE);
     } else {
       addBlock(VariableDIE, dwarf::DW_AT_location, 0, Block);
@@ -1213,7 +1227,7 @@ void CompileUnit::createGlobalVariableDIE(const MDNode *N) {
     addUInt(Block, 0, dwarf::DW_FORM_data1, dwarf::DW_OP_constu);
     SmallVector<Value*, 3> Idx(CE->op_begin()+1, CE->op_end());
     addUInt(Block, 0, dwarf::DW_FORM_udata, 
-                   Asm->getTargetData().getIndexedOffset(Ptr->getType(), Idx));
+                   Asm->getDataLayout().getIndexedOffset(Ptr->getType(), Idx));
     addUInt(Block, 0, dwarf::DW_FORM_data1, dwarf::DW_OP_plus);
     addBlock(VariableDIE, dwarf::DW_AT_location, 0, Block);
   }
@@ -1260,7 +1274,7 @@ void CompileUnit::constructArrayTypeDIE(DIE &Buffer,
                                         DICompositeType *CTy) {
   Buffer.setTag(dwarf::DW_TAG_array_type);
   if (CTy->getTag() == dwarf::DW_TAG_vector_type)
-    addUInt(&Buffer, dwarf::DW_AT_GNU_vector, dwarf::DW_FORM_flag, 1);
+    addFlag(&Buffer, dwarf::DW_AT_GNU_vector);
 
   // Emit derived type.
   addType(&Buffer, CTy->getTypeDerivedFrom());
@@ -1333,8 +1347,7 @@ DIE *CompileUnit::constructVariableDIE(DbgVariable *DV, bool isScopeAbstract) {
   }
 
   if (DV->isArtificial())
-    addUInt(VariableDie, dwarf::DW_AT_artificial,
-                        dwarf::DW_FORM_flag, 1);
+    addFlag(VariableDie, dwarf::DW_AT_artificial);
 
   if (isScopeAbstract) {
     DV->setDIE(VariableDie);
@@ -1446,7 +1459,7 @@ DIE *CompileUnit::createMemberDIE(DIDerivedType DT) {
     Offset -= FieldOffset;
 
     // Maybe we need to work from the other end.
-    if (Asm->getTargetData().isLittleEndian())
+    if (Asm->getDataLayout().isLittleEndian())
       Offset = FieldSize - (Offset + Size);
     addUInt(MemberDie, dwarf::DW_AT_bit_offset, 0, Offset);
 
diff --git a/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h b/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h
index b4ff9e8..fad9b6e 100644
--- a/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h
+++ b/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h
@@ -176,6 +176,9 @@ public:
   }
 public:
 
+  /// addFlag - Add a flag that is true to the DIE.
+  void addFlag(DIE *Die, unsigned Attribute);
+  
   /// addUInt - Add an unsigned integer attribute data and value.
   ///
   void addUInt(DIE *Die, unsigned Attribute, unsigned Form, uint64_t Integer);
@@ -280,8 +283,8 @@ public:
   /// for the given DITemplateTypeParameter.
   DIE *getOrCreateTemplateTypeParameterDIE(DITemplateTypeParameter TP);
 
-  /// getOrCreateTemplateValueParameterDIE - Find existing DIE or create new DIE 
-  /// for the given DITemplateValueParameter.
+  /// getOrCreateTemplateValueParameterDIE - Find existing DIE or create
+  /// new DIE for the given DITemplateValueParameter.
   DIE *getOrCreateTemplateValueParameterDIE(DITemplateValueParameter TVP);
 
   /// createDIEEntry - Creates a new DIEEntry to be a proxy for a debug
diff --git a/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
index 649684a..367b523 100644
--- a/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
+++ b/lib/CodeGen/AsmPrinter/DwarfDebug.cpp
@@ -27,7 +27,7 @@
 #include "llvm/MC/MCSection.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSymbol.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetFrameLowering.h"
 #include "llvm/Target/TargetLoweringObjectFile.h"
 #include "llvm/Target/TargetMachine.h"
@@ -54,9 +54,29 @@ static cl::opt<bool> UnknownLocations("use-unknown-locations", cl::Hidden,
      cl::desc("Make an absence of debug location information explicit."),
      cl::init(false));
 
-static cl::opt<bool> DwarfAccelTables("dwarf-accel-tables", cl::Hidden,
+namespace {
+  enum DefaultOnOff {
+    Default, Enable, Disable
+  };
+}
+
+static cl::opt<DefaultOnOff> DwarfAccelTables("dwarf-accel-tables", cl::Hidden,
      cl::desc("Output prototype dwarf accelerator tables."),
-     cl::init(false));
+     cl::values(
+                clEnumVal(Default, "Default for platform"),
+                clEnumVal(Enable, "Enabled"),
+                clEnumVal(Disable, "Disabled"),
+                clEnumValEnd),
+     cl::init(Default));
+
+static cl::opt<DefaultOnOff> DarwinGDBCompat("darwin-gdb-compat", cl::Hidden,
+     cl::desc("Compatibility with Darwin gdb."),
+     cl::values(
+                clEnumVal(Default, "Default for platform"),
+                clEnumVal(Enable, "Enabled"),
+                clEnumVal(Disable, "Disabled"),
+                clEnumValEnd),
+     cl::init(Default));
 
 namespace {
   const char *DWARFGroupName = "DWARF Emission";
@@ -135,10 +155,25 @@ DwarfDebug::DwarfDebug(AsmPrinter *A, Module *M)
   DwarfDebugRangeSectionSym = DwarfDebugLocSectionSym = 0;
   FunctionBeginSym = FunctionEndSym = 0;
 
-  // Turn on accelerator tables for Darwin.
-  if (Triple(M->getTargetTriple()).isOSDarwin())
-    DwarfAccelTables = true;
-  
+  // Turn on accelerator tables and older gdb compatibility
+  // for Darwin.
+  bool isDarwin = Triple(M->getTargetTriple()).isOSDarwin();
+  if (DarwinGDBCompat == Default) {
+    if (isDarwin)
+      isDarwinGDBCompat = true;
+    else
+      isDarwinGDBCompat = false;
+  } else
+    isDarwinGDBCompat = DarwinGDBCompat == Enable ? true : false;
+
+  if (DwarfAccelTables == Default) {
+    if (isDarwin)
+      hasDwarfAccelTables = true;
+    else
+      hasDwarfAccelTables = false;
+  } else
+    hasDwarfAccelTables = DwarfAccelTables == Enable ? true : false;
+
   {
     NamedRegionTimer T(DbgTimerName, DWARFGroupName, TimePassesIsEnabled);
     beginModule(M);
@@ -272,44 +307,51 @@ DIE *DwarfDebug::updateSubprogramScopeDIE(CompileUnit *SPCU,
   assert(SPDie && "Unable to find subprogram DIE!");
   DISubprogram SP(SPNode);
 
-  DISubprogram SPDecl = SP.getFunctionDeclaration();
-  if (!SPDecl.isSubprogram()) {
-    // There is not any need to generate specification DIE for a function
-    // defined at compile unit level. If a function is defined inside another
-    // function then gdb prefers the definition at top level and but does not
-    // expect specification DIE in parent function. So avoid creating
-    // specification DIE for a function defined inside a function.
-    if (SP.isDefinition() && !SP.getContext().isCompileUnit() &&
-        !SP.getContext().isFile() &&
-        !isSubprogramContext(SP.getContext())) {
-      SPCU->addUInt(SPDie, dwarf::DW_AT_declaration, dwarf::DW_FORM_flag, 1);
-      
-      // Add arguments.
-      DICompositeType SPTy = SP.getType();
-      DIArray Args = SPTy.getTypeArray();
-      unsigned SPTag = SPTy.getTag();
-      if (SPTag == dwarf::DW_TAG_subroutine_type)
-        for (unsigned i = 1, N = Args.getNumElements(); i < N; ++i) {
-          DIE *Arg = new DIE(dwarf::DW_TAG_formal_parameter);
-          DIType ATy = DIType(DIType(Args.getElement(i)));
-          SPCU->addType(Arg, ATy);
-          if (ATy.isArtificial())
-            SPCU->addUInt(Arg, dwarf::DW_AT_artificial, dwarf::DW_FORM_flag, 1);
-          SPDie->addChild(Arg);
-        }
-      DIE *SPDeclDie = SPDie;
-      SPDie = new DIE(dwarf::DW_TAG_subprogram);
-      SPCU->addDIEEntry(SPDie, dwarf::DW_AT_specification, dwarf::DW_FORM_ref4,
-                        SPDeclDie);
-      SPCU->addDie(SPDie);
-    }
-  }
-  // Pick up abstract subprogram DIE.
+  // If we're updating an abstract DIE, then we will be adding the children and
+  // object pointer later on. But what we don't want to do is process the
+  // concrete DIE twice.
   if (DIE *AbsSPDIE = AbstractSPDies.lookup(SPNode)) {
+    // Pick up abstract subprogram DIE.
     SPDie = new DIE(dwarf::DW_TAG_subprogram);
     SPCU->addDIEEntry(SPDie, dwarf::DW_AT_abstract_origin,
                       dwarf::DW_FORM_ref4, AbsSPDIE);
     SPCU->addDie(SPDie);
+  } else {
+    DISubprogram SPDecl = SP.getFunctionDeclaration();
+    if (!SPDecl.isSubprogram()) {
+      // There is not any need to generate specification DIE for a function
+      // defined at compile unit level. If a function is defined inside another
+      // function then gdb prefers the definition at top level and but does not
+      // expect specification DIE in parent function. So avoid creating
+      // specification DIE for a function defined inside a function.
+      if (SP.isDefinition() && !SP.getContext().isCompileUnit() &&
+          !SP.getContext().isFile() &&
+          !isSubprogramContext(SP.getContext())) {
+        SPCU->addFlag(SPDie, dwarf::DW_AT_declaration);
+
+        // Add arguments.
+        DICompositeType SPTy = SP.getType();
+        DIArray Args = SPTy.getTypeArray();
+        unsigned SPTag = SPTy.getTag();
+        if (SPTag == dwarf::DW_TAG_subroutine_type)
+          for (unsigned i = 1, N = Args.getNumElements(); i < N; ++i) {
+            DIE *Arg = new DIE(dwarf::DW_TAG_formal_parameter);
+            DIType ATy = DIType(Args.getElement(i));
+            SPCU->addType(Arg, ATy);
+            if (ATy.isArtificial())
+              SPCU->addFlag(Arg, dwarf::DW_AT_artificial);
+            if (ATy.isObjectPointer())
+              SPCU->addDIEEntry(SPDie, dwarf::DW_AT_object_pointer,
+                                dwarf::DW_FORM_ref4, Arg);
+            SPDie->addChild(Arg);
+          }
+        DIE *SPDeclDie = SPDie;
+        SPDie = new DIE(dwarf::DW_TAG_subprogram);
+        SPCU->addDIEEntry(SPDie, dwarf::DW_AT_specification, dwarf::DW_FORM_ref4,
+                          SPDeclDie);
+        SPCU->addDie(SPDie);
+      }
+    }
   }
 
   SPCU->addLabel(SPDie, dwarf::DW_AT_low_pc, dwarf::DW_FORM_addr,
@@ -346,7 +388,7 @@ DIE *DwarfDebug::constructLexicalScopeDIE(CompileUnit *TheCU,
     // DW_AT_ranges appropriately.
     TheCU->addUInt(ScopeDIE, dwarf::DW_AT_ranges, dwarf::DW_FORM_data4,
                    DebugRangeSymbols.size() 
-                   * Asm->getTargetData().getPointerSize());
+                   * Asm->getDataLayout().getPointerSize());
     for (SmallVector<InsnRange, 4>::const_iterator RI = Ranges.begin(),
          RE = Ranges.end(); RI != RE; ++RI) {
       DebugRangeSymbols.push_back(getLabelBeforeInsn(RI->first));
@@ -386,7 +428,7 @@ DIE *DwarfDebug::constructInlinedScopeDIE(CompileUnit *TheCU,
   DISubprogram InlinedSP = getDISubprogram(DS);
   DIE *OriginDIE = TheCU->getDIE(InlinedSP);
   if (!OriginDIE) {
-    DEBUG(dbgs() << "Unable to find original DIE for inlined subprogram.");
+    DEBUG(dbgs() << "Unable to find original DIE for an inlined subprogram.");
     return NULL;
   }
 
@@ -395,7 +437,7 @@ DIE *DwarfDebug::constructInlinedScopeDIE(CompileUnit *TheCU,
   const MCSymbol *EndLabel = getLabelAfterInsn(RI->second);
 
   if (StartLabel == 0 || EndLabel == 0) {
-    llvm_unreachable("Unexpected Start and End labels for a inlined scope!");
+    llvm_unreachable("Unexpected Start and End labels for an inlined scope!");
   }
   assert(StartLabel->isDefined() &&
          "Invalid starting label for an inlined scope!");
@@ -412,7 +454,7 @@ DIE *DwarfDebug::constructInlinedScopeDIE(CompileUnit *TheCU,
     // DW_AT_ranges appropriately.
     TheCU->addUInt(ScopeDIE, dwarf::DW_AT_ranges, dwarf::DW_FORM_data4,
                    DebugRangeSymbols.size() 
-                   * Asm->getTargetData().getPointerSize());
+                   * Asm->getDataLayout().getPointerSize());
     for (SmallVector<InsnRange, 4>::const_iterator RI = Ranges.begin(),
          RE = Ranges.end(); RI != RE; ++RI) {
       DebugRangeSymbols.push_back(getLabelBeforeInsn(RI->first));
@@ -461,21 +503,26 @@ DIE *DwarfDebug::constructScopeDIE(CompileUnit *TheCU, LexicalScope *Scope) {
     return NULL;
 
   SmallVector<DIE *, 8> Children;
+  DIE *ObjectPointer = NULL;
 
   // Collect arguments for current function.
   if (LScopes.isCurrentFunctionScope(Scope))
     for (unsigned i = 0, N = CurrentFnArguments.size(); i < N; ++i)
       if (DbgVariable *ArgDV = CurrentFnArguments[i])
         if (DIE *Arg = 
-            TheCU->constructVariableDIE(ArgDV, Scope->isAbstractScope()))
+            TheCU->constructVariableDIE(ArgDV, Scope->isAbstractScope())) {
           Children.push_back(Arg);
+          if (ArgDV->isObjectPointer()) ObjectPointer = Arg;
+        }
 
   // Collect lexical scope children first.
   const SmallVector<DbgVariable *, 8> &Variables = ScopeVariables.lookup(Scope);
   for (unsigned i = 0, N = Variables.size(); i < N; ++i)
     if (DIE *Variable = 
-        TheCU->constructVariableDIE(Variables[i], Scope->isAbstractScope()))
+        TheCU->constructVariableDIE(Variables[i], Scope->isAbstractScope())) {
       Children.push_back(Variable);
+      if (Variables[i]->isObjectPointer()) ObjectPointer = Variable;
+    }
   const SmallVector<LexicalScope *, 4> &Scopes = Scope->getChildren();
   for (unsigned j = 0, M = Scopes.size(); j < M; ++j)
     if (DIE *Nested = constructScopeDIE(TheCU, Scopes[j]))
@@ -509,6 +556,10 @@ DIE *DwarfDebug::constructScopeDIE(CompileUnit *TheCU, LexicalScope *Scope) {
          E = Children.end(); I != E; ++I)
     ScopeDIE->addChild(*I);
 
+  if (DS.isSubprogram() && ObjectPointer != NULL)
+    TheCU->addDIEEntry(ScopeDIE, dwarf::DW_AT_object_pointer,
+                       dwarf::DW_FORM_ref4, ObjectPointer);
+
   if (DS.isSubprogram())
     TheCU->addPubTypes(DISubprogram(DS));
 
@@ -556,7 +607,8 @@ CompileUnit *DwarfDebug::constructCompileUnit(const MDNode *N) {
   unsigned ID = GetOrCreateSourceID(FN, CompilationDir);
 
   DIE *Die = new DIE(dwarf::DW_TAG_compile_unit);
-  CompileUnit *NewCU = new CompileUnit(ID, DIUnit.getLanguage(), Die, Asm, this);
+  CompileUnit *NewCU = new CompileUnit(ID, DIUnit.getLanguage(), Die,
+                                       Asm, this);
   NewCU->addString(Die, dwarf::DW_AT_producer, DIUnit.getProducer());
   NewCU->addUInt(Die, dwarf::DW_AT_language, dwarf::DW_FORM_data2,
                  DIUnit.getLanguage());
@@ -575,7 +627,7 @@ CompileUnit *DwarfDebug::constructCompileUnit(const MDNode *N) {
   if (!CompilationDir.empty())
     NewCU->addString(Die, dwarf::DW_AT_comp_dir, CompilationDir);
   if (DIUnit.isOptimized())
-    NewCU->addUInt(Die, dwarf::DW_AT_APPLE_optimized, dwarf::DW_FORM_flag, 1);
+    NewCU->addFlag(Die, dwarf::DW_AT_APPLE_optimized);
 
   StringRef Flags = DIUnit.getFlags();
   if (!Flags.empty())
@@ -755,7 +807,7 @@ void DwarfDebug::endModule() {
         LexicalScope *Scope = 
           new LexicalScope(NULL, DIDescriptor(SP), NULL, false);
         DeadFnScopeMap[SP] = Scope;
-        
+
         // Construct subprogram DIE and add variables DIEs.
         CompileUnit *SPCU = CUMap.lookup(TheCU);
         assert(SPCU && "Unable to find Compile Unit!");
@@ -802,9 +854,9 @@ void DwarfDebug::endModule() {
   Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("data_end"));
 
   // End text sections.
-  for (unsigned i = 1, N = SectionMap.size(); i <= N; ++i) {
-    Asm->OutStreamer.SwitchSection(SectionMap[i]);
-    Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("section_end", i));
+  for (unsigned I = 0, E = SectionMap.size(); I != E; ++I) {
+    Asm->OutStreamer.SwitchSection(SectionMap[I]);
+    Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("section_end", I+1));
   }
 
   // Compute DIE offsets and sizes.
@@ -816,8 +868,8 @@ void DwarfDebug::endModule() {
   // Corresponding abbreviations into a abbrev section.
   emitAbbreviations();
 
-  // Emit info into a dwarf accelerator table sections.
-  if (DwarfAccelTables) {
+  // Emit info into the dwarf accelerator table sections.
+  if (useDwarfAccelTables()) {
     emitAccelNames();
     emitAccelObjC();
     emitAccelNamespaces();
@@ -825,7 +877,10 @@ void DwarfDebug::endModule() {
   }
   
   // Emit info into a debug pubtypes section.
-  emitDebugPubTypes();
+  // TODO: When we don't need the option anymore we can
+  // remove all of the code that adds to the table.
+  if (useDarwinGDBCompat())
+    emitDebugPubTypes();
 
   // Emit info into a debug loc section.
   emitDebugLoc();
@@ -840,7 +895,11 @@ void DwarfDebug::endModule() {
   emitDebugMacInfo();
 
   // Emit inline info.
-  emitDebugInlineInfo();
+  // TODO: When we don't need the option anymore we
+  // can remove all of the code that this section
+  // depends upon.
+  if (useDarwinGDBCompat())
+    emitDebugInlineInfo();
 
   // Emit info into a debug str section.
   emitDebugStr();
@@ -1014,7 +1073,7 @@ DwarfDebug::collectVariableInfo(const MachineFunction *MF,
     if (AbsVar)
       AbsVar->setMInsn(MInsn);
 
-    // Simple ranges that are fully coalesced.
+    // Simplify ranges that are fully coalesced.
     if (History.size() <= 1 || (History.size() == 2 &&
                                 MInsn->isIdenticalTo(History.back()))) {
       RegVar->setMInsn(MInsn);
@@ -1267,7 +1326,7 @@ void DwarfDebug::beginFunction(const MachineFunction *MF) {
             // Coalesce identical entries at the end of History.
             if (History.size() >= 2 &&
                 Prev->isIdenticalTo(History[History.size() - 2])) {
-              DEBUG(dbgs() << "Coalesce identical DBG_VALUE entries:\n"
+              DEBUG(dbgs() << "Coalescing identical DBG_VALUE entries:\n"
                     << "\t" << *Prev 
                     << "\t" << *History[History.size() - 2] << "\n");
               History.pop_back();
@@ -1283,7 +1342,7 @@ void DwarfDebug::beginFunction(const MachineFunction *MF) {
                 PrevMBB->getLastNonDebugInstr();
               if (LastMI == PrevMBB->end()) {
                 // Drop DBG_VALUE for empty range.
-                DEBUG(dbgs() << "Drop DBG_VALUE for empty range:\n"
+                DEBUG(dbgs() << "Dropping DBG_VALUE for empty range:\n"
                       << "\t" << *Prev << "\n");
                 History.pop_back();
               }
@@ -1300,9 +1359,10 @@ void DwarfDebug::beginFunction(const MachineFunction *MF) {
         if (!MI->isLabel())
           AtBlockEntry = false;
 
-        // First known non DBG_VALUE location marks beginning of function
-        // body.
-        if (PrologEndLoc.isUnknown() && !MI->getDebugLoc().isUnknown())
+        // First known non-DBG_VALUE and non-frame setup location marks
+        // the beginning of the function body.
+        if (!MI->getFlag(MachineInstr::FrameSetup) &&
+            (PrologEndLoc.isUnknown() && !MI->getDebugLoc().isUnknown()))
           PrologEndLoc = MI->getDebugLoc();
 
         // Check if the instruction clobbers any registers with debug vars.
@@ -1382,7 +1442,7 @@ void DwarfDebug::beginFunction(const MachineFunction *MF) {
                                        MF->getFunction()->getContext());
     recordSourceLine(FnStartDL.getLine(), FnStartDL.getCol(),
                      FnStartDL.getScope(MF->getFunction()->getContext()),
-                     DWARF2_LINE_DEFAULT_IS_STMT ? DWARF2_FLAG_IS_STMT : 0);
+                     0);
   }
 }
 
@@ -1439,8 +1499,7 @@ void DwarfDebug::endFunction(const MachineFunction *MF) {
   DIE *CurFnDIE = constructScopeDIE(TheCU, FnScope);
   
   if (!MF->getTarget().Options.DisableFramePointerElim(*MF))
-    TheCU->addUInt(CurFnDIE, dwarf::DW_AT_APPLE_omit_frame_ptr,
-                   dwarf::DW_FORM_flag, 1);
+    TheCU->addFlag(CurFnDIE, dwarf::DW_AT_APPLE_omit_frame_ptr);
 
   DebugFrames.push_back(FunctionDebugFrameInfo(Asm->getFunctionNumber(),
                                                MMI->getFrameMoves()));
@@ -1710,7 +1769,7 @@ void DwarfDebug::emitDebugInfo() {
     Asm->EmitSectionOffset(Asm->GetTempSymbol("abbrev_begin"),
                            DwarfAbbrevSectionSym);
     Asm->OutStreamer.AddComment("Address Size (in bytes)");
-    Asm->EmitInt8(Asm->getTargetData().getPointerSize());
+    Asm->EmitInt8(Asm->getDataLayout().getPointerSize());
 
     emitDIE(Die);
     Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("info_end", TheCU->getID()));
@@ -1756,14 +1815,14 @@ void DwarfDebug::emitEndOfLineMatrix(unsigned SectionEnd) {
   Asm->EmitInt8(0);
 
   Asm->OutStreamer.AddComment("Op size");
-  Asm->EmitInt8(Asm->getTargetData().getPointerSize() + 1);
+  Asm->EmitInt8(Asm->getDataLayout().getPointerSize() + 1);
   Asm->OutStreamer.AddComment("DW_LNE_set_address");
   Asm->EmitInt8(dwarf::DW_LNE_set_address);
 
   Asm->OutStreamer.AddComment("Section end label");
 
   Asm->OutStreamer.EmitSymbolValue(Asm->GetTempSymbol("section_end",SectionEnd),
-                                   Asm->getTargetData().getPointerSize(),
+                                   Asm->getDataLayout().getPointerSize(),
                                    0/*AddrSpace*/);
 
   // Mark end of matrix.
@@ -1992,7 +2051,7 @@ void DwarfDebug::emitDebugLoc() {
   // Start the dwarf loc section.
   Asm->OutStreamer.SwitchSection(
     Asm->getObjFileLowering().getDwarfLocSection());
-  unsigned char Size = Asm->getTargetData().getPointerSize();
+  unsigned char Size = Asm->getDataLayout().getPointerSize();
   Asm->OutStreamer.EmitLabel(Asm->GetTempSymbol("debug_loc", 0));
   unsigned index = 1;
   for (SmallVector<DotDebugLocEntry, 4>::iterator
@@ -2089,7 +2148,7 @@ void DwarfDebug::emitDebugRanges() {
   // Start the dwarf ranges section.
   Asm->OutStreamer.SwitchSection(
     Asm->getObjFileLowering().getDwarfRangesSection());
-  unsigned char Size = Asm->getTargetData().getPointerSize();
+  unsigned char Size = Asm->getDataLayout().getPointerSize();
   for (SmallVector<const MCSymbol *, 8>::iterator
          I = DebugRangeSymbols.begin(), E = DebugRangeSymbols.end();
        I != E; ++I) {
@@ -2147,7 +2206,7 @@ void DwarfDebug::emitDebugInlineInfo() {
   Asm->OutStreamer.AddComment("Dwarf Version");
   Asm->EmitInt16(dwarf::DWARF_VERSION);
   Asm->OutStreamer.AddComment("Address Size (in bytes)");
-  Asm->EmitInt8(Asm->getTargetData().getPointerSize());
+  Asm->EmitInt8(Asm->getDataLayout().getPointerSize());
 
   for (SmallVector<const MDNode *, 4>::iterator I = InlinedSPNodes.begin(),
          E = InlinedSPNodes.end(); I != E; ++I) {
@@ -2178,7 +2237,7 @@ void DwarfDebug::emitDebugInlineInfo() {
 
       if (Asm->isVerbose()) Asm->OutStreamer.AddComment("low_pc");
       Asm->OutStreamer.EmitSymbolValue(LI->first,
-                                       Asm->getTargetData().getPointerSize(),0);
+                                       Asm->getDataLayout().getPointerSize(),0);
     }
   }
 
diff --git a/lib/CodeGen/AsmPrinter/DwarfDebug.h b/lib/CodeGen/AsmPrinter/DwarfDebug.h
index d1d6512..61d9a51 100644
--- a/lib/CodeGen/AsmPrinter/DwarfDebug.h
+++ b/lib/CodeGen/AsmPrinter/DwarfDebug.h
@@ -21,9 +21,9 @@
 #include "llvm/MC/MachineLocation.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/FoldingSet.h"
+#include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/StringMap.h"
-#include "llvm/ADT/UniqueVector.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/DebugLoc.h"
 
@@ -96,7 +96,8 @@ typedef struct DotDebugLocEntry {
   DotDebugLocEntry(const MCSymbol *B, const MCSymbol *E, const ConstantFP *FPtr)
     : Begin(B), End(E), Variable(0), Merged(false), 
       Constant(true) { Constants.CFP = FPtr; EntryKind = E_ConstantFP; }
-  DotDebugLocEntry(const MCSymbol *B, const MCSymbol *E, const ConstantInt *IPtr)
+  DotDebugLocEntry(const MCSymbol *B, const MCSymbol *E,
+                   const ConstantInt *IPtr)
     : Begin(B), End(E), Variable(0), Merged(false), 
       Constant(true) { Constants.CIP = IPtr; EntryKind = E_ConstantInt; }
 
@@ -158,11 +159,19 @@ public:
   bool isArtificial()                const {
     if (Var.isArtificial())
       return true;
-    if (Var.getTag() == dwarf::DW_TAG_arg_variable
-        && getType().isArtificial())
+    if (getType().isArtificial())
       return true;
     return false;
   }
+
+  bool isObjectPointer()             const {
+    if (Var.isObjectPointer())
+      return true;
+    if (getType().isObjectPointer())
+      return true;
+    return false;
+  }
+  
   bool variableHasComplexAddress()   const {
     assert(Var.Verify() && "Invalid complex DbgVariable!");
     return Var.hasComplexAddress();
@@ -222,7 +231,7 @@ class DwarfDebug {
   
   /// SectionMap - Provides a unique id per text section.
   ///
-  UniqueVector<const MCSection*> SectionMap;
+  SetVector<const MCSection*> SectionMap;
 
   /// CurrentFnArguments - List of Arguments (DbgValues) for current function.
   SmallVector<DbgVariable *, 8> CurrentFnArguments;
@@ -307,6 +316,9 @@ class DwarfDebug {
   // table for the same directory as DW_at_comp_dir.
   StringRef CompilationDir;
 
+  // A holder for the DarwinGDBCompat flag so that the compile unit can use it.
+  bool isDarwinGDBCompat;
+  bool hasDwarfAccelTables;
 private:
 
   /// assignAbbrevNumber - Define a unique number for the abbreviation.
@@ -520,6 +532,11 @@ public:
   /// getStringPoolEntry - returns an entry into the string pool with the given
   /// string text.
   MCSymbol *getStringPoolEntry(StringRef Str);
+
+  /// useDarwinGDBCompat - returns whether or not to limit some of our debug
+  /// output to the limitations of darwin gdb.
+  bool useDarwinGDBCompat() { return isDarwinGDBCompat; }
+  bool useDwarfAccelTables() { return hasDwarfAccelTables; }
 };
 } // End of namespace llvm
 
diff --git a/lib/CodeGen/AsmPrinter/DwarfException.cpp b/lib/CodeGen/AsmPrinter/DwarfException.cpp
index 70cc2e5..08fb6b3 100644
--- a/lib/CodeGen/AsmPrinter/DwarfException.cpp
+++ b/lib/CodeGen/AsmPrinter/DwarfException.cpp
@@ -24,7 +24,7 @@
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/Target/Mangler.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetFrameLowering.h"
 #include "llvm/Target/TargetLoweringObjectFile.h"
 #include "llvm/Target/TargetMachine.h"
@@ -417,7 +417,7 @@ void DwarfException::EmitExceptionTable() {
     // that we're omitting that bit.
     TTypeEncoding = dwarf::DW_EH_PE_omit;
     // dwarf::DW_EH_PE_absptr
-    TypeFormatSize = Asm->getTargetData().getPointerSize();
+    TypeFormatSize = Asm->getDataLayout().getPointerSize();
   } else {
     // Okay, we have actual filters or typeinfos to emit.  As such, we need to
     // pick a type encoding for them.  We're about to emit a list of pointers to
diff --git a/lib/CodeGen/AsmPrinter/DwarfException.h b/lib/CodeGen/AsmPrinter/DwarfException.h
index 75f6056..fe9e493 100644
--- a/lib/CodeGen/AsmPrinter/DwarfException.h
+++ b/lib/CodeGen/AsmPrinter/DwarfException.h
@@ -43,26 +43,6 @@ protected:
   /// MMI - Collected machine module information.
   MachineModuleInfo *MMI;
 
-  /// EmitExceptionTable - Emit landing pads and actions.
-  ///
-  /// The general organization of the table is complex, but the basic concepts
-  /// are easy.  First there is a header which describes the location and
-  /// organization of the three components that follow.
-  ///  1. The landing pad site information describes the range of code covered
-  ///     by the try.  In our case it's an accumulation of the ranges covered
-  ///     by the invokes in the try.  There is also a reference to the landing
-  ///     pad that handles the exception once processed.  Finally an index into
-  ///     the actions table.
-  ///  2. The action table, in our case, is composed of pairs of type ids
-  ///     and next action offset.  Starting with the action index from the
-  ///     landing pad site, each type Id is checked for a match to the current
-  ///     exception.  If it matches then the exception and type id are passed
-  ///     on to the landing pad.  Otherwise the next action is looked up.  This
-  ///     chain is terminated with a next action of zero.  If no type id is
-  ///     found the frame is unwound and handling continues.
-  ///  3. Type id table contains references to all the C++ typeinfo for all
-  ///     catches in the function.  This tables is reversed indexed base 1.
-
   /// SharedTypeIds - How many leading type ids two landing pads have in common.
   static unsigned SharedTypeIds(const LandingPadInfo *L,
                                 const LandingPadInfo *R);
@@ -119,6 +99,26 @@ protected:
                             const RangeMapType &PadMap,
                             const SmallVectorImpl<const LandingPadInfo *> &LPs,
                             const SmallVectorImpl<unsigned> &FirstActions);
+
+  /// EmitExceptionTable - Emit landing pads and actions.
+  ///
+  /// The general organization of the table is complex, but the basic concepts
+  /// are easy.  First there is a header which describes the location and
+  /// organization of the three components that follow.
+  ///  1. The landing pad site information describes the range of code covered
+  ///     by the try.  In our case it's an accumulation of the ranges covered
+  ///     by the invokes in the try.  There is also a reference to the landing
+  ///     pad that handles the exception once processed.  Finally an index into
+  ///     the actions table.
+  ///  2. The action table, in our case, is composed of pairs of type ids
+  ///     and next action offset.  Starting with the action index from the
+  ///     landing pad site, each type Id is checked for a match to the current
+  ///     exception.  If it matches then the exception and type id are passed
+  ///     on to the landing pad.  Otherwise the next action is looked up.  This
+  ///     chain is terminated with a next action of zero.  If no type id is
+  ///     found the frame is unwound and handling continues.
+  ///  3. Type id table contains references to all the C++ typeinfo for all
+  ///     catches in the function.  This tables is reversed indexed base 1.
   void EmitExceptionTable();
 
 public:
diff --git a/lib/CodeGen/AsmPrinter/OcamlGCPrinter.cpp b/lib/CodeGen/AsmPrinter/OcamlGCPrinter.cpp
index 1153817..f7c0119 100644
--- a/lib/CodeGen/AsmPrinter/OcamlGCPrinter.cpp
+++ b/lib/CodeGen/AsmPrinter/OcamlGCPrinter.cpp
@@ -20,7 +20,7 @@
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/Target/Mangler.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetLoweringObjectFile.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/ADT/SmallString.h"
@@ -91,7 +91,7 @@ void OcamlGCMetadataPrinter::beginAssembly(AsmPrinter &AP) {
 /// either condition is detected in a function which uses the GC.
 ///
 void OcamlGCMetadataPrinter::finishAssembly(AsmPrinter &AP) {
-  unsigned IntPtrSize = AP.TM.getTargetData()->getPointerSize();
+  unsigned IntPtrSize = AP.TM.getDataLayout()->getPointerSize();
 
   AP.OutStreamer.SwitchSection(AP.getObjFileLowering().getTextSection());
   EmitCamlGlobal(getModule(), AP, "code_end");
diff --git a/lib/CodeGen/AsmPrinter/Win64Exception.cpp b/lib/CodeGen/AsmPrinter/Win64Exception.cpp
index b83aa5a..70742a8 100644
--- a/lib/CodeGen/AsmPrinter/Win64Exception.cpp
+++ b/lib/CodeGen/AsmPrinter/Win64Exception.cpp
@@ -24,7 +24,7 @@
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/Target/Mangler.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetFrameLowering.h"
 #include "llvm/Target/TargetLoweringObjectFile.h"
 #include "llvm/Target/TargetMachine.h"
diff --git a/lib/CodeGen/BranchFolding.cpp b/lib/CodeGen/BranchFolding.cpp
index fb65bb7..6f4c5a2 100644
--- a/lib/CodeGen/BranchFolding.cpp
+++ b/lib/CodeGen/BranchFolding.cpp
@@ -357,9 +357,8 @@ static unsigned ComputeCommonTailLength(MachineBasicBlock *MBB1,
   if (I1 == MBB1->begin() && I2 != MBB2->begin()) {
     --I2;
     while (I2->isDebugValue()) {
-      if (I2 == MBB2->begin()) {
+      if (I2 == MBB2->begin())
         return TailLen;
-        }
       --I2;
     }
     ++I2;
@@ -482,21 +481,19 @@ bool
 BranchFolder::MergePotentialsElt::operator<(const MergePotentialsElt &o) const {
   if (getHash() < o.getHash())
     return true;
-   else if (getHash() > o.getHash())
+  if (getHash() > o.getHash())
     return false;
-  else if (getBlock()->getNumber() < o.getBlock()->getNumber())
+  if (getBlock()->getNumber() < o.getBlock()->getNumber())
     return true;
-  else if (getBlock()->getNumber() > o.getBlock()->getNumber())
+  if (getBlock()->getNumber() > o.getBlock()->getNumber())
     return false;
-  else {
-    // _GLIBCXX_DEBUG checks strict weak ordering, which involves comparing
-    // an object with itself.
+  // _GLIBCXX_DEBUG checks strict weak ordering, which involves comparing
+  // an object with itself.
 #ifndef _GLIBCXX_DEBUG
-    llvm_unreachable("Predecessor appears twice");
+  llvm_unreachable("Predecessor appears twice");
 #else
-    return false;
+  return false;
 #endif
-  }
 }
 
 /// CountTerminators - Count the number of terminators in the given
@@ -574,7 +571,8 @@ static bool ProfitableToMerge(MachineBasicBlock *MBB1,
   // instructions that would be deleted in the merge.
   MachineFunction *MF = MBB1->getParent();
   if (EffectiveTailLen >= 2 &&
-      MF->getFunction()->hasFnAttr(Attribute::OptimizeForSize) &&
+      MF->getFunction()->getFnAttributes().
+        hasAttribute(Attributes::OptimizeForSize) &&
       (I1 == MBB1->begin() || I2 == MBB2->begin()))
     return true;
 
@@ -1554,8 +1552,7 @@ MachineBasicBlock::iterator findHoistingInsertPosAndDeps(MachineBasicBlock *MBB,
       for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI)
         Uses.insert(*AI);
     } else {
-      if (Uses.count(Reg)) {
-        Uses.erase(Reg);
+      if (Uses.erase(Reg)) {
         for (MCSubRegIterator SubRegs(Reg, TRI); SubRegs.isValid(); ++SubRegs)
           Uses.erase(*SubRegs); // Use sub-registers to be conservative
       }
diff --git a/lib/CodeGen/CMakeLists.txt b/lib/CodeGen/CMakeLists.txt
index 2e189ad..fa6d4e1 100644
--- a/lib/CodeGen/CMakeLists.txt
+++ b/lib/CodeGen/CMakeLists.txt
@@ -45,6 +45,7 @@ add_llvm_library(LLVMCodeGen
   MachineCopyPropagation.cpp
   MachineCSE.cpp
   MachineDominators.cpp
+  MachinePostDominators.cpp
   MachineFunction.cpp
   MachineFunctionAnalysis.cpp
   MachineFunctionPass.cpp
@@ -95,12 +96,14 @@ add_llvm_library(LLVMCodeGen
   SplitKit.cpp
   StackProtector.cpp
   StackSlotColoring.cpp
+  StackColoring.cpp
   StrongPHIElimination.cpp
   TailDuplication.cpp
   TargetFrameLoweringImpl.cpp
   TargetInstrInfoImpl.cpp
   TargetLoweringObjectFileImpl.cpp
   TargetOptionsImpl.cpp
+  TargetSchedule.cpp
   TwoAddressInstructionPass.cpp
   UnreachableBlockElim.cpp
   VirtRegMap.cpp
diff --git a/lib/CodeGen/CalcSpillWeights.cpp b/lib/CodeGen/CalcSpillWeights.cpp
index 939af3f..dee339a 100644
--- a/lib/CodeGen/CalcSpillWeights.cpp
+++ b/lib/CodeGen/CalcSpillWeights.cpp
@@ -9,7 +9,6 @@
 
 #define DEBUG_TYPE "calcspillweights"
 
-#include "llvm/Function.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/CodeGen/CalcSpillWeights.h"
 #include "llvm/CodeGen/LiveIntervalAnalysis.h"
@@ -42,8 +41,7 @@ void CalculateSpillWeights::getAnalysisUsage(AnalysisUsage &au) const {
 bool CalculateSpillWeights::runOnMachineFunction(MachineFunction &MF) {
 
   DEBUG(dbgs() << "********** Compute Spill Weights **********\n"
-               << "********** Function: "
-               << MF.getFunction()->getName() << '\n');
+               << "********** Function: " << MF.getName() << '\n');
 
   LiveIntervals &LIS = getAnalysis<LiveIntervals>();
   MachineRegisterInfo &MRI = MF.getRegInfo();
@@ -166,7 +164,7 @@ void VirtRegAuxInfo::CalculateWeightAndHint(LiveInterval &li) {
       continue;
     float hweight = Hint[hint] += weight;
     if (TargetRegisterInfo::isPhysicalRegister(hint)) {
-      if (hweight > bestPhys && LIS.isAllocatable(hint))
+      if (hweight > bestPhys && mri.isAllocatable(hint))
         bestPhys = hweight, hintPhys = hint;
     } else {
       if (hweight > bestVirt)
diff --git a/lib/CodeGen/CallingConvLower.cpp b/lib/CodeGen/CallingConvLower.cpp
index 0b747fd..22b9140 100644
--- a/lib/CodeGen/CallingConvLower.cpp
+++ b/lib/CodeGen/CallingConvLower.cpp
@@ -18,7 +18,7 @@
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetRegisterInfo.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetLowering.h"
 using namespace llvm;
@@ -50,7 +50,7 @@ void CCState::HandleByVal(unsigned ValNo, MVT ValVT,
   if (MinAlign > (int)Align)
     Align = MinAlign;
   MF.getFrameInfo()->ensureMaxAlignment(Align);
-  TM.getTargetLowering()->HandleByVal(this, Size);
+  TM.getTargetLowering()->HandleByVal(this, Size, Align);
   unsigned Offset = AllocateStack(Size, Align);
   addLoc(CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, LocInfo));
 }
diff --git a/lib/CodeGen/CodeGen.cpp b/lib/CodeGen/CodeGen.cpp
index fb2c2e8..a53f6f8 100644
--- a/lib/CodeGen/CodeGen.cpp
+++ b/lib/CodeGen/CodeGen.cpp
@@ -41,6 +41,7 @@ void llvm::initializeCodeGen(PassRegistry &Registry) {
   initializeMachineCopyPropagationPass(Registry);
   initializeMachineCSEPass(Registry);
   initializeMachineDominatorTreePass(Registry);
+  initializeMachinePostDominatorTreePass(Registry);
   initializeMachineLICMPass(Registry);
   initializeMachineLoopInfoPass(Registry);
   initializeMachineModuleInfoPass(Registry);
@@ -56,6 +57,7 @@ void llvm::initializeCodeGen(PassRegistry &Registry) {
   initializeRegisterCoalescerPass(Registry);
   initializeSlotIndexesPass(Registry);
   initializeStackProtectorPass(Registry);
+  initializeStackColoringPass(Registry);
   initializeStackSlotColoringPass(Registry);
   initializeStrongPHIEliminationPass(Registry);
   initializeTailDuplicatePassPass(Registry);
diff --git a/lib/CodeGen/CodePlacementOpt.cpp b/lib/CodeGen/CodePlacementOpt.cpp
index 99233df..d8e06c3 100644
--- a/lib/CodeGen/CodePlacementOpt.cpp
+++ b/lib/CodeGen/CodePlacementOpt.cpp
@@ -373,7 +373,7 @@ bool CodePlacementOpt::OptimizeIntraLoopEdges(MachineFunction &MF) {
 ///
 bool CodePlacementOpt::AlignLoops(MachineFunction &MF) {
   const Function *F = MF.getFunction();
-  if (F->hasFnAttr(Attribute::OptimizeForSize))
+  if (F->getFnAttributes().hasAttribute(Attributes::OptimizeForSize))
     return false;
 
   unsigned Align = TLI->getPrefLoopAlignment();
diff --git a/lib/CodeGen/CriticalAntiDepBreaker.cpp b/lib/CodeGen/CriticalAntiDepBreaker.cpp
index a9de1c749..377b471 100644
--- a/lib/CodeGen/CriticalAntiDepBreaker.cpp
+++ b/lib/CodeGen/CriticalAntiDepBreaker.cpp
@@ -527,7 +527,7 @@ BreakAntiDependencies(const std::vector<SUnit>& SUnits,
         if (Edge->getKind() == SDep::Anti) {
           AntiDepReg = Edge->getReg();
           assert(AntiDepReg != 0 && "Anti-dependence on reg0?");
-          if (!RegClassInfo.isAllocatable(AntiDepReg))
+          if (!MRI.isAllocatable(AntiDepReg))
             // Don't break anti-dependencies on non-allocatable registers.
             AntiDepReg = 0;
           else if (KeepRegs.test(AntiDepReg))
diff --git a/lib/CodeGen/DeadMachineInstructionElim.cpp b/lib/CodeGen/DeadMachineInstructionElim.cpp
index b4394e8..8964269 100644
--- a/lib/CodeGen/DeadMachineInstructionElim.cpp
+++ b/lib/CodeGen/DeadMachineInstructionElim.cpp
@@ -33,7 +33,6 @@ namespace {
     const MachineRegisterInfo *MRI;
     const TargetInstrInfo *TII;
     BitVector LivePhysRegs;
-    BitVector ReservedRegs;
 
   public:
     static char ID; // Pass identification, replacement for typeid
@@ -70,7 +69,7 @@ bool DeadMachineInstructionElim::isDead(const MachineInstr *MI) const {
       unsigned Reg = MO.getReg();
       if (TargetRegisterInfo::isPhysicalRegister(Reg)) {
         // Don't delete live physreg defs, or any reserved register defs.
-        if (LivePhysRegs.test(Reg) || ReservedRegs.test(Reg))
+        if (LivePhysRegs.test(Reg) || MRI->isReserved(Reg))
           return false;
       } else {
         if (!MRI->use_nodbg_empty(Reg))
@@ -90,9 +89,6 @@ bool DeadMachineInstructionElim::runOnMachineFunction(MachineFunction &MF) {
   TRI = MF.getTarget().getRegisterInfo();
   TII = MF.getTarget().getInstrInfo();
 
-  // Treat reserved registers as always live.
-  ReservedRegs = TRI->getReservedRegs(MF);
-
   // Loop over all instructions in all blocks, from bottom to top, so that it's
   // more likely that chains of dependent but ultimately dead instructions will
   // be cleaned up.
@@ -101,7 +97,7 @@ bool DeadMachineInstructionElim::runOnMachineFunction(MachineFunction &MF) {
     MachineBasicBlock *MBB = &*I;
 
     // Start out assuming that reserved registers are live out of this block.
-    LivePhysRegs = ReservedRegs;
+    LivePhysRegs = MRI->getReservedRegs();
 
     // Also add any explicit live-out physregs for this block.
     if (!MBB->empty() && MBB->back().isReturn())
diff --git a/lib/CodeGen/EarlyIfConversion.cpp b/lib/CodeGen/EarlyIfConversion.cpp
index f9347ef..d5d8404 100644
--- a/lib/CodeGen/EarlyIfConversion.cpp
+++ b/lib/CodeGen/EarlyIfConversion.cpp
@@ -18,7 +18,6 @@
 
 #define DEBUG_TYPE "early-ifcvt"
 #include "MachineTraceMetrics.h"
-#include "llvm/Function.h"
 #include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/PostOrderIterator.h"
 #include "llvm/ADT/SetVector.h"
@@ -32,9 +31,9 @@
 #include "llvm/CodeGen/MachineLoopInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/Passes.h"
-#include "llvm/MC/MCInstrItineraries.h"
 #include "llvm/Target/TargetInstrInfo.h"
 #include "llvm/Target/TargetRegisterInfo.h"
+#include "llvm/Target/TargetSubtargetInfo.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
@@ -775,11 +774,11 @@ bool EarlyIfConverter::tryConvertIf(MachineBasicBlock *MBB) {
 
 bool EarlyIfConverter::runOnMachineFunction(MachineFunction &MF) {
   DEBUG(dbgs() << "********** EARLY IF-CONVERSION **********\n"
-               << "********** Function: "
-               << ((Value*)MF.getFunction())->getName() << '\n');
+               << "********** Function: " << MF.getName() << '\n');
   TII = MF.getTarget().getInstrInfo();
   TRI = MF.getTarget().getRegisterInfo();
-  SchedModel = MF.getTarget().getInstrItineraryData()->SchedModel;
+  SchedModel =
+    MF.getTarget().getSubtarget<TargetSubtargetInfo>().getSchedModel();
   MRI = &MF.getRegInfo();
   DomTree = &getAnalysis<MachineDominatorTree>();
   Loops = getAnalysisIfAvailable<MachineLoopInfo>();
@@ -798,6 +797,5 @@ bool EarlyIfConverter::runOnMachineFunction(MachineFunction &MF) {
     if (tryConvertIf(I->getBlock()))
       Changed = true;
 
-  MF.verify(this, "After early if-conversion");
   return Changed;
 }
diff --git a/lib/CodeGen/ExecutionDepsFix.cpp b/lib/CodeGen/ExecutionDepsFix.cpp
index fee8e47..ed78f19 100644
--- a/lib/CodeGen/ExecutionDepsFix.cpp
+++ b/lib/CodeGen/ExecutionDepsFix.cpp
@@ -626,9 +626,12 @@ void ExeDepsFix::visitSoftInstr(MachineInstr *mi, unsigned mask) {
   }
   dv->Instrs.push_back(mi);
 
-  // Finally set all defs and non-collapsed uses to dv.
-  for (unsigned i = 0, e = mi->getDesc().getNumOperands(); i != e; ++i) {
-    MachineOperand &mo = mi->getOperand(i);
+  // Finally set all defs and non-collapsed uses to dv. We must iterate through
+  // all the operators, including imp-def ones.
+  for (MachineInstr::mop_iterator ii = mi->operands_begin(),
+                                  ee = mi->operands_end();
+                                  ii != ee; ++ii) {
+    MachineOperand &mo = *ii;
     if (!mo.isReg()) continue;
     int rx = regIndex(mo.getReg());
     if (rx < 0) continue;
@@ -654,7 +657,7 @@ bool ExeDepsFix::runOnMachineFunction(MachineFunction &mf) {
   bool anyregs = false;
   for (TargetRegisterClass::const_iterator I = RC->begin(), E = RC->end();
        I != E; ++I)
-    if (MF->getRegInfo().isPhysRegOrOverlapUsed(*I)) {
+    if (MF->getRegInfo().isPhysRegUsed(*I)) {
       anyregs = true;
       break;
     }
diff --git a/lib/CodeGen/ExpandPostRAPseudos.cpp b/lib/CodeGen/ExpandPostRAPseudos.cpp
index 7a17331..ffe4b63 100644
--- a/lib/CodeGen/ExpandPostRAPseudos.cpp
+++ b/lib/CodeGen/ExpandPostRAPseudos.cpp
@@ -14,7 +14,6 @@
 
 #define DEBUG_TYPE "postrapseudos"
 #include "llvm/CodeGen/Passes.h"
-#include "llvm/Function.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineInstr.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
@@ -190,8 +189,7 @@ bool ExpandPostRA::LowerCopy(MachineInstr *MI) {
 bool ExpandPostRA::runOnMachineFunction(MachineFunction &MF) {
   DEBUG(dbgs() << "Machine Function\n"
                << "********** EXPANDING POST-RA PSEUDO INSTRS **********\n"
-               << "********** Function: "
-               << MF.getFunction()->getName() << '\n');
+               << "********** Function: " << MF.getName() << '\n');
   TRI = MF.getTarget().getRegisterInfo();
   TII = MF.getTarget().getInstrInfo();
 
diff --git a/lib/CodeGen/GCStrategy.cpp b/lib/CodeGen/GCStrategy.cpp
index 506b5cf..f4755bb 100644
--- a/lib/CodeGen/GCStrategy.cpp
+++ b/lib/CodeGen/GCStrategy.cpp
@@ -20,6 +20,7 @@
 #include "llvm/IntrinsicInst.h"
 #include "llvm/Module.h"
 #include "llvm/Analysis/Dominators.h"
+#include "llvm/Analysis/DominatorInternals.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
@@ -387,9 +388,16 @@ void GCMachineCodeAnalysis::FindStackOffsets(MachineFunction &MF) {
   const TargetFrameLowering *TFI = TM->getFrameLowering();
   assert(TFI && "TargetRegisterInfo not available!");
 
-  for (GCFunctionInfo::roots_iterator RI = FI->roots_begin(),
-                                      RE = FI->roots_end(); RI != RE; ++RI)
-    RI->StackOffset = TFI->getFrameIndexOffset(MF, RI->Num);
+  for (GCFunctionInfo::roots_iterator RI = FI->roots_begin();
+       RI != FI->roots_end();) {
+    // If the root references a dead object, no need to keep it.
+    if (MF.getFrameInfo()->isDeadObjectIndex(RI->Num)) {
+      RI = FI->removeStackRoot(RI);
+    } else {
+      RI->StackOffset = TFI->getFrameIndexOffset(MF, RI->Num);
+      ++RI;
+    }
+  }
 }
 
 bool GCMachineCodeAnalysis::runOnMachineFunction(MachineFunction &MF) {
diff --git a/lib/CodeGen/IfConversion.cpp b/lib/CodeGen/IfConversion.cpp
index 4214ba1..31e36f0 100644
--- a/lib/CodeGen/IfConversion.cpp
+++ b/lib/CodeGen/IfConversion.cpp
@@ -13,7 +13,6 @@
 
 #define DEBUG_TYPE "ifcvt"
 #include "BranchFolding.h"
-#include "llvm/Function.h"
 #include "llvm/CodeGen/Passes.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineBranchProbabilityInfo.h"
@@ -282,7 +281,7 @@ bool IfConverter::runOnMachineFunction(MachineFunction &MF) {
   }
 
   DEBUG(dbgs() << "\nIfcvt: function (" << ++FnNum <<  ") \'"
-               << MF.getFunction()->getName() << "\'");
+               << MF.getName() << "\'");
 
   if (FnNum < IfCvtFnStart || (IfCvtFnStop != -1 && FnNum > IfCvtFnStop)) {
     DEBUG(dbgs() << " skipped\n");
@@ -997,14 +996,13 @@ static void UpdatePredRedefs(MachineInstr *MI, SmallSet<unsigned,4> &Redefs,
   }
   for (unsigned i = 0, e = Defs.size(); i != e; ++i) {
     unsigned Reg = Defs[i];
-    if (Redefs.count(Reg)) {
+    if (!Redefs.insert(Reg)) {
       if (AddImpUse)
         // Treat predicated update as read + write.
         MI->addOperand(MachineOperand::CreateReg(Reg, false/*IsDef*/,
                                               true/*IsImp*/,false/*IsKill*/,
                                               false/*IsDead*/,true/*IsUndef*/));
     } else {
-      Redefs.insert(Reg);
       for (MCSubRegIterator SubRegs(Reg, TRI); SubRegs.isValid(); ++SubRegs)
         Redefs.insert(*SubRegs);
     }
diff --git a/lib/CodeGen/InlineSpiller.cpp b/lib/CodeGen/InlineSpiller.cpp
index 07e37af..37828a7 100644
--- a/lib/CodeGen/InlineSpiller.cpp
+++ b/lib/CodeGen/InlineSpiller.cpp
@@ -613,7 +613,7 @@ MachineInstr *InlineSpiller::traceSiblingValue(unsigned UseReg, VNInfo *UseVNI,
     propagateSiblingValue(SVI);
   } while (!WorkList.empty());
 
-  // Look up the value we were looking for.  We already did this lokup at the
+  // Look up the value we were looking for.  We already did this lookup at the
   // top of the function, but SibValues may have been invalidated.
   SVI = SibValues.find(UseVNI);
   assert(SVI != SibValues.end() && "Didn't compute requested info");
@@ -863,7 +863,7 @@ bool InlineSpiller::reMaterializeFor(LiveInterval &VirtReg,
   // If the instruction also writes VirtReg.reg, it had better not require the
   // same register for uses and defs.
   SmallVector<std::pair<MachineInstr*, unsigned>, 8> Ops;
-  MIBundleOperands::RegInfo RI =
+  MIBundleOperands::VirtRegInfo RI =
     MIBundleOperands(MI).analyzeVirtReg(VirtReg.reg, &Ops);
   if (RI.Tied) {
     markValueUsed(&VirtReg, ParentVNI);
@@ -1142,7 +1142,7 @@ void InlineSpiller::spillAroundUses(unsigned Reg) {
 
     // Analyze instruction.
     SmallVector<std::pair<MachineInstr*, unsigned>, 8> Ops;
-    MIBundleOperands::RegInfo RI =
+    MIBundleOperands::VirtRegInfo RI =
       MIBundleOperands(MI).analyzeVirtReg(Reg, &Ops);
 
     // Find the slot index where this instruction reads and writes OldLI.
diff --git a/lib/CodeGen/IntrinsicLowering.cpp b/lib/CodeGen/IntrinsicLowering.cpp
index 8d2282a..6120ae56 100644
--- a/lib/CodeGen/IntrinsicLowering.cpp
+++ b/lib/CodeGen/IntrinsicLowering.cpp
@@ -21,7 +21,7 @@
 #include "llvm/Support/CallSite.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 using namespace llvm;
 
 template <class ArgIt>
@@ -457,7 +457,7 @@ void IntrinsicLowering::LowerIntrinsicCall(CallInst *CI) {
     break;   // Strip out annotate intrinsic
     
   case Intrinsic::memcpy: {
-    IntegerType *IntPtr = TD.getIntPtrType(Context);
+    Type *IntPtr = TD.getIntPtrType(Context);
     Value *Size = Builder.CreateIntCast(CI->getArgOperand(2), IntPtr,
                                         /* isSigned */ false);
     Value *Ops[3];
@@ -468,7 +468,7 @@ void IntrinsicLowering::LowerIntrinsicCall(CallInst *CI) {
     break;
   }
   case Intrinsic::memmove: {
-    IntegerType *IntPtr = TD.getIntPtrType(Context);
+    Type *IntPtr = TD.getIntPtrType(Context);
     Value *Size = Builder.CreateIntCast(CI->getArgOperand(2), IntPtr,
                                         /* isSigned */ false);
     Value *Ops[3];
@@ -479,7 +479,7 @@ void IntrinsicLowering::LowerIntrinsicCall(CallInst *CI) {
     break;
   }
   case Intrinsic::memset: {
-    IntegerType *IntPtr = TD.getIntPtrType(Context);
+    Type *IntPtr = TD.getIntPtrType(Context);
     Value *Size = Builder.CreateIntCast(CI->getArgOperand(2), IntPtr,
                                         /* isSigned */ false);
     Value *Ops[3];
diff --git a/lib/CodeGen/LLVMTargetMachine.cpp b/lib/CodeGen/LLVMTargetMachine.cpp
index cac0c83..24daafa 100644
--- a/lib/CodeGen/LLVMTargetMachine.cpp
+++ b/lib/CodeGen/LLVMTargetMachine.cpp
@@ -172,7 +172,7 @@ bool LLVMTargetMachine::addPassesToEmitFile(PassManagerBase &PM,
       const MCSubtargetInfo &STI = getSubtarget<MCSubtargetInfo>();
       MCE = getTarget().createMCCodeEmitter(*getInstrInfo(), MRI, STI,
                                             *Context);
-      MAB = getTarget().createMCAsmBackend(getTargetTriple());
+      MAB = getTarget().createMCAsmBackend(getTargetTriple(), TargetCPU);
     }
 
     MCStreamer *S = getTarget().createAsmStreamer(*Context, Out,
@@ -191,7 +191,7 @@ bool LLVMTargetMachine::addPassesToEmitFile(PassManagerBase &PM,
     // emission fails.
     MCCodeEmitter *MCE = getTarget().createMCCodeEmitter(*getInstrInfo(), MRI,
                                                          STI, *Context);
-    MCAsmBackend *MAB = getTarget().createMCAsmBackend(getTargetTriple());
+    MCAsmBackend *MAB = getTarget().createMCAsmBackend(getTargetTriple(), TargetCPU);
     if (MCE == 0 || MAB == 0)
       return true;
 
@@ -266,7 +266,7 @@ bool LLVMTargetMachine::addPassesToEmitMC(PassManagerBase &PM,
   const MCSubtargetInfo &STI = getSubtarget<MCSubtargetInfo>();
   MCCodeEmitter *MCE = getTarget().createMCCodeEmitter(*getInstrInfo(), MRI,
                                                        STI, *Ctx);
-  MCAsmBackend *MAB = getTarget().createMCAsmBackend(getTargetTriple());
+  MCAsmBackend *MAB = getTarget().createMCAsmBackend(getTargetTriple(), TargetCPU);
   if (MCE == 0 || MAB == 0)
     return true;
 
diff --git a/lib/CodeGen/LiveDebugVariables.cpp b/lib/CodeGen/LiveDebugVariables.cpp
index d631726..defc127 100644
--- a/lib/CodeGen/LiveDebugVariables.cpp
+++ b/lib/CodeGen/LiveDebugVariables.cpp
@@ -687,8 +687,7 @@ bool LDVImpl::runOnMachineFunction(MachineFunction &mf) {
   clear();
   LS.initialize(mf);
   DEBUG(dbgs() << "********** COMPUTING LIVE DEBUG VARIABLES: "
-               << ((Value*)mf.getFunction())->getName()
-               << " **********\n");
+               << mf.getName() << " **********\n");
 
   bool Changed = collectDebugValues(mf);
   computeIntervals();
diff --git a/lib/CodeGen/LiveInterval.cpp b/lib/CodeGen/LiveInterval.cpp
index 0a795e6..8585cbb 100644
--- a/lib/CodeGen/LiveInterval.cpp
+++ b/lib/CodeGen/LiveInterval.cpp
@@ -27,6 +27,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Target/TargetRegisterInfo.h"
+#include "RegisterCoalescer.h"
 #include <algorithm>
 using namespace llvm;
 
@@ -58,8 +59,16 @@ VNInfo *LiveInterval::createDeadDef(SlotIndex Def,
     return VNI;
   }
   if (SlotIndex::isSameInstr(Def, I->start)) {
-    assert(I->start == Def && "Cannot insert def, already live");
-    assert(I->valno->def == Def && "Inconsistent existing value def");
+    assert(I->valno->def == I->start && "Inconsistent existing value def");
+
+    // It is possible to have both normal and early-clobber defs of the same
+    // register on an instruction. It doesn't make a lot of sense, but it is
+    // possible to specify in inline assembly.
+    //
+    // Just convert everything to early-clobber.
+    Def = std::min(Def, I->start);
+    if (Def != I->start)
+      I->start = I->valno->def = Def;
     return I->valno;
   }
   assert(SlotIndex::isEarlierInstr(Def, I->start) && "Already live at def");
@@ -68,21 +77,6 @@ VNInfo *LiveInterval::createDeadDef(SlotIndex Def,
   return VNI;
 }
 
-/// killedInRange - Return true if the interval has kills in [Start,End).
-bool LiveInterval::killedInRange(SlotIndex Start, SlotIndex End) const {
-  Ranges::const_iterator r =
-    std::lower_bound(ranges.begin(), ranges.end(), End);
-
-  // Now r points to the first interval with start >= End, or ranges.end().
-  if (r == ranges.begin())
-    return false;
-
-  --r;
-  // Now r points to the last interval with end <= End.
-  // r->end is the kill point.
-  return r->end >= Start && r->end < End;
-}
-
 // overlaps - Return true if the intersection of the two live intervals is
 // not empty.
 //
@@ -142,6 +136,48 @@ bool LiveInterval::overlapsFrom(const LiveInterval& other,
   return false;
 }
 
+bool LiveInterval::overlaps(const LiveInterval &Other,
+                            const CoalescerPair &CP,
+                            const SlotIndexes &Indexes) const {
+  assert(!empty() && "empty interval");
+  if (Other.empty())
+    return false;
+
+  // Use binary searches to find initial positions.
+  const_iterator I = find(Other.beginIndex());
+  const_iterator IE = end();
+  if (I == IE)
+    return false;
+  const_iterator J = Other.find(I->start);
+  const_iterator JE = Other.end();
+  if (J == JE)
+    return false;
+
+  for (;;) {
+    // J has just been advanced to satisfy:
+    assert(J->end >= I->start);
+    // Check for an overlap.
+    if (J->start < I->end) {
+      // I and J are overlapping. Find the later start.
+      SlotIndex Def = std::max(I->start, J->start);
+      // Allow the overlap if Def is a coalescable copy.
+      if (Def.isBlock() ||
+          !CP.isCoalescable(Indexes.getInstructionFromIndex(Def)))
+        return true;
+    }
+    // Advance the iterator that ends first to check for more overlaps.
+    if (J->end > I->end) {
+      std::swap(I, J);
+      std::swap(IE, JE);
+    }
+    // Advance J until J->end >= I->start.
+    do
+      if (++J == JE)
+        return false;
+    while (J->end < I->start);
+  }
+}
+
 /// overlaps - Return true if the live interval overlaps a range specified
 /// by [Start, End).
 bool LiveInterval::overlaps(SlotIndex Start, SlotIndex End) const {
@@ -399,7 +435,7 @@ void LiveInterval::join(LiveInterval &Other,
 
   // If we have to apply a mapping to our base interval assignment, rewrite it
   // now.
-  if (MustMapCurValNos) {
+  if (MustMapCurValNos && !empty()) {
     // Map the first live range.
 
     iterator OutIt = begin();
@@ -673,27 +709,6 @@ VNInfo* LiveInterval::MergeValueNumberInto(VNInfo *V1, VNInfo *V2) {
   return V2;
 }
 
-void LiveInterval::Copy(const LiveInterval &RHS,
-                        MachineRegisterInfo *MRI,
-                        VNInfo::Allocator &VNInfoAllocator) {
-  ranges.clear();
-  valnos.clear();
-  std::pair<unsigned, unsigned> Hint = MRI->getRegAllocationHint(RHS.reg);
-  MRI->setRegAllocationHint(reg, Hint.first, Hint.second);
-
-  weight = RHS.weight;
-  for (unsigned i = 0, e = RHS.getNumValNums(); i != e; ++i) {
-    const VNInfo *VNI = RHS.getValNumInfo(i);
-    createValueCopy(VNI, VNInfoAllocator);
-  }
-  for (unsigned i = 0, e = RHS.ranges.size(); i != e; ++i) {
-    const LiveRange &LR = RHS.ranges[i];
-    addRange(LiveRange(LR.start, LR.end, getValNumInfo(LR.valno->id)));
-  }
-
-  verify();
-}
-
 unsigned LiveInterval::getSize() const {
   unsigned Sum = 0;
   for (const_iterator I = begin(), E = end(); I != E; ++I)
@@ -705,9 +720,11 @@ raw_ostream& llvm::operator<<(raw_ostream& os, const LiveRange &LR) {
   return os << '[' << LR.start << ',' << LR.end << ':' << LR.valno->id << ")";
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void LiveRange::dump() const {
   dbgs() << *this << "\n";
 }
+#endif
 
 void LiveInterval::print(raw_ostream &OS) const {
   if (empty())
@@ -740,9 +757,11 @@ void LiveInterval::print(raw_ostream &OS) const {
   }
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void LiveInterval::dump() const {
   dbgs() << *this << "\n";
 }
+#endif
 
 #ifndef NDEBUG
 void LiveInterval::verify() const {
diff --git a/lib/CodeGen/LiveIntervalAnalysis.cpp b/lib/CodeGen/LiveIntervalAnalysis.cpp
index d0f8ae1..4e75d89 100644
--- a/lib/CodeGen/LiveIntervalAnalysis.cpp
+++ b/lib/CodeGen/LiveIntervalAnalysis.cpp
@@ -34,6 +34,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/STLExtras.h"
 #include "LiveRangeCalc.h"
+#include "VirtRegMap.h"
 #include <algorithm>
 #include <limits>
 #include <cmath>
@@ -109,8 +110,6 @@ bool LiveIntervals::runOnMachineFunction(MachineFunction &fn) {
   DomTree = &getAnalysis<MachineDominatorTree>();
   if (!LRCalc)
     LRCalc = new LiveRangeCalc();
-  AllocatableRegs = TRI->getAllocatableSet(fn);
-  ReservedRegs = TRI->getReservedRegs(fn);
 
   // Allocate space for all virtual registers.
   VirtRegIntervals.resize(MRI->getNumVirtRegs());
@@ -147,6 +146,11 @@ void LiveIntervals::print(raw_ostream &OS, const Module* ) const {
       OS << PrintReg(Reg) << " = " << getInterval(Reg) << '\n';
   }
 
+  OS << "RegMasks:";
+  for (unsigned i = 0, e = RegMaskSlots.size(); i != e; ++i)
+    OS << ' ' << RegMaskSlots[i];
+  OS << '\n';
+
   printInstrs(OS);
 }
 
@@ -155,9 +159,11 @@ void LiveIntervals::printInstrs(raw_ostream &OS) const {
   MF->print(OS, Indexes);
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void LiveIntervals::dumpInstrs() const {
   printInstrs(dbgs());
 }
+#endif
 
 static
 bool MultipleDefsBySameMI(const MachineInstr &MI, unsigned MOIdx) {
@@ -382,8 +388,7 @@ void LiveIntervals::handleRegisterDef(MachineBasicBlock *MBB,
 /// which a variable is live
 void LiveIntervals::computeIntervals() {
   DEBUG(dbgs() << "********** COMPUTING LIVE INTERVALS **********\n"
-               << "********** Function: "
-               << ((Value*)MF->getFunction())->getName() << '\n');
+               << "********** Function: " << MF->getName() << '\n');
 
   RegMaskBlocks.resize(MF->getNumBlockIDs());
 
@@ -440,7 +445,7 @@ void LiveIntervals::computeIntervals() {
 
     // Compute the number of register mask instructions in this block.
     std::pair<unsigned, unsigned> &RMB = RegMaskBlocks[MBB->getNumber()];
-    RMB.second = RegMaskSlots.size() - RMB.first;;
+    RMB.second = RegMaskSlots.size() - RMB.first;
   }
 
   // Create empty intervals for registers defined by implicit_def's (except
@@ -497,7 +502,7 @@ void LiveIntervals::computeRegMasks() {
           RegMaskBits.push_back(MO->getRegMask());
       }
     // Compute the number of register mask instructions in this block.
-    RMB.second = RegMaskSlots.size() - RMB.first;;
+    RMB.second = RegMaskSlots.size() - RMB.first;
   }
 }
 
@@ -540,11 +545,11 @@ void LiveIntervals::computeRegUnitInterval(LiveInterval *LI) {
   // Ignore uses of reserved registers. We only track defs of those.
   for (MCRegUnitRootIterator Roots(Unit, TRI); Roots.isValid(); ++Roots) {
     unsigned Root = *Roots;
-    if (!isReserved(Root) && !MRI->reg_empty(Root))
+    if (!MRI->isReserved(Root) && !MRI->reg_empty(Root))
       LRCalc->extendToUses(LI, Root);
     for (MCSuperRegIterator Supers(Root, TRI); Supers.isValid(); ++Supers) {
       unsigned Reg = *Supers;
-      if (!isReserved(Reg) && !MRI->reg_empty(Reg))
+      if (!MRI->isReserved(Reg) && !MRI->reg_empty(Reg))
         LRCalc->extendToUses(LI, Reg);
     }
   }
@@ -729,17 +734,100 @@ bool LiveIntervals::shrinkToUses(LiveInterval *li,
   return CanSeparate;
 }
 
+void LiveIntervals::extendToIndices(LiveInterval *LI,
+                                    ArrayRef<SlotIndex> Indices) {
+  assert(LRCalc && "LRCalc not initialized.");
+  LRCalc->reset(MF, getSlotIndexes(), DomTree, &getVNInfoAllocator());
+  for (unsigned i = 0, e = Indices.size(); i != e; ++i)
+    LRCalc->extend(LI, Indices[i]);
+}
+
+void LiveIntervals::pruneValue(LiveInterval *LI, SlotIndex Kill,
+                               SmallVectorImpl<SlotIndex> *EndPoints) {
+  LiveRangeQuery LRQ(*LI, Kill);
+  VNInfo *VNI = LRQ.valueOut();
+  if (!VNI)
+    return;
+
+  MachineBasicBlock *KillMBB = Indexes->getMBBFromIndex(Kill);
+  SlotIndex MBBStart, MBBEnd;
+  tie(MBBStart, MBBEnd) = Indexes->getMBBRange(KillMBB);
+
+  // If VNI isn't live out from KillMBB, the value is trivially pruned.
+  if (LRQ.endPoint() < MBBEnd) {
+    LI->removeRange(Kill, LRQ.endPoint());
+    if (EndPoints) EndPoints->push_back(LRQ.endPoint());
+    return;
+  }
+
+  // VNI is live out of KillMBB.
+  LI->removeRange(Kill, MBBEnd);
+  if (EndPoints) EndPoints->push_back(MBBEnd);
+
+  // Find all blocks that are reachable from KillMBB without leaving VNI's live
+  // range. It is possible that KillMBB itself is reachable, so start a DFS
+  // from each successor.
+  typedef SmallPtrSet<MachineBasicBlock*, 9> VisitedTy;
+  VisitedTy Visited;
+  for (MachineBasicBlock::succ_iterator
+       SuccI = KillMBB->succ_begin(), SuccE = KillMBB->succ_end();
+       SuccI != SuccE; ++SuccI) {
+    for (df_ext_iterator<MachineBasicBlock*, VisitedTy>
+         I = df_ext_begin(*SuccI, Visited), E = df_ext_end(*SuccI, Visited);
+         I != E;) {
+      MachineBasicBlock *MBB = *I;
+
+      // Check if VNI is live in to MBB.
+      tie(MBBStart, MBBEnd) = Indexes->getMBBRange(MBB);
+      LiveRangeQuery LRQ(*LI, MBBStart);
+      if (LRQ.valueIn() != VNI) {
+        // This block isn't part of the VNI live range. Prune the search.
+        I.skipChildren();
+        continue;
+      }
+
+      // Prune the search if VNI is killed in MBB.
+      if (LRQ.endPoint() < MBBEnd) {
+        LI->removeRange(MBBStart, LRQ.endPoint());
+        if (EndPoints) EndPoints->push_back(LRQ.endPoint());
+        I.skipChildren();
+        continue;
+      }
+
+      // VNI is live through MBB.
+      LI->removeRange(MBBStart, MBBEnd);
+      if (EndPoints) EndPoints->push_back(MBBEnd);
+      ++I;
+    }
+  }
+}
 
 //===----------------------------------------------------------------------===//
 // Register allocator hooks.
 //
 
-void LiveIntervals::addKillFlags() {
+void LiveIntervals::addKillFlags(const VirtRegMap *VRM) {
+  // Keep track of regunit ranges.
+  SmallVector<std::pair<LiveInterval*, LiveInterval::iterator>, 8> RU;
+
   for (unsigned i = 0, e = MRI->getNumVirtRegs(); i != e; ++i) {
     unsigned Reg = TargetRegisterInfo::index2VirtReg(i);
     if (MRI->reg_nodbg_empty(Reg))
       continue;
     LiveInterval *LI = &getInterval(Reg);
+    if (LI->empty())
+      continue;
+
+    // Find the regunit intervals for the assigned register. They may overlap
+    // the virtual register live range, cancelling any kills.
+    RU.clear();
+    for (MCRegUnitIterator Units(VRM->getPhys(Reg), TRI); Units.isValid();
+         ++Units) {
+      LiveInterval *RUInt = &getRegUnit(*Units);
+      if (RUInt->empty())
+        continue;
+      RU.push_back(std::make_pair(RUInt, RUInt->find(LI->begin()->end)));
+    }
 
     // Every instruction that kills Reg corresponds to a live range end point.
     for (LiveInterval::iterator RI = LI->begin(), RE = LI->end(); RI != RE;
@@ -750,7 +838,32 @@ void LiveIntervals::addKillFlags() {
       MachineInstr *MI = getInstructionFromIndex(RI->end);
       if (!MI)
         continue;
-      MI->addRegisterKilled(Reg, NULL);
+
+      // Check if any of the reguints are live beyond the end of RI. That could
+      // happen when a physreg is defined as a copy of a virtreg:
+      //
+      //   %EAX = COPY %vreg5
+      //   FOO %vreg5         <--- MI, cancel kill because %EAX is live.
+      //   BAR %EAX<kill>
+      //
+      // There should be no kill flag on FOO when %vreg5 is rewritten as %EAX.
+      bool CancelKill = false;
+      for (unsigned u = 0, e = RU.size(); u != e; ++u) {
+        LiveInterval *RInt = RU[u].first;
+        LiveInterval::iterator &I = RU[u].second;
+        if (I == RInt->end())
+          continue;
+        I = RInt->advanceTo(I, RI->end);
+        if (I == RInt->end() || I->start >= RI->end)
+          continue;
+        // I is overlapping RI.
+        CancelKill = true;
+        break;
+      }
+      if (CancelKill)
+        MI->clearRegisterKills(Reg, NULL);
+      else
+        MI->addRegisterKilled(Reg, NULL);
     }
   }
 }
@@ -900,497 +1013,321 @@ private:
   LiveIntervals& LIS;
   const MachineRegisterInfo& MRI;
   const TargetRegisterInfo& TRI;
+  SlotIndex OldIdx;
   SlotIndex NewIdx;
-
-  typedef std::pair<LiveInterval*, LiveRange*> IntRangePair;
-  typedef DenseSet<IntRangePair> RangeSet;
-
-  struct RegRanges {
-    LiveRange* Use;
-    LiveRange* EC;
-    LiveRange* Dead;
-    LiveRange* Def;
-    RegRanges() : Use(0), EC(0), Dead(0), Def(0) {}
-  };
-  typedef DenseMap<unsigned, RegRanges> BundleRanges;
+  SmallPtrSet<LiveInterval*, 8> Updated;
+  bool UpdateFlags;
 
 public:
   HMEditor(LiveIntervals& LIS, const MachineRegisterInfo& MRI,
-           const TargetRegisterInfo& TRI, SlotIndex NewIdx)
-    : LIS(LIS), MRI(MRI), TRI(TRI), NewIdx(NewIdx) {}
-
-  // Update intervals for all operands of MI from OldIdx to NewIdx.
-  // This assumes that MI used to be at OldIdx, and now resides at
-  // NewIdx.
-  void moveAllRangesFrom(MachineInstr* MI, SlotIndex OldIdx) {
-    assert(NewIdx != OldIdx && "No-op move? That's a bit strange.");
-
-    // Collect the operands.
-    RangeSet Entering, Internal, Exiting;
-    bool hasRegMaskOp = false;
-    collectRanges(MI, Entering, Internal, Exiting, hasRegMaskOp, OldIdx);
-
-    // To keep the LiveRanges valid within an interval, move the ranges closest
-    // to the destination first. This prevents ranges from overlapping, to that
-    // APIs like removeRange still work.
-    if (NewIdx < OldIdx) {
-      moveAllEnteringFrom(OldIdx, Entering);
-      moveAllInternalFrom(OldIdx, Internal);
-      moveAllExitingFrom(OldIdx, Exiting);
-    }
-    else {
-      moveAllExitingFrom(OldIdx, Exiting);
-      moveAllInternalFrom(OldIdx, Internal);
-      moveAllEnteringFrom(OldIdx, Entering);
-    }
-
-    if (hasRegMaskOp)
-      updateRegMaskSlots(OldIdx);
-
-#ifndef NDEBUG
-    LIValidator validator;
-    validator = std::for_each(Entering.begin(), Entering.end(), validator);
-    validator = std::for_each(Internal.begin(), Internal.end(), validator);
-    validator = std::for_each(Exiting.begin(), Exiting.end(), validator);
-    assert(validator.rangesOk() && "moveAllOperandsFrom broke liveness.");
-#endif
-
+           const TargetRegisterInfo& TRI,
+           SlotIndex OldIdx, SlotIndex NewIdx, bool UpdateFlags)
+    : LIS(LIS), MRI(MRI), TRI(TRI), OldIdx(OldIdx), NewIdx(NewIdx),
+      UpdateFlags(UpdateFlags) {}
+
+  // FIXME: UpdateFlags is a workaround that creates live intervals for all
+  // physregs, even those that aren't needed for regalloc, in order to update
+  // kill flags. This is wasteful. Eventually, LiveVariables will strip all kill
+  // flags, and postRA passes will use a live register utility instead.
+  LiveInterval *getRegUnitLI(unsigned Unit) {
+    if (UpdateFlags)
+      return &LIS.getRegUnit(Unit);
+    return LIS.getCachedRegUnit(Unit);
   }
 
-  // Update intervals for all operands of MI to refer to BundleStart's
-  // SlotIndex.
-  void moveAllRangesInto(MachineInstr* MI, MachineInstr* BundleStart) {
-    if (MI == BundleStart)
-      return; // Bundling instr with itself - nothing to do.
-
-    SlotIndex OldIdx = LIS.getSlotIndexes()->getInstructionIndex(MI);
-    assert(LIS.getSlotIndexes()->getInstructionFromIndex(OldIdx) == MI &&
-           "SlotIndex <-> Instruction mapping broken for MI");
-
-    // Collect all ranges already in the bundle.
-    MachineBasicBlock::instr_iterator BII(BundleStart);
-    RangeSet Entering, Internal, Exiting;
-    bool hasRegMaskOp = false;
-    collectRanges(BII, Entering, Internal, Exiting, hasRegMaskOp, NewIdx);
-    assert(!hasRegMaskOp && "Can't have RegMask operand in bundle.");
-    for (++BII; &*BII == MI || BII->isInsideBundle(); ++BII) {
-      if (&*BII == MI)
+  /// Update all live ranges touched by MI, assuming a move from OldIdx to
+  /// NewIdx.
+  void updateAllRanges(MachineInstr *MI) {
+    DEBUG(dbgs() << "handleMove " << OldIdx << " -> " << NewIdx << ": " << *MI);
+    bool hasRegMask = false;
+    for (MIOperands MO(MI); MO.isValid(); ++MO) {
+      if (MO->isRegMask())
+        hasRegMask = true;
+      if (!MO->isReg())
         continue;
-      collectRanges(BII, Entering, Internal, Exiting, hasRegMaskOp, NewIdx);
-      assert(!hasRegMaskOp && "Can't have RegMask operand in bundle.");
-    }
-
-    BundleRanges BR = createBundleRanges(Entering, Internal, Exiting);
-
-    Entering.clear();
-    Internal.clear();
-    Exiting.clear();
-    collectRanges(MI, Entering, Internal, Exiting, hasRegMaskOp, OldIdx);
-    assert(!hasRegMaskOp && "Can't have RegMask operand in bundle.");
+      // Aggressively clear all kill flags.
+      // They are reinserted by VirtRegRewriter.
+      if (MO->isUse())
+        MO->setIsKill(false);
 
-    DEBUG(dbgs() << "Entering: " << Entering.size() << "\n");
-    DEBUG(dbgs() << "Internal: " << Internal.size() << "\n");
-    DEBUG(dbgs() << "Exiting: " << Exiting.size() << "\n");
-
-    moveAllEnteringFromInto(OldIdx, Entering, BR);
-    moveAllInternalFromInto(OldIdx, Internal, BR);
-    moveAllExitingFromInto(OldIdx, Exiting, BR);
-
-
-#ifndef NDEBUG
-    LIValidator validator;
-    validator = std::for_each(Entering.begin(), Entering.end(), validator);
-    validator = std::for_each(Internal.begin(), Internal.end(), validator);
-    validator = std::for_each(Exiting.begin(), Exiting.end(), validator);
-    assert(validator.rangesOk() && "moveAllOperandsInto broke liveness.");
-#endif
-  }
-
-private:
-
-#ifndef NDEBUG
-  class LIValidator {
-  private:
-    DenseSet<const LiveInterval*> Checked, Bogus;
-  public:
-    void operator()(const IntRangePair& P) {
-      const LiveInterval* LI = P.first;
-      if (Checked.count(LI))
-        return;
-      Checked.insert(LI);
-      if (LI->empty())
-        return;
-      SlotIndex LastEnd = LI->begin()->start;
-      for (LiveInterval::const_iterator LRI = LI->begin(), LRE = LI->end();
-           LRI != LRE; ++LRI) {
-        const LiveRange& LR = *LRI;
-        if (LastEnd > LR.start || LR.start >= LR.end)
-          Bogus.insert(LI);
-        LastEnd = LR.end;
-      }
-    }
-
-    bool rangesOk() const {
-      return Bogus.empty();
-    }
-  };
-#endif
-
-  // Collect IntRangePairs for all operands of MI that may need fixing.
-  // Treat's MI's index as OldIdx (regardless of what it is in SlotIndexes'
-  // maps).
-  void collectRanges(MachineInstr* MI, RangeSet& Entering, RangeSet& Internal,
-                     RangeSet& Exiting, bool& hasRegMaskOp, SlotIndex OldIdx) {
-    hasRegMaskOp = false;
-    for (MachineInstr::mop_iterator MOI = MI->operands_begin(),
-                                    MOE = MI->operands_end();
-         MOI != MOE; ++MOI) {
-      const MachineOperand& MO = *MOI;
-
-      if (MO.isRegMask()) {
-        hasRegMaskOp = true;
+      unsigned Reg = MO->getReg();
+      if (!Reg)
         continue;
-      }
-
-      if (!MO.isReg() || MO.getReg() == 0)
-        continue;
-
-      unsigned Reg = MO.getReg();
-
-      // TODO: Currently we're skipping uses that are reserved or have no
-      // interval, but we're not updating their kills. This should be
-      // fixed.
-      if (TargetRegisterInfo::isPhysicalRegister(Reg) && LIS.isReserved(Reg))
+      if (TargetRegisterInfo::isVirtualRegister(Reg)) {
+        updateRange(LIS.getInterval(Reg));
         continue;
-
-      // Collect ranges for register units. These live ranges are computed on
-      // demand, so just skip any that haven't been computed yet.
-      if (TargetRegisterInfo::isPhysicalRegister(Reg)) {
-        for (MCRegUnitIterator Units(Reg, &TRI); Units.isValid(); ++Units)
-          if (LiveInterval *LI = LIS.getCachedRegUnit(*Units))
-            collectRanges(MO, LI, Entering, Internal, Exiting, OldIdx);
-      } else {
-        // Collect ranges for individual virtual registers.
-        collectRanges(MO, &LIS.getInterval(Reg),
-                      Entering, Internal, Exiting, OldIdx);
       }
+
+      // For physregs, only update the regunits that actually have a
+      // precomputed live range.
+      for (MCRegUnitIterator Units(Reg, &TRI); Units.isValid(); ++Units)
+        if (LiveInterval *LI = getRegUnitLI(*Units))
+          updateRange(*LI);
     }
+    if (hasRegMask)
+      updateRegMaskSlots();
   }
 
-  void collectRanges(const MachineOperand &MO, LiveInterval *LI,
-                     RangeSet &Entering, RangeSet &Internal, RangeSet &Exiting,
-                     SlotIndex OldIdx) {
-    if (MO.readsReg()) {
-      LiveRange* LR = LI->getLiveRangeContaining(OldIdx);
-      if (LR != 0)
-        Entering.insert(std::make_pair(LI, LR));
-    }
-    if (MO.isDef()) {
-      LiveRange* LR = LI->getLiveRangeContaining(OldIdx.getRegSlot());
-      assert(LR != 0 && "No live range for def?");
-      if (LR->end > OldIdx.getDeadSlot())
-        Exiting.insert(std::make_pair(LI, LR));
+private:
+  /// Update a single live range, assuming an instruction has been moved from
+  /// OldIdx to NewIdx.
+  void updateRange(LiveInterval &LI) {
+    if (!Updated.insert(&LI))
+      return;
+    DEBUG({
+      dbgs() << "     ";
+      if (TargetRegisterInfo::isVirtualRegister(LI.reg))
+        dbgs() << PrintReg(LI.reg);
       else
-        Internal.insert(std::make_pair(LI, LR));
-    }
+        dbgs() << PrintRegUnit(LI.reg, &TRI);
+      dbgs() << ":\t" << LI << '\n';
+    });
+    if (SlotIndex::isEarlierInstr(OldIdx, NewIdx))
+      handleMoveDown(LI);
+    else
+      handleMoveUp(LI);
+    DEBUG(dbgs() << "        -->\t" << LI << '\n');
+    LI.verify();
   }
 
-  BundleRanges createBundleRanges(RangeSet& Entering,
-                                  RangeSet& Internal,
-                                  RangeSet& Exiting) {
-    BundleRanges BR;
-
-    for (RangeSet::iterator EI = Entering.begin(), EE = Entering.end();
-         EI != EE; ++EI) {
-      LiveInterval* LI = EI->first;
-      LiveRange* LR = EI->second;
-      BR[LI->reg].Use = LR;
-    }
+  /// Update LI to reflect an instruction has been moved downwards from OldIdx
+  /// to NewIdx.
+  ///
+  /// 1. Live def at OldIdx:
+  ///    Move def to NewIdx, assert endpoint after NewIdx.
+  ///
+  /// 2. Live def at OldIdx, killed at NewIdx:
+  ///    Change to dead def at NewIdx.
+  ///    (Happens when bundling def+kill together).
+  ///
+  /// 3. Dead def at OldIdx:
+  ///    Move def to NewIdx, possibly across another live value.
+  ///
+  /// 4. Def at OldIdx AND at NewIdx:
+  ///    Remove live range [OldIdx;NewIdx) and value defined at OldIdx.
+  ///    (Happens when bundling multiple defs together).
+  ///
+  /// 5. Value read at OldIdx, killed before NewIdx:
+  ///    Extend kill to NewIdx.
+  ///
+  void handleMoveDown(LiveInterval &LI) {
+    // First look for a kill at OldIdx.
+    LiveInterval::iterator I = LI.find(OldIdx.getBaseIndex());
+    LiveInterval::iterator E = LI.end();
+    // Is LI even live at OldIdx?
+    if (I == E || SlotIndex::isEarlierInstr(OldIdx, I->start))
+      return;
 
-    for (RangeSet::iterator II = Internal.begin(), IE = Internal.end();
-         II != IE; ++II) {
-      LiveInterval* LI = II->first;
-      LiveRange* LR = II->second;
-      if (LR->end.isDead()) {
-        BR[LI->reg].Dead = LR;
-      } else {
-        BR[LI->reg].EC = LR;
-      }
+    // Handle a live-in value.
+    if (!SlotIndex::isSameInstr(I->start, OldIdx)) {
+      bool isKill = SlotIndex::isSameInstr(OldIdx, I->end);
+      // If the live-in value already extends to NewIdx, there is nothing to do.
+      if (!SlotIndex::isEarlierInstr(I->end, NewIdx))
+        return;
+      // Aggressively remove all kill flags from the old kill point.
+      // Kill flags shouldn't be used while live intervals exist, they will be
+      // reinserted by VirtRegRewriter.
+      if (MachineInstr *KillMI = LIS.getInstructionFromIndex(I->end))
+        for (MIBundleOperands MO(KillMI); MO.isValid(); ++MO)
+          if (MO->isReg() && MO->isUse())
+            MO->setIsKill(false);
+      // Adjust I->end to reach NewIdx. This may temporarily make LI invalid by
+      // overlapping ranges. Case 5 above.
+      I->end = NewIdx.getRegSlot(I->end.isEarlyClobber());
+      // If this was a kill, there may also be a def. Otherwise we're done.
+      if (!isKill)
+        return;
+      ++I;
     }
 
-    for (RangeSet::iterator EI = Exiting.begin(), EE = Exiting.end();
-         EI != EE; ++EI) {
-      LiveInterval* LI = EI->first;
-      LiveRange* LR = EI->second;
-      BR[LI->reg].Def = LR;
+    // Check for a def at OldIdx.
+    if (I == E || !SlotIndex::isSameInstr(OldIdx, I->start))
+      return;
+    // We have a def at OldIdx.
+    VNInfo *DefVNI = I->valno;
+    assert(DefVNI->def == I->start && "Inconsistent def");
+    DefVNI->def = NewIdx.getRegSlot(I->start.isEarlyClobber());
+    // If the defined value extends beyond NewIdx, just move the def down.
+    // This is case 1 above.
+    if (SlotIndex::isEarlierInstr(NewIdx, I->end)) {
+      I->start = DefVNI->def;
+      return;
     }
-
-    return BR;
-  }
-
-  void moveKillFlags(unsigned reg, SlotIndex OldIdx, SlotIndex newKillIdx) {
-    MachineInstr* OldKillMI = LIS.getInstructionFromIndex(OldIdx);
-    if (!OldKillMI->killsRegister(reg))
-      return; // Bail out if we don't have kill flags on the old register.
-    MachineInstr* NewKillMI = LIS.getInstructionFromIndex(newKillIdx);
-    assert(OldKillMI->killsRegister(reg) && "Old 'kill' instr isn't a kill.");
-    assert(!NewKillMI->killsRegister(reg) &&
-           "New kill instr is already a kill.");
-    OldKillMI->clearRegisterKills(reg, &TRI);
-    NewKillMI->addRegisterKilled(reg, &TRI);
-  }
-
-  void updateRegMaskSlots(SlotIndex OldIdx) {
-    SmallVectorImpl<SlotIndex>::iterator RI =
-      std::lower_bound(LIS.RegMaskSlots.begin(), LIS.RegMaskSlots.end(),
-                       OldIdx);
-    assert(*RI == OldIdx && "No RegMask at OldIdx.");
-    *RI = NewIdx;
-    assert(*prior(RI) < *RI && *RI < *next(RI) &&
-           "RegSlots out of order. Did you move one call across another?");
-  }
-
-  // Return the last use of reg between NewIdx and OldIdx.
-  SlotIndex findLastUseBefore(unsigned Reg, SlotIndex OldIdx) {
-    SlotIndex LastUse = NewIdx;
-    for (MachineRegisterInfo::use_nodbg_iterator
-           UI = MRI.use_nodbg_begin(Reg),
-           UE = MRI.use_nodbg_end();
-         UI != UE; UI.skipInstruction()) {
-      const MachineInstr* MI = &*UI;
-      SlotIndex InstSlot = LIS.getSlotIndexes()->getInstructionIndex(MI);
-      if (InstSlot > LastUse && InstSlot < OldIdx)
-        LastUse = InstSlot;
+    // The remaining possibilities are now:
+    // 2. Live def at OldIdx, killed at NewIdx: isSameInstr(I->end, NewIdx).
+    // 3. Dead def at OldIdx: I->end = OldIdx.getDeadSlot().
+    // In either case, it is possible that there is an existing def at NewIdx.
+    assert((I->end == OldIdx.getDeadSlot() ||
+            SlotIndex::isSameInstr(I->end, NewIdx)) &&
+            "Cannot move def below kill");
+    LiveInterval::iterator NewI = LI.advanceTo(I, NewIdx.getRegSlot());
+    if (NewI != E && SlotIndex::isSameInstr(NewI->start, NewIdx)) {
+      // There is an existing def at NewIdx, case 4 above. The def at OldIdx is
+      // coalesced into that value.
+      assert(NewI->valno != DefVNI && "Multiple defs of value?");
+      LI.removeValNo(DefVNI);
+      return;
     }
-    return LastUse;
+    // There was no existing def at NewIdx. Turn *I into a dead def at NewIdx.
+    // If the def at OldIdx was dead, we allow it to be moved across other LI
+    // values. The new range should be placed immediately before NewI, move any
+    // intermediate ranges up.
+    assert(NewI != I && "Inconsistent iterators");
+    std::copy(llvm::next(I), NewI, I);
+    *llvm::prior(NewI) = LiveRange(DefVNI->def, NewIdx.getDeadSlot(), DefVNI);
   }
 
-  void moveEnteringUpFrom(SlotIndex OldIdx, IntRangePair& P) {
-    LiveInterval* LI = P.first;
-    LiveRange* LR = P.second;
-    bool LiveThrough = LR->end > OldIdx.getRegSlot();
-    if (LiveThrough)
+  /// Update LI to reflect an instruction has been moved upwards from OldIdx
+  /// to NewIdx.
+  ///
+  /// 1. Live def at OldIdx:
+  ///    Hoist def to NewIdx.
+  ///
+  /// 2. Dead def at OldIdx:
+  ///    Hoist def+end to NewIdx, possibly move across other values.
+  ///
+  /// 3. Dead def at OldIdx AND existing def at NewIdx:
+  ///    Remove value defined at OldIdx, coalescing it with existing value.
+  ///
+  /// 4. Live def at OldIdx AND existing def at NewIdx:
+  ///    Remove value defined at NewIdx, hoist OldIdx def to NewIdx.
+  ///    (Happens when bundling multiple defs together).
+  ///
+  /// 5. Value killed at OldIdx:
+  ///    Hoist kill to NewIdx, then scan for last kill between NewIdx and
+  ///    OldIdx.
+  ///
+  void handleMoveUp(LiveInterval &LI) {
+    // First look for a kill at OldIdx.
+    LiveInterval::iterator I = LI.find(OldIdx.getBaseIndex());
+    LiveInterval::iterator E = LI.end();
+    // Is LI even live at OldIdx?
+    if (I == E || SlotIndex::isEarlierInstr(OldIdx, I->start))
       return;
-    SlotIndex LastUse = findLastUseBefore(LI->reg, OldIdx);
-    if (LastUse != NewIdx)
-      moveKillFlags(LI->reg, NewIdx, LastUse);
-    LR->end = LastUse.getRegSlot();
-  }
 
-  void moveEnteringDownFrom(SlotIndex OldIdx, IntRangePair& P) {
-    LiveInterval* LI = P.first;
-    LiveRange* LR = P.second;
-    // Extend the LiveRange if NewIdx is past the end.
-    if (NewIdx > LR->end) {
-      // Move kill flags if OldIdx was not originally the end
-      // (otherwise LR->end points to an invalid slot).
-      if (LR->end.getRegSlot() != OldIdx.getRegSlot()) {
-        assert(LR->end > OldIdx && "LiveRange does not cover original slot");
-        moveKillFlags(LI->reg, LR->end, NewIdx);
+    // Handle a live-in value.
+    if (!SlotIndex::isSameInstr(I->start, OldIdx)) {
+      // If the live-in value isn't killed here, there is nothing to do.
+      if (!SlotIndex::isSameInstr(OldIdx, I->end))
+        return;
+      // Adjust I->end to end at NewIdx. If we are hoisting a kill above
+      // another use, we need to search for that use. Case 5 above.
+      I->end = NewIdx.getRegSlot(I->end.isEarlyClobber());
+      ++I;
+      // If OldIdx also defines a value, there couldn't have been another use.
+      if (I == E || !SlotIndex::isSameInstr(I->start, OldIdx)) {
+        // No def, search for the new kill.
+        // This can never be an early clobber kill since there is no def.
+        llvm::prior(I)->end = findLastUseBefore(LI.reg).getRegSlot();
+        return;
       }
-      LR->end = NewIdx.getRegSlot();
-    }
-  }
-
-  void moveAllEnteringFrom(SlotIndex OldIdx, RangeSet& Entering) {
-    bool GoingUp = NewIdx < OldIdx;
-
-    if (GoingUp) {
-      for (RangeSet::iterator EI = Entering.begin(), EE = Entering.end();
-           EI != EE; ++EI)
-        moveEnteringUpFrom(OldIdx, *EI);
-    } else {
-      for (RangeSet::iterator EI = Entering.begin(), EE = Entering.end();
-           EI != EE; ++EI)
-        moveEnteringDownFrom(OldIdx, *EI);
     }
-  }
-
-  void moveInternalFrom(SlotIndex OldIdx, IntRangePair& P) {
-    LiveInterval* LI = P.first;
-    LiveRange* LR = P.second;
-    assert(OldIdx < LR->start && LR->start < OldIdx.getDeadSlot() &&
-           LR->end <= OldIdx.getDeadSlot() &&
-           "Range should be internal to OldIdx.");
-    LiveRange Tmp(*LR);
-    Tmp.start = NewIdx.getRegSlot(LR->start.isEarlyClobber());
-    Tmp.valno->def = Tmp.start;
-    Tmp.end = LR->end.isDead() ? NewIdx.getDeadSlot() : NewIdx.getRegSlot();
-    LI->removeRange(*LR);
-    LI->addRange(Tmp);
-  }
-
-  void moveAllInternalFrom(SlotIndex OldIdx, RangeSet& Internal) {
-    for (RangeSet::iterator II = Internal.begin(), IE = Internal.end();
-         II != IE; ++II)
-      moveInternalFrom(OldIdx, *II);
-  }
-
-  void moveExitingFrom(SlotIndex OldIdx, IntRangePair& P) {
-    LiveRange* LR = P.second;
-    assert(OldIdx < LR->start && LR->start < OldIdx.getDeadSlot() &&
-           "Range should start in OldIdx.");
-    assert(LR->end > OldIdx.getDeadSlot() && "Range should exit OldIdx.");
-    SlotIndex NewStart = NewIdx.getRegSlot(LR->start.isEarlyClobber());
-    LR->start = NewStart;
-    LR->valno->def = NewStart;
-  }
-
-  void moveAllExitingFrom(SlotIndex OldIdx, RangeSet& Exiting) {
-    for (RangeSet::iterator EI = Exiting.begin(), EE = Exiting.end();
-         EI != EE; ++EI)
-      moveExitingFrom(OldIdx, *EI);
-  }
 
-  void moveEnteringUpFromInto(SlotIndex OldIdx, IntRangePair& P,
-                              BundleRanges& BR) {
-    LiveInterval* LI = P.first;
-    LiveRange* LR = P.second;
-    bool LiveThrough = LR->end > OldIdx.getRegSlot();
-    if (LiveThrough) {
-      assert((LR->start < NewIdx || BR[LI->reg].Def == LR) &&
-             "Def in bundle should be def range.");
-      assert((BR[LI->reg].Use == 0 || BR[LI->reg].Use == LR) &&
-             "If bundle has use for this reg it should be LR.");
-      BR[LI->reg].Use = LR;
+    // Now deal with the def at OldIdx.
+    assert(I != E && SlotIndex::isSameInstr(I->start, OldIdx) && "No def?");
+    VNInfo *DefVNI = I->valno;
+    assert(DefVNI->def == I->start && "Inconsistent def");
+    DefVNI->def = NewIdx.getRegSlot(I->start.isEarlyClobber());
+
+    // Check for an existing def at NewIdx.
+    LiveInterval::iterator NewI = LI.find(NewIdx.getRegSlot());
+    if (SlotIndex::isSameInstr(NewI->start, NewIdx)) {
+      assert(NewI->valno != DefVNI && "Same value defined more than once?");
+      // There is an existing def at NewIdx.
+      if (I->end.isDead()) {
+        // Case 3: Remove the dead def at OldIdx.
+        LI.removeValNo(DefVNI);
+        return;
+      }
+      // Case 4: Replace def at NewIdx with live def at OldIdx.
+      I->start = DefVNI->def;
+      LI.removeValNo(NewI->valno);
       return;
     }
 
-    SlotIndex LastUse = findLastUseBefore(LI->reg, OldIdx);
-    moveKillFlags(LI->reg, OldIdx, LastUse);
-
-    if (LR->start < NewIdx) {
-      // Becoming a new entering range.
-      assert(BR[LI->reg].Dead == 0 && BR[LI->reg].Def == 0 &&
-             "Bundle shouldn't be re-defining reg mid-range.");
-      assert((BR[LI->reg].Use == 0 || BR[LI->reg].Use == LR) &&
-             "Bundle shouldn't have different use range for same reg.");
-      LR->end = LastUse.getRegSlot();
-      BR[LI->reg].Use = LR;
-    } else {
-      // Becoming a new Dead-def.
-      assert(LR->start == NewIdx.getRegSlot(LR->start.isEarlyClobber()) &&
-             "Live range starting at unexpected slot.");
-      assert(BR[LI->reg].Def == LR && "Reg should have def range.");
-      assert(BR[LI->reg].Dead == 0 &&
-               "Can't have def and dead def of same reg in a bundle.");
-      LR->end = LastUse.getDeadSlot();
-      BR[LI->reg].Dead = BR[LI->reg].Def;
-      BR[LI->reg].Def = 0;
-    }
-  }
-
-  void moveEnteringDownFromInto(SlotIndex OldIdx, IntRangePair& P,
-                                BundleRanges& BR) {
-    LiveInterval* LI = P.first;
-    LiveRange* LR = P.second;
-    if (NewIdx > LR->end) {
-      // Range extended to bundle. Add to bundle uses.
-      // Note: Currently adds kill flags to bundle start.
-      assert(BR[LI->reg].Use == 0 &&
-             "Bundle already has use range for reg.");
-      moveKillFlags(LI->reg, LR->end, NewIdx);
-      LR->end = NewIdx.getRegSlot();
-      BR[LI->reg].Use = LR;
-    } else {
-      assert(BR[LI->reg].Use != 0 &&
-             "Bundle should already have a use range for reg.");
-    }
-  }
-
-  void moveAllEnteringFromInto(SlotIndex OldIdx, RangeSet& Entering,
-                               BundleRanges& BR) {
-    bool GoingUp = NewIdx < OldIdx;
-
-    if (GoingUp) {
-      for (RangeSet::iterator EI = Entering.begin(), EE = Entering.end();
-           EI != EE; ++EI)
-        moveEnteringUpFromInto(OldIdx, *EI, BR);
-    } else {
-      for (RangeSet::iterator EI = Entering.begin(), EE = Entering.end();
-           EI != EE; ++EI)
-        moveEnteringDownFromInto(OldIdx, *EI, BR);
+    // There is no existing def at NewIdx. Hoist DefVNI.
+    if (!I->end.isDead()) {
+      // Leave the end point of a live def.
+      I->start = DefVNI->def;
+      return;
     }
-  }
 
-  void moveInternalFromInto(SlotIndex OldIdx, IntRangePair& P,
-                            BundleRanges& BR) {
-    // TODO: Sane rules for moving ranges into bundles.
+    // DefVNI is a dead def. It may have been moved across other values in LI,
+    // so move I up to NewI. Slide [NewI;I) down one position.
+    std::copy_backward(NewI, I, llvm::next(I));
+    *NewI = LiveRange(DefVNI->def, NewIdx.getDeadSlot(), DefVNI);
   }
 
-  void moveAllInternalFromInto(SlotIndex OldIdx, RangeSet& Internal,
-                               BundleRanges& BR) {
-    for (RangeSet::iterator II = Internal.begin(), IE = Internal.end();
-         II != IE; ++II)
-      moveInternalFromInto(OldIdx, *II, BR);
+  void updateRegMaskSlots() {
+    SmallVectorImpl<SlotIndex>::iterator RI =
+      std::lower_bound(LIS.RegMaskSlots.begin(), LIS.RegMaskSlots.end(),
+                       OldIdx);
+    assert(RI != LIS.RegMaskSlots.end() && *RI == OldIdx.getRegSlot() &&
+           "No RegMask at OldIdx.");
+    *RI = NewIdx.getRegSlot();
+    assert((RI == LIS.RegMaskSlots.begin() ||
+            SlotIndex::isEarlierInstr(*llvm::prior(RI), *RI)) &&
+            "Cannot move regmask instruction above another call");
+    assert((llvm::next(RI) == LIS.RegMaskSlots.end() ||
+            SlotIndex::isEarlierInstr(*RI, *llvm::next(RI))) &&
+            "Cannot move regmask instruction below another call");
   }
 
-  void moveExitingFromInto(SlotIndex OldIdx, IntRangePair& P,
-                           BundleRanges& BR) {
-    LiveInterval* LI = P.first;
-    LiveRange* LR = P.second;
-
-    assert(LR->start.isRegister() &&
-           "Don't know how to merge exiting ECs into bundles yet.");
+  // Return the last use of reg between NewIdx and OldIdx.
+  SlotIndex findLastUseBefore(unsigned Reg) {
+    SlotIndex LastUse = NewIdx;
 
-    if (LR->end > NewIdx.getDeadSlot()) {
-      // This range is becoming an exiting range on the bundle.
-      // If there was an old dead-def of this reg, delete it.
-      if (BR[LI->reg].Dead != 0) {
-        LI->removeRange(*BR[LI->reg].Dead);
-        BR[LI->reg].Dead = 0;
+    if (TargetRegisterInfo::isVirtualRegister(Reg)) {
+      for (MachineRegisterInfo::use_nodbg_iterator
+             UI = MRI.use_nodbg_begin(Reg),
+             UE = MRI.use_nodbg_end();
+           UI != UE; UI.skipInstruction()) {
+        const MachineInstr* MI = &*UI;
+        SlotIndex InstSlot = LIS.getSlotIndexes()->getInstructionIndex(MI);
+        if (InstSlot > LastUse && InstSlot < OldIdx)
+          LastUse = InstSlot;
       }
-      assert(BR[LI->reg].Def == 0 &&
-             "Can't have two defs for the same variable exiting a bundle.");
-      LR->start = NewIdx.getRegSlot();
-      LR->valno->def = LR->start;
-      BR[LI->reg].Def = LR;
     } else {
-      // This range is becoming internal to the bundle.
-      assert(LR->end == NewIdx.getRegSlot() &&
-             "Can't bundle def whose kill is before the bundle");
-      if (BR[LI->reg].Dead || BR[LI->reg].Def) {
-        // Already have a def for this. Just delete range.
-        LI->removeRange(*LR);
-      } else {
-        // Make range dead, record.
-        LR->end = NewIdx.getDeadSlot();
-        BR[LI->reg].Dead = LR;
-        assert(BR[LI->reg].Use == LR &&
-               "Range becoming dead should currently be use.");
+      MachineInstr* MI = LIS.getSlotIndexes()->getInstructionFromIndex(NewIdx);
+      MachineBasicBlock::iterator MII(MI);
+      ++MII;
+      MachineBasicBlock* MBB = MI->getParent();
+      for (; MII != MBB->end() && LIS.getInstructionIndex(MII) < OldIdx; ++MII){
+        for (MachineInstr::mop_iterator MOI = MII->operands_begin(),
+                                        MOE = MII->operands_end();
+             MOI != MOE; ++MOI) {
+          const MachineOperand& mop = *MOI;
+          if (!mop.isReg() || mop.getReg() == 0 ||
+              TargetRegisterInfo::isVirtualRegister(mop.getReg()))
+            continue;
+
+          if (TRI.hasRegUnit(mop.getReg(), Reg))
+            LastUse = LIS.getInstructionIndex(MII);
+        }
       }
-      // In both cases the range is no longer a use on the bundle.
-      BR[LI->reg].Use = 0;
     }
+    return LastUse;
   }
-
-  void moveAllExitingFromInto(SlotIndex OldIdx, RangeSet& Exiting,
-                              BundleRanges& BR) {
-    for (RangeSet::iterator EI = Exiting.begin(), EE = Exiting.end();
-         EI != EE; ++EI)
-      moveExitingFromInto(OldIdx, *EI, BR);
-  }
-
 };
 
-void LiveIntervals::handleMove(MachineInstr* MI) {
+void LiveIntervals::handleMove(MachineInstr* MI, bool UpdateFlags) {
+  assert(!MI->isBundled() && "Can't handle bundled instructions yet.");
   SlotIndex OldIndex = Indexes->getInstructionIndex(MI);
   Indexes->removeMachineInstrFromMaps(MI);
-  SlotIndex NewIndex = MI->isInsideBundle() ?
-                        Indexes->getInstructionIndex(MI) :
-                        Indexes->insertMachineInstrInMaps(MI);
+  SlotIndex NewIndex = Indexes->insertMachineInstrInMaps(MI);
   assert(getMBBStartIdx(MI->getParent()) <= OldIndex &&
          OldIndex < getMBBEndIdx(MI->getParent()) &&
          "Cannot handle moves across basic block boundaries.");
-  assert(!MI->isBundled() && "Can't handle bundled instructions yet.");
 
-  HMEditor HME(*this, *MRI, *TRI, NewIndex);
-  HME.moveAllRangesFrom(MI, OldIndex);
+  HMEditor HME(*this, *MRI, *TRI, OldIndex, NewIndex, UpdateFlags);
+  HME.updateAllRanges(MI);
 }
 
 void LiveIntervals::handleMoveIntoBundle(MachineInstr* MI,
-                                         MachineInstr* BundleStart) {
+                                         MachineInstr* BundleStart,
+                                         bool UpdateFlags) {
+  SlotIndex OldIndex = Indexes->getInstructionIndex(MI);
   SlotIndex NewIndex = Indexes->getInstructionIndex(BundleStart);
-  HMEditor HME(*this, *MRI, *TRI, NewIndex);
-  HME.moveAllRangesInto(MI, BundleStart);
+  HMEditor HME(*this, *MRI, *TRI, OldIndex, NewIndex, UpdateFlags);
+  HME.updateAllRanges(MI);
 }
diff --git a/lib/CodeGen/LiveIntervalUnion.h b/lib/CodeGen/LiveIntervalUnion.h
index cd4e690..4d41fca 100644
--- a/lib/CodeGen/LiveIntervalUnion.h
+++ b/lib/CodeGen/LiveIntervalUnion.h
@@ -178,8 +178,8 @@ public:
     bool checkLoopInterference(MachineLoopRange*);
 
   private:
-    Query(const Query&);          // DO NOT IMPLEMENT
-    void operator=(const Query&); // DO NOT IMPLEMENT
+    Query(const Query&) LLVM_DELETED_FUNCTION;
+    void operator=(const Query&) LLVM_DELETED_FUNCTION;
   };
 
   // Array of LiveIntervalUnions.
diff --git a/lib/CodeGen/LiveRangeCalc.cpp b/lib/CodeGen/LiveRangeCalc.cpp
index d828f25..c3ff4f1 100644
--- a/lib/CodeGen/LiveRangeCalc.cpp
+++ b/lib/CodeGen/LiveRangeCalc.cpp
@@ -65,7 +65,11 @@ void LiveRangeCalc::extendToUses(LiveInterval *LI, unsigned Reg) {
   // Visit all operands that read Reg. This may include partial defs.
   for (MachineRegisterInfo::reg_nodbg_iterator I = MRI->reg_nodbg_begin(Reg),
        E = MRI->reg_nodbg_end(); I != E; ++I) {
-    const MachineOperand &MO = I.getOperand();
+    MachineOperand &MO = I.getOperand();
+    // Clear all kill flags. They will be reinserted after register allocation
+    // by LiveIntervalAnalysis::addKillFlags().
+    if (MO.isUse())
+      MO.setIsKill(false);
     if (!MO.readsReg())
       continue;
     // MI is reading Reg. We may have visited MI before if it happens to be
diff --git a/lib/CodeGen/LiveRangeEdit.cpp b/lib/CodeGen/LiveRangeEdit.cpp
index b4ce9aa..f8fbc7d 100644
--- a/lib/CodeGen/LiveRangeEdit.cpp
+++ b/lib/CodeGen/LiveRangeEdit.cpp
@@ -87,7 +87,7 @@ bool LiveRangeEdit::allUsesAvailableAt(const MachineInstr *OrigMI,
 
     // We can't remat physreg uses, unless it is a constant.
     if (TargetRegisterInfo::isPhysicalRegister(MO.getReg())) {
-      if (MRI.isConstantPhysReg(MO.getReg(), VRM->getMachineFunction()))
+      if (MRI.isConstantPhysReg(MO.getReg(), *OrigMI->getParent()->getParent()))
         continue;
       return false;
     }
@@ -96,6 +96,13 @@ bool LiveRangeEdit::allUsesAvailableAt(const MachineInstr *OrigMI,
     const VNInfo *OVNI = li.getVNInfoAt(OrigIdx);
     if (!OVNI)
       continue;
+
+    // Don't allow rematerialization immediately after the original def.
+    // It would be incorrect if OrigMI redefines the register.
+    // See PR14098.
+    if (SlotIndex::isSameInstr(OrigIdx, UseIdx))
+      return false;
+
     if (OVNI != li.getVNInfoAt(UseIdx))
       return false;
   }
@@ -249,7 +256,7 @@ void LiveRangeEdit::eliminateDeadDefs(SmallVectorImpl<MachineInstr*> &Dead,
         unsigned Reg = MOI->getReg();
         if (!TargetRegisterInfo::isVirtualRegister(Reg)) {
           // Check if MI reads any unreserved physregs.
-          if (Reg && MOI->readsReg() && !LIS.isReserved(Reg))
+          if (Reg && MOI->readsReg() && !MRI.isReserved(Reg))
             ReadsPhysRegs = true;
           continue;
         }
diff --git a/lib/CodeGen/LiveRegMatrix.cpp b/lib/CodeGen/LiveRegMatrix.cpp
index cdb1776..7f22478 100644
--- a/lib/CodeGen/LiveRegMatrix.cpp
+++ b/lib/CodeGen/LiveRegMatrix.cpp
@@ -13,6 +13,7 @@
 
 #define DEBUG_TYPE "regalloc"
 #include "LiveRegMatrix.h"
+#include "RegisterCoalescer.h"
 #include "VirtRegMap.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
@@ -117,8 +118,9 @@ bool LiveRegMatrix::checkRegUnitInterference(LiveInterval &VirtReg,
                                              unsigned PhysReg) {
   if (VirtReg.empty())
     return false;
+  CoalescerPair CP(VirtReg.reg, PhysReg, *TRI);
   for (MCRegUnitIterator Units(PhysReg, TRI); Units.isValid(); ++Units)
-    if (VirtReg.overlaps(LIS->getRegUnit(*Units)))
+    if (VirtReg.overlaps(LIS->getRegUnit(*Units), CP, *LIS->getSlotIndexes()))
       return true;
   return false;
 }
diff --git a/lib/CodeGen/LiveRegMatrix.h b/lib/CodeGen/LiveRegMatrix.h
index b3e2d7f..8f22c24 100644
--- a/lib/CodeGen/LiveRegMatrix.h
+++ b/lib/CodeGen/LiveRegMatrix.h
@@ -15,7 +15,7 @@
 // Register units are defined in MCRegisterInfo.h, they represent the smallest
 // unit of interference when dealing with overlapping physical registers. The
 // LiveRegMatrix is represented as a LiveIntervalUnion per register unit. When
-// a virtual register is assigned to a physicval register, the live range for
+// a virtual register is assigned to a physical register, the live range for
 // the virtual register is inserted into the LiveIntervalUnion for each regunit
 // in the physreg.
 //
diff --git a/lib/CodeGen/LiveStackAnalysis.cpp b/lib/CodeGen/LiveStackAnalysis.cpp
index 939e795..f0b522b 100644
--- a/lib/CodeGen/LiveStackAnalysis.cpp
+++ b/lib/CodeGen/LiveStackAnalysis.cpp
@@ -25,7 +25,10 @@
 using namespace llvm;
 
 char LiveStacks::ID = 0;
-INITIALIZE_PASS(LiveStacks, "livestacks",
+INITIALIZE_PASS_BEGIN(LiveStacks, "livestacks",
+                "Live Stack Slot Analysis", false, false)
+INITIALIZE_PASS_DEPENDENCY(SlotIndexes)
+INITIALIZE_PASS_END(LiveStacks, "livestacks",
                 "Live Stack Slot Analysis", false, false)
 
 char &llvm::LiveStacksID = LiveStacks::ID;
diff --git a/lib/CodeGen/LiveVariables.cpp b/lib/CodeGen/LiveVariables.cpp
index 348ed3a..6ea933d 100644
--- a/lib/CodeGen/LiveVariables.cpp
+++ b/lib/CodeGen/LiveVariables.cpp
@@ -65,6 +65,7 @@ LiveVariables::VarInfo::findKill(const MachineBasicBlock *MBB) const {
 }
 
 void LiveVariables::VarInfo::dump() const {
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   dbgs() << "  Alive in blocks: ";
   for (SparseBitVector<>::iterator I = AliveBlocks.begin(),
            E = AliveBlocks.end(); I != E; ++I)
@@ -77,6 +78,7 @@ void LiveVariables::VarInfo::dump() const {
       dbgs() << "\n    #" << i << ": " << *Kills[i];
     dbgs() << "\n";
   }
+#endif
 }
 
 /// getVarInfo - Get (possibly creating) a VarInfo object for the given vreg.
@@ -501,8 +503,6 @@ bool LiveVariables::runOnMachineFunction(MachineFunction &mf) {
   MRI = &mf.getRegInfo();
   TRI = MF->getTarget().getRegisterInfo();
 
-  ReservedRegisters = TRI->getReservedRegs(mf);
-
   unsigned NumRegs = TRI->getNumRegs();
   PhysRegDef  = new MachineInstr*[NumRegs];
   PhysRegUse  = new MachineInstr*[NumRegs];
@@ -586,7 +586,7 @@ bool LiveVariables::runOnMachineFunction(MachineFunction &mf) {
         unsigned MOReg = UseRegs[i];
         if (TargetRegisterInfo::isVirtualRegister(MOReg))
           HandleVirtRegUse(MOReg, MBB, MI);
-        else if (!ReservedRegisters[MOReg])
+        else if (!MRI->isReserved(MOReg))
           HandlePhysRegUse(MOReg, MI);
       }
 
@@ -599,7 +599,7 @@ bool LiveVariables::runOnMachineFunction(MachineFunction &mf) {
         unsigned MOReg = DefRegs[i];
         if (TargetRegisterInfo::isVirtualRegister(MOReg))
           HandleVirtRegDef(MOReg, MI);
-        else if (!ReservedRegisters[MOReg])
+        else if (!MRI->isReserved(MOReg))
           HandlePhysRegDef(MOReg, MI, Defs);
       }
       UpdatePhysRegDefs(MI, Defs);
@@ -806,18 +806,44 @@ void LiveVariables::addNewBlock(MachineBasicBlock *BB,
                                 MachineBasicBlock *SuccBB) {
   const unsigned NumNew = BB->getNumber();
 
-  // All registers used by PHI nodes in SuccBB must be live through BB.
-  for (MachineBasicBlock::iterator BBI = SuccBB->begin(),
-         BBE = SuccBB->end(); BBI != BBE && BBI->isPHI(); ++BBI)
+  SmallSet<unsigned, 16> Defs, Kills;
+
+  MachineBasicBlock::iterator BBI = SuccBB->begin(), BBE = SuccBB->end();
+  for (; BBI != BBE && BBI->isPHI(); ++BBI) {
+    // Record the def of the PHI node.
+    Defs.insert(BBI->getOperand(0).getReg());
+
+    // All registers used by PHI nodes in SuccBB must be live through BB.
     for (unsigned i = 1, e = BBI->getNumOperands(); i != e; i += 2)
       if (BBI->getOperand(i+1).getMBB() == BB)
         getVarInfo(BBI->getOperand(i).getReg()).AliveBlocks.set(NumNew);
+  }
+
+  // Record all vreg defs and kills of all instructions in SuccBB.
+  for (; BBI != BBE; ++BBI) {
+    for (MachineInstr::mop_iterator I = BBI->operands_begin(),
+         E = BBI->operands_end(); I != E; ++I) {
+      if (I->isReg() && TargetRegisterInfo::isVirtualRegister(I->getReg())) {
+        if (I->isDef())
+          Defs.insert(I->getReg());
+        else if (I->isKill())
+          Kills.insert(I->getReg());
+      }
+    }
+  }
 
   // Update info for all live variables
   for (unsigned i = 0, e = MRI->getNumVirtRegs(); i != e; ++i) {
     unsigned Reg = TargetRegisterInfo::index2VirtReg(i);
+
+    // If the Defs is defined in the successor it can't be live in BB.
+    if (Defs.count(Reg))
+      continue;
+
+    // If the register is either killed in or live through SuccBB it's also live
+    // through BB.
     VarInfo &VI = getVarInfo(Reg);
-    if (!VI.AliveBlocks.test(NumNew) && VI.isLiveIn(*SuccBB, Reg, *MRI))
+    if (Kills.count(Reg) || VI.AliveBlocks.test(SuccBB->getNumber()))
       VI.AliveBlocks.set(NumNew);
   }
 }
diff --git a/lib/CodeGen/MachineBasicBlock.cpp b/lib/CodeGen/MachineBasicBlock.cpp
index fa6b450..18d021d 100644
--- a/lib/CodeGen/MachineBasicBlock.cpp
+++ b/lib/CodeGen/MachineBasicBlock.cpp
@@ -21,7 +21,7 @@
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/Target/TargetRegisterInfo.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetInstrInfo.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Assembly/Writer.h"
@@ -145,7 +145,8 @@ MachineBasicBlock::iterator MachineBasicBlock::getFirstNonPHI() {
   instr_iterator I = instr_begin(), E = instr_end();
   while (I != E && I->isPHI())
     ++I;
-  assert(!I->isInsideBundle() && "First non-phi MI cannot be inside a bundle!");
+  assert((I == E || !I->isInsideBundle()) &&
+         "First non-phi MI cannot be inside a bundle!");
   return I;
 }
 
@@ -156,7 +157,7 @@ MachineBasicBlock::SkipPHIsAndLabels(MachineBasicBlock::iterator I) {
     ++I;
   // FIXME: This needs to change if we wish to bundle labels / dbg_values
   // inside the bundle.
-  assert(!I->isInsideBundle() &&
+  assert((I == E || !I->isInsideBundle()) &&
          "First non-phi / non-label instruction is inside a bundle!");
   return I;
 }
@@ -228,9 +229,11 @@ const MachineBasicBlock *MachineBasicBlock::getLandingPadSuccessor() const {
   return 0;
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void MachineBasicBlock::dump() const {
   print(dbgs());
 }
+#endif
 
 StringRef MachineBasicBlock::getName() const {
   if (const BasicBlock *LBB = getBasicBlock())
@@ -243,7 +246,7 @@ StringRef MachineBasicBlock::getName() const {
 std::string MachineBasicBlock::getFullName() const {
   std::string Name;
   if (getParent())
-    Name = (getParent()->getFunction()->getName() + ":").str();
+    Name = (getParent()->getName() + ":").str();
   if (getBasicBlock())
     Name += getBasicBlock()->getName();
   else
@@ -942,12 +945,11 @@ MachineBasicBlock::findDebugLoc(instr_iterator MBBI) {
 
 /// getSuccWeight - Return weight of the edge from this block to MBB.
 ///
-uint32_t MachineBasicBlock::getSuccWeight(const MachineBasicBlock *succ) const {
+uint32_t MachineBasicBlock::getSuccWeight(const_succ_iterator Succ) const {
   if (Weights.empty())
     return 0;
 
-  const_succ_iterator I = std::find(Successors.begin(), Successors.end(), succ);
-  return *getWeightIterator(I);
+  return *getWeightIterator(Succ);
 }
 
 /// getWeightIterator - Return wight iterator corresonding to the I successor
@@ -970,6 +972,80 @@ getWeightIterator(MachineBasicBlock::const_succ_iterator I) const {
   return Weights.begin() + index;
 }
 
+/// Return whether (physical) register "Reg" has been <def>ined and not <kill>ed
+/// as of just before "MI".
+/// 
+/// Search is localised to a neighborhood of
+/// Neighborhood instructions before (searching for defs or kills) and N
+/// instructions after (searching just for defs) MI.
+MachineBasicBlock::LivenessQueryResult
+MachineBasicBlock::computeRegisterLiveness(const TargetRegisterInfo *TRI,
+                                           unsigned Reg, MachineInstr *MI,
+                                           unsigned Neighborhood) {
+  
+  unsigned N = Neighborhood;
+  MachineBasicBlock *MBB = MI->getParent();
+
+  // Start by searching backwards from MI, looking for kills, reads or defs.
+
+  MachineBasicBlock::iterator I(MI);
+  // If this is the first insn in the block, don't search backwards.
+  if (I != MBB->begin()) {
+    do {
+      --I;
+
+      MachineOperandIteratorBase::PhysRegInfo Analysis =
+        MIOperands(I).analyzePhysReg(Reg, TRI);
+
+      if (Analysis.Kills)
+        // Register killed, so isn't live.
+        return LQR_Dead;
+
+      else if (Analysis.DefinesOverlap || Analysis.ReadsOverlap)
+        // Defined or read without a previous kill - live.
+        return (Analysis.Defines || Analysis.Reads) ? 
+          LQR_Live : LQR_OverlappingLive;
+
+    } while (I != MBB->begin() && --N > 0);
+  }
+
+  // Did we get to the start of the block?
+  if (I == MBB->begin()) {
+    // If so, the register's state is definitely defined by the live-in state.
+    for (MCRegAliasIterator RAI(Reg, TRI, /*IncludeSelf=*/true);
+         RAI.isValid(); ++RAI) {
+      if (MBB->isLiveIn(*RAI))
+        return (*RAI == Reg) ? LQR_Live : LQR_OverlappingLive;
+    }
+
+    return LQR_Dead;
+  }
+
+  N = Neighborhood;
+
+  // Try searching forwards from MI, looking for reads or defs.
+  I = MachineBasicBlock::iterator(MI);
+  // If this is the last insn in the block, don't search forwards.
+  if (I != MBB->end()) {
+    for (++I; I != MBB->end() && N > 0; ++I, --N) {
+      MachineOperandIteratorBase::PhysRegInfo Analysis =
+        MIOperands(I).analyzePhysReg(Reg, TRI);
+
+      if (Analysis.ReadsOverlap)
+        // Used, therefore must have been live.
+        return (Analysis.Reads) ?
+          LQR_Live : LQR_OverlappingLive;
+
+      else if (Analysis.DefinesOverlap)
+        // Defined (but not read) therefore cannot have been live.
+        return LQR_Dead;
+    }
+  }
+
+  // At this point we have no idea of the liveness of the register.
+  return LQR_Unknown;
+}
+
 void llvm::WriteAsOperand(raw_ostream &OS, const MachineBasicBlock *MBB,
                           bool t) {
   OS << "BB#" << MBB->getNumber();
diff --git a/lib/CodeGen/MachineBlockPlacement.cpp b/lib/CodeGen/MachineBlockPlacement.cpp
index c4dca2c..cd3f199 100644
--- a/lib/CodeGen/MachineBlockPlacement.cpp
+++ b/lib/CodeGen/MachineBlockPlacement.cpp
@@ -500,11 +500,10 @@ void MachineBlockPlacement::buildChain(
     assert(BB);
     assert(BlockToChain[BB] == &Chain);
     assert(*llvm::prior(Chain.end()) == BB);
-    MachineBasicBlock *BestSucc = 0;
 
     // Look for the best viable successor if there is one to place immediately
     // after this block.
-    BestSucc = selectBestSuccessor(BB, Chain, BlockFilter);
+    MachineBasicBlock *BestSucc = selectBestSuccessor(BB, Chain, BlockFilter);
 
     // If an immediate successor isn't available, look for the best viable
     // block among those we've identified as not violating the loop's CFG at
@@ -1014,7 +1013,8 @@ void MachineBlockPlacement::buildCFGChains(MachineFunction &F) {
   // exclusively on the loop info here so that we can align backedges in
   // unnatural CFGs and backedges that were introduced purely because of the
   // loop rotations done during this layout pass.
-  if (F.getFunction()->hasFnAttr(Attribute::OptimizeForSize))
+  if (F.getFunction()->getFnAttributes().
+        hasAttribute(Attributes::OptimizeForSize))
     return;
   unsigned Align = TLI->getPrefLoopAlignment();
   if (!Align)
diff --git a/lib/CodeGen/MachineBranchProbabilityInfo.cpp b/lib/CodeGen/MachineBranchProbabilityInfo.cpp
index 0cc1af0..4479211 100644
--- a/lib/CodeGen/MachineBranchProbabilityInfo.cpp
+++ b/lib/CodeGen/MachineBranchProbabilityInfo.cpp
@@ -38,7 +38,7 @@ getSumForBlock(const MachineBasicBlock *MBB, uint32_t &Scale) const {
   Scale = 1;
   for (MachineBasicBlock::const_succ_iterator I = MBB->succ_begin(),
        E = MBB->succ_end(); I != E; ++I) {
-    uint32_t Weight = getEdgeWeight(MBB, *I);
+    uint32_t Weight = getEdgeWeight(MBB, I);
     Sum += Weight;
   }
 
@@ -53,22 +53,30 @@ getSumForBlock(const MachineBasicBlock *MBB, uint32_t &Scale) const {
   Sum = 0;
   for (MachineBasicBlock::const_succ_iterator I = MBB->succ_begin(),
        E = MBB->succ_end(); I != E; ++I) {
-    uint32_t Weight = getEdgeWeight(MBB, *I);
+    uint32_t Weight = getEdgeWeight(MBB, I);
     Sum += Weight / Scale;
   }
   assert(Sum <= UINT32_MAX);
   return Sum;
 }
 
-uint32_t
-MachineBranchProbabilityInfo::getEdgeWeight(const MachineBasicBlock *Src,
-                                            const MachineBasicBlock *Dst) const {
+uint32_t MachineBranchProbabilityInfo::
+getEdgeWeight(const MachineBasicBlock *Src,
+              MachineBasicBlock::const_succ_iterator Dst) const {
   uint32_t Weight = Src->getSuccWeight(Dst);
   if (!Weight)
     return DEFAULT_WEIGHT;
   return Weight;
 }
 
+uint32_t MachineBranchProbabilityInfo::
+getEdgeWeight(const MachineBasicBlock *Src,
+              const MachineBasicBlock *Dst) const {
+  // This is a linear search. Try to use the const_succ_iterator version when
+  // possible.
+  return getEdgeWeight(Src, std::find(Src->succ_begin(), Src->succ_end(), Dst));
+}
+
 bool MachineBranchProbabilityInfo::isEdgeHot(MachineBasicBlock *Src,
                                              MachineBasicBlock *Dst) const {
   // Hot probability is at least 4/5 = 80%
@@ -82,7 +90,7 @@ MachineBranchProbabilityInfo::getHotSucc(MachineBasicBlock *MBB) const {
   MachineBasicBlock *MaxSucc = 0;
   for (MachineBasicBlock::const_succ_iterator I = MBB->succ_begin(),
        E = MBB->succ_end(); I != E; ++I) {
-    uint32_t Weight = getEdgeWeight(MBB, *I);
+    uint32_t Weight = getEdgeWeight(MBB, I);
     if (Weight > MaxWeight) {
       MaxWeight = Weight;
       MaxSucc = *I;
diff --git a/lib/CodeGen/MachineCSE.cpp b/lib/CodeGen/MachineCSE.cpp
index 896461f..dbc41de 100644
--- a/lib/CodeGen/MachineCSE.cpp
+++ b/lib/CodeGen/MachineCSE.cpp
@@ -63,8 +63,6 @@ namespace {
     virtual void releaseMemory() {
       ScopeMap.clear();
       Exps.clear();
-      AllocatableRegs.clear();
-      ReservedRegs.clear();
     }
 
   private:
@@ -78,8 +76,6 @@ namespace {
     ScopedHTType VNT;
     SmallVector<MachineInstr*, 64> Exps;
     unsigned CurrVN;
-    BitVector AllocatableRegs;
-    BitVector ReservedRegs;
 
     bool PerformTrivialCoalescing(MachineInstr *MI, MachineBasicBlock *MBB);
     bool isPhysDefTriviallyDead(unsigned Reg,
@@ -88,7 +84,8 @@ namespace {
     bool hasLivePhysRegDefUses(const MachineInstr *MI,
                                const MachineBasicBlock *MBB,
                                SmallSet<unsigned,8> &PhysRefs,
-                               SmallVector<unsigned,2> &PhysDefs) const;
+                               SmallVector<unsigned,2> &PhysDefs,
+                               bool &PhysUseDef) const;
     bool PhysRegDefsReach(MachineInstr *CSMI, MachineInstr *MI,
                           SmallSet<unsigned,8> &PhysRefs,
                           SmallVector<unsigned,2> &PhysDefs,
@@ -198,31 +195,52 @@ MachineCSE::isPhysDefTriviallyDead(unsigned Reg,
 bool MachineCSE::hasLivePhysRegDefUses(const MachineInstr *MI,
                                        const MachineBasicBlock *MBB,
                                        SmallSet<unsigned,8> &PhysRefs,
-                                       SmallVector<unsigned,2> &PhysDefs) const{
-  MachineBasicBlock::const_iterator I = MI; I = llvm::next(I);
+                                       SmallVector<unsigned,2> &PhysDefs,
+                                       bool &PhysUseDef) const{
+  // First, add all uses to PhysRefs.
   for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
     const MachineOperand &MO = MI->getOperand(i);
-    if (!MO.isReg())
+    if (!MO.isReg() || MO.isDef())
       continue;
     unsigned Reg = MO.getReg();
     if (!Reg)
       continue;
     if (TargetRegisterInfo::isVirtualRegister(Reg))
       continue;
-    // If the def is dead, it's ok. But the def may not marked "dead". That's
-    // common since this pass is run before livevariables. We can scan
-    // forward a few instructions and check if it is obviously dead.
-    if (MO.isDef() &&
-        (MO.isDead() || isPhysDefTriviallyDead(Reg, I, MBB->end())))
-      continue;
     // Reading constant physregs is ok.
     if (!MRI->isConstantPhysReg(Reg, *MBB->getParent()))
       for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI)
         PhysRefs.insert(*AI);
-    if (MO.isDef())
+  }
+
+  // Next, collect all defs into PhysDefs.  If any is already in PhysRefs
+  // (which currently contains only uses), set the PhysUseDef flag.
+  PhysUseDef = false;
+  MachineBasicBlock::const_iterator I = MI; I = llvm::next(I);
+  for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
+    const MachineOperand &MO = MI->getOperand(i);
+    if (!MO.isReg() || !MO.isDef())
+      continue;
+    unsigned Reg = MO.getReg();
+    if (!Reg)
+      continue;
+    if (TargetRegisterInfo::isVirtualRegister(Reg))
+      continue;
+    // Check against PhysRefs even if the def is "dead".
+    if (PhysRefs.count(Reg))
+      PhysUseDef = true;
+    // If the def is dead, it's ok. But the def may not marked "dead". That's
+    // common since this pass is run before livevariables. We can scan
+    // forward a few instructions and check if it is obviously dead.
+    if (!MO.isDead() && !isPhysDefTriviallyDead(Reg, I, MBB->end()))
       PhysDefs.push_back(Reg);
   }
 
+  // Finally, add all defs to PhysRefs as well.
+  for (unsigned i = 0, e = PhysDefs.size(); i != e; ++i)
+    for (MCRegAliasIterator AI(PhysDefs[i], TRI, true); AI.isValid(); ++AI)
+      PhysRefs.insert(*AI);
+
   return !PhysRefs.empty();
 }
 
@@ -242,7 +260,7 @@ bool MachineCSE::PhysRegDefsReach(MachineInstr *CSMI, MachineInstr *MI,
       return false;
 
     for (unsigned i = 0, e = PhysDefs.size(); i != e; ++i) {
-      if (AllocatableRegs.test(PhysDefs[i]) || ReservedRegs.test(PhysDefs[i]))
+      if (MRI->isAllocatable(PhysDefs[i]) || MRI->isReserved(PhysDefs[i]))
         // Avoid extending live range of physical registers if they are
         //allocatable or reserved.
         return false;
@@ -411,8 +429,8 @@ void MachineCSE::ExitScope(MachineBasicBlock *MBB) {
   DEBUG(dbgs() << "Exiting: " << MBB->getName() << '\n');
   DenseMap<MachineBasicBlock*, ScopeType*>::iterator SI = ScopeMap.find(MBB);
   assert(SI != ScopeMap.end());
-  ScopeMap.erase(SI);
   delete SI->second;
+  ScopeMap.erase(SI);
 }
 
 bool MachineCSE::ProcessBlock(MachineBasicBlock *MBB) {
@@ -463,16 +481,22 @@ bool MachineCSE::ProcessBlock(MachineBasicBlock *MBB) {
     bool CrossMBBPhysDef = false;
     SmallSet<unsigned, 8> PhysRefs;
     SmallVector<unsigned, 2> PhysDefs;
-    if (FoundCSE && hasLivePhysRegDefUses(MI, MBB, PhysRefs, PhysDefs)) {
+    bool PhysUseDef = false;
+    if (FoundCSE && hasLivePhysRegDefUses(MI, MBB, PhysRefs,
+                                          PhysDefs, PhysUseDef)) {
       FoundCSE = false;
 
       // ... Unless the CS is local or is in the sole predecessor block
       // and it also defines the physical register which is not clobbered
       // in between and the physical register uses were not clobbered.
-      unsigned CSVN = VNT.lookup(MI);
-      MachineInstr *CSMI = Exps[CSVN];
-      if (PhysRegDefsReach(CSMI, MI, PhysRefs, PhysDefs, CrossMBBPhysDef))
-        FoundCSE = true;
+      // This can never be the case if the instruction both uses and
+      // defines the same physical register, which was detected above.
+      if (!PhysUseDef) {
+        unsigned CSVN = VNT.lookup(MI);
+        MachineInstr *CSMI = Exps[CSVN];
+        if (PhysRegDefsReach(CSMI, MI, PhysRefs, PhysDefs, CrossMBBPhysDef))
+          FoundCSE = true;
+      }
     }
 
     if (!FoundCSE) {
@@ -635,7 +659,5 @@ bool MachineCSE::runOnMachineFunction(MachineFunction &MF) {
   MRI = &MF.getRegInfo();
   AA = &getAnalysis<AliasAnalysis>();
   DT = &getAnalysis<MachineDominatorTree>();
-  AllocatableRegs = TRI->getAllocatableSet(MF);
-  ReservedRegs = TRI->getReservedRegs(MF);
   return PerformCSE(DT->getRootNode());
 }
diff --git a/lib/CodeGen/MachineCopyPropagation.cpp b/lib/CodeGen/MachineCopyPropagation.cpp
index bac3aa2..4a79328 100644
--- a/lib/CodeGen/MachineCopyPropagation.cpp
+++ b/lib/CodeGen/MachineCopyPropagation.cpp
@@ -16,6 +16,7 @@
 #include "llvm/Pass.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/Target/TargetRegisterInfo.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -32,7 +33,7 @@ STATISTIC(NumDeletes, "Number of dead copies deleted");
 namespace {
   class MachineCopyPropagation : public MachineFunctionPass {
     const TargetRegisterInfo *TRI;
-    BitVector ReservedRegs;
+    MachineRegisterInfo *MRI;
 
   public:
     static char ID; // Pass identification, replacement for typeid
@@ -146,8 +147,8 @@ bool MachineCopyPropagation::CopyPropagateBlock(MachineBasicBlock &MBB) {
       DenseMap<unsigned, MachineInstr*>::iterator CI = AvailCopyMap.find(Src);
       if (CI != AvailCopyMap.end()) {
         MachineInstr *CopyMI = CI->second;
-        if (!ReservedRegs.test(Def) &&
-            (!ReservedRegs.test(Src) || NoInterveningSideEffect(CopyMI, MI)) &&
+        if (!MRI->isReserved(Def) &&
+            (!MRI->isReserved(Src) || NoInterveningSideEffect(CopyMI, MI)) &&
             isNopCopy(CopyMI, Def, Src, TRI)) {
           // The two copies cancel out and the source of the first copy
           // hasn't been overridden, eliminate the second one. e.g.
@@ -259,7 +260,7 @@ bool MachineCopyPropagation::CopyPropagateBlock(MachineBasicBlock &MBB) {
            DI = MaybeDeadCopies.begin(), DE = MaybeDeadCopies.end();
            DI != DE; ++DI) {
         unsigned Reg = (*DI)->getOperand(0).getReg();
-        if (ReservedRegs.test(Reg) || !MaskMO.clobbersPhysReg(Reg))
+        if (MRI->isReserved(Reg) || !MaskMO.clobbersPhysReg(Reg))
           continue;
         (*DI)->eraseFromParent();
         Changed = true;
@@ -296,7 +297,7 @@ bool MachineCopyPropagation::CopyPropagateBlock(MachineBasicBlock &MBB) {
     for (SmallSetVector<MachineInstr*, 8>::iterator
            DI = MaybeDeadCopies.begin(), DE = MaybeDeadCopies.end();
          DI != DE; ++DI) {
-      if (!ReservedRegs.test((*DI)->getOperand(0).getReg())) {
+      if (!MRI->isReserved((*DI)->getOperand(0).getReg())) {
         (*DI)->eraseFromParent();
         Changed = true;
         ++NumDeletes;
@@ -311,7 +312,7 @@ bool MachineCopyPropagation::runOnMachineFunction(MachineFunction &MF) {
   bool Changed = false;
 
   TRI = MF.getTarget().getRegisterInfo();
-  ReservedRegs = TRI->getReservedRegs(MF);
+  MRI = &MF.getRegInfo();
 
   for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I)
     Changed |= CopyPropagateBlock(*I);
diff --git a/lib/CodeGen/MachineFunction.cpp b/lib/CodeGen/MachineFunction.cpp
index d4aede8a..91d5211 100644
--- a/lib/CodeGen/MachineFunction.cpp
+++ b/lib/CodeGen/MachineFunction.cpp
@@ -28,7 +28,7 @@
 #include "llvm/MC/MCContext.h"
 #include "llvm/Analysis/ConstantFolding.h"
 #include "llvm/Support/Debug.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetLowering.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetFrameLowering.h"
@@ -59,13 +59,13 @@ MachineFunction::MachineFunction(const Function *F, const TargetMachine &TM,
     RegInfo = 0;
   MFInfo = 0;
   FrameInfo = new (Allocator) MachineFrameInfo(*TM.getFrameLowering());
-  if (Fn->hasFnAttr(Attribute::StackAlignment))
-    FrameInfo->ensureMaxAlignment(Attribute::getStackAlignmentFromAttrs(
-        Fn->getAttributes().getFnAttributes()));
-  ConstantPool = new (Allocator) MachineConstantPool(TM.getTargetData());
+  if (Fn->getFnAttributes().hasAttribute(Attributes::StackAlignment))
+    FrameInfo->ensureMaxAlignment(Fn->getAttributes().
+                                  getFnAttributes().getStackAlignment());
+  ConstantPool = new (Allocator) MachineConstantPool(TM.getDataLayout());
   Alignment = TM.getTargetLowering()->getMinFunctionAlignment();
   // FIXME: Shouldn't use pref alignment if explicit alignment is set on Fn.
-  if (!Fn->hasFnAttr(Attribute::OptimizeForSize))
+  if (!Fn->getFnAttributes().hasAttribute(Attributes::OptimizeForSize))
     Alignment = std::max(Alignment,
                          TM.getTargetLowering()->getPrefFunctionAlignment());
   FunctionNumber = FunctionNum;
@@ -284,12 +284,19 @@ MachineFunction::extractStoreMemRefs(MachineInstr::mmo_iterator Begin,
   return std::make_pair(Result, Result + Num);
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void MachineFunction::dump() const {
   print(dbgs());
 }
+#endif
+
+StringRef MachineFunction::getName() const {
+  assert(getFunction() && "No function!");
+  return getFunction()->getName();
+}
 
 void MachineFunction::print(raw_ostream &OS, SlotIndexes *Indexes) const {
-  OS << "# Machine code for function " << Fn->getName() << ": ";
+  OS << "# Machine code for function " << getName() << ": ";
   if (RegInfo) {
     OS << (RegInfo->isSSA() ? "SSA" : "Post SSA");
     if (!RegInfo->tracksLiveness())
@@ -334,7 +341,7 @@ void MachineFunction::print(raw_ostream &OS, SlotIndexes *Indexes) const {
     BB->print(OS, Indexes);
   }
 
-  OS << "\n# End machine code for function " << Fn->getName() << ".\n\n";
+  OS << "\n# End machine code for function " << getName() << ".\n\n";
 }
 
 namespace llvm {
@@ -344,7 +351,7 @@ namespace llvm {
   DOTGraphTraits (bool isSimple=false) : DefaultDOTGraphTraits(isSimple) {}
 
     static std::string getGraphName(const MachineFunction *F) {
-      return "CFG for '" + F->getFunction()->getName().str() + "' function";
+      return "CFG for '" + F->getName().str() + "' function";
     }
 
     std::string getNodeLabel(const MachineBasicBlock *Node,
@@ -377,7 +384,7 @@ namespace llvm {
 void MachineFunction::viewCFG() const
 {
 #ifndef NDEBUG
-  ViewGraph(this, "mf" + getFunction()->getName());
+  ViewGraph(this, "mf" + getName());
 #else
   errs() << "MachineFunction::viewCFG is only available in debug builds on "
          << "systems with Graphviz or gv!\n";
@@ -387,7 +394,7 @@ void MachineFunction::viewCFG() const
 void MachineFunction::viewCFGOnly() const
 {
 #ifndef NDEBUG
-  ViewGraph(this, "mf" + getFunction()->getName(), true);
+  ViewGraph(this, "mf" + getName(), true);
 #else
   errs() << "MachineFunction::viewCFGOnly is only available in debug builds on "
          << "systems with Graphviz or gv!\n";
@@ -453,7 +460,9 @@ int MachineFrameInfo::CreateFixedObject(uint64_t Size, int64_t SPOffset,
   unsigned StackAlign = TFI.getStackAlignment();
   unsigned Align = MinAlign(SPOffset, StackAlign);
   Objects.insert(Objects.begin(), StackObject(Size, Align, SPOffset, Immutable,
-                                              /*isSS*/false, false));
+                                              /*isSS*/   false,
+                                              /*NeedSP*/ false,
+                                              /*Alloca*/ 0));
   return -++NumFixedObjects;
 }
 
@@ -525,16 +534,18 @@ void MachineFrameInfo::print(const MachineFunction &MF, raw_ostream &OS) const{
   }
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void MachineFrameInfo::dump(const MachineFunction &MF) const {
   print(MF, dbgs());
 }
+#endif
 
 //===----------------------------------------------------------------------===//
 //  MachineJumpTableInfo implementation
 //===----------------------------------------------------------------------===//
 
 /// getEntrySize - Return the size of each entry in the jump table.
-unsigned MachineJumpTableInfo::getEntrySize(const TargetData &TD) const {
+unsigned MachineJumpTableInfo::getEntrySize(const DataLayout &TD) const {
   // The size of a jump table entry is 4 bytes unless the entry is just the
   // address of a block, in which case it is the pointer size.
   switch (getEntryKind()) {
@@ -553,7 +564,7 @@ unsigned MachineJumpTableInfo::getEntrySize(const TargetData &TD) const {
 }
 
 /// getEntryAlignment - Return the alignment of each entry in the jump table.
-unsigned MachineJumpTableInfo::getEntryAlignment(const TargetData &TD) const {
+unsigned MachineJumpTableInfo::getEntryAlignment(const DataLayout &TD) const {
   // The alignment of a jump table entry is the alignment of int32 unless the
   // entry is just the address of a block, in which case it is the pointer
   // alignment.
@@ -622,7 +633,9 @@ void MachineJumpTableInfo::print(raw_ostream &OS) const {
   OS << '\n';
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void MachineJumpTableInfo::dump() const { print(dbgs()); }
+#endif
 
 
 //===----------------------------------------------------------------------===//
@@ -657,7 +670,7 @@ MachineConstantPool::~MachineConstantPool() {
 /// CanShareConstantPoolEntry - Test whether the given two constants
 /// can be allocated the same constant pool entry.
 static bool CanShareConstantPoolEntry(const Constant *A, const Constant *B,
-                                      const TargetData *TD) {
+                                      const DataLayout *TD) {
   // Handle the trivial case quickly.
   if (A == B) return true;
 
@@ -681,7 +694,7 @@ static bool CanShareConstantPoolEntry(const Constant *A, const Constant *B,
   // Try constant folding a bitcast of both instructions to an integer.  If we
   // get two identical ConstantInt's, then we are good to share them.  We use
   // the constant folding APIs to do this so that we get the benefit of
-  // TargetData.
+  // DataLayout.
   if (isa<PointerType>(A->getType()))
     A = ConstantFoldInstOperands(Instruction::PtrToInt, IntTy,
                                  const_cast<Constant*>(A), TD);
@@ -749,10 +762,12 @@ void MachineConstantPool::print(raw_ostream &OS) const {
     if (Constants[i].isMachineConstantPoolEntry())
       Constants[i].Val.MachineCPVal->print(OS);
     else
-      OS << *(Value*)Constants[i].Val.ConstVal;
+      OS << *(const Value*)Constants[i].Val.ConstVal;
     OS << ", align=" << Constants[i].getAlignment();
     OS << "\n";
   }
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void MachineConstantPool::dump() const { print(dbgs()); }
+#endif
diff --git a/lib/CodeGen/MachineFunctionPrinterPass.cpp b/lib/CodeGen/MachineFunctionPrinterPass.cpp
index 0102ac7..ed94efb 100644
--- a/lib/CodeGen/MachineFunctionPrinterPass.cpp
+++ b/lib/CodeGen/MachineFunctionPrinterPass.cpp
@@ -51,7 +51,7 @@ struct MachineFunctionPrinterPass : public MachineFunctionPass {
 char MachineFunctionPrinterPass::ID = 0;
 }
 
-char &MachineFunctionPrinterPassID = MachineFunctionPrinterPass::ID;
+char &llvm::MachineFunctionPrinterPassID = MachineFunctionPrinterPass::ID;
 INITIALIZE_PASS(MachineFunctionPrinterPass, "print-machineinstrs",
                 "Machine Function Printer", false, false)
 
diff --git a/lib/CodeGen/MachineInstr.cpp b/lib/CodeGen/MachineInstr.cpp
index b166849..ce8d520 100644
--- a/lib/CodeGen/MachineInstr.cpp
+++ b/lib/CodeGen/MachineInstr.cpp
@@ -111,6 +111,7 @@ void MachineOperand::setIsDef(bool Val) {
 /// the specified value.  If an operand is known to be an immediate already,
 /// the setImm method should be used.
 void MachineOperand::ChangeToImmediate(int64_t ImmVal) {
+  assert((!isReg() || !isTied()) && "Cannot change a tied operand into an imm");
   // If this operand is currently a register operand, and if this is in a
   // function, deregister the operand from the register's use/def list.
   if (isReg() && isOnRegUseList())
@@ -136,7 +137,8 @@ void MachineOperand::ChangeToRegister(unsigned Reg, bool isDef, bool isImp,
         RegInfo = &MF->getRegInfo();
   // If this operand is already a register operand, remove it from the
   // register's use/def lists.
-  if (RegInfo && isReg())
+  bool WasReg = isReg();
+  if (RegInfo && WasReg)
     RegInfo->removeRegOperandFromUseList(this);
 
   // Change this to a register and set the reg#.
@@ -153,6 +155,9 @@ void MachineOperand::ChangeToRegister(unsigned Reg, bool isDef, bool isImp,
   IsDebug = isDebug;
   // Ensure isOnRegUseList() returns false.
   Contents.Reg.Prev = 0;
+  // Preserve the tie when the operand was already a register.
+  if (!WasReg)
+    TiedTo = 0;
 
   // If this operand is embedded in a function, add the operand to the
   // register's use/def list.
@@ -193,7 +198,8 @@ bool MachineOperand::isIdenticalTo(const MachineOperand &Other) const {
     return !strcmp(getSymbolName(), Other.getSymbolName()) &&
            getOffset() == Other.getOffset();
   case MachineOperand::MO_BlockAddress:
-    return getBlockAddress() == Other.getBlockAddress();
+    return getBlockAddress() == Other.getBlockAddress() &&
+           getOffset() == Other.getOffset();
   case MO_RegisterMask:
     return getRegMask() == Other.getRegMask();
   case MachineOperand::MO_MCSymbol:
@@ -208,8 +214,8 @@ bool MachineOperand::isIdenticalTo(const MachineOperand &Other) const {
 hash_code llvm::hash_value(const MachineOperand &MO) {
   switch (MO.getType()) {
   case MachineOperand::MO_Register:
-    return hash_combine(MO.getType(), MO.getTargetFlags(), MO.getReg(),
-                        MO.getSubReg(), MO.isDef());
+    // Register operands don't have target flags.
+    return hash_combine(MO.getType(), MO.getReg(), MO.getSubReg(), MO.isDef());
   case MachineOperand::MO_Immediate:
     return hash_combine(MO.getType(), MO.getTargetFlags(), MO.getImm());
   case MachineOperand::MO_CImmediate:
@@ -234,7 +240,7 @@ hash_code llvm::hash_value(const MachineOperand &MO) {
                         MO.getOffset());
   case MachineOperand::MO_BlockAddress:
     return hash_combine(MO.getType(), MO.getTargetFlags(),
-                        MO.getBlockAddress());
+                        MO.getBlockAddress(), MO.getOffset());
   case MachineOperand::MO_RegisterMask:
     return hash_combine(MO.getType(), MO.getTargetFlags(), MO.getRegMask());
   case MachineOperand::MO_Metadata:
@@ -262,7 +268,7 @@ void MachineOperand::print(raw_ostream &OS, const TargetMachine *TM) const {
     OS << PrintReg(getReg(), TRI, getSubReg());
 
     if (isDef() || isKill() || isDead() || isImplicit() || isUndef() ||
-        isInternalRead() || isEarlyClobber()) {
+        isInternalRead() || isEarlyClobber() || isTied()) {
       OS << '<';
       bool NeedComma = false;
       if (isDef()) {
@@ -282,27 +288,32 @@ void MachineOperand::print(raw_ostream &OS, const TargetMachine *TM) const {
           NeedComma = true;
       }
 
-      if (isKill() || isDead() || (isUndef() && isUse()) || isInternalRead()) {
+      if (isKill()) {
         if (NeedComma) OS << ',';
-        NeedComma = false;
-        if (isKill()) {
-          OS << "kill";
-          NeedComma = true;
-        }
-        if (isDead()) {
-          OS << "dead";
-          NeedComma = true;
-        }
-        if (isUndef() && isUse()) {
-          if (NeedComma) OS << ',';
-          OS << "undef";
-          NeedComma = true;
-        }
-        if (isInternalRead()) {
-          if (NeedComma) OS << ',';
-          OS << "internal";
-          NeedComma = true;
-        }
+        OS << "kill";
+        NeedComma = true;
+      }
+      if (isDead()) {
+        if (NeedComma) OS << ',';
+        OS << "dead";
+        NeedComma = true;
+      }
+      if (isUndef() && isUse()) {
+        if (NeedComma) OS << ',';
+        OS << "undef";
+        NeedComma = true;
+      }
+      if (isInternalRead()) {
+        if (NeedComma) OS << ',';
+        OS << "internal";
+        NeedComma = true;
+      }
+      if (isTied()) {
+        if (NeedComma) OS << ',';
+        OS << "tied";
+        if (TiedTo != 15)
+          OS << unsigned(TiedTo - 1);
+        NeedComma = true;
       }
       OS << '>';
     }
@@ -352,6 +363,7 @@ void MachineOperand::print(raw_ostream &OS, const TargetMachine *TM) const {
   case MachineOperand::MO_BlockAddress:
     OS << '<';
     WriteAsOperand(OS, getBlockAddress(), /*PrintType=*/false);
+    if (getOffset()) OS << "+" << getOffset();
     OS << '>';
     break;
   case MachineOperand::MO_RegisterMask:
@@ -528,20 +540,6 @@ void MachineInstr::addImplicitDefUseOperands() {
 /// MachineInstr ctor - This constructor creates a MachineInstr and adds the
 /// implicit operands. It reserves space for the number of operands specified by
 /// the MCInstrDesc.
-MachineInstr::MachineInstr(const MCInstrDesc &tid, bool NoImp)
-  : MCID(&tid), Flags(0), AsmPrinterFlags(0),
-    NumMemRefs(0), MemRefs(0), Parent(0) {
-  unsigned NumImplicitOps = 0;
-  if (!NoImp)
-    NumImplicitOps = MCID->getNumImplicitDefs() + MCID->getNumImplicitUses();
-  Operands.reserve(NumImplicitOps + MCID->getNumOperands());
-  if (!NoImp)
-    addImplicitDefUseOperands();
-  // Make sure that we get added to a machine basicblock
-  LeakDetector::addGarbageObject(this);
-}
-
-/// MachineInstr ctor - As above, but with a DebugLoc.
 MachineInstr::MachineInstr(const MCInstrDesc &tid, const DebugLoc dl,
                            bool NoImp)
   : MCID(&tid), Flags(0), AsmPrinterFlags(0),
@@ -559,21 +557,6 @@ MachineInstr::MachineInstr(const MCInstrDesc &tid, const DebugLoc dl,
 /// MachineInstr ctor - Work exactly the same as the ctor two above, except
 /// that the MachineInstr is created and added to the end of the specified
 /// basic block.
-MachineInstr::MachineInstr(MachineBasicBlock *MBB, const MCInstrDesc &tid)
-  : MCID(&tid), Flags(0), AsmPrinterFlags(0),
-    NumMemRefs(0), MemRefs(0), Parent(0) {
-  assert(MBB && "Cannot use inserting ctor with null basic block!");
-  unsigned NumImplicitOps =
-    MCID->getNumImplicitDefs() + MCID->getNumImplicitUses();
-  Operands.reserve(NumImplicitOps + MCID->getNumOperands());
-  addImplicitDefUseOperands();
-  // Make sure that we get added to a machine basicblock
-  LeakDetector::addGarbageObject(this);
-  MBB->push_back(this);  // Add instruction to end of basic block!
-}
-
-/// MachineInstr ctor - As above, but with a DebugLoc.
-///
 MachineInstr::MachineInstr(MachineBasicBlock *MBB, const DebugLoc dl,
                            const MCInstrDesc &tid)
   : MCID(&tid), Flags(0), AsmPrinterFlags(0),
@@ -673,6 +656,7 @@ void MachineInstr::addOperand(const MachineOperand &Op) {
   if (!isImpReg && !isInlineAsm()) {
     while (OpNo && Operands[OpNo-1].isReg() && Operands[OpNo-1].isImplicit()) {
       --OpNo;
+      assert(!Operands[OpNo].isTied() && "Cannot move tied operands");
       if (RegInfo)
         RegInfo->removeRegOperandFromUseList(&Operands[OpNo]);
     }
@@ -708,12 +692,25 @@ void MachineInstr::addOperand(const MachineOperand &Op) {
   if (Operands[OpNo].isReg()) {
     // Ensure isOnRegUseList() returns false, regardless of Op's status.
     Operands[OpNo].Contents.Reg.Prev = 0;
+    // Ignore existing ties. This is not a property that can be copied.
+    Operands[OpNo].TiedTo = 0;
     // Add the new operand to RegInfo.
     if (RegInfo)
       RegInfo->addRegOperandToUseList(&Operands[OpNo]);
-    // If the register operand is flagged as early, mark the operand as such.
-    if (MCID->getOperandConstraint(OpNo, MCOI::EARLY_CLOBBER) != -1)
-      Operands[OpNo].setIsEarlyClobber(true);
+    // The MCID operand information isn't accurate until we start adding
+    // explicit operands. The implicit operands are added first, then the
+    // explicits are inserted before them.
+    if (!isImpReg) {
+      // Tie uses to defs as indicated in MCInstrDesc.
+      if (Operands[OpNo].isUse()) {
+        int DefIdx = MCID->getOperandConstraint(OpNo, MCOI::TIED_TO);
+        if (DefIdx != -1)
+          tieOperands(DefIdx, OpNo);
+      }
+      // If the register operand is flagged as early, mark the operand as such.
+      if (MCID->getOperandConstraint(OpNo, MCOI::EARLY_CLOBBER) != -1)
+        Operands[OpNo].setIsEarlyClobber(true);
+    }
   }
 
   // Re-add all the implicit ops.
@@ -730,6 +727,7 @@ void MachineInstr::addOperand(const MachineOperand &Op) {
 ///
 void MachineInstr::RemoveOperand(unsigned OpNo) {
   assert(OpNo < Operands.size() && "Invalid operand number");
+  untieRegOperand(OpNo);
   MachineRegisterInfo *RegInfo = getRegInfo();
 
   // Special case removing the last one.
@@ -752,6 +750,13 @@ void MachineInstr::RemoveOperand(unsigned OpNo) {
     }
   }
 
+#ifndef NDEBUG
+  // Moving tied operands would break the ties.
+  for (unsigned i = OpNo + 1, e = Operands.size(); i != e; ++i)
+    if (Operands[i].isReg())
+      assert(!Operands[i].isTied() && "Cannot move tied operands");
+#endif
+
   Operands.erase(Operands.begin()+OpNo);
 
   if (RegInfo) {
@@ -935,6 +940,12 @@ bool MachineInstr::isStackAligningInlineAsm() const {
   return false;
 }
 
+InlineAsm::AsmDialect MachineInstr::getInlineAsmDialect() const {
+  assert(isInlineAsm() && "getInlineAsmDialect() only works for inline asms!");
+  unsigned ExtraInfo = getOperand(InlineAsm::MIOp_ExtraInfo).getImm();
+  return InlineAsm::AsmDialect((ExtraInfo & InlineAsm::Extra_AsmDialect) != 0);
+}
+
 int MachineInstr::findInlineAsmFlagIdx(unsigned OpIdx,
                                        unsigned *GroupNo) const {
   assert(isInlineAsm() && "Expected an inline asm instruction");
@@ -1004,9 +1015,10 @@ MachineInstr::getRegClassConstraint(unsigned OpIdx,
 unsigned MachineInstr::getBundleSize() const {
   assert(isBundle() && "Expecting a bundle");
 
-  MachineBasicBlock::const_instr_iterator I = *this;
+  const MachineBasicBlock *MBB = getParent();
+  MachineBasicBlock::const_instr_iterator I = *this, E = MBB->instr_end();
   unsigned Size = 0;
-  while ((++I)->isInsideBundle()) {
+  while ((++I != E) && I->isInsideBundle()) {
     ++Size;
   }
   assert(Size > 1 && "Malformed bundle");
@@ -1114,107 +1126,99 @@ int MachineInstr::findFirstPredOperandIdx() const {
   return -1;
 }
 
-/// isRegTiedToUseOperand - Given the index of a register def operand,
-/// check if the register def is tied to a source operand, due to either
-/// two-address elimination or inline assembly constraints. Returns the
-/// first tied use operand index by reference is UseOpIdx is not null.
-bool MachineInstr::
-isRegTiedToUseOperand(unsigned DefOpIdx, unsigned *UseOpIdx) const {
-  if (isInlineAsm()) {
-    assert(DefOpIdx > InlineAsm::MIOp_FirstOperand);
-    const MachineOperand &MO = getOperand(DefOpIdx);
-    if (!MO.isReg() || !MO.isDef() || MO.getReg() == 0)
-      return false;
-    // Determine the actual operand index that corresponds to this index.
-    unsigned DefNo = 0;
-    int FlagIdx = findInlineAsmFlagIdx(DefOpIdx, &DefNo);
-    if (FlagIdx < 0)
-      return false;
+// MachineOperand::TiedTo is 4 bits wide.
+const unsigned TiedMax = 15;
 
-    // Which part of the group is DefOpIdx?
-    unsigned DefPart = DefOpIdx - (FlagIdx + 1);
-
-    for (unsigned i = InlineAsm::MIOp_FirstOperand, e = getNumOperands();
-         i != e; ++i) {
-      const MachineOperand &FMO = getOperand(i);
-      if (!FMO.isImm())
-        continue;
-      if (i+1 >= e || !getOperand(i+1).isReg() || !getOperand(i+1).isUse())
-        continue;
-      unsigned Idx;
-      if (InlineAsm::isUseOperandTiedToDef(FMO.getImm(), Idx) &&
-          Idx == DefNo) {
-        if (UseOpIdx)
-          *UseOpIdx = (unsigned)i + 1 + DefPart;
-        return true;
-      }
-    }
-    return false;
+/// tieOperands - Mark operands at DefIdx and UseIdx as tied to each other.
+///
+/// Use and def operands can be tied together, indicated by a non-zero TiedTo
+/// field. TiedTo can have these values:
+///
+/// 0:              Operand is not tied to anything.
+/// 1 to TiedMax-1: Tied to getOperand(TiedTo-1).
+/// TiedMax:        Tied to an operand >= TiedMax-1.
+///
+/// The tied def must be one of the first TiedMax operands on a normal
+/// instruction. INLINEASM instructions allow more tied defs.
+///
+void MachineInstr::tieOperands(unsigned DefIdx, unsigned UseIdx) {
+  MachineOperand &DefMO = getOperand(DefIdx);
+  MachineOperand &UseMO = getOperand(UseIdx);
+  assert(DefMO.isDef() && "DefIdx must be a def operand");
+  assert(UseMO.isUse() && "UseIdx must be a use operand");
+  assert(!DefMO.isTied() && "Def is already tied to another use");
+  assert(!UseMO.isTied() && "Use is already tied to another def");
+
+  if (DefIdx < TiedMax)
+    UseMO.TiedTo = DefIdx + 1;
+  else {
+    // Inline asm can use the group descriptors to find tied operands, but on
+    // normal instruction, the tied def must be within the first TiedMax
+    // operands.
+    assert(isInlineAsm() && "DefIdx out of range");
+    UseMO.TiedTo = TiedMax;
   }
 
-  assert(getOperand(DefOpIdx).isDef() && "DefOpIdx is not a def!");
-  const MCInstrDesc &MCID = getDesc();
-  for (unsigned i = 0, e = MCID.getNumOperands(); i != e; ++i) {
-    const MachineOperand &MO = getOperand(i);
-    if (MO.isReg() && MO.isUse() &&
-        MCID.getOperandConstraint(i, MCOI::TIED_TO) == (int)DefOpIdx) {
-      if (UseOpIdx)
-        *UseOpIdx = (unsigned)i;
-      return true;
-    }
-  }
-  return false;
+  // UseIdx can be out of range, we'll search for it in findTiedOperandIdx().
+  DefMO.TiedTo = std::min(UseIdx + 1, TiedMax);
 }
 
-/// isRegTiedToDefOperand - Return true if the operand of the specified index
-/// is a register use and it is tied to an def operand. It also returns the def
-/// operand index by reference.
-bool MachineInstr::
-isRegTiedToDefOperand(unsigned UseOpIdx, unsigned *DefOpIdx) const {
-  if (isInlineAsm()) {
-    const MachineOperand &MO = getOperand(UseOpIdx);
-    if (!MO.isReg() || !MO.isUse() || MO.getReg() == 0)
-      return false;
+/// Given the index of a tied register operand, find the operand it is tied to.
+/// Defs are tied to uses and vice versa. Returns the index of the tied operand
+/// which must exist.
+unsigned MachineInstr::findTiedOperandIdx(unsigned OpIdx) const {
+  const MachineOperand &MO = getOperand(OpIdx);
+  assert(MO.isTied() && "Operand isn't tied");
 
-    // Find the flag operand corresponding to UseOpIdx
-    int FlagIdx = findInlineAsmFlagIdx(UseOpIdx);
-    if (FlagIdx < 0)
-      return false;
+  // Normally TiedTo is in range.
+  if (MO.TiedTo < TiedMax)
+    return MO.TiedTo - 1;
 
-    const MachineOperand &UFMO = getOperand(FlagIdx);
-    unsigned DefNo;
-    if (InlineAsm::isUseOperandTiedToDef(UFMO.getImm(), DefNo)) {
-      if (!DefOpIdx)
-        return true;
-
-      unsigned DefIdx = InlineAsm::MIOp_FirstOperand;
-      // Remember to adjust the index. First operand is asm string, second is
-      // the HasSideEffects and AlignStack bits, then there is a flag for each.
-      while (DefNo) {
-        const MachineOperand &FMO = getOperand(DefIdx);
-        assert(FMO.isImm());
-        // Skip over this def.
-        DefIdx += InlineAsm::getNumOperandRegisters(FMO.getImm()) + 1;
-        --DefNo;
-      }
-      *DefOpIdx = DefIdx + UseOpIdx - FlagIdx;
-      return true;
+  // Uses on normal instructions can be out of range.
+  if (!isInlineAsm()) {
+    // Normal tied defs must be in the 0..TiedMax-1 range.
+    if (MO.isUse())
+      return TiedMax - 1;
+    // MO is a def. Search for the tied use.
+    for (unsigned i = TiedMax - 1, e = getNumOperands(); i != e; ++i) {
+      const MachineOperand &UseMO = getOperand(i);
+      if (UseMO.isReg() && UseMO.isUse() && UseMO.TiedTo == OpIdx + 1)
+        return i;
     }
-    return false;
+    llvm_unreachable("Can't find tied use");
   }
 
-  const MCInstrDesc &MCID = getDesc();
-  if (UseOpIdx >= MCID.getNumOperands())
-    return false;
-  const MachineOperand &MO = getOperand(UseOpIdx);
-  if (!MO.isReg() || !MO.isUse())
-    return false;
-  int DefIdx = MCID.getOperandConstraint(UseOpIdx, MCOI::TIED_TO);
-  if (DefIdx == -1)
-    return false;
-  if (DefOpIdx)
-    *DefOpIdx = (unsigned)DefIdx;
-  return true;
+  // Now deal with inline asm by parsing the operand group descriptor flags.
+  // Find the beginning of each operand group.
+  SmallVector<unsigned, 8> GroupIdx;
+  unsigned OpIdxGroup = ~0u;
+  unsigned NumOps;
+  for (unsigned i = InlineAsm::MIOp_FirstOperand, e = getNumOperands(); i < e;
+       i += NumOps) {
+    const MachineOperand &FlagMO = getOperand(i);
+    assert(FlagMO.isImm() && "Invalid tied operand on inline asm");
+    unsigned CurGroup = GroupIdx.size();
+    GroupIdx.push_back(i);
+    NumOps = 1 + InlineAsm::getNumOperandRegisters(FlagMO.getImm());
+    // OpIdx belongs to this operand group.
+    if (OpIdx > i && OpIdx < i + NumOps)
+      OpIdxGroup = CurGroup;
+    unsigned TiedGroup;
+    if (!InlineAsm::isUseOperandTiedToDef(FlagMO.getImm(), TiedGroup))
+      continue;
+    // Operands in this group are tied to operands in TiedGroup which must be
+    // earlier. Find the number of operands between the two groups.
+    unsigned Delta = i - GroupIdx[TiedGroup];
+
+    // OpIdx is a use tied to TiedGroup.
+    if (OpIdxGroup == CurGroup)
+      return OpIdx - Delta;
+
+    // OpIdx is a def tied to this use group.
+    if (OpIdxGroup == TiedGroup)
+      return OpIdx + Delta;
+  }
+  llvm_unreachable("Invalid tied operand on inline asm");
 }
 
 /// clearKillInfo - Clears kill flags on all operands.
@@ -1292,7 +1296,12 @@ bool MachineInstr::isSafeToMove(const TargetInstrInfo *TII,
                                 AliasAnalysis *AA,
                                 bool &SawStore) const {
   // Ignore stuff that we obviously can't move.
-  if (mayStore() || isCall()) {
+  //
+  // Treat volatile loads as stores. This is not strictly necessary for
+  // volatiles, but it is required for atomic loads. It is not allowed to move
+  // a load across an atomic load with Ordering > Monotonic.
+  if (mayStore() || isCall() ||
+      (mayLoad() && hasOrderedMemoryRef())) {
     SawStore = true;
     return false;
   }
@@ -1308,8 +1317,8 @@ bool MachineInstr::isSafeToMove(const TargetInstrInfo *TII,
   // load.
   if (mayLoad() && !isInvariantLoad(AA))
     // Otherwise, this is a real load.  If there is a store between the load and
-    // end of block, or if the load is volatile, we can't move it.
-    return !SawStore && !hasVolatileMemoryRef();
+    // end of block, we can't move it.
+    return !SawStore;
 
   return true;
 }
@@ -1340,11 +1349,11 @@ bool MachineInstr::isSafeToReMat(const TargetInstrInfo *TII,
   return true;
 }
 
-/// hasVolatileMemoryRef - Return true if this instruction may have a
-/// volatile memory reference, or if the information describing the
-/// memory reference is not available. Return false if it is known to
-/// have no volatile memory references.
-bool MachineInstr::hasVolatileMemoryRef() const {
+/// hasOrderedMemoryRef - Return true if this instruction may have an ordered
+/// or volatile memory reference, or if the information describing the memory
+/// reference is not available. Return false if it is known to have no ordered
+/// memory references.
+bool MachineInstr::hasOrderedMemoryRef() const {
   // An instruction known never to access memory won't have a volatile access.
   if (!mayStore() &&
       !mayLoad() &&
@@ -1357,9 +1366,9 @@ bool MachineInstr::hasVolatileMemoryRef() const {
   if (memoperands_empty())
     return true;
 
-  // Check the memory reference information for volatile references.
+  // Check the memory reference information for ordered references.
   for (mmo_iterator I = memoperands_begin(), E = memoperands_end(); I != E; ++I)
-    if ((*I)->isVolatile())
+    if (!(*I)->isUnordered())
       return true;
 
   return false;
@@ -1461,7 +1470,9 @@ void MachineInstr::copyImplicitOps(const MachineInstr *MI) {
 }
 
 void MachineInstr::dump() const {
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   dbgs() << "  " << *this;
+#endif
 }
 
 static void printDebugLoc(DebugLoc DL, const MachineFunction *MF,
@@ -1540,6 +1551,10 @@ void MachineInstr::print(raw_ostream &OS, const TargetMachine *TM) const {
       OS << " [sideeffect]";
     if (ExtraInfo & InlineAsm::Extra_IsAlignStack)
       OS << " [alignstack]";
+    if (getInlineAsmDialect() == InlineAsm::AD_ATT)
+      OS << " [attdialect]";
+    if (getInlineAsmDialect() == InlineAsm::AD_Intel)
+      OS << " [inteldialect]";
 
     StartOp = AsmDescOp = InlineAsm::MIOp_FirstOperand;
     FirstOp = false;
diff --git a/lib/CodeGen/MachineInstrBundle.cpp b/lib/CodeGen/MachineInstrBundle.cpp
index b7de7bf..1f7fbfc 100644
--- a/lib/CodeGen/MachineInstrBundle.cpp
+++ b/lib/CodeGen/MachineInstrBundle.cpp
@@ -109,10 +109,10 @@ void llvm::finalizeBundle(MachineBasicBlock &MBB,
   MachineInstrBuilder MIB = BuildMI(MBB, FirstMI, FirstMI->getDebugLoc(),
                                     TII->get(TargetOpcode::BUNDLE));
 
-  SmallVector<unsigned, 8> LocalDefs;
-  SmallSet<unsigned, 8> LocalDefSet;
+  SmallVector<unsigned, 32> LocalDefs;
+  SmallSet<unsigned, 32> LocalDefSet;
   SmallSet<unsigned, 8> DeadDefSet;
-  SmallSet<unsigned, 8> KilledDefSet;
+  SmallSet<unsigned, 16> KilledDefSet;
   SmallVector<unsigned, 8> ExternUses;
   SmallSet<unsigned, 8> ExternUseSet;
   SmallSet<unsigned, 8> KilledUseSet;
@@ -181,7 +181,7 @@ void llvm::finalizeBundle(MachineBasicBlock &MBB,
     Defs.clear();
   }
 
-  SmallSet<unsigned, 8> Added;
+  SmallSet<unsigned, 32> Added;
   for (unsigned i = 0, e = LocalDefs.size(); i != e; ++i) {
     unsigned Reg = LocalDefs[i];
     if (Added.insert(Reg)) {
@@ -248,10 +248,10 @@ bool llvm::finalizeBundles(MachineFunction &MF) {
 // MachineOperand iterator
 //===----------------------------------------------------------------------===//
 
-MachineOperandIteratorBase::RegInfo
+MachineOperandIteratorBase::VirtRegInfo
 MachineOperandIteratorBase::analyzeVirtReg(unsigned Reg,
                     SmallVectorImpl<std::pair<MachineInstr*, unsigned> > *Ops) {
-  RegInfo RI = { false, false, false };
+  VirtRegInfo RI = { false, false, false };
   for(; isValid(); ++*this) {
     MachineOperand &MO = deref();
     if (!MO.isReg() || MO.getReg() != Reg)
@@ -276,3 +276,53 @@ MachineOperandIteratorBase::analyzeVirtReg(unsigned Reg,
   }
   return RI;
 }
+
+MachineOperandIteratorBase::PhysRegInfo
+MachineOperandIteratorBase::analyzePhysReg(unsigned Reg,
+                                           const TargetRegisterInfo *TRI) {
+  bool AllDefsDead = true;
+  PhysRegInfo PRI = {false, false, false, false, false, false, false};
+
+  assert(TargetRegisterInfo::isPhysicalRegister(Reg) &&
+         "analyzePhysReg not given a physical register!");
+  for (; isValid(); ++*this) {
+    MachineOperand &MO = deref();
+
+    if (MO.isRegMask() && MO.clobbersPhysReg(Reg))
+      PRI.Clobbers = true;    // Regmask clobbers Reg.
+
+    if (!MO.isReg())
+      continue;
+
+    unsigned MOReg = MO.getReg();
+    if (!MOReg || !TargetRegisterInfo::isPhysicalRegister(MOReg))
+      continue;
+
+    bool IsRegOrSuperReg = MOReg == Reg || TRI->isSubRegister(MOReg, Reg);
+    bool IsRegOrOverlapping = MOReg == Reg || TRI->regsOverlap(MOReg, Reg);
+
+    if (IsRegOrSuperReg && MO.readsReg()) {
+      // Reg or a super-reg is read, and perhaps killed also.
+      PRI.Reads = true;
+      PRI.Kills = MO.isKill();
+    } if (IsRegOrOverlapping && MO.readsReg()) {
+      PRI.ReadsOverlap = true;// Reg or an overlapping register is read.
+    }
+
+    if (!MO.isDef())
+      continue;
+
+    if (IsRegOrSuperReg) {
+      PRI.Defines = true;     // Reg or a super-register is defined.
+      if (!MO.isDead())
+        AllDefsDead = false;
+    }
+    if (IsRegOrOverlapping)
+      PRI.Clobbers = true;    // Reg or an overlapping reg is defined.
+  }
+
+  if (AllDefsDead && PRI.Defines)
+    PRI.DefinesDead = true;   // Reg or super-register was defined and was dead.
+
+  return PRI;
+}
diff --git a/lib/CodeGen/MachineLICM.cpp b/lib/CodeGen/MachineLICM.cpp
index efec481..169443e 100644
--- a/lib/CodeGen/MachineLICM.cpp
+++ b/lib/CodeGen/MachineLICM.cpp
@@ -334,7 +334,7 @@ bool MachineLICM::runOnMachineFunction(MachineFunction &MF) {
     DEBUG(dbgs() << "******** Pre-regalloc Machine LICM: ");
   else
     DEBUG(dbgs() << "******** Post-regalloc Machine LICM: ");
-  DEBUG(dbgs() << MF.getFunction()->getName() << " ********\n");
+  DEBUG(dbgs() << MF.getName() << " ********\n");
 
   if (PreRegAlloc) {
     // Estimate register pressure during pre-regalloc pass.
diff --git a/lib/CodeGen/MachineLoopInfo.cpp b/lib/CodeGen/MachineLoopInfo.cpp
index 9f3829e..27afeec 100644
--- a/lib/CodeGen/MachineLoopInfo.cpp
+++ b/lib/CodeGen/MachineLoopInfo.cpp
@@ -74,6 +74,8 @@ MachineBasicBlock *MachineLoop::getBottomBlock() {
   return BotMBB;
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void MachineLoop::dump() const {
   print(dbgs());
 }
+#endif
diff --git a/lib/CodeGen/MachineModuleInfo.cpp b/lib/CodeGen/MachineModuleInfo.cpp
index ea98b23..005bf78 100644
--- a/lib/CodeGen/MachineModuleInfo.cpp
+++ b/lib/CodeGen/MachineModuleInfo.cpp
@@ -25,7 +25,7 @@
 using namespace llvm;
 using namespace llvm::dwarf;
 
-// Handle the Pass registration stuff necessary to use TargetData's.
+// Handle the Pass registration stuff necessary to use DataLayout's.
 INITIALIZE_PASS(MachineModuleInfo, "machinemoduleinfo",
                 "Machine Module Information", false, false)
 char MachineModuleInfo::ID = 0;
diff --git a/lib/CodeGen/MachineModuleInfoImpls.cpp b/lib/CodeGen/MachineModuleInfoImpls.cpp
index 5ab56c0..a1c7e9f 100644
--- a/lib/CodeGen/MachineModuleInfoImpls.cpp
+++ b/lib/CodeGen/MachineModuleInfoImpls.cpp
@@ -21,8 +21,8 @@ using namespace llvm;
 //===----------------------------------------------------------------------===//
 
 // Out of line virtual method.
-void MachineModuleInfoMachO::Anchor() {}
-void MachineModuleInfoELF::Anchor() {}
+void MachineModuleInfoMachO::anchor() {}
+void MachineModuleInfoELF::anchor() {}
 
 static int SortSymbolPair(const void *LHS, const void *RHS) {
   typedef std::pair<MCSymbol*, MachineModuleInfoImpl::StubValueTy> PairTy;
diff --git a/lib/CodeGen/MachinePostDominators.cpp b/lib/CodeGen/MachinePostDominators.cpp
new file mode 100644
index 0000000..c3f6e92
--- /dev/null
+++ b/lib/CodeGen/MachinePostDominators.cpp
@@ -0,0 +1,55 @@
+//===- MachinePostDominators.cpp -Machine Post Dominator Calculation ------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements simple dominator construction algorithms for finding
+// post dominators on machine functions.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGen/MachinePostDominators.h"
+
+using namespace llvm;
+
+char MachinePostDominatorTree::ID = 0;
+
+//declare initializeMachinePostDominatorTreePass
+INITIALIZE_PASS(MachinePostDominatorTree, "machinepostdomtree",
+                "MachinePostDominator Tree Construction", true, true)
+
+MachinePostDominatorTree::MachinePostDominatorTree() : MachineFunctionPass(ID) {
+  initializeMachinePostDominatorTreePass(*PassRegistry::getPassRegistry());
+  DT = new DominatorTreeBase<MachineBasicBlock>(true); //true indicate
+                                                       // postdominator
+}
+
+FunctionPass *
+MachinePostDominatorTree::createMachinePostDominatorTreePass() {
+  return new MachinePostDominatorTree();
+}
+
+bool
+MachinePostDominatorTree::runOnMachineFunction(MachineFunction &F) {
+  DT->recalculate(F);
+  return false;
+}
+
+MachinePostDominatorTree::~MachinePostDominatorTree() {
+  delete DT;
+}
+
+void
+MachinePostDominatorTree::getAnalysisUsage(AnalysisUsage &AU) const {
+  AU.setPreservesAll();
+  MachineFunctionPass::getAnalysisUsage(AU);
+}
+
+void
+MachinePostDominatorTree::print(llvm::raw_ostream &OS, const Module *M) const {
+  DT->print(OS);
+}
diff --git a/lib/CodeGen/MachineRegisterInfo.cpp b/lib/CodeGen/MachineRegisterInfo.cpp
index 5fb938f..95d7a7d 100644
--- a/lib/CodeGen/MachineRegisterInfo.cpp
+++ b/lib/CodeGen/MachineRegisterInfo.cpp
@@ -21,7 +21,7 @@ MachineRegisterInfo::MachineRegisterInfo(const TargetRegisterInfo &TRI)
   : TRI(&TRI), IsSSA(true), TracksLiveness(true) {
   VRegInfo.reserve(256);
   RegAllocHints.reserve(256);
-  UsedPhysRegs.resize(TRI.getNumRegs());
+  UsedRegUnits.resize(TRI.getNumRegUnits());
   UsedPhysRegMask.resize(TRI.getNumRegs());
 
   // Create the physreg use/def lists.
@@ -32,7 +32,7 @@ MachineRegisterInfo::MachineRegisterInfo(const TargetRegisterInfo &TRI)
 MachineRegisterInfo::~MachineRegisterInfo() {
 #ifndef NDEBUG
   clearVirtRegs();
-  for (unsigned i = 0, e = UsedPhysRegs.size(); i != e; ++i)
+  for (unsigned i = 0, e = TRI->getNumRegs(); i != e; ++i)
     assert(!PhysRegUseDefLists[i] &&
            "PhysRegUseDefLists has entries after all instructions are deleted");
 #endif
@@ -306,22 +306,18 @@ void MachineRegisterInfo::dumpUses(unsigned Reg) const {
 
 void MachineRegisterInfo::freezeReservedRegs(const MachineFunction &MF) {
   ReservedRegs = TRI->getReservedRegs(MF);
+  assert(ReservedRegs.size() == TRI->getNumRegs() &&
+         "Invalid ReservedRegs vector from target");
 }
 
 bool MachineRegisterInfo::isConstantPhysReg(unsigned PhysReg,
                                             const MachineFunction &MF) const {
   assert(TargetRegisterInfo::isPhysicalRegister(PhysReg));
 
-  // Check if any overlapping register is modified.
+  // Check if any overlapping register is modified, or allocatable so it may be
+  // used later.
   for (MCRegAliasIterator AI(PhysReg, TRI, true); AI.isValid(); ++AI)
-    if (!def_empty(*AI))
-      return false;
-
-  // Check if any overlapping register is allocatable so it may be used later.
-  if (AllocatableRegs.empty())
-    AllocatableRegs = TRI->getAllocatableSet(MF);
-  for (MCRegAliasIterator AI(PhysReg, TRI, true); AI.isValid(); ++AI)
-    if (AllocatableRegs.test(*AI))
+    if (!def_empty(*AI) || isAllocatable(*AI))
       return false;
   return true;
 }
diff --git a/lib/CodeGen/MachineScheduler.cpp b/lib/CodeGen/MachineScheduler.cpp
index a1dc948..a4817d0 100644
--- a/lib/CodeGen/MachineScheduler.cpp
+++ b/lib/CodeGen/MachineScheduler.cpp
@@ -18,11 +18,8 @@
 #include "llvm/CodeGen/MachineScheduler.h"
 #include "llvm/CodeGen/Passes.h"
 #include "llvm/CodeGen/RegisterClassInfo.h"
-#include "llvm/CodeGen/RegisterPressure.h"
-#include "llvm/CodeGen/ScheduleDAGInstrs.h"
+#include "llvm/CodeGen/ScheduleDAGILP.h"
 #include "llvm/CodeGen/ScheduleHazardRecognizer.h"
-#include "llvm/Target/TargetInstrInfo.h"
-#include "llvm/MC/MCInstrItineraries.h"
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
@@ -35,10 +32,12 @@
 
 using namespace llvm;
 
-static cl::opt<bool> ForceTopDown("misched-topdown", cl::Hidden,
-                                  cl::desc("Force top-down list scheduling"));
-static cl::opt<bool> ForceBottomUp("misched-bottomup", cl::Hidden,
-                                  cl::desc("Force bottom-up list scheduling"));
+namespace llvm {
+cl::opt<bool> ForceTopDown("misched-topdown", cl::Hidden,
+                           cl::desc("Force top-down list scheduling"));
+cl::opt<bool> ForceBottomUp("misched-bottomup", cl::Hidden,
+                            cl::desc("Force bottom-up list scheduling"));
+}
 
 #ifndef NDEBUG
 static cl::opt<bool> ViewMISchedDAGs("view-misched-dags", cl::Hidden,
@@ -50,6 +49,15 @@ static cl::opt<unsigned> MISchedCutoff("misched-cutoff", cl::Hidden,
 static bool ViewMISchedDAGs = false;
 #endif // NDEBUG
 
+// Threshold to very roughly model an out-of-order processor's instruction
+// buffers. If the actual value of this threshold matters much in practice, then
+// it can be specified by the machine model. For now, it's an experimental
+// tuning knob to determine when and if it matters.
+static cl::opt<unsigned> ILPWindow("ilp-window", cl::Hidden,
+  cl::desc("Allow expected latency to exceed the critical path by N cycles "
+           "before attempting to balance ILP"),
+  cl::init(10U));
+
 //===----------------------------------------------------------------------===//
 // Machine Instruction Scheduling Pass and Registry
 //===----------------------------------------------------------------------===//
@@ -221,7 +229,7 @@ bool MachineScheduler::runOnMachineFunction(MachineFunction &mf) {
     // The Scheduler may insert instructions during either schedule() or
     // exitRegion(), even for empty regions. So the local iterators 'I' and
     // 'RegionEnd' are invalid across these calls.
-    unsigned RemainingCount = MBB->size();
+    unsigned RemainingInstrs = MBB->size();
     for(MachineBasicBlock::iterator RegionEnd = MBB->end();
         RegionEnd != MBB->begin(); RegionEnd = Scheduler->begin()) {
 
@@ -230,19 +238,19 @@ bool MachineScheduler::runOnMachineFunction(MachineFunction &mf) {
           || TII->isSchedulingBoundary(llvm::prior(RegionEnd), MBB, *MF)) {
         --RegionEnd;
         // Count the boundary instruction.
-        --RemainingCount;
+        --RemainingInstrs;
       }
 
       // The next region starts above the previous region. Look backward in the
       // instruction stream until we find the nearest boundary.
       MachineBasicBlock::iterator I = RegionEnd;
-      for(;I != MBB->begin(); --I, --RemainingCount) {
+      for(;I != MBB->begin(); --I, --RemainingInstrs) {
         if (TII->isSchedulingBoundary(llvm::prior(I), MBB, *MF))
           break;
       }
       // Notify the scheduler of the region, even if we may skip scheduling
       // it. Perhaps it still needs to be bundled.
-      Scheduler->enterRegion(MBB, I, RegionEnd, RemainingCount);
+      Scheduler->enterRegion(MBB, I, RegionEnd, RemainingInstrs);
 
       // Skip empty scheduling regions (0 or 1 schedulable instructions).
       if (I == RegionEnd || I == llvm::prior(RegionEnd)) {
@@ -252,11 +260,11 @@ bool MachineScheduler::runOnMachineFunction(MachineFunction &mf) {
         continue;
       }
       DEBUG(dbgs() << "********** MI Scheduling **********\n");
-      DEBUG(dbgs() << MF->getFunction()->getName()
+      DEBUG(dbgs() << MF->getName()
             << ":BB#" << MBB->getNumber() << "\n  From: " << *I << "    To: ";
             if (RegionEnd != MBB->end()) dbgs() << *RegionEnd;
             else dbgs() << "End";
-            dbgs() << " Remaining: " << RemainingCount << "\n");
+            dbgs() << " Remaining: " << RemainingInstrs << "\n");
 
       // Schedule a region: possibly reorder instructions.
       // This invalidates 'RegionEnd' and 'I'.
@@ -269,7 +277,7 @@ bool MachineScheduler::runOnMachineFunction(MachineFunction &mf) {
       // scheduler for the top of it's scheduled region.
       RegionEnd = Scheduler->begin();
     }
-    assert(RemainingCount == 0 && "Instruction count mismatch!");
+    assert(RemainingInstrs == 0 && "Instruction count mismatch!");
     Scheduler->finishBlock();
   }
   Scheduler->finalizeSchedule();
@@ -281,157 +289,20 @@ void MachineScheduler::print(raw_ostream &O, const Module* m) const {
   // unimplemented
 }
 
-//===----------------------------------------------------------------------===//
-// MachineSchedStrategy - Interface to a machine scheduling algorithm.
-//===----------------------------------------------------------------------===//
-
-namespace {
-class ScheduleDAGMI;
-
-/// MachineSchedStrategy - Interface used by ScheduleDAGMI to drive the selected
-/// scheduling algorithm.
-///
-/// If this works well and targets wish to reuse ScheduleDAGMI, we may expose it
-/// in ScheduleDAGInstrs.h
-class MachineSchedStrategy {
-public:
-  virtual ~MachineSchedStrategy() {}
-
-  /// Initialize the strategy after building the DAG for a new region.
-  virtual void initialize(ScheduleDAGMI *DAG) = 0;
-
-  /// Pick the next node to schedule, or return NULL. Set IsTopNode to true to
-  /// schedule the node at the top of the unscheduled region. Otherwise it will
-  /// be scheduled at the bottom.
-  virtual SUnit *pickNode(bool &IsTopNode) = 0;
-
-  /// Notify MachineSchedStrategy that ScheduleDAGMI has scheduled a node.
-  virtual void schedNode(SUnit *SU, bool IsTopNode) = 0;
-
-  /// When all predecessor dependencies have been resolved, free this node for
-  /// top-down scheduling.
-  virtual void releaseTopNode(SUnit *SU) = 0;
-  /// When all successor dependencies have been resolved, free this node for
-  /// bottom-up scheduling.
-  virtual void releaseBottomNode(SUnit *SU) = 0;
-};
-} // namespace
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+void ReadyQueue::dump() {
+  dbgs() << Name << ": ";
+  for (unsigned i = 0, e = Queue.size(); i < e; ++i)
+    dbgs() << Queue[i]->NodeNum << " ";
+  dbgs() << "\n";
+}
+#endif
 
 //===----------------------------------------------------------------------===//
 // ScheduleDAGMI - Base class for MachineInstr scheduling with LiveIntervals
 // preservation.
 //===----------------------------------------------------------------------===//
 
-namespace {
-/// ScheduleDAGMI is an implementation of ScheduleDAGInstrs that schedules
-/// machine instructions while updating LiveIntervals.
-class ScheduleDAGMI : public ScheduleDAGInstrs {
-  AliasAnalysis *AA;
-  RegisterClassInfo *RegClassInfo;
-  MachineSchedStrategy *SchedImpl;
-
-  MachineBasicBlock::iterator LiveRegionEnd;
-
-  /// Register pressure in this region computed by buildSchedGraph.
-  IntervalPressure RegPressure;
-  RegPressureTracker RPTracker;
-
-  /// List of pressure sets that exceed the target's pressure limit before
-  /// scheduling, listed in increasing set ID order. Each pressure set is paired
-  /// with its max pressure in the currently scheduled regions.
-  std::vector<PressureElement> RegionCriticalPSets;
-
-  /// The top of the unscheduled zone.
-  MachineBasicBlock::iterator CurrentTop;
-  IntervalPressure TopPressure;
-  RegPressureTracker TopRPTracker;
-
-  /// The bottom of the unscheduled zone.
-  MachineBasicBlock::iterator CurrentBottom;
-  IntervalPressure BotPressure;
-  RegPressureTracker BotRPTracker;
-
-#ifndef NDEBUG
-  /// The number of instructions scheduled so far. Used to cut off the
-  /// scheduler at the point determined by misched-cutoff.
-  unsigned NumInstrsScheduled;
-#endif
-public:
-  ScheduleDAGMI(MachineSchedContext *C, MachineSchedStrategy *S):
-    ScheduleDAGInstrs(*C->MF, *C->MLI, *C->MDT, /*IsPostRA=*/false, C->LIS),
-    AA(C->AA), RegClassInfo(C->RegClassInfo), SchedImpl(S),
-    RPTracker(RegPressure), CurrentTop(), TopRPTracker(TopPressure),
-    CurrentBottom(), BotRPTracker(BotPressure) {
-#ifndef NDEBUG
-    NumInstrsScheduled = 0;
-#endif
-  }
-
-  ~ScheduleDAGMI() {
-    delete SchedImpl;
-  }
-
-  MachineBasicBlock::iterator top() const { return CurrentTop; }
-  MachineBasicBlock::iterator bottom() const { return CurrentBottom; }
-
-  /// Implement the ScheduleDAGInstrs interface for handling the next scheduling
-  /// region. This covers all instructions in a block, while schedule() may only
-  /// cover a subset.
-  void enterRegion(MachineBasicBlock *bb,
-                   MachineBasicBlock::iterator begin,
-                   MachineBasicBlock::iterator end,
-                   unsigned endcount);
-
-  /// Implement ScheduleDAGInstrs interface for scheduling a sequence of
-  /// reorderable instructions.
-  void schedule();
-
-  /// Get current register pressure for the top scheduled instructions.
-  const IntervalPressure &getTopPressure() const { return TopPressure; }
-  const RegPressureTracker &getTopRPTracker() const { return TopRPTracker; }
-
-  /// Get current register pressure for the bottom scheduled instructions.
-  const IntervalPressure &getBotPressure() const { return BotPressure; }
-  const RegPressureTracker &getBotRPTracker() const { return BotRPTracker; }
-
-  /// Get register pressure for the entire scheduling region before scheduling.
-  const IntervalPressure &getRegPressure() const { return RegPressure; }
-
-  const std::vector<PressureElement> &getRegionCriticalPSets() const {
-    return RegionCriticalPSets;
-  }
-
-  /// getIssueWidth - Return the max instructions per scheduling group.
-  unsigned getIssueWidth() const {
-    return (InstrItins && InstrItins->SchedModel)
-      ? InstrItins->SchedModel->IssueWidth : 1;
-  }
-
-  /// getNumMicroOps - Return the number of issue slots required for this MI.
-  unsigned getNumMicroOps(MachineInstr *MI) const {
-    if (!InstrItins) return 1;
-    int UOps = InstrItins->getNumMicroOps(MI->getDesc().getSchedClass());
-    return (UOps >= 0) ? UOps : TII->getNumMicroOps(InstrItins, MI);
-  }
-
-protected:
-  void initRegPressure();
-  void updateScheduledPressure(std::vector<unsigned> NewMaxPressure);
-
-  void moveInstruction(MachineInstr *MI, MachineBasicBlock::iterator InsertPos);
-  bool checkSchedLimit();
-
-  void releaseRoots();
-
-  void releaseSucc(SUnit *SU, SDep *SuccEdge);
-  void releaseSuccessors(SUnit *SU);
-  void releasePred(SUnit *SU, SDep *PredEdge);
-  void releasePredecessors(SUnit *SU);
-
-  void placeDebugValues();
-};
-} // namespace
-
 /// ReleaseSucc - Decrement the NumPredsLeft count of a successor. When
 /// NumPredsLeft reaches zero, release the successor node.
 ///
@@ -498,7 +369,7 @@ void ScheduleDAGMI::moveInstruction(MachineInstr *MI,
   BB->splice(InsertPos, BB, MI);
 
   // Update LiveIntervals
-  LIS->handleMove(MI);
+  LIS->handleMove(MI, /*UpdateFlags=*/true);
 
   // Recede RegionBegin if an instruction moves above the first.
   if (RegionBegin == InsertPos)
@@ -565,6 +436,9 @@ void ScheduleDAGMI::initRegPressure() {
   std::vector<unsigned> RegionPressure = RPTracker.getPressure().MaxSetPressure;
   for (unsigned i = 0, e = RegionPressure.size(); i < e; ++i) {
     unsigned Limit = TRI->getRegPressureSetLimit(i);
+    DEBUG(dbgs() << TRI->getRegPressureSetName(i)
+          << "Limit " << Limit
+          << " Actual " << RegionPressure[i] << "\n");
     if (RegionPressure[i] > Limit)
       RegionCriticalPSets.push_back(PressureElement(i, 0));
   }
@@ -587,6 +461,74 @@ updateScheduledPressure(std::vector<unsigned> NewMaxPressure) {
   }
 }
 
+/// schedule - Called back from MachineScheduler::runOnMachineFunction
+/// after setting up the current scheduling region. [RegionBegin, RegionEnd)
+/// only includes instructions that have DAG nodes, not scheduling boundaries.
+///
+/// This is a skeletal driver, with all the functionality pushed into helpers,
+/// so that it can be easilly extended by experimental schedulers. Generally,
+/// implementing MachineSchedStrategy should be sufficient to implement a new
+/// scheduling algorithm. However, if a scheduler further subclasses
+/// ScheduleDAGMI then it will want to override this virtual method in order to
+/// update any specialized state.
+void ScheduleDAGMI::schedule() {
+  buildDAGWithRegPressure();
+
+  postprocessDAG();
+
+  DEBUG(for (unsigned su = 0, e = SUnits.size(); su != e; ++su)
+          SUnits[su].dumpAll(this));
+
+  if (ViewMISchedDAGs) viewGraph();
+
+  initQueues();
+
+  bool IsTopNode = false;
+  while (SUnit *SU = SchedImpl->pickNode(IsTopNode)) {
+    assert(!SU->isScheduled && "Node already scheduled");
+    if (!checkSchedLimit())
+      break;
+
+    scheduleMI(SU, IsTopNode);
+
+    updateQueues(SU, IsTopNode);
+  }
+  assert(CurrentTop == CurrentBottom && "Nonempty unscheduled zone.");
+
+  placeDebugValues();
+
+  DEBUG({
+      unsigned BBNum = top()->getParent()->getNumber();
+      dbgs() << "*** Final schedule for BB#" << BBNum << " ***\n";
+      dumpSchedule();
+      dbgs() << '\n';
+    });
+}
+
+/// Build the DAG and setup three register pressure trackers.
+void ScheduleDAGMI::buildDAGWithRegPressure() {
+  // Initialize the register pressure tracker used by buildSchedGraph.
+  RPTracker.init(&MF, RegClassInfo, LIS, BB, LiveRegionEnd);
+
+  // Account for liveness generate by the region boundary.
+  if (LiveRegionEnd != RegionEnd)
+    RPTracker.recede();
+
+  // Build the DAG, and compute current register pressure.
+  buildSchedGraph(AA, &RPTracker);
+  if (ViewMISchedDAGs) viewGraph();
+
+  // Initialize top/bottom trackers after computing region pressure.
+  initRegPressure();
+}
+
+/// Apply each ScheduleDAGMutation step in order.
+void ScheduleDAGMI::postprocessDAG() {
+  for (unsigned i = 0, e = Mutations.size(); i < e; ++i) {
+    Mutations[i]->apply(this);
+  }
+}
+
 // Release all DAG roots for scheduling.
 void ScheduleDAGMI::releaseRoots() {
   SmallVector<SUnit*, 16> BotRoots;
@@ -607,28 +549,10 @@ void ScheduleDAGMI::releaseRoots() {
     SchedImpl->releaseBottomNode(*I);
 }
 
-/// schedule - Called back from MachineScheduler::runOnMachineFunction
-/// after setting up the current scheduling region. [RegionBegin, RegionEnd)
-/// only includes instructions that have DAG nodes, not scheduling boundaries.
-void ScheduleDAGMI::schedule() {
-  // Initialize the register pressure tracker used by buildSchedGraph.
-  RPTracker.init(&MF, RegClassInfo, LIS, BB, LiveRegionEnd);
-
-  // Account for liveness generate by the region boundary.
-  if (LiveRegionEnd != RegionEnd)
-    RPTracker.recede();
-
-  // Build the DAG, and compute current register pressure.
-  buildSchedGraph(AA, &RPTracker);
-
-  // Initialize top/bottom trackers after computing region pressure.
-  initRegPressure();
-
-  DEBUG(for (unsigned su = 0, e = SUnits.size(); su != e; ++su)
-          SUnits[su].dumpAll(this));
-
-  if (ViewMISchedDAGs) viewGraph();
+/// Identify DAG roots and setup scheduler queues.
+void ScheduleDAGMI::initQueues() {
 
+  // Initialize the strategy before modifying the DAG.
   SchedImpl->initialize(this);
 
   // Release edges from the special Entry node or to the special Exit node.
@@ -638,61 +562,64 @@ void ScheduleDAGMI::schedule() {
   // Release all DAG roots for scheduling.
   releaseRoots();
 
+  SchedImpl->registerRoots();
+
   CurrentTop = nextIfDebug(RegionBegin, RegionEnd);
   CurrentBottom = RegionEnd;
-  bool IsTopNode = false;
-  while (SUnit *SU = SchedImpl->pickNode(IsTopNode)) {
-    if (!checkSchedLimit())
-      break;
-
-    // Move the instruction to its new location in the instruction stream.
-    MachineInstr *MI = SU->getInstr();
-
-    if (IsTopNode) {
-      assert(SU->isTopReady() && "node still has unscheduled dependencies");
-      if (&*CurrentTop == MI)
-        CurrentTop = nextIfDebug(++CurrentTop, CurrentBottom);
-      else {
-        moveInstruction(MI, CurrentTop);
-        TopRPTracker.setPos(MI);
-      }
+}
 
-      // Update top scheduled pressure.
-      TopRPTracker.advance();
-      assert(TopRPTracker.getPos() == CurrentTop && "out of sync");
-      updateScheduledPressure(TopRPTracker.getPressure().MaxSetPressure);
+/// Move an instruction and update register pressure.
+void ScheduleDAGMI::scheduleMI(SUnit *SU, bool IsTopNode) {
+  // Move the instruction to its new location in the instruction stream.
+  MachineInstr *MI = SU->getInstr();
 
-      // Release dependent instructions for scheduling.
-      releaseSuccessors(SU);
+  if (IsTopNode) {
+    assert(SU->isTopReady() && "node still has unscheduled dependencies");
+    if (&*CurrentTop == MI)
+      CurrentTop = nextIfDebug(++CurrentTop, CurrentBottom);
+    else {
+      moveInstruction(MI, CurrentTop);
+      TopRPTracker.setPos(MI);
     }
+
+    // Update top scheduled pressure.
+    TopRPTracker.advance();
+    assert(TopRPTracker.getPos() == CurrentTop && "out of sync");
+    updateScheduledPressure(TopRPTracker.getPressure().MaxSetPressure);
+  }
+  else {
+    assert(SU->isBottomReady() && "node still has unscheduled dependencies");
+    MachineBasicBlock::iterator priorII =
+      priorNonDebug(CurrentBottom, CurrentTop);
+    if (&*priorII == MI)
+      CurrentBottom = priorII;
     else {
-      assert(SU->isBottomReady() && "node still has unscheduled dependencies");
-      MachineBasicBlock::iterator priorII =
-        priorNonDebug(CurrentBottom, CurrentTop);
-      if (&*priorII == MI)
-        CurrentBottom = priorII;
-      else {
-        if (&*CurrentTop == MI) {
-          CurrentTop = nextIfDebug(++CurrentTop, priorII);
-          TopRPTracker.setPos(CurrentTop);
-        }
-        moveInstruction(MI, CurrentBottom);
-        CurrentBottom = MI;
+      if (&*CurrentTop == MI) {
+        CurrentTop = nextIfDebug(++CurrentTop, priorII);
+        TopRPTracker.setPos(CurrentTop);
       }
-      // Update bottom scheduled pressure.
-      BotRPTracker.recede();
-      assert(BotRPTracker.getPos() == CurrentBottom && "out of sync");
-      updateScheduledPressure(BotRPTracker.getPressure().MaxSetPressure);
-
-      // Release dependent instructions for scheduling.
-      releasePredecessors(SU);
+      moveInstruction(MI, CurrentBottom);
+      CurrentBottom = MI;
     }
-    SU->isScheduled = true;
-    SchedImpl->schedNode(SU, IsTopNode);
+    // Update bottom scheduled pressure.
+    BotRPTracker.recede();
+    assert(BotRPTracker.getPos() == CurrentBottom && "out of sync");
+    updateScheduledPressure(BotRPTracker.getPressure().MaxSetPressure);
   }
-  assert(CurrentTop == CurrentBottom && "Nonempty unscheduled zone.");
+}
 
-  placeDebugValues();
+/// Update scheduler queues after scheduling an instruction.
+void ScheduleDAGMI::updateQueues(SUnit *SU, bool IsTopNode) {
+  // Release dependent instructions for scheduling.
+  if (IsTopNode)
+    releaseSuccessors(SU);
+  else
+    releasePredecessors(SU);
+
+  SU->isScheduled = true;
+
+  // Notify the scheduling strategy after updating the DAG.
+  SchedImpl->schedNode(SU, IsTopNode);
 }
 
 /// Reinsert any remaining debug_values, just like the PostRA scheduler.
@@ -716,91 +643,146 @@ void ScheduleDAGMI::placeDebugValues() {
   FirstDbgValue = NULL;
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+void ScheduleDAGMI::dumpSchedule() const {
+  for (MachineBasicBlock::iterator MI = begin(), ME = end(); MI != ME; ++MI) {
+    if (SUnit *SU = getSUnit(&(*MI)))
+      SU->dump(this);
+    else
+      dbgs() << "Missing SUnit\n";
+  }
+}
+#endif
+
 //===----------------------------------------------------------------------===//
 // ConvergingScheduler - Implementation of the standard MachineSchedStrategy.
 //===----------------------------------------------------------------------===//
 
 namespace {
-/// ReadyQueue encapsulates vector of "ready" SUnits with basic convenience
-/// methods for pushing and removing nodes. ReadyQueue's are uniquely identified
-/// by an ID. SUnit::NodeQueueId is a mask of the ReadyQueues the SUnit is in.
-class ReadyQueue {
-  unsigned ID;
-  std::string Name;
-  std::vector<SUnit*> Queue;
-
+/// ConvergingScheduler shrinks the unscheduled zone using heuristics to balance
+/// the schedule.
+class ConvergingScheduler : public MachineSchedStrategy {
 public:
-  ReadyQueue(unsigned id, const Twine &name): ID(id), Name(name.str()) {}
-
-  unsigned getID() const { return ID; }
-
-  StringRef getName() const { return Name; }
-
-  // SU is in this queue if it's NodeQueueID is a superset of this ID.
-  bool isInQueue(SUnit *SU) const { return (SU->NodeQueueId & ID); }
-
-  bool empty() const { return Queue.empty(); }
-
-  unsigned size() const { return Queue.size(); }
-
-  typedef std::vector<SUnit*>::iterator iterator;
+  /// Represent the type of SchedCandidate found within a single queue.
+  /// pickNodeBidirectional depends on these listed by decreasing priority.
+  enum CandReason {
+    NoCand, SingleExcess, SingleCritical, ResourceReduce, ResourceDemand,
+    BotHeightReduce, BotPathReduce, TopDepthReduce, TopPathReduce,
+    SingleMax, MultiPressure, NextDefUse, NodeOrder};
 
-  iterator begin() { return Queue.begin(); }
+#ifndef NDEBUG
+  static const char *getReasonStr(ConvergingScheduler::CandReason Reason);
+#endif
 
-  iterator end() { return Queue.end(); }
+  /// Policy for scheduling the next instruction in the candidate's zone.
+  struct CandPolicy {
+    bool ReduceLatency;
+    unsigned ReduceResIdx;
+    unsigned DemandResIdx;
 
-  iterator find(SUnit *SU) {
-    return std::find(Queue.begin(), Queue.end(), SU);
-  }
+    CandPolicy(): ReduceLatency(false), ReduceResIdx(0), DemandResIdx(0) {}
+  };
 
-  void push(SUnit *SU) {
-    Queue.push_back(SU);
-    SU->NodeQueueId |= ID;
-  }
+  /// Status of an instruction's critical resource consumption.
+  struct SchedResourceDelta {
+    // Count critical resources in the scheduled region required by SU.
+    unsigned CritResources;
 
-  void remove(iterator I) {
-    (*I)->NodeQueueId &= ~ID;
-    *I = Queue.back();
-    Queue.pop_back();
-  }
+    // Count critical resources from another region consumed by SU.
+    unsigned DemandedResources;
 
-  void dump() {
-    dbgs() << Name << ": ";
-    for (unsigned i = 0, e = Queue.size(); i < e; ++i)
-      dbgs() << Queue[i]->NodeNum << " ";
-    dbgs() << "\n";
-  }
-};
+    SchedResourceDelta(): CritResources(0), DemandedResources(0) {}
 
-/// ConvergingScheduler shrinks the unscheduled zone using heuristics to balance
-/// the schedule.
-class ConvergingScheduler : public MachineSchedStrategy {
+    bool operator==(const SchedResourceDelta &RHS) const {
+      return CritResources == RHS.CritResources
+        && DemandedResources == RHS.DemandedResources;
+    }
+    bool operator!=(const SchedResourceDelta &RHS) const {
+      return !operator==(RHS);
+    }
+  };
 
   /// Store the state used by ConvergingScheduler heuristics, required for the
   /// lifetime of one invocation of pickNode().
   struct SchedCandidate {
+    CandPolicy Policy;
+
     // The best SUnit candidate.
     SUnit *SU;
 
+    // The reason for this candidate.
+    CandReason Reason;
+
     // Register pressure values for the best candidate.
     RegPressureDelta RPDelta;
 
-    SchedCandidate(): SU(NULL) {}
+    // Critical resource consumption of the best candidate.
+    SchedResourceDelta ResDelta;
+
+    SchedCandidate(const CandPolicy &policy)
+    : Policy(policy), SU(NULL), Reason(NoCand) {}
+
+    bool isValid() const { return SU; }
+
+    // Copy the status of another candidate without changing policy.
+    void setBest(SchedCandidate &Best) {
+      assert(Best.Reason != NoCand && "uninitialized Sched candidate");
+      SU = Best.SU;
+      Reason = Best.Reason;
+      RPDelta = Best.RPDelta;
+      ResDelta = Best.ResDelta;
+    }
+
+    void initResourceDelta(const ScheduleDAGMI *DAG,
+                           const TargetSchedModel *SchedModel);
+  };
+
+  /// Summarize the unscheduled region.
+  struct SchedRemainder {
+    // Critical path through the DAG in expected latency.
+    unsigned CriticalPath;
+
+    // Unscheduled resources
+    SmallVector<unsigned, 16> RemainingCounts;
+    // Critical resource for the unscheduled zone.
+    unsigned CritResIdx;
+    // Number of micro-ops left to schedule.
+    unsigned RemainingMicroOps;
+    // Is the unscheduled zone resource limited.
+    bool IsResourceLimited;
+
+    unsigned MaxRemainingCount;
+
+    void reset() {
+      CriticalPath = 0;
+      RemainingCounts.clear();
+      CritResIdx = 0;
+      RemainingMicroOps = 0;
+      IsResourceLimited = false;
+      MaxRemainingCount = 0;
+    }
+
+    SchedRemainder() { reset(); }
+
+    void init(ScheduleDAGMI *DAG, const TargetSchedModel *SchedModel);
   };
-  /// Represent the type of SchedCandidate found within a single queue.
-  enum CandResult {
-    NoCand, NodeOrder, SingleExcess, SingleCritical, SingleMax, MultiPressure };
 
   /// Each Scheduling boundary is associated with ready queues. It tracks the
-  /// current cycle in whichever direction at has moved, and maintains the state
+  /// current cycle in the direction of movement, and maintains the state
   /// of "hazards" and other interlocks at the current cycle.
   struct SchedBoundary {
     ScheduleDAGMI *DAG;
+    const TargetSchedModel *SchedModel;
+    SchedRemainder *Rem;
 
     ReadyQueue Available;
     ReadyQueue Pending;
     bool CheckPending;
 
+    // For heuristics, keep a list of the nodes that immediately depend on the
+    // most recently scheduled node.
+    SmallPtrSet<const SUnit*, 8> NextSUs;
+
     ScheduleHazardRecognizer *HazardRec;
 
     unsigned CurrCycle;
@@ -809,29 +791,88 @@ class ConvergingScheduler : public MachineSchedStrategy {
     /// MinReadyCycle - Cycle of the soonest available instruction.
     unsigned MinReadyCycle;
 
+    // The expected latency of the critical path in this scheduled zone.
+    unsigned ExpectedLatency;
+
+    // Resources used in the scheduled zone beyond this boundary.
+    SmallVector<unsigned, 16> ResourceCounts;
+
+    // Cache the critical resources ID in this scheduled zone.
+    unsigned CritResIdx;
+
+    // Is the scheduled region resource limited vs. latency limited.
+    bool IsResourceLimited;
+
+    unsigned ExpectedCount;
+
+    // Policy flag: attempt to find ILP until expected latency is covered.
+    bool ShouldIncreaseILP;
+
+#ifndef NDEBUG
     // Remember the greatest min operand latency.
     unsigned MaxMinLatency;
+#endif
+
+    void reset() {
+      Available.clear();
+      Pending.clear();
+      CheckPending = false;
+      NextSUs.clear();
+      HazardRec = 0;
+      CurrCycle = 0;
+      IssueCount = 0;
+      MinReadyCycle = UINT_MAX;
+      ExpectedLatency = 0;
+      ResourceCounts.resize(1);
+      assert(!ResourceCounts[0] && "nonzero count for bad resource");
+      CritResIdx = 0;
+      IsResourceLimited = false;
+      ExpectedCount = 0;
+      ShouldIncreaseILP = false;
+#ifndef NDEBUG
+      MaxMinLatency = 0;
+#endif
+      // Reserve a zero-count for invalid CritResIdx.
+      ResourceCounts.resize(1);
+    }
 
     /// Pending queues extend the ready queues with the same ID and the
     /// PendingFlag set.
     SchedBoundary(unsigned ID, const Twine &Name):
-      DAG(0), Available(ID, Name+".A"),
-      Pending(ID << ConvergingScheduler::LogMaxQID, Name+".P"),
-      CheckPending(false), HazardRec(0), CurrCycle(0), IssueCount(0),
-      MinReadyCycle(UINT_MAX), MaxMinLatency(0) {}
+      DAG(0), SchedModel(0), Rem(0), Available(ID, Name+".A"),
+      Pending(ID << ConvergingScheduler::LogMaxQID, Name+".P") {
+      reset();
+    }
 
     ~SchedBoundary() { delete HazardRec; }
 
+    void init(ScheduleDAGMI *dag, const TargetSchedModel *smodel,
+              SchedRemainder *rem);
+
     bool isTop() const {
       return Available.getID() == ConvergingScheduler::TopQID;
     }
 
+    unsigned getUnscheduledLatency(SUnit *SU) const {
+      if (isTop())
+        return SU->getHeight();
+      return SU->getDepth();
+    }
+
+    unsigned getCriticalCount() const {
+      return ResourceCounts[CritResIdx];
+    }
+
     bool checkHazard(SUnit *SU);
 
+    void checkILPPolicy();
+
     void releaseNode(SUnit *SU, unsigned ReadyCycle);
 
     void bumpCycle();
 
+    void countResource(unsigned PIdx, unsigned Cycles);
+
     void bumpNode(SUnit *SU);
 
     void releasePending();
@@ -841,10 +882,13 @@ class ConvergingScheduler : public MachineSchedStrategy {
     SUnit *pickOnlyChoice();
   };
 
+private:
   ScheduleDAGMI *DAG;
+  const TargetSchedModel *SchedModel;
   const TargetRegisterInfo *TRI;
 
   // State of the top and bottom scheduled instruction boundaries.
+  SchedRemainder Rem;
   SchedBoundary Top;
   SchedBoundary Bot;
 
@@ -857,7 +901,7 @@ public:
   };
 
   ConvergingScheduler():
-    DAG(0), TRI(0), Top(TopQID, "TopQ"), Bot(BotQID, "BotQ") {}
+    DAG(0), SchedModel(0), TRI(0), Top(TopQID, "TopQ"), Bot(BotQID, "BotQ") {}
 
   virtual void initialize(ScheduleDAGMI *dag);
 
@@ -869,28 +913,80 @@ public:
 
   virtual void releaseBottomNode(SUnit *SU);
 
+  virtual void registerRoots();
+
 protected:
-  SUnit *pickNodeBidrectional(bool &IsTopNode);
+  void balanceZones(
+    ConvergingScheduler::SchedBoundary &CriticalZone,
+    ConvergingScheduler::SchedCandidate &CriticalCand,
+    ConvergingScheduler::SchedBoundary &OppositeZone,
+    ConvergingScheduler::SchedCandidate &OppositeCand);
+
+  void checkResourceLimits(ConvergingScheduler::SchedCandidate &TopCand,
+                           ConvergingScheduler::SchedCandidate &BotCand);
+
+  void tryCandidate(SchedCandidate &Cand,
+                    SchedCandidate &TryCand,
+                    SchedBoundary &Zone,
+                    const RegPressureTracker &RPTracker,
+                    RegPressureTracker &TempTracker);
+
+  SUnit *pickNodeBidirectional(bool &IsTopNode);
+
+  void pickNodeFromQueue(SchedBoundary &Zone,
+                         const RegPressureTracker &RPTracker,
+                         SchedCandidate &Candidate);
 
-  CandResult pickNodeFromQueue(ReadyQueue &Q,
-                               const RegPressureTracker &RPTracker,
-                               SchedCandidate &Candidate);
 #ifndef NDEBUG
-  void traceCandidate(const char *Label, const ReadyQueue &Q, SUnit *SU,
-                      PressureElement P = PressureElement());
+  void traceCandidate(const SchedCandidate &Cand, const SchedBoundary &Zone);
 #endif
 };
 } // namespace
 
+void ConvergingScheduler::SchedRemainder::
+init(ScheduleDAGMI *DAG, const TargetSchedModel *SchedModel) {
+  reset();
+  if (!SchedModel->hasInstrSchedModel())
+    return;
+  RemainingCounts.resize(SchedModel->getNumProcResourceKinds());
+  for (std::vector<SUnit>::iterator
+         I = DAG->SUnits.begin(), E = DAG->SUnits.end(); I != E; ++I) {
+    const MCSchedClassDesc *SC = DAG->getSchedClass(&*I);
+    RemainingMicroOps += SchedModel->getNumMicroOps(I->getInstr(), SC);
+    for (TargetSchedModel::ProcResIter
+           PI = SchedModel->getWriteProcResBegin(SC),
+           PE = SchedModel->getWriteProcResEnd(SC); PI != PE; ++PI) {
+      unsigned PIdx = PI->ProcResourceIdx;
+      unsigned Factor = SchedModel->getResourceFactor(PIdx);
+      RemainingCounts[PIdx] += (Factor * PI->Cycles);
+    }
+  }
+}
+
+void ConvergingScheduler::SchedBoundary::
+init(ScheduleDAGMI *dag, const TargetSchedModel *smodel, SchedRemainder *rem) {
+  reset();
+  DAG = dag;
+  SchedModel = smodel;
+  Rem = rem;
+  if (SchedModel->hasInstrSchedModel())
+    ResourceCounts.resize(SchedModel->getNumProcResourceKinds());
+}
+
 void ConvergingScheduler::initialize(ScheduleDAGMI *dag) {
   DAG = dag;
+  SchedModel = DAG->getSchedModel();
   TRI = DAG->TRI;
-  Top.DAG = dag;
-  Bot.DAG = dag;
+  Rem.init(DAG, SchedModel);
+  Top.init(DAG, SchedModel, &Rem);
+  Bot.init(DAG, SchedModel, &Rem);
+
+  // Initialize resource counts.
 
-  // Initialize the HazardRecognizers.
+  // Initialize the HazardRecognizers. If itineraries don't exist, are empty, or
+  // are disabled, then these HazardRecs will be disabled.
+  const InstrItineraryData *Itin = SchedModel->getInstrItineraries();
   const TargetMachine &TM = DAG->MF.getTarget();
-  const InstrItineraryData *Itin = TM.getInstrItineraryData();
   Top.HazardRec = TM.getInstrInfo()->CreateTargetMIHazardRecognizer(Itin, DAG);
   Bot.HazardRec = TM.getInstrInfo()->CreateTargetMIHazardRecognizer(Itin, DAG);
 
@@ -905,13 +1001,12 @@ void ConvergingScheduler::releaseTopNode(SUnit *SU) {
   for (SUnit::succ_iterator I = SU->Preds.begin(), E = SU->Preds.end();
        I != E; ++I) {
     unsigned PredReadyCycle = I->getSUnit()->TopReadyCycle;
-    unsigned Latency =
-      DAG->computeOperandLatency(I->getSUnit(), SU, *I, /*FindMin=*/true);
+    unsigned MinLatency = I->getMinLatency();
 #ifndef NDEBUG
-    Top.MaxMinLatency = std::max(Latency, Top.MaxMinLatency);
+    Top.MaxMinLatency = std::max(MinLatency, Top.MaxMinLatency);
 #endif
-    if (SU->TopReadyCycle < PredReadyCycle + Latency)
-      SU->TopReadyCycle = PredReadyCycle + Latency;
+    if (SU->TopReadyCycle < PredReadyCycle + MinLatency)
+      SU->TopReadyCycle = PredReadyCycle + MinLatency;
   }
   Top.releaseNode(SU, SU->TopReadyCycle);
 }
@@ -925,17 +1020,27 @@ void ConvergingScheduler::releaseBottomNode(SUnit *SU) {
   for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
        I != E; ++I) {
     unsigned SuccReadyCycle = I->getSUnit()->BotReadyCycle;
-    unsigned Latency =
-      DAG->computeOperandLatency(SU, I->getSUnit(), *I, /*FindMin=*/true);
+    unsigned MinLatency = I->getMinLatency();
 #ifndef NDEBUG
-    Bot.MaxMinLatency = std::max(Latency, Bot.MaxMinLatency);
+    Bot.MaxMinLatency = std::max(MinLatency, Bot.MaxMinLatency);
 #endif
-    if (SU->BotReadyCycle < SuccReadyCycle + Latency)
-      SU->BotReadyCycle = SuccReadyCycle + Latency;
+    if (SU->BotReadyCycle < SuccReadyCycle + MinLatency)
+      SU->BotReadyCycle = SuccReadyCycle + MinLatency;
   }
   Bot.releaseNode(SU, SU->BotReadyCycle);
 }
 
+void ConvergingScheduler::registerRoots() {
+  Rem.CriticalPath = DAG->ExitSU.getDepth();
+  // Some roots may not feed into ExitSU. Check all of them in case.
+  for (std::vector<SUnit*>::const_iterator
+         I = Bot.Available.begin(), E = Bot.Available.end(); I != E; ++I) {
+    if ((*I)->getDepth() > Rem.CriticalPath)
+      Rem.CriticalPath = (*I)->getDepth();
+  }
+  DEBUG(dbgs() << "Critical Path: " << Rem.CriticalPath << '\n');
+}
+
 /// Does this SU have a hazard within the current instruction group.
 ///
 /// The scheduler supports two modes of hazard recognition. The first is the
@@ -953,14 +1058,27 @@ bool ConvergingScheduler::SchedBoundary::checkHazard(SUnit *SU) {
   if (HazardRec->isEnabled())
     return HazardRec->getHazardType(SU) != ScheduleHazardRecognizer::NoHazard;
 
-  if (IssueCount + DAG->getNumMicroOps(SU->getInstr()) > DAG->getIssueWidth())
+  unsigned uops = SchedModel->getNumMicroOps(SU->getInstr());
+  if ((IssueCount > 0) && (IssueCount + uops > SchedModel->getIssueWidth())) {
+    DEBUG(dbgs() << "  SU(" << SU->NodeNum << ") uops="
+          << SchedModel->getNumMicroOps(SU->getInstr()) << '\n');
     return true;
-
+  }
   return false;
 }
 
+/// If expected latency is covered, disable ILP policy.
+void ConvergingScheduler::SchedBoundary::checkILPPolicy() {
+  if (ShouldIncreaseILP
+      && (IsResourceLimited || ExpectedLatency <= CurrCycle)) {
+    ShouldIncreaseILP = false;
+    DEBUG(dbgs() << "Disable ILP: " << Available.getName() << '\n');
+  }
+}
+
 void ConvergingScheduler::SchedBoundary::releaseNode(SUnit *SU,
                                                      unsigned ReadyCycle) {
+
   if (ReadyCycle < MinReadyCycle)
     MinReadyCycle = ReadyCycle;
 
@@ -970,15 +1088,31 @@ void ConvergingScheduler::SchedBoundary::releaseNode(SUnit *SU,
     Pending.push(SU);
   else
     Available.push(SU);
+
+  // Record this node as an immediate dependent of the scheduled node.
+  NextSUs.insert(SU);
+
+  // If CriticalPath has been computed, then check if the unscheduled nodes
+  // exceed the ILP window. Before registerRoots, CriticalPath==0.
+  if (Rem->CriticalPath && (ExpectedLatency + getUnscheduledLatency(SU)
+                            > Rem->CriticalPath + ILPWindow)) {
+    ShouldIncreaseILP = true;
+    DEBUG(dbgs() << "Increase ILP: " << Available.getName() << " "
+          << ExpectedLatency << " + " << getUnscheduledLatency(SU) << '\n');
+  }
 }
 
 /// Move the boundary of scheduled code by one cycle.
 void ConvergingScheduler::SchedBoundary::bumpCycle() {
-  unsigned Width = DAG->getIssueWidth();
+  unsigned Width = SchedModel->getIssueWidth();
   IssueCount = (IssueCount <= Width) ? 0 : IssueCount - Width;
 
+  unsigned NextCycle = CurrCycle + 1;
   assert(MinReadyCycle < UINT_MAX && "MinReadyCycle uninitialized");
-  unsigned NextCycle = std::max(CurrCycle + 1, MinReadyCycle);
+  if (MinReadyCycle > NextCycle) {
+    IssueCount = 0;
+    NextCycle = MinReadyCycle;
+  }
 
   if (!HazardRec->isEnabled()) {
     // Bypass HazardRec virtual calls.
@@ -994,11 +1128,39 @@ void ConvergingScheduler::SchedBoundary::bumpCycle() {
     }
   }
   CheckPending = true;
+  IsResourceLimited = getCriticalCount() > std::max(ExpectedLatency, CurrCycle);
 
-  DEBUG(dbgs() << "*** " << Available.getName() << " cycle "
+  DEBUG(dbgs() << "  *** " << Available.getName() << " cycle "
         << CurrCycle << '\n');
 }
 
+/// Add the given processor resource to this scheduled zone.
+void ConvergingScheduler::SchedBoundary::countResource(unsigned PIdx,
+                                                       unsigned Cycles) {
+  unsigned Factor = SchedModel->getResourceFactor(PIdx);
+  DEBUG(dbgs() << "  " << SchedModel->getProcResource(PIdx)->Name
+        << " +(" << Cycles << "x" << Factor
+        << ") / " << SchedModel->getLatencyFactor() << '\n');
+
+  unsigned Count = Factor * Cycles;
+  ResourceCounts[PIdx] += Count;
+  assert(Rem->RemainingCounts[PIdx] >= Count && "resource double counted");
+  Rem->RemainingCounts[PIdx] -= Count;
+
+  // Reset MaxRemainingCount for sanity.
+  Rem->MaxRemainingCount = 0;
+
+  // Check if this resource exceeds the current critical resource by a full
+  // cycle. If so, it becomes the critical resource.
+  if ((int)(ResourceCounts[PIdx] - ResourceCounts[CritResIdx])
+      >= (int)SchedModel->getLatencyFactor()) {
+    CritResIdx = PIdx;
+    DEBUG(dbgs() << "  *** Critical resource "
+          << SchedModel->getProcResource(PIdx)->Name << " x"
+          << ResourceCounts[PIdx] << '\n');
+  }
+}
+
 /// Move the boundary of scheduled code by one SUnit.
 void ConvergingScheduler::SchedBoundary::bumpNode(SUnit *SU) {
   // Update the reservation table.
@@ -1010,11 +1172,38 @@ void ConvergingScheduler::SchedBoundary::bumpNode(SUnit *SU) {
     }
     HazardRec->EmitInstruction(SU);
   }
+  // Update resource counts and critical resource.
+  if (SchedModel->hasInstrSchedModel()) {
+    const MCSchedClassDesc *SC = DAG->getSchedClass(SU);
+    Rem->RemainingMicroOps -= SchedModel->getNumMicroOps(SU->getInstr(), SC);
+    for (TargetSchedModel::ProcResIter
+           PI = SchedModel->getWriteProcResBegin(SC),
+           PE = SchedModel->getWriteProcResEnd(SC); PI != PE; ++PI) {
+      countResource(PI->ProcResourceIdx, PI->Cycles);
+    }
+  }
+  if (isTop()) {
+    if (SU->getDepth() > ExpectedLatency)
+      ExpectedLatency = SU->getDepth();
+  }
+  else {
+    if (SU->getHeight() > ExpectedLatency)
+      ExpectedLatency = SU->getHeight();
+  }
+
+  IsResourceLimited = getCriticalCount() > std::max(ExpectedLatency, CurrCycle);
+
   // Check the instruction group dispatch limit.
   // TODO: Check if this SU must end a dispatch group.
-  IssueCount += DAG->getNumMicroOps(SU->getInstr());
-  if (IssueCount >= DAG->getIssueWidth()) {
-    DEBUG(dbgs() << "*** Max instrs at cycle " << CurrCycle << '\n');
+  IssueCount += SchedModel->getNumMicroOps(SU->getInstr());
+
+  // checkHazard prevents scheduling multiple instructions per cycle that exceed
+  // issue width. However, we commonly reach the maximum. In this case
+  // opportunistically bump the cycle to avoid uselessly checking everything in
+  // the readyQ. Furthermore, a single instruction may produce more than one
+  // cycle's worth of micro-ops.
+  if (IssueCount >= SchedModel->getIssueWidth()) {
+    DEBUG(dbgs() << "  *** Max instrs at cycle " << CurrCycle << '\n');
     bumpCycle();
   }
 }
@@ -1045,6 +1234,7 @@ void ConvergingScheduler::SchedBoundary::releasePending() {
     Pending.remove(Pending.begin()+i);
     --i; --e;
   }
+  DEBUG(if (!Pending.empty()) Pending.dump());
   CheckPending = false;
 }
 
@@ -1059,12 +1249,23 @@ void ConvergingScheduler::SchedBoundary::removeReady(SUnit *SU) {
 }
 
 /// If this queue only has one ready candidate, return it. As a side effect,
-/// advance the cycle until at least one node is ready. If multiple instructions
-/// are ready, return NULL.
+/// defer any nodes that now hit a hazard, and advance the cycle until at least
+/// one node is ready. If multiple instructions are ready, return NULL.
 SUnit *ConvergingScheduler::SchedBoundary::pickOnlyChoice() {
   if (CheckPending)
     releasePending();
 
+  if (IssueCount > 0) {
+    // Defer any ready instrs that now have a hazard.
+    for (ReadyQueue::iterator I = Available.begin(); I != Available.end();) {
+      if (checkHazard(*I)) {
+        Pending.push(*I);
+        I = Available.remove(I);
+        continue;
+      }
+      ++I;
+    }
+  }
   for (unsigned i = 0; Available.empty(); ++i) {
     assert(i <= (HazardRec->getMaxLookAhead() + MaxMinLatency) &&
            "permanent hazard"); (void)i;
@@ -1076,18 +1277,262 @@ SUnit *ConvergingScheduler::SchedBoundary::pickOnlyChoice() {
   return NULL;
 }
 
-#ifndef NDEBUG
-void ConvergingScheduler::traceCandidate(const char *Label, const ReadyQueue &Q,
-                                         SUnit *SU, PressureElement P) {
-  dbgs() << Label << " " << Q.getName() << " ";
-  if (P.isValid())
-    dbgs() << TRI->getRegPressureSetName(P.PSetID) << ":" << P.UnitIncrease
-           << " ";
-  else
-    dbgs() << "     ";
-  SU->dump(DAG);
+/// Record the candidate policy for opposite zones with different critical
+/// resources.
+///
+/// If the CriticalZone is latency limited, don't force a policy for the
+/// candidates here. Instead, When releasing each candidate, releaseNode
+/// compares the region's critical path to the candidate's height or depth and
+/// the scheduled zone's expected latency then sets ShouldIncreaseILP.
+void ConvergingScheduler::balanceZones(
+  ConvergingScheduler::SchedBoundary &CriticalZone,
+  ConvergingScheduler::SchedCandidate &CriticalCand,
+  ConvergingScheduler::SchedBoundary &OppositeZone,
+  ConvergingScheduler::SchedCandidate &OppositeCand) {
+
+  if (!CriticalZone.IsResourceLimited)
+    return;
+
+  SchedRemainder *Rem = CriticalZone.Rem;
+
+  // If the critical zone is overconsuming a resource relative to the
+  // remainder, try to reduce it.
+  unsigned RemainingCritCount =
+    Rem->RemainingCounts[CriticalZone.CritResIdx];
+  if ((int)(Rem->MaxRemainingCount - RemainingCritCount)
+      > (int)SchedModel->getLatencyFactor()) {
+    CriticalCand.Policy.ReduceResIdx = CriticalZone.CritResIdx;
+    DEBUG(dbgs() << "Balance " << CriticalZone.Available.getName() << " reduce "
+          << SchedModel->getProcResource(CriticalZone.CritResIdx)->Name
+          << '\n');
+  }
+  // If the other zone is underconsuming a resource relative to the full zone,
+  // try to increase it.
+  unsigned OppositeCount =
+    OppositeZone.ResourceCounts[CriticalZone.CritResIdx];
+  if ((int)(OppositeZone.ExpectedCount - OppositeCount)
+      > (int)SchedModel->getLatencyFactor()) {
+    OppositeCand.Policy.DemandResIdx = CriticalZone.CritResIdx;
+    DEBUG(dbgs() << "Balance " << OppositeZone.Available.getName() << " demand "
+          << SchedModel->getProcResource(OppositeZone.CritResIdx)->Name
+          << '\n');
+  }
+}
+
+/// Determine if the scheduled zones exceed resource limits or critical path and
+/// set each candidate's ReduceHeight policy accordingly.
+void ConvergingScheduler::checkResourceLimits(
+  ConvergingScheduler::SchedCandidate &TopCand,
+  ConvergingScheduler::SchedCandidate &BotCand) {
+
+  Bot.checkILPPolicy();
+  Top.checkILPPolicy();
+  if (Bot.ShouldIncreaseILP)
+    BotCand.Policy.ReduceLatency = true;
+  if (Top.ShouldIncreaseILP)
+    TopCand.Policy.ReduceLatency = true;
+
+  // Handle resource-limited regions.
+  if (Top.IsResourceLimited && Bot.IsResourceLimited
+      && Top.CritResIdx == Bot.CritResIdx) {
+    // If the scheduled critical resource in both zones is no longer the
+    // critical remaining resource, attempt to reduce resource height both ways.
+    if (Top.CritResIdx != Rem.CritResIdx) {
+      TopCand.Policy.ReduceResIdx = Top.CritResIdx;
+      BotCand.Policy.ReduceResIdx = Bot.CritResIdx;
+      DEBUG(dbgs() << "Reduce scheduled "
+            << SchedModel->getProcResource(Top.CritResIdx)->Name << '\n');
+    }
+    return;
+  }
+  // Handle latency-limited regions.
+  if (!Top.IsResourceLimited && !Bot.IsResourceLimited) {
+    // If the total scheduled expected latency exceeds the region's critical
+    // path then reduce latency both ways.
+    //
+    // Just because a zone is not resource limited does not mean it is latency
+    // limited. Unbuffered resource, such as max micro-ops may cause CurrCycle
+    // to exceed expected latency.
+    if ((Top.ExpectedLatency + Bot.ExpectedLatency >= Rem.CriticalPath)
+        && (Rem.CriticalPath > Top.CurrCycle + Bot.CurrCycle)) {
+      TopCand.Policy.ReduceLatency = true;
+      BotCand.Policy.ReduceLatency = true;
+      DEBUG(dbgs() << "Reduce scheduled latency " << Top.ExpectedLatency
+            << " + " << Bot.ExpectedLatency << '\n');
+    }
+    return;
+  }
+  // The critical resource is different in each zone, so request balancing.
+
+  // Compute the cost of each zone.
+  Rem.MaxRemainingCount = std::max(
+    Rem.RemainingMicroOps * SchedModel->getMicroOpFactor(),
+    Rem.RemainingCounts[Rem.CritResIdx]);
+  Top.ExpectedCount = std::max(Top.ExpectedLatency, Top.CurrCycle);
+  Top.ExpectedCount = std::max(
+    Top.getCriticalCount(),
+    Top.ExpectedCount * SchedModel->getLatencyFactor());
+  Bot.ExpectedCount = std::max(Bot.ExpectedLatency, Bot.CurrCycle);
+  Bot.ExpectedCount = std::max(
+    Bot.getCriticalCount(),
+    Bot.ExpectedCount * SchedModel->getLatencyFactor());
+
+  balanceZones(Top, TopCand, Bot, BotCand);
+  balanceZones(Bot, BotCand, Top, TopCand);
+}
+
+void ConvergingScheduler::SchedCandidate::
+initResourceDelta(const ScheduleDAGMI *DAG,
+                  const TargetSchedModel *SchedModel) {
+  if (!Policy.ReduceResIdx && !Policy.DemandResIdx)
+    return;
+
+  const MCSchedClassDesc *SC = DAG->getSchedClass(SU);
+  for (TargetSchedModel::ProcResIter
+         PI = SchedModel->getWriteProcResBegin(SC),
+         PE = SchedModel->getWriteProcResEnd(SC); PI != PE; ++PI) {
+    if (PI->ProcResourceIdx == Policy.ReduceResIdx)
+      ResDelta.CritResources += PI->Cycles;
+    if (PI->ProcResourceIdx == Policy.DemandResIdx)
+      ResDelta.DemandedResources += PI->Cycles;
+  }
+}
+
+/// Return true if this heuristic determines order.
+static bool tryLess(unsigned TryVal, unsigned CandVal,
+                    ConvergingScheduler::SchedCandidate &TryCand,
+                    ConvergingScheduler::SchedCandidate &Cand,
+                    ConvergingScheduler::CandReason Reason) {
+  if (TryVal < CandVal) {
+    TryCand.Reason = Reason;
+    return true;
+  }
+  if (TryVal > CandVal) {
+    if (Cand.Reason > Reason)
+      Cand.Reason = Reason;
+    return true;
+  }
+  return false;
+}
+static bool tryGreater(unsigned TryVal, unsigned CandVal,
+                       ConvergingScheduler::SchedCandidate &TryCand,
+                       ConvergingScheduler::SchedCandidate &Cand,
+                       ConvergingScheduler::CandReason Reason) {
+  if (TryVal > CandVal) {
+    TryCand.Reason = Reason;
+    return true;
+  }
+  if (TryVal < CandVal) {
+    if (Cand.Reason > Reason)
+      Cand.Reason = Reason;
+    return true;
+  }
+  return false;
+}
+
+/// Apply a set of heursitics to a new candidate. Heuristics are currently
+/// hierarchical. This may be more efficient than a graduated cost model because
+/// we don't need to evaluate all aspects of the model for each node in the
+/// queue. But it's really done to make the heuristics easier to debug and
+/// statistically analyze.
+///
+/// \param Cand provides the policy and current best candidate.
+/// \param TryCand refers to the next SUnit candidate, otherwise uninitialized.
+/// \param Zone describes the scheduled zone that we are extending.
+/// \param RPTracker describes reg pressure within the scheduled zone.
+/// \param TempTracker is a scratch pressure tracker to reuse in queries.
+void ConvergingScheduler::tryCandidate(SchedCandidate &Cand,
+                                       SchedCandidate &TryCand,
+                                       SchedBoundary &Zone,
+                                       const RegPressureTracker &RPTracker,
+                                       RegPressureTracker &TempTracker) {
+
+  // Always initialize TryCand's RPDelta.
+  TempTracker.getMaxPressureDelta(TryCand.SU->getInstr(), TryCand.RPDelta,
+                                  DAG->getRegionCriticalPSets(),
+                                  DAG->getRegPressure().MaxSetPressure);
+
+  // Initialize the candidate if needed.
+  if (!Cand.isValid()) {
+    TryCand.Reason = NodeOrder;
+    return;
+  }
+  // Avoid exceeding the target's limit.
+  if (tryLess(TryCand.RPDelta.Excess.UnitIncrease,
+              Cand.RPDelta.Excess.UnitIncrease, TryCand, Cand, SingleExcess))
+    return;
+  if (Cand.Reason == SingleExcess)
+    Cand.Reason = MultiPressure;
+
+  // Avoid increasing the max critical pressure in the scheduled region.
+  if (tryLess(TryCand.RPDelta.CriticalMax.UnitIncrease,
+              Cand.RPDelta.CriticalMax.UnitIncrease,
+              TryCand, Cand, SingleCritical))
+    return;
+  if (Cand.Reason == SingleCritical)
+    Cand.Reason = MultiPressure;
+
+  // Avoid critical resource consumption and balance the schedule.
+  TryCand.initResourceDelta(DAG, SchedModel);
+  if (tryLess(TryCand.ResDelta.CritResources, Cand.ResDelta.CritResources,
+              TryCand, Cand, ResourceReduce))
+    return;
+  if (tryGreater(TryCand.ResDelta.DemandedResources,
+                 Cand.ResDelta.DemandedResources,
+                 TryCand, Cand, ResourceDemand))
+    return;
+
+  // Avoid serializing long latency dependence chains.
+  if (Cand.Policy.ReduceLatency) {
+    if (Zone.isTop()) {
+      if (Cand.SU->getDepth() * SchedModel->getLatencyFactor()
+          > Zone.ExpectedCount) {
+        if (tryLess(TryCand.SU->getDepth(), Cand.SU->getDepth(),
+                    TryCand, Cand, TopDepthReduce))
+          return;
+      }
+      if (tryGreater(TryCand.SU->getHeight(), Cand.SU->getHeight(),
+                     TryCand, Cand, TopPathReduce))
+        return;
+    }
+    else {
+      if (Cand.SU->getHeight() * SchedModel->getLatencyFactor()
+          > Zone.ExpectedCount) {
+        if (tryLess(TryCand.SU->getHeight(), Cand.SU->getHeight(),
+                    TryCand, Cand, BotHeightReduce))
+          return;
+      }
+      if (tryGreater(TryCand.SU->getDepth(), Cand.SU->getDepth(),
+                     TryCand, Cand, BotPathReduce))
+        return;
+    }
+  }
+
+  // Avoid increasing the max pressure of the entire region.
+  if (tryLess(TryCand.RPDelta.CurrentMax.UnitIncrease,
+              Cand.RPDelta.CurrentMax.UnitIncrease, TryCand, Cand, SingleMax))
+    return;
+  if (Cand.Reason == SingleMax)
+    Cand.Reason = MultiPressure;
+
+  // Prefer immediate defs/users of the last scheduled instruction. This is a
+  // nice pressure avoidance strategy that also conserves the processor's
+  // register renaming resources and keeps the machine code readable.
+  if (Zone.NextSUs.count(TryCand.SU) && !Zone.NextSUs.count(Cand.SU)) {
+    TryCand.Reason = NextDefUse;
+    return;
+  }
+  if (!Zone.NextSUs.count(TryCand.SU) && Zone.NextSUs.count(Cand.SU)) {
+    if (Cand.Reason > NextDefUse)
+      Cand.Reason = NextDefUse;
+    return;
+  }
+  // Fall through to original instruction order.
+  if ((Zone.isTop() && TryCand.SU->NodeNum < Cand.SU->NodeNum)
+      || (!Zone.isTop() && TryCand.SU->NodeNum > Cand.SU->NodeNum)) {
+    TryCand.Reason = NodeOrder;
+  }
 }
-#endif
 
 /// pickNodeFromQueue helper that returns true if the LHS reg pressure effect is
 /// more desirable than RHS from scheduling standpoint.
@@ -1098,109 +1543,144 @@ static bool compareRPDelta(const RegPressureDelta &LHS,
   // have UnitIncrease==0, so are neutral.
 
   // Avoid increasing the max critical pressure in the scheduled region.
-  if (LHS.Excess.UnitIncrease != RHS.Excess.UnitIncrease)
+  if (LHS.Excess.UnitIncrease != RHS.Excess.UnitIncrease) {
+    DEBUG(dbgs() << "RP excess top - bot: "
+          << (LHS.Excess.UnitIncrease - RHS.Excess.UnitIncrease) << '\n');
     return LHS.Excess.UnitIncrease < RHS.Excess.UnitIncrease;
-
+  }
   // Avoid increasing the max critical pressure in the scheduled region.
-  if (LHS.CriticalMax.UnitIncrease != RHS.CriticalMax.UnitIncrease)
+  if (LHS.CriticalMax.UnitIncrease != RHS.CriticalMax.UnitIncrease) {
+    DEBUG(dbgs() << "RP critical top - bot: "
+          << (LHS.CriticalMax.UnitIncrease - RHS.CriticalMax.UnitIncrease)
+          << '\n');
     return LHS.CriticalMax.UnitIncrease < RHS.CriticalMax.UnitIncrease;
-
+  }
   // Avoid increasing the max pressure of the entire region.
-  if (LHS.CurrentMax.UnitIncrease != RHS.CurrentMax.UnitIncrease)
+  if (LHS.CurrentMax.UnitIncrease != RHS.CurrentMax.UnitIncrease) {
+    DEBUG(dbgs() << "RP current top - bot: "
+          << (LHS.CurrentMax.UnitIncrease - RHS.CurrentMax.UnitIncrease)
+          << '\n');
     return LHS.CurrentMax.UnitIncrease < RHS.CurrentMax.UnitIncrease;
-
+  }
   return false;
 }
 
+#ifndef NDEBUG
+const char *ConvergingScheduler::getReasonStr(
+  ConvergingScheduler::CandReason Reason) {
+  switch (Reason) {
+  case NoCand:         return "NOCAND    ";
+  case SingleExcess:   return "REG-EXCESS";
+  case SingleCritical: return "REG-CRIT  ";
+  case SingleMax:      return "REG-MAX   ";
+  case MultiPressure:  return "REG-MULTI ";
+  case ResourceReduce: return "RES-REDUCE";
+  case ResourceDemand: return "RES-DEMAND";
+  case TopDepthReduce: return "TOP-DEPTH ";
+  case TopPathReduce:  return "TOP-PATH  ";
+  case BotHeightReduce:return "BOT-HEIGHT";
+  case BotPathReduce:  return "BOT-PATH  ";
+  case NextDefUse:     return "DEF-USE   ";
+  case NodeOrder:      return "ORDER     ";
+  };
+  llvm_unreachable("Unknown reason!");
+}
+
+void ConvergingScheduler::traceCandidate(const SchedCandidate &Cand,
+                                         const SchedBoundary &Zone) {
+  const char *Label = getReasonStr(Cand.Reason);
+  PressureElement P;
+  unsigned ResIdx = 0;
+  unsigned Latency = 0;
+  switch (Cand.Reason) {
+  default:
+    break;
+  case SingleExcess:
+    P = Cand.RPDelta.Excess;
+    break;
+  case SingleCritical:
+    P = Cand.RPDelta.CriticalMax;
+    break;
+  case SingleMax:
+    P = Cand.RPDelta.CurrentMax;
+    break;
+  case ResourceReduce:
+    ResIdx = Cand.Policy.ReduceResIdx;
+    break;
+  case ResourceDemand:
+    ResIdx = Cand.Policy.DemandResIdx;
+    break;
+  case TopDepthReduce:
+    Latency = Cand.SU->getDepth();
+    break;
+  case TopPathReduce:
+    Latency = Cand.SU->getHeight();
+    break;
+  case BotHeightReduce:
+    Latency = Cand.SU->getHeight();
+    break;
+  case BotPathReduce:
+    Latency = Cand.SU->getDepth();
+    break;
+  }
+  dbgs() << Label << " " << Zone.Available.getName() << " ";
+  if (P.isValid())
+    dbgs() << TRI->getRegPressureSetName(P.PSetID) << ":" << P.UnitIncrease
+           << " ";
+  else
+    dbgs() << "     ";
+  if (ResIdx)
+    dbgs() << SchedModel->getProcResource(ResIdx)->Name << " ";
+  else
+    dbgs() << "        ";
+  if (Latency)
+    dbgs() << Latency << " cycles ";
+  else
+    dbgs() << "         ";
+  Cand.SU->dump(DAG);
+}
+#endif
+
 /// Pick the best candidate from the top queue.
 ///
 /// TODO: getMaxPressureDelta results can be mostly cached for each SUnit during
 /// DAG building. To adjust for the current scheduling location we need to
 /// maintain the number of vreg uses remaining to be top-scheduled.
-ConvergingScheduler::CandResult ConvergingScheduler::
-pickNodeFromQueue(ReadyQueue &Q, const RegPressureTracker &RPTracker,
-                  SchedCandidate &Candidate) {
+void ConvergingScheduler::pickNodeFromQueue(SchedBoundary &Zone,
+                                            const RegPressureTracker &RPTracker,
+                                            SchedCandidate &Cand) {
+  ReadyQueue &Q = Zone.Available;
+
   DEBUG(Q.dump());
 
   // getMaxPressureDelta temporarily modifies the tracker.
   RegPressureTracker &TempTracker = const_cast<RegPressureTracker&>(RPTracker);
 
-  // BestSU remains NULL if no top candidates beat the best existing candidate.
-  CandResult FoundCandidate = NoCand;
   for (ReadyQueue::iterator I = Q.begin(), E = Q.end(); I != E; ++I) {
-    RegPressureDelta RPDelta;
-    TempTracker.getMaxPressureDelta((*I)->getInstr(), RPDelta,
-                                    DAG->getRegionCriticalPSets(),
-                                    DAG->getRegPressure().MaxSetPressure);
-
-    // Initialize the candidate if needed.
-    if (!Candidate.SU) {
-      Candidate.SU = *I;
-      Candidate.RPDelta = RPDelta;
-      FoundCandidate = NodeOrder;
-      continue;
-    }
-    // Avoid exceeding the target's limit.
-    if (RPDelta.Excess.UnitIncrease < Candidate.RPDelta.Excess.UnitIncrease) {
-      DEBUG(traceCandidate("ECAND", Q, *I, RPDelta.Excess));
-      Candidate.SU = *I;
-      Candidate.RPDelta = RPDelta;
-      FoundCandidate = SingleExcess;
-      continue;
-    }
-    if (RPDelta.Excess.UnitIncrease > Candidate.RPDelta.Excess.UnitIncrease)
-      continue;
-    if (FoundCandidate == SingleExcess)
-      FoundCandidate = MultiPressure;
-
-    // Avoid increasing the max critical pressure in the scheduled region.
-    if (RPDelta.CriticalMax.UnitIncrease
-        < Candidate.RPDelta.CriticalMax.UnitIncrease) {
-      DEBUG(traceCandidate("PCAND", Q, *I, RPDelta.CriticalMax));
-      Candidate.SU = *I;
-      Candidate.RPDelta = RPDelta;
-      FoundCandidate = SingleCritical;
-      continue;
-    }
-    if (RPDelta.CriticalMax.UnitIncrease
-        > Candidate.RPDelta.CriticalMax.UnitIncrease)
-      continue;
-    if (FoundCandidate == SingleCritical)
-      FoundCandidate = MultiPressure;
-
-    // Avoid increasing the max pressure of the entire region.
-    if (RPDelta.CurrentMax.UnitIncrease
-        < Candidate.RPDelta.CurrentMax.UnitIncrease) {
-      DEBUG(traceCandidate("MCAND", Q, *I, RPDelta.CurrentMax));
-      Candidate.SU = *I;
-      Candidate.RPDelta = RPDelta;
-      FoundCandidate = SingleMax;
-      continue;
-    }
-    if (RPDelta.CurrentMax.UnitIncrease
-        > Candidate.RPDelta.CurrentMax.UnitIncrease)
-      continue;
-    if (FoundCandidate == SingleMax)
-      FoundCandidate = MultiPressure;
-
-    // Fall through to original instruction order.
-    // Only consider node order if Candidate was chosen from this Q.
-    if (FoundCandidate == NoCand)
-      continue;
 
-    if ((Q.getID() == TopQID && (*I)->NodeNum < Candidate.SU->NodeNum)
-        || (Q.getID() == BotQID && (*I)->NodeNum > Candidate.SU->NodeNum)) {
-      DEBUG(traceCandidate("NCAND", Q, *I));
-      Candidate.SU = *I;
-      Candidate.RPDelta = RPDelta;
-      FoundCandidate = NodeOrder;
+    SchedCandidate TryCand(Cand.Policy);
+    TryCand.SU = *I;
+    tryCandidate(Cand, TryCand, Zone, RPTracker, TempTracker);
+    if (TryCand.Reason != NoCand) {
+      // Initialize resource delta if needed in case future heuristics query it.
+      if (TryCand.ResDelta == SchedResourceDelta())
+        TryCand.initResourceDelta(DAG, SchedModel);
+      Cand.setBest(TryCand);
+      DEBUG(traceCandidate(Cand, Zone));
     }
+    TryCand.SU = *I;
   }
-  return FoundCandidate;
+}
+
+static void tracePick(const ConvergingScheduler::SchedCandidate &Cand,
+                      bool IsTop) {
+  DEBUG(dbgs() << "Pick " << (IsTop ? "top" : "bot")
+        << " SU(" << Cand.SU->NodeNum << ") "
+        << ConvergingScheduler::getReasonStr(Cand.Reason) << '\n');
 }
 
 /// Pick the best candidate node from either the top or bottom queue.
-SUnit *ConvergingScheduler::pickNodeBidrectional(bool &IsTopNode) {
+SUnit *ConvergingScheduler::pickNodeBidirectional(bool &IsTopNode) {
   // Schedule as far as possible in the direction of no choice. This is most
   // efficient, but also provides the best heuristics for CriticalPSets.
   if (SUnit *SU = Bot.pickOnlyChoice()) {
@@ -1211,11 +1691,14 @@ SUnit *ConvergingScheduler::pickNodeBidrectional(bool &IsTopNode) {
     IsTopNode = true;
     return SU;
   }
-  SchedCandidate BotCand;
+  CandPolicy NoPolicy;
+  SchedCandidate BotCand(NoPolicy);
+  SchedCandidate TopCand(NoPolicy);
+  checkResourceLimits(TopCand, BotCand);
+
   // Prefer bottom scheduling when heuristics are silent.
-  CandResult BotResult = pickNodeFromQueue(Bot.Available,
-                                           DAG->getBotRPTracker(), BotCand);
-  assert(BotResult != NoCand && "failed to find the first candidate");
+  pickNodeFromQueue(Bot, DAG->getBotRPTracker(), BotCand);
+  assert(BotCand.Reason != NoCand && "failed to find the first candidate");
 
   // If either Q has a single candidate that provides the least increase in
   // Excess pressure, we can immediately schedule from that Q.
@@ -1224,37 +1707,41 @@ SUnit *ConvergingScheduler::pickNodeBidrectional(bool &IsTopNode) {
   // affects picking from either Q. If scheduling in one direction must
   // increase pressure for one of the excess PSets, then schedule in that
   // direction first to provide more freedom in the other direction.
-  if (BotResult == SingleExcess || BotResult == SingleCritical) {
+  if (BotCand.Reason == SingleExcess || BotCand.Reason == SingleCritical) {
     IsTopNode = false;
+    tracePick(BotCand, IsTopNode);
     return BotCand.SU;
   }
   // Check if the top Q has a better candidate.
-  SchedCandidate TopCand;
-  CandResult TopResult = pickNodeFromQueue(Top.Available,
-                                           DAG->getTopRPTracker(), TopCand);
-  assert(TopResult != NoCand && "failed to find the first candidate");
+  pickNodeFromQueue(Top, DAG->getTopRPTracker(), TopCand);
+  assert(TopCand.Reason != NoCand && "failed to find the first candidate");
 
-  if (TopResult == SingleExcess || TopResult == SingleCritical) {
-    IsTopNode = true;
-    return TopCand.SU;
-  }
   // If either Q has a single candidate that minimizes pressure above the
   // original region's pressure pick it.
-  if (BotResult == SingleMax) {
+  if (TopCand.Reason <= SingleMax || BotCand.Reason <= SingleMax) {
+    if (TopCand.Reason < BotCand.Reason) {
+      IsTopNode = true;
+      tracePick(TopCand, IsTopNode);
+      return TopCand.SU;
+    }
     IsTopNode = false;
+    tracePick(BotCand, IsTopNode);
     return BotCand.SU;
   }
-  if (TopResult == SingleMax) {
+  // Check for a salient pressure difference and pick the best from either side.
+  if (compareRPDelta(TopCand.RPDelta, BotCand.RPDelta)) {
     IsTopNode = true;
+    tracePick(TopCand, IsTopNode);
     return TopCand.SU;
   }
-  // Check for a salient pressure difference and pick the best from either side.
-  if (compareRPDelta(TopCand.RPDelta, BotCand.RPDelta)) {
+  // Otherwise prefer the bottom candidate, in node order if all else failed.
+  if (TopCand.Reason < BotCand.Reason) {
     IsTopNode = true;
+    tracePick(TopCand, IsTopNode);
     return TopCand.SU;
   }
-  // Otherwise prefer the bottom candidate in node order.
   IsTopNode = false;
+  tracePick(BotCand, IsTopNode);
   return BotCand.SU;
 }
 
@@ -1266,33 +1753,34 @@ SUnit *ConvergingScheduler::pickNode(bool &IsTopNode) {
     return NULL;
   }
   SUnit *SU;
-  if (ForceTopDown) {
-    SU = Top.pickOnlyChoice();
-    if (!SU) {
-      SchedCandidate TopCand;
-      CandResult TopResult =
-        pickNodeFromQueue(Top.Available, DAG->getTopRPTracker(), TopCand);
-      assert(TopResult != NoCand && "failed to find the first candidate");
-      (void)TopResult;
-      SU = TopCand.SU;
+  do {
+    if (ForceTopDown) {
+      SU = Top.pickOnlyChoice();
+      if (!SU) {
+        CandPolicy NoPolicy;
+        SchedCandidate TopCand(NoPolicy);
+        pickNodeFromQueue(Top, DAG->getTopRPTracker(), TopCand);
+        assert(TopCand.Reason != NoCand && "failed to find the first candidate");
+        SU = TopCand.SU;
+      }
+      IsTopNode = true;
     }
-    IsTopNode = true;
-  }
-  else if (ForceBottomUp) {
-    SU = Bot.pickOnlyChoice();
-    if (!SU) {
-      SchedCandidate BotCand;
-      CandResult BotResult =
-        pickNodeFromQueue(Bot.Available, DAG->getBotRPTracker(), BotCand);
-      assert(BotResult != NoCand && "failed to find the first candidate");
-      (void)BotResult;
-      SU = BotCand.SU;
+    else if (ForceBottomUp) {
+      SU = Bot.pickOnlyChoice();
+      if (!SU) {
+        CandPolicy NoPolicy;
+        SchedCandidate BotCand(NoPolicy);
+        pickNodeFromQueue(Bot, DAG->getBotRPTracker(), BotCand);
+        assert(BotCand.Reason != NoCand && "failed to find the first candidate");
+        SU = BotCand.SU;
+      }
+      IsTopNode = false;
     }
-    IsTopNode = false;
-  }
-  else {
-    SU = pickNodeBidrectional(IsTopNode);
-  }
+    else {
+      SU = pickNodeBidirectional(IsTopNode);
+    }
+  } while (SU->isScheduled);
+
   if (SU->isTopReady())
     Top.removeReady(SU);
   if (SU->isBottomReady())
@@ -1331,6 +1819,86 @@ ConvergingSchedRegistry("converge", "Standard converging scheduler.",
                         createConvergingSched);
 
 //===----------------------------------------------------------------------===//
+// ILP Scheduler. Currently for experimental analysis of heuristics.
+//===----------------------------------------------------------------------===//
+
+namespace {
+/// \brief Order nodes by the ILP metric.
+struct ILPOrder {
+  ScheduleDAGILP *ILP;
+  bool MaximizeILP;
+
+  ILPOrder(ScheduleDAGILP *ilp, bool MaxILP): ILP(ilp), MaximizeILP(MaxILP) {}
+
+  /// \brief Apply a less-than relation on node priority.
+  bool operator()(const SUnit *A, const SUnit *B) const {
+    // Return true if A comes after B in the Q.
+    if (MaximizeILP)
+      return ILP->getILP(A) < ILP->getILP(B);
+    else
+      return ILP->getILP(A) > ILP->getILP(B);
+  }
+};
+
+/// \brief Schedule based on the ILP metric.
+class ILPScheduler : public MachineSchedStrategy {
+  ScheduleDAGILP ILP;
+  ILPOrder Cmp;
+
+  std::vector<SUnit*> ReadyQ;
+public:
+  ILPScheduler(bool MaximizeILP)
+  : ILP(/*BottomUp=*/true), Cmp(&ILP, MaximizeILP) {}
+
+  virtual void initialize(ScheduleDAGMI *DAG) {
+    ReadyQ.clear();
+    ILP.resize(DAG->SUnits.size());
+  }
+
+  virtual void registerRoots() {
+    for (std::vector<SUnit*>::const_iterator
+           I = ReadyQ.begin(), E = ReadyQ.end(); I != E; ++I) {
+      ILP.computeILP(*I);
+    }
+  }
+
+  /// Implement MachineSchedStrategy interface.
+  /// -----------------------------------------
+
+  virtual SUnit *pickNode(bool &IsTopNode) {
+    if (ReadyQ.empty()) return NULL;
+    pop_heap(ReadyQ.begin(), ReadyQ.end(), Cmp);
+    SUnit *SU = ReadyQ.back();
+    ReadyQ.pop_back();
+    IsTopNode = false;
+    DEBUG(dbgs() << "*** Scheduling " << *SU->getInstr()
+          << " ILP: " << ILP.getILP(SU) << '\n');
+    return SU;
+  }
+
+  virtual void schedNode(SUnit *, bool) {}
+
+  virtual void releaseTopNode(SUnit *) { /*only called for top roots*/ }
+
+  virtual void releaseBottomNode(SUnit *SU) {
+    ReadyQ.push_back(SU);
+    std::push_heap(ReadyQ.begin(), ReadyQ.end(), Cmp);
+  }
+};
+} // namespace
+
+static ScheduleDAGInstrs *createILPMaxScheduler(MachineSchedContext *C) {
+  return new ScheduleDAGMI(C, new ILPScheduler(true));
+}
+static ScheduleDAGInstrs *createILPMinScheduler(MachineSchedContext *C) {
+  return new ScheduleDAGMI(C, new ILPScheduler(false));
+}
+static MachineSchedRegistry ILPMaxRegistry(
+  "ilpmax", "Schedule bottom-up for max ILP", createILPMaxScheduler);
+static MachineSchedRegistry ILPMinRegistry(
+  "ilpmin", "Schedule bottom-up for min ILP", createILPMinScheduler);
+
+//===----------------------------------------------------------------------===//
 // Machine Instruction Shuffler for Correctness Testing
 //===----------------------------------------------------------------------===//
 
diff --git a/lib/CodeGen/MachineSink.cpp b/lib/CodeGen/MachineSink.cpp
index bc383cb..b117f8c 100644
--- a/lib/CodeGen/MachineSink.cpp
+++ b/lib/CodeGen/MachineSink.cpp
@@ -49,7 +49,6 @@ namespace {
     MachineDominatorTree *DT;   // Machine dominator tree
     MachineLoopInfo *LI;
     AliasAnalysis *AA;
-    BitVector AllocatableSet;   // Which physregs are allocatable?
 
     // Remember which edges have been considered for breaking.
     SmallSet<std::pair<MachineBasicBlock*,MachineBasicBlock*>, 8>
@@ -229,7 +228,6 @@ bool MachineSinking::runOnMachineFunction(MachineFunction &MF) {
   DT = &getAnalysis<MachineDominatorTree>();
   LI = &getAnalysis<MachineLoopInfo>();
   AA = &getAnalysis<AliasAnalysis>();
-  AllocatableSet = TRI->getAllocatableSet(MF);
 
   bool EverMadeChange = false;
 
diff --git a/lib/CodeGen/MachineTraceMetrics.cpp b/lib/CodeGen/MachineTraceMetrics.cpp
index 1a3aa60..9686b04 100644
--- a/lib/CodeGen/MachineTraceMetrics.cpp
+++ b/lib/CodeGen/MachineTraceMetrics.cpp
@@ -14,9 +14,10 @@
 #include "llvm/CodeGen/MachineLoopInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/Passes.h"
-#include "llvm/MC/MCInstrItineraries.h"
+#include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/Target/TargetInstrInfo.h"
 #include "llvm/Target/TargetRegisterInfo.h"
+#include "llvm/Target/TargetSubtargetInfo.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/ADT/PostOrderIterator.h"
@@ -50,9 +51,11 @@ bool MachineTraceMetrics::runOnMachineFunction(MachineFunction &Func) {
   MF = &Func;
   TII = MF->getTarget().getInstrInfo();
   TRI = MF->getTarget().getRegisterInfo();
-  ItinData = MF->getTarget().getInstrItineraryData();
   MRI = &MF->getRegInfo();
   Loops = &getAnalysis<MachineLoopInfo>();
+  const TargetSubtargetInfo &ST =
+    MF->getTarget().getSubtarget<TargetSubtargetInfo>();
+  SchedModel.init(*ST.getSchedModel(), &ST, TII);
   BlockInfo.resize(MF->getNumBlockIDs());
   return false;
 }
@@ -674,7 +677,7 @@ computeCrossBlockCriticalPath(const TraceBlockInfo &TBI) {
     const MachineInstr *DefMI = MTM.MRI->getVRegDef(LIR.Reg);
     // Ignore dependencies outside the current trace.
     const TraceBlockInfo &DefTBI = BlockInfo[DefMI->getParent()->getNumber()];
-    if (!DefTBI.hasValidDepth() || DefTBI.Head != TBI.Head)
+    if (!DefTBI.isEarlierInSameTrace(TBI))
       continue;
     unsigned Len = LIR.Height + Cycles[DefMI].Depth;
     MaxLen = std::max(MaxLen, Len);
@@ -737,16 +740,15 @@ computeInstrDepths(const MachineBasicBlock *MBB) {
         const TraceBlockInfo&DepTBI =
           BlockInfo[Dep.DefMI->getParent()->getNumber()];
         // Ignore dependencies from outside the current trace.
-        if (!DepTBI.hasValidDepth() || DepTBI.Head != TBI.Head)
+        if (!DepTBI.isEarlierInSameTrace(TBI))
           continue;
         assert(DepTBI.HasValidInstrDepths && "Inconsistent dependency");
         unsigned DepCycle = Cycles.lookup(Dep.DefMI).Depth;
         // Add latency if DefMI is a real instruction. Transients get latency 0.
         if (!Dep.DefMI->isTransient())
-          DepCycle += MTM.TII->computeOperandLatency(MTM.ItinData,
-                                                     Dep.DefMI, Dep.DefOp,
-                                                     UseMI, Dep.UseOp,
-                                                     /* FindMin = */ false);
+          DepCycle += MTM.SchedModel
+            .computeOperandLatency(Dep.DefMI, Dep.DefOp, UseMI, Dep.UseOp,
+                                   /* FindMin = */ false);
         Cycle = std::max(Cycle, DepCycle);
       }
       // Remember the instruction depth.
@@ -769,7 +771,7 @@ computeInstrDepths(const MachineBasicBlock *MBB) {
 // Height is the issue height computed from virtual register dependencies alone.
 static unsigned updatePhysDepsUpwards(const MachineInstr *MI, unsigned Height,
                                       SparseSet<LiveRegUnit> &RegUnits,
-                                      const InstrItineraryData *ItinData,
+                                      const TargetSchedModel &SchedModel,
                                       const TargetInstrInfo *TII,
                                       const TargetRegisterInfo *TRI) {
   SmallVector<unsigned, 8> ReadOps;
@@ -792,14 +794,10 @@ static unsigned updatePhysDepsUpwards(const MachineInstr *MI, unsigned Height,
       unsigned DepHeight = I->Cycle;
       if (!MI->isTransient()) {
         // We may not know the UseMI of this dependency, if it came from the
-        // live-in list.
-        if (I->MI)
-          DepHeight += TII->computeOperandLatency(ItinData,
-                                                  MI, MO.getOperandNo(),
-                                                  I->MI, I->Op);
-        else
-          // No UseMI. Just use the MI latency instead.
-          DepHeight += TII->getInstrLatency(ItinData, MI);
+        // live-in list. SchedModel can handle a NULL UseMI.
+        DepHeight += SchedModel
+          .computeOperandLatency(MI, MO.getOperandNo(), I->MI, I->Op,
+                                 /* FindMin = */ false);
       }
       Height = std::max(Height, DepHeight);
       // This regunit is dead above MI.
@@ -832,12 +830,12 @@ typedef DenseMap<const MachineInstr *, unsigned> MIHeightMap;
 static bool pushDepHeight(const DataDep &Dep,
                           const MachineInstr *UseMI, unsigned UseHeight,
                           MIHeightMap &Heights,
-                          const InstrItineraryData *ItinData,
+                          const TargetSchedModel &SchedModel,
                           const TargetInstrInfo *TII) {
   // Adjust height by Dep.DefMI latency.
   if (!Dep.DefMI->isTransient())
-    UseHeight += TII->computeOperandLatency(ItinData, Dep.DefMI, Dep.DefOp,
-                                            UseMI, Dep.UseOp);
+    UseHeight += SchedModel.computeOperandLatency(Dep.DefMI, Dep.DefOp,
+                                                  UseMI, Dep.UseOp, false);
 
   // Update Heights[DefMI] to be the maximum height seen.
   MIHeightMap::iterator I;
@@ -852,14 +850,14 @@ static bool pushDepHeight(const DataDep &Dep,
   return false;
 }
 
-/// Assuming that DefMI was used by Trace.back(), add it to the live-in lists
-/// of all the blocks in Trace. Stop when reaching the block that contains
-/// DefMI.
+/// Assuming that the virtual register defined by DefMI:DefOp was used by
+/// Trace.back(), add it to the live-in lists of all the blocks in Trace. Stop
+/// when reaching the block that contains DefMI.
 void MachineTraceMetrics::Ensemble::
-addLiveIns(const MachineInstr *DefMI,
+addLiveIns(const MachineInstr *DefMI, unsigned DefOp,
            ArrayRef<const MachineBasicBlock*> Trace) {
   assert(!Trace.empty() && "Trace should contain at least one block");
-  unsigned Reg = DefMI->getOperand(0).getReg();
+  unsigned Reg = DefMI->getOperand(DefOp).getReg();
   assert(TargetRegisterInfo::isVirtualRegister(Reg));
   const MachineBasicBlock *DefMBB = DefMI->getParent();
 
@@ -951,8 +949,8 @@ computeInstrHeights(const MachineBasicBlock *MBB) {
           unsigned Height = TBI.Succ ? Cycles.lookup(PHI).Height : 0;
           DEBUG(dbgs() << "pred\t" << Height << '\t' << *PHI);
           if (pushDepHeight(Deps.front(), PHI, Height,
-                            Heights, MTM.ItinData, MTM.TII))
-            addLiveIns(Deps.front().DefMI, Stack);
+                            Heights, MTM.SchedModel, MTM.TII))
+            addLiveIns(Deps.front().DefMI, Deps.front().DefOp, Stack);
         }
       }
     }
@@ -980,12 +978,12 @@ computeInstrHeights(const MachineBasicBlock *MBB) {
       // There may also be regunit dependencies to include in the height.
       if (HasPhysRegs)
         Cycle = updatePhysDepsUpwards(MI, Cycle, RegUnits,
-                                      MTM.ItinData, MTM.TII, MTM.TRI);
+                                      MTM.SchedModel, MTM.TII, MTM.TRI);
 
       // Update the required height of any virtual registers read by MI.
       for (unsigned i = 0, e = Deps.size(); i != e; ++i)
-        if (pushDepHeight(Deps[i], MI, Cycle, Heights, MTM.ItinData, MTM.TII))
-          addLiveIns(Deps[i].DefMI, Stack);
+        if (pushDepHeight(Deps[i], MI, Cycle, Heights, MTM.SchedModel, MTM.TII))
+          addLiveIns(Deps[i].DefMI, Deps[i].DefOp, Stack);
 
       InstrCycles &MICycles = Cycles[MI];
       MICycles.Height = Cycle;
@@ -1054,10 +1052,8 @@ MachineTraceMetrics::Trace::getPHIDepth(const MachineInstr *PHI) const {
   unsigned DepCycle = getInstrCycles(Dep.DefMI).Depth;
   // Add latency if DefMI is a real instruction. Transients get latency 0.
   if (!Dep.DefMI->isTransient())
-    DepCycle += TE.MTM.TII->computeOperandLatency(TE.MTM.ItinData,
-                                                  Dep.DefMI, Dep.DefOp,
-                                                  PHI, Dep.UseOp,
-                                                  /* FindMin = */ false);
+    DepCycle += TE.MTM.SchedModel
+      .computeOperandLatency(Dep.DefMI, Dep.DefOp, PHI, Dep.UseOp, false);
   return DepCycle;
 }
 
@@ -1068,9 +1064,8 @@ unsigned MachineTraceMetrics::Trace::getResourceDepth(bool Bottom) const {
   unsigned Instrs = TBI.InstrDepth;
   if (Bottom)
     Instrs += TE.MTM.BlockInfo[getBlockNum()].InstrCount;
-  if (const MCSchedModel *Model = TE.MTM.ItinData->SchedModel)
-    if (Model->IssueWidth != 0)
-      return Instrs / Model->IssueWidth;
+  if (unsigned IW = TE.MTM.SchedModel.getIssueWidth())
+    Instrs /= IW;
   // Assume issue width 1 without a schedule model.
   return Instrs;
 }
@@ -1080,9 +1075,8 @@ getResourceLength(ArrayRef<const MachineBasicBlock*> Extrablocks) const {
   unsigned Instrs = TBI.InstrDepth + TBI.InstrHeight;
   for (unsigned i = 0, e = Extrablocks.size(); i != e; ++i)
     Instrs += TE.MTM.getResources(Extrablocks[i])->InstrCount;
-  if (const MCSchedModel *Model = TE.MTM.ItinData->SchedModel)
-    if (Model->IssueWidth != 0)
-      return Instrs / Model->IssueWidth;
+  if (unsigned IW = TE.MTM.SchedModel.getIssueWidth())
+    Instrs /= IW;
   // Assume issue width 1 without a schedule model.
   return Instrs;
 }
diff --git a/lib/CodeGen/MachineTraceMetrics.h b/lib/CodeGen/MachineTraceMetrics.h
index c5b86f3..460730b 100644
--- a/lib/CodeGen/MachineTraceMetrics.h
+++ b/lib/CodeGen/MachineTraceMetrics.h
@@ -50,6 +50,7 @@
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/TargetSchedule.h"
 
 namespace llvm {
 
@@ -67,9 +68,9 @@ class MachineTraceMetrics : public MachineFunctionPass {
   const MachineFunction *MF;
   const TargetInstrInfo *TII;
   const TargetRegisterInfo *TRI;
-  const InstrItineraryData *ItinData;
   const MachineRegisterInfo *MRI;
   const MachineLoopInfo *Loops;
+  TargetSchedModel SchedModel;
 
 public:
   class Ensemble;
@@ -164,6 +165,14 @@ public:
     /// Invalidate height resources when a block below this one has changed.
     void invalidateHeight() { InstrHeight = ~0u; HasValidInstrHeights = false; }
 
+    /// Determine if this block belongs to the same trace as TBI and comes
+    /// before it in the trace.
+    /// Also returns true when TBI == this.
+    bool isEarlierInSameTrace(const TraceBlockInfo &TBI) const {
+      return hasValidDepth() && TBI.hasValidDepth() &&
+        Head == TBI.Head && InstrDepth <= TBI.InstrDepth;
+    }
+
     // Data-dependency-related information. Per-instruction depth and height
     // are computed from data dependencies in the current trace, using
     // itinerary data.
@@ -270,7 +279,7 @@ public:
     unsigned computeCrossBlockCriticalPath(const TraceBlockInfo&);
     void computeInstrDepths(const MachineBasicBlock*);
     void computeInstrHeights(const MachineBasicBlock*);
-    void addLiveIns(const MachineInstr *DefMI,
+    void addLiveIns(const MachineInstr *DefMI, unsigned DefOp,
                     ArrayRef<const MachineBasicBlock*> Trace);
 
   protected:
diff --git a/lib/CodeGen/MachineVerifier.cpp b/lib/CodeGen/MachineVerifier.cpp
index f745b41..69a3ae8 100644
--- a/lib/CodeGen/MachineVerifier.cpp
+++ b/lib/CodeGen/MachineVerifier.cpp
@@ -23,8 +23,9 @@
 // the verifier errors.
 //===----------------------------------------------------------------------===//
 
+#include "llvm/BasicBlock.h"
+#include "llvm/InlineAsm.h"
 #include "llvm/Instructions.h"
-#include "llvm/Function.h"
 #include "llvm/CodeGen/LiveIntervalAnalysis.h"
 #include "llvm/CodeGen/LiveVariables.h"
 #include "llvm/CodeGen/LiveStackAnalysis.h"
@@ -73,11 +74,12 @@ namespace {
     typedef SmallVector<const uint32_t*, 4> RegMaskVector;
     typedef DenseSet<unsigned> RegSet;
     typedef DenseMap<unsigned, const MachineInstr*> RegMap;
+    typedef SmallPtrSet<const MachineBasicBlock*, 8> BlockSet;
 
     const MachineInstr *FirstTerminator;
+    BlockSet FunctionBlocks;
 
     BitVector regsReserved;
-    BitVector regsAllocatable;
     RegSet regsLive;
     RegVector regsDefined, regsDead, regsKilled;
     RegMaskVector regMasks;
@@ -117,6 +119,9 @@ namespace {
       // block. This set is disjoint from regsLiveOut.
       RegSet vregsRequired;
 
+      // Set versions of block's predecessor and successor lists.
+      BlockSet Preds, Succs;
+
       BBInfo() : reachable(false) {}
 
       // Add register to vregsPassed if it belongs there. Return true if
@@ -180,7 +185,7 @@ namespace {
     }
 
     bool isAllocatable(unsigned Reg) {
-      return Reg < regsAllocatable.size() && regsAllocatable.test(Reg);
+      return Reg < TRI->getNumRegs() && MRI->isAllocatable(Reg);
     }
 
     // Analysis information if available
@@ -208,6 +213,8 @@ namespace {
     void report(const char *msg, const MachineBasicBlock *MBB,
                 const LiveInterval &LI);
 
+    void verifyInlineAsm(const MachineInstr *MI);
+
     void checkLiveness(const MachineOperand *MO, unsigned MONum);
     void markReachable(const MachineBasicBlock *MBB);
     void calcRegsPassed();
@@ -352,7 +359,7 @@ void MachineVerifier::report(const char *msg, const MachineFunction *MF) {
     MF->print(*OS, Indexes);
   }
   *OS << "*** Bad machine code: " << msg << " ***\n"
-      << "- function:    " << MF->getFunction()->getName() << "\n";
+      << "- function:    " << MF->getName() << "\n";
 }
 
 void MachineVerifier::report(const char *msg, const MachineBasicBlock *MBB) {
@@ -360,7 +367,7 @@ void MachineVerifier::report(const char *msg, const MachineBasicBlock *MBB) {
   report(msg, MBB->getParent());
   *OS << "- basic block: BB#" << MBB->getNumber()
       << ' ' << MBB->getName()
-      << " (" << (void*)MBB << ')';
+      << " (" << (const void*)MBB << ')';
   if (Indexes)
     *OS << " [" << Indexes->getMBBStartIdx(MBB)
         << ';' <<  Indexes->getMBBEndIdx(MBB) << ')';
@@ -419,7 +426,7 @@ void MachineVerifier::markReachable(const MachineBasicBlock *MBB) {
 
 void MachineVerifier::visitMachineFunctionBefore() {
   lastIndex = SlotIndex();
-  regsReserved = TRI->getReservedRegs(*MF);
+  regsReserved = MRI->getReservedRegs();
 
   // A sub-register of a reserved register is also reserved
   for (int Reg = regsReserved.find_first(); Reg>=0;
@@ -431,9 +438,23 @@ void MachineVerifier::visitMachineFunctionBefore() {
     }
   }
 
-  regsAllocatable = TRI->getAllocatableSet(*MF);
-
   markReachable(&MF->front());
+
+  // Build a set of the basic blocks in the function.
+  FunctionBlocks.clear();
+  for (MachineFunction::const_iterator
+       I = MF->begin(), E = MF->end(); I != E; ++I) {
+    FunctionBlocks.insert(I);
+    BBInfo &MInfo = MBBInfoMap[I];
+
+    MInfo.Preds.insert(I->pred_begin(), I->pred_end());
+    if (MInfo.Preds.size() != I->pred_size())
+      report("MBB has duplicate entries in its predecessor list.", I);
+
+    MInfo.Succs.insert(I->succ_begin(), I->succ_end());
+    if (MInfo.Succs.size() != I->succ_size())
+      report("MBB has duplicate entries in its successor list.", I);
+  }
 }
 
 // Does iterator point to a and b as the first two elements?
@@ -470,6 +491,25 @@ MachineVerifier::visitMachineBasicBlockBefore(const MachineBasicBlock *MBB) {
        E = MBB->succ_end(); I != E; ++I) {
     if ((*I)->isLandingPad())
       LandingPadSuccs.insert(*I);
+    if (!FunctionBlocks.count(*I))
+      report("MBB has successor that isn't part of the function.", MBB);
+    if (!MBBInfoMap[*I].Preds.count(MBB)) {
+      report("Inconsistent CFG", MBB);
+      *OS << "MBB is not in the predecessor list of the successor BB#"
+          << (*I)->getNumber() << ".\n";
+    }
+  }
+
+  // Check the predecessor list.
+  for (MachineBasicBlock::const_pred_iterator I = MBB->pred_begin(),
+       E = MBB->pred_end(); I != E; ++I) {
+    if (!FunctionBlocks.count(*I))
+      report("MBB has predecessor that isn't part of the function.", MBB);
+    if (!MBBInfoMap[*I].Succs.count(MBB)) {
+      report("Inconsistent CFG", MBB);
+      *OS << "MBB is not in the successor list of the predecessor BB#"
+          << (*I)->getNumber() << ".\n";
+    }
   }
 
   const MCAsmInfo *AsmInfo = TM->getMCAsmInfo();
@@ -540,7 +580,15 @@ MachineVerifier::visitMachineBasicBlockBefore(const MachineBasicBlock *MBB) {
       ++MBBI;
       if (MBBI == MF->end()) {
         report("MBB conditionally falls through out of function!", MBB);
-      } if (MBB->succ_size() != 2) {
+      } if (MBB->succ_size() == 1) {
+        // A conditional branch with only one successor is weird, but allowed.
+        if (&*MBBI != TBB)
+          report("MBB exits via conditional branch/fall-through but only has "
+                 "one CFG successor!", MBB);
+        else if (TBB != *MBB->succ_begin())
+          report("MBB exits via conditional branch/fall-through but the CFG "
+                 "successor don't match the actual successor!", MBB);
+      } else if (MBB->succ_size() != 2) {
         report("MBB exits via conditional branch/fall-through but doesn't have "
                "exactly two CFG successors!", MBB);
       } else if (!matchPair(MBB->succ_begin(), TBB, MBBI)) {
@@ -560,7 +608,15 @@ MachineVerifier::visitMachineBasicBlockBefore(const MachineBasicBlock *MBB) {
     } else if (TBB && FBB) {
       // Block conditionally branches somewhere, otherwise branches
       // somewhere else.
-      if (MBB->succ_size() != 2) {
+      if (MBB->succ_size() == 1) {
+        // A conditional branch with only one successor is weird, but allowed.
+        if (FBB != TBB)
+          report("MBB exits via conditional branch/branch through but only has "
+                 "one CFG successor!", MBB);
+        else if (TBB != *MBB->succ_begin())
+          report("MBB exits via conditional branch/branch through but the CFG "
+                 "successor don't match the actual successor!", MBB);
+      } else if (MBB->succ_size() != 2) {
         report("MBB exits via conditional branch/branch but doesn't have "
                "exactly two CFG successors!", MBB);
       } else if (!matchPair(MBB->succ_begin(), TBB, FBB)) {
@@ -639,6 +695,50 @@ void MachineVerifier::visitMachineBundleBefore(const MachineInstr *MI) {
   }
 }
 
+// The operands on an INLINEASM instruction must follow a template.
+// Verify that the flag operands make sense.
+void MachineVerifier::verifyInlineAsm(const MachineInstr *MI) {
+  // The first two operands on INLINEASM are the asm string and global flags.
+  if (MI->getNumOperands() < 2) {
+    report("Too few operands on inline asm", MI);
+    return;
+  }
+  if (!MI->getOperand(0).isSymbol())
+    report("Asm string must be an external symbol", MI);
+  if (!MI->getOperand(1).isImm())
+    report("Asm flags must be an immediate", MI);
+  // Allowed flags are Extra_HasSideEffects = 1, Extra_IsAlignStack = 2,
+  // Extra_AsmDialect = 4, Extra_MayLoad = 8, and Extra_MayStore = 16.
+  if (!isUInt<5>(MI->getOperand(1).getImm()))
+    report("Unknown asm flags", &MI->getOperand(1), 1);
+
+  assert(InlineAsm::MIOp_FirstOperand == 2 && "Asm format changed");
+
+  unsigned OpNo = InlineAsm::MIOp_FirstOperand;
+  unsigned NumOps;
+  for (unsigned e = MI->getNumOperands(); OpNo < e; OpNo += NumOps) {
+    const MachineOperand &MO = MI->getOperand(OpNo);
+    // There may be implicit ops after the fixed operands.
+    if (!MO.isImm())
+      break;
+    NumOps = 1 + InlineAsm::getNumOperandRegisters(MO.getImm());
+  }
+
+  if (OpNo > MI->getNumOperands())
+    report("Missing operands in last group", MI);
+
+  // An optional MDNode follows the groups.
+  if (OpNo < MI->getNumOperands() && MI->getOperand(OpNo).isMetadata())
+    ++OpNo;
+
+  // All trailing operands must be implicit registers.
+  for (unsigned e = MI->getNumOperands(); OpNo < e; ++OpNo) {
+    const MachineOperand &MO = MI->getOperand(OpNo);
+    if (!MO.isReg() || !MO.isImplicit())
+      report("Expected implicit register after groups", &MO, OpNo);
+  }
+}
+
 void MachineVerifier::visitMachineInstrBefore(const MachineInstr *MI) {
   const MCInstrDesc &MCID = MI->getDesc();
   if (MI->getNumOperands() < MCID.getNumOperands()) {
@@ -647,6 +747,10 @@ void MachineVerifier::visitMachineInstrBefore(const MachineInstr *MI) {
         << MI->getNumExplicitOperands() << " given.\n";
   }
 
+  // Check the tied operands.
+  if (MI->isInlineAsm())
+    verifyInlineAsm(MI);
+
   // Check the MachineMemOperands for basic consistency.
   for (MachineInstr::mmo_iterator I = MI->memoperands_begin(),
        E = MI->memoperands_end(); I != E; ++I) {
@@ -702,6 +806,17 @@ MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) {
       if (MO->isImplicit())
         report("Explicit operand marked as implicit", MO, MONum);
     }
+
+    int TiedTo = MCID.getOperandConstraint(MONum, MCOI::TIED_TO);
+    if (TiedTo != -1) {
+      if (!MO->isReg())
+        report("Tied use must be a register", MO, MONum);
+      else if (!MO->isTied())
+        report("Operand should be tied", MO, MONum);
+      else if (unsigned(TiedTo) != MI->findTiedOperandIdx(MONum))
+        report("Tied def doesn't match MCInstrDesc", MO, MONum);
+    } else if (MO->isReg() && MO->isTied())
+      report("Explicit operand should not be tied", MO, MONum);
   } else {
     // ARM adds %reg0 operands to indicate predicates. We'll allow that.
     if (MO->isReg() && !MO->isImplicit() && !MI->isVariadic() && MO->getReg())
@@ -716,6 +831,28 @@ MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) {
     if (MRI->tracksLiveness() && !MI->isDebugValue())
       checkLiveness(MO, MONum);
 
+    // Verify the consistency of tied operands.
+    if (MO->isTied()) {
+      unsigned OtherIdx = MI->findTiedOperandIdx(MONum);
+      const MachineOperand &OtherMO = MI->getOperand(OtherIdx);
+      if (!OtherMO.isReg())
+        report("Must be tied to a register", MO, MONum);
+      if (!OtherMO.isTied())
+        report("Missing tie flags on tied operand", MO, MONum);
+      if (MI->findTiedOperandIdx(OtherIdx) != MONum)
+        report("Inconsistent tie links", MO, MONum);
+      if (MONum < MCID.getNumDefs()) {
+        if (OtherIdx < MCID.getNumOperands()) {
+          if (-1 == MCID.getOperandConstraint(OtherIdx, MCOI::TIED_TO))
+            report("Explicit def tied to explicit use without tie constraint",
+                   MO, MONum);
+        } else {
+          if (!OtherMO.isImplicit())
+            report("Explicit def should be tied to implicit use", MO, MONum);
+        }
+      }
+    }
+
     // Verify two-address constraints after leaving SSA form.
     unsigned DefIdx;
     if (!MRI->isSSA() && MO->isUse() &&
diff --git a/lib/CodeGen/Passes.cpp b/lib/CodeGen/Passes.cpp
index cfa3eec..4ea21d4 100644
--- a/lib/CodeGen/Passes.cpp
+++ b/lib/CodeGen/Passes.cpp
@@ -49,8 +49,8 @@ static cl::opt<bool> DisableSSC("disable-ssc", cl::Hidden,
     cl::desc("Disable Stack Slot Coloring"));
 static cl::opt<bool> DisableMachineDCE("disable-machine-dce", cl::Hidden,
     cl::desc("Disable Machine Dead Code Elimination"));
-static cl::opt<bool> EnableEarlyIfConversion("enable-early-ifcvt", cl::Hidden,
-    cl::desc("Enable Early If-conversion"));
+static cl::opt<bool> DisableEarlyIfConversion("disable-early-ifcvt", cl::Hidden,
+    cl::desc("Disable Early If-conversion"));
 static cl::opt<bool> DisableMachineLICM("disable-machine-licm", cl::Hidden,
     cl::desc("Disable Machine LICM"));
 static cl::opt<bool> DisableMachineCSE("disable-machine-cse", cl::Hidden,
@@ -161,7 +161,7 @@ static AnalysisID overridePass(AnalysisID StandardID, AnalysisID TargetID) {
     return applyDisable(TargetID, DisableMachineDCE);
 
   if (StandardID == &EarlyIfConverterID)
-    return applyDisable(TargetID, !EnableEarlyIfConversion);
+    return applyDisable(TargetID, DisableEarlyIfConversion);
 
   if (StandardID == &MachineLICMID)
     return applyDisable(TargetID, DisableMachineLICM);
@@ -447,8 +447,8 @@ void TargetPassConfig::addMachinePasses() {
     const PassInfo *TPI = PR->getPassInfo(PrintMachineInstrs.getValue());
     const PassInfo *IPI = PR->getPassInfo(StringRef("print-machineinstrs"));
     assert (TPI && IPI && "Pass ID not registered!");
-    const char *TID = (char *)(TPI->getTypeInfo());
-    const char *IID = (char *)(IPI->getTypeInfo());
+    const char *TID = (const char *)(TPI->getTypeInfo());
+    const char *IID = (const char *)(IPI->getTypeInfo());
     insertPass(TID, IID);
   }
 
@@ -456,7 +456,8 @@ void TargetPassConfig::addMachinePasses() {
   printAndVerify("After Instruction Selection");
 
   // Expand pseudo-instructions emitted by ISel.
-  addPass(&ExpandISelPseudosID);
+  if (addPass(&ExpandISelPseudosID))
+    printAndVerify("After ExpandISelPseudos");
 
   // Add passes that optimize machine instructions in SSA form.
   if (getOptLevel() != CodeGenOpt::None) {
@@ -528,6 +529,10 @@ void TargetPassConfig::addMachineSSAOptimization() {
   // instructions dead.
   addPass(&OptimizePHIsID);
 
+  // This pass merges large allocas. StackSlotColoring is a different pass
+  // which merges spill slots.
+  addPass(&StackColoringID);
+
   // If the target requests it, assign local variables to stack slots relative
   // to one another and simplify frame index references where possible.
   addPass(&LocalStackSlotAllocationID);
diff --git a/lib/CodeGen/PeepholeOptimizer.cpp b/lib/CodeGen/PeepholeOptimizer.cpp
index 9099862..a795ac8 100644
--- a/lib/CodeGen/PeepholeOptimizer.cpp
+++ b/lib/CodeGen/PeepholeOptimizer.cpp
@@ -527,6 +527,11 @@ bool PeepholeOptimizer::runOnMachineFunction(MachineFunction &MF) {
         SeenMoveImm = true;
       } else {
         Changed |= optimizeExtInstr(MI, MBB, LocalMIs);
+        // optimizeExtInstr might have created new instructions after MI
+        // and before the already incremented MII. Adjust MII so that the
+        // next iteration sees the new instructions.
+        MII = MI;
+        ++MII;
         if (SeenMoveImm)
           Changed |= foldImmediate(MI, MBB, ImmDefRegs, ImmDefMIs);
       }
diff --git a/lib/CodeGen/PostRASchedulerList.cpp b/lib/CodeGen/PostRASchedulerList.cpp
index 7449ff5..d57bc73 100644
--- a/lib/CodeGen/PostRASchedulerList.cpp
+++ b/lib/CodeGen/PostRASchedulerList.cpp
@@ -240,6 +240,7 @@ void SchedulePostRATDList::exitRegion() {
   ScheduleDAGInstrs::exitRegion();
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 /// dumpSchedule - dump the scheduled Sequence.
 void SchedulePostRATDList::dumpSchedule() const {
   for (unsigned i = 0, e = Sequence.size(); i != e; i++) {
@@ -249,6 +250,7 @@ void SchedulePostRATDList::dumpSchedule() const {
       dbgs() << "**** NOOP ****\n";
   }
 }
+#endif
 
 bool PostRAScheduler::runOnMachineFunction(MachineFunction &Fn) {
   TII = Fn.getTarget().getInstrInfo();
@@ -298,7 +300,7 @@ bool PostRAScheduler::runOnMachineFunction(MachineFunction &Fn) {
       static int bbcnt = 0;
       if (bbcnt++ % DebugDiv != DebugMod)
         continue;
-      dbgs() << "*** DEBUG scheduling " << Fn.getFunction()->getName()
+      dbgs() << "*** DEBUG scheduling " << Fn.getName()
              << ":BB#" << MBB->getNumber() << " ***\n";
     }
 #endif
@@ -488,7 +490,6 @@ void SchedulePostRATDList::FixupKills(MachineBasicBlock *MBB) {
   DEBUG(dbgs() << "Fixup kills for BB#" << MBB->getNumber() << '\n');
 
   BitVector killedRegs(TRI->getNumRegs());
-  BitVector ReservedRegs = TRI->getReservedRegs(MF);
 
   StartBlockForKills(MBB);
 
@@ -529,7 +530,7 @@ void SchedulePostRATDList::FixupKills(MachineBasicBlock *MBB) {
       MachineOperand &MO = MI->getOperand(i);
       if (!MO.isReg() || !MO.isUse()) continue;
       unsigned Reg = MO.getReg();
-      if ((Reg == 0) || ReservedRegs.test(Reg)) continue;
+      if ((Reg == 0) || MRI.isReserved(Reg)) continue;
 
       bool kill = false;
       if (!killedRegs.test(Reg)) {
@@ -564,7 +565,7 @@ void SchedulePostRATDList::FixupKills(MachineBasicBlock *MBB) {
       MachineOperand &MO = MI->getOperand(i);
       if (!MO.isReg() || !MO.isUse() || MO.isUndef()) continue;
       unsigned Reg = MO.getReg();
-      if ((Reg == 0) || ReservedRegs.test(Reg)) continue;
+      if ((Reg == 0) || MRI.isReserved(Reg)) continue;
 
       LiveRegs.set(Reg);
 
diff --git a/lib/CodeGen/ProcessImplicitDefs.cpp b/lib/CodeGen/ProcessImplicitDefs.cpp
index 34d075c..e4e18c3 100644
--- a/lib/CodeGen/ProcessImplicitDefs.cpp
+++ b/lib/CodeGen/ProcessImplicitDefs.cpp
@@ -137,8 +137,7 @@ void ProcessImplicitDefs::processImplicitDef(MachineInstr *MI) {
 bool ProcessImplicitDefs::runOnMachineFunction(MachineFunction &MF) {
 
   DEBUG(dbgs() << "********** PROCESS IMPLICIT DEFS **********\n"
-               << "********** Function: "
-               << ((Value*)MF.getFunction())->getName() << '\n');
+               << "********** Function: " << MF.getName() << '\n');
 
   bool Changed = false;
 
diff --git a/lib/CodeGen/PrologEpilogInserter.cpp b/lib/CodeGen/PrologEpilogInserter.cpp
index c791ffb..77554d6 100644
--- a/lib/CodeGen/PrologEpilogInserter.cpp
+++ b/lib/CodeGen/PrologEpilogInserter.cpp
@@ -96,7 +96,7 @@ bool PEI::runOnMachineFunction(MachineFunction &Fn) {
   placeCSRSpillsAndRestores(Fn);
 
   // Add the code to save and restore the callee saved registers
-  if (!F->hasFnAttr(Attribute::Naked))
+  if (!F->getFnAttributes().hasAttribute(Attributes::Naked))
     insertCSRSpillsAndRestores(Fn);
 
   // Allow the target machine to make final modifications to the function
@@ -111,7 +111,7 @@ bool PEI::runOnMachineFunction(MachineFunction &Fn) {
   // called functions.  Because of this, calculateCalleeSavedRegisters()
   // must be called before this function in order to set the AdjustsStack
   // and MaxCallFrameSize variables.
-  if (!F->hasFnAttr(Attribute::Naked))
+  if (!F->getFnAttributes().hasAttribute(Attributes::Naked))
     insertPrologEpilogCode(Fn);
 
   // Replace all MO_FrameIndex operands with physical register references
@@ -221,13 +221,13 @@ void PEI::calculateCalleeSavedRegisters(MachineFunction &Fn) {
     return;
 
   // In Naked functions we aren't going to save any registers.
-  if (Fn.getFunction()->hasFnAttr(Attribute::Naked))
+  if (Fn.getFunction()->getFnAttributes().hasAttribute(Attributes::Naked))
     return;
 
   std::vector<CalleeSavedInfo> CSI;
   for (unsigned i = 0; CSRegs[i]; ++i) {
     unsigned Reg = CSRegs[i];
-    if (Fn.getRegInfo().isPhysRegOrOverlapUsed(Reg)) {
+    if (Fn.getRegInfo().isPhysRegUsed(Reg)) {
       // If the reg is modified, save it!
       CSI.push_back(CalleeSavedInfo(Reg));
     }
diff --git a/lib/CodeGen/RegAllocBasic.cpp b/lib/CodeGen/RegAllocBasic.cpp
index 3a03807..8a49609 100644
--- a/lib/CodeGen/RegAllocBasic.cpp
+++ b/lib/CodeGen/RegAllocBasic.cpp
@@ -20,7 +20,6 @@
 #include "VirtRegMap.h"
 #include "LiveRegMatrix.h"
 #include "llvm/Analysis/AliasAnalysis.h"
-#include "llvm/Function.h"
 #include "llvm/PassAnalysisSupport.h"
 #include "llvm/CodeGen/CalcSpillWeights.h"
 #include "llvm/CodeGen/LiveIntervalAnalysis.h"
@@ -273,7 +272,7 @@ unsigned RABasic::selectOrSplit(LiveInterval &VirtReg,
 bool RABasic::runOnMachineFunction(MachineFunction &mf) {
   DEBUG(dbgs() << "********** BASIC REGISTER ALLOCATION **********\n"
                << "********** Function: "
-               << ((Value*)mf.getFunction())->getName() << '\n');
+               << mf.getName() << '\n');
 
   MF = &mf;
   RegAllocBase::init(getAnalysis<VirtRegMap>(),
diff --git a/lib/CodeGen/RegAllocFast.cpp b/lib/CodeGen/RegAllocFast.cpp
index 6b3a48e..8892216 100644
--- a/lib/CodeGen/RegAllocFast.cpp
+++ b/lib/CodeGen/RegAllocFast.cpp
@@ -113,9 +113,11 @@ namespace {
     // PhysRegState - One of the RegState enums, or a virtreg.
     std::vector<unsigned> PhysRegState;
 
-    // UsedInInstr - BitVector of physregs that are used in the current
-    // instruction, and so cannot be allocated.
-    BitVector UsedInInstr;
+    typedef SparseSet<unsigned> UsedInInstrSet;
+
+    // UsedInInstr - Set of physregs that are used in the current instruction,
+    // and so cannot be allocated.
+    UsedInInstrSet UsedInInstr;
 
     // SkippedInstrs - Descriptors of instructions whose clobber list was
     // ignored because all registers were spilled. It is still necessary to
@@ -173,7 +175,7 @@ namespace {
                                        unsigned VirtReg, unsigned Hint);
     LiveRegMap::iterator reloadVirtReg(MachineInstr *MI, unsigned OpNum,
                                        unsigned VirtReg, unsigned Hint);
-    void spillAll(MachineInstr *MI);
+    void spillAll(MachineBasicBlock::iterator MI);
     bool setPhysReg(MachineInstr *MI, unsigned OpNum, unsigned PhysReg);
     void addRetOperands(MachineBasicBlock *MBB);
   };
@@ -312,7 +314,7 @@ void RAFast::spillVirtReg(MachineBasicBlock::iterator MI,
 }
 
 /// spillAll - Spill all dirty virtregs without killing them.
-void RAFast::spillAll(MachineInstr *MI) {
+void RAFast::spillAll(MachineBasicBlock::iterator MI) {
   if (LiveVirtRegs.empty()) return;
   isBulkSpilling = true;
   // The LiveRegMap is keyed by an unsigned (the virtreg number), so the order
@@ -340,7 +342,7 @@ void RAFast::usePhysReg(MachineOperand &MO) {
     PhysRegState[PhysReg] = regFree;
     // Fall through
   case regFree:
-    UsedInInstr.set(PhysReg);
+    UsedInInstr.insert(PhysReg);
     MO.setIsKill();
     return;
   default:
@@ -360,13 +362,13 @@ void RAFast::usePhysReg(MachineOperand &MO) {
              "Instruction is not using a subregister of a reserved register");
       // Leave the superregister in the working set.
       PhysRegState[Alias] = regFree;
-      UsedInInstr.set(Alias);
+      UsedInInstr.insert(Alias);
       MO.getParent()->addRegisterKilled(Alias, TRI, true);
       return;
     case regFree:
       if (TRI->isSuperRegister(PhysReg, Alias)) {
         // Leave the superregister in the working set.
-        UsedInInstr.set(Alias);
+        UsedInInstr.insert(Alias);
         MO.getParent()->addRegisterKilled(Alias, TRI, true);
         return;
       }
@@ -380,7 +382,7 @@ void RAFast::usePhysReg(MachineOperand &MO) {
 
   // All aliases are disabled, bring register into working set.
   PhysRegState[PhysReg] = regFree;
-  UsedInInstr.set(PhysReg);
+  UsedInInstr.insert(PhysReg);
   MO.setIsKill();
 }
 
@@ -389,7 +391,7 @@ void RAFast::usePhysReg(MachineOperand &MO) {
 /// reserved instead of allocated.
 void RAFast::definePhysReg(MachineInstr *MI, unsigned PhysReg,
                            RegState NewState) {
-  UsedInInstr.set(PhysReg);
+  UsedInInstr.insert(PhysReg);
   switch (unsigned VirtReg = PhysRegState[PhysReg]) {
   case regDisabled:
     break;
@@ -429,7 +431,7 @@ void RAFast::definePhysReg(MachineInstr *MI, unsigned PhysReg,
 // can be allocated directly.
 // Returns spillImpossible when PhysReg or an alias can't be spilled.
 unsigned RAFast::calcSpillCost(unsigned PhysReg) const {
-  if (UsedInInstr.test(PhysReg)) {
+  if (UsedInInstr.count(PhysReg)) {
     DEBUG(dbgs() << PrintReg(PhysReg, TRI) << " is already used in instr.\n");
     return spillImpossible;
   }
@@ -454,7 +456,7 @@ unsigned RAFast::calcSpillCost(unsigned PhysReg) const {
   unsigned Cost = 0;
   for (MCRegAliasIterator AI(PhysReg, TRI, false); AI.isValid(); ++AI) {
     unsigned Alias = *AI;
-    if (UsedInInstr.test(Alias))
+    if (UsedInInstr.count(Alias))
       return spillImpossible;
     switch (unsigned VirtReg = PhysRegState[Alias]) {
     case regDisabled:
@@ -509,7 +511,7 @@ RAFast::LiveRegMap::iterator RAFast::allocVirtReg(MachineInstr *MI,
 
   // Ignore invalid hints.
   if (Hint && (!TargetRegisterInfo::isPhysicalRegister(Hint) ||
-               !RC->contains(Hint) || !RegClassInfo.isAllocatable(Hint)))
+               !RC->contains(Hint) || !MRI->isAllocatable(Hint)))
     Hint = 0;
 
   // Take hint when possible.
@@ -530,7 +532,7 @@ RAFast::LiveRegMap::iterator RAFast::allocVirtReg(MachineInstr *MI,
   // First try to find a completely free register.
   for (ArrayRef<unsigned>::iterator I = AO.begin(), E = AO.end(); I != E; ++I) {
     unsigned PhysReg = *I;
-    if (PhysRegState[PhysReg] == regFree && !UsedInInstr.test(PhysReg)) {
+    if (PhysRegState[PhysReg] == regFree && !UsedInInstr.count(PhysReg)) {
       assignVirtToPhysReg(*LRI, PhysReg);
       return LRI;
     }
@@ -596,7 +598,7 @@ RAFast::defineVirtReg(MachineInstr *MI, unsigned OpNum,
   LRI->LastUse = MI;
   LRI->LastOpNum = OpNum;
   LRI->Dirty = true;
-  UsedInInstr.set(LRI->PhysReg);
+  UsedInInstr.insert(LRI->PhysReg);
   return LRI;
 }
 
@@ -646,7 +648,7 @@ RAFast::reloadVirtReg(MachineInstr *MI, unsigned OpNum,
   assert(LRI->PhysReg && "Register not assigned");
   LRI->LastUse = MI;
   LRI->LastOpNum = OpNum;
-  UsedInInstr.set(LRI->PhysReg);
+  UsedInInstr.insert(LRI->PhysReg);
   return LRI;
 }
 
@@ -708,7 +710,7 @@ void RAFast::handleThroughOperands(MachineInstr *MI,
     unsigned Reg = MO.getReg();
     if (!Reg || !TargetRegisterInfo::isPhysicalRegister(Reg)) continue;
     for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) {
-      UsedInInstr.set(*AI);
+      UsedInInstr.insert(*AI);
       if (ThroughRegs.count(PhysRegState[*AI]))
         definePhysReg(MI, *AI, regFree);
     }
@@ -756,7 +758,7 @@ void RAFast::handleThroughOperands(MachineInstr *MI,
   }
 
   // Restore UsedInInstr to a state usable for allocating normal virtual uses.
-  UsedInInstr.reset();
+  UsedInInstr.clear();
   for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
     MachineOperand &MO = MI->getOperand(i);
     if (!MO.isReg() || (MO.isDef() && !MO.isEarlyClobber())) continue;
@@ -764,12 +766,12 @@ void RAFast::handleThroughOperands(MachineInstr *MI,
     if (!Reg || !TargetRegisterInfo::isPhysicalRegister(Reg)) continue;
     DEBUG(dbgs() << "\tSetting " << PrintReg(Reg, TRI)
                  << " as used in instr\n");
-    UsedInInstr.set(Reg);
+    UsedInInstr.insert(Reg);
   }
 
   // Also mark PartialDefs as used to avoid reallocation.
   for (unsigned i = 0, e = PartialDefs.size(); i != e; ++i)
-    UsedInInstr.set(PartialDefs[i]);
+    UsedInInstr.insert(PartialDefs[i]);
 }
 
 /// addRetOperand - ensure that a return instruction has an operand for each
@@ -838,7 +840,7 @@ void RAFast::AllocateBasicBlock() {
   // Add live-in registers as live.
   for (MachineBasicBlock::livein_iterator I = MBB->livein_begin(),
          E = MBB->livein_end(); I != E; ++I)
-    if (RegClassInfo.isAllocatable(*I))
+    if (MRI->isAllocatable(*I))
       definePhysReg(MII, *I, regReserved);
 
   SmallVector<unsigned, 8> VirtDead;
@@ -942,7 +944,7 @@ void RAFast::AllocateBasicBlock() {
     }
 
     // Track registers used by instruction.
-    UsedInInstr.reset();
+    UsedInInstr.clear();
 
     // First scan.
     // Mark physreg uses and early clobbers as used.
@@ -954,6 +956,11 @@ void RAFast::AllocateBasicBlock() {
     bool hasPhysDefs = false;
     for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
       MachineOperand &MO = MI->getOperand(i);
+      // Make sure MRI knows about registers clobbered by regmasks.
+      if (MO.isRegMask()) {
+        MRI->addPhysRegsUsedFromRegMask(MO.getRegMask());
+        continue;
+      }
       if (!MO.isReg()) continue;
       unsigned Reg = MO.getReg();
       if (!Reg) continue;
@@ -970,7 +977,7 @@ void RAFast::AllocateBasicBlock() {
         }
         continue;
       }
-      if (!RegClassInfo.isAllocatable(Reg)) continue;
+      if (!MRI->isAllocatable(Reg)) continue;
       if (MO.isUse()) {
         usePhysReg(MO);
       } else if (MO.isEarlyClobber()) {
@@ -1016,11 +1023,13 @@ void RAFast::AllocateBasicBlock() {
       }
     }
 
-    MRI->addPhysRegsUsed(UsedInInstr);
+    for (UsedInInstrSet::iterator
+         I = UsedInInstr.begin(), E = UsedInInstr.end(); I != E; ++I)
+      MRI->setPhysRegUsed(*I);
 
     // Track registers defined by instruction - early clobbers and tied uses at
     // this point.
-    UsedInInstr.reset();
+    UsedInInstr.clear();
     if (hasEarlyClobbers) {
       for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
         MachineOperand &MO = MI->getOperand(i);
@@ -1030,7 +1039,7 @@ void RAFast::AllocateBasicBlock() {
         // Look for physreg defs and tied uses.
         if (!MO.isDef() && !MI->isRegTiedToDefOperand(i)) continue;
         for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI)
-          UsedInInstr.set(*AI);
+          UsedInInstr.insert(*AI);
       }
     }
 
@@ -1058,7 +1067,7 @@ void RAFast::AllocateBasicBlock() {
       unsigned Reg = MO.getReg();
 
       if (TargetRegisterInfo::isPhysicalRegister(Reg)) {
-        if (!RegClassInfo.isAllocatable(Reg)) continue;
+        if (!MRI->isAllocatable(Reg)) continue;
         definePhysReg(MI, Reg, (MO.isImplicit() || MO.isDead()) ?
                                regFree : regReserved);
         continue;
@@ -1080,7 +1089,9 @@ void RAFast::AllocateBasicBlock() {
       killVirtReg(VirtDead[i]);
     VirtDead.clear();
 
-    MRI->addPhysRegsUsed(UsedInInstr);
+    for (UsedInInstrSet::iterator
+         I = UsedInInstr.begin(), E = UsedInInstr.end(); I != E; ++I)
+      MRI->setPhysRegUsed(*I);
 
     if (CopyDst && CopyDst == CopySrc && CopyDstSub == CopySrcSub) {
       DEBUG(dbgs() << "-- coalescing: " << *MI);
@@ -1110,8 +1121,7 @@ void RAFast::AllocateBasicBlock() {
 ///
 bool RAFast::runOnMachineFunction(MachineFunction &Fn) {
   DEBUG(dbgs() << "********** FAST REGISTER ALLOCATION **********\n"
-               << "********** Function: "
-               << ((Value*)Fn.getFunction())->getName() << '\n');
+               << "********** Function: " << Fn.getName() << '\n');
   MF = &Fn;
   MRI = &MF->getRegInfo();
   TM = &Fn.getTarget();
@@ -1119,7 +1129,8 @@ bool RAFast::runOnMachineFunction(MachineFunction &Fn) {
   TII = TM->getInstrInfo();
   MRI->freezeReservedRegs(Fn);
   RegClassInfo.runOnMachineFunction(Fn);
-  UsedInInstr.resize(TRI->getNumRegs());
+  UsedInInstr.clear();
+  UsedInInstr.setUniverse(TRI->getNumRegs());
 
   assert(!MRI->isSSA() && "regalloc requires leaving SSA");
 
diff --git a/lib/CodeGen/RegAllocGreedy.cpp b/lib/CodeGen/RegAllocGreedy.cpp
index 6ac5428..06f69c1e 100644
--- a/lib/CodeGen/RegAllocGreedy.cpp
+++ b/lib/CodeGen/RegAllocGreedy.cpp
@@ -24,7 +24,6 @@
 #include "VirtRegMap.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/AliasAnalysis.h"
-#include "llvm/Function.h"
 #include "llvm/PassAnalysisSupport.h"
 #include "llvm/CodeGen/CalcSpillWeights.h"
 #include "llvm/CodeGen/EdgeBundles.h"
@@ -331,9 +330,9 @@ void RAGreedy::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.addPreserved<SlotIndexes>();
   AU.addRequired<LiveDebugVariables>();
   AU.addPreserved<LiveDebugVariables>();
-  AU.addRequired<CalculateSpillWeights>();
   AU.addRequired<LiveStacks>();
   AU.addPreserved<LiveStacks>();
+  AU.addRequired<CalculateSpillWeights>();
   AU.addRequired<MachineDominatorTree>();
   AU.addPreserved<MachineDominatorTree>();
   AU.addRequired<MachineLoopInfo>();
@@ -509,7 +508,7 @@ bool RAGreedy::shouldEvict(LiveInterval &A, bool IsHint,
 ///
 /// @param VirtReg Live range that is about to be assigned.
 /// @param PhysReg Desired register for assignment.
-/// @prarm IsHint  True when PhysReg is VirtReg's preferred register.
+/// @param IsHint  True when PhysReg is VirtReg's preferred register.
 /// @param MaxCost Only look for cheaper candidates and update with new cost
 ///                when returning true.
 /// @returns True when interference can be evicted cheaper than MaxCost.
@@ -1746,8 +1745,7 @@ unsigned RAGreedy::selectOrSplit(LiveInterval &VirtReg,
 
 bool RAGreedy::runOnMachineFunction(MachineFunction &mf) {
   DEBUG(dbgs() << "********** GREEDY REGISTER ALLOCATION **********\n"
-               << "********** Function: "
-               << ((Value*)mf.getFunction())->getName() << '\n');
+               << "********** Function: " << mf.getName() << '\n');
 
   MF = &mf;
   if (VerifyEnabled)
diff --git a/lib/CodeGen/RegAllocPBQP.cpp b/lib/CodeGen/RegAllocPBQP.cpp
index d0db26b..02ebce7 100644
--- a/lib/CodeGen/RegAllocPBQP.cpp
+++ b/lib/CodeGen/RegAllocPBQP.cpp
@@ -118,7 +118,6 @@ private:
   typedef std::vector<AllowedSet> AllowedSetMap;
   typedef std::pair<unsigned, unsigned> RegPair;
   typedef std::map<RegPair, PBQP::PBQPNum> CoalesceMap;
-  typedef std::vector<PBQP::Graph::NodeItr> NodeVector;
   typedef std::set<unsigned> RegSet;
 
 
@@ -192,7 +191,6 @@ std::auto_ptr<PBQPRAProblem> PBQPBuilder::build(MachineFunction *mf,
                                                 const MachineLoopInfo *loopInfo,
                                                 const RegSet &vregs) {
 
-  typedef std::vector<const LiveInterval*> LIVector;
   LiveIntervals *LIS = const_cast<LiveIntervals*>(lis);
   MachineRegisterInfo *mri = &mf->getRegInfo();
   const TargetRegisterInfo *tri = mf->getTarget().getRegisterInfo();
@@ -209,8 +207,6 @@ std::auto_ptr<PBQPRAProblem> PBQPBuilder::build(MachineFunction *mf,
     mri->setPhysRegUsed(Reg);
   }
 
-  BitVector reservedRegs = tri->getReservedRegs(*mf);
-
   // Iterate over vregs.
   for (RegSet::const_iterator vregItr = vregs.begin(), vregEnd = vregs.end();
        vregItr != vregEnd; ++vregItr) {
@@ -219,7 +215,7 @@ std::auto_ptr<PBQPRAProblem> PBQPBuilder::build(MachineFunction *mf,
     LiveInterval *vregLI = &LIS->getInterval(vreg);
 
     // Record any overlaps with regmask operands.
-    BitVector regMaskOverlaps(tri->getNumRegs());
+    BitVector regMaskOverlaps;
     LIS->checkRegMaskInterference(*vregLI, regMaskOverlaps);
 
     // Compute an initial allowed set for the current vreg.
@@ -228,7 +224,7 @@ std::auto_ptr<PBQPRAProblem> PBQPBuilder::build(MachineFunction *mf,
     ArrayRef<uint16_t> rawOrder = trc->getRawAllocationOrder(*mf);
     for (unsigned i = 0; i != rawOrder.size(); ++i) {
       unsigned preg = rawOrder[i];
-      if (reservedRegs.test(preg))
+      if (mri->isReserved(preg))
         continue;
 
       // vregLI crosses a regmask operand that clobbers preg.
@@ -358,7 +354,7 @@ std::auto_ptr<PBQPRAProblem> PBQPBuilderWithCoalescing::build(
                                                    loopInfo->getLoopDepth(mbb));
 
       if (cp.isPhys()) {
-        if (!lis->isAllocatable(dst)) {
+        if (!mf->getRegInfo().isAllocatable(dst)) {
           continue;
         }
 
@@ -433,6 +429,7 @@ void RegAllocPBQP::getAnalysisUsage(AnalysisUsage &au) const {
   au.addRequired<SlotIndexes>();
   au.addPreserved<SlotIndexes>();
   au.addRequired<LiveIntervals>();
+  au.addPreserved<LiveIntervals>();
   //au.addRequiredID(SplitCriticalEdgesID);
   if (customPassID)
     au.addRequiredID(*customPassID);
@@ -444,6 +441,7 @@ void RegAllocPBQP::getAnalysisUsage(AnalysisUsage &au) const {
   au.addRequired<MachineLoopInfo>();
   au.addPreserved<MachineLoopInfo>();
   au.addRequired<VirtRegMap>();
+  au.addPreserved<VirtRegMap>();
   MachineFunctionPass::getAnalysisUsage(au);
 }
 
@@ -556,7 +554,7 @@ bool RegAllocPBQP::runOnMachineFunction(MachineFunction &MF) {
 
   mri->freezeReservedRegs(MF);
 
-  DEBUG(dbgs() << "PBQP Register Allocating for " << mf->getFunction()->getName() << "\n");
+  DEBUG(dbgs() << "PBQP Register Allocating for " << mf->getName() << "\n");
 
   // Allocator main loop:
   //
@@ -570,11 +568,12 @@ bool RegAllocPBQP::runOnMachineFunction(MachineFunction &MF) {
   // Find the vreg intervals in need of allocation.
   findVRegIntervalsToAlloc();
 
+#ifndef NDEBUG
   const Function* func = mf->getFunction();
   std::string fqn =
     func->getParent()->getModuleIdentifier() + "." +
     func->getName().str();
-  (void)fqn;
+#endif
 
   // If there are non-empty intervals allocate them using pbqp.
   if (!vregsToAlloc.empty()) {
diff --git a/lib/CodeGen/RegisterClassInfo.cpp b/lib/CodeGen/RegisterClassInfo.cpp
index 652bc30..805d235 100644
--- a/lib/CodeGen/RegisterClassInfo.cpp
+++ b/lib/CodeGen/RegisterClassInfo.cpp
@@ -15,8 +15,9 @@
 //===----------------------------------------------------------------------===//
 
 #define DEBUG_TYPE "regalloc"
-#include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/RegisterClassInfo.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
@@ -57,10 +58,11 @@ void RegisterClassInfo::runOnMachineFunction(const MachineFunction &mf) {
   CalleeSaved = CSR;
 
   // Different reserved registers?
-  BitVector RR = TRI->getReservedRegs(*MF);
-  if (RR != Reserved)
+  const BitVector &RR = MF->getRegInfo().getReservedRegs();
+  if (Reserved.size() != RR.size() || RR != Reserved) {
     Update = true;
-  Reserved = RR;
+    Reserved = RR;
+  }
 
   // Invalidate cached information from previous function.
   if (Update)
diff --git a/lib/CodeGen/RegisterCoalescer.cpp b/lib/CodeGen/RegisterCoalescer.cpp
index 9906334..2538f10 100644
--- a/lib/CodeGen/RegisterCoalescer.cpp
+++ b/lib/CodeGen/RegisterCoalescer.cpp
@@ -55,6 +55,8 @@ STATISTIC(numCommutes , "Number of instruction commuting performed");
 STATISTIC(numExtends  , "Number of copies extended");
 STATISTIC(NumReMats   , "Number of instructions re-materialized");
 STATISTIC(NumInflated , "Number of register classes inflated");
+STATISTIC(NumLaneConflicts, "Number of dead lane conflicts tested");
+STATISTIC(NumLaneResolves,  "Number of dead lane conflicts resolved");
 
 static cl::opt<bool>
 EnableJoining("join-liveintervals",
@@ -123,6 +125,9 @@ namespace {
     /// can use this information below to update aliases.
     bool joinIntervals(CoalescerPair &CP);
 
+    /// Attempt joining two virtual registers. Return true on success.
+    bool joinVirtRegs(CoalescerPair &CP);
+
     /// Attempt joining with a reserved physreg.
     bool joinReservedPhysReg(CoalescerPair &CP);
 
@@ -193,12 +198,6 @@ INITIALIZE_PASS_END(RegisterCoalescer, "simple-register-coalescing",
 
 char RegisterCoalescer::ID = 0;
 
-static unsigned compose(const TargetRegisterInfo &tri, unsigned a, unsigned b) {
-  if (!a) return b;
-  if (!b) return a;
-  return tri.composeSubRegIndices(a, b);
-}
-
 static bool isMoveInstr(const TargetRegisterInfo &tri, const MachineInstr *MI,
                         unsigned &Src, unsigned &Dst,
                         unsigned &SrcSub, unsigned &DstSub) {
@@ -209,8 +208,8 @@ static bool isMoveInstr(const TargetRegisterInfo &tri, const MachineInstr *MI,
     SrcSub = MI->getOperand(1).getSubReg();
   } else if (MI->isSubregToReg()) {
     Dst = MI->getOperand(0).getReg();
-    DstSub = compose(tri, MI->getOperand(0).getSubReg(),
-                     MI->getOperand(3).getImm());
+    DstSub = tri.composeSubRegIndices(MI->getOperand(0).getSubReg(),
+                                      MI->getOperand(3).getImm());
     Src = MI->getOperand(2).getReg();
     SrcSub = MI->getOperand(2).getSubReg();
   } else
@@ -349,7 +348,8 @@ bool CoalescerPair::isCoalescable(const MachineInstr *MI) const {
     if (DstReg != Dst)
       return false;
     // Registers match, do the subregisters line up?
-    return compose(TRI, SrcIdx, SrcSub) == compose(TRI, DstIdx, DstSub);
+    return TRI.composeSubRegIndices(SrcIdx, SrcSub) ==
+           TRI.composeSubRegIndices(DstIdx, DstSub);
   }
 }
 
@@ -425,7 +425,8 @@ bool RegisterCoalescer::adjustCopiesBackFrom(const CoalescerPair &CP,
   // If AValNo is defined as a copy from IntB, we can potentially process this.
   // Get the instruction that defines this value number.
   MachineInstr *ACopyMI = LIS->getInstructionFromIndex(AValNo->def);
-  if (!CP.isCoalescable(ACopyMI))
+  // Don't allow any partial copies, even if isCoalescable() allows them.
+  if (!CP.isCoalescable(ACopyMI) || !ACopyMI->isFullCopy())
     return false;
 
   // Get the LiveRange in IntB that this value number starts with.
@@ -583,7 +584,7 @@ bool RegisterCoalescer::removeCopyByCommutingDef(const CoalescerPair &CP,
 
   MachineOperand &NewDstMO = DefMI->getOperand(NewDstIdx);
   unsigned NewReg = NewDstMO.getReg();
-  if (NewReg != IntB.reg || !NewDstMO.isKill())
+  if (NewReg != IntB.reg || !LiveRangeQuery(IntB, AValNo->def).isKill())
     return false;
 
   // Make sure there are no other definitions of IntB that would reach the
@@ -849,8 +850,17 @@ void RegisterCoalescer::updateRegDefsUses(unsigned SrcReg,
   // Update LiveDebugVariables.
   LDV->renameRegister(SrcReg, DstReg, SubIdx);
 
+  SmallPtrSet<MachineInstr*, 8> Visited;
   for (MachineRegisterInfo::reg_iterator I = MRI->reg_begin(SrcReg);
        MachineInstr *UseMI = I.skipInstruction();) {
+    // Each instruction can only be rewritten once because sub-register
+    // composition is not always idempotent. When SrcReg != DstReg, rewriting
+    // the UseMI operands removes them from the SrcReg use-def chain, but when
+    // SrcReg is DstReg we could encounter UseMI twice if it has multiple
+    // operands mentioning the virtual register.
+    if (SrcReg == DstReg && !Visited.insert(UseMI))
+      continue;
+
     SmallVector<unsigned,8> Ops;
     bool Reads, Writes;
     tie(Reads, Writes) = UseMI->readsWritesVirtualRegister(SrcReg, &Ops);
@@ -890,7 +900,7 @@ bool RegisterCoalescer::canJoinPhys(CoalescerPair &CP) {
   /// Always join simple intervals that are defined by a single copy from a
   /// reserved register. This doesn't increase register pressure, so it is
   /// always beneficial.
-  if (!RegClassInfo.isReserved(CP.getDstReg())) {
+  if (!MRI->isReserved(CP.getDstReg())) {
     DEBUG(dbgs() << "\tCan only merge into reserved registers.\n");
     return false;
   }
@@ -1065,7 +1075,7 @@ bool RegisterCoalescer::joinCopy(MachineInstr *CopyMI, bool &Again) {
 /// Attempt joining with a reserved physreg.
 bool RegisterCoalescer::joinReservedPhysReg(CoalescerPair &CP) {
   assert(CP.isPhys() && "Must be a physreg copy");
-  assert(RegClassInfo.isReserved(CP.getDstReg()) && "Not a reserved register");
+  assert(MRI->isReserved(CP.getDstReg()) && "Not a reserved register");
   LiveInterval &RHS = LIS->getInterval(CP.getSrcReg());
   DEBUG(dbgs() << "\t\tRHS = " << PrintReg(CP.getSrcReg()) << ' ' << RHS
                << '\n');
@@ -1102,347 +1112,797 @@ bool RegisterCoalescer::joinReservedPhysReg(CoalescerPair &CP) {
   return true;
 }
 
-/// ComputeUltimateVN - Assuming we are going to join two live intervals,
-/// compute what the resultant value numbers for each value in the input two
-/// ranges will be.  This is complicated by copies between the two which can
-/// and will commonly cause multiple value numbers to be merged into one.
-///
-/// VN is the value number that we're trying to resolve.  InstDefiningValue
-/// keeps track of the new InstDefiningValue assignment for the result
-/// LiveInterval.  ThisFromOther/OtherFromThis are sets that keep track of
-/// whether a value in this or other is a copy from the opposite set.
-/// ThisValNoAssignments/OtherValNoAssignments keep track of value #'s that have
-/// already been assigned.
-///
-/// ThisFromOther[x] - If x is defined as a copy from the other interval, this
-/// contains the value number the copy is from.
-///
-static unsigned ComputeUltimateVN(VNInfo *VNI,
-                                  SmallVector<VNInfo*, 16> &NewVNInfo,
-                                  DenseMap<VNInfo*, VNInfo*> &ThisFromOther,
-                                  DenseMap<VNInfo*, VNInfo*> &OtherFromThis,
-                                  SmallVector<int, 16> &ThisValNoAssignments,
-                                  SmallVector<int, 16> &OtherValNoAssignments) {
-  unsigned VN = VNI->id;
-
-  // If the VN has already been computed, just return it.
-  if (ThisValNoAssignments[VN] >= 0)
-    return ThisValNoAssignments[VN];
-  assert(ThisValNoAssignments[VN] != -2 && "Cyclic value numbers");
-
-  // If this val is not a copy from the other val, then it must be a new value
-  // number in the destination.
-  DenseMap<VNInfo*, VNInfo*>::iterator I = ThisFromOther.find(VNI);
-  if (I == ThisFromOther.end()) {
-    NewVNInfo.push_back(VNI);
-    return ThisValNoAssignments[VN] = NewVNInfo.size()-1;
-  }
-  VNInfo *OtherValNo = I->second;
-
-  // Otherwise, this *is* a copy from the RHS.  If the other side has already
-  // been computed, return it.
-  if (OtherValNoAssignments[OtherValNo->id] >= 0)
-    return ThisValNoAssignments[VN] = OtherValNoAssignments[OtherValNo->id];
-
-  // Mark this value number as currently being computed, then ask what the
-  // ultimate value # of the other value is.
-  ThisValNoAssignments[VN] = -2;
-  unsigned UltimateVN =
-    ComputeUltimateVN(OtherValNo, NewVNInfo, OtherFromThis, ThisFromOther,
-                      OtherValNoAssignments, ThisValNoAssignments);
-  return ThisValNoAssignments[VN] = UltimateVN;
-}
+//===----------------------------------------------------------------------===//
+//                 Interference checking and interval joining
+//===----------------------------------------------------------------------===//
+//
+// In the easiest case, the two live ranges being joined are disjoint, and
+// there is no interference to consider. It is quite common, though, to have
+// overlapping live ranges, and we need to check if the interference can be
+// resolved.
+//
+// The live range of a single SSA value forms a sub-tree of the dominator tree.
+// This means that two SSA values overlap if and only if the def of one value
+// is contained in the live range of the other value. As a special case, the
+// overlapping values can be defined at the same index.
+//
+// The interference from an overlapping def can be resolved in these cases:
+//
+// 1. Coalescable copies. The value is defined by a copy that would become an
+//    identity copy after joining SrcReg and DstReg. The copy instruction will
+//    be removed, and the value will be merged with the source value.
+//
+//    There can be several copies back and forth, causing many values to be
+//    merged into one. We compute a list of ultimate values in the joined live
+//    range as well as a mappings from the old value numbers.
+//
+// 2. IMPLICIT_DEF. This instruction is only inserted to ensure all PHI
+//    predecessors have a live out value. It doesn't cause real interference,
+//    and can be merged into the value it overlaps. Like a coalescable copy, it
+//    can be erased after joining.
+//
+// 3. Copy of external value. The overlapping def may be a copy of a value that
+//    is already in the other register. This is like a coalescable copy, but
+//    the live range of the source register must be trimmed after erasing the
+//    copy instruction:
+//
+//      %src = COPY %ext
+//      %dst = COPY %ext  <-- Remove this COPY, trim the live range of %ext.
+//
+// 4. Clobbering undefined lanes. Vector registers are sometimes built by
+//    defining one lane at a time:
+//
+//      %dst:ssub0<def,read-undef> = FOO
+//      %src = BAR
+//      %dst:ssub1<def> = COPY %src
+//
+//    The live range of %src overlaps the %dst value defined by FOO, but
+//    merging %src into %dst:ssub1 is only going to clobber the ssub1 lane
+//    which was undef anyway.
+//
+//    The value mapping is more complicated in this case. The final live range
+//    will have different value numbers for both FOO and BAR, but there is no
+//    simple mapping from old to new values. It may even be necessary to add
+//    new PHI values.
+//
+// 5. Clobbering dead lanes. A def may clobber a lane of a vector register that
+//    is live, but never read. This can happen because we don't compute
+//    individual live ranges per lane.
+//
+//      %dst<def> = FOO
+//      %src = BAR
+//      %dst:ssub1<def> = COPY %src
+//
+//    This kind of interference is only resolved locally. If the clobbered
+//    lane value escapes the block, the join is aborted.
 
+namespace {
+/// Track information about values in a single virtual register about to be
+/// joined. Objects of this class are always created in pairs - one for each
+/// side of the CoalescerPair.
+class JoinVals {
+  LiveInterval &LI;
+
+  // Location of this register in the final joined register.
+  // Either CP.DstIdx or CP.SrcIdx.
+  unsigned SubIdx;
+
+  // Values that will be present in the final live range.
+  SmallVectorImpl<VNInfo*> &NewVNInfo;
+
+  const CoalescerPair &CP;
+  LiveIntervals *LIS;
+  SlotIndexes *Indexes;
+  const TargetRegisterInfo *TRI;
+
+  // Value number assignments. Maps value numbers in LI to entries in NewVNInfo.
+  // This is suitable for passing to LiveInterval::join().
+  SmallVector<int, 8> Assignments;
+
+  // Conflict resolution for overlapping values.
+  enum ConflictResolution {
+    // No overlap, simply keep this value.
+    CR_Keep,
+
+    // Merge this value into OtherVNI and erase the defining instruction.
+    // Used for IMPLICIT_DEF, coalescable copies, and copies from external
+    // values.
+    CR_Erase,
+
+    // Merge this value into OtherVNI but keep the defining instruction.
+    // This is for the special case where OtherVNI is defined by the same
+    // instruction.
+    CR_Merge,
+
+    // Keep this value, and have it replace OtherVNI where possible. This
+    // complicates value mapping since OtherVNI maps to two different values
+    // before and after this def.
+    // Used when clobbering undefined or dead lanes.
+    CR_Replace,
+
+    // Unresolved conflict. Visit later when all values have been mapped.
+    CR_Unresolved,
+
+    // Unresolvable conflict. Abort the join.
+    CR_Impossible
+  };
 
-// Find out if we have something like
-// A = X
-// B = X
-// if so, we can pretend this is actually
-// A = X
-// B = A
-// which allows us to coalesce A and B.
-// VNI is the definition of B. LR is the life range of A that includes
-// the slot just before B. If we return true, we add "B = X" to DupCopies.
-// This implies that A dominates B.
-static bool RegistersDefinedFromSameValue(LiveIntervals &li,
-                                          const TargetRegisterInfo &tri,
-                                          CoalescerPair &CP,
-                                          VNInfo *VNI,
-                                          VNInfo *OtherVNI,
-                                     SmallVector<MachineInstr*, 8> &DupCopies) {
-  // FIXME: This is very conservative. For example, we don't handle
-  // physical registers.
-
-  MachineInstr *MI = li.getInstructionFromIndex(VNI->def);
-
-  if (!MI || CP.isPartial() || CP.isPhys())
-    return false;
+  // Per-value info for LI. The lane bit masks are all relative to the final
+  // joined register, so they can be compared directly between SrcReg and
+  // DstReg.
+  struct Val {
+    ConflictResolution Resolution;
 
-  unsigned A = CP.getDstReg();
-  if (!TargetRegisterInfo::isVirtualRegister(A))
-    return false;
+    // Lanes written by this def, 0 for unanalyzed values.
+    unsigned WriteLanes;
 
-  unsigned B = CP.getSrcReg();
-  if (!TargetRegisterInfo::isVirtualRegister(B))
-    return false;
+    // Lanes with defined values in this register. Other lanes are undef and
+    // safe to clobber.
+    unsigned ValidLanes;
 
-  MachineInstr *OtherMI = li.getInstructionFromIndex(OtherVNI->def);
-  if (!OtherMI)
-    return false;
+    // Value in LI being redefined by this def.
+    VNInfo *RedefVNI;
 
-  if (MI->isImplicitDef()) {
-    DupCopies.push_back(MI);
-    return true;
-  } else {
-    if (!MI->isFullCopy())
-      return false;
-    unsigned Src = MI->getOperand(1).getReg();
-    if (!TargetRegisterInfo::isVirtualRegister(Src))
-      return false;
-    if (!OtherMI->isFullCopy())
-      return false;
-    unsigned OtherSrc = OtherMI->getOperand(1).getReg();
-    if (!TargetRegisterInfo::isVirtualRegister(OtherSrc))
-      return false;
+    // Value in the other live range that overlaps this def, if any.
+    VNInfo *OtherVNI;
 
-    if (Src != OtherSrc)
-      return false;
+    // Is this value an IMPLICIT_DEF?
+    bool IsImplicitDef;
 
-    // If the copies use two different value numbers of X, we cannot merge
-    // A and B.
-    LiveInterval &SrcInt = li.getInterval(Src);
-    // getVNInfoBefore returns NULL for undef copies. In this case, the
-    // optimization is still safe.
-    if (SrcInt.getVNInfoBefore(OtherVNI->def) !=
-        SrcInt.getVNInfoBefore(VNI->def))
-      return false;
+    // True when the live range of this value will be pruned because of an
+    // overlapping CR_Replace value in the other live range.
+    bool Pruned;
 
-    DupCopies.push_back(MI);
-    return true;
-  }
-}
+    // True once Pruned above has been computed.
+    bool PrunedComputed;
 
-/// joinIntervals - Attempt to join these two intervals.  On failure, this
-/// returns false.
-bool RegisterCoalescer::joinIntervals(CoalescerPair &CP) {
-  // Handle physreg joins separately.
-  if (CP.isPhys())
-    return joinReservedPhysReg(CP);
+    Val() : Resolution(CR_Keep), WriteLanes(0), ValidLanes(0),
+            RedefVNI(0), OtherVNI(0), IsImplicitDef(false), Pruned(false),
+            PrunedComputed(false) {}
 
-  LiveInterval &RHS = LIS->getInterval(CP.getSrcReg());
-  DEBUG(dbgs() << "\t\tRHS = " << PrintReg(CP.getSrcReg()) << ' ' << RHS
-               << '\n');
+    bool isAnalyzed() const { return WriteLanes != 0; }
+  };
 
-  // Compute the final value assignment, assuming that the live ranges can be
-  // coalesced.
-  SmallVector<int, 16> LHSValNoAssignments;
-  SmallVector<int, 16> RHSValNoAssignments;
-  DenseMap<VNInfo*, VNInfo*> LHSValsDefinedFromRHS;
-  DenseMap<VNInfo*, VNInfo*> RHSValsDefinedFromLHS;
-  SmallVector<VNInfo*, 16> NewVNInfo;
+  // One entry per value number in LI.
+  SmallVector<Val, 8> Vals;
+
+  unsigned computeWriteLanes(const MachineInstr *DefMI, bool &Redef);
+  VNInfo *stripCopies(VNInfo *VNI);
+  ConflictResolution analyzeValue(unsigned ValNo, JoinVals &Other);
+  void computeAssignment(unsigned ValNo, JoinVals &Other);
+  bool taintExtent(unsigned, unsigned, JoinVals&,
+                   SmallVectorImpl<std::pair<SlotIndex, unsigned> >&);
+  bool usesLanes(MachineInstr *MI, unsigned, unsigned, unsigned);
+  bool isPrunedValue(unsigned ValNo, JoinVals &Other);
+
+public:
+  JoinVals(LiveInterval &li, unsigned subIdx,
+           SmallVectorImpl<VNInfo*> &newVNInfo,
+           const CoalescerPair &cp,
+           LiveIntervals *lis,
+           const TargetRegisterInfo *tri)
+    : LI(li), SubIdx(subIdx), NewVNInfo(newVNInfo), CP(cp), LIS(lis),
+      Indexes(LIS->getSlotIndexes()), TRI(tri),
+      Assignments(LI.getNumValNums(), -1), Vals(LI.getNumValNums())
+  {}
+
+  /// Analyze defs in LI and compute a value mapping in NewVNInfo.
+  /// Returns false if any conflicts were impossible to resolve.
+  bool mapValues(JoinVals &Other);
+
+  /// Try to resolve conflicts that require all values to be mapped.
+  /// Returns false if any conflicts were impossible to resolve.
+  bool resolveConflicts(JoinVals &Other);
+
+  /// Prune the live range of values in Other.LI where they would conflict with
+  /// CR_Replace values in LI. Collect end points for restoring the live range
+  /// after joining.
+  void pruneValues(JoinVals &Other, SmallVectorImpl<SlotIndex> &EndPoints);
+
+  /// Erase any machine instructions that have been coalesced away.
+  /// Add erased instructions to ErasedInstrs.
+  /// Add foreign virtual registers to ShrinkRegs if their live range ended at
+  /// the erased instrs.
+  void eraseInstrs(SmallPtrSet<MachineInstr*, 8> &ErasedInstrs,
+                   SmallVectorImpl<unsigned> &ShrinkRegs);
+
+  /// Get the value assignments suitable for passing to LiveInterval::join.
+  const int *getAssignments() const { return Assignments.data(); }
+};
+} // end anonymous namespace
+
+/// Compute the bitmask of lanes actually written by DefMI.
+/// Set Redef if there are any partial register definitions that depend on the
+/// previous value of the register.
+unsigned JoinVals::computeWriteLanes(const MachineInstr *DefMI, bool &Redef) {
+  unsigned L = 0;
+  for (ConstMIOperands MO(DefMI); MO.isValid(); ++MO) {
+    if (!MO->isReg() || MO->getReg() != LI.reg || !MO->isDef())
+      continue;
+    L |= TRI->getSubRegIndexLaneMask(
+           TRI->composeSubRegIndices(SubIdx, MO->getSubReg()));
+    if (MO->readsReg())
+      Redef = true;
+  }
+  return L;
+}
 
-  SmallVector<MachineInstr*, 8> DupCopies;
-  SmallVector<MachineInstr*, 8> DeadCopies;
+/// Find the ultimate value that VNI was copied from.
+VNInfo *JoinVals::stripCopies(VNInfo *VNI) {
+  while (!VNI->isPHIDef()) {
+    MachineInstr *MI = Indexes->getInstructionFromIndex(VNI->def);
+    assert(MI && "No defining instruction");
+    if (!MI->isFullCopy())
+      break;
+    unsigned Reg = MI->getOperand(1).getReg();
+    if (!TargetRegisterInfo::isVirtualRegister(Reg))
+      break;
+    LiveRangeQuery LRQ(LIS->getInterval(Reg), VNI->def);
+    if (!LRQ.valueIn())
+      break;
+    VNI = LRQ.valueIn();
+  }
+  return VNI;
+}
 
-  LiveInterval &LHS = LIS->getOrCreateInterval(CP.getDstReg());
-  DEBUG(dbgs() << "\t\tLHS = " << PrintReg(CP.getDstReg(), TRI) << ' ' << LHS
-               << '\n');
+/// Analyze ValNo in this live range, and set all fields of Vals[ValNo].
+/// Return a conflict resolution when possible, but leave the hard cases as
+/// CR_Unresolved.
+/// Recursively calls computeAssignment() on this and Other, guaranteeing that
+/// both OtherVNI and RedefVNI have been analyzed and mapped before returning.
+/// The recursion always goes upwards in the dominator tree, making loops
+/// impossible.
+JoinVals::ConflictResolution
+JoinVals::analyzeValue(unsigned ValNo, JoinVals &Other) {
+  Val &V = Vals[ValNo];
+  assert(!V.isAnalyzed() && "Value has already been analyzed!");
+  VNInfo *VNI = LI.getValNumInfo(ValNo);
+  if (VNI->isUnused()) {
+    V.WriteLanes = ~0u;
+    return CR_Keep;
+  }
 
-  // Loop over the value numbers of the LHS, seeing if any are defined from
-  // the RHS.
-  for (LiveInterval::vni_iterator i = LHS.vni_begin(), e = LHS.vni_end();
-       i != e; ++i) {
-    VNInfo *VNI = *i;
-    if (VNI->isUnused() || VNI->isPHIDef())
-      continue;
-    MachineInstr *MI = LIS->getInstructionFromIndex(VNI->def);
-    assert(MI && "Missing def");
-    if (!MI->isCopyLike() && !MI->isImplicitDef()) // Src not defined by a copy?
-      continue;
+  // Get the instruction defining this value, compute the lanes written.
+  const MachineInstr *DefMI = 0;
+  if (VNI->isPHIDef()) {
+    // Conservatively assume that all lanes in a PHI are valid.
+    V.ValidLanes = V.WriteLanes = TRI->getSubRegIndexLaneMask(SubIdx);
+  } else {
+    DefMI = Indexes->getInstructionFromIndex(VNI->def);
+    bool Redef = false;
+    V.ValidLanes = V.WriteLanes = computeWriteLanes(DefMI, Redef);
+
+    // If this is a read-modify-write instruction, there may be more valid
+    // lanes than the ones written by this instruction.
+    // This only covers partial redef operands. DefMI may have normal use
+    // operands reading the register. They don't contribute valid lanes.
+    //
+    // This adds ssub1 to the set of valid lanes in %src:
+    //
+    //   %src:ssub1<def> = FOO
+    //
+    // This leaves only ssub1 valid, making any other lanes undef:
+    //
+    //   %src:ssub1<def,read-undef> = FOO %src:ssub2
+    //
+    // The <read-undef> flag on the def operand means that old lane values are
+    // not important.
+    if (Redef) {
+      V.RedefVNI = LiveRangeQuery(LI, VNI->def).valueIn();
+      assert(V.RedefVNI && "Instruction is reading nonexistent value");
+      computeAssignment(V.RedefVNI->id, Other);
+      V.ValidLanes |= Vals[V.RedefVNI->id].ValidLanes;
+    }
 
-    // Figure out the value # from the RHS.
-    VNInfo *OtherVNI = RHS.getVNInfoBefore(VNI->def);
-    // The copy could be to an aliased physreg.
-    if (!OtherVNI)
-      continue;
+    // An IMPLICIT_DEF writes undef values.
+    if (DefMI->isImplicitDef()) {
+      V.IsImplicitDef = true;
+      V.ValidLanes &= ~V.WriteLanes;
+    }
+  }
 
-    // DstReg is known to be a register in the LHS interval.  If the src is
-    // from the RHS interval, we can use its value #.
-    if (CP.isCoalescable(MI))
-      DeadCopies.push_back(MI);
-    else if (!RegistersDefinedFromSameValue(*LIS, *TRI, CP, VNI, OtherVNI,
-                                            DupCopies))
-      continue;
+  // Find the value in Other that overlaps VNI->def, if any.
+  LiveRangeQuery OtherLRQ(Other.LI, VNI->def);
+
+  // It is possible that both values are defined by the same instruction, or
+  // the values are PHIs defined in the same block. When that happens, the two
+  // values should be merged into one, but not into any preceding value.
+  // The first value defined or visited gets CR_Keep, the other gets CR_Merge.
+  if (VNInfo *OtherVNI = OtherLRQ.valueDefined()) {
+    assert(SlotIndex::isSameInstr(VNI->def, OtherVNI->def) && "Broken LRQ");
+
+    // One value stays, the other is merged. Keep the earlier one, or the first
+    // one we see.
+    if (OtherVNI->def < VNI->def)
+      Other.computeAssignment(OtherVNI->id, *this);
+    else if (VNI->def < OtherVNI->def && OtherLRQ.valueIn()) {
+      // This is an early-clobber def overlapping a live-in value in the other
+      // register. Not mergeable.
+      V.OtherVNI = OtherLRQ.valueIn();
+      return CR_Impossible;
+    }
+    V.OtherVNI = OtherVNI;
+    Val &OtherV = Other.Vals[OtherVNI->id];
+    // Keep this value, check for conflicts when analyzing OtherVNI.
+    if (!OtherV.isAnalyzed())
+      return CR_Keep;
+    // Both sides have been analyzed now.
+    // Allow overlapping PHI values. Any real interference would show up in a
+    // predecessor, the PHI itself can't introduce any conflicts.
+    if (VNI->isPHIDef())
+      return CR_Merge;
+    if (V.ValidLanes & OtherV.ValidLanes)
+      // Overlapping lanes can't be resolved.
+      return CR_Impossible;
+    else
+      return CR_Merge;
+  }
 
-    LHSValsDefinedFromRHS[VNI] = OtherVNI;
+  // No simultaneous def. Is Other live at the def?
+  V.OtherVNI = OtherLRQ.valueIn();
+  if (!V.OtherVNI)
+    // No overlap, no conflict.
+    return CR_Keep;
+
+  assert(!SlotIndex::isSameInstr(VNI->def, V.OtherVNI->def) && "Broken LRQ");
+
+  // We have overlapping values, or possibly a kill of Other.
+  // Recursively compute assignments up the dominator tree.
+  Other.computeAssignment(V.OtherVNI->id, *this);
+  const Val &OtherV = Other.Vals[V.OtherVNI->id];
+
+  // Allow overlapping PHI values. Any real interference would show up in a
+  // predecessor, the PHI itself can't introduce any conflicts.
+  if (VNI->isPHIDef())
+    return CR_Replace;
+
+  // Check for simple erasable conflicts.
+  if (DefMI->isImplicitDef())
+    return CR_Erase;
+
+  // Include the non-conflict where DefMI is a coalescable copy that kills
+  // OtherVNI. We still want the copy erased and value numbers merged.
+  if (CP.isCoalescable(DefMI)) {
+    // Some of the lanes copied from OtherVNI may be undef, making them undef
+    // here too.
+    V.ValidLanes &= ~V.WriteLanes | OtherV.ValidLanes;
+    return CR_Erase;
   }
 
-  // Loop over the value numbers of the RHS, seeing if any are defined from
-  // the LHS.
-  for (LiveInterval::vni_iterator i = RHS.vni_begin(), e = RHS.vni_end();
-       i != e; ++i) {
-    VNInfo *VNI = *i;
-    if (VNI->isUnused() || VNI->isPHIDef())
-      continue;
-    MachineInstr *MI = LIS->getInstructionFromIndex(VNI->def);
-    assert(MI && "Missing def");
-    if (!MI->isCopyLike() && !MI->isImplicitDef()) // Src not defined by a copy?
-      continue;
+  // This may not be a real conflict if DefMI simply kills Other and defines
+  // VNI.
+  if (OtherLRQ.isKill() && OtherLRQ.endPoint() <= VNI->def)
+    return CR_Keep;
+
+  // Handle the case where VNI and OtherVNI can be proven to be identical:
+  //
+  //   %other = COPY %ext
+  //   %this  = COPY %ext <-- Erase this copy
+  //
+  if (DefMI->isFullCopy() && !CP.isPartial() &&
+      stripCopies(VNI) == stripCopies(V.OtherVNI))
+    return CR_Erase;
+
+  // If the lanes written by this instruction were all undef in OtherVNI, it is
+  // still safe to join the live ranges. This can't be done with a simple value
+  // mapping, though - OtherVNI will map to multiple values:
+  //
+  //   1 %dst:ssub0 = FOO                <-- OtherVNI
+  //   2 %src = BAR                      <-- VNI
+  //   3 %dst:ssub1 = COPY %src<kill>    <-- Eliminate this copy.
+  //   4 BAZ %dst<kill>
+  //   5 QUUX %src<kill>
+  //
+  // Here OtherVNI will map to itself in [1;2), but to VNI in [2;5). CR_Replace
+  // handles this complex value mapping.
+  if ((V.WriteLanes & OtherV.ValidLanes) == 0)
+    return CR_Replace;
+
+  // If the other live range is killed by DefMI and the live ranges are still
+  // overlapping, it must be because we're looking at an early clobber def:
+  //
+  //   %dst<def,early-clobber> = ASM %src<kill>
+  //
+  // In this case, it is illegal to merge the two live ranges since the early
+  // clobber def would clobber %src before it was read.
+  if (OtherLRQ.isKill()) {
+    // This case where the def doesn't overlap the kill is handled above.
+    assert(VNI->def.isEarlyClobber() &&
+           "Only early clobber defs can overlap a kill");
+    return CR_Impossible;
+  }
 
-    // Figure out the value # from the LHS.
-    VNInfo *OtherVNI = LHS.getVNInfoBefore(VNI->def);
-    // The copy could be to an aliased physreg.
-    if (!OtherVNI)
-      continue;
+  // VNI is clobbering live lanes in OtherVNI, but there is still the
+  // possibility that no instructions actually read the clobbered lanes.
+  // If we're clobbering all the lanes in OtherVNI, at least one must be read.
+  // Otherwise Other.LI wouldn't be live here.
+  if ((TRI->getSubRegIndexLaneMask(Other.SubIdx) & ~V.WriteLanes) == 0)
+    return CR_Impossible;
+
+  // We need to verify that no instructions are reading the clobbered lanes. To
+  // save compile time, we'll only check that locally. Don't allow the tainted
+  // value to escape the basic block.
+  MachineBasicBlock *MBB = Indexes->getMBBFromIndex(VNI->def);
+  if (OtherLRQ.endPoint() >= Indexes->getMBBEndIdx(MBB))
+    return CR_Impossible;
+
+  // There are still some things that could go wrong besides clobbered lanes
+  // being read, for example OtherVNI may be only partially redefined in MBB,
+  // and some clobbered lanes could escape the block. Save this analysis for
+  // resolveConflicts() when all values have been mapped. We need to know
+  // RedefVNI and WriteLanes for any later defs in MBB, and we can't compute
+  // that now - the recursive analyzeValue() calls must go upwards in the
+  // dominator tree.
+  return CR_Unresolved;
+}
 
-    // DstReg is known to be a register in the RHS interval.  If the src is
-    // from the LHS interval, we can use its value #.
-    if (CP.isCoalescable(MI))
-      DeadCopies.push_back(MI);
-    else if (!RegistersDefinedFromSameValue(*LIS, *TRI, CP, VNI, OtherVNI,
-                                            DupCopies))
-        continue;
+/// Compute the value assignment for ValNo in LI.
+/// This may be called recursively by analyzeValue(), but never for a ValNo on
+/// the stack.
+void JoinVals::computeAssignment(unsigned ValNo, JoinVals &Other) {
+  Val &V = Vals[ValNo];
+  if (V.isAnalyzed()) {
+    // Recursion should always move up the dominator tree, so ValNo is not
+    // supposed to reappear before it has been assigned.
+    assert(Assignments[ValNo] != -1 && "Bad recursion?");
+    return;
+  }
+  switch ((V.Resolution = analyzeValue(ValNo, Other))) {
+  case CR_Erase:
+  case CR_Merge:
+    // Merge this ValNo into OtherVNI.
+    assert(V.OtherVNI && "OtherVNI not assigned, can't merge.");
+    assert(Other.Vals[V.OtherVNI->id].isAnalyzed() && "Missing recursion");
+    Assignments[ValNo] = Other.Assignments[V.OtherVNI->id];
+    DEBUG(dbgs() << "\t\tmerge " << PrintReg(LI.reg) << ':' << ValNo << '@'
+                 << LI.getValNumInfo(ValNo)->def << " into "
+                 << PrintReg(Other.LI.reg) << ':' << V.OtherVNI->id << '@'
+                 << V.OtherVNI->def << " --> @"
+                 << NewVNInfo[Assignments[ValNo]]->def << '\n');
+    break;
+  case CR_Replace:
+  case CR_Unresolved:
+    // The other value is going to be pruned if this join is successful.
+    assert(V.OtherVNI && "OtherVNI not assigned, can't prune");
+    Other.Vals[V.OtherVNI->id].Pruned = true;
+    // Fall through.
+  default:
+    // This value number needs to go in the final joined live range.
+    Assignments[ValNo] = NewVNInfo.size();
+    NewVNInfo.push_back(LI.getValNumInfo(ValNo));
+    break;
+  }
+}
 
-    RHSValsDefinedFromLHS[VNI] = OtherVNI;
+bool JoinVals::mapValues(JoinVals &Other) {
+  for (unsigned i = 0, e = LI.getNumValNums(); i != e; ++i) {
+    computeAssignment(i, Other);
+    if (Vals[i].Resolution == CR_Impossible) {
+      DEBUG(dbgs() << "\t\tinterference at " << PrintReg(LI.reg) << ':' << i
+                   << '@' << LI.getValNumInfo(i)->def << '\n');
+      return false;
+    }
   }
+  return true;
+}
 
-  LHSValNoAssignments.resize(LHS.getNumValNums(), -1);
-  RHSValNoAssignments.resize(RHS.getNumValNums(), -1);
-  NewVNInfo.reserve(LHS.getNumValNums() + RHS.getNumValNums());
+/// Assuming ValNo is going to clobber some valid lanes in Other.LI, compute
+/// the extent of the tainted lanes in the block.
+///
+/// Multiple values in Other.LI can be affected since partial redefinitions can
+/// preserve previously tainted lanes.
+///
+///   1 %dst = VLOAD           <-- Define all lanes in %dst
+///   2 %src = FOO             <-- ValNo to be joined with %dst:ssub0
+///   3 %dst:ssub1 = BAR       <-- Partial redef doesn't clear taint in ssub0
+///   4 %dst:ssub0 = COPY %src <-- Conflict resolved, ssub0 wasn't read
+///
+/// For each ValNo in Other that is affected, add an (EndIndex, TaintedLanes)
+/// entry to TaintedVals.
+///
+/// Returns false if the tainted lanes extend beyond the basic block.
+bool JoinVals::
+taintExtent(unsigned ValNo, unsigned TaintedLanes, JoinVals &Other,
+            SmallVectorImpl<std::pair<SlotIndex, unsigned> > &TaintExtent) {
+  VNInfo *VNI = LI.getValNumInfo(ValNo);
+  MachineBasicBlock *MBB = Indexes->getMBBFromIndex(VNI->def);
+  SlotIndex MBBEnd = Indexes->getMBBEndIdx(MBB);
+
+  // Scan Other.LI from VNI.def to MBBEnd.
+  LiveInterval::iterator OtherI = Other.LI.find(VNI->def);
+  assert(OtherI != Other.LI.end() && "No conflict?");
+  do {
+    // OtherI is pointing to a tainted value. Abort the join if the tainted
+    // lanes escape the block.
+    SlotIndex End = OtherI->end;
+    if (End >= MBBEnd) {
+      DEBUG(dbgs() << "\t\ttaints global " << PrintReg(Other.LI.reg) << ':'
+                   << OtherI->valno->id << '@' << OtherI->start << '\n');
+      return false;
+    }
+    DEBUG(dbgs() << "\t\ttaints local " << PrintReg(Other.LI.reg) << ':'
+                 << OtherI->valno->id << '@' << OtherI->start
+                 << " to " << End << '\n');
+    // A dead def is not a problem.
+    if (End.isDead())
+      break;
+    TaintExtent.push_back(std::make_pair(End, TaintedLanes));
+
+    // Check for another def in the MBB.
+    if (++OtherI == Other.LI.end() || OtherI->start >= MBBEnd)
+      break;
+
+    // Lanes written by the new def are no longer tainted.
+    const Val &OV = Other.Vals[OtherI->valno->id];
+    TaintedLanes &= ~OV.WriteLanes;
+    if (!OV.RedefVNI)
+      break;
+  } while (TaintedLanes);
+  return true;
+}
 
-  for (LiveInterval::vni_iterator i = LHS.vni_begin(), e = LHS.vni_end();
-       i != e; ++i) {
-    VNInfo *VNI = *i;
-    unsigned VN = VNI->id;
-    if (LHSValNoAssignments[VN] >= 0 || VNI->isUnused())
+/// Return true if MI uses any of the given Lanes from Reg.
+/// This does not include partial redefinitions of Reg.
+bool JoinVals::usesLanes(MachineInstr *MI, unsigned Reg, unsigned SubIdx,
+                         unsigned Lanes) {
+  if (MI->isDebugValue())
+    return false;
+  for (ConstMIOperands MO(MI); MO.isValid(); ++MO) {
+    if (!MO->isReg() || MO->isDef() || MO->getReg() != Reg)
       continue;
-    ComputeUltimateVN(VNI, NewVNInfo,
-                      LHSValsDefinedFromRHS, RHSValsDefinedFromLHS,
-                      LHSValNoAssignments, RHSValNoAssignments);
-  }
-  for (LiveInterval::vni_iterator i = RHS.vni_begin(), e = RHS.vni_end();
-       i != e; ++i) {
-    VNInfo *VNI = *i;
-    unsigned VN = VNI->id;
-    if (RHSValNoAssignments[VN] >= 0 || VNI->isUnused())
+    if (!MO->readsReg())
       continue;
-    // If this value number isn't a copy from the LHS, it's a new number.
-    if (RHSValsDefinedFromLHS.find(VNI) == RHSValsDefinedFromLHS.end()) {
-      NewVNInfo.push_back(VNI);
-      RHSValNoAssignments[VN] = NewVNInfo.size()-1;
+    if (Lanes & TRI->getSubRegIndexLaneMask(
+                  TRI->composeSubRegIndices(SubIdx, MO->getSubReg())))
+      return true;
+  }
+  return false;
+}
+
+bool JoinVals::resolveConflicts(JoinVals &Other) {
+  for (unsigned i = 0, e = LI.getNumValNums(); i != e; ++i) {
+    Val &V = Vals[i];
+    assert (V.Resolution != CR_Impossible && "Unresolvable conflict");
+    if (V.Resolution != CR_Unresolved)
       continue;
-    }
+    DEBUG(dbgs() << "\t\tconflict at " << PrintReg(LI.reg) << ':' << i
+                 << '@' << LI.getValNumInfo(i)->def << '\n');
+    ++NumLaneConflicts;
+    assert(V.OtherVNI && "Inconsistent conflict resolution.");
+    VNInfo *VNI = LI.getValNumInfo(i);
+    const Val &OtherV = Other.Vals[V.OtherVNI->id];
+
+    // VNI is known to clobber some lanes in OtherVNI. If we go ahead with the
+    // join, those lanes will be tainted with a wrong value. Get the extent of
+    // the tainted lanes.
+    unsigned TaintedLanes = V.WriteLanes & OtherV.ValidLanes;
+    SmallVector<std::pair<SlotIndex, unsigned>, 8> TaintExtent;
+    if (!taintExtent(i, TaintedLanes, Other, TaintExtent))
+      // Tainted lanes would extend beyond the basic block.
+      return false;
 
-    ComputeUltimateVN(VNI, NewVNInfo,
-                      RHSValsDefinedFromLHS, LHSValsDefinedFromRHS,
-                      RHSValNoAssignments, LHSValNoAssignments);
-  }
+    assert(!TaintExtent.empty() && "There should be at least one conflict.");
 
-  // Armed with the mappings of LHS/RHS values to ultimate values, walk the
-  // interval lists to see if these intervals are coalescable.
-  LiveInterval::const_iterator I = LHS.begin();
-  LiveInterval::const_iterator IE = LHS.end();
-  LiveInterval::const_iterator J = RHS.begin();
-  LiveInterval::const_iterator JE = RHS.end();
-
-  // Collect interval end points that will no longer be kills.
-  SmallVector<MachineInstr*, 8> LHSOldKills;
-  SmallVector<MachineInstr*, 8> RHSOldKills;
-
-  // Skip ahead until the first place of potential sharing.
-  if (I != IE && J != JE) {
-    if (I->start < J->start) {
-      I = std::upper_bound(I, IE, J->start);
-      if (I != LHS.begin()) --I;
-    } else if (J->start < I->start) {
-      J = std::upper_bound(J, JE, I->start);
-      if (J != RHS.begin()) --J;
+    // Now look at the instructions from VNI->def to TaintExtent (inclusive).
+    MachineBasicBlock *MBB = Indexes->getMBBFromIndex(VNI->def);
+    MachineBasicBlock::iterator MI = MBB->begin();
+    if (!VNI->isPHIDef()) {
+      MI = Indexes->getInstructionFromIndex(VNI->def);
+      // No need to check the instruction defining VNI for reads.
+      ++MI;
     }
-  }
-
-  while (I != IE && J != JE) {
-    // Determine if these two live ranges overlap.
-    // If so, check value # info to determine if they are really different.
-    if (I->end > J->start && J->end > I->start) {
-      // If the live range overlap will map to the same value number in the
-      // result liverange, we can still coalesce them.  If not, we can't.
-      if (LHSValNoAssignments[I->valno->id] !=
-          RHSValNoAssignments[J->valno->id])
+    assert(!SlotIndex::isSameInstr(VNI->def, TaintExtent.front().first) &&
+           "Interference ends on VNI->def. Should have been handled earlier");
+    MachineInstr *LastMI =
+      Indexes->getInstructionFromIndex(TaintExtent.front().first);
+    assert(LastMI && "Range must end at a proper instruction");
+    unsigned TaintNum = 0;
+    for(;;) {
+      assert(MI != MBB->end() && "Bad LastMI");
+      if (usesLanes(MI, Other.LI.reg, Other.SubIdx, TaintedLanes)) {
+        DEBUG(dbgs() << "\t\ttainted lanes used by: " << *MI);
         return false;
-
-      // Extended live ranges should no longer be killed.
-      if (!I->end.isBlock() && I->end < J->end)
-        if (MachineInstr *MI = LIS->getInstructionFromIndex(I->end))
-          LHSOldKills.push_back(MI);
-      if (!J->end.isBlock() && J->end < I->end)
-        if (MachineInstr *MI = LIS->getInstructionFromIndex(J->end))
-          RHSOldKills.push_back(MI);
+      }
+      // LastMI is the last instruction to use the current value.
+      if (&*MI == LastMI) {
+        if (++TaintNum == TaintExtent.size())
+          break;
+        LastMI = Indexes->getInstructionFromIndex(TaintExtent[TaintNum].first);
+        assert(LastMI && "Range must end at a proper instruction");
+        TaintedLanes = TaintExtent[TaintNum].second;
+      }
+      ++MI;
     }
 
-    if (I->end < J->end)
-      ++I;
-    else
-      ++J;
-  }
-
-  // Clear kill flags where live ranges are extended.
-  while (!LHSOldKills.empty())
-    LHSOldKills.pop_back_val()->clearRegisterKills(LHS.reg, TRI);
-  while (!RHSOldKills.empty())
-    RHSOldKills.pop_back_val()->clearRegisterKills(RHS.reg, TRI);
-
-  if (LHSValNoAssignments.empty())
-    LHSValNoAssignments.push_back(-1);
-  if (RHSValNoAssignments.empty())
-    RHSValNoAssignments.push_back(-1);
-
-  // Now erase all the redundant copies.
-  for (unsigned i = 0, e = DeadCopies.size(); i != e; ++i) {
-    MachineInstr *MI = DeadCopies[i];
-    if (!ErasedInstrs.insert(MI))
-      continue;
-    DEBUG(dbgs() << "\t\terased:\t" << LIS->getInstructionIndex(MI)
-                 << '\t' << *MI);
-    LIS->RemoveMachineInstrFromMaps(MI);
-    MI->eraseFromParent();
+    // The tainted lanes are unused.
+    V.Resolution = CR_Replace;
+    ++NumLaneResolves;
   }
+  return true;
+}
 
-  SmallVector<unsigned, 8> SourceRegisters;
-  for (SmallVector<MachineInstr*, 8>::iterator I = DupCopies.begin(),
-         E = DupCopies.end(); I != E; ++I) {
-    MachineInstr *MI = *I;
-    if (!ErasedInstrs.insert(MI))
-      continue;
+// Determine if ValNo is a copy of a value number in LI or Other.LI that will
+// be pruned:
+//
+//   %dst = COPY %src
+//   %src = COPY %dst  <-- This value to be pruned.
+//   %dst = COPY %src  <-- This value is a copy of a pruned value.
+//
+bool JoinVals::isPrunedValue(unsigned ValNo, JoinVals &Other) {
+  Val &V = Vals[ValNo];
+  if (V.Pruned || V.PrunedComputed)
+    return V.Pruned;
+
+  if (V.Resolution != CR_Erase && V.Resolution != CR_Merge)
+    return V.Pruned;
+
+  // Follow copies up the dominator tree and check if any intermediate value
+  // has been pruned.
+  V.PrunedComputed = true;
+  V.Pruned = Other.isPrunedValue(V.OtherVNI->id, *this);
+  return V.Pruned;
+}
 
-    // If MI is a copy, then we have pretended that the assignment to B in
-    // A = X
-    // B = X
-    // was actually a copy from A. Now that we decided to coalesce A and B,
-    // transform the code into
-    // A = X
-    // In the case of the implicit_def, we just have to remove it.
-    if (!MI->isImplicitDef()) {
-      unsigned Src = MI->getOperand(1).getReg();
-      SourceRegisters.push_back(Src);
+void JoinVals::pruneValues(JoinVals &Other,
+                           SmallVectorImpl<SlotIndex> &EndPoints) {
+  for (unsigned i = 0, e = LI.getNumValNums(); i != e; ++i) {
+    SlotIndex Def = LI.getValNumInfo(i)->def;
+    switch (Vals[i].Resolution) {
+    case CR_Keep:
+      break;
+    case CR_Replace: {
+      // This value takes precedence over the value in Other.LI.
+      LIS->pruneValue(&Other.LI, Def, &EndPoints);
+      // Check if we're replacing an IMPLICIT_DEF value. The IMPLICIT_DEF
+      // instructions are only inserted to provide a live-out value for PHI
+      // predecessors, so the instruction should simply go away once its value
+      // has been replaced.
+      Val &OtherV = Other.Vals[Vals[i].OtherVNI->id];
+      bool EraseImpDef = OtherV.IsImplicitDef && OtherV.Resolution == CR_Keep;
+      if (!Def.isBlock()) {
+        // Remove <def,read-undef> flags. This def is now a partial redef.
+        // Also remove <def,dead> flags since the joined live range will
+        // continue past this instruction.
+        for (MIOperands MO(Indexes->getInstructionFromIndex(Def));
+             MO.isValid(); ++MO)
+          if (MO->isReg() && MO->isDef() && MO->getReg() == LI.reg) {
+            MO->setIsUndef(EraseImpDef);
+            MO->setIsDead(false);
+          }
+        // This value will reach instructions below, but we need to make sure
+        // the live range also reaches the instruction at Def.
+        if (!EraseImpDef)
+          EndPoints.push_back(Def);
+      }
+      DEBUG(dbgs() << "\t\tpruned " << PrintReg(Other.LI.reg) << " at " << Def
+                   << ": " << Other.LI << '\n');
+      break;
+    }
+    case CR_Erase:
+    case CR_Merge:
+      if (isPrunedValue(i, Other)) {
+        // This value is ultimately a copy of a pruned value in LI or Other.LI.
+        // We can no longer trust the value mapping computed by
+        // computeAssignment(), the value that was originally copied could have
+        // been replaced.
+        LIS->pruneValue(&LI, Def, &EndPoints);
+        DEBUG(dbgs() << "\t\tpruned all of " << PrintReg(LI.reg) << " at "
+                     << Def << ": " << LI << '\n');
+      }
+      break;
+    case CR_Unresolved:
+    case CR_Impossible:
+      llvm_unreachable("Unresolved conflicts");
     }
-    LIS->RemoveMachineInstrFromMaps(MI);
-    MI->eraseFromParent();
   }
+}
 
-  // If B = X was the last use of X in a liverange, we have to shrink it now
-  // that B = X is gone.
-  for (SmallVector<unsigned, 8>::iterator I = SourceRegisters.begin(),
-         E = SourceRegisters.end(); I != E; ++I) {
-    LIS->shrinkToUses(&LIS->getInterval(*I));
+void JoinVals::eraseInstrs(SmallPtrSet<MachineInstr*, 8> &ErasedInstrs,
+                           SmallVectorImpl<unsigned> &ShrinkRegs) {
+  for (unsigned i = 0, e = LI.getNumValNums(); i != e; ++i) {
+    // Get the def location before markUnused() below invalidates it.
+    SlotIndex Def = LI.getValNumInfo(i)->def;
+    switch (Vals[i].Resolution) {
+    case CR_Keep:
+      // If an IMPLICIT_DEF value is pruned, it doesn't serve a purpose any
+      // longer. The IMPLICIT_DEF instructions are only inserted by
+      // PHIElimination to guarantee that all PHI predecessors have a value.
+      if (!Vals[i].IsImplicitDef || !Vals[i].Pruned)
+        break;
+      // Remove value number i from LI. Note that this VNInfo is still present
+      // in NewVNInfo, so it will appear as an unused value number in the final
+      // joined interval.
+      LI.getValNumInfo(i)->markUnused();
+      LI.removeValNo(LI.getValNumInfo(i));
+      DEBUG(dbgs() << "\t\tremoved " << i << '@' << Def << ": " << LI << '\n');
+      // FALL THROUGH.
+
+    case CR_Erase: {
+      MachineInstr *MI = Indexes->getInstructionFromIndex(Def);
+      assert(MI && "No instruction to erase");
+      if (MI->isCopy()) {
+        unsigned Reg = MI->getOperand(1).getReg();
+        if (TargetRegisterInfo::isVirtualRegister(Reg) &&
+            Reg != CP.getSrcReg() && Reg != CP.getDstReg())
+          ShrinkRegs.push_back(Reg);
+      }
+      ErasedInstrs.insert(MI);
+      DEBUG(dbgs() << "\t\terased:\t" << Def << '\t' << *MI);
+      LIS->RemoveMachineInstrFromMaps(MI);
+      MI->eraseFromParent();
+      break;
+    }
+    default:
+      break;
+    }
   }
+}
+
+bool RegisterCoalescer::joinVirtRegs(CoalescerPair &CP) {
+  SmallVector<VNInfo*, 16> NewVNInfo;
+  LiveInterval &RHS = LIS->getInterval(CP.getSrcReg());
+  LiveInterval &LHS = LIS->getInterval(CP.getDstReg());
+  JoinVals RHSVals(RHS, CP.getSrcIdx(), NewVNInfo, CP, LIS, TRI);
+  JoinVals LHSVals(LHS, CP.getDstIdx(), NewVNInfo, CP, LIS, TRI);
+
+  DEBUG(dbgs() << "\t\tRHS = " << PrintReg(CP.getSrcReg()) << ' ' << RHS
+               << "\n\t\tLHS = " << PrintReg(CP.getDstReg()) << ' ' << LHS
+               << '\n');
+
+  // First compute NewVNInfo and the simple value mappings.
+  // Detect impossible conflicts early.
+  if (!LHSVals.mapValues(RHSVals) || !RHSVals.mapValues(LHSVals))
+    return false;
+
+  // Some conflicts can only be resolved after all values have been mapped.
+  if (!LHSVals.resolveConflicts(RHSVals) || !RHSVals.resolveConflicts(LHSVals))
+    return false;
 
-  // If we get here, we know that we can coalesce the live ranges.  Ask the
-  // intervals to coalesce themselves now.
-  LHS.join(RHS, &LHSValNoAssignments[0], &RHSValNoAssignments[0], NewVNInfo,
+  // All clear, the live ranges can be merged.
+
+  // The merging algorithm in LiveInterval::join() can't handle conflicting
+  // value mappings, so we need to remove any live ranges that overlap a
+  // CR_Replace resolution. Collect a set of end points that can be used to
+  // restore the live range after joining.
+  SmallVector<SlotIndex, 8> EndPoints;
+  LHSVals.pruneValues(RHSVals, EndPoints);
+  RHSVals.pruneValues(LHSVals, EndPoints);
+
+  // Erase COPY and IMPLICIT_DEF instructions. This may cause some external
+  // registers to require trimming.
+  SmallVector<unsigned, 8> ShrinkRegs;
+  LHSVals.eraseInstrs(ErasedInstrs, ShrinkRegs);
+  RHSVals.eraseInstrs(ErasedInstrs, ShrinkRegs);
+  while (!ShrinkRegs.empty())
+    LIS->shrinkToUses(&LIS->getInterval(ShrinkRegs.pop_back_val()));
+
+  // Join RHS into LHS.
+  LHS.join(RHS, LHSVals.getAssignments(), RHSVals.getAssignments(), NewVNInfo,
            MRI);
+
+  // Kill flags are going to be wrong if the live ranges were overlapping.
+  // Eventually, we should simply clear all kill flags when computing live
+  // ranges. They are reinserted after register allocation.
+  MRI->clearKillFlags(LHS.reg);
+  MRI->clearKillFlags(RHS.reg);
+
+  if (EndPoints.empty())
+    return true;
+
+  // Recompute the parts of the live range we had to remove because of
+  // CR_Replace conflicts.
+  DEBUG(dbgs() << "\t\trestoring liveness to " << EndPoints.size()
+               << " points: " << LHS << '\n');
+  LIS->extendToIndices(&LHS, EndPoints);
   return true;
 }
 
+/// joinIntervals - Attempt to join these two intervals.  On failure, this
+/// returns false.
+bool RegisterCoalescer::joinIntervals(CoalescerPair &CP) {
+  return CP.isPhys() ? joinReservedPhysReg(CP) : joinVirtRegs(CP);
+}
+
 namespace {
   // DepthMBBCompare - Comparison predicate that sort first based on the loop
   // depth of the basic block (the unsigned), and then on the MBB number.
@@ -1564,8 +2024,7 @@ bool RegisterCoalescer::runOnMachineFunction(MachineFunction &fn) {
   Loops = &getAnalysis<MachineLoopInfo>();
 
   DEBUG(dbgs() << "********** SIMPLE REGISTER COALESCING **********\n"
-               << "********** Function: "
-               << ((Value*)MF->getFunction())->getName() << '\n');
+               << "********** Function: " << MF->getName() << '\n');
 
   if (VerifyCoalescing)
     MF->verify(this, "Before register coalescing");
diff --git a/lib/CodeGen/RegisterCoalescer.h b/lib/CodeGen/RegisterCoalescer.h
index 8a6df98..47c3df1 100644
--- a/lib/CodeGen/RegisterCoalescer.h
+++ b/lib/CodeGen/RegisterCoalescer.h
@@ -63,6 +63,13 @@ namespace llvm {
       : TRI(tri), DstReg(0), SrcReg(0), DstIdx(0), SrcIdx(0),
         Partial(false), CrossClass(false), Flipped(false), NewRC(0) {}
 
+    /// Create a CoalescerPair representing a virtreg-to-physreg copy.
+    /// No need to call setRegisters().
+    CoalescerPair(unsigned VirtReg, unsigned PhysReg,
+                  const TargetRegisterInfo &tri)
+      : TRI(tri), DstReg(PhysReg), SrcReg(VirtReg), DstIdx(0), SrcIdx(0),
+        Partial(false), CrossClass(false), Flipped(false), NewRC(0) {}
+
     /// setRegisters - set registers to match the copy instruction MI. Return
     /// false if MI is not a coalescable copy instruction.
     bool setRegisters(const MachineInstr*);
diff --git a/lib/CodeGen/RegisterPressure.cpp b/lib/CodeGen/RegisterPressure.cpp
index 43448c8..543c426 100644
--- a/lib/CodeGen/RegisterPressure.cpp
+++ b/lib/CodeGen/RegisterPressure.cpp
@@ -63,7 +63,8 @@ void RegisterPressure::decrease(const TargetRegisterClass *RC,
   decreaseSetPressure(MaxSetPressure, RC, TRI);
 }
 
-void RegisterPressure::dump(const TargetRegisterInfo *TRI) {
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+void RegisterPressure::dump(const TargetRegisterInfo *TRI) const {
   dbgs() << "Live In: ";
   for (unsigned i = 0, e = LiveInRegs.size(); i < e; ++i)
     dbgs() << PrintReg(LiveInRegs[i], TRI) << " ";
@@ -78,6 +79,7 @@ void RegisterPressure::dump(const TargetRegisterInfo *TRI) {
              << '\n';
   }
 }
+#endif
 
 /// Increase the current pressure as impacted by these physical registers and
 /// bump the high water mark if needed.
@@ -320,10 +322,8 @@ struct RegisterOperands {
         if (findReg(MO.getReg(), isVReg, DeadDefs, TRI) == DeadDefs.end())
           DeadDefs.push_back(MO.getReg());
       }
-      else {
-        if (findReg(MO.getReg(), isVReg, Defs, TRI) == Defs.end())
-          Defs.push_back(MO.getReg());
-      }
+      else if (findReg(MO.getReg(), isVReg, Defs, TRI) == Defs.end())
+        Defs.push_back(MO.getReg());
     }
   }
 };
@@ -335,7 +335,7 @@ static void collectOperands(const MachineInstr *MI,
                             PhysRegOperands &PhysRegOpers,
                             VirtRegOperands &VirtRegOpers,
                             const TargetRegisterInfo *TRI,
-                            const RegisterClassInfo *RCI) {
+                            const MachineRegisterInfo *MRI) {
   for(ConstMIBundleOperands OperI(MI); OperI.isValid(); ++OperI) {
     const MachineOperand &MO = *OperI;
     if (!MO.isReg() || !MO.getReg())
@@ -343,7 +343,7 @@ static void collectOperands(const MachineInstr *MI,
 
     if (TargetRegisterInfo::isVirtualRegister(MO.getReg()))
       VirtRegOpers.collect(MO, TRI);
-    else if (RCI->isAllocatable(MO.getReg()))
+    else if (MRI->isAllocatable(MO.getReg()))
       PhysRegOpers.collect(MO, TRI);
   }
   // Remove redundant physreg dead defs.
@@ -449,7 +449,7 @@ bool RegPressureTracker::recede() {
 
   PhysRegOperands PhysRegOpers;
   VirtRegOperands VirtRegOpers;
-  collectOperands(CurrPos, PhysRegOpers, VirtRegOpers, TRI, RCI);
+  collectOperands(CurrPos, PhysRegOpers, VirtRegOpers, TRI, MRI);
 
   // Boost pressure for all dead defs together.
   increasePhysRegPressure(PhysRegOpers.DeadDefs);
@@ -522,7 +522,7 @@ bool RegPressureTracker::advance() {
 
   PhysRegOperands PhysRegOpers;
   VirtRegOperands VirtRegOpers;
-  collectOperands(CurrPos, PhysRegOpers, VirtRegOpers, TRI, RCI);
+  collectOperands(CurrPos, PhysRegOpers, VirtRegOpers, TRI, MRI);
 
   // Kill liveness at last uses.
   for (unsigned i = 0, e = PhysRegOpers.Uses.size(); i < e; ++i) {
@@ -664,7 +664,7 @@ void RegPressureTracker::bumpUpwardPressure(const MachineInstr *MI) {
   // Account for register pressure similar to RegPressureTracker::recede().
   PhysRegOperands PhysRegOpers;
   VirtRegOperands VirtRegOpers;
-  collectOperands(MI, PhysRegOpers, VirtRegOpers, TRI, RCI);
+  collectOperands(MI, PhysRegOpers, VirtRegOpers, TRI, MRI);
 
   // Boost max pressure for all dead defs together.
   // Since CurrSetPressure and MaxSetPressure
@@ -674,9 +674,16 @@ void RegPressureTracker::bumpUpwardPressure(const MachineInstr *MI) {
   decreaseVirtRegPressure(VirtRegOpers.DeadDefs);
 
   // Kill liveness at live defs.
-  decreasePhysRegPressure(PhysRegOpers.Defs);
-  decreaseVirtRegPressure(VirtRegOpers.Defs);
-
+  for (unsigned i = 0, e = PhysRegOpers.Defs.size(); i < e; ++i) {
+    unsigned Reg = PhysRegOpers.Defs[i];
+    if (!findReg(Reg, false, PhysRegOpers.Uses, TRI))
+      decreasePhysRegPressure(PhysRegOpers.Defs);
+  }
+  for (unsigned i = 0, e = VirtRegOpers.Defs.size(); i < e; ++i) {
+    unsigned Reg = VirtRegOpers.Defs[i];
+    if (!findReg(Reg, true, VirtRegOpers.Uses, TRI))
+      decreaseVirtRegPressure(VirtRegOpers.Defs);
+  }
   // Generate liveness for uses.
   for (unsigned i = 0, e = PhysRegOpers.Uses.size(); i < e; ++i) {
     unsigned Reg = PhysRegOpers.Uses[i];
@@ -750,7 +757,7 @@ void RegPressureTracker::bumpDownwardPressure(const MachineInstr *MI) {
   // Account for register pressure similar to RegPressureTracker::recede().
   PhysRegOperands PhysRegOpers;
   VirtRegOperands VirtRegOpers;
-  collectOperands(MI, PhysRegOpers, VirtRegOpers, TRI, RCI);
+  collectOperands(MI, PhysRegOpers, VirtRegOpers, TRI, MRI);
 
   // Kill liveness at last uses. Assume allocatable physregs are single-use
   // rather than checking LiveIntervals.
diff --git a/lib/CodeGen/RegisterScavenging.cpp b/lib/CodeGen/RegisterScavenging.cpp
index d673794..5ec6564 100644
--- a/lib/CodeGen/RegisterScavenging.cpp
+++ b/lib/CodeGen/RegisterScavenging.cpp
@@ -92,9 +92,6 @@ void RegScavenger::enterBasicBlock(MachineBasicBlock *mbb) {
     KillRegs.resize(NumPhysRegs);
     DefRegs.resize(NumPhysRegs);
 
-    // Create reserved registers bitvector.
-    ReservedRegs = TRI->getReservedRegs(MF);
-
     // Create callee-saved registers bitvector.
     CalleeSavedRegs.resize(NumPhysRegs);
     const uint16_t *CSRegs = TRI->getCalleeSavedRegs(&MF);
@@ -225,9 +222,9 @@ void RegScavenger::getRegsUsed(BitVector &used, bool includeReserved) {
   used = RegsAvailable;
   used.flip();
   if (includeReserved)
-    used |= ReservedRegs;
+    used |= MRI->getReservedRegs();
   else
-    used.reset(ReservedRegs);
+    used.reset(MRI->getReservedRegs());
 }
 
 unsigned RegScavenger::FindUnusedReg(const TargetRegisterClass *RC) const {
diff --git a/lib/CodeGen/ScheduleDAG.cpp b/lib/CodeGen/ScheduleDAG.cpp
index 752f8e4..9a65071 100644
--- a/lib/CodeGen/ScheduleDAG.cpp
+++ b/lib/CodeGen/ScheduleDAG.cpp
@@ -279,6 +279,7 @@ void SUnit::ComputeHeight() {
   } while (!WorkList.empty());
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 /// SUnit - Scheduling unit. It's an wrapper around either a single SDNode or
 /// a group of nodes flagged together.
 void SUnit::dump(const ScheduleDAG *G) const {
@@ -336,6 +337,7 @@ void SUnit::dumpAll(const ScheduleDAG *G) const {
   }
   dbgs() << "\n";
 }
+#endif
 
 #ifndef NDEBUG
 /// VerifyScheduledDAG - Verify that all SUnits were scheduled and that
diff --git a/lib/CodeGen/ScheduleDAGInstrs.cpp b/lib/CodeGen/ScheduleDAGInstrs.cpp
index 9c1dba3..a4d4a93 100644
--- a/lib/CodeGen/ScheduleDAGInstrs.cpp
+++ b/lib/CodeGen/ScheduleDAGInstrs.cpp
@@ -22,6 +22,7 @@
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/PseudoSourceValue.h"
 #include "llvm/CodeGen/RegisterPressure.h"
+#include "llvm/CodeGen/ScheduleDAGILP.h"
 #include "llvm/CodeGen/ScheduleDAGInstrs.h"
 #include "llvm/MC/MCInstrItineraries.h"
 #include "llvm/Target/TargetMachine.h"
@@ -30,6 +31,7 @@
 #include "llvm/Target/TargetSubtargetInfo.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/Format.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -44,14 +46,15 @@ ScheduleDAGInstrs::ScheduleDAGInstrs(MachineFunction &mf,
                                      const MachineDominatorTree &mdt,
                                      bool IsPostRAFlag,
                                      LiveIntervals *lis)
-  : ScheduleDAG(mf), MLI(mli), MDT(mdt), MFI(mf.getFrameInfo()),
-    InstrItins(mf.getTarget().getInstrItineraryData()), LIS(lis),
-    IsPostRA(IsPostRAFlag), UnitLatencies(false), CanHandleTerminators(false),
-    LoopRegs(MDT), FirstDbgValue(0) {
+  : ScheduleDAG(mf), MLI(mli), MDT(mdt), MFI(mf.getFrameInfo()), LIS(lis),
+    IsPostRA(IsPostRAFlag), CanHandleTerminators(false), FirstDbgValue(0) {
   assert((IsPostRA || LIS) && "PreRA scheduling requires LiveIntervals");
   DbgValues.clear();
   assert(!(IsPostRA && MRI.getNumVirtRegs()) &&
          "Virtual registers must be removed prior to PostRA scheduling");
+
+  const TargetSubtargetInfo &ST = TM.getSubtarget<TargetSubtargetInfo>();
+  SchedModel.init(*ST.getSchedModel(), &ST, TII);
 }
 
 /// getUnderlyingObjectFromInt - This is the function that does the work of
@@ -68,7 +71,7 @@ static const Value *getUnderlyingObjectFromInt(const Value *V) {
       // object. We don't have to worry about the case where the
       // object address is somehow being computed by the multiply,
       // because our callers only care when the result is an
-      // identifibale object.
+      // identifiable object.
       if (U->getOpcode() != Instruction::Add ||
           (!isa<ConstantInt>(U->getOperand(1)) &&
            Operator::getOpcode(U->getOperand(1)) != Instruction::Mul))
@@ -135,10 +138,6 @@ static const Value *getUnderlyingObjectForInstr(const MachineInstr *MI,
 
 void ScheduleDAGInstrs::startBlock(MachineBasicBlock *bb) {
   BB = bb;
-  LoopRegs.Deps.clear();
-  if (MachineLoop *ML = MLI.getLoopFor(BB))
-    if (BB == ML->getLoopLatch())
-      LoopRegs.VisitLoop(ML);
 }
 
 void ScheduleDAGInstrs::finishBlock() {
@@ -174,9 +173,6 @@ void ScheduleDAGInstrs::enterRegion(MachineBasicBlock *bb,
   EndIndex = endcount;
   MISUnitMap.clear();
 
-  // Check to see if the scheduler cares about latencies.
-  UnitLatencies = forceUnitLatencies();
-
   ScheduleDAG::clearDAG();
 }
 
@@ -209,7 +205,7 @@ void ScheduleDAGInstrs::addSchedBarrierDeps() {
       if (Reg == 0) continue;
 
       if (TRI->isPhysicalRegister(Reg))
-        Uses[Reg].push_back(&ExitSU);
+        Uses[Reg].push_back(PhysRegSUOper(&ExitSU, -1));
       else {
         assert(!IsPostRA && "Virtual register encountered after regalloc.");
         addVRegUseDeps(&ExitSU, i);
@@ -225,59 +221,44 @@ void ScheduleDAGInstrs::addSchedBarrierDeps() {
              E = (*SI)->livein_end(); I != E; ++I) {
         unsigned Reg = *I;
         if (!Uses.contains(Reg))
-          Uses[Reg].push_back(&ExitSU);
+          Uses[Reg].push_back(PhysRegSUOper(&ExitSU, -1));
       }
   }
 }
 
 /// MO is an operand of SU's instruction that defines a physical register. Add
 /// data dependencies from SU to any uses of the physical register.
-void ScheduleDAGInstrs::addPhysRegDataDeps(SUnit *SU,
-                                           const MachineOperand &MO) {
+void ScheduleDAGInstrs::addPhysRegDataDeps(SUnit *SU, unsigned OperIdx) {
+  const MachineOperand &MO = SU->getInstr()->getOperand(OperIdx);
   assert(MO.isDef() && "expect physreg def");
 
   // Ask the target if address-backscheduling is desirable, and if so how much.
   const TargetSubtargetInfo &ST = TM.getSubtarget<TargetSubtargetInfo>();
-  unsigned SpecialAddressLatency = ST.getSpecialAddressLatency();
-  unsigned DataLatency = SU->Latency;
 
   for (MCRegAliasIterator Alias(MO.getReg(), TRI, true);
        Alias.isValid(); ++Alias) {
     if (!Uses.contains(*Alias))
       continue;
-    std::vector<SUnit*> &UseList = Uses[*Alias];
+    std::vector<PhysRegSUOper> &UseList = Uses[*Alias];
     for (unsigned i = 0, e = UseList.size(); i != e; ++i) {
-      SUnit *UseSU = UseList[i];
+      SUnit *UseSU = UseList[i].SU;
       if (UseSU == SU)
         continue;
-      unsigned LDataLatency = DataLatency;
-      // Optionally add in a special extra latency for nodes that
-      // feed addresses.
-      // TODO: Perhaps we should get rid of
-      // SpecialAddressLatency and just move this into
-      // adjustSchedDependency for the targets that care about it.
-      if (SpecialAddressLatency != 0 && !UnitLatencies &&
-          UseSU != &ExitSU) {
-        MachineInstr *UseMI = UseSU->getInstr();
-        const MCInstrDesc &UseMCID = UseMI->getDesc();
-        int RegUseIndex = UseMI->findRegisterUseOperandIdx(*Alias);
-        assert(RegUseIndex >= 0 && "UseMI doesn't use register!");
-        if (RegUseIndex >= 0 &&
-            (UseMI->mayLoad() || UseMI->mayStore()) &&
-            (unsigned)RegUseIndex < UseMCID.getNumOperands() &&
-            UseMCID.OpInfo[RegUseIndex].isLookupPtrRegClass())
-          LDataLatency += SpecialAddressLatency;
-      }
-      // Adjust the dependence latency using operand def/use
-      // information (if any), and then allow the target to
-      // perform its own adjustments.
-      SDep dep(SU, SDep::Data, LDataLatency, *Alias);
-      if (!UnitLatencies) {
-        unsigned Latency = computeOperandLatency(SU, UseSU, dep);
-        dep.setLatency(Latency);
-
-        ST.adjustSchedDependency(SU, UseSU, dep);
-      }
+
+      SDep dep(SU, SDep::Data, *Alias);
+
+      // Adjust the dependence latency using operand def/use information,
+      // then allow the target to perform its own adjustments.
+      int UseOp = UseList[i].OpIdx;
+      MachineInstr *RegUse = UseOp < 0 ? 0 : UseSU->getInstr();
+      dep.setLatency(
+        SchedModel.computeOperandLatency(SU->getInstr(), OperIdx,
+                                         RegUse, UseOp, /*FindMin=*/false));
+      dep.setMinLatency(
+        SchedModel.computeOperandLatency(SU->getInstr(), OperIdx,
+                                         RegUse, UseOp, /*FindMin=*/true));
+
+      ST.adjustSchedDependency(SU, UseSU, dep);
       UseSU->addPred(dep);
     }
   }
@@ -301,20 +282,23 @@ void ScheduleDAGInstrs::addPhysRegDeps(SUnit *SU, unsigned OperIdx) {
        Alias.isValid(); ++Alias) {
     if (!Defs.contains(*Alias))
       continue;
-    std::vector<SUnit *> &DefList = Defs[*Alias];
+    std::vector<PhysRegSUOper> &DefList = Defs[*Alias];
     for (unsigned i = 0, e = DefList.size(); i != e; ++i) {
-      SUnit *DefSU = DefList[i];
+      SUnit *DefSU = DefList[i].SU;
       if (DefSU == &ExitSU)
         continue;
       if (DefSU != SU &&
           (Kind != SDep::Output || !MO.isDead() ||
            !DefSU->getInstr()->registerDefIsDead(*Alias))) {
         if (Kind == SDep::Anti)
-          DefSU->addPred(SDep(SU, Kind, 0, /*Reg=*/*Alias));
+          DefSU->addPred(SDep(SU, Kind, /*Reg=*/*Alias));
         else {
-          unsigned AOLat = TII->getOutputLatency(InstrItins, MI, OperIdx,
-                                                 DefSU->getInstr());
-          DefSU->addPred(SDep(SU, Kind, AOLat, /*Reg=*/*Alias));
+          SDep Dep(SU, Kind, /*Reg=*/*Alias);
+          unsigned OutLatency =
+            SchedModel.computeOutputLatency(MI, OperIdx, DefSU->getInstr());
+          Dep.setMinLatency(OutLatency);
+          Dep.setLatency(OutLatency);
+          DefSU->addPred(Dep);
         }
       }
     }
@@ -324,61 +308,14 @@ void ScheduleDAGInstrs::addPhysRegDeps(SUnit *SU, unsigned OperIdx) {
     // Either insert a new Reg2SUnits entry with an empty SUnits list, or
     // retrieve the existing SUnits list for this register's uses.
     // Push this SUnit on the use list.
-    Uses[MO.getReg()].push_back(SU);
+    Uses[MO.getReg()].push_back(PhysRegSUOper(SU, OperIdx));
   }
   else {
-    addPhysRegDataDeps(SU, MO);
+    addPhysRegDataDeps(SU, OperIdx);
 
     // Either insert a new Reg2SUnits entry with an empty SUnits list, or
     // retrieve the existing SUnits list for this register's defs.
-    std::vector<SUnit *> &DefList = Defs[MO.getReg()];
-
-    // If a def is going to wrap back around to the top of the loop,
-    // backschedule it.
-    if (!UnitLatencies && DefList.empty()) {
-      LoopDependencies::LoopDeps::iterator I = LoopRegs.Deps.find(MO.getReg());
-      if (I != LoopRegs.Deps.end()) {
-        const MachineOperand *UseMO = I->second.first;
-        unsigned Count = I->second.second;
-        const MachineInstr *UseMI = UseMO->getParent();
-        unsigned UseMOIdx = UseMO - &UseMI->getOperand(0);
-        const MCInstrDesc &UseMCID = UseMI->getDesc();
-        const TargetSubtargetInfo &ST =
-          TM.getSubtarget<TargetSubtargetInfo>();
-        unsigned SpecialAddressLatency = ST.getSpecialAddressLatency();
-        // TODO: If we knew the total depth of the region here, we could
-        // handle the case where the whole loop is inside the region but
-        // is large enough that the isScheduleHigh trick isn't needed.
-        if (UseMOIdx < UseMCID.getNumOperands()) {
-          // Currently, we only support scheduling regions consisting of
-          // single basic blocks. Check to see if the instruction is in
-          // the same region by checking to see if it has the same parent.
-          if (UseMI->getParent() != MI->getParent()) {
-            unsigned Latency = SU->Latency;
-            if (UseMCID.OpInfo[UseMOIdx].isLookupPtrRegClass())
-              Latency += SpecialAddressLatency;
-            // This is a wild guess as to the portion of the latency which
-            // will be overlapped by work done outside the current
-            // scheduling region.
-            Latency -= std::min(Latency, Count);
-            // Add the artificial edge.
-            ExitSU.addPred(SDep(SU, SDep::Order, Latency,
-                                /*Reg=*/0, /*isNormalMemory=*/false,
-                                /*isMustAlias=*/false,
-                                /*isArtificial=*/true));
-          } else if (SpecialAddressLatency > 0 &&
-                     UseMCID.OpInfo[UseMOIdx].isLookupPtrRegClass()) {
-            // The entire loop body is within the current scheduling region
-            // and the latency of this operation is assumed to be greater
-            // than the latency of the loop.
-            // TODO: Recursively mark data-edge predecessors as
-            //       isScheduleHigh too.
-            SU->isScheduleHigh = true;
-          }
-        }
-        LoopRegs.Deps.erase(I);
-      }
-    }
+    std::vector<PhysRegSUOper> &DefList = Defs[MO.getReg()];
 
     // clear this register's use list
     if (Uses.contains(MO.getReg()))
@@ -393,11 +330,11 @@ void ScheduleDAGInstrs::addPhysRegDeps(SUnit *SU, unsigned OperIdx) {
     // the block. Instead, we leave only one call at the back of the
     // DefList.
     if (SU->isCall) {
-      while (!DefList.empty() && DefList.back()->isCall)
+      while (!DefList.empty() && DefList.back().SU->isCall)
         DefList.pop_back();
     }
     // Defs are pushed in the order they are visited and never reordered.
-    DefList.push_back(SU);
+    DefList.push_back(PhysRegSUOper(SU, OperIdx));
   }
 }
 
@@ -430,9 +367,12 @@ void ScheduleDAGInstrs::addVRegDefDeps(SUnit *SU, unsigned OperIdx) {
   else {
     SUnit *DefSU = DefI->SU;
     if (DefSU != SU && DefSU != &ExitSU) {
-      unsigned OutLatency = TII->getOutputLatency(InstrItins, MI, OperIdx,
-                                                  DefSU->getInstr());
-      DefSU->addPred(SDep(SU, SDep::Output, OutLatency, Reg));
+      SDep Dep(SU, SDep::Output, Reg);
+      unsigned OutLatency =
+        SchedModel.computeOutputLatency(MI, OperIdx, DefSU->getInstr());
+      Dep.setMinLatency(OutLatency);
+      Dep.setLatency(OutLatency);
+      DefSU->addPred(Dep);
     }
     DefI->SU = SU;
   }
@@ -462,18 +402,17 @@ void ScheduleDAGInstrs::addVRegUseDeps(SUnit *SU, unsigned OperIdx) {
     if (DefSU) {
       // The reaching Def lives within this scheduling region.
       // Create a data dependence.
-      //
-      // TODO: Handle "special" address latencies cleanly.
-      SDep dep(DefSU, SDep::Data, DefSU->Latency, Reg);
-      if (!UnitLatencies) {
-        // Adjust the dependence latency using operand def/use information, then
-        // allow the target to perform its own adjustments.
-        unsigned Latency = computeOperandLatency(DefSU, SU, const_cast<SDep &>(dep));
-        dep.setLatency(Latency);
-
-        const TargetSubtargetInfo &ST = TM.getSubtarget<TargetSubtargetInfo>();
-        ST.adjustSchedDependency(DefSU, SU, const_cast<SDep &>(dep));
-      }
+      SDep dep(DefSU, SDep::Data, Reg);
+      // Adjust the dependence latency using operand def/use information, then
+      // allow the target to perform its own adjustments.
+      int DefOp = Def->findRegisterDefOperandIdx(Reg);
+      dep.setLatency(
+        SchedModel.computeOperandLatency(Def, DefOp, MI, OperIdx, false));
+      dep.setMinLatency(
+        SchedModel.computeOperandLatency(Def, DefOp, MI, OperIdx, true));
+
+      const TargetSubtargetInfo &ST = TM.getSubtarget<TargetSubtargetInfo>();
+      ST.adjustSchedDependency(DefSU, SU, const_cast<SDep &>(dep));
       SU->addPred(dep);
     }
   }
@@ -481,14 +420,14 @@ void ScheduleDAGInstrs::addVRegUseDeps(SUnit *SU, unsigned OperIdx) {
   // Add antidependence to the following def of the vreg it uses.
   VReg2SUnitMap::iterator DefI = VRegDefs.find(Reg);
   if (DefI != VRegDefs.end() && DefI->SU != SU)
-    DefI->SU->addPred(SDep(SU, SDep::Anti, 0, Reg));
+    DefI->SU->addPred(SDep(SU, SDep::Anti, Reg));
 }
 
 /// Return true if MI is an instruction we are unable to reason about
 /// (like a call or something with unmodeled side effects).
 static inline bool isGlobalMemoryObject(AliasAnalysis *AA, MachineInstr *MI) {
   if (MI->isCall() || MI->hasUnmodeledSideEffects() ||
-      (MI->hasVolatileMemoryRef() &&
+      (MI->hasOrderedMemoryRef() &&
        (!MI->mayLoad() || !MI->isInvariantLoad(AA))))
     return true;
   return false;
@@ -621,8 +560,7 @@ iterateChainSucc(AliasAnalysis *AA, const MachineFrameInfo *MFI,
   // and stop descending.
   if (*Depth > 200 ||
       MIsNeedChainEdge(AA, MFI, SUa->getInstr(), SUb->getInstr())) {
-    SUb->addPred(SDep(SUa, SDep::Order, /*Latency=*/0, /*Reg=*/0,
-                      /*isNormalMemory=*/true));
+    SUb->addPred(SDep(SUa, SDep::MayAliasMem));
     return *Depth;
   }
   // Track current depth.
@@ -653,9 +591,9 @@ static void adjustChainDeps(AliasAnalysis *AA, const MachineFrameInfo *MFI,
     if (SU == *I)
       continue;
     if (MIsNeedChainEdge(AA, MFI, SU->getInstr(), (*I)->getInstr())) {
-      unsigned Latency = ((*I)->getInstr()->mayLoad()) ? LatencyToLoad : 0;
-      (*I)->addPred(SDep(SU, SDep::Order, Latency, /*Reg=*/0,
-                         /*isNormalMemory=*/true));
+      SDep Dep(SU, SDep::MayAliasMem);
+      Dep.setLatency(((*I)->getInstr()->mayLoad()) ? LatencyToLoad : 0);
+      (*I)->addPred(Dep);
     }
     // Now go through all the chain successors and iterate from them.
     // Keep track of visited nodes.
@@ -678,9 +616,11 @@ void addChainDependency (AliasAnalysis *AA, const MachineFrameInfo *MFI,
   // If this is a false dependency,
   // do not add the edge, but rememeber the rejected node.
   if (!EnableAASchedMI ||
-      MIsNeedChainEdge(AA, MFI, SUa->getInstr(), SUb->getInstr()))
-    SUb->addPred(SDep(SUa, SDep::Order, TrueMemOrderLatency, /*Reg=*/0,
-                      isNormalMemory));
+      MIsNeedChainEdge(AA, MFI, SUa->getInstr(), SUb->getInstr())) {
+    SDep Dep(SUa, isNormalMemory ? SDep::MayAliasMem : SDep::Barrier);
+    Dep.setLatency(TrueMemOrderLatency);
+    SUb->addPred(Dep);
+  }
   else {
     // Duplicate entries should be ignored.
     RejectList.insert(SUb);
@@ -718,10 +658,7 @@ void ScheduleDAGInstrs::initSUnits() {
     SU->isCommutable = MI->isCommutable();
 
     // Assign the Latency field of SU using target-provided information.
-    if (UnitLatencies)
-      SU->Latency = 1;
-    else
-      computeLatency(SU);
+    SU->Latency = SchedModel.computeInstrLatency(SU->getInstr());
   }
 }
 
@@ -825,16 +762,19 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA,
       // references, even those that are known to not alias.
       for (std::map<const Value *, SUnit *>::iterator I =
              NonAliasMemDefs.begin(), E = NonAliasMemDefs.end(); I != E; ++I) {
-        I->second->addPred(SDep(SU, SDep::Order, /*Latency=*/0));
+        I->second->addPred(SDep(SU, SDep::Barrier));
       }
       for (std::map<const Value *, std::vector<SUnit *> >::iterator I =
              NonAliasMemUses.begin(), E = NonAliasMemUses.end(); I != E; ++I) {
-        for (unsigned i = 0, e = I->second.size(); i != e; ++i)
-          I->second[i]->addPred(SDep(SU, SDep::Order, TrueMemOrderLatency));
+        for (unsigned i = 0, e = I->second.size(); i != e; ++i) {
+          SDep Dep(SU, SDep::Barrier);
+          Dep.setLatency(TrueMemOrderLatency);
+          I->second[i]->addPred(Dep);
+        }
       }
       // Add SU to the barrier chain.
       if (BarrierChain)
-        BarrierChain->addPred(SDep(SU, SDep::Order, /*Latency=*/0));
+        BarrierChain->addPred(SDep(SU, SDep::Barrier));
       BarrierChain = SU;
       // This is a barrier event that acts as a pivotal node in the DAG,
       // so it is safe to clear list of exposed nodes.
@@ -922,7 +862,7 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA,
         // SU and barrier _could_ be reordered, they should not. In addition,
         // we have lost all RejectMemNodes below barrier.
         if (BarrierChain)
-          BarrierChain->addPred(SDep(SU, SDep::Order, /*Latency=*/0));
+          BarrierChain->addPred(SDep(SU, SDep::Barrier));
       } else {
         // Treat all other stores conservatively.
         goto new_alias_chain;
@@ -931,10 +871,7 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA,
       if (!ExitSU.isPred(SU))
         // Push store's up a bit to avoid them getting in between cmp
         // and branches.
-        ExitSU.addPred(SDep(SU, SDep::Order, 0,
-                            /*Reg=*/0, /*isNormalMemory=*/false,
-                            /*isMustAlias=*/false,
-                            /*isArtificial=*/true));
+        ExitSU.addPred(SDep(SU, SDep::Artificial));
     } else if (MI->mayLoad()) {
       bool MayAlias = true;
       if (MI->isInvariantLoad(AA)) {
@@ -969,7 +906,7 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA,
         if (MayAlias && AliasChain)
           addChainDependency(AA, MFI, SU, AliasChain, RejectMemNodes);
         if (BarrierChain)
-          BarrierChain->addPred(SDep(SU, SDep::Order, /*Latency=*/0));
+          BarrierChain->addPred(SDep(SU, SDep::Barrier));
       }
     }
   }
@@ -982,34 +919,10 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA,
   PendingLoads.clear();
 }
 
-void ScheduleDAGInstrs::computeLatency(SUnit *SU) {
-  // Compute the latency for the node. We only provide a default for missing
-  // itineraries. Empty itineraries still have latency properties.
-  if (!InstrItins) {
-    SU->Latency = 1;
-
-    // Simplistic target-independent heuristic: assume that loads take
-    // extra time.
-    if (SU->getInstr()->mayLoad())
-      SU->Latency += 2;
-  } else {
-    SU->Latency = TII->getInstrLatency(InstrItins, SU->getInstr());
-  }
-}
-
-unsigned ScheduleDAGInstrs::computeOperandLatency(SUnit *Def, SUnit *Use,
-                                                  const SDep& dep,
-                                                  bool FindMin) const {
-  // For a data dependency with a known register...
-  if ((dep.getKind() != SDep::Data) || (dep.getReg() == 0))
-    return 1;
-
-  return TII->computeOperandLatency(InstrItins, TRI, Def->getInstr(),
-                                    Use->getInstr(), dep.getReg(), FindMin);
-}
-
 void ScheduleDAGInstrs::dumpNode(const SUnit *SU) const {
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   SU->getInstr()->dump();
+#endif
 }
 
 std::string ScheduleDAGInstrs::getGraphNodeLabel(const SUnit *SU) const {
@@ -1029,3 +942,94 @@ std::string ScheduleDAGInstrs::getGraphNodeLabel(const SUnit *SU) const {
 std::string ScheduleDAGInstrs::getDAGName() const {
   return "dag." + BB->getFullName();
 }
+
+namespace {
+/// \brief Manage the stack used by a reverse depth-first search over the DAG.
+class SchedDAGReverseDFS {
+  std::vector<std::pair<const SUnit*, SUnit::const_pred_iterator> > DFSStack;
+public:
+  bool isComplete() const { return DFSStack.empty(); }
+
+  void follow(const SUnit *SU) {
+    DFSStack.push_back(std::make_pair(SU, SU->Preds.begin()));
+  }
+  void advance() { ++DFSStack.back().second; }
+
+  void backtrack() { DFSStack.pop_back(); }
+
+  const SUnit *getCurr() const { return DFSStack.back().first; }
+
+  SUnit::const_pred_iterator getPred() const { return DFSStack.back().second; }
+
+  SUnit::const_pred_iterator getPredEnd() const {
+    return getCurr()->Preds.end();
+  }
+};
+} // anonymous
+
+void ScheduleDAGILP::resize(unsigned NumSUnits) {
+  ILPValues.resize(NumSUnits);
+}
+
+ILPValue ScheduleDAGILP::getILP(const SUnit *SU) {
+  return ILPValues[SU->NodeNum];
+}
+
+// A leaf node has an ILP of 1/1.
+static ILPValue initILP(const SUnit *SU) {
+  unsigned Cnt = SU->getInstr()->isTransient() ? 0 : 1;
+  return ILPValue(Cnt, 1 + SU->getDepth());
+}
+
+/// Compute an ILP metric for all nodes in the subDAG reachable via depth-first
+/// search from this root.
+void ScheduleDAGILP::computeILP(const SUnit *Root) {
+  if (!IsBottomUp)
+    llvm_unreachable("Top-down ILP metric is unimplemnted");
+
+  SchedDAGReverseDFS DFS;
+  // Mark a node visited by validating it.
+  ILPValues[Root->NodeNum] = initILP(Root);
+  DFS.follow(Root);
+  for (;;) {
+    // Traverse the leftmost path as far as possible.
+    while (DFS.getPred() != DFS.getPredEnd()) {
+      const SUnit *PredSU = DFS.getPred()->getSUnit();
+      DFS.advance();
+      // If the pred is already valid, skip it.
+      if (ILPValues[PredSU->NodeNum].isValid())
+        continue;
+      ILPValues[PredSU->NodeNum] = initILP(PredSU);
+      DFS.follow(PredSU);
+    }
+    // Visit the top of the stack in postorder and backtrack.
+    unsigned PredCount = ILPValues[DFS.getCurr()->NodeNum].InstrCount;
+    DFS.backtrack();
+    if (DFS.isComplete())
+      break;
+    // Add the recently finished predecessor's bottom-up descendent count.
+    ILPValues[DFS.getCurr()->NodeNum].InstrCount += PredCount;
+  }
+}
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+void ILPValue::print(raw_ostream &OS) const {
+  if (!isValid())
+    OS << "BADILP";
+  OS << InstrCount << " / " << Cycles << " = "
+     << format("%g", ((double)InstrCount / Cycles));
+}
+
+void ILPValue::dump() const {
+  dbgs() << *this << '\n';
+}
+
+namespace llvm {
+
+raw_ostream &operator<<(raw_ostream &OS, const ILPValue &Val) {
+  Val.print(OS);
+  return OS;
+}
+
+} // namespace llvm
+#endif // !NDEBUG || LLVM_ENABLE_DUMP
diff --git a/lib/CodeGen/ScheduleDAGPrinter.cpp b/lib/CodeGen/ScheduleDAGPrinter.cpp
index 38feee9..6e781b1 100644
--- a/lib/CodeGen/ScheduleDAGPrinter.cpp
+++ b/lib/CodeGen/ScheduleDAGPrinter.cpp
@@ -12,7 +12,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Constants.h"
-#include "llvm/Function.h"
 #include "llvm/Assembly/Writer.h"
 #include "llvm/CodeGen/ScheduleDAG.h"
 #include "llvm/CodeGen/MachineConstantPool.h"
@@ -35,7 +34,7 @@ namespace llvm {
   DOTGraphTraits (bool isSimple=false) : DefaultDOTGraphTraits(isSimple) {}
 
     static std::string getGraphName(const ScheduleDAG *G) {
-      return G->MF.getFunction()->getName();
+      return G->MF.getName();
     }
 
     static bool renderGraphFromBottomUp() {
diff --git a/lib/CodeGen/ScoreboardHazardRecognizer.cpp b/lib/CodeGen/ScoreboardHazardRecognizer.cpp
index e675366..2cd84d6 100644
--- a/lib/CodeGen/ScoreboardHazardRecognizer.cpp
+++ b/lib/CodeGen/ScoreboardHazardRecognizer.cpp
@@ -89,6 +89,7 @@ void ScoreboardHazardRecognizer::Reset() {
   ReservedScoreboard.reset();
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void ScoreboardHazardRecognizer::Scoreboard::dump() const {
   dbgs() << "Scoreboard:\n";
 
@@ -104,6 +105,7 @@ void ScoreboardHazardRecognizer::Scoreboard::dump() const {
     dbgs() << '\n';
   }
 }
+#endif
 
 bool ScoreboardHazardRecognizer::atIssueLimit() const {
   if (IssueWidth == 0)
diff --git a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 4e29879..37d7731 100644
--- a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -23,7 +23,7 @@
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/Analysis/AliasAnalysis.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetLowering.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
@@ -194,6 +194,7 @@ namespace {
     SDValue visitOR(SDNode *N);
     SDValue visitXOR(SDNode *N);
     SDValue SimplifyVBinOp(SDNode *N);
+    SDValue SimplifyVUnaryOp(SDNode *N);
     SDValue visitSHL(SDNode *N);
     SDValue visitSRA(SDNode *N);
     SDValue visitSRL(SDNode *N);
@@ -269,6 +270,8 @@ namespace {
     SDValue ReduceLoadWidth(SDNode *N);
     SDValue ReduceLoadOpStoreWidth(SDNode *N);
     SDValue TransformFPLoadStorePair(SDNode *N);
+    SDValue reduceBuildVecExtToExtBuildVec(SDNode *N);
+    SDValue reduceBuildVecConvertToConvertBuildVec(SDNode *N);
 
     SDValue GetDemandedBits(SDValue V, const APInt &Mask);
 
@@ -300,6 +303,11 @@ namespace {
     /// looking for a better chain (aliasing node.)
     SDValue FindBetterChain(SDNode *N, SDValue Chain);
 
+    /// Merge consecutive store operations into a wide store.
+    /// This optimization uses wide integers or vectors when possible.
+    /// \return True if some memory operations were changed.
+    bool MergeConsecutiveStores(StoreSDNode *N);
+
   public:
     DAGCombiner(SelectionDAG &D, AliasAnalysis &A, CodeGenOpt::Level OL)
       : DAG(D), TLI(D.getTargetLoweringInfo()), Level(BeforeLegalizeTypes),
@@ -385,10 +393,6 @@ static char isNegatibleForFree(SDValue Op, bool LegalOperations,
                                const TargetLowering &TLI,
                                const TargetOptions *Options,
                                unsigned Depth = 0) {
-  // No compile time optimizations on this type.
-  if (Op.getValueType() == MVT::ppcf128)
-    return 0;
-
   // fneg is removable even if it has multiple uses.
   if (Op.getOpcode() == ISD::FNEG) return 2;
 
@@ -413,7 +417,7 @@ static char isNegatibleForFree(SDValue Op, bool LegalOperations,
         !TLI.isOperationLegalOrCustom(ISD::FSUB,  Op.getValueType()))
       return 0;
 
-    // fold (fsub (fadd A, B)) -> (fsub (fneg A), B)
+    // fold (fneg (fadd A, B)) -> (fsub (fneg A), B)
     if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI,
                                     Options, Depth + 1))
       return V;
@@ -1643,7 +1647,8 @@ SDValue DAGCombiner::visitSUB(SDNode *N) {
     return N0.getOperand(0);
   // fold C2-(A+C1) -> (C2-C1)-A
   if (N1.getOpcode() == ISD::ADD && N0C && N1C1) {
-    SDValue NewC = DAG.getConstant((N0C->getAPIntValue() - N1C1->getAPIntValue()), VT);
+    SDValue NewC = DAG.getConstant(N0C->getAPIntValue() - N1C1->getAPIntValue(),
+                                   VT);
     return DAG.getNode(ISD::SUB, N->getDebugLoc(), VT, NewC,
                        N1.getOperand(0));
   }
@@ -2345,16 +2350,19 @@ SDValue DAGCombiner::SimplifyBinOpWithSameOpcodeHands(SDNode *N) {
   // we don't want to undo this promotion.
   // We also handle SCALAR_TO_VECTOR because xor/or/and operations are cheaper
   // on scalars.
-  if ((N0.getOpcode() == ISD::BITCAST || N0.getOpcode() == ISD::SCALAR_TO_VECTOR)
-      && Level == AfterLegalizeTypes) {
+  if ((N0.getOpcode() == ISD::BITCAST ||
+       N0.getOpcode() == ISD::SCALAR_TO_VECTOR) &&
+      Level == AfterLegalizeTypes) {
     SDValue In0 = N0.getOperand(0);
     SDValue In1 = N1.getOperand(0);
     EVT In0Ty = In0.getValueType();
     EVT In1Ty = In1.getValueType();
-    // If both incoming values are integers, and the original types are the same.
+    DebugLoc DL = N->getDebugLoc();
+    // If both incoming values are integers, and the original types are the
+    // same.
     if (In0Ty.isInteger() && In1Ty.isInteger() && In0Ty == In1Ty) {
-      SDValue Op = DAG.getNode(N->getOpcode(), N->getDebugLoc(), In0Ty, In0, In1);
-      SDValue BC = DAG.getNode(N0.getOpcode(), N->getDebugLoc(), VT, Op);
+      SDValue Op = DAG.getNode(N->getOpcode(), DL, In0Ty, In0, In1);
+      SDValue BC = DAG.getNode(N0.getOpcode(), DL, VT, Op);
       AddToWorkList(Op.getNode());
       return BC;
     }
@@ -2496,8 +2504,18 @@ SDValue DAGCombiner::visitAND(SDNode *N) {
         // lanes of the constant together.
         EVT VT = Vector->getValueType(0);
         unsigned BitWidth = VT.getVectorElementType().getSizeInBits();
+
+        // If the splat value has been compressed to a bitlength lower
+        // than the size of the vector lane, we need to re-expand it to
+        // the lane size.
+        if (BitWidth > SplatBitSize)
+          for (SplatValue = SplatValue.zextOrTrunc(BitWidth);
+               SplatBitSize < BitWidth;
+               SplatBitSize = SplatBitSize * 2)
+            SplatValue |= SplatValue.shl(SplatBitSize);
+
         Constant = APInt::getAllOnesValue(BitWidth);
-        for (unsigned i = 0, n = VT.getVectorNumElements(); i < n; ++i)
+        for (unsigned i = 0, n = SplatBitSize/BitWidth; i < n; ++i)
           Constant &= SplatValue.lshr(i*BitWidth).zextOrTrunc(BitWidth);
       }
     }
@@ -2984,7 +3002,7 @@ SDValue DAGCombiner::MatchBSwapHWord(SDNode *N, SDValue N0, SDValue N1) {
   SDValue ShAmt = DAG.getConstant(16, getShiftAmountTy(VT));
   if (TLI.isOperationLegalOrCustom(ISD::ROTL, VT))
     return DAG.getNode(ISD::ROTL, N->getDebugLoc(), VT, BSwap, ShAmt);
-  else if (TLI.isOperationLegalOrCustom(ISD::ROTR, VT))
+  if (TLI.isOperationLegalOrCustom(ISD::ROTR, VT))
     return DAG.getNode(ISD::ROTR, N->getDebugLoc(), VT, BSwap, ShAmt);
   return DAG.getNode(ISD::OR, N->getDebugLoc(), VT,
                      DAG.getNode(ISD::SHL, N->getDebugLoc(), VT, BSwap, ShAmt),
@@ -3202,11 +3220,8 @@ SDNode *DAGCombiner::MatchRotate(SDValue LHS, SDValue RHS, DebugLoc DL) {
     if ((LShVal + RShVal) != OpSizeInBits)
       return 0;
 
-    SDValue Rot;
-    if (HasROTL)
-      Rot = DAG.getNode(ISD::ROTL, DL, VT, LHSShiftArg, LHSShiftAmt);
-    else
-      Rot = DAG.getNode(ISD::ROTR, DL, VT, LHSShiftArg, RHSShiftAmt);
+    SDValue Rot = DAG.getNode(HasROTL ? ISD::ROTL : ISD::ROTR, DL, VT,
+                              LHSShiftArg, HasROTL ? LHSShiftAmt : RHSShiftAmt);
 
     // If there is an AND of either shifted operand, apply it to the result.
     if (LHSMask.getNode() || RHSMask.getNode()) {
@@ -3239,12 +3254,8 @@ SDNode *DAGCombiner::MatchRotate(SDValue LHS, SDValue RHS, DebugLoc DL) {
     if (ConstantSDNode *SUBC =
           dyn_cast<ConstantSDNode>(RHSShiftAmt.getOperand(0))) {
       if (SUBC->getAPIntValue() == OpSizeInBits) {
-        if (HasROTL)
-          return DAG.getNode(ISD::ROTL, DL, VT,
-                             LHSShiftArg, LHSShiftAmt).getNode();
-        else
-          return DAG.getNode(ISD::ROTR, DL, VT,
-                             LHSShiftArg, RHSShiftAmt).getNode();
+        return DAG.getNode(HasROTL ? ISD::ROTL : ISD::ROTR, DL, VT, LHSShiftArg,
+                           HasROTL ? LHSShiftAmt : RHSShiftAmt).getNode();
       }
     }
   }
@@ -3256,25 +3267,21 @@ SDNode *DAGCombiner::MatchRotate(SDValue LHS, SDValue RHS, DebugLoc DL) {
     if (ConstantSDNode *SUBC =
           dyn_cast<ConstantSDNode>(LHSShiftAmt.getOperand(0))) {
       if (SUBC->getAPIntValue() == OpSizeInBits) {
-        if (HasROTR)
-          return DAG.getNode(ISD::ROTR, DL, VT,
-                             LHSShiftArg, RHSShiftAmt).getNode();
-        else
-          return DAG.getNode(ISD::ROTL, DL, VT,
-                             LHSShiftArg, LHSShiftAmt).getNode();
+        return DAG.getNode(HasROTR ? ISD::ROTR : ISD::ROTL, DL, VT, LHSShiftArg,
+                           HasROTR ? RHSShiftAmt : LHSShiftAmt).getNode();
       }
     }
   }
 
   // Look for sign/zext/any-extended or truncate cases:
-  if ((LHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND
-       || LHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND
-       || LHSShiftAmt.getOpcode() == ISD::ANY_EXTEND
-       || LHSShiftAmt.getOpcode() == ISD::TRUNCATE) &&
-      (RHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND
-       || RHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND
-       || RHSShiftAmt.getOpcode() == ISD::ANY_EXTEND
-       || RHSShiftAmt.getOpcode() == ISD::TRUNCATE)) {
+  if ((LHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND ||
+       LHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND ||
+       LHSShiftAmt.getOpcode() == ISD::ANY_EXTEND ||
+       LHSShiftAmt.getOpcode() == ISD::TRUNCATE) &&
+      (RHSShiftAmt.getOpcode() == ISD::SIGN_EXTEND ||
+       RHSShiftAmt.getOpcode() == ISD::ZERO_EXTEND ||
+       RHSShiftAmt.getOpcode() == ISD::ANY_EXTEND ||
+       RHSShiftAmt.getOpcode() == ISD::TRUNCATE)) {
     SDValue LExtOp0 = LHSShiftAmt.getOperand(0);
     SDValue RExtOp0 = RHSShiftAmt.getOperand(0);
     if (RExtOp0.getOpcode() == ISD::SUB &&
@@ -4046,7 +4053,8 @@ SDValue DAGCombiner::visitSELECT(SDNode *N) {
   if (VT.isInteger() &&
       (VT0 == MVT::i1 ||
        (VT0.isInteger() &&
-        TLI.getBooleanContents(false) == TargetLowering::ZeroOrOneBooleanContent)) &&
+        TLI.getBooleanContents(false) ==
+        TargetLowering::ZeroOrOneBooleanContent)) &&
       N1C && N2C && N1C->isNullValue() && N2C->getAPIntValue() == 1) {
     SDValue XORNode;
     if (VT == VT0)
@@ -4412,20 +4420,18 @@ SDValue DAGCombiner::visitSIGN_EXTEND(SDNode *N) {
       // If the desired elements are smaller or larger than the source
       // elements we can use a matching integer vector type and then
       // truncate/sign extend
-      else {
-        EVT MatchingElementType =
-          EVT::getIntegerVT(*DAG.getContext(),
-                            N0VT.getScalarType().getSizeInBits());
-        EVT MatchingVectorType =
-          EVT::getVectorVT(*DAG.getContext(), MatchingElementType,
-                           N0VT.getVectorNumElements());
+      EVT MatchingElementType =
+        EVT::getIntegerVT(*DAG.getContext(),
+                          N0VT.getScalarType().getSizeInBits());
+      EVT MatchingVectorType =
+        EVT::getVectorVT(*DAG.getContext(), MatchingElementType,
+                         N0VT.getVectorNumElements());
 
-        if (SVT == MatchingVectorType) {
-          SDValue VsetCC = DAG.getSetCC(N->getDebugLoc(), MatchingVectorType,
-                                 N0.getOperand(0), N0.getOperand(1),
-                                 cast<CondCodeSDNode>(N0.getOperand(2))->get());
-          return DAG.getSExtOrTrunc(VsetCC, N->getDebugLoc(), VT);
-        }
+      if (SVT == MatchingVectorType) {
+        SDValue VsetCC = DAG.getSetCC(N->getDebugLoc(), MatchingVectorType,
+                               N0.getOperand(0), N0.getOperand(1),
+                               cast<CondCodeSDNode>(N0.getOperand(2))->get());
+        return DAG.getSExtOrTrunc(VsetCC, N->getDebugLoc(), VT);
       }
     }
 
@@ -5235,13 +5241,12 @@ SDValue DAGCombiner::visitTRUNCATE(SDNode *N) {
       // if the source is smaller than the dest, we still need an extend
       return DAG.getNode(N0.getOpcode(), N->getDebugLoc(), VT,
                          N0.getOperand(0));
-    else if (N0.getOperand(0).getValueType().bitsGT(VT))
+    if (N0.getOperand(0).getValueType().bitsGT(VT))
       // if the source is larger than the dest, than we just need the truncate
       return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), VT, N0.getOperand(0));
-    else
-      // if the source and dest are the same type, we can drop both the extend
-      // and the truncate.
-      return N0.getOperand(0);
+    // if the source and dest are the same type, we can drop both the extend
+    // and the truncate.
+    return N0.getOperand(0);
   }
 
   // Fold extract-and-trunc into a narrow extract. For example:
@@ -5301,6 +5306,48 @@ SDValue DAGCombiner::visitTRUNCATE(SDNode *N) {
     if (Reduced.getNode())
       return Reduced;
   }
+  // fold (trunc (concat ... x ...)) -> (concat ..., (trunc x), ...)),
+  // where ... are all 'undef'.
+  if (N0.getOpcode() == ISD::CONCAT_VECTORS && !LegalTypes) {
+    SmallVector<EVT, 8> VTs;
+    SDValue V;
+    unsigned Idx = 0;
+    unsigned NumDefs = 0;
+
+    for (unsigned i = 0, e = N0.getNumOperands(); i != e; ++i) {
+      SDValue X = N0.getOperand(i);
+      if (X.getOpcode() != ISD::UNDEF) {
+        V = X;
+        Idx = i;
+        NumDefs++;
+      }
+      // Stop if more than one members are non-undef.
+      if (NumDefs > 1)
+        break;
+      VTs.push_back(EVT::getVectorVT(*DAG.getContext(),
+                                     VT.getVectorElementType(),
+                                     X.getValueType().getVectorNumElements()));
+    }
+
+    if (NumDefs == 0)
+      return DAG.getUNDEF(VT);
+
+    if (NumDefs == 1) {
+      assert(V.getNode() && "The single defined operand is empty!");
+      SmallVector<SDValue, 8> Opnds;
+      for (unsigned i = 0, e = VTs.size(); i != e; ++i) {
+        if (i != Idx) {
+          Opnds.push_back(DAG.getUNDEF(VTs[i]));
+          continue;
+        }
+        SDValue NV = DAG.getNode(ISD::TRUNCATE, V.getDebugLoc(), VTs[i], V);
+        AddToWorkList(NV.getNode());
+        Opnds.push_back(NV);
+      }
+      return DAG.getNode(ISD::CONCAT_VECTORS, N->getDebugLoc(), VT,
+                         &Opnds[0], Opnds.size());
+    }
+  }
 
   // Simplify the operands using demanded-bits information.
   if (!VT.isVector() &&
@@ -5338,7 +5385,7 @@ SDValue DAGCombiner::CombineConsecutiveLoads(SDNode *N, EVT VT) {
       !LD2->isVolatile() &&
       DAG.isConsecutiveLoad(LD2, LD1, LD1VT.getSizeInBits()/8, 1)) {
     unsigned Align = LD1->getAlignment();
-    unsigned NewAlign = TLI.getTargetData()->
+    unsigned NewAlign = TLI.getDataLayout()->
       getABITypeAlignment(VT.getTypeForEVT(*DAG.getContext()));
 
     if (NewAlign <= Align &&
@@ -5407,7 +5454,7 @@ SDValue DAGCombiner::visitBITCAST(SDNode *N) {
       !cast<LoadSDNode>(N0)->isVolatile() &&
       (!LegalOperations || TLI.isOperationLegal(ISD::LOAD, VT))) {
     LoadSDNode *LN0 = cast<LoadSDNode>(N0);
-    unsigned Align = TLI.getTargetData()->
+    unsigned Align = TLI.getDataLayout()->
       getABITypeAlignment(VT.getTypeForEVT(*DAG.getContext()));
     unsigned OrigAlign = LN0->getAlignment();
 
@@ -5430,7 +5477,8 @@ SDValue DAGCombiner::visitBITCAST(SDNode *N) {
   // This often reduces constant pool loads.
   if (((N0.getOpcode() == ISD::FNEG && !TLI.isFNegFree(VT)) ||
        (N0.getOpcode() == ISD::FABS && !TLI.isFAbsFree(VT))) &&
-      N0.getNode()->hasOneUse() && VT.isInteger() && !VT.isVector()) {
+      N0.getNode()->hasOneUse() && VT.isInteger() &&
+      !VT.isVector() && !N0.getValueType().isVector()) {
     SDValue NewConv = DAG.getNode(ISD::BITCAST, N0.getDebugLoc(), VT,
                                   N0.getOperand(0));
     AddToWorkList(NewConv.getNode());
@@ -5653,7 +5701,7 @@ SDValue DAGCombiner::visitFADD(SDNode *N) {
   }
 
   // fold (fadd c1, c2) -> c1 + c2
-  if (N0CFP && N1CFP && VT != MVT::ppcf128)
+  if (N0CFP && N1CFP)
     return DAG.getNode(ISD::FADD, N->getDebugLoc(), VT, N0, N1);
   // canonicalize constant to RHS
   if (N0CFP && !N1CFP)
@@ -5664,12 +5712,12 @@ SDValue DAGCombiner::visitFADD(SDNode *N) {
     return N0;
   // fold (fadd A, (fneg B)) -> (fsub A, B)
   if ((!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FSUB, VT)) &&
-      isNegatibleForFree(N1, LegalOperations, TLI, &DAG.getTarget().Options) == 2)
+    isNegatibleForFree(N1, LegalOperations, TLI, &DAG.getTarget().Options) == 2)
     return DAG.getNode(ISD::FSUB, N->getDebugLoc(), VT, N0,
                        GetNegatedExpression(N1, DAG, LegalOperations));
   // fold (fadd (fneg A), B) -> (fsub B, A)
   if ((!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FSUB, VT)) &&
-      isNegatibleForFree(N0, LegalOperations, TLI, &DAG.getTarget().Options) == 2)
+    isNegatibleForFree(N0, LegalOperations, TLI, &DAG.getTarget().Options) == 2)
     return DAG.getNode(ISD::FSUB, N->getDebugLoc(), VT, N1,
                        GetNegatedExpression(N0, DAG, LegalOperations));
 
@@ -5681,6 +5729,139 @@ SDValue DAGCombiner::visitFADD(SDNode *N) {
                        DAG.getNode(ISD::FADD, N->getDebugLoc(), VT,
                                    N0.getOperand(1), N1));
 
+  // If allow, fold (fadd (fneg x), x) -> 0.0
+  if (DAG.getTarget().Options.UnsafeFPMath &&
+      N0.getOpcode() == ISD::FNEG && N0.getOperand(0) == N1) {
+    return DAG.getConstantFP(0.0, VT);
+  }
+
+    // If allow, fold (fadd x, (fneg x)) -> 0.0
+  if (DAG.getTarget().Options.UnsafeFPMath &&
+      N1.getOpcode() == ISD::FNEG && N1.getOperand(0) == N0) {
+    return DAG.getConstantFP(0.0, VT);
+  }
+
+  // In unsafe math mode, we can fold chains of FADD's of the same value
+  // into multiplications.  This transform is not safe in general because
+  // we are reducing the number of rounding steps.
+  if (DAG.getTarget().Options.UnsafeFPMath &&
+      TLI.isOperationLegalOrCustom(ISD::FMUL, VT) &&
+      !N0CFP && !N1CFP) {
+    if (N0.getOpcode() == ISD::FMUL) {
+      ConstantFPSDNode *CFP00 = dyn_cast<ConstantFPSDNode>(N0.getOperand(0));
+      ConstantFPSDNode *CFP01 = dyn_cast<ConstantFPSDNode>(N0.getOperand(1));
+
+      // (fadd (fmul c, x), x) -> (fmul c+1, x)
+      if (CFP00 && !CFP01 && N0.getOperand(1) == N1) {
+        SDValue NewCFP = DAG.getNode(ISD::FADD, N->getDebugLoc(), VT,
+                                     SDValue(CFP00, 0),
+                                     DAG.getConstantFP(1.0, VT));
+        return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT,
+                           N1, NewCFP);
+      }
+
+      // (fadd (fmul x, c), x) -> (fmul c+1, x)
+      if (CFP01 && !CFP00 && N0.getOperand(0) == N1) {
+        SDValue NewCFP = DAG.getNode(ISD::FADD, N->getDebugLoc(), VT,
+                                     SDValue(CFP01, 0),
+                                     DAG.getConstantFP(1.0, VT));
+        return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT,
+                           N1, NewCFP);
+      }
+
+      // (fadd (fadd x, x), x) -> (fmul 3.0, x)
+      if (!CFP00 && !CFP01 && N0.getOperand(0) == N0.getOperand(1) &&
+          N0.getOperand(0) == N1) {
+        return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT,
+                           N1, DAG.getConstantFP(3.0, VT));
+      }
+
+      // (fadd (fmul c, x), (fadd x, x)) -> (fmul c+2, x)
+      if (CFP00 && !CFP01 && N1.getOpcode() == ISD::FADD &&
+          N1.getOperand(0) == N1.getOperand(1) &&
+          N0.getOperand(1) == N1.getOperand(0)) {
+        SDValue NewCFP = DAG.getNode(ISD::FADD, N->getDebugLoc(), VT,
+                                     SDValue(CFP00, 0),
+                                     DAG.getConstantFP(2.0, VT));
+        return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT,
+                           N0.getOperand(1), NewCFP);
+      }
+
+      // (fadd (fmul x, c), (fadd x, x)) -> (fmul c+2, x)
+      if (CFP01 && !CFP00 && N1.getOpcode() == ISD::FADD &&
+          N1.getOperand(0) == N1.getOperand(1) &&
+          N0.getOperand(0) == N1.getOperand(0)) {
+        SDValue NewCFP = DAG.getNode(ISD::FADD, N->getDebugLoc(), VT,
+                                     SDValue(CFP01, 0),
+                                     DAG.getConstantFP(2.0, VT));
+        return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT,
+                           N0.getOperand(0), NewCFP);
+      }
+    }
+
+    if (N1.getOpcode() == ISD::FMUL) {
+      ConstantFPSDNode *CFP10 = dyn_cast<ConstantFPSDNode>(N1.getOperand(0));
+      ConstantFPSDNode *CFP11 = dyn_cast<ConstantFPSDNode>(N1.getOperand(1));
+
+      // (fadd x, (fmul c, x)) -> (fmul c+1, x)
+      if (CFP10 && !CFP11 && N1.getOperand(1) == N0) {
+        SDValue NewCFP = DAG.getNode(ISD::FADD, N->getDebugLoc(), VT,
+                                     SDValue(CFP10, 0),
+                                     DAG.getConstantFP(1.0, VT));
+        return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT,
+                           N0, NewCFP);
+      }
+
+      // (fadd x, (fmul x, c)) -> (fmul c+1, x)
+      if (CFP11 && !CFP10 && N1.getOperand(0) == N0) {
+        SDValue NewCFP = DAG.getNode(ISD::FADD, N->getDebugLoc(), VT,
+                                     SDValue(CFP11, 0),
+                                     DAG.getConstantFP(1.0, VT));
+        return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT,
+                           N0, NewCFP);
+      }
+
+      // (fadd x, (fadd x, x)) -> (fmul 3.0, x)
+      if (!CFP10 && !CFP11 && N1.getOperand(0) == N1.getOperand(1) &&
+          N1.getOperand(0) == N0) {
+        return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT,
+                           N0, DAG.getConstantFP(3.0, VT));
+      }
+
+      // (fadd (fadd x, x), (fmul c, x)) -> (fmul c+2, x)
+      if (CFP10 && !CFP11 && N1.getOpcode() == ISD::FADD &&
+          N1.getOperand(0) == N1.getOperand(1) &&
+          N0.getOperand(1) == N1.getOperand(0)) {
+        SDValue NewCFP = DAG.getNode(ISD::FADD, N->getDebugLoc(), VT,
+                                     SDValue(CFP10, 0),
+                                     DAG.getConstantFP(2.0, VT));
+        return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT,
+                           N0.getOperand(1), NewCFP);
+      }
+
+      // (fadd (fadd x, x), (fmul x, c)) -> (fmul c+2, x)
+      if (CFP11 && !CFP10 && N1.getOpcode() == ISD::FADD &&
+          N1.getOperand(0) == N1.getOperand(1) &&
+          N0.getOperand(0) == N1.getOperand(0)) {
+        SDValue NewCFP = DAG.getNode(ISD::FADD, N->getDebugLoc(), VT,
+                                     SDValue(CFP11, 0),
+                                     DAG.getConstantFP(2.0, VT));
+        return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT,
+                           N0.getOperand(0), NewCFP);
+      }
+    }
+
+    // (fadd (fadd x, x), (fadd x, x)) -> (fmul 4.0, x)
+    if (N0.getOpcode() == ISD::FADD && N1.getOpcode() == ISD::FADD &&
+        N0.getOperand(0) == N0.getOperand(1) &&
+        N1.getOperand(0) == N1.getOperand(1) &&
+        N0.getOperand(0) == N1.getOperand(0)) {
+      return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT,
+                         N0.getOperand(0),
+                         DAG.getConstantFP(4.0, VT));
+    }
+  }
+
   // FADD -> FMA combines:
   if ((DAG.getTarget().Options.AllowFPOpFusion == FPOpFusion::Fast ||
        DAG.getTarget().Options.UnsafeFPMath) &&
@@ -5692,8 +5873,8 @@ SDValue DAGCombiner::visitFADD(SDNode *N) {
       return DAG.getNode(ISD::FMA, N->getDebugLoc(), VT,
                          N0.getOperand(0), N0.getOperand(1), N1);
     }
-  
-    // fold (fadd x, (fmul y, z)) -> (fma x, y, z)
+
+    // fold (fadd x, (fmul y, z)) -> (fma y, z, x)
     // Note: Commutes FADD operands.
     if (N1.getOpcode() == ISD::FMUL && N1->hasOneUse()) {
       return DAG.getNode(ISD::FMA, N->getDebugLoc(), VT,
@@ -5719,7 +5900,7 @@ SDValue DAGCombiner::visitFSUB(SDNode *N) {
   }
 
   // fold (fsub c1, c2) -> c1-c2
-  if (N0CFP && N1CFP && VT != MVT::ppcf128)
+  if (N0CFP && N1CFP)
     return DAG.getNode(ISD::FSUB, N->getDebugLoc(), VT, N0, N1);
   // fold (fsub A, 0) -> A
   if (DAG.getTarget().Options.UnsafeFPMath &&
@@ -5811,7 +5992,7 @@ SDValue DAGCombiner::visitFMUL(SDNode *N) {
   }
 
   // fold (fmul c1, c2) -> c1*c2
-  if (N0CFP && N1CFP && VT != MVT::ppcf128)
+  if (N0CFP && N1CFP)
     return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT, N0, N1);
   // canonicalize constant to RHS
   if (N0CFP && !N1CFP)
@@ -5867,7 +6048,14 @@ SDValue DAGCombiner::visitFMA(SDNode *N) {
   ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
   ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
   EVT VT = N->getValueType(0);
+  DebugLoc dl = N->getDebugLoc();
 
+  if (DAG.getTarget().Options.UnsafeFPMath) {
+    if (N0CFP && N0CFP->isZero())
+      return N2;
+    if (N1CFP && N1CFP->isZero())
+      return N2;
+  }
   if (N0CFP && N0CFP->isExactlyValue(1.0))
     return DAG.getNode(ISD::FADD, N->getDebugLoc(), VT, N1, N2);
   if (N1CFP && N1CFP->isExactlyValue(1.0))
@@ -5877,6 +6065,58 @@ SDValue DAGCombiner::visitFMA(SDNode *N) {
   if (N0CFP && !N1CFP)
     return DAG.getNode(ISD::FMA, N->getDebugLoc(), VT, N1, N0, N2);
 
+  // (fma x, c1, (fmul x, c2)) -> (fmul x, c1+c2)
+  if (DAG.getTarget().Options.UnsafeFPMath && N1CFP &&
+      N2.getOpcode() == ISD::FMUL &&
+      N0 == N2.getOperand(0) &&
+      N2.getOperand(1).getOpcode() == ISD::ConstantFP) {
+    return DAG.getNode(ISD::FMUL, dl, VT, N0,
+                       DAG.getNode(ISD::FADD, dl, VT, N1, N2.getOperand(1)));
+  }
+
+
+  // (fma (fmul x, c1), c2, y) -> (fma x, c1*c2, y)
+  if (DAG.getTarget().Options.UnsafeFPMath &&
+      N0.getOpcode() == ISD::FMUL && N1CFP &&
+      N0.getOperand(1).getOpcode() == ISD::ConstantFP) {
+    return DAG.getNode(ISD::FMA, dl, VT,
+                       N0.getOperand(0),
+                       DAG.getNode(ISD::FMUL, dl, VT, N1, N0.getOperand(1)),
+                       N2);
+  }
+
+  // (fma x, 1, y) -> (fadd x, y)
+  // (fma x, -1, y) -> (fadd (fneg x), y)
+  if (N1CFP) {
+    if (N1CFP->isExactlyValue(1.0))
+      return DAG.getNode(ISD::FADD, dl, VT, N0, N2);
+
+    if (N1CFP->isExactlyValue(-1.0) &&
+        (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT))) {
+      SDValue RHSNeg = DAG.getNode(ISD::FNEG, dl, VT, N0);
+      AddToWorkList(RHSNeg.getNode());
+      return DAG.getNode(ISD::FADD, dl, VT, N2, RHSNeg);
+    }
+  }
+
+  // (fma x, c, x) -> (fmul x, (c+1))
+  if (DAG.getTarget().Options.UnsafeFPMath && N1CFP && N0 == N2) {
+    return DAG.getNode(ISD::FMUL, dl, VT,
+                       N0,
+                       DAG.getNode(ISD::FADD, dl, VT,
+                                   N1, DAG.getConstantFP(1.0, VT)));
+  }
+
+  // (fma x, c, (fneg x)) -> (fmul x, (c-1))
+  if (DAG.getTarget().Options.UnsafeFPMath && N1CFP &&
+      N2.getOpcode() == ISD::FNEG && N2.getOperand(0) == N0) {
+    return DAG.getNode(ISD::FMUL, dl, VT,
+                       N0,
+                       DAG.getNode(ISD::FADD, dl, VT,
+                                   N1, DAG.getConstantFP(-1.0, VT)));
+  }
+
+
   return SDValue();
 }
 
@@ -5895,11 +6135,11 @@ SDValue DAGCombiner::visitFDIV(SDNode *N) {
   }
 
   // fold (fdiv c1, c2) -> c1/c2
-  if (N0CFP && N1CFP && VT != MVT::ppcf128)
+  if (N0CFP && N1CFP)
     return DAG.getNode(ISD::FDIV, N->getDebugLoc(), VT, N0, N1);
 
   // fold (fdiv X, c2) -> fmul X, 1/c2 if losing precision is acceptable.
-  if (N1CFP && VT != MVT::ppcf128 && DAG.getTarget().Options.UnsafeFPMath) {
+  if (N1CFP && DAG.getTarget().Options.UnsafeFPMath) {
     // Compute the reciprocal 1.0 / c2.
     APFloat N1APF = N1CFP->getValueAPF();
     APFloat Recip(N1APF.getSemantics(), 1); // 1.0
@@ -5942,7 +6182,7 @@ SDValue DAGCombiner::visitFREM(SDNode *N) {
   EVT VT = N->getValueType(0);
 
   // fold (frem c1, c2) -> fmod(c1,c2)
-  if (N0CFP && N1CFP && VT != MVT::ppcf128)
+  if (N0CFP && N1CFP)
     return DAG.getNode(ISD::FREM, N->getDebugLoc(), VT, N0, N1);
 
   return SDValue();
@@ -5955,7 +6195,7 @@ SDValue DAGCombiner::visitFCOPYSIGN(SDNode *N) {
   ConstantFPSDNode *N1CFP = dyn_cast<ConstantFPSDNode>(N1);
   EVT VT = N->getValueType(0);
 
-  if (N0CFP && N1CFP && VT != MVT::ppcf128)  // Constant fold
+  if (N0CFP && N1CFP)  // Constant fold
     return DAG.getNode(ISD::FCOPYSIGN, N->getDebugLoc(), VT, N0, N1);
 
   if (N1CFP) {
@@ -6005,7 +6245,7 @@ SDValue DAGCombiner::visitSINT_TO_FP(SDNode *N) {
   EVT OpVT = N0.getValueType();
 
   // fold (sint_to_fp c1) -> c1fp
-  if (N0C && OpVT != MVT::ppcf128 &&
+  if (N0C &&
       // ...but only if the target supports immediate floating-point values
       (!LegalOperations ||
        TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT)))
@@ -6062,7 +6302,7 @@ SDValue DAGCombiner::visitUINT_TO_FP(SDNode *N) {
   EVT OpVT = N0.getValueType();
 
   // fold (uint_to_fp c1) -> c1fp
-  if (N0C && OpVT != MVT::ppcf128 &&
+  if (N0C &&
       // ...but only if the target supports immediate floating-point values
       (!LegalOperations ||
        TLI.isOperationLegalOrCustom(llvm::ISD::ConstantFP, VT)))
@@ -6117,7 +6357,7 @@ SDValue DAGCombiner::visitFP_TO_UINT(SDNode *N) {
   EVT VT = N->getValueType(0);
 
   // fold (fp_to_uint c1fp) -> c1
-  if (N0CFP && VT != MVT::ppcf128)
+  if (N0CFP)
     return DAG.getNode(ISD::FP_TO_UINT, N->getDebugLoc(), VT, N0);
 
   return SDValue();
@@ -6130,7 +6370,7 @@ SDValue DAGCombiner::visitFP_ROUND(SDNode *N) {
   EVT VT = N->getValueType(0);
 
   // fold (fp_round c1fp) -> c1fp
-  if (N0CFP && N0.getValueType() != MVT::ppcf128)
+  if (N0CFP)
     return DAG.getNode(ISD::FP_ROUND, N->getDebugLoc(), VT, N0, N1);
 
   // fold (fp_round (fp_extend x)) -> x
@@ -6184,7 +6424,7 @@ SDValue DAGCombiner::visitFP_EXTEND(SDNode *N) {
     return SDValue();
 
   // fold (fp_extend c1fp) -> c1fp
-  if (N0CFP && VT != MVT::ppcf128)
+  if (N0CFP)
     return DAG.getNode(ISD::FP_EXTEND, N->getDebugLoc(), VT, N0);
 
   // Turn fp_extend(fp_round(X, 1)) -> x since the fp_round doesn't affect the
@@ -6225,6 +6465,11 @@ SDValue DAGCombiner::visitFNEG(SDNode *N) {
   SDValue N0 = N->getOperand(0);
   EVT VT = N->getValueType(0);
 
+  if (VT.isVector()) {
+    SDValue FoldedVOp = SimplifyVUnaryOp(N);
+    if (FoldedVOp.getNode()) return FoldedVOp;
+  }
+
   if (isNegatibleForFree(N0, LegalOperations, DAG.getTargetLoweringInfo(),
                          &DAG.getTarget().Options))
     return GetNegatedExpression(N0, DAG, LegalOperations);
@@ -6246,6 +6491,17 @@ SDValue DAGCombiner::visitFNEG(SDNode *N) {
     }
   }
 
+  // (fneg (fmul c, x)) -> (fmul -c, x)
+  if (N0.getOpcode() == ISD::FMUL) {
+    ConstantFPSDNode *CFP1 = dyn_cast<ConstantFPSDNode>(N0.getOperand(1));
+    if (CFP1) {
+      return DAG.getNode(ISD::FMUL, N->getDebugLoc(), VT,
+                         N0.getOperand(0),
+                         DAG.getNode(ISD::FNEG, N->getDebugLoc(), VT,
+                                     N0.getOperand(1)));
+    }
+  }
+
   return SDValue();
 }
 
@@ -6255,7 +6511,7 @@ SDValue DAGCombiner::visitFCEIL(SDNode *N) {
   EVT VT = N->getValueType(0);
 
   // fold (fceil c1) -> fceil(c1)
-  if (N0CFP && VT != MVT::ppcf128)
+  if (N0CFP)
     return DAG.getNode(ISD::FCEIL, N->getDebugLoc(), VT, N0);
 
   return SDValue();
@@ -6267,7 +6523,7 @@ SDValue DAGCombiner::visitFTRUNC(SDNode *N) {
   EVT VT = N->getValueType(0);
 
   // fold (ftrunc c1) -> ftrunc(c1)
-  if (N0CFP && VT != MVT::ppcf128)
+  if (N0CFP)
     return DAG.getNode(ISD::FTRUNC, N->getDebugLoc(), VT, N0);
 
   return SDValue();
@@ -6279,7 +6535,7 @@ SDValue DAGCombiner::visitFFLOOR(SDNode *N) {
   EVT VT = N->getValueType(0);
 
   // fold (ffloor c1) -> ffloor(c1)
-  if (N0CFP && VT != MVT::ppcf128)
+  if (N0CFP)
     return DAG.getNode(ISD::FFLOOR, N->getDebugLoc(), VT, N0);
 
   return SDValue();
@@ -6290,8 +6546,13 @@ SDValue DAGCombiner::visitFABS(SDNode *N) {
   ConstantFPSDNode *N0CFP = dyn_cast<ConstantFPSDNode>(N0);
   EVT VT = N->getValueType(0);
 
+  if (VT.isVector()) {
+    SDValue FoldedVOp = SimplifyVUnaryOp(N);
+    if (FoldedVOp.getNode()) return FoldedVOp;
+  }
+
   // fold (fabs c1) -> fabs(c1)
-  if (N0CFP && VT != MVT::ppcf128)
+  if (N0CFP)
     return DAG.getNode(ISD::FABS, N->getDebugLoc(), VT, N0);
   // fold (fabs (fabs x)) -> (fabs x)
   if (N0.getOpcode() == ISD::FABS)
@@ -6511,7 +6772,7 @@ static bool canFoldInAddressingMode(SDNode *N, SDNode *Use,
   } else
     return false;
 
-  TargetLowering::AddrMode AM;
+  AddrMode AM;
   if (N->getOpcode() == ISD::ADD) {
     ConstantSDNode *Offset = dyn_cast<ConstantSDNode>(N->getOperand(1));
     if (Offset)
@@ -7138,7 +7399,7 @@ SDValue DAGCombiner::ReduceLoadOpStoreWidth(SDNode *N) {
 
       unsigned NewAlign = MinAlign(LD->getAlignment(), PtrOff);
       Type *NewVTTy = NewVT.getTypeForEVT(*DAG.getContext());
-      if (NewAlign < TLI.getTargetData()->getABITypeAlignment(NewVTTy))
+      if (NewAlign < TLI.getDataLayout()->getABITypeAlignment(NewVTTy))
         return SDValue();
 
       SDValue NewPtr = DAG.getNode(ISD::ADD, LD->getDebugLoc(),
@@ -7200,7 +7461,7 @@ SDValue DAGCombiner::TransformFPLoadStorePair(SDNode *N) {
     unsigned LDAlign = LD->getAlignment();
     unsigned STAlign = ST->getAlignment();
     Type *IntVTTy = IntVT.getTypeForEVT(*DAG.getContext());
-    unsigned ABIAlign = TLI.getTargetData()->getABITypeAlignment(IntVTTy);
+    unsigned ABIAlign = TLI.getDataLayout()->getABITypeAlignment(IntVTTy);
     if (LDAlign < ABIAlign || STAlign < ABIAlign)
       return SDValue();
 
@@ -7225,6 +7486,433 @@ SDValue DAGCombiner::TransformFPLoadStorePair(SDNode *N) {
   return SDValue();
 }
 
+/// Returns the base pointer and an integer offset from that object.
+static std::pair<SDValue, int64_t> GetPointerBaseAndOffset(SDValue Ptr) {
+  if (Ptr->getOpcode() == ISD::ADD && isa<ConstantSDNode>(Ptr->getOperand(1))) {
+    int64_t Offset = cast<ConstantSDNode>(Ptr->getOperand(1))->getSExtValue();
+    SDValue Base = Ptr->getOperand(0);
+    return std::make_pair(Base, Offset);
+  }
+
+  return std::make_pair(Ptr, 0);
+}
+
+/// Holds a pointer to an LSBaseSDNode as well as information on where it
+/// is located in a sequence of memory operations connected by a chain.
+struct MemOpLink {
+  MemOpLink (LSBaseSDNode *N, int64_t Offset, unsigned Seq):
+    MemNode(N), OffsetFromBase(Offset), SequenceNum(Seq) { }
+  // Ptr to the mem node.
+  LSBaseSDNode *MemNode;
+  // Offset from the base ptr.
+  int64_t OffsetFromBase;
+  // What is the sequence number of this mem node.
+  // Lowest mem operand in the DAG starts at zero.
+  unsigned SequenceNum;
+};
+
+/// Sorts store nodes in a link according to their offset from a shared
+// base ptr.
+struct ConsecutiveMemoryChainSorter {
+  bool operator()(MemOpLink LHS, MemOpLink RHS) {
+    return LHS.OffsetFromBase < RHS.OffsetFromBase;
+  }
+};
+
+bool DAGCombiner::MergeConsecutiveStores(StoreSDNode* St) {
+  EVT MemVT = St->getMemoryVT();
+  int64_t ElementSizeBytes = MemVT.getSizeInBits()/8;
+
+  // Don't merge vectors into wider inputs.
+  if (MemVT.isVector() || !MemVT.isSimple())
+    return false;
+
+  // Perform an early exit check. Do not bother looking at stored values that
+  // are not constants or loads.
+  SDValue StoredVal = St->getValue();
+  bool IsLoadSrc = isa<LoadSDNode>(StoredVal);
+  if (!isa<ConstantSDNode>(StoredVal) && !isa<ConstantFPSDNode>(StoredVal) &&
+      !IsLoadSrc)
+    return false;
+
+  // Only look at ends of store sequences.
+  SDValue Chain = SDValue(St, 1);
+  if (Chain->hasOneUse() && Chain->use_begin()->getOpcode() == ISD::STORE)
+    return false;
+
+  // This holds the base pointer and the offset in bytes from the base pointer.
+  std::pair<SDValue, int64_t> BasePtr =
+      GetPointerBaseAndOffset(St->getBasePtr());
+
+  // We must have a base and an offset.
+  if (!BasePtr.first.getNode())
+    return false;
+
+  // Do not handle stores to undef base pointers.
+  if (BasePtr.first.getOpcode() == ISD::UNDEF)
+    return false;
+
+  SmallVector<MemOpLink, 8> StoreNodes;
+  // Walk up the chain and look for nodes with offsets from the same
+  // base pointer. Stop when reaching an instruction with a different kind
+  // or instruction which has a different base pointer.
+  unsigned Seq = 0;
+  StoreSDNode *Index = St;
+  while (Index) {
+    // If the chain has more than one use, then we can't reorder the mem ops.
+    if (Index != St && !SDValue(Index, 1)->hasOneUse())
+      break;
+
+    // Find the base pointer and offset for this memory node.
+    std::pair<SDValue, int64_t> Ptr =
+      GetPointerBaseAndOffset(Index->getBasePtr());
+
+    // Check that the base pointer is the same as the original one.
+    if (Ptr.first.getNode() != BasePtr.first.getNode())
+      break;
+
+    // Check that the alignment is the same.
+    if (Index->getAlignment() != St->getAlignment())
+      break;
+
+    // The memory operands must not be volatile.
+    if (Index->isVolatile() || Index->isIndexed())
+      break;
+
+    // No truncation.
+    if (StoreSDNode *St = dyn_cast<StoreSDNode>(Index))
+      if (St->isTruncatingStore())
+        break;
+
+    // The stored memory type must be the same.
+    if (Index->getMemoryVT() != MemVT)
+      break;
+
+    // We do not allow unaligned stores because we want to prevent overriding
+    // stores.
+    if (Index->getAlignment()*8 != MemVT.getSizeInBits())
+      break;
+
+    // We found a potential memory operand to merge.
+    StoreNodes.push_back(MemOpLink(Index, Ptr.second, Seq++));
+
+    // Move up the chain to the next memory operation.
+    Index = dyn_cast<StoreSDNode>(Index->getChain().getNode());
+  }
+
+  // Check if there is anything to merge.
+  if (StoreNodes.size() < 2)
+    return false;
+
+  // Sort the memory operands according to their distance from the base pointer.
+  std::sort(StoreNodes.begin(), StoreNodes.end(),
+            ConsecutiveMemoryChainSorter());
+
+  // Scan the memory operations on the chain and find the first non-consecutive
+  // store memory address.
+  unsigned LastConsecutiveStore = 0;
+  int64_t StartAddress = StoreNodes[0].OffsetFromBase;
+  for (unsigned i=1; i<StoreNodes.size(); ++i) {
+    int64_t CurrAddress = StoreNodes[i].OffsetFromBase;
+    if (CurrAddress - StartAddress != (ElementSizeBytes * i))
+      break;
+
+    // Mark this node as useful.
+    LastConsecutiveStore = i;
+  }
+
+  // The node with the lowest store address.
+  LSBaseSDNode *FirstInChain = StoreNodes[0].MemNode;
+
+  // Store the constants into memory as one consecutive store.
+  if (!IsLoadSrc) {
+    unsigned LastLegalType = 0;
+    unsigned LastLegalVectorType = 0;
+    bool NonZero = false;
+    for (unsigned i=0; i<LastConsecutiveStore+1; ++i) {
+      StoreSDNode *St  = cast<StoreSDNode>(StoreNodes[i].MemNode);
+      SDValue StoredVal = St->getValue();
+
+      if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(StoredVal)) {
+        NonZero |= !C->isNullValue();
+      } else if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(StoredVal)) {
+        NonZero |= !C->getConstantFPValue()->isNullValue();
+      } else {
+        // Non constant.
+        break;
+      }
+
+      // Find a legal type for the constant store.
+      unsigned StoreBW = (i+1) * ElementSizeBytes * 8;
+      EVT StoreTy = EVT::getIntegerVT(*DAG.getContext(), StoreBW);
+      if (TLI.isTypeLegal(StoreTy))
+        LastLegalType = i+1;
+
+      // Find a legal type for the vector store.
+      EVT Ty = EVT::getVectorVT(*DAG.getContext(), MemVT, i+1);
+      if (TLI.isTypeLegal(Ty))
+        LastLegalVectorType = i + 1;
+    }
+
+    // We only use vectors if the constant is known to be zero.
+    if (NonZero)
+      LastLegalVectorType = 0;
+
+    // Check if we found a legal integer type to store.
+    if (LastLegalType == 0 && LastLegalVectorType == 0)
+      return false;
+
+    bool UseVector = LastLegalVectorType > LastLegalType;
+    unsigned NumElem = UseVector ? LastLegalVectorType : LastLegalType;
+
+    // Make sure we have something to merge.
+    if (NumElem < 2)
+      return false;
+
+    unsigned EarliestNodeUsed = 0;
+    for (unsigned i=0; i < NumElem; ++i) {
+      // Find a chain for the new wide-store operand. Notice that some
+      // of the store nodes that we found may not be selected for inclusion
+      // in the wide store. The chain we use needs to be the chain of the
+      // earliest store node which is *used* and replaced by the wide store.
+      if (StoreNodes[i].SequenceNum > StoreNodes[EarliestNodeUsed].SequenceNum)
+        EarliestNodeUsed = i;
+    }
+
+    // The earliest Node in the DAG.
+    LSBaseSDNode *EarliestOp = StoreNodes[EarliestNodeUsed].MemNode;
+    DebugLoc DL = StoreNodes[0].MemNode->getDebugLoc();
+
+    SDValue StoredVal;
+    if (UseVector) {
+      // Find a legal type for the vector store.
+      EVT Ty = EVT::getVectorVT(*DAG.getContext(), MemVT, NumElem);
+      assert(TLI.isTypeLegal(Ty) && "Illegal vector store");
+      StoredVal = DAG.getConstant(0, Ty);
+    } else {
+      unsigned StoreBW = NumElem * ElementSizeBytes * 8;
+      APInt StoreInt(StoreBW, 0);
+
+      // Construct a single integer constant which is made of the smaller
+      // constant inputs.
+      bool IsLE = TLI.isLittleEndian();
+      for (unsigned i = 0; i < NumElem ; ++i) {
+        unsigned Idx = IsLE ?(NumElem - 1 - i) : i;
+        StoreSDNode *St  = cast<StoreSDNode>(StoreNodes[Idx].MemNode);
+        SDValue Val = St->getValue();
+        StoreInt<<=ElementSizeBytes*8;
+        if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val)) {
+          StoreInt|=C->getAPIntValue().zext(StoreBW);
+        } else if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val)) {
+          StoreInt|= C->getValueAPF().bitcastToAPInt().zext(StoreBW);
+        } else {
+          assert(false && "Invalid constant element type");
+        }
+      }
+
+      // Create the new Load and Store operations.
+      EVT StoreTy = EVT::getIntegerVT(*DAG.getContext(), StoreBW);
+      StoredVal = DAG.getConstant(StoreInt, StoreTy);
+    }
+
+    SDValue NewStore = DAG.getStore(EarliestOp->getChain(), DL, StoredVal,
+                                    FirstInChain->getBasePtr(),
+                                    FirstInChain->getPointerInfo(),
+                                    false, false,
+                                    FirstInChain->getAlignment());
+
+    // Replace the first store with the new store
+    CombineTo(EarliestOp, NewStore);
+    // Erase all other stores.
+    for (unsigned i = 0; i < NumElem ; ++i) {
+      if (StoreNodes[i].MemNode == EarliestOp)
+        continue;
+      StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode);
+      // ReplaceAllUsesWith will replace all uses that existed when it was
+      // called, but graph optimizations may cause new ones to appear. For
+      // example, the case in pr14333 looks like
+      //
+      //  St's chain -> St -> another store -> X
+      //
+      // And the only difference from St to the other store is the chain.
+      // When we change it's chain to be St's chain they become identical,
+      // get CSEed and the net result is that X is now a use of St.
+      // Since we know that St is redundant, just iterate.
+      while (!St->use_empty())
+        DAG.ReplaceAllUsesWith(SDValue(St, 0), St->getChain());
+      removeFromWorkList(St);
+      DAG.DeleteNode(St);
+    }
+
+    return true;
+  }
+
+  // Below we handle the case of multiple consecutive stores that
+  // come from multiple consecutive loads. We merge them into a single
+  // wide load and a single wide store.
+
+  // Look for load nodes which are used by the stored values.
+  SmallVector<MemOpLink, 8> LoadNodes;
+
+  // Find acceptable loads. Loads need to have the same chain (token factor),
+  // must not be zext, volatile, indexed, and they must be consecutive.
+  SDValue LdBasePtr;
+  for (unsigned i=0; i<LastConsecutiveStore+1; ++i) {
+    StoreSDNode *St  = cast<StoreSDNode>(StoreNodes[i].MemNode);
+    LoadSDNode *Ld = dyn_cast<LoadSDNode>(St->getValue());
+    if (!Ld) break;
+
+    // Loads must only have one use.
+    if (!Ld->hasNUsesOfValue(1, 0))
+      break;
+
+    // Check that the alignment is the same as the stores.
+    if (Ld->getAlignment() != St->getAlignment())
+      break;
+
+    // The memory operands must not be volatile.
+    if (Ld->isVolatile() || Ld->isIndexed())
+      break;
+
+    // We do not accept ext loads.
+    if (Ld->getExtensionType() != ISD::NON_EXTLOAD)
+      break;
+
+    // The stored memory type must be the same.
+    if (Ld->getMemoryVT() != MemVT)
+      break;
+
+    std::pair<SDValue, int64_t> LdPtr =
+    GetPointerBaseAndOffset(Ld->getBasePtr());
+
+    // If this is not the first ptr that we check.
+    if (LdBasePtr.getNode()) {
+      // The base ptr must be the same.
+      if (LdPtr.first != LdBasePtr)
+        break;
+    } else {
+      // Check that all other base pointers are the same as this one.
+      LdBasePtr = LdPtr.first;
+    }
+
+    // We found a potential memory operand to merge.
+    LoadNodes.push_back(MemOpLink(Ld, LdPtr.second, 0));
+  }
+
+  if (LoadNodes.size() < 2)
+    return false;
+
+  // Scan the memory operations on the chain and find the first non-consecutive
+  // load memory address. These variables hold the index in the store node
+  // array.
+  unsigned LastConsecutiveLoad = 0;
+  // This variable refers to the size and not index in the array.
+  unsigned LastLegalVectorType = 0;
+  unsigned LastLegalIntegerType = 0;
+  StartAddress = LoadNodes[0].OffsetFromBase;
+  SDValue FirstChain = LoadNodes[0].MemNode->getChain();
+  for (unsigned i = 1; i < LoadNodes.size(); ++i) {
+    // All loads much share the same chain.
+    if (LoadNodes[i].MemNode->getChain() != FirstChain)
+      break;
+    
+    int64_t CurrAddress = LoadNodes[i].OffsetFromBase;
+    if (CurrAddress - StartAddress != (ElementSizeBytes * i))
+      break;
+    LastConsecutiveLoad = i;
+
+    // Find a legal type for the vector store.
+    EVT StoreTy = EVT::getVectorVT(*DAG.getContext(), MemVT, i+1);
+    if (TLI.isTypeLegal(StoreTy))
+      LastLegalVectorType = i + 1;
+
+    // Find a legal type for the integer store.
+    unsigned StoreBW = (i+1) * ElementSizeBytes * 8;
+    StoreTy = EVT::getIntegerVT(*DAG.getContext(), StoreBW);
+    if (TLI.isTypeLegal(StoreTy))
+      LastLegalIntegerType = i + 1;
+  }
+
+  // Only use vector types if the vector type is larger than the integer type.
+  // If they are the same, use integers.
+  bool UseVectorTy = LastLegalVectorType > LastLegalIntegerType;
+  unsigned LastLegalType = std::max(LastLegalVectorType, LastLegalIntegerType);
+
+  // We add +1 here because the LastXXX variables refer to location while
+  // the NumElem refers to array/index size.
+  unsigned NumElem = std::min(LastConsecutiveStore, LastConsecutiveLoad) + 1;
+  NumElem = std::min(LastLegalType, NumElem);
+
+  if (NumElem < 2)
+    return false;
+
+  // The earliest Node in the DAG.
+  unsigned EarliestNodeUsed = 0;
+  LSBaseSDNode *EarliestOp = StoreNodes[EarliestNodeUsed].MemNode;
+  for (unsigned i=1; i<NumElem; ++i) {
+    // Find a chain for the new wide-store operand. Notice that some
+    // of the store nodes that we found may not be selected for inclusion
+    // in the wide store. The chain we use needs to be the chain of the
+    // earliest store node which is *used* and replaced by the wide store.
+    if (StoreNodes[i].SequenceNum > StoreNodes[EarliestNodeUsed].SequenceNum)
+      EarliestNodeUsed = i;
+  }
+
+  // Find if it is better to use vectors or integers to load and store
+  // to memory.
+  EVT JointMemOpVT;
+  if (UseVectorTy) {
+    JointMemOpVT = EVT::getVectorVT(*DAG.getContext(), MemVT, NumElem);
+  } else {
+    unsigned StoreBW = NumElem * ElementSizeBytes * 8;
+    JointMemOpVT = EVT::getIntegerVT(*DAG.getContext(), StoreBW);
+  }
+
+  DebugLoc LoadDL = LoadNodes[0].MemNode->getDebugLoc();
+  DebugLoc StoreDL = StoreNodes[0].MemNode->getDebugLoc();
+
+  LoadSDNode *FirstLoad = cast<LoadSDNode>(LoadNodes[0].MemNode);
+  SDValue NewLoad = DAG.getLoad(JointMemOpVT, LoadDL,
+                                FirstLoad->getChain(),
+                                FirstLoad->getBasePtr(),
+                                FirstLoad->getPointerInfo(),
+                                false, false, false,
+                                FirstLoad->getAlignment());
+
+  SDValue NewStore = DAG.getStore(EarliestOp->getChain(), StoreDL, NewLoad,
+                                  FirstInChain->getBasePtr(),
+                                  FirstInChain->getPointerInfo(), false, false,
+                                  FirstInChain->getAlignment());
+
+  // Replace one of the loads with the new load.
+  LoadSDNode *Ld = cast<LoadSDNode>(LoadNodes[0].MemNode);
+  DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1),
+                                SDValue(NewLoad.getNode(), 1));
+
+  // Remove the rest of the load chains.
+  for (unsigned i = 1; i < NumElem ; ++i) {
+    // Replace all chain users of the old load nodes with the chain of the new
+    // load node.
+    LoadSDNode *Ld = cast<LoadSDNode>(LoadNodes[i].MemNode);
+    DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1), Ld->getChain());
+  }
+
+  // Replace the first store with the new store.
+  CombineTo(EarliestOp, NewStore);
+  // Erase all other stores.
+  for (unsigned i = 0; i < NumElem ; ++i) {
+    // Remove all Store nodes.
+    if (StoreNodes[i].MemNode == EarliestOp)
+      continue;
+    StoreSDNode *St = cast<StoreSDNode>(StoreNodes[i].MemNode);
+    DAG.ReplaceAllUsesOfValueWith(SDValue(St, 0), St->getChain());
+    removeFromWorkList(St);
+    DAG.DeleteNode(St);
+  }
+
+  return true;
+}
+
 SDValue DAGCombiner::visitSTORE(SDNode *N) {
   StoreSDNode *ST  = cast<StoreSDNode>(N);
   SDValue Chain = ST->getChain();
@@ -7237,7 +7925,7 @@ SDValue DAGCombiner::visitSTORE(SDNode *N) {
       ST->isUnindexed()) {
     unsigned OrigAlign = ST->getAlignment();
     EVT SVT = Value.getOperand(0).getValueType();
-    unsigned Align = TLI.getTargetData()->
+    unsigned Align = TLI.getDataLayout()->
       getABITypeAlignment(SVT.getTypeForEVT(*DAG.getContext()));
     if (Align <= OrigAlign &&
         ((!LegalOperations && !ST->isVolatile()) ||
@@ -7426,6 +8114,11 @@ SDValue DAGCombiner::visitSTORE(SDNode *N) {
                              ST->getAlignment());
   }
 
+  // Only perform this optimization before the types are legal, because we
+  // don't want to perform this optimization on every DAGCombine invocation.
+  if (!LegalTypes && MergeConsecutiveStores(ST))
+    return SDValue(N, 0);
+
   return ReduceLoadOpStoreWidth(N);
 }
 
@@ -7504,9 +8197,9 @@ SDValue DAGCombiner::visitEXTRACT_VECTOR_ELT(SDNode *N) {
 
   // Transform: (EXTRACT_VECTOR_ELT( VECTOR_SHUFFLE )) -> EXTRACT_VECTOR_ELT.
   // We only perform this optimization before the op legalization phase because
-  // we may introduce new vector instructions which are not backed by TD patterns.
-  // For example on AVX, extracting elements from a wide vector without using
-  // extract_subvector.
+  // we may introduce new vector instructions which are not backed by TD
+  // patterns. For example on AVX, extracting elements from a wide vector
+  // without using extract_subvector.
   if (InVec.getOpcode() == ISD::VECTOR_SHUFFLE
       && ConstEltNo && !LegalOperations) {
     int Elt = cast<ConstantSDNode>(EltNo)->getZExtValue();
@@ -7625,7 +8318,7 @@ SDValue DAGCombiner::visitEXTRACT_VECTOR_ELT(SDNode *N) {
       // Check the resultant load doesn't need a higher alignment than the
       // original load.
       unsigned NewAlign =
-        TLI.getTargetData()
+        TLI.getDataLayout()
             ->getABITypeAlignment(LVT.getTypeForEVT(*DAG.getContext()));
 
       if (NewAlign > Align || !TLI.isOperationLegalOrCustom(ISD::LOAD, LVT))
@@ -7690,15 +8383,21 @@ SDValue DAGCombiner::visitEXTRACT_VECTOR_ELT(SDNode *N) {
   return SDValue();
 }
 
-SDValue DAGCombiner::visitBUILD_VECTOR(SDNode *N) {
+// Simplify (build_vec (ext )) to (bitcast (build_vec ))
+SDValue DAGCombiner::reduceBuildVecExtToExtBuildVec(SDNode *N) {
+  // We perform this optimization post type-legalization because
+  // the type-legalizer often scalarizes integer-promoted vectors.
+  // Performing this optimization before may create bit-casts which
+  // will be type-legalized to complex code sequences.
+  // We perform this optimization only before the operation legalizer because we
+  // may introduce illegal operations.
+  if (Level != AfterLegalizeVectorOps && Level != AfterLegalizeTypes)
+    return SDValue();
+
   unsigned NumInScalars = N->getNumOperands();
   DebugLoc dl = N->getDebugLoc();
   EVT VT = N->getValueType(0);
 
-  // A vector built entirely of undefs is undef.
-  if (ISD::allOperandsUndef(N))
-    return DAG.getUNDEF(VT);
-
   // Check to see if this is a BUILD_VECTOR of a bunch of values
   // which come from any_extend or zero_extend nodes. If so, we can create
   // a new BUILD_VECTOR using bit-casts which may enable other BUILD_VECTOR
@@ -7741,64 +8440,141 @@ SDValue DAGCombiner::visitBUILD_VECTOR(SDNode *N) {
   // In order to have valid types, all of the inputs must be extended from the
   // same source type and all of the inputs must be any or zero extend.
   // Scalar sizes must be a power of two.
-  EVT OutScalarTy = N->getValueType(0).getScalarType();
+  EVT OutScalarTy = VT.getScalarType();
   bool ValidTypes = SourceType != MVT::Other &&
                  isPowerOf2_32(OutScalarTy.getSizeInBits()) &&
                  isPowerOf2_32(SourceType.getSizeInBits());
 
-  // We perform this optimization post type-legalization because
-  // the type-legalizer often scalarizes integer-promoted vectors.
-  // Performing this optimization before may create bit-casts which
-  // will be type-legalized to complex code sequences.
-  // We perform this optimization only before the operation legalizer because we
-  // may introduce illegal operations.
   // Create a new simpler BUILD_VECTOR sequence which other optimizations can
   // turn into a single shuffle instruction.
-  if ((Level == AfterLegalizeVectorOps || Level == AfterLegalizeTypes) &&
-      ValidTypes) {
-    bool isLE = TLI.isLittleEndian();
-    unsigned ElemRatio = OutScalarTy.getSizeInBits()/SourceType.getSizeInBits();
-    assert(ElemRatio > 1 && "Invalid element size ratio");
-    SDValue Filler = AllAnyExt ? DAG.getUNDEF(SourceType):
-                                 DAG.getConstant(0, SourceType);
-
-    unsigned NewBVElems = ElemRatio * N->getValueType(0).getVectorNumElements();
-    SmallVector<SDValue, 8> Ops(NewBVElems, Filler);
-
-    // Populate the new build_vector
-    for (unsigned i=0; i < N->getNumOperands(); ++i) {
-      SDValue Cast = N->getOperand(i);
-      assert((Cast.getOpcode() == ISD::ANY_EXTEND ||
-              Cast.getOpcode() == ISD::ZERO_EXTEND ||
-              Cast.getOpcode() == ISD::UNDEF) && "Invalid cast opcode");
-      SDValue In;
-      if (Cast.getOpcode() == ISD::UNDEF)
-        In = DAG.getUNDEF(SourceType);
-      else
-        In = Cast->getOperand(0);
-      unsigned Index = isLE ? (i * ElemRatio) :
-                              (i * ElemRatio + (ElemRatio - 1));
+  if (!ValidTypes)
+    return SDValue();
+
+  bool isLE = TLI.isLittleEndian();
+  unsigned ElemRatio = OutScalarTy.getSizeInBits()/SourceType.getSizeInBits();
+  assert(ElemRatio > 1 && "Invalid element size ratio");
+  SDValue Filler = AllAnyExt ? DAG.getUNDEF(SourceType):
+                               DAG.getConstant(0, SourceType);
+
+  unsigned NewBVElems = ElemRatio * VT.getVectorNumElements();
+  SmallVector<SDValue, 8> Ops(NewBVElems, Filler);
+
+  // Populate the new build_vector
+  for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
+    SDValue Cast = N->getOperand(i);
+    assert((Cast.getOpcode() == ISD::ANY_EXTEND ||
+            Cast.getOpcode() == ISD::ZERO_EXTEND ||
+            Cast.getOpcode() == ISD::UNDEF) && "Invalid cast opcode");
+    SDValue In;
+    if (Cast.getOpcode() == ISD::UNDEF)
+      In = DAG.getUNDEF(SourceType);
+    else
+      In = Cast->getOperand(0);
+    unsigned Index = isLE ? (i * ElemRatio) :
+                            (i * ElemRatio + (ElemRatio - 1));
+
+    assert(Index < Ops.size() && "Invalid index");
+    Ops[Index] = In;
+  }
+
+  // The type of the new BUILD_VECTOR node.
+  EVT VecVT = EVT::getVectorVT(*DAG.getContext(), SourceType, NewBVElems);
+  assert(VecVT.getSizeInBits() == VT.getSizeInBits() &&
+         "Invalid vector size");
+  // Check if the new vector type is legal.
+  if (!isTypeLegal(VecVT)) return SDValue();
+
+  // Make the new BUILD_VECTOR.
+  SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, dl, VecVT, &Ops[0], Ops.size());
+
+  // The new BUILD_VECTOR node has the potential to be further optimized.
+  AddToWorkList(BV.getNode());
+  // Bitcast to the desired type.
+  return DAG.getNode(ISD::BITCAST, dl, VT, BV);
+}
+
+SDValue DAGCombiner::reduceBuildVecConvertToConvertBuildVec(SDNode *N) {
+  EVT VT = N->getValueType(0);
+
+  unsigned NumInScalars = N->getNumOperands();
+  DebugLoc dl = N->getDebugLoc();
+
+  EVT SrcVT = MVT::Other;
+  unsigned Opcode = ISD::DELETED_NODE;
+  unsigned NumDefs = 0;
 
-      assert(Index < Ops.size() && "Invalid index");
-      Ops[Index] = In;
+  for (unsigned i = 0; i != NumInScalars; ++i) {
+    SDValue In = N->getOperand(i);
+    unsigned Opc = In.getOpcode();
+
+    if (Opc == ISD::UNDEF)
+      continue;
+
+    // If all scalar values are floats and converted from integers.
+    if (Opcode == ISD::DELETED_NODE &&
+        (Opc == ISD::UINT_TO_FP || Opc == ISD::SINT_TO_FP)) {
+      Opcode = Opc;
+      // If not supported by target, bail out.
+      if (TLI.getOperationAction(Opcode, VT) != TargetLowering::Legal &&
+          TLI.getOperationAction(Opcode, VT) != TargetLowering::Custom)
+        return SDValue();
     }
+    if (Opc != Opcode)
+      return SDValue();
 
-    // The type of the new BUILD_VECTOR node.
-    EVT VecVT = EVT::getVectorVT(*DAG.getContext(), SourceType, NewBVElems);
-    assert(VecVT.getSizeInBits() == N->getValueType(0).getSizeInBits() &&
-           "Invalid vector size");
-    // Check if the new vector type is legal.
-    if (!isTypeLegal(VecVT)) return SDValue();
+    EVT InVT = In.getOperand(0).getValueType();
 
-    // Make the new BUILD_VECTOR.
-    SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(),
-                                 VecVT, &Ops[0], Ops.size());
+    // If all scalar values are typed differently, bail out. It's chosen to
+    // simplify BUILD_VECTOR of integer types.
+    if (SrcVT == MVT::Other)
+      SrcVT = InVT;
+    if (SrcVT != InVT)
+      return SDValue();
+    NumDefs++;
+  }
+
+  // If the vector has just one element defined, it's not worth to fold it into
+  // a vectorized one.
+  if (NumDefs < 2)
+    return SDValue();
 
-    // The new BUILD_VECTOR node has the potential to be further optimized.
-    AddToWorkList(BV.getNode());
-    // Bitcast to the desired type.
-    return DAG.getNode(ISD::BITCAST, dl, N->getValueType(0), BV);
+  assert((Opcode == ISD::UINT_TO_FP || Opcode == ISD::SINT_TO_FP)
+         && "Should only handle conversion from integer to float.");
+  assert(SrcVT != MVT::Other && "Cannot determine source type!");
+
+  EVT NVT = EVT::getVectorVT(*DAG.getContext(), SrcVT, NumInScalars);
+  SmallVector<SDValue, 8> Opnds;
+  for (unsigned i = 0; i != NumInScalars; ++i) {
+    SDValue In = N->getOperand(i);
+
+    if (In.getOpcode() == ISD::UNDEF)
+      Opnds.push_back(DAG.getUNDEF(SrcVT));
+    else
+      Opnds.push_back(In.getOperand(0));
   }
+  SDValue BV = DAG.getNode(ISD::BUILD_VECTOR, dl, NVT,
+                           &Opnds[0], Opnds.size());
+  AddToWorkList(BV.getNode());
+
+  return DAG.getNode(Opcode, dl, VT, BV);
+}
+
+SDValue DAGCombiner::visitBUILD_VECTOR(SDNode *N) {
+  unsigned NumInScalars = N->getNumOperands();
+  DebugLoc dl = N->getDebugLoc();
+  EVT VT = N->getValueType(0);
+
+  // A vector built entirely of undefs is undef.
+  if (ISD::allOperandsUndef(N))
+    return DAG.getUNDEF(VT);
+
+  SDValue V = reduceBuildVecExtToExtBuildVec(N);
+  if (V.getNode())
+    return V;
+
+  V = reduceBuildVecConvertToConvertBuildVec(N);
+  if (V.getNode())
+    return V;
 
   // Check to see if this is a BUILD_VECTOR of a bunch of EXTRACT_VECTOR_ELT
   // operations.  If so, and if the EXTRACT_VECTOR_ELT vector inputs come from
@@ -7876,15 +8652,22 @@ SDValue DAGCombiner::visitBUILD_VECTOR(SDNode *N) {
       if (VecIn1.getValueType().getSizeInBits()*2 != VT.getSizeInBits())
         return SDValue();
 
+      // If the input vector type has a different base type to the output
+      // vector type, bail out.
+      if (VecIn1.getValueType().getVectorElementType() !=
+          VT.getVectorElementType())
+        return SDValue();
+
       // Widen the input vector by adding undef values.
-      VecIn1 = DAG.getNode(ISD::CONCAT_VECTORS, N->getDebugLoc(), VT,
+      VecIn1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT,
                            VecIn1, DAG.getUNDEF(VecIn1.getValueType()));
     }
 
     // If VecIn2 is unused then change it to undef.
     VecIn2 = VecIn2.getNode() ? VecIn2 : DAG.getUNDEF(VT);
 
-    // Check that we were able to transform all incoming values to the same type.
+    // Check that we were able to transform all incoming values to the same
+    // type.
     if (VecIn2.getValueType() != VecIn1.getValueType() ||
         VecIn1.getValueType() != VT)
           return SDValue();
@@ -7897,7 +8680,7 @@ SDValue DAGCombiner::visitBUILD_VECTOR(SDNode *N) {
     SDValue Ops[2];
     Ops[0] = VecIn1;
     Ops[1] = VecIn2;
-    return DAG.getVectorShuffle(VT, N->getDebugLoc(), Ops[0], Ops[1], &Mask[0]);
+    return DAG.getVectorShuffle(VT, dl, Ops[0], Ops[1], &Mask[0]);
   }
 
   return SDValue();
@@ -7933,8 +8716,8 @@ SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode* N) {
       return SDValue();
 
     // Only handle cases where both indexes are constants with the same type.
-    ConstantSDNode *InsIdx = dyn_cast<ConstantSDNode>(N->getOperand(1));
-    ConstantSDNode *ExtIdx = dyn_cast<ConstantSDNode>(V->getOperand(2));
+    ConstantSDNode *ExtIdx = dyn_cast<ConstantSDNode>(N->getOperand(1));
+    ConstantSDNode *InsIdx = dyn_cast<ConstantSDNode>(V->getOperand(2));
 
     if (InsIdx && ExtIdx &&
         InsIdx->getValueType(0).getSizeInBits() <= 64 &&
@@ -7951,6 +8734,21 @@ SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode* N) {
     }
   }
 
+  if (V->getOpcode() == ISD::CONCAT_VECTORS) {
+    // Combine:
+    //    (extract_subvec (concat V1, V2, ...), i)
+    // Into:
+    //    Vi if possible
+    // Only operand 0 is checked as 'concat' assumes all inputs of the same type.
+    if (V->getOperand(0).getValueType() != NVT)
+      return SDValue();
+    unsigned Idx = dyn_cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
+    unsigned NumElems = NVT.getVectorNumElements();
+    assert((Idx % NumElems) == 0 &&
+           "IDX in concat is not a multiple of the result vector length.");
+    return V->getOperand(Idx / NumElems);
+  }
+
   return SDValue();
 }
 
@@ -8266,6 +9064,44 @@ SDValue DAGCombiner::SimplifyVBinOp(SDNode *N) {
   return SDValue();
 }
 
+/// SimplifyVUnaryOp - Visit a binary vector operation, like FABS/FNEG.
+SDValue DAGCombiner::SimplifyVUnaryOp(SDNode *N) {
+  // After legalize, the target may be depending on adds and other
+  // binary ops to provide legal ways to construct constants or other
+  // things. Simplifying them may result in a loss of legality.
+  if (LegalOperations) return SDValue();
+
+  assert(N->getValueType(0).isVector() &&
+         "SimplifyVUnaryOp only works on vectors!");
+
+  SDValue N0 = N->getOperand(0);
+
+  if (N0.getOpcode() != ISD::BUILD_VECTOR)
+    return SDValue();
+
+  // Operand is a BUILD_VECTOR node, see if we can constant fold it.
+  SmallVector<SDValue, 8> Ops;
+  for (unsigned i = 0, e = N0.getNumOperands(); i != e; ++i) {
+    SDValue Op = N0.getOperand(i);
+    if (Op.getOpcode() != ISD::UNDEF &&
+        Op.getOpcode() != ISD::ConstantFP)
+      break;
+    EVT EltVT = Op.getValueType();
+    SDValue FoldOp = DAG.getNode(N->getOpcode(), N0.getDebugLoc(), EltVT, Op);
+    if (FoldOp.getOpcode() != ISD::UNDEF &&
+        FoldOp.getOpcode() != ISD::ConstantFP)
+      break;
+    Ops.push_back(FoldOp);
+    AddToWorkList(FoldOp.getNode());
+  }
+
+  if (Ops.size() != N0.getNumOperands())
+    return SDValue();
+
+  return DAG.getNode(ISD::BUILD_VECTOR, N->getDebugLoc(),
+                     N0.getValueType(), &Ops[0], Ops.size());
+}
+
 SDValue DAGCombiner::SimplifySelect(DebugLoc DL, SDValue N0,
                                     SDValue N1, SDValue N2){
   assert(N0.getOpcode() ==ISD::SETCC && "First argument must be a SetCC node!");
@@ -8349,6 +9185,10 @@ bool DAGCombiner::SimplifySelectOps(SDNode *TheSelect, SDValue LHS,
       if ((LLD->hasAnyUseOfValue(1) && LLD->isPredecessorOf(CondNode)) ||
           (RLD->hasAnyUseOfValue(1) && RLD->isPredecessorOf(CondNode)))
         return false;
+      // The loads must not depend on one another.
+      if (LLD->isPredecessorOf(RLD) ||
+          RLD->isPredecessorOf(LLD))
+        return false;
       Addr = DAG.getNode(ISD::SELECT, TheSelect->getDebugLoc(),
                          LLD->getBasePtr().getValueType(),
                          TheSelect->getOperand(0), LLD->getBasePtr(),
@@ -8468,7 +9308,7 @@ SDValue DAGCombiner::SimplifySelectCC(DebugLoc DL, SDValue N0, SDValue N1,
           const_cast<ConstantFP*>(TV->getConstantFPValue())
         };
         Type *FPTy = Elts[0]->getType();
-        const TargetData &TD = *TLI.getTargetData();
+        const DataLayout &TD = *TLI.getDataLayout();
 
         // Create a ConstantArray of the two constants.
         Constant *CA = ConstantArray::get(ArrayType::get(FPTy, 2), Elts);
@@ -8583,34 +9423,38 @@ SDValue DAGCombiner::SimplifySelectCC(DebugLoc DL, SDValue N0, SDValue N1,
       return SDValue();
 
     // Get a SetCC of the condition
-    // FIXME: Should probably make sure that setcc is legal if we ever have a
-    // target where it isn't.
-    SDValue Temp, SCC;
-    // cast from setcc result type to select result type
-    if (LegalTypes) {
-      SCC  = DAG.getSetCC(DL, TLI.getSetCCResultType(N0.getValueType()),
-                          N0, N1, CC);
-      if (N2.getValueType().bitsLT(SCC.getValueType()))
-        Temp = DAG.getZeroExtendInReg(SCC, N2.getDebugLoc(), N2.getValueType());
-      else
+    // NOTE: Don't create a SETCC if it's not legal on this target.
+    if (!LegalOperations ||
+        TLI.isOperationLegal(ISD::SETCC,
+          LegalTypes ? TLI.getSetCCResultType(N0.getValueType()) : MVT::i1)) {
+      SDValue Temp, SCC;
+      // cast from setcc result type to select result type
+      if (LegalTypes) {
+        SCC  = DAG.getSetCC(DL, TLI.getSetCCResultType(N0.getValueType()),
+                            N0, N1, CC);
+        if (N2.getValueType().bitsLT(SCC.getValueType()))
+          Temp = DAG.getZeroExtendInReg(SCC, N2.getDebugLoc(),
+                                        N2.getValueType());
+        else
+          Temp = DAG.getNode(ISD::ZERO_EXTEND, N2.getDebugLoc(),
+                             N2.getValueType(), SCC);
+      } else {
+        SCC  = DAG.getSetCC(N0.getDebugLoc(), MVT::i1, N0, N1, CC);
         Temp = DAG.getNode(ISD::ZERO_EXTEND, N2.getDebugLoc(),
                            N2.getValueType(), SCC);
-    } else {
-      SCC  = DAG.getSetCC(N0.getDebugLoc(), MVT::i1, N0, N1, CC);
-      Temp = DAG.getNode(ISD::ZERO_EXTEND, N2.getDebugLoc(),
-                         N2.getValueType(), SCC);
-    }
+      }
 
-    AddToWorkList(SCC.getNode());
-    AddToWorkList(Temp.getNode());
+      AddToWorkList(SCC.getNode());
+      AddToWorkList(Temp.getNode());
 
-    if (N2C->getAPIntValue() == 1)
-      return Temp;
+      if (N2C->getAPIntValue() == 1)
+        return Temp;
 
-    // shl setcc result by log2 n2c
-    return DAG.getNode(ISD::SHL, DL, N2.getValueType(), Temp,
-                       DAG.getConstant(N2C->getAPIntValue().logBase2(),
-                                       getShiftAmountTy(Temp.getValueType())));
+      // shl setcc result by log2 n2c
+      return DAG.getNode(ISD::SHL, DL, N2.getValueType(), Temp,
+                         DAG.getConstant(N2C->getAPIntValue().logBase2(),
+                                         getShiftAmountTy(Temp.getValueType())));
+    }
   }
 
   // Check to see if this is the equivalent of setcc
@@ -8729,7 +9573,7 @@ SDValue DAGCombiner::BuildUDIV(SDNode *N) {
 // to alias with anything but itself.  Provides base object and offset as
 // results.
 static bool FindBaseOffset(SDValue Ptr, SDValue &Base, int64_t &Offset,
-                           const GlobalValue *&GV, void *&CV) {
+                           const GlobalValue *&GV, const void *&CV) {
   // Assume it is a primitive operation.
   Base = Ptr; Offset = 0; GV = 0; CV = 0;
 
@@ -8754,8 +9598,8 @@ static bool FindBaseOffset(SDValue Ptr, SDValue &Base, int64_t &Offset,
   // for ConstantSDNodes since the same constant pool entry may be represented
   // by multiple nodes with different offsets.
   if (ConstantPoolSDNode *C = dyn_cast<ConstantPoolSDNode>(Base)) {
-    CV = C->isMachineConstantPoolEntry() ? (void *)C->getMachineCPVal()
-                                         : (void *)C->getConstVal();
+    CV = C->isMachineConstantPoolEntry() ? (const void *)C->getMachineCPVal()
+                                         : (const void *)C->getConstVal();
     Offset += C->getOffset();
     return false;
   }
@@ -8780,7 +9624,7 @@ bool DAGCombiner::isAlias(SDValue Ptr1, int64_t Size1,
   SDValue Base1, Base2;
   int64_t Offset1, Offset2;
   const GlobalValue *GV1, *GV2;
-  void *CV1, *CV2;
+  const void *CV1, *CV2;
   bool isFrameIndex1 = FindBaseOffset(Ptr1, Base1, Offset1, GV1, CV1);
   bool isFrameIndex2 = FindBaseOffset(Ptr2, Base2, Offset2, GV2, CV2);
 
diff --git a/lib/CodeGen/SelectionDAG/FastISel.cpp b/lib/CodeGen/SelectionDAG/FastISel.cpp
index 683fac6..4854cf7 100644
--- a/lib/CodeGen/SelectionDAG/FastISel.cpp
+++ b/lib/CodeGen/SelectionDAG/FastISel.cpp
@@ -53,7 +53,7 @@
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/Analysis/Loads.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetInstrInfo.h"
 #include "llvm/Target/TargetLibraryInfo.h"
 #include "llvm/Target/TargetLowering.h"
@@ -1059,7 +1059,7 @@ FastISel::FastISel(FunctionLoweringInfo &funcInfo,
     MFI(*FuncInfo.MF->getFrameInfo()),
     MCP(*FuncInfo.MF->getConstantPool()),
     TM(FuncInfo.MF->getTarget()),
-    TD(*TM.getTargetData()),
+    TD(*TM.getDataLayout()),
     TII(*TM.getInstrInfo()),
     TLI(*TM.getTargetLowering()),
     TRI(*TM.getRegisterInfo()),
diff --git a/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp b/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp
index 3e18ea7..a418290 100644
--- a/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp
+++ b/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp
@@ -29,7 +29,7 @@
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/Target/TargetRegisterInfo.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetInstrInfo.h"
 #include "llvm/Target/TargetLowering.h"
 #include "llvm/Target/TargetOptions.h"
@@ -80,9 +80,9 @@ void FunctionLoweringInfo::set(const Function &fn, MachineFunction &mf) {
     if (const AllocaInst *AI = dyn_cast<AllocaInst>(I))
       if (const ConstantInt *CUI = dyn_cast<ConstantInt>(AI->getArraySize())) {
         Type *Ty = AI->getAllocatedType();
-        uint64_t TySize = TLI.getTargetData()->getTypeAllocSize(Ty);
+        uint64_t TySize = TLI.getDataLayout()->getTypeAllocSize(Ty);
         unsigned Align =
-          std::max((unsigned)TLI.getTargetData()->getPrefTypeAlignment(Ty),
+          std::max((unsigned)TLI.getDataLayout()->getPrefTypeAlignment(Ty),
                    AI->getAlignment());
 
         TySize *= CUI->getZExtValue();   // Get total allocated size.
@@ -97,7 +97,7 @@ void FunctionLoweringInfo::set(const Function &fn, MachineFunction &mf) {
             cast<ArrayType>(Ty)->getElementType()->isIntegerTy(8)));
         StaticAllocaMap[AI] =
           MF->getFrameInfo()->CreateStackObject(TySize, Align, false,
-                                                MayNeedSP);
+                                                MayNeedSP, AI);
       }
 
   for (; BB != EB; ++BB)
diff --git a/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
index 4488d27..a8381b2 100644
--- a/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
+++ b/lib/CodeGen/SelectionDAG/InstrEmitter.cpp
@@ -20,7 +20,7 @@
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetInstrInfo.h"
 #include "llvm/Target/TargetLowering.h"
@@ -55,7 +55,8 @@ unsigned InstrEmitter::CountResults(SDNode *Node) {
 ///
 /// Also count physreg RegisterSDNode and RegisterMaskSDNode operands preceding
 /// the chain and glue. These operands may be implicit on the machine instr.
-static unsigned countOperands(SDNode *Node, unsigned &NumImpUses) {
+static unsigned countOperands(SDNode *Node, unsigned NumExpUses,
+                              unsigned &NumImpUses) {
   unsigned N = Node->getNumOperands();
   while (N && Node->getOperand(N - 1).getValueType() == MVT::Glue)
     --N;
@@ -63,7 +64,8 @@ static unsigned countOperands(SDNode *Node, unsigned &NumImpUses) {
     --N; // Ignore chain if it exists.
 
   // Count RegisterSDNode and RegisterMaskSDNode operands for NumImpUses.
-  for (unsigned I = N; I; --I) {
+  NumImpUses = N - NumExpUses;
+  for (unsigned I = N; I > NumExpUses; --I) {
     if (isa<RegisterMaskSDNode>(Node->getOperand(I - 1)))
       continue;
     if (RegisterSDNode *RN = dyn_cast<RegisterSDNode>(Node->getOperand(I - 1)))
@@ -312,8 +314,6 @@ InstrEmitter::AddRegisterOperand(MachineInstr *MI, SDValue Op,
     const TargetRegisterClass *DstRC = 0;
     if (IIOpNum < II->getNumOperands())
       DstRC = TRI->getAllocatableClass(TII->getRegClass(*II,IIOpNum,TRI,*MF));
-    assert((DstRC || (MI->isVariadic() && IIOpNum >= MCID.getNumOperands())) &&
-           "Don't have operand info for this instruction!");
     if (DstRC && !MRI->constrainRegClass(VReg, DstRC, MinRCSize)) {
       unsigned NewVReg = MRI->createVirtualRegister(DstRC);
       BuildMI(*MBB, InsertPos, Op.getNode()->getDebugLoc(),
@@ -390,10 +390,10 @@ void InstrEmitter::AddOperand(MachineInstr *MI, SDValue Op,
     Type *Type = CP->getType();
     // MachineConstantPool wants an explicit alignment.
     if (Align == 0) {
-      Align = TM->getTargetData()->getPrefTypeAlignment(Type);
+      Align = TM->getDataLayout()->getPrefTypeAlignment(Type);
       if (Align == 0) {
         // Alignment of vector types.  FIXME!
-        Align = TM->getTargetData()->getTypeAllocSize(Type);
+        Align = TM->getDataLayout()->getTypeAllocSize(Type);
       }
     }
 
@@ -410,6 +410,7 @@ void InstrEmitter::AddOperand(MachineInstr *MI, SDValue Op,
                                             ES->getTargetFlags()));
   } else if (BlockAddressSDNode *BA = dyn_cast<BlockAddressSDNode>(Op)) {
     MI->addOperand(MachineOperand::CreateBA(BA->getBlockAddress(),
+                                            BA->getOffset(),
                                             BA->getTargetFlags()));
   } else if (TargetIndexSDNode *TI = dyn_cast<TargetIndexSDNode>(Op)) {
     MI->addOperand(MachineOperand::CreateTargetIndex(TI->getIndex(),
@@ -720,7 +721,8 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned,
   const MCInstrDesc &II = TII->get(Opc);
   unsigned NumResults = CountResults(Node);
   unsigned NumImpUses = 0;
-  unsigned NodeOperands = countOperands(Node, NumImpUses);
+  unsigned NodeOperands =
+    countOperands(Node, II.getNumOperands() - II.getNumDefs(), NumImpUses);
   bool HasPhysRegOuts = NumResults > II.getNumDefs() && II.getImplicitDefs()!=0;
 #ifndef NDEBUG
   unsigned NumMIOperands = NodeOperands + NumResults;
@@ -870,6 +872,17 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned,
     break;
   }
 
+  case ISD::LIFETIME_START:
+  case ISD::LIFETIME_END: {
+    unsigned TarOp = (Node->getOpcode() == ISD::LIFETIME_START) ?
+    TargetOpcode::LIFETIME_START : TargetOpcode::LIFETIME_END;
+
+    FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Node->getOperand(1));
+    BuildMI(*MBB, InsertPos, Node->getDebugLoc(), TII->get(TarOp))
+    .addFrameIndex(FI->getIndex());
+    break;
+  }
+
   case ISD::INLINEASM: {
     unsigned NumOps = Node->getNumOperands();
     if (Node->getOperand(NumOps-1).getValueType() == MVT::Glue)
@@ -884,25 +897,30 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned,
     const char *AsmStr = cast<ExternalSymbolSDNode>(AsmStrV)->getSymbol();
     MI->addOperand(MachineOperand::CreateES(AsmStr));
 
-    // Add the HasSideEffect and isAlignStack bits.
+    // Add the HasSideEffect, isAlignStack, AsmDialect, MayLoad and MayStore
+    // bits.
     int64_t ExtraInfo =
       cast<ConstantSDNode>(Node->getOperand(InlineAsm::Op_ExtraInfo))->
                           getZExtValue();
     MI->addOperand(MachineOperand::CreateImm(ExtraInfo));
 
+    // Remember to operand index of the group flags.
+    SmallVector<unsigned, 8> GroupIdx;
+
     // Add all of the operand registers to the instruction.
     for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) {
       unsigned Flags =
         cast<ConstantSDNode>(Node->getOperand(i))->getZExtValue();
-      unsigned NumVals = InlineAsm::getNumOperandRegisters(Flags);
+      const unsigned NumVals = InlineAsm::getNumOperandRegisters(Flags);
 
+      GroupIdx.push_back(MI->getNumOperands());
       MI->addOperand(MachineOperand::CreateImm(Flags));
       ++i;  // Skip the ID value.
 
       switch (InlineAsm::getKind(Flags)) {
       default: llvm_unreachable("Bad flags!");
         case InlineAsm::Kind_RegDef:
-        for (; NumVals; --NumVals, ++i) {
+        for (unsigned j = 0; j != NumVals; ++j, ++i) {
           unsigned Reg = cast<RegisterSDNode>(Node->getOperand(i))->getReg();
           // FIXME: Add dead flags for physical and virtual registers defined.
           // For now, mark physical register defs as implicit to help fast
@@ -913,7 +931,7 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned,
         break;
       case InlineAsm::Kind_RegDefEarlyClobber:
       case InlineAsm::Kind_Clobber:
-        for (; NumVals; --NumVals, ++i) {
+        for (unsigned j = 0; j != NumVals; ++j, ++i) {
           unsigned Reg = cast<RegisterSDNode>(Node->getOperand(i))->getReg();
           MI->addOperand(MachineOperand::CreateReg(Reg, /*isDef=*/ true,
                          /*isImp=*/ TargetRegisterInfo::isPhysicalRegister(Reg),
@@ -928,9 +946,20 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned,
       case InlineAsm::Kind_Mem:  // Addressing mode.
         // The addressing mode has been selected, just add all of the
         // operands to the machine instruction.
-        for (; NumVals; --NumVals, ++i)
+        for (unsigned j = 0; j != NumVals; ++j, ++i)
           AddOperand(MI, Node->getOperand(i), 0, 0, VRBaseMap,
                      /*IsDebug=*/false, IsClone, IsCloned);
+
+        // Manually set isTied bits.
+        if (InlineAsm::getKind(Flags) == InlineAsm::Kind_RegUse) {
+          unsigned DefGroup = 0;
+          if (InlineAsm::isUseOperandTiedToDef(Flags, DefGroup)) {
+            unsigned DefIdx = GroupIdx[DefGroup] + 1;
+            unsigned UseIdx = GroupIdx.back() + 1;
+            for (unsigned j = 0; j != NumVals; ++j)
+              MI->tieOperands(DefIdx + j, UseIdx + j);
+          }
+        }
         break;
       }
     }
diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 908ebb9..abf40b7 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -22,7 +22,7 @@
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/Target/TargetFrameLowering.h"
 #include "llvm/Target/TargetLowering.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -718,7 +718,7 @@ void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
           // expand it.
           if (!TLI.allowsUnalignedMemoryAccesses(ST->getMemoryVT())) {
             Type *Ty = ST->getMemoryVT().getTypeForEVT(*DAG.getContext());
-            unsigned ABIAlignment= TLI.getTargetData()->getABITypeAlignment(Ty);
+            unsigned ABIAlignment= TLI.getDataLayout()->getABITypeAlignment(Ty);
             if (ST->getAlignment() < ABIAlignment)
               ExpandUnalignedStore(cast<StoreSDNode>(Node),
                                    DAG, TLI, this);
@@ -824,7 +824,7 @@ void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
           // expand it.
           if (!TLI.allowsUnalignedMemoryAccesses(ST->getMemoryVT())) {
             Type *Ty = ST->getMemoryVT().getTypeForEVT(*DAG.getContext());
-            unsigned ABIAlignment= TLI.getTargetData()->getABITypeAlignment(Ty);
+            unsigned ABIAlignment= TLI.getDataLayout()->getABITypeAlignment(Ty);
             if (ST->getAlignment() < ABIAlignment)
               ExpandUnalignedStore(cast<StoreSDNode>(Node), DAG, TLI, this);
           }
@@ -869,25 +869,24 @@ void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
     switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
     default: llvm_unreachable("This action is not supported yet!");
     case TargetLowering::Legal:
-             // If this is an unaligned load and the target doesn't support it,
-             // expand it.
-             if (!TLI.allowsUnalignedMemoryAccesses(LD->getMemoryVT())) {
-               Type *Ty = LD->getMemoryVT().getTypeForEVT(*DAG.getContext());
-               unsigned ABIAlignment =
-                 TLI.getTargetData()->getABITypeAlignment(Ty);
-               if (LD->getAlignment() < ABIAlignment){
-                 ExpandUnalignedLoad(cast<LoadSDNode>(Node),
-                                     DAG, TLI, RVal, RChain);
-               }
-             }
-             break;
+      // If this is an unaligned load and the target doesn't support it,
+      // expand it.
+      if (!TLI.allowsUnalignedMemoryAccesses(LD->getMemoryVT())) {
+        Type *Ty = LD->getMemoryVT().getTypeForEVT(*DAG.getContext());
+        unsigned ABIAlignment =
+          TLI.getDataLayout()->getABITypeAlignment(Ty);
+        if (LD->getAlignment() < ABIAlignment){
+          ExpandUnalignedLoad(cast<LoadSDNode>(Node), DAG, TLI, RVal, RChain);
+        }
+      }
+      break;
     case TargetLowering::Custom: {
-             SDValue Res = TLI.LowerOperation(RVal, DAG);
-             if (Res.getNode()) {
-               RVal = Res;
-               RChain = Res.getValue(1);
-             }
-             break;
+      SDValue Res = TLI.LowerOperation(RVal, DAG);
+      if (Res.getNode()) {
+        RVal = Res;
+        RChain = Res.getValue(1);
+      }
+      break;
     }
     case TargetLowering::Promote: {
       // Only promote a load of vector type to another.
@@ -1060,7 +1059,7 @@ void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
                  Type *Ty =
                    LD->getMemoryVT().getTypeForEVT(*DAG.getContext());
                  unsigned ABIAlignment =
-                   TLI.getTargetData()->getABITypeAlignment(Ty);
+                   TLI.getDataLayout()->getABITypeAlignment(Ty);
                  if (LD->getAlignment() < ABIAlignment){
                    ExpandUnalignedLoad(cast<LoadSDNode>(Node),
                                        DAG, TLI, Value, Chain);
@@ -1241,6 +1240,19 @@ void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
     if (Action == TargetLowering::Legal)
       Action = TargetLowering::Custom;
     break;
+  case ISD::DEBUGTRAP:
+    Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
+    if (Action == TargetLowering::Expand) {
+      // replace ISD::DEBUGTRAP with ISD::TRAP
+      SDValue NewVal;
+      NewVal = DAG.getNode(ISD::TRAP, Node->getDebugLoc(), Node->getVTList(),
+                           Node->getOperand(0));
+      ReplaceNode(Node, NewVal.getNode());
+      LegalizeOp(NewVal.getNode());
+      return;
+    }
+    break;
+
   default:
     if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
       Action = TargetLowering::Legal;
@@ -1588,26 +1600,71 @@ void SelectionDAGLegalize::LegalizeSetCCCondCode(EVT VT,
     break;
   case TargetLowering::Expand: {
     ISD::CondCode CC1 = ISD::SETCC_INVALID, CC2 = ISD::SETCC_INVALID;
+    ISD::CondCode InvCC = ISD::SETCC_INVALID;
     unsigned Opc = 0;
     switch (CCCode) {
     default: llvm_unreachable("Don't know how to expand this condition!");
-    case ISD::SETOEQ: CC1 = ISD::SETEQ; CC2 = ISD::SETO;  Opc = ISD::AND; break;
-    case ISD::SETOGT: CC1 = ISD::SETGT; CC2 = ISD::SETO;  Opc = ISD::AND; break;
-    case ISD::SETOGE: CC1 = ISD::SETGE; CC2 = ISD::SETO;  Opc = ISD::AND; break;
-    case ISD::SETOLT: CC1 = ISD::SETLT; CC2 = ISD::SETO;  Opc = ISD::AND; break;
-    case ISD::SETOLE: CC1 = ISD::SETLE; CC2 = ISD::SETO;  Opc = ISD::AND; break;
-    case ISD::SETONE: CC1 = ISD::SETNE; CC2 = ISD::SETO;  Opc = ISD::AND; break;
-    case ISD::SETUEQ: CC1 = ISD::SETEQ; CC2 = ISD::SETUO; Opc = ISD::OR;  break;
-    case ISD::SETUGT: CC1 = ISD::SETGT; CC2 = ISD::SETUO; Opc = ISD::OR;  break;
-    case ISD::SETUGE: CC1 = ISD::SETGE; CC2 = ISD::SETUO; Opc = ISD::OR;  break;
-    case ISD::SETULT: CC1 = ISD::SETLT; CC2 = ISD::SETUO; Opc = ISD::OR;  break;
-    case ISD::SETULE: CC1 = ISD::SETLE; CC2 = ISD::SETUO; Opc = ISD::OR;  break;
-    case ISD::SETUNE: CC1 = ISD::SETNE; CC2 = ISD::SETUO; Opc = ISD::OR;  break;
-    // FIXME: Implement more expansions.
-    }
-
-    SDValue SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1);
-    SDValue SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2);
+    case ISD::SETO: 
+        assert(TLI.getCondCodeAction(ISD::SETOEQ, OpVT)
+            == TargetLowering::Legal
+            && "If SETO is expanded, SETOEQ must be legal!");
+        CC1 = ISD::SETOEQ; CC2 = ISD::SETOEQ; Opc = ISD::AND; break;
+    case ISD::SETUO:  
+        assert(TLI.getCondCodeAction(ISD::SETUNE, OpVT)
+            == TargetLowering::Legal
+            && "If SETUO is expanded, SETUNE must be legal!");
+        CC1 = ISD::SETUNE; CC2 = ISD::SETUNE; Opc = ISD::OR;  break;
+    case ISD::SETOEQ:
+    case ISD::SETOGT:
+    case ISD::SETOGE:
+    case ISD::SETOLT:
+    case ISD::SETOLE:
+    case ISD::SETONE: 
+    case ISD::SETUEQ: 
+    case ISD::SETUNE: 
+    case ISD::SETUGT: 
+    case ISD::SETUGE: 
+    case ISD::SETULT: 
+    case ISD::SETULE:
+        // If we are floating point, assign and break, otherwise fall through.
+        if (!OpVT.isInteger()) {
+          // We can use the 4th bit to tell if we are the unordered
+          // or ordered version of the opcode.
+          CC2 = ((unsigned)CCCode & 0x8U) ? ISD::SETUO : ISD::SETO;
+          Opc = ((unsigned)CCCode & 0x8U) ? ISD::OR : ISD::AND;
+          CC1 = (ISD::CondCode)(((int)CCCode & 0x7) | 0x10);
+          break;
+        }
+        // Fallthrough if we are unsigned integer.
+    case ISD::SETLE:
+    case ISD::SETGT:
+    case ISD::SETGE:
+    case ISD::SETLT:
+    case ISD::SETNE:
+    case ISD::SETEQ:
+      InvCC = ISD::getSetCCSwappedOperands(CCCode);
+      if (TLI.getCondCodeAction(InvCC, OpVT) == TargetLowering::Expand) {
+        // We only support using the inverted operation and not a
+        // different manner of supporting expanding these cases.
+        llvm_unreachable("Don't know how to expand this condition!");
+      }
+      LHS = DAG.getSetCC(dl, VT, RHS, LHS, InvCC);
+      RHS = SDValue();
+      CC = SDValue();
+      return;
+    }
+    
+    SDValue SetCC1, SetCC2;
+    if (CCCode != ISD::SETO && CCCode != ISD::SETUO) {
+      // If we aren't the ordered or unorder operation,
+      // then the pattern is (LHS CC1 RHS) Opc (LHS CC2 RHS).
+      SetCC1 = DAG.getSetCC(dl, VT, LHS, RHS, CC1);
+      SetCC2 = DAG.getSetCC(dl, VT, LHS, RHS, CC2);
+    } else {
+      // Otherwise, the pattern is (LHS CC1 LHS) Opc (RHS CC2 RHS)
+      SetCC1 = DAG.getSetCC(dl, VT, LHS, LHS, CC1);
+      SetCC2 = DAG.getSetCC(dl, VT, RHS, RHS, CC2);
+    }
     LHS = DAG.getNode(Opc, dl, VT, SetCC1, SetCC2);
     RHS = SDValue();
     CC  = SDValue();
@@ -1626,7 +1683,7 @@ SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp,
                                                DebugLoc dl) {
   // Create the stack frame object.
   unsigned SrcAlign =
-    TLI.getTargetData()->getPrefTypeAlignment(SrcOp.getValueType().
+    TLI.getDataLayout()->getPrefTypeAlignment(SrcOp.getValueType().
                                               getTypeForEVT(*DAG.getContext()));
   SDValue FIPtr = DAG.CreateStackTemporary(SlotVT, SrcAlign);
 
@@ -1638,7 +1695,7 @@ SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp,
   unsigned SlotSize = SlotVT.getSizeInBits();
   unsigned DestSize = DestVT.getSizeInBits();
   Type *DestType = DestVT.getTypeForEVT(*DAG.getContext());
-  unsigned DestAlign = TLI.getTargetData()->getPrefTypeAlignment(DestType);
+  unsigned DestAlign = TLI.getDataLayout()->getPrefTypeAlignment(DestType);
 
   // Emit a store to the stack slot.  Use a truncstore if the input value is
   // later than DestVT.
@@ -2042,7 +2099,7 @@ SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(bool isSigned,
                                                    SDValue Op0,
                                                    EVT DestVT,
                                                    DebugLoc dl) {
-  if (Op0.getValueType() == MVT::i32) {
+  if (Op0.getValueType() == MVT::i32 && TLI.isTypeLegal(MVT::f64)) {
     // simple 32-bit [signed|unsigned] integer to float/double expansion
 
     // Get the stack frame index of a 8 byte buffer.
@@ -2787,7 +2844,7 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) {
 
     // Increment the pointer, VAList, to the next vaarg
     Tmp3 = DAG.getNode(ISD::ADD, dl, TLI.getPointerTy(), VAList,
-                       DAG.getConstant(TLI.getTargetData()->
+                       DAG.getConstant(TLI.getDataLayout()->
                           getTypeAllocSize(VT.getTypeForEVT(*DAG.getContext())),
                                        TLI.getPointerTy()));
     // Store the incremented VAList to the legalized pointer
@@ -3109,6 +3166,8 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) {
     Tmp3 = Node->getOperand(1);
     if (TLI.isOperationLegalOrCustom(DivRemOpc, VT) ||
         (isDivRemLibcallAvailable(Node, isSigned, TLI) &&
+         // If div is legal, it's better to do the normal expansion
+         !TLI.isOperationLegalOrCustom(DivOpc, Node->getValueType(0)) &&
          useDivRem(Node, isSigned, false))) {
       Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Tmp2, Tmp3).getValue(1);
     } else if (TLI.isOperationLegalOrCustom(DivOpc, VT)) {
@@ -3366,7 +3425,7 @@ void SelectionDAGLegalize::ExpandNode(SDNode *Node) {
 
     EVT PTy = TLI.getPointerTy();
 
-    const TargetData &TD = *TLI.getTargetData();
+    const DataLayout &TD = *TLI.getDataLayout();
     unsigned EntrySize =
       DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
 
diff --git a/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
index e393896..92dc5a9 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
@@ -1245,32 +1245,30 @@ bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) {
   DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG); dbgs() << "\n");
   SDValue Res = SDValue();
 
-  if (TLI.getOperationAction(N->getOpcode(), N->getOperand(OpNo).getValueType())
-      == TargetLowering::Custom)
-    Res = TLI.LowerOperation(SDValue(N, 0), DAG);
-
-  if (Res.getNode() == 0) {
-    switch (N->getOpcode()) {
-    default:
-  #ifndef NDEBUG
-      dbgs() << "ExpandFloatOperand Op #" << OpNo << ": ";
-      N->dump(&DAG); dbgs() << "\n";
-  #endif
-      llvm_unreachable("Do not know how to expand this operator's operand!");
-
-    case ISD::BITCAST:         Res = ExpandOp_BITCAST(N); break;
-    case ISD::BUILD_VECTOR:    Res = ExpandOp_BUILD_VECTOR(N); break;
-    case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
-
-    case ISD::BR_CC:      Res = ExpandFloatOp_BR_CC(N); break;
-    case ISD::FP_ROUND:   Res = ExpandFloatOp_FP_ROUND(N); break;
-    case ISD::FP_TO_SINT: Res = ExpandFloatOp_FP_TO_SINT(N); break;
-    case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_UINT(N); break;
-    case ISD::SELECT_CC:  Res = ExpandFloatOp_SELECT_CC(N); break;
-    case ISD::SETCC:      Res = ExpandFloatOp_SETCC(N); break;
-    case ISD::STORE:      Res = ExpandFloatOp_STORE(cast<StoreSDNode>(N),
-                                                    OpNo); break;
-    }
+  // See if the target wants to custom expand this node.
+  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
+    return false;
+
+  switch (N->getOpcode()) {
+  default:
+#ifndef NDEBUG
+    dbgs() << "ExpandFloatOperand Op #" << OpNo << ": ";
+    N->dump(&DAG); dbgs() << "\n";
+#endif
+    llvm_unreachable("Do not know how to expand this operator's operand!");
+
+  case ISD::BITCAST:         Res = ExpandOp_BITCAST(N); break;
+  case ISD::BUILD_VECTOR:    Res = ExpandOp_BUILD_VECTOR(N); break;
+  case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
+
+  case ISD::BR_CC:      Res = ExpandFloatOp_BR_CC(N); break;
+  case ISD::FP_ROUND:   Res = ExpandFloatOp_FP_ROUND(N); break;
+  case ISD::FP_TO_SINT: Res = ExpandFloatOp_FP_TO_SINT(N); break;
+  case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_UINT(N); break;
+  case ISD::SELECT_CC:  Res = ExpandFloatOp_SELECT_CC(N); break;
+  case ISD::SETCC:      Res = ExpandFloatOp_SETCC(N); break;
+  case ISD::STORE:      Res = ExpandFloatOp_STORE(cast<StoreSDNode>(N),
+                                                  OpNo); break;
   }
 
   // If the result is null, the sub-method took care of registering results etc.
diff --git a/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index e8e968a..a370fae 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -644,8 +644,9 @@ SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
   EVT SmallVT = LHS.getValueType();
 
   // To determine if the result overflowed in a larger type, we extend the
-  // input to the larger type, do the multiply, then check the high bits of
-  // the result to see if the overflow happened.
+  // input to the larger type, do the multiply (checking if it overflows),
+  // then also check the high bits of the result to see if overflow happened
+  // there.
   if (N->getOpcode() == ISD::SMULO) {
     LHS = SExtPromotedInteger(LHS);
     RHS = SExtPromotedInteger(RHS);
@@ -653,24 +654,31 @@ SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
     LHS = ZExtPromotedInteger(LHS);
     RHS = ZExtPromotedInteger(RHS);
   }
-  SDValue Mul = DAG.getNode(ISD::MUL, DL, LHS.getValueType(), LHS, RHS);
+  SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1));
+  SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS);
 
-  // Overflow occurred iff the high part of the result does not
-  // zero/sign-extend the low part.
+  // Overflow occurred if it occurred in the larger type, or if the high part
+  // of the result does not zero/sign-extend the low part.  Check this second
+  // possibility first.
   SDValue Overflow;
   if (N->getOpcode() == ISD::UMULO) {
-    // Unsigned overflow occurred iff the high part is non-zero.
+    // Unsigned overflow occurred if the high part is non-zero.
     SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
                              DAG.getIntPtrConstant(SmallVT.getSizeInBits()));
     Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi,
                             DAG.getConstant(0, Hi.getValueType()), ISD::SETNE);
   } else {
-    // Signed overflow occurred iff the high part does not sign extend the low.
+    // Signed overflow occurred if the high part does not sign extend the low.
     SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(),
                                Mul, DAG.getValueType(SmallVT));
     Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE);
   }
 
+  // The only other way for overflow to occur is if the multiplication in the
+  // larger type itself overflowed.
+  Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow,
+                         SDValue(Mul.getNode(), 1));
+
   // Use the calculated overflow everywhere.
   ReplaceValueWith(SDValue(N, 1), Overflow);
   return Mul;
@@ -2253,32 +2261,35 @@ void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
 void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
                                           SDValue &Lo, SDValue &Hi) {
   EVT VT = N->getValueType(0);
-  Type *RetTy = VT.getTypeForEVT(*DAG.getContext());
-  EVT PtrVT = TLI.getPointerTy();
-  Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext());
   DebugLoc dl = N->getDebugLoc();
 
   // A divide for UMULO should be faster than a function call.
   if (N->getOpcode() == ISD::UMULO) {
     SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
-    DebugLoc DL = N->getDebugLoc();
 
-    SDValue MUL = DAG.getNode(ISD::MUL, DL, LHS.getValueType(), LHS, RHS);
+    SDValue MUL = DAG.getNode(ISD::MUL, dl, LHS.getValueType(), LHS, RHS);
     SplitInteger(MUL, Lo, Hi);
 
     // A divide for UMULO will be faster than a function call. Select to
     // make sure we aren't using 0.
     SDValue isZero = DAG.getSetCC(dl, TLI.getSetCCResultType(VT),
-                                  RHS, DAG.getConstant(0, VT), ISD::SETNE);
+                                  RHS, DAG.getConstant(0, VT), ISD::SETEQ);
     SDValue NotZero = DAG.getNode(ISD::SELECT, dl, VT, isZero,
                                   DAG.getConstant(1, VT), RHS);
-    SDValue DIV = DAG.getNode(ISD::UDIV, DL, LHS.getValueType(), MUL, NotZero);
-    SDValue Overflow;
-    Overflow = DAG.getSetCC(DL, N->getValueType(1), DIV, LHS, ISD::SETNE);
+    SDValue DIV = DAG.getNode(ISD::UDIV, dl, VT, MUL, NotZero);
+    SDValue Overflow = DAG.getSetCC(dl, N->getValueType(1), DIV, LHS,
+                                    ISD::SETNE);
+    Overflow = DAG.getNode(ISD::SELECT, dl, N->getValueType(1), isZero,
+                           DAG.getConstant(0, N->getValueType(1)),
+                           Overflow);
     ReplaceValueWith(SDValue(N, 1), Overflow);
     return;
   }
 
+  Type *RetTy = VT.getTypeForEVT(*DAG.getContext());
+  EVT PtrVT = TLI.getPointerTy();
+  Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext());
+  
   // Replace this with a libcall that will check overflow.
   RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
   if (VT == MVT::i32)
diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp
index 39337ff..644e36e 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp
@@ -15,7 +15,7 @@
 
 #include "LegalizeTypes.h"
 #include "llvm/CallingConv.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/ADT/SetVector.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ErrorHandling.h"
diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index 94fc976..20b7ce6 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -625,6 +625,7 @@ private:
   SDValue WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N);
   SDValue WidenVecRes_VSETCC(SDNode* N);
 
+  SDValue WidenVecRes_Ternary(SDNode *N);
   SDValue WidenVecRes_Binary(SDNode *N);
   SDValue WidenVecRes_Convert(SDNode *N);
   SDValue WidenVecRes_POWI(SDNode *N);
@@ -633,7 +634,7 @@ private:
   SDValue WidenVecRes_InregOp(SDNode *N);
 
   // Widen Vector Operand.
-  bool WidenVectorOperand(SDNode *N, unsigned ResNo);
+  bool WidenVectorOperand(SDNode *N, unsigned OpNo);
   SDValue WidenVecOp_BITCAST(SDNode *N);
   SDValue WidenVecOp_CONCAT_VECTORS(SDNode *N);
   SDValue WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N);
diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp b/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp
index 06f6bd6..6bcb3b2 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp
@@ -20,7 +20,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "LegalizeTypes.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 using namespace llvm;
 
 //===----------------------------------------------------------------------===//
@@ -94,14 +94,48 @@ void DAGTypeLegalizer::ExpandRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi) {
   if (InVT.isVector() && OutVT.isInteger()) {
     // Handle cases like i64 = BITCAST v1i64 on x86, where the operand
     // is legal but the result is not.
-    EVT NVT = EVT::getVectorVT(*DAG.getContext(), NOutVT, 2);
+    unsigned NumElems = 2;
+    EVT ElemVT = NOutVT;
+    EVT NVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, NumElems);
+
+    // If <ElemVT * N> is not a legal type, try <ElemVT/2 * (N*2)>.
+    while (!isTypeLegal(NVT)) {
+      unsigned NewSizeInBits = ElemVT.getSizeInBits() / 2;
+      // If the element size is smaller than byte, bail.
+      if (NewSizeInBits < 8)
+        break;
+      NumElems *= 2;
+      ElemVT = EVT::getIntegerVT(*DAG.getContext(), NewSizeInBits);
+      NVT = EVT::getVectorVT(*DAG.getContext(), ElemVT, NumElems);
+    }
 
     if (isTypeLegal(NVT)) {
       SDValue CastInOp = DAG.getNode(ISD::BITCAST, dl, NVT, InOp);
-      Lo = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NOutVT, CastInOp,
-                       DAG.getIntPtrConstant(0));
-      Hi = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NOutVT, CastInOp,
-                       DAG.getIntPtrConstant(1));
+
+      SmallVector<SDValue, 8> Vals;
+      for (unsigned i = 0; i < NumElems; ++i)
+        Vals.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ElemVT,
+                                   CastInOp, DAG.getIntPtrConstant(i)));
+
+      // Build Lo, Hi pair by pairing extracted elements if needed.
+      unsigned Slot = 0;
+      for (unsigned e = Vals.size(); e - Slot > 2; Slot += 2, e += 1) {
+        // Each iteration will BUILD_PAIR two nodes and append the result until
+        // there are only two nodes left, i.e. Lo and Hi.
+        SDValue LHS = Vals[Slot];
+        SDValue RHS = Vals[Slot + 1];
+
+        if (TLI.isBigEndian())
+          std::swap(LHS, RHS);
+
+        Vals.push_back(DAG.getNode(ISD::BUILD_PAIR, dl,
+                                   EVT::getIntegerVT(
+                                     *DAG.getContext(),
+                                     LHS.getValueType().getSizeInBits() << 1),
+                                   LHS, RHS));
+      }
+      Lo = Vals[Slot++];
+      Hi = Vals[Slot++];
 
       if (TLI.isBigEndian())
         std::swap(Lo, Hi);
@@ -116,7 +150,7 @@ void DAGTypeLegalizer::ExpandRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi) {
   // Create the stack frame object.  Make sure it is aligned for both
   // the source and expanded destination types.
   unsigned Alignment =
-    TLI.getTargetData()->getPrefTypeAlignment(NOutVT.
+    TLI.getDataLayout()->getPrefTypeAlignment(NOutVT.
                                               getTypeForEVT(*DAG.getContext()));
   SDValue StackPtr = DAG.CreateStackTemporary(InVT, Alignment);
   int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
diff --git a/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index 704f99b..22f8d51 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -64,6 +64,7 @@ class VectorLegalizer {
   // Implement vselect in terms of XOR, AND, OR when blend is not supported
   // by the target.
   SDValue ExpandVSELECT(SDValue Op);
+  SDValue ExpandSELECT(SDValue Op);
   SDValue ExpandLoad(SDValue Op);
   SDValue ExpandStore(SDValue Op);
   SDValue ExpandFNEG(SDValue Op);
@@ -220,6 +221,7 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
   case ISD::FRINT:
   case ISD::FNEARBYINT:
   case ISD::FFLOOR:
+  case ISD::FMA:
   case ISD::SIGN_EXTEND_INREG:
     QueryType = Node->getValueType(0);
     break;
@@ -260,6 +262,8 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
   case TargetLowering::Expand:
     if (Node->getOpcode() == ISD::VSELECT)
       Result = ExpandVSELECT(Op);
+    else if (Node->getOpcode() == ISD::SELECT)
+      Result = ExpandSELECT(Op);
     else if (Node->getOpcode() == ISD::UINT_TO_FP)
       Result = ExpandUINT_TO_FLOAT(Op);
     else if (Node->getOpcode() == ISD::FNEG)
@@ -435,6 +439,66 @@ SDValue VectorLegalizer::ExpandStore(SDValue Op) {
   return TF;
 }
 
+SDValue VectorLegalizer::ExpandSELECT(SDValue Op) {
+  // Lower a select instruction where the condition is a scalar and the
+  // operands are vectors. Lower this select to VSELECT and implement it
+  // using XOR AND OR. The selector bit is broadcasted. 
+  EVT VT = Op.getValueType();
+  DebugLoc DL = Op.getDebugLoc();
+
+  SDValue Mask = Op.getOperand(0);
+  SDValue Op1 = Op.getOperand(1);
+  SDValue Op2 = Op.getOperand(2);
+
+  assert(VT.isVector() && !Mask.getValueType().isVector()
+         && Op1.getValueType() == Op2.getValueType() && "Invalid type");
+
+  unsigned NumElem = VT.getVectorNumElements();
+
+  // If we can't even use the basic vector operations of
+  // AND,OR,XOR, we will have to scalarize the op.
+  // Notice that the operation may be 'promoted' which means that it is
+  // 'bitcasted' to another type which is handled.
+  // Also, we need to be able to construct a splat vector using BUILD_VECTOR.
+  if (TLI.getOperationAction(ISD::AND, VT) == TargetLowering::Expand ||
+      TLI.getOperationAction(ISD::XOR, VT) == TargetLowering::Expand ||
+      TLI.getOperationAction(ISD::OR,  VT) == TargetLowering::Expand ||
+      TLI.getOperationAction(ISD::BUILD_VECTOR,  VT) == TargetLowering::Expand)
+    return DAG.UnrollVectorOp(Op.getNode());
+
+  // Generate a mask operand.
+  EVT MaskTy = TLI.getSetCCResultType(VT);
+  assert(MaskTy.isVector() && "Invalid CC type");
+  assert(MaskTy.getSizeInBits() == Op1.getValueType().getSizeInBits()
+         && "Invalid mask size");
+
+  // What is the size of each element in the vector mask.
+  EVT BitTy = MaskTy.getScalarType();
+
+  Mask = DAG.getNode(ISD::SELECT, DL, BitTy, Mask,
+          DAG.getConstant(APInt::getAllOnesValue(BitTy.getSizeInBits()), BitTy),
+          DAG.getConstant(0, BitTy));
+
+  // Broadcast the mask so that the entire vector is all-one or all zero.
+  SmallVector<SDValue, 8> Ops(NumElem, Mask);
+  Mask = DAG.getNode(ISD::BUILD_VECTOR, DL, MaskTy, &Ops[0], Ops.size());
+
+  // Bitcast the operands to be the same type as the mask.
+  // This is needed when we select between FP types because
+  // the mask is a vector of integers.
+  Op1 = DAG.getNode(ISD::BITCAST, DL, MaskTy, Op1);
+  Op2 = DAG.getNode(ISD::BITCAST, DL, MaskTy, Op2);
+
+  SDValue AllOnes = DAG.getConstant(
+            APInt::getAllOnesValue(BitTy.getSizeInBits()), MaskTy);
+  SDValue NotMask = DAG.getNode(ISD::XOR, DL, MaskTy, Mask, AllOnes);
+
+  Op1 = DAG.getNode(ISD::AND, DL, MaskTy, Op1, Mask);
+  Op2 = DAG.getNode(ISD::AND, DL, MaskTy, Op2, NotMask);
+  SDValue Val = DAG.getNode(ISD::OR, DL, MaskTy, Op1, Op2);
+  return DAG.getNode(ISD::BITCAST, DL, Op.getValueType(), Val);
+}
+
 SDValue VectorLegalizer::ExpandVSELECT(SDValue Op) {
   // Implement VSELECT in terms of XOR, AND, OR
   // on platforms which do not support blend natively.
@@ -449,12 +513,17 @@ SDValue VectorLegalizer::ExpandVSELECT(SDValue Op) {
   // AND,OR,XOR, we will have to scalarize the op.
   // Notice that the operation may be 'promoted' which means that it is
   // 'bitcasted' to another type which is handled.
+  // This operation also isn't safe with AND, OR, XOR when the boolean
+  // type is 0/1 as we need an all ones vector constant to mask with.
+  // FIXME: Sign extend 1 to all ones if thats legal on the target.
   if (TLI.getOperationAction(ISD::AND, VT) == TargetLowering::Expand ||
       TLI.getOperationAction(ISD::XOR, VT) == TargetLowering::Expand ||
-      TLI.getOperationAction(ISD::OR,  VT) == TargetLowering::Expand)
+      TLI.getOperationAction(ISD::OR,  VT) == TargetLowering::Expand ||
+      TLI.getBooleanContents(true) !=
+      TargetLowering::ZeroOrNegativeOneBooleanContent)
     return DAG.UnrollVectorOp(Op.getNode());
 
-  assert(VT.getSizeInBits() == Op.getOperand(1).getValueType().getSizeInBits()
+  assert(VT.getSizeInBits() == Op1.getValueType().getSizeInBits()
          && "Invalid mask size");
   // Bitcast the operands to be the same type as the mask.
   // This is needed when we select between FP types because
diff --git a/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 4709202..d51a6eb 100644
--- a/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -21,7 +21,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "LegalizeTypes.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
 using namespace llvm;
@@ -749,7 +749,7 @@ void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo,
   SDValue EltPtr = GetVectorElementPointer(StackPtr, EltVT, Idx);
   Type *VecType = VecVT.getTypeForEVT(*DAG.getContext());
   unsigned Alignment =
-    TLI.getTargetData()->getPrefTypeAlignment(VecType);
+    TLI.getDataLayout()->getPrefTypeAlignment(VecType);
   Store = DAG.getTruncStore(Store, dl, Elt, EltPtr, MachinePointerInfo(), EltVT,
                             false, false, 0);
 
@@ -1366,6 +1366,9 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
   case ISD::FTRUNC:
     Res = WidenVecRes_Unary(N);
     break;
+  case ISD::FMA:
+    Res = WidenVecRes_Ternary(N);
+    break;
   }
 
   // If Res is null, the sub-method took care of registering the result.
@@ -1373,6 +1376,16 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
     SetWidenedVector(SDValue(N, ResNo), Res);
 }
 
+SDValue DAGTypeLegalizer::WidenVecRes_Ternary(SDNode *N) {
+  // Ternary op widening.
+  DebugLoc dl = N->getDebugLoc();
+  EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
+  SDValue InOp1 = GetWidenedVector(N->getOperand(0));
+  SDValue InOp2 = GetWidenedVector(N->getOperand(1));
+  SDValue InOp3 = GetWidenedVector(N->getOperand(2));
+  return DAG.getNode(N->getOpcode(), dl, WidenVT, InOp1, InOp2, InOp3);
+}
+
 SDValue DAGTypeLegalizer::WidenVecRes_Binary(SDNode *N) {
   // Binary op widening.
   unsigned Opcode = N->getOpcode();
@@ -2069,16 +2082,20 @@ SDValue DAGTypeLegalizer::WidenVecRes_VSETCC(SDNode *N) {
 //===----------------------------------------------------------------------===//
 // Widen Vector Operand
 //===----------------------------------------------------------------------===//
-bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned ResNo) {
-  DEBUG(dbgs() << "Widen node operand " << ResNo << ": ";
+bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) {
+  DEBUG(dbgs() << "Widen node operand " << OpNo << ": ";
         N->dump(&DAG);
         dbgs() << "\n");
   SDValue Res = SDValue();
 
+  // See if the target wants to custom widen this node.
+  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
+    return false;
+
   switch (N->getOpcode()) {
   default:
 #ifndef NDEBUG
-    dbgs() << "WidenVectorOperand op #" << ResNo << ": ";
+    dbgs() << "WidenVectorOperand op #" << OpNo << ": ";
     N->dump(&DAG);
     dbgs() << "\n";
 #endif
diff --git a/lib/CodeGen/SelectionDAG/SDNodeOrdering.h b/lib/CodeGen/SelectionDAG/SDNodeOrdering.h
index f88b26d..d2269f8 100644
--- a/lib/CodeGen/SelectionDAG/SDNodeOrdering.h
+++ b/lib/CodeGen/SelectionDAG/SDNodeOrdering.h
@@ -28,8 +28,8 @@ class SDNode;
 class SDNodeOrdering {
   DenseMap<const SDNode*, unsigned> OrderMap;
 
-  void operator=(const SDNodeOrdering&);   // Do not implement.
-  SDNodeOrdering(const SDNodeOrdering&);   // Do not implement.
+  void operator=(const SDNodeOrdering&) LLVM_DELETED_FUNCTION;
+  SDNodeOrdering(const SDNodeOrdering&) LLVM_DELETED_FUNCTION;
 public:
   SDNodeOrdering() {}
 
diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp
index b7ce48a..2ecdd89 100644
--- a/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp
+++ b/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp
@@ -13,11 +13,12 @@
 
 #define DEBUG_TYPE "pre-RA-sched"
 #include "ScheduleDAGSDNodes.h"
+#include "InstrEmitter.h"
 #include "llvm/InlineAsm.h"
 #include "llvm/CodeGen/SchedulerRegistry.h"
 #include "llvm/CodeGen/SelectionDAGISel.h"
 #include "llvm/Target/TargetRegisterInfo.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetInstrInfo.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/ADT/SmallSet.h"
@@ -34,6 +35,10 @@ STATISTIC(NumPRCopies,   "Number of physical copies");
 static RegisterScheduler
   fastDAGScheduler("fast", "Fast suboptimal list scheduling",
                    createFastDAGScheduler);
+static RegisterScheduler
+  linearizeDAGScheduler("linearize", "Linearize DAG, no scheduling",
+                        createDAGLinearizer);
+
 
 namespace {
   /// FastPriorityQueue - A degenerate priority queue that considers
@@ -331,7 +336,9 @@ SUnit *ScheduleDAGFast::CopyAndMoveSuccessors(SUnit *SU) {
       }
     }
     if (isNewLoad) {
-      AddPred(NewSU, SDep(LoadSU, SDep::Order, LoadSU->Latency));
+      SDep D(LoadSU, SDep::Barrier);
+      D.setLatency(LoadSU->Latency);
+      AddPred(NewSU, D);
     }
 
     ++NumUnfolds;
@@ -407,9 +414,12 @@ void ScheduleDAGFast::InsertCopiesAndMoveSuccs(SUnit *SU, unsigned Reg,
   for (unsigned i = 0, e = DelDeps.size(); i != e; ++i) {
     RemovePred(DelDeps[i].first, DelDeps[i].second);
   }
-
-  AddPred(CopyFromSU, SDep(SU, SDep::Data, SU->Latency, Reg));
-  AddPred(CopyToSU, SDep(CopyFromSU, SDep::Data, CopyFromSU->Latency, 0));
+  SDep FromDep(SU, SDep::Data, Reg);
+  FromDep.setLatency(SU->Latency);
+  AddPred(CopyFromSU, FromDep);
+  SDep ToDep(CopyFromSU, SDep::Data, 0);
+  ToDep.setLatency(CopyFromSU->Latency);
+  AddPred(CopyToSU, ToDep);
 
   Copies.push_back(CopyFromSU);
   Copies.push_back(CopyToSU);
@@ -586,18 +596,14 @@ void ScheduleDAGFast::ListScheduleBottomUp() {
           InsertCopiesAndMoveSuccs(LRDef, Reg, DestRC, RC, Copies);
           DEBUG(dbgs() << "Adding an edge from SU # " << TrySU->NodeNum
                        << " to SU #" << Copies.front()->NodeNum << "\n");
-          AddPred(TrySU, SDep(Copies.front(), SDep::Order, /*Latency=*/1,
-                              /*Reg=*/0, /*isNormalMemory=*/false,
-                              /*isMustAlias=*/false, /*isArtificial=*/true));
+          AddPred(TrySU, SDep(Copies.front(), SDep::Artificial));
           NewDef = Copies.back();
         }
 
         DEBUG(dbgs() << "Adding an edge from SU # " << NewDef->NodeNum
                      << " to SU #" << TrySU->NodeNum << "\n");
         LiveRegDefs[Reg] = NewDef;
-        AddPred(NewDef, SDep(TrySU, SDep::Order, /*Latency=*/1,
-                             /*Reg=*/0, /*isNormalMemory=*/false,
-                             /*isMustAlias=*/false, /*isArtificial=*/true));
+        AddPred(NewDef, SDep(TrySU, SDep::Artificial));
         TrySU->isAvailable = false;
         CurSU = NewDef;
       }
@@ -629,6 +635,155 @@ void ScheduleDAGFast::ListScheduleBottomUp() {
 #endif
 }
 
+
+namespace {
+//===----------------------------------------------------------------------===//
+// ScheduleDAGLinearize - No scheduling scheduler, it simply linearize the
+// DAG in topological order.
+// IMPORTANT: this may not work for targets with phyreg dependency.
+//
+class ScheduleDAGLinearize : public ScheduleDAGSDNodes {
+public:
+  ScheduleDAGLinearize(MachineFunction &mf) : ScheduleDAGSDNodes(mf) {}
+
+  void Schedule();
+
+  MachineBasicBlock *EmitSchedule(MachineBasicBlock::iterator &InsertPos);
+
+private:
+  std::vector<SDNode*> Sequence;
+  DenseMap<SDNode*, SDNode*> GluedMap;  // Cache glue to its user
+
+  void ScheduleNode(SDNode *N);
+};
+} // end anonymous namespace
+
+void ScheduleDAGLinearize::ScheduleNode(SDNode *N) {
+  if (N->getNodeId() != 0)
+    llvm_unreachable(0);
+
+  if (!N->isMachineOpcode() &&
+      (N->getOpcode() == ISD::EntryToken || isPassiveNode(N)))
+    // These nodes do not need to be translated into MIs.
+    return;
+
+  DEBUG(dbgs() << "\n*** Scheduling: ");
+  DEBUG(N->dump(DAG));
+  Sequence.push_back(N);
+
+  unsigned NumOps = N->getNumOperands();
+  if (unsigned NumLeft = NumOps) {
+    SDNode *GluedOpN = 0;
+    do {
+      const SDValue &Op = N->getOperand(NumLeft-1);
+      SDNode *OpN = Op.getNode();
+
+      if (NumLeft == NumOps && Op.getValueType() == MVT::Glue) {
+        // Schedule glue operand right above N.
+        GluedOpN = OpN;
+        assert(OpN->getNodeId() != 0 && "Glue operand not ready?");
+        OpN->setNodeId(0);
+        ScheduleNode(OpN);
+        continue;
+      }
+
+      if (OpN == GluedOpN)
+        // Glue operand is already scheduled.
+        continue;
+
+      DenseMap<SDNode*, SDNode*>::iterator DI = GluedMap.find(OpN);
+      if (DI != GluedMap.end() && DI->second != N)
+        // Users of glues are counted against the glued users.
+        OpN = DI->second;
+
+      unsigned Degree = OpN->getNodeId();
+      assert(Degree > 0 && "Predecessor over-released!");
+      OpN->setNodeId(--Degree);
+      if (Degree == 0)
+        ScheduleNode(OpN);
+    } while (--NumLeft);
+  }
+}
+
+/// findGluedUser - Find the representative use of a glue value by walking
+/// the use chain.
+static SDNode *findGluedUser(SDNode *N) {
+  while (SDNode *Glued = N->getGluedUser())
+    N = Glued;
+  return N;
+}
+
+void ScheduleDAGLinearize::Schedule() {
+  DEBUG(dbgs() << "********** DAG Linearization **********\n");
+
+  SmallVector<SDNode*, 8> Glues;
+  unsigned DAGSize = 0;
+  for (SelectionDAG::allnodes_iterator I = DAG->allnodes_begin(),
+         E = DAG->allnodes_end(); I != E; ++I) {
+    SDNode *N = I;
+
+    // Use node id to record degree.
+    unsigned Degree = N->use_size();
+    N->setNodeId(Degree);
+    unsigned NumVals = N->getNumValues();
+    if (NumVals && N->getValueType(NumVals-1) == MVT::Glue &&
+        N->hasAnyUseOfValue(NumVals-1)) {
+      SDNode *User = findGluedUser(N);
+      if (User) {
+        Glues.push_back(N);
+        GluedMap.insert(std::make_pair(N, User));
+      }
+    }
+
+    if (N->isMachineOpcode() ||
+        (N->getOpcode() != ISD::EntryToken && !isPassiveNode(N)))
+      ++DAGSize;
+  }
+
+  for (unsigned i = 0, e = Glues.size(); i != e; ++i) {
+    SDNode *Glue = Glues[i];
+    SDNode *GUser = GluedMap[Glue];
+    unsigned Degree = Glue->getNodeId();
+    unsigned UDegree = GUser->getNodeId();
+
+    // Glue user must be scheduled together with the glue operand. So other
+    // users of the glue operand must be treated as its users.
+    SDNode *ImmGUser = Glue->getGluedUser();
+    for (SDNode::use_iterator ui = Glue->use_begin(), ue = Glue->use_end();
+         ui != ue; ++ui)
+      if (*ui == ImmGUser)
+        --Degree;
+    GUser->setNodeId(UDegree + Degree);
+    Glue->setNodeId(1);
+  }
+
+  Sequence.reserve(DAGSize);
+  ScheduleNode(DAG->getRoot().getNode());
+}
+
+MachineBasicBlock*
+ScheduleDAGLinearize::EmitSchedule(MachineBasicBlock::iterator &InsertPos) {
+  InstrEmitter Emitter(BB, InsertPos);
+  DenseMap<SDValue, unsigned> VRBaseMap;
+
+  DEBUG({
+      dbgs() << "\n*** Final schedule ***\n";
+    });
+
+  // FIXME: Handle dbg_values.
+  unsigned NumNodes = Sequence.size();
+  for (unsigned i = 0; i != NumNodes; ++i) {
+    SDNode *N = Sequence[NumNodes-i-1];
+    DEBUG(N->dump(DAG));
+    Emitter.EmitNode(N, false, false, VRBaseMap);
+  }
+
+  DEBUG(dbgs() << '\n');
+
+  InsertPos = Emitter.getInsertPos();
+  return Emitter.getBlock();
+}
+
 //===----------------------------------------------------------------------===//
 //                         Public Constructor Functions
 //===----------------------------------------------------------------------===//
@@ -637,3 +792,8 @@ llvm::ScheduleDAGSDNodes *
 llvm::createFastDAGScheduler(SelectionDAGISel *IS, CodeGenOpt::Level) {
   return new ScheduleDAGFast(*IS->MF);
 }
+
+llvm::ScheduleDAGSDNodes *
+llvm::createDAGLinearizer(SelectionDAGISel *IS, CodeGenOpt::Level) {
+  return new ScheduleDAGLinearize(*IS->MF);
+}
diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp
index bf0a437..c554569 100644
--- a/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp
+++ b/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp
@@ -22,7 +22,7 @@
 #include "llvm/CodeGen/SelectionDAGISel.h"
 #include "llvm/CodeGen/ScheduleHazardRecognizer.h"
 #include "llvm/Target/TargetRegisterInfo.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetInstrInfo.h"
 #include "llvm/Target/TargetLowering.h"
@@ -656,6 +656,8 @@ void ScheduleDAGRRList::EmitNode(SUnit *SU) {
     break;
   case ISD::MERGE_VALUES:
   case ISD::TokenFactor:
+  case ISD::LIFETIME_START:
+  case ISD::LIFETIME_END:
   case ISD::CopyToReg:
   case ISD::CopyFromReg:
   case ISD::EH_LABEL:
@@ -1056,7 +1058,9 @@ SUnit *ScheduleDAGRRList::CopyAndMoveSuccessors(SUnit *SU) {
 
     // Add a data dependency to reflect that NewSU reads the value defined
     // by LoadSU.
-    AddPred(NewSU, SDep(LoadSU, SDep::Data, LoadSU->Latency));
+    SDep D(LoadSU, SDep::Data, 0);
+    D.setLatency(LoadSU->Latency);
+    AddPred(NewSU, D);
 
     if (isNewLoad)
       AvailableQueue->addNode(LoadSU);
@@ -1138,17 +1142,18 @@ void ScheduleDAGRRList::InsertCopiesAndMoveSuccs(SUnit *SU, unsigned Reg,
       // Avoid scheduling the def-side copy before other successors. Otherwise
       // we could introduce another physreg interference on the copy and
       // continue inserting copies indefinitely.
-      SDep D(CopyFromSU, SDep::Order, /*Latency=*/0,
-             /*Reg=*/0, /*isNormalMemory=*/false,
-             /*isMustAlias=*/false, /*isArtificial=*/true);
-      AddPred(SuccSU, D);
+      AddPred(SuccSU, SDep(CopyFromSU, SDep::Artificial));
     }
   }
   for (unsigned i = 0, e = DelDeps.size(); i != e; ++i)
     RemovePred(DelDeps[i].first, DelDeps[i].second);
 
-  AddPred(CopyFromSU, SDep(SU, SDep::Data, SU->Latency, Reg));
-  AddPred(CopyToSU, SDep(CopyFromSU, SDep::Data, CopyFromSU->Latency, 0));
+  SDep FromDep(SU, SDep::Data, Reg);
+  FromDep.setLatency(SU->Latency);
+  AddPred(CopyFromSU, FromDep);
+  SDep ToDep(CopyFromSU, SDep::Data, 0);
+  ToDep.setLatency(CopyFromSU->Latency);
+  AddPred(CopyToSU, ToDep);
 
   AvailableQueue->updateNode(SU);
   AvailableQueue->addNode(CopyFromSU);
@@ -1357,9 +1362,7 @@ SUnit *ScheduleDAGRRList::PickNodeToScheduleBottomUp() {
         if (!BtSU->isPending)
           AvailableQueue->remove(BtSU);
       }
-      AddPred(TrySU, SDep(BtSU, SDep::Order, /*Latency=*/1,
-                          /*Reg=*/0, /*isNormalMemory=*/false,
-                          /*isMustAlias=*/false, /*isArtificial=*/true));
+      AddPred(TrySU, SDep(BtSU, SDep::Artificial));
 
       // If one or more successors has been unscheduled, then the current
       // node is no longer avaialable. Schedule a successor that's now
@@ -1411,20 +1414,14 @@ SUnit *ScheduleDAGRRList::PickNodeToScheduleBottomUp() {
       InsertCopiesAndMoveSuccs(LRDef, Reg, DestRC, RC, Copies);
       DEBUG(dbgs() << "    Adding an edge from SU #" << TrySU->NodeNum
             << " to SU #" << Copies.front()->NodeNum << "\n");
-      AddPred(TrySU, SDep(Copies.front(), SDep::Order, /*Latency=*/1,
-                          /*Reg=*/0, /*isNormalMemory=*/false,
-                          /*isMustAlias=*/false,
-                          /*isArtificial=*/true));
+      AddPred(TrySU, SDep(Copies.front(), SDep::Artificial));
       NewDef = Copies.back();
     }
 
     DEBUG(dbgs() << "    Adding an edge from SU #" << NewDef->NodeNum
           << " to SU #" << TrySU->NodeNum << "\n");
     LiveRegDefs[Reg] = NewDef;
-    AddPred(NewDef, SDep(TrySU, SDep::Order, /*Latency=*/1,
-                         /*Reg=*/0, /*isNormalMemory=*/false,
-                         /*isMustAlias=*/false,
-                         /*isArtificial=*/true));
+    AddPred(NewDef, SDep(TrySU, SDep::Artificial));
     TrySU->isAvailable = false;
     CurSU = NewDef;
   }
@@ -1756,6 +1753,7 @@ public:
     return V;
   }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   void dump(ScheduleDAG *DAG) const {
     // Emulate pop() without clobbering NodeQueueIds.
     std::vector<SUnit*> DumpQueue = Queue;
@@ -1766,6 +1764,7 @@ public:
       SU->dump(DAG);
     }
   }
+#endif
 };
 
 typedef RegReductionPriorityQueue<bu_ls_rr_sort>
@@ -1893,6 +1892,7 @@ unsigned RegReductionPQBase::getNodePriority(const SUnit *SU) const {
 //===----------------------------------------------------------------------===//
 
 void RegReductionPQBase::dumpRegPressure() const {
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   for (TargetRegisterInfo::regclass_iterator I = TRI->regclass_begin(),
          E = TRI->regclass_end(); I != E; ++I) {
     const TargetRegisterClass *RC = *I;
@@ -1902,6 +1902,7 @@ void RegReductionPQBase::dumpRegPressure() const {
     DEBUG(dbgs() << RC->getName() << ": " << RP << " / " << RegLimit[Id]
           << '\n');
   }
+#endif
 }
 
 bool RegReductionPQBase::HighRegPressure(const SUnit *SU) const {
@@ -2930,10 +2931,7 @@ void RegReductionPQBase::AddPseudoTwoAddrDeps() {
             !scheduleDAG->IsReachable(SuccSU, SU)) {
           DEBUG(dbgs() << "    Adding a pseudo-two-addr edge from SU #"
                        << SU->NodeNum << " to SU #" << SuccSU->NodeNum << "\n");
-          scheduleDAG->AddPred(SU, SDep(SuccSU, SDep::Order, /*Latency=*/0,
-                                        /*Reg=*/0, /*isNormalMemory=*/false,
-                                        /*isMustAlias=*/false,
-                                        /*isArtificial=*/true));
+          scheduleDAG->AddPred(SU, SDep(SuccSU, SDep::Artificial));
         }
       }
     }
diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
index 748668c..a197fcb 100644
--- a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
+++ b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp
@@ -485,14 +485,15 @@ void ScheduleDAGSDNodes::AddSchedEdges() {
         if(isChain && OpN->getOpcode() == ISD::TokenFactor)
           OpLatency = 0;
 
-        const SDep &dep = SDep(OpSU, isChain ? SDep::Order : SDep::Data,
-                               OpLatency, PhysReg);
+        SDep Dep = isChain ? SDep(OpSU, SDep::Barrier)
+          : SDep(OpSU, SDep::Data, PhysReg);
+        Dep.setLatency(OpLatency);
         if (!isChain && !UnitLatencies) {
-          computeOperandLatency(OpN, N, i, const_cast<SDep &>(dep));
-          ST.adjustSchedDependency(OpSU, SU, const_cast<SDep &>(dep));
+          computeOperandLatency(OpN, N, i, Dep);
+          ST.adjustSchedDependency(OpSU, SU, Dep);
         }
 
-        if (!SU->addPred(dep) && !dep.isCtrl() && OpSU->NumRegDefsLeft > 1) {
+        if (!SU->addPred(Dep) && !Dep.isCtrl() && OpSU->NumRegDefsLeft > 1) {
           // Multiple register uses are combined in the same SUnit. For example,
           // we could have a set of glued nodes with all their defs consumed by
           // another set of glued nodes. Register pressure tracking sees this as
@@ -643,6 +644,7 @@ void ScheduleDAGSDNodes::computeOperandLatency(SDNode *Def, SDNode *Use,
 }
 
 void ScheduleDAGSDNodes::dumpNode(const SUnit *SU) const {
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   if (!SU->getNode()) {
     dbgs() << "PHYS REG COPY\n";
     return;
@@ -659,8 +661,10 @@ void ScheduleDAGSDNodes::dumpNode(const SUnit *SU) const {
     dbgs() << "\n";
     GluedNodes.pop_back();
   }
+#endif
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void ScheduleDAGSDNodes::dumpSchedule() const {
   for (unsigned i = 0, e = Sequence.size(); i != e; i++) {
     if (SUnit *SU = Sequence[i])
@@ -669,6 +673,7 @@ void ScheduleDAGSDNodes::dumpSchedule() const {
       dbgs() << "**** NOOP ****\n";
   }
 }
+#endif
 
 #ifndef NDEBUG
 /// VerifyScheduledSequence - Verify that all SUnits were scheduled and that
@@ -827,8 +832,7 @@ EmitSchedule(MachineBasicBlock::iterator &InsertPos) {
     }
 
     SmallVector<SDNode *, 4> GluedNodes;
-    for (SDNode *N = SU->getNode()->getGluedNode(); N;
-         N = N->getGluedNode())
+    for (SDNode *N = SU->getNode()->getGluedNode(); N; N = N->getGluedNode())
       GluedNodes.push_back(N);
     while (!GluedNodes.empty()) {
       SDNode *N = GluedNodes.back();
diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h
index 84e41fc..907356f 100644
--- a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h
+++ b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h
@@ -114,7 +114,8 @@ namespace llvm {
     /// EmitSchedule - Insert MachineInstrs into the MachineBasicBlock
     /// according to the order specified in Sequence.
     ///
-    MachineBasicBlock *EmitSchedule(MachineBasicBlock::iterator &InsertPos);
+    virtual MachineBasicBlock*
+    EmitSchedule(MachineBasicBlock::iterator &InsertPos);
 
     virtual void dumpNode(const SUnit *SU) const;
 
@@ -158,6 +159,12 @@ namespace llvm {
       void InitNodeNumDefs();
     };
 
+  protected:
+    /// ForceUnitLatencies - Return true if all scheduling edges should be given
+    /// a latency value of one.  The default is to return false; schedulers may
+    /// override this as needed.
+    virtual bool forceUnitLatencies() const { return false; }
+
   private:
     /// ClusterNeighboringLoads - Cluster loads from "near" addresses into
     /// combined SUnits.
diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp
index c851291..30f03ac 100644
--- a/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp
+++ b/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp
@@ -25,7 +25,7 @@
 #include "llvm/CodeGen/SchedulerRegistry.h"
 #include "llvm/CodeGen/SelectionDAGISel.h"
 #include "llvm/Target/TargetRegisterInfo.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetInstrInfo.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index f4fe892..f000ce3 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -29,7 +29,7 @@
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/Target/TargetRegisterInfo.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetLowering.h"
 #include "llvm/Target/TargetSelectionDAGInfo.h"
 #include "llvm/Target/TargetOptions.h"
@@ -91,11 +91,6 @@ bool ConstantFPSDNode::isValueValidForType(EVT VT,
                                            const APFloat& Val) {
   assert(VT.isFloatingPoint() && "Can only convert between FP types");
 
-  // PPC long double cannot be converted to any other type.
-  if (VT == MVT::ppcf128 ||
-      &Val.getSemantics() == &APFloat::PPCDoubleDouble)
-    return false;
-
   // convert modifies in place, so make a copy.
   APFloat Val2 = APFloat(Val);
   bool losesInfo;
@@ -136,13 +131,11 @@ bool ISD::isBuildVectorAllOnes(const SDNode *N) {
   // constants are.
   SDValue NotZero = N->getOperand(i);
   unsigned EltSize = N->getValueType(0).getVectorElementType().getSizeInBits();
-  if (isa<ConstantSDNode>(NotZero)) {
-    if (cast<ConstantSDNode>(NotZero)->getAPIntValue().countTrailingOnes() <
-        EltSize)
+  if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(NotZero)) {
+    if (CN->getAPIntValue().countTrailingOnes() < EltSize)
       return false;
-  } else if (isa<ConstantFPSDNode>(NotZero)) {
-    if (cast<ConstantFPSDNode>(NotZero)->getValueAPF()
-              .bitcastToAPInt().countTrailingOnes() < EltSize)
+  } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(NotZero)) {
+    if (CFPN->getValueAPF().bitcastToAPInt().countTrailingOnes() < EltSize)
       return false;
   } else
     return false;
@@ -179,11 +172,11 @@ bool ISD::isBuildVectorAllZeros(const SDNode *N) {
   // Do not accept build_vectors that aren't all constants or which have non-0
   // elements.
   SDValue Zero = N->getOperand(i);
-  if (isa<ConstantSDNode>(Zero)) {
-    if (!cast<ConstantSDNode>(Zero)->isNullValue())
+  if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Zero)) {
+    if (!CN->isNullValue())
       return false;
-  } else if (isa<ConstantFPSDNode>(Zero)) {
-    if (!cast<ConstantFPSDNode>(Zero)->getValueAPF().isPosZero())
+  } else if (ConstantFPSDNode *CFPN = dyn_cast<ConstantFPSDNode>(Zero)) {
+    if (!CFPN->getValueAPF().isPosZero())
       return false;
   } else
     return false;
@@ -494,8 +487,10 @@ static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) {
   }
   case ISD::TargetBlockAddress:
   case ISD::BlockAddress: {
-    ID.AddPointer(cast<BlockAddressSDNode>(N)->getBlockAddress());
-    ID.AddInteger(cast<BlockAddressSDNode>(N)->getTargetFlags());
+    const BlockAddressSDNode *BA = cast<BlockAddressSDNode>(N);
+    ID.AddPointer(BA->getBlockAddress());
+    ID.AddInteger(BA->getOffset());
+    ID.AddInteger(BA->getTargetFlags());
     break;
   }
   } // end switch (N->getOpcode())
@@ -883,7 +878,7 @@ unsigned SelectionDAG::getEVTAlignment(EVT VT) const {
                    PointerType::get(Type::getInt8Ty(*getContext()), 0) :
                    VT.getTypeForEVT(*getContext());
 
-  return TLI.getTargetData()->getABITypeAlignment(Ty);
+  return TLI.getDataLayout()->getABITypeAlignment(Ty);
 }
 
 // EntryNode could meaningfully have debug info if we can find it...
@@ -1097,10 +1092,9 @@ SDValue SelectionDAG::getGlobalAddress(const GlobalValue *GV, DebugLoc DL,
          "Cannot set target flags on target-independent globals");
 
   // Truncate (with sign-extension) the offset value to the pointer size.
-  EVT PTy = TLI.getPointerTy();
-  unsigned BitWidth = PTy.getSizeInBits();
+  unsigned BitWidth = TLI.getPointerTy().getSizeInBits();
   if (BitWidth < 64)
-    Offset = (Offset << (64 - BitWidth) >> (64 - BitWidth));
+    Offset = SignExtend64(Offset, BitWidth);
 
   const GlobalVariable *GVar = dyn_cast<GlobalVariable>(GV);
   if (!GVar) {
@@ -1174,7 +1168,7 @@ SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT,
   assert((TargetFlags == 0 || isTarget) &&
          "Cannot set target flags on target-independent globals");
   if (Alignment == 0)
-    Alignment = TLI.getTargetData()->getPrefTypeAlignment(C->getType());
+    Alignment = TLI.getDataLayout()->getPrefTypeAlignment(C->getType());
   unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
   FoldingSetNodeID ID;
   AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0);
@@ -1201,7 +1195,7 @@ SDValue SelectionDAG::getConstantPool(MachineConstantPoolValue *C, EVT VT,
   assert((TargetFlags == 0 || isTarget) &&
          "Cannot set target flags on target-independent globals");
   if (Alignment == 0)
-    Alignment = TLI.getTargetData()->getPrefTypeAlignment(C->getType());
+    Alignment = TLI.getDataLayout()->getPrefTypeAlignment(C->getType());
   unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool;
   FoldingSetNodeID ID;
   AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0);
@@ -1471,6 +1465,7 @@ SDValue SelectionDAG::getEHLabel(DebugLoc dl, SDValue Root, MCSymbol *Label) {
 
 
 SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
+                                      int64_t Offset,
                                       bool isTarget,
                                       unsigned char TargetFlags) {
   unsigned Opc = isTarget ? ISD::TargetBlockAddress : ISD::BlockAddress;
@@ -1478,12 +1473,14 @@ SDValue SelectionDAG::getBlockAddress(const BlockAddress *BA, EVT VT,
   FoldingSetNodeID ID;
   AddNodeIDNode(ID, Opc, getVTList(VT), 0, 0);
   ID.AddPointer(BA);
+  ID.AddInteger(Offset);
   ID.AddInteger(TargetFlags);
   void *IP = 0;
   if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP))
     return SDValue(E, 0);
 
-  SDNode *N = new (NodeAllocator) BlockAddressSDNode(Opc, VT, BA, TargetFlags);
+  SDNode *N = new (NodeAllocator) BlockAddressSDNode(Opc, VT, BA, Offset,
+                                                     TargetFlags);
   CSEMap.InsertNode(N, IP);
   AllNodes.push_back(N);
   return SDValue(N, 0);
@@ -1542,7 +1539,7 @@ SDValue SelectionDAG::CreateStackTemporary(EVT VT, unsigned minAlign) {
   unsigned ByteSize = VT.getStoreSize();
   Type *Ty = VT.getTypeForEVT(*getContext());
   unsigned StackAlign =
-  std::max((unsigned)TLI.getTargetData()->getPrefTypeAlignment(Ty), minAlign);
+  std::max((unsigned)TLI.getDataLayout()->getPrefTypeAlignment(Ty), minAlign);
 
   int FrameIdx = FrameInfo->CreateStackObject(ByteSize, StackAlign, false);
   return getFrameIndex(FrameIdx, TLI.getPointerTy());
@@ -1555,7 +1552,7 @@ SDValue SelectionDAG::CreateStackTemporary(EVT VT1, EVT VT2) {
                             VT2.getStoreSizeInBits())/8;
   Type *Ty1 = VT1.getTypeForEVT(*getContext());
   Type *Ty2 = VT2.getTypeForEVT(*getContext());
-  const TargetData *TD = TLI.getTargetData();
+  const DataLayout *TD = TLI.getDataLayout();
   unsigned Align = std::max(TD->getPrefTypeAlignment(Ty1),
                             TD->getPrefTypeAlignment(Ty2));
 
@@ -1610,10 +1607,6 @@ SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1,
   }
   if (ConstantFPSDNode *N1C = dyn_cast<ConstantFPSDNode>(N1.getNode())) {
     if (ConstantFPSDNode *N2C = dyn_cast<ConstantFPSDNode>(N2.getNode())) {
-      // No compile time operations on this type yet.
-      if (N1C->getValueType(0) == MVT::ppcf128)
-        return SDValue();
-
       APFloat::cmpResult R = N1C->getValueAPF().compare(N2C->getValueAPF());
       switch (Cond) {
       default: break;
@@ -2445,8 +2438,6 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL,
       return getConstant(Val.zextOrTrunc(VT.getSizeInBits()), VT);
     case ISD::UINT_TO_FP:
     case ISD::SINT_TO_FP: {
-      // No compile time operations on ppcf128.
-      if (VT == MVT::ppcf128) break;
       APFloat apf(APInt::getNullValue(VT.getSizeInBits()));
       (void)apf.convertFromAPInt(Val,
                                  Opcode==ISD::SINT_TO_FP,
@@ -2455,9 +2446,9 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL,
     }
     case ISD::BITCAST:
       if (VT == MVT::f32 && C->getValueType(0) == MVT::i32)
-        return getConstantFP(Val.bitsToFloat(), VT);
+        return getConstantFP(APFloat(Val), VT);
       else if (VT == MVT::f64 && C->getValueType(0) == MVT::i64)
-        return getConstantFP(Val.bitsToDouble(), VT);
+        return getConstantFP(APFloat(Val), VT);
       break;
     case ISD::BSWAP:
       return getConstant(Val.byteSwap(), VT);
@@ -2475,61 +2466,59 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL,
   // Constant fold unary operations with a floating point constant operand.
   if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Operand.getNode())) {
     APFloat V = C->getValueAPF();    // make copy
-    if (VT != MVT::ppcf128 && Operand.getValueType() != MVT::ppcf128) {
-      switch (Opcode) {
-      case ISD::FNEG:
-        V.changeSign();
+    switch (Opcode) {
+    case ISD::FNEG:
+      V.changeSign();
+      return getConstantFP(V, VT);
+    case ISD::FABS:
+      V.clearSign();
+      return getConstantFP(V, VT);
+    case ISD::FCEIL: {
+      APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive);
+      if (fs == APFloat::opOK || fs == APFloat::opInexact)
         return getConstantFP(V, VT);
-      case ISD::FABS:
-        V.clearSign();
+      break;
+    }
+    case ISD::FTRUNC: {
+      APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero);
+      if (fs == APFloat::opOK || fs == APFloat::opInexact)
         return getConstantFP(V, VT);
-      case ISD::FCEIL: {
-        APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardPositive);
-        if (fs == APFloat::opOK || fs == APFloat::opInexact)
-          return getConstantFP(V, VT);
-        break;
-      }
-      case ISD::FTRUNC: {
-        APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardZero);
-        if (fs == APFloat::opOK || fs == APFloat::opInexact)
-          return getConstantFP(V, VT);
-        break;
-      }
-      case ISD::FFLOOR: {
-        APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative);
-        if (fs == APFloat::opOK || fs == APFloat::opInexact)
-          return getConstantFP(V, VT);
-        break;
-      }
-      case ISD::FP_EXTEND: {
-        bool ignored;
-        // This can return overflow, underflow, or inexact; we don't care.
-        // FIXME need to be more flexible about rounding mode.
-        (void)V.convert(*EVTToAPFloatSemantics(VT),
-                        APFloat::rmNearestTiesToEven, &ignored);
+      break;
+    }
+    case ISD::FFLOOR: {
+      APFloat::opStatus fs = V.roundToIntegral(APFloat::rmTowardNegative);
+      if (fs == APFloat::opOK || fs == APFloat::opInexact)
         return getConstantFP(V, VT);
-      }
-      case ISD::FP_TO_SINT:
-      case ISD::FP_TO_UINT: {
-        integerPart x[2];
-        bool ignored;
-        assert(integerPartWidth >= 64);
-        // FIXME need to be more flexible about rounding mode.
-        APFloat::opStatus s = V.convertToInteger(x, VT.getSizeInBits(),
-                              Opcode==ISD::FP_TO_SINT,
-                              APFloat::rmTowardZero, &ignored);
-        if (s==APFloat::opInvalidOp)     // inexact is OK, in fact usual
-          break;
-        APInt api(VT.getSizeInBits(), x);
-        return getConstant(api, VT);
-      }
-      case ISD::BITCAST:
-        if (VT == MVT::i32 && C->getValueType(0) == MVT::f32)
-          return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), VT);
-        else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64)
-          return getConstant(V.bitcastToAPInt().getZExtValue(), VT);
+      break;
+    }
+    case ISD::FP_EXTEND: {
+      bool ignored;
+      // This can return overflow, underflow, or inexact; we don't care.
+      // FIXME need to be more flexible about rounding mode.
+      (void)V.convert(*EVTToAPFloatSemantics(VT),
+                      APFloat::rmNearestTiesToEven, &ignored);
+      return getConstantFP(V, VT);
+    }
+    case ISD::FP_TO_SINT:
+    case ISD::FP_TO_UINT: {
+      integerPart x[2];
+      bool ignored;
+      assert(integerPartWidth >= 64);
+      // FIXME need to be more flexible about rounding mode.
+      APFloat::opStatus s = V.convertToInteger(x, VT.getSizeInBits(),
+                            Opcode==ISD::FP_TO_SINT,
+                            APFloat::rmTowardZero, &ignored);
+      if (s==APFloat::opInvalidOp)     // inexact is OK, in fact usual
         break;
-      }
+      APInt api(VT.getSizeInBits(), x);
+      return getConstant(api, VT);
+    }
+    case ISD::BITCAST:
+      if (VT == MVT::i32 && C->getValueType(0) == MVT::f32)
+        return getConstant((uint32_t)V.bitcastToAPInt().getZExtValue(), VT);
+      else if (VT == MVT::i64 && C->getValueType(0) == MVT::f64)
+        return getConstant(V.bitcastToAPInt().getZExtValue(), VT);
+      break;
     }
   }
 
@@ -2817,6 +2806,24 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT,
         if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N2))
           if (CFP->getValueAPF().isZero())
             return N1;
+      } else if (Opcode == ISD::FMUL) {
+        ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(N1);
+        SDValue V = N2;
+
+        // If the first operand isn't the constant, try the second
+        if (!CFP) {
+          CFP = dyn_cast<ConstantFPSDNode>(N2);
+          V = N1;
+        }
+
+        if (CFP) {
+          // 0*x --> 0
+          if (CFP->isZero())
+            return SDValue(CFP,0);
+          // 1*x --> x
+          if (CFP->isExactlyValue(1.0))
+            return V;
+        }
       }
     }
     assert(VT.isFloatingPoint() && "This operator only applies to FP types!");
@@ -2935,17 +2942,13 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT,
     // expanding large vector constants.
     if (N2C && N1.getOpcode() == ISD::BUILD_VECTOR) {
       SDValue Elt = N1.getOperand(N2C->getZExtValue());
-      EVT VEltTy = N1.getValueType().getVectorElementType();
-      if (Elt.getValueType() != VEltTy) {
+
+      if (VT != Elt.getValueType())
         // If the vector element type is not legal, the BUILD_VECTOR operands
-        // are promoted and implicitly truncated.  Make that explicit here.
-        Elt = getNode(ISD::TRUNCATE, DL, VEltTy, Elt);
-      }
-      if (VT != VEltTy) {
-        // If the vector element type is not legal, the EXTRACT_VECTOR_ELT
-        // result is implicitly extended.
-        Elt = getNode(ISD::ANY_EXTEND, DL, VT, Elt);
-      }
+        // are promoted and implicitly truncated, and the result implicitly
+        // extended. Make that explicit here.
+        Elt = getAnyExtOrTrunc(Elt, DL, VT);
+
       return Elt;
     }
 
@@ -3036,7 +3039,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, DebugLoc DL, EVT VT,
       // Cannonicalize constant to RHS if commutative
       std::swap(N1CFP, N2CFP);
       std::swap(N1, N2);
-    } else if (N2CFP && VT != MVT::ppcf128) {
+    } else if (N2CFP) {
       APFloat V1 = N1CFP->getValueAPF(), V2 = N2CFP->getValueAPF();
       APFloat::opStatus s;
       switch (Opcode) {
@@ -3435,7 +3438,7 @@ static bool FindOptimalMemOpLowering(std::vector<EVT> &MemOps,
                                    DAG.getMachineFunction());
 
   if (VT == MVT::Other) {
-    if (DstAlign >= TLI.getTargetData()->getPointerPrefAlignment() ||
+    if (DstAlign >= TLI.getDataLayout()->getPointerPrefAlignment() ||
         TLI.allowsUnalignedMemoryAccesses(VT)) {
       VT = TLI.getPointerTy();
     } else {
@@ -3503,7 +3506,9 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, DebugLoc dl,
   bool DstAlignCanChange = false;
   MachineFunction &MF = DAG.getMachineFunction();
   MachineFrameInfo *MFI = MF.getFrameInfo();
-  bool OptSize = MF.getFunction()->hasFnAttr(Attribute::OptimizeForSize);
+  bool OptSize =
+    MF.getFunction()->getFnAttributes().
+      hasAttribute(Attributes::OptimizeForSize);
   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
   if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
     DstAlignCanChange = true;
@@ -3523,7 +3528,7 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, DebugLoc dl,
 
   if (DstAlignCanChange) {
     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
-    unsigned NewAlign = (unsigned) TLI.getTargetData()->getABITypeAlignment(Ty);
+    unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
     if (NewAlign > Align) {
       // Give the stack frame object a larger alignment if needed.
       if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
@@ -3596,7 +3601,8 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, DebugLoc dl,
   bool DstAlignCanChange = false;
   MachineFunction &MF = DAG.getMachineFunction();
   MachineFrameInfo *MFI = MF.getFrameInfo();
-  bool OptSize = MF.getFunction()->hasFnAttr(Attribute::OptimizeForSize);
+  bool OptSize = MF.getFunction()->getFnAttributes().
+    hasAttribute(Attributes::OptimizeForSize);
   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
   if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
     DstAlignCanChange = true;
@@ -3612,7 +3618,7 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, DebugLoc dl,
 
   if (DstAlignCanChange) {
     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
-    unsigned NewAlign = (unsigned) TLI.getTargetData()->getABITypeAlignment(Ty);
+    unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
     if (NewAlign > Align) {
       // Give the stack frame object a larger alignment if needed.
       if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
@@ -3674,7 +3680,8 @@ static SDValue getMemsetStores(SelectionDAG &DAG, DebugLoc dl,
   bool DstAlignCanChange = false;
   MachineFunction &MF = DAG.getMachineFunction();
   MachineFrameInfo *MFI = MF.getFrameInfo();
-  bool OptSize = MF.getFunction()->hasFnAttr(Attribute::OptimizeForSize);
+  bool OptSize = MF.getFunction()->getFnAttributes().
+    hasAttribute(Attributes::OptimizeForSize);
   FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Dst);
   if (FI && !MFI->isFixedObjectIndex(FI->getIndex()))
     DstAlignCanChange = true;
@@ -3687,7 +3694,7 @@ static SDValue getMemsetStores(SelectionDAG &DAG, DebugLoc dl,
 
   if (DstAlignCanChange) {
     Type *Ty = MemOps[0].getTypeForEVT(*DAG.getContext());
-    unsigned NewAlign = (unsigned) TLI.getTargetData()->getABITypeAlignment(Ty);
+    unsigned NewAlign = (unsigned) TLI.getDataLayout()->getABITypeAlignment(Ty);
     if (NewAlign > Align) {
       // Give the stack frame object a larger alignment if needed.
       if (MFI->getObjectAlignment(FI->getIndex()) < NewAlign)
@@ -3781,7 +3788,7 @@ SDValue SelectionDAG::getMemcpy(SDValue Chain, DebugLoc dl, SDValue Dst,
   // Emit a library call.
   TargetLowering::ArgListTy Args;
   TargetLowering::ArgListEntry Entry;
-  Entry.Ty = TLI.getTargetData()->getIntPtrType(*getContext());
+  Entry.Ty = TLI.getDataLayout()->getIntPtrType(*getContext());
   Entry.Node = Dst; Args.push_back(Entry);
   Entry.Node = Src; Args.push_back(Entry);
   Entry.Node = Size; Args.push_back(Entry);
@@ -3836,7 +3843,7 @@ SDValue SelectionDAG::getMemmove(SDValue Chain, DebugLoc dl, SDValue Dst,
   // Emit a library call.
   TargetLowering::ArgListTy Args;
   TargetLowering::ArgListEntry Entry;
-  Entry.Ty = TLI.getTargetData()->getIntPtrType(*getContext());
+  Entry.Ty = TLI.getDataLayout()->getIntPtrType(*getContext());
   Entry.Node = Dst; Args.push_back(Entry);
   Entry.Node = Src; Args.push_back(Entry);
   Entry.Node = Size; Args.push_back(Entry);
@@ -3885,7 +3892,7 @@ SDValue SelectionDAG::getMemset(SDValue Chain, DebugLoc dl, SDValue Dst,
     return Result;
 
   // Emit a library call.
-  Type *IntPtrTy = TLI.getTargetData()->getIntPtrType(*getContext());
+  Type *IntPtrTy = TLI.getDataLayout()->getIntPtrType(*getContext());
   TargetLowering::ArgListTy Args;
   TargetLowering::ArgListEntry Entry;
   Entry.Node = Dst; Entry.Ty = IntPtrTy;
@@ -3923,17 +3930,21 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
                                 SDValue Swp, MachinePointerInfo PtrInfo,
                                 unsigned Alignment,
                                 AtomicOrdering Ordering,
-                                SynchronizationScope SynchScope) {                                
+                                SynchronizationScope SynchScope) {
   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
     Alignment = getEVTAlignment(MemVT);
 
   MachineFunction &MF = getMachineFunction();
-  unsigned Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore;
 
+  // All atomics are load and store, except for ATMOIC_LOAD and ATOMIC_STORE.
   // For now, atomics are considered to be volatile always.
   // FIXME: Volatile isn't really correct; we should keep track of atomic
   // orderings in the memoperand.
-  Flags |= MachineMemOperand::MOVolatile;
+  unsigned Flags = MachineMemOperand::MOVolatile;
+  if (Opcode != ISD::ATOMIC_STORE)
+    Flags |= MachineMemOperand::MOLoad;
+  if (Opcode != ISD::ATOMIC_LOAD)
+    Flags |= MachineMemOperand::MOStore;
 
   MachineMemOperand *MMO =
     MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment);
@@ -3983,17 +3994,17 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
     Alignment = getEVTAlignment(MemVT);
 
   MachineFunction &MF = getMachineFunction();
-  // A monotonic store does not load; a release store "loads" in the sense
-  // that other stores cannot be sunk past it.
+  // An atomic store does not load. An atomic load does not store.
   // (An atomicrmw obviously both loads and stores.)
-  unsigned Flags = MachineMemOperand::MOStore;
-  if (Opcode != ISD::ATOMIC_STORE || Ordering > Monotonic)
-    Flags |= MachineMemOperand::MOLoad;
-
-  // For now, atomics are considered to be volatile always.
+  // For now, atomics are considered to be volatile always, and they are
+  // chained as such.
   // FIXME: Volatile isn't really correct; we should keep track of atomic
   // orderings in the memoperand.
-  Flags |= MachineMemOperand::MOVolatile;
+  unsigned Flags = MachineMemOperand::MOVolatile;
+  if (Opcode != ISD::ATOMIC_STORE)
+    Flags |= MachineMemOperand::MOLoad;
+  if (Opcode != ISD::ATOMIC_LOAD)
+    Flags |= MachineMemOperand::MOStore;
 
   MachineMemOperand *MMO =
     MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags,
@@ -4056,16 +4067,17 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
     Alignment = getEVTAlignment(MemVT);
 
   MachineFunction &MF = getMachineFunction();
-  // A monotonic load does not store; an acquire load "stores" in the sense
-  // that other loads cannot be hoisted past it.
-  unsigned Flags = MachineMemOperand::MOLoad;
-  if (Ordering > Monotonic)
-    Flags |= MachineMemOperand::MOStore;
-
-  // For now, atomics are considered to be volatile always.
+  // An atomic store does not load. An atomic load does not store.
+  // (An atomicrmw obviously both loads and stores.)
+  // For now, atomics are considered to be volatile always, and they are
+  // chained as such.
   // FIXME: Volatile isn't really correct; we should keep track of atomic
   // orderings in the memoperand.
-  Flags |= MachineMemOperand::MOVolatile;
+  unsigned Flags = MachineMemOperand::MOVolatile;
+  if (Opcode != ISD::ATOMIC_STORE)
+    Flags |= MachineMemOperand::MOLoad;
+  if (Opcode != ISD::ATOMIC_LOAD)
+    Flags |= MachineMemOperand::MOStore;
 
   MachineMemOperand *MMO =
     MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags,
@@ -4157,6 +4169,8 @@ SelectionDAG::getMemIntrinsicNode(unsigned Opcode, DebugLoc dl, SDVTList VTList,
   assert((Opcode == ISD::INTRINSIC_VOID ||
           Opcode == ISD::INTRINSIC_W_CHAIN ||
           Opcode == ISD::PREFETCH ||
+          Opcode == ISD::LIFETIME_START ||
+          Opcode == ISD::LIFETIME_END ||
           (Opcode <= INT_MAX &&
            (int)Opcode >= ISD::FIRST_TARGET_MEMORY_OPCODE)) &&
          "Opcode is not a memory-accessing opcode!");
@@ -4226,7 +4240,7 @@ SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
                       bool isVolatile, bool isNonTemporal, bool isInvariant,
                       unsigned Alignment, const MDNode *TBAAInfo,
                       const MDNode *Ranges) {
-  assert(Chain.getValueType() == MVT::Other && 
+  assert(Chain.getValueType() == MVT::Other &&
         "Invalid chain type");
   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
     Alignment = getEVTAlignment(VT);
@@ -4284,7 +4298,7 @@ SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
   AddNodeIDNode(ID, ISD::LOAD, VTs, Ops, 3);
   ID.AddInteger(MemVT.getRawBits());
   ID.AddInteger(encodeMemSDNodeFlags(ExtType, AM, MMO->isVolatile(),
-                                     MMO->isNonTemporal(), 
+                                     MMO->isNonTemporal(),
                                      MMO->isInvariant()));
   ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
   void *IP = 0;
@@ -4303,7 +4317,7 @@ SDValue SelectionDAG::getLoad(EVT VT, DebugLoc dl,
                               SDValue Chain, SDValue Ptr,
                               MachinePointerInfo PtrInfo,
                               bool isVolatile, bool isNonTemporal,
-                              bool isInvariant, unsigned Alignment, 
+                              bool isInvariant, unsigned Alignment,
                               const MDNode *TBAAInfo,
                               const MDNode *Ranges) {
   SDValue Undef = getUNDEF(Ptr.getValueType());
@@ -4332,7 +4346,7 @@ SelectionDAG::getIndexedLoad(SDValue OrigLoad, DebugLoc dl, SDValue Base,
          "Load is already a indexed load!");
   return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl,
                  LD->getChain(), Base, Offset, LD->getPointerInfo(),
-                 LD->getMemoryVT(), LD->isVolatile(), LD->isNonTemporal(), 
+                 LD->getMemoryVT(), LD->isVolatile(), LD->isNonTemporal(),
                  false, LD->getAlignment());
 }
 
@@ -4340,7 +4354,7 @@ SDValue SelectionDAG::getStore(SDValue Chain, DebugLoc dl, SDValue Val,
                                SDValue Ptr, MachinePointerInfo PtrInfo,
                                bool isVolatile, bool isNonTemporal,
                                unsigned Alignment, const MDNode *TBAAInfo) {
-  assert(Chain.getValueType() == MVT::Other && 
+  assert(Chain.getValueType() == MVT::Other &&
         "Invalid chain type");
   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
     Alignment = getEVTAlignment(Val.getValueType());
@@ -4365,7 +4379,7 @@ SDValue SelectionDAG::getStore(SDValue Chain, DebugLoc dl, SDValue Val,
 
 SDValue SelectionDAG::getStore(SDValue Chain, DebugLoc dl, SDValue Val,
                                SDValue Ptr, MachineMemOperand *MMO) {
-  assert(Chain.getValueType() == MVT::Other && 
+  assert(Chain.getValueType() == MVT::Other &&
         "Invalid chain type");
   EVT VT = Val.getValueType();
   SDVTList VTs = getVTList(MVT::Other);
@@ -4394,7 +4408,7 @@ SDValue SelectionDAG::getTruncStore(SDValue Chain, DebugLoc dl, SDValue Val,
                                     EVT SVT,bool isVolatile, bool isNonTemporal,
                                     unsigned Alignment,
                                     const MDNode *TBAAInfo) {
-  assert(Chain.getValueType() == MVT::Other && 
+  assert(Chain.getValueType() == MVT::Other &&
         "Invalid chain type");
   if (Alignment == 0)  // Ensure that codegen never sees alignment 0
     Alignment = getEVTAlignment(SVT);
@@ -4421,7 +4435,7 @@ SDValue SelectionDAG::getTruncStore(SDValue Chain, DebugLoc dl, SDValue Val,
                                     MachineMemOperand *MMO) {
   EVT VT = Val.getValueType();
 
-  assert(Chain.getValueType() == MVT::Other && 
+  assert(Chain.getValueType() == MVT::Other &&
         "Invalid chain type");
   if (VT == SVT)
     return getStore(Chain, dl, Val, Ptr, MMO);
@@ -6074,7 +6088,7 @@ unsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const {
     unsigned PtrWidth = TLI.getPointerTy().getSizeInBits();
     APInt KnownZero(PtrWidth, 0), KnownOne(PtrWidth, 0);
     llvm::ComputeMaskedBits(const_cast<GlobalValue*>(GV), KnownZero, KnownOne,
-                            TLI.getTargetData());
+                            TLI.getDataLayout());
     unsigned AlignBits = KnownZero.countTrailingOnes();
     unsigned Align = AlignBits ? 1 << std::min(31U, AlignBits) : 0;
     if (Align)
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index ba5bd79..3fbf7c2 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -19,6 +19,7 @@
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/ConstantFolding.h"
+#include "llvm/Analysis/ValueTracking.h"
 #include "llvm/Constants.h"
 #include "llvm/CallingConv.h"
 #include "llvm/DebugInfo.h"
@@ -43,7 +44,7 @@
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/SelectionDAG.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetFrameLowering.h"
 #include "llvm/Target/TargetInstrInfo.h"
 #include "llvm/Target/TargetIntrinsicInfo.h"
@@ -88,7 +89,7 @@ static const unsigned MaxParallelChains = 64;
 
 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, DebugLoc DL,
                                       const SDValue *Parts, unsigned NumParts,
-                                      EVT PartVT, EVT ValueVT);
+                                      EVT PartVT, EVT ValueVT, const Value *V);
 
 /// getCopyFromParts - Create a value that contains the specified legal parts
 /// combined into the value they represent.  If the parts combine to a type
@@ -98,9 +99,11 @@ static SDValue getCopyFromPartsVector(SelectionDAG &DAG, DebugLoc DL,
 static SDValue getCopyFromParts(SelectionDAG &DAG, DebugLoc DL,
                                 const SDValue *Parts,
                                 unsigned NumParts, EVT PartVT, EVT ValueVT,
+                                const Value *V,
                                 ISD::NodeType AssertOp = ISD::DELETED_NODE) {
   if (ValueVT.isVector())
-    return getCopyFromPartsVector(DAG, DL, Parts, NumParts, PartVT, ValueVT);
+    return getCopyFromPartsVector(DAG, DL, Parts, NumParts,
+                                  PartVT, ValueVT, V);
 
   assert(NumParts > 0 && "No parts to assemble!");
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
@@ -124,9 +127,9 @@ static SDValue getCopyFromParts(SelectionDAG &DAG, DebugLoc DL,
 
       if (RoundParts > 2) {
         Lo = getCopyFromParts(DAG, DL, Parts, RoundParts / 2,
-                              PartVT, HalfVT);
+                              PartVT, HalfVT, V);
         Hi = getCopyFromParts(DAG, DL, Parts + RoundParts / 2,
-                              RoundParts / 2, PartVT, HalfVT);
+                              RoundParts / 2, PartVT, HalfVT, V);
       } else {
         Lo = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[0]);
         Hi = DAG.getNode(ISD::BITCAST, DL, HalfVT, Parts[1]);
@@ -142,7 +145,7 @@ static SDValue getCopyFromParts(SelectionDAG &DAG, DebugLoc DL,
         unsigned OddParts = NumParts - RoundParts;
         EVT OddVT = EVT::getIntegerVT(*DAG.getContext(), OddParts * PartBits);
         Hi = getCopyFromParts(DAG, DL,
-                              Parts + RoundParts, OddParts, PartVT, OddVT);
+                              Parts + RoundParts, OddParts, PartVT, OddVT, V);
 
         // Combine the round and odd parts.
         Lo = Val;
@@ -171,7 +174,7 @@ static SDValue getCopyFromParts(SelectionDAG &DAG, DebugLoc DL,
       assert(ValueVT.isFloatingPoint() && PartVT.isInteger() &&
              !PartVT.isVector() && "Unexpected split");
       EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), ValueVT.getSizeInBits());
-      Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT);
+      Val = getCopyFromParts(DAG, DL, Parts, NumParts, PartVT, IntVT, V);
     }
   }
 
@@ -209,14 +212,14 @@ static SDValue getCopyFromParts(SelectionDAG &DAG, DebugLoc DL,
   llvm_unreachable("Unknown mismatch!");
 }
 
-/// getCopyFromParts - Create a value that contains the specified legal parts
-/// combined into the value they represent.  If the parts combine to a type
-/// larger then ValueVT then AssertOp can be used to specify whether the extra
-/// bits are known to be zero (ISD::AssertZext) or sign extended from ValueVT
-/// (ISD::AssertSext).
+/// getCopyFromPartsVector - Create a value that contains the specified legal
+/// parts combined into the value they represent.  If the parts combine to a
+/// type larger then ValueVT then AssertOp can be used to specify whether the
+/// extra bits are known to be zero (ISD::AssertZext) or sign extended from
+/// ValueVT (ISD::AssertSext).
 static SDValue getCopyFromPartsVector(SelectionDAG &DAG, DebugLoc DL,
                                       const SDValue *Parts, unsigned NumParts,
-                                      EVT PartVT, EVT ValueVT) {
+                                      EVT PartVT, EVT ValueVT, const Value *V) {
   assert(ValueVT.isVector() && "Not a vector value");
   assert(NumParts > 0 && "No parts to assemble!");
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
@@ -242,7 +245,7 @@ static SDValue getCopyFromPartsVector(SelectionDAG &DAG, DebugLoc DL,
       // as appropriate.
       for (unsigned i = 0; i != NumParts; ++i)
         Ops[i] = getCopyFromParts(DAG, DL, &Parts[i], 1,
-                                  PartVT, IntermediateVT);
+                                  PartVT, IntermediateVT, V);
     } else if (NumParts > 0) {
       // If the intermediate type was expanded, build the intermediate
       // operands from the parts.
@@ -251,7 +254,7 @@ static SDValue getCopyFromPartsVector(SelectionDAG &DAG, DebugLoc DL,
       unsigned Factor = NumParts / NumIntermediates;
       for (unsigned i = 0; i != NumIntermediates; ++i)
         Ops[i] = getCopyFromParts(DAG, DL, &Parts[i * Factor], Factor,
-                                  PartVT, IntermediateVT);
+                                  PartVT, IntermediateVT, V);
     }
 
     // Build a vector with BUILD_VECTOR or CONCAT_VECTORS from the
@@ -299,8 +302,19 @@ static SDValue getCopyFromPartsVector(SelectionDAG &DAG, DebugLoc DL,
     return DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
 
   // Handle cases such as i8 -> <1 x i1>
-  assert(ValueVT.getVectorNumElements() == 1 &&
-         "Only trivial scalar-to-vector conversions should get here!");
+  if (ValueVT.getVectorNumElements() != 1) {
+    LLVMContext &Ctx = *DAG.getContext();
+    Twine ErrMsg("non-trivial scalar-to-vector conversion");
+    if (const Instruction *I = dyn_cast_or_null<Instruction>(V)) {
+      if (const CallInst *CI = dyn_cast<CallInst>(I))
+        if (isa<InlineAsm>(CI->getCalledValue()))
+          ErrMsg = ErrMsg + ", possible invalid constraint for vector type";
+      Ctx.emitError(I, ErrMsg);
+    } else {
+      Ctx.emitError(ErrMsg);
+    }
+    report_fatal_error("Cannot handle scalar-to-vector conversion!");
+  }
 
   if (ValueVT.getVectorNumElements() == 1 &&
       ValueVT.getVectorElementType() != PartVT) {
@@ -312,25 +326,22 @@ static SDValue getCopyFromPartsVector(SelectionDAG &DAG, DebugLoc DL,
   return DAG.getNode(ISD::BUILD_VECTOR, DL, ValueVT, Val);
 }
 
-
-
-
 static void getCopyToPartsVector(SelectionDAG &DAG, DebugLoc dl,
                                  SDValue Val, SDValue *Parts, unsigned NumParts,
-                                 EVT PartVT);
+                                 EVT PartVT, const Value *V);
 
 /// getCopyToParts - Create a series of nodes that contain the specified value
 /// split into legal parts.  If the parts contain more bits than Val, then, for
 /// integers, ExtendKind can be used to specify how to generate the extra bits.
 static void getCopyToParts(SelectionDAG &DAG, DebugLoc DL,
                            SDValue Val, SDValue *Parts, unsigned NumParts,
-                           EVT PartVT,
+                           EVT PartVT, const Value *V,
                            ISD::NodeType ExtendKind = ISD::ANY_EXTEND) {
   EVT ValueVT = Val.getValueType();
 
   // Handle the vector case separately.
   if (ValueVT.isVector())
-    return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT);
+    return getCopyToPartsVector(DAG, DL, Val, Parts, NumParts, PartVT, V);
 
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   unsigned PartBits = PartVT.getSizeInBits();
@@ -382,7 +393,19 @@ static void getCopyToParts(SelectionDAG &DAG, DebugLoc DL,
          "Failed to tile the value with PartVT!");
 
   if (NumParts == 1) {
-    assert(PartVT == ValueVT && "Type conversion failed!");
+    if (PartVT != ValueVT) {
+      LLVMContext &Ctx = *DAG.getContext();
+      Twine ErrMsg("scalar-to-vector conversion failed");
+      if (const Instruction *I = dyn_cast_or_null<Instruction>(V)) {
+        if (const CallInst *CI = dyn_cast<CallInst>(I))
+          if (isa<InlineAsm>(CI->getCalledValue()))
+            ErrMsg = ErrMsg + ", possible invalid constraint for vector type";
+        Ctx.emitError(I, ErrMsg);
+      } else {
+        Ctx.emitError(ErrMsg);
+      }
+    }
+
     Parts[0] = Val;
     return;
   }
@@ -397,7 +420,7 @@ static void getCopyToParts(SelectionDAG &DAG, DebugLoc DL,
     unsigned OddParts = NumParts - RoundParts;
     SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val,
                                  DAG.getIntPtrConstant(RoundBits));
-    getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT);
+    getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V);
 
     if (TLI.isBigEndian())
       // The odd parts were reversed by getCopyToParts - unreverse them.
@@ -443,7 +466,7 @@ static void getCopyToParts(SelectionDAG &DAG, DebugLoc DL,
 /// value split into legal parts.
 static void getCopyToPartsVector(SelectionDAG &DAG, DebugLoc DL,
                                  SDValue Val, SDValue *Parts, unsigned NumParts,
-                                 EVT PartVT) {
+                                 EVT PartVT, const Value *V) {
   EVT ValueVT = Val.getValueType();
   assert(ValueVT.isVector() && "Not a vector");
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
@@ -529,7 +552,7 @@ static void getCopyToPartsVector(SelectionDAG &DAG, DebugLoc DL,
     // If the register was not expanded, promote or copy the value,
     // as appropriate.
     for (unsigned i = 0; i != NumParts; ++i)
-      getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT);
+      getCopyToParts(DAG, DL, Ops[i], &Parts[i], 1, PartVT, V);
   } else if (NumParts > 0) {
     // If the intermediate type was expanded, split each the value into
     // legal parts.
@@ -537,13 +560,10 @@ static void getCopyToPartsVector(SelectionDAG &DAG, DebugLoc DL,
            "Must expand into a divisible number of parts!");
     unsigned Factor = NumParts / NumIntermediates;
     for (unsigned i = 0; i != NumIntermediates; ++i)
-      getCopyToParts(DAG, DL, Ops[i], &Parts[i*Factor], Factor, PartVT);
+      getCopyToParts(DAG, DL, Ops[i], &Parts[i*Factor], Factor, PartVT, V);
   }
 }
 
-
-
-
 namespace {
   /// RegsForValue - This struct represents the registers (physical or virtual)
   /// that a particular set of values is assigned, and the type information
@@ -621,14 +641,15 @@ namespace {
     /// If the Flag pointer is NULL, no flag is used.
     SDValue getCopyFromRegs(SelectionDAG &DAG, FunctionLoweringInfo &FuncInfo,
                             DebugLoc dl,
-                            SDValue &Chain, SDValue *Flag) const;
+                            SDValue &Chain, SDValue *Flag,
+                            const Value *V = 0) const;
 
     /// getCopyToRegs - Emit a series of CopyToReg nodes that copies the
     /// specified value into the registers specified by this object.  This uses
     /// Chain/Flag as the input and updates them for the output Chain/Flag.
     /// If the Flag pointer is NULL, no flag is used.
     void getCopyToRegs(SDValue Val, SelectionDAG &DAG, DebugLoc dl,
-                       SDValue &Chain, SDValue *Flag) const;
+                       SDValue &Chain, SDValue *Flag, const Value *V) const;
 
     /// AddInlineAsmOperands - Add this value to the specified inlineasm node
     /// operand list.  This adds the code marker, matching input operand index
@@ -647,7 +668,8 @@ namespace {
 SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
                                       FunctionLoweringInfo &FuncInfo,
                                       DebugLoc dl,
-                                      SDValue &Chain, SDValue *Flag) const {
+                                      SDValue &Chain, SDValue *Flag,
+                                      const Value *V) const {
   // A Value with type {} or [0 x %t] needs no registers.
   if (ValueVTs.empty())
     return SDValue();
@@ -721,7 +743,7 @@ SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
     }
 
     Values[Value] = getCopyFromParts(DAG, dl, Parts.begin(),
-                                     NumRegs, RegisterVT, ValueVT);
+                                     NumRegs, RegisterVT, ValueVT, V);
     Part += NumRegs;
     Parts.clear();
   }
@@ -736,7 +758,8 @@ SDValue RegsForValue::getCopyFromRegs(SelectionDAG &DAG,
 /// Chain/Flag as the input and updates them for the output Chain/Flag.
 /// If the Flag pointer is NULL, no flag is used.
 void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, DebugLoc dl,
-                                 SDValue &Chain, SDValue *Flag) const {
+                                 SDValue &Chain, SDValue *Flag,
+                                 const Value *V) const {
   const TargetLowering &TLI = DAG.getTargetLoweringInfo();
 
   // Get the list of the values's legal parts.
@@ -748,7 +771,7 @@ void RegsForValue::getCopyToRegs(SDValue Val, SelectionDAG &DAG, DebugLoc dl,
     EVT RegisterVT = RegVTs[Value];
 
     getCopyToParts(DAG, dl, Val.getValue(Val.getResNo() + Value),
-                   &Parts[Part], NumParts, RegisterVT);
+                   &Parts[Part], NumParts, RegisterVT, V);
     Part += NumParts;
   }
 
@@ -824,7 +847,8 @@ void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis &aa,
   AA = &aa;
   GFI = gfi;
   LibInfo = li;
-  TD = DAG.getTarget().getTargetData();
+  TD = DAG.getTarget().getDataLayout();
+  Context = DAG.getContext();
   LPadToCallSiteMap.clear();
 }
 
@@ -992,7 +1016,7 @@ SDValue SelectionDAGBuilder::getValue(const Value *V) {
     unsigned InReg = It->second;
     RegsForValue RFV(*DAG.getContext(), TLI, InReg, V->getType());
     SDValue Chain = DAG.getEntryNode();
-    N = RFV.getCopyFromRegs(DAG, FuncInfo, getCurDebugLoc(), Chain, NULL);
+    N = RFV.getCopyFromRegs(DAG, FuncInfo, getCurDebugLoc(), Chain, NULL, V);
     resolveDanglingDebugInfo(V, N);
     return N;
   }
@@ -1147,7 +1171,7 @@ SDValue SelectionDAGBuilder::getValueImpl(const Value *V) {
     unsigned InReg = FuncInfo.InitializeRegForValue(Inst);
     RegsForValue RFV(*DAG.getContext(), TLI, InReg, Inst->getType());
     SDValue Chain = DAG.getEntryNode();
-    return RFV.getCopyFromRegs(DAG, FuncInfo, getCurDebugLoc(), Chain, NULL);
+    return RFV.getCopyFromRegs(DAG, FuncInfo, getCurDebugLoc(), Chain, NULL, V);
   }
 
   llvm_unreachable("Can't get register for value!");
@@ -1203,9 +1227,9 @@ void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
         ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
 
         const Function *F = I.getParent()->getParent();
-        if (F->paramHasAttr(0, Attribute::SExt))
+        if (F->getRetAttributes().hasAttribute(Attributes::SExt))
           ExtendKind = ISD::SIGN_EXTEND;
-        else if (F->paramHasAttr(0, Attribute::ZExt))
+        else if (F->getRetAttributes().hasAttribute(Attributes::ZExt))
           ExtendKind = ISD::ZERO_EXTEND;
 
         if (ExtendKind != ISD::ANY_EXTEND && VT.isInteger())
@@ -1216,11 +1240,11 @@ void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
         SmallVector<SDValue, 4> Parts(NumParts);
         getCopyToParts(DAG, getCurDebugLoc(),
                        SDValue(RetOp.getNode(), RetOp.getResNo() + j),
-                       &Parts[0], NumParts, PartVT, ExtendKind);
+                       &Parts[0], NumParts, PartVT, &I, ExtendKind);
 
         // 'inreg' on function refers to return value
         ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
-        if (F->paramHasAttr(0, Attribute::InReg))
+        if (F->getRetAttributes().hasAttribute(Attributes::InReg))
           Flags.setInReg();
 
         // Propagate extension type if any
@@ -1231,7 +1255,7 @@ void SelectionDAGBuilder::visitRet(const ReturnInst &I) {
 
         for (unsigned i = 0; i < NumParts; ++i) {
           Outs.push_back(ISD::OutputArg(Flags, Parts[i].getValueType(),
-                                        /*isfixed=*/true));
+                                        /*isfixed=*/true, 0, 0));
           OutVals.push_back(Parts[i]);
         }
       }
@@ -1601,7 +1625,10 @@ void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB,
 
   // Update successor info
   addSuccessorWithWeight(SwitchBB, CB.TrueBB, CB.TrueWeight);
-  addSuccessorWithWeight(SwitchBB, CB.FalseBB, CB.FalseWeight);
+  // TrueBB and FalseBB are always different unless the incoming IR is
+  // degenerate. This only happens when running llc on weird IR.
+  if (CB.TrueBB != CB.FalseBB)
+    addSuccessorWithWeight(SwitchBB, CB.FalseBB, CB.FalseWeight);
 
   // Set NextBlock to be the MBB immediately after the current one, if any.
   // This is used to avoid emitting unnecessary branches to the next block.
@@ -1762,6 +1789,7 @@ void SelectionDAGBuilder::visitBitTestHeader(BitTestBlock &B,
 /// visitBitTestCase - this function produces one "bit test"
 void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
                                            MachineBasicBlock* NextMBB,
+                                           uint32_t BranchWeightToNext,
                                            unsigned Reg,
                                            BitTestCase &B,
                                            MachineBasicBlock *SwitchBB) {
@@ -1799,8 +1827,10 @@ void SelectionDAGBuilder::visitBitTestCase(BitTestBlock &BB,
                        ISD::SETNE);
   }
 
-  addSuccessorWithWeight(SwitchBB, B.TargetBB);
-  addSuccessorWithWeight(SwitchBB, NextMBB);
+  // The branch weight from SwitchBB to B.TargetBB is B.ExtraWeight.
+  addSuccessorWithWeight(SwitchBB, B.TargetBB, B.ExtraWeight);
+  // The branch weight from SwitchBB to NextMBB is BranchWeightToNext.
+  addSuccessorWithWeight(SwitchBB, NextMBB, BranchWeightToNext);
 
   SDValue BrAnd = DAG.getNode(ISD::BRCOND, getCurDebugLoc(),
                               MVT::Other, getControlRoot(),
@@ -1923,6 +1953,7 @@ bool SelectionDAGBuilder::handleSmallSwitchRange(CaseRec& CR,
   if (++BBI != FuncInfo.MF->end())
     NextBlock = BBI;
 
+  BranchProbabilityInfo *BPI = FuncInfo.BPI;
   // If any two of the cases has the same destination, and if one value
   // is the same as the other, but has one bit unset that the other has set,
   // use bit manipulation to do two compares at once.  For example:
@@ -1956,8 +1987,12 @@ bool SelectionDAGBuilder::handleSmallSwitchRange(CaseRec& CR,
                                     ISD::SETEQ);
 
         // Update successor info.
-        addSuccessorWithWeight(SwitchBB, Small.BB);
-        addSuccessorWithWeight(SwitchBB, Default);
+        // Both Small and Big will jump to Small.BB, so we sum up the weights.
+        addSuccessorWithWeight(SwitchBB, Small.BB,
+                               Small.ExtraWeight + Big.ExtraWeight);
+        addSuccessorWithWeight(SwitchBB, Default,
+          // The default destination is the first successor in IR.
+          BPI ? BPI->getEdgeWeight(SwitchBB->getBasicBlock(), (unsigned)0) : 0);
 
         // Insert the true branch.
         SDValue BrCond = DAG.getNode(ISD::BRCOND, DL, MVT::Other,
@@ -1975,14 +2010,13 @@ bool SelectionDAGBuilder::handleSmallSwitchRange(CaseRec& CR,
   }
 
   // Order cases by weight so the most likely case will be checked first.
-  BranchProbabilityInfo *BPI = FuncInfo.BPI;
+  uint32_t UnhandledWeights = 0;
   if (BPI) {
     for (CaseItr I = CR.Range.first, IE = CR.Range.second; I != IE; ++I) {
-      uint32_t IWeight = BPI->getEdgeWeight(SwitchBB->getBasicBlock(),
-                                            I->BB->getBasicBlock());
+      uint32_t IWeight = I->ExtraWeight;
+      UnhandledWeights += IWeight;
       for (CaseItr J = CR.Range.first; J < I; ++J) {
-        uint32_t JWeight = BPI->getEdgeWeight(SwitchBB->getBasicBlock(),
-                                              J->BB->getBasicBlock());
+        uint32_t JWeight = J->ExtraWeight;
         if (IWeight > JWeight)
           std::swap(*I, *J);
       }
@@ -2031,10 +2065,12 @@ bool SelectionDAGBuilder::handleSmallSwitchRange(CaseRec& CR,
       LHS = I->Low; MHS = SV; RHS = I->High;
     }
 
-    uint32_t ExtraWeight = I->ExtraWeight;
+    // The false weight should be sum of all un-handled cases.
+    UnhandledWeights -= I->ExtraWeight;
     CaseBlock CB(CC, LHS, RHS, MHS, /* truebb */ I->BB, /* falsebb */ FallThrough,
                  /* me */ CurBlock,
-                 /* trueweight */ ExtraWeight / 2, /* falseweight */ ExtraWeight / 2);
+                 /* trueweight */ I->ExtraWeight,
+                 /* falseweight */ UnhandledWeights);
 
     // If emitting the first comparison, just call visitSwitchCase to emit the
     // code into the current block.  Otherwise, push the CaseBlock onto the
@@ -2079,7 +2115,7 @@ bool SelectionDAGBuilder::handleJTSwitchCase(CaseRec &CR,
   for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I)
     TSize += I->size();
 
-  if (!areJTsAllowed(TLI) || TSize.ult(4))
+  if (!areJTsAllowed(TLI) || TSize.ult(TLI.getMinimumJumpTableEntries()))
     return false;
 
   APInt Range = ComputeRange(First, Last);
@@ -2134,13 +2170,28 @@ bool SelectionDAGBuilder::handleJTSwitchCase(CaseRec &CR,
     }
   }
 
+  // Calculate weight for each unique destination in CR.
+  DenseMap<MachineBasicBlock*, uint32_t> DestWeights;
+  if (FuncInfo.BPI)
+    for (CaseItr I = CR.Range.first, E = CR.Range.second; I != E; ++I) {
+      DenseMap<MachineBasicBlock*, uint32_t>::iterator Itr =
+          DestWeights.find(I->BB);
+      if (Itr != DestWeights.end()) 
+        Itr->second += I->ExtraWeight;
+      else
+        DestWeights[I->BB] = I->ExtraWeight;
+    }
+
   // Update successor info. Add one edge to each unique successor.
   BitVector SuccsHandled(CR.CaseBB->getParent()->getNumBlockIDs());
   for (std::vector<MachineBasicBlock*>::iterator I = DestBBs.begin(),
          E = DestBBs.end(); I != E; ++I) {
     if (!SuccsHandled[(*I)->getNumber()]) {
       SuccsHandled[(*I)->getNumber()] = true;
-      addSuccessorWithWeight(JumpTableBB, *I);
+      DenseMap<MachineBasicBlock*, uint32_t>::iterator Itr =
+          DestWeights.find(*I);
+      addSuccessorWithWeight(JumpTableBB, *I,
+                             Itr != DestWeights.end() ? Itr->second : 0);
     }
   }
 
@@ -2371,7 +2422,7 @@ bool SelectionDAGBuilder::handleBitTestsSwitchCase(CaseRec& CR,
 
     if (i == count) {
       assert((count < 3) && "Too much destinations to test!");
-      CasesBits.push_back(CaseBits(0, Dest, 0));
+      CasesBits.push_back(CaseBits(0, Dest, 0, 0/*Weight*/));
       count++;
     }
 
@@ -2380,6 +2431,7 @@ bool SelectionDAGBuilder::handleBitTestsSwitchCase(CaseRec& CR,
 
     uint64_t lo = (lowValue - lowBound).getZExtValue();
     uint64_t hi = (highValue - lowBound).getZExtValue();
+    CasesBits[i].ExtraWeight += I->ExtraWeight;
 
     for (uint64_t j = lo; j <= hi; j++) {
       CasesBits[i].Mask |=  1ULL << j;
@@ -2407,7 +2459,7 @@ bool SelectionDAGBuilder::handleBitTestsSwitchCase(CaseRec& CR,
     CurMF->insert(BBI, CaseBB);
     BTC.push_back(BitTestCase(CasesBits[i].Mask,
                               CaseBB,
-                              CasesBits[i].BB));
+                              CasesBits[i].BB, CasesBits[i].ExtraWeight));
 
     // Put SV in a virtual register to make it available from the new blocks.
     ExportFromCurrentBlock(SV);
@@ -2435,30 +2487,25 @@ size_t SelectionDAGBuilder::Clusterify(CaseVector& Cases,
   
   Clusterifier TheClusterifier;
 
+  BranchProbabilityInfo *BPI = FuncInfo.BPI;
   // Start with "simple" cases
   for (SwitchInst::ConstCaseIt i = SI.case_begin(), e = SI.case_end();
        i != e; ++i) {
     const BasicBlock *SuccBB = i.getCaseSuccessor();
     MachineBasicBlock *SMBB = FuncInfo.MBBMap[SuccBB];
 
-    TheClusterifier.add(i.getCaseValueEx(), SMBB);
+    TheClusterifier.add(i.getCaseValueEx(), SMBB, 
+        BPI ? BPI->getEdgeWeight(SI.getParent(), i.getSuccessorIndex()) : 0);
   }
   
   TheClusterifier.optimize();
   
-  BranchProbabilityInfo *BPI = FuncInfo.BPI;
   size_t numCmps = 0;
   for (Clusterifier::RangeIterator i = TheClusterifier.begin(),
        e = TheClusterifier.end(); i != e; ++i, ++numCmps) {
     Clusterifier::Cluster &C = *i;
-    unsigned W = 0;
-    if (BPI) {
-      W = BPI->getEdgeWeight(SI.getParent(), C.second->getBasicBlock());
-      if (!W)
-        W = 16;
-      W *= C.first.Weight;
-      BPI->setEdgeWeight(SI.getParent(), C.second->getBasicBlock(), W);  
-    }
+    // Update edge weight for the cluster.
+    unsigned W = C.first.Weight;
 
     // FIXME: Currently work with ConstantInt based numbers.
     // Changing it to APInt based is a pretty heavy for this commit.
@@ -2540,9 +2587,10 @@ void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) {
     if (handleSmallSwitchRange(CR, WorkList, SV, Default, SwitchMBB))
       continue;
 
-    // If the switch has more than 5 blocks, and at least 40% dense, and the
+    // If the switch has more than N blocks, and is at least 40% dense, and the
     // target supports indirect branches, then emit a jump table rather than
     // lowering the switch to a binary tree of conditional branches.
+    // N defaults to 4 and is controlled via TLS.getMinimumJumpTableEntries().
     if (handleJTSwitchCase(CR, WorkList, SV, Default, SwitchMBB))
       continue;
 
@@ -2556,14 +2604,14 @@ void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) {
   MachineBasicBlock *IndirectBrMBB = FuncInfo.MBB;
 
   // Update machine-CFG edges with unique successors.
-  SmallVector<BasicBlock*, 32> succs;
-  succs.reserve(I.getNumSuccessors());
-  for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i)
-    succs.push_back(I.getSuccessor(i));
-  array_pod_sort(succs.begin(), succs.end());
-  succs.erase(std::unique(succs.begin(), succs.end()), succs.end());
-  for (unsigned i = 0, e = succs.size(); i != e; ++i) {
-    MachineBasicBlock *Succ = FuncInfo.MBBMap[succs[i]];
+  SmallSet<BasicBlock*, 32> Done;
+  for (unsigned i = 0, e = I.getNumSuccessors(); i != e; ++i) {
+    BasicBlock *BB = I.getSuccessor(i);
+    bool Inserted = Done.insert(BB);
+    if (!Inserted)
+        continue;
+
+    MachineBasicBlock *Succ = FuncInfo.MBBMap[BB];
     addSuccessorWithWeight(IndirectBrMBB, Succ);
   }
 
@@ -3160,9 +3208,9 @@ void SelectionDAGBuilder::visitAlloca(const AllocaInst &I) {
     return;   // getValue will auto-populate this.
 
   Type *Ty = I.getAllocatedType();
-  uint64_t TySize = TLI.getTargetData()->getTypeAllocSize(Ty);
+  uint64_t TySize = TLI.getDataLayout()->getTypeAllocSize(Ty);
   unsigned Align =
-    std::max((unsigned)TLI.getTargetData()->getPrefTypeAlignment(Ty),
+    std::max((unsigned)TLI.getDataLayout()->getPrefTypeAlignment(Ty),
              I.getAlignment());
 
   SDValue AllocSize = getValue(I.getArraySize());
@@ -3460,7 +3508,7 @@ void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) {
 
   SDValue InChain = getRoot();
 
-  EVT VT = EVT::getEVT(I.getType());
+  EVT VT = TLI.getValueType(I.getType());
 
   if (I.getAlignment() * 8 < VT.getSizeInBits())
     report_fatal_error("Cannot generate unaligned atomic load");
@@ -3490,7 +3538,7 @@ void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) {
 
   SDValue InChain = getRoot();
 
-  EVT VT = EVT::getEVT(I.getValueOperand()->getType());
+  EVT VT = TLI.getValueType(I.getValueOperand()->getType());
 
   if (I.getAlignment() * 8 < VT.getSizeInBits())
     report_fatal_error("Cannot generate unaligned atomic store");
@@ -4352,7 +4400,7 @@ static SDValue ExpandPowI(DebugLoc DL, SDValue LHS, SDValue RHS,
       return DAG.getConstantFP(1.0, LHS.getValueType());
 
     const Function *F = DAG.getMachineFunction().getFunction();
-    if (!F->hasFnAttr(Attribute::OptimizeForSize) ||
+    if (!F->getFnAttributes().hasAttribute(Attributes::OptimizeForSize) ||
         // If optimizing for size, don't insert too many multiplies.  This
         // inserts up to 5 multiplies.
         CountPopulation_32(Val)+Log2_32(Val) < 7) {
@@ -4850,7 +4898,21 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
     Res = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, DestVT,
                       getValue(I.getArgOperand(0)),
                       getValue(I.getArgOperand(1)),
-                      DAG.getConstant(Idx, MVT::i32));
+                      DAG.getIntPtrConstant(Idx));
+    setValue(&I, Res);
+    return 0;
+  }
+  case Intrinsic::x86_avx_vextractf128_pd_256:
+  case Intrinsic::x86_avx_vextractf128_ps_256:
+  case Intrinsic::x86_avx_vextractf128_si_256:
+  case Intrinsic::x86_avx2_vextracti128: {
+    DebugLoc dl = getCurDebugLoc();
+    EVT DestVT = TLI.getValueType(I.getType());
+    uint64_t Idx = (cast<ConstantInt>(I.getArgOperand(1))->getZExtValue() & 1) *
+                   DestVT.getVectorNumElements();
+    Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, DestVT,
+                      getValue(I.getArgOperand(0)),
+                      DAG.getIntPtrConstant(Idx));
     setValue(&I, Res);
     return 0;
   }
@@ -5113,10 +5175,13 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
     return 0;
   }
 
+  case Intrinsic::debugtrap:
   case Intrinsic::trap: {
     StringRef TrapFuncName = TM.Options.getTrapFunctionName();
     if (TrapFuncName.empty()) {
-      DAG.setRoot(DAG.getNode(ISD::TRAP, dl,MVT::Other, getRoot()));
+      ISD::NodeType Op = (Intrinsic == Intrinsic::trap) ? 
+        ISD::TRAP : ISD::DEBUGTRAP;
+      DAG.setRoot(DAG.getNode(Op, dl,MVT::Other, getRoot()));
       return 0;
     }
     TargetLowering::ArgListTy Args;
@@ -5131,10 +5196,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
     DAG.setRoot(Result.second);
     return 0;
   }
-  case Intrinsic::debugtrap: {
-    DAG.setRoot(DAG.getNode(ISD::DEBUGTRAP, dl,MVT::Other, getRoot()));
-    return 0;
-  }
+
   case Intrinsic::uadd_with_overflow:
   case Intrinsic::sadd_with_overflow:
   case Intrinsic::usub_with_overflow:
@@ -5177,14 +5239,40 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
                                         rw==1)); /* write */
     return 0;
   }
+  case Intrinsic::lifetime_start:
+  case Intrinsic::lifetime_end: {
+    bool IsStart = (Intrinsic == Intrinsic::lifetime_start);
+    // Stack coloring is not enabled in O0, discard region information.
+    if (TM.getOptLevel() == CodeGenOpt::None)
+      return 0;
+
+    SmallVector<Value *, 4> Allocas;
+    GetUnderlyingObjects(I.getArgOperand(1), Allocas, TD);
+
+    for (SmallVector<Value*, 4>::iterator Object = Allocas.begin(),
+         E = Allocas.end(); Object != E; ++Object) {
+      AllocaInst *LifetimeObject = dyn_cast_or_null<AllocaInst>(*Object);
 
+      // Could not find an Alloca.
+      if (!LifetimeObject)
+        continue;
+
+      int FI = FuncInfo.StaticAllocaMap[LifetimeObject];
+
+      SDValue Ops[2];
+      Ops[0] = getRoot();
+      Ops[1] = DAG.getFrameIndex(FI, TLI.getPointerTy(), true);
+      unsigned Opcode = (IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END);
+
+      Res = DAG.getNode(Opcode, dl, MVT::Other, Ops, 2);
+      DAG.setRoot(Res);
+    }
+  }
   case Intrinsic::invariant_start:
-  case Intrinsic::lifetime_start:
     // Discard region information.
     setValue(&I, DAG.getUNDEF(TLI.getPointerTy()));
     return 0;
   case Intrinsic::invariant_end:
-  case Intrinsic::lifetime_end:
     // Discard region information.
     return 0;
   case Intrinsic::donothing:
@@ -5220,9 +5308,9 @@ void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee,
   int DemoteStackIdx = -100;
 
   if (!CanLowerReturn) {
-    uint64_t TySize = TLI.getTargetData()->getTypeAllocSize(
+    uint64_t TySize = TLI.getDataLayout()->getTypeAllocSize(
                       FTy->getReturnType());
-    unsigned Align  = TLI.getTargetData()->getPrefTypeAlignment(
+    unsigned Align  = TLI.getDataLayout()->getPrefTypeAlignment(
                       FTy->getReturnType());
     MachineFunction &MF = DAG.getMachineFunction();
     DemoteStackIdx = MF.getFrameInfo()->CreateStackObject(TySize, Align, false);
@@ -5254,12 +5342,12 @@ void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee,
     Entry.Node = ArgNode; Entry.Ty = V->getType();
 
     unsigned attrInd = i - CS.arg_begin() + 1;
-    Entry.isSExt  = CS.paramHasAttr(attrInd, Attribute::SExt);
-    Entry.isZExt  = CS.paramHasAttr(attrInd, Attribute::ZExt);
-    Entry.isInReg = CS.paramHasAttr(attrInd, Attribute::InReg);
-    Entry.isSRet  = CS.paramHasAttr(attrInd, Attribute::StructRet);
-    Entry.isNest  = CS.paramHasAttr(attrInd, Attribute::Nest);
-    Entry.isByVal = CS.paramHasAttr(attrInd, Attribute::ByVal);
+    Entry.isSExt  = CS.paramHasAttr(attrInd, Attributes::SExt);
+    Entry.isZExt  = CS.paramHasAttr(attrInd, Attributes::ZExt);
+    Entry.isInReg = CS.paramHasAttr(attrInd, Attributes::InReg);
+    Entry.isSRet  = CS.paramHasAttr(attrInd, Attributes::StructRet);
+    Entry.isNest  = CS.paramHasAttr(attrInd, Attributes::Nest);
+    Entry.isByVal = CS.paramHasAttr(attrInd, Attributes::ByVal);
     Entry.Alignment = CS.getParamAlignment(attrInd);
     Args.push_back(Entry);
   }
@@ -5687,7 +5775,7 @@ public:
   /// MVT::Other.
   EVT getCallOperandValEVT(LLVMContext &Context,
                            const TargetLowering &TLI,
-                           const TargetData *TD) const {
+                           const DataLayout *TD) const {
     if (CallOperandVal == 0) return MVT::Other;
 
     if (isa<BasicBlock>(CallOperandVal))
@@ -5991,8 +6079,8 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
         // Otherwise, create a stack slot and emit a store to it before the
         // asm.
         Type *Ty = OpVal->getType();
-        uint64_t TySize = TLI.getTargetData()->getTypeAllocSize(Ty);
-        unsigned Align  = TLI.getTargetData()->getPrefTypeAlignment(Ty);
+        uint64_t TySize = TLI.getDataLayout()->getTypeAllocSize(Ty);
+        unsigned Align  = TLI.getDataLayout()->getPrefTypeAlignment(Ty);
         MachineFunction &MF = DAG.getMachineFunction();
         int SSFI = MF.getFrameInfo()->CreateStackObject(TySize, Align, false);
         SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI.getPointerTy());
@@ -6040,12 +6128,36 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
   const MDNode *SrcLoc = CS.getInstruction()->getMetadata("srcloc");
   AsmNodeOperands.push_back(DAG.getMDNode(SrcLoc));
 
-  // Remember the HasSideEffect and AlignStack bits as operand 3.
+  // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore
+  // bits as operand 3.
   unsigned ExtraInfo = 0;
   if (IA->hasSideEffects())
     ExtraInfo |= InlineAsm::Extra_HasSideEffects;
   if (IA->isAlignStack())
     ExtraInfo |= InlineAsm::Extra_IsAlignStack;
+  // Set the asm dialect.
+  ExtraInfo |= IA->getDialect() * InlineAsm::Extra_AsmDialect;
+
+  // Determine if this InlineAsm MayLoad or MayStore based on the constraints.
+  for (unsigned i = 0, e = TargetConstraints.size(); i != e; ++i) {
+    TargetLowering::AsmOperandInfo &OpInfo = TargetConstraints[i];
+
+    // Compute the constraint code and ConstraintType to use.
+    TLI.ComputeConstraintToUse(OpInfo, SDValue());
+
+    // Ideally, we would only check against memory constraints.  However, the
+    // meaning of an other constraint can be target-specific and we can't easily
+    // reason about it.  Therefore, be conservative and set MayLoad/MayStore
+    // for other constriants as well.
+    if (OpInfo.ConstraintType == TargetLowering::C_Memory ||
+        OpInfo.ConstraintType == TargetLowering::C_Other) {
+      if (OpInfo.Type == InlineAsm::isInput)
+        ExtraInfo |= InlineAsm::Extra_MayLoad;
+      else if (OpInfo.Type == InlineAsm::isOutput)
+        ExtraInfo |= InlineAsm::Extra_MayStore;
+    }
+  }
+
   AsmNodeOperands.push_back(DAG.getTargetConstant(ExtraInfo,
                                                   TLI.getPointerTy()));
 
@@ -6155,7 +6267,7 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
 
           // Use the produced MatchedRegs object to
           MatchedRegs.getCopyToRegs(InOperandVal, DAG, getCurDebugLoc(),
-                                    Chain, &Flag);
+                                    Chain, &Flag, CS.getInstruction());
           MatchedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse,
                                            true, OpInfo.getMatchedOperand(),
                                            DAG, AsmNodeOperands);
@@ -6237,7 +6349,7 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
       }
 
       OpInfo.AssignedRegs.getCopyToRegs(InOperandVal, DAG, getCurDebugLoc(),
-                                        Chain, &Flag);
+                                        Chain, &Flag, CS.getInstruction());
 
       OpInfo.AssignedRegs.AddInlineAsmOperands(InlineAsm::Kind_RegUse, false, 0,
                                                DAG, AsmNodeOperands);
@@ -6268,7 +6380,7 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
   // and set it as the value of the call.
   if (!RetValRegs.Regs.empty()) {
     SDValue Val = RetValRegs.getCopyFromRegs(DAG, FuncInfo, getCurDebugLoc(),
-                                             Chain, &Flag);
+                                             Chain, &Flag, CS.getInstruction());
 
     // FIXME: Why don't we do this for inline asms with MRVs?
     if (CS.getType()->isSingleValueType() && CS.getType()->isSized()) {
@@ -6308,7 +6420,7 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) {
     RegsForValue &OutRegs = IndirectStoresToEmit[i].first;
     const Value *Ptr = IndirectStoresToEmit[i].second;
     SDValue OutVal = OutRegs.getCopyFromRegs(DAG, FuncInfo, getCurDebugLoc(),
-                                             Chain, &Flag);
+                                             Chain, &Flag, IA);
     StoresToEmit.push_back(std::make_pair(OutVal, Ptr));
   }
 
@@ -6338,7 +6450,7 @@ void SelectionDAGBuilder::visitVAStart(const CallInst &I) {
 }
 
 void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) {
-  const TargetData &TD = *TLI.getTargetData();
+  const DataLayout &TD = *TLI.getDataLayout();
   SDValue V = DAG.getVAArg(TLI.getValueType(I.getType()), getCurDebugLoc(),
                            getRoot(), getValue(I.getOperand(0)),
                            DAG.getSrcValue(I.getOperand(0)),
@@ -6384,7 +6496,7 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
                            Args[i].Node.getResNo() + Value);
       ISD::ArgFlagsTy Flags;
       unsigned OriginalAlignment =
-        getTargetData()->getABITypeAlignment(ArgTy);
+        getDataLayout()->getABITypeAlignment(ArgTy);
 
       if (Args[i].isZExt)
         Flags.setZExt();
@@ -6398,7 +6510,7 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
         Flags.setByVal();
         PointerType *Ty = cast<PointerType>(Args[i].Ty);
         Type *ElementTy = Ty->getElementType();
-        Flags.setByValSize(getTargetData()->getTypeAllocSize(ElementTy));
+        Flags.setByValSize(getDataLayout()->getTypeAllocSize(ElementTy));
         // For ByVal, alignment should come from FE.  BE will guess if this
         // info is not there but there are cases it cannot get right.
         unsigned FrameAlign;
@@ -6423,12 +6535,13 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
         ExtendKind = ISD::ZERO_EXTEND;
 
       getCopyToParts(CLI.DAG, CLI.DL, Op, &Parts[0], NumParts,
-                     PartVT, ExtendKind);
+                     PartVT, CLI.CS ? CLI.CS->getInstruction() : 0, ExtendKind);
 
       for (unsigned j = 0; j != NumParts; ++j) {
         // if it isn't first piece, alignment must be 1
         ISD::OutputArg MyFlags(Flags, Parts[j].getValueType(),
-                               i < CLI.NumFixedArgs);
+                               i < CLI.NumFixedArgs,
+                               i, j*Parts[j].getValueType().getStoreSize());
         if (NumParts > 1 && j == 0)
           MyFlags.Flags.setSplit();
         else if (j != 0)
@@ -6504,7 +6617,7 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const {
     unsigned NumRegs = getNumRegisters(CLI.RetTy->getContext(), VT);
 
     ReturnValues.push_back(getCopyFromParts(CLI.DAG, CLI.DL, &InVals[CurReg],
-                                            NumRegs, RegisterVT, VT,
+                                            NumRegs, RegisterVT, VT, NULL,
                                             AssertOp));
     CurReg += NumRegs;
   }
@@ -6543,7 +6656,7 @@ SelectionDAGBuilder::CopyValueToVirtualRegister(const Value *V, unsigned Reg) {
 
   RegsForValue RFV(V->getContext(), TLI, Reg, V->getType());
   SDValue Chain = DAG.getEntryNode();
-  RFV.getCopyToRegs(Op, DAG, getCurDebugLoc(), Chain, 0);
+  RFV.getCopyToRegs(Op, DAG, getCurDebugLoc(), Chain, 0, V);
   PendingExports.push_back(Chain);
 }
 
@@ -6573,7 +6686,7 @@ void SelectionDAGISel::LowerArguments(const BasicBlock *LLVMBB) {
   const Function &F = *LLVMBB->getParent();
   SelectionDAG &DAG = SDB->DAG;
   DebugLoc dl = SDB->getCurDebugLoc();
-  const TargetData *TD = TLI.getTargetData();
+  const DataLayout *TD = TLI.getDataLayout();
   SmallVector<ISD::InputArg, 16> Ins;
 
   // Check whether the function can return without sret-demotion.
@@ -6591,7 +6704,7 @@ void SelectionDAGISel::LowerArguments(const BasicBlock *LLVMBB) {
     ISD::ArgFlagsTy Flags;
     Flags.setSRet();
     EVT RegisterVT = TLI.getRegisterType(*DAG.getContext(), ValueVTs[0]);
-    ISD::InputArg RetArg(Flags, RegisterVT, true);
+    ISD::InputArg RetArg(Flags, RegisterVT, true, 0, 0);
     Ins.push_back(RetArg);
   }
 
@@ -6610,15 +6723,15 @@ void SelectionDAGISel::LowerArguments(const BasicBlock *LLVMBB) {
       unsigned OriginalAlignment =
         TD->getABITypeAlignment(ArgTy);
 
-      if (F.paramHasAttr(Idx, Attribute::ZExt))
+      if (F.getParamAttributes(Idx).hasAttribute(Attributes::ZExt))
         Flags.setZExt();
-      if (F.paramHasAttr(Idx, Attribute::SExt))
+      if (F.getParamAttributes(Idx).hasAttribute(Attributes::SExt))
         Flags.setSExt();
-      if (F.paramHasAttr(Idx, Attribute::InReg))
+      if (F.getParamAttributes(Idx).hasAttribute(Attributes::InReg))
         Flags.setInReg();
-      if (F.paramHasAttr(Idx, Attribute::StructRet))
+      if (F.getParamAttributes(Idx).hasAttribute(Attributes::StructRet))
         Flags.setSRet();
-      if (F.paramHasAttr(Idx, Attribute::ByVal)) {
+      if (F.getParamAttributes(Idx).hasAttribute(Attributes::ByVal)) {
         Flags.setByVal();
         PointerType *Ty = cast<PointerType>(I->getType());
         Type *ElementTy = Ty->getElementType();
@@ -6632,14 +6745,15 @@ void SelectionDAGISel::LowerArguments(const BasicBlock *LLVMBB) {
           FrameAlign = TLI.getByValTypeAlignment(ElementTy);
         Flags.setByValAlign(FrameAlign);
       }
-      if (F.paramHasAttr(Idx, Attribute::Nest))
+      if (F.getParamAttributes(Idx).hasAttribute(Attributes::Nest))
         Flags.setNest();
       Flags.setOrigAlign(OriginalAlignment);
 
       EVT RegisterVT = TLI.getRegisterType(*CurDAG->getContext(), VT);
       unsigned NumRegs = TLI.getNumRegisters(*CurDAG->getContext(), VT);
       for (unsigned i = 0; i != NumRegs; ++i) {
-        ISD::InputArg MyFlags(Flags, RegisterVT, isArgValueUsed);
+        ISD::InputArg MyFlags(Flags, RegisterVT, isArgValueUsed,
+                              Idx-1, i*RegisterVT.getStoreSize());
         if (NumRegs > 1 && i == 0)
           MyFlags.Flags.setSplit();
         // if it isn't first piece, alignment must be 1
@@ -6685,7 +6799,7 @@ void SelectionDAGISel::LowerArguments(const BasicBlock *LLVMBB) {
     EVT RegVT = TLI.getRegisterType(*CurDAG->getContext(), VT);
     ISD::NodeType AssertOp = ISD::DELETED_NODE;
     SDValue ArgValue = getCopyFromParts(DAG, dl, &InVals[0], 1,
-                                        RegVT, VT, AssertOp);
+                                        RegVT, VT, NULL, AssertOp);
 
     MachineFunction& MF = SDB->DAG.getMachineFunction();
     MachineRegisterInfo& RegInfo = MF.getRegInfo();
@@ -6719,14 +6833,14 @@ void SelectionDAGISel::LowerArguments(const BasicBlock *LLVMBB) {
 
       if (!I->use_empty()) {
         ISD::NodeType AssertOp = ISD::DELETED_NODE;
-        if (F.paramHasAttr(Idx, Attribute::SExt))
+        if (F.getParamAttributes(Idx).hasAttribute(Attributes::SExt))
           AssertOp = ISD::AssertSext;
-        else if (F.paramHasAttr(Idx, Attribute::ZExt))
+        else if (F.getParamAttributes(Idx).hasAttribute(Attributes::ZExt))
           AssertOp = ISD::AssertZext;
 
         ArgValues.push_back(getCopyFromParts(DAG, dl, &InVals[i],
                                              NumParts, PartVT, VT,
-                                             AssertOp));
+                                             NULL, AssertOp));
       }
 
       i += NumParts;
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
index 4090002..9e46d96 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
@@ -66,7 +66,7 @@ class ShuffleVectorInst;
 class SIToFPInst;
 class StoreInst;
 class SwitchInst;
-class TargetData;
+class DataLayout;
 class TargetLibraryInfo;
 class TargetLowering;
 class TruncInst;
@@ -150,9 +150,11 @@ private:
     uint64_t Mask;
     MachineBasicBlock* BB;
     unsigned Bits;
+    uint32_t ExtraWeight;
 
-    CaseBits(uint64_t mask, MachineBasicBlock* bb, unsigned bits):
-      Mask(mask), BB(bb), Bits(bits) { }
+    CaseBits(uint64_t mask, MachineBasicBlock* bb, unsigned bits,
+             uint32_t Weight):
+      Mask(mask), BB(bb), Bits(bits), ExtraWeight(Weight) { }
   };
 
   typedef std::vector<Case>           CaseVector;
@@ -247,11 +249,13 @@ private:
   typedef std::pair<JumpTableHeader, JumpTable> JumpTableBlock;
 
   struct BitTestCase {
-    BitTestCase(uint64_t M, MachineBasicBlock* T, MachineBasicBlock* Tr):
-      Mask(M), ThisBB(T), TargetBB(Tr) { }
+    BitTestCase(uint64_t M, MachineBasicBlock* T, MachineBasicBlock* Tr,
+                uint32_t Weight):
+      Mask(M), ThisBB(T), TargetBB(Tr), ExtraWeight(Weight) { }
     uint64_t Mask;
     MachineBasicBlock *ThisBB;
     MachineBasicBlock *TargetBB;
+    uint32_t ExtraWeight;
   };
 
   typedef SmallVector<BitTestCase, 3> BitTestInfo;
@@ -281,7 +285,7 @@ public:
   const TargetMachine &TM;
   const TargetLowering &TLI;
   SelectionDAG &DAG;
-  const TargetData *TD;
+  const DataLayout *TD;
   AliasAnalysis *AA;
   const TargetLibraryInfo *LibInfo;
 
@@ -325,7 +329,7 @@ public:
                       CodeGenOpt::Level ol)
     : SDNodeOrder(0), TM(dag.getTarget()), TLI(dag.getTargetLoweringInfo()),
       DAG(dag), FuncInfo(funcinfo), OptLevel(ol),
-      HasTailCall(false), Context(dag.getContext()) {
+      HasTailCall(false) {
   }
 
   void init(GCFunctionInfo *gfi, AliasAnalysis &aa,
@@ -452,6 +456,7 @@ public:
   void visitBitTestHeader(BitTestBlock &B, MachineBasicBlock *SwitchBB);
   void visitBitTestCase(BitTestBlock &BB,
                         MachineBasicBlock* NextMBB,
+                        uint32_t BranchWeightToNext,
                         unsigned Reg,
                         BitTestCase &B,
                         MachineBasicBlock *SwitchBB);
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
index 13cd011..6f3ce7a 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -267,6 +267,8 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
   case ISD::STACKRESTORE:               return "stackrestore";
   case ISD::TRAP:                       return "trap";
   case ISD::DEBUGTRAP:                  return "debugtrap";
+  case ISD::LIFETIME_START:             return "lifetime.start";
+  case ISD::LIFETIME_END:               return "lifetime.end";
 
   // Bit manipulation
   case ISD::BSWAP:                      return "bswap";
@@ -331,7 +333,7 @@ void SDNode::dump(const SelectionDAG *G) const {
 }
 
 void SDNode::print_types(raw_ostream &OS, const SelectionDAG *G) const {
-  OS << (void*)this << ": ";
+  OS << (const void*)this << ": ";
 
   for (unsigned i = 0, e = getNumValues(); i != e; ++i) {
     if (i) OS << ",";
@@ -473,11 +475,16 @@ void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const {
     OS << "<" << *M->getMemOperand() << ">";
   } else if (const BlockAddressSDNode *BA =
                dyn_cast<BlockAddressSDNode>(this)) {
+    int64_t offset = BA->getOffset();
     OS << "<";
     WriteAsOperand(OS, BA->getBlockAddress()->getFunction(), false);
     OS << ", ";
     WriteAsOperand(OS, BA->getBlockAddress()->getBasicBlock(), false);
     OS << ">";
+    if (offset > 0)
+      OS << " + " << offset;
+    else
+      OS << " " << offset;
     if (unsigned int TF = BA->getTargetFlags())
       OS << " [TF=" << TF << ']';
   }
@@ -559,7 +566,7 @@ static void DumpNodesr(raw_ostream &OS, const SDNode *N, unsigned indent,
       child->printr(OS, G);
       once.insert(child);
     } else {         // Just the address. FIXME: also print the child's opcode.
-      OS << (void*)child;
+      OS << (const void*)child;
       if (unsigned RN = N->getOperand(i).getResNo())
         OS << ":" << RN;
     }
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index 4e5e3ba..c314fa5 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -474,6 +474,11 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) {
     MRI.replaceRegWith(From, To);
   }
 
+  // Freeze the set of reserved registers now that MachineFrameInfo has been
+  // set up. All the information required by getReservedRegs() should be
+  // available now.
+  MRI.freezeReservedRegs(*MF);
+
   // Release function-specific state. SDB and CurDAG are already cleared
   // at this point.
   FuncInfo->clear();
@@ -554,7 +559,7 @@ void SelectionDAGISel::CodeGenAndEmitDAG() {
 #endif
   {
     BlockNumber = FuncInfo->MBB->getNumber();
-    BlockName = MF->getFunction()->getName().str() + ":" +
+    BlockName = MF->getName().str() + ":" +
                 FuncInfo->MBB->getBasicBlock()->getName().str();
   }
   DEBUG(dbgs() << "Initial selection DAG: BB#" << BlockNumber
@@ -1209,7 +1214,12 @@ SelectionDAGISel::FinishBasicBlock() {
       CodeGenAndEmitDAG();
     }
 
+    uint32_t UnhandledWeight = 0;
+    for (unsigned j = 0, ej = SDB->BitTestCases[i].Cases.size(); j != ej; ++j)
+      UnhandledWeight += SDB->BitTestCases[i].Cases[j].ExtraWeight;
+
     for (unsigned j = 0, ej = SDB->BitTestCases[i].Cases.size(); j != ej; ++j) {
+      UnhandledWeight -= SDB->BitTestCases[i].Cases[j].ExtraWeight;
       // Set the current basic block to the mbb we wish to insert the code into
       FuncInfo->MBB = SDB->BitTestCases[i].Cases[j].ThisBB;
       FuncInfo->InsertPt = FuncInfo->MBB->end();
@@ -1217,12 +1227,14 @@ SelectionDAGISel::FinishBasicBlock() {
       if (j+1 != ej)
         SDB->visitBitTestCase(SDB->BitTestCases[i],
                               SDB->BitTestCases[i].Cases[j+1].ThisBB,
+                              UnhandledWeight,
                               SDB->BitTestCases[i].Reg,
                               SDB->BitTestCases[i].Cases[j],
                               FuncInfo->MBB);
       else
         SDB->visitBitTestCase(SDB->BitTestCases[i],
                               SDB->BitTestCases[i].Default,
+                              UnhandledWeight,
                               SDB->BitTestCases[i].Reg,
                               SDB->BitTestCases[i].Cases[j],
                               FuncInfo->MBB);
@@ -1794,10 +1806,13 @@ WalkChainUsers(const SDNode *ChainedNode,
         User->getOpcode() == ISD::HANDLENODE)  // Root of the graph.
       continue;
 
-    if (User->getOpcode() == ISD::CopyToReg ||
-        User->getOpcode() == ISD::CopyFromReg ||
-        User->getOpcode() == ISD::INLINEASM ||
-        User->getOpcode() == ISD::EH_LABEL) {
+    unsigned UserOpcode = User->getOpcode();
+    if (UserOpcode == ISD::CopyToReg ||
+        UserOpcode == ISD::CopyFromReg ||
+        UserOpcode == ISD::INLINEASM ||
+        UserOpcode == ISD::EH_LABEL ||
+        UserOpcode == ISD::LIFETIME_START ||
+        UserOpcode == ISD::LIFETIME_END) {
       // If their node ID got reset to -1 then they've already been selected.
       // Treat them like a MachineOpcode.
       if (User->getNodeId() == -1)
@@ -1994,7 +2009,7 @@ MorphNode(SDNode *Node, unsigned TargetOpc, SDVTList VTList,
   return Res;
 }
 
-/// CheckPatternPredicate - Implements OP_CheckPatternPredicate.
+/// CheckSame - Implements OP_CheckSame.
 LLVM_ATTRIBUTE_ALWAYS_INLINE static bool
 CheckSame(const unsigned char *MatcherTable, unsigned &MatcherIndex,
           SDValue N,
@@ -2213,6 +2228,8 @@ SelectCodeCommon(SDNode *NodeToMatch, const unsigned char *MatcherTable,
   case ISD::CopyFromReg:
   case ISD::CopyToReg:
   case ISD::EH_LABEL:
+  case ISD::LIFETIME_START:
+  case ISD::LIFETIME_END:
     NodeToMatch->setNodeId(-1); // Mark selected.
     return 0;
   case ISD::AssertSext:
@@ -2981,7 +2998,7 @@ void SelectionDAGISel::CannotYetSelect(SDNode *N) {
       N->getOpcode() != ISD::INTRINSIC_WO_CHAIN &&
       N->getOpcode() != ISD::INTRINSIC_VOID) {
     N->printrFull(Msg, CurDAG);
-    Msg << "\nIn function: " << MF->getFunction()->getName();
+    Msg << "\nIn function: " << MF->getName();
   } else {
     bool HasInputChain = N->getOperand(0).getValueType() == MVT::Other;
     unsigned iid =
diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGPrinter.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGPrinter.cpp
index 173ffac..3921635 100644
--- a/lib/CodeGen/SelectionDAG/SelectionDAGPrinter.cpp
+++ b/lib/CodeGen/SelectionDAG/SelectionDAGPrinter.cpp
@@ -14,7 +14,6 @@
 #include "ScheduleDAGSDNodes.h"
 #include "llvm/Constants.h"
 #include "llvm/DebugInfo.h"
-#include "llvm/Function.h"
 #include "llvm/Assembly/Writer.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/CodeGen/MachineConstantPool.h"
@@ -50,7 +49,7 @@ namespace llvm {
 
     template<typename EdgeIter>
     static std::string getEdgeSourceLabel(const void *Node, EdgeIter I) {
-      return itostr(I - SDNodeIterator::begin((SDNode *) Node));
+      return itostr(I - SDNodeIterator::begin((const SDNode *) Node));
     }
 
     /// edgeTargetsEdgeSource - This method returns true if this outgoing edge
@@ -73,7 +72,7 @@ namespace llvm {
     }
 
     static std::string getGraphName(const SelectionDAG *G) {
-      return G->getMachineFunction().getFunction()->getName();
+      return G->getMachineFunction().getName();
     }
 
     static bool renderGraphFromBottomUp() {
@@ -146,7 +145,7 @@ std::string DOTGraphTraits<SelectionDAG*>::getNodeLabel(const SDNode *Node,
 void SelectionDAG::viewGraph(const std::string &Title) {
 // This code is only for debugging!
 #ifndef NDEBUG
-  ViewGraph(this, "dag." + getMachineFunction().getFunction()->getName(),
+  ViewGraph(this, "dag." + getMachineFunction().getName(),
             false, Title);
 #else
   errs() << "SelectionDAG::viewGraph is only available in debug builds on "
diff --git a/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 6820175..49f55e2 100644
--- a/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -14,7 +14,7 @@
 #include "llvm/Target/TargetLowering.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCExpr.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetLoweringObjectFile.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetRegisterInfo.h"
@@ -515,7 +515,7 @@ static void InitCmpLibcallCCs(ISD::CondCode *CCs) {
 /// NOTE: The constructor takes ownership of TLOF.
 TargetLowering::TargetLowering(const TargetMachine &tm,
                                const TargetLoweringObjectFile *tlof)
-  : TM(tm), TD(TM.getTargetData()), TLOF(*tlof) {
+  : TM(tm), TD(TM.getDataLayout()), TLOF(*tlof) {
   // All operations default to being supported.
   memset(OpActions, 0, sizeof(OpActions));
   memset(LoadExtActions, 0, sizeof(LoadExtActions));
@@ -583,8 +583,13 @@ TargetLowering::TargetLowering(const TargetMachine &tm,
   // Default ISD::TRAP to expand (which turns it into abort).
   setOperationAction(ISD::TRAP, MVT::Other, Expand);
 
+  // On most systems, DEBUGTRAP and TRAP have no difference. The "Expand"
+  // here is to inform DAG Legalizer to replace DEBUGTRAP with TRAP.
+  //
+  setOperationAction(ISD::DEBUGTRAP, MVT::Other, Expand);
+
   IsLittleEndian = TD->isLittleEndian();
-  PointerTy = MVT::getIntegerVT(8*TD->getPointerSize());
+  PointerTy = MVT::getIntegerVT(8*TD->getPointerSize(0));
   memset(RegClassForVT, 0,MVT::LAST_VALUETYPE*sizeof(TargetRegisterClass*));
   memset(TargetDAGCombineArray, 0, array_lengthof(TargetDAGCombineArray));
   maxStoresPerMemset = maxStoresPerMemcpy = maxStoresPerMemmove = 8;
@@ -613,6 +618,7 @@ TargetLowering::TargetLowering(const TargetMachine &tm,
   ShouldFoldAtomicFences = false;
   InsertFencesForAtomic = false;
   SupportJumpTables = true;
+  MinimumJumpTableEntries = 4;
 
   InitLibcallNames(LibcallRoutineNames);
   InitCmpLibcallCCs(CmpLibcallCCs);
@@ -624,7 +630,7 @@ TargetLowering::~TargetLowering() {
 }
 
 MVT TargetLowering::getShiftAmountTy(EVT LHSTy) const {
-  return MVT::getIntegerVT(8*TD->getPointerSize());
+  return MVT::getIntegerVT(8*TD->getPointerSize(0));
 }
 
 /// canOpTrap - Returns true if the operation can trap for the value type.
@@ -772,7 +778,7 @@ void TargetLowering::computeRegisterProperties() {
       LegalIntReg = IntReg;
     } else {
       RegisterTypeForVT[IntReg] = TransformToType[IntReg] =
-        (MVT::SimpleValueType)LegalIntReg;
+        (const MVT::SimpleValueType)LegalIntReg;
       ValueTypeActions.setTypeAction(IVT, TypePromoteInteger);
     }
   }
@@ -898,10 +904,9 @@ const char *TargetLowering::getTargetNodeName(unsigned Opcode) const {
   return NULL;
 }
 
-
 EVT TargetLowering::getSetCCResultType(EVT VT) const {
   assert(!VT.isVector() && "No default SetCC type for vectors!");
-  return PointerTy.SimpleTy;
+  return getPointerTy(0).SimpleTy;
 }
 
 MVT::SimpleValueType TargetLowering::getCmpLibcallReturnType() const {
@@ -997,9 +1002,9 @@ void llvm::GetReturnInfo(Type* ReturnType, Attributes attr,
     EVT VT = ValueVTs[j];
     ISD::NodeType ExtendKind = ISD::ANY_EXTEND;
 
-    if (attr & Attribute::SExt)
+    if (attr.hasAttribute(Attributes::SExt))
       ExtendKind = ISD::SIGN_EXTEND;
-    else if (attr & Attribute::ZExt)
+    else if (attr.hasAttribute(Attributes::ZExt))
       ExtendKind = ISD::ZERO_EXTEND;
 
     // FIXME: C calling convention requires the return type to be promoted to
@@ -1017,18 +1022,17 @@ void llvm::GetReturnInfo(Type* ReturnType, Attributes attr,
 
     // 'inreg' on function refers to return value
     ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy();
-    if (attr & Attribute::InReg)
+    if (attr.hasAttribute(Attributes::InReg))
       Flags.setInReg();
 
     // Propagate extension type if any
-    if (attr & Attribute::SExt)
+    if (attr.hasAttribute(Attributes::SExt))
       Flags.setSExt();
-    else if (attr & Attribute::ZExt)
+    else if (attr.hasAttribute(Attributes::ZExt))
       Flags.setZExt();
 
-    for (unsigned i = 0; i < NumParts; ++i) {
-      Outs.push_back(ISD::OutputArg(Flags, PartVT, /*isFixed=*/true));
-    }
+    for (unsigned i = 0; i < NumParts; ++i)
+      Outs.push_back(ISD::OutputArg(Flags, PartVT, /*isFixed=*/true, 0, 0));
   }
 }
 
@@ -1062,7 +1066,7 @@ SDValue TargetLowering::getPICJumpTableRelocBase(SDValue Table,
 
   if ((JTEncoding == MachineJumpTableInfo::EK_GPRel64BlockAddress) ||
       (JTEncoding == MachineJumpTableInfo::EK_GPRel32BlockAddress))
-    return DAG.getGLOBAL_OFFSET_TABLE(getPointerTy());
+    return DAG.getGLOBAL_OFFSET_TABLE(getPointerTy(0));
 
   return Table;
 }
@@ -2441,7 +2445,7 @@ TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1,
   if (N0 == N1) {
     // The sext(setcc()) => setcc() optimization relies on the appropriate
     // constant being emitted.
-    uint64_t EqVal;
+    uint64_t EqVal = 0;
     switch (getBooleanContents(N0.getValueType().isVector())) {
     case UndefinedBooleanContent:
     case ZeroOrOneBooleanContent:
@@ -2954,8 +2958,9 @@ TargetLowering::AsmOperandInfoVector TargetLowering::ParseConstraints(
               EVT::getEVT(IntegerType::get(OpTy->getContext(), BitSize), true);
           break;
         }
-      } else if (dyn_cast<PointerType>(OpTy)) {
-        OpInfo.ConstraintVT = MVT::getIntegerVT(8*TD->getPointerSize());
+      } else if (PointerType *PT = dyn_cast<PointerType>(OpTy)) {
+        OpInfo.ConstraintVT = MVT::getIntegerVT(
+            8*TD->getPointerSize(PT->getAddressSpace()));
       } else {
         OpInfo.ConstraintVT = EVT::getEVT(OpTy, true);
       }
diff --git a/lib/CodeGen/SelectionDAG/TargetSelectionDAGInfo.cpp b/lib/CodeGen/SelectionDAG/TargetSelectionDAGInfo.cpp
index a081e3c..f769b44 100644
--- a/lib/CodeGen/SelectionDAG/TargetSelectionDAGInfo.cpp
+++ b/lib/CodeGen/SelectionDAG/TargetSelectionDAGInfo.cpp
@@ -16,7 +16,7 @@
 using namespace llvm;
 
 TargetSelectionDAGInfo::TargetSelectionDAGInfo(const TargetMachine &TM)
-  : TD(TM.getTargetData()) {
+  : TD(TM.getDataLayout()) {
 }
 
 TargetSelectionDAGInfo::~TargetSelectionDAGInfo() {
diff --git a/lib/CodeGen/ShrinkWrapping.cpp b/lib/CodeGen/ShrinkWrapping.cpp
index 21ae2f5..4fbe1b3 100644
--- a/lib/CodeGen/ShrinkWrapping.cpp
+++ b/lib/CodeGen/ShrinkWrapping.cpp
@@ -159,7 +159,7 @@ void PEI::initShrinkWrappingInfo() {
   // via --shrink-wrap-func=<funcname>.
 #ifndef NDEBUG
   if (ShrinkWrapFunc != "") {
-    std::string MFName = MF->getFunction()->getName().str();
+    std::string MFName = MF->getName().str();
     ShrinkWrapThisFunction = (MFName == ShrinkWrapFunc);
   }
 #endif
@@ -187,7 +187,7 @@ void PEI::placeCSRSpillsAndRestores(MachineFunction &Fn) {
 
   DEBUG(if (ShrinkWrapThisFunction) {
       dbgs() << "Place CSR spills/restores for "
-             << MF->getFunction()->getName() << "\n";
+             << MF->getName() << "\n";
     });
 
   if (calculateSets(Fn))
@@ -364,7 +364,7 @@ bool PEI::calculateSets(MachineFunction &Fn) {
   // If no CSRs used, we are done.
   if (CSI.empty()) {
     DEBUG(if (ShrinkWrapThisFunction)
-            dbgs() << "DISABLED: " << Fn.getFunction()->getName()
+            dbgs() << "DISABLED: " << Fn.getName()
                    << ": uses no callee-saved registers\n");
     return false;
   }
@@ -384,7 +384,7 @@ bool PEI::calculateSets(MachineFunction &Fn) {
   // implementation to functions with <= 500 MBBs.
   if (Fn.size() > 500) {
     DEBUG(if (ShrinkWrapThisFunction)
-            dbgs() << "DISABLED: " << Fn.getFunction()->getName()
+            dbgs() << "DISABLED: " << Fn.getName()
                    << ": too large (" << Fn.size() << " MBBs)\n");
     ShrinkWrapThisFunction = false;
   }
@@ -466,7 +466,7 @@ bool PEI::calculateSets(MachineFunction &Fn) {
   }
 
   if (allCSRUsesInEntryBlock) {
-    DEBUG(dbgs() << "DISABLED: " << Fn.getFunction()->getName()
+    DEBUG(dbgs() << "DISABLED: " << Fn.getName()
                  << ": all CSRs used in EntryBlock\n");
     ShrinkWrapThisFunction = false;
   } else {
@@ -478,7 +478,7 @@ bool PEI::calculateSets(MachineFunction &Fn) {
         allCSRsUsedInEntryFanout = false;
     }
     if (allCSRsUsedInEntryFanout) {
-      DEBUG(dbgs() << "DISABLED: " << Fn.getFunction()->getName()
+      DEBUG(dbgs() << "DISABLED: " << Fn.getName()
                    << ": all CSRs used in imm successors of EntryBlock\n");
       ShrinkWrapThisFunction = false;
     }
@@ -505,7 +505,7 @@ bool PEI::calculateSets(MachineFunction &Fn) {
       if (dominatesExitNodes) {
         CSRUsedInChokePoints |= CSRUsed[MBB];
         if (CSRUsedInChokePoints == UsedCSRegs) {
-          DEBUG(dbgs() << "DISABLED: " << Fn.getFunction()->getName()
+          DEBUG(dbgs() << "DISABLED: " << Fn.getName()
                        << ": all CSRs used in choke point(s) at "
                        << getBasicBlockName(MBB) << "\n");
           ShrinkWrapThisFunction = false;
@@ -521,7 +521,7 @@ bool PEI::calculateSets(MachineFunction &Fn) {
     return false;
 
   DEBUG({
-      dbgs() << "ENABLED: " << Fn.getFunction()->getName();
+      dbgs() << "ENABLED: " << Fn.getName();
       if (HasFastExitPath)
         dbgs() << " (fast exit path)";
       dbgs() << "\n";
@@ -861,7 +861,7 @@ void PEI::placeSpillsAndRestores(MachineFunction &Fn) {
   DEBUG(if (ShrinkWrapDebugging >= BasicInfo) {
       dbgs() << "-----------------------------------------------------------\n";
       dbgs() << "total iterations = " << iterations << " ( "
-           << Fn.getFunction()->getName()
+           << Fn.getName()
            << " " << numSRReducedThisFunc
            << " " << Fn.size()
            << " )\n";
@@ -984,7 +984,7 @@ void PEI::verifySpillRestorePlacement() {
       if (isReturnBlock(SBB) || SBB->succ_size() == 0) {
         if (restored != spilled) {
           CSRegSet notRestored = (spilled - restored);
-          DEBUG(dbgs() << MF->getFunction()->getName() << ": "
+          DEBUG(dbgs() << MF->getName() << ": "
                        << stringifyCSRegSet(notRestored)
                        << " spilled at " << getBasicBlockName(MBB)
                        << " are never restored on path to return "
@@ -1032,7 +1032,7 @@ void PEI::verifySpillRestorePlacement() {
     }
     if (spilled != restored) {
       CSRegSet notSpilled = (restored - spilled);
-      DEBUG(dbgs() << MF->getFunction()->getName() << ": "
+      DEBUG(dbgs() << MF->getName() << ": "
                    << stringifyCSRegSet(notSpilled)
                    << " restored at " << getBasicBlockName(MBB)
                    << " are never spilled\n");
diff --git a/lib/CodeGen/SjLjEHPrepare.cpp b/lib/CodeGen/SjLjEHPrepare.cpp
index 980bd74..4b566fc 100644
--- a/lib/CodeGen/SjLjEHPrepare.cpp
+++ b/lib/CodeGen/SjLjEHPrepare.cpp
@@ -30,7 +30,7 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetLowering.h"
 #include "llvm/Transforms/Scalar.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
@@ -191,58 +191,43 @@ setupFunctionContext(Function &F, ArrayRef<LandingPadInst*> LPads) {
   // that needs to be restored on all exits from the function. This is an alloca
   // because the value needs to be added to the global context list.
   unsigned Align =
-    TLI->getTargetData()->getPrefTypeAlignment(FunctionContextTy);
+    TLI->getDataLayout()->getPrefTypeAlignment(FunctionContextTy);
   FuncCtx =
     new AllocaInst(FunctionContextTy, 0, Align, "fn_context", EntryBB->begin());
 
   // Fill in the function context structure.
-  Type *Int32Ty = Type::getInt32Ty(F.getContext());
-  Value *Zero = ConstantInt::get(Int32Ty, 0);
-  Value *One = ConstantInt::get(Int32Ty, 1);
-  Value *Two = ConstantInt::get(Int32Ty, 2);
-  Value *Three = ConstantInt::get(Int32Ty, 3);
-  Value *Four = ConstantInt::get(Int32Ty, 4);
-
-  Value *Idxs[2] = { Zero, 0 };
-
   for (unsigned I = 0, E = LPads.size(); I != E; ++I) {
     LandingPadInst *LPI = LPads[I];
     IRBuilder<> Builder(LPI->getParent()->getFirstInsertionPt());
 
     // Reference the __data field.
-    Idxs[1] = Two;
-    Value *FCData = Builder.CreateGEP(FuncCtx, Idxs, "__data");
+    Value *FCData = Builder.CreateConstGEP2_32(FuncCtx, 0, 2, "__data");
 
     // The exception values come back in context->__data[0].
-    Idxs[1] = Zero;
-    Value *ExceptionAddr = Builder.CreateGEP(FCData, Idxs, "exception_gep");
+    Value *ExceptionAddr = Builder.CreateConstGEP2_32(FCData, 0, 0,
+                                                      "exception_gep");
     Value *ExnVal = Builder.CreateLoad(ExceptionAddr, true, "exn_val");
-    ExnVal = Builder.CreateIntToPtr(ExnVal, Type::getInt8PtrTy(F.getContext()));
+    ExnVal = Builder.CreateIntToPtr(ExnVal, Builder.getInt8PtrTy());
 
-    Idxs[1] = One;
-    Value *SelectorAddr = Builder.CreateGEP(FCData, Idxs, "exn_selector_gep");
+    Value *SelectorAddr = Builder.CreateConstGEP2_32(FCData, 0, 1,
+                                                     "exn_selector_gep");
     Value *SelVal = Builder.CreateLoad(SelectorAddr, true, "exn_selector_val");
 
     substituteLPadValues(LPI, ExnVal, SelVal);
   }
 
   // Personality function
-  Idxs[1] = Three;
+  IRBuilder<> Builder(EntryBB->getTerminator());
   if (!PersonalityFn)
     PersonalityFn = LPads[0]->getPersonalityFn();
-  Value *PersonalityFieldPtr =
-    GetElementPtrInst::Create(FuncCtx, Idxs, "pers_fn_gep",
-                              EntryBB->getTerminator());
-  new StoreInst(PersonalityFn, PersonalityFieldPtr, true,
-                EntryBB->getTerminator());
+  Value *PersonalityFieldPtr = Builder.CreateConstGEP2_32(FuncCtx, 0, 3,
+                                                          "pers_fn_gep");
+  Builder.CreateStore(PersonalityFn, PersonalityFieldPtr, /*isVolatile=*/true);
 
   // LSDA address
-  Value *LSDA = CallInst::Create(LSDAAddrFn, "lsda_addr",
-                                 EntryBB->getTerminator());
-  Idxs[1] = Four;
-  Value *LSDAFieldPtr = GetElementPtrInst::Create(FuncCtx, Idxs, "lsda_gep",
-                                                  EntryBB->getTerminator());
-  new StoreInst(LSDA, LSDAFieldPtr, true, EntryBB->getTerminator());
+  Value *LSDA = Builder.CreateCall(LSDAAddrFn, "lsda_addr");
+  Value *LSDAFieldPtr = Builder.CreateConstGEP2_32(FuncCtx, 0, 4, "lsda_gep");
+  Builder.CreateStore(LSDA, LSDAFieldPtr, /*isVolatile=*/true);
 
   return FuncCtx;
 }
@@ -417,48 +402,31 @@ bool SjLjEHPrepare::setupEntryBlockAndCallSites(Function &F) {
   Value *FuncCtx =
     setupFunctionContext(F, makeArrayRef(LPads.begin(), LPads.end()));
   BasicBlock *EntryBB = F.begin();
-  Type *Int32Ty = Type::getInt32Ty(F.getContext());
-
-  Value *Idxs[2] = {
-    ConstantInt::get(Int32Ty, 0), 0
-  };
+  IRBuilder<> Builder(EntryBB->getTerminator());
 
   // Get a reference to the jump buffer.
-  Idxs[1] = ConstantInt::get(Int32Ty, 5);
-  Value *JBufPtr = GetElementPtrInst::Create(FuncCtx, Idxs, "jbuf_gep",
-                                             EntryBB->getTerminator());
+  Value *JBufPtr = Builder.CreateConstGEP2_32(FuncCtx, 0, 5, "jbuf_gep");
 
   // Save the frame pointer.
-  Idxs[1] = ConstantInt::get(Int32Ty, 0);
-  Value *FramePtr = GetElementPtrInst::Create(JBufPtr, Idxs, "jbuf_fp_gep",
-                                              EntryBB->getTerminator());
+  Value *FramePtr = Builder.CreateConstGEP2_32(JBufPtr, 0, 0, "jbuf_fp_gep");
 
-  Value *Val = CallInst::Create(FrameAddrFn,
-                                ConstantInt::get(Int32Ty, 0),
-                                "fp",
-                                EntryBB->getTerminator());
-  new StoreInst(Val, FramePtr, true, EntryBB->getTerminator());
+  Value *Val = Builder.CreateCall(FrameAddrFn, Builder.getInt32(0), "fp");
+  Builder.CreateStore(Val, FramePtr, /*isVolatile=*/true);
 
   // Save the stack pointer.
-  Idxs[1] = ConstantInt::get(Int32Ty, 2);
-  Value *StackPtr = GetElementPtrInst::Create(JBufPtr, Idxs, "jbuf_sp_gep",
-                                              EntryBB->getTerminator());
+  Value *StackPtr = Builder.CreateConstGEP2_32(JBufPtr, 0, 2, "jbuf_sp_gep");
 
-  Val = CallInst::Create(StackAddrFn, "sp", EntryBB->getTerminator());
-  new StoreInst(Val, StackPtr, true, EntryBB->getTerminator());
+  Val = Builder.CreateCall(StackAddrFn, "sp");
+  Builder.CreateStore(Val, StackPtr, /*isVolatile=*/true);
 
   // Call the setjmp instrinsic. It fills in the rest of the jmpbuf.
-  Value *SetjmpArg = CastInst::Create(Instruction::BitCast, JBufPtr,
-                                      Type::getInt8PtrTy(F.getContext()), "",
-                                      EntryBB->getTerminator());
-  CallInst::Create(BuiltinSetjmpFn, SetjmpArg, "", EntryBB->getTerminator());
+  Value *SetjmpArg = Builder.CreateBitCast(JBufPtr, Builder.getInt8PtrTy());
+  Builder.CreateCall(BuiltinSetjmpFn, SetjmpArg);
 
   // Store a pointer to the function context so that the back-end will know
   // where to look for it.
-  Value *FuncCtxArg = CastInst::Create(Instruction::BitCast, FuncCtx,
-                                       Type::getInt8PtrTy(F.getContext()), "",
-                                       EntryBB->getTerminator());
-  CallInst::Create(FuncCtxFn, FuncCtxArg, "", EntryBB->getTerminator());
+  Value *FuncCtxArg = Builder.CreateBitCast(FuncCtx, Builder.getInt8PtrTy());
+  Builder.CreateCall(FuncCtxFn, FuncCtxArg);
 
   // At this point, we are all set up, update the invoke instructions to mark
   // their call_site values.
diff --git a/lib/CodeGen/SlotIndexes.cpp b/lib/CodeGen/SlotIndexes.cpp
index c8c3fb3..95faafab 100644
--- a/lib/CodeGen/SlotIndexes.cpp
+++ b/lib/CodeGen/SlotIndexes.cpp
@@ -143,6 +143,7 @@ void SlotIndexes::renumberIndexes(IndexList::iterator curItr) {
 }
 
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void SlotIndexes::dump() const {
   for (IndexList::const_iterator itr = indexList.begin();
        itr != indexList.end(); ++itr) {
@@ -159,6 +160,7 @@ void SlotIndexes::dump() const {
     dbgs() << "BB#" << i << "\t[" << MBBRanges[i].first << ';'
            << MBBRanges[i].second << ")\n";
 }
+#endif
 
 // Print a SlotIndex to a raw_ostream.
 void SlotIndex::print(raw_ostream &os) const {
@@ -168,9 +170,11 @@ void SlotIndex::print(raw_ostream &os) const {
     os << "invalid";
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 // Dump a SlotIndex to stderr.
 void SlotIndex::dump() const {
   print(dbgs());
   dbgs() << "\n";
 }
+#endif
 
diff --git a/lib/CodeGen/SplitKit.cpp b/lib/CodeGen/SplitKit.cpp
index 4a2b7ec..dca15ee 100644
--- a/lib/CodeGen/SplitKit.cpp
+++ b/lib/CodeGen/SplitKit.cpp
@@ -356,6 +356,7 @@ void SplitEditor::reset(LiveRangeEdit &LRE, ComplementSpillMode SM) {
   Edit->anyRematerializable(0);
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void SplitEditor::dump() const {
   if (RegAssign.empty()) {
     dbgs() << " empty\n";
@@ -366,6 +367,7 @@ void SplitEditor::dump() const {
     dbgs() << " [" << I.start() << ';' << I.stop() << "):" << I.value();
   dbgs() << '\n';
 }
+#endif
 
 VNInfo *SplitEditor::defValue(unsigned RegIdx,
                               const VNInfo *ParentVNI,
diff --git a/lib/CodeGen/StackColoring.cpp b/lib/CodeGen/StackColoring.cpp
new file mode 100644
index 0000000..1cbee84
--- /dev/null
+++ b/lib/CodeGen/StackColoring.cpp
@@ -0,0 +1,783 @@
+//===-- StackColoring.cpp -------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass implements the stack-coloring optimization that looks for
+// lifetime markers machine instructions (LIFESTART_BEGIN and LIFESTART_END),
+// which represent the possible lifetime of stack slots. It attempts to
+// merge disjoint stack slots and reduce the used stack space.
+// NOTE: This pass is not StackSlotColoring, which optimizes spill slots.
+//
+// TODO: In the future we plan to improve stack coloring in the following ways:
+// 1. Allow merging multiple small slots into a single larger slot at different
+//    offsets.
+// 2. Merge this pass with StackSlotColoring and allow merging of allocas with
+//    spill slots.
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "stackcoloring"
+#include "MachineTraceMetrics.h"
+#include "llvm/Function.h"
+#include "llvm/Module.h"
+#include "llvm/ADT/BitVector.h"
+#include "llvm/Analysis/Dominators.h"
+#include "llvm/Analysis/ValueTracking.h"
+#include "llvm/ADT/DepthFirstIterator.h"
+#include "llvm/ADT/PostOrderIterator.h"
+#include "llvm/ADT/SetVector.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/SparseSet.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/CodeGen/LiveInterval.h"
+#include "llvm/CodeGen/MachineLoopInfo.h"
+#include "llvm/CodeGen/MachineBranchProbabilityInfo.h"
+#include "llvm/CodeGen/MachineDominators.h"
+#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineLoopInfo.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/MachineMemOperand.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/CodeGen/SlotIndexes.h"
+#include "llvm/DebugInfo.h"
+#include "llvm/Instructions.h"
+#include "llvm/MC/MCInstrItineraries.h"
+#include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/Target/TargetRegisterInfo.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+
+static cl::opt<bool>
+DisableColoring("no-stack-coloring",
+        cl::init(false), cl::Hidden,
+        cl::desc("Disable stack coloring"));
+
+/// The user may write code that uses allocas outside of the declared lifetime
+/// zone. This can happen when the user returns a reference to a local
+/// data-structure. We can detect these cases and decide not to optimize the
+/// code. If this flag is enabled, we try to save the user.
+static cl::opt<bool>
+ProtectFromEscapedAllocas("protect-from-escaped-allocas",
+        cl::init(false), cl::Hidden,
+        cl::desc("Do not optimize lifetime zones that are broken"));
+
+STATISTIC(NumMarkerSeen,  "Number of lifetime markers found.");
+STATISTIC(StackSpaceSaved, "Number of bytes saved due to merging slots.");
+STATISTIC(StackSlotMerged, "Number of stack slot merged.");
+STATISTIC(EscapedAllocas,
+          "Number of allocas that escaped the lifetime region");
+
+//===----------------------------------------------------------------------===//
+//                           StackColoring Pass
+//===----------------------------------------------------------------------===//
+
+namespace {
+/// StackColoring - A machine pass for merging disjoint stack allocations,
+/// marked by the LIFETIME_START and LIFETIME_END pseudo instructions.
+class StackColoring : public MachineFunctionPass {
+  MachineFrameInfo *MFI;
+  MachineFunction *MF;
+
+  /// A class representing liveness information for a single basic block.
+  /// Each bit in the BitVector represents the liveness property
+  /// for a different stack slot.
+  struct BlockLifetimeInfo {
+    /// Which slots BEGINs in each basic block.
+    BitVector Begin;
+    /// Which slots ENDs in each basic block.
+    BitVector End;
+    /// Which slots are marked as LIVE_IN, coming into each basic block.
+    BitVector LiveIn;
+    /// Which slots are marked as LIVE_OUT, coming out of each basic block.
+    BitVector LiveOut;
+  };
+
+  /// Maps active slots (per bit) for each basic block.
+  DenseMap<MachineBasicBlock*, BlockLifetimeInfo> BlockLiveness;
+
+  /// Maps serial numbers to basic blocks.
+  DenseMap<MachineBasicBlock*, int> BasicBlocks;
+  /// Maps basic blocks to a serial number.
+  SmallVector<MachineBasicBlock*, 8> BasicBlockNumbering;
+
+  /// Maps liveness intervals for each slot.
+  SmallVector<LiveInterval*, 16> Intervals;
+  /// VNInfo is used for the construction of LiveIntervals.
+  VNInfo::Allocator VNInfoAllocator;
+  /// SlotIndex analysis object.
+  SlotIndexes *Indexes;
+
+  /// The list of lifetime markers found. These markers are to be removed
+  /// once the coloring is done.
+  SmallVector<MachineInstr*, 8> Markers;
+
+  /// SlotSizeSorter - A Sort utility for arranging stack slots according
+  /// to their size.
+  struct SlotSizeSorter {
+    MachineFrameInfo *MFI;
+    SlotSizeSorter(MachineFrameInfo *mfi) : MFI(mfi) { }
+    bool operator()(int LHS, int RHS) {
+      // We use -1 to denote a uninteresting slot. Place these slots at the end.
+      if (LHS == -1) return false;
+      if (RHS == -1) return true;
+      // Sort according to size.
+      return MFI->getObjectSize(LHS) > MFI->getObjectSize(RHS);
+  }
+};
+
+public:
+  static char ID;
+  StackColoring() : MachineFunctionPass(ID) {
+    initializeStackColoringPass(*PassRegistry::getPassRegistry());
+  }
+  void getAnalysisUsage(AnalysisUsage &AU) const;
+  bool runOnMachineFunction(MachineFunction &MF);
+
+private:
+  /// Debug.
+  void dump();
+
+  /// Removes all of the lifetime marker instructions from the function.
+  /// \returns true if any markers were removed.
+  bool removeAllMarkers();
+
+  /// Scan the machine function and find all of the lifetime markers.
+  /// Record the findings in the BEGIN and END vectors.
+  /// \returns the number of markers found.
+  unsigned collectMarkers(unsigned NumSlot);
+
+  /// Perform the dataflow calculation and calculate the lifetime for each of
+  /// the slots, based on the BEGIN/END vectors. Set the LifetimeLIVE_IN and
+  /// LifetimeLIVE_OUT maps that represent which stack slots are live coming
+  /// in and out blocks.
+  void calculateLocalLiveness();
+
+  /// Construct the LiveIntervals for the slots.
+  void calculateLiveIntervals(unsigned NumSlots);
+
+  /// Go over the machine function and change instructions which use stack
+  /// slots to use the joint slots.
+  void remapInstructions(DenseMap<int, int> &SlotRemap);
+
+  /// The input program may contain intructions which are not inside lifetime
+  /// markers. This can happen due to a bug in the compiler or due to a bug in
+  /// user code (for example, returning a reference to a local variable).
+  /// This procedure checks all of the instructions in the function and
+  /// invalidates lifetime ranges which do not contain all of the instructions
+  /// which access that frame slot.
+  void removeInvalidSlotRanges();
+
+  /// Map entries which point to other entries to their destination.
+  ///   A->B->C becomes A->C.
+   void expungeSlotMap(DenseMap<int, int> &SlotRemap, unsigned NumSlots);
+};
+} // end anonymous namespace
+
+char StackColoring::ID = 0;
+char &llvm::StackColoringID = StackColoring::ID;
+
+INITIALIZE_PASS_BEGIN(StackColoring,
+                   "stack-coloring", "Merge disjoint stack slots", false, false)
+INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
+INITIALIZE_PASS_DEPENDENCY(SlotIndexes)
+INITIALIZE_PASS_END(StackColoring,
+                   "stack-coloring", "Merge disjoint stack slots", false, false)
+
+void StackColoring::getAnalysisUsage(AnalysisUsage &AU) const {
+  AU.addRequired<MachineDominatorTree>();
+  AU.addPreserved<MachineDominatorTree>();
+  AU.addRequired<SlotIndexes>();
+  MachineFunctionPass::getAnalysisUsage(AU);
+}
+
+void StackColoring::dump() {
+  for (df_iterator<MachineFunction*> FI = df_begin(MF), FE = df_end(MF);
+       FI != FE; ++FI) {
+    unsigned Num = BasicBlocks[*FI];
+    DEBUG(dbgs()<<"Inspecting block #"<<Num<<" ["<<FI->getName()<<"]\n");
+    Num = 0;
+    DEBUG(dbgs()<<"BEGIN  : {");
+    for (unsigned i=0; i < BlockLiveness[*FI].Begin.size(); ++i)
+      DEBUG(dbgs()<<BlockLiveness[*FI].Begin.test(i)<<" ");
+    DEBUG(dbgs()<<"}\n");
+
+    DEBUG(dbgs()<<"END    : {");
+    for (unsigned i=0; i < BlockLiveness[*FI].End.size(); ++i)
+      DEBUG(dbgs()<<BlockLiveness[*FI].End.test(i)<<" ");
+
+    DEBUG(dbgs()<<"}\n");
+
+    DEBUG(dbgs()<<"LIVE_IN: {");
+    for (unsigned i=0; i < BlockLiveness[*FI].LiveIn.size(); ++i)
+      DEBUG(dbgs()<<BlockLiveness[*FI].LiveIn.test(i)<<" ");
+
+    DEBUG(dbgs()<<"}\n");
+    DEBUG(dbgs()<<"LIVEOUT: {");
+    for (unsigned i=0; i < BlockLiveness[*FI].LiveOut.size(); ++i)
+      DEBUG(dbgs()<<BlockLiveness[*FI].LiveOut.test(i)<<" ");
+    DEBUG(dbgs()<<"}\n");
+  }
+}
+
+unsigned StackColoring::collectMarkers(unsigned NumSlot) {
+  unsigned MarkersFound = 0;
+  // Scan the function to find all lifetime markers.
+  // NOTE: We use the a reverse-post-order iteration to ensure that we obtain a
+  // deterministic numbering, and because we'll need a post-order iteration
+  // later for solving the liveness dataflow problem.
+  for (df_iterator<MachineFunction*> FI = df_begin(MF), FE = df_end(MF);
+       FI != FE; ++FI) {
+
+    // Assign a serial number to this basic block.
+    BasicBlocks[*FI] = BasicBlockNumbering.size();
+    BasicBlockNumbering.push_back(*FI);
+
+    BlockLiveness[*FI].Begin.resize(NumSlot);
+    BlockLiveness[*FI].End.resize(NumSlot);
+
+    for (MachineBasicBlock::iterator BI = (*FI)->begin(), BE = (*FI)->end();
+         BI != BE; ++BI) {
+
+      if (BI->getOpcode() != TargetOpcode::LIFETIME_START &&
+          BI->getOpcode() != TargetOpcode::LIFETIME_END)
+        continue;
+
+      Markers.push_back(BI);
+
+      bool IsStart = BI->getOpcode() == TargetOpcode::LIFETIME_START;
+      MachineOperand &MI = BI->getOperand(0);
+      unsigned Slot = MI.getIndex();
+
+      MarkersFound++;
+
+      const AllocaInst *Allocation = MFI->getObjectAllocation(Slot);
+      if (Allocation) {
+        DEBUG(dbgs()<<"Found a lifetime marker for slot #"<<Slot<<
+              " with allocation: "<< Allocation->getName()<<"\n");
+      }
+
+      if (IsStart) {
+        BlockLiveness[*FI].Begin.set(Slot);
+      } else {
+        if (BlockLiveness[*FI].Begin.test(Slot)) {
+          // Allocas that start and end within a single block are handled
+          // specially when computing the LiveIntervals to avoid pessimizing
+          // the liveness propagation.
+          BlockLiveness[*FI].Begin.reset(Slot);
+        } else {
+          BlockLiveness[*FI].End.set(Slot);
+        }
+      }
+    }
+  }
+
+  // Update statistics.
+  NumMarkerSeen += MarkersFound;
+  return MarkersFound;
+}
+
+void StackColoring::calculateLocalLiveness() {
+  // Perform a standard reverse dataflow computation to solve for
+  // global liveness.  The BEGIN set here is equivalent to KILL in the standard
+  // formulation, and END is equivalent to GEN.  The result of this computation
+  // is a map from blocks to bitvectors where the bitvectors represent which
+  // allocas are live in/out of that block.
+  SmallPtrSet<MachineBasicBlock*, 8> BBSet(BasicBlockNumbering.begin(),
+                                           BasicBlockNumbering.end());
+  unsigned NumSSMIters = 0;
+  bool changed = true;
+  while (changed) {
+    changed = false;
+    ++NumSSMIters;
+
+    SmallPtrSet<MachineBasicBlock*, 8> NextBBSet;
+
+    for (SmallVector<MachineBasicBlock*, 8>::iterator
+         PI = BasicBlockNumbering.begin(), PE = BasicBlockNumbering.end();
+         PI != PE; ++PI) {
+
+      MachineBasicBlock *BB = *PI;
+      if (!BBSet.count(BB)) continue;
+
+      BitVector LocalLiveIn;
+      BitVector LocalLiveOut;
+
+      // Forward propagation from begins to ends.
+      for (MachineBasicBlock::pred_iterator PI = BB->pred_begin(),
+           PE = BB->pred_end(); PI != PE; ++PI)
+        LocalLiveIn |= BlockLiveness[*PI].LiveOut;
+      LocalLiveIn |= BlockLiveness[BB].End;
+      LocalLiveIn.reset(BlockLiveness[BB].Begin);
+
+      // Reverse propagation from ends to begins.
+      for (MachineBasicBlock::succ_iterator SI = BB->succ_begin(),
+           SE = BB->succ_end(); SI != SE; ++SI)
+        LocalLiveOut |= BlockLiveness[*SI].LiveIn;
+      LocalLiveOut |= BlockLiveness[BB].Begin;
+      LocalLiveOut.reset(BlockLiveness[BB].End);
+
+      LocalLiveIn |= LocalLiveOut;
+      LocalLiveOut |= LocalLiveIn;
+
+      // After adopting the live bits, we need to turn-off the bits which
+      // are de-activated in this block.
+      LocalLiveOut.reset(BlockLiveness[BB].End);
+      LocalLiveIn.reset(BlockLiveness[BB].Begin);
+
+      // If we have both BEGIN and END markers in the same basic block then
+      // we know that the BEGIN marker comes after the END, because we already
+      // handle the case where the BEGIN comes before the END when collecting
+      // the markers (and building the BEGIN/END vectore).
+      // Want to enable the LIVE_IN and LIVE_OUT of slots that have both
+      // BEGIN and END because it means that the value lives before and after
+      // this basic block.
+      BitVector LocalEndBegin = BlockLiveness[BB].End;
+      LocalEndBegin &= BlockLiveness[BB].Begin;
+      LocalLiveIn |= LocalEndBegin;
+      LocalLiveOut |= LocalEndBegin;
+
+      if (LocalLiveIn.test(BlockLiveness[BB].LiveIn)) {
+        changed = true;
+        BlockLiveness[BB].LiveIn |= LocalLiveIn;
+
+        for (MachineBasicBlock::pred_iterator PI = BB->pred_begin(),
+             PE = BB->pred_end(); PI != PE; ++PI)
+          NextBBSet.insert(*PI);
+      }
+
+      if (LocalLiveOut.test(BlockLiveness[BB].LiveOut)) {
+        changed = true;
+        BlockLiveness[BB].LiveOut |= LocalLiveOut;
+
+        for (MachineBasicBlock::succ_iterator SI = BB->succ_begin(),
+             SE = BB->succ_end(); SI != SE; ++SI)
+          NextBBSet.insert(*SI);
+      }
+    }
+
+    BBSet = NextBBSet;
+  }// while changed.
+}
+
+void StackColoring::calculateLiveIntervals(unsigned NumSlots) {
+  SmallVector<SlotIndex, 16> Starts;
+  SmallVector<SlotIndex, 16> Finishes;
+
+  // For each block, find which slots are active within this block
+  // and update the live intervals.
+  for (MachineFunction::iterator MBB = MF->begin(), MBBe = MF->end();
+       MBB != MBBe; ++MBB) {
+    Starts.clear();
+    Starts.resize(NumSlots);
+    Finishes.clear();
+    Finishes.resize(NumSlots);
+
+    // Create the interval for the basic blocks with lifetime markers in them.
+    for (SmallVector<MachineInstr*, 8>::iterator it = Markers.begin(),
+         e = Markers.end(); it != e; ++it) {
+      MachineInstr *MI = *it;
+      if (MI->getParent() != MBB)
+        continue;
+
+      assert((MI->getOpcode() == TargetOpcode::LIFETIME_START ||
+              MI->getOpcode() == TargetOpcode::LIFETIME_END) &&
+             "Invalid Lifetime marker");
+
+      bool IsStart = MI->getOpcode() == TargetOpcode::LIFETIME_START;
+      MachineOperand &Mo = MI->getOperand(0);
+      int Slot = Mo.getIndex();
+      assert(Slot >= 0 && "Invalid slot");
+
+      SlotIndex ThisIndex = Indexes->getInstructionIndex(MI);
+
+      if (IsStart) {
+        if (!Starts[Slot].isValid() || Starts[Slot] > ThisIndex)
+          Starts[Slot] = ThisIndex;
+      } else {
+        if (!Finishes[Slot].isValid() || Finishes[Slot] < ThisIndex)
+          Finishes[Slot] = ThisIndex;
+      }
+    }
+
+    // Create the interval of the blocks that we previously found to be 'alive'.
+    BitVector Alive = BlockLiveness[MBB].LiveIn;
+    Alive |= BlockLiveness[MBB].LiveOut;
+
+    if (Alive.any()) {
+      for (int pos = Alive.find_first(); pos != -1;
+           pos = Alive.find_next(pos)) {
+        if (!Starts[pos].isValid())
+          Starts[pos] = Indexes->getMBBStartIdx(MBB);
+        if (!Finishes[pos].isValid())
+          Finishes[pos] = Indexes->getMBBEndIdx(MBB);
+      }
+    }
+
+    for (unsigned i = 0; i < NumSlots; ++i) {
+      assert(Starts[i].isValid() == Finishes[i].isValid() && "Unmatched range");
+      if (!Starts[i].isValid())
+        continue;
+
+      assert(Starts[i] && Finishes[i] && "Invalid interval");
+      VNInfo *ValNum = Intervals[i]->getValNumInfo(0);
+      SlotIndex S = Starts[i];
+      SlotIndex F = Finishes[i];
+      if (S < F) {
+        // We have a single consecutive region.
+        Intervals[i]->addRange(LiveRange(S, F, ValNum));
+      } else {
+        // We have two non consecutive regions. This happens when
+        // LIFETIME_START appears after the LIFETIME_END marker.
+        SlotIndex NewStart = Indexes->getMBBStartIdx(MBB);
+        SlotIndex NewFin = Indexes->getMBBEndIdx(MBB);
+        Intervals[i]->addRange(LiveRange(NewStart, F, ValNum));
+        Intervals[i]->addRange(LiveRange(S, NewFin, ValNum));
+      }
+    }
+  }
+}
+
+bool StackColoring::removeAllMarkers() {
+  unsigned Count = 0;
+  for (unsigned i = 0; i < Markers.size(); ++i) {
+    Markers[i]->eraseFromParent();
+    Count++;
+  }
+  Markers.clear();
+
+  DEBUG(dbgs()<<"Removed "<<Count<<" markers.\n");
+  return Count;
+}
+
+void StackColoring::remapInstructions(DenseMap<int, int> &SlotRemap) {
+  unsigned FixedInstr = 0;
+  unsigned FixedMemOp = 0;
+  unsigned FixedDbg = 0;
+  MachineModuleInfo *MMI = &MF->getMMI();
+
+  // Remap debug information that refers to stack slots.
+  MachineModuleInfo::VariableDbgInfoMapTy &VMap = MMI->getVariableDbgInfo();
+  for (MachineModuleInfo::VariableDbgInfoMapTy::iterator VI = VMap.begin(),
+       VE = VMap.end(); VI != VE; ++VI) {
+    const MDNode *Var = VI->first;
+    if (!Var) continue;
+    std::pair<unsigned, DebugLoc> &VP = VI->second;
+    if (SlotRemap.count(VP.first)) {
+      DEBUG(dbgs()<<"Remapping debug info for ["<<Var->getName()<<"].\n");
+      VP.first = SlotRemap[VP.first];
+      FixedDbg++;
+    }
+  }
+
+  // Keep a list of *allocas* which need to be remapped.
+  DenseMap<const AllocaInst*, const AllocaInst*> Allocas;
+  for (DenseMap<int, int>::iterator it = SlotRemap.begin(),
+       e = SlotRemap.end(); it != e; ++it) {
+    const AllocaInst *From = MFI->getObjectAllocation(it->first);
+    const AllocaInst *To = MFI->getObjectAllocation(it->second);
+    assert(To && From && "Invalid allocation object");
+    Allocas[From] = To;
+  }
+
+  // Remap all instructions to the new stack slots.
+  MachineFunction::iterator BB, BBE;
+  MachineBasicBlock::iterator I, IE;
+  for (BB = MF->begin(), BBE = MF->end(); BB != BBE; ++BB)
+    for (I = BB->begin(), IE = BB->end(); I != IE; ++I) {
+
+      // Skip lifetime markers. We'll remove them soon.
+      if (I->getOpcode() == TargetOpcode::LIFETIME_START ||
+          I->getOpcode() == TargetOpcode::LIFETIME_END)
+        continue;
+
+      // Update the MachineMemOperand to use the new alloca.
+      for (MachineInstr::mmo_iterator MM = I->memoperands_begin(),
+           E = I->memoperands_end(); MM != E; ++MM) {
+        MachineMemOperand *MMO = *MM;
+
+        const Value *V = MMO->getValue();
+
+        if (!V)
+          continue;
+
+        // Climb up and find the original alloca.
+        V = GetUnderlyingObject(V);
+        // If we did not find one, or if the one that we found is not in our
+        // map, then move on.
+        if (!V || !isa<AllocaInst>(V)) {
+          // Clear mem operand since we don't know for sure that it doesn't
+          // alias a merged alloca.
+          MMO->setValue(0);
+          continue;
+        }
+        const AllocaInst *AI= cast<AllocaInst>(V);
+        if (!Allocas.count(AI))
+          continue;
+
+        MMO->setValue(Allocas[AI]);
+        FixedMemOp++;
+      }
+
+      // Update all of the machine instruction operands.
+      for (unsigned i = 0 ; i <  I->getNumOperands(); ++i) {
+        MachineOperand &MO = I->getOperand(i);
+
+        if (!MO.isFI())
+          continue;
+        int FromSlot = MO.getIndex();
+
+        // Don't touch arguments.
+        if (FromSlot<0)
+          continue;
+
+        // Only look at mapped slots.
+        if (!SlotRemap.count(FromSlot))
+          continue;
+
+        // In a debug build, check that the instruction that we are modifying is
+        // inside the expected live range. If the instruction is not inside
+        // the calculated range then it means that the alloca usage moved
+        // outside of the lifetime markers, or that the user has a bug.
+        // NOTE: Alloca address calculations which happen outside the lifetime
+        // zone are are okay, despite the fact that we don't have a good way
+        // for validating all of the usages of the calculation.
+#ifndef NDEBUG
+        bool TouchesMemory = I->mayLoad() || I->mayStore();
+        // If we *don't* protect the user from escaped allocas, don't bother
+        // validating the instructions.
+        if (!I->isDebugValue() && TouchesMemory && ProtectFromEscapedAllocas) {
+          SlotIndex Index = Indexes->getInstructionIndex(I);
+          LiveInterval *Interval = Intervals[FromSlot];
+          assert(Interval->find(Index) != Interval->end() &&
+               "Found instruction usage outside of live range.");
+        }
+#endif
+
+        // Fix the machine instructions.
+        int ToSlot = SlotRemap[FromSlot];
+        MO.setIndex(ToSlot);
+        FixedInstr++;
+      }
+    }
+
+  DEBUG(dbgs()<<"Fixed "<<FixedMemOp<<" machine memory operands.\n");
+  DEBUG(dbgs()<<"Fixed "<<FixedDbg<<" debug locations.\n");
+  DEBUG(dbgs()<<"Fixed "<<FixedInstr<<" machine instructions.\n");
+}
+
+void StackColoring::removeInvalidSlotRanges() {
+  MachineFunction::iterator BB, BBE;
+  MachineBasicBlock::iterator I, IE;
+  for (BB = MF->begin(), BBE = MF->end(); BB != BBE; ++BB)
+    for (I = BB->begin(), IE = BB->end(); I != IE; ++I) {
+
+      if (I->getOpcode() == TargetOpcode::LIFETIME_START ||
+          I->getOpcode() == TargetOpcode::LIFETIME_END || I->isDebugValue())
+        continue;
+
+      // Some intervals are suspicious! In some cases we find address
+      // calculations outside of the lifetime zone, but not actual memory
+      // read or write. Memory accesses outside of the lifetime zone are a clear
+      // violation, but address calculations are okay. This can happen when
+      // GEPs are hoisted outside of the lifetime zone.
+      // So, in here we only check instructions which can read or write memory.
+      if (!I->mayLoad() && !I->mayStore())
+        continue;
+
+      // Check all of the machine operands.
+      for (unsigned i = 0 ; i <  I->getNumOperands(); ++i) {
+        MachineOperand &MO = I->getOperand(i);
+
+        if (!MO.isFI())
+          continue;
+
+        int Slot = MO.getIndex();
+
+        if (Slot<0)
+          continue;
+
+        if (Intervals[Slot]->empty())
+          continue;
+
+        // Check that the used slot is inside the calculated lifetime range.
+        // If it is not, warn about it and invalidate the range.
+        LiveInterval *Interval = Intervals[Slot];
+        SlotIndex Index = Indexes->getInstructionIndex(I);
+        if (Interval->find(Index) == Interval->end()) {
+          Intervals[Slot]->clear();
+          DEBUG(dbgs()<<"Invalidating range #"<<Slot<<"\n");
+          EscapedAllocas++;
+        }
+      }
+    }
+}
+
+void StackColoring::expungeSlotMap(DenseMap<int, int> &SlotRemap,
+                                   unsigned NumSlots) {
+  // Expunge slot remap map.
+  for (unsigned i=0; i < NumSlots; ++i) {
+    // If we are remapping i
+    if (SlotRemap.count(i)) {
+      int Target = SlotRemap[i];
+      // As long as our target is mapped to something else, follow it.
+      while (SlotRemap.count(Target)) {
+        Target = SlotRemap[Target];
+        SlotRemap[i] = Target;
+      }
+    }
+  }
+}
+
+bool StackColoring::runOnMachineFunction(MachineFunction &Func) {
+  DEBUG(dbgs() << "********** Stack Coloring **********\n"
+               << "********** Function: "
+               << ((const Value*)Func.getFunction())->getName() << '\n');
+  MF = &Func;
+  MFI = MF->getFrameInfo();
+  Indexes = &getAnalysis<SlotIndexes>();
+  BlockLiveness.clear();
+  BasicBlocks.clear();
+  BasicBlockNumbering.clear();
+  Markers.clear();
+  Intervals.clear();
+  VNInfoAllocator.Reset();
+
+  unsigned NumSlots = MFI->getObjectIndexEnd();
+
+  // If there are no stack slots then there are no markers to remove.
+  if (!NumSlots)
+    return false;
+
+  SmallVector<int, 8> SortedSlots;
+
+  SortedSlots.reserve(NumSlots);
+  Intervals.reserve(NumSlots);
+
+  unsigned NumMarkers = collectMarkers(NumSlots);
+
+  unsigned TotalSize = 0;
+  DEBUG(dbgs()<<"Found "<<NumMarkers<<" markers and "<<NumSlots<<" slots\n");
+  DEBUG(dbgs()<<"Slot structure:\n");
+
+  for (int i=0; i < MFI->getObjectIndexEnd(); ++i) {
+    DEBUG(dbgs()<<"Slot #"<<i<<" - "<<MFI->getObjectSize(i)<<" bytes.\n");
+    TotalSize += MFI->getObjectSize(i);
+  }
+
+  DEBUG(dbgs()<<"Total Stack size: "<<TotalSize<<" bytes\n\n");
+
+  // Don't continue because there are not enough lifetime markers, or the
+  // stack is too small, or we are told not to optimize the slots.
+  if (NumMarkers < 2 || TotalSize < 16 || DisableColoring) {
+    DEBUG(dbgs()<<"Will not try to merge slots.\n");
+    return removeAllMarkers();
+  }
+
+  for (unsigned i=0; i < NumSlots; ++i) {
+    LiveInterval *LI = new LiveInterval(i, 0);
+    Intervals.push_back(LI);
+    LI->getNextValue(Indexes->getZeroIndex(), VNInfoAllocator);
+    SortedSlots.push_back(i);
+  }
+
+  // Calculate the liveness of each block.
+  calculateLocalLiveness();
+
+  // Propagate the liveness information.
+  calculateLiveIntervals(NumSlots);
+
+  // Search for allocas which are used outside of the declared lifetime
+  // markers.
+  if (ProtectFromEscapedAllocas)
+    removeInvalidSlotRanges();
+
+  // Maps old slots to new slots.
+  DenseMap<int, int> SlotRemap;
+  unsigned RemovedSlots = 0;
+  unsigned ReducedSize = 0;
+
+  // Do not bother looking at empty intervals.
+  for (unsigned I = 0; I < NumSlots; ++I) {
+    if (Intervals[SortedSlots[I]]->empty())
+      SortedSlots[I] = -1;
+  }
+
+  // This is a simple greedy algorithm for merging allocas. First, sort the
+  // slots, placing the largest slots first. Next, perform an n^2 scan and look
+  // for disjoint slots. When you find disjoint slots, merge the samller one
+  // into the bigger one and update the live interval. Remove the small alloca
+  // and continue.
+
+  // Sort the slots according to their size. Place unused slots at the end.
+  std::sort(SortedSlots.begin(), SortedSlots.end(), SlotSizeSorter(MFI));
+
+  bool Chanded = true;
+  while (Chanded) {
+    Chanded = false;
+    for (unsigned I = 0; I < NumSlots; ++I) {
+      if (SortedSlots[I] == -1)
+        continue;
+
+      for (unsigned J=I+1; J < NumSlots; ++J) {
+        if (SortedSlots[J] == -1)
+          continue;
+
+        int FirstSlot = SortedSlots[I];
+        int SecondSlot = SortedSlots[J];
+        LiveInterval *First = Intervals[FirstSlot];
+        LiveInterval *Second = Intervals[SecondSlot];
+        assert (!First->empty() && !Second->empty() && "Found an empty range");
+
+        // Merge disjoint slots.
+        if (!First->overlaps(*Second)) {
+          Chanded = true;
+          First->MergeRangesInAsValue(*Second, First->getValNumInfo(0));
+          SlotRemap[SecondSlot] = FirstSlot;
+          SortedSlots[J] = -1;
+          DEBUG(dbgs()<<"Merging #"<<FirstSlot<<" and slots #"<<
+                SecondSlot<<" together.\n");
+          unsigned MaxAlignment = std::max(MFI->getObjectAlignment(FirstSlot),
+                                           MFI->getObjectAlignment(SecondSlot));
+
+          assert(MFI->getObjectSize(FirstSlot) >=
+                 MFI->getObjectSize(SecondSlot) &&
+                 "Merging a small object into a larger one");
+
+          RemovedSlots+=1;
+          ReducedSize += MFI->getObjectSize(SecondSlot);
+          MFI->setObjectAlignment(FirstSlot, MaxAlignment);
+          MFI->RemoveStackObject(SecondSlot);
+        }
+      }
+    }
+  }// While changed.
+
+  // Record statistics.
+  StackSpaceSaved += ReducedSize;
+  StackSlotMerged += RemovedSlots;
+  DEBUG(dbgs()<<"Merge "<<RemovedSlots<<" slots. Saved "<<
+        ReducedSize<<" bytes\n");
+
+  // Scan the entire function and update all machine operands that use frame
+  // indices to use the remapped frame index.
+  expungeSlotMap(SlotRemap, NumSlots);
+  remapInstructions(SlotRemap);
+
+  // Release the intervals.
+  for (unsigned I = 0; I < NumSlots; ++I) {
+    delete Intervals[I];
+  }
+
+  return removeAllMarkers();
+}
diff --git a/lib/CodeGen/StackProtector.cpp b/lib/CodeGen/StackProtector.cpp
index f1eab1f..31e9ec0 100644
--- a/lib/CodeGen/StackProtector.cpp
+++ b/lib/CodeGen/StackProtector.cpp
@@ -26,18 +26,12 @@
 #include "llvm/Module.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/CommandLine.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetLowering.h"
+#include "llvm/Target/TargetOptions.h"
 #include "llvm/ADT/Triple.h"
 using namespace llvm;
 
-// SSPBufferSize - The lower bound for a buffer to be considered for stack
-// smashing protection.
-static cl::opt<unsigned>
-SSPBufferSize("stack-protector-buffer-size", cl::init(8),
-              cl::desc("Lower bound for a buffer to be considered for "
-                       "stack protection"));
-
 namespace {
   class StackProtector : public FunctionPass {
     /// TLI - Keep a pointer of a TargetLowering to consult for determining
@@ -61,6 +55,11 @@ namespace {
     /// check fails.
     BasicBlock *CreateFailBB();
 
+    /// ContainsProtectableArray - Check whether the type either is an array or
+    /// contains an array of sufficient size so that we need stack protectors
+    /// for it.
+    bool ContainsProtectableArray(Type *Ty, bool InStruct = false) const;
+
     /// RequiresStackProtector - Check whether or not this function needs a
     /// stack protector based upon the stack protector level.
     bool RequiresStackProtector() const;
@@ -100,21 +99,50 @@ bool StackProtector::runOnFunction(Function &Fn) {
   return InsertStackProtectors();
 }
 
+/// ContainsProtectableArray - Check whether the type either is an array or
+/// contains a char array of sufficient size so that we need stack protectors
+/// for it.
+bool StackProtector::ContainsProtectableArray(Type *Ty, bool InStruct) const {
+  if (!Ty) return false;
+  if (ArrayType *AT = dyn_cast<ArrayType>(Ty)) {
+    const TargetMachine &TM = TLI->getTargetMachine();
+    if (!AT->getElementType()->isIntegerTy(8)) {
+      Triple Trip(TM.getTargetTriple());
+
+      // If we're on a non-Darwin platform or we're inside of a structure, don't
+      // add stack protectors unless the array is a character array.
+      if (InStruct || !Trip.isOSDarwin())
+          return false;
+    }
+
+    // If an array has more than SSPBufferSize bytes of allocated space, then we
+    // emit stack protectors.
+    if (TM.Options.SSPBufferSize <= TLI->getDataLayout()->getTypeAllocSize(AT))
+      return true;
+  }
+
+  const StructType *ST = dyn_cast<StructType>(Ty);
+  if (!ST) return false;
+
+  for (StructType::element_iterator I = ST->element_begin(),
+         E = ST->element_end(); I != E; ++I)
+    if (ContainsProtectableArray(*I, true))
+      return true;
+
+  return false;
+}
+
 /// RequiresStackProtector - Check whether or not this function needs a stack
 /// protector based upon the stack protector level. The heuristic we use is to
 /// add a guard variable to functions that call alloca, and functions with
 /// buffers larger than SSPBufferSize bytes.
 bool StackProtector::RequiresStackProtector() const {
-  if (F->hasFnAttr(Attribute::StackProtectReq))
+  if (F->getFnAttributes().hasAttribute(Attributes::StackProtectReq))
     return true;
 
-  if (!F->hasFnAttr(Attribute::StackProtect))
+  if (!F->getFnAttributes().hasAttribute(Attributes::StackProtect))
     return false;
 
-  const TargetData *TD = TLI->getTargetData();
-  const TargetMachine &TM = TLI->getTargetMachine();
-  Triple Trip(TM.getTargetTriple());
-
   for (Function::iterator I = F->begin(), E = F->end(); I != E; ++I) {
     BasicBlock *BB = I;
 
@@ -126,17 +154,8 @@ bool StackProtector::RequiresStackProtector() const {
           // protectors.
           return true;
 
-        if (ArrayType *AT = dyn_cast<ArrayType>(AI->getAllocatedType())) {
-          // If we're on a non-Darwin platform, don't add stack protectors
-          // unless the array is a character array.
-          if (!Trip.isOSDarwin() && !AT->getElementType()->isIntegerTy(8))
-            continue;
-
-          // If an array has more than SSPBufferSize bytes of allocated space,
-          // then we emit stack protectors.
-          if (SSPBufferSize <= TD->getTypeAllocSize(AT))
-            return true;
-        }
+        if (ContainsProtectableArray(AI->getAllocatedType()))
+          return true;
       }
   }
 
diff --git a/lib/CodeGen/StackSlotColoring.cpp b/lib/CodeGen/StackSlotColoring.cpp
index 20da36e..d349abc 100644
--- a/lib/CodeGen/StackSlotColoring.cpp
+++ b/lib/CodeGen/StackSlotColoring.cpp
@@ -11,8 +11,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#define DEBUG_TYPE "stackcoloring"
-#include "llvm/Function.h"
+#define DEBUG_TYPE "stackslotcoloring"
 #include "llvm/Module.h"
 #include "llvm/CodeGen/Passes.h"
 #include "llvm/CodeGen/LiveIntervalAnalysis.h"
@@ -391,8 +390,7 @@ bool StackSlotColoring::RemoveDeadStores(MachineBasicBlock* MBB) {
 bool StackSlotColoring::runOnMachineFunction(MachineFunction &MF) {
   DEBUG({
       dbgs() << "********** Stack Slot Coloring **********\n"
-             << "********** Function: "
-             << MF.getFunction()->getName() << '\n';
+             << "********** Function: " << MF.getName() << '\n';
     });
 
   MFI = MF.getFrameInfo();
diff --git a/lib/CodeGen/StrongPHIElimination.cpp b/lib/CodeGen/StrongPHIElimination.cpp
index 5b06195..39fd600 100644
--- a/lib/CodeGen/StrongPHIElimination.cpp
+++ b/lib/CodeGen/StrongPHIElimination.cpp
@@ -404,9 +404,9 @@ bool StrongPHIElimination::runOnMachineFunction(MachineFunction &MF) {
 }
 
 void StrongPHIElimination::addReg(unsigned Reg) {
-  if (RegNodeMap.count(Reg))
-    return;
-  RegNodeMap[Reg] = new (Allocator) Node(Reg);
+  Node *&N = RegNodeMap[Reg];
+  if (!N)
+    N = new (Allocator) Node(Reg);
 }
 
 StrongPHIElimination::Node*
@@ -714,8 +714,9 @@ void StrongPHIElimination::InsertCopiesForPHI(MachineInstr *PHI,
         assert(getRegColor(CopyReg) == CopyReg);
       }
 
-      if (!InsertedSrcCopyMap.count(std::make_pair(PredBB, PHIColor)))
-        InsertedSrcCopyMap[std::make_pair(PredBB, PHIColor)] = CopyInstr;
+      // Insert into map if not already there.
+      InsertedSrcCopyMap.insert(std::make_pair(std::make_pair(PredBB, PHIColor),
+                                               CopyInstr));
     }
 
     SrcMO.setReg(CopyReg);
diff --git a/lib/CodeGen/TailDuplication.cpp b/lib/CodeGen/TailDuplication.cpp
index a813fa6..1497d1b 100644
--- a/lib/CodeGen/TailDuplication.cpp
+++ b/lib/CodeGen/TailDuplication.cpp
@@ -552,7 +552,8 @@ TailDuplicatePass::shouldTailDuplicate(const MachineFunction &MF,
   // compensate for the duplication.
   unsigned MaxDuplicateCount;
   if (TailDuplicateSize.getNumOccurrences() == 0 &&
-      MF.getFunction()->hasFnAttr(Attribute::OptimizeForSize))
+      MF.getFunction()->getFnAttributes().
+        hasAttribute(Attributes::OptimizeForSize))
     MaxDuplicateCount = 1;
   else
     MaxDuplicateCount = TailDuplicateSize;
diff --git a/lib/CodeGen/TargetInstrInfoImpl.cpp b/lib/CodeGen/TargetInstrInfoImpl.cpp
index ddee6b2..4439192 100644
--- a/lib/CodeGen/TargetInstrInfoImpl.cpp
+++ b/lib/CodeGen/TargetInstrInfoImpl.cpp
@@ -99,17 +99,8 @@ MachineInstr *TargetInstrInfoImpl::commuteInstruction(MachineInstr *MI,
 
   if (NewMI) {
     // Create a new instruction.
-    bool Reg0IsDead = HasDef ? MI->getOperand(0).isDead() : false;
     MachineFunction &MF = *MI->getParent()->getParent();
-    if (HasDef)
-      return BuildMI(MF, MI->getDebugLoc(), MI->getDesc())
-        .addReg(Reg0, RegState::Define | getDeadRegState(Reg0IsDead), SubReg0)
-        .addReg(Reg2, getKillRegState(Reg2IsKill), SubReg2)
-        .addReg(Reg1, getKillRegState(Reg1IsKill), SubReg1);
-    else
-      return BuildMI(MF, MI->getDebugLoc(), MI->getDesc())
-        .addReg(Reg2, getKillRegState(Reg2IsKill), SubReg2)
-        .addReg(Reg1, getKillRegState(Reg1IsKill), SubReg1);
+    MI = MF.CloneMachineInstr(MI);
   }
 
   if (HasDef) {
@@ -572,6 +563,8 @@ TargetInstrInfoImpl::getNumMicroOps(const InstrItineraryData *ItinData,
 /// Return the default expected latency for a def based on it's opcode.
 unsigned TargetInstrInfo::defaultDefLatency(const MCSchedModel *SchedModel,
                                             const MachineInstr *DefMI) const {
+  if (DefMI->isTransient())
+    return 0;
   if (DefMI->mayLoad())
     return SchedModel->LoadLatency;
   if (isHighLatencyDef(DefMI->getOpcode()))
@@ -615,13 +608,13 @@ getOperandLatency(const InstrItineraryData *ItinData,
 
 /// If we can determine the operand latency from the def only, without itinerary
 /// lookup, do so. Otherwise return -1.
-static int computeDefOperandLatency(
-  const TargetInstrInfo *TII, const InstrItineraryData *ItinData,
-  const MachineInstr *DefMI, bool FindMin) {
+int TargetInstrInfo::computeDefOperandLatency(
+  const InstrItineraryData *ItinData,
+  const MachineInstr *DefMI, bool FindMin) const {
 
   // Let the target hook getInstrLatency handle missing itineraries.
   if (!ItinData)
-    return TII->getInstrLatency(ItinData, DefMI);
+    return getInstrLatency(ItinData, DefMI);
 
   // Return a latency based on the itinerary properties and defining instruction
   // if possible. Some common subtargets don't require per-operand latency,
@@ -630,7 +623,7 @@ static int computeDefOperandLatency(
     // If MinLatency is valid, call getInstrLatency. This uses Stage latency if
     // it exists before defaulting to MinLatency.
     if (ItinData->SchedModel->MinLatency >= 0)
-      return TII->getInstrLatency(ItinData, DefMI);
+      return getInstrLatency(ItinData, DefMI);
 
     // If MinLatency is invalid, OperandLatency is interpreted as MinLatency.
     // For empty itineraries, short-cirtuit the check and default to one cycle.
@@ -638,29 +631,42 @@ static int computeDefOperandLatency(
       return 1;
   }
   else if(ItinData->isEmpty())
-    return TII->defaultDefLatency(ItinData->SchedModel, DefMI);
+    return defaultDefLatency(ItinData->SchedModel, DefMI);
 
   // ...operand lookup required
   return -1;
 }
 
 /// computeOperandLatency - Compute and return the latency of the given data
-/// dependent def and use when the operand indices are already known.
+/// dependent def and use when the operand indices are already known. UseMI may
+/// be NULL for an unknown use.
+///
+/// FindMin may be set to get the minimum vs. expected latency. Minimum
+/// latency is used for scheduling groups, while expected latency is for
+/// instruction cost and critical path.
 ///
-/// FindMin may be set to get the minimum vs. expected latency.
+/// Depending on the subtarget's itinerary properties, this may or may not need
+/// to call getOperandLatency(). For most subtargets, we don't need DefIdx or
+/// UseIdx to compute min latency.
 unsigned TargetInstrInfo::
 computeOperandLatency(const InstrItineraryData *ItinData,
                       const MachineInstr *DefMI, unsigned DefIdx,
                       const MachineInstr *UseMI, unsigned UseIdx,
                       bool FindMin) const {
 
-  int DefLatency = computeDefOperandLatency(this, ItinData, DefMI, FindMin);
+  int DefLatency = computeDefOperandLatency(ItinData, DefMI, FindMin);
   if (DefLatency >= 0)
     return DefLatency;
 
   assert(ItinData && !ItinData->isEmpty() && "computeDefOperandLatency fail");
 
-  int OperLatency = getOperandLatency(ItinData, DefMI, DefIdx, UseMI, UseIdx);
+  int OperLatency = 0;
+  if (UseMI)
+    OperLatency = getOperandLatency(ItinData, DefMI, DefIdx, UseMI, UseIdx);
+  else {
+    unsigned DefClass = DefMI->getDesc().getSchedClass();
+    OperLatency = ItinData->getOperandCycle(DefClass, DefIdx);
+  }
   if (OperLatency >= 0)
     return OperLatency;
 
@@ -673,77 +679,3 @@ computeOperandLatency(const InstrItineraryData *ItinData,
                             defaultDefLatency(ItinData->SchedModel, DefMI));
   return InstrLatency;
 }
-
-/// computeOperandLatency - Compute and return the latency of the given data
-/// dependent def and use. DefMI must be a valid def. UseMI may be NULL for an
-/// unknown use. Depending on the subtarget's itinerary properties, this may or
-/// may not need to call getOperandLatency().
-///
-/// FindMin may be set to get the minimum vs. expected latency. Minimum
-/// latency is used for scheduling groups, while expected latency is for
-/// instruction cost and critical path.
-///
-/// For most subtargets, we don't need DefIdx or UseIdx to compute min latency.
-/// DefMI must be a valid definition, but UseMI may be NULL for an unknown use.
-unsigned TargetInstrInfo::
-computeOperandLatency(const InstrItineraryData *ItinData,
-                      const TargetRegisterInfo *TRI,
-                      const MachineInstr *DefMI, const MachineInstr *UseMI,
-                      unsigned Reg, bool FindMin) const {
-
-  int DefLatency = computeDefOperandLatency(this, ItinData, DefMI, FindMin);
-  if (DefLatency >= 0)
-    return DefLatency;
-
-  assert(ItinData && !ItinData->isEmpty() && "computeDefOperandLatency fail");
-
-  // Find the definition of the register in the defining instruction.
-  int DefIdx = DefMI->findRegisterDefOperandIdx(Reg);
-  if (DefIdx != -1) {
-    const MachineOperand &MO = DefMI->getOperand(DefIdx);
-    if (MO.isReg() && MO.isImplicit() &&
-        DefIdx >= (int)DefMI->getDesc().getNumOperands()) {
-      // This is an implicit def, getOperandLatency() won't return the correct
-      // latency. e.g.
-      //   %D6<def>, %D7<def> = VLD1q16 %R2<kill>, 0, ..., %Q3<imp-def>
-      //   %Q1<def> = VMULv8i16 %Q1<kill>, %Q3<kill>, ...
-      // What we want is to compute latency between def of %D6/%D7 and use of
-      // %Q3 instead.
-      unsigned Op2 = DefMI->findRegisterDefOperandIdx(Reg, false, true, TRI);
-      if (DefMI->getOperand(Op2).isReg())
-        DefIdx = Op2;
-    }
-    // For all uses of the register, calculate the maxmimum latency
-    int OperLatency = -1;
-
-    // UseMI is null, then it must be a scheduling barrier.
-    if (!UseMI) {
-      unsigned DefClass = DefMI->getDesc().getSchedClass();
-      OperLatency = ItinData->getOperandCycle(DefClass, DefIdx);
-    }
-    else {
-      for (unsigned i = 0, e = UseMI->getNumOperands(); i != e; ++i) {
-        const MachineOperand &MO = UseMI->getOperand(i);
-        if (!MO.isReg() || !MO.isUse())
-          continue;
-        unsigned MOReg = MO.getReg();
-        if (MOReg != Reg)
-          continue;
-
-        int UseCycle = getOperandLatency(ItinData, DefMI, DefIdx, UseMI, i);
-        OperLatency = std::max(OperLatency, UseCycle);
-      }
-    }
-    // If we found an operand latency, we're done.
-    if (OperLatency >= 0)
-      return OperLatency;
-  }
-  // No operand latency was found.
-  unsigned InstrLatency = getInstrLatency(ItinData, DefMI);
-
-  // Expected latency is the max of the stage latency and itinerary props.
-  if (!FindMin)
-    InstrLatency = std::max(InstrLatency,
-                            defaultDefLatency(ItinData->SchedModel, DefMI));
-  return InstrLatency;
-}
diff --git a/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
index 2a2fa9e..8f5d770 100644
--- a/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
+++ b/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
@@ -27,7 +27,7 @@
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/Target/Mangler.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
 #include "llvm/Support/Dwarf.h"
@@ -77,9 +77,9 @@ void TargetLoweringObjectFileELF::emitPersonalityValue(MCStreamer &Streamer,
                                                     Flags,
                                                     SectionKind::getDataRel(),
                                                     0, Label->getName());
-  unsigned Size = TM.getTargetData()->getPointerSize();
+  unsigned Size = TM.getDataLayout()->getPointerSize();
   Streamer.SwitchSection(Sec);
-  Streamer.EmitValueToAlignment(TM.getTargetData()->getPointerABIAlignment());
+  Streamer.EmitValueToAlignment(TM.getDataLayout()->getPointerABIAlignment());
   Streamer.EmitSymbolAttribute(Label, MCSA_ELF_TypeObject);
   const MCExpr *E = MCConstantExpr::Create(Size, getContext());
   Streamer.EmitELFSize(Label, E);
@@ -247,7 +247,7 @@ SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind,
     // FIXME: this is getting the alignment of the character, not the
     // alignment of the global!
     unsigned Align =
-      TM.getTargetData()->getPreferredAlignment(cast<GlobalVariable>(GV));
+      TM.getDataLayout()->getPreferredAlignment(cast<GlobalVariable>(GV));
 
     const char *SizeSpec = ".rodata.str1.";
     if (Kind.isMergeable2ByteCString())
@@ -522,14 +522,14 @@ SelectSectionForGlobal(const GlobalValue *GV, SectionKind Kind,
 
   // FIXME: Alignment check should be handled by section classifier.
   if (Kind.isMergeable1ByteCString() &&
-      TM.getTargetData()->getPreferredAlignment(cast<GlobalVariable>(GV)) < 32)
+      TM.getDataLayout()->getPreferredAlignment(cast<GlobalVariable>(GV)) < 32)
     return CStringSection;
 
   // Do not put 16-bit arrays in the UString section if they have an
   // externally visible label, this runs into issues with certain linker
   // versions.
   if (Kind.isMergeable2ByteCString() && !GV->hasExternalLinkage() &&
-      TM.getTargetData()->getPreferredAlignment(cast<GlobalVariable>(GV)) < 32)
+      TM.getDataLayout()->getPreferredAlignment(cast<GlobalVariable>(GV)) < 32)
     return UStringSection;
 
   if (Kind.isMergeableConst()) {
diff --git a/lib/CodeGen/TargetSchedule.cpp b/lib/CodeGen/TargetSchedule.cpp
new file mode 100644
index 0000000..ca3b0e0
--- /dev/null
+++ b/lib/CodeGen/TargetSchedule.cpp
@@ -0,0 +1,306 @@
+//===-- llvm/Target/TargetSchedule.cpp - Sched Machine Model ----*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a wrapper around MCSchedModel that allows the interface
+// to benefit from information currently only available in TargetInstrInfo.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CodeGen/TargetSchedule.h"
+#include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/TargetRegisterInfo.h"
+#include "llvm/Target/TargetSubtargetInfo.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+
+static cl::opt<bool> EnableSchedModel("schedmodel", cl::Hidden, cl::init(true),
+  cl::desc("Use TargetSchedModel for latency lookup"));
+
+static cl::opt<bool> EnableSchedItins("scheditins", cl::Hidden, cl::init(true),
+  cl::desc("Use InstrItineraryData for latency lookup"));
+
+bool TargetSchedModel::hasInstrSchedModel() const {
+  return EnableSchedModel && SchedModel.hasInstrSchedModel();
+}
+
+bool TargetSchedModel::hasInstrItineraries() const {
+  return EnableSchedItins && !InstrItins.isEmpty();
+}
+
+static unsigned gcd(unsigned Dividend, unsigned Divisor) {
+  // Dividend and Divisor will be naturally swapped as needed.
+  while(Divisor) {
+    unsigned Rem = Dividend % Divisor;
+    Dividend = Divisor;
+    Divisor = Rem;
+  };
+  return Dividend;
+}
+static unsigned lcm(unsigned A, unsigned B) {
+  unsigned LCM = (uint64_t(A) * B) / gcd(A, B);
+  assert((LCM >= A && LCM >= B) && "LCM overflow");
+  return LCM;
+}
+
+void TargetSchedModel::init(const MCSchedModel &sm,
+                            const TargetSubtargetInfo *sti,
+                            const TargetInstrInfo *tii) {
+  SchedModel = sm;
+  STI = sti;
+  TII = tii;
+  STI->initInstrItins(InstrItins);
+
+  unsigned NumRes = SchedModel.getNumProcResourceKinds();
+  ResourceFactors.resize(NumRes);
+  ResourceLCM = SchedModel.IssueWidth;
+  for (unsigned Idx = 0; Idx < NumRes; ++Idx) {
+    unsigned NumUnits = SchedModel.getProcResource(Idx)->NumUnits;
+    if (NumUnits > 0)
+      ResourceLCM = lcm(ResourceLCM, NumUnits);
+  }
+  MicroOpFactor = ResourceLCM / SchedModel.IssueWidth;
+  for (unsigned Idx = 0; Idx < NumRes; ++Idx) {
+    unsigned NumUnits = SchedModel.getProcResource(Idx)->NumUnits;
+    ResourceFactors[Idx] = NumUnits ? (ResourceLCM / NumUnits) : 0;
+  }
+}
+
+unsigned TargetSchedModel::getNumMicroOps(const MachineInstr *MI,
+                                          const MCSchedClassDesc *SC) const {
+  if (hasInstrItineraries()) {
+    int UOps = InstrItins.getNumMicroOps(MI->getDesc().getSchedClass());
+    return (UOps >= 0) ? UOps : TII->getNumMicroOps(&InstrItins, MI);
+  }
+  if (hasInstrSchedModel()) {
+    if (!SC)
+      SC = resolveSchedClass(MI);
+    if (SC->isValid())
+      return SC->NumMicroOps;
+  }
+  return MI->isTransient() ? 0 : 1;
+}
+
+// The machine model may explicitly specify an invalid latency, which
+// effectively means infinite latency. Since users of the TargetSchedule API
+// don't know how to handle this, we convert it to a very large latency that is
+// easy to distinguish when debugging the DAG but won't induce overflow.
+static unsigned convertLatency(int Cycles) {
+  return Cycles >= 0 ? Cycles : 1000;
+}
+
+/// If we can determine the operand latency from the def only, without machine
+/// model or itinerary lookup, do so. Otherwise return -1.
+int TargetSchedModel::getDefLatency(const MachineInstr *DefMI,
+                                    bool FindMin) const {
+
+  // Return a latency based on the itinerary properties and defining instruction
+  // if possible. Some common subtargets don't require per-operand latency,
+  // especially for minimum latencies.
+  if (FindMin) {
+    // If MinLatency is invalid, then use the itinerary for MinLatency. If no
+    // itinerary exists either, then use single cycle latency.
+    if (SchedModel.MinLatency < 0 && !hasInstrItineraries()) {
+      return 1;
+    }
+    return SchedModel.MinLatency;
+  }
+  else if (!hasInstrSchedModel() && !hasInstrItineraries()) {
+    return TII->defaultDefLatency(&SchedModel, DefMI);
+  }
+  // ...operand lookup required
+  return -1;
+}
+
+/// Return the MCSchedClassDesc for this instruction. Some SchedClasses require
+/// evaluation of predicates that depend on instruction operands or flags.
+const MCSchedClassDesc *TargetSchedModel::
+resolveSchedClass(const MachineInstr *MI) const {
+
+  // Get the definition's scheduling class descriptor from this machine model.
+  unsigned SchedClass = MI->getDesc().getSchedClass();
+  const MCSchedClassDesc *SCDesc = SchedModel.getSchedClassDesc(SchedClass);
+
+#ifndef NDEBUG
+  unsigned NIter = 0;
+#endif
+  while (SCDesc->isVariant()) {
+    assert(++NIter < 6 && "Variants are nested deeper than the magic number");
+
+    SchedClass = STI->resolveSchedClass(SchedClass, MI, this);
+    SCDesc = SchedModel.getSchedClassDesc(SchedClass);
+  }
+  return SCDesc;
+}
+
+/// Find the def index of this operand. This index maps to the machine model and
+/// is independent of use operands. Def operands may be reordered with uses or
+/// merged with uses without affecting the def index (e.g. before/after
+/// regalloc). However, an instruction's def operands must never be reordered
+/// with respect to each other.
+static unsigned findDefIdx(const MachineInstr *MI, unsigned DefOperIdx) {
+  unsigned DefIdx = 0;
+  for (unsigned i = 0; i != DefOperIdx; ++i) {
+    const MachineOperand &MO = MI->getOperand(i);
+    if (MO.isReg() && MO.isDef())
+      ++DefIdx;
+  }
+  return DefIdx;
+}
+
+/// Find the use index of this operand. This is independent of the instruction's
+/// def operands.
+///
+/// Note that uses are not determined by the operand's isUse property, which
+/// is simply the inverse of isDef. Here we consider any readsReg operand to be
+/// a "use". The machine model allows an operand to be both a Def and Use.
+static unsigned findUseIdx(const MachineInstr *MI, unsigned UseOperIdx) {
+  unsigned UseIdx = 0;
+  for (unsigned i = 0; i != UseOperIdx; ++i) {
+    const MachineOperand &MO = MI->getOperand(i);
+    if (MO.isReg() && MO.readsReg())
+      ++UseIdx;
+  }
+  return UseIdx;
+}
+
+// Top-level API for clients that know the operand indices.
+unsigned TargetSchedModel::computeOperandLatency(
+  const MachineInstr *DefMI, unsigned DefOperIdx,
+  const MachineInstr *UseMI, unsigned UseOperIdx,
+  bool FindMin) const {
+
+  int DefLatency = getDefLatency(DefMI, FindMin);
+  if (DefLatency >= 0)
+    return DefLatency;
+
+  if (hasInstrItineraries()) {
+    int OperLatency = 0;
+    if (UseMI) {
+      OperLatency =
+        TII->getOperandLatency(&InstrItins, DefMI, DefOperIdx, UseMI, UseOperIdx);
+    }
+    else {
+      unsigned DefClass = DefMI->getDesc().getSchedClass();
+      OperLatency = InstrItins.getOperandCycle(DefClass, DefOperIdx);
+    }
+    if (OperLatency >= 0)
+      return OperLatency;
+
+    // No operand latency was found.
+    unsigned InstrLatency = TII->getInstrLatency(&InstrItins, DefMI);
+
+    // Expected latency is the max of the stage latency and itinerary props.
+    // Rather than directly querying InstrItins stage latency, we call a TII
+    // hook to allow subtargets to specialize latency. This hook is only
+    // applicable to the InstrItins model. InstrSchedModel should model all
+    // special cases without TII hooks.
+    if (!FindMin)
+      InstrLatency = std::max(InstrLatency,
+                              TII->defaultDefLatency(&SchedModel, DefMI));
+    return InstrLatency;
+  }
+  assert(!FindMin && hasInstrSchedModel() &&
+         "Expected a SchedModel for this cpu");
+  const MCSchedClassDesc *SCDesc = resolveSchedClass(DefMI);
+  unsigned DefIdx = findDefIdx(DefMI, DefOperIdx);
+  if (DefIdx < SCDesc->NumWriteLatencyEntries) {
+    // Lookup the definition's write latency in SubtargetInfo.
+    const MCWriteLatencyEntry *WLEntry =
+      STI->getWriteLatencyEntry(SCDesc, DefIdx);
+    unsigned WriteID = WLEntry->WriteResourceID;
+    unsigned Latency = convertLatency(WLEntry->Cycles);
+    if (!UseMI)
+      return Latency;
+
+    // Lookup the use's latency adjustment in SubtargetInfo.
+    const MCSchedClassDesc *UseDesc = resolveSchedClass(UseMI);
+    if (UseDesc->NumReadAdvanceEntries == 0)
+      return Latency;
+    unsigned UseIdx = findUseIdx(UseMI, UseOperIdx);
+    return Latency - STI->getReadAdvanceCycles(UseDesc, UseIdx, WriteID);
+  }
+  // If DefIdx does not exist in the model (e.g. implicit defs), then return
+  // unit latency (defaultDefLatency may be too conservative).
+#ifndef NDEBUG
+  if (SCDesc->isValid() && !DefMI->getOperand(DefOperIdx).isImplicit()
+      && !DefMI->getDesc().OpInfo[DefOperIdx].isOptionalDef()) {
+    std::string Err;
+    raw_string_ostream ss(Err);
+    ss << "DefIdx " << DefIdx << " exceeds machine model writes for "
+       << *DefMI;
+    report_fatal_error(ss.str());
+  }
+#endif
+  return DefMI->isTransient() ? 0 : 1;
+}
+
+unsigned TargetSchedModel::computeInstrLatency(const MachineInstr *MI) const {
+  // For the itinerary model, fall back to the old subtarget hook.
+  // Allow subtargets to compute Bundle latencies outside the machine model.
+  if (hasInstrItineraries() || MI->isBundle())
+    return TII->getInstrLatency(&InstrItins, MI);
+
+  if (hasInstrSchedModel()) {
+    const MCSchedClassDesc *SCDesc = resolveSchedClass(MI);
+    if (SCDesc->isValid()) {
+      unsigned Latency = 0;
+      for (unsigned DefIdx = 0, DefEnd = SCDesc->NumWriteLatencyEntries;
+           DefIdx != DefEnd; ++DefIdx) {
+        // Lookup the definition's write latency in SubtargetInfo.
+        const MCWriteLatencyEntry *WLEntry =
+          STI->getWriteLatencyEntry(SCDesc, DefIdx);
+        Latency = std::max(Latency, convertLatency(WLEntry->Cycles));
+      }
+      return Latency;
+    }
+  }
+  return TII->defaultDefLatency(&SchedModel, MI);
+}
+
+unsigned TargetSchedModel::
+computeOutputLatency(const MachineInstr *DefMI, unsigned DefOperIdx,
+                     const MachineInstr *DepMI) const {
+  // MinLatency == -1 is for in-order processors that always have unit
+  // MinLatency. MinLatency > 0 is for in-order processors with varying min
+  // latencies, but since this is not a RAW dep, we always use unit latency.
+  if (SchedModel.MinLatency != 0)
+    return 1;
+
+  // MinLatency == 0 indicates an out-of-order processor that can dispatch
+  // WAW dependencies in the same cycle.
+
+  // Treat predication as a data dependency for out-of-order cpus. In-order
+  // cpus do not need to treat predicated writes specially.
+  //
+  // TODO: The following hack exists because predication passes do not
+  // correctly append imp-use operands, and readsReg() strangely returns false
+  // for predicated defs.
+  unsigned Reg = DefMI->getOperand(DefOperIdx).getReg();
+  const MachineFunction &MF = *DefMI->getParent()->getParent();
+  const TargetRegisterInfo *TRI = MF.getTarget().getRegisterInfo();
+  if (!DepMI->readsRegister(Reg, TRI) && TII->isPredicated(DepMI))
+    return computeInstrLatency(DefMI);
+
+  // If we have a per operand scheduling model, check if this def is writing
+  // an unbuffered resource. If so, it treated like an in-order cpu.
+  if (hasInstrSchedModel()) {
+    const MCSchedClassDesc *SCDesc = resolveSchedClass(DefMI);
+    if (SCDesc->isValid()) {
+      for (const MCWriteProcResEntry *PRI = STI->getWriteProcResBegin(SCDesc),
+             *PRE = STI->getWriteProcResEnd(SCDesc); PRI != PRE; ++PRI) {
+        if (!SchedModel.getProcResource(PRI->ProcResourceIdx)->IsBuffered)
+          return 1;
+      }
+    }
+  }
+  return 0;
+}
diff --git a/lib/CodeGen/TwoAddressInstructionPass.cpp b/lib/CodeGen/TwoAddressInstructionPass.cpp
index aa601af..a9058bc 100644
--- a/lib/CodeGen/TwoAddressInstructionPass.cpp
+++ b/lib/CodeGen/TwoAddressInstructionPass.cpp
@@ -60,116 +60,108 @@ STATISTIC(NumReSchedUps,       "Number of instructions re-scheduled up");
 STATISTIC(NumReSchedDowns,     "Number of instructions re-scheduled down");
 
 namespace {
-  class TwoAddressInstructionPass : public MachineFunctionPass {
-    MachineFunction *MF;
-    const TargetInstrInfo *TII;
-    const TargetRegisterInfo *TRI;
-    const InstrItineraryData *InstrItins;
-    MachineRegisterInfo *MRI;
-    LiveVariables *LV;
-    SlotIndexes *Indexes;
-    LiveIntervals *LIS;
-    AliasAnalysis *AA;
-    CodeGenOpt::Level OptLevel;
-
-    // DistanceMap - Keep track the distance of a MI from the start of the
-    // current basic block.
-    DenseMap<MachineInstr*, unsigned> DistanceMap;
-
-    // SrcRegMap - A map from virtual registers to physical registers which
-    // are likely targets to be coalesced to due to copies from physical
-    // registers to virtual registers. e.g. v1024 = move r0.
-    DenseMap<unsigned, unsigned> SrcRegMap;
-
-    // DstRegMap - A map from virtual registers to physical registers which
-    // are likely targets to be coalesced to due to copies to physical
-    // registers from virtual registers. e.g. r1 = move v1024.
-    DenseMap<unsigned, unsigned> DstRegMap;
-
-    /// RegSequences - Keep track the list of REG_SEQUENCE instructions seen
-    /// during the initial walk of the machine function.
-    SmallVector<MachineInstr*, 16> RegSequences;
-
-    bool Sink3AddrInstruction(MachineBasicBlock *MBB, MachineInstr *MI,
-                              unsigned Reg,
-                              MachineBasicBlock::iterator OldPos);
-
-    bool NoUseAfterLastDef(unsigned Reg, MachineBasicBlock *MBB, unsigned Dist,
-                           unsigned &LastDef);
-
-    bool isProfitableToCommute(unsigned regA, unsigned regB, unsigned regC,
-                               MachineInstr *MI, MachineBasicBlock *MBB,
-                               unsigned Dist);
+class TwoAddressInstructionPass : public MachineFunctionPass {
+  MachineFunction *MF;
+  const TargetInstrInfo *TII;
+  const TargetRegisterInfo *TRI;
+  const InstrItineraryData *InstrItins;
+  MachineRegisterInfo *MRI;
+  LiveVariables *LV;
+  SlotIndexes *Indexes;
+  LiveIntervals *LIS;
+  AliasAnalysis *AA;
+  CodeGenOpt::Level OptLevel;
+
+  // The current basic block being processed.
+  MachineBasicBlock *MBB;
+
+  // DistanceMap - Keep track the distance of a MI from the start of the
+  // current basic block.
+  DenseMap<MachineInstr*, unsigned> DistanceMap;
+
+  // Set of already processed instructions in the current block.
+  SmallPtrSet<MachineInstr*, 8> Processed;
 
-    bool CommuteInstruction(MachineBasicBlock::iterator &mi,
-                            MachineFunction::iterator &mbbi,
-                            unsigned RegB, unsigned RegC, unsigned Dist);
+  // SrcRegMap - A map from virtual registers to physical registers which are
+  // likely targets to be coalesced to due to copies from physical registers to
+  // virtual registers. e.g. v1024 = move r0.
+  DenseMap<unsigned, unsigned> SrcRegMap;
 
-    bool isProfitableToConv3Addr(unsigned RegA, unsigned RegB);
+  // DstRegMap - A map from virtual registers to physical registers which are
+  // likely targets to be coalesced to due to copies to physical registers from
+  // virtual registers. e.g. r1 = move v1024.
+  DenseMap<unsigned, unsigned> DstRegMap;
 
-    bool ConvertInstTo3Addr(MachineBasicBlock::iterator &mi,
-                            MachineBasicBlock::iterator &nmi,
-                            MachineFunction::iterator &mbbi,
-                            unsigned RegA, unsigned RegB, unsigned Dist);
+  /// RegSequences - Keep track the list of REG_SEQUENCE instructions seen
+  /// during the initial walk of the machine function.
+  SmallVector<MachineInstr*, 16> RegSequences;
 
-    bool isDefTooClose(unsigned Reg, unsigned Dist,
-                       MachineInstr *MI, MachineBasicBlock *MBB);
+  bool sink3AddrInstruction(MachineInstr *MI, unsigned Reg,
+                            MachineBasicBlock::iterator OldPos);
 
-    bool RescheduleMIBelowKill(MachineBasicBlock *MBB,
-                               MachineBasicBlock::iterator &mi,
-                               MachineBasicBlock::iterator &nmi,
-                               unsigned Reg);
-    bool RescheduleKillAboveMI(MachineBasicBlock *MBB,
-                               MachineBasicBlock::iterator &mi,
-                               MachineBasicBlock::iterator &nmi,
-                               unsigned Reg);
+  bool noUseAfterLastDef(unsigned Reg, unsigned Dist, unsigned &LastDef);
 
-    bool TryInstructionTransform(MachineBasicBlock::iterator &mi,
-                                 MachineBasicBlock::iterator &nmi,
-                                 MachineFunction::iterator &mbbi,
-                                 unsigned SrcIdx, unsigned DstIdx,
-                                 unsigned Dist,
-                                 SmallPtrSet<MachineInstr*, 8> &Processed);
+  bool isProfitableToCommute(unsigned regA, unsigned regB, unsigned regC,
+                             MachineInstr *MI, unsigned Dist);
 
-    void ScanUses(unsigned DstReg, MachineBasicBlock *MBB,
-                  SmallPtrSet<MachineInstr*, 8> &Processed);
+  bool commuteInstruction(MachineBasicBlock::iterator &mi,
+                          unsigned RegB, unsigned RegC, unsigned Dist);
 
-    void ProcessCopy(MachineInstr *MI, MachineBasicBlock *MBB,
-                     SmallPtrSet<MachineInstr*, 8> &Processed);
+  bool isProfitableToConv3Addr(unsigned RegA, unsigned RegB);
 
-    typedef SmallVector<std::pair<unsigned, unsigned>, 4> TiedPairList;
-    typedef SmallDenseMap<unsigned, TiedPairList> TiedOperandMap;
-    bool collectTiedOperands(MachineInstr *MI, TiedOperandMap&);
-    void processTiedPairs(MachineInstr *MI, TiedPairList&, unsigned &Dist);
+  bool convertInstTo3Addr(MachineBasicBlock::iterator &mi,
+                          MachineBasicBlock::iterator &nmi,
+                          unsigned RegA, unsigned RegB, unsigned Dist);
 
-    void CoalesceExtSubRegs(SmallVector<unsigned,4> &Srcs, unsigned DstReg);
+  bool isDefTooClose(unsigned Reg, unsigned Dist, MachineInstr *MI);
 
-    /// EliminateRegSequences - Eliminate REG_SEQUENCE instructions as part
-    /// of the de-ssa process. This replaces sources of REG_SEQUENCE as
-    /// sub-register references of the register defined by REG_SEQUENCE.
-    bool EliminateRegSequences();
+  bool rescheduleMIBelowKill(MachineBasicBlock::iterator &mi,
+                             MachineBasicBlock::iterator &nmi,
+                             unsigned Reg);
+  bool rescheduleKillAboveMI(MachineBasicBlock::iterator &mi,
+                             MachineBasicBlock::iterator &nmi,
+                             unsigned Reg);
 
-  public:
-    static char ID; // Pass identification, replacement for typeid
-    TwoAddressInstructionPass() : MachineFunctionPass(ID) {
-      initializeTwoAddressInstructionPassPass(*PassRegistry::getPassRegistry());
-    }
+  bool tryInstructionTransform(MachineBasicBlock::iterator &mi,
+                               MachineBasicBlock::iterator &nmi,
+                               unsigned SrcIdx, unsigned DstIdx,
+                               unsigned Dist);
 
-    virtual void getAnalysisUsage(AnalysisUsage &AU) const {
-      AU.setPreservesCFG();
-      AU.addRequired<AliasAnalysis>();
-      AU.addPreserved<LiveVariables>();
-      AU.addPreserved<SlotIndexes>();
-      AU.addPreserved<LiveIntervals>();
-      AU.addPreservedID(MachineLoopInfoID);
-      AU.addPreservedID(MachineDominatorsID);
-      MachineFunctionPass::getAnalysisUsage(AU);
-    }
+  void scanUses(unsigned DstReg);
 
-    /// runOnMachineFunction - Pass entry point.
-    bool runOnMachineFunction(MachineFunction&);
-  };
-}
+  void processCopy(MachineInstr *MI);
+
+  typedef SmallVector<std::pair<unsigned, unsigned>, 4> TiedPairList;
+  typedef SmallDenseMap<unsigned, TiedPairList> TiedOperandMap;
+  bool collectTiedOperands(MachineInstr *MI, TiedOperandMap&);
+  void processTiedPairs(MachineInstr *MI, TiedPairList&, unsigned &Dist);
+
+  /// eliminateRegSequences - Eliminate REG_SEQUENCE instructions as part of
+  /// the de-ssa process. This replaces sources of REG_SEQUENCE as sub-register
+  /// references of the register defined by REG_SEQUENCE.
+  bool eliminateRegSequences();
+
+public:
+  static char ID; // Pass identification, replacement for typeid
+  TwoAddressInstructionPass() : MachineFunctionPass(ID) {
+    initializeTwoAddressInstructionPassPass(*PassRegistry::getPassRegistry());
+  }
+
+  virtual void getAnalysisUsage(AnalysisUsage &AU) const {
+    AU.setPreservesCFG();
+    AU.addRequired<AliasAnalysis>();
+    AU.addPreserved<LiveVariables>();
+    AU.addPreserved<SlotIndexes>();
+    AU.addPreserved<LiveIntervals>();
+    AU.addPreservedID(MachineLoopInfoID);
+    AU.addPreservedID(MachineDominatorsID);
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+
+  /// runOnMachineFunction - Pass entry point.
+  bool runOnMachineFunction(MachineFunction&);
+};
+} // end anonymous namespace
 
 char TwoAddressInstructionPass::ID = 0;
 INITIALIZE_PASS_BEGIN(TwoAddressInstructionPass, "twoaddressinstruction",
@@ -180,13 +172,13 @@ INITIALIZE_PASS_END(TwoAddressInstructionPass, "twoaddressinstruction",
 
 char &llvm::TwoAddressInstructionPassID = TwoAddressInstructionPass::ID;
 
-/// Sink3AddrInstruction - A two-address instruction has been converted to a
+/// sink3AddrInstruction - A two-address instruction has been converted to a
 /// three-address instruction to avoid clobbering a register. Try to sink it
 /// past the instruction that would kill the above mentioned register to reduce
 /// register pressure.
-bool TwoAddressInstructionPass::Sink3AddrInstruction(MachineBasicBlock *MBB,
-                                           MachineInstr *MI, unsigned SavedReg,
-                                           MachineBasicBlock::iterator OldPos) {
+bool TwoAddressInstructionPass::
+sink3AddrInstruction(MachineInstr *MI, unsigned SavedReg,
+                     MachineBasicBlock::iterator OldPos) {
   // FIXME: Shouldn't we be trying to do this before we three-addressify the
   // instruction?  After this transformation is done, we no longer need
   // the instruction to be in three-address form.
@@ -299,13 +291,12 @@ bool TwoAddressInstructionPass::Sink3AddrInstruction(MachineBasicBlock *MBB,
   return true;
 }
 
-/// NoUseAfterLastDef - Return true if there are no intervening uses between the
+/// noUseAfterLastDef - Return true if there are no intervening uses between the
 /// last instruction in the MBB that defines the specified register and the
 /// two-address instruction which is being processed. It also returns the last
 /// def location by reference
-bool TwoAddressInstructionPass::NoUseAfterLastDef(unsigned Reg,
-                                           MachineBasicBlock *MBB, unsigned Dist,
-                                           unsigned &LastDef) {
+bool TwoAddressInstructionPass::noUseAfterLastDef(unsigned Reg, unsigned Dist,
+                                                  unsigned &LastDef) {
   LastDef = 0;
   unsigned LastUse = Dist;
   for (MachineRegisterInfo::reg_iterator I = MRI->reg_begin(Reg),
@@ -465,10 +456,9 @@ regsAreCompatible(unsigned RegA, unsigned RegB, const TargetRegisterInfo *TRI) {
 /// isProfitableToCommute - Return true if it's potentially profitable to commute
 /// the two-address instruction that's being processed.
 bool
-TwoAddressInstructionPass::isProfitableToCommute(unsigned regA, unsigned regB,
-                                       unsigned regC,
-                                       MachineInstr *MI, MachineBasicBlock *MBB,
-                                       unsigned Dist) {
+TwoAddressInstructionPass::
+isProfitableToCommute(unsigned regA, unsigned regB, unsigned regC,
+                      MachineInstr *MI, unsigned Dist) {
   if (OptLevel == CodeGenOpt::None)
     return false;
 
@@ -516,13 +506,13 @@ TwoAddressInstructionPass::isProfitableToCommute(unsigned regA, unsigned regB,
   // If there is a use of regC between its last def (could be livein) and this
   // instruction, then bail.
   unsigned LastDefC = 0;
-  if (!NoUseAfterLastDef(regC, MBB, Dist, LastDefC))
+  if (!noUseAfterLastDef(regC, Dist, LastDefC))
     return false;
 
   // If there is a use of regB between its last def (could be livein) and this
   // instruction, then go ahead and make this transformation.
   unsigned LastDefB = 0;
-  if (!NoUseAfterLastDef(regB, MBB, Dist, LastDefB))
+  if (!noUseAfterLastDef(regB, Dist, LastDefB))
     return true;
 
   // Since there are no intervening uses for both registers, then commute
@@ -530,13 +520,12 @@ TwoAddressInstructionPass::isProfitableToCommute(unsigned regA, unsigned regB,
   return LastDefB && LastDefC && LastDefC > LastDefB;
 }
 
-/// CommuteInstruction - Commute a two-address instruction and update the basic
+/// commuteInstruction - Commute a two-address instruction and update the basic
 /// block, distance map, and live variables if needed. Return true if it is
 /// successful.
-bool
-TwoAddressInstructionPass::CommuteInstruction(MachineBasicBlock::iterator &mi,
-                               MachineFunction::iterator &mbbi,
-                               unsigned RegB, unsigned RegC, unsigned Dist) {
+bool TwoAddressInstructionPass::
+commuteInstruction(MachineBasicBlock::iterator &mi,
+                   unsigned RegB, unsigned RegC, unsigned Dist) {
   MachineInstr *MI = mi;
   DEBUG(dbgs() << "2addr: COMMUTING  : " << *MI);
   MachineInstr *NewMI = TII->commuteInstruction(MI);
@@ -555,8 +544,8 @@ TwoAddressInstructionPass::CommuteInstruction(MachineBasicBlock::iterator &mi,
     if (Indexes)
       Indexes->replaceMachineInstrInMaps(MI, NewMI);
 
-    mbbi->insert(mi, NewMI);           // Insert the new inst
-    mbbi->erase(mi);                   // Nuke the old inst.
+    MBB->insert(mi, NewMI);           // Insert the new inst
+    MBB->erase(mi);                   // Nuke the old inst.
     mi = NewMI;
     DistanceMap.insert(std::make_pair(NewMI, Dist));
   }
@@ -588,51 +577,51 @@ TwoAddressInstructionPass::isProfitableToConv3Addr(unsigned RegA,unsigned RegB){
   return (ToRegA && !regsAreCompatible(FromRegB, ToRegA, TRI));
 }
 
-/// ConvertInstTo3Addr - Convert the specified two-address instruction into a
+/// convertInstTo3Addr - Convert the specified two-address instruction into a
 /// three address one. Return true if this transformation was successful.
 bool
-TwoAddressInstructionPass::ConvertInstTo3Addr(MachineBasicBlock::iterator &mi,
+TwoAddressInstructionPass::convertInstTo3Addr(MachineBasicBlock::iterator &mi,
                                               MachineBasicBlock::iterator &nmi,
-                                              MachineFunction::iterator &mbbi,
                                               unsigned RegA, unsigned RegB,
                                               unsigned Dist) {
-  MachineInstr *NewMI = TII->convertToThreeAddress(mbbi, mi, LV);
-  if (NewMI) {
-    DEBUG(dbgs() << "2addr: CONVERTING 2-ADDR: " << *mi);
-    DEBUG(dbgs() << "2addr:         TO 3-ADDR: " << *NewMI);
-    bool Sunk = false;
+  // FIXME: Why does convertToThreeAddress() need an iterator reference?
+  MachineFunction::iterator MFI = MBB;
+  MachineInstr *NewMI = TII->convertToThreeAddress(MFI, mi, LV);
+  assert(MBB == MFI && "convertToThreeAddress changed iterator reference");
+  if (!NewMI)
+    return false;
 
-    if (Indexes)
-      Indexes->replaceMachineInstrInMaps(mi, NewMI);
+  DEBUG(dbgs() << "2addr: CONVERTING 2-ADDR: " << *mi);
+  DEBUG(dbgs() << "2addr:         TO 3-ADDR: " << *NewMI);
+  bool Sunk = false;
 
-    if (NewMI->findRegisterUseOperand(RegB, false, TRI))
-      // FIXME: Temporary workaround. If the new instruction doesn't
-      // uses RegB, convertToThreeAddress must have created more
-      // then one instruction.
-      Sunk = Sink3AddrInstruction(mbbi, NewMI, RegB, mi);
+  if (Indexes)
+    Indexes->replaceMachineInstrInMaps(mi, NewMI);
 
-    mbbi->erase(mi); // Nuke the old inst.
+  if (NewMI->findRegisterUseOperand(RegB, false, TRI))
+    // FIXME: Temporary workaround. If the new instruction doesn't
+    // uses RegB, convertToThreeAddress must have created more
+    // then one instruction.
+    Sunk = sink3AddrInstruction(NewMI, RegB, mi);
 
-    if (!Sunk) {
-      DistanceMap.insert(std::make_pair(NewMI, Dist));
-      mi = NewMI;
-      nmi = llvm::next(mi);
-    }
+  MBB->erase(mi); // Nuke the old inst.
 
-    // Update source and destination register maps.
-    SrcRegMap.erase(RegA);
-    DstRegMap.erase(RegB);
-    return true;
+  if (!Sunk) {
+    DistanceMap.insert(std::make_pair(NewMI, Dist));
+    mi = NewMI;
+    nmi = llvm::next(mi);
   }
 
-  return false;
+  // Update source and destination register maps.
+  SrcRegMap.erase(RegA);
+  DstRegMap.erase(RegB);
+  return true;
 }
 
-/// ScanUses - Scan forward recursively for only uses, update maps if the use
+/// scanUses - Scan forward recursively for only uses, update maps if the use
 /// is a copy or a two-address instruction.
 void
-TwoAddressInstructionPass::ScanUses(unsigned DstReg, MachineBasicBlock *MBB,
-                                    SmallPtrSet<MachineInstr*, 8> &Processed) {
+TwoAddressInstructionPass::scanUses(unsigned DstReg) {
   SmallVector<unsigned, 4> VirtRegPairs;
   bool IsDstPhys;
   bool IsCopy = false;
@@ -676,7 +665,7 @@ TwoAddressInstructionPass::ScanUses(unsigned DstReg, MachineBasicBlock *MBB,
   }
 }
 
-/// ProcessCopy - If the specified instruction is not yet processed, process it
+/// processCopy - If the specified instruction is not yet processed, process it
 /// if it's a copy. For a copy instruction, we find the physical registers the
 /// source and destination registers might be mapped to. These are kept in
 /// point-to maps used to determine future optimizations. e.g.
@@ -688,9 +677,7 @@ TwoAddressInstructionPass::ScanUses(unsigned DstReg, MachineBasicBlock *MBB,
 /// coalesced to r0 (from the input side). v1025 is mapped to r1. v1026 is
 /// potentially joined with r1 on the output side. It's worthwhile to commute
 /// 'add' to eliminate a copy.
-void TwoAddressInstructionPass::ProcessCopy(MachineInstr *MI,
-                                     MachineBasicBlock *MBB,
-                                     SmallPtrSet<MachineInstr*, 8> &Processed) {
+void TwoAddressInstructionPass::processCopy(MachineInstr *MI) {
   if (Processed.count(MI))
     return;
 
@@ -707,21 +694,20 @@ void TwoAddressInstructionPass::ProcessCopy(MachineInstr *MI,
       assert(SrcRegMap[DstReg] == SrcReg &&
              "Can't map to two src physical registers!");
 
-    ScanUses(DstReg, MBB, Processed);
+    scanUses(DstReg);
   }
 
   Processed.insert(MI);
   return;
 }
 
-/// RescheduleMIBelowKill - If there is one more local instruction that reads
+/// rescheduleMIBelowKill - If there is one more local instruction that reads
 /// 'Reg' and it kills 'Reg, consider moving the instruction below the kill
 /// instruction in order to eliminate the need for the copy.
-bool
-TwoAddressInstructionPass::RescheduleMIBelowKill(MachineBasicBlock *MBB,
-                                     MachineBasicBlock::iterator &mi,
-                                     MachineBasicBlock::iterator &nmi,
-                                     unsigned Reg) {
+bool TwoAddressInstructionPass::
+rescheduleMIBelowKill(MachineBasicBlock::iterator &mi,
+                      MachineBasicBlock::iterator &nmi,
+                      unsigned Reg) {
   // Bail immediately if we don't have LV available. We use it to find kills
   // efficiently.
   if (!LV)
@@ -853,8 +839,7 @@ TwoAddressInstructionPass::RescheduleMIBelowKill(MachineBasicBlock *MBB,
 /// isDefTooClose - Return true if the re-scheduling will put the given
 /// instruction too close to the defs of its register dependencies.
 bool TwoAddressInstructionPass::isDefTooClose(unsigned Reg, unsigned Dist,
-                                              MachineInstr *MI,
-                                              MachineBasicBlock *MBB) {
+                                              MachineInstr *MI) {
   for (MachineRegisterInfo::def_iterator DI = MRI->def_begin(Reg),
          DE = MRI->def_end(); DI != DE; ++DI) {
     MachineInstr *DefMI = &*DI;
@@ -873,15 +858,14 @@ bool TwoAddressInstructionPass::isDefTooClose(unsigned Reg, unsigned Dist,
   return false;
 }
 
-/// RescheduleKillAboveMI - If there is one more local instruction that reads
+/// rescheduleKillAboveMI - If there is one more local instruction that reads
 /// 'Reg' and it kills 'Reg, consider moving the kill instruction above the
 /// current two-address instruction in order to eliminate the need for the
 /// copy.
-bool
-TwoAddressInstructionPass::RescheduleKillAboveMI(MachineBasicBlock *MBB,
-                                     MachineBasicBlock::iterator &mi,
-                                     MachineBasicBlock::iterator &nmi,
-                                     unsigned Reg) {
+bool TwoAddressInstructionPass::
+rescheduleKillAboveMI(MachineBasicBlock::iterator &mi,
+                      MachineBasicBlock::iterator &nmi,
+                      unsigned Reg) {
   // Bail immediately if we don't have LV available. We use it to find kills
   // efficiently.
   if (!LV)
@@ -918,7 +902,7 @@ TwoAddressInstructionPass::RescheduleKillAboveMI(MachineBasicBlock *MBB,
     if (MO.isUse()) {
       if (!MOReg)
         continue;
-      if (isDefTooClose(MOReg, DI->second, MI, MBB))
+      if (isDefTooClose(MOReg, DI->second, MI))
         return false;
       if (MOReg == Reg && !MO.isKill())
         return false;
@@ -1006,18 +990,16 @@ TwoAddressInstructionPass::RescheduleKillAboveMI(MachineBasicBlock *MBB,
   return true;
 }
 
-/// TryInstructionTransform - For the case where an instruction has a single
+/// tryInstructionTransform - For the case where an instruction has a single
 /// pair of tied register operands, attempt some transformations that may
 /// either eliminate the tied operands or improve the opportunities for
 /// coalescing away the register copy.  Returns true if no copy needs to be
 /// inserted to untie mi's operands (either because they were untied, or
 /// because mi was rescheduled, and will be visited again later).
 bool TwoAddressInstructionPass::
-TryInstructionTransform(MachineBasicBlock::iterator &mi,
+tryInstructionTransform(MachineBasicBlock::iterator &mi,
                         MachineBasicBlock::iterator &nmi,
-                        MachineFunction::iterator &mbbi,
-                        unsigned SrcIdx, unsigned DstIdx, unsigned Dist,
-                        SmallPtrSet<MachineInstr*, 8> &Processed) {
+                        unsigned SrcIdx, unsigned DstIdx, unsigned Dist) {
   if (OptLevel == CodeGenOpt::None)
     return false;
 
@@ -1030,7 +1012,7 @@ TryInstructionTransform(MachineBasicBlock::iterator &mi,
   bool regBKilled = isKilled(MI, regB, MRI, TII);
 
   if (TargetRegisterInfo::isVirtualRegister(regA))
-    ScanUses(regA, &*mbbi, Processed);
+    scanUses(regA);
 
   // Check if it is profitable to commute the operands.
   unsigned SrcOp1, SrcOp2;
@@ -1051,7 +1033,7 @@ TryInstructionTransform(MachineBasicBlock::iterator &mi,
         // If C dies but B does not, swap the B and C operands.
         // This makes the live ranges of A and C joinable.
         TryCommute = true;
-      else if (isProfitableToCommute(regA, regB, regC, &MI, mbbi, Dist)) {
+      else if (isProfitableToCommute(regA, regB, regC, &MI, Dist)) {
         TryCommute = true;
         AggressiveCommute = true;
       }
@@ -1059,7 +1041,7 @@ TryInstructionTransform(MachineBasicBlock::iterator &mi,
   }
 
   // If it's profitable to commute, try to do so.
-  if (TryCommute && CommuteInstruction(mi, mbbi, regB, regC, Dist)) {
+  if (TryCommute && commuteInstruction(mi, regB, regC, Dist)) {
     ++NumCommuted;
     if (AggressiveCommute)
       ++NumAggrCommuted;
@@ -1068,7 +1050,7 @@ TryInstructionTransform(MachineBasicBlock::iterator &mi,
 
   // If there is one more use of regB later in the same MBB, consider
   // re-schedule this MI below it.
-  if (RescheduleMIBelowKill(mbbi, mi, nmi, regB)) {
+  if (rescheduleMIBelowKill(mi, nmi, regB)) {
     ++NumReSchedDowns;
     return true;
   }
@@ -1078,7 +1060,7 @@ TryInstructionTransform(MachineBasicBlock::iterator &mi,
     // three-address instruction.  Check if it is profitable.
     if (!regBKilled || isProfitableToConv3Addr(regA, regB)) {
       // Try to convert it.
-      if (ConvertInstTo3Addr(mi, nmi, mbbi, regA, regB, Dist)) {
+      if (convertInstTo3Addr(mi, nmi, regA, regB, Dist)) {
         ++NumConvertedTo3Addr;
         return true; // Done with this instruction.
       }
@@ -1087,7 +1069,7 @@ TryInstructionTransform(MachineBasicBlock::iterator &mi,
 
   // If there is one more use of regB later in the same MBB, consider
   // re-schedule it before this MI if it's legal.
-  if (RescheduleKillAboveMI(mbbi, mi, nmi, regB)) {
+  if (rescheduleKillAboveMI(mi, nmi, regB)) {
     ++NumReSchedUps;
     return true;
   }
@@ -1131,8 +1113,8 @@ TryInstructionTransform(MachineBasicBlock::iterator &mi,
 
         // Tentatively insert the instructions into the block so that they
         // look "normal" to the transformation logic.
-        mbbi->insert(mi, NewMIs[0]);
-        mbbi->insert(mi, NewMIs[1]);
+        MBB->insert(mi, NewMIs[0]);
+        MBB->insert(mi, NewMIs[1]);
 
         DEBUG(dbgs() << "2addr:    NEW LOAD: " << *NewMIs[0]
                      << "2addr:    NEW INST: " << *NewMIs[1]);
@@ -1142,8 +1124,7 @@ TryInstructionTransform(MachineBasicBlock::iterator &mi,
         unsigned NewSrcIdx = NewMIs[1]->findRegisterUseOperandIdx(regB);
         MachineBasicBlock::iterator NewMI = NewMIs[1];
         bool TransformSuccess =
-          TryInstructionTransform(NewMI, mi, mbbi,
-                                  NewSrcIdx, NewDstIdx, Dist, Processed);
+          tryInstructionTransform(NewMI, mi, NewSrcIdx, NewDstIdx, Dist);
         if (TransformSuccess ||
             NewMIs[1]->getOperand(NewSrcIdx).isKill()) {
           // Success, or at least we made an improvement. Keep the unfolded
@@ -1202,8 +1183,7 @@ bool TwoAddressInstructionPass::
 collectTiedOperands(MachineInstr *MI, TiedOperandMap &TiedOperands) {
   const MCInstrDesc &MCID = MI->getDesc();
   bool AnyOps = false;
-  unsigned NumOps = MI->isInlineAsm() ?
-    MI->getNumOperands() : MCID.getNumOperands();
+  unsigned NumOps = MI->getNumOperands();
 
   for (unsigned SrcIdx = 0; SrcIdx < NumOps; ++SrcIdx) {
     unsigned DstIdx = 0;
@@ -1373,22 +1353,21 @@ bool TwoAddressInstructionPass::runOnMachineFunction(MachineFunction &Func) {
 
   DEBUG(dbgs() << "********** REWRITING TWO-ADDR INSTRS **********\n");
   DEBUG(dbgs() << "********** Function: "
-        << MF->getFunction()->getName() << '\n');
+        << MF->getName() << '\n');
 
   // This pass takes the function out of SSA form.
   MRI->leaveSSA();
 
   TiedOperandMap TiedOperands;
-
-  SmallPtrSet<MachineInstr*, 8> Processed;
-  for (MachineFunction::iterator mbbi = MF->begin(), mbbe = MF->end();
-       mbbi != mbbe; ++mbbi) {
+  for (MachineFunction::iterator MBBI = MF->begin(), MBBE = MF->end();
+       MBBI != MBBE; ++MBBI) {
+    MBB = MBBI;
     unsigned Dist = 0;
     DistanceMap.clear();
     SrcRegMap.clear();
     DstRegMap.clear();
     Processed.clear();
-    for (MachineBasicBlock::iterator mi = mbbi->begin(), me = mbbi->end();
+    for (MachineBasicBlock::iterator mi = MBB->begin(), me = MBB->end();
          mi != me; ) {
       MachineBasicBlock::iterator nmi = llvm::next(mi);
       if (mi->isDebugValue()) {
@@ -1402,7 +1381,7 @@ bool TwoAddressInstructionPass::runOnMachineFunction(MachineFunction &Func) {
 
       DistanceMap.insert(std::make_pair(mi, ++Dist));
 
-      ProcessCopy(&*mi, &*mbbi, Processed);
+      processCopy(&*mi);
 
       // First scan through all the tied register uses in this instruction
       // and record a list of pairs of tied operands for each register.
@@ -1427,8 +1406,7 @@ bool TwoAddressInstructionPass::runOnMachineFunction(MachineFunction &Func) {
           unsigned SrcReg = mi->getOperand(SrcIdx).getReg();
           unsigned DstReg = mi->getOperand(DstIdx).getReg();
           if (SrcReg != DstReg &&
-              TryInstructionTransform(mi, nmi, mbbi, SrcIdx, DstIdx, Dist,
-                                      Processed)) {
+              tryInstructionTransform(mi, nmi, SrcIdx, DstIdx, Dist)) {
             // The tied operands have been eliminated or shifted further down the
             // block to ease elimination. Continue processing with 'nmi'.
             TiedOperands.clear();
@@ -1468,7 +1446,7 @@ bool TwoAddressInstructionPass::runOnMachineFunction(MachineFunction &Func) {
 
   // Eliminate REG_SEQUENCE instructions. Their whole purpose was to preseve
   // SSA form. It's now safe to de-SSA.
-  MadeChange |= EliminateRegSequences();
+  MadeChange |= eliminateRegSequences();
 
   return MadeChange;
 }
@@ -1515,127 +1493,6 @@ static MachineInstr *findFirstDef(unsigned Reg, MachineRegisterInfo *MRI) {
   return First;
 }
 
-/// CoalesceExtSubRegs - If a number of sources of the REG_SEQUENCE are
-/// EXTRACT_SUBREG from the same register and to the same virtual register
-/// with different sub-register indices, attempt to combine the
-/// EXTRACT_SUBREGs and pre-coalesce them. e.g.
-/// %reg1026<def> = VLDMQ %reg1025<kill>, 260, pred:14, pred:%reg0
-/// %reg1029:6<def> = EXTRACT_SUBREG %reg1026, 6
-/// %reg1029:5<def> = EXTRACT_SUBREG %reg1026<kill>, 5
-/// Since D subregs 5, 6 can combine to a Q register, we can coalesce
-/// reg1026 to reg1029.
-void
-TwoAddressInstructionPass::CoalesceExtSubRegs(SmallVector<unsigned,4> &Srcs,
-                                              unsigned DstReg) {
-  SmallSet<unsigned, 4> Seen;
-  for (unsigned i = 0, e = Srcs.size(); i != e; ++i) {
-    unsigned SrcReg = Srcs[i];
-    if (!Seen.insert(SrcReg))
-      continue;
-
-    // Check that the instructions are all in the same basic block.
-    MachineInstr *SrcDefMI = MRI->getUniqueVRegDef(SrcReg);
-    MachineInstr *DstDefMI = MRI->getUniqueVRegDef(DstReg);
-    if (!SrcDefMI || !DstDefMI ||
-        SrcDefMI->getParent() != DstDefMI->getParent())
-      continue;
-
-    // If there are no other uses than copies which feed into
-    // the reg_sequence, then we might be able to coalesce them.
-    bool CanCoalesce = true;
-    SmallVector<unsigned, 4> SrcSubIndices, DstSubIndices;
-    for (MachineRegisterInfo::use_nodbg_iterator
-           UI = MRI->use_nodbg_begin(SrcReg),
-           UE = MRI->use_nodbg_end(); UI != UE; ++UI) {
-      MachineInstr *UseMI = &*UI;
-      if (!UseMI->isCopy() || UseMI->getOperand(0).getReg() != DstReg) {
-        CanCoalesce = false;
-        break;
-      }
-      SrcSubIndices.push_back(UseMI->getOperand(1).getSubReg());
-      DstSubIndices.push_back(UseMI->getOperand(0).getSubReg());
-    }
-
-    if (!CanCoalesce || SrcSubIndices.size() < 2)
-      continue;
-
-    // Check that the source subregisters can be combined.
-    std::sort(SrcSubIndices.begin(), SrcSubIndices.end());
-    unsigned NewSrcSubIdx = 0;
-    if (!TRI->canCombineSubRegIndices(MRI->getRegClass(SrcReg), SrcSubIndices,
-                                      NewSrcSubIdx))
-      continue;
-
-    // Check that the destination subregisters can also be combined.
-    std::sort(DstSubIndices.begin(), DstSubIndices.end());
-    unsigned NewDstSubIdx = 0;
-    if (!TRI->canCombineSubRegIndices(MRI->getRegClass(DstReg), DstSubIndices,
-                                      NewDstSubIdx))
-      continue;
-
-    // If neither source nor destination can be combined to the full register,
-    // just give up.  This could be improved if it ever matters.
-    if (NewSrcSubIdx != 0 && NewDstSubIdx != 0)
-      continue;
-
-    // Now that we know that all the uses are extract_subregs and that those
-    // subregs can somehow be combined, scan all the extract_subregs again to
-    // make sure the subregs are in the right order and can be composed.
-    MachineInstr *SomeMI = 0;
-    CanCoalesce = true;
-    for (MachineRegisterInfo::use_nodbg_iterator
-           UI = MRI->use_nodbg_begin(SrcReg),
-           UE = MRI->use_nodbg_end(); UI != UE; ++UI) {
-      MachineInstr *UseMI = &*UI;
-      assert(UseMI->isCopy());
-      unsigned DstSubIdx = UseMI->getOperand(0).getSubReg();
-      unsigned SrcSubIdx = UseMI->getOperand(1).getSubReg();
-      assert(DstSubIdx != 0 && "missing subreg from RegSequence elimination");
-      if ((NewDstSubIdx == 0 &&
-           TRI->composeSubRegIndices(NewSrcSubIdx, DstSubIdx) != SrcSubIdx) ||
-          (NewSrcSubIdx == 0 &&
-           TRI->composeSubRegIndices(NewDstSubIdx, SrcSubIdx) != DstSubIdx)) {
-        CanCoalesce = false;
-        break;
-      }
-      // Keep track of one of the uses.  Preferably the first one which has a
-      // <def,undef> flag.
-      if (!SomeMI || UseMI->getOperand(0).isUndef())
-        SomeMI = UseMI;
-    }
-    if (!CanCoalesce)
-      continue;
-
-    // Insert a copy to replace the original.
-    MachineInstr *CopyMI = BuildMI(*SomeMI->getParent(), SomeMI,
-                                   SomeMI->getDebugLoc(),
-                                   TII->get(TargetOpcode::COPY))
-      .addReg(DstReg, RegState::Define |
-                      getUndefRegState(SomeMI->getOperand(0).isUndef()),
-              NewDstSubIdx)
-      .addReg(SrcReg, 0, NewSrcSubIdx);
-
-    // Remove all the old extract instructions.
-    for (MachineRegisterInfo::use_nodbg_iterator
-           UI = MRI->use_nodbg_begin(SrcReg),
-           UE = MRI->use_nodbg_end(); UI != UE; ) {
-      MachineInstr *UseMI = &*UI;
-      ++UI;
-      if (UseMI == CopyMI)
-        continue;
-      assert(UseMI->isCopy());
-      // Move any kills to the new copy or extract instruction.
-      if (UseMI->getOperand(1).isKill()) {
-        CopyMI->getOperand(1).setIsKill();
-        if (LV)
-          // Update live variables
-          LV->replaceKillInstruction(SrcReg, UseMI, &*CopyMI);
-      }
-      UseMI->eraseFromParent();
-    }
-  }
-}
-
 static bool HasOtherRegSequenceUses(unsigned Reg, MachineInstr *RegSeq,
                                     MachineRegisterInfo *MRI) {
   for (MachineRegisterInfo::use_iterator UI = MRI->use_begin(Reg),
@@ -1647,7 +1504,7 @@ static bool HasOtherRegSequenceUses(unsigned Reg, MachineInstr *RegSeq,
   return false;
 }
 
-/// EliminateRegSequences - Eliminate REG_SEQUENCE instructions as part
+/// eliminateRegSequences - Eliminate REG_SEQUENCE instructions as part
 /// of the de-ssa process. This replaces sources of REG_SEQUENCE as
 /// sub-register references of the register defined by REG_SEQUENCE. e.g.
 ///
@@ -1655,7 +1512,7 @@ static bool HasOtherRegSequenceUses(unsigned Reg, MachineInstr *RegSeq,
 /// %reg1031<def> = REG_SEQUENCE %reg1029<kill>, 5, %reg1030<kill>, 6
 /// =>
 /// %reg1031:5<def>, %reg1031:6<def> = VLD1q16 %reg1024<kill>, ...
-bool TwoAddressInstructionPass::EliminateRegSequences() {
+bool TwoAddressInstructionPass::eliminateRegSequences() {
   if (RegSequences.empty())
     return false;
 
@@ -1759,10 +1616,6 @@ bool TwoAddressInstructionPass::EliminateRegSequences() {
         if (MO.isReg() && MO.isDef() && MO.getReg() == DstReg)
           MO.setIsUndef();
       }
-      // Make sure there is a full non-subreg imp-def operand on the
-      // instruction.  This shouldn't be necessary, but it seems that at least
-      // RAFast requires it.
-      Def->addRegisterDefined(DstReg, TRI);
       DEBUG(dbgs() << "First def: " << *Def);
     }
 
@@ -1775,12 +1628,6 @@ bool TwoAddressInstructionPass::EliminateRegSequences() {
       DEBUG(dbgs() << "Eliminated: " << *MI);
       MI->eraseFromParent();
     }
-
-    // Try coalescing some EXTRACT_SUBREG instructions. This can create
-    // INSERT_SUBREG instructions that must have <undef> flags added by
-    // LiveIntervalAnalysis, so only run it when LiveVariables is available.
-    if (LV)
-      CoalesceExtSubRegs(RealSrcs, DstReg);
   }
 
   RegSequences.clear();
diff --git a/lib/CodeGen/VirtRegMap.cpp b/lib/CodeGen/VirtRegMap.cpp
index 93840f0..bb93bdc 100644
--- a/lib/CodeGen/VirtRegMap.cpp
+++ b/lib/CodeGen/VirtRegMap.cpp
@@ -19,8 +19,8 @@
 #define DEBUG_TYPE "regalloc"
 #include "VirtRegMap.h"
 #include "LiveDebugVariables.h"
-#include "llvm/Function.h"
 #include "llvm/CodeGen/LiveIntervalAnalysis.h"
+#include "llvm/CodeGen/LiveStackAnalysis.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
@@ -127,9 +127,11 @@ void VirtRegMap::print(raw_ostream &OS, const Module*) const {
   OS << '\n';
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void VirtRegMap::dump() const {
   print(dbgs());
 }
+#endif
 
 //===----------------------------------------------------------------------===//
 //                              VirtRegRewriter
@@ -170,6 +172,7 @@ INITIALIZE_PASS_BEGIN(VirtRegRewriter, "virtregrewriter",
 INITIALIZE_PASS_DEPENDENCY(SlotIndexes)
 INITIALIZE_PASS_DEPENDENCY(LiveIntervals)
 INITIALIZE_PASS_DEPENDENCY(LiveDebugVariables)
+INITIALIZE_PASS_DEPENDENCY(LiveStacks)
 INITIALIZE_PASS_DEPENDENCY(VirtRegMap)
 INITIALIZE_PASS_END(VirtRegRewriter, "virtregrewriter",
                     "Virtual Register Rewriter", false, false)
@@ -182,6 +185,8 @@ void VirtRegRewriter::getAnalysisUsage(AnalysisUsage &AU) const {
   AU.addRequired<SlotIndexes>();
   AU.addPreserved<SlotIndexes>();
   AU.addRequired<LiveDebugVariables>();
+  AU.addRequired<LiveStacks>();
+  AU.addPreserved<LiveStacks>();
   AU.addRequired<VirtRegMap>();
   MachineFunctionPass::getAnalysisUsage(AU);
 }
@@ -197,11 +202,11 @@ bool VirtRegRewriter::runOnMachineFunction(MachineFunction &fn) {
   VRM = &getAnalysis<VirtRegMap>();
   DEBUG(dbgs() << "********** REWRITE VIRTUAL REGISTERS **********\n"
                << "********** Function: "
-               << MF->getFunction()->getName() << '\n');
+               << MF->getName() << '\n');
   DEBUG(VRM->dump());
 
   // Add kill flags while we still have virtual registers.
-  LIS->addKillFlags();
+  LIS->addKillFlags(VRM);
 
   // Live-in lists on basic blocks are required for physregs.
   addMBBLiveIns();
@@ -252,9 +257,6 @@ void VirtRegRewriter::rewrite() {
   SmallVector<unsigned, 8> SuperDeads;
   SmallVector<unsigned, 8> SuperDefs;
   SmallVector<unsigned, 8> SuperKills;
-#ifndef NDEBUG
-  BitVector Reserved = TRI->getReservedRegs(*MF);
-#endif
 
   for (MachineFunction::iterator MBBI = MF->begin(), MBBE = MF->end();
        MBBI != MBBE; ++MBBI) {
@@ -278,7 +280,7 @@ void VirtRegRewriter::rewrite() {
         unsigned PhysReg = VRM->getPhys(VirtReg);
         assert(PhysReg != VirtRegMap::NO_PHYS_REG &&
                "Instruction uses unmapped VirtReg");
-        assert(!Reserved.test(PhysReg) && "Reserved register assignment");
+        assert(!MRI->isReserved(PhysReg) && "Reserved register assignment");
 
         // Preserve semantics of sub-register operands.
         if (MO.getSubReg()) {
diff --git a/lib/CodeGen/VirtRegMap.h b/lib/CodeGen/VirtRegMap.h
index c320985..7974dda 100644
--- a/lib/CodeGen/VirtRegMap.h
+++ b/lib/CodeGen/VirtRegMap.h
@@ -63,8 +63,8 @@ namespace llvm {
     /// createSpillSlot - Allocate a spill slot for RC from MFI.
     unsigned createSpillSlot(const TargetRegisterClass *RC);
 
-    VirtRegMap(const VirtRegMap&);     // DO NOT IMPLEMENT
-    void operator=(const VirtRegMap&); // DO NOT IMPLEMENT
+    VirtRegMap(const VirtRegMap&) LLVM_DELETED_FUNCTION;
+    void operator=(const VirtRegMap&) LLVM_DELETED_FUNCTION;
 
   public:
     static char ID;
diff --git a/lib/DebugInfo/CMakeLists.txt b/lib/DebugInfo/CMakeLists.txt
index 441f1e8..1e9e509 100644
--- a/lib/DebugInfo/CMakeLists.txt
+++ b/lib/DebugInfo/CMakeLists.txt
@@ -8,5 +8,6 @@ add_llvm_library(LLVMDebugInfo
   DWARFDebugAranges.cpp
   DWARFDebugInfoEntry.cpp
   DWARFDebugLine.cpp
+  DWARFDebugRangeList.cpp
   DWARFFormValue.cpp
   )
diff --git a/lib/DebugInfo/DIContext.cpp b/lib/DebugInfo/DIContext.cpp
index e2fd55f..691a92c 100644
--- a/lib/DebugInfo/DIContext.cpp
+++ b/lib/DebugInfo/DIContext.cpp
@@ -18,7 +18,10 @@ DIContext *DIContext::getDWARFContext(bool isLittleEndian,
                                       StringRef abbrevSection,
                                       StringRef aRangeSection,
                                       StringRef lineSection,
-                                      StringRef stringSection) {
+                                      StringRef stringSection,
+                                      StringRef rangeSection,
+                                      const RelocAddrMap &Map) {
   return new DWARFContextInMemory(isLittleEndian, infoSection, abbrevSection,
-                                  aRangeSection, lineSection, stringSection);
+                                  aRangeSection, lineSection, stringSection,
+                                  rangeSection, Map);
 }
diff --git a/lib/DebugInfo/DWARFCompileUnit.cpp b/lib/DebugInfo/DWARFCompileUnit.cpp
index b27d57b..bdd65b7 100644
--- a/lib/DebugInfo/DWARFCompileUnit.cpp
+++ b/lib/DebugInfo/DWARFCompileUnit.cpp
@@ -63,7 +63,7 @@ DWARFCompileUnit::extract(uint32_t offset, DataExtractor debug_info_data,
     Version = debug_info_data.getU16(&offset);
     bool abbrevsOK = debug_info_data.getU32(&offset) == abbrevs->getOffset();
     Abbrevs = abbrevs;
-    AddrSize = debug_info_data.getU8 (&offset);
+    AddrSize = debug_info_data.getU8(&offset);
 
     bool versionOK = DWARFContext::isSupportedVersion(Version);
     bool addrSizeOK = AddrSize == 4 || AddrSize == 8;
@@ -75,6 +75,15 @@ DWARFCompileUnit::extract(uint32_t offset, DataExtractor debug_info_data,
   return 0;
 }
 
+bool DWARFCompileUnit::extractRangeList(uint32_t RangeListOffset,
+                                        DWARFDebugRangeList &RangeList) const {
+  // Require that compile unit is extracted.
+  assert(DieArray.size() > 0);
+  DataExtractor RangesData(Context.getRangeSection(),
+                           Context.isLittleEndian(), AddrSize);
+  return RangeList.extract(RangesData, &RangeListOffset);
+}
+
 void DWARFCompileUnit::clear() {
   Offset = 0;
   Length = 0;
@@ -94,7 +103,9 @@ void DWARFCompileUnit::dump(raw_ostream &OS) {
      << " (next CU at " << format("0x%08x", getNextCompileUnitOffset())
      << ")\n";
 
-  getCompileUnitDIE(false)->dump(OS, this, -1U);
+  const DWARFDebugInfoEntryMinimal *CU = getCompileUnitDIE(false);
+  assert(CU && "Null Compile Unit?");
+  CU->dump(OS, this, -1U);
 }
 
 const char *DWARFCompileUnit::getCompilationDir() {
@@ -174,11 +185,11 @@ size_t DWARFCompileUnit::extractDIEsIfNeeded(bool cu_die_only) {
       addDIE(die);
       return 1;
     }
-    else if (depth == 0 && initial_die_array_size == 1) {
+    else if (depth == 0 && initial_die_array_size == 1)
       // Don't append the CU die as we already did that
-    } else {
-      addDIE (die);
-    }
+      ;
+    else
+      addDIE(die);
 
     const DWARFAbbreviationDeclaration *abbrDecl =
       die.getAbbreviationDeclarationPtr();
@@ -199,9 +210,9 @@ size_t DWARFCompileUnit::extractDIEsIfNeeded(bool cu_die_only) {
   // Give a little bit of info if we encounter corrupt DWARF (our offset
   // should always terminate at or before the start of the next compilation
   // unit header).
-  if (offset > next_cu_offset) {
-    fprintf (stderr, "warning: DWARF compile unit extends beyond its bounds cu 0x%8.8x at 0x%8.8x'\n", getOffset(), offset);
-  }
+  if (offset > next_cu_offset)
+    fprintf(stderr, "warning: DWARF compile unit extends beyond its"
+                    "bounds cu 0x%8.8x at 0x%8.8x'\n", getOffset(), offset);
 
   setDIERelations();
   return DieArray.size();
@@ -244,12 +255,21 @@ DWARFCompileUnit::buildAddressRangeTable(DWARFDebugAranges *debug_aranges,
     clearDIEs(true);
 }
 
-const DWARFDebugInfoEntryMinimal*
-DWARFCompileUnit::getFunctionDIEForAddress(int64_t address) {
+DWARFDebugInfoEntryMinimal::InlinedChain
+DWARFCompileUnit::getInlinedChainForAddress(uint64_t Address) {
+  // First, find a subprogram that contains the given address (the root
+  // of inlined chain).
   extractDIEsIfNeeded(false);
+  const DWARFDebugInfoEntryMinimal *SubprogramDIE = 0;
   for (size_t i = 0, n = DieArray.size(); i != n; i++) {
-    if (DieArray[i].addressRangeContainsAddress(this, address))
-      return &DieArray[i];
+    if (DieArray[i].isSubprogramDIE() &&
+        DieArray[i].addressRangeContainsAddress(this, Address)) {
+      SubprogramDIE = &DieArray[i];
+      break;
+    }
   }
-  return 0;
+  // Get inlined chain rooted at this subprogram DIE.
+  if (!SubprogramDIE)
+    return DWARFDebugInfoEntryMinimal::InlinedChain();
+  return SubprogramDIE->getInlinedChainForAddress(this, Address);
 }
diff --git a/lib/DebugInfo/DWARFCompileUnit.h b/lib/DebugInfo/DWARFCompileUnit.h
index b34a596..03e2862 100644
--- a/lib/DebugInfo/DWARFCompileUnit.h
+++ b/lib/DebugInfo/DWARFCompileUnit.h
@@ -12,6 +12,7 @@
 
 #include "DWARFDebugAbbrev.h"
 #include "DWARFDebugInfoEntry.h"
+#include "DWARFDebugRangeList.h"
 #include <vector>
 
 namespace llvm {
@@ -45,6 +46,11 @@ public:
   /// extractDIEsIfNeeded - Parses a compile unit and indexes its DIEs if it
   /// hasn't already been done. Returns the number of DIEs parsed at this call.
   size_t extractDIEsIfNeeded(bool cu_die_only);
+  /// extractRangeList - extracts the range list referenced by this compile
+  /// unit from .debug_ranges section. Returns true on success.
+  /// Requires that compile unit is already extracted.
+  bool extractRangeList(uint32_t RangeListOffset,
+                        DWARFDebugRangeList &RangeList) const;
   void clear();
   void dump(raw_ostream &OS);
   uint32_t getOffset() const { return Offset; }
@@ -106,11 +112,11 @@ public:
 
   void buildAddressRangeTable(DWARFDebugAranges *debug_aranges,
                               bool clear_dies_if_already_not_parsed);
-  /// getFunctionDIEForAddress - Returns pointer to parsed subprogram DIE,
-  /// address ranges of which contain the provided address,
-  /// or NULL if there is no such subprogram. The pointer
-  /// is valid until DWARFCompileUnit::clear() or clearDIEs() is called.
-  const DWARFDebugInfoEntryMinimal *getFunctionDIEForAddress(int64_t address);
+
+  /// getInlinedChainForAddress - fetches inlined chain for a given address.
+  /// Returns empty chain if there is no subprogram containing address.
+  DWARFDebugInfoEntryMinimal::InlinedChain getInlinedChainForAddress(
+      uint64_t Address);
 };
 
 }
diff --git a/lib/DebugInfo/DWARFContext.cpp b/lib/DebugInfo/DWARFContext.cpp
index 797662b..afd614c 100644
--- a/lib/DebugInfo/DWARFContext.cpp
+++ b/lib/DebugInfo/DWARFContext.cpp
@@ -17,6 +17,8 @@
 using namespace llvm;
 using namespace dwarf;
 
+typedef DWARFDebugLine::LineTable DWARFLineTable;
+
 void DWARFContext::dump(raw_ostream &OS) {
   OS << ".debug_abbrev contents:\n";
   getDebugAbbrev()->dump(OS);
@@ -32,15 +34,17 @@ void DWARFContext::dump(raw_ostream &OS) {
   while (set.extract(arangesData, &offset))
     set.dump(OS);
 
+  uint8_t savedAddressByteSize = 0;
   OS << "\n.debug_lines contents:\n";
   for (unsigned i = 0, e = getNumCompileUnits(); i != e; ++i) {
     DWARFCompileUnit *cu = getCompileUnitAtIndex(i);
+    savedAddressByteSize = cu->getAddressByteSize();
     unsigned stmtOffset =
       cu->getCompileUnitDIE()->getAttributeValueAsUnsigned(cu, DW_AT_stmt_list,
                                                            -1U);
     if (stmtOffset != -1U) {
       DataExtractor lineData(getLineSection(), isLittleEndian(),
-                             cu->getAddressByteSize());
+                             savedAddressByteSize);
       DWARFDebugLine::DumpingState state(OS);
       DWARFDebugLine::parseStatementTable(lineData, &stmtOffset, state);
     }
@@ -54,6 +58,18 @@ void DWARFContext::dump(raw_ostream &OS) {
     OS << format("0x%8.8x: \"%s\"\n", lastOffset, s);
     lastOffset = offset;
   }
+
+  OS << "\n.debug_ranges contents:\n";
+  // In fact, different compile units may have different address byte
+  // sizes, but for simplicity we just use the address byte size of the last
+  // compile unit (there is no easy and fast way to associate address range
+  // list and the compile unit it describes).
+  DataExtractor rangesData(getRangeSection(), isLittleEndian(),
+                           savedAddressByteSize);
+  offset = 0;
+  DWARFDebugRangeList rangeList;
+  while (rangeList.extract(rangesData, &offset))
+    rangeList.dump(OS);
 }
 
 const DWARFDebugAbbrev *DWARFContext::getDebugAbbrev() {
@@ -80,7 +96,7 @@ const DWARFDebugAranges *DWARFContext::getDebugAranges() {
   return Aranges.get();
 }
 
-const DWARFDebugLine::LineTable *
+const DWARFLineTable *
 DWARFContext::getLineTableForCompileUnit(DWARFCompileUnit *cu) {
   if (!Line)
     Line.reset(new DWARFDebugLine());
@@ -92,7 +108,7 @@ DWARFContext::getLineTableForCompileUnit(DWARFCompileUnit *cu) {
     return 0; // No line table for this compile unit.
 
   // See if the line table is cached.
-  if (const DWARFDebugLine::LineTable *lt = Line->getLineTable(stmtOffset))
+  if (const DWARFLineTable *lt = Line->getLineTable(stmtOffset))
     return lt;
 
   // We have to parse it first.
@@ -103,11 +119,11 @@ DWARFContext::getLineTableForCompileUnit(DWARFCompileUnit *cu) {
 
 void DWARFContext::parseCompileUnits() {
   uint32_t offset = 0;
-  const DataExtractor &debug_info_data = DataExtractor(getInfoSection(),
-                                                       isLittleEndian(), 0);
-  while (debug_info_data.isValidOffset(offset)) {
+  const DataExtractor &DIData = DataExtractor(getInfoSection(),
+                                              isLittleEndian(), 0);
+  while (DIData.isValidOffset(offset)) {
     CUs.push_back(DWARFCompileUnit(*this));
-    if (!CUs.back().extract(debug_info_data, &offset)) {
+    if (!CUs.back().extract(DIData, &offset)) {
       CUs.pop_back();
       break;
     }
@@ -131,75 +147,155 @@ namespace {
   };
 }
 
-DWARFCompileUnit *DWARFContext::getCompileUnitForOffset(uint32_t offset) {
+DWARFCompileUnit *DWARFContext::getCompileUnitForOffset(uint32_t Offset) {
   if (CUs.empty())
     parseCompileUnits();
 
-  DWARFCompileUnit *i = std::lower_bound(CUs.begin(), CUs.end(), offset,
-                                         OffsetComparator());
-  if (i != CUs.end())
-    return &*i;
+  DWARFCompileUnit *CU = std::lower_bound(CUs.begin(), CUs.end(), Offset,
+                                          OffsetComparator());
+  if (CU != CUs.end())
+    return &*CU;
   return 0;
 }
 
-DILineInfo DWARFContext::getLineInfoForAddress(uint64_t address,
-    DILineInfoSpecifier specifier) {
+DWARFCompileUnit *DWARFContext::getCompileUnitForAddress(uint64_t Address) {
   // First, get the offset of the compile unit.
-  uint32_t cuOffset = getDebugAranges()->findAddress(address);
+  uint32_t CUOffset = getDebugAranges()->findAddress(Address);
   // Retrieve the compile unit.
-  DWARFCompileUnit *cu = getCompileUnitForOffset(cuOffset);
-  if (!cu)
+  return getCompileUnitForOffset(CUOffset);
+}
+
+static bool getFileNameForCompileUnit(DWARFCompileUnit *CU,
+                                      const DWARFLineTable *LineTable,
+                                      uint64_t FileIndex,
+                                      bool NeedsAbsoluteFilePath,
+                                      std::string &FileName) {
+  if (CU == 0 ||
+      LineTable == 0 ||
+      !LineTable->getFileNameByIndex(FileIndex, NeedsAbsoluteFilePath,
+                                     FileName))
+    return false;
+  if (NeedsAbsoluteFilePath && sys::path::is_relative(FileName)) {
+    // We may still need to append compilation directory of compile unit.
+    SmallString<16> AbsolutePath;
+    if (const char *CompilationDir = CU->getCompilationDir()) {
+      sys::path::append(AbsolutePath, CompilationDir);
+    }
+    sys::path::append(AbsolutePath, FileName);
+    FileName = AbsolutePath.str();
+  }
+  return true;
+}
+
+static bool getFileLineInfoForCompileUnit(DWARFCompileUnit *CU,
+                                          const DWARFLineTable *LineTable,
+                                          uint64_t Address,
+                                          bool NeedsAbsoluteFilePath,
+                                          std::string &FileName,
+                                          uint32_t &Line, uint32_t &Column) {
+  if (CU == 0 || LineTable == 0)
+    return false;
+  // Get the index of row we're looking for in the line table.
+  uint32_t RowIndex = LineTable->lookupAddress(Address);
+  if (RowIndex == -1U)
+    return false;
+  // Take file number and line/column from the row.
+  const DWARFDebugLine::Row &Row = LineTable->Rows[RowIndex];
+  if (!getFileNameForCompileUnit(CU, LineTable, Row.File,
+                                 NeedsAbsoluteFilePath, FileName))
+    return false;
+  Line = Row.Line;
+  Column = Row.Column;
+  return true;
+}
+
+DILineInfo DWARFContext::getLineInfoForAddress(uint64_t Address,
+    DILineInfoSpecifier Specifier) {
+  DWARFCompileUnit *CU = getCompileUnitForAddress(Address);
+  if (!CU)
     return DILineInfo();
-  SmallString<16> fileName("<invalid>");
-  SmallString<16> functionName("<invalid>");
-  uint32_t line = 0;
-  uint32_t column = 0;
-  if (specifier.needs(DILineInfoSpecifier::FunctionName)) {
-    const DWARFDebugInfoEntryMinimal *function_die =
-        cu->getFunctionDIEForAddress(address);
-    if (function_die) {
-      if (const char *name = function_die->getSubprogramName(cu))
-        functionName = name;
+  std::string FileName = "<invalid>";
+  std::string FunctionName = "<invalid>";
+  uint32_t Line = 0;
+  uint32_t Column = 0;
+  if (Specifier.needs(DILineInfoSpecifier::FunctionName)) {
+    // The address may correspond to instruction in some inlined function,
+    // so we have to build the chain of inlined functions and take the
+    // name of the topmost function in it.
+    const DWARFDebugInfoEntryMinimal::InlinedChain &InlinedChain =
+        CU->getInlinedChainForAddress(Address);
+    if (InlinedChain.size() > 0) {
+      const DWARFDebugInfoEntryMinimal &TopFunctionDIE = InlinedChain[0];
+      if (const char *Name = TopFunctionDIE.getSubroutineName(CU))
+        FunctionName = Name;
     }
   }
-  if (specifier.needs(DILineInfoSpecifier::FileLineInfo)) {
-    // Get the line table for this compile unit.
-    const DWARFDebugLine::LineTable *lineTable = getLineTableForCompileUnit(cu);
-    if (lineTable) {
-      // Get the index of the row we're looking for in the line table.
-      uint32_t rowIndex = lineTable->lookupAddress(address);
-      if (rowIndex != -1U) {
-        const DWARFDebugLine::Row &row = lineTable->Rows[rowIndex];
-        // Take file/line info from the line table.
-        const DWARFDebugLine::FileNameEntry &fileNameEntry =
-            lineTable->Prologue.FileNames[row.File - 1];
-        fileName = fileNameEntry.Name;
-        if (specifier.needs(DILineInfoSpecifier::AbsoluteFilePath) &&
-            sys::path::is_relative(fileName.str())) {
-          // Append include directory of file (if it is present in line table)
-          // and compilation directory of compile unit to make path absolute.
-          const char *includeDir = 0;
-          if (uint64_t includeDirIndex = fileNameEntry.DirIdx) {
-            includeDir = lineTable->Prologue
-                         .IncludeDirectories[includeDirIndex - 1];
-          }
-          SmallString<16> absFileName;
-          if (includeDir == 0 || sys::path::is_relative(includeDir)) {
-            if (const char *compilationDir = cu->getCompilationDir())
-              sys::path::append(absFileName, compilationDir);
-          }
-          if (includeDir) {
-            sys::path::append(absFileName, includeDir);
-          }
-          sys::path::append(absFileName, fileName.str());
-          fileName = absFileName;
-        }
-        line = row.Line;
-        column = row.Column;
+  if (Specifier.needs(DILineInfoSpecifier::FileLineInfo)) {
+    const DWARFLineTable *LineTable = getLineTableForCompileUnit(CU);
+    const bool NeedsAbsoluteFilePath =
+        Specifier.needs(DILineInfoSpecifier::AbsoluteFilePath);
+    getFileLineInfoForCompileUnit(CU, LineTable, Address,
+                                  NeedsAbsoluteFilePath,
+                                  FileName, Line, Column);
+  }
+  return DILineInfo(StringRef(FileName), StringRef(FunctionName),
+                    Line, Column);
+}
+
+DIInliningInfo DWARFContext::getInliningInfoForAddress(uint64_t Address,
+    DILineInfoSpecifier Specifier) {
+  DWARFCompileUnit *CU = getCompileUnitForAddress(Address);
+  if (!CU)
+    return DIInliningInfo();
+
+  const DWARFDebugInfoEntryMinimal::InlinedChain &InlinedChain =
+      CU->getInlinedChainForAddress(Address);
+  if (InlinedChain.size() == 0)
+    return DIInliningInfo();
+
+  DIInliningInfo InliningInfo;
+  uint32_t CallFile = 0, CallLine = 0, CallColumn = 0;
+  const DWARFLineTable *LineTable = 0;
+  for (uint32_t i = 0, n = InlinedChain.size(); i != n; i++) {
+    const DWARFDebugInfoEntryMinimal &FunctionDIE = InlinedChain[i];
+    std::string FileName = "<invalid>";
+    std::string FunctionName = "<invalid>";
+    uint32_t Line = 0;
+    uint32_t Column = 0;
+    // Get function name if necessary.
+    if (Specifier.needs(DILineInfoSpecifier::FunctionName)) {
+      if (const char *Name = FunctionDIE.getSubroutineName(CU))
+        FunctionName = Name;
+    }
+    if (Specifier.needs(DILineInfoSpecifier::FileLineInfo)) {
+      const bool NeedsAbsoluteFilePath =
+          Specifier.needs(DILineInfoSpecifier::AbsoluteFilePath);
+      if (i == 0) {
+        // For the topmost frame, initialize the line table of this
+        // compile unit and fetch file/line info from it.
+        LineTable = getLineTableForCompileUnit(CU);
+        // For the topmost routine, get file/line info from line table.
+        getFileLineInfoForCompileUnit(CU, LineTable, Address,
+                                      NeedsAbsoluteFilePath,
+                                      FileName, Line, Column);
+      } else {
+        // Otherwise, use call file, call line and call column from
+        // previous DIE in inlined chain.
+        getFileNameForCompileUnit(CU, LineTable, CallFile,
+                                  NeedsAbsoluteFilePath, FileName);
+        Line = CallLine;
+        Column = CallColumn;
+      }
+      // Get call file/line/column of a current DIE.
+      if (i + 1 < n) {
+        FunctionDIE.getCallerFrame(CU, CallFile, CallLine, CallColumn);
       }
     }
+    DILineInfo Frame(StringRef(FileName), StringRef(FunctionName),
+                     Line, Column);
+    InliningInfo.addFrame(Frame);
   }
-  return DILineInfo(fileName, functionName, line, column);
+  return InliningInfo;
 }
 
 void DWARFContextInMemory::anchor() { }
diff --git a/lib/DebugInfo/DWARFContext.h b/lib/DebugInfo/DWARFContext.h
index e55a27e..4001792 100644
--- a/lib/DebugInfo/DWARFContext.h
+++ b/lib/DebugInfo/DWARFContext.h
@@ -13,6 +13,7 @@
 #include "DWARFCompileUnit.h"
 #include "DWARFDebugAranges.h"
 #include "DWARFDebugLine.h"
+#include "DWARFDebugRangeList.h"
 #include "llvm/DebugInfo/DIContext.h"
 #include "llvm/ADT/OwningPtr.h"
 #include "llvm/ADT/SmallVector.h"
@@ -25,21 +26,24 @@ namespace llvm {
 /// methods that a concrete implementation provides.
 class DWARFContext : public DIContext {
   bool IsLittleEndian;
+  const RelocAddrMap &RelocMap;
 
   SmallVector<DWARFCompileUnit, 1> CUs;
   OwningPtr<DWARFDebugAbbrev> Abbrev;
   OwningPtr<DWARFDebugAranges> Aranges;
   OwningPtr<DWARFDebugLine> Line;
 
-  DWARFContext(DWARFContext &); // = delete
-  DWARFContext &operator=(DWARFContext &); // = delete
+  DWARFContext(DWARFContext &) LLVM_DELETED_FUNCTION;
+  DWARFContext &operator=(DWARFContext &) LLVM_DELETED_FUNCTION;
 
   /// Read compile units from the debug_info section and store them in CUs.
   void parseCompileUnits();
 protected:
-  DWARFContext(bool isLittleEndian) : IsLittleEndian(isLittleEndian) {}
+  DWARFContext(bool isLittleEndian, const RelocAddrMap &Map) :
+    IsLittleEndian(isLittleEndian), RelocMap(Map) {}
 public:
   virtual void dump(raw_ostream &OS);
+
   /// Get the number of compile units in this context.
   unsigned getNumCompileUnits() {
     if (CUs.empty())
@@ -53,9 +57,6 @@ public:
     return &CUs[index];
   }
 
-  /// Return the compile unit that includes an offset (relative to .debug_info).
-  DWARFCompileUnit *getCompileUnitForOffset(uint32_t offset);
-
   /// Get a pointer to the parsed DebugAbbrev object.
   const DWARFDebugAbbrev *getDebugAbbrev();
 
@@ -66,22 +67,32 @@ public:
   const DWARFDebugLine::LineTable *
   getLineTableForCompileUnit(DWARFCompileUnit *cu);
 
-  virtual DILineInfo getLineInfoForAddress(uint64_t address,
-      DILineInfoSpecifier specifier = DILineInfoSpecifier());
+  virtual DILineInfo getLineInfoForAddress(uint64_t Address,
+      DILineInfoSpecifier Specifier = DILineInfoSpecifier());
+  virtual DIInliningInfo getInliningInfoForAddress(uint64_t Address,
+      DILineInfoSpecifier Specifier = DILineInfoSpecifier());
 
   bool isLittleEndian() const { return IsLittleEndian; }
+  const RelocAddrMap &relocMap() const { return RelocMap; }
 
   virtual StringRef getInfoSection() = 0;
   virtual StringRef getAbbrevSection() = 0;
   virtual StringRef getARangeSection() = 0;
   virtual StringRef getLineSection() = 0;
   virtual StringRef getStringSection() = 0;
+  virtual StringRef getRangeSection() = 0;
 
   static bool isSupportedVersion(unsigned version) {
     return version == 2 || version == 3;
   }
-};
+private:
+  /// Return the compile unit that includes an offset (relative to .debug_info).
+  DWARFCompileUnit *getCompileUnitForOffset(uint32_t Offset);
 
+  /// Return the compile unit which contains instruction with provided
+  /// address.
+  DWARFCompileUnit *getCompileUnitForAddress(uint64_t Address);
+};
 
 /// DWARFContextInMemory is the simplest possible implementation of a
 /// DWARFContext. It assumes all content is available in memory and stores
@@ -93,19 +104,23 @@ class DWARFContextInMemory : public DWARFContext {
   StringRef ARangeSection;
   StringRef LineSection;
   StringRef StringSection;
+  StringRef RangeSection;
 public:
   DWARFContextInMemory(bool isLittleEndian,
                        StringRef infoSection,
                        StringRef abbrevSection,
                        StringRef aRangeSection,
                        StringRef lineSection,
-                       StringRef stringSection)
-    : DWARFContext(isLittleEndian),
+                       StringRef stringSection,
+                       StringRef rangeSection,
+                       const RelocAddrMap &Map = RelocAddrMap())
+    : DWARFContext(isLittleEndian, Map),
       InfoSection(infoSection),
       AbbrevSection(abbrevSection),
       ARangeSection(aRangeSection),
       LineSection(lineSection),
-      StringSection(stringSection)
+      StringSection(stringSection),
+      RangeSection(rangeSection)
     {}
 
   virtual StringRef getInfoSection() { return InfoSection; }
@@ -113,6 +128,7 @@ public:
   virtual StringRef getARangeSection() { return ARangeSection; }
   virtual StringRef getLineSection() { return LineSection; }
   virtual StringRef getStringSection() { return StringSection; }
+  virtual StringRef getRangeSection() { return RangeSection; }
 };
 
 }
diff --git a/lib/DebugInfo/DWARFDebugAranges.cpp b/lib/DebugInfo/DWARFDebugAranges.cpp
index ef470e5..f9a34c9 100644
--- a/lib/DebugInfo/DWARFDebugAranges.cpp
+++ b/lib/DebugInfo/DWARFDebugAranges.cpp
@@ -62,7 +62,6 @@ bool DWARFDebugAranges::extract(DataExtractor debug_aranges_data) {
     uint32_t offset = 0;
 
     typedef std::vector<DWARFDebugArangeSet> SetCollection;
-    typedef SetCollection::const_iterator SetCollectionIter;
     SetCollection sets;
 
     DWARFDebugArangeSet set;
diff --git a/lib/DebugInfo/DWARFDebugInfoEntry.cpp b/lib/DebugInfo/DWARFDebugInfoEntry.cpp
index 429a36c..ab67464 100644
--- a/lib/DebugInfo/DWARFDebugInfoEntry.cpp
+++ b/lib/DebugInfo/DWARFDebugInfoEntry.cpp
@@ -1,4 +1,4 @@
-//===-- DWARFDebugInfoEntry.cpp --------------------------------------------===//
+//===-- DWARFDebugInfoEntry.cpp -------------------------------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -101,7 +101,7 @@ bool DWARFDebugInfoEntryMinimal::extractFast(const DWARFCompileUnit *cu,
   DataExtractor debug_info_data = cu->getDebugInfoExtractor();
   uint64_t abbrCode = debug_info_data.getULEB128(offset_ptr);
 
-  assert (fixed_form_sizes); // For best performance this should be specified!
+  assert(fixed_form_sizes); // For best performance this should be specified!
 
   if (abbrCode) {
     uint32_t offset = *offset_ptr;
@@ -126,6 +126,7 @@ bool DWARFDebugInfoEntryMinimal::extractFast(const DWARFCompileUnit *cu,
           switch (form) {
           // Blocks if inlined data that have a length field and the data bytes
           // inlined in the .debug_info.
+          case DW_FORM_exprloc:
           case DW_FORM_block:
             form_size = debug_info_data.getULEB128(&offset);
             break;
@@ -150,6 +151,11 @@ bool DWARFDebugInfoEntryMinimal::extractFast(const DWARFCompileUnit *cu,
             form_size = cu->getAddressByteSize();
             break;
 
+          // 0 sized form.
+          case DW_FORM_flag_present:
+            form_size = 0;
+            break;
+
           // 1 byte values
           case DW_FORM_data1:
           case DW_FORM_flag:
@@ -173,6 +179,7 @@ bool DWARFDebugInfoEntryMinimal::extractFast(const DWARFCompileUnit *cu,
           // 8 byte values
           case DW_FORM_data8:
           case DW_FORM_ref8:
+          case DW_FORM_ref_sig8:
             form_size = 8;
             break;
 
@@ -188,6 +195,13 @@ bool DWARFDebugInfoEntryMinimal::extractFast(const DWARFCompileUnit *cu,
             form = debug_info_data.getULEB128(&offset);
             break;
 
+          case DW_FORM_sec_offset:
+            if (cu->getAddressByteSize() == 4)
+              debug_info_data.getU32(offset_ptr);
+            else
+              debug_info_data.getU64(offset_ptr);
+            break;
+
           default:
             *offset_ptr = Offset;
             return false;
@@ -249,6 +263,7 @@ DWARFDebugInfoEntryMinimal::extract(const DWARFCompileUnit *cu,
               switch (form) {
               // Blocks if inlined data that have a length field and the data
               // bytes // inlined in the .debug_info
+              case DW_FORM_exprloc:
               case DW_FORM_block:
                 form_size = debug_info_data.getULEB128(&offset);
                 break;
@@ -273,6 +288,11 @@ DWARFDebugInfoEntryMinimal::extract(const DWARFCompileUnit *cu,
                 form_size = cu_addr_size;
                 break;
 
+              // 0 byte value
+              case DW_FORM_flag_present:
+                form_size = 0;
+                break;
+
               // 1 byte values
               case DW_FORM_data1:
               case DW_FORM_flag:
@@ -299,6 +319,7 @@ DWARFDebugInfoEntryMinimal::extract(const DWARFCompileUnit *cu,
               // 8 byte values
               case DW_FORM_data8:
               case DW_FORM_ref8:
+              case DW_FORM_ref_sig8:
                 form_size = 8;
                 break;
 
@@ -314,6 +335,13 @@ DWARFDebugInfoEntryMinimal::extract(const DWARFCompileUnit *cu,
                 form_is_indirect = true;
                 break;
 
+              case DW_FORM_sec_offset:
+                if (cu->getAddressByteSize() == 4)
+                  debug_info_data.getU32(offset_ptr);
+                else
+                  debug_info_data.getU64(offset_ptr);
+                break;
+
               default:
                 *offset_ptr = offset;
                 return false;
@@ -336,6 +364,16 @@ DWARFDebugInfoEntryMinimal::extract(const DWARFCompileUnit *cu,
   return false;
 }
 
+bool DWARFDebugInfoEntryMinimal::isSubprogramDIE() const {
+  return getTag() == DW_TAG_subprogram;
+}
+
+bool DWARFDebugInfoEntryMinimal::isSubroutineDIE() const {
+  uint32_t Tag = getTag();
+  return Tag == DW_TAG_subprogram ||
+         Tag == DW_TAG_inlined_subroutine;
+}
+
 uint32_t
 DWARFDebugInfoEntryMinimal::getAttributeValue(const DWARFCompileUnit *cu,
                                               const uint16_t attr,
@@ -373,9 +411,10 @@ DWARFDebugInfoEntryMinimal::getAttributeValue(const DWARFCompileUnit *cu,
 
 const char*
 DWARFDebugInfoEntryMinimal::getAttributeValueAsString(
-    const DWARFCompileUnit* cu,
-    const uint16_t attr,
-    const char* fail_value) const {
+                                                     const DWARFCompileUnit* cu,
+                                                     const uint16_t attr,
+                                                     const char* fail_value)
+                                                     const {
   DWARFFormValue form_value;
   if (getAttributeValue(cu, attr, form_value)) {
     DataExtractor stringExtractor(cu->getContext().getStringSection(),
@@ -387,9 +426,9 @@ DWARFDebugInfoEntryMinimal::getAttributeValueAsString(
 
 uint64_t
 DWARFDebugInfoEntryMinimal::getAttributeValueAsUnsigned(
-    const DWARFCompileUnit* cu,
-    const uint16_t attr,
-    uint64_t fail_value) const {
+                                                    const DWARFCompileUnit* cu,
+                                                    const uint16_t attr,
+                                                    uint64_t fail_value) const {
   DWARFFormValue form_value;
   if (getAttributeValue(cu, attr, form_value))
       return form_value.getUnsigned();
@@ -398,9 +437,9 @@ DWARFDebugInfoEntryMinimal::getAttributeValueAsUnsigned(
 
 int64_t
 DWARFDebugInfoEntryMinimal::getAttributeValueAsSigned(
-    const DWARFCompileUnit* cu,
-    const uint16_t attr,
-    int64_t fail_value) const {
+                                                     const DWARFCompileUnit* cu,
+                                                     const uint16_t attr,
+                                                     int64_t fail_value) const {
   DWARFFormValue form_value;
   if (getAttributeValue(cu, attr, form_value))
       return form_value.getSigned();
@@ -409,33 +448,42 @@ DWARFDebugInfoEntryMinimal::getAttributeValueAsSigned(
 
 uint64_t
 DWARFDebugInfoEntryMinimal::getAttributeValueAsReference(
-                                                  const DWARFCompileUnit* cu,
-                                                  const uint16_t attr,
-                                                  uint64_t fail_value) const {
+                                                     const DWARFCompileUnit* cu,
+                                                     const uint16_t attr,
+                                                     uint64_t fail_value)
+                                                     const {
   DWARFFormValue form_value;
   if (getAttributeValue(cu, attr, form_value))
       return form_value.getReference(cu);
   return fail_value;
 }
 
+bool DWARFDebugInfoEntryMinimal::getLowAndHighPC(const DWARFCompileUnit *CU,
+                                                 uint64_t &LowPC,
+                                                 uint64_t &HighPC) const {
+  HighPC = -1ULL;
+  LowPC = getAttributeValueAsUnsigned(CU, DW_AT_low_pc, -1ULL);
+  if (LowPC != -1ULL)
+    HighPC = getAttributeValueAsUnsigned(CU, DW_AT_high_pc, -1ULL);
+  return (HighPC != -1ULL);
+}
+
 void
-DWARFDebugInfoEntryMinimal::buildAddressRangeTable(const DWARFCompileUnit *cu,
-                                               DWARFDebugAranges *debug_aranges)
+DWARFDebugInfoEntryMinimal::buildAddressRangeTable(const DWARFCompileUnit *CU,
+                                               DWARFDebugAranges *DebugAranges)
                                                    const {
   if (AbbrevDecl) {
-    uint16_t tag = AbbrevDecl->getTag();
-    if (tag == DW_TAG_subprogram) {
-      uint64_t hi_pc = -1ULL;
-      uint64_t lo_pc = getAttributeValueAsUnsigned(cu, DW_AT_low_pc, -1ULL);
-      if (lo_pc != -1ULL)
-        hi_pc = getAttributeValueAsUnsigned(cu, DW_AT_high_pc, -1ULL);
-      if (hi_pc != -1ULL)
-        debug_aranges->appendRange(cu->getOffset(), lo_pc, hi_pc);
+    if (isSubprogramDIE()) {
+      uint64_t LowPC, HighPC;
+      if (getLowAndHighPC(CU, LowPC, HighPC)) {
+        DebugAranges->appendRange(CU->getOffset(), LowPC, HighPC);
+      }
+      // FIXME: try to append ranges from .debug_ranges section.
     }
 
     const DWARFDebugInfoEntryMinimal *child = getFirstChild();
     while (child) {
-      child->buildAddressRangeTable(cu, debug_aranges);
+      child->buildAddressRangeTable(CU, DebugAranges);
       child = child->getSibling();
     }
   }
@@ -443,51 +491,95 @@ DWARFDebugInfoEntryMinimal::buildAddressRangeTable(const DWARFCompileUnit *cu,
 
 bool
 DWARFDebugInfoEntryMinimal::addressRangeContainsAddress(
-    const DWARFCompileUnit *cu, const uint64_t address) const {
-  if (!isNULL() && getTag() == DW_TAG_subprogram) {
-    uint64_t hi_pc = -1ULL;
-    uint64_t lo_pc = getAttributeValueAsUnsigned(cu, DW_AT_low_pc, -1ULL);
-    if (lo_pc != -1ULL)
-      hi_pc = getAttributeValueAsUnsigned(cu, DW_AT_high_pc, -1ULL);
-    if (hi_pc != -1ULL) {
-      return (lo_pc <= address && address < hi_pc);
-    }
+                                                     const DWARFCompileUnit *CU,
+                                                     const uint64_t Address)
+                                                     const {
+  if (isNULL())
+    return false;
+  uint64_t LowPC, HighPC;
+  if (getLowAndHighPC(CU, LowPC, HighPC))
+    return (LowPC <= Address && Address <= HighPC);
+  // Try to get address ranges from .debug_ranges section.
+  uint32_t RangesOffset = getAttributeValueAsReference(CU, DW_AT_ranges, -1U);
+  if (RangesOffset != -1U) {
+    DWARFDebugRangeList RangeList;
+    if (CU->extractRangeList(RangesOffset, RangeList))
+      return RangeList.containsAddress(CU->getBaseAddress(), Address);
   }
   return false;
 }
 
 const char*
-DWARFDebugInfoEntryMinimal::getSubprogramName(
-    const DWARFCompileUnit *cu) const {
-  if (isNULL() || getTag() != DW_TAG_subprogram)
+DWARFDebugInfoEntryMinimal::getSubroutineName(const DWARFCompileUnit *CU)
+                                                                         const {
+  if (!isSubroutineDIE())
     return 0;
   // Try to get mangled name if possible.
   if (const char *name =
-      getAttributeValueAsString(cu, DW_AT_MIPS_linkage_name, 0))
+      getAttributeValueAsString(CU, DW_AT_MIPS_linkage_name, 0))
     return name;
-  if (const char *name = getAttributeValueAsString(cu, DW_AT_linkage_name, 0))
+  if (const char *name = getAttributeValueAsString(CU, DW_AT_linkage_name, 0))
     return name;
-  if (const char *name = getAttributeValueAsString(cu, DW_AT_name, 0))
+  if (const char *name = getAttributeValueAsString(CU, DW_AT_name, 0))
     return name;
   // Try to get name from specification DIE.
   uint32_t spec_ref =
-      getAttributeValueAsReference(cu, DW_AT_specification, -1U);
+      getAttributeValueAsReference(CU, DW_AT_specification, -1U);
   if (spec_ref != -1U) {
     DWARFDebugInfoEntryMinimal spec_die;
-    if (spec_die.extract(cu, &spec_ref)) {
-      if (const char *name = spec_die.getSubprogramName(cu))
+    if (spec_die.extract(CU, &spec_ref)) {
+      if (const char *name = spec_die.getSubroutineName(CU))
         return name;
     }
   }
   // Try to get name from abstract origin DIE.
   uint32_t abs_origin_ref =
-      getAttributeValueAsReference(cu, DW_AT_abstract_origin, -1U);
+      getAttributeValueAsReference(CU, DW_AT_abstract_origin, -1U);
   if (abs_origin_ref != -1U) {
     DWARFDebugInfoEntryMinimal abs_origin_die;
-    if (abs_origin_die.extract(cu, &abs_origin_ref)) {
-      if (const char *name = abs_origin_die.getSubprogramName(cu))
+    if (abs_origin_die.extract(CU, &abs_origin_ref)) {
+      if (const char *name = abs_origin_die.getSubroutineName(CU))
         return name;
     }
   }
   return 0;
 }
+
+void DWARFDebugInfoEntryMinimal::getCallerFrame(const DWARFCompileUnit *CU,
+                                                uint32_t &CallFile,
+                                                uint32_t &CallLine,
+                                                uint32_t &CallColumn) const {
+  CallFile = getAttributeValueAsUnsigned(CU, DW_AT_call_file, 0);
+  CallLine = getAttributeValueAsUnsigned(CU, DW_AT_call_line, 0);
+  CallColumn = getAttributeValueAsUnsigned(CU, DW_AT_call_column, 0);
+}
+
+DWARFDebugInfoEntryMinimal::InlinedChain
+DWARFDebugInfoEntryMinimal::getInlinedChainForAddress(
+                                                     const DWARFCompileUnit *CU,
+                                                     const uint64_t Address)
+                                                     const {
+  DWARFDebugInfoEntryMinimal::InlinedChain InlinedChain;
+  if (isNULL())
+    return InlinedChain;
+  for (const DWARFDebugInfoEntryMinimal *DIE = this; DIE; ) {
+    // Append current DIE to inlined chain only if it has correct tag
+    // (e.g. it is not a lexical block).
+    if (DIE->isSubroutineDIE()) {
+      InlinedChain.push_back(*DIE);
+    }
+    // Try to get child which also contains provided address.
+    const DWARFDebugInfoEntryMinimal *Child = DIE->getFirstChild();
+    while (Child) {
+      if (Child->addressRangeContainsAddress(CU, Address)) {
+        // Assume there is only one such child.
+        break;
+      }
+      Child = Child->getSibling();
+    }
+    DIE = Child;
+  }
+  // Reverse the obtained chain to make the root of inlined chain last.
+  std::reverse(InlinedChain.begin(), InlinedChain.end());
+  return InlinedChain;
+}
diff --git a/lib/DebugInfo/DWARFDebugInfoEntry.h b/lib/DebugInfo/DWARFDebugInfoEntry.h
index d5d86b9..9c1b2be 100644
--- a/lib/DebugInfo/DWARFDebugInfoEntry.h
+++ b/lib/DebugInfo/DWARFDebugInfoEntry.h
@@ -11,6 +11,7 @@
 #define LLVM_DEBUGINFO_DWARFDEBUGINFOENTRY_H
 
 #include "DWARFAbbreviationDeclaration.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/DataTypes.h"
 
 namespace llvm {
@@ -19,6 +20,7 @@ class DWARFDebugAranges;
 class DWARFCompileUnit;
 class DWARFContext;
 class DWARFFormValue;
+class DWARFInlinedSubroutineChain;
 
 /// DWARFDebugInfoEntryMinimal - A DIE with only the minimum required data.
 class DWARFDebugInfoEntryMinimal {
@@ -52,6 +54,13 @@ public:
 
   uint32_t getTag() const { return AbbrevDecl ? AbbrevDecl->getTag() : 0; }
   bool isNULL() const { return AbbrevDecl == 0; }
+
+  /// Returns true if DIE represents a subprogram (not inlined).
+  bool isSubprogramDIE() const;
+  /// Returns true if DIE represents a subprogram or an inlined
+  /// subroutine.
+  bool isSubroutineDIE() const;
+
   uint32_t getOffset() const { return Offset; }
   uint32_t getNumAttributes() const {
     return !isNULL() ? AbbrevDecl->getNumAttributes() : 0;
@@ -126,17 +135,40 @@ public:
                                     const uint16_t attr,
                                     int64_t fail_value) const;
 
-  void buildAddressRangeTable(const DWARFCompileUnit *cu,
-                              DWARFDebugAranges *debug_aranges) const;
-
-  bool addressRangeContainsAddress(const DWARFCompileUnit *cu,
-                                   const uint64_t address) const;
-
-  // If a DIE represents a subprogram, returns its mangled name
-  // (or short name, if mangled is missing). This name may be fetched
-  // from specification or abstract origin for this subprogram.
-  // Returns null if no name is found.
-  const char* getSubprogramName(const DWARFCompileUnit *cu) const;
+  /// Retrieves DW_AT_low_pc and DW_AT_high_pc from CU.
+  /// Returns true if both attributes are present.
+  bool getLowAndHighPC(const DWARFCompileUnit *CU,
+                       uint64_t &LowPC, uint64_t &HighPC) const;
+
+  void buildAddressRangeTable(const DWARFCompileUnit *CU,
+                              DWARFDebugAranges *DebugAranges) const;
+
+  bool addressRangeContainsAddress(const DWARFCompileUnit *CU,
+                                   const uint64_t Address) const;
+
+  /// If a DIE represents a subprogram (or inlined subroutine),
+  /// returns its mangled name (or short name, if mangled is missing).
+  /// This name may be fetched from specification or abstract origin
+  /// for this subprogram. Returns null if no name is found.
+  const char* getSubroutineName(const DWARFCompileUnit *CU) const;
+
+  /// Retrieves values of DW_AT_call_file, DW_AT_call_line and
+  /// DW_AT_call_column from DIE (or zeroes if they are missing).
+  void getCallerFrame(const DWARFCompileUnit *CU, uint32_t &CallFile,
+                      uint32_t &CallLine, uint32_t &CallColumn) const;
+
+  /// InlinedChain - represents a chain of inlined_subroutine
+  /// DIEs, (possibly ending with subprogram DIE), all of which are contained
+  /// in some concrete inlined instance tree. Address range for each DIE
+  /// (except the last DIE) in this chain is contained in address
+  /// range for next DIE in the chain.
+  typedef SmallVector<DWARFDebugInfoEntryMinimal, 4> InlinedChain;
+
+  /// Get inlined chain for a given address, rooted at the current DIE.
+  /// Returns empty chain if address is not contained in address range
+  /// of current DIE.
+  InlinedChain getInlinedChainForAddress(const DWARFCompileUnit *CU,
+                                         const uint64_t Address) const;
 };
 
 }
diff --git a/lib/DebugInfo/DWARFDebugLine.cpp b/lib/DebugInfo/DWARFDebugLine.cpp
index d99575d..267364a 100644
--- a/lib/DebugInfo/DWARFDebugLine.cpp
+++ b/lib/DebugInfo/DWARFDebugLine.cpp
@@ -10,6 +10,7 @@
 #include "DWARFDebugLine.h"
 #include "llvm/Support/Dwarf.h"
 #include "llvm/Support/Format.h"
+#include "llvm/Support/Path.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
 using namespace llvm;
@@ -513,3 +514,29 @@ DWARFDebugLine::LineTable::lookupAddress(uint64_t address) const {
   }
   return index;
 }
+
+bool
+DWARFDebugLine::LineTable::getFileNameByIndex(uint64_t FileIndex,
+                                              bool NeedsAbsoluteFilePath,
+                                              std::string &Result) const {
+  if (FileIndex == 0 || FileIndex > Prologue.FileNames.size())
+    return false;
+  const FileNameEntry &Entry = Prologue.FileNames[FileIndex - 1];
+  const char *FileName = Entry.Name;
+  if (!NeedsAbsoluteFilePath ||
+      sys::path::is_absolute(FileName)) {
+    Result = FileName;
+    return true;
+  }
+  SmallString<16> FilePath;
+  uint64_t IncludeDirIndex = Entry.DirIdx;
+  // Be defensive about the contents of Entry.
+  if (IncludeDirIndex > 0 &&
+      IncludeDirIndex <= Prologue.IncludeDirectories.size()) {
+    const char *IncludeDir = Prologue.IncludeDirectories[IncludeDirIndex - 1];
+    sys::path::append(FilePath, IncludeDir);
+  }
+  sys::path::append(FilePath, FileName);
+  Result = FilePath.str();
+  return true;
+}
diff --git a/lib/DebugInfo/DWARFDebugLine.h b/lib/DebugInfo/DWARFDebugLine.h
index 6382b45..586dd7e 100644
--- a/lib/DebugInfo/DWARFDebugLine.h
+++ b/lib/DebugInfo/DWARFDebugLine.h
@@ -12,6 +12,7 @@
 
 #include "llvm/Support/DataExtractor.h"
 #include <map>
+#include <string>
 #include <vector>
 
 namespace llvm {
@@ -174,6 +175,13 @@ public:
     // Returns the index of the row with file/line info for a given address,
     // or -1 if there is no such row.
     uint32_t lookupAddress(uint64_t address) const;
+
+    // Extracts filename by its index in filename table in prologue.
+    // Returns true on success.
+    bool getFileNameByIndex(uint64_t FileIndex,
+                            bool NeedsAbsoluteFilePath,
+                            std::string &Result) const;
+
     void dump(raw_ostream &OS) const;
 
     struct Prologue Prologue;
diff --git a/lib/DebugInfo/DWARFDebugRangeList.cpp b/lib/DebugInfo/DWARFDebugRangeList.cpp
new file mode 100644
index 0000000..1806bee
--- /dev/null
+++ b/lib/DebugInfo/DWARFDebugRangeList.cpp
@@ -0,0 +1,67 @@
+//===-- DWARFDebugRangesList.cpp ------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "DWARFDebugRangeList.h"
+#include "llvm/Support/Format.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+
+void DWARFDebugRangeList::clear() {
+  Offset = -1U;
+  AddressSize = 0;
+  Entries.clear();
+}
+
+bool DWARFDebugRangeList::extract(DataExtractor data, uint32_t *offset_ptr) {
+  clear();
+  if (!data.isValidOffset(*offset_ptr))
+    return false;
+  AddressSize = data.getAddressSize();
+  if (AddressSize != 4 && AddressSize != 8)
+    return false;
+  Offset = *offset_ptr;
+  while (true) {
+    RangeListEntry entry;
+    uint32_t prev_offset = *offset_ptr;
+    entry.StartAddress = data.getAddress(offset_ptr);
+    entry.EndAddress = data.getAddress(offset_ptr);
+    // Check that both values were extracted correctly.
+    if (*offset_ptr != prev_offset + 2 * AddressSize) {
+      clear();
+      return false;
+    }
+    if (entry.isEndOfListEntry())
+      break;
+    Entries.push_back(entry);
+  }
+  return true;
+}
+
+void DWARFDebugRangeList::dump(raw_ostream &OS) const {
+  for (int i = 0, n = Entries.size(); i != n; ++i) {
+    const char *format_str = (AddressSize == 4
+                              ? "%08x %08"  PRIx64 " %08"  PRIx64 "\n"
+                              : "%08x %016" PRIx64 " %016" PRIx64 "\n");
+    OS << format(format_str, Offset, Entries[i].StartAddress,
+                                     Entries[i].EndAddress);
+  }
+  OS << format("%08x <End of list>\n", Offset);
+}
+
+bool DWARFDebugRangeList::containsAddress(uint64_t BaseAddress,
+                                          uint64_t Address) const {
+  for (int i = 0, n = Entries.size(); i != n; ++i) {
+    if (Entries[i].isBaseAddressSelectionEntry(AddressSize))
+      BaseAddress = Entries[i].EndAddress;
+    else if (Entries[i].containsAddress(BaseAddress, Address))
+      return true;
+  }
+  return false;
+}
diff --git a/lib/DebugInfo/DWARFDebugRangeList.h b/lib/DebugInfo/DWARFDebugRangeList.h
new file mode 100644
index 0000000..4e34a91
--- /dev/null
+++ b/lib/DebugInfo/DWARFDebugRangeList.h
@@ -0,0 +1,78 @@
+//===-- DWARFDebugRangeList.h -----------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_DEBUGINFO_DWARFDEBUGRANGELIST_H
+#define LLVM_DEBUGINFO_DWARFDEBUGRANGELIST_H
+
+#include "llvm/Support/DataExtractor.h"
+#include <vector>
+
+namespace llvm {
+
+class raw_ostream;
+
+class DWARFDebugRangeList {
+public:
+  struct RangeListEntry {
+    // A beginning address offset. This address offset has the size of an
+    // address and is relative to the applicable base address of the
+    // compilation unit referencing this range list. It marks the beginning
+    // of an address range.
+    uint64_t StartAddress;
+    // An ending address offset. This address offset again has the size of
+    // an address and is relative to the applicable base address of the
+    // compilation unit referencing this range list. It marks the first
+    // address past the end of the address range. The ending address must
+    // be greater than or equal to the beginning address.
+    uint64_t EndAddress;
+    // The end of any given range list is marked by an end of list entry,
+    // which consists of a 0 for the beginning address offset
+    // and a 0 for the ending address offset.
+    bool isEndOfListEntry() const {
+      return (StartAddress == 0) && (EndAddress == 0);
+    }
+    // A base address selection entry consists of:
+    // 1. The value of the largest representable address offset
+    // (for example, 0xffffffff when the size of an address is 32 bits).
+    // 2. An address, which defines the appropriate base address for
+    // use in interpreting the beginning and ending address offsets of
+    // subsequent entries of the location list.
+    bool isBaseAddressSelectionEntry(uint8_t AddressSize) const {
+      assert(AddressSize == 4 || AddressSize == 8);
+      if (AddressSize == 4)
+        return StartAddress == -1U;
+      else
+        return StartAddress == -1ULL;
+    }
+    bool containsAddress(uint64_t BaseAddress, uint64_t Address) const {
+      return (BaseAddress + StartAddress <= Address) &&
+             (Address < BaseAddress + EndAddress);
+    }
+  };
+
+private:
+  // Offset in .debug_ranges section.
+  uint32_t Offset;
+  uint8_t AddressSize;
+  std::vector<RangeListEntry> Entries;
+
+public:
+  DWARFDebugRangeList() { clear(); }
+  void clear();
+  void dump(raw_ostream &OS) const;
+  bool extract(DataExtractor data, uint32_t *offset_ptr);
+  /// containsAddress - Returns true if range list contains the given
+  /// address. Has to be passed base address of the compile unit that
+  /// references this range list.
+  bool containsAddress(uint64_t BaseAddress, uint64_t Address) const;
+};
+
+}  // namespace llvm
+
+#endif  // LLVM_DEBUGINFO_DWARFDEBUGRANGELIST_H
diff --git a/lib/DebugInfo/DWARFFormValue.cpp b/lib/DebugInfo/DWARFFormValue.cpp
index ee2a3ab..fea9fd7 100644
--- a/lib/DebugInfo/DWARFFormValue.cpp
+++ b/lib/DebugInfo/DWARFFormValue.cpp
@@ -10,6 +10,7 @@
 #include "DWARFFormValue.h"
 #include "DWARFCompileUnit.h"
 #include "DWARFContext.h"
+#include "llvm/Support/Debug.h"
 #include "llvm/Support/Dwarf.h"
 #include "llvm/Support/Format.h"
 #include "llvm/Support/raw_ostream.h"
@@ -41,6 +42,10 @@ static const uint8_t form_sizes_addr4[] = {
   8, // 0x14 DW_FORM_ref8
   0, // 0x15 DW_FORM_ref_udata
   0, // 0x16 DW_FORM_indirect
+  4, // 0x17 DW_FORM_sec_offset
+  0, // 0x18 DW_FORM_exprloc
+  0, // 0x19 DW_FORM_flag_present
+  8, // 0x20 DW_FORM_ref_sig8
 };
 
 static const uint8_t form_sizes_addr8[] = {
@@ -67,6 +72,10 @@ static const uint8_t form_sizes_addr8[] = {
   8, // 0x14 DW_FORM_ref8
   0, // 0x15 DW_FORM_ref_udata
   0, // 0x16 DW_FORM_indirect
+  8, // 0x17 DW_FORM_sec_offset
+  0, // 0x18 DW_FORM_exprloc
+  0, // 0x19 DW_FORM_flag_present
+  8, // 0x20 DW_FORM_ref_sig8
 };
 
 const uint8_t *
@@ -90,9 +99,18 @@ DWARFFormValue::extractValue(DataExtractor data, uint32_t *offset_ptr,
     indirect = false;
     switch (Form) {
     case DW_FORM_addr:
-    case DW_FORM_ref_addr:
-      Value.uval = data.getUnsigned(offset_ptr, cu->getAddressByteSize());
+    case DW_FORM_ref_addr: {
+      RelocAddrMap::const_iterator AI
+        = cu->getContext().relocMap().find(*offset_ptr);
+      if (AI != cu->getContext().relocMap().end()) {
+        const std::pair<uint8_t, int64_t> &R = AI->second;
+        Value.uval = R.second;
+        *offset_ptr += R.first;
+      } else
+        Value.uval = data.getUnsigned(offset_ptr, cu->getAddressByteSize());
+    }
       break;
+    case DW_FORM_exprloc:
     case DW_FORM_block:
       Value.uval = data.getULEB128(offset_ptr);
       is_block = true;
@@ -129,9 +147,17 @@ DWARFFormValue::extractValue(DataExtractor data, uint32_t *offset_ptr,
     case DW_FORM_sdata:
       Value.sval = data.getSLEB128(offset_ptr);
       break;
-    case DW_FORM_strp:
-      Value.uval = data.getU32(offset_ptr);
+    case DW_FORM_strp: {
+      RelocAddrMap::const_iterator AI
+        = cu->getContext().relocMap().find(*offset_ptr);
+      if (AI != cu->getContext().relocMap().end()) {
+        const std::pair<uint8_t, int64_t> &R = AI->second;
+        Value.uval = R.second;
+        *offset_ptr += R.first;
+      } else
+        Value.uval = data.getU32(offset_ptr);
       break;
+    }
     case DW_FORM_udata:
     case DW_FORM_ref_udata:
       Value.uval = data.getULEB128(offset_ptr);
@@ -141,12 +167,24 @@ DWARFFormValue::extractValue(DataExtractor data, uint32_t *offset_ptr,
       // Set the string value to also be the data for inlined cstr form
       // values only so we can tell the differnence between DW_FORM_string
       // and DW_FORM_strp form values
-      Value.data = (uint8_t*)Value.cstr;
+      Value.data = (const uint8_t*)Value.cstr;
       break;
     case DW_FORM_indirect:
       Form = data.getULEB128(offset_ptr);
       indirect = true;
       break;
+    case DW_FORM_sec_offset:
+      if (cu->getAddressByteSize() == 4)
+        Value.uval = data.getU32(offset_ptr);
+      else
+        Value.uval = data.getU64(offset_ptr);
+      break;
+    case DW_FORM_flag_present:
+      Value.uval = 1;
+      break;
+    case DW_FORM_ref_sig8:
+      Value.uval = data.getU64(offset_ptr);
+      break;
     default:
       return false;
     }
@@ -179,6 +217,7 @@ DWARFFormValue::skipValue(uint16_t form, DataExtractor debug_info_data,
     switch (form) {
     // Blocks if inlined data that have a length field and the data bytes
     // inlined in the .debug_info
+    case DW_FORM_exprloc:
     case DW_FORM_block: {
       uint64_t size = debug_info_data.getULEB128(offset_ptr);
       *offset_ptr += size;
@@ -211,6 +250,10 @@ DWARFFormValue::skipValue(uint16_t form, DataExtractor debug_info_data,
       *offset_ptr += cu->getAddressByteSize();
       return true;
 
+    // 0 byte values - implied from the form.
+    case DW_FORM_flag_present:
+      return true;
+      
     // 1 byte values
     case DW_FORM_data1:
     case DW_FORM_flag:
@@ -234,6 +277,7 @@ DWARFFormValue::skipValue(uint16_t form, DataExtractor debug_info_data,
     // 8 byte values
     case DW_FORM_data8:
     case DW_FORM_ref8:
+    case DW_FORM_ref_sig8:
       *offset_ptr += 8;
       return true;
 
@@ -249,6 +293,15 @@ DWARFFormValue::skipValue(uint16_t form, DataExtractor debug_info_data,
       indirect = true;
       form = debug_info_data.getULEB128(offset_ptr);
       break;
+
+    // 4 for DWARF32, 8 for DWARF64.
+    case DW_FORM_sec_offset:
+      if (cu->getAddressByteSize() == 4)
+        *offset_ptr += 4;
+      else
+        *offset_ptr += 8;
+      return true;
+      
     default:
       return false;
     }
@@ -264,22 +317,26 @@ DWARFFormValue::dump(raw_ostream &OS, const DWARFCompileUnit *cu) const {
 
   switch (Form) {
   case DW_FORM_addr:      OS << format("0x%016" PRIx64, uvalue); break;
+  case DW_FORM_flag_present: OS << "true"; break;
   case DW_FORM_flag:
   case DW_FORM_data1:     OS << format("0x%02x", (uint8_t)uvalue); break;
   case DW_FORM_data2:     OS << format("0x%04x", (uint16_t)uvalue); break;
   case DW_FORM_data4:     OS << format("0x%08x", (uint32_t)uvalue); break;
+  case DW_FORM_ref_sig8:
   case DW_FORM_data8:     OS << format("0x%016" PRIx64, uvalue); break;
   case DW_FORM_string:
     OS << '"';
     OS.write_escaped(getAsCString(NULL));
     OS << '"';
     break;
+  case DW_FORM_exprloc:
   case DW_FORM_block:
   case DW_FORM_block1:
   case DW_FORM_block2:
   case DW_FORM_block4:
     if (uvalue > 0) {
       switch (Form) {
+      case DW_FORM_exprloc:
       case DW_FORM_block:  OS << format("<0x%" PRIx64 "> ", uvalue);     break;
       case DW_FORM_block1: OS << format("<0x%2.2x> ", (uint8_t)uvalue);  break;
       case DW_FORM_block2: OS << format("<0x%4.4x> ", (uint16_t)uvalue); break;
@@ -342,6 +399,14 @@ DWARFFormValue::dump(raw_ostream &OS, const DWARFCompileUnit *cu) const {
   case DW_FORM_indirect:
     OS << "DW_FORM_indirect";
     break;
+
+  case DW_FORM_sec_offset:
+    if (cu->getAddressByteSize() == 4)
+      OS << format("0x%08x", (uint32_t)uvalue);
+    else
+      OS << format("0x%016" PRIx64, uvalue);
+    break;
+    
   default:
     OS << format("DW_FORM(0x%4.4x)", Form);
     break;
@@ -404,6 +469,7 @@ const uint8_t *DWARFFormValue::BlockData() const {
 
 bool DWARFFormValue::isBlockForm(uint16_t form) {
   switch (form) {
+  case DW_FORM_exprloc:
   case DW_FORM_block:
   case DW_FORM_block1:
   case DW_FORM_block2:
diff --git a/lib/DebugInfo/DWARFFormValue.h b/lib/DebugInfo/DWARFFormValue.h
index 22ac011..c5b590d 100644
--- a/lib/DebugInfo/DWARFFormValue.h
+++ b/lib/DebugInfo/DWARFFormValue.h
@@ -52,7 +52,7 @@ public:
   bool extractValue(DataExtractor data, uint32_t *offset_ptr,
                     const DWARFCompileUnit *cu);
   bool isInlinedCStr() const {
-    return Value.data != NULL && Value.data == (uint8_t*)Value.cstr;
+    return Value.data != NULL && Value.data == (const uint8_t*)Value.cstr;
   }
   const uint8_t *BlockData() const;
   uint64_t getReference(const DWARFCompileUnit* cu) const;
diff --git a/lib/ExecutionEngine/ExecutionEngine.cpp b/lib/ExecutionEngine/ExecutionEngine.cpp
index a744d0c..05987f2 100644
--- a/lib/ExecutionEngine/ExecutionEngine.cpp
+++ b/lib/ExecutionEngine/ExecutionEngine.cpp
@@ -29,7 +29,7 @@
 #include "llvm/Support/DynamicLibrary.h"
 #include "llvm/Support/Host.h"
 #include "llvm/Support/TargetRegistry.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetMachine.h"
 #include <cmath>
 #include <cstring>
@@ -91,11 +91,11 @@ class GVMemoryBlock : public CallbackVH {
 public:
   /// \brief Returns the address the GlobalVariable should be written into.  The
   /// GVMemoryBlock object prefixes that.
-  static char *Create(const GlobalVariable *GV, const TargetData& TD) {
+  static char *Create(const GlobalVariable *GV, const DataLayout& TD) {
     Type *ElTy = GV->getType()->getElementType();
     size_t GVSize = (size_t)TD.getTypeAllocSize(ElTy);
     void *RawMemory = ::operator new(
-      TargetData::RoundUpAlignment(sizeof(GVMemoryBlock),
+      DataLayout::RoundUpAlignment(sizeof(GVMemoryBlock),
                                    TD.getPreferredAlignment(GV))
       + GVSize);
     new(RawMemory) GVMemoryBlock(GV);
@@ -113,7 +113,7 @@ public:
 }  // anonymous namespace
 
 char *ExecutionEngine::getMemoryForGV(const GlobalVariable *GV) {
-  return GVMemoryBlock::Create(GV, *getTargetData());
+  return GVMemoryBlock::Create(GV, *getDataLayout());
 }
 
 bool ExecutionEngine::removeModule(Module *M) {
@@ -267,7 +267,7 @@ public:
 void *ArgvArray::reset(LLVMContext &C, ExecutionEngine *EE,
                        const std::vector<std::string> &InputArgv) {
   clear();  // Free the old contents.
-  unsigned PtrSize = EE->getTargetData()->getPointerSize();
+  unsigned PtrSize = EE->getDataLayout()->getPointerSize();
   Array = new char[(InputArgv.size()+1)*PtrSize];
 
   DEBUG(dbgs() << "JIT: ARGV = " << (void*)Array << "\n");
@@ -342,7 +342,7 @@ void ExecutionEngine::runStaticConstructorsDestructors(bool isDtors) {
 #ifndef NDEBUG
 /// isTargetNullPtr - Return whether the target pointer stored at Loc is null.
 static bool isTargetNullPtr(ExecutionEngine *EE, void *Loc) {
-  unsigned PtrSize = EE->getTargetData()->getPointerSize();
+  unsigned PtrSize = EE->getDataLayout()->getPointerSize();
   for (unsigned i = 0; i < PtrSize; ++i)
     if (*(i + (uint8_t*)Loc))
       return false;
@@ -501,7 +501,8 @@ ExecutionEngine *EngineBuilder::create(TargetMachine *TM) {
     return 0;
   }
 
-  if ((WhichEngine & EngineKind::JIT) && ExecutionEngine::JITCtor == 0) {
+  if ((WhichEngine & EngineKind::JIT) && ExecutionEngine::JITCtor == 0 &&
+      ExecutionEngine::MCJITCtor == 0) {
     if (ErrorStr)
       *ErrorStr = "JIT has not been linked in.";
   }
@@ -643,15 +644,17 @@ GenericValue ExecutionEngine::getConstantValue(const Constant *C) {
     }
     case Instruction::PtrToInt: {
       GenericValue GV = getConstantValue(Op0);
-      uint32_t PtrWidth = TD->getPointerSizeInBits();
+      uint32_t PtrWidth = TD->getTypeSizeInBits(Op0->getType());
+      assert(PtrWidth <= 64 && "Bad pointer width");
       GV.IntVal = APInt(PtrWidth, uintptr_t(GV.PointerVal));
+      uint32_t IntWidth = TD->getTypeSizeInBits(CE->getType());
+      GV.IntVal = GV.IntVal.zextOrTrunc(IntWidth);
       return GV;
     }
     case Instruction::IntToPtr: {
       GenericValue GV = getConstantValue(Op0);
-      uint32_t PtrWidth = TD->getPointerSizeInBits();
-      if (PtrWidth != GV.IntVal.getBitWidth())
-        GV.IntVal = GV.IntVal.zextOrTrunc(PtrWidth);
+      uint32_t PtrWidth = TD->getTypeSizeInBits(CE->getType());
+      GV.IntVal = GV.IntVal.zextOrTrunc(PtrWidth);
       assert(GV.IntVal.getBitWidth() <= 64 && "Bad pointer width");
       GV.PointerVal = PointerTy(uintptr_t(GV.IntVal.getZExtValue()));
       return GV;
@@ -832,7 +835,7 @@ GenericValue ExecutionEngine::getConstantValue(const Constant *C) {
 static void StoreIntToMemory(const APInt &IntVal, uint8_t *Dst,
                              unsigned StoreBytes) {
   assert((IntVal.getBitWidth()+7)/8 >= StoreBytes && "Integer too small!");
-  uint8_t *Src = (uint8_t *)IntVal.getRawData();
+  const uint8_t *Src = (const uint8_t *)IntVal.getRawData();
 
   if (sys::isLittleEndianHost()) {
     // Little-endian host - the source is ordered from LSB to MSB.  Order the
@@ -855,7 +858,7 @@ static void StoreIntToMemory(const APInt &IntVal, uint8_t *Dst,
 
 void ExecutionEngine::StoreValueToMemory(const GenericValue &Val,
                                          GenericValue *Ptr, Type *Ty) {
-  const unsigned StoreBytes = getTargetData()->getTypeStoreSize(Ty);
+  const unsigned StoreBytes = getDataLayout()->getTypeStoreSize(Ty);
 
   switch (Ty->getTypeID()) {
   case Type::IntegerTyID:
@@ -881,7 +884,7 @@ void ExecutionEngine::StoreValueToMemory(const GenericValue &Val,
     dbgs() << "Cannot store value of type " << *Ty << "!\n";
   }
 
-  if (sys::isLittleEndianHost() != getTargetData()->isLittleEndian())
+  if (sys::isLittleEndianHost() != getDataLayout()->isLittleEndian())
     // Host and target are different endian - reverse the stored bytes.
     std::reverse((uint8_t*)Ptr, StoreBytes + (uint8_t*)Ptr);
 }
@@ -917,7 +920,7 @@ static void LoadIntFromMemory(APInt &IntVal, uint8_t *Src, unsigned LoadBytes) {
 void ExecutionEngine::LoadValueFromMemory(GenericValue &Result,
                                           GenericValue *Ptr,
                                           Type *Ty) {
-  const unsigned LoadBytes = getTargetData()->getTypeStoreSize(Ty);
+  const unsigned LoadBytes = getDataLayout()->getTypeStoreSize(Ty);
 
   switch (Ty->getTypeID()) {
   case Type::IntegerTyID:
@@ -958,20 +961,20 @@ void ExecutionEngine::InitializeMemory(const Constant *Init, void *Addr) {
   
   if (const ConstantVector *CP = dyn_cast<ConstantVector>(Init)) {
     unsigned ElementSize =
-      getTargetData()->getTypeAllocSize(CP->getType()->getElementType());
+      getDataLayout()->getTypeAllocSize(CP->getType()->getElementType());
     for (unsigned i = 0, e = CP->getNumOperands(); i != e; ++i)
       InitializeMemory(CP->getOperand(i), (char*)Addr+i*ElementSize);
     return;
   }
   
   if (isa<ConstantAggregateZero>(Init)) {
-    memset(Addr, 0, (size_t)getTargetData()->getTypeAllocSize(Init->getType()));
+    memset(Addr, 0, (size_t)getDataLayout()->getTypeAllocSize(Init->getType()));
     return;
   }
   
   if (const ConstantArray *CPA = dyn_cast<ConstantArray>(Init)) {
     unsigned ElementSize =
-      getTargetData()->getTypeAllocSize(CPA->getType()->getElementType());
+      getDataLayout()->getTypeAllocSize(CPA->getType()->getElementType());
     for (unsigned i = 0, e = CPA->getNumOperands(); i != e; ++i)
       InitializeMemory(CPA->getOperand(i), (char*)Addr+i*ElementSize);
     return;
@@ -979,7 +982,7 @@ void ExecutionEngine::InitializeMemory(const Constant *Init, void *Addr) {
   
   if (const ConstantStruct *CPS = dyn_cast<ConstantStruct>(Init)) {
     const StructLayout *SL =
-      getTargetData()->getStructLayout(cast<StructType>(CPS->getType()));
+      getDataLayout()->getStructLayout(cast<StructType>(CPS->getType()));
     for (unsigned i = 0, e = CPS->getNumOperands(); i != e; ++i)
       InitializeMemory(CPS->getOperand(i), (char*)Addr+SL->getElementOffset(i));
     return;
@@ -1126,7 +1129,7 @@ void ExecutionEngine::EmitGlobalVariable(const GlobalVariable *GV) {
     InitializeMemory(GV->getInitializer(), GA);
 
   Type *ElTy = GV->getType()->getElementType();
-  size_t GVSize = (size_t)getTargetData()->getTypeAllocSize(ElTy);
+  size_t GVSize = (size_t)getDataLayout()->getTypeAllocSize(ElTy);
   NumInitBytes += (unsigned)GVSize;
   ++NumGlobals;
 }
diff --git a/lib/ExecutionEngine/ExecutionEngineBindings.cpp b/lib/ExecutionEngine/ExecutionEngineBindings.cpp
index 75e680a..1e790e7 100644
--- a/lib/ExecutionEngine/ExecutionEngineBindings.cpp
+++ b/lib/ExecutionEngine/ExecutionEngineBindings.cpp
@@ -239,7 +239,7 @@ void *LLVMRecompileAndRelinkFunction(LLVMExecutionEngineRef EE, LLVMValueRef Fn)
 }
 
 LLVMTargetDataRef LLVMGetExecutionEngineTargetData(LLVMExecutionEngineRef EE) {
-  return wrap(unwrap(EE)->getTargetData());
+  return wrap(unwrap(EE)->getDataLayout());
 }
 
 void LLVMAddGlobalMapping(LLVMExecutionEngineRef EE, LLVMValueRef Global,
diff --git a/lib/ExecutionEngine/IntelJITEvents/CMakeLists.txt b/lib/ExecutionEngine/IntelJITEvents/CMakeLists.txt
index 7d67d0d..3483088 100644
--- a/lib/ExecutionEngine/IntelJITEvents/CMakeLists.txt
+++ b/lib/ExecutionEngine/IntelJITEvents/CMakeLists.txt
@@ -1,11 +1,6 @@
-
-include_directories( ${LLVM_INTEL_JITEVENTS_INCDIR} ${CMAKE_CURRENT_SOURCE_DIR}/.. )
-
-set(system_libs
-  ${system_libs}
-  jitprofiling
-  )
+include_directories( ${CMAKE_CURRENT_SOURCE_DIR}/.. )
 
 add_llvm_library(LLVMIntelJITEvents
   IntelJITEventListener.cpp
+  jitprofiling.c
   )
diff --git a/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp b/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp
index c11c17e..4cb0270 100644
--- a/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp
+++ b/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp
@@ -22,12 +22,12 @@
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/OwningPtr.h"
 #include "llvm/CodeGen/MachineFunction.h"
-#include "llvm/ExecutionEngine/IntelJITEventsWrapper.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/Errno.h"
 #include "llvm/Support/ValueHandle.h"
 #include "EventListenerCommon.h"
+#include "IntelJITEventsWrapper.h"
 
 using namespace llvm;
 using namespace llvm::jitprofiling;
@@ -37,13 +37,13 @@ namespace {
 class IntelJITEventListener : public JITEventListener {
   typedef DenseMap<void*, unsigned int> MethodIDMap;
 
-  IntelJITEventsWrapper& Wrapper;
+  OwningPtr<IntelJITEventsWrapper> Wrapper;
   MethodIDMap MethodIDs;
   FilenameCache Filenames;
 
 public:
-  IntelJITEventListener(IntelJITEventsWrapper& libraryWrapper)
-  : Wrapper(libraryWrapper) {
+  IntelJITEventListener(IntelJITEventsWrapper* libraryWrapper) {
+      Wrapper.reset(libraryWrapper);
   }
 
   ~IntelJITEventListener() {
@@ -54,6 +54,10 @@ public:
                                      const EmittedFunctionDetails &Details);
 
   virtual void NotifyFreeingMachineCode(void *OldPtr);
+
+  virtual void NotifyObjectEmitted(const ObjectImage &Obj);
+
+  virtual void NotifyFreeingObject(const ObjectImage &Obj);
 };
 
 static LineNumberInfo LineStartToIntelJITFormat(
@@ -94,7 +98,7 @@ static iJIT_Method_Load FunctionDescToIntelJITFormat(
 void IntelJITEventListener::NotifyFunctionEmitted(
     const Function &F, void *FnStart, size_t FnSize,
     const EmittedFunctionDetails &Details) {
-  iJIT_Method_Load FunctionMessage = FunctionDescToIntelJITFormat(Wrapper,
+  iJIT_Method_Load FunctionMessage = FunctionDescToIntelJITFormat(*Wrapper,
                                       F.getName().data(),
                                       reinterpret_cast<uint64_t>(FnStart),
                                       FnSize);
@@ -151,32 +155,36 @@ void IntelJITEventListener::NotifyFunctionEmitted(
     FunctionMessage.line_number_table = 0;
   }
 
-  Wrapper.iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED,
-                           &FunctionMessage);
+  Wrapper->iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED,
+                            &FunctionMessage);
   MethodIDs[FnStart] = FunctionMessage.method_id;
 }
 
 void IntelJITEventListener::NotifyFreeingMachineCode(void *FnStart) {
   MethodIDMap::iterator I = MethodIDs.find(FnStart);
   if (I != MethodIDs.end()) {
-    Wrapper.iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_UNLOAD_START, &I->second);
+    Wrapper->iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_UNLOAD_START, &I->second);
     MethodIDs.erase(I);
   }
 }
 
+void IntelJITEventListener::NotifyObjectEmitted(const ObjectImage &Obj) {
+}
+
+void IntelJITEventListener::NotifyFreeingObject(const ObjectImage &Obj) {
+}
+
 }  // anonymous namespace.
 
 namespace llvm {
 JITEventListener *JITEventListener::createIntelJITEventListener() {
-  static OwningPtr<IntelJITEventsWrapper> JITProfilingWrapper(
-                                            new IntelJITEventsWrapper);
-  return new IntelJITEventListener(*JITProfilingWrapper);
+  return new IntelJITEventListener(new IntelJITEventsWrapper);
 }
 
 // for testing
 JITEventListener *JITEventListener::createIntelJITEventListener(
                                       IntelJITEventsWrapper* TestImpl) {
-  return new IntelJITEventListener(*TestImpl);
+  return new IntelJITEventListener(TestImpl);
 }
 
 } // namespace llvm
diff --git a/lib/ExecutionEngine/IntelJITEvents/IntelJITEventsWrapper.h b/lib/ExecutionEngine/IntelJITEvents/IntelJITEventsWrapper.h
new file mode 100644
index 0000000..7ab08e1
--- /dev/null
+++ b/lib/ExecutionEngine/IntelJITEvents/IntelJITEventsWrapper.h
@@ -0,0 +1,102 @@
+//===-- IntelJITEventsWrapper.h - Intel JIT Events API Wrapper --*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines a wrapper for the Intel JIT Events API. It allows for the
+// implementation of the jitprofiling library to be swapped with an alternative
+// implementation (for testing). To include this file, you must have the
+// jitprofiling.h header available; it is available in Intel(R) VTune(TM)
+// Amplifier XE 2011.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef INTEL_JIT_EVENTS_WRAPPER_H
+#define INTEL_JIT_EVENTS_WRAPPER_H
+
+#include "jitprofiling.h"
+
+namespace llvm {
+
+class IntelJITEventsWrapper {
+  // Function pointer types for testing implementation of Intel jitprofiling
+  // library
+  typedef int (*NotifyEventPtr)(iJIT_JVM_EVENT, void*);
+  typedef void (*RegisterCallbackExPtr)(void *, iJIT_ModeChangedEx );
+  typedef iJIT_IsProfilingActiveFlags (*IsProfilingActivePtr)(void);
+  typedef void (*FinalizeThreadPtr)(void);
+  typedef void (*FinalizeProcessPtr)(void);
+  typedef unsigned int (*GetNewMethodIDPtr)(void);
+
+  NotifyEventPtr NotifyEventFunc;
+  RegisterCallbackExPtr RegisterCallbackExFunc;
+  IsProfilingActivePtr IsProfilingActiveFunc;
+  FinalizeThreadPtr FinalizeThreadFunc;
+  FinalizeProcessPtr FinalizeProcessFunc;
+  GetNewMethodIDPtr GetNewMethodIDFunc;
+
+public:
+  bool isAmplifierRunning() {
+    return iJIT_IsProfilingActive() == iJIT_SAMPLING_ON;
+  }
+
+  IntelJITEventsWrapper()
+  : NotifyEventFunc(::iJIT_NotifyEvent),
+    RegisterCallbackExFunc(::iJIT_RegisterCallbackEx),
+    IsProfilingActiveFunc(::iJIT_IsProfilingActive),
+    FinalizeThreadFunc(::FinalizeThread),
+    FinalizeProcessFunc(::FinalizeProcess),
+    GetNewMethodIDFunc(::iJIT_GetNewMethodID) {
+  }
+
+  IntelJITEventsWrapper(NotifyEventPtr NotifyEventImpl,
+                   RegisterCallbackExPtr RegisterCallbackExImpl,
+                   IsProfilingActivePtr IsProfilingActiveImpl,
+                   FinalizeThreadPtr FinalizeThreadImpl,
+                   FinalizeProcessPtr FinalizeProcessImpl,
+                   GetNewMethodIDPtr GetNewMethodIDImpl)
+  : NotifyEventFunc(NotifyEventImpl),
+    RegisterCallbackExFunc(RegisterCallbackExImpl),
+    IsProfilingActiveFunc(IsProfilingActiveImpl),
+    FinalizeThreadFunc(FinalizeThreadImpl),
+    FinalizeProcessFunc(FinalizeProcessImpl),
+    GetNewMethodIDFunc(GetNewMethodIDImpl) {
+  }
+
+  // Sends an event anncouncing that a function has been emitted
+  //   return values are event-specific.  See Intel documentation for details.
+  int  iJIT_NotifyEvent(iJIT_JVM_EVENT EventType, void *EventSpecificData) {
+    if (!NotifyEventFunc)
+      return -1;
+    return NotifyEventFunc(EventType, EventSpecificData);
+  }
+
+  // Registers a callback function to receive notice of profiling state changes
+  void iJIT_RegisterCallbackEx(void *UserData,
+                               iJIT_ModeChangedEx NewModeCallBackFuncEx) {
+    if (RegisterCallbackExFunc)
+      RegisterCallbackExFunc(UserData, NewModeCallBackFuncEx);
+  }
+
+  // Returns the current profiler mode
+  iJIT_IsProfilingActiveFlags iJIT_IsProfilingActive(void) {
+    if (!IsProfilingActiveFunc)
+      return iJIT_NOTHING_RUNNING;
+    return IsProfilingActiveFunc();
+  }
+
+  // Generates a locally unique method ID for use in code registration
+  unsigned int iJIT_GetNewMethodID(void) {
+    if (!GetNewMethodIDFunc)
+      return -1;
+    return GetNewMethodIDFunc();
+  }
+};
+
+} //namespace llvm
+
+#endif //INTEL_JIT_EVENTS_WRAPPER_H
diff --git a/lib/ExecutionEngine/IntelJITEvents/Makefile b/lib/ExecutionEngine/IntelJITEvents/Makefile
index ba75ac6..dcf3126 100644
--- a/lib/ExecutionEngine/IntelJITEvents/Makefile
+++ b/lib/ExecutionEngine/IntelJITEvents/Makefile
@@ -11,7 +11,8 @@ LIBRARYNAME = LLVMIntelJITEvents
 
 include $(LEVEL)/Makefile.config
 
-SOURCES := IntelJITEventListener.cpp
-CPPFLAGS += -I$(INTEL_JITEVENTS_INCDIR) -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
+SOURCES := IntelJITEventListener.cpp \
+  jitprofiling.c
+CPPFLAGS += -I$(PROJ_OBJ_DIR)/.. -I$(PROJ_SRC_DIR)/..
 
 include $(LLVM_SRC_ROOT)/Makefile.rules
diff --git a/lib/ExecutionEngine/IntelJITEvents/ittnotify_config.h b/lib/ExecutionEngine/IntelJITEvents/ittnotify_config.h
new file mode 100644
index 0000000..1f029fb
--- /dev/null
+++ b/lib/ExecutionEngine/IntelJITEvents/ittnotify_config.h
@@ -0,0 +1,454 @@
+/*===-- ittnotify_config.h - JIT Profiling API internal config-----*- C -*-===*
+ *
+ *                     The LLVM Compiler Infrastructure
+ *
+ * This file is distributed under the University of Illinois Open Source
+ * License. See LICENSE.TXT for details.
+ *
+ *===----------------------------------------------------------------------===*
+ *
+ * This file provides Intel(R) Performance Analyzer JIT (Just-In-Time) 
+ * Profiling API internal config.
+ *
+ * NOTE: This file comes in a style different from the rest of LLVM
+ * source base since  this is a piece of code shared from Intel(R)
+ * products.  Please do not reformat / re-style this code to make
+ * subsequent merges and contributions from the original source base eaiser.
+ *
+ *===----------------------------------------------------------------------===*/
+#ifndef _ITTNOTIFY_CONFIG_H_
+#define _ITTNOTIFY_CONFIG_H_
+
+/** @cond exclude_from_documentation */
+#ifndef ITT_OS_WIN
+#  define ITT_OS_WIN   1
+#endif /* ITT_OS_WIN */
+
+#ifndef ITT_OS_LINUX
+#  define ITT_OS_LINUX 2
+#endif /* ITT_OS_LINUX */
+
+#ifndef ITT_OS_MAC
+#  define ITT_OS_MAC   3
+#endif /* ITT_OS_MAC */
+
+#ifndef ITT_OS
+#  if defined WIN32 || defined _WIN32
+#    define ITT_OS ITT_OS_WIN
+#  elif defined( __APPLE__ ) && defined( __MACH__ )
+#    define ITT_OS ITT_OS_MAC
+#  else
+#    define ITT_OS ITT_OS_LINUX
+#  endif
+#endif /* ITT_OS */
+
+#ifndef ITT_PLATFORM_WIN
+#  define ITT_PLATFORM_WIN 1
+#endif /* ITT_PLATFORM_WIN */
+
+#ifndef ITT_PLATFORM_POSIX
+#  define ITT_PLATFORM_POSIX 2
+#endif /* ITT_PLATFORM_POSIX */
+
+#ifndef ITT_PLATFORM
+#  if ITT_OS==ITT_OS_WIN
+#    define ITT_PLATFORM ITT_PLATFORM_WIN
+#  else
+#    define ITT_PLATFORM ITT_PLATFORM_POSIX
+#  endif /* _WIN32 */
+#endif /* ITT_PLATFORM */
+
+#if defined(_UNICODE) && !defined(UNICODE)
+#define UNICODE
+#endif
+
+#include <stddef.h>
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+#include <tchar.h>
+#else  /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+#include <stdint.h>
+#if defined(UNICODE) || defined(_UNICODE)
+#include <wchar.h>
+#endif /* UNICODE || _UNICODE */
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+
+#ifndef CDECL
+#  if ITT_PLATFORM==ITT_PLATFORM_WIN
+#    define CDECL __cdecl
+#  else /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+#    if defined _M_X64 || defined _M_AMD64 || defined __x86_64__
+#      define CDECL /* not actual on x86_64 platform */
+#    else  /* _M_X64 || _M_AMD64 || __x86_64__ */
+#      define CDECL __attribute__ ((cdecl))
+#    endif /* _M_X64 || _M_AMD64 || __x86_64__ */
+#  endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+#endif /* CDECL */
+
+#ifndef STDCALL
+#  if ITT_PLATFORM==ITT_PLATFORM_WIN
+#    define STDCALL __stdcall
+#  else /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+#    if defined _M_X64 || defined _M_AMD64 || defined __x86_64__
+#      define STDCALL /* not supported on x86_64 platform */
+#    else  /* _M_X64 || _M_AMD64 || __x86_64__ */
+#      define STDCALL __attribute__ ((stdcall))
+#    endif /* _M_X64 || _M_AMD64 || __x86_64__ */
+#  endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+#endif /* STDCALL */
+
+#define ITTAPI    CDECL
+#define LIBITTAPI CDECL
+
+/* TODO: Temporary for compatibility! */
+#define ITTAPI_CALL    CDECL
+#define LIBITTAPI_CALL CDECL
+
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+/* use __forceinline (VC++ specific) */
+#define ITT_INLINE           __forceinline
+#define ITT_INLINE_ATTRIBUTE /* nothing */
+#else  /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+/*
+ * Generally, functions are not inlined unless optimization is specified.
+ * For functions declared inline, this attribute inlines the function even
+ * if no optimization level was specified.
+ */
+#ifdef __STRICT_ANSI__
+#define ITT_INLINE           static
+#else  /* __STRICT_ANSI__ */
+#define ITT_INLINE           static inline
+#endif /* __STRICT_ANSI__ */
+#define ITT_INLINE_ATTRIBUTE __attribute__ ((always_inline))
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+/** @endcond */
+
+#ifndef ITT_ARCH_IA32
+#  define ITT_ARCH_IA32  1
+#endif /* ITT_ARCH_IA32 */
+
+#ifndef ITT_ARCH_IA32E
+#  define ITT_ARCH_IA32E 2
+#endif /* ITT_ARCH_IA32E */
+
+#ifndef ITT_ARCH_IA64
+#  define ITT_ARCH_IA64  3
+#endif /* ITT_ARCH_IA64 */
+
+#ifndef ITT_ARCH
+#  if defined _M_X64 || defined _M_AMD64 || defined __x86_64__
+#    define ITT_ARCH ITT_ARCH_IA32E
+#  elif defined _M_IA64 || defined __ia64
+#    define ITT_ARCH ITT_ARCH_IA64
+#  else
+#    define ITT_ARCH ITT_ARCH_IA32
+#  endif
+#endif
+
+#ifdef __cplusplus
+#  define ITT_EXTERN_C extern "C"
+#else
+#  define ITT_EXTERN_C /* nothing */
+#endif /* __cplusplus */
+
+#define ITT_TO_STR_AUX(x) #x
+#define ITT_TO_STR(x)     ITT_TO_STR_AUX(x)
+
+#define __ITT_BUILD_ASSERT(expr, suffix) do { \
+    static char __itt_build_check_##suffix[(expr) ? 1 : -1]; \
+    __itt_build_check_##suffix[0] = 0; \
+} while(0)
+#define _ITT_BUILD_ASSERT(expr, suffix)  __ITT_BUILD_ASSERT((expr), suffix)
+#define ITT_BUILD_ASSERT(expr)           _ITT_BUILD_ASSERT((expr), __LINE__)
+
+#define ITT_MAGIC { 0xED, 0xAB, 0xAB, 0xEC, 0x0D, 0xEE, 0xDA, 0x30 }
+
+/* Replace with snapshot date YYYYMMDD for promotion build. */
+#define API_VERSION_BUILD    20111111
+
+#ifndef API_VERSION_NUM
+#define API_VERSION_NUM 0.0.0
+#endif /* API_VERSION_NUM */
+
+#define API_VERSION "ITT-API-Version " ITT_TO_STR(API_VERSION_NUM) \
+                                " (" ITT_TO_STR(API_VERSION_BUILD) ")"
+
+/* OS communication functions */
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+#include <windows.h>
+typedef HMODULE           lib_t;
+typedef DWORD             TIDT;
+typedef CRITICAL_SECTION  mutex_t;
+#define MUTEX_INITIALIZER { 0 }
+#define strong_alias(name, aliasname) /* empty for Windows */
+#else  /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+#include <dlfcn.h>
+#if defined(UNICODE) || defined(_UNICODE)
+#include <wchar.h>
+#endif /* UNICODE */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE 1 /* need for PTHREAD_MUTEX_RECURSIVE */
+#endif /* _GNU_SOURCE */
+#include <pthread.h>
+typedef void*             lib_t;
+typedef pthread_t         TIDT;
+typedef pthread_mutex_t   mutex_t;
+#define MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER
+#define _strong_alias(name, aliasname) \
+            extern __typeof (name) aliasname __attribute__ ((alias (#name)));
+#define strong_alias(name, aliasname) _strong_alias(name, aliasname)
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+#define __itt_get_proc(lib, name) GetProcAddress(lib, name)
+#define __itt_mutex_init(mutex)   InitializeCriticalSection(mutex)
+#define __itt_mutex_lock(mutex)   EnterCriticalSection(mutex)
+#define __itt_mutex_unlock(mutex) LeaveCriticalSection(mutex)
+#define __itt_load_lib(name)      LoadLibraryA(name)
+#define __itt_unload_lib(handle)  FreeLibrary(handle)
+#define __itt_system_error()      (int)GetLastError()
+#define __itt_fstrcmp(s1, s2)     lstrcmpA(s1, s2)
+#define __itt_fstrlen(s)          lstrlenA(s)
+#define __itt_fstrcpyn(s1, s2, l) lstrcpynA(s1, s2, l)
+#define __itt_fstrdup(s)          _strdup(s)
+#define __itt_thread_id()         GetCurrentThreadId()
+#define __itt_thread_yield()      SwitchToThread()
+#ifndef ITT_SIMPLE_INIT
+ITT_INLINE long 
+__itt_interlocked_increment(volatile long* ptr) ITT_INLINE_ATTRIBUTE;
+ITT_INLINE long __itt_interlocked_increment(volatile long* ptr)
+{
+    return InterlockedIncrement(ptr);
+}
+#endif /* ITT_SIMPLE_INIT */
+#else /* ITT_PLATFORM!=ITT_PLATFORM_WIN */
+#define __itt_get_proc(lib, name) dlsym(lib, name)
+#define __itt_mutex_init(mutex)   {\
+    pthread_mutexattr_t mutex_attr;                                         \
+    int error_code = pthread_mutexattr_init(&mutex_attr);                   \
+    if (error_code)                                                         \
+        __itt_report_error(__itt_error_system, "pthread_mutexattr_init",    \
+                           error_code);                                     \
+    error_code = pthread_mutexattr_settype(&mutex_attr,                     \
+                                           PTHREAD_MUTEX_RECURSIVE);        \
+    if (error_code)                                                         \
+        __itt_report_error(__itt_error_system, "pthread_mutexattr_settype", \
+                           error_code);                                     \
+    error_code = pthread_mutex_init(mutex, &mutex_attr);                    \
+    if (error_code)                                                         \
+        __itt_report_error(__itt_error_system, "pthread_mutex_init",        \
+                           error_code);                                     \
+    error_code = pthread_mutexattr_destroy(&mutex_attr);                    \
+    if (error_code)                                                         \
+        __itt_report_error(__itt_error_system, "pthread_mutexattr_destroy", \
+                           error_code);                                     \
+}
+#define __itt_mutex_lock(mutex)   pthread_mutex_lock(mutex)
+#define __itt_mutex_unlock(mutex) pthread_mutex_unlock(mutex)
+#define __itt_load_lib(name)      dlopen(name, RTLD_LAZY)
+#define __itt_unload_lib(handle)  dlclose(handle)
+#define __itt_system_error()      errno
+#define __itt_fstrcmp(s1, s2)     strcmp(s1, s2)
+#define __itt_fstrlen(s)          strlen(s)
+#define __itt_fstrcpyn(s1, s2, l) strncpy(s1, s2, l)
+#define __itt_fstrdup(s)          strdup(s)
+#define __itt_thread_id()         pthread_self()
+#define __itt_thread_yield()      sched_yield()
+#if ITT_ARCH==ITT_ARCH_IA64
+#ifdef __INTEL_COMPILER
+#define __TBB_machine_fetchadd4(addr, val) __fetchadd4_acq((void *)addr, val)
+#else  /* __INTEL_COMPILER */
+/* TODO: Add Support for not Intel compilers for IA64 */
+#endif /* __INTEL_COMPILER */
+#else /* ITT_ARCH!=ITT_ARCH_IA64 */
+ITT_INLINE long
+__TBB_machine_fetchadd4(volatile void* ptr, long addend) ITT_INLINE_ATTRIBUTE;
+ITT_INLINE long __TBB_machine_fetchadd4(volatile void* ptr, long addend)
+{
+    long result;
+    __asm__ __volatile__("lock\nxadd %0,%1"
+                          : "=r"(result),"=m"(*(long*)ptr)
+                          : "0"(addend), "m"(*(long*)ptr)
+                          : "memory");
+    return result;
+}
+#endif /* ITT_ARCH==ITT_ARCH_IA64 */
+#ifndef ITT_SIMPLE_INIT
+ITT_INLINE long 
+__itt_interlocked_increment(volatile long* ptr) ITT_INLINE_ATTRIBUTE;
+ITT_INLINE long __itt_interlocked_increment(volatile long* ptr)
+{
+    return __TBB_machine_fetchadd4(ptr, 1) + 1L;
+}
+#endif /* ITT_SIMPLE_INIT */
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+
+typedef enum {
+    __itt_collection_normal = 0,
+    __itt_collection_paused = 1
+} __itt_collection_state;
+
+typedef enum {
+    __itt_thread_normal  = 0,
+    __itt_thread_ignored = 1
+} __itt_thread_state;
+
+#pragma pack(push, 8)
+
+typedef struct ___itt_thread_info
+{
+    const char* nameA; /*!< Copy of original name in ASCII. */
+#if defined(UNICODE) || defined(_UNICODE)
+    const wchar_t* nameW; /*!< Copy of original name in UNICODE. */
+#else  /* UNICODE || _UNICODE */
+    void* nameW;
+#endif /* UNICODE || _UNICODE */
+    TIDT               tid;
+    __itt_thread_state state;   /*!< Thread state (paused or normal) */
+    int                extra1;  /*!< Reserved to the runtime */
+    void*              extra2;  /*!< Reserved to the runtime */
+    struct ___itt_thread_info* next;
+} __itt_thread_info;
+
+#include "ittnotify_types.h" /* For __itt_group_id definition */
+
+typedef struct ___itt_api_info_20101001
+{
+    const char*    name;
+    void**         func_ptr;
+    void*          init_func;
+    __itt_group_id group;
+}  __itt_api_info_20101001;
+
+typedef struct ___itt_api_info
+{
+    const char*    name;
+    void**         func_ptr;
+    void*          init_func;
+    void*          null_func;
+    __itt_group_id group;
+}  __itt_api_info;
+
+struct ___itt_domain;
+struct ___itt_string_handle;
+
+typedef struct ___itt_global
+{
+    unsigned char          magic[8];
+    unsigned long          version_major;
+    unsigned long          version_minor;
+    unsigned long          version_build;
+    volatile long          api_initialized;
+    volatile long          mutex_initialized;
+    volatile long          atomic_counter;
+    mutex_t                mutex;
+    lib_t                  lib;
+    void*                  error_handler;
+    const char**           dll_path_ptr;
+    __itt_api_info*        api_list_ptr;
+    struct ___itt_global*  next;
+    /* Joinable structures below */
+    __itt_thread_info*     thread_list;
+    struct ___itt_domain*  domain_list;
+    struct ___itt_string_handle* string_list;
+    __itt_collection_state state;
+} __itt_global;
+
+#pragma pack(pop)
+
+#define NEW_THREAD_INFO_W(gptr,h,h_tail,t,s,n) { \
+    h = (__itt_thread_info*)malloc(sizeof(__itt_thread_info)); \
+    if (h != NULL) { \
+        h->tid    = t; \
+        h->nameA  = NULL; \
+        h->nameW  = n ? _wcsdup(n) : NULL; \
+        h->state  = s; \
+        h->extra1 = 0;    /* reserved */ \
+        h->extra2 = NULL; /* reserved */ \
+        h->next   = NULL; \
+        if (h_tail == NULL) \
+            (gptr)->thread_list = h; \
+        else \
+            h_tail->next = h; \
+    } \
+}
+
+#define NEW_THREAD_INFO_A(gptr,h,h_tail,t,s,n) { \
+    h = (__itt_thread_info*)malloc(sizeof(__itt_thread_info)); \
+    if (h != NULL) { \
+        h->tid    = t; \
+        h->nameA  = n ? __itt_fstrdup(n) : NULL; \
+        h->nameW  = NULL; \
+        h->state  = s; \
+        h->extra1 = 0;    /* reserved */ \
+        h->extra2 = NULL; /* reserved */ \
+        h->next   = NULL; \
+        if (h_tail == NULL) \
+            (gptr)->thread_list = h; \
+        else \
+            h_tail->next = h; \
+    } \
+}
+
+#define NEW_DOMAIN_W(gptr,h,h_tail,name) { \
+    h = (__itt_domain*)malloc(sizeof(__itt_domain)); \
+    if (h != NULL) { \
+        h->flags  = 0;    /* domain is disabled by default */ \
+        h->nameA  = NULL; \
+        h->nameW  = name ? _wcsdup(name) : NULL; \
+        h->extra1 = 0;    /* reserved */ \
+        h->extra2 = NULL; /* reserved */ \
+        h->next   = NULL; \
+        if (h_tail == NULL) \
+            (gptr)->domain_list = h; \
+        else \
+            h_tail->next = h; \
+    } \
+}
+
+#define NEW_DOMAIN_A(gptr,h,h_tail,name) { \
+    h = (__itt_domain*)malloc(sizeof(__itt_domain)); \
+    if (h != NULL) { \
+        h->flags  = 0;    /* domain is disabled by default */ \
+        h->nameA  = name ? __itt_fstrdup(name) : NULL; \
+        h->nameW  = NULL; \
+        h->extra1 = 0;    /* reserved */ \
+        h->extra2 = NULL; /* reserved */ \
+        h->next   = NULL; \
+        if (h_tail == NULL) \
+            (gptr)->domain_list = h; \
+        else \
+            h_tail->next = h; \
+    } \
+}
+
+#define NEW_STRING_HANDLE_W(gptr,h,h_tail,name) { \
+    h = (__itt_string_handle*)malloc(sizeof(__itt_string_handle)); \
+    if (h != NULL) { \
+        h->strA   = NULL; \
+        h->strW   = name ? _wcsdup(name) : NULL; \
+        h->extra1 = 0;    /* reserved */ \
+        h->extra2 = NULL; /* reserved */ \
+        h->next   = NULL; \
+        if (h_tail == NULL) \
+            (gptr)->string_list = h; \
+        else \
+            h_tail->next = h; \
+    } \
+}
+
+#define NEW_STRING_HANDLE_A(gptr,h,h_tail,name) { \
+    h = (__itt_string_handle*)malloc(sizeof(__itt_string_handle)); \
+    if (h != NULL) { \
+        h->strA   = name ? __itt_fstrdup(name) : NULL; \
+        h->strW   = NULL; \
+        h->extra1 = 0;    /* reserved */ \
+        h->extra2 = NULL; /* reserved */ \
+        h->next   = NULL; \
+        if (h_tail == NULL) \
+            (gptr)->string_list = h; \
+        else \
+            h_tail->next = h; \
+    } \
+}
+
+#endif /* _ITTNOTIFY_CONFIG_H_ */
diff --git a/lib/ExecutionEngine/IntelJITEvents/ittnotify_types.h b/lib/ExecutionEngine/IntelJITEvents/ittnotify_types.h
new file mode 100644
index 0000000..5df752f
--- /dev/null
+++ b/lib/ExecutionEngine/IntelJITEvents/ittnotify_types.h
@@ -0,0 +1,70 @@
+/*===-- ittnotify_types.h - JIT Profiling API internal types--------*- C -*-===*
+ *
+ *                     The LLVM Compiler Infrastructure
+ *
+ * This file is distributed under the University of Illinois Open Source
+ * License. See LICENSE.TXT for details.
+ *
+ *===----------------------------------------------------------------------===*
+ *
+ * NOTE: This file comes in a style different from the rest of LLVM
+ * source base since  this is a piece of code shared from Intel(R)
+ * products.  Please do not reformat / re-style this code to make
+ * subsequent merges and contributions from the original source base eaiser.
+ *
+ *===----------------------------------------------------------------------===*/
+#ifndef _ITTNOTIFY_TYPES_H_
+#define _ITTNOTIFY_TYPES_H_
+
+typedef enum ___itt_group_id
+{
+    __itt_group_none      = 0,
+    __itt_group_legacy    = 1<<0,
+    __itt_group_control   = 1<<1,
+    __itt_group_thread    = 1<<2,
+    __itt_group_mark      = 1<<3,
+    __itt_group_sync      = 1<<4,
+    __itt_group_fsync     = 1<<5,
+    __itt_group_jit       = 1<<6,
+    __itt_group_model     = 1<<7,
+    __itt_group_splitter_min = 1<<7,
+    __itt_group_counter   = 1<<8,
+    __itt_group_frame     = 1<<9,
+    __itt_group_stitch    = 1<<10,
+    __itt_group_heap      = 1<<11,
+    __itt_group_splitter_max = 1<<12,
+    __itt_group_structure = 1<<12,
+    __itt_group_suppress = 1<<13,
+    __itt_group_all       = -1
+} __itt_group_id;
+
+#pragma pack(push, 8)
+
+typedef struct ___itt_group_list
+{
+    __itt_group_id id;
+    const char*    name;
+} __itt_group_list;
+
+#pragma pack(pop)
+
+#define ITT_GROUP_LIST(varname) \
+    static __itt_group_list varname[] = {       \
+        { __itt_group_all,       "all"       }, \
+        { __itt_group_control,   "control"   }, \
+        { __itt_group_thread,    "thread"    }, \
+        { __itt_group_mark,      "mark"      }, \
+        { __itt_group_sync,      "sync"      }, \
+        { __itt_group_fsync,     "fsync"     }, \
+        { __itt_group_jit,       "jit"       }, \
+        { __itt_group_model,     "model"     }, \
+        { __itt_group_counter,   "counter"   }, \
+        { __itt_group_frame,     "frame"     }, \
+        { __itt_group_stitch,    "stitch"    }, \
+        { __itt_group_heap,      "heap"      }, \
+        { __itt_group_structure, "structure" }, \
+        { __itt_group_suppress,  "suppress"  }, \
+        { __itt_group_none,      NULL        }  \
+    }
+
+#endif /* _ITTNOTIFY_TYPES_H_ */
diff --git a/lib/ExecutionEngine/IntelJITEvents/jitprofiling.c b/lib/ExecutionEngine/IntelJITEvents/jitprofiling.c
new file mode 100644
index 0000000..7b507de
--- /dev/null
+++ b/lib/ExecutionEngine/IntelJITEvents/jitprofiling.c
@@ -0,0 +1,481 @@
+/*===-- jitprofiling.c - JIT (Just-In-Time) Profiling API----------*- C -*-===*
+ *
+ *                     The LLVM Compiler Infrastructure
+ *
+ * This file is distributed under the University of Illinois Open Source
+ * License. See LICENSE.TXT for details.
+ *
+ *===----------------------------------------------------------------------===*
+ *
+ * This file provides Intel(R) Performance Analyzer JIT (Just-In-Time) 
+ * Profiling API implementation. 
+ *
+ * NOTE: This file comes in a style different from the rest of LLVM
+ * source base since  this is a piece of code shared from Intel(R)
+ * products.  Please do not reformat / re-style this code to make
+ * subsequent merges and contributions from the original source base eaiser.
+ *
+ *===----------------------------------------------------------------------===*/
+#include "ittnotify_config.h"
+
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+#include <windows.h>
+#pragma optimize("", off)
+#else  /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+#include <pthread.h>
+#include <dlfcn.h>
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+#include <malloc.h>
+#include <stdlib.h>
+
+#include "jitprofiling.h"
+
+static const char rcsid[] = "\n@(#) $Revision: 243501 $\n";
+
+#define DLL_ENVIRONMENT_VAR             "VS_PROFILER"
+
+#ifndef NEW_DLL_ENVIRONMENT_VAR
+#if ITT_ARCH==ITT_ARCH_IA32
+#define NEW_DLL_ENVIRONMENT_VAR	        "INTEL_JIT_PROFILER32"
+#else
+#define NEW_DLL_ENVIRONMENT_VAR	        "INTEL_JIT_PROFILER64"
+#endif
+#endif /* NEW_DLL_ENVIRONMENT_VAR */
+
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+#define DEFAULT_DLLNAME                 "JitPI.dll"
+HINSTANCE m_libHandle = NULL;
+#else  /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+#define DEFAULT_DLLNAME                 "libJitPI.so"
+void* m_libHandle = NULL;
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+
+/* default location of JIT profiling agent on Android */
+#define ANDROID_JIT_AGENT_PATH  "/data/intel/libittnotify.so"
+
+/* the function pointers */
+typedef unsigned int(*TPInitialize)(void);
+static TPInitialize FUNC_Initialize=NULL;
+
+typedef unsigned int(*TPNotify)(unsigned int, void*);
+static TPNotify FUNC_NotifyEvent=NULL;
+
+static iJIT_IsProfilingActiveFlags executionMode = iJIT_NOTHING_RUNNING;
+
+/* end collector dll part. */
+
+/* loadiJIT_Funcs() : this function is called just in the beginning 
+ *  and is responsible to load the functions from BistroJavaCollector.dll
+ * result:
+ *  on success: the functions loads, iJIT_DLL_is_missing=0, return value = 1
+ *  on failure: the functions are NULL, iJIT_DLL_is_missing=1, return value = 0
+ */ 
+static int loadiJIT_Funcs(void);
+
+/* global representing whether the BistroJavaCollector can't be loaded */
+static int iJIT_DLL_is_missing = 0;
+
+/* Virtual stack - the struct is used as a virtual stack for each thread.
+ * Every thread initializes with a stack of size INIT_TOP_STACK.
+ * Every method entry decreases from the current stack point,
+ * and when a thread stack reaches its top of stack (return from the global 
+ * function), the top of stack and the current stack increase. Notice that 
+ * when returning from a function the stack pointer is the address of 
+ * the function return.
+*/
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+static DWORD threadLocalStorageHandle = 0;
+#else  /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+static pthread_key_t threadLocalStorageHandle = (pthread_key_t)0;
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+
+#define INIT_TOP_Stack 10000
+
+typedef struct 
+{
+    unsigned int TopStack;
+    unsigned int CurrentStack;
+} ThreadStack, *pThreadStack;
+
+/* end of virtual stack. */
+
+/*
+ * The function for reporting virtual-machine related events to VTune.
+ * Note: when reporting iJVM_EVENT_TYPE_ENTER_NIDS, there is no need to fill 
+ * in the stack_id field in the iJIT_Method_NIDS structure, as VTune fills it.
+ * The return value in iJVM_EVENT_TYPE_ENTER_NIDS && 
+ * iJVM_EVENT_TYPE_LEAVE_NIDS events will be 0 in case of failure.
+ * in iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED event 
+ * it will be -1 if EventSpecificData == 0 otherwise it will be 0.
+*/
+
+ITT_EXTERN_C int JITAPI 
+iJIT_NotifyEvent(iJIT_JVM_EVENT event_type, void *EventSpecificData)
+{
+    int ReturnValue;
+
+    /*
+     * This section is for debugging outside of VTune. 
+     * It creates the environment variables that indicates call graph mode.
+     * If running outside of VTune remove the remark.
+     *
+     *
+     * static int firstTime = 1;
+     * char DoCallGraph[12] = "DoCallGraph";
+     * if (firstTime)
+     * {
+     * firstTime = 0;
+     * SetEnvironmentVariable( "BISTRO_COLLECTORS_DO_CALLGRAPH", DoCallGraph);
+     * }
+     *
+     * end of section.
+    */
+
+    /* initialization part - the functions have not been loaded yet. This part
+     *        will load the functions, and check if we are in Call Graph mode. 
+     *        (for special treatment).
+     */
+    if (!FUNC_NotifyEvent) 
+    {
+        if (iJIT_DLL_is_missing) 
+            return 0;
+
+        /* load the Function from the DLL */
+        if (!loadiJIT_Funcs()) 
+            return 0;
+
+        /* Call Graph initialization. */
+    }
+
+    /* If the event is method entry/exit, check that in the current mode 
+     * VTune is allowed to receive it
+     */
+    if ((event_type == iJVM_EVENT_TYPE_ENTER_NIDS || 
+         event_type == iJVM_EVENT_TYPE_LEAVE_NIDS) &&
+        (executionMode != iJIT_CALLGRAPH_ON))
+    {
+        return 0;
+    }
+    /* This section is performed when method enter event occurs.
+     * It updates the virtual stack, or creates it if this is the first 
+     * method entry in the thread. The stack pointer is decreased.
+     */
+    if (event_type == iJVM_EVENT_TYPE_ENTER_NIDS)
+    {
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+        pThreadStack threadStack = 
+            (pThreadStack)TlsGetValue (threadLocalStorageHandle);
+#else  /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+        pThreadStack threadStack = 
+            (pThreadStack)pthread_getspecific(threadLocalStorageHandle);
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+
+        /* check for use of reserved method IDs */
+        if ( ((piJIT_Method_NIDS) EventSpecificData)->method_id <= 999 )
+            return 0;
+
+        if (!threadStack)
+        {
+            /* initialize the stack. */
+            threadStack = (pThreadStack) calloc (sizeof(ThreadStack), 1);
+            threadStack->TopStack = INIT_TOP_Stack;
+            threadStack->CurrentStack = INIT_TOP_Stack;
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+            TlsSetValue(threadLocalStorageHandle,(void*)threadStack);
+#else  /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+            pthread_setspecific(threadLocalStorageHandle,(void*)threadStack);
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+        }
+
+        /* decrease the stack. */
+        ((piJIT_Method_NIDS) EventSpecificData)->stack_id = 
+            (threadStack->CurrentStack)--;
+    }
+
+    /* This section is performed when method leave event occurs
+     * It updates the virtual stack.
+     *    Increases the stack pointer.
+     *    If the stack pointer reached the top (left the global function)
+     *        increase the pointer and the top pointer.
+     */
+    if (event_type == iJVM_EVENT_TYPE_LEAVE_NIDS)
+    {
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+        pThreadStack threadStack = 
+           (pThreadStack)TlsGetValue (threadLocalStorageHandle);
+#else  /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+        pThreadStack threadStack = 
+            (pThreadStack)pthread_getspecific(threadLocalStorageHandle);
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+
+        /* check for use of reserved method IDs */
+        if ( ((piJIT_Method_NIDS) EventSpecificData)->method_id <= 999 )
+            return 0;
+
+        if (!threadStack)
+        {
+            /* Error: first report in this thread is method exit */
+            exit (1);
+        }
+
+        ((piJIT_Method_NIDS) EventSpecificData)->stack_id = 
+            ++(threadStack->CurrentStack) + 1;
+
+        if (((piJIT_Method_NIDS) EventSpecificData)->stack_id 
+               > threadStack->TopStack)
+            ((piJIT_Method_NIDS) EventSpecificData)->stack_id = 
+                (unsigned int)-1;
+    }
+
+    if (event_type == iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED)
+    {
+        /* check for use of reserved method IDs */
+        if ( ((piJIT_Method_Load) EventSpecificData)->method_id <= 999 )
+            return 0;
+    }
+
+    ReturnValue = (int)FUNC_NotifyEvent(event_type, EventSpecificData);   
+
+    return ReturnValue;
+}
+
+/* The new mode call back routine */
+ITT_EXTERN_C void JITAPI 
+iJIT_RegisterCallbackEx(void *userdata, iJIT_ModeChangedEx 
+                        NewModeCallBackFuncEx) 
+{
+    /* is it already missing... or the load of functions from the DLL failed */
+    if (iJIT_DLL_is_missing || !loadiJIT_Funcs())
+    {
+        /* then do not bother with notifications */
+        NewModeCallBackFuncEx(userdata, iJIT_NO_NOTIFICATIONS);  
+        /* Error: could not load JIT functions. */
+        return;
+    }
+    /* nothing to do with the callback */
+}
+
+/*
+ * This function allows the user to query in which mode, if at all, 
+ *VTune is running
+ */
+ITT_EXTERN_C iJIT_IsProfilingActiveFlags JITAPI iJIT_IsProfilingActive()
+{
+    if (!iJIT_DLL_is_missing)
+    {
+        loadiJIT_Funcs();
+    }
+
+    return executionMode;
+}
+
+/* this function loads the collector dll (BistroJavaCollector) 
+ * and the relevant functions.
+ * on success: all functions load,     iJIT_DLL_is_missing = 0, return value = 1
+ * on failure: all functions are NULL, iJIT_DLL_is_missing = 1, return value = 0
+ */ 
+static int loadiJIT_Funcs()
+{
+    static int bDllWasLoaded = 0;
+    char *dllName = (char*)rcsid; /* !! Just to avoid unused code elimination */
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+    DWORD dNameLength = 0;
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+
+    if(bDllWasLoaded)
+    {
+        /* dll was already loaded, no need to do it for the second time */
+        return 1;
+    }
+
+    /* Assumes that the DLL will not be found */
+    iJIT_DLL_is_missing = 1;
+    FUNC_NotifyEvent = NULL;
+
+    if (m_libHandle) 
+    {
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+        FreeLibrary(m_libHandle);
+#else  /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+        dlclose(m_libHandle);
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+        m_libHandle = NULL;
+    }
+
+    /* Try to get the dll name from the environment */
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+    dNameLength = GetEnvironmentVariableA(NEW_DLL_ENVIRONMENT_VAR, NULL, 0);
+    if (dNameLength)
+    {
+        DWORD envret = 0;
+        dllName = (char*)malloc(sizeof(char) * (dNameLength + 1));
+        envret = GetEnvironmentVariableA(NEW_DLL_ENVIRONMENT_VAR, 
+                                         dllName, dNameLength);
+        if (envret)
+        {
+            /* Try to load the dll from the PATH... */
+            m_libHandle = LoadLibraryExA(dllName, 
+                                         NULL, LOAD_WITH_ALTERED_SEARCH_PATH);
+        }
+        free(dllName);
+    } else {
+        /* Try to use old VS_PROFILER variable */
+        dNameLength = GetEnvironmentVariableA(DLL_ENVIRONMENT_VAR, NULL, 0);
+        if (dNameLength)
+        {
+            DWORD envret = 0;
+            dllName = (char*)malloc(sizeof(char) * (dNameLength + 1));
+            envret = GetEnvironmentVariableA(DLL_ENVIRONMENT_VAR, 
+                                             dllName, dNameLength);
+            if (envret)
+            {
+                /* Try to load the dll from the PATH... */
+                m_libHandle = LoadLibraryA(dllName);
+            }
+            free(dllName);
+        }
+    }
+#else  /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+    dllName = getenv(NEW_DLL_ENVIRONMENT_VAR);
+    if (!dllName)
+        dllName = getenv(DLL_ENVIRONMENT_VAR);
+#ifdef ANDROID
+    if (!dllName)
+        dllName = ANDROID_JIT_AGENT_PATH;
+#endif
+    if (dllName)
+    {
+        /* Try to load the dll from the PATH... */
+        m_libHandle = dlopen(dllName, RTLD_LAZY);
+    }
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+
+    if (!m_libHandle)
+    {
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+        m_libHandle = LoadLibraryA(DEFAULT_DLLNAME);
+#else  /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+        m_libHandle = dlopen(DEFAULT_DLLNAME, RTLD_LAZY);
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+    }
+
+    /* if the dll wasn't loaded - exit. */
+    if (!m_libHandle)
+    {
+        iJIT_DLL_is_missing = 1; /* don't try to initialize 
+                                  * JIT agent the second time 
+                                  */
+        return 0;
+    }
+
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+    FUNC_NotifyEvent = (TPNotify)GetProcAddress(m_libHandle, "NotifyEvent");
+#else  /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+    FUNC_NotifyEvent = (TPNotify)dlsym(m_libHandle, "NotifyEvent");
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+    if (!FUNC_NotifyEvent) 
+    {
+        FUNC_Initialize = NULL;
+        return 0;
+    }
+
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+    FUNC_Initialize = (TPInitialize)GetProcAddress(m_libHandle, "Initialize");
+#else  /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+    FUNC_Initialize = (TPInitialize)dlsym(m_libHandle, "Initialize");
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+    if (!FUNC_Initialize) 
+    {
+        FUNC_NotifyEvent = NULL;
+        return 0;
+    }
+
+    executionMode = (iJIT_IsProfilingActiveFlags)FUNC_Initialize();
+
+    bDllWasLoaded = 1;
+    iJIT_DLL_is_missing = 0; /* DLL is ok. */
+
+    /*
+     * Call Graph mode: init the thread local storage
+     * (need to store the virtual stack there).
+     */
+    if ( executionMode == iJIT_CALLGRAPH_ON )
+    {
+        /* Allocate a thread local storage slot for the thread "stack" */
+        if (!threadLocalStorageHandle)
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+            threadLocalStorageHandle = TlsAlloc();
+#else  /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+        pthread_key_create(&threadLocalStorageHandle, NULL);
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+    }
+
+    return 1;
+}
+
+/*
+ * This function should be called by the user whenever a thread ends, 
+ * to free the thread "virtual stack" storage
+ */
+ITT_EXTERN_C void JITAPI FinalizeThread()
+{
+    if (threadLocalStorageHandle)
+    {
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+        pThreadStack threadStack = 
+            (pThreadStack)TlsGetValue (threadLocalStorageHandle);
+#else  /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+        pThreadStack threadStack = 
+            (pThreadStack)pthread_getspecific(threadLocalStorageHandle);
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+        if (threadStack)
+        {
+            free (threadStack);
+            threadStack = NULL;
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+            TlsSetValue (threadLocalStorageHandle, threadStack);
+#else  /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+            pthread_setspecific(threadLocalStorageHandle, threadStack);
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+        }
+    }
+}
+
+/*
+ * This function should be called by the user when the process ends, 
+ * to free the local storage index
+*/
+ITT_EXTERN_C void JITAPI FinalizeProcess()
+{
+    if (m_libHandle) 
+    {
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+        FreeLibrary(m_libHandle);
+#else  /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+        dlclose(m_libHandle);
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+        m_libHandle = NULL;
+    }
+
+    if (threadLocalStorageHandle)
+#if ITT_PLATFORM==ITT_PLATFORM_WIN
+        TlsFree (threadLocalStorageHandle);
+#else  /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+    pthread_key_delete(threadLocalStorageHandle);
+#endif /* ITT_PLATFORM==ITT_PLATFORM_WIN */
+}
+
+/*
+ * This function should be called by the user for any method once.
+ * The function will return a unique method ID, the user should maintain 
+ * the ID for each method
+ */
+ITT_EXTERN_C unsigned int JITAPI iJIT_GetNewMethodID()
+{
+    static unsigned int methodID = 0x100000;
+
+    if (methodID == 0)
+        return 0;  /* ERROR : this is not a valid value */
+
+    return methodID++;
+}
diff --git a/lib/ExecutionEngine/IntelJITEvents/jitprofiling.h b/lib/ExecutionEngine/IntelJITEvents/jitprofiling.h
new file mode 100644
index 0000000..f08e287
--- /dev/null
+++ b/lib/ExecutionEngine/IntelJITEvents/jitprofiling.h
@@ -0,0 +1,259 @@
+/*===-- jitprofiling.h - JIT Profiling API-------------------------*- C -*-===*
+ *
+ *                     The LLVM Compiler Infrastructure
+ *
+ * This file is distributed under the University of Illinois Open Source
+ * License. See LICENSE.TXT for details.
+ *
+ *===----------------------------------------------------------------------===*
+ *
+ * This file provides Intel(R) Performance Analyzer JIT (Just-In-Time) 
+ * Profiling API declaration.
+ *
+ * NOTE: This file comes in a style different from the rest of LLVM
+ * source base since  this is a piece of code shared from Intel(R)
+ * products.  Please do not reformat / re-style this code to make
+ * subsequent merges and contributions from the original source base eaiser.
+ *
+ *===----------------------------------------------------------------------===*/
+#ifndef __JITPROFILING_H__
+#define __JITPROFILING_H__
+
+/*
+ * Various constants used by functions
+ */
+
+/* event notification */
+typedef enum iJIT_jvm_event
+{
+
+    /* shutdown  */
+    
+    /* 
+     * Program exiting EventSpecificData NA
+     */
+    iJVM_EVENT_TYPE_SHUTDOWN = 2, 
+
+    /* JIT profiling  */
+    
+    /* 
+     * issued after method code jitted into memory but before code is executed
+     * EventSpecificData is an iJIT_Method_Load
+     */
+    iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED=13,     
+
+    /* issued before unload. Method code will no longer be executed, but code 
+     * and info are still in memory. The VTune profiler may capture method 
+     * code only at this point EventSpecificData is iJIT_Method_Id
+     */
+    iJVM_EVENT_TYPE_METHOD_UNLOAD_START,         
+
+    /* Method Profiling */
+
+    /* method name, Id and stack is supplied 
+     * issued when a method is about to be entered EventSpecificData is 
+     * iJIT_Method_NIDS
+     */
+    iJVM_EVENT_TYPE_ENTER_NIDS = 19, 
+
+    /* method name, Id and stack is supplied 
+     * issued when a method is about to be left EventSpecificData is 
+     * iJIT_Method_NIDS
+     */
+    iJVM_EVENT_TYPE_LEAVE_NIDS               
+} iJIT_JVM_EVENT;
+
+typedef enum _iJIT_ModeFlags
+{
+    /* No need to Notify VTune, since VTune is not running */
+    iJIT_NO_NOTIFICATIONS          = 0x0000,     
+
+    /* when turned on the jit must call 
+     * iJIT_NotifyEvent
+     * (
+     *     iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED,
+     * )
+     * for all the method already jitted
+     */
+    iJIT_BE_NOTIFY_ON_LOAD         = 0x0001,     
+
+    /* when turned on the jit must call
+     * iJIT_NotifyEvent
+     * (
+     *     iJVM_EVENT_TYPE_METHOD_UNLOAD_FINISHED,
+     *  ) for all the method that are unloaded
+     */
+    iJIT_BE_NOTIFY_ON_UNLOAD       = 0x0002,     
+
+    /* when turned on the jit must instrument all
+     * the currently jited code with calls on
+     * method entries
+     */
+    iJIT_BE_NOTIFY_ON_METHOD_ENTRY = 0x0004,     
+
+    /* when turned on the jit must instrument all
+     * the currently jited code with calls
+     * on method exit
+     */
+    iJIT_BE_NOTIFY_ON_METHOD_EXIT  = 0x0008      
+
+} iJIT_ModeFlags;
+
+
+ /* Flags used by iJIT_IsProfilingActive() */
+typedef enum _iJIT_IsProfilingActiveFlags
+{
+    /* No profiler is running. Currently not used */
+    iJIT_NOTHING_RUNNING           = 0x0000,     
+
+    /* Sampling is running. This is the default value
+     * returned by iJIT_IsProfilingActive()
+     */
+    iJIT_SAMPLING_ON               = 0x0001,     
+    
+      /* Call Graph is running */
+    iJIT_CALLGRAPH_ON              = 0x0002
+
+} iJIT_IsProfilingActiveFlags;
+
+/* Enumerator for the environment of methods*/
+typedef enum _iJDEnvironmentType
+{
+    iJDE_JittingAPI = 2
+} iJDEnvironmentType;
+
+/**********************************
+ * Data structures for the events *
+ **********************************/
+
+/* structure for the events:
+ * iJVM_EVENT_TYPE_METHOD_UNLOAD_START
+ */
+
+typedef struct _iJIT_Method_Id
+{
+   /* Id of the method (same as the one passed in
+   * the iJIT_Method_Load struct
+   */
+    unsigned int       method_id;              
+
+} *piJIT_Method_Id, iJIT_Method_Id;
+
+
+/* structure for the events:
+ * iJVM_EVENT_TYPE_ENTER_NIDS,
+ * iJVM_EVENT_TYPE_LEAVE_NIDS,
+ * iJVM_EVENT_TYPE_EXCEPTION_OCCURRED_NIDS
+ */
+
+typedef struct _iJIT_Method_NIDS
+{
+    /* unique method ID */
+    unsigned int       method_id;              
+
+    /* NOTE: no need to fill this field, it's filled by VTune */
+    unsigned int       stack_id;               
+
+    /* method name (just the method, without the class) */
+    char*              method_name;            
+} *piJIT_Method_NIDS, iJIT_Method_NIDS;
+
+/* structures for the events:
+ * iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED
+ */
+
+typedef struct _LineNumberInfo
+{
+    /* x86 Offset from the begining of the method*/
+    unsigned int        Offset;                 
+    
+    /* source line number from the begining of the source file */
+    unsigned int        LineNumber;             
+
+} *pLineNumberInfo, LineNumberInfo;
+
+typedef struct _iJIT_Method_Load
+{
+    /* unique method ID - can be any unique value, (except 0 - 999) */
+    unsigned int        method_id;              
+
+    /* method name (can be with or without the class and signature, in any case
+     * the class name will be added to it)
+     */
+    char*               method_name;            
+
+    /* virtual address of that method - This determines the method range for the
+     * iJVM_EVENT_TYPE_ENTER/LEAVE_METHOD_ADDR events
+     */
+    void*               method_load_address;    
+
+    /* Size in memory - Must be exact */
+    unsigned int        method_size;            
+
+    /* Line Table size in number of entries - Zero if none */
+    unsigned int        line_number_size;       
+    
+    /* Pointer to the begining of the line numbers info array */
+    pLineNumberInfo     line_number_table;      
+
+    /* unique class ID */
+    unsigned int        class_id;               
+    
+    /* class file name */
+    char*               class_file_name;        
+
+    /* source file name */
+    char*               source_file_name;       
+
+    /* bits supplied by the user for saving in the JIT file */
+    void*               user_data;              
+
+    /* the size of the user data buffer */
+    unsigned int        user_data_size;         
+
+    /* NOTE: no need to fill this field, it's filled by VTune */
+    iJDEnvironmentType  env;                    
+
+} *piJIT_Method_Load, iJIT_Method_Load;
+
+/* API Functions */
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef CDECL
+#  if defined WIN32 || defined _WIN32
+#    define CDECL __cdecl
+#  else /* defined WIN32 || defined _WIN32 */
+#    if defined _M_X64 || defined _M_AMD64 || defined __x86_64__
+#      define CDECL /* not actual on x86_64 platform */
+#    else  /* _M_X64 || _M_AMD64 || __x86_64__ */
+#      define CDECL __attribute__ ((cdecl))
+#    endif /* _M_X64 || _M_AMD64 || __x86_64__ */
+#  endif /* defined WIN32 || defined _WIN32 */
+#endif /* CDECL */
+
+#define JITAPI CDECL
+
+/* called when the settings are changed with new settings */
+typedef void (*iJIT_ModeChangedEx)(void *UserData, iJIT_ModeFlags Flags);
+
+int JITAPI iJIT_NotifyEvent(iJIT_JVM_EVENT event_type, void *EventSpecificData);
+
+/* The new mode call back routine */
+void JITAPI iJIT_RegisterCallbackEx(void *userdata, 
+                                    iJIT_ModeChangedEx NewModeCallBackFuncEx);
+
+iJIT_IsProfilingActiveFlags JITAPI iJIT_IsProfilingActive(void);
+
+void JITAPI FinalizeThread(void);
+
+void JITAPI FinalizeProcess(void);
+
+unsigned int JITAPI iJIT_GetNewMethodID(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* __JITPROFILING_H__ */
diff --git a/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp b/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp
index 7a206eb..e16e2d1 100644
--- a/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp
+++ b/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp
@@ -25,7 +25,7 @@
 #include "llvm/Config/config.h"     // Detect libffi
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/DynamicLibrary.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Support/ManagedStatic.h"
 #include "llvm/Support/Mutex.h"
 #include <csignal>
@@ -180,7 +180,7 @@ static void *ffiValueFor(Type *Ty, const GenericValue &AV,
 
 static bool ffiInvoke(RawFunc Fn, Function *F,
                       const std::vector<GenericValue> &ArgVals,
-                      const TargetData *TD, GenericValue &Result) {
+                      const DataLayout *TD, GenericValue &Result) {
   ffi_cif cif;
   FunctionType *FTy = F->getFunctionType();
   const unsigned NumArgs = F->arg_size();
@@ -276,7 +276,7 @@ GenericValue Interpreter::callExternalFunction(Function *F,
   FunctionsLock->release();
 
   GenericValue Result;
-  if (RawFn != 0 && ffiInvoke(RawFn, F, ArgVals, getTargetData(), Result))
+  if (RawFn != 0 && ffiInvoke(RawFn, F, ArgVals, getDataLayout(), Result))
     return Result;
 #endif // USE_LIBFFI
 
@@ -376,7 +376,7 @@ GenericValue lle_X_sprintf(FunctionType *FT,
       case 'x': case 'X':
         if (HowLong >= 1) {
           if (HowLong == 1 &&
-              TheInterpreter->getTargetData()->getPointerSizeInBits() == 64 &&
+              TheInterpreter->getDataLayout()->getPointerSizeInBits() == 64 &&
               sizeof(long) < sizeof(int64_t)) {
             // Make sure we use %lld with a 64 bit argument because we might be
             // compiling LLI on a 32 bit compiler.
diff --git a/lib/ExecutionEngine/Interpreter/Interpreter.cpp b/lib/ExecutionEngine/Interpreter/Interpreter.cpp
index 43e3453..55152db 100644
--- a/lib/ExecutionEngine/Interpreter/Interpreter.cpp
+++ b/lib/ExecutionEngine/Interpreter/Interpreter.cpp
@@ -48,7 +48,7 @@ Interpreter::Interpreter(Module *M)
   : ExecutionEngine(M), TD(M) {
       
   memset(&ExitValue.Untyped, 0, sizeof(ExitValue.Untyped));
-  setTargetData(&TD);
+  setDataLayout(&TD);
   // Initialize the "backend"
   initializeExecutionEngine();
   initializeExternalFunctions();
diff --git a/lib/ExecutionEngine/Interpreter/Interpreter.h b/lib/ExecutionEngine/Interpreter/Interpreter.h
index 28c5775..72c42c1 100644
--- a/lib/ExecutionEngine/Interpreter/Interpreter.h
+++ b/lib/ExecutionEngine/Interpreter/Interpreter.h
@@ -17,7 +17,7 @@
 #include "llvm/Function.h"
 #include "llvm/ExecutionEngine/ExecutionEngine.h"
 #include "llvm/ExecutionEngine/GenericValue.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Support/CallSite.h"
 #include "llvm/Support/DataTypes.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -82,7 +82,7 @@ struct ExecutionContext {
 //
 class Interpreter : public ExecutionEngine, public InstVisitor<Interpreter> {
   GenericValue ExitValue;          // The return value of the called function
-  TargetData TD;
+  DataLayout TD;
   IntrinsicLowering *IL;
 
   // The runtime stack of executing code.  The top of the stack is the current
diff --git a/lib/ExecutionEngine/JIT/JIT.cpp b/lib/ExecutionEngine/JIT/JIT.cpp
index 97995ad..1ad3382 100644
--- a/lib/ExecutionEngine/JIT/JIT.cpp
+++ b/lib/ExecutionEngine/JIT/JIT.cpp
@@ -24,7 +24,7 @@
 #include "llvm/ExecutionEngine/GenericValue.h"
 #include "llvm/ExecutionEngine/JITEventListener.h"
 #include "llvm/ExecutionEngine/JITMemoryManager.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetJITInfo.h"
 #include "llvm/Support/Dwarf.h"
@@ -272,7 +272,7 @@ JIT::JIT(Module *M, TargetMachine &tm, TargetJITInfo &tji,
   : ExecutionEngine(M), TM(tm), TJI(tji),
     JMM(jmm ? jmm : JITMemoryManager::CreateDefaultMemManager()),
     AllocateGVsWithCode(GVsWithCode), isAlreadyCodeGenerating(false) {
-  setTargetData(TM.getTargetData());
+  setDataLayout(TM.getDataLayout());
 
   jitstate = new JITState(M);
 
@@ -285,7 +285,7 @@ JIT::JIT(Module *M, TargetMachine &tm, TargetJITInfo &tji,
   // Add target data
   MutexGuard locked(lock);
   FunctionPassManager &PM = jitstate->getPM(locked);
-  PM.add(new TargetData(*TM.getTargetData()));
+  PM.add(new DataLayout(*TM.getDataLayout()));
 
   // Turn the machine code intermediate representation into bytes in memory that
   // may be executed.
@@ -339,7 +339,7 @@ void JIT::addModule(Module *M) {
     jitstate = new JITState(M);
 
     FunctionPassManager &PM = jitstate->getPM(locked);
-    PM.add(new TargetData(*TM.getTargetData()));
+    PM.add(new DataLayout(*TM.getDataLayout()));
 
     // Turn the machine code intermediate representation into bytes in memory
     // that may be executed.
@@ -370,7 +370,7 @@ bool JIT::removeModule(Module *M) {
     jitstate = new JITState(Modules[0]);
 
     FunctionPassManager &PM = jitstate->getPM(locked);
-    PM.add(new TargetData(*TM.getTargetData()));
+    PM.add(new DataLayout(*TM.getDataLayout()));
 
     // Turn the machine code intermediate representation into bytes in memory
     // that may be executed.
@@ -815,8 +815,8 @@ char* JIT::getMemoryForGV(const GlobalVariable* GV) {
   // through the memory manager which puts them near the code but not in the
   // same buffer.
   Type *GlobalType = GV->getType()->getElementType();
-  size_t S = getTargetData()->getTypeAllocSize(GlobalType);
-  size_t A = getTargetData()->getPreferredAlignment(GV);
+  size_t S = getDataLayout()->getTypeAllocSize(GlobalType);
+  size_t A = getDataLayout()->getPreferredAlignment(GV);
   if (GV->isThreadLocal()) {
     MutexGuard locked(lock);
     Ptr = TJI.allocateThreadLocalMemory(S);
diff --git a/lib/ExecutionEngine/JIT/JITDwarfEmitter.cpp b/lib/ExecutionEngine/JIT/JITDwarfEmitter.cpp
index 42a136e..19c1979 100644
--- a/lib/ExecutionEngine/JIT/JITDwarfEmitter.cpp
+++ b/lib/ExecutionEngine/JIT/JITDwarfEmitter.cpp
@@ -24,7 +24,7 @@
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/Support/ErrorHandling.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetInstrInfo.h"
 #include "llvm/Target/TargetFrameLowering.h"
 #include "llvm/Target/TargetMachine.h"
@@ -42,7 +42,7 @@ unsigned char* JITDwarfEmitter::EmitDwarfTable(MachineFunction& F,
   assert(MMI && "MachineModuleInfo not registered!");
 
   const TargetMachine& TM = F.getTarget();
-  TD = TM.getTargetData();
+  TD = TM.getDataLayout();
   stackGrowthDirection = TM.getFrameLowering()->getStackGrowthDirection();
   RI = TM.getRegisterInfo();
   MAI = TM.getMCAsmInfo();
diff --git a/lib/ExecutionEngine/JIT/JITDwarfEmitter.h b/lib/ExecutionEngine/JIT/JITDwarfEmitter.h
index 8dc99ab..9cdbeac 100644
--- a/lib/ExecutionEngine/JIT/JITDwarfEmitter.h
+++ b/lib/ExecutionEngine/JIT/JITDwarfEmitter.h
@@ -23,12 +23,12 @@ class MachineFunction;
 class MachineModuleInfo;
 class MachineMove;
 class MCAsmInfo;
-class TargetData;
+class DataLayout;
 class TargetMachine;
 class TargetRegisterInfo;
 
 class JITDwarfEmitter {
-  const TargetData* TD;
+  const DataLayout* TD;
   JITCodeEmitter* JCE;
   const TargetRegisterInfo* RI;
   const MCAsmInfo *MAI;
diff --git a/lib/ExecutionEngine/JIT/JITEmitter.cpp b/lib/ExecutionEngine/JIT/JITEmitter.cpp
index ff3a9dc..ecafda7 100644
--- a/lib/ExecutionEngine/JIT/JITEmitter.cpp
+++ b/lib/ExecutionEngine/JIT/JITEmitter.cpp
@@ -30,7 +30,7 @@
 #include "llvm/ExecutionEngine/GenericValue.h"
 #include "llvm/ExecutionEngine/JITEventListener.h"
 #include "llvm/ExecutionEngine/JITMemoryManager.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetInstrInfo.h"
 #include "llvm/Target/TargetJITInfo.h"
 #include "llvm/Target/TargetMachine.h"
@@ -384,11 +384,6 @@ namespace {
       delete MemMgr;
     }
 
-    /// classof - Methods for support type inquiry through isa, cast, and
-    /// dyn_cast:
-    ///
-    static inline bool classof(const MachineCodeEmitter*) { return true; }
-
     JITResolver &getJITResolver() { return Resolver; }
 
     virtual void startFunction(MachineFunction &F);
@@ -763,7 +758,7 @@ void JITEmitter::processDebugLoc(DebugLoc DL, bool BeforePrintingInsn) {
 }
 
 static unsigned GetConstantPoolSizeInBytes(MachineConstantPool *MCP,
-                                           const TargetData *TD) {
+                                           const DataLayout *TD) {
   const std::vector<MachineConstantPoolEntry> &Constants = MCP->getConstants();
   if (Constants.empty()) return 0;
 
@@ -780,7 +775,7 @@ static unsigned GetConstantPoolSizeInBytes(MachineConstantPool *MCP,
 
 void JITEmitter::startFunction(MachineFunction &F) {
   DEBUG(dbgs() << "JIT: Starting CodeGen of Function "
-        << F.getFunction()->getName() << "\n");
+        << F.getName() << "\n");
 
   uintptr_t ActualSize = 0;
   // Set the memory writable, if it's not already
@@ -929,7 +924,7 @@ bool JITEmitter::finishFunction(MachineFunction &F) {
   PrevDL = DebugLoc();
 
   DEBUG(dbgs() << "JIT: Finished CodeGen of [" << (void*)FnStart
-        << "] Function: " << F.getFunction()->getName()
+        << "] Function: " << F.getName()
         << ": " << (FnEnd-FnStart) << " bytes of text, "
         << Relocations.size() << " relocations\n");
 
@@ -1058,7 +1053,7 @@ void JITEmitter::emitConstantPool(MachineConstantPool *MCP) {
   const std::vector<MachineConstantPoolEntry> &Constants = MCP->getConstants();
   if (Constants.empty()) return;
 
-  unsigned Size = GetConstantPoolSizeInBytes(MCP, TheJIT->getTargetData());
+  unsigned Size = GetConstantPoolSizeInBytes(MCP, TheJIT->getDataLayout());
   unsigned Align = MCP->getConstantPoolAlignment();
   ConstantPoolBase = allocateSpace(Size, Align);
   ConstantPool = MCP;
@@ -1087,7 +1082,7 @@ void JITEmitter::emitConstantPool(MachineConstantPool *MCP) {
           dbgs().write_hex(CAddr) << "]\n");
 
     Type *Ty = CPE.Val.ConstVal->getType();
-    Offset += TheJIT->getTargetData()->getTypeAllocSize(Ty);
+    Offset += TheJIT->getDataLayout()->getTypeAllocSize(Ty);
   }
 }
 
@@ -1104,14 +1099,14 @@ void JITEmitter::initJumpTableInfo(MachineJumpTableInfo *MJTI) {
   for (unsigned i = 0, e = JT.size(); i != e; ++i)
     NumEntries += JT[i].MBBs.size();
 
-  unsigned EntrySize = MJTI->getEntrySize(*TheJIT->getTargetData());
+  unsigned EntrySize = MJTI->getEntrySize(*TheJIT->getDataLayout());
 
   // Just allocate space for all the jump tables now.  We will fix up the actual
   // MBB entries in the tables after we emit the code for each block, since then
   // we will know the final locations of the MBBs in memory.
   JumpTable = MJTI;
   JumpTableBase = allocateSpace(NumEntries * EntrySize,
-                             MJTI->getEntryAlignment(*TheJIT->getTargetData()));
+                             MJTI->getEntryAlignment(*TheJIT->getDataLayout()));
 }
 
 void JITEmitter::emitJumpTableInfo(MachineJumpTableInfo *MJTI) {
@@ -1128,7 +1123,7 @@ void JITEmitter::emitJumpTableInfo(MachineJumpTableInfo *MJTI) {
   case MachineJumpTableInfo::EK_BlockAddress: {
     // EK_BlockAddress - Each entry is a plain address of block, e.g.:
     //     .word LBB123
-    assert(MJTI->getEntrySize(*TheJIT->getTargetData()) == sizeof(void*) &&
+    assert(MJTI->getEntrySize(*TheJIT->getDataLayout()) == sizeof(void*) &&
            "Cross JIT'ing?");
 
     // For each jump table, map each target in the jump table to the address of
@@ -1148,7 +1143,7 @@ void JITEmitter::emitJumpTableInfo(MachineJumpTableInfo *MJTI) {
   case MachineJumpTableInfo::EK_Custom32:
   case MachineJumpTableInfo::EK_GPRel32BlockAddress:
   case MachineJumpTableInfo::EK_LabelDifference32: {
-    assert(MJTI->getEntrySize(*TheJIT->getTargetData()) == 4&&"Cross JIT'ing?");
+    assert(MJTI->getEntrySize(*TheJIT->getDataLayout()) == 4&&"Cross JIT'ing?");
     // For each jump table, place the offset from the beginning of the table
     // to the target address.
     int *SlotPtr = (int*)JumpTableBase;
@@ -1224,7 +1219,7 @@ uintptr_t JITEmitter::getJumpTableEntryAddress(unsigned Index) const {
   const std::vector<MachineJumpTableEntry> &JT = JumpTable->getJumpTables();
   assert(Index < JT.size() && "Invalid jump table index!");
 
-  unsigned EntrySize = JumpTable->getEntrySize(*TheJIT->getTargetData());
+  unsigned EntrySize = JumpTable->getEntrySize(*TheJIT->getDataLayout());
 
   unsigned Offset = 0;
   for (unsigned i = 0; i < Index; ++i)
@@ -1265,15 +1260,13 @@ void *JIT::getPointerToFunctionOrStub(Function *F) {
     return Addr;
 
   // Get a stub if the target supports it.
-  assert(isa<JITEmitter>(JCE) && "Unexpected MCE?");
-  JITEmitter *JE = cast<JITEmitter>(getCodeEmitter());
+  JITEmitter *JE = static_cast<JITEmitter*>(getCodeEmitter());
   return JE->getJITResolver().getLazyFunctionStub(F);
 }
 
 void JIT::updateFunctionStub(Function *F) {
   // Get the empty stub we generated earlier.
-  assert(isa<JITEmitter>(JCE) && "Unexpected MCE?");
-  JITEmitter *JE = cast<JITEmitter>(getCodeEmitter());
+  JITEmitter *JE = static_cast<JITEmitter*>(getCodeEmitter());
   void *Stub = JE->getJITResolver().getLazyFunctionStub(F);
   void *Addr = getPointerToGlobalIfAvailable(F);
   assert(Addr != Stub && "Function must have non-stub address to be updated.");
@@ -1294,6 +1287,5 @@ void JIT::freeMachineCodeForFunction(Function *F) {
   updateGlobalMapping(F, 0);
 
   // Free the actual memory for the function body and related stuff.
-  assert(isa<JITEmitter>(JCE) && "Unexpected MCE?");
-  cast<JITEmitter>(JCE)->deallocateMemForFunction(F);
+  static_cast<JITEmitter*>(JCE)->deallocateMemForFunction(F);
 }
diff --git a/lib/ExecutionEngine/MCJIT/CMakeLists.txt b/lib/ExecutionEngine/MCJIT/CMakeLists.txt
index fef7176..2911a50 100644
--- a/lib/ExecutionEngine/MCJIT/CMakeLists.txt
+++ b/lib/ExecutionEngine/MCJIT/CMakeLists.txt
@@ -1,4 +1,3 @@
 add_llvm_library(LLVMMCJIT
   MCJIT.cpp
-  MCJITMemoryManager.cpp
   )
diff --git a/lib/ExecutionEngine/MCJIT/MCJIT.cpp b/lib/ExecutionEngine/MCJIT/MCJIT.cpp
index 739ffd7d8..752c5b7 100644
--- a/lib/ExecutionEngine/MCJIT/MCJIT.cpp
+++ b/lib/ExecutionEngine/MCJIT/MCJIT.cpp
@@ -8,18 +8,20 @@
 //===----------------------------------------------------------------------===//
 
 #include "MCJIT.h"
-#include "MCJITMemoryManager.h"
 #include "llvm/DerivedTypes.h"
 #include "llvm/Function.h"
 #include "llvm/ExecutionEngine/GenericValue.h"
-#include "llvm/ExecutionEngine/MCJIT.h"
+#include "llvm/ExecutionEngine/JITEventListener.h"
 #include "llvm/ExecutionEngine/JITMemoryManager.h"
+#include "llvm/ExecutionEngine/MCJIT.h"
+#include "llvm/ExecutionEngine/ObjectBuffer.h"
+#include "llvm/ExecutionEngine/ObjectImage.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/DynamicLibrary.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/MutexGuard.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 
 using namespace llvm;
 
@@ -44,24 +46,20 @@ ExecutionEngine *MCJIT::createJIT(Module *M,
   // FIXME: Don't do this here.
   sys::DynamicLibrary::LoadLibraryPermanently(0, NULL);
 
-  // If the target supports JIT code generation, create the JIT.
-  if (TargetJITInfo *TJ = TM->getJITInfo())
-    return new MCJIT(M, TM, *TJ, new MCJITMemoryManager(JMM), GVsWithCode);
-
-  if (ErrorStr)
-    *ErrorStr = "target does not support JIT code generation";
-  return 0;
+  return new MCJIT(M, TM, JMM, GVsWithCode);
 }
 
-MCJIT::MCJIT(Module *m, TargetMachine *tm, TargetJITInfo &tji,
-             RTDyldMemoryManager *MM, bool AllocateGVsWithCode)
-  : ExecutionEngine(m), TM(tm), Ctx(0), MemMgr(MM), Dyld(MM), 
-    isCompiled(false), M(m), OS(Buffer)  {
+MCJIT::MCJIT(Module *m, TargetMachine *tm, RTDyldMemoryManager *MM,
+             bool AllocateGVsWithCode)
+  : ExecutionEngine(m), TM(tm), Ctx(0), MemMgr(MM), Dyld(MM),
+    isCompiled(false), M(m)  {
 
-  setTargetData(TM->getTargetData());
+  setDataLayout(TM->getDataLayout());
 }
 
 MCJIT::~MCJIT() {
+  if (LoadedObject)
+    NotifyFreeingObject(*LoadedObject.get());
   delete MemMgr;
   delete TM;
 }
@@ -69,7 +67,7 @@ MCJIT::~MCJIT() {
 void MCJIT::emitObject(Module *m) {
   /// Currently, MCJIT only supports a single module and the module passed to
   /// this function call is expected to be the contained module.  The module
-  /// is passed as a parameter here to prepare for multiple module support in 
+  /// is passed as a parameter here to prepare for multiple module support in
   /// the future.
   assert(M == m);
 
@@ -84,41 +82,65 @@ void MCJIT::emitObject(Module *m) {
 
   PassManager PM;
 
-  PM.add(new TargetData(*TM->getTargetData()));
+  PM.add(new DataLayout(*TM->getDataLayout()));
+
+  // The RuntimeDyld will take ownership of this shortly
+  OwningPtr<ObjectBufferStream> Buffer(new ObjectBufferStream());
 
   // Turn the machine code intermediate representation into bytes in memory
   // that may be executed.
-  if (TM->addPassesToEmitMC(PM, Ctx, OS, false)) {
+  if (TM->addPassesToEmitMC(PM, Ctx, Buffer->getOStream(), false)) {
     report_fatal_error("Target does not support MC emission!");
   }
 
   // Initialize passes.
-  // FIXME: When we support multiple modules, we'll want to move the code
-  // gen and finalization out of the constructor here and do it more
-  // on-demand as part of getPointerToFunction().
   PM.run(*m);
-  // Flush the output buffer so the SmallVector gets its data.
-  OS.flush();
+  // Flush the output buffer to get the generated code into memory
+  Buffer->flush();
 
   // Load the object into the dynamic linker.
-  MemoryBuffer* MB = MemoryBuffer::getMemBuffer(StringRef(Buffer.data(),
-                                                          Buffer.size()),
-                                                "", false);
-  if (Dyld.loadObject(MB))
+  // handing off ownership of the buffer
+  LoadedObject.reset(Dyld.loadObject(Buffer.take()));
+  if (!LoadedObject)
     report_fatal_error(Dyld.getErrorString());
 
   // Resolve any relocations.
   Dyld.resolveRelocations();
 
+  // FIXME: Make this optional, maybe even move it to a JIT event listener
+  LoadedObject->registerWithDebugger();
+
+  NotifyObjectEmitted(*LoadedObject);
+
   // FIXME: Add support for per-module compilation state
   isCompiled = true;
 }
 
+// FIXME: Add a parameter to identify which object is being finalized when
+// MCJIT supports multiple modules.
+void MCJIT::finalizeObject() {
+  // If the module hasn't been compiled, just do that.
+  if (!isCompiled) {
+    // If the call to Dyld.resolveRelocations() is removed from emitObject()
+    // we'll need to do that here.
+    emitObject(M);
+    return;
+  }
+
+  // Resolve any relocations.
+  Dyld.resolveRelocations();
+}
+
 void *MCJIT::getPointerToBasicBlock(BasicBlock *BB) {
   report_fatal_error("not yet implemented");
 }
 
 void *MCJIT::getPointerToFunction(Function *F) {
+  // FIXME: This should really return a uint64_t since it's a pointer in the
+  // target address space, not our local address space. That's part of the
+  // ExecutionEngine interface, though. Fix that when the old JIT finally
+  // dies.
+
   // FIXME: Add support for per-module compilation state
   if (!isCompiled)
     emitObject(M);
@@ -132,10 +154,13 @@ void *MCJIT::getPointerToFunction(Function *F) {
 
   // FIXME: Should the Dyld be retaining module information? Probably not.
   // FIXME: Should we be using the mangler for this? Probably.
+  //
+  // This is the accessor for the target address, so make sure to check the
+  // load address of the symbol, not the local address.
   StringRef BaseName = F->getName();
   if (BaseName[0] == '\1')
-    return (void*)Dyld.getSymbolAddress(BaseName.substr(1));
-  return (void*)Dyld.getSymbolAddress((TM->getMCAsmInfo()->getGlobalPrefix()
+    return (void*)Dyld.getSymbolLoadAddress(BaseName.substr(1));
+  return (void*)Dyld.getSymbolLoadAddress((TM->getMCAsmInfo()->getGlobalPrefix()
                                        + BaseName).str());
 }
 
@@ -270,3 +295,33 @@ void *MCJIT::getPointerToNamedFunction(const std::string &Name,
   }
   return 0;
 }
+
+void MCJIT::RegisterJITEventListener(JITEventListener *L) {
+  if (L == NULL)
+    return;
+  MutexGuard locked(lock);
+  EventListeners.push_back(L);
+}
+void MCJIT::UnregisterJITEventListener(JITEventListener *L) {
+  if (L == NULL)
+    return;
+  MutexGuard locked(lock);
+  SmallVector<JITEventListener*, 2>::reverse_iterator I=
+      std::find(EventListeners.rbegin(), EventListeners.rend(), L);
+  if (I != EventListeners.rend()) {
+    std::swap(*I, EventListeners.back());
+    EventListeners.pop_back();
+  }
+}
+void MCJIT::NotifyObjectEmitted(const ObjectImage& Obj) {
+  MutexGuard locked(lock);
+  for (unsigned I = 0, S = EventListeners.size(); I < S; ++I) {
+    EventListeners[I]->NotifyObjectEmitted(Obj);
+  }
+}
+void MCJIT::NotifyFreeingObject(const ObjectImage& Obj) {
+  MutexGuard locked(lock);
+  for (unsigned I = 0, S = EventListeners.size(); I < S; ++I) {
+    EventListeners[I]->NotifyFreeingObject(Obj);
+  }
+}
diff --git a/lib/ExecutionEngine/MCJIT/MCJIT.h b/lib/ExecutionEngine/MCJIT/MCJIT.h
index 1d272e9..571080d 100644
--- a/lib/ExecutionEngine/MCJIT/MCJIT.h
+++ b/lib/ExecutionEngine/MCJIT/MCJIT.h
@@ -11,33 +11,32 @@
 #define LLVM_LIB_EXECUTIONENGINE_MCJIT_H
 
 #include "llvm/PassManager.h"
+#include "llvm/ADT/SmallVector.h"
 #include "llvm/ExecutionEngine/ExecutionEngine.h"
 #include "llvm/ExecutionEngine/RuntimeDyld.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/Support/raw_ostream.h"
 
 namespace llvm {
 
+class ObjectImage;
+
 // FIXME: This makes all kinds of horrible assumptions for the time being,
 // like only having one module, not needing to worry about multi-threading,
 // blah blah. Purely in get-it-up-and-limping mode for now.
 
 class MCJIT : public ExecutionEngine {
-  MCJIT(Module *M, TargetMachine *tm, TargetJITInfo &tji,
-        RTDyldMemoryManager *MemMgr, bool AllocateGVsWithCode);
+  MCJIT(Module *M, TargetMachine *tm, RTDyldMemoryManager *MemMgr,
+        bool AllocateGVsWithCode);
 
   TargetMachine *TM;
   MCContext *Ctx;
   RTDyldMemoryManager *MemMgr;
   RuntimeDyld Dyld;
+  SmallVector<JITEventListener*, 2> EventListeners;
 
   // FIXME: Add support for multiple modules
   bool isCompiled;
   Module *M;
-
-  // FIXME: Move these to a single container which manages JITed objects
-  SmallVector<char, 4096> Buffer; // Working buffer into which we JIT.
-  raw_svector_ostream OS;
+  OwningPtr<ObjectImage> LoadedObject;
 
 public:
   ~MCJIT();
@@ -45,6 +44,8 @@ public:
   /// @name ExecutionEngine interface implementation
   /// @{
 
+  virtual void finalizeObject();
+
   virtual void *getPointerToBasicBlock(BasicBlock *BB);
 
   virtual void *getPointerToFunction(Function *F);
@@ -71,10 +72,14 @@ public:
   /// Map the address of a JIT section as returned from the memory manager
   /// to the address in the target process as the running code will see it.
   /// This is the address which will be used for relocation resolution.
-  virtual void mapSectionAddress(void *LocalAddress, uint64_t TargetAddress) {
+  virtual void mapSectionAddress(const void *LocalAddress,
+                                 uint64_t TargetAddress) {
     Dyld.mapSectionAddress(LocalAddress, TargetAddress);
   }
 
+  virtual void RegisterJITEventListener(JITEventListener *L);
+  virtual void UnregisterJITEventListener(JITEventListener *L);
+
   /// @}
   /// @name (Private) Registration Interfaces
   /// @{
@@ -98,6 +103,9 @@ protected:
   /// is passed as a parameter here to prepare for multiple module support in 
   /// the future.
   void emitObject(Module *M);
+
+  void NotifyObjectEmitted(const ObjectImage& Obj);
+  void NotifyFreeingObject(const ObjectImage& Obj);
 };
 
 } // End llvm namespace
diff --git a/lib/ExecutionEngine/MCJIT/MCJITMemoryManager.cpp b/lib/ExecutionEngine/MCJIT/MCJITMemoryManager.cpp
deleted file mode 100644
index 457fe5e..0000000
--- a/lib/ExecutionEngine/MCJIT/MCJITMemoryManager.cpp
+++ /dev/null
@@ -1,14 +0,0 @@
-//==-- MCJITMemoryManager.cpp - Definition for the Memory Manager -*-C++ -*-==//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "MCJITMemoryManager.h"
-
-using namespace llvm;
-
-void MCJITMemoryManager::anchor() { }
diff --git a/lib/ExecutionEngine/MCJIT/MCJITMemoryManager.h b/lib/ExecutionEngine/MCJIT/MCJITMemoryManager.h
deleted file mode 100644
index 441aaeb..0000000
--- a/lib/ExecutionEngine/MCJIT/MCJITMemoryManager.h
+++ /dev/null
@@ -1,50 +0,0 @@
-//===-- MCJITMemoryManager.h - Definition for the Memory Manager ---C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIB_EXECUTIONENGINE_MCJITMEMORYMANAGER_H
-#define LLVM_LIB_EXECUTIONENGINE_MCJITMEMORYMANAGER_H
-
-#include "llvm/Module.h"
-#include "llvm/ExecutionEngine/JITMemoryManager.h"
-#include "llvm/ExecutionEngine/RuntimeDyld.h"
-#include <assert.h>
-
-namespace llvm {
-
-// The MCJIT memory manager is a layer between the standard JITMemoryManager
-// and the RuntimeDyld interface that maps objects, by name, onto their
-// matching LLVM IR counterparts in the module(s) being compiled.
-class MCJITMemoryManager : public RTDyldMemoryManager {
-  virtual void anchor();
-  OwningPtr<JITMemoryManager> JMM;
-
-public:
-  MCJITMemoryManager(JITMemoryManager *jmm) :
-    JMM(jmm?jmm:JITMemoryManager::CreateDefaultMemManager()) {}
-
-  uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment,
-                               unsigned SectionID) {
-    return JMM->allocateDataSection(Size, Alignment, SectionID);
-  }
-
-  uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment,
-                               unsigned SectionID) {
-    return JMM->allocateCodeSection(Size, Alignment, SectionID);
-  }
-
-  virtual void *getPointerToNamedFunction(const std::string &Name,
-                                          bool AbortOnFailure = true) {
-    return JMM->getPointerToNamedFunction(Name, AbortOnFailure);
-  }
-
-};
-
-} // End llvm namespace
-
-#endif
diff --git a/lib/ExecutionEngine/RuntimeDyld/GDBRegistrar.cpp b/lib/ExecutionEngine/RuntimeDyld/GDBRegistrar.cpp
index 8b50101..50cd072 100644
--- a/lib/ExecutionEngine/RuntimeDyld/GDBRegistrar.cpp
+++ b/lib/ExecutionEngine/RuntimeDyld/GDBRegistrar.cpp
@@ -78,12 +78,12 @@ public:
   /// Creates an entry in the JIT registry for the buffer @p Object,
   /// which must contain an object file in executable memory with any
   /// debug information for the debugger.
-  void registerObject(const MemoryBuffer &Object);
+  void registerObject(const ObjectBuffer &Object);
 
   /// Removes the internal registration of @p Object, and
   /// frees associated resources.
   /// Returns true if @p Object was found in ObjectBufferMap.
-  bool deregisterObject(const MemoryBuffer &Object);
+  bool deregisterObject(const ObjectBuffer &Object);
 
 private:
   /// Deregister the debug info for the given object file from the debugger
@@ -124,7 +124,7 @@ GDBJITRegistrar::~GDBJITRegistrar() {
   ObjectBufferMap.clear();
 }
 
-void GDBJITRegistrar::registerObject(const MemoryBuffer &Object) {
+void GDBJITRegistrar::registerObject(const ObjectBuffer &Object) {
 
   const char *Buffer = Object.getBufferStart();
   size_t      Size = Object.getBufferSize();
@@ -147,7 +147,7 @@ void GDBJITRegistrar::registerObject(const MemoryBuffer &Object) {
   }
 }
 
-bool GDBJITRegistrar::deregisterObject(const MemoryBuffer& Object) {
+bool GDBJITRegistrar::deregisterObject(const ObjectBuffer& Object) {
   const char *Buffer = Object.getBufferStart();
   RegisteredObjectBufferMap::iterator I = ObjectBufferMap.find(Buffer);
 
diff --git a/lib/ExecutionEngine/RuntimeDyld/JITRegistrar.h b/lib/ExecutionEngine/RuntimeDyld/JITRegistrar.h
index f964bc6..69e9dbe 100644
--- a/lib/ExecutionEngine/RuntimeDyld/JITRegistrar.h
+++ b/lib/ExecutionEngine/RuntimeDyld/JITRegistrar.h
@@ -10,7 +10,7 @@
 #ifndef LLVM_EXECUTION_ENGINE_JIT_REGISTRAR_H
 #define LLVM_EXECUTION_ENGINE_JIT_REGISTRAR_H
 
-#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/ExecutionEngine/ObjectBuffer.h"
 
 namespace llvm {
 
@@ -27,12 +27,12 @@ public:
   /// Creates an entry in the JIT registry for the buffer @p Object,
   /// which must contain an object file in executable memory with any
   /// debug information for the debugger.
-  virtual void registerObject(const MemoryBuffer &Object) = 0;
+  virtual void registerObject(const ObjectBuffer &Object) = 0;
 
   /// Removes the internal registration of @p Object, and
   /// frees associated resources.
   /// Returns true if @p Object was previously registered.
-  virtual bool deregisterObject(const MemoryBuffer &Object) = 0;
+  virtual bool deregisterObject(const ObjectBuffer &Object) = 0;
 
   /// Returns a reference to a GDB JIT registrar singleton
   static JITRegistrar& getGDBRegistrar();
diff --git a/lib/ExecutionEngine/RuntimeDyld/ObjectImage.h b/lib/ExecutionEngine/RuntimeDyld/ObjectImage.h
deleted file mode 100644
index c3e3572..0000000
--- a/lib/ExecutionEngine/RuntimeDyld/ObjectImage.h
+++ /dev/null
@@ -1,59 +0,0 @@
-//===---- ObjectImage.h - Format independent executuable object image -----===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file declares a file format independent ObjectImage class.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_RUNTIMEDYLD_OBJECT_IMAGE_H
-#define LLVM_RUNTIMEDYLD_OBJECT_IMAGE_H
-
-#include "llvm/Object/ObjectFile.h"
-
-namespace llvm {
-
-class ObjectImage {
-  ObjectImage(); // = delete
-  ObjectImage(const ObjectImage &other); // = delete
-protected:
-  object::ObjectFile *ObjFile;
-
-public:
-  ObjectImage(object::ObjectFile *Obj) { ObjFile = Obj; }
-  virtual ~ObjectImage() {}
-
-  virtual object::symbol_iterator begin_symbols() const
-              { return ObjFile->begin_symbols(); }
-  virtual object::symbol_iterator end_symbols() const
-              { return ObjFile->end_symbols(); }
-
-  virtual object::section_iterator begin_sections() const
-              { return ObjFile->begin_sections(); }
-  virtual object::section_iterator end_sections() const
-              { return ObjFile->end_sections(); }
-
-  virtual /* Triple::ArchType */ unsigned getArch() const
-              { return ObjFile->getArch(); }
-
-  // Subclasses can override these methods to update the image with loaded
-  // addresses for sections and common symbols
-  virtual void updateSectionAddress(const object::SectionRef &Sec,
-                                    uint64_t Addr) {}
-  virtual void updateSymbolAddress(const object::SymbolRef &Sym, uint64_t Addr)
-              {}
-
-  // Subclasses can override these methods to provide JIT debugging support
-  virtual void registerWithDebugger() {}
-  virtual void deregisterWithDebugger() {}
-};
-
-} // end namespace llvm
-
-#endif // LLVM_RUNTIMEDYLD_OBJECT_IMAGE_H
-
diff --git a/lib/ExecutionEngine/RuntimeDyld/ObjectImageCommon.h b/lib/ExecutionEngine/RuntimeDyld/ObjectImageCommon.h
new file mode 100644
index 0000000..17f3a21
--- /dev/null
+++ b/lib/ExecutionEngine/RuntimeDyld/ObjectImageCommon.h
@@ -0,0 +1,76 @@
+//===-- ObjectImageCommon.h - Format independent executuable object image -===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares a file format independent ObjectImage class.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_RUNTIMEDYLD_OBJECTIMAGECOMMON_H
+#define LLVM_RUNTIMEDYLD_OBJECTIMAGECOMMON_H
+
+#include "llvm/Object/ObjectFile.h"
+#include "llvm/ExecutionEngine/ObjectImage.h"
+#include "llvm/ExecutionEngine/ObjectBuffer.h"
+
+namespace llvm {
+
+class ObjectImageCommon : public ObjectImage {
+  ObjectImageCommon(); // = delete
+  ObjectImageCommon(const ObjectImageCommon &other); // = delete
+
+protected:
+  object::ObjectFile *ObjFile;
+
+  // This form of the constructor allows subclasses to use
+  // format-specific subclasses of ObjectFile directly
+  ObjectImageCommon(ObjectBuffer *Input, object::ObjectFile *Obj)
+  : ObjectImage(Input), // saves Input as Buffer and takes ownership
+    ObjFile(Obj)
+  {
+  }
+
+public:
+  ObjectImageCommon(ObjectBuffer* Input)
+  : ObjectImage(Input) // saves Input as Buffer and takes ownership
+  {
+    ObjFile = object::ObjectFile::createObjectFile(Buffer->getMemBuffer());
+  }
+  virtual ~ObjectImageCommon() { delete ObjFile; }
+
+  virtual object::symbol_iterator begin_symbols() const
+              { return ObjFile->begin_symbols(); }
+  virtual object::symbol_iterator end_symbols() const
+              { return ObjFile->end_symbols(); }
+
+  virtual object::section_iterator begin_sections() const
+              { return ObjFile->begin_sections(); }
+  virtual object::section_iterator end_sections() const
+              { return ObjFile->end_sections(); }
+
+  virtual /* Triple::ArchType */ unsigned getArch() const
+              { return ObjFile->getArch(); }
+
+  virtual StringRef getData() const { return ObjFile->getData(); }
+
+  // Subclasses can override these methods to update the image with loaded
+  // addresses for sections and common symbols
+  virtual void updateSectionAddress(const object::SectionRef &Sec,
+                                    uint64_t Addr) {}
+  virtual void updateSymbolAddress(const object::SymbolRef &Sym, uint64_t Addr)
+              {}
+
+  // Subclasses can override these methods to provide JIT debugging support
+  virtual void registerWithDebugger() {}
+  virtual void deregisterWithDebugger() {}
+};
+
+} // end namespace llvm
+
+#endif // LLVM_RUNTIMEDYLD_OBJECT_IMAGE_H
+
diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp
index b464040..f6dccb1 100644
--- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp
+++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp
@@ -12,10 +12,12 @@
 //===----------------------------------------------------------------------===//
 
 #define DEBUG_TYPE "dyld"
+#include "ObjectImageCommon.h"
 #include "RuntimeDyldImpl.h"
 #include "RuntimeDyldELF.h"
 #include "RuntimeDyldMachO.h"
 #include "llvm/Support/Path.h"
+#include "llvm/Support/MathExtras.h"
 
 using namespace llvm;
 using namespace llvm::object;
@@ -26,16 +28,6 @@ RuntimeDyldImpl::~RuntimeDyldImpl() {}
 
 namespace llvm {
 
-namespace {
-  // Helper for extensive error checking in debug builds.
-  error_code Check(error_code Err) {
-    if (Err) {
-      report_fatal_error(Err.message());
-    }
-    return Err;
-  }
-} // end anonymous namespace
-
 // Resolve the relocations for all symbols we currently know about.
 void RuntimeDyldImpl::resolveRelocations() {
   // First, resolve relocations associated with external symbols.
@@ -44,11 +36,15 @@ void RuntimeDyldImpl::resolveRelocations() {
   // Just iterate over the sections we have and resolve all the relocations
   // in them. Gross overkill, but it gets the job done.
   for (int i = 0, e = Sections.size(); i != e; ++i) {
-    reassignSectionAddress(i, Sections[i].LoadAddress);
+    uint64_t Addr = Sections[i].LoadAddress;
+    DEBUG(dbgs() << "Resolving relocations Section #" << i
+            << "\t" << format("%p", (uint8_t *)Addr)
+            << "\n");
+    resolveRelocationList(Relocations[i], Addr);
   }
 }
 
-void RuntimeDyldImpl::mapSectionAddress(void *LocalAddress,
+void RuntimeDyldImpl::mapSectionAddress(const void *LocalAddress,
                                         uint64_t TargetAddress) {
   for (unsigned i = 0, e = Sections.size(); i != e; ++i) {
     if (Sections[i].Address == LocalAddress) {
@@ -61,14 +57,11 @@ void RuntimeDyldImpl::mapSectionAddress(void *LocalAddress,
 
 // Subclasses can implement this method to create specialized image instances.
 // The caller owns the pointer that is returned.
-ObjectImage *RuntimeDyldImpl::createObjectImage(const MemoryBuffer *InputBuffer) {
-  ObjectFile *ObjFile = ObjectFile::createObjectFile(const_cast<MemoryBuffer*>
-                                                                 (InputBuffer));
-  ObjectImage *Obj = new ObjectImage(ObjFile);
-  return Obj;
+ObjectImage *RuntimeDyldImpl::createObjectImage(ObjectBuffer *InputBuffer) {
+  return new ObjectImageCommon(InputBuffer);
 }
 
-bool RuntimeDyldImpl::loadObject(const MemoryBuffer *InputBuffer) {
+ObjectImage *RuntimeDyldImpl::loadObject(ObjectBuffer *InputBuffer) {
   OwningPtr<ObjectImage> obj(createObjectImage(InputBuffer));
   if (!obj)
     report_fatal_error("Unable to create object image from memory buffer!");
@@ -80,9 +73,9 @@ bool RuntimeDyldImpl::loadObject(const MemoryBuffer *InputBuffer) {
   // Used sections from the object file
   ObjSectionToIDMap LocalSections;
 
-  // Common symbols requiring allocation, and the total size required to
-  // allocate all common symbols.
+  // Common symbols requiring allocation, with their sizes and alignments
   CommonSymbolMap CommonSymbols;
+  // Maximum required total memory to allocate all common symbols
   uint64_t CommonSize = 0;
 
   error_code err;
@@ -102,13 +95,15 @@ bool RuntimeDyldImpl::loadObject(const MemoryBuffer *InputBuffer) {
     bool isCommon = flags & SymbolRef::SF_Common;
     if (isCommon) {
       // Add the common symbols to a list.  We'll allocate them all below.
+      uint64_t Align = getCommonSymbolAlignment(*i);
       uint64_t Size = 0;
       Check(i->getSize(Size));
-      CommonSize += Size;
-      CommonSymbols[*i] = Size;
+      CommonSize += Size + Align;
+      CommonSymbols[*i] = CommonSymbolInfo(Size, Align);
     } else {
       if (SymType == object::SymbolRef::ST_Function ||
-          SymType == object::SymbolRef::ST_Data) {
+          SymType == object::SymbolRef::ST_Data ||
+          SymType == object::SymbolRef::ST_Unknown) {
         uint64_t FileOffset;
         StringRef SectionData;
         section_iterator si = obj->end_sections();
@@ -177,9 +172,7 @@ bool RuntimeDyldImpl::loadObject(const MemoryBuffer *InputBuffer) {
     }
   }
 
-  handleObjectLoaded(obj.take());
-
-  return false;
+  return obj.take();
 }
 
 void RuntimeDyldImpl::emitCommonSymbols(ObjectImage &Obj,
@@ -193,7 +186,7 @@ void RuntimeDyldImpl::emitCommonSymbols(ObjectImage &Obj,
   if (!Addr)
     report_fatal_error("Unable to allocate memory for common symbols!");
   uint64_t Offset = 0;
-  Sections.push_back(SectionEntry(Addr, TotalSize, TotalSize, 0));
+  Sections.push_back(SectionEntry(StringRef(), Addr, TotalSize, TotalSize, 0));
   memset(Addr, 0, TotalSize);
 
   DEBUG(dbgs() << "emitCommonSection SectionID: " << SectionID
@@ -204,11 +197,20 @@ void RuntimeDyldImpl::emitCommonSymbols(ObjectImage &Obj,
   // Assign the address of each symbol
   for (CommonSymbolMap::const_iterator it = CommonSymbols.begin(),
        itEnd = CommonSymbols.end(); it != itEnd; it++) {
+    uint64_t Size = it->second.first;
+    uint64_t Align = it->second.second;
     StringRef Name;
     it->first.getName(Name);
+    if (Align) {
+      // This symbol has an alignment requirement.
+      uint64_t AlignOffset = OffsetToAlignment((uint64_t)Addr, Align);
+      Addr += AlignOffset;
+      Offset += AlignOffset;
+      DEBUG(dbgs() << "Allocating common symbol " << Name << " address " <<
+                      format("%p\n", Addr));
+    }
     Obj.updateSymbolAddress(it->first, (uint64_t)Addr);
     SymbolTable[Name.data()] = SymbolLoc(SectionID, Offset);
-    uint64_t Size = it->second;
     Offset += Size;
     Addr += Size;
   }
@@ -236,10 +238,12 @@ unsigned RuntimeDyldImpl::emitSection(ObjectImage &Obj,
   bool IsVirtual;
   bool IsZeroInit;
   uint64_t DataSize;
+  StringRef Name;
   Check(Section.isRequiredForExecution(IsRequired));
   Check(Section.isVirtual(IsVirtual));
   Check(Section.isZeroInit(IsZeroInit));
   Check(Section.getSize(DataSize));
+  Check(Section.getName(Name));
 
   unsigned Allocate;
   unsigned SectionID = Sections.size();
@@ -267,6 +271,7 @@ unsigned RuntimeDyldImpl::emitSection(ObjectImage &Obj,
       memcpy(Addr, pData, DataSize);
 
     DEBUG(dbgs() << "emitSection SectionID: " << SectionID
+                 << " Name: " << Name
                  << " obj addr: " << format("%p", pData)
                  << " new addr: " << format("%p", Addr)
                  << " DataSize: " << DataSize
@@ -282,6 +287,7 @@ unsigned RuntimeDyldImpl::emitSection(ObjectImage &Obj,
     Allocate = 0;
     Addr = 0;
     DEBUG(dbgs() << "emitSection SectionID: " << SectionID
+                 << " Name: " << Name
                  << " obj addr: " << format("%p", data.data())
                  << " new addr: 0"
                  << " DataSize: " << DataSize
@@ -290,7 +296,8 @@ unsigned RuntimeDyldImpl::emitSection(ObjectImage &Obj,
                  << "\n");
   }
 
-  Sections.push_back(SectionEntry(Addr, Allocate, DataSize,(uintptr_t)pData));
+  Sections.push_back(SectionEntry(Name, Addr, Allocate, DataSize,
+				  (uintptr_t)pData));
   return SectionID;
 }
 
@@ -333,15 +340,49 @@ void RuntimeDyldImpl::addRelocationForSymbol(const RelocationEntry &RE,
 }
 
 uint8_t *RuntimeDyldImpl::createStubFunction(uint8_t *Addr) {
-  // TODO: There is only ARM far stub now. We should add the Thumb stub,
-  // and stubs for branches Thumb - ARM and ARM - Thumb.
   if (Arch == Triple::arm) {
+    // TODO: There is only ARM far stub now. We should add the Thumb stub,
+    // and stubs for branches Thumb - ARM and ARM - Thumb.
     uint32_t *StubAddr = (uint32_t*)Addr;
     *StubAddr = 0xe51ff004; // ldr pc,<label>
     return (uint8_t*)++StubAddr;
-  }
-  else
+  } else if (Arch == Triple::mipsel) {
+    uint32_t *StubAddr = (uint32_t*)Addr;
+    // 0:   3c190000        lui     t9,%hi(addr).
+    // 4:   27390000        addiu   t9,t9,%lo(addr).
+    // 8:   03200008        jr      t9.
+    // c:   00000000        nop.
+    const unsigned LuiT9Instr = 0x3c190000, AdduiT9Instr = 0x27390000;
+    const unsigned JrT9Instr = 0x03200008, NopInstr = 0x0;
+
+    *StubAddr = LuiT9Instr;
+    StubAddr++;
+    *StubAddr = AdduiT9Instr;
+    StubAddr++;
+    *StubAddr = JrT9Instr;
+    StubAddr++;
+    *StubAddr = NopInstr;
     return Addr;
+  } else if (Arch == Triple::ppc64) {
+    // PowerPC64 stub: the address points to a function descriptor
+    // instead of the function itself. Load the function address
+    // on r11 and sets it to control register. Also loads the function
+    // TOC in r2 and environment pointer to r11.
+    writeInt32BE(Addr,    0x3D800000); // lis   r12, highest(addr)
+    writeInt32BE(Addr+4,  0x618C0000); // ori   r12, higher(addr)
+    writeInt32BE(Addr+8,  0x798C07C6); // sldi  r12, r12, 32
+    writeInt32BE(Addr+12, 0x658C0000); // oris  r12, r12, h(addr)
+    writeInt32BE(Addr+16, 0x618C0000); // ori   r12, r12, l(addr)
+    writeInt32BE(Addr+20, 0xF8410028); // std   r2,  40(r1)
+    writeInt32BE(Addr+24, 0xE96C0000); // ld    r11, 0(r12)
+    writeInt32BE(Addr+28, 0xE84C0008); // ld    r2,  0(r12)
+    writeInt32BE(Addr+32, 0x7D6903A6); // mtctr r11
+    writeInt32BE(Addr+36, 0xE96C0010); // ld    r11, 16(r2)
+    writeInt32BE(Addr+40, 0x4E800420); // bctr
+
+    return Addr;
+  }
+  return Addr;
 }
 
 // Assign an address to a symbol name and resolve all the relocations
@@ -350,32 +391,30 @@ void RuntimeDyldImpl::reassignSectionAddress(unsigned SectionID,
                                              uint64_t Addr) {
   // The address to use for relocation resolution is not
   // the address of the local section buffer. We must be doing
-  // a remote execution environment of some sort. Re-apply any
-  // relocations referencing this section with the given address.
+  // a remote execution environment of some sort. Relocations can't
+  // be applied until all the sections have been moved.  The client must
+  // trigger this with a call to MCJIT::finalize() or
+  // RuntimeDyld::resolveRelocations().
   //
   // Addr is a uint64_t because we can't assume the pointer width
   // of the target is the same as that of the host. Just use a generic
   // "big enough" type.
   Sections[SectionID].LoadAddress = Addr;
-  DEBUG(dbgs() << "Resolving relocations Section #" << SectionID
-          << "\t" << format("%p", (uint8_t *)Addr)
-          << "\n");
-  resolveRelocationList(Relocations[SectionID], Addr);
 }
 
 void RuntimeDyldImpl::resolveRelocationEntry(const RelocationEntry &RE,
                                              uint64_t Value) {
-    // Ignore relocations for sections that were not loaded
-    if (Sections[RE.SectionID].Address != 0) {
-      uint8_t *Target = Sections[RE.SectionID].Address + RE.Offset;
-      DEBUG(dbgs() << "\tSectionID: " << RE.SectionID
-            << " + " << RE.Offset << " (" << format("%p", Target) << ")"
-            << " RelType: " << RE.RelType
-            << " Addend: " << RE.Addend
-            << "\n");
+  // Ignore relocations for sections that were not loaded
+  if (Sections[RE.SectionID].Address != 0) {
+    DEBUG(dbgs() << "\tSectionID: " << RE.SectionID
+          << " + " << RE.Offset << " ("
+          << format("%p", Sections[RE.SectionID].Address + RE.Offset) << ")"
+          << " RelType: " << RE.RelType
+          << " Addend: " << RE.Addend
+          << "\n");
 
-      resolveRelocation(Target, Sections[RE.SectionID].LoadAddress + RE.Offset,
-                        Value, RE.RelType, RE.Addend);
+    resolveRelocation(Sections[RE.SectionID], RE.Offset,
+                      Value, RE.RelType, RE.Addend);
   }
 }
 
@@ -420,7 +459,7 @@ RuntimeDyld::~RuntimeDyld() {
   delete Dyld;
 }
 
-bool RuntimeDyld::loadObject(MemoryBuffer *InputBuffer) {
+ObjectImage *RuntimeDyld::loadObject(ObjectBuffer *InputBuffer) {
   if (!Dyld) {
     sys::LLVMFileType type = sys::IdentifyFileType(
             InputBuffer->getBufferStart(),
@@ -462,6 +501,10 @@ void *RuntimeDyld::getSymbolAddress(StringRef Name) {
   return Dyld->getSymbolAddress(Name);
 }
 
+uint64_t RuntimeDyld::getSymbolLoadAddress(StringRef Name) {
+  return Dyld->getSymbolLoadAddress(Name);
+}
+
 void RuntimeDyld::resolveRelocations() {
   Dyld->resolveRelocations();
 }
@@ -471,7 +514,7 @@ void RuntimeDyld::reassignSectionAddress(unsigned SectionID,
   Dyld->reassignSectionAddress(SectionID, Addr);
 }
 
-void RuntimeDyld::mapSectionAddress(void *LocalAddress,
+void RuntimeDyld::mapSectionAddress(const void *LocalAddress,
                                     uint64_t TargetAddress) {
   Dyld->mapSectionAddress(LocalAddress, TargetAddress);
 }
diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp
index 75bb586..1ebcaf7 100644
--- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp
+++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp
@@ -12,21 +12,32 @@
 //===----------------------------------------------------------------------===//
 
 #define DEBUG_TYPE "dyld"
+#include "RuntimeDyldELF.h"
+#include "JITRegistrar.h"
+#include "ObjectImageCommon.h"
 #include "llvm/ADT/OwningPtr.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/IntervalMap.h"
-#include "RuntimeDyldELF.h"
 #include "llvm/Object/ObjectFile.h"
+#include "llvm/ExecutionEngine/ObjectImage.h"
+#include "llvm/ExecutionEngine/ObjectBuffer.h"
 #include "llvm/Support/ELF.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/Object/ELF.h"
-#include "JITRegistrar.h"
 using namespace llvm;
 using namespace llvm::object;
 
 namespace {
 
+static inline
+error_code check(error_code Err) {
+  if (Err) {
+    report_fatal_error(Err.message());
+  }
+  return Err;
+}
+
 template<support::endianness target_endianness, bool is64Bits>
 class DyldELFObject : public ELFObjectFile<target_endianness, is64Bits> {
   LLVM_ELF_IMPORT_TYPES(target_endianness, is64Bits)
@@ -36,25 +47,17 @@ class DyldELFObject : public ELFObjectFile<target_endianness, is64Bits> {
   typedef Elf_Rel_Impl<target_endianness, is64Bits, false> Elf_Rel;
   typedef Elf_Rel_Impl<target_endianness, is64Bits, true> Elf_Rela;
 
-  typedef typename ELFObjectFile<target_endianness, is64Bits>::
-    Elf_Ehdr Elf_Ehdr;
+  typedef Elf_Ehdr_Impl<target_endianness, is64Bits> Elf_Ehdr;
 
   typedef typename ELFDataTypeTypedefHelper<
           target_endianness, is64Bits>::value_type addr_type;
 
-protected:
-  // This duplicates the 'Data' member in the 'Binary' base class
-  // but it is necessary to workaround a bug in gcc 4.2
-  MemoryBuffer *InputData;
-
 public:
-  DyldELFObject(MemoryBuffer *Object, error_code &ec);
+  DyldELFObject(MemoryBuffer *Wrapper, error_code &ec);
 
   void updateSectionAddress(const SectionRef &Sec, uint64_t Addr);
   void updateSymbolAddress(const SymbolRef &Sym, uint64_t Addr);
 
-  const MemoryBuffer& getBuffer() const { return *InputData; }
-
   // Methods for type inquiry through isa, cast and dyn_cast
   static inline bool classof(const Binary *v) {
     return (isa<ELFObjectFile<target_endianness, is64Bits> >(v)
@@ -64,20 +67,18 @@ public:
       const ELFObjectFile<target_endianness, is64Bits> *v) {
     return v->isDyldType();
   }
-  static inline bool classof(const DyldELFObject *v) {
-    return true;
-  }
 };
 
 template<support::endianness target_endianness, bool is64Bits>
-class ELFObjectImage : public ObjectImage {
+class ELFObjectImage : public ObjectImageCommon {
   protected:
     DyldELFObject<target_endianness, is64Bits> *DyldObj;
     bool Registered;
 
   public:
-    ELFObjectImage(DyldELFObject<target_endianness, is64Bits> *Obj)
-    : ObjectImage(Obj),
+    ELFObjectImage(ObjectBuffer *Input,
+                   DyldELFObject<target_endianness, is64Bits> *Obj)
+    : ObjectImageCommon(Input, Obj),
       DyldObj(Obj),
       Registered(false) {}
 
@@ -100,20 +101,22 @@ class ELFObjectImage : public ObjectImage {
 
     virtual void registerWithDebugger()
     {
-      JITRegistrar::getGDBRegistrar().registerObject(DyldObj->getBuffer());
+      JITRegistrar::getGDBRegistrar().registerObject(*Buffer);
       Registered = true;
     }
     virtual void deregisterWithDebugger()
     {
-      JITRegistrar::getGDBRegistrar().deregisterObject(DyldObj->getBuffer());
+      JITRegistrar::getGDBRegistrar().deregisterObject(*Buffer);
     }
 };
 
+// The MemoryBuffer passed into this constructor is just a wrapper around the
+// actual memory.  Ultimately, the Binary parent class will take ownership of
+// this MemoryBuffer object but not the underlying memory.
 template<support::endianness target_endianness, bool is64Bits>
-DyldELFObject<target_endianness, is64Bits>::DyldELFObject(MemoryBuffer *Object,
+DyldELFObject<target_endianness, is64Bits>::DyldELFObject(MemoryBuffer *Wrapper,
                                                           error_code &ec)
-  : ELFObjectFile<target_endianness, is64Bits>(Object, ec),
-    InputData(Object) {
+  : ELFObjectFile<target_endianness, is64Bits>(Wrapper, ec) {
   this->isDyldELFObject = true;
 }
 
@@ -149,50 +152,43 @@ void DyldELFObject<target_endianness, is64Bits>::updateSymbolAddress(
 
 namespace llvm {
 
-ObjectImage *RuntimeDyldELF::createObjectImage(
-                                         const MemoryBuffer *ConstInputBuffer) {
-  MemoryBuffer *InputBuffer = const_cast<MemoryBuffer*>(ConstInputBuffer);
-  std::pair<unsigned char, unsigned char> Ident = getElfArchType(InputBuffer);
+ObjectImage *RuntimeDyldELF::createObjectImage(ObjectBuffer *Buffer) {
+  if (Buffer->getBufferSize() < ELF::EI_NIDENT)
+    llvm_unreachable("Unexpected ELF object size");
+  std::pair<unsigned char, unsigned char> Ident = std::make_pair(
+                         (uint8_t)Buffer->getBufferStart()[ELF::EI_CLASS],
+                         (uint8_t)Buffer->getBufferStart()[ELF::EI_DATA]);
   error_code ec;
 
   if (Ident.first == ELF::ELFCLASS32 && Ident.second == ELF::ELFDATA2LSB) {
     DyldELFObject<support::little, false> *Obj =
-           new DyldELFObject<support::little, false>(InputBuffer, ec);
-    return new ELFObjectImage<support::little, false>(Obj);
+           new DyldELFObject<support::little, false>(Buffer->getMemBuffer(), ec);
+    return new ELFObjectImage<support::little, false>(Buffer, Obj);
   }
   else if (Ident.first == ELF::ELFCLASS32 && Ident.second == ELF::ELFDATA2MSB) {
     DyldELFObject<support::big, false> *Obj =
-           new DyldELFObject<support::big, false>(InputBuffer, ec);
-    return new ELFObjectImage<support::big, false>(Obj);
+           new DyldELFObject<support::big, false>(Buffer->getMemBuffer(), ec);
+    return new ELFObjectImage<support::big, false>(Buffer, Obj);
   }
   else if (Ident.first == ELF::ELFCLASS64 && Ident.second == ELF::ELFDATA2MSB) {
     DyldELFObject<support::big, true> *Obj =
-           new DyldELFObject<support::big, true>(InputBuffer, ec);
-    return new ELFObjectImage<support::big, true>(Obj);
+           new DyldELFObject<support::big, true>(Buffer->getMemBuffer(), ec);
+    return new ELFObjectImage<support::big, true>(Buffer, Obj);
   }
   else if (Ident.first == ELF::ELFCLASS64 && Ident.second == ELF::ELFDATA2LSB) {
     DyldELFObject<support::little, true> *Obj =
-           new DyldELFObject<support::little, true>(InputBuffer, ec);
-    return new ELFObjectImage<support::little, true>(Obj);
+           new DyldELFObject<support::little, true>(Buffer->getMemBuffer(), ec);
+    return new ELFObjectImage<support::little, true>(Buffer, Obj);
   }
   else
     llvm_unreachable("Unexpected ELF format");
 }
 
-void RuntimeDyldELF::handleObjectLoaded(ObjectImage *Obj)
-{
-  Obj->registerWithDebugger();
-  // Save the loaded object.  It will deregister itself when deleted
-  LoadedObject = Obj;
-}
-
 RuntimeDyldELF::~RuntimeDyldELF() {
-  if (LoadedObject)
-    delete LoadedObject;
 }
 
-void RuntimeDyldELF::resolveX86_64Relocation(uint8_t *LocalAddress,
-                                             uint64_t FinalAddress,
+void RuntimeDyldELF::resolveX86_64Relocation(const SectionEntry &Section,
+                                             uint64_t Offset,
                                              uint64_t Value,
                                              uint32_t Type,
                                              int64_t Addend) {
@@ -201,8 +197,10 @@ void RuntimeDyldELF::resolveX86_64Relocation(uint8_t *LocalAddress,
     llvm_unreachable("Relocation type not implemented yet!");
   break;
   case ELF::R_X86_64_64: {
-    uint64_t *Target = (uint64_t*)(LocalAddress);
+    uint64_t *Target = reinterpret_cast<uint64_t*>(Section.Address + Offset);
     *Target = Value + Addend;
+    DEBUG(dbgs() << "Writing " << format("%p", (Value + Addend))
+                 << " at " << format("%p\n",Target));
     break;
   }
   case ELF::R_X86_64_32:
@@ -212,37 +210,52 @@ void RuntimeDyldELF::resolveX86_64Relocation(uint8_t *LocalAddress,
            (Type == ELF::R_X86_64_32S && 
              ((int64_t)Value <= INT32_MAX && (int64_t)Value >= INT32_MIN)));
     uint32_t TruncatedAddr = (Value & 0xFFFFFFFF);
-    uint32_t *Target = reinterpret_cast<uint32_t*>(LocalAddress);
+    uint32_t *Target = reinterpret_cast<uint32_t*>(Section.Address + Offset);
     *Target = TruncatedAddr;
+    DEBUG(dbgs() << "Writing " << format("%p", TruncatedAddr)
+                 << " at " << format("%p\n",Target));
     break;
   }
   case ELF::R_X86_64_PC32: {
-    uint32_t *Placeholder = reinterpret_cast<uint32_t*>(LocalAddress);
+    // Get the placeholder value from the generated object since
+    // a previous relocation attempt may have overwritten the loaded version
+    uint32_t *Placeholder = reinterpret_cast<uint32_t*>(Section.ObjAddress
+                                                                   + Offset);
+    uint32_t *Target = reinterpret_cast<uint32_t*>(Section.Address + Offset);
+    uint64_t  FinalAddress = Section.LoadAddress + Offset;
     int64_t RealOffset = *Placeholder + Value + Addend - FinalAddress;
     assert(RealOffset <= INT32_MAX && RealOffset >= INT32_MIN);
     int32_t TruncOffset = (RealOffset & 0xFFFFFFFF);
-    *Placeholder = TruncOffset;
+    *Target = TruncOffset;
     break;
   }
   }
 }
 
-void RuntimeDyldELF::resolveX86Relocation(uint8_t *LocalAddress,
-                                          uint32_t FinalAddress,
+void RuntimeDyldELF::resolveX86Relocation(const SectionEntry &Section,
+                                          uint64_t Offset,
                                           uint32_t Value,
                                           uint32_t Type,
                                           int32_t Addend) {
   switch (Type) {
   case ELF::R_386_32: {
-    uint32_t *Target = (uint32_t*)(LocalAddress);
-    uint32_t Placeholder = *Target;
-    *Target = Placeholder + Value + Addend;
+    // Get the placeholder value from the generated object since
+    // a previous relocation attempt may have overwritten the loaded version
+    uint32_t *Placeholder = reinterpret_cast<uint32_t*>(Section.ObjAddress
+                                                                   + Offset);
+    uint32_t *Target = reinterpret_cast<uint32_t*>(Section.Address + Offset);
+    *Target = *Placeholder + Value + Addend;
     break;
   }
   case ELF::R_386_PC32: {
-    uint32_t *Placeholder = reinterpret_cast<uint32_t*>(LocalAddress);
+    // Get the placeholder value from the generated object since
+    // a previous relocation attempt may have overwritten the loaded version
+    uint32_t *Placeholder = reinterpret_cast<uint32_t*>(Section.ObjAddress
+                                                                   + Offset);
+    uint32_t *Target = reinterpret_cast<uint32_t*>(Section.Address + Offset);
+    uint32_t  FinalAddress = ((Section.LoadAddress + Offset) & 0xFFFFFFFF);
     uint32_t RealOffset = *Placeholder + Value + Addend - FinalAddress;
-    *Placeholder = RealOffset;
+    *Target = RealOffset;
     break;
     }
     default:
@@ -253,16 +266,18 @@ void RuntimeDyldELF::resolveX86Relocation(uint8_t *LocalAddress,
   }
 }
 
-void RuntimeDyldELF::resolveARMRelocation(uint8_t *LocalAddress,
-                                          uint32_t FinalAddress,
+void RuntimeDyldELF::resolveARMRelocation(const SectionEntry &Section,
+                                          uint64_t Offset,
                                           uint32_t Value,
                                           uint32_t Type,
                                           int32_t Addend) {
   // TODO: Add Thumb relocations.
-  uint32_t* TargetPtr = (uint32_t*)LocalAddress;
+  uint32_t* TargetPtr = (uint32_t*)(Section.Address + Offset);
+  uint32_t FinalAddress = ((Section.LoadAddress + Offset) & 0xFFFFFFFF);
   Value += Addend;
 
-  DEBUG(dbgs() << "resolveARMRelocation, LocalAddress: " << LocalAddress
+  DEBUG(dbgs() << "resolveARMRelocation, LocalAddress: "
+               << Section.Address + Offset
                << " FinalAddress: " << format("%p",FinalAddress)
                << " Value: " << format("%x",Value)
                << " Type: " << format("%x",Type)
@@ -273,14 +288,19 @@ void RuntimeDyldELF::resolveARMRelocation(uint8_t *LocalAddress,
   default:
     llvm_unreachable("Not implemented relocation type!");
 
-  // Just write 32bit value to relocation address
+  // Write a 32bit value to relocation address, taking into account the 
+  // implicit addend encoded in the target.
   case ELF::R_ARM_ABS32 :
-    *TargetPtr = Value;
+    *TargetPtr += Value;
     break;
 
   // Write first 16 bit of 32 bit value to the mov instruction.
   // Last 4 bit should be shifted.
   case ELF::R_ARM_MOVW_ABS_NC :
+    // We are not expecting any other addend in the relocation address.
+    // Using 0x000F0FFF because MOVW has its 16 bit immediate split into 2 
+    // non-contiguous fields.
+    assert((*TargetPtr & 0x000F0FFF) == 0);
     Value = Value & 0xFFFF;
     *TargetPtr |= Value & 0xFFF;
     *TargetPtr |= ((Value >> 12) & 0xF) << 16;
@@ -289,6 +309,9 @@ void RuntimeDyldELF::resolveARMRelocation(uint8_t *LocalAddress,
   // Write last 16 bit of 32 bit value to the mov instruction.
   // Last 4 bit should be shifted.
   case ELF::R_ARM_MOVT_ABS :
+    // We are not expecting any other addend in the relocation address.
+    // Use 0x000F0FFF for the same reason as R_ARM_MOVW_ABS_NC.
+    assert((*TargetPtr & 0x000F0FFF) == 0);
     Value = (Value >> 16) & 0xFFFF;
     *TargetPtr |= Value & 0xFFF;
     *TargetPtr |= ((Value >> 12) & 0xF) << 16;
@@ -306,26 +329,250 @@ void RuntimeDyldELF::resolveARMRelocation(uint8_t *LocalAddress,
   }
 }
 
-void RuntimeDyldELF::resolveRelocation(uint8_t *LocalAddress,
-                                       uint64_t FinalAddress,
+void RuntimeDyldELF::resolveMIPSRelocation(const SectionEntry &Section,
+                                           uint64_t Offset,
+                                           uint32_t Value,
+                                           uint32_t Type,
+                                           int32_t Addend) {
+  uint32_t* TargetPtr = (uint32_t*)(Section.Address + Offset);
+  Value += Addend;
+
+  DEBUG(dbgs() << "resolveMipselocation, LocalAddress: "
+               << Section.Address + Offset
+               << " FinalAddress: "
+               << format("%p",Section.LoadAddress + Offset)
+               << " Value: " << format("%x",Value)
+               << " Type: " << format("%x",Type)
+               << " Addend: " << format("%x",Addend)
+               << "\n");
+
+  switch(Type) {
+  default:
+    llvm_unreachable("Not implemented relocation type!");
+    break;
+  case ELF::R_MIPS_32:
+    *TargetPtr = Value + (*TargetPtr);
+    break;
+  case ELF::R_MIPS_26:
+    *TargetPtr = ((*TargetPtr) & 0xfc000000) | (( Value & 0x0fffffff) >> 2);
+    break;
+  case ELF::R_MIPS_HI16:
+    // Get the higher 16-bits. Also add 1 if bit 15 is 1.
+    Value += ((*TargetPtr) & 0x0000ffff) << 16;
+    *TargetPtr = ((*TargetPtr) & 0xffff0000) |
+                 (((Value + 0x8000) >> 16) & 0xffff);
+    break;
+   case ELF::R_MIPS_LO16:
+    Value += ((*TargetPtr) & 0x0000ffff);
+    *TargetPtr = ((*TargetPtr) & 0xffff0000) | (Value & 0xffff);
+    break;
+   }
+}
+
+// Return the .TOC. section address to R_PPC64_TOC relocations.
+uint64_t RuntimeDyldELF::findPPC64TOC() const {
+  // The TOC consists of sections .got, .toc, .tocbss, .plt in that
+  // order. The TOC starts where the first of these sections starts.
+  SectionList::const_iterator it = Sections.begin();
+  SectionList::const_iterator ite = Sections.end();
+  for (; it != ite; ++it) {
+    if (it->Name == ".got" ||
+        it->Name == ".toc" ||
+        it->Name == ".tocbss" ||
+        it->Name == ".plt")
+      break;
+  }
+  if (it == ite) {
+    // This may happen for
+    // * references to TOC base base (sym@toc, .odp relocation) without
+    // a .toc directive.
+    // In this case just use the first section (which is usually
+    // the .odp) since the code won't reference the .toc base
+    // directly.
+    it = Sections.begin();
+  }
+  assert (it != ite);
+  // Per the ppc64-elf-linux ABI, The TOC base is TOC value plus 0x8000
+  // thus permitting a full 64 Kbytes segment.
+  return it->LoadAddress + 0x8000;
+}
+
+// Returns the sections and offset associated with the ODP entry referenced
+// by Symbol.
+void RuntimeDyldELF::findOPDEntrySection(ObjectImage &Obj,
+                                         ObjSectionToIDMap &LocalSections,
+                                         RelocationValueRef &Rel) {
+  // Get the ELF symbol value (st_value) to compare with Relocation offset in
+  // .opd entries
+
+  error_code err;
+  for (section_iterator si = Obj.begin_sections(),
+     se = Obj.end_sections(); si != se; si.increment(err)) {
+    StringRef SectionName;
+    check(si->getName(SectionName));
+    if (SectionName != ".opd")
+      continue;
+
+    for (relocation_iterator i = si->begin_relocations(),
+         e = si->end_relocations(); i != e;) {
+      check(err);
+
+      // The R_PPC64_ADDR64 relocation indicates the first field
+      // of a .opd entry
+      uint64_t TypeFunc;
+      check(i->getType(TypeFunc));
+      if (TypeFunc != ELF::R_PPC64_ADDR64) {
+        i.increment(err);
+        continue;
+      }
+
+      SymbolRef TargetSymbol;
+      uint64_t TargetSymbolOffset;
+      int64_t TargetAdditionalInfo;
+      check(i->getSymbol(TargetSymbol));
+      check(i->getOffset(TargetSymbolOffset));
+      check(i->getAdditionalInfo(TargetAdditionalInfo));
+
+      i = i.increment(err);
+      if (i == e)
+        break;
+      check(err);
+
+      // Just check if following relocation is a R_PPC64_TOC
+      uint64_t TypeTOC;
+      check(i->getType(TypeTOC));
+      if (TypeTOC != ELF::R_PPC64_TOC)
+        continue;
+
+      // Finally compares the Symbol value and the target symbol offset
+      // to check if this .opd entry refers to the symbol the relocation
+      // points to.
+      if (Rel.Addend != (intptr_t)TargetSymbolOffset)
+        continue;
+
+      section_iterator tsi(Obj.end_sections());
+      check(TargetSymbol.getSection(tsi));
+      Rel.SectionID = findOrEmitSection(Obj, (*tsi), true, LocalSections);
+      Rel.Addend = (intptr_t)TargetAdditionalInfo;
+      return;
+    }
+  }
+  llvm_unreachable("Attempting to get address of ODP entry!");
+}
+
+// Relocation masks following the #lo(value), #hi(value), #higher(value),
+// and #highest(value) macros defined in section 4.5.1. Relocation Types
+// in PPC-elf64abi document.
+//
+static inline
+uint16_t applyPPClo (uint64_t value)
+{
+  return value & 0xffff;
+}
+
+static inline
+uint16_t applyPPChi (uint64_t value)
+{
+  return (value >> 16) & 0xffff;
+}
+
+static inline
+uint16_t applyPPChigher (uint64_t value)
+{
+  return (value >> 32) & 0xffff;
+}
+
+static inline
+uint16_t applyPPChighest (uint64_t value)
+{
+  return (value >> 48) & 0xffff;
+}
+
+void RuntimeDyldELF::resolvePPC64Relocation(const SectionEntry &Section,
+                                            uint64_t Offset,
+                                            uint64_t Value,
+                                            uint32_t Type,
+                                            int64_t Addend) {
+  uint8_t* LocalAddress = Section.Address + Offset;
+  switch (Type) {
+  default:
+    llvm_unreachable("Relocation type not implemented yet!");
+  break;
+  case ELF::R_PPC64_ADDR16_LO :
+    writeInt16BE(LocalAddress, applyPPClo (Value + Addend));
+    break;
+  case ELF::R_PPC64_ADDR16_HI :
+    writeInt16BE(LocalAddress, applyPPChi (Value + Addend));
+    break;
+  case ELF::R_PPC64_ADDR16_HIGHER :
+    writeInt16BE(LocalAddress, applyPPChigher (Value + Addend));
+    break;
+  case ELF::R_PPC64_ADDR16_HIGHEST :
+    writeInt16BE(LocalAddress, applyPPChighest (Value + Addend));
+    break;
+  case ELF::R_PPC64_ADDR14 : {
+    assert(((Value + Addend) & 3) == 0);
+    // Preserve the AA/LK bits in the branch instruction
+    uint8_t aalk = *(LocalAddress+3);
+    writeInt16BE(LocalAddress + 2, (aalk & 3) | ((Value + Addend) & 0xfffc));
+  } break;
+  case ELF::R_PPC64_REL24 : {
+    uint64_t FinalAddress = (Section.LoadAddress + Offset);
+    int32_t delta = static_cast<int32_t>(Value - FinalAddress + Addend);
+    if (SignExtend32<24>(delta) != delta)
+      llvm_unreachable("Relocation R_PPC64_REL24 overflow");
+    // Generates a 'bl <address>' instruction
+    writeInt32BE(LocalAddress, 0x48000001 | (delta & 0x03FFFFFC));
+  } break;
+  case ELF::R_PPC64_ADDR64 :
+    writeInt64BE(LocalAddress, Value + Addend);
+    break;
+  case ELF::R_PPC64_TOC :
+    writeInt64BE(LocalAddress, findPPC64TOC());
+    break;
+  case ELF::R_PPC64_TOC16 : {
+    uint64_t TOCStart = findPPC64TOC();
+    Value = applyPPClo((Value + Addend) - TOCStart);
+    writeInt16BE(LocalAddress, applyPPClo(Value));
+  } break;
+  case ELF::R_PPC64_TOC16_DS : {
+    uint64_t TOCStart = findPPC64TOC();
+    Value = ((Value + Addend) - TOCStart);
+    writeInt16BE(LocalAddress, applyPPClo(Value));
+  } break;
+  }
+}
+
+
+void RuntimeDyldELF::resolveRelocation(const SectionEntry &Section,
+                                       uint64_t Offset,
                                        uint64_t Value,
                                        uint32_t Type,
                                        int64_t Addend) {
   switch (Arch) {
   case Triple::x86_64:
-    resolveX86_64Relocation(LocalAddress, FinalAddress, Value, Type, Addend);
+    resolveX86_64Relocation(Section, Offset, Value, Type, Addend);
     break;
   case Triple::x86:
-    resolveX86Relocation(LocalAddress, (uint32_t)(FinalAddress & 0xffffffffL),
+    resolveX86Relocation(Section, Offset,
                          (uint32_t)(Value & 0xffffffffL), Type,
                          (uint32_t)(Addend & 0xffffffffL));
     break;
   case Triple::arm:    // Fall through.
   case Triple::thumb:
-    resolveARMRelocation(LocalAddress, (uint32_t)(FinalAddress & 0xffffffffL),
+    resolveARMRelocation(Section, Offset,
                          (uint32_t)(Value & 0xffffffffL), Type,
                          (uint32_t)(Addend & 0xffffffffL));
     break;
+  case Triple::mips:    // Fall through.
+  case Triple::mipsel:
+    resolveMIPSRelocation(Section, Offset,
+                          (uint32_t)(Value & 0xffffffffL), Type,
+                          (uint32_t)(Addend & 0xffffffffL));
+    break;
+  case Triple::ppc64:
+    resolvePPC64Relocation(Section, Offset, Value, Type, Addend);
+    break;
   default: llvm_unreachable("Unsupported CPU type!");
   }
 }
@@ -350,6 +597,8 @@ void RuntimeDyldELF::processRelocationRef(const ObjRelocationInfo &Rel,
   RelocationValueRef Value;
   // First search for the symbol in the local symbol table
   SymbolTableMap::const_iterator lsi = Symbols.find(TargetName.data());
+  SymbolRef::Type SymType;
+  Symbol.getType(SymType);
   if (lsi != Symbols.end()) {
     Value.SectionID = lsi->second.first;
     Value.Addend = lsi->second.second;
@@ -361,8 +610,6 @@ void RuntimeDyldELF::processRelocationRef(const ObjRelocationInfo &Rel,
       Value.SectionID = gsi->second.first;
       Value.Addend = gsi->second.second;
     } else {
-      SymbolRef::Type SymType;
-      Symbol.getType(SymType);
       switch (SymType) {
         case SymbolRef::ST_Debug: {
           // TODO: Now ELF SymbolRef::ST_Debug = STT_SECTION, it's not obviously
@@ -373,7 +620,13 @@ void RuntimeDyldELF::processRelocationRef(const ObjRelocationInfo &Rel,
           if (si == Obj.end_sections())
             llvm_unreachable("Symbol section not found, bad object file format!");
           DEBUG(dbgs() << "\t\tThis is section symbol\n");
-          Value.SectionID = findOrEmitSection(Obj, (*si), true, ObjSectionToID);
+          // Default to 'true' in case isText fails (though it never does).
+          bool isCode = true;
+          si->isText(isCode);
+          Value.SectionID = findOrEmitSection(Obj, 
+                                              (*si), 
+                                              isCode, 
+                                              ObjSectionToID);
           Value.Addend = Addend;
           break;
         }
@@ -398,13 +651,12 @@ void RuntimeDyldELF::processRelocationRef(const ObjRelocationInfo &Rel,
     // This is an ARM branch relocation, need to use a stub function.
     DEBUG(dbgs() << "\t\tThis is an ARM branch relocation.");
     SectionEntry &Section = Sections[Rel.SectionID];
-    uint8_t *Target = Section.Address + Rel.Offset;
 
-    //  Look up for existing stub.
+    // Look for an existing stub.
     StubMap::const_iterator i = Stubs.find(Value);
     if (i != Stubs.end()) {
-      resolveRelocation(Target, (uint64_t)Target, (uint64_t)Section.Address +
-                        i->second, RelType, 0);
+        resolveRelocation(Section, Rel.Offset,
+                          (uint64_t)Section.Address + i->second, RelType, 0);
       DEBUG(dbgs() << " Stub function found\n");
     } else {
       // Create a new stub function.
@@ -419,10 +671,145 @@ void RuntimeDyldELF::processRelocationRef(const ObjRelocationInfo &Rel,
       else
         addRelocationForSection(RE, Value.SectionID);
 
-      resolveRelocation(Target, (uint64_t)Target, (uint64_t)Section.Address +
-                        Section.StubOffset, RelType, 0);
+      resolveRelocation(Section, Rel.Offset,
+                        (uint64_t)Section.Address + Section.StubOffset,
+                        RelType, 0);
       Section.StubOffset += getMaxStubSize();
     }
+  } else if (Arch == Triple::mipsel && RelType == ELF::R_MIPS_26) {
+    // This is an Mips branch relocation, need to use a stub function.
+    DEBUG(dbgs() << "\t\tThis is a Mips branch relocation.");
+    SectionEntry &Section = Sections[Rel.SectionID];
+    uint8_t *Target = Section.Address + Rel.Offset;
+    uint32_t *TargetAddress = (uint32_t *)Target;
+
+    // Extract the addend from the instruction.
+    uint32_t Addend = ((*TargetAddress) & 0x03ffffff) << 2;
+
+    Value.Addend += Addend;
+
+    //  Look up for existing stub.
+    StubMap::const_iterator i = Stubs.find(Value);
+    if (i != Stubs.end()) {
+      resolveRelocation(Section, Rel.Offset,
+                        (uint64_t)Section.Address + i->second, RelType, 0);
+      DEBUG(dbgs() << " Stub function found\n");
+    } else {
+      // Create a new stub function.
+      DEBUG(dbgs() << " Create a new stub function\n");
+      Stubs[Value] = Section.StubOffset;
+      uint8_t *StubTargetAddr = createStubFunction(Section.Address +
+                                                   Section.StubOffset);
+
+      // Creating Hi and Lo relocations for the filled stub instructions.
+      RelocationEntry REHi(Rel.SectionID,
+                           StubTargetAddr - Section.Address,
+                           ELF::R_MIPS_HI16, Value.Addend);
+      RelocationEntry RELo(Rel.SectionID,
+                           StubTargetAddr - Section.Address + 4,
+                           ELF::R_MIPS_LO16, Value.Addend);
+
+      if (Value.SymbolName) {
+        addRelocationForSymbol(REHi, Value.SymbolName);
+        addRelocationForSymbol(RELo, Value.SymbolName);
+      } else {
+        addRelocationForSection(REHi, Value.SectionID);
+        addRelocationForSection(RELo, Value.SectionID);
+      }
+
+      resolveRelocation(Section, Rel.Offset,
+                        (uint64_t)Section.Address + Section.StubOffset,
+                        RelType, 0);
+      Section.StubOffset += getMaxStubSize();
+    }
+  } else if (Arch == Triple::ppc64) {
+    if (RelType == ELF::R_PPC64_REL24) {
+      // A PPC branch relocation will need a stub function if the target is
+      // an external symbol (Symbol::ST_Unknown) or if the target address
+      // is not within the signed 24-bits branch address.
+      SectionEntry &Section = Sections[Rel.SectionID];
+      uint8_t *Target = Section.Address + Rel.Offset;
+      bool RangeOverflow = false;
+      if (SymType != SymbolRef::ST_Unknown) {
+        // A function call may points to the .opd entry, so the final symbol value
+        // in calculated based in the relocation values in .opd section.
+        findOPDEntrySection(Obj, ObjSectionToID, Value);
+        uint8_t *RelocTarget = Sections[Value.SectionID].Address + Value.Addend;
+        int32_t delta = static_cast<int32_t>(Target - RelocTarget);
+        // If it is within 24-bits branch range, just set the branch target
+        if (SignExtend32<24>(delta) == delta) {
+          RelocationEntry RE(Rel.SectionID, Rel.Offset, RelType, Value.Addend);
+          if (Value.SymbolName)
+            addRelocationForSymbol(RE, Value.SymbolName);
+          else
+            addRelocationForSection(RE, Value.SectionID);
+        } else {
+          RangeOverflow = true;
+        }
+      }
+      if (SymType == SymbolRef::ST_Unknown || RangeOverflow == true) {
+        // It is an external symbol (SymbolRef::ST_Unknown) or within a range
+        // larger than 24-bits.
+        StubMap::const_iterator i = Stubs.find(Value);
+        if (i != Stubs.end()) {
+          // Symbol function stub already created, just relocate to it
+          resolveRelocation(Section, Rel.Offset,
+                            (uint64_t)Section.Address + i->second, RelType, 0);
+          DEBUG(dbgs() << " Stub function found\n");
+        } else {
+          // Create a new stub function.
+          DEBUG(dbgs() << " Create a new stub function\n");
+          Stubs[Value] = Section.StubOffset;
+          uint8_t *StubTargetAddr = createStubFunction(Section.Address +
+                                                       Section.StubOffset);
+          RelocationEntry RE(Rel.SectionID, StubTargetAddr - Section.Address,
+                             ELF::R_PPC64_ADDR64, Value.Addend);
+
+          // Generates the 64-bits address loads as exemplified in section
+          // 4.5.1 in PPC64 ELF ABI.
+          RelocationEntry REhst(Rel.SectionID,
+                                StubTargetAddr - Section.Address + 2,
+                                ELF::R_PPC64_ADDR16_HIGHEST, Value.Addend);
+          RelocationEntry REhr(Rel.SectionID,
+                               StubTargetAddr - Section.Address + 6,
+                               ELF::R_PPC64_ADDR16_HIGHER, Value.Addend);
+          RelocationEntry REh(Rel.SectionID,
+                              StubTargetAddr - Section.Address + 14,
+                              ELF::R_PPC64_ADDR16_HI, Value.Addend);
+          RelocationEntry REl(Rel.SectionID,
+                              StubTargetAddr - Section.Address + 18,
+                              ELF::R_PPC64_ADDR16_LO, Value.Addend);
+
+          if (Value.SymbolName) {
+            addRelocationForSymbol(REhst, Value.SymbolName);
+            addRelocationForSymbol(REhr,  Value.SymbolName);
+            addRelocationForSymbol(REh,   Value.SymbolName);
+            addRelocationForSymbol(REl,   Value.SymbolName);
+          } else {
+            addRelocationForSection(REhst, Value.SectionID);
+            addRelocationForSection(REhr,  Value.SectionID);
+            addRelocationForSection(REh,   Value.SectionID);
+            addRelocationForSection(REl,   Value.SectionID);
+          }
+
+          resolveRelocation(Section, Rel.Offset,
+                            (uint64_t)Section.Address + Section.StubOffset,
+                            RelType, 0);
+          if (SymType == SymbolRef::ST_Unknown)
+            // Restore the TOC for external calls
+            writeInt32BE(Target+4, 0xE8410028); // ld r2,40(r1)
+          Section.StubOffset += getMaxStubSize();
+        }
+      }
+    } else {
+      RelocationEntry RE(Rel.SectionID, Rel.Offset, RelType, Value.Addend);
+      // Extra check to avoid relocation againt empty symbols (usually
+      // the R_PPC64_TOC).
+      if (Value.SymbolName && !TargetName.empty())
+        addRelocationForSymbol(RE, Value.SymbolName);
+      else
+        addRelocationForSection(RE, Value.SectionID);
+    }
   } else {
     RelocationEntry RE(Rel.SectionID, Rel.Offset, RelType, Value.Addend);
     if (Value.SymbolName)
@@ -432,8 +819,16 @@ void RuntimeDyldELF::processRelocationRef(const ObjRelocationInfo &Rel,
   }
 }
 
-bool RuntimeDyldELF::isCompatibleFormat(const MemoryBuffer *InputBuffer) const {
-  StringRef Magic = InputBuffer->getBuffer().slice(0, ELF::EI_NIDENT);
-  return (memcmp(Magic.data(), ELF::ElfMagic, strlen(ELF::ElfMagic))) == 0;
+unsigned RuntimeDyldELF::getCommonSymbolAlignment(const SymbolRef &Sym) {
+  // In ELF, the value of an SHN_COMMON symbol is its alignment requirement.
+  uint64_t Align;
+  Check(Sym.getValue(Align));
+  return Align;
+}
+
+bool RuntimeDyldELF::isCompatibleFormat(const ObjectBuffer *Buffer) const {
+  if (Buffer->getBufferSize() < strlen(ELF::ElfMagic))
+    return false;
+  return (memcmp(Buffer->getBufferStart(), ELF::ElfMagic, strlen(ELF::ElfMagic))) == 0;
 }
 } // namespace llvm
diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h
index e413f78..07e704b 100644
--- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h
+++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h
@@ -18,32 +18,52 @@
 
 using namespace llvm;
 
-
 namespace llvm {
+
+namespace {
+  // Helper for extensive error checking in debug builds.
+  error_code Check(error_code Err) {
+    if (Err) {
+      report_fatal_error(Err.message());
+    }
+    return Err;
+  }
+} // end anonymous namespace
+
 class RuntimeDyldELF : public RuntimeDyldImpl {
 protected:
-  ObjectImage *LoadedObject;
-
-  void resolveX86_64Relocation(uint8_t *LocalAddress,
-                               uint64_t FinalAddress,
+  void resolveX86_64Relocation(const SectionEntry &Section,
+                               uint64_t Offset,
                                uint64_t Value,
                                uint32_t Type,
                                int64_t Addend);
 
-  void resolveX86Relocation(uint8_t *LocalAddress,
-                            uint32_t FinalAddress,
+  void resolveX86Relocation(const SectionEntry &Section,
+                            uint64_t Offset,
                             uint32_t Value,
                             uint32_t Type,
                             int32_t Addend);
 
-  void resolveARMRelocation(uint8_t *LocalAddress,
-                            uint32_t FinalAddress,
+  void resolveARMRelocation(const SectionEntry &Section,
+                            uint64_t Offset,
                             uint32_t Value,
                             uint32_t Type,
                             int32_t Addend);
 
-  virtual void resolveRelocation(uint8_t *LocalAddress,
-                                 uint64_t FinalAddress,
+  void resolveMIPSRelocation(const SectionEntry &Section,
+                             uint64_t Offset,
+                             uint32_t Value,
+                             uint32_t Type,
+                             int32_t Addend);
+
+  void resolvePPC64Relocation(const SectionEntry &Section,
+                              uint64_t Offset,
+                              uint64_t Value,
+                              uint32_t Type,
+                              int64_t Addend);
+
+  virtual void resolveRelocation(const SectionEntry &Section,
+                                 uint64_t Offset,
                                  uint64_t Value,
                                  uint32_t Type,
                                  int64_t Addend);
@@ -54,16 +74,22 @@ protected:
                                     const SymbolTableMap &Symbols,
                                     StubMap &Stubs);
 
-  virtual ObjectImage *createObjectImage(const MemoryBuffer *InputBuffer);
-  virtual void handleObjectLoaded(ObjectImage *Obj);
+  unsigned getCommonSymbolAlignment(const SymbolRef &Sym);
+
+  virtual ObjectImage *createObjectImage(ObjectBuffer *InputBuffer);
+
+  uint64_t findPPC64TOC() const;
+  void findOPDEntrySection(ObjectImage &Obj,
+                           ObjSectionToIDMap &LocalSections,
+                           RelocationValueRef &Rel);
 
 public:
   RuntimeDyldELF(RTDyldMemoryManager *mm)
-      : RuntimeDyldImpl(mm), LoadedObject(0) {}
+      : RuntimeDyldImpl(mm) {}
 
   virtual ~RuntimeDyldELF();
 
-  bool isCompatibleFormat(const MemoryBuffer *InputBuffer) const;
+  bool isCompatibleFormat(const ObjectBuffer *Buffer) const;
 };
 
 } // end namespace llvm
diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h
index c38ca69..829fd6c 100644
--- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h
+++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h
@@ -14,8 +14,8 @@
 #ifndef LLVM_RUNTIME_DYLD_IMPL_H
 #define LLVM_RUNTIME_DYLD_IMPL_H
 
-#include "ObjectImage.h"
 #include "llvm/ExecutionEngine/RuntimeDyld.h"
+#include "llvm/ExecutionEngine/ObjectImage.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringMap.h"
@@ -24,6 +24,8 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/Format.h"
+#include "llvm/Support/Host.h"
+#include "llvm/Support/SwapByteOrder.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/system_error.h"
 #include <map>
@@ -33,7 +35,7 @@ using namespace llvm::object;
 
 namespace llvm {
 
-class MemoryBuffer;
+class ObjectBuffer;
 class Twine;
 
 
@@ -41,6 +43,9 @@ class Twine;
 /// linker.
 class SectionEntry {
 public:
+  /// Name - section name.
+  StringRef Name;
+
   /// Address - address in the linker's memory where the section resides.
   uint8_t *Address;
 
@@ -61,9 +66,9 @@ public:
   /// for calculating relocations in some object formats (like MachO).
   uintptr_t ObjAddress;
 
-  SectionEntry(uint8_t *address, size_t size, uintptr_t stubOffset,
-               uintptr_t objAddress)
-    : Address(address), Size(size), LoadAddress((uintptr_t)address),
+  SectionEntry(StringRef name, uint8_t *address, size_t size,
+	       uintptr_t stubOffset, uintptr_t objAddress)
+    : Name(name), Address(address), Size(size), LoadAddress((uintptr_t)address),
       StubOffset(stubOffset), ObjAddress(objAddress) {}
 };
 
@@ -135,8 +140,10 @@ protected:
   typedef StringMap<SymbolLoc> SymbolTableMap;
   SymbolTableMap GlobalSymbolTable;
 
-  // Keep a map of common symbols to their sizes
-  typedef std::map<SymbolRef, unsigned> CommonSymbolMap;
+  // Pair representing the size and alignment requirement for a common symbol.
+  typedef std::pair<unsigned, unsigned> CommonSymbolInfo;
+  // Keep a map of common symbols to their info pairs
+  typedef std::map<SymbolRef, CommonSymbolInfo> CommonSymbolMap;
 
   // For each symbol, keep a list of relocations based on it. Anytime
   // its address is reassigned (the JIT re-compiled the function, e.g.),
@@ -161,6 +168,10 @@ protected:
   inline unsigned getMaxStubSize() {
     if (Arch == Triple::arm || Arch == Triple::thumb)
       return 8; // 32-bit instruction and 32-bit address
+    else if (Arch == Triple::mipsel)
+      return 16;
+    else if (Arch == Triple::ppc64)
+      return 44;
     else
       return 0;
   }
@@ -175,10 +186,50 @@ protected:
     return true;
   }
 
+  uint64_t getSectionLoadAddress(unsigned SectionID) {
+    return Sections[SectionID].LoadAddress;
+  }
+
   uint8_t *getSectionAddress(unsigned SectionID) {
     return (uint8_t*)Sections[SectionID].Address;
   }
 
+  // Subclasses can override this method to get the alignment requirement of
+  // a common symbol. Returns no alignment requirement if not implemented.
+  virtual unsigned getCommonSymbolAlignment(const SymbolRef &Sym) {
+    return 0;
+  }
+
+
+  void writeInt16BE(uint8_t *Addr, uint16_t Value) {
+    if (sys::isLittleEndianHost())
+      Value = sys::SwapByteOrder(Value);
+    *Addr     = (Value >> 8) & 0xFF;
+    *(Addr+1) = Value & 0xFF;
+  }
+
+  void writeInt32BE(uint8_t *Addr, uint32_t Value) {
+    if (sys::isLittleEndianHost())
+      Value = sys::SwapByteOrder(Value);
+    *Addr     = (Value >> 24) & 0xFF;
+    *(Addr+1) = (Value >> 16) & 0xFF;
+    *(Addr+2) = (Value >> 8) & 0xFF;
+    *(Addr+3) = Value & 0xFF;
+  }
+
+  void writeInt64BE(uint8_t *Addr, uint64_t Value) {
+    if (sys::isLittleEndianHost())
+      Value = sys::SwapByteOrder(Value);
+    *Addr     = (Value >> 56) & 0xFF;
+    *(Addr+1) = (Value >> 48) & 0xFF;
+    *(Addr+2) = (Value >> 40) & 0xFF;
+    *(Addr+3) = (Value >> 32) & 0xFF;
+    *(Addr+4) = (Value >> 24) & 0xFF;
+    *(Addr+5) = (Value >> 16) & 0xFF;
+    *(Addr+6) = (Value >> 8) & 0xFF;
+    *(Addr+7) = Value & 0xFF;
+  }
+
   /// \brief Given the common symbols discovered in the object file, emit a
   /// new section for them and update the symbol mappings in the object and
   /// symbol table.
@@ -221,13 +272,14 @@ protected:
   void resolveRelocationEntry(const RelocationEntry &RE, uint64_t Value);
 
   /// \brief A object file specific relocation resolver
-  /// \param Address Address to apply the relocation action
+  /// \param Section The section where the relocation is being applied
+  /// \param Offset The offset into the section for this relocation
   /// \param Value Target symbol address to apply the relocation action
   /// \param Type object file specific relocation type
   /// \param Addend A constant addend used to compute the value to be stored
   ///        into the relocatable field
-  virtual void resolveRelocation(uint8_t *LocalAddress,
-                                 uint64_t FinalAddress,
+  virtual void resolveRelocation(const SectionEntry &Section,
+                                 uint64_t Offset,
                                  uint64_t Value,
                                  uint32_t Type,
                                  int64_t Addend) = 0;
@@ -242,19 +294,13 @@ protected:
 
   /// \brief Resolve relocations to external symbols.
   void resolveExternalSymbols();
-  virtual ObjectImage *createObjectImage(const MemoryBuffer *InputBuffer);
-  virtual void handleObjectLoaded(ObjectImage *Obj)
-  {
-    // Subclasses may choose to retain this image if they have a use for it
-    delete Obj;
-  }
-
+  virtual ObjectImage *createObjectImage(ObjectBuffer *InputBuffer);
 public:
   RuntimeDyldImpl(RTDyldMemoryManager *mm) : MemMgr(mm), HasError(false) {}
 
   virtual ~RuntimeDyldImpl();
 
-  bool loadObject(const MemoryBuffer *InputBuffer);
+  ObjectImage *loadObject(ObjectBuffer *InputBuffer);
 
   void *getSymbolAddress(StringRef Name) {
     // FIXME: Just look up as a function for now. Overly simple of course.
@@ -265,11 +311,20 @@ public:
     return getSectionAddress(Loc.first) + Loc.second;
   }
 
+  uint64_t getSymbolLoadAddress(StringRef Name) {
+    // FIXME: Just look up as a function for now. Overly simple of course.
+    // Work in progress.
+    if (GlobalSymbolTable.find(Name) == GlobalSymbolTable.end())
+      return 0;
+    SymbolLoc Loc = GlobalSymbolTable.lookup(Name);
+    return getSectionLoadAddress(Loc.first) + Loc.second;
+  }
+
   void resolveRelocations();
 
   void reassignSectionAddress(unsigned SectionID, uint64_t Addr);
 
-  void mapSectionAddress(void *LocalAddress, uint64_t TargetAddress);
+  void mapSectionAddress(const void *LocalAddress, uint64_t TargetAddress);
 
   // Is the linker in an error state?
   bool hasError() { return HasError; }
@@ -280,8 +335,7 @@ public:
   // Get the error message.
   StringRef getErrorString() { return ErrorStr; }
 
-  virtual bool isCompatibleFormat(const MemoryBuffer *InputBuffer) const = 0;
-
+  virtual bool isCompatibleFormat(const ObjectBuffer *Buffer) const = 0;
 };
 
 } // end namespace llvm
diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp
index 0e3a9d4..987c0c3 100644
--- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp
+++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp
@@ -21,11 +21,13 @@ using namespace llvm::object;
 
 namespace llvm {
 
-void RuntimeDyldMachO::resolveRelocation(uint8_t *LocalAddress,
-                                         uint64_t FinalAddress,
+void RuntimeDyldMachO::resolveRelocation(const SectionEntry &Section,
+                                         uint64_t Offset,
                                          uint64_t Value,
                                          uint32_t Type,
                                          int64_t Addend) {
+  uint8_t *LocalAddress = Section.Address + Offset;
+  uint64_t FinalAddress = Section.LoadAddress + Offset;
   bool isPCRel = (Type >> 24) & 1;
   unsigned MachoType = (Type >> 28) & 0xf;
   unsigned Size = 1 << ((Type >> 25) & 3);
@@ -57,7 +59,7 @@ void RuntimeDyldMachO::resolveRelocation(uint8_t *LocalAddress,
                           FinalAddress,
                           (uintptr_t)Value,
                           isPCRel,
-                          Type,
+                          MachoType,
                           Size,
                           Addend);
     break;
@@ -211,7 +213,6 @@ void RuntimeDyldMachO::processRelocationRef(const ObjRelocationInfo &Rel,
   uint32_t RelType = (uint32_t) (Rel.Type & 0xffffffffL);
   RelocationValueRef Value;
   SectionEntry &Section = Sections[Rel.SectionID];
-  uint8_t *Target = Section.Address + Rel.Offset;
 
   bool isExtern = (RelType >> 27) & 1;
   if (isExtern) {
@@ -246,7 +247,12 @@ void RuntimeDyldMachO::processRelocationRef(const ObjRelocationInfo &Rel,
     }
     assert(si != se && "No section containing relocation!");
     Value.SectionID = findOrEmitSection(Obj, *si, true, ObjSectionToID);
-    Value.Addend = *(const intptr_t *)Target;
+    Value.Addend = 0;
+    // FIXME: The size and type of the relocation determines if we can
+    // encode an Addend in the target location itself, and if so, how many
+    // bytes we should read in order to get it. We don't yet support doing
+    // that, and just assuming it's sizeof(intptr_t) is blatantly wrong.
+    //Value.Addend = *(const intptr_t *)Target;
     if (Value.Addend) {
       // The MachO addend is an offset from the current section.  We need it
       // to be an offset from the destination section
@@ -254,13 +260,13 @@ void RuntimeDyldMachO::processRelocationRef(const ObjRelocationInfo &Rel,
     }
   }
 
-  if (Arch == Triple::arm && RelType == macho::RIT_ARM_Branch24Bit) {
+  if (Arch == Triple::arm && (RelType & 0xf) == macho::RIT_ARM_Branch24Bit) {
     // This is an ARM branch relocation, need to use a stub function.
 
     //  Look up for existing stub.
     StubMap::const_iterator i = Stubs.find(Value);
     if (i != Stubs.end())
-      resolveRelocation(Target, (uint64_t)Target,
+      resolveRelocation(Section, Rel.Offset,
                         (uint64_t)Section.Address + i->second,
                         RelType, 0);
     else {
@@ -274,7 +280,7 @@ void RuntimeDyldMachO::processRelocationRef(const ObjRelocationInfo &Rel,
         addRelocationForSymbol(RE, Value.SymbolName);
       else
         addRelocationForSection(RE, Value.SectionID);
-      resolveRelocation(Target, (uint64_t)Target,
+      resolveRelocation(Section, Rel.Offset,
                         (uint64_t)Section.Address + Section.StubOffset,
                         RelType, 0);
       Section.StubOffset += getMaxStubSize();
@@ -290,8 +296,10 @@ void RuntimeDyldMachO::processRelocationRef(const ObjRelocationInfo &Rel,
 
 
 bool RuntimeDyldMachO::isCompatibleFormat(
-        const MemoryBuffer *InputBuffer) const {
-  StringRef Magic = InputBuffer->getBuffer().slice(0, 4);
+        const ObjectBuffer *InputBuffer) const {
+  if (InputBuffer->getBufferSize() < 4)
+    return false;
+  StringRef Magic(InputBuffer->getBufferStart(), 4);
   if (Magic == "\xFE\xED\xFA\xCE") return true;
   if (Magic == "\xCE\xFA\xED\xFE") return true;
   if (Magic == "\xFE\xED\xFA\xCF") return true;
diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h
index 707664c..fe3539d 100644
--- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h
+++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h
@@ -55,15 +55,15 @@ protected:
                                     StubMap &Stubs);
 
 public:
-  virtual void resolveRelocation(uint8_t *LocalAddress,
-                                 uint64_t FinalAddress,
+  virtual void resolveRelocation(const SectionEntry &Section,
+                                 uint64_t Offset,
                                  uint64_t Value,
                                  uint32_t Type,
                                  int64_t Addend);
 
   RuntimeDyldMachO(RTDyldMemoryManager *mm) : RuntimeDyldImpl(mm) {}
 
-  bool isCompatibleFormat(const MemoryBuffer *InputBuffer) const;
+  bool isCompatibleFormat(const ObjectBuffer *Buffer) const;
 };
 
 } // end namespace llvm
diff --git a/lib/ExecutionEngine/TargetSelect.cpp b/lib/ExecutionEngine/TargetSelect.cpp
index 7cdd669..8b6104f 100644
--- a/lib/ExecutionEngine/TargetSelect.cpp
+++ b/lib/ExecutionEngine/TargetSelect.cpp
@@ -26,7 +26,14 @@
 using namespace llvm;
 
 TargetMachine *EngineBuilder::selectTarget() {
-  Triple TT(LLVM_HOSTTRIPLE);
+  Triple TT;
+
+  // MCJIT can generate code for remote targets, but the old JIT and Interpreter
+  // must use the host architecture.
+  if (UseMCJIT && WhichEngine != EngineKind::Interpreter && M)
+    TT.setTriple(M->getTargetTriple());
+  else
+    TT.setTriple(LLVM_HOSTTRIPLE);
   return selectTarget(TT, MArch, MCPU, MAttrs);
 }
 
diff --git a/lib/MC/ELFObjectWriter.cpp b/lib/MC/ELFObjectWriter.cpp
index 7203b9a..eda0623 100644
--- a/lib/MC/ELFObjectWriter.cpp
+++ b/lib/MC/ELFObjectWriter.cpp
@@ -133,6 +133,11 @@ class ELFObjectWriter : public MCObjectWriter {
                                    bool IsPCRel) const {
       return TargetObjectWriter->ExplicitRelSym(Asm, Target, F, Fixup, IsPCRel);
     }
+    const MCSymbol *undefinedExplicitRelSym(const MCValue &Target,
+                                            const MCFixup &Fixup,
+                                            bool IsPCRel) const {
+      return TargetObjectWriter->undefinedExplicitRelSym(Target, Fixup, IsPCRel);
+    }
 
     bool is64Bit() const { return TargetObjectWriter->is64Bit(); }
     bool hasRelocationAddend() const {
@@ -270,9 +275,10 @@ class ELFObjectWriter : public MCObjectWriter {
 
     /// ComputeSymbolTable - Compute the symbol table data
     ///
-    /// \param StringTable [out] - The string table data.
-    /// \param StringIndexMap [out] - Map from symbol names to offsets in the
-    /// string table.
+    /// \param Asm - The assembler.
+    /// \param SectionIndexMap - Maps a section to its index.
+    /// \param RevGroupMap - Maps a signature symbol to the group section.
+    /// \param NumRegularSections - Number of non-relocation sections.
     void ComputeSymbolTable(MCAssembler &Asm,
                             const SectionIndexMapTy &SectionIndexMap,
                             RevGroupMapTy RevGroupMap,
@@ -638,7 +644,7 @@ const MCSymbol *ELFObjectWriter::SymbolToReloc(const MCAssembler &Asm,
   if (ASymbol.isUndefined()) {
     if (Renamed)
       return Renamed;
-    return &ASymbol;
+    return undefinedExplicitRelSym(Target, Fixup, IsPCRel);
   }
 
   if (SD.isExternal()) {
@@ -720,10 +726,13 @@ void ELFObjectWriter::RecordRelocation(const MCAssembler &Asm,
       MCSymbolData &SD = Asm.getSymbolData(ASymbol);
       MCFragment *F = SD.getFragment();
 
-      Index = F->getParent()->getOrdinal() + 1;
-
-      // Offset of the symbol in the section
-      Value += Layout.getSymbolOffset(&SD);
+      if (F) {
+        Index = F->getParent()->getOrdinal() + 1;
+        // Offset of the symbol in the section
+        Value += Layout.getSymbolOffset(&SD);
+      } else {
+        Index = 0;
+      }
     } else {
       if (Asm.getSymbolData(Symbol).getFlags() & ELF_Other_Weakref)
         WeakrefUsedInReloc.insert(RelocSymbol);
@@ -732,8 +741,7 @@ void ELFObjectWriter::RecordRelocation(const MCAssembler &Asm,
       Index = -1;
     }
     Addend = Value;
-    // Compensate for the addend on i386.
-    if (is64Bit())
+    if (hasRelocationAddend())
       Value = 0;
   }
 
diff --git a/lib/MC/MCAsmBackend.cpp b/lib/MC/MCAsmBackend.cpp
index 2e447b0..53960e7 100644
--- a/lib/MC/MCAsmBackend.cpp
+++ b/lib/MC/MCAsmBackend.cpp
@@ -12,12 +12,9 @@
 using namespace llvm;
 
 MCAsmBackend::MCAsmBackend()
-  : HasReliableSymbolDifference(false)
-{
-}
+  : HasReliableSymbolDifference(false), HasDataInCodeSupport(false) {}
 
-MCAsmBackend::~MCAsmBackend() {
-}
+MCAsmBackend::~MCAsmBackend() {}
 
 const MCFixupKindInfo &
 MCAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
diff --git a/lib/MC/MCAsmInfo.cpp b/lib/MC/MCAsmInfo.cpp
index 8da2e0e..7ea0f3b 100644
--- a/lib/MC/MCAsmInfo.cpp
+++ b/lib/MC/MCAsmInfo.cpp
@@ -68,8 +68,8 @@ MCAsmInfo::MCAsmInfo() {
   GlobalDirective = "\t.globl\t";
   HasSetDirective = true;
   HasAggressiveSymbolFolding = true;
-  LCOMMDirectiveType = LCOMM::None;
   COMMDirectiveAlignmentIsInBytes = true;
+  LCOMMDirectiveAlignmentType = LCOMM::NoAlignment;
   HasDotTypeDotSizeDirective = true;
   HasSingleParameterDotFile = true;
   HasNoDeadStrip = false;
diff --git a/lib/MC/MCAsmInfoCOFF.cpp b/lib/MC/MCAsmInfoCOFF.cpp
index 678e75a..fd79193 100644
--- a/lib/MC/MCAsmInfoCOFF.cpp
+++ b/lib/MC/MCAsmInfoCOFF.cpp
@@ -19,8 +19,10 @@ void MCAsmInfoCOFF::anchor() { }
 
 MCAsmInfoCOFF::MCAsmInfoCOFF() {
   GlobalPrefix = "_";
+  // MingW 4.5 and later support .comm with log2 alignment, but .lcomm uses byte
+  // alignment.
   COMMDirectiveAlignmentIsInBytes = false;
-  LCOMMDirectiveType = LCOMM::ByteAlignment;
+  LCOMMDirectiveAlignmentType = LCOMM::ByteAlignment;
   HasDotTypeDotSizeDirective = false;
   HasSingleParameterDotFile = false;
   PrivateGlobalPrefix = "L";  // Prefix for private global symbols
diff --git a/lib/MC/MCAsmInfoDarwin.cpp b/lib/MC/MCAsmInfoDarwin.cpp
index 8e0ac23..a0e3eba 100644
--- a/lib/MC/MCAsmInfoDarwin.cpp
+++ b/lib/MC/MCAsmInfoDarwin.cpp
@@ -32,6 +32,7 @@ MCAsmInfoDarwin::MCAsmInfoDarwin() {
 
   AlignmentIsInBytes = false;
   COMMDirectiveAlignmentIsInBytes = false;
+  LCOMMDirectiveAlignmentType = LCOMM::Log2Alignment;
   InlineAsmStart = " InlineAsm Start";
   InlineAsmEnd = " InlineAsm End";
 
diff --git a/lib/MC/MCAsmStreamer.cpp b/lib/MC/MCAsmStreamer.cpp
index 373df4b..17a6323 100644
--- a/lib/MC/MCAsmStreamer.cpp
+++ b/lib/MC/MCAsmStreamer.cpp
@@ -166,7 +166,7 @@ public:
   ///
   /// @param Symbol - The common symbol to emit.
   /// @param Size - The size of the common symbol.
-  /// @param Size - The alignment of the common symbol in bytes.
+  /// @param ByteAlignment - The alignment of the common symbol in bytes.
   virtual void EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size,
                                      unsigned ByteAlignment);
 
@@ -251,6 +251,7 @@ public:
   virtual void EmitPad(int64_t Offset);
   virtual void EmitRegSave(const SmallVectorImpl<unsigned> &RegList, bool);
 
+  virtual void EmitTCEntry(const MCSymbol &S);
 
   virtual void EmitInstruction(const MCInst &Inst);
 
@@ -517,13 +518,19 @@ void MCAsmStreamer::EmitCommonSymbol(MCSymbol *Symbol, uint64_t Size,
 /// @param Size - The size of the common symbol.
 void MCAsmStreamer::EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size,
                                           unsigned ByteAlign) {
-  assert(MAI.getLCOMMDirectiveType() != LCOMM::None &&
-         "Doesn't have .lcomm, can't emit it!");
   OS << "\t.lcomm\t" << *Symbol << ',' << Size;
   if (ByteAlign > 1) {
-    assert(MAI.getLCOMMDirectiveType() == LCOMM::ByteAlignment &&
-           "Alignment not supported on .lcomm!");
-    OS << ',' << ByteAlign;
+    switch (MAI.getLCOMMDirectiveAlignmentType()) {
+    case LCOMM::NoAlignment:
+      llvm_unreachable("alignment not supported on .lcomm!");
+    case LCOMM::ByteAlignment:
+      OS << ',' << ByteAlign;
+      break;
+    case LCOMM::Log2Alignment:
+      assert(isPowerOf2_32(ByteAlign) && "alignment must be a power of 2");
+      OS << ',' << Log2_32(ByteAlign);
+      break;
+    }
   }
   EmitEOL();
 }
@@ -1293,6 +1300,14 @@ void MCAsmStreamer::EmitRegSave(const SmallVectorImpl<unsigned> &RegList,
   EmitEOL();
 }
 
+void MCAsmStreamer::EmitTCEntry(const MCSymbol &S) {
+  OS << "\t.tc ";
+  OS << S.getName();
+  OS << "[TC],";
+  OS << S.getName();
+  EmitEOL();
+}
+
 void MCAsmStreamer::EmitInstruction(const MCInst &Inst) {
   assert(getCurrentSection() && "Cannot emit contents before setting section!");
 
diff --git a/lib/MC/MCAssembler.cpp b/lib/MC/MCAssembler.cpp
index 05519b5..726ec5a 100644
--- a/lib/MC/MCAssembler.cpp
+++ b/lib/MC/MCAssembler.cpp
@@ -199,8 +199,7 @@ MCAssembler::MCAssembler(MCContext &Context_, MCAsmBackend &Backend_,
                          MCCodeEmitter &Emitter_, MCObjectWriter &Writer_,
                          raw_ostream &OS_)
   : Context(Context_), Backend(Backend_), Emitter(Emitter_), Writer(Writer_),
-    OS(OS_), RelaxAll(false), NoExecStack(false), SubsectionsViaSymbols(false)
-{
+    OS(OS_), RelaxAll(false), NoExecStack(false), SubsectionsViaSymbols(false) {
 }
 
 MCAssembler::~MCAssembler() {
@@ -325,6 +324,12 @@ uint64_t MCAssembler::computeFragmentSize(const MCAsmLayout &Layout,
     const MCAlignFragment &AF = cast<MCAlignFragment>(F);
     unsigned Offset = Layout.getFragmentOffset(&AF);
     unsigned Size = OffsetToAlignment(Offset, AF.getAlignment());
+    // If we are padding with nops, force the padding to be larger than the
+    // minimum nop size.
+    if (Size > 0 && AF.hasEmitNops()) {
+      while (Size % getBackend().getMinimumNopSize())
+        Size += AF.getAlignment();
+    }
     if (Size > AF.getMaxBytesToEmit())
       return 0;
     return Size;
@@ -375,7 +380,7 @@ void MCAsmLayout::LayoutFragment(MCFragment *F) {
   LastValidFragment[F->getParent()] = F;
 }
 
-/// WriteFragmentData - Write the \arg F data to the output file.
+/// WriteFragmentData - Write the \p F data to the output file.
 static void WriteFragmentData(const MCAssembler &Asm, const MCAsmLayout &Layout,
                               const MCFragment &F) {
   MCObjectWriter *OW = &Asm.getWriter();
@@ -527,7 +532,7 @@ void MCAssembler::writeSectionData(const MCSectionData *SD,
   }
 
   uint64_t Start = getWriter().getStream().tell();
-  (void) Start;
+  (void)Start;
 
   for (MCSectionData::const_iterator it = SD->begin(),
          ie = SD->end(); it != ie; ++it)
@@ -824,6 +829,7 @@ raw_ostream &operator<<(raw_ostream &OS, const MCFixup &AF) {
 
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void MCFragment::dump() {
   raw_ostream &OS = llvm::errs();
 
@@ -964,6 +970,7 @@ void MCAssembler::dump() {
   }
   OS << "]>\n";
 }
+#endif
 
 // anchors for MC*Fragment vtables
 void MCDataFragment::anchor() { }
diff --git a/lib/MC/MCContext.cpp b/lib/MC/MCContext.cpp
index b5b14b9..477bd17 100644
--- a/lib/MC/MCContext.cpp
+++ b/lib/MC/MCContext.cpp
@@ -153,6 +153,12 @@ MCSymbol *MCContext::LookupSymbol(StringRef Name) const {
   return Symbols.lookup(Name);
 }
 
+MCSymbol *MCContext::LookupSymbol(const Twine &Name) const {
+  SmallString<128> NameSV;
+  Name.toVector(NameSV);
+  return LookupSymbol(NameSV.str());
+}
+
 //===----------------------------------------------------------------------===//
 // Section Management
 //===----------------------------------------------------------------------===//
diff --git a/lib/MC/MCDisassembler/Disassembler.cpp b/lib/MC/MCDisassembler/Disassembler.cpp
index 35f675d..5189c9da 100644
--- a/lib/MC/MCDisassembler/Disassembler.cpp
+++ b/lib/MC/MCDisassembler/Disassembler.cpp
@@ -184,3 +184,17 @@ size_t LLVMDisasmInstruction(LLVMDisasmContextRef DCR, uint8_t *Bytes,
   }
   llvm_unreachable("Invalid DecodeStatus!");
 }
+
+//
+// LLVMSetDisasmOptions() sets the disassembler's options.  It returns 1 if it
+// can set all the Options and 0 otherwise.
+//
+int LLVMSetDisasmOptions(LLVMDisasmContextRef DCR, uint64_t Options){
+  if (Options & LLVMDisassembler_Option_UseMarkup){
+      LLVMDisasmContext *DC = (LLVMDisasmContext *)DCR;
+      MCInstPrinter *IP = DC->getIP();
+      IP->setUseMarkup(1);
+      Options &= ~LLVMDisassembler_Option_UseMarkup;
+  }
+  return (Options == 0);
+}
diff --git a/lib/MC/MCDisassembler/EDDisassembler.cpp b/lib/MC/MCDisassembler/EDDisassembler.cpp
index 1226f1a..eed7a77 100644
--- a/lib/MC/MCDisassembler/EDDisassembler.cpp
+++ b/lib/MC/MCDisassembler/EDDisassembler.cpp
@@ -366,8 +366,9 @@ int EDDisassembler::parseInst(SmallVectorImpl<MCParsedAsmOperand*> &operands,
     instName = OpcodeToken.getString();
     instLoc = OpcodeToken.getLoc();
     
+    ParseInstructionInfo Info;
     if (NextToken.isNot(AsmToken::Eof) &&
-        TargetParser->ParseInstruction(instName, instLoc, operands))
+        TargetParser->ParseInstruction(Info, instName, instLoc, operands))
       ret = -1;
   } else {
     ret = -1;
diff --git a/lib/MC/MCDwarf.cpp b/lib/MC/MCDwarf.cpp
index 4c63e43..f71b266 100644
--- a/lib/MC/MCDwarf.cpp
+++ b/lib/MC/MCDwarf.cpp
@@ -425,9 +425,11 @@ void MCDwarfFile::print(raw_ostream &OS) const {
   OS << '"' << getName() << '"';
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void MCDwarfFile::dump() const {
   print(dbgs());
 }
+#endif
 
 // Utility function to write a tuple for .debug_abbrev.
 static void EmitAbbrev(MCStreamer *MCOS, uint64_t Name, uint64_t Form) {
diff --git a/lib/MC/MCELFObjectTargetWriter.cpp b/lib/MC/MCELFObjectTargetWriter.cpp
index 6eb6914..74cd042 100644
--- a/lib/MC/MCELFObjectTargetWriter.cpp
+++ b/lib/MC/MCELFObjectTargetWriter.cpp
@@ -9,6 +9,8 @@
 
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/MC/MCELFObjectWriter.h"
+#include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCValue.h"
 
 using namespace llvm;
 
@@ -35,6 +37,12 @@ const MCSymbol *MCELFObjectTargetWriter::ExplicitRelSym(const MCAssembler &Asm,
   return NULL;
 }
 
+const MCSymbol *MCELFObjectTargetWriter::undefinedExplicitRelSym(const MCValue &Target,
+                                                                 const MCFixup &Fixup,
+                                                                 bool IsPCRel) const {
+  const MCSymbol &Symbol = Target.getSymA()->getSymbol();
+  return &Symbol.AliasedSymbol();
+}
 
 void MCELFObjectTargetWriter::adjustFixupOffset(const MCFixup &Fixup,
                                                 uint64_t &RelocOffset) {
diff --git a/lib/MC/MCELFStreamer.cpp b/lib/MC/MCELFStreamer.cpp
index 2d342dc..14fbc1e 100644
--- a/lib/MC/MCELFStreamer.cpp
+++ b/lib/MC/MCELFStreamer.cpp
@@ -98,17 +98,13 @@ public:
                               uint64_t Size, unsigned ByteAlignment = 0) {
     llvm_unreachable("ELF doesn't support this directive");
   }
-  virtual void EmitBytes(StringRef Data, unsigned AddrSpace);
-  virtual void EmitValueToAlignment(unsigned ByteAlignment, int64_t Value = 0,
-                                    unsigned ValueSize = 1,
-                                    unsigned MaxBytesToEmit = 0);
-  virtual void EmitCodeAlignment(unsigned ByteAlignment,
-                                 unsigned MaxBytesToEmit = 0);
   virtual void EmitValueImpl(const MCExpr *Value, unsigned Size,
                              unsigned AddrSpace);
 
   virtual void EmitFileDirective(StringRef Filename);
 
+  virtual void EmitTCEntry(const MCSymbol &S);
+
   virtual void FinishImpl();
 
 private:
@@ -247,7 +243,6 @@ void MCELFStreamer::EmitSymbolAttribute(MCSymbol *Symbol,
   switch (Attribute) {
   case MCSA_LazyReference:
   case MCSA_Reference:
-  case MCSA_NoDeadStrip:
   case MCSA_SymbolResolver:
   case MCSA_PrivateExtern:
   case MCSA_WeakDefinition:
@@ -256,6 +251,7 @@ void MCELFStreamer::EmitSymbolAttribute(MCSymbol *Symbol,
   case MCSA_IndirectSymbol:
     llvm_unreachable("Invalid symbol attribute for ELF!");
 
+  case MCSA_NoDeadStrip:
   case MCSA_ELF_TypeGnuUniqueObject:
     // Ignore for now.
     break;
@@ -355,42 +351,6 @@ void MCELFStreamer::EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size,
   EmitCommonSymbol(Symbol, Size, ByteAlignment);
 }
 
-void MCELFStreamer::EmitBytes(StringRef Data, unsigned AddrSpace) {
-  // TODO: This is exactly the same as WinCOFFStreamer. Consider merging into
-  // MCObjectStreamer.
-  getOrCreateDataFragment()->getContents().append(Data.begin(), Data.end());
-}
-
-void MCELFStreamer::EmitValueToAlignment(unsigned ByteAlignment,
-                                           int64_t Value, unsigned ValueSize,
-                                           unsigned MaxBytesToEmit) {
-  // TODO: This is exactly the same as WinCOFFStreamer. Consider merging into
-  // MCObjectStreamer.
-  if (MaxBytesToEmit == 0)
-    MaxBytesToEmit = ByteAlignment;
-  new MCAlignFragment(ByteAlignment, Value, ValueSize, MaxBytesToEmit,
-                      getCurrentSectionData());
-
-  // Update the maximum alignment on the current section if necessary.
-  if (ByteAlignment > getCurrentSectionData()->getAlignment())
-    getCurrentSectionData()->setAlignment(ByteAlignment);
-}
-
-void MCELFStreamer::EmitCodeAlignment(unsigned ByteAlignment,
-                                        unsigned MaxBytesToEmit) {
-  // TODO: This is exactly the same as WinCOFFStreamer. Consider merging into
-  // MCObjectStreamer.
-  if (MaxBytesToEmit == 0)
-    MaxBytesToEmit = ByteAlignment;
-  MCAlignFragment *F = new MCAlignFragment(ByteAlignment, 0, 1, MaxBytesToEmit,
-                                           getCurrentSectionData());
-  F->setEmitNops(true);
-
-  // Update the maximum alignment on the current section if necessary.
-  if (ByteAlignment > getCurrentSectionData()->getAlignment())
-    getCurrentSectionData()->setAlignment(ByteAlignment);
-}
-
 void MCELFStreamer::EmitValueImpl(const MCExpr *Value, unsigned Size,
                                   unsigned AddrSpace) {
   fixSymbolsInTLSFixups(Value);
@@ -511,6 +471,12 @@ void MCELFStreamer::FinishImpl() {
   this->MCObjectStreamer::FinishImpl();
 }
 
+void MCELFStreamer::EmitTCEntry(const MCSymbol &S)
+{
+  // Creates a R_PPC64_TOC relocation
+  MCObjectStreamer::EmitSymbolValue(&S, 8, 0);
+}
+
 MCStreamer *llvm::createELFStreamer(MCContext &Context, MCAsmBackend &MAB,
                                     raw_ostream &OS, MCCodeEmitter *CE,
                                     bool RelaxAll, bool NoExecStack) {
diff --git a/lib/MC/MCExpr.cpp b/lib/MC/MCExpr.cpp
index 0eb7fcc..e033634 100644
--- a/lib/MC/MCExpr.cpp
+++ b/lib/MC/MCExpr.cpp
@@ -60,7 +60,8 @@ void MCExpr::print(raw_ostream &OS) const {
         SRE.getKind() == MCSymbolRefExpr::VK_ARM_GOTOFF ||
         SRE.getKind() == MCSymbolRefExpr::VK_ARM_TPOFF ||
         SRE.getKind() == MCSymbolRefExpr::VK_ARM_GOTTPOFF ||
-        SRE.getKind() == MCSymbolRefExpr::VK_ARM_TARGET1)
+        SRE.getKind() == MCSymbolRefExpr::VK_ARM_TARGET1 ||
+        SRE.getKind() == MCSymbolRefExpr::VK_ARM_TARGET2)
       OS << MCSymbolRefExpr::getVariantKindName(SRE.getKind());
     else if (SRE.getKind() != MCSymbolRefExpr::VK_None &&
              SRE.getKind() != MCSymbolRefExpr::VK_PPC_DARWIN_HA16 &&
@@ -136,10 +137,12 @@ void MCExpr::print(raw_ostream &OS) const {
   llvm_unreachable("Invalid expression kind!");
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void MCExpr::dump() const {
   print(dbgs());
   dbgs() << '\n';
 }
+#endif
 
 /* *** */
 
@@ -197,7 +200,9 @@ StringRef MCSymbolRefExpr::getVariantKindName(VariantKind Kind) {
   case VK_ARM_GOTTPOFF: return "(gottpoff)";
   case VK_ARM_TLSGD: return "(tlsgd)";
   case VK_ARM_TARGET1: return "(target1)";
-  case VK_PPC_TOC: return "toc";
+  case VK_ARM_TARGET2: return "(target2)";
+  case VK_PPC_TOC: return "tocbase";
+  case VK_PPC_TOC_ENTRY: return "toc";
   case VK_PPC_DARWIN_HA16: return "ha16";
   case VK_PPC_DARWIN_LO16: return "lo16";
   case VK_PPC_GAS_HA16: return "ha";
@@ -264,7 +269,7 @@ MCSymbolRefExpr::getVariantKindForName(StringRef Name) {
 
 /* *** */
 
-void MCTargetExpr::Anchor() {}
+void MCTargetExpr::anchor() {}
 
 /* *** */
 
diff --git a/lib/MC/MCInst.cpp b/lib/MC/MCInst.cpp
index 7bbfd2e..124cc14 100644
--- a/lib/MC/MCInst.cpp
+++ b/lib/MC/MCInst.cpp
@@ -32,10 +32,12 @@ void MCOperand::print(raw_ostream &OS, const MCAsmInfo *MAI) const {
   OS << ">";
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void MCOperand::dump() const {
   print(dbgs(), 0);
   dbgs() << "\n";
 }
+#endif
 
 void MCInst::print(raw_ostream &OS, const MCAsmInfo *MAI) const {
   OS << "<MCInst " << getOpcode();
@@ -62,7 +64,9 @@ void MCInst::dump_pretty(raw_ostream &OS, const MCAsmInfo *MAI,
   OS << ">";
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void MCInst::dump() const {
   print(dbgs(), 0);
   dbgs() << "\n";
 }
+#endif
diff --git a/lib/MC/MCInstPrinter.cpp b/lib/MC/MCInstPrinter.cpp
index 847bcc0..41d90ab 100644
--- a/lib/MC/MCInstPrinter.cpp
+++ b/lib/MC/MCInstPrinter.cpp
@@ -36,3 +36,17 @@ void MCInstPrinter::printAnnotation(raw_ostream &OS, StringRef Annot) {
       OS << " " << MAI.getCommentString() << " " << Annot;
   }
 }
+
+/// Utility functions to make adding mark ups simpler.
+StringRef MCInstPrinter::markup(StringRef s) const {
+  if (getUseMarkup())
+    return s;
+  else
+    return "";
+}
+StringRef MCInstPrinter::markup(StringRef a, StringRef b) const {
+  if (getUseMarkup())
+    return a;
+  else
+    return b;
+}
diff --git a/lib/MC/MCLabel.cpp b/lib/MC/MCLabel.cpp
index 9c0fc92..1d3022a 100644
--- a/lib/MC/MCLabel.cpp
+++ b/lib/MC/MCLabel.cpp
@@ -16,6 +16,8 @@ void MCLabel::print(raw_ostream &OS) const {
   OS << '"' << getInstance() << '"';
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void MCLabel::dump() const {
   print(dbgs());
 }
+#endif
diff --git a/lib/MC/MCMachOStreamer.cpp b/lib/MC/MCMachOStreamer.cpp
index b75fe2c..04b0e86 100644
--- a/lib/MC/MCMachOStreamer.cpp
+++ b/lib/MC/MCMachOStreamer.cpp
@@ -70,19 +70,11 @@ public:
     llvm_unreachable("macho doesn't support this directive");
   }
   virtual void EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size,
-                                     unsigned ByteAlignment) {
-    llvm_unreachable("macho doesn't support this directive");
-  }
+                                     unsigned ByteAlignment);
   virtual void EmitZerofill(const MCSection *Section, MCSymbol *Symbol = 0,
                             uint64_t Size = 0, unsigned ByteAlignment = 0);
   virtual void EmitTBSSSymbol(const MCSection *Section, MCSymbol *Symbol,
                               uint64_t Size, unsigned ByteAlignment = 0);
-  virtual void EmitBytes(StringRef Data, unsigned AddrSpace);
-  virtual void EmitValueToAlignment(unsigned ByteAlignment, int64_t Value = 0,
-                                    unsigned ValueSize = 1,
-                                    unsigned MaxBytesToEmit = 0);
-  virtual void EmitCodeAlignment(unsigned ByteAlignment,
-                                 unsigned MaxBytesToEmit = 0);
 
   virtual void EmitFileDirective(StringRef Filename) {
     // FIXME: Just ignore the .file; it isn't important enough to fail the
@@ -141,6 +133,8 @@ void MCMachOStreamer::EmitLabel(MCSymbol *Symbol) {
 }
 
 void MCMachOStreamer::EmitDataRegion(DataRegionData::KindTy Kind) {
+  if (!getAssembler().getBackend().hasDataInCodeSupport())
+    return;
   // Create a temporary label to mark the start of the data region.
   MCSymbol *Start = getContext().CreateTempSymbol();
   EmitLabel(Start);
@@ -151,6 +145,8 @@ void MCMachOStreamer::EmitDataRegion(DataRegionData::KindTy Kind) {
 }
 
 void MCMachOStreamer::EmitDataRegionEnd() {
+  if (!getAssembler().getBackend().hasDataInCodeSupport())
+    return;
   std::vector<DataRegionData> &Regions = getAssembler().getDataRegions();
   assert(Regions.size() && "Mismatched .end_data_region!");
   DataRegionData &Data = Regions.back();
@@ -325,6 +321,15 @@ void MCMachOStreamer::EmitCommonSymbol(MCSymbol *Symbol, uint64_t Size,
   SD.setCommon(Size, ByteAlignment);
 }
 
+void MCMachOStreamer::EmitLocalCommonSymbol(MCSymbol *Symbol, uint64_t Size,
+                                            unsigned ByteAlignment) {
+  // '.lcomm' is equivalent to '.zerofill'.
+  return EmitZerofill(getContext().getMachOSection("__DATA", "__bss",
+                                                   MCSectionMachO::S_ZEROFILL,
+                                                   0, SectionKind::getBSS()),
+                      Symbol, Size, ByteAlignment);
+}
+
 void MCMachOStreamer::EmitZerofill(const MCSection *Section, MCSymbol *Symbol,
                                    uint64_t Size, unsigned ByteAlignment) {
   MCSectionData &SectData = getAssembler().getOrCreateSectionData(*Section);
@@ -361,42 +366,6 @@ void MCMachOStreamer::EmitTBSSSymbol(const MCSection *Section, MCSymbol *Symbol,
   return;
 }
 
-void MCMachOStreamer::EmitBytes(StringRef Data, unsigned AddrSpace) {
-  // TODO: This is exactly the same as WinCOFFStreamer. Consider merging into
-  // MCObjectStreamer.
-  getOrCreateDataFragment()->getContents().append(Data.begin(), Data.end());
-}
-
-void MCMachOStreamer::EmitValueToAlignment(unsigned ByteAlignment,
-                                           int64_t Value, unsigned ValueSize,
-                                           unsigned MaxBytesToEmit) {
-  // TODO: This is exactly the same as WinCOFFStreamer. Consider merging into
-  // MCObjectStreamer.
-  if (MaxBytesToEmit == 0)
-    MaxBytesToEmit = ByteAlignment;
-  new MCAlignFragment(ByteAlignment, Value, ValueSize, MaxBytesToEmit,
-                      getCurrentSectionData());
-
-  // Update the maximum alignment on the current section if necessary.
-  if (ByteAlignment > getCurrentSectionData()->getAlignment())
-    getCurrentSectionData()->setAlignment(ByteAlignment);
-}
-
-void MCMachOStreamer::EmitCodeAlignment(unsigned ByteAlignment,
-                                        unsigned MaxBytesToEmit) {
-  // TODO: This is exactly the same as WinCOFFStreamer. Consider merging into
-  // MCObjectStreamer.
-  if (MaxBytesToEmit == 0)
-    MaxBytesToEmit = ByteAlignment;
-  MCAlignFragment *F = new MCAlignFragment(ByteAlignment, 0, 1, MaxBytesToEmit,
-                                           getCurrentSectionData());
-  F->setEmitNops(true);
-
-  // Update the maximum alignment on the current section if necessary.
-  if (ByteAlignment > getCurrentSectionData()->getAlignment())
-    getCurrentSectionData()->setAlignment(ByteAlignment);
-}
-
 void MCMachOStreamer::EmitInstToData(const MCInst &Inst) {
   MCDataFragment *DF = getOrCreateDataFragment();
 
diff --git a/lib/MC/MCObjectFileInfo.cpp b/lib/MC/MCObjectFileInfo.cpp
index 29b4a94..2e1604d 100644
--- a/lib/MC/MCObjectFileInfo.cpp
+++ b/lib/MC/MCObjectFileInfo.cpp
@@ -392,6 +392,18 @@ void MCObjectFileInfo::InitELFMCObjectFileInfo(Triple T) {
   DwarfMacroInfoSection =
     Ctx->getELFSection(".debug_macinfo", ELF::SHT_PROGBITS, 0,
                        SectionKind::getMetadata());
+  DwarfAccelNamesSection =
+    Ctx->getELFSection(".apple_names", ELF::SHT_PROGBITS, 0,
+                       SectionKind::getMetadata());
+  DwarfAccelObjCSection =
+    Ctx->getELFSection(".apple_objc", ELF::SHT_PROGBITS, 0,
+                       SectionKind::getMetadata());
+  DwarfAccelNamespaceSection =
+    Ctx->getELFSection(".apple_namespaces", ELF::SHT_PROGBITS, 0,
+                       SectionKind::getMetadata());
+  DwarfAccelTypesSection =
+    Ctx->getELFSection(".apple_types", ELF::SHT_PROGBITS, 0,
+                       SectionKind::getMetadata());
 }
 
 
@@ -430,12 +442,20 @@ void MCObjectFileInfo::InitCOFFMCObjectFileInfo(Triple T) {
   }
 
 
-  StaticDtorSection =
-    Ctx->getCOFFSection(".dtors",
-                        COFF::IMAGE_SCN_CNT_INITIALIZED_DATA |
-                        COFF::IMAGE_SCN_MEM_READ |
-                        COFF::IMAGE_SCN_MEM_WRITE,
-                        SectionKind::getDataRel());
+  if (T.getOS() == Triple::Win32) {
+    StaticDtorSection =
+      Ctx->getCOFFSection(".CRT$XTX",
+                          COFF::IMAGE_SCN_CNT_INITIALIZED_DATA |
+                          COFF::IMAGE_SCN_MEM_READ,
+                          SectionKind::getReadOnly());
+  } else {
+    StaticDtorSection =
+      Ctx->getCOFFSection(".dtors",
+                          COFF::IMAGE_SCN_CNT_INITIALIZED_DATA |
+                          COFF::IMAGE_SCN_MEM_READ |
+                          COFF::IMAGE_SCN_MEM_WRITE,
+                          SectionKind::getDataRel());
+  }
 
   // FIXME: We're emitting LSDA info into a readonly section on COFF, even
   // though it contains relocatable pointers.  In PIC mode, this is probably a
@@ -557,6 +577,7 @@ void MCObjectFileInfo::InitMCObjectFileInfo(StringRef TT, Reloc::Model relocm,
     Env = IsMachO;
     InitMachOMCObjectFileInfo(T);
   } else if ((Arch == Triple::x86 || Arch == Triple::x86_64) &&
+             (T.getEnvironment() != Triple::ELF) &&
              (T.getOS() == Triple::MinGW32 || T.getOS() == Triple::Cygwin ||
               T.getOS() == Triple::Win32)) {
     Env = IsCOFF;
diff --git a/lib/MC/MCObjectStreamer.cpp b/lib/MC/MCObjectStreamer.cpp
index bad7cfe..7746323 100644
--- a/lib/MC/MCObjectStreamer.cpp
+++ b/lib/MC/MCObjectStreamer.cpp
@@ -232,6 +232,31 @@ void MCObjectStreamer::EmitDwarfAdvanceFrameAddr(const MCSymbol *LastLabel,
   new MCDwarfCallFrameFragment(*AddrDelta, getCurrentSectionData());
 }
 
+void MCObjectStreamer::EmitBytes(StringRef Data, unsigned AddrSpace) {
+  assert(AddrSpace == 0 && "Address space must be 0!");
+  getOrCreateDataFragment()->getContents().append(Data.begin(), Data.end());
+}
+
+void MCObjectStreamer::EmitValueToAlignment(unsigned ByteAlignment,
+                                            int64_t Value,
+                                            unsigned ValueSize,
+                                            unsigned MaxBytesToEmit) {
+  if (MaxBytesToEmit == 0)
+    MaxBytesToEmit = ByteAlignment;
+  new MCAlignFragment(ByteAlignment, Value, ValueSize, MaxBytesToEmit,
+                      getCurrentSectionData());
+
+  // Update the maximum alignment on the current section if necessary.
+  if (ByteAlignment > getCurrentSectionData()->getAlignment())
+    getCurrentSectionData()->setAlignment(ByteAlignment);
+}
+
+void MCObjectStreamer::EmitCodeAlignment(unsigned ByteAlignment,
+                                         unsigned MaxBytesToEmit) {
+  EmitValueToAlignment(ByteAlignment, 0, 1, MaxBytesToEmit);
+  cast<MCAlignFragment>(getCurrentFragment())->setEmitNops(true);
+}
+
 bool MCObjectStreamer::EmitValueToOffset(const MCExpr *Offset,
                                          unsigned char Value) {
   int64_t Res;
@@ -258,12 +283,26 @@ bool MCObjectStreamer::EmitValueToOffset(const MCExpr *Offset,
 void MCObjectStreamer::EmitGPRel32Value(const MCExpr *Value) {
   MCDataFragment *DF = getOrCreateDataFragment();
 
-  DF->addFixup(MCFixup::Create(DF->getContents().size(),
-                               Value,
-                               FK_GPRel_4));
+  DF->addFixup(MCFixup::Create(DF->getContents().size(), Value, FK_GPRel_4));
   DF->getContents().resize(DF->getContents().size() + 4, 0);
 }
 
+// Associate GPRel32 fixup with data and resize data area
+void MCObjectStreamer::EmitGPRel64Value(const MCExpr *Value) {
+  MCDataFragment *DF = getOrCreateDataFragment();
+
+  DF->addFixup(MCFixup::Create(DF->getContents().size(), Value, FK_GPRel_4));
+  DF->getContents().resize(DF->getContents().size() + 8, 0);
+}
+
+void MCObjectStreamer::EmitFill(uint64_t NumBytes, uint8_t FillValue,
+                                unsigned AddrSpace) {
+  assert(AddrSpace == 0 && "Address space must be 0!");
+  // FIXME: A MCFillFragment would be more memory efficient but MCExpr has
+  //        problems evaluating expressions across multiple fragments.
+  getOrCreateDataFragment()->getContents().append(NumBytes, FillValue);
+}
+
 void MCObjectStreamer::FinishImpl() {
   // Dump out the dwarf file & directory tables and line tables.
   const MCSymbol *LineSectionSymbol = NULL;
diff --git a/lib/MC/MCParser/AsmLexer.cpp b/lib/MC/MCParser/AsmLexer.cpp
index c76052d..f93f685 100644
--- a/lib/MC/MCParser/AsmLexer.cpp
+++ b/lib/MC/MCParser/AsmLexer.cpp
@@ -396,8 +396,17 @@ AsmToken AsmLexer::LexToken() {
   case 0:
   case ' ':
   case '\t':
-    // Ignore whitespace.
-    return LexToken();
+    if (SkipSpace) {
+      // Ignore whitespace.
+      return LexToken();
+    } else {
+      int len = 1;
+      while (*CurPtr==' ' || *CurPtr=='\t') {
+        CurPtr++;
+        len++;
+      }
+      return AsmToken(AsmToken::Space, StringRef(TokStart, len));
+    }
   case '\n': // FALL THROUGH.
   case '\r':
     isAtStartOfLine = true;
diff --git a/lib/MC/MCParser/AsmParser.cpp b/lib/MC/MCParser/AsmParser.cpp
index b67c769..6f2e85e 100644
--- a/lib/MC/MCParser/AsmParser.cpp
+++ b/lib/MC/MCParser/AsmParser.cpp
@@ -19,6 +19,8 @@
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCDwarf.h"
 #include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCInstPrinter.h"
+#include "llvm/MC/MCInstrInfo.h"
 #include "llvm/MC/MCParser/AsmCond.h"
 #include "llvm/MC/MCParser/AsmLexer.h"
 #include "llvm/MC/MCParser/MCAsmParser.h"
@@ -35,6 +37,8 @@
 #include "llvm/Support/SourceMgr.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cctype>
+#include <set>
+#include <string>
 #include <vector>
 using namespace llvm;
 
@@ -42,12 +46,14 @@ static cl::opt<bool>
 FatalAssemblerWarnings("fatal-assembler-warnings",
                        cl::desc("Consider warnings as error"));
 
+MCAsmParserSemaCallback::~MCAsmParserSemaCallback() {} 
+
 namespace {
 
 /// \brief Helper class for tracking macro definitions.
 typedef std::vector<AsmToken> MacroArgument;
 typedef std::vector<MacroArgument> MacroArguments;
-typedef StringRef MacroParameter;
+typedef std::pair<StringRef, MacroArgument> MacroParameter;
 typedef std::vector<MacroParameter> MacroParameters;
 
 struct Macro {
@@ -80,12 +86,34 @@ public:
                      MemoryBuffer *I);
 };
 
+//struct AsmRewrite;
+struct ParseStatementInfo {
+  /// ParsedOperands - The parsed operands from the last parsed statement.
+  SmallVector<MCParsedAsmOperand*, 8> ParsedOperands;
+
+  /// Opcode - The opcode from the last parsed instruction.
+  unsigned Opcode;
+
+  SmallVectorImpl<AsmRewrite> *AsmRewrites;
+
+  ParseStatementInfo() : Opcode(~0U), AsmRewrites(0) {}
+  ParseStatementInfo(SmallVectorImpl<AsmRewrite> *rewrites)
+    : Opcode(~0), AsmRewrites(rewrites) {}
+
+  ~ParseStatementInfo() {
+    // Free any parsed operands.
+    for (unsigned i = 0, e = ParsedOperands.size(); i != e; ++i)
+      delete ParsedOperands[i];
+    ParsedOperands.clear();
+  }
+};
+
 /// \brief The concrete assembly parser instance.
 class AsmParser : public MCAsmParser {
   friend class GenericAsmParser;
 
-  AsmParser(const AsmParser &);   // DO NOT IMPLEMENT
-  void operator=(const AsmParser &);  // DO NOT IMPLEMENT
+  AsmParser(const AsmParser &) LLVM_DELETED_FUNCTION;
+  void operator=(const AsmParser &) LLVM_DELETED_FUNCTION;
 private:
   AsmLexer Lexer;
   MCContext &Ctx;
@@ -126,20 +154,27 @@ private:
   StringRef CppHashFilename;
   int64_t CppHashLineNumber;
   SMLoc CppHashLoc;
+  int CppHashBuf;
 
   /// AssemblerDialect. ~OU means unset value and use value provided by MAI.
   unsigned AssemblerDialect;
 
+  /// IsDarwin - is Darwin compatibility enabled?
+  bool IsDarwin;
+
+  /// ParsingInlineAsm - Are we parsing ms-style inline assembly?
+  bool ParsingInlineAsm;
+
 public:
   AsmParser(SourceMgr &SM, MCContext &Ctx, MCStreamer &Out,
             const MCAsmInfo &MAI);
-  ~AsmParser();
+  virtual ~AsmParser();
 
   virtual bool Run(bool NoInitialTextSection, bool NoFinalize = false);
 
-  void AddDirectiveHandler(MCAsmParserExtension *Object,
-                           StringRef Directive,
-                           DirectiveHandler Handler) {
+  virtual void AddDirectiveHandler(MCAsmParserExtension *Object,
+                                   StringRef Directive,
+                                   DirectiveHandler Handler) {
     DirectiveMap[Directive] = std::make_pair(Object, Handler);
   }
 
@@ -166,7 +201,19 @@ public:
   virtual bool Error(SMLoc L, const Twine &Msg,
                      ArrayRef<SMRange> Ranges = ArrayRef<SMRange>());
 
-  const AsmToken &Lex();
+  virtual const AsmToken &Lex();
+
+  void setParsingInlineAsm(bool V) { ParsingInlineAsm = V; }
+  bool isParsingInlineAsm() { return ParsingInlineAsm; }
+
+  bool ParseMSInlineAsm(void *AsmLoc, std::string &AsmString,
+                        unsigned &NumOutputs, unsigned &NumInputs,
+                        SmallVectorImpl<std::pair<void *,bool> > &OpDecls,
+                        SmallVectorImpl<std::string> &Constraints,
+                        SmallVectorImpl<std::string> &Clobbers,
+                        const MCInstrInfo *MII,
+                        const MCInstPrinter *IP,
+                        MCAsmParserSemaCallback &SI);
 
   bool ParseExpression(const MCExpr *&Res);
   virtual bool ParseExpression(const MCExpr *&Res, SMLoc &EndLoc);
@@ -178,7 +225,7 @@ public:
 private:
   void CheckForValidSection();
 
-  bool ParseStatement();
+  bool ParseStatement(ParseStatementInfo &Info);
   void EatToEndOfLine();
   bool ParseCppHashLineFilenameComment(const SMLoc &L);
 
@@ -202,26 +249,28 @@ private:
   /// This returns true on failure.
   bool ProcessIncbinFile(const std::string &Filename);
 
-  /// \brief Reset the current lexer position to that given by \arg Loc. The
+  /// \brief Reset the current lexer position to that given by \p Loc. The
   /// current token is not set; clients should ensure Lex() is called
   /// subsequently.
   void JumpToLoc(SMLoc Loc);
 
-  void EatToEndOfStatement();
+  virtual void EatToEndOfStatement();
 
-  bool ParseMacroArgument(MacroArgument &MA);
+  bool ParseMacroArgument(MacroArgument &MA,
+                          AsmToken::TokenKind &ArgumentDelimiter);
   bool ParseMacroArguments(const Macro *M, MacroArguments &A);
 
   /// \brief Parse up to the end of statement and a return the contents from the
   /// current token until the end of the statement; the current token on exit
   /// will be either the EndOfStatement or EOF.
-  StringRef ParseStringToEndOfStatement();
+  virtual StringRef ParseStringToEndOfStatement();
 
   /// \brief Parse until the end of a statement or a comma is encountered,
   /// return the contents from the current token up to the end or comma.
   StringRef ParseStringToComma();
 
-  bool ParseAssignment(StringRef Name, bool allow_redef);
+  bool ParseAssignment(StringRef Name, bool allow_redef,
+                       bool NoDeadStrip = false);
 
   bool ParsePrimaryExpr(const MCExpr *&Res, SMLoc &EndLoc);
   bool ParseBinOpRHS(unsigned Precedence, const MCExpr *&Res, SMLoc &EndLoc);
@@ -229,8 +278,8 @@ private:
   bool ParseBracketExpr(const MCExpr *&Res, SMLoc &EndLoc);
 
   /// ParseIdentifier - Parse an identifier or string (as a quoted identifier)
-  /// and set \arg Res to the identifier contents.
-  bool ParseIdentifier(StringRef &Res);
+  /// and set \p Res to the identifier contents.
+  virtual bool ParseIdentifier(StringRef &Res);
 
   // Directive Parsing.
 
@@ -282,6 +331,9 @@ private:
   bool ParseDirectiveIrp(SMLoc DirectiveLoc);  // ".irp"
   bool ParseDirectiveIrpc(SMLoc DirectiveLoc); // ".irpc"
   bool ParseDirectiveEndr(SMLoc DirectiveLoc); // ".endr"
+
+  // "_emit"
+  bool ParseDirectiveEmit(SMLoc DirectiveLoc, ParseStatementInfo &Info);
 };
 
 /// \brief Generic implementations of directive handling, etc. which is shared
@@ -406,8 +458,8 @@ AsmParser::AsmParser(SourceMgr &_SM, MCContext &_Ctx,
                      MCStreamer &_Out, const MCAsmInfo &_MAI)
   : Lexer(_MAI), Ctx(_Ctx), Out(_Out), MAI(_MAI), SrcMgr(_SM),
     GenericParser(new GenericAsmParser), PlatformParser(0),
-    CurBuffer(0), MacrosEnabled(true), CppHashLineNumber(0), 
-    AssemblerDialect(~0U) {
+    CurBuffer(0), MacrosEnabled(true), CppHashLineNumber(0),
+    AssemblerDialect(~0U), IsDarwin(false), ParsingInlineAsm(false) {
   // Save the old handler.
   SavedDiagHandler = SrcMgr.getDiagHandler();
   SavedDiagContext = SrcMgr.getDiagContext();
@@ -428,6 +480,7 @@ AsmParser::AsmParser(SourceMgr &_SM, MCContext &_Ctx,
   } else if (_MAI.hasSubsectionsViaSymbols()) {
     PlatformParser = createDarwinAsmParser();
     PlatformParser->Initialize(*this);
+    IsDarwin = true;
   } else {
     PlatformParser = createELFAsmParser();
     PlatformParser->Initialize(*this);
@@ -545,7 +598,8 @@ bool AsmParser::Run(bool NoInitialTextSection, bool NoFinalize) {
 
   // While we have input, parse each statement.
   while (Lexer.isNot(AsmToken::Eof)) {
-    if (!ParseStatement()) continue;
+    ParseStatementInfo Info;
+    if (!ParseStatement(Info)) continue;
 
     // We had an error, validate that one was emitted and recover by skipping to
     // the next line.
@@ -598,7 +652,7 @@ bool AsmParser::Run(bool NoInitialTextSection, bool NoFinalize) {
 }
 
 void AsmParser::CheckForValidSection() {
-  if (!getStreamer().getCurrentSection()) {
+  if (!ParsingInlineAsm && !getStreamer().getCurrentSection()) {
     TokError("expected section directive before assembly directive");
     Out.SwitchSection(Ctx.getMachOSection(
                         "__TEXT", "__text",
@@ -1024,14 +1078,11 @@ bool AsmParser::ParseBinOpRHS(unsigned Precedence, const MCExpr *&Res,
   }
 }
 
-
-
-
 /// ParseStatement:
 ///   ::= EndOfStatement
 ///   ::= Label* Directive ...Operands... EndOfStatement
 ///   ::= Label* Identifier OperandList* EndOfStatement
-bool AsmParser::ParseStatement() {
+bool AsmParser::ParseStatement(ParseStatementInfo &Info) {
   if (Lexer.is(AsmToken::EndOfStatement)) {
     Out.AddBlankLine();
     Lex();
@@ -1150,7 +1201,7 @@ bool AsmParser::ParseStatement() {
         return false;
     }
 
-    return ParseStatement();
+    return false;
   }
 
   case AsmToken::Equal:
@@ -1304,26 +1355,30 @@ bool AsmParser::ParseStatement() {
     return Error(IDLoc, "unknown directive");
   }
 
+  // _emit
+  if (ParsingInlineAsm && IDVal == "_emit")
+    return ParseDirectiveEmit(IDLoc, Info);
+
   CheckForValidSection();
 
   // Canonicalize the opcode to lower case.
-  SmallString<128> Opcode;
+  SmallString<128> OpcodeStr;
   for (unsigned i = 0, e = IDVal.size(); i != e; ++i)
-    Opcode.push_back(tolower(IDVal[i]));
+    OpcodeStr.push_back(tolower(IDVal[i]));
 
-  SmallVector<MCParsedAsmOperand*, 8> ParsedOperands;
-  bool HadError = getTargetParser().ParseInstruction(Opcode.str(), IDLoc,
-                                                     ParsedOperands);
+  ParseInstructionInfo IInfo(Info.AsmRewrites);
+  bool HadError = getTargetParser().ParseInstruction(IInfo, OpcodeStr.str(),
+                                                     IDLoc,Info.ParsedOperands);
 
   // Dump the parsed representation, if requested.
   if (getShowParsedOperands()) {
     SmallString<256> Str;
     raw_svector_ostream OS(Str);
     OS << "parsed instruction: [";
-    for (unsigned i = 0; i != ParsedOperands.size(); ++i) {
+    for (unsigned i = 0; i != Info.ParsedOperands.size(); ++i) {
       if (i != 0)
         OS << ", ";
-      ParsedOperands[i]->print(OS);
+      Info.ParsedOperands[i]->print(OS);
     }
     OS << "]";
 
@@ -1335,21 +1390,38 @@ bool AsmParser::ParseStatement() {
   // the instruction.
   if (!HadError && getContext().getGenDwarfForAssembly() &&
       getContext().getGenDwarfSection() == getStreamer().getCurrentSection() ) {
+
+     unsigned Line = SrcMgr.FindLineNumber(IDLoc, CurBuffer);
+
+     // If we previously parsed a cpp hash file line comment then make sure the
+     // current Dwarf File is for the CppHashFilename if not then emit the
+     // Dwarf File table for it and adjust the line number for the .loc.
+     const std::vector<MCDwarfFile *> &MCDwarfFiles =
+       getContext().getMCDwarfFiles();
+     if (CppHashFilename.size() != 0) {
+       if(MCDwarfFiles[getContext().getGenDwarfFileNumber()]->getName() !=
+          CppHashFilename)
+	 getStreamer().EmitDwarfFileDirective(
+	   getContext().nextGenDwarfFileNumber(), StringRef(), CppHashFilename);
+
+       unsigned CppHashLocLineNo = SrcMgr.FindLineNumber(CppHashLoc,CppHashBuf);
+       Line = CppHashLineNumber - 1 + (Line - CppHashLocLineNo);
+     }
+
     getStreamer().EmitDwarfLocDirective(getContext().getGenDwarfFileNumber(),
-                                        SrcMgr.FindLineNumber(IDLoc, CurBuffer),
-                                        0, DWARF2_LINE_DEFAULT_IS_STMT ?
+                                        Line, 0, DWARF2_LINE_DEFAULT_IS_STMT ?
                                         DWARF2_FLAG_IS_STMT : 0, 0, 0,
                                         StringRef());
   }
 
   // If parsing succeeded, match the instruction.
-  if (!HadError)
-    HadError = getTargetParser().MatchAndEmitInstruction(IDLoc, ParsedOperands,
-                                                         Out);
-
-  // Free any parsed operands.
-  for (unsigned i = 0, e = ParsedOperands.size(); i != e; ++i)
-    delete ParsedOperands[i];
+  if (!HadError) {
+    unsigned ErrorInfo;
+    HadError = getTargetParser().MatchAndEmitInstruction(IDLoc, Info.Opcode,
+                                                         Info.ParsedOperands,
+                                                         Out, ErrorInfo,
+                                                         ParsingInlineAsm);
+  }
 
   // Don't skip the rest of the line, the instruction parser is responsible for
   // that.
@@ -1394,6 +1466,7 @@ bool AsmParser::ParseCppHashLineFilenameComment(const SMLoc &L) {
   CppHashLoc = L;
   CppHashFilename = Filename;
   CppHashLineNumber = LineNumber;
+  CppHashBuf = CurBuffer;
 
   // Ignore any trailing characters, they're just comment.
   EatToEndOfLine();
@@ -1454,6 +1527,14 @@ void AsmParser::DiagHandler(const SMDiagnostic &Diag, void *Context) {
     NewDiag.print(0, OS);
 }
 
+// FIXME: This is mostly duplicated from the function in AsmLexer.cpp. The
+// difference being that that function accepts '@' as part of identifiers and
+// we can't do that. AsmLexer.cpp should probably be changed to handle
+// '@' as a special case when needed.
+static bool isIdentifierChar(char c) {
+  return isalnum(c) || c == '_' || c == '$' || c == '.';
+}
+
 bool AsmParser::expandMacro(raw_svector_ostream &OS, StringRef Body,
                             const MacroParameters &Parameters,
                             const MacroArguments &A,
@@ -1462,6 +1543,8 @@ bool AsmParser::expandMacro(raw_svector_ostream &OS, StringRef Body,
   if (NParameters != 0 && NParameters != A.size())
     return Error(L, "Wrong number of arguments");
 
+  // A macro without parameters is handled differently on Darwin:
+  // gas accepts no arguments and does no substitutions
   while (!Body.empty()) {
     // Scan for the next substitution.
     std::size_t End = Body.size(), Pos = 0;
@@ -1518,25 +1601,33 @@ bool AsmParser::expandMacro(raw_svector_ostream &OS, StringRef Body,
       Pos += 2;
     } else {
       unsigned I = Pos + 1;
-      while (isalnum(Body[I]) && I + 1 != End)
+      while (isIdentifierChar(Body[I]) && I + 1 != End)
         ++I;
 
       const char *Begin = Body.data() + Pos +1;
       StringRef Argument(Begin, I - (Pos +1));
       unsigned Index = 0;
       for (; Index < NParameters; ++Index)
-        if (Parameters[Index] == Argument)
+        if (Parameters[Index].first == Argument)
           break;
 
-      // FIXME: We should error at the macro definition.
-      if (Index == NParameters)
-        return Error(L, "Parameter not found");
-
-      for (MacroArgument::const_iterator it = A[Index].begin(),
-             ie = A[Index].end(); it != ie; ++it)
-        OS << it->getString();
+      if (Index == NParameters) {
+          if (Body[Pos+1] == '(' && Body[Pos+2] == ')')
+            Pos += 3;
+          else {
+            OS << '\\' << Argument;
+            Pos = I;
+          }
+      } else {
+        for (MacroArgument::const_iterator it = A[Index].begin(),
+               ie = A[Index].end(); it != ie; ++it)
+          if (it->getKind() == AsmToken::String)
+            OS << it->getStringContents();
+          else
+            OS << it->getString();
 
-      Pos += 1 + Argument.size();
+        Pos += 1 + Argument.size();
+      }
     }
     // Update the scan point.
     Body = Body.substr(Pos);
@@ -1551,24 +1642,97 @@ MacroInstantiation::MacroInstantiation(const Macro *M, SMLoc IL, SMLoc EL,
 {
 }
 
+static bool IsOperator(AsmToken::TokenKind kind)
+{
+  switch (kind)
+  {
+    default:
+      return false;
+    case AsmToken::Plus:
+    case AsmToken::Minus:
+    case AsmToken::Tilde:
+    case AsmToken::Slash:
+    case AsmToken::Star:
+    case AsmToken::Dot:
+    case AsmToken::Equal:
+    case AsmToken::EqualEqual:
+    case AsmToken::Pipe:
+    case AsmToken::PipePipe:
+    case AsmToken::Caret:
+    case AsmToken::Amp:
+    case AsmToken::AmpAmp:
+    case AsmToken::Exclaim:
+    case AsmToken::ExclaimEqual:
+    case AsmToken::Percent:
+    case AsmToken::Less:
+    case AsmToken::LessEqual:
+    case AsmToken::LessLess:
+    case AsmToken::LessGreater:
+    case AsmToken::Greater:
+    case AsmToken::GreaterEqual:
+    case AsmToken::GreaterGreater:
+      return true;
+  }
+}
+
 /// ParseMacroArgument - Extract AsmTokens for a macro argument.
 /// This is used for both default macro parameter values and the
 /// arguments in macro invocations
-bool AsmParser::ParseMacroArgument(MacroArgument &MA) {
+bool AsmParser::ParseMacroArgument(MacroArgument &MA,
+                                   AsmToken::TokenKind &ArgumentDelimiter) {
   unsigned ParenLevel = 0;
+  unsigned AddTokens = 0;
 
-  for (;;) {
-    SMLoc LastTokenLoc;
+  // gas accepts arguments separated by whitespace, except on Darwin
+  if (!IsDarwin)
+    Lexer.setSkipSpace(false);
 
-    if (Lexer.is(AsmToken::Eof) || Lexer.is(AsmToken::Equal))
+  for (;;) {
+    if (Lexer.is(AsmToken::Eof) || Lexer.is(AsmToken::Equal)) {
+      Lexer.setSkipSpace(true);
       return TokError("unexpected token in macro instantiation");
+    }
+
+    if (ParenLevel == 0 && Lexer.is(AsmToken::Comma)) {
+      // Spaces and commas cannot be mixed to delimit parameters
+      if (ArgumentDelimiter == AsmToken::Eof)
+        ArgumentDelimiter = AsmToken::Comma;
+      else if (ArgumentDelimiter != AsmToken::Comma) {
+        Lexer.setSkipSpace(true);
+        return TokError("expected ' ' for macro argument separator");
+      }
+      break;
+    }
+
+    if (Lexer.is(AsmToken::Space)) {
+      Lex(); // Eat spaces
+
+      // Spaces can delimit parameters, but could also be part an expression.
+      // If the token after a space is an operator, add the token and the next
+      // one into this argument
+      if (ArgumentDelimiter == AsmToken::Space ||
+          ArgumentDelimiter == AsmToken::Eof) {
+        if (IsOperator(Lexer.getKind())) {
+          // Check to see whether the token is used as an operator,
+          // or part of an identifier
+          const char *NextChar = getTok().getEndLoc().getPointer() + 1;
+          if (*NextChar == ' ')
+            AddTokens = 2;
+        }
+
+        if (!AddTokens && ParenLevel == 0) {
+          if (ArgumentDelimiter == AsmToken::Eof &&
+              !IsOperator(Lexer.getKind()))
+            ArgumentDelimiter = AsmToken::Space;
+          break;
+        }
+      }
+    }
 
     // HandleMacroEntry relies on not advancing the lexer here
     // to be able to fill in the remaining default parameter values
     if (Lexer.is(AsmToken::EndOfStatement))
       break;
-    if (ParenLevel == 0 && Lexer.is(AsmToken::Comma))
-      break;
 
     // Adjust the current parentheses level.
     if (Lexer.is(AsmToken::LParen))
@@ -1578,16 +1742,23 @@ bool AsmParser::ParseMacroArgument(MacroArgument &MA) {
 
     // Append the token to the current argument list.
     MA.push_back(getTok());
+    if (AddTokens)
+      AddTokens--;
     Lex();
   }
+
+  Lexer.setSkipSpace(true);
   if (ParenLevel != 0)
-    return TokError("unbalanced parenthesises in macro argument");
+    return TokError("unbalanced parentheses in macro argument");
   return false;
 }
 
 // Parse the macro instantiation arguments.
 bool AsmParser::ParseMacroArguments(const Macro *M, MacroArguments &A) {
   const unsigned NParameters = M ? M->Parameters.size() : 0;
+  // Argument delimiter is initially unknown. It will be set by
+  // ParseMacroArgument()
+  AsmToken::TokenKind ArgumentDelimiter = AsmToken::Eof;
 
   // Parse two kinds of macro invocations:
   // - macros defined without any parameters accept an arbitrary number of them
@@ -1596,13 +1767,30 @@ bool AsmParser::ParseMacroArguments(const Macro *M, MacroArguments &A) {
        ++Parameter) {
     MacroArgument MA;
 
-    if (ParseMacroArgument(MA))
+    if (ParseMacroArgument(MA, ArgumentDelimiter))
       return true;
 
-    A.push_back(MA);
+    if (!MA.empty() || !NParameters)
+      A.push_back(MA);
+    else if (NParameters) {
+      if (!M->Parameters[Parameter].second.empty())
+        A.push_back(M->Parameters[Parameter].second);
+    }
 
-    if (Lexer.is(AsmToken::EndOfStatement))
+    // At the end of the statement, fill in remaining arguments that have
+    // default values. If there aren't any, then the next argument is
+    // required but missing
+    if (Lexer.is(AsmToken::EndOfStatement)) {
+      if (NParameters && Parameter < NParameters - 1) {
+        if (M->Parameters[Parameter + 1].second.empty())
+          return TokError("macro argument '" +
+                          Twine(M->Parameters[Parameter + 1].first) +
+                          "' is missing");
+        else
+          continue;
+      }
       return false;
+    }
 
     if (Lexer.is(AsmToken::Comma))
       Lex();
@@ -1691,7 +1879,8 @@ static bool IsUsedIn(const MCSymbol *Sym, const MCExpr *Value) {
   llvm_unreachable("Unknown expr kind!");
 }
 
-bool AsmParser::ParseAssignment(StringRef Name, bool allow_redef) {
+bool AsmParser::ParseAssignment(StringRef Name, bool allow_redef,
+                                bool NoDeadStrip) {
   // FIXME: Use better location, we should use proper tokens.
   SMLoc EqualLoc = Lexer.getLoc();
 
@@ -1746,6 +1935,9 @@ bool AsmParser::ParseAssignment(StringRef Name, bool allow_redef) {
 
   // Do the assignment.
   Out.EmitAssignment(Sym, Value);
+  if (NoDeadStrip)
+    Out.EmitSymbolAttribute(Sym, MCSA_NoDeadStrip);
+
 
   return false;
 }
@@ -1803,7 +1995,7 @@ bool AsmParser::ParseDirectiveSet(StringRef IDVal, bool allow_redef) {
     return TokError("unexpected token in '" + Twine(IDVal) + "'");
   Lex();
 
-  return ParseAssignment(Name, allow_redef);
+  return ParseAssignment(Name, allow_redef, true);
 }
 
 bool AsmParser::ParseEscapedString(std::string &Data) {
@@ -2274,8 +2466,13 @@ bool AsmParser::ParseDirectiveComm(bool IsLocal) {
     if (ParseAbsoluteExpression(Pow2Alignment))
       return true;
 
+    LCOMM::LCOMMType LCOMM = Lexer.getMAI().getLCOMMDirectiveAlignmentType();
+    if (IsLocal && LCOMM == LCOMM::NoAlignment)
+      return Error(Pow2AlignmentLoc, "alignment not supported on this target");
+
     // If this target takes alignments in bytes (not log) validate and convert.
-    if (Lexer.getMAI().getAlignmentIsInBytes()) {
+    if ((!IsLocal && Lexer.getMAI().getCOMMDirectiveAlignmentIsInBytes()) ||
+        (IsLocal && LCOMM == LCOMM::ByteAlignment)) {
       if (!isPowerOf2_64(Pow2Alignment))
         return Error(Pow2AlignmentLoc, "alignment must be a power of 2");
       Pow2Alignment = Log2_64(Pow2Alignment);
@@ -2303,13 +2500,9 @@ bool AsmParser::ParseDirectiveComm(bool IsLocal) {
   if (!Sym->isUndefined())
     return Error(IDLoc, "invalid symbol redefinition");
 
-  // '.lcomm' is equivalent to '.zerofill'.
   // Create the Symbol as a common or local common with Size and Pow2Alignment
   if (IsLocal) {
-    getStreamer().EmitZerofill(Ctx.getMachOSection(
-                                 "__DATA", "__bss", MCSectionMachO::S_ZEROFILL,
-                                 0, SectionKind::getBSS()),
-                               Sym, Size, 1 << Pow2Alignment);
+    getStreamer().EmitLocalCommonSymbol(Sym, Size, 1 << Pow2Alignment);
     return false;
   }
 
@@ -3073,25 +3266,33 @@ bool GenericAsmParser::ParseDirectiveMacro(StringRef Directive,
                                            SMLoc DirectiveLoc) {
   StringRef Name;
   if (getParser().ParseIdentifier(Name))
-    return TokError("expected identifier in directive");
+    return TokError("expected identifier in '.macro' directive");
 
   MacroParameters Parameters;
+  // Argument delimiter is initially unknown. It will be set by
+  // ParseMacroArgument()
+  AsmToken::TokenKind ArgumentDelimiter = AsmToken::Eof;
   if (getLexer().isNot(AsmToken::EndOfStatement)) {
-    for(;;) {
-      StringRef Parameter;
-      if (getParser().ParseIdentifier(Parameter))
-        return TokError("expected identifier in directive");
+    for (;;) {
+      MacroParameter Parameter;
+      if (getParser().ParseIdentifier(Parameter.first))
+        return TokError("expected identifier in '.macro' directive");
+
+      if (getLexer().is(AsmToken::Equal)) {
+        Lex();
+        if (getParser().ParseMacroArgument(Parameter.second, ArgumentDelimiter))
+          return true;
+      }
+
       Parameters.push_back(Parameter);
 
-      if (getLexer().isNot(AsmToken::Comma))
+      if (getLexer().is(AsmToken::Comma))
+        Lex();
+      else if (getLexer().is(AsmToken::EndOfStatement))
         break;
-      Lex();
     }
   }
 
-  if (getLexer().isNot(AsmToken::EndOfStatement))
-    return TokError("unexpected token in '.macro' directive");
-
   // Eat the end of statement.
   Lex();
 
@@ -3296,7 +3497,7 @@ bool AsmParser::ParseDirectiveIrp(SMLoc DirectiveLoc) {
   MacroParameters Parameters;
   MacroParameter Parameter;
 
-  if (ParseIdentifier(Parameter))
+  if (ParseIdentifier(Parameter.first))
     return TokError("expected identifier in '.irp' directive");
 
   Parameters.push_back(Parameter);
@@ -3323,9 +3524,8 @@ bool AsmParser::ParseDirectiveIrp(SMLoc DirectiveLoc) {
   SmallString<256> Buf;
   raw_svector_ostream OS(Buf);
 
-  for (std::vector<MacroArgument>::iterator i = A.begin(), e = A.end(); i != e;
-       ++i) {
-    std::vector<MacroArgument> Args;
+  for (MacroArguments::iterator i = A.begin(), e = A.end(); i != e; ++i) {
+    MacroArguments Args;
     Args.push_back(*i);
 
     if (expandMacro(OS, M->Body, Parameters, Args, getTok().getLoc()))
@@ -3343,7 +3543,7 @@ bool AsmParser::ParseDirectiveIrpc(SMLoc DirectiveLoc) {
   MacroParameters Parameters;
   MacroParameter Parameter;
 
-  if (ParseIdentifier(Parameter))
+  if (ParseIdentifier(Parameter.first))
     return TokError("expected identifier in '.irpc' directive");
 
   Parameters.push_back(Parameter);
@@ -3393,7 +3593,7 @@ bool AsmParser::ParseDirectiveIrpc(SMLoc DirectiveLoc) {
 
 bool AsmParser::ParseDirectiveEndr(SMLoc DirectiveLoc) {
   if (ActiveMacros.empty())
-    return TokError("unexpected '.endr' directive, no current .rept");
+    return TokError("unmatched '.endr' directive");
 
   // The only .repl that should get here are the ones created by
   // InstantiateMacroLikeBody.
@@ -3403,6 +3603,214 @@ bool AsmParser::ParseDirectiveEndr(SMLoc DirectiveLoc) {
   return false;
 }
 
+bool AsmParser::ParseDirectiveEmit(SMLoc IDLoc, ParseStatementInfo &Info) {
+  const MCExpr *Value;
+  SMLoc ExprLoc = getLexer().getLoc();
+  if (ParseExpression(Value))
+    return true;
+  const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(Value);
+  if (!MCE)
+    return Error(ExprLoc, "unexpected expression in _emit");
+  uint64_t IntValue = MCE->getValue();
+  if (!isUIntN(8, IntValue) && !isIntN(8, IntValue))
+    return Error(ExprLoc, "literal value out of range for directive");
+
+  Info.AsmRewrites->push_back(AsmRewrite(AOK_Emit, IDLoc, 5));
+  return false;
+}
+
+bool AsmParser::ParseMSInlineAsm(void *AsmLoc, std::string &AsmString,
+                                 unsigned &NumOutputs, unsigned &NumInputs,
+                                 SmallVectorImpl<std::pair<void *, bool> > &OpDecls,
+                                 SmallVectorImpl<std::string> &Constraints,
+                                 SmallVectorImpl<std::string> &Clobbers,
+                                 const MCInstrInfo *MII,
+                                 const MCInstPrinter *IP,
+                                 MCAsmParserSemaCallback &SI) {
+  SmallVector<void *, 4> InputDecls;
+  SmallVector<void *, 4> OutputDecls;
+  SmallVector<bool, 4> InputDeclsOffsetOf;
+  SmallVector<bool, 4> OutputDeclsOffsetOf;
+  SmallVector<std::string, 4> InputConstraints;
+  SmallVector<std::string, 4> OutputConstraints;
+  std::set<std::string> ClobberRegs;
+
+  SmallVector<struct AsmRewrite, 4> AsmStrRewrites;
+
+  // Prime the lexer.
+  Lex();
+
+  // While we have input, parse each statement.
+  unsigned InputIdx = 0;
+  unsigned OutputIdx = 0;
+  while (getLexer().isNot(AsmToken::Eof)) {
+    ParseStatementInfo Info(&AsmStrRewrites);
+    if (ParseStatement(Info))
+      return true;
+
+    if (Info.Opcode != ~0U) {
+      const MCInstrDesc &Desc = MII->get(Info.Opcode);
+
+      // Build the list of clobbers, outputs and inputs.
+      for (unsigned i = 1, e = Info.ParsedOperands.size(); i != e; ++i) {
+        MCParsedAsmOperand *Operand = Info.ParsedOperands[i];
+
+        // Immediate.
+        if (Operand->isImm()) {
+          if (Operand->needAsmRewrite())
+            AsmStrRewrites.push_back(AsmRewrite(AOK_ImmPrefix,
+                                                Operand->getStartLoc()));
+          continue;
+        }
+
+        // Register operand.
+        if (Operand->isReg() && !Operand->isOffsetOf()) {
+          unsigned NumDefs = Desc.getNumDefs();
+          // Clobber.
+          if (NumDefs && Operand->getMCOperandNum() < NumDefs) {
+            std::string Reg;
+            raw_string_ostream OS(Reg);
+            IP->printRegName(OS, Operand->getReg());
+            ClobberRegs.insert(StringRef(OS.str()));
+          }
+          continue;
+        }
+
+        // Expr/Input or Output.
+        unsigned Size;
+        void *OpDecl = SI.LookupInlineAsmIdentifier(Operand->getName(), AsmLoc,
+                                                    Size);
+        if (OpDecl) {
+          bool isOutput = (i == 1) && Desc.mayStore();
+          if (!Operand->isOffsetOf() && Operand->needSizeDirective())
+            AsmStrRewrites.push_back(AsmRewrite(AOK_SizeDirective,
+                                                Operand->getStartLoc(),
+                                                /*Len*/0,
+                                                Operand->getMemSize()));
+          if (isOutput) {
+            std::string Constraint = "=";
+            ++InputIdx;
+            OutputDecls.push_back(OpDecl);
+            OutputDeclsOffsetOf.push_back(Operand->isOffsetOf());
+            Constraint += Operand->getConstraint().str();
+            OutputConstraints.push_back(Constraint);
+            AsmStrRewrites.push_back(AsmRewrite(AOK_Output,
+                                                Operand->getStartLoc(),
+                                                Operand->getNameLen()));
+          } else {
+            InputDecls.push_back(OpDecl);
+            InputDeclsOffsetOf.push_back(Operand->isOffsetOf());
+            InputConstraints.push_back(Operand->getConstraint().str());
+            AsmStrRewrites.push_back(AsmRewrite(AOK_Input,
+                                                Operand->getStartLoc(),
+                                                Operand->getNameLen()));
+          }
+        }
+      }
+    }
+  }
+
+  // Set the number of Outputs and Inputs.
+  NumOutputs = OutputDecls.size();
+  NumInputs = InputDecls.size();
+
+  // Set the unique clobbers.
+  for (std::set<std::string>::iterator I = ClobberRegs.begin(),
+         E = ClobberRegs.end(); I != E; ++I)
+    Clobbers.push_back(*I);
+
+  // Merge the various outputs and inputs.  Output are expected first.
+  if (NumOutputs || NumInputs) {
+    unsigned NumExprs = NumOutputs + NumInputs;
+    OpDecls.resize(NumExprs);
+    Constraints.resize(NumExprs);
+    // FIXME: Constraints are hard coded to 'm', but we need an 'r'
+    // constraint for offsetof.  This needs to be cleaned up!
+    for (unsigned i = 0; i < NumOutputs; ++i) {
+      OpDecls[i] = std::make_pair(OutputDecls[i], OutputDeclsOffsetOf[i]);
+      Constraints[i] = OutputDeclsOffsetOf[i] ? "=r" : OutputConstraints[i];
+    }
+    for (unsigned i = 0, j = NumOutputs; i < NumInputs; ++i, ++j) {
+      OpDecls[j] = std::make_pair(InputDecls[i], InputDeclsOffsetOf[i]);
+      Constraints[j] = InputDeclsOffsetOf[i] ? "r" : InputConstraints[i];
+    }
+  }
+
+  // Build the IR assembly string.
+  std::string AsmStringIR;
+  AsmRewriteKind PrevKind = AOK_Imm;
+  raw_string_ostream OS(AsmStringIR);
+  const char *Start = SrcMgr.getMemoryBuffer(0)->getBufferStart();
+  for (SmallVectorImpl<struct AsmRewrite>::iterator
+         I = AsmStrRewrites.begin(), E = AsmStrRewrites.end(); I != E; ++I) {
+    const char *Loc = (*I).Loc.getPointer();
+
+    AsmRewriteKind Kind = (*I).Kind;
+
+    // Emit everything up to the immediate/expression.  If the previous rewrite
+    // was a size directive, then this has already been done.
+    if (PrevKind != AOK_SizeDirective)
+      OS << StringRef(Start, Loc - Start);
+    PrevKind = Kind;
+
+    // Skip the original expression.
+    if (Kind == AOK_Skip) {
+      Start = Loc + (*I).Len;
+      continue;
+    }
+
+    // Rewrite expressions in $N notation.
+    switch (Kind) {
+    default: break;
+    case AOK_Imm:
+      OS << Twine("$$");
+      OS << (*I).Val;
+      break;
+    case AOK_ImmPrefix:
+      OS << Twine("$$");
+      break;
+    case AOK_Input:
+      OS << '$';
+      OS << InputIdx++;
+      break;
+    case AOK_Output:
+      OS << '$';
+      OS << OutputIdx++;
+      break;
+    case AOK_SizeDirective:
+      switch((*I).Val) {
+      default: break;
+      case 8:  OS << "byte ptr "; break;
+      case 16: OS << "word ptr "; break;
+      case 32: OS << "dword ptr "; break;
+      case 64: OS << "qword ptr "; break;
+      case 80: OS << "xword ptr "; break;
+      case 128: OS << "xmmword ptr "; break;
+      case 256: OS << "ymmword ptr "; break;
+      }
+      break;
+    case AOK_Emit:
+      OS << ".byte";
+      break;
+    case AOK_DotOperator:
+      OS << (*I).Val;
+      break;
+    }
+
+    // Skip the original expression.
+    if (Kind != AOK_SizeDirective)
+      Start = Loc + (*I).Len;
+  }
+
+  // Emit the remainder of the asm string.
+  const char *AsmEnd = SrcMgr.getMemoryBuffer(0)->getBufferEnd();
+  if (Start != AsmEnd)
+    OS << StringRef(Start, AsmEnd - Start);
+
+  AsmString = OS.str();
+  return false;
+}
+
 /// \brief Create an MCAsmParser instance.
 MCAsmParser *llvm::createMCAsmParser(SourceMgr &SM,
                                      MCContext &C, MCStreamer &Out,
diff --git a/lib/MC/MCParser/ELFAsmParser.cpp b/lib/MC/MCParser/ELFAsmParser.cpp
index 9316bb1..d55de1f 100644
--- a/lib/MC/MCParser/ELFAsmParser.cpp
+++ b/lib/MC/MCParser/ELFAsmParser.cpp
@@ -203,7 +203,7 @@ bool ELFAsmParser::ParseDirectiveSize(StringRef, SMLoc) {
   StringRef Name;
   if (getParser().ParseIdentifier(Name))
     return TokError("expected identifier in directive");
-  MCSymbol *Sym = getContext().GetOrCreateSymbol(Name);;
+  MCSymbol *Sym = getContext().GetOrCreateSymbol(Name);
 
   if (getLexer().isNot(AsmToken::Comma))
     return TokError("unexpected token in directive");
diff --git a/lib/MC/MCParser/MCAsmLexer.cpp b/lib/MC/MCParser/MCAsmLexer.cpp
index 3a3ff14..384b341 100644
--- a/lib/MC/MCParser/MCAsmLexer.cpp
+++ b/lib/MC/MCParser/MCAsmLexer.cpp
@@ -12,7 +12,8 @@
 
 using namespace llvm;
 
-MCAsmLexer::MCAsmLexer() : CurTok(AsmToken::Error, StringRef()), TokStart(0) {
+MCAsmLexer::MCAsmLexer() : CurTok(AsmToken::Error, StringRef()),
+                           TokStart(0), SkipSpace(true) {
 }
 
 MCAsmLexer::~MCAsmLexer() {
diff --git a/lib/MC/MCParser/MCAsmParser.cpp b/lib/MC/MCParser/MCAsmParser.cpp
index 3a825f0..6967fee 100644
--- a/lib/MC/MCParser/MCAsmParser.cpp
+++ b/lib/MC/MCParser/MCAsmParser.cpp
@@ -44,5 +44,7 @@ bool MCAsmParser::ParseExpression(const MCExpr *&Res) {
 }
 
 void MCParsedAsmOperand::dump() const {
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
   dbgs() << "  " << *this;
+#endif
 }
diff --git a/lib/MC/MCParser/MCTargetAsmParser.cpp b/lib/MC/MCParser/MCTargetAsmParser.cpp
index 6fb1ba4..60a3a3b 100644
--- a/lib/MC/MCParser/MCTargetAsmParser.cpp
+++ b/lib/MC/MCParser/MCTargetAsmParser.cpp
@@ -11,7 +11,7 @@
 using namespace llvm;
 
 MCTargetAsmParser::MCTargetAsmParser()
-  : AvailableFeatures(0)
+  : AvailableFeatures(0), ParsingInlineAsm(false)
 {
 }
 
diff --git a/lib/MC/MCRegisterInfo.cpp b/lib/MC/MCRegisterInfo.cpp
index 4d1aff3..5c71106 100644
--- a/lib/MC/MCRegisterInfo.cpp
+++ b/lib/MC/MCRegisterInfo.cpp
@@ -24,6 +24,8 @@ unsigned MCRegisterInfo::getMatchingSuperReg(unsigned Reg, unsigned SubIdx,
 }
 
 unsigned MCRegisterInfo::getSubReg(unsigned Reg, unsigned Idx) const {
+  assert(Idx && Idx < getNumSubRegIndices() &&
+         "This is not a subregister index");
   // Get a pointer to the corresponding SubRegIndices list. This list has the
   // name of each sub-register in the same order as MCSubRegIterator.
   const uint16_t *SRI = SubRegIndices + get(Reg).SubRegIndices;
@@ -34,6 +36,7 @@ unsigned MCRegisterInfo::getSubReg(unsigned Reg, unsigned Idx) const {
 }
 
 unsigned MCRegisterInfo::getSubRegIndex(unsigned Reg, unsigned SubReg) const {
+  assert(SubReg && SubReg < getNumRegs() && "This is not a register");
   // Get a pointer to the corresponding SubRegIndices list. This list has the
   // name of each sub-register in the same order as MCSubRegIterator.
   const uint16_t *SRI = SubRegIndices + get(Reg).SubRegIndices;
diff --git a/lib/MC/MCStreamer.cpp b/lib/MC/MCStreamer.cpp
index 0bac24d..afece0b 100644
--- a/lib/MC/MCStreamer.cpp
+++ b/lib/MC/MCStreamer.cpp
@@ -561,6 +561,10 @@ void MCStreamer::EmitRegSave(const SmallVectorImpl<unsigned> &RegList, bool) {
   abort();
 }
 
+void MCStreamer::EmitTCEntry(const MCSymbol &S) {
+  llvm_unreachable("Unsupported method");
+}
+
 /// EmitRawText - If this file is backed by an assembly streamer, this dumps
 /// the specified string in the output .s file.  This capability is
 /// indicated by the hasRawTextSupport() predicate.
diff --git a/lib/MC/MCSubtargetInfo.cpp b/lib/MC/MCSubtargetInfo.cpp
index 05c83f7..80a1f02 100644
--- a/lib/MC/MCSubtargetInfo.cpp
+++ b/lib/MC/MCSubtargetInfo.cpp
@@ -19,11 +19,28 @@ using namespace llvm;
 
 MCSchedModel MCSchedModel::DefaultSchedModel; // For unknown processors.
 
+/// InitMCProcessorInfo - Set or change the CPU (optionally supplemented
+/// with feature string). Recompute feature bits and scheduling model.
+void
+MCSubtargetInfo::InitMCProcessorInfo(StringRef CPU, StringRef FS) {
+  SubtargetFeatures Features(FS);
+  FeatureBits = Features.getFeatureBits(CPU, ProcDesc, NumProcs,
+                                        ProcFeatures, NumFeatures);
+
+  if (!CPU.empty())
+    CPUSchedModel = getSchedModelForCPU(CPU);
+  else
+    CPUSchedModel = &MCSchedModel::DefaultSchedModel;
+}
+
 void
 MCSubtargetInfo::InitMCSubtargetInfo(StringRef TT, StringRef CPU, StringRef FS,
                                      const SubtargetFeatureKV *PF,
                                      const SubtargetFeatureKV *PD,
                                      const SubtargetInfoKV *ProcSched,
+                                     const MCWriteProcResEntry *WPR,
+                                     const MCWriteLatencyEntry *WL,
+                                     const MCReadAdvanceEntry *RA,
                                      const InstrStage *IS,
                                      const unsigned *OC,
                                      const unsigned *FP,
@@ -31,26 +48,18 @@ MCSubtargetInfo::InitMCSubtargetInfo(StringRef TT, StringRef CPU, StringRef FS,
   TargetTriple = TT;
   ProcFeatures = PF;
   ProcDesc = PD;
-  ProcSchedModel = ProcSched;
+  ProcSchedModels = ProcSched;
+  WriteProcResTable = WPR;
+  WriteLatencyTable = WL;
+  ReadAdvanceTable = RA;
+
   Stages = IS;
   OperandCycles = OC;
   ForwardingPaths = FP;
   NumFeatures = NF;
   NumProcs = NP;
 
-  SubtargetFeatures Features(FS);
-  FeatureBits = Features.getFeatureBits(CPU, ProcDesc, NumProcs,
-                                        ProcFeatures, NumFeatures);
-}
-
-
-/// ReInitMCSubtargetInfo - Change CPU (and optionally supplemented with
-/// feature string) and recompute feature bits.
-uint64_t MCSubtargetInfo::ReInitMCSubtargetInfo(StringRef CPU, StringRef FS) {
-  SubtargetFeatures Features(FS);
-  FeatureBits = Features.getFeatureBits(CPU, ProcDesc, NumProcs,
-                                        ProcFeatures, NumFeatures);
-  return FeatureBits;
+  InitMCProcessorInfo(CPU, FS);
 }
 
 /// ToggleFeature - Toggle a feature and returns the re-computed feature
@@ -70,13 +79,13 @@ uint64_t MCSubtargetInfo::ToggleFeature(StringRef FS) {
 }
 
 
-MCSchedModel *
+const MCSchedModel *
 MCSubtargetInfo::getSchedModelForCPU(StringRef CPU) const {
-  assert(ProcSchedModel && "Processor machine model not available!");
+  assert(ProcSchedModels && "Processor machine model not available!");
 
 #ifndef NDEBUG
   for (size_t i = 1; i < NumProcs; i++) {
-    assert(strcmp(ProcSchedModel[i - 1].Key, ProcSchedModel[i].Key) < 0 &&
+    assert(strcmp(ProcSchedModels[i - 1].Key, ProcSchedModels[i].Key) < 0 &&
            "Processor machine model table is not sorted");
   }
 #endif
@@ -85,19 +94,25 @@ MCSubtargetInfo::getSchedModelForCPU(StringRef CPU) const {
   SubtargetInfoKV KV;
   KV.Key = CPU.data();
   const SubtargetInfoKV *Found =
-    std::lower_bound(ProcSchedModel, ProcSchedModel+NumProcs, KV);
-  if (Found == ProcSchedModel+NumProcs || StringRef(Found->Key) != CPU) {
+    std::lower_bound(ProcSchedModels, ProcSchedModels+NumProcs, KV);
+  if (Found == ProcSchedModels+NumProcs || StringRef(Found->Key) != CPU) {
     errs() << "'" << CPU
            << "' is not a recognized processor for this target"
            << " (ignoring processor)\n";
     return &MCSchedModel::DefaultSchedModel;
   }
   assert(Found->Value && "Missing processor SchedModel value");
-  return (MCSchedModel *)Found->Value;
+  return (const MCSchedModel *)Found->Value;
 }
 
 InstrItineraryData
 MCSubtargetInfo::getInstrItineraryForCPU(StringRef CPU) const {
-  MCSchedModel *SchedModel = getSchedModelForCPU(CPU);
+  const MCSchedModel *SchedModel = getSchedModelForCPU(CPU);
   return InstrItineraryData(SchedModel, Stages, OperandCycles, ForwardingPaths);
 }
+
+/// Initialize an InstrItineraryData instance.
+void MCSubtargetInfo::initInstrItins(InstrItineraryData &InstrItins) const {
+  InstrItins =
+    InstrItineraryData(CPUSchedModel, Stages, OperandCycles, ForwardingPaths);
+}
diff --git a/lib/MC/MCSymbol.cpp b/lib/MC/MCSymbol.cpp
index f7f9184..b973c57 100644
--- a/lib/MC/MCSymbol.cpp
+++ b/lib/MC/MCSymbol.cpp
@@ -26,7 +26,7 @@ static bool isAcceptableChar(char C) {
   return true;
 }
 
-/// NameNeedsQuoting - Return true if the identifier \arg Str needs quotes to be
+/// NameNeedsQuoting - Return true if the identifier \p Str needs quotes to be
 /// syntactically correct.
 static bool NameNeedsQuoting(StringRef Str) {
   assert(!Str.empty() && "Cannot create an empty MCSymbol");
@@ -76,6 +76,8 @@ void MCSymbol::print(raw_ostream &OS) const {
   OS << '"' << getName() << '"';
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void MCSymbol::dump() const {
   print(dbgs());
 }
+#endif
diff --git a/lib/MC/MCValue.cpp b/lib/MC/MCValue.cpp
index c6ea16c..4393777 100644
--- a/lib/MC/MCValue.cpp
+++ b/lib/MC/MCValue.cpp
@@ -31,6 +31,8 @@ void MCValue::print(raw_ostream &OS, const MCAsmInfo *MAI) const {
     OS << " + " << getConstant();
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void MCValue::dump() const {
   print(dbgs(), 0);
 }
+#endif
diff --git a/lib/MC/MachObjectWriter.cpp b/lib/MC/MachObjectWriter.cpp
index 5820a22..a94b214 100644
--- a/lib/MC/MachObjectWriter.cpp
+++ b/lib/MC/MachObjectWriter.cpp
@@ -68,6 +68,11 @@ uint64_t MachObjectWriter::getSymbolAddress(const MCSymbolData* SD,
 
   // If this is a variable, then recursively evaluate now.
   if (S.isVariable()) {
+    if (const MCConstantExpr *C =
+          dyn_cast<const MCConstantExpr>(S.getVariableValue()))
+      return C->getValue();
+
+
     MCValue Target;
     if (!S.getVariableValue()->EvaluateAsRelocatable(Target, Layout))
       report_fatal_error("unable to evaluate offset for variable '" +
@@ -140,8 +145,8 @@ void MachObjectWriter::WriteHeader(unsigned NumLoadCommands,
 
 /// WriteSegmentLoadCommand - Write a segment load command.
 ///
-/// \arg NumSections - The number of sections in this segment.
-/// \arg SectionDataSize - The total size of the sections.
+/// \param NumSections The number of sections in this segment.
+/// \param SectionDataSize The total size of the sections.
 void MachObjectWriter::WriteSegmentLoadCommand(unsigned NumSections,
                                                uint64_t VMSize,
                                                uint64_t SectionDataStartOffset,
@@ -315,11 +320,7 @@ void MachObjectWriter::WriteNlist(MachSymbolData &MSD,
 
   // Compute the symbol address.
   if (Symbol.isDefined()) {
-    if (Symbol.isAbsolute()) {
-      Address = cast<MCConstantExpr>(Symbol.getVariableValue())->getValue();
-    } else {
-      Address = getSymbolAddress(&Data, Layout);
-    }
+    Address = getSymbolAddress(&Data, Layout);
   } else if (Data.isCommon()) {
     // Common symbols are encoded with the size in the address
     // field, and their alignment in the flags.
@@ -396,8 +397,7 @@ void MachObjectWriter::BindIndirectSymbols(MCAssembler &Asm) {
       continue;
 
     // Initialize the section indirect symbol base, if necessary.
-    if (!IndirectSymBase.count(it->SectionData))
-      IndirectSymBase[it->SectionData] = IndirectIndex;
+    IndirectSymBase.insert(std::make_pair(it->SectionData, IndirectIndex));
 
     Asm.getOrCreateSymbolData(*it->Symbol);
   }
@@ -414,8 +414,7 @@ void MachObjectWriter::BindIndirectSymbols(MCAssembler &Asm) {
       continue;
 
     // Initialize the section indirect symbol base, if necessary.
-    if (!IndirectSymBase.count(it->SectionData))
-      IndirectSymBase[it->SectionData] = IndirectIndex;
+    IndirectSymBase.insert(std::make_pair(it->SectionData, IndirectIndex));
 
     // Set the symbol type to undefined lazy, but only on construction.
     //
@@ -559,6 +558,26 @@ void MachObjectWriter::computeSectionAddresses(const MCAssembler &Asm,
   }
 }
 
+void MachObjectWriter::markAbsoluteVariableSymbols(MCAssembler &Asm,
+                                                   const MCAsmLayout &Layout) {
+  for (MCAssembler::symbol_iterator i = Asm.symbol_begin(),
+                                    e = Asm.symbol_end();
+      i != e; ++i) {
+    MCSymbolData &SD = *i;
+    if (!SD.getSymbol().isVariable())
+      continue;
+
+    // Is the variable is a symbol difference (SA - SB + C) expression,
+    // and neither symbol is external, mark the variable as absolute.
+    const MCExpr *Expr = SD.getSymbol().getVariableValue();
+    MCValue Value;
+    if (Expr->EvaluateAsRelocatable(Value, Layout)) {
+      if (Value.getSymA() && Value.getSymB())
+        const_cast<MCSymbol*>(&SD.getSymbol())->setAbsolute();
+    }
+  }
+}
+
 void MachObjectWriter::ExecutePostLayoutBinding(MCAssembler &Asm,
                                                 const MCAsmLayout &Layout) {
   computeSectionAddresses(Asm, Layout);
@@ -566,6 +585,10 @@ void MachObjectWriter::ExecutePostLayoutBinding(MCAssembler &Asm,
   // Create symbol data for any indirect symbols.
   BindIndirectSymbols(Asm);
 
+  // Mark symbol difference expressions in variables (from .set or = directives)
+  // as absolute.
+  markAbsoluteVariableSymbols(Asm, Layout);
+
   // Compute symbol table information and bind symbol indices.
   ComputeSymbolTable(Asm, StringTable, LocalSymbolData, ExternalSymbolData,
                      UndefinedSymbolData);
@@ -797,8 +820,12 @@ void MachObjectWriter::WriteObject(MCAssembler &Asm,
          it = Asm.data_region_begin(), ie = Asm.data_region_end();
          it != ie; ++it) {
     const DataRegionData *Data = &(*it);
-    uint64_t Start = getSymbolAddress(&Layout.getAssembler().getSymbolData(*Data->Start), Layout);
-    uint64_t End = getSymbolAddress(&Layout.getAssembler().getSymbolData(*Data->End), Layout);
+    uint64_t Start =
+      getSymbolAddress(&Layout.getAssembler().getSymbolData(*Data->Start),
+                       Layout);
+    uint64_t End =
+      getSymbolAddress(&Layout.getAssembler().getSymbolData(*Data->End),
+                       Layout);
     DEBUG(dbgs() << "data in code region-- kind: " << Data->Kind
                  << "  start: " << Start << "(" << Data->Start->getName() << ")"
                  << "  end: " << End << "(" << Data->End->getName() << ")"
diff --git a/lib/MC/SubtargetFeature.cpp b/lib/MC/SubtargetFeature.cpp
index 0a44e77..7625abd 100644
--- a/lib/MC/SubtargetFeature.cpp
+++ b/lib/MC/SubtargetFeature.cpp
@@ -119,14 +119,15 @@ void SubtargetFeatures::AddFeature(const StringRef String,
 }
 
 /// Find KV in array using binary search.
-template<typename T> const T *Find(const StringRef S, const T *A, size_t L) {
+static const SubtargetFeatureKV *Find(StringRef S, const SubtargetFeatureKV *A,
+                                      size_t L) {
   // Make the lower bound element we're looking for
-  T KV;
+  SubtargetFeatureKV KV;
   KV.Key = S.data();
   // Determine the end of the array
-  const T *Hi = A + L;
+  const SubtargetFeatureKV *Hi = A + L;
   // Binary search the array
-  const T *F = std::lower_bound(A, Hi, KV);
+  const SubtargetFeatureKV *F = std::lower_bound(A, Hi, KV);
   // If not found then return NULL
   if (F == Hi || StringRef(F->Key) != S) return NULL;
   // Return the found array item
@@ -336,30 +337,6 @@ uint64_t SubtargetFeatures::getFeatureBits(const StringRef CPU,
   return Bits;
 }
 
-/// Get scheduling itinerary of a CPU.
-void *SubtargetFeatures::getItinerary(const StringRef CPU,
-                                      const SubtargetInfoKV *Table,
-                                      size_t TableSize) {
-  assert(Table && "missing table");
-#ifndef NDEBUG
-  for (size_t i = 1; i < TableSize; i++) {
-    assert(strcmp(Table[i - 1].Key, Table[i].Key) < 0 && "Table is not sorted");
-  }
-#endif
-
-  // Find entry
-  const SubtargetInfoKV *Entry = Find(CPU, Table, TableSize);
-
-  if (Entry) {
-    return Entry->Value;
-  } else {
-    errs() << "'" << CPU
-           << "' is not a recognized processor for this target"
-           << " (ignoring processor)\n";
-    return NULL;
-  }
-}
-
 /// print - Print feature string.
 ///
 void SubtargetFeatures::print(raw_ostream &OS) const {
@@ -368,11 +345,13 @@ void SubtargetFeatures::print(raw_ostream &OS) const {
   OS << "\n";
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 /// dump - Dump feature info.
 ///
 void SubtargetFeatures::dump() const {
   print(dbgs());
 }
+#endif
 
 /// getDefaultSubtargetFeatures - Return a string listing the features
 /// associated with the target triple.
diff --git a/lib/MC/WinCOFFStreamer.cpp b/lib/MC/WinCOFFStreamer.cpp
index b026277..702eec0 100644
--- a/lib/MC/WinCOFFStreamer.cpp
+++ b/lib/MC/WinCOFFStreamer.cpp
@@ -70,11 +70,6 @@ public:
                             uint64_t Size,unsigned ByteAlignment);
   virtual void EmitTBSSSymbol(const MCSection *Section, MCSymbol *Symbol,
                               uint64_t Size, unsigned ByteAlignment);
-  virtual void EmitBytes(StringRef Data, unsigned AddrSpace);
-  virtual void EmitValueToAlignment(unsigned ByteAlignment, int64_t Value,
-                                   unsigned ValueSize, unsigned MaxBytesToEmit);
-  virtual void EmitCodeAlignment(unsigned ByteAlignment,
-                                 unsigned MaxBytesToEmit);
   virtual void EmitFileDirective(StringRef Filename);
   virtual void EmitInstruction(const MCInst &Instruction);
   virtual void EmitWin64EHHandlerData();
@@ -333,43 +328,6 @@ void WinCOFFStreamer::EmitTBSSSymbol(const MCSection *Section, MCSymbol *Symbol,
   llvm_unreachable("not implemented");
 }
 
-void WinCOFFStreamer::EmitBytes(StringRef Data, unsigned AddrSpace) {
-  // TODO: This is copied exactly from the MachOStreamer. Consider merging into
-  // MCObjectStreamer?
-  getOrCreateDataFragment()->getContents().append(Data.begin(), Data.end());
-}
-
-void WinCOFFStreamer::EmitValueToAlignment(unsigned ByteAlignment,
-                                           int64_t Value,
-                                           unsigned ValueSize,
-                                           unsigned MaxBytesToEmit) {
-  // TODO: This is copied exactly from the MachOStreamer. Consider merging into
-  // MCObjectStreamer?
-  if (MaxBytesToEmit == 0)
-    MaxBytesToEmit = ByteAlignment;
-  new MCAlignFragment(ByteAlignment, Value, ValueSize, MaxBytesToEmit,
-                      getCurrentSectionData());
-
-  // Update the maximum alignment on the current section if necessary.
-  if (ByteAlignment > getCurrentSectionData()->getAlignment())
-    getCurrentSectionData()->setAlignment(ByteAlignment);
-}
-
-void WinCOFFStreamer::EmitCodeAlignment(unsigned ByteAlignment,
-                                        unsigned MaxBytesToEmit) {
-  // TODO: This is copied exactly from the MachOStreamer. Consider merging into
-  // MCObjectStreamer?
-  if (MaxBytesToEmit == 0)
-    MaxBytesToEmit = ByteAlignment;
-  MCAlignFragment *F = new MCAlignFragment(ByteAlignment, 0, 1, MaxBytesToEmit,
-                                           getCurrentSectionData());
-  F->setEmitNops(true);
-
-  // Update the maximum alignment on the current section if necessary.
-  if (ByteAlignment > getCurrentSectionData()->getAlignment())
-    getCurrentSectionData()->setAlignment(ByteAlignment);
-}
-
 void WinCOFFStreamer::EmitFileDirective(StringRef Filename) {
   // Ignore for now, linkers don't care, and proper debug
   // info will be a much large effort.
diff --git a/lib/Object/COFFObjectFile.cpp b/lib/Object/COFFObjectFile.cpp
index 8ab54c6..0b7ee34 100644
--- a/lib/Object/COFFObjectFile.cpp
+++ b/lib/Object/COFFObjectFile.cpp
@@ -288,6 +288,11 @@ error_code COFFObjectFile::getSymbolSection(DataRefImpl Symb,
   return object_error::success;
 }
 
+error_code COFFObjectFile::getSymbolValue(DataRefImpl Symb,
+                                          uint64_t &Val) const {
+  report_fatal_error("getSymbolValue unimplemented in COFFObjectFile");
+}
+
 error_code COFFObjectFile::getSectionNext(DataRefImpl Sec,
                                           SectionRef &Result) const {
   const coff_section *sec = toSec(Sec);
@@ -372,7 +377,14 @@ error_code COFFObjectFile::isSectionVirtual(DataRefImpl Sec,
 
 error_code COFFObjectFile::isSectionZeroInit(DataRefImpl Sec,
                                              bool &Result) const {
-  // FIXME: Unimplemented
+  // FIXME: Unimplemented.
+  Result = false;
+  return object_error::success;
+}
+
+error_code COFFObjectFile::isSectionReadOnlyData(DataRefImpl Sec,
+                                                bool &Result) const {
+  // FIXME: Unimplemented.
   Result = false;
   return object_error::success;
 }
diff --git a/lib/Object/MachOObjectFile.cpp b/lib/Object/MachOObjectFile.cpp
index d229671..45aeaac 100644
--- a/lib/Object/MachOObjectFile.cpp
+++ b/lib/Object/MachOObjectFile.cpp
@@ -363,6 +363,10 @@ error_code MachOObjectFile::getSymbolType(DataRefImpl Symb,
   return object_error::success;
 }
 
+error_code MachOObjectFile::getSymbolValue(DataRefImpl Symb,
+                                           uint64_t &Val) const {
+  report_fatal_error("getSymbolValue unimplemented in MachOObjectFile");
+}
 
 symbol_iterator MachOObjectFile::begin_symbols() const {
   // DRI.d.a = segment number; DRI.d.b = symbol index.
@@ -581,14 +585,14 @@ error_code MachOObjectFile::isSectionBSS(DataRefImpl DRI,
 
 error_code MachOObjectFile::isSectionRequiredForExecution(DataRefImpl Sec,
                                                           bool &Result) const {
-  // FIXME: Unimplemented
+  // FIXME: Unimplemented.
   Result = true;
   return object_error::success;
 }
 
 error_code MachOObjectFile::isSectionVirtual(DataRefImpl Sec,
-                                            bool &Result) const {
-  // FIXME: Unimplemented
+                                             bool &Result) const {
+  // FIXME: Unimplemented.
   Result = false;
   return object_error::success;
 }
@@ -612,6 +616,17 @@ error_code MachOObjectFile::isSectionZeroInit(DataRefImpl DRI,
   return object_error::success;
 }
 
+error_code MachOObjectFile::isSectionReadOnlyData(DataRefImpl Sec,
+                                                  bool &Result) const {
+  // Consider using the code from isSectionText to look for __const sections.
+  // Alternately, emit S_ATTR_PURE_INSTRUCTIONS and/or S_ATTR_SOME_INSTRUCTIONS
+  // to use section attributes to distinguish code from data.
+
+  // FIXME: Unimplemented.
+  Result = false;
+  return object_error::success;
+}
+
 error_code MachOObjectFile::sectionContainsSymbol(DataRefImpl Sec,
                                                   DataRefImpl Symb,
                                                   bool &Result) const {
diff --git a/lib/Support/APFloat.cpp b/lib/Support/APFloat.cpp
index ed261a4..7e8b4a3 100644
--- a/lib/Support/APFloat.cpp
+++ b/lib/Support/APFloat.cpp
@@ -46,22 +46,27 @@ namespace llvm {
     /* Number of bits in the significand.  This includes the integer
        bit.  */
     unsigned int precision;
-
-    /* True if arithmetic is supported.  */
-    unsigned int arithmeticOK;
   };
 
-  const fltSemantics APFloat::IEEEhalf = { 15, -14, 11, true };
-  const fltSemantics APFloat::IEEEsingle = { 127, -126, 24, true };
-  const fltSemantics APFloat::IEEEdouble = { 1023, -1022, 53, true };
-  const fltSemantics APFloat::IEEEquad = { 16383, -16382, 113, true };
-  const fltSemantics APFloat::x87DoubleExtended = { 16383, -16382, 64, true };
-  const fltSemantics APFloat::Bogus = { 0, 0, 0, true };
-
-  // The PowerPC format consists of two doubles.  It does not map cleanly
-  // onto the usual format above.  For now only storage of constants of
-  // this type is supported, no arithmetic.
-  const fltSemantics APFloat::PPCDoubleDouble = { 1023, -1022, 106, false };
+  const fltSemantics APFloat::IEEEhalf = { 15, -14, 11 };
+  const fltSemantics APFloat::IEEEsingle = { 127, -126, 24 };
+  const fltSemantics APFloat::IEEEdouble = { 1023, -1022, 53 };
+  const fltSemantics APFloat::IEEEquad = { 16383, -16382, 113 };
+  const fltSemantics APFloat::x87DoubleExtended = { 16383, -16382, 64 };
+  const fltSemantics APFloat::Bogus = { 0, 0, 0 };
+
+  /* The PowerPC format consists of two doubles.  It does not map cleanly
+     onto the usual format above.  It is approximated using twice the
+     mantissa bits.  Note that for exponents near the double minimum,
+     we no longer can represent the full 106 mantissa bits, so those
+     will be treated as denormal numbers.
+
+     FIXME: While this approximation is equivalent to what GCC uses for
+     compile-time arithmetic on PPC double-double numbers, it is not able
+     to represent all possible values held by a PPC double-double number,
+     for example: (long double) 1.0 + (long double) 0x1p-106
+     Should this be replaced by a full emulation of PPC double-double?  */
+  const fltSemantics APFloat::PPCDoubleDouble = { 1023, -1022 + 53, 53 + 53 };
 
   /* A tight upper bound on number of parts required to hold the value
      pow(5, power) is
@@ -116,12 +121,6 @@ hexDigitValue(unsigned int c)
   return -1U;
 }
 
-static inline void
-assertArithmeticOK(const llvm::fltSemantics &semantics) {
-  assert(semantics.arithmeticOK &&
-         "Compile-time arithmetic does not support these semantics");
-}
-
 /* Return the value of a decimal exponent of the form
    [+-]ddddddd.
 
@@ -196,8 +195,10 @@ totalExponent(StringRef::iterator p, StringRef::iterator end,
     assert(value < 10U && "Invalid character in exponent");
 
     unsignedExponent = unsignedExponent * 10 + value;
-    if (unsignedExponent > 32767)
+    if (unsignedExponent > 32767) {
       overflow = true;
+      break;
+    }
   }
 
   if (exponentAdjustment > 32767 || exponentAdjustment < -32768)
@@ -610,8 +611,6 @@ APFloat::assign(const APFloat &rhs)
   sign = rhs.sign;
   category = rhs.category;
   exponent = rhs.exponent;
-  sign2 = rhs.sign2;
-  exponent2 = rhs.exponent2;
   if (category == fcNormal || category == fcNaN)
     copySignificand(rhs);
 }
@@ -705,16 +704,10 @@ APFloat::bitwiseIsEqual(const APFloat &rhs) const {
       category != rhs.category ||
       sign != rhs.sign)
     return false;
-  if (semantics==(const llvm::fltSemantics*)&PPCDoubleDouble &&
-      sign2 != rhs.sign2)
-    return false;
   if (category==fcZero || category==fcInfinity)
     return true;
   else if (category==fcNormal && exponent!=rhs.exponent)
     return false;
-  else if (semantics==(const llvm::fltSemantics*)&PPCDoubleDouble &&
-           exponent2!=rhs.exponent2)
-    return false;
   else {
     int i= partCount();
     const integerPart* p=significandParts();
@@ -727,9 +720,7 @@ APFloat::bitwiseIsEqual(const APFloat &rhs) const {
   }
 }
 
-APFloat::APFloat(const fltSemantics &ourSemantics, integerPart value)
-  : exponent2(0), sign2(0) {
-  assertArithmeticOK(ourSemantics);
+APFloat::APFloat(const fltSemantics &ourSemantics, integerPart value) {
   initialize(&ourSemantics);
   sign = 0;
   zeroSignificand();
@@ -738,24 +729,19 @@ APFloat::APFloat(const fltSemantics &ourSemantics, integerPart value)
   normalize(rmNearestTiesToEven, lfExactlyZero);
 }
 
-APFloat::APFloat(const fltSemantics &ourSemantics) : exponent2(0), sign2(0) {
-  assertArithmeticOK(ourSemantics);
+APFloat::APFloat(const fltSemantics &ourSemantics) {
   initialize(&ourSemantics);
   category = fcZero;
   sign = false;
 }
 
-APFloat::APFloat(const fltSemantics &ourSemantics, uninitializedTag tag)
-  : exponent2(0), sign2(0) {
-  assertArithmeticOK(ourSemantics);
+APFloat::APFloat(const fltSemantics &ourSemantics, uninitializedTag tag) {
   // Allocates storage if necessary but does not initialize it.
   initialize(&ourSemantics);
 }
 
 APFloat::APFloat(const fltSemantics &ourSemantics,
-                 fltCategory ourCategory, bool negative)
-  : exponent2(0), sign2(0) {
-  assertArithmeticOK(ourSemantics);
+                 fltCategory ourCategory, bool negative) {
   initialize(&ourSemantics);
   category = ourCategory;
   sign = negative;
@@ -765,14 +751,12 @@ APFloat::APFloat(const fltSemantics &ourSemantics,
     makeNaN();
 }
 
-APFloat::APFloat(const fltSemantics &ourSemantics, StringRef text)
-  : exponent2(0), sign2(0) {
-  assertArithmeticOK(ourSemantics);
+APFloat::APFloat(const fltSemantics &ourSemantics, StringRef text) {
   initialize(&ourSemantics);
   convertFromString(text, rmNearestTiesToEven);
 }
 
-APFloat::APFloat(const APFloat &rhs) : exponent2(0), sign2(0) {
+APFloat::APFloat(const APFloat &rhs) {
   initialize(rhs.semantics);
   assign(rhs);
 }
@@ -1559,8 +1543,6 @@ APFloat::addOrSubtract(const APFloat &rhs, roundingMode rounding_mode,
 {
   opStatus fs;
 
-  assertArithmeticOK(*semantics);
-
   fs = addOrSubtractSpecials(rhs, subtract);
 
   /* This return code means it was not a simple case.  */
@@ -1605,7 +1587,6 @@ APFloat::multiply(const APFloat &rhs, roundingMode rounding_mode)
 {
   opStatus fs;
 
-  assertArithmeticOK(*semantics);
   sign ^= rhs.sign;
   fs = multiplySpecials(rhs);
 
@@ -1625,7 +1606,6 @@ APFloat::divide(const APFloat &rhs, roundingMode rounding_mode)
 {
   opStatus fs;
 
-  assertArithmeticOK(*semantics);
   sign ^= rhs.sign;
   fs = divideSpecials(rhs);
 
@@ -1647,7 +1627,6 @@ APFloat::remainder(const APFloat &rhs)
   APFloat V = *this;
   unsigned int origSign = sign;
 
-  assertArithmeticOK(*semantics);
   fs = V.divide(rhs, rmNearestTiesToEven);
   if (fs == opDivByZero)
     return fs;
@@ -1682,7 +1661,6 @@ APFloat::opStatus
 APFloat::mod(const APFloat &rhs, roundingMode rounding_mode)
 {
   opStatus fs;
-  assertArithmeticOK(*semantics);
   fs = modSpecials(rhs);
 
   if (category == fcNormal && rhs.category == fcNormal) {
@@ -1726,8 +1704,6 @@ APFloat::fusedMultiplyAdd(const APFloat &multiplicand,
 {
   opStatus fs;
 
-  assertArithmeticOK(*semantics);
-
   /* Post-multiplication sign, before addition.  */
   sign ^= multiplicand.sign;
 
@@ -1768,12 +1744,11 @@ APFloat::fusedMultiplyAdd(const APFloat &multiplicand,
 /* Rounding-mode corrrect round to integral value.  */
 APFloat::opStatus APFloat::roundToIntegral(roundingMode rounding_mode) {
   opStatus fs;
-  assertArithmeticOK(*semantics);
 
   // If the exponent is large enough, we know that this value is already
   // integral, and the arithmetic below would potentially cause it to saturate
   // to +/-Inf.  Bail out early instead.
-  if (exponent+1 >= (int)semanticsPrecision(*semantics))
+  if (category == fcNormal && exponent+1 >= (int)semanticsPrecision(*semantics))
     return opOK;
 
   // The algorithm here is quite simple: we add 2^(p-1), where p is the
@@ -1815,7 +1790,6 @@ APFloat::compare(const APFloat &rhs) const
 {
   cmpResult result;
 
-  assertArithmeticOK(*semantics);
   assert(semantics == rhs.semantics);
 
   switch (convolve(category, rhs.category)) {
@@ -1900,8 +1874,6 @@ APFloat::convert(const fltSemantics &toSemantics,
   int shift;
   const fltSemantics &fromSemantics = *semantics;
 
-  assertArithmeticOK(fromSemantics);
-  assertArithmeticOK(toSemantics);
   lostFraction = lfExactlyZero;
   newPartCount = partCountForBits(toSemantics.precision + 1);
   oldPartCount = partCount();
@@ -1986,8 +1958,6 @@ APFloat::convertToSignExtendedInteger(integerPart *parts, unsigned int width,
   const integerPart *src;
   unsigned int dstPartsCount, truncatedBits;
 
-  assertArithmeticOK(*semantics);
-
   *isExact = false;
 
   /* Handle the three special cases first.  */
@@ -2149,7 +2119,6 @@ APFloat::convertFromUnsignedParts(const integerPart *src,
   integerPart *dst;
   lostFraction lost_fraction;
 
-  assertArithmeticOK(*semantics);
   category = fcNormal;
   omsb = APInt::tcMSB(src, srcCount) + 1;
   dst = significandParts();
@@ -2200,7 +2169,6 @@ APFloat::convertFromSignExtendedInteger(const integerPart *src,
 {
   opStatus status;
 
-  assertArithmeticOK(*semantics);
   if (isSigned &&
       APInt::tcExtractBit(src, srcCount * integerPartWidth - 1)) {
     integerPart *copy;
@@ -2334,7 +2302,7 @@ APFloat::roundSignificandWithExponent(const integerPart *decSigParts,
                                       roundingMode rounding_mode)
 {
   unsigned int parts, pow5PartCount;
-  fltSemantics calcSemantics = { 32767, -32767, 0, true };
+  fltSemantics calcSemantics = { 32767, -32767, 0 };
   integerPart pow5Parts[maxPowerOfFiveParts];
   bool isNearest;
 
@@ -2526,7 +2494,6 @@ APFloat::convertFromDecimalString(StringRef str, roundingMode rounding_mode)
 APFloat::opStatus
 APFloat::convertFromString(StringRef str, roundingMode rounding_mode)
 {
-  assertArithmeticOK(*semantics);
   assert(!str.empty() && "Invalid string length");
 
   /* Handle a leading minus sign.  */
@@ -2578,8 +2545,6 @@ APFloat::convertToHexString(char *dst, unsigned int hexDigits,
 {
   char *p;
 
-  assertArithmeticOK(*semantics);
-
   p = dst;
   if (sign)
     *dst++ = '-';
@@ -2788,42 +2753,46 @@ APFloat::convertPPCDoubleDoubleAPFloatToAPInt() const
   assert(semantics == (const llvm::fltSemantics*)&PPCDoubleDouble);
   assert(partCount()==2);
 
-  uint64_t myexponent, mysignificand, myexponent2, mysignificand2;
-
-  if (category==fcNormal) {
-    myexponent = exponent + 1023; //bias
-    myexponent2 = exponent2 + 1023;
-    mysignificand = significandParts()[0];
-    mysignificand2 = significandParts()[1];
-    if (myexponent==1 && !(mysignificand & 0x10000000000000LL))
-      myexponent = 0;   // denormal
-    if (myexponent2==1 && !(mysignificand2 & 0x10000000000000LL))
-      myexponent2 = 0;   // denormal
-  } else if (category==fcZero) {
-    myexponent = 0;
-    mysignificand = 0;
-    myexponent2 = 0;
-    mysignificand2 = 0;
-  } else if (category==fcInfinity) {
-    myexponent = 0x7ff;
-    myexponent2 = 0;
-    mysignificand = 0;
-    mysignificand2 = 0;
+  uint64_t words[2];
+  opStatus fs;
+  bool losesInfo;
+
+  // Convert number to double.  To avoid spurious underflows, we re-
+  // normalize against the "double" minExponent first, and only *then*
+  // truncate the mantissa.  The result of that second conversion
+  // may be inexact, but should never underflow.
+  APFloat extended(*this);
+  fltSemantics extendedSemantics = *semantics;
+  extendedSemantics.minExponent = IEEEdouble.minExponent;
+  fs = extended.convert(extendedSemantics, rmNearestTiesToEven, &losesInfo);
+  assert(fs == opOK && !losesInfo);
+  (void)fs;
+
+  APFloat u(extended);
+  fs = u.convert(IEEEdouble, rmNearestTiesToEven, &losesInfo);
+  assert(fs == opOK || fs == opInexact);
+  (void)fs;
+  words[0] = *u.convertDoubleAPFloatToAPInt().getRawData();
+
+  // If conversion was exact or resulted in a special case, we're done;
+  // just set the second double to zero.  Otherwise, re-convert back to
+  // the extended format and compute the difference.  This now should
+  // convert exactly to double.
+  if (u.category == fcNormal && losesInfo) {
+    fs = u.convert(extendedSemantics, rmNearestTiesToEven, &losesInfo);
+    assert(fs == opOK && !losesInfo);
+    (void)fs;
+
+    APFloat v(extended);
+    v.subtract(u, rmNearestTiesToEven);
+    fs = v.convert(IEEEdouble, rmNearestTiesToEven, &losesInfo);
+    assert(fs == opOK && !losesInfo);
+    (void)fs;
+    words[1] = *v.convertDoubleAPFloatToAPInt().getRawData();
   } else {
-    assert(category == fcNaN && "Unknown category");
-    myexponent = 0x7ff;
-    mysignificand = significandParts()[0];
-    myexponent2 = exponent2;
-    mysignificand2 = significandParts()[1];
+    words[1] = 0;
   }
 
-  uint64_t words[2];
-  words[0] =  ((uint64_t)(sign & 1) << 63) |
-              ((myexponent & 0x7ff) <<  52) |
-              (mysignificand & 0xfffffffffffffLL);
-  words[1] =  ((uint64_t)(sign2 & 1) << 63) |
-              ((myexponent2 & 0x7ff) <<  52) |
-              (mysignificand2 & 0xfffffffffffffLL);
   return APInt(128, words);
 }
 
@@ -3043,47 +3012,23 @@ APFloat::initFromPPCDoubleDoubleAPInt(const APInt &api)
   assert(api.getBitWidth()==128);
   uint64_t i1 = api.getRawData()[0];
   uint64_t i2 = api.getRawData()[1];
-  uint64_t myexponent = (i1 >> 52) & 0x7ff;
-  uint64_t mysignificand = i1 & 0xfffffffffffffLL;
-  uint64_t myexponent2 = (i2 >> 52) & 0x7ff;
-  uint64_t mysignificand2 = i2 & 0xfffffffffffffLL;
+  opStatus fs;
+  bool losesInfo;
 
-  initialize(&APFloat::PPCDoubleDouble);
-  assert(partCount()==2);
+  // Get the first double and convert to our format.
+  initFromDoubleAPInt(APInt(64, i1));
+  fs = convert(PPCDoubleDouble, rmNearestTiesToEven, &losesInfo);
+  assert(fs == opOK && !losesInfo);
+  (void)fs;
 
-  sign = static_cast<unsigned int>(i1>>63);
-  sign2 = static_cast<unsigned int>(i2>>63);
-  if (myexponent==0 && mysignificand==0) {
-    // exponent, significand meaningless
-    // exponent2 and significand2 are required to be 0; we don't check
-    category = fcZero;
-  } else if (myexponent==0x7ff && mysignificand==0) {
-    // exponent, significand meaningless
-    // exponent2 and significand2 are required to be 0; we don't check
-    category = fcInfinity;
-  } else if (myexponent==0x7ff && mysignificand!=0) {
-    // exponent meaningless.  So is the whole second word, but keep it
-    // for determinism.
-    category = fcNaN;
-    exponent2 = myexponent2;
-    significandParts()[0] = mysignificand;
-    significandParts()[1] = mysignificand2;
-  } else {
-    category = fcNormal;
-    // Note there is no category2; the second word is treated as if it is
-    // fcNormal, although it might be something else considered by itself.
-    exponent = myexponent - 1023;
-    exponent2 = myexponent2 - 1023;
-    significandParts()[0] = mysignificand;
-    significandParts()[1] = mysignificand2;
-    if (myexponent==0)          // denormal
-      exponent = -1022;
-    else
-      significandParts()[0] |= 0x10000000000000LL;  // integer bit
-    if (myexponent2==0)
-      exponent2 = -1022;
-    else
-      significandParts()[1] |= 0x10000000000000LL;  // integer bit
+  // Unless we have a special case, add in second double.
+  if (category == fcNormal) {
+    APFloat v(APInt(64, i2));
+    fs = v.convert(PPCDoubleDouble, rmNearestTiesToEven, &losesInfo);
+    assert(fs == opOK && !losesInfo);
+    (void)fs;
+
+    add(v, rmNearestTiesToEven);
   }
 }
 
@@ -3309,15 +3254,15 @@ APFloat APFloat::getSmallestNormalized(const fltSemantics &Sem, bool Negative) {
   return Val;
 }
 
-APFloat::APFloat(const APInt& api, bool isIEEE) : exponent2(0), sign2(0) {
+APFloat::APFloat(const APInt& api, bool isIEEE) {
   initFromAPInt(api, isIEEE);
 }
 
-APFloat::APFloat(float f) : exponent2(0), sign2(0) {
+APFloat::APFloat(float f) {
   initFromAPInt(APInt::floatToBits(f));
 }
 
-APFloat::APFloat(double d) : exponent2(0), sign2(0) {
+APFloat::APFloat(double d) {
   initFromAPInt(APInt::doubleToBits(d));
 }
 
@@ -3608,11 +3553,6 @@ void APFloat::toString(SmallVectorImpl<char> &Str,
 }
 
 bool APFloat::getExactInverse(APFloat *inv) const {
-  // We can only guarantee the existence of an exact inverse for IEEE floats.
-  if (semantics != &IEEEhalf && semantics != &IEEEsingle &&
-      semantics != &IEEEdouble && semantics != &IEEEquad)
-    return false;
-
   // Special floats and denormals have no exact inverse.
   if (category != fcNormal)
     return false;
diff --git a/lib/Support/Atomic.cpp b/lib/Support/Atomic.cpp
index 3001f6c..9559ad7 100644
--- a/lib/Support/Atomic.cpp
+++ b/lib/Support/Atomic.cpp
@@ -21,11 +21,15 @@ using namespace llvm;
 #undef MemoryFence
 #endif
 
+#if defined(__GNUC__) || (defined(__IBMCPP__) && __IBMCPP__ >= 1210)
+#define GNU_ATOMICS
+#endif
+
 void sys::MemoryFence() {
 #if LLVM_HAS_ATOMICS == 0
   return;
 #else
-#  if defined(__GNUC__)
+#  if defined(GNU_ATOMICS)
   __sync_synchronize();
 #  elif defined(_MSC_VER)
   MemoryBarrier();
@@ -43,7 +47,7 @@ sys::cas_flag sys::CompareAndSwap(volatile sys::cas_flag* ptr,
   if (result == old_value)
     *ptr = new_value;
   return result;
-#elif defined(__GNUC__)
+#elif defined(GNU_ATOMICS)
   return __sync_val_compare_and_swap(ptr, old_value, new_value);
 #elif defined(_MSC_VER)
   return InterlockedCompareExchange(ptr, new_value, old_value);
@@ -56,7 +60,7 @@ sys::cas_flag sys::AtomicIncrement(volatile sys::cas_flag* ptr) {
 #if LLVM_HAS_ATOMICS == 0
   ++(*ptr);
   return *ptr;
-#elif defined(__GNUC__)
+#elif defined(GNU_ATOMICS)
   return __sync_add_and_fetch(ptr, 1);
 #elif defined(_MSC_VER)
   return InterlockedIncrement(ptr);
@@ -69,7 +73,7 @@ sys::cas_flag sys::AtomicDecrement(volatile sys::cas_flag* ptr) {
 #if LLVM_HAS_ATOMICS == 0
   --(*ptr);
   return *ptr;
-#elif defined(__GNUC__)
+#elif defined(GNU_ATOMICS)
   return __sync_sub_and_fetch(ptr, 1);
 #elif defined(_MSC_VER)
   return InterlockedDecrement(ptr);
@@ -82,7 +86,7 @@ sys::cas_flag sys::AtomicAdd(volatile sys::cas_flag* ptr, sys::cas_flag val) {
 #if LLVM_HAS_ATOMICS == 0
   *ptr += val;
   return *ptr;
-#elif defined(__GNUC__)
+#elif defined(GNU_ATOMICS)
   return __sync_add_and_fetch(ptr, val);
 #elif defined(_MSC_VER)
   return InterlockedExchangeAdd(ptr, val) + val;
diff --git a/lib/Support/CMakeLists.txt b/lib/Support/CMakeLists.txt
index 83baf60..6af0f4a 100644
--- a/lib/Support/CMakeLists.txt
+++ b/lib/Support/CMakeLists.txt
@@ -1,9 +1,3 @@
-## FIXME: This only requires RTTI because tblgen uses it.  Fix that.
-set(LLVM_REQUIRES_RTTI 1)
-if( MINGW )
-  set(LLVM_REQUIRES_EH 1)
-endif()
-
 add_llvm_library(LLVMSupport
   APFloat.cpp
   APInt.cpp
diff --git a/lib/Support/CommandLine.cpp b/lib/Support/CommandLine.cpp
index 593315d1..fc4f189 100644
--- a/lib/Support/CommandLine.cpp
+++ b/lib/Support/CommandLine.cpp
@@ -464,7 +464,7 @@ static void ParseCStringVector(std::vector<char *> &OutputVector,
 /// an environment variable (whose name is given in ENVVAR).
 ///
 void cl::ParseEnvironmentOptions(const char *progName, const char *envVar,
-                                 const char *Overview, bool ReadResponseFiles) {
+                                 const char *Overview) {
   // Check args.
   assert(progName && "Program name not specified");
   assert(envVar && "Environment variable name missing");
@@ -483,7 +483,7 @@ void cl::ParseEnvironmentOptions(const char *progName, const char *envVar,
   // and hand it off to ParseCommandLineOptions().
   ParseCStringVector(newArgv, envValue);
   int newArgc = static_cast<int>(newArgv.size());
-  ParseCommandLineOptions(newArgc, &newArgv[0], Overview, ReadResponseFiles);
+  ParseCommandLineOptions(newArgc, &newArgv[0], Overview);
 
   // Free all the strdup()ed strings.
   for (std::vector<char*>::iterator i = newArgv.begin(), e = newArgv.end();
@@ -529,7 +529,7 @@ static void ExpandResponseFiles(unsigned argc, const char*const* argv,
 }
 
 void cl::ParseCommandLineOptions(int argc, const char * const *argv,
-                                 const char *Overview, bool ReadResponseFiles) {
+                                 const char *Overview) {
   // Process all registered options.
   SmallVector<Option*, 4> PositionalOpts;
   SmallVector<Option*, 4> SinkOpts;
@@ -541,12 +541,10 @@ void cl::ParseCommandLineOptions(int argc, const char * const *argv,
 
   // Expand response files.
   std::vector<char*> newArgv;
-  if (ReadResponseFiles) {
-    newArgv.push_back(strdup(argv[0]));
-    ExpandResponseFiles(argc, argv, newArgv);
-    argv = &newArgv[0];
-    argc = static_cast<int>(newArgv.size());
-  }
+  newArgv.push_back(strdup(argv[0]));
+  ExpandResponseFiles(argc, argv, newArgv);
+  argv = &newArgv[0];
+  argc = static_cast<int>(newArgv.size());
 
   // Copy the program name into ProgName, making sure not to overflow it.
   std::string ProgName = sys::path::filename(argv[0]);
@@ -839,12 +837,10 @@ void cl::ParseCommandLineOptions(int argc, const char * const *argv,
   MoreHelp->clear();
 
   // Free the memory allocated by ExpandResponseFiles.
-  if (ReadResponseFiles) {
-    // Free all the strdup()ed strings.
-    for (std::vector<char*>::iterator i = newArgv.begin(), e = newArgv.end();
-         i != e; ++i)
-      free(*i);
-  }
+  // Free all the strdup()ed strings.
+  for (std::vector<char*>::iterator i = newArgv.begin(), e = newArgv.end();
+       i != e; ++i)
+    free(*i);
 
   // If we had an error processing our arguments, don't let the program execute
   if (ErrorParsing) exit(1);
diff --git a/lib/Support/DAGDeltaAlgorithm.cpp b/lib/Support/DAGDeltaAlgorithm.cpp
index 1e89c6a..34e82cf 100644
--- a/lib/Support/DAGDeltaAlgorithm.cpp
+++ b/lib/Support/DAGDeltaAlgorithm.cpp
@@ -122,7 +122,7 @@ private:
     DDA.UpdatedSearchState(Changes, Sets, Required);
   }
 
-  /// ExecuteOneTest - Execute a single test predicate on the change set \arg S.
+  /// ExecuteOneTest - Execute a single test predicate on the change set \p S.
   bool ExecuteOneTest(const changeset_ty &S) {
     // Check dependencies invariant.
     DEBUG({
@@ -143,8 +143,8 @@ public:
 
   changeset_ty Run();
 
-  /// GetTestResult - Get the test result for the active set \arg Changes with
-  /// \arg Required changes from the cache, executing the test if necessary.
+  /// GetTestResult - Get the test result for the active set \p Changes with
+  /// \p Required changes from the cache, executing the test if necessary.
   ///
   /// \param Changes - The set of active changes being minimized, which should
   /// have their pred closure included in the test.
@@ -163,11 +163,11 @@ class DeltaActiveSetHelper : public DeltaAlgorithm {
 protected:
   /// UpdatedSearchState - Callback used when the search state changes.
   virtual void UpdatedSearchState(const changeset_ty &Changes,
-                                  const changesetlist_ty &Sets) {
+                                  const changesetlist_ty &Sets) LLVM_OVERRIDE {
     DDAI.UpdatedSearchState(Changes, Sets, Required);
   }
 
-  virtual bool ExecuteOneTest(const changeset_ty &S) {
+  virtual bool ExecuteOneTest(const changeset_ty &S) LLVM_OVERRIDE {
     return DDAI.GetTestResult(S, Required);
   }
 
diff --git a/lib/Support/DataExtractor.cpp b/lib/Support/DataExtractor.cpp
index dc21155..3d5cce0 100644
--- a/lib/Support/DataExtractor.cpp
+++ b/lib/Support/DataExtractor.cpp
@@ -139,7 +139,7 @@ uint64_t DataExtractor::getULEB128(uint32_t *offset_ptr) const {
 
   while (isValidOffset(offset)) {
     byte = Data[offset++];
-    result |= (byte & 0x7f) << shift;
+    result |= uint64_t(byte & 0x7f) << shift;
     shift += 7;
     if ((byte & 0x80) == 0)
       break;
@@ -160,7 +160,7 @@ int64_t DataExtractor::getSLEB128(uint32_t *offset_ptr) const {
 
   while (isValidOffset(offset)) {
     byte = Data[offset++];
-    result |= (byte & 0x7f) << shift;
+    result |= uint64_t(byte & 0x7f) << shift;
     shift += 7;
     if ((byte & 0x80) == 0)
       break;
@@ -168,7 +168,7 @@ int64_t DataExtractor::getSLEB128(uint32_t *offset_ptr) const {
 
   // Sign bit of byte is 2nd high order bit (0x40)
   if (shift < 64 && (byte & 0x40))
-    result |= -(1 << shift);
+    result |= -(1ULL << shift);
 
   *offset_ptr = offset;
   return result;
diff --git a/lib/Support/DataStream.cpp b/lib/Support/DataStream.cpp
index 94d14a5..3a38e2a 100644
--- a/lib/Support/DataStream.cpp
+++ b/lib/Support/DataStream.cpp
@@ -58,7 +58,7 @@ public:
   virtual ~DataFileStreamer() {
     close(Fd);
   }
-  virtual size_t GetBytes(unsigned char *buf, size_t len) {
+  virtual size_t GetBytes(unsigned char *buf, size_t len) LLVM_OVERRIDE {
     NumStreamFetches++;
     return read(Fd, buf, len);
   }
diff --git a/lib/Support/DynamicLibrary.cpp b/lib/Support/DynamicLibrary.cpp
index fb02c07..45fec36 100644
--- a/lib/Support/DynamicLibrary.cpp
+++ b/lib/Support/DynamicLibrary.cpp
@@ -160,7 +160,7 @@ void* DynamicLibrary::SearchForAddressOfSymbol(const char *symbolName) {
 // On linux we have a weird situation. The stderr/out/in symbols are both
 // macros and global variables because of standards requirements. So, we
 // boldly use the EXPLICIT_SYMBOL macro without checking for a #define first.
-#if defined(__linux__)
+#if defined(__linux__) and !defined(__ANDROID__)
   {
     EXPLICIT_SYMBOL(stderr);
     EXPLICIT_SYMBOL(stdout);
diff --git a/lib/Support/Errno.cpp b/lib/Support/Errno.cpp
index dd218f6..730220f 100644
--- a/lib/Support/Errno.cpp
+++ b/lib/Support/Errno.cpp
@@ -13,6 +13,7 @@
 
 #include "llvm/Support/Errno.h"
 #include "llvm/Config/config.h"     // Get autoconf configuration settings
+#include "llvm/Support/raw_ostream.h"
 
 #if HAVE_STRING_H
 #include <string.h>
@@ -39,7 +40,7 @@ std::string StrError(int errnum) {
   const int MaxErrStrLen = 2000;
   char buffer[MaxErrStrLen];
   buffer[0] = '\0';
-  char* str = buffer;
+  std::string str;
 #ifdef HAVE_STRERROR_R
   // strerror_r is thread-safe.
   if (errnum)
@@ -49,21 +50,25 @@ std::string StrError(int errnum) {
     str = strerror_r(errnum,buffer,MaxErrStrLen-1);
 # else
     strerror_r(errnum,buffer,MaxErrStrLen-1);
+    str = buffer;
 # endif
 #elif HAVE_DECL_STRERROR_S // "Windows Secure API"
-    if (errnum)
+    if (errnum) {
       strerror_s(buffer, MaxErrStrLen - 1, errnum);
+      str = buffer;
+    }
 #elif defined(HAVE_STRERROR)
   // Copy the thread un-safe result of strerror into
   // the buffer as fast as possible to minimize impact
   // of collision of strerror in multiple threads.
   if (errnum)
-    strncpy(buffer,strerror(errnum),MaxErrStrLen-1);
-  buffer[MaxErrStrLen-1] = '\0';
+    str = strerror(errnum);
 #else
   // Strange that this system doesn't even have strerror
   // but, oh well, just use a generic message
-  sprintf(buffer, "Error #%d", errnum);
+  raw_string_ostream stream(str);
+  stream << "Error #" << errnum;
+  stream.flush();
 #endif
   return str;
 }
diff --git a/lib/Support/FoldingSet.cpp b/lib/Support/FoldingSet.cpp
index c6282c6..4d489a8 100644
--- a/lib/Support/FoldingSet.cpp
+++ b/lib/Support/FoldingSet.cpp
@@ -38,6 +38,14 @@ bool FoldingSetNodeIDRef::operator==(FoldingSetNodeIDRef RHS) const {
   return memcmp(Data, RHS.Data, Size*sizeof(*Data)) == 0;
 }
 
+/// Used to compare the "ordering" of two nodes as defined by the
+/// profiled bits and their ordering defined by memcmp().
+bool FoldingSetNodeIDRef::operator<(FoldingSetNodeIDRef RHS) const {
+  if (Size != RHS.Size)
+    return Size < RHS.Size;
+  return memcmp(Data, RHS.Data, Size*sizeof(*Data)) < 0;
+}
+
 //===----------------------------------------------------------------------===//
 // FoldingSetNodeID Implementation
 
@@ -152,6 +160,16 @@ bool FoldingSetNodeID::operator==(FoldingSetNodeIDRef RHS) const {
   return FoldingSetNodeIDRef(Bits.data(), Bits.size()) == RHS;
 }
 
+/// Used to compare the "ordering" of two nodes as defined by the
+/// profiled bits and their ordering defined by memcmp().
+bool FoldingSetNodeID::operator<(const FoldingSetNodeID &RHS)const{
+  return *this < FoldingSetNodeIDRef(RHS.Bits.data(), RHS.Bits.size());
+}
+
+bool FoldingSetNodeID::operator<(FoldingSetNodeIDRef RHS) const {
+  return FoldingSetNodeIDRef(Bits.data(), Bits.size()) < RHS;
+}
+
 /// Intern - Copy this node's data to a memory region allocated from the
 /// given allocator and return a FoldingSetNodeIDRef describing the
 /// interned data.
diff --git a/lib/Support/Host.cpp b/lib/Support/Host.cpp
index 9a2c39d..34e32b8 100644
--- a/lib/Support/Host.cpp
+++ b/lib/Support/Host.cpp
@@ -234,6 +234,8 @@ std::string sys::getHostCPUName() {
       case 37: // Intel Core i7, laptop version.
       case 44: // Intel Core i7 processor and Intel Xeon processor. All
                // processors are manufactured using the 32 nm process.
+      case 46: // Nehalem EX
+      case 47: // Westmere EX
         return "corei7";
 
       // SandyBridge:
@@ -303,6 +305,7 @@ std::string sys::getHostCPUName() {
         case 8:  return "k6-2";
         case 9:
         case 13: return "k6-3";
+        case 10: return "geode";
         default: return "pentium";
         }
       case 6:
@@ -500,6 +503,7 @@ std::string sys::getHostCPUName() {
           .Case("0xb76", "arm1176jz-s")
           .Case("0xc08", "cortex-a8")
           .Case("0xc09", "cortex-a9")
+          .Case("0xc0f", "cortex-a15")
           .Case("0xc20", "cortex-m0")
           .Case("0xc23", "cortex-m3")
           .Case("0xc24", "cortex-m4")
diff --git a/lib/Support/LockFileManager.cpp b/lib/Support/LockFileManager.cpp
index 64404a1..59bfcfc 100644
--- a/lib/Support/LockFileManager.cpp
+++ b/lib/Support/LockFileManager.cpp
@@ -49,7 +49,7 @@ LockFileManager::readLockFile(StringRef LockFileName) {
 }
 
 bool LockFileManager::processStillExecuting(StringRef Hostname, int PID) {
-#if LLVM_ON_UNIX
+#if LLVM_ON_UNIX && !defined(__ANDROID__)
   char MyHostname[256];
   MyHostname[255] = 0;
   MyHostname[0] = 0;
diff --git a/lib/Support/Makefile b/lib/Support/Makefile
index d68e500..4a2185d 100644
--- a/lib/Support/Makefile
+++ b/lib/Support/Makefile
@@ -11,9 +11,6 @@ LEVEL = ../..
 LIBRARYNAME = LLVMSupport
 BUILD_ARCHIVE = 1
 
-## FIXME: This only requires RTTI because tblgen uses it.  Fix that.
-REQUIRES_RTTI = 1
-
 EXTRA_DIST = Unix Win32 README.txt
 
 include $(LEVEL)/Makefile.common
diff --git a/lib/Support/Memory.cpp b/lib/Support/Memory.cpp
index 22f7494..12f0838 100644
--- a/lib/Support/Memory.cpp
+++ b/lib/Support/Memory.cpp
@@ -16,14 +16,6 @@
 #include "llvm/Support/Valgrind.h"
 #include "llvm/Config/config.h"
 
-#if defined(__mips__)
-#include <sys/cachectl.h>
-#endif
-
-namespace llvm {
-using namespace sys;
-}
-
 // Include the platform-specific parts of this class.
 #ifdef LLVM_ON_UNIX
 #include "Unix/Memory.inc"
@@ -31,51 +23,3 @@ using namespace sys;
 #ifdef LLVM_ON_WIN32
 #include "Windows/Memory.inc"
 #endif
-
-extern "C" void sys_icache_invalidate(const void *Addr, size_t len);
-
-/// InvalidateInstructionCache - Before the JIT can run a block of code
-/// that has been emitted it must invalidate the instruction cache on some
-/// platforms.
-void llvm::sys::Memory::InvalidateInstructionCache(const void *Addr,
-                                                   size_t Len) {
-
-// icache invalidation for PPC and ARM.
-#if defined(__APPLE__)
-
-#  if (defined(__POWERPC__) || defined (__ppc__) || \
-     defined(_POWER) || defined(_ARCH_PPC)) || defined(__arm__)
-  sys_icache_invalidate(const_cast<void *>(Addr), Len);
-#  endif
-
-#else
-
-#  if (defined(__POWERPC__) || defined (__ppc__) || \
-       defined(_POWER) || defined(_ARCH_PPC)) && defined(__GNUC__)
-  const size_t LineSize = 32;
-
-  const intptr_t Mask = ~(LineSize - 1);
-  const intptr_t StartLine = ((intptr_t) Addr) & Mask;
-  const intptr_t EndLine = ((intptr_t) Addr + Len + LineSize - 1) & Mask;
-
-  for (intptr_t Line = StartLine; Line < EndLine; Line += LineSize)
-    asm volatile("dcbf 0, %0" : : "r"(Line));
-  asm volatile("sync");
-
-  for (intptr_t Line = StartLine; Line < EndLine; Line += LineSize)
-    asm volatile("icbi 0, %0" : : "r"(Line));
-  asm volatile("isync");
-#  elif defined(__arm__) && defined(__GNUC__)
-  // FIXME: Can we safely always call this for __GNUC__ everywhere?
-  const char *Start = static_cast<const char *>(Addr);
-  const char *End = Start + Len;
-  __clear_cache(const_cast<char *>(Start), const_cast<char *>(End));
-#  elif defined(__mips__)
-  const char *Start = static_cast<const char *>(Addr);
-  cacheflush(const_cast<char *>(Start), Len, BCACHE);
-#  endif
-
-#endif  // end apple
-
-  ValgrindDiscardTranslations(Addr, Len);
-}
diff --git a/lib/Support/MemoryBuffer.cpp b/lib/Support/MemoryBuffer.cpp
index 992f03c..ec373e7 100644
--- a/lib/Support/MemoryBuffer.cpp
+++ b/lib/Support/MemoryBuffer.cpp
@@ -33,6 +33,9 @@
 #include <unistd.h>
 #else
 #include <io.h>
+#ifndef S_ISFIFO
+#define S_ISFIFO(x) (0)
+#endif
 #endif
 #include <fcntl.h>
 using namespace llvm;
@@ -81,12 +84,12 @@ public:
     init(InputData.begin(), InputData.end(), RequiresNullTerminator);
   }
 
-  virtual const char *getBufferIdentifier() const {
+  virtual const char *getBufferIdentifier() const LLVM_OVERRIDE {
      // The name is stored after the class itself.
     return reinterpret_cast<const char*>(this + 1);
   }
-  
-  virtual BufferKind getBufferKind() const {
+
+  virtual BufferKind getBufferKind() const LLVM_OVERRIDE {
     return MemoryBuffer_Malloc;
   }
 };
@@ -194,13 +197,34 @@ public:
     sys::Path::UnMapFilePages(reinterpret_cast<const char*>(RealStart),
                               RealSize);
   }
-  
-  virtual BufferKind getBufferKind() const {
+
+  virtual BufferKind getBufferKind() const LLVM_OVERRIDE {
     return MemoryBuffer_MMap;
   }
 };
 }
 
+static error_code getMemoryBufferForStream(int FD, 
+                                           StringRef BufferName,
+                                           OwningPtr<MemoryBuffer> &result) {
+  const ssize_t ChunkSize = 4096*4;
+  SmallString<ChunkSize> Buffer;
+  ssize_t ReadBytes;
+  // Read into Buffer until we hit EOF.
+  do {
+    Buffer.reserve(Buffer.size() + ChunkSize);
+    ReadBytes = read(FD, Buffer.end(), ChunkSize);
+    if (ReadBytes == -1) {
+      if (errno == EINTR) continue;
+      return error_code(errno, posix_category());
+    }
+    Buffer.set_size(Buffer.size() + ReadBytes);
+  } while (ReadBytes != 0);
+
+  result.reset(MemoryBuffer::getMemBufferCopy(Buffer, BufferName));
+  return error_code::success();
+}
+
 error_code MemoryBuffer::getFile(StringRef Filename,
                                  OwningPtr<MemoryBuffer> &result,
                                  int64_t FileSize,
@@ -297,6 +321,13 @@ error_code MemoryBuffer::getOpenFile(int FD, const char *Filename,
       if (fstat(FD, &FileInfo) == -1) {
         return error_code(errno, posix_category());
       }
+
+      // If this is a named pipe, we can't trust the size. Create the memory
+      // buffer by copying off the stream.
+      if (S_ISFIFO(FileInfo.st_mode)) {
+        return getMemoryBufferForStream(FD, Filename, result);
+      }
+
       FileSize = FileInfo.st_size;
     }
     MapSize = FileSize;
@@ -370,20 +401,5 @@ error_code MemoryBuffer::getSTDIN(OwningPtr<MemoryBuffer> &result) {
   // fallback if it fails.
   sys::Program::ChangeStdinToBinary();
 
-  const ssize_t ChunkSize = 4096*4;
-  SmallString<ChunkSize> Buffer;
-  ssize_t ReadBytes;
-  // Read into Buffer until we hit EOF.
-  do {
-    Buffer.reserve(Buffer.size() + ChunkSize);
-    ReadBytes = read(0, Buffer.end(), ChunkSize);
-    if (ReadBytes == -1) {
-      if (errno == EINTR) continue;
-      return error_code(errno, posix_category());
-    }
-    Buffer.set_size(Buffer.size() + ReadBytes);
-  } while (ReadBytes != 0);
-
-  result.reset(getMemBufferCopy(Buffer, "<stdin>"));
-  return error_code::success();
+  return getMemoryBufferForStream(0, "<stdin>", result);
 }
diff --git a/lib/Support/SmallVector.cpp b/lib/Support/SmallVector.cpp
index a89f149..f9c0e78 100644
--- a/lib/Support/SmallVector.cpp
+++ b/lib/Support/SmallVector.cpp
@@ -16,14 +16,15 @@ using namespace llvm;
 
 /// grow_pod - This is an implementation of the grow() method which only works
 /// on POD-like datatypes and is out of line to reduce code duplication.
-void SmallVectorBase::grow_pod(size_t MinSizeInBytes, size_t TSize) {
+void SmallVectorBase::grow_pod(void *FirstEl, size_t MinSizeInBytes,
+                               size_t TSize) {
   size_t CurSizeBytes = size_in_bytes();
   size_t NewCapacityInBytes = 2 * capacity_in_bytes() + TSize; // Always grow.
   if (NewCapacityInBytes < MinSizeInBytes)
     NewCapacityInBytes = MinSizeInBytes;
 
   void *NewElts;
-  if (this->isSmall()) {
+  if (BeginX == FirstEl) {
     NewElts = malloc(NewCapacityInBytes);
 
     // Copy the elements over.  No need to run dtors on PODs.
@@ -37,4 +38,3 @@ void SmallVectorBase::grow_pod(size_t MinSizeInBytes, size_t TSize) {
   this->BeginX = NewElts;
   this->CapacityX = (char*)this->BeginX + NewCapacityInBytes;
 }
-
diff --git a/lib/Support/StreamableMemoryObject.cpp b/lib/Support/StreamableMemoryObject.cpp
index fe3752a..59e27a2 100644
--- a/lib/Support/StreamableMemoryObject.cpp
+++ b/lib/Support/StreamableMemoryObject.cpp
@@ -8,6 +8,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Support/StreamableMemoryObject.h"
+#include "llvm/Support/Compiler.h"
 #include <cassert>
 #include <cstring>
 
@@ -23,18 +24,23 @@ public:
     assert(LastChar >= FirstChar && "Invalid start/end range");
   }
 
-  virtual uint64_t getBase() const { return 0; }
-  virtual uint64_t getExtent() const { return LastChar - FirstChar; }
-  virtual int readByte(uint64_t address, uint8_t* ptr) const;
+  virtual uint64_t getBase() const LLVM_OVERRIDE { return 0; }
+  virtual uint64_t getExtent() const LLVM_OVERRIDE {
+    return LastChar - FirstChar;
+  }
+  virtual int readByte(uint64_t address, uint8_t* ptr) const LLVM_OVERRIDE;
   virtual int readBytes(uint64_t address,
                         uint64_t size,
                         uint8_t* buf,
-                        uint64_t* copied) const;
-  virtual const uint8_t *getPointer(uint64_t address, uint64_t size) const;
-  virtual bool isValidAddress(uint64_t address) const {
+                        uint64_t* copied) const LLVM_OVERRIDE;
+  virtual const uint8_t *getPointer(uint64_t address,
+                                    uint64_t size) const LLVM_OVERRIDE;
+  virtual bool isValidAddress(uint64_t address) const LLVM_OVERRIDE {
     return validAddress(address);
   }
-  virtual bool isObjectEnd(uint64_t address) const {return objectEnd(address);}
+  virtual bool isObjectEnd(uint64_t address) const LLVM_OVERRIDE {
+    return objectEnd(address);
+  }
 
 private:
   const uint8_t* const FirstChar;
@@ -49,8 +55,8 @@ private:
     return static_cast<ptrdiff_t>(address) == LastChar - FirstChar;
   }
 
-  RawMemoryObject(const RawMemoryObject&);  // DO NOT IMPLEMENT
-  void operator=(const RawMemoryObject&);  // DO NOT IMPLEMENT
+  RawMemoryObject(const RawMemoryObject&) LLVM_DELETED_FUNCTION;
+  void operator=(const RawMemoryObject&) LLVM_DELETED_FUNCTION;
 };
 
 int RawMemoryObject::readByte(uint64_t address, uint8_t* ptr) const {
diff --git a/lib/Support/StringMap.cpp b/lib/Support/StringMap.cpp
index c2fc261..9ac1f86 100644
--- a/lib/Support/StringMap.cpp
+++ b/lib/Support/StringMap.cpp
@@ -13,6 +13,7 @@
 
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/Support/Compiler.h"
 #include <cassert>
 using namespace llvm;
 
@@ -69,7 +70,7 @@ unsigned StringMapImpl::LookupBucketFor(StringRef Name) {
   while (1) {
     StringMapEntryBase *BucketItem = TheTable[BucketNo];
     // If we found an empty bucket, this key isn't in the table yet, return it.
-    if (BucketItem == 0) {
+    if (LLVM_LIKELY(BucketItem == 0)) {
       // If we found a tombstone, we want to reuse the tombstone instead of an
       // empty bucket.  This reduces probing.
       if (FirstTombstone != -1) {
@@ -84,7 +85,7 @@ unsigned StringMapImpl::LookupBucketFor(StringRef Name) {
     if (BucketItem == getTombstoneVal()) {
       // Skip over tombstones.  However, remember the first one we see.
       if (FirstTombstone == -1) FirstTombstone = BucketNo;
-    } else if (HashTable[BucketNo] == FullHashValue) {
+    } else if (LLVM_LIKELY(HashTable[BucketNo] == FullHashValue)) {
       // If the full hash value matches, check deeply for a match.  The common
       // case here is that we are only looking at the buckets (for item info
       // being non-null and for the full hash value) not at the items.  This
@@ -123,12 +124,12 @@ int StringMapImpl::FindKey(StringRef Key) const {
   while (1) {
     StringMapEntryBase *BucketItem = TheTable[BucketNo];
     // If we found an empty bucket, this key isn't in the table yet, return.
-    if (BucketItem == 0)
+    if (LLVM_LIKELY(BucketItem == 0))
       return -1;
     
     if (BucketItem == getTombstoneVal()) {
       // Ignore tombstones.
-    } else if (HashTable[BucketNo] == FullHashValue) {
+    } else if (LLVM_LIKELY(HashTable[BucketNo] == FullHashValue)) {
       // If the full hash value matches, check deeply for a match.  The common
       // case here is that we are only looking at the buckets (for item info
       // being non-null and for the full hash value) not at the items.  This
diff --git a/lib/Support/StringRef.cpp b/lib/Support/StringRef.cpp
index 8aab4b2..f8e9208 100644
--- a/lib/Support/StringRef.cpp
+++ b/lib/Support/StringRef.cpp
@@ -350,8 +350,8 @@ bool llvm::getAsUnsignedInteger(StringRef Str, unsigned Radix,
     unsigned long long PrevResult = Result;
     Result = Result*Radix+CharVal;
 
-    // Check for overflow.
-    if (Result < PrevResult)
+    // Check for overflow by shifting back and seeing if bits were lost.
+    if (Result/Radix < PrevResult)
       return true;
 
     Str = Str.substr(1);
diff --git a/lib/Support/Triple.cpp b/lib/Support/Triple.cpp
index cca549d..c058c05 100644
--- a/lib/Support/Triple.cpp
+++ b/lib/Support/Triple.cpp
@@ -42,6 +42,8 @@ const char *Triple::getArchTypeName(ArchType Kind) {
   case nvptx64: return "nvptx64";
   case le32:    return "le32";
   case amdil:   return "amdil";
+  case spir:    return "spir";
+  case spir64:  return "spir64";
   }
 
   llvm_unreachable("Invalid ArchType!");
@@ -83,6 +85,8 @@ const char *Triple::getArchTypePrefix(ArchType Kind) {
   case nvptx64: return "nvptx";
   case le32:    return "le32";
   case amdil:   return "amdil";
+  case spir:    return "spir";
+  case spir64:  return "spir";
   }
 }
 
@@ -95,6 +99,8 @@ const char *Triple::getVendorTypeName(VendorType Kind) {
   case SCEI: return "scei";
   case BGP: return "bgp";
   case BGQ: return "bgq";
+  case Freescale: return "fsl";
+  case IBM: return "ibm";
   }
 
   llvm_unreachable("Invalid VendorType!");
@@ -125,6 +131,7 @@ const char *Triple::getOSTypeName(OSType Kind) {
   case NativeClient: return "nacl";
   case CNK: return "cnk";
   case Bitrig: return "bitrig";
+  case AIX: return "aix";
   }
 
   llvm_unreachable("Invalid OSType");
@@ -138,7 +145,8 @@ const char *Triple::getEnvironmentTypeName(EnvironmentType Kind) {
   case GNUEABI: return "gnueabi";
   case EABI: return "eabi";
   case MachO: return "macho";
-  case ANDROIDEABI: return "androideabi";
+  case Android: return "android";
+  case ELF: return "elf";
   }
 
   llvm_unreachable("Invalid EnvironmentType!");
@@ -170,40 +178,11 @@ Triple::ArchType Triple::getArchTypeForLLVMName(StringRef Name) {
     .Case("nvptx64", nvptx64)
     .Case("le32", le32)
     .Case("amdil", amdil)
+    .Case("spir", spir)
+    .Case("spir64", spir64)
     .Default(UnknownArch);
 }
 
-Triple::ArchType Triple::getArchTypeForDarwinArchName(StringRef Str) {
-  // See arch(3) and llvm-gcc's driver-driver.c. We don't implement support for
-  // archs which Darwin doesn't use.
-
-  // The matching this routine does is fairly pointless, since it is neither the
-  // complete architecture list, nor a reasonable subset. The problem is that
-  // historically the driver driver accepts this and also ties its -march=
-  // handling to the architecture name, so we need to be careful before removing
-  // support for it.
-
-  // This code must be kept in sync with Clang's Darwin specific argument
-  // translation.
-
-  return StringSwitch<ArchType>(Str)
-    .Cases("ppc", "ppc601", "ppc603", "ppc604", "ppc604e", Triple::ppc)
-    .Cases("ppc750", "ppc7400", "ppc7450", "ppc970", Triple::ppc)
-    .Case("ppc64", Triple::ppc64)
-    .Cases("i386", "i486", "i486SX", "i586", "i686", Triple::x86)
-    .Cases("pentium", "pentpro", "pentIIm3", "pentIIm5", "pentium4",
-           Triple::x86)
-    .Case("x86_64", Triple::x86_64)
-    // This is derived from the driver driver.
-    .Cases("arm", "armv4t", "armv5", "armv6", Triple::arm)
-    .Cases("armv7", "armv7f", "armv7k", "armv7s", "xscale", Triple::arm)
-    .Case("r600", Triple::r600)
-    .Case("nvptx", Triple::nvptx)
-    .Case("nvptx64", Triple::nvptx64)
-    .Case("amdil", Triple::amdil)
-    .Default(Triple::UnknownArch);
-}
-
 // Returns architecture name that is understood by the target assembler.
 const char *Triple::getArchNameForAssembler() {
   if (!isOSDarwin() && getVendor() != Triple::Apple)
@@ -225,6 +204,8 @@ const char *Triple::getArchNameForAssembler() {
     .Case("nvptx64", "nvptx64")
     .Case("le32", "le32")
     .Case("amdil", "amdil")
+    .Case("spir", "spir")
+    .Case("spir64", "spir64")
     .Default(NULL);
 }
 
@@ -259,6 +240,8 @@ static Triple::ArchType parseArch(StringRef ArchName) {
     .Case("nvptx64", Triple::nvptx64)
     .Case("le32", Triple::le32)
     .Case("amdil", Triple::amdil)
+    .Case("spir", Triple::spir)
+    .Case("spir64", Triple::spir64)
     .Default(Triple::UnknownArch);
 }
 
@@ -269,6 +252,8 @@ static Triple::VendorType parseVendor(StringRef VendorName) {
     .Case("scei", Triple::SCEI)
     .Case("bgp", Triple::BGP)
     .Case("bgq", Triple::BGQ)
+    .Case("fsl", Triple::Freescale)
+    .Case("ibm", Triple::IBM)
     .Default(Triple::UnknownVendor);
 }
 
@@ -295,6 +280,7 @@ static Triple::OSType parseOS(StringRef OSName) {
     .StartsWith("nacl", Triple::NativeClient)
     .StartsWith("cnk", Triple::CNK)
     .StartsWith("bitrig", Triple::Bitrig)
+    .StartsWith("aix", Triple::AIX)
     .Default(Triple::UnknownOS);
 }
 
@@ -305,7 +291,8 @@ static Triple::EnvironmentType parseEnvironment(StringRef EnvironmentName) {
     .StartsWith("gnueabi", Triple::GNUEABI)
     .StartsWith("gnu", Triple::GNU)
     .StartsWith("macho", Triple::MachO)
-    .StartsWith("androideabi", Triple::ANDROIDEABI)
+    .StartsWith("android", Triple::Android)
+    .StartsWith("elf", Triple::ELF)
     .Default(Triple::UnknownEnvironment);
 }
 
@@ -690,6 +677,7 @@ static unsigned getArchPointerBitWidth(llvm::Triple::ArchType Arch) {
   case llvm::Triple::thumb:
   case llvm::Triple::x86:
   case llvm::Triple::xcore:
+  case llvm::Triple::spir:
     return 32;
 
   case llvm::Triple::mips64:
@@ -698,6 +686,7 @@ static unsigned getArchPointerBitWidth(llvm::Triple::ArchType Arch) {
   case llvm::Triple::ppc64:
   case llvm::Triple::sparcv9:
   case llvm::Triple::x86_64:
+  case llvm::Triple::spir64:
     return 64;
   }
   llvm_unreachable("Invalid architecture value");
@@ -724,6 +713,7 @@ Triple Triple::get32BitArchVariant() const {
     break;
 
   case Triple::amdil:
+  case Triple::spir:
   case Triple::arm:
   case Triple::cellspu:
   case Triple::hexagon:
@@ -748,6 +738,7 @@ Triple Triple::get32BitArchVariant() const {
   case Triple::ppc64:     T.setArch(Triple::ppc);   break;
   case Triple::sparcv9:   T.setArch(Triple::sparc);   break;
   case Triple::x86_64:    T.setArch(Triple::x86);     break;
+  case Triple::spir64:    T.setArch(Triple::spir);    break;
   }
   return T;
 }
@@ -770,6 +761,7 @@ Triple Triple::get64BitArchVariant() const {
     T.setArch(UnknownArch);
     break;
 
+  case Triple::spir64:
   case Triple::mips64:
   case Triple::mips64el:
   case Triple::nvptx64:
@@ -785,6 +777,7 @@ Triple Triple::get64BitArchVariant() const {
   case Triple::ppc:     T.setArch(Triple::ppc64);     break;
   case Triple::sparc:   T.setArch(Triple::sparcv9);   break;
   case Triple::x86:     T.setArch(Triple::x86_64);    break;
+  case Triple::spir:    T.setArch(Triple::spir64);    break;
   }
   return T;
 }
diff --git a/lib/Support/Unix/Memory.inc b/lib/Support/Unix/Memory.inc
index 5a57a28..9a8abd2 100644
--- a/lib/Support/Unix/Memory.inc
+++ b/lib/Support/Unix/Memory.inc
@@ -13,6 +13,7 @@
 
 #include "Unix.h"
 #include "llvm/Support/DataTypes.h"
+#include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/Process.h"
 
 #ifdef HAVE_SYS_MMAN_H
@@ -23,14 +24,146 @@
 #include <mach/mach.h>
 #endif
 
+#if defined(__mips__)
+#  if defined(__OpenBSD__)
+#    include <mips64/sysarch.h>
+#  else
+#    include <sys/cachectl.h>
+#  endif
+#endif
+
+extern "C" void sys_icache_invalidate(const void *Addr, size_t len);
+
+namespace {
+
+int getPosixProtectionFlags(unsigned Flags) {
+  switch (Flags) {
+  case llvm::sys::Memory::MF_READ:
+    return PROT_READ;
+  case llvm::sys::Memory::MF_WRITE:
+    return PROT_WRITE;
+  case llvm::sys::Memory::MF_READ|llvm::sys::Memory::MF_WRITE:
+    return PROT_READ | PROT_WRITE;
+  case llvm::sys::Memory::MF_READ|llvm::sys::Memory::MF_EXEC:
+    return PROT_READ | PROT_EXEC;
+  case llvm::sys::Memory::MF_READ |
+	 llvm::sys::Memory::MF_WRITE |
+	 llvm::sys::Memory::MF_EXEC:
+    return PROT_READ | PROT_WRITE | PROT_EXEC;
+  case llvm::sys::Memory::MF_EXEC:
+    return PROT_EXEC;
+  default:
+    llvm_unreachable("Illegal memory protection flag specified!");
+  }
+  // Provide a default return value as required by some compilers.
+  return PROT_NONE;
+}
+
+} // namespace
+
+namespace llvm {
+namespace sys {
+
+MemoryBlock
+Memory::allocateMappedMemory(size_t NumBytes,
+                             const MemoryBlock *const NearBlock,
+                             unsigned PFlags,
+                             error_code &EC) {
+  EC = error_code::success();
+  if (NumBytes == 0)
+    return MemoryBlock();
+
+  static const size_t PageSize = Process::GetPageSize();
+  const size_t NumPages = (NumBytes+PageSize-1)/PageSize;
+
+  int fd = -1;
+#ifdef NEED_DEV_ZERO_FOR_MMAP
+  static int zero_fd = open("/dev/zero", O_RDWR);
+  if (zero_fd == -1) {
+    EC = error_code(errno, system_category());
+    return MemoryBlock();
+  }
+  fd = zero_fd;
+#endif
+
+  int MMFlags = MAP_PRIVATE |
+#ifdef HAVE_MMAP_ANONYMOUS
+  MAP_ANONYMOUS
+#else
+  MAP_ANON
+#endif
+  ; // Ends statement above
+
+  int Protect = getPosixProtectionFlags(PFlags);
+
+  // Use any near hint and the page size to set a page-aligned starting address
+  uintptr_t Start = NearBlock ? reinterpret_cast<uintptr_t>(NearBlock->base()) +
+                                      NearBlock->size() : 0;
+  if (Start && Start % PageSize)
+    Start += PageSize - Start % PageSize;
+
+  void *Addr = ::mmap(reinterpret_cast<void*>(Start), PageSize*NumPages,
+                      Protect, MMFlags, fd, 0);
+  if (Addr == MAP_FAILED) {
+    if (NearBlock) //Try again without a near hint
+      return allocateMappedMemory(NumBytes, 0, PFlags, EC);
+
+    EC = error_code(errno, system_category());
+    return MemoryBlock();
+  }
+
+  MemoryBlock Result;
+  Result.Address = Addr;
+  Result.Size = NumPages*PageSize;
+
+  if (PFlags & MF_EXEC)
+    Memory::InvalidateInstructionCache(Result.Address, Result.Size);
+
+  return Result;
+}
+
+error_code
+Memory::releaseMappedMemory(MemoryBlock &M) {
+  if (M.Address == 0 || M.Size == 0)
+    return error_code::success();
+
+  if (0 != ::munmap(M.Address, M.Size))
+    return error_code(errno, system_category());
+
+  M.Address = 0;
+  M.Size = 0;
+
+  return error_code::success();
+}
+
+error_code
+Memory::protectMappedMemory(const MemoryBlock &M, unsigned Flags) {
+  if (M.Address == 0 || M.Size == 0)
+    return error_code::success();
+
+  if (!Flags)
+    return error_code(EINVAL, generic_category());
+
+  int Protect = getPosixProtectionFlags(Flags);
+
+  int Result = ::mprotect(M.Address, M.Size, Protect);
+  if (Result != 0)
+    return error_code(errno, system_category());
+
+  if (Flags & MF_EXEC)
+    Memory::InvalidateInstructionCache(M.Address, M.Size);
+
+  return error_code::success();
+}
+
 /// AllocateRWX - Allocate a slab of memory with read/write/execute
 /// permissions.  This is typically used for JIT applications where we want
 /// to emit code to the memory then jump to it.  Getting this type of memory
 /// is very OS specific.
 ///
-llvm::sys::MemoryBlock
-llvm::sys::Memory::AllocateRWX(size_t NumBytes, const MemoryBlock* NearBlock,
-                               std::string *ErrMsg) {
+MemoryBlock
+Memory::AllocateRWX(size_t NumBytes, const MemoryBlock* NearBlock,
+                    std::string *ErrMsg) {
   if (NumBytes == 0) return MemoryBlock();
 
   size_t pageSize = Process::GetPageSize();
@@ -78,7 +211,7 @@ llvm::sys::Memory::AllocateRWX(size_t NumBytes, const MemoryBlock* NearBlock,
                                 VM_PROT_READ | VM_PROT_EXECUTE | VM_PROT_COPY);
   if (KERN_SUCCESS != kr) {
     MakeErrMsg(ErrMsg, "vm_protect max RX failed");
-    return sys::MemoryBlock();
+    return MemoryBlock();
   }
 
   kr = vm_protect(mach_task_self(), (vm_address_t)pa,
@@ -86,7 +219,7 @@ llvm::sys::Memory::AllocateRWX(size_t NumBytes, const MemoryBlock* NearBlock,
                   VM_PROT_READ | VM_PROT_WRITE);
   if (KERN_SUCCESS != kr) {
     MakeErrMsg(ErrMsg, "vm_protect RW failed");
-    return sys::MemoryBlock();
+    return MemoryBlock();
   }
 #endif
 
@@ -97,17 +230,17 @@ llvm::sys::Memory::AllocateRWX(size_t NumBytes, const MemoryBlock* NearBlock,
   return result;
 }
 
-bool llvm::sys::Memory::ReleaseRWX(MemoryBlock &M, std::string *ErrMsg) {
+bool Memory::ReleaseRWX(MemoryBlock &M, std::string *ErrMsg) {
   if (M.Address == 0 || M.Size == 0) return false;
   if (0 != ::munmap(M.Address, M.Size))
     return MakeErrMsg(ErrMsg, "Can't release RWX Memory");
   return false;
 }
 
-bool llvm::sys::Memory::setWritable (MemoryBlock &M, std::string *ErrMsg) {
+bool Memory::setWritable (MemoryBlock &M, std::string *ErrMsg) {
 #if defined(__APPLE__) && defined(__arm__)
   if (M.Address == 0 || M.Size == 0) return false;
-  sys::Memory::InvalidateInstructionCache(M.Address, M.Size);
+  Memory::InvalidateInstructionCache(M.Address, M.Size);
   kern_return_t kr = vm_protect(mach_task_self(), (vm_address_t)M.Address,
     (vm_size_t)M.Size, 0, VM_PROT_READ | VM_PROT_WRITE);
   return KERN_SUCCESS == kr;
@@ -116,10 +249,10 @@ bool llvm::sys::Memory::setWritable (MemoryBlock &M, std::string *ErrMsg) {
 #endif
 }
 
-bool llvm::sys::Memory::setExecutable (MemoryBlock &M, std::string *ErrMsg) {
+bool Memory::setExecutable (MemoryBlock &M, std::string *ErrMsg) {
 #if defined(__APPLE__) && defined(__arm__)
   if (M.Address == 0 || M.Size == 0) return false;
-  sys::Memory::InvalidateInstructionCache(M.Address, M.Size);
+  Memory::InvalidateInstructionCache(M.Address, M.Size);
   kern_return_t kr = vm_protect(mach_task_self(), (vm_address_t)M.Address,
     (vm_size_t)M.Size, 0, VM_PROT_READ | VM_PROT_EXECUTE | VM_PROT_COPY);
   return KERN_SUCCESS == kr;
@@ -128,7 +261,7 @@ bool llvm::sys::Memory::setExecutable (MemoryBlock &M, std::string *ErrMsg) {
 #endif
 }
 
-bool llvm::sys::Memory::setRangeWritable(const void *Addr, size_t Size) {
+bool Memory::setRangeWritable(const void *Addr, size_t Size) {
 #if defined(__APPLE__) && defined(__arm__)
   kern_return_t kr = vm_protect(mach_task_self(), (vm_address_t)Addr,
                                 (vm_size_t)Size, 0,
@@ -139,7 +272,7 @@ bool llvm::sys::Memory::setRangeWritable(const void *Addr, size_t Size) {
 #endif
 }
 
-bool llvm::sys::Memory::setRangeExecutable(const void *Addr, size_t Size) {
+bool Memory::setRangeExecutable(const void *Addr, size_t Size) {
 #if defined(__APPLE__) && defined(__arm__)
   kern_return_t kr = vm_protect(mach_task_self(), (vm_address_t)Addr,
                                 (vm_size_t)Size, 0,
@@ -149,3 +282,52 @@ bool llvm::sys::Memory::setRangeExecutable(const void *Addr, size_t Size) {
   return true;
 #endif
 }
+
+/// InvalidateInstructionCache - Before the JIT can run a block of code
+/// that has been emitted it must invalidate the instruction cache on some
+/// platforms.
+void Memory::InvalidateInstructionCache(const void *Addr,
+                                        size_t Len) {
+
+// icache invalidation for PPC and ARM.
+#if defined(__APPLE__)
+
+#  if (defined(__POWERPC__) || defined (__ppc__) || \
+     defined(_POWER) || defined(_ARCH_PPC)) || defined(__arm__)
+  sys_icache_invalidate(const_cast<void *>(Addr), Len);
+#  endif
+
+#else
+
+#  if (defined(__POWERPC__) || defined (__ppc__) || \
+       defined(_POWER) || defined(_ARCH_PPC)) && defined(__GNUC__)
+  const size_t LineSize = 32;
+
+  const intptr_t Mask = ~(LineSize - 1);
+  const intptr_t StartLine = ((intptr_t) Addr) & Mask;
+  const intptr_t EndLine = ((intptr_t) Addr + Len + LineSize - 1) & Mask;
+
+  for (intptr_t Line = StartLine; Line < EndLine; Line += LineSize)
+    asm volatile("dcbf 0, %0" : : "r"(Line));
+  asm volatile("sync");
+
+  for (intptr_t Line = StartLine; Line < EndLine; Line += LineSize)
+    asm volatile("icbi 0, %0" : : "r"(Line));
+  asm volatile("isync");
+#  elif defined(__arm__) && defined(__GNUC__)
+  // FIXME: Can we safely always call this for __GNUC__ everywhere?
+  const char *Start = static_cast<const char *>(Addr);
+  const char *End = Start + Len;
+  __clear_cache(const_cast<char *>(Start), const_cast<char *>(End));
+#  elif defined(__mips__)
+  const char *Start = static_cast<const char *>(Addr);
+  cacheflush(const_cast<char *>(Start), Len, BCACHE);
+#  endif
+
+#endif  // end apple
+
+  ValgrindDiscardTranslations(Addr, Len);
+}
+
+} // namespace sys
+} // namespace llvm
diff --git a/lib/Support/Unix/Path.inc b/lib/Support/Unix/Path.inc
index 6bddbdf..6a5ebb8 100644
--- a/lib/Support/Unix/Path.inc
+++ b/lib/Support/Unix/Path.inc
@@ -261,7 +261,8 @@ Path::GetCurrentDirectory() {
 }
 
 #if defined(__FreeBSD__) || defined (__NetBSD__) || defined(__Bitrig__) || \
-    defined(__OpenBSD__) || defined(__minix) || defined(__FreeBSD_kernel__)
+    defined(__OpenBSD__) || defined(__minix) || defined(__FreeBSD_kernel__) || \
+    defined(__linux__) || defined(__CYGWIN__)
 static int
 test_dir(char buf[PATH_MAX], char ret[PATH_MAX],
     const char *dir, const char *bin)
@@ -337,9 +338,17 @@ Path Path::GetMainExecutable(const char *argv0, void *MainAddr) {
     return Path(exe_path);
 #elif defined(__linux__) || defined(__CYGWIN__)
   char exe_path[MAXPATHLEN];
-  ssize_t len = readlink("/proc/self/exe", exe_path, sizeof(exe_path));
-  if (len >= 0)
-    return Path(StringRef(exe_path, len));
+  StringRef aPath("/proc/self/exe");
+  if (sys::fs::exists(aPath)) {
+      // /proc is not always mounted under Linux (chroot for example).
+      ssize_t len = readlink(aPath.str().c_str(), exe_path, sizeof(exe_path));
+      if (len >= 0)
+          return Path(StringRef(exe_path, len));
+  } else {
+      // Fall back to the classical detection.
+      if (getprogpath(exe_path, argv0) != NULL)
+          return Path(exe_path);
+  }
 #elif defined(HAVE_DLFCN_H)
   // Use dladdr to get executable path if available.
   Dl_info DLInfo;
diff --git a/lib/Support/Unix/Signals.inc b/lib/Support/Unix/Signals.inc
index 5195116..9e94068 100644
--- a/lib/Support/Unix/Signals.inc
+++ b/lib/Support/Unix/Signals.inc
@@ -121,17 +121,29 @@ static void UnregisterHandlers() {
 /// NB: This must be an async signal safe function. It cannot allocate or free
 /// memory, even in debug builds.
 static void RemoveFilesToRemove() {
-  // Note: avoid iterators in case of debug iterators that allocate or release
+  // We avoid iterators in case of debug iterators that allocate or release
   // memory.
   for (unsigned i = 0, e = FilesToRemove.size(); i != e; ++i) {
-    // Note that we don't want to use any external code here, and we don't care
-    // about errors. We're going to try as hard as we can as often as we need
-    // to to make these files go away. If these aren't files, too bad.
-    //
-    // We do however rely on a std::string implementation for which repeated
-    // calls to 'c_str()' don't allocate memory. We pre-call 'c_str()' on all
-    // of these strings to try to ensure this is safe.
-    unlink(FilesToRemove[i].c_str());
+    // We rely on a std::string implementation for which repeated calls to
+    // 'c_str()' don't allocate memory. We pre-call 'c_str()' on all of these
+    // strings to try to ensure this is safe.
+    const char *path = FilesToRemove[i].c_str();
+
+    // Get the status so we can determine if it's a file or directory. If we
+    // can't stat the file, ignore it.
+    struct stat buf;
+    if (stat(path, &buf) != 0)
+      continue;
+
+    // If this is not a regular file, ignore it. We want to prevent removal of
+    // special files like /dev/null, even if the compiler is being run with the
+    // super-user permissions.
+    if (!S_ISREG(buf.st_mode))
+      continue;
+  
+    // Otherwise, remove the file. We ignore any errors here as there is nothing
+    // else we can do.
+    unlink(path);
   }
 }
 
@@ -243,7 +255,7 @@ void llvm::sys::AddSignalHandler(void (*FnPtr)(void *), void *Cookie) {
 // On glibc systems we have the 'backtrace' function, which works nicely, but
 // doesn't demangle symbols.
 static void PrintStackTrace(void *) {
-#ifdef HAVE_BACKTRACE
+#if defined(HAVE_BACKTRACE) && defined(ENABLE_BACKTRACES)
   static void* StackTrace[256];
   // Use backtrace() to output a backtrace on Linux systems with glibc.
   int depth = backtrace(StackTrace,
@@ -293,7 +305,7 @@ static void PrintStackTrace(void *) {
 #endif
 }
 
-/// PrintStackTraceOnErrorSignal - When an error signal (such as SIBABRT or
+/// PrintStackTraceOnErrorSignal - When an error signal (such as SIGABRT or
 /// SIGSEGV) is delivered to the process, print a stack trace and then exit.
 void llvm::sys::PrintStackTraceOnErrorSignal() {
   AddSignalHandler(PrintStackTrace, 0);
@@ -305,10 +317,10 @@ void llvm::sys::PrintStackTraceOnErrorSignal() {
 
     exception_mask_t mask = EXC_MASK_CRASH;
 
-    kern_return_t ret = task_set_exception_ports(self, 
+    kern_return_t ret = task_set_exception_ports(self,
                              mask,
                              MACH_PORT_NULL,
-                             EXCEPTION_STATE_IDENTITY | MACH_EXCEPTION_CODES, 
+                             EXCEPTION_STATE_IDENTITY | MACH_EXCEPTION_CODES,
                              THREAD_STATE_NONE);
     (void)ret;
   }
diff --git a/lib/Support/Windows/Memory.inc b/lib/Support/Windows/Memory.inc
index fcc7283..cb80f28 100644
--- a/lib/Support/Windows/Memory.inc
+++ b/lib/Support/Windows/Memory.inc
@@ -12,51 +12,163 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "Windows.h"
 #include "llvm/Support/DataTypes.h"
+#include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/Process.h"
+#include "Windows.h"
+
+namespace {
+
+DWORD getWindowsProtectionFlags(unsigned Flags) {
+  switch (Flags) {
+  // Contrary to what you might expect, the Windows page protection flags
+  // are not a bitwise combination of RWX values
+  case llvm::sys::Memory::MF_READ:
+    return PAGE_READONLY;
+  case llvm::sys::Memory::MF_WRITE:
+    // Note: PAGE_WRITE is not supported by VirtualProtect
+    return PAGE_READWRITE;
+  case llvm::sys::Memory::MF_READ|llvm::sys::Memory::MF_WRITE:
+    return PAGE_READWRITE;
+  case llvm::sys::Memory::MF_READ|llvm::sys::Memory::MF_EXEC:
+    return PAGE_EXECUTE_READ;
+  case llvm::sys::Memory::MF_READ |
+         llvm::sys::Memory::MF_WRITE |
+         llvm::sys::Memory::MF_EXEC:
+    return PAGE_EXECUTE_READWRITE;
+  case llvm::sys::Memory::MF_EXEC:
+    return PAGE_EXECUTE;
+  default:
+    llvm_unreachable("Illegal memory protection flag specified!");
+  }
+  // Provide a default return value as required by some compilers.
+  return PAGE_NOACCESS;
+}
+
+size_t getAllocationGranularity() {
+  SYSTEM_INFO  Info;
+  ::GetSystemInfo(&Info);
+  if (Info.dwPageSize > Info.dwAllocationGranularity)
+    return Info.dwPageSize;
+  else
+    return Info.dwAllocationGranularity;
+}
+
+} // namespace
 
 namespace llvm {
-using namespace sys;
+namespace sys {
 
 //===----------------------------------------------------------------------===//
 //=== WARNING: Implementation here must contain only Win32 specific code
 //===          and must not be UNIX code
 //===----------------------------------------------------------------------===//
 
-MemoryBlock Memory::AllocateRWX(size_t NumBytes,
-                                const MemoryBlock *NearBlock,
-                                std::string *ErrMsg) {
-  if (NumBytes == 0) return MemoryBlock();
+MemoryBlock Memory::allocateMappedMemory(size_t NumBytes,
+                                         const MemoryBlock *const NearBlock,
+                                         unsigned Flags,
+                                         error_code &EC) {
+  EC = error_code::success();
+  if (NumBytes == 0)
+    return MemoryBlock();
+
+  // While we'd be happy to allocate single pages, the Windows allocation
+  // granularity may be larger than a single page (in practice, it is 64K)
+  // so mapping less than that will create an unreachable fragment of memory.
+  static const size_t Granularity = getAllocationGranularity();
+  const size_t NumBlocks = (NumBytes+Granularity-1)/Granularity;
 
-  static const size_t pageSize = Process::GetPageSize();
-  size_t NumPages = (NumBytes+pageSize-1)/pageSize;
+  uintptr_t Start = NearBlock ? reinterpret_cast<uintptr_t>(NearBlock->base()) +
+                                NearBlock->size()
+                           : NULL;
 
-  PVOID start = NearBlock ? static_cast<unsigned char *>(NearBlock->base()) +
-                                NearBlock->size() : NULL;
+  // If the requested address is not aligned to the allocation granularity,
+  // round up to get beyond NearBlock. VirtualAlloc would have rounded down.
+  if (Start && Start % Granularity != 0)
+    Start += Granularity - Start % Granularity;
 
-  void *pa = VirtualAlloc(start, NumPages*pageSize, MEM_RESERVE | MEM_COMMIT,
-                  PAGE_EXECUTE_READWRITE);
-  if (pa == NULL) {
+  DWORD Protect = getWindowsProtectionFlags(Flags);
+
+  void *PA = ::VirtualAlloc(reinterpret_cast<void*>(Start),
+                            NumBlocks*Granularity,
+                            MEM_RESERVE | MEM_COMMIT, Protect);
+  if (PA == NULL) {
     if (NearBlock) {
       // Try again without the NearBlock hint
-      return AllocateRWX(NumBytes, NULL, ErrMsg);
+      return allocateMappedMemory(NumBytes, NULL, Flags, EC);
     }
-    MakeErrMsg(ErrMsg, "Can't allocate RWX Memory: ");
+    EC = error_code(::GetLastError(), system_category());
     return MemoryBlock();
   }
 
-  MemoryBlock result;
-  result.Address = pa;
-  result.Size = NumPages*pageSize;
-  return result;
+  MemoryBlock Result;
+  Result.Address = PA;
+  Result.Size = NumBlocks*Granularity;
+                                 ;
+  if (Flags & MF_EXEC)
+    Memory::InvalidateInstructionCache(Result.Address, Result.Size);
+
+  return Result;
 }
 
-bool Memory::ReleaseRWX(MemoryBlock &M, std::string *ErrMsg) {
-  if (M.Address == 0 || M.Size == 0) return false;
+error_code Memory::releaseMappedMemory(MemoryBlock &M) {
+  if (M.Address == 0 || M.Size == 0)
+    return error_code::success();
+
   if (!VirtualFree(M.Address, 0, MEM_RELEASE))
-    return MakeErrMsg(ErrMsg, "Can't release RWX Memory: ");
-  return false;
+    return error_code(::GetLastError(), system_category());
+
+  M.Address = 0;
+  M.Size = 0;
+
+  return error_code::success();
+}
+
+error_code Memory::protectMappedMemory(const MemoryBlock &M,
+                                       unsigned Flags) {
+  if (M.Address == 0 || M.Size == 0)
+    return error_code::success();
+
+  DWORD Protect = getWindowsProtectionFlags(Flags);
+
+  DWORD OldFlags;
+  if (!VirtualProtect(M.Address, M.Size, Protect, &OldFlags))
+    return error_code(::GetLastError(), system_category());
+
+  if (Flags & MF_EXEC)
+    Memory::InvalidateInstructionCache(M.Address, M.Size);
+
+  return error_code::success();
+}
+
+/// InvalidateInstructionCache - Before the JIT can run a block of code
+/// that has been emitted it must invalidate the instruction cache on some
+/// platforms.
+void Memory::InvalidateInstructionCache(
+    const void *Addr, size_t Len) {
+  FlushInstructionCache(GetCurrentProcess(), Addr, Len);
+}
+
+
+MemoryBlock Memory::AllocateRWX(size_t NumBytes,
+                                const MemoryBlock *NearBlock,
+                                std::string *ErrMsg) {
+  MemoryBlock MB;
+  error_code EC;
+  MB = allocateMappedMemory(NumBytes, NearBlock,
+                            MF_READ|MF_WRITE|MF_EXEC, EC);
+  if (EC != error_code::success() && ErrMsg) {
+    MakeErrMsg(ErrMsg, EC.message());
+  }
+  return MB;
+}
+
+bool Memory::ReleaseRWX(MemoryBlock &M, std::string *ErrMsg) {
+  error_code EC = releaseMappedMemory(M);
+  if (EC == error_code::success())
+    return false;
+  MakeErrMsg(ErrMsg, EC.message());
+  return true;
 }
 
 static DWORD getProtection(const void *addr) {
@@ -93,7 +205,7 @@ bool Memory::setRangeWritable(const void *Addr, size_t Size) {
   }
 
   DWORD oldProt;
-  sys::Memory::InvalidateInstructionCache(Addr, Size);
+  Memory::InvalidateInstructionCache(Addr, Size);
   return ::VirtualProtect(const_cast<LPVOID>(Addr), Size, prot, &oldProt)
             == TRUE;
 }
@@ -112,9 +224,10 @@ bool Memory::setRangeExecutable(const void *Addr, size_t Size) {
   }
 
   DWORD oldProt;
-  sys::Memory::InvalidateInstructionCache(Addr, Size);
+  Memory::InvalidateInstructionCache(Addr, Size);
   return ::VirtualProtect(const_cast<LPVOID>(Addr), Size, prot, &oldProt)
             == TRUE;
 }
 
-}
+} // namespace sys
+} // namespace llvm
diff --git a/lib/Support/Windows/PathV2.inc b/lib/Support/Windows/PathV2.inc
index 696768b..3dfac66 100644
--- a/lib/Support/Windows/PathV2.inc
+++ b/lib/Support/Windows/PathV2.inc
@@ -794,7 +794,7 @@ mapped_file_region::mapped_file_region(const Twine &path,
   SmallVector<wchar_t, 128> path_utf16;
 
   // Convert path to UTF-16.
-  if (ec = UTF8ToUTF16(path.toStringRef(path_storage), path_utf16))
+  if ((ec = UTF8ToUTF16(path.toStringRef(path_storage), path_utf16)))
     return;
 
   // Get file handle for creating a file mapping.
diff --git a/lib/Support/YAMLParser.cpp b/lib/Support/YAMLParser.cpp
index 7c353c8..34df636 100644
--- a/lib/Support/YAMLParser.cpp
+++ b/lib/Support/YAMLParser.cpp
@@ -903,6 +903,7 @@ bool Scanner::consume(uint32_t Expected) {
 void Scanner::skip(uint32_t Distance) {
   Current += Distance;
   Column += Distance;
+  assert(Current <= End && "Skipped past the end");
 }
 
 bool Scanner::isBlankOrBreak(StringRef::iterator Position) {
@@ -1239,6 +1240,12 @@ bool Scanner::scanFlowScalar(bool IsDoubleQuoted) {
       }
     }
   }
+
+  if (Current == End) {
+    setError("Expected quote at end of scalar", Current);
+    return false;
+  }
+
   skip(1); // Skip ending quote.
   Token T;
   T.Kind = Token::TK_Scalar;
diff --git a/lib/Support/raw_ostream.cpp b/lib/Support/raw_ostream.cpp
index fa69c2d..7cd5364 100644
--- a/lib/Support/raw_ostream.cpp
+++ b/lib/Support/raw_ostream.cpp
@@ -266,8 +266,8 @@ void raw_ostream::flush_nonempty() {
 
 raw_ostream &raw_ostream::write(unsigned char C) {
   // Group exceptional cases into a single branch.
-  if (BUILTIN_EXPECT(OutBufCur >= OutBufEnd, false)) {
-    if (BUILTIN_EXPECT(!OutBufStart, false)) {
+  if (LLVM_UNLIKELY(OutBufCur >= OutBufEnd)) {
+    if (LLVM_UNLIKELY(!OutBufStart)) {
       if (BufferMode == Unbuffered) {
         write_impl(reinterpret_cast<char*>(&C), 1);
         return *this;
@@ -286,8 +286,8 @@ raw_ostream &raw_ostream::write(unsigned char C) {
 
 raw_ostream &raw_ostream::write(const char *Ptr, size_t Size) {
   // Group exceptional cases into a single branch.
-  if (BUILTIN_EXPECT(size_t(OutBufEnd - OutBufCur) < Size, false)) {
-    if (BUILTIN_EXPECT(!OutBufStart, false)) {
+  if (LLVM_UNLIKELY(size_t(OutBufEnd - OutBufCur) < Size)) {
+    if (LLVM_UNLIKELY(!OutBufStart)) {
       if (BufferMode == Unbuffered) {
         write_impl(Ptr, Size);
         return *this;
@@ -302,7 +302,7 @@ raw_ostream &raw_ostream::write(const char *Ptr, size_t Size) {
     // If the buffer is empty at this point we have a string that is larger
     // than the buffer. Directly write the chunk that is a multiple of the
     // preferred buffer size and put the remainder in the buffer.
-    if (BUILTIN_EXPECT(OutBufCur == OutBufStart, false)) {
+    if (LLVM_UNLIKELY(OutBufCur == OutBufStart)) {
       size_t BytesToWrite = Size - (Size % NumBytes);
       write_impl(Ptr, BytesToWrite);
       copy_to_buffer(Ptr + BytesToWrite, Size - BytesToWrite);
@@ -523,7 +523,7 @@ void raw_fd_ostream::write_impl(const char *Ptr, size_t Size) {
     ssize_t ret;
 
     // Check whether we should attempt to use atomic writes.
-    if (BUILTIN_EXPECT(!UseAtomicWrites, true)) {
+    if (LLVM_LIKELY(!UseAtomicWrites)) {
       ret = ::write(FD, Ptr, Size);
     } else {
       // Use ::writev() where available.
diff --git a/lib/Support/regexec.c b/lib/Support/regexec.c
index 0078616..bd5e72d 100644
--- a/lib/Support/regexec.c
+++ b/lib/Support/regexec.c
@@ -69,7 +69,7 @@
 #define	SETUP(v)	((v) = 0)
 #define	onestate	long
 #define	INIT(o, n)	((o) = (unsigned long)1 << (n))
-#define	INC(o)		((o) <<= 1)
+#define	INC(o)		((o) = (unsigned long)(o) << 1)
 #define	ISSTATEIN(v, o)	(((v) & (o)) != 0)
 /* some abbreviations; note that some of these know variable names! */
 /* do "if I'm here, I can also be there" etc without branches */
diff --git a/lib/Support/system_error.cpp b/lib/Support/system_error.cpp
index 56898de..2df223c 100644
--- a/lib/Support/system_error.cpp
+++ b/lib/Support/system_error.cpp
@@ -48,8 +48,8 @@ _do_message::message(int ev) const {
 
 class _generic_error_category : public _do_message {
 public:
-  virtual const char* name() const;
-  virtual std::string message(int ev) const;
+  virtual const char* name() const LLVM_OVERRIDE;
+  virtual std::string message(int ev) const LLVM_OVERRIDE;
 };
 
 const char*
@@ -74,9 +74,9 @@ generic_category() {
 
 class _system_error_category : public _do_message {
 public:
-  virtual const char* name() const;
-  virtual std::string message(int ev) const;
-  virtual error_condition default_error_condition(int ev) const;
+  virtual const char* name() const LLVM_OVERRIDE;
+  virtual std::string message(int ev) const LLVM_OVERRIDE;
+  virtual error_condition default_error_condition(int ev) const LLVM_OVERRIDE;
 };
 
 const char*
diff --git a/lib/TableGen/CMakeLists.txt b/lib/TableGen/CMakeLists.txt
index ba7bf14..935d674 100644
--- a/lib/TableGen/CMakeLists.txt
+++ b/lib/TableGen/CMakeLists.txt
@@ -1,13 +1,8 @@
-## FIXME: This only requires RTTI because tblgen uses it.  Fix that.
-set(LLVM_REQUIRES_RTTI 1)
-set(LLVM_REQUIRES_EH 1)
-
 add_llvm_library(LLVMTableGen
   Error.cpp
   Main.cpp
   Record.cpp
   StringMatcher.cpp
-  TableGenAction.cpp
   TableGenBackend.cpp
   TGLexer.cpp
   TGParser.cpp
diff --git a/lib/TableGen/Error.cpp b/lib/TableGen/Error.cpp
index 1463b68..0bb86b0 100644
--- a/lib/TableGen/Error.cpp
+++ b/lib/TableGen/Error.cpp
@@ -16,12 +16,25 @@
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/raw_ostream.h"
 
+#include <cstdlib>
+
 namespace llvm {
 
 SourceMgr SrcMgr;
 
-void PrintWarning(SMLoc WarningLoc, const Twine &Msg) {
-  SrcMgr.PrintMessage(WarningLoc, SourceMgr::DK_Warning, Msg);
+static void PrintMessage(ArrayRef<SMLoc> Loc, SourceMgr::DiagKind Kind,
+                         const Twine &Msg) {
+  SMLoc NullLoc;
+  if (Loc.empty())
+    Loc = NullLoc;
+  SrcMgr.PrintMessage(Loc.front(), Kind, Msg);
+  for (unsigned i = 1; i < Loc.size(); ++i)
+    SrcMgr.PrintMessage(Loc[i], SourceMgr::DK_Note,
+                        "instantiated from multiclass");
+}
+
+void PrintWarning(ArrayRef<SMLoc> WarningLoc, const Twine &Msg) {
+  PrintMessage(WarningLoc, SourceMgr::DK_Warning, Msg);
 }
 
 void PrintWarning(const char *Loc, const Twine &Msg) {
@@ -32,12 +45,8 @@ void PrintWarning(const Twine &Msg) {
   errs() << "warning:" << Msg << "\n";
 }
 
-void PrintWarning(const TGError &Warning) {
-  PrintWarning(Warning.getLoc(), Warning.getMessage());
-}
-
-void PrintError(SMLoc ErrorLoc, const Twine &Msg) {
-  SrcMgr.PrintMessage(ErrorLoc, SourceMgr::DK_Error, Msg);
+void PrintError(ArrayRef<SMLoc> ErrorLoc, const Twine &Msg) {
+  PrintMessage(ErrorLoc, SourceMgr::DK_Error, Msg);
 }
 
 void PrintError(const char *Loc, const Twine &Msg) {
@@ -48,8 +57,14 @@ void PrintError(const Twine &Msg) {
   errs() << "error:" << Msg << "\n";
 }
 
-void PrintError(const TGError &Error) {
-  PrintError(Error.getLoc(), Error.getMessage());
+void PrintFatalError(const std::string &Msg) {
+  PrintError(Twine(Msg));
+  std::exit(1);
+}
+
+void PrintFatalError(ArrayRef<SMLoc> ErrorLoc, const std::string &Msg) {
+  PrintError(ErrorLoc, Msg);
+  std::exit(1);
 }
 
 } // end namespace llvm
diff --git a/lib/TableGen/Main.cpp b/lib/TableGen/Main.cpp
index 7aeef56..d0ca756 100644
--- a/lib/TableGen/Main.cpp
+++ b/lib/TableGen/Main.cpp
@@ -22,8 +22,8 @@
 #include "llvm/Support/ToolOutputFile.h"
 #include "llvm/Support/system_error.h"
 #include "llvm/TableGen/Error.h"
+#include "llvm/TableGen/Main.h"
 #include "llvm/TableGen/Record.h"
-#include "llvm/TableGen/TableGenAction.h"
 #include <algorithm>
 #include <cstdio>
 using namespace llvm;
@@ -47,79 +47,79 @@ namespace {
               cl::value_desc("directory"), cl::Prefix);
 }
 
+/// \brief Create a dependency file for `-d` option.
+///
+/// This functionality is really only for the benefit of the build system.
+/// It is similar to GCC's `-M*` family of options.
+static int createDependencyFile(const TGParser &Parser, const char *argv0) {
+  if (OutputFilename == "-") {
+    errs() << argv0 << ": the option -d must be used together with -o\n";
+    return 1;
+  }
+  std::string Error;
+  tool_output_file DepOut(DependFilename.c_str(), Error);
+  if (!Error.empty()) {
+    errs() << argv0 << ": error opening " << DependFilename
+      << ":" << Error << "\n";
+    return 1;
+  }
+  DepOut.os() << OutputFilename << ":";
+  const std::vector<std::string> &Dependencies = Parser.getDependencies();
+  for (std::vector<std::string>::const_iterator I = Dependencies.begin(),
+                                                E = Dependencies.end();
+       I != E; ++I) {
+    DepOut.os() << " " << (*I);
+  }
+  DepOut.os() << "\n";
+  DepOut.keep();
+  return 0;
+}
+
 namespace llvm {
 
-int TableGenMain(char *argv0, TableGenAction &Action) {
+int TableGenMain(char *argv0, TableGenMainFn *MainFn) {
   RecordKeeper Records;
 
-  try {
-    // Parse the input file.
-    OwningPtr<MemoryBuffer> File;
-    if (error_code ec =
-          MemoryBuffer::getFileOrSTDIN(InputFilename.c_str(), File)) {
-      errs() << "Could not open input file '" << InputFilename << "': "
-             << ec.message() <<"\n";
-      return 1;
-    }
-    MemoryBuffer *F = File.take();
-
-    // Tell SrcMgr about this buffer, which is what TGParser will pick up.
-    SrcMgr.AddNewSourceBuffer(F, SMLoc());
-
-    // Record the location of the include directory so that the lexer can find
-    // it later.
-    SrcMgr.setIncludeDirs(IncludeDirs);
-
-    TGParser Parser(SrcMgr, Records);
-
-    if (Parser.ParseFile())
-      return 1;
-
-    std::string Error;
-    tool_output_file Out(OutputFilename.c_str(), Error);
-    if (!Error.empty()) {
-      errs() << argv0 << ": error opening " << OutputFilename
-        << ":" << Error << "\n";
-      return 1;
-    }
-    if (!DependFilename.empty()) {
-      if (OutputFilename == "-") {
-        errs() << argv0 << ": the option -d must be used together with -o\n";
-        return 1;
-      }
-      tool_output_file DepOut(DependFilename.c_str(), Error);
-      if (!Error.empty()) {
-        errs() << argv0 << ": error opening " << DependFilename
-          << ":" << Error << "\n";
-        return 1;
-      }
-      DepOut.os() << OutputFilename << ":";
-      const std::vector<std::string> &Dependencies = Parser.getDependencies();
-      for (std::vector<std::string>::const_iterator I = Dependencies.begin(),
-                                                    E = Dependencies.end();
-           I != E; ++I) {
-        DepOut.os() << " " << (*I);
-      }
-      DepOut.os() << "\n";
-      DepOut.keep();
-    }
-
-    if (Action(Out.os(), Records))
-      return 1;
-
-    // Declare success.
-    Out.keep();
-    return 0;
-
-  } catch (const TGError &Error) {
-    PrintError(Error);
-  } catch (const std::string &Error) {
-    PrintError(Error);
-  } catch (const char *Error) {
-    PrintError(Error);
-  } catch (...) {
-    errs() << argv0 << ": Unknown unexpected exception occurred.\n";
+  // Parse the input file.
+  OwningPtr<MemoryBuffer> File;
+  if (error_code ec =
+        MemoryBuffer::getFileOrSTDIN(InputFilename.c_str(), File)) {
+    errs() << "Could not open input file '" << InputFilename << "': "
+           << ec.message() <<"\n";
+    return 1;
+  }
+  MemoryBuffer *F = File.take();
+
+  // Tell SrcMgr about this buffer, which is what TGParser will pick up.
+  SrcMgr.AddNewSourceBuffer(F, SMLoc());
+
+  // Record the location of the include directory so that the lexer can find
+  // it later.
+  SrcMgr.setIncludeDirs(IncludeDirs);
+
+  TGParser Parser(SrcMgr, Records);
+
+  if (Parser.ParseFile())
+    return 1;
+
+  std::string Error;
+  tool_output_file Out(OutputFilename.c_str(), Error);
+  if (!Error.empty()) {
+    errs() << argv0 << ": error opening " << OutputFilename
+      << ":" << Error << "\n";
+    return 1;
   }
+  if (!DependFilename.empty()) {
+    if (int Ret = createDependencyFile(Parser, argv0))
+      return Ret;
+  }
+
+  if (MainFn(Out.os(), Records))
+    return 1;
+
+  // Declare success.
+  Out.keep();
+  return 0;
 
   return 1;
 }
diff --git a/lib/TableGen/Makefile b/lib/TableGen/Makefile
index 4472438..345db34 100644
--- a/lib/TableGen/Makefile
+++ b/lib/TableGen/Makefile
@@ -11,8 +11,4 @@ LEVEL = ../..
 LIBRARYNAME = LLVMTableGen
 BUILD_ARCHIVE = 1
 
-## FIXME: This only requires RTTI because tblgen uses it.  Fix that.
-REQUIRES_RTTI = 1
-REQUIRES_EH = 1
-
 include $(LEVEL)/Makefile.common
diff --git a/lib/TableGen/Record.cpp b/lib/TableGen/Record.cpp
index 99fdc1f..11feb435 100644
--- a/lib/TableGen/Record.cpp
+++ b/lib/TableGen/Record.cpp
@@ -112,7 +112,8 @@ Init *BitRecTy::convertValue(IntInit *II) {
 }
 
 Init *BitRecTy::convertValue(TypedInit *VI) {
-  if (dynamic_cast<BitRecTy*>(VI->getType()))
+  RecTy *Ty = VI->getType();
+  if (isa<BitRecTy>(Ty) || isa<BitsRecTy>(Ty) || isa<IntRecTy>(Ty))
     return VI;  // Accept variable if it is already of bit type!
   return 0;
 }
@@ -178,60 +179,15 @@ Init *BitsRecTy::convertValue(BitsInit *BI) {
 }
 
 Init *BitsRecTy::convertValue(TypedInit *VI) {
-  if (BitsRecTy *BRT = dynamic_cast<BitsRecTy*>(VI->getType()))
-    if (BRT->Size == Size) {
-      SmallVector<Init *, 16> NewBits(Size);
- 
-      for (unsigned i = 0; i != Size; ++i)
-        NewBits[i] = VarBitInit::get(VI, i);
-      return BitsInit::get(NewBits);
-    }
-
-  if (Size == 1 && dynamic_cast<BitRecTy*>(VI->getType()))
+  if (Size == 1 && isa<BitRecTy>(VI->getType()))
     return BitsInit::get(VI);
 
-  if (TernOpInit *Tern = dynamic_cast<TernOpInit*>(VI)) {
-    if (Tern->getOpcode() == TernOpInit::IF) {
-      Init *LHS = Tern->getLHS();
-      Init *MHS = Tern->getMHS();
-      Init *RHS = Tern->getRHS();
-
-      IntInit *MHSi = dynamic_cast<IntInit*>(MHS);
-      IntInit *RHSi = dynamic_cast<IntInit*>(RHS);
-
-      if (MHSi && RHSi) {
-        int64_t MHSVal = MHSi->getValue();
-        int64_t RHSVal = RHSi->getValue();
-
-        if (canFitInBitfield(MHSVal, Size) && canFitInBitfield(RHSVal, Size)) {
-          SmallVector<Init *, 16> NewBits(Size);
-
-          for (unsigned i = 0; i != Size; ++i)
-            NewBits[i] =
-              TernOpInit::get(TernOpInit::IF, LHS,
-                              IntInit::get((MHSVal & (1LL << i)) ? 1 : 0),
-                              IntInit::get((RHSVal & (1LL << i)) ? 1 : 0),
-                              VI->getType());
-
-          return BitsInit::get(NewBits);
-        }
-      } else {
-        BitsInit *MHSbs = dynamic_cast<BitsInit*>(MHS);
-        BitsInit *RHSbs = dynamic_cast<BitsInit*>(RHS);
-
-        if (MHSbs && RHSbs) {
-          SmallVector<Init *, 16> NewBits(Size);
-
-          for (unsigned i = 0; i != Size; ++i)
-            NewBits[i] = TernOpInit::get(TernOpInit::IF, LHS,
-                                         MHSbs->getBit(i),
-                                         RHSbs->getBit(i),
-                                         VI->getType());
+  if (VI->getType()->typeIsConvertibleTo(this)) {
+    SmallVector<Init *, 16> NewBits(Size);
 
-          return BitsInit::get(NewBits);
-        }
-      }
-    }
+    for (unsigned i = 0; i != Size; ++i)
+      NewBits[i] = VarBitInit::get(VI, i);
+    return BitsInit::get(NewBits);
   }
 
   return 0;
@@ -244,7 +200,7 @@ Init *IntRecTy::convertValue(BitInit *BI) {
 Init *IntRecTy::convertValue(BitsInit *BI) {
   int64_t Result = 0;
   for (unsigned i = 0, e = BI->getNumBits(); i != e; ++i)
-    if (BitInit *Bit = dynamic_cast<BitInit*>(BI->getBit(i))) {
+    if (BitInit *Bit = dyn_cast<BitInit>(BI->getBit(i))) {
       Result |= Bit->getValue() << i;
     } else {
       return 0;
@@ -285,7 +241,7 @@ Init *StringRecTy::convertValue(BinOpInit *BO) {
 
 
 Init *StringRecTy::convertValue(TypedInit *TI) {
-  if (dynamic_cast<StringRecTy*>(TI->getType()))
+  if (isa<StringRecTy>(TI->getType()))
     return TI;  // Accept variable if already of the right type!
   return 0;
 }
@@ -305,17 +261,15 @@ Init *ListRecTy::convertValue(ListInit *LI) {
     else
       return 0;
 
-  ListRecTy *LType = dynamic_cast<ListRecTy*>(LI->getType());
-  if (LType == 0) {
+  if (!isa<ListRecTy>(LI->getType()))
     return 0;
-  }
 
   return ListInit::get(Elements, this);
 }
 
 Init *ListRecTy::convertValue(TypedInit *TI) {
   // Ensure that TI is compatible with our class.
-  if (ListRecTy *LRT = dynamic_cast<ListRecTy*>(TI->getType()))
+  if (ListRecTy *LRT = dyn_cast<ListRecTy>(TI->getType()))
     if (LRT->getElementType()->typeIsConvertibleTo(getElementType()))
       return TI;
   return 0;
@@ -351,7 +305,7 @@ Init *DagRecTy::convertValue(BinOpInit *BO) {
 }
 
 RecordRecTy *RecordRecTy::get(Record *R) {
-  return &dynamic_cast<RecordRecTy&>(*R->getDefInit()->getType());
+  return dyn_cast<RecordRecTy>(R->getDefInit()->getType());
 }
 
 std::string RecordRecTy::getAsString() const {
@@ -367,7 +321,7 @@ Init *RecordRecTy::convertValue(DefInit *DI) {
 
 Init *RecordRecTy::convertValue(TypedInit *TI) {
   // Ensure that TI is compatible with Rec.
-  if (RecordRecTy *RRT = dynamic_cast<RecordRecTy*>(TI->getType()))
+  if (RecordRecTy *RRT = dyn_cast<RecordRecTy>(TI->getType()))
     if (RRT->getRecord()->isSubClassOf(getRecord()) ||
         RRT->getRecord() == getRecord())
       return TI;
@@ -386,57 +340,53 @@ bool RecordRecTy::baseClassOf(const RecordRecTy *RHS) const {
   return false;
 }
 
-
 /// resolveTypes - Find a common type that T1 and T2 convert to.
 /// Return 0 if no such type exists.
 ///
 RecTy *llvm::resolveTypes(RecTy *T1, RecTy *T2) {
-  if (!T1->typeIsConvertibleTo(T2)) {
-    if (!T2->typeIsConvertibleTo(T1)) {
-      // If one is a Record type, check superclasses
-      RecordRecTy *RecTy1 = dynamic_cast<RecordRecTy*>(T1);
-      if (RecTy1) {
-        // See if T2 inherits from a type T1 also inherits from
-        const std::vector<Record *> &T1SuperClasses =
-          RecTy1->getRecord()->getSuperClasses();
-        for(std::vector<Record *>::const_iterator i = T1SuperClasses.begin(),
-              iend = T1SuperClasses.end();
-            i != iend;
-            ++i) {
-          RecordRecTy *SuperRecTy1 = RecordRecTy::get(*i);
-          RecTy *NewType1 = resolveTypes(SuperRecTy1, T2);
-          if (NewType1 != 0) {
-            if (NewType1 != SuperRecTy1) {
-              delete SuperRecTy1;
-            }
-            return NewType1;
-          }
+  if (T1->typeIsConvertibleTo(T2))
+    return T2;
+  if (T2->typeIsConvertibleTo(T1))
+    return T1;
+
+  // If one is a Record type, check superclasses
+  if (RecordRecTy *RecTy1 = dyn_cast<RecordRecTy>(T1)) {
+    // See if T2 inherits from a type T1 also inherits from
+    const std::vector<Record *> &T1SuperClasses =
+      RecTy1->getRecord()->getSuperClasses();
+    for(std::vector<Record *>::const_iterator i = T1SuperClasses.begin(),
+          iend = T1SuperClasses.end();
+        i != iend;
+        ++i) {
+      RecordRecTy *SuperRecTy1 = RecordRecTy::get(*i);
+      RecTy *NewType1 = resolveTypes(SuperRecTy1, T2);
+      if (NewType1 != 0) {
+        if (NewType1 != SuperRecTy1) {
+          delete SuperRecTy1;
         }
+        return NewType1;
       }
-      RecordRecTy *RecTy2 = dynamic_cast<RecordRecTy*>(T2);
-      if (RecTy2) {
-        // See if T1 inherits from a type T2 also inherits from
-        const std::vector<Record *> &T2SuperClasses =
-          RecTy2->getRecord()->getSuperClasses();
-        for (std::vector<Record *>::const_iterator i = T2SuperClasses.begin(),
-              iend = T2SuperClasses.end();
-            i != iend;
-            ++i) {
-          RecordRecTy *SuperRecTy2 = RecordRecTy::get(*i);
-          RecTy *NewType2 = resolveTypes(T1, SuperRecTy2);
-          if (NewType2 != 0) {
-            if (NewType2 != SuperRecTy2) {
-              delete SuperRecTy2;
-            }
-            return NewType2;
-          }
+    }
+  }
+  if (RecordRecTy *RecTy2 = dyn_cast<RecordRecTy>(T2)) {
+    // See if T1 inherits from a type T2 also inherits from
+    const std::vector<Record *> &T2SuperClasses =
+      RecTy2->getRecord()->getSuperClasses();
+    for (std::vector<Record *>::const_iterator i = T2SuperClasses.begin(),
+          iend = T2SuperClasses.end();
+        i != iend;
+        ++i) {
+      RecordRecTy *SuperRecTy2 = RecordRecTy::get(*i);
+      RecTy *NewType2 = resolveTypes(T1, SuperRecTy2);
+      if (NewType2 != 0) {
+        if (NewType2 != SuperRecTy2) {
+          delete SuperRecTy2;
         }
+        return NewType2;
       }
-      return 0;
     }
-    return T2;
   }
-  return T1;
+  return 0;
 }
 
 
@@ -519,6 +469,15 @@ std::string BitsInit::getAsString() const {
   return Result + " }";
 }
 
+// Fix bit initializer to preserve the behavior that bit reference from a unset
+// bits initializer will resolve into VarBitInit to keep the field name and bit
+// number used in targets with fixed insn length.
+static Init *fixBitInit(const RecordVal *RV, Init *Before, Init *After) {
+  if (RV || After != UnsetInit::get())
+    return After;
+  return Before;
+}
+
 // resolveReferences - If there are any field references that refer to fields
 // that have been filled in, we can propagate the values now.
 //
@@ -526,16 +485,39 @@ Init *BitsInit::resolveReferences(Record &R, const RecordVal *RV) const {
   bool Changed = false;
   SmallVector<Init *, 16> NewBits(getNumBits());
 
-  for (unsigned i = 0, e = Bits.size(); i != e; ++i) {
-    Init *B;
-    Init *CurBit = getBit(i);
+  Init *CachedInit = 0;
+  Init *CachedBitVar = 0;
+  bool CachedBitVarChanged = false;
+
+  for (unsigned i = 0, e = getNumBits(); i != e; ++i) {
+    Init *CurBit = Bits[i];
+    Init *CurBitVar = CurBit->getBitVar();
 
-    do {
-      B = CurBit;
-      CurBit = CurBit->resolveReferences(R, RV);
-      Changed |= B != CurBit;
-    } while (B != CurBit);
     NewBits[i] = CurBit;
+
+    if (CurBitVar == CachedBitVar) {
+      if (CachedBitVarChanged) {
+        Init *Bit = CachedInit->getBit(CurBit->getBitNum());
+        NewBits[i] = fixBitInit(RV, CurBit, Bit);
+      }
+      continue;
+    }
+    CachedBitVar = CurBitVar;
+    CachedBitVarChanged = false;
+
+    Init *B;
+    do {
+      B = CurBitVar;
+      CurBitVar = CurBitVar->resolveReferences(R, RV);
+      CachedBitVarChanged |= B != CurBitVar;
+      Changed |= B != CurBitVar;
+    } while (B != CurBitVar);
+    CachedInit = CurBitVar;
+
+    if (CachedBitVarChanged) {
+      Init *Bit = CurBitVar->getBit(CurBit->getBitNum());
+      NewBits[i] = fixBitInit(RV, CurBit, Bit);
+    }
   }
 
   if (Changed)
@@ -613,7 +595,7 @@ ListInit *ListInit::get(ArrayRef<Init *> Range, RecTy *EltTy) {
 }
 
 void ListInit::Profile(FoldingSetNodeID &ID) const {
-  ListRecTy *ListType = dynamic_cast<ListRecTy *>(getType());
+  ListRecTy *ListType = dyn_cast<ListRecTy>(getType());
   assert(ListType && "Bad type for ListInit!");
   RecTy *EltTy = ListType->getElementType();
 
@@ -633,8 +615,9 @@ ListInit::convertInitListSlice(const std::vector<unsigned> &Elements) const {
 
 Record *ListInit::getElementAsRecord(unsigned i) const {
   assert(i < Values.size() && "List element index out of range!");
-  DefInit *DI = dynamic_cast<DefInit*>(Values[i]);
-  if (DI == 0) throw "Expected record in list!";
+  DefInit *DI = dyn_cast<DefInit>(Values[i]);
+  if (DI == 0)
+    PrintFatalError("Expected record in list!");
   return DI->getDef();
 }
 
@@ -668,7 +651,7 @@ Init *ListInit::resolveListElementReference(Record &R, const RecordVal *IRV,
   // If the element is set to some value, or if we are resolving a reference
   // to a specific variable and that variable is explicitly unset, then
   // replace the VarListElementInit with it.
-  if (IRV || !dynamic_cast<UnsetInit*>(E))
+  if (IRV || !isa<UnsetInit>(E))
     return E;
   return 0;
 }
@@ -682,30 +665,16 @@ std::string ListInit::getAsString() const {
   return Result + "]";
 }
 
-Init *OpInit::resolveBitReference(Record &R, const RecordVal *IRV,
-                                  unsigned Bit) const {
-  Init *Folded = Fold(&R, 0);
-
-  if (Folded != this) {
-    TypedInit *Typed = dynamic_cast<TypedInit *>(Folded);
-    if (Typed) {
-      return Typed->resolveBitReference(R, IRV, Bit);
-    }
-  }
-
-  return 0;
-}
-
 Init *OpInit::resolveListElementReference(Record &R, const RecordVal *IRV,
                                           unsigned Elt) const {
   Init *Resolved = resolveReferences(R, IRV);
-  OpInit *OResolved = dynamic_cast<OpInit *>(Resolved);
+  OpInit *OResolved = dyn_cast<OpInit>(Resolved);
   if (OResolved) {
     Resolved = OResolved->Fold(&R, 0);
   }
 
   if (Resolved != this) {
-    TypedInit *Typed = dynamic_cast<TypedInit *>(Resolved); 
+    TypedInit *Typed = dyn_cast<TypedInit>(Resolved);
     assert(Typed && "Expected typed init for list reference");
     if (Typed) {
       Init *New = Typed->resolveListElementReference(R, IRV, Elt);
@@ -718,6 +687,12 @@ Init *OpInit::resolveListElementReference(Record &R, const RecordVal *IRV,
   return 0;
 }
 
+Init *OpInit::getBit(unsigned Bit) const {
+  if (getType() == BitRecTy::get())
+    return const_cast<OpInit*>(this);
+  return VarBitInit::get(const_cast<OpInit*>(this), Bit);
+}
+
 UnOpInit *UnOpInit::get(UnaryOp opc, Init *lhs, RecTy *Type) {
   typedef std::pair<std::pair<unsigned, Init *>, RecTy *> Key;
 
@@ -735,30 +710,23 @@ Init *UnOpInit::Fold(Record *CurRec, MultiClass *CurMultiClass) const {
   switch (getOpcode()) {
   case CAST: {
     if (getType()->getAsString() == "string") {
-      StringInit *LHSs = dynamic_cast<StringInit*>(LHS);
-      if (LHSs) {
+      if (StringInit *LHSs = dyn_cast<StringInit>(LHS))
         return LHSs;
-      }
 
-      DefInit *LHSd = dynamic_cast<DefInit*>(LHS);
-      if (LHSd) {
+      if (DefInit *LHSd = dyn_cast<DefInit>(LHS))
         return StringInit::get(LHSd->getDef()->getName());
-      }
 
-      IntInit *LHSi = dynamic_cast<IntInit*>(LHS);
-      if (LHSi) {
+      if (IntInit *LHSi = dyn_cast<IntInit>(LHS))
         return StringInit::get(LHSi->getAsString());
-      }
     } else {
-      StringInit *LHSs = dynamic_cast<StringInit*>(LHS);
-      if (LHSs) {
+      if (StringInit *LHSs = dyn_cast<StringInit>(LHS)) {
         std::string Name = LHSs->getValue();
 
         // From TGParser::ParseIDValue
         if (CurRec) {
           if (const RecordVal *RV = CurRec->getValue(Name)) {
             if (RV->getType() != getType())
-              throw "type mismatch in cast";
+              PrintFatalError("type mismatch in cast");
             return VarInit::get(Name, RV->getType());
           }
 
@@ -770,7 +738,7 @@ Init *UnOpInit::Fold(Record *CurRec, MultiClass *CurMultiClass) const {
             assert(RV && "Template arg doesn't exist??");
 
             if (RV->getType() != getType())
-              throw "type mismatch in cast";
+              PrintFatalError("type mismatch in cast");
 
             return VarInit::get(TemplateArgName, RV->getType());
           }
@@ -784,7 +752,7 @@ Init *UnOpInit::Fold(Record *CurRec, MultiClass *CurMultiClass) const {
             assert(RV && "Template arg doesn't exist??");
 
             if (RV->getType() != getType())
-              throw "type mismatch in cast";
+              PrintFatalError("type mismatch in cast");
 
             return VarInit::get(MCName, RV->getType());
           }
@@ -793,14 +761,14 @@ Init *UnOpInit::Fold(Record *CurRec, MultiClass *CurMultiClass) const {
         if (Record *D = (CurRec->getRecords()).getDef(Name))
           return DefInit::get(D);
 
-        throw TGError(CurRec->getLoc(), "Undefined reference:'" + Name + "'\n");
+        PrintFatalError(CurRec->getLoc(),
+                        "Undefined reference:'" + Name + "'\n");
       }
     }
     break;
   }
   case HEAD: {
-    ListInit *LHSl = dynamic_cast<ListInit*>(LHS);
-    if (LHSl) {
+    if (ListInit *LHSl = dyn_cast<ListInit>(LHS)) {
       if (LHSl->getSize() == 0) {
         assert(0 && "Empty list in car");
         return 0;
@@ -810,8 +778,7 @@ Init *UnOpInit::Fold(Record *CurRec, MultiClass *CurMultiClass) const {
     break;
   }
   case TAIL: {
-    ListInit *LHSl = dynamic_cast<ListInit*>(LHS);
-    if (LHSl) {
+    if (ListInit *LHSl = dyn_cast<ListInit>(LHS)) {
       if (LHSl->getSize() == 0) {
         assert(0 && "Empty list in cdr");
         return 0;
@@ -828,16 +795,14 @@ Init *UnOpInit::Fold(Record *CurRec, MultiClass *CurMultiClass) const {
     break;
   }
   case EMPTY: {
-    ListInit *LHSl = dynamic_cast<ListInit*>(LHS);
-    if (LHSl) {
+    if (ListInit *LHSl = dyn_cast<ListInit>(LHS)) {
       if (LHSl->getSize() == 0) {
         return IntInit::get(1);
       } else {
         return IntInit::get(0);
       }
     }
-    StringInit *LHSs = dynamic_cast<StringInit*>(LHS);
-    if (LHSs) {
+    if (StringInit *LHSs = dyn_cast<StringInit>(LHS)) {
       if (LHSs->getValue().empty()) {
         return IntInit::get(1);
       } else {
@@ -891,13 +856,13 @@ BinOpInit *BinOpInit::get(BinaryOp opc, Init *lhs,
 Init *BinOpInit::Fold(Record *CurRec, MultiClass *CurMultiClass) const {
   switch (getOpcode()) {
   case CONCAT: {
-    DagInit *LHSs = dynamic_cast<DagInit*>(LHS);
-    DagInit *RHSs = dynamic_cast<DagInit*>(RHS);
+    DagInit *LHSs = dyn_cast<DagInit>(LHS);
+    DagInit *RHSs = dyn_cast<DagInit>(RHS);
     if (LHSs && RHSs) {
-      DefInit *LOp = dynamic_cast<DefInit*>(LHSs->getOperator());
-      DefInit *ROp = dynamic_cast<DefInit*>(RHSs->getOperator());
+      DefInit *LOp = dyn_cast<DefInit>(LHSs->getOperator());
+      DefInit *ROp = dyn_cast<DefInit>(RHSs->getOperator());
       if (LOp == 0 || ROp == 0 || LOp->getDef() != ROp->getDef())
-        throw "Concated Dag operators do not match!";
+        PrintFatalError("Concated Dag operators do not match!");
       std::vector<Init*> Args;
       std::vector<std::string> ArgNames;
       for (unsigned i = 0, e = LHSs->getNumArgs(); i != e; ++i) {
@@ -913,8 +878,8 @@ Init *BinOpInit::Fold(Record *CurRec, MultiClass *CurMultiClass) const {
     break;
   }
   case STRCONCAT: {
-    StringInit *LHSs = dynamic_cast<StringInit*>(LHS);
-    StringInit *RHSs = dynamic_cast<StringInit*>(RHS);
+    StringInit *LHSs = dyn_cast<StringInit>(LHS);
+    StringInit *RHSs = dyn_cast<StringInit>(RHS);
     if (LHSs && RHSs)
       return StringInit::get(LHSs->getValue() + RHSs->getValue());
     break;
@@ -922,16 +887,16 @@ Init *BinOpInit::Fold(Record *CurRec, MultiClass *CurMultiClass) const {
   case EQ: {
     // try to fold eq comparison for 'bit' and 'int', otherwise fallback
     // to string objects.
-    IntInit* L =
-      dynamic_cast<IntInit*>(LHS->convertInitializerTo(IntRecTy::get()));
-    IntInit* R =
-      dynamic_cast<IntInit*>(RHS->convertInitializerTo(IntRecTy::get()));
+    IntInit *L =
+      dyn_cast_or_null<IntInit>(LHS->convertInitializerTo(IntRecTy::get()));
+    IntInit *R =
+      dyn_cast_or_null<IntInit>(RHS->convertInitializerTo(IntRecTy::get()));
 
     if (L && R)
       return IntInit::get(L->getValue() == R->getValue());
 
-    StringInit *LHSs = dynamic_cast<StringInit*>(LHS);
-    StringInit *RHSs = dynamic_cast<StringInit*>(RHS);
+    StringInit *LHSs = dyn_cast<StringInit>(LHS);
+    StringInit *RHSs = dyn_cast<StringInit>(RHS);
 
     // Make sure we've resolved
     if (LHSs && RHSs)
@@ -942,8 +907,8 @@ Init *BinOpInit::Fold(Record *CurRec, MultiClass *CurMultiClass) const {
   case SHL:
   case SRA:
   case SRL: {
-    IntInit *LHSi = dynamic_cast<IntInit*>(LHS);
-    IntInit *RHSi = dynamic_cast<IntInit*>(RHS);
+    IntInit *LHSi = dyn_cast<IntInit>(LHS);
+    IntInit *RHSi = dyn_cast<IntInit>(RHS);
     if (LHSi && RHSi) {
       int64_t LHSv = LHSi->getValue(), RHSv = RHSi->getValue();
       int64_t Result;
@@ -1016,7 +981,7 @@ static Init *EvaluateOperation(OpInit *RHSo, Init *LHS, Init *Arg,
                                MultiClass *CurMultiClass) {
   std::vector<Init *> NewOperands;
 
-  TypedInit *TArg = dynamic_cast<TypedInit*>(Arg);
+  TypedInit *TArg = dyn_cast<TypedInit>(Arg);
 
   // If this is a dag, recurse
   if (TArg && TArg->getType()->getAsString() == "dag") {
@@ -1030,7 +995,7 @@ static Init *EvaluateOperation(OpInit *RHSo, Init *LHS, Init *Arg,
   }
 
   for (int i = 0; i < RHSo->getNumOperands(); ++i) {
-    OpInit *RHSoo = dynamic_cast<OpInit*>(RHSo->getOperand(i));
+    OpInit *RHSoo = dyn_cast<OpInit>(RHSo->getOperand(i));
 
     if (RHSoo) {
       Init *Result = EvaluateOperation(RHSoo, LHS, Arg,
@@ -1058,25 +1023,21 @@ static Init *EvaluateOperation(OpInit *RHSo, Init *LHS, Init *Arg,
 
 static Init *ForeachHelper(Init *LHS, Init *MHS, Init *RHS, RecTy *Type,
                            Record *CurRec, MultiClass *CurMultiClass) {
-  DagInit *MHSd = dynamic_cast<DagInit*>(MHS);
-  ListInit *MHSl = dynamic_cast<ListInit*>(MHS);
+  DagInit *MHSd = dyn_cast<DagInit>(MHS);
+  ListInit *MHSl = dyn_cast<ListInit>(MHS);
 
-  DagRecTy *DagType = dynamic_cast<DagRecTy*>(Type);
-  ListRecTy *ListType = dynamic_cast<ListRecTy*>(Type);
-
-  OpInit *RHSo = dynamic_cast<OpInit*>(RHS);
+  OpInit *RHSo = dyn_cast<OpInit>(RHS);
 
   if (!RHSo) {
-    throw TGError(CurRec->getLoc(), "!foreach requires an operator\n");
+    PrintFatalError(CurRec->getLoc(), "!foreach requires an operator\n");
   }
 
-  TypedInit *LHSt = dynamic_cast<TypedInit*>(LHS);
+  TypedInit *LHSt = dyn_cast<TypedInit>(LHS);
 
-  if (!LHSt) {
-    throw TGError(CurRec->getLoc(), "!foreach requires typed variable\n");
-  }
+  if (!LHSt)
+    PrintFatalError(CurRec->getLoc(), "!foreach requires typed variable\n");
 
-  if ((MHSd && DagType) || (MHSl && ListType)) {
+  if ((MHSd && isa<DagRecTy>(Type)) || (MHSl && isa<ListRecTy>(Type))) {
     if (MHSd) {
       Init *Val = MHSd->getOperator();
       Init *Result = EvaluateOperation(RHSo, LHS, Val,
@@ -1139,17 +1100,17 @@ static Init *ForeachHelper(Init *LHS, Init *MHS, Init *RHS, RecTy *Type,
 Init *TernOpInit::Fold(Record *CurRec, MultiClass *CurMultiClass) const {
   switch (getOpcode()) {
   case SUBST: {
-    DefInit *LHSd = dynamic_cast<DefInit*>(LHS);
-    VarInit *LHSv = dynamic_cast<VarInit*>(LHS);
-    StringInit *LHSs = dynamic_cast<StringInit*>(LHS);
+    DefInit *LHSd = dyn_cast<DefInit>(LHS);
+    VarInit *LHSv = dyn_cast<VarInit>(LHS);
+    StringInit *LHSs = dyn_cast<StringInit>(LHS);
 
-    DefInit *MHSd = dynamic_cast<DefInit*>(MHS);
-    VarInit *MHSv = dynamic_cast<VarInit*>(MHS);
-    StringInit *MHSs = dynamic_cast<StringInit*>(MHS);
+    DefInit *MHSd = dyn_cast<DefInit>(MHS);
+    VarInit *MHSv = dyn_cast<VarInit>(MHS);
+    StringInit *MHSs = dyn_cast<StringInit>(MHS);
 
-    DefInit *RHSd = dynamic_cast<DefInit*>(RHS);
-    VarInit *RHSv = dynamic_cast<VarInit*>(RHS);
-    StringInit *RHSs = dynamic_cast<StringInit*>(RHS);
+    DefInit *RHSd = dyn_cast<DefInit>(RHS);
+    VarInit *RHSv = dyn_cast<VarInit>(RHS);
+    StringInit *RHSs = dyn_cast<StringInit>(RHS);
 
     if ((LHSd && MHSd && RHSd)
         || (LHSv && MHSv && RHSv)
@@ -1197,9 +1158,9 @@ Init *TernOpInit::Fold(Record *CurRec, MultiClass *CurMultiClass) const {
   }
 
   case IF: {
-    IntInit *LHSi = dynamic_cast<IntInit*>(LHS);
+    IntInit *LHSi = dyn_cast<IntInit>(LHS);
     if (Init *I = LHS->convertInitializerTo(IntRecTy::get()))
-      LHSi = dynamic_cast<IntInit*>(I);
+      LHSi = dyn_cast<IntInit>(I);
     if (LHSi) {
       if (LHSi->getValue()) {
         return MHS;
@@ -1219,9 +1180,9 @@ Init *TernOpInit::resolveReferences(Record &R,
   Init *lhs = LHS->resolveReferences(R, RV);
 
   if (Opc == IF && lhs != LHS) {
-    IntInit *Value = dynamic_cast<IntInit*>(lhs);
+    IntInit *Value = dyn_cast<IntInit>(lhs);
     if (Init *I = lhs->convertInitializerTo(IntRecTy::get()))
-      Value = dynamic_cast<IntInit*>(I);
+      Value = dyn_cast<IntInit>(I);
     if (Value != 0) {
       // Short-circuit
       if (Value->getValue()) {
@@ -1257,19 +1218,15 @@ std::string TernOpInit::getAsString() const {
 }
 
 RecTy *TypedInit::getFieldType(const std::string &FieldName) const {
-  RecordRecTy *RecordType = dynamic_cast<RecordRecTy *>(getType());
-  if (RecordType) {
-    RecordVal *Field = RecordType->getRecord()->getValue(FieldName);
-    if (Field) {
+  if (RecordRecTy *RecordType = dyn_cast<RecordRecTy>(getType()))
+    if (RecordVal *Field = RecordType->getRecord()->getValue(FieldName))
       return Field->getType();
-    }
-  }
   return 0;
 }
 
 Init *
 TypedInit::convertInitializerBitRange(const std::vector<unsigned> &Bits) const {
-  BitsRecTy *T = dynamic_cast<BitsRecTy*>(getType());
+  BitsRecTy *T = dyn_cast<BitsRecTy>(getType());
   if (T == 0) return 0;  // Cannot subscript a non-bits variable.
   unsigned NumBits = T->getNumBits();
 
@@ -1285,7 +1242,7 @@ TypedInit::convertInitializerBitRange(const std::vector<unsigned> &Bits) const {
 
 Init *
 TypedInit::convertInitListSlice(const std::vector<unsigned> &Elements) const {
-  ListRecTy *T = dynamic_cast<ListRecTy*>(getType());
+  ListRecTy *T = dyn_cast<ListRecTy>(getType());
   if (T == 0) return 0;  // Cannot subscript a non-list variable.
 
   if (Elements.size() == 1)
@@ -1318,31 +1275,15 @@ VarInit *VarInit::get(Init *VN, RecTy *T) {
 }
 
 const std::string &VarInit::getName() const {
-  StringInit *NameString =
-    dynamic_cast<StringInit *>(getNameInit());
+  StringInit *NameString = dyn_cast<StringInit>(getNameInit());
   assert(NameString && "VarInit name is not a string!");
   return NameString->getValue();
 }
 
-Init *VarInit::resolveBitReference(Record &R, const RecordVal *IRV,
-                                   unsigned Bit) const {
-  if (R.isTemplateArg(getNameInit())) return 0;
-  if (IRV && IRV->getNameInit() != getNameInit()) return 0;
-
-  RecordVal *RV = R.getValue(getNameInit());
-  assert(RV && "Reference to a non-existent variable?");
-  assert(dynamic_cast<BitsInit*>(RV->getValue()));
-  BitsInit *BI = (BitsInit*)RV->getValue();
-
-  assert(Bit < BI->getNumBits() && "Bit reference out of range!");
-  Init *B = BI->getBit(Bit);
-
-  // If the bit is set to some value, or if we are resolving a reference to a
-  // specific variable and that variable is explicitly unset, then replace the
-  // VarBitInit with it.
-  if (IRV || !dynamic_cast<UnsetInit*>(B))
-    return B;
-  return 0;
+Init *VarInit::getBit(unsigned Bit) const {
+  if (getType() == BitRecTy::get())
+    return const_cast<VarInit*>(this);
+  return VarBitInit::get(const_cast<VarInit*>(this), Bit);
 }
 
 Init *VarInit::resolveListElementReference(Record &R,
@@ -1353,9 +1294,9 @@ Init *VarInit::resolveListElementReference(Record &R,
 
   RecordVal *RV = R.getValue(getNameInit());
   assert(RV && "Reference to a non-existent variable?");
-  ListInit *LI = dynamic_cast<ListInit*>(RV->getValue());
+  ListInit *LI = dyn_cast<ListInit>(RV->getValue());
   if (!LI) {
-    TypedInit *VI = dynamic_cast<TypedInit*>(RV->getValue());
+    TypedInit *VI = dyn_cast<TypedInit>(RV->getValue());
     assert(VI && "Invalid list element!");
     return VarListElementInit::get(VI, Elt);
   }
@@ -1366,14 +1307,14 @@ Init *VarInit::resolveListElementReference(Record &R,
   // If the element is set to some value, or if we are resolving a reference
   // to a specific variable and that variable is explicitly unset, then
   // replace the VarListElementInit with it.
-  if (IRV || !dynamic_cast<UnsetInit*>(E))
+  if (IRV || !isa<UnsetInit>(E))
     return E;
   return 0;
 }
 
 
 RecTy *VarInit::getFieldType(const std::string &FieldName) const {
-  if (RecordRecTy *RTy = dynamic_cast<RecordRecTy*>(getType()))
+  if (RecordRecTy *RTy = dyn_cast<RecordRecTy>(getType()))
     if (const RecordVal *RV = RTy->getRecord()->getValue(FieldName))
       return RV->getType();
   return 0;
@@ -1381,9 +1322,9 @@ RecTy *VarInit::getFieldType(const std::string &FieldName) const {
 
 Init *VarInit::getFieldInit(Record &R, const RecordVal *RV,
                             const std::string &FieldName) const {
-  if (dynamic_cast<RecordRecTy*>(getType()))
+  if (isa<RecordRecTy>(getType()))
     if (const RecordVal *Val = R.getValue(VarName)) {
-      if (RV != Val && (RV || dynamic_cast<UnsetInit*>(Val->getValue())))
+      if (RV != Val && (RV || isa<UnsetInit>(Val->getValue())))
         return 0;
       Init *TheInit = Val->getValue();
       assert(TheInit != this && "Infinite loop detected!");
@@ -1402,7 +1343,7 @@ Init *VarInit::getFieldInit(Record &R, const RecordVal *RV,
 ///
 Init *VarInit::resolveReferences(Record &R, const RecordVal *RV) const {
   if (RecordVal *Val = R.getValue(VarName))
-    if (RV == Val || (RV == 0 && !dynamic_cast<UnsetInit*>(Val->getValue())))
+    if (RV == Val || (RV == 0 && !isa<UnsetInit>(Val->getValue())))
       return Val->getValue();
   return const_cast<VarInit *>(this);
 }
@@ -1425,9 +1366,11 @@ std::string VarBitInit::getAsString() const {
 }
 
 Init *VarBitInit::resolveReferences(Record &R, const RecordVal *RV) const {
-  if (Init *I = getVariable()->resolveBitReference(R, RV, getBitNum()))
-    return I;
-  return const_cast<VarBitInit *>(this);
+  Init *I = TI->resolveReferences(R, RV);
+  if (TI != I)
+    return I->getBit(getBitNum());
+
+  return const_cast<VarBitInit*>(this);
 }
 
 VarListElementInit *VarListElementInit::get(TypedInit *T,
@@ -1456,11 +1399,10 @@ VarListElementInit::resolveReferences(Record &R, const RecordVal *RV) const {
   return const_cast<VarListElementInit *>(this);
 }
 
-Init *VarListElementInit::resolveBitReference(Record &R, const RecordVal *RV,
-                                              unsigned Bit) const {
-  // FIXME: This should be implemented, to support references like:
-  // bit B = AA[0]{1};
-  return 0;
+Init *VarListElementInit::getBit(unsigned Bit) const {
+  if (getType() == BitRecTy::get())
+    return const_cast<VarListElementInit*>(this);
+  return VarBitInit::get(const_cast<VarListElementInit*>(this), Bit);
 }
 
 Init *VarListElementInit:: resolveListElementReference(Record &R,
@@ -1469,8 +1411,7 @@ Init *VarListElementInit:: resolveListElementReference(Record &R,
   Init *Result = TI->resolveListElementReference(R, RV, Element);
   
   if (Result) {
-    TypedInit *TInit = dynamic_cast<TypedInit *>(Result);
-    if (TInit) {
+    if (TypedInit *TInit = dyn_cast<TypedInit>(Result)) {
       Init *Result2 = TInit->resolveListElementReference(R, RV, Elt);
       if (Result2) return Result2;
       return new VarListElementInit(TInit, Elt);
@@ -1513,30 +1454,23 @@ FieldInit *FieldInit::get(Init *R, const std::string &FN) {
   return I;
 }
 
-Init *FieldInit::resolveBitReference(Record &R, const RecordVal *RV,
-                                     unsigned Bit) const {
-  if (Init *BitsVal = Rec->getFieldInit(R, RV, FieldName))
-    if (BitsInit *BI = dynamic_cast<BitsInit*>(BitsVal)) {
-      assert(Bit < BI->getNumBits() && "Bit reference out of range!");
-      Init *B = BI->getBit(Bit);
-
-      if (dynamic_cast<BitInit*>(B))  // If the bit is set.
-        return B;                     // Replace the VarBitInit with it.
-    }
-  return 0;
+Init *FieldInit::getBit(unsigned Bit) const {
+  if (getType() == BitRecTy::get())
+    return const_cast<FieldInit*>(this);
+  return VarBitInit::get(const_cast<FieldInit*>(this), Bit);
 }
 
 Init *FieldInit::resolveListElementReference(Record &R, const RecordVal *RV,
                                              unsigned Elt) const {
   if (Init *ListVal = Rec->getFieldInit(R, RV, FieldName))
-    if (ListInit *LI = dynamic_cast<ListInit*>(ListVal)) {
+    if (ListInit *LI = dyn_cast<ListInit>(ListVal)) {
       if (Elt >= LI->getSize()) return 0;
       Init *E = LI->getElement(Elt);
 
       // If the element is set to some value, or if we are resolving a
       // reference to a specific variable and that variable is explicitly
       // unset, then replace the VarListElementInit with it.
-      if (RV || !dynamic_cast<UnsetInit*>(E))
+      if (RV || !isa<UnsetInit>(E))
         return E;
     }
   return 0;
@@ -1665,7 +1599,7 @@ RecordVal::RecordVal(const std::string &N, RecTy *T, unsigned P)
 }
 
 const std::string &RecordVal::getName() const {
-  StringInit *NameString = dynamic_cast<StringInit *>(Name);
+  StringInit *NameString = dyn_cast<StringInit>(Name);
   assert(NameString && "RecordVal name is not a string!");
   return NameString->getValue();
 }
@@ -1695,12 +1629,11 @@ void Record::init() {
 
 void Record::checkName() {
   // Ensure the record name has string type.
-  const TypedInit *TypedName = dynamic_cast<const TypedInit *>(Name);
+  const TypedInit *TypedName = dyn_cast<const TypedInit>(Name);
   assert(TypedName && "Record name is not typed!");
   RecTy *Type = TypedName->getType();
-  if (dynamic_cast<StringRecTy *>(Type) == 0) {
-    throw TGError(getLoc(), "Record name is not a string!");
-  }
+  if (!isa<StringRecTy>(Type))
+    PrintFatalError(getLoc(), "Record name is not a string!");
 }
 
 DefInit *Record::getDefInit() {
@@ -1710,8 +1643,7 @@ DefInit *Record::getDefInit() {
 }
 
 const std::string &Record::getName() const {
-  const StringInit *NameString =
-    dynamic_cast<const StringInit *>(Name);
+  const StringInit *NameString = dyn_cast<StringInit>(Name);
   assert(NameString && "Record name is not a string!");
   return NameString->getValue();
 }
@@ -1751,7 +1683,15 @@ void Record::resolveReferencesTo(const RecordVal *RV) {
     if (RV == &Values[i]) // Skip resolve the same field as the given one
       continue;
     if (Init *V = Values[i].getValue())
-      Values[i].setValue(V->resolveReferences(*this, RV));
+      if (Values[i].setValue(V->resolveReferences(*this, RV)))
+        PrintFatalError(getLoc(), "Invalid value is found when setting '"
+                      + Values[i].getNameInitAsString()
+                      + "' after resolving references"
+                      + (RV ? " against '" + RV->getNameInitAsString()
+                              + "' of ("
+                              + RV->getValue()->getAsUnquotedString() + ")"
+                            : "")
+                      + "\n");
   }
   Init *OldName = getNameInit();
   Init *NewName = Name->resolveReferences(*this, RV);
@@ -1799,184 +1739,201 @@ raw_ostream &llvm::operator<<(raw_ostream &OS, const Record &R) {
 }
 
 /// getValueInit - Return the initializer for a value with the specified name,
-/// or throw an exception if the field does not exist.
+/// or abort if the field does not exist.
 ///
 Init *Record::getValueInit(StringRef FieldName) const {
   const RecordVal *R = getValue(FieldName);
   if (R == 0 || R->getValue() == 0)
-    throw "Record `" + getName() + "' does not have a field named `" +
-      FieldName.str() + "'!\n";
+    PrintFatalError(getLoc(), "Record `" + getName() +
+      "' does not have a field named `" + FieldName.str() + "'!\n");
   return R->getValue();
 }
 
 
 /// getValueAsString - This method looks up the specified field and returns its
-/// value as a string, throwing an exception if the field does not exist or if
+/// value as a string, aborts if the field does not exist or if
 /// the value is not a string.
 ///
 std::string Record::getValueAsString(StringRef FieldName) const {
   const RecordVal *R = getValue(FieldName);
   if (R == 0 || R->getValue() == 0)
-    throw "Record `" + getName() + "' does not have a field named `" +
-          FieldName.str() + "'!\n";
+    PrintFatalError(getLoc(), "Record `" + getName() +
+      "' does not have a field named `" + FieldName.str() + "'!\n");
 
-  if (StringInit *SI = dynamic_cast<StringInit*>(R->getValue()))
+  if (StringInit *SI = dyn_cast<StringInit>(R->getValue()))
     return SI->getValue();
-  throw "Record `" + getName() + "', field `" + FieldName.str() +
-        "' does not have a string initializer!";
+  PrintFatalError(getLoc(), "Record `" + getName() + "', field `" +
+    FieldName.str() + "' does not have a string initializer!");
 }
 
 /// getValueAsBitsInit - This method looks up the specified field and returns
-/// its value as a BitsInit, throwing an exception if the field does not exist
-/// or if the value is not the right type.
+/// its value as a BitsInit, aborts if the field does not exist or if
+/// the value is not the right type.
 ///
 BitsInit *Record::getValueAsBitsInit(StringRef FieldName) const {
   const RecordVal *R = getValue(FieldName);
   if (R == 0 || R->getValue() == 0)
-    throw "Record `" + getName() + "' does not have a field named `" +
-          FieldName.str() + "'!\n";
+    PrintFatalError(getLoc(), "Record `" + getName() +
+      "' does not have a field named `" + FieldName.str() + "'!\n");
 
-  if (BitsInit *BI = dynamic_cast<BitsInit*>(R->getValue()))
+  if (BitsInit *BI = dyn_cast<BitsInit>(R->getValue()))
     return BI;
-  throw "Record `" + getName() + "', field `" + FieldName.str() +
-        "' does not have a BitsInit initializer!";
+  PrintFatalError(getLoc(), "Record `" + getName() + "', field `" +
+    FieldName.str() + "' does not have a BitsInit initializer!");
 }
 
 /// getValueAsListInit - This method looks up the specified field and returns
-/// its value as a ListInit, throwing an exception if the field does not exist
-/// or if the value is not the right type.
+/// its value as a ListInit, aborting if the field does not exist or if
+/// the value is not the right type.
 ///
 ListInit *Record::getValueAsListInit(StringRef FieldName) const {
   const RecordVal *R = getValue(FieldName);
   if (R == 0 || R->getValue() == 0)
-    throw "Record `" + getName() + "' does not have a field named `" +
-          FieldName.str() + "'!\n";
+    PrintFatalError(getLoc(), "Record `" + getName() +
+      "' does not have a field named `" + FieldName.str() + "'!\n");
 
-  if (ListInit *LI = dynamic_cast<ListInit*>(R->getValue()))
+  if (ListInit *LI = dyn_cast<ListInit>(R->getValue()))
     return LI;
-  throw "Record `" + getName() + "', field `" + FieldName.str() +
-        "' does not have a list initializer!";
+  PrintFatalError(getLoc(), "Record `" + getName() + "', field `" +
+    FieldName.str() + "' does not have a list initializer!");
 }
 
 /// getValueAsListOfDefs - This method looks up the specified field and returns
-/// its value as a vector of records, throwing an exception if the field does
-/// not exist or if the value is not the right type.
+/// its value as a vector of records, aborting if the field does not exist
+/// or if the value is not the right type.
 ///
 std::vector<Record*>
 Record::getValueAsListOfDefs(StringRef FieldName) const {
   ListInit *List = getValueAsListInit(FieldName);
   std::vector<Record*> Defs;
   for (unsigned i = 0; i < List->getSize(); i++) {
-    if (DefInit *DI = dynamic_cast<DefInit*>(List->getElement(i))) {
+    if (DefInit *DI = dyn_cast<DefInit>(List->getElement(i))) {
       Defs.push_back(DI->getDef());
     } else {
-      throw "Record `" + getName() + "', field `" + FieldName.str() +
-            "' list is not entirely DefInit!";
+      PrintFatalError(getLoc(), "Record `" + getName() + "', field `" +
+        FieldName.str() + "' list is not entirely DefInit!");
     }
   }
   return Defs;
 }
 
 /// getValueAsInt - This method looks up the specified field and returns its
-/// value as an int64_t, throwing an exception if the field does not exist or if
-/// the value is not the right type.
+/// value as an int64_t, aborting if the field does not exist or if the value
+/// is not the right type.
 ///
 int64_t Record::getValueAsInt(StringRef FieldName) const {
   const RecordVal *R = getValue(FieldName);
   if (R == 0 || R->getValue() == 0)
-    throw "Record `" + getName() + "' does not have a field named `" +
-          FieldName.str() + "'!\n";
+    PrintFatalError(getLoc(), "Record `" + getName() +
+      "' does not have a field named `" + FieldName.str() + "'!\n");
 
-  if (IntInit *II = dynamic_cast<IntInit*>(R->getValue()))
+  if (IntInit *II = dyn_cast<IntInit>(R->getValue()))
     return II->getValue();
-  throw "Record `" + getName() + "', field `" + FieldName.str() +
-        "' does not have an int initializer!";
+  PrintFatalError(getLoc(), "Record `" + getName() + "', field `" +
+    FieldName.str() + "' does not have an int initializer!");
 }
 
 /// getValueAsListOfInts - This method looks up the specified field and returns
-/// its value as a vector of integers, throwing an exception if the field does
-/// not exist or if the value is not the right type.
+/// its value as a vector of integers, aborting if the field does not exist or
+/// if the value is not the right type.
 ///
 std::vector<int64_t>
 Record::getValueAsListOfInts(StringRef FieldName) const {
   ListInit *List = getValueAsListInit(FieldName);
   std::vector<int64_t> Ints;
   for (unsigned i = 0; i < List->getSize(); i++) {
-    if (IntInit *II = dynamic_cast<IntInit*>(List->getElement(i))) {
+    if (IntInit *II = dyn_cast<IntInit>(List->getElement(i))) {
       Ints.push_back(II->getValue());
     } else {
-      throw "Record `" + getName() + "', field `" + FieldName.str() +
-            "' does not have a list of ints initializer!";
+      PrintFatalError(getLoc(), "Record `" + getName() + "', field `" +
+        FieldName.str() + "' does not have a list of ints initializer!");
     }
   }
   return Ints;
 }
 
 /// getValueAsListOfStrings - This method looks up the specified field and
-/// returns its value as a vector of strings, throwing an exception if the
-/// field does not exist or if the value is not the right type.
+/// returns its value as a vector of strings, aborting if the field does not
+/// exist or if the value is not the right type.
 ///
 std::vector<std::string>
 Record::getValueAsListOfStrings(StringRef FieldName) const {
   ListInit *List = getValueAsListInit(FieldName);
   std::vector<std::string> Strings;
   for (unsigned i = 0; i < List->getSize(); i++) {
-    if (StringInit *II = dynamic_cast<StringInit*>(List->getElement(i))) {
+    if (StringInit *II = dyn_cast<StringInit>(List->getElement(i))) {
       Strings.push_back(II->getValue());
     } else {
-      throw "Record `" + getName() + "', field `" + FieldName.str() +
-            "' does not have a list of strings initializer!";
+      PrintFatalError(getLoc(), "Record `" + getName() + "', field `" +
+        FieldName.str() + "' does not have a list of strings initializer!");
     }
   }
   return Strings;
 }
 
 /// getValueAsDef - This method looks up the specified field and returns its
-/// value as a Record, throwing an exception if the field does not exist or if
-/// the value is not the right type.
+/// value as a Record, aborting if the field does not exist or if the value
+/// is not the right type.
 ///
 Record *Record::getValueAsDef(StringRef FieldName) const {
   const RecordVal *R = getValue(FieldName);
   if (R == 0 || R->getValue() == 0)
-    throw "Record `" + getName() + "' does not have a field named `" +
-      FieldName.str() + "'!\n";
+    PrintFatalError(getLoc(), "Record `" + getName() +
+      "' does not have a field named `" + FieldName.str() + "'!\n");
 
-  if (DefInit *DI = dynamic_cast<DefInit*>(R->getValue()))
+  if (DefInit *DI = dyn_cast<DefInit>(R->getValue()))
     return DI->getDef();
-  throw "Record `" + getName() + "', field `" + FieldName.str() +
-        "' does not have a def initializer!";
+  PrintFatalError(getLoc(), "Record `" + getName() + "', field `" +
+    FieldName.str() + "' does not have a def initializer!");
 }
 
 /// getValueAsBit - This method looks up the specified field and returns its
-/// value as a bit, throwing an exception if the field does not exist or if
-/// the value is not the right type.
+/// value as a bit, aborting if the field does not exist or if the value is
+/// not the right type.
 ///
 bool Record::getValueAsBit(StringRef FieldName) const {
   const RecordVal *R = getValue(FieldName);
   if (R == 0 || R->getValue() == 0)
-    throw "Record `" + getName() + "' does not have a field named `" +
-      FieldName.str() + "'!\n";
+    PrintFatalError(getLoc(), "Record `" + getName() +
+      "' does not have a field named `" + FieldName.str() + "'!\n");
 
-  if (BitInit *BI = dynamic_cast<BitInit*>(R->getValue()))
+  if (BitInit *BI = dyn_cast<BitInit>(R->getValue()))
     return BI->getValue();
-  throw "Record `" + getName() + "', field `" + FieldName.str() +
-        "' does not have a bit initializer!";
+  PrintFatalError(getLoc(), "Record `" + getName() + "', field `" +
+    FieldName.str() + "' does not have a bit initializer!");
+}
+
+bool Record::getValueAsBitOrUnset(StringRef FieldName, bool &Unset) const {
+  const RecordVal *R = getValue(FieldName);
+  if (R == 0 || R->getValue() == 0)
+    PrintFatalError(getLoc(), "Record `" + getName() +
+      "' does not have a field named `" + FieldName.str() + "'!\n");
+
+  if (R->getValue() == UnsetInit::get()) {
+    Unset = true;
+    return false;
+  }
+  Unset = false;
+  if (BitInit *BI = dyn_cast<BitInit>(R->getValue()))
+    return BI->getValue();
+  PrintFatalError(getLoc(), "Record `" + getName() + "', field `" +
+    FieldName.str() + "' does not have a bit initializer!");
 }
 
 /// getValueAsDag - This method looks up the specified field and returns its
-/// value as an Dag, throwing an exception if the field does not exist or if
-/// the value is not the right type.
+/// value as an Dag, aborting if the field does not exist or if the value is
+/// not the right type.
 ///
 DagInit *Record::getValueAsDag(StringRef FieldName) const {
   const RecordVal *R = getValue(FieldName);
   if (R == 0 || R->getValue() == 0)
-    throw "Record `" + getName() + "' does not have a field named `" +
-      FieldName.str() + "'!\n";
+    PrintFatalError(getLoc(), "Record `" + getName() +
+      "' does not have a field named `" + FieldName.str() + "'!\n");
 
-  if (DagInit *DI = dynamic_cast<DagInit*>(R->getValue()))
+  if (DagInit *DI = dyn_cast<DagInit>(R->getValue()))
     return DI;
-  throw "Record `" + getName() + "', field `" + FieldName.str() +
-        "' does not have a dag initializer!";
+  PrintFatalError(getLoc(), "Record `" + getName() + "', field `" +
+    FieldName.str() + "' does not have a dag initializer!");
 }
 
 
@@ -2019,7 +1976,7 @@ std::vector<Record*>
 RecordKeeper::getAllDerivedDefinitions(const std::string &ClassName) const {
   Record *Class = getClass(ClassName);
   if (!Class)
-    throw "ERROR: Couldn't find the `" + ClassName + "' class!\n";
+    PrintFatalError("ERROR: Couldn't find the `" + ClassName + "' class!\n");
 
   std::vector<Record*> Defs;
   for (std::map<std::string, Record*>::const_iterator I = getDefs().begin(),
@@ -2034,7 +1991,7 @@ RecordKeeper::getAllDerivedDefinitions(const std::string &ClassName) const {
 /// to CurRec's name.
 Init *llvm::QualifyName(Record &CurRec, MultiClass *CurMultiClass,
                         Init *Name, const std::string &Scoper) {
-  RecTy *Type = dynamic_cast<TypedInit *>(Name)->getType();
+  RecTy *Type = dyn_cast<TypedInit>(Name)->getType();
 
   BinOpInit *NewName =
     BinOpInit::get(BinOpInit::STRCONCAT, 
diff --git a/lib/TableGen/TGParser.cpp b/lib/TableGen/TGParser.cpp
index b9c7ff6..b1f9f72 100644
--- a/lib/TableGen/TGParser.cpp
+++ b/lib/TableGen/TGParser.cpp
@@ -93,7 +93,7 @@ bool TGParser::SetValue(Record *CurRec, SMLoc Loc, Init *ValName,
   // Do not allow assignments like 'X = X'.  This will just cause infinite loops
   // in the resolution machinery.
   if (BitList.empty())
-    if (VarInit *VI = dynamic_cast<VarInit*>(V))
+    if (VarInit *VI = dyn_cast<VarInit>(V))
       if (VI->getNameInit() == ValName)
         return false;
 
@@ -102,7 +102,7 @@ bool TGParser::SetValue(Record *CurRec, SMLoc Loc, Init *ValName,
   // initializer.
   //
   if (!BitList.empty()) {
-    BitsInit *CurVal = dynamic_cast<BitsInit*>(RV->getValue());
+    BitsInit *CurVal = dyn_cast<BitsInit>(RV->getValue());
     if (CurVal == 0)
       return Error(Loc, "Value '" + ValName->getAsUnquotedString()
                    + "' is not a bits type");
@@ -110,12 +110,11 @@ bool TGParser::SetValue(Record *CurRec, SMLoc Loc, Init *ValName,
     // Convert the incoming value to a bits type of the appropriate size...
     Init *BI = V->convertInitializerTo(BitsRecTy::get(BitList.size()));
     if (BI == 0) {
-      V->convertInitializerTo(BitsRecTy::get(BitList.size()));
       return Error(Loc, "Initializer is not compatible with bit range");
     }
 
     // We should have a BitsInit type now.
-    BitsInit *BInit = dynamic_cast<BitsInit*>(BI);
+    BitsInit *BInit = dyn_cast<BitsInit>(BI);
     assert(BInit != 0);
 
     SmallVector<Init *, 16> NewBits(CurVal->getNumBits());
@@ -311,7 +310,7 @@ bool TGParser::ProcessForeachDefs(Record *CurRec, SMLoc Loc, IterSet &IterVals){
   if (IterVals.size() != Loops.size()) {
     assert(IterVals.size() < Loops.size());
     ForeachLoop &CurLoop = Loops[IterVals.size()];
-    ListInit *List = dynamic_cast<ListInit *>(CurLoop.ListValue);
+    ListInit *List = dyn_cast<ListInit>(CurLoop.ListValue);
     if (List == 0) {
       Error(Loc, "Loop list is not a list");
       return true;
@@ -336,7 +335,7 @@ bool TGParser::ProcessForeachDefs(Record *CurRec, SMLoc Loc, IterSet &IterVals){
   // Set the iterator values now.
   for (unsigned i = 0, e = IterVals.size(); i != e; ++i) {
     VarInit *IterVar = IterVals[i].IterVar;
-    TypedInit *IVal = dynamic_cast<TypedInit *>(IterVals[i].IterValue);
+    TypedInit *IVal = dyn_cast<TypedInit>(IterVals[i].IterValue);
     if (IVal == 0) {
       Error(Loc, "foreach iterator value is untyped");
       return true;
@@ -407,8 +406,7 @@ Init *TGParser::ParseObjectName(MultiClass *CurMultiClass) {
 
   RecTy *Type = 0;
   if (CurRec) {
-    const TypedInit *CurRecName =
-      dynamic_cast<const TypedInit *>(CurRec->getNameInit());
+    const TypedInit *CurRecName = dyn_cast<TypedInit>(CurRec->getNameInit());
     if (!CurRecName) {
       TokError("Record name is not typed!");
       return 0;
@@ -781,7 +779,7 @@ Init *TGParser::ParseIDValue(Record *CurRec,
   for (LoopVector::iterator i = Loops.begin(), iend = Loops.end();
        i != iend;
        ++i) {
-    VarInit *IterVar = dynamic_cast<VarInit *>(i->IterVar);
+    VarInit *IterVar = dyn_cast<VarInit>(i->IterVar);
     if (IterVar && IterVar->getName() == Name)
       return IterVar;
   }
@@ -856,16 +854,16 @@ Init *TGParser::ParseOperation(Record *CurRec) {
     if (Code == UnOpInit::HEAD
         || Code == UnOpInit::TAIL
         || Code == UnOpInit::EMPTY) {
-      ListInit *LHSl = dynamic_cast<ListInit*>(LHS);
-      StringInit *LHSs = dynamic_cast<StringInit*>(LHS);
-      TypedInit *LHSt = dynamic_cast<TypedInit*>(LHS);
+      ListInit *LHSl = dyn_cast<ListInit>(LHS);
+      StringInit *LHSs = dyn_cast<StringInit>(LHS);
+      TypedInit *LHSt = dyn_cast<TypedInit>(LHS);
       if (LHSl == 0 && LHSs == 0 && LHSt == 0) {
         TokError("expected list or string type argument in unary operator");
         return 0;
       }
       if (LHSt) {
-        ListRecTy *LType = dynamic_cast<ListRecTy*>(LHSt->getType());
-        StringRecTy *SType = dynamic_cast<StringRecTy*>(LHSt->getType());
+        ListRecTy *LType = dyn_cast<ListRecTy>(LHSt->getType());
+        StringRecTy *SType = dyn_cast<StringRecTy>(LHSt->getType());
         if (LType == 0 && SType == 0) {
           TokError("expected list or string type argumnet in unary operator");
           return 0;
@@ -885,7 +883,7 @@ Init *TGParser::ParseOperation(Record *CurRec) {
         }
         if (LHSl) {
           Init *Item = LHSl->getElement(0);
-          TypedInit *Itemt = dynamic_cast<TypedInit*>(Item);
+          TypedInit *Itemt = dyn_cast<TypedInit>(Item);
           if (Itemt == 0) {
             TokError("untyped list element in unary operator");
             return 0;
@@ -897,7 +895,7 @@ Init *TGParser::ParseOperation(Record *CurRec) {
           }
         } else {
           assert(LHSt && "expected list type argument in unary operator");
-          ListRecTy *LType = dynamic_cast<ListRecTy*>(LHSt->getType());
+          ListRecTy *LType = dyn_cast<ListRecTy>(LHSt->getType());
           if (LType == 0) {
             TokError("expected list type argumnet in unary operator");
             return 0;
@@ -1044,35 +1042,28 @@ Init *TGParser::ParseOperation(Record *CurRec) {
     switch (LexCode) {
     default: llvm_unreachable("Unhandled code!");
     case tgtok::XIf: {
-      // FIXME: The `!if' operator doesn't handle non-TypedInit well at
-      // all. This can be made much more robust.
-      TypedInit *MHSt = dynamic_cast<TypedInit*>(MHS);
-      TypedInit *RHSt = dynamic_cast<TypedInit*>(RHS);
-
       RecTy *MHSTy = 0;
       RecTy *RHSTy = 0;
 
-      if (MHSt == 0 && RHSt == 0) {
-        BitsInit *MHSbits = dynamic_cast<BitsInit*>(MHS);
-        BitsInit *RHSbits = dynamic_cast<BitsInit*>(RHS);
-
-        if (MHSbits && RHSbits &&
-            MHSbits->getNumBits() == RHSbits->getNumBits()) {
-          Type = BitRecTy::get();
-          break;
-        } else {
-          BitInit *MHSbit = dynamic_cast<BitInit*>(MHS);
-          BitInit *RHSbit = dynamic_cast<BitInit*>(RHS);
-
-          if (MHSbit && RHSbit) {
-            Type = BitRecTy::get();
-            break;
-          }
-        }
-      } else if (MHSt != 0 && RHSt != 0) {
+      if (TypedInit *MHSt = dyn_cast<TypedInit>(MHS))
         MHSTy = MHSt->getType();
+      if (BitsInit *MHSbits = dyn_cast<BitsInit>(MHS))
+        MHSTy = BitsRecTy::get(MHSbits->getNumBits());
+      if (isa<BitInit>(MHS))
+        MHSTy = BitRecTy::get();
+
+      if (TypedInit *RHSt = dyn_cast<TypedInit>(RHS))
         RHSTy = RHSt->getType();
-      }
+      if (BitsInit *RHSbits = dyn_cast<BitsInit>(RHS))
+        RHSTy = BitsRecTy::get(RHSbits->getNumBits());
+      if (isa<BitInit>(RHS))
+        RHSTy = BitRecTy::get();
+
+      // For UnsetInit, it's typed from the other hand.
+      if (isa<UnsetInit>(MHS))
+        MHSTy = RHSTy;
+      if (isa<UnsetInit>(RHS))
+        RHSTy = MHSTy;
 
       if (!MHSTy || !RHSTy) {
         TokError("could not get type for !if");
@@ -1090,7 +1081,7 @@ Init *TGParser::ParseOperation(Record *CurRec) {
       break;
     }
     case tgtok::XForEach: {
-      TypedInit *MHSt = dynamic_cast<TypedInit *>(MHS);
+      TypedInit *MHSt = dyn_cast<TypedInit>(MHS);
       if (MHSt == 0) {
         TokError("could not get type for !foreach");
         return 0;
@@ -1099,7 +1090,7 @@ Init *TGParser::ParseOperation(Record *CurRec) {
       break;
     }
     case tgtok::XSubst: {
-      TypedInit *RHSt = dynamic_cast<TypedInit *>(RHS);
+      TypedInit *RHSt = dyn_cast<TypedInit>(RHS);
       if (RHSt == 0) {
         TokError("could not get type for !subst");
         return 0;
@@ -1278,7 +1269,7 @@ Init *TGParser::ParseSimpleValue(Record *CurRec, RecTy *ItemType,
     ListRecTy *GivenListTy = 0;
 
     if (ItemType != 0) {
-      ListRecTy *ListType = dynamic_cast<ListRecTy*>(ItemType);
+      ListRecTy *ListType = dyn_cast<ListRecTy>(ItemType);
       if (ListType == 0) {
         std::stringstream s;
         s << "Type mismatch for list, expected list type, got "
@@ -1323,7 +1314,7 @@ Init *TGParser::ParseSimpleValue(Record *CurRec, RecTy *ItemType,
     for (std::vector<Init *>::iterator i = Vals.begin(), ie = Vals.end();
          i != ie;
          ++i) {
-      TypedInit *TArg = dynamic_cast<TypedInit*>(*i);
+      TypedInit *TArg = dyn_cast<TypedInit>(*i);
       if (TArg == 0) {
         TokError("Untyped list element");
         return 0;
@@ -1506,7 +1497,7 @@ Init *TGParser::ParseValue(Record *CurRec, RecTy *ItemType, IDParseMode Mode) {
       // Create a !strconcat() operation, first casting each operand to
       // a string if necessary.
 
-      TypedInit *LHS = dynamic_cast<TypedInit *>(Result);
+      TypedInit *LHS = dyn_cast<TypedInit>(Result);
       if (!LHS) {
         Error(PasteLoc, "LHS of paste is not typed!");
         return 0;
@@ -1533,7 +1524,7 @@ Init *TGParser::ParseValue(Record *CurRec, RecTy *ItemType, IDParseMode Mode) {
 
       default:
         Init *RHSResult = ParseValue(CurRec, ItemType, ParseNameMode);
-        RHS = dynamic_cast<TypedInit *>(RHSResult);
+        RHS = dyn_cast<TypedInit>(RHSResult);
         if (!RHS) {
           Error(PasteLoc, "RHS of paste is not typed!");
           return 0;
@@ -1724,13 +1715,13 @@ VarInit *TGParser::ParseForeachDeclaration(ListInit *&ForeachListValue) {
   default: TokError("Unknown token when expecting a range list"); return 0;
   case tgtok::l_square: { // '[' ValueList ']'
     Init *List = ParseSimpleValue(0, 0, ParseForeachMode);
-    ForeachListValue = dynamic_cast<ListInit*>(List);
+    ForeachListValue = dyn_cast<ListInit>(List);
     if (ForeachListValue == 0) {
       TokError("Expected a Value list");
       return 0;
     }
     RecTy *ValueType = ForeachListValue->getType();
-    ListRecTy *ListType = dynamic_cast<ListRecTy *>(ValueType);
+    ListRecTy *ListType = dyn_cast<ListRecTy>(ValueType);
     if (ListType == 0) {
       TokError("Value list is not of list type");
       return 0;
@@ -2265,7 +2256,7 @@ InstantiateMulticlassDef(MultiClass &MC,
 
   Init *DefName = DefProto->getNameInit();
 
-  StringInit *DefNameString = dynamic_cast<StringInit *>(DefName);
+  StringInit *DefNameString = dyn_cast<StringInit>(DefName);
 
   if (DefNameString != 0) {
     // We have a fully expanded string so there are no operators to
@@ -2277,7 +2268,10 @@ InstantiateMulticlassDef(MultiClass &MC,
                      DefName, StringRecTy::get())->Fold(DefProto, &MC);
   }
 
-  Record *CurRec = new Record(DefName, DefmPrefixLoc, Records);
+  // Make a trail of SMLocs from the multiclass instantiations.
+  SmallVector<SMLoc, 4> Locs(1, DefmPrefixLoc);
+  Locs.append(DefProto->getLoc().begin(), DefProto->getLoc().end());
+  Record *CurRec = new Record(DefName, Locs, Records);
 
   SubClassReference Ref;
   Ref.RefLoc = DefmPrefixLoc;
diff --git a/lib/TableGen/TGParser.h b/lib/TableGen/TGParser.h
index 3d2c72c..9c2ad43 100644
--- a/lib/TableGen/TGParser.h
+++ b/lib/TableGen/TGParser.h
@@ -30,7 +30,7 @@ namespace llvm {
   struct MultiClass;
   struct SubClassReference;
   struct SubMultiClassReference;
-  
+
   struct LetRecord {
     std::string Name;
     std::vector<unsigned> Bits;
@@ -41,7 +41,7 @@ namespace llvm {
       : Name(N), Bits(B), Value(V), Loc(L) {
     }
   };
-  
+
   /// ForeachLoop - Record the iteration state associated with a for loop.
   /// This is used to instantiate items in the loop body.
   struct ForeachLoop {
@@ -56,13 +56,13 @@ class TGParser {
   TGLexer Lex;
   std::vector<std::vector<LetRecord> > LetStack;
   std::map<std::string, MultiClass*> MultiClasses;
-  
+
   /// Loops - Keep track of any foreach loops we are within.
   ///
   typedef std::vector<ForeachLoop> LoopVector;
   LoopVector Loops;
 
-  /// CurMultiClass - If we are parsing a 'multiclass' definition, this is the 
+  /// CurMultiClass - If we are parsing a 'multiclass' definition, this is the
   /// current value.
   MultiClass *CurMultiClass;
 
@@ -82,13 +82,13 @@ class TGParser {
   };
 
 public:
-  TGParser(SourceMgr &SrcMgr, RecordKeeper &records) : 
+  TGParser(SourceMgr &SrcMgr, RecordKeeper &records) :
     Lex(SrcMgr), CurMultiClass(0), Records(records) {}
-  
+
   /// ParseFile - Main entrypoint for parsing a tblgen file.  These parser
   /// routines return true on error, or false on success.
   bool ParseFile();
-  
+
   bool Error(SMLoc L, const Twine &Msg) const {
     PrintError(L, Msg);
     return true;
@@ -102,9 +102,9 @@ public:
 
 private:  // Semantic analysis methods.
   bool AddValue(Record *TheRec, SMLoc Loc, const RecordVal &RV);
-  bool SetValue(Record *TheRec, SMLoc Loc, Init *ValName, 
+  bool SetValue(Record *TheRec, SMLoc Loc, Init *ValName,
                 const std::vector<unsigned> &BitList, Init *V);
-  bool SetValue(Record *TheRec, SMLoc Loc, const std::string &ValName, 
+  bool SetValue(Record *TheRec, SMLoc Loc, const std::string &ValName,
                 const std::vector<unsigned> &BitList, Init *V) {
     return SetValue(TheRec, Loc, StringInit::get(ValName), BitList, V);
   }
@@ -170,7 +170,8 @@ private:  // Parser methods.
                          IDParseMode Mode = ParseValueMode);
   Init *ParseValue(Record *CurRec, RecTy *ItemType = 0,
                    IDParseMode Mode = ParseValueMode);
-  std::vector<Init*> ParseValueList(Record *CurRec, Record *ArgsRec = 0, RecTy *EltTy = 0);
+  std::vector<Init*> ParseValueList(Record *CurRec, Record *ArgsRec = 0,
+                                    RecTy *EltTy = 0);
   std::vector<std::pair<llvm::Init*, std::string> > ParseDagArgList(Record *);
   bool ParseOptionalRangeList(std::vector<unsigned> &Ranges);
   bool ParseOptionalBitList(std::vector<unsigned> &Ranges);
@@ -184,7 +185,7 @@ private:  // Parser methods.
   MultiClass *ParseMultiClassID();
   Record *ParseDefmID();
 };
-  
+
 } // end namespace llvm
 
 #endif
diff --git a/lib/TableGen/TableGenAction.cpp b/lib/TableGen/TableGenAction.cpp
deleted file mode 100644
index 54e5083..0000000
--- a/lib/TableGen/TableGenAction.cpp
+++ /dev/null
@@ -1,15 +0,0 @@
-//===- TableGenAction.cpp - defines TableGenAction --------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/TableGen/TableGenAction.h"
-
-using namespace llvm;
-
-void TableGenAction::anchor() { }
-
diff --git a/lib/Target/ARM/ARM.h b/lib/Target/ARM/ARM.h
index 2a1e8e4..1446bbb 100644
--- a/lib/Target/ARM/ARM.h
+++ b/lib/Target/ARM/ARM.h
@@ -37,6 +37,7 @@ FunctionPass *createARMJITCodeEmitterPass(ARMBaseTargetMachine &TM,
 
 FunctionPass *createARMLoadStoreOptimizationPass(bool PreAlloc = false);
 FunctionPass *createARMExpandPseudoPass();
+FunctionPass *createARMGlobalBaseRegPass();
 FunctionPass *createARMGlobalMergePass(const TargetLowering* tli);
 FunctionPass *createARMConstantIslandPass();
 FunctionPass *createMLxExpansionPass();
diff --git a/lib/Target/ARM/ARM.td b/lib/Target/ARM/ARM.td
index 69e2346..23974ad 100644
--- a/lib/Target/ARM/ARM.td
+++ b/lib/Target/ARM/ARM.td
@@ -32,9 +32,6 @@ def FeatureVFP2 : SubtargetFeature<"vfp2", "HasVFPv2", "true",
 def FeatureVFP3 : SubtargetFeature<"vfp3", "HasVFPv3", "true",
                                    "Enable VFP3 instructions",
                                    [FeatureVFP2]>;
-def FeatureVFP4 : SubtargetFeature<"vfp4", "HasVFPv4", "true",
-                                   "Enable VFP4 instructions",
-                                   [FeatureVFP3]>;
 def FeatureNEON : SubtargetFeature<"neon", "HasNEON", "true",
                                    "Enable NEON instructions",
                                    [FeatureVFP3]>;
@@ -44,10 +41,16 @@ def FeatureNoARM  : SubtargetFeature<"noarm", "NoARM", "true",
                                      "Does not support ARM mode execution">;
 def FeatureFP16   : SubtargetFeature<"fp16", "HasFP16", "true",
                                      "Enable half-precision floating point">;
+def FeatureVFP4   : SubtargetFeature<"vfp4", "HasVFPv4", "true",
+                                     "Enable VFP4 instructions",
+                                     [FeatureVFP3, FeatureFP16]>;
 def FeatureD16    : SubtargetFeature<"d16", "HasD16", "true",
                                      "Restrict VFP3 to 16 double registers">;
 def FeatureHWDiv  : SubtargetFeature<"hwdiv", "HasHardwareDivide", "true",
                                      "Enable divide instructions">;
+def FeatureHWDivARM  : SubtargetFeature<"hwdiv-arm",
+                                        "HasHardwareDivideInARM", "true",
+                                      "Enable divide instructions in ARM mode">;
 def FeatureT2XtPk : SubtargetFeature<"t2xtpk", "HasT2ExtractPack", "true",
                                  "Enable Thumb2 extract and pack instructions">;
 def FeatureDB     : SubtargetFeature<"db", "HasDataBarrier", "true",
@@ -139,6 +142,18 @@ def ProcA9      : SubtargetFeature<"a9", "ARMProcFamily", "CortexA9",
                                    [FeatureVMLxForwarding,
                                     FeatureT2XtPk, FeatureFP16,
                                     FeatureAvoidPartialCPSR]>;
+def ProcSwift   : SubtargetFeature<"swift", "ARMProcFamily", "Swift",
+                                   "Swift ARM processors",
+                                   [FeatureNEONForFP, FeatureT2XtPk,
+                                    FeatureVFP4, FeatureMP, FeatureHWDiv,
+                                    FeatureHWDivARM, FeatureAvoidPartialCPSR,
+                                    FeatureHasSlowFPVMLx]>;
+
+// FIXME: It has not been determined if A15 has these features.
+def ProcA15      : SubtargetFeature<"a15", "ARMProcFamily", "CortexA15",
+                                   "Cortex-A15 ARM processors",
+                                   [FeatureT2XtPk, FeatureFP16,
+                                    FeatureAvoidPartialCPSR]>;
 
 class ProcNoItin<string Name, list<SubtargetFeature> Features>
  : Processor<Name, NoItineraries, Features>;
@@ -214,6 +229,10 @@ def : ProcessorModel<"cortex-a9-mp", CortexA9Model,
                                     [ProcA9, HasV7Ops, FeatureNEON, FeatureDB,
                                      FeatureDSPThumb2, FeatureMP,
                                      FeatureHasRAS]>;
+// FIXME: A15 has currently the same ProcessorModel as A9.
+def : ProcessorModel<"cortex-a15",   CortexA9Model,
+                                    [ProcA15, HasV7Ops, FeatureNEON, FeatureDB,
+                                     FeatureDSPThumb2, FeatureHasRAS]>;
 
 // V7M Processors.
 def : ProcNoItin<"cortex-m3",       [HasV7Ops,
@@ -227,6 +246,12 @@ def : ProcNoItin<"cortex-m4",       [HasV7Ops,
                                      FeatureT2XtPk, FeatureVFP4,
                                      FeatureVFPOnlySP, FeatureMClass]>;
 
+// Swift uArch Processors.
+def : ProcessorModel<"swift",       SwiftModel,
+                                    [ProcSwift, HasV7Ops, FeatureNEON,
+                                     FeatureDB, FeatureDSPThumb2,
+                                     FeatureHasRAS]>;
+
 //===----------------------------------------------------------------------===//
 // Register File Description
 //===----------------------------------------------------------------------===//
diff --git a/lib/Target/ARM/ARMAsmPrinter.cpp b/lib/Target/ARM/ARMAsmPrinter.cpp
index e9e2803..d439d1d 100644
--- a/lib/Target/ARM/ARMAsmPrinter.cpp
+++ b/lib/Target/ARM/ARMAsmPrinter.cpp
@@ -23,6 +23,8 @@
 #include "InstPrinter/ARMInstPrinter.h"
 #include "MCTargetDesc/ARMAddressingModes.h"
 #include "MCTargetDesc/ARMMCExpr.h"
+#include "llvm/ADT/SetVector.h"
+#include "llvm/ADT/SmallString.h"
 #include "llvm/Constants.h"
 #include "llvm/DebugInfo.h"
 #include "llvm/Module.h"
@@ -40,9 +42,8 @@
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/Target/Mangler.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetMachine.h"
-#include "llvm/ADT/SmallString.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -302,7 +303,7 @@ void ARMAsmPrinter::EmitFunctionEntryLabel() {
 }
 
 void ARMAsmPrinter::EmitXXStructor(const Constant *CV) {
-  uint64_t Size = TM.getTargetData()->getTypeAllocSize(CV->getType());
+  uint64_t Size = TM.getDataLayout()->getTypeAllocSize(CV->getType());
   assert(Size && "C++ constructor pointer had zero size!");
 
   const GlobalValue *GV = dyn_cast<GlobalValue>(CV->stripPointerCasts());
@@ -389,16 +390,6 @@ void ARMAsmPrinter::printOperand(const MachineInstr *MI, int OpNum,
 //===--------------------------------------------------------------------===//
 
 MCSymbol *ARMAsmPrinter::
-GetARMSetPICJumpTableLabel2(unsigned uid, unsigned uid2,
-                            const MachineBasicBlock *MBB) const {
-  SmallString<60> Name;
-  raw_svector_ostream(Name) << MAI->getPrivateGlobalPrefix()
-    << getFunctionNumber() << '_' << uid << '_' << uid2
-    << "_set_" << MBB->getNumber();
-  return OutContext.GetOrCreateSymbol(Name.str());
-}
-
-MCSymbol *ARMAsmPrinter::
 GetARMJTIPICJumpTableLabel2(unsigned uid, unsigned uid2) const {
   SmallString<60> Name;
   raw_svector_ostream(Name) << MAI->getPrivateGlobalPrefix() << "JTI"
@@ -592,9 +583,24 @@ void ARMAsmPrinter::EmitStartOfAsmFile(Module &M) {
       const TargetLoweringObjectFileMachO &TLOFMacho =
         static_cast<const TargetLoweringObjectFileMachO &>(
           getObjFileLowering());
-      OutStreamer.SwitchSection(TLOFMacho.getTextSection());
-      OutStreamer.SwitchSection(TLOFMacho.getTextCoalSection());
-      OutStreamer.SwitchSection(TLOFMacho.getConstTextCoalSection());
+
+      // Collect the set of sections our functions will go into.
+      SetVector<const MCSection *, SmallVector<const MCSection *, 8>,
+        SmallPtrSet<const MCSection *, 8> > TextSections;
+      // Default text section comes first.
+      TextSections.insert(TLOFMacho.getTextSection());
+      // Now any user defined text sections from function attributes.
+      for (Module::iterator F = M.begin(), e = M.end(); F != e; ++F)
+        if (!F->isDeclaration() && !F->hasAvailableExternallyLinkage())
+          TextSections.insert(TLOFMacho.SectionForGlobal(F, Mang, TM));
+      // Now the coalescable sections.
+      TextSections.insert(TLOFMacho.getTextCoalSection());
+      TextSections.insert(TLOFMacho.getConstTextCoalSection());
+
+      // Emit the sections in the .s file header to fix the order.
+      for (unsigned i = 0, e = TextSections.size(); i != e; ++i)
+        OutStreamer.SwitchSection(TextSections[i]);
+
       if (RelocM == Reloc::DynamicNoPIC) {
         const MCSection *sect =
           OutContext.getMachOSection("__TEXT", "__symbol_stub4",
@@ -743,13 +749,28 @@ void ARMAsmPrinter::emitAttributes() {
     AttrEmitter->EmitAttribute(ARMBuildAttrs::THUMB_ISA_use,
                                ARMBuildAttrs::Allowed);
   } else if (CPUString == "generic") {
-    // FIXME: Why these defaults?
-    AttrEmitter->EmitAttribute(ARMBuildAttrs::CPU_arch, ARMBuildAttrs::v4T);
+    // For a generic CPU, we assume a standard v7a architecture in Subtarget.
+    AttrEmitter->EmitAttribute(ARMBuildAttrs::CPU_arch, ARMBuildAttrs::v7);
+    AttrEmitter->EmitAttribute(ARMBuildAttrs::CPU_arch_profile,
+                               ARMBuildAttrs::ApplicationProfile);
     AttrEmitter->EmitAttribute(ARMBuildAttrs::ARM_ISA_use,
                                ARMBuildAttrs::Allowed);
     AttrEmitter->EmitAttribute(ARMBuildAttrs::THUMB_ISA_use,
-                               ARMBuildAttrs::Allowed);
-  }
+                               ARMBuildAttrs::AllowThumb32);
+  } else if (Subtarget->hasV7Ops()) {
+    AttrEmitter->EmitAttribute(ARMBuildAttrs::CPU_arch, ARMBuildAttrs::v7);
+    AttrEmitter->EmitAttribute(ARMBuildAttrs::THUMB_ISA_use,
+                               ARMBuildAttrs::AllowThumb32);
+  } else if (Subtarget->hasV6T2Ops())
+    AttrEmitter->EmitAttribute(ARMBuildAttrs::CPU_arch, ARMBuildAttrs::v6T2);
+  else if (Subtarget->hasV6Ops())
+    AttrEmitter->EmitAttribute(ARMBuildAttrs::CPU_arch, ARMBuildAttrs::v6);
+  else if (Subtarget->hasV5TEOps())
+    AttrEmitter->EmitAttribute(ARMBuildAttrs::CPU_arch, ARMBuildAttrs::v5TE);
+  else if (Subtarget->hasV5TOps())
+    AttrEmitter->EmitAttribute(ARMBuildAttrs::CPU_arch, ARMBuildAttrs::v5T);
+  else if (Subtarget->hasV4TOps())
+    AttrEmitter->EmitAttribute(ARMBuildAttrs::CPU_arch, ARMBuildAttrs::v4T);
 
   if (Subtarget->hasNEON() && emitFPU) {
     /* NEON is not exactly a VFP architecture, but GAS emit one of
@@ -893,7 +914,7 @@ MCSymbol *ARMAsmPrinter::GetARMGVSymbol(const GlobalValue *GV) {
 
 void ARMAsmPrinter::
 EmitMachineConstantPoolValue(MachineConstantPoolValue *MCPV) {
-  int Size = TM.getTargetData()->getTypeAllocSize(MCPV->getType());
+  int Size = TM.getDataLayout()->getTypeAllocSize(MCPV->getType());
 
   ARMConstantPoolValue *ACPV = static_cast<ARMConstantPoolValue*>(MCPV);
 
@@ -1091,16 +1112,6 @@ static void populateADROperands(MCInst &Inst, unsigned Dest,
   Inst.addOperand(MCOperand::CreateReg(ccreg));
 }
 
-void ARMAsmPrinter::EmitPatchedInstruction(const MachineInstr *MI,
-                                           unsigned Opcode) {
-  MCInst TmpInst;
-
-  // Emit the instruction as usual, just patch the opcode.
-  LowerARMMachineInstrToMCInst(MI, TmpInst, *this);
-  TmpInst.setOpcode(Opcode);
-  OutStreamer.EmitInstruction(TmpInst);
-}
-
 void ARMAsmPrinter::EmitUnwindingInstruction(const MachineInstr *MI) {
   assert(MI->getFlag(MachineInstr::FrameSetup) &&
       "Only instruction which are involved into frame setup code are allowed");
@@ -1402,31 +1413,6 @@ void ARMAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     }
     return;
   }
-  case ARM::t2BMOVPCB_CALL: {
-    {
-      MCInst TmpInst;
-      TmpInst.setOpcode(ARM::tMOVr);
-      TmpInst.addOperand(MCOperand::CreateReg(ARM::LR));
-      TmpInst.addOperand(MCOperand::CreateReg(ARM::PC));
-      // Add predicate operands.
-      TmpInst.addOperand(MCOperand::CreateImm(ARMCC::AL));
-      TmpInst.addOperand(MCOperand::CreateReg(0));
-      OutStreamer.EmitInstruction(TmpInst);
-    }
-    {
-      MCInst TmpInst;
-      TmpInst.setOpcode(ARM::t2B);
-      const GlobalValue *GV = MI->getOperand(0).getGlobal();
-      MCSymbol *GVSym = Mang->getSymbol(GV);
-      const MCExpr *GVSymExpr = MCSymbolRefExpr::Create(GVSym, OutContext);
-      TmpInst.addOperand(MCOperand::CreateExpr(GVSymExpr));
-      // Add predicate operands.
-      TmpInst.addOperand(MCOperand::CreateImm(ARMCC::AL));
-      TmpInst.addOperand(MCOperand::CreateReg(0));
-      OutStreamer.EmitInstruction(TmpInst);
-    }
-    return;
-  }
   case ARM::MOVi16_ga_pcrel:
   case ARM::t2MOVi16_ga_pcrel: {
     MCInst TmpInst;
diff --git a/lib/Target/ARM/ARMAsmPrinter.h b/lib/Target/ARM/ARMAsmPrinter.h
index 3555e8f5..c875b2c 100644
--- a/lib/Target/ARM/ARMAsmPrinter.h
+++ b/lib/Target/ARM/ARMAsmPrinter.h
@@ -53,7 +53,7 @@ public:
       Subtarget = &TM.getSubtarget<ARMSubtarget>();
     }
 
-  virtual const char *getPassName() const {
+  virtual const char *getPassName() const LLVM_OVERRIDE {
     return "ARM Assembly Printer";
   }
 
@@ -62,22 +62,24 @@ public:
 
   virtual bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
                                unsigned AsmVariant, const char *ExtraCode,
-                               raw_ostream &O);
+                               raw_ostream &O) LLVM_OVERRIDE;
   virtual bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum,
-                                     unsigned AsmVariant,
-                                     const char *ExtraCode, raw_ostream &O);
+                                     unsigned AsmVariant, const char *ExtraCode,
+                                     raw_ostream &O) LLVM_OVERRIDE;
 
   void EmitJumpTable(const MachineInstr *MI);
   void EmitJump2Table(const MachineInstr *MI);
-  virtual void EmitInstruction(const MachineInstr *MI);
-  bool runOnMachineFunction(MachineFunction &F);
+  virtual void EmitInstruction(const MachineInstr *MI) LLVM_OVERRIDE;
+  virtual bool runOnMachineFunction(MachineFunction &F) LLVM_OVERRIDE;
 
-  virtual void EmitConstantPool() {} // we emit constant pools customly!
-  virtual void EmitFunctionBodyEnd();
-  virtual void EmitFunctionEntryLabel();
-  void EmitStartOfAsmFile(Module &M);
-  void EmitEndOfAsmFile(Module &M);
-  void EmitXXStructor(const Constant *CV);
+  virtual void EmitConstantPool() LLVM_OVERRIDE {
+    // we emit constant pools customly!
+  }
+  virtual void EmitFunctionBodyEnd() LLVM_OVERRIDE;
+  virtual void EmitFunctionEntryLabel() LLVM_OVERRIDE;
+  virtual void EmitStartOfAsmFile(Module &M) LLVM_OVERRIDE;
+  virtual void EmitEndOfAsmFile(Module &M) LLVM_OVERRIDE;
+  virtual void EmitXXStructor(const Constant *CV) LLVM_OVERRIDE;
 
   // lowerOperand - Convert a MachineOperand into the equivalent MCOperand.
   bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp);
@@ -101,12 +103,13 @@ private:
 public:
   void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS);
 
-  MachineLocation getDebugValueLocation(const MachineInstr *MI) const;
+  virtual MachineLocation
+    getDebugValueLocation(const MachineInstr *MI) const LLVM_OVERRIDE;
 
   /// EmitDwarfRegOp - Emit dwarf register operation.
-  virtual void EmitDwarfRegOp(const MachineLocation &MLoc) const;
+  virtual void EmitDwarfRegOp(const MachineLocation &MLoc) const LLVM_OVERRIDE;
 
-  virtual unsigned getISAEncoding() {
+  virtual unsigned getISAEncoding() LLVM_OVERRIDE {
     // ARM/Darwin adds ISA to the DWARF info for each function.
     if (!Subtarget->isTargetDarwin())
       return 0;
@@ -114,18 +117,19 @@ public:
       ARM::DW_ISA_ARM_thumb : ARM::DW_ISA_ARM_arm;
   }
 
+private:
   MCOperand GetSymbolRef(const MachineOperand &MO, const MCSymbol *Symbol);
-  MCSymbol *GetARMSetPICJumpTableLabel2(unsigned uid, unsigned uid2,
-                                        const MachineBasicBlock *MBB) const;
   MCSymbol *GetARMJTIPICJumpTableLabel2(unsigned uid, unsigned uid2) const;
 
   MCSymbol *GetARMSJLJEHLabel(void) const;
 
   MCSymbol *GetARMGVSymbol(const GlobalValue *GV);
 
+public:
   /// EmitMachineConstantPoolValue - Print a machine constantpool value to
   /// the .s file.
-  virtual void EmitMachineConstantPoolValue(MachineConstantPoolValue *MCPV);
+  virtual void
+    EmitMachineConstantPoolValue(MachineConstantPoolValue *MCPV) LLVM_OVERRIDE;
 };
 } // end namespace llvm
 
diff --git a/lib/Target/ARM/ARMBaseInstrInfo.cpp b/lib/Target/ARM/ARMBaseInstrInfo.cpp
index 1cc5a17..3c7bb24 100644
--- a/lib/Target/ARM/ARMBaseInstrInfo.cpp
+++ b/lib/Target/ARM/ARMBaseInstrInfo.cpp
@@ -49,6 +49,11 @@ static cl::opt<bool>
 WidenVMOVS("widen-vmovs", cl::Hidden, cl::init(true),
            cl::desc("Widen ARM vmovs to vmovd when possible"));
 
+static cl::opt<unsigned>
+SwiftPartialUpdateClearance("swift-partial-update-clearance",
+     cl::Hidden, cl::init(12),
+     cl::desc("Clearance before partial register updates"));
+
 /// ARM_MLxEntry - Record information about MLA / MLS instructions.
 struct ARM_MLxEntry {
   uint16_t MLxOpc;     // MLA / MLS opcode
@@ -683,7 +688,7 @@ void ARMBaseInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
   // Handle register classes that require multiple instructions.
   unsigned BeginIdx = 0;
   unsigned SubRegs = 0;
-  unsigned Spacing = 1;
+  int Spacing = 1;
 
   // Use VORRq when possible.
   if (ARM::QQPRRegClass.contains(DestReg, SrcReg))
@@ -697,6 +702,8 @@ void ARMBaseInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
     Opc = ARM::VMOVD, BeginIdx = ARM::dsub_0, SubRegs = 3;
   else if (ARM::DQuadRegClass.contains(DestReg, SrcReg))
     Opc = ARM::VMOVD, BeginIdx = ARM::dsub_0, SubRegs = 4;
+  else if (ARM::GPRPairRegClass.contains(DestReg, SrcReg))
+    Opc = ARM::MOVr, BeginIdx = ARM::gsub_0, SubRegs = 2;
 
   else if (ARM::DPairSpcRegClass.contains(DestReg, SrcReg))
     Opc = ARM::VMOVD, BeginIdx = ARM::dsub_0, SubRegs = 2, Spacing = 2;
@@ -705,27 +712,38 @@ void ARMBaseInstrInfo::copyPhysReg(MachineBasicBlock &MBB,
   else if (ARM::DQuadSpcRegClass.contains(DestReg, SrcReg))
     Opc = ARM::VMOVD, BeginIdx = ARM::dsub_0, SubRegs = 4, Spacing = 2;
 
-  if (Opc) {
-    const TargetRegisterInfo *TRI = &getRegisterInfo();
-    MachineInstrBuilder Mov;
-    for (unsigned i = 0; i != SubRegs; ++i) {
-      unsigned Dst = TRI->getSubReg(DestReg, BeginIdx + i*Spacing);
-      unsigned Src = TRI->getSubReg(SrcReg,  BeginIdx + i*Spacing);
-      assert(Dst && Src && "Bad sub-register");
-      Mov = AddDefaultPred(BuildMI(MBB, I, I->getDebugLoc(), get(Opc), Dst)
-                             .addReg(Src));
-      // VORR takes two source operands.
-      if (Opc == ARM::VORRq)
-        Mov.addReg(Src);
-    }
-    // Add implicit super-register defs and kills to the last instruction.
-    Mov->addRegisterDefined(DestReg, TRI);
-    if (KillSrc)
-      Mov->addRegisterKilled(SrcReg, TRI);
-    return;
-  }
+  assert(Opc && "Impossible reg-to-reg copy");
+
+  const TargetRegisterInfo *TRI = &getRegisterInfo();
+  MachineInstrBuilder Mov;
 
-  llvm_unreachable("Impossible reg-to-reg copy");
+  // Copy register tuples backward when the first Dest reg overlaps with SrcReg.
+  if (TRI->regsOverlap(SrcReg, TRI->getSubReg(DestReg, BeginIdx))) {
+    BeginIdx = BeginIdx + ((SubRegs-1)*Spacing);
+    Spacing = -Spacing;
+  }
+#ifndef NDEBUG
+  SmallSet<unsigned, 4> DstRegs;
+#endif
+  for (unsigned i = 0; i != SubRegs; ++i) {
+    unsigned Dst = TRI->getSubReg(DestReg, BeginIdx + i*Spacing);
+    unsigned Src = TRI->getSubReg(SrcReg,  BeginIdx + i*Spacing);
+    assert(Dst && Src && "Bad sub-register");
+#ifndef NDEBUG
+    assert(!DstRegs.count(Src) && "destructive vector copy");
+    DstRegs.insert(Dst);
+#endif
+    Mov = BuildMI(MBB, I, I->getDebugLoc(), get(Opc), Dst)
+      .addReg(Src);
+    // VORR takes two source operands.
+    if (Opc == ARM::VORRq)
+      Mov.addReg(Src);
+    Mov = AddDefaultPred(Mov);
+  }
+  // Add implicit super-register defs and kills to the last instruction.
+  Mov->addRegisterDefined(DestReg, TRI);
+  if (KillSrc)
+    Mov->addRegisterKilled(SrcReg, TRI);
 }
 
 static const
@@ -775,6 +793,13 @@ storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
         AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VSTRD))
                    .addReg(SrcReg, getKillRegState(isKill))
                    .addFrameIndex(FI).addImm(0).addMemOperand(MMO));
+      } else if (ARM::GPRPairRegClass.hasSubClassEq(RC)) {
+        MachineInstrBuilder MIB =
+          AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::STMIA))
+                       .addFrameIndex(FI))
+                       .addMemOperand(MMO);
+          MIB = AddDReg(MIB, SrcReg, ARM::gsub_0, getKillRegState(isKill), TRI);
+                AddDReg(MIB, SrcReg, ARM::gsub_1, 0, TRI);
       } else
         llvm_unreachable("Unknown reg class!");
       break;
@@ -922,6 +947,7 @@ loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
   DebugLoc DL;
   if (I != MBB.end()) DL = I->getDebugLoc();
   MachineFunction &MF = *MBB.getParent();
+  ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
   MachineFrameInfo &MFI = *MF.getFrameInfo();
   unsigned Align = MFI.getObjectAlignment(FI);
   MachineMemOperand *MMO =
@@ -947,6 +973,15 @@ loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
     if (ARM::DPRRegClass.hasSubClassEq(RC)) {
       AddDefaultPred(BuildMI(MBB, I, DL, get(ARM::VLDRD), DestReg)
                    .addFrameIndex(FI).addImm(0).addMemOperand(MMO));
+    } else if (ARM::GPRPairRegClass.hasSubClassEq(RC)) {
+      unsigned LdmOpc = AFI->isThumbFunction() ? ARM::t2LDMIA : ARM::LDMIA;
+      MachineInstrBuilder MIB =
+        AddDefaultPred(BuildMI(MBB, I, DL, get(LdmOpc))
+                    .addFrameIndex(FI).addImm(0).addMemOperand(MMO));
+      MIB = AddDReg(MIB, DestReg, ARM::gsub_0, RegState::DefineNoRead, TRI);
+      MIB = AddDReg(MIB, DestReg, ARM::gsub_1, RegState::DefineNoRead, TRI);
+      if (TargetRegisterInfo::isPhysicalRegister(DestReg))
+        MIB.addReg(DestReg, RegState::ImplicitDefine);
     } else
       llvm_unreachable("Unknown reg class!");
     break;
@@ -1378,7 +1413,6 @@ bool ARMBaseInstrInfo::areLoadsFromSameBasePtr(SDNode *Load1, SDNode *Load2,
   case ARM::VLDRD:
   case ARM::VLDRS:
   case ARM::t2LDRi8:
-  case ARM::t2LDRDi8:
   case ARM::t2LDRSHi8:
   case ARM::t2LDRi12:
   case ARM::t2LDRSHi12:
@@ -1517,6 +1551,14 @@ isProfitableToIfCvt(MachineBasicBlock &TMBB,
   return (TCycles + FCycles + TExtra + FExtra) <= UnpredCost;
 }
 
+bool
+ARMBaseInstrInfo::isProfitableToUnpredicate(MachineBasicBlock &TMBB,
+                                            MachineBasicBlock &FMBB) const {
+  // Reduce false anti-dependencies to let Swift's out-of-order execution
+  // engine do its thing.
+  return Subtarget.isSwift();
+}
+
 /// getInstrPredicate - If instruction is predicated, returns its predicate
 /// condition, otherwise returns AL. It also returns the condition code
 /// register by reference.
@@ -1569,71 +1611,41 @@ ARMBaseInstrInfo::commuteInstruction(MachineInstr *MI, bool NewMI) const {
 }
 
 /// Identify instructions that can be folded into a MOVCC instruction, and
-/// return the corresponding opcode for the predicated pseudo-instruction.
-static unsigned canFoldIntoMOVCC(unsigned Reg, MachineInstr *&MI,
-                                 const MachineRegisterInfo &MRI) {
+/// return the defining instruction.
+static MachineInstr *canFoldIntoMOVCC(unsigned Reg,
+                                      const MachineRegisterInfo &MRI,
+                                      const TargetInstrInfo *TII) {
   if (!TargetRegisterInfo::isVirtualRegister(Reg))
     return 0;
   if (!MRI.hasOneNonDBGUse(Reg))
     return 0;
-  MI = MRI.getVRegDef(Reg);
+  MachineInstr *MI = MRI.getVRegDef(Reg);
   if (!MI)
     return 0;
+  // MI is folded into the MOVCC by predicating it.
+  if (!MI->isPredicable())
+    return 0;
   // Check if MI has any non-dead defs or physreg uses. This also detects
   // predicated instructions which will be reading CPSR.
   for (unsigned i = 1, e = MI->getNumOperands(); i != e; ++i) {
     const MachineOperand &MO = MI->getOperand(i);
+    // Reject frame index operands, PEI can't handle the predicated pseudos.
+    if (MO.isFI() || MO.isCPI() || MO.isJTI())
+      return 0;
     if (!MO.isReg())
       continue;
+    // MI can't have any tied operands, that would conflict with predication.
+    if (MO.isTied())
+      return 0;
     if (TargetRegisterInfo::isPhysicalRegister(MO.getReg()))
       return 0;
     if (MO.isDef() && !MO.isDead())
       return 0;
   }
-  switch (MI->getOpcode()) {
-  default: return 0;
-  case ARM::ANDri:   return ARM::ANDCCri;
-  case ARM::ANDrr:   return ARM::ANDCCrr;
-  case ARM::ANDrsi:  return ARM::ANDCCrsi;
-  case ARM::ANDrsr:  return ARM::ANDCCrsr;
-  case ARM::t2ANDri: return ARM::t2ANDCCri;
-  case ARM::t2ANDrr: return ARM::t2ANDCCrr;
-  case ARM::t2ANDrs: return ARM::t2ANDCCrs;
-  case ARM::EORri:   return ARM::EORCCri;
-  case ARM::EORrr:   return ARM::EORCCrr;
-  case ARM::EORrsi:  return ARM::EORCCrsi;
-  case ARM::EORrsr:  return ARM::EORCCrsr;
-  case ARM::t2EORri: return ARM::t2EORCCri;
-  case ARM::t2EORrr: return ARM::t2EORCCrr;
-  case ARM::t2EORrs: return ARM::t2EORCCrs;
-  case ARM::ORRri:   return ARM::ORRCCri;
-  case ARM::ORRrr:   return ARM::ORRCCrr;
-  case ARM::ORRrsi:  return ARM::ORRCCrsi;
-  case ARM::ORRrsr:  return ARM::ORRCCrsr;
-  case ARM::t2ORRri: return ARM::t2ORRCCri;
-  case ARM::t2ORRrr: return ARM::t2ORRCCrr;
-  case ARM::t2ORRrs: return ARM::t2ORRCCrs;
-
-  // ARM ADD/SUB
-  case ARM::ADDri:   return ARM::ADDCCri;
-  case ARM::ADDrr:   return ARM::ADDCCrr;
-  case ARM::ADDrsi:  return ARM::ADDCCrsi;
-  case ARM::ADDrsr:  return ARM::ADDCCrsr;
-  case ARM::SUBri:   return ARM::SUBCCri;
-  case ARM::SUBrr:   return ARM::SUBCCrr;
-  case ARM::SUBrsi:  return ARM::SUBCCrsi;
-  case ARM::SUBrsr:  return ARM::SUBCCrsr;
-
-  // Thumb2 ADD/SUB
-  case ARM::t2ADDri:   return ARM::t2ADDCCri;
-  case ARM::t2ADDri12: return ARM::t2ADDCCri12;
-  case ARM::t2ADDrr:   return ARM::t2ADDCCrr;
-  case ARM::t2ADDrs:   return ARM::t2ADDCCrs;
-  case ARM::t2SUBri:   return ARM::t2SUBCCri;
-  case ARM::t2SUBri12: return ARM::t2SUBCCri12;
-  case ARM::t2SUBrr:   return ARM::t2SUBCCrr;
-  case ARM::t2SUBrs:   return ARM::t2SUBCCrs;
-  }
+  bool DontMoveAcrossStores = true;
+  if (!MI->isSafeToMove(TII, /* AliasAnalysis = */ 0, DontMoveAcrossStores))
+    return 0;
+  return MI;
 }
 
 bool ARMBaseInstrInfo::analyzeSelect(const MachineInstr *MI,
@@ -1662,19 +1674,18 @@ MachineInstr *ARMBaseInstrInfo::optimizeSelect(MachineInstr *MI,
   assert((MI->getOpcode() == ARM::MOVCCr || MI->getOpcode() == ARM::t2MOVCCr) &&
          "Unknown select instruction");
   const MachineRegisterInfo &MRI = MI->getParent()->getParent()->getRegInfo();
-  MachineInstr *DefMI = 0;
-  unsigned Opc = canFoldIntoMOVCC(MI->getOperand(2).getReg(), DefMI, MRI);
-  bool Invert = !Opc;
-  if (!Opc)
-    Opc = canFoldIntoMOVCC(MI->getOperand(1).getReg(), DefMI, MRI);
-  if (!Opc)
+  MachineInstr *DefMI = canFoldIntoMOVCC(MI->getOperand(2).getReg(), MRI, this);
+  bool Invert = !DefMI;
+  if (!DefMI)
+    DefMI = canFoldIntoMOVCC(MI->getOperand(1).getReg(), MRI, this);
+  if (!DefMI)
     return 0;
 
   // Create a new predicated version of DefMI.
   // Rfalse is the first use.
   MachineInstrBuilder NewMI = BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
-                                      get(Opc), MI->getOperand(0).getReg())
-    .addOperand(MI->getOperand(Invert ? 2 : 1));
+                                      DefMI->getDesc(),
+                                      MI->getOperand(0).getReg());
 
   // Copy all the DefMI operands, excluding its (null) predicate.
   const MCInstrDesc &DefDesc = DefMI->getDesc();
@@ -1693,6 +1704,15 @@ MachineInstr *ARMBaseInstrInfo::optimizeSelect(MachineInstr *MI,
   if (NewMI->hasOptionalDef())
     AddDefaultCC(NewMI);
 
+  // The output register value when the predicate is false is an implicit
+  // register operand tied to the first def.
+  // The tie makes the register allocator ensure the FalseReg is allocated the
+  // same register as operand 0.
+  MachineOperand FalseReg = MI->getOperand(Invert ? 2 : 1);
+  FalseReg.setImplicit();
+  NewMI->addOperand(FalseReg);
+  NewMI->tieOperands(0, NewMI->getNumOperands() - 1);
+
   // The caller will erase MI, but not DefMI.
   DefMI->eraseFromParent();
   return NewMI;
@@ -2039,13 +2059,14 @@ optimizeCompareInstr(MachineInstr *CmpInstr, unsigned SrcReg, unsigned SrcReg2,
 
   // Masked compares sometimes use the same register as the corresponding 'and'.
   if (CmpMask != ~0) {
-    if (!isSuitableForMask(MI, SrcReg, CmpMask, false)) {
+    if (!isSuitableForMask(MI, SrcReg, CmpMask, false) || isPredicated(MI)) {
       MI = 0;
       for (MachineRegisterInfo::use_iterator UI = MRI->use_begin(SrcReg),
            UE = MRI->use_end(); UI != UE; ++UI) {
         if (UI->getParent() != CmpInstr->getParent()) continue;
         MachineInstr *PotentialAND = &*UI;
-        if (!isSuitableForMask(PotentialAND, SrcReg, CmpMask, true))
+        if (!isSuitableForMask(PotentialAND, SrcReg, CmpMask, true) ||
+            isPredicated(PotentialAND))
           continue;
         MI = PotentialAND;
         break;
@@ -2111,6 +2132,10 @@ optimizeCompareInstr(MachineInstr *CmpInstr, unsigned SrcReg, unsigned SrcReg2,
   // The single candidate is called MI.
   if (!MI) MI = Sub;
 
+  // We can't use a predicated instruction - it doesn't always write the flags.
+  if (isPredicated(MI))
+    return false;
+
   switch (MI->getOpcode()) {
   default: break;
   case ARM::RSBrr:
@@ -2217,6 +2242,7 @@ optimizeCompareInstr(MachineInstr *CmpInstr, unsigned SrcReg, unsigned SrcReg2,
     // Toggle the optional operand to CPSR.
     MI->getOperand(5).setReg(ARM::CPSR);
     MI->getOperand(5).setIsDef(true);
+    assert(!isPredicated(MI) && "Can't use flags from predicated instruction");
     CmpInstr->eraseFromParent();
 
     // Modify the condition code of operands in OperandsToUpdate.
@@ -2347,6 +2373,260 @@ bool ARMBaseInstrInfo::FoldImmediate(MachineInstr *UseMI,
   return true;
 }
 
+static unsigned getNumMicroOpsSwiftLdSt(const InstrItineraryData *ItinData,
+                                        const MachineInstr *MI) {
+  switch (MI->getOpcode()) {
+  default: {
+    const MCInstrDesc &Desc = MI->getDesc();
+    int UOps = ItinData->getNumMicroOps(Desc.getSchedClass());
+    assert(UOps >= 0 && "bad # UOps");
+    return UOps;
+  }
+
+  case ARM::LDRrs:
+  case ARM::LDRBrs:
+  case ARM::STRrs:
+  case ARM::STRBrs: {
+    unsigned ShOpVal = MI->getOperand(3).getImm();
+    bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub;
+    unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
+    if (!isSub &&
+        (ShImm == 0 ||
+         ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
+          ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)))
+      return 1;
+    return 2;
+  }
+
+  case ARM::LDRH:
+  case ARM::STRH: {
+    if (!MI->getOperand(2).getReg())
+      return 1;
+
+    unsigned ShOpVal = MI->getOperand(3).getImm();
+    bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub;
+    unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
+    if (!isSub &&
+        (ShImm == 0 ||
+         ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
+          ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)))
+      return 1;
+    return 2;
+  }
+
+  case ARM::LDRSB:
+  case ARM::LDRSH:
+    return (ARM_AM::getAM3Op(MI->getOperand(3).getImm()) == ARM_AM::sub) ? 3:2;
+
+  case ARM::LDRSB_POST:
+  case ARM::LDRSH_POST: {
+    unsigned Rt = MI->getOperand(0).getReg();
+    unsigned Rm = MI->getOperand(3).getReg();
+    return (Rt == Rm) ? 4 : 3;
+  }
+
+  case ARM::LDR_PRE_REG:
+  case ARM::LDRB_PRE_REG: {
+    unsigned Rt = MI->getOperand(0).getReg();
+    unsigned Rm = MI->getOperand(3).getReg();
+    if (Rt == Rm)
+      return 3;
+    unsigned ShOpVal = MI->getOperand(4).getImm();
+    bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub;
+    unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
+    if (!isSub &&
+        (ShImm == 0 ||
+         ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
+          ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)))
+      return 2;
+    return 3;
+  }
+
+  case ARM::STR_PRE_REG:
+  case ARM::STRB_PRE_REG: {
+    unsigned ShOpVal = MI->getOperand(4).getImm();
+    bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub;
+    unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
+    if (!isSub &&
+        (ShImm == 0 ||
+         ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
+          ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)))
+      return 2;
+    return 3;
+  }
+
+  case ARM::LDRH_PRE:
+  case ARM::STRH_PRE: {
+    unsigned Rt = MI->getOperand(0).getReg();
+    unsigned Rm = MI->getOperand(3).getReg();
+    if (!Rm)
+      return 2;
+    if (Rt == Rm)
+      return 3;
+    return (ARM_AM::getAM3Op(MI->getOperand(4).getImm()) == ARM_AM::sub)
+      ? 3 : 2;
+  }
+
+  case ARM::LDR_POST_REG:
+  case ARM::LDRB_POST_REG:
+  case ARM::LDRH_POST: {
+    unsigned Rt = MI->getOperand(0).getReg();
+    unsigned Rm = MI->getOperand(3).getReg();
+    return (Rt == Rm) ? 3 : 2;
+  }
+
+  case ARM::LDR_PRE_IMM:
+  case ARM::LDRB_PRE_IMM:
+  case ARM::LDR_POST_IMM:
+  case ARM::LDRB_POST_IMM:
+  case ARM::STRB_POST_IMM:
+  case ARM::STRB_POST_REG:
+  case ARM::STRB_PRE_IMM:
+  case ARM::STRH_POST:
+  case ARM::STR_POST_IMM:
+  case ARM::STR_POST_REG:
+  case ARM::STR_PRE_IMM:
+    return 2;
+
+  case ARM::LDRSB_PRE:
+  case ARM::LDRSH_PRE: {
+    unsigned Rm = MI->getOperand(3).getReg();
+    if (Rm == 0)
+      return 3;
+    unsigned Rt = MI->getOperand(0).getReg();
+    if (Rt == Rm)
+      return 4;
+    unsigned ShOpVal = MI->getOperand(4).getImm();
+    bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub;
+    unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
+    if (!isSub &&
+        (ShImm == 0 ||
+         ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
+          ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)))
+      return 3;
+    return 4;
+  }
+
+  case ARM::LDRD: {
+    unsigned Rt = MI->getOperand(0).getReg();
+    unsigned Rn = MI->getOperand(2).getReg();
+    unsigned Rm = MI->getOperand(3).getReg();
+    if (Rm)
+      return (ARM_AM::getAM3Op(MI->getOperand(4).getImm()) == ARM_AM::sub) ?4:3;
+    return (Rt == Rn) ? 3 : 2;
+  }
+
+  case ARM::STRD: {
+    unsigned Rm = MI->getOperand(3).getReg();
+    if (Rm)
+      return (ARM_AM::getAM3Op(MI->getOperand(4).getImm()) == ARM_AM::sub) ?4:3;
+    return 2;
+  }
+
+  case ARM::LDRD_POST:
+  case ARM::t2LDRD_POST:
+    return 3;
+
+  case ARM::STRD_POST:
+  case ARM::t2STRD_POST:
+    return 4;
+
+  case ARM::LDRD_PRE: {
+    unsigned Rt = MI->getOperand(0).getReg();
+    unsigned Rn = MI->getOperand(3).getReg();
+    unsigned Rm = MI->getOperand(4).getReg();
+    if (Rm)
+      return (ARM_AM::getAM3Op(MI->getOperand(5).getImm()) == ARM_AM::sub) ?5:4;
+    return (Rt == Rn) ? 4 : 3;
+  }
+
+  case ARM::t2LDRD_PRE: {
+    unsigned Rt = MI->getOperand(0).getReg();
+    unsigned Rn = MI->getOperand(3).getReg();
+    return (Rt == Rn) ? 4 : 3;
+  }
+
+  case ARM::STRD_PRE: {
+    unsigned Rm = MI->getOperand(4).getReg();
+    if (Rm)
+      return (ARM_AM::getAM3Op(MI->getOperand(5).getImm()) == ARM_AM::sub) ?5:4;
+    return 3;
+  }
+
+  case ARM::t2STRD_PRE:
+    return 3;
+
+  case ARM::t2LDR_POST:
+  case ARM::t2LDRB_POST:
+  case ARM::t2LDRB_PRE:
+  case ARM::t2LDRSBi12:
+  case ARM::t2LDRSBi8:
+  case ARM::t2LDRSBpci:
+  case ARM::t2LDRSBs:
+  case ARM::t2LDRH_POST:
+  case ARM::t2LDRH_PRE:
+  case ARM::t2LDRSBT:
+  case ARM::t2LDRSB_POST:
+  case ARM::t2LDRSB_PRE:
+  case ARM::t2LDRSH_POST:
+  case ARM::t2LDRSH_PRE:
+  case ARM::t2LDRSHi12:
+  case ARM::t2LDRSHi8:
+  case ARM::t2LDRSHpci:
+  case ARM::t2LDRSHs:
+    return 2;
+
+  case ARM::t2LDRDi8: {
+    unsigned Rt = MI->getOperand(0).getReg();
+    unsigned Rn = MI->getOperand(2).getReg();
+    return (Rt == Rn) ? 3 : 2;
+  }
+
+  case ARM::t2STRB_POST:
+  case ARM::t2STRB_PRE:
+  case ARM::t2STRBs:
+  case ARM::t2STRDi8:
+  case ARM::t2STRH_POST:
+  case ARM::t2STRH_PRE:
+  case ARM::t2STRHs:
+  case ARM::t2STR_POST:
+  case ARM::t2STR_PRE:
+  case ARM::t2STRs:
+    return 2;
+  }
+}
+
+// Return the number of 32-bit words loaded by LDM or stored by STM. If this
+// can't be easily determined return 0 (missing MachineMemOperand).
+//
+// FIXME: The current MachineInstr design does not support relying on machine
+// mem operands to determine the width of a memory access. Instead, we expect
+// the target to provide this information based on the instruction opcode and
+// operands. However, using MachineMemOperand is a the best solution now for
+// two reasons:
+//
+// 1) getNumMicroOps tries to infer LDM memory width from the total number of MI
+// operands. This is much more dangerous than using the MachineMemOperand
+// sizes because CodeGen passes can insert/remove optional machine operands. In
+// fact, it's totally incorrect for preRA passes and appears to be wrong for
+// postRA passes as well.
+//
+// 2) getNumLDMAddresses is only used by the scheduling machine model and any
+// machine model that calls this should handle the unknown (zero size) case.
+//
+// Long term, we should require a target hook that verifies MachineMemOperand
+// sizes during MC lowering. That target hook should be local to MC lowering
+// because we can't ensure that it is aware of other MI forms. Doing this will
+// ensure that MachineMemOperands are correctly propagated through all passes.
+unsigned ARMBaseInstrInfo::getNumLDMAddresses(const MachineInstr *MI) const {
+  unsigned Size = 0;
+  for (MachineInstr::mmo_iterator I = MI->memoperands_begin(),
+         E = MI->memoperands_end(); I != E; ++I) {
+    Size += (*I)->getSize();
+  }
+  return Size / 4;
+}
+
 unsigned
 ARMBaseInstrInfo::getNumMicroOps(const InstrItineraryData *ItinData,
                                  const MachineInstr *MI) const {
@@ -2356,8 +2636,12 @@ ARMBaseInstrInfo::getNumMicroOps(const InstrItineraryData *ItinData,
   const MCInstrDesc &Desc = MI->getDesc();
   unsigned Class = Desc.getSchedClass();
   int ItinUOps = ItinData->getNumMicroOps(Class);
-  if (ItinUOps >= 0)
+  if (ItinUOps >= 0) {
+    if (Subtarget.isSwift() && (Desc.mayLoad() || Desc.mayStore()))
+      return getNumMicroOpsSwiftLdSt(ItinData, MI);
+
     return ItinUOps;
+  }
 
   unsigned Opc = MI->getOpcode();
   switch (Opc) {
@@ -2426,7 +2710,43 @@ ARMBaseInstrInfo::getNumMicroOps(const InstrItineraryData *ItinData,
   case ARM::t2STMIA_UPD:
   case ARM::t2STMDB_UPD: {
     unsigned NumRegs = MI->getNumOperands() - Desc.getNumOperands() + 1;
-    if (Subtarget.isCortexA8()) {
+    if (Subtarget.isSwift()) {
+      // rdar://8402126
+      int UOps = 1 + NumRegs;  // One for address computation, one for each ld / st.
+      switch (Opc) {
+      default: break;
+      case ARM::VLDMDIA_UPD:
+      case ARM::VLDMDDB_UPD:
+      case ARM::VLDMSIA_UPD:
+      case ARM::VLDMSDB_UPD:
+      case ARM::VSTMDIA_UPD:
+      case ARM::VSTMDDB_UPD:
+      case ARM::VSTMSIA_UPD:
+      case ARM::VSTMSDB_UPD:
+      case ARM::LDMIA_UPD:
+      case ARM::LDMDA_UPD:
+      case ARM::LDMDB_UPD:
+      case ARM::LDMIB_UPD:
+      case ARM::STMIA_UPD:
+      case ARM::STMDA_UPD:
+      case ARM::STMDB_UPD:
+      case ARM::STMIB_UPD:
+      case ARM::tLDMIA_UPD:
+      case ARM::tSTMIA_UPD:
+      case ARM::t2LDMIA_UPD:
+      case ARM::t2LDMDB_UPD:
+      case ARM::t2STMIA_UPD:
+      case ARM::t2STMDB_UPD:
+        ++UOps; // One for base register writeback.
+        break;
+      case ARM::LDMIA_RET:
+      case ARM::tPOP_RET:
+      case ARM::t2LDMIA_RET:
+        UOps += 2; // One for base reg wb, one for write to pc.
+        break;
+      }
+      return UOps;
+    } else if (Subtarget.isCortexA8()) {
       if (NumRegs < 4)
         return 2;
       // 4 registers would be issued: 2, 2.
@@ -2435,7 +2755,7 @@ ARMBaseInstrInfo::getNumMicroOps(const InstrItineraryData *ItinData,
       if (NumRegs % 2)
         ++A8UOps;
       return A8UOps;
-    } else if (Subtarget.isCortexA9()) {
+    } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) {
       int A9UOps = (NumRegs / 2);
       // If there are odd number of registers or if it's not 64-bit aligned,
       // then it takes an extra AGU (Address Generation Unit) cycle.
@@ -2468,7 +2788,7 @@ ARMBaseInstrInfo::getVLDMDefCycle(const InstrItineraryData *ItinData,
     DefCycle = RegNo / 2 + 1;
     if (RegNo % 2)
       ++DefCycle;
-  } else if (Subtarget.isCortexA9()) {
+  } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) {
     DefCycle = RegNo;
     bool isSLoad = false;
 
@@ -2512,7 +2832,7 @@ ARMBaseInstrInfo::getLDMDefCycle(const InstrItineraryData *ItinData,
       DefCycle = 1;
     // Result latency is issue cycle + 2: E2.
     DefCycle += 2;
-  } else if (Subtarget.isCortexA9()) {
+  } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) {
     DefCycle = (RegNo / 2);
     // If there are odd number of registers or if it's not 64-bit aligned,
     // then it takes an extra AGU (Address Generation Unit) cycle.
@@ -2543,7 +2863,7 @@ ARMBaseInstrInfo::getVSTMUseCycle(const InstrItineraryData *ItinData,
     UseCycle = RegNo / 2 + 1;
     if (RegNo % 2)
       ++UseCycle;
-  } else if (Subtarget.isCortexA9()) {
+  } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) {
     UseCycle = RegNo;
     bool isSStore = false;
 
@@ -2584,7 +2904,7 @@ ARMBaseInstrInfo::getSTMUseCycle(const InstrItineraryData *ItinData,
       UseCycle = 2;
     // Read in E3.
     UseCycle += 2;
-  } else if (Subtarget.isCortexA9()) {
+  } else if (Subtarget.isLikeA9() || Subtarget.isSwift()) {
     UseCycle = (RegNo / 2);
     // If there are odd number of registers or if it's not 64-bit aligned,
     // then it takes an extra AGU (Address Generation Unit) cycle.
@@ -2769,7 +3089,7 @@ static int adjustDefLatency(const ARMSubtarget &Subtarget,
                             const MachineInstr *DefMI,
                             const MCInstrDesc *DefMCID, unsigned DefAlign) {
   int Adjust = 0;
-  if (Subtarget.isCortexA8() || Subtarget.isCortexA9()) {
+  if (Subtarget.isCortexA8() || Subtarget.isLikeA9()) {
     // FIXME: Shifter op hack: no shift (i.e. [r +/- r]) or [r + r << 2]
     // variants are one cycle cheaper.
     switch (DefMCID->getOpcode()) {
@@ -2794,9 +3114,40 @@ static int adjustDefLatency(const ARMSubtarget &Subtarget,
       break;
     }
     }
+  } else if (Subtarget.isSwift()) {
+    // FIXME: Properly handle all of the latency adjustments for address
+    // writeback.
+    switch (DefMCID->getOpcode()) {
+    default: break;
+    case ARM::LDRrs:
+    case ARM::LDRBrs: {
+      unsigned ShOpVal = DefMI->getOperand(3).getImm();
+      bool isSub = ARM_AM::getAM2Op(ShOpVal) == ARM_AM::sub;
+      unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
+      if (!isSub &&
+          (ShImm == 0 ||
+           ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
+            ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl)))
+        Adjust -= 2;
+      else if (!isSub &&
+               ShImm == 1 && ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsr)
+        --Adjust;
+      break;
+    }
+    case ARM::t2LDRs:
+    case ARM::t2LDRBs:
+    case ARM::t2LDRHs:
+    case ARM::t2LDRSHs: {
+      // Thumb2 mode: lsl only.
+      unsigned ShAmt = DefMI->getOperand(3).getImm();
+      if (ShAmt == 0 || ShAmt == 1 || ShAmt == 2 || ShAmt == 3)
+        Adjust -= 2;
+      break;
+    }
+    }
   }
 
-  if (DefAlign < 8 && Subtarget.isCortexA9()) {
+  if (DefAlign < 8 && Subtarget.isLikeA9()) {
     switch (DefMCID->getOpcode()) {
     default: break;
     case ARM::VLD1q8:
@@ -2954,7 +3305,7 @@ ARMBaseInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
   if (Reg == ARM::CPSR) {
     if (DefMI->getOpcode() == ARM::FMSTAT) {
       // fpscr -> cpsr stalls over 20 cycles on A8 (and earlier?)
-      return Subtarget.isCortexA9() ? 1 : 20;
+      return Subtarget.isLikeA9() ? 1 : 20;
     }
 
     // CPSR set and branch can be paired in the same cycle.
@@ -2970,7 +3321,8 @@ ARMBaseInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
     // instructions).
     if (Latency > 0 && Subtarget.isThumb2()) {
       const MachineFunction *MF = DefMI->getParent()->getParent();
-      if (MF->getFunction()->hasFnAttr(Attribute::OptimizeForSize))
+      if (MF->getFunction()->getFnAttributes().
+            hasAttribute(Attributes::OptimizeForSize))
         --Latency;
     }
     return Latency;
@@ -3020,7 +3372,7 @@ ARMBaseInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
 
   if (!UseNode->isMachineOpcode()) {
     int Latency = ItinData->getOperandCycle(DefMCID.getSchedClass(), DefIdx);
-    if (Subtarget.isCortexA9())
+    if (Subtarget.isLikeA9() || Subtarget.isSwift())
       return Latency <= 2 ? 1 : Latency - 1;
     else
       return Latency <= 3 ? 1 : Latency - 2;
@@ -3037,7 +3389,7 @@ ARMBaseInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
                                   UseMCID, UseIdx, UseAlign);
 
   if (Latency > 1 &&
-      (Subtarget.isCortexA8() || Subtarget.isCortexA9())) {
+      (Subtarget.isCortexA8() || Subtarget.isLikeA9())) {
     // FIXME: Shifter op hack: no shift (i.e. [r +/- r]) or [r + r << 2]
     // variants are one cycle cheaper.
     switch (DefMCID.getOpcode()) {
@@ -3064,9 +3416,36 @@ ARMBaseInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
       break;
     }
     }
+  } else if (DefIdx == 0 && Latency > 2 && Subtarget.isSwift()) {
+    // FIXME: Properly handle all of the latency adjustments for address
+    // writeback.
+    switch (DefMCID.getOpcode()) {
+    default: break;
+    case ARM::LDRrs:
+    case ARM::LDRBrs: {
+      unsigned ShOpVal =
+        cast<ConstantSDNode>(DefNode->getOperand(2))->getZExtValue();
+      unsigned ShImm = ARM_AM::getAM2Offset(ShOpVal);
+      if (ShImm == 0 ||
+          ((ShImm == 1 || ShImm == 2 || ShImm == 3) &&
+           ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsl))
+        Latency -= 2;
+      else if (ShImm == 1 && ARM_AM::getAM2ShiftOpc(ShOpVal) == ARM_AM::lsr)
+        --Latency;
+      break;
+    }
+    case ARM::t2LDRs:
+    case ARM::t2LDRBs:
+    case ARM::t2LDRHs:
+    case ARM::t2LDRSHs: {
+      // Thumb2 mode: lsl 0-3 only.
+      Latency -= 2;
+      break;
+    }
+    }
   }
 
-  if (DefAlign < 8 && Subtarget.isCortexA9())
+  if (DefAlign < 8 && Subtarget.isLikeA9())
     switch (DefMCID.getOpcode()) {
     default: break;
     case ARM::VLD1q8:
@@ -3190,18 +3569,6 @@ ARMBaseInstrInfo::getOperandLatency(const InstrItineraryData *ItinData,
   return Latency;
 }
 
-unsigned
-ARMBaseInstrInfo::getOutputLatency(const InstrItineraryData *ItinData,
-                                   const MachineInstr *DefMI, unsigned DefIdx,
-                                   const MachineInstr *DepMI) const {
-  unsigned Reg = DefMI->getOperand(DefIdx).getReg();
-  if (DepMI->readsRegister(Reg, &getRegisterInfo()) || !isPredicated(DepMI))
-    return 1;
-
-  // If the second MI is predicated, then there is an implicit use dependency.
-  return getInstrLatency(ItinData, DefMI);
-}
-
 unsigned ARMBaseInstrInfo::getInstrLatency(const InstrItineraryData *ItinData,
                                            const MachineInstr *MI,
                                            unsigned *PredCost) const {
@@ -3359,11 +3726,12 @@ ARMBaseInstrInfo::getExecutionDomain(const MachineInstr *MI) const {
   if (MI->getOpcode() == ARM::VMOVD && !isPredicated(MI))
     return std::make_pair(ExeVFP, (1<<ExeVFP) | (1<<ExeNEON));
 
-  // Cortex-A9 is particularly picky about mixing the two and wants these
+  // A9-like cores are particularly picky about mixing the two and want these
   // converted.
-  if (Subtarget.isCortexA9() && !isPredicated(MI) &&
+  if (Subtarget.isLikeA9() && !isPredicated(MI) &&
       (MI->getOpcode() == ARM::VMOVRS ||
-       MI->getOpcode() == ARM::VMOVSR))
+       MI->getOpcode() == ARM::VMOVSR ||
+       MI->getOpcode() == ARM::VMOVS))
     return std::make_pair(ExeVFP, (1<<ExeVFP) | (1<<ExeNEON));
 
   // No other instructions can be swizzled, so just determine their domain.
@@ -3383,13 +3751,70 @@ ARMBaseInstrInfo::getExecutionDomain(const MachineInstr *MI) const {
   return std::make_pair(ExeGeneric, 0);
 }
 
+static unsigned getCorrespondingDRegAndLane(const TargetRegisterInfo *TRI,
+                                            unsigned SReg, unsigned &Lane) {
+  unsigned DReg = TRI->getMatchingSuperReg(SReg, ARM::ssub_0, &ARM::DPRRegClass);
+  Lane = 0;
+
+  if (DReg != ARM::NoRegister)
+   return DReg;
+
+  Lane = 1;
+  DReg = TRI->getMatchingSuperReg(SReg, ARM::ssub_1, &ARM::DPRRegClass);
+
+  assert(DReg && "S-register with no D super-register?");
+  return DReg;
+}
+
+/// getImplicitSPRUseForDPRUse - Given a use of a DPR register and lane,
+/// set ImplicitSReg to a register number that must be marked as implicit-use or
+/// zero if no register needs to be defined as implicit-use.
+///
+/// If the function cannot determine if an SPR should be marked implicit use or
+/// not, it returns false.
+///
+/// This function handles cases where an instruction is being modified from taking
+/// an SPR to a DPR[Lane]. A use of the DPR is being added, which may conflict
+/// with an earlier def of an SPR corresponding to DPR[Lane^1] (i.e. the other
+/// lane of the DPR).
+///
+/// If the other SPR is defined, an implicit-use of it should be added. Else,
+/// (including the case where the DPR itself is defined), it should not.
+///
+static bool getImplicitSPRUseForDPRUse(const TargetRegisterInfo *TRI,
+                                       MachineInstr *MI,
+                                       unsigned DReg, unsigned Lane,
+                                       unsigned &ImplicitSReg) {
+  // If the DPR is defined or used already, the other SPR lane will be chained
+  // correctly, so there is nothing to be done.
+  if (MI->definesRegister(DReg, TRI) || MI->readsRegister(DReg, TRI)) {
+    ImplicitSReg = 0;
+    return true;
+  }
+
+  // Otherwise we need to go searching to see if the SPR is set explicitly.
+  ImplicitSReg = TRI->getSubReg(DReg,
+                                (Lane & 1) ? ARM::ssub_0 : ARM::ssub_1);
+  MachineBasicBlock::LivenessQueryResult LQR =
+    MI->getParent()->computeRegisterLiveness(TRI, ImplicitSReg, MI);
+
+  if (LQR == MachineBasicBlock::LQR_Live)
+    return true;
+  else if (LQR == MachineBasicBlock::LQR_Unknown)
+    return false;
+
+  // If the register is known not to be live, there is no need to add an
+  // implicit-use.
+  ImplicitSReg = 0;
+  return true;
+}
+
 void
 ARMBaseInstrInfo::setExecutionDomain(MachineInstr *MI, unsigned Domain) const {
   unsigned DstReg, SrcReg, DReg;
   unsigned Lane;
   MachineInstrBuilder MIB(MI);
   const TargetRegisterInfo *TRI = &getRegisterInfo();
-  bool isKill;
   switch (MI->getOpcode()) {
     default:
       llvm_unreachable("cannot handle opcode!");
@@ -3400,82 +3825,294 @@ ARMBaseInstrInfo::setExecutionDomain(MachineInstr *MI, unsigned Domain) const {
 
       // Zap the predicate operands.
       assert(!isPredicated(MI) && "Cannot predicate a VORRd");
-      MI->RemoveOperand(3);
-      MI->RemoveOperand(2);
 
-      // Change to a VORRd which requires two identical use operands.
-      MI->setDesc(get(ARM::VORRd));
+      // Source instruction is %DDst = VMOVD %DSrc, 14, %noreg (; implicits)
+      DstReg = MI->getOperand(0).getReg();
+      SrcReg = MI->getOperand(1).getReg();
 
-      // Add the extra source operand and new predicates.
-      // This will go before any implicit ops.
-      AddDefaultPred(MachineInstrBuilder(MI).addOperand(MI->getOperand(1)));
+      for (unsigned i = MI->getDesc().getNumOperands(); i; --i)
+        MI->RemoveOperand(i-1);
+
+      // Change to a %DDst = VORRd %DSrc, %DSrc, 14, %noreg (; implicits)
+      MI->setDesc(get(ARM::VORRd));
+      AddDefaultPred(MIB.addReg(DstReg, RegState::Define)
+                        .addReg(SrcReg)
+                        .addReg(SrcReg));
       break;
     case ARM::VMOVRS:
       if (Domain != ExeNEON)
         break;
       assert(!isPredicated(MI) && "Cannot predicate a VGETLN");
 
+      // Source instruction is %RDst = VMOVRS %SSrc, 14, %noreg (; implicits)
       DstReg = MI->getOperand(0).getReg();
       SrcReg = MI->getOperand(1).getReg();
 
-      DReg = TRI->getMatchingSuperReg(SrcReg, ARM::ssub_0, &ARM::DPRRegClass);
-      Lane = 0;
-      if (DReg == ARM::NoRegister) {
-        DReg = TRI->getMatchingSuperReg(SrcReg, ARM::ssub_1, &ARM::DPRRegClass);
-        Lane = 1;
-        assert(DReg && "S-register with no D super-register?");
-      }
+      for (unsigned i = MI->getDesc().getNumOperands(); i; --i)
+        MI->RemoveOperand(i-1);
 
-      MI->RemoveOperand(3);
-      MI->RemoveOperand(2);
-      MI->RemoveOperand(1);
+      DReg = getCorrespondingDRegAndLane(TRI, SrcReg, Lane);
 
+      // Convert to %RDst = VGETLNi32 %DSrc, Lane, 14, %noreg (; imps)
+      // Note that DSrc has been widened and the other lane may be undef, which
+      // contaminates the entire register.
       MI->setDesc(get(ARM::VGETLNi32));
-      MIB.addReg(DReg);
-      MIB.addImm(Lane);
+      AddDefaultPred(MIB.addReg(DstReg, RegState::Define)
+                        .addReg(DReg, RegState::Undef)
+                        .addImm(Lane));
 
-      MIB->getOperand(1).setIsUndef();
+      // The old source should be an implicit use, otherwise we might think it
+      // was dead before here.
       MIB.addReg(SrcReg, RegState::Implicit);
-
-      AddDefaultPred(MIB);
       break;
-    case ARM::VMOVSR:
+    case ARM::VMOVSR: {
       if (Domain != ExeNEON)
         break;
       assert(!isPredicated(MI) && "Cannot predicate a VSETLN");
 
+      // Source instruction is %SDst = VMOVSR %RSrc, 14, %noreg (; implicits)
       DstReg = MI->getOperand(0).getReg();
       SrcReg = MI->getOperand(1).getReg();
-      DReg = TRI->getMatchingSuperReg(DstReg, ARM::ssub_0, &ARM::DPRRegClass);
-      Lane = 0;
-      if (DReg == ARM::NoRegister) {
-        DReg = TRI->getMatchingSuperReg(DstReg, ARM::ssub_1, &ARM::DPRRegClass);
-        Lane = 1;
-        assert(DReg && "S-register with no D super-register?");
-      }
-      isKill = MI->getOperand(0).isKill();
 
-      MI->RemoveOperand(3);
-      MI->RemoveOperand(2);
-      MI->RemoveOperand(1);
-      MI->RemoveOperand(0);
+      DReg = getCorrespondingDRegAndLane(TRI, DstReg, Lane);
+
+      unsigned ImplicitSReg;
+      if (!getImplicitSPRUseForDPRUse(TRI, MI, DReg, Lane, ImplicitSReg))
+        break;
+
+      for (unsigned i = MI->getDesc().getNumOperands(); i; --i)
+        MI->RemoveOperand(i-1);
 
+      // Convert to %DDst = VSETLNi32 %DDst, %RSrc, Lane, 14, %noreg (; imps)
+      // Again DDst may be undefined at the beginning of this instruction.
       MI->setDesc(get(ARM::VSETLNi32));
-      MIB.addReg(DReg);
-      MIB.addReg(DReg);
-      MIB.addReg(SrcReg);
-      MIB.addImm(Lane);
+      MIB.addReg(DReg, RegState::Define)
+         .addReg(DReg, getUndefRegState(!MI->readsRegister(DReg, TRI)))
+         .addReg(SrcReg)
+         .addImm(Lane);
+      AddDefaultPred(MIB);
+
+      // The narrower destination must be marked as set to keep previous chains
+      // in place.
+      MIB.addReg(DstReg, RegState::Define | RegState::Implicit);
+      if (ImplicitSReg != 0)
+        MIB.addReg(ImplicitSReg, RegState::Implicit);
+      break;
+    }
+    case ARM::VMOVS: {
+      if (Domain != ExeNEON)
+        break;
+
+      // Source instruction is %SDst = VMOVS %SSrc, 14, %noreg (; implicits)
+      DstReg = MI->getOperand(0).getReg();
+      SrcReg = MI->getOperand(1).getReg();
+
+      unsigned DstLane = 0, SrcLane = 0, DDst, DSrc;
+      DDst = getCorrespondingDRegAndLane(TRI, DstReg, DstLane);
+      DSrc = getCorrespondingDRegAndLane(TRI, SrcReg, SrcLane);
 
-      MIB->getOperand(1).setIsUndef();
+      unsigned ImplicitSReg;
+      if (!getImplicitSPRUseForDPRUse(TRI, MI, DSrc, SrcLane, ImplicitSReg))
+        break;
 
-      if (isKill)
-        MIB->addRegisterKilled(DstReg, TRI, true);
-      MIB->addRegisterDefined(DstReg, TRI);
+      for (unsigned i = MI->getDesc().getNumOperands(); i; --i)
+        MI->RemoveOperand(i-1);
+
+      if (DSrc == DDst) {
+        // Destination can be:
+        //     %DDst = VDUPLN32d %DDst, Lane, 14, %noreg (; implicits)
+        MI->setDesc(get(ARM::VDUPLN32d));
+        MIB.addReg(DDst, RegState::Define)
+           .addReg(DDst, getUndefRegState(!MI->readsRegister(DDst, TRI)))
+           .addImm(SrcLane);
+        AddDefaultPred(MIB);
+
+        // Neither the source or the destination are naturally represented any
+        // more, so add them in manually.
+        MIB.addReg(DstReg, RegState::Implicit | RegState::Define);
+        MIB.addReg(SrcReg, RegState::Implicit);
+        if (ImplicitSReg != 0)
+          MIB.addReg(ImplicitSReg, RegState::Implicit);
+        break;
+      }
 
+      // In general there's no single instruction that can perform an S <-> S
+      // move in NEON space, but a pair of VEXT instructions *can* do the
+      // job. It turns out that the VEXTs needed will only use DSrc once, with
+      // the position based purely on the combination of lane-0 and lane-1
+      // involved. For example
+      //     vmov s0, s2 -> vext.32 d0, d0, d1, #1  vext.32 d0, d0, d0, #1
+      //     vmov s1, s3 -> vext.32 d0, d1, d0, #1  vext.32 d0, d0, d0, #1
+      //     vmov s0, s3 -> vext.32 d0, d0, d0, #1  vext.32 d0, d1, d0, #1
+      //     vmov s1, s2 -> vext.32 d0, d0, d0, #1  vext.32 d0, d0, d1, #1
+      //
+      // Pattern of the MachineInstrs is:
+      //     %DDst = VEXTd32 %DSrc1, %DSrc2, Lane, 14, %noreg (;implicits)
+      MachineInstrBuilder NewMIB;
+      NewMIB = BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
+                       get(ARM::VEXTd32), DDst);
+
+      // On the first instruction, both DSrc and DDst may be <undef> if present.
+      // Specifically when the original instruction didn't have them as an
+      // <imp-use>.
+      unsigned CurReg = SrcLane == 1 && DstLane == 1 ? DSrc : DDst;
+      bool CurUndef = !MI->readsRegister(CurReg, TRI);
+      NewMIB.addReg(CurReg, getUndefRegState(CurUndef));
+
+      CurReg = SrcLane == 0 && DstLane == 0 ? DSrc : DDst;
+      CurUndef = !MI->readsRegister(CurReg, TRI);
+      NewMIB.addReg(CurReg, getUndefRegState(CurUndef));
+
+      NewMIB.addImm(1);
+      AddDefaultPred(NewMIB);
+
+      if (SrcLane == DstLane)
+        NewMIB.addReg(SrcReg, RegState::Implicit);
+
+      MI->setDesc(get(ARM::VEXTd32));
+      MIB.addReg(DDst, RegState::Define);
+
+      // On the second instruction, DDst has definitely been defined above, so
+      // it is not <undef>. DSrc, if present, can be <undef> as above.
+      CurReg = SrcLane == 1 && DstLane == 0 ? DSrc : DDst;
+      CurUndef = CurReg == DSrc && !MI->readsRegister(CurReg, TRI);
+      MIB.addReg(CurReg, getUndefRegState(CurUndef));
+
+      CurReg = SrcLane == 0 && DstLane == 1 ? DSrc : DDst;
+      CurUndef = CurReg == DSrc && !MI->readsRegister(CurReg, TRI);
+      MIB.addReg(CurReg, getUndefRegState(CurUndef));
+
+      MIB.addImm(1);
       AddDefaultPred(MIB);
+
+      if (SrcLane != DstLane)
+        MIB.addReg(SrcReg, RegState::Implicit);
+
+      // As before, the original destination is no longer represented, add it
+      // implicitly.
+      MIB.addReg(DstReg, RegState::Define | RegState::Implicit);
+      if (ImplicitSReg != 0)
+        MIB.addReg(ImplicitSReg, RegState::Implicit);
       break;
+    }
+  }
+
+}
+
+//===----------------------------------------------------------------------===//
+// Partial register updates
+//===----------------------------------------------------------------------===//
+//
+// Swift renames NEON registers with 64-bit granularity.  That means any
+// instruction writing an S-reg implicitly reads the containing D-reg.  The
+// problem is mostly avoided by translating f32 operations to v2f32 operations
+// on D-registers, but f32 loads are still a problem.
+//
+// These instructions can load an f32 into a NEON register:
+//
+// VLDRS - Only writes S, partial D update.
+// VLD1LNd32 - Writes all D-regs, explicit partial D update, 2 uops.
+// VLD1DUPd32 - Writes all D-regs, no partial reg update, 2 uops.
+//
+// FCONSTD can be used as a dependency-breaking instruction.
+
+
+unsigned ARMBaseInstrInfo::
+getPartialRegUpdateClearance(const MachineInstr *MI,
+                             unsigned OpNum,
+                             const TargetRegisterInfo *TRI) const {
+  // Only Swift has partial register update problems.
+  if (!SwiftPartialUpdateClearance || !Subtarget.isSwift())
+    return 0;
+
+  assert(TRI && "Need TRI instance");
+
+  const MachineOperand &MO = MI->getOperand(OpNum);
+  if (MO.readsReg())
+    return 0;
+  unsigned Reg = MO.getReg();
+  int UseOp = -1;
+
+  switch(MI->getOpcode()) {
+    // Normal instructions writing only an S-register.
+  case ARM::VLDRS:
+  case ARM::FCONSTS:
+  case ARM::VMOVSR:
+    // rdar://problem/8791586
+  case ARM::VMOVv8i8:
+  case ARM::VMOVv4i16:
+  case ARM::VMOVv2i32:
+  case ARM::VMOVv2f32:
+  case ARM::VMOVv1i64:
+    UseOp = MI->findRegisterUseOperandIdx(Reg, false, TRI);
+    break;
+
+    // Explicitly reads the dependency.
+  case ARM::VLD1LNd32:
+    UseOp = 1;
+    break;
+  default:
+    return 0;
+  }
+
+  // If this instruction actually reads a value from Reg, there is no unwanted
+  // dependency.
+  if (UseOp != -1 && MI->getOperand(UseOp).readsReg())
+    return 0;
+
+  // We must be able to clobber the whole D-reg.
+  if (TargetRegisterInfo::isVirtualRegister(Reg)) {
+    // Virtual register must be a foo:ssub_0<def,undef> operand.
+    if (!MO.getSubReg() || MI->readsVirtualRegister(Reg))
+      return 0;
+  } else if (ARM::SPRRegClass.contains(Reg)) {
+    // Physical register: MI must define the full D-reg.
+    unsigned DReg = TRI->getMatchingSuperReg(Reg, ARM::ssub_0,
+                                             &ARM::DPRRegClass);
+    if (!DReg || !MI->definesRegister(DReg, TRI))
+      return 0;
   }
 
+  // MI has an unwanted D-register dependency.
+  // Avoid defs in the previous N instructrions.
+  return SwiftPartialUpdateClearance;
+}
+
+// Break a partial register dependency after getPartialRegUpdateClearance
+// returned non-zero.
+void ARMBaseInstrInfo::
+breakPartialRegDependency(MachineBasicBlock::iterator MI,
+                          unsigned OpNum,
+                          const TargetRegisterInfo *TRI) const {
+  assert(MI && OpNum < MI->getDesc().getNumDefs() && "OpNum is not a def");
+  assert(TRI && "Need TRI instance");
+
+  const MachineOperand &MO = MI->getOperand(OpNum);
+  unsigned Reg = MO.getReg();
+  assert(TargetRegisterInfo::isPhysicalRegister(Reg) &&
+         "Can't break virtual register dependencies.");
+  unsigned DReg = Reg;
+
+  // If MI defines an S-reg, find the corresponding D super-register.
+  if (ARM::SPRRegClass.contains(Reg)) {
+    DReg = ARM::D0 + (Reg - ARM::S0) / 2;
+    assert(TRI->isSuperRegister(Reg, DReg) && "Register enums broken");
+  }
+
+  assert(ARM::DPRRegClass.contains(DReg) && "Can only break D-reg deps");
+  assert(MI->definesRegister(DReg, TRI) && "MI doesn't clobber full D-reg");
+
+  // FIXME: In some cases, VLDRS can be changed to a VLD1DUPd32 which defines
+  // the full D-register by loading the same value to both lanes.  The
+  // instruction is micro-coded with 2 uops, so don't do this until we can
+  // properly schedule micro-coded instuctions.  The dispatcher stalls cause
+  // too big regressions.
+
+  // Insert the dependency-breaking FCONSTD before MI.
+  // 96 is the encoding of 0.5, but the actual value doesn't matter here.
+  AddDefaultPred(BuildMI(*MI->getParent(), MI, MI->getDebugLoc(),
+                         get(ARM::FCONSTD), DReg).addImm(96));
+  MI->addRegisterKilled(DReg, TRI, true);
 }
 
 bool ARMBaseInstrInfo::hasNOP() const {
diff --git a/lib/Target/ARM/ARMBaseInstrInfo.h b/lib/Target/ARM/ARMBaseInstrInfo.h
index 92e5ee8..6f38e35 100644
--- a/lib/Target/ARM/ARMBaseInstrInfo.h
+++ b/lib/Target/ARM/ARMBaseInstrInfo.h
@@ -182,10 +182,13 @@ public:
   virtual bool isProfitableToDupForIfCvt(MachineBasicBlock &MBB,
                                          unsigned NumCycles,
                                          const BranchProbability
-                                           &Probability) const {
+                                         &Probability) const {
     return NumCycles == 1;
   }
 
+  virtual bool isProfitableToUnpredicate(MachineBasicBlock &TMBB,
+                                         MachineBasicBlock &FMBB) const;
+
   /// analyzeCompare - For a comparison instruction, return the source registers
   /// in SrcReg and SrcReg2 if having two register operands, and the value it
   /// compares against in CmpValue. Return true if the comparison instruction
@@ -226,15 +229,18 @@ public:
                         SDNode *DefNode, unsigned DefIdx,
                         SDNode *UseNode, unsigned UseIdx) const;
 
-  virtual unsigned getOutputLatency(const InstrItineraryData *ItinData,
-                                    const MachineInstr *DefMI, unsigned DefIdx,
-                                    const MachineInstr *DepMI) const;
-
   /// VFP/NEON execution domains.
   std::pair<uint16_t, uint16_t>
   getExecutionDomain(const MachineInstr *MI) const;
   void setExecutionDomain(MachineInstr *MI, unsigned Domain) const;
 
+  unsigned getPartialRegUpdateClearance(const MachineInstr*, unsigned,
+                                        const TargetRegisterInfo*) const;
+  void breakPartialRegDependency(MachineBasicBlock::iterator, unsigned,
+                                 const TargetRegisterInfo *TRI) const;
+  /// Get the number of addresses by LDM or VLDM or zero for unknown.
+  unsigned getNumLDMAddresses(const MachineInstr *MI) const;
+
 private:
   unsigned getInstBundleLength(const MachineInstr *MI) const;
 
diff --git a/lib/Target/ARM/ARMBaseRegisterInfo.cpp b/lib/Target/ARM/ARMBaseRegisterInfo.cpp
index 9deb96e..e5b300f 100644
--- a/lib/Target/ARM/ARMBaseRegisterInfo.cpp
+++ b/lib/Target/ARM/ARMBaseRegisterInfo.cpp
@@ -84,6 +84,11 @@ ARMBaseRegisterInfo::getCallPreservedMask(CallingConv::ID) const {
     ? CSR_iOS_RegMask : CSR_AAPCS_RegMask;
 }
 
+const uint32_t*
+ARMBaseRegisterInfo::getNoPreservedMask() const {
+  return CSR_NoRegs_RegMask;
+}
+
 BitVector ARMBaseRegisterInfo::
 getReservedRegs(const MachineFunction &MF) const {
   const TargetFrameLowering *TFI = MF.getTarget().getFrameLowering();
@@ -106,148 +111,12 @@ getReservedRegs(const MachineFunction &MF) const {
     for (unsigned i = 0; i != 16; ++i)
       Reserved.set(ARM::D16 + i);
   }
-  return Reserved;
-}
-
-bool ARMBaseRegisterInfo::isReservedReg(const MachineFunction &MF,
-                                        unsigned Reg) const {
-  const TargetFrameLowering *TFI = MF.getTarget().getFrameLowering();
-
-  switch (Reg) {
-  default: break;
-  case ARM::SP:
-  case ARM::PC:
-    return true;
-  case ARM::R6:
-    if (hasBasePointer(MF))
-      return true;
-    break;
-  case ARM::R7:
-  case ARM::R11:
-    if (FramePtr == Reg && TFI->hasFP(MF))
-      return true;
-    break;
-  case ARM::R9:
-    return STI.isR9Reserved();
-  }
-
-  return false;
-}
+  const TargetRegisterClass *RC  = &ARM::GPRPairRegClass;
+  for(TargetRegisterClass::iterator I = RC->begin(), E = RC->end(); I!=E; ++I)
+    for (MCSubRegIterator SI(*I, this); SI.isValid(); ++SI)
+      if (Reserved.test(*SI)) Reserved.set(*I);
 
-bool
-ARMBaseRegisterInfo::canCombineSubRegIndices(const TargetRegisterClass *RC,
-                                          SmallVectorImpl<unsigned> &SubIndices,
-                                          unsigned &NewSubIdx) const {
-
-  unsigned Size = RC->getSize() * 8;
-  if (Size < 6)
-    return 0;
-
-  NewSubIdx = 0;  // Whole register.
-  unsigned NumRegs = SubIndices.size();
-  if (NumRegs == 8) {
-    // 8 D registers -> 1 QQQQ register.
-    return (Size == 512 &&
-            SubIndices[0] == ARM::dsub_0 &&
-            SubIndices[1] == ARM::dsub_1 &&
-            SubIndices[2] == ARM::dsub_2 &&
-            SubIndices[3] == ARM::dsub_3 &&
-            SubIndices[4] == ARM::dsub_4 &&
-            SubIndices[5] == ARM::dsub_5 &&
-            SubIndices[6] == ARM::dsub_6 &&
-            SubIndices[7] == ARM::dsub_7);
-  } else if (NumRegs == 4) {
-    if (SubIndices[0] == ARM::qsub_0) {
-      // 4 Q registers -> 1 QQQQ register.
-      return (Size == 512 &&
-              SubIndices[1] == ARM::qsub_1 &&
-              SubIndices[2] == ARM::qsub_2 &&
-              SubIndices[3] == ARM::qsub_3);
-    } else if (SubIndices[0] == ARM::dsub_0) {
-      // 4 D registers -> 1 QQ register.
-      if (Size >= 256 &&
-          SubIndices[1] == ARM::dsub_1 &&
-          SubIndices[2] == ARM::dsub_2 &&
-          SubIndices[3] == ARM::dsub_3) {
-        if (Size == 512)
-          NewSubIdx = ARM::qqsub_0;
-        return true;
-      }
-    } else if (SubIndices[0] == ARM::dsub_4) {
-      // 4 D registers -> 1 QQ register (2nd).
-      if (Size == 512 &&
-          SubIndices[1] == ARM::dsub_5 &&
-          SubIndices[2] == ARM::dsub_6 &&
-          SubIndices[3] == ARM::dsub_7) {
-        NewSubIdx = ARM::qqsub_1;
-        return true;
-      }
-    } else if (SubIndices[0] == ARM::ssub_0) {
-      // 4 S registers -> 1 Q register.
-      if (Size >= 128 &&
-          SubIndices[1] == ARM::ssub_1 &&
-          SubIndices[2] == ARM::ssub_2 &&
-          SubIndices[3] == ARM::ssub_3) {
-        if (Size >= 256)
-          NewSubIdx = ARM::qsub_0;
-        return true;
-      }
-    }
-  } else if (NumRegs == 2) {
-    if (SubIndices[0] == ARM::qsub_0) {
-      // 2 Q registers -> 1 QQ register.
-      if (Size >= 256 && SubIndices[1] == ARM::qsub_1) {
-        if (Size == 512)
-          NewSubIdx = ARM::qqsub_0;
-        return true;
-      }
-    } else if (SubIndices[0] == ARM::qsub_2) {
-      // 2 Q registers -> 1 QQ register (2nd).
-      if (Size == 512 && SubIndices[1] == ARM::qsub_3) {
-        NewSubIdx = ARM::qqsub_1;
-        return true;
-      }
-    } else if (SubIndices[0] == ARM::dsub_0) {
-      // 2 D registers -> 1 Q register.
-      if (Size >= 128 && SubIndices[1] == ARM::dsub_1) {
-        if (Size >= 256)
-          NewSubIdx = ARM::qsub_0;
-        return true;
-      }
-    } else if (SubIndices[0] == ARM::dsub_2) {
-      // 2 D registers -> 1 Q register (2nd).
-      if (Size >= 256 && SubIndices[1] == ARM::dsub_3) {
-        NewSubIdx = ARM::qsub_1;
-        return true;
-      }
-    } else if (SubIndices[0] == ARM::dsub_4) {
-      // 2 D registers -> 1 Q register (3rd).
-      if (Size == 512 && SubIndices[1] == ARM::dsub_5) {
-        NewSubIdx = ARM::qsub_2;
-        return true;
-      }
-    } else if (SubIndices[0] == ARM::dsub_6) {
-      // 2 D registers -> 1 Q register (3rd).
-      if (Size == 512 && SubIndices[1] == ARM::dsub_7) {
-        NewSubIdx = ARM::qsub_3;
-        return true;
-      }
-    } else if (SubIndices[0] == ARM::ssub_0) {
-      // 2 S registers -> 1 D register.
-      if (SubIndices[1] == ARM::ssub_1) {
-        if (Size >= 128)
-          NewSubIdx = ARM::dsub_0;
-        return true;
-      }
-    } else if (SubIndices[0] == ARM::ssub_2) {
-      // 2 S registers -> 1 D register (2nd).
-      if (Size >= 128 && SubIndices[1] == ARM::ssub_3) {
-        NewSubIdx = ARM::dsub_1;
-        return true;
-      }
-    }
-  }
-  return false;
+  return Reserved;
 }
 
 const TargetRegisterClass*
@@ -263,6 +132,7 @@ ARMBaseRegisterInfo::getLargestLegalSuperClass(const TargetRegisterClass *RC)
     case ARM::QPRRegClassID:
     case ARM::QQPRRegClassID:
     case ARM::QQQQPRRegClassID:
+    case ARM::GPRPairRegClassID:
       return Super;
     }
     Super = *I++;
@@ -476,7 +346,7 @@ ARMBaseRegisterInfo::UpdateRegAllocHint(unsigned Reg, unsigned NewReg,
 bool
 ARMBaseRegisterInfo::avoidWriteAfterWrite(const TargetRegisterClass *RC) const {
   // CortexA9 has a Write-after-write hazard for NEON registers.
-  if (!STI.isCortexA9())
+  if (!STI.isLikeA9())
     return false;
 
   switch (RC->getID()) {
@@ -561,8 +431,9 @@ needsStackRealignment(const MachineFunction &MF) const {
   const MachineFrameInfo *MFI = MF.getFrameInfo();
   const Function *F = MF.getFunction();
   unsigned StackAlign = MF.getTarget().getFrameLowering()->getStackAlignment();
-  bool requiresRealignment = ((MFI->getMaxAlignment() > StackAlign) ||
-                               F->hasFnAttr(Attribute::StackAlignment));
+  bool requiresRealignment =
+    ((MFI->getMaxAlignment() > StackAlign) ||
+     F->getFnAttributes().hasAttribute(Attributes::StackAlignment));
 
   return requiresRealignment && canRealignStack(MF);
 }
@@ -595,6 +466,7 @@ unsigned ARMBaseRegisterInfo::getEHHandlerRegister() const {
 
 unsigned ARMBaseRegisterInfo::getRegisterPairEven(unsigned Reg,
                                               const MachineFunction &MF) const {
+  const MachineRegisterInfo &MRI = MF.getRegInfo();
   switch (Reg) {
   default: break;
   // Return 0 if either register of the pair is a special register.
@@ -603,10 +475,10 @@ unsigned ARMBaseRegisterInfo::getRegisterPairEven(unsigned Reg,
   case ARM::R3: return ARM::R2;
   case ARM::R5: return ARM::R4;
   case ARM::R7:
-    return (isReservedReg(MF, ARM::R7) || isReservedReg(MF, ARM::R6))
+    return (MRI.isReserved(ARM::R7) || MRI.isReserved(ARM::R6))
       ? 0 : ARM::R6;
-  case ARM::R9: return isReservedReg(MF, ARM::R9)  ? 0 :ARM::R8;
-  case ARM::R11: return isReservedReg(MF, ARM::R11) ? 0 : ARM::R10;
+  case ARM::R9: return MRI.isReserved(ARM::R9)  ? 0 :ARM::R8;
+  case ARM::R11: return MRI.isReserved(ARM::R11) ? 0 : ARM::R10;
 
   case ARM::S1: return ARM::S0;
   case ARM::S3: return ARM::S2;
@@ -648,6 +520,7 @@ unsigned ARMBaseRegisterInfo::getRegisterPairEven(unsigned Reg,
 
 unsigned ARMBaseRegisterInfo::getRegisterPairOdd(unsigned Reg,
                                              const MachineFunction &MF) const {
+  const MachineRegisterInfo &MRI = MF.getRegInfo();
   switch (Reg) {
   default: break;
   // Return 0 if either register of the pair is a special register.
@@ -656,10 +529,10 @@ unsigned ARMBaseRegisterInfo::getRegisterPairOdd(unsigned Reg,
   case ARM::R2: return ARM::R3;
   case ARM::R4: return ARM::R5;
   case ARM::R6:
-    return (isReservedReg(MF, ARM::R7) || isReservedReg(MF, ARM::R6))
+    return (MRI.isReserved(ARM::R7) || MRI.isReserved(ARM::R6))
       ? 0 : ARM::R7;
-  case ARM::R8: return isReservedReg(MF, ARM::R9)  ? 0 :ARM::R9;
-  case ARM::R10: return isReservedReg(MF, ARM::R11) ? 0 : ARM::R11;
+  case ARM::R8: return MRI.isReserved(ARM::R9)  ? 0 :ARM::R9;
+  case ARM::R10: return MRI.isReserved(ARM::R11) ? 0 : ARM::R11;
 
   case ARM::S0: return ARM::S1;
   case ARM::S2: return ARM::S3;
diff --git a/lib/Target/ARM/ARMBaseRegisterInfo.h b/lib/Target/ARM/ARMBaseRegisterInfo.h
index da29f7e..e2bdd04 100644
--- a/lib/Target/ARM/ARMBaseRegisterInfo.h
+++ b/lib/Target/ARM/ARMBaseRegisterInfo.h
@@ -96,19 +96,10 @@ public:
   /// Code Generation virtual methods...
   const uint16_t *getCalleeSavedRegs(const MachineFunction *MF = 0) const;
   const uint32_t *getCallPreservedMask(CallingConv::ID) const;
+  const uint32_t *getNoPreservedMask() const;
 
   BitVector getReservedRegs(const MachineFunction &MF) const;
 
-  /// canCombineSubRegIndices - Given a register class and a list of
-  /// subregister indices, return true if it's possible to combine the
-  /// subregister indices into one that corresponds to a larger
-  /// subregister. Return the new subregister index by reference. Note the
-  /// new index may be zero if the given subregisters can be combined to
-  /// form the whole register.
-  virtual bool canCombineSubRegIndices(const TargetRegisterClass *RC,
-                                       SmallVectorImpl<unsigned> &SubIndices,
-                                       unsigned &NewSubIdx) const;
-
   const TargetRegisterClass*
   getPointerRegClass(const MachineFunction &MF, unsigned Kind = 0) const;
   const TargetRegisterClass*
@@ -170,8 +161,6 @@ public:
                                  unsigned MIFlags = MachineInstr::NoFlags)const;
 
   /// Code Generation virtual methods...
-  virtual bool isReservedReg(const MachineFunction &MF, unsigned Reg) const;
-
   virtual bool requiresRegisterScavenging(const MachineFunction &MF) const;
 
   virtual bool trackLivenessAfterRegAlloc(const MachineFunction &MF) const;
diff --git a/lib/Target/ARM/ARMCallingConv.td b/lib/Target/ARM/ARMCallingConv.td
index bda1517..b378b96 100644
--- a/lib/Target/ARM/ARMCallingConv.td
+++ b/lib/Target/ARM/ARMCallingConv.td
@@ -190,6 +190,8 @@ def RetCC_ARM_AAPCS_VFP : CallingConv<[
 // Callee-saved register lists.
 //===----------------------------------------------------------------------===//
 
+def CSR_NoRegs : CalleeSavedRegs<(add)>;
+
 def CSR_AAPCS : CalleeSavedRegs<(add LR, R11, R10, R9, R8, R7, R6, R5, R4,
                                      (sequence "D%u", 15, 8))>;
 
diff --git a/lib/Target/ARM/ARMCodeEmitter.cpp b/lib/Target/ARM/ARMCodeEmitter.cpp
index e81b4cc..6adbf4f 100644
--- a/lib/Target/ARM/ARMCodeEmitter.cpp
+++ b/lib/Target/ARM/ARMCodeEmitter.cpp
@@ -47,7 +47,7 @@ namespace {
   class ARMCodeEmitter : public MachineFunctionPass {
     ARMJITInfo                *JTI;
     const ARMBaseInstrInfo    *II;
-    const TargetData          *TD;
+    const DataLayout          *TD;
     const ARMSubtarget        *Subtarget;
     TargetMachine             &TM;
     JITCodeEmitter            &MCE;
@@ -67,7 +67,7 @@ namespace {
     ARMCodeEmitter(TargetMachine &tm, JITCodeEmitter &mce)
       : MachineFunctionPass(ID), JTI(0),
         II((const ARMBaseInstrInfo *)tm.getInstrInfo()),
-        TD(tm.getTargetData()), TM(tm),
+        TD(tm.getDataLayout()), TM(tm),
         MCE(mce), MCPEs(0), MJTEs(0),
         IsPIC(TM.getRelocationModel() == Reloc::PIC_), IsThumb(false) {}
 
@@ -376,7 +376,7 @@ bool ARMCodeEmitter::runOnMachineFunction(MachineFunction &MF) {
          "JIT relocation model must be set to static or default!");
   JTI = ((ARMBaseTargetMachine &)MF.getTarget()).getJITInfo();
   II = (const ARMBaseInstrInfo *)MF.getTarget().getInstrInfo();
-  TD = MF.getTarget().getTargetData();
+  TD = MF.getTarget().getDataLayout();
   Subtarget = &TM.getSubtarget<ARMSubtarget>();
   MCPEs = &MF.getConstantPool()->getConstants();
   MJTEs = 0;
@@ -389,7 +389,7 @@ bool ARMCodeEmitter::runOnMachineFunction(MachineFunction &MF) {
 
   do {
     DEBUG(errs() << "JITTing function '"
-          << MF.getFunction()->getName() << "'\n");
+          << MF.getName() << "'\n");
     MCE.startFunction(MF);
     for (MachineFunction::iterator MBB = MF.begin(), E = MF.end();
          MBB != E; ++MBB) {
diff --git a/lib/Target/ARM/ARMConstantIslandPass.cpp b/lib/Target/ARM/ARMConstantIslandPass.cpp
index a953985..a57368f 100644
--- a/lib/Target/ARM/ARMConstantIslandPass.cpp
+++ b/lib/Target/ARM/ARMConstantIslandPass.cpp
@@ -22,7 +22,7 @@
 #include "llvm/CodeGen/MachineFunctionPass.h"
 #include "llvm/CodeGen/MachineJumpTableInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -528,7 +528,7 @@ ARMConstantIslands::doInitialPlacement(std::vector<MachineInstr*> &CPEMIs) {
   // identity mapping of CPI's to CPE's.
   const std::vector<MachineConstantPoolEntry> &CPs = MCP->getConstants();
 
-  const TargetData &TD = *MF->getTarget().getTargetData();
+  const DataLayout &TD = *MF->getTarget().getDataLayout();
   for (unsigned i = 0, e = CPs.size(); i != e; ++i) {
     unsigned Size = TD.getTypeAllocSize(CPs[i].getType());
     assert(Size >= 4 && "Too small constant pool entry");
@@ -1388,10 +1388,9 @@ bool ARMConstantIslands::handleConstantPoolUser(unsigned CPUserIndex) {
     // If the original WaterList entry was "new water" on this iteration,
     // propagate that to the new island.  This is just keeping NewWaterList
     // updated to match the WaterList, which will be updated below.
-    if (NewWaterList.count(WaterBB)) {
-      NewWaterList.erase(WaterBB);
+    if (NewWaterList.erase(WaterBB))
       NewWaterList.insert(NewIsland);
-    }
+
     // The new CPE goes before the following block (NewMBB).
     NewMBB = llvm::next(MachineFunction::iterator(WaterBB));
 
diff --git a/lib/Target/ARM/ARMConstantPoolValue.h b/lib/Target/ARM/ARMConstantPoolValue.h
index 6b98d44..ae531c4 100644
--- a/lib/Target/ARM/ARMConstantPoolValue.h
+++ b/lib/Target/ARM/ARMConstantPoolValue.h
@@ -102,8 +102,6 @@ public:
   virtual void print(raw_ostream &O) const;
   void print(raw_ostream *O) const { if (O) print(*O); }
   void dump() const;
-
-  static bool classof(const ARMConstantPoolValue *) { return true; }
 };
 
 inline raw_ostream &operator<<(raw_ostream &O, const ARMConstantPoolValue &V) {
@@ -158,7 +156,6 @@ public:
   static bool classof(const ARMConstantPoolValue *APV) {
     return APV->isGlobalValue() || APV->isBlockAddress() || APV->isLSDA();
   }
-  static bool classof(const ARMConstantPoolConstant *) { return true; }
 };
 
 /// ARMConstantPoolSymbol - ARM-specific constantpool values for external
@@ -192,7 +189,6 @@ public:
   static bool classof(const ARMConstantPoolValue *ACPV) {
     return ACPV->isExtSymbol();
   }
-  static bool classof(const ARMConstantPoolSymbol *) { return true; }
 };
 
 /// ARMConstantPoolMBB - ARM-specific constantpool value of a machine basic
@@ -225,7 +221,6 @@ public:
   static bool classof(const ARMConstantPoolValue *ACPV) {
     return ACPV->isMachineBasicBlock();
   }
-  static bool classof(const ARMConstantPoolMBB *) { return true; }
 };
 
 } // End llvm namespace
diff --git a/lib/Target/ARM/ARMELFWriterInfo.cpp b/lib/Target/ARM/ARMELFWriterInfo.cpp
deleted file mode 100644
index f671317..0000000
--- a/lib/Target/ARM/ARMELFWriterInfo.cpp
+++ /dev/null
@@ -1,78 +0,0 @@
-//===-- ARMELFWriterInfo.cpp - ELF Writer Info for the ARM backend --------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements ELF writer information for the ARM backend.
-//
-//===----------------------------------------------------------------------===//
-
-#include "ARMELFWriterInfo.h"
-#include "ARMRelocations.h"
-#include "llvm/Function.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Target/TargetData.h"
-#include "llvm/Target/TargetMachine.h"
-#include "llvm/Support/ELF.h"
-
-using namespace llvm;
-
-//===----------------------------------------------------------------------===//
-//  Implementation of the ARMELFWriterInfo class
-//===----------------------------------------------------------------------===//
-
-ARMELFWriterInfo::ARMELFWriterInfo(TargetMachine &TM)
-  : TargetELFWriterInfo(TM.getTargetData()->getPointerSizeInBits() == 64,
-                        TM.getTargetData()->isLittleEndian()) {
-}
-
-ARMELFWriterInfo::~ARMELFWriterInfo() {}
-
-unsigned ARMELFWriterInfo::getRelocationType(unsigned MachineRelTy) const {
-  switch (MachineRelTy) {
-  case ARM::reloc_arm_absolute:
-  case ARM::reloc_arm_relative:
-  case ARM::reloc_arm_cp_entry:
-  case ARM::reloc_arm_vfp_cp_entry:
-  case ARM::reloc_arm_machine_cp_entry:
-  case ARM::reloc_arm_jt_base:
-  case ARM::reloc_arm_pic_jt:
-    llvm_unreachable("unsupported ARM relocation type");
-
-  case ARM::reloc_arm_branch: return ELF::R_ARM_CALL;
-  case ARM::reloc_arm_movt:   return ELF::R_ARM_MOVT_ABS;
-  case ARM::reloc_arm_movw:   return ELF::R_ARM_MOVW_ABS_NC;
-  default:
-    llvm_unreachable("unknown ARM relocation type");
-  }
-}
-
-long int ARMELFWriterInfo::getDefaultAddendForRelTy(unsigned RelTy,
-                                                    long int Modifier) const {
-  llvm_unreachable("ARMELFWriterInfo::getDefaultAddendForRelTy() not "
-                   "implemented");
-}
-
-unsigned ARMELFWriterInfo::getRelocationTySize(unsigned RelTy) const {
-  llvm_unreachable("ARMELFWriterInfo::getRelocationTySize() not implemented");
-}
-
-bool ARMELFWriterInfo::isPCRelativeRel(unsigned RelTy) const {
-  llvm_unreachable("ARMELFWriterInfo::isPCRelativeRel() not implemented");
-}
-
-unsigned ARMELFWriterInfo::getAbsoluteLabelMachineRelTy() const {
-  llvm_unreachable("ARMELFWriterInfo::getAbsoluteLabelMachineRelTy() not "
-                   "implemented");
-}
-
-long int ARMELFWriterInfo::computeRelocation(unsigned SymOffset,
-                                             unsigned RelOffset,
-                                             unsigned RelTy) const {
-  llvm_unreachable("ARMELFWriterInfo::getAbsoluteLabelMachineRelTy() not "
-                   "implemented");
-}
diff --git a/lib/Target/ARM/ARMELFWriterInfo.h b/lib/Target/ARM/ARMELFWriterInfo.h
deleted file mode 100644
index 6a84f8a..0000000
--- a/lib/Target/ARM/ARMELFWriterInfo.h
+++ /dev/null
@@ -1,59 +0,0 @@
-//===-- ARMELFWriterInfo.h - ELF Writer Info for ARM ------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements ELF writer information for the ARM backend.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef ARM_ELF_WRITER_INFO_H
-#define ARM_ELF_WRITER_INFO_H
-
-#include "llvm/Target/TargetELFWriterInfo.h"
-
-namespace llvm {
-  class TargetMachine;
-
-  class ARMELFWriterInfo : public TargetELFWriterInfo {
-  public:
-    ARMELFWriterInfo(TargetMachine &TM);
-    virtual ~ARMELFWriterInfo();
-
-    /// getRelocationType - Returns the target specific ELF Relocation type.
-    /// 'MachineRelTy' contains the object code independent relocation type
-    virtual unsigned getRelocationType(unsigned MachineRelTy) const;
-
-    /// hasRelocationAddend - True if the target uses an addend in the
-    /// ELF relocation entry.
-    virtual bool hasRelocationAddend() const { return false; }
-
-    /// getDefaultAddendForRelTy - Gets the default addend value for a
-    /// relocation entry based on the target ELF relocation type.
-    virtual long int getDefaultAddendForRelTy(unsigned RelTy,
-                                              long int Modifier = 0) const;
-
-    /// getRelTySize - Returns the size of relocatable field in bits
-    virtual unsigned getRelocationTySize(unsigned RelTy) const;
-
-    /// isPCRelativeRel - True if the relocation type is pc relative
-    virtual bool isPCRelativeRel(unsigned RelTy) const;
-
-    /// getJumpTableRelocationTy - Returns the machine relocation type used
-    /// to reference a jumptable.
-    virtual unsigned getAbsoluteLabelMachineRelTy() const;
-
-    /// computeRelocation - Some relocatable fields could be relocated
-    /// directly, avoiding the relocation symbol emission, compute the
-    /// final relocation value for this symbol.
-    virtual long int computeRelocation(unsigned SymOffset, unsigned RelOffset,
-                                       unsigned RelTy) const;
-  };
-
-} // end llvm namespace
-
-#endif // ARM_ELF_WRITER_INFO_H
diff --git a/lib/Target/ARM/ARMExpandPseudoInsts.cpp b/lib/Target/ARM/ARMExpandPseudoInsts.cpp
index 15bb32e..8c45e0b 100644
--- a/lib/Target/ARM/ARMExpandPseudoInsts.cpp
+++ b/lib/Target/ARM/ARMExpandPseudoInsts.cpp
@@ -103,9 +103,9 @@ namespace {
     bool IsLoad;
     bool isUpdating;
     bool hasWritebackOperand;
-    NEONRegSpacing RegSpacing;
-    unsigned char NumRegs; // D registers loaded or stored
-    unsigned char RegElts; // elements per D register; used for lane ops
+    uint8_t RegSpacing; // One of type NEONRegSpacing
+    uint8_t NumRegs; // D registers loaded or stored
+    uint8_t RegElts; // elements per D register; used for lane ops
     // FIXME: Temporary flag to denote whether the real instruction takes
     // a single register (like the encoding) or all of the registers in
     // the list (like the asm syntax and the isel DAG). When all definitions
@@ -377,7 +377,7 @@ void ARMExpandPseudo::ExpandVLD(MachineBasicBlock::iterator &MBBI) {
 
   const NEONLdStTableEntry *TableEntry = LookupNEONLdSt(MI.getOpcode());
   assert(TableEntry && TableEntry->IsLoad && "NEONLdStTable lookup failed");
-  NEONRegSpacing RegSpc = TableEntry->RegSpacing;
+  NEONRegSpacing RegSpc = (NEONRegSpacing)TableEntry->RegSpacing;
   unsigned NumRegs = TableEntry->NumRegs;
 
   MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
@@ -442,7 +442,7 @@ void ARMExpandPseudo::ExpandVST(MachineBasicBlock::iterator &MBBI) {
 
   const NEONLdStTableEntry *TableEntry = LookupNEONLdSt(MI.getOpcode());
   assert(TableEntry && !TableEntry->IsLoad && "NEONLdStTable lookup failed");
-  NEONRegSpacing RegSpc = TableEntry->RegSpacing;
+  NEONRegSpacing RegSpc = (NEONRegSpacing)TableEntry->RegSpacing;
   unsigned NumRegs = TableEntry->NumRegs;
 
   MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(),
@@ -493,7 +493,7 @@ void ARMExpandPseudo::ExpandLaneOp(MachineBasicBlock::iterator &MBBI) {
 
   const NEONLdStTableEntry *TableEntry = LookupNEONLdSt(MI.getOpcode());
   assert(TableEntry && "NEONLdStTable lookup failed");
-  NEONRegSpacing RegSpc = TableEntry->RegSpacing;
+  NEONRegSpacing RegSpc = (NEONRegSpacing)TableEntry->RegSpacing;
   unsigned NumRegs = TableEntry->NumRegs;
   unsigned RegElts = TableEntry->RegElts;
 
@@ -777,9 +777,7 @@ bool ARMExpandPseudo::ExpandMI(MachineBasicBlock &MBB,
       MI.eraseFromParent();
       return true;
     }
-    case ARM::Int_eh_sjlj_dispatchsetup:
-    case ARM::Int_eh_sjlj_dispatchsetup_nofp:
-    case ARM::tInt_eh_sjlj_dispatchsetup: {
+    case ARM::Int_eh_sjlj_dispatchsetup: {
       MachineFunction &MF = *MI.getParent()->getParent();
       const ARMBaseInstrInfo *AII =
         static_cast<const ARMBaseInstrInfo*>(TII);
diff --git a/lib/Target/ARM/ARMFastISel.cpp b/lib/Target/ARM/ARMFastISel.cpp
index bf9d16e..6611862 100644
--- a/lib/Target/ARM/ARMFastISel.cpp
+++ b/lib/Target/ARM/ARMFastISel.cpp
@@ -40,7 +40,7 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/GetElementPtrTypeIterator.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetInstrInfo.h"
 #include "llvm/Target/TargetLowering.h"
 #include "llvm/Target/TargetMachine.h"
@@ -100,51 +100,53 @@ class ARMFastISel : public FastISel {
     }
 
     // Code from FastISel.cpp.
-    virtual unsigned FastEmitInst_(unsigned MachineInstOpcode,
-                                   const TargetRegisterClass *RC);
-    virtual unsigned FastEmitInst_r(unsigned MachineInstOpcode,
-                                    const TargetRegisterClass *RC,
-                                    unsigned Op0, bool Op0IsKill);
-    virtual unsigned FastEmitInst_rr(unsigned MachineInstOpcode,
-                                     const TargetRegisterClass *RC,
-                                     unsigned Op0, bool Op0IsKill,
-                                     unsigned Op1, bool Op1IsKill);
-    virtual unsigned FastEmitInst_rrr(unsigned MachineInstOpcode,
-                                      const TargetRegisterClass *RC,
-                                      unsigned Op0, bool Op0IsKill,
-                                      unsigned Op1, bool Op1IsKill,
-                                      unsigned Op2, bool Op2IsKill);
-    virtual unsigned FastEmitInst_ri(unsigned MachineInstOpcode,
-                                     const TargetRegisterClass *RC,
-                                     unsigned Op0, bool Op0IsKill,
-                                     uint64_t Imm);
-    virtual unsigned FastEmitInst_rf(unsigned MachineInstOpcode,
-                                     const TargetRegisterClass *RC,
-                                     unsigned Op0, bool Op0IsKill,
-                                     const ConstantFP *FPImm);
-    virtual unsigned FastEmitInst_rri(unsigned MachineInstOpcode,
-                                      const TargetRegisterClass *RC,
-                                      unsigned Op0, bool Op0IsKill,
-                                      unsigned Op1, bool Op1IsKill,
-                                      uint64_t Imm);
-    virtual unsigned FastEmitInst_i(unsigned MachineInstOpcode,
-                                    const TargetRegisterClass *RC,
-                                    uint64_t Imm);
-    virtual unsigned FastEmitInst_ii(unsigned MachineInstOpcode,
-                                     const TargetRegisterClass *RC,
-                                     uint64_t Imm1, uint64_t Imm2);
-
-    virtual unsigned FastEmitInst_extractsubreg(MVT RetVT,
-                                                unsigned Op0, bool Op0IsKill,
-                                                uint32_t Idx);
+  private:
+    unsigned FastEmitInst_(unsigned MachineInstOpcode,
+                           const TargetRegisterClass *RC);
+    unsigned FastEmitInst_r(unsigned MachineInstOpcode,
+                            const TargetRegisterClass *RC,
+                            unsigned Op0, bool Op0IsKill);
+    unsigned FastEmitInst_rr(unsigned MachineInstOpcode,
+                             const TargetRegisterClass *RC,
+                             unsigned Op0, bool Op0IsKill,
+                             unsigned Op1, bool Op1IsKill);
+    unsigned FastEmitInst_rrr(unsigned MachineInstOpcode,
+                              const TargetRegisterClass *RC,
+                              unsigned Op0, bool Op0IsKill,
+                              unsigned Op1, bool Op1IsKill,
+                              unsigned Op2, bool Op2IsKill);
+    unsigned FastEmitInst_ri(unsigned MachineInstOpcode,
+                             const TargetRegisterClass *RC,
+                             unsigned Op0, bool Op0IsKill,
+                             uint64_t Imm);
+    unsigned FastEmitInst_rf(unsigned MachineInstOpcode,
+                             const TargetRegisterClass *RC,
+                             unsigned Op0, bool Op0IsKill,
+                             const ConstantFP *FPImm);
+    unsigned FastEmitInst_rri(unsigned MachineInstOpcode,
+                              const TargetRegisterClass *RC,
+                              unsigned Op0, bool Op0IsKill,
+                              unsigned Op1, bool Op1IsKill,
+                              uint64_t Imm);
+    unsigned FastEmitInst_i(unsigned MachineInstOpcode,
+                            const TargetRegisterClass *RC,
+                            uint64_t Imm);
+    unsigned FastEmitInst_ii(unsigned MachineInstOpcode,
+                             const TargetRegisterClass *RC,
+                             uint64_t Imm1, uint64_t Imm2);
+
+    unsigned FastEmitInst_extractsubreg(MVT RetVT,
+                                        unsigned Op0, bool Op0IsKill,
+                                        uint32_t Idx);
 
     // Backend specific FastISel code.
+  private:
     virtual bool TargetSelectInstruction(const Instruction *I);
     virtual unsigned TargetMaterializeConstant(const Constant *C);
     virtual unsigned TargetMaterializeAlloca(const AllocaInst *AI);
     virtual bool TryToFoldLoad(MachineInstr *MI, unsigned OpNo,
                                const LoadInst *LI);
-
+  private:
   #include "ARMGenFastISel.inc"
 
     // Instruction selection routines.
@@ -192,6 +194,7 @@ class ARMFastISel : public FastISel {
     unsigned ARMMoveToFPReg(EVT VT, unsigned SrcReg);
     unsigned ARMMoveToIntReg(EVT VT, unsigned SrcReg);
     unsigned ARMSelectCallOp(bool UseReg);
+    unsigned ARMLowerPICELF(const GlobalValue *GV, unsigned Align, EVT VT);
 
     // Call handling routines.
   private:
@@ -615,11 +618,11 @@ unsigned ARMFastISel::ARMMaterializeGV(const GlobalValue *GV, EVT VT) {
   if (VT != MVT::i32) return 0;
 
   Reloc::Model RelocM = TM.getRelocationModel();
-
-  // TODO: Need more magic for ARM PIC.
-  if (!isThumb2 && (RelocM == Reloc::PIC_)) return 0;
-
-  unsigned DestReg = createResultReg(TLI.getRegClassFor(VT));
+  bool IsIndirect = Subtarget->GVIsIndirectSymbol(GV, RelocM);
+  const TargetRegisterClass *RC = isThumb2 ?
+    (const TargetRegisterClass*)&ARM::rGPRRegClass :
+    (const TargetRegisterClass*)&ARM::GPRRegClass;
+  unsigned DestReg = createResultReg(RC);
 
   // Use movw+movt when possible, it avoids constant pool entries.
   // Darwin targets don't support movt with Reloc::Static, see
@@ -649,6 +652,9 @@ unsigned ARMFastISel::ARMMaterializeGV(const GlobalValue *GV, EVT VT) {
       Align = TD.getTypeAllocSize(GV->getType());
     }
 
+    if (Subtarget->isTargetELF() && RelocM == Reloc::PIC_)
+      return ARMLowerPICELF(GV, Align, VT);
+
     // Grab index.
     unsigned PCAdj = (RelocM != Reloc::PIC_) ? 0 :
       (Subtarget->isThumb() ? 4 : 8);
@@ -666,17 +672,30 @@ unsigned ARMFastISel::ARMMaterializeGV(const GlobalValue *GV, EVT VT) {
         .addConstantPoolIndex(Idx);
       if (RelocM == Reloc::PIC_)
         MIB.addImm(Id);
+      AddOptionalDefs(MIB);
     } else {
       // The extra immediate is for addrmode2.
       MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(ARM::LDRcp),
                     DestReg)
         .addConstantPoolIndex(Idx)
         .addImm(0);
+      AddOptionalDefs(MIB);
+
+      if (RelocM == Reloc::PIC_) {
+        unsigned Opc = IsIndirect ? ARM::PICLDR : ARM::PICADD;
+        unsigned NewDestReg = createResultReg(TLI.getRegClassFor(VT));
+
+        MachineInstrBuilder MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt,
+                                          DL, TII.get(Opc), NewDestReg)
+                                  .addReg(DestReg)
+                                  .addImm(Id);
+        AddOptionalDefs(MIB);
+        return NewDestReg;
+      }
     }
-    AddOptionalDefs(MIB);
   }
 
-  if (Subtarget->GVIsIndirectSymbol(GV, RelocM)) {
+  if (IsIndirect) {
     MachineInstrBuilder MIB;
     unsigned NewDestReg = createResultReg(TLI.getRegClassFor(VT));
     if (isThumb2)
@@ -1009,6 +1028,9 @@ bool ARMFastISel::ARMEmitLoad(EVT VT, unsigned &ResultReg, Address &Addr,
       RC = &ARM::GPRRegClass;
       break;
     case MVT::i16:
+      if (Alignment && Alignment < 2 && !Subtarget->allowsUnalignedMem())
+        return false;
+
       if (isThumb2) {
         if (Addr.Offset < 0 && Addr.Offset > -256 && Subtarget->hasV6T2Ops())
           Opc = isZExt ? ARM::t2LDRHi8 : ARM::t2LDRSHi8;
@@ -1021,6 +1043,9 @@ bool ARMFastISel::ARMEmitLoad(EVT VT, unsigned &ResultReg, Address &Addr,
       RC = &ARM::GPRRegClass;
       break;
     case MVT::i32:
+      if (Alignment && Alignment < 4 && !Subtarget->allowsUnalignedMem())
+        return false;
+
       if (isThumb2) {
         if (Addr.Offset < 0 && Addr.Offset > -256 && Subtarget->hasV6T2Ops())
           Opc = ARM::t2LDRi8;
@@ -1127,6 +1152,9 @@ bool ARMFastISel::ARMEmitStore(EVT VT, unsigned SrcReg, Address &Addr,
       }
       break;
     case MVT::i16:
+      if (Alignment && Alignment < 2 && !Subtarget->allowsUnalignedMem())
+        return false;
+
       if (isThumb2) {
         if (Addr.Offset < 0 && Addr.Offset > -256 && Subtarget->hasV6T2Ops())
           StrOpc = ARM::t2STRHi8;
@@ -1138,6 +1166,9 @@ bool ARMFastISel::ARMEmitStore(EVT VT, unsigned SrcReg, Address &Addr,
       }
       break;
     case MVT::i32:
+      if (Alignment && Alignment < 4 && !Subtarget->allowsUnalignedMem())
+        return false;
+
       if (isThumb2) {
         if (Addr.Offset < 0 && Addr.Offset > -256 && Subtarget->hasV6T2Ops())
           StrOpc = ARM::t2STRi8;
@@ -1360,6 +1391,11 @@ bool ARMFastISel::SelectIndirectBr(const Instruction *I) {
   unsigned Opc = isThumb2 ? ARM::tBRIND : ARM::BX;
   AddOptionalDefs(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(Opc))
                   .addReg(AddrReg));
+
+  const IndirectBrInst *IB = cast<IndirectBrInst>(I);
+  for (unsigned i = 0, e = IB->getNumSuccessors(); i != e; ++i)
+    FuncInfo.MBB->addSuccessor(FuncInfo.MBBMap[IB->getSuccessor(i)]);
+
   return true;
 }
 
@@ -2210,25 +2246,17 @@ bool ARMFastISel::ARMEmitLibcall(const Instruction *I, RTLIB::Libcall Call) {
   unsigned CallOpc = ARMSelectCallOp(EnableARMLongCalls);
   MachineInstrBuilder MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt,
                                     DL, TII.get(CallOpc));
-  if (isThumb2) {
-    // Explicitly adding the predicate here.
+  // BL / BLX don't take a predicate, but tBL / tBLX do.
+  if (isThumb2)
     AddDefaultPred(MIB);
-    if (EnableARMLongCalls)
-      MIB.addReg(CalleeReg);
-    else
-      MIB.addExternalSymbol(TLI.getLibcallName(Call));
-  } else {
-    if (EnableARMLongCalls)
-      MIB.addReg(CalleeReg);
-    else
-      MIB.addExternalSymbol(TLI.getLibcallName(Call));
+  if (EnableARMLongCalls)
+    MIB.addReg(CalleeReg);
+  else
+    MIB.addExternalSymbol(TLI.getLibcallName(Call));
 
-    // Explicitly adding the predicate here.
-    AddDefaultPred(MIB);
-  }
   // Add implicit physical register uses to the call.
   for (unsigned i = 0, e = RegArgs.size(); i != e; ++i)
-    MIB.addReg(RegArgs[i]);
+    MIB.addReg(RegArgs[i], RegState::Implicit);
 
   // Add a register mask with the call-preserved registers.
   // Proper defs for return values will be added by setPhysRegsDeadExcept().
@@ -2300,16 +2328,16 @@ bool ARMFastISel::SelectCall(const Instruction *I,
 
     ISD::ArgFlagsTy Flags;
     unsigned AttrInd = i - CS.arg_begin() + 1;
-    if (CS.paramHasAttr(AttrInd, Attribute::SExt))
+    if (CS.paramHasAttr(AttrInd, Attributes::SExt))
       Flags.setSExt();
-    if (CS.paramHasAttr(AttrInd, Attribute::ZExt))
+    if (CS.paramHasAttr(AttrInd, Attributes::ZExt))
       Flags.setZExt();
 
     // FIXME: Only handle *easy* calls for now.
-    if (CS.paramHasAttr(AttrInd, Attribute::InReg) ||
-        CS.paramHasAttr(AttrInd, Attribute::StructRet) ||
-        CS.paramHasAttr(AttrInd, Attribute::Nest) ||
-        CS.paramHasAttr(AttrInd, Attribute::ByVal))
+    if (CS.paramHasAttr(AttrInd, Attributes::InReg) ||
+        CS.paramHasAttr(AttrInd, Attributes::StructRet) ||
+        CS.paramHasAttr(AttrInd, Attributes::Nest) ||
+        CS.paramHasAttr(AttrInd, Attributes::ByVal))
       return false;
 
     Type *ArgTy = (*i)->getType();
@@ -2356,30 +2384,20 @@ bool ARMFastISel::SelectCall(const Instruction *I,
   unsigned CallOpc = ARMSelectCallOp(UseReg);
   MachineInstrBuilder MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt,
                                     DL, TII.get(CallOpc));
-  if(isThumb2) {
-    // Explicitly adding the predicate here.
-    AddDefaultPred(MIB);
-    if (UseReg)
-      MIB.addReg(CalleeReg);
-    else if (!IntrMemName)
-      MIB.addGlobalAddress(GV, 0, 0);
-    else
-      MIB.addExternalSymbol(IntrMemName, 0);
-  } else {
-    if (UseReg)
-      MIB.addReg(CalleeReg);
-    else if (!IntrMemName)
-      MIB.addGlobalAddress(GV, 0, 0);
-    else
-      MIB.addExternalSymbol(IntrMemName, 0);
 
-    // Explicitly adding the predicate here.
+  // ARM calls don't take a predicate, but tBL / tBLX do.
+  if(isThumb2)
     AddDefaultPred(MIB);
-  }
+  if (UseReg)
+    MIB.addReg(CalleeReg);
+  else if (!IntrMemName)
+    MIB.addGlobalAddress(GV, 0, 0);
+  else
+    MIB.addExternalSymbol(IntrMemName, 0);
 
   // Add implicit physical register uses to the call.
   for (unsigned i = 0, e = RegArgs.size(); i != e; ++i)
-    MIB.addReg(RegArgs[i]);
+    MIB.addReg(RegArgs[i], RegState::Implicit);
 
   // Add a register mask with the call-preserved registers.
   // Proper defs for return values will be added by setPhysRegsDeadExcept().
@@ -2648,7 +2666,7 @@ bool ARMFastISel::SelectShift(const Instruction *I,
   unsigned Reg1 = getRegForValue(Src1Value);
   if (Reg1 == 0) return false;
 
-  unsigned Reg2;
+  unsigned Reg2 = 0;
   if (Opc == ARM::MOVsr) {
     Reg2 = getRegForValue(Src2Value);
     if (Reg2 == 0) return false;
@@ -2790,6 +2808,47 @@ bool ARMFastISel::TryToFoldLoad(MachineInstr *MI, unsigned OpNo,
   return true;
 }
 
+unsigned ARMFastISel::ARMLowerPICELF(const GlobalValue *GV,
+                                     unsigned Align, EVT VT) {
+  bool UseGOTOFF = GV->hasLocalLinkage() || GV->hasHiddenVisibility();
+  ARMConstantPoolConstant *CPV =
+    ARMConstantPoolConstant::Create(GV, UseGOTOFF ? ARMCP::GOTOFF : ARMCP::GOT);
+  unsigned Idx = MCP.getConstantPoolIndex(CPV, Align);
+
+  unsigned Opc;
+  unsigned DestReg1 = createResultReg(TLI.getRegClassFor(VT));
+  // Load value.
+  if (isThumb2) {
+    AddOptionalDefs(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL,
+                            TII.get(ARM::t2LDRpci), DestReg1)
+                    .addConstantPoolIndex(Idx));
+    Opc = UseGOTOFF ? ARM::t2ADDrr : ARM::t2LDRs;
+  } else {
+    // The extra immediate is for addrmode2.
+    AddOptionalDefs(BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt,
+                            DL, TII.get(ARM::LDRcp), DestReg1)
+                    .addConstantPoolIndex(Idx).addImm(0));
+    Opc = UseGOTOFF ? ARM::ADDrr : ARM::LDRrs;
+  }
+
+  unsigned GlobalBaseReg = AFI->getGlobalBaseReg();
+  if (GlobalBaseReg == 0) {
+    GlobalBaseReg = MRI.createVirtualRegister(TLI.getRegClassFor(VT));
+    AFI->setGlobalBaseReg(GlobalBaseReg);
+  }
+
+  unsigned DestReg2 = createResultReg(TLI.getRegClassFor(VT));
+  MachineInstrBuilder MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt,
+                                    DL, TII.get(Opc), DestReg2)
+                            .addReg(DestReg1)
+                            .addReg(GlobalBaseReg);
+  if (!UseGOTOFF)
+    MIB.addImm(0);
+  AddOptionalDefs(MIB);
+
+  return DestReg2;
+}
+
 namespace llvm {
   FastISel *ARM::createFastISel(FunctionLoweringInfo &funcInfo,
                                 const TargetLibraryInfo *libInfo) {
diff --git a/lib/Target/ARM/ARMFrameLowering.cpp b/lib/Target/ARM/ARMFrameLowering.cpp
index aee72d2..9392497 100644
--- a/lib/Target/ARM/ARMFrameLowering.cpp
+++ b/lib/Target/ARM/ARMFrameLowering.cpp
@@ -153,7 +153,8 @@ void ARMFrameLowering::emitPrologue(MachineFunction &MF) const {
   int FramePtrSpillFI = 0;
   int D8SpillFI = 0;
 
-  // All calls are tail calls in GHC calling conv, and functions have no prologue/epilogue.
+  // All calls are tail calls in GHC calling conv, and functions have no
+  // prologue/epilogue.
   if (MF.getFunction()->getCallingConv() == CallingConv::GHC)
     return;
 
@@ -360,7 +361,8 @@ void ARMFrameLowering::emitEpilogue(MachineFunction &MF,
   int NumBytes = (int)MFI->getStackSize();
   unsigned FramePtr = RegInfo->getFrameRegister(MF);
 
-  // All calls are tail calls in GHC calling conv, and functions have no prologue/epilogue.
+  // All calls are tail calls in GHC calling conv, and functions have no
+  // prologue/epilogue.
   if (MF.getFunction()->getCallingConv() == CallingConv::GHC)
     return;
 
@@ -1151,7 +1153,7 @@ static void checkNumAlignedDPRCS2Regs(MachineFunction &MF) {
     return;
 
   // Naked functions don't spill callee-saved registers.
-  if (MF.getFunction()->hasFnAttr(Attribute::Naked))
+  if (MF.getFunction()->getFnAttributes().hasAttribute(Attributes::Naked))
     return;
 
   // We are planning to use NEON instructions vst1 / vld1.
@@ -1176,7 +1178,7 @@ static void checkNumAlignedDPRCS2Regs(MachineFunction &MF) {
   MachineRegisterInfo &MRI = MF.getRegInfo();
   unsigned NumSpills = 0;
   for (; NumSpills < 8; ++NumSpills)
-    if (!MRI.isPhysRegOrOverlapUsed(ARM::D8 + NumSpills))
+    if (!MRI.isPhysRegUsed(ARM::D8 + NumSpills))
       break;
 
   // Don't do this for just one d-register. It's not worth it.
@@ -1209,6 +1211,7 @@ ARMFrameLowering::processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
     *static_cast<const ARMBaseInstrInfo*>(MF.getTarget().getInstrInfo());
   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
   MachineFrameInfo *MFI = MF.getFrameInfo();
+  MachineRegisterInfo &MRI = MF.getRegInfo();
   unsigned FramePtr = RegInfo->getFrameRegister(MF);
 
   // Spill R4 if Thumb2 function requires stack realignment - it will be used as
@@ -1218,12 +1221,12 @@ ARMFrameLowering::processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
   // FIXME: It will be better just to find spare register here.
   if (AFI->isThumb2Function() &&
       (MFI->hasVarSizedObjects() || RegInfo->needsStackRealignment(MF)))
-    MF.getRegInfo().setPhysRegUsed(ARM::R4);
+    MRI.setPhysRegUsed(ARM::R4);
 
   if (AFI->isThumb1OnlyFunction()) {
     // Spill LR if Thumb1 function uses variable length argument lists.
     if (AFI->getVarArgsRegSaveSize() > 0)
-      MF.getRegInfo().setPhysRegUsed(ARM::LR);
+      MRI.setPhysRegUsed(ARM::LR);
 
     // Spill R4 if Thumb1 epilogue has to restore SP from FP. We don't know
     // for sure what the stack size will be, but for this, an estimate is good
@@ -1233,7 +1236,7 @@ ARMFrameLowering::processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
     // FIXME: It will be better just to find spare register here.
     unsigned StackSize = estimateStackSize(MF);
     if (MFI->hasVarSizedObjects() || StackSize > 508)
-      MF.getRegInfo().setPhysRegUsed(ARM::R4);
+      MRI.setPhysRegUsed(ARM::R4);
   }
 
   // See if we can spill vector registers to aligned stack.
@@ -1241,7 +1244,7 @@ ARMFrameLowering::processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
 
   // Spill the BasePtr if it's used.
   if (RegInfo->hasBasePointer(MF))
-    MF.getRegInfo().setPhysRegUsed(RegInfo->getBaseRegister());
+    MRI.setPhysRegUsed(RegInfo->getBaseRegister());
 
   // Don't spill FP if the frame can be eliminated. This is determined
   // by scanning the callee-save registers to see if any is used.
@@ -1249,7 +1252,7 @@ ARMFrameLowering::processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
   for (unsigned i = 0; CSRegs[i]; ++i) {
     unsigned Reg = CSRegs[i];
     bool Spilled = false;
-    if (MF.getRegInfo().isPhysRegOrOverlapUsed(Reg)) {
+    if (MRI.isPhysRegUsed(Reg)) {
       Spilled = true;
       CanEliminateFrame = false;
     }
@@ -1338,7 +1341,7 @@ ARMFrameLowering::processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
     // If LR is not spilled, but at least one of R4, R5, R6, and R7 is spilled.
     // Spill LR as well so we can fold BX_RET to the registers restore (LDM).
     if (!LRSpilled && CS1Spilled) {
-      MF.getRegInfo().setPhysRegUsed(ARM::LR);
+      MRI.setPhysRegUsed(ARM::LR);
       NumGPRSpills++;
       UnspilledCS1GPRs.erase(std::find(UnspilledCS1GPRs.begin(),
                                     UnspilledCS1GPRs.end(), (unsigned)ARM::LR));
@@ -1347,7 +1350,7 @@ ARMFrameLowering::processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
     }
 
     if (hasFP(MF)) {
-      MF.getRegInfo().setPhysRegUsed(FramePtr);
+      MRI.setPhysRegUsed(FramePtr);
       NumGPRSpills++;
     }
 
@@ -1362,16 +1365,16 @@ ARMFrameLowering::processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
           // Don't spill high register if the function is thumb1
           if (!AFI->isThumb1OnlyFunction() ||
               isARMLowRegister(Reg) || Reg == ARM::LR) {
-            MF.getRegInfo().setPhysRegUsed(Reg);
-            if (!RegInfo->isReservedReg(MF, Reg))
+            MRI.setPhysRegUsed(Reg);
+            if (!MRI.isReserved(Reg))
               ExtraCSSpill = true;
             break;
           }
         }
       } else if (!UnspilledCS2GPRs.empty() && !AFI->isThumb1OnlyFunction()) {
         unsigned Reg = UnspilledCS2GPRs.front();
-        MF.getRegInfo().setPhysRegUsed(Reg);
-        if (!RegInfo->isReservedReg(MF, Reg))
+        MRI.setPhysRegUsed(Reg);
+        if (!MRI.isReserved(Reg))
           ExtraCSSpill = true;
       }
     }
@@ -1389,7 +1392,7 @@ ARMFrameLowering::processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
       while (NumExtras && !UnspilledCS1GPRs.empty()) {
         unsigned Reg = UnspilledCS1GPRs.back();
         UnspilledCS1GPRs.pop_back();
-        if (!RegInfo->isReservedReg(MF, Reg) &&
+        if (!MRI.isReserved(Reg) &&
             (!AFI->isThumb1OnlyFunction() || isARMLowRegister(Reg) ||
              Reg == ARM::LR)) {
           Extras.push_back(Reg);
@@ -1401,7 +1404,7 @@ ARMFrameLowering::processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
         while (NumExtras && !UnspilledCS2GPRs.empty()) {
           unsigned Reg = UnspilledCS2GPRs.back();
           UnspilledCS2GPRs.pop_back();
-          if (!RegInfo->isReservedReg(MF, Reg)) {
+          if (!MRI.isReserved(Reg)) {
             Extras.push_back(Reg);
             NumExtras--;
           }
@@ -1409,7 +1412,7 @@ ARMFrameLowering::processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
       }
       if (Extras.size() && NumExtras == 0) {
         for (unsigned i = 0, e = Extras.size(); i != e; ++i) {
-          MF.getRegInfo().setPhysRegUsed(Extras[i]);
+          MRI.setPhysRegUsed(Extras[i]);
         }
       } else if (!AFI->isThumb1OnlyFunction()) {
         // note: Thumb1 functions spill to R12, not the stack.  Reserve a slot
@@ -1423,7 +1426,7 @@ ARMFrameLowering::processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
   }
 
   if (ForceLRSpill) {
-    MF.getRegInfo().setPhysRegUsed(ARM::LR);
+    MRI.setPhysRegUsed(ARM::LR);
     AFI->setLRIsSpilledForFarJump(true);
   }
 }
diff --git a/lib/Target/ARM/ARMHazardRecognizer.cpp b/lib/Target/ARM/ARMHazardRecognizer.cpp
index a5fd15b..1240169 100644
--- a/lib/Target/ARM/ARMHazardRecognizer.cpp
+++ b/lib/Target/ARM/ARMHazardRecognizer.cpp
@@ -47,7 +47,7 @@ ARMHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
       // Skip over one non-VFP / NEON instruction.
       if (!LastMI->isBarrier() &&
           // On A9, AGU and NEON/FPU are muxed.
-          !(STI.isCortexA9() && (LastMI->mayLoad() || LastMI->mayStore())) &&
+          !(STI.isLikeA9() && (LastMI->mayLoad() || LastMI->mayStore())) &&
           (LastMCID.TSFlags & ARMII::DomainMask) == ARMII::DomainGeneral) {
         MachineBasicBlock::iterator I = LastMI;
         if (I != LastMI->getParent()->begin()) {
diff --git a/lib/Target/ARM/ARMISelDAGToDAG.cpp b/lib/Target/ARM/ARMISelDAGToDAG.cpp
index a3a6c31..efd6d2b 100644
--- a/lib/Target/ARM/ARMISelDAGToDAG.cpp
+++ b/lib/Target/ARM/ARMISelDAGToDAG.cpp
@@ -239,7 +239,6 @@ private:
 
   /// SelectCMOVOp - Select CMOV instructions for ARM.
   SDNode *SelectCMOVOp(SDNode *N);
-  SDNode *SelectConditionalOp(SDNode *N);
   SDNode *SelectT2CMOVShiftOp(SDNode *N, SDValue FalseVal, SDValue TrueVal,
                               ARMCC::CondCodes CCVal, SDValue CCR,
                               SDValue InFlag);
@@ -306,7 +305,7 @@ static bool isOpcWithIntImmediate(SDNode *N, unsigned Opc, unsigned& Imm) {
 }
 
 /// \brief Check whether a particular node is a constant value representable as
-/// (N * Scale) where (N in [\arg RangeMin, \arg RangeMax).
+/// (N * Scale) where (N in [\p RangeMin, \p RangeMax).
 ///
 /// \param ScaledConstant [out] - On success, the pre-scaled constant value.
 static bool isScaledConstantInRange(SDValue Node, int Scale,
@@ -337,7 +336,8 @@ bool ARMDAGToDAGISel::hasNoVMLxHazardUse(SDNode *N) const {
   if (!CheckVMLxHazard)
     return true;
 
-  if (!Subtarget->isCortexA8() && !Subtarget->isCortexA9())
+  if (!Subtarget->isCortexA8() && !Subtarget->isLikeA9() &&
+      !Subtarget->isSwift())
     return true;
 
   if (!N->hasOneUse())
@@ -375,12 +375,13 @@ bool ARMDAGToDAGISel::hasNoVMLxHazardUse(SDNode *N) const {
 bool ARMDAGToDAGISel::isShifterOpProfitable(const SDValue &Shift,
                                             ARM_AM::ShiftOpc ShOpcVal,
                                             unsigned ShAmt) {
-  if (!Subtarget->isCortexA9())
+  if (!Subtarget->isLikeA9() && !Subtarget->isSwift())
     return true;
   if (Shift.hasOneUse())
     return true;
   // R << 2 is free.
-  return ShOpcVal == ARM_AM::lsl && ShAmt == 2;
+  return ShOpcVal == ARM_AM::lsl &&
+         (ShAmt == 2 || (Subtarget->isSwift() && ShAmt == 1));
 }
 
 bool ARMDAGToDAGISel::SelectImmShifterOperand(SDValue N,
@@ -487,7 +488,7 @@ bool ARMDAGToDAGISel::SelectAddrModeImm12(SDValue N,
 bool ARMDAGToDAGISel::SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset,
                                       SDValue &Opc) {
   if (N.getOpcode() == ISD::MUL &&
-      (!Subtarget->isCortexA9() || N.hasOneUse())) {
+      ((!Subtarget->isLikeA9() && !Subtarget->isSwift()) || N.hasOneUse())) {
     if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
       // X * [3,5,9] -> X + X * [2,4,8] etc.
       int RHSC = (int)RHS->getZExtValue();
@@ -551,7 +552,8 @@ bool ARMDAGToDAGISel::SelectLdStSOReg(SDValue N, SDValue &Base, SDValue &Offset,
 
   // Try matching (R shl C) + (R).
   if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift &&
-      !(Subtarget->isCortexA9() || N.getOperand(0).hasOneUse())) {
+      !(Subtarget->isLikeA9() || Subtarget->isSwift() ||
+        N.getOperand(0).hasOneUse())) {
     ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0).getOpcode());
     if (ShOpcVal != ARM_AM::no_shift) {
       // Check to see if the RHS of the shift is a constant, if not, we can't
@@ -585,7 +587,7 @@ AddrMode2Type ARMDAGToDAGISel::SelectAddrMode2Worker(SDValue N,
                                                      SDValue &Offset,
                                                      SDValue &Opc) {
   if (N.getOpcode() == ISD::MUL &&
-      (!Subtarget->isCortexA9() || N.hasOneUse())) {
+      (!(Subtarget->isLikeA9() || Subtarget->isSwift()) || N.hasOneUse())) {
     if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(N.getOperand(1))) {
       // X * [3,5,9] -> X + X * [2,4,8] etc.
       int RHSC = (int)RHS->getZExtValue();
@@ -651,7 +653,7 @@ AddrMode2Type ARMDAGToDAGISel::SelectAddrMode2Worker(SDValue N,
     }
   }
 
-  if (Subtarget->isCortexA9() && !N.hasOneUse()) {
+  if ((Subtarget->isLikeA9() || Subtarget->isSwift()) && !N.hasOneUse()) {
     // Compute R +/- (R << N) and reuse it.
     Base = N;
     Offset = CurDAG->getRegister(0, MVT::i32);
@@ -689,7 +691,8 @@ AddrMode2Type ARMDAGToDAGISel::SelectAddrMode2Worker(SDValue N,
 
   // Try matching (R shl C) + (R).
   if (N.getOpcode() != ISD::SUB && ShOpcVal == ARM_AM::no_shift &&
-      !(Subtarget->isCortexA9() || N.getOperand(0).hasOneUse())) {
+      !(Subtarget->isLikeA9() || Subtarget->isSwift() ||
+        N.getOperand(0).hasOneUse())) {
     ShOpcVal = ARM_AM::getShiftOpcForNode(N.getOperand(0).getOpcode());
     if (ShOpcVal != ARM_AM::no_shift) {
       // Check to see if the RHS of the shift is a constant, if not, we can't
@@ -2363,121 +2366,6 @@ SDNode *ARMDAGToDAGISel::SelectCMOVOp(SDNode *N) {
   return CurDAG->SelectNodeTo(N, Opc, VT, Ops, 5);
 }
 
-SDNode *ARMDAGToDAGISel::SelectConditionalOp(SDNode *N) {
-  SDValue FalseVal = N->getOperand(0);
-  SDValue TrueVal  = N->getOperand(1);
-  ARMCC::CondCodes CCVal =
-    (ARMCC::CondCodes)cast<ConstantSDNode>(N->getOperand(2))->getZExtValue();
-  SDValue CCR = N->getOperand(3);
-  assert(CCR.getOpcode() == ISD::Register);
-  SDValue InFlag = N->getOperand(4);
-  SDValue CC = CurDAG->getTargetConstant(CCVal, MVT::i32);
-  SDValue Reg0 = CurDAG->getRegister(0, MVT::i32);
-
-  if (Subtarget->isThumb()) {
-    SDValue CPTmp0;
-    SDValue CPTmp1;
-    if (SelectT2ShifterOperandReg(TrueVal, CPTmp0, CPTmp1)) {
-      unsigned Opc;
-      switch (N->getOpcode()) {
-      default: llvm_unreachable("Unexpected node");
-      case ARMISD::CAND: Opc = ARM::t2ANDCCrs; break;
-      case ARMISD::COR:  Opc = ARM::t2ORRCCrs; break;
-      case ARMISD::CXOR: Opc = ARM::t2EORCCrs; break;
-      }
-      SDValue Ops[] = {
-        FalseVal, FalseVal, CPTmp0, CPTmp1, CC, CCR, Reg0, InFlag
-      };
-      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 8);
-    }
-
-    ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal);
-    if (T) {
-      unsigned TrueImm = T->getZExtValue();
-      if (is_t2_so_imm(TrueImm)) {
-        unsigned Opc;
-        switch (N->getOpcode()) {
-        default: llvm_unreachable("Unexpected node");
-        case ARMISD::CAND: Opc = ARM::t2ANDCCri; break;
-        case ARMISD::COR:  Opc = ARM::t2ORRCCri; break;
-        case ARMISD::CXOR: Opc = ARM::t2EORCCri; break;
-        }
-        SDValue True = CurDAG->getTargetConstant(TrueImm, MVT::i32);
-        SDValue Ops[] = { FalseVal, FalseVal, True, CC, CCR, Reg0, InFlag };
-        return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 7);
-      }
-    }
-
-    unsigned Opc;
-    switch (N->getOpcode()) {
-    default: llvm_unreachable("Unexpected node");
-    case ARMISD::CAND: Opc = ARM::t2ANDCCrr; break;
-    case ARMISD::COR:  Opc = ARM::t2ORRCCrr; break;
-    case ARMISD::CXOR: Opc = ARM::t2EORCCrr; break;
-    }
-    SDValue Ops[] = { FalseVal, FalseVal, TrueVal, CC, CCR, Reg0, InFlag };
-    return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 7);
-  }
-
-  SDValue CPTmp0;
-  SDValue CPTmp1;
-  SDValue CPTmp2;
-  if (SelectImmShifterOperand(TrueVal, CPTmp0, CPTmp2)) {
-    unsigned Opc;
-    switch (N->getOpcode()) {
-    default: llvm_unreachable("Unexpected node");
-    case ARMISD::CAND: Opc = ARM::ANDCCrsi; break;
-    case ARMISD::COR:  Opc = ARM::ORRCCrsi; break;
-    case ARMISD::CXOR: Opc = ARM::EORCCrsi; break;
-    }
-    SDValue Ops[] = {
-      FalseVal, FalseVal, CPTmp0, CPTmp2, CC, CCR, Reg0, InFlag
-    };
-    return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 8);
-  }
-
-  if (SelectRegShifterOperand(TrueVal, CPTmp0, CPTmp1, CPTmp2)) {
-    unsigned Opc;
-    switch (N->getOpcode()) {
-    default: llvm_unreachable("Unexpected node");
-    case ARMISD::CAND: Opc = ARM::ANDCCrsr; break;
-    case ARMISD::COR:  Opc = ARM::ORRCCrsr; break;
-    case ARMISD::CXOR: Opc = ARM::EORCCrsr; break;
-    }
-    SDValue Ops[] = {
-      FalseVal, FalseVal, CPTmp0, CPTmp1, CPTmp2, CC, CCR, Reg0, InFlag
-    };
-    return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 9);
-  }
-
-  ConstantSDNode *T = dyn_cast<ConstantSDNode>(TrueVal);
-  if (T) {
-    unsigned TrueImm = T->getZExtValue();
-    if (is_so_imm(TrueImm)) {
-      unsigned Opc;
-      switch (N->getOpcode()) {
-      default: llvm_unreachable("Unexpected node");
-      case ARMISD::CAND: Opc = ARM::ANDCCri; break;
-      case ARMISD::COR:  Opc = ARM::ORRCCri; break;
-      case ARMISD::CXOR: Opc = ARM::EORCCri; break;
-      }
-      SDValue True = CurDAG->getTargetConstant(TrueImm, MVT::i32);
-      SDValue Ops[] = { FalseVal, FalseVal, True, CC, CCR, Reg0, InFlag };
-      return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 7);
-    }
-  }
-
-  unsigned Opc;
-  switch (N->getOpcode()) {
-  default: llvm_unreachable("Unexpected node");
-  case ARMISD::CAND: Opc = ARM::ANDCCrr; break;
-  case ARMISD::COR:  Opc = ARM::ORRCCrr; break;
-  case ARMISD::CXOR: Opc = ARM::EORCCrr; break;
-  }
-  SDValue Ops[] = { FalseVal, FalseVal, TrueVal, CC, CCR, Reg0, InFlag };
-  return CurDAG->SelectNodeTo(N, Opc, MVT::i32, Ops, 7);
-}
-
 /// Target-specific DAG combining for ISD::XOR.
 /// Target-independent combining lowers SELECT_CC nodes of the form
 /// select_cc setg[ge] X,  0,  X, -X
@@ -2753,6 +2641,38 @@ SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
                                     dl, MVT::i32, MVT::i32, Ops, 5);
     }
   }
+  case ARMISD::UMLAL:{
+    if (Subtarget->isThumb()) {
+      SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
+                        N->getOperand(3), getAL(CurDAG),
+                        CurDAG->getRegister(0, MVT::i32)};
+      return CurDAG->getMachineNode(ARM::t2UMLAL, dl, MVT::i32, MVT::i32, Ops, 6);
+    }else{
+      SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
+                        N->getOperand(3), getAL(CurDAG),
+                        CurDAG->getRegister(0, MVT::i32),
+                        CurDAG->getRegister(0, MVT::i32) };
+      return CurDAG->getMachineNode(Subtarget->hasV6Ops() ?
+                                      ARM::UMLAL : ARM::UMLALv5,
+                                      dl, MVT::i32, MVT::i32, Ops, 7);
+    }
+  }
+  case ARMISD::SMLAL:{
+    if (Subtarget->isThumb()) {
+      SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
+                        N->getOperand(3), getAL(CurDAG),
+                        CurDAG->getRegister(0, MVT::i32)};
+      return CurDAG->getMachineNode(ARM::t2SMLAL, dl, MVT::i32, MVT::i32, Ops, 6);
+    }else{
+      SDValue Ops[] = { N->getOperand(0), N->getOperand(1), N->getOperand(2),
+                        N->getOperand(3), getAL(CurDAG),
+                        CurDAG->getRegister(0, MVT::i32),
+                        CurDAG->getRegister(0, MVT::i32) };
+      return CurDAG->getMachineNode(Subtarget->hasV6Ops() ?
+                                      ARM::SMLAL : ARM::SMLALv5,
+                                      dl, MVT::i32, MVT::i32, Ops, 7);
+    }
+  }
   case ISD::LOAD: {
     SDNode *ResNode = 0;
     if (Subtarget->isThumb() && Subtarget->hasThumb2())
@@ -2805,10 +2725,6 @@ SDNode *ARMDAGToDAGISel::Select(SDNode *N) {
   }
   case ARMISD::CMOV:
     return SelectCMOVOp(N);
-  case ARMISD::CAND:
-  case ARMISD::COR:
-  case ARMISD::CXOR:
-    return SelectConditionalOp(N);
   case ARMISD::VZIP: {
     unsigned Opc = 0;
     EVT VT = N->getValueType(0);
diff --git a/lib/Target/ARM/ARMISelLowering.cpp b/lib/Target/ARM/ARMISelLowering.cpp
index 190ca07..ff99b04 100644
--- a/lib/Target/ARM/ARMISelLowering.cpp
+++ b/lib/Target/ARM/ARMISelLowering.cpp
@@ -122,6 +122,7 @@ void ARMTargetLowering::addTypeForNEON(MVT VT, MVT PromotedLdStVT,
   setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal);
   setOperationAction(ISD::SELECT,            VT, Expand);
   setOperationAction(ISD::SELECT_CC,         VT, Expand);
+  setOperationAction(ISD::VSELECT,           VT, Expand);
   setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand);
   if (VT.isInteger()) {
     setOperationAction(ISD::SHL, VT, Custom);
@@ -514,6 +515,7 @@ ARMTargetLowering::ARMTargetLowering(TargetMachine &TM)
     setOperationAction(ISD::FLOG10, MVT::v4f32, Expand);
     setOperationAction(ISD::FEXP, MVT::v4f32, Expand);
     setOperationAction(ISD::FEXP2, MVT::v4f32, Expand);
+    setOperationAction(ISD::FFLOOR, MVT::v4f32, Expand);
 
     // Neon does not support some operations on v1i64 and v2i64 types.
     setOperationAction(ISD::MUL, MVT::v1i64, Expand);
@@ -566,6 +568,11 @@ ARMTargetLowering::ARMTargetLowering(TargetMachine &TM)
     }
   }
 
+  // ARM and Thumb2 support UMLAL/SMLAL.
+  if (!Subtarget->isThumb1Only())
+    setTargetDAGCombine(ISD::ADDC);
+
+
   computeRegisterProperties();
 
   // ARM does not have f32 extending load.
@@ -629,9 +636,9 @@ ARMTargetLowering::ARMTargetLowering(TargetMachine &TM)
   if (!Subtarget->hasV6Ops())
     setOperationAction(ISD::BSWAP, MVT::i32, Expand);
 
-  // These are expanded into libcalls.
-  if (!Subtarget->hasDivide() || !Subtarget->isThumb2()) {
-    // v7M has a hardware divider
+  if (!(Subtarget->hasDivide() && Subtarget->isThumb2()) &&
+      !(Subtarget->hasDivideInARMMode() && !Subtarget->isThumb())) {
+    // These are expanded into libcalls if the cpu doesn't have HW divider.
     setOperationAction(ISD::SDIV,  MVT::i32, Expand);
     setOperationAction(ISD::UDIV,  MVT::i32, Expand);
   }
@@ -791,12 +798,9 @@ ARMTargetLowering::ARMTargetLowering(TargetMachine &TM)
   setTargetDAGCombine(ISD::ADD);
   setTargetDAGCombine(ISD::SUB);
   setTargetDAGCombine(ISD::MUL);
-
-  if (Subtarget->hasV6T2Ops() || Subtarget->hasNEON()) {
-    setTargetDAGCombine(ISD::AND);
-    setTargetDAGCombine(ISD::OR);
-    setTargetDAGCombine(ISD::XOR);
-  }
+  setTargetDAGCombine(ISD::AND);
+  setTargetDAGCombine(ISD::OR);
+  setTargetDAGCombine(ISD::XOR);
 
   if (Subtarget->hasV6Ops())
     setTargetDAGCombine(ISD::SRL);
@@ -821,7 +825,7 @@ ARMTargetLowering::ARMTargetLowering(TargetMachine &TM)
   benefitFromCodePlacementOpt = true;
 
   // Prefer likely predicted branches to selects on out-of-order cores.
-  predictableSelectIsExpensive = Subtarget->isCortexA9();
+  predictableSelectIsExpensive = Subtarget->isLikeA9();
 
   setMinFunctionAlignment(Subtarget->isThumb() ? 1 : 2);
 }
@@ -898,9 +902,6 @@ const char *ARMTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case ARMISD::FMSTAT:        return "ARMISD::FMSTAT";
 
   case ARMISD::CMOV:          return "ARMISD::CMOV";
-  case ARMISD::CAND:          return "ARMISD::CAND";
-  case ARMISD::COR:           return "ARMISD::COR";
-  case ARMISD::CXOR:          return "ARMISD::CXOR";
 
   case ARMISD::RBIT:          return "ARMISD::RBIT";
 
@@ -984,6 +985,8 @@ const char *ARMTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case ARMISD::VTBL2:         return "ARMISD::VTBL2";
   case ARMISD::VMULLs:        return "ARMISD::VMULLs";
   case ARMISD::VMULLu:        return "ARMISD::VMULLu";
+  case ARMISD::UMLAL:         return "ARMISD::UMLAL";
+  case ARMISD::SMLAL:         return "ARMISD::SMLAL";
   case ARMISD::BUILD_VECTOR:  return "ARMISD::BUILD_VECTOR";
   case ARMISD::FMAX:          return "ARMISD::FMAX";
   case ARMISD::FMIN:          return "ARMISD::FMIN";
@@ -1591,19 +1594,19 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
 
   // FIXME: handle tail calls differently.
   unsigned CallOpc;
+  bool HasMinSizeAttr = MF.getFunction()->getFnAttributes().
+    hasAttribute(Attributes::MinSize);
   if (Subtarget->isThumb()) {
     if ((!isDirect || isARMFunc) && !Subtarget->hasV5TOps())
       CallOpc = ARMISD::CALL_NOLINK;
-    else if (doesNotRet && isDirect && !isARMFunc &&
-             Subtarget->hasRAS() && !Subtarget->isThumb1Only())
-      // "mov lr, pc; b _foo" to avoid confusing the RSP
-      CallOpc = ARMISD::CALL_NOLINK;
     else
       CallOpc = isARMFunc ? ARMISD::CALL : ARMISD::tCALL;
   } else {
-    if (!isDirect && !Subtarget->hasV5TOps()) {
+    if (!isDirect && !Subtarget->hasV5TOps())
       CallOpc = ARMISD::CALL_NOLINK;
-    } else if (doesNotRet && isDirect && Subtarget->hasRAS())
+    else if (doesNotRet && isDirect && Subtarget->hasRAS() &&
+               // Emit regular call when code size is the priority
+               !HasMinSizeAttr)
       // "mov lr, pc; b _foo" to avoid confusing the RSP
       CallOpc = ARMISD::CALL_NOLINK;
     else
@@ -1653,22 +1656,31 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
 /// and then confiscate the rest of the parameter registers to insure
 /// this.
 void
-ARMTargetLowering::HandleByVal(CCState *State, unsigned &size) const {
+ARMTargetLowering::HandleByVal(
+    CCState *State, unsigned &size, unsigned Align) const {
   unsigned reg = State->AllocateReg(GPRArgRegs, 4);
   assert((State->getCallOrPrologue() == Prologue ||
           State->getCallOrPrologue() == Call) &&
          "unhandled ParmContext");
   if ((!State->isFirstByValRegValid()) &&
       (ARM::R0 <= reg) && (reg <= ARM::R3)) {
-    State->setFirstByValReg(reg);
-    // At a call site, a byval parameter that is split between
-    // registers and memory needs its size truncated here.  In a
-    // function prologue, such byval parameters are reassembled in
-    // memory, and are not truncated.
-    if (State->getCallOrPrologue() == Call) {
-      unsigned excess = 4 * (ARM::R4 - reg);
-      assert(size >= excess && "expected larger existing stack allocation");
-      size -= excess;
+    if (Subtarget->isAAPCS_ABI() && Align > 4) {
+      unsigned AlignInRegs = Align / 4;
+      unsigned Waste = (ARM::R4 - reg) % AlignInRegs;
+      for (unsigned i = 0; i < Waste; ++i)
+        reg = State->AllocateReg(GPRArgRegs, 4);
+    }
+    if (reg != 0) {
+      State->setFirstByValReg(reg);
+      // At a call site, a byval parameter that is split between
+      // registers and memory needs its size truncated here.  In a
+      // function prologue, such byval parameters are reassembled in
+      // memory, and are not truncated.
+      if (State->getCallOrPrologue() == Call) {
+        unsigned excess = 4 * (ARM::R4 - reg);
+        assert(size >= excess && "expected larger existing stack allocation");
+        size -= excess;
+      }
     }
   }
   // Confiscate any remaining parameter registers to preclude their
@@ -1801,6 +1813,14 @@ ARMTargetLowering::IsEligibleForTailCallOptimization(SDValue Callee,
     }
   }
 
+  // If Caller's vararg or byval argument has been split between registers and
+  // stack, do not perform tail call, since part of the argument is in caller's
+  // local frame.
+  const ARMFunctionInfo *AFI_Caller = DAG.getMachineFunction().
+                                      getInfo<ARMFunctionInfo>();
+  if (AFI_Caller->getVarArgsRegSaveSize())
+    return false;
+
   // If the callee takes no arguments then go on to check the results of the
   // call.
   if (!Outs.empty()) {
@@ -2532,7 +2552,10 @@ ARMTargetLowering::computeRegArea(CCState &CCInfo, MachineFunction &MF,
 void
 ARMTargetLowering::VarArgStyleRegisters(CCState &CCInfo, SelectionDAG &DAG,
                                         DebugLoc dl, SDValue &Chain,
-                                        unsigned ArgOffset) const {
+                                        const Value *OrigArg,
+                                        unsigned OffsetFromOrigArg,
+                                        unsigned ArgOffset,
+                                        bool ForceMutable) const {
   MachineFunction &MF = DAG.getMachineFunction();
   MachineFrameInfo *MFI = MF.getFrameInfo();
   ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
@@ -2559,7 +2582,7 @@ ARMTargetLowering::VarArgStyleRegisters(CCState &CCInfo, SelectionDAG &DAG,
                                     getPointerTy());
 
     SmallVector<SDValue, 4> MemOps;
-    for (; firstRegToSaveIndex < 4; ++firstRegToSaveIndex) {
+    for (unsigned i = 0; firstRegToSaveIndex < 4; ++firstRegToSaveIndex, ++i) {
       const TargetRegisterClass *RC;
       if (AFI->isThumb1OnlyFunction())
         RC = &ARM::tGPRRegClass;
@@ -2570,7 +2593,7 @@ ARMTargetLowering::VarArgStyleRegisters(CCState &CCInfo, SelectionDAG &DAG,
       SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i32);
       SDValue Store =
         DAG.getStore(Val.getValue(1), dl, Val, FIN,
-                 MachinePointerInfo::getFixedStack(AFI->getVarArgsFrameIndex()),
+                     MachinePointerInfo(OrigArg, OffsetFromOrigArg + 4*i),
                      false, false, 0);
       MemOps.push_back(Store);
       FIN = DAG.getNode(ISD::ADD, dl, getPointerTy(), FIN,
@@ -2581,7 +2604,8 @@ ARMTargetLowering::VarArgStyleRegisters(CCState &CCInfo, SelectionDAG &DAG,
                           &MemOps[0], MemOps.size());
   } else
     // This will point to the next argument passed via stack.
-    AFI->setVarArgsFrameIndex(MFI->CreateFixedObject(4, ArgOffset, true));
+    AFI->setVarArgsFrameIndex(
+        MFI->CreateFixedObject(4, ArgOffset, !ForceMutable));
 }
 
 SDValue
@@ -2604,14 +2628,16 @@ ARMTargetLowering::LowerFormalArguments(SDValue Chain,
   CCInfo.AnalyzeFormalArguments(Ins,
                                 CCAssignFnForNode(CallConv, /* Return*/ false,
                                                   isVarArg));
-
+  
   SmallVector<SDValue, 16> ArgValues;
   int lastInsIndex = -1;
-
   SDValue ArgValue;
+  Function::const_arg_iterator CurOrigArg = MF.getFunction()->arg_begin();
+  unsigned CurArgIdx = 0;
   for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
     CCValAssign &VA = ArgLocs[i];
-
+    std::advance(CurOrigArg, Ins[VA.getValNo()].OrigArgIndex - CurArgIdx);
+    CurArgIdx = Ins[VA.getValNo()].OrigArgIndex;
     // Arguments stored in registers.
     if (VA.isRegLoc()) {
       EVT RegVT = VA.getLocVT();
@@ -2705,14 +2731,20 @@ ARMTargetLowering::LowerFormalArguments(SDValue Chain,
           // Since they could be overwritten by lowering of arguments in case of
           // a tail call.
           if (Flags.isByVal()) {
-            unsigned VARegSize, VARegSaveSize;
-            computeRegArea(CCInfo, MF, VARegSize, VARegSaveSize);
-            VarArgStyleRegisters(CCInfo, DAG, dl, Chain, 0);
-            unsigned Bytes = Flags.getByValSize() - VARegSize;
-            if (Bytes == 0) Bytes = 1; // Don't create zero-sized stack objects.
-            int FI = MFI->CreateFixedObject(Bytes,
-                                            VA.getLocMemOffset(), false);
-            InVals.push_back(DAG.getFrameIndex(FI, getPointerTy()));
+            ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
+            if (!AFI->getVarArgsFrameIndex()) {
+              VarArgStyleRegisters(CCInfo, DAG,
+                                   dl, Chain, CurOrigArg,
+                                   Ins[VA.getValNo()].PartOffset,
+                                   VA.getLocMemOffset(),
+                                   true /*force mutable frames*/);
+              int VAFrameIndex = AFI->getVarArgsFrameIndex();
+              InVals.push_back(DAG.getFrameIndex(VAFrameIndex, getPointerTy()));
+            } else {
+              int FI = MFI->CreateFixedObject(Flags.getByValSize(),
+                                              VA.getLocMemOffset(), false);
+              InVals.push_back(DAG.getFrameIndex(FI, getPointerTy()));              
+            }
           } else {
             int FI = MFI->CreateFixedObject(VA.getLocVT().getSizeInBits()/8,
                                             VA.getLocMemOffset(), true);
@@ -2730,7 +2762,8 @@ ARMTargetLowering::LowerFormalArguments(SDValue Chain,
 
   // varargs
   if (isVarArg)
-    VarArgStyleRegisters(CCInfo, DAG, dl, Chain, CCInfo.getNextStackOffset());
+    VarArgStyleRegisters(CCInfo, DAG, dl, Chain, 0, 0,
+                         CCInfo.getNextStackOffset());
 
   return Chain;
 }
@@ -3890,6 +3923,36 @@ SDValue ARMTargetLowering::LowerConstantFP(SDValue Op, SelectionDAG &DAG,
   return SDValue();
 }
 
+// check if an VEXT instruction can handle the shuffle mask when the
+// vector sources of the shuffle are the same.
+static bool isSingletonVEXTMask(ArrayRef<int> M, EVT VT, unsigned &Imm) {
+  unsigned NumElts = VT.getVectorNumElements();
+
+  // Assume that the first shuffle index is not UNDEF.  Fail if it is.
+  if (M[0] < 0)
+    return false;
+
+  Imm = M[0];
+
+  // If this is a VEXT shuffle, the immediate value is the index of the first
+  // element.  The other shuffle indices must be the successive elements after
+  // the first one.
+  unsigned ExpectedElt = Imm;
+  for (unsigned i = 1; i < NumElts; ++i) {
+    // Increment the expected index.  If it wraps around, just follow it
+    // back to index zero and keep going.
+    ++ExpectedElt;
+    if (ExpectedElt == NumElts)
+      ExpectedElt = 0;
+
+    if (M[i] < 0) continue; // ignore UNDEF indices
+    if (ExpectedElt != static_cast<unsigned>(M[i]))
+      return false;
+  }
+
+  return true;
+}
+
 
 static bool isVEXTMask(ArrayRef<int> M, EVT VT,
                        bool &ReverseVEXT, unsigned &Imm) {
@@ -4157,10 +4220,21 @@ SDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
   }
 
   // Scan through the operands to see if only one value is used.
+  //
+  // As an optimisation, even if more than one value is used it may be more
+  // profitable to splat with one value then change some lanes.
+  //
+  // Heuristically we decide to do this if the vector has a "dominant" value,
+  // defined as splatted to more than half of the lanes.
   unsigned NumElts = VT.getVectorNumElements();
   bool isOnlyLowElement = true;
   bool usesOnlyOneValue = true;
+  bool hasDominantValue = false;
   bool isConstant = true;
+
+  // Map of the number of times a particular SDValue appears in the
+  // element list.
+  DenseMap<SDValue, unsigned> ValueCounts;
   SDValue Value;
   for (unsigned i = 0; i < NumElts; ++i) {
     SDValue V = Op.getOperand(i);
@@ -4171,13 +4245,21 @@ SDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
     if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
       isConstant = false;
 
-    if (!Value.getNode())
+    ValueCounts.insert(std::make_pair(V, 0));
+    unsigned &Count = ValueCounts[V];
+    
+    // Is this value dominant? (takes up more than half of the lanes)
+    if (++Count > (NumElts / 2)) {
+      hasDominantValue = true;
       Value = V;
-    else if (V != Value)
-      usesOnlyOneValue = false;
+    }
   }
+  if (ValueCounts.size() != 1)
+    usesOnlyOneValue = false;
+  if (!Value.getNode() && ValueCounts.size() > 0)
+    Value = ValueCounts.begin()->first;
 
-  if (!Value.getNode())
+  if (ValueCounts.size() == 0)
     return DAG.getUNDEF(VT);
 
   if (isOnlyLowElement)
@@ -4187,9 +4269,51 @@ SDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
 
   // Use VDUP for non-constant splats.  For f32 constant splats, reduce to
   // i32 and try again.
-  if (usesOnlyOneValue && EltSize <= 32) {
-    if (!isConstant)
-      return DAG.getNode(ARMISD::VDUP, dl, VT, Value);
+  if (hasDominantValue && EltSize <= 32) {
+    if (!isConstant) {
+      SDValue N;
+
+      // If we are VDUPing a value that comes directly from a vector, that will
+      // cause an unnecessary move to and from a GPR, where instead we could
+      // just use VDUPLANE.
+      if (Value->getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
+        // We need to create a new undef vector to use for the VDUPLANE if the
+        // size of the vector from which we get the value is different than the
+        // size of the vector that we need to create. We will insert the element
+        // such that the register coalescer will remove unnecessary copies.
+        if (VT != Value->getOperand(0).getValueType()) {
+          ConstantSDNode *constIndex;
+          constIndex = dyn_cast<ConstantSDNode>(Value->getOperand(1));
+          assert(constIndex && "The index is not a constant!");
+          unsigned index = constIndex->getAPIntValue().getLimitedValue() %
+                             VT.getVectorNumElements();
+          N =  DAG.getNode(ARMISD::VDUPLANE, dl, VT,
+                 DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, DAG.getUNDEF(VT),
+                        Value, DAG.getConstant(index, MVT::i32)),
+                           DAG.getConstant(index, MVT::i32));
+        } else {
+          N = DAG.getNode(ARMISD::VDUPLANE, dl, VT,
+                        Value->getOperand(0), Value->getOperand(1));
+        }
+      }
+      else
+        N = DAG.getNode(ARMISD::VDUP, dl, VT, Value);
+
+      if (!usesOnlyOneValue) {
+        // The dominant value was splatted as 'N', but we now have to insert
+        // all differing elements.
+        for (unsigned I = 0; I < NumElts; ++I) {
+          if (Op.getOperand(I) == Value)
+            continue;
+          SmallVector<SDValue, 3> Ops;
+          Ops.push_back(N);
+          Ops.push_back(Op.getOperand(I));
+          Ops.push_back(DAG.getConstant(I, MVT::i32));
+          N = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, &Ops[0], 3);
+        }
+      }
+      return N;
+    }
     if (VT.getVectorElementType().isFloatingPoint()) {
       SmallVector<SDValue, 8> Ops;
       for (unsigned i = 0; i < NumElts; ++i)
@@ -4201,9 +4325,11 @@ SDValue ARMTargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
       if (Val.getNode())
         return DAG.getNode(ISD::BITCAST, dl, VT, Val);
     }
-    SDValue Val = IsSingleInstrConstant(Value, DAG, ST, dl);
-    if (Val.getNode())
-      return DAG.getNode(ARMISD::VDUP, dl, VT, Val);
+    if (usesOnlyOneValue) {
+      SDValue Val = IsSingleInstrConstant(Value, DAG, ST, dl);
+      if (isConstant && Val.getNode())
+        return DAG.getNode(ARMISD::VDUP, dl, VT, Val); 
+    }
   }
 
   // If all elements are constants and the case above didn't get hit, fall back
@@ -4586,6 +4712,12 @@ static SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) {
     if (isVREVMask(ShuffleMask, VT, 16))
       return DAG.getNode(ARMISD::VREV16, dl, VT, V1);
 
+    if (V2->getOpcode() == ISD::UNDEF &&
+        isSingletonVEXTMask(ShuffleMask, VT, Imm)) {
+      return DAG.getNode(ARMISD::VEXT, dl, VT, V1, V1,
+                         DAG.getConstant(Imm, MVT::i32));
+    }
+
     // Check for Neon shuffles that modify both input vectors in place.
     // If both results are used, i.e., if there are two shuffles with the same
     // source operands and with masks corresponding to both results of one of
@@ -5421,7 +5553,7 @@ ARMTargetLowering::EmitAtomicBinary(MachineInstr *MI, MachineBasicBlock *BB,
   exitMBB->transferSuccessorsAndUpdatePHIs(BB);
 
   const TargetRegisterClass *TRC = isThumb2 ?
-    (const TargetRegisterClass*)&ARM::tGPRRegClass :
+    (const TargetRegisterClass*)&ARM::rGPRRegClass :
     (const TargetRegisterClass*)&ARM::GPRRegClass;
   unsigned scratch = MRI.createVirtualRegister(TRC);
   unsigned scratch2 = (!BinOpcode) ? incr : MRI.createVirtualRegister(TRC);
@@ -5532,7 +5664,7 @@ ARMTargetLowering::EmitAtomicBinaryMinMax(MachineInstr *MI,
   exitMBB->transferSuccessorsAndUpdatePHIs(BB);
 
   const TargetRegisterClass *TRC = isThumb2 ?
-    (const TargetRegisterClass*)&ARM::tGPRRegClass :
+    (const TargetRegisterClass*)&ARM::rGPRRegClass :
     (const TargetRegisterClass*)&ARM::GPRRegClass;
   unsigned scratch = MRI.createVirtualRegister(TRC);
   unsigned scratch2 = MRI.createVirtualRegister(TRC);
@@ -5546,7 +5678,7 @@ ARMTargetLowering::EmitAtomicBinaryMinMax(MachineInstr *MI,
   //   ldrex dest, ptr
   //   (sign extend dest, if required)
   //   cmp dest, incr
-  //   cmov.cond scratch2, dest, incr
+  //   cmov.cond scratch2, incr, dest
   //   strex scratch, scratch2, ptr
   //   cmp scratch, #0
   //   bne- loopMBB
@@ -5569,7 +5701,7 @@ ARMTargetLowering::EmitAtomicBinaryMinMax(MachineInstr *MI,
   AddDefaultPred(BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2CMPrr : ARM::CMPrr))
                  .addReg(oldval).addReg(incr));
   BuildMI(BB, dl, TII->get(isThumb2 ? ARM::t2MOVCCr : ARM::MOVCCr), scratch2)
-         .addReg(oldval).addReg(incr).addImm(Cond).addReg(ARM::CPSR);
+         .addReg(incr).addReg(oldval).addImm(Cond).addReg(ARM::CPSR);
 
   MIB = BuildMI(BB, dl, TII->get(strOpc), scratch).addReg(scratch2).addReg(ptr);
   if (strOpc == ARM::t2STREX)
@@ -5939,12 +6071,15 @@ EmitSjLjDispatchBlock(MachineInstr *MI, MachineBasicBlock *MBB) const {
                              MachineMemOperand::MOLoad |
                              MachineMemOperand::MOVolatile, 4, 4);
 
-  if (AFI->isThumb1OnlyFunction())
-    BuildMI(DispatchBB, dl, TII->get(ARM::tInt_eh_sjlj_dispatchsetup));
-  else if (!Subtarget->hasVFP2())
-    BuildMI(DispatchBB, dl, TII->get(ARM::Int_eh_sjlj_dispatchsetup_nofp));
-  else
-    BuildMI(DispatchBB, dl, TII->get(ARM::Int_eh_sjlj_dispatchsetup));
+  MachineInstrBuilder MIB;
+  MIB = BuildMI(DispatchBB, dl, TII->get(ARM::Int_eh_sjlj_dispatchsetup));
+
+  const ARMBaseInstrInfo *AII = static_cast<const ARMBaseInstrInfo*>(TII);
+  const ARMBaseRegisterInfo &RI = AII->getRegisterInfo();
+
+  // Add a register mask with no preserved registers.  This results in all
+  // registers being marked as clobbered.
+  MIB.addRegMask(RI.getNoPreservedMask());
 
   unsigned NumLPads = LPadList.size();
   if (Subtarget->isThumb2()) {
@@ -6016,9 +6151,9 @@ EmitSjLjDispatchBlock(MachineInstr *MI, MachineBasicBlock *MBB) const {
       const Constant *C = ConstantInt::get(Int32Ty, NumLPads);
 
       // MachineConstantPool wants an explicit alignment.
-      unsigned Align = getTargetData()->getPrefTypeAlignment(Int32Ty);
+      unsigned Align = getDataLayout()->getPrefTypeAlignment(Int32Ty);
       if (Align == 0)
-        Align = getTargetData()->getTypeAllocSize(C->getType());
+        Align = getDataLayout()->getTypeAllocSize(C->getType());
       unsigned Idx = ConstantPool->getConstantPoolIndex(C, Align);
 
       unsigned VReg1 = MRI->createVirtualRegister(TRC);
@@ -6105,9 +6240,9 @@ EmitSjLjDispatchBlock(MachineInstr *MI, MachineBasicBlock *MBB) const {
       const Constant *C = ConstantInt::get(Int32Ty, NumLPads);
 
       // MachineConstantPool wants an explicit alignment.
-      unsigned Align = getTargetData()->getPrefTypeAlignment(Int32Ty);
+      unsigned Align = getDataLayout()->getPrefTypeAlignment(Int32Ty);
       if (Align == 0)
-        Align = getTargetData()->getTypeAllocSize(C->getType());
+        Align = getDataLayout()->getTypeAllocSize(C->getType());
       unsigned Idx = ConstantPool->getConstantPoolIndex(C, Align);
 
       unsigned VReg1 = MRI->createVirtualRegister(TRC);
@@ -6154,18 +6289,15 @@ EmitSjLjDispatchBlock(MachineInstr *MI, MachineBasicBlock *MBB) const {
   }
 
   // Add the jump table entries as successors to the MBB.
-  MachineBasicBlock *PrevMBB = 0;
+  SmallPtrSet<MachineBasicBlock*, 8> SeenMBBs;
   for (std::vector<MachineBasicBlock*>::iterator
          I = LPadList.begin(), E = LPadList.end(); I != E; ++I) {
     MachineBasicBlock *CurMBB = *I;
-    if (PrevMBB != CurMBB)
+    if (SeenMBBs.insert(CurMBB))
       DispContBB->addSuccessor(CurMBB);
-    PrevMBB = CurMBB;
   }
 
   // N.B. the order the invoke BBs are processed in doesn't matter here.
-  const ARMBaseInstrInfo *AII = static_cast<const ARMBaseInstrInfo*>(TII);
-  const ARMBaseRegisterInfo &RI = AII->getRegisterInfo();
   const uint16_t *SavedRegs = RI.getCalleeSavedRegs(MF);
   SmallVector<MachineBasicBlock*, 64> MBBLPads;
   for (SmallPtrSet<MachineBasicBlock*, 64>::iterator
@@ -6279,7 +6411,8 @@ EmitStructByval(MachineInstr *MI, MachineBasicBlock *BB) const {
     UnitSize = 2;
   } else {
     // Check whether we can use NEON instructions.
-    if (!MF->getFunction()->hasFnAttr(Attribute::NoImplicitFloat) &&
+    if (!MF->getFunction()->getFnAttributes().
+          hasAttribute(Attributes::NoImplicitFloat) &&
         Subtarget->hasNEON()) {
       if ((Align % 16 == 0) && SizeVal >= 16) {
         ldrOpc = ARM::VLD1q32wb_fixed;
@@ -6364,7 +6497,8 @@ EmitStructByval(MachineInstr *MI, MachineBasicBlock *BB) const {
       } else {
         AddDefaultPred(BuildMI(*BB, MI, dl,
           TII->get(ldrOpc),scratch)
-          .addReg(srcOut, RegState::Define).addReg(srcIn).addImm(1));
+          .addReg(srcOut, RegState::Define).addReg(srcIn)
+          .addReg(0).addImm(1));
 
         AddDefaultPred(BuildMI(*BB, MI, dl, TII->get(strOpc), destOut)
           .addReg(scratch).addReg(destIn)
@@ -6427,9 +6561,9 @@ EmitStructByval(MachineInstr *MI, MachineBasicBlock *BB) const {
     const Constant *C = ConstantInt::get(Int32Ty, LoopSize);
 
     // MachineConstantPool wants an explicit alignment.
-    unsigned Align = getTargetData()->getPrefTypeAlignment(Int32Ty);
+    unsigned Align = getDataLayout()->getPrefTypeAlignment(Int32Ty);
     if (Align == 0)
-      Align = getTargetData()->getTypeAllocSize(C->getType());
+      Align = getDataLayout()->getTypeAllocSize(C->getType());
     unsigned Idx = ConstantPool->getConstantPoolIndex(C, Align);
 
     AddDefaultPred(BuildMI(BB, dl, TII->get(ARM::LDRcp))
@@ -6981,73 +7115,131 @@ static inline bool isZeroOrAllOnes(SDValue N, bool AllOnes) {
   return AllOnes ? C->isAllOnesValue() : C->isNullValue();
 }
 
+// Return true if N is conditionally 0 or all ones.
+// Detects these expressions where cc is an i1 value:
+//
+//   (select cc 0, y)   [AllOnes=0]
+//   (select cc y, 0)   [AllOnes=0]
+//   (zext cc)          [AllOnes=0]
+//   (sext cc)          [AllOnes=0/1]
+//   (select cc -1, y)  [AllOnes=1]
+//   (select cc y, -1)  [AllOnes=1]
+//
+// Invert is set when N is the null/all ones constant when CC is false.
+// OtherOp is set to the alternative value of N.
+static bool isConditionalZeroOrAllOnes(SDNode *N, bool AllOnes,
+                                       SDValue &CC, bool &Invert,
+                                       SDValue &OtherOp,
+                                       SelectionDAG &DAG) {
+  switch (N->getOpcode()) {
+  default: return false;
+  case ISD::SELECT: {
+    CC = N->getOperand(0);
+    SDValue N1 = N->getOperand(1);
+    SDValue N2 = N->getOperand(2);
+    if (isZeroOrAllOnes(N1, AllOnes)) {
+      Invert = false;
+      OtherOp = N2;
+      return true;
+    }
+    if (isZeroOrAllOnes(N2, AllOnes)) {
+      Invert = true;
+      OtherOp = N1;
+      return true;
+    }
+    return false;
+  }
+  case ISD::ZERO_EXTEND:
+    // (zext cc) can never be the all ones value.
+    if (AllOnes)
+      return false;
+    // Fall through.
+  case ISD::SIGN_EXTEND: {
+    EVT VT = N->getValueType(0);
+    CC = N->getOperand(0);
+    if (CC.getValueType() != MVT::i1)
+      return false;
+    Invert = !AllOnes;
+    if (AllOnes)
+      // When looking for an AllOnes constant, N is an sext, and the 'other'
+      // value is 0.
+      OtherOp = DAG.getConstant(0, VT);
+    else if (N->getOpcode() == ISD::ZERO_EXTEND)
+      // When looking for a 0 constant, N can be zext or sext.
+      OtherOp = DAG.getConstant(1, VT);
+    else
+      OtherOp = DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), VT);
+    return true;
+  }
+  }
+}
+
 // Combine a constant select operand into its use:
 //
-//   (add (select cc, 0, c), x) -> (select cc, x, (add, x, c))
-//   (sub x, (select cc, 0, c)) -> (select cc, x, (sub, x, c))
+//   (add (select cc, 0, c), x)  -> (select cc, x, (add, x, c))
+//   (sub x, (select cc, 0, c))  -> (select cc, x, (sub, x, c))
+//   (and (select cc, -1, c), x) -> (select cc, x, (and, x, c))  [AllOnes=1]
+//   (or  (select cc, 0, c), x)  -> (select cc, x, (or, x, c))
+//   (xor (select cc, 0, c), x)  -> (select cc, x, (xor, x, c))
 //
 // The transform is rejected if the select doesn't have a constant operand that
-// is null.
+// is null, or all ones when AllOnes is set.
+//
+// Also recognize sext/zext from i1:
+//
+//   (add (zext cc), x) -> (select cc (add x, 1), x)
+//   (add (sext cc), x) -> (select cc (add x, -1), x)
+//
+// These transformations eventually create predicated instructions.
 //
 // @param N       The node to transform.
 // @param Slct    The N operand that is a select.
 // @param OtherOp The other N operand (x above).
 // @param DCI     Context.
+// @param AllOnes Require the select constant to be all ones instead of null.
 // @returns The new node, or SDValue() on failure.
 static
 SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp,
-                            TargetLowering::DAGCombinerInfo &DCI) {
+                            TargetLowering::DAGCombinerInfo &DCI,
+                            bool AllOnes = false) {
   SelectionDAG &DAG = DCI.DAG;
-  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
   EVT VT = N->getValueType(0);
-  unsigned Opc = N->getOpcode();
-  bool isSlctCC = Slct.getOpcode() == ISD::SELECT_CC;
-  SDValue LHS = isSlctCC ? Slct.getOperand(2) : Slct.getOperand(1);
-  SDValue RHS = isSlctCC ? Slct.getOperand(3) : Slct.getOperand(2);
-  ISD::CondCode CC = ISD::SETCC_INVALID;
-
-  if (isSlctCC) {
-    CC = cast<CondCodeSDNode>(Slct.getOperand(4))->get();
-  } else {
-    SDValue CCOp = Slct.getOperand(0);
-    if (CCOp.getOpcode() == ISD::SETCC)
-      CC = cast<CondCodeSDNode>(CCOp.getOperand(2))->get();
-  }
-
-  bool DoXform = false;
-  bool InvCC = false;
-  assert ((Opc == ISD::ADD || (Opc == ISD::SUB && Slct == N->getOperand(1))) &&
-          "Bad input!");
+  SDValue NonConstantVal;
+  SDValue CCOp;
+  bool SwapSelectOps;
+  if (!isConditionalZeroOrAllOnes(Slct.getNode(), AllOnes, CCOp, SwapSelectOps,
+                                  NonConstantVal, DAG))
+    return SDValue();
 
-  if (isZeroOrAllOnes(LHS, false)) {
-    DoXform = true;
-  } else if (CC != ISD::SETCC_INVALID && isZeroOrAllOnes(RHS, false)) {
-    std::swap(LHS, RHS);
-    SDValue Op0 = Slct.getOperand(0);
-    EVT OpVT = isSlctCC ? Op0.getValueType() : Op0.getOperand(0).getValueType();
-    bool isInt = OpVT.isInteger();
-    CC = ISD::getSetCCInverse(CC, isInt);
+  // Slct is now know to be the desired identity constant when CC is true.
+  SDValue TrueVal = OtherOp;
+  SDValue FalseVal = DAG.getNode(N->getOpcode(), N->getDebugLoc(), VT,
+                                 OtherOp, NonConstantVal);
+  // Unless SwapSelectOps says CC should be false.
+  if (SwapSelectOps)
+    std::swap(TrueVal, FalseVal);
 
-    if (!TLI.isCondCodeLegal(CC, OpVT))
-      return SDValue();         // Inverse operator isn't legal.
+  return DAG.getNode(ISD::SELECT, N->getDebugLoc(), VT,
+                     CCOp, TrueVal, FalseVal);
+}
 
-    DoXform = true;
-    InvCC = true;
+// Attempt combineSelectAndUse on each operand of a commutative operator N.
+static
+SDValue combineSelectAndUseCommutative(SDNode *N, bool AllOnes,
+                                       TargetLowering::DAGCombinerInfo &DCI) {
+  SDValue N0 = N->getOperand(0);
+  SDValue N1 = N->getOperand(1);
+  if (N0.getNode()->hasOneUse()) {
+    SDValue Result = combineSelectAndUse(N, N0, N1, DCI, AllOnes);
+    if (Result.getNode())
+      return Result;
   }
-
-  if (!DoXform)
-    return SDValue();
-
-  SDValue Result = DAG.getNode(Opc, RHS.getDebugLoc(), VT, OtherOp, RHS);
-  if (isSlctCC)
-    return DAG.getSelectCC(N->getDebugLoc(), OtherOp, Result,
-                           Slct.getOperand(0), Slct.getOperand(1), CC);
-  SDValue CCOp = Slct.getOperand(0);
-  if (InvCC)
-    CCOp = DAG.getSetCC(Slct.getDebugLoc(), CCOp.getValueType(),
-                        CCOp.getOperand(0), CCOp.getOperand(1), CC);
-  return DAG.getNode(ISD::SELECT, N->getDebugLoc(), VT,
-                     CCOp, OtherOp, Result);
+  if (N1.getNode()->hasOneUse()) {
+    SDValue Result = combineSelectAndUse(N, N1, N0, DCI, AllOnes);
+    if (Result.getNode())
+      return Result;
+  }
+  return SDValue();
 }
 
 // AddCombineToVPADDL- For pair-wise add on neon, use the vpaddl instruction
@@ -7139,6 +7331,154 @@ static SDValue AddCombineToVPADDL(SDNode *N, SDValue N0, SDValue N1,
   return DAG.getNode(ISD::TRUNCATE, N->getDebugLoc(), VT, tmp);
 }
 
+static SDValue findMUL_LOHI(SDValue V) {
+  if (V->getOpcode() == ISD::UMUL_LOHI ||
+      V->getOpcode() == ISD::SMUL_LOHI)
+    return V;
+  return SDValue();
+}
+
+static SDValue AddCombineTo64bitMLAL(SDNode *AddcNode,
+                                     TargetLowering::DAGCombinerInfo &DCI,
+                                     const ARMSubtarget *Subtarget) {
+
+  if (Subtarget->isThumb1Only()) return SDValue();
+
+  // Only perform the checks after legalize when the pattern is available.
+  if (DCI.isBeforeLegalize()) return SDValue();
+
+  // Look for multiply add opportunities.
+  // The pattern is a ISD::UMUL_LOHI followed by two add nodes, where
+  // each add nodes consumes a value from ISD::UMUL_LOHI and there is
+  // a glue link from the first add to the second add.
+  // If we find this pattern, we can replace the U/SMUL_LOHI, ADDC, and ADDE by
+  // a S/UMLAL instruction.
+  //          loAdd   UMUL_LOHI
+  //            \    / :lo    \ :hi
+  //             \  /          \          [no multiline comment]
+  //              ADDC         |  hiAdd
+  //                 \ :glue  /  /
+  //                  \      /  /
+  //                    ADDE
+  //
+  assert(AddcNode->getOpcode() == ISD::ADDC && "Expect an ADDC");
+  SDValue AddcOp0 = AddcNode->getOperand(0);
+  SDValue AddcOp1 = AddcNode->getOperand(1);
+
+  // Check if the two operands are from the same mul_lohi node.
+  if (AddcOp0.getNode() == AddcOp1.getNode())
+    return SDValue();
+
+  assert(AddcNode->getNumValues() == 2 &&
+         AddcNode->getValueType(0) == MVT::i32 &&
+         AddcNode->getValueType(1) == MVT::Glue &&
+         "Expect ADDC with two result values: i32, glue");
+
+  // Check that the ADDC adds the low result of the S/UMUL_LOHI.
+  if (AddcOp0->getOpcode() != ISD::UMUL_LOHI &&
+      AddcOp0->getOpcode() != ISD::SMUL_LOHI &&
+      AddcOp1->getOpcode() != ISD::UMUL_LOHI &&
+      AddcOp1->getOpcode() != ISD::SMUL_LOHI)
+    return SDValue();
+
+  // Look for the glued ADDE.
+  SDNode* AddeNode = AddcNode->getGluedUser();
+  if (AddeNode == NULL)
+    return SDValue();
+
+  // Make sure it is really an ADDE.
+  if (AddeNode->getOpcode() != ISD::ADDE)
+    return SDValue();
+
+  assert(AddeNode->getNumOperands() == 3 &&
+         AddeNode->getOperand(2).getValueType() == MVT::Glue &&
+         "ADDE node has the wrong inputs");
+
+  // Check for the triangle shape.
+  SDValue AddeOp0 = AddeNode->getOperand(0);
+  SDValue AddeOp1 = AddeNode->getOperand(1);
+
+  // Make sure that the ADDE operands are not coming from the same node.
+  if (AddeOp0.getNode() == AddeOp1.getNode())
+    return SDValue();
+
+  // Find the MUL_LOHI node walking up ADDE's operands.
+  bool IsLeftOperandMUL = false;
+  SDValue MULOp = findMUL_LOHI(AddeOp0);
+  if (MULOp == SDValue())
+   MULOp = findMUL_LOHI(AddeOp1);
+  else
+    IsLeftOperandMUL = true;
+  if (MULOp == SDValue())
+     return SDValue();
+
+  // Figure out the right opcode.
+  unsigned Opc = MULOp->getOpcode();
+  unsigned FinalOpc = (Opc == ISD::SMUL_LOHI) ? ARMISD::SMLAL : ARMISD::UMLAL;
+
+  // Figure out the high and low input values to the MLAL node.
+  SDValue* HiMul = &MULOp;
+  SDValue* HiAdd = NULL;
+  SDValue* LoMul = NULL;
+  SDValue* LowAdd = NULL;
+
+  if (IsLeftOperandMUL)
+    HiAdd = &AddeOp1;
+  else
+    HiAdd = &AddeOp0;
+
+
+  if (AddcOp0->getOpcode() == Opc) {
+    LoMul = &AddcOp0;
+    LowAdd = &AddcOp1;
+  }
+  if (AddcOp1->getOpcode() == Opc) {
+    LoMul = &AddcOp1;
+    LowAdd = &AddcOp0;
+  }
+
+  if (LoMul == NULL)
+    return SDValue();
+
+  if (LoMul->getNode() != HiMul->getNode())
+    return SDValue();
+
+  // Create the merged node.
+  SelectionDAG &DAG = DCI.DAG;
+
+  // Build operand list.
+  SmallVector<SDValue, 8> Ops;
+  Ops.push_back(LoMul->getOperand(0));
+  Ops.push_back(LoMul->getOperand(1));
+  Ops.push_back(*LowAdd);
+  Ops.push_back(*HiAdd);
+
+  SDValue MLALNode =  DAG.getNode(FinalOpc, AddcNode->getDebugLoc(),
+                                 DAG.getVTList(MVT::i32, MVT::i32),
+                                 &Ops[0], Ops.size());
+
+  // Replace the ADDs' nodes uses by the MLA node's values.
+  SDValue HiMLALResult(MLALNode.getNode(), 1);
+  DAG.ReplaceAllUsesOfValueWith(SDValue(AddeNode, 0), HiMLALResult);
+
+  SDValue LoMLALResult(MLALNode.getNode(), 0);
+  DAG.ReplaceAllUsesOfValueWith(SDValue(AddcNode, 0), LoMLALResult);
+
+  // Return original node to notify the driver to stop replacing.
+  SDValue resNode(AddcNode, 0);
+  return resNode;
+}
+
+/// PerformADDCCombine - Target-specific dag combine transform from
+/// ISD::ADDC, ISD::ADDE, and ISD::MUL_LOHI to MLAL.
+static SDValue PerformADDCCombine(SDNode *N,
+                                 TargetLowering::DAGCombinerInfo &DCI,
+                                 const ARMSubtarget *Subtarget) {
+
+  return AddCombineTo64bitMLAL(N, DCI, Subtarget);
+
+}
+
 /// PerformADDCombineWithOperands - Try DAG combinations for an ADD with
 /// operands N0 and N1.  This is a helper for PerformADDCombine that is
 /// called with the default operands, and if that fails, with commuted
@@ -7153,7 +7493,7 @@ static SDValue PerformADDCombineWithOperands(SDNode *N, SDValue N0, SDValue N1,
     return Result;
 
   // fold (add (select cc, 0, c), x) -> (select cc, x, (add, x, c))
-  if (N0.getOpcode() == ISD::SELECT && N0.getNode()->hasOneUse()) {
+  if (N0.getNode()->hasOneUse()) {
     SDValue Result = combineSelectAndUse(N, N0, N1, DCI);
     if (Result.getNode()) return Result;
   }
@@ -7185,7 +7525,7 @@ static SDValue PerformSUBCombine(SDNode *N,
   SDValue N1 = N->getOperand(1);
 
   // fold (sub x, (select cc, 0, c)) -> (select cc, x, (sub, x, c))
-  if (N1.getOpcode() == ISD::SELECT && N1.getNode()->hasOneUse()) {
+  if (N1.getNode()->hasOneUse()) {
     SDValue Result = combineSelectAndUse(N, N1, N0, DCI);
     if (Result.getNode()) return Result;
   }
@@ -7313,41 +7653,6 @@ static SDValue PerformMULCombine(SDNode *N,
   return SDValue();
 }
 
-static bool isCMOVWithZeroOrAllOnesLHS(SDValue N, bool AllOnes) {
-  return N.getOpcode() == ARMISD::CMOV && N.getNode()->hasOneUse() &&
-    isZeroOrAllOnes(N.getOperand(0), AllOnes);
-}
-
-/// formConditionalOp - Combine an operation with a conditional move operand
-/// to form a conditional op. e.g. (or x, (cmov 0, y, cond)) => (or.cond x, y)
-/// (and x, (cmov -1, y, cond)) => (and.cond, x, y)
-static SDValue formConditionalOp(SDNode *N, SelectionDAG &DAG,
-                                 bool Commutable) {
-  SDValue N0 = N->getOperand(0);
-  SDValue N1 = N->getOperand(1);
-
-  bool isAND = N->getOpcode() == ISD::AND;
-  bool isCand = isCMOVWithZeroOrAllOnesLHS(N1, isAND);
-  if (!isCand && Commutable) {
-    isCand = isCMOVWithZeroOrAllOnesLHS(N0, isAND);
-    if (isCand)
-      std::swap(N0, N1);
-  }
-  if (!isCand)
-    return SDValue();
-
-  unsigned Opc = 0;
-  switch (N->getOpcode()) {
-  default: llvm_unreachable("Unexpected node");
-  case ISD::AND: Opc = ARMISD::CAND; break;
-  case ISD::OR:  Opc = ARMISD::COR; break;
-  case ISD::XOR: Opc = ARMISD::CXOR; break;
-  }
-  return DAG.getNode(Opc, N->getDebugLoc(), N->getValueType(0), N0,
-                     N1.getOperand(1), N1.getOperand(2), N1.getOperand(3),
-                     N1.getOperand(4));
-}
-
 static SDValue PerformANDCombine(SDNode *N,
                                  TargetLowering::DAGCombinerInfo &DCI,
                                  const ARMSubtarget *Subtarget) {
@@ -7382,10 +7687,10 @@ static SDValue PerformANDCombine(SDNode *N,
   }
 
   if (!Subtarget->isThumb1Only()) {
-    // (and x, (cmov -1, y, cond)) => (and.cond x, y)
-    SDValue CAND = formConditionalOp(N, DAG, true);
-    if (CAND.getNode())
-      return CAND;
+    // fold (and (select cc, -1, c), x) -> (select cc, x, (and, x, c))
+    SDValue Result = combineSelectAndUseCommutative(N, true, DCI);
+    if (Result.getNode())
+      return Result;
   }
 
   return SDValue();
@@ -7425,13 +7730,12 @@ static SDValue PerformORCombine(SDNode *N,
   }
 
   if (!Subtarget->isThumb1Only()) {
-    // (or x, (cmov 0, y, cond)) => (or.cond x, y)
-    SDValue COR = formConditionalOp(N, DAG, true);
-    if (COR.getNode())
-      return COR;
+    // fold (or (select cc, 0, c), x) -> (select cc, x, (or, x, c))
+    SDValue Result = combineSelectAndUseCommutative(N, false, DCI);
+    if (Result.getNode())
+      return Result;
   }
 
-
   // The code below optimizes (or (and X, Y), Z).
   // The AND operand needs to have a single user to make these optimizations
   // profitable.
@@ -7593,10 +7897,10 @@ static SDValue PerformXORCombine(SDNode *N,
     return SDValue();
 
   if (!Subtarget->isThumb1Only()) {
-    // (xor x, (cmov 0, y, cond)) => (xor.cond x, y)
-    SDValue CXOR = formConditionalOp(N, DAG, true);
-    if (CXOR.getNode())
-      return CXOR;
+    // fold (xor (select cc, 0, c), x) -> (select cc, x, (xor, x, c))
+    SDValue Result = combineSelectAndUseCommutative(N, false, DCI);
+    if (Result.getNode())
+      return Result;
   }
 
   return SDValue();
@@ -8746,6 +9050,7 @@ SDValue ARMTargetLowering::PerformDAGCombine(SDNode *N,
                                              DAGCombinerInfo &DCI) const {
   switch (N->getOpcode()) {
   default: break;
+  case ISD::ADDC:       return PerformADDCCombine(N, DCI, Subtarget);
   case ISD::ADD:        return PerformADDCombine(N, DCI, Subtarget);
   case ISD::SUB:        return PerformSUBCombine(N, DCI);
   case ISD::MUL:        return PerformMULCombine(N, DCI, Subtarget);
@@ -8807,8 +9112,8 @@ bool ARMTargetLowering::isDesirableToTransformToIntegerOp(unsigned Opc,
 }
 
 bool ARMTargetLowering::allowsUnalignedMemoryAccesses(EVT VT) const {
-  if (!Subtarget->allowsUnalignedMem())
-    return false;
+  // The AllowsUnaliged flag models the SCTLR.A setting in ARM cpus
+  bool AllowsUnaligned = Subtarget->allowsUnalignedMem();
 
   switch (VT.getSimpleVT().SimpleTy) {
   default:
@@ -8816,10 +9121,14 @@ bool ARMTargetLowering::allowsUnalignedMemoryAccesses(EVT VT) const {
   case MVT::i8:
   case MVT::i16:
   case MVT::i32:
-    return true;
+    // Unaligned access can use (for example) LRDB, LRDH, LDR
+    return AllowsUnaligned;
   case MVT::f64:
-    return Subtarget->hasNEON();
-  // FIXME: VLD1 etc with standard alignment is legal.
+  case MVT::v2f64:
+    // For any little-endian targets with neon, we can support unaligned ld/st
+    // of D and Q (e.g. {D0,D1}) registers by using vld1.i8/vst1.i8.
+    // A big-endian target may also explictly support unaligned accesses
+    return Subtarget->hasNEON() && (AllowsUnaligned || isLittleEndian());
   }
 }
 
@@ -8838,7 +9147,7 @@ EVT ARMTargetLowering::getOptimalMemOpType(uint64_t Size,
 
   // See if we can use NEON instructions for this...
   if (IsZeroVal &&
-      !F->hasFnAttr(Attribute::NoImplicitFloat) &&
+      !F->getFnAttributes().hasAttribute(Attributes::NoImplicitFloat) &&
       Subtarget->hasNEON()) {
     if (memOpAlign(SrcAlign, DstAlign, 16) && Size >= 16) {
       return MVT::v4i32;
@@ -9632,7 +9941,7 @@ bool ARMTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
   case Intrinsic::arm_neon_vld4lane: {
     Info.opc = ISD::INTRINSIC_W_CHAIN;
     // Conservatively set memVT to the entire set of vectors loaded.
-    uint64_t NumElts = getTargetData()->getTypeAllocSize(I.getType()) / 8;
+    uint64_t NumElts = getDataLayout()->getTypeAllocSize(I.getType()) / 8;
     Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts);
     Info.ptrVal = I.getArgOperand(0);
     Info.offset = 0;
@@ -9657,7 +9966,7 @@ bool ARMTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
       Type *ArgTy = I.getArgOperand(ArgI)->getType();
       if (!ArgTy->isVectorTy())
         break;
-      NumElts += getTargetData()->getTypeAllocSize(ArgTy) / 8;
+      NumElts += getDataLayout()->getTypeAllocSize(ArgTy) / 8;
     }
     Info.memVT = EVT::getVectorVT(I.getType()->getContext(), MVT::i64, NumElts);
     Info.ptrVal = I.getArgOperand(0);
diff --git a/lib/Target/ARM/ARMISelLowering.h b/lib/Target/ARM/ARMISelLowering.h
index 51d1205..4eb3b2c 100644
--- a/lib/Target/ARM/ARMISelLowering.h
+++ b/lib/Target/ARM/ARMISelLowering.h
@@ -63,9 +63,6 @@ namespace llvm {
       FMSTAT,       // ARM fmstat instruction.
 
       CMOV,         // ARM conditional move instructions.
-      CAND,         // ARM conditional and instructions.
-      COR,          // ARM conditional or instructions.
-      CXOR,         // ARM conditional xor instructions.
 
       BCC_i64,
 
@@ -176,6 +173,9 @@ namespace llvm {
       VMULLs,       // ...signed
       VMULLu,       // ...unsigned
 
+      UMLAL,        // 64bit Unsigned Accumulate Multiply
+      SMLAL,        // 64bit Signed Accumulate Multiply
+
       // Operands of the standard BUILD_VECTOR node are not legalized, which
       // is fine if BUILD_VECTORs are always lowered to shuffles or other
       // operations, but for ARM some BUILD_VECTORs are legal as-is and their
@@ -260,6 +260,11 @@ namespace llvm {
 
     virtual const char *getTargetNodeName(unsigned Opcode) const;
 
+    virtual bool isSelectSupported(SelectSupportKind Kind) const {
+      // ARM does not support scalar condition selects on vectors.
+      return (Kind != ScalarCondVectorVal);
+    }
+
     /// getSetCCResultType - Return the value type to use for ISD::SETCC.
     virtual EVT getSetCCResultType(EVT VT) const;
 
@@ -461,7 +466,11 @@ namespace llvm {
                            SmallVectorImpl<SDValue> &InVals) const;
 
     void VarArgStyleRegisters(CCState &CCInfo, SelectionDAG &DAG,
-                              DebugLoc dl, SDValue &Chain, unsigned ArgOffset)
+                              DebugLoc dl, SDValue &Chain,
+                              const Value *OrigArg,
+                              unsigned OffsetFromOrigArg,
+                              unsigned ArgOffset,
+                              bool ForceMutable = false)
       const;
 
     void computeRegArea(CCState &CCInfo, MachineFunction &MF,
@@ -472,7 +481,7 @@ namespace llvm {
                 SmallVectorImpl<SDValue> &InVals) const;
 
     /// HandleByVal - Target-specific cleanup for ByVal support.
-    virtual void HandleByVal(CCState *, unsigned &) const;
+    virtual void HandleByVal(CCState *, unsigned &, unsigned) const;
 
     /// IsEligibleForTailCallOptimization - Check whether the call is eligible
     /// for tail call optimization. Targets which want to do tail call
diff --git a/lib/Target/ARM/ARMInstrFormats.td b/lib/Target/ARM/ARMInstrFormats.td
index c8966fb..67a6820 100644
--- a/lib/Target/ARM/ARMInstrFormats.td
+++ b/lib/Target/ARM/ARMInstrFormats.td
@@ -846,6 +846,23 @@ class AMiscA1I<bits<8> opcod, bits<4> opc7_4, dag oops, dag iops,
   let Inst{3-0}   = Rm;
 }
 
+// Division instructions.
+class ADivA1I<bits<3> opcod, dag oops, dag iops,
+              InstrItinClass itin, string opc, string asm, list<dag> pattern>
+  : I<oops, iops, AddrModeNone, 4, IndexModeNone, ArithMiscFrm, itin,
+      opc, asm, "", pattern> {
+  bits<4> Rd;
+  bits<4> Rn;
+  bits<4> Rm;
+  let Inst{27-23} = 0b01110;
+  let Inst{22-20} = opcod;
+  let Inst{19-16} = Rd;
+  let Inst{15-12} = 0b1111;
+  let Inst{11-8}  = Rm;
+  let Inst{7-4}   = 0b0001;
+  let Inst{3-0}   = Rn;
+}
+
 // PKH instructions
 def PKHLSLAsmOperand : ImmAsmOperand {
   let Name = "PKHLSLImm";
@@ -893,6 +910,10 @@ class ARMV5TPat<dag pattern, dag result> : Pat<pattern, result> {
 class ARMV5TEPat<dag pattern, dag result> : Pat<pattern, result> {
   list<Predicate> Predicates = [IsARM, HasV5TE];
 }
+// ARMV5MOPat - Same as ARMV5TEPat with UseMulOps.
+class ARMV5MOPat<dag pattern, dag result> : Pat<pattern, result> {
+  list<Predicate> Predicates = [IsARM, HasV5TE, UseMulOps];
+}
 class ARMV6Pat<dag pattern, dag result> : Pat<pattern, result> {
   list<Predicate> Predicates = [IsARM, HasV6];
 }
diff --git a/lib/Target/ARM/ARMInstrInfo.cpp b/lib/Target/ARM/ARMInstrInfo.cpp
index 31b0c41..a0b6f24 100644
--- a/lib/Target/ARM/ARMInstrInfo.cpp
+++ b/lib/Target/ARM/ARMInstrInfo.cpp
@@ -13,13 +13,17 @@
 
 #include "ARMInstrInfo.h"
 #include "ARM.h"
+#include "ARMConstantPoolValue.h"
 #include "ARMMachineFunctionInfo.h"
+#include "ARMTargetMachine.h"
 #include "MCTargetDesc/ARMAddressingModes.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/CodeGen/LiveVariables.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineJumpTableInfo.h"
+#include "llvm/Function.h"
+#include "llvm/GlobalVariable.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCInst.h"
 using namespace llvm;
@@ -84,3 +88,61 @@ unsigned ARMInstrInfo::getUnindexedOpcode(unsigned Opc) const {
 
   return 0;
 }
+
+namespace {
+  /// ARMCGBR - Create Global Base Reg pass. This initializes the PIC
+  /// global base register for ARM ELF.
+  struct ARMCGBR : public MachineFunctionPass {
+    static char ID;
+    ARMCGBR() : MachineFunctionPass(ID) {}
+
+    virtual bool runOnMachineFunction(MachineFunction &MF) {
+      ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>();
+      if (AFI->getGlobalBaseReg() == 0)
+        return false;
+
+      const ARMTargetMachine *TM =
+        static_cast<const ARMTargetMachine *>(&MF.getTarget());
+      if (TM->getRelocationModel() != Reloc::PIC_)
+        return false;
+
+      LLVMContext* Context = &MF.getFunction()->getContext();
+      GlobalValue *GV = new GlobalVariable(Type::getInt32Ty(*Context), false,
+                                           GlobalValue::ExternalLinkage, 0,
+                                           "_GLOBAL_OFFSET_TABLE_");
+      unsigned Id = AFI->createPICLabelUId();
+      ARMConstantPoolValue *CPV = ARMConstantPoolConstant::Create(GV, Id);
+      unsigned Align = TM->getDataLayout()->getPrefTypeAlignment(GV->getType());
+      unsigned Idx = MF.getConstantPool()->getConstantPoolIndex(CPV, Align);
+
+      MachineBasicBlock &FirstMBB = MF.front();
+      MachineBasicBlock::iterator MBBI = FirstMBB.begin();
+      DebugLoc DL = FirstMBB.findDebugLoc(MBBI);
+      unsigned GlobalBaseReg = AFI->getGlobalBaseReg();
+      unsigned Opc = TM->getSubtarget<ARMSubtarget>().isThumb2() ?
+                     ARM::t2LDRpci : ARM::LDRcp;
+      const TargetInstrInfo &TII = *TM->getInstrInfo();
+      MachineInstrBuilder MIB = BuildMI(FirstMBB, MBBI, DL,
+                                        TII.get(Opc), GlobalBaseReg)
+                                .addConstantPoolIndex(Idx);
+      if (Opc == ARM::LDRcp)
+        MIB.addImm(0);
+      AddDefaultPred(MIB);
+
+      return true;
+    }
+
+    virtual const char *getPassName() const {
+      return "ARM PIC Global Base Reg Initialization";
+    }
+
+    virtual void getAnalysisUsage(AnalysisUsage &AU) const {
+      AU.setPreservesCFG();
+      MachineFunctionPass::getAnalysisUsage(AU);
+    }
+  };
+}
+
+char ARMCGBR::ID = 0;
+FunctionPass*
+llvm::createARMGlobalBaseRegPass() { return new ARMCGBR(); }
diff --git a/lib/Target/ARM/ARMInstrInfo.td b/lib/Target/ARM/ARMInstrInfo.td
index 992aba5..df2e55e 100644
--- a/lib/Target/ARM/ARMInstrInfo.td
+++ b/lib/Target/ARM/ARMInstrInfo.td
@@ -83,6 +83,13 @@ def SDTBinaryArithWithFlagsInOut : SDTypeProfile<2, 3,
                                              SDTCisInt<0>,
                                              SDTCisVT<1, i32>,
                                              SDTCisVT<4, i32>]>;
+
+def SDT_ARM64bitmlal : SDTypeProfile<2,4, [ SDTCisVT<0, i32>, SDTCisVT<1, i32>,
+                                        SDTCisVT<2, i32>, SDTCisVT<3, i32>,
+                                        SDTCisVT<4, i32>, SDTCisVT<5, i32> ] >;
+def ARMUmlal         : SDNode<"ARMISD::UMLAL", SDT_ARM64bitmlal>;
+def ARMSmlal         : SDNode<"ARMISD::SMLAL", SDT_ARM64bitmlal>;
+
 // Node definitions.
 def ARMWrapper       : SDNode<"ARMISD::Wrapper",     SDTIntUnaryOp>;
 def ARMWrapperDYN    : SDNode<"ARMISD::WrapperDYN",  SDTIntUnaryOp>;
@@ -90,9 +97,10 @@ def ARMWrapperPIC    : SDNode<"ARMISD::WrapperPIC",  SDTIntUnaryOp>;
 def ARMWrapperJT     : SDNode<"ARMISD::WrapperJT",   SDTIntBinOp>;
 
 def ARMcallseq_start : SDNode<"ISD::CALLSEQ_START", SDT_ARMCallSeqStart,
-                              [SDNPHasChain, SDNPOutGlue]>;
+                              [SDNPHasChain, SDNPSideEffect, SDNPOutGlue]>;
 def ARMcallseq_end   : SDNode<"ISD::CALLSEQ_END",   SDT_ARMCallSeqEnd,
-                              [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue]>;
+                              [SDNPHasChain, SDNPSideEffect,
+                               SDNPOptInGlue, SDNPOutGlue]>;
 def ARMcopystructbyval : SDNode<"ARMISD::COPY_STRUCT_BYVAL" ,
                                 SDT_ARMStructByVal,
                                 [SDNPHasChain, SDNPInGlue, SDNPOutGlue,
@@ -148,14 +156,16 @@ def ARMsube          : SDNode<"ARMISD::SUBE",  SDTBinaryArithWithFlagsInOut>;
 
 def ARMthread_pointer: SDNode<"ARMISD::THREAD_POINTER", SDT_ARMThreadPointer>;
 def ARMeh_sjlj_setjmp: SDNode<"ARMISD::EH_SJLJ_SETJMP",
-                               SDT_ARMEH_SJLJ_Setjmp, [SDNPHasChain]>;
+                               SDT_ARMEH_SJLJ_Setjmp,
+                               [SDNPHasChain, SDNPSideEffect]>;
 def ARMeh_sjlj_longjmp: SDNode<"ARMISD::EH_SJLJ_LONGJMP",
-                               SDT_ARMEH_SJLJ_Longjmp, [SDNPHasChain]>;
+                               SDT_ARMEH_SJLJ_Longjmp,
+                               [SDNPHasChain, SDNPSideEffect]>;
 
 def ARMMemBarrier     : SDNode<"ARMISD::MEMBARRIER", SDT_ARMMEMBARRIER,
-                               [SDNPHasChain]>;
+                               [SDNPHasChain, SDNPSideEffect]>;
 def ARMMemBarrierMCR  : SDNode<"ARMISD::MEMBARRIER_MCR", SDT_ARMMEMBARRIER,
-                               [SDNPHasChain]>;
+                               [SDNPHasChain, SDNPSideEffect]>;
 def ARMPreload        : SDNode<"ARMISD::PRELOAD", SDT_ARMPREFETCH,
                                [SDNPHasChain, SDNPMayLoad, SDNPMayStore]>;
 
@@ -197,6 +207,8 @@ def HasFP16          : Predicate<"Subtarget->hasFP16()">,
                                  AssemblerPredicate<"FeatureFP16","half-float">;
 def HasDivide        : Predicate<"Subtarget->hasDivide()">,
                                  AssemblerPredicate<"FeatureHWDiv", "divide">;
+def HasDivideInARM   : Predicate<"Subtarget->hasDivideInARMMode()">,
+                                 AssemblerPredicate<"FeatureHWDivARM">;
 def HasT2ExtractPack : Predicate<"Subtarget->hasT2ExtractPack()">,
                                  AssemblerPredicate<"FeatureT2XtPk",
                                                      "pack/extract">;
@@ -232,6 +244,7 @@ def IsNaCl           : Predicate<"Subtarget->isTargetNaCl()">;
 def UseMovt          : Predicate<"Subtarget->useMovt()">;
 def DontUseMovt      : Predicate<"!Subtarget->useMovt()">;
 def UseFPVMLx        : Predicate<"Subtarget->useFPVMLx()">;
+def UseMulOps        : Predicate<"Subtarget->useMulOps()">;
 
 // Prefer fused MAC for fp mul + add over fp VMLA / VMLS if they are available.
 // But only select them if more precision in FP computation is allowed.
@@ -242,6 +255,20 @@ def UseFusedMAC      : Predicate<"(TM.Options.AllowFPOpFusion =="
 def DontUseFusedMAC  : Predicate<"!Subtarget->hasVFP4() || "
                                  "Subtarget->isTargetDarwin()">;
 
+// VGETLNi32 is microcoded on Swift - prefer VMOV.
+def HasFastVGETLNi32 : Predicate<"!Subtarget->isSwift()">;
+def HasSlowVGETLNi32 : Predicate<"Subtarget->isSwift()">;
+
+// VDUP.32 is microcoded on Swift - prefer VMOV.
+def HasFastVDUP32 : Predicate<"!Subtarget->isSwift()">;
+def HasSlowVDUP32 : Predicate<"Subtarget->isSwift()">;
+
+// Cortex-A9 prefers VMOVSR to VMOVDRR even when using NEON for scalar FP, as
+// this allows more effective execution domain optimization. See
+// setExecutionDomain().
+def UseVMOVSR : Predicate<"Subtarget->isCortexA9() || !Subtarget->useNEONForSinglePrecisionFP()">;
+def DontUseVMOVSR : Predicate<"!Subtarget->isCortexA9() && Subtarget->useNEONForSinglePrecisionFP()">;
+
 def IsLE             : Predicate<"TLI.isLittleEndian()">;
 def IsBE             : Predicate<"TLI.isBigEndian()">;
 
@@ -256,15 +283,13 @@ class RegConstraint<string C> {
 //  ARM specific transformation functions and pattern fragments.
 //
 
-// imm_neg_XFORM - Return a imm value packed into the format described for
-// imm_neg defs below.
+// imm_neg_XFORM - Return the negation of an i32 immediate value.
 def imm_neg_XFORM : SDNodeXForm<imm, [{
   return CurDAG->getTargetConstant(-(int)N->getZExtValue(), MVT::i32);
 }]>;
 
-// so_imm_not_XFORM - Return a so_imm value packed into the format described for
-// so_imm_not def below.
-def so_imm_not_XFORM : SDNodeXForm<imm, [{
+// imm_not_XFORM - Return the complement of a i32 immediate value.
+def imm_not_XFORM : SDNodeXForm<imm, [{
   return CurDAG->getTargetConstant(~(int)N->getZExtValue(), MVT::i32);
 }]>;
 
@@ -275,7 +300,7 @@ def imm16_31 : ImmLeaf<i32, [{
 
 def so_imm_neg_asmoperand : AsmOperandClass { let Name = "ARMSOImmNeg"; }
 def so_imm_neg : Operand<i32>, PatLeaf<(imm), [{
-    int64_t Value = -(int)N->getZExtValue();
+    unsigned Value = -(unsigned)N->getZExtValue();
     return Value && ARM_AM::getSOImmVal(Value) != -1;
   }], imm_neg_XFORM> {
   let ParserMatchClass = so_imm_neg_asmoperand;
@@ -287,7 +312,7 @@ def so_imm_neg : Operand<i32>, PatLeaf<(imm), [{
 def so_imm_not_asmoperand : AsmOperandClass { let Name = "ARMSOImmNot"; }
 def so_imm_not : Operand<i32>, PatLeaf<(imm), [{
     return ARM_AM::getSOImmVal(~(uint32_t)N->getZExtValue()) != -1;
-  }], so_imm_not_XFORM> {
+  }], imm_not_XFORM> {
   let ParserMatchClass = so_imm_not_asmoperand;
 }
 
@@ -1791,12 +1816,15 @@ def ADR : AI1<{0,?,?,0}, (outs GPR:$Rd), (ins adrlabel:$label),
   let Inst{15-12} = Rd;
   let Inst{11-0} = label{11-0};
 }
+
+let hasSideEffects = 1 in {
 def LEApcrel : ARMPseudoInst<(outs GPR:$Rd), (ins i32imm:$label, pred:$p),
                     4, IIC_iALUi, []>;
 
 def LEApcrelJT : ARMPseudoInst<(outs GPR:$Rd),
                       (ins i32imm:$label, nohash_imm:$id, pred:$p),
                       4, IIC_iALUi, []>;
+}
 
 //===----------------------------------------------------------------------===//
 //  Control Flow Instructions.
@@ -3079,15 +3107,19 @@ def : ARMPat<(ARMaddc GPR:$src, so_imm_neg:$imm),
              (SUBSri  GPR:$src, so_imm_neg:$imm)>;
 
 def : ARMPat<(add     GPR:$src, imm0_65535_neg:$imm),
-             (SUBrr   GPR:$src, (MOVi16 (imm_neg_XFORM imm:$imm)))>;
+             (SUBrr   GPR:$src, (MOVi16 (imm_neg_XFORM imm:$imm)))>,
+             Requires<[IsARM, HasV6T2]>;
 def : ARMPat<(ARMaddc GPR:$src, imm0_65535_neg:$imm),
-             (SUBSrr  GPR:$src, (MOVi16 (imm_neg_XFORM imm:$imm)))>;
+             (SUBSrr  GPR:$src, (MOVi16 (imm_neg_XFORM imm:$imm)))>,
+             Requires<[IsARM, HasV6T2]>;
 
 // The with-carry-in form matches bitwise not instead of the negation.
 // Effectively, the inverse interpretation of the carry flag already accounts
 // for part of the negation.
 def : ARMPat<(ARMadde GPR:$src, so_imm_not:$imm, CPSR),
              (SBCri   GPR:$src, so_imm_not:$imm)>;
+def : ARMPat<(ARMadde GPR:$src, imm0_65535_neg:$imm, CPSR),
+             (SBCrr   GPR:$src, (MOVi16 (imm_not_XFORM imm:$imm)))>;
 
 // Note: These are implemented in C++ code, because they have to generate
 // ADD/SUBrs instructions, which use a complex pattern that a xform function
@@ -3399,6 +3431,18 @@ class AsMul1I64<bits<7> opcod, dag oops, dag iops, InstrItinClass itin,
   let Inst{11-8}  = Rm;
   let Inst{3-0}   = Rn;
 }
+class AsMla1I64<bits<7> opcod, dag oops, dag iops, InstrItinClass itin,
+             string opc, string asm, list<dag> pattern>
+  : AsMul1I<opcod, oops, iops, itin, opc, asm, pattern> {
+  bits<4> RdLo;
+  bits<4> RdHi;
+  bits<4> Rm;
+  bits<4> Rn;
+  let Inst{19-16} = RdHi;
+  let Inst{15-12} = RdLo;
+  let Inst{11-8}  = Rm;
+  let Inst{3-0}   = Rn;
+}
 
 // FIXME: The v5 pseudos are only necessary for the additional Constraint
 //        property. Remove them when it's possible to add those properties
@@ -3419,13 +3463,13 @@ def MULv5: ARMPseudoExpand<(outs GPRnopc:$Rd), (ins GPRnopc:$Rn, GPRnopc:$Rm,
                            4, IIC_iMUL32,
                [(set GPRnopc:$Rd, (mul GPRnopc:$Rn, GPRnopc:$Rm))],
                (MUL GPRnopc:$Rd, GPRnopc:$Rn, GPRnopc:$Rm, pred:$p, cc_out:$s)>,
-               Requires<[IsARM, NoV6]>;
+               Requires<[IsARM, NoV6, UseMulOps]>;
 }
 
 def MLA  : AsMul1I32<0b0000001, (outs GPR:$Rd), (ins GPR:$Rn, GPR:$Rm, GPR:$Ra),
                      IIC_iMAC32, "mla", "\t$Rd, $Rn, $Rm, $Ra",
                    [(set GPR:$Rd, (add (mul GPR:$Rn, GPR:$Rm), GPR:$Ra))]>,
-                   Requires<[IsARM, HasV6]> {
+                   Requires<[IsARM, HasV6, UseMulOps]> {
   bits<4> Ra;
   let Inst{15-12} = Ra;
 }
@@ -3441,7 +3485,7 @@ def MLAv5: ARMPseudoExpand<(outs GPR:$Rd),
 def MLS  : AMul1I<0b0000011, (outs GPR:$Rd), (ins GPR:$Rn, GPR:$Rm, GPR:$Ra),
                    IIC_iMAC32, "mls", "\t$Rd, $Rn, $Rm, $Ra",
                    [(set GPR:$Rd, (sub GPR:$Ra, (mul GPR:$Rn, GPR:$Rm)))]>,
-                   Requires<[IsARM, HasV6T2]> {
+                   Requires<[IsARM, HasV6T2, UseMulOps]> {
   bits<4> Rd;
   bits<4> Rm;
   bits<4> Rn;
@@ -3481,14 +3525,14 @@ def UMULLv5 : ARMPseudoExpand<(outs GPR:$RdLo, GPR:$RdHi),
 }
 
 // Multiply + accumulate
-def SMLAL : AsMul1I64<0b0000111, (outs GPR:$RdLo, GPR:$RdHi),
-                               (ins GPR:$Rn, GPR:$Rm), IIC_iMAC64,
+def SMLAL : AsMla1I64<0b0000111, (outs GPR:$RdLo, GPR:$RdHi),
+                        (ins GPR:$Rn, GPR:$Rm, GPR:$RLo, GPR:$RHi), IIC_iMAC64,
                     "smlal", "\t$RdLo, $RdHi, $Rn, $Rm", []>,
-                    Requires<[IsARM, HasV6]>;
-def UMLAL : AsMul1I64<0b0000101, (outs GPR:$RdLo, GPR:$RdHi),
-                               (ins GPR:$Rn, GPR:$Rm), IIC_iMAC64,
+         RegConstraint<"$RLo = $RdLo, $RHi = $RdHi">, Requires<[IsARM, HasV6]>;
+def UMLAL : AsMla1I64<0b0000101, (outs GPR:$RdLo, GPR:$RdHi),
+                        (ins GPR:$Rn, GPR:$Rm, GPR:$RLo, GPR:$RHi), IIC_iMAC64,
                     "umlal", "\t$RdLo, $RdHi, $Rn, $Rm", []>,
-                    Requires<[IsARM, HasV6]>;
+         RegConstraint<"$RLo = $RdLo, $RHi = $RdHi">, Requires<[IsARM, HasV6]>;
 
 def UMAAL : AMul1I <0b0000010, (outs GPR:$RdLo, GPR:$RdHi),
                                (ins GPR:$Rn, GPR:$Rm), IIC_iMAC64,
@@ -3504,17 +3548,22 @@ def UMAAL : AMul1I <0b0000010, (outs GPR:$RdLo, GPR:$RdHi),
   let Inst{3-0}   = Rn;
 }
 
-let Constraints = "@earlyclobber $RdLo,@earlyclobber $RdHi" in {
+let Constraints = "$RLo = $RdLo,$RHi = $RdHi" in {
 def SMLALv5 : ARMPseudoExpand<(outs GPR:$RdLo, GPR:$RdHi),
-                              (ins GPR:$Rn, GPR:$Rm, pred:$p, cc_out:$s),
+                (ins GPR:$Rn, GPR:$Rm, GPR:$RLo, GPR:$RHi, pred:$p, cc_out:$s),
                               4, IIC_iMAC64, [],
-          (SMLAL GPR:$RdLo, GPR:$RdHi, GPR:$Rn, GPR:$Rm, pred:$p, cc_out:$s)>,
+             (SMLAL GPR:$RdLo, GPR:$RdHi, GPR:$Rn, GPR:$Rm, GPR:$RLo, GPR:$RHi,
+                           pred:$p, cc_out:$s)>,
                            Requires<[IsARM, NoV6]>;
 def UMLALv5 : ARMPseudoExpand<(outs GPR:$RdLo, GPR:$RdHi),
-                              (ins GPR:$Rn, GPR:$Rm, pred:$p, cc_out:$s),
+                (ins GPR:$Rn, GPR:$Rm, GPR:$RLo, GPR:$RHi, pred:$p, cc_out:$s),
                               4, IIC_iMAC64, [],
-          (UMLAL GPR:$RdLo, GPR:$RdHi, GPR:$Rn, GPR:$Rm, pred:$p, cc_out:$s)>,
+             (UMLAL GPR:$RdLo, GPR:$RdHi, GPR:$Rn, GPR:$Rm, GPR:$RLo, GPR:$RHi,
+                           pred:$p, cc_out:$s)>,
                            Requires<[IsARM, NoV6]>;
+}
+
+let Constraints = "@earlyclobber $RdLo,@earlyclobber $RdHi" in {
 def UMAALv5 : ARMPseudoExpand<(outs GPR:$RdLo, GPR:$RdHi),
                               (ins GPR:$Rn, GPR:$Rm, pred:$p),
                               4, IIC_iMAC64, [],
@@ -3542,7 +3591,7 @@ def SMMLA : AMul2Ia <0b0111010, 0b0001, (outs GPR:$Rd),
                (ins GPR:$Rn, GPR:$Rm, GPR:$Ra),
                IIC_iMAC32, "smmla", "\t$Rd, $Rn, $Rm, $Ra",
                [(set GPR:$Rd, (add (mulhs GPR:$Rn, GPR:$Rm), GPR:$Ra))]>,
-            Requires<[IsARM, HasV6]>;
+            Requires<[IsARM, HasV6, UseMulOps]>;
 
 def SMMLAR : AMul2Ia <0b0111010, 0b0011, (outs GPR:$Rd),
                (ins GPR:$Rn, GPR:$Rm, GPR:$Ra),
@@ -3552,7 +3601,7 @@ def SMMLAR : AMul2Ia <0b0111010, 0b0011, (outs GPR:$Rd),
 def SMMLS : AMul2Ia <0b0111010, 0b1101, (outs GPR:$Rd),
                (ins GPR:$Rn, GPR:$Rm, GPR:$Ra),
                IIC_iMAC32, "smmls", "\t$Rd, $Rn, $Rm, $Ra", []>,
-            Requires<[IsARM, HasV6]>;
+            Requires<[IsARM, HasV6, UseMulOps]>;
 
 def SMMLSR : AMul2Ia <0b0111010, 0b1111, (outs GPR:$Rd),
                (ins GPR:$Rn, GPR:$Rm, GPR:$Ra),
@@ -3606,7 +3655,7 @@ multiclass AI_smla<string opc, PatFrag opnode> {
               [(set GPRnopc:$Rd, (add GPR:$Ra,
                                (opnode (sext_inreg GPRnopc:$Rn, i16),
                                        (sext_inreg GPRnopc:$Rm, i16))))]>,
-           Requires<[IsARM, HasV5TE]>;
+           Requires<[IsARM, HasV5TE, UseMulOps]>;
 
   def BT : AMulxyIa<0b0001000, 0b10, (outs GPRnopc:$Rd),
               (ins GPRnopc:$Rn, GPRnopc:$Rm, GPR:$Ra),
@@ -3614,7 +3663,7 @@ multiclass AI_smla<string opc, PatFrag opnode> {
               [(set GPRnopc:$Rd,
                     (add GPR:$Ra, (opnode (sext_inreg GPRnopc:$Rn, i16),
                                           (sra GPRnopc:$Rm, (i32 16)))))]>,
-           Requires<[IsARM, HasV5TE]>;
+           Requires<[IsARM, HasV5TE, UseMulOps]>;
 
   def TB : AMulxyIa<0b0001000, 0b01, (outs GPRnopc:$Rd),
               (ins GPRnopc:$Rn, GPRnopc:$Rm, GPR:$Ra),
@@ -3622,7 +3671,7 @@ multiclass AI_smla<string opc, PatFrag opnode> {
               [(set GPRnopc:$Rd,
                     (add GPR:$Ra, (opnode (sra GPRnopc:$Rn, (i32 16)),
                                           (sext_inreg GPRnopc:$Rm, i16))))]>,
-           Requires<[IsARM, HasV5TE]>;
+           Requires<[IsARM, HasV5TE, UseMulOps]>;
 
   def TT : AMulxyIa<0b0001000, 0b11, (outs GPRnopc:$Rd),
               (ins GPRnopc:$Rn, GPRnopc:$Rm, GPR:$Ra),
@@ -3630,7 +3679,7 @@ multiclass AI_smla<string opc, PatFrag opnode> {
              [(set GPRnopc:$Rd,
                    (add GPR:$Ra, (opnode (sra GPRnopc:$Rn, (i32 16)),
                                          (sra GPRnopc:$Rm, (i32 16)))))]>,
-            Requires<[IsARM, HasV5TE]>;
+            Requires<[IsARM, HasV5TE, UseMulOps]>;
 
   def WB : AMulxyIa<0b0001001, 0b00, (outs GPRnopc:$Rd),
               (ins GPRnopc:$Rn, GPRnopc:$Rm, GPR:$Ra),
@@ -3638,7 +3687,7 @@ multiclass AI_smla<string opc, PatFrag opnode> {
               [(set GPRnopc:$Rd,
                     (add GPR:$Ra, (sra (opnode GPRnopc:$Rn,
                                   (sext_inreg GPRnopc:$Rm, i16)), (i32 16))))]>,
-           Requires<[IsARM, HasV5TE]>;
+           Requires<[IsARM, HasV5TE, UseMulOps]>;
 
   def WT : AMulxyIa<0b0001001, 0b10, (outs GPRnopc:$Rd),
               (ins GPRnopc:$Rn, GPRnopc:$Rm, GPR:$Ra),
@@ -3646,7 +3695,7 @@ multiclass AI_smla<string opc, PatFrag opnode> {
               [(set GPRnopc:$Rd,
                  (add GPR:$Ra, (sra (opnode GPRnopc:$Rn,
                                     (sra GPRnopc:$Rm, (i32 16))), (i32 16))))]>,
-            Requires<[IsARM, HasV5TE]>;
+            Requires<[IsARM, HasV5TE, UseMulOps]>;
   }
 }
 
@@ -3749,6 +3798,19 @@ defm SMUA : AI_sdml<0, "smua">;
 defm SMUS : AI_sdml<1, "smus">;
 
 //===----------------------------------------------------------------------===//
+//  Division Instructions (ARMv7-A with virtualization extension)
+//
+def SDIV : ADivA1I<0b001, (outs GPR:$Rd), (ins GPR:$Rn, GPR:$Rm), IIC_iDIV,
+                   "sdiv", "\t$Rd, $Rn, $Rm",
+                   [(set GPR:$Rd, (sdiv GPR:$Rn, GPR:$Rm))]>,
+           Requires<[IsARM, HasDivideInARM]>;
+
+def UDIV : ADivA1I<0b011, (outs GPR:$Rd), (ins GPR:$Rn, GPR:$Rm), IIC_iDIV,
+                   "udiv", "\t$Rd, $Rn, $Rm",
+                   [(set GPR:$Rd, (udiv GPR:$Rn, GPR:$Rm))]>,
+           Requires<[IsARM, HasDivideInARM]>;
+
+//===----------------------------------------------------------------------===//
 //  Misc. Arithmetic Instructions.
 //
 
@@ -3986,48 +4048,6 @@ def MVNCCi : ARMPseudoInst<(outs GPR:$Rd),
  [/*(set GPR:$Rd, (ARMcmov GPR:$false, so_imm_not:$imm, imm:$cc, CCR:$ccr))*/]>,
                 RegConstraint<"$false = $Rd">;
 
-// Conditional instructions
-multiclass AsI1_bincc_irs<Instruction iri, Instruction irr, Instruction irsi,
-                          Instruction irsr,
-                          InstrItinClass iii, InstrItinClass iir,
-                          InstrItinClass iis> {
-  def ri  : ARMPseudoExpand<(outs GPR:$Rd),
-                            (ins GPR:$Rfalse, GPR:$Rn, so_imm:$imm,
-                                 pred:$p, cc_out:$s),
-                            4, iii, [],
-                       (iri GPR:$Rd, GPR:$Rn, so_imm:$imm, pred:$p, cc_out:$s)>,
-                            RegConstraint<"$Rfalse = $Rd">;
-  def rr  : ARMPseudoExpand<(outs GPR:$Rd),
-                            (ins GPR:$Rfalse, GPR:$Rn, GPR:$Rm,
-                                 pred:$p, cc_out:$s),
-                            4, iir, [],
-                           (irr GPR:$Rd, GPR:$Rn, GPR:$Rm, pred:$p, cc_out:$s)>,
-                            RegConstraint<"$Rfalse = $Rd">;
-  def rsi : ARMPseudoExpand<(outs GPR:$Rd),
-                            (ins GPR:$Rfalse, GPR:$Rn, so_reg_imm:$shift,
-                                 pred:$p, cc_out:$s),
-                            4, iis, [],
-                (irsi GPR:$Rd, GPR:$Rn, so_reg_imm:$shift, pred:$p, cc_out:$s)>,
-                            RegConstraint<"$Rfalse = $Rd">;
-  def rsr : ARMPseudoExpand<(outs GPRnopc:$Rd),
-                           (ins GPRnopc:$Rfalse, GPRnopc:$Rn, so_reg_reg:$shift,
-                                pred:$p, cc_out:$s),
-                            4, iis, [],
-                (irsr GPR:$Rd, GPR:$Rn, so_reg_reg:$shift, pred:$p, cc_out:$s)>,
-                            RegConstraint<"$Rfalse = $Rd">;
-}
-
-defm ANDCC : AsI1_bincc_irs<ANDri, ANDrr, ANDrsi, ANDrsr,
-                            IIC_iBITi, IIC_iBITr, IIC_iBITsr>;
-defm ORRCC : AsI1_bincc_irs<ORRri, ORRrr, ORRrsi, ORRrsr,
-                            IIC_iBITi, IIC_iBITr, IIC_iBITsr>;
-defm EORCC : AsI1_bincc_irs<EORri, EORrr, EORrsi, EORrsr,
-                            IIC_iBITi, IIC_iBITr, IIC_iBITsr>;
-defm ADDCC : AsI1_bincc_irs<ADDri, ADDrr, ADDrsi, ADDrsr,
-                            IIC_iBITi, IIC_iBITr, IIC_iBITsr>;
-defm SUBCC : AsI1_bincc_irs<SUBri, SUBrr, SUBrsi, SUBrsr,
-                            IIC_iBITi, IIC_iBITr, IIC_iBITsr>;
-
 } // neverHasSideEffects
 
 
@@ -4723,21 +4743,13 @@ def Int_eh_sjlj_longjmp : PseudoInst<(outs), (ins GPR:$src, GPR:$scratch),
                                 Requires<[IsARM, IsIOS]>;
 }
 
-// eh.sjlj.dispatchsetup pseudo-instructions.
-// These pseudos are used for both ARM and Thumb2. Any differences are
-// handled when the pseudo is expanded (which happens before any passes
-// that need the instruction size).
-let Defs =
-  [ R0,  R1,  R2,  R3,  R4,  R5,  R6,  R7,  R8,  R9,  R10, R11, R12, LR, CPSR,
-    Q0, Q1, Q2, Q3, Q4, Q5, Q6, Q7, Q8, Q9, Q10, Q11, Q12, Q13, Q14, Q15 ],
-  isBarrier = 1 in
+// eh.sjlj.dispatchsetup pseudo-instruction.
+// This pseudo is used for both ARM and Thumb. Any differences are handled when
+// the pseudo is expanded (which happens before any passes that need the
+// instruction size).
+let isBarrier = 1 in
 def Int_eh_sjlj_dispatchsetup : PseudoInst<(outs), (ins), NoItinerary, []>;
 
-let Defs =
-  [ R0,  R1,  R2,  R3,  R4,  R5,  R6,  R7,  R8,  R9,  R10, R11, R12, LR, CPSR ],
-  isBarrier = 1 in
-def Int_eh_sjlj_dispatchsetup_nofp : PseudoInst<(outs), (ins), NoItinerary, []>;
-
 
 //===----------------------------------------------------------------------===//
 // Non-Instruction Patterns
@@ -4841,32 +4853,32 @@ def : ARMV5TEPat<(sra (mul GPR:$a, (sra (shl GPR:$b, (i32 16)), (i32 16))),
 def : ARMV5TEPat<(sra (mul GPR:$a, sext_16_node:$b), (i32 16)),
                  (SMULWB GPR:$a, GPR:$b)>;
 
-def : ARMV5TEPat<(add GPR:$acc,
+def : ARMV5MOPat<(add GPR:$acc,
                       (mul (sra (shl GPR:$a, (i32 16)), (i32 16)),
                            (sra (shl GPR:$b, (i32 16)), (i32 16)))),
                  (SMLABB GPR:$a, GPR:$b, GPR:$acc)>;
-def : ARMV5TEPat<(add GPR:$acc,
+def : ARMV5MOPat<(add GPR:$acc,
                       (mul sext_16_node:$a, sext_16_node:$b)),
                  (SMLABB GPR:$a, GPR:$b, GPR:$acc)>;
-def : ARMV5TEPat<(add GPR:$acc,
+def : ARMV5MOPat<(add GPR:$acc,
                       (mul (sra (shl GPR:$a, (i32 16)), (i32 16)),
                            (sra GPR:$b, (i32 16)))),
                  (SMLABT GPR:$a, GPR:$b, GPR:$acc)>;
-def : ARMV5TEPat<(add GPR:$acc,
+def : ARMV5MOPat<(add GPR:$acc,
                       (mul sext_16_node:$a, (sra GPR:$b, (i32 16)))),
                  (SMLABT GPR:$a, GPR:$b, GPR:$acc)>;
-def : ARMV5TEPat<(add GPR:$acc,
+def : ARMV5MOPat<(add GPR:$acc,
                       (mul (sra GPR:$a, (i32 16)),
                            (sra (shl GPR:$b, (i32 16)), (i32 16)))),
                  (SMLATB GPR:$a, GPR:$b, GPR:$acc)>;
-def : ARMV5TEPat<(add GPR:$acc,
+def : ARMV5MOPat<(add GPR:$acc,
                       (mul (sra GPR:$a, (i32 16)), sext_16_node:$b)),
                  (SMLATB GPR:$a, GPR:$b, GPR:$acc)>;
-def : ARMV5TEPat<(add GPR:$acc,
+def : ARMV5MOPat<(add GPR:$acc,
                       (sra (mul GPR:$a, (sra (shl GPR:$b, (i32 16)), (i32 16))),
                            (i32 16))),
                  (SMLAWB GPR:$a, GPR:$b, GPR:$acc)>;
-def : ARMV5TEPat<(add GPR:$acc,
+def : ARMV5MOPat<(add GPR:$acc,
                       (sra (mul GPR:$a, sext_16_node:$b), (i32 16))),
                  (SMLAWB GPR:$a, GPR:$b, GPR:$acc)>;
 
diff --git a/lib/Target/ARM/ARMInstrNEON.td b/lib/Target/ARM/ARMInstrNEON.td
index 048d340..3cf213c 100644
--- a/lib/Target/ARM/ARMInstrNEON.td
+++ b/lib/Target/ARM/ARMInstrNEON.td
@@ -398,6 +398,20 @@ def VecListFourQWordIndexed : Operand<i32> {
   let MIOperandInfo = (ops DPR:$Vd, i32imm:$idx);
 }
 
+def dword_alignedload : PatFrag<(ops node:$ptr), (load node:$ptr), [{
+  return cast<LoadSDNode>(N)->getAlignment() >= 8;
+}]>;
+def dword_alignedstore : PatFrag<(ops node:$val, node:$ptr),
+                                 (store node:$val, node:$ptr), [{
+  return cast<StoreSDNode>(N)->getAlignment() >= 8;
+}]>;
+def word_alignedload : PatFrag<(ops node:$ptr), (load node:$ptr), [{
+  return cast<LoadSDNode>(N)->getAlignment() == 4;
+}]>;
+def word_alignedstore : PatFrag<(ops node:$val, node:$ptr),
+                                 (store node:$val, node:$ptr), [{
+  return cast<StoreSDNode>(N)->getAlignment() == 4;
+}]>;
 def hword_alignedload : PatFrag<(ops node:$ptr), (load node:$ptr), [{
   return cast<LoadSDNode>(N)->getAlignment() == 2;
 }]>;
@@ -1980,7 +1994,7 @@ def VST1LNd8  : VST1LN<0b0000, {?,?,?,0}, "8", v8i8, truncstorei8,
 def VST1LNd16 : VST1LN<0b0100, {?,?,0,?}, "16", v4i16, truncstorei16,
                        NEONvgetlaneu, addrmode6> {
   let Inst{7-6} = lane{1-0};
-  let Inst{4}   = Rn{5};
+  let Inst{4}   = Rn{4};
 }
 
 def VST1LNd32 : VST1LN<0b1000, {?,0,?,?}, "32", v2i32, store, extractelt,
@@ -2023,7 +2037,7 @@ def VST1LNd8_UPD  : VST1LNWB<0b0000, {?,?,?,0}, "8", v8i8, post_truncsti8,
 def VST1LNd16_UPD : VST1LNWB<0b0100, {?,?,0,?}, "16", v4i16, post_truncsti16,
                              NEONvgetlaneu, addrmode6> {
   let Inst{7-6} = lane{1-0};
-  let Inst{4}   = Rn{5};
+  let Inst{4}   = Rn{4};
 }
 def VST1LNd32_UPD : VST1LNWB<0b1000, {?,0,?,?}, "32", v2i32, post_store,
                              extractelt, addrmode6oneL32> {
@@ -2273,6 +2287,25 @@ def : Pat<(f64 (non_word_alignedload addrmode6:$addr)),
 def : Pat<(non_word_alignedstore (f64 DPR:$value), addrmode6:$addr),
           (VST1d64 addrmode6:$addr, DPR:$value)>, Requires<[IsBE]>;
 
+// Use vld1/vst1 for Q and QQ. Also use them for unaligned v2f64
+// load / store if it's legal.
+def : Pat<(v2f64 (dword_alignedload addrmode6:$addr)),
+          (VLD1q64 addrmode6:$addr)>;
+def : Pat<(dword_alignedstore (v2f64 QPR:$value), addrmode6:$addr),
+          (VST1q64 addrmode6:$addr, QPR:$value)>;
+def : Pat<(v2f64 (word_alignedload addrmode6:$addr)),
+          (VLD1q32 addrmode6:$addr)>;
+def : Pat<(word_alignedstore (v2f64 QPR:$value), addrmode6:$addr),
+          (VST1q32 addrmode6:$addr, QPR:$value)>;
+def : Pat<(v2f64 (hword_alignedload addrmode6:$addr)),
+          (VLD1q16 addrmode6:$addr)>, Requires<[IsLE]>;
+def : Pat<(hword_alignedstore (v2f64 QPR:$value), addrmode6:$addr),
+          (VST1q16 addrmode6:$addr, QPR:$value)>, Requires<[IsLE]>;
+def : Pat<(v2f64 (byte_alignedload addrmode6:$addr)),
+          (VLD1q8 addrmode6:$addr)>, Requires<[IsLE]>;
+def : Pat<(byte_alignedstore (v2f64 QPR:$value), addrmode6:$addr),
+          (VST1q8 addrmode6:$addr, QPR:$value)>, Requires<[IsLE]>;
+
 //===----------------------------------------------------------------------===//
 // NEON pattern fragments
 //===----------------------------------------------------------------------===//
@@ -4455,10 +4488,36 @@ def  VBSLd    : N3VX<1, 0, 0b01, 0b0001, 0, 1, (outs DPR:$Vd),
                      "vbsl", "$Vd, $Vn, $Vm", "$src1 = $Vd",
                      [(set DPR:$Vd,
                            (v2i32 (NEONvbsl DPR:$src1, DPR:$Vn, DPR:$Vm)))]>;
+def : Pat<(v8i8 (int_arm_neon_vbsl (v8i8 DPR:$src1),
+                                   (v8i8 DPR:$Vn), (v8i8 DPR:$Vm))),
+          (VBSLd DPR:$src1, DPR:$Vn, DPR:$Vm)>,
+        Requires<[HasNEON]>;
+def : Pat<(v4i16 (int_arm_neon_vbsl (v4i16 DPR:$src1),
+                                    (v4i16 DPR:$Vn), (v4i16 DPR:$Vm))),
+          (VBSLd DPR:$src1, DPR:$Vn, DPR:$Vm)>,
+        Requires<[HasNEON]>;
+def : Pat<(v2i32 (int_arm_neon_vbsl (v2i32 DPR:$src1),
+                                    (v2i32 DPR:$Vn), (v2i32 DPR:$Vm))),
+          (VBSLd DPR:$src1, DPR:$Vn, DPR:$Vm)>,
+        Requires<[HasNEON]>;
+def : Pat<(v2f32 (int_arm_neon_vbsl (v2f32 DPR:$src1),
+                                    (v2f32 DPR:$Vn), (v2f32 DPR:$Vm))),
+          (VBSLd DPR:$src1, DPR:$Vn, DPR:$Vm)>,
+        Requires<[HasNEON]>;
+def : Pat<(v1i64 (int_arm_neon_vbsl (v1i64 DPR:$src1),
+                                    (v1i64 DPR:$Vn), (v1i64 DPR:$Vm))),
+          (VBSLd DPR:$src1, DPR:$Vn, DPR:$Vm)>,
+        Requires<[HasNEON]>;
 
 def : Pat<(v2i32 (or (and DPR:$Vn, DPR:$Vd),
                      (and DPR:$Vm, (vnotd DPR:$Vd)))),
-          (VBSLd DPR:$Vd, DPR:$Vn, DPR:$Vm)>;
+          (VBSLd DPR:$Vd, DPR:$Vn, DPR:$Vm)>,
+        Requires<[HasNEON]>;
+
+def : Pat<(v1i64 (or (and DPR:$Vn, DPR:$Vd),
+                     (and DPR:$Vm, (vnotd DPR:$Vd)))),
+          (VBSLd DPR:$Vd, DPR:$Vn, DPR:$Vm)>,
+        Requires<[HasNEON]>;
 
 def  VBSLq    : N3VX<1, 0, 0b01, 0b0001, 1, 1, (outs QPR:$Vd),
                      (ins QPR:$src1, QPR:$Vn, QPR:$Vm),
@@ -4467,9 +4526,35 @@ def  VBSLq    : N3VX<1, 0, 0b01, 0b0001, 1, 1, (outs QPR:$Vd),
                      [(set QPR:$Vd,
                            (v4i32 (NEONvbsl QPR:$src1, QPR:$Vn, QPR:$Vm)))]>;
 
+def : Pat<(v16i8 (int_arm_neon_vbsl (v16i8 QPR:$src1),
+                                   (v16i8 QPR:$Vn), (v16i8 QPR:$Vm))),
+          (VBSLq QPR:$src1, QPR:$Vn, QPR:$Vm)>,
+        Requires<[HasNEON]>;
+def : Pat<(v8i16 (int_arm_neon_vbsl (v8i16 QPR:$src1),
+                                    (v8i16 QPR:$Vn), (v8i16 QPR:$Vm))),
+          (VBSLq QPR:$src1, QPR:$Vn, QPR:$Vm)>,
+        Requires<[HasNEON]>;
+def : Pat<(v4i32 (int_arm_neon_vbsl (v4i32 QPR:$src1),
+                                    (v4i32 QPR:$Vn), (v4i32 QPR:$Vm))),
+          (VBSLq QPR:$src1, QPR:$Vn, QPR:$Vm)>,
+        Requires<[HasNEON]>;
+def : Pat<(v4f32 (int_arm_neon_vbsl (v4f32 QPR:$src1),
+                                    (v4f32 QPR:$Vn), (v4f32 QPR:$Vm))),
+          (VBSLq QPR:$src1, QPR:$Vn, QPR:$Vm)>,
+        Requires<[HasNEON]>;
+def : Pat<(v2i64 (int_arm_neon_vbsl (v2i64 QPR:$src1),
+                                    (v2i64 QPR:$Vn), (v2i64 QPR:$Vm))),
+          (VBSLq QPR:$src1, QPR:$Vn, QPR:$Vm)>,
+        Requires<[HasNEON]>;
+
 def : Pat<(v4i32 (or (and QPR:$Vn, QPR:$Vd),
                      (and QPR:$Vm, (vnotq QPR:$Vd)))),
-          (VBSLq QPR:$Vd, QPR:$Vn, QPR:$Vm)>;
+          (VBSLq QPR:$Vd, QPR:$Vn, QPR:$Vm)>,
+        Requires<[HasNEON]>;
+def : Pat<(v2i64 (or (and QPR:$Vn, QPR:$Vd),
+                     (and QPR:$Vm, (vnotq QPR:$Vd)))),
+          (VBSLq QPR:$Vd, QPR:$Vn, QPR:$Vm)>,
+        Requires<[HasNEON]>;
 
 //   VBIF     : Vector Bitwise Insert if False
 //              like VBSL but with: "vbif $dst, $src3, $src1", "$src2 = $dst",
@@ -4983,7 +5068,8 @@ def VGETLNi32 : NVGetLane<{1,1,1,0,0,0,?,1}, 0b1011, 0b00,
                           (outs GPR:$R), (ins DPR:$V, VectorIndex32:$lane),
                           IIC_VMOVSI, "vmov", "32", "$R, $V$lane",
                           [(set GPR:$R, (extractelt (v2i32 DPR:$V),
-                                           imm:$lane))]> {
+                                           imm:$lane))]>,
+                Requires<[HasNEON, HasFastVGETLNi32]> {
   let Inst{21} = lane{0};
 }
 // def VGETLNf32: see FMRDH and FMRDL in ARMInstrVFP.td
@@ -5006,7 +5092,16 @@ def : Pat<(NEONvgetlaneu (v8i16 QPR:$src), imm:$lane),
 def : Pat<(extractelt (v4i32 QPR:$src), imm:$lane),
           (VGETLNi32 (v2i32 (EXTRACT_SUBREG QPR:$src,
                              (DSubReg_i32_reg imm:$lane))),
-                     (SubReg_i32_lane imm:$lane))>;
+                     (SubReg_i32_lane imm:$lane))>,
+      Requires<[HasNEON, HasFastVGETLNi32]>;
+def : Pat<(extractelt (v2i32 DPR:$src), imm:$lane),
+          (COPY_TO_REGCLASS
+            (i32 (EXTRACT_SUBREG DPR:$src, (SSubReg_f32_reg imm:$lane))), GPR)>,
+      Requires<[HasNEON, HasSlowVGETLNi32]>;
+def : Pat<(extractelt (v4i32 QPR:$src), imm:$lane),
+          (COPY_TO_REGCLASS
+            (i32 (EXTRACT_SUBREG QPR:$src, (SSubReg_f32_reg imm:$lane))), GPR)>,
+      Requires<[HasNEON, HasSlowVGETLNi32]>;
 def : Pat<(extractelt (v2f32 DPR:$src1), imm:$src2),
           (EXTRACT_SUBREG (v2f32 (COPY_TO_REGCLASS (v2f32 DPR:$src1),DPR_VFP2)),
                           (SSubReg_f32_reg imm:$src2))>;
@@ -5117,14 +5212,23 @@ class VDUPQ<bits<8> opcod1, bits<2> opcod3, string Dt, ValueType Ty>
 
 def  VDUP8d   : VDUPD<0b11101100, 0b00, "8", v8i8>;
 def  VDUP16d  : VDUPD<0b11101000, 0b01, "16", v4i16>;
-def  VDUP32d  : VDUPD<0b11101000, 0b00, "32", v2i32>;
+def  VDUP32d  : VDUPD<0b11101000, 0b00, "32", v2i32>,
+                Requires<[HasNEON, HasFastVDUP32]>;
 def  VDUP8q   : VDUPQ<0b11101110, 0b00, "8", v16i8>;
 def  VDUP16q  : VDUPQ<0b11101010, 0b01, "16", v8i16>;
 def  VDUP32q  : VDUPQ<0b11101010, 0b00, "32", v4i32>;
 
-def : Pat<(v2f32 (NEONvdup (f32 (bitconvert GPR:$R)))), (VDUP32d GPR:$R)>;
+// NEONvdup patterns for uarchs with fast VDUP.32.
+def : Pat<(v2f32 (NEONvdup (f32 (bitconvert GPR:$R)))), (VDUP32d GPR:$R)>,
+      Requires<[HasNEON,HasFastVDUP32]>;
 def : Pat<(v4f32 (NEONvdup (f32 (bitconvert GPR:$R)))), (VDUP32q GPR:$R)>;
 
+// NEONvdup patterns for uarchs with slow VDUP.32 - use VMOVDRR instead.
+def : Pat<(v2i32 (NEONvdup (i32 GPR:$R))), (VMOVDRR GPR:$R, GPR:$R)>,
+      Requires<[HasNEON,HasSlowVDUP32]>;
+def : Pat<(v2f32 (NEONvdup (f32 (bitconvert GPR:$R)))), (VMOVDRR GPR:$R, GPR:$R)>,
+      Requires<[HasNEON,HasSlowVDUP32]>;
+
 //   VDUP     : Vector Duplicate Lane (from scalar to all elements)
 
 class VDUPLND<bits<4> op19_16, string OpcodeStr, string Dt,
@@ -5561,6 +5665,11 @@ def : N2VSPat<arm_ftoui, VCVTf2ud>;
 def : N2VSPat<arm_sitof, VCVTs2fd>;
 def : N2VSPat<arm_uitof, VCVTu2fd>;
 
+// Prefer VMOVDRR for i32 -> f32 bitcasts, it can write all DPR registers.
+def : Pat<(f32 (bitconvert GPR:$a)),
+          (EXTRACT_SUBREG (VMOVDRR GPR:$a, GPR:$a), ssub_0)>,
+        Requires<[HasNEON, DontUseVMOVSR]>;
+
 //===----------------------------------------------------------------------===//
 // Non-Instruction Patterns
 //===----------------------------------------------------------------------===//
diff --git a/lib/Target/ARM/ARMInstrThumb.td b/lib/Target/ARM/ARMInstrThumb.td
index 554f6d9..ae7a5c0 100644
--- a/lib/Target/ARM/ARMInstrThumb.td
+++ b/lib/Target/ARM/ARMInstrThumb.td
@@ -223,6 +223,7 @@ def t_addrmode_sp : Operand<i32>,
 def t_addrmode_pc : Operand<i32> {
   let EncoderMethod = "getAddrModePCOpValue";
   let DecoderMethod = "DecodeThumbAddrModePC";
+  let PrintMethod = "printThumbLdrLabelOperand";
 }
 
 //===----------------------------------------------------------------------===//
@@ -1200,6 +1201,7 @@ let neverHasSideEffects = 1, isReMaterializable = 1 in
 def tLEApcrel   : tPseudoInst<(outs tGPR:$Rd), (ins i32imm:$label, pred:$p),
                               2, IIC_iALUi, []>;
 
+let hasSideEffects = 1 in
 def tLEApcrelJT : tPseudoInst<(outs tGPR:$Rd),
                               (ins i32imm:$label, nohash_imm:$id, pred:$p),
                               2, IIC_iALUi, []>;
@@ -1245,10 +1247,6 @@ def tInt_eh_sjlj_longjmp : XI<(outs), (ins GPR:$src, GPR:$scratch),
                               [(ARMeh_sjlj_longjmp GPR:$src, GPR:$scratch)]>,
                              Requires<[IsThumb, IsIOS]>;
 
-let Defs = [ R0,  R1,  R2,  R3,  R4,  R5,  R6,  R7, R12, CPSR ],
-    isBarrier = 1 in
-def tInt_eh_sjlj_dispatchsetup : PseudoInst<(outs), (ins), NoItinerary, []>;
-
 //===----------------------------------------------------------------------===//
 // Non-Instruction Patterns
 //
diff --git a/lib/Target/ARM/ARMInstrThumb2.td b/lib/Target/ARM/ARMInstrThumb2.td
index 8ecf009..002d64a 100644
--- a/lib/Target/ARM/ARMInstrThumb2.td
+++ b/lib/Target/ARM/ARMInstrThumb2.td
@@ -159,7 +159,7 @@ def t2addrmode_imm12 : Operand<i32>,
 // t2ldrlabel  := imm12
 def t2ldrlabel : Operand<i32> {
   let EncoderMethod = "getAddrModeImm12OpValue";
-  let PrintMethod = "printT2LdrLabelOperand";
+  let PrintMethod = "printThumbLdrLabelOperand";
 }
 
 def t2ldr_pcrel_imm12_asmoperand : AsmOperandClass {let Name = "MemPCRelImm12";}
@@ -523,6 +523,23 @@ class T2MulLong<bits<3> opc22_20, bits<4> opc7_4,
   let Inst{7-4}   = opc7_4;
   let Inst{3-0}   = Rm;
 }
+class T2MlaLong<bits<3> opc22_20, bits<4> opc7_4,
+                dag oops, dag iops, InstrItinClass itin,
+                string opc, string asm, list<dag> pattern>
+  : T2I<oops, iops, itin, opc, asm, pattern> {
+  bits<4> RdLo;
+  bits<4> RdHi;
+  bits<4> Rn;
+  bits<4> Rm;
+
+  let Inst{31-23} = 0b111110111;
+  let Inst{22-20} = opc22_20;
+  let Inst{19-16} = Rn;
+  let Inst{15-12} = RdLo;
+  let Inst{11-8}  = RdHi;
+  let Inst{7-4}   = opc7_4;
+  let Inst{3-0}   = Rm;
+}
 
 
 /// T2I_bin_irs - Defines a set of (op reg, {so_imm|r|so_reg}) patterns for a
@@ -757,33 +774,6 @@ multiclass T2I_bin_ii12rs<bits<3> op23_21, string opc, PatFrag opnode,
      let Inst{24} = 1;
      let Inst{23-21} = op23_21;
    }
-
-   // Predicated versions.
-   def CCri : t2PseudoExpand<(outs GPRnopc:$Rd),
-                             (ins GPRnopc:$Rfalse, GPRnopc:$Rn, t2_so_imm:$imm,
-                                  pred:$p, cc_out:$s), 4, IIC_iALUi, [],
-                             (!cast<Instruction>(NAME#ri) GPRnopc:$Rd,
-                              GPRnopc:$Rn, t2_so_imm:$imm, pred:$p, cc_out:$s)>,
-              RegConstraint<"$Rfalse = $Rd">;
-   def CCri12 : t2PseudoExpand<(outs GPRnopc:$Rd),
-                             (ins GPRnopc:$Rfalse, GPR:$Rn, imm0_4095:$imm,
-                                  pred:$p),
-                             4, IIC_iALUi, [],
-                             (!cast<Instruction>(NAME#ri12) GPRnopc:$Rd,
-                              GPR:$Rn, imm0_4095:$imm, pred:$p)>,
-                RegConstraint<"$Rfalse = $Rd">;
-   def CCrr : t2PseudoExpand<(outs GPRnopc:$Rd),
-                             (ins GPRnopc:$Rfalse, GPRnopc:$Rn, rGPR:$Rm,
-                                  pred:$p, cc_out:$s), 4, IIC_iALUr, [],
-                             (!cast<Instruction>(NAME#rr) GPRnopc:$Rd,
-                              GPRnopc:$Rn, rGPR:$Rm, pred:$p, cc_out:$s)>,
-              RegConstraint<"$Rfalse = $Rd">;
-   def CCrs : t2PseudoExpand<(outs GPRnopc:$Rd),
-                             (ins GPRnopc:$Rfalse, GPRnopc:$Rn, t2_so_reg:$Rm,
-                                  pred:$p, cc_out:$s), 4, IIC_iALUsi, [],
-                             (!cast<Instruction>(NAME#rs) GPRnopc:$Rd,
-                              GPRnopc:$Rn, t2_so_reg:$Rm, pred:$p, cc_out:$s)>,
-              RegConstraint<"$Rfalse = $Rd">;
 }
 
 /// T2I_adde_sube_irs - Defines a set of (op reg, {so_imm|r|so_reg}) patterns
@@ -1200,6 +1190,7 @@ def t2ADR : T2PCOneRegImm<(outs rGPR:$Rd),
 let neverHasSideEffects = 1, isReMaterializable = 1 in
 def t2LEApcrel   : t2PseudoInst<(outs rGPR:$Rd), (ins i32imm:$label, pred:$p),
                                 4, IIC_iALUi, []>;
+let hasSideEffects = 1 in
 def t2LEApcrelJT : t2PseudoInst<(outs rGPR:$Rd),
                                 (ins i32imm:$label, nohash_imm:$id, pred:$p),
                                 4, IIC_iALUi,
@@ -1962,7 +1953,7 @@ def : T2Pat<(ARMadde    rGPR:$src, imm0_255_not:$imm, CPSR),
 def : T2Pat<(ARMadde    rGPR:$src, t2_so_imm_not:$imm, CPSR),
             (t2SBCri    rGPR:$src, t2_so_imm_not:$imm)>;
 def : T2Pat<(ARMadde    rGPR:$src, imm0_65535_neg:$imm, CPSR),
-            (t2SBCrr    rGPR:$src, (t2MOVi16 (imm_neg_XFORM imm:$imm)))>;
+            (t2SBCrr    rGPR:$src, (t2MOVi16 (imm_not_XFORM imm:$imm)))>;
 
 // Select Bytes -- for disassembly only
 
@@ -2405,7 +2396,8 @@ def t2MUL: T2ThreeReg<(outs rGPR:$Rd), (ins rGPR:$Rn, rGPR:$Rm), IIC_iMUL32,
 def t2MLA: T2FourReg<
                 (outs rGPR:$Rd), (ins rGPR:$Rn, rGPR:$Rm, rGPR:$Ra), IIC_iMAC32,
                 "mla", "\t$Rd, $Rn, $Rm, $Ra",
-                [(set rGPR:$Rd, (add (mul rGPR:$Rn, rGPR:$Rm), rGPR:$Ra))]> {
+                [(set rGPR:$Rd, (add (mul rGPR:$Rn, rGPR:$Rm), rGPR:$Ra))]>,
+           Requires<[IsThumb2, UseMulOps]> {
   let Inst{31-27} = 0b11111;
   let Inst{26-23} = 0b0110;
   let Inst{22-20} = 0b000;
@@ -2415,7 +2407,8 @@ def t2MLA: T2FourReg<
 def t2MLS: T2FourReg<
                 (outs rGPR:$Rd), (ins rGPR:$Rn, rGPR:$Rm, rGPR:$Ra), IIC_iMAC32,
                 "mls", "\t$Rd, $Rn, $Rm, $Ra",
-                [(set rGPR:$Rd, (sub rGPR:$Ra, (mul rGPR:$Rn, rGPR:$Rm)))]> {
+                [(set rGPR:$Rd, (sub rGPR:$Ra, (mul rGPR:$Rn, rGPR:$Rm)))]>,
+           Requires<[IsThumb2, UseMulOps]> {
   let Inst{31-27} = 0b11111;
   let Inst{26-23} = 0b0110;
   let Inst{22-20} = 0b000;
@@ -2437,15 +2430,17 @@ def t2UMULL : T2MulLong<0b010, 0b0000,
 } // isCommutable
 
 // Multiply + accumulate
-def t2SMLAL : T2MulLong<0b100, 0b0000,
+def t2SMLAL : T2MlaLong<0b100, 0b0000,
                   (outs rGPR:$RdLo, rGPR:$RdHi),
-                  (ins rGPR:$Rn, rGPR:$Rm), IIC_iMAC64,
-                  "smlal", "\t$RdLo, $RdHi, $Rn, $Rm", []>;
+                  (ins rGPR:$Rn, rGPR:$Rm, rGPR:$RLo, rGPR:$RHi), IIC_iMAC64,
+                  "smlal", "\t$RdLo, $RdHi, $Rn, $Rm", []>,
+                  RegConstraint<"$RLo = $RdLo, $RHi = $RdHi">;
 
-def t2UMLAL : T2MulLong<0b110, 0b0000,
+def t2UMLAL : T2MlaLong<0b110, 0b0000,
                   (outs rGPR:$RdLo, rGPR:$RdHi),
-                  (ins rGPR:$Rn, rGPR:$Rm), IIC_iMAC64,
-                  "umlal", "\t$RdLo, $RdHi, $Rn, $Rm", []>;
+                  (ins rGPR:$Rn, rGPR:$Rm, rGPR:$RLo, rGPR:$RHi), IIC_iMAC64,
+                  "umlal", "\t$RdLo, $RdHi, $Rn, $Rm", []>,
+                  RegConstraint<"$RLo = $RdLo, $RHi = $RdHi">;
 
 def t2UMAAL : T2MulLong<0b110, 0b0110,
                   (outs rGPR:$RdLo, rGPR:$RdHi),
@@ -2482,7 +2477,7 @@ def t2SMMLA : T2FourReg<
         (outs rGPR:$Rd), (ins rGPR:$Rn, rGPR:$Rm, rGPR:$Ra), IIC_iMAC32,
                 "smmla", "\t$Rd, $Rn, $Rm, $Ra",
                 [(set rGPR:$Rd, (add (mulhs rGPR:$Rm, rGPR:$Rn), rGPR:$Ra))]>,
-          Requires<[IsThumb2, HasThumb2DSP]> {
+              Requires<[IsThumb2, HasThumb2DSP, UseMulOps]> {
   let Inst{31-27} = 0b11111;
   let Inst{26-23} = 0b0110;
   let Inst{22-20} = 0b101;
@@ -2503,7 +2498,7 @@ def t2SMMLS: T2FourReg<
         (outs rGPR:$Rd), (ins rGPR:$Rn, rGPR:$Rm, rGPR:$Ra), IIC_iMAC32,
                 "smmls", "\t$Rd, $Rn, $Rm, $Ra",
                 [(set rGPR:$Rd, (sub rGPR:$Ra, (mulhs rGPR:$Rn, rGPR:$Rm)))]>,
-          Requires<[IsThumb2, HasThumb2DSP]> {
+             Requires<[IsThumb2, HasThumb2DSP, UseMulOps]> {
   let Inst{31-27} = 0b11111;
   let Inst{26-23} = 0b0110;
   let Inst{22-20} = 0b110;
@@ -2608,7 +2603,7 @@ multiclass T2I_smla<string opc, PatFrag opnode> {
               [(set rGPR:$Rd, (add rGPR:$Ra,
                                (opnode (sext_inreg rGPR:$Rn, i16),
                                        (sext_inreg rGPR:$Rm, i16))))]>,
-          Requires<[IsThumb2, HasThumb2DSP]> {
+           Requires<[IsThumb2, HasThumb2DSP, UseMulOps]> {
     let Inst{31-27} = 0b11111;
     let Inst{26-23} = 0b0110;
     let Inst{22-20} = 0b001;
@@ -2621,7 +2616,7 @@ multiclass T2I_smla<string opc, PatFrag opnode> {
              !strconcat(opc, "bt"), "\t$Rd, $Rn, $Rm, $Ra",
              [(set rGPR:$Rd, (add rGPR:$Ra, (opnode (sext_inreg rGPR:$Rn, i16),
                                                  (sra rGPR:$Rm, (i32 16)))))]>,
-          Requires<[IsThumb2, HasThumb2DSP]> {
+           Requires<[IsThumb2, HasThumb2DSP, UseMulOps]> {
     let Inst{31-27} = 0b11111;
     let Inst{26-23} = 0b0110;
     let Inst{22-20} = 0b001;
@@ -2634,7 +2629,7 @@ multiclass T2I_smla<string opc, PatFrag opnode> {
               !strconcat(opc, "tb"), "\t$Rd, $Rn, $Rm, $Ra",
               [(set rGPR:$Rd, (add rGPR:$Ra, (opnode (sra rGPR:$Rn, (i32 16)),
                                                (sext_inreg rGPR:$Rm, i16))))]>,
-          Requires<[IsThumb2, HasThumb2DSP]> {
+           Requires<[IsThumb2, HasThumb2DSP, UseMulOps]> {
     let Inst{31-27} = 0b11111;
     let Inst{26-23} = 0b0110;
     let Inst{22-20} = 0b001;
@@ -2647,7 +2642,7 @@ multiclass T2I_smla<string opc, PatFrag opnode> {
               !strconcat(opc, "tt"), "\t$Rd, $Rn, $Rm, $Ra",
              [(set rGPR:$Rd, (add rGPR:$Ra, (opnode (sra rGPR:$Rn, (i32 16)),
                                                  (sra rGPR:$Rm, (i32 16)))))]>,
-          Requires<[IsThumb2, HasThumb2DSP]> {
+           Requires<[IsThumb2, HasThumb2DSP, UseMulOps]> {
     let Inst{31-27} = 0b11111;
     let Inst{26-23} = 0b0110;
     let Inst{22-20} = 0b001;
@@ -2660,7 +2655,7 @@ multiclass T2I_smla<string opc, PatFrag opnode> {
               !strconcat(opc, "wb"), "\t$Rd, $Rn, $Rm, $Ra",
               [(set rGPR:$Rd, (add rGPR:$Ra, (sra (opnode rGPR:$Rn,
                                     (sext_inreg rGPR:$Rm, i16)), (i32 16))))]>,
-          Requires<[IsThumb2, HasThumb2DSP]> {
+           Requires<[IsThumb2, HasThumb2DSP, UseMulOps]> {
     let Inst{31-27} = 0b11111;
     let Inst{26-23} = 0b0110;
     let Inst{22-20} = 0b011;
@@ -2673,7 +2668,7 @@ multiclass T2I_smla<string opc, PatFrag opnode> {
               !strconcat(opc, "wt"), "\t$Rd, $Rn, $Rm, $Ra",
               [(set rGPR:$Rd, (add rGPR:$Ra, (sra (opnode rGPR:$Rn,
                                       (sra rGPR:$Rm, (i32 16))), (i32 16))))]>,
-          Requires<[IsThumb2, HasThumb2DSP]> {
+           Requires<[IsThumb2, HasThumb2DSP, UseMulOps]> {
     let Inst{31-27} = 0b11111;
     let Inst{26-23} = 0b0110;
     let Inst{22-20} = 0b011;
@@ -2767,7 +2762,7 @@ def t2SMLSLDX : T2FourReg_mac<1, 0b101, 0b1101, (outs rGPR:$Ra,rGPR:$Rd),
 //  Division Instructions.
 //  Signed and unsigned division on v7-M
 //
-def t2SDIV : T2ThreeReg<(outs rGPR:$Rd), (ins rGPR:$Rn, rGPR:$Rm), IIC_iALUi,
+def t2SDIV : T2ThreeReg<(outs rGPR:$Rd), (ins rGPR:$Rn, rGPR:$Rm), IIC_iDIV,
                  "sdiv", "\t$Rd, $Rn, $Rm",
                  [(set rGPR:$Rd, (sdiv rGPR:$Rn, rGPR:$Rm))]>,
                  Requires<[HasDivide, IsThumb2]> {
@@ -2778,7 +2773,7 @@ def t2SDIV : T2ThreeReg<(outs rGPR:$Rd), (ins rGPR:$Rn, rGPR:$Rm), IIC_iALUi,
   let Inst{7-4} = 0b1111;
 }
 
-def t2UDIV : T2ThreeReg<(outs rGPR:$Rd), (ins rGPR:$Rn, rGPR:$Rm), IIC_iALUi,
+def t2UDIV : T2ThreeReg<(outs rGPR:$Rd), (ins rGPR:$Rn, rGPR:$Rm), IIC_iDIV,
                  "udiv", "\t$Rd, $Rn, $Rm",
                  [(set rGPR:$Rd, (udiv rGPR:$Rn, rGPR:$Rm))]>,
                  Requires<[HasDivide, IsThumb2]> {
@@ -3049,37 +3044,6 @@ def t2MOVCCror : T2I_movcc_sh<0b11, (outs rGPR:$Rd),
                  RegConstraint<"$false = $Rd">;
 } // isCodeGenOnly = 1
 
-multiclass T2I_bincc_irs<Instruction iri, Instruction irr, Instruction irs,
-                   InstrItinClass iii, InstrItinClass iir, InstrItinClass iis> {
-   // shifted imm
-   def ri : t2PseudoExpand<(outs rGPR:$Rd),
-                           (ins rGPR:$Rfalse, rGPR:$Rn, t2_so_imm:$imm,
-                                pred:$p, cc_out:$s),
-                           4, iii, [],
-                  (iri rGPR:$Rd, rGPR:$Rn, t2_so_imm:$imm, pred:$p, cc_out:$s)>,
-                           RegConstraint<"$Rfalse = $Rd">;
-   // register
-   def rr : t2PseudoExpand<(outs rGPR:$Rd),
-                           (ins rGPR:$Rfalse, rGPR:$Rn, rGPR:$Rm,
-                                pred:$p, cc_out:$s),
-                           4, iir, [],
-                        (irr rGPR:$Rd, rGPR:$Rn, rGPR:$Rm, pred:$p, cc_out:$s)>,
-                           RegConstraint<"$Rfalse = $Rd">;
-   // shifted register
-   def rs : t2PseudoExpand<(outs rGPR:$Rd),
-                           (ins rGPR:$Rfalse, rGPR:$Rn, t2_so_reg:$ShiftedRm,
-                                pred:$p, cc_out:$s),
-                           4, iis, [],
-            (irs rGPR:$Rd, rGPR:$Rn, t2_so_reg:$ShiftedRm, pred:$p, cc_out:$s)>,
-                           RegConstraint<"$Rfalse = $Rd">;
-} // T2I_bincc_irs
-
-defm t2ANDCC : T2I_bincc_irs<t2ANDri, t2ANDrr, t2ANDrs,
-                             IIC_iBITi, IIC_iBITr, IIC_iBITsi>;
-defm t2ORRCC : T2I_bincc_irs<t2ORRri, t2ORRrr, t2ORRrs,
-                             IIC_iBITi, IIC_iBITr, IIC_iBITsi>;
-defm t2EORCC : T2I_bincc_irs<t2EORri, t2EORrr, t2EORrs,
-                             IIC_iBITi, IIC_iBITr, IIC_iBITsi>;
 } // neverHasSideEffects
 
 //===----------------------------------------------------------------------===//
@@ -3281,11 +3245,11 @@ def t2B   : T2I<(outs), (ins uncondbrtarget:$target), IIC_Br,
   let Inst{15-14} = 0b10;
   let Inst{12} = 1;
 
-  bits<20> target;
+  bits<24> target;
   let Inst{26} = target{19};
   let Inst{11} = target{18};
   let Inst{13} = target{17};
-  let Inst{21-16} = target{16-11};
+  let Inst{25-16} = target{20-11};
   let Inst{10-0} = target{10-0};
   let DecoderMethod = "DecodeT2BInstruction";
 }
@@ -3367,20 +3331,6 @@ let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1 in {
                  Requires<[IsThumb2, IsIOS]>;
 }
 
-let isCall = 1, Defs = [LR], Uses = [SP] in {
-  // mov lr, pc; b if callee is marked noreturn to avoid confusing the
-  // return stack predictor.
-  def t2BMOVPCB_CALL : tPseudoInst<(outs),
-                                   (ins t_bltarget:$func),
-                               6, IIC_Br, [(ARMcall_nolink tglobaladdr:$func)]>,
-                        Requires<[IsThumb]>;
-}
-
-// Direct calls
-def : T2Pat<(ARMcall_nolink texternalsym:$func),
-            (t2BMOVPCB_CALL texternalsym:$func)>,
-      Requires<[IsThumb]>;
-
 // IT block
 let Defs = [ITSTATE] in
 def t2IT : Thumb2XI<(outs), (ins it_pred:$cc, it_mask:$mask),
diff --git a/lib/Target/ARM/ARMInstrVFP.td b/lib/Target/ARM/ARMInstrVFP.td
index eb7eaa6..b5a896c 100644
--- a/lib/Target/ARM/ARMInstrVFP.td
+++ b/lib/Target/ARM/ARMInstrVFP.td
@@ -450,11 +450,11 @@ def VCVTBSH: ASuI<0b11101, 0b11, 0b0011, 0b01, 0, (outs SPR:$Sd), (ins SPR:$Sm),
                  /* FIXME */ IIC_fpCVTHS, "vcvtb", ".f16.f32\t$Sd, $Sm",
                  [/* For disassembly only; pattern left blank */]>;
 
-def : ARMPat<(f32_to_f16 SPR:$a),
-             (i32 (COPY_TO_REGCLASS (VCVTBSH SPR:$a), GPR))>;
+def : Pat<(f32_to_f16 SPR:$a),
+          (i32 (COPY_TO_REGCLASS (VCVTBSH SPR:$a), GPR))>;
 
-def : ARMPat<(f16_to_f32 GPR:$a),
-             (VCVTBHS (COPY_TO_REGCLASS GPR:$a, SPR))>;
+def : Pat<(f16_to_f32 GPR:$a),
+          (VCVTBHS (COPY_TO_REGCLASS GPR:$a, SPR))>;
 
 def VCVTTHS: ASuI<0b11101, 0b11, 0b0010, 0b11, 0, (outs SPR:$Sd), (ins SPR:$Sm),
                  /* FIXME */ IIC_fpCVTSH, "vcvtt", ".f32.f16\t$Sd, $Sm",
@@ -523,10 +523,12 @@ def VMOVRS : AVConv2I<0b11100001, 0b1010,
   let D = VFPNeonDomain;
 }
 
+// Bitcast i32 -> f32.  NEON prefers to use VMOVDRR.
 def VMOVSR : AVConv4I<0b11100000, 0b1010,
                       (outs SPR:$Sn), (ins GPR:$Rt),
                       IIC_fpMOVIS, "vmov", "\t$Sn, $Rt",
-                      [(set SPR:$Sn, (bitconvert GPR:$Rt))]> {
+                      [(set SPR:$Sn, (bitconvert GPR:$Rt))]>,
+             Requires<[HasVFP2, UseVMOVSR]> {
   // Instruction operands.
   bits<5> Sn;
   bits<4> Rt;
diff --git a/lib/Target/ARM/ARMJITInfo.cpp b/lib/Target/ARM/ARMJITInfo.cpp
index 3f99cce..254d8f6 100644
--- a/lib/Target/ARM/ARMJITInfo.cpp
+++ b/lib/Target/ARM/ARMJITInfo.cpp
@@ -168,7 +168,7 @@ void *ARMJITInfo::emitFunctionStub(const Function* F, void *Fn,
       intptr_t LazyPtr = getIndirectSymAddr(Fn);
       if (!LazyPtr) {
         // In PIC mode, the function stub is loading a lazy-ptr.
-        LazyPtr= (intptr_t)emitGlobalValueIndirectSym((GlobalValue*)F, Fn, JCE);
+        LazyPtr= (intptr_t)emitGlobalValueIndirectSym((const GlobalValue*)F, Fn, JCE);
         DEBUG(if (F)
                 errs() << "JIT: Indirect symbol emitted at [" << LazyPtr
                        << "] for GV '" << F->getName() << "'\n";
diff --git a/lib/Target/ARM/ARMLoadStoreOptimizer.cpp b/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
index 897ceb6..0185289 100644
--- a/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
+++ b/lib/Target/ARM/ARMLoadStoreOptimizer.cpp
@@ -27,7 +27,7 @@
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/RegisterScavenging.h"
 #include "llvm/CodeGen/SelectionDAGNodes.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetInstrInfo.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetRegisterInfo.h"
@@ -1448,7 +1448,7 @@ namespace {
     static char ID;
     ARMPreAllocLoadStoreOpt() : MachineFunctionPass(ID) {}
 
-    const TargetData *TD;
+    const DataLayout *TD;
     const TargetInstrInfo *TII;
     const TargetRegisterInfo *TRI;
     const ARMSubtarget *STI;
@@ -1478,7 +1478,7 @@ namespace {
 }
 
 bool ARMPreAllocLoadStoreOpt::runOnMachineFunction(MachineFunction &Fn) {
-  TD  = Fn.getTarget().getTargetData();
+  TD  = Fn.getTarget().getDataLayout();
   TII = Fn.getTarget().getInstrInfo();
   TRI = Fn.getTarget().getRegisterInfo();
   STI = &Fn.getTarget().getSubtarget<ARMSubtarget>();
diff --git a/lib/Target/ARM/ARMMachineFunctionInfo.h b/lib/Target/ARM/ARMMachineFunctionInfo.h
index f1c8fc8..c0ac04b 100644
--- a/lib/Target/ARM/ARMMachineFunctionInfo.h
+++ b/lib/Target/ARM/ARMMachineFunctionInfo.h
@@ -108,6 +108,11 @@ class ARMFunctionInfo : public MachineFunctionInfo {
   /// pass.
   DenseMap<unsigned, unsigned> CPEClones;
 
+  /// GlobalBaseReg - keeps track of the virtual register initialized for
+  /// use as the global base register. This is used for PIC in some PIC
+  /// relocation models.
+  unsigned GlobalBaseReg;
+
 public:
   ARMFunctionInfo() :
     isThumb(false),
@@ -119,7 +124,7 @@ public:
     GPRCS1Frames(0), GPRCS2Frames(0), DPRCSFrames(0),
     NumAlignedDPRCS2Regs(0),
     JumpTableUId(0), PICLabelUId(0),
-    VarArgsFrameIndex(0), HasITBlocks(false) {}
+    VarArgsFrameIndex(0), HasITBlocks(false), GlobalBaseReg(0) {}
 
   explicit ARMFunctionInfo(MachineFunction &MF) :
     isThumb(MF.getTarget().getSubtarget<ARMSubtarget>().isThumb()),
@@ -130,7 +135,7 @@ public:
     GPRCS1Size(0), GPRCS2Size(0), DPRCSSize(0),
     GPRCS1Frames(32), GPRCS2Frames(32), DPRCSFrames(32),
     JumpTableUId(0), PICLabelUId(0),
-    VarArgsFrameIndex(0), HasITBlocks(false) {}
+    VarArgsFrameIndex(0), HasITBlocks(false), GlobalBaseReg(0) {}
 
   bool isThumbFunction() const { return isThumb; }
   bool isThumb1OnlyFunction() const { return isThumb && !hasThumb2; }
@@ -249,6 +254,9 @@ public:
   bool hasITBlocks() const { return HasITBlocks; }
   void setHasITBlocks(bool h) { HasITBlocks = h; }
 
+  unsigned getGlobalBaseReg() const { return GlobalBaseReg; }
+  void setGlobalBaseReg(unsigned Reg) { GlobalBaseReg = Reg; }
+
   void recordCPEClone(unsigned CPIdx, unsigned CPCloneIdx) {
     if (!CPEClones.insert(std::make_pair(CPCloneIdx, CPIdx)).second)
       assert(0 && "Duplicate entries!");
diff --git a/lib/Target/ARM/ARMRegisterInfo.td b/lib/Target/ARM/ARMRegisterInfo.td
index 6f974fd..b0f576b 100644
--- a/lib/Target/ARM/ARMRegisterInfo.td
+++ b/lib/Target/ARM/ARMRegisterInfo.td
@@ -49,6 +49,9 @@ def ssub_0  : SubRegIndex;
 def ssub_1  : SubRegIndex;
 def ssub_2  : SubRegIndex<[dsub_1, ssub_0]>;
 def ssub_3  : SubRegIndex<[dsub_1, ssub_1]>;
+
+def gsub_0  : SubRegIndex;
+def gsub_1  : SubRegIndex;
 // Let TableGen synthesize the remaining 12 ssub_* indices.
 // We don't need to name them.
 }
@@ -247,11 +250,16 @@ def CCR : RegisterClass<"ARM", [i32], 32, (add CPSR)> {
 }
 
 // Scalar single precision floating point register class..
-def SPR : RegisterClass<"ARM", [f32], 32, (sequence "S%u", 0, 31)>;
+// FIXME: Allocation order changed to s0, s2, s4, ... as a quick hack to
+// avoid partial-write dependencies on D registers (S registers are
+// renamed as portions of D registers).
+def SPR : RegisterClass<"ARM", [f32], 32, (add (decimate
+                                                (sequence "S%u", 0, 31), 2),
+                                               (sequence "S%u", 0, 31))>;
 
 // Subset of SPR which can be used as a source of NEON scalars for 16-bit
 // operations
-def SPR_8 : RegisterClass<"ARM", [f32], 32, (trunc SPR, 16)>;
+def SPR_8 : RegisterClass<"ARM", [f32], 32, (sequence "S%u", 0, 15)>;
 
 // Scalar double precision floating point / generic 64-bit vector register
 // class.
@@ -308,6 +316,17 @@ def DPair : RegisterClass<"ARM", [v16i8, v8i16, v4i32, v2i64, v4f32, v2f64],
   let AltOrderSelect = [{ return 1; }];
 }
 
+// Pseudo-registers representing even-odd pairs of GPRs from R1 to R13/SP.
+// These are needed by instructions (e.g. ldrexd/strexd) requiring even-odd GPRs.
+def Tuples2R : RegisterTuples<[gsub_0, gsub_1],
+                              [(add R0, R2, R4, R6, R8, R10, R12),
+                               (add R1, R3, R5, R7, R9, R11, SP)]>;
+
+// Register class representing a pair of even-odd GPRs.
+def GPRPair : RegisterClass<"ARM", [untyped], 64, (add Tuples2R)> {
+  let Size = 64; // 2 x 32 bits, we have no predefined type of that size.
+}
+
 // Pseudo-registers representing 3 consecutive D registers.
 def Tuples3D : RegisterTuples<[dsub_0, dsub_1, dsub_2],
                               [(shl DPR, 0),
diff --git a/lib/Target/ARM/ARMSchedule.td b/lib/Target/ARM/ARMSchedule.td
index 81d2fa3..02196d0 100644
--- a/lib/Target/ARM/ARMSchedule.td
+++ b/lib/Target/ARM/ARMSchedule.td
@@ -55,6 +55,7 @@ def IIC_iMUL32     : InstrItinClass;
 def IIC_iMAC32     : InstrItinClass;
 def IIC_iMUL64     : InstrItinClass;
 def IIC_iMAC64     : InstrItinClass;
+def IIC_iDIV     : InstrItinClass;
 def IIC_iLoad_i    : InstrItinClass;
 def IIC_iLoad_r    : InstrItinClass;
 def IIC_iLoad_si   : InstrItinClass;
@@ -261,3 +262,4 @@ def IIC_VTBX4      : InstrItinClass;
 include "ARMScheduleV6.td"
 include "ARMScheduleA8.td"
 include "ARMScheduleA9.td"
+include "ARMScheduleSwift.td"
diff --git a/lib/Target/ARM/ARMScheduleA9.td b/lib/Target/ARM/ARMScheduleA9.td
index 7bc590f..404634f 100644
--- a/lib/Target/ARM/ARMScheduleA9.td
+++ b/lib/Target/ARM/ARMScheduleA9.td
@@ -1876,8 +1876,9 @@ def CortexA9Itineraries : ProcessorItineraries<
 ]>;
 
 // ===---------------------------------------------------------------------===//
-// This following definitions describe the simple machine model which
-// will replace itineraries.
+// The following definitions describe the simpler per-operand machine model.
+// This works with MachineScheduler and will eventually replace itineraries.
+
 
 // Cortex-A9 machine model for scheduling and other instruction cost heuristics.
 def CortexA9Model : SchedMachineModel {
@@ -1891,5 +1892,595 @@ def CortexA9Model : SchedMachineModel {
   let Itineraries = CortexA9Itineraries;
 }
 
-// TODO: Add Cortex-A9 processor and scheduler resources.
+//===----------------------------------------------------------------------===//
+// Define each kind of processor resource and number available.
+
+def A9UnitALU : ProcResource<2>;
+def A9UnitMul : ProcResource<1> { let Super = A9UnitALU; }
+def A9UnitAGU : ProcResource<1>;
+def A9UnitLS  : ProcResource<1>;
+def A9UnitFP  : ProcResource<1> { let Buffered = 0; }
+def A9UnitB   : ProcResource<1>;
+
+//===----------------------------------------------------------------------===//
+// Define scheduler read/write types with their resources and latency on A9.
+
+// Consume an issue slot, but no processor resources. This is useful when all
+// other writes associated with the operand have NumMicroOps = 0.
+def A9WriteIssue : SchedWriteRes<[]> { let Latency = 0; }
+
+// Write an integer register.
+def A9WriteI : SchedWriteRes<[A9UnitALU]>;
+// Write an integer shifted-by register
+def A9WriteIsr : SchedWriteRes<[A9UnitALU]> { let Latency = 2; }
+
+// Basic ALU.
+def A9WriteA : SchedWriteRes<[A9UnitALU]>;
+// ALU with operand shifted by immediate.
+def A9WriteAsi : SchedWriteRes<[A9UnitALU]> { let Latency = 2; }
+// ALU with operand shifted by register.
+def A9WriteAsr : SchedWriteRes<[A9UnitALU]> { let Latency = 3; }
+
+// Multiplication
+def A9WriteM   : SchedWriteRes<[A9UnitMul, A9UnitMul]> { let Latency = 4; }
+def A9WriteMHi : SchedWriteRes<[A9UnitMul]> { let Latency = 5;
+                                              let NumMicroOps = 0; }
+def A9WriteM16   : SchedWriteRes<[A9UnitMul]> { let Latency = 3; }
+def A9WriteM16Hi : SchedWriteRes<[A9UnitMul]> { let Latency = 4;
+                                                let NumMicroOps = 0; }
+
+// Floating-point
+// Only one FP or AGU instruction may issue per cycle. We model this
+// by having FP instructions consume the AGU resource.
+def A9WriteF      : SchedWriteRes<[A9UnitFP, A9UnitAGU]> { let Latency = 4; }
+def A9WriteFMov   : SchedWriteRes<[A9UnitFP, A9UnitAGU]> { let Latency = 1; }
+def A9WriteFMulS  : SchedWriteRes<[A9UnitFP, A9UnitAGU]> { let Latency = 5; }
+def A9WriteFMulD  : SchedWriteRes<[A9UnitFP, A9UnitAGU]> { let Latency = 6; }
+def A9WriteFMAS   : SchedWriteRes<[A9UnitFP, A9UnitAGU]> { let Latency = 8; }
+def A9WriteFMAD   : SchedWriteRes<[A9UnitFP, A9UnitAGU]> { let Latency = 9; }
+def A9WriteFDivS  : SchedWriteRes<[A9UnitFP, A9UnitAGU]> { let Latency = 15; }
+def A9WriteFDivD  : SchedWriteRes<[A9UnitFP, A9UnitAGU]> { let Latency = 25; }
+def A9WriteFSqrtS : SchedWriteRes<[A9UnitFP, A9UnitAGU]> { let Latency = 17; }
+def A9WriteFSqrtD : SchedWriteRes<[A9UnitFP, A9UnitAGU]> { let Latency = 32; }
+
+// NEON has an odd mix of latencies. Simply name the write types by latency.
+def A9WriteV1 : SchedWriteRes<[A9UnitFP, A9UnitAGU]> { let Latency = 1; }
+def A9WriteV2 : SchedWriteRes<[A9UnitFP, A9UnitAGU]> { let Latency = 2; }
+def A9WriteV3 : SchedWriteRes<[A9UnitFP, A9UnitAGU]> { let Latency = 3; }
+def A9WriteV4 : SchedWriteRes<[A9UnitFP, A9UnitAGU]> { let Latency = 4; }
+def A9WriteV5 : SchedWriteRes<[A9UnitFP, A9UnitAGU]> { let Latency = 5; }
+def A9WriteV6 : SchedWriteRes<[A9UnitFP, A9UnitAGU]> { let Latency = 6; }
+def A9WriteV7 : SchedWriteRes<[A9UnitFP, A9UnitAGU]> { let Latency = 7; }
+def A9WriteV9 : SchedWriteRes<[A9UnitFP, A9UnitAGU]> { let Latency = 9; }
+def A9WriteV10 : SchedWriteRes<[A9UnitFP, A9UnitAGU]> { let Latency = 10; }
+
+// Reserve A9UnitFP for 2 consecutive cycles.
+def A9Write2V4 : SchedWriteRes<[A9UnitFP, A9UnitAGU]> {
+  let Latency = 4;
+  let ResourceCycles = [2];
+}
+def A9Write2V7 : SchedWriteRes<[A9UnitFP, A9UnitAGU]> {
+  let Latency = 7;
+  let ResourceCycles = [2];
+}
+def A9Write2V9 : SchedWriteRes<[A9UnitFP, A9UnitAGU]> {
+  let Latency = 9;
+  let ResourceCycles = [2];
+}
+
+// Branches don't have a def operand but still consume resources.
+def A9WriteB : SchedWriteRes<[A9UnitB]>;
+
+// Address generation.
+def A9WriteAdr : SchedWriteRes<[A9UnitAGU]> { let NumMicroOps = 0; }
+
+// Load Integer.
+def A9WriteL : SchedWriteRes<[A9UnitLS]> { let Latency = 3; }
+// Load the upper 32-bits using the same micro-op.
+def A9WriteLHi : SchedWriteRes<[]> { let Latency = 3;
+                                     let NumMicroOps = 0; }
+// Offset shifted by register.
+def A9WriteLsi : SchedWriteRes<[A9UnitLS]> { let Latency = 4; }
+// Load (and zero extend) a byte.
+def A9WriteLb : SchedWriteRes<[A9UnitLS]> { let Latency = 4; }
+def A9WriteLbsi : SchedWriteRes<[A9UnitLS]> { let Latency = 5; }
+
+// Load or Store Float, aligned.
+def A9WriteLSfp : SchedWriteRes<[A9UnitLS, A9UnitFP]> { let Latency = 1; }
+
+// Store Integer.
+def A9WriteS : SchedWriteRes<[A9UnitLS]>;
+
+//===----------------------------------------------------------------------===//
+// Define resources dynamically for load multiple variants.
+
+// Define helpers for extra latency without consuming resources.
+def A9WriteCycle1 : SchedWriteRes<[]> { let Latency = 1; let NumMicroOps = 0; }
+foreach NumCycles = 2-8 in {
+def A9WriteCycle#NumCycles : WriteSequence<[A9WriteCycle1], NumCycles>;
+} // foreach NumCycles
+
+// Define TII for use in SchedVariant Predicates.
+def : PredicateProlog<[{
+  const ARMBaseInstrInfo *TII =
+    static_cast<const ARMBaseInstrInfo*>(SchedModel->getInstrInfo());
+  (void)TII;
+}]>;
+
+// Define address generation sequences and predicates for 8 flavors of LDMs.
+foreach NumAddr = 1-8 in {
+
+// Define A9WriteAdr1-8 as a sequence of A9WriteAdr with additive
+// latency for instructions that generate multiple loads or stores.
+def A9WriteAdr#NumAddr : WriteSequence<[A9WriteAdr], NumAddr>;
+
+// Define a predicate to select the LDM based on number of memory addresses.
+def A9LMAdr#NumAddr#Pred :
+  SchedPredicate<"TII->getNumLDMAddresses(MI) == "#NumAddr>;
+
+} // foreach NumAddr
+
+// Fall-back for unknown LDMs.
+def A9LMUnknownPred : SchedPredicate<"TII->getNumLDMAddresses(MI) == 0">;
+
+// LDM/VLDM/VLDn address generation latency & resources.
+// Dynamically select the A9WriteAdrN sequence using a predicate.
+def A9WriteLMAdr : SchedWriteVariant<[
+  SchedVar<A9LMAdr1Pred, [A9WriteAdr1]>,
+  SchedVar<A9LMAdr2Pred, [A9WriteAdr2]>,
+  SchedVar<A9LMAdr3Pred, [A9WriteAdr3]>,
+  SchedVar<A9LMAdr4Pred, [A9WriteAdr4]>,
+  SchedVar<A9LMAdr5Pred, [A9WriteAdr5]>,
+  SchedVar<A9LMAdr6Pred, [A9WriteAdr6]>,
+  SchedVar<A9LMAdr7Pred, [A9WriteAdr7]>,
+  SchedVar<A9LMAdr8Pred, [A9WriteAdr8]>,
+  // For unknown LDM/VLDM/VSTM, assume 2 32-bit registers.
+  SchedVar<A9LMUnknownPred, [A9WriteAdr2]>]>;
+
+// Define LDM Resources.
+// These take no issue resource, so they can be combined with other
+// writes like WriteB.
+// A9WriteLMLo takes a single LS resource and 2 cycles.
+def A9WriteLMLo : SchedWriteRes<[A9UnitLS]> { let Latency = 2;
+                                              let NumMicroOps = 0; }
+// Assuming aligned access, the upper half of each pair is free with
+// the same latency.
+def A9WriteLMHi : SchedWriteRes<[]> { let Latency = 2;
+                                      let NumMicroOps = 0; }
+// Each A9WriteL#N variant adds N cycles of latency without consuming
+// additional resources.
+foreach NumAddr = 1-8 in {
+def A9WriteL#NumAddr : WriteSequence<
+  [A9WriteLMLo, !cast<SchedWrite>("A9WriteCycle"#NumAddr)]>;
+def A9WriteL#NumAddr#Hi : WriteSequence<
+  [A9WriteLMHi, !cast<SchedWrite>("A9WriteCycle"#NumAddr)]>;
+}
+
+//===----------------------------------------------------------------------===//
+// LDM: Load multiple into 32-bit integer registers.
+
+// A9WriteLM variants expand into a pair of writes for each 64-bit
+// value loaded. When the number of registers is odd, the last
+// A9WriteLnHi is naturally ignored because the instruction has no
+// following def operands.  These variants take no issue resource, so
+// they may need to be part of a WriteSequence that includes A9WriteIssue.
+def A9WriteLM : SchedWriteVariant<[
+  SchedVar<A9LMAdr1Pred, [A9WriteL1, A9WriteL1Hi]>,
+  SchedVar<A9LMAdr2Pred, [A9WriteL1, A9WriteL1Hi,
+                          A9WriteL2, A9WriteL2Hi]>,
+  SchedVar<A9LMAdr3Pred, [A9WriteL1, A9WriteL1Hi,
+                          A9WriteL2, A9WriteL2Hi,
+                          A9WriteL3, A9WriteL3Hi]>,
+  SchedVar<A9LMAdr4Pred, [A9WriteL1, A9WriteL1Hi,
+                          A9WriteL2, A9WriteL2Hi,
+                          A9WriteL3, A9WriteL3Hi,
+                          A9WriteL4, A9WriteL4Hi]>,
+  SchedVar<A9LMAdr5Pred, [A9WriteL1, A9WriteL1Hi,
+                          A9WriteL2, A9WriteL2Hi,
+                          A9WriteL3, A9WriteL3Hi,
+                          A9WriteL4, A9WriteL4Hi,
+                          A9WriteL5, A9WriteL5Hi]>,
+  SchedVar<A9LMAdr6Pred, [A9WriteL1, A9WriteL1Hi,
+                          A9WriteL2, A9WriteL2Hi,
+                          A9WriteL3, A9WriteL3Hi,
+                          A9WriteL4, A9WriteL4Hi,
+                          A9WriteL5, A9WriteL5Hi,
+                          A9WriteL6, A9WriteL6Hi]>,
+  SchedVar<A9LMAdr7Pred, [A9WriteL1, A9WriteL1Hi,
+                          A9WriteL2, A9WriteL2Hi,
+                          A9WriteL3, A9WriteL3Hi,
+                          A9WriteL4, A9WriteL4Hi,
+                          A9WriteL5, A9WriteL5Hi,
+                          A9WriteL6, A9WriteL6Hi,
+                          A9WriteL7, A9WriteL7Hi]>,
+  SchedVar<A9LMAdr8Pred, [A9WriteL1, A9WriteL1Hi,
+                          A9WriteL2, A9WriteL2Hi,
+                          A9WriteL3, A9WriteL3Hi,
+                          A9WriteL4, A9WriteL4Hi,
+                          A9WriteL5, A9WriteL5Hi,
+                          A9WriteL6, A9WriteL6Hi,
+                          A9WriteL7, A9WriteL7Hi,
+                          A9WriteL8, A9WriteL8Hi]>,
+  // For unknown LDMs, define the maximum number of writes, but only
+  // make the first two consume resources.
+  SchedVar<A9LMUnknownPred, [A9WriteL1, A9WriteL1Hi,
+                             A9WriteL2, A9WriteL2Hi,
+                             A9WriteL3Hi, A9WriteL3Hi,
+                             A9WriteL4Hi, A9WriteL4Hi,
+                             A9WriteL5Hi, A9WriteL5Hi,
+                             A9WriteL6Hi, A9WriteL6Hi,
+                             A9WriteL7Hi, A9WriteL7Hi,
+                             A9WriteL8Hi, A9WriteL8Hi]>]> {
+  let Variadic = 1;
+}
+
+//===----------------------------------------------------------------------===//
+// VFP Load/Store Multiple Variants, and NEON VLDn/VSTn support.
+
+// A9WriteLfpOp is the same as A9WriteLSfp but takes no issue resources
+// so can be used in WriteSequences for in single-issue instructions that
+// encapsulate multiple loads.
+def A9WriteLfpOp : SchedWriteRes<[A9UnitLS, A9UnitFP]> {
+  let Latency = 1;
+  let NumMicroOps = 0;
+}
+
+foreach NumAddr = 1-8 in {
+
+// Helper for A9WriteLfp1-8: A sequence of fp loads with no micro-ops.
+def A9WriteLfp#NumAddr#Seq : WriteSequence<[A9WriteLfpOp], NumAddr>;
+
+// A9WriteLfp1-8 definitions are statically expanded into a sequence of
+// A9WriteLfpOps with additive latency that takes a single issue slot.
+// Used directly to describe NEON VLDn.
+def A9WriteLfp#NumAddr : WriteSequence<
+  [A9WriteIssue, !cast<SchedWrite>("A9WriteLfp"#NumAddr#Seq)]>;
+
+// A9WriteLfp1-8Mov adds a cycle of latency and FP resource for
+// permuting loaded values.
+def A9WriteLfp#NumAddr#Mov : WriteSequence<
+  [A9WriteF, !cast<SchedWrite>("A9WriteLfp"#NumAddr#Seq)]>;
+
+} // foreach NumAddr
+
+// Define VLDM/VSTM PreRA resources.
+// A9WriteLMfpPreRA are dynamically expanded into the correct
+// A9WriteLfp1-8 sequence based on a predicate. This supports the
+// preRA VLDM variants in which all 64-bit loads are written to the
+// same tuple of either single or double precision registers.
+def A9WriteLMfpPreRA : SchedWriteVariant<[
+  SchedVar<A9LMAdr1Pred, [A9WriteLfp1]>,
+  SchedVar<A9LMAdr2Pred, [A9WriteLfp2]>,
+  SchedVar<A9LMAdr3Pred, [A9WriteLfp3]>,
+  SchedVar<A9LMAdr4Pred, [A9WriteLfp4]>,
+  SchedVar<A9LMAdr5Pred, [A9WriteLfp5]>,
+  SchedVar<A9LMAdr6Pred, [A9WriteLfp6]>,
+  SchedVar<A9LMAdr7Pred, [A9WriteLfp7]>,
+  SchedVar<A9LMAdr8Pred, [A9WriteLfp8]>,
+  // For unknown VLDM/VSTM PreRA, assume 2xS registers.
+  SchedVar<A9LMUnknownPred, [A9WriteLfp2]>]>;
+
+// Define VLDM/VSTM PostRA Resources.
+// A9WriteLMfpLo takes a LS and FP resource and one issue slot but no latency.
+def A9WriteLMfpLo : SchedWriteRes<[A9UnitLS, A9UnitFP]> { let Latency = 0; }
+
+foreach NumAddr = 1-8 in {
+
+// Each A9WriteL#N variant adds N cycles of latency without consuming
+// additional resources.
+def A9WriteLMfp#NumAddr : WriteSequence<
+  [A9WriteLMfpLo, !cast<SchedWrite>("A9WriteCycle"#NumAddr)]>;
+
+// Assuming aligned access, the upper half of each pair is free with
+// the same latency.
+def A9WriteLMfp#NumAddr#Hi : WriteSequence<
+  [A9WriteLMHi, !cast<SchedWrite>("A9WriteCycle"#NumAddr)]>;
+
+} // foreach NumAddr
+
+// VLDM PostRA Variants. These variants expand A9WriteLMfpPostRA into a
+// pair of writes for each 64-bit data loaded. When the number of
+// registers is odd, the last WriteLMfpnHi is naturally ignored because
+// the instruction has no following def operands.
+def A9WriteLMfpPostRA : SchedWriteVariant<[
+  SchedVar<A9LMAdr1Pred, [A9WriteLMfp1, A9WriteLMfp1Hi]>,
+  SchedVar<A9LMAdr2Pred, [A9WriteLMfp1, A9WriteLMfp1Hi,
+                          A9WriteLMfp2, A9WriteLMfp2Hi]>,
+  SchedVar<A9LMAdr3Pred, [A9WriteLMfp1, A9WriteLMfp1Hi,
+                          A9WriteLMfp2, A9WriteLMfp2Hi,
+                          A9WriteLMfp3, A9WriteLMfp3Hi]>,
+  SchedVar<A9LMAdr4Pred, [A9WriteLMfp1, A9WriteLMfp1Hi,
+                          A9WriteLMfp2, A9WriteLMfp2Hi,
+                          A9WriteLMfp3, A9WriteLMfp3Hi,
+                          A9WriteLMfp4, A9WriteLMfp4Hi]>,
+  SchedVar<A9LMAdr5Pred, [A9WriteLMfp1, A9WriteLMfp1Hi,
+                          A9WriteLMfp2, A9WriteLMfp2Hi,
+                          A9WriteLMfp3, A9WriteLMfp3Hi,
+                          A9WriteLMfp4, A9WriteLMfp4Hi,
+                          A9WriteLMfp5, A9WriteLMfp5Hi]>,
+  SchedVar<A9LMAdr6Pred, [A9WriteLMfp1, A9WriteLMfp1Hi,
+                          A9WriteLMfp2, A9WriteLMfp2Hi,
+                          A9WriteLMfp3, A9WriteLMfp3Hi,
+                          A9WriteLMfp4, A9WriteLMfp4Hi,
+                          A9WriteLMfp5, A9WriteLMfp5Hi,
+                          A9WriteLMfp6, A9WriteLMfp6Hi]>,
+  SchedVar<A9LMAdr7Pred, [A9WriteLMfp1, A9WriteLMfp1Hi,
+                          A9WriteLMfp2, A9WriteLMfp2Hi,
+                          A9WriteLMfp3, A9WriteLMfp3Hi,
+                          A9WriteLMfp4, A9WriteLMfp4Hi,
+                          A9WriteLMfp5, A9WriteLMfp5Hi,
+                          A9WriteLMfp6, A9WriteLMfp6Hi,
+                          A9WriteLMfp7, A9WriteLMfp7Hi]>,
+  SchedVar<A9LMAdr8Pred, [A9WriteLMfp1, A9WriteLMfp1Hi,
+                          A9WriteLMfp2, A9WriteLMfp2Hi,
+                          A9WriteLMfp3, A9WriteLMfp3Hi,
+                          A9WriteLMfp4, A9WriteLMfp4Hi,
+                          A9WriteLMfp5, A9WriteLMfp5Hi,
+                          A9WriteLMfp6, A9WriteLMfp6Hi,
+                          A9WriteLMfp7, A9WriteLMfp7Hi,
+                          A9WriteLMfp8, A9WriteLMfp8Hi]>,
+  // For unknown LDMs, define the maximum number of writes, but only
+  // make the first two consume resources.
+  SchedVar<A9LMUnknownPred, [A9WriteLMfp1, A9WriteLMfp1Hi,
+                             A9WriteLMfp2, A9WriteLMfp2Hi,
+                             A9WriteLMfp3Hi, A9WriteLMfp3Hi,
+                             A9WriteLMfp4Hi, A9WriteLMfp4Hi,
+                             A9WriteLMfp5Hi, A9WriteLMfp5Hi,
+                             A9WriteLMfp6Hi, A9WriteLMfp6Hi,
+                             A9WriteLMfp7Hi, A9WriteLMfp7Hi,
+                             A9WriteLMfp8Hi, A9WriteLMfp8Hi]>]> {
+  let Variadic = 1;
+}
+
+// Distinguish between our multiple MI-level forms of the same
+// VLDM/VSTM instructions.
+def A9PreRA : SchedPredicate<
+  "TargetRegisterInfo::isVirtualRegister(MI->getOperand(0).getReg())">;
+def A9PostRA : SchedPredicate<
+  "TargetRegisterInfo::isPhysicalRegister(MI->getOperand(0).getReg())">;
+
+// VLDM represents all destination registers as a single register
+// tuple, unlike LDM. So the number of write operands is not variadic.
+def A9WriteLMfp : SchedWriteVariant<[
+  SchedVar<A9PreRA, [A9WriteLMfpPreRA]>,
+  SchedVar<A9PostRA, [A9WriteLMfpPostRA]>]>;
+
+//===----------------------------------------------------------------------===//
+// Resources for other (non LDM/VLDM) Variants.
+
+// These mov immediate writers are unconditionally expanded with
+// additive latency.
+def A9WriteI2 : WriteSequence<[A9WriteI, A9WriteI]>;
+def A9WriteI2pc : WriteSequence<[A9WriteI, A9WriteI, A9WriteA]>;
+def A9WriteI2ld  : WriteSequence<[A9WriteI, A9WriteI, A9WriteL]>;
+
+// Some ALU operations can read loaded integer values one cycle early.
+def A9ReadA : SchedReadAdvance<1,
+  [A9WriteL, A9WriteLHi, A9WriteLsi, A9WriteLb, A9WriteLbsi,
+   A9WriteL1, A9WriteL2, A9WriteL3, A9WriteL4,
+   A9WriteL5, A9WriteL6, A9WriteL7, A9WriteL8,
+   A9WriteL1Hi, A9WriteL2Hi, A9WriteL3Hi, A9WriteL4Hi,
+   A9WriteL5Hi, A9WriteL6Hi, A9WriteL7Hi, A9WriteL8Hi]>;
+
+// Read types for operands that are unconditionally read in cycle N
+// after the instruction issues, decreases producer latency by N-1.
+def A9Read2 : SchedReadAdvance<1>;
+def A9Read3 : SchedReadAdvance<2>;
+def A9Read4 : SchedReadAdvance<3>;
+
+//===----------------------------------------------------------------------===//
+// Map itinerary classes to scheduler read/write resources per operand.
+//
+// For ARM, we piggyback scheduler resources on the Itinerary classes
+// to avoid perturbing the existing instruction definitions.
+
+// This table follows the ARM Cortex-A9 Technical Reference Manuals,
+// mostly in order.
+let SchedModel = CortexA9Model in {
+
+def :ItinRW<[A9WriteI], [IIC_iMOVi,IIC_iMOVr,IIC_iMOVsi,
+                         IIC_iMVNi,IIC_iMVNsi,
+                         IIC_iCMOVi,IIC_iCMOVr,IIC_iCMOVsi]>;
+def :ItinRW<[A9WriteI,A9ReadA],[IIC_iMVNr]>;
+def :ItinRW<[A9WriteIsr], [IIC_iMOVsr,IIC_iMVNsr,IIC_iCMOVsr]>;
+
+def :ItinRW<[A9WriteI2],   [IIC_iMOVix2,IIC_iCMOVix2]>;
+def :ItinRW<[A9WriteI2pc], [IIC_iMOVix2addpc]>;
+def :ItinRW<[A9WriteI2ld], [IIC_iMOVix2ld]>;
+
+def :ItinRW<[A9WriteA], [IIC_iBITi,IIC_iBITr,IIC_iUNAr,IIC_iTSTi,IIC_iTSTr]>;
+def :ItinRW<[A9WriteA, A9ReadA], [IIC_iALUi, IIC_iCMPi, IIC_iCMPsi]>;
+def :ItinRW<[A9WriteA, A9ReadA, A9ReadA],[IIC_iALUr,IIC_iCMPr]>;
+def :ItinRW<[A9WriteAsi], [IIC_iBITsi,IIC_iUNAsi,IIC_iEXTr,IIC_iTSTsi]>;
+def :ItinRW<[A9WriteAsi, A9ReadA], [IIC_iALUsi]>;
+def :ItinRW<[A9WriteAsi, ReadDefault, A9ReadA], [IIC_iALUsir]>; // RSB
+def :ItinRW<[A9WriteAsr], [IIC_iBITsr,IIC_iTSTsr,IIC_iEXTAr,IIC_iEXTAsr]>;
+def :ItinRW<[A9WriteAsr, A9ReadA], [IIC_iALUsr,IIC_iCMPsr]>;
+
+// A9WriteHi ignored for MUL32.
+def :ItinRW<[A9WriteM, A9WriteMHi], [IIC_iMUL32,IIC_iMAC32,
+                                     IIC_iMUL64,IIC_iMAC64]>;
+// FIXME: SMLALxx needs itin classes
+def :ItinRW<[A9WriteM16, A9WriteM16Hi], [IIC_iMUL16,IIC_iMAC16]>;
+
+// TODO: For floating-point ops, we model the pipeline forwarding
+// latencies here. WAW latencies are sometimes longer.
+
+def :ItinRW<[A9WriteFMov], [IIC_fpSTAT, IIC_fpMOVIS, IIC_fpMOVID, IIC_fpMOVSI,
+                            IIC_fpUNA32, IIC_fpUNA64,
+                            IIC_fpCMP32, IIC_fpCMP64]>;
+def :ItinRW<[A9WriteFMov, A9WriteFMov], [IIC_fpMOVDI]>;
+def :ItinRW<[A9WriteF], [IIC_fpCVTSD, IIC_fpCVTDS, IIC_fpCVTSH, IIC_fpCVTHS,
+                         IIC_fpCVTIS, IIC_fpCVTID, IIC_fpCVTSI, IIC_fpCVTDI,
+                         IIC_fpALU32, IIC_fpALU64]>;
+def :ItinRW<[A9WriteFMulS], [IIC_fpMUL32]>;
+def :ItinRW<[A9WriteFMulD], [IIC_fpMUL64]>;
+def :ItinRW<[A9WriteFMAS], [IIC_fpMAC32]>;
+def :ItinRW<[A9WriteFMAD], [IIC_fpMAC64]>;
+def :ItinRW<[A9WriteFDivS], [IIC_fpDIV32]>;
+def :ItinRW<[A9WriteFDivD], [IIC_fpDIV64]>;
+def :ItinRW<[A9WriteFSqrtS], [IIC_fpSQRT32]>;
+def :ItinRW<[A9WriteFSqrtD], [IIC_fpSQRT64]>;
+
+def :ItinRW<[A9WriteB], [IIC_Br]>;
+
+// A9 PLD is processed in a dedicated unit.
+def :ItinRW<[], [IIC_Preload]>;
+
+// Note: We must assume that loads are aligned, since the machine
+// model cannot know this statically and A9 ignores alignment hints.
+
+// A9WriteAdr consumes AGU regardless address writeback. But it's
+// latency is only relevant for users of an updated address.
+def :ItinRW<[A9WriteL, A9WriteAdr], [IIC_iLoad_i,IIC_iLoad_r,
+                                     IIC_iLoad_iu,IIC_iLoad_ru]>;
+def :ItinRW<[A9WriteLsi, A9WriteAdr], [IIC_iLoad_si,IIC_iLoad_siu]>;
+def :ItinRW<[A9WriteLb, A9WriteAdr2], [IIC_iLoad_bh_i,IIC_iLoad_bh_r,
+                                       IIC_iLoad_bh_iu,IIC_iLoad_bh_ru]>;
+def :ItinRW<[A9WriteLbsi, A9WriteAdr2], [IIC_iLoad_bh_si,IIC_iLoad_bh_siu]>;
+def :ItinRW<[A9WriteL, A9WriteLHi, A9WriteAdr], [IIC_iLoad_d_i,IIC_iLoad_d_r,
+                                            IIC_iLoad_d_ru]>;
+// Store either has no def operands, or the one def for address writeback.
+def :ItinRW<[A9WriteAdr, A9WriteS], [IIC_iStore_i, IIC_iStore_r,
+                                     IIC_iStore_iu, IIC_iStore_ru,
+                                     IIC_iStore_d_i, IIC_iStore_d_r,
+                                     IIC_iStore_d_ru]>;
+def :ItinRW<[A9WriteAdr2, A9WriteS], [IIC_iStore_si, IIC_iStore_siu,
+                                      IIC_iStore_bh_i, IIC_iStore_bh_r,
+                                      IIC_iStore_bh_iu, IIC_iStore_bh_ru]>;
+def :ItinRW<[A9WriteAdr3, A9WriteS], [IIC_iStore_bh_si, IIC_iStore_bh_siu]>;
+
+// A9WriteML will be expanded into a separate write for each def
+// operand. Address generation consumes resources, but A9WriteLMAdr
+// is listed after all def operands, so has no effective latency.
+//
+// Note: A9WriteLM expands into an even number of def operands. The
+// actual number of def operands may be less by one.
+def :ItinRW<[A9WriteLM, A9WriteLMAdr, A9WriteIssue], [IIC_iLoad_m, IIC_iPop]>;
+
+// Load multiple with address writeback has an extra def operand in
+// front of the loaded registers.
+//
+// Reuse the load-multiple variants for store-multiple because the
+// resources are identical, For stores only the address writeback
+// has a def operand so the WriteL latencies are unused.
+def :ItinRW<[A9WriteLMAdr, A9WriteLM, A9WriteIssue], [IIC_iLoad_mu,
+                                                      IIC_iStore_m,
+                                                      IIC_iStore_mu]>;
+def :ItinRW<[A9WriteLM, A9WriteLMAdr, A9WriteB], [IIC_iLoad_mBr, IIC_iPop_Br]>;
+def :ItinRW<[A9WriteL, A9WriteAdr, A9WriteA], [IIC_iLoadiALU]>;
+
+def :ItinRW<[A9WriteLSfp, A9WriteAdr], [IIC_fpLoad32, IIC_fpLoad64]>;
+
+def :ItinRW<[A9WriteLMfp, A9WriteLMAdr], [IIC_fpLoad_m]>;
+def :ItinRW<[A9WriteLMAdr, A9WriteLMfp], [IIC_fpLoad_mu]>;
+def :ItinRW<[A9WriteAdr, A9WriteLSfp], [IIC_fpStore32, IIC_fpStore64,
+                                        IIC_fpStore_m, IIC_fpStore_mu]>;
+
+// Note: Unlike VLDM, VLD1 expects the writeback operand after the
+// normal writes.
+def :ItinRW<[A9WriteLfp1, A9WriteAdr1], [IIC_VLD1, IIC_VLD1u,
+                                         IIC_VLD1x2, IIC_VLD1x2u]>;
+def :ItinRW<[A9WriteLfp2, A9WriteAdr2], [IIC_VLD1x3, IIC_VLD1x3u,
+                                         IIC_VLD1x4, IIC_VLD1x4u,
+                                         IIC_VLD4dup, IIC_VLD4dupu]>;
+def :ItinRW<[A9WriteLfp1Mov, A9WriteAdr1], [IIC_VLD1dup, IIC_VLD1dupu,
+                                            IIC_VLD2, IIC_VLD2u,
+                                            IIC_VLD2dup, IIC_VLD2dupu]>;
+def :ItinRW<[A9WriteLfp2Mov, A9WriteAdr1], [IIC_VLD1ln, IIC_VLD1lnu,
+                                            IIC_VLD2x2, IIC_VLD2x2u,
+                                            IIC_VLD2ln, IIC_VLD2lnu]>;
+def :ItinRW<[A9WriteLfp3Mov, A9WriteAdr3], [IIC_VLD3, IIC_VLD3u,
+                                            IIC_VLD3dup, IIC_VLD3dupu]>;
+def :ItinRW<[A9WriteLfp4Mov, A9WriteAdr4], [IIC_VLD4, IIC_VLD4u,
+                                            IIC_VLD4ln, IIC_VLD4lnu]>;
+def :ItinRW<[A9WriteLfp5Mov, A9WriteAdr5], [IIC_VLD3ln, IIC_VLD3lnu]>;
+
+// Vector stores use similar resources to vector loads, so use the
+// same write types. The address write must be first for stores with
+// address writeback.
+def :ItinRW<[A9WriteAdr1, A9WriteLfp1], [IIC_VST1, IIC_VST1u,
+                                         IIC_VST1x2, IIC_VST1x2u,
+                                         IIC_VST1ln, IIC_VST1lnu,
+                                         IIC_VST2, IIC_VST2u,
+                                         IIC_VST2x2, IIC_VST2x2u,
+                                         IIC_VST2ln, IIC_VST2lnu]>;
+def :ItinRW<[A9WriteAdr2, A9WriteLfp2], [IIC_VST1x3, IIC_VST1x3u,
+                                         IIC_VST1x4, IIC_VST1x4u,
+                                         IIC_VST3, IIC_VST3u,
+                                         IIC_VST3ln, IIC_VST3lnu,
+                                         IIC_VST4, IIC_VST4u,
+                                         IIC_VST4ln, IIC_VST4lnu]>;
+
+// NEON moves.
+def :ItinRW<[A9WriteV2], [IIC_VMOVSI, IIC_VMOVDI, IIC_VMOVD, IIC_VMOVQ]>;
+def :ItinRW<[A9WriteV1], [IIC_VMOV, IIC_VMOVIS, IIC_VMOVID]>;
+def :ItinRW<[A9WriteV3], [IIC_VMOVISL, IIC_VMOVN]>;
+
+// NEON integer arithmetic
+//
+// VADD/VAND/VORR/VEOR/VBIC/VORN/VBIT/VBIF/VBSL
+def :ItinRW<[A9WriteV3, A9Read2, A9Read2], [IIC_VBINiD, IIC_VBINiQ]>;
+// VSUB/VMVN/VCLSD/VCLZD/VCNTD
+def :ItinRW<[A9WriteV3, A9Read2], [IIC_VSUBiD, IIC_VSUBiQ, IIC_VCNTiD]>;
+// VADDL/VSUBL/VNEG are mapped later under IIC_SHLi.
+// ...
+// VHADD/VRHADD/VQADD/VTST/VADH/VRADH
+def :ItinRW<[A9WriteV4, A9Read2, A9Read2], [IIC_VBINi4D, IIC_VBINi4Q]>;
+// VSBH/VRSBH/VHSUB/VQSUB/VABD/VCEQ/VCGE/VCGT/VMAX/VMIN/VPMAX/VPMIN/VABDL
+def :ItinRW<[A9WriteV4, A9Read2], [IIC_VSUBi4D, IIC_VSUBi4Q]>;
+// VQNEG/VQABS
+def :ItinRW<[A9WriteV4], [IIC_VQUNAiD, IIC_VQUNAiQ]>;
+// VABS
+def :ItinRW<[A9WriteV4, A9Read2], [IIC_VUNAiD, IIC_VUNAiQ]>;
+// VPADD/VPADDL are mapped later under IIC_SHLi.
+// ...
+// VCLSQ/VCLZQ/VCNTQ, takes two cycles.
+def :ItinRW<[A9Write2V4, A9Read3], [IIC_VCNTiQ]>;
+// VMOVimm/VMVNimm/VORRimm/VBICimm
+def :ItinRW<[A9WriteV3], [IIC_VMOVImm]>;
+def :ItinRW<[A9WriteV6, A9Read3, A9Read2], [IIC_VABAD, IIC_VABAQ]>;
+def :ItinRW<[A9WriteV6, A9Read3], [IIC_VPALiD, IIC_VPALiQ]>;
+
+// NEON integer multiply
+//
+// Note: these don't quite match the timing docs, but they do match
+// the original A9 itinerary.
+def :ItinRW<[A9WriteV6, A9Read2, A9Read2], [IIC_VMULi16D]>;
+def :ItinRW<[A9WriteV7, A9Read2, A9Read2], [IIC_VMULi16Q]>;
+def :ItinRW<[A9Write2V7, A9Read2], [IIC_VMULi32D]>;
+def :ItinRW<[A9Write2V9, A9Read2], [IIC_VMULi32Q]>;
+def :ItinRW<[A9WriteV6, A9Read3, A9Read2, A9Read2], [IIC_VMACi16D]>;
+def :ItinRW<[A9WriteV7, A9Read3, A9Read2, A9Read2], [IIC_VMACi16Q]>;
+def :ItinRW<[A9Write2V7, A9Read3, A9Read2], [IIC_VMACi32D]>;
+def :ItinRW<[A9Write2V9, A9Read3, A9Read2], [IIC_VMACi32Q]>;
+
+// NEON integer shift
+// TODO: Q,Q,Q shifts should actually reserve FP for 2 cycles.
+def :ItinRW<[A9WriteV3], [IIC_VSHLiD, IIC_VSHLiQ]>;
+def :ItinRW<[A9WriteV4], [IIC_VSHLi4D, IIC_VSHLi4Q]>;
+
+// NEON permute
+def :ItinRW<[A9WriteV2], [IIC_VPERMD, IIC_VPERMQ, IIC_VEXTD]>;
+def :ItinRW<[A9WriteV3, A9WriteV4, ReadDefault, A9Read2],
+            [IIC_VPERMQ3, IIC_VEXTQ]>;
+def :ItinRW<[A9WriteV3, A9Read2], [IIC_VTB1]>;
+def :ItinRW<[A9WriteV3, A9Read2, A9Read2], [IIC_VTB2]>;
+def :ItinRW<[A9WriteV4, A9Read2, A9Read2, A9Read3], [IIC_VTB3]>;
+def :ItinRW<[A9WriteV4, A9Read2, A9Read2, A9Read3, A9Read3], [IIC_VTB4]>;
+def :ItinRW<[A9WriteV3, ReadDefault, A9Read2], [IIC_VTBX1]>;
+def :ItinRW<[A9WriteV3, ReadDefault, A9Read2, A9Read2], [IIC_VTBX2]>;
+def :ItinRW<[A9WriteV4, ReadDefault, A9Read2, A9Read2, A9Read3], [IIC_VTBX3]>;
+def :ItinRW<[A9WriteV4, ReadDefault, A9Read2, A9Read2, A9Read3, A9Read3],
+            [IIC_VTBX4]>;
 
+// NEON floating-point
+def :ItinRW<[A9WriteV5, A9Read2, A9Read2], [IIC_VBIND]>;
+def :ItinRW<[A9WriteV6, A9Read2, A9Read2], [IIC_VBINQ]>;
+def :ItinRW<[A9WriteV5, A9Read2], [IIC_VUNAD, IIC_VFMULD]>;
+def :ItinRW<[A9WriteV6, A9Read2], [IIC_VUNAQ, IIC_VFMULQ]>;
+def :ItinRW<[A9WriteV9, A9Read3, A9Read2], [IIC_VMACD, IIC_VFMACD]>;
+def :ItinRW<[A9WriteV10, A9Read3, A9Read2], [IIC_VMACQ, IIC_VFMACQ]>;
+def :ItinRW<[A9WriteV9, A9Read2, A9Read2], [IIC_VRECSD]>;
+def :ItinRW<[A9WriteV10, A9Read2, A9Read2], [IIC_VRECSQ]>;
+} // SchedModel = CortexA9Model
diff --git a/lib/Target/ARM/ARMScheduleSwift.td b/lib/Target/ARM/ARMScheduleSwift.td
new file mode 100644
index 0000000..e9bc3e0
--- /dev/null
+++ b/lib/Target/ARM/ARMScheduleSwift.td
@@ -0,0 +1,1085 @@
+//=- ARMScheduleSwift.td - Swift Scheduling Definitions -*- tablegen -*----===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the itinerary class data for the Swift processor..
+//
+//===----------------------------------------------------------------------===//
+
+// ===---------------------------------------------------------------------===//
+// This section contains legacy support for itineraries. This is
+// required until SD and PostRA schedulers are replaced by MachineScheduler.
+
+def SW_DIS0 : FuncUnit;
+def SW_DIS1 : FuncUnit;
+def SW_DIS2 : FuncUnit;
+
+def SW_ALU0 : FuncUnit;
+def SW_ALU1 : FuncUnit;
+def SW_LS   : FuncUnit;
+def SW_IDIV : FuncUnit;
+def SW_FDIV : FuncUnit;
+
+// FIXME: Need bypasses.
+// FIXME: Model the multiple stages of IIC_iMOVix2, IIC_iMOVix2addpc, and
+//        IIC_iMOVix2ld better.
+// FIXME: Model the special immediate shifts that are not microcoded.
+// FIXME: Do we need to model the fact that uses of r15 in a micro-op force it
+//        to issue on pipe 1?
+// FIXME: Model the pipelined behavior of CMP / TST instructions.
+// FIXME: Better model the microcode stages of multiply instructions, especially
+//        conditional variants.
+// FIXME: Add preload instruction when it is documented.
+// FIXME: Model non-pipelined nature of FP div / sqrt unit.
+
+def SwiftItineraries : ProcessorItineraries<
+  [SW_DIS0, SW_DIS1, SW_DIS2, SW_ALU0, SW_ALU1, SW_LS, SW_IDIV, SW_FDIV], [], [
+  //
+  // Move instructions, unconditional
+  InstrItinData<IIC_iMOVi   , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                              [1]>,
+  InstrItinData<IIC_iMOVr   , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                              [1]>,
+  InstrItinData<IIC_iMOVsi  , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                              [1]>,
+  InstrItinData<IIC_iMOVsr  , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                              [1]>,
+  InstrItinData<IIC_iMOVix2 , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0, SW_ALU1]>,
+                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                              [2]>,
+  InstrItinData<IIC_iMOVix2addpc,[InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                                  InstrStage<1, [SW_ALU0, SW_ALU1]>,
+                                  InstrStage<1, [SW_ALU0, SW_ALU1]>,
+                                  InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                                 [3]>,
+  InstrItinData<IIC_iMOVix2ld,[InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0, SW_ALU1]>,
+                               InstrStage<1, [SW_ALU0, SW_ALU1]>,
+                               InstrStage<1, [SW_LS]>],
+                              [5]>,
+  //
+  // MVN instructions
+  InstrItinData<IIC_iMVNi   , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                              [1]>,
+  InstrItinData<IIC_iMVNr   , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                              [1]>,
+  InstrItinData<IIC_iMVNsi  , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                              [1]>,
+  InstrItinData<IIC_iMVNsr  , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                              [1]>,
+  //
+  // No operand cycles
+  InstrItinData<IIC_iALUx   , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0, SW_ALU1]>]>,
+  //
+  // Binary Instructions that produce a result
+  InstrItinData<IIC_iALUi , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                             InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                            [1, 1]>,
+  InstrItinData<IIC_iALUr , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                             InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                            [1, 1, 1]>,
+  InstrItinData<IIC_iALUsi, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                             InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                            [2, 1, 1]>,
+  InstrItinData<IIC_iALUsir,[InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                             InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                            [2, 1, 1]>,
+  InstrItinData<IIC_iALUsr, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                             InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                            [2, 1, 1, 1]>,
+  //
+  // Bitwise Instructions that produce a result
+  InstrItinData<IIC_iBITi , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                             InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                            [1, 1]>,
+  InstrItinData<IIC_iBITr , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                             InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                            [1, 1, 1]>,
+  InstrItinData<IIC_iBITsi, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                             InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                            [2, 1, 1]>,
+  InstrItinData<IIC_iBITsr, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                             InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                            [2, 1, 1, 1]>,
+  //
+  // Unary Instructions that produce a result
+
+  // CLZ, RBIT, etc.
+  InstrItinData<IIC_iUNAr , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                             InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                            [1, 1]>,
+
+  // BFC, BFI, UBFX, SBFX
+  InstrItinData<IIC_iUNAsi, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                             InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                            [2, 1]>,
+
+  //
+  // Zero and sign extension instructions
+  InstrItinData<IIC_iEXTr , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                             InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                            [1, 1]>,
+  InstrItinData<IIC_iEXTAr, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                             InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                            [1, 1, 1]>,
+  InstrItinData<IIC_iEXTAsr,[InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                             InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                            [1, 1, 1, 1]>,
+  //
+  // Compare instructions
+  InstrItinData<IIC_iCMPi   , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                              [1]>,
+  InstrItinData<IIC_iCMPr   , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                              [1, 1]>,
+  InstrItinData<IIC_iCMPsi  , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<2, [SW_ALU0, SW_ALU1]>],
+                              [1, 1]>,
+  InstrItinData<IIC_iCMPsr  , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<2, [SW_ALU0, SW_ALU1]>],
+                              [1, 1, 1]>,
+  //
+  // Test instructions
+  InstrItinData<IIC_iTSTi   , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                              [1]>,
+  InstrItinData<IIC_iTSTr   , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                              [1, 1]>,
+  InstrItinData<IIC_iTSTsi  , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<2, [SW_ALU0, SW_ALU1]>],
+                              [1, 1]>,
+  InstrItinData<IIC_iTSTsr  , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<2, [SW_ALU0, SW_ALU1]>],
+                              [1, 1, 1]>,
+  //
+  // Move instructions, conditional
+  // FIXME: Correctly model the extra input dep on the destination.
+  InstrItinData<IIC_iCMOVi  , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                              [1]>,
+  InstrItinData<IIC_iCMOVr  , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                              [1, 1]>,
+  InstrItinData<IIC_iCMOVsi , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                              [1, 1]>,
+  InstrItinData<IIC_iCMOVsr , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                              [2, 1, 1]>,
+  InstrItinData<IIC_iCMOVix2, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0, SW_ALU1]>,
+                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                              [2]>,
+
+  // Integer multiply pipeline
+  //
+  InstrItinData<IIC_iMUL16  , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [3, 1, 1]>,
+  InstrItinData<IIC_iMAC16  , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [3, 1, 1, 1]>,
+  InstrItinData<IIC_iMUL32  , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [4, 1, 1]>,
+  InstrItinData<IIC_iMAC32  , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [4, 1, 1, 1]>,
+  InstrItinData<IIC_iMUL64  , [InstrStage<1, [SW_DIS0], 0>,
+                               InstrStage<1, [SW_DIS1], 0>,
+                               InstrStage<1, [SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0], 1>,
+                               InstrStage<1, [SW_ALU0], 3>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [5, 5, 1, 1]>,
+  InstrItinData<IIC_iMAC64  , [InstrStage<1, [SW_DIS0], 0>,
+                               InstrStage<1, [SW_DIS1], 0>,
+                               InstrStage<1, [SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0], 1>,
+                               InstrStage<1, [SW_ALU0], 1>,
+                               InstrStage<1, [SW_ALU0, SW_ALU1], 3>,
+                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                              [5, 6, 1, 1]>,
+  //
+  // Integer divide
+  InstrItinData<IIC_iDIV  , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                             InstrStage<1, [SW_ALU0], 0>,
+                             InstrStage<14, [SW_IDIV]>],
+                            [14, 1, 1]>,
+
+  // Integer load pipeline
+  // FIXME: The timings are some rough approximations
+  //
+  // Immediate offset
+  InstrItinData<IIC_iLoad_i   , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                                 InstrStage<1, [SW_LS]>],
+                                [3, 1]>,
+  InstrItinData<IIC_iLoad_bh_i, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                                 InstrStage<1, [SW_LS]>],
+                                [3, 1]>,
+  InstrItinData<IIC_iLoad_d_i , [InstrStage<1, [SW_DIS0], 0>,
+                                 InstrStage<1, [SW_DIS1], 0>,
+                                 InstrStage<1, [SW_LS], 1>,
+                                 InstrStage<1, [SW_LS]>],
+                                [3, 4, 1]>,
+  //
+  // Register offset
+  InstrItinData<IIC_iLoad_r   , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                                 InstrStage<1, [SW_LS]>],
+                                [3, 1, 1]>,
+  InstrItinData<IIC_iLoad_bh_r, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                                 InstrStage<1, [SW_LS]>],
+                                [3, 1, 1]>,
+  InstrItinData<IIC_iLoad_d_r , [InstrStage<1, [SW_DIS0], 0>,
+                                 InstrStage<1, [SW_DIS1], 0>,
+                                 InstrStage<1, [SW_DIS2], 0>,
+                                 InstrStage<1, [SW_LS], 1>,
+                                 InstrStage<1, [SW_LS], 3>,
+                                 InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                                [3, 4, 1, 1]>,
+  //
+  // Scaled register offset
+  InstrItinData<IIC_iLoad_si  , [InstrStage<1, [SW_DIS0], 0>,
+                                 InstrStage<1, [SW_DIS1], 0>,
+                                 InstrStage<1, [SW_ALU0, SW_ALU1], 2>,
+                                 InstrStage<1, [SW_LS]>],
+                                [5, 1, 1]>,
+  InstrItinData<IIC_iLoad_bh_si,[InstrStage<1, [SW_DIS0], 0>,
+                                 InstrStage<1, [SW_DIS1], 0>,
+                                 InstrStage<1, [SW_ALU0, SW_ALU1], 2>,
+                                 InstrStage<1, [SW_LS]>],
+                                [5, 1, 1]>,
+  //
+  // Immediate offset with update
+  InstrItinData<IIC_iLoad_iu  , [InstrStage<1, [SW_DIS0], 0>,
+                                 InstrStage<1, [SW_DIS1], 0>,
+                                 InstrStage<1, [SW_ALU0, SW_ALU1], 1>,
+                                 InstrStage<1, [SW_LS]>],
+                                [3, 1, 1]>,
+  InstrItinData<IIC_iLoad_bh_iu,[InstrStage<1, [SW_DIS0], 0>,
+                                 InstrStage<1, [SW_DIS1], 0>,
+                                 InstrStage<1, [SW_ALU0, SW_ALU1], 1>,
+                                 InstrStage<1, [SW_LS]>],
+                                [3, 1, 1]>,
+  //
+  // Register offset with update
+  InstrItinData<IIC_iLoad_ru  , [InstrStage<1, [SW_DIS0], 0>,
+                                 InstrStage<1, [SW_DIS1], 0>,
+                                 InstrStage<1, [SW_ALU0], 1>,
+                                 InstrStage<1, [SW_LS]>],
+                                [3, 1, 1, 1]>,
+  InstrItinData<IIC_iLoad_bh_ru,[InstrStage<1, [SW_DIS0], 0>,
+                                 InstrStage<1, [SW_DIS1], 0>,
+                                 InstrStage<1, [SW_ALU0], 1>,
+                                 InstrStage<1, [SW_LS]>],
+                                [3, 1, 1, 1]>,
+  InstrItinData<IIC_iLoad_d_ru, [InstrStage<1, [SW_DIS0], 0>,
+                                 InstrStage<1, [SW_DIS1], 0>,
+                                 InstrStage<1, [SW_DIS2], 0>,
+                                 InstrStage<1, [SW_ALU0, SW_ALU1], 0>,
+                                 InstrStage<1, [SW_LS], 3>,
+                                 InstrStage<1, [SW_LS], 0>,
+                                 InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                                [3, 4, 1, 1]>,
+  //
+  // Scaled register offset with update
+  InstrItinData<IIC_iLoad_siu , [InstrStage<1, [SW_DIS0], 0>,
+                                 InstrStage<1, [SW_DIS1], 0>,
+                                 InstrStage<1, [SW_DIS2], 0>,
+                                 InstrStage<1, [SW_ALU0, SW_ALU1], 2>,
+                                 InstrStage<1, [SW_LS], 3>,
+                                 InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                                [5, 3, 1, 1]>,
+  InstrItinData<IIC_iLoad_bh_siu,[InstrStage<1, [SW_DIS0], 0>,
+                                  InstrStage<1, [SW_DIS1], 0>,
+                                  InstrStage<1, [SW_DIS2], 0>,
+                                  InstrStage<1, [SW_ALU0, SW_ALU1], 2>,
+                                  InstrStage<1, [SW_LS], 0>,
+                                  InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                                [5, 3, 1, 1]>,
+  //
+  // Load multiple, def is the 5th operand.
+  // FIXME: This assumes 3 to 4 registers.
+  InstrItinData<IIC_iLoad_m  , [InstrStage<1, [SW_DIS0], 0>,
+                                InstrStage<1, [SW_DIS1], 0>,
+                                InstrStage<1, [SW_DIS2], 0>,
+                                InstrStage<1, [SW_ALU0, SW_ALU1], 1>,
+                                InstrStage<1, [SW_LS]>],
+                               [1, 1, 1, 1, 3], [], -1>, // dynamic uops
+
+  //
+  // Load multiple + update, defs are the 1st and 5th operands.
+  InstrItinData<IIC_iLoad_mu , [InstrStage<1, [SW_DIS0], 0>,
+                                InstrStage<1, [SW_DIS1], 0>,
+                                InstrStage<1, [SW_DIS2], 0>,
+                                InstrStage<1, [SW_ALU0, SW_ALU1], 0>,
+                                InstrStage<1, [SW_LS], 3>,
+                                InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                               [2, 1, 1, 1, 3], [], -1>, // dynamic uops
+  //
+  // Load multiple plus branch
+  InstrItinData<IIC_iLoad_mBr, [InstrStage<1, [SW_DIS0], 0>,
+                                InstrStage<1, [SW_DIS1], 0>,
+                                InstrStage<1, [SW_DIS2], 0>,
+                                InstrStage<1, [SW_ALU0, SW_ALU1], 1>,
+                                InstrStage<1, [SW_LS]>],
+                               [1, 1, 1, 1, 3], [], -1>, // dynamic uops
+  //
+  // Pop, def is the 3rd operand.
+  InstrItinData<IIC_iPop  ,    [InstrStage<1, [SW_DIS0], 0>,
+                                InstrStage<1, [SW_DIS1], 0>,
+                                InstrStage<1, [SW_ALU0, SW_ALU1], 1>,
+                                InstrStage<1, [SW_LS]>],
+                               [1, 1, 3], [], -1>, // dynamic uops
+  //
+  // Pop + branch, def is the 3rd operand.
+  InstrItinData<IIC_iPop_Br,   [InstrStage<1, [SW_DIS0], 0>,
+                                InstrStage<1, [SW_DIS1], 0>,
+                                InstrStage<1, [SW_ALU0, SW_ALU1], 1>,
+                                InstrStage<1, [SW_LS]>],
+                               [1, 1, 3], [], -1>, // dynamic uops
+
+  //
+  // iLoadi + iALUr for t2LDRpci_pic.
+  InstrItinData<IIC_iLoadiALU, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                                InstrStage<1, [SW_LS], 3>,
+                                InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                               [4, 1]>,
+
+  // Integer store pipeline
+  ///
+  // Immediate offset
+  InstrItinData<IIC_iStore_i  , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                                 InstrStage<1, [SW_LS]>],
+                                [1, 1]>,
+  InstrItinData<IIC_iStore_bh_i,[InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                                 InstrStage<1, [SW_LS]>],
+                                [1, 1]>,
+  InstrItinData<IIC_iStore_d_i, [InstrStage<1, [SW_DIS0], 0>,
+                                 InstrStage<1, [SW_DIS1], 0>,
+                                 InstrStage<1, [SW_DIS2], 0>,
+                                 InstrStage<1, [SW_LS], 0>,
+                                 InstrStage<1, [SW_ALU0, SW_ALU1], 1>,
+                                 InstrStage<1, [SW_LS]>],
+                                [1, 1]>,
+  //
+  // Register offset
+  InstrItinData<IIC_iStore_r  , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                                 InstrStage<1, [SW_LS]>],
+                                [1, 1, 1]>,
+  InstrItinData<IIC_iStore_bh_r,[InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                                 InstrStage<1, [SW_LS]>],
+                                [1, 1, 1]>,
+  InstrItinData<IIC_iStore_d_r, [InstrStage<1, [SW_DIS0], 0>,
+                                 InstrStage<1, [SW_DIS1], 0>,
+                                 InstrStage<1, [SW_DIS2], 0>,
+                                 InstrStage<1, [SW_LS], 0>,
+                                 InstrStage<1, [SW_ALU0, SW_ALU1], 1>,
+                                 InstrStage<1, [SW_LS]>],
+                                [1, 1, 1]>,
+  //
+  // Scaled register offset
+  InstrItinData<IIC_iStore_si ,  [InstrStage<1, [SW_DIS0], 0>,
+                                  InstrStage<1, [SW_DIS1], 0>,
+                                  InstrStage<1, [SW_ALU0, SW_ALU1], 2>,
+                                  InstrStage<1, [SW_LS]>],
+                                 [1, 1, 1]>,
+  InstrItinData<IIC_iStore_bh_si,[InstrStage<1, [SW_DIS0], 0>,
+                                  InstrStage<1, [SW_DIS1], 0>,
+                                  InstrStage<1, [SW_ALU0, SW_ALU1], 2>,
+                                  InstrStage<1, [SW_LS]>],
+                                 [1, 1, 1]>,
+  //
+  // Immediate offset with update
+  InstrItinData<IIC_iStore_iu ,  [InstrStage<1, [SW_DIS0], 0>,
+                                  InstrStage<1, [SW_DIS1], 0>,
+                                  InstrStage<1, [SW_ALU0, SW_ALU1], 1>,
+                                  InstrStage<1, [SW_LS]>],
+                                 [1, 1, 1]>,
+  InstrItinData<IIC_iStore_bh_iu,[InstrStage<1, [SW_DIS0], 0>,
+                                  InstrStage<1, [SW_DIS1], 0>,
+                                  InstrStage<1, [SW_ALU0, SW_ALU1], 1>,
+                                  InstrStage<1, [SW_LS]>],
+                                 [1, 1, 1]>,
+  //
+  // Register offset with update
+  InstrItinData<IIC_iStore_ru ,  [InstrStage<1, [SW_DIS0], 0>,
+                                  InstrStage<1, [SW_DIS1], 0>,
+                                  InstrStage<1, [SW_ALU0, SW_ALU1], 1>,
+                                  InstrStage<1, [SW_LS]>],
+                                 [1, 1, 1, 1]>,
+  InstrItinData<IIC_iStore_bh_ru,[InstrStage<1, [SW_DIS0], 0>,
+                                  InstrStage<1, [SW_DIS1], 0>,
+                                  InstrStage<1, [SW_ALU0, SW_ALU1], 1>,
+                                  InstrStage<1, [SW_LS]>],
+                                 [1, 1, 1, 1]>,
+  InstrItinData<IIC_iStore_d_ru, [InstrStage<1, [SW_DIS0], 0>,
+                                  InstrStage<1, [SW_DIS1], 0>,
+                                  InstrStage<1, [SW_ALU0, SW_ALU1], 1>,
+                                  InstrStage<1, [SW_LS]>],
+                                 [1, 1, 1, 1]>,
+  //
+  // Scaled register offset with update
+  InstrItinData<IIC_iStore_siu,    [InstrStage<1, [SW_DIS0], 0>,
+                                    InstrStage<1, [SW_DIS1], 0>,
+                                    InstrStage<1, [SW_ALU0, SW_ALU1], 2>,
+                                    InstrStage<1, [SW_LS], 0>,
+                                    InstrStage<1, [SW_ALU0, SW_ALU1], 1>],
+                                   [3, 1, 1, 1]>,
+  InstrItinData<IIC_iStore_bh_siu, [InstrStage<1, [SW_DIS0], 0>,
+                                    InstrStage<1, [SW_DIS1], 0>,
+                                    InstrStage<1, [SW_ALU0, SW_ALU1], 2>,
+                                    InstrStage<1, [SW_LS], 0>,
+                                    InstrStage<1, [SW_ALU0, SW_ALU1], 1>],
+                                   [3, 1, 1, 1]>,
+  //
+  // Store multiple
+  InstrItinData<IIC_iStore_m , [InstrStage<1, [SW_DIS0], 0>,
+                                InstrStage<1, [SW_DIS1], 0>,
+                                InstrStage<1, [SW_DIS2], 0>,
+                                InstrStage<1, [SW_ALU0, SW_ALU1], 1>,
+                                InstrStage<1, [SW_LS], 1>,
+                                InstrStage<1, [SW_ALU0, SW_ALU1], 1>,
+                                InstrStage<1, [SW_LS], 1>,
+                                InstrStage<1, [SW_ALU0, SW_ALU1], 1>,
+                                InstrStage<1, [SW_LS]>],
+                                [], [], -1>, // dynamic uops
+  //
+  // Store multiple + update
+  InstrItinData<IIC_iStore_mu, [InstrStage<1, [SW_DIS0], 0>,
+                                InstrStage<1, [SW_DIS1], 0>,
+                                InstrStage<1, [SW_DIS2], 0>,
+                                InstrStage<1, [SW_ALU0, SW_ALU1], 1>,
+                                InstrStage<1, [SW_LS], 1>,
+                                InstrStage<1, [SW_ALU0, SW_ALU1], 1>,
+                                InstrStage<1, [SW_LS], 1>,
+                                InstrStage<1, [SW_ALU0, SW_ALU1], 1>,
+                                InstrStage<1, [SW_LS]>],
+                               [2], [], -1>, // dynamic uops
+
+  //
+  // Preload
+  InstrItinData<IIC_Preload,   [InstrStage<1, [SW_DIS0], 0>], [1, 1]>,
+
+  // Branch
+  //
+  // no delay slots, so the latency of a branch is unimportant
+  InstrItinData<IIC_Br       , [InstrStage<1, [SW_DIS0], 0>]>,
+
+  // FP Special Register to Integer Register File Move
+  InstrItinData<IIC_fpSTAT , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                              InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                             [1]>,
+  //
+  // Single-precision FP Unary
+  //
+  // Most floating-point moves get issued on ALU0.
+  InstrItinData<IIC_fpUNA32 , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [2, 1]>,
+  //
+  // Double-precision FP Unary
+  InstrItinData<IIC_fpUNA64 , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [2, 1]>,
+
+  //
+  // Single-precision FP Compare
+  InstrItinData<IIC_fpCMP32 , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [1, 1]>,
+  //
+  // Double-precision FP Compare
+  InstrItinData<IIC_fpCMP64 , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [1, 1]>,
+  //
+  // Single to Double FP Convert
+  InstrItinData<IIC_fpCVTSD , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [4, 1]>,
+  //
+  // Double to Single FP Convert
+  InstrItinData<IIC_fpCVTDS , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [4, 1]>,
+
+  //
+  // Single to Half FP Convert
+  InstrItinData<IIC_fpCVTSH , [InstrStage<1, [SW_DIS0], 0>,
+                               InstrStage<1, [SW_DIS1], 0>,
+                               InstrStage<1, [SW_ALU1], 4>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [6, 1]>,
+  //
+  // Half to Single FP Convert
+  InstrItinData<IIC_fpCVTHS , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [4, 1]>,
+
+  //
+  // Single-Precision FP to Integer Convert
+  InstrItinData<IIC_fpCVTSI , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [4, 1]>,
+  //
+  // Double-Precision FP to Integer Convert
+  InstrItinData<IIC_fpCVTDI , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [4, 1]>,
+  //
+  // Integer to Single-Precision FP Convert
+  InstrItinData<IIC_fpCVTIS , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [4, 1]>,
+  //
+  // Integer to Double-Precision FP Convert
+  InstrItinData<IIC_fpCVTID , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [4, 1]>,
+  //
+  // Single-precision FP ALU
+  InstrItinData<IIC_fpALU32 , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [2, 1, 1]>,
+  //
+  // Double-precision FP ALU
+  InstrItinData<IIC_fpALU64 , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [2, 1, 1]>,
+  //
+  // Single-precision FP Multiply
+  InstrItinData<IIC_fpMUL32 , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [4, 1, 1]>,
+  //
+  // Double-precision FP Multiply
+  InstrItinData<IIC_fpMUL64 , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [6, 1, 1]>,
+  //
+  // Single-precision FP MAC
+  InstrItinData<IIC_fpMAC32 , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [8, 1, 1]>,
+  //
+  // Double-precision FP MAC
+  InstrItinData<IIC_fpMAC64 , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [12, 1, 1]>,
+  //
+  // Single-precision Fused FP MAC
+  InstrItinData<IIC_fpFMAC32, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [8, 1, 1]>,
+  //
+  // Double-precision Fused FP MAC
+  InstrItinData<IIC_fpFMAC64, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [12, 1, 1]>,
+  //
+  // Single-precision FP DIV
+  InstrItinData<IIC_fpDIV32 , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1], 0>,
+                               InstrStage<15, [SW_FDIV]>],
+                              [17, 1, 1]>,
+  //
+  // Double-precision FP DIV
+  InstrItinData<IIC_fpDIV64 , [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1], 0>,
+                               InstrStage<30, [SW_FDIV]>],
+                              [32, 1, 1]>,
+  //
+  // Single-precision FP SQRT
+  InstrItinData<IIC_fpSQRT32, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1], 0>,
+                               InstrStage<15, [SW_FDIV]>],
+                              [17, 1]>,
+  //
+  // Double-precision FP SQRT
+  InstrItinData<IIC_fpSQRT64, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1], 0>,
+                               InstrStage<30, [SW_FDIV]>],
+                              [32, 1, 1]>,
+
+  //
+  // Integer to Single-precision Move
+  InstrItinData<IIC_fpMOVIS,  [InstrStage<1, [SW_DIS0], 0>,
+                               InstrStage<1, [SW_DIS1], 0>,
+                               InstrStage<1, [SW_LS], 4>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [6, 1]>,
+  //
+  // Integer to Double-precision Move
+  InstrItinData<IIC_fpMOVID,  [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_LS]>],
+                              [4, 1]>,
+  //
+  // Single-precision to Integer Move
+  InstrItinData<IIC_fpMOVSI,  [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_LS]>],
+                              [3, 1]>,
+  //
+  // Double-precision to Integer Move
+  InstrItinData<IIC_fpMOVDI,  [InstrStage<1, [SW_DIS0], 0>,
+                               InstrStage<1, [SW_DIS1], 0>,
+                               InstrStage<1, [SW_LS], 3>,
+                               InstrStage<1, [SW_LS]>],
+                              [3, 4, 1]>,
+  //
+  // Single-precision FP Load
+  InstrItinData<IIC_fpLoad32, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_LS]>],
+                              [4, 1]>,
+  //
+  // Double-precision FP Load
+  InstrItinData<IIC_fpLoad64, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_LS]>],
+                              [4, 1]>,
+  //
+  // FP Load Multiple
+  // FIXME: Assumes a single Q register.
+  InstrItinData<IIC_fpLoad_m, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_LS]>],
+                              [1, 1, 1, 4], [], -1>, // dynamic uops
+  //
+  // FP Load Multiple + update
+  // FIXME: Assumes a single Q register.
+  InstrItinData<IIC_fpLoad_mu,[InstrStage<1, [SW_DIS0], 0>,
+                               InstrStage<1, [SW_DIS1], 0>,
+                               InstrStage<1, [SW_LS], 4>,
+                               InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                              [2, 1, 1, 1, 4], [], -1>, // dynamic uops
+  //
+  // Single-precision FP Store
+  InstrItinData<IIC_fpStore32,[InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_LS]>],
+                              [1, 1]>,
+  //
+  // Double-precision FP Store
+  InstrItinData<IIC_fpStore64,[InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_LS]>],
+                              [1, 1]>,
+  //
+  // FP Store Multiple
+  // FIXME: Assumes a single Q register.
+  InstrItinData<IIC_fpStore_m,[InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_LS]>],
+                              [1, 1, 1], [], -1>, // dynamic uops
+  //
+  // FP Store Multiple + update
+  // FIXME: Assumes a single Q register.
+  InstrItinData<IIC_fpStore_mu,[InstrStage<1, [SW_DIS0], 0>,
+                                InstrStage<1, [SW_DIS1], 0>,
+                                InstrStage<1, [SW_LS], 4>,
+                                InstrStage<1, [SW_ALU0, SW_ALU1]>],
+                               [2, 1, 1, 1], [], -1>, // dynamic uops
+  // NEON
+  //
+  // Double-register Integer Unary
+  InstrItinData<IIC_VUNAiD,   [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [4, 1]>,
+  //
+  // Quad-register Integer Unary
+  InstrItinData<IIC_VUNAiQ,   [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [4, 1]>,
+  //
+  // Double-register Integer Q-Unary
+  InstrItinData<IIC_VQUNAiD,  [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [4, 1]>,
+  //
+  // Quad-register Integer CountQ-Unary
+  InstrItinData<IIC_VQUNAiQ,  [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [4, 1]>,
+  //
+  // Double-register Integer Binary
+  InstrItinData<IIC_VBINiD,   [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [2, 1, 1]>,
+  //
+  // Quad-register Integer Binary
+  InstrItinData<IIC_VBINiQ,   [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [2, 1, 1]>,
+  //
+  // Double-register Integer Subtract
+  InstrItinData<IIC_VSUBiD,   [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [2, 1, 1]>,
+  //
+  // Quad-register Integer Subtract
+  InstrItinData<IIC_VSUBiQ,   [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [2, 1, 1]>,
+  //
+  // Double-register Integer Shift
+  InstrItinData<IIC_VSHLiD,   [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [2, 1, 1]>,
+  //
+  // Quad-register Integer Shift
+  InstrItinData<IIC_VSHLiQ,   [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [2, 1, 1]>,
+  //
+  // Double-register Integer Shift (4 cycle)
+  InstrItinData<IIC_VSHLi4D,  [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [4, 1, 1]>,
+  //
+  // Quad-register Integer Shift (4 cycle)
+  InstrItinData<IIC_VSHLi4Q,  [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [4, 1, 1]>,
+  //
+  // Double-register Integer Binary (4 cycle)
+  InstrItinData<IIC_VBINi4D,  [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [4, 1, 1]>,
+  //
+  // Quad-register Integer Binary (4 cycle)
+  InstrItinData<IIC_VBINi4Q,  [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [4, 1, 1]>,
+  //
+  // Double-register Integer Subtract (4 cycle)
+  InstrItinData<IIC_VSUBi4D,  [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [4, 1, 1]>,
+  //
+  // Quad-register Integer Subtract (4 cycle)
+  InstrItinData<IIC_VSUBi4Q,  [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [4, 1, 1]>,
+
+  //
+  // Double-register Integer Count
+  InstrItinData<IIC_VCNTiD,   [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [2, 1, 1]>,
+  //
+  // Quad-register Integer Count
+  InstrItinData<IIC_VCNTiQ,   [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [2, 1, 1]>,
+  //
+  // Double-register Absolute Difference and Accumulate
+  InstrItinData<IIC_VABAD,    [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [4, 1, 1, 1]>,
+  //
+  // Quad-register Absolute Difference and Accumulate
+  InstrItinData<IIC_VABAQ,    [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [4, 1, 1, 1]>,
+  //
+  // Double-register Integer Pair Add Long
+  InstrItinData<IIC_VPALiD,   [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [4, 1, 1]>,
+  //
+  // Quad-register Integer Pair Add Long
+  InstrItinData<IIC_VPALiQ,   [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [4, 1, 1]>,
+
+  //
+  // Double-register Integer Multiply (.8, .16)
+  InstrItinData<IIC_VMULi16D, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [4, 1, 1]>,
+  //
+  // Quad-register Integer Multiply (.8, .16)
+  InstrItinData<IIC_VMULi16Q, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [4, 1, 1]>,
+
+  //
+  // Double-register Integer Multiply (.32)
+  InstrItinData<IIC_VMULi32D, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [4, 1, 1]>,
+  //
+  // Quad-register Integer Multiply (.32)
+  InstrItinData<IIC_VMULi32Q, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [4, 1, 1]>,
+  //
+  // Double-register Integer Multiply-Accumulate (.8, .16)
+  InstrItinData<IIC_VMACi16D, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [4, 1, 1, 1]>,
+  //
+  // Double-register Integer Multiply-Accumulate (.32)
+  InstrItinData<IIC_VMACi32D, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [4, 1, 1, 1]>,
+  //
+  // Quad-register Integer Multiply-Accumulate (.8, .16)
+  InstrItinData<IIC_VMACi16Q, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [4, 1, 1, 1]>,
+  //
+  // Quad-register Integer Multiply-Accumulate (.32)
+  InstrItinData<IIC_VMACi32Q, [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [4, 1, 1, 1]>,
+
+  //
+  // Move
+  InstrItinData<IIC_VMOV,     [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [2, 1]>,
+  //
+  // Move Immediate
+  InstrItinData<IIC_VMOVImm,  [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [2]>,
+  //
+  // Double-register Permute Move
+  InstrItinData<IIC_VMOVD,    [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [2, 1]>,
+  //
+  // Quad-register Permute Move
+  InstrItinData<IIC_VMOVQ,    [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [2, 1]>,
+  //
+  // Integer to Single-precision Move
+  InstrItinData<IIC_VMOVIS ,  [InstrStage<1, [SW_DIS0], 0>,
+                               InstrStage<1, [SW_DIS1], 0>,
+                               InstrStage<1, [SW_LS], 4>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [6, 1]>,
+  //
+  // Integer to Double-precision Move
+  InstrItinData<IIC_VMOVID ,  [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_LS]>],
+                              [4, 1, 1]>,
+  //
+  // Single-precision to Integer Move
+  InstrItinData<IIC_VMOVSI ,  [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_LS]>],
+                              [3, 1]>,
+  //
+  // Double-precision to Integer Move
+  InstrItinData<IIC_VMOVDI ,  [InstrStage<1, [SW_DIS0], 0>,
+                               InstrStage<1, [SW_DIS1], 0>,
+                               InstrStage<1, [SW_LS], 3>,
+                               InstrStage<1, [SW_LS]>],
+                              [3, 4, 1]>,
+  //
+  // Integer to Lane Move
+  // FIXME: I think this is correct, but it is not clear from the tuning guide.
+  InstrItinData<IIC_VMOVISL , [InstrStage<1, [SW_DIS0], 0>,
+                               InstrStage<1, [SW_DIS1], 0>,
+                               InstrStage<1, [SW_LS], 4>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [6, 1]>,
+
+  //
+  // Vector narrow move
+  InstrItinData<IIC_VMOVN,    [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [2, 1]>,
+  //
+  // Double-register FP Unary
+  // FIXME: VRECPE / VRSQRTE has a longer latency than VABS, which is used here,
+  //        and they issue on a different pipeline.
+  InstrItinData<IIC_VUNAD,    [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [2, 1]>,
+  //
+  // Quad-register FP Unary
+  // FIXME: VRECPE / VRSQRTE has a longer latency than VABS, which is used here,
+  //        and they issue on a different pipeline.
+  InstrItinData<IIC_VUNAQ,    [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [2, 1]>,
+  //
+  // Double-register FP Binary
+  // FIXME: We're using this itin for many instructions.
+  InstrItinData<IIC_VBIND,    [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [4, 1, 1]>,
+
+  //
+  // VPADD, etc.
+  InstrItinData<IIC_VPBIND,   [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [4, 1, 1]>,
+  //
+  // Double-register FP VMUL
+  InstrItinData<IIC_VFMULD,   [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [4, 1, 1]>,
+  //
+  // Quad-register FP Binary
+  InstrItinData<IIC_VBINQ,    [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU0]>],
+                              [4, 1, 1]>,
+  //
+  // Quad-register FP VMUL
+  InstrItinData<IIC_VFMULQ,   [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [4, 1, 1]>,
+  //
+  // Double-register FP Multiple-Accumulate
+  InstrItinData<IIC_VMACD,    [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [8, 1, 1]>,
+  //
+  // Quad-register FP Multiple-Accumulate
+  InstrItinData<IIC_VMACQ,    [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [8, 1, 1]>,
+  //
+  // Double-register Fused FP Multiple-Accumulate
+  InstrItinData<IIC_VFMACD,   [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [8, 1, 1]>,
+  //
+  // Quad-register FusedF P Multiple-Accumulate
+  InstrItinData<IIC_VFMACQ,   [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [8, 1, 1]>,
+  //
+  // Double-register Reciprical Step
+  InstrItinData<IIC_VRECSD,   [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [8, 1, 1]>,
+  //
+  // Quad-register Reciprical Step
+  InstrItinData<IIC_VRECSQ,   [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [8, 1, 1]>,
+  //
+  // Double-register Permute
+  // FIXME: The latencies are unclear from the documentation.
+  InstrItinData<IIC_VPERMD,   [InstrStage<1, [SW_DIS0], 0>,
+                               InstrStage<1, [SW_DIS1], 0>,
+                               InstrStage<1, [SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1], 2>,
+                               InstrStage<1, [SW_ALU1], 2>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [3, 4, 3, 4]>,
+  //
+  // Quad-register Permute
+  // FIXME: The latencies are unclear from the documentation.
+  InstrItinData<IIC_VPERMQ,   [InstrStage<1, [SW_DIS0], 0>,
+                               InstrStage<1, [SW_DIS1], 0>,
+                               InstrStage<1, [SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1], 2>,
+                               InstrStage<1, [SW_ALU1], 2>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [3, 4, 3, 4]>,
+  //
+  // Quad-register Permute (3 cycle issue on A9)
+  InstrItinData<IIC_VPERMQ3,  [InstrStage<1, [SW_DIS0], 0>,
+                               InstrStage<1, [SW_DIS1], 0>,
+                               InstrStage<1, [SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1], 2>,
+                               InstrStage<1, [SW_ALU1], 2>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [3, 4, 3, 4]>,
+
+  //
+  // Double-register VEXT
+  InstrItinData<IIC_VEXTD,    [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [2, 1, 1]>,
+  //
+  // Quad-register VEXT
+  InstrItinData<IIC_VEXTQ,    [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [2, 1, 1]>,
+  //
+  // VTB
+  InstrItinData<IIC_VTB1,     [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [2, 1, 1]>,
+  InstrItinData<IIC_VTB2,     [InstrStage<1, [SW_DIS0], 0>,
+                               InstrStage<1, [SW_DIS1], 0>,
+                               InstrStage<1, [SW_ALU1], 2>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [4, 1, 3, 3]>,
+  InstrItinData<IIC_VTB3,     [InstrStage<1, [SW_DIS0], 0>,
+                               InstrStage<1, [SW_DIS1], 0>,
+                               InstrStage<1, [SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1], 2>,
+                               InstrStage<1, [SW_ALU1], 2>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [6, 1, 3, 5, 5]>,
+  InstrItinData<IIC_VTB4,     [InstrStage<1, [SW_DIS0], 0>,
+                               InstrStage<1, [SW_DIS1], 0>,
+                               InstrStage<1, [SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1], 2>,
+                               InstrStage<1, [SW_ALU1], 2>,
+                               InstrStage<1, [SW_ALU1], 2>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [8, 1, 3, 5, 7, 7]>,
+  //
+  // VTBX
+  InstrItinData<IIC_VTBX1,    [InstrStage<1, [SW_DIS0, SW_DIS1, SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [2, 1, 1]>,
+  InstrItinData<IIC_VTBX2,    [InstrStage<1, [SW_DIS0], 0>,
+                               InstrStage<1, [SW_DIS1], 0>,
+                               InstrStage<1, [SW_ALU1], 2>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [4, 1, 3, 3]>,
+  InstrItinData<IIC_VTBX3,    [InstrStage<1, [SW_DIS0], 0>,
+                               InstrStage<1, [SW_DIS1], 0>,
+                               InstrStage<1, [SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1], 2>,
+                               InstrStage<1, [SW_ALU1], 2>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [6, 1, 3, 5, 5]>,
+  InstrItinData<IIC_VTBX4,    [InstrStage<1, [SW_DIS0], 0>,
+                               InstrStage<1, [SW_DIS1], 0>,
+                               InstrStage<1, [SW_DIS2], 0>,
+                               InstrStage<1, [SW_ALU1], 2>,
+                               InstrStage<1, [SW_ALU1], 2>,
+                               InstrStage<1, [SW_ALU1], 2>,
+                               InstrStage<1, [SW_ALU1]>],
+                              [8, 1, 3, 5, 7, 7]>
+]>;
+
+// ===---------------------------------------------------------------------===//
+// This following definitions describe the simple machine model which
+// will replace itineraries.
+
+// Swift machine model for scheduling and other instruction cost heuristics.
+def SwiftModel : SchedMachineModel {
+  let IssueWidth = 3; // 3 micro-ops are dispatched per cycle.
+  let MinLatency = 0; // Data dependencies are allowed within dispatch groups.
+  let LoadLatency = 3;
+
+  let Itineraries = SwiftItineraries;
+}
+
+// TODO: Add Swift processor and scheduler resources.
diff --git a/lib/Target/ARM/ARMSelectionDAGInfo.cpp b/lib/Target/ARM/ARMSelectionDAGInfo.cpp
index 31d5d38..b33b3c9 100644
--- a/lib/Target/ARM/ARMSelectionDAGInfo.cpp
+++ b/lib/Target/ARM/ARMSelectionDAGInfo.cpp
@@ -155,7 +155,7 @@ EmitTargetCodeForMemset(SelectionDAG &DAG, DebugLoc dl,
   TargetLowering::ArgListEntry Entry;
 
   // First argument: data pointer
-  Type *IntPtrTy = TLI.getTargetData()->getIntPtrType(*DAG.getContext());
+  Type *IntPtrTy = TLI.getDataLayout()->getIntPtrType(*DAG.getContext());
   Entry.Node = Dst;
   Entry.Ty = IntPtrTy;
   Args.push_back(Entry);
diff --git a/lib/Target/ARM/ARMSubtarget.cpp b/lib/Target/ARM/ARMSubtarget.cpp
index 4762854..bcc9db4 100644
--- a/lib/Target/ARM/ARMSubtarget.cpp
+++ b/lib/Target/ARM/ARMSubtarget.cpp
@@ -13,8 +13,9 @@
 
 #include "ARMSubtarget.h"
 #include "ARMBaseRegisterInfo.h"
+#include "ARMBaseInstrInfo.h"
 #include "llvm/GlobalValue.h"
-#include "llvm/Target/TargetSubtargetInfo.h"
+#include "llvm/Target/TargetInstrInfo.h"
 #include "llvm/Support/CommandLine.h"
 
 #define GET_SUBTARGETINFO_TARGET_DESC
@@ -31,6 +32,10 @@ static cl::opt<bool>
 DarwinUseMOVT("arm-darwin-use-movt", cl::init(true), cl::Hidden);
 
 static cl::opt<bool>
+UseFusedMulOps("arm-use-mulops",
+               cl::init(true), cl::Hidden);
+
+static cl::opt<bool>
 StrictAlign("arm-strict-align", cl::Hidden,
             cl::desc("Disallow all unaligned memory accesses"));
 
@@ -49,6 +54,7 @@ ARMSubtarget::ARMSubtarget(const std::string &TT, const std::string &CPU,
   , HasVFPv4(false)
   , HasNEON(false)
   , UseNEONForSinglePrecisionFP(false)
+  , UseMulOps(UseFusedMulOps)
   , SlowFPVMLx(false)
   , HasVMLxForwarding(false)
   , SlowFPBrcc(false)
@@ -63,6 +69,7 @@ ARMSubtarget::ARMSubtarget(const std::string &TT, const std::string &CPU,
   , HasFP16(false)
   , HasD16(false)
   , HasHardwareDivide(false)
+  , HasHardwareDivideInARM(false)
   , HasT2ExtractPack(false)
   , HasDataBarrier(false)
   , Pref32BitThumb(false)
diff --git a/lib/Target/ARM/ARMSubtarget.h b/lib/Target/ARM/ARMSubtarget.h
index b394061..8e6b650 100644
--- a/lib/Target/ARM/ARMSubtarget.h
+++ b/lib/Target/ARM/ARMSubtarget.h
@@ -30,7 +30,7 @@ class StringRef;
 class ARMSubtarget : public ARMGenSubtargetInfo {
 protected:
   enum ARMProcFamilyEnum {
-    Others, CortexA8, CortexA9
+    Others, CortexA8, CortexA9, CortexA15, Swift
   };
 
   /// ARMProcFamily - ARM processor family: Cortex-A8, Cortex-A9, and others.
@@ -57,6 +57,10 @@ protected:
   /// determine if NEON should actually be used.
   bool UseNEONForSinglePrecisionFP;
 
+  /// UseMulOps - True if non-microcoded fused integer multiply-add and
+  /// multiply-subtract instructions should be used.
+  bool UseMulOps;
+
   /// SlowFPVMLx - If the VFP2 / NEON instructions are available, indicates
   /// whether the FP VML[AS] instructions are slow (if so, don't use them).
   bool SlowFPVMLx;
@@ -107,6 +111,9 @@ protected:
   /// HasHardwareDivide - True if subtarget supports [su]div
   bool HasHardwareDivide;
 
+  /// HasHardwareDivideInARM - True if subtarget supports [su]div in ARM mode
+  bool HasHardwareDivideInARM;
+
   /// HasT2ExtractPack - True if subtarget supports thumb2 extract/pack
   /// instructions.
   bool HasT2ExtractPack;
@@ -199,7 +206,10 @@ protected:
 
   bool isCortexA8() const { return ARMProcFamily == CortexA8; }
   bool isCortexA9() const { return ARMProcFamily == CortexA9; }
+  bool isCortexA15() const { return ARMProcFamily == CortexA15; }
+  bool isSwift()    const { return ARMProcFamily == Swift; }
   bool isCortexM3() const { return CPUString == "cortex-m3"; }
+  bool isLikeA9() const { return isCortexA9() || isCortexA15(); }
 
   bool hasARMOps() const { return !NoARM; }
 
@@ -211,8 +221,10 @@ protected:
     return hasNEON() && UseNEONForSinglePrecisionFP; }
 
   bool hasDivide() const { return HasHardwareDivide; }
+  bool hasDivideInARMMode() const { return HasHardwareDivideInARM; }
   bool hasT2ExtractPack() const { return HasT2ExtractPack; }
   bool hasDataBarrier() const { return HasDataBarrier; }
+  bool useMulOps() const { return UseMulOps; }
   bool useFPVMLx() const { return !SlowFPVMLx; }
   bool hasVMLxForwarding() const { return HasVMLxForwarding; }
   bool isFPBrccSlow() const { return SlowFPBrcc; }
diff --git a/lib/Target/ARM/ARMTargetMachine.cpp b/lib/Target/ARM/ARMTargetMachine.cpp
index 171c9ad..b486d4f 100644
--- a/lib/Target/ARM/ARMTargetMachine.cpp
+++ b/lib/Target/ARM/ARMTargetMachine.cpp
@@ -60,7 +60,7 @@ ARMTargetMachine::ARMTargetMachine(const Target &T, StringRef TT,
                                    CodeGenOpt::Level OL)
   : ARMBaseTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL),
     InstrInfo(Subtarget),
-    DataLayout(Subtarget.isAPCS_ABI() ?
+    DL(Subtarget.isAPCS_ABI() ?
                std::string("e-p:32:32-f64:32:64-i64:32:64-"
                            "v128:32:128-v64:32:64-n32-S32") :
                Subtarget.isAAPCS_ABI() ?
@@ -68,10 +68,10 @@ ARMTargetMachine::ARMTargetMachine(const Target &T, StringRef TT,
                            "v128:64:128-v64:64:64-n32-S64") :
                std::string("e-p:32:32-f64:64:64-i64:64:64-"
                            "v128:64:128-v64:64:64-n32-S32")),
-    ELFWriterInfo(*this),
     TLInfo(*this),
     TSInfo(*this),
-    FrameLowering(Subtarget) {
+    FrameLowering(Subtarget),
+    STTI(&TLInfo), VTTI(&TLInfo) {
   if (!Subtarget.hasARMOps())
     report_fatal_error("CPU: '" + Subtarget.getCPUString() + "' does not "
                        "support ARM mode execution!");
@@ -88,7 +88,7 @@ ThumbTargetMachine::ThumbTargetMachine(const Target &T, StringRef TT,
     InstrInfo(Subtarget.hasThumb2()
               ? ((ARMBaseInstrInfo*)new Thumb2InstrInfo(Subtarget))
               : ((ARMBaseInstrInfo*)new Thumb1InstrInfo(Subtarget))),
-    DataLayout(Subtarget.isAPCS_ABI() ?
+    DL(Subtarget.isAPCS_ABI() ?
                std::string("e-p:32:32-f64:32:64-i64:32:64-"
                            "i16:16:32-i8:8:32-i1:8:32-"
                            "v128:32:128-v64:32:64-a:0:32-n32-S32") :
@@ -99,12 +99,12 @@ ThumbTargetMachine::ThumbTargetMachine(const Target &T, StringRef TT,
                std::string("e-p:32:32-f64:64:64-i64:64:64-"
                            "i16:16:32-i8:8:32-i1:8:32-"
                            "v128:64:128-v64:64:64-a:0:32-n32-S32")),
-    ELFWriterInfo(*this),
     TLInfo(*this),
     TSInfo(*this),
     FrameLowering(Subtarget.hasThumb2()
               ? new ARMFrameLowering(Subtarget)
-              : (ARMFrameLowering*)new Thumb1FrameLowering(Subtarget)) {
+              : (ARMFrameLowering*)new Thumb1FrameLowering(Subtarget)),
+    STTI(&TLInfo), VTTI(&TLInfo) {
 }
 
 namespace {
@@ -143,6 +143,11 @@ bool ARMPassConfig::addPreISel() {
 
 bool ARMPassConfig::addInstSelector() {
   addPass(createARMISelDag(getARMTargetMachine(), getOptLevel()));
+
+  const ARMSubtarget *Subtarget = &getARMSubtarget();
+  if (Subtarget->isTargetELF() && !Subtarget->isThumb1Only() &&
+      TM->Options.EnableFastISel)
+    addPass(createARMGlobalBaseRegPass());
   return false;
 }
 
@@ -150,7 +155,7 @@ bool ARMPassConfig::addPreRegAlloc() {
   // FIXME: temporarily disabling load / store optimization pass for Thumb1.
   if (getOptLevel() != CodeGenOpt::None && !getARMSubtarget().isThumb1Only())
     addPass(createARMLoadStoreOptimizationPass(true));
-  if (getOptLevel() != CodeGenOpt::None && getARMSubtarget().isCortexA9())
+  if (getOptLevel() != CodeGenOpt::None && getARMSubtarget().isLikeA9())
     addPass(createMLxExpansionPass());
   return true;
 }
diff --git a/lib/Target/ARM/ARMTargetMachine.h b/lib/Target/ARM/ARMTargetMachine.h
index abcdb24..ebdd5b4 100644
--- a/lib/Target/ARM/ARMTargetMachine.h
+++ b/lib/Target/ARM/ARMTargetMachine.h
@@ -15,7 +15,6 @@
 #define ARMTARGETMACHINE_H
 
 #include "ARMInstrInfo.h"
-#include "ARMELFWriterInfo.h"
 #include "ARMFrameLowering.h"
 #include "ARMJITInfo.h"
 #include "ARMSubtarget.h"
@@ -25,7 +24,8 @@
 #include "Thumb1FrameLowering.h"
 #include "Thumb2InstrInfo.h"
 #include "llvm/Target/TargetMachine.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/Target/TargetTransformImpl.h"
+#include "llvm/DataLayout.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/ADT/OwningPtr.h"
 
@@ -62,11 +62,12 @@ public:
 class ARMTargetMachine : public ARMBaseTargetMachine {
   virtual void anchor();
   ARMInstrInfo        InstrInfo;
-  const TargetData    DataLayout;       // Calculates type size & alignment
-  ARMELFWriterInfo    ELFWriterInfo;
+  const DataLayout    DL;       // Calculates type size & alignment
   ARMTargetLowering   TLInfo;
   ARMSelectionDAGInfo TSInfo;
   ARMFrameLowering    FrameLowering;
+  ScalarTargetTransformImpl STTI;
+  VectorTargetTransformImpl VTTI;
  public:
   ARMTargetMachine(const Target &T, StringRef TT,
                    StringRef CPU, StringRef FS,
@@ -88,12 +89,14 @@ class ARMTargetMachine : public ARMBaseTargetMachine {
   virtual const ARMFrameLowering *getFrameLowering() const {
     return &FrameLowering;
   }
-
-  virtual const ARMInstrInfo     *getInstrInfo() const { return &InstrInfo; }
-  virtual const TargetData       *getTargetData() const { return &DataLayout; }
-  virtual const ARMELFWriterInfo *getELFWriterInfo() const {
-    return Subtarget.isTargetELF() ? &ELFWriterInfo : 0;
+  virtual const ScalarTargetTransformInfo *getScalarTargetTransformInfo()const {
+    return &STTI;
+  }
+  virtual const VectorTargetTransformInfo *getVectorTargetTransformInfo()const {
+    return &VTTI;
   }
+  virtual const ARMInstrInfo     *getInstrInfo() const { return &InstrInfo; }
+  virtual const DataLayout       *getDataLayout() const { return &DL; }
 };
 
 /// ThumbTargetMachine - Thumb target machine.
@@ -104,12 +107,13 @@ class ThumbTargetMachine : public ARMBaseTargetMachine {
   virtual void anchor();
   // Either Thumb1InstrInfo or Thumb2InstrInfo.
   OwningPtr<ARMBaseInstrInfo> InstrInfo;
-  const TargetData    DataLayout;   // Calculates type size & alignment
-  ARMELFWriterInfo    ELFWriterInfo;
+  const DataLayout    DL;   // Calculates type size & alignment
   ARMTargetLowering   TLInfo;
   ARMSelectionDAGInfo TSInfo;
   // Either Thumb1FrameLowering or ARMFrameLowering.
   OwningPtr<ARMFrameLowering> FrameLowering;
+  ScalarTargetTransformImpl STTI;
+  VectorTargetTransformImpl VTTI;
 public:
   ThumbTargetMachine(const Target &T, StringRef TT,
                      StringRef CPU, StringRef FS,
@@ -138,10 +142,13 @@ public:
   virtual const ARMFrameLowering *getFrameLowering() const {
     return FrameLowering.get();
   }
-  virtual const TargetData       *getTargetData() const { return &DataLayout; }
-  virtual const ARMELFWriterInfo *getELFWriterInfo() const {
-    return Subtarget.isTargetELF() ? &ELFWriterInfo : 0;
+  virtual const ScalarTargetTransformInfo *getScalarTargetTransformInfo()const {
+    return &STTI;
+  }
+  virtual const VectorTargetTransformInfo *getVectorTargetTransformInfo()const {
+    return &VTTI;
   }
+  virtual const DataLayout       *getDataLayout() const { return &DL; }
 };
 
 } // end namespace llvm
diff --git a/lib/Target/ARM/AsmParser/ARMAsmParser.cpp b/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
index 3a5957b..c61e3bd 100644
--- a/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
+++ b/lib/Target/ARM/AsmParser/ARMAsmParser.cpp
@@ -181,49 +181,44 @@ class ARMAsmParser : public MCTargetAsmParser {
   OperandMatchResultTy parseVectorLane(VectorLaneTy &LaneKind, unsigned &Index);
 
   // Asm Match Converter Methods
-  bool cvtT2LdrdPre(MCInst &Inst, unsigned Opcode,
-                    const SmallVectorImpl<MCParsedAsmOperand*> &);
-  bool cvtT2StrdPre(MCInst &Inst, unsigned Opcode,
-                    const SmallVectorImpl<MCParsedAsmOperand*> &);
-  bool cvtLdWriteBackRegT2AddrModeImm8(MCInst &Inst, unsigned Opcode,
+  void cvtT2LdrdPre(MCInst &Inst, const SmallVectorImpl<MCParsedAsmOperand*> &);
+  void cvtT2StrdPre(MCInst &Inst, const SmallVectorImpl<MCParsedAsmOperand*> &);
+  void cvtLdWriteBackRegT2AddrModeImm8(MCInst &Inst,
                                   const SmallVectorImpl<MCParsedAsmOperand*> &);
-  bool cvtStWriteBackRegT2AddrModeImm8(MCInst &Inst, unsigned Opcode,
+  void cvtStWriteBackRegT2AddrModeImm8(MCInst &Inst,
                                   const SmallVectorImpl<MCParsedAsmOperand*> &);
-  bool cvtLdWriteBackRegAddrMode2(MCInst &Inst, unsigned Opcode,
+  void cvtLdWriteBackRegAddrMode2(MCInst &Inst,
                                   const SmallVectorImpl<MCParsedAsmOperand*> &);
-  bool cvtLdWriteBackRegAddrModeImm12(MCInst &Inst, unsigned Opcode,
+  void cvtLdWriteBackRegAddrModeImm12(MCInst &Inst,
                                   const SmallVectorImpl<MCParsedAsmOperand*> &);
-  bool cvtStWriteBackRegAddrModeImm12(MCInst &Inst, unsigned Opcode,
+  void cvtStWriteBackRegAddrModeImm12(MCInst &Inst,
                                   const SmallVectorImpl<MCParsedAsmOperand*> &);
-  bool cvtStWriteBackRegAddrMode2(MCInst &Inst, unsigned Opcode,
+  void cvtStWriteBackRegAddrMode2(MCInst &Inst,
                                   const SmallVectorImpl<MCParsedAsmOperand*> &);
-  bool cvtStWriteBackRegAddrMode3(MCInst &Inst, unsigned Opcode,
+  void cvtStWriteBackRegAddrMode3(MCInst &Inst,
                                   const SmallVectorImpl<MCParsedAsmOperand*> &);
-  bool cvtLdExtTWriteBackImm(MCInst &Inst, unsigned Opcode,
+  void cvtLdExtTWriteBackImm(MCInst &Inst,
                              const SmallVectorImpl<MCParsedAsmOperand*> &);
-  bool cvtLdExtTWriteBackReg(MCInst &Inst, unsigned Opcode,
+  void cvtLdExtTWriteBackReg(MCInst &Inst,
                              const SmallVectorImpl<MCParsedAsmOperand*> &);
-  bool cvtStExtTWriteBackImm(MCInst &Inst, unsigned Opcode,
+  void cvtStExtTWriteBackImm(MCInst &Inst,
                              const SmallVectorImpl<MCParsedAsmOperand*> &);
-  bool cvtStExtTWriteBackReg(MCInst &Inst, unsigned Opcode,
+  void cvtStExtTWriteBackReg(MCInst &Inst,
                              const SmallVectorImpl<MCParsedAsmOperand*> &);
-  bool cvtLdrdPre(MCInst &Inst, unsigned Opcode,
-                  const SmallVectorImpl<MCParsedAsmOperand*> &);
-  bool cvtStrdPre(MCInst &Inst, unsigned Opcode,
-                  const SmallVectorImpl<MCParsedAsmOperand*> &);
-  bool cvtLdWriteBackRegAddrMode3(MCInst &Inst, unsigned Opcode,
+  void cvtLdrdPre(MCInst &Inst, const SmallVectorImpl<MCParsedAsmOperand*> &);
+  void cvtStrdPre(MCInst &Inst, const SmallVectorImpl<MCParsedAsmOperand*> &);
+  void cvtLdWriteBackRegAddrMode3(MCInst &Inst,
                                   const SmallVectorImpl<MCParsedAsmOperand*> &);
-  bool cvtThumbMultiply(MCInst &Inst, unsigned Opcode,
+  void cvtThumbMultiply(MCInst &Inst,
                         const SmallVectorImpl<MCParsedAsmOperand*> &);
-  bool cvtVLDwbFixed(MCInst &Inst, unsigned Opcode,
+  void cvtVLDwbFixed(MCInst &Inst,
                      const SmallVectorImpl<MCParsedAsmOperand*> &);
-  bool cvtVLDwbRegister(MCInst &Inst, unsigned Opcode,
+  void cvtVLDwbRegister(MCInst &Inst,
                         const SmallVectorImpl<MCParsedAsmOperand*> &);
-  bool cvtVSTwbFixed(MCInst &Inst, unsigned Opcode,
+  void cvtVSTwbFixed(MCInst &Inst,
                      const SmallVectorImpl<MCParsedAsmOperand*> &);
-  bool cvtVSTwbRegister(MCInst &Inst, unsigned Opcode,
+  void cvtVSTwbRegister(MCInst &Inst,
                         const SmallVectorImpl<MCParsedAsmOperand*> &);
-
   bool validateInstruction(MCInst &Inst,
                            const SmallVectorImpl<MCParsedAsmOperand*> &Ops);
   bool processInstruction(MCInst &Inst,
@@ -258,15 +253,17 @@ public:
 
   // Implementation of the MCTargetAsmParser interface:
   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc);
-  bool ParseInstruction(StringRef Name, SMLoc NameLoc,
+  bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
+                        SMLoc NameLoc,
                         SmallVectorImpl<MCParsedAsmOperand*> &Operands);
   bool ParseDirective(AsmToken DirectiveID);
 
   unsigned checkTargetMatchPredicate(MCInst &Inst);
 
-  bool MatchAndEmitInstruction(SMLoc IDLoc,
+  bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
                                SmallVectorImpl<MCParsedAsmOperand*> &Operands,
-                               MCStreamer &Out);
+                               MCStreamer &Out, unsigned &ErrorInfo,
+                               bool MatchingInlineAsm);
 };
 } // end anonymous namespace
 
@@ -486,7 +483,8 @@ public:
   SMLoc getStartLoc() const { return StartLoc; }
   /// getEndLoc - Get the location of the last token of this operand.
   SMLoc getEndLoc() const { return EndLoc; }
-
+  /// getLocRange - Get the range between the first and last token of this
+  /// operand.
   SMRange getLocRange() const { return SMRange(StartLoc, EndLoc); }
 
   ARMCC::CondCodes getCondCode() const {
@@ -862,7 +860,7 @@ public:
   bool isSPRRegList() const { return Kind == k_SPRRegisterList; }
   bool isToken() const { return Kind == k_Token; }
   bool isMemBarrierOpt() const { return Kind == k_MemBarrierOpt; }
-  bool isMemory() const { return Kind == k_Memory; }
+  bool isMem() const { return Kind == k_Memory; }
   bool isShifterImm() const { return Kind == k_ShifterImmediate; }
   bool isRegShiftedReg() const { return Kind == k_ShiftedRegister; }
   bool isRegShiftedImm() const { return Kind == k_ShiftedImmediate; }
@@ -873,14 +871,14 @@ public:
     return Kind == k_PostIndexRegister && PostIdxReg.ShiftTy ==ARM_AM::no_shift;
   }
   bool isMemNoOffset(bool alignOK = false) const {
-    if (!isMemory())
+    if (!isMem())
       return false;
     // No offset of any kind.
     return Memory.OffsetRegNum == 0 && Memory.OffsetImm == 0 &&
      (alignOK || Memory.Alignment == 0);
   }
   bool isMemPCRelImm12() const {
-    if (!isMemory() || Memory.OffsetRegNum != 0 || Memory.Alignment != 0)
+    if (!isMem() || Memory.OffsetRegNum != 0 || Memory.Alignment != 0)
       return false;
     // Base register must be PC.
     if (Memory.BaseRegNum != ARM::PC)
@@ -894,7 +892,7 @@ public:
     return isMemNoOffset(true);
   }
   bool isAddrMode2() const {
-    if (!isMemory() || Memory.Alignment != 0) return false;
+    if (!isMem() || Memory.Alignment != 0) return false;
     // Check for register offset.
     if (Memory.OffsetRegNum) return true;
     // Immediate offset in range [-4095, 4095].
@@ -916,7 +914,7 @@ public:
     // and we reject it.
     if (isImm() && !isa<MCConstantExpr>(getImm()))
       return true;
-    if (!isMemory() || Memory.Alignment != 0) return false;
+    if (!isMem() || Memory.Alignment != 0) return false;
     // No shifts are legal for AM3.
     if (Memory.ShiftType != ARM_AM::no_shift) return false;
     // Check for register offset.
@@ -946,7 +944,7 @@ public:
     // and we reject it.
     if (isImm() && !isa<MCConstantExpr>(getImm()))
       return true;
-    if (!isMemory() || Memory.Alignment != 0) return false;
+    if (!isMem() || Memory.Alignment != 0) return false;
     // Check for register offset.
     if (Memory.OffsetRegNum) return false;
     // Immediate offset in range [-1020, 1020] and a multiple of 4.
@@ -956,25 +954,25 @@ public:
       Val == INT32_MIN;
   }
   bool isMemTBB() const {
-    if (!isMemory() || !Memory.OffsetRegNum || Memory.isNegative ||
+    if (!isMem() || !Memory.OffsetRegNum || Memory.isNegative ||
         Memory.ShiftType != ARM_AM::no_shift || Memory.Alignment != 0)
       return false;
     return true;
   }
   bool isMemTBH() const {
-    if (!isMemory() || !Memory.OffsetRegNum || Memory.isNegative ||
+    if (!isMem() || !Memory.OffsetRegNum || Memory.isNegative ||
         Memory.ShiftType != ARM_AM::lsl || Memory.ShiftImm != 1 ||
         Memory.Alignment != 0 )
       return false;
     return true;
   }
   bool isMemRegOffset() const {
-    if (!isMemory() || !Memory.OffsetRegNum || Memory.Alignment != 0)
+    if (!isMem() || !Memory.OffsetRegNum || Memory.Alignment != 0)
       return false;
     return true;
   }
   bool isT2MemRegOffset() const {
-    if (!isMemory() || !Memory.OffsetRegNum || Memory.isNegative ||
+    if (!isMem() || !Memory.OffsetRegNum || Memory.isNegative ||
         Memory.Alignment != 0)
       return false;
     // Only lsl #{0, 1, 2, 3} allowed.
@@ -987,14 +985,14 @@ public:
   bool isMemThumbRR() const {
     // Thumb reg+reg addressing is simple. Just two registers, a base and
     // an offset. No shifts, negations or any other complicating factors.
-    if (!isMemory() || !Memory.OffsetRegNum || Memory.isNegative ||
+    if (!isMem() || !Memory.OffsetRegNum || Memory.isNegative ||
         Memory.ShiftType != ARM_AM::no_shift || Memory.Alignment != 0)
       return false;
     return isARMLowRegister(Memory.BaseRegNum) &&
       (!Memory.OffsetRegNum || isARMLowRegister(Memory.OffsetRegNum));
   }
   bool isMemThumbRIs4() const {
-    if (!isMemory() || Memory.OffsetRegNum != 0 ||
+    if (!isMem() || Memory.OffsetRegNum != 0 ||
         !isARMLowRegister(Memory.BaseRegNum) || Memory.Alignment != 0)
       return false;
     // Immediate offset, multiple of 4 in range [0, 124].
@@ -1003,7 +1001,7 @@ public:
     return Val >= 0 && Val <= 124 && (Val % 4) == 0;
   }
   bool isMemThumbRIs2() const {
-    if (!isMemory() || Memory.OffsetRegNum != 0 ||
+    if (!isMem() || Memory.OffsetRegNum != 0 ||
         !isARMLowRegister(Memory.BaseRegNum) || Memory.Alignment != 0)
       return false;
     // Immediate offset, multiple of 4 in range [0, 62].
@@ -1012,7 +1010,7 @@ public:
     return Val >= 0 && Val <= 62 && (Val % 2) == 0;
   }
   bool isMemThumbRIs1() const {
-    if (!isMemory() || Memory.OffsetRegNum != 0 ||
+    if (!isMem() || Memory.OffsetRegNum != 0 ||
         !isARMLowRegister(Memory.BaseRegNum) || Memory.Alignment != 0)
       return false;
     // Immediate offset in range [0, 31].
@@ -1021,7 +1019,7 @@ public:
     return Val >= 0 && Val <= 31;
   }
   bool isMemThumbSPI() const {
-    if (!isMemory() || Memory.OffsetRegNum != 0 ||
+    if (!isMem() || Memory.OffsetRegNum != 0 ||
         Memory.BaseRegNum != ARM::SP || Memory.Alignment != 0)
       return false;
     // Immediate offset, multiple of 4 in range [0, 1020].
@@ -1035,7 +1033,7 @@ public:
     // and we reject it.
     if (isImm() && !isa<MCConstantExpr>(getImm()))
       return true;
-    if (!isMemory() || Memory.OffsetRegNum != 0 || Memory.Alignment != 0)
+    if (!isMem() || Memory.OffsetRegNum != 0 || Memory.Alignment != 0)
       return false;
     // Immediate offset a multiple of 4 in range [-1020, 1020].
     if (!Memory.OffsetImm) return true;
@@ -1044,7 +1042,7 @@ public:
     return (Val >= -1020 && Val <= 1020 && (Val & 3) == 0) || Val == INT32_MIN;
   }
   bool isMemImm0_1020s4Offset() const {
-    if (!isMemory() || Memory.OffsetRegNum != 0 || Memory.Alignment != 0)
+    if (!isMem() || Memory.OffsetRegNum != 0 || Memory.Alignment != 0)
       return false;
     // Immediate offset a multiple of 4 in range [0, 1020].
     if (!Memory.OffsetImm) return true;
@@ -1052,7 +1050,7 @@ public:
     return Val >= 0 && Val <= 1020 && (Val & 3) == 0;
   }
   bool isMemImm8Offset() const {
-    if (!isMemory() || Memory.OffsetRegNum != 0 || Memory.Alignment != 0)
+    if (!isMem() || Memory.OffsetRegNum != 0 || Memory.Alignment != 0)
       return false;
     // Base reg of PC isn't allowed for these encodings.
     if (Memory.BaseRegNum == ARM::PC) return false;
@@ -1062,7 +1060,7 @@ public:
     return (Val == INT32_MIN) || (Val > -256 && Val < 256);
   }
   bool isMemPosImm8Offset() const {
-    if (!isMemory() || Memory.OffsetRegNum != 0 || Memory.Alignment != 0)
+    if (!isMem() || Memory.OffsetRegNum != 0 || Memory.Alignment != 0)
       return false;
     // Immediate offset in range [0, 255].
     if (!Memory.OffsetImm) return true;
@@ -1070,7 +1068,7 @@ public:
     return Val >= 0 && Val < 256;
   }
   bool isMemNegImm8Offset() const {
-    if (!isMemory() || Memory.OffsetRegNum != 0 || Memory.Alignment != 0)
+    if (!isMem() || Memory.OffsetRegNum != 0 || Memory.Alignment != 0)
       return false;
     // Base reg of PC isn't allowed for these encodings.
     if (Memory.BaseRegNum == ARM::PC) return false;
@@ -1080,7 +1078,7 @@ public:
     return (Val == INT32_MIN) || (Val > -256 && Val < 0);
   }
   bool isMemUImm12Offset() const {
-    if (!isMemory() || Memory.OffsetRegNum != 0 || Memory.Alignment != 0)
+    if (!isMem() || Memory.OffsetRegNum != 0 || Memory.Alignment != 0)
       return false;
     // Immediate offset in range [0, 4095].
     if (!Memory.OffsetImm) return true;
@@ -1094,7 +1092,7 @@ public:
     if (isImm() && !isa<MCConstantExpr>(getImm()))
       return true;
 
-    if (!isMemory() || Memory.OffsetRegNum != 0 || Memory.Alignment != 0)
+    if (!isMem() || Memory.OffsetRegNum != 0 || Memory.Alignment != 0)
       return false;
     // Immediate offset in range [-4095, 4095].
     if (!Memory.OffsetImm) return true;
@@ -3376,7 +3374,8 @@ ARMAsmParser::OperandMatchResultTy ARMAsmParser::
 parseMSRMaskOperand(SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   SMLoc S = Parser.getTok().getLoc();
   const AsmToken &Tok = Parser.getTok();
-  assert(Tok.is(AsmToken::Identifier) && "Token is not an Identifier");
+  if (!Tok.is(AsmToken::Identifier))
+    return MatchOperand_NoMatch;
   StringRef Mask = Tok.getString();
 
   if (isMClass()) {
@@ -3880,8 +3879,8 @@ parseAM3Offset(SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
 /// cvtT2LdrdPre - Convert parsed operands to MCInst.
 /// Needed here because the Asm Gen Matcher can't handle properly tied operands
 /// when they refer multiple MIOperands inside a single one.
-bool ARMAsmParser::
-cvtT2LdrdPre(MCInst &Inst, unsigned Opcode,
+void ARMAsmParser::
+cvtT2LdrdPre(MCInst &Inst,
              const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   // Rt, Rt2
   ((ARMOperand*)Operands[2])->addRegOperands(Inst, 1);
@@ -3892,14 +3891,13 @@ cvtT2LdrdPre(MCInst &Inst, unsigned Opcode,
   ((ARMOperand*)Operands[4])->addMemImm8s4OffsetOperands(Inst, 2);
   // pred
   ((ARMOperand*)Operands[1])->addCondCodeOperands(Inst, 2);
-  return true;
 }
 
 /// cvtT2StrdPre - Convert parsed operands to MCInst.
 /// Needed here because the Asm Gen Matcher can't handle properly tied operands
 /// when they refer multiple MIOperands inside a single one.
-bool ARMAsmParser::
-cvtT2StrdPre(MCInst &Inst, unsigned Opcode,
+void ARMAsmParser::
+cvtT2StrdPre(MCInst &Inst,
              const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   // Create a writeback register dummy placeholder.
   Inst.addOperand(MCOperand::CreateReg(0));
@@ -3910,14 +3908,13 @@ cvtT2StrdPre(MCInst &Inst, unsigned Opcode,
   ((ARMOperand*)Operands[4])->addMemImm8s4OffsetOperands(Inst, 2);
   // pred
   ((ARMOperand*)Operands[1])->addCondCodeOperands(Inst, 2);
-  return true;
 }
 
 /// cvtLdWriteBackRegT2AddrModeImm8 - Convert parsed operands to MCInst.
 /// Needed here because the Asm Gen Matcher can't handle properly tied operands
 /// when they refer multiple MIOperands inside a single one.
-bool ARMAsmParser::
-cvtLdWriteBackRegT2AddrModeImm8(MCInst &Inst, unsigned Opcode,
+void ARMAsmParser::
+cvtLdWriteBackRegT2AddrModeImm8(MCInst &Inst,
                          const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   ((ARMOperand*)Operands[2])->addRegOperands(Inst, 1);
 
@@ -3926,28 +3923,26 @@ cvtLdWriteBackRegT2AddrModeImm8(MCInst &Inst, unsigned Opcode,
 
   ((ARMOperand*)Operands[3])->addMemImm8OffsetOperands(Inst, 2);
   ((ARMOperand*)Operands[1])->addCondCodeOperands(Inst, 2);
-  return true;
 }
 
 /// cvtStWriteBackRegT2AddrModeImm8 - Convert parsed operands to MCInst.
 /// Needed here because the Asm Gen Matcher can't handle properly tied operands
 /// when they refer multiple MIOperands inside a single one.
-bool ARMAsmParser::
-cvtStWriteBackRegT2AddrModeImm8(MCInst &Inst, unsigned Opcode,
+void ARMAsmParser::
+cvtStWriteBackRegT2AddrModeImm8(MCInst &Inst,
                          const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   // Create a writeback register dummy placeholder.
   Inst.addOperand(MCOperand::CreateImm(0));
   ((ARMOperand*)Operands[2])->addRegOperands(Inst, 1);
   ((ARMOperand*)Operands[3])->addMemImm8OffsetOperands(Inst, 2);
   ((ARMOperand*)Operands[1])->addCondCodeOperands(Inst, 2);
-  return true;
 }
 
 /// cvtLdWriteBackRegAddrMode2 - Convert parsed operands to MCInst.
 /// Needed here because the Asm Gen Matcher can't handle properly tied operands
 /// when they refer multiple MIOperands inside a single one.
-bool ARMAsmParser::
-cvtLdWriteBackRegAddrMode2(MCInst &Inst, unsigned Opcode,
+void ARMAsmParser::
+cvtLdWriteBackRegAddrMode2(MCInst &Inst,
                          const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   ((ARMOperand*)Operands[2])->addRegOperands(Inst, 1);
 
@@ -3956,14 +3951,13 @@ cvtLdWriteBackRegAddrMode2(MCInst &Inst, unsigned Opcode,
 
   ((ARMOperand*)Operands[3])->addAddrMode2Operands(Inst, 3);
   ((ARMOperand*)Operands[1])->addCondCodeOperands(Inst, 2);
-  return true;
 }
 
 /// cvtLdWriteBackRegAddrModeImm12 - Convert parsed operands to MCInst.
 /// Needed here because the Asm Gen Matcher can't handle properly tied operands
 /// when they refer multiple MIOperands inside a single one.
-bool ARMAsmParser::
-cvtLdWriteBackRegAddrModeImm12(MCInst &Inst, unsigned Opcode,
+void ARMAsmParser::
+cvtLdWriteBackRegAddrModeImm12(MCInst &Inst,
                          const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   ((ARMOperand*)Operands[2])->addRegOperands(Inst, 1);
 
@@ -3972,57 +3966,53 @@ cvtLdWriteBackRegAddrModeImm12(MCInst &Inst, unsigned Opcode,
 
   ((ARMOperand*)Operands[3])->addMemImm12OffsetOperands(Inst, 2);
   ((ARMOperand*)Operands[1])->addCondCodeOperands(Inst, 2);
-  return true;
 }
 
 
 /// cvtStWriteBackRegAddrModeImm12 - Convert parsed operands to MCInst.
 /// Needed here because the Asm Gen Matcher can't handle properly tied operands
 /// when they refer multiple MIOperands inside a single one.
-bool ARMAsmParser::
-cvtStWriteBackRegAddrModeImm12(MCInst &Inst, unsigned Opcode,
+void ARMAsmParser::
+cvtStWriteBackRegAddrModeImm12(MCInst &Inst,
                          const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   // Create a writeback register dummy placeholder.
   Inst.addOperand(MCOperand::CreateImm(0));
   ((ARMOperand*)Operands[2])->addRegOperands(Inst, 1);
   ((ARMOperand*)Operands[3])->addMemImm12OffsetOperands(Inst, 2);
   ((ARMOperand*)Operands[1])->addCondCodeOperands(Inst, 2);
-  return true;
 }
 
 /// cvtStWriteBackRegAddrMode2 - Convert parsed operands to MCInst.
 /// Needed here because the Asm Gen Matcher can't handle properly tied operands
 /// when they refer multiple MIOperands inside a single one.
-bool ARMAsmParser::
-cvtStWriteBackRegAddrMode2(MCInst &Inst, unsigned Opcode,
+void ARMAsmParser::
+cvtStWriteBackRegAddrMode2(MCInst &Inst,
                          const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   // Create a writeback register dummy placeholder.
   Inst.addOperand(MCOperand::CreateImm(0));
   ((ARMOperand*)Operands[2])->addRegOperands(Inst, 1);
   ((ARMOperand*)Operands[3])->addAddrMode2Operands(Inst, 3);
   ((ARMOperand*)Operands[1])->addCondCodeOperands(Inst, 2);
-  return true;
 }
 
 /// cvtStWriteBackRegAddrMode3 - Convert parsed operands to MCInst.
 /// Needed here because the Asm Gen Matcher can't handle properly tied operands
 /// when they refer multiple MIOperands inside a single one.
-bool ARMAsmParser::
-cvtStWriteBackRegAddrMode3(MCInst &Inst, unsigned Opcode,
+void ARMAsmParser::
+cvtStWriteBackRegAddrMode3(MCInst &Inst,
                          const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   // Create a writeback register dummy placeholder.
   Inst.addOperand(MCOperand::CreateImm(0));
   ((ARMOperand*)Operands[2])->addRegOperands(Inst, 1);
   ((ARMOperand*)Operands[3])->addAddrMode3Operands(Inst, 3);
   ((ARMOperand*)Operands[1])->addCondCodeOperands(Inst, 2);
-  return true;
 }
 
 /// cvtLdExtTWriteBackImm - Convert parsed operands to MCInst.
 /// Needed here because the Asm Gen Matcher can't handle properly tied operands
 /// when they refer multiple MIOperands inside a single one.
-bool ARMAsmParser::
-cvtLdExtTWriteBackImm(MCInst &Inst, unsigned Opcode,
+void ARMAsmParser::
+cvtLdExtTWriteBackImm(MCInst &Inst,
                       const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   // Rt
   ((ARMOperand*)Operands[2])->addRegOperands(Inst, 1);
@@ -4034,14 +4024,13 @@ cvtLdExtTWriteBackImm(MCInst &Inst, unsigned Opcode,
   ((ARMOperand*)Operands[4])->addPostIdxImm8Operands(Inst, 1);
   // pred
   ((ARMOperand*)Operands[1])->addCondCodeOperands(Inst, 2);
-  return true;
 }
 
 /// cvtLdExtTWriteBackReg - Convert parsed operands to MCInst.
 /// Needed here because the Asm Gen Matcher can't handle properly tied operands
 /// when they refer multiple MIOperands inside a single one.
-bool ARMAsmParser::
-cvtLdExtTWriteBackReg(MCInst &Inst, unsigned Opcode,
+void ARMAsmParser::
+cvtLdExtTWriteBackReg(MCInst &Inst,
                       const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   // Rt
   ((ARMOperand*)Operands[2])->addRegOperands(Inst, 1);
@@ -4053,14 +4042,13 @@ cvtLdExtTWriteBackReg(MCInst &Inst, unsigned Opcode,
   ((ARMOperand*)Operands[4])->addPostIdxRegOperands(Inst, 2);
   // pred
   ((ARMOperand*)Operands[1])->addCondCodeOperands(Inst, 2);
-  return true;
 }
 
 /// cvtStExtTWriteBackImm - Convert parsed operands to MCInst.
 /// Needed here because the Asm Gen Matcher can't handle properly tied operands
 /// when they refer multiple MIOperands inside a single one.
-bool ARMAsmParser::
-cvtStExtTWriteBackImm(MCInst &Inst, unsigned Opcode,
+void ARMAsmParser::
+cvtStExtTWriteBackImm(MCInst &Inst,
                       const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   // Create a writeback register dummy placeholder.
   Inst.addOperand(MCOperand::CreateImm(0));
@@ -4072,14 +4060,13 @@ cvtStExtTWriteBackImm(MCInst &Inst, unsigned Opcode,
   ((ARMOperand*)Operands[4])->addPostIdxImm8Operands(Inst, 1);
   // pred
   ((ARMOperand*)Operands[1])->addCondCodeOperands(Inst, 2);
-  return true;
 }
 
 /// cvtStExtTWriteBackReg - Convert parsed operands to MCInst.
 /// Needed here because the Asm Gen Matcher can't handle properly tied operands
 /// when they refer multiple MIOperands inside a single one.
-bool ARMAsmParser::
-cvtStExtTWriteBackReg(MCInst &Inst, unsigned Opcode,
+void ARMAsmParser::
+cvtStExtTWriteBackReg(MCInst &Inst,
                       const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   // Create a writeback register dummy placeholder.
   Inst.addOperand(MCOperand::CreateImm(0));
@@ -4091,14 +4078,13 @@ cvtStExtTWriteBackReg(MCInst &Inst, unsigned Opcode,
   ((ARMOperand*)Operands[4])->addPostIdxRegOperands(Inst, 2);
   // pred
   ((ARMOperand*)Operands[1])->addCondCodeOperands(Inst, 2);
-  return true;
 }
 
 /// cvtLdrdPre - Convert parsed operands to MCInst.
 /// Needed here because the Asm Gen Matcher can't handle properly tied operands
 /// when they refer multiple MIOperands inside a single one.
-bool ARMAsmParser::
-cvtLdrdPre(MCInst &Inst, unsigned Opcode,
+void ARMAsmParser::
+cvtLdrdPre(MCInst &Inst,
            const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   // Rt, Rt2
   ((ARMOperand*)Operands[2])->addRegOperands(Inst, 1);
@@ -4109,14 +4095,13 @@ cvtLdrdPre(MCInst &Inst, unsigned Opcode,
   ((ARMOperand*)Operands[4])->addAddrMode3Operands(Inst, 3);
   // pred
   ((ARMOperand*)Operands[1])->addCondCodeOperands(Inst, 2);
-  return true;
 }
 
 /// cvtStrdPre - Convert parsed operands to MCInst.
 /// Needed here because the Asm Gen Matcher can't handle properly tied operands
 /// when they refer multiple MIOperands inside a single one.
-bool ARMAsmParser::
-cvtStrdPre(MCInst &Inst, unsigned Opcode,
+void ARMAsmParser::
+cvtStrdPre(MCInst &Inst,
            const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   // Create a writeback register dummy placeholder.
   Inst.addOperand(MCOperand::CreateImm(0));
@@ -4127,40 +4112,27 @@ cvtStrdPre(MCInst &Inst, unsigned Opcode,
   ((ARMOperand*)Operands[4])->addAddrMode3Operands(Inst, 3);
   // pred
   ((ARMOperand*)Operands[1])->addCondCodeOperands(Inst, 2);
-  return true;
 }
 
 /// cvtLdWriteBackRegAddrMode3 - Convert parsed operands to MCInst.
 /// Needed here because the Asm Gen Matcher can't handle properly tied operands
 /// when they refer multiple MIOperands inside a single one.
-bool ARMAsmParser::
-cvtLdWriteBackRegAddrMode3(MCInst &Inst, unsigned Opcode,
+void ARMAsmParser::
+cvtLdWriteBackRegAddrMode3(MCInst &Inst,
                          const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   ((ARMOperand*)Operands[2])->addRegOperands(Inst, 1);
   // Create a writeback register dummy placeholder.
   Inst.addOperand(MCOperand::CreateImm(0));
   ((ARMOperand*)Operands[3])->addAddrMode3Operands(Inst, 3);
   ((ARMOperand*)Operands[1])->addCondCodeOperands(Inst, 2);
-  return true;
 }
 
-/// cvtThumbMultiple- Convert parsed operands to MCInst.
+/// cvtThumbMultiply - Convert parsed operands to MCInst.
 /// Needed here because the Asm Gen Matcher can't handle properly tied operands
 /// when they refer multiple MIOperands inside a single one.
-bool ARMAsmParser::
-cvtThumbMultiply(MCInst &Inst, unsigned Opcode,
+void ARMAsmParser::
+cvtThumbMultiply(MCInst &Inst,
            const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
-  // The second source operand must be the same register as the destination
-  // operand.
-  if (Operands.size() == 6 &&
-      (((ARMOperand*)Operands[3])->getReg() !=
-       ((ARMOperand*)Operands[5])->getReg()) &&
-      (((ARMOperand*)Operands[3])->getReg() !=
-       ((ARMOperand*)Operands[4])->getReg())) {
-    Error(Operands[3]->getStartLoc(),
-          "destination register must match source register");
-    return false;
-  }
   ((ARMOperand*)Operands[3])->addRegOperands(Inst, 1);
   ((ARMOperand*)Operands[1])->addCCOutOperands(Inst, 1);
   // If we have a three-operand form, make sure to set Rn to be the operand
@@ -4173,12 +4145,10 @@ cvtThumbMultiply(MCInst &Inst, unsigned Opcode,
   ((ARMOperand*)Operands[RegOp])->addRegOperands(Inst, 1);
   Inst.addOperand(Inst.getOperand(0));
   ((ARMOperand*)Operands[2])->addCondCodeOperands(Inst, 2);
-
-  return true;
 }
 
-bool ARMAsmParser::
-cvtVLDwbFixed(MCInst &Inst, unsigned Opcode,
+void ARMAsmParser::
+cvtVLDwbFixed(MCInst &Inst,
               const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   // Vd
   ((ARMOperand*)Operands[3])->addVecListOperands(Inst, 1);
@@ -4188,11 +4158,10 @@ cvtVLDwbFixed(MCInst &Inst, unsigned Opcode,
   ((ARMOperand*)Operands[4])->addAlignedMemoryOperands(Inst, 2);
   // pred
   ((ARMOperand*)Operands[1])->addCondCodeOperands(Inst, 2);
-  return true;
 }
 
-bool ARMAsmParser::
-cvtVLDwbRegister(MCInst &Inst, unsigned Opcode,
+void ARMAsmParser::
+cvtVLDwbRegister(MCInst &Inst,
                  const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   // Vd
   ((ARMOperand*)Operands[3])->addVecListOperands(Inst, 1);
@@ -4204,11 +4173,10 @@ cvtVLDwbRegister(MCInst &Inst, unsigned Opcode,
   ((ARMOperand*)Operands[5])->addRegOperands(Inst, 1);
   // pred
   ((ARMOperand*)Operands[1])->addCondCodeOperands(Inst, 2);
-  return true;
 }
 
-bool ARMAsmParser::
-cvtVSTwbFixed(MCInst &Inst, unsigned Opcode,
+void ARMAsmParser::
+cvtVSTwbFixed(MCInst &Inst,
               const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   // Create a writeback register dummy placeholder.
   Inst.addOperand(MCOperand::CreateImm(0));
@@ -4218,11 +4186,10 @@ cvtVSTwbFixed(MCInst &Inst, unsigned Opcode,
   ((ARMOperand*)Operands[3])->addVecListOperands(Inst, 1);
   // pred
   ((ARMOperand*)Operands[1])->addCondCodeOperands(Inst, 2);
-  return true;
 }
 
-bool ARMAsmParser::
-cvtVSTwbRegister(MCInst &Inst, unsigned Opcode,
+void ARMAsmParser::
+cvtVSTwbRegister(MCInst &Inst,
                  const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   // Create a writeback register dummy placeholder.
   Inst.addOperand(MCOperand::CreateImm(0));
@@ -4234,7 +4201,6 @@ cvtVSTwbRegister(MCInst &Inst, unsigned Opcode,
   ((ARMOperand*)Operands[3])->addVecListOperands(Inst, 1);
   // pred
   ((ARMOperand*)Operands[1])->addCondCodeOperands(Inst, 2);
-  return true;
 }
 
 /// Parse an ARM memory expression, return false if successful else return true
@@ -4471,6 +4437,12 @@ bool ARMAsmParser::parseMemRegOffsetShift(ARM_AM::ShiftOpc &St,
         ((St == ARM_AM::lsl || St == ARM_AM::ror) && Imm > 31) ||
         ((St == ARM_AM::lsr || St == ARM_AM::asr) && Imm > 32))
       return Error(Loc, "immediate shift value out of range");
+    // If <ShiftTy> #0, turn it into a no_shift.
+    if (Imm == 0)
+      St = ARM_AM::lsl;
+    // For consistency, treat lsr #32 and asr #32 as having immediate value 0.
+    if (Imm == 32)
+      Imm = 0;
     Amount = Imm;
   }
 
@@ -4648,7 +4620,7 @@ bool ARMAsmParser::parseOperand(SmallVectorImpl<MCParsedAsmOperand*> &Operands,
       return true;
 
     const MCExpr *ExprVal = ARMMCExpr::Create(RefKind, SubExprVal,
-                                                   getContext());
+                                              getContext());
     E = SMLoc::getFromPointer(Parser.getTok().getLoc().getPointer() - 1);
     Operands.push_back(ARMOperand::CreateImm(ExprVal, S, E));
     return false;
@@ -4983,7 +4955,8 @@ static bool doesIgnoreDataTypeSuffix(StringRef Mnemonic, StringRef DT) {
 
 static void applyMnemonicAliases(StringRef &Mnemonic, unsigned Features);
 /// Parse an arm instruction mnemonic followed by its operands.
-bool ARMAsmParser::ParseInstruction(StringRef Name, SMLoc NameLoc,
+bool ARMAsmParser::ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
+                                    SMLoc NameLoc,
                                SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   // Apply mnemonic aliases before doing anything else, as the destination
   // mnemnonic may include suffices and we want to handle them normally.
@@ -5377,6 +5350,25 @@ validateInstruction(MCInst &Inst,
                    "in register list");
     break;
   }
+  case ARM::tMUL: {
+    // The second source operand must be the same register as the destination
+    // operand.
+    //
+    // In this case, we must directly check the parsed operands because the
+    // cvtThumbMultiply() function is written in such a way that it guarantees
+    // this first statement is always true for the new Inst.  Essentially, the
+    // destination is unconditionally copied into the second source operand
+    // without checking to see if it matches what we actually parsed.
+    if (Operands.size() == 6 &&
+        (((ARMOperand*)Operands[3])->getReg() !=
+         ((ARMOperand*)Operands[5])->getReg()) &&
+        (((ARMOperand*)Operands[3])->getReg() !=
+         ((ARMOperand*)Operands[4])->getReg())) {
+      return Error(Operands[3]->getStartLoc(),
+                   "destination register must match source register");
+    }
+    break;
+  }
   // Like for ldm/stm, push and pop have hi-reg handling version in Thumb2,
   // so only issue a diagnostic for thumb1. The instructions will be
   // switched to the t2 encodings in processInstruction() if necessary.
@@ -5678,6 +5670,20 @@ bool ARMAsmParser::
 processInstruction(MCInst &Inst,
                    const SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   switch (Inst.getOpcode()) {
+  // Alias for alternate form of 'ADR Rd, #imm' instruction.
+  case ARM::ADDri: {
+    if (Inst.getOperand(1).getReg() != ARM::PC ||
+        Inst.getOperand(5).getReg() != 0)
+      return false;
+    MCInst TmpInst;
+    TmpInst.setOpcode(ARM::ADR);
+    TmpInst.addOperand(Inst.getOperand(0));
+    TmpInst.addOperand(Inst.getOperand(2));
+    TmpInst.addOperand(Inst.getOperand(3));
+    TmpInst.addOperand(Inst.getOperand(4));
+    Inst = TmpInst;
+    return true;
+  }
   // Aliases for alternate PC+imm syntax of LDR instructions.
   case ARM::t2LDRpcrel:
     Inst.setOpcode(ARM::t2LDRpci);
@@ -7471,13 +7477,14 @@ unsigned ARMAsmParser::checkTargetMatchPredicate(MCInst &Inst) {
 
 static const char *getSubtargetFeatureName(unsigned Val);
 bool ARMAsmParser::
-MatchAndEmitInstruction(SMLoc IDLoc,
+MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
                         SmallVectorImpl<MCParsedAsmOperand*> &Operands,
-                        MCStreamer &Out) {
+                        MCStreamer &Out, unsigned &ErrorInfo,
+                        bool MatchingInlineAsm) {
   MCInst Inst;
-  unsigned ErrorInfo;
   unsigned MatchResult;
-  MatchResult = MatchInstructionImpl(Operands, Inst, ErrorInfo);
+  MatchResult = MatchInstructionImpl(Operands, Inst, ErrorInfo,
+                                     MatchingInlineAsm);
   switch (MatchResult) {
   default: break;
   case Match_Success:
@@ -7540,9 +7547,6 @@ MatchAndEmitInstruction(SMLoc IDLoc,
   case Match_MnemonicFail:
     return Error(IDLoc, "invalid instruction",
                  ((ARMOperand*)Operands[0])->getLocRange());
-  case Match_ConversionFail:
-    // The converter function will have already emitted a diagnostic.
-    return true;
   case Match_RequiresNotITBlock:
     return Error(IDLoc, "flag setting instruction only valid outside IT block");
   case Match_RequiresITBlock:
diff --git a/lib/Target/ARM/CMakeLists.txt b/lib/Target/ARM/CMakeLists.txt
index ac916cc..377bd92 100644
--- a/lib/Target/ARM/CMakeLists.txt
+++ b/lib/Target/ARM/CMakeLists.txt
@@ -22,7 +22,6 @@ add_llvm_target(ARMCodeGen
   ARMCodeEmitter.cpp
   ARMConstantIslandPass.cpp
   ARMConstantPoolValue.cpp
-  ARMELFWriterInfo.cpp
   ARMExpandPseudoInsts.cpp
   ARMFastISel.cpp
   ARMFrameLowering.cpp
diff --git a/lib/Target/ARM/Disassembler/ARMDisassembler.cpp b/lib/Target/ARM/Disassembler/ARMDisassembler.cpp
index c90751d..f00142d 100644
--- a/lib/Target/ARM/Disassembler/ARMDisassembler.cpp
+++ b/lib/Target/ARM/Disassembler/ARMDisassembler.cpp
@@ -525,8 +525,9 @@ static bool tryAddingSymbolicOperand(uint64_t Address, int32_t Value,
     else
        ReferenceType = LLVMDisassembler_ReferenceType_InOut_None;
     const char *ReferenceName;
-    const char *Name = SymbolLookUp(DisInfo, Value, &ReferenceType, Address,
-                                    &ReferenceName);
+    uint64_t SymbolValue = 0x00000000ffffffffULL & Value;
+    const char *Name = SymbolLookUp(DisInfo, SymbolValue, &ReferenceType,
+                                    Address, &ReferenceName);
     if (Name) {
       SymbolicOp.AddSymbol.Name = Name;
       SymbolicOp.AddSymbol.Present = true;
@@ -1523,6 +1524,8 @@ DecodeAddrMode2IdxInstruction(MCInst &Inst, unsigned Insn,
         return MCDisassembler::Fail;
     }
     unsigned amt = fieldFromInstruction(Insn, 7, 5);
+    if (Opc == ARM_AM::ror && amt == 0)
+      Opc = ARM_AM::rrx;
     unsigned imm = ARM_AM::getAM2Opc(Op, amt, Opc, idx_mode);
 
     Inst.addOperand(MCOperand::CreateImm(imm));
@@ -1564,6 +1567,9 @@ static DecodeStatus DecodeSORegMemOperand(MCInst &Inst, unsigned Val,
       break;
   }
 
+  if (ShOp == ARM_AM::ror && imm == 0)
+    ShOp = ARM_AM::rrx;
+
   if (!Check(S, DecodeGPRRegisterClass(Inst, Rn, Address, Decoder)))
     return MCDisassembler::Fail;
   if (!Check(S, DecodeGPRRegisterClass(Inst, Rm, Address, Decoder)))
@@ -2089,16 +2095,28 @@ static DecodeStatus DecodeAddrMode7Operand(MCInst &Inst, unsigned Val,
 static DecodeStatus
 DecodeT2BInstruction(MCInst &Inst, unsigned Insn,
                      uint64_t Address, const void *Decoder) {
-  DecodeStatus S = MCDisassembler::Success;
-  unsigned imm = (fieldFromInstruction(Insn, 0, 11) << 0) |
-                 (fieldFromInstruction(Insn, 11, 1) << 18) |
-                 (fieldFromInstruction(Insn, 13, 1) << 17) |
-                 (fieldFromInstruction(Insn, 16, 6) << 11) |
-                 (fieldFromInstruction(Insn, 26, 1) << 19);
-  if (!tryAddingSymbolicOperand(Address, Address + SignExtend32<20>(imm<<1) + 4,
+  DecodeStatus Status = MCDisassembler::Success;
+
+  // Note the J1 and J2 values are from the encoded instruction.  So here
+  // change them to I1 and I2 values via as documented:
+  // I1 = NOT(J1 EOR S);
+  // I2 = NOT(J2 EOR S);
+  // and build the imm32 with one trailing zero as documented:
+  // imm32 = SignExtend(S:I1:I2:imm10:imm11:'0', 32);
+  unsigned S = fieldFromInstruction(Insn, 26, 1);
+  unsigned J1 = fieldFromInstruction(Insn, 13, 1);
+  unsigned J2 = fieldFromInstruction(Insn, 11, 1);
+  unsigned I1 = !(J1 ^ S);
+  unsigned I2 = !(J2 ^ S);
+  unsigned imm10 = fieldFromInstruction(Insn, 16, 10);
+  unsigned imm11 = fieldFromInstruction(Insn, 0, 11);
+  unsigned tmp = (S << 23) | (I1 << 22) | (I2 << 21) | (imm10 << 11) | imm11;
+  int imm32 = SignExtend32<24>(tmp << 1);
+  if (!tryAddingSymbolicOperand(Address, Address + imm32 + 4,
                                 true, 4, Inst, Decoder))
-    Inst.addOperand(MCOperand::CreateImm(SignExtend32<20>(imm << 1)));
-  return S;
+    Inst.addOperand(MCOperand::CreateImm(imm32));
+
+  return Status;
 }
 
 static DecodeStatus
@@ -2701,6 +2719,8 @@ static DecodeStatus DecodeVLD1DupInstruction(MCInst &Inst, unsigned Insn,
   unsigned align = fieldFromInstruction(Insn, 4, 1);
   unsigned size = fieldFromInstruction(Insn, 6, 2);
 
+  if (size == 0 && align == 1)
+    return MCDisassembler::Fail;
   align *= (1 << size);
 
   switch (Inst.getOpcode()) {
@@ -2831,6 +2851,8 @@ static DecodeStatus DecodeVLD4DupInstruction(MCInst &Inst, unsigned Insn,
   unsigned align = fieldFromInstruction(Insn, 4, 1);
 
   if (size == 0x3) {
+    if (align == 0)
+      return MCDisassembler::Fail;
     size = 4;
     align = 16;
   } else {
@@ -3170,7 +3192,7 @@ static DecodeStatus DecodeT2Imm8S4(MCInst &Inst, unsigned Val,
     int imm = Val & 0xFF;
 
     if (!(Val & 0x100)) imm *= -1;
-    Inst.addOperand(MCOperand::CreateImm(imm << 2));
+    Inst.addOperand(MCOperand::CreateImm(imm * 4));
   }
 
   return MCDisassembler::Success;
@@ -3710,8 +3732,16 @@ static DecodeStatus DecodeVLD1LN(MCInst &Inst, unsigned Insn,
       if (fieldFromInstruction(Insn, 6, 1))
         return MCDisassembler::Fail; // UNDEFINED
       index = fieldFromInstruction(Insn, 7, 1);
-      if (fieldFromInstruction(Insn, 4, 2) != 0)
-        align = 4;
+
+      switch (fieldFromInstruction(Insn, 4, 2)) {
+        case 0 :
+          align = 0; break;
+        case 3:
+          align = 4; break;
+        default:
+          return MCDisassembler::Fail;
+      }
+      break;
   }
 
   if (!Check(S, DecodeDPRRegisterClass(Inst, Rd, Address, Decoder)))
@@ -3769,8 +3799,16 @@ static DecodeStatus DecodeVST1LN(MCInst &Inst, unsigned Insn,
       if (fieldFromInstruction(Insn, 6, 1))
         return MCDisassembler::Fail; // UNDEFINED
       index = fieldFromInstruction(Insn, 7, 1);
-      if (fieldFromInstruction(Insn, 4, 2) != 0)
-        align = 4;
+
+      switch (fieldFromInstruction(Insn, 4, 2)) {
+        case 0: 
+          align = 0; break;
+        case 3:
+          align = 4; break;
+        default:
+          return MCDisassembler::Fail;
+      }
+      break;
   }
 
   if (Rm != 0xF) { // Writeback
@@ -4090,8 +4128,15 @@ static DecodeStatus DecodeVLD4LN(MCInst &Inst, unsigned Insn,
         inc = 2;
       break;
     case 2:
-      if (fieldFromInstruction(Insn, 4, 2))
-        align = 4 << fieldFromInstruction(Insn, 4, 2);
+      switch (fieldFromInstruction(Insn, 4, 2)) {
+        case 0:
+          align = 0; break;
+        case 3:
+          return MCDisassembler::Fail;
+        default:
+          align = 4 << fieldFromInstruction(Insn, 4, 2); break;
+      }
+
       index = fieldFromInstruction(Insn, 7, 1);
       if (fieldFromInstruction(Insn, 6, 1))
         inc = 2;
@@ -4164,8 +4209,15 @@ static DecodeStatus DecodeVST4LN(MCInst &Inst, unsigned Insn,
         inc = 2;
       break;
     case 2:
-      if (fieldFromInstruction(Insn, 4, 2))
-        align = 4 << fieldFromInstruction(Insn, 4, 2);
+      switch (fieldFromInstruction(Insn, 4, 2)) {
+        case 0:
+          align = 0; break;
+        case 3:
+          return MCDisassembler::Fail;
+        default:
+          align = 4 << fieldFromInstruction(Insn, 4, 2); break;
+      }
+
       index = fieldFromInstruction(Insn, 7, 1);
       if (fieldFromInstruction(Insn, 6, 1))
         inc = 2;
diff --git a/lib/Target/ARM/InstPrinter/ARMInstPrinter.cpp b/lib/Target/ARM/InstPrinter/ARMInstPrinter.cpp
index 8b9109e..dcc41d9 100644
--- a/lib/Target/ARM/InstPrinter/ARMInstPrinter.cpp
+++ b/lib/Target/ARM/InstPrinter/ARMInstPrinter.cpp
@@ -29,11 +29,33 @@ using namespace llvm;
 ///
 /// getSORegOffset returns an integer from 0-31, representing '32' as 0.
 static unsigned translateShiftImm(unsigned imm) {
+  // lsr #32 and asr #32 exist, but should be encoded as a 0.
+  assert((imm & ~0x1f) == 0 && "Invalid shift encoding");
+
   if (imm == 0)
     return 32;
   return imm;
 }
 
+/// Prints the shift value with an immediate value.
+static void printRegImmShift(raw_ostream &O, ARM_AM::ShiftOpc ShOpc,
+                          unsigned ShImm, bool UseMarkup) {
+  if (ShOpc == ARM_AM::no_shift || (ShOpc == ARM_AM::lsl && !ShImm))
+    return;
+  O << ", ";
+
+  assert (!(ShOpc == ARM_AM::ror && !ShImm) && "Cannot have ror #0");
+  O << getShiftOpcStr(ShOpc);
+
+  if (ShOpc != ARM_AM::rrx) {
+    O << " ";
+    if (UseMarkup)
+      O << "<imm:";
+    O << "#" << translateShiftImm(ShImm);
+    if (UseMarkup)
+      O << ">";
+  }
+}
 
 ARMInstPrinter::ARMInstPrinter(const MCAsmInfo &MAI,
                                const MCInstrInfo &MII,
@@ -45,7 +67,9 @@ ARMInstPrinter::ARMInstPrinter(const MCAsmInfo &MAI,
 }
 
 void ARMInstPrinter::printRegName(raw_ostream &OS, unsigned RegNo) const {
-  OS << getRegisterName(RegNo);
+  OS << markup("<reg:")
+     << getRegisterName(RegNo)
+     << markup(">");
 }
 
 void ARMInstPrinter::printInst(const MCInst *MI, raw_ostream &O,
@@ -85,10 +109,13 @@ void ARMInstPrinter::printInst(const MCInst *MI, raw_ostream &O,
     printSBitModifierOperand(MI, 6, O);
     printPredicateOperand(MI, 4, O);
 
-    O << '\t' << getRegisterName(Dst.getReg())
-      << ", " << getRegisterName(MO1.getReg());
+    O << '\t';
+    printRegName(O, Dst.getReg());
+    O << ", ";
+    printRegName(O, MO1.getReg());
 
-    O << ", " << getRegisterName(MO2.getReg());
+    O << ", ";
+    printRegName(O, MO2.getReg());
     assert(ARM_AM::getSORegOffset(MO3.getImm()) == 0);
     printAnnotation(O, Annot);
     return;
@@ -104,15 +131,20 @@ void ARMInstPrinter::printInst(const MCInst *MI, raw_ostream &O,
     printSBitModifierOperand(MI, 5, O);
     printPredicateOperand(MI, 3, O);
 
-    O << '\t' << getRegisterName(Dst.getReg())
-      << ", " << getRegisterName(MO1.getReg());
+    O << '\t';
+    printRegName(O, Dst.getReg());
+    O << ", ";
+    printRegName(O, MO1.getReg());
 
     if (ARM_AM::getSORegShOp(MO2.getImm()) == ARM_AM::rrx) {
       printAnnotation(O, Annot);
       return;
     }
 
-    O << ", #" << translateShiftImm(ARM_AM::getSORegOffset(MO2.getImm()));
+    O << ", "
+      << markup("<imm:")
+      << "#" << translateShiftImm(ARM_AM::getSORegOffset(MO2.getImm()))
+      << markup(">");
     printAnnotation(O, Annot);
     return;
   }
@@ -136,7 +168,9 @@ void ARMInstPrinter::printInst(const MCInst *MI, raw_ostream &O,
       MI->getOperand(3).getImm() == -4) {
     O << '\t' << "push";
     printPredicateOperand(MI, 4, O);
-    O << "\t{" << getRegisterName(MI->getOperand(1).getReg()) << "}";
+    O << "\t{";
+    printRegName(O, MI->getOperand(1).getReg());
+    O << "}";
     printAnnotation(O, Annot);
     return;
   }
@@ -159,7 +193,9 @@ void ARMInstPrinter::printInst(const MCInst *MI, raw_ostream &O,
       MI->getOperand(4).getImm() == 4) {
     O << '\t' << "pop";
     printPredicateOperand(MI, 5, O);
-    O << "\t{" << getRegisterName(MI->getOperand(0).getReg()) << "}";
+    O << "\t{";
+    printRegName(O, MI->getOperand(0).getReg());
+    O << "}";
     printAnnotation(O, Annot);
     return;
   }
@@ -198,7 +234,8 @@ void ARMInstPrinter::printInst(const MCInst *MI, raw_ostream &O,
     O << "\tldm";
 
     printPredicateOperand(MI, 1, O);
-    O << '\t' << getRegisterName(BaseReg);
+    O << '\t';
+    printRegName(O, BaseReg);
     if (Writeback) O << "!";
     O << ", ";
     printRegisterList(MI, 3, O);
@@ -224,9 +261,11 @@ void ARMInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
   const MCOperand &Op = MI->getOperand(OpNo);
   if (Op.isReg()) {
     unsigned Reg = Op.getReg();
-    O << getRegisterName(Reg);
+    printRegName(O, Reg);
   } else if (Op.isImm()) {
-    O << '#' << Op.getImm();
+    O << markup("<imm:")
+      << '#' << Op.getImm()
+      << markup(">");
   } else {
     assert(Op.isExpr() && "unknown operand kind in printOperand");
     // If a symbolic branch target was added as a constant expression then print
@@ -244,13 +283,16 @@ void ARMInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
   }
 }
 
-void ARMInstPrinter::printT2LdrLabelOperand(const MCInst *MI, unsigned OpNum,
-                                       raw_ostream &O) {
+void ARMInstPrinter::printThumbLdrLabelOperand(const MCInst *MI, unsigned OpNum,
+                                               raw_ostream &O) {
   const MCOperand &MO1 = MI->getOperand(OpNum);
   if (MO1.isExpr())
     O << *MO1.getExpr();
-  else if (MO1.isImm())
-    O << "[pc, #" << MO1.getImm() << "]";
+  else if (MO1.isImm()) {
+    O << markup("<mem:") << "[pc, "
+      << markup("<imm:") << "#" << MO1.getImm()
+      << markup(">]>", "]");
+  }
   else
     llvm_unreachable("Unknown LDR label operand?");
 }
@@ -266,7 +308,7 @@ void ARMInstPrinter::printSORegRegOperand(const MCInst *MI, unsigned OpNum,
   const MCOperand &MO2 = MI->getOperand(OpNum+1);
   const MCOperand &MO3 = MI->getOperand(OpNum+2);
 
-  O << getRegisterName(MO1.getReg());
+  printRegName(O, MO1.getReg());
 
   // Print the shift opc.
   ARM_AM::ShiftOpc ShOpc = ARM_AM::getSORegShOp(MO3.getImm());
@@ -274,7 +316,8 @@ void ARMInstPrinter::printSORegRegOperand(const MCInst *MI, unsigned OpNum,
   if (ShOpc == ARM_AM::rrx)
     return;
 
-  O << ' ' << getRegisterName(MO2.getReg());
+  O << ' ';
+  printRegName(O, MO2.getReg());
   assert(ARM_AM::getSORegOffset(MO3.getImm()) == 0);
 }
 
@@ -283,14 +326,11 @@ void ARMInstPrinter::printSORegImmOperand(const MCInst *MI, unsigned OpNum,
   const MCOperand &MO1 = MI->getOperand(OpNum);
   const MCOperand &MO2 = MI->getOperand(OpNum+1);
 
-  O << getRegisterName(MO1.getReg());
+  printRegName(O, MO1.getReg());
 
   // Print the shift opc.
-  ARM_AM::ShiftOpc ShOpc = ARM_AM::getSORegShOp(MO2.getImm());
-  O << ", " << ARM_AM::getShiftOpcStr(ShOpc);
-  if (ShOpc == ARM_AM::rrx)
-    return;
-  O << " #" << translateShiftImm(ARM_AM::getSORegOffset(MO2.getImm()));
+  printRegImmShift(O, ARM_AM::getSORegShOp(MO2.getImm()),
+                   ARM_AM::getSORegOffset(MO2.getImm()), UseMarkup);
 }
 
 
@@ -304,67 +344,51 @@ void ARMInstPrinter::printAM2PreOrOffsetIndexOp(const MCInst *MI, unsigned Op,
   const MCOperand &MO2 = MI->getOperand(Op+1);
   const MCOperand &MO3 = MI->getOperand(Op+2);
 
-  O << "[" << getRegisterName(MO1.getReg());
+  O << markup("<mem:") << "[";
+  printRegName(O, MO1.getReg());
 
   if (!MO2.getReg()) {
-    if (ARM_AM::getAM2Offset(MO3.getImm())) // Don't print +0.
-      O << ", #"
+    if (ARM_AM::getAM2Offset(MO3.getImm())) { // Don't print +0.
+      O << ", "
+        << markup("<imm:")
+        << "#"
         << ARM_AM::getAddrOpcStr(ARM_AM::getAM2Op(MO3.getImm()))
-        << ARM_AM::getAM2Offset(MO3.getImm());
-    O << "]";
-    return;
-  }
-
-  O << ", "
-    << ARM_AM::getAddrOpcStr(ARM_AM::getAM2Op(MO3.getImm()))
-    << getRegisterName(MO2.getReg());
-
-  if (unsigned ShImm = ARM_AM::getAM2Offset(MO3.getImm()))
-    O << ", "
-    << ARM_AM::getShiftOpcStr(ARM_AM::getAM2ShiftOpc(MO3.getImm()))
-    << " #" << ShImm;
-  O << "]";
-}
-
-void ARMInstPrinter::printAM2PostIndexOp(const MCInst *MI, unsigned Op,
-                                         raw_ostream &O) {
-  const MCOperand &MO1 = MI->getOperand(Op);
-  const MCOperand &MO2 = MI->getOperand(Op+1);
-  const MCOperand &MO3 = MI->getOperand(Op+2);
-
-  O << "[" << getRegisterName(MO1.getReg()) << "], ";
-
-  if (!MO2.getReg()) {
-    unsigned ImmOffs = ARM_AM::getAM2Offset(MO3.getImm());
-    O << '#'
-      << ARM_AM::getAddrOpcStr(ARM_AM::getAM2Op(MO3.getImm()))
-      << ImmOffs;
+        << ARM_AM::getAM2Offset(MO3.getImm())
+        << markup(">");
+    }
+    O << "]" << markup(">");
     return;
   }
 
-  O << ARM_AM::getAddrOpcStr(ARM_AM::getAM2Op(MO3.getImm()))
-    << getRegisterName(MO2.getReg());
+  O << ", ";
+  O << ARM_AM::getAddrOpcStr(ARM_AM::getAM2Op(MO3.getImm()));
+  printRegName(O, MO2.getReg());
 
-  if (unsigned ShImm = ARM_AM::getAM2Offset(MO3.getImm()))
-    O << ", "
-    << ARM_AM::getShiftOpcStr(ARM_AM::getAM2ShiftOpc(MO3.getImm()))
-    << " #" << ShImm;
+  printRegImmShift(O, ARM_AM::getAM2ShiftOpc(MO3.getImm()),
+                   ARM_AM::getAM2Offset(MO3.getImm()), UseMarkup);
+  O << "]" << markup(">");
 }
 
 void ARMInstPrinter::printAddrModeTBB(const MCInst *MI, unsigned Op,
                                            raw_ostream &O) {
   const MCOperand &MO1 = MI->getOperand(Op);
   const MCOperand &MO2 = MI->getOperand(Op+1);
-  O << "[" << getRegisterName(MO1.getReg()) << ", "
-    << getRegisterName(MO2.getReg()) << "]";
+  O << markup("<mem:") << "[";
+  printRegName(O, MO1.getReg());
+  O << ", ";
+  printRegName(O, MO2.getReg());
+  O << "]" << markup(">");
 }
 
 void ARMInstPrinter::printAddrModeTBH(const MCInst *MI, unsigned Op,
                                            raw_ostream &O) {
   const MCOperand &MO1 = MI->getOperand(Op);
   const MCOperand &MO2 = MI->getOperand(Op+1);
-  O << "[" << getRegisterName(MO1.getReg()) << ", "
-    << getRegisterName(MO2.getReg()) << ", lsl #1]";
+  O << markup("<mem:") << "[";
+  printRegName(O, MO1.getReg());
+  O << ", ";
+  printRegName(O, MO2.getReg());
+  O << ", lsl " << markup("<imm:") << "#1" << markup(">") << "]" << markup(">");
 }
 
 void ARMInstPrinter::printAddrMode2Operand(const MCInst *MI, unsigned Op,
@@ -376,13 +400,13 @@ void ARMInstPrinter::printAddrMode2Operand(const MCInst *MI, unsigned Op,
     return;
   }
 
+#ifndef NDEBUG
   const MCOperand &MO3 = MI->getOperand(Op+2);
   unsigned IdxMode = ARM_AM::getAM2IdxMode(MO3.getImm());
+  assert(IdxMode != ARMII::IndexModePost &&
+         "Should be pre or offset index op");
+#endif
 
-  if (IdxMode == ARMII::IndexModePost) {
-    printAM2PostIndexOp(MI, Op, O);
-    return;
-  }
   printAM2PreOrOffsetIndexOp(MI, Op, O);
 }
 
@@ -394,19 +418,18 @@ void ARMInstPrinter::printAddrMode2OffsetOperand(const MCInst *MI,
 
   if (!MO1.getReg()) {
     unsigned ImmOffs = ARM_AM::getAM2Offset(MO2.getImm());
-    O << '#'
-      << ARM_AM::getAddrOpcStr(ARM_AM::getAM2Op(MO2.getImm()))
-      << ImmOffs;
+    O << markup("<imm:")
+      << '#' << ARM_AM::getAddrOpcStr(ARM_AM::getAM2Op(MO2.getImm()))
+      << ImmOffs
+      << markup(">");
     return;
   }
 
-  O << ARM_AM::getAddrOpcStr(ARM_AM::getAM2Op(MO2.getImm()))
-    << getRegisterName(MO1.getReg());
+  O << ARM_AM::getAddrOpcStr(ARM_AM::getAM2Op(MO2.getImm()));
+  printRegName(O, MO1.getReg());
 
-  if (unsigned ShImm = ARM_AM::getAM2Offset(MO2.getImm()))
-    O << ", "
-    << ARM_AM::getShiftOpcStr(ARM_AM::getAM2ShiftOpc(MO2.getImm()))
-    << " #" << ShImm;
+  printRegImmShift(O, ARM_AM::getAM2ShiftOpc(MO2.getImm()),
+                   ARM_AM::getAM2Offset(MO2.getImm()), UseMarkup);
 }
 
 //===--------------------------------------------------------------------===//
@@ -419,18 +442,22 @@ void ARMInstPrinter::printAM3PostIndexOp(const MCInst *MI, unsigned Op,
   const MCOperand &MO2 = MI->getOperand(Op+1);
   const MCOperand &MO3 = MI->getOperand(Op+2);
 
-  O << "[" << getRegisterName(MO1.getReg()) << "], ";
+  O << markup("<mem:") << "[";
+  printRegName(O, MO1.getReg());
+  O << "], " << markup(">");
 
   if (MO2.getReg()) {
-    O << (char)ARM_AM::getAM3Op(MO3.getImm())
-    << getRegisterName(MO2.getReg());
+    O << (char)ARM_AM::getAM3Op(MO3.getImm());
+    printRegName(O, MO2.getReg());
     return;
   }
 
   unsigned ImmOffs = ARM_AM::getAM3Offset(MO3.getImm());
-  O << '#'
+  O << markup("<imm:")
+    << '#'
     << ARM_AM::getAddrOpcStr(ARM_AM::getAM3Op(MO3.getImm()))
-    << ImmOffs;
+    << ImmOffs
+    << markup(">");
 }
 
 void ARMInstPrinter::printAM3PreOrOffsetIndexOp(const MCInst *MI, unsigned Op,
@@ -439,23 +466,29 @@ void ARMInstPrinter::printAM3PreOrOffsetIndexOp(const MCInst *MI, unsigned Op,
   const MCOperand &MO2 = MI->getOperand(Op+1);
   const MCOperand &MO3 = MI->getOperand(Op+2);
 
-  O << '[' << getRegisterName(MO1.getReg());
+  O << markup("<mem:") << '[';
+  printRegName(O, MO1.getReg());
 
   if (MO2.getReg()) {
-    O << ", " << getAddrOpcStr(ARM_AM::getAM3Op(MO3.getImm()))
-      << getRegisterName(MO2.getReg()) << ']';
+    O << ", " << getAddrOpcStr(ARM_AM::getAM3Op(MO3.getImm()));
+    printRegName(O, MO2.getReg());
+    O << ']' << markup(">");
     return;
   }
 
-  //If the op is sub we have to print the immediate even if it is 0 
+  //If the op is sub we have to print the immediate even if it is 0
   unsigned ImmOffs = ARM_AM::getAM3Offset(MO3.getImm());
   ARM_AM::AddrOpc op = ARM_AM::getAM3Op(MO3.getImm());
- 
-  if (ImmOffs || (op == ARM_AM::sub))
-    O << ", #"
+
+  if (ImmOffs || (op == ARM_AM::sub)) {
+    O << ", "
+      << markup("<imm:")
+      << "#"
       << ARM_AM::getAddrOpcStr(op)
-      << ImmOffs;
-  O << ']';
+      << ImmOffs
+      << markup(">");
+  }
+  O << ']' << markup(">");
 }
 
 void ARMInstPrinter::printAddrMode3Operand(const MCInst *MI, unsigned Op,
@@ -483,15 +516,15 @@ void ARMInstPrinter::printAddrMode3OffsetOperand(const MCInst *MI,
   const MCOperand &MO2 = MI->getOperand(OpNum+1);
 
   if (MO1.getReg()) {
-    O << getAddrOpcStr(ARM_AM::getAM3Op(MO2.getImm()))
-      << getRegisterName(MO1.getReg());
+    O << getAddrOpcStr(ARM_AM::getAM3Op(MO2.getImm()));
+    printRegName(O, MO1.getReg());
     return;
   }
 
   unsigned ImmOffs = ARM_AM::getAM3Offset(MO2.getImm());
-  O << '#'
-    << ARM_AM::getAddrOpcStr(ARM_AM::getAM3Op(MO2.getImm()))
-    << ImmOffs;
+  O << markup("<imm:")
+    << '#' << ARM_AM::getAddrOpcStr(ARM_AM::getAM3Op(MO2.getImm())) << ImmOffs
+    << markup(">");
 }
 
 void ARMInstPrinter::printPostIdxImm8Operand(const MCInst *MI,
@@ -499,7 +532,9 @@ void ARMInstPrinter::printPostIdxImm8Operand(const MCInst *MI,
                                              raw_ostream &O) {
   const MCOperand &MO = MI->getOperand(OpNum);
   unsigned Imm = MO.getImm();
-  O << '#' << ((Imm & 256) ? "" : "-") << (Imm & 0xff);
+  O << markup("<imm:")
+    << '#' << ((Imm & 256) ? "" : "-") << (Imm & 0xff)
+    << markup(">");
 }
 
 void ARMInstPrinter::printPostIdxRegOperand(const MCInst *MI, unsigned OpNum,
@@ -507,7 +542,8 @@ void ARMInstPrinter::printPostIdxRegOperand(const MCInst *MI, unsigned OpNum,
   const MCOperand &MO1 = MI->getOperand(OpNum);
   const MCOperand &MO2 = MI->getOperand(OpNum+1);
 
-  O << (MO2.getImm() ? "" : "-") << getRegisterName(MO1.getReg());
+  O << (MO2.getImm() ? "" : "-");
+  printRegName(O, MO1.getReg());
 }
 
 void ARMInstPrinter::printPostIdxImm8s4Operand(const MCInst *MI,
@@ -515,7 +551,9 @@ void ARMInstPrinter::printPostIdxImm8s4Operand(const MCInst *MI,
                                              raw_ostream &O) {
   const MCOperand &MO = MI->getOperand(OpNum);
   unsigned Imm = MO.getImm();
-  O << '#' << ((Imm & 256) ? "" : "-") << ((Imm & 0xff) << 2);
+  O << markup("<imm:")
+    << '#' << ((Imm & 256) ? "" : "-") << ((Imm & 0xff) << 2)
+    << markup(">");
 }
 
 
@@ -536,16 +574,20 @@ void ARMInstPrinter::printAddrMode5Operand(const MCInst *MI, unsigned OpNum,
     return;
   }
 
-  O << "[" << getRegisterName(MO1.getReg());
+  O << markup("<mem:") << "[";
+  printRegName(O, MO1.getReg());
 
   unsigned ImmOffs = ARM_AM::getAM5Offset(MO2.getImm());
   unsigned Op = ARM_AM::getAM5Op(MO2.getImm());
   if (ImmOffs || Op == ARM_AM::sub) {
-    O << ", #"
+    O << ", "
+      << markup("<imm:")
+      << "#"
       << ARM_AM::getAddrOpcStr(ARM_AM::getAM5Op(MO2.getImm()))
-      << ImmOffs * 4;
+      << ImmOffs * 4
+      << markup(">");
   }
-  O << "]";
+  O << "]" << markup(">");
 }
 
 void ARMInstPrinter::printAddrMode6Operand(const MCInst *MI, unsigned OpNum,
@@ -553,18 +595,21 @@ void ARMInstPrinter::printAddrMode6Operand(const MCInst *MI, unsigned OpNum,
   const MCOperand &MO1 = MI->getOperand(OpNum);
   const MCOperand &MO2 = MI->getOperand(OpNum+1);
 
-  O << "[" << getRegisterName(MO1.getReg());
+  O << markup("<mem:") << "[";
+  printRegName(O, MO1.getReg());
   if (MO2.getImm()) {
     // FIXME: Both darwin as and GNU as violate ARM docs here.
     O << ", :" << (MO2.getImm() << 3);
   }
-  O << "]";
+  O << "]" << markup(">");
 }
 
 void ARMInstPrinter::printAddrMode7Operand(const MCInst *MI, unsigned OpNum,
                                            raw_ostream &O) {
   const MCOperand &MO1 = MI->getOperand(OpNum);
-  O << "[" << getRegisterName(MO1.getReg()) << "]";
+  O << markup("<mem:") << "[";
+  printRegName(O, MO1.getReg());
+  O << "]" << markup(">");
 }
 
 void ARMInstPrinter::printAddrMode6OffsetOperand(const MCInst *MI,
@@ -573,8 +618,10 @@ void ARMInstPrinter::printAddrMode6OffsetOperand(const MCInst *MI,
   const MCOperand &MO = MI->getOperand(OpNum);
   if (MO.getReg() == 0)
     O << "!";
-  else
-    O << ", " << getRegisterName(MO.getReg());
+  else {
+    O << ", ";
+    printRegName(O, MO.getReg());
+  }
 }
 
 void ARMInstPrinter::printBitfieldInvMaskImmOperand(const MCInst *MI,
@@ -585,7 +632,9 @@ void ARMInstPrinter::printBitfieldInvMaskImmOperand(const MCInst *MI,
   int32_t lsb = CountTrailingZeros_32(v);
   int32_t width = (32 - CountLeadingZeros_32 (v)) - lsb;
   assert(MO.isImm() && "Not a valid bf_inv_mask_imm value!");
-  O << '#' << lsb << ", #" << width;
+  O << markup("<imm:") << '#' << lsb << markup(">")
+    << ", "
+    << markup("<imm:") << '#' << width << markup(">");
 }
 
 void ARMInstPrinter::printMemBOption(const MCInst *MI, unsigned OpNum,
@@ -599,10 +648,18 @@ void ARMInstPrinter::printShiftImmOperand(const MCInst *MI, unsigned OpNum,
   unsigned ShiftOp = MI->getOperand(OpNum).getImm();
   bool isASR = (ShiftOp & (1 << 5)) != 0;
   unsigned Amt = ShiftOp & 0x1f;
-  if (isASR)
-    O << ", asr #" << (Amt == 0 ? 32 : Amt);
-  else if (Amt)
-    O << ", lsl #" << Amt;
+  if (isASR) {
+    O << ", asr "
+      << markup("<imm:")
+      << "#" << (Amt == 0 ? 32 : Amt)
+      << markup(">");
+  }
+  else if (Amt) {
+    O << ", lsl "
+      << markup("<imm:")
+      << "#" << Amt
+      << markup(">");
+  }
 }
 
 void ARMInstPrinter::printPKHLSLShiftImm(const MCInst *MI, unsigned OpNum,
@@ -611,7 +668,7 @@ void ARMInstPrinter::printPKHLSLShiftImm(const MCInst *MI, unsigned OpNum,
   if (Imm == 0)
     return;
   assert(Imm > 0 && Imm < 32 && "Invalid PKH shift immediate value!");
-  O << ", lsl #" << Imm;
+  O << ", lsl " << markup("<imm:") << "#" << Imm << markup(">");
 }
 
 void ARMInstPrinter::printPKHASRShiftImm(const MCInst *MI, unsigned OpNum,
@@ -621,7 +678,7 @@ void ARMInstPrinter::printPKHASRShiftImm(const MCInst *MI, unsigned OpNum,
   if (Imm == 0)
     Imm = 32;
   assert(Imm > 0 && Imm <= 32 && "Invalid PKH shift immediate value!");
-  O << ", asr #" << Imm;
+  O << ", asr " << markup("<imm:") << "#" << Imm << markup(">");
 }
 
 void ARMInstPrinter::printRegisterList(const MCInst *MI, unsigned OpNum,
@@ -629,7 +686,7 @@ void ARMInstPrinter::printRegisterList(const MCInst *MI, unsigned OpNum,
   O << "{";
   for (unsigned i = OpNum, e = MI->getNumOperands(); i != e; ++i) {
     if (i != OpNum) O << ", ";
-    O << getRegisterName(MI->getOperand(i).getReg());
+    printRegName(O, MI->getOperand(i).getReg());
   }
   O << "}";
 }
@@ -803,23 +860,29 @@ void ARMInstPrinter::printAdrLabelOperand(const MCInst *MI, unsigned OpNum,
 
   int32_t OffImm = (int32_t)MO.getImm();
 
+  O << markup("<imm:");
   if (OffImm == INT32_MIN)
     O << "#-0";
   else if (OffImm < 0)
     O << "#-" << -OffImm;
   else
     O << "#" << OffImm;
+  O << markup(">");
 }
 
 void ARMInstPrinter::printThumbS4ImmOperand(const MCInst *MI, unsigned OpNum,
                                             raw_ostream &O) {
-  O << "#" << MI->getOperand(OpNum).getImm() * 4;
+  O << markup("<imm:")
+    << "#" << MI->getOperand(OpNum).getImm() * 4
+    << markup(">");
 }
 
 void ARMInstPrinter::printThumbSRImm(const MCInst *MI, unsigned OpNum,
                                      raw_ostream &O) {
   unsigned Imm = MI->getOperand(OpNum).getImm();
-  O << "#" << (Imm == 0 ? 32 : Imm);
+  O << markup("<imm:")
+    << "#" << (Imm == 0 ? 32 : Imm)
+    << markup(">");
 }
 
 void ARMInstPrinter::printThumbITMask(const MCInst *MI, unsigned OpNum,
@@ -849,10 +912,13 @@ void ARMInstPrinter::printThumbAddrModeRROperand(const MCInst *MI, unsigned Op,
     return;
   }
 
-  O << "[" << getRegisterName(MO1.getReg());
-  if (unsigned RegNum = MO2.getReg())
-    O << ", " << getRegisterName(RegNum);
-  O << "]";
+  O << markup("<mem:") << "[";
+  printRegName(O, MO1.getReg());
+  if (unsigned RegNum = MO2.getReg()) {
+    O << ", ";
+    printRegName(O, RegNum);
+  }
+  O << "]" << markup(">");
 }
 
 void ARMInstPrinter::printThumbAddrModeImm5SOperand(const MCInst *MI,
@@ -867,10 +933,15 @@ void ARMInstPrinter::printThumbAddrModeImm5SOperand(const MCInst *MI,
     return;
   }
 
-  O << "[" << getRegisterName(MO1.getReg());
-  if (unsigned ImmOffs = MO2.getImm())
-    O << ", #" << ImmOffs * Scale;
-  O << "]";
+  O << markup("<mem:") << "[";
+  printRegName(O, MO1.getReg());
+  if (unsigned ImmOffs = MO2.getImm()) {
+    O << ", "
+      << markup("<imm:")
+      << "#" << ImmOffs * Scale
+      << markup(">");
+  }
+  O << "]" << markup(">");
 }
 
 void ARMInstPrinter::printThumbAddrModeImm5S1Operand(const MCInst *MI,
@@ -906,14 +977,12 @@ void ARMInstPrinter::printT2SOOperand(const MCInst *MI, unsigned OpNum,
   const MCOperand &MO2 = MI->getOperand(OpNum+1);
 
   unsigned Reg = MO1.getReg();
-  O << getRegisterName(Reg);
+  printRegName(O, Reg);
 
   // Print the shift opc.
   assert(MO2.isImm() && "Not a valid t2_so_reg value!");
-  ARM_AM::ShiftOpc ShOpc = ARM_AM::getSORegShOp(MO2.getImm());
-  O << ", " << ARM_AM::getShiftOpcStr(ShOpc);
-  if (ShOpc != ARM_AM::rrx)
-    O << " #" << translateShiftImm(ARM_AM::getSORegOffset(MO2.getImm()));
+  printRegImmShift(O, ARM_AM::getSORegShOp(MO2.getImm()),
+                   ARM_AM::getSORegOffset(MO2.getImm()), UseMarkup);
 }
 
 void ARMInstPrinter::printAddrModeImm12Operand(const MCInst *MI, unsigned OpNum,
@@ -926,18 +995,27 @@ void ARMInstPrinter::printAddrModeImm12Operand(const MCInst *MI, unsigned OpNum,
     return;
   }
 
-  O << "[" << getRegisterName(MO1.getReg());
+  O << markup("<mem:") << "[";
+  printRegName(O, MO1.getReg());
 
   int32_t OffImm = (int32_t)MO2.getImm();
   bool isSub = OffImm < 0;
   // Special value for #-0. All others are normal.
   if (OffImm == INT32_MIN)
     OffImm = 0;
-  if (isSub)
-    O << ", #-" << -OffImm;
-  else if (OffImm > 0)
-    O << ", #" << OffImm;
-  O << "]";
+  if (isSub) {
+    O << ", "
+      << markup("<imm:") 
+      << "#-" << -OffImm
+      << markup(">");
+  }
+  else if (OffImm > 0) {
+    O << ", "
+      << markup("<imm:") 
+      << "#" << OffImm
+      << markup(">");
+  }
+  O << "]" << markup(">");
 }
 
 void ARMInstPrinter::printT2AddrModeImm8Operand(const MCInst *MI,
@@ -946,17 +1024,24 @@ void ARMInstPrinter::printT2AddrModeImm8Operand(const MCInst *MI,
   const MCOperand &MO1 = MI->getOperand(OpNum);
   const MCOperand &MO2 = MI->getOperand(OpNum+1);
 
-  O << "[" << getRegisterName(MO1.getReg());
+  O << markup("<mem:") << "[";
+  printRegName(O, MO1.getReg());
 
   int32_t OffImm = (int32_t)MO2.getImm();
   // Don't print +0.
+  if (OffImm != 0)
+    O << ", ";
+  if (OffImm != 0 && UseMarkup)
+    O << "<imm:";
   if (OffImm == INT32_MIN)
-    O << ", #-0";
+    O << "#-0";
   else if (OffImm < 0)
-    O << ", #-" << -OffImm;
+    O << "#-" << -OffImm;
   else if (OffImm > 0)
-    O << ", #" << OffImm;
-  O << "]";
+    O << "#" << OffImm;
+  if (OffImm != 0 && UseMarkup)
+    O << ">";
+  O << "]" << markup(">");
 }
 
 void ARMInstPrinter::printT2AddrModeImm8s4Operand(const MCInst *MI,
@@ -970,20 +1055,27 @@ void ARMInstPrinter::printT2AddrModeImm8s4Operand(const MCInst *MI,
     return;
   }
 
-  O << "[" << getRegisterName(MO1.getReg());
+  O << markup("<mem:") << "[";
+  printRegName(O, MO1.getReg());
 
   int32_t OffImm = (int32_t)MO2.getImm();
 
   assert(((OffImm & 0x3) == 0) && "Not a valid immediate!");
 
   // Don't print +0.
+  if (OffImm != 0)
+    O << ", ";
+  if (OffImm != 0 && UseMarkup)
+    O << "<imm:";
   if (OffImm == INT32_MIN)
-    O << ", #-0";
+    O << "#-0";
   else if (OffImm < 0)
-    O << ", #-" << -OffImm;
+    O << "#-" << -OffImm;
   else if (OffImm > 0)
-    O << ", #" << OffImm;
-  O << "]";
+    O << "#" << OffImm;
+  if (OffImm != 0 && UseMarkup)
+    O << ">";
+  O << "]" << markup(">");
 }
 
 void ARMInstPrinter::printT2AddrModeImm0_1020s4Operand(const MCInst *MI,
@@ -992,10 +1084,15 @@ void ARMInstPrinter::printT2AddrModeImm0_1020s4Operand(const MCInst *MI,
   const MCOperand &MO1 = MI->getOperand(OpNum);
   const MCOperand &MO2 = MI->getOperand(OpNum+1);
 
-  O << "[" << getRegisterName(MO1.getReg());
-  if (MO2.getImm())
-    O << ", #" << MO2.getImm() * 4;
-  O << "]";
+  O << markup("<mem:") << "[";
+  printRegName(O, MO1.getReg());
+  if (MO2.getImm()) {
+    O << ", "
+      << markup("<imm:")
+      << "#" << MO2.getImm() * 4
+      << markup(">");
+  }
+  O << "]" << markup(">");
 }
 
 void ARMInstPrinter::printT2AddrModeImm8OffsetOperand(const MCInst *MI,
@@ -1003,11 +1100,12 @@ void ARMInstPrinter::printT2AddrModeImm8OffsetOperand(const MCInst *MI,
                                                       raw_ostream &O) {
   const MCOperand &MO1 = MI->getOperand(OpNum);
   int32_t OffImm = (int32_t)MO1.getImm();
-  // Don't print +0.
+  O << ", " << markup("<imm:");
   if (OffImm < 0)
-    O << ", #-" << -OffImm;
+    O << "#-" << -OffImm;
   else
-    O << ", #" << OffImm;
+    O << "#" << OffImm;
+  O << markup(">");
 }
 
 void ARMInstPrinter::printT2AddrModeImm8s4OffsetOperand(const MCInst *MI,
@@ -1019,12 +1117,18 @@ void ARMInstPrinter::printT2AddrModeImm8s4OffsetOperand(const MCInst *MI,
   assert(((OffImm & 0x3) == 0) && "Not a valid immediate!");
 
   // Don't print +0.
+  if (OffImm != 0)
+    O << ", ";
+  if (OffImm != 0 && UseMarkup)
+    O << "<imm:";
   if (OffImm == INT32_MIN)
-    O << ", #-0";
+    O << "#-0";
   else if (OffImm < 0)
-    O << ", #-" << -OffImm;
+    O << "#-" << -OffImm;
   else if (OffImm > 0)
-    O << ", #" << OffImm;
+    O << "#" << OffImm;
+  if (OffImm != 0 && UseMarkup)
+    O << ">";
 }
 
 void ARMInstPrinter::printT2AddrModeSoRegOperand(const MCInst *MI,
@@ -1034,23 +1138,30 @@ void ARMInstPrinter::printT2AddrModeSoRegOperand(const MCInst *MI,
   const MCOperand &MO2 = MI->getOperand(OpNum+1);
   const MCOperand &MO3 = MI->getOperand(OpNum+2);
 
-  O << "[" << getRegisterName(MO1.getReg());
+  O << markup("<mem:") << "[";
+  printRegName(O, MO1.getReg());
 
   assert(MO2.getReg() && "Invalid so_reg load / store address!");
-  O << ", " << getRegisterName(MO2.getReg());
+  O << ", ";
+  printRegName(O, MO2.getReg());
 
   unsigned ShAmt = MO3.getImm();
   if (ShAmt) {
     assert(ShAmt <= 3 && "Not a valid Thumb2 addressing mode!");
-    O << ", lsl #" << ShAmt;
+    O << ", lsl "
+      << markup("<imm:")
+      << "#" << ShAmt
+      << markup(">");
   }
-  O << "]";
+  O << "]" << markup(">");
 }
 
 void ARMInstPrinter::printFPImmOperand(const MCInst *MI, unsigned OpNum,
                                        raw_ostream &O) {
   const MCOperand &MO = MI->getOperand(OpNum);
-  O << '#' << ARM_AM::getFPImmFloat(MO.getImm());
+  O << markup("<imm:")
+    << '#' << ARM_AM::getFPImmFloat(MO.getImm())
+    << markup(">");
 }
 
 void ARMInstPrinter::printNEONModImmOperand(const MCInst *MI, unsigned OpNum,
@@ -1058,14 +1169,18 @@ void ARMInstPrinter::printNEONModImmOperand(const MCInst *MI, unsigned OpNum,
   unsigned EncodedImm = MI->getOperand(OpNum).getImm();
   unsigned EltBits;
   uint64_t Val = ARM_AM::decodeNEONModImm(EncodedImm, EltBits);
-  O << "#0x";
+  O << markup("<imm:")
+    << "#0x";
   O.write_hex(Val);
+  O << markup(">");
 }
 
 void ARMInstPrinter::printImmPlusOneOperand(const MCInst *MI, unsigned OpNum,
                                             raw_ostream &O) {
   unsigned Imm = MI->getOperand(OpNum).getImm();
-  O << "#" << Imm + 1;
+  O << markup("<imm:")
+    << "#" << Imm + 1
+    << markup(">");
 }
 
 void ARMInstPrinter::printRotImmOperand(const MCInst *MI, unsigned OpNum,
@@ -1073,23 +1188,30 @@ void ARMInstPrinter::printRotImmOperand(const MCInst *MI, unsigned OpNum,
   unsigned Imm = MI->getOperand(OpNum).getImm();
   if (Imm == 0)
     return;
-  O << ", ror #";
+  O << ", ror "
+    << markup("<imm:")
+    << "#";
   switch (Imm) {
   default: assert (0 && "illegal ror immediate!");
   case 1: O << "8"; break;
   case 2: O << "16"; break;
   case 3: O << "24"; break;
   }
+  O << markup(">");
 }
 
 void ARMInstPrinter::printFBits16(const MCInst *MI, unsigned OpNum,
                                   raw_ostream &O) {
-  O << "#" << 16 - MI->getOperand(OpNum).getImm();
+  O << markup("<imm:")
+    << "#" << 16 - MI->getOperand(OpNum).getImm()
+    << markup(">");
 }
 
 void ARMInstPrinter::printFBits32(const MCInst *MI, unsigned OpNum,
                                   raw_ostream &O) {
-  O << "#" << 32 - MI->getOperand(OpNum).getImm();
+  O << markup("<imm:")
+    << "#" << 32 - MI->getOperand(OpNum).getImm()
+    << markup(">");
 }
 
 void ARMInstPrinter::printVectorIndex(const MCInst *MI, unsigned OpNum,
@@ -1099,7 +1221,9 @@ void ARMInstPrinter::printVectorIndex(const MCInst *MI, unsigned OpNum,
 
 void ARMInstPrinter::printVectorListOne(const MCInst *MI, unsigned OpNum,
                                         raw_ostream &O) {
-  O << "{" << getRegisterName(MI->getOperand(OpNum).getReg()) << "}";
+  O << "{";
+  printRegName(O, MI->getOperand(OpNum).getReg());
+  O << "}";
 }
 
 void ARMInstPrinter::printVectorListTwo(const MCInst *MI, unsigned OpNum,
@@ -1107,7 +1231,11 @@ void ARMInstPrinter::printVectorListTwo(const MCInst *MI, unsigned OpNum,
   unsigned Reg = MI->getOperand(OpNum).getReg();
   unsigned Reg0 = MRI.getSubReg(Reg, ARM::dsub_0);
   unsigned Reg1 = MRI.getSubReg(Reg, ARM::dsub_1);
-  O << "{" << getRegisterName(Reg0) << ", " << getRegisterName(Reg1) << "}";
+  O << "{";
+  printRegName(O, Reg0);
+  O << ", ";
+  printRegName(O, Reg1);
+  O << "}";
 }
 
 void ARMInstPrinter::printVectorListTwoSpaced(const MCInst *MI,
@@ -1116,7 +1244,11 @@ void ARMInstPrinter::printVectorListTwoSpaced(const MCInst *MI,
   unsigned Reg = MI->getOperand(OpNum).getReg();
   unsigned Reg0 = MRI.getSubReg(Reg, ARM::dsub_0);
   unsigned Reg1 = MRI.getSubReg(Reg, ARM::dsub_2);
-  O << "{" << getRegisterName(Reg0) << ", " << getRegisterName(Reg1) << "}";
+  O << "{";
+  printRegName(O, Reg0);
+  O << ", ";
+  printRegName(O, Reg1);
+  O << "}";
 }
 
 void ARMInstPrinter::printVectorListThree(const MCInst *MI, unsigned OpNum,
@@ -1124,9 +1256,13 @@ void ARMInstPrinter::printVectorListThree(const MCInst *MI, unsigned OpNum,
   // Normally, it's not safe to use register enum values directly with
   // addition to get the next register, but for VFP registers, the
   // sort order is guaranteed because they're all of the form D<n>.
-  O << "{" << getRegisterName(MI->getOperand(OpNum).getReg()) << ", "
-    << getRegisterName(MI->getOperand(OpNum).getReg() + 1) << ", "
-    << getRegisterName(MI->getOperand(OpNum).getReg() + 2) << "}";
+  O << "{";
+  printRegName(O, MI->getOperand(OpNum).getReg());
+  O << ", ";
+  printRegName(O, MI->getOperand(OpNum).getReg() + 1);
+  O << ", ";
+  printRegName(O, MI->getOperand(OpNum).getReg() + 2);
+  O << "}";
 }
 
 void ARMInstPrinter::printVectorListFour(const MCInst *MI, unsigned OpNum,
@@ -1134,16 +1270,23 @@ void ARMInstPrinter::printVectorListFour(const MCInst *MI, unsigned OpNum,
   // Normally, it's not safe to use register enum values directly with
   // addition to get the next register, but for VFP registers, the
   // sort order is guaranteed because they're all of the form D<n>.
-  O << "{" << getRegisterName(MI->getOperand(OpNum).getReg()) << ", "
-    << getRegisterName(MI->getOperand(OpNum).getReg() + 1) << ", "
-    << getRegisterName(MI->getOperand(OpNum).getReg() + 2) << ", "
-    << getRegisterName(MI->getOperand(OpNum).getReg() + 3) << "}";
+  O << "{";
+  printRegName(O, MI->getOperand(OpNum).getReg());
+  O << ", ";
+  printRegName(O, MI->getOperand(OpNum).getReg() + 1);
+  O << ", ";
+  printRegName(O, MI->getOperand(OpNum).getReg() + 2);
+  O << ", ";
+  printRegName(O, MI->getOperand(OpNum).getReg() + 3);
+  O << "}";
 }
 
 void ARMInstPrinter::printVectorListOneAllLanes(const MCInst *MI,
                                                 unsigned OpNum,
                                                 raw_ostream &O) {
-  O << "{" << getRegisterName(MI->getOperand(OpNum).getReg()) << "[]}";
+  O << "{";
+  printRegName(O, MI->getOperand(OpNum).getReg());
+  O << "[]}";
 }
 
 void ARMInstPrinter::printVectorListTwoAllLanes(const MCInst *MI,
@@ -1152,7 +1295,11 @@ void ARMInstPrinter::printVectorListTwoAllLanes(const MCInst *MI,
   unsigned Reg = MI->getOperand(OpNum).getReg();
   unsigned Reg0 = MRI.getSubReg(Reg, ARM::dsub_0);
   unsigned Reg1 = MRI.getSubReg(Reg, ARM::dsub_1);
-  O << "{" << getRegisterName(Reg0) << "[], " << getRegisterName(Reg1) << "[]}";
+  O << "{";
+  printRegName(O, Reg0);
+  O << "[], ";
+  printRegName(O, Reg1);
+  O << "[]}";
 }
 
 void ARMInstPrinter::printVectorListThreeAllLanes(const MCInst *MI,
@@ -1161,9 +1308,13 @@ void ARMInstPrinter::printVectorListThreeAllLanes(const MCInst *MI,
   // Normally, it's not safe to use register enum values directly with
   // addition to get the next register, but for VFP registers, the
   // sort order is guaranteed because they're all of the form D<n>.
-  O << "{" << getRegisterName(MI->getOperand(OpNum).getReg()) << "[], "
-    << getRegisterName(MI->getOperand(OpNum).getReg() + 1) << "[], "
-    << getRegisterName(MI->getOperand(OpNum).getReg() + 2) << "[]}";
+  O << "{";
+  printRegName(O, MI->getOperand(OpNum).getReg());
+  O << "[], ";
+  printRegName(O, MI->getOperand(OpNum).getReg() + 1);
+  O << "[], ";
+  printRegName(O, MI->getOperand(OpNum).getReg() + 2);
+  O << "[]}";
 }
 
 void ARMInstPrinter::printVectorListFourAllLanes(const MCInst *MI,
@@ -1172,10 +1323,15 @@ void ARMInstPrinter::printVectorListFourAllLanes(const MCInst *MI,
   // Normally, it's not safe to use register enum values directly with
   // addition to get the next register, but for VFP registers, the
   // sort order is guaranteed because they're all of the form D<n>.
-  O << "{" << getRegisterName(MI->getOperand(OpNum).getReg()) << "[], "
-    << getRegisterName(MI->getOperand(OpNum).getReg() + 1) << "[], "
-    << getRegisterName(MI->getOperand(OpNum).getReg() + 2) << "[], "
-    << getRegisterName(MI->getOperand(OpNum).getReg() + 3) << "[]}";
+  O << "{";
+  printRegName(O, MI->getOperand(OpNum).getReg());
+  O << "[], ";
+  printRegName(O, MI->getOperand(OpNum).getReg() + 1);
+  O << "[], ";
+  printRegName(O, MI->getOperand(OpNum).getReg() + 2);
+  O << "[], ";
+  printRegName(O, MI->getOperand(OpNum).getReg() + 3);
+  O << "[]}";
 }
 
 void ARMInstPrinter::printVectorListTwoSpacedAllLanes(const MCInst *MI,
@@ -1184,7 +1340,11 @@ void ARMInstPrinter::printVectorListTwoSpacedAllLanes(const MCInst *MI,
   unsigned Reg = MI->getOperand(OpNum).getReg();
   unsigned Reg0 = MRI.getSubReg(Reg, ARM::dsub_0);
   unsigned Reg1 = MRI.getSubReg(Reg, ARM::dsub_2);
-  O << "{" << getRegisterName(Reg0) << "[], " << getRegisterName(Reg1) << "[]}";
+  O << "{";
+  printRegName(O, Reg0);
+  O << "[], ";
+  printRegName(O, Reg1);
+  O << "[]}";
 }
 
 void ARMInstPrinter::printVectorListThreeSpacedAllLanes(const MCInst *MI,
@@ -1193,9 +1353,13 @@ void ARMInstPrinter::printVectorListThreeSpacedAllLanes(const MCInst *MI,
   // Normally, it's not safe to use register enum values directly with
   // addition to get the next register, but for VFP registers, the
   // sort order is guaranteed because they're all of the form D<n>.
-  O << "{" << getRegisterName(MI->getOperand(OpNum).getReg()) << "[], "
-    << getRegisterName(MI->getOperand(OpNum).getReg() + 2) << "[], "
-    << getRegisterName(MI->getOperand(OpNum).getReg() + 4) << "[]}";
+  O << "{";
+  printRegName(O, MI->getOperand(OpNum).getReg());
+  O  << "[], ";
+  printRegName(O, MI->getOperand(OpNum).getReg() + 2);
+  O << "[], ";
+  printRegName(O, MI->getOperand(OpNum).getReg() + 4);
+  O << "[]}";
 }
 
 void ARMInstPrinter::printVectorListFourSpacedAllLanes(const MCInst *MI,
@@ -1204,10 +1368,15 @@ void ARMInstPrinter::printVectorListFourSpacedAllLanes(const MCInst *MI,
   // Normally, it's not safe to use register enum values directly with
   // addition to get the next register, but for VFP registers, the
   // sort order is guaranteed because they're all of the form D<n>.
-  O << "{" << getRegisterName(MI->getOperand(OpNum).getReg()) << "[], "
-    << getRegisterName(MI->getOperand(OpNum).getReg() + 2) << "[], "
-    << getRegisterName(MI->getOperand(OpNum).getReg() + 4) << "[], "
-    << getRegisterName(MI->getOperand(OpNum).getReg() + 6) << "[]}";
+  O << "{";
+  printRegName(O, MI->getOperand(OpNum).getReg());
+  O << "[], ";
+  printRegName(O, MI->getOperand(OpNum).getReg() + 2);
+  O << "[], ";
+  printRegName(O, MI->getOperand(OpNum).getReg() + 4);
+  O << "[], ";
+  printRegName(O, MI->getOperand(OpNum).getReg() + 6);
+  O << "[]}";
 }
 
 void ARMInstPrinter::printVectorListThreeSpaced(const MCInst *MI,
@@ -1216,9 +1385,13 @@ void ARMInstPrinter::printVectorListThreeSpaced(const MCInst *MI,
   // Normally, it's not safe to use register enum values directly with
   // addition to get the next register, but for VFP registers, the
   // sort order is guaranteed because they're all of the form D<n>.
-  O << "{" << getRegisterName(MI->getOperand(OpNum).getReg()) << ", "
-    << getRegisterName(MI->getOperand(OpNum).getReg() + 2) << ", "
-    << getRegisterName(MI->getOperand(OpNum).getReg() + 4) << "}";
+  O << "{";
+  printRegName(O, MI->getOperand(OpNum).getReg());
+  O << ", ";
+  printRegName(O, MI->getOperand(OpNum).getReg() + 2);
+  O << ", ";
+  printRegName(O, MI->getOperand(OpNum).getReg() + 4);
+  O << "}";
 }
 
 void ARMInstPrinter::printVectorListFourSpaced(const MCInst *MI,
@@ -1227,8 +1400,13 @@ void ARMInstPrinter::printVectorListFourSpaced(const MCInst *MI,
   // Normally, it's not safe to use register enum values directly with
   // addition to get the next register, but for VFP registers, the
   // sort order is guaranteed because they're all of the form D<n>.
-  O << "{" << getRegisterName(MI->getOperand(OpNum).getReg()) << ", "
-    << getRegisterName(MI->getOperand(OpNum).getReg() + 2) << ", "
-    << getRegisterName(MI->getOperand(OpNum).getReg() + 4) << ", "
-    << getRegisterName(MI->getOperand(OpNum).getReg() + 6) << "}";
+  O << "{";
+  printRegName(O, MI->getOperand(OpNum).getReg());
+  O << ", ";
+  printRegName(O, MI->getOperand(OpNum).getReg() + 2);
+  O << ", ";
+  printRegName(O, MI->getOperand(OpNum).getReg() + 4);
+  O << ", ";
+  printRegName(O, MI->getOperand(OpNum).getReg() + 6);
+  O << "}";
 }
diff --git a/lib/Target/ARM/InstPrinter/ARMInstPrinter.h b/lib/Target/ARM/InstPrinter/ARMInstPrinter.h
index 73d7bfd..b7bab5f 100644
--- a/lib/Target/ARM/InstPrinter/ARMInstPrinter.h
+++ b/lib/Target/ARM/InstPrinter/ARMInstPrinter.h
@@ -126,7 +126,8 @@ public:
   void printRotImmOperand(const MCInst *MI, unsigned OpNum, raw_ostream &O);
 
   void printPCLabel(const MCInst *MI, unsigned OpNum, raw_ostream &O);
-  void printT2LdrLabelOperand(const MCInst *MI, unsigned OpNum, raw_ostream &O);
+  void printThumbLdrLabelOperand(const MCInst *MI, unsigned OpNum,
+                                 raw_ostream &O);
   void printFBits16(const MCInst *MI, unsigned OpNum, raw_ostream &O);
   void printFBits32(const MCInst *MI, unsigned OpNum, raw_ostream &O);
   void printVectorIndex(const MCInst *MI, unsigned OpNum, raw_ostream &O);
diff --git a/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp b/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
index ac6ce64..1ba6ab0 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
+++ b/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp
@@ -593,7 +593,9 @@ public:
   const object::mach::CPUSubtypeARM Subtype;
   DarwinARMAsmBackend(const Target &T, const StringRef TT,
                       object::mach::CPUSubtypeARM st)
-    : ARMAsmBackend(T, TT), Subtype(st) { }
+    : ARMAsmBackend(T, TT), Subtype(st) {
+      HasDataInCodeSupport = true;
+    }
 
   MCObjectWriter *createObjectWriter(raw_ostream &OS) const {
     return createARMMachObjectWriter(OS, /*Is64Bit=*/false,
@@ -674,7 +676,7 @@ void DarwinARMAsmBackend::applyFixup(const MCFixup &Fixup, char *Data,
 
 } // end anonymous namespace
 
-MCAsmBackend *llvm::createARMAsmBackend(const Target &T, StringRef TT) {
+MCAsmBackend *llvm::createARMAsmBackend(const Target &T, StringRef TT, StringRef CPU) {
   Triple TheTriple(TT);
 
   if (TheTriple.isOSDarwin()) {
@@ -687,6 +689,15 @@ MCAsmBackend *llvm::createARMAsmBackend(const Target &T, StringRef TT) {
     else if (TheTriple.getArchName() == "armv6" ||
         TheTriple.getArchName() == "thumbv6")
       return new DarwinARMAsmBackend(T, TT, object::mach::CSARM_V6);
+    else if (TheTriple.getArchName() == "armv7f" ||
+        TheTriple.getArchName() == "thumbv7f")
+      return new DarwinARMAsmBackend(T, TT, object::mach::CSARM_V7F);
+    else if (TheTriple.getArchName() == "armv7k" ||
+        TheTriple.getArchName() == "thumbv7k")
+      return new DarwinARMAsmBackend(T, TT, object::mach::CSARM_V7K);
+    else if (TheTriple.getArchName() == "armv7s" ||
+        TheTriple.getArchName() == "thumbv7s")
+      return new DarwinARMAsmBackend(T, TT, object::mach::CSARM_V7S);
     return new DarwinARMAsmBackend(T, TT, object::mach::CSARM_V7);
   }
 
diff --git a/lib/Target/ARM/MCTargetDesc/ARMELFObjectWriter.cpp b/lib/Target/ARM/MCTargetDesc/ARMELFObjectWriter.cpp
index 7d6acbc..99e4f71 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMELFObjectWriter.cpp
+++ b/lib/Target/ARM/MCTargetDesc/ARMELFObjectWriter.cpp
@@ -194,6 +194,10 @@ unsigned ARMELFObjectWriter::GetRelocTypeInner(const MCValue &Target,
     case ARM::fixup_arm_uncondbranch:
       Type = ELF::R_ARM_JUMP24;
       break;
+    case ARM::fixup_t2_condbranch:
+    case ARM::fixup_t2_uncondbranch:
+      Type = ELF::R_ARM_THM_JUMP24;
+      break;
     case ARM::fixup_arm_movt_hi16:
     case ARM::fixup_arm_movt_hi16_pcrel:
       Type = ELF::R_ARM_MOVT_PREL;
@@ -242,6 +246,9 @@ unsigned ARMELFObjectWriter::GetRelocTypeInner(const MCValue &Target,
       case MCSymbolRefExpr::VK_ARM_TARGET1:
         Type = ELF::R_ARM_TARGET1;
         break;
+      case MCSymbolRefExpr::VK_ARM_TARGET2:
+        Type = ELF::R_ARM_TARGET2;
+        break;
       } 
       break;
     case ARM::fixup_arm_ldst_pcrel_12:
diff --git a/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.cpp b/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.cpp
index d32805e..c1aab9c 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.cpp
+++ b/lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.cpp
@@ -50,7 +50,6 @@ ARMELFMCAsmInfo::ARMELFMCAsmInfo() {
   Code32Directive = ".code\t32";
 
   WeakRefDirective = "\t.weak\t";
-  LCOMMDirectiveType = LCOMM::NoAlignment;
 
   HasLEB128 = true;
   SupportsDebugInformation = true;
diff --git a/lib/Target/ARM/MCTargetDesc/ARMMCCodeEmitter.cpp b/lib/Target/ARM/MCTargetDesc/ARMMCCodeEmitter.cpp
index 94f1082..d0e127a 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMMCCodeEmitter.cpp
+++ b/lib/Target/ARM/MCTargetDesc/ARMMCCodeEmitter.cpp
@@ -35,8 +35,8 @@ STATISTIC(MCNumCPRelocations, "Number of constant pool relocations created.");
 
 namespace {
 class ARMMCCodeEmitter : public MCCodeEmitter {
-  ARMMCCodeEmitter(const ARMMCCodeEmitter &); // DO NOT IMPLEMENT
-  void operator=(const ARMMCCodeEmitter &); // DO NOT IMPLEMENT
+  ARMMCCodeEmitter(const ARMMCCodeEmitter &) LLVM_DELETED_FUNCTION;
+  void operator=(const ARMMCCodeEmitter &) LLVM_DELETED_FUNCTION;
   const MCInstrInfo &MCII;
   const MCSubtargetInfo &STI;
   const MCContext &CTX;
@@ -783,7 +783,7 @@ getT2Imm8s4OpValue(const MCInst &MI, unsigned OpIdx,
 
   // Immediate is always encoded as positive. The 'U' bit controls add vs sub.
   if (Imm8 < 0)
-    Imm8 = -Imm8;
+    Imm8 = -(uint32_t)Imm8;
 
   // Scaled by 4.
   Imm8 /= 4;
@@ -934,6 +934,10 @@ getLdStSORegOpValue(const MCInst &MI, unsigned OpIdx,
   ARM_AM::ShiftOpc ShOp = ARM_AM::getAM2ShiftOpc(MO2.getImm());
   unsigned SBits = getShiftOp(ShOp);
 
+  // While "lsr #32" and "asr #32" exist, they are encoded with a 0 in the shift
+  // amount. However, it would be an easy mistake to make so check here.
+  assert((ShImm & ~0x1f) == 0 && "Out of range shift amount");
+
   // {16-13} = Rn
   // {12}    = isAdd
   // {11-0}  = shifter
diff --git a/lib/Target/ARM/MCTargetDesc/ARMMCExpr.h b/lib/Target/ARM/MCTargetDesc/ARMMCExpr.h
index a727e08..b404e6c 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMMCExpr.h
+++ b/lib/Target/ARM/MCTargetDesc/ARMMCExpr.h
@@ -28,7 +28,7 @@ private:
 
   explicit ARMMCExpr(VariantKind _Kind, const MCExpr *_Expr)
     : Kind(_Kind), Expr(_Expr) {}
-  
+
 public:
   /// @name Construction
   /// @{
@@ -67,9 +67,6 @@ public:
   static bool classof(const MCExpr *E) {
     return E->getKind() == MCExpr::Target;
   }
-  
-  static bool classof(const ARMMCExpr *) { return true; }
-
 };
 } // end namespace llvm
 
diff --git a/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.cpp b/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.cpp
index 5df84c8..00ffc94 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.cpp
+++ b/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.cpp
@@ -71,6 +71,14 @@ std::string ARM_MC::ParseARMTriple(StringRef TT, StringRef CPU) {
         else
           // Use CPU to figure out the exact features.
           ARMArchFeature = "+v7";
+      } else if (Len >= Idx+2 && TT[Idx+1] == 's') {
+        if (NoCPU)
+          // v7s: FeatureNEON, FeatureDB, FeatureDSPThumb2, FeatureT2XtPk
+          //      Swift
+          ARMArchFeature = "+v7,+swift,+neon,+db,+t2dsp,+t2xtpk";
+        else
+          // Use CPU to figure out the exact features.
+          ARMArchFeature = "+v7";
       } else {
         // v7 CPUs have lots of different feature sets. If no CPU is specified,
         // then assume v7a (e.g. cortex-a8) feature set. Otherwise, return
diff --git a/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.h b/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.h
index 510302d..a89981e 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.h
+++ b/lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.h
@@ -46,7 +46,7 @@ MCCodeEmitter *createARMMCCodeEmitter(const MCInstrInfo &MCII,
                                       const MCSubtargetInfo &STI,
                                       MCContext &Ctx);
 
-MCAsmBackend *createARMAsmBackend(const Target &T, StringRef TT);
+MCAsmBackend *createARMAsmBackend(const Target &T, StringRef TT, StringRef CPU);
 
 /// createARMELFObjectWriter - Construct an ELF Mach-O object writer.
 MCObjectWriter *createARMELFObjectWriter(raw_ostream &OS,
diff --git a/lib/Target/ARM/MCTargetDesc/ARMMachObjectWriter.cpp b/lib/Target/ARM/MCTargetDesc/ARMMachObjectWriter.cpp
index a51e0fa..2154c93 100644
--- a/lib/Target/ARM/MCTargetDesc/ARMMachObjectWriter.cpp
+++ b/lib/Target/ARM/MCTargetDesc/ARMMachObjectWriter.cpp
@@ -41,6 +41,12 @@ class ARMMachObjectWriter : public MCMachObjectTargetWriter {
                                         const MCFixup &Fixup, MCValue Target,
                                         uint64_t &FixedValue);
 
+  bool requiresExternRelocation(MachObjectWriter *Writer,
+                                const MCAssembler &Asm,
+                                const MCFragment &Fragment,
+                                unsigned RelocType, const MCSymbolData *SD,
+                                uint64_t FixedValue);
+
 public:
   ARMMachObjectWriter(bool Is64Bit, uint32_t CPUType,
                       uint32_t CPUSubtype)
@@ -305,6 +311,46 @@ void ARMMachObjectWriter::RecordARMScatteredRelocation(MachObjectWriter *Writer,
   Writer->addRelocation(Fragment->getParent(), MRE);
 }
 
+bool ARMMachObjectWriter::requiresExternRelocation(MachObjectWriter *Writer,
+                                                   const MCAssembler &Asm,
+                                                   const MCFragment &Fragment,
+                                                   unsigned RelocType,
+                                                   const MCSymbolData *SD,
+                                                   uint64_t FixedValue) {
+  // Most cases can be identified purely from the symbol.
+  if (Writer->doesSymbolRequireExternRelocation(SD))
+    return true;
+  int64_t Value = (int64_t)FixedValue;  // The displacement is signed.
+  int64_t Range;
+  switch (RelocType) {
+  default:
+    return false;
+  case macho::RIT_ARM_Branch24Bit:
+    // PC pre-adjustment of 8 for these instructions.
+    Value -= 8;
+    // ARM BL/BLX has a 25-bit offset.
+    Range = 0x1ffffff;
+    break;
+  case macho::RIT_ARM_ThumbBranch22Bit:
+    // PC pre-adjustment of 4 for these instructions.
+    Value -= 4;
+    // Thumb BL/BLX has a 24-bit offset.
+    Range = 0xffffff;
+  }
+  // BL/BLX also use external relocations when an internal relocation
+  // would result in the target being out of range. This gives the linker
+  // enough information to generate a branch island.
+  const MCSectionData &SymSD = Asm.getSectionData(
+    SD->getSymbol().getSection());
+  Value += Writer->getSectionAddress(&SymSD);
+  Value -= Writer->getSectionAddress(Fragment.getParent());
+  // If the resultant value would be out of range for an internal relocation,
+  // use an external instead.
+  if (Value > Range || Value < -(Range + 1))
+    return true;
+  return false;
+}
+
 void ARMMachObjectWriter::RecordRelocation(MachObjectWriter *Writer,
                                            const MCAssembler &Asm,
                                            const MCAsmLayout &Layout,
@@ -373,7 +419,8 @@ void ARMMachObjectWriter::RecordRelocation(MachObjectWriter *Writer,
     }
 
     // Check whether we need an external or internal relocation.
-    if (Writer->doesSymbolRequireExternRelocation(SD)) {
+    if (requiresExternRelocation(Writer, Asm, *Fragment, RelocType, SD,
+                                 FixedValue)) {
       IsExtern = 1;
       Index = SD->getIndex();
 
@@ -410,7 +457,7 @@ void ARMMachObjectWriter::RecordRelocation(MachObjectWriter *Writer,
   if (Type == macho::RIT_ARM_Half) {
     // The other-half value only gets populated for the movt and movw
     // relocation entries.
-    uint32_t Value = 0;;
+    uint32_t Value = 0;
     switch ((unsigned)Fixup.getKind()) {
     default: break;
     case ARM::fixup_arm_movw_lo16:
diff --git a/lib/Target/ARM/MLxExpansionPass.cpp b/lib/Target/ARM/MLxExpansionPass.cpp
index ad60e32..70643bc 100644
--- a/lib/Target/ARM/MLxExpansionPass.cpp
+++ b/lib/Target/ARM/MLxExpansionPass.cpp
@@ -51,7 +51,8 @@ namespace {
     const TargetRegisterInfo *TRI;
     MachineRegisterInfo *MRI;
 
-    bool isA9;
+    bool isLikeA9;
+    bool isSwift;
     unsigned MIIdx;
     MachineInstr* LastMIs[4];
     SmallPtrSet<MachineInstr*, 4> IgnoreStall;
@@ -60,6 +61,7 @@ namespace {
     void pushStack(MachineInstr *MI);
     MachineInstr *getAccDefMI(MachineInstr *MI) const;
     unsigned getDefReg(MachineInstr *MI) const;
+    bool hasLoopHazard(MachineInstr *MI) const;
     bool hasRAWHazard(unsigned Reg, MachineInstr *MI) const;
     bool FindMLxHazard(MachineInstr *MI);
     void ExpandFPMLxInstruction(MachineBasicBlock &MBB, MachineInstr *MI,
@@ -135,6 +137,50 @@ unsigned MLxExpansion::getDefReg(MachineInstr *MI) const {
   return Reg;
 }
 
+/// hasLoopHazard - Check whether an MLx instruction is chained to itself across
+/// a single-MBB loop.
+bool MLxExpansion::hasLoopHazard(MachineInstr *MI) const {
+  unsigned Reg = MI->getOperand(1).getReg();
+  if (TargetRegisterInfo::isPhysicalRegister(Reg))
+    return false;
+
+  MachineBasicBlock *MBB = MI->getParent();
+  MachineInstr *DefMI = MRI->getVRegDef(Reg);
+  while (true) {
+outer_continue:
+    if (DefMI->getParent() != MBB)
+      break;
+
+    if (DefMI->isPHI()) {
+      for (unsigned i = 1, e = DefMI->getNumOperands(); i < e; i += 2) {
+        if (DefMI->getOperand(i + 1).getMBB() == MBB) {
+          unsigned SrcReg = DefMI->getOperand(i).getReg();
+          if (TargetRegisterInfo::isVirtualRegister(SrcReg)) {
+            DefMI = MRI->getVRegDef(SrcReg);
+            goto outer_continue;
+          }
+        }
+      }
+    } else if (DefMI->isCopyLike()) {
+      Reg = DefMI->getOperand(1).getReg();
+      if (TargetRegisterInfo::isVirtualRegister(Reg)) {
+        DefMI = MRI->getVRegDef(Reg);
+        continue;
+      }
+    } else if (DefMI->isInsertSubreg()) {
+      Reg = DefMI->getOperand(2).getReg();
+      if (TargetRegisterInfo::isVirtualRegister(Reg)) {
+        DefMI = MRI->getVRegDef(Reg);
+        continue;
+      }
+    }
+
+    break;
+  }
+
+  return DefMI == MI;
+}
+
 bool MLxExpansion::hasRAWHazard(unsigned Reg, MachineInstr *MI) const {
   // FIXME: Detect integer instructions properly.
   const MCInstrDesc &MCID = MI->getDesc();
@@ -149,6 +195,19 @@ bool MLxExpansion::hasRAWHazard(unsigned Reg, MachineInstr *MI) const {
   return false;
 }
 
+static bool isFpMulInstruction(unsigned Opcode) {
+  switch (Opcode) {
+  case ARM::VMULS:
+  case ARM::VMULfd:
+  case ARM::VMULfq:
+  case ARM::VMULD:
+  case ARM::VMULslfd:
+  case ARM::VMULslfq:
+    return true;
+  default:
+    return false;
+  }
+}
 
 bool MLxExpansion::FindMLxHazard(MachineInstr *MI) {
   if (NumExpand >= ExpandLimit)
@@ -171,6 +230,12 @@ bool MLxExpansion::FindMLxHazard(MachineInstr *MI) {
     return true;
   }
 
+  // On Swift, we mostly care about hazards from multiplication instructions
+  // writing the accumulator and the pipelining of loop iterations by out-of-
+  // order execution. 
+  if (isSwift)
+    return isFpMulInstruction(DefMI->getOpcode()) || hasLoopHazard(MI);
+
   if (IgnoreStall.count(MI))
     return false;
 
@@ -179,8 +244,8 @@ bool MLxExpansion::FindMLxHazard(MachineInstr *MI) {
   // preserves the in-order retirement of the instructions.
   // Look at the next few instructions, if *most* of them can cause hazards,
   // then the scheduler can't *fix* this, we'd better break up the VMLA.
-  unsigned Limit1 = isA9 ? 1 : 4;
-  unsigned Limit2 = isA9 ? 1 : 4;
+  unsigned Limit1 = isLikeA9 ? 1 : 4;
+  unsigned Limit2 = isLikeA9 ? 1 : 4;
   for (unsigned i = 1; i <= 4; ++i) {
     int Idx = ((int)MIIdx - i + 4) % 4;
     MachineInstr *NextMI = LastMIs[Idx];
@@ -316,7 +381,8 @@ bool MLxExpansion::runOnMachineFunction(MachineFunction &Fn) {
   TRI = Fn.getTarget().getRegisterInfo();
   MRI = &Fn.getRegInfo();
   const ARMSubtarget *STI = &Fn.getTarget().getSubtarget<ARMSubtarget>();
-  isA9 = STI->isCortexA9();
+  isLikeA9 = STI->isLikeA9() || STI->isSwift();
+  isSwift = STI->isSwift();
 
   bool Modified = false;
   for (MachineFunction::iterator MFI = Fn.begin(), E = Fn.end(); MFI != E;
diff --git a/lib/Target/CMakeLists.txt b/lib/Target/CMakeLists.txt
index 5913a9c..a85acaa 100644
--- a/lib/Target/CMakeLists.txt
+++ b/lib/Target/CMakeLists.txt
@@ -1,8 +1,6 @@
 add_llvm_library(LLVMTarget
   Mangler.cpp
   Target.cpp
-  TargetData.cpp
-  TargetELFWriterInfo.cpp
   TargetInstrInfo.cpp
   TargetIntrinsicInfo.cpp
   TargetJITInfo.cpp
@@ -12,6 +10,7 @@ add_llvm_library(LLVMTarget
   TargetMachineC.cpp
   TargetRegisterInfo.cpp
   TargetSubtargetInfo.cpp
+  TargetTransformImpl.cpp
   )
 
 foreach(t ${LLVM_TARGETS_TO_BUILD})
diff --git a/lib/Target/CellSPU/SPUAsmPrinter.cpp b/lib/Target/CellSPU/SPUAsmPrinter.cpp
index 03d5a9a..3396e8b 100644
--- a/lib/Target/CellSPU/SPUAsmPrinter.cpp
+++ b/lib/Target/CellSPU/SPUAsmPrinter.cpp
@@ -130,8 +130,7 @@ namespace {
     void
     printS10ImmOperand(const MachineInstr *MI, unsigned OpNo, raw_ostream &O)
     {
-      short value = (short) (((int) MI->getOperand(OpNo).getImm() << 16)
-                             >> 16);
+      short value = MI->getOperand(OpNo).getImm();
       assert((value >= -(1 << 9) && value <= (1 << 9) - 1)
              && "Invalid s10 argument");
       O << value;
@@ -140,8 +139,7 @@ namespace {
     void
     printU10ImmOperand(const MachineInstr *MI, unsigned OpNo, raw_ostream &O)
     {
-      short value = (short) (((int) MI->getOperand(OpNo).getImm() << 16)
-                             >> 16);
+      short value = MI->getOperand(OpNo).getImm();
       assert((value <= (1 << 10) - 1) && "Invalid u10 argument");
       O << value;
     }
diff --git a/lib/Target/CellSPU/SPUFrameLowering.cpp b/lib/Target/CellSPU/SPUFrameLowering.cpp
index fac806e1..f011995 100644
--- a/lib/Target/CellSPU/SPUFrameLowering.cpp
+++ b/lib/Target/CellSPU/SPUFrameLowering.cpp
@@ -22,7 +22,7 @@
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/RegisterScavenging.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetOptions.h"
 #include "llvm/Support/CommandLine.h"
 using namespace llvm;
diff --git a/lib/Target/CellSPU/SPUISelDAGToDAG.cpp b/lib/Target/CellSPU/SPUISelDAGToDAG.cpp
index c27caea..5d50610 100644
--- a/lib/Target/CellSPU/SPUISelDAGToDAG.cpp
+++ b/lib/Target/CellSPU/SPUISelDAGToDAG.cpp
@@ -67,8 +67,8 @@ namespace {
 
   //! ConstantSDNode predicate for signed 16-bit values
   /*!
-    \arg CN The constant SelectionDAG node holding the value
-    \arg Imm The returned 16-bit value, if returning true
+    \param CN The constant SelectionDAG node holding the value
+    \param Imm The returned 16-bit value, if returning true
 
     This predicate tests the value in \a CN to see whether it can be
     represented as a 16-bit, sign-extended quantity. Returns true if
@@ -83,12 +83,10 @@ namespace {
       return true;
     } else if (vt == MVT::i32) {
       int32_t i_val = (int32_t) CN->getZExtValue();
-      short s_val = (short) i_val;
-      return i_val == s_val;
+      return i_val == SignExtend32<16>(i_val);
     } else {
       int64_t i_val = (int64_t) CN->getZExtValue();
-      short s_val = (short) i_val;
-      return i_val == s_val;
+      return i_val == SignExtend64<16>(i_val);
     }
   }
 
@@ -99,9 +97,10 @@ namespace {
     EVT vt = FPN->getValueType(0);
     if (vt == MVT::f32) {
       int val = FloatToBits(FPN->getValueAPF().convertToFloat());
-      int sval = (int) ((val << 16) >> 16);
-      Imm = (short) val;
-      return val == sval;
+      if (val == SignExtend32<16>(val)) {
+        Imm = (short) val;
+        return true;
+      }
     }
 
     return false;
@@ -306,10 +305,10 @@ namespace {
 }
 
 /*!
- \arg Op The ISD instruction operand
- \arg N The address to be tested
- \arg Base The base address
- \arg Index The base address index
+ \param Op The ISD instruction operand
+ \param N The address to be tested
+ \param Base The base address
+ \param Index The base address index
  */
 bool
 SPUDAGToDAGISel::SelectAFormAddr(SDNode *Op, SDValue N, SDValue &Base,
@@ -376,10 +375,10 @@ SPUDAGToDAGISel::SelectDForm2Addr(SDNode *Op, SDValue N, SDValue &Disp,
 }
 
 /*!
-  \arg Op The ISD instruction (ignored)
-  \arg N The address to be tested
-  \arg Base Base address register/pointer
-  \arg Index Base address index
+  \param Op The ISD instruction (ignored)
+  \param N The address to be tested
+  \param Base Base address register/pointer
+  \param Index Base address index
 
   Examine the input address by a base register plus a signed 10-bit
   displacement, [r+I10] (D-form address).
@@ -542,10 +541,10 @@ SPUDAGToDAGISel::DFormAddressPredicate(SDNode *Op, SDValue N, SDValue &Base,
 }
 
 /*!
-  \arg Op The ISD instruction operand
-  \arg N The address operand
-  \arg Base The base pointer operand
-  \arg Index The offset/index operand
+  \param Op The ISD instruction operand
+  \param N The address operand
+  \param Base The base pointer operand
+  \param Index The offset/index operand
 
   If the address \a N can be expressed as an A-form or D-form address, returns
   false.  Otherwise, creates two operands, Base and Index that will become the
@@ -570,7 +569,7 @@ SPUDAGToDAGISel::SelectXFormAddr(SDNode *Op, SDValue N, SDValue &Base,
  Utility function to use with COPY_TO_REGCLASS instructions. Returns a SDValue
  to be used as the last parameter of a
 CurDAG->getMachineNode(COPY_TO_REGCLASS,..., ) function call
- \arg VT the value type for which we want a register class
+ \param VT the value type for which we want a register class
 */
 SDValue SPUDAGToDAGISel::getRC( MVT VT ) {
   switch( VT.SimpleTy ) {
diff --git a/lib/Target/CellSPU/SPUSubtarget.h b/lib/Target/CellSPU/SPUSubtarget.h
index 7c4aa14..27d28b2 100644
--- a/lib/Target/CellSPU/SPUSubtarget.h
+++ b/lib/Target/CellSPU/SPUSubtarget.h
@@ -80,9 +80,9 @@ namespace llvm {
       return UseLargeMem;
     }
 
-    /// getTargetDataString - Return the pointer size and type alignment
+    /// getDataLayoutString - Return the pointer size and type alignment
     /// properties of this subtarget.
-    const char *getTargetDataString() const {
+    const char *getDataLayoutString() const {
       return "E-p:32:32:128-f64:64:128-f32:32:128-i64:32:128-i32:32:128"
              "-i16:16:128-i8:8:128-i1:8:128-a:0:128-v64:64:128-v128:128:128"
              "-s:128:128-n32:64";
diff --git a/lib/Target/CellSPU/SPUTargetMachine.cpp b/lib/Target/CellSPU/SPUTargetMachine.cpp
index 54764f1..9183165 100644
--- a/lib/Target/CellSPU/SPUTargetMachine.cpp
+++ b/lib/Target/CellSPU/SPUTargetMachine.cpp
@@ -38,12 +38,13 @@ SPUTargetMachine::SPUTargetMachine(const Target &T, StringRef TT,
                                    CodeGenOpt::Level OL)
   : LLVMTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL),
     Subtarget(TT, CPU, FS),
-    DataLayout(Subtarget.getTargetDataString()),
+    DL(Subtarget.getDataLayoutString()),
     InstrInfo(*this),
     FrameLowering(Subtarget),
     TLInfo(*this),
     TSInfo(*this),
-    InstrItins(Subtarget.getInstrItineraryData()) {
+    InstrItins(Subtarget.getInstrItineraryData()),
+    STTI(&TLInfo), VTTI(&TLInfo) {
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/lib/Target/CellSPU/SPUTargetMachine.h b/lib/Target/CellSPU/SPUTargetMachine.h
index 3e5d38c..7f53ea6 100644
--- a/lib/Target/CellSPU/SPUTargetMachine.h
+++ b/lib/Target/CellSPU/SPUTargetMachine.h
@@ -20,7 +20,8 @@
 #include "SPUSelectionDAGInfo.h"
 #include "SPUFrameLowering.h"
 #include "llvm/Target/TargetMachine.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/Target/TargetTransformImpl.h"
+#include "llvm/DataLayout.h"
 
 namespace llvm {
 
@@ -28,12 +29,14 @@ namespace llvm {
 ///
 class SPUTargetMachine : public LLVMTargetMachine {
   SPUSubtarget        Subtarget;
-  const TargetData    DataLayout;
+  const DataLayout    DL;
   SPUInstrInfo        InstrInfo;
   SPUFrameLowering    FrameLowering;
   SPUTargetLowering   TLInfo;
   SPUSelectionDAGInfo TSInfo;
   InstrItineraryData  InstrItins;
+  ScalarTargetTransformImpl STTI;
+  VectorTargetTransformImpl VTTI;
 public:
   SPUTargetMachine(const Target &T, StringRef TT,
                    StringRef CPU, StringRef FS, const TargetOptions &Options,
@@ -70,13 +73,19 @@ public:
     return &InstrInfo.getRegisterInfo();
   }
 
-  virtual const TargetData *getTargetData() const {
-    return &DataLayout;
+  virtual const DataLayout *getDataLayout() const {
+    return &DL;
   }
 
   virtual const InstrItineraryData *getInstrItineraryData() const {
     return &InstrItins;
   }
+  virtual const ScalarTargetTransformInfo *getScalarTargetTransformInfo()const {
+    return &STTI;
+  }
+  virtual const VectorTargetTransformInfo *getVectorTargetTransformInfo()const {
+    return &VTTI;
+  }
 
   // Pass Pipeline Configuration
   virtual TargetPassConfig *createPassConfig(PassManagerBase &PM);
diff --git a/lib/Target/CppBackend/CPPBackend.cpp b/lib/Target/CppBackend/CPPBackend.cpp
index c8e757b..5c90990 100644
--- a/lib/Target/CppBackend/CPPBackend.cpp
+++ b/lib/Target/CppBackend/CPPBackend.cpp
@@ -285,14 +285,14 @@ void CppWriter::printLinkageType(GlobalValue::LinkageTypes LT) {
     Out << "GlobalValue::LinkerPrivateLinkage"; break;
   case GlobalValue::LinkerPrivateWeakLinkage:
     Out << "GlobalValue::LinkerPrivateWeakLinkage"; break;
-  case GlobalValue::LinkerPrivateWeakDefAutoLinkage:
-    Out << "GlobalValue::LinkerPrivateWeakDefAutoLinkage"; break;
   case GlobalValue::AvailableExternallyLinkage:
     Out << "GlobalValue::AvailableExternallyLinkage "; break;
   case GlobalValue::LinkOnceAnyLinkage:
     Out << "GlobalValue::LinkOnceAnyLinkage "; break;
   case GlobalValue::LinkOnceODRLinkage:
     Out << "GlobalValue::LinkOnceODRLinkage "; break;
+  case GlobalValue::LinkOnceODRAutoHideLinkage:
+    Out << "GlobalValue::LinkOnceODRAutoHideLinkage"; break;
   case GlobalValue::WeakAnyLinkage:
     Out << "GlobalValue::WeakAnyLinkage"; break;
   case GlobalValue::WeakODRLinkage:
@@ -474,13 +474,15 @@ void CppWriter::printAttributes(const AttrListPtr &PAL,
     Out << "AttributeWithIndex PAWI;"; nl(Out);
     for (unsigned i = 0; i < PAL.getNumSlots(); ++i) {
       unsigned index = PAL.getSlot(i).Index;
-      Attributes attrs = PAL.getSlot(i).Attrs;
-      Out << "PAWI.Index = " << index << "U; PAWI.Attrs = Attribute::None ";
-#define HANDLE_ATTR(X)                 \
-      if (attrs & Attribute::X)      \
-        Out << " | Attribute::" #X;  \
-      attrs &= ~Attribute::X;
-      
+      AttrBuilder attrs(PAL.getSlot(i).Attrs);
+      Out << "PAWI.Index = " << index << "U;\n";
+      Out << " {\n    AttrBuilder B;\n";
+
+#define HANDLE_ATTR(X)                                     \
+      if (attrs.hasAttribute(Attributes::X))               \
+        Out << "    B.addAttribute(Attributes::" #X ");\n"; \
+      attrs.removeAttribute(Attributes::X);
+
       HANDLE_ATTR(SExt);
       HANDLE_ATTR(ZExt);
       HANDLE_ATTR(NoReturn);
@@ -505,19 +507,18 @@ void CppWriter::printAttributes(const AttrListPtr &PAL,
       HANDLE_ATTR(ReturnsTwice);
       HANDLE_ATTR(UWTable);
       HANDLE_ATTR(NonLazyBind);
+      HANDLE_ATTR(MinSize);
 #undef HANDLE_ATTR
-      if (attrs & Attribute::StackAlignment)
-        Out << " | Attribute::constructStackAlignmentFromInt("
-            << Attribute::getStackAlignmentFromAttrs(attrs)
-            << ")"; 
-      attrs &= ~Attribute::StackAlignment;
-      assert(attrs == 0 && "Unhandled attribute!");
-      Out << ";";
+      if (attrs.hasAttribute(Attributes::StackAlignment))
+        Out << "    B.addStackAlignmentAttr(" << attrs.getStackAlignment() << ")\n";
+      attrs.removeAttribute(Attributes::StackAlignment);
+      assert(!attrs.hasAttributes() && "Unhandled attribute!");
+      Out << "    PAWI.Attrs = Attributes::get(mod->getContext(), B);\n }";
       nl(Out);
       Out << "Attrs.push_back(PAWI);";
       nl(Out);
     }
-    Out << name << "_PAL = AttrListPtr::get(Attrs);";
+    Out << name << "_PAL = AttrListPtr::get(mod->getContext(), Attrs);";
     nl(Out);
     out(); nl(Out);
     Out << '}'; nl(Out);
diff --git a/lib/Target/CppBackend/CPPTargetMachine.h b/lib/Target/CppBackend/CPPTargetMachine.h
index 9cbe798..30d765d 100644
--- a/lib/Target/CppBackend/CPPTargetMachine.h
+++ b/lib/Target/CppBackend/CPPTargetMachine.h
@@ -15,7 +15,7 @@
 #define CPPTARGETMACHINE_H
 
 #include "llvm/Target/TargetMachine.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 
 namespace llvm {
 
@@ -35,7 +35,7 @@ struct CPPTargetMachine : public TargetMachine {
                                    AnalysisID StartAfter,
                                    AnalysisID StopAfter);
 
-  virtual const TargetData *getTargetData() const { return 0; }
+  virtual const DataLayout *getDataLayout() const { return 0; }
 };
 
 extern Target TheCppBackendTarget;
diff --git a/lib/Target/Hexagon/CMakeLists.txt b/lib/Target/Hexagon/CMakeLists.txt
index 1f2d8ac..306084b 100644
--- a/lib/Target/Hexagon/CMakeLists.txt
+++ b/lib/Target/Hexagon/CMakeLists.txt
@@ -16,6 +16,7 @@ add_llvm_target(HexagonCodeGen
   HexagonExpandPredSpillCode.cpp
   HexagonFrameLowering.cpp
   HexagonHardwareLoops.cpp
+  HexagonMachineScheduler.cpp
   HexagonMCInstLower.cpp
   HexagonInstrInfo.cpp
   HexagonISelDAGToDAG.cpp
diff --git a/lib/Target/Hexagon/HexagonAsmPrinter.cpp b/lib/Target/Hexagon/HexagonAsmPrinter.cpp
index 5fa4740..c15bce6 100644
--- a/lib/Target/Hexagon/HexagonAsmPrinter.cpp
+++ b/lib/Target/Hexagon/HexagonAsmPrinter.cpp
@@ -46,7 +46,7 @@
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/TargetRegistry.h"
 #include "llvm/Target/Mangler.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetLoweringObjectFile.h"
 #include "llvm/Target/TargetRegisterInfo.h"
 #include "llvm/Target/TargetInstrInfo.h"
diff --git a/lib/Target/Hexagon/HexagonCallingConvLower.cpp b/lib/Target/Hexagon/HexagonCallingConvLower.cpp
index ba8e679..73f9d9a 100644
--- a/lib/Target/Hexagon/HexagonCallingConvLower.cpp
+++ b/lib/Target/Hexagon/HexagonCallingConvLower.cpp
@@ -16,7 +16,7 @@
 #include "HexagonCallingConvLower.h"
 #include "Hexagon.h"
 #include "llvm/Target/TargetRegisterInfo.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
diff --git a/lib/Target/Hexagon/HexagonISelLowering.cpp b/lib/Target/Hexagon/HexagonISelLowering.cpp
index 703a128..1c891f1 100644
--- a/lib/Target/Hexagon/HexagonISelLowering.cpp
+++ b/lib/Target/Hexagon/HexagonISelLowering.cpp
@@ -1350,6 +1350,8 @@ HexagonTargetLowering::HexagonTargetLowering(HexagonTargetMachine
     } else {
       setOperationAction(ISD::BR_JT, MVT::Other, Expand);
     }
+    // Increase jump tables cutover to 5, was 4.
+    setMinimumJumpTableEntries(5);
 
     setOperationAction(ISD::BR_CC, MVT::i32, Expand);
 
diff --git a/lib/Target/Hexagon/HexagonInstrFormats.td b/lib/Target/Hexagon/HexagonInstrFormats.td
index e472d49..a64c7a1 100644
--- a/lib/Target/Hexagon/HexagonInstrFormats.td
+++ b/lib/Target/Hexagon/HexagonInstrFormats.td
@@ -56,6 +56,16 @@ class InstHexagon<dag outs, dag ins, string asmstr, list<dag> pattern,
   bits<1> isPredicated = 0;
   let TSFlags{6} = isPredicated;
 
+  // Dot new value store instructions.
+  bits<1> isNVStore = 0;
+  let TSFlags{8} = isNVStore;
+
+  // Fields used for relation models.
+  string BaseOpcode = "";
+  string CextOpcode = "";
+  string PredSense = "";
+  string PNewValue = "";
+  string InputType = "";    // Input is "imm" or "reg" type.
   // *** The code above must match HexagonBaseInfo.h ***
 }
 
diff --git a/lib/Target/Hexagon/HexagonInstrInfo.cpp b/lib/Target/Hexagon/HexagonInstrInfo.cpp
index c8f933d..8435440 100644
--- a/lib/Target/Hexagon/HexagonInstrInfo.cpp
+++ b/lib/Target/Hexagon/HexagonInstrInfo.cpp
@@ -25,6 +25,7 @@
 #include "llvm/CodeGen/PseudoSourceValue.h"
 #include "llvm/Support/MathExtras.h"
 #define GET_INSTRINFO_CTOR
+#define GET_INSTRMAP_INFO
 #include "HexagonGenInstrInfo.inc"
 #include "HexagonGenDFAPacketizer.inc"
 
@@ -1915,6 +1916,15 @@ unsigned HexagonInstrInfo::getInvertedPredicatedOpcode(const int Opc) const {
 
 int HexagonInstrInfo::
 getMatchingCondBranchOpcode(int Opc, bool invertPredicate) const {
+  enum Hexagon::PredSense inPredSense;
+  inPredSense = invertPredicate ? Hexagon::PredSense_false :
+                                  Hexagon::PredSense_true;
+  int CondOpcode = Hexagon::getPredOpcode(Opc, inPredSense);
+  if (CondOpcode >= 0) // Valid Conditional opcode/instruction
+    return CondOpcode;
+
+  // This switch case will be removed once all the instructions have been
+  // modified to use relation maps.
   switch(Opc) {
   case Hexagon::TFR:
     return !invertPredicate ? Hexagon::TFR_cPt :
@@ -1934,24 +1944,6 @@ getMatchingCondBranchOpcode(int Opc, bool invertPredicate) const {
   case Hexagon::JMP_EQriPt_nv_V4:
     return !invertPredicate ? Hexagon::JMP_EQriPt_nv_V4 :
                               Hexagon::JMP_EQriNotPt_nv_V4;
-  case Hexagon::ADD_ri:
-    return !invertPredicate ? Hexagon::ADD_ri_cPt :
-                              Hexagon::ADD_ri_cNotPt;
-  case Hexagon::ADD_rr:
-    return !invertPredicate ? Hexagon::ADD_rr_cPt :
-                              Hexagon::ADD_rr_cNotPt;
-  case Hexagon::XOR_rr:
-    return !invertPredicate ? Hexagon::XOR_rr_cPt :
-                              Hexagon::XOR_rr_cNotPt;
-  case Hexagon::AND_rr:
-    return !invertPredicate ? Hexagon::AND_rr_cPt :
-                              Hexagon::AND_rr_cNotPt;
-  case Hexagon::OR_rr:
-    return !invertPredicate ? Hexagon::OR_rr_cPt :
-                              Hexagon::OR_rr_cNotPt;
-  case Hexagon::SUB_rr:
-    return !invertPredicate ? Hexagon::SUB_rr_cPt :
-                              Hexagon::SUB_rr_cNotPt;
   case Hexagon::COMBINE_rr:
     return !invertPredicate ? Hexagon::COMBINE_rr_cPt :
                               Hexagon::COMBINE_rr_cNotPt;
diff --git a/lib/Target/Hexagon/HexagonInstrInfo.td b/lib/Target/Hexagon/HexagonInstrInfo.td
index c0c0df6..1d4a706 100644
--- a/lib/Target/Hexagon/HexagonInstrInfo.td
+++ b/lib/Target/Hexagon/HexagonInstrInfo.td
@@ -15,6 +15,18 @@ include "HexagonInstrFormats.td"
 include "HexagonImmediates.td"
 
 //===----------------------------------------------------------------------===//
+// Classes used for relation maps.
+//===----------------------------------------------------------------------===//
+// PredRel - Filter class used to relate non-predicated instructions with their
+// predicated forms.
+class PredRel;
+// PredNewRel - Filter class used to relate predicated instructions with their
+// predicate-new forms.
+class PredNewRel: PredRel;
+// ImmRegRel - Filter class used to relate instructions having reg-reg form
+// with their reg-imm counterparts.
+class ImmRegRel;
+//===----------------------------------------------------------------------===//
 // Hexagon Instruction Predicate Definitions.
 //===----------------------------------------------------------------------===//
 def HasV2T                      : Predicate<"Subtarget.hasV2TOps()">;
@@ -148,37 +160,91 @@ multiclass CMP32_ri_s8<string OpcStr, PatFrag OpNode> {
 }
 
 //===----------------------------------------------------------------------===//
-// ALU32/ALU +
+// ALU32/ALU (Instructions with register-register form)
 //===----------------------------------------------------------------------===//
-// Add.
-let isCommutable = 1, isPredicable = 1 in
-def ADD_rr : ALU32_rr<(outs IntRegs:$dst),
-            (ins IntRegs:$src1, IntRegs:$src2),
-            "$dst = add($src1, $src2)",
-            [(set (i32 IntRegs:$dst), (add (i32 IntRegs:$src1),
-                                           (i32 IntRegs:$src2)))]>;
+multiclass ALU32_Pbase<string mnemonic, bit isNot,
+                       bit isPredNew> {
 
-let isPredicable = 1 in
-def ADD_ri : ALU32_ri<(outs IntRegs:$dst),
-            (ins IntRegs:$src1, s16Imm:$src2),
-            "$dst = add($src1, #$src2)",
-            [(set (i32 IntRegs:$dst), (add (i32 IntRegs:$src1),
-                                           s16ImmPred:$src2))]>;
+  let PNewValue = #!if(isPredNew, "new", "") in
+  def #NAME# : ALU32_rr<(outs IntRegs:$dst),
+            (ins PredRegs:$src1, IntRegs:$src2, IntRegs: $src3),
+            !if(isNot, "if (!$src1", "if ($src1")#!if(isPredNew,".new) $dst = ",
+            ") $dst = ")#mnemonic#"($src2, $src3)",
+            []>;
+}
 
-// Logical operations.
-let isPredicable = 1 in
-def XOR_rr : ALU32_rr<(outs IntRegs:$dst),
-            (ins IntRegs:$src1, IntRegs:$src2),
-            "$dst = xor($src1, $src2)",
-            [(set (i32 IntRegs:$dst), (xor (i32 IntRegs:$src1),
-                                           (i32 IntRegs:$src2)))]>;
+multiclass ALU32_Pred<string mnemonic, bit PredNot> {
+  let PredSense = #!if(PredNot, "false", "true") in {
+    defm _c#NAME# : ALU32_Pbase<mnemonic, PredNot, 0>;
+    // Predicate new
+    defm _cdn#NAME# : ALU32_Pbase<mnemonic, PredNot, 1>;
+  }
+}
 
-let isCommutable = 1, isPredicable = 1 in
-def AND_rr : ALU32_rr<(outs IntRegs:$dst),
+let InputType = "reg" in
+multiclass ALU32_base<string mnemonic, string CextOp, SDNode OpNode> {
+  let CextOpcode = CextOp, BaseOpcode = CextOp#_rr in {
+    let isPredicable = 1 in
+    def #NAME# : ALU32_rr<(outs IntRegs:$dst),
             (ins IntRegs:$src1, IntRegs:$src2),
-            "$dst = and($src1, $src2)",
-            [(set (i32 IntRegs:$dst), (and (i32 IntRegs:$src1),
-                                           (i32 IntRegs:$src2)))]>;
+            "$dst = "#mnemonic#"($src1, $src2)",
+            [(set (i32 IntRegs:$dst), (OpNode (i32 IntRegs:$src1),
+                                              (i32 IntRegs:$src2)))]>;
+
+    let neverHasSideEffects = 1, isPredicated = 1 in {
+      defm Pt : ALU32_Pred<mnemonic, 0>;
+      defm NotPt : ALU32_Pred<mnemonic, 1>;
+    }
+  }
+}
+
+let isCommutable = 1 in {
+  defm ADD_rr : ALU32_base<"add", "ADD", add>, ImmRegRel, PredNewRel;
+  defm AND_rr : ALU32_base<"and", "AND", and>, ImmRegRel, PredNewRel;
+  defm XOR_rr : ALU32_base<"xor", "XOR", xor>, ImmRegRel, PredNewRel;
+  defm OR_rr  : ALU32_base<"or", "OR", or>, ImmRegRel, PredNewRel;
+}
+
+defm SUB_rr : ALU32_base<"sub", "SUB", sub>, ImmRegRel, PredNewRel;
+
+//===----------------------------------------------------------------------===//
+// ALU32/ALU (ADD with register-immediate form)
+//===----------------------------------------------------------------------===//
+multiclass ALU32ri_Pbase<string mnemonic, bit isNot, bit isPredNew> {
+  let PNewValue = #!if(isPredNew, "new", "") in
+  def #NAME# : ALU32_ri<(outs IntRegs:$dst),
+            (ins PredRegs:$src1, IntRegs:$src2, s8Imm: $src3),
+            !if(isNot, "if (!$src1", "if ($src1")#!if(isPredNew,".new) $dst = ",
+            ") $dst = ")#mnemonic#"($src2, #$src3)",
+            []>;
+}
+
+multiclass ALU32ri_Pred<string mnemonic, bit PredNot> {
+  let PredSense = #!if(PredNot, "false", "true") in {
+    defm _c#NAME# : ALU32ri_Pbase<mnemonic, PredNot, 0>;
+    // Predicate new
+    defm _cdn#NAME# : ALU32ri_Pbase<mnemonic, PredNot, 1>;
+  }
+}
+
+let InputType = "imm" in
+multiclass ALU32ri_base<string mnemonic, string CextOp, SDNode OpNode> {
+  let CextOpcode = CextOp, BaseOpcode = CextOp#_ri in {
+    let isPredicable = 1 in
+    def #NAME# : ALU32_ri<(outs IntRegs:$dst),
+            (ins IntRegs:$src1, s16Imm:$src2),
+            "$dst = "#mnemonic#"($src1, #$src2)",
+            [(set (i32 IntRegs:$dst), (OpNode (i32 IntRegs:$src1),
+                                              (s16ImmPred:$src2)))]>;
+
+    let neverHasSideEffects = 1, isPredicated = 1 in {
+      defm Pt : ALU32ri_Pred<mnemonic, 0>;
+      defm NotPt : ALU32ri_Pred<mnemonic, 1>;
+    }
+  }
+}
+
+defm ADD_ri : ALU32ri_base<"add", "ADD", add>, ImmRegRel, PredNewRel;
 
 def OR_ri : ALU32_ri<(outs IntRegs:$dst),
             (ins IntRegs:$src1, s10Imm:$src2),
@@ -197,13 +263,6 @@ def AND_ri : ALU32_ri<(outs IntRegs:$dst),
             [(set (i32 IntRegs:$dst), (and (i32 IntRegs:$src1),
                                            s10ImmPred:$src2))]>;
 
-let isCommutable = 1, isPredicable = 1 in
-def OR_rr : ALU32_rr<(outs IntRegs:$dst),
-            (ins IntRegs:$src1, IntRegs:$src2),
-            "$dst = or($src1, $src2)",
-            [(set (i32 IntRegs:$dst), (or (i32 IntRegs:$src1),
-                                          (i32 IntRegs:$src2)))]>;
-
 // Negate.
 def NEG : ALU32_rr<(outs IntRegs:$dst), (ins IntRegs:$src1),
           "$dst = neg($src1)",
@@ -214,14 +273,6 @@ def NOP : ALU32_rr<(outs), (ins),
           "nop",
           []>;
 
-// Subtract.
-let isPredicable = 1 in
-def SUB_rr : ALU32_rr<(outs IntRegs:$dst),
-            (ins IntRegs:$src1, IntRegs:$src2),
-            "$dst = sub($src1, $src2)",
-            [(set (i32 IntRegs:$dst), (sub (i32 IntRegs:$src1),
-                                           (i32 IntRegs:$src2)))]>;
-
 // Rd32=sub(#s10,Rs32)
 def SUB_ri : ALU32_ri<(outs IntRegs:$dst),
             (ins s10Imm:$src1, IntRegs:$src2),
@@ -348,56 +399,6 @@ def ZXTH : ALU32_rr<(outs IntRegs:$dst), (ins IntRegs:$src1),
 // ALU32/PRED +
 //===----------------------------------------------------------------------===//
 
-// Conditional add.
-let neverHasSideEffects = 1, isPredicated = 1 in
-def ADD_ri_cPt : ALU32_ri<(outs IntRegs:$dst),
-            (ins PredRegs:$src1, IntRegs:$src2, s8Imm:$src3),
-            "if ($src1) $dst = add($src2, #$src3)",
-            []>;
-
-let neverHasSideEffects = 1, isPredicated = 1 in
-def ADD_ri_cNotPt : ALU32_ri<(outs IntRegs:$dst),
-            (ins PredRegs:$src1, IntRegs:$src2, s8Imm:$src3),
-            "if (!$src1) $dst = add($src2, #$src3)",
-            []>;
-
-let neverHasSideEffects = 1, isPredicated = 1 in
-def ADD_ri_cdnPt : ALU32_ri<(outs IntRegs:$dst),
-            (ins PredRegs:$src1, IntRegs:$src2, s8Imm:$src3),
-            "if ($src1.new) $dst = add($src2, #$src3)",
-            []>;
-
-let neverHasSideEffects = 1, isPredicated = 1 in
-def ADD_ri_cdnNotPt : ALU32_ri<(outs IntRegs:$dst),
-            (ins PredRegs:$src1, IntRegs:$src2, s8Imm:$src3),
-            "if (!$src1.new) $dst = add($src2, #$src3)",
-            []>;
-
-let neverHasSideEffects = 1, isPredicated = 1 in
-def ADD_rr_cPt : ALU32_rr<(outs IntRegs:$dst),
-            (ins PredRegs:$src1, IntRegs:$src2, IntRegs:$src3),
-            "if ($src1) $dst = add($src2, $src3)",
-            []>;
-
-let neverHasSideEffects = 1, isPredicated = 1 in
-def ADD_rr_cNotPt : ALU32_rr<(outs IntRegs:$dst),
-            (ins PredRegs:$src1, IntRegs:$src2, IntRegs:$src3),
-            "if (!$src1) $dst = add($src2, $src3)",
-            []>;
-
-let neverHasSideEffects = 1, isPredicated = 1 in
-def ADD_rr_cdnPt : ALU32_rr<(outs IntRegs:$dst),
-            (ins PredRegs:$src1, IntRegs:$src2, IntRegs:$src3),
-            "if ($src1.new) $dst = add($src2, $src3)",
-            []>;
-
-let neverHasSideEffects = 1, isPredicated = 1 in
-def ADD_rr_cdnNotPt : ALU32_rr<(outs IntRegs:$dst),
-            (ins PredRegs:$src1, IntRegs:$src2, IntRegs:$src3),
-            "if (!$src1.new) $dst = add($src2, $src3)",
-            []>;
-
-
 // Conditional combine.
 
 let neverHasSideEffects = 1, isPredicated = 1 in
@@ -424,108 +425,6 @@ def COMBINE_rr_cdnNotPt : ALU32_rr<(outs DoubleRegs:$dst),
             "if (!$src1.new) $dst = combine($src2, $src3)",
             []>;
 
-// Conditional logical operations.
-
-let isPredicated = 1 in
-def XOR_rr_cPt : ALU32_rr<(outs IntRegs:$dst),
-            (ins PredRegs:$src1, IntRegs:$src2, IntRegs:$src3),
-            "if ($src1) $dst = xor($src2, $src3)",
-            []>;
-
-let isPredicated = 1 in
-def XOR_rr_cNotPt : ALU32_rr<(outs IntRegs:$dst),
-            (ins PredRegs:$src1, IntRegs:$src2, IntRegs:$src3),
-            "if (!$src1) $dst = xor($src2, $src3)",
-            []>;
-
-let isPredicated = 1 in
-def XOR_rr_cdnPt : ALU32_rr<(outs IntRegs:$dst),
-            (ins PredRegs:$src1, IntRegs:$src2, IntRegs:$src3),
-            "if ($src1.new) $dst = xor($src2, $src3)",
-            []>;
-
-let isPredicated = 1 in
-def XOR_rr_cdnNotPt : ALU32_rr<(outs IntRegs:$dst),
-            (ins PredRegs:$src1, IntRegs:$src2, IntRegs:$src3),
-            "if (!$src1.new) $dst = xor($src2, $src3)",
-            []>;
-
-let isPredicated = 1 in
-def AND_rr_cPt : ALU32_rr<(outs IntRegs:$dst),
-            (ins PredRegs:$src1, IntRegs:$src2, IntRegs:$src3),
-            "if ($src1) $dst = and($src2, $src3)",
-            []>;
-
-let isPredicated = 1 in
-def AND_rr_cNotPt : ALU32_rr<(outs IntRegs:$dst),
-            (ins PredRegs:$src1, IntRegs:$src2, IntRegs:$src3),
-            "if (!$src1) $dst = and($src2, $src3)",
-            []>;
-
-let isPredicated = 1 in
-def AND_rr_cdnPt : ALU32_rr<(outs IntRegs:$dst),
-            (ins PredRegs:$src1, IntRegs:$src2, IntRegs:$src3),
-            "if ($src1.new) $dst = and($src2, $src3)",
-            []>;
-
-let isPredicated = 1 in
-def AND_rr_cdnNotPt : ALU32_rr<(outs IntRegs:$dst),
-            (ins PredRegs:$src1, IntRegs:$src2, IntRegs:$src3),
-            "if (!$src1.new) $dst = and($src2, $src3)",
-            []>;
-
-let isPredicated = 1 in
-def OR_rr_cPt : ALU32_rr<(outs IntRegs:$dst),
-            (ins PredRegs:$src1, IntRegs:$src2, IntRegs:$src3),
-            "if ($src1) $dst = or($src2, $src3)",
-            []>;
-
-let isPredicated = 1 in
-def OR_rr_cNotPt : ALU32_rr<(outs IntRegs:$dst),
-            (ins PredRegs:$src1, IntRegs:$src2, IntRegs:$src3),
-            "if (!$src1) $dst = or($src2, $src3)",
-            []>;
-
-let isPredicated = 1 in
-def OR_rr_cdnPt : ALU32_rr<(outs IntRegs:$dst),
-            (ins PredRegs:$src1, IntRegs:$src2, IntRegs:$src3),
-            "if ($src1.new) $dst = or($src2, $src3)",
-            []>;
-
-let isPredicated = 1 in
-def OR_rr_cdnNotPt : ALU32_rr<(outs IntRegs:$dst),
-            (ins PredRegs:$src1, IntRegs:$src2, IntRegs:$src3),
-            "if (!$src1.new) $dst = or($src2, $src3)",
-            []>;
-
-
-// Conditional subtract.
-
-let isPredicated = 1 in
-def SUB_rr_cPt : ALU32_rr<(outs IntRegs:$dst),
-            (ins PredRegs:$src1, IntRegs:$src2, IntRegs:$src3),
-            "if ($src1) $dst = sub($src2, $src3)",
-            []>;
-
-let isPredicated = 1 in
-def SUB_rr_cNotPt : ALU32_rr<(outs IntRegs:$dst),
-            (ins PredRegs:$src1, IntRegs:$src2, IntRegs:$src3),
-            "if (!$src1) $dst = sub($src2, $src3)",
-            []>;
-
-let isPredicated = 1 in
-def SUB_rr_cdnPt : ALU32_rr<(outs IntRegs:$dst),
-            (ins PredRegs:$src1, IntRegs:$src2, IntRegs:$src3),
-            "if ($src1.new) $dst = sub($src2, $src3)",
-            []>;
-
-let isPredicated = 1 in
-def SUB_rr_cdnNotPt : ALU32_rr<(outs IntRegs:$dst),
-            (ins PredRegs:$src1, IntRegs:$src2, IntRegs:$src3),
-            "if (!$src1.new) $dst = sub($src2, $src3)",
-            []>;
-
-
 // Conditional transfer.
 let neverHasSideEffects = 1, isPredicated = 1 in
 def TFR_cPt : ALU32_rr<(outs IntRegs:$dst), (ins PredRegs:$src1, IntRegs:$src2),
@@ -3546,4 +3445,31 @@ include "HexagonInstrInfoV5.td"
 // V5 Instructions -
 //===----------------------------------------------------------------------===//
 
+//===----------------------------------------------------------------------===//
+// Generate mapping table to relate non-predicate instructions with their
+// predicated formats - true and false.
+//
+
+def getPredOpcode : InstrMapping {
+  let FilterClass = "PredRel";
+  // Instructions with the same BaseOpcode and isNVStore values form a row.
+  let RowFields = ["BaseOpcode", "isNVStore", "PNewValue"];
+  // Instructions with the same predicate sense form a column.
+  let ColFields = ["PredSense"];
+  // The key column is the unpredicated instructions.
+  let KeyCol = [""];
+  // Value columns are PredSense=true and PredSense=false
+  let ValueCols = [["true"], ["false"]];
+}
 
+//===----------------------------------------------------------------------===//
+// Generate mapping table to relate predicated instructions with their .new
+// format.
+//
+def getPredNewOpcode : InstrMapping {
+  let FilterClass = "PredNewRel";
+  let RowFields = ["BaseOpcode", "PredSense", "isNVStore"];
+  let ColFields = ["PNewValue"];
+  let KeyCol = [""];
+  let ValueCols = [["new"]];
+}
diff --git a/lib/Target/Hexagon/HexagonMachineScheduler.cpp b/lib/Target/Hexagon/HexagonMachineScheduler.cpp
new file mode 100644
index 0000000..0e9ef48
--- /dev/null
+++ b/lib/Target/Hexagon/HexagonMachineScheduler.cpp
@@ -0,0 +1,681 @@
+//===- HexagonMachineScheduler.cpp - MI Scheduler for Hexagon -------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// MachineScheduler schedules machine instructions after phi elimination. It
+// preserves LiveIntervals so it can be invoked before register allocation.
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "misched"
+
+#include "HexagonMachineScheduler.h"
+
+#include <queue>
+
+using namespace llvm;
+
+/// Platform specific modifications to DAG.
+void VLIWMachineScheduler::postprocessDAG() {
+  SUnit* LastSequentialCall = NULL;
+  // Currently we only catch the situation when compare gets scheduled
+  // before preceding call.
+  for (unsigned su = 0, e = SUnits.size(); su != e; ++su) {
+    // Remember the call.
+    if (SUnits[su].getInstr()->isCall())
+      LastSequentialCall = &(SUnits[su]);
+    // Look for a compare that defines a predicate.
+    else if (SUnits[su].getInstr()->isCompare() && LastSequentialCall)
+      SUnits[su].addPred(SDep(LastSequentialCall, SDep::Barrier));
+  }
+}
+
+/// Check if scheduling of this SU is possible
+/// in the current packet.
+/// It is _not_ precise (statefull), it is more like
+/// another heuristic. Many corner cases are figured
+/// empirically.
+bool VLIWResourceModel::isResourceAvailable(SUnit *SU) {
+  if (!SU || !SU->getInstr())
+    return false;
+
+  // First see if the pipeline could receive this instruction
+  // in the current cycle.
+  switch (SU->getInstr()->getOpcode()) {
+  default:
+    if (!ResourcesModel->canReserveResources(SU->getInstr()))
+      return false;
+  case TargetOpcode::EXTRACT_SUBREG:
+  case TargetOpcode::INSERT_SUBREG:
+  case TargetOpcode::SUBREG_TO_REG:
+  case TargetOpcode::REG_SEQUENCE:
+  case TargetOpcode::IMPLICIT_DEF:
+  case TargetOpcode::COPY:
+  case TargetOpcode::INLINEASM:
+    break;
+  }
+
+  // Now see if there are no other dependencies to instructions already
+  // in the packet.
+  for (unsigned i = 0, e = Packet.size(); i != e; ++i) {
+    if (Packet[i]->Succs.size() == 0)
+      continue;
+    for (SUnit::const_succ_iterator I = Packet[i]->Succs.begin(),
+         E = Packet[i]->Succs.end(); I != E; ++I) {
+      // Since we do not add pseudos to packets, might as well
+      // ignore order dependencies.
+      if (I->isCtrl())
+        continue;
+
+      if (I->getSUnit() == SU)
+        return false;
+    }
+  }
+  return true;
+}
+
+/// Keep track of available resources.
+bool VLIWResourceModel::reserveResources(SUnit *SU) {
+  bool startNewCycle = false;
+  // Artificially reset state.
+  if (!SU) {
+    ResourcesModel->clearResources();
+    Packet.clear();
+    TotalPackets++;
+    return false;
+  }
+  // If this SU does not fit in the packet
+  // start a new one.
+  if (!isResourceAvailable(SU)) {
+    ResourcesModel->clearResources();
+    Packet.clear();
+    TotalPackets++;
+    startNewCycle = true;
+  }
+
+  switch (SU->getInstr()->getOpcode()) {
+  default:
+    ResourcesModel->reserveResources(SU->getInstr());
+    break;
+  case TargetOpcode::EXTRACT_SUBREG:
+  case TargetOpcode::INSERT_SUBREG:
+  case TargetOpcode::SUBREG_TO_REG:
+  case TargetOpcode::REG_SEQUENCE:
+  case TargetOpcode::IMPLICIT_DEF:
+  case TargetOpcode::KILL:
+  case TargetOpcode::PROLOG_LABEL:
+  case TargetOpcode::EH_LABEL:
+  case TargetOpcode::COPY:
+  case TargetOpcode::INLINEASM:
+    break;
+  }
+  Packet.push_back(SU);
+
+#ifndef NDEBUG
+  DEBUG(dbgs() << "Packet[" << TotalPackets << "]:\n");
+  for (unsigned i = 0, e = Packet.size(); i != e; ++i) {
+    DEBUG(dbgs() << "\t[" << i << "] SU(");
+    DEBUG(dbgs() << Packet[i]->NodeNum << ")\t");
+    DEBUG(Packet[i]->getInstr()->dump());
+  }
+#endif
+
+  // If packet is now full, reset the state so in the next cycle
+  // we start fresh.
+  if (Packet.size() >= SchedModel->getIssueWidth()) {
+    ResourcesModel->clearResources();
+    Packet.clear();
+    TotalPackets++;
+    startNewCycle = true;
+  }
+
+  return startNewCycle;
+}
+
+/// schedule - Called back from MachineScheduler::runOnMachineFunction
+/// after setting up the current scheduling region. [RegionBegin, RegionEnd)
+/// only includes instructions that have DAG nodes, not scheduling boundaries.
+void VLIWMachineScheduler::schedule() {
+  DEBUG(dbgs()
+        << "********** MI Converging Scheduling VLIW BB#" << BB->getNumber()
+        << " " << BB->getName()
+        << " in_func " << BB->getParent()->getFunction()->getName()
+        << " at loop depth "  << MLI.getLoopDepth(BB)
+        << " \n");
+
+  buildDAGWithRegPressure();
+
+  // Postprocess the DAG to add platform specific artificial dependencies.
+  postprocessDAG();
+
+  // To view Height/Depth correctly, they should be accessed at least once.
+  DEBUG(unsigned maxH = 0;
+        for (unsigned su = 0, e = SUnits.size(); su != e; ++su)
+          if (SUnits[su].getHeight() > maxH)
+            maxH = SUnits[su].getHeight();
+        dbgs() << "Max Height " << maxH << "\n";);
+  DEBUG(unsigned maxD = 0;
+        for (unsigned su = 0, e = SUnits.size(); su != e; ++su)
+          if (SUnits[su].getDepth() > maxD)
+            maxD = SUnits[su].getDepth();
+        dbgs() << "Max Depth " << maxD << "\n";);
+  DEBUG(for (unsigned su = 0, e = SUnits.size(); su != e; ++su)
+          SUnits[su].dumpAll(this));
+
+  initQueues();
+
+  bool IsTopNode = false;
+  while (SUnit *SU = SchedImpl->pickNode(IsTopNode)) {
+    if (!checkSchedLimit())
+      break;
+
+    scheduleMI(SU, IsTopNode);
+
+    updateQueues(SU, IsTopNode);
+  }
+  assert(CurrentTop == CurrentBottom && "Nonempty unscheduled zone.");
+
+  placeDebugValues();
+}
+
+void ConvergingVLIWScheduler::initialize(ScheduleDAGMI *dag) {
+  DAG = static_cast<VLIWMachineScheduler*>(dag);
+  SchedModel = DAG->getSchedModel();
+  TRI = DAG->TRI;
+  Top.init(DAG, SchedModel);
+  Bot.init(DAG, SchedModel);
+
+  // Initialize the HazardRecognizers. If itineraries don't exist, are empty, or
+  // are disabled, then these HazardRecs will be disabled.
+  const InstrItineraryData *Itin = DAG->getSchedModel()->getInstrItineraries();
+  const TargetMachine &TM = DAG->MF.getTarget();
+  Top.HazardRec = TM.getInstrInfo()->CreateTargetMIHazardRecognizer(Itin, DAG);
+  Bot.HazardRec = TM.getInstrInfo()->CreateTargetMIHazardRecognizer(Itin, DAG);
+
+  Top.ResourceModel = new VLIWResourceModel(TM, DAG->getSchedModel());
+  Bot.ResourceModel = new VLIWResourceModel(TM, DAG->getSchedModel());
+
+  assert((!llvm::ForceTopDown || !llvm::ForceBottomUp) &&
+         "-misched-topdown incompatible with -misched-bottomup");
+}
+
+void ConvergingVLIWScheduler::releaseTopNode(SUnit *SU) {
+  if (SU->isScheduled)
+    return;
+
+  for (SUnit::succ_iterator I = SU->Preds.begin(), E = SU->Preds.end();
+       I != E; ++I) {
+    unsigned PredReadyCycle = I->getSUnit()->TopReadyCycle;
+    unsigned MinLatency = I->getMinLatency();
+#ifndef NDEBUG
+    Top.MaxMinLatency = std::max(MinLatency, Top.MaxMinLatency);
+#endif
+    if (SU->TopReadyCycle < PredReadyCycle + MinLatency)
+      SU->TopReadyCycle = PredReadyCycle + MinLatency;
+  }
+  Top.releaseNode(SU, SU->TopReadyCycle);
+}
+
+void ConvergingVLIWScheduler::releaseBottomNode(SUnit *SU) {
+  if (SU->isScheduled)
+    return;
+
+  assert(SU->getInstr() && "Scheduled SUnit must have instr");
+
+  for (SUnit::succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
+       I != E; ++I) {
+    unsigned SuccReadyCycle = I->getSUnit()->BotReadyCycle;
+    unsigned MinLatency = I->getMinLatency();
+#ifndef NDEBUG
+    Bot.MaxMinLatency = std::max(MinLatency, Bot.MaxMinLatency);
+#endif
+    if (SU->BotReadyCycle < SuccReadyCycle + MinLatency)
+      SU->BotReadyCycle = SuccReadyCycle + MinLatency;
+  }
+  Bot.releaseNode(SU, SU->BotReadyCycle);
+}
+
+/// Does this SU have a hazard within the current instruction group.
+///
+/// The scheduler supports two modes of hazard recognition. The first is the
+/// ScheduleHazardRecognizer API. It is a fully general hazard recognizer that
+/// supports highly complicated in-order reservation tables
+/// (ScoreboardHazardRecognizer) and arbitrary target-specific logic.
+///
+/// The second is a streamlined mechanism that checks for hazards based on
+/// simple counters that the scheduler itself maintains. It explicitly checks
+/// for instruction dispatch limitations, including the number of micro-ops that
+/// can dispatch per cycle.
+///
+/// TODO: Also check whether the SU must start a new group.
+bool ConvergingVLIWScheduler::SchedBoundary::checkHazard(SUnit *SU) {
+  if (HazardRec->isEnabled())
+    return HazardRec->getHazardType(SU) != ScheduleHazardRecognizer::NoHazard;
+
+  unsigned uops = SchedModel->getNumMicroOps(SU->getInstr());
+  if (IssueCount + uops > SchedModel->getIssueWidth())
+    return true;
+
+  return false;
+}
+
+void ConvergingVLIWScheduler::SchedBoundary::releaseNode(SUnit *SU,
+                                                     unsigned ReadyCycle) {
+  if (ReadyCycle < MinReadyCycle)
+    MinReadyCycle = ReadyCycle;
+
+  // Check for interlocks first. For the purpose of other heuristics, an
+  // instruction that cannot issue appears as if it's not in the ReadyQueue.
+  if (ReadyCycle > CurrCycle || checkHazard(SU))
+
+    Pending.push(SU);
+  else
+    Available.push(SU);
+}
+
+/// Move the boundary of scheduled code by one cycle.
+void ConvergingVLIWScheduler::SchedBoundary::bumpCycle() {
+  unsigned Width = SchedModel->getIssueWidth();
+  IssueCount = (IssueCount <= Width) ? 0 : IssueCount - Width;
+
+  assert(MinReadyCycle < UINT_MAX && "MinReadyCycle uninitialized");
+  unsigned NextCycle = std::max(CurrCycle + 1, MinReadyCycle);
+
+  if (!HazardRec->isEnabled()) {
+    // Bypass HazardRec virtual calls.
+    CurrCycle = NextCycle;
+  } else {
+    // Bypass getHazardType calls in case of long latency.
+    for (; CurrCycle != NextCycle; ++CurrCycle) {
+      if (isTop())
+        HazardRec->AdvanceCycle();
+      else
+        HazardRec->RecedeCycle();
+    }
+  }
+  CheckPending = true;
+
+  DEBUG(dbgs() << "*** " << Available.getName() << " cycle "
+        << CurrCycle << '\n');
+}
+
+/// Move the boundary of scheduled code by one SUnit.
+void ConvergingVLIWScheduler::SchedBoundary::bumpNode(SUnit *SU) {
+  bool startNewCycle = false;
+
+  // Update the reservation table.
+  if (HazardRec->isEnabled()) {
+    if (!isTop() && SU->isCall) {
+      // Calls are scheduled with their preceding instructions. For bottom-up
+      // scheduling, clear the pipeline state before emitting.
+      HazardRec->Reset();
+    }
+    HazardRec->EmitInstruction(SU);
+  }
+
+  // Update DFA model.
+  startNewCycle = ResourceModel->reserveResources(SU);
+
+  // Check the instruction group dispatch limit.
+  // TODO: Check if this SU must end a dispatch group.
+  IssueCount += SchedModel->getNumMicroOps(SU->getInstr());
+  if (startNewCycle) {
+    DEBUG(dbgs() << "*** Max instrs at cycle " << CurrCycle << '\n');
+    bumpCycle();
+  }
+  else
+    DEBUG(dbgs() << "*** IssueCount " << IssueCount
+          << " at cycle " << CurrCycle << '\n');
+}
+
+/// Release pending ready nodes in to the available queue. This makes them
+/// visible to heuristics.
+void ConvergingVLIWScheduler::SchedBoundary::releasePending() {
+  // If the available queue is empty, it is safe to reset MinReadyCycle.
+  if (Available.empty())
+    MinReadyCycle = UINT_MAX;
+
+  // Check to see if any of the pending instructions are ready to issue.  If
+  // so, add them to the available queue.
+  for (unsigned i = 0, e = Pending.size(); i != e; ++i) {
+    SUnit *SU = *(Pending.begin()+i);
+    unsigned ReadyCycle = isTop() ? SU->TopReadyCycle : SU->BotReadyCycle;
+
+    if (ReadyCycle < MinReadyCycle)
+      MinReadyCycle = ReadyCycle;
+
+    if (ReadyCycle > CurrCycle)
+      continue;
+
+    if (checkHazard(SU))
+      continue;
+
+    Available.push(SU);
+    Pending.remove(Pending.begin()+i);
+    --i; --e;
+  }
+  CheckPending = false;
+}
+
+/// Remove SU from the ready set for this boundary.
+void ConvergingVLIWScheduler::SchedBoundary::removeReady(SUnit *SU) {
+  if (Available.isInQueue(SU))
+    Available.remove(Available.find(SU));
+  else {
+    assert(Pending.isInQueue(SU) && "bad ready count");
+    Pending.remove(Pending.find(SU));
+  }
+}
+
+/// If this queue only has one ready candidate, return it. As a side effect,
+/// advance the cycle until at least one node is ready. If multiple instructions
+/// are ready, return NULL.
+SUnit *ConvergingVLIWScheduler::SchedBoundary::pickOnlyChoice() {
+  if (CheckPending)
+    releasePending();
+
+  for (unsigned i = 0; Available.empty(); ++i) {
+    assert(i <= (HazardRec->getMaxLookAhead() + MaxMinLatency) &&
+           "permanent hazard"); (void)i;
+    ResourceModel->reserveResources(0);
+    bumpCycle();
+    releasePending();
+  }
+  if (Available.size() == 1)
+    return *Available.begin();
+  return NULL;
+}
+
+#ifndef NDEBUG
+void ConvergingVLIWScheduler::traceCandidate(const char *Label,
+                                             const ReadyQueue &Q,
+                                             SUnit *SU, PressureElement P) {
+  dbgs() << Label << " " << Q.getName() << " ";
+  if (P.isValid())
+    dbgs() << TRI->getRegPressureSetName(P.PSetID) << ":" << P.UnitIncrease
+           << " ";
+  else
+    dbgs() << "     ";
+  SU->dump(DAG);
+}
+#endif
+
+/// getSingleUnscheduledPred - If there is exactly one unscheduled predecessor
+/// of SU, return it, otherwise return null.
+static SUnit *getSingleUnscheduledPred(SUnit *SU) {
+  SUnit *OnlyAvailablePred = 0;
+  for (SUnit::const_pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
+       I != E; ++I) {
+    SUnit &Pred = *I->getSUnit();
+    if (!Pred.isScheduled) {
+      // We found an available, but not scheduled, predecessor.  If it's the
+      // only one we have found, keep track of it... otherwise give up.
+      if (OnlyAvailablePred && OnlyAvailablePred != &Pred)
+        return 0;
+      OnlyAvailablePred = &Pred;
+    }
+  }
+  return OnlyAvailablePred;
+}
+
+/// getSingleUnscheduledSucc - If there is exactly one unscheduled successor
+/// of SU, return it, otherwise return null.
+static SUnit *getSingleUnscheduledSucc(SUnit *SU) {
+  SUnit *OnlyAvailableSucc = 0;
+  for (SUnit::const_succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
+       I != E; ++I) {
+    SUnit &Succ = *I->getSUnit();
+    if (!Succ.isScheduled) {
+      // We found an available, but not scheduled, successor.  If it's the
+      // only one we have found, keep track of it... otherwise give up.
+      if (OnlyAvailableSucc && OnlyAvailableSucc != &Succ)
+        return 0;
+      OnlyAvailableSucc = &Succ;
+    }
+  }
+  return OnlyAvailableSucc;
+}
+
+// Constants used to denote relative importance of
+// heuristic components for cost computation.
+static const unsigned PriorityOne = 200;
+static const unsigned PriorityTwo = 100;
+static const unsigned PriorityThree = 50;
+static const unsigned PriorityFour = 20;
+static const unsigned ScaleTwo = 10;
+static const unsigned FactorOne = 2;
+
+/// Single point to compute overall scheduling cost.
+/// TODO: More heuristics will be used soon.
+int ConvergingVLIWScheduler::SchedulingCost(ReadyQueue &Q, SUnit *SU,
+                                            SchedCandidate &Candidate,
+                                            RegPressureDelta &Delta,
+                                            bool verbose) {
+  // Initial trivial priority.
+  int ResCount = 1;
+
+  // Do not waste time on a node that is already scheduled.
+  if (!SU || SU->isScheduled)
+    return ResCount;
+
+  // Forced priority is high.
+  if (SU->isScheduleHigh)
+    ResCount += PriorityOne;
+
+  // Critical path first.
+  if (Q.getID() == TopQID) {
+    ResCount += (SU->getHeight() * ScaleTwo);
+
+    // If resources are available for it, multiply the
+    // chance of scheduling.
+    if (Top.ResourceModel->isResourceAvailable(SU))
+      ResCount <<= FactorOne;
+  } else {
+    ResCount += (SU->getDepth() * ScaleTwo);
+
+    // If resources are available for it, multiply the
+    // chance of scheduling.
+    if (Bot.ResourceModel->isResourceAvailable(SU))
+      ResCount <<= FactorOne;
+  }
+
+  unsigned NumNodesBlocking = 0;
+  if (Q.getID() == TopQID) {
+    // How many SUs does it block from scheduling?
+    // Look at all of the successors of this node.
+    // Count the number of nodes that
+    // this node is the sole unscheduled node for.
+    for (SUnit::const_succ_iterator I = SU->Succs.begin(), E = SU->Succs.end();
+         I != E; ++I)
+      if (getSingleUnscheduledPred(I->getSUnit()) == SU)
+        ++NumNodesBlocking;
+  } else {
+    // How many unscheduled predecessors block this node?
+    for (SUnit::const_pred_iterator I = SU->Preds.begin(), E = SU->Preds.end();
+         I != E; ++I)
+      if (getSingleUnscheduledSucc(I->getSUnit()) == SU)
+        ++NumNodesBlocking;
+  }
+  ResCount += (NumNodesBlocking * ScaleTwo);
+
+  // Factor in reg pressure as a heuristic.
+  ResCount -= (Delta.Excess.UnitIncrease*PriorityThree);
+  ResCount -= (Delta.CriticalMax.UnitIncrease*PriorityThree);
+
+  DEBUG(if (verbose) dbgs() << " Total(" << ResCount << ")");
+
+  return ResCount;
+}
+
+/// Pick the best candidate from the top queue.
+///
+/// TODO: getMaxPressureDelta results can be mostly cached for each SUnit during
+/// DAG building. To adjust for the current scheduling location we need to
+/// maintain the number of vreg uses remaining to be top-scheduled.
+ConvergingVLIWScheduler::CandResult ConvergingVLIWScheduler::
+pickNodeFromQueue(ReadyQueue &Q, const RegPressureTracker &RPTracker,
+                  SchedCandidate &Candidate) {
+  DEBUG(Q.dump());
+
+  // getMaxPressureDelta temporarily modifies the tracker.
+  RegPressureTracker &TempTracker = const_cast<RegPressureTracker&>(RPTracker);
+
+  // BestSU remains NULL if no top candidates beat the best existing candidate.
+  CandResult FoundCandidate = NoCand;
+  for (ReadyQueue::iterator I = Q.begin(), E = Q.end(); I != E; ++I) {
+    RegPressureDelta RPDelta;
+    TempTracker.getMaxPressureDelta((*I)->getInstr(), RPDelta,
+                                    DAG->getRegionCriticalPSets(),
+                                    DAG->getRegPressure().MaxSetPressure);
+
+    int CurrentCost = SchedulingCost(Q, *I, Candidate, RPDelta, false);
+
+    // Initialize the candidate if needed.
+    if (!Candidate.SU) {
+      Candidate.SU = *I;
+      Candidate.RPDelta = RPDelta;
+      Candidate.SCost = CurrentCost;
+      FoundCandidate = NodeOrder;
+      continue;
+    }
+
+    // Best cost.
+    if (CurrentCost > Candidate.SCost) {
+      DEBUG(traceCandidate("CCAND", Q, *I));
+      Candidate.SU = *I;
+      Candidate.RPDelta = RPDelta;
+      Candidate.SCost = CurrentCost;
+      FoundCandidate = BestCost;
+      continue;
+    }
+
+    // Fall through to original instruction order.
+    // Only consider node order if Candidate was chosen from this Q.
+    if (FoundCandidate == NoCand)
+      continue;
+  }
+  return FoundCandidate;
+}
+
+/// Pick the best candidate node from either the top or bottom queue.
+SUnit *ConvergingVLIWScheduler::pickNodeBidrectional(bool &IsTopNode) {
+  // Schedule as far as possible in the direction of no choice. This is most
+  // efficient, but also provides the best heuristics for CriticalPSets.
+  if (SUnit *SU = Bot.pickOnlyChoice()) {
+    IsTopNode = false;
+    return SU;
+  }
+  if (SUnit *SU = Top.pickOnlyChoice()) {
+    IsTopNode = true;
+    return SU;
+  }
+  SchedCandidate BotCand;
+  // Prefer bottom scheduling when heuristics are silent.
+  CandResult BotResult = pickNodeFromQueue(Bot.Available,
+                                           DAG->getBotRPTracker(), BotCand);
+  assert(BotResult != NoCand && "failed to find the first candidate");
+
+  // If either Q has a single candidate that provides the least increase in
+  // Excess pressure, we can immediately schedule from that Q.
+  //
+  // RegionCriticalPSets summarizes the pressure within the scheduled region and
+  // affects picking from either Q. If scheduling in one direction must
+  // increase pressure for one of the excess PSets, then schedule in that
+  // direction first to provide more freedom in the other direction.
+  if (BotResult == SingleExcess || BotResult == SingleCritical) {
+    IsTopNode = false;
+    return BotCand.SU;
+  }
+  // Check if the top Q has a better candidate.
+  SchedCandidate TopCand;
+  CandResult TopResult = pickNodeFromQueue(Top.Available,
+                                           DAG->getTopRPTracker(), TopCand);
+  assert(TopResult != NoCand && "failed to find the first candidate");
+
+  if (TopResult == SingleExcess || TopResult == SingleCritical) {
+    IsTopNode = true;
+    return TopCand.SU;
+  }
+  // If either Q has a single candidate that minimizes pressure above the
+  // original region's pressure pick it.
+  if (BotResult == SingleMax) {
+    IsTopNode = false;
+    return BotCand.SU;
+  }
+  if (TopResult == SingleMax) {
+    IsTopNode = true;
+    return TopCand.SU;
+  }
+  if (TopCand.SCost > BotCand.SCost) {
+    IsTopNode = true;
+    return TopCand.SU;
+  }
+  // Otherwise prefer the bottom candidate in node order.
+  IsTopNode = false;
+  return BotCand.SU;
+}
+
+/// Pick the best node to balance the schedule. Implements MachineSchedStrategy.
+SUnit *ConvergingVLIWScheduler::pickNode(bool &IsTopNode) {
+  if (DAG->top() == DAG->bottom()) {
+    assert(Top.Available.empty() && Top.Pending.empty() &&
+           Bot.Available.empty() && Bot.Pending.empty() && "ReadyQ garbage");
+    return NULL;
+  }
+  SUnit *SU;
+  if (llvm::ForceTopDown) {
+    SU = Top.pickOnlyChoice();
+    if (!SU) {
+      SchedCandidate TopCand;
+      CandResult TopResult =
+        pickNodeFromQueue(Top.Available, DAG->getTopRPTracker(), TopCand);
+      assert(TopResult != NoCand && "failed to find the first candidate");
+      (void)TopResult;
+      SU = TopCand.SU;
+    }
+    IsTopNode = true;
+  } else if (llvm::ForceBottomUp) {
+    SU = Bot.pickOnlyChoice();
+    if (!SU) {
+      SchedCandidate BotCand;
+      CandResult BotResult =
+        pickNodeFromQueue(Bot.Available, DAG->getBotRPTracker(), BotCand);
+      assert(BotResult != NoCand && "failed to find the first candidate");
+      (void)BotResult;
+      SU = BotCand.SU;
+    }
+    IsTopNode = false;
+  } else {
+    SU = pickNodeBidrectional(IsTopNode);
+  }
+  if (SU->isTopReady())
+    Top.removeReady(SU);
+  if (SU->isBottomReady())
+    Bot.removeReady(SU);
+
+  DEBUG(dbgs() << "*** " << (IsTopNode ? "Top" : "Bottom")
+        << " Scheduling Instruction in cycle "
+        << (IsTopNode ? Top.CurrCycle : Bot.CurrCycle) << '\n';
+        SU->dump(DAG));
+  return SU;
+}
+
+/// Update the scheduler's state after scheduling a node. This is the same node
+/// that was just returned by pickNode(). However, VLIWMachineScheduler needs
+/// to update it's state based on the current cycle before MachineSchedStrategy
+/// does.
+void ConvergingVLIWScheduler::schedNode(SUnit *SU, bool IsTopNode) {
+  if (IsTopNode) {
+    SU->TopReadyCycle = Top.CurrCycle;
+    Top.bumpNode(SU);
+  } else {
+    SU->BotReadyCycle = Bot.CurrCycle;
+    Bot.bumpNode(SU);
+  }
+}
+
diff --git a/lib/Target/Hexagon/HexagonMachineScheduler.h b/lib/Target/Hexagon/HexagonMachineScheduler.h
new file mode 100644
index 0000000..fe0242a
--- /dev/null
+++ b/lib/Target/Hexagon/HexagonMachineScheduler.h
@@ -0,0 +1,244 @@
+//===-- HexagonMachineScheduler.h - Custom Hexagon MI scheduler.      ----===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Custom Hexagon MI scheduler.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef HEXAGONASMPRINTER_H
+#define HEXAGONASMPRINTER_H
+
+#include "llvm/CodeGen/LiveIntervalAnalysis.h"
+#include "llvm/CodeGen/MachineScheduler.h"
+#include "llvm/CodeGen/Passes.h"
+#include "llvm/CodeGen/RegisterClassInfo.h"
+#include "llvm/CodeGen/RegisterPressure.h"
+#include "llvm/CodeGen/ResourcePriorityQueue.h"
+#include "llvm/CodeGen/ScheduleDAGInstrs.h"
+#include "llvm/CodeGen/ScheduleHazardRecognizer.h"
+#include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/Target/TargetInstrInfo.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/ADT/OwningPtr.h"
+#include "llvm/ADT/PriorityQueue.h"
+
+using namespace llvm;
+
+namespace llvm {
+//===----------------------------------------------------------------------===//
+// ConvergingVLIWScheduler - Implementation of the standard
+// MachineSchedStrategy.
+//===----------------------------------------------------------------------===//
+
+class VLIWResourceModel {
+  /// ResourcesModel - Represents VLIW state.
+  /// Not limited to VLIW targets per say, but assumes
+  /// definition of DFA by a target.
+  DFAPacketizer *ResourcesModel;
+
+  const TargetSchedModel *SchedModel;
+
+  /// Local packet/bundle model. Purely
+  /// internal to the MI schedulre at the time.
+  std::vector<SUnit*> Packet;
+
+  /// Total packets created.
+  unsigned TotalPackets;
+
+public:
+VLIWResourceModel(const TargetMachine &TM, const TargetSchedModel *SM) :
+    SchedModel(SM), TotalPackets(0) {
+    ResourcesModel = TM.getInstrInfo()->CreateTargetScheduleState(&TM,NULL);
+
+    // This hard requirement could be relaxed,
+    // but for now do not let it proceed.
+    assert(ResourcesModel && "Unimplemented CreateTargetScheduleState.");
+
+    Packet.resize(SchedModel->getIssueWidth());
+    Packet.clear();
+    ResourcesModel->clearResources();
+  }
+
+  ~VLIWResourceModel() {
+    delete ResourcesModel;
+  }
+
+  void resetPacketState() {
+    Packet.clear();
+  }
+
+  void resetDFA() {
+    ResourcesModel->clearResources();
+  }
+
+  void reset() {
+    Packet.clear();
+    ResourcesModel->clearResources();
+  }
+
+  bool isResourceAvailable(SUnit *SU);
+  bool reserveResources(SUnit *SU);
+  unsigned getTotalPackets() const { return TotalPackets; }
+};
+
+/// Extend the standard ScheduleDAGMI to provide more context and override the
+/// top-level schedule() driver.
+class VLIWMachineScheduler : public ScheduleDAGMI {
+public:
+  VLIWMachineScheduler(MachineSchedContext *C, MachineSchedStrategy *S):
+    ScheduleDAGMI(C, S) {}
+
+  /// Schedule - This is called back from ScheduleDAGInstrs::Run() when it's
+  /// time to do some work.
+  virtual void schedule();
+  /// Perform platform specific DAG postprocessing.
+  void postprocessDAG();
+};
+
+/// ConvergingVLIWScheduler shrinks the unscheduled zone using heuristics
+/// to balance the schedule.
+class ConvergingVLIWScheduler : public MachineSchedStrategy {
+
+  /// Store the state used by ConvergingVLIWScheduler heuristics, required
+  ///  for the lifetime of one invocation of pickNode().
+  struct SchedCandidate {
+    // The best SUnit candidate.
+    SUnit *SU;
+
+    // Register pressure values for the best candidate.
+    RegPressureDelta RPDelta;
+
+    // Best scheduling cost.
+    int SCost;
+
+    SchedCandidate(): SU(NULL), SCost(0) {}
+  };
+  /// Represent the type of SchedCandidate found within a single queue.
+  enum CandResult {
+    NoCand, NodeOrder, SingleExcess, SingleCritical, SingleMax, MultiPressure,
+    BestCost};
+
+  /// Each Scheduling boundary is associated with ready queues. It tracks the
+  /// current cycle in whichever direction at has moved, and maintains the state
+  /// of "hazards" and other interlocks at the current cycle.
+  struct SchedBoundary {
+    VLIWMachineScheduler *DAG;
+    const TargetSchedModel *SchedModel;
+
+    ReadyQueue Available;
+    ReadyQueue Pending;
+    bool CheckPending;
+
+    ScheduleHazardRecognizer *HazardRec;
+    VLIWResourceModel *ResourceModel;
+
+    unsigned CurrCycle;
+    unsigned IssueCount;
+
+    /// MinReadyCycle - Cycle of the soonest available instruction.
+    unsigned MinReadyCycle;
+
+    // Remember the greatest min operand latency.
+    unsigned MaxMinLatency;
+
+    /// Pending queues extend the ready queues with the same ID and the
+    /// PendingFlag set.
+    SchedBoundary(unsigned ID, const Twine &Name):
+      DAG(0), SchedModel(0), Available(ID, Name+".A"),
+      Pending(ID << ConvergingVLIWScheduler::LogMaxQID, Name+".P"),
+      CheckPending(false), HazardRec(0), ResourceModel(0),
+      CurrCycle(0), IssueCount(0),
+      MinReadyCycle(UINT_MAX), MaxMinLatency(0) {}
+
+    ~SchedBoundary() {
+      delete ResourceModel;
+      delete HazardRec;
+    }
+
+    void init(VLIWMachineScheduler *dag, const TargetSchedModel *smodel) {
+      DAG = dag;
+      SchedModel = smodel;
+    }
+
+    bool isTop() const {
+      return Available.getID() == ConvergingVLIWScheduler::TopQID;
+    }
+
+    bool checkHazard(SUnit *SU);
+
+    void releaseNode(SUnit *SU, unsigned ReadyCycle);
+
+    void bumpCycle();
+
+    void bumpNode(SUnit *SU);
+
+    void releasePending();
+
+    void removeReady(SUnit *SU);
+
+    SUnit *pickOnlyChoice();
+  };
+
+  VLIWMachineScheduler *DAG;
+  const TargetSchedModel *SchedModel;
+  const TargetRegisterInfo *TRI;
+
+  // State of the top and bottom scheduled instruction boundaries.
+  SchedBoundary Top;
+  SchedBoundary Bot;
+
+public:
+  /// SUnit::NodeQueueId: 0 (none), 1 (top), 2 (bot), 3 (both)
+  enum {
+    TopQID = 1,
+    BotQID = 2,
+    LogMaxQID = 2
+  };
+
+  ConvergingVLIWScheduler():
+    DAG(0), SchedModel(0), TRI(0), Top(TopQID, "TopQ"), Bot(BotQID, "BotQ") {}
+
+  virtual void initialize(ScheduleDAGMI *dag);
+
+  virtual SUnit *pickNode(bool &IsTopNode);
+
+  virtual void schedNode(SUnit *SU, bool IsTopNode);
+
+  virtual void releaseTopNode(SUnit *SU);
+
+  virtual void releaseBottomNode(SUnit *SU);
+
+  unsigned ReportPackets() {
+    return Top.ResourceModel->getTotalPackets() +
+           Bot.ResourceModel->getTotalPackets();
+  }
+
+protected:
+  SUnit *pickNodeBidrectional(bool &IsTopNode);
+
+  int SchedulingCost(ReadyQueue &Q,
+                     SUnit *SU, SchedCandidate &Candidate,
+                     RegPressureDelta &Delta, bool verbose);
+
+  CandResult pickNodeFromQueue(ReadyQueue &Q,
+                               const RegPressureTracker &RPTracker,
+                               SchedCandidate &Candidate);
+#ifndef NDEBUG
+  void traceCandidate(const char *Label, const ReadyQueue &Q, SUnit *SU,
+                      PressureElement P = PressureElement());
+#endif
+};
+
+} // namespace
+
+
+#endif
diff --git a/lib/Target/Hexagon/HexagonNewValueJump.cpp b/lib/Target/Hexagon/HexagonNewValueJump.cpp
index 7ece408..1e91c39 100644
--- a/lib/Target/Hexagon/HexagonNewValueJump.cpp
+++ b/lib/Target/Hexagon/HexagonNewValueJump.cpp
@@ -337,7 +337,7 @@ bool HexagonNewValueJump::runOnMachineFunction(MachineFunction &MF) {
 
   DEBUG(dbgs() << "********** Hexagon New Value Jump **********\n"
                << "********** Function: "
-               << MF.getFunction()->getName() << "\n");
+               << MF.getName() << "\n");
 
 #if 0
   // for now disable this, if we move NewValueJump before register
diff --git a/lib/Target/Hexagon/HexagonPeephole.cpp b/lib/Target/Hexagon/HexagonPeephole.cpp
index 55cbc09..a295015 100644
--- a/lib/Target/Hexagon/HexagonPeephole.cpp
+++ b/lib/Target/Hexagon/HexagonPeephole.cpp
@@ -109,6 +109,7 @@ bool HexagonPeephole::runOnMachineFunction(MachineFunction &MF) {
   MRI = &MF.getRegInfo();
 
   DenseMap<unsigned, unsigned> PeepholeMap;
+  DenseMap<unsigned, std::pair<unsigned, unsigned> > PeepholeDoubleRegsMap;
 
   if (DisableHexagonPeephole) return false;
 
@@ -117,6 +118,7 @@ bool HexagonPeephole::runOnMachineFunction(MachineFunction &MF) {
        MBBb != MBBe; ++MBBb) {
     MachineBasicBlock* MBB = MBBb;
     PeepholeMap.clear();
+    PeepholeDoubleRegsMap.clear();
 
     // Traverse the basic block.
     for (MachineBasicBlock::iterator MII = MBB->begin(); MII != MBB->end();
@@ -140,6 +142,24 @@ bool HexagonPeephole::runOnMachineFunction(MachineFunction &MF) {
         }
       }
 
+      // Look for this sequence below
+      // %vregDoubleReg1 = LSRd_ri %vregDoubleReg0, 32
+      // %vregIntReg = COPY %vregDoubleReg1:subreg_loreg.
+      // and convert into
+      // %vregIntReg = COPY %vregDoubleReg0:subreg_hireg.
+      if (MI->getOpcode() == Hexagon::LSRd_ri) {
+        assert(MI->getNumOperands() == 3);
+        MachineOperand &Dst = MI->getOperand(0);
+        MachineOperand &Src1 = MI->getOperand(1);
+        MachineOperand &Src2 = MI->getOperand(2);
+        if (Src2.getImm() != 32)
+          continue;
+        unsigned DstReg = Dst.getReg();
+        unsigned SrcReg = Src1.getReg();
+        PeepholeDoubleRegsMap[DstReg] =
+          std::make_pair(*&SrcReg, 1/*Hexagon::subreg_hireg*/);
+      }
+
       // Look for P=NOT(P).
       if (!DisablePNotP &&
           (MI->getOpcode() == Hexagon::NOT_p)) {
@@ -178,6 +198,21 @@ bool HexagonPeephole::runOnMachineFunction(MachineFunction &MF) {
             // Change the 1st operand.
             MI->RemoveOperand(1);
             MI->addOperand(MachineOperand::CreateReg(PeepholeSrc, false));
+          } else  {
+            DenseMap<unsigned, std::pair<unsigned, unsigned> >::iterator DI =
+              PeepholeDoubleRegsMap.find(SrcReg);
+            if (DI != PeepholeDoubleRegsMap.end()) {
+              std::pair<unsigned,unsigned> PeepholeSrc = DI->second;
+              MI->RemoveOperand(1);
+              MI->addOperand(MachineOperand::CreateReg(PeepholeSrc.first,
+                                                       false /*isDef*/,
+                                                       false /*isImp*/,
+                                                       false /*isKill*/,
+                                                       false /*isDead*/,
+                                                       false /*isUndef*/,
+                                                       false /*isEarlyClobber*/,
+                                                       PeepholeSrc.second));
+            }
           }
         }
       }
diff --git a/lib/Target/Hexagon/HexagonRegisterInfo.cpp b/lib/Target/Hexagon/HexagonRegisterInfo.cpp
index 2c23674..3742486 100644
--- a/lib/Target/Hexagon/HexagonRegisterInfo.cpp
+++ b/lib/Target/Hexagon/HexagonRegisterInfo.cpp
@@ -310,6 +310,58 @@ void HexagonRegisterInfo::getInitialFrameState(std::vector<MachineMove>
   Moves.push_back(MachineMove(0, Dst, Src));
 }
 
+// Get the weight in units of pressure for this register class.
+const RegClassWeight &
+HexagonRegisterInfo::getRegClassWeight(const TargetRegisterClass *RC) const {
+  // Each TargetRegisterClass has a per register weight, and weight
+  // limit which must be less than the limits of its pressure sets.
+  static const RegClassWeight RCWeightTable[] = {
+    {1, 32}, // IntRegs
+    {1, 8},  // CRRegs
+    {1, 4},  // PredRegs
+    {2, 16}, // DoubleRegs
+    {0, 0} };
+  return RCWeightTable[RC->getID()];
+}
+
+/// Get the number of dimensions of register pressure.
+unsigned HexagonRegisterInfo::getNumRegPressureSets() const {
+  return 4;
+}
+
+/// Get the name of this register unit pressure set.
+const char *HexagonRegisterInfo::getRegPressureSetName(unsigned Idx) const {
+  static const char *const RegPressureSetName[] = {
+    "IntRegsRegSet",
+    "CRRegsRegSet",
+    "PredRegsRegSet",
+    "DoubleRegsRegSet"
+  };
+  assert((Idx < 4) && "Index out of bounds");
+  return RegPressureSetName[Idx];
+}
+
+/// Get the register unit pressure limit for this dimension.
+/// This limit must be adjusted dynamically for reserved registers.
+unsigned HexagonRegisterInfo::getRegPressureSetLimit(unsigned Idx) const {
+  static const int RegPressureLimit [] = { 16, 4, 2, 8 };
+  assert((Idx < 4) && "Index out of bounds");
+  return RegPressureLimit[Idx];
+}
+
+const int*
+HexagonRegisterInfo::getRegClassPressureSets(const TargetRegisterClass *RC)
+  const {
+  static const int RCSetsTable[] = {
+    0,  -1,  // IntRegs
+    1,  -1,  // CRRegs
+    2,  -1,  // PredRegs
+    0,  -1,  // DoubleRegs
+    -1 };
+  static const unsigned RCSetStartTable[] = { 0, 2, 4, 6, 0 };
+  unsigned SetListStart = RCSetStartTable[RC->getID()];
+  return &RCSetsTable[SetListStart];
+}
 unsigned HexagonRegisterInfo::getEHExceptionRegister() const {
   llvm_unreachable("What is the exception register");
 }
diff --git a/lib/Target/Hexagon/HexagonRegisterInfo.h b/lib/Target/Hexagon/HexagonRegisterInfo.h
index 85355ae..8820d13 100644
--- a/lib/Target/Hexagon/HexagonRegisterInfo.h
+++ b/lib/Target/Hexagon/HexagonRegisterInfo.h
@@ -87,6 +87,11 @@ struct HexagonRegisterInfo : public HexagonGenRegisterInfo {
   // Exception handling queries.
   unsigned getEHExceptionRegister() const;
   unsigned getEHHandlerRegister() const;
+  const RegClassWeight &getRegClassWeight(const TargetRegisterClass *RC) const;
+  unsigned getNumRegPressureSets() const;
+  const char *getRegPressureSetName(unsigned Idx) const;
+  unsigned getRegPressureSetLimit(unsigned Idx) const;
+  const int* getRegClassPressureSets(const TargetRegisterClass *RC) const;
 };
 
 } // end namespace llvm
diff --git a/lib/Target/Hexagon/HexagonRemoveSZExtArgs.cpp b/lib/Target/Hexagon/HexagonRemoveSZExtArgs.cpp
index 2468f0b..4d93dd1 100644
--- a/lib/Target/Hexagon/HexagonRemoveSZExtArgs.cpp
+++ b/lib/Target/Hexagon/HexagonRemoveSZExtArgs.cpp
@@ -50,7 +50,7 @@ bool HexagonRemoveExtendArgs::runOnFunction(Function &F) {
   unsigned Idx = 1;
   for (Function::arg_iterator AI = F.arg_begin(), AE = F.arg_end(); AI != AE;
        ++AI, ++Idx) {
-    if (F.paramHasAttr(Idx, Attribute::SExt)) {
+    if (F.getParamAttributes(Idx).hasAttribute(Attributes::SExt)) {
       Argument* Arg = AI;
       if (!isa<PointerType>(Arg->getType())) {
         for (Instruction::use_iterator UI = Arg->use_begin();
diff --git a/lib/Target/Hexagon/HexagonSchedule.td b/lib/Target/Hexagon/HexagonSchedule.td
index d1076b8..b5ff69a 100644
--- a/lib/Target/Hexagon/HexagonSchedule.td
+++ b/lib/Target/Hexagon/HexagonSchedule.td
@@ -47,6 +47,7 @@ def HexagonModel : SchedMachineModel {
   // Max issue per cycle == bundle width.
   let IssueWidth = 4;
   let Itineraries = HexagonItineraries;
+  let LoadLatency = 1;
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/lib/Target/Hexagon/HexagonScheduleV4.td b/lib/Target/Hexagon/HexagonScheduleV4.td
index 9b41126..5668ae8 100644
--- a/lib/Target/Hexagon/HexagonScheduleV4.td
+++ b/lib/Target/Hexagon/HexagonScheduleV4.td
@@ -58,6 +58,7 @@ def HexagonModelV4 : SchedMachineModel {
   // Max issue per cycle == bundle width.
   let IssueWidth = 4;
   let Itineraries = HexagonItinerariesV4;
+  let LoadLatency = 1;
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/lib/Target/Hexagon/HexagonSubtarget.cpp b/lib/Target/Hexagon/HexagonSubtarget.cpp
index 5d087db..4bacb8f 100644
--- a/lib/Target/Hexagon/HexagonSubtarget.cpp
+++ b/lib/Target/Hexagon/HexagonSubtarget.cpp
@@ -40,28 +40,27 @@ EnableIEEERndNear(
 
 HexagonSubtarget::HexagonSubtarget(StringRef TT, StringRef CPU, StringRef FS):
   HexagonGenSubtargetInfo(TT, CPU, FS),
-  HexagonArchVersion(V2),
   CPUString(CPU.str()) {
-  ParseSubtargetFeatures(CPU, FS);
 
-  switch(HexagonArchVersion) {
-  case HexagonSubtarget::V2:
-    break;
-  case HexagonSubtarget::V3:
-    EnableV3 = true;
-    break;
-  case HexagonSubtarget::V4:
-    break;
-  case HexagonSubtarget::V5:
-    break;
-  default:
-    // If the programmer has not specified a Hexagon version, default
-    // to -mv4.
+  // If the programmer has not specified a Hexagon version, default to -mv4.
+  if (CPUString.empty())
     CPUString = "hexagonv4";
-    HexagonArchVersion = HexagonSubtarget::V4;
-    break;
+
+  if (CPUString == "hexagonv2") {
+    HexagonArchVersion = V2;
+  } else if (CPUString == "hexagonv3") {
+    EnableV3 = true;
+    HexagonArchVersion = V3;
+  } else if (CPUString == "hexagonv4") {
+    HexagonArchVersion = V4;
+  } else if (CPUString == "hexagonv5") {
+    HexagonArchVersion = V5;
+  } else {
+    llvm_unreachable("Unrecognized Hexagon processor version");
   }
 
+  ParseSubtargetFeatures(CPUString, FS);
+
   // Initialize scheduling itinerary for the specified CPU.
   InstrItins = getInstrItineraryForCPU(CPUString);
 
diff --git a/lib/Target/Hexagon/HexagonTargetMachine.cpp b/lib/Target/Hexagon/HexagonTargetMachine.cpp
index a7b291f..30866e9 100644
--- a/lib/Target/Hexagon/HexagonTargetMachine.cpp
+++ b/lib/Target/Hexagon/HexagonTargetMachine.cpp
@@ -14,6 +14,7 @@
 #include "HexagonTargetMachine.h"
 #include "Hexagon.h"
 #include "HexagonISelLowering.h"
+#include "HexagonMachineScheduler.h"
 #include "llvm/Module.h"
 #include "llvm/CodeGen/Passes.h"
 #include "llvm/PassManager.h"
@@ -29,6 +30,11 @@ opt<bool> DisableHardwareLoops(
                         "disable-hexagon-hwloops", cl::Hidden,
                         cl::desc("Disable Hardware Loops for Hexagon target"));
 
+static cl::
+opt<bool> DisableHexagonMISched("disable-hexagon-misched",
+                                cl::Hidden, cl::ZeroOrMore, cl::init(false),
+                                cl::desc("Disable Hexagon MI Scheduling"));
+
 /// HexagonTargetMachineModule - Note that this is used on hosts that
 /// cannot link in a library unless there are references into the
 /// library.  In particular, it seems that it is not possible to get
@@ -42,6 +48,13 @@ extern "C" void LLVMInitializeHexagonTarget() {
   RegisterTargetMachine<HexagonTargetMachine> X(TheHexagonTarget);
 }
 
+static ScheduleDAGInstrs *createVLIWMachineSched(MachineSchedContext *C) {
+  return new VLIWMachineScheduler(C, new ConvergingVLIWScheduler());
+}
+
+static MachineSchedRegistry
+SchedCustomRegistry("hexagon", "Run Hexagon's custom scheduler",
+                    createVLIWMachineSched);
 
 /// HexagonTargetMachine ctor - Create an ILP32 architecture model.
 ///
@@ -55,13 +68,14 @@ HexagonTargetMachine::HexagonTargetMachine(const Target &T, StringRef TT,
                                            CodeModel::Model CM,
                                            CodeGenOpt::Level OL)
   : LLVMTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL),
-    DataLayout("e-p:32:32:32-"
+    DL("e-p:32:32:32-"
                 "i64:64:64-i32:32:32-i16:16:16-i1:32:32-"
                 "f64:64:64-f32:32:32-a0:0-n32") ,
     Subtarget(TT, CPU, FS), InstrInfo(Subtarget), TLInfo(*this),
     TSInfo(*this),
     FrameLowering(Subtarget),
-    InstrItins(&Subtarget.getInstrItineraryData()) {
+    InstrItins(&Subtarget.getInstrItineraryData()),
+    STTI(&TLInfo), VTTI(&TLInfo) {
   setMCUseCFI(false);
 }
 
@@ -74,7 +88,7 @@ bool HexagonTargetMachine::addPassesForOptimizations(PassManagerBase &PM) {
   PM.add(createDeadCodeEliminationPass());
   PM.add(createConstantPropagationPass());
   PM.add(createLoopUnrollPass());
-  PM.add(createLoopStrengthReducePass(getTargetLowering()));
+  PM.add(createLoopStrengthReducePass());
   return true;
 }
 
@@ -83,7 +97,13 @@ namespace {
 class HexagonPassConfig : public TargetPassConfig {
 public:
   HexagonPassConfig(HexagonTargetMachine *TM, PassManagerBase &PM)
-    : TargetPassConfig(TM, PM) {}
+    : TargetPassConfig(TM, PM) {
+    // Enable MI scheduler.
+    if (!DisableHexagonMISched) {
+      enablePass(&MachineSchedulerID);
+      MachineSchedRegistry::setDefault(createVLIWMachineSched);
+    }
+  }
 
   HexagonTargetMachine &getHexagonTargetMachine() const {
     return getTM<HexagonTargetMachine>();
diff --git a/lib/Target/Hexagon/HexagonTargetMachine.h b/lib/Target/Hexagon/HexagonTargetMachine.h
index 0336965..7a4215c 100644
--- a/lib/Target/Hexagon/HexagonTargetMachine.h
+++ b/lib/Target/Hexagon/HexagonTargetMachine.h
@@ -20,20 +20,23 @@
 #include "HexagonSelectionDAGInfo.h"
 #include "HexagonFrameLowering.h"
 #include "llvm/Target/TargetMachine.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
+#include "llvm/Target/TargetTransformImpl.h"
 
 namespace llvm {
 
 class Module;
 
 class HexagonTargetMachine : public LLVMTargetMachine {
-  const TargetData DataLayout;       // Calculates type size & alignment.
+  const DataLayout DL;       // Calculates type size & alignment.
   HexagonSubtarget Subtarget;
   HexagonInstrInfo InstrInfo;
   HexagonTargetLowering TLInfo;
   HexagonSelectionDAGInfo TSInfo;
   HexagonFrameLowering FrameLowering;
   const InstrItineraryData* InstrItins;
+  ScalarTargetTransformImpl STTI;
+  VectorTargetTransformImpl VTTI;
 
 public:
   HexagonTargetMachine(const Target &T, StringRef TT,StringRef CPU,
@@ -68,7 +71,15 @@ public:
     return &TSInfo;
   }
 
-  virtual const TargetData       *getTargetData() const { return &DataLayout; }
+  virtual const ScalarTargetTransformInfo *getScalarTargetTransformInfo()const {
+    return &STTI;
+  }
+
+  virtual const VectorTargetTransformInfo *getVectorTargetTransformInfo()const {
+    return &VTTI;
+  }
+
+  virtual const DataLayout       *getDataLayout() const { return &DL; }
   static unsigned getModuleMatchQuality(const Module &M);
 
   // Pass Pipeline Configuration.
diff --git a/lib/Target/Hexagon/HexagonTargetObjectFile.cpp b/lib/Target/Hexagon/HexagonTargetObjectFile.cpp
index 32cc709..f4d7761 100644
--- a/lib/Target/Hexagon/HexagonTargetObjectFile.cpp
+++ b/lib/Target/Hexagon/HexagonTargetObjectFile.cpp
@@ -16,7 +16,7 @@
 #include "HexagonTargetMachine.h"
 #include "llvm/Function.h"
 #include "llvm/GlobalVariable.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/DerivedTypes.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/Support/ELF.h"
@@ -73,7 +73,7 @@ IsGlobalInSmallSection(const GlobalValue *GV, const TargetMachine &TM,
 
   if (Kind.isBSS() || Kind.isDataNoRel() || Kind.isCommon()) {
     Type *Ty = GV->getType()->getElementType();
-    return IsInSmallSection(TM.getTargetData()->getTypeAllocSize(Ty));
+    return IsInSmallSection(TM.getDataLayout()->getTypeAllocSize(Ty));
   }
 
   return false;
diff --git a/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp b/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp
index a03ed03..3d5f685 100644
--- a/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp
+++ b/lib/Target/Hexagon/HexagonVLIWPacketizer.cpp
@@ -3474,8 +3474,8 @@ bool HexagonPacketizerList::isLegalToPacketizeTogether(SUnit *SUI, SUnit *SUJ) {
       // 1. Two loads unless they are volatile.
       // 2. Two stores in V4 unless they are volatile.
       else if ((DepType == SDep::Order) &&
-               !I->hasVolatileMemoryRef() &&
-               !J->hasVolatileMemoryRef()) {
+               !I->hasOrderedMemoryRef() &&
+               !J->hasOrderedMemoryRef()) {
         if (QRI->Subtarget.hasV4TOps() &&
             // hexagonv4 allows dual store.
             MCIDI.mayStore() && MCIDJ.mayStore()) {
diff --git a/lib/Target/Hexagon/HexagonVarargsCallingConvention.h b/lib/Target/Hexagon/HexagonVarargsCallingConvention.h
index 9305c27..c607b5d 100644
--- a/lib/Target/Hexagon/HexagonVarargsCallingConvention.h
+++ b/lib/Target/Hexagon/HexagonVarargsCallingConvention.h
@@ -75,9 +75,9 @@ static bool CC_Hexagon32_VarArgs(unsigned ValNo, EVT ValVT,
 
   const Type* ArgTy = LocVT.getTypeForEVT(State.getContext());
   unsigned Alignment =
-    State.getTarget().getTargetData()->getABITypeAlignment(ArgTy);
+    State.getTarget().getDataLayout()->getABITypeAlignment(ArgTy);
   unsigned Size =
-    State.getTarget().getTargetData()->getTypeSizeInBits(ArgTy) / 8;
+    State.getTarget().getDataLayout()->getTypeSizeInBits(ArgTy) / 8;
 
   // If it's passed by value, then we need the size of the aggregate not of
   // the pointer.
@@ -130,9 +130,9 @@ static bool RetCC_Hexagon32_VarArgs(unsigned ValNo, EVT ValVT,
 
   const Type* ArgTy = LocVT.getTypeForEVT(State.getContext());
   unsigned Alignment =
-    State.getTarget().getTargetData()->getABITypeAlignment(ArgTy);
+    State.getTarget().getDataLayout()->getABITypeAlignment(ArgTy);
   unsigned Size =
-    State.getTarget().getTargetData()->getTypeSizeInBits(ArgTy) / 8;
+    State.getTarget().getDataLayout()->getTypeSizeInBits(ArgTy) / 8;
 
   unsigned Offset3 = State.AllocateStack(Size, Alignment);
   State.addLoc(CCValAssign::getMem(ValNo, ValVT.getSimpleVT(), Offset3,
diff --git a/lib/Target/Hexagon/MCTargetDesc/HexagonMCAsmInfo.cpp b/lib/Target/Hexagon/MCTargetDesc/HexagonMCAsmInfo.cpp
index d6e6c36..86f75d1 100644
--- a/lib/Target/Hexagon/MCTargetDesc/HexagonMCAsmInfo.cpp
+++ b/lib/Target/Hexagon/MCTargetDesc/HexagonMCAsmInfo.cpp
@@ -24,7 +24,7 @@ HexagonMCAsmInfo::HexagonMCAsmInfo(const Target &T, StringRef TT) {
   HasLEB128 = true;
 
   PrivateGlobalPrefix = ".L";
-  LCOMMDirectiveType = LCOMM::ByteAlignment;
+  LCOMMDirectiveAlignmentType = LCOMM::ByteAlignment;
   InlineAsmStart = "# InlineAsm Start";
   InlineAsmEnd = "# InlineAsm End";
   ZeroDirective = "\t.space\t";
diff --git a/lib/Target/MBlaze/AsmParser/MBlazeAsmParser.cpp b/lib/Target/MBlaze/AsmParser/MBlazeAsmParser.cpp
index 38fb0e8..f7809ca 100644
--- a/lib/Target/MBlaze/AsmParser/MBlazeAsmParser.cpp
+++ b/lib/Target/MBlaze/AsmParser/MBlazeAsmParser.cpp
@@ -44,9 +44,10 @@ class MBlazeAsmParser : public MCTargetAsmParser {
 
   bool ParseDirectiveWord(unsigned Size, SMLoc L);
 
-  bool MatchAndEmitInstruction(SMLoc IDLoc,
+  bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
                                SmallVectorImpl<MCParsedAsmOperand*> &Operands,
-                               MCStreamer &Out);
+                               MCStreamer &Out, unsigned &ErrorInfo,
+                               bool MatchingInlineAsm);
 
   /// @name Auto-generated Match Functions
   /// {
@@ -56,12 +57,12 @@ class MBlazeAsmParser : public MCTargetAsmParser {
 
   /// }
 
-
 public:
   MBlazeAsmParser(MCSubtargetInfo &_STI, MCAsmParser &_Parser)
     : MCTargetAsmParser(), Parser(_Parser) {}
 
-  virtual bool ParseInstruction(StringRef Name, SMLoc NameLoc,
+  virtual bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
+                                SMLoc NameLoc,
                                 SmallVectorImpl<MCParsedAsmOperand*> &Operands);
 
   virtual bool ParseDirective(AsmToken DirectiveID);
@@ -313,14 +314,13 @@ static unsigned MatchRegisterName(StringRef Name);
 /// }
 //
 bool MBlazeAsmParser::
-MatchAndEmitInstruction(SMLoc IDLoc,
+MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
                         SmallVectorImpl<MCParsedAsmOperand*> &Operands,
-                        MCStreamer &Out) {
+                        MCStreamer &Out, unsigned &ErrorInfo,
+                        bool MatchingInlineAsm) {
   MCInst Inst;
-  SMLoc ErrorLoc;
-  unsigned ErrorInfo;
-
-  switch (MatchInstructionImpl(Operands, Inst, ErrorInfo)) {
+  switch (MatchInstructionImpl(Operands, Inst, ErrorInfo,
+                               MatchingInlineAsm)) {
   default: break;
   case Match_Success:
     Out.EmitInstruction(Inst);
@@ -329,10 +329,8 @@ MatchAndEmitInstruction(SMLoc IDLoc,
     return Error(IDLoc, "instruction use requires an option to be enabled");
   case Match_MnemonicFail:
       return Error(IDLoc, "unrecognized instruction mnemonic");
-  case Match_ConversionFail:
-    return Error(IDLoc, "unable to convert operands to instruction");
-  case Match_InvalidOperand:
-    ErrorLoc = IDLoc;
+  case Match_InvalidOperand: {
+    SMLoc ErrorLoc = IDLoc;
     if (ErrorInfo != ~0U) {
       if (ErrorInfo >= Operands.size())
         return Error(IDLoc, "too few operands for instruction");
@@ -343,6 +341,7 @@ MatchAndEmitInstruction(SMLoc IDLoc,
 
     return Error(ErrorLoc, "invalid operand for instruction");
   }
+  }
 
   llvm_unreachable("Implement any new match types added!");
 }
@@ -479,7 +478,7 @@ ParseOperand(SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
 
 /// Parse an mblaze instruction mnemonic followed by its operands.
 bool MBlazeAsmParser::
-ParseInstruction(StringRef Name, SMLoc NameLoc,
+ParseInstruction(ParseInstructionInfo &Info, StringRef Name, SMLoc NameLoc,
                  SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
   // The first operands is the token for the instruction name
   size_t dotLoc = Name.find('.');
diff --git a/lib/Target/MBlaze/CMakeLists.txt b/lib/Target/MBlaze/CMakeLists.txt
index 6c3e8b6..0bf93d7 100644
--- a/lib/Target/MBlaze/CMakeLists.txt
+++ b/lib/Target/MBlaze/CMakeLists.txt
@@ -27,7 +27,6 @@ add_llvm_target(MBlazeCodeGen
   MBlazeSelectionDAGInfo.cpp
   MBlazeAsmPrinter.cpp
   MBlazeMCInstLower.cpp
-  MBlazeELFWriterInfo.cpp
   )
 
 add_dependencies(LLVMMBlazeCodeGen intrinsics_gen)
diff --git a/lib/Target/MBlaze/MBlazeAsmPrinter.cpp b/lib/Target/MBlaze/MBlazeAsmPrinter.cpp
index e9f340f..b679a31 100644
--- a/lib/Target/MBlaze/MBlazeAsmPrinter.cpp
+++ b/lib/Target/MBlaze/MBlazeAsmPrinter.cpp
@@ -34,7 +34,7 @@
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/Target/Mangler.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetLoweringObjectFile.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
diff --git a/lib/Target/MBlaze/MBlazeELFWriterInfo.cpp b/lib/Target/MBlaze/MBlazeELFWriterInfo.cpp
deleted file mode 100644
index e3c7236..0000000
--- a/lib/Target/MBlaze/MBlazeELFWriterInfo.cpp
+++ /dev/null
@@ -1,107 +0,0 @@
-//===-- MBlazeELFWriterInfo.cpp - ELF Writer Info for the MBlaze backend --===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements ELF writer information for the MBlaze backend.
-//
-//===----------------------------------------------------------------------===//
-
-#include "MBlazeELFWriterInfo.h"
-#include "MBlazeRelocations.h"
-#include "llvm/Function.h"
-#include "llvm/Support/ELF.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Target/TargetData.h"
-#include "llvm/Target/TargetMachine.h"
-
-using namespace llvm;
-
-//===----------------------------------------------------------------------===//
-//  Implementation of the MBlazeELFWriterInfo class
-//===----------------------------------------------------------------------===//
-
-MBlazeELFWriterInfo::MBlazeELFWriterInfo(TargetMachine &TM)
-  : TargetELFWriterInfo(TM.getTargetData()->getPointerSizeInBits() == 64,
-                        TM.getTargetData()->isLittleEndian()) {
-}
-
-MBlazeELFWriterInfo::~MBlazeELFWriterInfo() {}
-
-unsigned MBlazeELFWriterInfo::getRelocationType(unsigned MachineRelTy) const {
-  switch (MachineRelTy) {
-  case MBlaze::reloc_pcrel_word:
-    return ELF::R_MICROBLAZE_64_PCREL;
-  case MBlaze::reloc_absolute_word:
-    return ELF::R_MICROBLAZE_NONE;
-  default:
-    llvm_unreachable("unknown mblaze machine relocation type");
-  }
-}
-
-long int MBlazeELFWriterInfo::getDefaultAddendForRelTy(unsigned RelTy,
-                                                    long int Modifier) const {
-  switch (RelTy) {
-  case ELF::R_MICROBLAZE_32_PCREL:
-    return Modifier - 4;
-  case ELF::R_MICROBLAZE_32:
-    return Modifier;
-  default:
-    llvm_unreachable("unknown mblaze relocation type");
-  }
-}
-
-unsigned MBlazeELFWriterInfo::getRelocationTySize(unsigned RelTy) const {
-  // FIXME: Most of these sizes are guesses based on the name
-  switch (RelTy) {
-  case ELF::R_MICROBLAZE_32:
-  case ELF::R_MICROBLAZE_32_PCREL:
-  case ELF::R_MICROBLAZE_32_PCREL_LO:
-  case ELF::R_MICROBLAZE_32_LO:
-  case ELF::R_MICROBLAZE_SRO32:
-  case ELF::R_MICROBLAZE_SRW32:
-  case ELF::R_MICROBLAZE_32_SYM_OP_SYM:
-  case ELF::R_MICROBLAZE_GOTOFF_32:
-    return 32;
-
-  case ELF::R_MICROBLAZE_64_PCREL:
-  case ELF::R_MICROBLAZE_64:
-  case ELF::R_MICROBLAZE_GOTPC_64:
-  case ELF::R_MICROBLAZE_GOT_64:
-  case ELF::R_MICROBLAZE_PLT_64:
-  case ELF::R_MICROBLAZE_GOTOFF_64:
-    return 64;
-  }
-
-  return 0;
-}
-
-bool MBlazeELFWriterInfo::isPCRelativeRel(unsigned RelTy) const {
-  // FIXME: Most of these are guesses based on the name
-  switch (RelTy) {
-  case ELF::R_MICROBLAZE_32_PCREL:
-  case ELF::R_MICROBLAZE_64_PCREL:
-  case ELF::R_MICROBLAZE_32_PCREL_LO:
-  case ELF::R_MICROBLAZE_GOTPC_64:
-    return true;
-  }
-
-  return false;
-}
-
-unsigned MBlazeELFWriterInfo::getAbsoluteLabelMachineRelTy() const {
-  return MBlaze::reloc_absolute_word;
-}
-
-long int MBlazeELFWriterInfo::computeRelocation(unsigned SymOffset,
-                                                unsigned RelOffset,
-                                                unsigned RelTy) const {
-  assert((RelTy == ELF::R_MICROBLAZE_32_PCREL ||
-          RelTy == ELF::R_MICROBLAZE_64_PCREL) &&
-         "computeRelocation unknown for this relocation type");
-  return SymOffset - (RelOffset + 4);
-}
diff --git a/lib/Target/MBlaze/MBlazeELFWriterInfo.h b/lib/Target/MBlaze/MBlazeELFWriterInfo.h
deleted file mode 100644
index a314eb7..0000000
--- a/lib/Target/MBlaze/MBlazeELFWriterInfo.h
+++ /dev/null
@@ -1,59 +0,0 @@
-//===-- MBlazeELFWriterInfo.h - ELF Writer Info for MBlaze ------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements ELF writer information for the MBlaze backend.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef MBLAZE_ELF_WRITER_INFO_H
-#define MBLAZE_ELF_WRITER_INFO_H
-
-#include "llvm/Target/TargetELFWriterInfo.h"
-
-namespace llvm {
-  class TargetMachine;
-
-  class MBlazeELFWriterInfo : public TargetELFWriterInfo {
-  public:
-    MBlazeELFWriterInfo(TargetMachine &TM);
-    virtual ~MBlazeELFWriterInfo();
-
-    /// getRelocationType - Returns the target specific ELF Relocation type.
-    /// 'MachineRelTy' contains the object code independent relocation type
-    virtual unsigned getRelocationType(unsigned MachineRelTy) const;
-
-    /// hasRelocationAddend - True if the target uses an addend in the
-    /// ELF relocation entry.
-    virtual bool hasRelocationAddend() const { return false; }
-
-    /// getDefaultAddendForRelTy - Gets the default addend value for a
-    /// relocation entry based on the target ELF relocation type.
-    virtual long int getDefaultAddendForRelTy(unsigned RelTy,
-                                              long int Modifier = 0) const;
-
-    /// getRelTySize - Returns the size of relocatable field in bits
-    virtual unsigned getRelocationTySize(unsigned RelTy) const;
-
-    /// isPCRelativeRel - True if the relocation type is pc relative
-    virtual bool isPCRelativeRel(unsigned RelTy) const;
-
-    /// getJumpTableRelocationTy - Returns the machine relocation type used
-    /// to reference a jumptable.
-    virtual unsigned getAbsoluteLabelMachineRelTy() const;
-
-    /// computeRelocation - Some relocatable fields could be relocated
-    /// directly, avoiding the relocation symbol emission, compute the
-    /// final relocation value for this symbol.
-    virtual long int computeRelocation(unsigned SymOffset, unsigned RelOffset,
-                                       unsigned RelTy) const;
-  };
-
-} // end llvm namespace
-
-#endif // MBLAZE_ELF_WRITER_INFO_H
diff --git a/lib/Target/MBlaze/MBlazeFrameLowering.cpp b/lib/Target/MBlaze/MBlazeFrameLowering.cpp
index d2f14a5..9e467bf 100644
--- a/lib/Target/MBlaze/MBlazeFrameLowering.cpp
+++ b/lib/Target/MBlaze/MBlazeFrameLowering.cpp
@@ -23,7 +23,7 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetOptions.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
diff --git a/lib/Target/MBlaze/MBlazeIntrinsicInfo.cpp b/lib/Target/MBlaze/MBlazeIntrinsicInfo.cpp
index 91aaf94..1c2e3b2 100644
--- a/lib/Target/MBlaze/MBlazeIntrinsicInfo.cpp
+++ b/lib/Target/MBlaze/MBlazeIntrinsicInfo.cpp
@@ -83,7 +83,7 @@ bool MBlazeIntrinsicInfo::isOverloaded(unsigned IntrID) const {
 #undef GET_INTRINSIC_OVERLOAD_TABLE
 }
 
-/// This defines the "getAttributes(ID id)" method.
+/// This defines the "getAttributes(LLVMContext &C, ID id)" method.
 #define GET_INTRINSIC_ATTRIBUTES
 #include "MBlazeGenIntrinsics.inc"
 #undef GET_INTRINSIC_ATTRIBUTES
@@ -104,7 +104,8 @@ Function *MBlazeIntrinsicInfo::getDeclaration(Module *M, unsigned IntrID,
                                                 Type **Tys,
                                                 unsigned numTy) const {
   assert(!isOverloaded(IntrID) && "MBlaze intrinsics are not overloaded");
-  AttrListPtr AList = getAttributes((mblazeIntrinsic::ID) IntrID);
+  AttrListPtr AList = getAttributes(M->getContext(),
+                                    (mblazeIntrinsic::ID) IntrID);
   return cast<Function>(M->getOrInsertFunction(getName(IntrID),
                                                getType(M->getContext(), IntrID),
                                                AList));
diff --git a/lib/Target/MBlaze/MBlazeRegisterInfo.cpp b/lib/Target/MBlaze/MBlazeRegisterInfo.cpp
index 46f5207..daa76e8 100644
--- a/lib/Target/MBlaze/MBlazeRegisterInfo.cpp
+++ b/lib/Target/MBlaze/MBlazeRegisterInfo.cpp
@@ -140,7 +140,7 @@ eliminateFrameIndex(MachineBasicBlock::iterator II, int SPAdj,
 
   unsigned oi = i == 2 ? 1 : 2;
 
-  DEBUG(dbgs() << "\nFunction : " << MF.getFunction()->getName() << "\n";
+  DEBUG(dbgs() << "\nFunction : " << MF.getName() << "\n";
         dbgs() << "<--------->\n" << MI);
 
   int FrameIndex = MI.getOperand(i).getIndex();
diff --git a/lib/Target/MBlaze/MBlazeTargetMachine.cpp b/lib/Target/MBlaze/MBlazeTargetMachine.cpp
index 5f82f14..f180652 100644
--- a/lib/Target/MBlaze/MBlazeTargetMachine.cpp
+++ b/lib/Target/MBlaze/MBlazeTargetMachine.cpp
@@ -38,11 +38,12 @@ MBlazeTargetMachine(const Target &T, StringRef TT,
                     CodeGenOpt::Level OL)
   : LLVMTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL),
     Subtarget(TT, CPU, FS),
-    DataLayout("E-p:32:32:32-i8:8:8-i16:16:16"),
+    DL("E-p:32:32:32-i8:8:8-i16:16:16"),
     InstrInfo(*this),
     FrameLowering(Subtarget),
-    TLInfo(*this), TSInfo(*this), ELFWriterInfo(*this),
-    InstrItins(Subtarget.getInstrItineraryData()) {
+    TLInfo(*this), TSInfo(*this),
+    InstrItins(Subtarget.getInstrItineraryData()),
+    STTI(&TLInfo), VTTI(&TLInfo) {
 }
 
 namespace {
diff --git a/lib/Target/MBlaze/MBlazeTargetMachine.h b/lib/Target/MBlaze/MBlazeTargetMachine.h
index 1647a21..a8df4e6 100644
--- a/lib/Target/MBlaze/MBlazeTargetMachine.h
+++ b/lib/Target/MBlaze/MBlazeTargetMachine.h
@@ -20,25 +20,26 @@
 #include "MBlazeSelectionDAGInfo.h"
 #include "MBlazeIntrinsicInfo.h"
 #include "MBlazeFrameLowering.h"
-#include "MBlazeELFWriterInfo.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/Target/TargetMachine.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetFrameLowering.h"
+#include "llvm/Target/TargetTransformImpl.h"
 
 namespace llvm {
   class formatted_raw_ostream;
 
   class MBlazeTargetMachine : public LLVMTargetMachine {
     MBlazeSubtarget        Subtarget;
-    const TargetData       DataLayout; // Calculates type size & alignment
+    const DataLayout       DL; // Calculates type size & alignment
     MBlazeInstrInfo        InstrInfo;
     MBlazeFrameLowering    FrameLowering;
     MBlazeTargetLowering   TLInfo;
     MBlazeSelectionDAGInfo TSInfo;
     MBlazeIntrinsicInfo    IntrinsicInfo;
-    MBlazeELFWriterInfo    ELFWriterInfo;
     InstrItineraryData     InstrItins;
+    ScalarTargetTransformImpl STTI;
+    VectorTargetTransformImpl VTTI;
 
   public:
     MBlazeTargetMachine(const Target &T, StringRef TT,
@@ -59,8 +60,8 @@ namespace llvm {
     virtual const MBlazeSubtarget *getSubtargetImpl() const
     { return &Subtarget; }
 
-    virtual const TargetData *getTargetData() const
-    { return &DataLayout;}
+    virtual const DataLayout *getDataLayout() const
+    { return &DL;}
 
     virtual const MBlazeRegisterInfo *getRegisterInfo() const
     { return &InstrInfo.getRegisterInfo(); }
@@ -74,9 +75,10 @@ namespace llvm {
     const TargetIntrinsicInfo *getIntrinsicInfo() const
     { return &IntrinsicInfo; }
 
-    virtual const MBlazeELFWriterInfo *getELFWriterInfo() const {
-      return &ELFWriterInfo;
-    }
+    virtual const ScalarTargetTransformInfo *getScalarTargetTransformInfo()const
+    { return &STTI; }
+    virtual const VectorTargetTransformInfo *getVectorTargetTransformInfo()const
+    { return &VTTI; }
 
     // Pass Pipeline Configuration
     virtual TargetPassConfig *createPassConfig(PassManagerBase &PM);
diff --git a/lib/Target/MBlaze/MBlazeTargetObjectFile.cpp b/lib/Target/MBlaze/MBlazeTargetObjectFile.cpp
index f66ea30..899c74ee 100644
--- a/lib/Target/MBlaze/MBlazeTargetObjectFile.cpp
+++ b/lib/Target/MBlaze/MBlazeTargetObjectFile.cpp
@@ -13,7 +13,7 @@
 #include "llvm/GlobalVariable.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCSectionELF.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ELF.h"
@@ -70,7 +70,7 @@ IsGlobalInSmallSection(const GlobalValue *GV, const TargetMachine &TM,
     return false;
 
   Type *Ty = GV->getType()->getElementType();
-  return IsInSmallSection(TM.getTargetData()->getTypeAllocSize(Ty));
+  return IsInSmallSection(TM.getDataLayout()->getTypeAllocSize(Ty));
 }
 
 const MCSection *MBlazeTargetObjectFile::
diff --git a/lib/Target/MBlaze/MCTargetDesc/MBlazeAsmBackend.cpp b/lib/Target/MBlaze/MCTargetDesc/MBlazeAsmBackend.cpp
index f383fec..44feeb4 100644
--- a/lib/Target/MBlaze/MCTargetDesc/MBlazeAsmBackend.cpp
+++ b/lib/Target/MBlaze/MCTargetDesc/MBlazeAsmBackend.cpp
@@ -156,7 +156,8 @@ void ELFMBlazeAsmBackend::applyFixup(const MCFixup &Fixup, char *Data,
 }
 } // end anonymous namespace
 
-MCAsmBackend *llvm::createMBlazeAsmBackend(const Target &T, StringRef TT) {
+MCAsmBackend *llvm::createMBlazeAsmBackend(const Target &T, StringRef TT,
+                                           StringRef CPU) {
   Triple TheTriple(TT);
 
   if (TheTriple.isOSDarwin())
diff --git a/lib/Target/MBlaze/MCTargetDesc/MBlazeMCCodeEmitter.cpp b/lib/Target/MBlaze/MCTargetDesc/MBlazeMCCodeEmitter.cpp
index bfd11a0..2b71d9d 100644
--- a/lib/Target/MBlaze/MCTargetDesc/MBlazeMCCodeEmitter.cpp
+++ b/lib/Target/MBlaze/MCTargetDesc/MBlazeMCCodeEmitter.cpp
@@ -29,8 +29,8 @@ STATISTIC(MCNumEmitted, "Number of MC instructions emitted");
 
 namespace {
 class MBlazeMCCodeEmitter : public MCCodeEmitter {
-  MBlazeMCCodeEmitter(const MBlazeMCCodeEmitter &); // DO NOT IMPLEMENT
-  void operator=(const MBlazeMCCodeEmitter &); // DO NOT IMPLEMENT
+  MBlazeMCCodeEmitter(const MBlazeMCCodeEmitter &) LLVM_DELETED_FUNCTION;
+  void operator=(const MBlazeMCCodeEmitter &) LLVM_DELETED_FUNCTION;
   const MCInstrInfo &MCII;
 
 public:
diff --git a/lib/Target/MBlaze/MCTargetDesc/MBlazeMCTargetDesc.h b/lib/Target/MBlaze/MCTargetDesc/MBlazeMCTargetDesc.h
index 7cc96c6..7bc7d8f 100644
--- a/lib/Target/MBlaze/MCTargetDesc/MBlazeMCTargetDesc.h
+++ b/lib/Target/MBlaze/MCTargetDesc/MBlazeMCTargetDesc.h
@@ -35,7 +35,8 @@ MCCodeEmitter *createMBlazeMCCodeEmitter(const MCInstrInfo &MCII,
                                          const MCSubtargetInfo &STI,
                                          MCContext &Ctx);
 
-MCAsmBackend *createMBlazeAsmBackend(const Target &T, StringRef TT);
+MCAsmBackend *createMBlazeAsmBackend(const Target &T, StringRef TT,
+                                     StringRef CPU);
 
 MCObjectWriter *createMBlazeELFObjectWriter(raw_ostream &OS, uint8_t OSABI);
 } // End llvm namespace
diff --git a/lib/Target/MSP430/MSP430FrameLowering.cpp b/lib/Target/MSP430/MSP430FrameLowering.cpp
index 61d7f2b..2e170f1 100644
--- a/lib/Target/MSP430/MSP430FrameLowering.cpp
+++ b/lib/Target/MSP430/MSP430FrameLowering.cpp
@@ -20,7 +20,7 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetOptions.h"
 #include "llvm/Support/CommandLine.h"
 
@@ -221,3 +221,17 @@ MSP430FrameLowering::restoreCalleeSavedRegisters(MachineBasicBlock &MBB,
 
   return true;
 }
+
+void
+MSP430FrameLowering::processFunctionBeforeFrameFinalized(MachineFunction &MF)
+                                                                         const {
+  const TargetFrameLowering *TFI = MF.getTarget().getFrameLowering();
+
+  // Create a frame entry for the FPW register that must be saved.
+  if (TFI->hasFP(MF)) {
+    int FrameIdx = MF.getFrameInfo()->CreateFixedObject(2, -4, true);
+    (void)FrameIdx;
+    assert(FrameIdx == MF.getFrameInfo()->getObjectIndexBegin() &&
+           "Slot for FPW register must be last in order to be found!");
+  }
+}
diff --git a/lib/Target/MSP430/MSP430FrameLowering.h b/lib/Target/MSP430/MSP430FrameLowering.h
index b636827..cb02545 100644
--- a/lib/Target/MSP430/MSP430FrameLowering.h
+++ b/lib/Target/MSP430/MSP430FrameLowering.h
@@ -46,6 +46,7 @@ public:
 
   bool hasFP(const MachineFunction &MF) const;
   bool hasReservedCallFrame(const MachineFunction &MF) const;
+  void processFunctionBeforeFrameFinalized(MachineFunction &MF) const;
 };
 
 } // End llvm namespace
diff --git a/lib/Target/MSP430/MSP430ISelDAGToDAG.cpp b/lib/Target/MSP430/MSP430ISelDAGToDAG.cpp
index 5430d43..5efc6a3 100644
--- a/lib/Target/MSP430/MSP430ISelDAGToDAG.cpp
+++ b/lib/Target/MSP430/MSP430ISelDAGToDAG.cpp
@@ -274,8 +274,8 @@ bool MSP430DAGToDAGISel::SelectAddr(SDValue N,
   else if (AM.JT != -1)
     Disp = CurDAG->getTargetJumpTable(AM.JT, MVT::i16, 0/*AM.SymbolFlags*/);
   else if (AM.BlockAddr)
-    Disp = CurDAG->getBlockAddress(AM.BlockAddr, MVT::i32,
-                                   true, 0/*AM.SymbolFlags*/);
+    Disp = CurDAG->getTargetBlockAddress(AM.BlockAddr, MVT::i32, 0,
+                                         0/*AM.SymbolFlags*/);
   else
     Disp = CurDAG->getTargetConstant(AM.Disp, MVT::i16);
 
diff --git a/lib/Target/MSP430/MSP430ISelLowering.cpp b/lib/Target/MSP430/MSP430ISelLowering.cpp
index f8b7e14..fc677ae 100644
--- a/lib/Target/MSP430/MSP430ISelLowering.cpp
+++ b/lib/Target/MSP430/MSP430ISelLowering.cpp
@@ -61,7 +61,7 @@ MSP430TargetLowering::MSP430TargetLowering(MSP430TargetMachine &tm) :
   TargetLowering(tm, new TargetLoweringObjectFileELF()),
   Subtarget(*tm.getSubtargetImpl()) {
 
-  TD = getTargetData();
+  TD = getDataLayout();
 
   // Set up the register classes.
   addRegisterClass(MVT::i8,  &MSP430::GR8RegClass);
@@ -655,7 +655,7 @@ SDValue MSP430TargetLowering::LowerBlockAddress(SDValue Op,
                                                 SelectionDAG &DAG) const {
   DebugLoc dl = Op.getDebugLoc();
   const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress();
-  SDValue Result = DAG.getBlockAddress(BA, getPointerTy(), /*isTarget=*/true);
+  SDValue Result = DAG.getTargetBlockAddress(BA, getPointerTy());
 
   return DAG.getNode(MSP430ISD::Wrapper, dl, getPointerTy(), Result);
 }
diff --git a/lib/Target/MSP430/MSP430ISelLowering.h b/lib/Target/MSP430/MSP430ISelLowering.h
index d8ad02f..991304c 100644
--- a/lib/Target/MSP430/MSP430ISelLowering.h
+++ b/lib/Target/MSP430/MSP430ISelLowering.h
@@ -169,7 +169,7 @@ namespace llvm {
                                             SelectionDAG &DAG) const;
 
     const MSP430Subtarget &Subtarget;
-    const TargetData *TD;
+    const DataLayout *TD;
   };
 } // namespace llvm
 
diff --git a/lib/Target/MSP430/MSP430RegisterInfo.cpp b/lib/Target/MSP430/MSP430RegisterInfo.cpp
index aed46a2..9ae238f 100644
--- a/lib/Target/MSP430/MSP430RegisterInfo.cpp
+++ b/lib/Target/MSP430/MSP430RegisterInfo.cpp
@@ -220,20 +220,6 @@ MSP430RegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
   MI.getOperand(i+1).ChangeToImmediate(Offset);
 }
 
-void
-MSP430RegisterInfo::processFunctionBeforeFrameFinalized(MachineFunction &MF)
-                                                                         const {
-  const TargetFrameLowering *TFI = MF.getTarget().getFrameLowering();
-
-  // Create a frame entry for the FPW register that must be saved.
-  if (TFI->hasFP(MF)) {
-    int FrameIdx = MF.getFrameInfo()->CreateFixedObject(2, -4, true);
-    (void)FrameIdx;
-    assert(FrameIdx == MF.getFrameInfo()->getObjectIndexBegin() &&
-           "Slot for FPW register must be last in order to be found!");
-  }
-}
-
 unsigned MSP430RegisterInfo::getFrameRegister(const MachineFunction &MF) const {
   const TargetFrameLowering *TFI = MF.getTarget().getFrameLowering();
 
diff --git a/lib/Target/MSP430/MSP430RegisterInfo.h b/lib/Target/MSP430/MSP430RegisterInfo.h
index 9ee0a03..64a43bc 100644
--- a/lib/Target/MSP430/MSP430RegisterInfo.h
+++ b/lib/Target/MSP430/MSP430RegisterInfo.h
@@ -49,8 +49,6 @@ public:
   void eliminateFrameIndex(MachineBasicBlock::iterator II,
                            int SPAdj, RegScavenger *RS = NULL) const;
 
-  void processFunctionBeforeFrameFinalized(MachineFunction &MF) const;
-
   // Debug information queries.
   unsigned getFrameRegister(const MachineFunction &MF) const;
 };
diff --git a/lib/Target/MSP430/MSP430TargetMachine.cpp b/lib/Target/MSP430/MSP430TargetMachine.cpp
index 817001d..13e37b3 100644
--- a/lib/Target/MSP430/MSP430TargetMachine.cpp
+++ b/lib/Target/MSP430/MSP430TargetMachine.cpp
@@ -33,10 +33,10 @@ MSP430TargetMachine::MSP430TargetMachine(const Target &T,
                                          CodeGenOpt::Level OL)
   : LLVMTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL),
     Subtarget(TT, CPU, FS),
-    // FIXME: Check TargetData string.
-    DataLayout("e-p:16:16:16-i8:8:8-i16:16:16-i32:16:32-n8:16"),
+    // FIXME: Check DataLayout string.
+    DL("e-p:16:16:16-i8:8:8-i16:16:16-i32:16:32-n8:16"),
     InstrInfo(*this), TLInfo(*this), TSInfo(*this),
-    FrameLowering(Subtarget) { }
+    FrameLowering(Subtarget), STTI(&TLInfo), VTTI(&TLInfo) { }
 
 namespace {
 /// MSP430 Code Generator Pass Configuration Options.
diff --git a/lib/Target/MSP430/MSP430TargetMachine.h b/lib/Target/MSP430/MSP430TargetMachine.h
index f54146b..186172e 100644
--- a/lib/Target/MSP430/MSP430TargetMachine.h
+++ b/lib/Target/MSP430/MSP430TargetMachine.h
@@ -21,9 +21,10 @@
 #include "MSP430SelectionDAGInfo.h"
 #include "MSP430RegisterInfo.h"
 #include "MSP430Subtarget.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetFrameLowering.h"
 #include "llvm/Target/TargetMachine.h"
+#include "llvm/Target/TargetTransformImpl.h"
 
 namespace llvm {
 
@@ -31,11 +32,13 @@ namespace llvm {
 ///
 class MSP430TargetMachine : public LLVMTargetMachine {
   MSP430Subtarget        Subtarget;
-  const TargetData       DataLayout;       // Calculates type size & alignment
+  const DataLayout       DL;       // Calculates type size & alignment
   MSP430InstrInfo        InstrInfo;
   MSP430TargetLowering   TLInfo;
   MSP430SelectionDAGInfo TSInfo;
   MSP430FrameLowering    FrameLowering;
+  ScalarTargetTransformImpl STTI;
+  VectorTargetTransformImpl VTTI;
 
 public:
   MSP430TargetMachine(const Target &T, StringRef TT,
@@ -47,7 +50,7 @@ public:
     return &FrameLowering;
   }
   virtual const MSP430InstrInfo *getInstrInfo() const  { return &InstrInfo; }
-  virtual const TargetData *getTargetData() const     { return &DataLayout;}
+  virtual const DataLayout *getDataLayout() const     { return &DL;}
   virtual const MSP430Subtarget *getSubtargetImpl() const { return &Subtarget; }
 
   virtual const TargetRegisterInfo *getRegisterInfo() const {
@@ -61,7 +64,12 @@ public:
   virtual const MSP430SelectionDAGInfo* getSelectionDAGInfo() const {
     return &TSInfo;
   }
-
+  virtual const ScalarTargetTransformInfo *getScalarTargetTransformInfo()const {
+    return &STTI;
+  }
+  virtual const VectorTargetTransformInfo *getVectorTargetTransformInfo()const {
+    return &VTTI;
+  }
   virtual TargetPassConfig *createPassConfig(PassManagerBase &PM);
 }; // MSP430TargetMachine.
 
diff --git a/lib/Target/Mangler.cpp b/lib/Target/Mangler.cpp
index 786a0c5..539a1f7 100644
--- a/lib/Target/Mangler.cpp
+++ b/lib/Target/Mangler.cpp
@@ -14,7 +14,7 @@
 #include "llvm/Target/Mangler.h"
 #include "llvm/DerivedTypes.h"
 #include "llvm/Function.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/Support/raw_ostream.h"
@@ -44,7 +44,7 @@ static void MangleLetter(SmallVectorImpl<char> &OutName, unsigned char C) {
   OutName.push_back('_');
 }
 
-/// NameNeedsEscaping - Return true if the identifier \arg Str needs quotes
+/// NameNeedsEscaping - Return true if the identifier \p Str needs quotes
 /// for this assembler.
 static bool NameNeedsEscaping(StringRef Str, const MCAsmInfo &MAI) {
   assert(!Str.empty() && "Cannot create an empty MCSymbol");
@@ -157,7 +157,7 @@ void Mangler::getNameWithPrefix(SmallVectorImpl<char> &OutName,
 /// a suffix on their name indicating the number of words of arguments they
 /// take.
 static void AddFastCallStdCallSuffix(SmallVectorImpl<char> &OutName,
-                                     const Function *F, const TargetData &TD) {
+                                     const Function *F, const DataLayout &TD) {
   // Calculate arguments size total.
   unsigned ArgWords = 0;
   for (Function::const_arg_iterator AI = F->arg_begin(), AE = F->arg_end();
@@ -183,8 +183,7 @@ void Mangler::getNameWithPrefix(SmallVectorImpl<char> &OutName,
   ManglerPrefixTy PrefixTy = Mangler::Default;
   if (GV->hasPrivateLinkage() || isImplicitlyPrivate)
     PrefixTy = Mangler::Private;
-  else if (GV->hasLinkerPrivateLinkage() || GV->hasLinkerPrivateWeakLinkage() ||
-           GV->hasLinkerPrivateWeakDefAutoLinkage())
+  else if (GV->hasLinkerPrivateLinkage() || GV->hasLinkerPrivateWeakLinkage())
     PrefixTy = Mangler::LinkerPrivate;
   
   // If this global has a name, handle it simply.
diff --git a/lib/Target/Mips/AsmParser/CMakeLists.txt b/lib/Target/Mips/AsmParser/CMakeLists.txt
index 6c7343b..28f5219 100644
--- a/lib/Target/Mips/AsmParser/CMakeLists.txt
+++ b/lib/Target/Mips/AsmParser/CMakeLists.txt
@@ -1,3 +1,4 @@
+include_directories( ${CMAKE_CURRENT_BINARY_DIR}/.. ${CMAKE_CURRENT_SOURCE_DIR}/.. )
 add_llvm_library(LLVMMipsAsmParser
   MipsAsmParser.cpp
   )
diff --git a/lib/Target/Mips/AsmParser/MipsAsmParser.cpp b/lib/Target/Mips/AsmParser/MipsAsmParser.cpp
index 58b5590..67b5248 100644
--- a/lib/Target/Mips/AsmParser/MipsAsmParser.cpp
+++ b/lib/Target/Mips/AsmParser/MipsAsmParser.cpp
@@ -8,53 +8,1316 @@
 //===----------------------------------------------------------------------===//
 
 #include "MCTargetDesc/MipsMCTargetDesc.h"
+#include "MipsRegisterInfo.h"
+#include "llvm/ADT/StringSwitch.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCStreamer.h"
+#include "llvm/MC/MCSubtargetInfo.h"
+#include "llvm/MC/MCSymbol.h"
 #include "llvm/MC/MCParser/MCAsmLexer.h"
+#include "llvm/MC/MCParser/MCParsedAsmOperand.h"
 #include "llvm/MC/MCTargetAsmParser.h"
 #include "llvm/Support/TargetRegistry.h"
 
 using namespace llvm;
 
 namespace {
+class MipsAssemblerOptions {
+public:
+  MipsAssemblerOptions():
+    aTReg(1), reorder(true), macro(true) {
+  }
+
+  unsigned getATRegNum() {return aTReg;}
+  bool setATReg(unsigned Reg);
+
+  bool isReorder() {return reorder;}
+  void setReorder() {reorder = true;}
+  void setNoreorder() {reorder = false;}
+
+  bool isMacro() {return macro;}
+  void setMacro() {macro = true;}
+  void setNomacro() {macro = false;}
+
+private:
+  unsigned aTReg;
+  bool reorder;
+  bool macro;
+};
+}
+
+namespace {
 class MipsAsmParser : public MCTargetAsmParser {
-  bool MatchAndEmitInstruction(SMLoc IDLoc,
+
+  enum FpFormatTy {
+    FP_FORMAT_NONE = -1,
+    FP_FORMAT_S,
+    FP_FORMAT_D,
+    FP_FORMAT_L,
+    FP_FORMAT_W
+  } FpFormat;
+
+  MCSubtargetInfo &STI;
+  MCAsmParser &Parser;
+  MipsAssemblerOptions Options;
+
+
+#define GET_ASSEMBLER_HEADER
+#include "MipsGenAsmMatcher.inc"
+
+  bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
                                SmallVectorImpl<MCParsedAsmOperand*> &Operands,
-                               MCStreamer &Out);
+                               MCStreamer &Out, unsigned &ErrorInfo,
+                               bool MatchingInlineAsm);
 
   bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc);
 
-  bool ParseInstruction(StringRef Name, SMLoc NameLoc,
-                                SmallVectorImpl<MCParsedAsmOperand*> &Operands);
+  bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
+                        SMLoc NameLoc,
+                        SmallVectorImpl<MCParsedAsmOperand*> &Operands);
+
+  bool parseMathOperation(StringRef Name, SMLoc NameLoc,
+                        SmallVectorImpl<MCParsedAsmOperand*> &Operands);
 
   bool ParseDirective(AsmToken DirectiveID);
 
+  MipsAsmParser::OperandMatchResultTy
+  parseMemOperand(SmallVectorImpl<MCParsedAsmOperand*>&);
+
+  bool ParseOperand(SmallVectorImpl<MCParsedAsmOperand*> &,
+                    StringRef Mnemonic);
+
+  int tryParseRegister(StringRef Mnemonic);
+
+  bool tryParseRegisterOperand(SmallVectorImpl<MCParsedAsmOperand*> &Operands,
+                               StringRef Mnemonic);
+
+  bool needsExpansion(MCInst &Inst);
+
+  void expandInstruction(MCInst &Inst, SMLoc IDLoc,
+                         SmallVectorImpl<MCInst> &Instructions);
+  void expandLoadImm(MCInst &Inst, SMLoc IDLoc,
+                     SmallVectorImpl<MCInst> &Instructions);
+  void expandLoadAddressImm(MCInst &Inst, SMLoc IDLoc,
+                            SmallVectorImpl<MCInst> &Instructions);
+  void expandLoadAddressReg(MCInst &Inst, SMLoc IDLoc,
+                            SmallVectorImpl<MCInst> &Instructions);
+  bool reportParseError(StringRef ErrorMsg);
+
+  bool parseMemOffset(const MCExpr *&Res);
+  bool parseRelocOperand(const MCExpr *&Res);
+
+  bool parseDirectiveSet();
+
+  bool parseSetAtDirective();
+  bool parseSetNoAtDirective();
+  bool parseSetMacroDirective();
+  bool parseSetNoMacroDirective();
+  bool parseSetReorderDirective();
+  bool parseSetNoReorderDirective();
+
+  MCSymbolRefExpr::VariantKind getVariantKind(StringRef Symbol);
+
+  bool isMips64() const {
+    return (STI.getFeatureBits() & Mips::FeatureMips64) != 0;
+  }
+
+  bool isFP64() const {
+    return (STI.getFeatureBits() & Mips::FeatureFP64Bit) != 0;
+  }
+
+  int matchRegisterName(StringRef Symbol);
+
+  int matchRegisterByNumber(unsigned RegNum, StringRef Mnemonic);
+
+  void setFpFormat(FpFormatTy Format) {
+    FpFormat = Format;
+  }
+
+  void setDefaultFpFormat();
+
+  void setFpFormat(StringRef Format);
+
+  FpFormatTy getFpFormat() {return FpFormat;}
+
+  bool requestsDoubleOperand(StringRef Mnemonic);
+
+  unsigned getReg(int RC,int RegNo);
+
+  unsigned getATReg();
 public:
   MipsAsmParser(MCSubtargetInfo &sti, MCAsmParser &parser)
-    : MCTargetAsmParser() {
+    : MCTargetAsmParser(), STI(sti), Parser(parser) {
+    // Initialize the set of available features.
+    setAvailableFeatures(ComputeAvailableFeatures(STI.getFeatureBits()));
+  }
+
+  MCAsmParser &getParser() const { return Parser; }
+  MCAsmLexer &getLexer() const { return Parser.getLexer(); }
+
+};
+}
+
+namespace {
+
+/// MipsOperand - Instances of this class represent a parsed Mips machine
+/// instruction.
+class MipsOperand : public MCParsedAsmOperand {
+
+  enum KindTy {
+    k_CondCode,
+    k_CoprocNum,
+    k_Immediate,
+    k_Memory,
+    k_PostIndexRegister,
+    k_Register,
+    k_Token
+  } Kind;
+
+  MipsOperand(KindTy K) : MCParsedAsmOperand(), Kind(K) {}
+
+  union {
+    struct {
+      const char *Data;
+      unsigned Length;
+    } Tok;
+
+    struct {
+      unsigned RegNum;
+    } Reg;
+
+    struct {
+      const MCExpr *Val;
+    } Imm;
+
+    struct {
+      unsigned Base;
+      const MCExpr *Off;
+    } Mem;
+  };
+
+  SMLoc StartLoc, EndLoc;
+
+public:
+  void addRegOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+    Inst.addOperand(MCOperand::CreateReg(getReg()));
+  }
+
+  void addExpr(MCInst &Inst, const MCExpr *Expr) const{
+    // Add as immediate when possible.  Null MCExpr = 0.
+    if (Expr == 0)
+      Inst.addOperand(MCOperand::CreateImm(0));
+    else if (const MCConstantExpr *CE = dyn_cast<MCConstantExpr>(Expr))
+      Inst.addOperand(MCOperand::CreateImm(CE->getValue()));
+    else
+      Inst.addOperand(MCOperand::CreateExpr(Expr));
+  }
+
+  void addImmOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 1 && "Invalid number of operands!");
+    const MCExpr *Expr = getImm();
+    addExpr(Inst,Expr);
+  }
+
+  void addMemOperands(MCInst &Inst, unsigned N) const {
+    assert(N == 2 && "Invalid number of operands!");
+
+    Inst.addOperand(MCOperand::CreateReg(getMemBase()));
+
+    const MCExpr *Expr = getMemOff();
+    addExpr(Inst,Expr);
+  }
+
+  bool isReg() const { return Kind == k_Register; }
+  bool isImm() const { return Kind == k_Immediate; }
+  bool isToken() const { return Kind == k_Token; }
+  bool isMem() const { return Kind == k_Memory; }
+
+  StringRef getToken() const {
+    assert(Kind == k_Token && "Invalid access!");
+    return StringRef(Tok.Data, Tok.Length);
+  }
+
+  unsigned getReg() const {
+    assert((Kind == k_Register) && "Invalid access!");
+    return Reg.RegNum;
+  }
+
+  const MCExpr *getImm() const {
+    assert((Kind == k_Immediate) && "Invalid access!");
+    return Imm.Val;
+  }
+
+  unsigned getMemBase() const {
+    assert((Kind == k_Memory) && "Invalid access!");
+    return Mem.Base;
+  }
+
+  const MCExpr *getMemOff() const {
+    assert((Kind == k_Memory) && "Invalid access!");
+    return Mem.Off;
+  }
+
+  static MipsOperand *CreateToken(StringRef Str, SMLoc S) {
+    MipsOperand *Op = new MipsOperand(k_Token);
+    Op->Tok.Data = Str.data();
+    Op->Tok.Length = Str.size();
+    Op->StartLoc = S;
+    Op->EndLoc = S;
+    return Op;
   }
 
+  static MipsOperand *CreateReg(unsigned RegNum, SMLoc S, SMLoc E) {
+    MipsOperand *Op = new MipsOperand(k_Register);
+    Op->Reg.RegNum = RegNum;
+    Op->StartLoc = S;
+    Op->EndLoc = E;
+    return Op;
+  }
+
+  static MipsOperand *CreateImm(const MCExpr *Val, SMLoc S, SMLoc E) {
+    MipsOperand *Op = new MipsOperand(k_Immediate);
+    Op->Imm.Val = Val;
+    Op->StartLoc = S;
+    Op->EndLoc = E;
+    return Op;
+  }
+
+  static MipsOperand *CreateMem(unsigned Base, const MCExpr *Off,
+                                 SMLoc S, SMLoc E) {
+    MipsOperand *Op = new MipsOperand(k_Memory);
+    Op->Mem.Base = Base;
+    Op->Mem.Off = Off;
+    Op->StartLoc = S;
+    Op->EndLoc = E;
+    return Op;
+  }
+
+  /// getStartLoc - Get the location of the first token of this operand.
+  SMLoc getStartLoc() const { return StartLoc; }
+  /// getEndLoc - Get the location of the last token of this operand.
+  SMLoc getEndLoc() const { return EndLoc; }
+
+  virtual void print(raw_ostream &OS) const {
+    llvm_unreachable("unimplemented!");
+  }
 };
 }
 
+bool MipsAsmParser::needsExpansion(MCInst &Inst) {
+
+  switch(Inst.getOpcode()) {
+    case Mips::LoadImm32Reg:
+    case Mips::LoadAddr32Imm:
+    case Mips::LoadAddr32Reg:
+      return true;
+    default:
+      return false;
+  }
+}
+
+void MipsAsmParser::expandInstruction(MCInst &Inst, SMLoc IDLoc,
+                        SmallVectorImpl<MCInst> &Instructions){
+  switch(Inst.getOpcode()) {
+    case Mips::LoadImm32Reg:
+      return expandLoadImm(Inst, IDLoc, Instructions);
+    case Mips::LoadAddr32Imm:
+      return expandLoadAddressImm(Inst,IDLoc,Instructions);
+    case Mips::LoadAddr32Reg:
+      return expandLoadAddressReg(Inst,IDLoc,Instructions);
+    }
+}
+
+void MipsAsmParser::expandLoadImm(MCInst &Inst, SMLoc IDLoc,
+                                  SmallVectorImpl<MCInst> &Instructions){
+  MCInst tmpInst;
+  const MCOperand &ImmOp = Inst.getOperand(1);
+  assert(ImmOp.isImm() && "expected immediate operand kind");
+  const MCOperand &RegOp = Inst.getOperand(0);
+  assert(RegOp.isReg() && "expected register operand kind");
+
+  int ImmValue = ImmOp.getImm();
+  tmpInst.setLoc(IDLoc);
+  if ( 0 <= ImmValue && ImmValue <= 65535) {
+    // for 0 <= j <= 65535.
+    // li d,j => ori d,$zero,j
+    tmpInst.setOpcode(isMips64() ? Mips::ORi64 : Mips::ORi);
+    tmpInst.addOperand(MCOperand::CreateReg(RegOp.getReg()));
+    tmpInst.addOperand(
+              MCOperand::CreateReg(isMips64() ? Mips::ZERO_64 : Mips::ZERO));
+    tmpInst.addOperand(MCOperand::CreateImm(ImmValue));
+    Instructions.push_back(tmpInst);
+  } else if ( ImmValue < 0 && ImmValue >= -32768) {
+    // for -32768 <= j < 0.
+    // li d,j => addiu d,$zero,j
+    tmpInst.setOpcode(Mips::ADDiu); //TODO:no ADDiu64 in td files?
+    tmpInst.addOperand(MCOperand::CreateReg(RegOp.getReg()));
+    tmpInst.addOperand(
+              MCOperand::CreateReg(isMips64() ? Mips::ZERO_64 : Mips::ZERO));
+    tmpInst.addOperand(MCOperand::CreateImm(ImmValue));
+    Instructions.push_back(tmpInst);
+  } else {
+    // for any other value of j that is representable as a 32-bit integer.
+    // li d,j => lui d,hi16(j)
+    //           ori d,d,lo16(j)
+    tmpInst.setOpcode(isMips64() ? Mips::LUi64 : Mips::LUi);
+    tmpInst.addOperand(MCOperand::CreateReg(RegOp.getReg()));
+    tmpInst.addOperand(MCOperand::CreateImm((ImmValue & 0xffff0000) >> 16));
+    Instructions.push_back(tmpInst);
+    tmpInst.clear();
+    tmpInst.setOpcode(isMips64() ? Mips::ORi64 : Mips::ORi);
+    tmpInst.addOperand(MCOperand::CreateReg(RegOp.getReg()));
+    tmpInst.addOperand(MCOperand::CreateReg(RegOp.getReg()));
+    tmpInst.addOperand(MCOperand::CreateImm(ImmValue & 0xffff));
+    tmpInst.setLoc(IDLoc);
+    Instructions.push_back(tmpInst);
+  }
+}
+
+void MipsAsmParser::expandLoadAddressReg(MCInst &Inst, SMLoc IDLoc,
+                                         SmallVectorImpl<MCInst> &Instructions){
+  MCInst tmpInst;
+  const MCOperand &ImmOp = Inst.getOperand(2);
+  assert(ImmOp.isImm() && "expected immediate operand kind");
+  const MCOperand &SrcRegOp = Inst.getOperand(1);
+  assert(SrcRegOp.isReg() && "expected register operand kind");
+  const MCOperand &DstRegOp = Inst.getOperand(0);
+  assert(DstRegOp.isReg() && "expected register operand kind");
+  int ImmValue = ImmOp.getImm();
+  if ( -32768 <= ImmValue && ImmValue <= 65535) {
+    //for -32768 <= j <= 65535.
+    //la d,j(s) => addiu d,s,j
+    tmpInst.setOpcode(Mips::ADDiu); //TODO:no ADDiu64 in td files?
+    tmpInst.addOperand(MCOperand::CreateReg(DstRegOp.getReg()));
+    tmpInst.addOperand(MCOperand::CreateReg(SrcRegOp.getReg()));
+    tmpInst.addOperand(MCOperand::CreateImm(ImmValue));
+    Instructions.push_back(tmpInst);
+  } else {
+    //for any other value of j that is representable as a 32-bit integer.
+    //la d,j(s) => lui d,hi16(j)
+    //             ori d,d,lo16(j)
+    //             addu d,d,s
+    tmpInst.setOpcode(isMips64()?Mips::LUi64:Mips::LUi);
+    tmpInst.addOperand(MCOperand::CreateReg(DstRegOp.getReg()));
+    tmpInst.addOperand(MCOperand::CreateImm((ImmValue & 0xffff0000) >> 16));
+    Instructions.push_back(tmpInst);
+    tmpInst.clear();
+    tmpInst.setOpcode(isMips64()?Mips::ORi64:Mips::ORi);
+    tmpInst.addOperand(MCOperand::CreateReg(DstRegOp.getReg()));
+    tmpInst.addOperand(MCOperand::CreateReg(DstRegOp.getReg()));
+    tmpInst.addOperand(MCOperand::CreateImm(ImmValue & 0xffff));
+    Instructions.push_back(tmpInst);
+    tmpInst.clear();
+    tmpInst.setOpcode(Mips::ADDu);
+    tmpInst.addOperand(MCOperand::CreateReg(DstRegOp.getReg()));
+    tmpInst.addOperand(MCOperand::CreateReg(DstRegOp.getReg()));
+    tmpInst.addOperand(MCOperand::CreateReg(SrcRegOp.getReg()));
+    Instructions.push_back(tmpInst);
+  }
+}
+
+void MipsAsmParser::expandLoadAddressImm(MCInst &Inst, SMLoc IDLoc,
+                                         SmallVectorImpl<MCInst> &Instructions){
+  MCInst tmpInst;
+  const MCOperand &ImmOp = Inst.getOperand(1);
+  assert(ImmOp.isImm() && "expected immediate operand kind");
+  const MCOperand &RegOp = Inst.getOperand(0);
+  assert(RegOp.isReg() && "expected register operand kind");
+  int ImmValue = ImmOp.getImm();
+  if ( -32768 <= ImmValue && ImmValue <= 65535) {
+    //for -32768 <= j <= 65535.
+    //la d,j => addiu d,$zero,j
+    tmpInst.setOpcode(Mips::ADDiu);
+    tmpInst.addOperand(MCOperand::CreateReg(RegOp.getReg()));
+    tmpInst.addOperand(
+              MCOperand::CreateReg(isMips64()?Mips::ZERO_64:Mips::ZERO));
+    tmpInst.addOperand(MCOperand::CreateImm(ImmValue));
+    Instructions.push_back(tmpInst);
+  } else {
+    //for any other value of j that is representable as a 32-bit integer.
+    //la d,j => lui d,hi16(j)
+    //          ori d,d,lo16(j)
+    tmpInst.setOpcode(isMips64()?Mips::LUi64:Mips::LUi);
+    tmpInst.addOperand(MCOperand::CreateReg(RegOp.getReg()));
+    tmpInst.addOperand(MCOperand::CreateImm((ImmValue & 0xffff0000) >> 16));
+    Instructions.push_back(tmpInst);
+    tmpInst.clear();
+    tmpInst.setOpcode(isMips64()?Mips::ORi64:Mips::ORi);
+    tmpInst.addOperand(MCOperand::CreateReg(RegOp.getReg()));
+    tmpInst.addOperand(MCOperand::CreateReg(RegOp.getReg()));
+    tmpInst.addOperand(MCOperand::CreateImm(ImmValue & 0xffff));
+    Instructions.push_back(tmpInst);
+  }
+}
+
 bool MipsAsmParser::
-MatchAndEmitInstruction(SMLoc IDLoc,
+MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
                         SmallVectorImpl<MCParsedAsmOperand*> &Operands,
-                        MCStreamer &Out) {
+                        MCStreamer &Out, unsigned &ErrorInfo,
+                        bool MatchingInlineAsm) {
+  MCInst Inst;
+  unsigned MatchResult = MatchInstructionImpl(Operands, Inst, ErrorInfo,
+                                              MatchingInlineAsm);
+
+  switch (MatchResult) {
+  default: break;
+  case Match_Success: {
+    if (needsExpansion(Inst)) {
+      SmallVector<MCInst, 4> Instructions;
+      expandInstruction(Inst, IDLoc, Instructions);
+      for(unsigned i =0; i < Instructions.size(); i++){
+        Out.EmitInstruction(Instructions[i]);
+      }
+    } else {
+        Inst.setLoc(IDLoc);
+        Out.EmitInstruction(Inst);
+      }
+    return false;
+  }
+  case Match_MissingFeature:
+    Error(IDLoc, "instruction requires a CPU feature not currently enabled");
+    return true;
+  case Match_InvalidOperand: {
+    SMLoc ErrorLoc = IDLoc;
+    if (ErrorInfo != ~0U) {
+      if (ErrorInfo >= Operands.size())
+        return Error(IDLoc, "too few operands for instruction");
+
+      ErrorLoc = ((MipsOperand*)Operands[ErrorInfo])->getStartLoc();
+      if (ErrorLoc == SMLoc()) ErrorLoc = IDLoc;
+    }
+
+    return Error(ErrorLoc, "invalid operand for instruction");
+  }
+  case Match_MnemonicFail:
+    return Error(IDLoc, "invalid instruction");
+  }
+  return true;
+}
+
+int MipsAsmParser::matchRegisterName(StringRef Name) {
+
+   int CC;
+   if (!isMips64())
+    CC = StringSwitch<unsigned>(Name)
+      .Case("zero",  Mips::ZERO)
+      .Case("a0",  Mips::A0)
+      .Case("a1",  Mips::A1)
+      .Case("a2",  Mips::A2)
+      .Case("a3",  Mips::A3)
+      .Case("v0",  Mips::V0)
+      .Case("v1",  Mips::V1)
+      .Case("s0",  Mips::S0)
+      .Case("s1",  Mips::S1)
+      .Case("s2",  Mips::S2)
+      .Case("s3",  Mips::S3)
+      .Case("s4",  Mips::S4)
+      .Case("s5",  Mips::S5)
+      .Case("s6",  Mips::S6)
+      .Case("s7",  Mips::S7)
+      .Case("k0",  Mips::K0)
+      .Case("k1",  Mips::K1)
+      .Case("sp",  Mips::SP)
+      .Case("fp",  Mips::FP)
+      .Case("gp",  Mips::GP)
+      .Case("ra",  Mips::RA)
+      .Case("t0",  Mips::T0)
+      .Case("t1",  Mips::T1)
+      .Case("t2",  Mips::T2)
+      .Case("t3",  Mips::T3)
+      .Case("t4",  Mips::T4)
+      .Case("t5",  Mips::T5)
+      .Case("t6",  Mips::T6)
+      .Case("t7",  Mips::T7)
+      .Case("t8",  Mips::T8)
+      .Case("t9",  Mips::T9)
+      .Case("at",  Mips::AT)
+      .Case("fcc0",  Mips::FCC0)
+      .Default(-1);
+   else
+    CC = StringSwitch<unsigned>(Name)
+      .Case("zero", Mips::ZERO_64)
+      .Case("at", Mips::AT_64)
+      .Case("v0", Mips::V0_64)
+      .Case("v1", Mips::V1_64)
+      .Case("a0", Mips::A0_64)
+      .Case("a1", Mips::A1_64)
+      .Case("a2", Mips::A2_64)
+      .Case("a3", Mips::A3_64)
+      .Case("a4", Mips::T0_64)
+      .Case("a5", Mips::T1_64)
+      .Case("a6", Mips::T2_64)
+      .Case("a7", Mips::T3_64)
+      .Case("t4", Mips::T4_64)
+      .Case("t5", Mips::T5_64)
+      .Case("t6", Mips::T6_64)
+      .Case("t7", Mips::T7_64)
+      .Case("s0", Mips::S0_64)
+      .Case("s1", Mips::S1_64)
+      .Case("s2", Mips::S2_64)
+      .Case("s3", Mips::S3_64)
+      .Case("s4", Mips::S4_64)
+      .Case("s5", Mips::S5_64)
+      .Case("s6", Mips::S6_64)
+      .Case("s7", Mips::S7_64)
+      .Case("t8", Mips::T8_64)
+      .Case("t9", Mips::T9_64)
+      .Case("kt0", Mips::K0_64)
+      .Case("kt1", Mips::K1_64)
+      .Case("gp", Mips::GP_64)
+      .Case("sp", Mips::SP_64)
+      .Case("fp", Mips::FP_64)
+      .Case("s8", Mips::FP_64)
+      .Case("ra", Mips::RA_64)
+      .Default(-1);
+
+  if (CC != -1)
+    return CC;
+
+  if (Name[0] == 'f') {
+    StringRef NumString = Name.substr(1);
+    unsigned IntVal;
+    if( NumString.getAsInteger(10, IntVal))
+      return -1; // not integer
+    if (IntVal > 31)
+      return -1;
+
+    FpFormatTy Format = getFpFormat();
+
+    if (Format == FP_FORMAT_S || Format == FP_FORMAT_W)
+      return getReg(Mips::FGR32RegClassID, IntVal);
+    if (Format == FP_FORMAT_D) {
+      if(isFP64()) {
+        return getReg(Mips::FGR64RegClassID, IntVal);
+      }
+      // only even numbers available as register pairs
+      if (( IntVal > 31) || (IntVal%2 !=  0))
+        return -1;
+      return getReg(Mips::AFGR64RegClassID, IntVal/2);
+    }
+  }
+
+  return -1;
+}
+void MipsAsmParser::setDefaultFpFormat() {
+
+  if (isMips64() || isFP64())
+    FpFormat = FP_FORMAT_D;
+  else
+    FpFormat = FP_FORMAT_S;
+}
+
+bool MipsAsmParser::requestsDoubleOperand(StringRef Mnemonic){
+
+  bool IsDouble = StringSwitch<bool>(Mnemonic.lower())
+    .Case("ldxc1", true)
+    .Case("ldc1",  true)
+    .Case("sdxc1", true)
+    .Case("sdc1",  true)
+    .Default(false);
+
+  return IsDouble;
+}
+void MipsAsmParser::setFpFormat(StringRef Format) {
+
+  FpFormat = StringSwitch<FpFormatTy>(Format.lower())
+    .Case(".s",  FP_FORMAT_S)
+    .Case(".d",  FP_FORMAT_D)
+    .Case(".l",  FP_FORMAT_L)
+    .Case(".w",  FP_FORMAT_W)
+    .Default(FP_FORMAT_NONE);
+}
+
+bool MipsAssemblerOptions::setATReg(unsigned Reg) {
+  if (Reg > 31)
+    return false;
+
+  aTReg = Reg;
   return true;
 }
 
+unsigned MipsAsmParser::getATReg() {
+  unsigned Reg = Options.getATRegNum();
+  if (isMips64())
+    return getReg(Mips::CPU64RegsRegClassID,Reg);
+  
+  return getReg(Mips::CPURegsRegClassID,Reg);
+}
+
+unsigned MipsAsmParser::getReg(int RC,int RegNo) {
+  return *(getContext().getRegisterInfo().getRegClass(RC).begin() + RegNo);
+}
+
+int MipsAsmParser::matchRegisterByNumber(unsigned RegNum, StringRef Mnemonic) {
+
+  if (Mnemonic.lower() == "rdhwr") {
+    // at the moment only hwreg29 is supported
+    if (RegNum != 29)
+      return -1;
+    return Mips::HWR29;
+  }
+
+  if (RegNum > 31)
+    return -1;
+
+  // MIPS64 registers are numbered 1 after the 32-bit equivalents
+  return getReg(Mips::CPURegsRegClassID, RegNum) + isMips64();
+}
+
+int MipsAsmParser::tryParseRegister(StringRef Mnemonic) {
+  const AsmToken &Tok = Parser.getTok();
+  int RegNum = -1;
+
+  if (Tok.is(AsmToken::Identifier)) {
+    std::string lowerCase = Tok.getString().lower();
+    RegNum = matchRegisterName(lowerCase);
+  } else if (Tok.is(AsmToken::Integer))
+    RegNum = matchRegisterByNumber(static_cast<unsigned>(Tok.getIntVal()),
+                                   Mnemonic.lower());
+    else
+      return RegNum;  //error
+  // 64 bit div operations require Mips::ZERO instead of MIPS::ZERO_64
+  if (isMips64() && RegNum == Mips::ZERO_64) {
+    if (Mnemonic.find("ddiv") != StringRef::npos)
+      RegNum = Mips::ZERO;
+  }
+  return RegNum;
+}
+
 bool MipsAsmParser::
-ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) {
+  tryParseRegisterOperand(SmallVectorImpl<MCParsedAsmOperand*> &Operands,
+                          StringRef Mnemonic){
+
+  SMLoc S = Parser.getTok().getLoc();
+  int RegNo = -1;
+
+  // FIXME: we should make a more generic method for CCR
+  if ((Mnemonic == "cfc1" || Mnemonic == "ctc1")
+      && Operands.size() == 2 && Parser.getTok().is(AsmToken::Integer)){
+    RegNo = Parser.getTok().getIntVal();  // get the int value
+    // at the moment only fcc0 is supported
+    if (RegNo ==  0)
+      RegNo = Mips::FCC0;
+  } else
+    RegNo = tryParseRegister(Mnemonic);
+  if (RegNo == -1)
+    return true;
+
+  Operands.push_back(MipsOperand::CreateReg(RegNo, S,
+      Parser.getTok().getLoc()));
+  Parser.Lex(); // Eat register token.
+  return false;
+}
+
+bool MipsAsmParser::ParseOperand(SmallVectorImpl<MCParsedAsmOperand*>&Operands,
+                                 StringRef Mnemonic) {
+  // Check if the current operand has a custom associated parser, if so, try to
+  // custom parse the operand, or fallback to the general approach.
+  OperandMatchResultTy ResTy = MatchOperandParserImpl(Operands, Mnemonic);
+  if (ResTy == MatchOperand_Success)
+    return false;
+  // If there wasn't a custom match, try the generic matcher below. Otherwise,
+  // there was a match, but an error occurred, in which case, just return that
+  // the operand parsing failed.
+  if (ResTy == MatchOperand_ParseFail)
+    return true;
+
+  switch (getLexer().getKind()) {
+  default:
+    Error(Parser.getTok().getLoc(), "unexpected token in operand");
+    return true;
+  case AsmToken::Dollar: {
+    // parse register
+    SMLoc S = Parser.getTok().getLoc();
+    Parser.Lex(); // Eat dollar token.
+    // parse register operand
+    if (!tryParseRegisterOperand(Operands, Mnemonic)) {
+      if (getLexer().is(AsmToken::LParen)) {
+        // check if it is indexed addressing operand
+        Operands.push_back(MipsOperand::CreateToken("(", S));
+        Parser.Lex(); // eat parenthesis
+        if (getLexer().isNot(AsmToken::Dollar))
+          return true;
+
+        Parser.Lex(); // eat dollar
+        if (tryParseRegisterOperand(Operands, Mnemonic))
+          return true;
+
+        if (!getLexer().is(AsmToken::RParen))
+          return true;
+
+        S = Parser.getTok().getLoc();
+        Operands.push_back(MipsOperand::CreateToken(")", S));
+        Parser.Lex();
+      }
+      return false;
+    }
+    // maybe it is a symbol reference
+    StringRef Identifier;
+    if (Parser.ParseIdentifier(Identifier))
+      return true;
+
+    SMLoc E = SMLoc::getFromPointer(Parser.getTok().getLoc().getPointer() - 1);
+
+    MCSymbol *Sym = getContext().GetOrCreateSymbol("$" + Identifier);
+
+    // Otherwise create a symbol ref.
+    const MCExpr *Res = MCSymbolRefExpr::Create(Sym, MCSymbolRefExpr::VK_None,
+                                                getContext());
+
+    Operands.push_back(MipsOperand::CreateImm(Res, S, E));
+    return false;
+  }
+  case AsmToken::Identifier:
+  case AsmToken::LParen:
+  case AsmToken::Minus:
+  case AsmToken::Plus:
+  case AsmToken::Integer:
+  case AsmToken::String: {
+     // quoted label names
+    const MCExpr *IdVal;
+    SMLoc S = Parser.getTok().getLoc();
+    if (getParser().ParseExpression(IdVal))
+      return true;
+    SMLoc E = SMLoc::getFromPointer(Parser.getTok().getLoc().getPointer() - 1);
+    Operands.push_back(MipsOperand::CreateImm(IdVal, S, E));
+    return false;
+  }
+  case AsmToken::Percent: {
+    // it is a symbol reference or constant expression
+    const MCExpr *IdVal;
+    SMLoc S = Parser.getTok().getLoc(); // start location of the operand
+    if (parseRelocOperand(IdVal))
+      return true;
+
+    SMLoc E = SMLoc::getFromPointer(Parser.getTok().getLoc().getPointer() - 1);
+
+    Operands.push_back(MipsOperand::CreateImm(IdVal, S, E));
+    return false;
+  } // case AsmToken::Percent
+  } // switch(getLexer().getKind())
+  return true;
+}
+
+bool MipsAsmParser::parseRelocOperand(const MCExpr *&Res) {
+
+  Parser.Lex(); // eat % token
+  const AsmToken &Tok = Parser.getTok(); // get next token, operation
+  if (Tok.isNot(AsmToken::Identifier))
+    return true;
+
+  std::string Str = Tok.getIdentifier().str();
+
+  Parser.Lex(); // eat identifier
+  // now make expression from the rest of the operand
+  const MCExpr *IdVal;
+  SMLoc EndLoc;
+
+  if (getLexer().getKind() == AsmToken::LParen) {
+    while (1) {
+      Parser.Lex(); // eat '(' token
+      if (getLexer().getKind() == AsmToken::Percent) {
+        Parser.Lex(); // eat % token
+        const AsmToken &nextTok = Parser.getTok();
+        if (nextTok.isNot(AsmToken::Identifier))
+          return true;
+        Str += "(%";
+        Str += nextTok.getIdentifier();
+        Parser.Lex(); // eat identifier
+        if (getLexer().getKind() != AsmToken::LParen)
+          return true;
+      } else
+        break;
+    }
+    if (getParser().ParseParenExpression(IdVal,EndLoc))
+      return true;
+
+    while (getLexer().getKind() == AsmToken::RParen)
+      Parser.Lex(); // eat ')' token
+
+  } else
+    return true; // parenthesis must follow reloc operand
+
+  // Check the type of the expression
+  if (const MCConstantExpr *MCE = dyn_cast<MCConstantExpr>(IdVal)) {
+    // it's a constant, evaluate lo or hi value
+    int Val = MCE->getValue();
+    if (Str == "lo") {
+      Val = Val & 0xffff;
+    } else if (Str == "hi") {
+      Val = (Val & 0xffff0000) >> 16;
+    }
+    Res = MCConstantExpr::Create(Val, getContext());
+    return false;
+  }
+
+  if (const MCSymbolRefExpr *MSRE = dyn_cast<MCSymbolRefExpr>(IdVal)) {
+    // it's a symbol, create symbolic expression from symbol
+    StringRef Symbol = MSRE->getSymbol().getName();
+    MCSymbolRefExpr::VariantKind VK = getVariantKind(Str);
+    Res = MCSymbolRefExpr::Create(Symbol,VK,getContext());
+    return false;
+  }
   return true;
 }
 
+bool MipsAsmParser::ParseRegister(unsigned &RegNo, SMLoc &StartLoc,
+                                  SMLoc &EndLoc) {
+
+  StartLoc = Parser.getTok().getLoc();
+  RegNo = tryParseRegister("");
+  EndLoc = Parser.getTok().getLoc();
+  return (RegNo == (unsigned)-1);
+}
+
+bool MipsAsmParser::parseMemOffset(const MCExpr *&Res) {
+
+  SMLoc S;
+
+  switch(getLexer().getKind()) {
+  default:
+    return true;
+  case AsmToken::Integer:
+  case AsmToken::Minus:
+  case AsmToken::Plus:
+    return (getParser().ParseExpression(Res));
+  case AsmToken::Percent:
+    return parseRelocOperand(Res);
+  case AsmToken::LParen:
+    return false;  // it's probably assuming 0
+  }
+  return true;
+}
+
+MipsAsmParser::OperandMatchResultTy MipsAsmParser::parseMemOperand(
+               SmallVectorImpl<MCParsedAsmOperand*>&Operands) {
+
+  const MCExpr *IdVal = 0;
+  SMLoc S;
+  // first operand is the offset
+  S = Parser.getTok().getLoc();
+
+  if (parseMemOffset(IdVal))
+    return MatchOperand_ParseFail;
+
+  const AsmToken &Tok = Parser.getTok(); // get next token
+  if (Tok.isNot(AsmToken::LParen)) {
+    MipsOperand *Mnemonic = static_cast<MipsOperand*>(Operands[0]);
+    if (Mnemonic->getToken() == "la") {
+      SMLoc E = SMLoc::getFromPointer(Parser.getTok().getLoc().getPointer()-1);
+      Operands.push_back(MipsOperand::CreateImm(IdVal, S, E));
+      return MatchOperand_Success;
+    }
+    Error(Parser.getTok().getLoc(), "'(' expected");
+    return MatchOperand_ParseFail;
+  }
+
+  Parser.Lex(); // Eat '(' token.
+
+  const AsmToken &Tok1 = Parser.getTok(); // get next token
+  if (Tok1.is(AsmToken::Dollar)) {
+    Parser.Lex(); // Eat '$' token.
+    if (tryParseRegisterOperand(Operands,"")) {
+      Error(Parser.getTok().getLoc(), "unexpected token in operand");
+      return MatchOperand_ParseFail;
+    }
+
+  } else {
+    Error(Parser.getTok().getLoc(), "unexpected token in operand");
+    return MatchOperand_ParseFail;
+  }
+
+  const AsmToken &Tok2 = Parser.getTok(); // get next token
+  if (Tok2.isNot(AsmToken::RParen)) {
+    Error(Parser.getTok().getLoc(), "')' expected");
+    return MatchOperand_ParseFail;
+  }
+
+  SMLoc E = SMLoc::getFromPointer(Parser.getTok().getLoc().getPointer() - 1);
+
+  Parser.Lex(); // Eat ')' token.
+
+  if (IdVal == 0)
+    IdVal = MCConstantExpr::Create(0, getContext());
+
+  // now replace register operand with the mem operand
+  MipsOperand* op = static_cast<MipsOperand*>(Operands.back());
+  int RegNo = op->getReg();
+  // remove register from operands
+  Operands.pop_back();
+  // and add memory operand
+  Operands.push_back(MipsOperand::CreateMem(RegNo, IdVal, S, E));
+  delete op;
+  return MatchOperand_Success;
+}
+
+MCSymbolRefExpr::VariantKind MipsAsmParser::getVariantKind(StringRef Symbol) {
+
+  MCSymbolRefExpr::VariantKind VK
+                   = StringSwitch<MCSymbolRefExpr::VariantKind>(Symbol)
+    .Case("hi",          MCSymbolRefExpr::VK_Mips_ABS_HI)
+    .Case("lo",          MCSymbolRefExpr::VK_Mips_ABS_LO)
+    .Case("gp_rel",      MCSymbolRefExpr::VK_Mips_GPREL)
+    .Case("call16",      MCSymbolRefExpr::VK_Mips_GOT_CALL)
+    .Case("got",         MCSymbolRefExpr::VK_Mips_GOT)
+    .Case("tlsgd",       MCSymbolRefExpr::VK_Mips_TLSGD)
+    .Case("tlsldm",      MCSymbolRefExpr::VK_Mips_TLSLDM)
+    .Case("dtprel_hi",   MCSymbolRefExpr::VK_Mips_DTPREL_HI)
+    .Case("dtprel_lo",   MCSymbolRefExpr::VK_Mips_DTPREL_LO)
+    .Case("gottprel",    MCSymbolRefExpr::VK_Mips_GOTTPREL)
+    .Case("tprel_hi",    MCSymbolRefExpr::VK_Mips_TPREL_HI)
+    .Case("tprel_lo",    MCSymbolRefExpr::VK_Mips_TPREL_LO)
+    .Case("got_disp",    MCSymbolRefExpr::VK_Mips_GOT_DISP)
+    .Case("got_page",    MCSymbolRefExpr::VK_Mips_GOT_PAGE)
+    .Case("got_ofst",    MCSymbolRefExpr::VK_Mips_GOT_OFST)
+    .Case("hi(%neg(%gp_rel",    MCSymbolRefExpr::VK_Mips_GPOFF_HI)
+    .Case("lo(%neg(%gp_rel",    MCSymbolRefExpr::VK_Mips_GPOFF_LO)
+    .Default(MCSymbolRefExpr::VK_None);
+
+  return VK;
+}
+
+static int ConvertCcString(StringRef CondString) {
+  int CC = StringSwitch<unsigned>(CondString)
+      .Case(".f",    0)
+      .Case(".un",   1)
+      .Case(".eq",   2)
+      .Case(".ueq",  3)
+      .Case(".olt",  4)
+      .Case(".ult",  5)
+      .Case(".ole",  6)
+      .Case(".ule",  7)
+      .Case(".sf",   8)
+      .Case(".ngle", 9)
+      .Case(".seq",  10)
+      .Case(".ngl",  11)
+      .Case(".lt",   12)
+      .Case(".nge",  13)
+      .Case(".le",   14)
+      .Case(".ngt",  15)
+      .Default(-1);
+
+  return CC;
+}
+
 bool MipsAsmParser::
-ParseInstruction(StringRef Name, SMLoc NameLoc,
+parseMathOperation(StringRef Name, SMLoc NameLoc,
+                   SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
+  // split the format
+  size_t Start = Name.find('.'), Next = Name.rfind('.');
+  StringRef Format1 = Name.slice(Start, Next);
+  // and add the first format to the operands
+  Operands.push_back(MipsOperand::CreateToken(Format1, NameLoc));
+  // now for the second format
+  StringRef Format2 = Name.slice(Next, StringRef::npos);
+  Operands.push_back(MipsOperand::CreateToken(Format2, NameLoc));
+
+  // set the format for the first register
+  setFpFormat(Format1);
+
+  // Read the remaining operands.
+  if (getLexer().isNot(AsmToken::EndOfStatement)) {
+    // Read the first operand.
+    if (ParseOperand(Operands, Name)) {
+      SMLoc Loc = getLexer().getLoc();
+      Parser.EatToEndOfStatement();
+      return Error(Loc, "unexpected token in argument list");
+    }
+
+    if (getLexer().isNot(AsmToken::Comma)) {
+      SMLoc Loc = getLexer().getLoc();
+      Parser.EatToEndOfStatement();
+      return Error(Loc, "unexpected token in argument list");
+
+    }
+    Parser.Lex();  // Eat the comma.
+
+    //set the format for the first register
+    setFpFormat(Format2);
+
+    // Parse and remember the operand.
+    if (ParseOperand(Operands, Name)) {
+      SMLoc Loc = getLexer().getLoc();
+      Parser.EatToEndOfStatement();
+      return Error(Loc, "unexpected token in argument list");
+    }
+  }
+
+  if (getLexer().isNot(AsmToken::EndOfStatement)) {
+    SMLoc Loc = getLexer().getLoc();
+    Parser.EatToEndOfStatement();
+    return Error(Loc, "unexpected token in argument list");
+  }
+
+  Parser.Lex(); // Consume the EndOfStatement
+  return false;
+}
+
+bool MipsAsmParser::
+ParseInstruction(ParseInstructionInfo &Info, StringRef Name, SMLoc NameLoc,
                  SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
+  // floating point instructions: should register be treated as double?
+  if (requestsDoubleOperand(Name)) {
+    setFpFormat(FP_FORMAT_D);
+  Operands.push_back(MipsOperand::CreateToken(Name, NameLoc));
+  }
+  else {
+    setDefaultFpFormat();
+    // Create the leading tokens for the mnemonic, split by '.' characters.
+    size_t Start = 0, Next = Name.find('.');
+    StringRef Mnemonic = Name.slice(Start, Next);
+
+    Operands.push_back(MipsOperand::CreateToken(Mnemonic, NameLoc));
+
+    if (Next != StringRef::npos) {
+      // there is a format token in mnemonic
+      // StringRef Rest = Name.slice(Next, StringRef::npos);
+      size_t Dot = Name.find('.', Next+1);
+      StringRef Format = Name.slice(Next, Dot);
+      if (Dot == StringRef::npos) //only one '.' in a string, it's a format
+        Operands.push_back(MipsOperand::CreateToken(Format, NameLoc));
+      else {
+        if (Name.startswith("c.")){
+          // floating point compare, add '.' and immediate represent for cc
+          Operands.push_back(MipsOperand::CreateToken(".", NameLoc));
+          int Cc = ConvertCcString(Format);
+          if (Cc == -1) {
+            return Error(NameLoc, "Invalid conditional code");
+          }
+          SMLoc E = SMLoc::getFromPointer(
+              Parser.getTok().getLoc().getPointer() -1 );
+          Operands.push_back(MipsOperand::CreateImm(
+              MCConstantExpr::Create(Cc, getContext()), NameLoc, E));
+        } else {
+          // trunc, ceil, floor ...
+          return parseMathOperation(Name, NameLoc, Operands);
+        }
+
+        // the rest is a format
+        Format = Name.slice(Dot, StringRef::npos);
+        Operands.push_back(MipsOperand::CreateToken(Format, NameLoc));
+      }
+
+      setFpFormat(Format);
+    }
+  }
+
+  // Read the remaining operands.
+  if (getLexer().isNot(AsmToken::EndOfStatement)) {
+    // Read the first operand.
+    if (ParseOperand(Operands, Name)) {
+      SMLoc Loc = getLexer().getLoc();
+      Parser.EatToEndOfStatement();
+      return Error(Loc, "unexpected token in argument list");
+    }
+
+    while (getLexer().is(AsmToken::Comma) ) {
+      Parser.Lex();  // Eat the comma.
+
+      // Parse and remember the operand.
+      if (ParseOperand(Operands, Name)) {
+        SMLoc Loc = getLexer().getLoc();
+        Parser.EatToEndOfStatement();
+        return Error(Loc, "unexpected token in argument list");
+      }
+    }
+  }
+
+  if (getLexer().isNot(AsmToken::EndOfStatement)) {
+    SMLoc Loc = getLexer().getLoc();
+    Parser.EatToEndOfStatement();
+    return Error(Loc, "unexpected token in argument list");
+  }
+
+  Parser.Lex(); // Consume the EndOfStatement
+  return false;
+}
+
+bool MipsAsmParser::reportParseError(StringRef ErrorMsg) {
+   SMLoc Loc = getLexer().getLoc();
+   Parser.EatToEndOfStatement();
+   return Error(Loc, ErrorMsg);
+}
+
+bool MipsAsmParser::parseSetNoAtDirective() {
+  // line should look like:
+  //  .set noat
+  // set at reg to 0
+  Options.setATReg(0);
+  // eat noat
+  Parser.Lex();
+  // if this is not the end of the statement, report error
+  if (getLexer().isNot(AsmToken::EndOfStatement)) {
+    reportParseError("unexpected token in statement");
+    return false;
+  }
+  Parser.Lex(); // Consume the EndOfStatement
+  return false;
+}
+bool MipsAsmParser::parseSetAtDirective() {
+  // line can be
+  //  .set at - defaults to $1
+  // or .set at=$reg
+  getParser().Lex();
+  if (getLexer().is(AsmToken::EndOfStatement)) {
+    Options.setATReg(1);
+    Parser.Lex(); // Consume the EndOfStatement
+    return false;
+  } else if (getLexer().is(AsmToken::Equal)) {
+    getParser().Lex(); //eat '='
+    if (getLexer().isNot(AsmToken::Dollar)) {
+      reportParseError("unexpected token in statement");
+      return false;
+    }
+    Parser.Lex(); // eat '$'
+    if (getLexer().isNot(AsmToken::Integer)) {
+      reportParseError("unexpected token in statement");
+      return false;
+    }
+    const AsmToken &Reg = Parser.getTok();
+    if (!Options.setATReg(Reg.getIntVal())) {
+      reportParseError("unexpected token in statement");
+      return false;
+    }
+    getParser().Lex(); //eat reg
+
+    if (getLexer().isNot(AsmToken::EndOfStatement)) {
+      reportParseError("unexpected token in statement");
+      return false;
+     }
+    Parser.Lex(); // Consume the EndOfStatement
+    return false;
+  } else {
+    reportParseError("unexpected token in statement");
+    return false;
+  }
+}
+
+bool MipsAsmParser::parseSetReorderDirective() {
+  Parser.Lex();
+  // if this is not the end of the statement, report error
+  if (getLexer().isNot(AsmToken::EndOfStatement)) {
+    reportParseError("unexpected token in statement");
+    return false;
+  }
+  Options.setReorder();
+  Parser.Lex(); // Consume the EndOfStatement
+  return false;
+}
+
+bool MipsAsmParser::parseSetNoReorderDirective() {
+    Parser.Lex();
+    // if this is not the end of the statement, report error
+    if (getLexer().isNot(AsmToken::EndOfStatement)) {
+      reportParseError("unexpected token in statement");
+      return false;
+    }
+    Options.setNoreorder();
+    Parser.Lex(); // Consume the EndOfStatement
+    return false;
+}
+
+bool MipsAsmParser::parseSetMacroDirective() {
+  Parser.Lex();
+  // if this is not the end of the statement, report error
+  if (getLexer().isNot(AsmToken::EndOfStatement)) {
+    reportParseError("unexpected token in statement");
+    return false;
+  }
+  Options.setMacro();
+  Parser.Lex(); // Consume the EndOfStatement
+  return false;
+}
+
+bool MipsAsmParser::parseSetNoMacroDirective() {
+  Parser.Lex();
+  // if this is not the end of the statement, report error
+  if (getLexer().isNot(AsmToken::EndOfStatement)) {
+    reportParseError("`noreorder' must be set before `nomacro'");
+    return false;
+  }
+  if (Options.isReorder()) {
+    reportParseError("`noreorder' must be set before `nomacro'");
+    return false;
+  }
+  Options.setNomacro();
+  Parser.Lex(); // Consume the EndOfStatement
+  return false;
+}
+bool MipsAsmParser::parseDirectiveSet() {
+
+  // get next token
+  const AsmToken &Tok = Parser.getTok();
+
+  if (Tok.getString() == "noat") {
+    return parseSetNoAtDirective();
+  } else if (Tok.getString() == "at") {
+    return parseSetAtDirective();
+  } else if (Tok.getString() == "reorder") {
+    return parseSetReorderDirective();
+  } else if (Tok.getString() == "noreorder") {
+    return parseSetNoReorderDirective();
+  } else if (Tok.getString() == "macro") {
+    return parseSetMacroDirective();
+  } else if (Tok.getString() == "nomacro") {
+    return parseSetNoMacroDirective();
+  } else if (Tok.getString() == "nomips16") {
+    // ignore this directive for now
+    Parser.EatToEndOfStatement();
+    return false;
+  } else if (Tok.getString() == "nomicromips") {
+    // ignore this directive for now
+    Parser.EatToEndOfStatement();
+    return false;
+  }
   return true;
 }
 
-bool MipsAsmParser::
-ParseDirective(AsmToken DirectiveID) {
+bool MipsAsmParser::ParseDirective(AsmToken DirectiveID) {
+
+  if (DirectiveID.getString() == ".ent") {
+    // ignore this directive for now
+    Parser.Lex();
+    return false;
+  }
+
+  if (DirectiveID.getString() == ".end") {
+    // ignore this directive for now
+    Parser.Lex();
+    return false;
+  }
+
+  if (DirectiveID.getString() == ".frame") {
+    // ignore this directive for now
+    Parser.EatToEndOfStatement();
+    return false;
+  }
+
+  if (DirectiveID.getString() == ".set") {
+    return parseDirectiveSet();
+  }
+
+  if (DirectiveID.getString() == ".fmask") {
+    // ignore this directive for now
+    Parser.EatToEndOfStatement();
+    return false;
+  }
+
+  if (DirectiveID.getString() == ".mask") {
+    // ignore this directive for now
+    Parser.EatToEndOfStatement();
+    return false;
+  }
+
+  if (DirectiveID.getString() == ".gpword") {
+    // ignore this directive for now
+    Parser.EatToEndOfStatement();
+    return false;
+  }
+
   return true;
 }
 
@@ -64,3 +1327,7 @@ extern "C" void LLVMInitializeMipsAsmParser() {
   RegisterMCAsmParser<MipsAsmParser> A(TheMips64Target);
   RegisterMCAsmParser<MipsAsmParser> B(TheMips64elTarget);
 }
+
+#define GET_REGISTER_MATCHER
+#define GET_MATCHER_IMPLEMENTATION
+#include "MipsGenAsmMatcher.inc"
diff --git a/lib/Target/Mips/CMakeLists.txt b/lib/Target/Mips/CMakeLists.txt
index aab8a01..ef56e75 100644
--- a/lib/Target/Mips/CMakeLists.txt
+++ b/lib/Target/Mips/CMakeLists.txt
@@ -10,6 +10,8 @@ tablegen(LLVM MipsGenDAGISel.inc -gen-dag-isel)
 tablegen(LLVM MipsGenCallingConv.inc -gen-callingconv)
 tablegen(LLVM MipsGenSubtargetInfo.inc -gen-subtarget)
 tablegen(LLVM MipsGenEDInfo.inc -gen-enhanced-disassembly-info)
+tablegen(LLVM MipsGenAsmMatcher.inc -gen-asm-matcher)
+tablegen(LLVM MipsGenMCPseudoLowering.inc -gen-pseudo-lowering)
 add_public_tablegen_target(MipsCommonTableGen)
 
 add_llvm_target(MipsCodeGen
diff --git a/lib/Target/Mips/Disassembler/MipsDisassembler.cpp b/lib/Target/Mips/Disassembler/MipsDisassembler.cpp
index aa57472..82dbcc5 100644
--- a/lib/Target/Mips/Disassembler/MipsDisassembler.cpp
+++ b/lib/Target/Mips/Disassembler/MipsDisassembler.cpp
@@ -108,6 +108,11 @@ static DecodeStatus DecodeCPURegsRegisterClass(MCInst &Inst,
                                                uint64_t Address,
                                                const void *Decoder);
 
+static DecodeStatus DecodeDSPRegsRegisterClass(MCInst &Inst,
+                                               unsigned RegNo,
+                                               uint64_t Address,
+                                               const void *Decoder);
+
 static DecodeStatus DecodeFGR64RegisterClass(MCInst &Inst,
                                              unsigned RegNo,
                                              uint64_t Address,
@@ -138,6 +143,11 @@ static DecodeStatus DecodeHWRegs64RegisterClass(MCInst &Inst,
                                                 uint64_t Address,
                                                 const void *Decoder);
 
+static DecodeStatus DecodeACRegsRegisterClass(MCInst &Inst,
+                                              unsigned RegNo,
+                                              uint64_t Address,
+                                              const void *Decoder);
+
 static DecodeStatus DecodeBranchTarget(MCInst &Inst,
                                        unsigned Offset,
                                        uint64_t Address,
@@ -346,6 +356,13 @@ static DecodeStatus DecodeCPURegsRegisterClass(MCInst &Inst,
   return MCDisassembler::Success;
 }
 
+static DecodeStatus DecodeDSPRegsRegisterClass(MCInst &Inst,
+                                               unsigned RegNo,
+                                               uint64_t Address,
+                                               const void *Decoder) {
+  return DecodeCPURegsRegisterClass(Inst, RegNo, Address, Decoder);
+}
+
 static DecodeStatus DecodeFGR64RegisterClass(MCInst &Inst,
                                              unsigned RegNo,
                                              uint64_t Address,
@@ -463,6 +480,18 @@ static DecodeStatus DecodeHWRegs64RegisterClass(MCInst &Inst,
   return MCDisassembler::Success;
 }
 
+static DecodeStatus DecodeACRegsRegisterClass(MCInst &Inst,
+                                              unsigned RegNo,
+                                              uint64_t Address,
+                                              const void *Decoder) {
+  if (RegNo >= 4)
+    return MCDisassembler::Fail;
+
+  unsigned Reg = getReg(Decoder, Mips::ACRegsRegClassID, RegNo);
+  Inst.addOperand(MCOperand::CreateReg(Reg));
+  return MCDisassembler::Success;
+}
+
 static DecodeStatus DecodeBranchTarget(MCInst &Inst,
                                        unsigned Offset,
                                        uint64_t Address,
diff --git a/lib/Target/Mips/MCTargetDesc/CMakeLists.txt b/lib/Target/Mips/MCTargetDesc/CMakeLists.txt
index fa23150..be5d7e4 100644
--- a/lib/Target/Mips/MCTargetDesc/CMakeLists.txt
+++ b/lib/Target/Mips/MCTargetDesc/CMakeLists.txt
@@ -1,5 +1,6 @@
 add_llvm_library(LLVMMipsDesc
   MipsAsmBackend.cpp
+  MipsDirectObjLower.cpp
   MipsMCAsmInfo.cpp
   MipsMCCodeEmitter.cpp
   MipsMCTargetDesc.cpp
diff --git a/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp b/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp
index 18961fd..9a35bb6 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp
+++ b/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp
@@ -92,7 +92,7 @@ public:
       MCELFObjectTargetWriter::getOSABI(OSType), IsLittle, Is64Bit);
   }
 
-  /// ApplyFixup - Apply the \arg Value for given \arg Fixup into the provided
+  /// ApplyFixup - Apply the \p Value for given \p Fixup into the provided
   /// data fragment, at the offset specified by the fixup and following the
   /// fixup kind as appropriate.
   void applyFixup(const MCFixup &Fixup, char *Data, unsigned DataSize,
@@ -217,7 +217,7 @@ public:
   ///
   /// \param Inst - The instruction to relax, which may be the same
   /// as the output.
-  /// \parm Res [output] - On return, the relaxed instruction.
+  /// \param [out] Res On return, the relaxed instruction.
   void relaxInstruction(const MCInst &Inst, MCInst &Res) const {
   }
 
@@ -244,22 +244,26 @@ public:
 } // namespace
 
 // MCAsmBackend
-MCAsmBackend *llvm::createMipsAsmBackendEL32(const Target &T, StringRef TT) {
+MCAsmBackend *llvm::createMipsAsmBackendEL32(const Target &T, StringRef TT,
+                                             StringRef CPU) {
   return new MipsAsmBackend(T, Triple(TT).getOS(),
                             /*IsLittle*/true, /*Is64Bit*/false);
 }
 
-MCAsmBackend *llvm::createMipsAsmBackendEB32(const Target &T, StringRef TT) {
+MCAsmBackend *llvm::createMipsAsmBackendEB32(const Target &T, StringRef TT,
+                                             StringRef CPU) {
   return new MipsAsmBackend(T, Triple(TT).getOS(),
                             /*IsLittle*/false, /*Is64Bit*/false);
 }
 
-MCAsmBackend *llvm::createMipsAsmBackendEL64(const Target &T, StringRef TT) {
+MCAsmBackend *llvm::createMipsAsmBackendEL64(const Target &T, StringRef TT,
+                                             StringRef CPU) {
   return new MipsAsmBackend(T, Triple(TT).getOS(),
                             /*IsLittle*/true, /*Is64Bit*/true);
 }
 
-MCAsmBackend *llvm::createMipsAsmBackendEB64(const Target &T, StringRef TT) {
+MCAsmBackend *llvm::createMipsAsmBackendEB64(const Target &T, StringRef TT,
+                                             StringRef CPU) {
   return new MipsAsmBackend(T, Triple(TT).getOS(),
                             /*IsLittle*/false, /*Is64Bit*/true);
 }
diff --git a/lib/Target/Mips/MCTargetDesc/MipsBaseInfo.h b/lib/Target/Mips/MCTargetDesc/MipsBaseInfo.h
index 234455e..233214b 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsBaseInfo.h
+++ b/lib/Target/Mips/MCTargetDesc/MipsBaseInfo.h
@@ -122,14 +122,16 @@ inline static unsigned getMipsRegisterNumbering(unsigned RegEnum)
 {
   switch (RegEnum) {
   case Mips::ZERO: case Mips::ZERO_64: case Mips::F0: case Mips::D0_64:
-  case Mips::D0:
+  case Mips::D0:   case Mips::FCC0:    case Mips::AC0:
     return 0;
   case Mips::AT: case Mips::AT_64: case Mips::F1: case Mips::D1_64:
+  case Mips::AC1:
     return 1;
   case Mips::V0: case Mips::V0_64: case Mips::F2: case Mips::D2_64:
-  case Mips::D1:
+  case Mips::D1: case Mips::AC2:
     return 2;
   case Mips::V1: case Mips::V1_64: case Mips::F3: case Mips::D3_64:
+  case Mips::AC3:
     return 3;
   case Mips::A0: case Mips::A0_64: case Mips::F4: case Mips::D4_64:
   case Mips::D2:
diff --git a/lib/Target/Mips/MCTargetDesc/MipsDirectObjLower.cpp b/lib/Target/Mips/MCTargetDesc/MipsDirectObjLower.cpp
new file mode 100644
index 0000000..15c4282
--- /dev/null
+++ b/lib/Target/Mips/MCTargetDesc/MipsDirectObjLower.cpp
@@ -0,0 +1,81 @@
+//===-- MipsDirectObjLower.cpp - Mips LLVM direct object lowering -----===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains code to lower Mips MCInst records that are normally
+// left to the assembler to lower such as large shifts.
+//
+//===----------------------------------------------------------------------===//
+#include "MipsInstrInfo.h"
+#include "MCTargetDesc/MipsDirectObjLower.h"
+#include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCStreamer.h"
+
+using namespace llvm;
+
+// If the D<shift> instruction has a shift amount that is greater
+// than 31 (checked in calling routine), lower it to a D<shift>32 instruction
+void Mips::LowerLargeShift(MCInst& Inst) {
+
+  assert(Inst.getNumOperands() == 3 && "Invalid no. of operands for shift!");
+  assert(Inst.getOperand(2).isImm());
+
+  int64_t Shift = Inst.getOperand(2).getImm();
+  if (Shift <= 31)
+    return; // Do nothing
+  Shift -= 32;
+
+  // saminus32
+  Inst.getOperand(2).setImm(Shift);
+
+  switch (Inst.getOpcode()) {
+  default:
+    // Calling function is not synchronized
+    llvm_unreachable("Unexpected shift instruction");
+  case Mips::DSLL:
+    Inst.setOpcode(Mips::DSLL32);
+    return;
+  case Mips::DSRL:
+    Inst.setOpcode(Mips::DSRL32);
+    return;
+  case Mips::DSRA:
+    Inst.setOpcode(Mips::DSRA32);
+    return;
+  }
+}
+
+// Pick a DEXT or DINS instruction variant based on the pos and size operands
+void Mips::LowerDextDins(MCInst& InstIn) {
+  int Opcode = InstIn.getOpcode();
+
+  if (Opcode == Mips::DEXT)
+    assert(InstIn.getNumOperands() == 4 &&
+           "Invalid no. of machine operands for DEXT!");
+  else // Only DEXT and DINS are possible
+    assert(InstIn.getNumOperands() == 5 &&
+           "Invalid no. of machine operands for DINS!");
+
+  assert(InstIn.getOperand(2).isImm());
+  int64_t pos = InstIn.getOperand(2).getImm();
+  assert(InstIn.getOperand(3).isImm());
+  int64_t size = InstIn.getOperand(3).getImm();
+
+  if (size <= 32) {
+    if (pos < 32)  // DEXT/DINS, do nothing
+      return;
+    // DEXTU/DINSU
+    InstIn.getOperand(2).setImm(pos - 32);
+    InstIn.setOpcode((Opcode == Mips::DEXT) ? Mips::DEXTU : Mips::DINSU);
+    return;
+  }
+  // DEXTM/DINSM
+  assert(pos < 32 && "DEXT/DINS cannot have both size and pos > 32");
+  InstIn.getOperand(3).setImm(size - 32);
+  InstIn.setOpcode((Opcode == Mips::DEXT) ? Mips::DEXTM : Mips::DINSM);
+  return;
+}
diff --git a/lib/Target/Mips/MCTargetDesc/MipsDirectObjLower.h b/lib/Target/Mips/MCTargetDesc/MipsDirectObjLower.h
new file mode 100644
index 0000000..8813cc9
--- /dev/null
+++ b/lib/Target/Mips/MCTargetDesc/MipsDirectObjLower.h
@@ -0,0 +1,28 @@
+//===-- MipsDirectObjLower.h - Mips LLVM direct object lowering *- C++ -*--===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MIPSDIRECTOBJLOWER_H
+#define MIPSDIRECTOBJLOWER_H
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/Compiler.h"
+
+namespace llvm {
+  class MCInst;
+  class MCStreamer;
+
+  namespace Mips {
+  /// MipsDirectObjLower - This name space is used to lower MCInstr in cases
+  //                       where the assembler usually finishes the lowering
+  //                       such as large shifts.
+    void LowerLargeShift(MCInst &Inst);
+    void LowerDextDins(MCInst &Inst);
+  }
+}
+
+#endif
diff --git a/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp b/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp
index 8e84b3f..5d240fe 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp
+++ b/lib/Target/Mips/MCTargetDesc/MipsELFObjectWriter.cpp
@@ -34,7 +34,8 @@ namespace {
 
   class MipsELFObjectWriter : public MCELFObjectTargetWriter {
   public:
-    MipsELFObjectWriter(bool _is64Bit, uint8_t OSABI, bool _isN64);
+    MipsELFObjectWriter(bool _is64Bit, uint8_t OSABI,
+                        bool _isN64, bool IsLittleEndian);
 
     virtual ~MipsELFObjectWriter();
 
@@ -53,9 +54,9 @@ namespace {
 }
 
 MipsELFObjectWriter::MipsELFObjectWriter(bool _is64Bit, uint8_t OSABI,
-                                         bool _isN64)
+                                         bool _isN64, bool IsLittleEndian)
   : MCELFObjectTargetWriter(_is64Bit, OSABI, ELF::EM_MIPS,
-                            /*HasRelocationAddend*/ false,
+                            /*HasRelocationAddend*/ (_isN64) ? true : false,
                             /*IsN64*/ _isN64) {}
 
 MipsELFObjectWriter::~MipsELFObjectWriter() {}
@@ -274,6 +275,7 @@ MCObjectWriter *llvm::createMipsELFObjectWriter(raw_ostream &OS,
                                                 bool IsLittleEndian,
                                                 bool Is64Bit) {
   MCELFObjectTargetWriter *MOTW = new MipsELFObjectWriter(Is64Bit, OSABI,
-                                                (Is64Bit) ? true : false);
+                                                (Is64Bit) ? true : false,
+                                                IsLittleEndian);
   return createELFObjectWriter(MOTW, OS, IsLittleEndian);
 }
diff --git a/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.cpp b/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.cpp
index 8dab62d..7fbdae0 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.cpp
+++ b/lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.cpp
@@ -13,6 +13,7 @@
 //
 #define DEBUG_TYPE "mccodeemitter"
 #include "MCTargetDesc/MipsBaseInfo.h"
+#include "MCTargetDesc/MipsDirectObjLower.h"
 #include "MCTargetDesc/MipsFixupKinds.h"
 #include "MCTargetDesc/MipsMCTargetDesc.h"
 #include "llvm/ADT/APFloat.h"
@@ -29,17 +30,14 @@ using namespace llvm;
 
 namespace {
 class MipsMCCodeEmitter : public MCCodeEmitter {
-  MipsMCCodeEmitter(const MipsMCCodeEmitter &); // DO NOT IMPLEMENT
-  void operator=(const MipsMCCodeEmitter &); // DO NOT IMPLEMENT
+  MipsMCCodeEmitter(const MipsMCCodeEmitter &) LLVM_DELETED_FUNCTION;
+  void operator=(const MipsMCCodeEmitter &) LLVM_DELETED_FUNCTION;
   const MCInstrInfo &MCII;
-  const MCSubtargetInfo &STI;
-  MCContext &Ctx;
   bool IsLittleEndian;
 
 public:
-  MipsMCCodeEmitter(const MCInstrInfo &mcii, const MCSubtargetInfo &sti,
-                    MCContext &ctx, bool IsLittle) :
-            MCII(mcii), STI(sti) , Ctx(ctx), IsLittleEndian(IsLittle) {}
+  MipsMCCodeEmitter(const MCInstrInfo &mcii, bool IsLittle) :
+            MCII(mcii), IsLittleEndian(IsLittle) {}
 
   ~MipsMCCodeEmitter() {}
 
@@ -95,7 +93,7 @@ MCCodeEmitter *llvm::createMipsMCCodeEmitterEB(const MCInstrInfo &MCII,
                                                const MCSubtargetInfo &STI,
                                                MCContext &Ctx)
 {
-  return new MipsMCCodeEmitter(MCII, STI, Ctx, false);
+  return new MipsMCCodeEmitter(MCII, false);
 }
 
 MCCodeEmitter *llvm::createMipsMCCodeEmitterEL(const MCInstrInfo &MCII,
@@ -103,7 +101,7 @@ MCCodeEmitter *llvm::createMipsMCCodeEmitterEL(const MCInstrInfo &MCII,
                                                const MCSubtargetInfo &STI,
                                                MCContext &Ctx)
 {
-  return new MipsMCCodeEmitter(MCII, STI, Ctx, true);
+  return new MipsMCCodeEmitter(MCII, true);
 }
 
 /// EncodeInstruction - Emit the instruction.
@@ -112,16 +110,35 @@ void MipsMCCodeEmitter::
 EncodeInstruction(const MCInst &MI, raw_ostream &OS,
                   SmallVectorImpl<MCFixup> &Fixups) const
 {
-  uint32_t Binary = getBinaryCodeForInstr(MI, Fixups);
+
+  // Non-pseudo instructions that get changed for direct object
+  // only based on operand values.
+  // If this list of instructions get much longer we will move
+  // the check to a function call. Until then, this is more efficient.
+  MCInst TmpInst = MI;
+  switch (MI.getOpcode()) {
+  // If shift amount is >= 32 it the inst needs to be lowered further
+  case Mips::DSLL:
+  case Mips::DSRL:
+  case Mips::DSRA:
+    Mips::LowerLargeShift(TmpInst);
+    break;
+    // Double extract instruction is chosen by pos and size operands
+  case Mips::DEXT:
+  case Mips::DINS:
+    Mips::LowerDextDins(TmpInst);
+  }
+
+  uint32_t Binary = getBinaryCodeForInstr(TmpInst, Fixups);
 
   // Check for unimplemented opcodes.
-  // Unfortunately in MIPS both NOT and SLL will come in with Binary == 0
+  // Unfortunately in MIPS both NOP and SLL will come in with Binary == 0
   // so we have to special check for them.
-  unsigned Opcode = MI.getOpcode();
+  unsigned Opcode = TmpInst.getOpcode();
   if ((Opcode != Mips::NOP) && (Opcode != Mips::SLL) && !Binary)
     llvm_unreachable("unimplemented opcode in EncodeInstruction()");
 
-  const MCInstrDesc &Desc = MCII.get(MI.getOpcode());
+  const MCInstrDesc &Desc = MCII.get(TmpInst.getOpcode());
   uint64_t TSFlags = Desc.TSFlags;
 
   // Pseudo instructions don't get encoded and shouldn't be here
@@ -129,8 +146,10 @@ EncodeInstruction(const MCInst &MI, raw_ostream &OS,
   if ((TSFlags & MipsII::FormMask) == MipsII::Pseudo)
     llvm_unreachable("Pseudo opcode found in EncodeInstruction()");
 
-  // For now all instructions are 4 bytes
-  int Size = 4; // FIXME: Have Desc.getSize() return the correct value!
+  // Get byte count of instruction
+  unsigned Size = Desc.getSize();
+  if (!Size)
+    llvm_unreachable("Desc.getSize() returns 0");
 
   EmitInstruction(Binary, Size, OS);
 }
@@ -143,7 +162,11 @@ getBranchTargetOpValue(const MCInst &MI, unsigned OpNo,
                        SmallVectorImpl<MCFixup> &Fixups) const {
 
   const MCOperand &MO = MI.getOperand(OpNo);
-  assert(MO.isExpr() && "getBranchTargetOpValue expects only expressions");
+
+  // If the destination is an immediate, we have nothing to do.
+  if (MO.isImm()) return MO.getImm();
+  assert(MO.isExpr() &&
+         "getBranchTargetOpValue expects only expressions or immediates");
 
   const MCExpr *Expr = MO.getExpr();
   Fixups.push_back(MCFixup::Create(0, Expr,
@@ -159,7 +182,10 @@ getJumpTargetOpValue(const MCInst &MI, unsigned OpNo,
                      SmallVectorImpl<MCFixup> &Fixups) const {
 
   const MCOperand &MO = MI.getOperand(OpNo);
-  assert(MO.isExpr() && "getJumpTargetOpValue expects only expressions");
+  // If the destination is an immediate, we have nothing to do.
+  if (MO.isImm()) return MO.getImm();
+  assert(MO.isExpr() &&
+         "getJumpTargetOpValue expects only expressions or an immediate");
 
   const MCExpr *Expr = MO.getExpr();
   Fixups.push_back(MCFixup::Create(0, Expr,
diff --git a/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.h b/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.h
index bfcc2a2..71954a4 100644
--- a/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.h
+++ b/lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.h
@@ -42,10 +42,14 @@ MCCodeEmitter *createMipsMCCodeEmitterEL(const MCInstrInfo &MCII,
                                          const MCSubtargetInfo &STI,
                                          MCContext &Ctx);
 
-MCAsmBackend *createMipsAsmBackendEB32(const Target &T, StringRef TT);
-MCAsmBackend *createMipsAsmBackendEL32(const Target &T, StringRef TT);
-MCAsmBackend *createMipsAsmBackendEB64(const Target &T, StringRef TT);
-MCAsmBackend *createMipsAsmBackendEL64(const Target &T, StringRef TT);
+MCAsmBackend *createMipsAsmBackendEB32(const Target &T, StringRef TT,
+                                       StringRef CPU);
+MCAsmBackend *createMipsAsmBackendEL32(const Target &T, StringRef TT,
+                                       StringRef CPU);
+MCAsmBackend *createMipsAsmBackendEB64(const Target &T, StringRef TT,
+                                       StringRef CPU);
+MCAsmBackend *createMipsAsmBackendEL64(const Target &T, StringRef TT,
+                                       StringRef CPU);
 
 MCObjectWriter *createMipsELFObjectWriter(raw_ostream &OS,
                                           uint8_t OSABI,
diff --git a/lib/Target/Mips/Makefile b/lib/Target/Mips/Makefile
index 596f071..bd8c517 100644
--- a/lib/Target/Mips/Makefile
+++ b/lib/Target/Mips/Makefile
@@ -16,7 +16,9 @@ BUILT_SOURCES = MipsGenRegisterInfo.inc MipsGenInstrInfo.inc \
                 MipsGenAsmWriter.inc MipsGenCodeEmitter.inc \
                 MipsGenDAGISel.inc MipsGenCallingConv.inc \
                 MipsGenSubtargetInfo.inc MipsGenMCCodeEmitter.inc \
-                MipsGenEDInfo.inc MipsGenDisassemblerTables.inc
+                MipsGenEDInfo.inc MipsGenDisassemblerTables.inc \
+                MipsGenMCPseudoLowering.inc MipsGenAsmMatcher.inc
+
 DIRS = InstPrinter Disassembler AsmParser TargetInfo MCTargetDesc
 
 include $(LEVEL)/Makefile.common
diff --git a/lib/Target/Mips/Mips.td b/lib/Target/Mips/Mips.td
index 90f7942..90c01d5 100644
--- a/lib/Target/Mips/Mips.td
+++ b/lib/Target/Mips/Mips.td
@@ -77,6 +77,10 @@ def FeatureMips64r2    : SubtargetFeature<"mips64r2", "MipsArchVersion",
 def FeatureMips16  : SubtargetFeature<"mips16", "InMips16Mode", "true",
                                       "Mips16 mode">;
 
+def FeatureDSP : SubtargetFeature<"dsp", "HasDSP", "true", "Mips DSP ASE">;
+def FeatureDSPR2 : SubtargetFeature<"dspr2", "HasDSPR2", "true",
+                                    "Mips DSP-R2 ASE", [FeatureDSP]>;
+
 //===----------------------------------------------------------------------===//
 // Mips processors supported.
 //===----------------------------------------------------------------------===//
@@ -95,9 +99,20 @@ def MipsAsmWriter : AsmWriter {
   bit isMCAsmWriter = 1;
 }
 
+def MipsAsmParser : AsmParser {
+  let ShouldEmitMatchRegisterName = 0;
+}
+
+def MipsAsmParserVariant : AsmParserVariant {
+  int Variant = 0;
+
+  // Recognize hard coded registers.
+  string RegisterPrefix = "$";
+}
+
 def Mips : Target {
   let InstructionSet = MipsInstrInfo;
-
+  let AssemblyParsers = [MipsAsmParser];
   let AssemblyWriters = [MipsAsmWriter];
+  let AssemblyParserVariants = [MipsAsmParserVariant];
 }
-
diff --git a/lib/Target/Mips/Mips16FrameLowering.cpp b/lib/Target/Mips/Mips16FrameLowering.cpp
index 030042f..4e6b21f 100644
--- a/lib/Target/Mips/Mips16FrameLowering.cpp
+++ b/lib/Target/Mips/Mips16FrameLowering.cpp
@@ -20,7 +20,7 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetOptions.h"
 #include "llvm/Support/CommandLine.h"
 
@@ -41,6 +41,11 @@ void Mips16FrameLowering::emitPrologue(MachineFunction &MF) const {
   // Adjust stack.
   if (isInt<16>(-StackSize))
     BuildMI(MBB, MBBI, dl, TII.get(Mips::SaveRaF16)).addImm(StackSize);
+
+  if (hasFP(MF))
+    BuildMI(MBB, MBBI, dl, TII.get(Mips::MoveR3216), Mips::S0)
+      .addReg(Mips::SP);
+
 }
 
 void Mips16FrameLowering::emitEpilogue(MachineFunction &MF,
@@ -55,6 +60,10 @@ void Mips16FrameLowering::emitEpilogue(MachineFunction &MF,
   if (!StackSize)
     return;
 
+  if (hasFP(MF))
+    BuildMI(MBB, MBBI, dl, TII.get(Mips::Move32R16), Mips::SP)
+      .addReg(Mips::S0);
+
   // Adjust stack.
   if (isInt<16>(StackSize))
     // assumes stacksize multiple of 8
@@ -66,19 +75,58 @@ spillCalleeSavedRegisters(MachineBasicBlock &MBB,
                           MachineBasicBlock::iterator MI,
                           const std::vector<CalleeSavedInfo> &CSI,
                           const TargetRegisterInfo *TRI) const {
-  // FIXME: implement.
+  MachineFunction *MF = MBB.getParent();
+  MachineBasicBlock *EntryBlock = MF->begin();
+
+  //
+  // Registers RA, S0,S1 are the callee saved registers and they
+  // will be saved with the "save" instruction
+  // during emitPrologue
+  //
+  for (unsigned i = 0, e = CSI.size(); i != e; ++i) {
+    // Add the callee-saved register as live-in. Do not add if the register is
+    // RA and return address is taken, because it has already been added in
+    // method MipsTargetLowering::LowerRETURNADDR.
+    // It's killed at the spill, unless the register is RA and return address
+    // is taken.
+    unsigned Reg = CSI[i].getReg();
+    bool IsRAAndRetAddrIsTaken = (Reg == Mips::RA)
+      && MF->getFrameInfo()->isReturnAddressTaken();
+    if (!IsRAAndRetAddrIsTaken)
+      EntryBlock->addLiveIn(Reg);
+  }
+
+  return true;
+}
+
+bool Mips16FrameLowering::restoreCalleeSavedRegisters(MachineBasicBlock &MBB,
+                                          MachineBasicBlock::iterator MI,
+                                       const std::vector<CalleeSavedInfo> &CSI,
+                                       const TargetRegisterInfo *TRI) const {
+  //
+  // Registers RA,S0,S1 are the callee saved registers and they will be restored
+  // with the restore instruction during emitEpilogue.
+  // We need to override this virtual function, otherwise llvm will try and
+  // restore the registers on it's on from the stack.
+  //
+
   return true;
 }
 
 bool
 Mips16FrameLowering::hasReservedCallFrame(const MachineFunction &MF) const {
-  // FIXME: implement.
-  return true;
+  const MachineFrameInfo *MFI = MF.getFrameInfo();
+  // Reserve call frame if the size of the maximum call frame fits into 15-bit
+  // immediate field and there are no variable sized objects on the stack.
+  return isInt<15>(MFI->getMaxCallFrameSize()) && !MFI->hasVarSizedObjects();
 }
 
 void Mips16FrameLowering::
 processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
                                      RegScavenger *RS) const {
+  MF.getRegInfo().setPhysRegUsed(Mips::RA);
+  MF.getRegInfo().setPhysRegUsed(Mips::S0);
+  MF.getRegInfo().setPhysRegUsed(Mips::S1);
 }
 
 const MipsFrameLowering *
diff --git a/lib/Target/Mips/Mips16FrameLowering.h b/lib/Target/Mips/Mips16FrameLowering.h
index 25cc37b..01db71e 100644
--- a/lib/Target/Mips/Mips16FrameLowering.h
+++ b/lib/Target/Mips/Mips16FrameLowering.h
@@ -32,6 +32,11 @@ public:
                                  const std::vector<CalleeSavedInfo> &CSI,
                                  const TargetRegisterInfo *TRI) const;
 
+  bool restoreCalleeSavedRegisters(MachineBasicBlock &MBB,
+                                   MachineBasicBlock::iterator MI,
+                                   const std::vector<CalleeSavedInfo> &CSI,
+                                   const TargetRegisterInfo *TRI) const;
+
   bool hasReservedCallFrame(const MachineFunction &MF) const;
 
   void processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
diff --git a/lib/Target/Mips/Mips16InstrInfo.cpp b/lib/Target/Mips/Mips16InstrInfo.cpp
index 2bc286b..619646b 100644
--- a/lib/Target/Mips/Mips16InstrInfo.cpp
+++ b/lib/Target/Mips/Mips16InstrInfo.cpp
@@ -25,7 +25,7 @@
 using namespace llvm;
 
 Mips16InstrInfo::Mips16InstrInfo(MipsTargetMachine &tm)
-  : MipsInstrInfo(tm, /* FIXME: set mips16 unconditional br */ 0),
+  : MipsInstrInfo(tm, Mips::BimmX16),
     RI(*tm.getSubtargetImpl(), *this) {}
 
 const MipsRegisterInfo &Mips16InstrInfo::getRegisterInfo() const {
@@ -58,12 +58,22 @@ void Mips16InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
                                   MachineBasicBlock::iterator I, DebugLoc DL,
                                   unsigned DestReg, unsigned SrcReg,
                                   bool KillSrc) const {
-  unsigned Opc = 0, ZeroReg = 0;
+  unsigned Opc = 0;
+
+  if (Mips::CPU16RegsRegClass.contains(DestReg) &&
+      Mips::CPURegsRegClass.contains(SrcReg))
+    Opc = Mips::MoveR3216;
+  else if (Mips::CPURegsRegClass.contains(DestReg) &&
+           Mips::CPU16RegsRegClass.contains(SrcReg))
+    Opc = Mips::Move32R16;
+  else if ((SrcReg == Mips::HI) &&
+           (Mips::CPU16RegsRegClass.contains(DestReg)))
+    Opc = Mips::Mfhi16, SrcReg = 0;
+
+  else if ((SrcReg == Mips::LO) &&
+           (Mips::CPU16RegsRegClass.contains(DestReg)))
+    Opc = Mips::Mflo16, SrcReg = 0;
 
-  if (Mips::CPURegsRegClass.contains(DestReg)) { // Copy to CPU Reg.
-    if (Mips::CPURegsRegClass.contains(SrcReg))
-      Opc = Mips::Mov32R16;
-  }
 
   assert(Opc && "Cannot copy registers");
 
@@ -72,9 +82,6 @@ void Mips16InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
   if (DestReg)
     MIB.addReg(DestReg, RegState::Define);
 
-  if (ZeroReg)
-    MIB.addReg(ZeroReg);
-
   if (SrcReg)
     MIB.addReg(SrcReg, getKillRegState(KillSrc));
 }
@@ -84,7 +91,15 @@ storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
                     unsigned SrcReg, bool isKill, int FI,
                     const TargetRegisterClass *RC,
                     const TargetRegisterInfo *TRI) const {
-  assert(false && "Implement this function.");
+  DebugLoc DL;
+  if (I != MBB.end()) DL = I->getDebugLoc();
+  MachineMemOperand *MMO = GetMemOperand(MBB, FI, MachineMemOperand::MOStore);
+  unsigned Opc = 0;
+  if (Mips::CPU16RegsRegClass.hasSubClassEq(RC))
+    Opc = Mips::SwRxSpImmX16;
+  assert(Opc && "Register class not handled!");
+  BuildMI(MBB, I, DL, get(Opc)).addReg(SrcReg, getKillRegState(isKill))
+    .addFrameIndex(FI).addImm(0).addMemOperand(MMO);
 }
 
 void Mips16InstrInfo::
@@ -92,7 +107,16 @@ loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator I,
                      unsigned DestReg, int FI,
                      const TargetRegisterClass *RC,
                      const TargetRegisterInfo *TRI) const {
-  assert(false && "Implement this function.");
+  DebugLoc DL;
+  if (I != MBB.end()) DL = I->getDebugLoc();
+  MachineMemOperand *MMO = GetMemOperand(MBB, FI, MachineMemOperand::MOLoad);
+  unsigned Opc = 0;
+
+  if (Mips::CPU16RegsRegClass.hasSubClassEq(RC))
+    Opc = Mips::LwRxSpImmX16;
+  assert(Opc && "Register class not handled!");
+  BuildMI(MBB, I, DL, get(Opc), DestReg).addFrameIndex(FI).addImm(0)
+    .addMemOperand(MMO);
 }
 
 bool Mips16InstrInfo::expandPostRAPseudo(MachineBasicBlock::iterator MI) const {
@@ -102,7 +126,7 @@ bool Mips16InstrInfo::expandPostRAPseudo(MachineBasicBlock::iterator MI) const {
   default:
     return false;
   case Mips::RetRA16:
-    ExpandRetRA16(MBB, MI, Mips::JrRa16);
+    ExpandRetRA16(MBB, MI, Mips::JrcRa16);
     break;
   }
 
@@ -113,12 +137,55 @@ bool Mips16InstrInfo::expandPostRAPseudo(MachineBasicBlock::iterator MI) const {
 /// GetOppositeBranchOpc - Return the inverse of the specified
 /// opcode, e.g. turning BEQ to BNE.
 unsigned Mips16InstrInfo::GetOppositeBranchOpc(unsigned Opc) const {
+  switch (Opc) {
+  default:  llvm_unreachable("Illegal opcode!");
+  case Mips::BeqzRxImmX16: return Mips::BnezRxImmX16;
+  case Mips::BnezRxImmX16: return Mips::BeqzRxImmX16;
+  case Mips::BteqzT8CmpX16: return Mips::BtnezT8CmpX16;
+  case Mips::BteqzT8SltX16: return Mips::BtnezT8SltX16;
+  case Mips::BteqzT8SltiX16: return Mips::BtnezT8SltiX16;
+  case Mips::BtnezX16: return Mips::BteqzX16;
+  case Mips::BtnezT8CmpiX16: return Mips::BteqzT8CmpiX16;
+  case Mips::BtnezT8SltuX16: return Mips::BteqzT8SltuX16;
+  case Mips::BtnezT8SltiuX16: return Mips::BteqzT8SltiuX16;
+  case Mips::BteqzX16: return Mips::BtnezX16;
+  case Mips::BteqzT8CmpiX16: return Mips::BtnezT8CmpiX16;
+  case Mips::BteqzT8SltuX16: return Mips::BtnezT8SltuX16;
+  case Mips::BteqzT8SltiuX16: return Mips::BtnezT8SltiuX16;
+  case Mips::BtnezT8CmpX16: return Mips::BteqzT8CmpX16;
+  case Mips::BtnezT8SltX16: return Mips::BteqzT8SltX16;
+  case Mips::BtnezT8SltiX16: return Mips::BteqzT8SltiX16;
+  }
   assert(false && "Implement this function.");
   return 0;
 }
 
+/// Adjust SP by Amount bytes.
+void Mips16InstrInfo::adjustStackPtr(unsigned SP, int64_t Amount,
+                                     MachineBasicBlock &MBB,
+                                     MachineBasicBlock::iterator I) const {
+  DebugLoc DL = I != MBB.end() ? I->getDebugLoc() : DebugLoc();
+  if (isInt<16>(Amount)) {
+    if (Amount < 0)
+      BuildMI(MBB, I, DL, get(Mips::SaveDecSpF16)). addImm(-Amount);
+    else if (Amount > 0)
+      BuildMI(MBB, I, DL, get(Mips::RestoreIncSpF16)).addImm(Amount);
+  }
+  else
+    // not implemented for large values yet
+    assert(false && "adjust stack pointer amount exceeded");
+}
+
 unsigned Mips16InstrInfo::GetAnalyzableBrOpc(unsigned Opc) const {
-  return 0;
+  return (Opc == Mips::BeqzRxImmX16   || Opc == Mips::BimmX16  ||
+          Opc == Mips::BnezRxImmX16   || Opc == Mips::BteqzX16 ||
+          Opc == Mips::BteqzT8CmpX16  || Opc == Mips::BteqzT8CmpiX16 ||
+          Opc == Mips::BteqzT8SltX16  || Opc == Mips::BteqzT8SltuX16  ||
+          Opc == Mips::BteqzT8SltiX16 || Opc == Mips::BteqzT8SltiuX16 ||
+          Opc == Mips::BtnezX16       || Opc == Mips::BtnezT8CmpX16 ||
+          Opc == Mips::BtnezT8CmpiX16 || Opc == Mips::BtnezT8SltX16 ||
+          Opc == Mips::BtnezT8SltuX16 || Opc == Mips::BtnezT8SltiX16 ||
+          Opc == Mips::BtnezT8SltiuX16 ) ? Opc : 0;
 }
 
 void Mips16InstrInfo::ExpandRetRA16(MachineBasicBlock &MBB,
diff --git a/lib/Target/Mips/Mips16InstrInfo.h b/lib/Target/Mips/Mips16InstrInfo.h
index 260c5b6..e06ccfe 100644
--- a/lib/Target/Mips/Mips16InstrInfo.h
+++ b/lib/Target/Mips/Mips16InstrInfo.h
@@ -64,6 +64,10 @@ public:
 
   virtual unsigned GetOppositeBranchOpc(unsigned Opc) const;
 
+  /// Adjust SP by Amount bytes.
+  void adjustStackPtr(unsigned SP, int64_t Amount, MachineBasicBlock &MBB,
+                      MachineBasicBlock::iterator I) const;
+
 private:
   virtual unsigned GetAnalyzableBrOpc(unsigned Opc) const;
 
diff --git a/lib/Target/Mips/Mips16InstrInfo.td b/lib/Target/Mips/Mips16InstrInfo.td
index 94cf984..5defc75 100644
--- a/lib/Target/Mips/Mips16InstrInfo.td
+++ b/lib/Target/Mips/Mips16InstrInfo.td
@@ -10,21 +10,74 @@
 // This file describes Mips16 instructions.
 //
 //===----------------------------------------------------------------------===//
+//
+//
+// Mips Address
+//
+def addr16 :
+  ComplexPattern<iPTR, 3, "SelectAddr16", [frameindex], [SDNPWantParent]>;
 
 //
-// RRR-type instruction format
+// Address operand
+def mem16 : Operand<i32> {
+  let PrintMethod = "printMemOperand";
+  let MIOperandInfo = (ops CPU16Regs, simm16, CPU16Regs);
+  let EncoderMethod = "getMemEncoding";
+}
+
+def mem16_ea : Operand<i32> {
+  let PrintMethod = "printMemOperandEA";
+  let MIOperandInfo = (ops CPU16Regs, simm16);
+  let EncoderMethod = "getMemEncoding";
+}
+
+//
+// Compare a register and immediate and place result in CC
+// Implicit use of T8
 //
+// EXT-CCRR Instruction format
+//
+class FEXT_CCRXI16_ins<bits<5> _op, string asmstr,
+                       InstrItinClass itin>:
+  FEXT_RI16<_op, (outs CPU16Regs:$cc), (ins CPU16Regs:$rx, simm16:$imm),
+            !strconcat(asmstr, "\t$rx, $imm\n\tmove\t$cc, $$t8"), [], itin> {
+  let isCodeGenOnly=1;
+}
 
-class FRRR16_ins<bits<2> _f, string asmstr,  InstrItinClass itin> :
-  FRRR16<_f, (outs CPU16Regs:$rz), (ins CPU16Regs:$rx, CPU16Regs:$ry),
-         !strconcat(asmstr, "\t$rz, $rx, $ry"), [], itin>;
+//
+// EXT-I instruction format
+//
+class FEXT_I16_ins<bits<5> eop, string asmstr, InstrItinClass itin> :
+  FEXT_I16<eop, (outs), (ins brtarget:$imm16),
+           !strconcat(asmstr, "\t$imm16"),[], itin>;
 
 //
-// I8_MOV32R instruction format (used only by MOV32R instruction)
+// EXT-I8 instruction format
 //
-class FI8_MOV32R16_ins<string asmstr, InstrItinClass itin>:
-  FI8_MOV32R16<(outs CPURegs:$r32), (ins CPU16Regs:$rz),
-               !strconcat(asmstr,  "\t$r32, $rz"), [], itin>;
+
+class FEXT_I816_ins_base<bits<3> _func, string asmstr,
+                         string asmstr2, InstrItinClass itin>:
+  FEXT_I816<_func, (outs), (ins uimm16:$imm), !strconcat(asmstr, asmstr2),
+            [], itin>;
+
+class FEXT_I816_ins<bits<3> _func, string asmstr,
+                    InstrItinClass itin>:
+  FEXT_I816_ins_base<_func, asmstr, "\t$imm", itin>;
+
+//
+// Assembler formats in alphabetical order.
+// Natural and pseudos are mixed together.
+//
+// Compare two registers and place result in CC
+// Implicit use of T8
+//
+// CC-RR Instruction format
+//
+class FCCRR16_ins<bits<5> f, string asmstr, InstrItinClass itin> :
+  FRR16<f, (outs CPU16Regs:$cc), (ins CPU16Regs:$rx, CPU16Regs:$ry),
+        !strconcat(asmstr, "\t$rx, $ry\n\tmove\t$cc, $$t8"), [], itin> {
+  let isCodeGenOnly=1;
+}
 
 //
 // EXT-RI instruction format
@@ -42,6 +95,10 @@ class FEXT_RI16_ins<bits<5> _op, string asmstr,
 class FEXT_RI16_PC_ins<bits<5> _op, string asmstr, InstrItinClass itin>:
   FEXT_RI16_ins_base<_op, asmstr, "\t$rx, $$pc, $imm", itin>;
 
+class FEXT_RI16_B_ins<bits<5> _op, string asmstr,
+                      InstrItinClass itin>:
+  FEXT_RI16<_op, (outs), (ins  CPU16Regs:$rx, brtarget:$imm),
+            !strconcat(asmstr, "\t$rx, $imm"), [], itin>;
 
 class FEXT_2RI16_ins<bits<5> _op, string asmstr,
                      InstrItinClass itin>:
@@ -51,6 +108,104 @@ class FEXT_2RI16_ins<bits<5> _op, string asmstr,
 }
 
 
+// this has an explicit sp argument that we ignore to work around a problem
+// in the compiler
+class FEXT_RI16_SP_explicit_ins<bits<5> _op, string asmstr,
+                                InstrItinClass itin>:
+  FEXT_RI16<_op, (outs CPU16Regs:$rx), (ins CPUSPReg:$ry, simm16:$imm),
+            !strconcat(asmstr, "\t$rx, $imm ( $ry ); "), [], itin>;
+
+//
+// EXT-RRI instruction format
+//
+
+class FEXT_RRI16_mem_ins<bits<5> op, string asmstr, Operand MemOpnd,
+                         InstrItinClass itin>:
+  FEXT_RRI16<op, (outs CPU16Regs:$ry), (ins  MemOpnd:$addr),
+             !strconcat(asmstr, "\t$ry, $addr"), [], itin>;
+
+class FEXT_RRI16_mem2_ins<bits<5> op, string asmstr, Operand MemOpnd,
+                          InstrItinClass itin>:
+  FEXT_RRI16<op, (outs ), (ins  CPU16Regs:$ry, MemOpnd:$addr),
+             !strconcat(asmstr, "\t$ry, $addr"), [], itin>;
+
+//
+//
+// EXT-RRI-A instruction format
+//
+
+class FEXT_RRI_A16_mem_ins<bits<1> op, string asmstr, Operand MemOpnd,
+                           InstrItinClass itin>:
+  FEXT_RRI_A16<op, (outs CPU16Regs:$ry), (ins  MemOpnd:$addr),
+               !strconcat(asmstr, "\t$ry, $addr"), [], itin>;
+
+//
+// EXT-SHIFT instruction format
+//
+class FEXT_SHIFT16_ins<bits<2> _f, string asmstr, InstrItinClass itin>:
+  FEXT_SHIFT16<_f, (outs CPU16Regs:$rx), (ins CPU16Regs:$ry, shamt:$sa),
+               !strconcat(asmstr, "\t$rx, $ry, $sa"), [], itin>;
+
+//
+// EXT-T8I8
+//
+class FEXT_T8I816_ins<bits<3> _func, string asmstr, string asmstr2,
+                      InstrItinClass itin>:
+  FEXT_I816<_func, (outs),
+            (ins CPU16Regs:$rx, CPU16Regs:$ry, brtarget:$imm),
+            !strconcat(asmstr2, !strconcat("\t$rx, $ry\n\t",
+            !strconcat(asmstr, "\t$imm"))),[], itin> {
+  let isCodeGenOnly=1;
+}
+
+//
+// EXT-T8I8I
+//
+class FEXT_T8I8I16_ins<bits<3> _func, string asmstr, string asmstr2,
+                       InstrItinClass itin>:
+  FEXT_I816<_func, (outs),
+            (ins CPU16Regs:$rx, simm16:$imm, brtarget:$targ),
+            !strconcat(asmstr2, !strconcat("\t$rx, $imm\n\t",
+            !strconcat(asmstr, "\t$targ"))), [], itin> {
+  let isCodeGenOnly=1;
+}
+//
+
+
+//
+// I8_MOVR32 instruction format (used only by the MOVR32 instructio
+//
+class FI8_MOVR3216_ins<string asmstr, InstrItinClass itin>:
+       FI8_MOVR3216<(outs CPU16Regs:$rz), (ins CPURegs:$r32),
+       !strconcat(asmstr,  "\t$rz, $r32"), [], itin>;
+
+//
+// I8_MOV32R instruction format (used only by MOV32R instruction)
+//
+
+class FI8_MOV32R16_ins<string asmstr, InstrItinClass itin>:
+  FI8_MOV32R16<(outs CPURegs:$r32), (ins CPU16Regs:$rz),
+               !strconcat(asmstr,  "\t$r32, $rz"), [], itin>;
+
+//
+// This are pseudo formats for multiply
+// This first one can be changed to non pseudo now.
+//
+// MULT
+//
+class FMULT16_ins<string asmstr, InstrItinClass itin> :
+  MipsPseudo16<(outs), (ins CPU16Regs:$rx, CPU16Regs:$ry),
+               !strconcat(asmstr, "\t$rx, $ry"), []>;
+
+//
+// MULT-LO
+//
+class FMULT16_LO_ins<string asmstr, InstrItinClass itin> :
+  MipsPseudo16<(outs CPU16Regs:$rz), (ins CPU16Regs:$rx, CPU16Regs:$ry),
+               !strconcat(asmstr, "\t$rx, $ry\n\tmflo\t$rz"), []> {
+  let isCodeGenOnly=1;
+}
+
 //
 // RR-type instruction format
 //
@@ -60,6 +215,27 @@ class FRR16_ins<bits<5> f, string asmstr, InstrItinClass itin> :
         !strconcat(asmstr, "\t$rx, $ry"), [], itin> {
 }
 
+class FRRTR16_ins<bits<5> f, string asmstr, InstrItinClass itin> :
+  FRR16<f, (outs CPU16Regs:$rz), (ins CPU16Regs:$rx, CPU16Regs:$ry),
+        !strconcat(asmstr, "\t$rx, $ry\n\tmove\t$rz, $$t8"), [], itin> ;
+
+//
+// maybe refactor but need a $zero as a dummy first parameter
+//
+class FRR16_div_ins<bits<5> f, string asmstr, InstrItinClass itin> :
+  FRR16<f, (outs ), (ins CPU16Regs:$rx, CPU16Regs:$ry),
+        !strconcat(asmstr, "\t$$zero, $rx, $ry"), [], itin> ;
+
+class FUnaryRR16_ins<bits<5> f, string asmstr, InstrItinClass itin> :
+  FRR16<f, (outs CPU16Regs:$rx), (ins CPU16Regs:$ry),
+        !strconcat(asmstr, "\t$rx, $ry"), [], itin> ;
+
+
+class FRR16_M_ins<bits<5> f, string asmstr,
+                  InstrItinClass itin> :
+  FRR16<f, (outs CPU16Regs:$rx), (ins),
+        !strconcat(asmstr, "\t$rx"), [], itin>;
+
 class FRxRxRy16_ins<bits<5> f, string asmstr,
                     InstrItinClass itin> :
   FRR16<f, (outs CPU16Regs:$rz), (ins CPU16Regs:$rx, CPU16Regs:$ry),
@@ -74,35 +250,109 @@ class FRR16_JALRC_RA_only_ins<bits<1> nd_, bits<1> l_,
   FRR16_JALRC<nd_, l_, 1, (outs), (ins), !strconcat(asmstr, "\t $$ra"),
               [], itin> ;
 
+
+class FRR16_JALRC_ins<bits<1> nd, bits<1> l, bits<1> ra,
+                      string asmstr, InstrItinClass itin>:
+  FRR16_JALRC<nd, l, ra, (outs), (ins CPU16Regs:$rx), 
+              !strconcat(asmstr, "\t $rx"), [], itin> ;
+
 //
-// EXT-RRI instruction format
+// RRR-type instruction format
 //
 
-class FEXT_RRI16_mem_ins<bits<5> op, string asmstr, Operand MemOpnd,
-                         InstrItinClass itin>:
-  FEXT_RRI16<op, (outs CPU16Regs:$ry), (ins  MemOpnd:$addr),
-             !strconcat(asmstr, "\t$ry, $addr"), [], itin>;
+class FRRR16_ins<bits<2> _f, string asmstr,  InstrItinClass itin> :
+  FRRR16<_f, (outs CPU16Regs:$rz), (ins CPU16Regs:$rx, CPU16Regs:$ry),
+         !strconcat(asmstr, "\t$rz, $rx, $ry"), [], itin>;
 
-class FEXT_RRI16_mem2_ins<bits<5> op, string asmstr, Operand MemOpnd,
-                          InstrItinClass itin>:
-  FEXT_RRI16<op, (outs ), (ins  CPU16Regs:$ry, MemOpnd:$addr),
-             !strconcat(asmstr, "\t$ry, $addr"), [], itin>;
+//
+// These Sel patterns support the generation of conditional move
+// pseudo instructions.
+//
+// The nomenclature uses the components making up the pseudo and may
+// be a bit counter intuitive when compared with the end result we seek.
+// For example using a bqez in the example directly below results in the
+// conditional move being done if the tested register is not zero.
+// I considered in easier to check by keeping the pseudo consistent with
+// it's components but it could have been done differently.
+//
+// The simplest case is when can test and operand directly and do the
+// conditional move based on a simple mips16 conditional
+//  branch instruction.
+// for example:
+// if $op == beqz or bnez:
+//
+// $op1 $rt, .+4
+// move $rd, $rs
+//
+// if $op == beqz, then if $rt != 0, then the conditional assignment
+// $rd = $rs is done.
 
+// if $op == bnez, then if $rt == 0, then the conditional assignment
+// $rd = $rs is done.
 //
-// EXT-SHIFT instruction format
+// So this pseudo class only has one operand, i.e. op
 //
-class FEXT_SHIFT16_ins<bits<2> _f, string asmstr, InstrItinClass itin>:
-  FEXT_SHIFT16<_f, (outs CPU16Regs:$rx), (ins CPU16Regs:$ry, shamt:$sa),
-               !strconcat(asmstr, "\t$rx, $ry, $sa"), [], itin>;
+class Sel<bits<5> f1, string op, InstrItinClass itin>:
+  MipsInst16_32<(outs CPU16Regs:$rd_), (ins CPU16Regs:$rd, CPU16Regs:$rs,
+                CPU16Regs:$rt),
+                !strconcat(op, "\t$rt, .+4\n\t\n\tmove $rd, $rs"), [], itin,
+                Pseudo16> {
+  let isCodeGenOnly=1;
+  let Constraints = "$rd = $rd_";
+}
 
 //
-// Address operand
-def mem16 : Operand<i32> {
-  let PrintMethod = "printMemOperand";
-  let MIOperandInfo = (ops CPU16Regs, simm16);
-  let EncoderMethod = "getMemEncoding";
+// The next two instruction classes allow for an operand which tests
+// two operands and returns a value in register T8 and
+//then does a conditional branch based on the value of T8
+//
+
+// op2 can be cmpi or slti/sltiu
+// op1 can bteqz or btnez
+// the operands for op2 are a register and a signed constant
+//
+// $op2 $t, $imm  ;test register t and branch conditionally
+// $op1 .+4       ;op1 is a conditional branch
+// move $rd, $rs
+//
+//
+class SeliT<bits<5> f1, string op1, bits<5> f2, string op2,
+                 InstrItinClass itin>:
+  MipsInst16_32<(outs CPU16Regs:$rd_), (ins CPU16Regs:$rd, CPU16Regs:$rs,
+                                        CPU16Regs:$rl, simm16:$imm),
+                 !strconcat(op2,
+                 !strconcat("\t$rl, $imm\n\t",
+                 !strconcat(op1, "\t.+4\n\tmove $rd, $rs"))), [], itin,
+                 Pseudo16> {
+  let isCodeGenOnly=1;
+  let Constraints = "$rd = $rd_";
+}
+
+//
+// op2 can be cmp or slt/sltu
+// op1 can be bteqz or btnez
+// the operands for op2 are two registers
+// op1 is a conditional branch
+//
+//
+// $op2 $rl, $rr  ;test registers rl,rr
+// $op1 .+4       ;op2 is a conditional branch
+// move $rd, $rs
+//
+//
+class SelT<bits<5> f1, string op1, bits<5> f2, string op2,
+           InstrItinClass itin>:
+  MipsInst16_32<(outs CPU16Regs:$rd_), (ins CPU16Regs:$rd, CPU16Regs:$rs,
+                CPU16Regs:$rl, CPU16Regs:$rr),
+                !strconcat(op2,
+                !strconcat("\t$rl, $rr\n\t",
+                !strconcat(op1, "\t.+4\n\tmove $rd, $rs"))), [], itin,
+                Pseudo16> {
+  let isCodeGenOnly=1;
+  let Constraints = "$rd = $rd_";
 }
 
+
 //
 // Some general instruction class info
 //
@@ -115,6 +365,24 @@ class ArithLogic16Defs<bit isCom=0> {
   bit neverHasSideEffects = 1;
 }
 
+class branch16 {
+  bit isBranch = 1;
+  bit isTerminator = 1;
+  bit isBarrier = 1;
+}
+
+class cbranch16 {
+  bit isBranch = 1;
+  bit isTerminator = 1;
+}
+
+class MayLoad {
+  bit mayLoad = 1;
+}
+
+class MayStore {
+  bit mayStore = 1;
+}
 //
 
 // Format: ADDIU rx, immediate MIPS16e
@@ -126,6 +394,9 @@ def AddiuRxImmX16: FEXT_RI16_ins<0b01001, "addiu", IIAlu>;
 def AddiuRxRxImmX16: FEXT_2RI16_ins<0b01001, "addiu", IIAlu>,
   ArithLogic16Defs<0>;
 
+def AddiuRxRyOffMemX16:
+  FEXT_RRI_A16_mem_ins<0, "addiu", mem16_ea, IIAlu>;
+
 //
 
 // Format: ADDIU rx, pc, immediate MIPS16e
@@ -148,6 +419,87 @@ def AdduRxRyRz16: FRRR16_ins<01, "addu", IIAlu>, ArithLogic16Defs<1>;
 
 def AndRxRxRy16: FRxRxRy16_ins<0b01100, "and", IIAlu>, ArithLogic16Defs<1>;
 
+
+//
+// Format: BEQZ rx, offset MIPS16e
+// Purpose: Branch on Equal to Zero (Extended)
+// To test a GPR then do a PC-relative conditional branch.
+//
+def BeqzRxImmX16: FEXT_RI16_B_ins<0b00100, "beqz", IIAlu>, cbranch16;
+
+// Format: B offset MIPS16e
+// Purpose: Unconditional Branch
+// To do an unconditional PC-relative branch.
+//
+def BimmX16: FEXT_I16_ins<0b00010, "b", IIAlu>, branch16;
+
+//
+// Format: BNEZ rx, offset MIPS16e
+// Purpose: Branch on Not Equal to Zero (Extended)
+// To test a GPR then do a PC-relative conditional branch.
+//
+def BnezRxImmX16: FEXT_RI16_B_ins<0b00101, "bnez", IIAlu>, cbranch16;
+
+//
+// Format: BTEQZ offset MIPS16e
+// Purpose: Branch on T Equal to Zero (Extended)
+// To test special register T then do a PC-relative conditional branch.
+//
+def BteqzX16: FEXT_I816_ins<0b000, "bteqz", IIAlu>, cbranch16;
+
+def BteqzT8CmpX16: FEXT_T8I816_ins<0b000, "bteqz", "cmp", IIAlu>, cbranch16;
+
+def BteqzT8CmpiX16: FEXT_T8I8I16_ins<0b000, "bteqz", "cmpi", IIAlu>,
+  cbranch16;
+
+def BteqzT8SltX16: FEXT_T8I816_ins<0b000, "bteqz", "slt", IIAlu>, cbranch16;
+
+def BteqzT8SltuX16: FEXT_T8I816_ins<0b000, "bteqz", "sltu", IIAlu>, cbranch16;
+
+def BteqzT8SltiX16: FEXT_T8I8I16_ins<0b000, "bteqz", "slti", IIAlu>, cbranch16;
+
+def BteqzT8SltiuX16: FEXT_T8I8I16_ins<0b000, "bteqz", "sltiu", IIAlu>,
+  cbranch16;
+
+//
+// Format: BTNEZ offset MIPS16e
+// Purpose: Branch on T Not Equal to Zero (Extended)
+// To test special register T then do a PC-relative conditional branch.
+//
+def BtnezX16: FEXT_I816_ins<0b001, "btnez", IIAlu> ,cbranch16;
+
+def BtnezT8CmpX16: FEXT_T8I816_ins<0b000, "btnez", "cmp", IIAlu>, cbranch16;
+
+def BtnezT8CmpiX16: FEXT_T8I8I16_ins<0b000, "btnez", "cmpi", IIAlu>, cbranch16;
+
+def BtnezT8SltX16: FEXT_T8I816_ins<0b000, "btnez", "slt", IIAlu>, cbranch16;
+
+def BtnezT8SltuX16: FEXT_T8I816_ins<0b000, "btnez", "sltu", IIAlu>, cbranch16;
+
+def BtnezT8SltiX16: FEXT_T8I8I16_ins<0b000, "btnez", "slti", IIAlu>, cbranch16;
+
+def BtnezT8SltiuX16: FEXT_T8I8I16_ins<0b000, "btnez", "sltiu", IIAlu>,
+  cbranch16;
+
+//
+// Format: DIV rx, ry MIPS16e
+// Purpose: Divide Word
+// To divide 32-bit signed integers.
+//
+def DivRxRy16: FRR16_div_ins<0b11010, "div", IIAlu> {
+  let Defs = [HI, LO];
+}
+
+//
+// Format: DIVU rx, ry MIPS16e
+// Purpose: Divide Unsigned Word
+// To divide 32-bit unsigned integers.
+//
+def DivuRxRy16: FRR16_div_ins<0b11011, "divu", IIAlu> {
+  let Defs = [HI, LO];
+}
+
+
 //
 // Format: JR ra MIPS16e
 // Purpose: Jump Register Through Register ra
@@ -155,35 +507,56 @@ def AndRxRxRy16: FRxRxRy16_ins<0b01100, "and", IIAlu>, ArithLogic16Defs<1>;
 // address register.
 //
 
-def JrRa16: FRR16_JALRC_RA_only_ins<0, 0, "jr", IIAlu>;
+def JrRa16: FRR16_JALRC_RA_only_ins<0, 0, "jr", IIAlu> {
+  let isBranch = 1;
+  let isIndirectBranch = 1;
+  let hasDelaySlot = 1;
+  let isTerminator=1;
+  let isBarrier=1;
+}
+
+def JrcRa16: FRR16_JALRC_RA_only_ins<0, 0, "jrc", IIAlu> {
+  let isBranch = 1;
+  let isIndirectBranch = 1;
+  let isTerminator=1;
+  let isBarrier=1;
+}
 
+def JrcRx16: FRR16_JALRC_ins<1, 1, 0, "jrc", IIAlu> {
+  let isBranch = 1;
+  let isIndirectBranch = 1;
+  let isTerminator=1;
+  let isBarrier=1;
+}
 //
 // Format: LB ry, offset(rx) MIPS16e
 // Purpose: Load Byte (Extended)
 // To load a byte from memory as a signed value.
 //
-def LbRxRyOffMemX16: FEXT_RRI16_mem_ins<0b10011, "lb", mem16, IIAlu>;
+def LbRxRyOffMemX16: FEXT_RRI16_mem_ins<0b10011, "lb", mem16, IILoad>, MayLoad;
 
 //
 // Format: LBU ry, offset(rx) MIPS16e
 // Purpose: Load Byte Unsigned (Extended)
 // To load a byte from memory as a unsigned value.
 //
-def LbuRxRyOffMemX16: FEXT_RRI16_mem_ins<0b10100, "lbu", mem16, IIAlu>;
+def LbuRxRyOffMemX16:
+  FEXT_RRI16_mem_ins<0b10100, "lbu", mem16, IILoad>, MayLoad;
 
 //
 // Format: LH ry, offset(rx) MIPS16e
 // Purpose: Load Halfword signed (Extended)
 // To load a halfword from memory as a signed value.
 //
-def LhRxRyOffMemX16: FEXT_RRI16_mem_ins<0b10100, "lh", mem16, IIAlu>;
+def LhRxRyOffMemX16: FEXT_RRI16_mem_ins<0b10100, "lh", mem16, IILoad>, MayLoad;
 
 //
 // Format: LHU ry, offset(rx) MIPS16e
 // Purpose: Load Halfword unsigned (Extended)
 // To load a halfword from memory as an unsigned value.
 //
-def LhuRxRyOffMemX16: FEXT_RRI16_mem_ins<0b10100, "lhu", mem16, IIAlu>;
+def LhuRxRyOffMemX16:
+  FEXT_RRI16_mem_ins<0b10100, "lhu", mem16, IILoad>, MayLoad;
 
 //
 // Format: LI rx, immediate MIPS16e
@@ -197,28 +570,98 @@ def LiRxImmX16: FEXT_RI16_ins<0b01101, "li", IIAlu>;
 // Purpose: Load Word (Extended)
 // To load a word from memory as a signed value.
 //
-def LwRxRyOffMemX16: FEXT_RRI16_mem_ins<0b10011, "lw", mem16, IIAlu>;
+def LwRxRyOffMemX16: FEXT_RRI16_mem_ins<0b10011, "lw", mem16, IILoad>, MayLoad;
+
+// Format: LW rx, offset(sp) MIPS16e
+// Purpose: Load Word (SP-Relative, Extended)
+// To load an SP-relative word from memory as a signed value.
+//
+def LwRxSpImmX16: FEXT_RI16_SP_explicit_ins<0b10110, "lw", IILoad>, MayLoad;
 
 //
 // Format: MOVE r32, rz MIPS16e
 // Purpose: Move
 // To move the contents of a GPR to a GPR.
 //
-def Mov32R16: FI8_MOV32R16_ins<"move", IIAlu>;
+def Move32R16: FI8_MOV32R16_ins<"move", IIAlu>;
+
+//
+// Format: MOVE ry, r32 MIPS16e
+//Purpose: Move
+// To move the contents of a GPR to a GPR.
+//
+def MoveR3216: FI8_MOVR3216_ins<"move", IIAlu>;
+
+//
+// Format: MFHI rx MIPS16e
+// Purpose: Move From HI Register
+// To copy the special purpose HI register to a GPR.
+//
+def Mfhi16: FRR16_M_ins<0b10000, "mfhi", IIAlu> {
+  let Uses = [HI];
+  let neverHasSideEffects = 1;
+}
+
+//
+// Format: MFLO rx MIPS16e
+// Purpose: Move From LO Register
+// To copy the special purpose LO register to a GPR.
+//
+def Mflo16: FRR16_M_ins<0b10010, "mflo", IIAlu> {
+  let Uses = [LO];
+  let neverHasSideEffects = 1;
+}
+
+//
+// Pseudo Instruction for mult
+//
+def MultRxRy16:  FMULT16_ins<"mult",  IIAlu> {
+  let isCommutable = 1;
+  let neverHasSideEffects = 1;
+  let Defs = [HI, LO];
+}
+
+def MultuRxRy16: FMULT16_ins<"multu", IIAlu> {
+  let isCommutable = 1;
+  let neverHasSideEffects = 1;
+  let Defs = [HI, LO];
+}
+
+//
+// Format: MULT rx, ry MIPS16e
+// Purpose: Multiply Word
+// To multiply 32-bit signed integers.
+//
+def MultRxRyRz16: FMULT16_LO_ins<"mult", IIAlu> {
+  let isCommutable = 1;
+  let neverHasSideEffects = 1;
+  let Defs = [HI, LO];
+}
+
+//
+// Format: MULTU rx, ry MIPS16e
+// Purpose: Multiply Unsigned Word
+// To multiply 32-bit unsigned integers.
+//
+def MultuRxRyRz16: FMULT16_LO_ins<"multu", IIAlu> {
+  let isCommutable = 1;
+  let neverHasSideEffects = 1;
+  let Defs = [HI, LO];
+}
 
 //
 // Format: NEG rx, ry MIPS16e
 // Purpose: Negate
 // To negate an integer value.
 //
-def NegRxRy16: FRR16_ins<0b11101, "neg", IIAlu>;
+def NegRxRy16: FUnaryRR16_ins<0b11101, "neg", IIAlu>;
 
 //
 // Format: NOT rx, ry MIPS16e
 // Purpose: Not
 // To complement an integer value
 //
-def NotRxRy16: FRR16_ins<0b01111, "not", IIAlu>;
+def NotRxRy16: FUnaryRR16_ins<0b01111, "not", IIAlu>;
 
 //
 // Format: OR rx, ry MIPS16e
@@ -240,10 +683,22 @@ def OrRxRxRy16: FRxRxRy16_ins<0b01101, "or", IIAlu>, ArithLogic16Defs<1>;
 // for direct object emitter, encoding needs to be adjusted for the
 // frame size
 //
-let ra=1, s=0,s0=0,s1=0 in
+let ra=1, s=0,s0=1,s1=1 in
 def RestoreRaF16:
   FI8_SVRS16<0b1, (outs), (ins uimm16:$frame_size),
-             "restore \t$$ra, $frame_size", [], IILoad >;
+             "restore\t$$ra,  $$s0, $$s1, $frame_size", [], IILoad >, MayLoad {
+  let isCodeGenOnly = 1;
+}
+
+// Use Restore to increment SP since SP is not a Mip 16 register, this
+// is an easy way to do that which does not require a register.
+//
+let ra=0, s=0,s0=0,s1=0 in
+def RestoreIncSpF16:
+  FI8_SVRS16<0b1, (outs), (ins uimm16:$frame_size),
+             "restore\t$frame_size", [], IILoad >, MayLoad {
+  let isCodeGenOnly = 1;
+}
 
 //
 // Format: SAVE {ra,}{s0/s1/s0-1,}{framesize} (All arguments are optional)
@@ -252,24 +707,152 @@ def RestoreRaF16:
 // To set up a stack frame on entry to a subroutine,
 // saving return address and static registers, and adjusting stack
 //
-let ra=1, s=1,s0=0,s1=0 in
+let ra=1, s=1,s0=1,s1=1 in
 def SaveRaF16:
   FI8_SVRS16<0b1, (outs), (ins uimm16:$frame_size),
-             "save \t$$ra, $frame_size", [], IILoad >;
+             "save\t$$ra, $$s0, $$s1, $frame_size", [], IIStore >, MayStore {
+  let isCodeGenOnly = 1;
+}
 
 //
+// Use Save to decrement the SP by a constant since SP is not
+// a Mips16 register.
+//
+let ra=0, s=0,s0=0,s1=0 in
+def SaveDecSpF16:
+  FI8_SVRS16<0b1, (outs), (ins uimm16:$frame_size),
+             "save\t$frame_size", [], IIStore >, MayStore {
+  let isCodeGenOnly = 1;
+}
+//
 // Format: SB ry, offset(rx) MIPS16e
 // Purpose: Store Byte (Extended)
 // To store a byte to memory.
 //
-def SbRxRyOffMemX16: FEXT_RRI16_mem2_ins<0b11000, "sb", mem16, IIAlu>;
+def SbRxRyOffMemX16:
+  FEXT_RRI16_mem2_ins<0b11000, "sb", mem16, IIStore>, MayStore;
 
 //
+// The Sel(T) instructions are pseudos
+// T means that they use T8 implicitly.
+//
+//
+// Format: SelBeqZ rd, rs, rt
+// Purpose: if rt==0, do nothing
+//          else rs = rt
+//
+def SelBeqZ: Sel<0b00100, "beqz", IIAlu>;
+
+//
+// Format:  SelTBteqZCmp rd, rs, rl, rr
+// Purpose: b = Cmp rl, rr.
+//          If b==0 then do nothing.
+//          if b!=0 then rd = rs
+//
+def SelTBteqZCmp: SelT<0b000, "bteqz", 0b01010, "cmp", IIAlu>;
+
+//
+// Format:  SelTBteqZCmpi rd, rs, rl, rr
+// Purpose: b = Cmpi rl, imm.
+//          If b==0 then do nothing.
+//          if b!=0 then rd = rs
+//
+def SelTBteqZCmpi: SeliT<0b000, "bteqz", 0b01110, "cmpi", IIAlu>;
+
+//
+// Format:  SelTBteqZSlt rd, rs, rl, rr
+// Purpose: b = Slt rl, rr.
+//          If b==0 then do nothing.
+//          if b!=0 then rd = rs
+//
+def SelTBteqZSlt: SelT<0b000, "bteqz", 0b00010, "slt", IIAlu>;
+
+//
+// Format:  SelTBteqZSlti rd, rs, rl, rr
+// Purpose: b = Slti rl, imm.
+//          If b==0 then do nothing.
+//          if b!=0 then rd = rs
+//
+def SelTBteqZSlti: SeliT<0b000, "bteqz", 0b01010, "slti", IIAlu>;
+
+//
+// Format:  SelTBteqZSltu rd, rs, rl, rr
+// Purpose: b = Sltu rl, rr.
+//          If b==0 then do nothing.
+//          if b!=0 then rd = rs
+//
+def SelTBteqZSltu: SelT<0b000, "bteqz", 0b00011, "sltu", IIAlu>;
+
+//
+// Format:  SelTBteqZSltiu rd, rs, rl, rr
+// Purpose: b = Sltiu rl, imm.
+//          If b==0 then do nothing.
+//          if b!=0 then rd = rs
+//
+def SelTBteqZSltiu: SeliT<0b000, "bteqz", 0b01011, "sltiu", IIAlu>;
+
+//
+// Format: SelBnez rd, rs, rt
+// Purpose: if rt!=0, do nothing
+//          else rs = rt
+//
+def SelBneZ: Sel<0b00101, "bnez", IIAlu>;
+
+//
+// Format:  SelTBtneZCmp rd, rs, rl, rr
+// Purpose: b = Cmp rl, rr.
+//          If b!=0 then do nothing.
+//          if b0=0 then rd = rs
+//
+def SelTBtneZCmp: SelT<0b001, "btnez", 0b01010, "cmp", IIAlu>;
+
+//
+// Format:  SelTBtnezCmpi rd, rs, rl, rr
+// Purpose: b = Cmpi rl, imm.
+//          If b!=0 then do nothing.
+//          if b==0 then rd = rs
+//
+def SelTBtneZCmpi: SeliT<0b000, "btnez", 0b01110, "cmpi", IIAlu>;
+
+//
+// Format:  SelTBtneZSlt rd, rs, rl, rr
+// Purpose: b = Slt rl, rr.
+//          If b!=0 then do nothing.
+//          if b==0 then rd = rs
+//
+def SelTBtneZSlt: SelT<0b001, "btnez", 0b00010, "slt", IIAlu>;
+
+//
+// Format:  SelTBtneZSlti rd, rs, rl, rr
+// Purpose: b = Slti rl, imm.
+//          If b!=0 then do nothing.
+//          if b==0 then rd = rs
+//
+def SelTBtneZSlti: SeliT<0b001, "btnez", 0b01010, "slti", IIAlu>;
+
+//
+// Format:  SelTBtneZSltu rd, rs, rl, rr
+// Purpose: b = Sltu rl, rr.
+//          If b!=0 then do nothing.
+//          if b==0 then rd = rs
+//
+def SelTBtneZSltu: SelT<0b001, "btnez", 0b00011, "sltu", IIAlu>;
+
+//
+// Format:  SelTBtneZSltiu rd, rs, rl, rr
+// Purpose: b = Slti rl, imm.
+//          If b!=0 then do nothing.
+//          if b==0 then rd = rs
+//
+def SelTBtneZSltiu: SeliT<0b001, "btnez", 0b01011, "sltiu", IIAlu>;
+//
+//
 // Format: SH ry, offset(rx) MIPS16e
 // Purpose: Store Halfword (Extended)
 // To store a halfword to memory.
 //
-def ShRxRyOffMemX16: FEXT_RRI16_mem2_ins<0b11001, "sh", mem16, IIAlu>;
+def ShRxRyOffMemX16:
+  FEXT_RRI16_mem2_ins<0b11001, "sh", mem16, IIStore>, MayStore;
 
 //
 // Format: SLL rx, ry, sa MIPS16e
@@ -285,8 +868,40 @@ def SllX16: FEXT_SHIFT16_ins<0b00, "sll", IIAlu>;
 //
 def SllvRxRy16 : FRxRxRy16_ins<0b00100, "sllv", IIAlu>;
 
+//
+// Format: SLTI rx, immediate MIPS16e
+// Purpose: Set on Less Than Immediate (Extended)
+// To record the result of a less-than comparison with a constant.
+//
+def SltiCCRxImmX16: FEXT_CCRXI16_ins<0b01010, "slti", IIAlu>;
 
 //
+// Format: SLTIU rx, immediate MIPS16e
+// Purpose: Set on Less Than Immediate Unsigned (Extended)
+// To record the result of a less-than comparison with a constant.
+//
+def SltiuCCRxImmX16: FEXT_CCRXI16_ins<0b01011, "sltiu", IIAlu>;
+
+//
+// Format: SLT rx, ry MIPS16e
+// Purpose: Set on Less Than
+// To record the result of a less-than comparison.
+//
+def SltRxRy16: FRR16_ins<0b00010, "slt", IIAlu>;
+
+def SltCCRxRy16: FCCRR16_ins<0b00010, "slt", IIAlu>;
+
+// Format: SLTU rx, ry MIPS16e
+// Purpose: Set on Less Than Unsigned
+// To record the result of an unsigned less-than comparison.
+//
+def SltuRxRyRz16: FRRTR16_ins<0b00011, "sltu", IIAlu> {
+  let isCodeGenOnly=1;
+}
+
+
+def SltuCCRxRy16: FCCRR16_ins<0b00011, "sltu", IIAlu>;
+//
 // Format: SRAV ry, rx MIPS16e
 // Purpose: Shift Word Right Arithmetic Variable
 // To execute an arithmetic right-shift of a word by a variable
@@ -333,9 +948,18 @@ def SubuRxRyRz16: FRRR16_ins<0b11, "subu", IIAlu>, ArithLogic16Defs<0>;
 // Purpose: Store Word (Extended)
 // To store a word to memory.
 //
-def SwRxRyOffMemX16: FEXT_RRI16_mem2_ins<0b11011, "sw", mem16, IIAlu>;
+def SwRxRyOffMemX16:
+  FEXT_RRI16_mem2_ins<0b11011, "sw", mem16, IIStore>, MayStore;
 
 //
+// Format: SW rx, offset(sp) MIPS16e
+// Purpose: Store Word rx (SP-Relative)
+// To store an SP-relative word to memory.
+//
+def SwRxSpImmX16: FEXT_RI16_SP_explicit_ins<0b11010, "sw", IIStore>, MayStore;
+
+//
+//
 // Format: XOR rx, ry MIPS16e
 // Purpose: Xor
 // To do a bitwise logical XOR.
@@ -361,6 +985,7 @@ class ArithLogic16_pat<SDNode OpNode, Instruction I> :
 
 def: ArithLogic16_pat<add, AdduRxRyRz16>;
 def: ArithLogic16_pat<and, AndRxRxRy16>;
+def: ArithLogic16_pat<mul, MultRxRyRz16>;
 def: ArithLogic16_pat<or, OrRxRxRy16>;
 def: ArithLogic16_pat<sub, SubuRxRyRz16>;
 def: ArithLogic16_pat<xor, XorRxRxRy16>;
@@ -385,35 +1010,533 @@ def: shift_rotate_reg16_pat<sra, SravRxRy16>;
 def: shift_rotate_reg16_pat<srl, SrlvRxRy16>;
 
 class LoadM16_pat<PatFrag OpNode, Instruction I> :
-  Mips16Pat<(OpNode addr:$addr), (I addr:$addr)>;
+  Mips16Pat<(OpNode addr16:$addr), (I addr16:$addr)>;
 
 def: LoadM16_pat<sextloadi8, LbRxRyOffMemX16>;
 def: LoadM16_pat<zextloadi8, LbuRxRyOffMemX16>;
-def: LoadM16_pat<sextloadi16_a, LhRxRyOffMemX16>;
-def: LoadM16_pat<zextloadi16_a, LhuRxRyOffMemX16>;
-def: LoadM16_pat<load_a, LwRxRyOffMemX16>;
+def: LoadM16_pat<sextloadi16, LhRxRyOffMemX16>;
+def: LoadM16_pat<zextloadi16, LhuRxRyOffMemX16>;
+def: LoadM16_pat<load, LwRxRyOffMemX16>;
 
 class StoreM16_pat<PatFrag OpNode, Instruction I> :
-  Mips16Pat<(OpNode CPU16Regs:$r, addr:$addr), (I CPU16Regs:$r, addr:$addr)>;
+  Mips16Pat<(OpNode CPU16Regs:$r, addr16:$addr),
+            (I CPU16Regs:$r, addr16:$addr)>;
 
 def: StoreM16_pat<truncstorei8, SbRxRyOffMemX16>;
-def: StoreM16_pat<truncstorei16_a, ShRxRyOffMemX16>;
-def: StoreM16_pat<store_a, SwRxRyOffMemX16>;
+def: StoreM16_pat<truncstorei16, ShRxRyOffMemX16>;
+def: StoreM16_pat<store, SwRxRyOffMemX16>;
+
+// Unconditional branch
+class UncondBranch16_pat<SDNode OpNode, Instruction I>:
+  Mips16Pat<(OpNode bb:$imm16), (I bb:$imm16)> {
+    let Predicates = [RelocPIC, InMips16Mode];
+  }
+
+// Indirect branch
+def: Mips16Pat<
+  (brind CPU16Regs:$rs), 
+  (JrcRx16 CPU16Regs:$rs)>;  
 
 
 // Jump and Link (Call)
-let isCall=1, hasDelaySlot=1 in
+let isCall=1, hasDelaySlot=0 in
 def JumpLinkReg16:
   FRR16_JALRC<0, 0, 0, (outs), (ins CPU16Regs:$rs),
-              "jalr \t$rs", [(MipsJmpLink CPU16Regs:$rs)], IIBranch>;
+              "jalrc \t$rs", [(MipsJmpLink CPU16Regs:$rs)], IIBranch>;
 
 // Mips16 pseudos
 let isReturn=1, isTerminator=1, hasDelaySlot=1, isBarrier=1, hasCtrlDep=1,
   hasExtraSrcRegAllocReq = 1 in
 def RetRA16 : MipsPseudo16<(outs), (ins), "", [(MipsRet)]>;
 
+
+// setcc patterns
+
+class SetCC_R16<PatFrag cond_op, Instruction I>:
+  Mips16Pat<(cond_op CPU16Regs:$rx, CPU16Regs:$ry),
+            (I CPU16Regs:$rx, CPU16Regs:$ry)>;
+
+class SetCC_I16<PatFrag cond_op, PatLeaf imm_type, Instruction I>:
+  Mips16Pat<(cond_op CPU16Regs:$rx, imm_type:$imm16),
+            (I CPU16Regs:$rx, imm_type:$imm16)>;
+
+
+def: Mips16Pat<(i32  addr16:$addr),
+               (AddiuRxRyOffMemX16  addr16:$addr)>;
+
+
+// Large (>16 bit) immediate loads
+def : Mips16Pat<(i32 imm:$imm),
+                (OrRxRxRy16 (SllX16 (LiRxImmX16 (HI16 imm:$imm)), 16),
+                (LiRxImmX16 (LO16 imm:$imm)))>;
+
+// Carry MipsPatterns
+def : Mips16Pat<(subc CPU16Regs:$lhs, CPU16Regs:$rhs),
+                (SubuRxRyRz16 CPU16Regs:$lhs, CPU16Regs:$rhs)>;
+def : Mips16Pat<(addc CPU16Regs:$lhs, CPU16Regs:$rhs),
+                (AdduRxRyRz16 CPU16Regs:$lhs, CPU16Regs:$rhs)>;
+def : Mips16Pat<(addc  CPU16Regs:$src, immSExt16:$imm),
+                (AddiuRxRxImmX16 CPU16Regs:$src, imm:$imm)>;
+
+//
+// Some branch conditional patterns are not generated by llvm at this time.
+// Some are for seemingly arbitrary reasons not used: i.e. with signed number
+// comparison they are used and for unsigned a different pattern is used.
+// I am pushing upstream from the full mips16 port and it seemed that I needed
+// these earlier and the mips32 port has these but now I cannot create test
+// cases that use these patterns. While I sort this all out I will leave these
+// extra patterns commented out and if I can be sure they are really not used,
+// I will delete the code. I don't want to check the code in uncommented without
+// a valid test case. In some cases, the compiler is generating patterns with
+// setcc instead and earlier I had implemented setcc first so may have masked
+// the problem. The setcc variants are suboptimal for mips16 so I may wantto
+// figure out how to enable the brcond patterns or else possibly new
+// combinations of of brcond and setcc.
+//
+//
+// bcond-seteq
+//
+def: Mips16Pat
+  <(brcond (i32 (seteq CPU16Regs:$rx, CPU16Regs:$ry)), bb:$imm16),
+   (BteqzT8CmpX16 CPU16Regs:$rx, CPU16Regs:$ry,  bb:$imm16)
+  >;
+
+
+def: Mips16Pat
+  <(brcond (i32 (seteq CPU16Regs:$rx, immZExt16:$imm)), bb:$targ16),
+   (BteqzT8CmpiX16 CPU16Regs:$rx, immSExt16:$imm,  bb:$targ16)
+  >;
+
+def: Mips16Pat
+  <(brcond (i32 (seteq CPU16Regs:$rx, 0)), bb:$targ16),
+   (BeqzRxImmX16 CPU16Regs:$rx, bb:$targ16)
+  >;
+
+//
+// bcond-setgt (do we need to have this pair of setlt, setgt??)
+//
+def: Mips16Pat
+  <(brcond (i32 (setgt CPU16Regs:$rx, CPU16Regs:$ry)), bb:$imm16),
+   (BtnezT8SltX16 CPU16Regs:$ry, CPU16Regs:$rx,  bb:$imm16)
+  >;
+
+//
+// bcond-setge
+//
+def: Mips16Pat
+  <(brcond (i32 (setge CPU16Regs:$rx, CPU16Regs:$ry)), bb:$imm16),
+   (BteqzT8SltX16 CPU16Regs:$rx, CPU16Regs:$ry,  bb:$imm16)
+  >;
+
+//
+// never called because compiler transforms a >= k to a > (k-1)
+def: Mips16Pat
+  <(brcond (i32 (setge CPU16Regs:$rx, immSExt16:$imm)), bb:$imm16),
+   (BteqzT8SltiX16 CPU16Regs:$rx, immSExt16:$imm,  bb:$imm16)
+  >;
+
+//
+// bcond-setlt
+//
+def: Mips16Pat
+  <(brcond (i32 (setlt CPU16Regs:$rx, CPU16Regs:$ry)), bb:$imm16),
+   (BtnezT8SltX16 CPU16Regs:$rx, CPU16Regs:$ry,  bb:$imm16)
+  >;
+
+def: Mips16Pat
+  <(brcond (i32 (setlt CPU16Regs:$rx, immSExt16:$imm)), bb:$imm16),
+   (BtnezT8SltiX16 CPU16Regs:$rx, immSExt16:$imm,  bb:$imm16)
+  >;
+
+//
+// bcond-setle
+//
+def: Mips16Pat
+  <(brcond (i32 (setle CPU16Regs:$rx, CPU16Regs:$ry)), bb:$imm16),
+   (BteqzT8SltX16 CPU16Regs:$ry, CPU16Regs:$rx,  bb:$imm16)
+  >;
+
+//
+// bcond-setne
+//
+def: Mips16Pat
+  <(brcond (i32 (setne CPU16Regs:$rx, CPU16Regs:$ry)), bb:$imm16),
+   (BtnezT8CmpX16 CPU16Regs:$rx, CPU16Regs:$ry,  bb:$imm16)
+  >;
+
+def: Mips16Pat
+  <(brcond (i32 (setne CPU16Regs:$rx, immZExt16:$imm)), bb:$targ16),
+   (BtnezT8CmpiX16 CPU16Regs:$rx, immSExt16:$imm,  bb:$targ16)
+  >;
+
+def: Mips16Pat
+  <(brcond (i32 (setne CPU16Regs:$rx, 0)), bb:$targ16),
+   (BnezRxImmX16 CPU16Regs:$rx, bb:$targ16)
+  >;
+
+//
+// This needs to be there but I forget which code will generate it
+//
+def: Mips16Pat
+  <(brcond CPU16Regs:$rx, bb:$targ16),
+   (BnezRxImmX16 CPU16Regs:$rx, bb:$targ16)
+  >;
+
+//
+
+//
+// bcond-setugt
+//
+//def: Mips16Pat
+//  <(brcond (i32 (setugt CPU16Regs:$rx, CPU16Regs:$ry)), bb:$imm16),
+//   (BtnezT8SltuX16 CPU16Regs:$ry, CPU16Regs:$rx,  bb:$imm16)
+//  >;
+
+//
+// bcond-setuge
+//
+//def: Mips16Pat
+//  <(brcond (i32 (setuge CPU16Regs:$rx, CPU16Regs:$ry)), bb:$imm16),
+//   (BteqzT8SltuX16 CPU16Regs:$rx, CPU16Regs:$ry,  bb:$imm16)
+//  >;
+
+
+//
+// bcond-setult
+//
+//def: Mips16Pat
+//  <(brcond (i32 (setult CPU16Regs:$rx, CPU16Regs:$ry)), bb:$imm16),
+//   (BtnezT8SltuX16 CPU16Regs:$rx, CPU16Regs:$ry,  bb:$imm16)
+//  >;
+
+def: UncondBranch16_pat<br, BimmX16>;
+
 // Small immediates
+def: Mips16Pat<(i32 immSExt16:$in),
+               (AddiuRxRxImmX16 (Move32R16 ZERO), immSExt16:$in)>;
+
 def: Mips16Pat<(i32 immZExt16:$in), (LiRxImmX16 immZExt16:$in)>;
 
+//
+// MipsDivRem
+//
+def: Mips16Pat
+  <(MipsDivRem CPU16Regs:$rx, CPU16Regs:$ry),
+   (DivRxRy16 CPU16Regs:$rx, CPU16Regs:$ry)>;
+
+//
+// MipsDivRemU
+//
+def: Mips16Pat
+  <(MipsDivRemU CPU16Regs:$rx, CPU16Regs:$ry),
+   (DivuRxRy16 CPU16Regs:$rx, CPU16Regs:$ry)>;
+
+//  signed a,b
+//  x = (a>=b)?x:y
+//
+//  if !(a < b) x = y
+//
+def : Mips16Pat<(select (i32 (setge CPU16Regs:$a, CPU16Regs:$b)),
+                 CPU16Regs:$x, CPU16Regs:$y),
+                (SelTBteqZSlt CPU16Regs:$x, CPU16Regs:$y,
+                 CPU16Regs:$a, CPU16Regs:$b)>;
+
+//  signed a,b
+//  x = (a>b)?x:y
+//
+//  if  (b < a) x = y
+//
+def : Mips16Pat<(select (i32 (setgt CPU16Regs:$a, CPU16Regs:$b)),
+                 CPU16Regs:$x, CPU16Regs:$y),
+                (SelTBtneZSlt CPU16Regs:$x, CPU16Regs:$y,
+                 CPU16Regs:$b, CPU16Regs:$a)>;
+
+// unsigned a,b
+// x = (a>=b)?x:y
+//
+// if !(a < b) x = y;
+//
+def : Mips16Pat<
+  (select (i32 (setuge CPU16Regs:$a, CPU16Regs:$b)),
+   CPU16Regs:$x, CPU16Regs:$y),
+  (SelTBteqZSltu CPU16Regs:$x, CPU16Regs:$y,
+   CPU16Regs:$a, CPU16Regs:$b)>;
+
+//  unsigned a,b
+//  x = (a>b)?x:y
+//
+//  if (b < a) x = y
+//
+def : Mips16Pat<(select (i32 (setugt CPU16Regs:$a, CPU16Regs:$b)),
+                 CPU16Regs:$x, CPU16Regs:$y),
+                (SelTBtneZSltu CPU16Regs:$x, CPU16Regs:$y,
+                 CPU16Regs:$b, CPU16Regs:$a)>;
+
+// signed
+// x = (a >= k)?x:y
+// due to an llvm optimization, i don't think that this will ever
+// be used. This is transformed into x = (a > k-1)?x:y
+//
+//
+
+//def : Mips16Pat<
+//  (select (i32 (setge CPU16Regs:$lhs, immSExt16:$rhs)),
+//   CPU16Regs:$T, CPU16Regs:$F),
+//  (SelTBteqZSlti CPU16Regs:$T, CPU16Regs:$F,
+//   CPU16Regs:$lhs, immSExt16:$rhs)>;
+
+//def : Mips16Pat<
+//  (select (i32 (setuge CPU16Regs:$lhs, immSExt16:$rhs)),
+//   CPU16Regs:$T, CPU16Regs:$F),
+//  (SelTBteqZSltiu CPU16Regs:$T, CPU16Regs:$F,
+//   CPU16Regs:$lhs, immSExt16:$rhs)>;
+
+// signed
+// x = (a < k)?x:y
+//
+// if !(a < k) x = y;
+//
+def : Mips16Pat<
+  (select (i32 (setlt CPU16Regs:$a, immSExt16:$b)),
+   CPU16Regs:$x, CPU16Regs:$y),
+  (SelTBtneZSlti CPU16Regs:$x, CPU16Regs:$y,
+   CPU16Regs:$a, immSExt16:$b)>;
+
+
+//
+//
+// signed
+// x = (a <= b)? x : y
+//
+// if  (b < a) x = y
+//
+def : Mips16Pat<(select (i32 (setle CPU16Regs:$a, CPU16Regs:$b)),
+                 CPU16Regs:$x, CPU16Regs:$y),
+                (SelTBteqZSlt CPU16Regs:$x, CPU16Regs:$y,
+                 CPU16Regs:$b, CPU16Regs:$a)>;
+
+//
+// unnsigned
+// x = (a <= b)? x : y
+//
+// if  (b < a) x = y
+//
+def : Mips16Pat<(select (i32 (setule CPU16Regs:$a, CPU16Regs:$b)),
+                 CPU16Regs:$x, CPU16Regs:$y),
+                (SelTBteqZSltu CPU16Regs:$x, CPU16Regs:$y,
+                 CPU16Regs:$b, CPU16Regs:$a)>;
+
+//
+// signed/unsigned
+// x = (a == b)? x : y
+//
+// if (a != b) x = y
+//
+def : Mips16Pat<(select (i32 (seteq CPU16Regs:$a, CPU16Regs:$b)),
+                 CPU16Regs:$x, CPU16Regs:$y),
+                (SelTBteqZCmp CPU16Regs:$x, CPU16Regs:$y,
+                 CPU16Regs:$b, CPU16Regs:$a)>;
+
+//
+// signed/unsigned
+// x = (a == 0)? x : y
+//
+// if (a != 0) x = y
+//
+def : Mips16Pat<(select (i32 (seteq CPU16Regs:$a, 0)),
+                 CPU16Regs:$x, CPU16Regs:$y),
+                (SelBeqZ CPU16Regs:$x, CPU16Regs:$y,
+                 CPU16Regs:$a)>;
+
+
+//
+// signed/unsigned
+// x = (a == k)? x : y
+//
+// if (a != k) x = y
+//
+def : Mips16Pat<(select (i32 (seteq CPU16Regs:$a, immZExt16:$k)),
+                 CPU16Regs:$x, CPU16Regs:$y),
+                (SelTBteqZCmpi CPU16Regs:$x, CPU16Regs:$y,
+                 CPU16Regs:$a, immZExt16:$k)>;
+
+
+//
+// signed/unsigned
+// x = (a != b)? x : y
+//
+// if (a == b) x = y
+//
+//
+def : Mips16Pat<(select (i32 (setne CPU16Regs:$a, CPU16Regs:$b)),
+                 CPU16Regs:$x, CPU16Regs:$y),
+                (SelTBtneZCmp CPU16Regs:$x, CPU16Regs:$y,
+                 CPU16Regs:$b, CPU16Regs:$a)>;
+
+//
+// signed/unsigned
+// x = (a != 0)? x : y
+//
+// if (a == 0) x = y
+//
+def : Mips16Pat<(select (i32 (setne CPU16Regs:$a, 0)),
+                 CPU16Regs:$x, CPU16Regs:$y),
+                (SelBneZ CPU16Regs:$x, CPU16Regs:$y,
+                 CPU16Regs:$a)>;
+
+// signed/unsigned
+// x = (a)? x : y
+//
+// if (!a) x = y
+//
+def : Mips16Pat<(select  CPU16Regs:$a,
+                 CPU16Regs:$x, CPU16Regs:$y),
+      (SelBneZ CPU16Regs:$x, CPU16Regs:$y,
+       CPU16Regs:$a)>;
+
+
+//
+// signed/unsigned
+// x = (a != k)? x : y
+//
+// if (a == k) x = y
+//
+def : Mips16Pat<(select (i32 (setne CPU16Regs:$a, immZExt16:$k)),
+                 CPU16Regs:$x, CPU16Regs:$y),
+                (SelTBtneZCmpi CPU16Regs:$x, CPU16Regs:$y,
+                 CPU16Regs:$a, immZExt16:$k)>;
+
+//
+// When writing C code to test setxx these patterns,
+// some will be transformed into
+// other things. So we test using C code but using -O3 and -O0
+//
+// seteq
+//
+def : Mips16Pat
+  <(seteq CPU16Regs:$lhs,CPU16Regs:$rhs),
+   (SltiuCCRxImmX16 (XorRxRxRy16 CPU16Regs:$lhs, CPU16Regs:$rhs), 1)>;
+
+def : Mips16Pat
+  <(seteq CPU16Regs:$lhs, 0),
+   (SltiuCCRxImmX16 CPU16Regs:$lhs, 1)>;
+
+
+//
+// setge
+//
+
+def: Mips16Pat
+  <(setge CPU16Regs:$lhs, CPU16Regs:$rhs),
+   (XorRxRxRy16 (SltCCRxRy16 CPU16Regs:$lhs, CPU16Regs:$rhs),
+   (LiRxImmX16 1))>;
+
+//
+// For constants, llvm transforms this to:
+// x > (k -1) and then reverses the operands to use setlt. So this pattern
+// is not used now by the compiler. (Presumably checking that k-1 does not
+// overflow). The compiler never uses this at a the current time, due to
+// other optimizations.
+//
+//def: Mips16Pat
+//  <(setge CPU16Regs:$lhs, immSExt16:$rhs),
+//   (XorRxRxRy16 (SltiCCRxImmX16 CPU16Regs:$lhs, immSExt16:$rhs),
+//   (LiRxImmX16 1))>;
+
+// This catches the x >= -32768 case by transforming it to  x > -32769
+//
+def: Mips16Pat
+  <(setgt CPU16Regs:$lhs, -32769),
+   (XorRxRxRy16 (SltiCCRxImmX16 CPU16Regs:$lhs, -32768),
+   (LiRxImmX16 1))>;
+
+//
+// setgt
+//
+//
+
+def: Mips16Pat
+  <(setgt CPU16Regs:$lhs, CPU16Regs:$rhs),
+   (SltCCRxRy16 CPU16Regs:$rhs, CPU16Regs:$lhs)>;
+
+//
+// setle
+//
+def: Mips16Pat
+  <(setle CPU16Regs:$lhs, CPU16Regs:$rhs),
+   (XorRxRxRy16 (SltCCRxRy16 CPU16Regs:$rhs, CPU16Regs:$lhs), (LiRxImmX16 1))>;
+
+//
+// setlt
+//
+def: SetCC_R16<setlt, SltCCRxRy16>;
+
+def: SetCC_I16<setlt, immSExt16, SltiCCRxImmX16>;
+
+//
+// setne
+//
+def : Mips16Pat
+  <(setne CPU16Regs:$lhs,CPU16Regs:$rhs),
+   (SltuCCRxRy16 (LiRxImmX16 0),
+   (XorRxRxRy16 CPU16Regs:$lhs, CPU16Regs:$rhs))>;
+
+
+//
+// setuge
+//
+def: Mips16Pat
+  <(setuge CPU16Regs:$lhs, CPU16Regs:$rhs),
+   (XorRxRxRy16 (SltuCCRxRy16 CPU16Regs:$lhs, CPU16Regs:$rhs),
+   (LiRxImmX16 1))>;
+
+// this pattern will never be used because the compiler will transform
+// x >= k to x > (k - 1) and then use SLT
+//
+//def: Mips16Pat
+//  <(setuge CPU16Regs:$lhs, immZExt16:$rhs),
+//   (XorRxRxRy16 (SltiuCCRxImmX16 CPU16Regs:$lhs, immZExt16:$rhs),
+//   (LiRxImmX16 1))>;
+
+//
+// setugt
+//
+def: Mips16Pat
+  <(setugt CPU16Regs:$lhs, CPU16Regs:$rhs),
+   (SltuCCRxRy16 CPU16Regs:$rhs, CPU16Regs:$lhs)>;
+
+//
+// setule
+//
+def: Mips16Pat
+  <(setule CPU16Regs:$lhs, CPU16Regs:$rhs),
+   (XorRxRxRy16 (SltuCCRxRy16 CPU16Regs:$rhs, CPU16Regs:$lhs), (LiRxImmX16 1))>;
+
+//
+// setult
+//
+def: SetCC_R16<setult, SltuCCRxRy16>;
+
+def: SetCC_I16<setult, immSExt16, SltiuCCRxImmX16>;
+
 def: Mips16Pat<(add CPU16Regs:$hi, (MipsLo tglobaladdr:$lo)),
                (AddiuRxRxImmX16 CPU16Regs:$hi, tglobaladdr:$lo)>;
+
+// hi/lo relocs
+
+def : Mips16Pat<(MipsHi tglobaltlsaddr:$in), 
+                (SllX16 (LiRxImmX16 tglobaltlsaddr:$in), 16)>;
+
+// wrapper_pic
+class Wrapper16Pat<SDNode node, Instruction ADDiuOp, RegisterClass RC>:
+  Mips16Pat<(MipsWrapper RC:$gp, node:$in),
+            (ADDiuOp RC:$gp, node:$in)>;
+
+
+def : Wrapper16Pat<tglobaladdr, AddiuRxRxImmX16, CPU16Regs>;
+def : Wrapper16Pat<tglobaltlsaddr, AddiuRxRxImmX16, CPU16Regs>;
+
+def : Mips16Pat<(i32 (extloadi8   addr16:$src)),
+                (LbuRxRyOffMemX16  addr16:$src)>;
+def : Mips16Pat<(i32 (extloadi16  addr16:$src)),
+                (LhuRxRyOffMemX16  addr16:$src)>;
\ No newline at end of file
diff --git a/lib/Target/Mips/Mips16RegisterInfo.cpp b/lib/Target/Mips/Mips16RegisterInfo.cpp
index c15d1bf..d7397a3 100644
--- a/lib/Target/Mips/Mips16RegisterInfo.cpp
+++ b/lib/Target/Mips/Mips16RegisterInfo.cpp
@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "Mips16RegisterInfo.h"
+#include "Mips16InstrInfo.h"
 #include "Mips.h"
 #include "MipsAnalyzeImmediate.h"
 #include "MipsInstrInfo.h"
@@ -39,15 +40,27 @@
 using namespace llvm;
 
 Mips16RegisterInfo::Mips16RegisterInfo(const MipsSubtarget &ST,
-                                       const TargetInstrInfo &TII)
-  : MipsRegisterInfo(ST, TII) {}
+    const Mips16InstrInfo &I)
+  : MipsRegisterInfo(ST), TII(I) {}
 
 // This function eliminate ADJCALLSTACKDOWN,
 // ADJCALLSTACKUP pseudo instructions
 void Mips16RegisterInfo::
 eliminateCallFramePseudoInstr(MachineFunction &MF, MachineBasicBlock &MBB,
                               MachineBasicBlock::iterator I) const {
-  // Simply discard ADJCALLSTACKDOWN, ADJCALLSTACKUP instructions.
+  const TargetFrameLowering *TFI = MF.getTarget().getFrameLowering();
+
+  if (!TFI->hasReservedCallFrame(MF)) {
+    int64_t Amount = I->getOperand(0).getImm();
+
+    if (I->getOpcode() == Mips::ADJCALLSTACKDOWN)
+      Amount = -Amount;
+
+    const Mips16InstrInfo *II = static_cast<const Mips16InstrInfo*>(&TII);
+
+    II->adjustStackPtr(Mips::SP, Amount, MBB, I);
+  }
+
   MBB.erase(I);
 }
 
@@ -55,57 +68,60 @@ void Mips16RegisterInfo::eliminateFI(MachineBasicBlock::iterator II,
                                      unsigned OpNo, int FrameIndex,
                                      uint64_t StackSize,
                                      int64_t SPOffset) const {
-      MachineInstr &MI = *II;
-      MachineFunction &MF = *MI.getParent()->getParent();
-      MachineFrameInfo *MFI = MF.getFrameInfo();
-      MipsFunctionInfo *MipsFI = MF.getInfo<MipsFunctionInfo>();
-
-      const std::vector<CalleeSavedInfo> &CSI = MFI->getCalleeSavedInfo();
-      int MinCSFI = 0;
-      int MaxCSFI = -1;
-
-      if (CSI.size()) {
-        MinCSFI = CSI[0].getFrameIdx();
-        MaxCSFI = CSI[CSI.size() - 1].getFrameIdx();
-      }
-
-      // The following stack frame objects are always
-      // referenced relative to $sp:
-      //  1. Outgoing arguments.
-      //  2. Pointer to dynamically allocated stack space.
-      //  3. Locations for callee-saved registers.
-      // Everything else is referenced relative to whatever register
-      // getFrameRegister() returns.
-      unsigned FrameReg;
-
-      if (MipsFI->isOutArgFI(FrameIndex) ||
-         (FrameIndex >= MinCSFI && FrameIndex <= MaxCSFI))
-        FrameReg = Subtarget.isABI_N64() ? Mips::SP_64 : Mips::SP;
+  MachineInstr &MI = *II;
+  MachineFunction &MF = *MI.getParent()->getParent();
+  MachineFrameInfo *MFI = MF.getFrameInfo();
+
+  const std::vector<CalleeSavedInfo> &CSI = MFI->getCalleeSavedInfo();
+  int MinCSFI = 0;
+  int MaxCSFI = -1;
+
+  if (CSI.size()) {
+    MinCSFI = CSI[0].getFrameIdx();
+    MaxCSFI = CSI[CSI.size() - 1].getFrameIdx();
+  }
+
+  // The following stack frame objects are always
+  // referenced relative to $sp:
+  //  1. Outgoing arguments.
+  //  2. Pointer to dynamically allocated stack space.
+  //  3. Locations for callee-saved registers.
+  // Everything else is referenced relative to whatever register
+  // getFrameRegister() returns.
+  unsigned FrameReg;
+
+  if (FrameIndex >= MinCSFI && FrameIndex <= MaxCSFI)
+    FrameReg = Mips::SP;
+  else {
+    const TargetFrameLowering *TFI = MF.getTarget().getFrameLowering();
+    if (TFI->hasFP(MF)) {
+      FrameReg = Mips::S0;
+    }
+    else {
+      if ((MI.getNumOperands()> OpNo+2) && MI.getOperand(OpNo+2).isReg())
+        FrameReg = MI.getOperand(OpNo+2).getReg();
       else
-        FrameReg = getFrameRegister(MF);
-
-      // Calculate final offset.
-      // - There is no need to change the offset if the frame object
-      //   is one of the
-      //   following: an outgoing argument, pointer to a dynamically allocated
-      //   stack space or a $gp restore location,
-      // - If the frame object is any of the following,
-      //   its offset must be adjusted
-      //   by adding the size of the stack:
-      //   incoming argument, callee-saved register location or local variable.
-      int64_t Offset;
-
-      if (MipsFI->isOutArgFI(FrameIndex))
-        Offset = SPOffset;
-      else
-        Offset = SPOffset + (int64_t)StackSize;
-
-      Offset    += MI.getOperand(OpNo + 1).getImm();
-
-      DEBUG(errs() << "Offset     : " << Offset << "\n" << "<--------->\n");
-
-      MI.getOperand(OpNo).ChangeToRegister(FrameReg, false);
-      MI.getOperand(OpNo + 1).ChangeToImmediate(Offset);
+        FrameReg = Mips::SP;
+    }
+  }
+  // Calculate final offset.
+  // - There is no need to change the offset if the frame object
+  //   is one of the
+  //   following: an outgoing argument, pointer to a dynamically allocated
+  //   stack space or a $gp restore location,
+  // - If the frame object is any of the following,
+  //   its offset must be adjusted
+  //   by adding the size of the stack:
+  //   incoming argument, callee-saved register location or local variable.
+  int64_t Offset;
+  Offset = SPOffset + (int64_t)StackSize;
+  Offset += MI.getOperand(OpNo + 1).getImm();
+
+
+  DEBUG(errs() << "Offset     : " << Offset << "\n" << "<--------->\n");
+
+  MI.getOperand(OpNo).ChangeToRegister(FrameReg, false);
+  MI.getOperand(OpNo + 1).ChangeToImmediate(Offset);
 
 
 }
diff --git a/lib/Target/Mips/Mips16RegisterInfo.h b/lib/Target/Mips/Mips16RegisterInfo.h
index 3f4b3a7..153def2 100644
--- a/lib/Target/Mips/Mips16RegisterInfo.h
+++ b/lib/Target/Mips/Mips16RegisterInfo.h
@@ -17,11 +17,12 @@
 #include "MipsRegisterInfo.h"
 
 namespace llvm {
+class Mips16InstrInfo;
 
 class Mips16RegisterInfo : public MipsRegisterInfo {
+  const Mips16InstrInfo &TII;
 public:
-  Mips16RegisterInfo(const MipsSubtarget &Subtarget,
-                     const TargetInstrInfo &TII);
+  Mips16RegisterInfo(const MipsSubtarget &Subtarget, const Mips16InstrInfo &TII);
 
   void eliminateCallFramePseudoInstr(MachineFunction &MF,
                                      MachineBasicBlock &MBB,
diff --git a/lib/Target/Mips/Mips64InstrInfo.td b/lib/Target/Mips/Mips64InstrInfo.td
index 20fc178..a611168 100644
--- a/lib/Target/Mips/Mips64InstrInfo.td
+++ b/lib/Target/Mips/Mips64InstrInfo.td
@@ -83,8 +83,10 @@ let usesCustomInserter = 1, Predicates = [HasMips64, HasStandardEncoding],
 //===----------------------------------------------------------------------===//
 let DecoderNamespace = "Mips64" in {
 /// Arithmetic Instructions (ALU Immediate)
-def DADDiu   : ArithLogicI<0x19, "daddiu", add, simm16_64, immSExt16,
+def DADDi    : ArithOverflowI<0x18, "daddi", add, simm16_64, immSExt16,
                            CPU64Regs>;
+def DADDiu   : ArithLogicI<0x19, "daddiu", add, simm16_64, immSExt16,
+                           CPU64Regs>, IsAsCheapAsAMove;
 def DANDi    : ArithLogicI<0x0c, "andi", and, uimm16_64, immZExt16, CPU64Regs>;
 def SLTi64   : SetCC_I<0x0a, "slti", setlt, simm16_64, immSExt16, CPU64Regs>;
 def SLTiu64  : SetCC_I<0x0b, "sltiu", setult, simm16_64, immSExt16, CPU64Regs>;
@@ -93,6 +95,7 @@ def XORi64   : ArithLogicI<0x0e, "xori", xor, uimm16_64, immZExt16, CPU64Regs>;
 def LUi64    : LoadUpper<0x0f, "lui", CPU64Regs, uimm16_64>;
 
 /// Arithmetic Instructions (3-Operand, R-Type)
+def DADD     : ArithOverflowR<0x00, 0x2C, "dadd", IIAlu, CPU64Regs, 1>;
 def DADDu    : ArithLogicR<0x00, 0x2d, "daddu", add, IIAlu, CPU64Regs, 1>;
 def DSUBu    : ArithLogicR<0x00, 0x2f, "dsubu", sub, IIAlu, CPU64Regs>;
 def SLT64    : SetCC_R<0x00, 0x2a, "slt", setlt, CPU64Regs>;
@@ -110,9 +113,9 @@ def DSLLV    : shift_rotate_reg<0x14, 0x00, "dsllv", shl, CPU64Regs>;
 def DSRLV    : shift_rotate_reg<0x16, 0x00, "dsrlv", srl, CPU64Regs>;
 def DSRAV    : shift_rotate_reg<0x17, 0x00, "dsrav", sra, CPU64Regs>;
 let Pattern = []<dag> in {
-def DSLL32   : shift_rotate_imm64<0x3c, 0x00, "dsll32", shl>;
-def DSRL32   : shift_rotate_imm64<0x3e, 0x00, "dsrl32", srl>;
-def DSRA32   : shift_rotate_imm64<0x3f, 0x00, "dsra32", sra>;
+  def DSLL32   : shift_rotate_imm64<0x3c, 0x00, "dsll32", shl>;
+  def DSRL32   : shift_rotate_imm64<0x3e, 0x00, "dsrl32", srl>;
+  def DSRA32   : shift_rotate_imm64<0x3f, 0x00, "dsra32", sra>;
 }
 }
 // Rotate Instructions
@@ -127,24 +130,15 @@ let DecoderNamespace = "Mips64" in {
 ///  aligned
 defm LB64    : LoadM64<0x20, "lb",  sextloadi8>;
 defm LBu64   : LoadM64<0x24, "lbu", zextloadi8>;
-defm LH64    : LoadM64<0x21, "lh",  sextloadi16_a>;
-defm LHu64   : LoadM64<0x25, "lhu", zextloadi16_a>;
-defm LW64    : LoadM64<0x23, "lw",  sextloadi32_a>;
-defm LWu64   : LoadM64<0x27, "lwu", zextloadi32_a>;
+defm LH64    : LoadM64<0x21, "lh",  sextloadi16>;
+defm LHu64   : LoadM64<0x25, "lhu", zextloadi16>;
+defm LW64    : LoadM64<0x23, "lw",  sextloadi32>;
+defm LWu64   : LoadM64<0x27, "lwu", zextloadi32>;
 defm SB64    : StoreM64<0x28, "sb", truncstorei8>;
-defm SH64    : StoreM64<0x29, "sh", truncstorei16_a>;
-defm SW64    : StoreM64<0x2b, "sw", truncstorei32_a>;
-defm LD      : LoadM64<0x37, "ld",  load_a>;
-defm SD      : StoreM64<0x3f, "sd", store_a>;
-
-///  unaligned
-defm ULH64     : LoadM64<0x21, "ulh",  sextloadi16_u, 1>;
-defm ULHu64    : LoadM64<0x25, "ulhu", zextloadi16_u, 1>;
-defm ULW64     : LoadM64<0x23, "ulw",  sextloadi32_u, 1>;
-defm USH64     : StoreM64<0x29, "ush", truncstorei16_u, 1>;
-defm USW64     : StoreM64<0x2b, "usw", truncstorei32_u, 1>;
-defm ULD       : LoadM64<0x37, "uld",  load_u, 1>;
-defm USD       : StoreM64<0x3f, "usd", store_u, 1>;
+defm SH64    : StoreM64<0x29, "sh", truncstorei16>;
+defm SW64    : StoreM64<0x2b, "sw", truncstorei32>;
+defm LD      : LoadM64<0x37, "ld",  load>;
+defm SD      : StoreM64<0x3f, "sd", store>;
 
 /// load/store left/right
 let isCodeGenOnly = 1 in {
@@ -183,6 +177,7 @@ def BLTZ64 : CBranchZero<0x01, 0, "bltz", setlt, CPU64Regs>;
 }
 let DecoderNamespace = "Mips64" in
 def JALR64 : JumpLinkReg<0x00, 0x09, "jalr", CPU64Regs>;
+def TAILCALL64_R : JumpFR<CPU64Regs, MipsTailCall>, IsTailCall;
 
 let DecoderNamespace = "Mips64" in {
 /// Multiply and Divide Instructions.
@@ -217,7 +212,15 @@ let DecoderNamespace = "Mips64" in {
 def RDHWR64 : ReadHardware<CPU64Regs, HWRegs64>;
 
 def DEXT : ExtBase<3, "dext", CPU64Regs>;
+let Pattern = []<dag> in {
+  def DEXTU : ExtBase<2, "dextu", CPU64Regs>;
+  def DEXTM : ExtBase<1, "dextm", CPU64Regs>;
+}
 def DINS : InsBase<7, "dins", CPU64Regs>;
+let Pattern = []<dag> in {
+  def DINSU : InsBase<6, "dinsu", CPU64Regs>;
+  def DINSM : InsBase<5, "dinsm", CPU64Regs>;
+}
 
 let isCodeGenOnly = 1, rs = 0, shamt = 0 in {
   def DSLL64_32 : FR<0x00, 0x3c, (outs CPU64Regs:$rd), (ins CPURegs:$rt),
@@ -236,21 +239,14 @@ let isCodeGenOnly = 1, rs = 0, shamt = 0 in {
 let Predicates = [NotN64, HasStandardEncoding] in {
   def : MipsPat<(i64 (extloadi1  addr:$src)), (LB64 addr:$src)>;
   def : MipsPat<(i64 (extloadi8  addr:$src)), (LB64 addr:$src)>;
-  def : MipsPat<(i64 (extloadi16_a addr:$src)), (LH64 addr:$src)>;
-  def : MipsPat<(i64 (extloadi16_u addr:$src)), (ULH64 addr:$src)>;
-  def : MipsPat<(i64 (extloadi32_a addr:$src)), (LW64 addr:$src)>;
-  def : MipsPat<(i64 (extloadi32_u addr:$src)), (ULW64 addr:$src)>;
-  def : MipsPat<(zextloadi32_u addr:$a), (DSRL (DSLL (ULW64 addr:$a), 32), 32)>;
+  def : MipsPat<(i64 (extloadi16 addr:$src)), (LH64 addr:$src)>;
+  def : MipsPat<(i64 (extloadi32 addr:$src)), (LW64 addr:$src)>;
 }
 let Predicates = [IsN64, HasStandardEncoding] in {
   def : MipsPat<(i64 (extloadi1  addr:$src)), (LB64_P8 addr:$src)>;
   def : MipsPat<(i64 (extloadi8  addr:$src)), (LB64_P8 addr:$src)>;
-  def : MipsPat<(i64 (extloadi16_a addr:$src)), (LH64_P8 addr:$src)>;
-  def : MipsPat<(i64 (extloadi16_u addr:$src)), (ULH64_P8 addr:$src)>;
-  def : MipsPat<(i64 (extloadi32_a addr:$src)), (LW64_P8 addr:$src)>;
-  def : MipsPat<(i64 (extloadi32_u addr:$src)), (ULW64_P8 addr:$src)>;
-  def : MipsPat<(zextloadi32_u addr:$a),
-                (DSRL (DSLL (ULW64_P8 addr:$a), 32), 32)>;
+  def : MipsPat<(i64 (extloadi16 addr:$src)), (LH64_P8 addr:$src)>;
+  def : MipsPat<(i64 (extloadi32 addr:$src)), (LW64_P8 addr:$src)>;
 }
 
 // hi/lo relocs
@@ -315,3 +311,38 @@ def : MipsPat<(i64 (sext_inreg CPU64Regs:$src, i32)),
 
 // bswap MipsPattern
 def : MipsPat<(bswap CPU64Regs:$rt), (DSHD (DSBH CPU64Regs:$rt))>;
+
+//===----------------------------------------------------------------------===//
+// Instruction aliases
+//===----------------------------------------------------------------------===//
+def : InstAlias<"move $dst,$src", (DADD CPU64Regs:$dst,CPU64Regs:$src,ZERO_64)>;
+
+/// Move between CPU and coprocessor registers
+let DecoderNamespace = "Mips64" in {
+def MFC0_3OP64  : MFC3OP<0x10, 0, (outs CPU64Regs:$rt), 
+                       (ins CPU64Regs:$rd, uimm16:$sel),"mfc0\t$rt, $rd, $sel">;
+def MTC0_3OP64  : MFC3OP<0x10, 4, (outs CPU64Regs:$rd, uimm16:$sel),
+                       (ins CPU64Regs:$rt),"mtc0\t$rt, $rd, $sel">;
+def MFC2_3OP64  : MFC3OP<0x12, 0, (outs CPU64Regs:$rt),
+                       (ins CPU64Regs:$rd, uimm16:$sel),"mfc2\t$rt, $rd, $sel">;
+def MTC2_3OP64  : MFC3OP<0x12, 4, (outs CPU64Regs:$rd, uimm16:$sel),
+                       (ins CPU64Regs:$rt),"mtc2\t$rt, $rd, $sel">;
+def DMFC0_3OP64  : MFC3OP<0x10, 1, (outs CPU64Regs:$rt), 
+                       (ins CPU64Regs:$rd, uimm16:$sel),"dmfc0\t$rt, $rd, $sel">;
+def DMTC0_3OP64  : MFC3OP<0x10, 5, (outs CPU64Regs:$rd, uimm16:$sel),
+                       (ins CPU64Regs:$rt),"dmtc0\t$rt, $rd, $sel">;
+def DMFC2_3OP64  : MFC3OP<0x12, 1, (outs CPU64Regs:$rt),
+                       (ins CPU64Regs:$rd, uimm16:$sel),"dmfc2\t$rt, $rd, $sel">;
+def DMTC2_3OP64  : MFC3OP<0x12, 5, (outs CPU64Regs:$rd, uimm16:$sel),
+                       (ins CPU64Regs:$rt),"dmtc2\t$rt, $rd, $sel">;
+}
+// Two operand (implicit 0 selector) versions:
+def : InstAlias<"mfc0 $rt, $rd", (MFC0_3OP64 CPU64Regs:$rt, CPU64Regs:$rd, 0)>;
+def : InstAlias<"mtc0 $rt, $rd", (MTC0_3OP64 CPU64Regs:$rd, 0, CPU64Regs:$rt)>;
+def : InstAlias<"mfc2 $rt, $rd", (MFC2_3OP64 CPU64Regs:$rt, CPU64Regs:$rd, 0)>;
+def : InstAlias<"mtc2 $rt, $rd", (MTC2_3OP64 CPU64Regs:$rd, 0, CPU64Regs:$rt)>;
+def : InstAlias<"dmfc0 $rt, $rd", (DMFC0_3OP64 CPU64Regs:$rt, CPU64Regs:$rd, 0)>;
+def : InstAlias<"dmtc0 $rt, $rd", (DMTC0_3OP64 CPU64Regs:$rd, 0, CPU64Regs:$rt)>;
+def : InstAlias<"dmfc2 $rt, $rd", (DMFC2_3OP64 CPU64Regs:$rt, CPU64Regs:$rd, 0)>;
+def : InstAlias<"dmtc2 $rt, $rd", (DMTC2_3OP64 CPU64Regs:$rd, 0, CPU64Regs:$rt)>;
+
diff --git a/lib/Target/Mips/MipsAnalyzeImmediate.cpp b/lib/Target/Mips/MipsAnalyzeImmediate.cpp
index dc8fbd0..99b163e 100644
--- a/lib/Target/Mips/MipsAnalyzeImmediate.cpp
+++ b/lib/Target/Mips/MipsAnalyzeImmediate.cpp
@@ -91,7 +91,7 @@ void MipsAnalyzeImmediate::ReplaceADDiuSLLWithLUi(InstSeq &Seq) {
 
   // Sign-extend and shift operand of ADDiu and see if it still fits in 16-bit.
   int64_t Imm = SignExtend64<16>(Seq[0].ImmOpnd);
-  int64_t ShiftedImm = Imm << (Seq[1].ImmOpnd - 16);
+  int64_t ShiftedImm = (uint64_t)Imm << (Seq[1].ImmOpnd - 16);
 
   if (!isInt<16>(ShiftedImm))
     return;
diff --git a/lib/Target/Mips/MipsAsmPrinter.cpp b/lib/Target/Mips/MipsAsmPrinter.cpp
index 00ff754..bf2818d 100644
--- a/lib/Target/Mips/MipsAsmPrinter.cpp
+++ b/lib/Target/Mips/MipsAsmPrinter.cpp
@@ -37,7 +37,7 @@
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/TargetRegistry.h"
 #include "llvm/Target/Mangler.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetLoweringObjectFile.h"
 #include "llvm/Target/TargetOptions.h"
 
@@ -49,6 +49,13 @@ bool MipsAsmPrinter::runOnMachineFunction(MachineFunction &MF) {
   return true;
 }
 
+bool MipsAsmPrinter::lowerOperand(const MachineOperand &MO, MCOperand &MCOp) {
+  MCOp = MCInstLowering.LowerOperand(MO);
+  return MCOp.isValid();
+}
+
+#include "MipsGenMCPseudoLowering.inc"
+
 void MipsAsmPrinter::EmitInstruction(const MachineInstr *MI) {
   if (MI->isDebugValue()) {
     SmallString<128> Str;
@@ -58,24 +65,9 @@ void MipsAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     return;
   }
 
-  // Direct object specific instruction lowering
-  if (!OutStreamer.hasRawTextSupport())
-    switch (MI->getOpcode()) {
-    case Mips::DSLL:
-    case Mips::DSRL:
-    case Mips::DSRA:
-      assert(MI->getNumOperands() == 3 &&
-             "Invalid no. of machine operands for shift!");
-      assert(MI->getOperand(2).isImm());
-      int64_t Shift = MI->getOperand(2).getImm();
-      if (Shift > 31) {
-        MCInst TmpInst0;
-        MCInstLowering.LowerLargeShift(MI, TmpInst0, Shift - 32);
-        OutStreamer.EmitInstruction(TmpInst0);
-        return;
-      }
-      break;
-    }
+  // Do any auto-generated pseudo lowerings.
+  if (emitPseudoExpansionLowering(OutStreamer, MI))
+    return;
 
   MachineBasicBlock::const_instr_iterator I = MI;
   MachineBasicBlock::const_instr_iterator E = MI->getParent()->instr_end();
@@ -83,8 +75,9 @@ void MipsAsmPrinter::EmitInstruction(const MachineInstr *MI) {
   do {
     MCInst TmpInst0;
     MCInstLowering.Lower(I++, TmpInst0);
+
     OutStreamer.EmitInstruction(TmpInst0);
-  } while ((I != E) && I->isInsideBundle());
+  } while ((I != E) && I->isInsideBundle()); // Delay slot check
 }
 
 //===----------------------------------------------------------------------===//
@@ -214,7 +207,7 @@ const char *MipsAsmPrinter::getCurrentABIString() const {
   case MipsSubtarget::N32:  return "abiN32";
   case MipsSubtarget::N64:  return "abi64";
   case MipsSubtarget::EABI: return "eabi32"; // TODO: handle eabi64
-  default: llvm_unreachable("Unknown Mips ABI");;
+  default: llvm_unreachable("Unknown Mips ABI");
   }
 }
 
@@ -246,8 +239,7 @@ void MipsAsmPrinter::EmitFunctionBodyStart() {
 
     OutStreamer.EmitRawText(StringRef("\t.set\tnoreorder"));
     OutStreamer.EmitRawText(StringRef("\t.set\tnomacro"));
-    if (MipsFI->getEmitNOAT())
-      OutStreamer.EmitRawText(StringRef("\t.set\tnoat"));
+    OutStreamer.EmitRawText(StringRef("\t.set\tnoat"));
   }
 }
 
@@ -258,9 +250,7 @@ void MipsAsmPrinter::EmitFunctionBodyEnd() {
   // always be at the function end, and we can't emit and
   // break with BB logic.
   if (OutStreamer.hasRawTextSupport()) {
-    if (MipsFI->getEmitNOAT())
-      OutStreamer.EmitRawText(StringRef("\t.set\tat"));
-
+    OutStreamer.EmitRawText(StringRef("\t.set\tat"));
     OutStreamer.EmitRawText(StringRef("\t.set\tmacro"));
     OutStreamer.EmitRawText(StringRef("\t.set\treorder"));
     OutStreamer.EmitRawText("\t.end\t" + Twine(CurrentFnSym->getName()));
diff --git a/lib/Target/Mips/MipsAsmPrinter.h b/lib/Target/Mips/MipsAsmPrinter.h
index 562bf9c..94d8bfa 100644
--- a/lib/Target/Mips/MipsAsmPrinter.h
+++ b/lib/Target/Mips/MipsAsmPrinter.h
@@ -32,6 +32,14 @@ class LLVM_LIBRARY_VISIBILITY MipsAsmPrinter : public AsmPrinter {
 
   void EmitInstrWithMacroNoAT(const MachineInstr *MI);
 
+private:
+  // tblgen'erated function.
+  bool emitPseudoExpansionLowering(MCStreamer &OutStreamer,
+                                   const MachineInstr *MI);
+
+  // lowerOperand - Convert a MachineOperand into the equivalent MCOperand.
+  bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp);
+
 public:
 
   const MipsSubtarget *Subtarget;
diff --git a/lib/Target/Mips/MipsCallingConv.td b/lib/Target/Mips/MipsCallingConv.td
index 19213fa..78cf140 100644
--- a/lib/Target/Mips/MipsCallingConv.td
+++ b/lib/Target/Mips/MipsCallingConv.td
@@ -35,9 +35,6 @@ def RetCC_MipsO32 : CallingConv<[
 //===----------------------------------------------------------------------===//
 
 def CC_MipsN : CallingConv<[
-   // Handles byval parameters.
-  CCIfByVal<CCCustom<"CC_Mips64Byval">>,
-
   // Promote i8/i16 arguments to i32.
   CCIfType<[i8, i16], CCPromoteToType<i32>>,
 
@@ -72,9 +69,6 @@ def CC_MipsN : CallingConv<[
 // N32/64 variable arguments.
 // All arguments are passed in integer registers.
 def CC_MipsN_VarArg : CallingConv<[
-   // Handles byval parameters.
-  CCIfByVal<CCCustom<"CC_Mips64Byval">>,
-
   // Promote i8/i16 arguments to i32.
   CCIfType<[i8, i16], CCPromoteToType<i32>>,
 
@@ -211,12 +205,6 @@ def CC_Mips_FastCC : CallingConv<[
 // Mips Calling Convention Dispatch
 //===----------------------------------------------------------------------===//
 
-def CC_Mips : CallingConv<[
-  CCIfSubtarget<"isABI_EABI()", CCDelegateTo<CC_MipsEABI>>,
-  CCIfSubtarget<"isABI_N32()", CCDelegateTo<CC_MipsN>>,
-  CCIfSubtarget<"isABI_N64()", CCDelegateTo<CC_MipsN>>
-]>;
-
 def RetCC_Mips : CallingConv<[
   CCIfSubtarget<"isABI_EABI()", CCDelegateTo<RetCC_MipsEABI>>,
   CCIfSubtarget<"isABI_N32()", CCDelegateTo<RetCC_MipsN>>,
diff --git a/lib/Target/Mips/MipsCodeEmitter.cpp b/lib/Target/Mips/MipsCodeEmitter.cpp
index cb7022b..4bfccd8 100644
--- a/lib/Target/Mips/MipsCodeEmitter.cpp
+++ b/lib/Target/Mips/MipsCodeEmitter.cpp
@@ -30,7 +30,6 @@
 #include "llvm/CodeGen/Passes.h"
 #include "llvm/Constants.h"
 #include "llvm/DerivedTypes.h"
-#include "llvm/Function.h"
 #include "llvm/PassManager.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -48,7 +47,7 @@ namespace {
 class MipsCodeEmitter : public MachineFunctionPass {
   MipsJITInfo *JTI;
   const MipsInstrInfo *II;
-  const TargetData *TD;
+  const DataLayout *TD;
   const MipsSubtarget *Subtarget;
   TargetMachine &TM;
   JITCodeEmitter &MCE;
@@ -67,7 +66,7 @@ class MipsCodeEmitter : public MachineFunctionPass {
     MipsCodeEmitter(TargetMachine &tm, JITCodeEmitter &mce) :
       MachineFunctionPass(ID), JTI(0),
       II((const MipsInstrInfo *) tm.getInstrInfo()),
-      TD(tm.getTargetData()), TM(tm), MCE(mce), MCPEs(0), MJTEs(0),
+      TD(tm.getDataLayout()), TM(tm), MCE(mce), MCPEs(0), MJTEs(0),
       IsPIC(TM.getRelocationModel() == Reloc::PIC_) {
     }
 
@@ -129,7 +128,7 @@ char MipsCodeEmitter::ID = 0;
 bool MipsCodeEmitter::runOnMachineFunction(MachineFunction &MF) {
   JTI = ((MipsTargetMachine&) MF.getTarget()).getJITInfo();
   II = ((const MipsTargetMachine&) MF.getTarget()).getInstrInfo();
-  TD = ((const MipsTargetMachine&) MF.getTarget()).getTargetData();
+  TD = ((const MipsTargetMachine&) MF.getTarget()).getDataLayout();
   Subtarget = &TM.getSubtarget<MipsSubtarget> ();
   MCPEs = &MF.getConstantPool()->getConstants();
   MJTEs = 0;
@@ -139,7 +138,7 @@ bool MipsCodeEmitter::runOnMachineFunction(MachineFunction &MF) {
 
   do {
     DEBUG(errs() << "JITTing function '"
-        << MF.getFunction()->getName() << "'\n");
+        << MF.getName() << "'\n");
     MCE.startFunction(MF);
 
     for (MachineFunction::iterator MBB = MF.begin(), E = MF.end();
@@ -219,15 +218,9 @@ unsigned MipsCodeEmitter::getMachineOpValue(const MachineInstr &MI,
     return getMipsRegisterNumbering(MO.getReg());
   else if (MO.isImm())
     return static_cast<unsigned>(MO.getImm());
-  else if (MO.isGlobal()) {
-    if (MI.getOpcode() == Mips::ULW || MI.getOpcode() == Mips::USW ||
-          MI.getOpcode() == Mips::ULH || MI.getOpcode() == Mips::ULHu)
-      emitGlobalAddressUnaligned(MO.getGlobal(), getRelocation(MI, MO), 4);
-    else if (MI.getOpcode() == Mips::USH)
-      emitGlobalAddressUnaligned(MO.getGlobal(), getRelocation(MI, MO), 8);
-    else
-      emitGlobalAddress(MO.getGlobal(), getRelocation(MI, MO), true);
-  } else if (MO.isSymbol())
+  else if (MO.isGlobal())
+    emitGlobalAddress(MO.getGlobal(), getRelocation(MI, MO), true);
+  else if (MO.isSymbol())
     emitExternalSymbolAddress(MO.getSymbolName(), getRelocation(MI, MO));
   else if (MO.isCPI())
     emitConstPoolAddress(MO.getIndex(), getRelocation(MI, MO));
@@ -384,29 +377,8 @@ void MipsCodeEmitter::emitInstruction(const MachineInstr &MI) {
   if ((MI.getDesc().TSFlags & MipsII::FormMask) == MipsII::Pseudo)
     return;
 
-
-  switch (MI.getOpcode()) {
-  case Mips::USW:
-    NumEmitted += emitUSW(MI);
-    break;
-  case Mips::ULW:
-    NumEmitted += emitULW(MI);
-    break;
-  case Mips::ULH:
-    NumEmitted += emitULH(MI);
-    break;
-  case Mips::ULHu:
-    NumEmitted += emitULHu(MI);
-    break;
-  case Mips::USH:
-    NumEmitted += emitUSH(MI);
-    break;
-
-  default:
-    emitWordLE(getBinaryCodeForInstr(MI));
-    ++NumEmitted;  // Keep track of the # of mi's emitted
-    break;
-  }
+  emitWordLE(getBinaryCodeForInstr(MI));
+  ++NumEmitted;  // Keep track of the # of mi's emitted
 
   MCE.processDebugLoc(MI.getDebugLoc(), false);
 }
diff --git a/lib/Target/Mips/MipsDSPInstrFormats.td b/lib/Target/Mips/MipsDSPInstrFormats.td
new file mode 100644
index 0000000..8e01d06
--- /dev/null
+++ b/lib/Target/Mips/MipsDSPInstrFormats.td
@@ -0,0 +1,309 @@
+//===- MipsDSPInstrFormats.td - Mips Instruction Formats ---*- tablegen -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+def HasDSP : Predicate<"Subtarget.hasDSP()">,
+             AssemblerPredicate<"FeatureDSP">;
+def HasDSPR2 : Predicate<"Subtarget.hasDSPR2()">,
+               AssemblerPredicate<"FeatureDSPR2">;
+
+// Fields.
+class Field6<bits<6> val> {
+  bits<6> V = val;
+}
+
+def SPECIAL3_OPCODE : Field6<0b011111>;
+def REGIMM_OPCODE : Field6<0b000001>;
+
+class DSPInst : MipsInst<(outs), (ins), "", [], NoItinerary, FrmOther> {
+  let Predicates = [HasDSP];
+}
+
+class PseudoDSP<dag outs, dag ins, list<dag> pattern>:
+  MipsPseudo<outs, ins, "", pattern> {
+  let Predicates = [HasDSP];
+}
+
+// ADDU.QB sub-class format.
+class ADDU_QB_FMT<bits<5> op> : DSPInst {
+  bits<5> rd;
+  bits<5> rs;
+  bits<5> rt;
+
+  let Opcode = SPECIAL3_OPCODE.V;
+
+  let Inst{25-21} = rs;
+  let Inst{20-16} = rt;
+  let Inst{15-11} = rd;
+  let Inst{10-6}  = op;
+  let Inst{5-0}   = 0b010000;
+}
+
+class RADDU_W_QB_FMT<bits<5> op> : DSPInst {
+  bits<5> rd;
+  bits<5> rs;
+
+  let Opcode = SPECIAL3_OPCODE.V;
+
+  let Inst{25-21} = rs;
+  let Inst{20-16} = 0;
+  let Inst{15-11} = rd;
+  let Inst{10-6}  = op;
+  let Inst{5-0}   = 0b010000;
+}
+
+// CMPU.EQ.QB sub-class format.
+class CMP_EQ_QB_R2_FMT<bits<5> op> : DSPInst {
+  bits<5> rs;
+  bits<5> rt;
+
+  let Opcode = SPECIAL3_OPCODE.V;
+
+  let Inst{25-21} = rs;
+  let Inst{20-16} = rt;
+  let Inst{15-11} = 0;
+  let Inst{10-6}  = op;
+  let Inst{5-0}   = 0b010001;
+}
+
+class CMP_EQ_QB_R3_FMT<bits<5> op> : DSPInst {
+  bits<5> rs;
+  bits<5> rt;
+  bits<5> rd;
+
+  let Opcode = SPECIAL3_OPCODE.V;
+
+  let Inst{25-21} = rs;
+  let Inst{20-16} = rt;
+  let Inst{15-11} = rd;
+  let Inst{10-6}  = op;
+  let Inst{5-0}   = 0b010001;
+}
+
+class PRECR_SRA_PH_W_FMT<bits<5> op> : DSPInst {
+  bits<5> rs;
+  bits<5> rt;
+  bits<5> sa;
+
+  let Opcode = SPECIAL3_OPCODE.V;
+
+  let Inst{25-21} = rs;
+  let Inst{20-16} = rt;
+  let Inst{15-11} = sa;
+  let Inst{10-6}  = op;
+  let Inst{5-0}   = 0b010001;
+}
+
+// ABSQ_S.PH sub-class format.
+class ABSQ_S_PH_R2_FMT<bits<5> op> : DSPInst {
+  bits<5> rd;
+  bits<5> rt;
+
+  let Opcode = SPECIAL3_OPCODE.V;
+
+  let Inst{25-21} = 0;
+  let Inst{20-16} = rt;
+  let Inst{15-11} = rd;
+  let Inst{10-6}  = op;
+  let Inst{5-0}   = 0b010010;
+}
+
+
+class REPL_FMT<bits<5> op> : DSPInst {
+  bits<5> rd;
+  bits<10> imm;
+
+  let Opcode = SPECIAL3_OPCODE.V;
+
+  let Inst{25-16} = imm;
+  let Inst{15-11} = rd;
+  let Inst{10-6}  = op;
+  let Inst{5-0}   = 0b010010;
+}
+
+// SHLL.QB sub-class format.
+class SHLL_QB_FMT<bits<5> op> : DSPInst {
+  bits<5> rd;
+  bits<5> rt;
+  bits<5> rs_sa;
+
+  let Opcode = SPECIAL3_OPCODE.V;
+
+  let Inst{25-21} = rs_sa;
+  let Inst{20-16} = rt;
+  let Inst{15-11} = rd;
+  let Inst{10-6}  = op;
+  let Inst{5-0}   = 0b010011;
+}
+
+// LX sub-class format.
+class LX_FMT<bits<5> op> : DSPInst {
+  bits<5> rd;
+  bits<5> base;
+  bits<5> index;
+
+  let Opcode = SPECIAL3_OPCODE.V;
+
+  let Inst{25-21} = base;
+  let Inst{20-16} = index;
+  let Inst{15-11} = rd;
+  let Inst{10-6}  = op;
+  let Inst{5-0} = 0b001010;
+}
+
+// ADDUH.QB sub-class format.
+class ADDUH_QB_FMT<bits<5> op> : DSPInst {
+  bits<5> rd;
+  bits<5> rs;
+  bits<5> rt;
+
+  let Opcode = SPECIAL3_OPCODE.V;
+
+  let Inst{25-21} = rs;
+  let Inst{20-16} = rt;
+  let Inst{15-11} = rd;
+  let Inst{10-6} = op;
+  let Inst{5-0} = 0b011000;
+}
+
+// APPEND sub-class format.
+class APPEND_FMT<bits<5> op> : DSPInst {
+  bits<5> rt;
+  bits<5> rs;
+  bits<5> sa;
+
+  let Opcode = SPECIAL3_OPCODE.V;
+
+  let Inst{25-21} = rs;
+  let Inst{20-16} = rt;
+  let Inst{15-11} = sa;
+  let Inst{10-6} = op;
+  let Inst{5-0} = 0b110001;
+}
+
+// DPA.W.PH sub-class format.
+class DPA_W_PH_FMT<bits<5> op> : DSPInst {
+  bits<2> ac;
+  bits<5> rs;
+  bits<5> rt;
+
+  let Opcode = SPECIAL3_OPCODE.V;
+
+  let Inst{25-21} = rs;
+  let Inst{20-16} = rt;
+  let Inst{15-13} = 0;
+  let Inst{12-11} = ac;
+  let Inst{10-6}  = op;
+  let Inst{5-0} = 0b110000;
+}
+
+// MULT sub-class format.
+class MULT_FMT<bits<6> opcode, bits<6> funct> : DSPInst {
+  bits<2> ac;
+  bits<5> rs;
+  bits<5> rt;
+
+  let Opcode = opcode;
+
+  let Inst{25-21} = rs;
+  let Inst{20-16} = rt;
+  let Inst{15-13} = 0;
+  let Inst{12-11} = ac;
+  let Inst{10-6}  = 0;
+  let Inst{5-0} = funct;
+}
+
+// EXTR.W sub-class format (type 1).
+class EXTR_W_TY1_FMT<bits<5> op> : DSPInst {
+  bits<5> rt;
+  bits<2> ac;
+  bits<5> shift_rs;
+
+  let Opcode = SPECIAL3_OPCODE.V;
+
+  let Inst{25-21} = shift_rs;
+  let Inst{20-16} = rt;
+  let Inst{15-13} = 0;
+  let Inst{12-11} = ac;
+  let Inst{10-6} = op;
+  let Inst{5-0} = 0b111000;
+}
+
+// SHILO sub-class format.
+class SHILO_R1_FMT<bits<5> op> : DSPInst {
+  bits<2> ac;
+  bits<6> shift;
+
+  let Opcode = SPECIAL3_OPCODE.V;
+
+  let Inst{25-20} = shift;
+  let Inst{19-13} = 0;
+  let Inst{12-11} = ac;
+  let Inst{10-6} = op;
+  let Inst{5-0} = 0b111000;
+}
+
+class SHILO_R2_FMT<bits<5> op> : DSPInst {
+  bits<2> ac;
+  bits<5> rs;
+
+  let Opcode = SPECIAL3_OPCODE.V;
+
+  let Inst{25-21} = rs;
+  let Inst{20-13} = 0;
+  let Inst{12-11} = ac;
+  let Inst{10-6} = op;
+  let Inst{5-0} = 0b111000;
+}
+
+class RDDSP_FMT<bits<5> op> : DSPInst {
+  bits<5> rd;
+  bits<10> mask;
+
+  let Opcode = SPECIAL3_OPCODE.V;
+
+  let Inst{25-16} = mask;
+  let Inst{15-11} = rd;
+  let Inst{10-6} = op;
+  let Inst{5-0} = 0b111000;
+}
+
+class WRDSP_FMT<bits<5> op> : DSPInst {
+  bits<5> rs;
+  bits<10> mask;
+
+  let Opcode = SPECIAL3_OPCODE.V;
+
+  let Inst{25-21} = rs;
+  let Inst{20-11} = mask;
+  let Inst{10-6} = op;
+  let Inst{5-0} = 0b111000;
+}
+
+class BPOSGE32_FMT<bits<5> op> : DSPInst {
+  bits<16> offset;
+
+  let Opcode = REGIMM_OPCODE.V;
+
+  let Inst{25-21} = 0;
+  let Inst{20-16} = op;
+  let Inst{15-0} = offset;
+}
+
+// INSV sub-class format.
+class INSV_FMT<bits<6> op> : DSPInst {
+  bits<5> rt;
+  bits<5> rs;
+
+  let Opcode = SPECIAL3_OPCODE.V;
+
+  let Inst{25-21} = rs;
+  let Inst{20-16} = rt;
+  let Inst{15-6} = 0;
+  let Inst{5-0} = op;
+}
diff --git a/lib/Target/Mips/MipsDSPInstrInfo.td b/lib/Target/Mips/MipsDSPInstrInfo.td
new file mode 100644
index 0000000..ef94028
--- /dev/null
+++ b/lib/Target/Mips/MipsDSPInstrInfo.td
@@ -0,0 +1,1319 @@
+//===- MipsDSPInstrInfo.td - DSP ASE instructions -*- tablegen ------------*-=//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file describes Mips DSP ASE instructions.
+//
+//===----------------------------------------------------------------------===//
+
+// ImmLeaf
+def immZExt2 : ImmLeaf<i32, [{return isUInt<2>(Imm);}]>;
+def immZExt3 : ImmLeaf<i32, [{return isUInt<3>(Imm);}]>;
+def immZExt4 : ImmLeaf<i32, [{return isUInt<4>(Imm);}]>;
+def immZExt8 : ImmLeaf<i32, [{return isUInt<8>(Imm);}]>;
+def immZExt10 : ImmLeaf<i32, [{return isUInt<10>(Imm);}]>;
+def immSExt6 : ImmLeaf<i32, [{return isInt<6>(Imm);}]>;
+
+// Mips-specific dsp nodes
+def SDT_MipsExtr : SDTypeProfile<1, 1, [SDTCisVT<0, i32>, SDTCisSameAs<0, 1>]>;
+def SDT_MipsShilo : SDTypeProfile<0, 1, [SDTCisVT<0, i32>]>;
+def SDT_MipsDPA : SDTypeProfile<0, 2, [SDTCisVT<0, i32>, SDTCisSameAs<0, 1>]>;
+
+class MipsDSPBase<string Opc, SDTypeProfile Prof> :
+  SDNode<!strconcat("MipsISD::", Opc), Prof,
+         [SDNPHasChain, SDNPInGlue, SDNPOutGlue]>;
+
+class MipsDSPSideEffectBase<string Opc, SDTypeProfile Prof> :
+  SDNode<!strconcat("MipsISD::", Opc), Prof,
+         [SDNPHasChain, SDNPInGlue, SDNPOutGlue, SDNPSideEffect]>;
+
+def MipsEXTP : MipsDSPSideEffectBase<"EXTP", SDT_MipsExtr>;
+def MipsEXTPDP : MipsDSPSideEffectBase<"EXTPDP", SDT_MipsExtr>;
+def MipsEXTR_S_H : MipsDSPSideEffectBase<"EXTR_S_H", SDT_MipsExtr>;
+def MipsEXTR_W : MipsDSPSideEffectBase<"EXTR_W", SDT_MipsExtr>;
+def MipsEXTR_R_W : MipsDSPSideEffectBase<"EXTR_R_W", SDT_MipsExtr>;
+def MipsEXTR_RS_W : MipsDSPSideEffectBase<"EXTR_RS_W", SDT_MipsExtr>;
+
+def MipsSHILO : MipsDSPBase<"SHILO", SDT_MipsShilo>;
+def MipsMTHLIP : MipsDSPBase<"MTHLIP", SDT_MipsShilo>;
+
+def MipsMULSAQ_S_W_PH : MipsDSPSideEffectBase<"MULSAQ_S_W_PH", SDT_MipsDPA>;
+def MipsMAQ_S_W_PHL : MipsDSPSideEffectBase<"MAQ_S_W_PHL", SDT_MipsDPA>;
+def MipsMAQ_S_W_PHR : MipsDSPSideEffectBase<"MAQ_S_W_PHR", SDT_MipsDPA>;
+def MipsMAQ_SA_W_PHL : MipsDSPSideEffectBase<"MAQ_SA_W_PHL", SDT_MipsDPA>;
+def MipsMAQ_SA_W_PHR : MipsDSPSideEffectBase<"MAQ_SA_W_PHR", SDT_MipsDPA>;
+
+def MipsDPAU_H_QBL : MipsDSPBase<"DPAU_H_QBL", SDT_MipsDPA>;
+def MipsDPAU_H_QBR : MipsDSPBase<"DPAU_H_QBR", SDT_MipsDPA>;
+def MipsDPSU_H_QBL : MipsDSPBase<"DPSU_H_QBL", SDT_MipsDPA>;
+def MipsDPSU_H_QBR : MipsDSPBase<"DPSU_H_QBR", SDT_MipsDPA>;
+def MipsDPAQ_S_W_PH : MipsDSPSideEffectBase<"DPAQ_S_W_PH", SDT_MipsDPA>;
+def MipsDPSQ_S_W_PH : MipsDSPSideEffectBase<"DPSQ_S_W_PH", SDT_MipsDPA>;
+def MipsDPAQ_SA_L_W : MipsDSPSideEffectBase<"DPAQ_SA_L_W", SDT_MipsDPA>;
+def MipsDPSQ_SA_L_W : MipsDSPSideEffectBase<"DPSQ_SA_L_W", SDT_MipsDPA>;
+
+def MipsDPA_W_PH : MipsDSPBase<"DPA_W_PH", SDT_MipsDPA>;
+def MipsDPS_W_PH : MipsDSPBase<"DPS_W_PH", SDT_MipsDPA>;
+def MipsDPAQX_S_W_PH : MipsDSPSideEffectBase<"DPAQX_S_W_PH", SDT_MipsDPA>;
+def MipsDPAQX_SA_W_PH : MipsDSPSideEffectBase<"DPAQX_SA_W_PH", SDT_MipsDPA>;
+def MipsDPAX_W_PH : MipsDSPBase<"DPAX_W_PH", SDT_MipsDPA>;
+def MipsDPSX_W_PH : MipsDSPBase<"DPSX_W_PH", SDT_MipsDPA>;
+def MipsDPSQX_S_W_PH : MipsDSPSideEffectBase<"DPSQX_S_W_PH", SDT_MipsDPA>;
+def MipsDPSQX_SA_W_PH : MipsDSPSideEffectBase<"DPSQX_SA_W_PH", SDT_MipsDPA>;
+def MipsMULSA_W_PH : MipsDSPBase<"MULSA_W_PH", SDT_MipsDPA>;
+
+def MipsMULT : MipsDSPBase<"MULT", SDT_MipsDPA>;
+def MipsMULTU : MipsDSPBase<"MULTU", SDT_MipsDPA>;
+def MipsMADD_DSP : MipsDSPBase<"MADD_DSP", SDT_MipsDPA>;
+def MipsMADDU_DSP : MipsDSPBase<"MADDU_DSP", SDT_MipsDPA>;
+def MipsMSUB_DSP : MipsDSPBase<"MSUB_DSP", SDT_MipsDPA>;
+def MipsMSUBU_DSP : MipsDSPBase<"MSUBU_DSP", SDT_MipsDPA>;
+
+// Flags.
+class IsCommutable {
+  bit isCommutable = 1;
+}
+
+class UseAC {
+  list<Register> Uses = [AC0];
+}
+
+class UseDSPCtrl {
+  list<Register> Uses = [DSPCtrl];
+}
+
+class ClearDefs {
+  list<Register> Defs = [];
+}
+
+// Instruction encoding.
+class ADDU_QB_ENC : ADDU_QB_FMT<0b00000>;
+class ADDU_S_QB_ENC : ADDU_QB_FMT<0b00100>;
+class SUBU_QB_ENC : ADDU_QB_FMT<0b00001>;
+class SUBU_S_QB_ENC : ADDU_QB_FMT<0b00101>;
+class ADDQ_PH_ENC : ADDU_QB_FMT<0b01010>;
+class ADDQ_S_PH_ENC : ADDU_QB_FMT<0b01110>;
+class SUBQ_PH_ENC : ADDU_QB_FMT<0b01011>;
+class SUBQ_S_PH_ENC : ADDU_QB_FMT<0b01111>;
+class ADDQ_S_W_ENC : ADDU_QB_FMT<0b10110>;
+class SUBQ_S_W_ENC : ADDU_QB_FMT<0b10111>;
+class ADDSC_ENC : ADDU_QB_FMT<0b10000>;
+class ADDWC_ENC : ADDU_QB_FMT<0b10001>;
+class MODSUB_ENC : ADDU_QB_FMT<0b10010>;
+class RADDU_W_QB_ENC : RADDU_W_QB_FMT<0b10100>;
+class ABSQ_S_PH_ENC : ABSQ_S_PH_R2_FMT<0b01001>;
+class ABSQ_S_W_ENC : ABSQ_S_PH_R2_FMT<0b10001>;
+class PRECRQ_QB_PH_ENC : CMP_EQ_QB_R3_FMT<0b01100>;
+class PRECRQ_PH_W_ENC : CMP_EQ_QB_R3_FMT<0b10100>;
+class PRECRQ_RS_PH_W_ENC : CMP_EQ_QB_R3_FMT<0b10101>;
+class PRECRQU_S_QB_PH_ENC : CMP_EQ_QB_R3_FMT<0b01111>;
+class PRECEQ_W_PHL_ENC : ABSQ_S_PH_R2_FMT<0b01100>;
+class PRECEQ_W_PHR_ENC : ABSQ_S_PH_R2_FMT<0b01101>;
+class PRECEQU_PH_QBL_ENC : ABSQ_S_PH_R2_FMT<0b00100>;
+class PRECEQU_PH_QBR_ENC : ABSQ_S_PH_R2_FMT<0b00101>;
+class PRECEQU_PH_QBLA_ENC : ABSQ_S_PH_R2_FMT<0b00110>;
+class PRECEQU_PH_QBRA_ENC : ABSQ_S_PH_R2_FMT<0b00111>;
+class PRECEU_PH_QBL_ENC : ABSQ_S_PH_R2_FMT<0b11100>;
+class PRECEU_PH_QBR_ENC : ABSQ_S_PH_R2_FMT<0b11101>;
+class PRECEU_PH_QBLA_ENC : ABSQ_S_PH_R2_FMT<0b11110>;
+class PRECEU_PH_QBRA_ENC : ABSQ_S_PH_R2_FMT<0b11111>;
+class SHLL_QB_ENC : SHLL_QB_FMT<0b00000>;
+class SHLLV_QB_ENC : SHLL_QB_FMT<0b00010>;
+class SHRL_QB_ENC : SHLL_QB_FMT<0b00001>;
+class SHRLV_QB_ENC : SHLL_QB_FMT<0b00011>;
+class SHLL_PH_ENC : SHLL_QB_FMT<0b01000>;
+class SHLLV_PH_ENC : SHLL_QB_FMT<0b01010>;
+class SHLL_S_PH_ENC : SHLL_QB_FMT<0b01100>;
+class SHLLV_S_PH_ENC : SHLL_QB_FMT<0b01110>;
+class SHRA_PH_ENC : SHLL_QB_FMT<0b01001>;
+class SHRAV_PH_ENC : SHLL_QB_FMT<0b01011>;
+class SHRA_R_PH_ENC : SHLL_QB_FMT<0b01101>;
+class SHRAV_R_PH_ENC : SHLL_QB_FMT<0b01111>;
+class SHLL_S_W_ENC : SHLL_QB_FMT<0b10100>;
+class SHLLV_S_W_ENC : SHLL_QB_FMT<0b10110>;
+class SHRA_R_W_ENC : SHLL_QB_FMT<0b10101>;
+class SHRAV_R_W_ENC : SHLL_QB_FMT<0b10111>;
+class MULEU_S_PH_QBL_ENC : ADDU_QB_FMT<0b00110>;
+class MULEU_S_PH_QBR_ENC : ADDU_QB_FMT<0b00111>;
+class MULEQ_S_W_PHL_ENC : ADDU_QB_FMT<0b11100>;
+class MULEQ_S_W_PHR_ENC : ADDU_QB_FMT<0b11101>;
+class MULQ_RS_PH_ENC : ADDU_QB_FMT<0b11111>;
+class MULSAQ_S_W_PH_ENC : DPA_W_PH_FMT<0b00110>;
+class MAQ_S_W_PHL_ENC : DPA_W_PH_FMT<0b10100>;
+class MAQ_S_W_PHR_ENC : DPA_W_PH_FMT<0b10110>;
+class MAQ_SA_W_PHL_ENC : DPA_W_PH_FMT<0b10000>;
+class MAQ_SA_W_PHR_ENC : DPA_W_PH_FMT<0b10010>;
+class DPAU_H_QBL_ENC : DPA_W_PH_FMT<0b00011>;
+class DPAU_H_QBR_ENC : DPA_W_PH_FMT<0b00111>;
+class DPSU_H_QBL_ENC : DPA_W_PH_FMT<0b01011>;
+class DPSU_H_QBR_ENC : DPA_W_PH_FMT<0b01111>;
+class DPAQ_S_W_PH_ENC : DPA_W_PH_FMT<0b00100>;
+class DPSQ_S_W_PH_ENC : DPA_W_PH_FMT<0b00101>;
+class DPAQ_SA_L_W_ENC : DPA_W_PH_FMT<0b01100>;
+class DPSQ_SA_L_W_ENC : DPA_W_PH_FMT<0b01101>;
+class MULT_DSP_ENC : MULT_FMT<0b000000, 0b011000>;
+class MULTU_DSP_ENC : MULT_FMT<0b000000, 0b011001>;
+class MADD_DSP_ENC : MULT_FMT<0b011100, 0b000000>;
+class MADDU_DSP_ENC : MULT_FMT<0b011100, 0b000001>;
+class MSUB_DSP_ENC : MULT_FMT<0b011100, 0b000100>;
+class MSUBU_DSP_ENC : MULT_FMT<0b011100, 0b000101>;
+class CMPU_EQ_QB_ENC : CMP_EQ_QB_R2_FMT<0b00000>;
+class CMPU_LT_QB_ENC : CMP_EQ_QB_R2_FMT<0b00001>;
+class CMPU_LE_QB_ENC : CMP_EQ_QB_R2_FMT<0b00010>;
+class CMPGU_EQ_QB_ENC : CMP_EQ_QB_R3_FMT<0b00100>;
+class CMPGU_LT_QB_ENC : CMP_EQ_QB_R3_FMT<0b00101>;
+class CMPGU_LE_QB_ENC : CMP_EQ_QB_R3_FMT<0b00110>;
+class CMP_EQ_PH_ENC : CMP_EQ_QB_R2_FMT<0b01000>;
+class CMP_LT_PH_ENC : CMP_EQ_QB_R2_FMT<0b01001>;
+class CMP_LE_PH_ENC : CMP_EQ_QB_R2_FMT<0b01010>;
+class BITREV_ENC : ABSQ_S_PH_R2_FMT<0b11011>;
+class PACKRL_PH_ENC : CMP_EQ_QB_R3_FMT<0b01110>;
+class REPL_QB_ENC : REPL_FMT<0b00010>;
+class REPL_PH_ENC : REPL_FMT<0b01010>;
+class REPLV_QB_ENC : ABSQ_S_PH_R2_FMT<0b00011>;
+class REPLV_PH_ENC : ABSQ_S_PH_R2_FMT<0b01011>;
+class PICK_QB_ENC : CMP_EQ_QB_R3_FMT<0b00011>;
+class PICK_PH_ENC : CMP_EQ_QB_R3_FMT<0b01011>;
+class LWX_ENC : LX_FMT<0b00000>;
+class LHX_ENC : LX_FMT<0b00100>;
+class LBUX_ENC : LX_FMT<0b00110>;
+class BPOSGE32_ENC : BPOSGE32_FMT<0b11100>;
+class INSV_ENC : INSV_FMT<0b001100>;
+
+class EXTP_ENC : EXTR_W_TY1_FMT<0b00010>;
+class EXTPV_ENC : EXTR_W_TY1_FMT<0b00011>;
+class EXTPDP_ENC : EXTR_W_TY1_FMT<0b01010>;
+class EXTPDPV_ENC : EXTR_W_TY1_FMT<0b01011>;
+class EXTR_W_ENC : EXTR_W_TY1_FMT<0b00000>;
+class EXTRV_W_ENC : EXTR_W_TY1_FMT<0b00001>;
+class EXTR_R_W_ENC : EXTR_W_TY1_FMT<0b00100>;
+class EXTRV_R_W_ENC : EXTR_W_TY1_FMT<0b00101>;
+class EXTR_RS_W_ENC : EXTR_W_TY1_FMT<0b00110>;
+class EXTRV_RS_W_ENC : EXTR_W_TY1_FMT<0b00111>;
+class EXTR_S_H_ENC : EXTR_W_TY1_FMT<0b01110>;
+class EXTRV_S_H_ENC : EXTR_W_TY1_FMT<0b01111>;
+class SHILO_ENC : SHILO_R1_FMT<0b11010>;
+class SHILOV_ENC : SHILO_R2_FMT<0b11011>;
+class MTHLIP_ENC : SHILO_R2_FMT<0b11111>;
+
+class RDDSP_ENC : RDDSP_FMT<0b10010>;
+class WRDSP_ENC : WRDSP_FMT<0b10011>;
+class ADDU_PH_ENC : ADDU_QB_FMT<0b01000>;
+class ADDU_S_PH_ENC : ADDU_QB_FMT<0b01100>;
+class SUBU_PH_ENC : ADDU_QB_FMT<0b01001>;
+class SUBU_S_PH_ENC : ADDU_QB_FMT<0b01101>;
+class CMPGDU_EQ_QB_ENC : CMP_EQ_QB_R3_FMT<0b11000>;
+class CMPGDU_LT_QB_ENC : CMP_EQ_QB_R3_FMT<0b11001>;
+class CMPGDU_LE_QB_ENC : CMP_EQ_QB_R3_FMT<0b11010>;
+class ABSQ_S_QB_ENC : ABSQ_S_PH_R2_FMT<0b00001>;
+class ADDUH_QB_ENC : ADDUH_QB_FMT<0b00000>;
+class ADDUH_R_QB_ENC : ADDUH_QB_FMT<0b00010>;
+class SUBUH_QB_ENC : ADDUH_QB_FMT<0b00001>;
+class SUBUH_R_QB_ENC : ADDUH_QB_FMT<0b00011>;
+class ADDQH_PH_ENC : ADDUH_QB_FMT<0b01000>;
+class ADDQH_R_PH_ENC : ADDUH_QB_FMT<0b01010>;
+class SUBQH_PH_ENC : ADDUH_QB_FMT<0b01001>;
+class SUBQH_R_PH_ENC : ADDUH_QB_FMT<0b01011>;
+class ADDQH_W_ENC : ADDUH_QB_FMT<0b10000>;
+class ADDQH_R_W_ENC : ADDUH_QB_FMT<0b10010>;
+class SUBQH_W_ENC : ADDUH_QB_FMT<0b10001>;
+class SUBQH_R_W_ENC : ADDUH_QB_FMT<0b10011>;
+class MUL_PH_ENC : ADDUH_QB_FMT<0b01100>;
+class MUL_S_PH_ENC : ADDUH_QB_FMT<0b01110>;
+class MULQ_S_W_ENC : ADDUH_QB_FMT<0b10110>;
+class MULQ_RS_W_ENC : ADDUH_QB_FMT<0b10111>;
+class MULQ_S_PH_ENC : ADDU_QB_FMT<0b11110>;
+class DPA_W_PH_ENC : DPA_W_PH_FMT<0b00000>;
+class DPS_W_PH_ENC : DPA_W_PH_FMT<0b00001>;
+class DPAQX_S_W_PH_ENC : DPA_W_PH_FMT<0b11000>;
+class DPAQX_SA_W_PH_ENC : DPA_W_PH_FMT<0b11010>;
+class DPAX_W_PH_ENC : DPA_W_PH_FMT<0b01000>;
+class DPSX_W_PH_ENC : DPA_W_PH_FMT<0b01001>;
+class DPSQX_S_W_PH_ENC : DPA_W_PH_FMT<0b11001>;
+class DPSQX_SA_W_PH_ENC : DPA_W_PH_FMT<0b11011>;
+class MULSA_W_PH_ENC : DPA_W_PH_FMT<0b00010>;
+class PRECR_QB_PH_ENC : CMP_EQ_QB_R3_FMT<0b01101>;
+class PRECR_SRA_PH_W_ENC : PRECR_SRA_PH_W_FMT<0b11110>;
+class PRECR_SRA_R_PH_W_ENC : PRECR_SRA_PH_W_FMT<0b11111>;
+class SHRA_QB_ENC : SHLL_QB_FMT<0b00100>;
+class SHRAV_QB_ENC : SHLL_QB_FMT<0b00110>;
+class SHRA_R_QB_ENC : SHLL_QB_FMT<0b00101>;
+class SHRAV_R_QB_ENC : SHLL_QB_FMT<0b00111>;
+class SHRL_PH_ENC : SHLL_QB_FMT<0b11001>;
+class SHRLV_PH_ENC : SHLL_QB_FMT<0b11011>;
+class APPEND_ENC : APPEND_FMT<0b00000>;
+class BALIGN_ENC : APPEND_FMT<0b10000>;
+class PREPEND_ENC : APPEND_FMT<0b00001>;
+
+// Instruction desc.
+class ADDU_QB_DESC_BASE<string instr_asm, SDPatternOperator OpNode,
+                        InstrItinClass itin, RegisterClass RCD,
+                        RegisterClass RCS,  RegisterClass RCT = RCS> {
+  dag OutOperandList = (outs RCD:$rd);
+  dag InOperandList = (ins RCS:$rs, RCT:$rt);
+  string AsmString = !strconcat(instr_asm, "\t$rd, $rs, $rt");
+  list<dag> Pattern = [(set RCD:$rd, (OpNode RCS:$rs, RCT:$rt))];
+  InstrItinClass Itinerary = itin;
+  list<Register> Defs = [DSPCtrl];
+}
+
+class RADDU_W_QB_DESC_BASE<string instr_asm, SDPatternOperator OpNode,
+                           InstrItinClass itin, RegisterClass RCD,
+                           RegisterClass RCS = RCD> {
+  dag OutOperandList = (outs RCD:$rd);
+  dag InOperandList = (ins RCS:$rs);
+  string AsmString = !strconcat(instr_asm, "\t$rd, $rs");
+  list<dag> Pattern = [(set RCD:$rd, (OpNode RCS:$rs))];
+  InstrItinClass Itinerary = itin;
+  list<Register> Defs = [DSPCtrl];
+}
+
+class CMP_EQ_QB_R2_DESC_BASE<string instr_asm, SDPatternOperator OpNode,
+                             InstrItinClass itin, RegisterClass RCS,
+                             RegisterClass RCT = RCS> {
+  dag OutOperandList = (outs);
+  dag InOperandList = (ins RCS:$rs, RCT:$rt);
+  string AsmString = !strconcat(instr_asm, "\t$rs, $rt");
+  list<dag> Pattern = [(OpNode RCS:$rs, RCT:$rt)];
+  InstrItinClass Itinerary = itin;
+  list<Register> Defs = [DSPCtrl];
+}
+
+class CMP_EQ_QB_R3_DESC_BASE<string instr_asm, SDPatternOperator OpNode,
+                             InstrItinClass itin, RegisterClass RCD,
+                             RegisterClass RCS,  RegisterClass RCT = RCS> {
+  dag OutOperandList = (outs RCD:$rd);
+  dag InOperandList = (ins RCS:$rs, RCT:$rt);
+  string AsmString = !strconcat(instr_asm, "\t$rd, $rs, $rt");
+  list<dag> Pattern = [(set RCD:$rd, (OpNode RCS:$rs, RCT:$rt))];
+  InstrItinClass Itinerary = itin;
+  list<Register> Defs = [DSPCtrl];
+}
+
+class PRECR_SRA_PH_W_DESC_BASE<string instr_asm, SDPatternOperator OpNode,
+                               InstrItinClass itin, RegisterClass RCT,
+                               RegisterClass RCS = RCT> {
+  dag OutOperandList = (outs RCT:$rt);
+  dag InOperandList = (ins RCS:$rs, shamt:$sa, RCS:$src);
+  string AsmString = !strconcat(instr_asm, "\t$rt, $rs, $sa");
+  list<dag> Pattern = [(set RCT:$rt, (OpNode RCS:$src, RCS:$rs, immZExt5:$sa))];
+  InstrItinClass Itinerary = itin;
+  list<Register> Defs = [DSPCtrl];
+  string Constraints = "$src = $rt";
+}
+
+class ABSQ_S_PH_R2_DESC_BASE<string instr_asm, SDPatternOperator OpNode,
+                             InstrItinClass itin, RegisterClass RCD,
+                             RegisterClass RCT = RCD> {
+  dag OutOperandList = (outs RCD:$rd);
+  dag InOperandList = (ins RCT:$rt);
+  string AsmString = !strconcat(instr_asm, "\t$rd, $rt");
+  list<dag> Pattern = [(set RCD:$rd, (OpNode RCT:$rt))];
+  InstrItinClass Itinerary = itin;
+  list<Register> Defs = [DSPCtrl];
+}
+
+class REPL_DESC_BASE<string instr_asm, SDPatternOperator OpNode,
+                     ImmLeaf immPat, InstrItinClass itin, RegisterClass RC> {
+  dag OutOperandList = (outs RC:$rd);
+  dag InOperandList = (ins uimm16:$imm);
+  string AsmString = !strconcat(instr_asm, "\t$rd, $imm");
+  list<dag> Pattern = [(set RC:$rd, (OpNode immPat:$imm))];
+  InstrItinClass Itinerary = itin;
+  list<Register> Defs = [DSPCtrl];
+}
+
+class SHLL_QB_R3_DESC_BASE<string instr_asm, SDPatternOperator OpNode,
+                           InstrItinClass itin, RegisterClass RC> {
+  dag OutOperandList = (outs RC:$rd);
+  dag InOperandList =  (ins RC:$rt, CPURegs:$rs_sa);
+  string AsmString = !strconcat(instr_asm, "\t$rd, $rt, $rs_sa");
+  list<dag> Pattern = [(set RC:$rd, (OpNode RC:$rt, CPURegs:$rs_sa))];
+  InstrItinClass Itinerary = itin;
+  list<Register> Defs = [DSPCtrl];
+}
+
+class SHLL_QB_R2_DESC_BASE<string instr_asm, SDPatternOperator OpNode,
+                           SDPatternOperator ImmPat, InstrItinClass itin,
+                           RegisterClass RC> {
+  dag OutOperandList = (outs RC:$rd);
+  dag InOperandList = (ins RC:$rt, uimm16:$rs_sa);
+  string AsmString = !strconcat(instr_asm, "\t$rd, $rt, $rs_sa");
+  list<dag> Pattern = [(set RC:$rd, (OpNode RC:$rt, ImmPat:$rs_sa))];
+  InstrItinClass Itinerary = itin;
+  list<Register> Defs = [DSPCtrl];
+}
+
+class LX_DESC_BASE<string instr_asm, SDPatternOperator OpNode,
+                   InstrItinClass itin> {
+  dag OutOperandList = (outs CPURegs:$rd);
+  dag InOperandList = (ins CPURegs:$base, CPURegs:$index);
+  string AsmString = !strconcat(instr_asm, "\t$rd, ${index}(${base})");
+  list<dag> Pattern = [(set CPURegs:$rd,
+                       (OpNode CPURegs:$base, CPURegs:$index))];
+  InstrItinClass Itinerary = itin;
+  list<Register> Defs = [DSPCtrl];
+  bit mayLoad = 1;
+}
+
+class ADDUH_QB_DESC_BASE<string instr_asm, SDPatternOperator OpNode,
+                         InstrItinClass itin, RegisterClass RCD,
+                         RegisterClass RCS = RCD,  RegisterClass RCT = RCD> {
+  dag OutOperandList = (outs RCD:$rd);
+  dag InOperandList = (ins RCS:$rs, RCT:$rt);
+  string AsmString = !strconcat(instr_asm, "\t$rd, $rs, $rt");
+  list<dag> Pattern = [(set RCD:$rd, (OpNode RCS:$rs, RCT:$rt))];
+  InstrItinClass Itinerary = itin;
+  list<Register> Defs = [DSPCtrl];
+}
+
+class APPEND_DESC_BASE<string instr_asm, SDPatternOperator OpNode,
+                       SDPatternOperator ImmOp, InstrItinClass itin> {
+  dag OutOperandList = (outs CPURegs:$rt);
+  dag InOperandList = (ins CPURegs:$rs, shamt:$sa, CPURegs:$src);
+  string AsmString = !strconcat(instr_asm, "\t$rt, $rs, $sa");
+  list<dag> Pattern =  [(set CPURegs:$rt,
+                        (OpNode CPURegs:$src, CPURegs:$rs, ImmOp:$sa))];
+  InstrItinClass Itinerary = itin;
+  list<Register> Defs = [DSPCtrl];
+  string Constraints = "$src = $rt";
+}
+
+class EXTR_W_TY1_R2_DESC_BASE<string instr_asm, SDPatternOperator OpNode,
+                              InstrItinClass itin> {
+  dag OutOperandList = (outs CPURegs:$rt);
+  dag InOperandList = (ins ACRegs:$ac, CPURegs:$shift_rs);
+  string AsmString = !strconcat(instr_asm, "\t$rt, $ac, $shift_rs");
+  InstrItinClass Itinerary = itin;
+  list<Register> Defs = [DSPCtrl];
+}
+
+class EXTR_W_TY1_R1_DESC_BASE<string instr_asm, SDPatternOperator OpNode,
+                              InstrItinClass itin> {
+  dag OutOperandList = (outs CPURegs:$rt);
+  dag InOperandList = (ins ACRegs:$ac, uimm16:$shift_rs);
+  string AsmString = !strconcat(instr_asm, "\t$rt, $ac, $shift_rs");
+  InstrItinClass Itinerary = itin;
+  list<Register> Defs = [DSPCtrl];
+}
+
+class SHILO_R1_PSEUDO_BASE<SDPatternOperator OpNode, InstrItinClass itin,
+                           Instruction realinst> :
+  PseudoDSP<(outs), (ins simm16:$shift), [(OpNode immSExt6:$shift)]>,
+  PseudoInstExpansion<(realinst AC0, simm16:$shift)> {
+  list<Register> Defs = [DSPCtrl, AC0];
+  list<Register> Uses = [AC0];
+  InstrItinClass Itinerary = itin;
+}
+
+class SHILO_R1_DESC_BASE<string instr_asm> {
+  dag OutOperandList = (outs ACRegs:$ac);
+  dag InOperandList = (ins simm16:$shift);
+  string AsmString = !strconcat(instr_asm, "\t$ac, $shift");
+}
+
+class SHILO_R2_PSEUDO_BASE<SDPatternOperator OpNode, InstrItinClass itin,
+                           Instruction realinst> :
+  PseudoDSP<(outs), (ins CPURegs:$rs), [(OpNode CPURegs:$rs)]>,
+  PseudoInstExpansion<(realinst AC0, CPURegs:$rs)> {
+  list<Register> Defs = [DSPCtrl, AC0];
+  list<Register> Uses = [AC0];
+  InstrItinClass Itinerary = itin;
+}
+
+class SHILO_R2_DESC_BASE<string instr_asm> {
+  dag OutOperandList = (outs ACRegs:$ac);
+  dag InOperandList = (ins CPURegs:$rs);
+  string AsmString = !strconcat(instr_asm, "\t$ac, $rs");
+}
+
+class MTHLIP_DESC_BASE<string instr_asm> {
+  dag OutOperandList = (outs ACRegs:$ac);
+  dag InOperandList = (ins CPURegs:$rs);
+  string AsmString = !strconcat(instr_asm, "\t$rs, $ac");
+}
+
+class RDDSP_DESC_BASE<string instr_asm, SDPatternOperator OpNode,
+                      InstrItinClass itin> {
+  dag OutOperandList = (outs CPURegs:$rd);
+  dag InOperandList = (ins uimm16:$mask);
+  string AsmString = !strconcat(instr_asm, "\t$rd, $mask");
+  list<dag> Pattern = [(set CPURegs:$rd, (OpNode immZExt10:$mask))];
+  InstrItinClass Itinerary = itin;
+  list<Register> Uses = [DSPCtrl];
+}
+
+class WRDSP_DESC_BASE<string instr_asm, SDPatternOperator OpNode,
+                      InstrItinClass itin> {
+  dag OutOperandList = (outs);
+  dag InOperandList = (ins CPURegs:$rs, uimm16:$mask);
+  string AsmString = !strconcat(instr_asm, "\t$rs, $mask");
+  list<dag> Pattern = [(OpNode CPURegs:$rs, immZExt10:$mask)];
+  InstrItinClass Itinerary = itin;
+  list<Register> Defs = [DSPCtrl];
+}
+
+class DPA_W_PH_PSEUDO_BASE<SDPatternOperator OpNode, InstrItinClass itin,
+                           Instruction realinst> :
+  PseudoDSP<(outs), (ins CPURegs:$rs, CPURegs:$rt),
+            [(OpNode CPURegs:$rs, CPURegs:$rt)]>,
+  PseudoInstExpansion<(realinst AC0, CPURegs:$rs, CPURegs:$rt)> {
+  list<Register> Defs = [DSPCtrl, AC0];
+  list<Register> Uses = [AC0];
+  InstrItinClass Itinerary = itin;
+}
+
+class DPA_W_PH_DESC_BASE<string instr_asm> {
+  dag OutOperandList = (outs ACRegs:$ac);
+  dag InOperandList = (ins CPURegs:$rs, CPURegs:$rt);
+  string AsmString = !strconcat(instr_asm, "\t$ac, $rs, $rt");
+}
+
+class MULT_PSEUDO_BASE<SDPatternOperator OpNode, InstrItinClass itin,
+                       Instruction realinst> :
+  PseudoDSP<(outs), (ins CPURegs:$rs, CPURegs:$rt),
+            [(OpNode CPURegs:$rs, CPURegs:$rt)]>,
+  PseudoInstExpansion<(realinst AC0, CPURegs:$rs, CPURegs:$rt)> {
+  list<Register> Defs = [DSPCtrl, AC0];
+  InstrItinClass Itinerary = itin;
+}
+
+class MULT_DESC_BASE<string instr_asm> {
+  dag OutOperandList = (outs ACRegs:$ac);
+  dag InOperandList = (ins CPURegs:$rs, CPURegs:$rt);
+  string AsmString = !strconcat(instr_asm, "\t$ac, $rs, $rt");
+}
+
+class BPOSGE32_PSEUDO_DESC_BASE<SDPatternOperator OpNode, InstrItinClass itin> :
+  MipsPseudo<(outs CPURegs:$dst), (ins), "", [(set CPURegs:$dst, (OpNode))]> {
+  list<Register> Uses = [DSPCtrl];
+  bit usesCustomInserter = 1;
+}
+
+class BPOSGE32_DESC_BASE<string instr_asm, InstrItinClass itin> {
+  dag OutOperandList = (outs);
+  dag InOperandList = (ins brtarget:$offset);
+  string AsmString = !strconcat(instr_asm, "\t$offset");
+  InstrItinClass Itinerary = itin;
+  list<Register> Uses = [DSPCtrl];
+  bit isBranch = 1;
+  bit isTerminator = 1;
+  bit hasDelaySlot = 1;
+}
+
+class INSV_DESC_BASE<string instr_asm, SDPatternOperator OpNode,
+                     InstrItinClass itin> {
+  dag OutOperandList = (outs CPURegs:$rt);
+  dag InOperandList = (ins CPURegs:$src, CPURegs:$rs);
+  string AsmString = !strconcat(instr_asm, "\t$rt, $rs");
+  list<dag> Pattern = [(set CPURegs:$rt, (OpNode CPURegs:$src, CPURegs:$rs))];
+  InstrItinClass Itinerary = itin;
+  list<Register> Uses = [DSPCtrl];
+  string Constraints = "$src = $rt";
+}
+
+//===----------------------------------------------------------------------===//
+// MIPS DSP Rev 1
+//===----------------------------------------------------------------------===//
+
+// Addition/subtraction
+class ADDU_QB_DESC : ADDU_QB_DESC_BASE<"addu.qb", int_mips_addu_qb, NoItinerary,
+                                       DSPRegs, DSPRegs>, IsCommutable;
+
+class ADDU_S_QB_DESC : ADDU_QB_DESC_BASE<"addu_s.qb", int_mips_addu_s_qb,
+                                         NoItinerary, DSPRegs, DSPRegs>,
+                       IsCommutable;
+
+class SUBU_QB_DESC : ADDU_QB_DESC_BASE<"subu.qb", int_mips_subu_qb, NoItinerary,
+                                       DSPRegs, DSPRegs>;
+
+class SUBU_S_QB_DESC : ADDU_QB_DESC_BASE<"subu_s.qb", int_mips_subu_s_qb,
+                                         NoItinerary, DSPRegs, DSPRegs>;
+
+class ADDQ_PH_DESC : ADDU_QB_DESC_BASE<"addq.ph", int_mips_addq_ph, NoItinerary,
+                                       DSPRegs, DSPRegs>, IsCommutable;
+
+class ADDQ_S_PH_DESC : ADDU_QB_DESC_BASE<"addq_s.ph", int_mips_addq_s_ph,
+                                         NoItinerary, DSPRegs, DSPRegs>,
+                       IsCommutable;
+
+class SUBQ_PH_DESC : ADDU_QB_DESC_BASE<"subq.ph", int_mips_subq_ph, NoItinerary,
+                                       DSPRegs, DSPRegs>;
+
+class SUBQ_S_PH_DESC : ADDU_QB_DESC_BASE<"subq_s.ph", int_mips_subq_s_ph,
+                                         NoItinerary, DSPRegs, DSPRegs>;
+
+class ADDQ_S_W_DESC : ADDU_QB_DESC_BASE<"addq_s.w", int_mips_addq_s_w,
+                                        NoItinerary, CPURegs, CPURegs>,
+                      IsCommutable;
+
+class SUBQ_S_W_DESC : ADDU_QB_DESC_BASE<"subq_s.w", int_mips_subq_s_w,
+                                        NoItinerary, CPURegs, CPURegs>;
+
+class ADDSC_DESC : ADDU_QB_DESC_BASE<"addsc", int_mips_addsc, NoItinerary,
+                                     CPURegs, CPURegs>, IsCommutable;
+
+class ADDWC_DESC : ADDU_QB_DESC_BASE<"addwc", int_mips_addwc, NoItinerary,
+                                     CPURegs, CPURegs>,
+                   IsCommutable, UseDSPCtrl;
+
+class MODSUB_DESC : ADDU_QB_DESC_BASE<"modsub", int_mips_modsub, NoItinerary,
+                                      CPURegs, CPURegs>, ClearDefs;
+
+class RADDU_W_QB_DESC : RADDU_W_QB_DESC_BASE<"raddu.w.qb", int_mips_raddu_w_qb,
+                                             NoItinerary, CPURegs, DSPRegs>,
+                        ClearDefs;
+
+// Absolute value
+class ABSQ_S_PH_DESC : ABSQ_S_PH_R2_DESC_BASE<"absq_s.ph", int_mips_absq_s_ph,
+                                              NoItinerary, DSPRegs>;
+
+class ABSQ_S_W_DESC : ABSQ_S_PH_R2_DESC_BASE<"absq_s.w", int_mips_absq_s_w,
+                                             NoItinerary, CPURegs>;
+
+// Precision reduce/expand
+class PRECRQ_QB_PH_DESC : CMP_EQ_QB_R3_DESC_BASE<"precrq.qb.ph",
+                                                 int_mips_precrq_qb_ph,
+                                                 NoItinerary, DSPRegs, DSPRegs>,
+                          ClearDefs;
+
+class PRECRQ_PH_W_DESC : CMP_EQ_QB_R3_DESC_BASE<"precrq.ph.w",
+                                                int_mips_precrq_ph_w,
+                                                NoItinerary, DSPRegs, CPURegs>,
+                         ClearDefs;
+
+class PRECRQ_RS_PH_W_DESC : CMP_EQ_QB_R3_DESC_BASE<"precrq_rs.ph.w",
+                                                   int_mips_precrq_rs_ph_w,
+                                                   NoItinerary, DSPRegs,
+                                                   CPURegs>;
+
+class PRECRQU_S_QB_PH_DESC : CMP_EQ_QB_R3_DESC_BASE<"precrqu_s.qb.ph",
+                                                    int_mips_precrqu_s_qb_ph,
+                                                    NoItinerary, DSPRegs,
+                                                    DSPRegs>;
+
+class PRECEQ_W_PHL_DESC : ABSQ_S_PH_R2_DESC_BASE<"preceq.w.phl",
+                                                 int_mips_preceq_w_phl,
+                                                 NoItinerary, CPURegs, DSPRegs>,
+                          ClearDefs;
+
+class PRECEQ_W_PHR_DESC : ABSQ_S_PH_R2_DESC_BASE<"preceq.w.phr",
+                                                 int_mips_preceq_w_phr,
+                                                 NoItinerary, CPURegs, DSPRegs>,
+                          ClearDefs;
+
+class PRECEQU_PH_QBL_DESC : ABSQ_S_PH_R2_DESC_BASE<"precequ.ph.qbl",
+                                                   int_mips_precequ_ph_qbl,
+                                                   NoItinerary, DSPRegs>,
+                            ClearDefs;
+
+class PRECEQU_PH_QBR_DESC : ABSQ_S_PH_R2_DESC_BASE<"precequ.ph.qbr",
+                                                   int_mips_precequ_ph_qbr,
+                                                   NoItinerary, DSPRegs>,
+                            ClearDefs;
+
+class PRECEQU_PH_QBLA_DESC : ABSQ_S_PH_R2_DESC_BASE<"precequ.ph.qbla",
+                                                    int_mips_precequ_ph_qbla,
+                                                    NoItinerary, DSPRegs>,
+                             ClearDefs;
+
+class PRECEQU_PH_QBRA_DESC : ABSQ_S_PH_R2_DESC_BASE<"precequ.ph.qbra",
+                                                    int_mips_precequ_ph_qbra,
+                                                    NoItinerary, DSPRegs>,
+                             ClearDefs;
+
+class PRECEU_PH_QBL_DESC : ABSQ_S_PH_R2_DESC_BASE<"preceu.ph.qbl",
+                                                  int_mips_preceu_ph_qbl,
+                                                  NoItinerary, DSPRegs>,
+                           ClearDefs;
+
+class PRECEU_PH_QBR_DESC : ABSQ_S_PH_R2_DESC_BASE<"preceu.ph.qbr",
+                                                  int_mips_preceu_ph_qbr,
+                                                  NoItinerary, DSPRegs>,
+                           ClearDefs;
+
+class PRECEU_PH_QBLA_DESC : ABSQ_S_PH_R2_DESC_BASE<"preceu.ph.qbla",
+                                                   int_mips_preceu_ph_qbla,
+                                                   NoItinerary, DSPRegs>,
+                            ClearDefs;
+
+class PRECEU_PH_QBRA_DESC : ABSQ_S_PH_R2_DESC_BASE<"preceu.ph.qbra",
+                                                   int_mips_preceu_ph_qbra,
+                                                   NoItinerary, DSPRegs>,
+                            ClearDefs;
+
+// Shift
+class SHLL_QB_DESC : SHLL_QB_R2_DESC_BASE<"shll.qb", int_mips_shll_qb, immZExt3,
+                                          NoItinerary, DSPRegs>;
+
+class SHLLV_QB_DESC : SHLL_QB_R3_DESC_BASE<"shllv.qb", int_mips_shll_qb,
+                                           NoItinerary, DSPRegs>;
+
+class SHRL_QB_DESC : SHLL_QB_R2_DESC_BASE<"shrl.qb", int_mips_shrl_qb, immZExt3,
+                                          NoItinerary, DSPRegs>, ClearDefs;
+
+class SHRLV_QB_DESC : SHLL_QB_R3_DESC_BASE<"shrlv.qb", int_mips_shrl_qb,
+                                           NoItinerary, DSPRegs>, ClearDefs;
+
+class SHLL_PH_DESC : SHLL_QB_R2_DESC_BASE<"shll.ph", int_mips_shll_ph, immZExt4,
+                                          NoItinerary, DSPRegs>;
+
+class SHLLV_PH_DESC : SHLL_QB_R3_DESC_BASE<"shllv.ph", int_mips_shll_ph,
+                                           NoItinerary, DSPRegs>;
+
+class SHLL_S_PH_DESC : SHLL_QB_R2_DESC_BASE<"shll_s.ph", int_mips_shll_s_ph,
+                                            immZExt4, NoItinerary, DSPRegs>;
+
+class SHLLV_S_PH_DESC : SHLL_QB_R3_DESC_BASE<"shllv_s.ph", int_mips_shll_s_ph,
+                                             NoItinerary, DSPRegs>;
+
+class SHRA_PH_DESC : SHLL_QB_R2_DESC_BASE<"shra.ph", int_mips_shra_ph, immZExt4,
+                                          NoItinerary, DSPRegs>, ClearDefs;
+
+class SHRAV_PH_DESC : SHLL_QB_R3_DESC_BASE<"shrav.ph", int_mips_shra_ph,
+                                           NoItinerary, DSPRegs>, ClearDefs;
+
+class SHRA_R_PH_DESC : SHLL_QB_R2_DESC_BASE<"shra_r.ph", int_mips_shra_r_ph,
+                                            immZExt4, NoItinerary, DSPRegs>,
+                       ClearDefs;
+
+class SHRAV_R_PH_DESC : SHLL_QB_R3_DESC_BASE<"shrav_r.ph", int_mips_shra_r_ph,
+                                             NoItinerary, DSPRegs>, ClearDefs;
+
+class SHLL_S_W_DESC : SHLL_QB_R2_DESC_BASE<"shll_s.w", int_mips_shll_s_w,
+                                           immZExt5, NoItinerary, CPURegs>;
+
+class SHLLV_S_W_DESC : SHLL_QB_R3_DESC_BASE<"shllv_s.w", int_mips_shll_s_w,
+                                            NoItinerary, CPURegs>;
+
+class SHRA_R_W_DESC : SHLL_QB_R2_DESC_BASE<"shra_r.w", int_mips_shra_r_w,
+                                           immZExt5, NoItinerary, CPURegs>,
+                      ClearDefs;
+
+class SHRAV_R_W_DESC : SHLL_QB_R3_DESC_BASE<"shrav_r.w", int_mips_shra_r_w,
+                                            NoItinerary, CPURegs>;
+
+// Multiplication
+class MULEU_S_PH_QBL_DESC : ADDU_QB_DESC_BASE<"muleu_s.ph.qbl",
+                                              int_mips_muleu_s_ph_qbl,
+                                              NoItinerary, DSPRegs, DSPRegs>;
+
+class MULEU_S_PH_QBR_DESC : ADDU_QB_DESC_BASE<"muleu_s.ph.qbr",
+                                              int_mips_muleu_s_ph_qbr,
+                                              NoItinerary, DSPRegs, DSPRegs>;
+
+class MULEQ_S_W_PHL_DESC : ADDU_QB_DESC_BASE<"muleq_s.w.phl",
+                                             int_mips_muleq_s_w_phl,
+                                             NoItinerary, CPURegs, DSPRegs>,
+                           IsCommutable;
+
+class MULEQ_S_W_PHR_DESC : ADDU_QB_DESC_BASE<"muleq_s.w.phr",
+                                             int_mips_muleq_s_w_phr,
+                                             NoItinerary, CPURegs, DSPRegs>,
+                           IsCommutable;
+
+class MULQ_RS_PH_DESC : ADDU_QB_DESC_BASE<"mulq_rs.ph", int_mips_mulq_rs_ph,
+                                          NoItinerary, DSPRegs, DSPRegs>,
+                        IsCommutable;
+
+class MULSAQ_S_W_PH_DESC : DPA_W_PH_DESC_BASE<"mulsaq_s.w.ph">;
+
+class MAQ_S_W_PHL_DESC : DPA_W_PH_DESC_BASE<"maq_s.w.phl">;
+
+class MAQ_S_W_PHR_DESC : DPA_W_PH_DESC_BASE<"maq_s.w.phr">;
+
+class MAQ_SA_W_PHL_DESC : DPA_W_PH_DESC_BASE<"maq_sa.w.phl">;
+
+class MAQ_SA_W_PHR_DESC : DPA_W_PH_DESC_BASE<"maq_sa.w.phr">;
+
+// Dot product with accumulate/subtract
+class DPAU_H_QBL_DESC : DPA_W_PH_DESC_BASE<"dpau.h.qbl">;
+
+class DPAU_H_QBR_DESC : DPA_W_PH_DESC_BASE<"dpau.h.qbr">;
+
+class DPSU_H_QBL_DESC : DPA_W_PH_DESC_BASE<"dpsu.h.qbl">;
+
+class DPSU_H_QBR_DESC : DPA_W_PH_DESC_BASE<"dpsu.h.qbr">;
+
+class DPAQ_S_W_PH_DESC : DPA_W_PH_DESC_BASE<"dpaq_s.w.ph">;
+
+class DPSQ_S_W_PH_DESC : DPA_W_PH_DESC_BASE<"dpsq_s.w.ph">;
+
+class DPAQ_SA_L_W_DESC : DPA_W_PH_DESC_BASE<"dpaq_sa.l.w">;
+
+class DPSQ_SA_L_W_DESC : DPA_W_PH_DESC_BASE<"dpsq_sa.l.w">;
+
+class MULT_DSP_DESC : MULT_DESC_BASE<"mult">;
+
+class MULTU_DSP_DESC : MULT_DESC_BASE<"multu">;
+
+class MADD_DSP_DESC : MULT_DESC_BASE<"madd">;
+
+class MADDU_DSP_DESC : MULT_DESC_BASE<"maddu">;
+
+class MSUB_DSP_DESC : MULT_DESC_BASE<"msub">;
+
+class MSUBU_DSP_DESC : MULT_DESC_BASE<"msubu">;
+
+// Comparison
+class CMPU_EQ_QB_DESC : CMP_EQ_QB_R2_DESC_BASE<"cmpu.eq.qb",
+                                               int_mips_cmpu_eq_qb, NoItinerary,
+                                               DSPRegs>, IsCommutable;
+
+class CMPU_LT_QB_DESC : CMP_EQ_QB_R2_DESC_BASE<"cmpu.lt.qb",
+                                               int_mips_cmpu_lt_qb, NoItinerary,
+                                               DSPRegs>, IsCommutable;
+
+class CMPU_LE_QB_DESC : CMP_EQ_QB_R2_DESC_BASE<"cmpu.le.qb",
+                                               int_mips_cmpu_le_qb, NoItinerary,
+                                               DSPRegs>, IsCommutable;
+
+class CMPGU_EQ_QB_DESC : CMP_EQ_QB_R3_DESC_BASE<"cmpgu.eq.qb",
+                                                int_mips_cmpgu_eq_qb,
+                                                NoItinerary, CPURegs, DSPRegs>,
+                         IsCommutable;
+
+class CMPGU_LT_QB_DESC : CMP_EQ_QB_R3_DESC_BASE<"cmpgu.lt.qb",
+                                                int_mips_cmpgu_lt_qb,
+                                                NoItinerary, CPURegs, DSPRegs>,
+                         IsCommutable;
+
+class CMPGU_LE_QB_DESC : CMP_EQ_QB_R3_DESC_BASE<"cmpgu.le.qb",
+                                                int_mips_cmpgu_le_qb,
+                                                NoItinerary, CPURegs, DSPRegs>,
+                         IsCommutable;
+
+class CMP_EQ_PH_DESC : CMP_EQ_QB_R2_DESC_BASE<"cmp.eq.ph", int_mips_cmp_eq_ph,
+                                              NoItinerary, DSPRegs>,
+                       IsCommutable;
+
+class CMP_LT_PH_DESC : CMP_EQ_QB_R2_DESC_BASE<"cmp.lt.ph", int_mips_cmp_lt_ph,
+                                              NoItinerary, DSPRegs>,
+                       IsCommutable;
+
+class CMP_LE_PH_DESC : CMP_EQ_QB_R2_DESC_BASE<"cmp.le.ph", int_mips_cmp_le_ph,
+                                              NoItinerary, DSPRegs>,
+                       IsCommutable;
+
+// Misc
+class BITREV_DESC : ABSQ_S_PH_R2_DESC_BASE<"bitrev", int_mips_bitrev,
+                                           NoItinerary, CPURegs>, ClearDefs;
+
+class PACKRL_PH_DESC : CMP_EQ_QB_R3_DESC_BASE<"packrl.ph", int_mips_packrl_ph,
+                                              NoItinerary, DSPRegs, DSPRegs>,
+                       ClearDefs;
+
+class REPL_QB_DESC : REPL_DESC_BASE<"repl.qb", int_mips_repl_qb, immZExt8,
+                                    NoItinerary, DSPRegs>, ClearDefs;
+
+class REPL_PH_DESC : REPL_DESC_BASE<"repl.ph", int_mips_repl_ph, immZExt10,
+                                    NoItinerary, DSPRegs>, ClearDefs;
+
+class REPLV_QB_DESC : ABSQ_S_PH_R2_DESC_BASE<"replv.qb", int_mips_repl_qb,
+                                             NoItinerary, DSPRegs, CPURegs>,
+                      ClearDefs;
+
+class REPLV_PH_DESC : ABSQ_S_PH_R2_DESC_BASE<"replv.ph", int_mips_repl_ph,
+                                             NoItinerary, DSPRegs, CPURegs>,
+                      ClearDefs;
+
+class PICK_QB_DESC : CMP_EQ_QB_R3_DESC_BASE<"pick.qb", int_mips_pick_qb,
+                                            NoItinerary, DSPRegs, DSPRegs>,
+                     ClearDefs, UseDSPCtrl;
+
+class PICK_PH_DESC : CMP_EQ_QB_R3_DESC_BASE<"pick.ph", int_mips_pick_ph,
+                                            NoItinerary, DSPRegs, DSPRegs>,
+                     ClearDefs, UseDSPCtrl;
+
+class LWX_DESC : LX_DESC_BASE<"lwx", int_mips_lwx, NoItinerary>, ClearDefs;
+
+class LHX_DESC : LX_DESC_BASE<"lhx", int_mips_lhx, NoItinerary>, ClearDefs;
+
+class LBUX_DESC : LX_DESC_BASE<"lbux", int_mips_lbux, NoItinerary>, ClearDefs;
+
+class BPOSGE32_DESC : BPOSGE32_DESC_BASE<"bposge32", NoItinerary>;
+
+// Extr
+class EXTP_DESC : EXTR_W_TY1_R1_DESC_BASE<"extp", MipsEXTP, NoItinerary>;
+
+class EXTPV_DESC : EXTR_W_TY1_R2_DESC_BASE<"extpv", MipsEXTP, NoItinerary>;
+
+class EXTPDP_DESC : EXTR_W_TY1_R1_DESC_BASE<"extpdp", MipsEXTPDP, NoItinerary>;
+
+class EXTPDPV_DESC : EXTR_W_TY1_R2_DESC_BASE<"extpdpv", MipsEXTPDP,
+                                             NoItinerary>;
+
+class EXTR_W_DESC : EXTR_W_TY1_R1_DESC_BASE<"extr.w", MipsEXTR_W, NoItinerary>;
+
+class EXTRV_W_DESC : EXTR_W_TY1_R2_DESC_BASE<"extrv.w", MipsEXTR_W,
+                                             NoItinerary>;
+
+class EXTR_R_W_DESC : EXTR_W_TY1_R1_DESC_BASE<"extr_r.w", MipsEXTR_R_W,
+                                              NoItinerary>;
+
+class EXTRV_R_W_DESC : EXTR_W_TY1_R2_DESC_BASE<"extrv_r.w", MipsEXTR_R_W,
+                                               NoItinerary>;
+
+class EXTR_RS_W_DESC : EXTR_W_TY1_R1_DESC_BASE<"extr_rs.w", MipsEXTR_RS_W,
+                                               NoItinerary>;
+
+class EXTRV_RS_W_DESC : EXTR_W_TY1_R2_DESC_BASE<"extrv_rs.w", MipsEXTR_RS_W,
+                                                NoItinerary>;
+
+class EXTR_S_H_DESC : EXTR_W_TY1_R1_DESC_BASE<"extr_s.h", MipsEXTR_S_H,
+                                              NoItinerary>;
+
+class EXTRV_S_H_DESC : EXTR_W_TY1_R2_DESC_BASE<"extrv_s.h", MipsEXTR_S_H,
+                                               NoItinerary>;
+
+class SHILO_DESC : SHILO_R1_DESC_BASE<"shilo">;
+
+class SHILOV_DESC : SHILO_R2_DESC_BASE<"shilov">;
+
+class MTHLIP_DESC : MTHLIP_DESC_BASE<"mthlip">;
+
+class RDDSP_DESC : RDDSP_DESC_BASE<"rddsp", int_mips_rddsp, NoItinerary>;
+
+class WRDSP_DESC : WRDSP_DESC_BASE<"wrdsp", int_mips_wrdsp, NoItinerary>;
+
+class INSV_DESC : INSV_DESC_BASE<"insv", int_mips_insv, NoItinerary>;
+
+//===----------------------------------------------------------------------===//
+// MIPS DSP Rev 2
+// Addition/subtraction
+class ADDU_PH_DESC : ADDU_QB_DESC_BASE<"addu.ph", int_mips_addu_ph, NoItinerary,
+                                       DSPRegs, DSPRegs>, IsCommutable;
+
+class ADDU_S_PH_DESC : ADDU_QB_DESC_BASE<"addu_s.ph", int_mips_addu_s_ph,
+                                         NoItinerary, DSPRegs, DSPRegs>,
+                       IsCommutable;
+
+class SUBU_PH_DESC : ADDU_QB_DESC_BASE<"subu.ph", int_mips_subu_ph, NoItinerary,
+                                       DSPRegs, DSPRegs>;
+
+class SUBU_S_PH_DESC : ADDU_QB_DESC_BASE<"subu_s.ph", int_mips_subu_s_ph,
+                                         NoItinerary, DSPRegs, DSPRegs>;
+
+class ADDUH_QB_DESC : ADDUH_QB_DESC_BASE<"adduh.qb", int_mips_adduh_qb,
+                                         NoItinerary, DSPRegs>,
+                      ClearDefs, IsCommutable;
+
+class ADDUH_R_QB_DESC : ADDUH_QB_DESC_BASE<"adduh_r.qb", int_mips_adduh_r_qb,
+                                           NoItinerary, DSPRegs>,
+                        ClearDefs, IsCommutable;
+
+class SUBUH_QB_DESC : ADDUH_QB_DESC_BASE<"subuh.qb", int_mips_subuh_qb,
+                                         NoItinerary, DSPRegs>, ClearDefs;
+
+class SUBUH_R_QB_DESC : ADDUH_QB_DESC_BASE<"subuh_r.qb", int_mips_subuh_r_qb,
+                                           NoItinerary, DSPRegs>, ClearDefs;
+
+class ADDQH_PH_DESC : ADDUH_QB_DESC_BASE<"addqh.ph", int_mips_addqh_ph,
+                                         NoItinerary, DSPRegs>,
+                      ClearDefs, IsCommutable;
+
+class ADDQH_R_PH_DESC : ADDUH_QB_DESC_BASE<"addqh_r.ph", int_mips_addqh_r_ph,
+                                           NoItinerary, DSPRegs>,
+                        ClearDefs, IsCommutable;
+
+class SUBQH_PH_DESC : ADDUH_QB_DESC_BASE<"subqh.ph", int_mips_subqh_ph,
+                                         NoItinerary, DSPRegs>, ClearDefs;
+
+class SUBQH_R_PH_DESC : ADDUH_QB_DESC_BASE<"subqh_r.ph", int_mips_subqh_r_ph,
+                                           NoItinerary, DSPRegs>, ClearDefs;
+
+class ADDQH_W_DESC : ADDUH_QB_DESC_BASE<"addqh.w", int_mips_addqh_w,
+                                        NoItinerary, CPURegs>,
+                     ClearDefs, IsCommutable;
+
+class ADDQH_R_W_DESC : ADDUH_QB_DESC_BASE<"addqh_r.w", int_mips_addqh_r_w,
+                                          NoItinerary, CPURegs>,
+                       ClearDefs, IsCommutable;
+
+class SUBQH_W_DESC : ADDUH_QB_DESC_BASE<"subqh.w", int_mips_subqh_w,
+                                        NoItinerary, CPURegs>, ClearDefs;
+
+class SUBQH_R_W_DESC : ADDUH_QB_DESC_BASE<"subqh_r.w", int_mips_subqh_r_w,
+                                          NoItinerary, CPURegs>, ClearDefs;
+
+// Comparison
+class CMPGDU_EQ_QB_DESC : CMP_EQ_QB_R3_DESC_BASE<"cmpgdu.eq.qb",
+                                                 int_mips_cmpgdu_eq_qb,
+                                                 NoItinerary, CPURegs, DSPRegs>,
+                          IsCommutable;
+
+class CMPGDU_LT_QB_DESC : CMP_EQ_QB_R3_DESC_BASE<"cmpgdu.lt.qb",
+                                                 int_mips_cmpgdu_lt_qb,
+                                                 NoItinerary, CPURegs, DSPRegs>,
+                          IsCommutable;
+
+class CMPGDU_LE_QB_DESC : CMP_EQ_QB_R3_DESC_BASE<"cmpgdu.le.qb",
+                                                 int_mips_cmpgdu_le_qb,
+                                                 NoItinerary, CPURegs, DSPRegs>,
+                          IsCommutable;
+
+// Absolute
+class ABSQ_S_QB_DESC : ABSQ_S_PH_R2_DESC_BASE<"absq_s.qb", int_mips_absq_s_qb,
+                                              NoItinerary, DSPRegs>;
+
+// Multiplication
+class MUL_PH_DESC : ADDUH_QB_DESC_BASE<"mul.ph", int_mips_mul_ph, NoItinerary,
+                                       DSPRegs>, IsCommutable;
+
+class MUL_S_PH_DESC : ADDUH_QB_DESC_BASE<"mul_s.ph", int_mips_mul_s_ph,
+                                         NoItinerary, DSPRegs>, IsCommutable;
+
+class MULQ_S_W_DESC : ADDUH_QB_DESC_BASE<"mulq_s.w", int_mips_mulq_s_w,
+                                         NoItinerary, CPURegs>, IsCommutable;
+
+class MULQ_RS_W_DESC : ADDUH_QB_DESC_BASE<"mulq_rs.w", int_mips_mulq_rs_w,
+                                          NoItinerary, CPURegs>, IsCommutable;
+
+class MULQ_S_PH_DESC : ADDU_QB_DESC_BASE<"mulq_s.ph", int_mips_mulq_s_ph,
+                                         NoItinerary, DSPRegs, DSPRegs>,
+                       IsCommutable;
+
+// Dot product with accumulate/subtract
+class DPA_W_PH_DESC : DPA_W_PH_DESC_BASE<"dpa.w.ph">;
+
+class DPS_W_PH_DESC : DPA_W_PH_DESC_BASE<"dps.w.ph">;
+
+class DPAQX_S_W_PH_DESC : DPA_W_PH_DESC_BASE<"dpaqx_s.w.ph">;
+
+class DPAQX_SA_W_PH_DESC : DPA_W_PH_DESC_BASE<"dpaqx_sa.w.ph">;
+
+class DPAX_W_PH_DESC : DPA_W_PH_DESC_BASE<"dpax.w.ph">;
+
+class DPSX_W_PH_DESC : DPA_W_PH_DESC_BASE<"dpsx.w.ph">;
+
+class DPSQX_S_W_PH_DESC : DPA_W_PH_DESC_BASE<"dpsqx_s.w.ph">;
+
+class DPSQX_SA_W_PH_DESC : DPA_W_PH_DESC_BASE<"dpsqx_sa.w.ph">;
+
+class MULSA_W_PH_DESC : DPA_W_PH_DESC_BASE<"mulsa.w.ph">;
+
+// Precision reduce/expand
+class PRECR_QB_PH_DESC : CMP_EQ_QB_R3_DESC_BASE<"precr.qb.ph",
+                                                int_mips_precr_qb_ph,
+                                                NoItinerary, DSPRegs, DSPRegs>;
+
+class PRECR_SRA_PH_W_DESC : PRECR_SRA_PH_W_DESC_BASE<"precr_sra.ph.w",
+                                                     int_mips_precr_sra_ph_w,
+                                                     NoItinerary, DSPRegs,
+                                                     CPURegs>, ClearDefs;
+
+class PRECR_SRA_R_PH_W_DESC : PRECR_SRA_PH_W_DESC_BASE<"precr_sra_r.ph.w",
+                                                      int_mips_precr_sra_r_ph_w,
+                                                       NoItinerary, DSPRegs,
+                                                       CPURegs>, ClearDefs;
+
+// Shift
+class SHRA_QB_DESC : SHLL_QB_R2_DESC_BASE<"shra.qb", int_mips_shra_qb, immZExt3,
+                                          NoItinerary, DSPRegs>, ClearDefs;
+
+class SHRAV_QB_DESC : SHLL_QB_R3_DESC_BASE<"shrav.qb", int_mips_shra_qb,
+                                           NoItinerary, DSPRegs>, ClearDefs;
+
+class SHRA_R_QB_DESC : SHLL_QB_R2_DESC_BASE<"shra_r.qb", int_mips_shra_r_qb,
+                                            immZExt3, NoItinerary, DSPRegs>,
+                       ClearDefs;
+
+class SHRAV_R_QB_DESC : SHLL_QB_R3_DESC_BASE<"shrav_r.qb", int_mips_shra_r_qb,
+                                             NoItinerary, DSPRegs>, ClearDefs;
+
+class SHRL_PH_DESC : SHLL_QB_R2_DESC_BASE<"shrl.ph", int_mips_shrl_ph, immZExt4,
+                                          NoItinerary, DSPRegs>, ClearDefs;
+
+class SHRLV_PH_DESC : SHLL_QB_R3_DESC_BASE<"shrlv.ph", int_mips_shrl_ph,
+                                           NoItinerary, DSPRegs>, ClearDefs;
+
+// Misc
+class APPEND_DESC : APPEND_DESC_BASE<"append", int_mips_append, immZExt5,
+                                     NoItinerary>, ClearDefs;
+
+class BALIGN_DESC : APPEND_DESC_BASE<"balign", int_mips_balign, immZExt2,
+                                     NoItinerary>, ClearDefs;
+
+class PREPEND_DESC : APPEND_DESC_BASE<"prepend", int_mips_prepend, immZExt5,
+                                      NoItinerary>, ClearDefs;
+
+// Pseudos.
+def BPOSGE32_PSEUDO : BPOSGE32_PSEUDO_DESC_BASE<int_mips_bposge32, NoItinerary>;
+
+// Instruction defs.
+// MIPS DSP Rev 1
+def ADDU_QB : ADDU_QB_ENC, ADDU_QB_DESC;
+def ADDU_S_QB : ADDU_S_QB_ENC, ADDU_S_QB_DESC;
+def SUBU_QB : SUBU_QB_ENC, SUBU_QB_DESC;
+def SUBU_S_QB : SUBU_S_QB_ENC, SUBU_S_QB_DESC;
+def ADDQ_PH : ADDQ_PH_ENC, ADDQ_PH_DESC;
+def ADDQ_S_PH : ADDQ_S_PH_ENC, ADDQ_S_PH_DESC;
+def SUBQ_PH : SUBQ_PH_ENC, SUBQ_PH_DESC;
+def SUBQ_S_PH : SUBQ_S_PH_ENC, SUBQ_S_PH_DESC;
+def ADDQ_S_W : ADDQ_S_W_ENC, ADDQ_S_W_DESC;
+def SUBQ_S_W : SUBQ_S_W_ENC, SUBQ_S_W_DESC;
+def ADDSC : ADDSC_ENC, ADDSC_DESC;
+def ADDWC : ADDWC_ENC, ADDWC_DESC;
+def MODSUB : MODSUB_ENC, MODSUB_DESC;
+def RADDU_W_QB : RADDU_W_QB_ENC, RADDU_W_QB_DESC;
+def ABSQ_S_PH : ABSQ_S_PH_ENC, ABSQ_S_PH_DESC;
+def ABSQ_S_W : ABSQ_S_W_ENC, ABSQ_S_W_DESC;
+def PRECRQ_QB_PH : PRECRQ_QB_PH_ENC, PRECRQ_QB_PH_DESC;
+def PRECRQ_PH_W : PRECRQ_PH_W_ENC, PRECRQ_PH_W_DESC;
+def PRECRQ_RS_PH_W : PRECRQ_RS_PH_W_ENC, PRECRQ_RS_PH_W_DESC;
+def PRECRQU_S_QB_PH : PRECRQU_S_QB_PH_ENC, PRECRQU_S_QB_PH_DESC;
+def PRECEQ_W_PHL : PRECEQ_W_PHL_ENC, PRECEQ_W_PHL_DESC;
+def PRECEQ_W_PHR : PRECEQ_W_PHR_ENC, PRECEQ_W_PHR_DESC;
+def PRECEQU_PH_QBL : PRECEQU_PH_QBL_ENC, PRECEQU_PH_QBL_DESC;
+def PRECEQU_PH_QBR : PRECEQU_PH_QBR_ENC, PRECEQU_PH_QBR_DESC;
+def PRECEQU_PH_QBLA : PRECEQU_PH_QBLA_ENC, PRECEQU_PH_QBLA_DESC;
+def PRECEQU_PH_QBRA : PRECEQU_PH_QBRA_ENC, PRECEQU_PH_QBRA_DESC;
+def PRECEU_PH_QBL : PRECEU_PH_QBL_ENC, PRECEU_PH_QBL_DESC;
+def PRECEU_PH_QBR : PRECEU_PH_QBR_ENC, PRECEU_PH_QBR_DESC;
+def PRECEU_PH_QBLA : PRECEU_PH_QBLA_ENC, PRECEU_PH_QBLA_DESC;
+def PRECEU_PH_QBRA : PRECEU_PH_QBRA_ENC, PRECEU_PH_QBRA_DESC;
+def SHLL_QB : SHLL_QB_ENC, SHLL_QB_DESC;
+def SHLLV_QB : SHLLV_QB_ENC, SHLLV_QB_DESC;
+def SHRL_QB : SHRL_QB_ENC, SHRL_QB_DESC;
+def SHRLV_QB : SHRLV_QB_ENC, SHRLV_QB_DESC;
+def SHLL_PH : SHLL_PH_ENC, SHLL_PH_DESC;
+def SHLLV_PH : SHLLV_PH_ENC, SHLLV_PH_DESC;
+def SHLL_S_PH : SHLL_S_PH_ENC, SHLL_S_PH_DESC;
+def SHLLV_S_PH : SHLLV_S_PH_ENC, SHLLV_S_PH_DESC;
+def SHRA_PH : SHRA_PH_ENC, SHRA_PH_DESC;
+def SHRAV_PH : SHRAV_PH_ENC, SHRAV_PH_DESC;
+def SHRA_R_PH : SHRA_R_PH_ENC, SHRA_R_PH_DESC;
+def SHRAV_R_PH : SHRAV_R_PH_ENC, SHRAV_R_PH_DESC;
+def SHLL_S_W : SHLL_S_W_ENC, SHLL_S_W_DESC;
+def SHLLV_S_W : SHLLV_S_W_ENC, SHLLV_S_W_DESC;
+def SHRA_R_W : SHRA_R_W_ENC, SHRA_R_W_DESC;
+def SHRAV_R_W : SHRAV_R_W_ENC, SHRAV_R_W_DESC;
+def MULEU_S_PH_QBL : MULEU_S_PH_QBL_ENC, MULEU_S_PH_QBL_DESC;
+def MULEU_S_PH_QBR : MULEU_S_PH_QBR_ENC, MULEU_S_PH_QBR_DESC;
+def MULEQ_S_W_PHL : MULEQ_S_W_PHL_ENC, MULEQ_S_W_PHL_DESC;
+def MULEQ_S_W_PHR : MULEQ_S_W_PHR_ENC, MULEQ_S_W_PHR_DESC;
+def MULQ_RS_PH : MULQ_RS_PH_ENC, MULQ_RS_PH_DESC;
+def MULSAQ_S_W_PH : MULSAQ_S_W_PH_ENC, MULSAQ_S_W_PH_DESC;
+def MAQ_S_W_PHL : MAQ_S_W_PHL_ENC, MAQ_S_W_PHL_DESC;
+def MAQ_S_W_PHR : MAQ_S_W_PHR_ENC, MAQ_S_W_PHR_DESC;
+def MAQ_SA_W_PHL : MAQ_SA_W_PHL_ENC, MAQ_SA_W_PHL_DESC;
+def MAQ_SA_W_PHR : MAQ_SA_W_PHR_ENC, MAQ_SA_W_PHR_DESC;
+def DPAU_H_QBL : DPAU_H_QBL_ENC, DPAU_H_QBL_DESC;
+def DPAU_H_QBR : DPAU_H_QBR_ENC, DPAU_H_QBR_DESC;
+def DPSU_H_QBL : DPSU_H_QBL_ENC, DPSU_H_QBL_DESC;
+def DPSU_H_QBR : DPSU_H_QBR_ENC, DPSU_H_QBR_DESC;
+def DPAQ_S_W_PH : DPAQ_S_W_PH_ENC, DPAQ_S_W_PH_DESC;
+def DPSQ_S_W_PH : DPSQ_S_W_PH_ENC, DPSQ_S_W_PH_DESC;
+def DPAQ_SA_L_W : DPAQ_SA_L_W_ENC, DPAQ_SA_L_W_DESC;
+def DPSQ_SA_L_W : DPSQ_SA_L_W_ENC, DPSQ_SA_L_W_DESC;
+def MULT_DSP : MULT_DSP_ENC, MULT_DSP_DESC;
+def MULTU_DSP : MULTU_DSP_ENC, MULTU_DSP_DESC;
+def MADD_DSP : MADD_DSP_ENC, MADD_DSP_DESC;
+def MADDU_DSP : MADDU_DSP_ENC, MADDU_DSP_DESC;
+def MSUB_DSP : MSUB_DSP_ENC, MSUB_DSP_DESC;
+def MSUBU_DSP : MSUBU_DSP_ENC, MSUBU_DSP_DESC;
+def CMPU_EQ_QB : CMPU_EQ_QB_ENC, CMPU_EQ_QB_DESC;
+def CMPU_LT_QB : CMPU_LT_QB_ENC, CMPU_LT_QB_DESC;
+def CMPU_LE_QB : CMPU_LE_QB_ENC, CMPU_LE_QB_DESC;
+def CMPGU_EQ_QB : CMPGU_EQ_QB_ENC, CMPGU_EQ_QB_DESC;
+def CMPGU_LT_QB : CMPGU_LT_QB_ENC, CMPGU_LT_QB_DESC;
+def CMPGU_LE_QB : CMPGU_LE_QB_ENC, CMPGU_LE_QB_DESC;
+def CMP_EQ_PH : CMP_EQ_PH_ENC, CMP_EQ_PH_DESC;
+def CMP_LT_PH : CMP_LT_PH_ENC, CMP_LT_PH_DESC;
+def CMP_LE_PH : CMP_LE_PH_ENC, CMP_LE_PH_DESC;
+def BITREV : BITREV_ENC, BITREV_DESC;
+def PACKRL_PH : PACKRL_PH_ENC, PACKRL_PH_DESC;
+def REPL_QB : REPL_QB_ENC, REPL_QB_DESC;
+def REPL_PH : REPL_PH_ENC, REPL_PH_DESC;
+def REPLV_QB : REPLV_QB_ENC, REPLV_QB_DESC;
+def REPLV_PH : REPLV_PH_ENC, REPLV_PH_DESC;
+def PICK_QB : PICK_QB_ENC, PICK_QB_DESC;
+def PICK_PH : PICK_PH_ENC, PICK_PH_DESC;
+def LWX : LWX_ENC, LWX_DESC;
+def LHX : LHX_ENC, LHX_DESC;
+def LBUX : LBUX_ENC, LBUX_DESC;
+def BPOSGE32 : BPOSGE32_ENC, BPOSGE32_DESC;
+def INSV : INSV_ENC, INSV_DESC;
+def EXTP : EXTP_ENC, EXTP_DESC;
+def EXTPV : EXTPV_ENC, EXTPV_DESC;
+def EXTPDP : EXTPDP_ENC, EXTPDP_DESC;
+def EXTPDPV : EXTPDPV_ENC, EXTPDPV_DESC;
+def EXTR_W : EXTR_W_ENC, EXTR_W_DESC;
+def EXTRV_W : EXTRV_W_ENC, EXTRV_W_DESC;
+def EXTR_R_W : EXTR_R_W_ENC, EXTR_R_W_DESC;
+def EXTRV_R_W : EXTRV_R_W_ENC, EXTRV_R_W_DESC;
+def EXTR_RS_W : EXTR_RS_W_ENC, EXTR_RS_W_DESC;
+def EXTRV_RS_W : EXTRV_RS_W_ENC, EXTRV_RS_W_DESC;
+def EXTR_S_H : EXTR_S_H_ENC, EXTR_S_H_DESC;
+def EXTRV_S_H : EXTRV_S_H_ENC, EXTRV_S_H_DESC;
+def SHILO : SHILO_ENC, SHILO_DESC;
+def SHILOV : SHILOV_ENC, SHILOV_DESC;
+def MTHLIP : MTHLIP_ENC, MTHLIP_DESC;
+def RDDSP : RDDSP_ENC, RDDSP_DESC;
+def WRDSP : WRDSP_ENC, WRDSP_DESC;
+
+// MIPS DSP Rev 2
+let Predicates = [HasDSPR2] in {
+
+def ADDU_PH : ADDU_PH_ENC, ADDU_PH_DESC;
+def ADDU_S_PH : ADDU_S_PH_ENC, ADDU_S_PH_DESC;
+def SUBU_PH : SUBU_PH_ENC, SUBU_PH_DESC;
+def SUBU_S_PH : SUBU_S_PH_ENC, SUBU_S_PH_DESC;
+def CMPGDU_EQ_QB : CMPGDU_EQ_QB_ENC, CMPGDU_EQ_QB_DESC;
+def CMPGDU_LT_QB : CMPGDU_LT_QB_ENC, CMPGDU_LT_QB_DESC;
+def CMPGDU_LE_QB : CMPGDU_LE_QB_ENC, CMPGDU_LE_QB_DESC;
+def ABSQ_S_QB : ABSQ_S_QB_ENC, ABSQ_S_QB_DESC;
+def ADDUH_QB : ADDUH_QB_ENC, ADDUH_QB_DESC;
+def ADDUH_R_QB : ADDUH_R_QB_ENC, ADDUH_R_QB_DESC;
+def SUBUH_QB : SUBUH_QB_ENC, SUBUH_QB_DESC;
+def SUBUH_R_QB : SUBUH_R_QB_ENC, SUBUH_R_QB_DESC;
+def ADDQH_PH : ADDQH_PH_ENC, ADDQH_PH_DESC;
+def ADDQH_R_PH : ADDQH_R_PH_ENC, ADDQH_R_PH_DESC;
+def SUBQH_PH : SUBQH_PH_ENC, SUBQH_PH_DESC;
+def SUBQH_R_PH : SUBQH_R_PH_ENC, SUBQH_R_PH_DESC;
+def ADDQH_W : ADDQH_W_ENC, ADDQH_W_DESC;
+def ADDQH_R_W : ADDQH_R_W_ENC, ADDQH_R_W_DESC;
+def SUBQH_W : SUBQH_W_ENC, SUBQH_W_DESC;
+def SUBQH_R_W : SUBQH_R_W_ENC, SUBQH_R_W_DESC;
+def MUL_PH : MUL_PH_ENC, MUL_PH_DESC;
+def MUL_S_PH : MUL_S_PH_ENC, MUL_S_PH_DESC;
+def MULQ_S_W : MULQ_S_W_ENC, MULQ_S_W_DESC;
+def MULQ_RS_W : MULQ_RS_W_ENC, MULQ_RS_W_DESC;
+def MULQ_S_PH : MULQ_S_PH_ENC, MULQ_S_PH_DESC;
+def DPA_W_PH : DPA_W_PH_ENC, DPA_W_PH_DESC;
+def DPS_W_PH : DPS_W_PH_ENC, DPS_W_PH_DESC;
+def DPAQX_S_W_PH : DPAQX_S_W_PH_ENC, DPAQX_S_W_PH_DESC;
+def DPAQX_SA_W_PH : DPAQX_SA_W_PH_ENC, DPAQX_SA_W_PH_DESC;
+def DPAX_W_PH : DPAX_W_PH_ENC, DPAX_W_PH_DESC;
+def DPSX_W_PH : DPSX_W_PH_ENC, DPSX_W_PH_DESC;
+def DPSQX_S_W_PH : DPSQX_S_W_PH_ENC, DPSQX_S_W_PH_DESC;
+def DPSQX_SA_W_PH : DPSQX_SA_W_PH_ENC, DPSQX_SA_W_PH_DESC;
+def MULSA_W_PH : MULSA_W_PH_ENC, MULSA_W_PH_DESC;
+def PRECR_QB_PH : PRECR_QB_PH_ENC, PRECR_QB_PH_DESC;
+def PRECR_SRA_PH_W : PRECR_SRA_PH_W_ENC, PRECR_SRA_PH_W_DESC;
+def PRECR_SRA_R_PH_W : PRECR_SRA_R_PH_W_ENC, PRECR_SRA_R_PH_W_DESC;
+def SHRA_QB : SHRA_QB_ENC, SHRA_QB_DESC;
+def SHRAV_QB : SHRAV_QB_ENC, SHRAV_QB_DESC;
+def SHRA_R_QB : SHRA_R_QB_ENC, SHRA_R_QB_DESC;
+def SHRAV_R_QB : SHRAV_R_QB_ENC, SHRAV_R_QB_DESC;
+def SHRL_PH : SHRL_PH_ENC, SHRL_PH_DESC;
+def SHRLV_PH : SHRLV_PH_ENC, SHRLV_PH_DESC;
+def APPEND : APPEND_ENC, APPEND_DESC;
+def BALIGN : BALIGN_ENC, BALIGN_DESC;
+def PREPEND : PREPEND_ENC, PREPEND_DESC;
+
+}
+
+// Pseudos.
+def MULSAQ_S_W_PH_PSEUDO : DPA_W_PH_PSEUDO_BASE<MipsMULSAQ_S_W_PH, NoItinerary,
+                                                MULSAQ_S_W_PH>;
+def MAQ_S_W_PHL_PSEUDO : DPA_W_PH_PSEUDO_BASE<MipsMAQ_S_W_PHL, NoItinerary,
+                                              MAQ_S_W_PHL>;
+def MAQ_S_W_PHR_PSEUDO : DPA_W_PH_PSEUDO_BASE<MipsMAQ_S_W_PHR, NoItinerary,
+                                              MAQ_S_W_PHR>;
+def MAQ_SA_W_PHL_PSEUDO : DPA_W_PH_PSEUDO_BASE<MipsMAQ_SA_W_PHL, NoItinerary,
+                                               MAQ_SA_W_PHL>;
+def MAQ_SA_W_PHR_PSEUDO : DPA_W_PH_PSEUDO_BASE<MipsMAQ_SA_W_PHR, NoItinerary,
+                                               MAQ_SA_W_PHR>;
+def DPAU_H_QBL_PSEUDO : DPA_W_PH_PSEUDO_BASE<MipsDPAU_H_QBL, NoItinerary,
+                                             DPAU_H_QBL>;
+def DPAU_H_QBR_PSEUDO : DPA_W_PH_PSEUDO_BASE<MipsDPAU_H_QBR, NoItinerary,
+                                             DPAU_H_QBR>;
+def DPSU_H_QBL_PSEUDO : DPA_W_PH_PSEUDO_BASE<MipsDPSU_H_QBL, NoItinerary,
+                                             DPSU_H_QBL>;
+def DPSU_H_QBR_PSEUDO : DPA_W_PH_PSEUDO_BASE<MipsDPSU_H_QBR, NoItinerary,
+                                             DPSU_H_QBR>;
+def DPAQ_S_W_PH_PSEUDO : DPA_W_PH_PSEUDO_BASE<MipsDPAQ_S_W_PH, NoItinerary,
+                                              DPAQ_S_W_PH>;
+def DPSQ_S_W_PH_PSEUDO : DPA_W_PH_PSEUDO_BASE<MipsDPSQ_S_W_PH, NoItinerary,
+                                              DPSQ_S_W_PH>;
+def DPAQ_SA_L_W_PSEUDO : DPA_W_PH_PSEUDO_BASE<MipsDPAQ_SA_L_W, NoItinerary,
+                                              DPAQ_SA_L_W>;
+def DPSQ_SA_L_W_PSEUDO : DPA_W_PH_PSEUDO_BASE<MipsDPSQ_SA_L_W, NoItinerary,
+                                              DPSQ_SA_L_W>;
+
+def MULT_DSP_PSEUDO : MULT_PSEUDO_BASE<MipsMULT, NoItinerary, MULT_DSP>,
+                      IsCommutable;
+def MULTU_DSP_PSEUDO : MULT_PSEUDO_BASE<MipsMULTU, NoItinerary, MULTU_DSP>,
+                       IsCommutable;
+def MADD_DSP_PSEUDO : MULT_PSEUDO_BASE<MipsMADD_DSP, NoItinerary, MADD_DSP>,
+                      IsCommutable, UseAC;
+def MADDU_DSP_PSEUDO : MULT_PSEUDO_BASE<MipsMADDU_DSP, NoItinerary, MADDU_DSP>,
+                       IsCommutable, UseAC;
+def MSUB_DSP_PSEUDO : MULT_PSEUDO_BASE<MipsMSUB_DSP, NoItinerary, MSUB_DSP>,
+                      UseAC;
+def MSUBU_DSP_PSEUDO : MULT_PSEUDO_BASE<MipsMSUBU_DSP, NoItinerary, MSUBU_DSP>,
+                       UseAC;
+
+def SHILO_PSEUDO : SHILO_R1_PSEUDO_BASE<MipsSHILO, NoItinerary, SHILO>;
+def SHILOV_PSEUDO : SHILO_R2_PSEUDO_BASE<MipsSHILO, NoItinerary, SHILOV>;
+def MTHLIP_PSEUDO : SHILO_R2_PSEUDO_BASE<MipsMTHLIP, NoItinerary, MTHLIP>;
+
+let Predicates = [HasDSPR2] in {
+
+def DPA_W_PH_PSEUDO : DPA_W_PH_PSEUDO_BASE<MipsDPA_W_PH, NoItinerary, DPA_W_PH>;
+def DPS_W_PH_PSEUDO : DPA_W_PH_PSEUDO_BASE<MipsDPS_W_PH, NoItinerary, DPS_W_PH>;
+def DPAQX_S_W_PH_PSEUDO : DPA_W_PH_PSEUDO_BASE<MipsDPAQX_S_W_PH, NoItinerary,
+                                               DPAQX_S_W_PH>;
+def DPAQX_SA_W_PH_PSEUDO : DPA_W_PH_PSEUDO_BASE<MipsDPAQX_SA_W_PH, NoItinerary,
+                                                DPAQX_SA_W_PH>;
+def DPAX_W_PH_PSEUDO : DPA_W_PH_PSEUDO_BASE<MipsDPAX_W_PH, NoItinerary,
+                                            DPAX_W_PH>;
+def DPSX_W_PH_PSEUDO : DPA_W_PH_PSEUDO_BASE<MipsDPSX_W_PH, NoItinerary,
+                                            DPSX_W_PH>;
+def DPSQX_S_W_PH_PSEUDO : DPA_W_PH_PSEUDO_BASE<MipsDPSQX_S_W_PH, NoItinerary,
+                                               DPSQX_S_W_PH>;
+def DPSQX_SA_W_PH_PSEUDO : DPA_W_PH_PSEUDO_BASE<MipsDPSQX_SA_W_PH, NoItinerary,
+                                                DPSQX_SA_W_PH>;
+def MULSA_W_PH_PSEUDO : DPA_W_PH_PSEUDO_BASE<MipsMULSA_W_PH, NoItinerary,
+                                             MULSA_W_PH>;
+
+}
+
+// Patterns.
+class DSPPat<dag pattern, dag result, Predicate pred = HasDSP> :
+  Pat<pattern, result>, Requires<[pred]>;
+
+class BitconvertPat<ValueType DstVT, ValueType SrcVT, RegisterClass DstRC,
+                    RegisterClass SrcRC> :
+   DSPPat<(DstVT (bitconvert (SrcVT SrcRC:$src))),
+          (COPY_TO_REGCLASS SrcRC:$src, DstRC)>;
+
+def : BitconvertPat<i32, v2i16, CPURegs, DSPRegs>;
+def : BitconvertPat<i32, v4i8, CPURegs, DSPRegs>;
+def : BitconvertPat<v2i16, i32, DSPRegs, CPURegs>;
+def : BitconvertPat<v4i8, i32, DSPRegs, CPURegs>;
+
+def : DSPPat<(v2i16 (load addr:$a)),
+             (v2i16 (COPY_TO_REGCLASS (LW addr:$a), DSPRegs))>;
+def : DSPPat<(v4i8 (load addr:$a)),
+             (v4i8 (COPY_TO_REGCLASS (LW addr:$a), DSPRegs))>;
+def : DSPPat<(store (v2i16 DSPRegs:$val), addr:$a),
+             (SW (COPY_TO_REGCLASS DSPRegs:$val, CPURegs), addr:$a)>;
+def : DSPPat<(store (v4i8 DSPRegs:$val), addr:$a),
+             (SW (COPY_TO_REGCLASS DSPRegs:$val, CPURegs), addr:$a)>;
+
+// Extr patterns.
+class EXTR_W_TY1_R2_Pat<SDPatternOperator OpNode, Instruction Instr> :
+  DSPPat<(i32 (OpNode CPURegs:$rs)), (Instr AC0, CPURegs:$rs)>;
+
+class EXTR_W_TY1_R1_Pat<SDPatternOperator OpNode, Instruction Instr> :
+  DSPPat<(i32 (OpNode immZExt5:$shift)), (Instr AC0, immZExt5:$shift)>;
+
+def : EXTR_W_TY1_R1_Pat<MipsEXTP, EXTP>;
+def : EXTR_W_TY1_R2_Pat<MipsEXTP, EXTPV>;
+def : EXTR_W_TY1_R1_Pat<MipsEXTPDP, EXTPDP>;
+def : EXTR_W_TY1_R2_Pat<MipsEXTPDP, EXTPDPV>;
+def : EXTR_W_TY1_R1_Pat<MipsEXTR_W, EXTR_W>;
+def : EXTR_W_TY1_R2_Pat<MipsEXTR_W, EXTRV_W>;
+def : EXTR_W_TY1_R1_Pat<MipsEXTR_R_W, EXTR_R_W>;
+def : EXTR_W_TY1_R2_Pat<MipsEXTR_R_W, EXTRV_R_W>;
+def : EXTR_W_TY1_R1_Pat<MipsEXTR_RS_W, EXTR_RS_W>;
+def : EXTR_W_TY1_R2_Pat<MipsEXTR_RS_W, EXTRV_RS_W>;
+def : EXTR_W_TY1_R1_Pat<MipsEXTR_S_H, EXTR_S_H>;
+def : EXTR_W_TY1_R2_Pat<MipsEXTR_S_H, EXTRV_S_H>;
diff --git a/lib/Target/Mips/MipsDelaySlotFiller.cpp b/lib/Target/Mips/MipsDelaySlotFiller.cpp
index 2bba8a3..e3c8ed7 100644
--- a/lib/Target/Mips/MipsDelaySlotFiller.cpp
+++ b/lib/Target/Mips/MipsDelaySlotFiller.cpp
@@ -30,10 +30,11 @@ STATISTIC(FilledSlots, "Number of delay slots filled");
 STATISTIC(UsefulSlots, "Number of delay slots filled with instructions that"
                        " are not NOP.");
 
-static cl::opt<bool> EnableDelaySlotFiller(
-  "enable-mips-delay-filler",
+static cl::opt<bool> DisableDelaySlotFiller(
+  "disable-mips-delay-filler",
   cl::init(false),
-  cl::desc("Fill the Mips delay slots useful instructions."),
+  cl::desc("Disable the delay slot filler, which attempts to fill the Mips"
+           "delay slots with useful instructions."),
   cl::Hidden);
 
 // This option can be used to silence complaints by machine verifier passes.
@@ -114,7 +115,9 @@ runOnMachineBasicBlock(MachineBasicBlock &MBB) {
 
       InstrIter D;
 
-      if (EnableDelaySlotFiller && findDelayInstr(MBB, I, D)) {
+      // Delay slot filling is disabled at -O0.
+      if (!DisableDelaySlotFiller && (TM.getOptLevel() != CodeGenOpt::None) &&
+          findDelayInstr(MBB, I, D)) {
         MBB.splice(llvm::next(I), &MBB, D);
         ++UsefulSlots;
       } else
diff --git a/lib/Target/Mips/MipsFrameLowering.cpp b/lib/Target/Mips/MipsFrameLowering.cpp
index 8c0474b..2cad2a6 100644
--- a/lib/Target/Mips/MipsFrameLowering.cpp
+++ b/lib/Target/Mips/MipsFrameLowering.cpp
@@ -23,7 +23,7 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetOptions.h"
 #include "llvm/Support/CommandLine.h"
 
@@ -98,3 +98,37 @@ bool MipsFrameLowering::hasFP(const MachineFunction &MF) const {
   return MF.getTarget().Options.DisableFramePointerElim(MF) ||
       MFI->hasVarSizedObjects() || MFI->isFrameAddressTaken();
 }
+
+uint64_t MipsFrameLowering::estimateStackSize(const MachineFunction &MF) const {
+  const MachineFrameInfo *MFI = MF.getFrameInfo();
+  const TargetRegisterInfo &TRI = *MF.getTarget().getRegisterInfo();
+
+  int64_t Offset = 0;
+
+  // Iterate over fixed sized objects.
+  for (int I = MFI->getObjectIndexBegin(); I != 0; ++I)
+    Offset = std::max(Offset, -MFI->getObjectOffset(I));
+
+  // Conservatively assume all callee-saved registers will be saved.
+  for (const uint16_t *R = TRI.getCalleeSavedRegs(&MF); *R; ++R) {
+    unsigned Size = TRI.getMinimalPhysRegClass(*R)->getSize();
+    Offset = RoundUpToAlignment(Offset + Size, Size);
+  }
+
+  unsigned MaxAlign = MFI->getMaxAlignment();
+
+  // Check that MaxAlign is not zero if there is a stack object that is not a
+  // callee-saved spill.
+  assert(!MFI->getObjectIndexEnd() || MaxAlign);
+
+  // Iterate over other objects.
+  for (unsigned I = 0, E = MFI->getObjectIndexEnd(); I != E; ++I)
+    Offset = RoundUpToAlignment(Offset + MFI->getObjectSize(I), MaxAlign);
+
+  // Call frame.
+  if (MFI->adjustsStack() && hasReservedCallFrame(MF))
+    Offset = RoundUpToAlignment(Offset + MFI->getMaxCallFrameSize(),
+                                std::max(MaxAlign, getStackAlignment()));
+
+  return RoundUpToAlignment(Offset, getStackAlignment());
+}
diff --git a/lib/Target/Mips/MipsFrameLowering.h b/lib/Target/Mips/MipsFrameLowering.h
index ed7b7fe..df52d92 100644
--- a/lib/Target/Mips/MipsFrameLowering.h
+++ b/lib/Target/Mips/MipsFrameLowering.h
@@ -34,6 +34,9 @@ public:
                                          const MipsSubtarget &ST);
 
   bool hasFP(const MachineFunction &MF) const;
+
+protected:
+  uint64_t estimateStackSize(const MachineFunction &MF) const;
 };
 
 /// Create MipsInstrInfo objects.
diff --git a/lib/Target/Mips/MipsISelDAGToDAG.cpp b/lib/Target/Mips/MipsISelDAGToDAG.cpp
index 5a97c17..c5fca7f 100644
--- a/lib/Target/Mips/MipsISelDAGToDAG.cpp
+++ b/lib/Target/Mips/MipsISelDAGToDAG.cpp
@@ -86,6 +86,10 @@ private:
 
   SDNode *getGlobalBaseReg();
 
+  SDValue getMips16SPAliasReg();
+
+  void getMips16SPRefReg(SDNode *parent, SDValue &AliasReg);
+
   std::pair<SDNode*, SDNode*> SelectMULT(SDNode *N, unsigned Opc, DebugLoc dl,
                                          EVT Ty, bool HasLo, bool HasHi);
 
@@ -94,6 +98,9 @@ private:
   // Complex Pattern.
   bool SelectAddr(SDNode *Parent, SDValue N, SDValue &Base, SDValue &Offset);
 
+  bool SelectAddr16(SDNode *Parent, SDValue N, SDValue &Base, SDValue &Offset,
+       SDValue &Alias);
+
   // getImm - Return a target constant with the specified value.
   inline SDValue getImm(const SDNode *Node, unsigned Imm) {
     return CurDAG->getTargetConstant(Imm, Node->getValueType(0));
@@ -102,6 +109,7 @@ private:
   void ProcessFunctionAfterISel(MachineFunction &MF);
   bool ReplaceUsesWithZeroReg(MachineRegisterInfo *MRI, const MachineInstr&);
   void InitGlobalBaseReg(MachineFunction &MF);
+  void InitMips16SPAliasReg(MachineFunction &MF);
 
   virtual bool SelectInlineAsmMemoryOperand(const SDValue &Op,
                                             char ConstraintCode,
@@ -220,6 +228,26 @@ void MipsDAGToDAGISel::InitGlobalBaseReg(MachineFunction &MF) {
     .addReg(Mips::V0).addReg(Mips::T9);
 }
 
+// Insert instructions to initialize the Mips16 SP Alias register in the
+// first MBB of the function.
+//
+void MipsDAGToDAGISel::InitMips16SPAliasReg(MachineFunction &MF) {
+  MipsFunctionInfo *MipsFI = MF.getInfo<MipsFunctionInfo>();
+
+  if (!MipsFI->mips16SPAliasRegSet())
+    return;
+
+  MachineBasicBlock &MBB = MF.front();
+  MachineBasicBlock::iterator I = MBB.begin();
+  const TargetInstrInfo &TII = *MF.getTarget().getInstrInfo();
+  DebugLoc DL = I != MBB.end() ? I->getDebugLoc() : DebugLoc();
+  unsigned Mips16SPAliasReg = MipsFI->getMips16SPAliasReg();
+
+  BuildMI(MBB, I, DL, TII.get(Mips::MoveR3216), Mips16SPAliasReg)
+    .addReg(Mips::SP);
+}
+
+
 bool MipsDAGToDAGISel::ReplaceUsesWithZeroReg(MachineRegisterInfo *MRI,
                                               const MachineInstr& MI) {
   unsigned DstReg = 0, ZeroReg = 0;
@@ -260,6 +288,7 @@ bool MipsDAGToDAGISel::ReplaceUsesWithZeroReg(MachineRegisterInfo *MRI,
 
 void MipsDAGToDAGISel::ProcessFunctionAfterISel(MachineFunction &MF) {
   InitGlobalBaseReg(MF);
+  InitMips16SPAliasReg(MF);
 
   MachineRegisterInfo *MRI = &MF.getRegInfo();
 
@@ -284,6 +313,14 @@ SDNode *MipsDAGToDAGISel::getGlobalBaseReg() {
   return CurDAG->getRegister(GlobalBaseReg, TLI.getPointerTy()).getNode();
 }
 
+/// getMips16SPAliasReg - Output the instructions required to put the
+/// SP into a Mips16 accessible aliased register.
+SDValue MipsDAGToDAGISel::getMips16SPAliasReg() {
+  unsigned Mips16SPAliasReg =
+    MF->getInfo<MipsFunctionInfo>()->getMips16SPAliasReg();
+  return CurDAG->getRegister(Mips16SPAliasReg, TLI.getPointerTy());
+}
+
 /// ComplexPattern used on MipsInstrInfo
 /// Used on Mips Load/Store instructions
 bool MipsDAGToDAGISel::
@@ -337,8 +374,9 @@ SelectAddr(SDNode *Parent, SDValue Addr, SDValue &Base, SDValue &Offset) {
     // Generate:
     //  lui $2, %hi($CPI1_0)
     //  lwc1 $f0, %lo($CPI1_0)($2)
-    if (Addr.getOperand(1).getOpcode() == MipsISD::Lo) {
-      SDValue LoVal = Addr.getOperand(1), Opnd0 = LoVal.getOperand(0);
+    if (Addr.getOperand(1).getOpcode() == MipsISD::Lo ||
+        Addr.getOperand(1).getOpcode() == MipsISD::GPRel) {
+      SDValue Opnd0 = Addr.getOperand(1).getOperand(0);
       if (isa<ConstantPoolSDNode>(Opnd0) || isa<GlobalAddressSDNode>(Opnd0) ||
           isa<JumpTableSDNode>(Opnd0)) {
         Base = Addr.getOperand(0);
@@ -361,6 +399,115 @@ SelectAddr(SDNode *Parent, SDValue Addr, SDValue &Base, SDValue &Offset) {
   return true;
 }
 
+void MipsDAGToDAGISel::getMips16SPRefReg(SDNode *Parent, SDValue &AliasReg) {
+  SDValue AliasFPReg = CurDAG->getRegister(Mips::S0, TLI.getPointerTy());
+  if (Parent) {
+    switch (Parent->getOpcode()) {
+      case ISD::LOAD: {
+        LoadSDNode *SD = dyn_cast<LoadSDNode>(Parent);
+        switch (SD->getMemoryVT().getSizeInBits()) {
+        case 8:
+        case 16:
+          AliasReg = TM.getFrameLowering()->hasFP(*MF)?
+            AliasFPReg: getMips16SPAliasReg();
+          return;
+        }
+        break;
+      }
+      case ISD::STORE: {
+        StoreSDNode *SD = dyn_cast<StoreSDNode>(Parent);
+        switch (SD->getMemoryVT().getSizeInBits()) {
+        case 8:
+        case 16:
+          AliasReg = TM.getFrameLowering()->hasFP(*MF)?
+            AliasFPReg: getMips16SPAliasReg();
+          return;
+        }
+        break;
+      }
+    }
+  }
+  AliasReg = CurDAG->getRegister(Mips::SP, TLI.getPointerTy());
+  return;
+
+}
+bool MipsDAGToDAGISel::SelectAddr16(
+  SDNode *Parent, SDValue Addr, SDValue &Base, SDValue &Offset,
+  SDValue &Alias) {
+  EVT ValTy = Addr.getValueType();
+
+  Alias = CurDAG->getTargetConstant(0, ValTy);
+
+  // if Address is FI, get the TargetFrameIndex.
+  if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>(Addr)) {
+    Base   = CurDAG->getTargetFrameIndex(FIN->getIndex(), ValTy);
+    Offset = CurDAG->getTargetConstant(0, ValTy);
+    getMips16SPRefReg(Parent, Alias);
+    return true;
+  }
+  // on PIC code Load GA
+  if (Addr.getOpcode() == MipsISD::Wrapper) {
+    Base   = Addr.getOperand(0);
+    Offset = Addr.getOperand(1);
+    return true;
+  }
+  if (TM.getRelocationModel() != Reloc::PIC_) {
+    if ((Addr.getOpcode() == ISD::TargetExternalSymbol ||
+        Addr.getOpcode() == ISD::TargetGlobalAddress))
+      return false;
+  }
+  // Addresses of the form FI+const or FI|const
+  if (CurDAG->isBaseWithConstantOffset(Addr)) {
+    ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Addr.getOperand(1));
+    if (isInt<16>(CN->getSExtValue())) {
+
+      // If the first operand is a FI, get the TargetFI Node
+      if (FrameIndexSDNode *FIN = dyn_cast<FrameIndexSDNode>
+                                  (Addr.getOperand(0))) {
+        Base = CurDAG->getTargetFrameIndex(FIN->getIndex(), ValTy);
+        getMips16SPRefReg(Parent, Alias);
+      }
+      else
+        Base = Addr.getOperand(0);
+
+      Offset = CurDAG->getTargetConstant(CN->getZExtValue(), ValTy);
+      return true;
+    }
+  }
+  // Operand is a result from an ADD.
+  if (Addr.getOpcode() == ISD::ADD) {
+    // When loading from constant pools, load the lower address part in
+    // the instruction itself. Example, instead of:
+    //  lui $2, %hi($CPI1_0)
+    //  addiu $2, $2, %lo($CPI1_0)
+    //  lwc1 $f0, 0($2)
+    // Generate:
+    //  lui $2, %hi($CPI1_0)
+    //  lwc1 $f0, %lo($CPI1_0)($2)
+    if (Addr.getOperand(1).getOpcode() == MipsISD::Lo ||
+        Addr.getOperand(1).getOpcode() == MipsISD::GPRel) {
+      SDValue Opnd0 = Addr.getOperand(1).getOperand(0);
+      if (isa<ConstantPoolSDNode>(Opnd0) || isa<GlobalAddressSDNode>(Opnd0) ||
+          isa<JumpTableSDNode>(Opnd0)) {
+        Base = Addr.getOperand(0);
+        Offset = Opnd0;
+        return true;
+      }
+    }
+
+    // If an indexed floating point load/store can be emitted, return false.
+    const LSBaseSDNode *LS = dyn_cast<LSBaseSDNode>(Parent);
+
+    if (LS &&
+        (LS->getMemoryVT() == MVT::f32 || LS->getMemoryVT() == MVT::f64) &&
+        Subtarget.hasMips32r2Or64())
+      return false;
+  }
+  Base   = Addr;
+  Offset = CurDAG->getTargetConstant(0, ValTy);
+  return true;
+}
+
 /// Select multiply instructions.
 std::pair<SDNode*, SDNode*>
 MipsDAGToDAGISel::SelectMULT(SDNode *N, unsigned Opc, DebugLoc dl, EVT Ty,
@@ -371,14 +518,16 @@ MipsDAGToDAGISel::SelectMULT(SDNode *N, unsigned Opc, DebugLoc dl, EVT Ty,
   SDValue InFlag = SDValue(Mul, 0);
 
   if (HasLo) {
-    Lo = CurDAG->getMachineNode(Ty == MVT::i32 ? Mips::MFLO : Mips::MFLO64, dl,
-                                Ty, MVT::Glue, InFlag);
+    unsigned Opcode = Subtarget.inMips16Mode() ? Mips::Mflo16 :
+      (Ty == MVT::i32 ? Mips::MFLO : Mips::MFLO64);
+    Lo = CurDAG->getMachineNode(Opcode, dl, Ty, MVT::Glue, InFlag);
     InFlag = SDValue(Lo, 1);
   }
-  if (HasHi)
-    Hi = CurDAG->getMachineNode(Ty == MVT::i32 ? Mips::MFHI : Mips::MFHI64, dl,
-                                Ty, InFlag);
-
+  if (HasHi) {
+    unsigned Opcode = Subtarget.inMips16Mode() ? Mips::Mfhi16 :
+      (Ty == MVT::i32 ? Mips::MFHI : Mips::MFHI64);
+    Hi = CurDAG->getMachineNode(Opcode, dl, Ty, InFlag);
+  }
   return std::make_pair(Lo, Hi);
 }
 
@@ -410,6 +559,7 @@ SDNode* MipsDAGToDAGISel::Select(SDNode *Node) {
 
   case ISD::SUBE:
   case ISD::ADDE: {
+    bool inMips16Mode = Subtarget.inMips16Mode();
     SDValue InFlag = Node->getOperand(2), CmpLHS;
     unsigned Opc = InFlag.getOpcode(); (void)Opc;
     assert(((Opc == ISD::ADDC || Opc == ISD::ADDE) ||
@@ -419,10 +569,16 @@ SDNode* MipsDAGToDAGISel::Select(SDNode *Node) {
     unsigned MOp;
     if (Opcode == ISD::ADDE) {
       CmpLHS = InFlag.getValue(0);
-      MOp = Mips::ADDu;
+      if (inMips16Mode)
+        MOp = Mips::AdduRxRyRz16;
+      else
+        MOp = Mips::ADDu;
     } else {
       CmpLHS = InFlag.getOperand(0);
-      MOp = Mips::SUBu;
+      if (inMips16Mode)
+        MOp = Mips::SubuRxRyRz16;
+      else
+        MOp = Mips::SUBu;
     }
 
     SDValue Ops[] = { CmpLHS, InFlag.getOperand(1) };
@@ -431,8 +587,11 @@ SDNode* MipsDAGToDAGISel::Select(SDNode *Node) {
     SDValue RHS = Node->getOperand(1);
 
     EVT VT = LHS.getValueType();
-    SDNode *Carry = CurDAG->getMachineNode(Mips::SLTu, dl, VT, Ops, 2);
-    SDNode *AddCarry = CurDAG->getMachineNode(Mips::ADDu, dl, VT,
+
+    unsigned Sltu_op = inMips16Mode? Mips::SltuRxRyRz16: Mips::SLTu;
+    SDNode *Carry = CurDAG->getMachineNode(Sltu_op, dl, VT, Ops, 2);
+    unsigned Addu_op = inMips16Mode? Mips::AdduRxRyRz16 : Mips::ADDu;
+    SDNode *AddCarry = CurDAG->getMachineNode(Addu_op, dl, VT,
                                               SDValue(Carry,0), RHS);
 
     return CurDAG->SelectNodeTo(Node, MOp, VT, MVT::Glue,
@@ -442,8 +601,13 @@ SDNode* MipsDAGToDAGISel::Select(SDNode *Node) {
   /// Mul with two results
   case ISD::SMUL_LOHI:
   case ISD::UMUL_LOHI: {
-    if (NodeTy == MVT::i32)
-      MultOpc = (Opcode == ISD::UMUL_LOHI ? Mips::MULTu : Mips::MULT);
+    if (NodeTy == MVT::i32) {
+      if (Subtarget.inMips16Mode())
+        MultOpc = (Opcode == ISD::UMUL_LOHI ? Mips::MultuRxRy16 :
+                   Mips::MultRxRy16);
+      else
+        MultOpc = (Opcode == ISD::UMUL_LOHI ? Mips::MULTu : Mips::MULT);
+    }
     else
       MultOpc = (Opcode == ISD::UMUL_LOHI ? Mips::DMULTu : Mips::DMULT);
 
@@ -469,8 +633,13 @@ SDNode* MipsDAGToDAGISel::Select(SDNode *Node) {
   }
   case ISD::MULHS:
   case ISD::MULHU: {
-    if (NodeTy == MVT::i32)
-      MultOpc = (Opcode == ISD::MULHU ? Mips::MULTu : Mips::MULT);
+    if (NodeTy == MVT::i32) {
+      if (Subtarget.inMips16Mode())
+        MultOpc = (Opcode == ISD::MULHU ?
+                   Mips::MultuRxRy16 : Mips::MultRxRy16);
+      else
+        MultOpc = (Opcode == ISD::MULHU ? Mips::MULTu : Mips::MULT);
+    }
     else
       MultOpc = (Opcode == ISD::MULHU ? Mips::DMULTu : Mips::DMULT);
 
@@ -539,6 +708,15 @@ SDNode* MipsDAGToDAGISel::Select(SDNode *Node) {
     return RegOpnd;
   }
 
+#ifndef NDEBUG
+  case ISD::LOAD:
+  case ISD::STORE:
+    assert(cast<MemSDNode>(Node)->getMemoryVT().getSizeInBits() / 8 <=
+           cast<MemSDNode>(Node)->getAlignment() &&
+           "Unexpected unaligned loads/stores.");
+    break;
+#endif
+
   case MipsISD::ThreadPointer: {
     EVT PtrVT = TLI.getPointerTy();
     unsigned RdhwrOpc, SrcReg, DestReg;
diff --git a/lib/Target/Mips/MipsISelLowering.cpp b/lib/Target/Mips/MipsISelLowering.cpp
index c5207c6..e225b6c 100644
--- a/lib/Target/Mips/MipsISelLowering.cpp
+++ b/lib/Target/Mips/MipsISelLowering.cpp
@@ -25,6 +25,7 @@
 #include "llvm/GlobalVariable.h"
 #include "llvm/Intrinsics.h"
 #include "llvm/CallingConv.h"
+#include "llvm/ADT/Statistic.h"
 #include "llvm/CodeGen/CallingConvLower.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
 #include "llvm/CodeGen/MachineFunction.h"
@@ -32,12 +33,33 @@
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/SelectionDAGISel.h"
 #include "llvm/CodeGen/ValueTypes.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/raw_ostream.h"
 
 using namespace llvm;
 
+STATISTIC(NumTailCalls, "Number of tail calls");
+
+static cl::opt<bool>
+EnableMipsTailCalls("enable-mips-tail-calls", cl::Hidden,
+                    cl::desc("MIPS: Enable tail calls."), cl::init(false));
+
+static const uint16_t O32IntRegs[4] = {
+  Mips::A0, Mips::A1, Mips::A2, Mips::A3
+};
+
+static const uint16_t Mips64IntRegs[8] = {
+  Mips::A0_64, Mips::A1_64, Mips::A2_64, Mips::A3_64,
+  Mips::T0_64, Mips::T1_64, Mips::T2_64, Mips::T3_64
+};
+
+static const uint16_t Mips64DPRegs[8] = {
+  Mips::D12_64, Mips::D13_64, Mips::D14_64, Mips::D15_64,
+  Mips::D16_64, Mips::D17_64, Mips::D18_64, Mips::D19_64
+};
+
 // If I is a shifted mask, set the size (Size) and the first bit of the
 // mask (Pos), and return true.
 // For example, if I is 0x003ff800, (Pos, Size) = (11, 11).
@@ -58,6 +80,7 @@ static SDValue GetGlobalReg(SelectionDAG &DAG, EVT Ty) {
 const char *MipsTargetLowering::getTargetNodeName(unsigned Opcode) const {
   switch (Opcode) {
   case MipsISD::JmpLink:           return "MipsISD::JmpLink";
+  case MipsISD::TailCall:          return "MipsISD::TailCall";
   case MipsISD::Hi:                return "MipsISD::Hi";
   case MipsISD::Lo:                return "MipsISD::Lo";
   case MipsISD::GPRel:             return "MipsISD::GPRel";
@@ -89,6 +112,20 @@ const char *MipsTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case MipsISD::LDR:               return "MipsISD::LDR";
   case MipsISD::SDL:               return "MipsISD::SDL";
   case MipsISD::SDR:               return "MipsISD::SDR";
+  case MipsISD::EXTP:              return "MipsISD::EXTP";
+  case MipsISD::EXTPDP:            return "MipsISD::EXTPDP";
+  case MipsISD::EXTR_S_H:          return "MipsISD::EXTR_S_H";
+  case MipsISD::EXTR_W:            return "MipsISD::EXTR_W";
+  case MipsISD::EXTR_R_W:          return "MipsISD::EXTR_R_W";
+  case MipsISD::EXTR_RS_W:         return "MipsISD::EXTR_RS_W";
+  case MipsISD::SHILO:             return "MipsISD::SHILO";
+  case MipsISD::MTHLIP:            return "MipsISD::MTHLIP";
+  case MipsISD::MULT:              return "MipsISD::MULT";
+  case MipsISD::MULTU:             return "MipsISD::MULTU";
+  case MipsISD::MADD_DSP:          return "MipsISD::MADD_DSPDSP";
+  case MipsISD::MADDU_DSP:         return "MipsISD::MADDU_DSP";
+  case MipsISD::MSUB_DSP:          return "MipsISD::MSUB_DSP";
+  case MipsISD::MSUBU_DSP:         return "MipsISD::MSUBU_DSP";
   default:                         return NULL;
   }
 }
@@ -113,7 +150,22 @@ MipsTargetLowering(MipsTargetMachine &TM)
 
   if (Subtarget->inMips16Mode()) {
     addRegisterClass(MVT::i32, &Mips::CPU16RegsRegClass);
-    addRegisterClass(MVT::i32, &Mips::CPURARegRegClass);
+  }
+
+  if (Subtarget->hasDSP()) {
+    MVT::SimpleValueType VecTys[2] = {MVT::v2i16, MVT::v4i8};
+
+    for (unsigned i = 0; i < array_lengthof(VecTys); ++i) {
+      addRegisterClass(VecTys[i], &Mips::DSPRegsRegClass);
+
+      // Expand all builtin opcodes.
+      for (unsigned Opc = 0; Opc < ISD::BUILTIN_OP_END; ++Opc)
+        setOperationAction(Opc, VecTys[i], Expand);
+
+      setOperationAction(ISD::LOAD, VecTys[i], Legal);
+      setOperationAction(ISD::STORE, VecTys[i], Legal);
+      setOperationAction(ISD::BITCAST, VecTys[i], Legal);
+    }
   }
 
   if (!TM.Options.UseSoftFloat) {
@@ -160,10 +212,18 @@ MipsTargetLowering(MipsTargetMachine &TM)
   setOperationAction(ISD::VASTART,            MVT::Other, Custom);
   setOperationAction(ISD::FCOPYSIGN,          MVT::f32,   Custom);
   setOperationAction(ISD::FCOPYSIGN,          MVT::f64,   Custom);
-  setOperationAction(ISD::MEMBARRIER,         MVT::Other, Custom);
-  setOperationAction(ISD::ATOMIC_FENCE,       MVT::Other, Custom);
-  setOperationAction(ISD::LOAD,               MVT::i32, Custom);
-  setOperationAction(ISD::STORE,              MVT::i32, Custom);
+  if (Subtarget->inMips16Mode()) {
+    setOperationAction(ISD::MEMBARRIER,         MVT::Other, Expand);
+    setOperationAction(ISD::ATOMIC_FENCE,       MVT::Other, Expand);
+  }
+  else {
+    setOperationAction(ISD::MEMBARRIER,         MVT::Other, Custom);
+    setOperationAction(ISD::ATOMIC_FENCE,       MVT::Other, Custom);
+  }
+  if (!Subtarget->inMips16Mode()) {
+    setOperationAction(ISD::LOAD,               MVT::i32, Custom);
+    setOperationAction(ISD::STORE,              MVT::i32, Custom);
+  }
 
   if (!TM.Options.NoNaNsFPMath) {
     setOperationAction(ISD::FABS,             MVT::f32,   Custom);
@@ -187,6 +247,10 @@ MipsTargetLowering(MipsTargetMachine &TM)
     setOperationAction(ISD::SRL_PARTS,          MVT::i32,   Custom);
   }
 
+  setOperationAction(ISD::ADD,                MVT::i32,   Custom);
+  if (HasMips64)
+    setOperationAction(ISD::ADD,                MVT::i64,   Custom);
+
   setOperationAction(ISD::SDIV, MVT::i32, Expand);
   setOperationAction(ISD::SREM, MVT::i32, Expand);
   setOperationAction(ISD::UDIV, MVT::i32, Expand);
@@ -254,6 +318,9 @@ MipsTargetLowering(MipsTargetMachine &TM)
   setOperationAction(ISD::VACOPY,            MVT::Other, Expand);
   setOperationAction(ISD::VAEND,             MVT::Other, Expand);
 
+  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i64, Custom);
+  setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i64, Custom);
+
   // Use the default for now
   setOperationAction(ISD::STACKSAVE,         MVT::Other, Expand);
   setOperationAction(ISD::STACKRESTORE,      MVT::Other, Expand);
@@ -263,6 +330,21 @@ MipsTargetLowering(MipsTargetMachine &TM)
   setOperationAction(ISD::ATOMIC_STORE,      MVT::i32,    Expand);
   setOperationAction(ISD::ATOMIC_STORE,      MVT::i64,    Expand);
 
+  if (Subtarget->inMips16Mode()) {
+    setOperationAction(ISD::ATOMIC_CMP_SWAP,       MVT::i32,    Expand);
+    setOperationAction(ISD::ATOMIC_SWAP,           MVT::i32,    Expand);
+    setOperationAction(ISD::ATOMIC_LOAD_ADD,       MVT::i32,    Expand);
+    setOperationAction(ISD::ATOMIC_LOAD_SUB,       MVT::i32,    Expand);
+    setOperationAction(ISD::ATOMIC_LOAD_AND,       MVT::i32,    Expand);
+    setOperationAction(ISD::ATOMIC_LOAD_OR,        MVT::i32,    Expand);
+    setOperationAction(ISD::ATOMIC_LOAD_XOR,       MVT::i32,    Expand);
+    setOperationAction(ISD::ATOMIC_LOAD_NAND,      MVT::i32,    Expand);
+    setOperationAction(ISD::ATOMIC_LOAD_MIN,       MVT::i32,    Expand);
+    setOperationAction(ISD::ATOMIC_LOAD_MAX,       MVT::i32,    Expand);
+    setOperationAction(ISD::ATOMIC_LOAD_UMIN,      MVT::i32,    Expand);
+    setOperationAction(ISD::ATOMIC_LOAD_UMAX,      MVT::i32,    Expand);
+  }
+
   setInsertFencesForAtomic(true);
 
   if (!Subtarget->hasSEInReg()) {
@@ -310,6 +392,9 @@ MipsTargetLowering(MipsTargetMachine &TM)
 bool MipsTargetLowering::allowsUnalignedMemoryAccesses(EVT VT) const {
   MVT::SimpleValueType SVT = VT.getSimpleVT().SimpleTy;
 
+  if (Subtarget->inMips16Mode())
+    return false;
+
   switch (SVT) {
   case MVT::i64:
   case MVT::i32:
@@ -785,6 +870,26 @@ SDValue  MipsTargetLowering::PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI)
   return SDValue();
 }
 
+void
+MipsTargetLowering::LowerOperationWrapper(SDNode *N,
+                                          SmallVectorImpl<SDValue> &Results,
+                                          SelectionDAG &DAG) const {
+  SDValue Res = LowerOperation(SDValue(N, 0), DAG);
+
+  for (unsigned I = 0, E = Res->getNumValues(); I != E; ++I)
+    Results.push_back(Res.getValue(I));
+}
+
+void
+MipsTargetLowering::ReplaceNodeResults(SDNode *N,
+                                       SmallVectorImpl<SDValue> &Results,
+                                       SelectionDAG &DAG) const {
+  SDValue Res = LowerOperation(SDValue(N, 0), DAG);
+
+  for (unsigned I = 0, E = Res->getNumValues(); I != E; ++I)
+    Results.push_back(Res.getValue(I));
+}
+
 SDValue MipsTargetLowering::
 LowerOperation(SDValue Op, SelectionDAG &DAG) const
 {
@@ -811,6 +916,9 @@ LowerOperation(SDValue Op, SelectionDAG &DAG) const
     case ISD::SRL_PARTS:          return LowerShiftRightParts(Op, DAG, false);
     case ISD::LOAD:               return LowerLOAD(Op, DAG);
     case ISD::STORE:              return LowerSTORE(Op, DAG);
+    case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
+    case ISD::INTRINSIC_W_CHAIN:  return LowerINTRINSIC_W_CHAIN(Op, DAG);
+    case ISD::ADD:                return LowerADD(Op, DAG);
   }
   return SDValue();
 }
@@ -919,6 +1027,70 @@ static MachineBasicBlock* ExpandCondMov(MachineInstr *MI, MachineBasicBlock *BB,
   return BB;
 }
 */
+
+MachineBasicBlock *
+MipsTargetLowering::EmitBPOSGE32(MachineInstr *MI, MachineBasicBlock *BB) const{
+  // $bb:
+  //  bposge32_pseudo $vr0
+  //  =>
+  // $bb:
+  //  bposge32 $tbb
+  // $fbb:
+  //  li $vr2, 0
+  //  b $sink
+  // $tbb:
+  //  li $vr1, 1
+  // $sink:
+  //  $vr0 = phi($vr2, $fbb, $vr1, $tbb)
+
+  MachineRegisterInfo &RegInfo = BB->getParent()->getRegInfo();
+  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
+  const TargetRegisterClass *RC = &Mips::CPURegsRegClass;
+  DebugLoc DL = MI->getDebugLoc();
+  const BasicBlock *LLVM_BB = BB->getBasicBlock();
+  MachineFunction::iterator It = llvm::next(MachineFunction::iterator(BB));
+  MachineFunction *F = BB->getParent();
+  MachineBasicBlock *FBB = F->CreateMachineBasicBlock(LLVM_BB);
+  MachineBasicBlock *TBB = F->CreateMachineBasicBlock(LLVM_BB);
+  MachineBasicBlock *Sink  = F->CreateMachineBasicBlock(LLVM_BB);
+  F->insert(It, FBB);
+  F->insert(It, TBB);
+  F->insert(It, Sink);
+
+  // Transfer the remainder of BB and its successor edges to Sink.
+  Sink->splice(Sink->begin(), BB, llvm::next(MachineBasicBlock::iterator(MI)),
+               BB->end());
+  Sink->transferSuccessorsAndUpdatePHIs(BB);
+
+  // Add successors.
+  BB->addSuccessor(FBB);
+  BB->addSuccessor(TBB);
+  FBB->addSuccessor(Sink);
+  TBB->addSuccessor(Sink);
+
+  // Insert the real bposge32 instruction to $BB.
+  BuildMI(BB, DL, TII->get(Mips::BPOSGE32)).addMBB(TBB);
+
+  // Fill $FBB.
+  unsigned VR2 = RegInfo.createVirtualRegister(RC);
+  BuildMI(*FBB, FBB->end(), DL, TII->get(Mips::ADDiu), VR2)
+    .addReg(Mips::ZERO).addImm(0);
+  BuildMI(*FBB, FBB->end(), DL, TII->get(Mips::B)).addMBB(Sink);
+
+  // Fill $TBB.
+  unsigned VR1 = RegInfo.createVirtualRegister(RC);
+  BuildMI(*TBB, TBB->end(), DL, TII->get(Mips::ADDiu), VR1)
+    .addReg(Mips::ZERO).addImm(1);
+
+  // Insert phi function to $Sink.
+  BuildMI(*Sink, Sink->begin(), DL, TII->get(Mips::PHI),
+          MI->getOperand(0).getReg())
+    .addReg(VR2).addMBB(FBB).addReg(VR1).addMBB(TBB);
+
+  MI->eraseFromParent();   // The pseudo instruction is gone now.
+  return Sink;
+}
+
 MachineBasicBlock *
 MipsTargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI,
                                                 MachineBasicBlock *BB) const {
@@ -1027,6 +1199,8 @@ MipsTargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI,
   case Mips::ATOMIC_CMP_SWAP_I64:
   case Mips::ATOMIC_CMP_SWAP_I64_P8:
     return EmitAtomicCmpSwap(MI, BB, 8);
+  case Mips::BPOSGE32_PSEUDO:
+    return EmitBPOSGE32(MI, BB);
   }
 }
 
@@ -1571,15 +1745,16 @@ SDValue MipsTargetLowering::LowerGlobalAddress(SDValue Op,
   if (getTargetMachine().getRelocationModel() != Reloc::PIC_ && !IsN64) {
     SDVTList VTs = DAG.getVTList(MVT::i32);
 
-    MipsTargetObjectFile &TLOF = (MipsTargetObjectFile&)getObjFileLowering();
+    const MipsTargetObjectFile &TLOF =
+      (const MipsTargetObjectFile&)getObjFileLowering();
 
     // %gp_rel relocation
     if (TLOF.IsGlobalInSmallSection(GV, getTargetMachine())) {
       SDValue GA = DAG.getTargetGlobalAddress(GV, dl, MVT::i32, 0,
                                               MipsII::MO_GPREL);
       SDValue GPRelNode = DAG.getNode(MipsISD::GPRel, dl, VTs, &GA, 1);
-      SDValue GOT = DAG.getGLOBAL_OFFSET_TABLE(MVT::i32);
-      return DAG.getNode(ISD::ADD, dl, MVT::i32, GOT, GPRelNode);
+      SDValue GPReg = DAG.getRegister(Mips::GP, MVT::i32);
+      return DAG.getNode(ISD::ADD, dl, MVT::i32, GPReg, GPRelNode);
     }
     // %hi/%lo relocation
     SDValue GAHi = DAG.getTargetGlobalAddress(GV, dl, MVT::i32, 0,
@@ -1620,8 +1795,10 @@ SDValue MipsTargetLowering::LowerBlockAddress(SDValue Op,
 
   if (getTargetMachine().getRelocationModel() != Reloc::PIC_ && !IsN64) {
     // %hi/%lo relocation
-    SDValue BAHi = DAG.getBlockAddress(BA, MVT::i32, true, MipsII::MO_ABS_HI);
-    SDValue BALo = DAG.getBlockAddress(BA, MVT::i32, true, MipsII::MO_ABS_LO);
+    SDValue BAHi =
+      DAG.getTargetBlockAddress(BA, MVT::i32, 0, MipsII::MO_ABS_HI);
+    SDValue BALo =
+      DAG.getTargetBlockAddress(BA, MVT::i32, 0, MipsII::MO_ABS_LO);
     SDValue Hi = DAG.getNode(MipsISD::Hi, dl, MVT::i32, BAHi);
     SDValue Lo = DAG.getNode(MipsISD::Lo, dl, MVT::i32, BALo);
     return DAG.getNode(ISD::ADD, dl, MVT::i32, Hi, Lo);
@@ -1630,10 +1807,10 @@ SDValue MipsTargetLowering::LowerBlockAddress(SDValue Op,
   EVT ValTy = Op.getValueType();
   unsigned GOTFlag = HasMips64 ? MipsII::MO_GOT_PAGE : MipsII::MO_GOT;
   unsigned OFSTFlag = HasMips64 ? MipsII::MO_GOT_OFST : MipsII::MO_ABS_LO;
-  SDValue BAGOTOffset = DAG.getBlockAddress(BA, ValTy, true, GOTFlag);
+  SDValue BAGOTOffset = DAG.getTargetBlockAddress(BA, ValTy, 0, GOTFlag);
   BAGOTOffset = DAG.getNode(MipsISD::Wrapper, dl, ValTy,
                             GetGlobalReg(DAG, ValTy), BAGOTOffset);
-  SDValue BALOOffset = DAG.getBlockAddress(BA, ValTy, true, OFSTFlag);
+  SDValue BALOOffset = DAG.getTargetBlockAddress(BA, ValTy, 0, OFSTFlag);
   SDValue Load = DAG.getLoad(ValTy, dl, DAG.getEntryNode(), BAGOTOffset,
                              MachinePointerInfo(), false, false, false, 0);
   SDValue Lo = DAG.getNode(MipsISD::Lo, dl, ValTy, BALOOffset);
@@ -2224,6 +2401,172 @@ SDValue MipsTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
   return CreateStoreLR(MipsISD::SDR, DAG, SD, SDL, IsLittle ? 0 : 7);
 }
 
+// This function expands mips intrinsic nodes which have 64-bit input operands
+// or output values.
+//
+// out64 = intrinsic-node in64
+// =>
+// lo = copy (extract-element (in64, 0))
+// hi = copy (extract-element (in64, 1))
+// mips-specific-node
+// v0 = copy lo
+// v1 = copy hi
+// out64 = merge-values (v0, v1)
+//
+static SDValue LowerDSPIntr(SDValue Op, SelectionDAG &DAG,
+                            unsigned Opc, bool HasI64In, bool HasI64Out) {
+  DebugLoc DL = Op.getDebugLoc();
+  bool HasChainIn = Op->getOperand(0).getValueType() == MVT::Other;
+  SDValue Chain = HasChainIn ? Op->getOperand(0) : DAG.getEntryNode();
+  SmallVector<SDValue, 3> Ops;
+
+  if (HasI64In) {
+    SDValue InLo = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32,
+                               Op->getOperand(1 + HasChainIn),
+                               DAG.getConstant(0, MVT::i32));
+    SDValue InHi = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i32,
+                               Op->getOperand(1 + HasChainIn),
+                               DAG.getConstant(1, MVT::i32));
+
+    Chain = DAG.getCopyToReg(Chain, DL, Mips::LO, InLo, SDValue());
+    Chain = DAG.getCopyToReg(Chain, DL, Mips::HI, InHi, Chain.getValue(1));
+
+    Ops.push_back(Chain);
+    Ops.append(Op->op_begin() + HasChainIn + 2, Op->op_end());
+    Ops.push_back(Chain.getValue(1));
+  } else {
+    Ops.push_back(Chain);
+    Ops.append(Op->op_begin() + HasChainIn + 1, Op->op_end());
+  }
+
+  if (!HasI64Out)
+    return DAG.getNode(Opc, DL, Op->value_begin(), Op->getNumValues(),
+                       Ops.begin(), Ops.size());
+
+  SDValue Intr = DAG.getNode(Opc, DL, DAG.getVTList(MVT::Other, MVT::Glue),
+                             Ops.begin(), Ops.size());
+  SDValue OutLo = DAG.getCopyFromReg(Intr.getValue(0), DL, Mips::LO, MVT::i32,
+                                     Intr.getValue(1));
+  SDValue OutHi = DAG.getCopyFromReg(OutLo.getValue(1), DL, Mips::HI, MVT::i32,
+                                     OutLo.getValue(2));
+  SDValue Out = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, OutLo, OutHi);
+
+  if (!HasChainIn)
+    return Out;
+
+  SDValue Vals[] = { Out, OutHi.getValue(1) };
+  return DAG.getMergeValues(Vals, 2, DL);
+}
+
+SDValue MipsTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
+                                                    SelectionDAG &DAG) const {
+  switch (cast<ConstantSDNode>(Op->getOperand(0))->getZExtValue()) {
+  default:
+    return SDValue();
+  case Intrinsic::mips_shilo:
+    return LowerDSPIntr(Op, DAG, MipsISD::SHILO, true, true);
+  case Intrinsic::mips_dpau_h_qbl:
+    return LowerDSPIntr(Op, DAG, MipsISD::DPAU_H_QBL, true, true);
+  case Intrinsic::mips_dpau_h_qbr:
+    return LowerDSPIntr(Op, DAG, MipsISD::DPAU_H_QBR, true, true);
+  case Intrinsic::mips_dpsu_h_qbl:
+    return LowerDSPIntr(Op, DAG, MipsISD::DPSU_H_QBL, true, true);
+  case Intrinsic::mips_dpsu_h_qbr:
+    return LowerDSPIntr(Op, DAG, MipsISD::DPSU_H_QBR, true, true);
+  case Intrinsic::mips_dpa_w_ph:
+    return LowerDSPIntr(Op, DAG, MipsISD::DPA_W_PH, true, true);
+  case Intrinsic::mips_dps_w_ph:
+    return LowerDSPIntr(Op, DAG, MipsISD::DPS_W_PH, true, true);
+  case Intrinsic::mips_dpax_w_ph:
+    return LowerDSPIntr(Op, DAG, MipsISD::DPAX_W_PH, true, true);
+  case Intrinsic::mips_dpsx_w_ph:
+    return LowerDSPIntr(Op, DAG, MipsISD::DPSX_W_PH, true, true);
+  case Intrinsic::mips_mulsa_w_ph:
+    return LowerDSPIntr(Op, DAG, MipsISD::MULSA_W_PH, true, true);
+  case Intrinsic::mips_mult:
+    return LowerDSPIntr(Op, DAG, MipsISD::MULT, false, true);
+  case Intrinsic::mips_multu:
+    return LowerDSPIntr(Op, DAG, MipsISD::MULTU, false, true);
+  case Intrinsic::mips_madd:
+    return LowerDSPIntr(Op, DAG, MipsISD::MADD_DSP, true, true);
+  case Intrinsic::mips_maddu:
+    return LowerDSPIntr(Op, DAG, MipsISD::MADDU_DSP, true, true);
+  case Intrinsic::mips_msub:
+    return LowerDSPIntr(Op, DAG, MipsISD::MSUB_DSP, true, true);
+  case Intrinsic::mips_msubu:
+    return LowerDSPIntr(Op, DAG, MipsISD::MSUBU_DSP, true, true);
+  }
+}
+
+SDValue MipsTargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
+                                                   SelectionDAG &DAG) const {
+  switch (cast<ConstantSDNode>(Op->getOperand(1))->getZExtValue()) {
+  default:
+    return SDValue();
+  case Intrinsic::mips_extp:
+    return LowerDSPIntr(Op, DAG, MipsISD::EXTP, true, false);
+  case Intrinsic::mips_extpdp:
+    return LowerDSPIntr(Op, DAG, MipsISD::EXTPDP, true, false);
+  case Intrinsic::mips_extr_w:
+    return LowerDSPIntr(Op, DAG, MipsISD::EXTR_W, true, false);
+  case Intrinsic::mips_extr_r_w:
+    return LowerDSPIntr(Op, DAG, MipsISD::EXTR_R_W, true, false);
+  case Intrinsic::mips_extr_rs_w:
+    return LowerDSPIntr(Op, DAG, MipsISD::EXTR_RS_W, true, false);
+  case Intrinsic::mips_extr_s_h:
+    return LowerDSPIntr(Op, DAG, MipsISD::EXTR_S_H, true, false);
+  case Intrinsic::mips_mthlip:
+    return LowerDSPIntr(Op, DAG, MipsISD::MTHLIP, true, true);
+  case Intrinsic::mips_mulsaq_s_w_ph:
+    return LowerDSPIntr(Op, DAG, MipsISD::MULSAQ_S_W_PH, true, true);
+  case Intrinsic::mips_maq_s_w_phl:
+    return LowerDSPIntr(Op, DAG, MipsISD::MAQ_S_W_PHL, true, true);
+  case Intrinsic::mips_maq_s_w_phr:
+    return LowerDSPIntr(Op, DAG, MipsISD::MAQ_S_W_PHR, true, true);
+  case Intrinsic::mips_maq_sa_w_phl:
+    return LowerDSPIntr(Op, DAG, MipsISD::MAQ_SA_W_PHL, true, true);
+  case Intrinsic::mips_maq_sa_w_phr:
+    return LowerDSPIntr(Op, DAG, MipsISD::MAQ_SA_W_PHR, true, true);
+  case Intrinsic::mips_dpaq_s_w_ph:
+    return LowerDSPIntr(Op, DAG, MipsISD::DPAQ_S_W_PH, true, true);
+  case Intrinsic::mips_dpsq_s_w_ph:
+    return LowerDSPIntr(Op, DAG, MipsISD::DPSQ_S_W_PH, true, true);
+  case Intrinsic::mips_dpaq_sa_l_w:
+    return LowerDSPIntr(Op, DAG, MipsISD::DPAQ_SA_L_W, true, true);
+  case Intrinsic::mips_dpsq_sa_l_w:
+    return LowerDSPIntr(Op, DAG, MipsISD::DPSQ_SA_L_W, true, true);
+  case Intrinsic::mips_dpaqx_s_w_ph:
+    return LowerDSPIntr(Op, DAG, MipsISD::DPAQX_S_W_PH, true, true);
+  case Intrinsic::mips_dpaqx_sa_w_ph:
+    return LowerDSPIntr(Op, DAG, MipsISD::DPAQX_SA_W_PH, true, true);
+  case Intrinsic::mips_dpsqx_s_w_ph:
+    return LowerDSPIntr(Op, DAG, MipsISD::DPSQX_S_W_PH, true, true);
+  case Intrinsic::mips_dpsqx_sa_w_ph:
+    return LowerDSPIntr(Op, DAG, MipsISD::DPSQX_SA_W_PH, true, true);
+  }
+}
+
+SDValue MipsTargetLowering::LowerADD(SDValue Op, SelectionDAG &DAG) const {
+  if (Op->getOperand(0).getOpcode() != ISD::FRAMEADDR
+      || cast<ConstantSDNode>
+        (Op->getOperand(0).getOperand(0))->getZExtValue() != 0
+      || Op->getOperand(1).getOpcode() != ISD::FRAME_TO_ARGS_OFFSET)
+    return SDValue();
+
+  // The pattern
+  //   (add (frameaddr 0), (frame_to_args_offset))
+  // results from lowering llvm.eh.dwarf.cfa intrinsic. Transform it to
+  //   (add FrameObject, 0)
+  // where FrameObject is a fixed StackObject with offset 0 which points to
+  // the old stack pointer.
+  MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
+  EVT ValTy = Op->getValueType(0);
+  int FI = MFI->CreateFixedObject(Op.getValueSizeInBits() / 8, 0, false);
+  SDValue InArgsAddr = DAG.getFrameIndex(FI, ValTy);
+  return DAG.getNode(ISD::ADD, Op->getDebugLoc(), ValTy, InArgsAddr,
+                     DAG.getConstant(0, ValTy));
+}
+
 //===----------------------------------------------------------------------===//
 //                      Calling Convention Implementation
 //===----------------------------------------------------------------------===//
@@ -2259,16 +2602,9 @@ static bool CC_MipsO32(unsigned ValNo, MVT ValVT,
       Mips::D6, Mips::D7
   };
 
-  // ByVal Args
-  if (ArgFlags.isByVal()) {
-    State.HandleByVal(ValNo, ValVT, LocVT, LocInfo,
-                      1 /*MinSize*/, 4 /*MinAlign*/, ArgFlags);
-    unsigned NextReg = (State.getNextStackOffset() + 3) / 4;
-    for (unsigned r = State.getFirstUnallocated(IntRegs, IntRegsSize);
-         r < std::min(IntRegsSize, NextReg); ++r)
-      State.AllocateReg(IntRegs[r]);
-    return false;
-  }
+  // Do not process byval args here.
+  if (ArgFlags.isByVal())
+    return true;
 
   // Promote i8 and i16
   if (LocVT == MVT::i8 || LocVT == MVT::i16) {
@@ -2323,279 +2659,72 @@ static bool CC_MipsO32(unsigned ValNo, MVT ValVT,
   } else
     llvm_unreachable("Cannot handle this ValVT.");
 
-  unsigned SizeInBytes = ValVT.getSizeInBits() >> 3;
-  unsigned Offset = State.AllocateStack(SizeInBytes, OrigAlign);
-
-  if (!Reg)
+  if (!Reg) {
+    unsigned Offset = State.AllocateStack(ValVT.getSizeInBits() >> 3,
+                                          OrigAlign);
     State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, LocInfo));
-  else
+  } else
     State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
 
-  return false; // CC must always match
-}
-
-static const uint16_t Mips64IntRegs[8] =
-  {Mips::A0_64, Mips::A1_64, Mips::A2_64, Mips::A3_64,
-   Mips::T0_64, Mips::T1_64, Mips::T2_64, Mips::T3_64};
-static const uint16_t Mips64DPRegs[8] =
-  {Mips::D12_64, Mips::D13_64, Mips::D14_64, Mips::D15_64,
-   Mips::D16_64, Mips::D17_64, Mips::D18_64, Mips::D19_64};
-
-static bool CC_Mips64Byval(unsigned ValNo, MVT ValVT, MVT LocVT,
-                           CCValAssign::LocInfo LocInfo,
-                           ISD::ArgFlagsTy ArgFlags, CCState &State) {
-  unsigned Align = std::max(ArgFlags.getByValAlign(), (unsigned)8);
-  unsigned Size  = (ArgFlags.getByValSize() + 7) / 8 * 8;
-  unsigned FirstIdx = State.getFirstUnallocated(Mips64IntRegs, 8);
-
-  assert(Align <= 16 && "Cannot handle alignments larger than 16.");
-
-  // If byval is 16-byte aligned, the first arg register must be even.
-  if ((Align == 16) && (FirstIdx % 2)) {
-    State.AllocateReg(Mips64IntRegs[FirstIdx], Mips64DPRegs[FirstIdx]);
-    ++FirstIdx;
-  }
-
-  // Mark the registers allocated.
-  for (unsigned I = FirstIdx; Size && (I < 8); Size -= 8, ++I)
-    State.AllocateReg(Mips64IntRegs[I], Mips64DPRegs[I]);
-
-  // Allocate space on caller's stack.
-  unsigned Offset = State.AllocateStack(Size, Align);
-
-  if (FirstIdx < 8)
-    State.addLoc(CCValAssign::getReg(ValNo, ValVT, Mips64IntRegs[FirstIdx],
-                                     LocVT, LocInfo));
-  else
-    State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, LocInfo));
-
-  return true;
+  return false;
 }
 
 #include "MipsGenCallingConv.inc"
 
-static void
-AnalyzeMips64CallOperands(CCState &CCInfo,
-                          const SmallVectorImpl<ISD::OutputArg> &Outs) {
-  unsigned NumOps = Outs.size();
-  for (unsigned i = 0; i != NumOps; ++i) {
-    MVT ArgVT = Outs[i].VT;
-    ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
-    bool R;
-
-    if (Outs[i].IsFixed)
-      R = CC_MipsN(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo);
-    else
-      R = CC_MipsN_VarArg(i, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo);
-
-    if (R) {
-#ifndef NDEBUG
-      dbgs() << "Call operand #" << i << " has unhandled type "
-             << EVT(ArgVT).getEVTString();
-#endif
-      llvm_unreachable(0);
-    }
-  }
-}
-
 //===----------------------------------------------------------------------===//
 //                  Call Calling Convention Implementation
 //===----------------------------------------------------------------------===//
 
 static const unsigned O32IntRegsSize = 4;
 
-static const uint16_t O32IntRegs[] = {
-  Mips::A0, Mips::A1, Mips::A2, Mips::A3
-};
-
 // Return next O32 integer argument register.
 static unsigned getNextIntArgReg(unsigned Reg) {
   assert((Reg == Mips::A0) || (Reg == Mips::A2));
   return (Reg == Mips::A0) ? Mips::A1 : Mips::A3;
 }
 
-// Write ByVal Arg to arg registers and stack.
-static void
-WriteByValArg(SDValue Chain, DebugLoc dl,
-              SmallVector<std::pair<unsigned, SDValue>, 16> &RegsToPass,
-              SmallVector<SDValue, 8> &MemOpChains, SDValue StackPtr,
-              MachineFrameInfo *MFI, SelectionDAG &DAG, SDValue Arg,
-              const CCValAssign &VA, const ISD::ArgFlagsTy &Flags,
-              MVT PtrType, bool isLittle) {
-  unsigned LocMemOffset = VA.getLocMemOffset();
-  unsigned Offset = 0;
-  uint32_t RemainingSize = Flags.getByValSize();
-  unsigned ByValAlign = Flags.getByValAlign();
-
-  // Copy the first 4 words of byval arg to registers A0 - A3.
-  // FIXME: Use a stricter alignment if it enables better optimization in passes
-  //        run later.
-  for (; RemainingSize >= 4 && LocMemOffset < 4 * 4;
-       Offset += 4, RemainingSize -= 4, LocMemOffset += 4) {
-    SDValue LoadPtr = DAG.getNode(ISD::ADD, dl, MVT::i32, Arg,
-                                  DAG.getConstant(Offset, MVT::i32));
-    SDValue LoadVal = DAG.getLoad(MVT::i32, dl, Chain, LoadPtr,
-                                  MachinePointerInfo(), false, false, false,
-                                  std::min(ByValAlign, (unsigned )4));
-    MemOpChains.push_back(LoadVal.getValue(1));
-    unsigned DstReg = O32IntRegs[LocMemOffset / 4];
-    RegsToPass.push_back(std::make_pair(DstReg, LoadVal));
-  }
-
-  if (RemainingSize == 0)
-    return;
+/// IsEligibleForTailCallOptimization - Check whether the call is eligible
+/// for tail call optimization.
+bool MipsTargetLowering::
+IsEligibleForTailCallOptimization(const MipsCC &MipsCCInfo,
+                                  unsigned NextStackOffset,
+                                  const MipsFunctionInfo& FI) const {
+  if (!EnableMipsTailCalls)
+    return false;
 
-  // If there still is a register available for argument passing, write the
-  // remaining part of the structure to it using subword loads and shifts.
-  if (LocMemOffset < 4 * 4) {
-    assert(RemainingSize <= 3 && RemainingSize >= 1 &&
-           "There must be one to three bytes remaining.");
-    unsigned LoadSize = (RemainingSize == 3 ? 2 : RemainingSize);
-    SDValue LoadPtr = DAG.getNode(ISD::ADD, dl, MVT::i32, Arg,
-                                  DAG.getConstant(Offset, MVT::i32));
-    unsigned Alignment = std::min(ByValAlign, (unsigned )4);
-    SDValue LoadVal = DAG.getExtLoad(ISD::ZEXTLOAD, dl, MVT::i32, Chain,
-                                     LoadPtr, MachinePointerInfo(),
-                                     MVT::getIntegerVT(LoadSize * 8), false,
-                                     false, Alignment);
-    MemOpChains.push_back(LoadVal.getValue(1));
-
-    // If target is big endian, shift it to the most significant half-word or
-    // byte.
-    if (!isLittle)
-      LoadVal = DAG.getNode(ISD::SHL, dl, MVT::i32, LoadVal,
-                            DAG.getConstant(32 - LoadSize * 8, MVT::i32));
-
-    Offset += LoadSize;
-    RemainingSize -= LoadSize;
-
-    // Read second subword if necessary.
-    if (RemainingSize != 0)  {
-      assert(RemainingSize == 1 && "There must be one byte remaining.");
-      LoadPtr = DAG.getNode(ISD::ADD, dl, MVT::i32, Arg,
-                            DAG.getConstant(Offset, MVT::i32));
-      unsigned Alignment = std::min(ByValAlign, (unsigned )2);
-      SDValue Subword = DAG.getExtLoad(ISD::ZEXTLOAD, dl, MVT::i32, Chain,
-                                       LoadPtr, MachinePointerInfo(),
-                                       MVT::i8, false, false, Alignment);
-      MemOpChains.push_back(Subword.getValue(1));
-      // Insert the loaded byte to LoadVal.
-      // FIXME: Use INS if supported by target.
-      unsigned ShiftAmt = isLittle ? 16 : 8;
-      SDValue Shift = DAG.getNode(ISD::SHL, dl, MVT::i32, Subword,
-                                  DAG.getConstant(ShiftAmt, MVT::i32));
-      LoadVal = DAG.getNode(ISD::OR, dl, MVT::i32, LoadVal, Shift);
-    }
+  // No tail call optimization for mips16.
+  if (Subtarget->inMips16Mode())
+    return false;
 
-    unsigned DstReg = O32IntRegs[LocMemOffset / 4];
-    RegsToPass.push_back(std::make_pair(DstReg, LoadVal));
-    return;
-  }
+  // Return false if either the callee or caller has a byval argument.
+  if (MipsCCInfo.hasByValArg() || FI.hasByvalArg())
+    return false;
 
-  // Copy remaining part of byval arg using memcpy.
-  SDValue Src = DAG.getNode(ISD::ADD, dl, MVT::i32, Arg,
-                            DAG.getConstant(Offset, MVT::i32));
-  SDValue Dst = DAG.getNode(ISD::ADD, dl, MVT::i32, StackPtr,
-                            DAG.getIntPtrConstant(LocMemOffset));
-  Chain = DAG.getMemcpy(Chain, dl, Dst, Src,
-                        DAG.getConstant(RemainingSize, MVT::i32),
-                        std::min(ByValAlign, (unsigned)4),
-                        /*isVolatile=*/false, /*AlwaysInline=*/false,
-                        MachinePointerInfo(0), MachinePointerInfo(0));
-  MemOpChains.push_back(Chain);
+  // Return true if the callee's argument area is no larger than the
+  // caller's.
+  return NextStackOffset <= FI.getIncomingArgSize();
 }
 
-// Copy Mips64 byVal arg to registers and stack.
-void static
-PassByValArg64(SDValue Chain, DebugLoc dl,
-               SmallVector<std::pair<unsigned, SDValue>, 16> &RegsToPass,
-               SmallVector<SDValue, 8> &MemOpChains, SDValue StackPtr,
-               MachineFrameInfo *MFI, SelectionDAG &DAG, SDValue Arg,
-               const CCValAssign &VA, const ISD::ArgFlagsTy &Flags,
-               EVT PtrTy, bool isLittle) {
-  unsigned ByValSize = Flags.getByValSize();
-  unsigned Alignment = std::min(Flags.getByValAlign(), (unsigned)8);
-  bool IsRegLoc = VA.isRegLoc();
-  unsigned Offset = 0; // Offset in # of bytes from the beginning of struct.
-  unsigned LocMemOffset = 0;
-  unsigned MemCpySize = ByValSize;
-
-  if (!IsRegLoc)
-    LocMemOffset = VA.getLocMemOffset();
-  else {
-    const uint16_t *Reg = std::find(Mips64IntRegs, Mips64IntRegs + 8,
-                                    VA.getLocReg());
-    const uint16_t *RegEnd = Mips64IntRegs + 8;
-
-    // Copy double words to registers.
-    for (; (Reg != RegEnd) && (ByValSize >= Offset + 8); ++Reg, Offset += 8) {
-      SDValue LoadPtr = DAG.getNode(ISD::ADD, dl, PtrTy, Arg,
-                                    DAG.getConstant(Offset, PtrTy));
-      SDValue LoadVal = DAG.getLoad(MVT::i64, dl, Chain, LoadPtr,
-                                    MachinePointerInfo(), false, false, false,
-                                    Alignment);
-      MemOpChains.push_back(LoadVal.getValue(1));
-      RegsToPass.push_back(std::make_pair(*Reg, LoadVal));
-    }
-
-    // Return if the struct has been fully copied.
-    if (!(MemCpySize = ByValSize - Offset))
-      return;
-
-    // If there is an argument register available, copy the remainder of the
-    // byval argument with sub-doubleword loads and shifts.
-    if (Reg != RegEnd) {
-      assert((ByValSize < Offset + 8) &&
-             "Size of the remainder should be smaller than 8-byte.");
-      SDValue Val;
-      for (unsigned LoadSize = 4; Offset < ByValSize; LoadSize /= 2) {
-        unsigned RemSize = ByValSize - Offset;
-
-        if (RemSize < LoadSize)
-          continue;
-
-        SDValue LoadPtr = DAG.getNode(ISD::ADD, dl, PtrTy, Arg,
-                                      DAG.getConstant(Offset, PtrTy));
-        SDValue LoadVal =
-          DAG.getExtLoad(ISD::ZEXTLOAD, dl, MVT::i64, Chain, LoadPtr,
-                         MachinePointerInfo(), MVT::getIntegerVT(LoadSize * 8),
-                         false, false, Alignment);
-        MemOpChains.push_back(LoadVal.getValue(1));
-
-        // Offset in number of bits from double word boundary.
-        unsigned OffsetDW = (Offset % 8) * 8;
-        unsigned Shamt = isLittle ? OffsetDW : 64 - (OffsetDW + LoadSize * 8);
-        SDValue Shift = DAG.getNode(ISD::SHL, dl, MVT::i64, LoadVal,
-                                    DAG.getConstant(Shamt, MVT::i32));
-
-        Val = Val.getNode() ? DAG.getNode(ISD::OR, dl, MVT::i64, Val, Shift) :
-                              Shift;
-        Offset += LoadSize;
-        Alignment = std::min(Alignment, LoadSize);
-      }
-
-      RegsToPass.push_back(std::make_pair(*Reg, Val));
-      return;
-    }
+SDValue
+MipsTargetLowering::passArgOnStack(SDValue StackPtr, unsigned Offset,
+                                   SDValue Chain, SDValue Arg, DebugLoc DL,
+                                   bool IsTailCall, SelectionDAG &DAG) const {
+  if (!IsTailCall) {
+    SDValue PtrOff = DAG.getNode(ISD::ADD, DL, getPointerTy(), StackPtr,
+                                 DAG.getIntPtrConstant(Offset));
+    return DAG.getStore(Chain, DL, Arg, PtrOff, MachinePointerInfo(), false,
+                        false, 0);
   }
 
-  assert(MemCpySize && "MemCpySize must not be zero.");
-
-  // Copy remainder of byval arg to it with memcpy.
-  SDValue Src = DAG.getNode(ISD::ADD, dl, PtrTy, Arg,
-                            DAG.getConstant(Offset, PtrTy));
-  SDValue Dst = DAG.getNode(ISD::ADD, dl, MVT::i64, StackPtr,
-                            DAG.getIntPtrConstant(LocMemOffset));
-  Chain = DAG.getMemcpy(Chain, dl, Dst, Src,
-                        DAG.getConstant(MemCpySize, PtrTy), Alignment,
-                        /*isVolatile=*/false, /*AlwaysInline=*/false,
-                        MachinePointerInfo(0), MachinePointerInfo(0));
-  MemOpChains.push_back(Chain);
+  MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo();
+  int FI = MFI->CreateFixedObject(Arg.getValueSizeInBits() / 8, Offset, false);
+  SDValue FIN = DAG.getFrameIndex(FI, getPointerTy());
+  return DAG.getStore(Chain, DL, Arg, FIN, MachinePointerInfo(),
+                      /*isVolatile=*/ true, false, 0);
 }
 
 /// LowerCall - functions arguments are copied from virtual regs to
 /// (physical regs)/(stack frame), CALLSEQ_START and CALLSEQ_END are emitted.
-/// TODO: isTailCall.
 SDValue
 MipsTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
                               SmallVectorImpl<SDValue> &InVals) const {
@@ -2610,56 +2739,49 @@ MipsTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   CallingConv::ID CallConv              = CLI.CallConv;
   bool isVarArg                         = CLI.IsVarArg;
 
-  // MIPs target does not yet support tail call optimization.
-  isTailCall = false;
-
   MachineFunction &MF = DAG.getMachineFunction();
   MachineFrameInfo *MFI = MF.getFrameInfo();
   const TargetFrameLowering *TFL = MF.getTarget().getFrameLowering();
   bool IsPIC = getTargetMachine().getRelocationModel() == Reloc::PIC_;
-  MipsFunctionInfo *MipsFI = MF.getInfo<MipsFunctionInfo>();
 
   // Analyze operands of the call, assigning locations to each operand.
   SmallVector<CCValAssign, 16> ArgLocs;
   CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(),
                  getTargetMachine(), ArgLocs, *DAG.getContext());
+  MipsCC MipsCCInfo(CallConv, isVarArg, IsO32, CCInfo);
 
-  if (CallConv == CallingConv::Fast)
-    CCInfo.AnalyzeCallOperands(Outs, CC_Mips_FastCC);
-  else if (IsO32)
-    CCInfo.AnalyzeCallOperands(Outs, CC_MipsO32);
-  else if (HasMips64)
-    AnalyzeMips64CallOperands(CCInfo, Outs);
-  else
-    CCInfo.AnalyzeCallOperands(Outs, CC_Mips);
+  MipsCCInfo.analyzeCallOperands(Outs);
 
   // Get a count of how many bytes are to be pushed on the stack.
   unsigned NextStackOffset = CCInfo.getNextStackOffset();
-  unsigned StackAlignment = TFL->getStackAlignment();
-  NextStackOffset = RoundUpToAlignment(NextStackOffset, StackAlignment);
 
-  // Update size of the maximum argument space.
-  // For O32, a minimum of four words (16 bytes) of argument space is
-  // allocated.
-  if (IsO32 && (CallConv != CallingConv::Fast))
-    NextStackOffset = std::max(NextStackOffset, (unsigned)16);
+  // Check if it's really possible to do a tail call.
+  if (isTailCall)
+    isTailCall =
+      IsEligibleForTailCallOptimization(MipsCCInfo, NextStackOffset,
+                                        *MF.getInfo<MipsFunctionInfo>());
+
+  if (isTailCall)
+    ++NumTailCalls;
 
   // Chain is the output chain of the last Load/Store or CopyToReg node.
   // ByValChain is the output chain of the last Memcpy node created for copying
   // byval arguments to the stack.
+  unsigned StackAlignment = TFL->getStackAlignment();
+  NextStackOffset = RoundUpToAlignment(NextStackOffset, StackAlignment);
   SDValue NextStackOffsetVal = DAG.getIntPtrConstant(NextStackOffset, true);
-  Chain = DAG.getCALLSEQ_START(Chain, NextStackOffsetVal);
+
+  if (!isTailCall)
+    Chain = DAG.getCALLSEQ_START(Chain, NextStackOffsetVal);
 
   SDValue StackPtr = DAG.getCopyFromReg(Chain, dl,
                                         IsN64 ? Mips::SP_64 : Mips::SP,
                                         getPointerTy());
 
-  if (MipsFI->getMaxCallFrameSize() < NextStackOffset)
-    MipsFI->setMaxCallFrameSize(NextStackOffset);
-
   // With EABI is it possible to have 16 args on registers.
   SmallVector<std::pair<unsigned, SDValue>, 16> RegsToPass;
   SmallVector<SDValue, 8> MemOpChains;
+  MipsCC::byval_iterator ByValArg = MipsCCInfo.byval_begin();
 
   // Walk the register/memloc assignments, inserting copies/loads.
   for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
@@ -2672,14 +2794,12 @@ MipsTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     if (Flags.isByVal()) {
       assert(Flags.getByValSize() &&
              "ByVal args of size 0 should have been ignored by front-end.");
-      if (IsO32)
-        WriteByValArg(Chain, dl, RegsToPass, MemOpChains, StackPtr,
-                      MFI, DAG, Arg, VA, Flags, getPointerTy(),
-                      Subtarget->isLittle());
-      else
-        PassByValArg64(Chain, dl, RegsToPass, MemOpChains, StackPtr,
-                       MFI, DAG, Arg, VA, Flags, getPointerTy(),
-                       Subtarget->isLittle());
+      assert(ByValArg != MipsCCInfo.byval_end());
+      assert(!isTailCall &&
+             "Do not tail-call optimize if there is a byval argument.");
+      passByValArg(Chain, dl, RegsToPass, MemOpChains, StackPtr, MFI, DAG, Arg,
+                   MipsCCInfo, *ByValArg, Flags, Subtarget->isLittle());
+      ++ByValArg;
       continue;
     }
 
@@ -2729,10 +2849,8 @@ MipsTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
 
     // emit ISD::STORE whichs stores the
     // parameter value to a stack Location
-    SDValue PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(), StackPtr,
-                                 DAG.getIntPtrConstant(VA.getLocMemOffset()));
-    MemOpChains.push_back(DAG.getStore(Chain, dl, Arg, PtrOff,
-                                       MachinePointerInfo(), false, false, 0));
+    MemOpChains.push_back(passArgOnStack(StackPtr, VA.getLocMemOffset(),
+                                         Chain, Arg, dl, isTailCall, DAG));
   }
 
   // Transform all store nodes into one single node because all store
@@ -2861,6 +2979,9 @@ MipsTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   if (InFlag.getNode())
     Ops.push_back(InFlag);
 
+  if (isTailCall)
+    return DAG.getNode(MipsISD::TailCall, dl, MVT::Other, &Ops[0], Ops.size());
+
   Chain  = DAG.getNode(MipsISD::JmpLink, dl, NodeTys, &Ops[0], Ops.size());
   InFlag = Chain.getValue(1);
 
@@ -2904,70 +3025,6 @@ MipsTargetLowering::LowerCallResult(SDValue Chain, SDValue InFlag,
 //===----------------------------------------------------------------------===//
 //             Formal Arguments Calling Convention Implementation
 //===----------------------------------------------------------------------===//
-static void ReadByValArg(MachineFunction &MF, SDValue Chain, DebugLoc dl,
-                         std::vector<SDValue> &OutChains,
-                         SelectionDAG &DAG, unsigned NumWords, SDValue FIN,
-                         const CCValAssign &VA, const ISD::ArgFlagsTy &Flags,
-                         const Argument *FuncArg) {
-  unsigned LocMem = VA.getLocMemOffset();
-  unsigned FirstWord = LocMem / 4;
-
-  // copy register A0 - A3 to frame object
-  for (unsigned i = 0; i < NumWords; ++i) {
-    unsigned CurWord = FirstWord + i;
-    if (CurWord >= O32IntRegsSize)
-      break;
-
-    unsigned SrcReg = O32IntRegs[CurWord];
-    unsigned Reg = AddLiveIn(MF, SrcReg, &Mips::CPURegsRegClass);
-    SDValue StorePtr = DAG.getNode(ISD::ADD, dl, MVT::i32, FIN,
-                                   DAG.getConstant(i * 4, MVT::i32));
-    SDValue Store = DAG.getStore(Chain, dl, DAG.getRegister(Reg, MVT::i32),
-                                 StorePtr, MachinePointerInfo(FuncArg, i * 4),
-                                 false, false, 0);
-    OutChains.push_back(Store);
-  }
-}
-
-// Create frame object on stack and copy registers used for byval passing to it.
-static unsigned
-CopyMips64ByValRegs(MachineFunction &MF, SDValue Chain, DebugLoc dl,
-                    std::vector<SDValue> &OutChains, SelectionDAG &DAG,
-                    const CCValAssign &VA, const ISD::ArgFlagsTy &Flags,
-                    MachineFrameInfo *MFI, bool IsRegLoc,
-                    SmallVectorImpl<SDValue> &InVals, MipsFunctionInfo *MipsFI,
-                    EVT PtrTy, const Argument *FuncArg) {
-  const uint16_t *Reg = Mips64IntRegs + 8;
-  int FOOffset; // Frame object offset from virtual frame pointer.
-
-  if (IsRegLoc) {
-    Reg = std::find(Mips64IntRegs, Mips64IntRegs + 8, VA.getLocReg());
-    FOOffset = (Reg - Mips64IntRegs) * 8 - 8 * 8;
-  }
-  else
-    FOOffset = VA.getLocMemOffset();
-
-  // Create frame object.
-  unsigned NumRegs = (Flags.getByValSize() + 7) / 8;
-  unsigned LastFI = MFI->CreateFixedObject(NumRegs * 8, FOOffset, true);
-  SDValue FIN = DAG.getFrameIndex(LastFI, PtrTy);
-  InVals.push_back(FIN);
-
-  // Copy arg registers.
-  for (unsigned I = 0; (Reg != Mips64IntRegs + 8) && (I < NumRegs);
-       ++Reg, ++I) {
-    unsigned VReg = AddLiveIn(MF, *Reg, &Mips::CPU64RegsRegClass);
-    SDValue StorePtr = DAG.getNode(ISD::ADD, dl, PtrTy, FIN,
-                                   DAG.getConstant(I * 8, PtrTy));
-    SDValue Store = DAG.getStore(Chain, dl, DAG.getRegister(VReg, MVT::i64),
-                                 StorePtr, MachinePointerInfo(FuncArg, I * 8),
-                                 false, false, 0);
-    OutChains.push_back(Store);
-  }
-
-  return LastFI;
-}
-
 /// LowerFormalArguments - transform physical registers into virtual registers
 /// and generate load operations for arguments places on the stack.
 SDValue
@@ -2991,20 +3048,21 @@ MipsTargetLowering::LowerFormalArguments(SDValue Chain,
   SmallVector<CCValAssign, 16> ArgLocs;
   CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(),
                  getTargetMachine(), ArgLocs, *DAG.getContext());
+  MipsCC MipsCCInfo(CallConv, isVarArg, IsO32, CCInfo);
 
-  if (CallConv == CallingConv::Fast)
-    CCInfo.AnalyzeFormalArguments(Ins, CC_Mips_FastCC);
-  else if (IsO32)
-    CCInfo.AnalyzeFormalArguments(Ins, CC_MipsO32);
-  else
-    CCInfo.AnalyzeFormalArguments(Ins, CC_Mips);
+  MipsCCInfo.analyzeFormalArguments(Ins);
+  MipsFI->setFormalArgInfo(CCInfo.getNextStackOffset(),
+                           MipsCCInfo.hasByValArg());
 
   Function::const_arg_iterator FuncArg =
     DAG.getMachineFunction().getFunction()->arg_begin();
-  int LastFI = 0;// MipsFI->LastInArgFI is 0 at the entry of this function.
+  unsigned CurArgIdx = 0;
+  MipsCC::byval_iterator ByValArg = MipsCCInfo.byval_begin();
 
-  for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i, ++FuncArg) {
+  for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
     CCValAssign &VA = ArgLocs[i];
+    std::advance(FuncArg, Ins[i].OrigArgIndex - CurArgIdx);
+    CurArgIdx = Ins[i].OrigArgIndex;
     EVT ValVT = VA.getValVT();
     ISD::ArgFlagsTy Flags = Ins[i].Flags;
     bool IsRegLoc = VA.isRegLoc();
@@ -3012,18 +3070,10 @@ MipsTargetLowering::LowerFormalArguments(SDValue Chain,
     if (Flags.isByVal()) {
       assert(Flags.getByValSize() &&
              "ByVal args of size 0 should have been ignored by front-end.");
-      if (IsO32) {
-        unsigned NumWords = (Flags.getByValSize() + 3) / 4;
-        LastFI = MFI->CreateFixedObject(NumWords * 4, VA.getLocMemOffset(),
-                                        true);
-        SDValue FIN = DAG.getFrameIndex(LastFI, getPointerTy());
-        InVals.push_back(FIN);
-        ReadByValArg(MF, Chain, dl, OutChains, DAG, NumWords, FIN, VA, Flags,
-                     &*FuncArg);
-      } else // N32/64
-        LastFI = CopyMips64ByValRegs(MF, Chain, dl, OutChains, DAG, VA, Flags,
-                                     MFI, IsRegLoc, InVals, MipsFI,
-                                     getPointerTy(), &*FuncArg);
+      assert(ByValArg != MipsCCInfo.byval_end());
+      copyByValRegs(Chain, dl, OutChains, DAG, Flags, InVals, &*FuncArg,
+                    MipsCCInfo, *ByValArg);
+      ++ByValArg;
       continue;
     }
 
@@ -3085,13 +3135,13 @@ MipsTargetLowering::LowerFormalArguments(SDValue Chain,
       assert(VA.isMemLoc());
 
       // The stack pointer offset is relative to the caller stack frame.
-      LastFI = MFI->CreateFixedObject(ValVT.getSizeInBits()/8,
+      int FI = MFI->CreateFixedObject(ValVT.getSizeInBits()/8,
                                       VA.getLocMemOffset(), true);
 
       // Create load nodes to retrieve arguments from the stack
-      SDValue FIN = DAG.getFrameIndex(LastFI, getPointerTy());
+      SDValue FIN = DAG.getFrameIndex(FI, getPointerTy());
       InVals.push_back(DAG.getLoad(ValVT, dl, Chain, FIN,
-                                   MachinePointerInfo::getFixedStack(LastFI),
+                                   MachinePointerInfo::getFixedStack(FI),
                                    false, false, false, 0));
     }
   }
@@ -3102,55 +3152,16 @@ MipsTargetLowering::LowerFormalArguments(SDValue Chain,
   if (DAG.getMachineFunction().getFunction()->hasStructRetAttr()) {
     unsigned Reg = MipsFI->getSRetReturnReg();
     if (!Reg) {
-      Reg = MF.getRegInfo().createVirtualRegister(getRegClassFor(MVT::i32));
+      Reg = MF.getRegInfo().
+        createVirtualRegister(getRegClassFor(IsN64 ? MVT::i64 : MVT::i32));
       MipsFI->setSRetReturnReg(Reg);
     }
     SDValue Copy = DAG.getCopyToReg(DAG.getEntryNode(), dl, Reg, InVals[0]);
     Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Copy, Chain);
   }
 
-  if (isVarArg) {
-    unsigned NumOfRegs = IsO32 ? 4 : 8;
-    const uint16_t *ArgRegs = IsO32 ? O32IntRegs : Mips64IntRegs;
-    unsigned Idx = CCInfo.getFirstUnallocated(ArgRegs, NumOfRegs);
-    int FirstRegSlotOffset = IsO32 ? 0 : -64 ; // offset of $a0's slot.
-    const TargetRegisterClass *RC = IsO32 ?
-      (const TargetRegisterClass*)&Mips::CPURegsRegClass :
-      (const TargetRegisterClass*)&Mips::CPU64RegsRegClass;
-    unsigned RegSize = RC->getSize();
-    int RegSlotOffset = FirstRegSlotOffset + Idx * RegSize;
-
-    // Offset of the first variable argument from stack pointer.
-    int FirstVaArgOffset;
-
-    if (IsO32 || (Idx == NumOfRegs)) {
-      FirstVaArgOffset =
-        (CCInfo.getNextStackOffset() + RegSize - 1) / RegSize * RegSize;
-    } else
-      FirstVaArgOffset = RegSlotOffset;
-
-    // Record the frame index of the first variable argument
-    // which is a value necessary to VASTART.
-    LastFI = MFI->CreateFixedObject(RegSize, FirstVaArgOffset, true);
-    MipsFI->setVarArgsFrameIndex(LastFI);
-
-    // Copy the integer registers that have not been used for argument passing
-    // to the argument register save area. For O32, the save area is allocated
-    // in the caller's stack frame, while for N32/64, it is allocated in the
-    // callee's stack frame.
-    for (int StackOffset = RegSlotOffset;
-         Idx < NumOfRegs; ++Idx, StackOffset += RegSize) {
-      unsigned Reg = AddLiveIn(DAG.getMachineFunction(), ArgRegs[Idx], RC);
-      SDValue ArgValue = DAG.getCopyFromReg(Chain, dl, Reg,
-                                            MVT::getIntegerVT(RegSize * 8));
-      LastFI = MFI->CreateFixedObject(RegSize, StackOffset, true);
-      SDValue PtrOff = DAG.getFrameIndex(LastFI, getPointerTy());
-      OutChains.push_back(DAG.getStore(Chain, dl, ArgValue, PtrOff,
-                                       MachinePointerInfo(), false, false, 0));
-    }
-  }
-
-  MipsFI->setLastInArgFI(LastFI);
+  if (isVarArg)
+    writeVarArgRegs(OutChains, MipsCCInfo, Chain, dl, DAG);
 
   // All stores are grouped in one node to allow the matching between
   // the size of Ins and InVals. This only happens when on varg functions
@@ -3167,6 +3178,17 @@ MipsTargetLowering::LowerFormalArguments(SDValue Chain,
 //               Return Value Calling Convention Implementation
 //===----------------------------------------------------------------------===//
 
+bool
+MipsTargetLowering::CanLowerReturn(CallingConv::ID CallConv,
+                                   MachineFunction &MF, bool isVarArg,
+                                   const SmallVectorImpl<ISD::OutputArg> &Outs,
+                                   LLVMContext &Context) const {
+  SmallVector<CCValAssign, 16> RVLocs;
+  CCState CCInfo(CallConv, isVarArg, MF, getTargetMachine(),
+                 RVLocs, Context);
+  return CCInfo.CheckReturn(Outs, RetCC_Mips);
+}
+
 SDValue
 MipsTargetLowering::LowerReturn(SDValue Chain,
                                 CallingConv::ID CallConv, bool isVarArg,
@@ -3219,9 +3241,11 @@ MipsTargetLowering::LowerReturn(SDValue Chain,
     if (!Reg)
       llvm_unreachable("sret virtual register not created in the entry block");
     SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg, getPointerTy());
+    unsigned V0 = IsN64 ? Mips::V0_64 : Mips::V0;
 
-    Chain = DAG.getCopyToReg(Chain, dl, Mips::V0, Val, Flag);
+    Chain = DAG.getCopyToReg(Chain, dl, V0, Val, Flag);
     Flag = Chain.getValue(1);
+    MF.getRegInfo().addLiveOut(V0);
   }
 
   // Return on Mips is always a "jr $ra"
@@ -3325,8 +3349,11 @@ getRegForInlineAsmConstraint(const std::string &Constraint, EVT VT) const
     case 'd': // Address register. Same as 'r' unless generating MIPS16 code.
     case 'y': // Same as 'r'. Exists for compatibility.
     case 'r':
-      if (VT == MVT::i32 || VT == MVT::i16 || VT == MVT::i8)
+      if (VT == MVT::i32 || VT == MVT::i16 || VT == MVT::i8) {
+        if (Subtarget->inMips16Mode())
+          return std::make_pair(0U, &Mips::CPU16RegsRegClass);
         return std::make_pair(0U, &Mips::CPURegsRegClass);
+      }
       if (VT == MVT::i64 && !HasMips64)
         return std::make_pair(0U, &Mips::CPURegsRegClass);
       if (VT == MVT::i64 && HasMips64)
@@ -3485,3 +3512,316 @@ unsigned MipsTargetLowering::getJumpTableEncoding() const {
 
   return TargetLowering::getJumpTableEncoding();
 }
+
+MipsTargetLowering::MipsCC::MipsCC(CallingConv::ID CallConv, bool IsVarArg,
+                                   bool IsO32, CCState &Info) : CCInfo(Info) {
+  UseRegsForByval = true;
+
+  if (IsO32) {
+    RegSize = 4;
+    NumIntArgRegs = array_lengthof(O32IntRegs);
+    ReservedArgArea = 16;
+    IntArgRegs = ShadowRegs = O32IntRegs;
+    FixedFn = VarFn = CC_MipsO32;
+  } else {
+    RegSize = 8;
+    NumIntArgRegs = array_lengthof(Mips64IntRegs);
+    ReservedArgArea = 0;
+    IntArgRegs = Mips64IntRegs;
+    ShadowRegs = Mips64DPRegs;
+    FixedFn = CC_MipsN;
+    VarFn = CC_MipsN_VarArg;
+  }
+
+  if (CallConv == CallingConv::Fast) {
+    assert(!IsVarArg);
+    UseRegsForByval = false;
+    ReservedArgArea = 0;
+    FixedFn = VarFn = CC_Mips_FastCC;
+  }
+
+  // Pre-allocate reserved argument area.
+  CCInfo.AllocateStack(ReservedArgArea, 1);
+}
+
+void MipsTargetLowering::MipsCC::
+analyzeCallOperands(const SmallVectorImpl<ISD::OutputArg> &Args) {
+  unsigned NumOpnds = Args.size();
+
+  for (unsigned I = 0; I != NumOpnds; ++I) {
+    MVT ArgVT = Args[I].VT;
+    ISD::ArgFlagsTy ArgFlags = Args[I].Flags;
+    bool R;
+
+    if (ArgFlags.isByVal()) {
+      handleByValArg(I, ArgVT, ArgVT, CCValAssign::Full, ArgFlags);
+      continue;
+    }
+
+    if (Args[I].IsFixed)
+      R = FixedFn(I, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo);
+    else
+      R = VarFn(I, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo);
+
+    if (R) {
+#ifndef NDEBUG
+      dbgs() << "Call operand #" << I << " has unhandled type "
+             << EVT(ArgVT).getEVTString();
+#endif
+      llvm_unreachable(0);
+    }
+  }
+}
+
+void MipsTargetLowering::MipsCC::
+analyzeFormalArguments(const SmallVectorImpl<ISD::InputArg> &Args) {
+  unsigned NumArgs = Args.size();
+
+  for (unsigned I = 0; I != NumArgs; ++I) {
+    MVT ArgVT = Args[I].VT;
+    ISD::ArgFlagsTy ArgFlags = Args[I].Flags;
+
+    if (ArgFlags.isByVal()) {
+      handleByValArg(I, ArgVT, ArgVT, CCValAssign::Full, ArgFlags);
+      continue;
+    }
+
+    if (!FixedFn(I, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo))
+      continue;
+
+#ifndef NDEBUG
+    dbgs() << "Formal Arg #" << I << " has unhandled type "
+           << EVT(ArgVT).getEVTString();
+#endif
+    llvm_unreachable(0);
+  }
+}
+
+void
+MipsTargetLowering::MipsCC::handleByValArg(unsigned ValNo, MVT ValVT,
+                                           MVT LocVT,
+                                           CCValAssign::LocInfo LocInfo,
+                                           ISD::ArgFlagsTy ArgFlags) {
+  assert(ArgFlags.getByValSize() && "Byval argument's size shouldn't be 0.");
+
+  struct ByValArgInfo ByVal;
+  unsigned ByValSize = RoundUpToAlignment(ArgFlags.getByValSize(), RegSize);
+  unsigned Align = std::min(std::max(ArgFlags.getByValAlign(), RegSize),
+                            RegSize * 2);
+
+  if (UseRegsForByval)
+    allocateRegs(ByVal, ByValSize, Align);
+
+  // Allocate space on caller's stack.
+  ByVal.Address = CCInfo.AllocateStack(ByValSize - RegSize * ByVal.NumRegs,
+                                       Align);
+  CCInfo.addLoc(CCValAssign::getMem(ValNo, ValVT, ByVal.Address, LocVT,
+                                    LocInfo));
+  ByValArgs.push_back(ByVal);
+}
+
+void MipsTargetLowering::MipsCC::allocateRegs(ByValArgInfo &ByVal,
+                                              unsigned ByValSize,
+                                              unsigned Align) {
+  assert(!(ByValSize % RegSize) && !(Align % RegSize) &&
+         "Byval argument's size and alignment should be a multiple of"
+         "RegSize.");
+
+  ByVal.FirstIdx = CCInfo.getFirstUnallocated(IntArgRegs, NumIntArgRegs);
+
+  // If Align > RegSize, the first arg register must be even.
+  if ((Align > RegSize) && (ByVal.FirstIdx % 2)) {
+    CCInfo.AllocateReg(IntArgRegs[ByVal.FirstIdx], ShadowRegs[ByVal.FirstIdx]);
+    ++ByVal.FirstIdx;
+  }
+
+  // Mark the registers allocated.
+  for (unsigned I = ByVal.FirstIdx; ByValSize && (I < NumIntArgRegs);
+       ByValSize -= RegSize, ++I, ++ByVal.NumRegs)
+    CCInfo.AllocateReg(IntArgRegs[I], ShadowRegs[I]);
+}
+
+void MipsTargetLowering::
+copyByValRegs(SDValue Chain, DebugLoc DL, std::vector<SDValue> &OutChains,
+              SelectionDAG &DAG, const ISD::ArgFlagsTy &Flags,
+              SmallVectorImpl<SDValue> &InVals, const Argument *FuncArg,
+              const MipsCC &CC, const ByValArgInfo &ByVal) const {
+  MachineFunction &MF = DAG.getMachineFunction();
+  MachineFrameInfo *MFI = MF.getFrameInfo();
+  unsigned RegAreaSize = ByVal.NumRegs * CC.regSize();
+  unsigned FrameObjSize = std::max(Flags.getByValSize(), RegAreaSize);
+  int FrameObjOffset;
+
+  if (RegAreaSize)
+    FrameObjOffset = (int)CC.reservedArgArea() -
+      (int)((CC.numIntArgRegs() - ByVal.FirstIdx) * CC.regSize());
+  else
+    FrameObjOffset = ByVal.Address;
+
+  // Create frame object.
+  EVT PtrTy = getPointerTy();
+  int FI = MFI->CreateFixedObject(FrameObjSize, FrameObjOffset, true);
+  SDValue FIN = DAG.getFrameIndex(FI, PtrTy);
+  InVals.push_back(FIN);
+
+  if (!ByVal.NumRegs)
+    return;
+
+  // Copy arg registers.
+  EVT RegTy = MVT::getIntegerVT(CC.regSize() * 8);
+  const TargetRegisterClass *RC = getRegClassFor(RegTy);
+
+  for (unsigned I = 0; I < ByVal.NumRegs; ++I) {
+    unsigned ArgReg = CC.intArgRegs()[ByVal.FirstIdx + I];
+    unsigned VReg = AddLiveIn(MF, ArgReg, RC);
+    unsigned Offset = I * CC.regSize();
+    SDValue StorePtr = DAG.getNode(ISD::ADD, DL, PtrTy, FIN,
+                                   DAG.getConstant(Offset, PtrTy));
+    SDValue Store = DAG.getStore(Chain, DL, DAG.getRegister(VReg, RegTy),
+                                 StorePtr, MachinePointerInfo(FuncArg, Offset),
+                                 false, false, 0);
+    OutChains.push_back(Store);
+  }
+}
+
+// Copy byVal arg to registers and stack.
+void MipsTargetLowering::
+passByValArg(SDValue Chain, DebugLoc DL,
+             SmallVector<std::pair<unsigned, SDValue>, 16> &RegsToPass,
+             SmallVector<SDValue, 8> &MemOpChains, SDValue StackPtr,
+             MachineFrameInfo *MFI, SelectionDAG &DAG, SDValue Arg,
+             const MipsCC &CC, const ByValArgInfo &ByVal,
+             const ISD::ArgFlagsTy &Flags, bool isLittle) const {
+  unsigned ByValSize = Flags.getByValSize();
+  unsigned Offset = 0; // Offset in # of bytes from the beginning of struct.
+  unsigned RegSize = CC.regSize();
+  unsigned Alignment = std::min(Flags.getByValAlign(), RegSize);
+  EVT PtrTy = getPointerTy(), RegTy = MVT::getIntegerVT(RegSize * 8);
+
+  if (ByVal.NumRegs) {
+    const uint16_t *ArgRegs = CC.intArgRegs();
+    bool LeftoverBytes = (ByVal.NumRegs * RegSize > ByValSize);
+    unsigned I = 0;
+
+    // Copy words to registers.
+    for (; I < ByVal.NumRegs - LeftoverBytes; ++I, Offset += RegSize) {
+      SDValue LoadPtr = DAG.getNode(ISD::ADD, DL, PtrTy, Arg,
+                                    DAG.getConstant(Offset, PtrTy));
+      SDValue LoadVal = DAG.getLoad(RegTy, DL, Chain, LoadPtr,
+                                    MachinePointerInfo(), false, false, false,
+                                    Alignment);
+      MemOpChains.push_back(LoadVal.getValue(1));
+      unsigned ArgReg = ArgRegs[ByVal.FirstIdx + I];
+      RegsToPass.push_back(std::make_pair(ArgReg, LoadVal));
+    }
+
+    // Return if the struct has been fully copied.
+    if (ByValSize == Offset)
+      return;
+
+    // Copy the remainder of the byval argument with sub-word loads and shifts.
+    if (LeftoverBytes) {
+      assert((ByValSize > Offset) && (ByValSize < Offset + RegSize) &&
+             "Size of the remainder should be smaller than RegSize.");
+      SDValue Val;
+
+      for (unsigned LoadSize = RegSize / 2, TotalSizeLoaded = 0;
+           Offset < ByValSize; LoadSize /= 2) {
+        unsigned RemSize = ByValSize - Offset;
+
+        if (RemSize < LoadSize)
+          continue;
+
+        // Load subword.
+        SDValue LoadPtr = DAG.getNode(ISD::ADD, DL, PtrTy, Arg,
+                                      DAG.getConstant(Offset, PtrTy));
+        SDValue LoadVal =
+          DAG.getExtLoad(ISD::ZEXTLOAD, DL, RegTy, Chain, LoadPtr,
+                         MachinePointerInfo(), MVT::getIntegerVT(LoadSize * 8),
+                         false, false, Alignment);
+        MemOpChains.push_back(LoadVal.getValue(1));
+
+        // Shift the loaded value.
+        unsigned Shamt;
+
+        if (isLittle)
+          Shamt = TotalSizeLoaded;
+        else
+          Shamt = (RegSize - (TotalSizeLoaded + LoadSize)) * 8;
+
+        SDValue Shift = DAG.getNode(ISD::SHL, DL, RegTy, LoadVal,
+                                    DAG.getConstant(Shamt, MVT::i32));
+
+        if (Val.getNode())
+          Val = DAG.getNode(ISD::OR, DL, RegTy, Val, Shift);
+        else
+          Val = Shift;
+
+        Offset += LoadSize;
+        TotalSizeLoaded += LoadSize;
+        Alignment = std::min(Alignment, LoadSize);
+      }
+
+      unsigned ArgReg = ArgRegs[ByVal.FirstIdx + I];
+      RegsToPass.push_back(std::make_pair(ArgReg, Val));
+      return;
+    }
+  }
+
+  // Copy remainder of byval arg to it with memcpy.
+  unsigned MemCpySize = ByValSize - Offset;
+  SDValue Src = DAG.getNode(ISD::ADD, DL, PtrTy, Arg,
+                            DAG.getConstant(Offset, PtrTy));
+  SDValue Dst = DAG.getNode(ISD::ADD, DL, PtrTy, StackPtr,
+                            DAG.getIntPtrConstant(ByVal.Address));
+  Chain = DAG.getMemcpy(Chain, DL, Dst, Src,
+                        DAG.getConstant(MemCpySize, PtrTy), Alignment,
+                        /*isVolatile=*/false, /*AlwaysInline=*/false,
+                        MachinePointerInfo(0), MachinePointerInfo(0));
+  MemOpChains.push_back(Chain);
+}
+
+void
+MipsTargetLowering::writeVarArgRegs(std::vector<SDValue> &OutChains,
+                                    const MipsCC &CC, SDValue Chain,
+                                    DebugLoc DL, SelectionDAG &DAG) const {
+  unsigned NumRegs = CC.numIntArgRegs();
+  const uint16_t *ArgRegs = CC.intArgRegs();
+  const CCState &CCInfo = CC.getCCInfo();
+  unsigned Idx = CCInfo.getFirstUnallocated(ArgRegs, NumRegs);
+  unsigned RegSize = CC.regSize();
+  EVT RegTy = MVT::getIntegerVT(RegSize * 8);
+  const TargetRegisterClass *RC = getRegClassFor(RegTy);
+  MachineFunction &MF = DAG.getMachineFunction();
+  MachineFrameInfo *MFI = MF.getFrameInfo();
+  MipsFunctionInfo *MipsFI = MF.getInfo<MipsFunctionInfo>();
+
+  // Offset of the first variable argument from stack pointer.
+  int VaArgOffset;
+
+  if (NumRegs == Idx)
+    VaArgOffset = RoundUpToAlignment(CCInfo.getNextStackOffset(), RegSize);
+  else
+    VaArgOffset =
+      (int)CC.reservedArgArea() - (int)(RegSize * (NumRegs - Idx));
+
+  // Record the frame index of the first variable argument
+  // which is a value necessary to VASTART.
+  int FI = MFI->CreateFixedObject(RegSize, VaArgOffset, true);
+  MipsFI->setVarArgsFrameIndex(FI);
+
+  // Copy the integer registers that have not been used for argument passing
+  // to the argument register save area. For O32, the save area is allocated
+  // in the caller's stack frame, while for N32/64, it is allocated in the
+  // callee's stack frame.
+  for (unsigned I = Idx; I < NumRegs; ++I, VaArgOffset += RegSize) {
+    unsigned Reg = AddLiveIn(MF, ArgRegs[I], RC);
+    SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, Reg, RegTy);
+    FI = MFI->CreateFixedObject(RegSize, VaArgOffset, true);
+    SDValue PtrOff = DAG.getFrameIndex(FI, getPointerTy());
+    SDValue Store = DAG.getStore(Chain, DL, ArgValue, PtrOff,
+                                 MachinePointerInfo(), false, false, 0);
+    cast<StoreSDNode>(Store.getNode())->getMemOperand()->setValue(0);
+    OutChains.push_back(Store);
+  }
+}
diff --git a/lib/Target/Mips/MipsISelLowering.h b/lib/Target/Mips/MipsISelLowering.h
index 95ea8fa..43f97e8 100644
--- a/lib/Target/Mips/MipsISelLowering.h
+++ b/lib/Target/Mips/MipsISelLowering.h
@@ -17,6 +17,7 @@
 
 #include "Mips.h"
 #include "MipsSubtarget.h"
+#include "llvm/CodeGen/CallingConvLower.h"
 #include "llvm/CodeGen/SelectionDAG.h"
 #include "llvm/Target/TargetLowering.h"
 
@@ -29,6 +30,9 @@ namespace llvm {
       // Jump and link (call)
       JmpLink,
 
+      // Tail call
+      TailCall,
+
       // Get the Higher 16 bits from a 32-bit immediate
       // No relation with Mips Hi register
       Hi,
@@ -81,6 +85,47 @@ namespace llvm {
       Ext,
       Ins,
 
+      // EXTR.W instrinsic nodes.
+      EXTP,
+      EXTPDP,
+      EXTR_S_H,
+      EXTR_W,
+      EXTR_R_W,
+      EXTR_RS_W,
+      SHILO,
+      MTHLIP,
+
+      // DPA.W intrinsic nodes.
+      MULSAQ_S_W_PH,
+      MAQ_S_W_PHL,
+      MAQ_S_W_PHR,
+      MAQ_SA_W_PHL,
+      MAQ_SA_W_PHR,
+      DPAU_H_QBL,
+      DPAU_H_QBR,
+      DPSU_H_QBL,
+      DPSU_H_QBR,
+      DPAQ_S_W_PH,
+      DPSQ_S_W_PH,
+      DPAQ_SA_L_W,
+      DPSQ_SA_L_W,
+      DPA_W_PH,
+      DPS_W_PH,
+      DPAQX_S_W_PH,
+      DPAQX_SA_W_PH,
+      DPAX_W_PH,
+      DPSX_W_PH,
+      DPSQX_S_W_PH,
+      DPSQX_SA_W_PH,
+      MULSA_W_PH,
+
+      MULT,
+      MULTU,
+      MADD_DSP,
+      MADDU_DSP,
+      MSUB_DSP,
+      MSUBU_DSP,
+
       // Load/Store Left/Right nodes.
       LWL = ISD::FIRST_TARGET_MEMORY_OPCODE,
       LWR,
@@ -96,6 +141,7 @@ namespace llvm {
   //===--------------------------------------------------------------------===//
   // TargetLowering Implementation
   //===--------------------------------------------------------------------===//
+  class MipsFunctionInfo;
 
   class MipsTargetLowering : public TargetLowering  {
   public:
@@ -105,9 +151,19 @@ namespace llvm {
 
     virtual bool allowsUnalignedMemoryAccesses (EVT VT) const;
 
+    virtual void LowerOperationWrapper(SDNode *N,
+                                       SmallVectorImpl<SDValue> &Results,
+                                       SelectionDAG &DAG) const;
+
     /// LowerOperation - Provide custom lowering hooks for some operations.
     virtual SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const;
 
+    /// ReplaceNodeResults - Replace the results of node with an illegal result
+    /// type with new values built out of custom code.
+    ///
+    virtual void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue>&Results,
+                                    SelectionDAG &DAG) const;
+
     /// getTargetNodeName - This method returns the name of a target specific
     //  DAG node.
     virtual const char *getTargetNodeName(unsigned Opcode) const;
@@ -117,6 +173,69 @@ namespace llvm {
 
     virtual SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const;
   private:
+
+    /// ByValArgInfo - Byval argument information.
+    struct ByValArgInfo {
+      unsigned FirstIdx; // Index of the first register used.
+      unsigned NumRegs;  // Number of registers used for this argument.
+      unsigned Address;  // Offset of the stack area used to pass this argument.
+
+      ByValArgInfo() : FirstIdx(0), NumRegs(0), Address(0) {}
+    };
+
+    /// MipsCC - This class provides methods used to analyze formal and call
+    /// arguments and inquire about calling convention information.
+    class MipsCC {
+    public:
+      MipsCC(CallingConv::ID CallConv, bool IsVarArg, bool IsO32,
+             CCState &Info);
+
+      void analyzeCallOperands(const SmallVectorImpl<ISD::OutputArg> &Outs);
+      void analyzeFormalArguments(const SmallVectorImpl<ISD::InputArg> &Ins);
+      void handleByValArg(unsigned ValNo, MVT ValVT, MVT LocVT,
+                          CCValAssign::LocInfo LocInfo,
+                          ISD::ArgFlagsTy ArgFlags);
+
+      const CCState &getCCInfo() const { return CCInfo; }
+
+      /// hasByValArg - Returns true if function has byval arguments.
+      bool hasByValArg() const { return !ByValArgs.empty(); }
+
+      /// useRegsForByval - Returns true if the calling convention allows the
+      /// use of registers to pass byval arguments.
+      bool useRegsForByval() const { return UseRegsForByval; }
+
+      /// regSize - Size (in number of bits) of integer registers.
+      unsigned regSize() const { return RegSize; }
+
+      /// numIntArgRegs - Number of integer registers available for calls.
+      unsigned numIntArgRegs() const { return NumIntArgRegs; }
+
+      /// reservedArgArea - The size of the area the caller reserves for
+      /// register arguments. This is 16-byte if ABI is O32.
+      unsigned reservedArgArea() const { return ReservedArgArea; }
+
+      /// intArgRegs - Pointer to array of integer registers.
+      const uint16_t *intArgRegs() const { return IntArgRegs; }
+
+      typedef SmallVector<ByValArgInfo, 2>::const_iterator byval_iterator;
+      byval_iterator byval_begin() const { return ByValArgs.begin(); }
+      byval_iterator byval_end() const { return ByValArgs.end(); }
+
+    private:
+      void allocateRegs(ByValArgInfo &ByVal, unsigned ByValSize,
+                        unsigned Align);
+
+      CCState &CCInfo;
+      bool UseRegsForByval;
+      unsigned RegSize;
+      unsigned NumIntArgRegs;
+      unsigned ReservedArgArea;
+      const uint16_t *IntArgRegs, *ShadowRegs;
+      SmallVector<ByValArgInfo, 2> ByValArgs;
+      llvm::CCAssignFn *FixedFn, *VarFn;
+    };
+
     // Subtarget Info
     const MipsSubtarget *Subtarget;
 
@@ -151,6 +270,39 @@ namespace llvm {
                                  bool IsSRA) const;
     SDValue LowerLOAD(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG) const;
+    SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
+    SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) const;
+    SDValue LowerADD(SDValue Op, SelectionDAG &DAG) const;
+
+    /// IsEligibleForTailCallOptimization - Check whether the call is eligible
+    /// for tail call optimization.
+    bool IsEligibleForTailCallOptimization(const MipsCC &MipsCCInfo,
+                                           unsigned NextStackOffset,
+                                           const MipsFunctionInfo& FI) const;
+
+    /// copyByValArg - Copy argument registers which were used to pass a byval
+    /// argument to the stack. Create a stack frame object for the byval
+    /// argument.
+    void copyByValRegs(SDValue Chain, DebugLoc DL,
+                       std::vector<SDValue> &OutChains, SelectionDAG &DAG,
+                       const ISD::ArgFlagsTy &Flags,
+                       SmallVectorImpl<SDValue> &InVals,
+                       const Argument *FuncArg,
+                       const MipsCC &CC, const ByValArgInfo &ByVal) const;
+
+    /// passByValArg - Pass a byval argument in registers or on stack.
+    void passByValArg(SDValue Chain, DebugLoc DL,
+                      SmallVector<std::pair<unsigned, SDValue>, 16> &RegsToPass,
+                      SmallVector<SDValue, 8> &MemOpChains, SDValue StackPtr,
+                      MachineFrameInfo *MFI, SelectionDAG &DAG, SDValue Arg,
+                      const MipsCC &CC, const ByValArgInfo &ByVal,
+                      const ISD::ArgFlagsTy &Flags, bool isLittle) const;
+
+    /// writeVarArgRegs - Write variable function arguments passed in registers
+    /// to the stack. Also create a stack frame object for the first variable
+    /// argument.
+    void writeVarArgRegs(std::vector<SDValue> &OutChains, const MipsCC &CC,
+                         SDValue Chain, DebugLoc DL, SelectionDAG &DAG) const;
 
     virtual SDValue
       LowerFormalArguments(SDValue Chain,
@@ -159,10 +311,20 @@ namespace llvm {
                            DebugLoc dl, SelectionDAG &DAG,
                            SmallVectorImpl<SDValue> &InVals) const;
 
+    SDValue passArgOnStack(SDValue StackPtr, unsigned Offset, SDValue Chain,
+                           SDValue Arg, DebugLoc DL, bool IsTailCall,
+                           SelectionDAG &DAG) const;
+
     virtual SDValue
       LowerCall(TargetLowering::CallLoweringInfo &CLI,
                 SmallVectorImpl<SDValue> &InVals) const;
 
+    virtual bool
+      CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF,
+                     bool isVarArg,
+                     const SmallVectorImpl<ISD::OutputArg> &Outs,
+                     LLVMContext &Context) const;
+
     virtual SDValue
       LowerReturn(SDValue Chain,
                   CallingConv::ID CallConv, bool isVarArg,
@@ -209,6 +371,8 @@ namespace llvm {
 
     virtual unsigned getJumpTableEncoding() const;
 
+    MachineBasicBlock *EmitBPOSGE32(MachineInstr *MI,
+                                    MachineBasicBlock *BB) const;
     MachineBasicBlock *EmitAtomicBinary(MachineInstr *MI, MachineBasicBlock *BB,
                     unsigned Size, unsigned BinOpcode, bool Nand = false) const;
     MachineBasicBlock *EmitAtomicBinaryPartword(MachineInstr *MI,
diff --git a/lib/Target/Mips/MipsInstrFPU.td b/lib/Target/Mips/MipsInstrFPU.td
index 3e78c45..33ee020 100644
--- a/lib/Target/Mips/MipsInstrFPU.td
+++ b/lib/Target/Mips/MipsInstrFPU.td
@@ -90,20 +90,20 @@ def fpimm0neg : PatLeaf<(fpimm), [{
 let DecoderMethod = "DecodeFMem" in {
 class FPLoad<bits<6> op, string opstr, RegisterClass RC, Operand MemOpnd>:
   FMem<op, (outs RC:$ft), (ins MemOpnd:$addr),
-      !strconcat(opstr, "\t$ft, $addr"), [(set RC:$ft, (load_a addr:$addr))],
+      !strconcat(opstr, "\t$ft, $addr"), [(set RC:$ft, (load addr:$addr))],
       IILoad>;
 
 // FP store.
 class FPStore<bits<6> op, string opstr, RegisterClass RC, Operand MemOpnd>:
   FMem<op, (outs), (ins RC:$ft, MemOpnd:$addr),
-      !strconcat(opstr, "\t$ft, $addr"), [(store_a RC:$ft, addr:$addr)],
+      !strconcat(opstr, "\t$ft, $addr"), [(store RC:$ft, addr:$addr)],
       IIStore>;
 }
 // FP indexed load.
 class FPIdxLoad<bits<6> funct, string opstr, RegisterClass DRC,
                 RegisterClass PRC, SDPatternOperator FOp = null_frag>:
   FFMemIdx<funct, (outs DRC:$fd), (ins PRC:$base, PRC:$index),
-           !strconcat(opstr, "\t$fd, $index($base)"),
+           !strconcat(opstr, "\t$fd, ${index}(${base})"),
            [(set DRC:$fd, (FOp (add PRC:$base, PRC:$index)))]> {
   let fs = 0;
 }
@@ -112,7 +112,7 @@ class FPIdxLoad<bits<6> funct, string opstr, RegisterClass DRC,
 class FPIdxStore<bits<6> funct, string opstr, RegisterClass DRC,
                  RegisterClass PRC, SDPatternOperator FOp= null_frag>:
   FFMemIdx<funct, (outs), (ins DRC:$fs, PRC:$base, PRC:$index),
-           !strconcat(opstr, "\t$fs, $index($base)"),
+           !strconcat(opstr, "\t$fs, ${index}(${base})"),
            [(FOp DRC:$fs, (add PRC:$base, PRC:$index))]> {
   let fd = 0;
 }
@@ -182,20 +182,21 @@ defm CEIL_W  : FFR1_W_M<0xe, "ceil">;
 defm CEIL_L  : FFR1_L_M<0xa, "ceil">;
 defm FLOOR_W : FFR1_W_M<0xf, "floor">;
 defm FLOOR_L : FFR1_L_M<0xb, "floor">;
-defm CVT_W   : FFR1_W_M<0x24, "cvt">;
+defm CVT_W   : FFR1_W_M<0x24, "cvt">, NeverHasSideEffects;
 //defm CVT_L   : FFR1_L_M<0x25, "cvt">;
 
-def CVT_S_W : FFR1<0x20, 20, "cvt", "s.w", FGR32, FGR32>;
-def CVT_L_S : FFR1<0x25, 16, "cvt", "l.s", FGR64, FGR32>;
-def CVT_L_D64: FFR1<0x25, 17, "cvt", "l.d", FGR64, FGR64>;
+def CVT_S_W : FFR1<0x20, 20, "cvt", "s.w", FGR32, FGR32>, NeverHasSideEffects;
+def CVT_L_S : FFR1<0x25, 16, "cvt", "l.s", FGR64, FGR32>, NeverHasSideEffects;
+def CVT_L_D64: FFR1<0x25, 17, "cvt", "l.d", FGR64, FGR64>, NeverHasSideEffects;
 
-let Predicates = [NotFP64bit, HasStandardEncoding] in {
+let Predicates = [NotFP64bit, HasStandardEncoding], neverHasSideEffects = 1 in {
   def CVT_S_D32 : FFR1<0x20, 17, "cvt", "s.d", FGR32, AFGR64>;
   def CVT_D32_W : FFR1<0x21, 20, "cvt", "d.w", AFGR64, FGR32>;
   def CVT_D32_S : FFR1<0x21, 16, "cvt", "d.s", AFGR64, FGR32>;
 }
 
-let Predicates = [IsFP64bit, HasStandardEncoding], DecoderNamespace = "Mips64" in {
+let Predicates = [IsFP64bit, HasStandardEncoding], DecoderNamespace = "Mips64",
+    neverHasSideEffects = 1 in {
  def CVT_S_D64 : FFR1<0x20, 17, "cvt", "s.d", FGR32, FGR64>;
  def CVT_S_L   : FFR1<0x20, 21, "cvt", "s.l", FGR32, FGR64>;
  def CVT_D64_W : FFR1<0x21, 20, "cvt", "d.w", FGR64, FGR32>;
@@ -282,26 +283,26 @@ let Predicates = [NotN64, NotMips64, HasStandardEncoding] in {
 
 // Indexed loads and stores.
 let Predicates = [HasMips32r2Or64, HasStandardEncoding] in {
-  def LWXC1 : FPIdxLoad<0x0, "lwxc1", FGR32, CPURegs, load_a>;
-  def SWXC1 : FPIdxStore<0x8, "swxc1", FGR32, CPURegs, store_a>;
+  def LWXC1 : FPIdxLoad<0x0, "lwxc1", FGR32, CPURegs, load>;
+  def SWXC1 : FPIdxStore<0x8, "swxc1", FGR32, CPURegs, store>;
 }
 
 let Predicates = [HasMips32r2, NotMips64, HasStandardEncoding] in {
-  def LDXC1 : FPIdxLoad<0x1, "ldxc1", AFGR64, CPURegs, load_a>;
-  def SDXC1 : FPIdxStore<0x9, "sdxc1", AFGR64, CPURegs, store_a>;
+  def LDXC1 : FPIdxLoad<0x1, "ldxc1", AFGR64, CPURegs, load>;
+  def SDXC1 : FPIdxStore<0x9, "sdxc1", AFGR64, CPURegs, store>;
 }
 
 let Predicates = [HasMips64, NotN64, HasStandardEncoding], DecoderNamespace="Mips64" in {
-  def LDXC164 : FPIdxLoad<0x1, "ldxc1", FGR64, CPURegs, load_a>;
-  def SDXC164 : FPIdxStore<0x9, "sdxc1", FGR64, CPURegs, store_a>;
+  def LDXC164 : FPIdxLoad<0x1, "ldxc1", FGR64, CPURegs, load>;
+  def SDXC164 : FPIdxStore<0x9, "sdxc1", FGR64, CPURegs, store>;
 }
 
 // n64
 let Predicates = [IsN64, HasStandardEncoding], isCodeGenOnly=1 in {
-  def LWXC1_P8   : FPIdxLoad<0x0, "lwxc1", FGR32, CPU64Regs, load_a>;
-  def LDXC164_P8 : FPIdxLoad<0x1, "ldxc1", FGR64, CPU64Regs, load_a>;
-  def SWXC1_P8   : FPIdxStore<0x8, "swxc1", FGR32, CPU64Regs, store_a>;
-  def SDXC164_P8 : FPIdxStore<0x9, "sdxc1", FGR64, CPU64Regs, store_a>;
+  def LWXC1_P8   : FPIdxLoad<0x0, "lwxc1", FGR32, CPU64Regs, load>;
+  def LDXC164_P8 : FPIdxLoad<0x1, "ldxc1", FGR64, CPU64Regs, load>;
+  def SWXC1_P8   : FPIdxStore<0x8, "swxc1", FGR32, CPU64Regs, store>;
+  def SDXC164_P8 : FPIdxStore<0x9, "sdxc1", FGR64, CPU64Regs, store>;
 }
 
 // Load/store doubleword indexed unaligned.
diff --git a/lib/Target/Mips/MipsInstrFormats.td b/lib/Target/Mips/MipsInstrFormats.td
index 8feb853..1ecbdc2 100644
--- a/lib/Target/Mips/MipsInstrFormats.td
+++ b/lib/Target/Mips/MipsInstrFormats.td
@@ -92,6 +92,14 @@ class PseudoSE<dag outs, dag ins, string asmstr, list<dag> pattern>:
   let Predicates = [HasStandardEncoding];
 }
 
+// Pseudo-instructions for alternate assembly syntax (never used by codegen).
+// These are aliases that require C++ handling to convert to the target
+// instruction, while InstAliases can be handled directly by tblgen.
+class MipsAsmPseudoInst<dag outs, dag ins, string asmstr>:
+  MipsInst<outs, ins, asmstr, [], IIPseudo, Pseudo> {
+  let isPseudo = 1;
+  let Pattern = [];
+}
 //===----------------------------------------------------------------------===//
 // Format R instruction class in Mips : <|opcode|rs|rt|rd|shamt|funct|>
 //===----------------------------------------------------------------------===//
@@ -163,6 +171,27 @@ class FJ<bits<6> op, dag outs, dag ins, string asmstr, list<dag> pattern,
   let Inst{25-0} = addr;
 }
 
+ //===----------------------------------------------------------------------===//
+// MFC instruction class in Mips : <|op|mf|rt|rd|0000000|sel|>
+//===----------------------------------------------------------------------===//
+class MFC3OP<bits<6> op, bits<5> _mfmt, dag outs, dag ins, string asmstr>:
+  InstSE<outs, ins, asmstr, [], NoItinerary, FrmFR>
+{
+  bits<5> mfmt;
+  bits<5> rt;
+  bits<5> rd;
+  bits<3> sel;
+
+  let Opcode = op;
+  let mfmt = _mfmt;
+
+  let Inst{25-21} = mfmt;
+  let Inst{20-16} = rt;
+  let Inst{15-11} = rd;
+  let Inst{10-3}  = 0;
+  let Inst{2-0}   = sel;
+}
+
 //===----------------------------------------------------------------------===//
 //
 //  FLOATING POINT INSTRUCTION FORMATS
diff --git a/lib/Target/Mips/MipsInstrInfo.cpp b/lib/Target/Mips/MipsInstrInfo.cpp
index 50e3eb5..ca80d43 100644
--- a/lib/Target/Mips/MipsInstrInfo.cpp
+++ b/lib/Target/Mips/MipsInstrInfo.cpp
@@ -95,6 +95,7 @@ bool MipsInstrInfo::AnalyzeBranch(MachineBasicBlock &MBB,
                                   SmallVectorImpl<MachineOperand> &Cond,
                                   bool AllowModify) const
 {
+
   MachineBasicBlock::reverse_iterator I = MBB.rbegin(), REnd = MBB.rend();
 
   // Skip all the debug instructions.
@@ -177,9 +178,14 @@ void MipsInstrInfo::BuildCondBr(MachineBasicBlock &MBB,
   const MCInstrDesc &MCID = get(Opc);
   MachineInstrBuilder MIB = BuildMI(&MBB, DL, MCID);
 
-  for (unsigned i = 1; i < Cond.size(); ++i)
-    MIB.addReg(Cond[i].getReg());
-
+  for (unsigned i = 1; i < Cond.size(); ++i) {
+    if (Cond[i].isReg())
+      MIB.addReg(Cond[i].getReg());
+    else if (Cond[i].isImm())
+      MIB.addImm(Cond[i].getImm());
+    else
+       assert(true && "Cannot copy operand");
+  }
   MIB.addMBB(TBB);
 }
 
@@ -262,46 +268,3 @@ unsigned MipsInstrInfo::GetInstSizeInBytes(const MachineInstr *MI) const {
   }
   }
 }
-
-unsigned
-llvm::Mips::loadImmediate(int64_t Imm, bool IsN64, const TargetInstrInfo &TII,
-                          MachineBasicBlock& MBB,
-                          MachineBasicBlock::iterator II, DebugLoc DL,
-                          bool LastInstrIsADDiu,
-                          MipsAnalyzeImmediate::Inst *LastInst) {
-  MipsAnalyzeImmediate AnalyzeImm;
-  unsigned Size = IsN64 ? 64 : 32;
-  unsigned LUi = IsN64 ? Mips::LUi64 : Mips::LUi;
-  unsigned ZEROReg = IsN64 ? Mips::ZERO_64 : Mips::ZERO;
-  unsigned ATReg = IsN64 ? Mips::AT_64 : Mips::AT;
-
-  const MipsAnalyzeImmediate::InstSeq &Seq =
-    AnalyzeImm.Analyze(Imm, Size, LastInstrIsADDiu);
-  MipsAnalyzeImmediate::InstSeq::const_iterator Inst = Seq.begin();
-
-  if (LastInst && (Seq.size() == 1)) {
-    *LastInst = *Inst;
-    return 0;
-  }
-
-  // The first instruction can be a LUi, which is different from other
-  // instructions (ADDiu, ORI and SLL) in that it does not have a register
-  // operand.
-  if (Inst->Opc == LUi)
-    BuildMI(MBB, II, DL, TII.get(LUi), ATReg)
-      .addImm(SignExtend64<16>(Inst->ImmOpnd));
-  else
-    BuildMI(MBB, II, DL, TII.get(Inst->Opc), ATReg).addReg(ZEROReg)
-      .addImm(SignExtend64<16>(Inst->ImmOpnd));
-
-  // Build the remaining instructions in Seq. Skip the last instruction if
-  // LastInst is not 0.
-  for (++Inst; Inst != Seq.end() - !!LastInst; ++Inst)
-    BuildMI(MBB, II, DL, TII.get(Inst->Opc), ATReg).addReg(ATReg)
-      .addImm(SignExtend64<16>(Inst->ImmOpnd));
-
-  if (LastInst)
-    *LastInst = *Inst;
-
-  return Seq.size() - !!LastInst;
-}
diff --git a/lib/Target/Mips/MipsInstrInfo.h b/lib/Target/Mips/MipsInstrInfo.h
index 7d56259..aca2bc7 100644
--- a/lib/Target/Mips/MipsInstrInfo.h
+++ b/lib/Target/Mips/MipsInstrInfo.h
@@ -88,18 +88,6 @@ private:
                    const SmallVectorImpl<MachineOperand>& Cond) const;
 };
 
-namespace Mips {
-  /// Emit a series of instructions to load an immediate. All instructions
-  /// except for the last one are emitted. The function returns the number of
-  /// MachineInstrs generated. The opcode-immediate pair of the last
-  /// instruction is returned in LastInst, if it is not 0.
-  unsigned
-  loadImmediate(int64_t Imm, bool IsN64, const TargetInstrInfo &TII,
-                MachineBasicBlock& MBB, MachineBasicBlock::iterator II,
-                DebugLoc DL, bool LastInstrIsADDiu,
-                MipsAnalyzeImmediate::Inst *LastInst);
-}
-
 /// Create MipsInstrInfo objects.
 const MipsInstrInfo *createMips16InstrInfo(MipsTargetMachine &TM);
 const MipsInstrInfo *createMipsSEInstrInfo(MipsTargetMachine &TM);
diff --git a/lib/Target/Mips/MipsInstrInfo.td b/lib/Target/Mips/MipsInstrInfo.td
index da15d4d..f16b5f9 100644
--- a/lib/Target/Mips/MipsInstrInfo.td
+++ b/lib/Target/Mips/MipsInstrInfo.td
@@ -52,6 +52,10 @@ def MipsJmpLink : SDNode<"MipsISD::JmpLink",SDT_MipsJmpLink,
                          [SDNPHasChain, SDNPOutGlue, SDNPOptInGlue,
                           SDNPVariadic]>;
 
+// Tail call
+def MipsTailCall : SDNode<"MipsISD::TailCall", SDT_MipsJmpLink,
+                          [SDNPHasChain, SDNPOptInGlue, SDNPVariadic]>;
+
 // Hi and Lo nodes are used to handle global addresses. Used on
 // MipsISelLowering to lower stuff like GlobalAddress, ExternalSymbol
 // static model. (nothing to do with Mips Registers Hi and Lo)
@@ -74,9 +78,10 @@ def MipsRet : SDNode<"MipsISD::Ret", SDTNone, [SDNPHasChain, SDNPOptInGlue]>;
 
 // These are target-independent nodes, but have target-specific formats.
 def callseq_start : SDNode<"ISD::CALLSEQ_START", SDT_MipsCallSeqStart,
-                           [SDNPHasChain, SDNPOutGlue]>;
+                           [SDNPHasChain, SDNPSideEffect, SDNPOutGlue]>;
 def callseq_end   : SDNode<"ISD::CALLSEQ_END", SDT_MipsCallSeqEnd,
-                           [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue]>;
+                           [SDNPHasChain, SDNPSideEffect,
+                            SDNPOptInGlue, SDNPOutGlue]>;
 
 // MAdd*/MSub* nodes
 def MipsMAdd      : SDNode<"MipsISD::MAdd", SDT_MipsMAddMSub,
@@ -110,7 +115,7 @@ def MipsWrapper    : SDNode<"MipsISD::Wrapper", SDTIntBinOp>;
 def MipsDynAlloc  : SDNode<"MipsISD::DynAlloc", SDT_MipsDynAlloc,
                            [SDNPHasChain, SDNPInGlue]>;
 
-def MipsSync : SDNode<"MipsISD::Sync", SDT_Sync, [SDNPHasChain]>;
+def MipsSync : SDNode<"MipsISD::Sync", SDT_Sync, [SDNPHasChain,SDNPSideEffect]>;
 
 def MipsExt :  SDNode<"MipsISD::Ext", SDT_Ext>;
 def MipsIns :  SDNode<"MipsISD::Ins", SDT_Ins>;
@@ -174,6 +179,35 @@ class MipsPat<dag pattern, dag result> : Pat<pattern, result> {
   let Predicates = [HasStandardEncoding];
 }
 
+class IsBranch {
+  bit isBranch = 1;
+}
+
+class IsReturn {
+  bit isReturn = 1;
+}
+
+class IsCall {
+  bit isCall = 1;
+}
+
+class IsTailCall {
+  bit isCall = 1;
+  bit isTerminator = 1;
+  bit isReturn = 1;
+  bit isBarrier = 1;
+  bit hasExtraSrcRegAllocReq = 1;
+  bit isCodeGenOnly = 1;
+}
+
+class IsAsCheapAsAMove {
+  bit isAsCheapAsAMove = 1;
+}
+
+class NeverHasSideEffects {
+  bit neverHasSideEffects = 1;
+}
+
 //===----------------------------------------------------------------------===//
 // Instruction format superclass
 //===----------------------------------------------------------------------===//
@@ -208,17 +242,24 @@ def uimm16      : Operand<i32> {
   let PrintMethod = "printUnsignedImm";
 }
 
+def MipsMemAsmOperand : AsmOperandClass {
+  let Name = "Mem";
+  let ParserMethod = "parseMemOperand";
+}
+
 // Address operand
 def mem : Operand<i32> {
   let PrintMethod = "printMemOperand";
   let MIOperandInfo = (ops CPURegs, simm16);
   let EncoderMethod = "getMemEncoding";
+  let ParserMatchClass = MipsMemAsmOperand;
 }
 
 def mem64 : Operand<i64> {
   let PrintMethod = "printMemOperand";
   let MIOperandInfo = (ops CPU64Regs, simm16_64);
   let EncoderMethod = "getMemEncoding";
+  let ParserMatchClass = MipsMemAsmOperand;
 }
 
 def mem_ea : Operand<i32> {
@@ -285,57 +326,25 @@ def addr :
   ComplexPattern<iPTR, 2, "SelectAddr", [frameindex], [SDNPWantParent]>;
 
 //===----------------------------------------------------------------------===//
-// Pattern fragment for load/store
+// Instructions specific format
 //===----------------------------------------------------------------------===//
-class UnalignedLoad<PatFrag Node> :
-  PatFrag<(ops node:$ptr), (Node node:$ptr), [{
-  LoadSDNode *LD = cast<LoadSDNode>(N);
-  return LD->getMemoryVT().getSizeInBits()/8 > LD->getAlignment();
-}]>;
 
-class AlignedLoad<PatFrag Node> :
-  PatFrag<(ops node:$ptr), (Node node:$ptr), [{
-  LoadSDNode *LD = cast<LoadSDNode>(N);
-  return LD->getMemoryVT().getSizeInBits()/8 <= LD->getAlignment();
-}]>;
-
-class UnalignedStore<PatFrag Node> :
-  PatFrag<(ops node:$val, node:$ptr), (Node node:$val, node:$ptr), [{
-  StoreSDNode *SD = cast<StoreSDNode>(N);
-  return SD->getMemoryVT().getSizeInBits()/8 > SD->getAlignment();
-}]>;
+/// Move Control Registers From/To CPU Registers
+def MFC0_3OP  : MFC3OP<0x10, 0, (outs CPURegs:$rt),
+                       (ins CPURegs:$rd, uimm16:$sel),"mfc0\t$rt, $rd, $sel">;
+def : InstAlias<"mfc0 $rt, $rd", (MFC0_3OP CPURegs:$rt, CPURegs:$rd, 0)>;
 
-class AlignedStore<PatFrag Node> :
-  PatFrag<(ops node:$val, node:$ptr), (Node node:$val, node:$ptr), [{
-  StoreSDNode *SD = cast<StoreSDNode>(N);
-  return SD->getMemoryVT().getSizeInBits()/8 <= SD->getAlignment();
-}]>;
+def MTC0_3OP  : MFC3OP<0x10, 4, (outs CPURegs:$rd, uimm16:$sel),
+                       (ins CPURegs:$rt),"mtc0\t$rt, $rd, $sel">;
+def : InstAlias<"mtc0 $rt, $rd", (MTC0_3OP CPURegs:$rd, 0, CPURegs:$rt)>;
 
-// Load/Store PatFrags.
-def sextloadi16_a   : AlignedLoad<sextloadi16>;
-def zextloadi16_a   : AlignedLoad<zextloadi16>;
-def extloadi16_a    : AlignedLoad<extloadi16>;
-def load_a          : AlignedLoad<load>;
-def sextloadi32_a   : AlignedLoad<sextloadi32>;
-def zextloadi32_a   : AlignedLoad<zextloadi32>;
-def extloadi32_a    : AlignedLoad<extloadi32>;
-def truncstorei16_a : AlignedStore<truncstorei16>;
-def store_a         : AlignedStore<store>;
-def truncstorei32_a : AlignedStore<truncstorei32>;
-def sextloadi16_u   : UnalignedLoad<sextloadi16>;
-def zextloadi16_u   : UnalignedLoad<zextloadi16>;
-def extloadi16_u    : UnalignedLoad<extloadi16>;
-def load_u          : UnalignedLoad<load>;
-def sextloadi32_u   : UnalignedLoad<sextloadi32>;
-def zextloadi32_u   : UnalignedLoad<zextloadi32>;
-def extloadi32_u    : UnalignedLoad<extloadi32>;
-def truncstorei16_u : UnalignedStore<truncstorei16>;
-def store_u         : UnalignedStore<store>;
-def truncstorei32_u : UnalignedStore<truncstorei32>;
+def MFC2_3OP  : MFC3OP<0x12, 0, (outs CPURegs:$rt),
+                       (ins CPURegs:$rd, uimm16:$sel),"mfc2\t$rt, $rd, $sel">;
+def : InstAlias<"mfc2 $rt, $rd", (MFC2_3OP CPURegs:$rt, CPURegs:$rd, 0)>;
 
-//===----------------------------------------------------------------------===//
-// Instructions specific format
-//===----------------------------------------------------------------------===//
+def MTC2_3OP  : MFC3OP<0x12, 4, (outs CPURegs:$rd, uimm16:$sel),
+                       (ins CPURegs:$rt),"mtc2\t$rt, $rd, $sel">;
+def : InstAlias<"mtc2 $rt, $rd", (MTC2_3OP CPURegs:$rd, 0, CPURegs:$rt)>;
 
 // Arithmetic and logical instructions with 3 register operands.
 class ArithLogicR<bits<6> op, bits<6> func, string instr_asm, SDNode OpNode,
@@ -416,7 +425,7 @@ class shift_rotate_reg<bits<6> func, bits<5> isRotate, string instr_asm,
 // Load Upper Imediate
 class LoadUpper<bits<6> op, string instr_asm, RegisterClass RC, Operand Imm>:
   FI<op, (outs RC:$rt), (ins Imm:$imm16),
-     !strconcat(instr_asm, "\t$rt, $imm16"), [], IIAlu> {
+     !strconcat(instr_asm, "\t$rt, $imm16"), [], IIAlu>, IsAsCheapAsAMove {
   let rs = 0;
   let neverHasSideEffects = 1;
   let isReMaterializable = 1;
@@ -597,14 +606,13 @@ class SetCC_I<bits<6> op, string instr_asm, PatFrag cond_op, Operand Od,
      IIAlu>;
 
 // Jump
-class JumpFJ<bits<6> op, string instr_asm>:
-  FJ<op, (outs), (ins jmptarget:$target),
-     !strconcat(instr_asm, "\t$target"), [(br bb:$target)], IIBranch> {
-  let isBranch=1;
+class JumpFJ<bits<6> op, DAGOperand opnd, string instr_asm,
+             SDPatternOperator operator, SDPatternOperator targetoperator>:
+  FJ<op, (outs), (ins opnd:$target), !strconcat(instr_asm, "\t$target"),
+     [(operator targetoperator:$target)], IIBranch> {
   let isTerminator=1;
   let isBarrier=1;
   let hasDelaySlot = 1;
-  let Predicates = [RelocStatic, HasStandardEncoding];
   let DecoderMethod = "DecodeJumpTarget";
   let Defs = [AT];
 }
@@ -625,21 +633,21 @@ class UncondBranch<bits<6> op, string instr_asm>:
 
 // Base class for indirect branch and return instruction classes.
 let isTerminator=1, isBarrier=1, hasDelaySlot = 1 in
-class JumpFR<RegisterClass RC, list<dag> pattern>:
-  FR<0, 0x8, (outs), (ins RC:$rs), "jr\t$rs", pattern, IIBranch> {
+class JumpFR<RegisterClass RC, SDPatternOperator operator = null_frag>:
+  FR<0, 0x8, (outs), (ins RC:$rs), "jr\t$rs", [(operator RC:$rs)], IIBranch> {
   let rt = 0;
   let rd = 0;
   let shamt = 0;
 }
 
 // Indirect branch
-class IndirectBranch<RegisterClass RC>: JumpFR<RC, [(brind RC:$rs)]> {
+class IndirectBranch<RegisterClass RC>: JumpFR<RC, brind> {
   let isBranch = 1;
   let isIndirectBranch = 1;
 }
 
 // Return instruction
-class RetBase<RegisterClass RC>: JumpFR<RC, []> {
+class RetBase<RegisterClass RC>: JumpFR<RC> {
   let isReturn = 1;
   let isCodeGenOnly = 1;
   let hasCtrlDep = 1;
@@ -905,12 +913,28 @@ let usesCustomInserter = 1 in {
 // Instruction definition
 //===----------------------------------------------------------------------===//
 
+class LoadImm32< string instr_asm, Operand Od, RegisterClass RC> :
+  MipsAsmPseudoInst<(outs RC:$rt), (ins Od:$imm32),
+                     !strconcat(instr_asm, "\t$rt, $imm32")> ;
+def LoadImm32Reg : LoadImm32<"li", shamt,CPURegs>;
+
+class LoadAddress<string instr_asm, Operand MemOpnd, RegisterClass RC> :
+  MipsAsmPseudoInst<(outs RC:$rt), (ins MemOpnd:$addr),
+                     !strconcat(instr_asm, "\t$rt, $addr")> ;
+def LoadAddr32Reg : LoadAddress<"la", mem, CPURegs>;
+
+class LoadAddressImm<string instr_asm, Operand Od, RegisterClass RC> :
+  MipsAsmPseudoInst<(outs RC:$rt), (ins Od:$imm32),
+                     !strconcat(instr_asm, "\t$rt, $imm32")> ;
+def LoadAddr32Imm : LoadAddressImm<"la", shamt,CPURegs>;
+
 //===----------------------------------------------------------------------===//
 // MipsI Instructions
 //===----------------------------------------------------------------------===//
 
 /// Arithmetic Instructions (ALU Immediate)
-def ADDiu   : ArithLogicI<0x09, "addiu", add, simm16, immSExt16, CPURegs>;
+def ADDiu   : ArithLogicI<0x09, "addiu", add, simm16, immSExt16, CPURegs>,
+              IsAsCheapAsAMove;
 def ADDi    : ArithOverflowI<0x08, "addi", add, simm16, immSExt16, CPURegs>;
 def SLTi    : SetCC_I<0x0a, "slti", setlt, simm16, immSExt16, CPURegs>;
 def SLTiu   : SetCC_I<0x0b, "sltiu", setult, simm16, immSExt16, CPURegs>;
@@ -949,19 +973,12 @@ let Predicates = [HasMips32r2, HasStandardEncoding] in {
 ///  aligned
 defm LB      : LoadM32<0x20, "lb",  sextloadi8>;
 defm LBu     : LoadM32<0x24, "lbu", zextloadi8>;
-defm LH      : LoadM32<0x21, "lh",  sextloadi16_a>;
-defm LHu     : LoadM32<0x25, "lhu", zextloadi16_a>;
-defm LW      : LoadM32<0x23, "lw",  load_a>;
+defm LH      : LoadM32<0x21, "lh",  sextloadi16>;
+defm LHu     : LoadM32<0x25, "lhu", zextloadi16>;
+defm LW      : LoadM32<0x23, "lw",  load>;
 defm SB      : StoreM32<0x28, "sb", truncstorei8>;
-defm SH      : StoreM32<0x29, "sh", truncstorei16_a>;
-defm SW      : StoreM32<0x2b, "sw", store_a>;
-
-///  unaligned
-defm ULH     : LoadM32<0x21, "ulh",  sextloadi16_u, 1>;
-defm ULHu    : LoadM32<0x25, "ulhu", zextloadi16_u, 1>;
-defm ULW     : LoadM32<0x23, "ulw",  load_u, 1>;
-defm USH     : StoreM32<0x29, "ush", truncstorei16_u, 1>;
-defm USW     : StoreM32<0x2b, "usw", store_u, 1>;
+defm SH      : StoreM32<0x29, "sh", truncstorei16>;
+defm SW      : StoreM32<0x2b, "sw", store>;
 
 /// load/store left/right
 defm LWL : LoadLeftRightM32<0x22, "lwl", MipsLWL>;
@@ -996,7 +1013,8 @@ def SC_P8 : SCBase<0x38, "sc", CPURegs, mem64>,
 }
 
 /// Jump and Branch Instructions
-def J       : JumpFJ<0x02, "j">;
+def J       : JumpFJ<0x02, jmptarget, "j", br, bb>,
+              Requires<[RelocStatic, HasStandardEncoding]>, IsBranch;
 def JR      : IndirectBranch<CPURegs>;
 def B       : UncondBranch<0x04, "b">;
 def BEQ     : CBranch<0x04, "beq", seteq, CPURegs>;
@@ -1014,6 +1032,8 @@ def JAL  : JumpLink<0x03, "jal">;
 def JALR : JumpLinkReg<0x00, 0x09, "jalr", CPURegs>;
 def BGEZAL  : BranchLink<"bgezal", 0x11, CPURegs>;
 def BLTZAL  : BranchLink<"bltzal", 0x10, CPURegs>;
+def TAILCALL : JumpFJ<0x02, calltarget, "j", MipsTailCall, imm>, IsTailCall;
+def TAILCALL_R : JumpFR<CPURegs, MipsTailCall>, IsTailCall;
 
 def RET : RetBase<CPURegs>;
 
@@ -1072,6 +1092,26 @@ def EXT : ExtBase<0, "ext", CPURegs>;
 def INS : InsBase<4, "ins", CPURegs>;
 
 //===----------------------------------------------------------------------===//
+// Instruction aliases
+//===----------------------------------------------------------------------===//
+def : InstAlias<"move $dst,$src", (ADD CPURegs:$dst,CPURegs:$src,ZERO)>;
+def : InstAlias<"bal $offset", (BGEZAL RA,brtarget:$offset)>;
+def : InstAlias<"addu $rs,$rt,$imm",
+                (ADDiu CPURegs:$rs,CPURegs:$rt,simm16:$imm)>;
+def : InstAlias<"add $rs,$rt,$imm",
+                (ADDi CPURegs:$rs,CPURegs:$rt,simm16:$imm)>;
+def : InstAlias<"and $rs,$rt,$imm",
+                (ANDi CPURegs:$rs,CPURegs:$rt,simm16:$imm)>;
+def : InstAlias<"j $rs", (JR CPURegs:$rs)>;
+def : InstAlias<"not $rt,$rs", (NOR CPURegs:$rt,CPURegs:$rs,ZERO)>;
+def : InstAlias<"neg $rt,$rs", (SUB CPURegs:$rt,ZERO,CPURegs:$rs)>;
+def : InstAlias<"negu $rt,$rs", (SUBu CPURegs:$rt,ZERO,CPURegs:$rs)>;
+def : InstAlias<"slt $rs,$rt,$imm",
+                (SLTi CPURegs:$rs,CPURegs:$rt,simm16:$imm)>;
+def : InstAlias<"xor $rs,$rt,$imm",
+                (XORi CPURegs:$rs,CPURegs:$rt,simm16:$imm)>;
+
+//===----------------------------------------------------------------------===//
 //  Arbitrary patterns that map to one or more instructions
 //===----------------------------------------------------------------------===//
 
@@ -1103,6 +1143,11 @@ def : MipsPat<(MipsJmpLink (i32 texternalsym:$dst)),
 //def : MipsPat<(MipsJmpLink CPURegs:$dst),
 //              (JALR CPURegs:$dst)>;
 
+// Tail call
+def : MipsPat<(MipsTailCall (iPTR tglobaladdr:$dst)),
+              (TAILCALL tglobaladdr:$dst)>;
+def : MipsPat<(MipsTailCall (iPTR texternalsym:$dst)),
+              (TAILCALL texternalsym:$dst)>;
 // hi/lo relocs
 def : MipsPat<(MipsHi tglobaladdr:$in), (LUi tglobaladdr:$in)>;
 def : MipsPat<(MipsHi tblockaddress:$in), (LUi tblockaddress:$in)>;
@@ -1153,24 +1198,20 @@ def : MipsPat<(not CPURegs:$in),
 let Predicates = [NotN64, HasStandardEncoding] in {
   def : MipsPat<(i32 (extloadi1  addr:$src)), (LBu addr:$src)>;
   def : MipsPat<(i32 (extloadi8  addr:$src)), (LBu addr:$src)>;
-  def : MipsPat<(i32 (extloadi16_a addr:$src)), (LHu addr:$src)>;
-  def : MipsPat<(i32 (extloadi16_u addr:$src)), (ULHu addr:$src)>;
+  def : MipsPat<(i32 (extloadi16 addr:$src)), (LHu addr:$src)>;
 }
 let Predicates = [IsN64, HasStandardEncoding] in {
   def : MipsPat<(i32 (extloadi1  addr:$src)), (LBu_P8 addr:$src)>;
   def : MipsPat<(i32 (extloadi8  addr:$src)), (LBu_P8 addr:$src)>;
-  def : MipsPat<(i32 (extloadi16_a addr:$src)), (LHu_P8 addr:$src)>;
-  def : MipsPat<(i32 (extloadi16_u addr:$src)), (ULHu_P8 addr:$src)>;
+  def : MipsPat<(i32 (extloadi16 addr:$src)), (LHu_P8 addr:$src)>;
 }
 
 // peepholes
 let Predicates = [NotN64, HasStandardEncoding] in {
-  def : MipsPat<(store_a (i32 0), addr:$dst), (SW ZERO, addr:$dst)>;
-  def : MipsPat<(store_u (i32 0), addr:$dst), (USW ZERO, addr:$dst)>;
+  def : MipsPat<(store (i32 0), addr:$dst), (SW ZERO, addr:$dst)>;
 }
 let Predicates = [IsN64, HasStandardEncoding] in {
-  def : MipsPat<(store_a (i32 0), addr:$dst), (SW_P8 ZERO, addr:$dst)>;
-  def : MipsPat<(store_u (i32 0), addr:$dst), (USW_P8 ZERO, addr:$dst)>;
+  def : MipsPat<(store (i32 0), addr:$dst), (SW_P8 ZERO, addr:$dst)>;
 }
 
 // brcond patterns
@@ -1265,3 +1306,8 @@ include "MipsCondMov.td"
 
 include "Mips16InstrFormats.td"
 include "Mips16InstrInfo.td"
+
+// DSP
+include "MipsDSPInstrFormats.td"
+include "MipsDSPInstrInfo.td"
+
diff --git a/lib/Target/Mips/MipsLongBranch.cpp b/lib/Target/Mips/MipsLongBranch.cpp
index f78203f..5d9f0cf 100644
--- a/lib/Target/Mips/MipsLongBranch.cpp
+++ b/lib/Target/Mips/MipsLongBranch.cpp
@@ -10,6 +10,10 @@
 // This pass expands a branch or jump instruction into a long branch if its
 // offset is too large to fit into its immediate field.
 //
+// FIXME: 
+// 1. Fix pc-region jump instructions which cross 256MB segment boundaries. 
+// 2. If program has inline assembly statements whose size cannot be
+//    determined accurately, load branch target addresses from the GOT. 
 //===----------------------------------------------------------------------===//
 
 #define DEBUG_TYPE "mips-long-branch"
@@ -48,7 +52,7 @@ namespace {
   typedef MachineBasicBlock::reverse_iterator ReverseIter;
 
   struct MBBInfo {
-    uint64_t Size;
+    uint64_t Size, Address;
     bool HasLongBranch;
     MachineInstr *Br;
 
@@ -61,7 +65,10 @@ namespace {
     static char ID;
     MipsLongBranch(TargetMachine &tm)
       : MachineFunctionPass(ID), TM(tm),
-        TII(static_cast<const MipsInstrInfo*>(tm.getInstrInfo())) {}
+        TII(static_cast<const MipsInstrInfo*>(tm.getInstrInfo())),
+        IsPIC(TM.getRelocationModel() == Reloc::PIC_),
+        ABI(TM.getSubtarget<MipsSubtarget>().getTargetABI()),
+        LongBranchSeqSize(!IsPIC ? 2 : (ABI == MipsSubtarget::N64 ? 13 : 9)) {}
 
     virtual const char *getPassName() const {
       return "Mips Long Branch";
@@ -81,6 +88,9 @@ namespace {
     const MipsInstrInfo *TII;
     MachineFunction *MF;
     SmallVector<MBBInfo, 16> MBBInfos;
+    bool IsPIC;
+    unsigned ABI;
+    unsigned LongBranchSeqSize;
   };
 
   char MipsLongBranch::ID = 0;
@@ -230,12 +240,6 @@ void MipsLongBranch::replaceBranch(MachineBasicBlock &MBB, Iter Br,
 
 // Expand branch instructions to long branches.
 void MipsLongBranch::expandToLongBranch(MBBInfo &I) {
-  I.HasLongBranch = true;
-
-  bool IsPIC = TM.getRelocationModel() == Reloc::PIC_;
-  unsigned ABI = TM.getSubtarget<MipsSubtarget>().getTargetABI();
-  bool N64 = ABI == MipsSubtarget::N64;
-
   MachineBasicBlock::iterator Pos;
   MachineBasicBlock *MBB = I.Br->getParent(), *TgtMBB = getTargetMBB(*I.Br);
   DebugLoc DL = I.Br->getDebugLoc();
@@ -248,101 +252,105 @@ void MipsLongBranch::expandToLongBranch(MBBInfo &I) {
   MBB->addSuccessor(LongBrMBB);
 
   if (IsPIC) {
-    // $longbr:
-    //  addiu $sp, $sp, -regsize * 2
-    //  sw $ra, 0($sp)
-    //  bal $baltgt
-    //  sw $a3, regsize($sp)
-    // $baltgt:
-    //  lui $a3, %hi($baltgt)
-    //  lui $at, %hi($tgt)
-    //  addiu $a3, $a3, %lo($baltgt)
-    //  addiu $at, $at, %lo($tgt)
-    //  subu $at, $at, $a3
-    //  addu $at, $ra, $at
-    //
-    //  if n64:
-    //   lui $a3, %highest($baltgt)
-    //   lui $ra, %highest($tgt)
-    //   addiu $a3, $a3, %higher($baltgt)
-    //   addiu $ra, $ra, %higher($tgt)
-    //   dsll $a3, $a3, 32
-    //   dsll $ra, $ra, 32
-    //   subu $at, $at, $a3
-    //   addu $at, $at, $ra
-    //
-    //  lw $ra, 0($sp)
-    //  lw $a3, regsize($sp)
-    //  jr $at
-    //  addiu $sp, $sp, regsize * 2
-    // $fallthrough:
-    //
-    MF->getInfo<MipsFunctionInfo>()->setEmitNOAT();
     MachineBasicBlock *BalTgtMBB = MF->CreateMachineBasicBlock(BB);
     MF->insert(FallThroughMBB, BalTgtMBB);
     LongBrMBB->addSuccessor(BalTgtMBB);
     BalTgtMBB->addSuccessor(TgtMBB);
 
-    int RegSize = N64 ? 8 : 4;
-    unsigned AT = N64 ? Mips::AT_64 : Mips::AT;
-    unsigned A3 = N64 ? Mips::A3_64 : Mips::A3;
-    unsigned SP = N64 ? Mips::SP_64 : Mips::SP;
-    unsigned RA = N64 ? Mips::RA_64 : Mips::RA;
-    unsigned Load = N64 ? Mips::LD_P8 : Mips::LW;
-    unsigned Store = N64 ? Mips::SD_P8 : Mips::SW;
-    unsigned LUi = N64 ? Mips::LUi64 : Mips::LUi;
-    unsigned ADDiu = N64 ? Mips::DADDiu : Mips::ADDiu;
-    unsigned ADDu = N64 ? Mips::DADDu : Mips::ADDu;
-    unsigned SUBu = N64 ? Mips::SUBu : Mips::SUBu;
-    unsigned JR = N64 ? Mips::JR64 : Mips::JR;
-
-    Pos = LongBrMBB->begin();
-
-    BuildMI(*LongBrMBB, Pos, DL, TII->get(ADDiu), SP).addReg(SP)
-      .addImm(-RegSize * 2);
-    BuildMI(*LongBrMBB, Pos, DL, TII->get(Store)).addReg(RA).addReg(SP)
-      .addImm(0);
-    BuildMI(*LongBrMBB, Pos, DL, TII->get(Mips::BAL_BR)).addMBB(BalTgtMBB);
-    BuildMI(*LongBrMBB, Pos, DL, TII->get(Store)).addReg(A3).addReg(SP)
-      .addImm(RegSize)->setIsInsideBundle();
-
-    Pos = BalTgtMBB->begin();
-
-    BuildMI(*BalTgtMBB, Pos, DL, TII->get(LUi), A3)
-      .addMBB(BalTgtMBB, MipsII::MO_ABS_HI);
-    BuildMI(*BalTgtMBB, Pos, DL, TII->get(LUi), AT)
-      .addMBB(TgtMBB, MipsII::MO_ABS_HI);
-    BuildMI(*BalTgtMBB, Pos, DL, TII->get(ADDiu), A3).addReg(A3)
-      .addMBB(BalTgtMBB, MipsII::MO_ABS_LO);
-    BuildMI(*BalTgtMBB, Pos, DL, TII->get(ADDiu), AT).addReg(AT)
-      .addMBB(TgtMBB, MipsII::MO_ABS_LO);
-    BuildMI(*BalTgtMBB, Pos, DL, TII->get(SUBu), AT).addReg(AT).addReg(A3);
-    BuildMI(*BalTgtMBB, Pos, DL, TII->get(ADDu), AT).addReg(RA).addReg(AT);
-
-    if (N64) {
-      BuildMI(*BalTgtMBB, Pos, DL, TII->get(LUi), A3)
-        .addMBB(BalTgtMBB, MipsII::MO_HIGHEST);
-      BuildMI(*BalTgtMBB, Pos, DL, TII->get(LUi), RA)
-        .addMBB(TgtMBB, MipsII::MO_HIGHEST);
-      BuildMI(*BalTgtMBB, Pos, DL, TII->get(ADDiu), A3).addReg(A3)
-        .addMBB(BalTgtMBB, MipsII::MO_HIGHER);
-      BuildMI(*BalTgtMBB, Pos, DL, TII->get(ADDiu), RA).addReg(RA)
-        .addMBB(TgtMBB, MipsII::MO_HIGHER);
-      BuildMI(*BalTgtMBB, Pos, DL, TII->get(Mips::DSLL), A3).addReg(A3)
-        .addImm(32);
-      BuildMI(*BalTgtMBB, Pos, DL, TII->get(Mips::DSLL), RA).addReg(RA)
-        .addImm(32);
-      BuildMI(*BalTgtMBB, Pos, DL, TII->get(SUBu), AT).addReg(AT).addReg(A3);
-      BuildMI(*BalTgtMBB, Pos, DL, TII->get(ADDu), AT).addReg(AT).addReg(RA);
-      I.Size += 4 * 8;
+    int64_t TgtAddress = MBBInfos[TgtMBB->getNumber()].Address;
+    int64_t Offset = TgtAddress - (I.Address + I.Size - 20);
+    int64_t Lo = SignExtend64<16>(Offset & 0xffff);
+    int64_t Hi = SignExtend64<16>(((Offset + 0x8000) >> 16) & 0xffff);
+
+    if (ABI != MipsSubtarget::N64) {
+      // $longbr:
+      //  addiu $sp, $sp, -8
+      //  sw $ra, 0($sp)
+      //  bal $baltgt
+      //  lui $at, %hi($tgt - $baltgt)
+      // $baltgt:
+      //  addiu $at, $at, %lo($tgt - $baltgt)
+      //  addu $at, $ra, $at
+      //  lw $ra, 0($sp)
+      //  jr $at
+      //  addiu $sp, $sp, 8
+      // $fallthrough:
+      //
+
+      Pos = LongBrMBB->begin();
+
+      BuildMI(*LongBrMBB, Pos, DL, TII->get(Mips::ADDiu), Mips::SP)
+        .addReg(Mips::SP).addImm(-8);
+      BuildMI(*LongBrMBB, Pos, DL, TII->get(Mips::SW)).addReg(Mips::RA)
+        .addReg(Mips::SP).addImm(0);
+      BuildMI(*LongBrMBB, Pos, DL, TII->get(Mips::BAL_BR)).addMBB(BalTgtMBB);
+      BuildMI(*LongBrMBB, Pos, DL, TII->get(Mips::LUi), Mips::AT).addImm(Hi)
+        ->setIsInsideBundle();
+
+      Pos = BalTgtMBB->begin();
+
+      BuildMI(*BalTgtMBB, Pos, DL, TII->get(Mips::ADDiu), Mips::AT)
+        .addReg(Mips::AT).addImm(Lo);
+      BuildMI(*BalTgtMBB, Pos, DL, TII->get(Mips::ADDu), Mips::AT)
+        .addReg(Mips::RA).addReg(Mips::AT);
+      BuildMI(*BalTgtMBB, Pos, DL, TII->get(Mips::LW), Mips::RA)
+        .addReg(Mips::SP).addImm(0);
+      BuildMI(*BalTgtMBB, Pos, DL, TII->get(Mips::JR)).addReg(Mips::AT);
+      BuildMI(*BalTgtMBB, Pos, DL, TII->get(Mips::ADDiu), Mips::SP)
+        .addReg(Mips::SP).addImm(8)->setIsInsideBundle();
+    } else {
+      // $longbr:
+      //  daddiu $sp, $sp, -16
+      //  sd $ra, 0($sp)
+      //  lui64 $at, %highest($tgt - $baltgt)
+      //  daddiu $at, $at, %higher($tgt - $baltgt)
+      //  dsll $at, $at, 16
+      //  daddiu $at, $at, %hi($tgt - $baltgt)
+      //  bal $baltgt
+      //  dsll $at, $at, 16
+      // $baltgt:
+      //  daddiu $at, $at, %lo($tgt - $baltgt)
+      //  daddu $at, $ra, $at
+      //  ld $ra, 0($sp)
+      //  jr64 $at
+      //  daddiu $sp, $sp, 16
+      // $fallthrough:
+      //
+
+      int64_t Higher = SignExtend64<16>(((Offset + 0x80008000) >> 32) & 0xffff);
+      int64_t Highest =
+        SignExtend64<16>(((Offset + 0x800080008000LL) >> 48) & 0xffff);
+
+      Pos = LongBrMBB->begin();
+
+      BuildMI(*LongBrMBB, Pos, DL, TII->get(Mips::DADDiu), Mips::SP_64)
+        .addReg(Mips::SP_64).addImm(-16);
+      BuildMI(*LongBrMBB, Pos, DL, TII->get(Mips::SD)).addReg(Mips::RA_64)
+        .addReg(Mips::SP_64).addImm(0);
+      BuildMI(*LongBrMBB, Pos, DL, TII->get(Mips::LUi64), Mips::AT_64)
+        .addImm(Highest);
+      BuildMI(*LongBrMBB, Pos, DL, TII->get(Mips::DADDiu), Mips::AT_64)
+        .addReg(Mips::AT_64).addImm(Higher);
+      BuildMI(*LongBrMBB, Pos, DL, TII->get(Mips::DSLL), Mips::AT_64)
+        .addReg(Mips::AT_64).addImm(16);
+      BuildMI(*LongBrMBB, Pos, DL, TII->get(Mips::DADDiu), Mips::AT_64)
+        .addReg(Mips::AT_64).addImm(Hi);
+      BuildMI(*LongBrMBB, Pos, DL, TII->get(Mips::BAL_BR)).addMBB(BalTgtMBB);
+      BuildMI(*LongBrMBB, Pos, DL, TII->get(Mips::DSLL), Mips::AT_64)
+        .addReg(Mips::AT_64).addImm(16)->setIsInsideBundle();
+
+      Pos = BalTgtMBB->begin();
+
+      BuildMI(*BalTgtMBB, Pos, DL, TII->get(Mips::DADDiu), Mips::AT_64)
+        .addReg(Mips::AT_64).addImm(Lo);
+      BuildMI(*BalTgtMBB, Pos, DL, TII->get(Mips::DADDu), Mips::AT_64)
+        .addReg(Mips::RA_64).addReg(Mips::AT_64);
+      BuildMI(*BalTgtMBB, Pos, DL, TII->get(Mips::LD), Mips::RA_64)
+        .addReg(Mips::SP_64).addImm(0);
+      BuildMI(*BalTgtMBB, Pos, DL, TII->get(Mips::JR64)).addReg(Mips::AT_64);
+      BuildMI(*BalTgtMBB, Pos, DL, TII->get(Mips::DADDiu), Mips::SP_64)
+        .addReg(Mips::SP_64).addImm(16)->setIsInsideBundle();
     }
-
-    BuildMI(*BalTgtMBB, Pos, DL, TII->get(Load), RA).addReg(SP).addImm(0);
-    BuildMI(*BalTgtMBB, Pos, DL, TII->get(Load), A3).addReg(SP).addImm(RegSize);
-    BuildMI(*BalTgtMBB, Pos, DL, TII->get(JR)).addReg(AT);
-    BuildMI(*BalTgtMBB, Pos, DL, TII->get(ADDiu), SP).addReg(SP)
-      .addImm(RegSize * 2)->setIsInsideBundle();
-    I.Size += 4 * 14;
   } else {
     // $longbr:
     //  j $tgt
@@ -353,7 +361,6 @@ void MipsLongBranch::expandToLongBranch(MBBInfo &I) {
     LongBrMBB->addSuccessor(TgtMBB);
     BuildMI(*LongBrMBB, Pos, DL, TII->get(Mips::J)).addMBB(TgtMBB);
     BuildMI(*LongBrMBB, Pos, DL, TII->get(Mips::NOP))->setIsInsideBundle();
-    I.Size += 4 * 2;
   }
 
   if (I.Br->isUnconditionalBranch()) {
@@ -401,19 +408,34 @@ bool MipsLongBranch::runOnMachineFunction(MachineFunction &F) {
       if (!I->Br || I->HasLongBranch)
         continue;
 
-      if (!ForceLongBranch)
-        // Check if offset fits into 16-bit immediate field of branches.
-        if (isInt<16>(computeOffset(I->Br) / 4))
-          continue;
+      // Check if offset fits into 16-bit immediate field of branches.
+      if (!ForceLongBranch && isInt<16>(computeOffset(I->Br) / 4))
+        continue;
 
-      expandToLongBranch(*I);
+      I->HasLongBranch = true;
+      I->Size += LongBranchSeqSize * 4;
       ++LongBranches;
       EverMadeChange = MadeChange = true;
     }
   }
 
-  if (EverMadeChange)
-    MF->RenumberBlocks();
+  if (!EverMadeChange)
+    return true;
+
+  // Compute basic block addresses.
+  if (TM.getRelocationModel() == Reloc::PIC_) {
+    uint64_t Address = 0;
+
+    for (I = MBBInfos.begin(); I != E; Address += I->Size, ++I)
+      I->Address = Address;
+  }
+
+  // Do the expansion.
+  for (I = MBBInfos.begin(); I != E; ++I)
+    if (I->HasLongBranch)
+      expandToLongBranch(*I);
+
+  MF->RenumberBlocks();
 
   return true;
 }
diff --git a/lib/Target/Mips/MipsMCInstLower.cpp b/lib/Target/Mips/MipsMCInstLower.cpp
index d4c5e6d..5fa6339 100644
--- a/lib/Target/Mips/MipsMCInstLower.cpp
+++ b/lib/Target/Mips/MipsMCInstLower.cpp
@@ -11,7 +11,6 @@
 // MCInst records.
 //
 //===----------------------------------------------------------------------===//
-
 #include "MipsMCInstLower.h"
 #include "MipsAsmPrinter.h"
 #include "MipsInstrInfo.h"
@@ -161,31 +160,3 @@ void MipsMCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const {
   }
 }
 
-// If the D<shift> instruction has a shift amount that is greater
-// than 31 (checked in calling routine), lower it to a D<shift>32 instruction
-void MipsMCInstLower::LowerLargeShift(const MachineInstr *MI,
-                                      MCInst& Inst,
-                                      int64_t Shift) {
-  // rt
-  Inst.addOperand(LowerOperand(MI->getOperand(0)));
-  // rd
-  Inst.addOperand(LowerOperand(MI->getOperand(1)));
-  // saminus32
-  Inst.addOperand(MCOperand::CreateImm(Shift));
-
-  switch (MI->getOpcode()) {
-  default:
-    // Calling function is not synchronized
-    llvm_unreachable("Unexpected shift instruction");
-    break;
-  case Mips::DSLL:
-    Inst.setOpcode(Mips::DSLL32);
-    break;
-  case Mips::DSRL:
-    Inst.setOpcode(Mips::DSRL32);
-    break;
-  case Mips::DSRA:
-    Inst.setOpcode(Mips::DSRA32);
-    break;
-  }
-}
diff --git a/lib/Target/Mips/MipsMCInstLower.h b/lib/Target/Mips/MipsMCInstLower.h
index 0abb996..c4a6016 100644
--- a/lib/Target/Mips/MipsMCInstLower.h
+++ b/lib/Target/Mips/MipsMCInstLower.h
@@ -33,12 +33,11 @@ public:
   MipsMCInstLower(MipsAsmPrinter &asmprinter);
   void Initialize(Mangler *mang, MCContext *C);
   void Lower(const MachineInstr *MI, MCInst &OutMI) const;
-  void LowerLargeShift(const MachineInstr *MI, MCInst &Inst, int64_t Shift);
+  MCOperand LowerOperand(const MachineOperand& MO, unsigned offset = 0) const;
 
 private:
   MCOperand LowerSymbolOperand(const MachineOperand &MO,
                                MachineOperandType MOTy, unsigned Offset) const;
-  MCOperand LowerOperand(const MachineOperand& MO, unsigned offset = 0) const;
 };
 }
 
diff --git a/lib/Target/Mips/MipsMachineFunction.cpp b/lib/Target/Mips/MipsMachineFunction.cpp
index 362173e..5ff19ab 100644
--- a/lib/Target/Mips/MipsMachineFunction.cpp
+++ b/lib/Target/Mips/MipsMachineFunction.cpp
@@ -43,4 +43,17 @@ unsigned MipsFunctionInfo::getGlobalBaseReg() {
   return GlobalBaseReg = MF.getRegInfo().createVirtualRegister(RC);
 }
 
+bool MipsFunctionInfo::mips16SPAliasRegSet() const {
+  return Mips16SPAliasReg;
+}
+unsigned MipsFunctionInfo::getMips16SPAliasReg() {
+  // Return if it has already been initialized.
+  if (Mips16SPAliasReg)
+    return Mips16SPAliasReg;
+
+  const TargetRegisterClass *RC;
+  RC=(const TargetRegisterClass*)&Mips::CPU16RegsRegClass;
+  return Mips16SPAliasReg = MF.getRegInfo().createVirtualRegister(RC);
+}
+
 void MipsFunctionInfo::anchor() { }
diff --git a/lib/Target/Mips/MipsMachineFunction.h b/lib/Target/Mips/MipsMachineFunction.h
index df3c4c0..bb45f92 100644
--- a/lib/Target/Mips/MipsMachineFunction.h
+++ b/lib/Target/Mips/MipsMachineFunction.h
@@ -39,55 +39,45 @@ class MipsFunctionInfo : public MachineFunctionInfo {
   /// relocation models.
   unsigned GlobalBaseReg;
 
+  /// Mips16SPAliasReg - keeps track of the virtual register initialized for
+  /// use as an alias for SP for use in load/store of halfword/byte from/to
+  /// the stack
+  unsigned Mips16SPAliasReg;
+
   /// VarArgsFrameIndex - FrameIndex for start of varargs area.
   int VarArgsFrameIndex;
 
-  // Range of frame object indices.
-  // InArgFIRange: Range of indices of all frame objects created during call to
-  //               LowerFormalArguments.
-  // OutArgFIRange: Range of indices of all frame objects created during call to
-  //                LowerCall except for the frame object for restoring $gp.
-  std::pair<int, int> InArgFIRange, OutArgFIRange;
-  unsigned MaxCallFrameSize;
+  /// True if function has a byval argument.
+  bool HasByvalArg;
 
-  bool EmitNOAT;
+  /// Size of incoming argument area.
+  unsigned IncomingArgSize;
 
 public:
   MipsFunctionInfo(MachineFunction& MF)
-  : MF(MF), SRetReturnReg(0), GlobalBaseReg(0),
-    VarArgsFrameIndex(0), InArgFIRange(std::make_pair(-1, 0)),
-    OutArgFIRange(std::make_pair(-1, 0)), MaxCallFrameSize(0), EmitNOAT(false)
+   : MF(MF), SRetReturnReg(0), GlobalBaseReg(0), Mips16SPAliasReg(0),
+     VarArgsFrameIndex(0)
   {}
 
-  bool isInArgFI(int FI) const {
-    return FI <= InArgFIRange.first && FI >= InArgFIRange.second;
-  }
-  void setLastInArgFI(int FI) { InArgFIRange.second = FI; }
-
-  bool isOutArgFI(int FI) const {
-    return FI <= OutArgFIRange.first && FI >= OutArgFIRange.second;
-  }
-  void extendOutArgFIRange(int FirstFI, int LastFI) {
-    if (!OutArgFIRange.second)
-      // this must be the first time this function was called.
-      OutArgFIRange.first = FirstFI;
-    OutArgFIRange.second = LastFI;
-  }
-
   unsigned getSRetReturnReg() const { return SRetReturnReg; }
   void setSRetReturnReg(unsigned Reg) { SRetReturnReg = Reg; }
 
   bool globalBaseRegSet() const;
   unsigned getGlobalBaseReg();
 
+  bool mips16SPAliasRegSet() const;
+  unsigned getMips16SPAliasReg();
+
   int getVarArgsFrameIndex() const { return VarArgsFrameIndex; }
   void setVarArgsFrameIndex(int Index) { VarArgsFrameIndex = Index; }
 
-  unsigned getMaxCallFrameSize() const { return MaxCallFrameSize; }
-  void setMaxCallFrameSize(unsigned S) { MaxCallFrameSize = S; }
+  bool hasByvalArg() const { return HasByvalArg; }
+  void setFormalArgInfo(unsigned Size, bool HasByval) {
+    IncomingArgSize = Size;
+    HasByvalArg = HasByval;
+  }
 
-  bool getEmitNOAT() const { return EmitNOAT; }
-  void setEmitNOAT() { EmitNOAT = true; }
+  unsigned getIncomingArgSize() const { return IncomingArgSize; }
 };
 
 } // end of namespace llvm
diff --git a/lib/Target/Mips/MipsRegisterInfo.cpp b/lib/Target/Mips/MipsRegisterInfo.cpp
index ae6ae3a..d8e0dd4 100644
--- a/lib/Target/Mips/MipsRegisterInfo.cpp
+++ b/lib/Target/Mips/MipsRegisterInfo.cpp
@@ -22,7 +22,6 @@
 #include "llvm/Constants.h"
 #include "llvm/DebugInfo.h"
 #include "llvm/Type.h"
-#include "llvm/Function.h"
 #include "llvm/CodeGen/ValueTypes.h"
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineFunction.h"
@@ -43,9 +42,8 @@
 
 using namespace llvm;
 
-MipsRegisterInfo::MipsRegisterInfo(const MipsSubtarget &ST,
-                                   const TargetInstrInfo &tii)
-  : MipsGenRegisterInfo(Mips::RA), Subtarget(ST), TII(tii) {}
+MipsRegisterInfo::MipsRegisterInfo(const MipsSubtarget &ST)
+  : MipsGenRegisterInfo(Mips::RA), Subtarget(ST) {}
 
 unsigned MipsRegisterInfo::getPICCallReg() { return Mips::T9; }
 
@@ -83,11 +81,11 @@ MipsRegisterInfo::getCallPreservedMask(CallingConv::ID) const {
 BitVector MipsRegisterInfo::
 getReservedRegs(const MachineFunction &MF) const {
   static const uint16_t ReservedCPURegs[] = {
-    Mips::ZERO, Mips::AT, Mips::K0, Mips::K1, Mips::SP
+    Mips::ZERO, Mips::K0, Mips::K1, Mips::SP
   };
 
   static const uint16_t ReservedCPU64Regs[] = {
-    Mips::ZERO_64, Mips::AT_64, Mips::K0_64, Mips::K1_64, Mips::SP_64
+    Mips::ZERO_64, Mips::K0_64, Mips::K1_64, Mips::SP_64
   };
 
   BitVector Reserved(getNumRegs());
@@ -96,41 +94,49 @@ getReservedRegs(const MachineFunction &MF) const {
   for (unsigned I = 0; I < array_lengthof(ReservedCPURegs); ++I)
     Reserved.set(ReservedCPURegs[I]);
 
-  if (Subtarget.hasMips64()) {
-    for (unsigned I = 0; I < array_lengthof(ReservedCPU64Regs); ++I)
-      Reserved.set(ReservedCPU64Regs[I]);
+  for (unsigned I = 0; I < array_lengthof(ReservedCPU64Regs); ++I)
+    Reserved.set(ReservedCPU64Regs[I]);
 
+  if (Subtarget.hasMips64()) {
     // Reserve all registers in AFGR64.
     for (RegIter Reg = Mips::AFGR64RegClass.begin(),
          EReg = Mips::AFGR64RegClass.end(); Reg != EReg; ++Reg)
       Reserved.set(*Reg);
   } else {
-    // Reserve all registers in CPU64Regs & FGR64.
-    for (RegIter Reg = Mips::CPU64RegsRegClass.begin(),
-         EReg = Mips::CPU64RegsRegClass.end(); Reg != EReg; ++Reg)
-      Reserved.set(*Reg);
-
+    // Reserve all registers in FGR64.
     for (RegIter Reg = Mips::FGR64RegClass.begin(),
          EReg = Mips::FGR64RegClass.end(); Reg != EReg; ++Reg)
       Reserved.set(*Reg);
   }
-
   // Reserve FP if this function should have a dedicated frame pointer register.
   if (MF.getTarget().getFrameLowering()->hasFP(MF)) {
-    Reserved.set(Mips::FP);
-    Reserved.set(Mips::FP_64);
+    if (Subtarget.inMips16Mode())
+      Reserved.set(Mips::S0);
+    else {
+      Reserved.set(Mips::FP);
+      Reserved.set(Mips::FP_64);
+    }
   }
 
   // Reserve hardware registers.
   Reserved.set(Mips::HWR29);
   Reserved.set(Mips::HWR29_64);
 
+  // Reserve DSP control register.
+  Reserved.set(Mips::DSPCtrl);
+
   // Reserve RA if in mips16 mode.
   if (Subtarget.inMips16Mode()) {
     Reserved.set(Mips::RA);
     Reserved.set(Mips::RA_64);
   }
 
+  // Reserve GP if small section is used.
+  if (Subtarget.useSmallSection()) {
+    Reserved.set(Mips::GP);
+    Reserved.set(Mips::GP_64);
+  }
+
   return Reserved;
 }
 
@@ -160,7 +166,7 @@ eliminateFrameIndex(MachineBasicBlock::iterator II, int SPAdj,
            "Instr doesn't have FrameIndex operand!");
   }
 
-  DEBUG(errs() << "\nFunction : " << MF.getFunction()->getName() << "\n";
+  DEBUG(errs() << "\nFunction : " << MF.getName() << "\n";
         errs() << "<--------->\n" << MI);
 
   int FrameIndex = MI.getOperand(i).getIndex();
@@ -179,8 +185,12 @@ getFrameRegister(const MachineFunction &MF) const {
   const TargetFrameLowering *TFI = MF.getTarget().getFrameLowering();
   bool IsN64 = Subtarget.isABI_N64();
 
-  return TFI->hasFP(MF) ? (IsN64 ? Mips::FP_64 : Mips::FP) :
-                          (IsN64 ? Mips::SP_64 : Mips::SP);
+  if (Subtarget.inMips16Mode())
+    return TFI->hasFP(MF) ? Mips::S0 : Mips::SP;
+  else
+    return TFI->hasFP(MF) ? (IsN64 ? Mips::FP_64 : Mips::FP) :
+                            (IsN64 ? Mips::SP_64 : Mips::SP);
+
 }
 
 unsigned MipsRegisterInfo::
diff --git a/lib/Target/Mips/MipsRegisterInfo.h b/lib/Target/Mips/MipsRegisterInfo.h
index 9a05e94..78adf7f 100644
--- a/lib/Target/Mips/MipsRegisterInfo.h
+++ b/lib/Target/Mips/MipsRegisterInfo.h
@@ -22,16 +22,14 @@
 
 namespace llvm {
 class MipsSubtarget;
-class TargetInstrInfo;
 class Type;
 
 class MipsRegisterInfo : public MipsGenRegisterInfo {
 protected:
   const MipsSubtarget &Subtarget;
-  const TargetInstrInfo &TII;
 
 public:
-  MipsRegisterInfo(const MipsSubtarget &Subtarget, const TargetInstrInfo &tii);
+  MipsRegisterInfo(const MipsSubtarget &Subtarget);
 
   /// getRegisterNumbering - Given the enum value for some register, e.g.
   /// Mips::RA, return the number that it corresponds to (e.g. 31).
diff --git a/lib/Target/Mips/MipsRegisterInfo.td b/lib/Target/Mips/MipsRegisterInfo.td
index b255e42..391c19e 100644
--- a/lib/Target/Mips/MipsRegisterInfo.td
+++ b/lib/Target/Mips/MipsRegisterInfo.td
@@ -14,6 +14,8 @@ let Namespace = "Mips" in {
 def sub_fpeven : SubRegIndex;
 def sub_fpodd  : SubRegIndex;
 def sub_32     : SubRegIndex;
+def sub_lo     : SubRegIndex;
+def sub_hi     : SubRegIndex;
 }
 
 // We have banks of 32 registers each.
@@ -71,7 +73,7 @@ class HWR<bits<5> num, string n> : MipsReg<n> {
 let Namespace = "Mips" in {
   // General Purpose Registers
   def ZERO : MipsGPRReg< 0, "zero">, DwarfRegNum<[0]>;
-  def AT   : MipsGPRReg< 1, "at">,   DwarfRegNum<[1]>;
+  def AT   : MipsGPRReg< 1, "1">,    DwarfRegNum<[1]>;
   def V0   : MipsGPRReg< 2, "2">,    DwarfRegNum<[2]>;
   def V1   : MipsGPRReg< 3, "3">,    DwarfRegNum<[3]>;
   def A0   : MipsGPRReg< 4, "4">,    DwarfRegNum<[4]>;
@@ -105,7 +107,7 @@ let Namespace = "Mips" in {
 
   // General Purpose 64-bit Registers
   def ZERO_64 : Mips64GPRReg< 0, "zero", [ZERO]>, DwarfRegNum<[0]>;
-  def AT_64   : Mips64GPRReg< 1, "at",   [AT]>, DwarfRegNum<[1]>;
+  def AT_64   : Mips64GPRReg< 1, "1",    [AT]>, DwarfRegNum<[1]>;
   def V0_64   : Mips64GPRReg< 2, "2",    [V0]>, DwarfRegNum<[2]>;
   def V1_64   : Mips64GPRReg< 3, "3",    [V1]>, DwarfRegNum<[3]>;
   def A0_64   : Mips64GPRReg< 4, "4",    [A0]>, DwarfRegNum<[4]>;
@@ -239,16 +241,29 @@ let Namespace = "Mips" in {
   // fcc0 register
   def FCC0 : Register<"fcc0">;
 
+  // PC register
+  def PC : Register<"pc">;
+
   // Hardware register $29
   def HWR29 : Register<"29">;
   def HWR29_64 : Register<"29">;
+
+  // Accum registers
+  let SubRegIndices = [sub_lo, sub_hi] in
+  def AC0 : RegisterWithSubRegs<"ac0", [LO, HI]>;
+  def AC1 : Register<"ac1">;
+  def AC2 : Register<"ac2">;
+  def AC3 : Register<"ac3">;
+
+  def DSPCtrl : Register<"dspctrl">;
 }
 
 //===----------------------------------------------------------------------===//
 // Register Classes
 //===----------------------------------------------------------------------===//
 
-def CPURegs : RegisterClass<"Mips", [i32], 32, (add
+class CPURegsClass<list<ValueType> regTypes> :
+  RegisterClass<"Mips", regTypes, 32, (add
   // Reserved
   ZERO, AT,
   // Return Values and Arguments
@@ -262,6 +277,9 @@ def CPURegs : RegisterClass<"Mips", [i32], 32, (add
   // Reserved
   K0, K1, GP, SP, FP, RA)>;
 
+def CPURegs : CPURegsClass<[i32]>;
+def DSPRegs : CPURegsClass<[v4i8, v2i16]>;
+
 def CPU64Regs : RegisterClass<"Mips", [i64], 64, (add
 // Reserved
   ZERO_64, AT_64,
@@ -284,6 +302,7 @@ def CPU16Regs : RegisterClass<"Mips", [i32], 32, (add
 
 def CPURAReg : RegisterClass<"Mips", [i32], 32, (add RA)>;
 
+def CPUSPReg : RegisterClass<"Mips", [i32], 32, (add SP)>;
 
 // 64bit fp:
 // * FGR64  - 32 64-bit registers
@@ -319,3 +338,5 @@ def HILO64 : RegisterClass<"Mips", [i64], 64, (add HI64, LO64)>;
 def HWRegs : RegisterClass<"Mips", [i32], 32, (add HWR29)>;
 def HWRegs64 : RegisterClass<"Mips", [i64], 32, (add HWR29_64)>;
 
+// Accumulator Registers
+def ACRegs : RegisterClass<"Mips", [i64], 64, (sequence "AC%u", 0, 3)>;
diff --git a/lib/Target/Mips/MipsSEFrameLowering.cpp b/lib/Target/Mips/MipsSEFrameLowering.cpp
index 1c59847..03f5176 100644
--- a/lib/Target/Mips/MipsSEFrameLowering.cpp
+++ b/lib/Target/Mips/MipsSEFrameLowering.cpp
@@ -22,7 +22,8 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/CodeGen/RegisterScavenging.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetOptions.h"
 #include "llvm/Support/CommandLine.h"
 
@@ -202,6 +203,19 @@ processFunctionBeforeCalleeSavedScan(MachineFunction &MF,
   // Mark $fp as used if function has dedicated frame pointer.
   if (hasFP(MF))
     MRI.setPhysRegUsed(FP);
+
+  // Set scavenging frame index if necessary.
+  uint64_t MaxSPOffset = MF.getInfo<MipsFunctionInfo>()->getIncomingArgSize() +
+    estimateStackSize(MF);
+
+  if (isInt<16>(MaxSPOffset))
+    return;
+
+  const TargetRegisterClass *RC = STI.isABI_N64() ?
+    &Mips::CPU64RegsRegClass : &Mips::CPURegsRegClass;
+  int FI = MF.getFrameInfo()->CreateStackObject(RC->getSize(),
+                                                RC->getAlignment(), false);
+  RS->setScavengingFrameIndex(FI);
 }
 
 const MipsFrameLowering *
diff --git a/lib/Target/Mips/MipsSEInstrInfo.cpp b/lib/Target/Mips/MipsSEInstrInfo.cpp
index eeb1de3..fb0f9df 100644
--- a/lib/Target/Mips/MipsSEInstrInfo.cpp
+++ b/lib/Target/Mips/MipsSEInstrInfo.cpp
@@ -260,14 +260,55 @@ void MipsSEInstrInfo::adjustStackPtr(unsigned SP, int64_t Amount,
   if (isInt<16>(Amount))// addi sp, sp, amount
     BuildMI(MBB, I, DL, get(ADDiu), SP).addReg(SP).addImm(Amount);
   else { // Expand immediate that doesn't fit in 16-bit.
-    unsigned ATReg = STI.isABI_N64() ? Mips::AT_64 : Mips::AT;
-
-    MBB.getParent()->getInfo<MipsFunctionInfo>()->setEmitNOAT();
-    Mips::loadImmediate(Amount, STI.isABI_N64(), *this, MBB, I, DL, false, 0);
-    BuildMI(MBB, I, DL, get(ADDu), SP).addReg(SP).addReg(ATReg);
+    unsigned Reg = loadImmediate(Amount, MBB, I, DL, 0);
+    BuildMI(MBB, I, DL, get(ADDu), SP).addReg(SP).addReg(Reg, RegState::Kill);
   }
 }
 
+/// This function generates the sequence of instructions needed to get the
+/// result of adding register REG and immediate IMM.
+unsigned
+MipsSEInstrInfo::loadImmediate(int64_t Imm, MachineBasicBlock &MBB,
+                               MachineBasicBlock::iterator II, DebugLoc DL,
+                               unsigned *NewImm) const {
+  MipsAnalyzeImmediate AnalyzeImm;
+  const MipsSubtarget &STI = TM.getSubtarget<MipsSubtarget>();
+  MachineRegisterInfo &RegInfo = MBB.getParent()->getRegInfo();
+  unsigned Size = STI.isABI_N64() ? 64 : 32;
+  unsigned LUi = STI.isABI_N64() ? Mips::LUi64 : Mips::LUi;
+  unsigned ZEROReg = STI.isABI_N64() ? Mips::ZERO_64 : Mips::ZERO;
+  const TargetRegisterClass *RC = STI.isABI_N64() ?
+    &Mips::CPU64RegsRegClass : &Mips::CPURegsRegClass;
+  bool LastInstrIsADDiu = NewImm;
+
+  const MipsAnalyzeImmediate::InstSeq &Seq =
+    AnalyzeImm.Analyze(Imm, Size, LastInstrIsADDiu);
+  MipsAnalyzeImmediate::InstSeq::const_iterator Inst = Seq.begin();
+
+  assert(Seq.size() && (!LastInstrIsADDiu || (Seq.size() > 1)));
+
+  // The first instruction can be a LUi, which is different from other
+  // instructions (ADDiu, ORI and SLL) in that it does not have a register
+  // operand.
+  unsigned Reg = RegInfo.createVirtualRegister(RC);
+
+  if (Inst->Opc == LUi)
+    BuildMI(MBB, II, DL, get(LUi), Reg).addImm(SignExtend64<16>(Inst->ImmOpnd));
+  else
+    BuildMI(MBB, II, DL, get(Inst->Opc), Reg).addReg(ZEROReg)
+      .addImm(SignExtend64<16>(Inst->ImmOpnd));
+
+  // Build the remaining instructions in Seq.
+  for (++Inst; Inst != Seq.end() - LastInstrIsADDiu; ++Inst)
+    BuildMI(MBB, II, DL, get(Inst->Opc), Reg).addReg(Reg, RegState::Kill)
+      .addImm(SignExtend64<16>(Inst->ImmOpnd));
+
+  if (LastInstrIsADDiu)
+    *NewImm = Inst->ImmOpnd;
+
+  return Reg;
+}
+
 unsigned MipsSEInstrInfo::GetAnalyzableBrOpc(unsigned Opc) const {
   return (Opc == Mips::BEQ    || Opc == Mips::BNE    || Opc == Mips::BGTZ   ||
           Opc == Mips::BGEZ   || Opc == Mips::BLTZ   || Opc == Mips::BLEZ   ||
diff --git a/lib/Target/Mips/MipsSEInstrInfo.h b/lib/Target/Mips/MipsSEInstrInfo.h
index 346e74d..55b78b2 100644
--- a/lib/Target/Mips/MipsSEInstrInfo.h
+++ b/lib/Target/Mips/MipsSEInstrInfo.h
@@ -15,7 +15,6 @@
 #define MIPSSEINSTRUCTIONINFO_H
 
 #include "MipsInstrInfo.h"
-#include "MipsAnalyzeImmediate.h"
 #include "MipsSERegisterInfo.h"
 
 namespace llvm {
@@ -70,6 +69,13 @@ public:
   void adjustStackPtr(unsigned SP, int64_t Amount, MachineBasicBlock &MBB,
                       MachineBasicBlock::iterator I) const;
 
+  /// Emit a series of instructions to load an immediate. If NewImm is a
+  /// non-NULL parameter, the last instruction is not emitted, but instead
+  /// its immediate operand is returned in NewImm.
+  unsigned loadImmediate(int64_t Imm, MachineBasicBlock &MBB,
+                         MachineBasicBlock::iterator II, DebugLoc DL,
+                         unsigned *NewImm) const;
+
 private:
   virtual unsigned GetAnalyzableBrOpc(unsigned Opc) const;
 
diff --git a/lib/Target/Mips/MipsSERegisterInfo.cpp b/lib/Target/Mips/MipsSERegisterInfo.cpp
index 043a1ef..56b9ba9 100644
--- a/lib/Target/Mips/MipsSERegisterInfo.cpp
+++ b/lib/Target/Mips/MipsSERegisterInfo.cpp
@@ -26,6 +26,7 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/Target/TargetFrameLowering.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
@@ -40,8 +41,18 @@
 using namespace llvm;
 
 MipsSERegisterInfo::MipsSERegisterInfo(const MipsSubtarget &ST,
-                                       const TargetInstrInfo &TII)
-  : MipsRegisterInfo(ST, TII) {}
+                                       const MipsSEInstrInfo &I)
+  : MipsRegisterInfo(ST), TII(I) {}
+
+bool MipsSERegisterInfo::
+requiresRegisterScavenging(const MachineFunction &MF) const {
+  return true;
+}
+
+bool MipsSERegisterInfo::
+requiresFrameIndexScavenging(const MachineFunction &MF) const {
+  return true;
+}
 
 // This function eliminate ADJCALLSTACKDOWN,
 // ADJCALLSTACKUP pseudo instructions
@@ -72,7 +83,6 @@ void MipsSERegisterInfo::eliminateFI(MachineBasicBlock::iterator II,
   MachineInstr &MI = *II;
   MachineFunction &MF = *MI.getParent()->getParent();
   MachineFrameInfo *MFI = MF.getFrameInfo();
-  MipsFunctionInfo *MipsFI = MF.getInfo<MipsFunctionInfo>();
 
   const std::vector<CalleeSavedInfo> &CSI = MFI->getCalleeSavedInfo();
   int MinCSFI = 0;
@@ -91,8 +101,7 @@ void MipsSERegisterInfo::eliminateFI(MachineBasicBlock::iterator II,
   // getFrameRegister() returns.
   unsigned FrameReg;
 
-  if (MipsFI->isOutArgFI(FrameIndex) ||
-      (FrameIndex >= MinCSFI && FrameIndex <= MaxCSFI))
+  if (FrameIndex >= MinCSFI && FrameIndex <= MaxCSFI)
     FrameReg = Subtarget.isABI_N64() ? Mips::SP_64 : Mips::SP;
   else
     FrameReg = getFrameRegister(MF);
@@ -104,14 +113,11 @@ void MipsSERegisterInfo::eliminateFI(MachineBasicBlock::iterator II,
   // - If the frame object is any of the following, its offset must be adjusted
   //   by adding the size of the stack:
   //   incoming argument, callee-saved register location or local variable.
+  bool IsKill = false;
   int64_t Offset;
 
-  if (MipsFI->isOutArgFI(FrameIndex))
-    Offset = SPOffset;
-  else
-    Offset = SPOffset + (int64_t)StackSize;
-
-  Offset    += MI.getOperand(OpNo + 1).getImm();
+  Offset = SPOffset + (int64_t)StackSize;
+  Offset += MI.getOperand(OpNo + 1).getImm();
 
   DEBUG(errs() << "Offset     : " << Offset << "\n" << "<--------->\n");
 
@@ -121,18 +127,17 @@ void MipsSERegisterInfo::eliminateFI(MachineBasicBlock::iterator II,
     MachineBasicBlock &MBB = *MI.getParent();
     DebugLoc DL = II->getDebugLoc();
     unsigned ADDu = Subtarget.isABI_N64() ? Mips::DADDu : Mips::ADDu;
-    unsigned ATReg = Subtarget.isABI_N64() ? Mips::AT_64 : Mips::AT;
-    MipsAnalyzeImmediate::Inst LastInst(0, 0);
+    unsigned NewImm;
 
-    MipsFI->setEmitNOAT();
-    Mips::loadImmediate(Offset, Subtarget.isABI_N64(), TII, MBB, II, DL, true,
-                        &LastInst);
-    BuildMI(MBB, II, DL, TII.get(ADDu), ATReg).addReg(FrameReg).addReg(ATReg);
+    unsigned Reg = TII.loadImmediate(Offset, MBB, II, DL, &NewImm);
+    BuildMI(MBB, II, DL, TII.get(ADDu), Reg).addReg(FrameReg)
+      .addReg(Reg, RegState::Kill);
 
-    FrameReg = ATReg;
-    Offset = SignExtend64<16>(LastInst.ImmOpnd);
+    FrameReg = Reg;
+    Offset = SignExtend64<16>(NewImm);
+    IsKill = true;
   }
 
-  MI.getOperand(OpNo).ChangeToRegister(FrameReg, false);
+  MI.getOperand(OpNo).ChangeToRegister(FrameReg, false, false, IsKill);
   MI.getOperand(OpNo + 1).ChangeToImmediate(Offset);
 }
diff --git a/lib/Target/Mips/MipsSERegisterInfo.h b/lib/Target/Mips/MipsSERegisterInfo.h
index 4b17b33..7437bd3 100644
--- a/lib/Target/Mips/MipsSERegisterInfo.h
+++ b/lib/Target/Mips/MipsSERegisterInfo.h
@@ -18,11 +18,18 @@
 #include "MipsRegisterInfo.h"
 
 namespace llvm {
+class MipsSEInstrInfo;
 
 class MipsSERegisterInfo : public MipsRegisterInfo {
+  const MipsSEInstrInfo &TII;
+
 public:
   MipsSERegisterInfo(const MipsSubtarget &Subtarget,
-                     const TargetInstrInfo &TII);
+                     const MipsSEInstrInfo &TII);
+
+  bool requiresRegisterScavenging(const MachineFunction &MF) const;
+
+  bool requiresFrameIndexScavenging(const MachineFunction &MF) const;
 
   void eliminateCallFramePseudoInstr(MachineFunction &MF,
                                      MachineBasicBlock &MBB,
diff --git a/lib/Target/Mips/MipsSubtarget.cpp b/lib/Target/Mips/MipsSubtarget.cpp
index 11ff809..930af4d 100644
--- a/lib/Target/Mips/MipsSubtarget.cpp
+++ b/lib/Target/Mips/MipsSubtarget.cpp
@@ -25,12 +25,14 @@ using namespace llvm;
 void MipsSubtarget::anchor() { }
 
 MipsSubtarget::MipsSubtarget(const std::string &TT, const std::string &CPU,
-                             const std::string &FS, bool little) :
+                             const std::string &FS, bool little,
+                             Reloc::Model RM) :
   MipsGenSubtargetInfo(TT, CPU, FS),
   MipsArchVersion(Mips32), MipsABI(UnknownABI), IsLittle(little),
   IsSingleFloat(false), IsFP64bit(false), IsGP64bit(false), HasVFPU(false),
   IsLinux(true), HasSEInReg(false), HasCondMov(false), HasMulDivAdd(false),
-  HasMinMax(false), HasSwap(false), HasBitCount(false), InMips16Mode(false)
+  HasMinMax(false), HasSwap(false), HasBitCount(false), InMips16Mode(false),
+  HasDSP(false), HasDSPR2(false), IsAndroid(false)
 {
   std::string CPUName = CPU;
   if (CPUName.empty())
@@ -54,6 +56,9 @@ MipsSubtarget::MipsSubtarget(const std::string &TT, const std::string &CPU,
   // Is the target system Linux ?
   if (TT.find("linux") == std::string::npos)
     IsLinux = false;
+
+  // Set UseSmallSection.
+  UseSmallSection = !IsLinux && (RM == Reloc::Static);
 }
 
 bool
diff --git a/lib/Target/Mips/MipsSubtarget.h b/lib/Target/Mips/MipsSubtarget.h
index ba15362..ff69237 100644
--- a/lib/Target/Mips/MipsSubtarget.h
+++ b/lib/Target/Mips/MipsSubtarget.h
@@ -65,6 +65,9 @@ protected:
   // isLinux - Target system is Linux. Is false we consider ELFOS for now.
   bool IsLinux;
 
+  // UseSmallSection - Small section is used.
+  bool UseSmallSection;
+
   /// Features related to the presence of specific instructions.
 
   // HasSEInReg - SEB and SEH (signext in register) instructions.
@@ -89,6 +92,9 @@ protected:
   // InMips16 -- can process Mips16 instructions
   bool InMips16Mode;
 
+  // HasDSP, HasDSPR2 -- supports DSP ASE.
+  bool HasDSP, HasDSPR2;
+
   // IsAndroid -- target is android
   bool IsAndroid;
 
@@ -109,7 +115,7 @@ public:
   /// This constructor initializes the data members to match that
   /// of the specified triple.
   MipsSubtarget(const std::string &TT, const std::string &CPU,
-                const std::string &FS, bool little);
+                const std::string &FS, bool little, Reloc::Model RM);
 
   /// ParseSubtargetFeatures - Parses features string setting specified
   /// subtarget options.  Definition of function is auto generated by tblgen.
@@ -131,8 +137,11 @@ public:
   bool isNotSingleFloat() const { return !IsSingleFloat; }
   bool hasVFPU() const { return HasVFPU; }
   bool inMips16Mode() const { return InMips16Mode; }
+  bool hasDSP() const { return HasDSP; }
+  bool hasDSPR2() const { return HasDSPR2; }
   bool isAndroid() const { return IsAndroid; }
   bool isLinux() const { return IsLinux; }
+  bool useSmallSection() const { return UseSmallSection; }
 
   bool hasStandardEncoding() const { return !inMips16Mode(); }
 
diff --git a/lib/Target/Mips/MipsTargetMachine.cpp b/lib/Target/Mips/MipsTargetMachine.cpp
index 03a024a..983ee21 100644
--- a/lib/Target/Mips/MipsTargetMachine.cpp
+++ b/lib/Target/Mips/MipsTargetMachine.cpp
@@ -42,8 +42,8 @@ MipsTargetMachine(const Target &T, StringRef TT,
                   CodeGenOpt::Level OL,
                   bool isLittle)
   : LLVMTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL),
-    Subtarget(TT, CPU, FS, isLittle),
-    DataLayout(isLittle ?
+    Subtarget(TT, CPU, FS, isLittle, RM),
+    DL(isLittle ?
                (Subtarget.isABI_N64() ?
                 "e-p:64:64:64-i8:8:32-i16:16:32-i64:64:64-f128:128:128-n32" :
                 "e-p:32:32:32-i8:8:32-i16:16:32-i64:64:64-n32") :
@@ -52,7 +52,8 @@ MipsTargetMachine(const Target &T, StringRef TT,
                 "E-p:32:32:32-i8:8:32-i16:16:32-i64:64:64-n32")),
     InstrInfo(MipsInstrInfo::create(*this)),
     FrameLowering(MipsFrameLowering::create(*this, Subtarget)),
-    TLInfo(*this), TSInfo(*this), JITInfo() {
+    TLInfo(*this), TSInfo(*this), JITInfo(),
+    STTI(&TLInfo), VTTI(&TLInfo) {
 }
 
 void MipsebTargetMachine::anchor() { }
diff --git a/lib/Target/Mips/MipsTargetMachine.h b/lib/Target/Mips/MipsTargetMachine.h
index 21b49e6..b54f5ce 100644
--- a/lib/Target/Mips/MipsTargetMachine.h
+++ b/lib/Target/Mips/MipsTargetMachine.h
@@ -21,8 +21,9 @@
 #include "MipsSelectionDAGInfo.h"
 #include "MipsSubtarget.h"
 #include "llvm/Target/TargetMachine.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetFrameLowering.h"
+#include "llvm/Target/TargetTransformImpl.h"
 
 namespace llvm {
 class formatted_raw_ostream;
@@ -30,12 +31,14 @@ class MipsRegisterInfo;
 
 class MipsTargetMachine : public LLVMTargetMachine {
   MipsSubtarget       Subtarget;
-  const TargetData    DataLayout; // Calculates type size & alignment
+  const DataLayout    DL; // Calculates type size & alignment
   const MipsInstrInfo *InstrInfo;
   const MipsFrameLowering *FrameLowering;
   MipsTargetLowering  TLInfo;
   MipsSelectionDAGInfo TSInfo;
   MipsJITInfo JITInfo;
+  ScalarTargetTransformImpl STTI;
+  VectorTargetTransformImpl VTTI;
 
 public:
   MipsTargetMachine(const Target &T, StringRef TT,
@@ -52,8 +55,8 @@ public:
   { return FrameLowering; }
   virtual const MipsSubtarget *getSubtargetImpl() const
   { return &Subtarget; }
-  virtual const TargetData *getTargetData()    const
-  { return &DataLayout;}
+  virtual const DataLayout *getDataLayout()    const
+  { return &DL;}
   virtual MipsJITInfo *getJITInfo()
   { return &JITInfo; }
 
@@ -69,6 +72,13 @@ public:
     return &TSInfo;
   }
 
+  virtual const ScalarTargetTransformInfo *getScalarTargetTransformInfo()const {
+    return &STTI;
+  }
+  virtual const VectorTargetTransformInfo *getVectorTargetTransformInfo()const {
+    return &VTTI;
+  }
+
   // Pass Pipeline Configuration
   virtual TargetPassConfig *createPassConfig(PassManagerBase &PM);
   virtual bool addCodeEmitter(PassManagerBase &PM, JITCodeEmitter &JCE);
diff --git a/lib/Target/Mips/MipsTargetObjectFile.cpp b/lib/Target/Mips/MipsTargetObjectFile.cpp
index 04dc60a..881908b 100644
--- a/lib/Target/Mips/MipsTargetObjectFile.cpp
+++ b/lib/Target/Mips/MipsTargetObjectFile.cpp
@@ -13,7 +13,7 @@
 #include "llvm/GlobalVariable.h"
 #include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCSectionELF.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/ELF.h"
@@ -26,6 +26,7 @@ SSThreshold("mips-ssection-threshold", cl::Hidden,
 
 void MipsTargetObjectFile::Initialize(MCContext &Ctx, const TargetMachine &TM){
   TargetLoweringObjectFileELF::Initialize(Ctx, TM);
+  InitializeELF(TM.Options.UseInitArray);
 
   SmallDataSection =
     getContext().getELFSection(".sdata", ELF::SHT_PROGBITS,
@@ -60,9 +61,10 @@ bool MipsTargetObjectFile::
 IsGlobalInSmallSection(const GlobalValue *GV, const TargetMachine &TM,
                        SectionKind Kind) const {
 
-  // Only use small section for non linux targets.
   const MipsSubtarget &Subtarget = TM.getSubtarget<MipsSubtarget>();
-  if (Subtarget.isLinux())
+
+  // Return if small section is not available.
+  if (!Subtarget.useSmallSection())
     return false;
 
   // Only global variables, not functions.
@@ -80,7 +82,7 @@ IsGlobalInSmallSection(const GlobalValue *GV, const TargetMachine &TM,
     return false;
 
   Type *Ty = GV->getType()->getElementType();
-  return IsInSmallSection(TM.getTargetData()->getTypeAllocSize(Ty));
+  return IsInSmallSection(TM.getDataLayout()->getTypeAllocSize(Ty));
 }
 
 
diff --git a/lib/Target/NVPTX/NVPTX.td b/lib/Target/NVPTX/NVPTX.td
index ae7710e..7aee359 100644
--- a/lib/Target/NVPTX/NVPTX.td
+++ b/lib/Target/NVPTX/NVPTX.td
@@ -24,7 +24,30 @@ include "NVPTXInstrInfo.td"
 // - Need at least one feature to avoid generating zero sized array by
 //   TableGen in NVPTXGenSubtarget.inc.
 //===----------------------------------------------------------------------===//
-def FeatureDummy  : SubtargetFeature<"dummy", "dummy", "true", "">;
+
+// SM Versions
+def SM10 : SubtargetFeature<"sm_10", "SmVersion", "10",
+                            "Target SM 1.0">;
+def SM11 : SubtargetFeature<"sm_11", "SmVersion", "11",
+                            "Target SM 1.1">;
+def SM12 : SubtargetFeature<"sm_12", "SmVersion", "12",
+                            "Target SM 1.2">;
+def SM13 : SubtargetFeature<"sm_13", "SmVersion", "13",
+                            "Target SM 1.3">;
+def SM20 : SubtargetFeature<"sm_20", "SmVersion", "20",
+                            "Target SM 2.0">;
+def SM21 : SubtargetFeature<"sm_21", "SmVersion", "21",
+                            "Target SM 2.1">;
+def SM30 : SubtargetFeature<"sm_30", "SmVersion", "30",
+                            "Target SM 3.0">;
+def SM35 : SubtargetFeature<"sm_35", "SmVersion", "35",
+                            "Target SM 3.5">;
+
+// PTX Versions
+def PTX30 : SubtargetFeature<"ptx30", "PTXVersion", "30",
+                             "Use PTX version 3.0">;
+def PTX31 : SubtargetFeature<"ptx31", "PTXVersion", "31",
+                             "Use PTX version 3.1">;
 
 //===----------------------------------------------------------------------===//
 // NVPTX supported processors.
@@ -33,7 +56,14 @@ def FeatureDummy  : SubtargetFeature<"dummy", "dummy", "true", "">;
 class Proc<string Name, list<SubtargetFeature> Features>
  : Processor<Name, NoItineraries, Features>;
 
-def : Proc<"sm_10", [FeatureDummy]>;
+def : Proc<"sm_10", [SM10]>;
+def : Proc<"sm_11", [SM11]>;
+def : Proc<"sm_12", [SM12]>;
+def : Proc<"sm_13", [SM13]>;
+def : Proc<"sm_20", [SM20]>;
+def : Proc<"sm_21", [SM21]>;
+def : Proc<"sm_30", [SM30]>;
+def : Proc<"sm_35", [SM35]>;
 
 
 def NVPTXInstrInfo : InstrInfo {
diff --git a/lib/Target/NVPTX/NVPTXAllocaHoisting.h b/lib/Target/NVPTX/NVPTXAllocaHoisting.h
index 24b3bd5..c7cabf6 100644
--- a/lib/Target/NVPTX/NVPTXAllocaHoisting.h
+++ b/lib/Target/NVPTX/NVPTXAllocaHoisting.h
@@ -16,7 +16,7 @@
 
 #include "llvm/CodeGen/MachineFunctionAnalysis.h"
 #include "llvm/Pass.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 
 namespace llvm {
 
@@ -31,7 +31,7 @@ public:
   NVPTXAllocaHoisting() : FunctionPass(ID) {}
 
   void getAnalysisUsage(AnalysisUsage &AU) const {
-    AU.addRequired<TargetData>();
+    AU.addRequired<DataLayout>();
     AU.addPreserved<MachineFunctionAnalysis>();
   }
 
diff --git a/lib/Target/NVPTX/NVPTXAsmPrinter.cpp b/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
index f2b9616..0a885ce 100644
--- a/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
+++ b/lib/Target/NVPTX/NVPTXAsmPrinter.cpp
@@ -68,7 +68,54 @@ static cl::opt<bool, true>InterleaveSrc("nvptx-emit-src",
                                         cl::location(llvm::InterleaveSrcInPtx));
 
 
+namespace {
+/// DiscoverDependentGlobals - Return a set of GlobalVariables on which \p V
+/// depends.
+void DiscoverDependentGlobals(Value *V,
+                              DenseSet<GlobalVariable*> &Globals) {
+  if (GlobalVariable *GV = dyn_cast<GlobalVariable>(V))
+    Globals.insert(GV);
+  else {
+    if (User *U = dyn_cast<User>(V)) {
+      for (unsigned i = 0, e = U->getNumOperands(); i != e; ++i) {
+        DiscoverDependentGlobals(U->getOperand(i), Globals);
+      }
+    }
+  }
+}
 
+/// VisitGlobalVariableForEmission - Add \p GV to the list of GlobalVariable
+/// instances to be emitted, but only after any dependents have been added
+/// first.
+void VisitGlobalVariableForEmission(GlobalVariable *GV,
+                                    SmallVectorImpl<GlobalVariable*> &Order,
+                                    DenseSet<GlobalVariable*> &Visited,
+                                    DenseSet<GlobalVariable*> &Visiting) {
+  // Have we already visited this one?
+  if (Visited.count(GV)) return;
+
+  // Do we have a circular dependency?
+  if (Visiting.count(GV))
+    report_fatal_error("Circular dependency found in global variable set");
+
+  // Start visiting this global
+  Visiting.insert(GV);
+
+  // Make sure we visit all dependents first
+  DenseSet<GlobalVariable*> Others;
+  for (unsigned i = 0, e = GV->getNumOperands(); i != e; ++i)
+    DiscoverDependentGlobals(GV->getOperand(i), Others);
+  
+  for (DenseSet<GlobalVariable*>::iterator I = Others.begin(),
+       E = Others.end(); I != E; ++I)
+    VisitGlobalVariableForEmission(*I, Order, Visited, Visiting);
+
+  // Now we can visit ourself
+  Order.push_back(GV);
+  Visited.insert(GV);
+  Visiting.erase(GV);
+}
+}
 
 // @TODO: This is a copy from AsmPrinter.cpp.  The function is static, so we
 // cannot just link to the existing version.
@@ -98,10 +145,10 @@ const MCExpr *nvptx::LowerConstant(const Constant *CV, AsmPrinter &AP) {
   switch (CE->getOpcode()) {
   default:
     // If the code isn't optimized, there may be outstanding folding
-    // opportunities. Attempt to fold the expression using TargetData as a
+    // opportunities. Attempt to fold the expression using DataLayout as a
     // last resort before giving up.
     if (Constant *C =
-        ConstantFoldConstantExpression(CE, AP.TM.getTargetData()))
+        ConstantFoldConstantExpression(CE, AP.TM.getDataLayout()))
       if (C != CE)
         return LowerConstant(C, AP);
 
@@ -115,7 +162,7 @@ const MCExpr *nvptx::LowerConstant(const Constant *CV, AsmPrinter &AP) {
         report_fatal_error(OS.str());
     }
   case Instruction::GetElementPtr: {
-    const TargetData &TD = *AP.TM.getTargetData();
+    const DataLayout &TD = *AP.TM.getDataLayout();
     // Generate a symbolic expression for the byte address
     const Constant *PtrVal = CE->getOperand(0);
     SmallVector<Value*, 8> IdxVec(CE->op_begin()+1, CE->op_end());
@@ -145,7 +192,7 @@ const MCExpr *nvptx::LowerConstant(const Constant *CV, AsmPrinter &AP) {
     return LowerConstant(CE->getOperand(0), AP);
 
   case Instruction::IntToPtr: {
-    const TargetData &TD = *AP.TM.getTargetData();
+    const DataLayout &TD = *AP.TM.getDataLayout();
     // Handle casts to pointers by changing them into casts to the appropriate
     // integer type.  This promotes constant folding and simplifies this code.
     Constant *Op = CE->getOperand(0);
@@ -155,7 +202,7 @@ const MCExpr *nvptx::LowerConstant(const Constant *CV, AsmPrinter &AP) {
   }
 
   case Instruction::PtrToInt: {
-    const TargetData &TD = *AP.TM.getTargetData();
+    const DataLayout &TD = *AP.TM.getDataLayout();
     // Support only foldable casts to/from pointers that can be eliminated by
     // changing the pointer to the appropriately sized integer type.
     Constant *Op = CE->getOperand(0);
@@ -270,7 +317,7 @@ void NVPTXAsmPrinter::EmitInstruction(const MachineInstr *MI) {
 void NVPTXAsmPrinter::printReturnValStr(const Function *F,
                                         raw_ostream &O)
 {
-  const TargetData *TD = TM.getTargetData();
+  const DataLayout *TD = TM.getDataLayout();
   const TargetLowering *TLI = TM.getTargetLowering();
 
   Type *Ty = F->getReturnType();
@@ -874,7 +921,7 @@ bool NVPTXAsmPrinter::doInitialization (Module &M) {
   const_cast<TargetLoweringObjectFile&>(getObjFileLowering())
           .Initialize(OutContext, TM);
 
-  Mang = new Mangler(OutContext, *TM.getTargetData());
+  Mang = new Mangler(OutContext, *TM.getDataLayout());
 
   // Emit header before any dwarf directives are emitted below.
   emitHeader(M, OS1);
@@ -893,10 +940,27 @@ bool NVPTXAsmPrinter::doInitialization (Module &M) {
 
   emitDeclarations(M, OS2);
 
-  // Print out module-level global variables here.
+  // As ptxas does not support forward references of globals, we need to first
+  // sort the list of module-level globals in def-use order. We visit each
+  // global variable in order, and ensure that we emit it *after* its dependent
+  // globals. We use a little extra memory maintaining both a set and a list to
+  // have fast searches while maintaining a strict ordering.
+  SmallVector<GlobalVariable*,8> Globals;
+  DenseSet<GlobalVariable*> GVVisited;
+  DenseSet<GlobalVariable*> GVVisiting;
+
+  // Visit each global variable, in order
   for (Module::global_iterator I = M.global_begin(), E = M.global_end();
-      I != E; ++I)
-    printModuleLevelGV(I, OS2);
+       I != E; ++I)
+    VisitGlobalVariableForEmission(I, Globals, GVVisited, GVVisiting);
+
+  assert(GVVisited.size() == M.getGlobalList().size() && 
+         "Missed a global variable");
+  assert(GVVisiting.size() == 0 && "Did not fully process a global variable");
+
+  // Print out module-level global variables in proper order
+  for (unsigned i = 0, e = Globals.size(); i != e; ++i)
+    printModuleLevelGV(Globals[i], OS2);
 
   OS2 << '\n';
 
@@ -910,7 +974,8 @@ void NVPTXAsmPrinter::emitHeader (Module &M, raw_ostream &O) {
   O << "//\n";
   O << "\n";
 
-  O << ".version 3.0\n";
+  unsigned PTXVersion = nvptxSubtarget.getPTXVersion();
+  O << ".version " << (PTXVersion / 10) << "." << (PTXVersion % 10) << "\n";
 
   O << ".target ";
   O << nvptxSubtarget.getTargetName();
@@ -1023,7 +1088,7 @@ void NVPTXAsmPrinter::printModuleLevelGV(GlobalVariable* GVar, raw_ostream &O,
       return;
   }
 
-  const TargetData *TD = TM.getTargetData();
+  const DataLayout *TD = TM.getDataLayout();
 
   // GlobalVariables are always constant pointers themselves.
   const PointerType *PTy = GVar->getType();
@@ -1296,7 +1361,7 @@ std::string NVPTXAsmPrinter::getPTXFundamentalTypeStr(const Type *Ty,
 void NVPTXAsmPrinter::emitPTXGlobalVariable(const GlobalVariable* GVar,
                                             raw_ostream &O) {
 
-  const TargetData *TD = TM.getTargetData();
+  const DataLayout *TD = TM.getDataLayout();
 
   // GlobalVariables are always constant pointers themselves.
   const PointerType *PTy = GVar->getType();
@@ -1342,7 +1407,7 @@ void NVPTXAsmPrinter::emitPTXGlobalVariable(const GlobalVariable* GVar,
 
 
 static unsigned int
-getOpenCLAlignment(const TargetData *TD,
+getOpenCLAlignment(const DataLayout *TD,
                    Type *Ty) {
   if (Ty->isPrimitiveType() || Ty->isIntegerTy() || isa<PointerType>(Ty))
     return TD->getPrefTypeAlignment(Ty);
@@ -1421,7 +1486,7 @@ void NVPTXAsmPrinter::printParamName(int paramIndex, raw_ostream &O) {
 
 void NVPTXAsmPrinter::emitFunctionParamList(const Function *F,
                                             raw_ostream &O) {
-  const TargetData *TD = TM.getTargetData();
+  const DataLayout *TD = TM.getDataLayout();
   const AttrListPtr &PAL = F->getAttributes();
   const TargetLowering *TLI = TM.getTargetLowering();
   Function::const_arg_iterator I, E;
@@ -1456,7 +1521,8 @@ void NVPTXAsmPrinter::emitFunctionParamList(const Function *F,
       continue;
     }
 
-    if (PAL.paramHasAttr(paramIndex+1, Attribute::ByVal) == false) {
+    if (PAL.getParamAttributes(paramIndex+1).
+          hasAttribute(Attributes::ByVal) == false) {
       // Just a scalar
       const PointerType *PTy = dyn_cast<PointerType>(Ty);
       if (isKernelFunc) {
@@ -1524,6 +1590,9 @@ void NVPTXAsmPrinter::emitFunctionParamList(const Function *F,
       // <a> = PAL.getparamalignment
       // size = typeallocsize of element type
       unsigned align = PAL.getParamAlignment(paramIndex+1);
+      if (align == 0)
+        align = TD->getABITypeAlignment(ETy);
+
       unsigned sz = TD->getTypeAllocSize(ETy);
       O << "\t.param .align " << align
           << " .b8 ";
@@ -1714,7 +1783,7 @@ void NVPTXAsmPrinter::printScalarConstant(Constant *CPV, raw_ostream &O) {
 void NVPTXAsmPrinter::bufferLEByte(Constant *CPV, int Bytes,
                                    AggBuffer *aggBuffer) {
 
-  const TargetData *TD = TM.getTargetData();
+  const DataLayout *TD = TM.getDataLayout();
 
   if (isa<UndefValue>(CPV) || CPV->isNullValue()) {
     int s = TD->getTypeAllocSize(CPV->getType());
@@ -1843,7 +1912,7 @@ void NVPTXAsmPrinter::bufferLEByte(Constant *CPV, int Bytes,
 
 void NVPTXAsmPrinter::bufferAggregateConstant(Constant *CPV,
                                               AggBuffer *aggBuffer) {
-  const TargetData *TD = TM.getTargetData();
+  const DataLayout *TD = TM.getDataLayout();
   int Bytes;
 
   // Old constants
diff --git a/lib/Target/NVPTX/NVPTXISelLowering.cpp b/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 6ea10ea..f1a99d7 100644
--- a/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -174,10 +174,11 @@ NVPTXTargetLowering::NVPTXTargetLowering(NVPTXTargetMachine &TM)
   setTruncStoreAction(MVT::f64, MVT::f32, Expand);
 
   // PTX does not support load / store predicate registers
-  setOperationAction(ISD::LOAD, MVT::i1, Expand);
+  setOperationAction(ISD::LOAD, MVT::i1, Custom);
+  setOperationAction(ISD::STORE, MVT::i1, Custom);
+
   setLoadExtAction(ISD::SEXTLOAD, MVT::i1, Promote);
   setLoadExtAction(ISD::ZEXTLOAD, MVT::i1, Promote);
-  setOperationAction(ISD::STORE, MVT::i1, Expand);
   setTruncStoreAction(MVT::i64, MVT::i1, Expand);
   setTruncStoreAction(MVT::i32, MVT::i1, Expand);
   setTruncStoreAction(MVT::i16, MVT::i1, Expand);
@@ -402,7 +403,7 @@ std::string NVPTXTargetLowering::getPrototype(Type *retTy,
 
     if (isABI) {
       unsigned align = Outs[i].Flags.getByValAlign();
-      unsigned sz = getTargetData()->getTypeAllocSize(ETy);
+      unsigned sz = getDataLayout()->getTypeAllocSize(ETy);
       O << ".param .align " << align
           << " .b8 ";
       O << "_";
@@ -655,11 +656,11 @@ NVPTXTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
       else {
         if (Func) { // direct call
           if (!llvm::getAlign(*(CS->getCalledFunction()), 0, retAlignment))
-            retAlignment = getTargetData()->getABITypeAlignment(retTy);
+            retAlignment = getDataLayout()->getABITypeAlignment(retTy);
         } else { // indirect call
           const CallInst *CallI = dyn_cast<CallInst>(CS->getInstruction());
           if (!llvm::getAlign(*CallI, 0, retAlignment))
-            retAlignment = getTargetData()->getABITypeAlignment(retTy);
+            retAlignment = getDataLayout()->getABITypeAlignment(retTy);
         }
         SDVTList DeclareRetVTs = DAG.getVTList(MVT::Other, MVT::Glue);
         SDValue DeclareRetOps[] = { Chain, DAG.getConstant(retAlignment,
@@ -856,11 +857,64 @@ LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::EXTRACT_SUBVECTOR:
     return Op;
   case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, DAG);
+  case ISD::STORE: return LowerSTORE(Op, DAG);
+  case ISD::LOAD: return LowerLOAD(Op, DAG);
   default:
     llvm_unreachable("Custom lowering not defined for operation");
   }
 }
 
+
+// v = ld i1* addr
+//   =>
+// v1 = ld i8* addr
+// v = trunc v1 to i1
+SDValue NVPTXTargetLowering::
+LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
+  SDNode *Node = Op.getNode();
+  LoadSDNode *LD = cast<LoadSDNode>(Node);
+  DebugLoc dl = Node->getDebugLoc();
+  assert(LD->getExtensionType() == ISD::NON_EXTLOAD) ;
+  assert(Node->getValueType(0) == MVT::i1 &&
+         "Custom lowering for i1 load only");
+  SDValue newLD = DAG.getLoad(MVT::i8, dl, LD->getChain(), LD->getBasePtr(),
+                              LD->getPointerInfo(),
+                              LD->isVolatile(), LD->isNonTemporal(),
+                              LD->isInvariant(),
+                              LD->getAlignment());
+  SDValue result = DAG.getNode(ISD::TRUNCATE, dl, MVT::i1, newLD);
+  // The legalizer (the caller) is expecting two values from the legalized
+  // load, so we build a MergeValues node for it. See ExpandUnalignedLoad()
+  // in LegalizeDAG.cpp which also uses MergeValues.
+  SDValue Ops[] = {result, LD->getChain()};
+  return DAG.getMergeValues(Ops, 2, dl);
+}
+
+// st i1 v, addr
+//    =>
+// v1 = zxt v to i8
+// st i8, addr
+SDValue NVPTXTargetLowering::
+LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
+  SDNode *Node = Op.getNode();
+  DebugLoc dl = Node->getDebugLoc();
+  StoreSDNode *ST = cast<StoreSDNode>(Node);
+  SDValue Tmp1 = ST->getChain();
+  SDValue Tmp2 = ST->getBasePtr();
+  SDValue Tmp3 = ST->getValue();
+  assert(Tmp3.getValueType() == MVT::i1 && "Custom lowering for i1 store only");
+  unsigned Alignment = ST->getAlignment();
+  bool isVolatile = ST->isVolatile();
+  bool isNonTemporal = ST->isNonTemporal();
+  Tmp3 = DAG.getNode(ISD::ZERO_EXTEND, dl,
+                     MVT::i8, Tmp3);
+  SDValue Result = DAG.getStore(Tmp1, dl, Tmp3, Tmp2,
+                                ST->getPointerInfo(), isVolatile,
+                                isNonTemporal, Alignment);
+  return Result;
+}
+
+
 SDValue
 NVPTXTargetLowering::getExtSymb(SelectionDAG &DAG, const char *inname, int idx,
                                 EVT v) const {
@@ -916,7 +970,7 @@ NVPTXTargetLowering::LowerFormalArguments(SDValue Chain,
                                           DebugLoc dl, SelectionDAG &DAG,
                                        SmallVectorImpl<SDValue> &InVals) const {
   MachineFunction &MF = DAG.getMachineFunction();
-  const TargetData *TD = getTargetData();
+  const DataLayout *TD = getDataLayout();
 
   const Function *F = MF.getFunction();
   const AttrListPtr &PAL = F->getAttributes();
@@ -965,7 +1019,7 @@ NVPTXTargetLowering::LowerFormalArguments(SDValue Chain,
     // to newly created nodes. The SDNOdes for params have to
     // appear in the same order as their order of appearance
     // in the original function. "idx+1" holds that order.
-    if (PAL.paramHasAttr(i+1, Attribute::ByVal) == false) {
+    if (PAL.getParamAttributes(i+1).hasAttribute(Attributes::ByVal) == false) {
       // A plain scalar.
       if (isABI || isKernel) {
         // If ABI, load from the param symbol
diff --git a/lib/Target/NVPTX/NVPTXISelLowering.h b/lib/Target/NVPTX/NVPTXISelLowering.h
index 86246e6..94a177c 100644
--- a/lib/Target/NVPTX/NVPTXISelLowering.h
+++ b/lib/Target/NVPTX/NVPTXISelLowering.h
@@ -138,6 +138,9 @@ private:
   SDValue getParamHelpSymbol(SelectionDAG &DAG, int idx);
 
   SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const;
+
+  SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG) const;
+  SDValue LowerLOAD(SDValue Op, SelectionDAG &DAG) const;
 };
 } // namespace llvm
 
diff --git a/lib/Target/NVPTX/NVPTXLowerAggrCopies.cpp b/lib/Target/NVPTX/NVPTXLowerAggrCopies.cpp
index 56b2372..9273931 100644
--- a/lib/Target/NVPTX/NVPTXLowerAggrCopies.cpp
+++ b/lib/Target/NVPTX/NVPTXLowerAggrCopies.cpp
@@ -21,7 +21,7 @@
 #include "llvm/LLVMContext.h"
 #include "llvm/Module.h"
 #include "llvm/Support/InstIterator.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 
 using namespace llvm;
 
@@ -110,7 +110,7 @@ bool NVPTXLowerAggrCopies::runOnFunction(Function &F) {
   SmallVector<MemTransferInst *, 4> aggrMemcpys;
   SmallVector<MemSetInst *, 4> aggrMemsets;
 
-  TargetData *TD = &getAnalysis<TargetData>();
+  DataLayout *TD = &getAnalysis<DataLayout>();
   LLVMContext &Context = F.getParent()->getContext();
 
   //
diff --git a/lib/Target/NVPTX/NVPTXLowerAggrCopies.h b/lib/Target/NVPTX/NVPTXLowerAggrCopies.h
index ac7f150..b150c69 100644
--- a/lib/Target/NVPTX/NVPTXLowerAggrCopies.h
+++ b/lib/Target/NVPTX/NVPTXLowerAggrCopies.h
@@ -17,7 +17,7 @@
 
 #include "llvm/Pass.h"
 #include "llvm/CodeGen/MachineFunctionAnalysis.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 
 namespace llvm {
 
@@ -28,7 +28,7 @@ struct NVPTXLowerAggrCopies : public FunctionPass {
   NVPTXLowerAggrCopies() : FunctionPass(ID) {}
 
   void getAnalysisUsage(AnalysisUsage &AU) const {
-    AU.addRequired<TargetData>();
+    AU.addRequired<DataLayout>();
     AU.addPreserved<MachineFunctionAnalysis>();
   }
 
diff --git a/lib/Target/NVPTX/NVPTXSubtarget.cpp b/lib/Target/NVPTX/NVPTXSubtarget.cpp
index 6aadd43..7b62cce 100644
--- a/lib/Target/NVPTX/NVPTXSubtarget.cpp
+++ b/lib/Target/NVPTX/NVPTXSubtarget.cpp
@@ -34,16 +34,18 @@ DriverInterface(cl::desc("Choose driver interface:"),
 
 NVPTXSubtarget::NVPTXSubtarget(const std::string &TT, const std::string &CPU,
                                const std::string &FS, bool is64Bit)
-:NVPTXGenSubtargetInfo(TT, "", FS), // Don't pass CPU to subtarget,
- // because we don't register all
- // nvptx targets.
- Is64Bit(is64Bit) {
+: NVPTXGenSubtargetInfo(TT, CPU, FS),
+  Is64Bit(is64Bit),
+  PTXVersion(0),
+  SmVersion(10) {
 
   drvInterface = DriverInterface;
 
   // Provide the default CPU if none
   std::string defCPU = "sm_10";
 
+  ParseSubtargetFeatures((CPU.empty() ? defCPU : CPU), FS);
+
   // Get the TargetName from the FS if available
   if (FS.empty() && CPU.empty())
     TargetName = defCPU;
@@ -52,6 +54,12 @@ NVPTXSubtarget::NVPTXSubtarget(const std::string &TT, const std::string &CPU,
   else
     llvm_unreachable("we are not using FeatureStr");
 
-  // Set up the SmVersion
-  SmVersion = atoi(TargetName.c_str()+3);
+  // We default to PTX 3.1, but we cannot just default to it in the initializer
+  // since the attribute parser checks if the given option is >= the default.
+  // So if we set ptx31 as the default, the ptx30 attribute would never match.
+  // Instead, we use 0 as the default and manually set 31 if the default is
+  // used.
+  if (PTXVersion == 0) {
+    PTXVersion = 31;
+  }
 }
diff --git a/lib/Target/NVPTX/NVPTXSubtarget.h b/lib/Target/NVPTX/NVPTXSubtarget.h
index 8f2a629..3cfd971 100644
--- a/lib/Target/NVPTX/NVPTXSubtarget.h
+++ b/lib/Target/NVPTX/NVPTXSubtarget.h
@@ -25,13 +25,17 @@
 namespace llvm {
 
 class NVPTXSubtarget : public NVPTXGenSubtargetInfo {
-
-  unsigned int SmVersion;
+  
   std::string TargetName;
   NVPTX::DrvInterface drvInterface;
-  bool dummy; // For the 'dummy' feature, see NVPTX.td
   bool Is64Bit;
 
+  // PTX version x.y is represented as 10*x+y, e.g. 3.1 == 31
+  unsigned PTXVersion;
+
+  // SM version x.y is represented as 10*x+y, e.g. 3.1 == 31
+  unsigned int SmVersion;
+
 public:
   /// This constructor initializes the data members to match that
   /// of the specified module.
@@ -69,6 +73,8 @@ public:
   NVPTX::DrvInterface getDrvInterface() const { return drvInterface; }
   std::string getTargetName() const { return TargetName; }
 
+  unsigned getPTXVersion() const { return PTXVersion; }
+
   void ParseSubtargetFeatures(StringRef CPU, StringRef FS);
 
   std::string getDataLayout() const {
diff --git a/lib/Target/NVPTX/NVPTXTargetMachine.cpp b/lib/Target/NVPTX/NVPTXTargetMachine.cpp
index 433f415..cbb4900 100644
--- a/lib/Target/NVPTX/NVPTXTargetMachine.cpp
+++ b/lib/Target/NVPTX/NVPTXTargetMachine.cpp
@@ -32,7 +32,7 @@
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetInstrInfo.h"
 #include "llvm/Target/TargetLowering.h"
 #include "llvm/Target/TargetLoweringObjectFile.h"
@@ -71,8 +71,9 @@ NVPTXTargetMachine::NVPTXTargetMachine(const Target &T,
                                        bool is64bit)
 : LLVMTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL),
   Subtarget(TT, CPU, FS, is64bit),
-  DataLayout(Subtarget.getDataLayout()),
-  InstrInfo(*this), TLInfo(*this), TSInfo(*this), FrameLowering(*this,is64bit)
+  DL(Subtarget.getDataLayout()),
+  InstrInfo(*this), TLInfo(*this), TSInfo(*this), FrameLowering(*this,is64bit),
+  STTI(&TLInfo), VTTI(&TLInfo)
 /*FrameInfo(TargetFrameInfo::StackGrowsUp, 8, 0)*/ {
 }
 
diff --git a/lib/Target/NVPTX/NVPTXTargetMachine.h b/lib/Target/NVPTX/NVPTXTargetMachine.h
index b3f9cac..11bc9d4 100644
--- a/lib/Target/NVPTX/NVPTXTargetMachine.h
+++ b/lib/Target/NVPTX/NVPTXTargetMachine.h
@@ -21,10 +21,11 @@
 #include "NVPTXSubtarget.h"
 #include "NVPTXFrameLowering.h"
 #include "ManagedStringPool.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetFrameLowering.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetSelectionDAGInfo.h"
+#include "llvm/Target/TargetTransformImpl.h"
 
 namespace llvm {
 
@@ -32,7 +33,7 @@ namespace llvm {
 ///
 class NVPTXTargetMachine : public LLVMTargetMachine {
   NVPTXSubtarget        Subtarget;
-  const TargetData      DataLayout;       // Calculates type size & alignment
+  const DataLayout      DL;       // Calculates type size & alignment
   NVPTXInstrInfo        InstrInfo;
   NVPTXTargetLowering   TLInfo;
   TargetSelectionDAGInfo   TSInfo;
@@ -44,6 +45,9 @@ class NVPTXTargetMachine : public LLVMTargetMachine {
   // Hold Strings that can be free'd all together with NVPTXTargetMachine
   ManagedStringPool     ManagedStrPool;
 
+  ScalarTargetTransformImpl STTI;
+  VectorTargetTransformImpl VTTI;
+
   //bool addCommonCodeGenPasses(PassManagerBase &, CodeGenOpt::Level,
   //                            bool DisableVerify, MCContext *&OutCtx);
 
@@ -58,7 +62,7 @@ public:
     return &FrameLowering;
   }
   virtual const NVPTXInstrInfo *getInstrInfo() const  { return &InstrInfo; }
-  virtual const TargetData *getTargetData() const     { return &DataLayout;}
+  virtual const DataLayout *getDataLayout() const     { return &DL;}
   virtual const NVPTXSubtarget *getSubtargetImpl() const { return &Subtarget;}
 
   virtual const NVPTXRegisterInfo *getRegisterInfo() const {
@@ -72,6 +76,12 @@ public:
   virtual const TargetSelectionDAGInfo *getSelectionDAGInfo() const {
     return &TSInfo;
   }
+  virtual const ScalarTargetTransformInfo *getScalarTargetTransformInfo()const {
+    return &STTI;
+  }
+  virtual const VectorTargetTransformInfo *getVectorTargetTransformInfo()const {
+    return &VTTI;
+  }
 
   //virtual bool addInstSelector(PassManagerBase &PM,
   //                             CodeGenOpt::Level OptLevel);
diff --git a/lib/Target/PowerPC/InstPrinter/PPCInstPrinter.cpp b/lib/Target/PowerPC/InstPrinter/PPCInstPrinter.cpp
index d175e3e..3d58306 100644
--- a/lib/Target/PowerPC/InstPrinter/PPCInstPrinter.cpp
+++ b/lib/Target/PowerPC/InstPrinter/PPCInstPrinter.cpp
@@ -136,21 +136,21 @@ void PPCInstPrinter::printPredicateOperand(const MCInst *MI, unsigned OpNo,
 
 void PPCInstPrinter::printS5ImmOperand(const MCInst *MI, unsigned OpNo,
                                        raw_ostream &O) {
-  char Value = MI->getOperand(OpNo).getImm();
-  Value = (Value << (32-5)) >> (32-5);
+  int Value = MI->getOperand(OpNo).getImm();
+  Value = SignExtend32<5>(Value);
   O << (int)Value;
 }
 
 void PPCInstPrinter::printU5ImmOperand(const MCInst *MI, unsigned OpNo,
                                        raw_ostream &O) {
-  unsigned char Value = MI->getOperand(OpNo).getImm();
+  unsigned int Value = MI->getOperand(OpNo).getImm();
   assert(Value <= 31 && "Invalid u5imm argument!");
   O << (unsigned int)Value;
 }
 
 void PPCInstPrinter::printU6ImmOperand(const MCInst *MI, unsigned OpNo,
                                        raw_ostream &O) {
-  unsigned char Value = MI->getOperand(OpNo).getImm();
+  unsigned int Value = MI->getOperand(OpNo).getImm();
   assert(Value <= 63 && "Invalid u6imm argument!");
   O << (unsigned int)Value;
 }
diff --git a/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp b/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
index 48de583..87ecb13 100644
--- a/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
+++ b/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp
@@ -29,9 +29,14 @@ static unsigned adjustFixupValue(unsigned Kind, uint64_t Value) {
   case FK_Data_1:
   case FK_Data_2:
   case FK_Data_4:
+  case FK_Data_8:
+  case PPC::fixup_ppc_toc:
     return Value;
+  case PPC::fixup_ppc_lo14:
+  case PPC::fixup_ppc_toc16_ds:
+    return (Value & 0xffff) << 2;
   case PPC::fixup_ppc_brcond14:
-    return Value & 0x3ffc;
+    return Value & 0xfffc;
   case PPC::fixup_ppc_br24:
     return Value & 0x3fffffc;
 #if 0
@@ -41,6 +46,7 @@ static unsigned adjustFixupValue(unsigned Kind, uint64_t Value) {
   case PPC::fixup_ppc_ha16:
     return ((Value >> 16) + ((Value & 0x8000) ? 1 : 0)) & 0xffff;
   case PPC::fixup_ppc_lo16:
+  case PPC::fixup_ppc_toc16:
     return Value & 0xffff;
   }
 }
@@ -72,7 +78,10 @@ public:
       { "fixup_ppc_brcond14",    16,     14,   MCFixupKindInfo::FKF_IsPCRel },
       { "fixup_ppc_lo16",        16,     16,   0 },
       { "fixup_ppc_ha16",        16,     16,   0 },
-      { "fixup_ppc_lo14",        16,     14,   0 }
+      { "fixup_ppc_lo14",        16,     14,   0 },
+      { "fixup_ppc_toc",          0,     64,   0 },
+      { "fixup_ppc_toc16",       16,     16,   0 },
+      { "fixup_ppc_toc16_ds",    16,     14,   0 }
     };
 
     if (Kind < FirstTargetFixupKind)
@@ -181,7 +190,7 @@ namespace {
 
 
 
-MCAsmBackend *llvm::createPPCAsmBackend(const Target &T, StringRef TT) {
+MCAsmBackend *llvm::createPPCAsmBackend(const Target &T, StringRef TT, StringRef CPU) {
   if (Triple(TT).isOSDarwin())
     return new DarwinPPCAsmBackend(T);
 
diff --git a/lib/Target/PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp b/lib/Target/PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp
index a197981..dc93f71 100644
--- a/lib/Target/PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp
+++ b/lib/Target/PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp
@@ -11,6 +11,8 @@
 #include "MCTargetDesc/PPCMCTargetDesc.h"
 #include "llvm/MC/MCELFObjectWriter.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCValue.h"
 
 using namespace llvm;
 
@@ -21,9 +23,15 @@ namespace {
 
     virtual ~PPCELFObjectWriter();
   protected:
+    virtual unsigned getRelocTypeInner(const MCValue &Target,
+                                       const MCFixup &Fixup,
+                                       bool IsPCRel) const;
     virtual unsigned GetRelocType(const MCValue &Target, const MCFixup &Fixup,
                                   bool IsPCRel, bool IsRelocWithSymbol,
                                   int64_t Addend) const;
+    virtual const MCSymbol *undefinedExplicitRelSym(const MCValue &Target,
+                                                    const MCFixup &Fixup,
+                                                    bool IsPCRel) const;
     virtual void adjustFixupOffset(const MCFixup &Fixup, uint64_t &RelocOffset);
   };
 }
@@ -36,11 +44,13 @@ PPCELFObjectWriter::PPCELFObjectWriter(bool Is64Bit, uint8_t OSABI)
 PPCELFObjectWriter::~PPCELFObjectWriter() {
 }
 
-unsigned PPCELFObjectWriter::GetRelocType(const MCValue &Target,
-                                             const MCFixup &Fixup,
-                                             bool IsPCRel,
-                                             bool IsRelocWithSymbol,
-                                             int64_t Addend) const {
+unsigned PPCELFObjectWriter::getRelocTypeInner(const MCValue &Target,
+                                               const MCFixup &Fixup,
+                                               bool IsPCRel) const
+{
+  MCSymbolRefExpr::VariantKind Modifier = Target.isAbsolute() ?
+    MCSymbolRefExpr::VK_None : Target.getSymA()->getKind();
+
   // determine the type of the relocation
   unsigned Type;
   if (IsPCRel) {
@@ -61,17 +71,53 @@ unsigned PPCELFObjectWriter::GetRelocType(const MCValue &Target,
       Type = ELF::R_PPC_ADDR24;
       break;
     case PPC::fixup_ppc_brcond14:
-      Type = ELF::R_PPC_ADDR14_BRTAKEN; // XXX: or BRNTAKEN?_
+      Type = ELF::R_PPC_ADDR14; // XXX: or BRNTAKEN?_
       break;
     case PPC::fixup_ppc_ha16:
-      Type = ELF::R_PPC_ADDR16_HA;
+      switch (Modifier) {
+      default: llvm_unreachable("Unsupported Modifier");
+      case MCSymbolRefExpr::VK_PPC_TPREL16_HA:
+        Type = ELF::R_PPC_TPREL16_HA;
+        break;
+      case MCSymbolRefExpr::VK_None:
+        Type = ELF::R_PPC_ADDR16_HA;
+	break;
+      }
       break;
     case PPC::fixup_ppc_lo16:
-      Type = ELF::R_PPC_ADDR16_LO;
+      switch (Modifier) {
+      default: llvm_unreachable("Unsupported Modifier");
+      case MCSymbolRefExpr::VK_PPC_TPREL16_LO:
+        Type = ELF::R_PPC_TPREL16_LO;
+        break;
+      case MCSymbolRefExpr::VK_None:
+        Type = ELF::R_PPC_ADDR16_LO;
+	break;
+      }
       break;
     case PPC::fixup_ppc_lo14:
       Type = ELF::R_PPC_ADDR14;
       break;
+    case PPC::fixup_ppc_toc:
+      Type = ELF::R_PPC64_TOC;
+      break;
+    case PPC::fixup_ppc_toc16:
+      Type = ELF::R_PPC64_TOC16;
+      break;
+    case PPC::fixup_ppc_toc16_ds:
+      Type = ELF::R_PPC64_TOC16_DS;
+      break;
+    case FK_Data_8:
+      switch (Modifier) {
+      default: llvm_unreachable("Unsupported Modifier");
+      case MCSymbolRefExpr::VK_PPC_TOC:
+        Type = ELF::R_PPC64_TOC;
+        break;
+      case MCSymbolRefExpr::VK_None:
+        Type = ELF::R_PPC64_ADDR64;
+	break;
+      }
+      break;
     case FK_Data_4:
       Type = ELF::R_PPC_ADDR32;
       break;
@@ -83,11 +129,41 @@ unsigned PPCELFObjectWriter::GetRelocType(const MCValue &Target,
   return Type;
 }
 
+unsigned PPCELFObjectWriter::GetRelocType(const MCValue &Target,
+                                          const MCFixup &Fixup,
+                                          bool IsPCRel,
+                                          bool IsRelocWithSymbol,
+                                          int64_t Addend) const {
+  return getRelocTypeInner(Target, Fixup, IsPCRel);
+}
+
+const MCSymbol *PPCELFObjectWriter::undefinedExplicitRelSym(const MCValue &Target,
+                                                            const MCFixup &Fixup,
+                                                            bool IsPCRel) const {
+  assert(Target.getSymA() && "SymA cannot be 0");
+  const MCSymbol &Symbol = Target.getSymA()->getSymbol().AliasedSymbol();
+
+  unsigned RelocType = getRelocTypeInner(Target, Fixup, IsPCRel);
+
+  // The .odp creation emits a relocation against the symbol ".TOC." which
+  // create a R_PPC64_TOC relocation. However the relocation symbol name
+  // in final object creation should be NULL, since the symbol does not
+  // really exist, it is just the reference to TOC base for the current
+  // object file.
+  bool EmitThisSym = RelocType != ELF::R_PPC64_TOC;
+
+  if (EmitThisSym && !Symbol.isTemporary())
+    return &Symbol;
+  return NULL;
+}
+
 void PPCELFObjectWriter::
 adjustFixupOffset(const MCFixup &Fixup, uint64_t &RelocOffset) {
   switch ((unsigned)Fixup.getKind()) {
     case PPC::fixup_ppc_ha16:
     case PPC::fixup_ppc_lo16:
+    case PPC::fixup_ppc_toc16:
+    case PPC::fixup_ppc_toc16_ds:
       RelocOffset += 2;
       break;
     default:
diff --git a/lib/Target/PowerPC/MCTargetDesc/PPCFixupKinds.h b/lib/Target/PowerPC/MCTargetDesc/PPCFixupKinds.h
index b3c889e..37b265e 100644
--- a/lib/Target/PowerPC/MCTargetDesc/PPCFixupKinds.h
+++ b/lib/Target/PowerPC/MCTargetDesc/PPCFixupKinds.h
@@ -34,6 +34,16 @@ enum Fixups {
   /// fixup_ppc_lo14 - A 14-bit fixup corresponding to lo16(_foo) for instrs
   /// like 'std'.
   fixup_ppc_lo14,
+
+  /// fixup_ppc_toc - Insert value of TOC base (.TOC.).
+  fixup_ppc_toc,
+
+  /// fixup_ppc_toc16 - A 16-bit signed fixup relative to the TOC base.
+  fixup_ppc_toc16,
+
+  /// fixup_ppc_toc16_ds - A 14-bit signed fixup relative to the TOC base with
+  /// implied 2 zero bits
+  fixup_ppc_toc16_ds,
   
   // Marker
   LastTargetFixupKind,
diff --git a/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.cpp b/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.cpp
index 245b457..215aa40 100644
--- a/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.cpp
+++ b/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.cpp
@@ -59,12 +59,10 @@ PPCLinuxMCAsmInfo::PPCLinuxMCAsmInfo(bool is64Bit) {
   HasLEB128 = true;  // Target asm supports leb128 directives (little-endian)
 
   // Exceptions handling
-  if (!is64Bit)
-    ExceptionsType = ExceptionHandling::DwarfCFI;
+  ExceptionsType = ExceptionHandling::DwarfCFI;
     
   ZeroDirective = "\t.space\t";
   Data64bitsDirective = is64Bit ? "\t.quad\t" : 0;
-  LCOMMDirectiveType = LCOMM::NoAlignment;
   AssemblerDialect = 0;           // Old-Style mnemonics.
 }
 
diff --git a/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp b/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp
index f652422..2118302 100644
--- a/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp
+++ b/lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp
@@ -15,7 +15,9 @@
 #include "MCTargetDesc/PPCBaseInfo.h"
 #include "MCTargetDesc/PPCFixupKinds.h"
 #include "llvm/MC/MCCodeEmitter.h"
+#include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCInstrInfo.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -25,16 +27,28 @@ STATISTIC(MCNumEmitted, "Number of MC instructions emitted");
 
 namespace {
 class PPCMCCodeEmitter : public MCCodeEmitter {
-  PPCMCCodeEmitter(const PPCMCCodeEmitter &); // DO NOT IMPLEMENT
-  void operator=(const PPCMCCodeEmitter &);   // DO NOT IMPLEMENT
-  
+  PPCMCCodeEmitter(const PPCMCCodeEmitter &) LLVM_DELETED_FUNCTION;
+  void operator=(const PPCMCCodeEmitter &) LLVM_DELETED_FUNCTION;
+
+  const MCSubtargetInfo &STI;
+  Triple TT;
+
 public:
   PPCMCCodeEmitter(const MCInstrInfo &mcii, const MCSubtargetInfo &sti,
-                   MCContext &ctx) {
+                   MCContext &ctx)
+    : STI(sti), TT(STI.getTargetTriple()) {
   }
   
   ~PPCMCCodeEmitter() {}
 
+  bool is64BitMode() const {
+    return (STI.getFeatureBits() & PPC::Feature64Bit) != 0;
+  }
+
+  bool isSVR4ABI() const {
+    return TT.isMacOSX() == 0;
+  }
+
   unsigned getDirectBrEncoding(const MCInst &MI, unsigned OpNo,
                                SmallVectorImpl<MCFixup> &Fixups) const;
   unsigned getCondBrEncoding(const MCInst &MI, unsigned OpNo,
@@ -61,11 +75,19 @@ public:
                                  SmallVectorImpl<MCFixup> &Fixups) const;
   void EncodeInstruction(const MCInst &MI, raw_ostream &OS,
                          SmallVectorImpl<MCFixup> &Fixups) const {
-    unsigned Bits = getBinaryCodeForInstr(MI, Fixups);
+    uint64_t Bits = getBinaryCodeForInstr(MI, Fixups);
+
+    // BL8_NOPELF and BLA8_NOP_ELF is both size of 8 bacause of the
+    // following 'nop'.
+    unsigned Size = 4; // FIXME: Have Desc.getSize() return the correct value!
+    unsigned Opcode = MI.getOpcode();
+    if (Opcode == PPC::BL8_NOP_ELF || Opcode == PPC::BLA8_NOP_ELF)
+      Size = 8;
     
     // Output the constant in big endian byte order.
-    for (unsigned i = 0; i != 4; ++i) {
-      OS << (char)(Bits >> 24);
+    int ShiftValue = (Size * 8) - 8;
+    for (unsigned i = 0; i != Size; ++i) {
+      OS << (char)(Bits >> ShiftValue);
       Bits <<= 8;
     }
     
@@ -140,8 +162,12 @@ unsigned PPCMCCodeEmitter::getMemRIEncoding(const MCInst &MI, unsigned OpNo,
     return (getMachineOpValue(MI, MO, Fixups) & 0xFFFF) | RegBits;
   
   // Add a fixup for the displacement field.
-  Fixups.push_back(MCFixup::Create(0, MO.getExpr(),
-                                   (MCFixupKind)PPC::fixup_ppc_lo16));
+  if (isSVR4ABI() && is64BitMode())
+    Fixups.push_back(MCFixup::Create(0, MO.getExpr(),
+                                     (MCFixupKind)PPC::fixup_ppc_toc16));
+  else
+    Fixups.push_back(MCFixup::Create(0, MO.getExpr(),
+                                     (MCFixupKind)PPC::fixup_ppc_lo16));
   return RegBits;
 }
 
@@ -158,8 +184,12 @@ unsigned PPCMCCodeEmitter::getMemRIXEncoding(const MCInst &MI, unsigned OpNo,
     return (getMachineOpValue(MI, MO, Fixups) & 0x3FFF) | RegBits;
   
   // Add a fixup for the branch target.
-  Fixups.push_back(MCFixup::Create(0, MO.getExpr(),
-                                   (MCFixupKind)PPC::fixup_ppc_lo14));
+  if (isSVR4ABI() && is64BitMode())
+    Fixups.push_back(MCFixup::Create(0, MO.getExpr(),
+                                     (MCFixupKind)PPC::fixup_ppc_toc16_ds));
+  else
+    Fixups.push_back(MCFixup::Create(0, MO.getExpr(),
+                                     (MCFixupKind)PPC::fixup_ppc_lo14));
   return RegBits;
 }
 
@@ -168,7 +198,9 @@ unsigned PPCMCCodeEmitter::
 get_crbitm_encoding(const MCInst &MI, unsigned OpNo,
                     SmallVectorImpl<MCFixup> &Fixups) const {
   const MCOperand &MO = MI.getOperand(OpNo);
-  assert((MI.getOpcode() == PPC::MTCRF || MI.getOpcode() == PPC::MFOCRF) &&
+  assert((MI.getOpcode() == PPC::MTCRF || 
+          MI.getOpcode() == PPC::MFOCRF ||
+          MI.getOpcode() == PPC::MTCRF8) &&
          (MO.getReg() >= PPC::CR0 && MO.getReg() <= PPC::CR7));
   return 0x80 >> getPPCRegisterNumbering(MO.getReg());
 }
diff --git a/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp b/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp
index 6568e82..4c2578d 100644
--- a/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp
+++ b/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp
@@ -70,7 +70,7 @@ static MCAsmInfo *createPPCMCAsmInfo(const Target &T, StringRef TT) {
 
   // Initial state of the frame pointer is R1.
   MachineLocation Dst(MachineLocation::VirtualFP);
-  MachineLocation Src(PPC::R1, 0);
+  MachineLocation Src(isPPC64? PPC::X1 : PPC::R1, 0);
   MAI->addInitialFrameState(0, Dst, Src);
 
   return MAI;
diff --git a/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.h b/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.h
index 7162e15..a0e4cf3 100644
--- a/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.h
+++ b/lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.h
@@ -36,7 +36,7 @@ MCCodeEmitter *createPPCMCCodeEmitter(const MCInstrInfo &MCII,
                                       const MCSubtargetInfo &STI,
                                       MCContext &Ctx);
 
-MCAsmBackend *createPPCAsmBackend(const Target &T, StringRef TT);
+MCAsmBackend *createPPCAsmBackend(const Target &T, StringRef TT, StringRef CPU);
 
 /// createPPCELFObjectWriter - Construct an PPC ELF object writer.
 MCObjectWriter *createPPCELFObjectWriter(raw_ostream &OS,
diff --git a/lib/Target/PowerPC/PPC.td b/lib/Target/PowerPC/PPC.td
index b7f1688..cb15dad 100644
--- a/lib/Target/PowerPC/PPC.td
+++ b/lib/Target/PowerPC/PPC.td
@@ -35,6 +35,10 @@ def Directive970 : SubtargetFeature<"", "DarwinDirective", "PPC::DIR_970", "">;
 def Directive32  : SubtargetFeature<"", "DarwinDirective", "PPC::DIR_32", "">;
 def Directive64  : SubtargetFeature<"", "DarwinDirective", "PPC::DIR_64", "">;
 def DirectiveA2  : SubtargetFeature<"", "DarwinDirective", "PPC::DIR_A2", "">;
+def DirectiveE500mc : SubtargetFeature<"", "DarwinDirective",
+                                       "PPC::DIR_E500mc", "">;
+def DirectiveE5500  : SubtargetFeature<"", "DarwinDirective", 
+                                       "PPC::DIR_E5500", "">;
 def DirectivePwr6: SubtargetFeature<"", "DarwinDirective", "PPC::DIR_PWR6", "">;
 def DirectivePwr7: SubtargetFeature<"", "DarwinDirective", "PPC::DIR_PWR7", "">;
 
@@ -94,6 +98,12 @@ def : Processor<"g5", G5Itineraries,
                   [Directive970, FeatureAltivec,
                    FeatureMFOCRF, FeatureFSqrt, FeatureSTFIWX,
                    Feature64Bit /*, Feature64BitRegs */]>;
+def : ProcessorModel<"e500mc", PPCE500mcModel,
+                  [DirectiveE500mc, FeatureMFOCRF,
+                   FeatureSTFIWX, FeatureBookE, FeatureISEL]>;
+def : ProcessorModel<"e5500", PPCE5500Model,
+                  [DirectiveE5500, FeatureMFOCRF, Feature64Bit,
+                   FeatureSTFIWX, FeatureBookE, FeatureISEL]>;
 def : Processor<"a2", PPCA2Itineraries, [DirectiveA2, FeatureBookE,
                                          FeatureMFOCRF, FeatureFSqrt,
                                          FeatureSTFIWX, FeatureISEL,
diff --git a/lib/Target/PowerPC/PPCAsmPrinter.cpp b/lib/Target/PowerPC/PPCAsmPrinter.cpp
index f76b89c..15d690b 100644
--- a/lib/Target/PowerPC/PPCAsmPrinter.cpp
+++ b/lib/Target/PowerPC/PPCAsmPrinter.cpp
@@ -54,12 +54,13 @@
 #include "llvm/Support/ELF.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/MapVector.h"
 using namespace llvm;
 
 namespace {
   class PPCAsmPrinter : public AsmPrinter {
   protected:
-    DenseMap<MCSymbol*, MCSymbol*> TOC;
+    MapVector<MCSymbol*, MCSymbol*> TOC;
     const PPCSubtarget &Subtarget;
     uint64_t TOCLabelID;
   public:
@@ -109,6 +110,8 @@ namespace {
     bool doFinalization(Module &M);
 
     virtual void EmitFunctionEntryLabel();
+
+    void EmitFunctionBodyEnd();
   };
 
   /// PPCDarwinAsmPrinter - PowerPC assembly printer, customized for Darwin/Mac
@@ -282,8 +285,22 @@ bool PPCAsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNo,
                                           unsigned AsmVariant,
                                           const char *ExtraCode,
                                           raw_ostream &O) {
-  if (ExtraCode && ExtraCode[0])
-    return true; // Unknown modifier.
+  if (ExtraCode && ExtraCode[0]) {
+    if (ExtraCode[1] != 0) return true; // Unknown modifier.
+
+    switch (ExtraCode[0]) {
+    default: return true;  // Unknown modifier.
+    case 'y': // A memory reference for an X-form instruction
+      {
+        const char *RegName = "r0";
+        if (!Subtarget.isDarwin()) RegName = stripRegisterPrefix(RegName);
+        O << RegName << ", ";
+        printOperand(MI, OpNo, O);
+        return false;
+      }
+    }
+  }
+
   assert(MI->getOperand(OpNo).isReg());
   O << "0(";
   printOperand(MI, OpNo, O);
@@ -345,23 +362,37 @@ void PPCAsmPrinter::EmitInstruction(const MachineInstr *MI) {
     OutStreamer.EmitLabel(PICBase);
     return;
   }
+  case PPC::LDtocJTI:
+  case PPC::LDtocCPT:
   case PPC::LDtoc: {
     // Transform %X3 = LDtoc <ga:@min1>, %X2
     LowerPPCMachineInstrToMCInst(MI, TmpInst, *this, Subtarget.isDarwin());
-      
+
     // Change the opcode to LD, and the global address operand to be a
     // reference to the TOC entry we will synthesize later.
     TmpInst.setOpcode(PPC::LD);
     const MachineOperand &MO = MI->getOperand(1);
-    assert(MO.isGlobal());
-      
-    // Map symbol -> label of TOC entry.
-    MCSymbol *&TOCEntry = TOC[Mang->getSymbol(MO.getGlobal())];
-    if (TOCEntry == 0)
-      TOCEntry = GetTempSymbol("C", TOCLabelID++);
-      
+
+    // Map symbol -> label of TOC entry
+    assert(MO.isGlobal() || MO.isCPI() || MO.isJTI());
+    MCSymbol *MOSymbol = 0;
+    if (MO.isGlobal())
+      MOSymbol = Mang->getSymbol(MO.getGlobal());
+    else if (MO.isCPI())
+      MOSymbol = GetCPISymbol(MO.getIndex());
+    else if (MO.isJTI())
+      MOSymbol = GetJTISymbol(MO.getIndex());
+    MCSymbol *&TOCEntry = TOC[MOSymbol];
+    // To avoid name clash check if the name already exists.
+    while (TOCEntry == 0) {
+      if (OutContext.LookupSymbol(Twine(MAI->getPrivateGlobalPrefix()) +
+                                  "C" + Twine(TOCLabelID++)) == 0) {
+        TOCEntry = GetTempSymbol("C", TOCLabelID);
+      }
+    }
+
     const MCExpr *Exp =
-      MCSymbolRefExpr::Create(TOCEntry, MCSymbolRefExpr::VK_PPC_TOC,
+      MCSymbolRefExpr::Create(TOCEntry, MCSymbolRefExpr::VK_PPC_TOC_ENTRY,
                               OutContext);
     TmpInst.getOperand(1) = MCOperand::CreateExpr(Exp);
     OutStreamer.EmitInstruction(TmpInst);
@@ -404,11 +435,17 @@ void PPCLinuxAsmPrinter::EmitFunctionEntryLabel() {
   OutStreamer.EmitValueToAlignment(8);
   MCSymbol *Symbol1 = 
     OutContext.GetOrCreateSymbol(".L." + Twine(CurrentFnSym->getName()));
-  MCSymbol *Symbol2 = OutContext.GetOrCreateSymbol(StringRef(".TOC.@tocbase"));
+  // Generates a R_PPC64_ADDR64 (from FK_DATA_8) relocation for the function
+  // entry point.
   OutStreamer.EmitValue(MCSymbolRefExpr::Create(Symbol1, OutContext),
-                        Subtarget.isPPC64() ? 8 : 4/*size*/, 0/*addrspace*/);
-  OutStreamer.EmitValue(MCSymbolRefExpr::Create(Symbol2, OutContext),
-                        Subtarget.isPPC64() ? 8 : 4/*size*/, 0/*addrspace*/);
+                        8/*size*/, 0/*addrspace*/);
+  MCSymbol *Symbol2 = OutContext.GetOrCreateSymbol(StringRef(".TOC."));
+  // Generates a R_PPC64_TOC relocation for TOC base insertion.
+  OutStreamer.EmitValue(MCSymbolRefExpr::Create(Symbol2,
+                        MCSymbolRefExpr::VK_PPC_TOC, OutContext),
+                        8/*size*/, 0/*addrspace*/);
+  // Emit a null environment pointer.
+  OutStreamer.EmitIntValue(0, 8 /* size */, 0 /* addrspace */);
   OutStreamer.SwitchSection(Current);
 
   MCSymbol *RealFnSym = OutContext.GetOrCreateSymbol(
@@ -419,7 +456,7 @@ void PPCLinuxAsmPrinter::EmitFunctionEntryLabel() {
 
 
 bool PPCLinuxAsmPrinter::doFinalization(Module &M) {
-  const TargetData *TD = TM.getTargetData();
+  const DataLayout *TD = TM.getDataLayout();
 
   bool isPPC64 = TD->getPointerSizeInBits() == 64;
 
@@ -429,18 +466,34 @@ bool PPCLinuxAsmPrinter::doFinalization(Module &M) {
         SectionKind::getReadOnly());
     OutStreamer.SwitchSection(Section);
 
-    // FIXME: This is nondeterminstic!
-    for (DenseMap<MCSymbol*, MCSymbol*>::iterator I = TOC.begin(),
+    for (MapVector<MCSymbol*, MCSymbol*>::iterator I = TOC.begin(),
          E = TOC.end(); I != E; ++I) {
       OutStreamer.EmitLabel(I->second);
-      OutStreamer.EmitRawText("\t.tc " + Twine(I->first->getName()) +
-                              "[TC]," + I->first->getName());
+      MCSymbol *S = OutContext.GetOrCreateSymbol(I->first->getName());
+      OutStreamer.EmitTCEntry(*S);
     }
   }
 
   return AsmPrinter::doFinalization(M);
 }
 
+/// EmitFunctionBodyEnd - Print the traceback table before the .size
+/// directive.
+///
+void PPCLinuxAsmPrinter::EmitFunctionBodyEnd() {
+  // Only the 64-bit target requires a traceback table.  For now,
+  // we only emit the word of zeroes that GDB requires to find
+  // the end of the function, and zeroes for the eight-byte
+  // mandatory fields.
+  // FIXME: We should fill in the eight-byte mandatory fields as described in
+  // the PPC64 ELF ABI (this is a low-priority item because GDB does not
+  // currently make use of these fields).
+  if (Subtarget.isPPC64()) {
+    OutStreamer.EmitIntValue(0, 4/*size*/);
+    OutStreamer.EmitIntValue(0, 8/*size*/);
+  }
+}
+
 void PPCDarwinAsmPrinter::EmitStartOfAsmFile(Module &M) {
   static const char *const CPUDirectives[] = {
     "",
@@ -453,6 +506,8 @@ void PPCDarwinAsmPrinter::EmitStartOfAsmFile(Module &M) {
     "ppc750",
     "ppc970",
     "ppcA2",
+    "ppce500mc",
+    "ppce5500",
     "power6",
     "power7",
     "ppc64"
@@ -508,7 +563,7 @@ static MCSymbol *GetAnonSym(MCSymbol *Sym, MCContext &Ctx) {
 
 void PPCDarwinAsmPrinter::
 EmitFunctionStubs(const MachineModuleInfoMachO::SymbolListTy &Stubs) {
-  bool isPPC64 = TM.getTargetData()->getPointerSizeInBits() == 64;
+  bool isPPC64 = TM.getDataLayout()->getPointerSizeInBits() == 64;
   
   const TargetLoweringObjectFileMachO &TLOFMacho = 
     static_cast<const TargetLoweringObjectFileMachO &>(getObjFileLowering());
@@ -603,7 +658,7 @@ EmitFunctionStubs(const MachineModuleInfoMachO::SymbolListTy &Stubs) {
 
 
 bool PPCDarwinAsmPrinter::doFinalization(Module &M) {
-  bool isPPC64 = TM.getTargetData()->getPointerSizeInBits() == 64;
+  bool isPPC64 = TM.getDataLayout()->getPointerSizeInBits() == 64;
 
   // Darwin/PPC always uses mach-o.
   const TargetLoweringObjectFileMachO &TLOFMacho = 
diff --git a/lib/Target/PowerPC/PPCCallingConv.td b/lib/Target/PowerPC/PPCCallingConv.td
index b2b5364..3f87e88 100644
--- a/lib/Target/PowerPC/PPCCallingConv.td
+++ b/lib/Target/PowerPC/PPCCallingConv.td
@@ -12,12 +12,19 @@
 //
 //===----------------------------------------------------------------------===//
 
+/// CCIfSubtarget - Match if the current subtarget has a feature F.
+class CCIfSubtarget<string F, CCAction A>
+ : CCIf<!strconcat("State.getTarget().getSubtarget<PPCSubtarget>().", F), A>;
+
 //===----------------------------------------------------------------------===//
 // Return Value Calling Convention
 //===----------------------------------------------------------------------===//
 
 // Return-value convention for PowerPC
 def RetCC_PPC : CallingConv<[
+  // On PPC64, integer return values are always promoted to i64
+  CCIfType<[i32], CCIfSubtarget<"isPPC64()", CCPromoteToType<i64>>>,
+
   CCIfType<[i32], CCAssignToReg<[R3, R4, R5, R6, R7, R8, R9, R10]>>,
   CCIfType<[i64], CCAssignToReg<[X3, X4, X5, X6]>>,
   
diff --git a/lib/Target/PowerPC/PPCFrameLowering.cpp b/lib/Target/PowerPC/PPCFrameLowering.cpp
index c24afa9..caf7bf2 100644
--- a/lib/Target/PowerPC/PPCFrameLowering.cpp
+++ b/lib/Target/PowerPC/PPCFrameLowering.cpp
@@ -13,6 +13,7 @@
 
 #include "PPCFrameLowering.h"
 #include "PPCInstrInfo.h"
+#include "PPCInstrBuilder.h"
 #include "PPCMachineFunctionInfo.h"
 #include "llvm/Function.h"
 #include "llvm/CodeGen/MachineFrameInfo.h"
@@ -49,6 +50,11 @@ static const uint16_t VRRegNo[] = {
 /// to manipulate the VRSAVE register, even though it uses vector registers.
 /// This can happen when the only registers used are known to be live in or out
 /// of the function.  Remove all of the VRSAVE related code from the function.
+/// FIXME: The removal of the code results in a compile failure at -O0 when the
+/// function contains a function call, as the GPR containing original VRSAVE
+/// contents is spilled and reloaded around the call.  Without the prolog code,
+/// the spill instruction refers to an undefined register.  This code needs
+/// to account for all uses of that GPR.
 static void RemoveVRSaveCode(MachineInstr *MI) {
   MachineBasicBlock *Entry = MI->getParent();
   MachineFunction *MF = Entry->getParent();
@@ -168,6 +174,11 @@ static void HandleVRSaveUpdate(MachineInstr *MI, const TargetInstrInfo &TII) {
   MI->eraseFromParent();
 }
 
+static bool spillsCR(const MachineFunction &MF) {
+  const PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
+  return FuncInfo->isCRSpilled();
+}
+
 /// determineFrameLayout - Determine the size of the frame and maximum call
 /// frame size.
 void PPCFrameLowering::determineFrameLayout(MachineFunction &MF) const {
@@ -184,13 +195,22 @@ void PPCFrameLowering::determineFrameLayout(MachineFunction &MF) const {
 
   // If we are a leaf function, and use up to 224 bytes of stack space,
   // don't have a frame pointer, calls, or dynamic alloca then we do not need
-  // to adjust the stack pointer (we fit in the Red Zone).
-  bool DisableRedZone = MF.getFunction()->hasFnAttr(Attribute::NoRedZone);
-  // FIXME SVR4 The 32-bit SVR4 ABI has no red zone.
+  // to adjust the stack pointer (we fit in the Red Zone).  For 64-bit
+  // SVR4, we also require a stack frame if we need to spill the CR,
+  // since this spill area is addressed relative to the stack pointer.
+  bool DisableRedZone = MF.getFunction()->getFnAttributes().
+    hasAttribute(Attributes::NoRedZone);
+  // FIXME SVR4 The 32-bit SVR4 ABI has no red zone.  However, it can
+  // still generate stackless code if all local vars are reg-allocated.
+  // Try: (FrameSize <= 224
+  //       || (FrameSize == 0 && Subtarget.isPPC32 && Subtarget.isSVR4ABI()))
   if (!DisableRedZone &&
       FrameSize <= 224 &&                          // Fits in red zone.
       !MFI->hasVarSizedObjects() &&                // No dynamic alloca.
       !MFI->adjustsStack() &&                      // No calls.
+      !(Subtarget.isPPC64() &&                     // No 64-bit SVR4 CRsave.
+	Subtarget.isSVR4ABI()
+	&& spillsCR(MF)) &&
       (!ALIGN_STACK || MaxAlign <= TargetAlign)) { // No special alignment.
     // No need for frame
     MFI->setStackSize(0);
@@ -241,7 +261,7 @@ bool PPCFrameLowering::needsFP(const MachineFunction &MF) const {
 
   // Naked functions have no stack frame pushed, so we don't have a frame
   // pointer.
-  if (MF.getFunction()->hasFnAttr(Attribute::Naked))
+  if (MF.getFunction()->getFnAttributes().hasAttribute(Attributes::Naked))
     return false;
 
   return MF.getTarget().Options.DisableFramePointerElim(MF) ||
@@ -268,12 +288,13 @@ void PPCFrameLowering::emitPrologue(MachineFunction &MF) const {
 
   // Scan the prolog, looking for an UPDATE_VRSAVE instruction.  If we find it,
   // process it.
-  for (unsigned i = 0; MBBI != MBB.end(); ++i, ++MBBI) {
-    if (MBBI->getOpcode() == PPC::UPDATE_VRSAVE) {
-      HandleVRSaveUpdate(MBBI, TII);
-      break;
+  if (!Subtarget.isSVR4ABI())
+    for (unsigned i = 0; MBBI != MBB.end(); ++i, ++MBBI) {
+      if (MBBI->getOpcode() == PPC::UPDATE_VRSAVE) {
+        HandleVRSaveUpdate(MBBI, TII);
+        break;
+      }
     }
-  }
 
   // Move MBBI back to the beginning of the function.
   MBBI = MBB.begin();
@@ -488,7 +509,6 @@ void PPCFrameLowering::emitPrologue(MachineFunction &MF) const {
     // Add callee saved registers to move list.
     const std::vector<CalleeSavedInfo> &CSI = MFI->getCalleeSavedInfo();
     for (unsigned I = 0, E = CSI.size(); I != E; ++I) {
-      int Offset = MFI->getObjectOffset(CSI[I].getFrameIdx());
       unsigned Reg = CSI[I].getReg();
       if (Reg == PPC::LR || Reg == PPC::LR8 || Reg == PPC::RM) continue;
 
@@ -497,6 +517,25 @@ void PPCFrameLowering::emitPrologue(MachineFunction &MF) const {
       if (PPC::CRBITRCRegClass.contains(Reg))
         continue;
 
+      // For SVR4, don't emit a move for the CR spill slot if we haven't
+      // spilled CRs.
+      if (Subtarget.isSVR4ABI()
+	  && (PPC::CR2 <= Reg && Reg <= PPC::CR4)
+	  && !spillsCR(MF))
+	continue;
+
+      // For 64-bit SVR4 when we have spilled CRs, the spill location
+      // is SP+8, not a frame-relative slot.
+      if (Subtarget.isSVR4ABI()
+	  && Subtarget.isPPC64()
+	  && (PPC::CR2 <= Reg && Reg <= PPC::CR4)) {
+	MachineLocation CSDst(PPC::X1, 8);
+	MachineLocation CSSrc(PPC::CR2);
+	Moves.push_back(MachineMove(Label, CSDst, CSSrc));
+	continue;
+      }
+
+      int Offset = MFI->getObjectOffset(CSI[I].getFrameIdx());
       MachineLocation CSDst(MachineLocation::VirtualFP, Offset);
       MachineLocation CSSrc(Reg);
       Moves.push_back(MachineMove(Label, CSDst, CSSrc));
@@ -714,11 +753,6 @@ void PPCFrameLowering::emitEpilogue(MachineFunction &MF,
   }
 }
 
-static bool spillsCR(const MachineFunction &MF) {
-  const PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
-  return FuncInfo->isCRSpilled();
-}
-
 /// MustSaveLR - Return true if this function requires that we save the LR
 /// register onto the stack in the prolog and restore it in the epilog of the
 /// function.
@@ -808,7 +842,6 @@ void PPCFrameLowering::processFunctionBeforeFrameFinalized(MachineFunction &MF)
   bool HasGPSaveArea = false;
   bool HasG8SaveArea = false;
   bool HasFPSaveArea = false;
-  bool HasCRSaveArea = false;
   bool HasVRSAVESaveArea = false;
   bool HasVRSaveArea = false;
 
@@ -843,10 +876,9 @@ void PPCFrameLowering::processFunctionBeforeFrameFinalized(MachineFunction &MF)
       if (Reg < MinFPR) {
         MinFPR = Reg;
       }
-// FIXME SVR4: Disable CR save area for now.
     } else if (PPC::CRBITRCRegClass.contains(Reg) ||
                PPC::CRRCRegClass.contains(Reg)) {
-//      HasCRSaveArea = true;
+      ; // do nothing, as we already know whether CRs are spilled
     } else if (PPC::VRSAVERCRegClass.contains(Reg)) {
       HasVRSAVESaveArea = true;
     } else if (PPC::VRRCRegClass.contains(Reg)) {
@@ -926,16 +958,21 @@ void PPCFrameLowering::processFunctionBeforeFrameFinalized(MachineFunction &MF)
     }
   }
 
-  // The CR save area is below the general register save area.
-  if (HasCRSaveArea) {
-    // FIXME SVR4: Is it actually possible to have multiple elements in CSI
-    //             which have the CR/CRBIT register class?
+  // For 32-bit only, the CR save area is below the general register
+  // save area.  For 64-bit SVR4, the CR save area is addressed relative
+  // to the stack pointer and hence does not need an adjustment here.
+  // Only CR2 (the first nonvolatile spilled) has an associated frame
+  // index so that we have a single uniform save area.
+  if (spillsCR(MF) && !(Subtarget.isPPC64() && Subtarget.isSVR4ABI())) {
     // Adjust the frame index of the CR spill slot.
     for (unsigned i = 0, e = CSI.size(); i != e; ++i) {
       unsigned Reg = CSI[i].getReg();
 
-      if (PPC::CRBITRCRegClass.contains(Reg) ||
-          PPC::CRRCRegClass.contains(Reg)) {
+      if ((Subtarget.isSVR4ABI() && Reg == PPC::CR2)
+	  // Leave Darwin logic as-is.
+	  || (!Subtarget.isSVR4ABI() &&
+	      (PPC::CRBITRCRegClass.contains(Reg) ||
+	       PPC::CRRCRegClass.contains(Reg)))) {
         int FI = CSI[i].getFrameIdx();
 
         FFI->setObjectOffset(FI, LowerBound + FFI->getObjectOffset(FI));
@@ -973,3 +1010,184 @@ void PPCFrameLowering::processFunctionBeforeFrameFinalized(MachineFunction &MF)
     }
   }
 }
+
+bool 
+PPCFrameLowering::spillCalleeSavedRegisters(MachineBasicBlock &MBB,
+				     MachineBasicBlock::iterator MI,
+				     const std::vector<CalleeSavedInfo> &CSI,
+				     const TargetRegisterInfo *TRI) const {
+
+  // Currently, this function only handles SVR4 32- and 64-bit ABIs.
+  // Return false otherwise to maintain pre-existing behavior.
+  if (!Subtarget.isSVR4ABI())
+    return false;
+
+  MachineFunction *MF = MBB.getParent();
+  const PPCInstrInfo &TII =
+    *static_cast<const PPCInstrInfo*>(MF->getTarget().getInstrInfo());
+  DebugLoc DL;
+  bool CRSpilled = false;
+  
+  for (unsigned i = 0, e = CSI.size(); i != e; ++i) {
+    unsigned Reg = CSI[i].getReg();
+    // CR2 through CR4 are the nonvolatile CR fields.
+    bool IsCRField = PPC::CR2 <= Reg && Reg <= PPC::CR4;
+
+    if (CRSpilled && IsCRField)
+      continue;
+
+    // Add the callee-saved register as live-in; it's killed at the spill.
+    MBB.addLiveIn(Reg);
+
+    // Insert the spill to the stack frame.
+    if (IsCRField) {
+      CRSpilled = true;
+      // The first time we see a CR field, store the whole CR into the
+      // save slot via GPR12 (available in the prolog for 32- and 64-bit).
+      if (Subtarget.isPPC64()) {
+	// 64-bit:  SP+8
+	MBB.insert(MI, BuildMI(*MF, DL, TII.get(PPC::MFCR), PPC::X12));
+	MBB.insert(MI, BuildMI(*MF, DL, TII.get(PPC::STW))
+			       .addReg(PPC::X12,
+				       getKillRegState(true))
+			       .addImm(8)
+			       .addReg(PPC::X1));
+      } else {
+	// 32-bit:  FP-relative.  Note that we made sure CR2-CR4 all have
+	// the same frame index in PPCRegisterInfo::hasReservedSpillSlot.
+	MBB.insert(MI, BuildMI(*MF, DL, TII.get(PPC::MFCR), PPC::R12));
+	MBB.insert(MI, addFrameReference(BuildMI(*MF, DL, TII.get(PPC::STW))
+					 .addReg(PPC::R12,
+						 getKillRegState(true)),
+					 CSI[i].getFrameIdx()));
+      }
+      
+      // Record that we spill the CR in this function.
+      PPCFunctionInfo *FuncInfo = MF->getInfo<PPCFunctionInfo>();
+      FuncInfo->setSpillsCR();
+    } else {
+      const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
+      TII.storeRegToStackSlot(MBB, MI, Reg, true,
+			      CSI[i].getFrameIdx(), RC, TRI);
+    }
+  }
+  return true;
+}
+
+static void
+restoreCRs(bool isPPC64, bool CR2Spilled, bool CR3Spilled, bool CR4Spilled,
+	   MachineBasicBlock &MBB, MachineBasicBlock::iterator MI,
+	   const std::vector<CalleeSavedInfo> &CSI, unsigned CSIIndex) {
+
+  MachineFunction *MF = MBB.getParent();
+  const PPCInstrInfo &TII =
+    *static_cast<const PPCInstrInfo*>(MF->getTarget().getInstrInfo());
+  DebugLoc DL;
+  unsigned RestoreOp, MoveReg;
+
+  if (isPPC64) {
+    // 64-bit:  SP+8
+    MBB.insert(MI, BuildMI(*MF, DL, TII.get(PPC::LWZ), PPC::X12)
+	       .addImm(8)
+	       .addReg(PPC::X1));
+    RestoreOp = PPC::MTCRF8;
+    MoveReg = PPC::X12;
+  } else {
+    // 32-bit:  FP-relative
+    MBB.insert(MI, addFrameReference(BuildMI(*MF, DL, TII.get(PPC::LWZ),
+					     PPC::R12),
+				     CSI[CSIIndex].getFrameIdx()));
+    RestoreOp = PPC::MTCRF;
+    MoveReg = PPC::R12;
+  }
+  
+  if (CR2Spilled)
+    MBB.insert(MI, BuildMI(*MF, DL, TII.get(RestoreOp), PPC::CR2)
+	       .addReg(MoveReg));
+
+  if (CR3Spilled)
+    MBB.insert(MI, BuildMI(*MF, DL, TII.get(RestoreOp), PPC::CR3)
+	       .addReg(MoveReg));
+
+  if (CR4Spilled)
+    MBB.insert(MI, BuildMI(*MF, DL, TII.get(RestoreOp), PPC::CR4)
+	       .addReg(MoveReg));
+}
+
+bool 
+PPCFrameLowering::restoreCalleeSavedRegisters(MachineBasicBlock &MBB,
+					MachineBasicBlock::iterator MI,
+				        const std::vector<CalleeSavedInfo> &CSI,
+					const TargetRegisterInfo *TRI) const {
+
+  // Currently, this function only handles SVR4 32- and 64-bit ABIs.
+  // Return false otherwise to maintain pre-existing behavior.
+  if (!Subtarget.isSVR4ABI())
+    return false;
+
+  MachineFunction *MF = MBB.getParent();
+  const PPCInstrInfo &TII =
+    *static_cast<const PPCInstrInfo*>(MF->getTarget().getInstrInfo());
+  bool CR2Spilled = false;
+  bool CR3Spilled = false;
+  bool CR4Spilled = false;
+  unsigned CSIIndex = 0;
+
+  // Initialize insertion-point logic; we will be restoring in reverse
+  // order of spill.
+  MachineBasicBlock::iterator I = MI, BeforeI = I;
+  bool AtStart = I == MBB.begin();
+
+  if (!AtStart)
+    --BeforeI;
+
+  for (unsigned i = 0, e = CSI.size(); i != e; ++i) {
+    unsigned Reg = CSI[i].getReg();
+
+    if (Reg == PPC::CR2) {
+      CR2Spilled = true;
+      // The spill slot is associated only with CR2, which is the
+      // first nonvolatile spilled.  Save it here.
+      CSIIndex = i;
+      continue;
+    } else if (Reg == PPC::CR3) {
+      CR3Spilled = true;
+      continue;
+    } else if (Reg == PPC::CR4) {
+      CR4Spilled = true;
+      continue;
+    } else {
+      // When we first encounter a non-CR register after seeing at
+      // least one CR register, restore all spilled CRs together.
+      if ((CR2Spilled || CR3Spilled || CR4Spilled)
+	  && !(PPC::CR2 <= Reg && Reg <= PPC::CR4)) {
+	restoreCRs(Subtarget.isPPC64(), CR2Spilled, CR3Spilled, CR4Spilled,
+		   MBB, I, CSI, CSIIndex);
+	CR2Spilled = CR3Spilled = CR4Spilled = false;
+      }
+
+      // Default behavior for non-CR saves.
+      const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg);
+      TII.loadRegFromStackSlot(MBB, I, Reg, CSI[i].getFrameIdx(),
+			       RC, TRI);
+      assert(I != MBB.begin() &&
+	     "loadRegFromStackSlot didn't insert any code!");
+      }
+
+    // Insert in reverse order.
+    if (AtStart)
+      I = MBB.begin();
+    else {
+      I = BeforeI;
+      ++I;
+    }	    
+  }
+
+  // If we haven't yet spilled the CRs, do so now.
+  if (CR2Spilled || CR3Spilled || CR4Spilled)
+    restoreCRs(Subtarget.isPPC64(), CR2Spilled, CR3Spilled, CR4Spilled,
+	       MBB, I, CSI, CSIIndex);
+
+  return true;
+}
+
diff --git a/lib/Target/PowerPC/PPCFrameLowering.h b/lib/Target/PowerPC/PPCFrameLowering.h
index d708541..4d957b9 100644
--- a/lib/Target/PowerPC/PPCFrameLowering.h
+++ b/lib/Target/PowerPC/PPCFrameLowering.h
@@ -45,6 +45,16 @@ public:
                                             RegScavenger *RS = NULL) const;
   void processFunctionBeforeFrameFinalized(MachineFunction &MF) const;
 
+  bool spillCalleeSavedRegisters(MachineBasicBlock &MBB,
+                                 MachineBasicBlock::iterator MI,
+                                 const std::vector<CalleeSavedInfo> &CSI,
+                                 const TargetRegisterInfo *TRI) const;
+
+  bool restoreCalleeSavedRegisters(MachineBasicBlock &MBB,
+                                   MachineBasicBlock::iterator MI,
+                                   const std::vector<CalleeSavedInfo> &CSI,
+                                   const TargetRegisterInfo *TRI) const;
+
   /// targetHandlesStackFrameRounding - Returns true if the target is
   /// responsible for rounding up the stack frame (probably at emitPrologue
   /// time).
@@ -170,23 +180,11 @@ public:
       {PPC::R15, -68},
       {PPC::R14, -72},
 
-      // CR save area offset.
-      // FIXME SVR4: Disable CR save area for now.
-//      {PPC::CR2, -4},
-//      {PPC::CR3, -4},
-//      {PPC::CR4, -4},
-//      {PPC::CR2LT, -4},
-//      {PPC::CR2GT, -4},
-//      {PPC::CR2EQ, -4},
-//      {PPC::CR2UN, -4},
-//      {PPC::CR3LT, -4},
-//      {PPC::CR3GT, -4},
-//      {PPC::CR3EQ, -4},
-//      {PPC::CR3UN, -4},
-//      {PPC::CR4LT, -4},
-//      {PPC::CR4GT, -4},
-//      {PPC::CR4EQ, -4},
-//      {PPC::CR4UN, -4},
+      // CR save area offset.  We map each of the nonvolatile CR fields
+      // to the slot for CR2, which is the first of the nonvolatile CR
+      // fields to be assigned, so that we only allocate one save slot.
+      // See PPCRegisterInfo::hasReservedSpillSlot() for more information.
+      {PPC::CR2, -4},
 
       // VRSAVE save area offset.
       {PPC::VRSAVE, -4},
@@ -228,27 +226,6 @@ public:
       {PPC::F14, -144},
 
       // General register save area offsets.
-      // FIXME 64-bit SVR4: Are 32-bit registers actually allocated in 64-bit
-      //                    mode?
-      {PPC::R31, -4},
-      {PPC::R30, -12},
-      {PPC::R29, -20},
-      {PPC::R28, -28},
-      {PPC::R27, -36},
-      {PPC::R26, -44},
-      {PPC::R25, -52},
-      {PPC::R24, -60},
-      {PPC::R23, -68},
-      {PPC::R22, -76},
-      {PPC::R21, -84},
-      {PPC::R20, -92},
-      {PPC::R19, -100},
-      {PPC::R18, -108},
-      {PPC::R17, -116},
-      {PPC::R16, -124},
-      {PPC::R15, -132},
-      {PPC::R14, -140},
-
       {PPC::X31, -8},
       {PPC::X30, -16},
       {PPC::X29, -24},
@@ -268,24 +245,6 @@ public:
       {PPC::X15, -136},
       {PPC::X14, -144},
 
-      // CR save area offset.
-      // FIXME SVR4: Disable CR save area for now.
-//      {PPC::CR2, -4},
-//      {PPC::CR3, -4},
-//      {PPC::CR4, -4},
-//      {PPC::CR2LT, -4},
-//      {PPC::CR2GT, -4},
-//      {PPC::CR2EQ, -4},
-//      {PPC::CR2UN, -4},
-//      {PPC::CR3LT, -4},
-//      {PPC::CR3GT, -4},
-//      {PPC::CR3EQ, -4},
-//      {PPC::CR3UN, -4},
-//      {PPC::CR4LT, -4},
-//      {PPC::CR4GT, -4},
-//      {PPC::CR4EQ, -4},
-//      {PPC::CR4UN, -4},
-
       // VRSAVE save area offset.
       {PPC::VRSAVE, -4},
 
diff --git a/lib/Target/PowerPC/PPCISelDAGToDAG.cpp b/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
index a00f686..254fea6 100644
--- a/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
+++ b/lib/Target/PowerPC/PPCISelDAGToDAG.cpp
@@ -53,7 +53,9 @@ namespace {
       GlobalBaseReg = 0;
       SelectionDAGISel::runOnMachineFunction(MF);
 
-      InsertVRSaveCode(MF);
+      if (!PPCSubTarget.isSVR4ABI())
+        InsertVRSaveCode(MF);
+
       return true;
     }
 
@@ -621,6 +623,88 @@ static unsigned getCRIdxForSetCC(ISD::CondCode CC, bool &Invert, int &Other) {
   }
 }
 
+// getVCmpInst: return the vector compare instruction for the specified
+// vector type and condition code. Since this is for altivec specific code,
+// only support the altivec types (v16i8, v8i16, v4i32, and v4f32).
+static unsigned int getVCmpInst(MVT::SimpleValueType VecVT, ISD::CondCode CC) {
+  switch (CC) {
+    case ISD::SETEQ:
+    case ISD::SETUEQ:
+    case ISD::SETNE:
+    case ISD::SETUNE:
+      if (VecVT == MVT::v16i8)
+        return PPC::VCMPEQUB;
+      else if (VecVT == MVT::v8i16)
+        return PPC::VCMPEQUH;
+      else if (VecVT == MVT::v4i32)
+        return PPC::VCMPEQUW;
+      // v4f32 != v4f32 could be translate to unordered not equal
+      else if (VecVT == MVT::v4f32)
+        return PPC::VCMPEQFP;
+      break;
+    case ISD::SETLT:
+    case ISD::SETGT:
+    case ISD::SETLE:
+    case ISD::SETGE:
+      if (VecVT == MVT::v16i8)
+        return PPC::VCMPGTSB;
+      else if (VecVT == MVT::v8i16)
+        return PPC::VCMPGTSH;
+      else if (VecVT == MVT::v4i32)
+        return PPC::VCMPGTSW;
+      else if (VecVT == MVT::v4f32)
+        return PPC::VCMPGTFP;
+      break;
+    case ISD::SETULT:
+    case ISD::SETUGT:
+    case ISD::SETUGE:
+    case ISD::SETULE:
+      if (VecVT == MVT::v16i8)
+        return PPC::VCMPGTUB;
+      else if (VecVT == MVT::v8i16)
+        return PPC::VCMPGTUH;
+      else if (VecVT == MVT::v4i32)
+        return PPC::VCMPGTUW;
+      break;
+    case ISD::SETOEQ:
+      if (VecVT == MVT::v4f32)
+        return PPC::VCMPEQFP;
+      break;
+    case ISD::SETOLT:
+    case ISD::SETOGT:
+    case ISD::SETOLE:
+      if (VecVT == MVT::v4f32)
+        return PPC::VCMPGTFP;
+      break;
+    case ISD::SETOGE:
+      if (VecVT == MVT::v4f32)
+        return PPC::VCMPGEFP;
+      break;
+    default:
+      break;
+  }
+  llvm_unreachable("Invalid integer vector compare condition");
+}
+
+// getVCmpEQInst: return the equal compare instruction for the specified vector
+// type. Since this is for altivec specific code, only support the altivec
+// types (v16i8, v8i16, v4i32, and v4f32).
+static unsigned int getVCmpEQInst(MVT::SimpleValueType VecVT) {
+  switch (VecVT) {
+    case MVT::v16i8:
+      return PPC::VCMPEQUB;
+    case MVT::v8i16:
+      return PPC::VCMPEQUH;
+    case MVT::v4i32:
+      return PPC::VCMPEQUW;
+    case MVT::v4f32:
+      return PPC::VCMPEQFP;
+    default:
+      llvm_unreachable("Invalid integer vector compare condition");
+  }
+}
+
+
 SDNode *PPCDAGToDAGISel::SelectSETCC(SDNode *N) {
   DebugLoc dl = N->getDebugLoc();
   unsigned Imm;
@@ -701,10 +785,67 @@ SDNode *PPCDAGToDAGISel::SelectSETCC(SDNode *N) {
     }
   }
 
+  SDValue LHS = N->getOperand(0);
+  SDValue RHS = N->getOperand(1);
+
+  // Altivec Vector compare instructions do not set any CR register by default and
+  // vector compare operations return the same type as the operands.
+  if (LHS.getValueType().isVector()) {
+    EVT VecVT = LHS.getValueType();
+    MVT::SimpleValueType VT = VecVT.getSimpleVT().SimpleTy;
+    unsigned int VCmpInst = getVCmpInst(VT, CC);
+
+    switch (CC) {
+      case ISD::SETEQ:
+      case ISD::SETOEQ:
+      case ISD::SETUEQ:
+        return CurDAG->SelectNodeTo(N, VCmpInst, VecVT, LHS, RHS);
+      case ISD::SETNE:
+      case ISD::SETONE:
+      case ISD::SETUNE: {
+        SDValue VCmp(CurDAG->getMachineNode(VCmpInst, dl, VecVT, LHS, RHS), 0);
+        return CurDAG->SelectNodeTo(N, PPC::VNOR, VecVT, VCmp, VCmp);
+      } 
+      case ISD::SETLT:
+      case ISD::SETOLT:
+      case ISD::SETULT:
+        return CurDAG->SelectNodeTo(N, VCmpInst, VecVT, RHS, LHS);
+      case ISD::SETGT:
+      case ISD::SETOGT:
+      case ISD::SETUGT:
+        return CurDAG->SelectNodeTo(N, VCmpInst, VecVT, LHS, RHS);
+      case ISD::SETGE:
+      case ISD::SETOGE:
+      case ISD::SETUGE: {
+        // Small optimization: Altivec provides a 'Vector Compare Greater Than
+        // or Equal To' instruction (vcmpgefp), so in this case there is no
+        // need for extra logic for the equal compare.
+        if (VecVT.getSimpleVT().isFloatingPoint()) {
+          return CurDAG->SelectNodeTo(N, VCmpInst, VecVT, LHS, RHS);
+        } else {
+          SDValue VCmpGT(CurDAG->getMachineNode(VCmpInst, dl, VecVT, LHS, RHS), 0);
+          unsigned int VCmpEQInst = getVCmpEQInst(VT);
+          SDValue VCmpEQ(CurDAG->getMachineNode(VCmpEQInst, dl, VecVT, LHS, RHS), 0);
+          return CurDAG->SelectNodeTo(N, PPC::VOR, VecVT, VCmpGT, VCmpEQ);
+        }
+      }
+      case ISD::SETLE:
+      case ISD::SETOLE:
+      case ISD::SETULE: {
+        SDValue VCmpLE(CurDAG->getMachineNode(VCmpInst, dl, VecVT, RHS, LHS), 0);
+        unsigned int VCmpEQInst = getVCmpEQInst(VT);
+        SDValue VCmpEQ(CurDAG->getMachineNode(VCmpEQInst, dl, VecVT, LHS, RHS), 0);
+        return CurDAG->SelectNodeTo(N, PPC::VOR, VecVT, VCmpLE, VCmpEQ);
+      }
+      default:
+        llvm_unreachable("Invalid vector compare type: should be expanded by legalize");
+    }
+  }
+
   bool Inv;
   int OtherCondIdx;
   unsigned Idx = getCRIdxForSetCC(CC, Inv, OtherCondIdx);
-  SDValue CCReg = SelectCC(N->getOperand(0), N->getOperand(1), CC, dl);
+  SDValue CCReg = SelectCC(LHS, RHS, CC, dl);
   SDValue IntCR;
 
   // Force the ccreg into CR7.
@@ -717,7 +858,7 @@ SDNode *PPCDAGToDAGISel::SelectSETCC(SDNode *N) {
   if (PPCSubTarget.hasMFOCRF() && OtherCondIdx == -1)
     IntCR = SDValue(CurDAG->getMachineNode(PPC::MFOCRF, dl, MVT::i32, CR7Reg,
                                            CCReg), 0);
- else
+  else
     IntCR = SDValue(CurDAG->getMachineNode(PPC::MFCRpseud, dl, MVT::i32,
                                            CR7Reg, CCReg), 0);
 
@@ -975,6 +1116,7 @@ SDNode *PPCDAGToDAGISel::Select(SDNode *N) {
 
   case ISD::AND: {
     unsigned Imm, Imm2, SH, MB, ME;
+    uint64_t Imm64;
 
     // If this is an and of a value rotated between 0 and 31 bits and then and'd
     // with a mask, emit rlwinm
@@ -993,6 +1135,14 @@ SDNode *PPCDAGToDAGISel::Select(SDNode *N) {
       SDValue Ops[] = { Val, getI32Imm(0), getI32Imm(MB), getI32Imm(ME) };
       return CurDAG->SelectNodeTo(N, PPC::RLWINM, MVT::i32, Ops, 4);
     }
+    // If this is a 64-bit zero-extension mask, emit rldicl.
+    if (isInt64Immediate(N->getOperand(1).getNode(), Imm64) &&
+        isMask_64(Imm64)) {
+      SDValue Val = N->getOperand(0);
+      MB = 64 - CountTrailingOnes_64(Imm64);
+      SDValue Ops[] = { Val, getI32Imm(0), getI32Imm(MB) };
+      return CurDAG->SelectNodeTo(N, PPC::RLDICL, MVT::i64, Ops, 3);
+    }
     // AND X, 0 -> 0, not "rlwinm 32".
     if (isInt32Immediate(N->getOperand(1), Imm) && (Imm == 0)) {
       ReplaceUses(SDValue(N, 0), N->getOperand(1));
diff --git a/lib/Target/PowerPC/PPCISelLowering.cpp b/lib/Target/PowerPC/PPCISelLowering.cpp
index 61d44c5..adf78d5 100644
--- a/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -361,6 +361,22 @@ PPCTargetLowering::PPCTargetLowering(PPCTargetMachine &TM)
       setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Expand);
       setOperationAction(ISD::CTTZ, VT, Expand);
       setOperationAction(ISD::CTTZ_ZERO_UNDEF, VT, Expand);
+      setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand);
+
+      for (unsigned j = (unsigned)MVT::FIRST_VECTOR_VALUETYPE;
+           j <= (unsigned)MVT::LAST_VECTOR_VALUETYPE; ++j) {
+        MVT::SimpleValueType InnerVT = (MVT::SimpleValueType)j;
+        setTruncStoreAction(VT, InnerVT, Expand);
+      }
+      setLoadExtAction(ISD::SEXTLOAD, VT, Expand);
+      setLoadExtAction(ISD::ZEXTLOAD, VT, Expand);
+      setLoadExtAction(ISD::EXTLOAD, VT, Expand);
+    }
+
+    for (unsigned i = (unsigned)MVT::FIRST_FP_VECTOR_VALUETYPE;
+         i <= (unsigned)MVT::LAST_FP_VECTOR_VALUETYPE; ++i) {
+      MVT::SimpleValueType VT = (MVT::SimpleValueType)i;
+      setOperationAction(ISD::FSQRT, VT, Expand);
     }
 
     // We can custom expand all VECTOR_SHUFFLEs to VPERM, others we can handle
@@ -373,6 +389,10 @@ PPCTargetLowering::PPCTargetLowering(PPCTargetMachine &TM)
     setOperationAction(ISD::LOAD  , MVT::v4i32, Legal);
     setOperationAction(ISD::SELECT, MVT::v4i32, Expand);
     setOperationAction(ISD::STORE , MVT::v4i32, Legal);
+    setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Legal);
+    setOperationAction(ISD::FP_TO_UINT, MVT::v4i32, Legal);
+    setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Legal);
+    setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Legal);
 
     addRegisterClass(MVT::v4f32, &PPC::VRRCRegClass);
     addRegisterClass(MVT::v4i32, &PPC::VRRCRegClass);
@@ -392,6 +412,14 @@ PPCTargetLowering::PPCTargetLowering(PPCTargetMachine &TM)
     setOperationAction(ISD::BUILD_VECTOR, MVT::v8i16, Custom);
     setOperationAction(ISD::BUILD_VECTOR, MVT::v4i32, Custom);
     setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom);
+
+    // Altivec does not contain unordered floating-point compare instructions
+    setCondCodeAction(ISD::SETUO, MVT::v4f32, Expand);
+    setCondCodeAction(ISD::SETUEQ, MVT::v4f32, Expand);
+    setCondCodeAction(ISD::SETUGT, MVT::v4f32, Expand);
+    setCondCodeAction(ISD::SETUGE, MVT::v4f32, Expand);
+    setCondCodeAction(ISD::SETULT, MVT::v4f32, Expand);
+    setCondCodeAction(ISD::SETULE, MVT::v4f32, Expand);
   }
 
   if (Subtarget->has64BitSupport()) {
@@ -449,6 +477,21 @@ PPCTargetLowering::PPCTargetLowering(PPCTargetMachine &TM)
   setSchedulingPreference(Sched::Hybrid);
 
   computeRegisterProperties();
+
+  // The Freescale cores does better with aggressive inlining of memcpy and
+  // friends. Gcc uses same threshold of 128 bytes (= 32 word stores).
+  if (Subtarget->getDarwinDirective() == PPC::DIR_E500mc ||
+      Subtarget->getDarwinDirective() == PPC::DIR_E5500) {
+    maxStoresPerMemset = 32;
+    maxStoresPerMemsetOptSize = 16;
+    maxStoresPerMemcpy = 32;
+    maxStoresPerMemcpyOptSize = 8;
+    maxStoresPerMemmove = 32;
+    maxStoresPerMemmoveOptSize = 8;
+
+    setPrefFunctionAlignment(4);
+    benefitFromCodePlacementOpt = true;
+  }
 }
 
 /// getByValTypeAlignment - Return the desired alignment for ByVal aggregate
@@ -517,11 +560,15 @@ const char *PPCTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case PPCISD::FADDRTZ:         return "PPCISD::FADDRTZ";
   case PPCISD::MTFSF:           return "PPCISD::MTFSF";
   case PPCISD::TC_RETURN:       return "PPCISD::TC_RETURN";
+  case PPCISD::CR6SET:          return "PPCISD::CR6SET";
+  case PPCISD::CR6UNSET:        return "PPCISD::CR6UNSET";
   }
 }
 
 EVT PPCTargetLowering::getSetCCResultType(EVT VT) const {
-  return MVT::i32;
+  if (!VT.isVector())
+    return MVT::i32;
+  return VT.changeVectorElementTypeToInteger();
 }
 
 //===----------------------------------------------------------------------===//
@@ -811,14 +858,13 @@ SDValue PPC::get_VSPLTI_elt(SDNode *N, unsigned ByteSize, SelectionDAG &DAG) {
   }
 
   // Properly sign extend the value.
-  int ShAmt = (4-ByteSize)*8;
-  int MaskVal = ((int)Value << ShAmt) >> ShAmt;
+  int MaskVal = SignExtend32(Value, ByteSize * 8);
 
   // If this is zero, don't match, zero matches ISD::isBuildVectorAllZeros.
   if (MaskVal == 0) return SDValue();
 
   // Finally, if this value fits in a 5 bit sext field, return it
-  if (((MaskVal << (32-5)) >> (32-5)) == MaskVal)
+  if (SignExtend32<5>(MaskVal) == MaskVal)
     return DAG.getTargetConstant(MaskVal, MVT::i32);
   return SDValue();
 }
@@ -1204,6 +1250,14 @@ SDValue PPCTargetLowering::LowerConstantPool(SDValue Op,
   ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op);
   const Constant *C = CP->getConstVal();
 
+  // 64-bit SVR4 ABI code is always position-independent.
+  // The actual address of the GlobalValue is stored in the TOC.
+  if (PPCSubTarget.isSVR4ABI() && PPCSubTarget.isPPC64()) {
+    SDValue GA = DAG.getTargetConstantPool(C, PtrVT, CP->getAlignment(), 0);
+    return DAG.getNode(PPCISD::TOC_ENTRY, CP->getDebugLoc(), MVT::i64, GA,
+                       DAG.getRegister(PPC::X2, MVT::i64));
+  }
+
   unsigned MOHiFlag, MOLoFlag;
   bool isPIC = GetLabelAccessInfo(DAG.getTarget(), MOHiFlag, MOLoFlag);
   SDValue CPIHi =
@@ -1217,6 +1271,14 @@ SDValue PPCTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
   EVT PtrVT = Op.getValueType();
   JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);
 
+  // 64-bit SVR4 ABI code is always position-independent.
+  // The actual address of the GlobalValue is stored in the TOC.
+  if (PPCSubTarget.isSVR4ABI() && PPCSubTarget.isPPC64()) {
+    SDValue GA = DAG.getTargetJumpTable(JT->getIndex(), PtrVT);
+    return DAG.getNode(PPCISD::TOC_ENTRY, JT->getDebugLoc(), MVT::i64, GA,
+                       DAG.getRegister(PPC::X2, MVT::i64));
+  }
+
   unsigned MOHiFlag, MOLoFlag;
   bool isPIC = GetLabelAccessInfo(DAG.getTarget(), MOHiFlag, MOLoFlag);
   SDValue JTIHi = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, MOHiFlag);
@@ -1232,8 +1294,8 @@ SDValue PPCTargetLowering::LowerBlockAddress(SDValue Op,
 
   unsigned MOHiFlag, MOLoFlag;
   bool isPIC = GetLabelAccessInfo(DAG.getTarget(), MOHiFlag, MOLoFlag);
-  SDValue TgtBAHi = DAG.getBlockAddress(BA, PtrVT, /*isTarget=*/true, MOHiFlag);
-  SDValue TgtBALo = DAG.getBlockAddress(BA, PtrVT, /*isTarget=*/true, MOLoFlag);
+  SDValue TgtBAHi = DAG.getTargetBlockAddress(BA, PtrVT, 0, MOHiFlag);
+  SDValue TgtBALo = DAG.getTargetBlockAddress(BA, PtrVT, 0, MOLoFlag);
   return LowerLabelRef(TgtBAHi, TgtBALo, isPIC, DAG);
 }
 
@@ -1441,7 +1503,7 @@ SDValue PPCTargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG,
                               MachinePointerInfo(),
                               MVT::i32, false, false, 0);
 
-  return DAG.getLoad(VT, dl, InChain, Result, MachinePointerInfo(), 
+  return DAG.getLoad(VT, dl, InChain, Result, MachinePointerInfo(),
                      false, false, false, 0);
 }
 
@@ -1461,7 +1523,7 @@ SDValue PPCTargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
   EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
   bool isPPC64 = (PtrVT == MVT::i64);
   Type *IntPtrTy =
-    DAG.getTargetLoweringInfo().getTargetData()->getIntPtrType(
+    DAG.getTargetLoweringInfo().getDataLayout()->getIntPtrType(
                                                              *DAG.getContext());
 
   TargetLowering::ArgListTy Args;
@@ -1684,9 +1746,13 @@ PPCTargetLowering::LowerFormalArguments(SDValue Chain,
                                         DebugLoc dl, SelectionDAG &DAG,
                                         SmallVectorImpl<SDValue> &InVals)
                                           const {
-  if (PPCSubTarget.isSVR4ABI() && !PPCSubTarget.isPPC64()) {
-    return LowerFormalArguments_SVR4(Chain, CallConv, isVarArg, Ins,
-                                     dl, DAG, InVals);
+  if (PPCSubTarget.isSVR4ABI()) {
+    if (PPCSubTarget.isPPC64())
+      return LowerFormalArguments_64SVR4(Chain, CallConv, isVarArg, Ins,
+                                         dl, DAG, InVals);
+    else
+      return LowerFormalArguments_32SVR4(Chain, CallConv, isVarArg, Ins,
+                                         dl, DAG, InVals);
   } else {
     return LowerFormalArguments_Darwin(Chain, CallConv, isVarArg, Ins,
                                        dl, DAG, InVals);
@@ -1694,7 +1760,7 @@ PPCTargetLowering::LowerFormalArguments(SDValue Chain,
 }
 
 SDValue
-PPCTargetLowering::LowerFormalArguments_SVR4(
+PPCTargetLowering::LowerFormalArguments_32SVR4(
                                       SDValue Chain,
                                       CallingConv::ID CallConv, bool isVarArg,
                                       const SmallVectorImpl<ISD::InputArg>
@@ -1911,6 +1977,334 @@ PPCTargetLowering::LowerFormalArguments_SVR4(
   return Chain;
 }
 
+// PPC64 passes i8, i16, and i32 values in i64 registers. Promote
+// value to MVT::i64 and then truncate to the correct register size.
+SDValue
+PPCTargetLowering::extendArgForPPC64(ISD::ArgFlagsTy Flags, EVT ObjectVT,
+                                     SelectionDAG &DAG, SDValue ArgVal,
+                                     DebugLoc dl) const {
+  if (Flags.isSExt())
+    ArgVal = DAG.getNode(ISD::AssertSext, dl, MVT::i64, ArgVal,
+                         DAG.getValueType(ObjectVT));
+  else if (Flags.isZExt())
+    ArgVal = DAG.getNode(ISD::AssertZext, dl, MVT::i64, ArgVal,
+                         DAG.getValueType(ObjectVT));
+  
+  return DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, ArgVal);
+}
+
+// Set the size that is at least reserved in caller of this function.  Tail
+// call optimized functions' reserved stack space needs to be aligned so that
+// taking the difference between two stack areas will result in an aligned
+// stack.
+void
+PPCTargetLowering::setMinReservedArea(MachineFunction &MF, SelectionDAG &DAG,
+                                      unsigned nAltivecParamsAtEnd,
+                                      unsigned MinReservedArea,
+                                      bool isPPC64) const {
+  PPCFunctionInfo *FI = MF.getInfo<PPCFunctionInfo>();
+  // Add the Altivec parameters at the end, if needed.
+  if (nAltivecParamsAtEnd) {
+    MinReservedArea = ((MinReservedArea+15)/16)*16;
+    MinReservedArea += 16*nAltivecParamsAtEnd;
+  }
+  MinReservedArea =
+    std::max(MinReservedArea,
+             PPCFrameLowering::getMinCallFrameSize(isPPC64, true));
+  unsigned TargetAlign
+    = DAG.getMachineFunction().getTarget().getFrameLowering()->
+        getStackAlignment();
+  unsigned AlignMask = TargetAlign-1;
+  MinReservedArea = (MinReservedArea + AlignMask) & ~AlignMask;
+  FI->setMinReservedArea(MinReservedArea);
+}
+
+SDValue
+PPCTargetLowering::LowerFormalArguments_64SVR4(
+                                      SDValue Chain,
+                                      CallingConv::ID CallConv, bool isVarArg,
+                                      const SmallVectorImpl<ISD::InputArg>
+                                        &Ins,
+                                      DebugLoc dl, SelectionDAG &DAG,
+                                      SmallVectorImpl<SDValue> &InVals) const {
+  // TODO: add description of PPC stack frame format, or at least some docs.
+  //
+  MachineFunction &MF = DAG.getMachineFunction();
+  MachineFrameInfo *MFI = MF.getFrameInfo();
+  PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
+
+  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
+  // Potential tail calls could cause overwriting of argument stack slots.
+  bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt &&
+                       (CallConv == CallingConv::Fast));
+  unsigned PtrByteSize = 8;
+
+  unsigned ArgOffset = PPCFrameLowering::getLinkageSize(true, true);
+  // Area that is at least reserved in caller of this function.
+  unsigned MinReservedArea = ArgOffset;
+
+  static const uint16_t GPR[] = {
+    PPC::X3, PPC::X4, PPC::X5, PPC::X6,
+    PPC::X7, PPC::X8, PPC::X9, PPC::X10,
+  };
+
+  static const uint16_t *FPR = GetFPR();
+
+  static const uint16_t VR[] = {
+    PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8,
+    PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13
+  };
+
+  const unsigned Num_GPR_Regs = array_lengthof(GPR);
+  const unsigned Num_FPR_Regs = 13;
+  const unsigned Num_VR_Regs  = array_lengthof(VR);
+
+  unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0;
+
+  // Add DAG nodes to load the arguments or copy them out of registers.  On
+  // entry to a function on PPC, the arguments start after the linkage area,
+  // although the first ones are often in registers.
+
+  SmallVector<SDValue, 8> MemOps;
+  unsigned nAltivecParamsAtEnd = 0;
+  Function::const_arg_iterator FuncArg = MF.getFunction()->arg_begin();
+  for (unsigned ArgNo = 0, e = Ins.size(); ArgNo != e; ++ArgNo, ++FuncArg) {
+    SDValue ArgVal;
+    bool needsLoad = false;
+    EVT ObjectVT = Ins[ArgNo].VT;
+    unsigned ObjSize = ObjectVT.getSizeInBits()/8;
+    unsigned ArgSize = ObjSize;
+    ISD::ArgFlagsTy Flags = Ins[ArgNo].Flags;
+
+    unsigned CurArgOffset = ArgOffset;
+
+    // Varargs or 64 bit Altivec parameters are padded to a 16 byte boundary.
+    if (ObjectVT==MVT::v4f32 || ObjectVT==MVT::v4i32 ||
+        ObjectVT==MVT::v8i16 || ObjectVT==MVT::v16i8) {
+      if (isVarArg) {
+        MinReservedArea = ((MinReservedArea+15)/16)*16;
+        MinReservedArea += CalculateStackSlotSize(ObjectVT,
+                                                  Flags,
+                                                  PtrByteSize);
+      } else
+        nAltivecParamsAtEnd++;
+    } else
+      // Calculate min reserved area.
+      MinReservedArea += CalculateStackSlotSize(Ins[ArgNo].VT,
+                                                Flags,
+                                                PtrByteSize);
+
+    // FIXME the codegen can be much improved in some cases.
+    // We do not have to keep everything in memory.
+    if (Flags.isByVal()) {
+      // ObjSize is the true size, ArgSize rounded up to multiple of registers.
+      ObjSize = Flags.getByValSize();
+      ArgSize = ((ObjSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
+      // Empty aggregate parameters do not take up registers.  Examples:
+      //   struct { } a;
+      //   union  { } b;
+      //   int c[0];
+      // etc.  However, we have to provide a place-holder in InVals, so
+      // pretend we have an 8-byte item at the current address for that
+      // purpose.
+      if (!ObjSize) {
+        int FI = MFI->CreateFixedObject(PtrByteSize, ArgOffset, true);
+        SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
+        InVals.push_back(FIN);
+        continue;
+      }
+      // All aggregates smaller than 8 bytes must be passed right-justified.
+      if (ObjSize < PtrByteSize)
+        CurArgOffset = CurArgOffset + (PtrByteSize - ObjSize);
+      // The value of the object is its address.
+      int FI = MFI->CreateFixedObject(ObjSize, CurArgOffset, true);
+      SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
+      InVals.push_back(FIN);
+
+      if (ObjSize < 8) {
+        if (GPR_idx != Num_GPR_Regs) {
+          unsigned VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass);
+          SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT);
+          SDValue Store;
+
+          if (ObjSize==1 || ObjSize==2 || ObjSize==4) {
+            EVT ObjType = (ObjSize == 1 ? MVT::i8 :
+                           (ObjSize == 2 ? MVT::i16 : MVT::i32));
+            Store = DAG.getTruncStore(Val.getValue(1), dl, Val, FIN,
+                                      MachinePointerInfo(FuncArg, CurArgOffset),
+                                      ObjType, false, false, 0);
+          } else {
+            // For sizes that don't fit a truncating store (3, 5, 6, 7),
+            // store the whole register as-is to the parameter save area
+            // slot.  The address of the parameter was already calculated
+            // above (InVals.push_back(FIN)) to be the right-justified
+            // offset within the slot.  For this store, we need a new
+            // frame index that points at the beginning of the slot.
+            int FI = MFI->CreateFixedObject(PtrByteSize, ArgOffset, true);
+            SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
+            Store = DAG.getStore(Val.getValue(1), dl, Val, FIN,
+                                 MachinePointerInfo(FuncArg, ArgOffset),
+                                 false, false, 0);
+          }
+
+          MemOps.push_back(Store);
+          ++GPR_idx;
+        }
+        // Whether we copied from a register or not, advance the offset
+        // into the parameter save area by a full doubleword.
+        ArgOffset += PtrByteSize;
+        continue;
+      }
+
+      for (unsigned j = 0; j < ArgSize; j += PtrByteSize) {
+        // Store whatever pieces of the object are in registers
+        // to memory.  ArgOffset will be the address of the beginning
+        // of the object.
+        if (GPR_idx != Num_GPR_Regs) {
+          unsigned VReg;
+          VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass);
+          int FI = MFI->CreateFixedObject(PtrByteSize, ArgOffset, true);
+          SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
+          SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT);
+          SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN,
+                                       MachinePointerInfo(FuncArg, ArgOffset),
+                                       false, false, 0);
+          MemOps.push_back(Store);
+          ++GPR_idx;
+          ArgOffset += PtrByteSize;
+        } else {
+          ArgOffset += ArgSize - j;
+          break;
+        }
+      }
+      continue;
+    }
+
+    switch (ObjectVT.getSimpleVT().SimpleTy) {
+    default: llvm_unreachable("Unhandled argument type!");
+    case MVT::i32:
+    case MVT::i64:
+      if (GPR_idx != Num_GPR_Regs) {
+        unsigned VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass);
+        ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64);
+
+        if (ObjectVT == MVT::i32)
+          // PPC64 passes i8, i16, and i32 values in i64 registers. Promote
+          // value to MVT::i64 and then truncate to the correct register size.
+          ArgVal = extendArgForPPC64(Flags, ObjectVT, DAG, ArgVal, dl);
+
+        ++GPR_idx;
+      } else {
+        needsLoad = true;
+        ArgSize = PtrByteSize;
+      }
+      ArgOffset += 8;
+      break;
+
+    case MVT::f32:
+    case MVT::f64:
+      // Every 8 bytes of argument space consumes one of the GPRs available for
+      // argument passing.
+      if (GPR_idx != Num_GPR_Regs) {
+        ++GPR_idx;
+      }
+      if (FPR_idx != Num_FPR_Regs) {
+        unsigned VReg;
+
+        if (ObjectVT == MVT::f32)
+          VReg = MF.addLiveIn(FPR[FPR_idx], &PPC::F4RCRegClass);
+        else
+          VReg = MF.addLiveIn(FPR[FPR_idx], &PPC::F8RCRegClass);
+
+        ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT);
+        ++FPR_idx;
+      } else {
+        needsLoad = true;
+        ArgSize = PtrByteSize;
+      }
+
+      ArgOffset += 8;
+      break;
+    case MVT::v4f32:
+    case MVT::v4i32:
+    case MVT::v8i16:
+    case MVT::v16i8:
+      // Note that vector arguments in registers don't reserve stack space,
+      // except in varargs functions.
+      if (VR_idx != Num_VR_Regs) {
+        unsigned VReg = MF.addLiveIn(VR[VR_idx], &PPC::VRRCRegClass);
+        ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, ObjectVT);
+        if (isVarArg) {
+          while ((ArgOffset % 16) != 0) {
+            ArgOffset += PtrByteSize;
+            if (GPR_idx != Num_GPR_Regs)
+              GPR_idx++;
+          }
+          ArgOffset += 16;
+          GPR_idx = std::min(GPR_idx+4, Num_GPR_Regs); // FIXME correct for ppc64?
+        }
+        ++VR_idx;
+      } else {
+        // Vectors are aligned.
+        ArgOffset = ((ArgOffset+15)/16)*16;
+        CurArgOffset = ArgOffset;
+        ArgOffset += 16;
+        needsLoad = true;
+      }
+      break;
+    }
+
+    // We need to load the argument to a virtual register if we determined
+    // above that we ran out of physical registers of the appropriate type.
+    if (needsLoad) {
+      int FI = MFI->CreateFixedObject(ObjSize,
+                                      CurArgOffset + (ArgSize - ObjSize),
+                                      isImmutable);
+      SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
+      ArgVal = DAG.getLoad(ObjectVT, dl, Chain, FIN, MachinePointerInfo(),
+                           false, false, false, 0);
+    }
+
+    InVals.push_back(ArgVal);
+  }
+
+  // Set the size that is at least reserved in caller of this function.  Tail
+  // call optimized functions' reserved stack space needs to be aligned so that
+  // taking the difference between two stack areas will result in an aligned
+  // stack.
+  setMinReservedArea(MF, DAG, nAltivecParamsAtEnd, MinReservedArea, true);
+
+  // If the function takes variable number of arguments, make a frame index for
+  // the start of the first vararg value... for expansion of llvm.va_start.
+  if (isVarArg) {
+    int Depth = ArgOffset;
+
+    FuncInfo->setVarArgsFrameIndex(
+      MFI->CreateFixedObject(PtrByteSize, Depth, true));
+    SDValue FIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
+
+    // If this function is vararg, store any remaining integer argument regs
+    // to their spots on the stack so that they may be loaded by deferencing the
+    // result of va_next.
+    for (; GPR_idx != Num_GPR_Regs; ++GPR_idx) {
+      unsigned VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass);
+      SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT);
+      SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN,
+                                   MachinePointerInfo(), false, false, 0);
+      MemOps.push_back(Store);
+      // Increment the address by four for the next argument to store
+      SDValue PtrOff = DAG.getConstant(PtrByteSize, PtrVT);
+      FIN = DAG.getNode(ISD::ADD, dl, PtrOff.getValueType(), FIN, PtrOff);
+    }
+  }
+
+  if (!MemOps.empty())
+    Chain = DAG.getNode(ISD::TokenFactor, dl,
+                        MVT::Other, &MemOps[0], MemOps.size());
+
+  return Chain;
+}
+
 SDValue
 PPCTargetLowering::LowerFormalArguments_Darwin(
                                       SDValue Chain,
@@ -1987,10 +2381,12 @@ PPCTargetLowering::LowerFormalArguments_Darwin(
       default: llvm_unreachable("Unhandled argument type!");
       case MVT::i32:
       case MVT::f32:
-        VecArgOffset += isPPC64 ? 8 : 4;
+        VecArgOffset += 4;
         break;
       case MVT::i64:  // PPC64
       case MVT::f64:
+        // FIXME: We are guaranteed to be !isPPC64 at this point.
+        // Does MVT::i64 apply?
         VecArgOffset += 8;
         break;
       case MVT::v4f32:
@@ -2013,7 +2409,8 @@ PPCTargetLowering::LowerFormalArguments_Darwin(
 
   SmallVector<SDValue, 8> MemOps;
   unsigned nAltivecParamsAtEnd = 0;
-  for (unsigned ArgNo = 0, e = Ins.size(); ArgNo != e; ++ArgNo) {
+  Function::const_arg_iterator FuncArg = MF.getFunction()->arg_begin();
+  for (unsigned ArgNo = 0, e = Ins.size(); ArgNo != e; ++ArgNo, ++FuncArg) {
     SDValue ArgVal;
     bool needsLoad = false;
     EVT ObjectVT = Ins[ArgNo].VT;
@@ -2061,10 +2458,11 @@ PPCTargetLowering::LowerFormalArguments_Darwin(
           else
             VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::GPRCRegClass);
           SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT);
+          EVT ObjType = ObjSize == 1 ? MVT::i8 : MVT::i16;
           SDValue Store = DAG.getTruncStore(Val.getValue(1), dl, Val, FIN,
-                                            MachinePointerInfo(),
-                                            ObjSize==1 ? MVT::i8 : MVT::i16,
-                                            false, false, 0);
+                                            MachinePointerInfo(FuncArg,
+                                              CurArgOffset),
+                                            ObjType, false, false, 0);
           MemOps.push_back(Store);
           ++GPR_idx;
         }
@@ -2075,8 +2473,8 @@ PPCTargetLowering::LowerFormalArguments_Darwin(
       }
       for (unsigned j = 0; j < ArgSize; j += PtrByteSize) {
         // Store whatever pieces of the object are in registers
-        // to memory.  ArgVal will be address of the beginning of
-        // the object.
+        // to memory.  ArgOffset will be the address of the beginning
+        // of the object.
         if (GPR_idx != Num_GPR_Regs) {
           unsigned VReg;
           if (isPPC64)
@@ -2087,7 +2485,7 @@ PPCTargetLowering::LowerFormalArguments_Darwin(
           SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
           SDValue Val = DAG.getCopyFromReg(Chain, dl, VReg, PtrVT);
           SDValue Store = DAG.getStore(Val.getValue(1), dl, Val, FIN,
-                                       MachinePointerInfo(),
+                                       MachinePointerInfo(FuncArg, ArgOffset),
                                        false, false, 0);
           MemOps.push_back(Store);
           ++GPR_idx;
@@ -2122,18 +2520,10 @@ PPCTargetLowering::LowerFormalArguments_Darwin(
         unsigned VReg = MF.addLiveIn(GPR[GPR_idx], &PPC::G8RCRegClass);
         ArgVal = DAG.getCopyFromReg(Chain, dl, VReg, MVT::i64);
 
-        if (ObjectVT == MVT::i32) {
+        if (ObjectVT == MVT::i32)
           // PPC64 passes i8, i16, and i32 values in i64 registers. Promote
           // value to MVT::i64 and then truncate to the correct register size.
-          if (Flags.isSExt())
-            ArgVal = DAG.getNode(ISD::AssertSext, dl, MVT::i64, ArgVal,
-                                 DAG.getValueType(ObjectVT));
-          else if (Flags.isZExt())
-            ArgVal = DAG.getNode(ISD::AssertZext, dl, MVT::i64, ArgVal,
-                                 DAG.getValueType(ObjectVT));
-
-          ArgVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, ArgVal);
-        }
+          ArgVal = extendArgForPPC64(Flags, ObjectVT, DAG, ArgVal, dl);
 
         ++GPR_idx;
       } else {
@@ -2220,23 +2610,10 @@ PPCTargetLowering::LowerFormalArguments_Darwin(
   }
 
   // Set the size that is at least reserved in caller of this function.  Tail
-  // call optimized function's reserved stack space needs to be aligned so that
+  // call optimized functions' reserved stack space needs to be aligned so that
   // taking the difference between two stack areas will result in an aligned
   // stack.
-  PPCFunctionInfo *FI = MF.getInfo<PPCFunctionInfo>();
-  // Add the Altivec parameters at the end, if needed.
-  if (nAltivecParamsAtEnd) {
-    MinReservedArea = ((MinReservedArea+15)/16)*16;
-    MinReservedArea += 16*nAltivecParamsAtEnd;
-  }
-  MinReservedArea =
-    std::max(MinReservedArea,
-             PPCFrameLowering::getMinCallFrameSize(isPPC64, true));
-  unsigned TargetAlign = DAG.getMachineFunction().getTarget().getFrameLowering()->
-    getStackAlignment();
-  unsigned AlignMask = TargetAlign-1;
-  MinReservedArea = (MinReservedArea + AlignMask) & ~AlignMask;
-  FI->setMinReservedArea(MinReservedArea);
+  setMinReservedArea(MF, DAG, nAltivecParamsAtEnd, MinReservedArea, isPPC64);
 
   // If the function takes variable number of arguments, make a frame index for
   // the start of the first vararg value... for expansion of llvm.va_start.
@@ -2276,8 +2653,8 @@ PPCTargetLowering::LowerFormalArguments_Darwin(
   return Chain;
 }
 
-/// CalculateParameterAndLinkageAreaSize - Get the size of the paramter plus
-/// linkage area for the Darwin ABI.
+/// CalculateParameterAndLinkageAreaSize - Get the size of the parameter plus
+/// linkage area for the Darwin ABI, or the 64-bit SVR4 ABI.
 static unsigned
 CalculateParameterAndLinkageAreaSize(SelectionDAG &DAG,
                                      bool isPPC64,
@@ -2408,7 +2785,7 @@ static SDNode *isBLACompatibleAddress(SDValue Op, SelectionDAG &DAG) {
 
   int Addr = C->getZExtValue();
   if ((Addr & 3) != 0 ||  // Low 2 bits are implicitly zero.
-      (Addr << 6 >> 6) != Addr)
+      SignExtend32<26>(Addr) != Addr)
     return 0;  // Top 6 bits have to be sext of immediate.
 
   return DAG.getConstant((int)C->getZExtValue() >> 2,
@@ -2686,7 +3063,7 @@ unsigned PrepareCall(SelectionDAG &DAG, SDValue &Callee, SDValue &InFlag,
       // Thus for a call through a function pointer, the following actions need
       // to be performed:
       //   1. Save the TOC of the caller in the TOC save area of its stack
-      //      frame (this is done in LowerCall_Darwin()).
+      //      frame (this is done in LowerCall_Darwin() or LowerCall_64SVR4()).
       //   2. Load the address of the function entry point from the function
       //      descriptor.
       //   3. Load the TOC of the callee from the function descriptor into r2.
@@ -2776,6 +3153,15 @@ unsigned PrepareCall(SelectionDAG &DAG, SDValue &Callee, SDValue &InFlag,
   return CallOpc;
 }
 
+static
+bool isLocalCall(const SDValue &Callee)
+{
+  if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee))
+    return !G->getGlobal()->isDeclaration() &&
+           !G->getGlobal()->isWeakForLinker();
+  return false;
+}
+
 SDValue
 PPCTargetLowering::LowerCallResult(SDValue Chain, SDValue InFlag,
                                    CallingConv::ID CallConv, bool isVarArg,
@@ -2791,12 +3177,32 @@ PPCTargetLowering::LowerCallResult(SDValue Chain, SDValue InFlag,
   // Copy all of the result registers out of their specified physreg.
   for (unsigned i = 0, e = RVLocs.size(); i != e; ++i) {
     CCValAssign &VA = RVLocs[i];
-    EVT VT = VA.getValVT();
     assert(VA.isRegLoc() && "Can only return in registers!");
-    Chain = DAG.getCopyFromReg(Chain, dl,
-                               VA.getLocReg(), VT, InFlag).getValue(1);
-    InVals.push_back(Chain.getValue(0));
-    InFlag = Chain.getValue(2);
+
+    SDValue Val = DAG.getCopyFromReg(Chain, dl,
+                                     VA.getLocReg(), VA.getLocVT(), InFlag);
+    Chain = Val.getValue(1);
+    InFlag = Val.getValue(2);
+
+    switch (VA.getLocInfo()) {
+    default: llvm_unreachable("Unknown loc info!");
+    case CCValAssign::Full: break;
+    case CCValAssign::AExt:
+      Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val);
+      break;
+    case CCValAssign::ZExt:
+      Val = DAG.getNode(ISD::AssertZext, dl, VA.getLocVT(), Val,
+                        DAG.getValueType(VA.getValVT()));
+      Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val);
+      break;
+    case CCValAssign::SExt:
+      Val = DAG.getNode(ISD::AssertSext, dl, VA.getLocVT(), Val,
+                        DAG.getValueType(VA.getValVT()));
+      Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val);
+      break;
+    }
+
+    InVals.push_back(Val);
   }
 
   return Chain;
@@ -2819,6 +3225,10 @@ PPCTargetLowering::FinishCall(CallingConv::ID CallConv, DebugLoc dl,
                                  isTailCall, RegsToPass, Ops, NodeTys,
                                  PPCSubTarget);
 
+  // Add implicit use of CR bit 6 for 32-bit SVR4 vararg calls
+  if (isVarArg && PPCSubTarget.isSVR4ABI() && !PPCSubTarget.isPPC64())
+    Ops.push_back(DAG.getRegister(PPC::CR1EQ, MVT::i32));
+
   // When performing tail call optimization the callee pops its arguments off
   // the stack. Account for this here so these bytes can be pushed back on in
   // PPCRegisterInfo::eliminateCallFramePseudoInstr.
@@ -2880,8 +3290,8 @@ PPCTargetLowering::FinishCall(CallingConv::ID CallConv, DebugLoc dl,
       // from allocating it), resulting in an additional register being
       // allocated and an unnecessary move instruction being generated.
       needsTOCRestore = true;
-    } else if (CallOpc == PPCISD::CALL_SVR4) {
-      // Otherwise insert NOP.
+    } else if ((CallOpc == PPCISD::CALL_SVR4) && !isLocalCall(Callee)) {
+      // Otherwise insert NOP for non-local calls.
       CallOpc = PPCISD::CALL_NOP_SVR4;
     }
   }
@@ -2923,10 +3333,16 @@ PPCTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     isTailCall = IsEligibleForTailCallOptimization(Callee, CallConv, isVarArg,
                                                    Ins, DAG);
 
-  if (PPCSubTarget.isSVR4ABI() && !PPCSubTarget.isPPC64())
-    return LowerCall_SVR4(Chain, Callee, CallConv, isVarArg,
-                          isTailCall, Outs, OutVals, Ins,
-                          dl, DAG, InVals);
+  if (PPCSubTarget.isSVR4ABI()) {
+    if (PPCSubTarget.isPPC64())
+      return LowerCall_64SVR4(Chain, Callee, CallConv, isVarArg,
+                              isTailCall, Outs, OutVals, Ins,
+                              dl, DAG, InVals);
+    else
+      return LowerCall_32SVR4(Chain, Callee, CallConv, isVarArg,
+                              isTailCall, Outs, OutVals, Ins,
+                              dl, DAG, InVals);
+  }
 
   return LowerCall_Darwin(Chain, Callee, CallConv, isVarArg,
                           isTailCall, Outs, OutVals, Ins,
@@ -2934,15 +3350,15 @@ PPCTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
 }
 
 SDValue
-PPCTargetLowering::LowerCall_SVR4(SDValue Chain, SDValue Callee,
-                                  CallingConv::ID CallConv, bool isVarArg,
-                                  bool isTailCall,
-                                  const SmallVectorImpl<ISD::OutputArg> &Outs,
-                                  const SmallVectorImpl<SDValue> &OutVals,
-                                  const SmallVectorImpl<ISD::InputArg> &Ins,
-                                  DebugLoc dl, SelectionDAG &DAG,
-                                  SmallVectorImpl<SDValue> &InVals) const {
-  // See PPCTargetLowering::LowerFormalArguments_SVR4() for a description
+PPCTargetLowering::LowerCall_32SVR4(SDValue Chain, SDValue Callee,
+                                    CallingConv::ID CallConv, bool isVarArg,
+                                    bool isTailCall,
+                                    const SmallVectorImpl<ISD::OutputArg> &Outs,
+                                    const SmallVectorImpl<SDValue> &OutVals,
+                                    const SmallVectorImpl<ISD::InputArg> &Ins,
+                                    DebugLoc dl, SelectionDAG &DAG,
+                                    SmallVectorImpl<SDValue> &InVals) const {
+  // See PPCTargetLowering::LowerFormalArguments_32SVR4() for a description
   // of the 32-bit SVR4 ABI stack frame layout.
 
   assert((CallConv == CallingConv::C ||
@@ -3042,73 +3458,455 @@ PPCTargetLowering::LowerCall_SVR4(SDValue Chain, SDValue Callee,
 
   SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
   SmallVector<TailCallArgumentInfo, 8> TailCallArguments;
-  SmallVector<SDValue, 8> MemOpChains;
+  SmallVector<SDValue, 8> MemOpChains;
+
+  bool seenFloatArg = false;
+  // Walk the register/memloc assignments, inserting copies/loads.
+  for (unsigned i = 0, j = 0, e = ArgLocs.size();
+       i != e;
+       ++i) {
+    CCValAssign &VA = ArgLocs[i];
+    SDValue Arg = OutVals[i];
+    ISD::ArgFlagsTy Flags = Outs[i].Flags;
+
+    if (Flags.isByVal()) {
+      // Argument is an aggregate which is passed by value, thus we need to
+      // create a copy of it in the local variable space of the current stack
+      // frame (which is the stack frame of the caller) and pass the address of
+      // this copy to the callee.
+      assert((j < ByValArgLocs.size()) && "Index out of bounds!");
+      CCValAssign &ByValVA = ByValArgLocs[j++];
+      assert((VA.getValNo() == ByValVA.getValNo()) && "ValNo mismatch!");
+
+      // Memory reserved in the local variable space of the callers stack frame.
+      unsigned LocMemOffset = ByValVA.getLocMemOffset();
+
+      SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset);
+      PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(), StackPtr, PtrOff);
+
+      // Create a copy of the argument in the local area of the current
+      // stack frame.
+      SDValue MemcpyCall =
+        CreateCopyOfByValArgument(Arg, PtrOff,
+                                  CallSeqStart.getNode()->getOperand(0),
+                                  Flags, DAG, dl);
+
+      // This must go outside the CALLSEQ_START..END.
+      SDValue NewCallSeqStart = DAG.getCALLSEQ_START(MemcpyCall,
+                           CallSeqStart.getNode()->getOperand(1));
+      DAG.ReplaceAllUsesWith(CallSeqStart.getNode(),
+                             NewCallSeqStart.getNode());
+      Chain = CallSeqStart = NewCallSeqStart;
+
+      // Pass the address of the aggregate copy on the stack either in a
+      // physical register or in the parameter list area of the current stack
+      // frame to the callee.
+      Arg = PtrOff;
+    }
+
+    if (VA.isRegLoc()) {
+      seenFloatArg |= VA.getLocVT().isFloatingPoint();
+      // Put argument in a physical register.
+      RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
+    } else {
+      // Put argument in the parameter list area of the current stack frame.
+      assert(VA.isMemLoc());
+      unsigned LocMemOffset = VA.getLocMemOffset();
+
+      if (!isTailCall) {
+        SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset);
+        PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(), StackPtr, PtrOff);
+
+        MemOpChains.push_back(DAG.getStore(Chain, dl, Arg, PtrOff,
+                                           MachinePointerInfo(),
+                                           false, false, 0));
+      } else {
+        // Calculate and remember argument location.
+        CalculateTailCallArgDest(DAG, MF, false, Arg, SPDiff, LocMemOffset,
+                                 TailCallArguments);
+      }
+    }
+  }
+
+  if (!MemOpChains.empty())
+    Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
+                        &MemOpChains[0], MemOpChains.size());
+
+  // Build a sequence of copy-to-reg nodes chained together with token chain
+  // and flag operands which copy the outgoing args into the appropriate regs.
+  SDValue InFlag;
+  for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
+    Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
+                             RegsToPass[i].second, InFlag);
+    InFlag = Chain.getValue(1);
+  }
+
+  // Set CR bit 6 to true if this is a vararg call with floating args passed in
+  // registers.
+  if (isVarArg) {
+    SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue);
+    SDValue Ops[] = { Chain, InFlag };
+
+    Chain = DAG.getNode(seenFloatArg ? PPCISD::CR6SET : PPCISD::CR6UNSET,
+                        dl, VTs, Ops, InFlag.getNode() ? 2 : 1);
+
+    InFlag = Chain.getValue(1);
+  }
+
+  if (isTailCall)
+    PrepareTailCall(DAG, InFlag, Chain, dl, false, SPDiff, NumBytes, LROp, FPOp,
+                    false, TailCallArguments);
+
+  return FinishCall(CallConv, dl, isTailCall, isVarArg, DAG,
+                    RegsToPass, InFlag, Chain, Callee, SPDiff, NumBytes,
+                    Ins, InVals);
+}
+
+// Copy an argument into memory, being careful to do this outside the
+// call sequence for the call to which the argument belongs.
+SDValue
+PPCTargetLowering::createMemcpyOutsideCallSeq(SDValue Arg, SDValue PtrOff,
+                                              SDValue CallSeqStart,
+                                              ISD::ArgFlagsTy Flags,
+                                              SelectionDAG &DAG,
+                                              DebugLoc dl) const {
+  SDValue MemcpyCall = CreateCopyOfByValArgument(Arg, PtrOff,
+                        CallSeqStart.getNode()->getOperand(0),
+                        Flags, DAG, dl);
+  // The MEMCPY must go outside the CALLSEQ_START..END.
+  SDValue NewCallSeqStart = DAG.getCALLSEQ_START(MemcpyCall,
+                             CallSeqStart.getNode()->getOperand(1));
+  DAG.ReplaceAllUsesWith(CallSeqStart.getNode(),
+                         NewCallSeqStart.getNode());
+  return NewCallSeqStart;
+}
+
+SDValue
+PPCTargetLowering::LowerCall_64SVR4(SDValue Chain, SDValue Callee,
+                                    CallingConv::ID CallConv, bool isVarArg,
+                                    bool isTailCall,
+                                    const SmallVectorImpl<ISD::OutputArg> &Outs,
+                                    const SmallVectorImpl<SDValue> &OutVals,
+                                    const SmallVectorImpl<ISD::InputArg> &Ins,
+                                    DebugLoc dl, SelectionDAG &DAG,
+                                    SmallVectorImpl<SDValue> &InVals) const {
+
+  unsigned NumOps = Outs.size();
+
+  EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
+  unsigned PtrByteSize = 8;
+
+  MachineFunction &MF = DAG.getMachineFunction();
+
+  // Mark this function as potentially containing a function that contains a
+  // tail call. As a consequence the frame pointer will be used for dynamicalloc
+  // and restoring the callers stack pointer in this functions epilog. This is
+  // done because by tail calling the called function might overwrite the value
+  // in this function's (MF) stack pointer stack slot 0(SP).
+  if (getTargetMachine().Options.GuaranteedTailCallOpt &&
+      CallConv == CallingConv::Fast)
+    MF.getInfo<PPCFunctionInfo>()->setHasFastCall();
+
+  unsigned nAltivecParamsAtEnd = 0;
+
+  // Count how many bytes are to be pushed on the stack, including the linkage
+  // area, and parameter passing area.  We start with at least 48 bytes, which
+  // is reserved space for [SP][CR][LR][3 x unused].
+  // NOTE: For PPC64, nAltivecParamsAtEnd always remains zero as a result
+  // of this call.
+  unsigned NumBytes =
+    CalculateParameterAndLinkageAreaSize(DAG, true, isVarArg, CallConv,
+                                         Outs, OutVals, nAltivecParamsAtEnd);
+
+  // Calculate by how many bytes the stack has to be adjusted in case of tail
+  // call optimization.
+  int SPDiff = CalculateTailCallSPDiff(DAG, isTailCall, NumBytes);
+
+  // To protect arguments on the stack from being clobbered in a tail call,
+  // force all the loads to happen before doing any other lowering.
+  if (isTailCall)
+    Chain = DAG.getStackArgumentTokenFactor(Chain);
+
+  // Adjust the stack pointer for the new arguments...
+  // These operations are automatically eliminated by the prolog/epilog pass
+  Chain = DAG.getCALLSEQ_START(Chain, DAG.getIntPtrConstant(NumBytes, true));
+  SDValue CallSeqStart = Chain;
+
+  // Load the return address and frame pointer so it can be move somewhere else
+  // later.
+  SDValue LROp, FPOp;
+  Chain = EmitTailCallLoadFPAndRetAddr(DAG, SPDiff, Chain, LROp, FPOp, true,
+                                       dl);
+
+  // Set up a copy of the stack pointer for use loading and storing any
+  // arguments that may not fit in the registers available for argument
+  // passing.
+  SDValue StackPtr = DAG.getRegister(PPC::X1, MVT::i64);
+
+  // Figure out which arguments are going to go in registers, and which in
+  // memory.  Also, if this is a vararg function, floating point operations
+  // must be stored to our stack, and loaded into integer regs as well, if
+  // any integer regs are available for argument passing.
+  unsigned ArgOffset = PPCFrameLowering::getLinkageSize(true, true);
+  unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0;
+
+  static const uint16_t GPR[] = {
+    PPC::X3, PPC::X4, PPC::X5, PPC::X6,
+    PPC::X7, PPC::X8, PPC::X9, PPC::X10,
+  };
+  static const uint16_t *FPR = GetFPR();
+
+  static const uint16_t VR[] = {
+    PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8,
+    PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13
+  };
+  const unsigned NumGPRs = array_lengthof(GPR);
+  const unsigned NumFPRs = 13;
+  const unsigned NumVRs  = array_lengthof(VR);
+
+  SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
+  SmallVector<TailCallArgumentInfo, 8> TailCallArguments;
 
-  bool seenFloatArg = false;
-  // Walk the register/memloc assignments, inserting copies/loads.
-  for (unsigned i = 0, j = 0, e = ArgLocs.size();
-       i != e;
-       ++i) {
-    CCValAssign &VA = ArgLocs[i];
+  SmallVector<SDValue, 8> MemOpChains;
+  for (unsigned i = 0; i != NumOps; ++i) {
     SDValue Arg = OutVals[i];
     ISD::ArgFlagsTy Flags = Outs[i].Flags;
 
+    // PtrOff will be used to store the current argument to the stack if a
+    // register cannot be found for it.
+    SDValue PtrOff;
+
+    PtrOff = DAG.getConstant(ArgOffset, StackPtr.getValueType());
+
+    PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff);
+
+    // Promote integers to 64-bit values.
+    if (Arg.getValueType() == MVT::i32) {
+      // FIXME: Should this use ANY_EXTEND if neither sext nor zext?
+      unsigned ExtOp = Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
+      Arg = DAG.getNode(ExtOp, dl, MVT::i64, Arg);
+    }
+
+    // FIXME memcpy is used way more than necessary.  Correctness first.
+    // Note: "by value" is code for passing a structure by value, not
+    // basic types.
     if (Flags.isByVal()) {
-      // Argument is an aggregate which is passed by value, thus we need to
-      // create a copy of it in the local variable space of the current stack
-      // frame (which is the stack frame of the caller) and pass the address of
-      // this copy to the callee.
-      assert((j < ByValArgLocs.size()) && "Index out of bounds!");
-      CCValAssign &ByValVA = ByValArgLocs[j++];
-      assert((VA.getValNo() == ByValVA.getValNo()) && "ValNo mismatch!");
+      // Note: Size includes alignment padding, so
+      //   struct x { short a; char b; }
+      // will have Size = 4.  With #pragma pack(1), it will have Size = 3.
+      // These are the proper values we need for right-justifying the
+      // aggregate in a parameter register.
+      unsigned Size = Flags.getByValSize();
 
-      // Memory reserved in the local variable space of the callers stack frame.
-      unsigned LocMemOffset = ByValVA.getLocMemOffset();
+      // An empty aggregate parameter takes up no storage and no
+      // registers.
+      if (Size == 0)
+        continue;
 
-      SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset);
-      PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(), StackPtr, PtrOff);
+      // All aggregates smaller than 8 bytes must be passed right-justified.
+      if (Size==1 || Size==2 || Size==4) {
+        EVT VT = (Size==1) ? MVT::i8 : ((Size==2) ? MVT::i16 : MVT::i32);
+        if (GPR_idx != NumGPRs) {
+          SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, PtrVT, Chain, Arg,
+                                        MachinePointerInfo(), VT,
+                                        false, false, 0);
+          MemOpChains.push_back(Load.getValue(1));
+          RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load));
 
-      // Create a copy of the argument in the local area of the current
-      // stack frame.
-      SDValue MemcpyCall =
-        CreateCopyOfByValArgument(Arg, PtrOff,
-                                  CallSeqStart.getNode()->getOperand(0),
-                                  Flags, DAG, dl);
+          ArgOffset += PtrByteSize;
+          continue;
+        }
+      }
 
-      // This must go outside the CALLSEQ_START..END.
-      SDValue NewCallSeqStart = DAG.getCALLSEQ_START(MemcpyCall,
-                           CallSeqStart.getNode()->getOperand(1));
-      DAG.ReplaceAllUsesWith(CallSeqStart.getNode(),
-                             NewCallSeqStart.getNode());
-      Chain = CallSeqStart = NewCallSeqStart;
+      if (GPR_idx == NumGPRs && Size < 8) {
+        SDValue Const = DAG.getConstant(PtrByteSize - Size,
+                                        PtrOff.getValueType());
+        SDValue AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, Const);
+        Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, AddPtr,
+                                                          CallSeqStart,
+                                                          Flags, DAG, dl);
+        ArgOffset += PtrByteSize;
+        continue;
+      }
+      // Copy entire object into memory.  There are cases where gcc-generated
+      // code assumes it is there, even if it could be put entirely into
+      // registers.  (This is not what the doc says.)
 
-      // Pass the address of the aggregate copy on the stack either in a
-      // physical register or in the parameter list area of the current stack
-      // frame to the callee.
-      Arg = PtrOff;
+      // FIXME: The above statement is likely due to a misunderstanding of the
+      // documents.  All arguments must be copied into the parameter area BY
+      // THE CALLEE in the event that the callee takes the address of any
+      // formal argument.  That has not yet been implemented.  However, it is
+      // reasonable to use the stack area as a staging area for the register
+      // load.
+
+      // Skip this for small aggregates, as we will use the same slot for a
+      // right-justified copy, below.
+      if (Size >= 8)
+        Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, PtrOff,
+                                                          CallSeqStart,
+                                                          Flags, DAG, dl);
+
+      // When a register is available, pass a small aggregate right-justified.
+      if (Size < 8 && GPR_idx != NumGPRs) {
+        // The easiest way to get this right-justified in a register
+        // is to copy the structure into the rightmost portion of a
+        // local variable slot, then load the whole slot into the
+        // register.
+        // FIXME: The memcpy seems to produce pretty awful code for
+        // small aggregates, particularly for packed ones.
+        // FIXME: It would be preferable to use the slot in the 
+        // parameter save area instead of a new local variable.
+        SDValue Const = DAG.getConstant(8 - Size, PtrOff.getValueType());
+        SDValue AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, Const);
+        Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, AddPtr,
+                                                          CallSeqStart,
+                                                          Flags, DAG, dl);
+
+        // Load the slot into the register.
+        SDValue Load = DAG.getLoad(PtrVT, dl, Chain, PtrOff,
+                                   MachinePointerInfo(),
+                                   false, false, false, 0);
+        MemOpChains.push_back(Load.getValue(1));
+        RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load));
+
+        // Done with this argument.
+        ArgOffset += PtrByteSize;
+        continue;
+      }
+
+      // For aggregates larger than PtrByteSize, copy the pieces of the
+      // object that fit into registers from the parameter save area.
+      for (unsigned j=0; j<Size; j+=PtrByteSize) {
+        SDValue Const = DAG.getConstant(j, PtrOff.getValueType());
+        SDValue AddArg = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, Const);
+        if (GPR_idx != NumGPRs) {
+          SDValue Load = DAG.getLoad(PtrVT, dl, Chain, AddArg,
+                                     MachinePointerInfo(),
+                                     false, false, false, 0);
+          MemOpChains.push_back(Load.getValue(1));
+          RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load));
+          ArgOffset += PtrByteSize;
+        } else {
+          ArgOffset += ((Size - j + PtrByteSize-1)/PtrByteSize)*PtrByteSize;
+          break;
+        }
+      }
+      continue;
     }
 
-    if (VA.isRegLoc()) {
-      seenFloatArg |= VA.getLocVT().isFloatingPoint();
-      // Put argument in a physical register.
-      RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
-    } else {
-      // Put argument in the parameter list area of the current stack frame.
-      assert(VA.isMemLoc());
-      unsigned LocMemOffset = VA.getLocMemOffset();
+    switch (Arg.getValueType().getSimpleVT().SimpleTy) {
+    default: llvm_unreachable("Unexpected ValueType for argument!");
+    case MVT::i32:
+    case MVT::i64:
+      if (GPR_idx != NumGPRs) {
+        RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Arg));
+      } else {
+        LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset,
+                         true, isTailCall, false, MemOpChains,
+                         TailCallArguments, dl);
+      }
+      ArgOffset += PtrByteSize;
+      break;
+    case MVT::f32:
+    case MVT::f64:
+      if (FPR_idx != NumFPRs) {
+        RegsToPass.push_back(std::make_pair(FPR[FPR_idx++], Arg));
 
-      if (!isTailCall) {
-        SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset);
-        PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(), StackPtr, PtrOff);
+        if (isVarArg) {
+          // A single float or an aggregate containing only a single float
+          // must be passed right-justified in the stack doubleword, and
+          // in the GPR, if one is available.
+          SDValue StoreOff;
+          if (Arg.getValueType().getSimpleVT().SimpleTy == MVT::f32) {
+            SDValue ConstFour = DAG.getConstant(4, PtrOff.getValueType());
+            StoreOff = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, ConstFour);
+          } else
+            StoreOff = PtrOff;
 
-        MemOpChains.push_back(DAG.getStore(Chain, dl, Arg, PtrOff,
-                                           MachinePointerInfo(),
-                                           false, false, 0));
+          SDValue Store = DAG.getStore(Chain, dl, Arg, StoreOff,
+                                       MachinePointerInfo(), false, false, 0);
+          MemOpChains.push_back(Store);
+
+          // Float varargs are always shadowed in available integer registers
+          if (GPR_idx != NumGPRs) {
+            SDValue Load = DAG.getLoad(PtrVT, dl, Store, PtrOff,
+                                       MachinePointerInfo(), false, false,
+                                       false, 0);
+            MemOpChains.push_back(Load.getValue(1));
+            RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load));
+          }
+        } else if (GPR_idx != NumGPRs)
+          // If we have any FPRs remaining, we may also have GPRs remaining.
+          ++GPR_idx;
       } else {
-        // Calculate and remember argument location.
-        CalculateTailCallArgDest(DAG, MF, false, Arg, SPDiff, LocMemOffset,
-                                 TailCallArguments);
+        // Single-precision floating-point values are mapped to the
+        // second (rightmost) word of the stack doubleword.
+        if (Arg.getValueType() == MVT::f32) {
+          SDValue ConstFour = DAG.getConstant(4, PtrOff.getValueType());
+          PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, ConstFour);
+        }
+
+        LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset,
+                         true, isTailCall, false, MemOpChains,
+                         TailCallArguments, dl);
+      }
+      ArgOffset += 8;
+      break;
+    case MVT::v4f32:
+    case MVT::v4i32:
+    case MVT::v8i16:
+    case MVT::v16i8:
+      if (isVarArg) {
+        // These go aligned on the stack, or in the corresponding R registers
+        // when within range.  The Darwin PPC ABI doc claims they also go in
+        // V registers; in fact gcc does this only for arguments that are
+        // prototyped, not for those that match the ...  We do it for all
+        // arguments, seems to work.
+        while (ArgOffset % 16 !=0) {
+          ArgOffset += PtrByteSize;
+          if (GPR_idx != NumGPRs)
+            GPR_idx++;
+        }
+        // We could elide this store in the case where the object fits
+        // entirely in R registers.  Maybe later.
+        PtrOff = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr,
+                            DAG.getConstant(ArgOffset, PtrVT));
+        SDValue Store = DAG.getStore(Chain, dl, Arg, PtrOff,
+                                     MachinePointerInfo(), false, false, 0);
+        MemOpChains.push_back(Store);
+        if (VR_idx != NumVRs) {
+          SDValue Load = DAG.getLoad(MVT::v4f32, dl, Store, PtrOff,
+                                     MachinePointerInfo(),
+                                     false, false, false, 0);
+          MemOpChains.push_back(Load.getValue(1));
+          RegsToPass.push_back(std::make_pair(VR[VR_idx++], Load));
+        }
+        ArgOffset += 16;
+        for (unsigned i=0; i<16; i+=PtrByteSize) {
+          if (GPR_idx == NumGPRs)
+            break;
+          SDValue Ix = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff,
+                                  DAG.getConstant(i, PtrVT));
+          SDValue Load = DAG.getLoad(PtrVT, dl, Store, Ix, MachinePointerInfo(),
+                                     false, false, false, 0);
+          MemOpChains.push_back(Load.getValue(1));
+          RegsToPass.push_back(std::make_pair(GPR[GPR_idx++], Load));
+        }
+        break;
+      }
+
+      // Non-varargs Altivec params generally go in registers, but have
+      // stack space allocated at the end.
+      if (VR_idx != NumVRs) {
+        // Doesn't have GPR space allocated.
+        RegsToPass.push_back(std::make_pair(VR[VR_idx++], Arg));
+      } else {
+        LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset,
+                         true, isTailCall, true, MemOpChains,
+                         TailCallArguments, dl);
+        ArgOffset += 16;
       }
+      break;
     }
   }
 
@@ -3116,12 +3914,24 @@ PPCTargetLowering::LowerCall_SVR4(SDValue Chain, SDValue Callee,
     Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
                         &MemOpChains[0], MemOpChains.size());
 
-  // Set CR6 to true if this is a vararg call with floating args passed in
-  // registers.
-  if (isVarArg) {
-    SDValue SetCR(DAG.getMachineNode(seenFloatArg ? PPC::CRSET : PPC::CRUNSET,
-                                     dl, MVT::i32), 0);
-    RegsToPass.push_back(std::make_pair(unsigned(PPC::CR1EQ), SetCR));
+  // Check if this is an indirect call (MTCTR/BCTRL).
+  // See PrepareCall() for more information about calls through function
+  // pointers in the 64-bit SVR4 ABI.
+  if (!isTailCall &&
+      !dyn_cast<GlobalAddressSDNode>(Callee) &&
+      !dyn_cast<ExternalSymbolSDNode>(Callee) &&
+      !isBLACompatibleAddress(Callee, DAG)) {
+    // Load r2 into a virtual register and store it to the TOC save area.
+    SDValue Val = DAG.getCopyFromReg(Chain, dl, PPC::X2, MVT::i64);
+    // TOC save area offset.
+    SDValue PtrOff = DAG.getIntPtrConstant(40);
+    SDValue AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff);
+    Chain = DAG.getStore(Val.getValue(1), dl, Val, AddPtr, MachinePointerInfo(),
+                         false, false, 0);
+    // R12 must contain the address of an indirect callee.  This does not
+    // mean the MTCTR instruction must use R12; it's easier to model this
+    // as an extra parameter, so do that.
+    RegsToPass.push_back(std::make_pair((unsigned)PPC::X12, Callee));
   }
 
   // Build a sequence of copy-to-reg nodes chained together with token chain
@@ -3134,8 +3944,8 @@ PPCTargetLowering::LowerCall_SVR4(SDValue Chain, SDValue Callee,
   }
 
   if (isTailCall)
-    PrepareTailCall(DAG, InFlag, Chain, dl, false, SPDiff, NumBytes, LROp, FPOp,
-                    false, TailCallArguments);
+    PrepareTailCall(DAG, InFlag, Chain, dl, true, SPDiff, NumBytes, LROp,
+                    FPOp, true, TailCallArguments);
 
   return FinishCall(CallConv, dl, isTailCall, isVarArg, DAG,
                     RegsToPass, InFlag, Chain, Callee, SPDiff, NumBytes,
@@ -3152,7 +3962,7 @@ PPCTargetLowering::LowerCall_Darwin(SDValue Chain, SDValue Callee,
                                     DebugLoc dl, SelectionDAG &DAG,
                                     SmallVectorImpl<SDValue> &InVals) const {
 
-  unsigned NumOps  = Outs.size();
+  unsigned NumOps = Outs.size();
 
   EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy();
   bool isPPC64 = PtrVT == MVT::i64;
@@ -3259,11 +4069,13 @@ PPCTargetLowering::LowerCall_Darwin(SDValue Chain, SDValue Callee,
     }
 
     // FIXME memcpy is used way more than necessary.  Correctness first.
+    // Note: "by value" is code for passing a structure by value, not
+    // basic types.
     if (Flags.isByVal()) {
       unsigned Size = Flags.getByValSize();
+      // Very small objects are passed right-justified.  Everything else is
+      // passed left-justified.
       if (Size==1 || Size==2) {
-        // Very small objects are passed right-justified.
-        // Everything else is passed left-justified.
         EVT VT = (Size==1) ? MVT::i8 : MVT::i16;
         if (GPR_idx != NumGPRs) {
           SDValue Load = DAG.getExtLoad(ISD::EXTLOAD, dl, PtrVT, Chain, Arg,
@@ -3274,17 +4086,12 @@ PPCTargetLowering::LowerCall_Darwin(SDValue Chain, SDValue Callee,
 
           ArgOffset += PtrByteSize;
         } else {
-          SDValue Const = DAG.getConstant(4 - Size, PtrOff.getValueType());
+          SDValue Const = DAG.getConstant(PtrByteSize - Size,
+                                          PtrOff.getValueType());
           SDValue AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, PtrOff, Const);
-          SDValue MemcpyCall = CreateCopyOfByValArgument(Arg, AddPtr,
-                                CallSeqStart.getNode()->getOperand(0),
-                                Flags, DAG, dl);
-          // This must go outside the CALLSEQ_START..END.
-          SDValue NewCallSeqStart = DAG.getCALLSEQ_START(MemcpyCall,
-                               CallSeqStart.getNode()->getOperand(1));
-          DAG.ReplaceAllUsesWith(CallSeqStart.getNode(),
-                                 NewCallSeqStart.getNode());
-          Chain = CallSeqStart = NewCallSeqStart;
+          Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, AddPtr,
+                                                            CallSeqStart,
+                                                            Flags, DAG, dl);
           ArgOffset += PtrByteSize;
         }
         continue;
@@ -3292,15 +4099,13 @@ PPCTargetLowering::LowerCall_Darwin(SDValue Chain, SDValue Callee,
       // Copy entire object into memory.  There are cases where gcc-generated
       // code assumes it is there, even if it could be put entirely into
       // registers.  (This is not what the doc says.)
-      SDValue MemcpyCall = CreateCopyOfByValArgument(Arg, PtrOff,
-                            CallSeqStart.getNode()->getOperand(0),
-                            Flags, DAG, dl);
-      // This must go outside the CALLSEQ_START..END.
-      SDValue NewCallSeqStart = DAG.getCALLSEQ_START(MemcpyCall,
-                           CallSeqStart.getNode()->getOperand(1));
-      DAG.ReplaceAllUsesWith(CallSeqStart.getNode(), NewCallSeqStart.getNode());
-      Chain = CallSeqStart = NewCallSeqStart;
-      // And copy the pieces of it that fit into registers.
+      Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, PtrOff,
+                                                        CallSeqStart,
+                                                        Flags, DAG, dl);
+
+      // For small aggregates (Darwin only) and aggregates >= PtrByteSize,
+      // copy the pieces of the object that fit into registers from the
+      // parameter save area.
       for (unsigned j=0; j<Size; j+=PtrByteSize) {
         SDValue Const = DAG.getConstant(j, PtrOff.getValueType());
         SDValue AddArg = DAG.getNode(ISD::ADD, dl, PtrVT, Arg, Const);
@@ -3369,11 +4174,10 @@ PPCTargetLowering::LowerCall_Darwin(SDValue Chain, SDValue Callee,
               !isPPC64)  // PPC64 has 64-bit GPR's obviously :)
             ++GPR_idx;
         }
-      } else {
+      } else
         LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset,
                          isPPC64, isTailCall, false, MemOpChains,
                          TailCallArguments, dl);
-      }
       if (isPPC64)
         ArgOffset += 8;
       else
@@ -3468,22 +4272,6 @@ PPCTargetLowering::LowerCall_Darwin(SDValue Chain, SDValue Callee,
     Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
                         &MemOpChains[0], MemOpChains.size());
 
-  // Check if this is an indirect call (MTCTR/BCTRL).
-  // See PrepareCall() for more information about calls through function
-  // pointers in the 64-bit SVR4 ABI.
-  if (!isTailCall && isPPC64 && PPCSubTarget.isSVR4ABI() &&
-      !dyn_cast<GlobalAddressSDNode>(Callee) &&
-      !dyn_cast<ExternalSymbolSDNode>(Callee) &&
-      !isBLACompatibleAddress(Callee, DAG)) {
-    // Load r2 into a virtual register and store it to the TOC save area.
-    SDValue Val = DAG.getCopyFromReg(Chain, dl, PPC::X2, MVT::i64);
-    // TOC save area offset.
-    SDValue PtrOff = DAG.getIntPtrConstant(40);
-    SDValue AddPtr = DAG.getNode(ISD::ADD, dl, PtrVT, StackPtr, PtrOff);
-    Chain = DAG.getStore(Val.getValue(1), dl, Val, AddPtr, MachinePointerInfo(),
-                         false, false, 0);
-  }
-
   // On Darwin, R12 must contain the address of an indirect callee.  This does
   // not mean the MTCTR instruction must use R12; it's easier to model this as
   // an extra parameter, so do that.
@@ -3548,8 +4336,24 @@ PPCTargetLowering::LowerReturn(SDValue Chain,
   for (unsigned i = 0; i != RVLocs.size(); ++i) {
     CCValAssign &VA = RVLocs[i];
     assert(VA.isRegLoc() && "Can only return in registers!");
-    Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(),
-                             OutVals[i], Flag);
+
+    SDValue Arg = OutVals[i];
+
+    switch (VA.getLocInfo()) {
+    default: llvm_unreachable("Unknown loc info!");
+    case CCValAssign::Full: break;
+    case CCValAssign::AExt:
+      Arg = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), Arg);
+      break;
+    case CCValAssign::ZExt:
+      Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), Arg);
+      break;
+    case CCValAssign::SExt:
+      Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), Arg);
+      break;
+    }
+
+    Chain = DAG.getCopyToReg(Chain, dl, VA.getLocReg(), Arg, Flag);
     Flag = Chain.getValue(1);
   }
 
@@ -3781,7 +4585,52 @@ SDValue PPCTargetLowering::LowerSINT_TO_FP(SDValue Op,
     return SDValue();
 
   if (Op.getOperand(0).getValueType() == MVT::i64) {
-    SDValue Bits = DAG.getNode(ISD::BITCAST, dl, MVT::f64, Op.getOperand(0));
+    SDValue SINT = Op.getOperand(0);
+    // When converting to single-precision, we actually need to convert
+    // to double-precision first and then round to single-precision.
+    // To avoid double-rounding effects during that operation, we have
+    // to prepare the input operand.  Bits that might be truncated when
+    // converting to double-precision are replaced by a bit that won't
+    // be lost at this stage, but is below the single-precision rounding
+    // position.
+    //
+    // However, if -enable-unsafe-fp-math is in effect, accept double
+    // rounding to avoid the extra overhead.
+    if (Op.getValueType() == MVT::f32 &&
+        !DAG.getTarget().Options.UnsafeFPMath) {
+
+      // Twiddle input to make sure the low 11 bits are zero.  (If this
+      // is the case, we are guaranteed the value will fit into the 53 bit
+      // mantissa of an IEEE double-precision value without rounding.)
+      // If any of those low 11 bits were not zero originally, make sure
+      // bit 12 (value 2048) is set instead, so that the final rounding
+      // to single-precision gets the correct result.
+      SDValue Round = DAG.getNode(ISD::AND, dl, MVT::i64,
+                                  SINT, DAG.getConstant(2047, MVT::i64));
+      Round = DAG.getNode(ISD::ADD, dl, MVT::i64,
+                          Round, DAG.getConstant(2047, MVT::i64));
+      Round = DAG.getNode(ISD::OR, dl, MVT::i64, Round, SINT);
+      Round = DAG.getNode(ISD::AND, dl, MVT::i64,
+                          Round, DAG.getConstant(-2048, MVT::i64));
+
+      // However, we cannot use that value unconditionally: if the magnitude
+      // of the input value is small, the bit-twiddling we did above might
+      // end up visibly changing the output.  Fortunately, in that case, we
+      // don't need to twiddle bits since the original input will convert
+      // exactly to double-precision floating-point already.  Therefore,
+      // construct a conditional to use the original value if the top 11
+      // bits are all sign-bit copies, and use the rounded value computed
+      // above otherwise.
+      SDValue Cond = DAG.getNode(ISD::SRA, dl, MVT::i64,
+                                 SINT, DAG.getConstant(53, MVT::i32));
+      Cond = DAG.getNode(ISD::ADD, dl, MVT::i64,
+                         Cond, DAG.getConstant(1, MVT::i64));
+      Cond = DAG.getSetCC(dl, MVT::i32,
+                          Cond, DAG.getConstant(1, MVT::i64), ISD::SETUGT);
+
+      SINT = DAG.getNode(ISD::SELECT, dl, MVT::i64, Cond, Round, SINT);
+    }
+    SDValue Bits = DAG.getNode(ISD::BITCAST, dl, MVT::f64, SINT);
     SDValue FP = DAG.getNode(PPCISD::FCFID, dl, MVT::f64, Bits);
     if (Op.getValueType() == MVT::f32)
       FP = DAG.getNode(ISD::FP_ROUND, dl,
@@ -4126,7 +4975,7 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
     unsigned TypeShiftAmt = i & (SplatBitSize-1);
 
     // vsplti + shl self.
-    if (SextVal == (i << (int)TypeShiftAmt)) {
+    if (SextVal == (int)((unsigned)i << TypeShiftAmt)) {
       SDValue Res = BuildSplatI(i, SplatSize, MVT::Other, DAG, dl);
       static const unsigned IIDs[] = { // Intrinsic to use for each size.
         Intrinsic::ppc_altivec_vslb, Intrinsic::ppc_altivec_vslh, 0,
@@ -4171,17 +5020,17 @@ SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
     }
 
     // t = vsplti c, result = vsldoi t, t, 1
-    if (SextVal == ((i << 8) | (i < 0 ? 0xFF : 0))) {
+    if (SextVal == (int)(((unsigned)i << 8) | (i < 0 ? 0xFF : 0))) {
       SDValue T = BuildSplatI(i, SplatSize, MVT::v16i8, DAG, dl);
       return BuildVSLDOI(T, T, 1, Op.getValueType(), DAG, dl);
     }
     // t = vsplti c, result = vsldoi t, t, 2
-    if (SextVal == ((i << 16) | (i < 0 ? 0xFFFF : 0))) {
+    if (SextVal == (int)(((unsigned)i << 16) | (i < 0 ? 0xFFFF : 0))) {
       SDValue T = BuildSplatI(i, SplatSize, MVT::v16i8, DAG, dl);
       return BuildVSLDOI(T, T, 2, Op.getValueType(), DAG, dl);
     }
     // t = vsplti c, result = vsldoi t, t, 3
-    if (SextVal == ((i << 24) | (i < 0 ? 0xFFFFFF : 0))) {
+    if (SextVal == (int)(((unsigned)i << 24) | (i < 0 ? 0xFFFFFF : 0))) {
       SDValue T = BuildSplatI(i, SplatSize, MVT::v16i8, DAG, dl);
       return BuildVSLDOI(T, T, 3, Op.getValueType(), DAG, dl);
     }
@@ -5630,6 +6479,14 @@ PPCTargetLowering::getConstraintType(const std::string &Constraint) const {
     case 'v':
     case 'y':
       return C_RegisterClass;
+    case 'Z':
+      // FIXME: While Z does indicate a memory constraint, it specifically
+      // indicates an r+r address (used in conjunction with the 'y' modifier
+      // in the replacement string). Currently, we're forcing the base
+      // register to be r0 in the asm printer (which is interpreted as zero)
+      // and forming the complete address in the second register. This is
+      // suboptimal.
+      return C_Memory;
     }
   }
   return TargetLowering::getConstraintType(Constraint);
@@ -5672,6 +6529,9 @@ PPCTargetLowering::getSingleConstraintMatchWeight(
   case 'y':
     weight = CW_Register;
     break;
+  case 'Z':
+    weight = CW_Memory;
+    break;
   }
   return weight;
 }
@@ -5688,9 +6548,9 @@ PPCTargetLowering::getRegForInlineAsmConstraint(const std::string &Constraint,
         return std::make_pair(0U, &PPC::G8RCRegClass);
       return std::make_pair(0U, &PPC::GPRCRegClass);
     case 'f':
-      if (VT == MVT::f32)
+      if (VT == MVT::f32 || VT == MVT::i32)
         return std::make_pair(0U, &PPC::F4RCRegClass);
-      if (VT == MVT::f64)
+      if (VT == MVT::f64 || VT == MVT::i64)
         return std::make_pair(0U, &PPC::F8RCRegClass);
       break;
     case 'v':
@@ -5870,7 +6730,8 @@ SDValue PPCTargetLowering::LowerFRAMEADDR(SDValue Op,
   bool is31 = (getTargetMachine().Options.DisableFramePointerElim(MF) ||
                MFI->hasVarSizedObjects()) &&
                   MFI->getStackSize() &&
-                  !MF.getFunction()->hasFnAttr(Attribute::Naked);
+                  !MF.getFunction()->getFnAttributes().
+                    hasAttribute(Attributes::Naked);
   unsigned FrameReg = isPPC64 ? (is31 ? PPC::X31 : PPC::X1) :
                                 (is31 ? PPC::R31 : PPC::R1);
   SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg,
diff --git a/lib/Target/PowerPC/PPCISelLowering.h b/lib/Target/PowerPC/PPCISelLowering.h
index b0a013b..b3c7f9c 100644
--- a/lib/Target/PowerPC/PPCISelLowering.h
+++ b/lib/Target/PowerPC/PPCISelLowering.h
@@ -174,6 +174,10 @@ namespace llvm {
       ///   operand #3 optional in flag
       TC_RETURN,
 
+      /// ch, gl = CR6[UN]SET ch, inglue - Toggle CR bit 6 for SVR4 vararg calls
+      CR6SET,
+      CR6UNSET,
+
       /// STD_32 - This is the STD instruction for use with "32-bit" registers.
       STD_32 = ISD::FIRST_TARGET_MEMORY_OPCODE,
 
@@ -463,20 +467,41 @@ namespace llvm {
                   DebugLoc dl, SelectionDAG &DAG) const;
 
     SDValue
+      extendArgForPPC64(ISD::ArgFlagsTy Flags, EVT ObjectVT, SelectionDAG &DAG,
+                        SDValue ArgVal, DebugLoc dl) const;
+
+    void
+      setMinReservedArea(MachineFunction &MF, SelectionDAG &DAG,
+                         unsigned nAltivecParamsAtEnd,
+                         unsigned MinReservedArea, bool isPPC64) const;
+
+    SDValue
       LowerFormalArguments_Darwin(SDValue Chain,
                                   CallingConv::ID CallConv, bool isVarArg,
                                   const SmallVectorImpl<ISD::InputArg> &Ins,
                                   DebugLoc dl, SelectionDAG &DAG,
                                   SmallVectorImpl<SDValue> &InVals) const;
     SDValue
-      LowerFormalArguments_SVR4(SDValue Chain,
-                                CallingConv::ID CallConv, bool isVarArg,
-                                const SmallVectorImpl<ISD::InputArg> &Ins,
-                                DebugLoc dl, SelectionDAG &DAG,
-                                SmallVectorImpl<SDValue> &InVals) const;
+      LowerFormalArguments_64SVR4(SDValue Chain,
+                                  CallingConv::ID CallConv, bool isVarArg,
+                                  const SmallVectorImpl<ISD::InputArg> &Ins,
+                                  DebugLoc dl, SelectionDAG &DAG,
+                                  SmallVectorImpl<SDValue> &InVals) const;
+    SDValue
+      LowerFormalArguments_32SVR4(SDValue Chain,
+                                  CallingConv::ID CallConv, bool isVarArg,
+                                  const SmallVectorImpl<ISD::InputArg> &Ins,
+                                  DebugLoc dl, SelectionDAG &DAG,
+                                  SmallVectorImpl<SDValue> &InVals) const;
+
+    SDValue
+      createMemcpyOutsideCallSeq(SDValue Arg, SDValue PtrOff,
+                                 SDValue CallSeqStart, ISD::ArgFlagsTy Flags,
+                                 SelectionDAG &DAG, DebugLoc dl) const;
 
     SDValue
-      LowerCall_Darwin(SDValue Chain, SDValue Callee, CallingConv::ID CallConv,
+      LowerCall_Darwin(SDValue Chain, SDValue Callee,
+                       CallingConv::ID CallConv,
                        bool isVarArg, bool isTailCall,
                        const SmallVectorImpl<ISD::OutputArg> &Outs,
                        const SmallVectorImpl<SDValue> &OutVals,
@@ -484,13 +509,22 @@ namespace llvm {
                        DebugLoc dl, SelectionDAG &DAG,
                        SmallVectorImpl<SDValue> &InVals) const;
     SDValue
-    LowerCall_SVR4(SDValue Chain, SDValue Callee, CallingConv::ID CallConv,
-                   bool isVarArg, bool isTailCall,
-                   const SmallVectorImpl<ISD::OutputArg> &Outs,
-                   const SmallVectorImpl<SDValue> &OutVals,
-                   const SmallVectorImpl<ISD::InputArg> &Ins,
-                   DebugLoc dl, SelectionDAG &DAG,
-                   SmallVectorImpl<SDValue> &InVals) const;
+      LowerCall_64SVR4(SDValue Chain, SDValue Callee,
+                       CallingConv::ID CallConv,
+                       bool isVarArg, bool isTailCall,
+                       const SmallVectorImpl<ISD::OutputArg> &Outs,
+                       const SmallVectorImpl<SDValue> &OutVals,
+                       const SmallVectorImpl<ISD::InputArg> &Ins,
+                       DebugLoc dl, SelectionDAG &DAG,
+                       SmallVectorImpl<SDValue> &InVals) const;
+    SDValue
+    LowerCall_32SVR4(SDValue Chain, SDValue Callee, CallingConv::ID CallConv,
+                     bool isVarArg, bool isTailCall,
+                     const SmallVectorImpl<ISD::OutputArg> &Outs,
+                     const SmallVectorImpl<SDValue> &OutVals,
+                     const SmallVectorImpl<ISD::InputArg> &Ins,
+                     DebugLoc dl, SelectionDAG &DAG,
+                     SmallVectorImpl<SDValue> &InVals) const;
   };
 }
 
diff --git a/lib/Target/PowerPC/PPCInstr64Bit.td b/lib/Target/PowerPC/PPCInstr64Bit.td
index 39778a5..9711452 100644
--- a/lib/Target/PowerPC/PPCInstr64Bit.td
+++ b/lib/Target/PowerPC/PPCInstr64Bit.td
@@ -29,6 +29,9 @@ def symbolLo64 : Operand<i64> {
   let PrintMethod = "printSymbolLo";
   let EncoderMethod = "getLO16Encoding";
 }
+def tocentry : Operand<iPTR> {
+  let MIOperandInfo = (ops i32imm:$imm);
+}
 
 //===----------------------------------------------------------------------===//
 // 64-bit transformation functions.
@@ -60,7 +63,7 @@ def HI48_64 : SDNodeXForm<imm, [{
 //
 
 let Defs = [LR8] in
-  def MovePCtoLR8 : Pseudo<(outs), (ins), "", []>,
+  def MovePCtoLR8 : Pseudo<(outs), (ins), "#MovePCtoLR8", []>,
                     PPC970_Unit_BRU;
 
 // Darwin ABI Calls.
@@ -138,31 +141,31 @@ def : Pat<(PPCnop),
 let usesCustomInserter = 1 in {
   let Defs = [CR0] in {
     def ATOMIC_LOAD_ADD_I64 : Pseudo<
-      (outs G8RC:$dst), (ins memrr:$ptr, G8RC:$incr), "",
+      (outs G8RC:$dst), (ins memrr:$ptr, G8RC:$incr), "#ATOMIC_LOAD_ADD_I64",
       [(set G8RC:$dst, (atomic_load_add_64 xoaddr:$ptr, G8RC:$incr))]>;
     def ATOMIC_LOAD_SUB_I64 : Pseudo<
-      (outs G8RC:$dst), (ins memrr:$ptr, G8RC:$incr), "",
+      (outs G8RC:$dst), (ins memrr:$ptr, G8RC:$incr), "#ATOMIC_LOAD_SUB_I64",
       [(set G8RC:$dst, (atomic_load_sub_64 xoaddr:$ptr, G8RC:$incr))]>;
     def ATOMIC_LOAD_OR_I64 : Pseudo<
-      (outs G8RC:$dst), (ins memrr:$ptr, G8RC:$incr), "",
+      (outs G8RC:$dst), (ins memrr:$ptr, G8RC:$incr), "#ATOMIC_LOAD_OR_I64",
       [(set G8RC:$dst, (atomic_load_or_64 xoaddr:$ptr, G8RC:$incr))]>;
     def ATOMIC_LOAD_XOR_I64 : Pseudo<
-      (outs G8RC:$dst), (ins memrr:$ptr, G8RC:$incr), "",
+      (outs G8RC:$dst), (ins memrr:$ptr, G8RC:$incr), "#ATOMIC_LOAD_XOR_I64",
       [(set G8RC:$dst, (atomic_load_xor_64 xoaddr:$ptr, G8RC:$incr))]>;
     def ATOMIC_LOAD_AND_I64 : Pseudo<
-      (outs G8RC:$dst), (ins memrr:$ptr, G8RC:$incr), "",
+      (outs G8RC:$dst), (ins memrr:$ptr, G8RC:$incr), "#ATOMIC_LOAD_AND_i64",
       [(set G8RC:$dst, (atomic_load_and_64 xoaddr:$ptr, G8RC:$incr))]>;
     def ATOMIC_LOAD_NAND_I64 : Pseudo<
-      (outs G8RC:$dst), (ins memrr:$ptr, G8RC:$incr), "",
+      (outs G8RC:$dst), (ins memrr:$ptr, G8RC:$incr), "#ATOMIC_LOAD_NAND_I64",
       [(set G8RC:$dst, (atomic_load_nand_64 xoaddr:$ptr, G8RC:$incr))]>;
 
     def ATOMIC_CMP_SWAP_I64 : Pseudo<
-      (outs G8RC:$dst), (ins memrr:$ptr, G8RC:$old, G8RC:$new), "",
+      (outs G8RC:$dst), (ins memrr:$ptr, G8RC:$old, G8RC:$new), "#ATOMIC_CMP_SWAP_I64",
       [(set G8RC:$dst, 
                     (atomic_cmp_swap_64 xoaddr:$ptr, G8RC:$old, G8RC:$new))]>;
 
     def ATOMIC_SWAP_I64 : Pseudo<
-      (outs G8RC:$dst), (ins memrr:$ptr, G8RC:$new), "",
+      (outs G8RC:$dst), (ins memrr:$ptr, G8RC:$new), "#ATOMIC_SWAP_I64",
       [(set G8RC:$dst, (atomic_swap_64 xoaddr:$ptr, G8RC:$new))]>;
   }
 }
@@ -231,10 +234,10 @@ def : Pat<(PPCtc_return CTRRC8:$dst, imm:$imm),
 
 let isBranch = 1, isTerminator = 1, hasCtrlDep = 1, PPC970_Unit = 7 in {
   let Defs = [CTR8], Uses = [CTR8] in {
-    def BDZ8  : IForm_ext<16, 18, 0, 0, (outs), (ins condbrtarget:$dst),
-                         "bdz $dst",  BrB, []>;
-    def BDNZ8 : IForm_ext<16, 16, 0, 0, (outs), (ins condbrtarget:$dst),
-                         "bdnz $dst", BrB, []>;
+    def BDZ8  : BForm_1<16, 18, 0, 0, (outs), (ins condbrtarget:$dst),
+                        "bdz $dst">;
+    def BDNZ8 : BForm_1<16, 16, 0, 0, (outs), (ins condbrtarget:$dst),
+                        "bdnz $dst">;
   }
 }
 
@@ -244,7 +247,7 @@ def MTCRF8 : XFXForm_5<31, 144, (outs crbitm:$FXM), (ins G8RC:$rS),
             PPC970_MicroCode, PPC970_Unit_CRU;
 
 def MFCR8pseud: XFXForm_3<31, 19, (outs G8RC:$rT), (ins crbitm:$FXM),
-                       "", SprMFCR>,
+                       "#MFCR8pseud", SprMFCR>,
             PPC970_MicroCode, PPC970_Unit_CRU;
             
 def MFCR8 : XFXForm_3<31, 19, (outs G8RC:$rT), (ins),
@@ -275,7 +278,7 @@ def MFTB8 : XFXForm_1_ext<31, 339, 268, (outs G8RC:$rT), (ins),
 // the POWER3.
 
 let Defs = [X1], Uses = [X1] in
-def DYNALLOC8 : Pseudo<(outs G8RC:$result), (ins G8RC:$negsize, memri:$fpsi),"",
+def DYNALLOC8 : Pseudo<(outs G8RC:$result), (ins G8RC:$negsize, memri:$fpsi),"#DYNALLOC8",
                        [(set G8RC:$result,
                              (PPCdynalloc G8RC:$negsize, iaddr:$fpsi))]>;
 
@@ -296,12 +299,14 @@ def MFLR8  : XFXForm_1_ext<31, 339, 8, (outs G8RC:$rT), (ins),
 
 let PPC970_Unit = 1 in {  // FXU Operations.
 
+let isReMaterializable = 1, isAsCheapAsAMove = 1, isMoveImm = 1 in {
 def LI8  : DForm_2_r0<14, (outs G8RC:$rD), (ins symbolLo64:$imm),
                       "li $rD, $imm", IntSimple,
                       [(set G8RC:$rD, immSExt16:$imm)]>;
 def LIS8 : DForm_2_r0<15, (outs G8RC:$rD), (ins symbolHi64:$imm),
                       "lis $rD, $imm", IntSimple,
                       [(set G8RC:$rD, imm16ShiftedSExt:$imm)]>;
+}
 
 // Logical ops.
 def NAND8: XForm_6<31, 476, (outs G8RC:$rA), (ins G8RC:$rS, G8RC:$rB),
@@ -459,7 +464,7 @@ def EXTSW_32_64 : XForm_11<31, 986, (outs G8RC:$rA), (ins GPRC:$rS),
 
 let Defs = [CARRY] in {
 def SRADI  : XSForm_1<31, 413, (outs G8RC:$rA), (ins G8RC:$rS, u6imm:$SH),
-                      "sradi $rA, $rS, $SH", IntRotateD,
+                      "sradi $rA, $rS, $SH", IntRotateDI,
                       [(set G8RC:$rA, (sra G8RC:$rS, (i32 imm:$SH)))]>, isPPC64;
 }
 def CNTLZD : XForm_11<31, 58, (outs G8RC:$rA), (ins G8RC:$rS),
@@ -482,23 +487,23 @@ def MULLD : XOForm_1<31, 233, 0, (outs G8RC:$rT), (ins G8RC:$rA, G8RC:$rB),
 let isCommutable = 1 in {
 def RLDIMI : MDForm_1<30, 3,
                       (outs G8RC:$rA), (ins G8RC:$rSi, G8RC:$rS, u6imm:$SH, u6imm:$MB),
-                      "rldimi $rA, $rS, $SH, $MB", IntRotateD,
+                      "rldimi $rA, $rS, $SH, $MB", IntRotateDI,
                       []>, isPPC64, RegConstraint<"$rSi = $rA">,
                       NoEncode<"$rSi">;
 }
 
 // Rotate instructions.
 def RLDCL  : MDForm_1<30, 0,
-                      (outs G8RC:$rA), (ins G8RC:$rS, GPRC:$rB, u6imm:$MB),
-                      "rldcl $rA, $rS, $rB, $MB", IntRotateD,
+                      (outs G8RC:$rA), (ins G8RC:$rS, GPRC:$rB, u6imm:$MBE),
+                      "rldcl $rA, $rS, $rB, $MBE", IntRotateD,
                       []>, isPPC64;
 def RLDICL : MDForm_1<30, 0,
-                      (outs G8RC:$rA), (ins G8RC:$rS, u6imm:$SH, u6imm:$MB),
-                      "rldicl $rA, $rS, $SH, $MB", IntRotateD,
+                      (outs G8RC:$rA), (ins G8RC:$rS, u6imm:$SH, u6imm:$MBE),
+                      "rldicl $rA, $rS, $SH, $MBE", IntRotateDI,
                       []>, isPPC64;
 def RLDICR : MDForm_1<30, 1,
-                      (outs G8RC:$rA), (ins G8RC:$rS, u6imm:$SH, u6imm:$ME),
-                      "rldicr $rA, $rS, $SH, $ME", IntRotateD,
+                      (outs G8RC:$rA), (ins G8RC:$rS, u6imm:$SH, u6imm:$MBE),
+                      "rldicr $rA, $rS, $SH, $MBE", IntRotateDI,
                       []>, isPPC64;
 
 def RLWINM8 : MForm_2<21,
@@ -506,7 +511,7 @@ def RLWINM8 : MForm_2<21,
                      "rlwinm $rA, $rS, $SH, $MB, $ME", IntGeneral,
                      []>;
 
-def ISEL8   : AForm_1<31, 15,
+def ISEL8   : AForm_4<31, 15,
                      (outs G8RC:$rT), (ins G8RC:$rA, G8RC:$rB, pred:$cond),
                      "isel $rT, $rA, $rB, $cond", IntGeneral,
                      []>;
@@ -541,19 +546,19 @@ def LWAX : XForm_1<31, 341, (outs G8RC:$rD), (ins memrr:$src),
 let mayLoad = 1 in
 def LHAU8 : DForm_1a<43, (outs G8RC:$rD, ptr_rc:$ea_result), (ins symbolLo:$disp,
                             ptr_rc:$rA),
-                    "lhau $rD, $disp($rA)", LdStLoad,
+                    "lhau $rD, $disp($rA)", LdStLHAU,
                     []>, RegConstraint<"$rA = $ea_result">,
                     NoEncode<"$ea_result">;
 // NO LWAU!
 
 def LHAUX8 : XForm_1<31, 375, (outs G8RC:$rD, ptr_rc:$ea_result),
                     (ins memrr:$addr),
-                    "lhaux $rD, $addr", LdStLoad,
+                    "lhaux $rD, $addr", LdStLHAU,
                     []>, RegConstraint<"$addr.offreg = $ea_result">,
                     NoEncode<"$ea_result">;
-def LWAUX : XForm_1<31, 375, (outs G8RC:$rD, ptr_rc:$ea_result),
+def LWAUX : XForm_1<31, 373, (outs G8RC:$rD, ptr_rc:$ea_result),
                     (ins memrr:$addr),
-                    "lwaux $rD, $addr", LdStLoad,
+                    "lwaux $rD, $addr", LdStLHAU,
                     []>, RegConstraint<"$addr.offreg = $ea_result">,
                     NoEncode<"$ea_result">, isPPC64;
 }
@@ -584,31 +589,31 @@ def LWZX8 : XForm_1<31,  23, (outs G8RC:$rD), (ins memrr:$src),
 // Update forms.
 let mayLoad = 1 in {
 def LBZU8 : DForm_1<35, (outs G8RC:$rD, ptr_rc:$ea_result), (ins memri:$addr),
-                    "lbzu $rD, $addr", LdStLoad,
+                    "lbzu $rD, $addr", LdStLoadUpd,
                     []>, RegConstraint<"$addr.reg = $ea_result">,
                     NoEncode<"$ea_result">;
 def LHZU8 : DForm_1<41, (outs G8RC:$rD, ptr_rc:$ea_result), (ins memri:$addr),
-                    "lhzu $rD, $addr", LdStLoad,
+                    "lhzu $rD, $addr", LdStLoadUpd,
                     []>, RegConstraint<"$addr.reg = $ea_result">,
                     NoEncode<"$ea_result">;
 def LWZU8 : DForm_1<33, (outs G8RC:$rD, ptr_rc:$ea_result), (ins memri:$addr),
-                    "lwzu $rD, $addr", LdStLoad,
+                    "lwzu $rD, $addr", LdStLoadUpd,
                     []>, RegConstraint<"$addr.reg = $ea_result">,
                     NoEncode<"$ea_result">;
 
 def LBZUX8 : XForm_1<31, 119, (outs G8RC:$rD, ptr_rc:$ea_result),
                    (ins memrr:$addr),
-                   "lbzux $rD, $addr", LdStLoad,
+                   "lbzux $rD, $addr", LdStLoadUpd,
                    []>, RegConstraint<"$addr.offreg = $ea_result">,
                    NoEncode<"$ea_result">;
-def LHZUX8 : XForm_1<31, 331, (outs G8RC:$rD, ptr_rc:$ea_result),
+def LHZUX8 : XForm_1<31, 311, (outs G8RC:$rD, ptr_rc:$ea_result),
                    (ins memrr:$addr),
-                   "lhzux $rD, $addr", LdStLoad,
+                   "lhzux $rD, $addr", LdStLoadUpd,
                    []>, RegConstraint<"$addr.offreg = $ea_result">,
                    NoEncode<"$ea_result">;
 def LWZUX8 : XForm_1<31, 55, (outs G8RC:$rD, ptr_rc:$ea_result),
                    (ins memrr:$addr),
-                   "lwzux $rD, $addr", LdStLoad,
+                   "lwzux $rD, $addr", LdStLoadUpd,
                    []>, RegConstraint<"$addr.offreg = $ea_result">,
                    NoEncode<"$ea_result">;
 }
@@ -621,18 +626,26 @@ def LD   : DSForm_1<58, 0, (outs G8RC:$rD), (ins memrix:$src),
                     "ld $rD, $src", LdStLD,
                     [(set G8RC:$rD, (load ixaddr:$src))]>, isPPC64;
 def LDtoc: Pseudo<(outs G8RC:$rD), (ins tocentry:$disp, G8RC:$reg),
-                  "",
+                  "#LDtoc",
                   [(set G8RC:$rD,
                      (PPCtoc_entry tglobaladdr:$disp, G8RC:$reg))]>, isPPC64;
+def LDtocJTI: Pseudo<(outs G8RC:$rD), (ins tocentry:$disp, G8RC:$reg),
+                  "#LDtocJTI",
+                  [(set G8RC:$rD,
+                     (PPCtoc_entry tjumptable:$disp, G8RC:$reg))]>, isPPC64;
+def LDtocCPT: Pseudo<(outs G8RC:$rD), (ins tocentry:$disp, G8RC:$reg),
+                  "#LDtocCPT",
+                  [(set G8RC:$rD,
+                     (PPCtoc_entry tconstpool:$disp, G8RC:$reg))]>, isPPC64;
 
 let hasSideEffects = 1 in { 
-let RST = 2, DS_RA = 0 in // FIXME: Should be a pseudo.
-def LDinto_toc: DSForm_1<58, 0, (outs), (ins G8RC:$reg),
+let RST = 2, DS = 2 in
+def LDinto_toc: DSForm_1a<58, 0, (outs), (ins G8RC:$reg),
                     "ld 2, 8($reg)", LdStLD,
                     [(PPCload_toc G8RC:$reg)]>, isPPC64;
                     
-let RST = 2, DS_RA = 0 in // FIXME: Should be a pseudo.
-def LDtoc_restore : DSForm_1<58, 0, (outs), (ins),
+let RST = 2, DS = 10, RA = 1 in
+def LDtoc_restore : DSForm_1a<58, 0, (outs), (ins),
                     "ld 2, 40(1)", LdStLD,
                     [(PPCtoc_restore)]>, isPPC64;
 }
@@ -642,13 +655,13 @@ def LDX  : XForm_1<31,  21, (outs G8RC:$rD), (ins memrr:$src),
                    
 let mayLoad = 1 in
 def LDU  : DSForm_1<58, 1, (outs G8RC:$rD, ptr_rc:$ea_result), (ins memrix:$addr),
-                    "ldu $rD, $addr", LdStLD,
+                    "ldu $rD, $addr", LdStLDU,
                     []>, RegConstraint<"$addr.reg = $ea_result">, isPPC64,
                     NoEncode<"$ea_result">;
 
 def LDUX : XForm_1<31, 53, (outs G8RC:$rD, ptr_rc:$ea_result),
                    (ins memrr:$addr),
-                   "ldux $rD, $addr", LdStLoad,
+                   "ldux $rD, $addr", LdStLDU,
                    []>, RegConstraint<"$addr.offreg = $ea_result">,
                    NoEncode<"$ea_result">, isPPC64;
 }
@@ -693,16 +706,16 @@ def STDX  : XForm_8<31, 149, (outs), (ins G8RC:$rS, memrr:$dst),
 
 let PPC970_Unit = 2 in {
 
-def STBU8 : DForm_1a<38, (outs ptr_rc:$ea_res), (ins G8RC:$rS,
+def STBU8 : DForm_1a<39, (outs ptr_rc:$ea_res), (ins G8RC:$rS,
                              symbolLo:$ptroff, ptr_rc:$ptrreg),
-                    "stbu $rS, $ptroff($ptrreg)", LdStStore,
+                    "stbu $rS, $ptroff($ptrreg)", LdStStoreUpd,
                     [(set ptr_rc:$ea_res,
                           (pre_truncsti8 G8RC:$rS, ptr_rc:$ptrreg, 
                                          iaddroff:$ptroff))]>,
                     RegConstraint<"$ptrreg = $ea_res">, NoEncode<"$ea_res">;
 def STHU8 : DForm_1a<45, (outs ptr_rc:$ea_res), (ins G8RC:$rS,
                              symbolLo:$ptroff, ptr_rc:$ptrreg),
-                    "sthu $rS, $ptroff($ptrreg)", LdStStore,
+                    "sthu $rS, $ptroff($ptrreg)", LdStStoreUpd,
                     [(set ptr_rc:$ea_res,
                         (pre_truncsti16 G8RC:$rS, ptr_rc:$ptrreg, 
                                         iaddroff:$ptroff))]>,
@@ -710,7 +723,7 @@ def STHU8 : DForm_1a<45, (outs ptr_rc:$ea_res), (ins G8RC:$rS,
 
 def STWU8 : DForm_1a<37, (outs ptr_rc:$ea_res), (ins G8RC:$rS,
                              symbolLo:$ptroff, ptr_rc:$ptrreg),
-                    "stwu $rS, $ptroff($ptrreg)", LdStStore,
+                    "stwu $rS, $ptroff($ptrreg)", LdStStoreUpd,
                     [(set ptr_rc:$ea_res,
                           (pre_truncsti32 G8RC:$rS, ptr_rc:$ptrreg,
                                           iaddroff:$ptroff))]>,
@@ -718,7 +731,7 @@ def STWU8 : DForm_1a<37, (outs ptr_rc:$ea_res), (ins G8RC:$rS,
 
 def STDU : DSForm_1a<62, 1, (outs ptr_rc:$ea_res), (ins G8RC:$rS,
                                         s16immX4:$ptroff, ptr_rc:$ptrreg),
-                    "stdu $rS, $ptroff($ptrreg)", LdStSTD,
+                    "stdu $rS, $ptroff($ptrreg)", LdStSTDU,
                     [(set ptr_rc:$ea_res, (pre_store G8RC:$rS, ptr_rc:$ptrreg, 
                                                      iaddroff:$ptroff))]>,
                     RegConstraint<"$ptrreg = $ea_res">, NoEncode<"$ea_res">,
@@ -727,7 +740,7 @@ def STDU : DSForm_1a<62, 1, (outs ptr_rc:$ea_res), (ins G8RC:$rS,
 
 def STBUX8 : XForm_8<31, 247, (outs ptr_rc:$ea_res),
                               (ins G8RC:$rS, ptr_rc:$ptroff, ptr_rc:$ptrreg),
-                    "stbux $rS, $ptroff, $ptrreg", LdStStore,
+                    "stbux $rS, $ptroff, $ptrreg", LdStStoreUpd,
                     [(set ptr_rc:$ea_res,
                        (pre_truncsti8 G8RC:$rS,
                                       ptr_rc:$ptrreg, xaddroff:$ptroff))]>,
@@ -736,7 +749,7 @@ def STBUX8 : XForm_8<31, 247, (outs ptr_rc:$ea_res),
 
 def STHUX8 : XForm_8<31, 439, (outs ptr_rc:$ea_res),
                               (ins G8RC:$rS, ptr_rc:$ptroff, ptr_rc:$ptrreg),
-                    "sthux $rS, $ptroff, $ptrreg", LdStStore,
+                    "sthux $rS, $ptroff, $ptrreg", LdStStoreUpd,
                     [(set ptr_rc:$ea_res,
                        (pre_truncsti16 G8RC:$rS,
                                        ptr_rc:$ptrreg, xaddroff:$ptroff))]>,
@@ -745,7 +758,7 @@ def STHUX8 : XForm_8<31, 439, (outs ptr_rc:$ea_res),
 
 def STWUX8 : XForm_8<31, 183, (outs ptr_rc:$ea_res),
                               (ins G8RC:$rS, ptr_rc:$ptroff, ptr_rc:$ptrreg),
-                    "stwux $rS, $ptroff, $ptrreg", LdStStore,
+                    "stwux $rS, $ptroff, $ptrreg", LdStStoreUpd,
                     [(set ptr_rc:$ea_res,
                        (pre_truncsti32 G8RC:$rS,
                                        ptr_rc:$ptrreg, xaddroff:$ptroff))]>,
@@ -754,7 +767,7 @@ def STWUX8 : XForm_8<31, 183, (outs ptr_rc:$ea_res),
 
 def STDUX : XForm_8<31, 181, (outs ptr_rc:$ea_res),
                               (ins G8RC:$rS, ptr_rc:$ptroff, ptr_rc:$ptrreg),
-                    "stdux $rS, $ptroff, $ptrreg", LdStStore,
+                    "stdux $rS, $ptroff, $ptrreg", LdStSTDU,
                     [(set ptr_rc:$ea_res,
                        (pre_store G8RC:$rS, ptr_rc:$ptrreg, xaddroff:$ptroff))]>,
                     RegConstraint<"$ptroff = $ea_res">, NoEncode<"$ea_res">,
diff --git a/lib/Target/PowerPC/PPCInstrAltivec.td b/lib/Target/PowerPC/PPCInstrAltivec.td
index b0b8423..ba58c3e 100644
--- a/lib/Target/PowerPC/PPCInstrAltivec.td
+++ b/lib/Target/PowerPC/PPCInstrAltivec.td
@@ -340,6 +340,28 @@ def VCTUXS : VXForm_1<906, (outs VRRC:$vD), (ins u5imm:$UIMM, VRRC:$vB),
                       "vctuxs $vD, $vB, $UIMM", VecFP,
                       [(set VRRC:$vD,
                              (int_ppc_altivec_vctuxs VRRC:$vB, imm:$UIMM))]>;
+
+// Defines with the UIM field set to 0 for floating-point
+// to integer (fp_to_sint/fp_to_uint) conversions and integer
+// to floating-point (sint_to_fp/uint_to_fp) conversions.
+let VA = 0 in {
+def VCFSX_0 : VXForm_1<842, (outs VRRC:$vD), (ins VRRC:$vB),
+                       "vcfsx $vD, $vB, 0", VecFP,
+                       [(set VRRC:$vD,
+                             (int_ppc_altivec_vcfsx VRRC:$vB, 0))]>;
+def VCTUXS_0 : VXForm_1<906, (outs VRRC:$vD), (ins VRRC:$vB),
+                        "vctuxs $vD, $vB, 0", VecFP,
+                        [(set VRRC:$vD,
+                               (int_ppc_altivec_vctuxs VRRC:$vB, 0))]>;
+def VCFUX_0 : VXForm_1<778, (outs VRRC:$vD), (ins VRRC:$vB),
+                       "vcfux $vD, $vB, 0", VecFP,
+                       [(set VRRC:$vD,
+                               (int_ppc_altivec_vcfux VRRC:$vB, 0))]>;
+def VCTSXS_0 : VXForm_1<970, (outs VRRC:$vD), (ins VRRC:$vB),
+                      "vctsxs $vD, $vB, 0", VecFP,
+                      [(set VRRC:$vD,
+                             (int_ppc_altivec_vctsxs VRRC:$vB, 0))]>;
+}
 def VEXPTEFP : VX2_Int<394, "vexptefp", int_ppc_altivec_vexptefp>;
 def VLOGEFP  : VX2_Int<458, "vlogefp",  int_ppc_altivec_vlogefp>;
 
@@ -689,3 +711,13 @@ def : Pat<(v8i16 (sra (v8i16 VRRC:$vA), (v8i16 VRRC:$vB))),
           (v8i16 (VSRAH VRRC:$vA, VRRC:$vB))>;
 def : Pat<(v4i32 (sra (v4i32 VRRC:$vA), (v4i32 VRRC:$vB))),
           (v4i32 (VSRAW VRRC:$vA, VRRC:$vB))>;
+
+// Float to integer and integer to float conversions
+def : Pat<(v4i32 (fp_to_sint (v4f32 VRRC:$vA))),
+           (VCTSXS_0 VRRC:$vA)>;
+def : Pat<(v4i32 (fp_to_uint (v4f32 VRRC:$vA))),
+           (VCTUXS_0 VRRC:$vA)>;
+def : Pat<(v4f32 (sint_to_fp (v4i32 VRRC:$vA))),
+           (VCFSX_0 VRRC:$vA)>;
+def : Pat<(v4f32 (uint_to_fp (v4i32 VRRC:$vA))),
+           (VCFUX_0 VRRC:$vA)>;
diff --git a/lib/Target/PowerPC/PPCInstrFormats.td b/lib/Target/PowerPC/PPCInstrFormats.td
index a41a027..c3c171c 100644
--- a/lib/Target/PowerPC/PPCInstrFormats.td
+++ b/lib/Target/PowerPC/PPCInstrFormats.td
@@ -94,12 +94,6 @@ class IForm<bits<6> opcode, bit aa, bit lk, dag OOL, dag IOL, string asmstr,
   let Inst{31}    = lk;
 }
 
-class IForm_ext<bits<6> opcode, bits<5> bo, bit aa, bit lk, dag OOL, dag IOL,
-            string asmstr, InstrItinClass itin, list<dag> pattern>
-         : IForm<opcode, aa, lk, OOL, IOL, asmstr, itin, pattern> {
-  let LI{0-4} = bo;
-}
-
 // 1.7.2 B-Form
 class BForm<bits<6> opcode, bit aa, bit lk, dag OOL, dag IOL, string asmstr>
   : I<opcode, OOL, IOL, asmstr, BrB> {
@@ -118,6 +112,13 @@ class BForm<bits<6> opcode, bit aa, bit lk, dag OOL, dag IOL, string asmstr>
   let Inst{31}    = lk;
 }
 
+class BForm_1<bits<6> opcode, bits<5> bo, bit aa, bit lk, dag OOL, dag IOL,
+             string asmstr>
+  : BForm<opcode, aa, lk, OOL, IOL, asmstr> {
+  let BIBO{4-0} = bo;
+  let BIBO{6-5} = 0;
+  let CR = 0;
+}
 
 // 1.7.4 D-Form
 class DForm_base<bits<6> opcode, dag OOL, dag IOL, string asmstr,
@@ -625,9 +626,9 @@ class XFXForm_5<bits<6> opcode, bits<10> xo, dag OOL, dag IOL, string asmstr,
                 InstrItinClass itin> 
   : I<opcode, OOL, IOL, asmstr, itin> {
   bits<8>  FXM;
-  bits<5>  ST;
+  bits<5>  rS;
    
-  let Inst{6-10}  = ST;
+  let Inst{6-10}  = rS;
   let Inst{11}    = 0;
   let Inst{12-19} = FXM;
   let Inst{20}    = 0;
@@ -666,7 +667,7 @@ class XFLForm<bits<6> opcode, bits<10> xo, dag OOL, dag IOL, string asmstr,
                       string cstr, InstrItinClass itin, list<dag>pattern>
   : I<opcode, OOL, IOL, asmstr, itin> {
   bits<8> FM;
-  bits<5> RT;
+  bits<5> rT;
 
   bit RC = 0;    // set by isDOT
   let Pattern = pattern;
@@ -675,7 +676,7 @@ class XFLForm<bits<6> opcode, bits<10> xo, dag OOL, dag IOL, string asmstr,
   let Inst{6} = 0;
   let Inst{7-14}  = FM;
   let Inst{15} = 0;
-  let Inst{16-20} = RT;
+  let Inst{16-20} = rT;
   let Inst{21-30} = xo;
   let Inst{31}    = RC;
 }
@@ -758,6 +759,26 @@ class AForm_3<bits<6> opcode, bits<5> xo, dag OOL, dag IOL, string asmstr,
   let FRB = 0;
 }
 
+class AForm_4<bits<6> opcode, bits<5> xo, dag OOL, dag IOL, string asmstr, 
+              InstrItinClass itin, list<dag> pattern>
+         : I<opcode, OOL, IOL, asmstr, itin> {
+  bits<5> RT;
+  bits<5> RA;
+  bits<5> RB;
+  bits<7> BIBO;  // 2 bits of BI and 5 bits of BO (must be 12).
+  bits<3> CR;
+
+  let Pattern = pattern;
+
+  let Inst{6-10}  = RT;
+  let Inst{11-15} = RA;
+  let Inst{16-20} = RB;
+  let Inst{21-23} = CR;
+  let Inst{24-25} = BIBO{6-5};
+  let Inst{26-30} = xo;
+  let Inst{31}    = 0;
+}
+
 // 1.7.13 M-Form
 class MForm_1<bits<6> opcode, dag OOL, dag IOL, string asmstr,
               InstrItinClass itin, list<dag> pattern>
diff --git a/lib/Target/PowerPC/PPCInstrInfo.cpp b/lib/Target/PowerPC/PPCInstrInfo.cpp
index 47f09dc..d9d6844 100644
--- a/lib/Target/PowerPC/PPCInstrInfo.cpp
+++ b/lib/Target/PowerPC/PPCInstrInfo.cpp
@@ -54,7 +54,8 @@ ScheduleHazardRecognizer *PPCInstrInfo::CreateTargetHazardRecognizer(
   const TargetMachine *TM,
   const ScheduleDAG *DAG) const {
   unsigned Directive = TM->getSubtarget<PPCSubtarget>().getDarwinDirective();
-  if (Directive == PPC::DIR_440 || Directive == PPC::DIR_A2) {
+  if (Directive == PPC::DIR_440 || Directive == PPC::DIR_A2 ||
+      Directive == PPC::DIR_E500mc || Directive == PPC::DIR_E5500) {
     const InstrItineraryData *II = TM->getInstrItineraryData();
     return new PPCScoreboardHazardRecognizer(II, DAG);
   }
@@ -70,7 +71,8 @@ ScheduleHazardRecognizer *PPCInstrInfo::CreateTargetPostRAHazardRecognizer(
   unsigned Directive = TM.getSubtarget<PPCSubtarget>().getDarwinDirective();
 
   // Most subtargets use a PPC970 recognizer.
-  if (Directive != PPC::DIR_440 && Directive != PPC::DIR_A2) {
+  if (Directive != PPC::DIR_440 && Directive != PPC::DIR_A2 &&
+      Directive != PPC::DIR_E500mc && Directive != PPC::DIR_E5500) {
     const TargetInstrInfo *TII = TM.getInstrInfo();
     assert(TII && "No InstrInfo?");
 
@@ -568,12 +570,15 @@ PPCInstrInfo::StoreRegToStackSlot(MachineFunction &MF,
     // STVX VAL, 0, R0
     //
     // FIXME: We use R0 here, because it isn't available for RA.
-    NewMIs.push_back(addFrameReference(BuildMI(MF, DL, get(PPC::ADDI), PPC::R0),
+    bool Is64Bit = TM.getSubtargetImpl()->isPPC64();
+    unsigned Instr = Is64Bit ? PPC::ADDI8 : PPC::ADDI;
+    unsigned GPR0  = Is64Bit ? PPC::X0    : PPC::R0;
+    NewMIs.push_back(addFrameReference(BuildMI(MF, DL, get(Instr), GPR0),
                                        FrameIdx, 0, 0));
     NewMIs.push_back(BuildMI(MF, DL, get(PPC::STVX))
                      .addReg(SrcReg, getKillRegState(isKill))
-                     .addReg(PPC::R0)
-                     .addReg(PPC::R0));
+                     .addReg(GPR0)
+                     .addReg(GPR0));
   } else {
     llvm_unreachable("Unknown regclass!");
   }
@@ -705,10 +710,13 @@ PPCInstrInfo::LoadRegFromStackSlot(MachineFunction &MF, DebugLoc DL,
     // Dest = LVX 0, R0
     //
     // FIXME: We use R0 here, because it isn't available for RA.
-    NewMIs.push_back(addFrameReference(BuildMI(MF, DL, get(PPC::ADDI), PPC::R0),
+    bool Is64Bit = TM.getSubtargetImpl()->isPPC64();
+    unsigned Instr = Is64Bit ? PPC::ADDI8 : PPC::ADDI;
+    unsigned GPR0  = Is64Bit ? PPC::X0    : PPC::R0;
+    NewMIs.push_back(addFrameReference(BuildMI(MF, DL, get(Instr), GPR0),
                                        FrameIdx, 0, 0));
-    NewMIs.push_back(BuildMI(MF, DL, get(PPC::LVX),DestReg).addReg(PPC::R0)
-                     .addReg(PPC::R0));
+    NewMIs.push_back(BuildMI(MF, DL, get(PPC::LVX),DestReg).addReg(GPR0)
+                     .addReg(GPR0));
   } else {
     llvm_unreachable("Unknown regclass!");
   }
diff --git a/lib/Target/PowerPC/PPCInstrInfo.td b/lib/Target/PowerPC/PPCInstrInfo.td
index f57f0c9..6ee045a 100644
--- a/lib/Target/PowerPC/PPCInstrInfo.td
+++ b/lib/Target/PowerPC/PPCInstrInfo.td
@@ -123,9 +123,11 @@ def PPCnop : SDNode<"PPCISD::NOP", SDT_PPCnop, [SDNPInGlue, SDNPOutGlue]>;
 def PPCload   : SDNode<"PPCISD::LOAD", SDTypeProfile<1, 1, []>,
                        [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue]>;
 def PPCload_toc : SDNode<"PPCISD::LOAD_TOC", SDTypeProfile<0, 1, []>,
-                          [SDNPHasChain, SDNPInGlue, SDNPOutGlue]>;
+                          [SDNPHasChain, SDNPSideEffect,
+                           SDNPInGlue, SDNPOutGlue]>;
 def PPCtoc_restore : SDNode<"PPCISD::TOC_RESTORE", SDTypeProfile<0, 0, []>,
-                            [SDNPHasChain, SDNPInGlue, SDNPOutGlue]>;
+                            [SDNPHasChain, SDNPSideEffect,
+                             SDNPInGlue, SDNPOutGlue]>;
 def PPCmtctr      : SDNode<"PPCISD::MTCTR", SDT_PPCCall,
                            [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue]>;
 def PPCbctrl_Darwin  : SDNode<"PPCISD::BCTRL_Darwin", SDTNone,
@@ -153,6 +155,12 @@ def PPClbrx       : SDNode<"PPCISD::LBRX", SDT_PPClbrx,
 def PPCstbrx      : SDNode<"PPCISD::STBRX", SDT_PPCstbrx,
                            [SDNPHasChain, SDNPMayStore]>;
 
+// Instructions to set/unset CR bit 6 for SVR4 vararg calls
+def PPCcr6set   : SDNode<"PPCISD::CR6SET", SDTNone,
+                         [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue]>;
+def PPCcr6unset : SDNode<"PPCISD::CR6UNSET", SDTNone,
+                         [SDNPHasChain, SDNPOptInGlue, SDNPOutGlue]>;
+
 // Instructions to support atomic operations
 def PPClarx      : SDNode<"PPCISD::LARX", SDT_PPClarx,
                           [SDNPHasChain, SDNPMayLoad]>;
@@ -330,9 +338,6 @@ def memrix : Operand<iPTR> {   // memri where the imm is shifted 2 bits.
   let MIOperandInfo = (ops i32imm:$imm, ptr_rc:$reg);
   let EncoderMethod = "getMemRIXEncoding";
 }
-def tocentry : Operand<iPTR> {
-  let MIOperandInfo = (ops i32imm:$imm);
-}
 
 // PowerPC Predicate operand.  20 = (0<<5)|20 = always, CR0 is a dummy reg
 // that doesn't matter.
@@ -364,9 +369,9 @@ def IsBookE  : Predicate<"PPCSubTarget.isBookE()">;
 
 let hasCtrlDep = 1 in {
 let Defs = [R1], Uses = [R1] in {
-def ADJCALLSTACKDOWN : Pseudo<(outs), (ins u16imm:$amt), "",
+def ADJCALLSTACKDOWN : Pseudo<(outs), (ins u16imm:$amt), "#ADJCALLSTACKDOWN $amt",
                               [(callseq_start timm:$amt)]>;
-def ADJCALLSTACKUP   : Pseudo<(outs), (ins u16imm:$amt1, u16imm:$amt2), "",
+def ADJCALLSTACKUP   : Pseudo<(outs), (ins u16imm:$amt1, u16imm:$amt2), "#ADJCALLSTACKUP $amt1 $amt2",
                               [(callseq_end timm:$amt1, timm:$amt2)]>;
 }
 
@@ -375,7 +380,7 @@ def UPDATE_VRSAVE    : Pseudo<(outs GPRC:$rD), (ins GPRC:$rS),
 }
 
 let Defs = [R1], Uses = [R1] in
-def DYNALLOC : Pseudo<(outs GPRC:$result), (ins GPRC:$negsize, memri:$fpsi), "",
+def DYNALLOC : Pseudo<(outs GPRC:$result), (ins GPRC:$negsize, memri:$fpsi), "#DYNALLOC",
                        [(set GPRC:$result,
                              (PPCdynalloc GPRC:$negsize, iaddr:$fpsi))]>;
                          
@@ -384,19 +389,19 @@ def DYNALLOC : Pseudo<(outs GPRC:$result), (ins GPRC:$negsize, memri:$fpsi), "",
 let usesCustomInserter = 1,    // Expanded after instruction selection.
     PPC970_Single = 1 in {
   def SELECT_CC_I4 : Pseudo<(outs GPRC:$dst), (ins CRRC:$cond, GPRC:$T, GPRC:$F,
-                              i32imm:$BROPC), "",
+                              i32imm:$BROPC), "#SELECT_CC_I4",
                               []>;
   def SELECT_CC_I8 : Pseudo<(outs G8RC:$dst), (ins CRRC:$cond, G8RC:$T, G8RC:$F,
-                              i32imm:$BROPC), "",
+                              i32imm:$BROPC), "#SELECT_CC_I8",
                               []>;
   def SELECT_CC_F4  : Pseudo<(outs F4RC:$dst), (ins CRRC:$cond, F4RC:$T, F4RC:$F,
-                              i32imm:$BROPC), "",
+                              i32imm:$BROPC), "#SELECT_CC_F4",
                               []>;
   def SELECT_CC_F8  : Pseudo<(outs F8RC:$dst), (ins CRRC:$cond, F8RC:$T, F8RC:$F,
-                              i32imm:$BROPC), "",
+                              i32imm:$BROPC), "#SELECT_CC_F8",
                               []>;
   def SELECT_CC_VRRC: Pseudo<(outs VRRC:$dst), (ins CRRC:$cond, VRRC:$T, VRRC:$F,
-                              i32imm:$BROPC), "",
+                              i32imm:$BROPC), "#SELECT_CC_VRRC",
                               []>;
 }
 
@@ -404,16 +409,16 @@ let usesCustomInserter = 1,    // Expanded after instruction selection.
 // scavenge a register for it.
 let mayStore = 1 in
 def SPILL_CR : Pseudo<(outs), (ins CRRC:$cond, memri:$F),
-                     "", []>;
+                     "#SPILL_CR", []>;
 
 // RESTORE_CR - Indicate that we're restoring the CR register (previously
 // spilled), so we'll need to scavenge a register for it.
 let mayLoad = 1 in
 def RESTORE_CR : Pseudo<(outs CRRC:$cond), (ins memri:$F),
-                     "", []>;
+                     "#RESTORE_CR", []>;
 
 let isTerminator = 1, isBarrier = 1, PPC970_Unit = 7 in {
-  let isReturn = 1, Uses = [LR, RM] in
+  let isCodeGenOnly = 1, isReturn = 1, Uses = [LR, RM] in
     def BLR : XLForm_2_br<19, 16, 0, (outs), (ins pred:$p),
                           "b${p:cc}lr ${p:reg}", BrB, 
                           [(retflag)]>;
@@ -422,7 +427,7 @@ let isTerminator = 1, isBarrier = 1, PPC970_Unit = 7 in {
 }
 
 let Defs = [LR] in
-  def MovePCtoLR : Pseudo<(outs), (ins), "", []>,
+  def MovePCtoLR : Pseudo<(outs), (ins), "#MovePCtoLR", []>,
                    PPC970_Unit_BRU;
 
 let isBranch = 1, isTerminator = 1, hasCtrlDep = 1, PPC970_Unit = 7 in {
@@ -434,16 +439,17 @@ let isBranch = 1, isTerminator = 1, hasCtrlDep = 1, PPC970_Unit = 7 in {
 
   // BCC represents an arbitrary conditional branch on a predicate.
   // FIXME: should be able to write a pattern for PPCcondbranch, but can't use
-  // a two-value operand where a dag node expects two operands. :( 
-  def BCC : BForm<16, 0, 0, (outs), (ins pred:$cond, condbrtarget:$dst),
-                  "b${cond:cc} ${cond:reg}, $dst"
-                  /*[(PPCcondbranch CRRC:$crS, imm:$opc, bb:$dst)]*/>;
+  // a two-value operand where a dag node expects two operands. :(
+  let isCodeGenOnly = 1 in
+    def BCC : BForm<16, 0, 0, (outs), (ins pred:$cond, condbrtarget:$dst),
+                    "b${cond:cc} ${cond:reg}, $dst"
+                    /*[(PPCcondbranch CRRC:$crS, imm:$opc, bb:$dst)]*/>;
 
   let Defs = [CTR], Uses = [CTR] in {
-    def BDZ  : IForm_ext<16, 18, 0, 0, (outs), (ins condbrtarget:$dst),
-                         "bdz $dst",  BrB, []>;
-    def BDNZ : IForm_ext<16, 16, 0, 0, (outs), (ins condbrtarget:$dst),
-                         "bdnz $dst", BrB, []>;
+    def BDZ  : BForm_1<16, 18, 0, 0, (outs), (ins condbrtarget:$dst),
+                       "bdz $dst">;
+    def BDNZ : BForm_1<16, 16, 0, 0, (outs), (ins condbrtarget:$dst),
+                       "bdnz $dst">;
   }
 }
 
@@ -559,81 +565,81 @@ def : Pat<(prefetch xoaddr:$dst, (i32 0), imm, (i32 1)),
 let usesCustomInserter = 1 in {
   let Defs = [CR0] in {
     def ATOMIC_LOAD_ADD_I8 : Pseudo<
-      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$incr), "",
+      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$incr), "#ATOMIC_LOAD_ADD_I8",
       [(set GPRC:$dst, (atomic_load_add_8 xoaddr:$ptr, GPRC:$incr))]>;
     def ATOMIC_LOAD_SUB_I8 : Pseudo<
-      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$incr), "",
+      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$incr), "#ATOMIC_LOAD_SUB_I8",
       [(set GPRC:$dst, (atomic_load_sub_8 xoaddr:$ptr, GPRC:$incr))]>;
     def ATOMIC_LOAD_AND_I8 : Pseudo<
-      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$incr), "",
+      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$incr), "#ATOMIC_LOAD_AND_I8",
       [(set GPRC:$dst, (atomic_load_and_8 xoaddr:$ptr, GPRC:$incr))]>;
     def ATOMIC_LOAD_OR_I8 : Pseudo<
-      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$incr), "",
+      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$incr), "#ATOMIC_LOAD_OR_I8",
       [(set GPRC:$dst, (atomic_load_or_8 xoaddr:$ptr, GPRC:$incr))]>;
     def ATOMIC_LOAD_XOR_I8 : Pseudo<
-      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$incr), "",
+      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$incr), "ATOMIC_LOAD_XOR_I8",
       [(set GPRC:$dst, (atomic_load_xor_8 xoaddr:$ptr, GPRC:$incr))]>;
     def ATOMIC_LOAD_NAND_I8 : Pseudo<
-      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$incr), "",
+      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$incr), "#ATOMIC_LOAD_NAND_I8",
       [(set GPRC:$dst, (atomic_load_nand_8 xoaddr:$ptr, GPRC:$incr))]>;
     def ATOMIC_LOAD_ADD_I16 : Pseudo<
-      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$incr), "",
+      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$incr), "#ATOMIC_LOAD_ADD_I16",
       [(set GPRC:$dst, (atomic_load_add_16 xoaddr:$ptr, GPRC:$incr))]>;
     def ATOMIC_LOAD_SUB_I16 : Pseudo<
-      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$incr), "",
+      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$incr), "#ATOMIC_LOAD_SUB_I16",
       [(set GPRC:$dst, (atomic_load_sub_16 xoaddr:$ptr, GPRC:$incr))]>;
     def ATOMIC_LOAD_AND_I16 : Pseudo<
-      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$incr), "",
+      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$incr), "#ATOMIC_LOAD_AND_I16",
       [(set GPRC:$dst, (atomic_load_and_16 xoaddr:$ptr, GPRC:$incr))]>;
     def ATOMIC_LOAD_OR_I16 : Pseudo<
-      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$incr), "",
+      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$incr), "#ATOMIC_LOAD_OR_I16",
       [(set GPRC:$dst, (atomic_load_or_16 xoaddr:$ptr, GPRC:$incr))]>;
     def ATOMIC_LOAD_XOR_I16 : Pseudo<
-      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$incr), "",
+      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$incr), "#ATOMIC_LOAD_XOR_I16",
       [(set GPRC:$dst, (atomic_load_xor_16 xoaddr:$ptr, GPRC:$incr))]>;
     def ATOMIC_LOAD_NAND_I16 : Pseudo<
-      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$incr), "",
+      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$incr), "#ATOMIC_LOAD_NAND_I16",
       [(set GPRC:$dst, (atomic_load_nand_16 xoaddr:$ptr, GPRC:$incr))]>;
     def ATOMIC_LOAD_ADD_I32 : Pseudo<
-      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$incr), "",
+      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$incr), "#ATOMIC_LOAD_ADD_I32",
       [(set GPRC:$dst, (atomic_load_add_32 xoaddr:$ptr, GPRC:$incr))]>;
     def ATOMIC_LOAD_SUB_I32 : Pseudo<
-      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$incr), "",
+      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$incr), "#ATOMIC_LOAD_SUB_I32",
       [(set GPRC:$dst, (atomic_load_sub_32 xoaddr:$ptr, GPRC:$incr))]>;
     def ATOMIC_LOAD_AND_I32 : Pseudo<
-      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$incr), "",
+      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$incr), "#ATOMIC_LOAD_AND_I32",
       [(set GPRC:$dst, (atomic_load_and_32 xoaddr:$ptr, GPRC:$incr))]>;
     def ATOMIC_LOAD_OR_I32 : Pseudo<
-      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$incr), "",
+      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$incr), "#ATOMIC_LOAD_OR_I32",
       [(set GPRC:$dst, (atomic_load_or_32 xoaddr:$ptr, GPRC:$incr))]>;
     def ATOMIC_LOAD_XOR_I32 : Pseudo<
-      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$incr), "",
+      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$incr), "#ATOMIC_LOAD_XOR_I32",
       [(set GPRC:$dst, (atomic_load_xor_32 xoaddr:$ptr, GPRC:$incr))]>;
     def ATOMIC_LOAD_NAND_I32 : Pseudo<
-      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$incr), "",
+      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$incr), "#ATOMIC_LOAD_NAND_I32",
       [(set GPRC:$dst, (atomic_load_nand_32 xoaddr:$ptr, GPRC:$incr))]>;
 
     def ATOMIC_CMP_SWAP_I8 : Pseudo<
-      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$old, GPRC:$new), "",
+      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$old, GPRC:$new), "#ATOMIC_CMP_SWAP_I8",
       [(set GPRC:$dst, 
                     (atomic_cmp_swap_8 xoaddr:$ptr, GPRC:$old, GPRC:$new))]>;
     def ATOMIC_CMP_SWAP_I16 : Pseudo<
-      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$old, GPRC:$new), "",
+      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$old, GPRC:$new), "#ATOMIC_CMP_SWAP_I16 $dst $ptr $old $new",
       [(set GPRC:$dst, 
                     (atomic_cmp_swap_16 xoaddr:$ptr, GPRC:$old, GPRC:$new))]>;
     def ATOMIC_CMP_SWAP_I32 : Pseudo<
-      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$old, GPRC:$new), "",
+      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$old, GPRC:$new), "#ATOMIC_CMP_SWAP_I32 $dst $ptr $old $new",
       [(set GPRC:$dst, 
                     (atomic_cmp_swap_32 xoaddr:$ptr, GPRC:$old, GPRC:$new))]>;
 
     def ATOMIC_SWAP_I8 : Pseudo<
-      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$new), "",
+      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$new), "#ATOMIC_SWAP_i8",
       [(set GPRC:$dst, (atomic_swap_8 xoaddr:$ptr, GPRC:$new))]>;
     def ATOMIC_SWAP_I16 : Pseudo<
-      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$new), "",
+      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$new), "#ATOMIC_SWAP_I16",
       [(set GPRC:$dst, (atomic_swap_16 xoaddr:$ptr, GPRC:$new))]>;
     def ATOMIC_SWAP_I32 : Pseudo<
-      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$new), "",
+      (outs GPRC:$dst), (ins memrr:$ptr, GPRC:$new), "#ATOMIC_SWAP_I32",
       [(set GPRC:$dst, (atomic_swap_32 xoaddr:$ptr, GPRC:$new))]>;
   }
 }
@@ -673,7 +679,7 @@ def LWZ : DForm_1<32, (outs GPRC:$rD), (ins memri:$src),
                   [(set GPRC:$rD, (load iaddr:$src))]>;
 
 def LFS : DForm_1<48, (outs F4RC:$rD), (ins memri:$src),
-                  "lfs $rD, $src", LdStLFDU,
+                  "lfs $rD, $src", LdStLFD,
                   [(set F4RC:$rD, (load iaddr:$src))]>;
 def LFD : DForm_1<50, (outs F8RC:$rD), (ins memri:$src),
                   "lfd $rD, $src", LdStLFD,
@@ -683,32 +689,32 @@ def LFD : DForm_1<50, (outs F8RC:$rD), (ins memri:$src),
 // Unindexed (r+i) Loads with Update (preinc).
 let mayLoad = 1 in {
 def LBZU : DForm_1<35, (outs GPRC:$rD, ptr_rc:$ea_result), (ins memri:$addr),
-                   "lbzu $rD, $addr", LdStLoad,
+                   "lbzu $rD, $addr", LdStLoadUpd,
                    []>, RegConstraint<"$addr.reg = $ea_result">,
                    NoEncode<"$ea_result">;
 
 def LHAU : DForm_1<43, (outs GPRC:$rD, ptr_rc:$ea_result), (ins memri:$addr),
-                   "lhau $rD, $addr", LdStLoad,
+                   "lhau $rD, $addr", LdStLHAU,
                    []>, RegConstraint<"$addr.reg = $ea_result">,
                    NoEncode<"$ea_result">;
 
 def LHZU : DForm_1<41, (outs GPRC:$rD, ptr_rc:$ea_result), (ins memri:$addr),
-                   "lhzu $rD, $addr", LdStLoad,
+                   "lhzu $rD, $addr", LdStLoadUpd,
                    []>, RegConstraint<"$addr.reg = $ea_result">,
                    NoEncode<"$ea_result">;
 
 def LWZU : DForm_1<33, (outs GPRC:$rD, ptr_rc:$ea_result), (ins memri:$addr),
-                   "lwzu $rD, $addr", LdStLoad,
+                   "lwzu $rD, $addr", LdStLoadUpd,
                    []>, RegConstraint<"$addr.reg = $ea_result">,
                    NoEncode<"$ea_result">;
 
 def LFSU : DForm_1<49, (outs F4RC:$rD, ptr_rc:$ea_result), (ins memri:$addr),
-                  "lfs $rD, $addr", LdStLFDU,
+                  "lfsu $rD, $addr", LdStLFDU,
                   []>, RegConstraint<"$addr.reg = $ea_result">,
                    NoEncode<"$ea_result">;
 
 def LFDU : DForm_1<51, (outs F8RC:$rD, ptr_rc:$ea_result), (ins memri:$addr),
-                  "lfd $rD, $addr", LdStLFD,
+                  "lfdu $rD, $addr", LdStLFDU,
                   []>, RegConstraint<"$addr.reg = $ea_result">,
                    NoEncode<"$ea_result">;
 
@@ -716,37 +722,37 @@ def LFDU : DForm_1<51, (outs F8RC:$rD, ptr_rc:$ea_result), (ins memri:$addr),
 // Indexed (r+r) Loads with Update (preinc).
 def LBZUX : XForm_1<31, 119, (outs GPRC:$rD, ptr_rc:$ea_result),
                    (ins memrr:$addr),
-                   "lbzux $rD, $addr", LdStLoad,
+                   "lbzux $rD, $addr", LdStLoadUpd,
                    []>, RegConstraint<"$addr.offreg = $ea_result">,
                    NoEncode<"$ea_result">;
 
 def LHAUX : XForm_1<31, 375, (outs GPRC:$rD, ptr_rc:$ea_result),
                    (ins memrr:$addr),
-                   "lhaux $rD, $addr", LdStLoad,
+                   "lhaux $rD, $addr", LdStLHAU,
                    []>, RegConstraint<"$addr.offreg = $ea_result">,
                    NoEncode<"$ea_result">;
 
-def LHZUX : XForm_1<31, 331, (outs GPRC:$rD, ptr_rc:$ea_result),
+def LHZUX : XForm_1<31, 311, (outs GPRC:$rD, ptr_rc:$ea_result),
                    (ins memrr:$addr),
-                   "lhzux $rD, $addr", LdStLoad,
+                   "lhzux $rD, $addr", LdStLoadUpd,
                    []>, RegConstraint<"$addr.offreg = $ea_result">,
                    NoEncode<"$ea_result">;
 
 def LWZUX : XForm_1<31, 55, (outs GPRC:$rD, ptr_rc:$ea_result),
                    (ins memrr:$addr),
-                   "lwzux $rD, $addr", LdStLoad,
+                   "lwzux $rD, $addr", LdStLoadUpd,
                    []>, RegConstraint<"$addr.offreg = $ea_result">,
                    NoEncode<"$ea_result">;
 
 def LFSUX : XForm_1<31, 567, (outs F4RC:$rD, ptr_rc:$ea_result),
                    (ins memrr:$addr),
-                   "lfsux $rD, $addr", LdStLoad,
+                   "lfsux $rD, $addr", LdStLFDU,
                    []>, RegConstraint<"$addr.offreg = $ea_result">,
                    NoEncode<"$ea_result">;
 
 def LFDUX : XForm_1<31, 631, (outs F8RC:$rD, ptr_rc:$ea_result),
                    (ins memrr:$addr),
-                   "lfdux $rD, $addr", LdStLoad,
+                   "lfdux $rD, $addr", LdStLFDU,
                    []>, RegConstraint<"$addr.offreg = $ea_result">,
                    NoEncode<"$ea_result">;
 }
@@ -778,10 +784,10 @@ def LWBRX : XForm_1<31,  534, (outs GPRC:$rD), (ins memrr:$src),
                    [(set GPRC:$rD, (PPClbrx xoaddr:$src, i32))]>;
 
 def LFSX   : XForm_25<31, 535, (outs F4RC:$frD), (ins memrr:$src),
-                      "lfsx $frD, $src", LdStLFDU,
+                      "lfsx $frD, $src", LdStLFD,
                       [(set F4RC:$frD, (load xaddr:$src))]>;
 def LFDX   : XForm_25<31, 599, (outs F8RC:$frD), (ins memrr:$src),
-                      "lfdx $frD, $src", LdStLFDU,
+                      "lfdx $frD, $src", LdStLFD,
                       [(set F8RC:$frD, (load xaddr:$src))]>;
 }
 
@@ -801,10 +807,10 @@ def STW  : DForm_1<36, (outs), (ins GPRC:$rS, memri:$src),
                    "stw $rS, $src", LdStStore,
                    [(store GPRC:$rS, iaddr:$src)]>;
 def STFS : DForm_1<52, (outs), (ins F4RC:$rS, memri:$dst),
-                   "stfs $rS, $dst", LdStUX,
+                   "stfs $rS, $dst", LdStSTFD,
                    [(store F4RC:$rS, iaddr:$dst)]>;
 def STFD : DForm_1<54, (outs), (ins F8RC:$rS, memri:$dst),
-                   "stfd $rS, $dst", LdStUX,
+                   "stfd $rS, $dst", LdStSTFD,
                    [(store F8RC:$rS, iaddr:$dst)]>;
 }
 
@@ -812,33 +818,33 @@ def STFD : DForm_1<54, (outs), (ins F8RC:$rS, memri:$dst),
 let PPC970_Unit = 2 in {
 def STBU  : DForm_1a<39, (outs ptr_rc:$ea_res), (ins GPRC:$rS,
                              symbolLo:$ptroff, ptr_rc:$ptrreg),
-                    "stbu $rS, $ptroff($ptrreg)", LdStStore,
+                    "stbu $rS, $ptroff($ptrreg)", LdStStoreUpd,
                     [(set ptr_rc:$ea_res,
                           (pre_truncsti8 GPRC:$rS, ptr_rc:$ptrreg, 
                                          iaddroff:$ptroff))]>,
                     RegConstraint<"$ptrreg = $ea_res">, NoEncode<"$ea_res">;
 def STHU  : DForm_1a<45, (outs ptr_rc:$ea_res), (ins GPRC:$rS,
                              symbolLo:$ptroff, ptr_rc:$ptrreg),
-                    "sthu $rS, $ptroff($ptrreg)", LdStStore,
+                    "sthu $rS, $ptroff($ptrreg)", LdStStoreUpd,
                     [(set ptr_rc:$ea_res,
                         (pre_truncsti16 GPRC:$rS, ptr_rc:$ptrreg, 
                                         iaddroff:$ptroff))]>,
                     RegConstraint<"$ptrreg = $ea_res">, NoEncode<"$ea_res">;
 def STWU  : DForm_1a<37, (outs ptr_rc:$ea_res), (ins GPRC:$rS,
                              symbolLo:$ptroff, ptr_rc:$ptrreg),
-                    "stwu $rS, $ptroff($ptrreg)", LdStStore,
+                    "stwu $rS, $ptroff($ptrreg)", LdStStoreUpd,
                     [(set ptr_rc:$ea_res, (pre_store GPRC:$rS, ptr_rc:$ptrreg, 
                                                      iaddroff:$ptroff))]>,
                     RegConstraint<"$ptrreg = $ea_res">, NoEncode<"$ea_res">;
 def STFSU : DForm_1a<37, (outs ptr_rc:$ea_res), (ins F4RC:$rS,
                              symbolLo:$ptroff, ptr_rc:$ptrreg),
-                    "stfsu $rS, $ptroff($ptrreg)", LdStStore,
+                    "stfsu $rS, $ptroff($ptrreg)", LdStSTFDU,
                     [(set ptr_rc:$ea_res, (pre_store F4RC:$rS,  ptr_rc:$ptrreg, 
                                           iaddroff:$ptroff))]>,
                     RegConstraint<"$ptrreg = $ea_res">, NoEncode<"$ea_res">;
 def STFDU : DForm_1a<37, (outs ptr_rc:$ea_res), (ins F8RC:$rS,
                              symbolLo:$ptroff, ptr_rc:$ptrreg),
-                    "stfdu $rS, $ptroff($ptrreg)", LdStStore,
+                    "stfdu $rS, $ptroff($ptrreg)", LdStSTFDU,
                     [(set ptr_rc:$ea_res, (pre_store F8RC:$rS, ptr_rc:$ptrreg, 
                                           iaddroff:$ptroff))]>,
                     RegConstraint<"$ptrreg = $ea_res">, NoEncode<"$ea_res">;
@@ -863,7 +869,7 @@ def STWX  : XForm_8<31, 151, (outs), (ins GPRC:$rS, memrr:$dst),
  
 def STBUX : XForm_8<31, 247, (outs ptr_rc:$ea_res),
                              (ins GPRC:$rS, ptr_rc:$ptroff, ptr_rc:$ptrreg),
-                   "stbux $rS, $ptroff, $ptrreg", LdStStore,
+                   "stbux $rS, $ptroff, $ptrreg", LdStStoreUpd,
                    [(set ptr_rc:$ea_res,
                       (pre_truncsti8 GPRC:$rS,
                                      ptr_rc:$ptrreg, xaddroff:$ptroff))]>,
@@ -872,7 +878,7 @@ def STBUX : XForm_8<31, 247, (outs ptr_rc:$ea_res),
  
 def STHUX : XForm_8<31, 439, (outs ptr_rc:$ea_res),
                              (ins GPRC:$rS, ptr_rc:$ptroff, ptr_rc:$ptrreg),
-                   "sthux $rS, $ptroff, $ptrreg", LdStStore,
+                   "sthux $rS, $ptroff, $ptrreg", LdStStoreUpd,
                    [(set ptr_rc:$ea_res,
                       (pre_truncsti16 GPRC:$rS,
                                       ptr_rc:$ptrreg, xaddroff:$ptroff))]>,
@@ -881,7 +887,7 @@ def STHUX : XForm_8<31, 439, (outs ptr_rc:$ea_res),
                  
 def STWUX : XForm_8<31, 183, (outs ptr_rc:$ea_res),
                              (ins GPRC:$rS, ptr_rc:$ptroff, ptr_rc:$ptrreg),
-                   "stwux $rS, $ptroff, $ptrreg", LdStStore,
+                   "stwux $rS, $ptroff, $ptrreg", LdStStoreUpd,
                    [(set ptr_rc:$ea_res,
                       (pre_store GPRC:$rS, ptr_rc:$ptrreg, xaddroff:$ptroff))]>,
                    RegConstraint<"$ptroff = $ea_res">, NoEncode<"$ea_res">,
@@ -889,7 +895,7 @@ def STWUX : XForm_8<31, 183, (outs ptr_rc:$ea_res),
 
 def STFSUX : XForm_8<31, 695, (outs ptr_rc:$ea_res),
                               (ins F4RC:$rS, ptr_rc:$ptroff, ptr_rc:$ptrreg),
-                    "stfsux $rS, $ptroff, $ptrreg", LdStStore,
+                    "stfsux $rS, $ptroff, $ptrreg", LdStSTFDU,
                     [(set ptr_rc:$ea_res,
                        (pre_store F4RC:$rS, ptr_rc:$ptrreg, xaddroff:$ptroff))]>,
                     RegConstraint<"$ptroff = $ea_res">, NoEncode<"$ea_res">,
@@ -897,7 +903,7 @@ def STFSUX : XForm_8<31, 695, (outs ptr_rc:$ea_res),
 
 def STFDUX : XForm_8<31, 759, (outs ptr_rc:$ea_res),
                               (ins F8RC:$rS, ptr_rc:$ptroff, ptr_rc:$ptrreg),
-                    "stfdux $rS, $ptroff, $ptrreg", LdStStore,
+                    "stfdux $rS, $ptroff, $ptrreg", LdStSTFDU,
                     [(set ptr_rc:$ea_res,
                        (pre_store F8RC:$rS, ptr_rc:$ptrreg, xaddroff:$ptroff))]>,
                     RegConstraint<"$ptroff = $ea_res">, NoEncode<"$ea_res">,
@@ -913,14 +919,14 @@ def STWBRX: XForm_8<31, 662, (outs), (ins GPRC:$rS, memrr:$dst),
                    PPC970_DGroup_Cracked;
 
 def STFIWX: XForm_28<31, 983, (outs), (ins F8RC:$frS, memrr:$dst),
-                     "stfiwx $frS, $dst", LdStUX,
+                     "stfiwx $frS, $dst", LdStSTFD,
                      [(PPCstfiwx F8RC:$frS, xoaddr:$dst)]>;
                      
 def STFSX : XForm_28<31, 663, (outs), (ins F4RC:$frS, memrr:$dst),
-                     "stfsx $frS, $dst", LdStUX,
+                     "stfsx $frS, $dst", LdStSTFD,
                      [(store F4RC:$frS, xaddr:$dst)]>;
 def STFDX : XForm_28<31, 727, (outs), (ins F8RC:$frS, memrr:$dst),
-                     "stfdx $frS, $dst", LdStUX,
+                     "stfdx $frS, $dst", LdStSTFD,
                      [(store F8RC:$frS, xaddr:$dst)]>;
 }
 
@@ -964,7 +970,7 @@ def SUBFIC : DForm_2< 8, (outs GPRC:$rD), (ins GPRC:$rA, s16imm:$imm),
                      [(set GPRC:$rD, (subc immSExt16:$imm, GPRC:$rA))]>;
 }
 
-let isReMaterializable = 1 in {
+let isReMaterializable = 1, isAsCheapAsAMove = 1, isMoveImm = 1 in {
   def LI  : DForm_2_r0<14, (outs GPRC:$rD), (ins symbolLo:$imm),
                        "li $rD, $imm", IntSimple,
                        [(set GPRC:$rD, immSExt16:$imm)]>;
@@ -1143,6 +1149,16 @@ def CRUNSET: XLForm_1_ext<19, 193, (outs CRBITRC:$dst), (ins),
               "crxor $dst, $dst, $dst", BrCR,
               []>;
 
+let Defs = [CR1EQ], CRD = 6 in {
+def CR6SET  : XLForm_1_ext<19, 289, (outs), (ins),
+              "creqv 6, 6, 6", BrCR,
+              [(PPCcr6set)]>;
+
+def CR6UNSET: XLForm_1_ext<19, 193, (outs), (ins),
+              "crxor 6, 6, 6", BrCR,
+              [(PPCcr6unset)]>;
+}
+
 // XFX-Form instructions.  Instructions that deal with SPRs.
 //
 let Uses = [CTR] in {
@@ -1192,7 +1208,7 @@ def MTCRF : XFXForm_5<31, 144, (outs crbitm:$FXM), (ins GPRC:$rS),
 //
 // FIXME: Make this a real Pseudo instruction when the JIT switches to MC.
 def MFCRpseud: XFXForm_3<31, 19, (outs GPRC:$rT), (ins crbitm:$FXM),
-                       "", SprMFCR>,
+                       "#MFCRpseud", SprMFCR>,
             PPC970_MicroCode, PPC970_Unit_CRU;
             
 def MFCR : XFXForm_3<31, 19, (outs GPRC:$rT), (ins),
@@ -1233,7 +1249,7 @@ let Uses = [RM] in {
                PPC970_DGroup_Single, PPC970_Unit_FPU;
   def FADDrtz: AForm_2<63, 21,
                       (outs F8RC:$FRT), (ins F8RC:$FRA, F8RC:$FRB),
-                      "fadd $FRT, $FRA, $FRB", FPGeneral,
+                      "fadd $FRT, $FRA, $FRB", FPAddSub,
                       [(set F8RC:$FRT, (PPCfaddrtz F8RC:$FRA, F8RC:$FRB))]>,
                PPC970_DGroup_Single, PPC970_Unit_FPU;
 }
@@ -1364,7 +1380,7 @@ def FSELS : AForm_1<63, 23,
 let Uses = [RM] in {
   def FADD  : AForm_2<63, 21,
                       (outs F8RC:$FRT), (ins F8RC:$FRA, F8RC:$FRB),
-                      "fadd $FRT, $FRA, $FRB", FPGeneral,
+                      "fadd $FRT, $FRA, $FRB", FPAddSub,
                       [(set F8RC:$FRT, (fadd F8RC:$FRA, F8RC:$FRB))]>;
   def FADDS : AForm_2<59, 21,
                       (outs F4RC:$FRT), (ins F4RC:$FRA, F4RC:$FRB),
@@ -1379,16 +1395,16 @@ let Uses = [RM] in {
                       "fdivs $FRT, $FRA, $FRB", FPDivS,
                       [(set F4RC:$FRT, (fdiv F4RC:$FRA, F4RC:$FRB))]>;
   def FMUL  : AForm_3<63, 25,
-                      (outs F8RC:$FRT), (ins F8RC:$FRA, F8RC:$FRB),
-                      "fmul $FRT, $FRA, $FRB", FPFused,
-                      [(set F8RC:$FRT, (fmul F8RC:$FRA, F8RC:$FRB))]>;
+                      (outs F8RC:$FRT), (ins F8RC:$FRA, F8RC:$FRC),
+                      "fmul $FRT, $FRA, $FRC", FPFused,
+                      [(set F8RC:$FRT, (fmul F8RC:$FRA, F8RC:$FRC))]>;
   def FMULS : AForm_3<59, 25,
-                      (outs F4RC:$FRT), (ins F4RC:$FRA, F4RC:$FRB),
-                      "fmuls $FRT, $FRA, $FRB", FPGeneral,
-                      [(set F4RC:$FRT, (fmul F4RC:$FRA, F4RC:$FRB))]>;
+                      (outs F4RC:$FRT), (ins F4RC:$FRA, F4RC:$FRC),
+                      "fmuls $FRT, $FRA, $FRC", FPGeneral,
+                      [(set F4RC:$FRT, (fmul F4RC:$FRA, F4RC:$FRC))]>;
   def FSUB  : AForm_2<63, 20,
                       (outs F8RC:$FRT), (ins F8RC:$FRA, F8RC:$FRB),
-                      "fsub $FRT, $FRA, $FRB", FPGeneral,
+                      "fsub $FRT, $FRA, $FRB", FPAddSub,
                       [(set F8RC:$FRT, (fsub F8RC:$FRA, F8RC:$FRB))]>;
   def FSUBS : AForm_2<59, 20,
                       (outs F4RC:$FRT), (ins F4RC:$FRA, F4RC:$FRB),
@@ -1398,7 +1414,7 @@ let Uses = [RM] in {
 }
 
 let PPC970_Unit = 1 in {  // FXU Operations.
-  def ISEL  : AForm_1<31, 15,
+  def ISEL  : AForm_4<31, 15,
                      (outs GPRC:$rT), (ins GPRC:$rA, GPRC:$rB, pred:$cond),
                      "isel $rT, $rA, $rB, $cond", IntGeneral,
                      []>;
diff --git a/lib/Target/PowerPC/PPCRegisterInfo.cpp b/lib/Target/PowerPC/PPCRegisterInfo.cpp
index ab8bf1f..459c358 100644
--- a/lib/Target/PowerPC/PPCRegisterInfo.cpp
+++ b/lib/Target/PowerPC/PPCRegisterInfo.cpp
@@ -71,7 +71,7 @@ PPCRegisterInfo::PPCRegisterInfo(const PPCSubtarget &ST,
   : PPCGenRegisterInfo(ST.isPPC64() ? PPC::LR8 : PPC::LR,
                        ST.isPPC64() ? 0 : 1,
                        ST.isPPC64() ? 0 : 1),
-    Subtarget(ST), TII(tii) {
+    Subtarget(ST), TII(tii), CRSpillFrameIdx(0) {
   ImmToIdxMap[PPC::LD]   = PPC::LDX;    ImmToIdxMap[PPC::STD]  = PPC::STDX;
   ImmToIdxMap[PPC::LBZ]  = PPC::LBZX;   ImmToIdxMap[PPC::STB]  = PPC::STBX;
   ImmToIdxMap[PPC::LHZ]  = PPC::LHZX;   ImmToIdxMap[PPC::LHA]  = PPC::LHAX;
@@ -111,10 +111,15 @@ PPCRegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
     return Subtarget.isPPC64() ? CSR_Darwin64_SaveList :
                                  CSR_Darwin32_SaveList;
 
+  // For 32-bit SVR4, also initialize the frame index associated with
+  // the CR spill slot.
+  if (!Subtarget.isPPC64())
+    CRSpillFrameIdx = 0;
+
   return Subtarget.isPPC64() ? CSR_SVR464_SaveList : CSR_SVR432_SaveList;
 }
 
-const unsigned*
+const uint32_t*
 PPCRegisterInfo::getCallPreservedMask(CallingConv::ID CC) const {
   if (Subtarget.isDarwinABI())
     return Subtarget.isPPC64() ? CSR_Darwin64_RegMask :
@@ -477,6 +482,31 @@ void PPCRegisterInfo::lowerCRRestore(MachineBasicBlock::iterator II,
   MBB.erase(II);
 }
 
+bool
+PPCRegisterInfo::hasReservedSpillSlot(const MachineFunction &MF,
+				      unsigned Reg, int &FrameIdx) const {
+
+  // For the nonvolatile condition registers (CR2, CR3, CR4) in an SVR4
+  // ABI, return true to prevent allocating an additional frame slot.
+  // For 64-bit, the CR save area is at SP+8; the value of FrameIdx = 0
+  // is arbitrary and will be subsequently ignored.  For 32-bit, we must
+  // create exactly one stack slot and return its FrameIdx for all
+  // nonvolatiles.
+  if (Subtarget.isSVR4ABI() && PPC::CR2 <= Reg && Reg <= PPC::CR4) {
+    if (Subtarget.isPPC64()) {
+      FrameIdx = 0;
+    } else if (CRSpillFrameIdx) {
+      FrameIdx = CRSpillFrameIdx;
+    } else {
+      MachineFrameInfo *MFI = ((MachineFunction &)MF).getFrameInfo();
+      FrameIdx = MFI->CreateFixedObject((uint64_t)4, (int64_t)-4, true);
+      CRSpillFrameIdx = FrameIdx;
+    }
+    return true;
+  }
+  return false;
+}
+
 void
 PPCRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
                                      int SPAdj, RegScavenger *RS) const {
@@ -566,7 +596,7 @@ PPCRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
   // to Offset to get the correct offset.
   // Naked functions have stack size 0, although getStackSize may not reflect that
   // because we didn't call all the pieces that compute it for naked functions.
-  if (!MF.getFunction()->hasFnAttr(Attribute::Naked))
+  if (!MF.getFunction()->getFnAttributes().hasAttribute(Attributes::Naked))
     Offset += MFI->getStackSize();
 
   // If we can, encode the offset directly into the instruction.  If this is a
diff --git a/lib/Target/PowerPC/PPCRegisterInfo.h b/lib/Target/PowerPC/PPCRegisterInfo.h
index 152c36d..a8fd796 100644
--- a/lib/Target/PowerPC/PPCRegisterInfo.h
+++ b/lib/Target/PowerPC/PPCRegisterInfo.h
@@ -30,6 +30,7 @@ class PPCRegisterInfo : public PPCGenRegisterInfo {
   std::map<unsigned, unsigned> ImmToIdxMap;
   const PPCSubtarget &Subtarget;
   const TargetInstrInfo &TII;
+  mutable int CRSpillFrameIdx;
 public:
   PPCRegisterInfo(const PPCSubtarget &SubTarget, const TargetInstrInfo &tii);
   
@@ -43,7 +44,7 @@ public:
 
   /// Code Generation virtual methods...
   const uint16_t *getCalleeSavedRegs(const MachineFunction* MF = 0) const;
-  const unsigned *getCallPreservedMask(CallingConv::ID CC) const;
+  const uint32_t *getCallPreservedMask(CallingConv::ID CC) const;
 
   BitVector getReservedRegs(const MachineFunction &MF) const;
 
@@ -65,6 +66,8 @@ public:
                        int SPAdj, RegScavenger *RS) const;
   void lowerCRRestore(MachineBasicBlock::iterator II, unsigned FrameIndex,
                        int SPAdj, RegScavenger *RS) const;
+  bool hasReservedSpillSlot(const MachineFunction &MF, unsigned Reg,
+			    int &FrameIdx) const;
   void eliminateFrameIndex(MachineBasicBlock::iterator II,
                            int SPAdj, RegScavenger *RS = NULL) const;
 
diff --git a/lib/Target/PowerPC/PPCSchedule.td b/lib/Target/PowerPC/PPCSchedule.td
index 6a6ccb9..660c0c3 100644
--- a/lib/Target/PowerPC/PPCSchedule.td
+++ b/lib/Target/PowerPC/PPCSchedule.td
@@ -40,6 +40,7 @@ def IntMulHWU    : InstrItinClass;
 def IntMulLI     : InstrItinClass;
 def IntRFID      : InstrItinClass;
 def IntRotateD   : InstrItinClass;
+def IntRotateDI  : InstrItinClass;
 def IntRotate    : InstrItinClass;
 def IntShift     : InstrItinClass;
 def IntTrapD     : InstrItinClass;
@@ -52,15 +53,18 @@ def LdStDCBA     : InstrItinClass;
 def LdStDCBF     : InstrItinClass;
 def LdStDCBI     : InstrItinClass;
 def LdStLoad     : InstrItinClass;
+def LdStLoadUpd  : InstrItinClass;
 def LdStStore    : InstrItinClass;
+def LdStStoreUpd : InstrItinClass;
 def LdStDSS      : InstrItinClass;
 def LdStICBI     : InstrItinClass;
-def LdStUX       : InstrItinClass;
 def LdStLD       : InstrItinClass;
+def LdStLDU      : InstrItinClass;
 def LdStLDARX    : InstrItinClass;
 def LdStLFD      : InstrItinClass;
 def LdStLFDU     : InstrItinClass;
 def LdStLHA      : InstrItinClass;
+def LdStLHAU     : InstrItinClass;
 def LdStLMW      : InstrItinClass;
 def LdStLVecX    : InstrItinClass;
 def LdStLWA      : InstrItinClass;
@@ -69,6 +73,9 @@ def LdStSLBIA    : InstrItinClass;
 def LdStSLBIE    : InstrItinClass;
 def LdStSTD      : InstrItinClass;
 def LdStSTDCX    : InstrItinClass;
+def LdStSTDU     : InstrItinClass;
+def LdStSTFD     : InstrItinClass;
+def LdStSTFDU    : InstrItinClass;
 def LdStSTVEBX   : InstrItinClass;
 def LdStSTWCX    : InstrItinClass;
 def LdStSync     : InstrItinClass;
@@ -86,6 +93,7 @@ def SprMTSRIN    : InstrItinClass;
 def SprRFI       : InstrItinClass;
 def SprSC        : InstrItinClass;
 def FPGeneral    : InstrItinClass;
+def FPAddSub     : InstrItinClass;
 def FPCompare    : InstrItinClass;
 def FPDivD       : InstrItinClass;
 def FPDivS       : InstrItinClass;
@@ -110,6 +118,8 @@ include "PPCScheduleG4.td"
 include "PPCScheduleG4Plus.td"
 include "PPCScheduleG5.td"
 include "PPCScheduleA2.td"
+include "PPCScheduleE500mc.td"
+include "PPCScheduleE5500.td"
 
 //===----------------------------------------------------------------------===//
 // Instruction to itinerary class map - When add new opcodes to the supported
@@ -171,7 +181,7 @@ include "PPCScheduleA2.td"
 //    extsh      IntSimple
 //    extsw      IntSimple
 //    fabs       FPGeneral
-//    fadd       FPGeneral
+//    fadd       FPAddSub
 //    fadds      FPGeneral
 //    fcfid      FPGeneral
 //    fcmpo      FPCompare
@@ -201,35 +211,35 @@ include "PPCScheduleA2.td"
 //    fsel       FPGeneral
 //    fsqrt      FPSqrt
 //    fsqrts     FPSqrt
-//    fsub       FPGeneral
+//    fsub       FPAddSub
 //    fsubs      FPGeneral
 //    icbi       LdStICBI
 //    isync      SprISYNC
 //    lbz        LdStLoad
-//    lbzu       LdStLoad
-//    lbzux      LdStUX
+//    lbzu       LdStLoadUpd
+//    lbzux      LdStLoadUpd
 //    lbzx       LdStLoad
 //    ld         LdStLD
 //    ldarx      LdStLDARX
-//    ldu        LdStLD
-//    ldux       LdStLD
+//    ldu        LdStLDU
+//    ldux       LdStLDU
 //    ldx        LdStLD
 //    lfd        LdStLFD
 //    lfdu       LdStLFDU
 //    lfdux      LdStLFDU
-//    lfdx       LdStLFDU
-//    lfs        LdStLFDU
+//    lfdx       LdStLFD
+//    lfs        LdStLFD
 //    lfsu       LdStLFDU
 //    lfsux      LdStLFDU
-//    lfsx       LdStLFDU
+//    lfsx       LdStLFD
 //    lha        LdStLHA
-//    lhau       LdStLHA
-//    lhaux      LdStLHA
+//    lhau       LdStLHAU
+//    lhaux      LdStLHAU
 //    lhax       LdStLHA
 //    lhbrx      LdStLoad
 //    lhz        LdStLoad
-//    lhzu       LdStLoad
-//    lhzux      LdStUX
+//    lhzu       LdStLoadUpd
+//    lhzux      LdStLoadUpd
 //    lhzx       LdStLoad
 //    lmw        LdStLMW
 //    lswi       LdStLMW
@@ -243,12 +253,12 @@ include "PPCScheduleA2.td"
 //    lvxl       LdStLVecX
 //    lwa        LdStLWA
 //    lwarx      LdStLWARX
-//    lwaux      LdStLHA
+//    lwaux      LdStLHAU
 //    lwax       LdStLHA
 //    lwbrx      LdStLoad
 //    lwz        LdStLoad
-//    lwzu       LdStLoad
-//    lwzux      LdStUX
+//    lwzu       LdStLoadUpd
+//    lwzux      LdStLoadUpd
 //    lwzx       LdStLoad
 //    mcrf       BrMCR
 //    mcrfs      FPGeneral
@@ -292,10 +302,10 @@ include "PPCScheduleA2.td"
 //    rfid       IntRFID
 //    rldcl      IntRotateD
 //    rldcr      IntRotateD
-//    rldic      IntRotateD
-//    rldicl     IntRotateD
-//    rldicr     IntRotateD
-//    rldimi     IntRotateD
+//    rldic      IntRotateDI
+//    rldicl     IntRotateDI
+//    rldicr     IntRotateDI
+//    rldimi     IntRotateDI
 //    rlwimi     IntRotate
 //    rlwinm     IntGeneral
 //    rlwnm      IntGeneral
@@ -305,33 +315,33 @@ include "PPCScheduleA2.td"
 //    sld        IntRotateD
 //    slw        IntGeneral
 //    srad       IntRotateD
-//    sradi      IntRotateD
+//    sradi      IntRotateDI
 //    sraw       IntShift
 //    srawi      IntShift
 //    srd        IntRotateD
 //    srw        IntGeneral
 //    stb        LdStStore
-//    stbu       LdStStore
-//    stbux      LdStStore
+//    stbu       LdStStoreUpd
+//    stbux      LdStStoreUpd
 //    stbx       LdStStore
 //    std        LdStSTD
 //    stdcx.     LdStSTDCX
-//    stdu       LdStSTD
-//    stdux      LdStSTD
+//    stdu       LdStSTDU
+//    stdux      LdStSTDU
 //    stdx       LdStSTD
-//    stfd       LdStUX
-//    stfdu      LdStUX
-//    stfdux     LdStUX
-//    stfdx      LdStUX
-//    stfiwx     LdStUX
-//    stfs       LdStUX
-//    stfsu      LdStUX
-//    stfsux     LdStUX
-//    stfsx      LdStUX
+//    stfd       LdStSTFD
+//    stfdu      LdStSTFDU
+//    stfdux     LdStSTFDU
+//    stfdx      LdStSTFD
+//    stfiwx     LdStSTFD
+//    stfs       LdStSTFD
+//    stfsu      LdStSTFDU
+//    stfsux     LdStSTFDU
+//    stfsx      LdStSTFD
 //    sth        LdStStore
 //    sthbrx     LdStStore
-//    sthu       LdStStore
-//    sthux      LdStStore
+//    sthu       LdStStoreUpd
+//    sthux      LdStStoreUpd
 //    sthx       LdStStore
 //    stmw       LdStLMW
 //    stswi      LdStLMW
@@ -344,8 +354,8 @@ include "PPCScheduleA2.td"
 //    stw        LdStStore
 //    stwbrx     LdStStore
 //    stwcx.     LdStSTWCX
-//    stwu       LdStStore
-//    stwux      LdStStore
+//    stwu       LdStStoreUpd
+//    stwux      LdStStoreUpd
 //    stwx       LdStStore
 //    subf       IntGeneral
 //    subfc      IntGeneral
diff --git a/lib/Target/PowerPC/PPCSchedule440.td b/lib/Target/PowerPC/PPCSchedule440.td
index cd0fb70..37b6eac 100644
--- a/lib/Target/PowerPC/PPCSchedule440.td
+++ b/lib/Target/PowerPC/PPCSchedule440.td
@@ -288,6 +288,15 @@ def PPC440Itineraries : ProcessorItineraries<
                                InstrStage<2, [LWB]>],
                               [9, 5],
                               [GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<LdStLoadUpd , [InstrStage<1, [IFTH1, IFTH2]>,
+                               InstrStage<1, [PDCD1, PDCD2]>,
+                               InstrStage<1, [DISS1, DISS2]>,
+                               InstrStage<1, [LRACC]>,
+                               InstrStage<1, [AGEN]>,
+                               InstrStage<1, [CRD]>,
+                               InstrStage<2, [LWB]>],
+                              [9, 5],
+                              [GPR_Bypass, GPR_Bypass]>,                              
   InstrItinData<LdStStore   , [InstrStage<1, [IFTH1, IFTH2]>,
                                InstrStage<1, [PDCD1, PDCD2]>,
                                InstrStage<1, [DISS1, DISS2]>,
@@ -297,6 +306,15 @@ def PPC440Itineraries : ProcessorItineraries<
                                InstrStage<2, [LWB]>],
                               [8, 5],
                               [NoBypass, GPR_Bypass]>,
+  InstrItinData<LdStStoreUpd, [InstrStage<1, [IFTH1, IFTH2]>,
+                               InstrStage<1, [PDCD1, PDCD2]>,
+                               InstrStage<1, [DISS1, DISS2]>,
+                               InstrStage<1, [LRACC]>,
+                               InstrStage<1, [AGEN]>,
+                               InstrStage<1, [CRD]>,
+                               InstrStage<2, [LWB]>],
+                              [8, 5],
+                              [NoBypass, GPR_Bypass]>,                              
   InstrItinData<LdStICBI    , [InstrStage<1, [IFTH1, IFTH2]>,
                                InstrStage<1, [PDCD1, PDCD2]>,
                                InstrStage<1, [DISS1, DISS2]>,
@@ -306,7 +324,7 @@ def PPC440Itineraries : ProcessorItineraries<
                                InstrStage<1, [LWB]>],
                               [8, 5],
                               [NoBypass, GPR_Bypass]>,
-  InstrItinData<LdStUX      , [InstrStage<1, [IFTH1, IFTH2]>,
+  InstrItinData<LdStSTFD    , [InstrStage<1, [IFTH1, IFTH2]>,
                                InstrStage<1, [PDCD1, PDCD2]>,
                                InstrStage<1, [DISS1, DISS2]>,
                                InstrStage<1, [LRACC]>,
@@ -315,6 +333,15 @@ def PPC440Itineraries : ProcessorItineraries<
                                InstrStage<1, [LWB]>],
                               [8, 5, 5],
                               [NoBypass, GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<LdStSTFDU   , [InstrStage<1, [IFTH1, IFTH2]>,
+                               InstrStage<1, [PDCD1, PDCD2]>,
+                               InstrStage<1, [DISS1, DISS2]>,
+                               InstrStage<1, [LRACC]>,
+                               InstrStage<1, [AGEN]>,
+                               InstrStage<1, [CRD]>,
+                               InstrStage<1, [LWB]>],
+                              [8, 5, 5],
+                              [NoBypass, GPR_Bypass, GPR_Bypass]>,                              
   InstrItinData<LdStLFD     , [InstrStage<1, [IFTH1, IFTH2]>,
                                InstrStage<1, [PDCD1, PDCD2]>,
                                InstrStage<1, [DISS1, DISS2]>,
@@ -342,6 +369,15 @@ def PPC440Itineraries : ProcessorItineraries<
                                InstrStage<1, [LWB]>],
                               [8, 5],
                               [NoBypass, GPR_Bypass]>,
+  InstrItinData<LdStLHAU    , [InstrStage<1, [IFTH1, IFTH2]>,
+                               InstrStage<1, [PDCD1, PDCD2]>,
+                               InstrStage<1, [DISS1, DISS2]>,
+                               InstrStage<1, [LRACC]>,
+                               InstrStage<1, [AGEN]>,
+                               InstrStage<1, [CRD]>,
+                               InstrStage<1, [LWB]>],
+                              [8, 5],
+                              [NoBypass, GPR_Bypass]>,                              
   InstrItinData<LdStLMW     , [InstrStage<1, [IFTH1, IFTH2]>,
                                InstrStage<1, [PDCD1, PDCD2]>,
                                InstrStage<1, [DISS1, DISS2]>,
@@ -371,6 +407,15 @@ def PPC440Itineraries : ProcessorItineraries<
                                InstrStage<2, [LWB]>],
                               [8, 5],
                               [NoBypass, GPR_Bypass]>,
+  InstrItinData<LdStSTDU    , [InstrStage<1, [IFTH1, IFTH2]>,
+                               InstrStage<1, [PDCD1, PDCD2]>,
+                               InstrStage<1, [DISS1, DISS2]>,
+                               InstrStage<1, [LRACC]>,
+                               InstrStage<1, [AGEN]>,
+                               InstrStage<1, [CRD]>,
+                               InstrStage<2, [LWB]>],
+                              [8, 5],
+                              [NoBypass, GPR_Bypass]>,                              
   InstrItinData<LdStSTDCX   , [InstrStage<1, [IFTH1, IFTH2]>,
                                InstrStage<1, [PDCD1, PDCD2]>,
                                InstrStage<1, [DISS1]>,
@@ -537,6 +582,19 @@ def PPC440Itineraries : ProcessorItineraries<
                                InstrStage<1, [FWB]>],
                               [10, 4, 4],
                               [FPR_Bypass, FPR_Bypass, FPR_Bypass]>,
+  InstrItinData<FPAddSub    , [InstrStage<1, [IFTH1, IFTH2]>,
+                               InstrStage<1, [PDCD1, PDCD2]>,
+                               InstrStage<1, [DISS1, DISS2]>,
+                               InstrStage<1, [FRACC]>,
+                               InstrStage<1, [FEXE1]>,
+                               InstrStage<1, [FEXE2]>,
+                               InstrStage<1, [FEXE3]>,
+                               InstrStage<1, [FEXE4]>,
+                               InstrStage<1, [FEXE5]>,
+                               InstrStage<1, [FEXE6]>,
+                               InstrStage<1, [FWB]>],
+                              [10, 4, 4],
+                              [FPR_Bypass, FPR_Bypass, FPR_Bypass]>,
   InstrItinData<FPCompare   , [InstrStage<1, [IFTH1, IFTH2]>,
                                InstrStage<1, [PDCD1, PDCD2]>,
                                InstrStage<1, [DISS1, DISS2]>,
diff --git a/lib/Target/PowerPC/PPCScheduleA2.td b/lib/Target/PowerPC/PPCScheduleA2.td
index 4d4a5d0..ba63b5c 100644
--- a/lib/Target/PowerPC/PPCScheduleA2.td
+++ b/lib/Target/PowerPC/PPCScheduleA2.td
@@ -181,6 +181,17 @@ def PPCA2Itineraries : ProcessorItineraries<
                                InstrStage<1, [XEX5]>, InstrStage<1, [XEX6]>],
                               [10, 7, 7],
                               [GPR_Bypass, GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<IntRotateDI , [InstrStage<4,
+                                 [IU0to3_0, IU0to3_1, IU0to3_2, IU0to3_3]>,
+                               InstrStage<1, [IU4_0, IU4_1, IU4_2, IU4_3,
+                                              IU4_4, IU4_5, IU4_6, IU4_7]>,
+                               InstrStage<1, [IU5]>, InstrStage<1, [IU6]>,
+                               InstrStage<1, [RF0]>, InstrStage<1, [XRF1]>,
+                               InstrStage<1, [XEX1]>, InstrStage<1, [XEX2]>,
+                               InstrStage<1, [XEX3]>, InstrStage<1, [XEX4]>,
+                               InstrStage<1, [XEX5]>, InstrStage<1, [XEX6]>],
+                              [10, 7, 7],
+                              [GPR_Bypass, GPR_Bypass, GPR_Bypass]>,                              
   InstrItinData<IntShift    , [InstrStage<4,
                                  [IU0to3_0, IU0to3_1, IU0to3_2, IU0to3_3]>,
                                InstrStage<1, [IU4_0, IU4_1, IU4_2, IU4_3,
@@ -302,7 +313,18 @@ def PPCA2Itineraries : ProcessorItineraries<
                                InstrStage<1, [XEX5]>, InstrStage<1, [XEX6]>],
                               [14, 7],
                               [GPR_Bypass, GPR_Bypass]>,
-  InstrItinData<LdStLD      , [InstrStage<4,
+  InstrItinData<LdStLoadUpd , [InstrStage<4,
+                                 [IU0to3_0, IU0to3_1, IU0to3_2, IU0to3_3]>,
+                               InstrStage<1, [IU4_0, IU4_1, IU4_2, IU4_3,
+                                              IU4_4, IU4_5, IU4_6, IU4_7]>,
+                               InstrStage<1, [IU5]>, InstrStage<1, [IU6]>,
+                               InstrStage<1, [RF0]>, InstrStage<1, [XRF1]>,
+                               InstrStage<1, [XEX1]>, InstrStage<1, [XEX2]>,
+                               InstrStage<1, [XEX3]>, InstrStage<1, [XEX4]>,
+                               InstrStage<1, [XEX5]>, InstrStage<1, [XEX6]>],
+                              [14, 7],
+                              [GPR_Bypass, GPR_Bypass]>,                              
+  InstrItinData<LdStLDU     , [InstrStage<4,
                                  [IU0to3_0, IU0to3_1, IU0to3_2, IU0to3_3]>,
                                InstrStage<1, [IU4_0, IU4_1, IU4_2, IU4_3,
                                               IU4_4, IU4_5, IU4_6, IU4_7]>,
@@ -324,6 +346,17 @@ def PPCA2Itineraries : ProcessorItineraries<
                                InstrStage<1, [XEX5]>, InstrStage<1, [XEX6]>],
                               [13, 7],
                               [GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<LdStStoreUpd, [InstrStage<4,
+                                 [IU0to3_0, IU0to3_1, IU0to3_2, IU0to3_3]>,
+                               InstrStage<1, [IU4_0, IU4_1, IU4_2, IU4_3,
+                                              IU4_4, IU4_5, IU4_6, IU4_7]>,
+                               InstrStage<1, [IU5]>, InstrStage<1, [IU6]>,
+                               InstrStage<1, [RF0]>, InstrStage<1, [XRF1]>,
+                               InstrStage<1, [XEX1]>, InstrStage<1, [XEX2]>,
+                               InstrStage<1, [XEX3]>, InstrStage<1, [XEX4]>,
+                               InstrStage<1, [XEX5]>, InstrStage<1, [XEX6]>],
+                              [13, 7],
+                              [GPR_Bypass, GPR_Bypass]>,
   InstrItinData<LdStICBI    , [InstrStage<4,
                                  [IU0to3_0, IU0to3_1, IU0to3_2, IU0to3_3]>,
                                InstrStage<1, [IU4_0, IU4_1, IU4_2, IU4_3,
@@ -335,7 +368,7 @@ def PPCA2Itineraries : ProcessorItineraries<
                                InstrStage<1, [XEX5]>, InstrStage<1, [XEX6]>],
                               [14, 7],
                               [NoBypass, GPR_Bypass]>,
-  InstrItinData<LdStUX      , [InstrStage<4,
+  InstrItinData<LdStSTFD    , [InstrStage<4,
                                  [IU0to3_0, IU0to3_1, IU0to3_2, IU0to3_3]>,
                                InstrStage<1, [IU4_0, IU4_1, IU4_2, IU4_3,
                                               IU4_4, IU4_5, IU4_6, IU4_7]>,
@@ -346,6 +379,17 @@ def PPCA2Itineraries : ProcessorItineraries<
                                InstrStage<1, [XEX5]>, InstrStage<1, [XEX6]>],
                               [14, 7, 7],
                               [NoBypass, FPR_Bypass, FPR_Bypass]>,
+  InstrItinData<LdStSTFDU   , [InstrStage<4,
+                                 [IU0to3_0, IU0to3_1, IU0to3_2, IU0to3_3]>,
+                               InstrStage<1, [IU4_0, IU4_1, IU4_2, IU4_3,
+                                              IU4_4, IU4_5, IU4_6, IU4_7]>,
+                               InstrStage<1, [IU5]>, InstrStage<1, [IU6]>,
+                               InstrStage<1, [RF0]>, InstrStage<1, [XRF1]>,
+                               InstrStage<1, [XEX1]>, InstrStage<1, [XEX2]>,
+                               InstrStage<1, [XEX3]>, InstrStage<1, [XEX4]>,
+                               InstrStage<1, [XEX5]>, InstrStage<1, [XEX6]>],
+                              [14, 7, 7],
+                              [NoBypass, FPR_Bypass, FPR_Bypass]>,                              
   InstrItinData<LdStLFD     , [InstrStage<4,
                                  [IU0to3_0, IU0to3_1, IU0to3_2, IU0to3_3]>,
                                InstrStage<1, [IU4_0, IU4_1, IU4_2, IU4_3,
@@ -379,6 +423,17 @@ def PPCA2Itineraries : ProcessorItineraries<
                                InstrStage<1, [XEX5]>, InstrStage<1, [XEX6]>],
                               [14, 7],
                               [NoBypass, GPR_Bypass]>,
+  InstrItinData<LdStLHAU    , [InstrStage<4,
+                                 [IU0to3_0, IU0to3_1, IU0to3_2, IU0to3_3]>,
+                               InstrStage<1, [IU4_0, IU4_1, IU4_2, IU4_3,
+                                              IU4_4, IU4_5, IU4_6, IU4_7]>,
+                               InstrStage<1, [IU5]>, InstrStage<1, [IU6]>,
+                               InstrStage<1, [RF0]>, InstrStage<1, [XRF1]>,
+                               InstrStage<1, [XEX1]>, InstrStage<1, [XEX2]>,
+                               InstrStage<1, [XEX3]>, InstrStage<1, [XEX4]>,
+                               InstrStage<1, [XEX5]>, InstrStage<1, [XEX6]>],
+                              [14, 7],
+                              [NoBypass, GPR_Bypass]>,
   InstrItinData<LdStLMW     , [InstrStage<4,
                                  [IU0to3_0, IU0to3_1, IU0to3_2, IU0to3_3]>,
                                InstrStage<1, [IU4_0, IU4_1, IU4_2, IU4_3,
@@ -412,6 +467,17 @@ def PPCA2Itineraries : ProcessorItineraries<
                                InstrStage<1, [XEX5]>, InstrStage<1, [XEX6]>],
                               [13, 7],
                               [GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<LdStSTDU    , [InstrStage<4,
+                                 [IU0to3_0, IU0to3_1, IU0to3_2, IU0to3_3]>,
+                               InstrStage<1, [IU4_0, IU4_1, IU4_2, IU4_3,
+                                              IU4_4, IU4_5, IU4_6, IU4_7]>,
+                               InstrStage<1, [IU5]>, InstrStage<1, [IU6]>,
+                               InstrStage<1, [RF0]>, InstrStage<1, [XRF1]>,
+                               InstrStage<1, [XEX1]>, InstrStage<1, [XEX2]>,
+                               InstrStage<1, [XEX3]>, InstrStage<1, [XEX4]>,
+                               InstrStage<1, [XEX5]>, InstrStage<1, [XEX6]>],
+                              [13, 7],
+                              [GPR_Bypass, GPR_Bypass]>,                              
   InstrItinData<LdStSTDCX   , [InstrStage<4,
                                  [IU0to3_0, IU0to3_1, IU0to3_2, IU0to3_3]>,
                                InstrStage<1, [IU4_0, IU4_1, IU4_2, IU4_3,
@@ -593,6 +659,17 @@ def PPCA2Itineraries : ProcessorItineraries<
                                InstrStage<1, [FEX5]>, InstrStage<1, [FEX6]>],
                               [15, 7, 7],
                               [FPR_Bypass, FPR_Bypass, FPR_Bypass]>,
+  InstrItinData<FPAddSub    , [InstrStage<4,
+                                 [IU0to3_0, IU0to3_1, IU0to3_2, IU0to3_3]>,
+                               InstrStage<1, [IU4_0, IU4_1, IU4_2, IU4_3,
+                                              IU4_4, IU4_5, IU4_6, IU4_7]>,
+                               InstrStage<1, [IU5]>, InstrStage<1, [IU6]>,
+                               InstrStage<1, [RF0]>, InstrStage<1, [FRF1]>,
+                               InstrStage<1, [FEX1]>, InstrStage<1, [FEX2]>,
+                               InstrStage<1, [FEX3]>, InstrStage<1, [FEX4]>,
+                               InstrStage<1, [FEX5]>, InstrStage<1, [FEX6]>],
+                              [15, 7, 7],
+                              [FPR_Bypass, FPR_Bypass, FPR_Bypass]>,
   InstrItinData<FPCompare   , [InstrStage<4,
                                  [IU0to3_0, IU0to3_1, IU0to3_2, IU0to3_3]>,
                                InstrStage<1, [IU4_0, IU4_1, IU4_2, IU4_3,
diff --git a/lib/Target/PowerPC/PPCScheduleE500mc.td b/lib/Target/PowerPC/PPCScheduleE500mc.td
new file mode 100644
index 0000000..9bb779a
--- /dev/null
+++ b/lib/Target/PowerPC/PPCScheduleE500mc.td
@@ -0,0 +1,265 @@
+//===-- PPCScheduleE500mc.td - e500mc Scheduling Defs ------*- tablegen -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the itinerary class data for the Freescale e500mc 32-bit 
+// Power processor.
+// 
+// All information is derived from the "e500mc Core Reference Manual",
+// Freescale Document Number E500MCRM, Rev. 1, 03/2012.
+//
+//===----------------------------------------------------------------------===//
+// Relevant functional units in the Freescale e500mc core:
+//
+//  * Decode & Dispatch
+//    Can dispatch up to 2 instructions per clock cycle to either the GPR Issue
+//    queues (GIQx), FP Issue Queue (FIQ), or Branch issue queue (BIQ).
+def DIS0 : FuncUnit; // Dispatch stage - insn 1
+def DIS1 : FuncUnit; // Dispatch stage - insn 2
+
+//  * Execute
+//    6 pipelined execution units: SFX0, SFX1, BU, FPU, LSU, CFX.
+//    Some instructions can only execute in SFX0 but not SFX1.
+//    The CFX has a bypass path, allowing non-divide instructions to execute 
+//    while a divide instruction is executed.
+def SFX0  : FuncUnit; // Simple unit 0
+def SFX1  : FuncUnit; // Simple unit 1
+def BU    : FuncUnit; // Branch unit
+def CFX_DivBypass 
+          : FuncUnit; // CFX divide bypass path
+def CFX_0 : FuncUnit; // CFX pipeline
+def LSU_0 : FuncUnit; // LSU pipeline
+def FPU_0 : FuncUnit; // FPU pipeline
+
+def PPCE500mcItineraries : ProcessorItineraries<
+  [DIS0, DIS1, SFX0, SFX1, BU, CFX_DivBypass, CFX_0, LSU_0, FPU_0],
+  [CR_Bypass, GPR_Bypass, FPR_Bypass], [
+  InstrItinData<IntSimple   , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [SFX0, SFX1]>],
+                              [4, 1, 1], // Latency = 1
+                              [GPR_Bypass, GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<IntGeneral  , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [SFX0, SFX1]>],
+                              [4, 1, 1], // Latency = 1
+                              [GPR_Bypass, GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<IntCompare  , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [SFX0, SFX1]>],
+                              [5, 1, 1], // Latency = 1 or 2
+                              [CR_Bypass, GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<IntDivW     , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [CFX_0], 0>,
+                               InstrStage<14, [CFX_DivBypass]>],
+                              [17, 1, 1], // Latency=4..35, Repeat= 4..35
+                              [GPR_Bypass, GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<IntMFFS     , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<8, [FPU_0]>],
+                              [11], // Latency = 8
+                              [FPR_Bypass]>,
+  InstrItinData<IntMTFSB0   , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<8, [FPU_0]>],
+                              [11, 1, 1], // Latency = 8
+                              [NoBypass, NoBypass, NoBypass]>,
+  InstrItinData<IntMulHW    , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [CFX_0]>],
+                              [7, 1, 1], // Latency = 4, Repeat rate = 1
+                              [GPR_Bypass, GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<IntMulHWU   , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [CFX_0]>],
+                              [7, 1, 1], // Latency = 4, Repeat rate = 1
+                              [GPR_Bypass, GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<IntMulLI    , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [CFX_0]>],
+                              [7, 1, 1], // Latency = 4, Repeat rate = 1
+                              [GPR_Bypass, GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<IntRotate   , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [SFX0, SFX1]>],
+                              [4, 1, 1], // Latency = 1
+                              [GPR_Bypass, GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<IntShift    , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [SFX0, SFX1]>],
+                              [4, 1, 1], // Latency = 1
+                              [GPR_Bypass, GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<IntTrapW    , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<2, [SFX0]>],
+                              [5, 1], // Latency = 2, Repeat rate = 2
+                              [GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<BrB         , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [BU]>],
+                              [4, 1], // Latency = 1
+                              [NoBypass, GPR_Bypass]>,
+  InstrItinData<BrCR        , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [BU]>],
+                              [4, 1, 1], // Latency = 1
+                              [CR_Bypass, CR_Bypass, CR_Bypass]>,
+  InstrItinData<BrMCR       , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [BU]>],
+                              [4, 1], // Latency = 1
+                              [CR_Bypass, CR_Bypass]>,
+  InstrItinData<BrMCRX      , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [SFX0, SFX1]>],
+                              [4, 1, 1], // Latency = 1
+                              [CR_Bypass, GPR_Bypass]>,
+  InstrItinData<LdStDCBA    , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [LSU_0]>],
+                              [6, 1], // Latency = 3, Repeat rate = 1
+                              [GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<LdStDCBF    , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [LSU_0]>],
+                              [6, 1], // Latency = 3
+                              [GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<LdStDCBI    , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [LSU_0]>],
+                              [6, 1], // Latency = 3
+                              [GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<LdStLoad    , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [LSU_0]>],
+                              [6, 1], // Latency = 3
+                              [GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<LdStLoadUpd , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [SFX0, SFX1], 0>,
+                               InstrStage<1, [LSU_0]>],
+                              [6, 1], // Latency = 3
+                              [GPR_Bypass, GPR_Bypass],
+                              2>, // 2 micro-ops                              
+  InstrItinData<LdStStore   , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [LSU_0]>],
+                              [6, 1], // Latency = 3
+                              [NoBypass, GPR_Bypass]>,
+  InstrItinData<LdStStoreUpd, [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [SFX0, SFX1], 0>,
+                               InstrStage<1, [LSU_0]>],
+                              [6, 1], // Latency = 3
+                              [NoBypass, GPR_Bypass],
+                              2>, // 2 micro-ops                              
+  InstrItinData<LdStICBI    , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [LSU_0]>],
+                              [6, 1], // Latency = 3
+                              [NoBypass, GPR_Bypass]>,
+  InstrItinData<LdStSTFD    , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [LSU_0]>],
+                              [6, 1, 1], // Latency = 3
+                              [GPR_Bypass, GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<LdStSTFDU   , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [SFX0, SFX1], 0>,
+                               InstrStage<1, [LSU_0]>],
+                              [6, 1, 1], // Latency = 3
+                              [GPR_Bypass, GPR_Bypass, GPR_Bypass],
+                              2>, // 2 micro-ops                              
+  InstrItinData<LdStLFD     , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [LSU_0]>],
+                              [7, 1, 1], // Latency = 4
+                              [FPR_Bypass, GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<LdStLFDU    , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [SFX0, SFX1], 0>,
+                               InstrStage<1, [LSU_0]>],
+                              [7, 1, 1], // Latency = 4
+                              [FPR_Bypass, GPR_Bypass, GPR_Bypass],
+                              2>, // 2 micro-ops
+  InstrItinData<LdStLHA     , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [LSU_0]>],
+                              [6, 1], // Latency = 3
+                              [GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<LdStLHAU    , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [SFX0, SFX1], 0>,
+                               InstrStage<1, [LSU_0]>],
+                              [6, 1], // Latency = 3
+                              [GPR_Bypass, GPR_Bypass]>,                              
+  InstrItinData<LdStLMW     , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [LSU_0]>],
+                              [7, 1], // Latency = r+3
+                              [NoBypass, GPR_Bypass]>,
+  InstrItinData<LdStLWARX   , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<3, [LSU_0]>],
+                              [6, 1, 1], // Latency = 3, Repeat rate = 3
+                              [GPR_Bypass, GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<LdStSTWCX   , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [LSU_0]>],
+                              [6, 1], // Latency = 3
+                              [NoBypass, GPR_Bypass]>,
+  InstrItinData<LdStSync    , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [LSU_0]>]>,
+  InstrItinData<SprMFSR     , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<4, [SFX0]>],
+                              [7, 1],
+                              [GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<SprMTMSR    , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<2, [SFX0, SFX1]>],
+                              [5, 1], // Latency = 2, Repeat rate = 4
+                              [GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<SprMTSR     , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [SFX0]>],
+                              [5, 1],
+                              [NoBypass, GPR_Bypass]>,
+  InstrItinData<SprTLBSYNC  , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [LSU_0], 0>]>,
+  InstrItinData<SprMFCR     , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<5, [SFX0]>],
+                              [8, 1],
+                              [GPR_Bypass, CR_Bypass]>,
+  InstrItinData<SprMFMSR    , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<4, [SFX0]>],
+                              [7, 1], // Latency = 4, Repeat rate = 4
+                              [GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<SprMFSPR    , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [SFX0, SFX1]>],
+                              [4, 1], // Latency = 1, Repeat rate = 1
+                              [GPR_Bypass, CR_Bypass]>,
+  InstrItinData<SprMFTB     , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<4, [SFX0]>],
+                              [7, 1], // Latency = 4, Repeat rate = 4
+                              [NoBypass, GPR_Bypass]>,
+  InstrItinData<SprMTSPR    , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [SFX0, SFX1]>],
+                              [4, 1], // Latency = 1, Repeat rate = 1
+                              [CR_Bypass, GPR_Bypass]>,
+  InstrItinData<SprMTSRIN   , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [SFX0]>],
+                              [4, 1],
+                              [NoBypass, GPR_Bypass]>,
+  InstrItinData<FPGeneral   , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<2, [FPU_0]>],
+                              [11, 1, 1], // Latency = 8, Repeat rate = 2 
+                              [FPR_Bypass, FPR_Bypass, FPR_Bypass]>,
+  InstrItinData<FPAddSub    , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<4, [FPU_0]>],
+                              [13, 1, 1], // Latency = 10, Repeat rate = 4 
+                              [FPR_Bypass, FPR_Bypass, FPR_Bypass]>,                              
+  InstrItinData<FPCompare   , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<2, [FPU_0]>],
+                              [11, 1, 1], // Latency = 8, Repeat rate = 2
+                              [CR_Bypass, FPR_Bypass, FPR_Bypass]>,
+  InstrItinData<FPDivD      , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<68, [FPU_0]>],
+                              [71, 1, 1], // Latency = 68, Repeat rate = 68
+                              [FPR_Bypass, FPR_Bypass, FPR_Bypass]>,
+  InstrItinData<FPDivS      , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<38, [FPU_0]>],
+                              [41, 1, 1], // Latency = 38, Repeat rate = 38
+                              [FPR_Bypass, FPR_Bypass, FPR_Bypass]>,
+  InstrItinData<FPFused     , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<4, [FPU_0]>],
+                              [13, 1, 1, 1], // Latency = 10, Repeat rate = 4
+                              [FPR_Bypass, FPR_Bypass, FPR_Bypass, FPR_Bypass]>,
+  InstrItinData<FPRes       , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<38, [FPU_0]>],
+                              [41, 1], // Latency = 38, Repeat rate = 38
+                              [FPR_Bypass, FPR_Bypass]>
+]>;
+
+// ===---------------------------------------------------------------------===//
+// e500mc machine model for scheduling and other instruction cost heuristics.
+
+def PPCE500mcModel : SchedMachineModel {
+  let IssueWidth = 2;  // 2 micro-ops are dispatched per cycle.
+  let MinLatency = -1; // OperandCycles are interpreted as MinLatency.
+  let LoadLatency = 5; // Optimistic load latency assuming bypass.
+                       // This is overriden by OperandCycles if the
+                       // Itineraries are queried instead.
+
+  let Itineraries = PPCE500mcItineraries;
+}
diff --git a/lib/Target/PowerPC/PPCScheduleE5500.td b/lib/Target/PowerPC/PPCScheduleE5500.td
new file mode 100644
index 0000000..d7e11ac
--- /dev/null
+++ b/lib/Target/PowerPC/PPCScheduleE5500.td
@@ -0,0 +1,309 @@
+//===-- PPCScheduleE500mc.td - e5500 Scheduling Defs -------*- tablegen -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the itinerary class data for the Freescale e5500 64-bit 
+// Power processor.
+// 
+// All information is derived from the "e5500 Core Reference Manual",
+// Freescale Document Number e5500RM, Rev. 1, 03/2012.
+//
+//===----------------------------------------------------------------------===//
+// Relevant functional units in the Freescale e5500 core
+// (These are the same as for the e500mc)
+//
+//  * Decode & Dispatch
+//    Can dispatch up to 2 instructions per clock cycle to either the GPR Issue
+//    queues (GIQx), FP Issue Queue (FIQ), or Branch issue queue (BIQ).
+// def DIS0 : FuncUnit;
+// def DIS1 : FuncUnit;
+
+//  * Execute
+//    6 pipelined execution units: SFX0, SFX1, BU, FPU, LSU, CFX.
+//    The CFX has a bypass path, allowing non-divide instructions to execute 
+//    while a divide instruction is being executed.
+// def SFX0  : FuncUnit; // Simple unit 0
+// def SFX1  : FuncUnit; // Simple unit 1
+// def BU    : FuncUnit; // Branch unit
+// def CFX_DivBypass 
+//           : FuncUnit; // CFX divide bypass path
+// def CFX_0 : FuncUnit; // CFX pipeline stage 0
+
+def CFX_1 : FuncUnit; // CFX pipeline stage 1 
+
+// def LSU_0 : FuncUnit; // LSU pipeline
+// def FPU_0 : FuncUnit; // FPU pipeline
+
+
+def PPCE5500Itineraries : ProcessorItineraries<
+  [DIS0, DIS1, SFX0, SFX1, BU, CFX_DivBypass, CFX_0, CFX_1,
+   LSU_0, FPU_0],
+  [CR_Bypass, GPR_Bypass, FPR_Bypass], [
+  InstrItinData<IntSimple   , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [SFX0, SFX1]>],
+                              [5, 2, 2], // Latency = 1
+                              [GPR_Bypass, GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<IntGeneral  , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [SFX0, SFX1]>],
+                              [5, 2, 2], // Latency = 1
+                              [GPR_Bypass, GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<IntCompare  , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [SFX0, SFX1]>],
+                              [6, 2, 2], // Latency = 1 or 2
+                              [CR_Bypass, GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<IntDivD     , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [CFX_0], 0>,
+                               InstrStage<26, [CFX_DivBypass]>],
+                              [30, 2, 2], // Latency= 4..26, Repeat rate= 4..26
+                              [GPR_Bypass, GPR_Bypass, GPR_Bypass]>,                              
+  InstrItinData<IntDivW     , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [CFX_0], 0>,
+                               InstrStage<16, [CFX_DivBypass]>],
+                              [20, 2, 2], // Latency= 4..16, Repeat rate= 4..16
+                              [GPR_Bypass, GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<IntMFFS     , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [FPU_0]>],
+                              [11], // Latency = 7, Repeat rate = 1
+                              [FPR_Bypass]>,
+  InstrItinData<IntMTFSB0   , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<7, [FPU_0]>],
+                              [11, 2, 2], // Latency = 7, Repeat rate = 7
+                              [NoBypass, NoBypass, NoBypass]>,
+  InstrItinData<IntMulHD    , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [CFX_0], 0>,
+                               InstrStage<2, [CFX_1]>],
+                              [9, 2, 2], // Latency = 4..7, Repeat rate = 2..4
+                              [GPR_Bypass, GPR_Bypass, GPR_Bypass]>,                              
+  InstrItinData<IntMulHW    , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [CFX_0], 0>,
+                               InstrStage<1, [CFX_1]>],
+                              [8, 2, 2], // Latency = 4, Repeat rate = 1
+                              [GPR_Bypass, GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<IntMulHWU   , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [CFX_0], 0>,
+                               InstrStage<1, [CFX_1]>],
+                              [8, 2, 2], // Latency = 4, Repeat rate = 1
+                              [GPR_Bypass, GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<IntMulLI    , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [CFX_0], 0>,
+                               InstrStage<2, [CFX_1]>],
+                              [8, 2, 2], // Latency = 4 or 5, Repeat = 2
+                              [GPR_Bypass, GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<IntRotate   , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [SFX0, SFX1]>],
+                              [5, 2, 2], // Latency = 1
+                              [GPR_Bypass, GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<IntRotateD  , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<2, [SFX0, SFX1]>],
+                              [6, 2, 2], // Latency = 2, Repeat rate = 2
+                              [GPR_Bypass, GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<IntRotateDI , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [SFX0, SFX1]>],
+                              [5, 2, 2], // Latency = 1, Repeat rate = 1
+                              [GPR_Bypass, GPR_Bypass, GPR_Bypass]>,                                                            
+  InstrItinData<IntShift    , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<2, [SFX0, SFX1]>],
+                              [6, 2, 2], // Latency = 2, Repeat rate = 2
+                              [GPR_Bypass, GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<IntTrapW    , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<2, [SFX0]>],
+                              [6, 2], // Latency = 2, Repeat rate = 2
+                              [GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<BrB         , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [BU]>],
+                              [5, 2], // Latency = 1
+                              [NoBypass, GPR_Bypass]>,
+  InstrItinData<BrCR        , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [BU]>],
+                              [5, 2, 2], // Latency = 1
+                              [CR_Bypass, CR_Bypass, CR_Bypass]>,
+  InstrItinData<BrMCR       , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [BU]>],
+                              [5, 2], // Latency = 1
+                              [CR_Bypass, CR_Bypass]>,
+  InstrItinData<BrMCRX      , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [CFX_0]>],
+                              [5, 2, 2], // Latency = 1
+                              [CR_Bypass, GPR_Bypass]>,
+  InstrItinData<LdStDCBA    , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [LSU_0]>],
+                              [7, 2], // Latency = 3, Repeat rate = 1
+                              [GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<LdStDCBF    , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [LSU_0]>],
+                              [7, 2], // Latency = 3, Repeat rate = 1
+                              [GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<LdStDCBI    , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [LSU_0]>],
+                              [7, 2], // Latency = 3, Repeat rate = 1
+                              [GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<LdStLoad    , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [LSU_0]>],
+                              [7, 2], // Latency = 3
+                              [GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<LdStLoadUpd , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [SFX0, SFX1], 0>,
+                               InstrStage<1, [LSU_0]>],
+                              [7, 2], // Latency = 3, Repeat rate = 1
+                              [GPR_Bypass, GPR_Bypass],
+                              2>, // 2 micro-ops
+  InstrItinData<LdStLD      , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [LSU_0]>],
+                              [7, 2], // Latency = 3, Repeat rate = 1
+                              [GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<LdStLDARX   , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<3, [LSU_0]>],
+                              [7, 2], // Latency = 3, Repeat rate = 3
+                              [GPR_Bypass, GPR_Bypass]>,                              
+  InstrItinData<LdStLDU     , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [SFX0, SFX1], 0>,
+                               InstrStage<1, [LSU_0]>],
+                              [7, 2], // Latency = 3, Repeat rate = 1
+                              [GPR_Bypass, GPR_Bypass],
+                              2>, // 2 micro-ops
+  InstrItinData<LdStStore   , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [LSU_0]>],
+                              [7, 2], // Latency = 3, Repeat rate = 1
+                              [NoBypass, GPR_Bypass]>,
+  InstrItinData<LdStStoreUpd, [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [SFX0, SFX1], 0>,
+                               InstrStage<1, [LSU_0]>],
+                              [7, 2], // Latency = 3, Repeat rate = 1
+                              [NoBypass, GPR_Bypass],
+                              2>, // 2 micro-ops                              
+  InstrItinData<LdStICBI    , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [LSU_0]>],
+                              [7, 2], // Latency = 3, Repeat rate = 1
+                              [NoBypass, GPR_Bypass]>,
+  InstrItinData<LdStSTFD    , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [LSU_0]>],
+                              [7, 2, 2], // Latency = 3, Repeat rate = 1
+                              [GPR_Bypass, GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<LdStSTFDU   , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [SFX0, SFX1], 0>,
+                               InstrStage<1, [LSU_0]>],
+                              [7, 2, 2], // Latency = 3, Repeat rate = 1
+                              [GPR_Bypass, GPR_Bypass, GPR_Bypass],
+                              2>, // 2 micro-ops                              
+  InstrItinData<LdStLFD     , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [LSU_0]>],
+                              [8, 2, 2], // Latency = 4, Repeat rate = 1
+                              [FPR_Bypass, GPR_Bypass, GPR_Bypass],
+                              2>, // 2 micro-ops
+  InstrItinData<LdStLFDU    , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [SFX0, SFX1], 0>,
+                               InstrStage<1, [LSU_0]>],
+                              [8, 2, 2], // Latency = 4, Repeat rate = 1
+                              [FPR_Bypass, GPR_Bypass, GPR_Bypass],
+                              2>, // 2 micro-ops
+  InstrItinData<LdStLHA     , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [LSU_0]>],
+                              [7, 2], // Latency = 3
+                              [GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<LdStLHAU    , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [SFX0, SFX1], 0>,
+                               InstrStage<1, [LSU_0]>],
+                              [7, 2], // Latency = 3, Repeat rate = 1
+                              [GPR_Bypass, GPR_Bypass],
+                              2>, // 2 micro-ops                              
+  InstrItinData<LdStLMW     , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<4, [LSU_0]>],
+                              [8, 2], // Latency = r+3, Repeat rate = r+3
+                              [NoBypass, GPR_Bypass]>,
+  InstrItinData<LdStLWARX   , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<3, [LSU_0]>],
+                              [7, 2, 2], // Latency = 3, Repeat rate = 3
+                              [GPR_Bypass, GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<LdStSTD     , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [LSU_0]>],
+                              [7, 2], // Latency = 3, Repeat rate = 1                              
+                              [NoBypass, GPR_Bypass]>,
+  InstrItinData<LdStSTDCX   , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [LSU_0]>],
+                              [7, 2], // Latency = 3, Repeat rate = 1                              
+                              [NoBypass, GPR_Bypass]>,                              
+  InstrItinData<LdStSTDU    , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [SFX0, SFX1], 0>,
+                               InstrStage<1, [LSU_0]>],
+                              [7, 2], // Latency = 3, Repeat rate = 1
+                              [NoBypass, GPR_Bypass],
+                              2>, // 2 micro-ops                              
+  InstrItinData<LdStSTWCX   , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [LSU_0]>],
+                              [7, 2], // Latency = 3, Repeat rate = 1
+                              [NoBypass, GPR_Bypass]>,
+  InstrItinData<LdStSync    , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [LSU_0]>]>,
+  InstrItinData<SprMTMSR    , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<2, [CFX_0]>],
+                              [6, 2], // Latency = 2, Repeat rate = 4
+                              [GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<SprTLBSYNC  , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [LSU_0], 0>]>,
+  InstrItinData<SprMFCR     , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<5, [CFX_0]>],
+                              [9, 2], // Latency = 5, Repeat rate = 5
+                              [GPR_Bypass, CR_Bypass]>,
+  InstrItinData<SprMFMSR    , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<4, [SFX0]>],
+                              [8, 2], // Latency = 4, Repeat rate = 4
+                              [GPR_Bypass, GPR_Bypass]>,
+  InstrItinData<SprMFSPR    , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [CFX_0]>],
+                              [5], // Latency = 1, Repeat rate = 1
+                              [GPR_Bypass]>,
+  InstrItinData<SprMFTB     , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<4, [CFX_0]>],
+                              [8, 2], // Latency = 4, Repeat rate = 4
+                              [NoBypass, GPR_Bypass]>,
+  InstrItinData<SprMTSPR    , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [SFX0, SFX1]>],
+                              [5], // Latency = 1, Repeat rate = 1
+                              [GPR_Bypass]>,
+  InstrItinData<FPGeneral   , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [FPU_0]>],
+                              [11, 2, 2], // Latency = 7, Repeat rate = 1 
+                              [FPR_Bypass, FPR_Bypass, FPR_Bypass]>,
+  InstrItinData<FPAddSub    , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [FPU_0]>],
+                              [11, 2, 2], // Latency = 7, Repeat rate = 1 
+                              [FPR_Bypass, FPR_Bypass, FPR_Bypass]>,                              
+  InstrItinData<FPCompare   , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [FPU_0]>],
+                              [11, 2, 2], // Latency = 7, Repeat rate = 1
+                              [CR_Bypass, FPR_Bypass, FPR_Bypass]>,
+  InstrItinData<FPDivD      , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<31, [FPU_0]>],
+                              [39, 2, 2], // Latency = 35, Repeat rate = 31
+                              [FPR_Bypass, FPR_Bypass, FPR_Bypass]>,
+  InstrItinData<FPDivS      , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<16, [FPU_0]>],
+                              [24, 2, 2], // Latency = 20, Repeat rate = 16 
+                              [FPR_Bypass, FPR_Bypass, FPR_Bypass]>,
+  InstrItinData<FPFused     , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<1, [FPU_0]>],
+                              [11, 2, 2, 2], // Latency = 7, Repeat rate = 1
+                              [FPR_Bypass, FPR_Bypass, FPR_Bypass, FPR_Bypass]>,
+  InstrItinData<FPRes       , [InstrStage<1, [DIS0, DIS1], 0>,
+                               InstrStage<2, [FPU_0]>],
+                              [12, 2], // Latency = 8, Repeat rate = 2
+                              [FPR_Bypass, FPR_Bypass]>
+]>;
+
+// ===---------------------------------------------------------------------===//
+// e5500 machine model for scheduling and other instruction cost heuristics.
+
+def PPCE5500Model : SchedMachineModel {
+  let IssueWidth = 2;  // 2 micro-ops are dispatched per cycle.
+  let MinLatency = -1; // OperandCycles are interpreted as MinLatency.
+  let LoadLatency = 6; // Optimistic load latency assuming bypass.
+                       // This is overriden by OperandCycles if the
+                       // Itineraries are queried instead.
+
+  let Itineraries = PPCE5500Itineraries;
+}
diff --git a/lib/Target/PowerPC/PPCScheduleG3.td b/lib/Target/PowerPC/PPCScheduleG3.td
index 61e89ed..72a0a39 100644
--- a/lib/Target/PowerPC/PPCScheduleG3.td
+++ b/lib/Target/PowerPC/PPCScheduleG3.td
@@ -34,12 +34,16 @@ def G3Itineraries : ProcessorItineraries<
   InstrItinData<LdStDCBF    , [InstrStage<3, [SLU]>]>,
   InstrItinData<LdStDCBI    , [InstrStage<3, [SLU]>]>,
   InstrItinData<LdStLoad    , [InstrStage<2, [SLU]>]>,
+  InstrItinData<LdStLoadUpd , [InstrStage<2, [SLU]>]>,  
   InstrItinData<LdStStore   , [InstrStage<2, [SLU]>]>,
+  InstrItinData<LdStStoreUpd, [InstrStage<2, [SLU]>]>,  
   InstrItinData<LdStICBI    , [InstrStage<3, [SLU]>]>,
-  InstrItinData<LdStUX      , [InstrStage<2, [SLU]>]>,
+  InstrItinData<LdStSTFD    , [InstrStage<2, [SLU]>]>,
+  InstrItinData<LdStSTFDU   , [InstrStage<2, [SLU]>]>,
   InstrItinData<LdStLFD     , [InstrStage<2, [SLU]>]>,
   InstrItinData<LdStLFDU    , [InstrStage<2, [SLU]>]>,
   InstrItinData<LdStLHA     , [InstrStage<2, [SLU]>]>,
+  InstrItinData<LdStLHAU    , [InstrStage<2, [SLU]>]>,  
   InstrItinData<LdStLMW     , [InstrStage<34, [SLU]>]>,
   InstrItinData<LdStLWARX   , [InstrStage<3, [SLU]>]>,
   InstrItinData<LdStSTWCX   , [InstrStage<8, [SLU]>]>,
@@ -58,6 +62,7 @@ def G3Itineraries : ProcessorItineraries<
   InstrItinData<SprRFI      , [InstrStage<2, [SRU]>]>,
   InstrItinData<SprSC       , [InstrStage<2, [SRU]>]>,
   InstrItinData<FPGeneral   , [InstrStage<1, [FPU1]>]>,
+  InstrItinData<FPAddSub    , [InstrStage<1, [FPU1]>]>,
   InstrItinData<FPCompare   , [InstrStage<1, [FPU1]>]>,
   InstrItinData<FPDivD      , [InstrStage<31, [FPU1]>]>,
   InstrItinData<FPDivS      , [InstrStage<17, [FPU1]>]>,
diff --git a/lib/Target/PowerPC/PPCScheduleG4.td b/lib/Target/PowerPC/PPCScheduleG4.td
index e19ddfa..fc9120d 100644
--- a/lib/Target/PowerPC/PPCScheduleG4.td
+++ b/lib/Target/PowerPC/PPCScheduleG4.td
@@ -33,13 +33,17 @@ def G4Itineraries : ProcessorItineraries<
   InstrItinData<LdStDCBF    , [InstrStage<2, [SLU]>]>,
   InstrItinData<LdStDCBI    , [InstrStage<2, [SLU]>]>,
   InstrItinData<LdStLoad    , [InstrStage<2, [SLU]>]>,
+  InstrItinData<LdStLoadUpd , [InstrStage<2, [SLU]>]>,
   InstrItinData<LdStStore   , [InstrStage<2, [SLU]>]>,
+  InstrItinData<LdStStoreUpd, [InstrStage<2, [SLU]>]>,
   InstrItinData<LdStDSS     , [InstrStage<2, [SLU]>]>,
   InstrItinData<LdStICBI    , [InstrStage<2, [SLU]>]>,
-  InstrItinData<LdStUX      , [InstrStage<2, [SLU]>]>,
+  InstrItinData<LdStSTFD    , [InstrStage<2, [SLU]>]>,
+  InstrItinData<LdStSTFDU   , [InstrStage<2, [SLU]>]>,
   InstrItinData<LdStLFD     , [InstrStage<2, [SLU]>]>,
   InstrItinData<LdStLFDU    , [InstrStage<2, [SLU]>]>,
   InstrItinData<LdStLHA     , [InstrStage<2, [SLU]>]>,
+  InstrItinData<LdStLHAU    , [InstrStage<2, [SLU]>]>, 
   InstrItinData<LdStLMW     , [InstrStage<34, [SLU]>]>,
   InstrItinData<LdStLVecX   , [InstrStage<2, [SLU]>]>,
   InstrItinData<LdStLWARX   , [InstrStage<3, [SLU]>]>,
@@ -60,6 +64,7 @@ def G4Itineraries : ProcessorItineraries<
   InstrItinData<SprRFI      , [InstrStage<2, [SRU]>]>,
   InstrItinData<SprSC       , [InstrStage<2, [SRU]>]>,
   InstrItinData<FPGeneral   , [InstrStage<1, [FPU1]>]>,
+  InstrItinData<FPAddSub    , [InstrStage<1, [FPU1]>]>,
   InstrItinData<FPCompare   , [InstrStage<1, [FPU1]>]>,
   InstrItinData<FPDivD      , [InstrStage<31, [FPU1]>]>,
   InstrItinData<FPDivS      , [InstrStage<17, [FPU1]>]>,
diff --git a/lib/Target/PowerPC/PPCScheduleG4Plus.td b/lib/Target/PowerPC/PPCScheduleG4Plus.td
index e7446cb..a4e82ce 100644
--- a/lib/Target/PowerPC/PPCScheduleG4Plus.td
+++ b/lib/Target/PowerPC/PPCScheduleG4Plus.td
@@ -36,19 +36,24 @@ def G4PlusItineraries : ProcessorItineraries<
   InstrItinData<LdStDCBF    , [InstrStage<3, [SLU]>]>,
   InstrItinData<LdStDCBI    , [InstrStage<3, [SLU]>]>,
   InstrItinData<LdStLoad    , [InstrStage<3, [SLU]>]>,
+  InstrItinData<LdStLoadUpd , [InstrStage<3, [SLU]>]>,
   InstrItinData<LdStStore   , [InstrStage<3, [SLU]>]>,
+  InstrItinData<LdStStoreUpd, [InstrStage<3, [SLU]>]>,
   InstrItinData<LdStDSS     , [InstrStage<3, [SLU]>]>,
   InstrItinData<LdStICBI    , [InstrStage<3, [IU2]>]>,
-  InstrItinData<LdStUX      , [InstrStage<3, [SLU]>]>,
+  InstrItinData<LdStSTFD    , [InstrStage<3, [SLU]>]>,
+  InstrItinData<LdStSTFDU   , [InstrStage<3, [SLU]>]>,
   InstrItinData<LdStLFD     , [InstrStage<4, [SLU]>]>,
   InstrItinData<LdStLFDU    , [InstrStage<4, [SLU]>]>,
   InstrItinData<LdStLHA     , [InstrStage<3, [SLU]>]>,
+  InstrItinData<LdStLHAU    , [InstrStage<3, [SLU]>]>,  
   InstrItinData<LdStLMW     , [InstrStage<37, [SLU]>]>,
   InstrItinData<LdStLVecX   , [InstrStage<3, [SLU]>]>,
   InstrItinData<LdStLWA     , [InstrStage<3, [SLU]>]>,
   InstrItinData<LdStLWARX   , [InstrStage<3, [SLU]>]>,
   InstrItinData<LdStSTD     , [InstrStage<3, [SLU]>]>,
   InstrItinData<LdStSTDCX   , [InstrStage<3, [SLU]>]>,
+  InstrItinData<LdStSTDU    , [InstrStage<3, [SLU]>]>,  
   InstrItinData<LdStSTVEBX  , [InstrStage<3, [SLU]>]>,
   InstrItinData<LdStSTWCX   , [InstrStage<3, [SLU]>]>,
   InstrItinData<LdStSync    , [InstrStage<35, [SLU]>]>,
@@ -66,6 +71,7 @@ def G4PlusItineraries : ProcessorItineraries<
   InstrItinData<SprRFI      , [InstrStage<1, [IU1, IU2, IU3, IU4]>]>,
   InstrItinData<SprSC       , [InstrStage<0, [IU1, IU2, IU3, IU4]>]>,
   InstrItinData<FPGeneral   , [InstrStage<5, [FPU1]>]>,
+  InstrItinData<FPAddSub    , [InstrStage<5, [FPU1]>]>,  
   InstrItinData<FPCompare   , [InstrStage<5, [FPU1]>]>,
   InstrItinData<FPDivD      , [InstrStage<35, [FPU1]>]>,
   InstrItinData<FPDivS      , [InstrStage<21, [FPU1]>]>,
diff --git a/lib/Target/PowerPC/PPCScheduleG5.td b/lib/Target/PowerPC/PPCScheduleG5.td
index 1371499..7c02ea0 100644
--- a/lib/Target/PowerPC/PPCScheduleG5.td
+++ b/lib/Target/PowerPC/PPCScheduleG5.td
@@ -27,6 +27,7 @@ def G5Itineraries : ProcessorItineraries<
   InstrItinData<IntMulLI    , [InstrStage<4, [IU1, IU2]>]>,
   InstrItinData<IntRFID     , [InstrStage<1, [IU2]>]>,
   InstrItinData<IntRotateD  , [InstrStage<2, [IU1, IU2]>]>,
+  InstrItinData<IntRotateDI , [InstrStage<2, [IU1, IU2]>]>,  
   InstrItinData<IntRotate   , [InstrStage<4, [IU1, IU2]>]>,
   InstrItinData<IntShift    , [InstrStage<2, [IU1, IU2]>]>,
   InstrItinData<IntTrapD    , [InstrStage<1, [IU1, IU2]>]>,
@@ -37,15 +38,20 @@ def G5Itineraries : ProcessorItineraries<
   InstrItinData<BrMCRX      , [InstrStage<3, [BPU]>]>,
   InstrItinData<LdStDCBF    , [InstrStage<3, [SLU]>]>,
   InstrItinData<LdStLoad    , [InstrStage<3, [SLU]>]>,
+  InstrItinData<LdStLoadUpd , [InstrStage<3, [SLU]>]>,  
   InstrItinData<LdStStore   , [InstrStage<3, [SLU]>]>,
+  InstrItinData<LdStStoreUpd, [InstrStage<3, [SLU]>]>,  
   InstrItinData<LdStDSS     , [InstrStage<10, [SLU]>]>,
   InstrItinData<LdStICBI    , [InstrStage<40, [SLU]>]>,
-  InstrItinData<LdStUX      , [InstrStage<4, [SLU]>]>,
+  InstrItinData<LdStSTFD    , [InstrStage<4, [SLU]>]>,
+  InstrItinData<LdStSTFDU   , [InstrStage<4, [SLU]>]>,  
   InstrItinData<LdStLD      , [InstrStage<3, [SLU]>]>,
+  InstrItinData<LdStLDU     , [InstrStage<3, [SLU]>]>,
   InstrItinData<LdStLDARX   , [InstrStage<11, [SLU]>]>,
   InstrItinData<LdStLFD     , [InstrStage<3, [SLU]>]>,
   InstrItinData<LdStLFDU    , [InstrStage<5, [SLU]>]>,
   InstrItinData<LdStLHA     , [InstrStage<5, [SLU]>]>,
+  InstrItinData<LdStLHAU    , [InstrStage<5, [SLU]>]>,  
   InstrItinData<LdStLMW     , [InstrStage<64, [SLU]>]>,
   InstrItinData<LdStLVecX   , [InstrStage<3, [SLU]>]>,
   InstrItinData<LdStLWA     , [InstrStage<5, [SLU]>]>,
@@ -53,6 +59,7 @@ def G5Itineraries : ProcessorItineraries<
   InstrItinData<LdStSLBIA   , [InstrStage<40, [SLU]>]>, // needs work
   InstrItinData<LdStSLBIE   , [InstrStage<2, [SLU]>]>,
   InstrItinData<LdStSTD     , [InstrStage<3, [SLU]>]>,
+  InstrItinData<LdStSTDU    , [InstrStage<3, [SLU]>]>,
   InstrItinData<LdStSTDCX   , [InstrStage<11, [SLU]>]>,
   InstrItinData<LdStSTVEBX  , [InstrStage<5, [SLU]>]>,
   InstrItinData<LdStSTWCX   , [InstrStage<11, [SLU]>]>,
@@ -69,6 +76,7 @@ def G5Itineraries : ProcessorItineraries<
   InstrItinData<SprMTSPR    , [InstrStage<8, [IU2]>]>,
   InstrItinData<SprSC       , [InstrStage<1, [IU2]>]>,
   InstrItinData<FPGeneral   , [InstrStage<6, [FPU1, FPU2]>]>,
+  InstrItinData<FPAddSub    , [InstrStage<6, [FPU1, FPU2]>]>,
   InstrItinData<FPCompare   , [InstrStage<8, [FPU1, FPU2]>]>,
   InstrItinData<FPDivD      , [InstrStage<33, [FPU1, FPU2]>]>,
   InstrItinData<FPDivS      , [InstrStage<33, [FPU1, FPU2]>]>,
diff --git a/lib/Target/PowerPC/PPCSubtarget.cpp b/lib/Target/PowerPC/PPCSubtarget.cpp
index bb193ac..9c8cb92 100644
--- a/lib/Target/PowerPC/PPCSubtarget.cpp
+++ b/lib/Target/PowerPC/PPCSubtarget.cpp
@@ -54,19 +54,26 @@ PPCSubtarget::PPCSubtarget(const std::string &TT, const std::string &CPU,
     CPUName = sys::getHostCPUName();
 #endif
 
-  // Parse features string.
-  ParseSubtargetFeatures(CPUName, FS);
-
   // Initialize scheduling itinerary for the specified CPU.
   InstrItins = getInstrItineraryForCPU(CPUName);
 
+  // Make sure 64-bit features are available when CPUname is generic
+  std::string FullFS = FS;
+
   // If we are generating code for ppc64, verify that options make sense.
   if (is64Bit) {
     Has64BitSupport = true;
     // Silently force 64-bit register use on ppc64.
     Use64BitRegs = true;
+    if (!FullFS.empty())
+      FullFS = "+64bit," + FullFS;
+    else
+      FullFS = "+64bit";
   }
-  
+
+  // Parse features string.
+  ParseSubtargetFeatures(CPUName, FullFS);
+
   // If the user requested use of 64-bit regs, but the cpu selected doesn't
   // support it, ignore.
   if (use64BitRegs() && !has64BitSupport())
diff --git a/lib/Target/PowerPC/PPCSubtarget.h b/lib/Target/PowerPC/PPCSubtarget.h
index 0207c83..b9e22f4 100644
--- a/lib/Target/PowerPC/PPCSubtarget.h
+++ b/lib/Target/PowerPC/PPCSubtarget.h
@@ -33,32 +33,34 @@ namespace PPC {
   enum {
     DIR_NONE,
     DIR_32,
-    DIR_440, 
-    DIR_601, 
-    DIR_602, 
-    DIR_603, 
+    DIR_440,
+    DIR_601,
+    DIR_602,
+    DIR_603,
     DIR_7400,
-    DIR_750, 
-    DIR_970, 
+    DIR_750,
+    DIR_970,
     DIR_A2,
+    DIR_E500mc,
+    DIR_E5500,
     DIR_PWR6,
     DIR_PWR7,
-    DIR_64  
+    DIR_64
   };
 }
 
 class GlobalValue;
 class TargetMachine;
-  
+
 class PPCSubtarget : public PPCGenSubtargetInfo {
 protected:
   /// stackAlignment - The minimum alignment known to hold of the stack frame on
   /// entry to the function and which must be maintained by every function.
   unsigned StackAlignment;
-  
+
   /// Selected instruction itineraries (one entry per itinerary class.)
   InstrItineraryData InstrItins;
-  
+
   /// Which cpu directive was used.
   unsigned DarwinDirective;
 
@@ -74,7 +76,7 @@ protected:
   bool IsBookE;
   bool HasLazyResolverStubs;
   bool IsJITCodeModel;
-  
+
   /// TargetTriple - What processor and OS we're targeting.
   Triple TargetTriple;
 
@@ -84,11 +86,11 @@ public:
   ///
   PPCSubtarget(const std::string &TT, const std::string &CPU,
                const std::string &FS, bool is64Bit);
-  
-  /// ParseSubtargetFeatures - Parses features string setting specified 
+
+  /// ParseSubtargetFeatures - Parses features string setting specified
   /// subtarget options.  Definition of function is auto generated by tblgen.
   void ParseSubtargetFeatures(StringRef CPU, StringRef FS);
-  
+
   /// SetJITMode - This is called to inform the subtarget info that we are
   /// producing code for the JIT.
   void SetJITMode();
@@ -97,20 +99,27 @@ public:
   /// stack frame on entry to the function and which must be maintained by every
   /// function for this subtarget.
   unsigned getStackAlignment() const { return StackAlignment; }
-  
+
   /// getDarwinDirective - Returns the -m directive specified for the cpu.
   ///
   unsigned getDarwinDirective() const { return DarwinDirective; }
-  
-  /// getInstrItins - Return the instruction itineraies based on subtarget 
+
+  /// getInstrItins - Return the instruction itineraies based on subtarget
   /// selection.
   const InstrItineraryData &getInstrItineraryData() const { return InstrItins; }
 
-  /// getTargetDataString - Return the pointer size and type alignment
+  /// getDataLayoutString - Return the pointer size and type alignment
   /// properties of this subtarget.
-  const char *getTargetDataString() const {
+  const char *getDataLayoutString() const {
     // Note, the alignment values for f64 and i64 on ppc64 in Darwin
     // documentation are wrong; these are correct (i.e. "what gcc does").
+    if (isPPC64() && isSVR4ABI()) {
+      if (TargetTriple.getOS() == llvm::Triple::FreeBSD)
+        return "E-p:64:64-f64:64:64-i64:64:64-f128:64:64-v128:128:128-n32:64";
+      else
+        return "E-p:64:64-f64:64:64-i64:64:64-f128:128:128-v128:128:128-n32:64";
+    }
+
     return isPPC64() ? "E-p:64:64-f64:64:64-i64:64:64-f128:64:128-n32:64"
                      : "E-p:32:32-f64:64:64-i64:64:64-f128:64:128-n32";
   }
@@ -118,22 +127,22 @@ public:
   /// isPPC64 - Return true if we are generating code for 64-bit pointer mode.
   ///
   bool isPPC64() const { return IsPPC64; }
-  
+
   /// has64BitSupport - Return true if the selected CPU supports 64-bit
   /// instructions, regardless of whether we are in 32-bit or 64-bit mode.
   bool has64BitSupport() const { return Has64BitSupport; }
-  
+
   /// use64BitRegs - Return true if in 64-bit mode or if we should use 64-bit
   /// registers in 32-bit mode when possible.  This can only true if
   /// has64BitSupport() returns true.
   bool use64BitRegs() const { return Use64BitRegs; }
-  
+
   /// hasLazyResolverStub - Return true if accesses to the specified global have
   /// to go through a dyld lazy resolution stub.  This means that an extra load
   /// is required to get the address of the global.
-  bool hasLazyResolverStub(const GlobalValue *GV, 
+  bool hasLazyResolverStub(const GlobalValue *GV,
                            const TargetMachine &TM) const;
-  
+
   // isJITCodeModel - True if we're generating code for the JIT
   bool isJITCodeModel() const { return IsJITCodeModel; }
 
diff --git a/lib/Target/PowerPC/PPCTargetMachine.cpp b/lib/Target/PowerPC/PPCTargetMachine.cpp
index 9805112..3fc977e 100644
--- a/lib/Target/PowerPC/PPCTargetMachine.cpp
+++ b/lib/Target/PowerPC/PPCTargetMachine.cpp
@@ -40,10 +40,11 @@ PPCTargetMachine::PPCTargetMachine(const Target &T, StringRef TT,
                                    bool is64Bit)
   : LLVMTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL),
     Subtarget(TT, CPU, FS, is64Bit),
-    DataLayout(Subtarget.getTargetDataString()), InstrInfo(*this),
+    DL(Subtarget.getDataLayoutString()), InstrInfo(*this),
     FrameLowering(Subtarget), JITInfo(*this, is64Bit),
     TLInfo(*this), TSInfo(*this),
-    InstrItins(Subtarget.getInstrItineraryData()) {
+    InstrItins(Subtarget.getInstrItineraryData()),
+    STTI(&TLInfo), VTTI(&TLInfo) {
 
   // The binutils for the BG/P are too old for CFI.
   if (Subtarget.isBGP())
diff --git a/lib/Target/PowerPC/PPCTargetMachine.h b/lib/Target/PowerPC/PPCTargetMachine.h
index 7da2b0c..c168433 100644
--- a/lib/Target/PowerPC/PPCTargetMachine.h
+++ b/lib/Target/PowerPC/PPCTargetMachine.h
@@ -21,7 +21,8 @@
 #include "PPCISelLowering.h"
 #include "PPCSelectionDAGInfo.h"
 #include "llvm/Target/TargetMachine.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/Target/TargetTransformImpl.h"
+#include "llvm/DataLayout.h"
 
 namespace llvm {
 
@@ -29,13 +30,15 @@ namespace llvm {
 ///
 class PPCTargetMachine : public LLVMTargetMachine {
   PPCSubtarget        Subtarget;
-  const TargetData    DataLayout;       // Calculates type size & alignment
+  const DataLayout    DL;       // Calculates type size & alignment
   PPCInstrInfo        InstrInfo;
   PPCFrameLowering    FrameLowering;
   PPCJITInfo          JITInfo;
   PPCTargetLowering   TLInfo;
   PPCSelectionDAGInfo TSInfo;
   InstrItineraryData  InstrItins;
+  ScalarTargetTransformImpl STTI;
+  VectorTargetTransformImpl VTTI;
 
 public:
   PPCTargetMachine(const Target &T, StringRef TT,
@@ -58,11 +61,17 @@ public:
     return &InstrInfo.getRegisterInfo();
   }
 
-  virtual const TargetData    *getTargetData() const    { return &DataLayout; }
+  virtual const DataLayout    *getDataLayout() const    { return &DL; }
   virtual const PPCSubtarget  *getSubtargetImpl() const { return &Subtarget; }
   virtual const InstrItineraryData *getInstrItineraryData() const {
     return &InstrItins;
   }
+  virtual const ScalarTargetTransformInfo *getScalarTargetTransformInfo()const {
+    return &STTI;
+  }
+  virtual const VectorTargetTransformInfo *getVectorTargetTransformInfo()const {
+    return &VTTI;
+  }
 
   // Pass Pipeline Configuration
   virtual TargetPassConfig *createPassConfig(PassManagerBase &PM);
diff --git a/lib/Target/README.txt b/lib/Target/README.txt
index cbfa4cf..8165f5b 100644
--- a/lib/Target/README.txt
+++ b/lib/Target/README.txt
@@ -152,7 +152,7 @@ stuff too.
 
 //===---------------------------------------------------------------------===//
 
-For vector types, TargetData.cpp::getTypeInfo() returns alignment that is equal
+For vector types, DataLayout.cpp::getTypeInfo() returns alignment that is equal
 to the type size. It works but can be overly conservative as the alignment of
 specific vector types are target dependent.
 
@@ -2367,8 +2367,3 @@ unsigned foo(unsigned x, unsigned y) { return x > y && x != 0; }
 should fold to x > y.
 
 //===---------------------------------------------------------------------===//
-
-int f(double x) { return __builtin_fabs(x) < 0.0; }
-should fold to false.
-
-//===---------------------------------------------------------------------===//
diff --git a/lib/Target/Sparc/SparcFrameLowering.cpp b/lib/Target/Sparc/SparcFrameLowering.cpp
index 1c5c89e..716c79f 100644
--- a/lib/Target/Sparc/SparcFrameLowering.cpp
+++ b/lib/Target/Sparc/SparcFrameLowering.cpp
@@ -20,7 +20,7 @@
 #include "llvm/CodeGen/MachineInstrBuilder.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetOptions.h"
 #include "llvm/Support/CommandLine.h"
 
diff --git a/lib/Target/Sparc/SparcISelLowering.cpp b/lib/Target/Sparc/SparcISelLowering.cpp
index 79f7ebd..8e5619e 100644
--- a/lib/Target/Sparc/SparcISelLowering.cpp
+++ b/lib/Target/Sparc/SparcISelLowering.cpp
@@ -637,7 +637,7 @@ SparcTargetLowering::getSRetArgSize(SelectionDAG &DAG, SDValue Callee) const
 
   PointerType *Ty = cast<PointerType>(CalleeFn->arg_begin()->getType());
   Type *ElementTy = Ty->getElementType();
-  return getTargetData()->getTypeAllocSize(ElementTy);
+  return getDataLayout()->getTypeAllocSize(ElementTy);
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/lib/Target/Sparc/SparcInstrInfo.td b/lib/Target/Sparc/SparcInstrInfo.td
index 15541ef..e64c140 100644
--- a/lib/Target/Sparc/SparcInstrInfo.td
+++ b/lib/Target/Sparc/SparcInstrInfo.td
@@ -129,7 +129,7 @@ def retflag       : SDNode<"SPISD::RET_FLAG", SDT_SPRet,
                            [SDNPHasChain, SDNPOptInGlue]>;
 
 def flushw        : SDNode<"SPISD::FLUSHW", SDTNone,
-                           [SDNPHasChain]>;
+                           [SDNPHasChain, SDNPSideEffect, SDNPMayStore]>;
 
 def getPCX        : Operand<i32> {
   let PrintMethod = "printGetPCX";
diff --git a/lib/Target/Sparc/SparcTargetMachine.cpp b/lib/Target/Sparc/SparcTargetMachine.cpp
index 9ee12ed..45c9624 100644
--- a/lib/Target/Sparc/SparcTargetMachine.cpp
+++ b/lib/Target/Sparc/SparcTargetMachine.cpp
@@ -33,10 +33,10 @@ SparcTargetMachine::SparcTargetMachine(const Target &T, StringRef TT,
                                        bool is64bit)
   : LLVMTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL),
     Subtarget(TT, CPU, FS, is64bit),
-    DataLayout(Subtarget.getDataLayout()),
+    DL(Subtarget.getDataLayout()),
     InstrInfo(Subtarget),
     TLInfo(*this), TSInfo(*this),
-    FrameLowering(Subtarget) {
+    FrameLowering(Subtarget), STTI(&TLInfo), VTTI(&TLInfo) {
 }
 
 namespace {
diff --git a/lib/Target/Sparc/SparcTargetMachine.h b/lib/Target/Sparc/SparcTargetMachine.h
index b2cc624..0fbe2d7 100644
--- a/lib/Target/Sparc/SparcTargetMachine.h
+++ b/lib/Target/Sparc/SparcTargetMachine.h
@@ -20,18 +20,21 @@
 #include "SparcSelectionDAGInfo.h"
 #include "SparcSubtarget.h"
 #include "llvm/Target/TargetMachine.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetFrameLowering.h"
+#include "llvm/Target/TargetTransformImpl.h"
 
 namespace llvm {
 
 class SparcTargetMachine : public LLVMTargetMachine {
   SparcSubtarget Subtarget;
-  const TargetData DataLayout;       // Calculates type size & alignment
+  const DataLayout DL;       // Calculates type size & alignment
   SparcInstrInfo InstrInfo;
   SparcTargetLowering TLInfo;
   SparcSelectionDAGInfo TSInfo;
   SparcFrameLowering FrameLowering;
+  ScalarTargetTransformImpl STTI;
+  VectorTargetTransformImpl VTTI;
 public:
   SparcTargetMachine(const Target &T, StringRef TT,
                      StringRef CPU, StringRef FS, const TargetOptions &Options,
@@ -52,7 +55,13 @@ public:
   virtual const SparcSelectionDAGInfo* getSelectionDAGInfo() const {
     return &TSInfo;
   }
-  virtual const TargetData       *getTargetData() const { return &DataLayout; }
+  virtual const ScalarTargetTransformInfo *getScalarTargetTransformInfo()const {
+    return &STTI;
+  }
+  virtual const VectorTargetTransformInfo *getVectorTargetTransformInfo()const {
+    return &VTTI;
+  }
+  virtual const DataLayout       *getDataLayout() const { return &DL; }
 
   // Pass Pipeline Configuration
   virtual TargetPassConfig *createPassConfig(PassManagerBase &PM);
diff --git a/lib/Target/Target.cpp b/lib/Target/Target.cpp
index a2b83bc..393178a 100644
--- a/lib/Target/Target.cpp
+++ b/lib/Target/Target.cpp
@@ -16,7 +16,7 @@
 #include "llvm-c/Initialization.h"
 #include "llvm/InitializePasses.h"
 #include "llvm/PassManager.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetLibraryInfo.h"
 #include "llvm/LLVMContext.h"
 #include <cstring>
@@ -24,8 +24,9 @@
 using namespace llvm;
 
 void llvm::initializeTarget(PassRegistry &Registry) {
-  initializeTargetDataPass(Registry);
+  initializeDataLayoutPass(Registry);
   initializeTargetLibraryInfoPass(Registry);
+  initializeTargetTransformInfoPass(Registry);
 }
 
 void LLVMInitializeTarget(LLVMPassRegistryRef R) {
@@ -33,11 +34,11 @@ void LLVMInitializeTarget(LLVMPassRegistryRef R) {
 }
 
 LLVMTargetDataRef LLVMCreateTargetData(const char *StringRep) {
-  return wrap(new TargetData(StringRep));
+  return wrap(new DataLayout(StringRep));
 }
 
 void LLVMAddTargetData(LLVMTargetDataRef TD, LLVMPassManagerRef PM) {
-  unwrap(PM)->add(new TargetData(*unwrap(TD)));
+  unwrap(PM)->add(new DataLayout(*unwrap(TD)));
 }
 
 void LLVMAddTargetLibraryInfo(LLVMTargetLibraryInfoRef TLI,
@@ -55,13 +56,21 @@ LLVMByteOrdering LLVMByteOrder(LLVMTargetDataRef TD) {
 }
 
 unsigned LLVMPointerSize(LLVMTargetDataRef TD) {
-  return unwrap(TD)->getPointerSize();
+  return unwrap(TD)->getPointerSize(0);
+}
+
+unsigned LLVMPointerSizeForAS(LLVMTargetDataRef TD, unsigned AS) {
+  return unwrap(TD)->getPointerSize(AS);
 }
 
 LLVMTypeRef LLVMIntPtrType(LLVMTargetDataRef TD) {
   return wrap(unwrap(TD)->getIntPtrType(getGlobalContext()));
 }
 
+LLVMTypeRef LLVMIntPtrTypeForAS(LLVMTargetDataRef TD, unsigned AS) {
+  return wrap(unwrap(TD)->getIntPtrType(getGlobalContext(), AS));
+}
+
 unsigned long long LLVMSizeOfTypeInBits(LLVMTargetDataRef TD, LLVMTypeRef Ty) {
   return unwrap(TD)->getTypeSizeInBits(unwrap(Ty));
 }
diff --git a/lib/Target/TargetData.cpp b/lib/Target/TargetData.cpp
deleted file mode 100644
index cc6dc1e..0000000
--- a/lib/Target/TargetData.cpp
+++ /dev/null
@@ -1,663 +0,0 @@
-//===-- TargetData.cpp - Data size & alignment routines --------------------==//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines target properties related to datatype size/offset/alignment
-// information.
-//
-// This structure should be created once, filled in if the defaults are not
-// correct and then passed around by const&.  None of the members functions
-// require modification to the object.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Target/TargetData.h"
-#include "llvm/Constants.h"
-#include "llvm/DerivedTypes.h"
-#include "llvm/Module.h"
-#include "llvm/Support/GetElementPtrTypeIterator.h"
-#include "llvm/Support/MathExtras.h"
-#include "llvm/Support/ManagedStatic.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/raw_ostream.h"
-#include "llvm/Support/Mutex.h"
-#include "llvm/ADT/DenseMap.h"
-#include <algorithm>
-#include <cstdlib>
-using namespace llvm;
-
-// Handle the Pass registration stuff necessary to use TargetData's.
-
-// Register the default SparcV9 implementation...
-INITIALIZE_PASS(TargetData, "targetdata", "Target Data Layout", false, true)
-char TargetData::ID = 0;
-
-//===----------------------------------------------------------------------===//
-// Support for StructLayout
-//===----------------------------------------------------------------------===//
-
-StructLayout::StructLayout(StructType *ST, const TargetData &TD) {
-  assert(!ST->isOpaque() && "Cannot get layout of opaque structs");
-  StructAlignment = 0;
-  StructSize = 0;
-  NumElements = ST->getNumElements();
-
-  // Loop over each of the elements, placing them in memory.
-  for (unsigned i = 0, e = NumElements; i != e; ++i) {
-    Type *Ty = ST->getElementType(i);
-    unsigned TyAlign = ST->isPacked() ? 1 : TD.getABITypeAlignment(Ty);
-
-    // Add padding if necessary to align the data element properly.
-    if ((StructSize & (TyAlign-1)) != 0)
-      StructSize = TargetData::RoundUpAlignment(StructSize, TyAlign);
-
-    // Keep track of maximum alignment constraint.
-    StructAlignment = std::max(TyAlign, StructAlignment);
-
-    MemberOffsets[i] = StructSize;
-    StructSize += TD.getTypeAllocSize(Ty); // Consume space for this data item
-  }
-
-  // Empty structures have alignment of 1 byte.
-  if (StructAlignment == 0) StructAlignment = 1;
-
-  // Add padding to the end of the struct so that it could be put in an array
-  // and all array elements would be aligned correctly.
-  if ((StructSize & (StructAlignment-1)) != 0)
-    StructSize = TargetData::RoundUpAlignment(StructSize, StructAlignment);
-}
-
-
-/// getElementContainingOffset - Given a valid offset into the structure,
-/// return the structure index that contains it.
-unsigned StructLayout::getElementContainingOffset(uint64_t Offset) const {
-  const uint64_t *SI =
-    std::upper_bound(&MemberOffsets[0], &MemberOffsets[NumElements], Offset);
-  assert(SI != &MemberOffsets[0] && "Offset not in structure type!");
-  --SI;
-  assert(*SI <= Offset && "upper_bound didn't work");
-  assert((SI == &MemberOffsets[0] || *(SI-1) <= Offset) &&
-         (SI+1 == &MemberOffsets[NumElements] || *(SI+1) > Offset) &&
-         "Upper bound didn't work!");
-
-  // Multiple fields can have the same offset if any of them are zero sized.
-  // For example, in { i32, [0 x i32], i32 }, searching for offset 4 will stop
-  // at the i32 element, because it is the last element at that offset.  This is
-  // the right one to return, because anything after it will have a higher
-  // offset, implying that this element is non-empty.
-  return SI-&MemberOffsets[0];
-}
-
-//===----------------------------------------------------------------------===//
-// TargetAlignElem, TargetAlign support
-//===----------------------------------------------------------------------===//
-
-TargetAlignElem
-TargetAlignElem::get(AlignTypeEnum align_type, unsigned abi_align,
-                     unsigned pref_align, uint32_t bit_width) {
-  assert(abi_align <= pref_align && "Preferred alignment worse than ABI!");
-  TargetAlignElem retval;
-  retval.AlignType = align_type;
-  retval.ABIAlign = abi_align;
-  retval.PrefAlign = pref_align;
-  retval.TypeBitWidth = bit_width;
-  return retval;
-}
-
-bool
-TargetAlignElem::operator==(const TargetAlignElem &rhs) const {
-  return (AlignType == rhs.AlignType
-          && ABIAlign == rhs.ABIAlign
-          && PrefAlign == rhs.PrefAlign
-          && TypeBitWidth == rhs.TypeBitWidth);
-}
-
-const TargetAlignElem
-TargetData::InvalidAlignmentElem = { (AlignTypeEnum)0xFF, 0, 0, 0 };
-
-//===----------------------------------------------------------------------===//
-//                       TargetData Class Implementation
-//===----------------------------------------------------------------------===//
-
-/// getInt - Get an integer ignoring errors.
-static int getInt(StringRef R) {
-  int Result = 0;
-  R.getAsInteger(10, Result);
-  return Result;
-}
-
-void TargetData::init() {
-  initializeTargetDataPass(*PassRegistry::getPassRegistry());
-
-  LayoutMap = 0;
-  LittleEndian = false;
-  PointerMemSize = 8;
-  PointerABIAlign = 8;
-  PointerPrefAlign = PointerABIAlign;
-  StackNaturalAlign = 0;
-
-  // Default alignments
-  setAlignment(INTEGER_ALIGN,   1,  1, 1);   // i1
-  setAlignment(INTEGER_ALIGN,   1,  1, 8);   // i8
-  setAlignment(INTEGER_ALIGN,   2,  2, 16);  // i16
-  setAlignment(INTEGER_ALIGN,   4,  4, 32);  // i32
-  setAlignment(INTEGER_ALIGN,   4,  8, 64);  // i64
-  setAlignment(FLOAT_ALIGN,     2,  2, 16);  // half
-  setAlignment(FLOAT_ALIGN,     4,  4, 32);  // float
-  setAlignment(FLOAT_ALIGN,     8,  8, 64);  // double
-  setAlignment(FLOAT_ALIGN,    16, 16, 128); // ppcf128, quad, ...
-  setAlignment(VECTOR_ALIGN,    8,  8, 64);  // v2i32, v1i64, ...
-  setAlignment(VECTOR_ALIGN,   16, 16, 128); // v16i8, v8i16, v4i32, ...
-  setAlignment(AGGREGATE_ALIGN, 0,  8,  0);  // struct
-}
-
-std::string TargetData::parseSpecifier(StringRef Desc, TargetData *td) {
-
-  if (td)
-    td->init();
-
-  while (!Desc.empty()) {
-    std::pair<StringRef, StringRef> Split = Desc.split('-');
-    StringRef Token = Split.first;
-    Desc = Split.second;
-
-    if (Token.empty())
-      continue;
-
-    Split = Token.split(':');
-    StringRef Specifier = Split.first;
-    Token = Split.second;
-
-    assert(!Specifier.empty() && "Can't be empty here");
-
-    switch (Specifier[0]) {
-    case 'E':
-      if (td)
-        td->LittleEndian = false;
-      break;
-    case 'e':
-      if (td)
-        td->LittleEndian = true;
-      break;
-    case 'p': {
-      // Pointer size.
-      Split = Token.split(':');
-      int PointerMemSizeBits = getInt(Split.first);
-      if (PointerMemSizeBits < 0 || PointerMemSizeBits % 8 != 0)
-        return "invalid pointer size, must be a positive 8-bit multiple";
-      if (td)
-        td->PointerMemSize = PointerMemSizeBits / 8;
-
-      // Pointer ABI alignment.
-      Split = Split.second.split(':');
-      int PointerABIAlignBits = getInt(Split.first);
-      if (PointerABIAlignBits < 0 || PointerABIAlignBits % 8 != 0) {
-        return "invalid pointer ABI alignment, "
-               "must be a positive 8-bit multiple";
-      }
-      if (td)
-        td->PointerABIAlign = PointerABIAlignBits / 8;
-
-      // Pointer preferred alignment.
-      Split = Split.second.split(':');
-      int PointerPrefAlignBits = getInt(Split.first);
-      if (PointerPrefAlignBits < 0 || PointerPrefAlignBits % 8 != 0) {
-        return "invalid pointer preferred alignment, "
-               "must be a positive 8-bit multiple";
-      }
-      if (td) {
-        td->PointerPrefAlign = PointerPrefAlignBits / 8;
-        if (td->PointerPrefAlign == 0)
-          td->PointerPrefAlign = td->PointerABIAlign;
-      }
-      break;
-    }
-    case 'i':
-    case 'v':
-    case 'f':
-    case 'a':
-    case 's': {
-      AlignTypeEnum AlignType;
-      char field = Specifier[0];
-      switch (field) {
-      default:
-      case 'i': AlignType = INTEGER_ALIGN; break;
-      case 'v': AlignType = VECTOR_ALIGN; break;
-      case 'f': AlignType = FLOAT_ALIGN; break;
-      case 'a': AlignType = AGGREGATE_ALIGN; break;
-      case 's': AlignType = STACK_ALIGN; break;
-      }
-      int Size = getInt(Specifier.substr(1));
-      if (Size < 0) {
-        return std::string("invalid ") + field + "-size field, "
-               "must be positive";
-      }
-
-      Split = Token.split(':');
-      int ABIAlignBits = getInt(Split.first);
-      if (ABIAlignBits < 0 || ABIAlignBits % 8 != 0) {
-        return std::string("invalid ") + field +"-abi-alignment field, "
-               "must be a positive 8-bit multiple";
-      }
-      unsigned ABIAlign = ABIAlignBits / 8;
-
-      Split = Split.second.split(':');
-
-      int PrefAlignBits = getInt(Split.first);
-      if (PrefAlignBits < 0 || PrefAlignBits % 8 != 0) {
-        return std::string("invalid ") + field +"-preferred-alignment field, "
-               "must be a positive 8-bit multiple";
-      }
-      unsigned PrefAlign = PrefAlignBits / 8;
-      if (PrefAlign == 0)
-        PrefAlign = ABIAlign;
-      
-      if (td)
-        td->setAlignment(AlignType, ABIAlign, PrefAlign, Size);
-      break;
-    }
-    case 'n':  // Native integer types.
-      Specifier = Specifier.substr(1);
-      do {
-        int Width = getInt(Specifier);
-        if (Width <= 0) {
-          return std::string("invalid native integer size \'") + Specifier.str() +
-                 "\', must be a positive integer.";
-        }
-        if (td && Width != 0)
-          td->LegalIntWidths.push_back(Width);
-        Split = Token.split(':');
-        Specifier = Split.first;
-        Token = Split.second;
-      } while (!Specifier.empty() || !Token.empty());
-      break;
-    case 'S': { // Stack natural alignment.
-      int StackNaturalAlignBits = getInt(Specifier.substr(1));
-      if (StackNaturalAlignBits < 0 || StackNaturalAlignBits % 8 != 0) {
-        return "invalid natural stack alignment (S-field), "
-               "must be a positive 8-bit multiple";
-      }
-      if (td)
-        td->StackNaturalAlign = StackNaturalAlignBits / 8;
-      break;
-    }
-    default:
-      break;
-    }
-  }
-
-  return "";
-}
-
-/// Default ctor.
-///
-/// @note This has to exist, because this is a pass, but it should never be
-/// used.
-TargetData::TargetData() : ImmutablePass(ID) {
-  report_fatal_error("Bad TargetData ctor used.  "
-                    "Tool did not specify a TargetData to use?");
-}
-
-TargetData::TargetData(const Module *M)
-  : ImmutablePass(ID) {
-  std::string errMsg = parseSpecifier(M->getDataLayout(), this);
-  assert(errMsg == "" && "Module M has malformed target data layout string.");
-  (void)errMsg;
-}
-
-void
-TargetData::setAlignment(AlignTypeEnum align_type, unsigned abi_align,
-                         unsigned pref_align, uint32_t bit_width) {
-  assert(abi_align <= pref_align && "Preferred alignment worse than ABI!");
-  for (unsigned i = 0, e = Alignments.size(); i != e; ++i) {
-    if (Alignments[i].AlignType == align_type &&
-        Alignments[i].TypeBitWidth == bit_width) {
-      // Update the abi, preferred alignments.
-      Alignments[i].ABIAlign = abi_align;
-      Alignments[i].PrefAlign = pref_align;
-      return;
-    }
-  }
-
-  Alignments.push_back(TargetAlignElem::get(align_type, abi_align,
-                                            pref_align, bit_width));
-}
-
-/// getAlignmentInfo - Return the alignment (either ABI if ABIInfo = true or
-/// preferred if ABIInfo = false) the target wants for the specified datatype.
-unsigned TargetData::getAlignmentInfo(AlignTypeEnum AlignType,
-                                      uint32_t BitWidth, bool ABIInfo,
-                                      Type *Ty) const {
-  // Check to see if we have an exact match and remember the best match we see.
-  int BestMatchIdx = -1;
-  int LargestInt = -1;
-  for (unsigned i = 0, e = Alignments.size(); i != e; ++i) {
-    if (Alignments[i].AlignType == AlignType &&
-        Alignments[i].TypeBitWidth == BitWidth)
-      return ABIInfo ? Alignments[i].ABIAlign : Alignments[i].PrefAlign;
-
-    // The best match so far depends on what we're looking for.
-     if (AlignType == INTEGER_ALIGN &&
-         Alignments[i].AlignType == INTEGER_ALIGN) {
-      // The "best match" for integers is the smallest size that is larger than
-      // the BitWidth requested.
-      if (Alignments[i].TypeBitWidth > BitWidth && (BestMatchIdx == -1 ||
-           Alignments[i].TypeBitWidth < Alignments[BestMatchIdx].TypeBitWidth))
-        BestMatchIdx = i;
-      // However, if there isn't one that's larger, then we must use the
-      // largest one we have (see below)
-      if (LargestInt == -1 ||
-          Alignments[i].TypeBitWidth > Alignments[LargestInt].TypeBitWidth)
-        LargestInt = i;
-    }
-  }
-
-  // Okay, we didn't find an exact solution.  Fall back here depending on what
-  // is being looked for.
-  if (BestMatchIdx == -1) {
-    // If we didn't find an integer alignment, fall back on most conservative.
-    if (AlignType == INTEGER_ALIGN) {
-      BestMatchIdx = LargestInt;
-    } else {
-      assert(AlignType == VECTOR_ALIGN && "Unknown alignment type!");
-
-      // By default, use natural alignment for vector types. This is consistent
-      // with what clang and llvm-gcc do.
-      unsigned Align = getTypeAllocSize(cast<VectorType>(Ty)->getElementType());
-      Align *= cast<VectorType>(Ty)->getNumElements();
-      // If the alignment is not a power of 2, round up to the next power of 2.
-      // This happens for non-power-of-2 length vectors.
-      if (Align & (Align-1))
-        Align = NextPowerOf2(Align);
-      return Align;
-    }
-  }
-
-  // Since we got a "best match" index, just return it.
-  return ABIInfo ? Alignments[BestMatchIdx].ABIAlign
-                 : Alignments[BestMatchIdx].PrefAlign;
-}
-
-namespace {
-
-class StructLayoutMap {
-  typedef DenseMap<StructType*, StructLayout*> LayoutInfoTy;
-  LayoutInfoTy LayoutInfo;
-
-public:
-  virtual ~StructLayoutMap() {
-    // Remove any layouts.
-    for (LayoutInfoTy::iterator I = LayoutInfo.begin(), E = LayoutInfo.end();
-         I != E; ++I) {
-      StructLayout *Value = I->second;
-      Value->~StructLayout();
-      free(Value);
-    }
-  }
-
-  StructLayout *&operator[](StructType *STy) {
-    return LayoutInfo[STy];
-  }
-
-  // for debugging...
-  virtual void dump() const {}
-};
-
-} // end anonymous namespace
-
-TargetData::~TargetData() {
-  delete static_cast<StructLayoutMap*>(LayoutMap);
-}
-
-const StructLayout *TargetData::getStructLayout(StructType *Ty) const {
-  if (!LayoutMap)
-    LayoutMap = new StructLayoutMap();
-
-  StructLayoutMap *STM = static_cast<StructLayoutMap*>(LayoutMap);
-  StructLayout *&SL = (*STM)[Ty];
-  if (SL) return SL;
-
-  // Otherwise, create the struct layout.  Because it is variable length, we
-  // malloc it, then use placement new.
-  int NumElts = Ty->getNumElements();
-  StructLayout *L =
-    (StructLayout *)malloc(sizeof(StructLayout)+(NumElts-1) * sizeof(uint64_t));
-
-  // Set SL before calling StructLayout's ctor.  The ctor could cause other
-  // entries to be added to TheMap, invalidating our reference.
-  SL = L;
-
-  new (L) StructLayout(Ty, *this);
-
-  return L;
-}
-
-std::string TargetData::getStringRepresentation() const {
-  std::string Result;
-  raw_string_ostream OS(Result);
-
-  OS << (LittleEndian ? "e" : "E")
-     << "-p:" << PointerMemSize*8 << ':' << PointerABIAlign*8
-     << ':' << PointerPrefAlign*8
-     << "-S" << StackNaturalAlign*8;
-
-  for (unsigned i = 0, e = Alignments.size(); i != e; ++i) {
-    const TargetAlignElem &AI = Alignments[i];
-    OS << '-' << (char)AI.AlignType << AI.TypeBitWidth << ':'
-       << AI.ABIAlign*8 << ':' << AI.PrefAlign*8;
-  }
-
-  if (!LegalIntWidths.empty()) {
-    OS << "-n" << (unsigned)LegalIntWidths[0];
-
-    for (unsigned i = 1, e = LegalIntWidths.size(); i != e; ++i)
-      OS << ':' << (unsigned)LegalIntWidths[i];
-  }
-  return OS.str();
-}
-
-
-uint64_t TargetData::getTypeSizeInBits(Type *Ty) const {
-  assert(Ty->isSized() && "Cannot getTypeInfo() on a type that is unsized!");
-  switch (Ty->getTypeID()) {
-  case Type::LabelTyID:
-  case Type::PointerTyID:
-    return getPointerSizeInBits();
-  case Type::ArrayTyID: {
-    ArrayType *ATy = cast<ArrayType>(Ty);
-    return getTypeAllocSizeInBits(ATy->getElementType())*ATy->getNumElements();
-  }
-  case Type::StructTyID:
-    // Get the layout annotation... which is lazily created on demand.
-    return getStructLayout(cast<StructType>(Ty))->getSizeInBits();
-  case Type::IntegerTyID:
-    return cast<IntegerType>(Ty)->getBitWidth();
-  case Type::VoidTyID:
-    return 8;
-  case Type::HalfTyID:
-    return 16;
-  case Type::FloatTyID:
-    return 32;
-  case Type::DoubleTyID:
-  case Type::X86_MMXTyID:
-    return 64;
-  case Type::PPC_FP128TyID:
-  case Type::FP128TyID:
-    return 128;
-  // In memory objects this is always aligned to a higher boundary, but
-  // only 80 bits contain information.
-  case Type::X86_FP80TyID:
-    return 80;
-  case Type::VectorTyID:
-    return cast<VectorType>(Ty)->getBitWidth();
-  default:
-    llvm_unreachable("TargetData::getTypeSizeInBits(): Unsupported type");
-  }
-}
-
-/*!
-  \param abi_or_pref Flag that determines which alignment is returned. true
-  returns the ABI alignment, false returns the preferred alignment.
-  \param Ty The underlying type for which alignment is determined.
-
-  Get the ABI (\a abi_or_pref == true) or preferred alignment (\a abi_or_pref
-  == false) for the requested type \a Ty.
- */
-unsigned TargetData::getAlignment(Type *Ty, bool abi_or_pref) const {
-  int AlignType = -1;
-
-  assert(Ty->isSized() && "Cannot getTypeInfo() on a type that is unsized!");
-  switch (Ty->getTypeID()) {
-  // Early escape for the non-numeric types.
-  case Type::LabelTyID:
-  case Type::PointerTyID:
-    return (abi_or_pref
-            ? getPointerABIAlignment()
-            : getPointerPrefAlignment());
-  case Type::ArrayTyID:
-    return getAlignment(cast<ArrayType>(Ty)->getElementType(), abi_or_pref);
-
-  case Type::StructTyID: {
-    // Packed structure types always have an ABI alignment of one.
-    if (cast<StructType>(Ty)->isPacked() && abi_or_pref)
-      return 1;
-
-    // Get the layout annotation... which is lazily created on demand.
-    const StructLayout *Layout = getStructLayout(cast<StructType>(Ty));
-    unsigned Align = getAlignmentInfo(AGGREGATE_ALIGN, 0, abi_or_pref, Ty);
-    return std::max(Align, Layout->getAlignment());
-  }
-  case Type::IntegerTyID:
-  case Type::VoidTyID:
-    AlignType = INTEGER_ALIGN;
-    break;
-  case Type::HalfTyID:
-  case Type::FloatTyID:
-  case Type::DoubleTyID:
-  // PPC_FP128TyID and FP128TyID have different data contents, but the
-  // same size and alignment, so they look the same here.
-  case Type::PPC_FP128TyID:
-  case Type::FP128TyID:
-  case Type::X86_FP80TyID:
-    AlignType = FLOAT_ALIGN;
-    break;
-  case Type::X86_MMXTyID:
-  case Type::VectorTyID:
-    AlignType = VECTOR_ALIGN;
-    break;
-  default:
-    llvm_unreachable("Bad type for getAlignment!!!");
-  }
-
-  return getAlignmentInfo((AlignTypeEnum)AlignType, getTypeSizeInBits(Ty),
-                          abi_or_pref, Ty);
-}
-
-unsigned TargetData::getABITypeAlignment(Type *Ty) const {
-  return getAlignment(Ty, true);
-}
-
-/// getABIIntegerTypeAlignment - Return the minimum ABI-required alignment for
-/// an integer type of the specified bitwidth.
-unsigned TargetData::getABIIntegerTypeAlignment(unsigned BitWidth) const {
-  return getAlignmentInfo(INTEGER_ALIGN, BitWidth, true, 0);
-}
-
-
-unsigned TargetData::getCallFrameTypeAlignment(Type *Ty) const {
-  for (unsigned i = 0, e = Alignments.size(); i != e; ++i)
-    if (Alignments[i].AlignType == STACK_ALIGN)
-      return Alignments[i].ABIAlign;
-
-  return getABITypeAlignment(Ty);
-}
-
-unsigned TargetData::getPrefTypeAlignment(Type *Ty) const {
-  return getAlignment(Ty, false);
-}
-
-unsigned TargetData::getPreferredTypeAlignmentShift(Type *Ty) const {
-  unsigned Align = getPrefTypeAlignment(Ty);
-  assert(!(Align & (Align-1)) && "Alignment is not a power of two!");
-  return Log2_32(Align);
-}
-
-/// getIntPtrType - Return an unsigned integer type that is the same size or
-/// greater to the host pointer size.
-IntegerType *TargetData::getIntPtrType(LLVMContext &C) const {
-  return IntegerType::get(C, getPointerSizeInBits());
-}
-
-
-uint64_t TargetData::getIndexedOffset(Type *ptrTy,
-                                      ArrayRef<Value *> Indices) const {
-  Type *Ty = ptrTy;
-  assert(Ty->isPointerTy() && "Illegal argument for getIndexedOffset()");
-  uint64_t Result = 0;
-
-  generic_gep_type_iterator<Value* const*>
-    TI = gep_type_begin(ptrTy, Indices);
-  for (unsigned CurIDX = 0, EndIDX = Indices.size(); CurIDX != EndIDX;
-       ++CurIDX, ++TI) {
-    if (StructType *STy = dyn_cast<StructType>(*TI)) {
-      assert(Indices[CurIDX]->getType() ==
-             Type::getInt32Ty(ptrTy->getContext()) &&
-             "Illegal struct idx");
-      unsigned FieldNo = cast<ConstantInt>(Indices[CurIDX])->getZExtValue();
-
-      // Get structure layout information...
-      const StructLayout *Layout = getStructLayout(STy);
-
-      // Add in the offset, as calculated by the structure layout info...
-      Result += Layout->getElementOffset(FieldNo);
-
-      // Update Ty to refer to current element
-      Ty = STy->getElementType(FieldNo);
-    } else {
-      // Update Ty to refer to current element
-      Ty = cast<SequentialType>(Ty)->getElementType();
-
-      // Get the array index and the size of each array element.
-      if (int64_t arrayIdx = cast<ConstantInt>(Indices[CurIDX])->getSExtValue())
-        Result += (uint64_t)arrayIdx * getTypeAllocSize(Ty);
-    }
-  }
-
-  return Result;
-}
-
-/// getPreferredAlignment - Return the preferred alignment of the specified
-/// global.  This includes an explicitly requested alignment (if the global
-/// has one).
-unsigned TargetData::getPreferredAlignment(const GlobalVariable *GV) const {
-  Type *ElemType = GV->getType()->getElementType();
-  unsigned Alignment = getPrefTypeAlignment(ElemType);
-  unsigned GVAlignment = GV->getAlignment();
-  if (GVAlignment >= Alignment) {
-    Alignment = GVAlignment;
-  } else if (GVAlignment != 0) {
-    Alignment = std::max(GVAlignment, getABITypeAlignment(ElemType));
-  }
-
-  if (GV->hasInitializer() && GVAlignment == 0) {
-    if (Alignment < 16) {
-      // If the global is not external, see if it is large.  If so, give it a
-      // larger alignment.
-      if (getTypeSizeInBits(ElemType) > 128)
-        Alignment = 16;    // 16-byte alignment.
-    }
-  }
-  return Alignment;
-}
-
-/// getPreferredAlignmentLog - Return the preferred alignment of the
-/// specified global, returned in log form.  This includes an explicitly
-/// requested alignment (if the global has one).
-unsigned TargetData::getPreferredAlignmentLog(const GlobalVariable *GV) const {
-  return Log2_32(getPreferredAlignment(GV));
-}
diff --git a/lib/Target/TargetELFWriterInfo.cpp b/lib/Target/TargetELFWriterInfo.cpp
deleted file mode 100644
index a661ee9..0000000
--- a/lib/Target/TargetELFWriterInfo.cpp
+++ /dev/null
@@ -1,25 +0,0 @@
-//===-- lib/Target/TargetELFWriterInfo.cpp - ELF Writer Info --0-*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements the TargetELFWriterInfo class.
-//
-//===----------------------------------------------------------------------===//
-
-#include "llvm/Function.h"
-#include "llvm/Target/TargetELFWriterInfo.h"
-#include "llvm/Target/TargetData.h"
-#include "llvm/Target/TargetMachine.h"
-using namespace llvm;
-
-TargetELFWriterInfo::TargetELFWriterInfo(bool is64Bit_, bool isLittleEndian_) :
-  is64Bit(is64Bit_), isLittleEndian(isLittleEndian_) {
-}
-
-TargetELFWriterInfo::~TargetELFWriterInfo() {}
-
diff --git a/lib/Target/TargetLibraryInfo.cpp b/lib/Target/TargetLibraryInfo.cpp
index 8e215a7..6d4eab1 100644
--- a/lib/Target/TargetLibraryInfo.cpp
+++ b/lib/Target/TargetLibraryInfo.cpp
@@ -24,6 +24,16 @@ void TargetLibraryInfo::anchor() { }
 
 const char* TargetLibraryInfo::StandardNames[LibFunc::NumLibFuncs] =
   {
+    "_ZdaPv",
+    "_ZdlPv",
+    "_Znaj",
+    "_ZnajRKSt9nothrow_t",
+    "_Znam",
+    "_ZnamRKSt9nothrow_t",
+    "_Znwj",
+    "_ZnwjRKSt9nothrow_t",
+    "_Znwm",
+    "_ZnwmRKSt9nothrow_t",
     "__cxa_atexit",
     "__cxa_guard_abort",
     "__cxa_guard_acquire",
@@ -31,16 +41,29 @@ const char* TargetLibraryInfo::StandardNames[LibFunc::NumLibFuncs] =
     "__memcpy_chk",
     "acos",
     "acosf",
+    "acosh",
+    "acoshf",
+    "acoshl",
     "acosl",
     "asin",
     "asinf",
+    "asinh",
+    "asinhf",
+    "asinhl",
     "asinl",
     "atan",
     "atan2",
     "atan2f",
     "atan2l",
     "atanf",
+    "atanh",
+    "atanhf",
+    "atanhl",
     "atanl",
+    "calloc",
+    "cbrt",
+    "cbrtf",
+    "cbrtl",
     "ceil",
     "ceilf",
     "ceill",
@@ -54,6 +77,9 @@ const char* TargetLibraryInfo::StandardNames[LibFunc::NumLibFuncs] =
     "coshl",
     "cosl",
     "exp",
+    "exp10",
+    "exp10f",
+    "exp10l",
     "exp2",
     "exp2f",
     "exp2l",
@@ -74,6 +100,7 @@ const char* TargetLibraryInfo::StandardNames[LibFunc::NumLibFuncs] =
     "fmodl",
     "fputc",
     "fputs",
+    "free",
     "fwrite",
     "iprintf",
     "log",
@@ -86,8 +113,12 @@ const char* TargetLibraryInfo::StandardNames[LibFunc::NumLibFuncs] =
     "log2",
     "log2f",
     "log2l",
+    "logb",
+    "logbf",
+    "logbl",
     "logf",
     "logl",
+    "malloc",
     "memchr",
     "memcmp",
     "memcpy",
@@ -97,11 +128,14 @@ const char* TargetLibraryInfo::StandardNames[LibFunc::NumLibFuncs] =
     "nearbyint",
     "nearbyintf",
     "nearbyintl",
+    "posix_memalign",
     "pow",
     "powf",
     "powl",
     "putchar",
     "puts",
+    "realloc",
+    "reallocf",
     "rint",
     "rintf",
     "rintl",
@@ -118,14 +152,30 @@ const char* TargetLibraryInfo::StandardNames[LibFunc::NumLibFuncs] =
     "sqrt",
     "sqrtf",
     "sqrtl",
+    "stpcpy",
     "strcat",
     "strchr",
+    "strcmp",
     "strcpy",
+    "strcspn",
+    "strdup",
     "strlen",
     "strncat",
     "strncmp",
     "strncpy",
+    "strndup",
     "strnlen",
+    "strpbrk",
+    "strrchr",
+    "strspn",
+    "strstr",
+    "strtod",
+    "strtof",
+    "strtol",
+    "strtold",
+    "strtoll",
+    "strtoul",
+    "strtoull",
     "tan",
     "tanf",
     "tanh",
@@ -134,7 +184,8 @@ const char* TargetLibraryInfo::StandardNames[LibFunc::NumLibFuncs] =
     "tanl",
     "trunc",
     "truncf",
-    "truncl"
+    "truncl",
+    "valloc"
   };
 
 /// initialize - Initialize the set of available library functions based on the
@@ -205,6 +256,21 @@ static void initialize(TargetLibraryInfo &TLI, const Triple &T,
     TLI.setUnavailable(LibFunc::tanhl);
 
     // Win32 only has C89 math
+    TLI.setUnavailable(LibFunc::acosh);
+    TLI.setUnavailable(LibFunc::acoshf);
+    TLI.setUnavailable(LibFunc::acoshl);
+    TLI.setUnavailable(LibFunc::asinh);
+    TLI.setUnavailable(LibFunc::asinhf);
+    TLI.setUnavailable(LibFunc::asinhl);
+    TLI.setUnavailable(LibFunc::atanh);
+    TLI.setUnavailable(LibFunc::atanhf);
+    TLI.setUnavailable(LibFunc::atanhl);
+    TLI.setUnavailable(LibFunc::cbrt);
+    TLI.setUnavailable(LibFunc::cbrtf);
+    TLI.setUnavailable(LibFunc::cbrtl);
+    TLI.setUnavailable(LibFunc::exp10);
+    TLI.setUnavailable(LibFunc::exp10f);
+    TLI.setUnavailable(LibFunc::exp10l);
     TLI.setUnavailable(LibFunc::exp2);
     TLI.setUnavailable(LibFunc::exp2f);
     TLI.setUnavailable(LibFunc::exp2l);
@@ -217,6 +283,9 @@ static void initialize(TargetLibraryInfo &TLI, const Triple &T,
     TLI.setUnavailable(LibFunc::log1p);
     TLI.setUnavailable(LibFunc::log1pf);
     TLI.setUnavailable(LibFunc::log1pl);
+    TLI.setUnavailable(LibFunc::logb);
+    TLI.setUnavailable(LibFunc::logbf);
+    TLI.setUnavailable(LibFunc::logbl);
     TLI.setUnavailable(LibFunc::nearbyint);
     TLI.setUnavailable(LibFunc::nearbyintf);
     TLI.setUnavailable(LibFunc::nearbyintl);
@@ -254,6 +323,10 @@ static void initialize(TargetLibraryInfo &TLI, const Triple &T,
       TLI.setUnavailable(LibFunc::tanf);
       TLI.setUnavailable(LibFunc::tanhf);
     }
+
+    // Win32 does *not* provide stpcpy.  It is provided on POSIX systems:
+    // http://pubs.opengroup.org/onlinepubs/9699919799/functions/stpcpy.html
+    TLI.setUnavailable(LibFunc::stpcpy);
   }
 }
 
diff --git a/lib/Target/TargetLoweringObjectFile.cpp b/lib/Target/TargetLoweringObjectFile.cpp
index b74a0bd..9d7e2b8 100644
--- a/lib/Target/TargetLoweringObjectFile.cpp
+++ b/lib/Target/TargetLoweringObjectFile.cpp
@@ -22,7 +22,7 @@
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/Target/Mangler.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
 #include "llvm/Support/Dwarf.h"
@@ -184,7 +184,7 @@ SectionKind TargetLoweringObjectFile::getKindForGlobal(const GlobalValue *GV,
       // Otherwise, just drop it into a mergable constant section.  If we have
       // a section for this size, use it, otherwise use the arbitrary sized
       // mergable section.
-      switch (TM.getTargetData()->getTypeAllocSize(C->getType())) {
+      switch (TM.getDataLayout()->getTypeAllocSize(C->getType())) {
       case 4:  return SectionKind::getMergeableConst4();
       case 8:  return SectionKind::getMergeableConst8();
       case 16: return SectionKind::getMergeableConst16();
diff --git a/lib/Target/TargetMachineC.cpp b/lib/Target/TargetMachineC.cpp
index d6bba8b..f69c2ab 100644
--- a/lib/Target/TargetMachineC.cpp
+++ b/lib/Target/TargetMachineC.cpp
@@ -14,7 +14,7 @@
 #include "llvm-c/Core.h"
 #include "llvm-c/Target.h"
 #include "llvm-c/TargetMachine.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/raw_ostream.h"
@@ -146,7 +146,7 @@ char* LLVMGetTargetMachineFeatureString(LLVMTargetMachineRef T) {
 }
 
 LLVMTargetDataRef LLVMGetTargetMachineData(LLVMTargetMachineRef T) {
-  return wrap(unwrap(T)->getTargetData());
+  return wrap(unwrap(T)->getDataLayout());
 }
 
 LLVMBool LLVMTargetMachineEmitToFile(LLVMTargetMachineRef T, LLVMModuleRef M,
@@ -158,14 +158,14 @@ LLVMBool LLVMTargetMachineEmitToFile(LLVMTargetMachineRef T, LLVMModuleRef M,
 
   std::string error;
 
-  const TargetData* td = TM->getTargetData();
+  const DataLayout* td = TM->getDataLayout();
 
   if (!td) {
-    error = "No TargetData in TargetMachine";
+    error = "No DataLayout in TargetMachine";
     *ErrorMessage = strdup(error.c_str());
     return true;
   }
-  pass.add(new TargetData(*td));
+  pass.add(new DataLayout(*td));
 
   TargetMachine::CodeGenFileType ft;
   switch (codegen) {
@@ -184,7 +184,7 @@ LLVMBool LLVMTargetMachineEmitToFile(LLVMTargetMachineRef T, LLVMModuleRef M,
   }
 
   if (TM->addPassesToEmitFile(pass, destf, ft)) {
-    error = "No TargetData in TargetMachine";
+    error = "No DataLayout in TargetMachine";
     *ErrorMessage = strdup(error.c_str());
     return true;
   }
diff --git a/lib/Target/TargetRegisterInfo.cpp b/lib/Target/TargetRegisterInfo.cpp
index 2395f2b..be8b582 100644
--- a/lib/Target/TargetRegisterInfo.cpp
+++ b/lib/Target/TargetRegisterInfo.cpp
@@ -20,8 +20,10 @@ using namespace llvm;
 
 TargetRegisterInfo::TargetRegisterInfo(const TargetRegisterInfoDesc *ID,
                              regclass_iterator RCB, regclass_iterator RCE,
-                             const char *const *subregindexnames)
-  : InfoDesc(ID), SubRegIndexNames(subregindexnames),
+                             const char *const *SRINames,
+                             const unsigned *SRILaneMasks)
+  : InfoDesc(ID), SubRegIndexNames(SRINames),
+    SubRegIndexLaneMasks(SRILaneMasks),
     RegClassBegin(RCB), RegClassEnd(RCE) {
 }
 
diff --git a/lib/Target/TargetTransformImpl.cpp b/lib/Target/TargetTransformImpl.cpp
new file mode 100644
index 0000000..b36e6f8
--- /dev/null
+++ b/lib/Target/TargetTransformImpl.cpp
@@ -0,0 +1,353 @@
+// llvm/Target/TargetTransformImpl.cpp - Target Loop Trans Info ---*- C++ -*-=//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Target/TargetTransformImpl.h"
+#include "llvm/Target/TargetLowering.h"
+#include <utility>
+
+using namespace llvm;
+
+//===----------------------------------------------------------------------===//
+//
+// Calls used by scalar transformations.
+//
+//===----------------------------------------------------------------------===//
+
+bool ScalarTargetTransformImpl::isLegalAddImmediate(int64_t imm) const {
+  return TLI->isLegalAddImmediate(imm);
+}
+
+bool ScalarTargetTransformImpl::isLegalICmpImmediate(int64_t imm) const {
+  return TLI->isLegalICmpImmediate(imm);
+}
+
+bool ScalarTargetTransformImpl::isLegalAddressingMode(const AddrMode &AM,
+                                                      Type *Ty) const {
+  return TLI->isLegalAddressingMode(AM, Ty);
+}
+
+bool ScalarTargetTransformImpl::isTruncateFree(Type *Ty1, Type *Ty2) const {
+  return TLI->isTruncateFree(Ty1, Ty2);
+}
+
+bool ScalarTargetTransformImpl::isTypeLegal(Type *Ty) const {
+  EVT T = TLI->getValueType(Ty);
+  return TLI->isTypeLegal(T);
+}
+
+unsigned ScalarTargetTransformImpl::getJumpBufAlignment() const {
+  return TLI->getJumpBufAlignment();
+}
+
+unsigned ScalarTargetTransformImpl::getJumpBufSize() const {
+  return TLI->getJumpBufSize();
+}
+
+bool ScalarTargetTransformImpl::shouldBuildLookupTables() const {
+  return TLI->supportJumpTables() &&
+      (TLI->isOperationLegalOrCustom(ISD::BR_JT, MVT::Other) ||
+       TLI->isOperationLegalOrCustom(ISD::BRIND, MVT::Other));
+}
+
+//===----------------------------------------------------------------------===//
+//
+// Calls used by the vectorizers.
+//
+//===----------------------------------------------------------------------===//
+int VectorTargetTransformImpl::InstructionOpcodeToISD(unsigned Opcode) const {
+  enum InstructionOpcodes {
+#define HANDLE_INST(NUM, OPCODE, CLASS) OPCODE = NUM,
+#define LAST_OTHER_INST(NUM) InstructionOpcodesCount = NUM
+#include "llvm/Instruction.def"
+  };
+  switch (static_cast<InstructionOpcodes>(Opcode)) {
+  case Ret:            return 0;
+  case Br:             return 0;
+  case Switch:         return 0;
+  case IndirectBr:     return 0;
+  case Invoke:         return 0;
+  case Resume:         return 0;
+  case Unreachable:    return 0;
+  case Add:            return ISD::ADD;
+  case FAdd:           return ISD::FADD;
+  case Sub:            return ISD::SUB;
+  case FSub:           return ISD::FSUB;
+  case Mul:            return ISD::MUL;
+  case FMul:           return ISD::FMUL;
+  case UDiv:           return ISD::UDIV;
+  case SDiv:           return ISD::UDIV;
+  case FDiv:           return ISD::FDIV;
+  case URem:           return ISD::UREM;
+  case SRem:           return ISD::SREM;
+  case FRem:           return ISD::FREM;
+  case Shl:            return ISD::SHL;
+  case LShr:           return ISD::SRL;
+  case AShr:           return ISD::SRA;
+  case And:            return ISD::AND;
+  case Or:             return ISD::OR;
+  case Xor:            return ISD::XOR;
+  case Alloca:         return 0;
+  case Load:           return ISD::LOAD;
+  case Store:          return ISD::STORE;
+  case GetElementPtr:  return 0;
+  case Fence:          return 0;
+  case AtomicCmpXchg:  return 0;
+  case AtomicRMW:      return 0;
+  case Trunc:          return ISD::TRUNCATE;
+  case ZExt:           return ISD::ZERO_EXTEND;
+  case SExt:           return ISD::SIGN_EXTEND;
+  case FPToUI:         return ISD::FP_TO_UINT;
+  case FPToSI:         return ISD::FP_TO_SINT;
+  case UIToFP:         return ISD::UINT_TO_FP;
+  case SIToFP:         return ISD::SINT_TO_FP;
+  case FPTrunc:        return ISD::FP_ROUND;
+  case FPExt:          return ISD::FP_EXTEND;
+  case PtrToInt:       return ISD::BITCAST;
+  case IntToPtr:       return ISD::BITCAST;
+  case BitCast:        return ISD::BITCAST;
+  case ICmp:           return ISD::SETCC;
+  case FCmp:           return ISD::SETCC;
+  case PHI:            return 0;
+  case Call:           return 0;
+  case Select:         return ISD::SELECT;
+  case UserOp1:        return 0;
+  case UserOp2:        return 0;
+  case VAArg:          return 0;
+  case ExtractElement: return ISD::EXTRACT_VECTOR_ELT;
+  case InsertElement:  return ISD::INSERT_VECTOR_ELT;
+  case ShuffleVector:  return ISD::VECTOR_SHUFFLE;
+  case ExtractValue:   return ISD::MERGE_VALUES;
+  case InsertValue:    return ISD::MERGE_VALUES;
+  case LandingPad:     return 0;
+  }
+
+  llvm_unreachable("Unknown instruction type encountered!");
+}
+
+std::pair<unsigned, MVT>
+VectorTargetTransformImpl::getTypeLegalizationCost(Type *Ty) const {
+
+  LLVMContext &C = Ty->getContext();
+  EVT MTy = TLI->getValueType(Ty);
+
+  unsigned Cost = 1;
+  // We keep legalizing the type until we find a legal kind. We assume that
+  // the only operation that costs anything is the split. After splitting
+  // we need to handle two types.
+  while (true) {
+    TargetLowering::LegalizeKind LK = TLI->getTypeConversion(C, MTy);
+
+    if (LK.first == TargetLowering::TypeLegal)
+      return std::make_pair(Cost, MTy.getSimpleVT());
+
+    if (LK.first == TargetLowering::TypeSplitVector ||
+        LK.first == TargetLowering::TypeExpandInteger)
+      Cost *= 2;
+
+    // Keep legalizing the type.
+    MTy = LK.second;
+  }
+}
+
+unsigned
+VectorTargetTransformImpl::getScalarizationOverhead(Type *Ty,
+                                                    bool Insert,
+                                                    bool Extract) const {
+  assert (Ty->isVectorTy() && "Can only scalarize vectors");
+  unsigned Cost = 0;
+
+  for (int i = 0, e = Ty->getVectorNumElements(); i < e; ++i) {
+    if (Insert)
+      Cost += getVectorInstrCost(Instruction::InsertElement, Ty, i);
+    if (Extract)
+      Cost += getVectorInstrCost(Instruction::ExtractElement, Ty, i);
+  }
+
+  return Cost;
+}
+
+unsigned VectorTargetTransformImpl::getArithmeticInstrCost(unsigned Opcode,
+                                                           Type *Ty) const {
+  // Check if any of the operands are vector operands.
+  int ISD = InstructionOpcodeToISD(Opcode);
+  assert(ISD && "Invalid opcode");
+
+  std::pair<unsigned, MVT> LT = getTypeLegalizationCost(Ty);
+
+  if (!TLI->isOperationExpand(ISD, LT.second)) {
+    // The operation is legal. Assume it costs 1. Multiply
+    // by the type-legalization overhead.
+    return LT.first * 1;
+  }
+
+  // Else, assume that we need to scalarize this op.
+  if (Ty->isVectorTy()) {
+    unsigned Num = Ty->getVectorNumElements();
+    unsigned Cost = getArithmeticInstrCost(Opcode, Ty->getScalarType());
+    // return the cost of multiple scalar invocation plus the cost of inserting
+    // and extracting the values.
+    return getScalarizationOverhead(Ty, true, true) + Num * Cost;
+  }
+
+  // We don't know anything about this scalar instruction.
+  return 1;
+}
+
+unsigned VectorTargetTransformImpl::getBroadcastCost(Type *Tp) const {
+  return 1;
+}
+
+unsigned VectorTargetTransformImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
+                                  Type *Src) const {
+  int ISD = InstructionOpcodeToISD(Opcode);
+  assert(ISD && "Invalid opcode");
+
+  std::pair<unsigned, MVT> SrcLT = getTypeLegalizationCost(Src);
+  std::pair<unsigned, MVT> DstLT = getTypeLegalizationCost(Dst);
+
+  // Handle scalar conversions.
+  if (!Src->isVectorTy() && !Dst->isVectorTy()) {
+
+    // Scalar bitcasts are usually free.
+    if (Opcode == Instruction::BitCast)
+      return 0;
+
+    if (Opcode == Instruction::Trunc &&
+        TLI->isTruncateFree(SrcLT.second, DstLT.second))
+      return 0;
+
+    if (Opcode == Instruction::ZExt &&
+        TLI->isZExtFree(SrcLT.second, DstLT.second))
+      return 0;
+
+    // Just check the op cost. If the operation is legal then assume it costs 1.
+    if (!TLI->isOperationExpand(ISD, DstLT.second))
+      return  1;
+
+    // Assume that illegal scalar instruction are expensive.
+    return 4;
+  }
+
+  // Check vector-to-vector casts.
+  if (Dst->isVectorTy() && Src->isVectorTy()) {
+
+    // If the cast is between same-sized registers, then the check is simple.
+    if (SrcLT.first == DstLT.first &&
+        SrcLT.second.getSizeInBits() == DstLT.second.getSizeInBits()) {
+
+      // Bitcast between types that are legalized to the same type are free.
+      if (Opcode == Instruction::BitCast || Opcode == Instruction::Trunc)
+        return 0;
+
+      // Assume that Zext is done using AND.
+      if (Opcode == Instruction::ZExt)
+        return 1;
+
+      // Assume that sext is done using SHL and SRA.
+      if (Opcode == Instruction::SExt)
+        return 2;
+
+      // Just check the op cost. If the operation is legal then assume it costs
+      // 1 and multiply by the type-legalization overhead.
+      if (!TLI->isOperationExpand(ISD, DstLT.second))
+        return SrcLT.first * 1;
+    }
+
+    // If we are converting vectors and the operation is illegal, or
+    // if the vectors are legalized to different types, estimate the
+    // scalarization costs.
+    unsigned Num = Dst->getVectorNumElements();
+    unsigned Cost = getCastInstrCost(Opcode, Dst->getScalarType(),
+                                     Src->getScalarType());
+
+    // Return the cost of multiple scalar invocation plus the cost of
+    // inserting and extracting the values.
+    return getScalarizationOverhead(Dst, true, true) + Num * Cost;
+  }
+
+  // We already handled vector-to-vector and scalar-to-scalar conversions. This 
+  // is where we handle bitcast between vectors and scalars. We need to assume
+  //  that the conversion is scalarized in one way or another.
+  if (Opcode == Instruction::BitCast)
+    // Illegal bitcasts are done by storing and loading from a stack slot.
+    return (Src->isVectorTy()? getScalarizationOverhead(Src, false, true):0) +
+           (Dst->isVectorTy()? getScalarizationOverhead(Dst, true, false):0);
+
+  llvm_unreachable("Unhandled cast");
+ }
+
+unsigned VectorTargetTransformImpl::getCFInstrCost(unsigned Opcode) const {
+  return 1;
+}
+
+unsigned VectorTargetTransformImpl::getCmpSelInstrCost(unsigned Opcode,
+                                                       Type *ValTy,
+                                                       Type *CondTy) const {
+  int ISD = InstructionOpcodeToISD(Opcode);
+  assert(ISD && "Invalid opcode");
+
+  // Selects on vectors are actually vector selects.
+  if (ISD == ISD::SELECT) {
+    assert(CondTy && "CondTy must exist");
+    if (CondTy->isVectorTy())
+      ISD = ISD::VSELECT;
+  }
+
+  std::pair<unsigned, MVT> LT = getTypeLegalizationCost(ValTy);
+
+  if (!TLI->isOperationExpand(ISD, LT.second)) {
+    // The operation is legal. Assume it costs 1. Multiply
+    // by the type-legalization overhead.
+    return LT.first * 1;
+  }
+
+  // Otherwise, assume that the cast is scalarized.
+  if (ValTy->isVectorTy()) {
+    unsigned Num = ValTy->getVectorNumElements();
+    if (CondTy)
+      CondTy = CondTy->getScalarType();
+    unsigned Cost = getCmpSelInstrCost(Opcode, ValTy->getScalarType(),
+                                       CondTy);
+
+    // Return the cost of multiple scalar invocation plus the cost of inserting
+    // and extracting the values.
+    return getScalarizationOverhead(ValTy, true, false) + Num * Cost;
+  }
+
+  // Unknown scalar opcode.
+  return 1;
+}
+
+unsigned VectorTargetTransformImpl::getVectorInstrCost(unsigned Opcode,
+                                                       Type *Val,
+                                                       unsigned Index) const {
+  return 1;
+}
+
+unsigned
+VectorTargetTransformImpl::getInstrCost(unsigned Opcode, Type *Ty1,
+                                        Type *Ty2) const {
+  return 1;
+}
+
+unsigned
+VectorTargetTransformImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
+                                           unsigned Alignment,
+                                           unsigned AddressSpace) const {
+  std::pair<unsigned, MVT> LT = getTypeLegalizationCost(Src);
+
+  // Assume that all loads of legal types cost 1.
+  return LT.first;
+}
+
+unsigned
+VectorTargetTransformImpl::getNumberOfParts(Type *Tp) const {
+  std::pair<unsigned, MVT> LT = getTypeLegalizationCost(Tp);
+  return LT.first;
+}
diff --git a/lib/Target/X86/AsmParser/X86AsmLexer.cpp b/lib/Target/X86/AsmParser/X86AsmLexer.cpp
index 2794e60..66ad353 100644
--- a/lib/Target/X86/AsmParser/X86AsmLexer.cpp
+++ b/lib/Target/X86/AsmParser/X86AsmLexer.cpp
@@ -18,19 +18,19 @@
 using namespace llvm;
 
 namespace {
-  
+
 class X86AsmLexer : public MCTargetAsmLexer {
   const MCAsmInfo &AsmInfo;
-  
+
   bool tentativeIsValid;
   AsmToken tentativeToken;
-  
+
   const AsmToken &lexTentative() {
     tentativeToken = getLexer()->Lex();
     tentativeIsValid = true;
     return tentativeToken;
   }
-  
+
   const AsmToken &lexDefinite() {
     if (tentativeIsValid) {
       tentativeIsValid = false;
@@ -38,7 +38,7 @@ class X86AsmLexer : public MCTargetAsmLexer {
     }
     return getLexer()->Lex();
   }
-  
+
   AsmToken LexTokenATT();
   AsmToken LexTokenIntel();
 protected:
@@ -47,7 +47,7 @@ protected:
       SetError(SMLoc(), "No MCAsmLexer installed");
       return AsmToken(AsmToken::Error, "", 0);
     }
-    
+
     switch (AsmInfo.getAssemblerDialect()) {
     default:
       SetError(SMLoc(), "Unhandled dialect");
@@ -71,33 +71,32 @@ public:
 
 AsmToken X86AsmLexer::LexTokenATT() {
   AsmToken lexedToken = lexDefinite();
-  
+
   switch (lexedToken.getKind()) {
   default:
     return lexedToken;
   case AsmToken::Error:
     SetError(Lexer->getErrLoc(), Lexer->getErr());
     return lexedToken;
-      
+
   case AsmToken::Percent: {
     const AsmToken &nextToken = lexTentative();
     if (nextToken.getKind() != AsmToken::Identifier)
       return lexedToken;
 
-      
     if (unsigned regID = MatchRegisterName(nextToken.getString())) {
       lexDefinite();
-        
+
       // FIXME: This is completely wrong when there is a space or other
       // punctuation between the % and the register name.
       StringRef regStr(lexedToken.getString().data(),
-                       lexedToken.getString().size() + 
+                       lexedToken.getString().size() +
                        nextToken.getString().size());
-      
-      return AsmToken(AsmToken::Register, regStr, 
+
+      return AsmToken(AsmToken::Register, regStr,
                       static_cast<int64_t>(regID));
     }
-    
+
     // Match register name failed.  If this is "db[0-7]", match it as an alias
     // for dr[0-7].
     if (nextToken.getString().size() == 3 &&
@@ -113,29 +112,29 @@ AsmToken X86AsmLexer::LexTokenATT() {
       case '6': RegNo = X86::DR6; break;
       case '7': RegNo = X86::DR7; break;
       }
-      
+
       if (RegNo != -1) {
         lexDefinite();
 
         // FIXME: This is completely wrong when there is a space or other
         // punctuation between the % and the register name.
         StringRef regStr(lexedToken.getString().data(),
-                         lexedToken.getString().size() + 
+                         lexedToken.getString().size() +
                          nextToken.getString().size());
-        return AsmToken(AsmToken::Register, regStr, 
+        return AsmToken(AsmToken::Register, regStr,
                         static_cast<int64_t>(RegNo));
       }
     }
-      
-   
+
+
     return lexedToken;
-  }    
+  }
   }
 }
 
 AsmToken X86AsmLexer::LexTokenIntel() {
   const AsmToken &lexedToken = lexDefinite();
-  
+
   switch(lexedToken.getKind()) {
   default:
     return lexedToken;
@@ -144,7 +143,7 @@ AsmToken X86AsmLexer::LexTokenIntel() {
     return lexedToken;
   case AsmToken::Identifier: {
     unsigned regID = MatchRegisterName(lexedToken.getString().lower());
-    
+
     if (regID)
       return AsmToken(AsmToken::Register,
                       lexedToken.getString(),
diff --git a/lib/Target/X86/AsmParser/X86AsmParser.cpp b/lib/Target/X86/AsmParser/X86AsmParser.cpp
index fbbaa9500..ce446e7 100644
--- a/lib/Target/X86/AsmParser/X86AsmParser.cpp
+++ b/lib/Target/X86/AsmParser/X86AsmParser.cpp
@@ -11,12 +11,14 @@
 #include "llvm/MC/MCTargetAsmParser.h"
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCExpr.h"
+#include "llvm/MC/MCSymbol.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCRegisterInfo.h"
 #include "llvm/MC/MCSubtargetInfo.h"
 #include "llvm/MC/MCParser/MCAsmLexer.h"
 #include "llvm/MC/MCParser/MCAsmParser.h"
 #include "llvm/MC/MCParser/MCParsedAsmOperand.h"
+#include "llvm/ADT/APFloat.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringSwitch.h"
@@ -33,13 +35,16 @@ struct X86Operand;
 class X86AsmParser : public MCTargetAsmParser {
   MCSubtargetInfo &STI;
   MCAsmParser &Parser;
+  ParseInstructionInfo *InstInfo;
 private:
   MCAsmParser &getParser() const { return Parser; }
 
   MCAsmLexer &getLexer() const { return Parser.getLexer(); }
 
   bool Error(SMLoc L, const Twine &Msg,
-             ArrayRef<SMRange> Ranges = ArrayRef<SMRange>()) {
+             ArrayRef<SMRange> Ranges = ArrayRef<SMRange>(),
+             bool MatchingInlineAsm = false) {
+    if (MatchingInlineAsm) return true;
     return Parser.Error(L, Msg, Ranges);
   }
 
@@ -51,23 +56,25 @@ private:
   X86Operand *ParseOperand();
   X86Operand *ParseATTOperand();
   X86Operand *ParseIntelOperand();
-  X86Operand *ParseIntelMemOperand();
+  X86Operand *ParseIntelOffsetOfOperator(SMLoc StartLoc);
+  X86Operand *ParseIntelTypeOperator(SMLoc StartLoc);
+  X86Operand *ParseIntelMemOperand(unsigned SegReg, SMLoc StartLoc);
   X86Operand *ParseIntelBracExpression(unsigned SegReg, unsigned Size);
   X86Operand *ParseMemOperand(unsigned SegReg, SMLoc StartLoc);
 
+  bool ParseIntelDotOperator(const MCExpr *Disp, const MCExpr **NewDisp,
+                             SmallString<64> &Err);
+
   bool ParseDirectiveWord(unsigned Size, SMLoc L);
   bool ParseDirectiveCode(StringRef IDVal, SMLoc L);
 
   bool processInstruction(MCInst &Inst,
                           const SmallVectorImpl<MCParsedAsmOperand*> &Ops);
 
-  bool MatchAndEmitInstruction(SMLoc IDLoc,
+  bool MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
                                SmallVectorImpl<MCParsedAsmOperand*> &Operands,
-                               MCStreamer &Out);
-
-  bool MatchInstruction(SMLoc IDLoc,
-                        SmallVectorImpl<MCParsedAsmOperand*> &Operands,
-                        SmallVectorImpl<MCInst> &MCInsts);
+                               MCStreamer &Out, unsigned &ErrorInfo,
+                               bool MatchingInlineAsm);
 
   /// isSrcOp - Returns true if operand is either (%rsi) or %ds:%(rsi)
   /// in 64bit mode or (%esi) or %es:(%esi) in 32bit mode.
@@ -96,14 +103,15 @@ private:
 
 public:
   X86AsmParser(MCSubtargetInfo &sti, MCAsmParser &parser)
-    : MCTargetAsmParser(), STI(sti), Parser(parser) {
+    : MCTargetAsmParser(), STI(sti), Parser(parser), InstInfo(0) {
 
     // Initialize the set of available features.
     setAvailableFeatures(ComputeAvailableFeatures(STI.getFeatureBits()));
   }
   virtual bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc);
 
-  virtual bool ParseInstruction(StringRef Name, SMLoc NameLoc,
+  virtual bool ParseInstruction(ParseInstructionInfo &Info, StringRef Name,
+                                SMLoc NameLoc,
                                 SmallVectorImpl<MCParsedAsmOperand*> &Operands);
 
   virtual bool ParseDirective(AsmToken DirectiveID);
@@ -159,6 +167,7 @@ struct X86Operand : public MCParsedAsmOperand {
   } Kind;
 
   SMLoc StartLoc, EndLoc;
+  SMLoc OffsetOfLoc;
 
   union {
     struct {
@@ -172,6 +181,7 @@ struct X86Operand : public MCParsedAsmOperand {
 
     struct {
       const MCExpr *Val;
+      bool NeedAsmRewrite;
     } Imm;
 
     struct {
@@ -181,6 +191,7 @@ struct X86Operand : public MCParsedAsmOperand {
       unsigned IndexReg;
       unsigned Scale;
       unsigned Size;
+      bool NeedSizeDir;
     } Mem;
   };
 
@@ -191,8 +202,11 @@ struct X86Operand : public MCParsedAsmOperand {
   SMLoc getStartLoc() const { return StartLoc; }
   /// getEndLoc - Get the location of the last token of this operand.
   SMLoc getEndLoc() const { return EndLoc; }
-
+  /// getLocRange - Get the range between the first and last token of this
+  /// operand.
   SMRange getLocRange() const { return SMRange(StartLoc, EndLoc); }
+  /// getOffsetOfLoc - Get the location of the offset operator.
+  SMLoc getOffsetOfLoc() const { return OffsetOfLoc; }
 
   virtual void print(raw_ostream &OS) const {}
 
@@ -216,6 +230,11 @@ struct X86Operand : public MCParsedAsmOperand {
     return Imm.Val;
   }
 
+  bool needAsmRewrite() const {
+    assert(Kind == Immediate && "Invalid access!");
+    return Imm.NeedAsmRewrite;
+  }
+
   const MCExpr *getMemDisp() const {
     assert(Kind == Memory && "Invalid access!");
     return Mem.Disp;
@@ -312,6 +331,20 @@ struct X86Operand : public MCParsedAsmOperand {
     return isImmSExti64i32Value(CE->getValue());
   }
 
+  unsigned getMemSize() const {
+    assert(Kind == Memory && "Invalid access!");
+    return Mem.Size;
+  }
+
+  bool isOffsetOf() const {
+    return OffsetOfLoc.getPointer();
+  }
+
+  bool needSizeDirective() const {
+    assert(Kind == Memory && "Invalid access!");
+    return Mem.NeedSizeDir;
+  }
+
   bool isMem() const { return Kind == Memory; }
   bool isMem8() const {
     return Kind == Memory && (!Mem.Size || Mem.Size == 8);
@@ -437,21 +470,25 @@ struct X86Operand : public MCParsedAsmOperand {
     return Res;
   }
 
-  static X86Operand *CreateReg(unsigned RegNo, SMLoc StartLoc, SMLoc EndLoc) {
+  static X86Operand *CreateReg(unsigned RegNo, SMLoc StartLoc, SMLoc EndLoc,
+                               SMLoc OffsetOfLoc = SMLoc()) {
     X86Operand *Res = new X86Operand(Register, StartLoc, EndLoc);
     Res->Reg.RegNo = RegNo;
+    Res->OffsetOfLoc = OffsetOfLoc;
     return Res;
   }
 
-  static X86Operand *CreateImm(const MCExpr *Val, SMLoc StartLoc, SMLoc EndLoc){
+  static X86Operand *CreateImm(const MCExpr *Val, SMLoc StartLoc, SMLoc EndLoc,
+                               bool NeedRewrite = true){
     X86Operand *Res = new X86Operand(Immediate, StartLoc, EndLoc);
     Res->Imm.Val = Val;
+    Res->Imm.NeedAsmRewrite = NeedRewrite;
     return Res;
   }
 
   /// Create an absolute memory operand.
-  static X86Operand *CreateMem(const MCExpr *Disp, SMLoc StartLoc,
-                               SMLoc EndLoc, unsigned Size = 0) {
+  static X86Operand *CreateMem(const MCExpr *Disp, SMLoc StartLoc, SMLoc EndLoc,
+                               unsigned Size = 0, bool NeedSizeDir = false){
     X86Operand *Res = new X86Operand(Memory, StartLoc, EndLoc);
     Res->Mem.SegReg   = 0;
     Res->Mem.Disp     = Disp;
@@ -459,6 +496,7 @@ struct X86Operand : public MCParsedAsmOperand {
     Res->Mem.IndexReg = 0;
     Res->Mem.Scale    = 1;
     Res->Mem.Size     = Size;
+    Res->Mem.NeedSizeDir = NeedSizeDir;
     return Res;
   }
 
@@ -466,7 +504,7 @@ struct X86Operand : public MCParsedAsmOperand {
   static X86Operand *CreateMem(unsigned SegReg, const MCExpr *Disp,
                                unsigned BaseReg, unsigned IndexReg,
                                unsigned Scale, SMLoc StartLoc, SMLoc EndLoc,
-                               unsigned Size = 0) {
+                               unsigned Size = 0, bool NeedSizeDir = false) {
     // We should never just have a displacement, that should be parsed as an
     // absolute memory operand.
     assert((SegReg || BaseReg || IndexReg) && "Invalid memory operand!");
@@ -481,6 +519,7 @@ struct X86Operand : public MCParsedAsmOperand {
     Res->Mem.IndexReg = IndexReg;
     Res->Mem.Scale    = Scale;
     Res->Mem.Size     = Size;
+    Res->Mem.NeedSizeDir = NeedSizeDir;
     return Res;
   }
 };
@@ -510,12 +549,13 @@ bool X86AsmParser::isDstOp(X86Operand &Op) {
 bool X86AsmParser::ParseRegister(unsigned &RegNo,
                                  SMLoc &StartLoc, SMLoc &EndLoc) {
   RegNo = 0;
-  if (!isParsingIntelSyntax()) {
-    const AsmToken &TokPercent = Parser.getTok();
-    assert(TokPercent.is(AsmToken::Percent) && "Invalid token kind!");
-    StartLoc = TokPercent.getLoc();
+  const AsmToken &PercentTok = Parser.getTok();
+  StartLoc = PercentTok.getLoc();
+
+  // If we encounter a %, ignore it. This code handles registers with and
+  // without the prefix, unprefixed registers can occur in cfi directives.
+  if (!isParsingIntelSyntax() && PercentTok.is(AsmToken::Percent))
     Parser.Lex(); // Eat percent token.
-  }
 
   const AsmToken &Tok = Parser.getTok();
   if (Tok.isNot(AsmToken::Identifier)) {
@@ -621,23 +661,25 @@ X86Operand *X86AsmParser::ParseOperand() {
 
 /// getIntelMemOperandSize - Return intel memory operand size.
 static unsigned getIntelMemOperandSize(StringRef OpStr) {
-  unsigned Size = 0;
-  if (OpStr == "BYTE") Size = 8;
-  if (OpStr == "WORD") Size = 16;
-  if (OpStr == "DWORD") Size = 32;
-  if (OpStr == "QWORD") Size = 64;
-  if (OpStr == "XWORD") Size = 80;
-  if (OpStr == "XMMWORD") Size = 128;
-  if (OpStr == "YMMWORD") Size = 256;
+  unsigned Size = StringSwitch<unsigned>(OpStr)
+    .Cases("BYTE", "byte", 8)
+    .Cases("WORD", "word", 16)
+    .Cases("DWORD", "dword", 32)
+    .Cases("QWORD", "qword", 64)
+    .Cases("XWORD", "xword", 80)
+    .Cases("XMMWORD", "xmmword", 128)
+    .Cases("YMMWORD", "ymmword", 256)
+    .Default(0);
   return Size;
 }
 
-X86Operand *X86AsmParser::ParseIntelBracExpression(unsigned SegReg,
+X86Operand *X86AsmParser::ParseIntelBracExpression(unsigned SegReg, 
                                                    unsigned Size) {
   unsigned BaseReg = 0, IndexReg = 0, Scale = 1;
-  SMLoc Start = Parser.getTok().getLoc(), End;
+  const AsmToken &Tok = Parser.getTok();
+  SMLoc Start = Tok.getLoc(), End;
 
-  const MCExpr *Disp = MCConstantExpr::Create(0, getParser().getContext());
+  const MCExpr *Disp = MCConstantExpr::Create(0, getContext());
   // Parse [ BaseReg + Scale*IndexReg + Disp ] or [ symbol ]
 
   // Eat '['
@@ -653,15 +695,17 @@ X86Operand *X86AsmParser::ParseIntelBracExpression(unsigned SegReg,
       if (getLexer().isNot(AsmToken::RBrac))
         return ErrorOperand(Start, "Expected ']' token!");
       Parser.Lex();
+      End = Tok.getLoc();
       return X86Operand::CreateMem(Disp, Start, End, Size);
     }
   } else if (getLexer().is(AsmToken::Integer)) {
-      int64_t Val = Parser.getTok().getIntVal();
+      int64_t Val = Tok.getIntVal();
       Parser.Lex();
-      SMLoc Loc = Parser.getTok().getLoc();
+      SMLoc Loc = Tok.getLoc();
       if (getLexer().is(AsmToken::RBrac)) {
         // Handle '[' number ']'
         Parser.Lex();
+        End = Tok.getLoc();
         const MCExpr *Disp = MCConstantExpr::Create(Val, getContext());
         if (SegReg)
           return X86Operand::CreateMem(SegReg, Disp, 0, 0, Scale,
@@ -670,7 +714,7 @@ X86Operand *X86AsmParser::ParseIntelBracExpression(unsigned SegReg,
       } else if (getLexer().is(AsmToken::Star)) {
         // Handle '[' Scale*IndexReg ']'
         Parser.Lex();
-        SMLoc IdxRegLoc = Parser.getTok().getLoc();
+        SMLoc IdxRegLoc = Tok.getLoc();
         if (ParseRegister(IndexReg, IdxRegLoc, End))
           return ErrorOperand(IdxRegLoc, "Expected register");
         Scale = Val;
@@ -678,16 +722,27 @@ X86Operand *X86AsmParser::ParseIntelBracExpression(unsigned SegReg,
         return ErrorOperand(Loc, "Unexpected token");
   }
 
-  if (getLexer().is(AsmToken::Plus) || getLexer().is(AsmToken::Minus)) {
-    bool isPlus = getLexer().is(AsmToken::Plus);
+  // Parse ][ as a plus.
+  bool ExpectRBrac = true;
+  if (getLexer().is(AsmToken::RBrac)) {
+    ExpectRBrac = false;
     Parser.Lex();
-    SMLoc PlusLoc = Parser.getTok().getLoc();
+    End = Tok.getLoc();
+  }
+
+  if (getLexer().is(AsmToken::Plus) || getLexer().is(AsmToken::Minus) ||
+      getLexer().is(AsmToken::LBrac)) {
+    ExpectRBrac = true;
+    bool isPlus = getLexer().is(AsmToken::Plus) ||
+      getLexer().is(AsmToken::LBrac);
+    Parser.Lex(); 
+    SMLoc PlusLoc = Tok.getLoc();
     if (getLexer().is(AsmToken::Integer)) {
-      int64_t Val = Parser.getTok().getIntVal();
+      int64_t Val = Tok.getIntVal();
       Parser.Lex();
       if (getLexer().is(AsmToken::Star)) {
         Parser.Lex();
-        SMLoc IdxRegLoc = Parser.getTok().getLoc();
+        SMLoc IdxRegLoc = Tok.getLoc();
         if (ParseRegister(IndexReg, IdxRegLoc, End))
           return ErrorOperand(IdxRegLoc, "Expected register");
         Scale = Val;
@@ -698,21 +753,48 @@ X86Operand *X86AsmParser::ParseIntelBracExpression(unsigned SegReg,
         return ErrorOperand(PlusLoc, "unexpected token after +");
     } else if (getLexer().is(AsmToken::Identifier)) {
       // This could be an index register or a displacement expression.
-      End = Parser.getTok().getLoc();
+      End = Tok.getLoc();
       if (!IndexReg)
         ParseRegister(IndexReg, Start, End);
       else if (getParser().ParseExpression(Disp, End)) return 0;
     }
   }
+  
+  // Parse ][ as a plus.
+  if (getLexer().is(AsmToken::RBrac)) {
+    ExpectRBrac = false;
+    Parser.Lex();
+    End = Tok.getLoc();
+    if (getLexer().is(AsmToken::LBrac)) {
+      ExpectRBrac = true;
+      Parser.Lex();
+      if (getParser().ParseExpression(Disp, End))
+        return 0;
+    }
+  } else if (ExpectRBrac) {
+      if (getParser().ParseExpression(Disp, End))
+        return 0;
+  }
 
-  if (getLexer().isNot(AsmToken::RBrac))
-    if (getParser().ParseExpression(Disp, End)) return 0;
+  if (ExpectRBrac) {
+    if (getLexer().isNot(AsmToken::RBrac))
+      return ErrorOperand(End, "expected ']' token!");
+    Parser.Lex();
+    End = Tok.getLoc();
+  }
 
-  End = Parser.getTok().getLoc();
-  if (getLexer().isNot(AsmToken::RBrac))
-    return ErrorOperand(End, "expected ']' token!");
-  Parser.Lex();
-  End = Parser.getTok().getLoc();
+  // Parse the dot operator (e.g., [ebx].foo.bar).
+  if (Tok.getString().startswith(".")) {
+    SmallString<64> Err;
+    const MCExpr *NewDisp;
+    if (ParseIntelDotOperator(Disp, &NewDisp, Err))
+      return ErrorOperand(Tok.getLoc(), Err);
+    
+    Parser.Lex();  // Eat the field.
+    Disp = NewDisp;
+  }
+
+  End = Tok.getLoc();
 
   // handle [-42]
   if (!BaseReg && !IndexReg)
@@ -723,15 +805,15 @@ X86Operand *X86AsmParser::ParseIntelBracExpression(unsigned SegReg,
 }
 
 /// ParseIntelMemOperand - Parse intel style memory operand.
-X86Operand *X86AsmParser::ParseIntelMemOperand() {
+X86Operand *X86AsmParser::ParseIntelMemOperand(unsigned SegReg, SMLoc Start) {
   const AsmToken &Tok = Parser.getTok();
-  SMLoc Start = Parser.getTok().getLoc(), End;
-  unsigned SegReg = 0;
+  SMLoc End;
 
   unsigned Size = getIntelMemOperandSize(Tok.getString());
   if (Size) {
     Parser.Lex();
-    assert (Tok.getString() == "PTR" && "Unexpected token!");
+    assert ((Tok.getString() == "PTR" || Tok.getString() == "ptr") &&
+            "Unexpected token!");
     Parser.Lex();
   }
 
@@ -750,12 +832,164 @@ X86Operand *X86AsmParser::ParseIntelMemOperand() {
 
   const MCExpr *Disp = MCConstantExpr::Create(0, getParser().getContext());
   if (getParser().ParseExpression(Disp, End)) return 0;
-  return X86Operand::CreateMem(Disp, Start, End, Size);
+  End = Parser.getTok().getLoc();
+
+  bool NeedSizeDir = false;
+  if (!Size && isParsingInlineAsm()) {
+    if (const MCSymbolRefExpr *SymRef = dyn_cast<MCSymbolRefExpr>(Disp)) {
+      const MCSymbol &Sym = SymRef->getSymbol();
+      // FIXME: The SemaLookup will fail if the name is anything other then an
+      // identifier.
+      // FIXME: Pass a valid SMLoc.
+      SemaCallback->LookupInlineAsmIdentifier(Sym.getName(), NULL, Size);
+      NeedSizeDir = Size > 0;
+    }
+  }
+  if (!isParsingInlineAsm())
+    return X86Operand::CreateMem(Disp, Start, End, Size);
+  else
+    // When parsing inline assembly we set the base register to a non-zero value
+    // as we don't know the actual value at this time.  This is necessary to
+    // get the matching correct in some cases.
+    return X86Operand::CreateMem(/*SegReg*/0, Disp, /*BaseReg*/1, /*IndexReg*/0,
+                                 /*Scale*/1, Start, End, Size, NeedSizeDir);
+}
+
+/// Parse the '.' operator.
+bool X86AsmParser::ParseIntelDotOperator(const MCExpr *Disp,
+                                         const MCExpr **NewDisp,
+                                         SmallString<64> &Err) {
+  AsmToken Tok = *&Parser.getTok();
+  uint64_t OrigDispVal, DotDispVal;
+
+  // FIXME: Handle non-constant expressions.
+  if (const MCConstantExpr *OrigDisp = dyn_cast<MCConstantExpr>(Disp)) {
+    OrigDispVal = OrigDisp->getValue();
+  } else {
+    Err = "Non-constant offsets are not supported!";
+    return true;
+  }
+
+  // Drop the '.'.
+  StringRef DotDispStr = Tok.getString().drop_front(1);
+
+  // .Imm gets lexed as a real.
+  if (Tok.is(AsmToken::Real)) {
+    APInt DotDisp;
+    DotDispStr.getAsInteger(10, DotDisp);
+    DotDispVal = DotDisp.getZExtValue();
+  } else if (Tok.is(AsmToken::Identifier)) {
+    // We should only see an identifier when parsing the original inline asm.
+    // The front-end should rewrite this in terms of immediates.
+    assert (isParsingInlineAsm() && "Unexpected field name!");
+
+    unsigned DotDisp;
+    std::pair<StringRef, StringRef> BaseMember = DotDispStr.split('.');
+    if (SemaCallback->LookupInlineAsmField(BaseMember.first, BaseMember.second,
+                                           DotDisp)) {
+      Err = "Unable to lookup field reference!";
+      return true;
+    }
+    DotDispVal = DotDisp;
+  } else {
+    Err = "Unexpected token type!";
+    return true;
+  }
+
+  if (isParsingInlineAsm() && Tok.is(AsmToken::Identifier)) {
+    SMLoc Loc = SMLoc::getFromPointer(DotDispStr.data());
+    unsigned Len = DotDispStr.size();
+    unsigned Val = OrigDispVal + DotDispVal;
+    InstInfo->AsmRewrites->push_back(AsmRewrite(AOK_DotOperator, Loc, Len,
+                                                Val));
+  }
+
+  *NewDisp = MCConstantExpr::Create(OrigDispVal + DotDispVal, getContext());
+  return false;
+}
+
+/// Parse the 'offset' operator.  This operator is used to specify the
+/// location rather then the content of a variable.
+X86Operand *X86AsmParser::ParseIntelOffsetOfOperator(SMLoc Start) {
+  SMLoc OffsetOfLoc = Start;
+  Parser.Lex(); // Eat offset.
+  Start = Parser.getTok().getLoc();
+  assert (Parser.getTok().is(AsmToken::Identifier) && "Expected an identifier");
+
+  SMLoc End;
+  const MCExpr *Val;
+  if (getParser().ParseExpression(Val, End))
+    return ErrorOperand(Start, "Unable to parse expression!");
+
+  End = Parser.getTok().getLoc();
+
+  // Don't emit the offset operator.
+  InstInfo->AsmRewrites->push_back(AsmRewrite(AOK_Skip, OffsetOfLoc, 7));
+
+  // The offset operator will have an 'r' constraint, thus we need to create
+  // register operand to ensure proper matching.  Just pick a GPR based on
+  // the size of a pointer.
+  unsigned RegNo = is64BitMode() ? X86::RBX : X86::EBX;
+  return X86Operand::CreateReg(RegNo, Start, End, OffsetOfLoc);
+}
+
+/// Parse the 'TYPE' operator.  The TYPE operator returns the size of a C or
+/// C++ type or variable. If the variable is an array, TYPE returns the size of
+/// a single element of the array.
+X86Operand *X86AsmParser::ParseIntelTypeOperator(SMLoc Start) {
+  SMLoc TypeLoc = Start;
+  Parser.Lex(); // Eat offset.
+  Start = Parser.getTok().getLoc();
+  assert (Parser.getTok().is(AsmToken::Identifier) && "Expected an identifier");
+
+  SMLoc End;
+  const MCExpr *Val;
+  if (getParser().ParseExpression(Val, End))
+    return 0;
+
+  End = Parser.getTok().getLoc();
+
+  unsigned Size = 0;
+  if (const MCSymbolRefExpr *SymRef = dyn_cast<MCSymbolRefExpr>(Val)) {
+    const MCSymbol &Sym = SymRef->getSymbol();
+    // FIXME: The SemaLookup will fail if the name is anything other then an
+    // identifier.
+    // FIXME: Pass a valid SMLoc.
+    if (!SemaCallback->LookupInlineAsmIdentifier(Sym.getName(), NULL, Size))
+      return ErrorOperand(Start, "Unable to lookup TYPE of expr!");
+
+    Size /= 8; // Size is in terms of bits, but we want bytes in the context.
+  }
+
+  // Rewrite the type operator and the C or C++ type or variable in terms of an
+  // immediate.  E.g. TYPE foo -> $$4
+  unsigned Len = End.getPointer() - TypeLoc.getPointer();
+  InstInfo->AsmRewrites->push_back(AsmRewrite(AOK_Imm, TypeLoc, Len, Size));
+
+  const MCExpr *Imm = MCConstantExpr::Create(Size, getContext());
+  return X86Operand::CreateImm(Imm, Start, End, /*NeedAsmRewrite*/false);
 }
 
 X86Operand *X86AsmParser::ParseIntelOperand() {
   SMLoc Start = Parser.getTok().getLoc(), End;
 
+  // offset operator.
+  StringRef AsmTokStr = Parser.getTok().getString();
+  if ((AsmTokStr == "offset" || AsmTokStr == "OFFSET") &&
+      isParsingInlineAsm())
+    return ParseIntelOffsetOfOperator(Start);
+
+  // Type directive.
+  if ((AsmTokStr == "type" || AsmTokStr == "TYPE") &&
+      isParsingInlineAsm())
+    return ParseIntelTypeOperator(Start);
+
+  // Unsupported directives.
+  if (isParsingIntelSyntax() &&
+      (AsmTokStr == "size" || AsmTokStr == "SIZE" ||
+       AsmTokStr == "length" || AsmTokStr == "LENGTH"))
+      return ErrorOperand(Start, "Unsupported directive!");
+
   // immediate.
   if (getLexer().is(AsmToken::Integer) || getLexer().is(AsmToken::Real) ||
       getLexer().is(AsmToken::Minus)) {
@@ -769,12 +1003,17 @@ X86Operand *X86AsmParser::ParseIntelOperand() {
   // register
   unsigned RegNo = 0;
   if (!ParseRegister(RegNo, Start, End)) {
-    End = Parser.getTok().getLoc();
-    return X86Operand::CreateReg(RegNo, Start, End);
+    // If this is a segment register followed by a ':', then this is the start
+    // of a memory reference, otherwise this is a normal register reference.
+    if (getLexer().isNot(AsmToken::Colon))
+      return X86Operand::CreateReg(RegNo, Start, Parser.getTok().getLoc());
+
+    getParser().Lex(); // Eat the colon.
+    return ParseIntelMemOperand(RegNo, Start);
   }
 
   // mem operand
-  return ParseIntelMemOperand();
+  return ParseIntelMemOperand(0, Start);
 }
 
 X86Operand *X86AsmParser::ParseATTOperand() {
@@ -972,8 +1211,9 @@ X86Operand *X86AsmParser::ParseMemOperand(unsigned SegReg, SMLoc MemStart) {
 }
 
 bool X86AsmParser::
-ParseInstruction(StringRef Name, SMLoc NameLoc,
+ParseInstruction(ParseInstructionInfo &Info, StringRef Name, SMLoc NameLoc,
                  SmallVectorImpl<MCParsedAsmOperand*> &Operands) {
+  InstInfo = &Info;
   StringRef PatchedName = Name;
 
   // FIXME: Hack to recognize setneb as setne.
@@ -1509,28 +1749,18 @@ processInstruction(MCInst &Inst,
 }
 
 bool X86AsmParser::
-MatchAndEmitInstruction(SMLoc IDLoc,
+MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
                         SmallVectorImpl<MCParsedAsmOperand*> &Operands,
-                        MCStreamer &Out) {
-  SmallVector<MCInst, 2> Insts;
-  bool Error = MatchInstruction(IDLoc, Operands, Insts);
-  if (!Error)
-    for (unsigned i = 0, e = Insts.size(); i != e; ++i)
-      Out.EmitInstruction(Insts[i]);
-  return Error;
-}
-
-bool X86AsmParser::
-MatchInstruction(SMLoc IDLoc,
-                 SmallVectorImpl<MCParsedAsmOperand*> &Operands,
-                 SmallVectorImpl<MCInst> &MCInsts) {
+                        MCStreamer &Out, unsigned &ErrorInfo,
+                        bool MatchingInlineAsm) {
   assert(!Operands.empty() && "Unexpect empty operand list!");
   X86Operand *Op = static_cast<X86Operand*>(Operands[0]);
   assert(Op->isToken() && "Leading operand should always be a mnemonic!");
+  ArrayRef<SMRange> EmptyRanges = ArrayRef<SMRange>();
 
   // First, handle aliases that expand to multiple instructions.
   // FIXME: This should be replaced with a real .td file alias mechanism.
-  // Also, MatchInstructionImpl should do actually *do* the EmitInstruction
+  // Also, MatchInstructionImpl should actually *do* the EmitInstruction
   // call.
   if (Op->getToken() == "fstsw" || Op->getToken() == "fstcw" ||
       Op->getToken() == "fstsww" || Op->getToken() == "fstcww" ||
@@ -1539,7 +1769,8 @@ MatchInstruction(SMLoc IDLoc,
     MCInst Inst;
     Inst.setOpcode(X86::WAIT);
     Inst.setLoc(IDLoc);
-    MCInsts.push_back(Inst);
+    if (!MatchingInlineAsm)
+      Out.EmitInstruction(Inst);
 
     const char *Repl =
       StringSwitch<const char*>(Op->getToken())
@@ -1558,28 +1789,30 @@ MatchInstruction(SMLoc IDLoc,
   }
 
   bool WasOriginallyInvalidOperand = false;
-  unsigned OrigErrorInfo;
   MCInst Inst;
 
   // First, try a direct match.
-  switch (MatchInstructionImpl(Operands, Inst, OrigErrorInfo,
+  switch (MatchInstructionImpl(Operands, Inst,
+                               ErrorInfo, MatchingInlineAsm,
                                isParsingIntelSyntax())) {
   default: break;
   case Match_Success:
     // Some instructions need post-processing to, for example, tweak which
     // encoding is selected. Loop on it while changes happen so the
     // individual transformations can chain off each other.
-    while (processInstruction(Inst, Operands))
-      ;
+    if (!MatchingInlineAsm)
+      while (processInstruction(Inst, Operands))
+        ;
 
     Inst.setLoc(IDLoc);
-    MCInsts.push_back(Inst);
+    if (!MatchingInlineAsm)
+      Out.EmitInstruction(Inst);
+    Opcode = Inst.getOpcode();
     return false;
   case Match_MissingFeature:
-    Error(IDLoc, "instruction requires a CPU feature not currently enabled");
+    Error(IDLoc, "instruction requires a CPU feature not currently enabled",
+          EmptyRanges, MatchingInlineAsm);
     return true;
-  case Match_ConversionFail:
-    return Error(IDLoc, "unable to convert operands to instruction");
   case Match_InvalidOperand:
     WasOriginallyInvalidOperand = true;
     break;
@@ -1612,13 +1845,17 @@ MatchInstruction(SMLoc IDLoc,
   unsigned ErrorInfoIgnore;
   unsigned Match1, Match2, Match3, Match4;
 
-  Match1 = MatchInstructionImpl(Operands, Inst, ErrorInfoIgnore);
+  Match1 = MatchInstructionImpl(Operands, Inst, ErrorInfoIgnore,
+                                isParsingIntelSyntax());
   Tmp[Base.size()] = Suffixes[1];
-  Match2 = MatchInstructionImpl(Operands, Inst, ErrorInfoIgnore);
+  Match2 = MatchInstructionImpl(Operands, Inst, ErrorInfoIgnore,
+                                isParsingIntelSyntax());
   Tmp[Base.size()] = Suffixes[2];
-  Match3 = MatchInstructionImpl(Operands, Inst, ErrorInfoIgnore);
+  Match3 = MatchInstructionImpl(Operands, Inst, ErrorInfoIgnore,
+                                isParsingIntelSyntax());
   Tmp[Base.size()] = Suffixes[3];
-  Match4 = MatchInstructionImpl(Operands, Inst, ErrorInfoIgnore);
+  Match4 = MatchInstructionImpl(Operands, Inst, ErrorInfoIgnore,
+                                isParsingIntelSyntax());
 
   // Restore the old token.
   Op->setTokenValue(Base);
@@ -1631,7 +1868,9 @@ MatchInstruction(SMLoc IDLoc,
     (Match3 == Match_Success) + (Match4 == Match_Success);
   if (NumSuccessfulMatches == 1) {
     Inst.setLoc(IDLoc);
-    MCInsts.push_back(Inst);
+    if (!MatchingInlineAsm)
+      Out.EmitInstruction(Inst);
+    Opcode = Inst.getOpcode();
     return false;
   }
 
@@ -1658,7 +1897,7 @@ MatchInstruction(SMLoc IDLoc,
       OS << "'" << Base << MatchChars[i] << "'";
     }
     OS << ")";
-    Error(IDLoc, OS.str());
+    Error(IDLoc, OS.str(), EmptyRanges, MatchingInlineAsm);
     return true;
   }
 
@@ -1669,31 +1908,36 @@ MatchInstruction(SMLoc IDLoc,
   if ((Match1 == Match_MnemonicFail) && (Match2 == Match_MnemonicFail) &&
       (Match3 == Match_MnemonicFail) && (Match4 == Match_MnemonicFail)) {
     if (!WasOriginallyInvalidOperand) {
+      ArrayRef<SMRange> Ranges = MatchingInlineAsm ? EmptyRanges :
+        Op->getLocRange();
       return Error(IDLoc, "invalid instruction mnemonic '" + Base + "'",
-                   Op->getLocRange());
+                   Ranges, MatchingInlineAsm);
     }
 
     // Recover location info for the operand if we know which was the problem.
-    if (OrigErrorInfo != ~0U) {
-      if (OrigErrorInfo >= Operands.size())
-        return Error(IDLoc, "too few operands for instruction");
+    if (ErrorInfo != ~0U) {
+      if (ErrorInfo >= Operands.size())
+        return Error(IDLoc, "too few operands for instruction",
+                     EmptyRanges, MatchingInlineAsm);
 
-      X86Operand *Operand = (X86Operand*)Operands[OrigErrorInfo];
+      X86Operand *Operand = (X86Operand*)Operands[ErrorInfo];
       if (Operand->getStartLoc().isValid()) {
         SMRange OperandRange = Operand->getLocRange();
         return Error(Operand->getStartLoc(), "invalid operand for instruction",
-                     OperandRange);
+                     OperandRange, MatchingInlineAsm);
       }
     }
 
-    return Error(IDLoc, "invalid operand for instruction");
+    return Error(IDLoc, "invalid operand for instruction", EmptyRanges,
+                 MatchingInlineAsm);
   }
 
   // If one instruction matched with a missing feature, report this as a
   // missing feature.
   if ((Match1 == Match_MissingFeature) + (Match2 == Match_MissingFeature) +
       (Match3 == Match_MissingFeature) + (Match4 == Match_MissingFeature) == 1){
-    Error(IDLoc, "instruction requires a CPU feature not currently enabled");
+    Error(IDLoc, "instruction requires a CPU feature not currently enabled",
+          EmptyRanges, MatchingInlineAsm);
     return true;
   }
 
@@ -1701,12 +1945,14 @@ MatchInstruction(SMLoc IDLoc,
   // operand failure.
   if ((Match1 == Match_InvalidOperand) + (Match2 == Match_InvalidOperand) +
       (Match3 == Match_InvalidOperand) + (Match4 == Match_InvalidOperand) == 1){
-    Error(IDLoc, "invalid operand for instruction");
+    Error(IDLoc, "invalid operand for instruction", EmptyRanges,
+          MatchingInlineAsm);
     return true;
   }
 
   // If all of these were an outright failure, report it in a useless way.
-  Error(IDLoc, "unknown use of instruction mnemonic without a size suffix");
+  Error(IDLoc, "unknown use of instruction mnemonic without a size suffix",
+        EmptyRanges, MatchingInlineAsm);
   return true;
 }
 
@@ -1717,7 +1963,10 @@ bool X86AsmParser::ParseDirective(AsmToken DirectiveID) {
     return ParseDirectiveWord(2, DirectiveID.getLoc());
   else if (IDVal.startswith(".code"))
     return ParseDirectiveCode(IDVal, DirectiveID.getLoc());
-  else if (IDVal.startswith(".intel_syntax")) {
+  else if (IDVal.startswith(".att_syntax")) {
+    getParser().setAssemblerDialect(0);
+    return false;
+  } else if (IDVal.startswith(".intel_syntax")) {
     getParser().setAssemblerDialect(1);
     if (getLexer().isNot(AsmToken::EndOfStatement)) {
       if(Parser.getTok().getString() == "noprefix") {
diff --git a/lib/Target/X86/CMakeLists.txt b/lib/Target/X86/CMakeLists.txt
index b886d46..f4d03a6 100644
--- a/lib/Target/X86/CMakeLists.txt
+++ b/lib/Target/X86/CMakeLists.txt
@@ -17,7 +17,6 @@ set(sources
   X86AsmPrinter.cpp
   X86COFFMachineModuleInfo.cpp
   X86CodeEmitter.cpp
-  X86ELFWriterInfo.cpp
   X86FastISel.cpp
   X86FloatingPoint.cpp
   X86FrameLowering.cpp
diff --git a/lib/Target/X86/Disassembler/X86Disassembler.cpp b/lib/Target/X86/Disassembler/X86Disassembler.cpp
index 5039887..f136927 100644
--- a/lib/Target/X86/Disassembler/X86Disassembler.cpp
+++ b/lib/Target/X86/Disassembler/X86Disassembler.cpp
@@ -44,7 +44,7 @@ void x86DisassemblerDebug(const char *file,
   dbgs() << file << ":" << line << ": " << s;
 }
 
-const char *x86DisassemblerGetInstrName(unsigned Opcode, void *mii) {
+const char *x86DisassemblerGetInstrName(unsigned Opcode, const void *mii) {
   const MCInstrInfo *MII = static_cast<const MCInstrInfo *>(mii);
   return MII->getName(Opcode);
 }
@@ -95,8 +95,8 @@ const EDInstInfo *X86GenericDisassembler::getEDInfo() const {
 ///                   be a pointer to a MemoryObject.
 /// @param byte     - A pointer to the byte to be read.
 /// @param address  - The address to be read.
-static int regionReader(void* arg, uint8_t* byte, uint64_t address) {
-  MemoryObject* region = static_cast<MemoryObject*>(arg);
+static int regionReader(const void* arg, uint8_t* byte, uint64_t address) {
+  const MemoryObject* region = static_cast<const MemoryObject*>(arg);
   return region->readByte(address, byte);
 }
 
@@ -135,10 +135,10 @@ X86GenericDisassembler::getInstruction(MCInst &instr,
   
   int ret = decodeInstruction(&internalInstr,
                               regionReader,
-                              (void*)&region,
+                              (const void*)&region,
                               loggerFn,
                               (void*)&vStream,
-                              (void*)MII,
+                              (const void*)MII,
                               address,
                               fMode);
 
@@ -379,6 +379,8 @@ static void translateImmediate(MCInst &mcInst, uint64_t immediate,
   }
 
   switch (type) {
+  case TYPE_XMM32:
+  case TYPE_XMM64:
   case TYPE_XMM128:
     mcInst.addOperand(MCOperand::CreateReg(X86::XMM0 + (immediate >> 4)));
     return;
diff --git a/lib/Target/X86/Disassembler/X86Disassembler.h b/lib/Target/X86/Disassembler/X86Disassembler.h
index 0dbfa26..981701f 100644
--- a/lib/Target/X86/Disassembler/X86Disassembler.h
+++ b/lib/Target/X86/Disassembler/X86Disassembler.h
@@ -78,7 +78,7 @@
   uint16_t operands;
 
 #define INSTRUCTION_IDS               \
-  unsigned instructionIDs;
+  uint16_t instructionIDs;
 
 #include "X86DisassemblerDecoderCommon.h"
 
diff --git a/lib/Target/X86/Disassembler/X86DisassemblerDecoder.c b/lib/Target/X86/Disassembler/X86DisassemblerDecoder.c
index 0c92912..85d8a99 100644
--- a/lib/Target/X86/Disassembler/X86DisassemblerDecoder.c
+++ b/lib/Target/X86/Disassembler/X86DisassemblerDecoder.c
@@ -138,6 +138,10 @@ static InstrUID decode(OpcodeType type,
     if (modFromModRM(modRM) == 0x3)
       return modRMTable[dec->instructionIDs+((modRM & 0x38) >> 3)+8];
     return modRMTable[dec->instructionIDs+((modRM & 0x38) >> 3)];
+  case MODRM_SPLITMISC:
+    if (modFromModRM(modRM) == 0x3)
+      return modRMTable[dec->instructionIDs+(modRM & 0x3f)+8];
+    return modRMTable[dec->instructionIDs+((modRM & 0x38) >> 3)];
   case MODRM_FULL:
     return modRMTable[dec->instructionIDs+modRM];
   }
@@ -200,7 +204,7 @@ static void unconsumeByte(struct InternalInstruction* insn) {
                              insn->readerCursor + offset);        \
       if (ret)                                                    \
         return ret;                                               \
-      combined = combined | ((type)byte << ((type)offset * 8));   \
+      combined = combined | ((uint64_t)byte << (offset * 8));     \
     }                                                             \
     *ptr = combined;                                              \
     insn->readerCursor += sizeof(type);                           \
@@ -690,7 +694,7 @@ static int getIDWithAttrMask(uint16_t* instructionID,
  * @param orig  - The instruction that is not 16-bit
  * @param equiv - The instruction that is 16-bit
  */
-static BOOL is16BitEquvalent(const char* orig, const char* equiv) {
+static BOOL is16BitEquivalent(const char* orig, const char* equiv) {
   off_t i;
   
   for (i = 0;; i++) {
@@ -719,7 +723,7 @@ static BOOL is16BitEquvalent(const char* orig, const char* equiv) {
  * @return      - 0 if the ModR/M could be read when needed or was not needed;
  *                nonzero otherwise.
  */
-static int getID(struct InternalInstruction* insn, void *miiArg) {
+static int getID(struct InternalInstruction* insn, const void *miiArg) {
   uint8_t attrMask;
   uint16_t instructionID;
   
@@ -856,7 +860,7 @@ static int getID(struct InternalInstruction* insn, void *miiArg) {
     specWithOpSizeName =
       x86DisassemblerGetInstrName(instructionIDWithOpsize, miiArg);
 
-    if (is16BitEquvalent(specName, specWithOpSizeName)) {
+    if (is16BitEquivalent(specName, specWithOpSizeName)) {
       insn->instructionID = instructionIDWithOpsize;
       insn->spec = specifierForUID(instructionIDWithOpsize);
     } else {
@@ -1621,10 +1625,10 @@ static int readOperands(struct InternalInstruction* insn) {
  */
 int decodeInstruction(struct InternalInstruction* insn,
                       byteReader_t reader,
-                      void* readerArg,
+                      const void* readerArg,
                       dlog_t logger,
                       void* loggerArg,
-                      void* miiArg,
+                      const void* miiArg,
                       uint64_t startLoc,
                       DisassemblerMode mode) {
   memset(insn, 0, sizeof(struct InternalInstruction));
diff --git a/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h b/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h
index 797703f..407ead3 100644
--- a/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h
+++ b/lib/Target/X86/Disassembler/X86DisassemblerDecoder.h
@@ -24,7 +24,7 @@ extern "C" {
   uint16_t operands;
 
 #define INSTRUCTION_IDS     \
-  unsigned instructionIDs;
+  uint16_t instructionIDs;
 
 #include "X86DisassemblerDecoderCommon.h"
 
@@ -403,7 +403,7 @@ typedef uint8_t BOOL;
  *                  be read from.
  * @return        - -1 if the byte cannot be read for any reason; 0 otherwise.
  */
-typedef int (*byteReader_t)(void* arg, uint8_t* byte, uint64_t address);
+typedef int (*byteReader_t)(const void* arg, uint8_t* byte, uint64_t address);
 
 /*
  * dlog_t - Type for the logging function that the consumer can provide to
@@ -422,7 +422,7 @@ struct InternalInstruction {
   /* Reader interface (C) */
   byteReader_t reader;
   /* Opaque value passed to the reader */
-  void* readerArg;
+  const void* readerArg;
   /* The address of the next byte to read via the reader */
   uint64_t readerCursor;
 
@@ -561,10 +561,10 @@ struct InternalInstruction {
  */
 int decodeInstruction(struct InternalInstruction* insn,
                       byteReader_t reader,
-                      void* readerArg,
+                      const void* readerArg,
                       dlog_t logger,
                       void* loggerArg,
-                      void* miiArg,
+                      const void* miiArg,
                       uint64_t startLoc,
                       DisassemblerMode mode);
 
@@ -579,7 +579,7 @@ void x86DisassemblerDebug(const char *file,
                           unsigned line,
                           const char *s);
 
-const char *x86DisassemblerGetInstrName(unsigned Opcode, void *mii);
+const char *x86DisassemblerGetInstrName(unsigned Opcode, const void *mii);
 
 #ifdef __cplusplus
 }
diff --git a/lib/Target/X86/Disassembler/X86DisassemblerDecoderCommon.h b/lib/Target/X86/Disassembler/X86DisassemblerDecoderCommon.h
index b0a0e1e..23dfe4b 100644
--- a/lib/Target/X86/Disassembler/X86DisassemblerDecoderCommon.h
+++ b/lib/Target/X86/Disassembler/X86DisassemblerDecoderCommon.h
@@ -160,6 +160,10 @@ typedef uint16_t InstrUID;
  * MODRM_SPLITRM  - If the ModR/M byte is between 0x00 and 0xbf, the opcode
  *                  corresponds to one instruction; otherwise, it corresponds to
  *                  a different instruction.
+ * MODRM_SPLITMISC- If the ModR/M byte is between 0x00 and 0xbf, ModR/M byte
+ *                  divided by 8 is used to select instruction; otherwise, each
+ *                  value of the ModR/M byte could correspond to a different
+ *                  instruction.
  * MODRM_SPLITREG - ModR/M byte divided by 8 is used to select instruction. This
                     corresponds to instructions that use reg field as opcode
  * MODRM_FULL     - Potentially, each value of the ModR/M byte could correspond
@@ -169,6 +173,7 @@ typedef uint16_t InstrUID;
 #define MODRMTYPES            \
   ENUM_ENTRY(MODRM_ONEENTRY)  \
   ENUM_ENTRY(MODRM_SPLITRM)   \
+  ENUM_ENTRY(MODRM_SPLITMISC)  \
   ENUM_ENTRY(MODRM_SPLITREG)  \
   ENUM_ENTRY(MODRM_FULL)
 
diff --git a/lib/Target/X86/InstPrinter/X86ATTInstPrinter.cpp b/lib/Target/X86/InstPrinter/X86ATTInstPrinter.cpp
index 5118e4c..a4bd114 100644
--- a/lib/Target/X86/InstPrinter/X86ATTInstPrinter.cpp
+++ b/lib/Target/X86/InstPrinter/X86ATTInstPrinter.cpp
@@ -15,6 +15,7 @@
 #define DEBUG_TYPE "asm-printer"
 #include "X86ATTInstPrinter.h"
 #include "X86InstComments.h"
+#include "MCTargetDesc/X86BaseInfo.h"
 #include "MCTargetDesc/X86MCTargetDesc.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCAsmInfo.h"
@@ -33,11 +34,19 @@ using namespace llvm;
 
 void X86ATTInstPrinter::printRegName(raw_ostream &OS,
                                      unsigned RegNo) const {
-  OS << '%' << getRegisterName(RegNo);
+  OS << markup("<reg:")
+     << '%' << getRegisterName(RegNo)
+     << markup(">");
 }
 
 void X86ATTInstPrinter::printInst(const MCInst *MI, raw_ostream &OS,
                                   StringRef Annot) {
+  const MCInstrDesc &Desc = MII.get(MI->getOpcode());
+  uint64_t TSFlags = Desc.TSFlags;
+
+  if (TSFlags & X86II::LOCK)
+    OS << "\tlock\n";
+
   // Try to print any aliases first.
   if (!printAliasInstr(MI, OS))
     printInstruction(MI, OS);
@@ -52,7 +61,8 @@ void X86ATTInstPrinter::printInst(const MCInst *MI, raw_ostream &OS,
 
 void X86ATTInstPrinter::printSSECC(const MCInst *MI, unsigned Op,
                                    raw_ostream &O) {
-  switch (MI->getOperand(Op).getImm()) {
+  int64_t Imm = MI->getOperand(Op).getImm() & 0xf;
+  switch (Imm) {
   default: llvm_unreachable("Invalid ssecc argument!");
   case    0: O << "eq"; break;
   case    1: O << "lt"; break;
@@ -70,6 +80,30 @@ void X86ATTInstPrinter::printSSECC(const MCInst *MI, unsigned Op,
   case  0xd: O << "ge"; break;
   case  0xe: O << "gt"; break;
   case  0xf: O << "true"; break;
+  }
+}
+
+void X86ATTInstPrinter::printAVXCC(const MCInst *MI, unsigned Op,
+                                   raw_ostream &O) {
+  int64_t Imm = MI->getOperand(Op).getImm() & 0x1f;
+  switch (Imm) {
+  default: llvm_unreachable("Invalid avxcc argument!");
+  case    0: O << "eq"; break;
+  case    1: O << "lt"; break;
+  case    2: O << "le"; break;
+  case    3: O << "unord"; break;
+  case    4: O << "neq"; break;
+  case    5: O << "nlt"; break;
+  case    6: O << "nle"; break;
+  case    7: O << "ord"; break;
+  case    8: O << "eq_uq"; break;
+  case    9: O << "nge"; break;
+  case  0xa: O << "ngt"; break;
+  case  0xb: O << "false"; break;
+  case  0xc: O << "neq_oq"; break;
+  case  0xd: O << "ge"; break;
+  case  0xe: O << "gt"; break;
+  case  0xf: O << "true"; break;
   case 0x10: O << "eq_os"; break;
   case 0x11: O << "lt_oq"; break;
   case 0x12: O << "le_oq"; break;
@@ -89,12 +123,12 @@ void X86ATTInstPrinter::printSSECC(const MCInst *MI, unsigned Op,
   }
 }
 
-/// print_pcrel_imm - This is used to print an immediate value that ends up
+/// printPCRelImm - This is used to print an immediate value that ends up
 /// being encoded as a pc-relative value (e.g. for jumps and calls).  These
 /// print slightly differently than normal immediates.  For example, a $ is not
 /// emitted.
-void X86ATTInstPrinter::print_pcrel_imm(const MCInst *MI, unsigned OpNo,
-                                        raw_ostream &O) {
+void X86ATTInstPrinter::printPCRelImm(const MCInst *MI, unsigned OpNo,
+                                      raw_ostream &O) {
   const MCOperand &Op = MI->getOperand(OpNo);
   if (Op.isImm())
     O << Op.getImm();
@@ -119,17 +153,21 @@ void X86ATTInstPrinter::printOperand(const MCInst *MI, unsigned OpNo,
                                      raw_ostream &O) {
   const MCOperand &Op = MI->getOperand(OpNo);
   if (Op.isReg()) {
-    O << '%' << getRegisterName(Op.getReg());
+    printRegName(O, Op.getReg());
   } else if (Op.isImm()) {
     // Print X86 immediates as signed values.
-    O << '$' << (int64_t)Op.getImm();
+    O << markup("<imm:")
+      << '$' << (int64_t)Op.getImm()
+      << markup(">");
     
     if (CommentStream && (Op.getImm() > 255 || Op.getImm() < -256))
       *CommentStream << format("imm = 0x%" PRIX64 "\n", (uint64_t)Op.getImm());
     
   } else {
     assert(Op.isExpr() && "unknown operand kind in printOperand");
-    O << '$' << *Op.getExpr();
+    O << markup("<imm:")
+      << '$' << *Op.getExpr()
+      << markup(">");
   }
 }
 
@@ -140,6 +178,8 @@ void X86ATTInstPrinter::printMemReference(const MCInst *MI, unsigned Op,
   const MCOperand &DispSpec = MI->getOperand(Op+3);
   const MCOperand &SegReg = MI->getOperand(Op+4);
   
+  O << markup("<mem:");
+
   // If this has a segment register, print it.
   if (SegReg.getReg()) {
     printOperand(MI, Op+4, O);
@@ -164,9 +204,15 @@ void X86ATTInstPrinter::printMemReference(const MCInst *MI, unsigned Op,
       O << ',';
       printOperand(MI, Op+2, O);
       unsigned ScaleVal = MI->getOperand(Op+1).getImm();
-      if (ScaleVal != 1)
-        O << ',' << ScaleVal;
+      if (ScaleVal != 1) {
+        O << ','
+	  << markup("<imm:")
+          << ScaleVal
+	  << markup(">");
+      }
     }
     O << ')';
   }
+
+  O << markup(">");
 }
diff --git a/lib/Target/X86/InstPrinter/X86ATTInstPrinter.h b/lib/Target/X86/InstPrinter/X86ATTInstPrinter.h
index 2e00bff..8e09183 100644
--- a/lib/Target/X86/InstPrinter/X86ATTInstPrinter.h
+++ b/lib/Target/X86/InstPrinter/X86ATTInstPrinter.h
@@ -40,7 +40,8 @@ public:
   void printOperand(const MCInst *MI, unsigned OpNo, raw_ostream &OS);
   void printMemReference(const MCInst *MI, unsigned Op, raw_ostream &OS);
   void printSSECC(const MCInst *MI, unsigned Op, raw_ostream &OS);
-  void print_pcrel_imm(const MCInst *MI, unsigned OpNo, raw_ostream &OS);
+  void printAVXCC(const MCInst *MI, unsigned Op, raw_ostream &OS);
+  void printPCRelImm(const MCInst *MI, unsigned OpNo, raw_ostream &OS);
   
   void printopaquemem(const MCInst *MI, unsigned OpNo, raw_ostream &O) {
     printMemReference(MI, OpNo, O);
diff --git a/lib/Target/X86/InstPrinter/X86IntelInstPrinter.cpp b/lib/Target/X86/InstPrinter/X86IntelInstPrinter.cpp
index 4ea662c..d67aec7 100644
--- a/lib/Target/X86/InstPrinter/X86IntelInstPrinter.cpp
+++ b/lib/Target/X86/InstPrinter/X86IntelInstPrinter.cpp
@@ -1,4 +1,4 @@
-//===-- X86IntelInstPrinter.cpp - AT&T assembly instruction printing ------===//
+//===-- X86IntelInstPrinter.cpp - Intel assembly instruction printing -----===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This file includes code for rendering MCInst instances as AT&T-style
+// This file includes code for rendering MCInst instances as Intel-style
 // assembly.
 //
 //===----------------------------------------------------------------------===//
@@ -15,6 +15,7 @@
 #define DEBUG_TYPE "asm-printer"
 #include "X86IntelInstPrinter.h"
 #include "X86InstComments.h"
+#include "MCTargetDesc/X86BaseInfo.h"
 #include "MCTargetDesc/X86MCTargetDesc.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCExpr.h"
@@ -32,6 +33,12 @@ void X86IntelInstPrinter::printRegName(raw_ostream &OS, unsigned RegNo) const {
 
 void X86IntelInstPrinter::printInst(const MCInst *MI, raw_ostream &OS,
                                     StringRef Annot) {
+  const MCInstrDesc &Desc = MII.get(MI->getOpcode());
+  uint64_t TSFlags = Desc.TSFlags;
+
+  if (TSFlags & X86II::LOCK)
+    OS << "\tlock\n";
+
   printInstruction(MI, OS);
 
   // Next always print the annotation.
@@ -44,7 +51,8 @@ void X86IntelInstPrinter::printInst(const MCInst *MI, raw_ostream &OS,
 
 void X86IntelInstPrinter::printSSECC(const MCInst *MI, unsigned Op,
                                      raw_ostream &O) {
-  switch (MI->getOperand(Op).getImm()) {
+  int64_t Imm = MI->getOperand(Op).getImm() & 0xf;
+  switch (Imm) {
   default: llvm_unreachable("Invalid ssecc argument!");
   case    0: O << "eq"; break;
   case    1: O << "lt"; break;
@@ -62,6 +70,30 @@ void X86IntelInstPrinter::printSSECC(const MCInst *MI, unsigned Op,
   case  0xd: O << "ge"; break;
   case  0xe: O << "gt"; break;
   case  0xf: O << "true"; break;
+  }
+}
+
+void X86IntelInstPrinter::printAVXCC(const MCInst *MI, unsigned Op,
+                                     raw_ostream &O) {
+  int64_t Imm = MI->getOperand(Op).getImm() & 0x1f;
+  switch (Imm) {
+  default: llvm_unreachable("Invalid avxcc argument!");
+  case    0: O << "eq"; break;
+  case    1: O << "lt"; break;
+  case    2: O << "le"; break;
+  case    3: O << "unord"; break;
+  case    4: O << "neq"; break;
+  case    5: O << "nlt"; break;
+  case    6: O << "nle"; break;
+  case    7: O << "ord"; break;
+  case    8: O << "eq_uq"; break;
+  case    9: O << "nge"; break;
+  case  0xa: O << "ngt"; break;
+  case  0xb: O << "false"; break;
+  case  0xc: O << "neq_oq"; break;
+  case  0xd: O << "ge"; break;
+  case  0xe: O << "gt"; break;
+  case  0xf: O << "true"; break;
   case 0x10: O << "eq_os"; break;
   case 0x11: O << "lt_oq"; break;
   case 0x12: O << "le_oq"; break;
@@ -78,14 +110,13 @@ void X86IntelInstPrinter::printSSECC(const MCInst *MI, unsigned Op,
   case 0x1d: O << "ge_oq"; break;
   case 0x1e: O << "gt_oq"; break;
   case 0x1f: O << "true_us"; break;
-
   }
 }
 
-/// print_pcrel_imm - This is used to print an immediate value that ends up
+/// printPCRelImm - This is used to print an immediate value that ends up
 /// being encoded as a pc-relative value.
-void X86IntelInstPrinter::print_pcrel_imm(const MCInst *MI, unsigned OpNo,
-                                          raw_ostream &O) {
+void X86IntelInstPrinter::printPCRelImm(const MCInst *MI, unsigned OpNo,
+                                        raw_ostream &O) {
   const MCOperand &Op = MI->getOperand(OpNo);
   if (Op.isImm())
     O << Op.getImm();
@@ -153,8 +184,7 @@ void X86IntelInstPrinter::printMemReference(const MCInst *MI, unsigned Op,
     printOperand(MI, Op+2, O);
     NeedPlus = true;
   }
-  
-  
+
   if (!DispSpec.isImm()) {
     if (NeedPlus) O << " + ";
     assert(DispSpec.isExpr() && "non-immediate displacement for LEA?");
diff --git a/lib/Target/X86/InstPrinter/X86IntelInstPrinter.h b/lib/Target/X86/InstPrinter/X86IntelInstPrinter.h
index 4f5938d..bb769eb 100644
--- a/lib/Target/X86/InstPrinter/X86IntelInstPrinter.h
+++ b/lib/Target/X86/InstPrinter/X86IntelInstPrinter.h
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This class prints an X86 MCInst to intel style .s file syntax.
+// This class prints an X86 MCInst to Intel style .s file syntax.
 //
 //===----------------------------------------------------------------------===//
 
@@ -37,7 +37,8 @@ public:
   void printOperand(const MCInst *MI, unsigned OpNo, raw_ostream &O);
   void printMemReference(const MCInst *MI, unsigned Op, raw_ostream &O);
   void printSSECC(const MCInst *MI, unsigned Op, raw_ostream &O);
-  void print_pcrel_imm(const MCInst *MI, unsigned OpNo, raw_ostream &O);
+  void printAVXCC(const MCInst *MI, unsigned Op, raw_ostream &O);
+  void printPCRelImm(const MCInst *MI, unsigned OpNo, raw_ostream &O);
   
   void printopaquemem(const MCInst *MI, unsigned OpNo, raw_ostream &O) {
     O << "OPAQUE PTR ";
diff --git a/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp b/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
index 32e40fe..467edad 100644
--- a/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
+++ b/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp
@@ -66,9 +66,10 @@ public:
 };
 
 class X86AsmBackend : public MCAsmBackend {
+  StringRef CPU;
 public:
-  X86AsmBackend(const Target &T)
-    : MCAsmBackend() {}
+  X86AsmBackend(const Target &T, StringRef _CPU)
+    : MCAsmBackend(), CPU(_CPU) {}
 
   unsigned getNumFixupKinds() const {
     return X86::NumTargetFixupKinds;
@@ -278,9 +279,9 @@ void X86AsmBackend::relaxInstruction(const MCInst &Inst, MCInst &Res) const {
   Res.setOpcode(RelaxedOp);
 }
 
-/// writeNopData - Write optimal nops to the output file for the \arg Count
+/// writeNopData - Write optimal nops to the output file for the \p Count
 /// bytes.  This returns the number of bytes written.  It may return 0 if
-/// the \arg Count is more than the maximum optimal nops.
+/// the \p Count is more than the maximum optimal nops.
 bool X86AsmBackend::writeNopData(uint64_t Count, MCObjectWriter *OW) const {
   static const uint8_t Nops[10][10] = {
     // nop
@@ -305,6 +306,15 @@ bool X86AsmBackend::writeNopData(uint64_t Count, MCObjectWriter *OW) const {
     {0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00},
   };
 
+  // This CPU doesnt support long nops. If needed add more.
+  // FIXME: Can we get this from the subtarget somehow?
+  if (CPU == "generic" || CPU == "i386" || CPU == "i486" || CPU == "i586" ||
+      CPU == "pentium" || CPU == "pentium-mmx" || CPU == "geode") {
+    for (uint64_t i = 0; i < Count; ++i)
+      OW->Write8(0x90);
+    return true;
+  }
+
   // Write an optimal sequence for the first 15 bytes.
   const uint64_t OptimalCount = (Count < 16) ? Count : 15;
   const uint64_t Prefixes = OptimalCount <= 10 ? 0 : OptimalCount - 10;
@@ -327,8 +337,8 @@ namespace {
 class ELFX86AsmBackend : public X86AsmBackend {
 public:
   uint8_t OSABI;
-  ELFX86AsmBackend(const Target &T, uint8_t _OSABI)
-    : X86AsmBackend(T), OSABI(_OSABI) {
+  ELFX86AsmBackend(const Target &T, uint8_t _OSABI, StringRef CPU)
+    : X86AsmBackend(T, CPU), OSABI(_OSABI) {
     HasReliableSymbolDifference = true;
   }
 
@@ -340,21 +350,21 @@ public:
 
 class ELFX86_32AsmBackend : public ELFX86AsmBackend {
 public:
-  ELFX86_32AsmBackend(const Target &T, uint8_t OSABI)
-    : ELFX86AsmBackend(T, OSABI) {}
+  ELFX86_32AsmBackend(const Target &T, uint8_t OSABI, StringRef CPU)
+    : ELFX86AsmBackend(T, OSABI, CPU) {}
 
   MCObjectWriter *createObjectWriter(raw_ostream &OS) const {
-    return createX86ELFObjectWriter(OS, /*Is64Bit*/ false, OSABI);
+    return createX86ELFObjectWriter(OS, /*IsELF64*/ false, OSABI, ELF::EM_386);
   }
 };
 
 class ELFX86_64AsmBackend : public ELFX86AsmBackend {
 public:
-  ELFX86_64AsmBackend(const Target &T, uint8_t OSABI)
-    : ELFX86AsmBackend(T, OSABI) {}
+  ELFX86_64AsmBackend(const Target &T, uint8_t OSABI, StringRef CPU)
+    : ELFX86AsmBackend(T, OSABI, CPU) {}
 
   MCObjectWriter *createObjectWriter(raw_ostream &OS) const {
-    return createX86ELFObjectWriter(OS, /*Is64Bit*/ true, OSABI);
+    return createX86ELFObjectWriter(OS, /*IsELF64*/ true, OSABI, ELF::EM_X86_64);
   }
 };
 
@@ -362,8 +372,8 @@ class WindowsX86AsmBackend : public X86AsmBackend {
   bool Is64Bit;
 
 public:
-  WindowsX86AsmBackend(const Target &T, bool is64Bit)
-    : X86AsmBackend(T)
+  WindowsX86AsmBackend(const Target &T, bool is64Bit, StringRef CPU)
+    : X86AsmBackend(T, CPU)
     , Is64Bit(is64Bit) {
   }
 
@@ -374,14 +384,14 @@ public:
 
 class DarwinX86AsmBackend : public X86AsmBackend {
 public:
-  DarwinX86AsmBackend(const Target &T)
-    : X86AsmBackend(T) { }
+  DarwinX86AsmBackend(const Target &T, StringRef CPU)
+    : X86AsmBackend(T, CPU) { }
 };
 
 class DarwinX86_32AsmBackend : public DarwinX86AsmBackend {
 public:
-  DarwinX86_32AsmBackend(const Target &T)
-    : DarwinX86AsmBackend(T) {}
+  DarwinX86_32AsmBackend(const Target &T, StringRef CPU)
+    : DarwinX86AsmBackend(T, CPU) {}
 
   MCObjectWriter *createObjectWriter(raw_ostream &OS) const {
     return createX86MachObjectWriter(OS, /*Is64Bit=*/false,
@@ -392,8 +402,8 @@ public:
 
 class DarwinX86_64AsmBackend : public DarwinX86AsmBackend {
 public:
-  DarwinX86_64AsmBackend(const Target &T)
-    : DarwinX86AsmBackend(T) {
+  DarwinX86_64AsmBackend(const Target &T, StringRef CPU)
+    : DarwinX86AsmBackend(T, CPU) {
     HasReliableSymbolDifference = true;
   }
 
@@ -439,28 +449,28 @@ public:
 
 } // end anonymous namespace
 
-MCAsmBackend *llvm::createX86_32AsmBackend(const Target &T, StringRef TT) {
+MCAsmBackend *llvm::createX86_32AsmBackend(const Target &T, StringRef TT, StringRef CPU) {
   Triple TheTriple(TT);
 
   if (TheTriple.isOSDarwin() || TheTriple.getEnvironment() == Triple::MachO)
-    return new DarwinX86_32AsmBackend(T);
+    return new DarwinX86_32AsmBackend(T, CPU);
 
-  if (TheTriple.isOSWindows())
-    return new WindowsX86AsmBackend(T, false);
+  if (TheTriple.isOSWindows() && TheTriple.getEnvironment() != Triple::ELF)
+    return new WindowsX86AsmBackend(T, false, CPU);
 
   uint8_t OSABI = MCELFObjectTargetWriter::getOSABI(TheTriple.getOS());
-  return new ELFX86_32AsmBackend(T, OSABI);
+  return new ELFX86_32AsmBackend(T, OSABI, CPU);
 }
 
-MCAsmBackend *llvm::createX86_64AsmBackend(const Target &T, StringRef TT) {
+MCAsmBackend *llvm::createX86_64AsmBackend(const Target &T, StringRef TT, StringRef CPU) {
   Triple TheTriple(TT);
 
   if (TheTriple.isOSDarwin() || TheTriple.getEnvironment() == Triple::MachO)
-    return new DarwinX86_64AsmBackend(T);
+    return new DarwinX86_64AsmBackend(T, CPU);
 
-  if (TheTriple.isOSWindows())
-    return new WindowsX86AsmBackend(T, true);
+  if (TheTriple.isOSWindows() && TheTriple.getEnvironment() != Triple::ELF)
+    return new WindowsX86AsmBackend(T, true, CPU);
 
   uint8_t OSABI = MCELFObjectTargetWriter::getOSABI(TheTriple.getOS());
-  return new ELFX86_64AsmBackend(T, OSABI);
+  return new ELFX86_64AsmBackend(T, OSABI, CPU);
 }
diff --git a/lib/Target/X86/MCTargetDesc/X86BaseInfo.h b/lib/Target/X86/MCTargetDesc/X86BaseInfo.h
index db597fb..7ea1961 100644
--- a/lib/Target/X86/MCTargetDesc/X86BaseInfo.h
+++ b/lib/Target/X86/MCTargetDesc/X86BaseInfo.h
@@ -276,9 +276,9 @@ namespace X86II {
     MRM_C1 = 33, MRM_C2 = 34, MRM_C3 = 35, MRM_C4 = 36,
     MRM_C8 = 37, MRM_C9 = 38, MRM_E8 = 39, MRM_F0 = 40,
     MRM_F8 = 41, MRM_F9 = 42, MRM_D0 = 45, MRM_D1 = 46,
-    MRM_D4 = 47, MRM_D8 = 48, MRM_D9 = 49, MRM_DA = 50,
-    MRM_DB = 51, MRM_DC = 52, MRM_DD = 53, MRM_DE = 54,
-    MRM_DF = 55,
+    MRM_D4 = 47, MRM_D5 = 48, MRM_D8 = 49, MRM_D9 = 50,
+    MRM_DA = 51, MRM_DB = 52, MRM_DC = 53, MRM_DD = 54,
+    MRM_DE = 55, MRM_DF = 56,
 
     /// RawFrmImm8 - This is used for the ENTER instruction, which has two
     /// immediates, the first of which is a 16-bit immediate (specified by
@@ -580,11 +580,11 @@ namespace X86II {
     case X86II::MRM_E8: case X86II::MRM_F0:
     case X86II::MRM_F8: case X86II::MRM_F9:
     case X86II::MRM_D0: case X86II::MRM_D1:
-    case X86II::MRM_D4: case X86II::MRM_D8:
-    case X86II::MRM_D9: case X86II::MRM_DA:
-    case X86II::MRM_DB: case X86II::MRM_DC:
-    case X86II::MRM_DD: case X86II::MRM_DE:
-    case X86II::MRM_DF:
+    case X86II::MRM_D4: case X86II::MRM_D5:
+    case X86II::MRM_D8: case X86II::MRM_D9:
+    case X86II::MRM_DA: case X86II::MRM_DB:
+    case X86II::MRM_DC: case X86II::MRM_DD:
+    case X86II::MRM_DE: case X86II::MRM_DF:
       return -1;
     }
   }
diff --git a/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp b/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp
index 5a42a80..de80dd8 100644
--- a/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp
+++ b/lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp
@@ -20,7 +20,7 @@ using namespace llvm;
 namespace {
   class X86ELFObjectWriter : public MCELFObjectTargetWriter {
   public:
-    X86ELFObjectWriter(bool is64Bit, uint8_t OSABI);
+    X86ELFObjectWriter(bool IsELF64, uint8_t OSABI, uint16_t EMachine);
 
     virtual ~X86ELFObjectWriter();
   protected:
@@ -30,10 +30,11 @@ namespace {
   };
 }
 
-X86ELFObjectWriter::X86ELFObjectWriter(bool Is64Bit, uint8_t OSABI)
-  : MCELFObjectTargetWriter(Is64Bit, OSABI,
-                            Is64Bit ?  ELF::EM_X86_64 : ELF::EM_386,
-                            /*HasRelocationAddend*/ Is64Bit) {}
+X86ELFObjectWriter::X86ELFObjectWriter(bool IsELF64, uint8_t OSABI,
+                                       uint16_t EMachine)
+  : MCELFObjectTargetWriter(IsELF64, OSABI, EMachine,
+                            // Only i386 uses Rel instead of RelA.
+                            /*HasRelocationAddend*/ EMachine != ELF::EM_386) {}
 
 X86ELFObjectWriter::~X86ELFObjectWriter()
 {}
@@ -48,7 +49,7 @@ unsigned X86ELFObjectWriter::GetRelocType(const MCValue &Target,
   MCSymbolRefExpr::VariantKind Modifier = Target.isAbsolute() ?
     MCSymbolRefExpr::VK_None : Target.getSymA()->getKind();
   unsigned Type;
-  if (is64Bit()) {
+  if (getEMachine() == ELF::EM_X86_64) {
     if (IsPCRel) {
       switch ((unsigned)Fixup.getKind()) {
       default: llvm_unreachable("invalid fixup kind!");
@@ -130,7 +131,7 @@ unsigned X86ELFObjectWriter::GetRelocType(const MCValue &Target,
       case FK_Data_1: Type = ELF::R_X86_64_8; break;
       }
     }
-  } else {
+  } else if (getEMachine() == ELF::EM_386) {
     if (IsPCRel) {
       switch ((unsigned)Fixup.getKind()) {
       default: llvm_unreachable("invalid fixup kind!");
@@ -210,15 +211,17 @@ unsigned X86ELFObjectWriter::GetRelocType(const MCValue &Target,
       case FK_Data_1: Type = ELF::R_386_8; break;
       }
     }
-  }
+  } else
+    llvm_unreachable("Unsupported ELF machine type.");
 
   return Type;
 }
 
 MCObjectWriter *llvm::createX86ELFObjectWriter(raw_ostream &OS,
-                                               bool Is64Bit,
-                                               uint8_t OSABI) {
+                                               bool IsELF64,
+                                               uint8_t OSABI,
+                                               uint16_t EMachine) {
   MCELFObjectTargetWriter *MOTW =
-    new X86ELFObjectWriter(Is64Bit, OSABI);
+    new X86ELFObjectWriter(IsELF64, OSABI, EMachine);
   return createELFObjectWriter(MOTW, OS,  /*IsLittleEndian=*/true);
 }
diff --git a/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp b/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp
index b0acd7d..16488eb 100644
--- a/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp
+++ b/lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp
@@ -34,6 +34,10 @@ AsmWriterFlavor("x86-asm-syntax", cl::init(ATT),
              clEnumValN(Intel, "intel", "Emit Intel-style assembly"),
              clEnumValEnd));
 
+static cl::opt<bool>
+MarkedJTDataRegions("mark-data-regions", cl::init(false),
+  cl::desc("Mark code section jump table data regions."),
+  cl::Hidden);
 
 void X86MCAsmInfoDarwin::anchor() { }
 
@@ -59,6 +63,7 @@ X86MCAsmInfoDarwin::X86MCAsmInfoDarwin(const Triple &T) {
 
   SupportsDebugInformation = true;
   DwarfUsesInlineInfoSection = true;
+  UseDataRegionDirectives = MarkedJTDataRegions;
 
   // Exceptions handling
   ExceptionsType = ExceptionHandling::DwarfCFI;
diff --git a/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp b/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
index 4a38324..122204a 100644
--- a/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
+++ b/lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp
@@ -16,6 +16,7 @@
 #include "MCTargetDesc/X86BaseInfo.h"
 #include "MCTargetDesc/X86FixupKinds.h"
 #include "llvm/MC/MCCodeEmitter.h"
+#include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCExpr.h"
 #include "llvm/MC/MCInst.h"
 #include "llvm/MC/MCInstrInfo.h"
@@ -28,8 +29,8 @@ using namespace llvm;
 
 namespace {
 class X86MCCodeEmitter : public MCCodeEmitter {
-  X86MCCodeEmitter(const X86MCCodeEmitter &); // DO NOT IMPLEMENT
-  void operator=(const X86MCCodeEmitter &); // DO NOT IMPLEMENT
+  X86MCCodeEmitter(const X86MCCodeEmitter &) LLVM_DELETED_FUNCTION;
+  void operator=(const X86MCCodeEmitter &) LLVM_DELETED_FUNCTION;
   const MCInstrInfo &MCII;
   const MCSubtargetInfo &STI;
   MCContext &Ctx;
@@ -51,8 +52,8 @@ public:
     return (STI.getFeatureBits() & X86::Mode64Bit) == 0;
   }
 
-  static unsigned GetX86RegNum(const MCOperand &MO) {
-    return X86_MC::getX86RegNum(MO.getReg());
+  unsigned GetX86RegNum(const MCOperand &MO) const {
+    return Ctx.getRegisterInfo().getEncodingValue(MO.getReg()) & 0x7;
   }
 
   // On regular x86, both XMM0-XMM7 and XMM8-XMM15 are encoded in the range
@@ -64,8 +65,8 @@ public:
   //  VEX.VVVV    => XMM9 => ~9
   //
   // See table 4-35 of Intel AVX Programming Reference for details.
-  static unsigned char getVEXRegisterEncoding(const MCInst &MI,
-                                              unsigned OpNum) {
+  unsigned char getVEXRegisterEncoding(const MCInst &MI,
+                                       unsigned OpNum) const {
     unsigned SrcReg = MI.getOperand(OpNum).getReg();
     unsigned SrcRegNum = GetX86RegNum(MI.getOperand(OpNum));
     if (X86II::isX86_64ExtendedReg(SrcReg))
@@ -560,15 +561,6 @@ void X86MCCodeEmitter::EmitVEXOpcodePrefix(uint64_t TSFlags, unsigned &CurByte,
   }
 
 
-  // Set the vector length to 256-bit if YMM0-YMM15 is used
-  for (unsigned i = 0; i != MI.getNumOperands(); ++i) {
-    if (!MI.getOperand(i).isReg())
-      continue;
-    unsigned SrcReg = MI.getOperand(i).getReg();
-    if (SrcReg >= X86::YMM0 && SrcReg <= X86::YMM15)
-      VEX_L = 1;
-  }
-
   // Classify VEX_B, VEX_4V, VEX_R, VEX_X
   unsigned NumOps = Desc.getNumOperands();
   unsigned CurOp = 0;
@@ -1129,13 +1121,13 @@ EncodeInstruction(const MCInst &MI, raw_ostream &OS,
   case X86II::MRM_C3: case X86II::MRM_C4:
   case X86II::MRM_C8: case X86II::MRM_C9:
   case X86II::MRM_D0: case X86II::MRM_D1:
-  case X86II::MRM_D4: case X86II::MRM_D8:
-  case X86II::MRM_D9: case X86II::MRM_DA:
-  case X86II::MRM_DB: case X86II::MRM_DC:
-  case X86II::MRM_DD: case X86II::MRM_DE:
-  case X86II::MRM_DF: case X86II::MRM_E8:
-  case X86II::MRM_F0: case X86II::MRM_F8:
-  case X86II::MRM_F9:
+  case X86II::MRM_D4: case X86II::MRM_D5:
+  case X86II::MRM_D8: case X86II::MRM_D9:
+  case X86II::MRM_DA: case X86II::MRM_DB:
+  case X86II::MRM_DC: case X86II::MRM_DD:
+  case X86II::MRM_DE: case X86II::MRM_DF:
+  case X86II::MRM_E8: case X86II::MRM_F0:
+  case X86II::MRM_F8: case X86II::MRM_F9:
     EmitByte(BaseOpcode, CurByte, OS);
 
     unsigned char MRM;
@@ -1150,6 +1142,7 @@ EncodeInstruction(const MCInst &MI, raw_ostream &OS,
     case X86II::MRM_D0: MRM = 0xD0; break;
     case X86II::MRM_D1: MRM = 0xD1; break;
     case X86II::MRM_D4: MRM = 0xD4; break;
+    case X86II::MRM_D5: MRM = 0xD5; break;
     case X86II::MRM_D8: MRM = 0xD8; break;
     case X86II::MRM_D9: MRM = 0xD9; break;
     case X86II::MRM_DA: MRM = 0xDA; break;
diff --git a/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp b/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp
index 3482363..287c9f1 100644
--- a/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp
+++ b/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp
@@ -209,117 +209,10 @@ unsigned X86_MC::getDwarfRegFlavour(StringRef TT, bool isEH) {
   return DWARFFlavour::X86_32_Generic;
 }
 
-/// getX86RegNum - This function maps LLVM register identifiers to their X86
-/// specific numbering, which is used in various places encoding instructions.
-unsigned X86_MC::getX86RegNum(unsigned RegNo) {
-  switch(RegNo) {
-  case X86::RAX: case X86::EAX: case X86::AX: case X86::AL: return N86::EAX;
-  case X86::RCX: case X86::ECX: case X86::CX: case X86::CL: return N86::ECX;
-  case X86::RDX: case X86::EDX: case X86::DX: case X86::DL: return N86::EDX;
-  case X86::RBX: case X86::EBX: case X86::BX: case X86::BL: return N86::EBX;
-  case X86::RSP: case X86::ESP: case X86::SP: case X86::SPL: case X86::AH:
-    return N86::ESP;
-  case X86::RBP: case X86::EBP: case X86::BP: case X86::BPL: case X86::CH:
-    return N86::EBP;
-  case X86::RSI: case X86::ESI: case X86::SI: case X86::SIL: case X86::DH:
-    return N86::ESI;
-  case X86::RDI: case X86::EDI: case X86::DI: case X86::DIL: case X86::BH:
-    return N86::EDI;
-
-  case X86::R8:  case X86::R8D:  case X86::R8W:  case X86::R8B:
-    return N86::EAX;
-  case X86::R9:  case X86::R9D:  case X86::R9W:  case X86::R9B:
-    return N86::ECX;
-  case X86::R10: case X86::R10D: case X86::R10W: case X86::R10B:
-    return N86::EDX;
-  case X86::R11: case X86::R11D: case X86::R11W: case X86::R11B:
-    return N86::EBX;
-  case X86::R12: case X86::R12D: case X86::R12W: case X86::R12B:
-    return N86::ESP;
-  case X86::R13: case X86::R13D: case X86::R13W: case X86::R13B:
-    return N86::EBP;
-  case X86::R14: case X86::R14D: case X86::R14W: case X86::R14B:
-    return N86::ESI;
-  case X86::R15: case X86::R15D: case X86::R15W: case X86::R15B:
-    return N86::EDI;
-
-  case X86::ST0: case X86::ST1: case X86::ST2: case X86::ST3:
-  case X86::ST4: case X86::ST5: case X86::ST6: case X86::ST7:
-    return RegNo-X86::ST0;
-
-  case X86::XMM0: case X86::XMM8:
-  case X86::YMM0: case X86::YMM8: case X86::MM0:
-    return 0;
-  case X86::XMM1: case X86::XMM9:
-  case X86::YMM1: case X86::YMM9: case X86::MM1:
-    return 1;
-  case X86::XMM2: case X86::XMM10:
-  case X86::YMM2: case X86::YMM10: case X86::MM2:
-    return 2;
-  case X86::XMM3: case X86::XMM11:
-  case X86::YMM3: case X86::YMM11: case X86::MM3:
-    return 3;
-  case X86::XMM4: case X86::XMM12:
-  case X86::YMM4: case X86::YMM12: case X86::MM4:
-    return 4;
-  case X86::XMM5: case X86::XMM13:
-  case X86::YMM5: case X86::YMM13: case X86::MM5:
-    return 5;
-  case X86::XMM6: case X86::XMM14:
-  case X86::YMM6: case X86::YMM14: case X86::MM6:
-    return 6;
-  case X86::XMM7: case X86::XMM15:
-  case X86::YMM7: case X86::YMM15: case X86::MM7:
-    return 7;
-
-  case X86::ES: return 0;
-  case X86::CS: return 1;
-  case X86::SS: return 2;
-  case X86::DS: return 3;
-  case X86::FS: return 4;
-  case X86::GS: return 5;
-
-  case X86::CR0: case X86::CR8 : case X86::DR0: return 0;
-  case X86::CR1: case X86::CR9 : case X86::DR1: return 1;
-  case X86::CR2: case X86::CR10: case X86::DR2: return 2;
-  case X86::CR3: case X86::CR11: case X86::DR3: return 3;
-  case X86::CR4: case X86::CR12: case X86::DR4: return 4;
-  case X86::CR5: case X86::CR13: case X86::DR5: return 5;
-  case X86::CR6: case X86::CR14: case X86::DR6: return 6;
-  case X86::CR7: case X86::CR15: case X86::DR7: return 7;
-
-  // Pseudo index registers are equivalent to a "none"
-  // scaled index (See Intel Manual 2A, table 2-3)
-  case X86::EIZ:
-  case X86::RIZ:
-    return 4;
-
-  default:
-    assert((int(RegNo) > 0) && "Unknown physical register!");
-    return 0;
-  }
-}
-
 void X86_MC::InitLLVM2SEHRegisterMapping(MCRegisterInfo *MRI) {
   // FIXME: TableGen these.
   for (unsigned Reg = X86::NoRegister+1; Reg < X86::NUM_TARGET_REGS; ++Reg) {
-    int SEH = X86_MC::getX86RegNum(Reg);
-    switch (Reg) {
-    case X86::R8:  case X86::R8D:  case X86::R8W:  case X86::R8B:
-    case X86::R9:  case X86::R9D:  case X86::R9W:  case X86::R9B:
-    case X86::R10: case X86::R10D: case X86::R10W: case X86::R10B:
-    case X86::R11: case X86::R11D: case X86::R11W: case X86::R11B:
-    case X86::R12: case X86::R12D: case X86::R12W: case X86::R12B:
-    case X86::R13: case X86::R13D: case X86::R13W: case X86::R13B:
-    case X86::R14: case X86::R14D: case X86::R14W: case X86::R14B:
-    case X86::R15: case X86::R15D: case X86::R15W: case X86::R15B:
-    case X86::XMM8:  case X86::XMM9:  case X86::XMM10: case X86::XMM11:
-    case X86::XMM12: case X86::XMM13: case X86::XMM14: case X86::XMM15:
-    case X86::YMM8:  case X86::YMM9:  case X86::YMM10: case X86::YMM11:
-    case X86::YMM12: case X86::YMM13: case X86::YMM14: case X86::YMM15:
-      SEH += 8;
-      break;
-    }
+    unsigned SEH = MRI->getEncodingValue(Reg);
     MRI->mapLLVMRegToSEHReg(Reg, SEH);
   }
 }
@@ -379,11 +272,15 @@ static MCAsmInfo *createX86MCAsmInfo(const Target &T, StringRef TT) {
       MAI = new X86_64MCAsmInfoDarwin(TheTriple);
     else
       MAI = new X86MCAsmInfoDarwin(TheTriple);
+  } else if (TheTriple.getEnvironment() == Triple::ELF) {
+    // Force the use of an ELF container.
+    MAI = new X86ELFMCAsmInfo(TheTriple);
   } else if (TheTriple.getOS() == Triple::Win32) {
     MAI = new X86MCAsmInfoMicrosoft(TheTriple);
   } else if (TheTriple.getOS() == Triple::MinGW32 || TheTriple.getOS() == Triple::Cygwin) {
     MAI = new X86MCAsmInfoGNUCOFF(TheTriple);
   } else {
+    // The default is ELF.
     MAI = new X86ELFMCAsmInfo(TheTriple);
   }
 
@@ -465,7 +362,7 @@ static MCStreamer *createMCStreamer(const Target &T, StringRef TT,
   if (TheTriple.isOSDarwin() || TheTriple.getEnvironment() == Triple::MachO)
     return createMachOStreamer(Ctx, MAB, _OS, _Emitter, RelaxAll);
 
-  if (TheTriple.isOSWindows())
+  if (TheTriple.isOSWindows() && TheTriple.getEnvironment() != Triple::ELF)
     return createWinCOFFStreamer(Ctx, MAB, *_Emitter, _OS, RelaxAll);
 
   return createELFStreamer(Ctx, MAB, _OS, _Emitter, RelaxAll, NoExecStack);
diff --git a/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.h b/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.h
index 4650069..981aa1a 100644
--- a/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.h
+++ b/lib/Target/X86/MCTargetDesc/X86MCTargetDesc.h
@@ -64,8 +64,6 @@ namespace X86_MC {
 
   unsigned getDwarfRegFlavour(StringRef TT, bool isEH);
 
-  unsigned getX86RegNum(unsigned RegNo);
-
   void InitLLVM2SEHRegisterMapping(MCRegisterInfo *MRI);
 
   /// createX86MCSubtargetInfo - Create a X86 MCSubtargetInfo instance.
@@ -80,8 +78,8 @@ MCCodeEmitter *createX86MCCodeEmitter(const MCInstrInfo &MCII,
                                       const MCSubtargetInfo &STI,
                                       MCContext &Ctx);
 
-MCAsmBackend *createX86_32AsmBackend(const Target &T, StringRef TT);
-MCAsmBackend *createX86_64AsmBackend(const Target &T, StringRef TT);
+MCAsmBackend *createX86_32AsmBackend(const Target &T, StringRef TT, StringRef CPU);
+MCAsmBackend *createX86_64AsmBackend(const Target &T, StringRef TT, StringRef CPU);
 
 /// createX86MachObjectWriter - Construct an X86 Mach-O object writer.
 MCObjectWriter *createX86MachObjectWriter(raw_ostream &OS,
@@ -91,8 +89,9 @@ MCObjectWriter *createX86MachObjectWriter(raw_ostream &OS,
 
 /// createX86ELFObjectWriter - Construct an X86 ELF object writer.
 MCObjectWriter *createX86ELFObjectWriter(raw_ostream &OS,
-                                         bool Is64Bit,
-                                         uint8_t OSABI);
+                                         bool IsELF64,
+                                         uint8_t OSABI,
+                                         uint16_t EMachine);
 /// createX86WinCOFFObjectWriter - Construct an X86 Win COFF object writer.
 MCObjectWriter *createX86WinCOFFObjectWriter(raw_ostream &OS, bool Is64Bit);
 } // End llvm namespace
diff --git a/lib/Target/X86/MCTargetDesc/X86MachObjectWriter.cpp b/lib/Target/X86/MCTargetDesc/X86MachObjectWriter.cpp
index f0f1982..7ff058e 100644
--- a/lib/Target/X86/MCTargetDesc/X86MachObjectWriter.cpp
+++ b/lib/Target/X86/MCTargetDesc/X86MachObjectWriter.cpp
@@ -11,11 +11,13 @@
 #include "MCTargetDesc/X86MCTargetDesc.h"
 #include "llvm/MC/MCAssembler.h"
 #include "llvm/MC/MCAsmLayout.h"
+#include "llvm/MC/MCContext.h"
 #include "llvm/MC/MCMachObjectWriter.h"
 #include "llvm/MC/MCSectionMachO.h"
 #include "llvm/MC/MCValue.h"
 #include "llvm/ADT/Twine.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/Format.h"
 #include "llvm/Object/MachOFormat.h"
 
 using namespace llvm;
@@ -23,7 +25,7 @@ using namespace llvm::object;
 
 namespace {
 class X86MachObjectWriter : public MCMachObjectTargetWriter {
-  void RecordScatteredRelocation(MachObjectWriter *Writer,
+  bool RecordScatteredRelocation(MachObjectWriter *Writer,
                                  const MCAssembler &Asm,
                                  const MCAsmLayout &Layout,
                                  const MCFragment *Fragment,
@@ -335,7 +337,7 @@ void X86MachObjectWriter::RecordX86_64Relocation(MachObjectWriter *Writer,
   Writer->addRelocation(Fragment->getParent(), MRE);
 }
 
-void X86MachObjectWriter::RecordScatteredRelocation(MachObjectWriter *Writer,
+bool X86MachObjectWriter::RecordScatteredRelocation(MachObjectWriter *Writer,
                                                     const MCAssembler &Asm,
                                                     const MCAsmLayout &Layout,
                                                     const MCFragment *Fragment,
@@ -381,6 +383,19 @@ void X86MachObjectWriter::RecordScatteredRelocation(MachObjectWriter *Writer,
   // Relocations are written out in reverse order, so the PAIR comes first.
   if (Type == macho::RIT_Difference ||
       Type == macho::RIT_Generic_LocalDifference) {
+    // If the offset is too large to fit in a scattered relocation,
+    // we're hosed. It's an unfortunate limitation of the MachO format.
+    if (FixupOffset > 0xffffff) {
+      char Buffer[32];
+      format("0x%x", FixupOffset).print(Buffer, sizeof(Buffer));
+      Asm.getContext().FatalError(Fixup.getLoc(),
+                         Twine("Section too large, can't encode "
+                                "r_address (") + Buffer +
+                         ") into 24 bits of scattered "
+                         "relocation entry.");
+      llvm_unreachable("fatal error returned?!");
+    }
+
     macho::RelocationEntry MRE;
     MRE.Word0 = ((0         <<  0) |
                  (macho::RIT_Pair  << 24) |
@@ -389,6 +404,16 @@ void X86MachObjectWriter::RecordScatteredRelocation(MachObjectWriter *Writer,
                  macho::RF_Scattered);
     MRE.Word1 = Value2;
     Writer->addRelocation(Fragment->getParent(), MRE);
+  } else {
+    // If the offset is more than 24-bits, it won't fit in a scattered
+    // relocation offset field, so we fall back to using a non-scattered
+    // relocation. This is a bit risky, as if the offset reaches out of
+    // the block and the linker is doing scattered loading on this
+    // symbol, things can go badly.
+    //
+    // Required for 'as' compatibility.
+    if (FixupOffset > 0xffffff)
+      return false;
   }
 
   macho::RelocationEntry MRE;
@@ -399,6 +424,7 @@ void X86MachObjectWriter::RecordScatteredRelocation(MachObjectWriter *Writer,
                macho::RF_Scattered);
   MRE.Word1 = Value;
   Writer->addRelocation(Fragment->getParent(), MRE);
+  return true;
 }
 
 void X86MachObjectWriter::RecordTLVPRelocation(MachObjectWriter *Writer,
@@ -469,9 +495,11 @@ void X86MachObjectWriter::RecordX86Relocation(MachObjectWriter *Writer,
   // If this is a difference or a defined symbol plus an offset, then we need a
   // scattered relocation entry. Differences always require scattered
   // relocations.
-  if (Target.getSymB())
-    return RecordScatteredRelocation(Writer, Asm, Layout, Fragment, Fixup,
-                                     Target, Log2Size, FixedValue);
+  if (Target.getSymB()) {
+    RecordScatteredRelocation(Writer, Asm, Layout, Fragment, Fixup,
+                              Target, Log2Size, FixedValue);
+    return;
+  }
 
   // Get the symbol data, if any.
   MCSymbolData *SD = 0;
@@ -483,9 +511,13 @@ void X86MachObjectWriter::RecordX86Relocation(MachObjectWriter *Writer,
   uint32_t Offset = Target.getConstant();
   if (IsPCRel)
     Offset += 1 << Log2Size;
-  if (Offset && SD && !Writer->doesSymbolRequireExternRelocation(SD))
-    return RecordScatteredRelocation(Writer, Asm, Layout, Fragment, Fixup,
-                                     Target, Log2Size, FixedValue);
+  // Try to record the scattered relocation if needed. Fall back to non
+  // scattered if necessary (see comments in RecordScatteredRelocation()
+  // for details).
+  if (Offset && SD && !Writer->doesSymbolRequireExternRelocation(SD) &&
+      RecordScatteredRelocation(Writer, Asm, Layout, Fragment, Fixup,
+                                Target, Log2Size, FixedValue))
+    return;
 
   // See <reloc.h>.
   uint32_t FixupOffset = Layout.getFragmentOffset(Fragment)+Fixup.getOffset();
diff --git a/lib/Target/X86/README-SSE.txt b/lib/Target/X86/README-SSE.txt
index 624e56f..4011035 100644
--- a/lib/Target/X86/README-SSE.txt
+++ b/lib/Target/X86/README-SSE.txt
@@ -941,3 +941,15 @@ and inversion with an rsqrtss instruction, which computes 1/sqrt faster at the
 cost of reduced accuracy.
 
 //===---------------------------------------------------------------------===//
+
+This function should be matched to haddpd when the appropriate CPU is enabled:
+
+#include <x86intrin.h>
+double f (__m128d p) {
+  return p[0] + p[1];
+}
+
+similarly, v[0]-v[1] should match to hsubpd, and {v[0]-v[1], w[0]-w[1]} should
+turn into hsubpd also.
+
+//===---------------------------------------------------------------------===//
diff --git a/lib/Target/X86/X86.td b/lib/Target/X86/X86.td
index 18e6b7c..8ad0bc0 100644
--- a/lib/Target/X86/X86.td
+++ b/lib/Target/X86/X86.td
@@ -118,8 +118,13 @@ def FeatureBMI     : SubtargetFeature<"bmi", "HasBMI", "true",
                                       "Support BMI instructions">;
 def FeatureBMI2    : SubtargetFeature<"bmi2", "HasBMI2", "true",
                                       "Support BMI2 instructions">;
+def FeatureRTM     : SubtargetFeature<"rtm", "HasRTM", "true",
+                                      "Support RTM instructions">;
 def FeatureLeaForSP : SubtargetFeature<"lea-sp", "UseLeaForSP", "true",
                                      "Use LEA for adjusting the stack pointer">;
+def FeatureSlowDivide : SubtargetFeature<"idiv-to-divb",
+                          "HasSlowDivide", "true",
+                          "Use small divide for positive values less than 256">;
 
 //===----------------------------------------------------------------------===//
 // X86 processors supported.
@@ -159,8 +164,9 @@ def : Proc<"core2",           [FeatureSSSE3, FeatureCMPXCHG16B,
                                FeatureSlowBTMem]>;
 def : Proc<"penryn",          [FeatureSSE41, FeatureCMPXCHG16B,
                                FeatureSlowBTMem]>;
-def : AtomProc<"atom",        [ProcIntelAtom, FeatureSSE3, FeatureCMPXCHG16B,
-                               FeatureMOVBE, FeatureSlowBTMem, FeatureLeaForSP]>;
+def : AtomProc<"atom",        [ProcIntelAtom, FeatureSSSE3, FeatureCMPXCHG16B,
+                               FeatureMOVBE, FeatureSlowBTMem, FeatureLeaForSP,
+                               FeatureSlowDivide]>;
 // "Arrandale" along with corei3 and corei5
 def : Proc<"corei7",          [FeatureSSE42, FeatureCMPXCHG16B,
                                FeatureSlowBTMem, FeatureFastUAMem,
@@ -188,7 +194,8 @@ def : Proc<"core-avx2",       [FeatureAVX2, FeatureCMPXCHG16B, FeaturePOPCNT,
                                FeatureAES, FeaturePCLMUL, FeatureRDRAND,
                                FeatureF16C, FeatureFSGSBase,
                                FeatureMOVBE, FeatureLZCNT, FeatureBMI,
-                               FeatureBMI2, FeatureFMA]>;
+                               FeatureBMI2, FeatureFMA,
+                               FeatureRTM]>;
 
 def : Proc<"k6",              [FeatureMMX]>;
 def : Proc<"k6-2",            [Feature3DNow]>;
@@ -227,6 +234,7 @@ def : Proc<"bdver2",          [FeatureXOP, FeatureFMA4, FeatureCMPXCHG16B,
                                FeatureAES, FeaturePCLMUL,
                                FeatureF16C, FeatureLZCNT,
                                FeaturePOPCNT, FeatureBMI, FeatureFMA]>;
+def : Proc<"geode",           [Feature3DNowA]>;
 
 def : Proc<"winchip-c6",      [FeatureMMX]>;
 def : Proc<"winchip2",        [Feature3DNow]>;
diff --git a/lib/Target/X86/X86AsmPrinter.cpp b/lib/Target/X86/X86AsmPrinter.cpp
index db71e27..fdd7125 100644
--- a/lib/Target/X86/X86AsmPrinter.cpp
+++ b/lib/Target/X86/X86AsmPrinter.cpp
@@ -13,7 +13,6 @@
 //===----------------------------------------------------------------------===//
 
 #include "X86AsmPrinter.h"
-#include "X86MCInstLower.h"
 #include "X86.h"
 #include "X86COFFMachineModuleInfo.h"
 #include "X86MachineFunctionInfo.h"
@@ -206,10 +205,10 @@ void X86AsmPrinter::printSymbolOperand(const MachineOperand &MO,
   }
 }
 
-/// print_pcrel_imm - This is used to print an immediate value that ends up
+/// printPCRelImm - This is used to print an immediate value that ends up
 /// being encoded as a pc-relative value.  These print slightly differently, for
 /// example, a $ is not emitted.
-void X86AsmPrinter::print_pcrel_imm(const MachineInstr *MI, unsigned OpNo,
+void X86AsmPrinter::printPCRelImm(const MachineInstr *MI, unsigned OpNo,
                                     raw_ostream &O) {
   const MachineOperand &MO = MI->getOperand(OpNo);
   switch (MO.getType()) {
@@ -233,15 +232,17 @@ void X86AsmPrinter::print_pcrel_imm(const MachineInstr *MI, unsigned OpNo,
 
 
 void X86AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNo,
-                                 raw_ostream &O, const char *Modifier) {
+                                 raw_ostream &O, const char *Modifier,
+                                 unsigned AsmVariant) {
   const MachineOperand &MO = MI->getOperand(OpNo);
   switch (MO.getType()) {
   default: llvm_unreachable("unknown operand type!");
   case MachineOperand::MO_Register: {
-    O << '%';
+    // FIXME: Enumerating AsmVariant, so we can remove magic number.
+    if (AsmVariant == 0) O << '%';
     unsigned Reg = MO.getReg();
     if (Modifier && strncmp(Modifier, "subreg", strlen("subreg")) == 0) {
-      EVT VT = (strcmp(Modifier+6,"64") == 0) ?
+      MVT::SimpleValueType VT = (strcmp(Modifier+6,"64") == 0) ?
         MVT::i64 : ((strcmp(Modifier+6, "32") == 0) ? MVT::i32 :
                     ((strcmp(Modifier+6,"16") == 0) ? MVT::i16 : MVT::i8));
       Reg = getX86SubSuperRegister(Reg, VT);
@@ -265,46 +266,6 @@ void X86AsmPrinter::printOperand(const MachineInstr *MI, unsigned OpNo,
   }
 }
 
-void X86AsmPrinter::printSSECC(const MachineInstr *MI, unsigned Op,
-                               raw_ostream &O) {
-  unsigned char value = MI->getOperand(Op).getImm();
-  switch (value) {
-  default: llvm_unreachable("Invalid ssecc argument!");
-  case    0: O << "eq"; break;
-  case    1: O << "lt"; break;
-  case    2: O << "le"; break;
-  case    3: O << "unord"; break;
-  case    4: O << "neq"; break;
-  case    5: O << "nlt"; break;
-  case    6: O << "nle"; break;
-  case    7: O << "ord"; break;
-  case    8: O << "eq_uq"; break;
-  case    9: O << "nge"; break;
-  case  0xa: O << "ngt"; break;
-  case  0xb: O << "false"; break;
-  case  0xc: O << "neq_oq"; break;
-  case  0xd: O << "ge"; break;
-  case  0xe: O << "gt"; break;
-  case  0xf: O << "true"; break;
-  case 0x10: O << "eq_os"; break;
-  case 0x11: O << "lt_oq"; break;
-  case 0x12: O << "le_oq"; break;
-  case 0x13: O << "unord_s"; break;
-  case 0x14: O << "neq_us"; break;
-  case 0x15: O << "nlt_uq"; break;
-  case 0x16: O << "nle_uq"; break;
-  case 0x17: O << "ord_s"; break;
-  case 0x18: O << "eq_us"; break;
-  case 0x19: O << "nge_uq"; break;
-  case 0x1a: O << "ngt_uq"; break;
-  case 0x1b: O << "false_os"; break;
-  case 0x1c: O << "neq_os"; break;
-  case 0x1d: O << "ge_oq"; break;
-  case 0x1e: O << "gt_oq"; break;
-  case 0x1f: O << "true_us"; break;
-  }
-}
-
 void X86AsmPrinter::printLeaMemReference(const MachineInstr *MI, unsigned Op,
                                          raw_ostream &O, const char *Modifier) {
   const MachineOperand &BaseReg  = MI->getOperand(Op);
@@ -363,10 +324,51 @@ void X86AsmPrinter::printMemReference(const MachineInstr *MI, unsigned Op,
   printLeaMemReference(MI, Op, O, Modifier);
 }
 
-void X86AsmPrinter::printPICLabel(const MachineInstr *MI, unsigned Op,
-                                  raw_ostream &O) {
-  O << *MF->getPICBaseSymbol() << '\n';
-  O << *MF->getPICBaseSymbol() << ':';
+void X86AsmPrinter::printIntelMemReference(const MachineInstr *MI, unsigned Op,
+                                           raw_ostream &O, const char *Modifier,
+                                           unsigned AsmVariant){
+  const MachineOperand &BaseReg  = MI->getOperand(Op);
+  unsigned ScaleVal = MI->getOperand(Op+1).getImm();
+  const MachineOperand &IndexReg = MI->getOperand(Op+2);
+  const MachineOperand &DispSpec = MI->getOperand(Op+3);
+  const MachineOperand &SegReg   = MI->getOperand(Op+4);
+  
+  // If this has a segment register, print it.
+  if (SegReg.getReg()) {
+    printOperand(MI, Op+4, O, Modifier, AsmVariant);
+    O << ':';
+  }
+  
+  O << '[';
+  
+  bool NeedPlus = false;
+  if (BaseReg.getReg()) {
+    printOperand(MI, Op, O, Modifier, AsmVariant);
+    NeedPlus = true;
+  }
+  
+  if (IndexReg.getReg()) {
+    if (NeedPlus) O << " + ";
+    if (ScaleVal != 1)
+      O << ScaleVal << '*';
+    printOperand(MI, Op+2, O, Modifier, AsmVariant);
+    NeedPlus = true;
+  }
+
+  assert (DispSpec.isImm() && "Displacement is not an immediate!");
+  int64_t DispVal = DispSpec.getImm();
+  if (DispVal || (!IndexReg.getReg() && !BaseReg.getReg())) {
+    if (NeedPlus) {
+      if (DispVal > 0)
+        O << " + ";
+      else {
+        O << " - ";
+        DispVal = -DispVal;
+      }
+    }
+    O << DispVal;
+  }  
+  O << ']';
 }
 
 bool X86AsmPrinter::printAsmMRegister(const MachineOperand &MO, char Mode,
@@ -457,7 +459,7 @@ bool X86AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
       return false;
 
     case 'P': // This is the operand of a call, treat specially.
-      print_pcrel_imm(MI, OpNo, O);
+      printPCRelImm(MI, OpNo, O);
       return false;
 
     case 'n':  // Negate the immediate or print a '-' before the operand.
@@ -471,7 +473,7 @@ bool X86AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
     }
   }
 
-  printOperand(MI, OpNo, O);
+  printOperand(MI, OpNo, O, /*Modifier*/ 0, AsmVariant);
   return false;
 }
 
@@ -479,6 +481,11 @@ bool X86AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI,
                                           unsigned OpNo, unsigned AsmVariant,
                                           const char *ExtraCode,
                                           raw_ostream &O) {
+  if (AsmVariant) {
+    printIntelMemReference(MI, OpNo, O);
+    return false;
+  }
+
   if (ExtraCode && ExtraCode[0]) {
     if (ExtraCode[1] != 0) return true; // Unknown modifier.
 
@@ -680,7 +687,7 @@ void X86AsmPrinter::EmitEndOfAsmFile(Module &M) {
     MachineModuleInfoELF::SymbolListTy Stubs = MMIELF.GetGVStubList();
     if (!Stubs.empty()) {
       OutStreamer.SwitchSection(TLOFELF.getDataRelSection());
-      const TargetData *TD = TM.getTargetData();
+      const DataLayout *TD = TM.getDataLayout();
 
       for (unsigned i = 0, e = Stubs.size(); i != e; ++i) {
         OutStreamer.EmitLabel(Stubs[i].first);
diff --git a/lib/Target/X86/X86AsmPrinter.h b/lib/Target/X86/X86AsmPrinter.h
index 35386cd..61eb14e 100644
--- a/lib/Target/X86/X86AsmPrinter.h
+++ b/lib/Target/X86/X86AsmPrinter.h
@@ -34,47 +34,48 @@ class LLVM_LIBRARY_VISIBILITY X86AsmPrinter : public AsmPrinter {
     Subtarget = &TM.getSubtarget<X86Subtarget>();
   }
 
-  virtual const char *getPassName() const {
+  virtual const char *getPassName() const LLVM_OVERRIDE {
     return "X86 AT&T-Style Assembly Printer";
   }
 
   const X86Subtarget &getSubtarget() const { return *Subtarget; }
 
-  virtual void EmitStartOfAsmFile(Module &M);
+  virtual void EmitStartOfAsmFile(Module &M) LLVM_OVERRIDE;
 
-  virtual void EmitEndOfAsmFile(Module &M);
+  virtual void EmitEndOfAsmFile(Module &M) LLVM_OVERRIDE;
 
-  virtual void EmitInstruction(const MachineInstr *MI);
+  virtual void EmitInstruction(const MachineInstr *MI) LLVM_OVERRIDE;
 
   void printSymbolOperand(const MachineOperand &MO, raw_ostream &O);
 
   // These methods are used by the tablegen'erated instruction printer.
   void printOperand(const MachineInstr *MI, unsigned OpNo, raw_ostream &O,
-                    const char *Modifier = 0);
-  void print_pcrel_imm(const MachineInstr *MI, unsigned OpNo, raw_ostream &O);
+                    const char *Modifier = 0, unsigned AsmVariant = 0);
+  void printPCRelImm(const MachineInstr *MI, unsigned OpNo, raw_ostream &O);
 
   bool printAsmMRegister(const MachineOperand &MO, char Mode, raw_ostream &O);
-  bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
-                       unsigned AsmVariant, const char *ExtraCode,
-                       raw_ostream &OS);
-  bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNo,
-                             unsigned AsmVariant, const char *ExtraCode,
-                             raw_ostream &OS);
-
-  void printMachineInstruction(const MachineInstr *MI);
-  void printSSECC(const MachineInstr *MI, unsigned Op, raw_ostream &O);
+  virtual bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
+                               unsigned AsmVariant, const char *ExtraCode,
+                               raw_ostream &OS) LLVM_OVERRIDE;
+  virtual bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNo,
+                                     unsigned AsmVariant, const char *ExtraCode,
+                                     raw_ostream &OS) LLVM_OVERRIDE;
+
   void printMemReference(const MachineInstr *MI, unsigned Op, raw_ostream &O,
                          const char *Modifier=NULL);
   void printLeaMemReference(const MachineInstr *MI, unsigned Op, raw_ostream &O,
                             const char *Modifier=NULL);
 
-  void printPICLabel(const MachineInstr *MI, unsigned Op, raw_ostream &O);
+  void printIntelMemReference(const MachineInstr *MI, unsigned Op,
+                              raw_ostream &O, const char *Modifier=NULL,
+                              unsigned AsmVariant = 1);
 
-  bool runOnMachineFunction(MachineFunction &F);
+  virtual bool runOnMachineFunction(MachineFunction &F) LLVM_OVERRIDE;
 
   void PrintDebugValueComment(const MachineInstr *MI, raw_ostream &OS);
 
-  MachineLocation getDebugValueLocation(const MachineInstr *MI) const;
+  virtual MachineLocation
+    getDebugValueLocation(const MachineInstr *MI) const LLVM_OVERRIDE;
 };
 
 } // end namespace llvm
diff --git a/lib/Target/X86/X86COFFMachineModuleInfo.h b/lib/Target/X86/X86COFFMachineModuleInfo.h
index 471eb31..a5a8dc1 100644
--- a/lib/Target/X86/X86COFFMachineModuleInfo.h
+++ b/lib/Target/X86/X86COFFMachineModuleInfo.h
@@ -20,7 +20,7 @@
 
 namespace llvm {
   class X86MachineFunctionInfo;
-  class TargetData;
+  class DataLayout;
 
 /// X86COFFMachineModuleInfo - This is a MachineModuleInfoImpl implementation
 /// for X86 COFF targets.
diff --git a/lib/Target/X86/X86CallingConv.td b/lib/Target/X86/X86CallingConv.td
index a6d2709..6786756 100644
--- a/lib/Target/X86/X86CallingConv.td
+++ b/lib/Target/X86/X86CallingConv.td
@@ -88,6 +88,21 @@ def RetCC_X86_32_Fast : CallingConv<[
   CCDelegateTo<RetCC_X86Common>
 ]>;
 
+// Intel_OCL_BI return-value convention.
+def RetCC_Intel_OCL_BI : CallingConv<[
+  // Vector types are returned in XMM0,XMM1,XMMM2 and XMM3.
+  CCIfType<[f32, f64, v4i32, v2i64, v4f32, v2f64],
+            CCAssignToReg<[XMM0,XMM1,XMM2,XMM3]>>,
+
+  // 256-bit FP vectors
+  // No more than 4 registers
+  CCIfType<[v8f32, v4f64, v8i32, v4i64],
+            CCAssignToReg<[YMM0,YMM1,YMM2,YMM3]>>,
+
+  // i32, i64 in the standard way
+  CCDelegateTo<RetCC_X86Common>
+]>;
+
 // X86-64 C return-value convention.
 def RetCC_X86_64_C : CallingConv<[
   // The X86-64 calling convention always returns FP values in XMM0.
@@ -128,6 +143,10 @@ def RetCC_X86_64 : CallingConv<[
 
 // This is the return-value convention used for the entire X86 backend.
 def RetCC_X86 : CallingConv<[
+
+  // Check if this is the Intel OpenCL built-ins calling convention
+  CCIfCC<"CallingConv::Intel_OCL_BI", CCDelegateTo<RetCC_Intel_OCL_BI>>,
+
   CCIfSubtarget<"is64Bit()", CCDelegateTo<RetCC_X86_64>>,
   CCDelegateTo<RetCC_X86_32>
 ]>;
@@ -235,6 +254,29 @@ def CC_X86_Win64_C : CallingConv<[
   CCIfType<[f80], CCAssignToStack<0, 0>>
 ]>;
 
+// X86-64 Intel OpenCL built-ins calling convention.
+def CC_Intel_OCL_BI : CallingConv<[
+  CCIfType<[i32], CCIfSubtarget<"isTargetWin32()", CCAssignToStack<4, 4>>>,
+
+  CCIfType<[i32], CCIfSubtarget<"isTargetWin64()", CCAssignToReg<[ECX, EDX, R8D, R9D]>>>,
+  CCIfType<[i64], CCIfSubtarget<"isTargetWin64()", CCAssignToReg<[RCX, RDX, R8,  R9 ]>>>,
+
+  CCIfType<[i32], CCAssignToReg<[EDI, ESI, EDX, ECX]>>,
+  CCIfType<[i64], CCAssignToReg<[RDI, RSI, RDX, RCX]>>,
+
+ // The SSE vector arguments are passed in XMM registers.
+  CCIfType<[f32, f64, v4i32, v2i64, v4f32, v2f64],
+           CCAssignToReg<[XMM0, XMM1, XMM2, XMM3]>>,
+  
+  // The 256-bit vector arguments are passed in YMM registers.
+  CCIfType<[v8f32, v4f64, v8i32, v4i64],
+                CCAssignToReg<[YMM0, YMM1, YMM2, YMM3]>>,
+  
+  CCIfSubtarget<"isTargetWin64()", CCDelegateTo<CC_X86_Win64_C>>,
+  CCDelegateTo<CC_X86_64_C>
+]>;
+
+
 def CC_X86_64_GHC : CallingConv<[
   // Promote i8/i16/i32 arguments to i64.
   CCIfType<[i8, i16, i32], CCPromoteToType<i64>>,
@@ -324,7 +366,7 @@ def CC_X86_32_FastCall : CallingConv<[
   CCIfNest<CCAssignToReg<[EAX]>>,
 
   // The first 2 integer arguments are passed in ECX/EDX
-  CCIfType<[i32], CCAssignToReg<[ECX, EDX]>>,
+  CCIfInReg<CCIfType<[i32], CCAssignToReg<[ECX, EDX]>>>,
 
   // Otherwise, same as everything else.
   CCDelegateTo<CC_X86_32_Common>
@@ -408,6 +450,7 @@ def CC_X86_64 : CallingConv<[
 
 // This is the argument convention used for the entire X86 backend.
 def CC_X86 : CallingConv<[
+  CCIfCC<"CallingConv::Intel_OCL_BI", CCDelegateTo<CC_Intel_OCL_BI>>,
   CCIfSubtarget<"is64Bit()", CCDelegateTo<CC_X86_64>>,
   CCDelegateTo<CC_X86_32>
 ]>;
@@ -426,3 +469,17 @@ def CSR_64EHRet : CalleeSavedRegs<(add RAX, RDX, CSR_64)>;
 
 def CSR_Win64 : CalleeSavedRegs<(add RBX, RBP, RDI, RSI, R12, R13, R14, R15,
                                      (sequence "XMM%u", 6, 15))>;
+
+
+// Standard C + YMM6-15
+def CSR_Win64_Intel_OCL_BI_AVX : CalleeSavedRegs<(add RBX, RBP, RDI, RSI, R12,
+                                                  R13, R14, R15, 
+                                                  (sequence "YMM%u", 6, 15))>;
+
+//Standard C + XMM 8-15
+def CSR_64_Intel_OCL_BI       : CalleeSavedRegs<(add CSR_64,
+                                                 (sequence "XMM%u", 8, 15))>;
+
+//Standard C + YMM 8-15
+def CSR_64_Intel_OCL_BI_AVX    : CalleeSavedRegs<(add CSR_64,
+                                                  (sequence "YMM%u", 8, 15))>;
diff --git a/lib/Target/X86/X86CodeEmitter.cpp b/lib/Target/X86/X86CodeEmitter.cpp
index d705049..44db563 100644
--- a/lib/Target/X86/X86CodeEmitter.cpp
+++ b/lib/Target/X86/X86CodeEmitter.cpp
@@ -26,7 +26,6 @@
 #include "llvm/CodeGen/MachineInstr.h"
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/Passes.h"
-#include "llvm/Function.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/MC/MCCodeEmitter.h"
 #include "llvm/MC/MCExpr.h"
@@ -43,7 +42,7 @@ namespace {
   template<class CodeEmitter>
   class Emitter : public MachineFunctionPass {
     const X86InstrInfo  *II;
-    const TargetData    *TD;
+    const DataLayout    *TD;
     X86TargetMachine    &TM;
     CodeEmitter         &MCE;
     MachineModuleInfo   *MMI;
@@ -57,7 +56,7 @@ namespace {
       MCE(mce), PICBaseOffset(0), Is64BitMode(false),
       IsPIC(TM.getRelocationModel() == Reloc::PIC_) {}
     Emitter(X86TargetMachine &tm, CodeEmitter &mce,
-            const X86InstrInfo &ii, const TargetData &td, bool is64)
+            const X86InstrInfo &ii, const DataLayout &td, bool is64)
       : MachineFunctionPass(ID), II(&ii), TD(&td), TM(tm),
       MCE(mce), PICBaseOffset(0), Is64BitMode(is64),
       IsPIC(TM.getRelocationModel() == Reloc::PIC_) {}
@@ -110,6 +109,14 @@ namespace {
     void emitMemModRMByte(const MachineInstr &MI,
                           unsigned Op, unsigned RegOpcodeField,
                           intptr_t PCAdj = 0);
+
+    unsigned getX86RegNum(unsigned RegNo) const {
+      const TargetRegisterInfo *TRI = TM.getRegisterInfo();
+      return TRI->getEncodingValue(RegNo) & 0x7;
+    }
+
+    unsigned char getVEXRegisterEncoding(const MachineInstr &MI,
+                                         unsigned OpNum) const;
   };
 
 template<class CodeEmitter>
@@ -129,13 +136,12 @@ bool Emitter<CodeEmitter>::runOnMachineFunction(MachineFunction &MF) {
   MCE.setModuleInfo(MMI);
 
   II = TM.getInstrInfo();
-  TD = TM.getTargetData();
+  TD = TM.getDataLayout();
   Is64BitMode = TM.getSubtarget<X86Subtarget>().is64Bit();
   IsPIC = TM.getRelocationModel() == Reloc::PIC_;
 
   do {
-    DEBUG(dbgs() << "JITTing function '"
-          << MF.getFunction()->getName() << "'\n");
+    DEBUG(dbgs() << "JITTing function '" << MF.getName() << "'\n");
     MCE.startFunction(MF);
     for (MachineFunction::iterator MBB = MF.begin(), E = MF.end();
          MBB != E; ++MBB) {
@@ -365,7 +371,7 @@ inline static unsigned char ModRMByte(unsigned Mod, unsigned RegOpcode,
 template<class CodeEmitter>
 void Emitter<CodeEmitter>::emitRegModRMByte(unsigned ModRMReg,
                                             unsigned RegOpcodeFld){
-  MCE.emitByte(ModRMByte(3, RegOpcodeFld, X86_MC::getX86RegNum(ModRMReg)));
+  MCE.emitByte(ModRMByte(3, RegOpcodeFld, getX86RegNum(ModRMReg)));
 }
 
 template<class CodeEmitter>
@@ -503,7 +509,7 @@ void Emitter<CodeEmitter>::emitMemModRMByte(const MachineInstr &MI,
   // 2-7) and absolute references.
   unsigned BaseRegNo = -1U;
   if (BaseReg != 0 && BaseReg != X86::RIP)
-    BaseRegNo = X86_MC::getX86RegNum(BaseReg);
+    BaseRegNo = getX86RegNum(BaseReg);
 
   if (// The SIB byte must be used if there is an index register.
       IndexReg.getReg() == 0 &&
@@ -579,15 +585,15 @@ void Emitter<CodeEmitter>::emitMemModRMByte(const MachineInstr &MI,
     // Manual 2A, table 2-7. The displacement has already been output.
     unsigned IndexRegNo;
     if (IndexReg.getReg())
-      IndexRegNo = X86_MC::getX86RegNum(IndexReg.getReg());
+      IndexRegNo = getX86RegNum(IndexReg.getReg());
     else // Examples: [ESP+1*<noreg>+4] or [scaled idx]+disp32 (MOD=0,BASE=5)
       IndexRegNo = 4;
     emitSIBByte(SS, IndexRegNo, 5);
   } else {
-    unsigned BaseRegNo = X86_MC::getX86RegNum(BaseReg);
+    unsigned BaseRegNo = getX86RegNum(BaseReg);
     unsigned IndexRegNo;
     if (IndexReg.getReg())
-      IndexRegNo = X86_MC::getX86RegNum(IndexReg.getReg());
+      IndexRegNo = getX86RegNum(IndexReg.getReg());
     else
       IndexRegNo = 4;   // For example [ESP+1*<noreg>+4]
     emitSIBByte(SS, IndexRegNo, BaseRegNo);
@@ -758,10 +764,12 @@ void Emitter<CodeEmitter>::emitOpcodePrefix(uint64_t TSFlags,
 //  VEX.VVVV    => XMM9 => ~9
 //
 // See table 4-35 of Intel AVX Programming Reference for details.
-static unsigned char getVEXRegisterEncoding(const MachineInstr &MI,
-                                            unsigned OpNum) {
+template<class CodeEmitter>
+unsigned char
+Emitter<CodeEmitter>::getVEXRegisterEncoding(const MachineInstr &MI,
+                                             unsigned OpNum) const {
   unsigned SrcReg = MI.getOperand(OpNum).getReg();
-  unsigned SrcRegNum = X86_MC::getX86RegNum(MI.getOperand(OpNum).getReg());
+  unsigned SrcRegNum = getX86RegNum(MI.getOperand(OpNum).getReg());
   if (X86II::isX86_64ExtendedReg(SrcReg))
     SrcRegNum |= 8;
 
@@ -923,17 +931,6 @@ void Emitter<CodeEmitter>::emitVEXOpcodePrefix(uint64_t TSFlags,
   }
 
 
-  // Set the vector length to 256-bit if YMM0-YMM15 is used
-  for (unsigned i = 0; i != MI.getNumOperands(); ++i) {
-    if (!MI.getOperand(i).isReg())
-      continue;
-    if (MI.getOperand(i).isImplicit())
-      continue;
-    unsigned SrcReg = MI.getOperand(i).getReg();
-    if (SrcReg >= X86::YMM0 && SrcReg <= X86::YMM15)
-      VEX_L = 1;
-  }
-
   // Classify VEX_B, VEX_4V, VEX_R, VEX_X
   unsigned NumOps = Desc->getNumOperands();
   unsigned CurOp = 0;
@@ -1248,7 +1245,7 @@ void Emitter<CodeEmitter>::emitInstruction(MachineInstr &MI,
 
   case X86II::AddRegFrm: {
     MCE.emitByte(BaseOpcode +
-                 X86_MC::getX86RegNum(MI.getOperand(CurOp++).getReg()));
+                 getX86RegNum(MI.getOperand(CurOp++).getReg()));
 
     if (CurOp == NumOps)
       break;
@@ -1283,7 +1280,7 @@ void Emitter<CodeEmitter>::emitInstruction(MachineInstr &MI,
   case X86II::MRMDestReg: {
     MCE.emitByte(BaseOpcode);
     emitRegModRMByte(MI.getOperand(CurOp).getReg(),
-                     X86_MC::getX86RegNum(MI.getOperand(CurOp+1).getReg()));
+                     getX86RegNum(MI.getOperand(CurOp+1).getReg()));
     CurOp += 2;
     break;
   }
@@ -1294,7 +1291,7 @@ void Emitter<CodeEmitter>::emitInstruction(MachineInstr &MI,
     if (HasVEX_4V) // Skip 1st src (which is encoded in VEX_VVVV)
       SrcRegNum++;
     emitMemModRMByte(MI, CurOp,
-                X86_MC::getX86RegNum(MI.getOperand(SrcRegNum).getReg()));
+                     getX86RegNum(MI.getOperand(SrcRegNum).getReg()));
     CurOp = SrcRegNum + 1;
     break;
   }
@@ -1310,7 +1307,7 @@ void Emitter<CodeEmitter>::emitInstruction(MachineInstr &MI,
       ++SrcRegNum;
 
     emitRegModRMByte(MI.getOperand(SrcRegNum).getReg(),
-                     X86_MC::getX86RegNum(MI.getOperand(CurOp).getReg()));
+                     getX86RegNum(MI.getOperand(CurOp).getReg()));
     // 2 operands skipped with HasMemOp4, compensate accordingly
     CurOp = HasMemOp4 ? SrcRegNum : SrcRegNum + 1;
     if (HasVEX_4VOp3)
@@ -1332,7 +1329,7 @@ void Emitter<CodeEmitter>::emitInstruction(MachineInstr &MI,
     intptr_t PCAdj = (CurOp + AddrOperands + 1 != NumOps) ?
       X86II::getSizeOfImm(Desc->TSFlags) : 0;
     emitMemModRMByte(MI, FirstMemOp,
-                     X86_MC::getX86RegNum(MI.getOperand(CurOp).getReg()),PCAdj);
+                     getX86RegNum(MI.getOperand(CurOp).getReg()),PCAdj);
     CurOp += AddrOperands + 1;
     if (HasVEX_4VOp3)
       ++CurOp;
@@ -1422,7 +1419,7 @@ void Emitter<CodeEmitter>::emitInstruction(MachineInstr &MI,
     MCE.emitByte(BaseOpcode);
     // Duplicate register, used by things like MOV8r0 (aka xor reg,reg).
     emitRegModRMByte(MI.getOperand(CurOp).getReg(),
-                     X86_MC::getX86RegNum(MI.getOperand(CurOp).getReg()));
+                     getX86RegNum(MI.getOperand(CurOp).getReg()));
     ++CurOp;
     break;
 
@@ -1455,7 +1452,7 @@ void Emitter<CodeEmitter>::emitInstruction(MachineInstr &MI,
       const MachineOperand &MO = MI.getOperand(HasMemOp4 ? MemOp4_I8IMMOperand
                                                          : CurOp);
       ++CurOp;
-      unsigned RegNum = X86_MC::getX86RegNum(MO.getReg()) << 4;
+      unsigned RegNum = getX86RegNum(MO.getReg()) << 4;
       if (X86II::isX86_64ExtendedReg(MO.getReg()))
         RegNum |= 1 << 7;
       // If there is an additional 5th operand it must be an immediate, which
diff --git a/lib/Target/X86/X86ELFWriterInfo.cpp b/lib/Target/X86/X86ELFWriterInfo.cpp
deleted file mode 100644
index c1a49a7..0000000
--- a/lib/Target/X86/X86ELFWriterInfo.cpp
+++ /dev/null
@@ -1,147 +0,0 @@
-//===-- X86ELFWriterInfo.cpp - ELF Writer Info for the X86 backend --------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements ELF writer information for the X86 backend.
-//
-//===----------------------------------------------------------------------===//
-
-#include "X86ELFWriterInfo.h"
-#include "X86Relocations.h"
-#include "llvm/Function.h"
-#include "llvm/Support/ELF.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Target/TargetData.h"
-#include "llvm/Target/TargetMachine.h"
-
-using namespace llvm;
-
-//===----------------------------------------------------------------------===//
-//  Implementation of the X86ELFWriterInfo class
-//===----------------------------------------------------------------------===//
-
-X86ELFWriterInfo::X86ELFWriterInfo(bool is64Bit_, bool isLittleEndian_)
-  : TargetELFWriterInfo(is64Bit_, isLittleEndian_) {
-    EMachine = is64Bit ? EM_X86_64 : EM_386;
-  }
-
-X86ELFWriterInfo::~X86ELFWriterInfo() {}
-
-unsigned X86ELFWriterInfo::getRelocationType(unsigned MachineRelTy) const {
-  if (is64Bit) {
-    switch(MachineRelTy) {
-    case X86::reloc_pcrel_word:
-      return ELF::R_X86_64_PC32;
-    case X86::reloc_absolute_word:
-      return ELF::R_X86_64_32;
-    case X86::reloc_absolute_word_sext:
-      return ELF::R_X86_64_32S;
-    case X86::reloc_absolute_dword:
-      return ELF::R_X86_64_64;
-    case X86::reloc_picrel_word:
-    default:
-      llvm_unreachable("unknown x86_64 machine relocation type");
-    }
-  } else {
-    switch(MachineRelTy) {
-    case X86::reloc_pcrel_word:
-      return ELF::R_386_PC32;
-    case X86::reloc_absolute_word:
-      return ELF::R_386_32;
-    case X86::reloc_absolute_word_sext:
-    case X86::reloc_absolute_dword:
-    case X86::reloc_picrel_word:
-    default:
-      llvm_unreachable("unknown x86 machine relocation type");
-    }
-  }
-}
-
-long int X86ELFWriterInfo::getDefaultAddendForRelTy(unsigned RelTy,
-                                                    long int Modifier) const {
-  if (is64Bit) {
-    switch(RelTy) {
-    case ELF::R_X86_64_PC32: return Modifier - 4;
-    case ELF::R_X86_64_32:
-    case ELF::R_X86_64_32S:
-    case ELF::R_X86_64_64:
-      return Modifier;
-    default:
-      llvm_unreachable("unknown x86_64 relocation type");
-    }
-  } else {
-    switch(RelTy) {
-    case ELF::R_386_PC32: return Modifier - 4;
-    case ELF::R_386_32: return Modifier;
-    default:
-      llvm_unreachable("unknown x86 relocation type");
-    }
-  }
-}
-
-unsigned X86ELFWriterInfo::getRelocationTySize(unsigned RelTy) const {
-  if (is64Bit) {
-    switch(RelTy) {
-    case ELF::R_X86_64_PC32:
-    case ELF::R_X86_64_32:
-    case ELF::R_X86_64_32S:
-        return 32;
-    case ELF::R_X86_64_64:
-        return 64;
-    default:
-      llvm_unreachable("unknown x86_64 relocation type");
-    }
-  } else {
-    switch(RelTy) {
-    case ELF::R_386_PC32:
-    case ELF::R_386_32:
-        return 32;
-    default:
-      llvm_unreachable("unknown x86 relocation type");
-    }
-  }
-}
-
-bool X86ELFWriterInfo::isPCRelativeRel(unsigned RelTy) const {
-  if (is64Bit) {
-    switch(RelTy) {
-    case ELF::R_X86_64_PC32:
-        return true;
-    case ELF::R_X86_64_32:
-    case ELF::R_X86_64_32S:
-    case ELF::R_X86_64_64:
-        return false;
-    default:
-      llvm_unreachable("unknown x86_64 relocation type");
-    }
-  } else {
-    switch(RelTy) {
-    case ELF::R_386_PC32:
-        return true;
-    case ELF::R_386_32:
-        return false;
-    default:
-      llvm_unreachable("unknown x86 relocation type");
-    }
-  }
-}
-
-unsigned X86ELFWriterInfo::getAbsoluteLabelMachineRelTy() const {
-  return is64Bit ?
-    X86::reloc_absolute_dword : X86::reloc_absolute_word;
-}
-
-long int X86ELFWriterInfo::computeRelocation(unsigned SymOffset,
-                                             unsigned RelOffset,
-                                             unsigned RelTy) const {
-
-  if (RelTy == ELF::R_X86_64_PC32 || RelTy == ELF::R_386_PC32)
-    return SymOffset - (RelOffset + 4);
-
-  llvm_unreachable("computeRelocation unknown for this relocation type");
-}
diff --git a/lib/Target/X86/X86ELFWriterInfo.h b/lib/Target/X86/X86ELFWriterInfo.h
deleted file mode 100644
index a45b5bb..0000000
--- a/lib/Target/X86/X86ELFWriterInfo.h
+++ /dev/null
@@ -1,59 +0,0 @@
-//===-- X86ELFWriterInfo.h - ELF Writer Info for X86 ------------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements ELF writer information for the X86 backend.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef X86_ELF_WRITER_INFO_H
-#define X86_ELF_WRITER_INFO_H
-
-#include "llvm/Target/TargetELFWriterInfo.h"
-
-namespace llvm {
-
-  class X86ELFWriterInfo : public TargetELFWriterInfo {
-
-  public:
-    X86ELFWriterInfo(bool is64Bit_, bool isLittleEndian_);
-    virtual ~X86ELFWriterInfo();
-
-    /// getRelocationType - Returns the target specific ELF Relocation type.
-    /// 'MachineRelTy' contains the object code independent relocation type
-    virtual unsigned getRelocationType(unsigned MachineRelTy) const;
-
-    /// hasRelocationAddend - True if the target uses an addend in the
-    /// ELF relocation entry.
-    virtual bool hasRelocationAddend() const { return is64Bit ? true : false; }
-
-    /// getDefaultAddendForRelTy - Gets the default addend value for a
-    /// relocation entry based on the target ELF relocation type.
-    virtual long int getDefaultAddendForRelTy(unsigned RelTy,
-                                              long int Modifier = 0) const;
-
-    /// getRelTySize - Returns the size of relocatable field in bits
-    virtual unsigned getRelocationTySize(unsigned RelTy) const;
-
-    /// isPCRelativeRel - True if the relocation type is pc relative
-    virtual bool isPCRelativeRel(unsigned RelTy) const;
-
-    /// getJumpTableRelocationTy - Returns the machine relocation type used
-    /// to reference a jumptable.
-    virtual unsigned getAbsoluteLabelMachineRelTy() const;
-
-    /// computeRelocation - Some relocatable fields could be relocated
-    /// directly, avoiding the relocation symbol emission, compute the
-    /// final relocation value for this symbol.
-    virtual long int computeRelocation(unsigned SymOffset, unsigned RelOffset,
-                                       unsigned RelTy) const;
-  };
-
-} // end llvm namespace
-
-#endif // X86_ELF_WRITER_INFO_H
diff --git a/lib/Target/X86/X86FastISel.cpp b/lib/Target/X86/X86FastISel.cpp
index e5952aa..d4627c7 100644
--- a/lib/Target/X86/X86FastISel.cpp
+++ b/lib/Target/X86/X86FastISel.cpp
@@ -45,9 +45,9 @@ class X86FastISel : public FastISel {
   /// make the right decision when generating code for different targets.
   const X86Subtarget *Subtarget;
 
-  /// StackPtr - Register used as the stack pointer.
+  /// RegInfo - X86 register info.
   ///
-  unsigned StackPtr;
+  const X86RegisterInfo *RegInfo;
 
   /// X86ScalarSSEf32, X86ScalarSSEf64 - Select between SSE or x87
   /// floating point ops.
@@ -61,9 +61,9 @@ public:
                        const TargetLibraryInfo *libInfo)
     : FastISel(funcInfo, libInfo) {
     Subtarget = &TM.getSubtarget<X86Subtarget>();
-    StackPtr = Subtarget->is64Bit() ? X86::RSP : X86::ESP;
     X86ScalarSSEf64 = Subtarget->hasSSE2();
     X86ScalarSSEf32 = Subtarget->hasSSE1();
+    RegInfo = static_cast<const X86RegisterInfo*>(TM.getRegisterInfo());
   }
 
   virtual bool TargetSelectInstruction(const Instruction *I);
@@ -710,6 +710,8 @@ bool X86FastISel::X86SelectStore(const Instruction *I) {
 bool X86FastISel::X86SelectRet(const Instruction *I) {
   const ReturnInst *Ret = cast<ReturnInst>(I);
   const Function &F = *I->getParent()->getParent();
+  const X86MachineFunctionInfo *X86MFInfo =
+      FuncInfo.MF->getInfo<X86MachineFunctionInfo>();
 
   if (!FuncInfo.CanLowerReturn)
     return false;
@@ -724,8 +726,7 @@ bool X86FastISel::X86SelectRet(const Instruction *I) {
     return false;
 
   // Don't handle popping bytes on return for now.
-  if (FuncInfo.MF->getInfo<X86MachineFunctionInfo>()
-        ->getBytesToPopOnReturn() != 0)
+  if (X86MFInfo->getBytesToPopOnReturn() != 0)
     return 0;
 
   // fastcc with -tailcallopt is intended to provide a guaranteed
@@ -809,6 +810,19 @@ bool X86FastISel::X86SelectRet(const Instruction *I) {
     MRI.addLiveOut(VA.getLocReg());
   }
 
+  // The x86-64 ABI for returning structs by value requires that we copy
+  // the sret argument into %rax for the return. We saved the argument into
+  // a virtual register in the entry block, so now we copy the value out
+  // and into %rax.
+  if (Subtarget->is64Bit() && F.hasStructRetAttr()) {
+    unsigned Reg = X86MFInfo->getSRetReturnReg();
+    assert(Reg &&
+           "SRetReturnReg should have been set in LowerFormalArguments()!");
+    BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(TargetOpcode::COPY),
+            X86::RAX).addReg(Reg);
+    MRI.addLiveOut(X86::RAX);
+  }
+
   // Now emit the RET.
   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DL, TII.get(X86::RET));
   return true;
@@ -1527,9 +1541,9 @@ static unsigned computeBytesPoppedByCallee(const X86Subtarget &Subtarget,
   CallingConv::ID CC = CS.getCallingConv();
   if (CC == CallingConv::Fast || CC == CallingConv::GHC)
     return 0;
-  if (!CS.paramHasAttr(1, Attribute::StructRet))
+  if (!CS.paramHasAttr(1, Attributes::StructRet))
     return 0;
-  if (CS.paramHasAttr(1, Attribute::InReg))
+  if (CS.paramHasAttr(1, Attributes::InReg))
     return 0;
   return 4;
 }
@@ -1608,12 +1622,12 @@ bool X86FastISel::DoSelectCall(const Instruction *I, const char *MemIntName) {
     Value *ArgVal = *i;
     ISD::ArgFlagsTy Flags;
     unsigned AttrInd = i - CS.arg_begin() + 1;
-    if (CS.paramHasAttr(AttrInd, Attribute::SExt))
+    if (CS.paramHasAttr(AttrInd, Attributes::SExt))
       Flags.setSExt();
-    if (CS.paramHasAttr(AttrInd, Attribute::ZExt))
+    if (CS.paramHasAttr(AttrInd, Attributes::ZExt))
       Flags.setZExt();
 
-    if (CS.paramHasAttr(AttrInd, Attribute::ByVal)) {
+    if (CS.paramHasAttr(AttrInd, Attributes::ByVal)) {
       PointerType *Ty = cast<PointerType>(ArgVal->getType());
       Type *ElementTy = Ty->getElementType();
       unsigned FrameSize = TD.getTypeAllocSize(ElementTy);
@@ -1627,9 +1641,9 @@ bool X86FastISel::DoSelectCall(const Instruction *I, const char *MemIntName) {
         return false;
     }
 
-    if (CS.paramHasAttr(AttrInd, Attribute::InReg))
+    if (CS.paramHasAttr(AttrInd, Attributes::InReg))
       Flags.setInReg();
-    if (CS.paramHasAttr(AttrInd, Attribute::Nest))
+    if (CS.paramHasAttr(AttrInd, Attributes::Nest))
       Flags.setNest();
 
     // If this is an i1/i8/i16 argument, promote to i32 to avoid an extra
@@ -1771,7 +1785,7 @@ bool X86FastISel::DoSelectCall(const Instruction *I, const char *MemIntName) {
     } else {
       unsigned LocMemOffset = VA.getLocMemOffset();
       X86AddressMode AM;
-      AM.Base.Reg = StackPtr;
+      AM.Base.Reg = RegInfo->getStackRegister();
       AM.Disp = LocMemOffset;
       const Value *ArgVal = ArgVals[VA.getValNo()];
       ISD::ArgFlagsTy Flags = ArgFlags[VA.getValNo()];
@@ -1897,11 +1911,11 @@ bool X86FastISel::DoSelectCall(const Instruction *I, const char *MemIntName) {
       ISD::InputArg MyFlags;
       MyFlags.VT = RegisterVT.getSimpleVT();
       MyFlags.Used = !CS.getInstruction()->use_empty();
-      if (CS.paramHasAttr(0, Attribute::SExt))
+      if (CS.paramHasAttr(0, Attributes::SExt))
         MyFlags.Flags.setSExt();
-      if (CS.paramHasAttr(0, Attribute::ZExt))
+      if (CS.paramHasAttr(0, Attributes::ZExt))
         MyFlags.Flags.setZExt();
-      if (CS.paramHasAttr(0, Attribute::InReg))
+      if (CS.paramHasAttr(0, Attributes::InReg))
         MyFlags.Flags.setInReg();
       Ins.push_back(MyFlags);
     }
@@ -2014,13 +2028,17 @@ X86FastISel::TargetSelectInstruction(const Instruction *I)  {
 unsigned X86FastISel::TargetMaterializeConstant(const Constant *C) {
   MVT VT;
   if (!isTypeLegal(C->getType(), VT))
-    return false;
+    return 0;
+
+  // Can't handle alternate code models yet.
+  if (TM.getCodeModel() != CodeModel::Small)
+    return 0;
 
   // Get opcode and regclass of the output for the given load instruction.
   unsigned Opc = 0;
   const TargetRegisterClass *RC = NULL;
   switch (VT.SimpleTy) {
-  default: return false;
+  default: return 0;
   case MVT::i8:
     Opc = X86::MOV8rm;
     RC  = &X86::GR8RegClass;
@@ -2058,7 +2076,7 @@ unsigned X86FastISel::TargetMaterializeConstant(const Constant *C) {
     break;
   case MVT::f80:
     // No f80 support yet.
-    return false;
+    return 0;
   }
 
   // Materialize addresses with LEA instructions.
diff --git a/lib/Target/X86/X86FloatingPoint.cpp b/lib/Target/X86/X86FloatingPoint.cpp
index 955c75a..791f598 100644
--- a/lib/Target/X86/X86FloatingPoint.cpp
+++ b/lib/Target/X86/X86FloatingPoint.cpp
@@ -171,6 +171,7 @@ namespace {
     // Shuffle live registers to match the expectations of successor blocks.
     void finishBlockStack();
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
     void dumpStack() const {
       dbgs() << "Stack contents:";
       for (unsigned i = 0; i != StackTop; ++i) {
@@ -181,6 +182,7 @@ namespace {
         dbgs() << ", ST" << i << " in FP" << unsigned(PendingST[i]);
       dbgs() << "\n";
     }
+#endif
 
     /// getSlot - Return the stack slot number a particular register number is
     /// in.
@@ -575,8 +577,8 @@ namespace {
     friend bool operator<(const TableEntry &TE, unsigned V) {
       return TE.from < V;
     }
-    friend bool LLVM_ATTRIBUTE_USED operator<(unsigned V,
-                                              const TableEntry &TE) {
+    friend bool LLVM_ATTRIBUTE_UNUSED operator<(unsigned V,
+                                                const TableEntry &TE) {
       return V < TE.from;
     }
   };
diff --git a/lib/Target/X86/X86FrameLowering.cpp b/lib/Target/X86/X86FrameLowering.cpp
index 2238688..369589d 100644
--- a/lib/Target/X86/X86FrameLowering.cpp
+++ b/lib/Target/X86/X86FrameLowering.cpp
@@ -25,7 +25,7 @@
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/MC/MCAsmInfo.h"
 #include "llvm/MC/MCSymbol.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetOptions.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/ADT/SmallSet.h"
@@ -313,11 +313,11 @@ void X86FrameLowering::emitCalleeSavedFrameMoves(MachineFunction &MF,
   if (CSI.empty()) return;
 
   std::vector<MachineMove> &Moves = MMI.getFrameMoves();
-  const TargetData *TD = TM.getTargetData();
+  const X86RegisterInfo *RegInfo = TM.getRegisterInfo();
   bool HasFP = hasFP(MF);
 
   // Calculate amount of bytes used for return address storing.
-  int stackGrowth = -TD->getPointerSize();
+  int stackGrowth = -RegInfo->getSlotSize();
 
   // FIXME: This is dirty hack. The code itself is pretty mess right now.
   // It should be rewritten from scratch and generalized sometimes.
@@ -674,7 +674,7 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF) const {
   // function, and use up to 128 bytes of stack space, don't have a frame
   // pointer, calls, or dynamic alloca then we do not need to adjust the
   // stack pointer (we fit in the Red Zone).
-  if (Is64Bit && !Fn->hasFnAttr(Attribute::NoRedZone) &&
+  if (Is64Bit && !Fn->getFnAttributes().hasAttribute(Attributes::NoRedZone) &&
       !RegInfo->needsStackRealignment(MF) &&
       !MFI->hasVarSizedObjects() &&                     // No dynamic alloca.
       !MFI->adjustsStack() &&                           // No calls.
@@ -715,9 +715,8 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF) const {
   //        ELSE                        => DW_CFA_offset_extended
 
   std::vector<MachineMove> &Moves = MMI.getFrameMoves();
-  const TargetData *TD = MF.getTarget().getTargetData();
   uint64_t NumBytes = 0;
-  int stackGrowth = -TD->getPointerSize();
+  int stackGrowth = -SlotSize;
 
   if (HasFP) {
     // Calculate required stack adjustment.
@@ -836,8 +835,6 @@ void X86FrameLowering::emitPrologue(MachineFunction &MF) const {
     MI->getOperand(3).setIsDead();
   }
 
-  DL = MBB.findDebugLoc(MBBI);
-
   // If there is an SUB32ri of ESP immediately before this instruction, merge
   // the two. This can be the case when tail call elimination is enabled and
   // the callee has more arguments then the caller.
diff --git a/lib/Target/X86/X86ISelDAGToDAG.cpp b/lib/Target/X86/X86ISelDAGToDAG.cpp
index 27195b4..99f5574 100644
--- a/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -100,6 +100,7 @@ namespace {
       Base_Reg = Reg;
     }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
     void dump() {
       dbgs() << "X86ISelAddressMode " << this << '\n';
       dbgs() << "Base_Reg ";
@@ -133,6 +134,7 @@ namespace {
         dbgs() << "nul";
       dbgs() << " JT" << JT << " Align" << Align << '\n';
     }
+#endif
   };
 }
 
@@ -189,7 +191,6 @@ namespace {
     SDNode *Select(SDNode *N);
     SDNode *SelectGather(SDNode *N, unsigned Opc);
     SDNode *SelectAtomic64(SDNode *Node, unsigned Opc);
-    SDNode *SelectAtomicLoadAdd(SDNode *Node, EVT NVT);
     SDNode *SelectAtomicLoadArith(SDNode *Node, EVT NVT);
 
     bool FoldOffsetIntoAddress(uint64_t Offset, X86ISelAddressMode &AM);
@@ -244,13 +245,15 @@ namespace {
       else if (AM.CP)
         Disp = CurDAG->getTargetConstantPool(AM.CP, MVT::i32,
                                              AM.Align, AM.Disp, AM.SymbolFlags);
-      else if (AM.ES)
+      else if (AM.ES) {
+        assert(!AM.Disp && "Non-zero displacement is ignored with ES.");
         Disp = CurDAG->getTargetExternalSymbol(AM.ES, MVT::i32, AM.SymbolFlags);
-      else if (AM.JT != -1)
+      } else if (AM.JT != -1) {
+        assert(!AM.Disp && "Non-zero displacement is ignored with JT.");
         Disp = CurDAG->getTargetJumpTable(AM.JT, MVT::i32, AM.SymbolFlags);
-      else if (AM.BlockAddr)
-        Disp = CurDAG->getBlockAddress(AM.BlockAddr, MVT::i32,
-                                       true, AM.SymbolFlags);
+      } else if (AM.BlockAddr)
+        Disp = CurDAG->getTargetBlockAddress(AM.BlockAddr, MVT::i32, AM.Disp,
+                                             AM.SymbolFlags);
       else
         Disp = CurDAG->getTargetConstant(AM.Disp, MVT::i32);
 
@@ -359,7 +362,7 @@ X86DAGToDAGISel::IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const {
 /// MoveBelowCallOrigChain - Replace the original chain operand of the call with
 /// load's chain operand and move load below the call's chain operand.
 static void MoveBelowOrigChain(SelectionDAG *CurDAG, SDValue Load,
-                                  SDValue Call, SDValue OrigChain) {
+                               SDValue Call, SDValue OrigChain) {
   SmallVector<SDValue, 8> Ops;
   SDValue Chain = OrigChain.getOperand(0);
   if (Chain.getNode() == Load.getNode())
@@ -383,11 +386,13 @@ static void MoveBelowOrigChain(SelectionDAG *CurDAG, SDValue Load,
   CurDAG->UpdateNodeOperands(OrigChain.getNode(), &Ops[0], Ops.size());
   CurDAG->UpdateNodeOperands(Load.getNode(), Call.getOperand(0),
                              Load.getOperand(1), Load.getOperand(2));
+
+  unsigned NumOps = Call.getNode()->getNumOperands();
   Ops.clear();
   Ops.push_back(SDValue(Load.getNode(), 1));
-  for (unsigned i = 1, e = Call.getNode()->getNumOperands(); i != e; ++i)
+  for (unsigned i = 1, e = NumOps; i != e; ++i)
     Ops.push_back(Call.getOperand(i));
-  CurDAG->UpdateNodeOperands(Call.getNode(), &Ops[0], Ops.size());
+  CurDAG->UpdateNodeOperands(Call.getNode(), &Ops[0], NumOps);
 }
 
 /// isCalleeLoad - Return true if call address is a load and it can be
@@ -396,6 +401,10 @@ static void MoveBelowOrigChain(SelectionDAG *CurDAG, SDValue Load,
 /// In the case of a tail call, there isn't a callseq node between the call
 /// chain and the load.
 static bool isCalleeLoad(SDValue Callee, SDValue &Chain, bool HasCallSeq) {
+  // The transformation is somewhat dangerous if the call's chain was glued to
+  // the call. After MoveBelowOrigChain the load is moved between the call and
+  // the chain, this can create a cycle if the load is not folded. So it is
+  // *really* important that we are sure the load will be folded.
   if (Callee.getNode() == Chain.getNode() || !Callee.hasOneUse())
     return false;
   LoadSDNode *LD = dyn_cast<LoadSDNode>(Callee.getNode());
@@ -425,7 +434,8 @@ static bool isCalleeLoad(SDValue Callee, SDValue &Chain, bool HasCallSeq) {
 
 void X86DAGToDAGISel::PreprocessISelDAG() {
   // OptForSize is used in pattern predicates that isel is matching.
-  OptForSize = MF->getFunction()->hasFnAttr(Attribute::OptimizeForSize);
+  OptForSize = MF->getFunction()->getFnAttributes().
+    hasAttribute(Attributes::OptimizeForSize);
 
   for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(),
        E = CurDAG->allnodes_end(); I != E; ) {
@@ -433,7 +443,10 @@ void X86DAGToDAGISel::PreprocessISelDAG() {
 
     if (OptLevel != CodeGenOpt::None &&
         (N->getOpcode() == X86ISD::CALL ||
-         N->getOpcode() == X86ISD::TC_RETURN)) {
+         (N->getOpcode() == X86ISD::TC_RETURN &&
+          // Only does this if load can be foled into TC_RETURN.
+          (Subtarget->is64Bit() ||
+           getTargetMachine().getRelocationModel() != Reloc::PIC_)))) {
       /// Also try moving call address load from outside callseq_start to just
       /// before the call to allow it to be folded.
       ///
@@ -652,10 +665,16 @@ bool X86DAGToDAGISel::MatchWrapper(SDValue N, X86ISelAddressMode &AM) {
     } else if (JumpTableSDNode *J = dyn_cast<JumpTableSDNode>(N0)) {
       AM.JT = J->getIndex();
       AM.SymbolFlags = J->getTargetFlags();
-    } else {
-      AM.BlockAddr = cast<BlockAddressSDNode>(N0)->getBlockAddress();
-      AM.SymbolFlags = cast<BlockAddressSDNode>(N0)->getTargetFlags();
-    }
+    } else if (BlockAddressSDNode *BA = dyn_cast<BlockAddressSDNode>(N0)) {
+      X86ISelAddressMode Backup = AM;
+      AM.BlockAddr = BA->getBlockAddress();
+      AM.SymbolFlags = BA->getTargetFlags();
+      if (FoldOffsetIntoAddress(BA->getOffset(), AM)) {
+        AM = Backup;
+        return true;
+      }
+    } else
+      llvm_unreachable("Unhandled symbol reference node.");
 
     if (N.getOpcode() == X86ISD::WrapperRIP)
       AM.setBaseReg(CurDAG->getRegister(X86::RIP, MVT::i64));
@@ -684,10 +703,12 @@ bool X86DAGToDAGISel::MatchWrapper(SDValue N, X86ISelAddressMode &AM) {
     } else if (JumpTableSDNode *J = dyn_cast<JumpTableSDNode>(N0)) {
       AM.JT = J->getIndex();
       AM.SymbolFlags = J->getTargetFlags();
-    } else {
-      AM.BlockAddr = cast<BlockAddressSDNode>(N0)->getBlockAddress();
-      AM.SymbolFlags = cast<BlockAddressSDNode>(N0)->getTargetFlags();
-    }
+    } else if (BlockAddressSDNode *BA = dyn_cast<BlockAddressSDNode>(N0)) {
+      AM.BlockAddr = BA->getBlockAddress();
+      AM.Disp += BA->getOffset();
+      AM.SymbolFlags = BA->getTargetFlags();
+    } else
+      llvm_unreachable("Unhandled symbol reference node.");
     return false;
   }
 
@@ -1011,7 +1032,7 @@ bool X86DAGToDAGISel::MatchAddressRecursively(SDValue N, X86ISelAddressMode &AM,
           AM.IndexReg = ShVal.getNode()->getOperand(0);
           ConstantSDNode *AddVal =
             cast<ConstantSDNode>(ShVal.getNode()->getOperand(1));
-          uint64_t Disp = AddVal->getSExtValue() << Val;
+          uint64_t Disp = (uint64_t)AddVal->getSExtValue() << Val;
           if (!FoldOffsetIntoAddress(Disp, AM))
             return false;
         }
@@ -1281,7 +1302,9 @@ bool X86DAGToDAGISel::SelectAddr(SDNode *Parent, SDValue N, SDValue &Base,
       // that are not a MemSDNode, and thus don't have proper addrspace info.
       Parent->getOpcode() != ISD::INTRINSIC_W_CHAIN && // unaligned loads, fixme
       Parent->getOpcode() != ISD::INTRINSIC_VOID && // nontemporal stores
-      Parent->getOpcode() != X86ISD::TLSCALL) { // Fixme
+      Parent->getOpcode() != X86ISD::TLSCALL && // Fixme
+      Parent->getOpcode() != X86ISD::EH_SJLJ_SETJMP && // setjmp
+      Parent->getOpcode() != X86ISD::EH_SJLJ_LONGJMP) { // longjmp
     unsigned AddrSpace =
       cast<MemSDNode>(Parent)->getPointerInfo().getAddrSpace();
     // AddrSpace 256 -> GS, 257 -> FS.
@@ -1468,6 +1491,7 @@ SDNode *X86DAGToDAGISel::SelectAtomic64(SDNode *Node, unsigned Opc) {
   SDValue In1 = Node->getOperand(1);
   SDValue In2L = Node->getOperand(2);
   SDValue In2H = Node->getOperand(3);
+
   SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
   if (!SelectAddr(Node, In1, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4))
     return NULL;
@@ -1481,159 +1505,13 @@ SDNode *X86DAGToDAGISel::SelectAtomic64(SDNode *Node, unsigned Opc) {
   return ResNode;
 }
 
-// FIXME: Figure out some way to unify this with the 'or' and other code
-// below.
-SDNode *X86DAGToDAGISel::SelectAtomicLoadAdd(SDNode *Node, EVT NVT) {
-  if (Node->hasAnyUseOfValue(0))
-    return 0;
-
-  // Optimize common patterns for __sync_add_and_fetch and
-  // __sync_sub_and_fetch where the result is not used. This allows us
-  // to use "lock" version of add, sub, inc, dec instructions.
-  // FIXME: Do not use special instructions but instead add the "lock"
-  // prefix to the target node somehow. The extra information will then be
-  // transferred to machine instruction and it denotes the prefix.
-  SDValue Chain = Node->getOperand(0);
-  SDValue Ptr = Node->getOperand(1);
-  SDValue Val = Node->getOperand(2);
-  SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
-  if (!SelectAddr(Node, Ptr, Tmp0, Tmp1, Tmp2, Tmp3, Tmp4))
-    return 0;
-
-  bool isInc = false, isDec = false, isSub = false, isCN = false;
-  ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val);
-  if (CN && CN->getSExtValue() == (int32_t)CN->getSExtValue()) {
-    isCN = true;
-    int64_t CNVal = CN->getSExtValue();
-    if (CNVal == 1)
-      isInc = true;
-    else if (CNVal == -1)
-      isDec = true;
-    else if (CNVal >= 0)
-      Val = CurDAG->getTargetConstant(CNVal, NVT);
-    else {
-      isSub = true;
-      Val = CurDAG->getTargetConstant(-CNVal, NVT);
-    }
-  } else if (Val.hasOneUse() &&
-             Val.getOpcode() == ISD::SUB &&
-             X86::isZeroNode(Val.getOperand(0))) {
-    isSub = true;
-    Val = Val.getOperand(1);
-  }
-
-  DebugLoc dl = Node->getDebugLoc();
-  unsigned Opc = 0;
-  switch (NVT.getSimpleVT().SimpleTy) {
-  default: return 0;
-  case MVT::i8:
-    if (isInc)
-      Opc = X86::LOCK_INC8m;
-    else if (isDec)
-      Opc = X86::LOCK_DEC8m;
-    else if (isSub) {
-      if (isCN)
-        Opc = X86::LOCK_SUB8mi;
-      else
-        Opc = X86::LOCK_SUB8mr;
-    } else {
-      if (isCN)
-        Opc = X86::LOCK_ADD8mi;
-      else
-        Opc = X86::LOCK_ADD8mr;
-    }
-    break;
-  case MVT::i16:
-    if (isInc)
-      Opc = X86::LOCK_INC16m;
-    else if (isDec)
-      Opc = X86::LOCK_DEC16m;
-    else if (isSub) {
-      if (isCN) {
-        if (immSext8(Val.getNode()))
-          Opc = X86::LOCK_SUB16mi8;
-        else
-          Opc = X86::LOCK_SUB16mi;
-      } else
-        Opc = X86::LOCK_SUB16mr;
-    } else {
-      if (isCN) {
-        if (immSext8(Val.getNode()))
-          Opc = X86::LOCK_ADD16mi8;
-        else
-          Opc = X86::LOCK_ADD16mi;
-      } else
-        Opc = X86::LOCK_ADD16mr;
-    }
-    break;
-  case MVT::i32:
-    if (isInc)
-      Opc = X86::LOCK_INC32m;
-    else if (isDec)
-      Opc = X86::LOCK_DEC32m;
-    else if (isSub) {
-      if (isCN) {
-        if (immSext8(Val.getNode()))
-          Opc = X86::LOCK_SUB32mi8;
-        else
-          Opc = X86::LOCK_SUB32mi;
-      } else
-        Opc = X86::LOCK_SUB32mr;
-    } else {
-      if (isCN) {
-        if (immSext8(Val.getNode()))
-          Opc = X86::LOCK_ADD32mi8;
-        else
-          Opc = X86::LOCK_ADD32mi;
-      } else
-        Opc = X86::LOCK_ADD32mr;
-    }
-    break;
-  case MVT::i64:
-    if (isInc)
-      Opc = X86::LOCK_INC64m;
-    else if (isDec)
-      Opc = X86::LOCK_DEC64m;
-    else if (isSub) {
-      Opc = X86::LOCK_SUB64mr;
-      if (isCN) {
-        if (immSext8(Val.getNode()))
-          Opc = X86::LOCK_SUB64mi8;
-        else if (i64immSExt32(Val.getNode()))
-          Opc = X86::LOCK_SUB64mi32;
-      }
-    } else {
-      Opc = X86::LOCK_ADD64mr;
-      if (isCN) {
-        if (immSext8(Val.getNode()))
-          Opc = X86::LOCK_ADD64mi8;
-        else if (i64immSExt32(Val.getNode()))
-          Opc = X86::LOCK_ADD64mi32;
-      }
-    }
-    break;
-  }
-
-  SDValue Undef = SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
-                                                 dl, NVT), 0);
-  MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
-  MemOp[0] = cast<MemSDNode>(Node)->getMemOperand();
-  if (isInc || isDec) {
-    SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain };
-    SDValue Ret = SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops, 6), 0);
-    cast<MachineSDNode>(Ret)->setMemRefs(MemOp, MemOp + 1);
-    SDValue RetVals[] = { Undef, Ret };
-    return CurDAG->getMergeValues(RetVals, 2, dl).getNode();
-  } else {
-    SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Val, Chain };
-    SDValue Ret = SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops, 7), 0);
-    cast<MachineSDNode>(Ret)->setMemRefs(MemOp, MemOp + 1);
-    SDValue RetVals[] = { Undef, Ret };
-    return CurDAG->getMergeValues(RetVals, 2, dl).getNode();
-  }
-}
-
+/// Atomic opcode table
+///
 enum AtomicOpc {
+  ADD,
+  SUB,
+  INC,
+  DEC,
   OR,
   AND,
   XOR,
@@ -1657,6 +1535,58 @@ enum AtomicSz {
 
 static const uint16_t AtomicOpcTbl[AtomicOpcEnd][AtomicSzEnd] = {
   {
+    X86::LOCK_ADD8mi,
+    X86::LOCK_ADD8mr,
+    X86::LOCK_ADD16mi8,
+    X86::LOCK_ADD16mi,
+    X86::LOCK_ADD16mr,
+    X86::LOCK_ADD32mi8,
+    X86::LOCK_ADD32mi,
+    X86::LOCK_ADD32mr,
+    X86::LOCK_ADD64mi8,
+    X86::LOCK_ADD64mi32,
+    X86::LOCK_ADD64mr,
+  },
+  {
+    X86::LOCK_SUB8mi,
+    X86::LOCK_SUB8mr,
+    X86::LOCK_SUB16mi8,
+    X86::LOCK_SUB16mi,
+    X86::LOCK_SUB16mr,
+    X86::LOCK_SUB32mi8,
+    X86::LOCK_SUB32mi,
+    X86::LOCK_SUB32mr,
+    X86::LOCK_SUB64mi8,
+    X86::LOCK_SUB64mi32,
+    X86::LOCK_SUB64mr,
+  },
+  {
+    0,
+    X86::LOCK_INC8m,
+    0,
+    0,
+    X86::LOCK_INC16m,
+    0,
+    0,
+    X86::LOCK_INC32m,
+    0,
+    0,
+    X86::LOCK_INC64m,
+  },
+  {
+    0,
+    X86::LOCK_DEC8m,
+    0,
+    0,
+    X86::LOCK_DEC16m,
+    0,
+    0,
+    X86::LOCK_DEC32m,
+    0,
+    0,
+    X86::LOCK_DEC64m,
+  },
+  {
     X86::LOCK_OR8mi,
     X86::LOCK_OR8mr,
     X86::LOCK_OR16mi8,
@@ -1667,7 +1597,7 @@ static const uint16_t AtomicOpcTbl[AtomicOpcEnd][AtomicSzEnd] = {
     X86::LOCK_OR32mr,
     X86::LOCK_OR64mi8,
     X86::LOCK_OR64mi32,
-    X86::LOCK_OR64mr
+    X86::LOCK_OR64mr,
   },
   {
     X86::LOCK_AND8mi,
@@ -1680,7 +1610,7 @@ static const uint16_t AtomicOpcTbl[AtomicOpcEnd][AtomicSzEnd] = {
     X86::LOCK_AND32mr,
     X86::LOCK_AND64mi8,
     X86::LOCK_AND64mi32,
-    X86::LOCK_AND64mr
+    X86::LOCK_AND64mr,
   },
   {
     X86::LOCK_XOR8mi,
@@ -1693,18 +1623,74 @@ static const uint16_t AtomicOpcTbl[AtomicOpcEnd][AtomicSzEnd] = {
     X86::LOCK_XOR32mr,
     X86::LOCK_XOR64mi8,
     X86::LOCK_XOR64mi32,
-    X86::LOCK_XOR64mr
+    X86::LOCK_XOR64mr,
   }
 };
 
+// Return the target constant operand for atomic-load-op and do simple
+// translations, such as from atomic-load-add to lock-sub. The return value is
+// one of the following 3 cases:
+// + target-constant, the operand could be supported as a target constant.
+// + empty, the operand is not needed any more with the new op selected.
+// + non-empty, otherwise.
+static SDValue getAtomicLoadArithTargetConstant(SelectionDAG *CurDAG,
+                                                DebugLoc dl,
+                                                enum AtomicOpc &Op, EVT NVT,
+                                                SDValue Val) {
+  if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val)) {
+    int64_t CNVal = CN->getSExtValue();
+    // Quit if not 32-bit imm.
+    if ((int32_t)CNVal != CNVal)
+      return Val;
+    // For atomic-load-add, we could do some optimizations.
+    if (Op == ADD) {
+      // Translate to INC/DEC if ADD by 1 or -1.
+      if ((CNVal == 1) || (CNVal == -1)) {
+        Op = (CNVal == 1) ? INC : DEC;
+        // No more constant operand after being translated into INC/DEC.
+        return SDValue();
+      }
+      // Translate to SUB if ADD by negative value.
+      if (CNVal < 0) {
+        Op = SUB;
+        CNVal = -CNVal;
+      }
+    }
+    return CurDAG->getTargetConstant(CNVal, NVT);
+  }
+
+  // If the value operand is single-used, try to optimize it.
+  if (Op == ADD && Val.hasOneUse()) {
+    // Translate (atomic-load-add ptr (sub 0 x)) back to (lock-sub x).
+    if (Val.getOpcode() == ISD::SUB && X86::isZeroNode(Val.getOperand(0))) {
+      Op = SUB;
+      return Val.getOperand(1);
+    }
+    // A special case for i16, which needs truncating as, in most cases, it's
+    // promoted to i32. We will translate
+    // (atomic-load-add (truncate (sub 0 x))) to (lock-sub (EXTRACT_SUBREG x))
+    if (Val.getOpcode() == ISD::TRUNCATE && NVT == MVT::i16 &&
+        Val.getOperand(0).getOpcode() == ISD::SUB &&
+        X86::isZeroNode(Val.getOperand(0).getOperand(0))) {
+      Op = SUB;
+      Val = Val.getOperand(0);
+      return CurDAG->getTargetExtractSubreg(X86::sub_16bit, dl, NVT,
+                                            Val.getOperand(1));
+    }
+  }
+
+  return Val;
+}
+
 SDNode *X86DAGToDAGISel::SelectAtomicLoadArith(SDNode *Node, EVT NVT) {
   if (Node->hasAnyUseOfValue(0))
     return 0;
 
+  DebugLoc dl = Node->getDebugLoc();
+
   // Optimize common patterns for __sync_or_and_fetch and similar arith
   // operations where the result is not used. This allows us to use the "lock"
   // version of the arithmetic instruction.
-  // FIXME: Same as for 'add' and 'sub', try to merge those down here.
   SDValue Chain = Node->getOperand(0);
   SDValue Ptr = Node->getOperand(1);
   SDValue Val = Node->getOperand(2);
@@ -1715,6 +1701,8 @@ SDNode *X86DAGToDAGISel::SelectAtomicLoadArith(SDNode *Node, EVT NVT) {
   // Which index into the table.
   enum AtomicOpc Op;
   switch (Node->getOpcode()) {
+    default:
+      return 0;
     case ISD::ATOMIC_LOAD_OR:
       Op = OR;
       break;
@@ -1724,16 +1712,14 @@ SDNode *X86DAGToDAGISel::SelectAtomicLoadArith(SDNode *Node, EVT NVT) {
     case ISD::ATOMIC_LOAD_XOR:
       Op = XOR;
       break;
-    default:
-      return 0;
-  }
-
-  bool isCN = false;
-  ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val);
-  if (CN && (int32_t)CN->getSExtValue() == CN->getSExtValue()) {
-    isCN = true;
-    Val = CurDAG->getTargetConstant(CN->getSExtValue(), NVT);
+    case ISD::ATOMIC_LOAD_ADD:
+      Op = ADD;
+      break;
   }
+  
+  Val = getAtomicLoadArithTargetConstant(CurDAG, dl, Op, NVT, Val);
+  bool isUnOp = !Val.getNode();
+  bool isCN = Val.getNode() && (Val.getOpcode() == ISD::TargetConstant);
 
   unsigned Opc = 0;
   switch (NVT.getSimpleVT().SimpleTy) {
@@ -1775,13 +1761,20 @@ SDNode *X86DAGToDAGISel::SelectAtomicLoadArith(SDNode *Node, EVT NVT) {
 
   assert(Opc != 0 && "Invalid arith lock transform!");
 
-  DebugLoc dl = Node->getDebugLoc();
+  SDValue Ret;
   SDValue Undef = SDValue(CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF,
                                                  dl, NVT), 0);
   MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1);
   MemOp[0] = cast<MemSDNode>(Node)->getMemOperand();
-  SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Val, Chain };
-  SDValue Ret = SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops, 7), 0);
+  if (isUnOp) {
+    SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Chain };
+    Ret = SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops,
+                                         array_lengthof(Ops)), 0);
+  } else {
+    SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, Val, Chain };
+    Ret = SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops,
+                                         array_lengthof(Ops)), 0);
+  }
   cast<MachineSDNode>(Ret)->setMemRefs(MemOp, MemOp + 1);
   SDValue RetVals[] = { Undef, Ret };
   return CurDAG->getMergeValues(RetVals, 2, dl).getNode();
@@ -2059,6 +2052,10 @@ SDNode *X86DAGToDAGISel::Select(SDNode *Node) {
   case X86ISD::ATOMSUB64_DAG:
   case X86ISD::ATOMNAND64_DAG:
   case X86ISD::ATOMAND64_DAG:
+  case X86ISD::ATOMMAX64_DAG:
+  case X86ISD::ATOMMIN64_DAG:
+  case X86ISD::ATOMUMAX64_DAG:
+  case X86ISD::ATOMUMIN64_DAG:
   case X86ISD::ATOMSWAP64_DAG: {
     unsigned Opc;
     switch (Opcode) {
@@ -2069,6 +2066,10 @@ SDNode *X86DAGToDAGISel::Select(SDNode *Node) {
     case X86ISD::ATOMSUB64_DAG:  Opc = X86::ATOMSUB6432;  break;
     case X86ISD::ATOMNAND64_DAG: Opc = X86::ATOMNAND6432; break;
     case X86ISD::ATOMAND64_DAG:  Opc = X86::ATOMAND6432;  break;
+    case X86ISD::ATOMMAX64_DAG:  Opc = X86::ATOMMAX6432;  break;
+    case X86ISD::ATOMMIN64_DAG:  Opc = X86::ATOMMIN6432;  break;
+    case X86ISD::ATOMUMAX64_DAG: Opc = X86::ATOMUMAX6432; break;
+    case X86ISD::ATOMUMIN64_DAG: Opc = X86::ATOMUMIN6432; break;
     case X86ISD::ATOMSWAP64_DAG: Opc = X86::ATOMSWAP6432; break;
     }
     SDNode *RetVal = SelectAtomic64(Node, Opc);
@@ -2077,15 +2078,10 @@ SDNode *X86DAGToDAGISel::Select(SDNode *Node) {
     break;
   }
 
-  case ISD::ATOMIC_LOAD_ADD: {
-    SDNode *RetVal = SelectAtomicLoadAdd(Node, NVT);
-    if (RetVal)
-      return RetVal;
-    break;
-  }
   case ISD::ATOMIC_LOAD_XOR:
   case ISD::ATOMIC_LOAD_AND:
-  case ISD::ATOMIC_LOAD_OR: {
+  case ISD::ATOMIC_LOAD_OR:
+  case ISD::ATOMIC_LOAD_ADD: {
     SDNode *RetVal = SelectAtomicLoadArith(Node, NVT);
     if (RetVal)
       return RetVal;
@@ -2116,7 +2112,8 @@ SDNode *X86DAGToDAGISel::Select(SDNode *Node) {
 
     // Make sure that we don't change the operation by removing bits.
     // This only matters for OR and XOR, AND is unaffected.
-    if (Opcode != ISD::AND && ((Val >> ShlVal) << ShlVal) != Val)
+    uint64_t RemovedBitsMask = (1ULL << ShlVal) - 1;
+    if (Opcode != ISD::AND && (Val & RemovedBitsMask) != 0)
       break;
 
     unsigned ShlOp, Op;
@@ -2199,13 +2196,16 @@ SDNode *X86DAGToDAGISel::Select(SDNode *Node) {
     SDValue N1 = Node->getOperand(1);
 
     bool isSigned = Opcode == ISD::SMUL_LOHI;
+    bool hasBMI2 = Subtarget->hasBMI2();
     if (!isSigned) {
       switch (NVT.getSimpleVT().SimpleTy) {
       default: llvm_unreachable("Unsupported VT!");
       case MVT::i8:  Opc = X86::MUL8r;  MOpc = X86::MUL8m;  break;
       case MVT::i16: Opc = X86::MUL16r; MOpc = X86::MUL16m; break;
-      case MVT::i32: Opc = X86::MUL32r; MOpc = X86::MUL32m; break;
-      case MVT::i64: Opc = X86::MUL64r; MOpc = X86::MUL64m; break;
+      case MVT::i32: Opc = hasBMI2 ? X86::MULX32rr : X86::MUL32r;
+                     MOpc = hasBMI2 ? X86::MULX32rm : X86::MUL32m; break;
+      case MVT::i64: Opc = hasBMI2 ? X86::MULX64rr : X86::MUL64r;
+                     MOpc = hasBMI2 ? X86::MULX64rm : X86::MUL64m; break;
       }
     } else {
       switch (NVT.getSimpleVT().SimpleTy) {
@@ -2217,13 +2217,31 @@ SDNode *X86DAGToDAGISel::Select(SDNode *Node) {
       }
     }
 
-    unsigned LoReg, HiReg;
-    switch (NVT.getSimpleVT().SimpleTy) {
-    default: llvm_unreachable("Unsupported VT!");
-    case MVT::i8:  LoReg = X86::AL;  HiReg = X86::AH;  break;
-    case MVT::i16: LoReg = X86::AX;  HiReg = X86::DX;  break;
-    case MVT::i32: LoReg = X86::EAX; HiReg = X86::EDX; break;
-    case MVT::i64: LoReg = X86::RAX; HiReg = X86::RDX; break;
+    unsigned SrcReg, LoReg, HiReg;
+    switch (Opc) {
+    default: llvm_unreachable("Unknown MUL opcode!");
+    case X86::IMUL8r:
+    case X86::MUL8r:
+      SrcReg = LoReg = X86::AL; HiReg = X86::AH;
+      break;
+    case X86::IMUL16r:
+    case X86::MUL16r:
+      SrcReg = LoReg = X86::AX; HiReg = X86::DX;
+      break;
+    case X86::IMUL32r:
+    case X86::MUL32r:
+      SrcReg = LoReg = X86::EAX; HiReg = X86::EDX;
+      break;
+    case X86::IMUL64r:
+    case X86::MUL64r:
+      SrcReg = LoReg = X86::RAX; HiReg = X86::RDX;
+      break;
+    case X86::MULX32rr:
+      SrcReg = X86::EDX; LoReg = HiReg = 0;
+      break;
+    case X86::MULX64rr:
+      SrcReg = X86::RDX; LoReg = HiReg = 0;
+      break;
     }
 
     SDValue Tmp0, Tmp1, Tmp2, Tmp3, Tmp4;
@@ -2235,22 +2253,47 @@ SDNode *X86DAGToDAGISel::Select(SDNode *Node) {
         std::swap(N0, N1);
     }
 
-    SDValue InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, LoReg,
+    SDValue InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, SrcReg,
                                           N0, SDValue()).getValue(1);
+    SDValue ResHi, ResLo;
 
     if (foldedLoad) {
+      SDValue Chain;
       SDValue Ops[] = { Tmp0, Tmp1, Tmp2, Tmp3, Tmp4, N1.getOperand(0),
                         InFlag };
-      SDNode *CNode =
-        CurDAG->getMachineNode(MOpc, dl, MVT::Other, MVT::Glue, Ops,
-                               array_lengthof(Ops));
-      InFlag = SDValue(CNode, 1);
+      if (MOpc == X86::MULX32rm || MOpc == X86::MULX64rm) {
+        SDVTList VTs = CurDAG->getVTList(NVT, NVT, MVT::Other, MVT::Glue);
+        SDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops,
+                                               array_lengthof(Ops));
+        ResHi = SDValue(CNode, 0);
+        ResLo = SDValue(CNode, 1);
+        Chain = SDValue(CNode, 2);
+        InFlag = SDValue(CNode, 3);
+      } else {
+        SDVTList VTs = CurDAG->getVTList(MVT::Other, MVT::Glue);
+        SDNode *CNode = CurDAG->getMachineNode(MOpc, dl, VTs, Ops,
+                                               array_lengthof(Ops));
+        Chain = SDValue(CNode, 0);
+        InFlag = SDValue(CNode, 1);
+      }
 
       // Update the chain.
-      ReplaceUses(N1.getValue(1), SDValue(CNode, 0));
+      ReplaceUses(N1.getValue(1), Chain);
     } else {
-      SDNode *CNode = CurDAG->getMachineNode(Opc, dl, MVT::Glue, N1, InFlag);
-      InFlag = SDValue(CNode, 0);
+      SDValue Ops[] = { N1, InFlag };
+      if (Opc == X86::MULX32rr || Opc == X86::MULX64rr) {
+        SDVTList VTs = CurDAG->getVTList(NVT, NVT, MVT::Glue);
+        SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops,
+                                               array_lengthof(Ops));
+        ResHi = SDValue(CNode, 0);
+        ResLo = SDValue(CNode, 1);
+        InFlag = SDValue(CNode, 2);
+      } else {
+        SDVTList VTs = CurDAG->getVTList(MVT::Glue);
+        SDNode *CNode = CurDAG->getMachineNode(Opc, dl, VTs, Ops,
+                                               array_lengthof(Ops));
+        InFlag = SDValue(CNode, 0);
+      }
     }
 
     // Prevent use of AH in a REX instruction by referencing AX instead.
@@ -2275,19 +2318,25 @@ SDNode *X86DAGToDAGISel::Select(SDNode *Node) {
     }
     // Copy the low half of the result, if it is needed.
     if (!SDValue(Node, 0).use_empty()) {
-      SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
-                                              LoReg, NVT, InFlag);
-      InFlag = Result.getValue(2);
-      ReplaceUses(SDValue(Node, 0), Result);
-      DEBUG(dbgs() << "=> "; Result.getNode()->dump(CurDAG); dbgs() << '\n');
+      if (ResLo.getNode() == 0) {
+        assert(LoReg && "Register for low half is not defined!");
+        ResLo = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, LoReg, NVT,
+                                       InFlag);
+        InFlag = ResLo.getValue(2);
+      }
+      ReplaceUses(SDValue(Node, 0), ResLo);
+      DEBUG(dbgs() << "=> "; ResLo.getNode()->dump(CurDAG); dbgs() << '\n');
     }
     // Copy the high half of the result, if it is needed.
     if (!SDValue(Node, 1).use_empty()) {
-      SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
-                                              HiReg, NVT, InFlag);
-      InFlag = Result.getValue(2);
-      ReplaceUses(SDValue(Node, 1), Result);
-      DEBUG(dbgs() << "=> "; Result.getNode()->dump(CurDAG); dbgs() << '\n');
+      if (ResHi.getNode() == 0) {
+        assert(HiReg && "Register for high half is not defined!");
+        ResHi = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl, HiReg, NVT,
+                                       InFlag);
+        InFlag = ResHi.getValue(2);
+      }
+      ReplaceUses(SDValue(Node, 1), ResHi);
+      DEBUG(dbgs() << "=> "; ResHi.getNode()->dump(CurDAG); dbgs() << '\n');
     }
 
     return NULL;
@@ -2488,7 +2537,13 @@ SDNode *X86DAGToDAGISel::Select(SDNode *Node) {
                                                         MVT::i8, Reg);
 
         // Emit a testb.
-        return CurDAG->getMachineNode(X86::TEST8ri, dl, MVT::i32, Subreg, Imm);
+        SDNode *NewNode = CurDAG->getMachineNode(X86::TEST8ri, dl, MVT::i32,
+                                                 Subreg, Imm);
+        // Replace SUB|CMP with TEST, since SUB has two outputs while TEST has
+        // one, do not call ReplaceAllUsesWith.
+        ReplaceUses(SDValue(Node, (Opcode == X86ISD::SUB ? 1 : 0)),
+                    SDValue(NewNode, 0));
+        return NULL;
       }
 
       // For example, "testl %eax, $2048" to "testb %ah, $8".
@@ -2519,8 +2574,13 @@ SDNode *X86DAGToDAGISel::Select(SDNode *Node) {
         // Emit a testb.  The EXTRACT_SUBREG becomes a COPY that can only
         // target GR8_NOREX registers, so make sure the register class is
         // forced.
-        return CurDAG->getMachineNode(X86::TEST8ri_NOREX, dl, MVT::i32,
-                                      Subreg, ShiftedImm);
+        SDNode *NewNode = CurDAG->getMachineNode(X86::TEST8ri_NOREX, dl,
+                                                 MVT::i32, Subreg, ShiftedImm);
+        // Replace SUB|CMP with TEST, since SUB has two outputs while TEST has
+        // one, do not call ReplaceAllUsesWith.
+        ReplaceUses(SDValue(Node, (Opcode == X86ISD::SUB ? 1 : 0)),
+                    SDValue(NewNode, 0));
+        return NULL;
       }
 
       // For example, "testl %eax, $32776" to "testw %ax, $32776".
@@ -2536,7 +2596,13 @@ SDNode *X86DAGToDAGISel::Select(SDNode *Node) {
                                                         MVT::i16, Reg);
 
         // Emit a testw.
-        return CurDAG->getMachineNode(X86::TEST16ri, dl, MVT::i32, Subreg, Imm);
+        SDNode *NewNode = CurDAG->getMachineNode(X86::TEST16ri, dl, MVT::i32,
+                                                 Subreg, Imm);
+        // Replace SUB|CMP with TEST, since SUB has two outputs while TEST has
+        // one, do not call ReplaceAllUsesWith.
+        ReplaceUses(SDValue(Node, (Opcode == X86ISD::SUB ? 1 : 0)),
+                    SDValue(NewNode, 0));
+        return NULL;
       }
 
       // For example, "testq %rax, $268468232" to "testl %eax, $268468232".
@@ -2552,7 +2618,13 @@ SDNode *X86DAGToDAGISel::Select(SDNode *Node) {
                                                         MVT::i32, Reg);
 
         // Emit a testl.
-        return CurDAG->getMachineNode(X86::TEST32ri, dl, MVT::i32, Subreg, Imm);
+        SDNode *NewNode = CurDAG->getMachineNode(X86::TEST32ri, dl, MVT::i32,
+                                                 Subreg, Imm);
+        // Replace SUB|CMP with TEST, since SUB has two outputs while TEST has
+        // one, do not call ReplaceAllUsesWith.
+        ReplaceUses(SDValue(Node, (Opcode == X86ISD::SUB ? 1 : 0)),
+                    SDValue(NewNode, 0));
+        return NULL;
       }
     }
     break;
@@ -2607,85 +2679,6 @@ SDNode *X86DAGToDAGISel::Select(SDNode *Node) {
 
     return Result;
   }
-
-  // FIXME: Custom handling because TableGen doesn't support multiple implicit
-  // defs in an instruction pattern
-  case X86ISD::PCMPESTRI: {
-    SDValue N0 = Node->getOperand(0);
-    SDValue N1 = Node->getOperand(1);
-    SDValue N2 = Node->getOperand(2);
-    SDValue N3 = Node->getOperand(3);
-    SDValue N4 = Node->getOperand(4);
-
-    // Make sure last argument is a constant
-    ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(N4);
-    if (!Cst)
-      break;
-
-    uint64_t Imm = Cst->getZExtValue();
-
-    SDValue InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl,
-                                          X86::EAX, N1, SDValue()).getValue(1);
-    InFlag = CurDAG->getCopyToReg(CurDAG->getEntryNode(), dl, X86::EDX,
-                                  N3, InFlag).getValue(1);
-
-    SDValue Ops[] = { N0, N2, getI8Imm(Imm), InFlag };
-    unsigned Opc = Subtarget->hasAVX() ? X86::VPCMPESTRIrr :
-                                         X86::PCMPESTRIrr;
-    InFlag = SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Glue, Ops,
-                                            array_lengthof(Ops)), 0);
-
-    if (!SDValue(Node, 0).use_empty()) {
-      SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
-                                              X86::ECX, NVT, InFlag);
-      InFlag = Result.getValue(2);
-      ReplaceUses(SDValue(Node, 0), Result);
-    }
-    if (!SDValue(Node, 1).use_empty()) {
-      SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
-                                              X86::EFLAGS, NVT, InFlag);
-      InFlag = Result.getValue(2);
-      ReplaceUses(SDValue(Node, 1), Result);
-    }
-
-    return NULL;
-  }
-
-  // FIXME: Custom handling because TableGen doesn't support multiple implicit
-  // defs in an instruction pattern
-  case X86ISD::PCMPISTRI: {
-    SDValue N0 = Node->getOperand(0);
-    SDValue N1 = Node->getOperand(1);
-    SDValue N2 = Node->getOperand(2);
-
-    // Make sure last argument is a constant
-    ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(N2);
-    if (!Cst)
-      break;
-
-    uint64_t Imm = Cst->getZExtValue();
-
-    SDValue Ops[] = { N0, N1, getI8Imm(Imm) };
-    unsigned Opc = Subtarget->hasAVX() ? X86::VPCMPISTRIrr :
-                                         X86::PCMPISTRIrr;
-    SDValue InFlag = SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Glue, Ops,
-                                                    array_lengthof(Ops)), 0);
-
-    if (!SDValue(Node, 0).use_empty()) {
-      SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
-                                              X86::ECX, NVT, InFlag);
-      InFlag = Result.getValue(2);
-      ReplaceUses(SDValue(Node, 0), Result);
-    }
-    if (!SDValue(Node, 1).use_empty()) {
-      SDValue Result = CurDAG->getCopyFromReg(CurDAG->getEntryNode(), dl,
-                                              X86::EFLAGS, NVT, InFlag);
-      InFlag = Result.getValue(2);
-      ReplaceUses(SDValue(Node, 1), Result);
-    }
-
-    return NULL;
-  }
   }
 
   SDNode *ResNode = SelectCode(Node);
diff --git a/lib/Target/X86/X86ISelLowering.cpp b/lib/Target/X86/X86ISelLowering.cpp
index c77355f..b35fb51 100644
--- a/lib/Target/X86/X86ISelLowering.cpp
+++ b/lib/Target/X86/X86ISelLowering.cpp
@@ -85,7 +85,7 @@ static SDValue Extract128BitVector(SDValue Vec, unsigned IdxVal,
   unsigned NormalizedIdxVal = (((IdxVal * ElVT.getSizeInBits()) / 128)
                                * ElemsPerChunk);
 
-  SDValue VecIdx = DAG.getConstant(NormalizedIdxVal, MVT::i32);
+  SDValue VecIdx = DAG.getIntPtrConstant(NormalizedIdxVal);
   SDValue Result = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, ResultVT, Vec,
                                VecIdx);
 
@@ -118,7 +118,7 @@ static SDValue Insert128BitVector(SDValue Result, SDValue Vec,
   unsigned NormalizedIdxVal = (((IdxVal * ElVT.getSizeInBits())/128)
                                * ElemsPerChunk);
 
-  SDValue VecIdx = DAG.getConstant(NormalizedIdxVal, MVT::i32);
+  SDValue VecIdx = DAG.getIntPtrConstant(NormalizedIdxVal);
   return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResultVT, Result, Vec,
                      VecIdx);
 }
@@ -158,10 +158,9 @@ X86TargetLowering::X86TargetLowering(X86TargetMachine &TM)
   Subtarget = &TM.getSubtarget<X86Subtarget>();
   X86ScalarSSEf64 = Subtarget->hasSSE2();
   X86ScalarSSEf32 = Subtarget->hasSSE1();
-  X86StackPtr = Subtarget->is64Bit() ? X86::RSP : X86::ESP;
 
   RegInfo = TM.getRegisterInfo();
-  TD = getTargetData();
+  TD = getDataLayout();
 
   // Set up the TargetLowering object.
   static const MVT IntVTs[] = { MVT::i8, MVT::i16, MVT::i32, MVT::i64 };
@@ -180,7 +179,11 @@ X86TargetLowering::X86TargetLowering(X86TargetMachine &TM)
     setSchedulingPreference(Sched::ILP);
   else
     setSchedulingPreference(Sched::RegPressure);
-  setStackPointerRegisterToSaveRestore(X86StackPtr);
+  setStackPointerRegisterToSaveRestore(RegInfo->getStackRegister());
+
+  // Bypass i32 with i8 on Atom when compiling with O2
+  if (Subtarget->hasSlowDivide() && TM.getOptLevel() >= CodeGenOpt::Default)
+    addBypassSlowDiv(32, 8);
 
   if (Subtarget->isTargetWindows() && !Subtarget->isTargetCygMing()) {
     // Setup Windows compiler runtime calls.
@@ -453,6 +456,14 @@ X86TargetLowering::X86TargetLowering(X86TargetMachine &TM)
     setOperationAction(ISD::SETCC         , MVT::i64  , Custom);
   }
   setOperationAction(ISD::EH_RETURN       , MVT::Other, Custom);
+  // NOTE: EH_SJLJ_SETJMP/_LONGJMP supported here is NOT intened to support
+  // SjLj exception handling but a light-weight setjmp/longjmp replacement to
+  // support continuation, user-level threading, and etc.. As a result, no
+  // other SjLj exception interfaces are implemented and please don't build
+  // your own exception handling based on them.
+  // LLVM/Clang supports zero-cost DWARF exception handling.
+  setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom);
+  setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom);
 
   // Darwin ABI issue.
   setOperationAction(ISD::ConstantPool    , MVT::i32  , Custom);
@@ -510,6 +521,10 @@ X86TargetLowering::X86TargetLowering(X86TargetMachine &TM)
     setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i64, Custom);
     setOperationAction(ISD::ATOMIC_LOAD_NAND, MVT::i64, Custom);
     setOperationAction(ISD::ATOMIC_SWAP, MVT::i64, Custom);
+    setOperationAction(ISD::ATOMIC_LOAD_MAX, MVT::i64, Custom);
+    setOperationAction(ISD::ATOMIC_LOAD_MIN, MVT::i64, Custom);
+    setOperationAction(ISD::ATOMIC_LOAD_UMAX, MVT::i64, Custom);
+    setOperationAction(ISD::ATOMIC_LOAD_UMIN, MVT::i64, Custom);
   }
 
   if (Subtarget->hasCmpxchg16b()) {
@@ -541,6 +556,7 @@ X86TargetLowering::X86TargetLowering(X86TargetMachine &TM)
   setOperationAction(ISD::ADJUST_TRAMPOLINE, MVT::Other, Custom);
 
   setOperationAction(ISD::TRAP, MVT::Other, Legal);
+  setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
 
   // VASTART needs to be custom lowered to use the VarArgsFrameIndex
   setOperationAction(ISD::VASTART           , MVT::Other, Custom);
@@ -643,7 +659,9 @@ X86TargetLowering::X86TargetLowering(X86TargetMachine &TM)
     setOperationAction(ISD::FCOPYSIGN, MVT::f32, Expand);
 
     if (!TM.Options.UnsafeFPMath) {
+      setOperationAction(ISD::FSIN           , MVT::f32  , Expand);
       setOperationAction(ISD::FSIN           , MVT::f64  , Expand);
+      setOperationAction(ISD::FCOS           , MVT::f32  , Expand);
       setOperationAction(ISD::FCOS           , MVT::f64  , Expand);
     }
     addLegalFPImmediate(APFloat(+0.0)); // FLD0
@@ -735,6 +753,7 @@ X86TargetLowering::X86TargetLowering(X86TargetMachine &TM)
     setOperationAction(ISD::FPOWI, (MVT::SimpleValueType)VT, Expand);
     setOperationAction(ISD::FSQRT, (MVT::SimpleValueType)VT, Expand);
     setOperationAction(ISD::FCOPYSIGN, (MVT::SimpleValueType)VT, Expand);
+    setOperationAction(ISD::FFLOOR, (MVT::SimpleValueType)VT, Expand);
     setOperationAction(ISD::SMUL_LOHI, (MVT::SimpleValueType)VT, Expand);
     setOperationAction(ISD::UMUL_LOHI, (MVT::SimpleValueType)VT, Expand);
     setOperationAction(ISD::SDIVREM, (MVT::SimpleValueType)VT, Expand);
@@ -824,6 +843,7 @@ X86TargetLowering::X86TargetLowering(X86TargetMachine &TM)
     setOperationAction(ISD::FDIV,               MVT::v4f32, Legal);
     setOperationAction(ISD::FSQRT,              MVT::v4f32, Legal);
     setOperationAction(ISD::FNEG,               MVT::v4f32, Custom);
+    setOperationAction(ISD::FABS,               MVT::v4f32, Custom);
     setOperationAction(ISD::LOAD,               MVT::v4f32, Legal);
     setOperationAction(ISD::BUILD_VECTOR,       MVT::v4f32, Custom);
     setOperationAction(ISD::VECTOR_SHUFFLE,     MVT::v4f32, Custom);
@@ -857,6 +877,7 @@ X86TargetLowering::X86TargetLowering(X86TargetMachine &TM)
     setOperationAction(ISD::FDIV,               MVT::v2f64, Legal);
     setOperationAction(ISD::FSQRT,              MVT::v2f64, Legal);
     setOperationAction(ISD::FNEG,               MVT::v2f64, Custom);
+    setOperationAction(ISD::FABS,               MVT::v2f64, Custom);
 
     setOperationAction(ISD::SETCC,              MVT::v2i64, Custom);
     setOperationAction(ISD::SETCC,              MVT::v16i8, Custom);
@@ -925,6 +946,18 @@ X86TargetLowering::X86TargetLowering(X86TargetMachine &TM)
 
     setOperationAction(ISD::FP_TO_SINT,         MVT::v4i32, Legal);
     setOperationAction(ISD::SINT_TO_FP,         MVT::v4i32, Legal);
+
+    setOperationAction(ISD::UINT_TO_FP,         MVT::v4i8,  Custom);
+    setOperationAction(ISD::UINT_TO_FP,         MVT::v4i16, Custom);
+    // As there is no 64-bit GPR available, we need build a special custom
+    // sequence to convert from v2i32 to v2f32.
+    if (!Subtarget->is64Bit())
+      setOperationAction(ISD::UINT_TO_FP,       MVT::v2f32, Custom);
+
+    setOperationAction(ISD::FP_EXTEND,          MVT::v2f32, Custom);
+    setOperationAction(ISD::FP_ROUND,           MVT::v2f32, Custom);
+
+    setLoadExtAction(ISD::EXTLOAD,              MVT::v2f32, Legal);
   }
 
   if (Subtarget->hasSSE41()) {
@@ -939,6 +972,9 @@ X86TargetLowering::X86TargetLowering(X86TargetMachine &TM)
     setOperationAction(ISD::FRINT,              MVT::f64,   Legal);
     setOperationAction(ISD::FNEARBYINT,         MVT::f64,   Legal);
 
+    setOperationAction(ISD::FFLOOR,             MVT::v4f32, Legal);
+    setOperationAction(ISD::FFLOOR,             MVT::v2f64, Legal);
+
     // FIXME: Do we need to handle scalar-to-vector here?
     setOperationAction(ISD::MUL,                MVT::v4i32, Legal);
 
@@ -1016,19 +1052,33 @@ X86TargetLowering::X86TargetLowering(X86TargetMachine &TM)
     setOperationAction(ISD::FMUL,               MVT::v8f32, Legal);
     setOperationAction(ISD::FDIV,               MVT::v8f32, Legal);
     setOperationAction(ISD::FSQRT,              MVT::v8f32, Legal);
+    setOperationAction(ISD::FFLOOR,             MVT::v8f32, Legal);
     setOperationAction(ISD::FNEG,               MVT::v8f32, Custom);
+    setOperationAction(ISD::FABS,               MVT::v8f32, Custom);
 
     setOperationAction(ISD::FADD,               MVT::v4f64, Legal);
     setOperationAction(ISD::FSUB,               MVT::v4f64, Legal);
     setOperationAction(ISD::FMUL,               MVT::v4f64, Legal);
     setOperationAction(ISD::FDIV,               MVT::v4f64, Legal);
     setOperationAction(ISD::FSQRT,              MVT::v4f64, Legal);
+    setOperationAction(ISD::FFLOOR,             MVT::v4f64, Legal);
     setOperationAction(ISD::FNEG,               MVT::v4f64, Custom);
+    setOperationAction(ISD::FABS,               MVT::v4f64, Custom);
+
+    setOperationAction(ISD::TRUNCATE,           MVT::v8i16, Custom);
+
+    setOperationAction(ISD::FP_TO_SINT,         MVT::v8i16, Custom);
 
     setOperationAction(ISD::FP_TO_SINT,         MVT::v8i32, Legal);
     setOperationAction(ISD::SINT_TO_FP,         MVT::v8i32, Legal);
     setOperationAction(ISD::FP_ROUND,           MVT::v4f32, Legal);
 
+    setOperationAction(ISD::ZERO_EXTEND,        MVT::v8i32, Custom);
+    setOperationAction(ISD::UINT_TO_FP,         MVT::v8i8,  Custom);
+    setOperationAction(ISD::UINT_TO_FP,         MVT::v8i16, Custom);
+
+    setLoadExtAction(ISD::EXTLOAD,              MVT::v4f32, Legal);
+
     setOperationAction(ISD::SRL,               MVT::v16i16, Custom);
     setOperationAction(ISD::SRL,               MVT::v32i8, Custom);
 
@@ -1052,7 +1102,7 @@ X86TargetLowering::X86TargetLowering(X86TargetMachine &TM)
     setOperationAction(ISD::VSELECT,           MVT::v8i32, Legal);
     setOperationAction(ISD::VSELECT,           MVT::v8f32, Legal);
 
-    if (Subtarget->hasFMA()) {
+    if (Subtarget->hasFMA() || Subtarget->hasFMA4()) {
       setOperationAction(ISD::FMA,             MVT::v8f32, Custom);
       setOperationAction(ISD::FMA,             MVT::v4f64, Custom);
       setOperationAction(ISD::FMA,             MVT::v4f32, Custom);
@@ -1217,10 +1267,8 @@ X86TargetLowering::X86TargetLowering(X86TargetMachine &TM)
   setTargetDAGCombine(ISD::ANY_EXTEND);
   setTargetDAGCombine(ISD::SIGN_EXTEND);
   setTargetDAGCombine(ISD::TRUNCATE);
-  setTargetDAGCombine(ISD::UINT_TO_FP);
   setTargetDAGCombine(ISD::SINT_TO_FP);
   setTargetDAGCombine(ISD::SETCC);
-  setTargetDAGCombine(ISD::FP_TO_SINT);
   if (Subtarget->is64Bit())
     setTargetDAGCombine(ISD::MUL);
   setTargetDAGCombine(ISD::XOR);
@@ -1318,7 +1366,7 @@ X86TargetLowering::getOptimalMemOpType(uint64_t Size,
   // cases like PR2962.  This should be removed when PR2962 is fixed.
   const Function *F = MF.getFunction();
   if (IsZeroVal &&
-      !F->hasFnAttr(Attribute::NoImplicitFloat)) {
+      !F->getFnAttributes().hasAttribute(Attributes::NoImplicitFloat)) {
     if (Size >= 16 &&
         (Subtarget->isUnalignedMemAccessFast() ||
          ((DstAlign == 0 || DstAlign >= 16) &&
@@ -1986,7 +2034,8 @@ X86TargetLowering::LowerFormalArguments(SDValue Chain,
       unsigned NumIntRegs = CCInfo.getFirstUnallocated(GPR64ArgRegs,
                                                        TotalNumIntRegs);
 
-      bool NoImplicitFloatOps = Fn->hasFnAttr(Attribute::NoImplicitFloat);
+      bool NoImplicitFloatOps = Fn->getFnAttributes().
+        hasAttribute(Attributes::NoImplicitFloat);
       assert(!(NumXMMRegs && !Subtarget->hasSSE1()) &&
              "SSE register cannot be used when SSE is disabled!");
       assert(!(NumXMMRegs && MF.getTarget().Options.UseSoftFloat &&
@@ -2134,16 +2183,14 @@ X86TargetLowering::EmitTailCallLoadRetAddr(SelectionDAG &DAG,
 /// optimization is performed and it is required (FPDiff!=0).
 static SDValue
 EmitTailCallStoreRetAddr(SelectionDAG & DAG, MachineFunction &MF,
-                         SDValue Chain, SDValue RetAddrFrIdx,
-                         bool Is64Bit, int FPDiff, DebugLoc dl) {
+                         SDValue Chain, SDValue RetAddrFrIdx, EVT PtrVT,
+                         unsigned SlotSize, int FPDiff, DebugLoc dl) {
   // Store the return address to the appropriate stack slot.
   if (!FPDiff) return Chain;
   // Calculate the new stack slot for the return address.
-  int SlotSize = Is64Bit ? 8 : 4;
   int NewReturnAddrFI =
     MF.getFrameInfo()->CreateFixedObject(SlotSize, FPDiff-SlotSize, false);
-  EVT VT = Is64Bit ? MVT::i64 : MVT::i32;
-  SDValue NewRetAddrFrIdx = DAG.getFrameIndex(NewReturnAddrFI, VT);
+  SDValue NewRetAddrFrIdx = DAG.getFrameIndex(NewReturnAddrFI, PtrVT);
   Chain = DAG.getStore(Chain, dl, RetAddrFrIdx, NewRetAddrFrIdx,
                        MachinePointerInfo::getFixedStack(NewReturnAddrFI),
                        false, false, 0);
@@ -2178,7 +2225,7 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     // Check if it's really possible to do a tail call.
     isTailCall = IsEligibleForTailCallOptimization(Callee, CallConv,
                     isVarArg, SR != NotStructReturn,
-                    MF.getFunction()->hasStructRetAttr(),
+                    MF.getFunction()->hasStructRetAttr(), CLI.RetTy,
                     Outs, OutVals, Ins, DAG);
 
     // Sibcalls are automatically detected tailcalls which do not require
@@ -2218,14 +2265,15 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
   int FPDiff = 0;
   if (isTailCall && !IsSibcall) {
     // Lower arguments at fp - stackoffset + fpdiff.
-    unsigned NumBytesCallerPushed =
-      MF.getInfo<X86MachineFunctionInfo>()->getBytesToPopOnReturn();
+    X86MachineFunctionInfo *X86Info = MF.getInfo<X86MachineFunctionInfo>();
+    unsigned NumBytesCallerPushed = X86Info->getBytesToPopOnReturn();
+
     FPDiff = NumBytesCallerPushed - NumBytes;
 
     // Set the delta of movement of the returnaddr stackslot.
     // But only set if delta is greater than previous delta.
-    if (FPDiff < (MF.getInfo<X86MachineFunctionInfo>()->getTCReturnAddrDelta()))
-      MF.getInfo<X86MachineFunctionInfo>()->setTCReturnAddrDelta(FPDiff);
+    if (FPDiff < X86Info->getTCReturnAddrDelta())
+      X86Info->setTCReturnAddrDelta(FPDiff);
   }
 
   if (!IsSibcall)
@@ -2302,7 +2350,8 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
     } else if (!IsSibcall && (!isTailCall || isByVal)) {
       assert(VA.isMemLoc());
       if (StackPtr.getNode() == 0)
-        StackPtr = DAG.getCopyFromReg(Chain, dl, X86StackPtr, getPointerTy());
+        StackPtr = DAG.getCopyFromReg(Chain, dl, RegInfo->getStackRegister(),
+                                      getPointerTy());
       MemOpChains.push_back(LowerMemOpCallTo(Chain, StackPtr, Arg,
                                              dl, DAG, VA, Flags));
     }
@@ -2390,7 +2439,8 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
           // Copy relative to framepointer.
           SDValue Source = DAG.getIntPtrConstant(VA.getLocMemOffset());
           if (StackPtr.getNode() == 0)
-            StackPtr = DAG.getCopyFromReg(Chain, dl, X86StackPtr,
+            StackPtr = DAG.getCopyFromReg(Chain, dl,
+                                          RegInfo->getStackRegister(),
                                           getPointerTy());
           Source = DAG.getNode(ISD::ADD, dl, getPointerTy(), StackPtr, Source);
 
@@ -2412,7 +2462,8 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
                           &MemOpChains2[0], MemOpChains2.size());
 
     // Store the return address to the appropriate stack slot.
-    Chain = EmitTailCallStoreRetAddr(DAG, MF, Chain, RetAddrFrIdx, Is64Bit,
+    Chain = EmitTailCallStoreRetAddr(DAG, MF, Chain, RetAddrFrIdx,
+                                     getPointerTy(), RegInfo->getSlotSize(),
                                      FPDiff, dl);
   }
 
@@ -2462,7 +2513,8 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
         OpFlags = X86II::MO_DARWIN_STUB;
       } else if (Subtarget->isPICStyleRIPRel() &&
                  isa<Function>(GV) &&
-                 cast<Function>(GV)->hasFnAttr(Attribute::NonLazyBind)) {
+                 cast<Function>(GV)->getFnAttributes().
+                   hasAttribute(Attributes::NonLazyBind)) {
         // If the function is marked as non-lazy, generate an indirect call
         // which loads from the GOT directly. This avoids runtime overhead
         // at the cost of eager binding (and one extra byte of encoding).
@@ -2623,7 +2675,7 @@ X86TargetLowering::GetAlignedArgumentStackSize(unsigned StackSize,
   unsigned StackAlignment = TFI.getStackAlignment();
   uint64_t AlignMask = StackAlignment - 1;
   int64_t Offset = StackSize;
-  uint64_t SlotSize = TD->getPointerSize();
+  unsigned SlotSize = RegInfo->getSlotSize();
   if ( (Offset & AlignMask) <= (StackAlignment - SlotSize) ) {
     // Number smaller than 12 so just add the difference.
     Offset += ((StackAlignment - SlotSize) - (Offset & AlignMask));
@@ -2698,6 +2750,7 @@ X86TargetLowering::IsEligibleForTailCallOptimization(SDValue Callee,
                                                      bool isVarArg,
                                                      bool isCalleeStructRet,
                                                      bool isCallerStructRet,
+                                                     Type *RetTy,
                                     const SmallVectorImpl<ISD::OutputArg> &Outs,
                                     const SmallVectorImpl<SDValue> &OutVals,
                                     const SmallVectorImpl<ISD::InputArg> &Ins,
@@ -2709,6 +2762,13 @@ X86TargetLowering::IsEligibleForTailCallOptimization(SDValue Callee,
   // If -tailcallopt is specified, make fastcc functions tail-callable.
   const MachineFunction &MF = DAG.getMachineFunction();
   const Function *CallerF = DAG.getMachineFunction().getFunction();
+
+  // If the function return type is x86_fp80 and the callee return type is not,
+  // then the FP_EXTEND of the call result is not a nop. It's not safe to
+  // perform a tailcall optimization here.
+  if (CallerF->getReturnType()->isX86_FP80Ty() && !RetTy->isX86_FP80Ty())
+    return false;
+
   CallingConv::ID CallerCC = CallerF->getCallingConv();
   bool CCMatch = CallerCC == CalleeCC;
 
@@ -2832,7 +2892,7 @@ X86TargetLowering::IsEligibleForTailCallOptimization(SDValue Callee,
       MachineFrameInfo *MFI = MF.getFrameInfo();
       const MachineRegisterInfo *MRI = &MF.getRegInfo();
       const X86InstrInfo *TII =
-        ((X86TargetMachine&)getTargetMachine()).getInstrInfo();
+        ((const X86TargetMachine&)getTargetMachine()).getInstrInfo();
       for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
         CCValAssign &VA = ArgLocs[i];
         SDValue Arg = OutVals[i];
@@ -2983,7 +3043,7 @@ SDValue X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) const {
 
   if (ReturnAddrIndex == 0) {
     // Set up a frame object for the return address.
-    uint64_t SlotSize = TD->getPointerSize();
+    unsigned SlotSize = RegInfo->getSlotSize();
     ReturnAddrIndex = MF.getFrameInfo()->CreateFixedObject(SlotSize, -SlotSize,
                                                            false);
     FuncInfo->setRAIndex(ReturnAddrIndex);
@@ -3506,25 +3566,26 @@ SDValue Compact8x32ShuffleNode(ShuffleVectorSDNode *SVOp,
     if (!isUndefOrEqual(Mask[i], MaskToOptimizeOdd[i]))
       MatchOddMask = false;
   }
-  static const int CompactionMaskEven[] = {0, 2, -1, -1, 4, 6, -1, -1};
-  static const int CompactionMaskOdd [] = {1, 3, -1, -1, 5, 7, -1, -1};
 
-  const int *CompactionMask;
-  if (MatchEvenMask)
-    CompactionMask = CompactionMaskEven;
-  else if (MatchOddMask)
-    CompactionMask = CompactionMaskOdd;
-  else
+  if (!MatchEvenMask && !MatchOddMask)
     return SDValue();
 
   SDValue UndefNode = DAG.getNode(ISD::UNDEF, dl, VT);
 
-  SDValue Op0 = DAG.getVectorShuffle(VT, dl, SVOp->getOperand(0),
-                                     UndefNode, CompactionMask);
-  SDValue Op1 = DAG.getVectorShuffle(VT, dl, SVOp->getOperand(1),
-                                     UndefNode, CompactionMask);
-  static const int UnpackMask[] = {0, 8, 1, 9, 4, 12, 5, 13};
-  return DAG.getVectorShuffle(VT, dl, Op0, Op1, UnpackMask);
+  SDValue Op0 = SVOp->getOperand(0);
+  SDValue Op1 = SVOp->getOperand(1);
+
+  if (MatchEvenMask) {
+    // Shift the second operand right to 32 bits.
+    static const int ShiftRightMask[] = {-1, 0, -1, 2, -1, 4, -1, 6 };
+    Op1 = DAG.getVectorShuffle(VT, dl, Op1, UndefNode, ShiftRightMask);
+  } else {
+    // Shift the first operand left to 32 bits.
+    static const int ShiftLeftMask[] = {1, -1, 3, -1, 5, -1, 7, -1 };
+    Op0 = DAG.getVectorShuffle(VT, dl, Op0, UndefNode, ShiftLeftMask);
+  }
+  static const int BlendMask[] = {0, 9, 2, 11, 4, 13, 6, 15};
+  return DAG.getVectorShuffle(VT, dl, Op0, Op1, BlendMask);
 }
 
 /// isUNPCKLMask - Return true if the specified VECTOR_SHUFFLE operand
@@ -4575,7 +4636,6 @@ static SDValue getShuffleScalarElt(SDNode *N, unsigned Index, SelectionDAG &DAG,
     MVT ShufVT = V.getValueType().getSimpleVT();
     unsigned NumElems = ShufVT.getVectorNumElements();
     SmallVector<int, 16> ShuffleMask;
-    SDValue ImmN;
     bool IsUnary;
 
     if (!getTargetShuffleMask(N, ShufVT, ShuffleMask, IsUnary))
@@ -4977,6 +5037,18 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, SmallVectorImpl<SDValue> &Elts,
                                 LDBase->getAlignment(),
                                 false/*isVolatile*/, true/*ReadMem*/,
                                 false/*WriteMem*/);
+
+    // Make sure the newly-created LOAD is in the same position as LDBase in
+    // terms of dependency. We create a TokenFactor for LDBase and ResNode, and
+    // update uses of LDBase's output chain to use the TokenFactor.
+    if (LDBase->hasAnyUseOfValue(1)) {
+      SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other,
+                             SDValue(LDBase, 1), SDValue(ResNode.getNode(), 1));
+      DAG.ReplaceAllUsesOfValueWith(SDValue(LDBase, 1), NewChain);
+      DAG.UpdateNodeOperands(NewChain.getNode(), SDValue(LDBase, 1),
+                             SDValue(ResNode.getNode(), 1));
+    }
+
     return DAG.getNode(ISD::BITCAST, DL, VT, ResNode);
   }
   return SDValue();
@@ -4990,7 +5062,7 @@ static SDValue EltsFromConsecutiveLoads(EVT VT, SmallVectorImpl<SDValue> &Elts,
 /// The VBROADCAST node is returned when a pattern is found,
 /// or SDValue() otherwise.
 SDValue
-X86TargetLowering::LowerVectorBroadcast(SDValue &Op, SelectionDAG &DAG) const {
+X86TargetLowering::LowerVectorBroadcast(SDValue Op, SelectionDAG &DAG) const {
   if (!Subtarget->hasAVX())
     return SDValue();
 
@@ -5114,80 +5186,78 @@ X86TargetLowering::LowerVectorBroadcast(SDValue &Op, SelectionDAG &DAG) const {
   return SDValue();
 }
 
-// LowerVectorFpExtend - Recognize the scalarized FP_EXTEND from v2f32 to v2f64
-// and convert it into X86ISD::VFPEXT due to the current ISD::FP_EXTEND has the
-// constraint of matching input/output vector elements.
 SDValue
-X86TargetLowering::LowerVectorFpExtend(SDValue &Op, SelectionDAG &DAG) const {
-  DebugLoc DL = Op.getDebugLoc();
-  SDNode *N = Op.getNode();
+X86TargetLowering::buildFromShuffleMostly(SDValue Op, SelectionDAG &DAG) const {
   EVT VT = Op.getValueType();
-  unsigned NumElts = Op.getNumOperands();
 
-  // Check supported types and sub-targets.
-  //
-  // Only v2f32 -> v2f64 needs special handling.
-  if (VT != MVT::v2f64 || !Subtarget->hasSSE2())
+  // Skip if insert_vec_elt is not supported.
+  if (!isOperationLegalOrCustom(ISD::INSERT_VECTOR_ELT, VT))
     return SDValue();
 
-  SDValue VecIn;
-  EVT VecInVT;
-  SmallVector<int, 8> Mask;
-  EVT SrcVT = MVT::Other;
+  DebugLoc DL = Op.getDebugLoc();
+  unsigned NumElems = Op.getNumOperands();
 
-  // Check the patterns could be translated into X86vfpext.
-  for (unsigned i = 0; i < NumElts; ++i) {
-    SDValue In = N->getOperand(i);
-    unsigned Opcode = In.getOpcode();
+  SDValue VecIn1;
+  SDValue VecIn2;
+  SmallVector<unsigned, 4> InsertIndices;
+  SmallVector<int, 8> Mask(NumElems, -1);
+
+  for (unsigned i = 0; i != NumElems; ++i) {
+    unsigned Opc = Op.getOperand(i).getOpcode();
 
-    // Skip if the element is undefined.
-    if (Opcode == ISD::UNDEF) {
-      Mask.push_back(-1);
+    if (Opc == ISD::UNDEF)
       continue;
-    }
 
-    // Quit if one of the elements is not defined from 'fpext'.
-    if (Opcode != ISD::FP_EXTEND)
-      return SDValue();
+    if (Opc != ISD::EXTRACT_VECTOR_ELT) {
+      // Quit if more than 1 elements need inserting.
+      if (InsertIndices.size() > 1)
+        return SDValue();
+
+      InsertIndices.push_back(i);
+      continue;
+    }
 
-    // Check how the source of 'fpext' is defined.
-    SDValue L2In = In.getOperand(0);
-    EVT L2InVT = L2In.getValueType();
+    SDValue ExtractedFromVec = Op.getOperand(i).getOperand(0);
+    SDValue ExtIdx = Op.getOperand(i).getOperand(1);
 
-    // Check the original type
-    if (SrcVT == MVT::Other)
-      SrcVT = L2InVT;
-    else if (SrcVT != L2InVT) // Quit if non-homogenous typed.
+    // Quit if extracted from vector of different type.
+    if (ExtractedFromVec.getValueType() != VT)
       return SDValue();
 
-    // Check whether the value being 'fpext'ed is extracted from the same
-    // source.
-    Opcode = L2In.getOpcode();
-
-    // Quit if it's not extracted with a constant index.
-    if (Opcode != ISD::EXTRACT_VECTOR_ELT ||
-        !isa<ConstantSDNode>(L2In.getOperand(1)))
+    // Quit if non-constant index.
+    if (!isa<ConstantSDNode>(ExtIdx))
       return SDValue();
 
-    SDValue ExtractedFromVec = L2In.getOperand(0);
+    if (VecIn1.getNode() == 0)
+      VecIn1 = ExtractedFromVec;
+    else if (VecIn1 != ExtractedFromVec) {
+      if (VecIn2.getNode() == 0)
+        VecIn2 = ExtractedFromVec;
+      else if (VecIn2 != ExtractedFromVec)
+        // Quit if more than 2 vectors to shuffle
+        return SDValue();
+    }
 
-    if (VecIn.getNode() == 0) {
-      VecIn = ExtractedFromVec;
-      VecInVT = ExtractedFromVec.getValueType();
-    } else if (VecIn != ExtractedFromVec) // Quit if built from more than 1 vec.
-      return SDValue();
+    unsigned Idx = cast<ConstantSDNode>(ExtIdx)->getZExtValue();
 
-    Mask.push_back(cast<ConstantSDNode>(L2In.getOperand(1))->getZExtValue());
+    if (ExtractedFromVec == VecIn1)
+      Mask[i] = Idx;
+    else if (ExtractedFromVec == VecIn2)
+      Mask[i] = Idx + NumElems;
   }
 
-  // Fill the remaining mask as undef.
-  for (unsigned i = NumElts; i < VecInVT.getVectorNumElements(); ++i)
-    Mask.push_back(-1);
+  if (VecIn1.getNode() == 0)
+    return SDValue();
 
-  return DAG.getNode(X86ISD::VFPEXT, DL, VT,
-                     DAG.getVectorShuffle(VecInVT, DL,
-                                          VecIn, DAG.getUNDEF(VecInVT),
-                                          &Mask[0]));
+  VecIn2 = VecIn2.getNode() ? VecIn2 : DAG.getUNDEF(VT);
+  SDValue NV = DAG.getVectorShuffle(VT, DL, VecIn1, VecIn2, &Mask[0]);
+  for (unsigned i = 0, e = InsertIndices.size(); i != e; ++i) {
+    unsigned Idx = InsertIndices[i];
+    NV = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, NV, Op.getOperand(Idx),
+                     DAG.getIntPtrConstant(Idx));
+  }
+
+  return NV;
 }
 
 SDValue
@@ -5222,10 +5292,6 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
   if (Broadcast.getNode())
     return Broadcast;
 
-  SDValue FpExt = LowerVectorFpExtend(Op, DAG);
-  if (FpExt.getNode())
-    return FpExt;
-
   unsigned EVTBits = ExtVT.getSizeInBits();
 
   unsigned NumZero  = 0;
@@ -5470,6 +5536,11 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
     if (LD.getNode())
       return LD;
 
+    // Check for a build vector from mostly shuffle plus few inserting.
+    SDValue Sh = buildFromShuffleMostly(Op, DAG);
+    if (Sh.getNode())
+      return Sh;
+
     // For SSE 4.1, use insertps to put the high elements into the low element.
     if (getSubtarget()->hasSSE41()) {
       SDValue Result;
@@ -5536,8 +5607,7 @@ static SDValue LowerAVXCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) {
   return Concat128BitVectors(V1, V2, ResVT, NumElems, DAG, dl);
 }
 
-SDValue
-X86TargetLowering::LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const {
+static SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) {
   assert(Op.getNumOperands() == 2);
 
   // 256-bit AVX can use the vinsertf128 instruction to create 256-bit vectors
@@ -5546,9 +5616,9 @@ X86TargetLowering::LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const {
 }
 
 // Try to lower a shuffle node into a simple blend instruction.
-static SDValue LowerVECTOR_SHUFFLEtoBlend(ShuffleVectorSDNode *SVOp,
-                                          const X86Subtarget *Subtarget,
-                                          SelectionDAG &DAG) {
+static SDValue
+LowerVECTOR_SHUFFLEtoBlend(ShuffleVectorSDNode *SVOp,
+                           const X86Subtarget *Subtarget, SelectionDAG &DAG) {
   SDValue V1 = SVOp->getOperand(0);
   SDValue V2 = SVOp->getOperand(1);
   DebugLoc dl = SVOp->getDebugLoc();
@@ -5618,9 +5688,9 @@ static SDValue LowerVECTOR_SHUFFLEtoBlend(ShuffleVectorSDNode *SVOp,
 // 2. [ssse3] 1 x pshufb
 // 3. [ssse3] 2 x pshufb + 1 x por
 // 4. [all]   mov + pshuflw + pshufhw + N x (pextrw + pinsrw)
-SDValue
-X86TargetLowering::LowerVECTOR_SHUFFLEv8i16(SDValue Op,
-                                            SelectionDAG &DAG) const {
+static SDValue
+LowerVECTOR_SHUFFLEv8i16(SDValue Op, const X86Subtarget *Subtarget,
+                         SelectionDAG &DAG) {
   ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
   SDValue V1 = SVOp->getOperand(0);
   SDValue V2 = SVOp->getOperand(1);
@@ -5877,8 +5947,6 @@ SDValue LowerVECTOR_SHUFFLEv16i8(ShuffleVectorSDNode *SVOp,
   DebugLoc dl = SVOp->getDebugLoc();
   ArrayRef<int> MaskVals = SVOp->getMask();
 
-  bool V2IsUndef = V2.getOpcode() == ISD::UNDEF;
-
   // If we have SSSE3, case 1 is generated when all result bytes come from
   // one of  the inputs.  Otherwise, case 2 is generated.  If no SSSE3 is
   // present, fall back to case 3.
@@ -5902,7 +5970,11 @@ SDValue LowerVECTOR_SHUFFLEv16i8(ShuffleVectorSDNode *SVOp,
     V1 = DAG.getNode(X86ISD::PSHUFB, dl, MVT::v16i8, V1,
                      DAG.getNode(ISD::BUILD_VECTOR, dl,
                                  MVT::v16i8, &pshufbMask[0], 16));
-    if (V2IsUndef)
+
+    // As PSHUFB will zero elements with negative indices, it's safe to ignore
+    // the 2nd operand if it's undefined or zero.
+    if (V2.getOpcode() == ISD::UNDEF ||
+        ISD::isBuildVectorAllZeros(V2.getNode()))
       return V1;
 
     // Calculate the shuffle mask for the second input, shuffle it, and
@@ -5988,6 +6060,51 @@ SDValue LowerVECTOR_SHUFFLEv16i8(ShuffleVectorSDNode *SVOp,
   return DAG.getNode(ISD::BITCAST, dl, MVT::v16i8, NewV);
 }
 
+// v32i8 shuffles - Translate to VPSHUFB if possible.
+static
+SDValue LowerVECTOR_SHUFFLEv32i8(ShuffleVectorSDNode *SVOp,
+                                 const X86Subtarget *Subtarget,
+                                 SelectionDAG &DAG) {
+  EVT VT = SVOp->getValueType(0);
+  SDValue V1 = SVOp->getOperand(0);
+  SDValue V2 = SVOp->getOperand(1);
+  DebugLoc dl = SVOp->getDebugLoc();
+  SmallVector<int, 32> MaskVals(SVOp->getMask().begin(), SVOp->getMask().end());
+
+  bool V2IsUndef = V2.getOpcode() == ISD::UNDEF;
+  bool V1IsAllZero = ISD::isBuildVectorAllZeros(V1.getNode());
+  bool V2IsAllZero = ISD::isBuildVectorAllZeros(V2.getNode());
+
+  // VPSHUFB may be generated if
+  // (1) one of input vector is undefined or zeroinitializer.
+  // The mask value 0x80 puts 0 in the corresponding slot of the vector.
+  // And (2) the mask indexes don't cross the 128-bit lane.
+  if (VT != MVT::v32i8 || !Subtarget->hasAVX2() ||
+      (!V2IsUndef && !V2IsAllZero && !V1IsAllZero))
+    return SDValue();
+
+  if (V1IsAllZero && !V2IsAllZero) {
+    CommuteVectorShuffleMask(MaskVals, 32);
+    V1 = V2;
+  }
+  SmallVector<SDValue, 32> pshufbMask;
+  for (unsigned i = 0; i != 32; i++) {
+    int EltIdx = MaskVals[i];
+    if (EltIdx < 0 || EltIdx >= 32)
+      EltIdx = 0x80;
+    else {
+      if ((EltIdx >= 16 && i < 16) || (EltIdx < 16 && i >= 16))
+        // Cross lane is not allowed.
+        return SDValue();
+      EltIdx &= 0xf;
+    }
+    pshufbMask.push_back(DAG.getConstant(EltIdx, MVT::i8));
+  }
+  return DAG.getNode(X86ISD::PSHUFB, dl, MVT::v32i8, V1,
+                      DAG.getNode(ISD::BUILD_VECTOR, dl,
+                                  MVT::v32i8, &pshufbMask[0], 32));
+}
+
 /// RewriteAsNarrowerShuffle - Try rewriting v8i16 and v16i8 shuffles as 4 wide
 /// ones, or rewriting v4i32 / v4f32 as 2 wide ones if possible. This can be
 /// done when every pair / quad of shuffle mask elements point to elements in
@@ -6322,17 +6439,17 @@ LowerVECTOR_SHUFFLE_128v4(ShuffleVectorSDNode *SVOp, SelectionDAG &DAG) {
 }
 
 static bool MayFoldVectorLoad(SDValue V) {
-  if (V.hasOneUse() && V.getOpcode() == ISD::BITCAST)
+  while (V.hasOneUse() && V.getOpcode() == ISD::BITCAST)
     V = V.getOperand(0);
+
   if (V.hasOneUse() && V.getOpcode() == ISD::SCALAR_TO_VECTOR)
     V = V.getOperand(0);
   if (V.hasOneUse() && V.getOpcode() == ISD::BUILD_VECTOR &&
       V.getNumOperands() == 2 && V.getOperand(1).getOpcode() == ISD::UNDEF)
     // BUILD_VECTOR (load), undef
     V = V.getOperand(0);
-  if (MayFoldLoad(V))
-    return true;
-  return false;
+
+  return MayFoldLoad(V);
 }
 
 // FIXME: the version above should always be used. Since there's
@@ -6455,6 +6572,81 @@ SDValue getMOVLP(SDValue &Op, DebugLoc &dl, SelectionDAG &DAG, bool HasSSE2) {
                               getShuffleSHUFImmediate(SVOp), DAG);
 }
 
+// Reduce a vector shuffle to zext.
+SDValue
+X86TargetLowering::lowerVectorIntExtend(SDValue Op, SelectionDAG &DAG) const {
+  // PMOVZX is only available from SSE41.
+  if (!Subtarget->hasSSE41())
+    return SDValue();
+
+  EVT VT = Op.getValueType();
+
+  // Only AVX2 support 256-bit vector integer extending.
+  if (!Subtarget->hasAVX2() && VT.is256BitVector())
+    return SDValue();
+
+  ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
+  DebugLoc DL = Op.getDebugLoc();
+  SDValue V1 = Op.getOperand(0);
+  SDValue V2 = Op.getOperand(1);
+  unsigned NumElems = VT.getVectorNumElements();
+
+  // Extending is an unary operation and the element type of the source vector
+  // won't be equal to or larger than i64.
+  if (V2.getOpcode() != ISD::UNDEF || !VT.isInteger() ||
+      VT.getVectorElementType() == MVT::i64)
+    return SDValue();
+
+  // Find the expansion ratio, e.g. expanding from i8 to i32 has a ratio of 4.
+  unsigned Shift = 1; // Start from 2, i.e. 1 << 1.
+  while ((1U << Shift) < NumElems) {
+    if (SVOp->getMaskElt(1U << Shift) == 1)
+      break;
+    Shift += 1;
+    // The maximal ratio is 8, i.e. from i8 to i64.
+    if (Shift > 3)
+      return SDValue();
+  }
+
+  // Check the shuffle mask.
+  unsigned Mask = (1U << Shift) - 1;
+  for (unsigned i = 0; i != NumElems; ++i) {
+    int EltIdx = SVOp->getMaskElt(i);
+    if ((i & Mask) != 0 && EltIdx != -1)
+      return SDValue();
+    if ((i & Mask) == 0 && (unsigned)EltIdx != (i >> Shift))
+      return SDValue();
+  }
+
+  unsigned NBits = VT.getVectorElementType().getSizeInBits() << Shift;
+  EVT NeVT = EVT::getIntegerVT(*DAG.getContext(), NBits);
+  EVT NVT = EVT::getVectorVT(*DAG.getContext(), NeVT, NumElems >> Shift);
+
+  if (!isTypeLegal(NVT))
+    return SDValue();
+
+  // Simplify the operand as it's prepared to be fed into shuffle.
+  unsigned SignificantBits = NVT.getSizeInBits() >> Shift;
+  if (V1.getOpcode() == ISD::BITCAST &&
+      V1.getOperand(0).getOpcode() == ISD::SCALAR_TO_VECTOR &&
+      V1.getOperand(0).getOperand(0).getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
+      V1.getOperand(0)
+        .getOperand(0).getValueType().getSizeInBits() == SignificantBits) {
+    // (bitcast (sclr2vec (ext_vec_elt x))) -> (bitcast x)
+    SDValue V = V1.getOperand(0).getOperand(0).getOperand(0);
+    ConstantSDNode *CIdx =
+      dyn_cast<ConstantSDNode>(V1.getOperand(0).getOperand(0).getOperand(1));
+    // If it's foldable, i.e. normal load with single use, we will let code
+    // selection to fold it. Otherwise, we will short the conversion sequence.
+    if (CIdx && CIdx->getZExtValue() == 0 &&
+        (!ISD::isNormalLoad(V.getNode()) || !V.hasOneUse()))
+      V1 = DAG.getNode(ISD::BITCAST, DL, V1.getValueType(), V);
+  }
+
+  return DAG.getNode(ISD::BITCAST, DL, VT,
+                     DAG.getNode(X86ISD::VZEXT, DL, NVT, V1));
+}
+
 SDValue
 X86TargetLowering::NormalizeVectorShuffle(SDValue Op, SelectionDAG &DAG) const {
   ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
@@ -6485,6 +6677,11 @@ X86TargetLowering::NormalizeVectorShuffle(SDValue Op, SelectionDAG &DAG) const {
     return PromoteSplat(SVOp, DAG);
   }
 
+  // Check integer expanding shuffles.
+  SDValue NewOp = lowerVectorIntExtend(Op, DAG);
+  if (NewOp.getNode())
+    return NewOp;
+
   // If the shuffle can be profitably rewritten as a narrower shuffle, then
   // do it!
   if (VT == MVT::v8i16  || VT == MVT::v16i8 ||
@@ -6534,7 +6731,8 @@ X86TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const {
   bool HasAVX    = Subtarget->hasAVX();
   bool HasAVX2   = Subtarget->hasAVX2();
   MachineFunction &MF = DAG.getMachineFunction();
-  bool OptForSize = MF.getFunction()->hasFnAttr(Attribute::OptimizeForSize);
+  bool OptForSize = MF.getFunction()->getFnAttributes().
+    hasAttribute(Attributes::OptimizeForSize);
 
   assert(VT.getSizeInBits() != 64 && "Can't lower MMX shuffles");
 
@@ -6803,7 +7001,7 @@ X86TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const {
 
   // Handle v8i16 specifically since SSE can do byte extraction and insertion.
   if (VT == MVT::v8i16) {
-    SDValue NewOp = LowerVECTOR_SHUFFLEv8i16(Op, DAG);
+    SDValue NewOp = LowerVECTOR_SHUFFLEv8i16(Op, Subtarget, DAG);
     if (NewOp.getNode())
       return NewOp;
   }
@@ -6814,6 +7012,12 @@ X86TargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const {
       return NewOp;
   }
 
+  if (VT == MVT::v32i8) {
+    SDValue NewOp = LowerVECTOR_SHUFFLEv32i8(SVOp, Subtarget, DAG);
+    if (NewOp.getNode())
+      return NewOp;
+  }
+
   // Handle all 128-bit wide vectors with 4 elements, and match them with
   // several different shuffle types.
   if (NumElems == 4 && VT.is128BitVector())
@@ -6837,9 +7041,9 @@ X86TargetLowering::LowerEXTRACT_VECTOR_ELT_SSE4(SDValue Op,
 
   if (VT.getSizeInBits() == 8) {
     SDValue Extract = DAG.getNode(X86ISD::PEXTRB, dl, MVT::i32,
-                                    Op.getOperand(0), Op.getOperand(1));
+                                  Op.getOperand(0), Op.getOperand(1));
     SDValue Assert  = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Extract,
-                                    DAG.getValueType(VT));
+                                  DAG.getValueType(VT));
     return DAG.getNode(ISD::TRUNCATE, dl, VT, Assert);
   }
 
@@ -6854,9 +7058,9 @@ X86TargetLowering::LowerEXTRACT_VECTOR_ELT_SSE4(SDValue Op,
                                                  Op.getOperand(0)),
                                      Op.getOperand(1)));
     SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, MVT::i32,
-                                    Op.getOperand(0), Op.getOperand(1));
+                                  Op.getOperand(0), Op.getOperand(1));
     SDValue Assert  = DAG.getNode(ISD::AssertZext, dl, MVT::i32, Extract,
-                                    DAG.getValueType(VT));
+                                  DAG.getValueType(VT));
     return DAG.getNode(ISD::TRUNCATE, dl, VT, Assert);
   }
 
@@ -6940,9 +7144,9 @@ X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
     // Transform it so it match pextrw which produces a 32-bit result.
     EVT EltVT = MVT::i32;
     SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, EltVT,
-                                    Op.getOperand(0), Op.getOperand(1));
+                                  Op.getOperand(0), Op.getOperand(1));
     SDValue Assert  = DAG.getNode(ISD::AssertZext, dl, EltVT, Extract,
-                                    DAG.getValueType(VT));
+                                  DAG.getValueType(VT));
     return DAG.getNode(ISD::TRUNCATE, dl, VT, Assert);
   }
 
@@ -7085,8 +7289,7 @@ X86TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const {
   return SDValue();
 }
 
-SDValue
-X86TargetLowering::LowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) const {
+static SDValue LowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) {
   LLVMContext *Context = DAG.getContext();
   DebugLoc dl = Op.getDebugLoc();
   EVT OpVT = Op.getValueType();
@@ -7118,8 +7321,8 @@ X86TargetLowering::LowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) const {
 // Lower a node with an EXTRACT_SUBVECTOR opcode.  This may result in
 // a simple subregister reference or explicit instructions to grab
 // upper bits of a vector.
-SDValue
-X86TargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const {
+static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, const X86Subtarget *Subtarget,
+                                      SelectionDAG &DAG) {
   if (Subtarget->hasAVX()) {
     DebugLoc dl = Op.getNode()->getDebugLoc();
     SDValue Vec = Op.getNode()->getOperand(0);
@@ -7138,8 +7341,8 @@ X86TargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const {
 // Lower a node with an INSERT_SUBVECTOR opcode.  This may result in a
 // simple superregister reference or explicit instructions to insert
 // the upper bits of a vector.
-SDValue
-X86TargetLowering::LowerINSERT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const {
+static SDValue LowerINSERT_SUBVECTOR(SDValue Op, const X86Subtarget *Subtarget,
+                                     SelectionDAG &DAG) {
   if (Subtarget->hasAVX()) {
     DebugLoc dl = Op.getNode()->getDebugLoc();
     SDValue Vec = Op.getNode()->getOperand(0);
@@ -7282,9 +7485,10 @@ X86TargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const {
     Subtarget->ClassifyBlockAddressReference();
   CodeModel::Model M = getTargetMachine().getCodeModel();
   const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress();
+  int64_t Offset = cast<BlockAddressSDNode>(Op)->getOffset();
   DebugLoc dl = Op.getDebugLoc();
-  SDValue Result = DAG.getBlockAddress(BA, getPointerTy(),
-                                       /*isTarget=*/true, OpFlags);
+  SDValue Result = DAG.getTargetBlockAddress(BA, getPointerTy(), Offset,
+                                             OpFlags);
 
   if (Subtarget->isPICStyleRIPRel() &&
       (M == CodeModel::Small || M == CodeModel::Kernel))
@@ -7393,8 +7597,8 @@ LowerToTLSGeneralDynamicModel32(GlobalAddressSDNode *GA, SelectionDAG &DAG,
   SDValue InFlag;
   DebugLoc dl = GA->getDebugLoc();  // ? function entry point might be better
   SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, X86::EBX,
-                                     DAG.getNode(X86ISD::GlobalBaseReg,
-                                                 DebugLoc(), PtrVT), InFlag);
+                                   DAG.getNode(X86ISD::GlobalBaseReg,
+                                               DebugLoc(), PtrVT), InFlag);
   InFlag = Chain.getValue(1);
 
   return GetTLSADDR(DAG, Chain, GA, &InFlag, PtrVT, X86::EAX, X86II::MO_TLSGD);
@@ -7895,11 +8099,29 @@ SDValue X86TargetLowering::LowerUINT_TO_FP_i32(SDValue Op,
   return Sub;
 }
 
+SDValue X86TargetLowering::lowerUINT_TO_FP_vec(SDValue Op,
+                                               SelectionDAG &DAG) const {
+  SDValue N0 = Op.getOperand(0);
+  EVT SVT = N0.getValueType();
+  DebugLoc dl = Op.getDebugLoc();
+
+  assert((SVT == MVT::v4i8 || SVT == MVT::v4i16 ||
+          SVT == MVT::v8i8 || SVT == MVT::v8i16) &&
+         "Custom UINT_TO_FP is not supported!");
+
+  EVT NVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, SVT.getVectorNumElements());
+  return DAG.getNode(ISD::SINT_TO_FP, dl, Op.getValueType(),
+                     DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N0));
+}
+
 SDValue X86TargetLowering::LowerUINT_TO_FP(SDValue Op,
                                            SelectionDAG &DAG) const {
   SDValue N0 = Op.getOperand(0);
   DebugLoc dl = Op.getDebugLoc();
 
+  if (Op.getValueType().isVector())
+    return lowerUINT_TO_FP_vec(Op, DAG);
+
   // Since UINT_TO_FP is legal (it's marked custom), dag combiner won't
   // optimize it to a SINT_TO_FP when the sign bit is known zero. Perform
   // the optimization here.
@@ -8073,10 +8295,66 @@ FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG, bool IsSigned, bool IsReplace) co
   }
 }
 
+SDValue X86TargetLowering::lowerZERO_EXTEND(SDValue Op, SelectionDAG &DAG) const {
+  DebugLoc DL = Op.getDebugLoc();
+  EVT VT = Op.getValueType();
+  SDValue In = Op.getOperand(0);
+  EVT SVT = In.getValueType();
+
+  if (!VT.is256BitVector() || !SVT.is128BitVector() ||
+      VT.getVectorNumElements() != SVT.getVectorNumElements())
+    return SDValue();
+
+  assert(Subtarget->hasAVX() && "256-bit vector is observed without AVX!");
+
+  // AVX2 has better support of integer extending.
+  if (Subtarget->hasAVX2())
+    return DAG.getNode(X86ISD::VZEXT, DL, VT, In);
+
+  SDValue Lo = DAG.getNode(X86ISD::VZEXT, DL, MVT::v4i32, In);
+  static const int Mask[] = {4, 5, 6, 7, -1, -1, -1, -1};
+  SDValue Hi = DAG.getNode(X86ISD::VZEXT, DL, MVT::v4i32,
+                           DAG.getVectorShuffle(MVT::v8i16, DL, In, DAG.getUNDEF(MVT::v8i16), &Mask[0]));
+
+  return DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i32, Lo, Hi);
+}
+
+SDValue X86TargetLowering::lowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const {
+  DebugLoc DL = Op.getDebugLoc();
+  EVT VT = Op.getValueType();
+  EVT SVT = Op.getOperand(0).getValueType();
+
+  if (!VT.is128BitVector() || !SVT.is256BitVector() ||
+      VT.getVectorNumElements() != SVT.getVectorNumElements())
+    return SDValue();
+
+  assert(Subtarget->hasAVX() && "256-bit vector is observed without AVX!");
+
+  unsigned NumElems = VT.getVectorNumElements();
+  EVT NVT = EVT::getVectorVT(*DAG.getContext(), VT.getVectorElementType(),
+                             NumElems * 2);
+
+  SDValue In = Op.getOperand(0);
+  SmallVector<int, 16> MaskVec(NumElems * 2, -1);
+  // Prepare truncation shuffle mask
+  for (unsigned i = 0; i != NumElems; ++i)
+    MaskVec[i] = i * 2;
+  SDValue V = DAG.getVectorShuffle(NVT, DL,
+                                   DAG.getNode(ISD::BITCAST, DL, NVT, In),
+                                   DAG.getUNDEF(NVT), &MaskVec[0]);
+  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, V,
+                     DAG.getIntPtrConstant(0));
+}
+
 SDValue X86TargetLowering::LowerFP_TO_SINT(SDValue Op,
                                            SelectionDAG &DAG) const {
-  if (Op.getValueType().isVector())
+  if (Op.getValueType().isVector()) {
+    if (Op.getValueType() == MVT::v8i16)
+      return DAG.getNode(ISD::TRUNCATE, Op.getDebugLoc(), Op.getValueType(),
+                         DAG.getNode(ISD::FP_TO_SINT, Op.getDebugLoc(),
+                                     MVT::v8i32, Op.getOperand(0)));
     return SDValue();
+  }
 
   std::pair<SDValue,SDValue> Vals = FP_TO_INTHelper(Op, DAG,
     /*IsSigned=*/ true, /*IsReplace=*/ false);
@@ -8111,26 +8389,49 @@ SDValue X86TargetLowering::LowerFP_TO_UINT(SDValue Op,
   return FIST;
 }
 
-SDValue X86TargetLowering::LowerFABS(SDValue Op,
-                                     SelectionDAG &DAG) const {
+SDValue X86TargetLowering::lowerFP_EXTEND(SDValue Op,
+                                          SelectionDAG &DAG) const {
+  DebugLoc DL = Op.getDebugLoc();
+  EVT VT = Op.getValueType();
+  SDValue In = Op.getOperand(0);
+  EVT SVT = In.getValueType();
+
+  assert(SVT == MVT::v2f32 && "Only customize MVT::v2f32 type legalization!");
+
+  return DAG.getNode(X86ISD::VFPEXT, DL, VT,
+                     DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4f32,
+                                 In, DAG.getUNDEF(SVT)));
+}
+
+SDValue X86TargetLowering::LowerFABS(SDValue Op, SelectionDAG &DAG) const {
   LLVMContext *Context = DAG.getContext();
   DebugLoc dl = Op.getDebugLoc();
   EVT VT = Op.getValueType();
   EVT EltVT = VT;
-  if (VT.isVector())
+  unsigned NumElts = VT == MVT::f64 ? 2 : 4;
+  if (VT.isVector()) {
     EltVT = VT.getVectorElementType();
-  Constant *C;
-  if (EltVT == MVT::f64) {
-    C = ConstantVector::getSplat(2,
-                ConstantFP::get(*Context, APFloat(APInt(64, ~(1ULL << 63)))));
-  } else {
-    C = ConstantVector::getSplat(4,
-               ConstantFP::get(*Context, APFloat(APInt(32, ~(1U << 31)))));
+    NumElts = VT.getVectorNumElements();
   }
-  SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 16);
+  Constant *C;
+  if (EltVT == MVT::f64)
+    C = ConstantFP::get(*Context, APFloat(APInt(64, ~(1ULL << 63))));
+  else
+    C = ConstantFP::get(*Context, APFloat(APInt(32, ~(1U << 31))));
+  C = ConstantVector::getSplat(NumElts, C);
+  SDValue CPIdx = DAG.getConstantPool(C, getPointerTy());
+  unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
   SDValue Mask = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx,
                              MachinePointerInfo::getConstantPool(),
-                             false, false, false, 16);
+                             false, false, false, Alignment);
+  if (VT.isVector()) {
+    MVT ANDVT = VT.is128BitVector() ? MVT::v2i64 : MVT::v4i64;
+    return DAG.getNode(ISD::BITCAST, dl, VT,
+                       DAG.getNode(ISD::AND, dl, ANDVT,
+                                   DAG.getNode(ISD::BITCAST, dl, ANDVT,
+                                               Op.getOperand(0)),
+                                   DAG.getNode(ISD::BITCAST, dl, ANDVT, Mask)));
+  }
   return DAG.getNode(X86ISD::FAND, dl, VT, Op.getOperand(0), Mask);
 }
 
@@ -8150,10 +8451,11 @@ SDValue X86TargetLowering::LowerFNEG(SDValue Op, SelectionDAG &DAG) const {
   else
     C = ConstantFP::get(*Context, APFloat(APInt(32, 1U << 31)));
   C = ConstantVector::getSplat(NumElts, C);
-  SDValue CPIdx = DAG.getConstantPool(C, getPointerTy(), 16);
+  SDValue CPIdx = DAG.getConstantPool(C, getPointerTy());
+  unsigned Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlignment();
   SDValue Mask = DAG.getLoad(VT, dl, DAG.getEntryNode(), CPIdx,
                              MachinePointerInfo::getConstantPool(),
-                             false, false, false, 16);
+                             false, false, false, Alignment);
   if (VT.isVector()) {
     MVT XORVT = VT.is128BitVector() ? MVT::v2i64 : MVT::v4i64;
     return DAG.getNode(ISD::BITCAST, dl, VT,
@@ -8239,7 +8541,7 @@ SDValue X86TargetLowering::LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const {
   return DAG.getNode(X86ISD::FOR, dl, VT, Val, SignBit);
 }
 
-SDValue X86TargetLowering::LowerFGETSIGN(SDValue Op, SelectionDAG &DAG) const {
+static SDValue LowerFGETSIGN(SDValue Op, SelectionDAG &DAG) {
   SDValue N0 = Op.getOperand(0);
   DebugLoc dl = Op.getDebugLoc();
   EVT VT = Op.getValueType();
@@ -8250,6 +8552,98 @@ SDValue X86TargetLowering::LowerFGETSIGN(SDValue Op, SelectionDAG &DAG) const {
   return DAG.getNode(ISD::AND, dl, VT, xFGETSIGN, DAG.getConstant(1, VT));
 }
 
+// LowerVectorAllZeroTest - Check whether an OR'd tree is PTEST-able.
+//
+SDValue X86TargetLowering::LowerVectorAllZeroTest(SDValue Op, SelectionDAG &DAG) const {
+  assert(Op.getOpcode() == ISD::OR && "Only check OR'd tree.");
+
+  if (!Subtarget->hasSSE41())
+    return SDValue();
+
+  if (!Op->hasOneUse())
+    return SDValue();
+
+  SDNode *N = Op.getNode();
+  DebugLoc DL = N->getDebugLoc();
+
+  SmallVector<SDValue, 8> Opnds;
+  DenseMap<SDValue, unsigned> VecInMap;
+  EVT VT = MVT::Other;
+
+  // Recognize a special case where a vector is casted into wide integer to
+  // test all 0s.
+  Opnds.push_back(N->getOperand(0));
+  Opnds.push_back(N->getOperand(1));
+
+  for (unsigned Slot = 0, e = Opnds.size(); Slot < e; ++Slot) {
+    SmallVector<SDValue, 8>::const_iterator I = Opnds.begin() + Slot;
+    // BFS traverse all OR'd operands.
+    if (I->getOpcode() == ISD::OR) {
+      Opnds.push_back(I->getOperand(0));
+      Opnds.push_back(I->getOperand(1));
+      // Re-evaluate the number of nodes to be traversed.
+      e += 2; // 2 more nodes (LHS and RHS) are pushed.
+      continue;
+    }
+
+    // Quit if a non-EXTRACT_VECTOR_ELT
+    if (I->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
+      return SDValue();
+
+    // Quit if without a constant index.
+    SDValue Idx = I->getOperand(1);
+    if (!isa<ConstantSDNode>(Idx))
+      return SDValue();
+
+    SDValue ExtractedFromVec = I->getOperand(0);
+    DenseMap<SDValue, unsigned>::iterator M = VecInMap.find(ExtractedFromVec);
+    if (M == VecInMap.end()) {
+      VT = ExtractedFromVec.getValueType();
+      // Quit if not 128/256-bit vector.
+      if (!VT.is128BitVector() && !VT.is256BitVector())
+        return SDValue();
+      // Quit if not the same type.
+      if (VecInMap.begin() != VecInMap.end() &&
+          VT != VecInMap.begin()->first.getValueType())
+        return SDValue();
+      M = VecInMap.insert(std::make_pair(ExtractedFromVec, 0)).first;
+    }
+    M->second |= 1U << cast<ConstantSDNode>(Idx)->getZExtValue();
+  }
+
+  assert((VT.is128BitVector() || VT.is256BitVector()) &&
+         "Not extracted from 128-/256-bit vector.");
+
+  unsigned FullMask = (1U << VT.getVectorNumElements()) - 1U;
+  SmallVector<SDValue, 8> VecIns;
+
+  for (DenseMap<SDValue, unsigned>::const_iterator
+        I = VecInMap.begin(), E = VecInMap.end(); I != E; ++I) {
+    // Quit if not all elements are used.
+    if (I->second != FullMask)
+      return SDValue();
+    VecIns.push_back(I->first);
+  }
+
+  EVT TestVT = VT.is128BitVector() ? MVT::v2i64 : MVT::v4i64;
+
+  // Cast all vectors into TestVT for PTEST.
+  for (unsigned i = 0, e = VecIns.size(); i < e; ++i)
+    VecIns[i] = DAG.getNode(ISD::BITCAST, DL, TestVT, VecIns[i]);
+
+  // If more than one full vectors are evaluated, OR them first before PTEST.
+  for (unsigned Slot = 0, e = VecIns.size(); e - Slot > 1; Slot += 2, e += 1) {
+    // Each iteration will OR 2 nodes and append the result until there is only
+    // 1 node left, i.e. the final OR'd value of all vectors.
+    SDValue LHS = VecIns[Slot];
+    SDValue RHS = VecIns[Slot + 1];
+    VecIns.push_back(DAG.getNode(ISD::OR, DL, TestVT, LHS, RHS));
+  }
+
+  return DAG.getNode(X86ISD::PTEST, DL, MVT::i32,
+                     VecIns.back(), VecIns.back());
+}
+
 /// Emit nodes that will be selected as "test Op0,Op0", or something
 /// equivalent.
 SDValue X86TargetLowering::EmitTest(SDValue Op, unsigned X86CC,
@@ -8283,7 +8677,33 @@ SDValue X86TargetLowering::EmitTest(SDValue Op, unsigned X86CC,
 
   unsigned Opcode = 0;
   unsigned NumOperands = 0;
-  switch (Op.getNode()->getOpcode()) {
+
+  // Truncate operations may prevent the merge of the SETCC instruction
+  // and the arithmetic intruction before it. Attempt to truncate the operands
+  // of the arithmetic instruction and use a reduced bit-width instruction.
+  bool NeedTruncation = false;
+  SDValue ArithOp = Op;
+  if (Op->getOpcode() == ISD::TRUNCATE && Op->hasOneUse()) {
+    SDValue Arith = Op->getOperand(0);
+    // Both the trunc and the arithmetic op need to have one user each.
+    if (Arith->hasOneUse())
+      switch (Arith.getOpcode()) {
+        default: break;
+        case ISD::ADD:
+        case ISD::SUB:
+        case ISD::AND:
+        case ISD::OR:
+        case ISD::XOR: {
+          NeedTruncation = true;
+          ArithOp = Arith;
+        }
+      }
+  }
+
+  // NOTICE: In the code below we use ArithOp to hold the arithmetic operation
+  // which may be the result of a CAST.  We use the variable 'Op', which is the
+  // non-casted variable when we check for possible users.
+  switch (ArithOp.getOpcode()) {
   case ISD::ADD:
     // Due to an isel shortcoming, be conservative if this add is likely to be
     // selected as part of a load-modify-store instruction. When the root node
@@ -8303,7 +8723,7 @@ SDValue X86TargetLowering::EmitTest(SDValue Op, unsigned X86CC,
         goto default_case;
 
     if (ConstantSDNode *C =
-        dyn_cast<ConstantSDNode>(Op.getNode()->getOperand(1))) {
+        dyn_cast<ConstantSDNode>(ArithOp.getNode()->getOperand(1))) {
       // An add of one will be selected as an INC.
       if (C->getAPIntValue() == 1) {
         Opcode = X86ISD::INC;
@@ -8339,7 +8759,7 @@ SDValue X86TargetLowering::EmitTest(SDValue Op, unsigned X86CC,
 
       if (User->getOpcode() != ISD::BRCOND &&
           User->getOpcode() != ISD::SETCC &&
-          (User->getOpcode() != ISD::SELECT || UOpNo != 0)) {
+          !(User->getOpcode() == ISD::SELECT && UOpNo == 0)) {
         NonFlagUse = true;
         break;
       }
@@ -8360,14 +8780,20 @@ SDValue X86TargetLowering::EmitTest(SDValue Op, unsigned X86CC,
         goto default_case;
 
     // Otherwise use a regular EFLAGS-setting instruction.
-    switch (Op.getNode()->getOpcode()) {
+    switch (ArithOp.getOpcode()) {
     default: llvm_unreachable("unexpected operator!");
-    case ISD::SUB:
-      Opcode = X86ISD::SUB;
-      break;
-    case ISD::OR:  Opcode = X86ISD::OR;  break;
+    case ISD::SUB: Opcode = X86ISD::SUB; break;
     case ISD::XOR: Opcode = X86ISD::XOR; break;
     case ISD::AND: Opcode = X86ISD::AND; break;
+    case ISD::OR: {
+      if (!NeedTruncation && (X86CC == X86::COND_E || X86CC == X86::COND_NE)) {
+        SDValue EFLAGS = LowerVectorAllZeroTest(Op, DAG);
+        if (EFLAGS.getNode())
+          return EFLAGS;
+      }
+      Opcode = X86ISD::OR;
+      break;
+    }
     }
 
     NumOperands = 2;
@@ -8385,19 +8811,40 @@ SDValue X86TargetLowering::EmitTest(SDValue Op, unsigned X86CC,
     break;
   }
 
+  // If we found that truncation is beneficial, perform the truncation and
+  // update 'Op'.
+  if (NeedTruncation) {
+    EVT VT = Op.getValueType();
+    SDValue WideVal = Op->getOperand(0);
+    EVT WideVT = WideVal.getValueType();
+    unsigned ConvertedOp = 0;
+    // Use a target machine opcode to prevent further DAGCombine
+    // optimizations that may separate the arithmetic operations
+    // from the setcc node.
+    switch (WideVal.getOpcode()) {
+      default: break;
+      case ISD::ADD: ConvertedOp = X86ISD::ADD; break;
+      case ISD::SUB: ConvertedOp = X86ISD::SUB; break;
+      case ISD::AND: ConvertedOp = X86ISD::AND; break;
+      case ISD::OR:  ConvertedOp = X86ISD::OR;  break;
+      case ISD::XOR: ConvertedOp = X86ISD::XOR; break;
+    }
+
+    if (ConvertedOp) {
+      const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+      if (TLI.isOperationLegal(WideVal.getOpcode(), WideVT)) {
+        SDValue V0 = DAG.getNode(ISD::TRUNCATE, dl, VT, WideVal.getOperand(0));
+        SDValue V1 = DAG.getNode(ISD::TRUNCATE, dl, VT, WideVal.getOperand(1));
+        Op = DAG.getNode(ConvertedOp, dl, VT, V0, V1);
+      }
+    }
+  }
+
   if (Opcode == 0)
     // Emit a CMP with 0, which is the TEST pattern.
     return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
                        DAG.getConstant(0, Op.getValueType()));
 
-  if (Opcode == X86ISD::CMP) {
-    SDValue New = DAG.getNode(Opcode, dl, MVT::i32, Op.getOperand(0),
-                              Op.getOperand(1));
-    // We can't replace usage of SUB with CMP.
-    // The SUB node will be removed later because there is no use of it.
-    return SDValue(New.getNode(), 0);
-  }
-
   SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
   SmallVector<SDValue, 4> Ops;
   for (unsigned i = 0; i != NumOperands; ++i)
@@ -8956,6 +9403,21 @@ SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
     }
   }
 
+  // X86 doesn't have an i8 cmov. If both operands are the result of a truncate
+  // widen the cmov and push the truncate through. This avoids introducing a new
+  // branch during isel and doesn't add any extensions.
+  if (Op.getValueType() == MVT::i8 &&
+      Op1.getOpcode() == ISD::TRUNCATE && Op2.getOpcode() == ISD::TRUNCATE) {
+    SDValue T1 = Op1.getOperand(0), T2 = Op2.getOperand(0);
+    if (T1.getValueType() == T2.getValueType() &&
+        // Blacklist CopyFromReg to avoid partial register stalls.
+        T1.getOpcode() != ISD::CopyFromReg && T2.getOpcode()!=ISD::CopyFromReg){
+      SDVTList VTs = DAG.getVTList(T1.getValueType(), MVT::Glue);
+      SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, VTs, T2, T1, CC, Cond);
+      return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Cmov);
+    }
+  }
+
   // X86ISD::CMOV means set the result (which is operand 1) to the RHS if
   // condition is true.
   SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue);
@@ -9310,7 +9772,8 @@ X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
     Chain = DAG.getNode(X86ISD::WIN_ALLOCA, dl, NodeTys, Chain, Flag);
     Flag = Chain.getValue(1);
 
-    Chain = DAG.getCopyFromReg(Chain, dl, X86StackPtr, SPTy).getValue(1);
+    Chain = DAG.getCopyFromReg(Chain, dl, RegInfo->getStackRegister(),
+                               SPTy).getValue(1);
 
     SDValue Ops1[2] = { Chain.getValue(0), Chain };
     return DAG.getMergeValues(Ops1, 2, dl);
@@ -9393,7 +9856,7 @@ SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
 
   EVT ArgVT = Op.getNode()->getValueType(0);
   Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
-  uint32_t ArgSize = getTargetData()->getTypeAllocSize(ArgTy);
+  uint32_t ArgSize = getDataLayout()->getTypeAllocSize(ArgTy);
   uint8_t ArgMode;
 
   // Decide which area this value should be read from.
@@ -9413,7 +9876,8 @@ SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
     // Sanity Check: Make sure using fp_offset makes sense.
     assert(!getTargetMachine().Options.UseSoftFloat &&
            !(DAG.getMachineFunction()
-                .getFunction()->hasFnAttr(Attribute::NoImplicitFloat)) &&
+                .getFunction()->getFnAttributes()
+                .hasAttribute(Attributes::NoImplicitFloat)) &&
            Subtarget->hasSSE1());
   }
 
@@ -9444,7 +9908,8 @@ SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
                      false, false, false, 0);
 }
 
-SDValue X86TargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) const {
+static SDValue LowerVACOPY(SDValue Op, const X86Subtarget *Subtarget,
+                           SelectionDAG &DAG) {
   // X86-64 va_list is a struct { i32, i32, i8*, i8* }.
   assert(Subtarget->is64Bit() && "This code only handles 64-bit va_copy!");
   SDValue Chain = Op.getOperand(0);
@@ -9505,8 +9970,7 @@ static SDValue getTargetVShiftNode(unsigned Opc, DebugLoc dl, EVT VT,
   return DAG.getNode(Opc, dl, VT, SrcOp, ShAmt);
 }
 
-SDValue
-X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const {
+static SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) {
   DebugLoc dl = Op.getDebugLoc();
   unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
   switch (IntNo) {
@@ -9894,62 +10358,6 @@ X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const
                                Op.getOperand(1), Op.getOperand(2), DAG);
   }
 
-  // Fix vector shift instructions where the last operand is a non-immediate
-  // i32 value.
-  case Intrinsic::x86_mmx_pslli_w:
-  case Intrinsic::x86_mmx_pslli_d:
-  case Intrinsic::x86_mmx_pslli_q:
-  case Intrinsic::x86_mmx_psrli_w:
-  case Intrinsic::x86_mmx_psrli_d:
-  case Intrinsic::x86_mmx_psrli_q:
-  case Intrinsic::x86_mmx_psrai_w:
-  case Intrinsic::x86_mmx_psrai_d: {
-    SDValue ShAmt = Op.getOperand(2);
-    if (isa<ConstantSDNode>(ShAmt))
-      return SDValue();
-
-    unsigned NewIntNo;
-    switch (IntNo) {
-    default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
-    case Intrinsic::x86_mmx_pslli_w:
-      NewIntNo = Intrinsic::x86_mmx_psll_w;
-      break;
-    case Intrinsic::x86_mmx_pslli_d:
-      NewIntNo = Intrinsic::x86_mmx_psll_d;
-      break;
-    case Intrinsic::x86_mmx_pslli_q:
-      NewIntNo = Intrinsic::x86_mmx_psll_q;
-      break;
-    case Intrinsic::x86_mmx_psrli_w:
-      NewIntNo = Intrinsic::x86_mmx_psrl_w;
-      break;
-    case Intrinsic::x86_mmx_psrli_d:
-      NewIntNo = Intrinsic::x86_mmx_psrl_d;
-      break;
-    case Intrinsic::x86_mmx_psrli_q:
-      NewIntNo = Intrinsic::x86_mmx_psrl_q;
-      break;
-    case Intrinsic::x86_mmx_psrai_w:
-      NewIntNo = Intrinsic::x86_mmx_psra_w;
-      break;
-    case Intrinsic::x86_mmx_psrai_d:
-      NewIntNo = Intrinsic::x86_mmx_psra_d;
-      break;
-    }
-
-    // The vector shift intrinsics with scalars uses 32b shift amounts but
-    // the sse2/mmx shift instructions reads 64 bits. Set the upper 32 bits
-    // to be zero.
-    ShAmt =  DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v2i32, ShAmt,
-                         DAG.getConstant(0, MVT::i32));
-// FIXME this must be lowered to get rid of the invalid type.
-
-    EVT VT = Op.getValueType();
-    ShAmt = DAG.getNode(ISD::BITCAST, dl, VT, ShAmt);
-    return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, dl, VT,
-                       DAG.getConstant(NewIntNo, MVT::i32),
-                       Op.getOperand(1), ShAmt);
-  }
   case Intrinsic::x86_sse42_pcmpistria128:
   case Intrinsic::x86_sse42_pcmpestria128:
   case Intrinsic::x86_sse42_pcmpistric128:
@@ -10028,13 +10436,80 @@ X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const
     SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
     return DAG.getNode(Opcode, dl, VTs, NewOps.data(), NewOps.size());
   }
-  }
-}
-
-SDValue
-X86TargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) const {
-  DebugLoc dl = Op.getDebugLoc();
-  unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
+  case Intrinsic::x86_fma_vfmadd_ps:
+  case Intrinsic::x86_fma_vfmadd_pd:
+  case Intrinsic::x86_fma_vfmsub_ps:
+  case Intrinsic::x86_fma_vfmsub_pd:
+  case Intrinsic::x86_fma_vfnmadd_ps:
+  case Intrinsic::x86_fma_vfnmadd_pd:
+  case Intrinsic::x86_fma_vfnmsub_ps:
+  case Intrinsic::x86_fma_vfnmsub_pd:
+  case Intrinsic::x86_fma_vfmaddsub_ps:
+  case Intrinsic::x86_fma_vfmaddsub_pd:
+  case Intrinsic::x86_fma_vfmsubadd_ps:
+  case Intrinsic::x86_fma_vfmsubadd_pd:
+  case Intrinsic::x86_fma_vfmadd_ps_256:
+  case Intrinsic::x86_fma_vfmadd_pd_256:
+  case Intrinsic::x86_fma_vfmsub_ps_256:
+  case Intrinsic::x86_fma_vfmsub_pd_256:
+  case Intrinsic::x86_fma_vfnmadd_ps_256:
+  case Intrinsic::x86_fma_vfnmadd_pd_256:
+  case Intrinsic::x86_fma_vfnmsub_ps_256:
+  case Intrinsic::x86_fma_vfnmsub_pd_256:
+  case Intrinsic::x86_fma_vfmaddsub_ps_256:
+  case Intrinsic::x86_fma_vfmaddsub_pd_256:
+  case Intrinsic::x86_fma_vfmsubadd_ps_256:
+  case Intrinsic::x86_fma_vfmsubadd_pd_256: {
+    unsigned Opc;
+    switch (IntNo) {
+    default: llvm_unreachable("Impossible intrinsic");  // Can't reach here.
+    case Intrinsic::x86_fma_vfmadd_ps:
+    case Intrinsic::x86_fma_vfmadd_pd:
+    case Intrinsic::x86_fma_vfmadd_ps_256:
+    case Intrinsic::x86_fma_vfmadd_pd_256:
+      Opc = X86ISD::FMADD;
+      break;
+    case Intrinsic::x86_fma_vfmsub_ps:
+    case Intrinsic::x86_fma_vfmsub_pd:
+    case Intrinsic::x86_fma_vfmsub_ps_256:
+    case Intrinsic::x86_fma_vfmsub_pd_256:
+      Opc = X86ISD::FMSUB;
+      break;
+    case Intrinsic::x86_fma_vfnmadd_ps:
+    case Intrinsic::x86_fma_vfnmadd_pd:
+    case Intrinsic::x86_fma_vfnmadd_ps_256:
+    case Intrinsic::x86_fma_vfnmadd_pd_256:
+      Opc = X86ISD::FNMADD;
+      break;
+    case Intrinsic::x86_fma_vfnmsub_ps:
+    case Intrinsic::x86_fma_vfnmsub_pd:
+    case Intrinsic::x86_fma_vfnmsub_ps_256:
+    case Intrinsic::x86_fma_vfnmsub_pd_256:
+      Opc = X86ISD::FNMSUB;
+      break;
+    case Intrinsic::x86_fma_vfmaddsub_ps:
+    case Intrinsic::x86_fma_vfmaddsub_pd:
+    case Intrinsic::x86_fma_vfmaddsub_ps_256:
+    case Intrinsic::x86_fma_vfmaddsub_pd_256:
+      Opc = X86ISD::FMADDSUB;
+      break;
+    case Intrinsic::x86_fma_vfmsubadd_ps:
+    case Intrinsic::x86_fma_vfmsubadd_pd:
+    case Intrinsic::x86_fma_vfmsubadd_ps_256:
+    case Intrinsic::x86_fma_vfmsubadd_pd_256:
+      Opc = X86ISD::FMSUBADD;
+      break;
+    }
+
+    return DAG.getNode(Opc, dl, Op.getValueType(), Op.getOperand(1),
+                       Op.getOperand(2), Op.getOperand(3));
+  }
+  }
+}
+
+static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) {
+  DebugLoc dl = Op.getDebugLoc();
+  unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
   switch (IntNo) {
   default: return SDValue();    // Don't custom lower most intrinsics.
 
@@ -10070,21 +10545,21 @@ SDValue X86TargetLowering::LowerRETURNADDR(SDValue Op,
 
   unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
   DebugLoc dl = Op.getDebugLoc();
+  EVT PtrVT = getPointerTy();
 
   if (Depth > 0) {
     SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
     SDValue Offset =
-      DAG.getConstant(TD->getPointerSize(),
-                      Subtarget->is64Bit() ? MVT::i64 : MVT::i32);
-    return DAG.getLoad(getPointerTy(), dl, DAG.getEntryNode(),
-                       DAG.getNode(ISD::ADD, dl, getPointerTy(),
+      DAG.getConstant(RegInfo->getSlotSize(), PtrVT);
+    return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(),
+                       DAG.getNode(ISD::ADD, dl, PtrVT,
                                    FrameAddr, Offset),
                        MachinePointerInfo(), false, false, false, 0);
   }
 
   // Just load the return address.
   SDValue RetAddrFI = getReturnAddressFrameIndex(DAG);
-  return DAG.getLoad(getPointerTy(), dl, DAG.getEntryNode(),
+  return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(),
                      RetAddrFI, MachinePointerInfo(), false, false, false, 0);
 }
 
@@ -10106,7 +10581,7 @@ SDValue X86TargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
 
 SDValue X86TargetLowering::LowerFRAME_TO_ARGS_OFFSET(SDValue Op,
                                                      SelectionDAG &DAG) const {
-  return DAG.getIntPtrConstant(2*TD->getPointerSize());
+  return DAG.getIntPtrConstant(2 * RegInfo->getSlotSize());
 }
 
 SDValue X86TargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const {
@@ -10121,7 +10596,7 @@ SDValue X86TargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const {
   unsigned StoreAddrReg = (Subtarget->is64Bit() ? X86::RCX : X86::ECX);
 
   SDValue StoreAddr = DAG.getNode(ISD::ADD, dl, getPointerTy(), Frame,
-                                  DAG.getIntPtrConstant(TD->getPointerSize()));
+                                  DAG.getIntPtrConstant(RegInfo->getSlotSize()));
   StoreAddr = DAG.getNode(ISD::ADD, dl, getPointerTy(), StoreAddr, Offset);
   Chain = DAG.getStore(Chain, dl, Handler, StoreAddr, MachinePointerInfo(),
                        false, false, 0);
@@ -10132,8 +10607,22 @@ SDValue X86TargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const {
                      Chain, DAG.getRegister(StoreAddrReg, getPointerTy()));
 }
 
-SDValue X86TargetLowering::LowerADJUST_TRAMPOLINE(SDValue Op,
-                                                  SelectionDAG &DAG) const {
+SDValue X86TargetLowering::lowerEH_SJLJ_SETJMP(SDValue Op,
+                                               SelectionDAG &DAG) const {
+  DebugLoc DL = Op.getDebugLoc();
+  return DAG.getNode(X86ISD::EH_SJLJ_SETJMP, DL,
+                     DAG.getVTList(MVT::i32, MVT::Other),
+                     Op.getOperand(0), Op.getOperand(1));
+}
+
+SDValue X86TargetLowering::lowerEH_SJLJ_LONGJMP(SDValue Op,
+                                                SelectionDAG &DAG) const {
+  DebugLoc DL = Op.getDebugLoc();
+  return DAG.getNode(X86ISD::EH_SJLJ_LONGJMP, DL, MVT::Other,
+                     Op.getOperand(0), Op.getOperand(1));
+}
+
+static SDValue LowerADJUST_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) {
   return Op.getOperand(0);
 }
 
@@ -10146,6 +10635,7 @@ SDValue X86TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
   DebugLoc dl  = Op.getDebugLoc();
 
   const Value *TrmpAddr = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
+  const TargetRegisterInfo* TRI = getTargetMachine().getRegisterInfo();
 
   if (Subtarget->is64Bit()) {
     SDValue OutChains[6];
@@ -10154,8 +10644,8 @@ SDValue X86TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
     const unsigned char JMP64r  = 0xFF; // 64-bit jmp through register opcode.
     const unsigned char MOV64ri = 0xB8; // X86::MOV64ri opcode.
 
-    const unsigned char N86R10 = X86_MC::getX86RegNum(X86::R10);
-    const unsigned char N86R11 = X86_MC::getX86RegNum(X86::R11);
+    const unsigned char N86R10 = TRI->getEncodingValue(X86::R10) & 0x7;
+    const unsigned char N86R11 = TRI->getEncodingValue(X86::R11) & 0x7;
 
     const unsigned char REX_WB = 0x40 | 0x08 | 0x01; // REX prefix
 
@@ -10228,7 +10718,7 @@ SDValue X86TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
 
         for (FunctionType::param_iterator I = FTy->param_begin(),
              E = FTy->param_end(); I != E; ++I, ++Idx)
-          if (Attrs.paramHasAttr(Idx, Attribute::InReg))
+          if (Attrs.getParamAttributes(Idx).hasAttribute(Attributes::InReg))
             // FIXME: should only count parameters that are lowered to integers.
             InRegCount += (TD->getTypeSizeInBits(*I) + 31) / 32;
 
@@ -10257,7 +10747,7 @@ SDValue X86TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
 
     // This is storing the opcode for MOV32ri.
     const unsigned char MOV32ri = 0xB8; // X86::MOV32ri's opcode byte.
-    const unsigned char N86Reg = X86_MC::getX86RegNum(NestReg);
+    const unsigned char N86Reg = TRI->getEncodingValue(NestReg) & 0x7;
     OutChains[0] = DAG.getStore(Root, dl,
                                 DAG.getConstant(MOV32ri|N86Reg, MVT::i8),
                                 Trmp, MachinePointerInfo(TrmpAddr),
@@ -10356,7 +10846,7 @@ SDValue X86TargetLowering::LowerFLT_ROUNDS_(SDValue Op,
                       ISD::TRUNCATE : ISD::ZERO_EXTEND), DL, VT, RetVal);
 }
 
-SDValue X86TargetLowering::LowerCTLZ(SDValue Op, SelectionDAG &DAG) const {
+static SDValue LowerCTLZ(SDValue Op, SelectionDAG &DAG) {
   EVT VT = Op.getValueType();
   EVT OpVT = VT;
   unsigned NumBits = VT.getSizeInBits();
@@ -10390,8 +10880,7 @@ SDValue X86TargetLowering::LowerCTLZ(SDValue Op, SelectionDAG &DAG) const {
   return Op;
 }
 
-SDValue X86TargetLowering::LowerCTLZ_ZERO_UNDEF(SDValue Op,
-                                                SelectionDAG &DAG) const {
+static SDValue LowerCTLZ_ZERO_UNDEF(SDValue Op, SelectionDAG &DAG) {
   EVT VT = Op.getValueType();
   EVT OpVT = VT;
   unsigned NumBits = VT.getSizeInBits();
@@ -10416,7 +10905,7 @@ SDValue X86TargetLowering::LowerCTLZ_ZERO_UNDEF(SDValue Op,
   return Op;
 }
 
-SDValue X86TargetLowering::LowerCTTZ(SDValue Op, SelectionDAG &DAG) const {
+static SDValue LowerCTTZ(SDValue Op, SelectionDAG &DAG) {
   EVT VT = Op.getValueType();
   unsigned NumBits = VT.getSizeInBits();
   DebugLoc dl = Op.getDebugLoc();
@@ -10465,21 +10954,22 @@ static SDValue Lower256IntArith(SDValue Op, SelectionDAG &DAG) {
                      DAG.getNode(Op.getOpcode(), dl, NewVT, LHS2, RHS2));
 }
 
-SDValue X86TargetLowering::LowerADD(SDValue Op, SelectionDAG &DAG) const {
+static SDValue LowerADD(SDValue Op, SelectionDAG &DAG) {
   assert(Op.getValueType().is256BitVector() &&
          Op.getValueType().isInteger() &&
          "Only handle AVX 256-bit vector integer operation");
   return Lower256IntArith(Op, DAG);
 }
 
-SDValue X86TargetLowering::LowerSUB(SDValue Op, SelectionDAG &DAG) const {
+static SDValue LowerSUB(SDValue Op, SelectionDAG &DAG) {
   assert(Op.getValueType().is256BitVector() &&
          Op.getValueType().isInteger() &&
          "Only handle AVX 256-bit vector integer operation");
   return Lower256IntArith(Op, DAG);
 }
 
-SDValue X86TargetLowering::LowerMUL(SDValue Op, SelectionDAG &DAG) const {
+static SDValue LowerMUL(SDValue Op, const X86Subtarget *Subtarget,
+                        SelectionDAG &DAG) {
   EVT VT = Op.getValueType();
 
   // Decompose 256-bit ops into smaller 128-bit ops.
@@ -10754,7 +11244,7 @@ SDValue X86TargetLowering::LowerShift(SDValue Op, SelectionDAG &DAG) const {
   return SDValue();
 }
 
-SDValue X86TargetLowering::LowerXALUO(SDValue Op, SelectionDAG &DAG) const {
+static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) {
   // Lower the "add/sub/mul with overflow" instruction into a regular ins plus
   // a "setcc" instruction that checks the overflow flag. The "brcond" lowering
   // looks for this combo and may remove the "setcc" instruction if the "setcc"
@@ -10869,7 +11359,7 @@ SDValue X86TargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,
         LHS1 = DAG.getNode(Op.getOpcode(), dl, NewVT, LHS1, Extra);
         LHS2 = DAG.getNode(Op.getOpcode(), dl, NewVT, LHS2, Extra);
 
-        return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, LHS1, LHS2);;
+        return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, LHS1, LHS2);
       }
       // fall through
     case MVT::v4i32:
@@ -10882,7 +11372,8 @@ SDValue X86TargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,
 }
 
 
-SDValue X86TargetLowering::LowerMEMBARRIER(SDValue Op, SelectionDAG &DAG) const{
+static SDValue LowerMEMBARRIER(SDValue Op, const X86Subtarget *Subtarget,
+                              SelectionDAG &DAG) {
   DebugLoc dl = Op.getDebugLoc();
 
   // Go ahead and emit the fence on x86-64 even if we asked for no-sse2.
@@ -10927,8 +11418,8 @@ SDValue X86TargetLowering::LowerMEMBARRIER(SDValue Op, SelectionDAG &DAG) const{
   return DAG.getNode(X86ISD::MFENCE, dl, MVT::Other, Op.getOperand(0));
 }
 
-SDValue X86TargetLowering::LowerATOMIC_FENCE(SDValue Op,
-                                             SelectionDAG &DAG) const {
+static SDValue LowerATOMIC_FENCE(SDValue Op, const X86Subtarget *Subtarget,
+                                 SelectionDAG &DAG) {
   DebugLoc dl = Op.getDebugLoc();
   AtomicOrdering FenceOrdering = static_cast<AtomicOrdering>(
     cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue());
@@ -10966,7 +11457,8 @@ SDValue X86TargetLowering::LowerATOMIC_FENCE(SDValue Op,
 }
 
 
-SDValue X86TargetLowering::LowerCMP_SWAP(SDValue Op, SelectionDAG &DAG) const {
+static SDValue LowerCMP_SWAP(SDValue Op, const X86Subtarget *Subtarget,
+                             SelectionDAG &DAG) {
   EVT T = Op.getValueType();
   DebugLoc DL = Op.getDebugLoc();
   unsigned Reg = 0;
@@ -10997,8 +11489,8 @@ SDValue X86TargetLowering::LowerCMP_SWAP(SDValue Op, SelectionDAG &DAG) const {
   return cpOut;
 }
 
-SDValue X86TargetLowering::LowerREADCYCLECOUNTER(SDValue Op,
-                                                 SelectionDAG &DAG) const {
+static SDValue LowerREADCYCLECOUNTER(SDValue Op, const X86Subtarget *Subtarget,
+                                     SelectionDAG &DAG) {
   assert(Subtarget->is64Bit() && "Result not type legalized?");
   SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
   SDValue TheChain = Op.getOperand(0);
@@ -11016,8 +11508,7 @@ SDValue X86TargetLowering::LowerREADCYCLECOUNTER(SDValue Op,
   return DAG.getMergeValues(Ops, 2, dl);
 }
 
-SDValue X86TargetLowering::LowerBITCAST(SDValue Op,
-                                            SelectionDAG &DAG) const {
+SDValue X86TargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const {
   EVT SrcVT = Op.getOperand(0).getValueType();
   EVT DstVT = Op.getValueType();
   assert(Subtarget->is64Bit() && !Subtarget->hasSSE2() &&
@@ -11037,7 +11528,7 @@ SDValue X86TargetLowering::LowerBITCAST(SDValue Op,
   return SDValue();
 }
 
-SDValue X86TargetLowering::LowerLOAD_SUB(SDValue Op, SelectionDAG &DAG) const {
+static SDValue LowerLOAD_SUB(SDValue Op, SelectionDAG &DAG) {
   SDNode *Node = Op.getNode();
   DebugLoc dl = Node->getDebugLoc();
   EVT T = Node->getValueType(0);
@@ -11110,9 +11601,9 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   switch (Op.getOpcode()) {
   default: llvm_unreachable("Should not custom lower this!");
   case ISD::SIGN_EXTEND_INREG:  return LowerSIGN_EXTEND_INREG(Op,DAG);
-  case ISD::MEMBARRIER:         return LowerMEMBARRIER(Op,DAG);
-  case ISD::ATOMIC_FENCE:       return LowerATOMIC_FENCE(Op,DAG);
-  case ISD::ATOMIC_CMP_SWAP:    return LowerCMP_SWAP(Op,DAG);
+  case ISD::MEMBARRIER:         return LowerMEMBARRIER(Op, Subtarget, DAG);
+  case ISD::ATOMIC_FENCE:       return LowerATOMIC_FENCE(Op, Subtarget, DAG);
+  case ISD::ATOMIC_CMP_SWAP:    return LowerCMP_SWAP(Op, Subtarget, DAG);
   case ISD::ATOMIC_LOAD_SUB:    return LowerLOAD_SUB(Op,DAG);
   case ISD::ATOMIC_STORE:       return LowerATOMIC_STORE(Op,DAG);
   case ISD::BUILD_VECTOR:       return LowerBUILD_VECTOR(Op, DAG);
@@ -11120,8 +11611,8 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::VECTOR_SHUFFLE:     return LowerVECTOR_SHUFFLE(Op, DAG);
   case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
   case ISD::INSERT_VECTOR_ELT:  return LowerINSERT_VECTOR_ELT(Op, DAG);
-  case ISD::EXTRACT_SUBVECTOR:  return LowerEXTRACT_SUBVECTOR(Op, DAG);
-  case ISD::INSERT_SUBVECTOR:   return LowerINSERT_SUBVECTOR(Op, DAG);
+  case ISD::EXTRACT_SUBVECTOR:  return LowerEXTRACT_SUBVECTOR(Op,Subtarget,DAG);
+  case ISD::INSERT_SUBVECTOR:   return LowerINSERT_SUBVECTOR(Op, Subtarget,DAG);
   case ISD::SCALAR_TO_VECTOR:   return LowerSCALAR_TO_VECTOR(Op, DAG);
   case ISD::ConstantPool:       return LowerConstantPool(Op, DAG);
   case ISD::GlobalAddress:      return LowerGlobalAddress(Op, DAG);
@@ -11133,8 +11624,11 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::SRL_PARTS:          return LowerShiftParts(Op, DAG);
   case ISD::SINT_TO_FP:         return LowerSINT_TO_FP(Op, DAG);
   case ISD::UINT_TO_FP:         return LowerUINT_TO_FP(Op, DAG);
+  case ISD::TRUNCATE:           return lowerTRUNCATE(Op, DAG);
+  case ISD::ZERO_EXTEND:        return lowerZERO_EXTEND(Op, DAG);
   case ISD::FP_TO_SINT:         return LowerFP_TO_SINT(Op, DAG);
   case ISD::FP_TO_UINT:         return LowerFP_TO_UINT(Op, DAG);
+  case ISD::FP_EXTEND:          return lowerFP_EXTEND(Op, DAG);
   case ISD::FABS:               return LowerFABS(Op, DAG);
   case ISD::FNEG:               return LowerFNEG(Op, DAG);
   case ISD::FCOPYSIGN:          return LowerFCOPYSIGN(Op, DAG);
@@ -11145,7 +11639,7 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::JumpTable:          return LowerJumpTable(Op, DAG);
   case ISD::VASTART:            return LowerVASTART(Op, DAG);
   case ISD::VAARG:              return LowerVAARG(Op, DAG);
-  case ISD::VACOPY:             return LowerVACOPY(Op, DAG);
+  case ISD::VACOPY:             return LowerVACOPY(Op, Subtarget, DAG);
   case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
   case ISD::INTRINSIC_W_CHAIN:  return LowerINTRINSIC_W_CHAIN(Op, DAG);
   case ISD::RETURNADDR:         return LowerRETURNADDR(Op, DAG);
@@ -11154,13 +11648,15 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
                                 return LowerFRAME_TO_ARGS_OFFSET(Op, DAG);
   case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
   case ISD::EH_RETURN:          return LowerEH_RETURN(Op, DAG);
+  case ISD::EH_SJLJ_SETJMP:     return lowerEH_SJLJ_SETJMP(Op, DAG);
+  case ISD::EH_SJLJ_LONGJMP:    return lowerEH_SJLJ_LONGJMP(Op, DAG);
   case ISD::INIT_TRAMPOLINE:    return LowerINIT_TRAMPOLINE(Op, DAG);
   case ISD::ADJUST_TRAMPOLINE:  return LowerADJUST_TRAMPOLINE(Op, DAG);
   case ISD::FLT_ROUNDS_:        return LowerFLT_ROUNDS_(Op, DAG);
   case ISD::CTLZ:               return LowerCTLZ(Op, DAG);
   case ISD::CTLZ_ZERO_UNDEF:    return LowerCTLZ_ZERO_UNDEF(Op, DAG);
   case ISD::CTTZ:               return LowerCTTZ(Op, DAG);
-  case ISD::MUL:                return LowerMUL(Op, DAG);
+  case ISD::MUL:                return LowerMUL(Op, Subtarget, DAG);
   case ISD::SRA:
   case ISD::SRL:
   case ISD::SHL:                return LowerShift(Op, DAG);
@@ -11170,7 +11666,7 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
   case ISD::USUBO:
   case ISD::SMULO:
   case ISD::UMULO:              return LowerXALUO(Op, DAG);
-  case ISD::READCYCLECOUNTER:   return LowerREADCYCLECOUNTER(Op, DAG);
+  case ISD::READCYCLECOUNTER:   return LowerREADCYCLECOUNTER(Op, Subtarget,DAG);
   case ISD::BITCAST:            return LowerBITCAST(Op, DAG);
   case ISD::ADDC:
   case ISD::ADDE:
@@ -11263,6 +11759,27 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
     }
     return;
   }
+  case ISD::UINT_TO_FP: {
+    if (N->getOperand(0).getValueType() != MVT::v2i32 &&
+        N->getValueType(0) != MVT::v2f32)
+      return;
+    SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v2i64,
+                                 N->getOperand(0));
+    SDValue Bias = DAG.getConstantFP(BitsToDouble(0x4330000000000000ULL),
+                                     MVT::f64);
+    SDValue VBias = DAG.getNode(ISD::BUILD_VECTOR, dl, MVT::v2f64, Bias, Bias);
+    SDValue Or = DAG.getNode(ISD::OR, dl, MVT::v2i64, ZExtIn,
+                             DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, VBias));
+    Or = DAG.getNode(ISD::BITCAST, dl, MVT::v2f64, Or);
+    SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, Or, VBias);
+    Results.push_back(DAG.getNode(X86ISD::VFPROUND, dl, MVT::v4f32, Sub));
+    return;
+  }
+  case ISD::FP_ROUND: {
+    SDValue V = DAG.getNode(X86ISD::VFPROUND, dl, MVT::v4f32, N->getOperand(0));
+    Results.push_back(V);
+    return;
+  }
   case ISD::READCYCLECOUNTER: {
     SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
     SDValue TheChain = N->getOperand(0);
@@ -11330,6 +11847,10 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
   case ISD::ATOMIC_LOAD_OR:
   case ISD::ATOMIC_LOAD_SUB:
   case ISD::ATOMIC_LOAD_XOR:
+  case ISD::ATOMIC_LOAD_MAX:
+  case ISD::ATOMIC_LOAD_MIN:
+  case ISD::ATOMIC_LOAD_UMAX:
+  case ISD::ATOMIC_LOAD_UMIN:
   case ISD::ATOMIC_SWAP: {
     unsigned Opc;
     switch (N->getOpcode()) {
@@ -11352,6 +11873,18 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
     case ISD::ATOMIC_LOAD_XOR:
       Opc = X86ISD::ATOMXOR64_DAG;
       break;
+    case ISD::ATOMIC_LOAD_MAX:
+      Opc = X86ISD::ATOMMAX64_DAG;
+      break;
+    case ISD::ATOMIC_LOAD_MIN:
+      Opc = X86ISD::ATOMMIN64_DAG;
+      break;
+    case ISD::ATOMIC_LOAD_UMAX:
+      Opc = X86ISD::ATOMUMAX64_DAG;
+      break;
+    case ISD::ATOMIC_LOAD_UMIN:
+      Opc = X86ISD::ATOMUMIN64_DAG;
+      break;
     case ISD::ATOMIC_SWAP:
       Opc = X86ISD::ATOMSWAP64_DAG;
       break;
@@ -11418,11 +11951,15 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
   case X86ISD::FHSUB:              return "X86ISD::FHSUB";
   case X86ISD::FMAX:               return "X86ISD::FMAX";
   case X86ISD::FMIN:               return "X86ISD::FMIN";
+  case X86ISD::FMAXC:              return "X86ISD::FMAXC";
+  case X86ISD::FMINC:              return "X86ISD::FMINC";
   case X86ISD::FRSQRT:             return "X86ISD::FRSQRT";
   case X86ISD::FRCP:               return "X86ISD::FRCP";
   case X86ISD::TLSADDR:            return "X86ISD::TLSADDR";
   case X86ISD::TLSBASEADDR:        return "X86ISD::TLSBASEADDR";
   case X86ISD::TLSCALL:            return "X86ISD::TLSCALL";
+  case X86ISD::EH_SJLJ_SETJMP:     return "X86ISD::EH_SJLJ_SETJMP";
+  case X86ISD::EH_SJLJ_LONGJMP:    return "X86ISD::EH_SJLJ_LONGJMP";
   case X86ISD::EH_RETURN:          return "X86ISD::EH_RETURN";
   case X86ISD::TC_RETURN:          return "X86ISD::TC_RETURN";
   case X86ISD::FNSTCW16m:          return "X86ISD::FNSTCW16m";
@@ -11438,7 +11975,10 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
   case X86ISD::VZEXT_MOVL:         return "X86ISD::VZEXT_MOVL";
   case X86ISD::VSEXT_MOVL:         return "X86ISD::VSEXT_MOVL";
   case X86ISD::VZEXT_LOAD:         return "X86ISD::VZEXT_LOAD";
+  case X86ISD::VZEXT:              return "X86ISD::VZEXT";
+  case X86ISD::VSEXT:              return "X86ISD::VSEXT";
   case X86ISD::VFPEXT:             return "X86ISD::VFPEXT";
+  case X86ISD::VFPROUND:           return "X86ISD::VFPROUND";
   case X86ISD::VSHLDQ:             return "X86ISD::VSHLDQ";
   case X86ISD::VSRLDQ:             return "X86ISD::VSRLDQ";
   case X86ISD::VSHL:               return "X86ISD::VSHL";
@@ -11505,6 +12045,8 @@ const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
   case X86ISD::FNMSUB:             return "X86ISD::FNMSUB";
   case X86ISD::FMADDSUB:           return "X86ISD::FMADDSUB";
   case X86ISD::FMSUBADD:           return "X86ISD::FMSUBADD";
+  case X86ISD::PCMPESTRI:          return "X86ISD::PCMPESTRI";
+  case X86ISD::PCMPISTRI:          return "X86ISD::PCMPISTRI";
   }
 }
 
@@ -11653,430 +12195,724 @@ X86TargetLowering::isVectorClearMaskLegal(const SmallVectorImpl<int> &Mask,
 //                           X86 Scheduler Hooks
 //===----------------------------------------------------------------------===//
 
-// private utility function
-MachineBasicBlock *
-X86TargetLowering::EmitAtomicBitwiseWithCustomInserter(MachineInstr *bInstr,
-                                                       MachineBasicBlock *MBB,
-                                                       unsigned regOpc,
-                                                       unsigned immOpc,
-                                                       unsigned LoadOpc,
-                                                       unsigned CXchgOpc,
-                                                       unsigned notOpc,
-                                                       unsigned EAXreg,
-                                                 const TargetRegisterClass *RC,
-                                                       bool Invert) const {
-  // For the atomic bitwise operator, we generate
-  //   thisMBB:
-  //   newMBB:
-  //     ld  t1 = [bitinstr.addr]
-  //     op  t2 = t1, [bitinstr.val]
-  //     not t3 = t2  (if Invert)
-  //     mov EAX = t1
-  //     lcs dest = [bitinstr.addr], t3  [EAX is implicit]
-  //     bz  newMBB
-  //     fallthrough -->nextMBB
-  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
-  const BasicBlock *LLVM_BB = MBB->getBasicBlock();
-  MachineFunction::iterator MBBIter = MBB;
-  ++MBBIter;
+/// Utility function to emit xbegin specifying the start of an RTM region.
+static MachineBasicBlock *EmitXBegin(MachineInstr *MI, MachineBasicBlock *MBB,
+                                     const TargetInstrInfo *TII) {
+  DebugLoc DL = MI->getDebugLoc();
+
+  const BasicBlock *BB = MBB->getBasicBlock();
+  MachineFunction::iterator I = MBB;
+  ++I;
+
+  // For the v = xbegin(), we generate
+  //
+  // thisMBB:
+  //  xbegin sinkMBB
+  //
+  // mainMBB:
+  //  eax = -1
+  //
+  // sinkMBB:
+  //  v = eax
 
-  /// First build the CFG
-  MachineFunction *F = MBB->getParent();
   MachineBasicBlock *thisMBB = MBB;
-  MachineBasicBlock *newMBB = F->CreateMachineBasicBlock(LLVM_BB);
-  MachineBasicBlock *nextMBB = F->CreateMachineBasicBlock(LLVM_BB);
-  F->insert(MBBIter, newMBB);
-  F->insert(MBBIter, nextMBB);
-
-  // Transfer the remainder of thisMBB and its successor edges to nextMBB.
-  nextMBB->splice(nextMBB->begin(), thisMBB,
-                  llvm::next(MachineBasicBlock::iterator(bInstr)),
-                  thisMBB->end());
-  nextMBB->transferSuccessorsAndUpdatePHIs(thisMBB);
-
-  // Update thisMBB to fall through to newMBB
-  thisMBB->addSuccessor(newMBB);
-
-  // newMBB jumps to itself and fall through to nextMBB
-  newMBB->addSuccessor(nextMBB);
-  newMBB->addSuccessor(newMBB);
-
-  // Insert instructions into newMBB based on incoming instruction
-  assert(bInstr->getNumOperands() < X86::AddrNumOperands + 4 &&
-         "unexpected number of operands");
-  DebugLoc dl = bInstr->getDebugLoc();
-  MachineOperand& destOper = bInstr->getOperand(0);
-  MachineOperand* argOpers[2 + X86::AddrNumOperands];
-  int numArgs = bInstr->getNumOperands() - 1;
-  for (int i=0; i < numArgs; ++i)
-    argOpers[i] = &bInstr->getOperand(i+1);
-
-  // x86 address has 4 operands: base, index, scale, and displacement
-  int lastAddrIndx = X86::AddrNumOperands - 1; // [0,3]
-  int valArgIndx = lastAddrIndx + 1;
-
-  unsigned t1 = F->getRegInfo().createVirtualRegister(RC);
-  MachineInstrBuilder MIB = BuildMI(newMBB, dl, TII->get(LoadOpc), t1);
-  for (int i=0; i <= lastAddrIndx; ++i)
-    (*MIB).addOperand(*argOpers[i]);
-
-  unsigned t2 = F->getRegInfo().createVirtualRegister(RC);
-  assert((argOpers[valArgIndx]->isReg() ||
-          argOpers[valArgIndx]->isImm()) &&
-         "invalid operand");
-  if (argOpers[valArgIndx]->isReg())
-    MIB = BuildMI(newMBB, dl, TII->get(regOpc), t2);
-  else
-    MIB = BuildMI(newMBB, dl, TII->get(immOpc), t2);
-  MIB.addReg(t1);
-  (*MIB).addOperand(*argOpers[valArgIndx]);
+  MachineFunction *MF = MBB->getParent();
+  MachineBasicBlock *mainMBB = MF->CreateMachineBasicBlock(BB);
+  MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB);
+  MF->insert(I, mainMBB);
+  MF->insert(I, sinkMBB);
 
-  unsigned t3 = F->getRegInfo().createVirtualRegister(RC);
-  if (Invert) {
-    MIB = BuildMI(newMBB, dl, TII->get(notOpc), t3).addReg(t2);
-  }
-  else
-    t3 = t2;
+  // Transfer the remainder of BB and its successor edges to sinkMBB.
+  sinkMBB->splice(sinkMBB->begin(), MBB,
+                  llvm::next(MachineBasicBlock::iterator(MI)), MBB->end());
+  sinkMBB->transferSuccessorsAndUpdatePHIs(MBB);
+
+  // thisMBB:
+  //  xbegin sinkMBB
+  //  # fallthrough to mainMBB
+  //  # abortion to sinkMBB
+  BuildMI(thisMBB, DL, TII->get(X86::XBEGIN_4)).addMBB(sinkMBB);
+  thisMBB->addSuccessor(mainMBB);
+  thisMBB->addSuccessor(sinkMBB);
+
+  // mainMBB:
+  //  EAX = -1
+  BuildMI(mainMBB, DL, TII->get(X86::MOV32ri), X86::EAX).addImm(-1);
+  mainMBB->addSuccessor(sinkMBB);
+
+  // sinkMBB:
+  // EAX is live into the sinkMBB
+  sinkMBB->addLiveIn(X86::EAX);
+  BuildMI(*sinkMBB, sinkMBB->begin(), DL,
+          TII->get(TargetOpcode::COPY), MI->getOperand(0).getReg())
+    .addReg(X86::EAX);
 
-  MIB = BuildMI(newMBB, dl, TII->get(TargetOpcode::COPY), EAXreg);
-  MIB.addReg(t1);
+  MI->eraseFromParent();
+  return sinkMBB;
+}
 
-  MIB = BuildMI(newMBB, dl, TII->get(CXchgOpc));
-  for (int i=0; i <= lastAddrIndx; ++i)
-    (*MIB).addOperand(*argOpers[i]);
-  MIB.addReg(t3);
-  assert(bInstr->hasOneMemOperand() && "Unexpected number of memoperand");
-  (*MIB).setMemRefs(bInstr->memoperands_begin(),
-                    bInstr->memoperands_end());
+// Get CMPXCHG opcode for the specified data type.
+static unsigned getCmpXChgOpcode(EVT VT) {
+  switch (VT.getSimpleVT().SimpleTy) {
+  case MVT::i8:  return X86::LCMPXCHG8;
+  case MVT::i16: return X86::LCMPXCHG16;
+  case MVT::i32: return X86::LCMPXCHG32;
+  case MVT::i64: return X86::LCMPXCHG64;
+  default:
+    break;
+  }
+  llvm_unreachable("Invalid operand size!");
+}
 
-  MIB = BuildMI(newMBB, dl, TII->get(TargetOpcode::COPY), destOper.getReg());
-  MIB.addReg(EAXreg);
+// Get LOAD opcode for the specified data type.
+static unsigned getLoadOpcode(EVT VT) {
+  switch (VT.getSimpleVT().SimpleTy) {
+  case MVT::i8:  return X86::MOV8rm;
+  case MVT::i16: return X86::MOV16rm;
+  case MVT::i32: return X86::MOV32rm;
+  case MVT::i64: return X86::MOV64rm;
+  default:
+    break;
+  }
+  llvm_unreachable("Invalid operand size!");
+}
 
-  // insert branch
-  BuildMI(newMBB, dl, TII->get(X86::JNE_4)).addMBB(newMBB);
+// Get opcode of the non-atomic one from the specified atomic instruction.
+static unsigned getNonAtomicOpcode(unsigned Opc) {
+  switch (Opc) {
+  case X86::ATOMAND8:  return X86::AND8rr;
+  case X86::ATOMAND16: return X86::AND16rr;
+  case X86::ATOMAND32: return X86::AND32rr;
+  case X86::ATOMAND64: return X86::AND64rr;
+  case X86::ATOMOR8:   return X86::OR8rr;
+  case X86::ATOMOR16:  return X86::OR16rr;
+  case X86::ATOMOR32:  return X86::OR32rr;
+  case X86::ATOMOR64:  return X86::OR64rr;
+  case X86::ATOMXOR8:  return X86::XOR8rr;
+  case X86::ATOMXOR16: return X86::XOR16rr;
+  case X86::ATOMXOR32: return X86::XOR32rr;
+  case X86::ATOMXOR64: return X86::XOR64rr;
+  }
+  llvm_unreachable("Unhandled atomic-load-op opcode!");
+}
+
+// Get opcode of the non-atomic one from the specified atomic instruction with
+// extra opcode.
+static unsigned getNonAtomicOpcodeWithExtraOpc(unsigned Opc,
+                                               unsigned &ExtraOpc) {
+  switch (Opc) {
+  case X86::ATOMNAND8:  ExtraOpc = X86::NOT8r;   return X86::AND8rr;
+  case X86::ATOMNAND16: ExtraOpc = X86::NOT16r;  return X86::AND16rr;
+  case X86::ATOMNAND32: ExtraOpc = X86::NOT32r;  return X86::AND32rr;
+  case X86::ATOMNAND64: ExtraOpc = X86::NOT64r;  return X86::AND64rr;
+  case X86::ATOMMAX8:   ExtraOpc = X86::CMP8rr;  return X86::CMOVL32rr;
+  case X86::ATOMMAX16:  ExtraOpc = X86::CMP16rr; return X86::CMOVL16rr;
+  case X86::ATOMMAX32:  ExtraOpc = X86::CMP32rr; return X86::CMOVL32rr;
+  case X86::ATOMMAX64:  ExtraOpc = X86::CMP64rr; return X86::CMOVL64rr;
+  case X86::ATOMMIN8:   ExtraOpc = X86::CMP8rr;  return X86::CMOVG32rr;
+  case X86::ATOMMIN16:  ExtraOpc = X86::CMP16rr; return X86::CMOVG16rr;
+  case X86::ATOMMIN32:  ExtraOpc = X86::CMP32rr; return X86::CMOVG32rr;
+  case X86::ATOMMIN64:  ExtraOpc = X86::CMP64rr; return X86::CMOVG64rr;
+  case X86::ATOMUMAX8:  ExtraOpc = X86::CMP8rr;  return X86::CMOVB32rr;
+  case X86::ATOMUMAX16: ExtraOpc = X86::CMP16rr; return X86::CMOVB16rr;
+  case X86::ATOMUMAX32: ExtraOpc = X86::CMP32rr; return X86::CMOVB32rr;
+  case X86::ATOMUMAX64: ExtraOpc = X86::CMP64rr; return X86::CMOVB64rr;
+  case X86::ATOMUMIN8:  ExtraOpc = X86::CMP8rr;  return X86::CMOVA32rr;
+  case X86::ATOMUMIN16: ExtraOpc = X86::CMP16rr; return X86::CMOVA16rr;
+  case X86::ATOMUMIN32: ExtraOpc = X86::CMP32rr; return X86::CMOVA32rr;
+  case X86::ATOMUMIN64: ExtraOpc = X86::CMP64rr; return X86::CMOVA64rr;
+  }
+  llvm_unreachable("Unhandled atomic-load-op opcode!");
+}
+
+// Get opcode of the non-atomic one from the specified atomic instruction for
+// 64-bit data type on 32-bit target.
+static unsigned getNonAtomic6432Opcode(unsigned Opc, unsigned &HiOpc) {
+  switch (Opc) {
+  case X86::ATOMAND6432:  HiOpc = X86::AND32rr; return X86::AND32rr;
+  case X86::ATOMOR6432:   HiOpc = X86::OR32rr;  return X86::OR32rr;
+  case X86::ATOMXOR6432:  HiOpc = X86::XOR32rr; return X86::XOR32rr;
+  case X86::ATOMADD6432:  HiOpc = X86::ADC32rr; return X86::ADD32rr;
+  case X86::ATOMSUB6432:  HiOpc = X86::SBB32rr; return X86::SUB32rr;
+  case X86::ATOMSWAP6432: HiOpc = X86::MOV32rr; return X86::MOV32rr;
+  case X86::ATOMMAX6432:  HiOpc = X86::SETLr;   return X86::SETLr;
+  case X86::ATOMMIN6432:  HiOpc = X86::SETGr;   return X86::SETGr;
+  case X86::ATOMUMAX6432: HiOpc = X86::SETBr;   return X86::SETBr;
+  case X86::ATOMUMIN6432: HiOpc = X86::SETAr;   return X86::SETAr;
+  }
+  llvm_unreachable("Unhandled atomic-load-op opcode!");
+}
+
+// Get opcode of the non-atomic one from the specified atomic instruction for
+// 64-bit data type on 32-bit target with extra opcode.
+static unsigned getNonAtomic6432OpcodeWithExtraOpc(unsigned Opc,
+                                                   unsigned &HiOpc,
+                                                   unsigned &ExtraOpc) {
+  switch (Opc) {
+  case X86::ATOMNAND6432:
+    ExtraOpc = X86::NOT32r;
+    HiOpc = X86::AND32rr;
+    return X86::AND32rr;
+  }
+  llvm_unreachable("Unhandled atomic-load-op opcode!");
+}
 
-  bInstr->eraseFromParent();   // The pseudo instruction is gone now.
-  return nextMBB;
+// Get pseudo CMOV opcode from the specified data type.
+static unsigned getPseudoCMOVOpc(EVT VT) {
+  switch (VT.getSimpleVT().SimpleTy) {
+  case MVT::i8:  return X86::CMOV_GR8;
+  case MVT::i16: return X86::CMOV_GR16;
+  case MVT::i32: return X86::CMOV_GR32;
+  default:
+    break;
+  }
+  llvm_unreachable("Unknown CMOV opcode!");
 }
 
-// private utility function:  64 bit atomics on 32 bit host.
+// EmitAtomicLoadArith - emit the code sequence for pseudo atomic instructions.
+// They will be translated into a spin-loop or compare-exchange loop from
+//
+//    ...
+//    dst = atomic-fetch-op MI.addr, MI.val
+//    ...
+//
+// to
+//
+//    ...
+//    EAX = LOAD MI.addr
+// loop:
+//    t1 = OP MI.val, EAX
+//    LCMPXCHG [MI.addr], t1, [EAX is implicitly used & defined]
+//    JNE loop
+// sink:
+//    dst = EAX
+//    ...
 MachineBasicBlock *
-X86TargetLowering::EmitAtomicBit6432WithCustomInserter(MachineInstr *bInstr,
-                                                       MachineBasicBlock *MBB,
-                                                       unsigned regOpcL,
-                                                       unsigned regOpcH,
-                                                       unsigned immOpcL,
-                                                       unsigned immOpcH,
-                                                       bool Invert) const {
-  // For the atomic bitwise operator, we generate
-  //   thisMBB (instructions are in pairs, except cmpxchg8b)
-  //     ld t1,t2 = [bitinstr.addr]
-  //   newMBB:
-  //     out1, out2 = phi (thisMBB, t1/t2) (newMBB, t3/t4)
-  //     op  t5, t6 <- out1, out2, [bitinstr.val]
-  //      (for SWAP, substitute:  mov t5, t6 <- [bitinstr.val])
-  //     neg t7, t8 < t5, t6  (if Invert)
-  //     mov ECX, EBX <- t5, t6
-  //     mov EAX, EDX <- t1, t2
-  //     cmpxchg8b [bitinstr.addr]  [EAX, EDX, EBX, ECX implicit]
-  //     mov t3, t4 <- EAX, EDX
-  //     bz  newMBB
-  //     result in out1, out2
-  //     fallthrough -->nextMBB
-
-  const TargetRegisterClass *RC = &X86::GR32RegClass;
-  const unsigned LoadOpc = X86::MOV32rm;
-  const unsigned NotOpc = X86::NOT32r;
+X86TargetLowering::EmitAtomicLoadArith(MachineInstr *MI,
+                                       MachineBasicBlock *MBB) const {
   const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
-  const BasicBlock *LLVM_BB = MBB->getBasicBlock();
-  MachineFunction::iterator MBBIter = MBB;
-  ++MBBIter;
+  DebugLoc DL = MI->getDebugLoc();
+
+  MachineFunction *MF = MBB->getParent();
+  MachineRegisterInfo &MRI = MF->getRegInfo();
+
+  const BasicBlock *BB = MBB->getBasicBlock();
+  MachineFunction::iterator I = MBB;
+  ++I;
+
+  assert(MI->getNumOperands() <= X86::AddrNumOperands + 2 &&
+         "Unexpected number of operands");
+
+  assert(MI->hasOneMemOperand() &&
+         "Expected atomic-load-op to have one memoperand");
+
+  // Memory Reference
+  MachineInstr::mmo_iterator MMOBegin = MI->memoperands_begin();
+  MachineInstr::mmo_iterator MMOEnd = MI->memoperands_end();
+
+  unsigned DstReg, SrcReg;
+  unsigned MemOpndSlot;
+
+  unsigned CurOp = 0;
+
+  DstReg = MI->getOperand(CurOp++).getReg();
+  MemOpndSlot = CurOp;
+  CurOp += X86::AddrNumOperands;
+  SrcReg = MI->getOperand(CurOp++).getReg();
+
+  const TargetRegisterClass *RC = MRI.getRegClass(DstReg);
+  MVT::SimpleValueType VT = *RC->vt_begin();
+  unsigned AccPhyReg = getX86SubSuperRegister(X86::EAX, VT);
+
+  unsigned LCMPXCHGOpc = getCmpXChgOpcode(VT);
+  unsigned LOADOpc = getLoadOpcode(VT);
+
+  // For the atomic load-arith operator, we generate
+  //
+  //  thisMBB:
+  //    EAX = LOAD [MI.addr]
+  //  mainMBB:
+  //    t1 = OP MI.val, EAX
+  //    LCMPXCHG [MI.addr], t1, [EAX is implicitly used & defined]
+  //    JNE mainMBB
+  //  sinkMBB:
 
-  /// First build the CFG
-  MachineFunction *F = MBB->getParent();
   MachineBasicBlock *thisMBB = MBB;
-  MachineBasicBlock *newMBB = F->CreateMachineBasicBlock(LLVM_BB);
-  MachineBasicBlock *nextMBB = F->CreateMachineBasicBlock(LLVM_BB);
-  F->insert(MBBIter, newMBB);
-  F->insert(MBBIter, nextMBB);
-
-  // Transfer the remainder of thisMBB and its successor edges to nextMBB.
-  nextMBB->splice(nextMBB->begin(), thisMBB,
-                  llvm::next(MachineBasicBlock::iterator(bInstr)),
-                  thisMBB->end());
-  nextMBB->transferSuccessorsAndUpdatePHIs(thisMBB);
-
-  // Update thisMBB to fall through to newMBB
-  thisMBB->addSuccessor(newMBB);
-
-  // newMBB jumps to itself and fall through to nextMBB
-  newMBB->addSuccessor(nextMBB);
-  newMBB->addSuccessor(newMBB);
-
-  DebugLoc dl = bInstr->getDebugLoc();
-  // Insert instructions into newMBB based on incoming instruction
-  // There are 8 "real" operands plus 9 implicit def/uses, ignored here.
-  assert(bInstr->getNumOperands() < X86::AddrNumOperands + 14 &&
-         "unexpected number of operands");
-  MachineOperand& dest1Oper = bInstr->getOperand(0);
-  MachineOperand& dest2Oper = bInstr->getOperand(1);
-  MachineOperand* argOpers[2 + X86::AddrNumOperands];
-  for (int i=0; i < 2 + X86::AddrNumOperands; ++i) {
-    argOpers[i] = &bInstr->getOperand(i+2);
-
-    // We use some of the operands multiple times, so conservatively just
-    // clear any kill flags that might be present.
-    if (argOpers[i]->isReg() && argOpers[i]->isUse())
-      argOpers[i]->setIsKill(false);
-  }
-
-  // x86 address has 5 operands: base, index, scale, displacement, and segment.
-  int lastAddrIndx = X86::AddrNumOperands - 1; // [0,3]
-
-  unsigned t1 = F->getRegInfo().createVirtualRegister(RC);
-  MachineInstrBuilder MIB = BuildMI(thisMBB, dl, TII->get(LoadOpc), t1);
-  for (int i=0; i <= lastAddrIndx; ++i)
-    (*MIB).addOperand(*argOpers[i]);
-  unsigned t2 = F->getRegInfo().createVirtualRegister(RC);
-  MIB = BuildMI(thisMBB, dl, TII->get(LoadOpc), t2);
-  // add 4 to displacement.
-  for (int i=0; i <= lastAddrIndx-2; ++i)
-    (*MIB).addOperand(*argOpers[i]);
-  MachineOperand newOp3 = *(argOpers[3]);
-  if (newOp3.isImm())
-    newOp3.setImm(newOp3.getImm()+4);
-  else
-    newOp3.setOffset(newOp3.getOffset()+4);
-  (*MIB).addOperand(newOp3);
-  (*MIB).addOperand(*argOpers[lastAddrIndx]);
-
-  // t3/4 are defined later, at the bottom of the loop
-  unsigned t3 = F->getRegInfo().createVirtualRegister(RC);
-  unsigned t4 = F->getRegInfo().createVirtualRegister(RC);
-  BuildMI(newMBB, dl, TII->get(X86::PHI), dest1Oper.getReg())
-    .addReg(t1).addMBB(thisMBB).addReg(t3).addMBB(newMBB);
-  BuildMI(newMBB, dl, TII->get(X86::PHI), dest2Oper.getReg())
-    .addReg(t2).addMBB(thisMBB).addReg(t4).addMBB(newMBB);
-
-  // The subsequent operations should be using the destination registers of
-  // the PHI instructions.
-  t1 = dest1Oper.getReg();
-  t2 = dest2Oper.getReg();
-
-  int valArgIndx = lastAddrIndx + 1;
-  assert((argOpers[valArgIndx]->isReg() ||
-          argOpers[valArgIndx]->isImm()) &&
-         "invalid operand");
-  unsigned t5 = F->getRegInfo().createVirtualRegister(RC);
-  unsigned t6 = F->getRegInfo().createVirtualRegister(RC);
-  if (argOpers[valArgIndx]->isReg())
-    MIB = BuildMI(newMBB, dl, TII->get(regOpcL), t5);
-  else
-    MIB = BuildMI(newMBB, dl, TII->get(immOpcL), t5);
-  if (regOpcL != X86::MOV32rr)
-    MIB.addReg(t1);
-  (*MIB).addOperand(*argOpers[valArgIndx]);
-  assert(argOpers[valArgIndx + 1]->isReg() ==
-         argOpers[valArgIndx]->isReg());
-  assert(argOpers[valArgIndx + 1]->isImm() ==
-         argOpers[valArgIndx]->isImm());
-  if (argOpers[valArgIndx + 1]->isReg())
-    MIB = BuildMI(newMBB, dl, TII->get(regOpcH), t6);
-  else
-    MIB = BuildMI(newMBB, dl, TII->get(immOpcH), t6);
-  if (regOpcH != X86::MOV32rr)
-    MIB.addReg(t2);
-  (*MIB).addOperand(*argOpers[valArgIndx + 1]);
-
-  unsigned t7, t8;
-  if (Invert) {
-    t7 = F->getRegInfo().createVirtualRegister(RC);
-    t8 = F->getRegInfo().createVirtualRegister(RC);
-    MIB = BuildMI(newMBB, dl, TII->get(NotOpc), t7).addReg(t5);
-    MIB = BuildMI(newMBB, dl, TII->get(NotOpc), t8).addReg(t6);
-  } else {
-    t7 = t5;
-    t8 = t6;
+  MachineBasicBlock *mainMBB = MF->CreateMachineBasicBlock(BB);
+  MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB);
+  MF->insert(I, mainMBB);
+  MF->insert(I, sinkMBB);
+
+  MachineInstrBuilder MIB;
+
+  // Transfer the remainder of BB and its successor edges to sinkMBB.
+  sinkMBB->splice(sinkMBB->begin(), MBB,
+                  llvm::next(MachineBasicBlock::iterator(MI)), MBB->end());
+  sinkMBB->transferSuccessorsAndUpdatePHIs(MBB);
+
+  // thisMBB:
+  MIB = BuildMI(thisMBB, DL, TII->get(LOADOpc), AccPhyReg);
+  for (unsigned i = 0; i < X86::AddrNumOperands; ++i)
+    MIB.addOperand(MI->getOperand(MemOpndSlot + i));
+  MIB.setMemRefs(MMOBegin, MMOEnd);
+
+  thisMBB->addSuccessor(mainMBB);
+
+  // mainMBB:
+  MachineBasicBlock *origMainMBB = mainMBB;
+  mainMBB->addLiveIn(AccPhyReg);
+
+  // Copy AccPhyReg as it is used more than once.
+  unsigned AccReg = MRI.createVirtualRegister(RC);
+  BuildMI(mainMBB, DL, TII->get(TargetOpcode::COPY), AccReg)
+    .addReg(AccPhyReg);
+
+  unsigned t1 = MRI.createVirtualRegister(RC);
+  unsigned Opc = MI->getOpcode();
+  switch (Opc) {
+  default:
+    llvm_unreachable("Unhandled atomic-load-op opcode!");
+  case X86::ATOMAND8:
+  case X86::ATOMAND16:
+  case X86::ATOMAND32:
+  case X86::ATOMAND64:
+  case X86::ATOMOR8:
+  case X86::ATOMOR16:
+  case X86::ATOMOR32:
+  case X86::ATOMOR64:
+  case X86::ATOMXOR8:
+  case X86::ATOMXOR16:
+  case X86::ATOMXOR32:
+  case X86::ATOMXOR64: {
+    unsigned ARITHOpc = getNonAtomicOpcode(Opc);
+    BuildMI(mainMBB, DL, TII->get(ARITHOpc), t1).addReg(SrcReg)
+      .addReg(AccReg);
+    break;
+  }
+  case X86::ATOMNAND8:
+  case X86::ATOMNAND16:
+  case X86::ATOMNAND32:
+  case X86::ATOMNAND64: {
+    unsigned t2 = MRI.createVirtualRegister(RC);
+    unsigned NOTOpc;
+    unsigned ANDOpc = getNonAtomicOpcodeWithExtraOpc(Opc, NOTOpc);
+    BuildMI(mainMBB, DL, TII->get(ANDOpc), t2).addReg(SrcReg)
+      .addReg(AccReg);
+    BuildMI(mainMBB, DL, TII->get(NOTOpc), t1).addReg(t2);
+    break;
+  }
+  case X86::ATOMMAX8:
+  case X86::ATOMMAX16:
+  case X86::ATOMMAX32:
+  case X86::ATOMMAX64:
+  case X86::ATOMMIN8:
+  case X86::ATOMMIN16:
+  case X86::ATOMMIN32:
+  case X86::ATOMMIN64:
+  case X86::ATOMUMAX8:
+  case X86::ATOMUMAX16:
+  case X86::ATOMUMAX32:
+  case X86::ATOMUMAX64:
+  case X86::ATOMUMIN8:
+  case X86::ATOMUMIN16:
+  case X86::ATOMUMIN32:
+  case X86::ATOMUMIN64: {
+    unsigned CMPOpc;
+    unsigned CMOVOpc = getNonAtomicOpcodeWithExtraOpc(Opc, CMPOpc);
+
+    BuildMI(mainMBB, DL, TII->get(CMPOpc))
+      .addReg(SrcReg)
+      .addReg(AccReg);
+
+    if (Subtarget->hasCMov()) {
+      if (VT != MVT::i8) {
+        // Native support
+        BuildMI(mainMBB, DL, TII->get(CMOVOpc), t1)
+          .addReg(SrcReg)
+          .addReg(AccReg);
+      } else {
+        // Promote i8 to i32 to use CMOV32
+        const TargetRegisterClass *RC32 = getRegClassFor(MVT::i32);
+        unsigned SrcReg32 = MRI.createVirtualRegister(RC32);
+        unsigned AccReg32 = MRI.createVirtualRegister(RC32);
+        unsigned t2 = MRI.createVirtualRegister(RC32);
+
+        unsigned Undef = MRI.createVirtualRegister(RC32);
+        BuildMI(mainMBB, DL, TII->get(TargetOpcode::IMPLICIT_DEF), Undef);
+
+        BuildMI(mainMBB, DL, TII->get(TargetOpcode::INSERT_SUBREG), SrcReg32)
+          .addReg(Undef)
+          .addReg(SrcReg)
+          .addImm(X86::sub_8bit);
+        BuildMI(mainMBB, DL, TII->get(TargetOpcode::INSERT_SUBREG), AccReg32)
+          .addReg(Undef)
+          .addReg(AccReg)
+          .addImm(X86::sub_8bit);
+
+        BuildMI(mainMBB, DL, TII->get(CMOVOpc), t2)
+          .addReg(SrcReg32)
+          .addReg(AccReg32);
+
+        BuildMI(mainMBB, DL, TII->get(TargetOpcode::COPY), t1)
+          .addReg(t2, 0, X86::sub_8bit);
+      }
+    } else {
+      // Use pseudo select and lower them.
+      assert((VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32) &&
+             "Invalid atomic-load-op transformation!");
+      unsigned SelOpc = getPseudoCMOVOpc(VT);
+      X86::CondCode CC = X86::getCondFromCMovOpc(CMOVOpc);
+      assert(CC != X86::COND_INVALID && "Invalid atomic-load-op transformation!");
+      MIB = BuildMI(mainMBB, DL, TII->get(SelOpc), t1)
+              .addReg(SrcReg).addReg(AccReg)
+              .addImm(CC);
+      mainMBB = EmitLoweredSelect(MIB, mainMBB);
+    }
+    break;
+  }
   }
 
-  MIB = BuildMI(newMBB, dl, TII->get(TargetOpcode::COPY), X86::EAX);
-  MIB.addReg(t1);
-  MIB = BuildMI(newMBB, dl, TII->get(TargetOpcode::COPY), X86::EDX);
-  MIB.addReg(t2);
+  // Copy AccPhyReg back from virtual register.
+  BuildMI(mainMBB, DL, TII->get(TargetOpcode::COPY), AccPhyReg)
+    .addReg(AccReg);
 
-  MIB = BuildMI(newMBB, dl, TII->get(TargetOpcode::COPY), X86::EBX);
-  MIB.addReg(t7);
-  MIB = BuildMI(newMBB, dl, TII->get(TargetOpcode::COPY), X86::ECX);
-  MIB.addReg(t8);
+  MIB = BuildMI(mainMBB, DL, TII->get(LCMPXCHGOpc));
+  for (unsigned i = 0; i < X86::AddrNumOperands; ++i)
+    MIB.addOperand(MI->getOperand(MemOpndSlot + i));
+  MIB.addReg(t1);
+  MIB.setMemRefs(MMOBegin, MMOEnd);
 
-  MIB = BuildMI(newMBB, dl, TII->get(X86::LCMPXCHG8B));
-  for (int i=0; i <= lastAddrIndx; ++i)
-    (*MIB).addOperand(*argOpers[i]);
+  BuildMI(mainMBB, DL, TII->get(X86::JNE_4)).addMBB(origMainMBB);
 
-  assert(bInstr->hasOneMemOperand() && "Unexpected number of memoperand");
-  (*MIB).setMemRefs(bInstr->memoperands_begin(),
-                    bInstr->memoperands_end());
+  mainMBB->addSuccessor(origMainMBB);
+  mainMBB->addSuccessor(sinkMBB);
 
-  MIB = BuildMI(newMBB, dl, TII->get(TargetOpcode::COPY), t3);
-  MIB.addReg(X86::EAX);
-  MIB = BuildMI(newMBB, dl, TII->get(TargetOpcode::COPY), t4);
-  MIB.addReg(X86::EDX);
+  // sinkMBB:
+  sinkMBB->addLiveIn(AccPhyReg);
 
-  // insert branch
-  BuildMI(newMBB, dl, TII->get(X86::JNE_4)).addMBB(newMBB);
+  BuildMI(*sinkMBB, sinkMBB->begin(), DL,
+          TII->get(TargetOpcode::COPY), DstReg)
+    .addReg(AccPhyReg);
 
-  bInstr->eraseFromParent();   // The pseudo instruction is gone now.
-  return nextMBB;
+  MI->eraseFromParent();
+  return sinkMBB;
 }
 
-// private utility function
+// EmitAtomicLoadArith6432 - emit the code sequence for pseudo atomic
+// instructions. They will be translated into a spin-loop or compare-exchange
+// loop from
+//
+//    ...
+//    dst = atomic-fetch-op MI.addr, MI.val
+//    ...
+//
+// to
+//
+//    ...
+//    EAX = LOAD [MI.addr + 0]
+//    EDX = LOAD [MI.addr + 4]
+// loop:
+//    EBX = OP MI.val.lo, EAX
+//    ECX = OP MI.val.hi, EDX
+//    LCMPXCHG8B [MI.addr], [ECX:EBX & EDX:EAX are implicitly used and EDX:EAX is implicitly defined]
+//    JNE loop
+// sink:
+//    dst = EDX:EAX
+//    ...
 MachineBasicBlock *
-X86TargetLowering::EmitAtomicMinMaxWithCustomInserter(MachineInstr *mInstr,
-                                                      MachineBasicBlock *MBB,
-                                                      unsigned cmovOpc) const {
-  // For the atomic min/max operator, we generate
-  //   thisMBB:
-  //   newMBB:
-  //     ld t1 = [min/max.addr]
-  //     mov t2 = [min/max.val]
-  //     cmp  t1, t2
-  //     cmov[cond] t2 = t1
-  //     mov EAX = t1
-  //     lcs dest = [bitinstr.addr], t2  [EAX is implicit]
-  //     bz   newMBB
-  //     fallthrough -->nextMBB
-  //
+X86TargetLowering::EmitAtomicLoadArith6432(MachineInstr *MI,
+                                           MachineBasicBlock *MBB) const {
   const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
-  const BasicBlock *LLVM_BB = MBB->getBasicBlock();
-  MachineFunction::iterator MBBIter = MBB;
-  ++MBBIter;
+  DebugLoc DL = MI->getDebugLoc();
+
+  MachineFunction *MF = MBB->getParent();
+  MachineRegisterInfo &MRI = MF->getRegInfo();
+
+  const BasicBlock *BB = MBB->getBasicBlock();
+  MachineFunction::iterator I = MBB;
+  ++I;
+
+  assert(MI->getNumOperands() <= X86::AddrNumOperands + 4 &&
+         "Unexpected number of operands");
+
+  assert(MI->hasOneMemOperand() &&
+         "Expected atomic-load-op32 to have one memoperand");
+
+  // Memory Reference
+  MachineInstr::mmo_iterator MMOBegin = MI->memoperands_begin();
+  MachineInstr::mmo_iterator MMOEnd = MI->memoperands_end();
+
+  unsigned DstLoReg, DstHiReg;
+  unsigned SrcLoReg, SrcHiReg;
+  unsigned MemOpndSlot;
+
+  unsigned CurOp = 0;
+
+  DstLoReg = MI->getOperand(CurOp++).getReg();
+  DstHiReg = MI->getOperand(CurOp++).getReg();
+  MemOpndSlot = CurOp;
+  CurOp += X86::AddrNumOperands;
+  SrcLoReg = MI->getOperand(CurOp++).getReg();
+  SrcHiReg = MI->getOperand(CurOp++).getReg();
+
+  const TargetRegisterClass *RC = &X86::GR32RegClass;
+  const TargetRegisterClass *RC8 = &X86::GR8RegClass;
+
+  unsigned LCMPXCHGOpc = X86::LCMPXCHG8B;
+  unsigned LOADOpc = X86::MOV32rm;
+
+  // For the atomic load-arith operator, we generate
+  //
+  //  thisMBB:
+  //    EAX = LOAD [MI.addr + 0]
+  //    EDX = LOAD [MI.addr + 4]
+  //  mainMBB:
+  //    EBX = OP MI.vallo, EAX
+  //    ECX = OP MI.valhi, EDX
+  //    LCMPXCHG8B [MI.addr], [ECX:EBX & EDX:EAX are implicitly used and EDX:EAX is implicitly defined]
+  //    JNE mainMBB
+  //  sinkMBB:
 
-  /// First build the CFG
-  MachineFunction *F = MBB->getParent();
   MachineBasicBlock *thisMBB = MBB;
-  MachineBasicBlock *newMBB = F->CreateMachineBasicBlock(LLVM_BB);
-  MachineBasicBlock *nextMBB = F->CreateMachineBasicBlock(LLVM_BB);
-  F->insert(MBBIter, newMBB);
-  F->insert(MBBIter, nextMBB);
-
-  // Transfer the remainder of thisMBB and its successor edges to nextMBB.
-  nextMBB->splice(nextMBB->begin(), thisMBB,
-                  llvm::next(MachineBasicBlock::iterator(mInstr)),
-                  thisMBB->end());
-  nextMBB->transferSuccessorsAndUpdatePHIs(thisMBB);
-
-  // Update thisMBB to fall through to newMBB
-  thisMBB->addSuccessor(newMBB);
-
-  // newMBB jumps to newMBB and fall through to nextMBB
-  newMBB->addSuccessor(nextMBB);
-  newMBB->addSuccessor(newMBB);
-
-  DebugLoc dl = mInstr->getDebugLoc();
-  // Insert instructions into newMBB based on incoming instruction
-  assert(mInstr->getNumOperands() < X86::AddrNumOperands + 4 &&
-         "unexpected number of operands");
-  MachineOperand& destOper = mInstr->getOperand(0);
-  MachineOperand* argOpers[2 + X86::AddrNumOperands];
-  int numArgs = mInstr->getNumOperands() - 1;
-  for (int i=0; i < numArgs; ++i)
-    argOpers[i] = &mInstr->getOperand(i+1);
-
-  // x86 address has 4 operands: base, index, scale, and displacement
-  int lastAddrIndx = X86::AddrNumOperands - 1; // [0,3]
-  int valArgIndx = lastAddrIndx + 1;
-
-  unsigned t1 = F->getRegInfo().createVirtualRegister(&X86::GR32RegClass);
-  MachineInstrBuilder MIB = BuildMI(newMBB, dl, TII->get(X86::MOV32rm), t1);
-  for (int i=0; i <= lastAddrIndx; ++i)
-    (*MIB).addOperand(*argOpers[i]);
-
-  // We only support register and immediate values
-  assert((argOpers[valArgIndx]->isReg() ||
-          argOpers[valArgIndx]->isImm()) &&
-         "invalid operand");
-
-  unsigned t2 = F->getRegInfo().createVirtualRegister(&X86::GR32RegClass);
-  if (argOpers[valArgIndx]->isReg())
-    MIB = BuildMI(newMBB, dl, TII->get(TargetOpcode::COPY), t2);
-  else
-    MIB = BuildMI(newMBB, dl, TII->get(X86::MOV32rr), t2);
-  (*MIB).addOperand(*argOpers[valArgIndx]);
+  MachineBasicBlock *mainMBB = MF->CreateMachineBasicBlock(BB);
+  MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB);
+  MF->insert(I, mainMBB);
+  MF->insert(I, sinkMBB);
 
-  MIB = BuildMI(newMBB, dl, TII->get(TargetOpcode::COPY), X86::EAX);
-  MIB.addReg(t1);
+  MachineInstrBuilder MIB;
 
-  MIB = BuildMI(newMBB, dl, TII->get(X86::CMP32rr));
-  MIB.addReg(t1);
-  MIB.addReg(t2);
+  // Transfer the remainder of BB and its successor edges to sinkMBB.
+  sinkMBB->splice(sinkMBB->begin(), MBB,
+                  llvm::next(MachineBasicBlock::iterator(MI)), MBB->end());
+  sinkMBB->transferSuccessorsAndUpdatePHIs(MBB);
+
+  // thisMBB:
+  // Lo
+  MIB = BuildMI(thisMBB, DL, TII->get(LOADOpc), X86::EAX);
+  for (unsigned i = 0; i < X86::AddrNumOperands; ++i)
+    MIB.addOperand(MI->getOperand(MemOpndSlot + i));
+  MIB.setMemRefs(MMOBegin, MMOEnd);
+  // Hi
+  MIB = BuildMI(thisMBB, DL, TII->get(LOADOpc), X86::EDX);
+  for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
+    if (i == X86::AddrDisp)
+      MIB.addDisp(MI->getOperand(MemOpndSlot + i), 4); // 4 == sizeof(i32)
+    else
+      MIB.addOperand(MI->getOperand(MemOpndSlot + i));
+  }
+  MIB.setMemRefs(MMOBegin, MMOEnd);
 
-  // Generate movc
-  unsigned t3 = F->getRegInfo().createVirtualRegister(&X86::GR32RegClass);
-  MIB = BuildMI(newMBB, dl, TII->get(cmovOpc),t3);
-  MIB.addReg(t2);
-  MIB.addReg(t1);
+  thisMBB->addSuccessor(mainMBB);
 
-  // Cmp and exchange if none has modified the memory location
-  MIB = BuildMI(newMBB, dl, TII->get(X86::LCMPXCHG32));
-  for (int i=0; i <= lastAddrIndx; ++i)
-    (*MIB).addOperand(*argOpers[i]);
-  MIB.addReg(t3);
-  assert(mInstr->hasOneMemOperand() && "Unexpected number of memoperand");
-  (*MIB).setMemRefs(mInstr->memoperands_begin(),
-                    mInstr->memoperands_end());
+  // mainMBB:
+  MachineBasicBlock *origMainMBB = mainMBB;
+  mainMBB->addLiveIn(X86::EAX);
+  mainMBB->addLiveIn(X86::EDX);
 
-  MIB = BuildMI(newMBB, dl, TII->get(TargetOpcode::COPY), destOper.getReg());
-  MIB.addReg(X86::EAX);
+  // Copy EDX:EAX as they are used more than once.
+  unsigned LoReg = MRI.createVirtualRegister(RC);
+  unsigned HiReg = MRI.createVirtualRegister(RC);
+  BuildMI(mainMBB, DL, TII->get(TargetOpcode::COPY), LoReg).addReg(X86::EAX);
+  BuildMI(mainMBB, DL, TII->get(TargetOpcode::COPY), HiReg).addReg(X86::EDX);
 
-  // insert branch
-  BuildMI(newMBB, dl, TII->get(X86::JNE_4)).addMBB(newMBB);
+  unsigned t1L = MRI.createVirtualRegister(RC);
+  unsigned t1H = MRI.createVirtualRegister(RC);
 
-  mInstr->eraseFromParent();   // The pseudo instruction is gone now.
-  return nextMBB;
+  unsigned Opc = MI->getOpcode();
+  switch (Opc) {
+  default:
+    llvm_unreachable("Unhandled atomic-load-op6432 opcode!");
+  case X86::ATOMAND6432:
+  case X86::ATOMOR6432:
+  case X86::ATOMXOR6432:
+  case X86::ATOMADD6432:
+  case X86::ATOMSUB6432: {
+    unsigned HiOpc;
+    unsigned LoOpc = getNonAtomic6432Opcode(Opc, HiOpc);
+    BuildMI(mainMBB, DL, TII->get(LoOpc), t1L).addReg(LoReg).addReg(SrcLoReg);
+    BuildMI(mainMBB, DL, TII->get(HiOpc), t1H).addReg(HiReg).addReg(SrcHiReg);
+    break;
+  }
+  case X86::ATOMNAND6432: {
+    unsigned HiOpc, NOTOpc;
+    unsigned LoOpc = getNonAtomic6432OpcodeWithExtraOpc(Opc, HiOpc, NOTOpc);
+    unsigned t2L = MRI.createVirtualRegister(RC);
+    unsigned t2H = MRI.createVirtualRegister(RC);
+    BuildMI(mainMBB, DL, TII->get(LoOpc), t2L).addReg(SrcLoReg).addReg(LoReg);
+    BuildMI(mainMBB, DL, TII->get(HiOpc), t2H).addReg(SrcHiReg).addReg(HiReg);
+    BuildMI(mainMBB, DL, TII->get(NOTOpc), t1L).addReg(t2L);
+    BuildMI(mainMBB, DL, TII->get(NOTOpc), t1H).addReg(t2H);
+    break;
+  }
+  case X86::ATOMMAX6432:
+  case X86::ATOMMIN6432:
+  case X86::ATOMUMAX6432:
+  case X86::ATOMUMIN6432: {
+    unsigned HiOpc;
+    unsigned LoOpc = getNonAtomic6432Opcode(Opc, HiOpc);
+    unsigned cL = MRI.createVirtualRegister(RC8);
+    unsigned cH = MRI.createVirtualRegister(RC8);
+    unsigned cL32 = MRI.createVirtualRegister(RC);
+    unsigned cH32 = MRI.createVirtualRegister(RC);
+    unsigned cc = MRI.createVirtualRegister(RC);
+    // cl := cmp src_lo, lo
+    BuildMI(mainMBB, DL, TII->get(X86::CMP32rr))
+      .addReg(SrcLoReg).addReg(LoReg);
+    BuildMI(mainMBB, DL, TII->get(LoOpc), cL);
+    BuildMI(mainMBB, DL, TII->get(X86::MOVZX32rr8), cL32).addReg(cL);
+    // ch := cmp src_hi, hi
+    BuildMI(mainMBB, DL, TII->get(X86::CMP32rr))
+      .addReg(SrcHiReg).addReg(HiReg);
+    BuildMI(mainMBB, DL, TII->get(HiOpc), cH);
+    BuildMI(mainMBB, DL, TII->get(X86::MOVZX32rr8), cH32).addReg(cH);
+    // cc := if (src_hi == hi) ? cl : ch;
+    if (Subtarget->hasCMov()) {
+      BuildMI(mainMBB, DL, TII->get(X86::CMOVE32rr), cc)
+        .addReg(cH32).addReg(cL32);
+    } else {
+      MIB = BuildMI(mainMBB, DL, TII->get(X86::CMOV_GR32), cc)
+              .addReg(cH32).addReg(cL32)
+              .addImm(X86::COND_E);
+      mainMBB = EmitLoweredSelect(MIB, mainMBB);
+    }
+    BuildMI(mainMBB, DL, TII->get(X86::TEST32rr)).addReg(cc).addReg(cc);
+    if (Subtarget->hasCMov()) {
+      BuildMI(mainMBB, DL, TII->get(X86::CMOVNE32rr), t1L)
+        .addReg(SrcLoReg).addReg(LoReg);
+      BuildMI(mainMBB, DL, TII->get(X86::CMOVNE32rr), t1H)
+        .addReg(SrcHiReg).addReg(HiReg);
+    } else {
+      MIB = BuildMI(mainMBB, DL, TII->get(X86::CMOV_GR32), t1L)
+              .addReg(SrcLoReg).addReg(LoReg)
+              .addImm(X86::COND_NE);
+      mainMBB = EmitLoweredSelect(MIB, mainMBB);
+      MIB = BuildMI(mainMBB, DL, TII->get(X86::CMOV_GR32), t1H)
+              .addReg(SrcHiReg).addReg(HiReg)
+              .addImm(X86::COND_NE);
+      mainMBB = EmitLoweredSelect(MIB, mainMBB);
+    }
+    break;
+  }
+  case X86::ATOMSWAP6432: {
+    unsigned HiOpc;
+    unsigned LoOpc = getNonAtomic6432Opcode(Opc, HiOpc);
+    BuildMI(mainMBB, DL, TII->get(LoOpc), t1L).addReg(SrcLoReg);
+    BuildMI(mainMBB, DL, TII->get(HiOpc), t1H).addReg(SrcHiReg);
+    break;
+  }
+  }
+
+  // Copy EDX:EAX back from HiReg:LoReg
+  BuildMI(mainMBB, DL, TII->get(TargetOpcode::COPY), X86::EAX).addReg(LoReg);
+  BuildMI(mainMBB, DL, TII->get(TargetOpcode::COPY), X86::EDX).addReg(HiReg);
+  // Copy ECX:EBX from t1H:t1L
+  BuildMI(mainMBB, DL, TII->get(TargetOpcode::COPY), X86::EBX).addReg(t1L);
+  BuildMI(mainMBB, DL, TII->get(TargetOpcode::COPY), X86::ECX).addReg(t1H);
+
+  MIB = BuildMI(mainMBB, DL, TII->get(LCMPXCHGOpc));
+  for (unsigned i = 0; i < X86::AddrNumOperands; ++i)
+    MIB.addOperand(MI->getOperand(MemOpndSlot + i));
+  MIB.setMemRefs(MMOBegin, MMOEnd);
+
+  BuildMI(mainMBB, DL, TII->get(X86::JNE_4)).addMBB(origMainMBB);
+
+  mainMBB->addSuccessor(origMainMBB);
+  mainMBB->addSuccessor(sinkMBB);
+
+  // sinkMBB:
+  sinkMBB->addLiveIn(X86::EAX);
+  sinkMBB->addLiveIn(X86::EDX);
+
+  BuildMI(*sinkMBB, sinkMBB->begin(), DL,
+          TII->get(TargetOpcode::COPY), DstLoReg)
+    .addReg(X86::EAX);
+  BuildMI(*sinkMBB, sinkMBB->begin(), DL,
+          TII->get(TargetOpcode::COPY), DstHiReg)
+    .addReg(X86::EDX);
+
+  MI->eraseFromParent();
+  return sinkMBB;
 }
 
 // FIXME: When we get size specific XMM0 registers, i.e. XMM0_V16I8
 // or XMM0_V32I8 in AVX all of this code can be replaced with that
 // in the .td file.
-MachineBasicBlock *
-X86TargetLowering::EmitPCMP(MachineInstr *MI, MachineBasicBlock *BB,
-                            unsigned numArgs, bool memArg) const {
-  assert(Subtarget->hasSSE42() &&
-         "Target must have SSE4.2 or AVX features enabled");
+static MachineBasicBlock *EmitPCMPSTRM(MachineInstr *MI, MachineBasicBlock *BB,
+                                       const TargetInstrInfo *TII) {
+  unsigned Opc;
+  switch (MI->getOpcode()) {
+  default: llvm_unreachable("illegal opcode!");
+  case X86::PCMPISTRM128REG:  Opc = X86::PCMPISTRM128rr;  break;
+  case X86::VPCMPISTRM128REG: Opc = X86::VPCMPISTRM128rr; break;
+  case X86::PCMPISTRM128MEM:  Opc = X86::PCMPISTRM128rm;  break;
+  case X86::VPCMPISTRM128MEM: Opc = X86::VPCMPISTRM128rm; break;
+  case X86::PCMPESTRM128REG:  Opc = X86::PCMPESTRM128rr;  break;
+  case X86::VPCMPESTRM128REG: Opc = X86::VPCMPESTRM128rr; break;
+  case X86::PCMPESTRM128MEM:  Opc = X86::PCMPESTRM128rm;  break;
+  case X86::VPCMPESTRM128MEM: Opc = X86::VPCMPESTRM128rm; break;
+  }
 
   DebugLoc dl = MI->getDebugLoc();
-  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
+  MachineInstrBuilder MIB = BuildMI(*BB, MI, dl, TII->get(Opc));
+
+  unsigned NumArgs = MI->getNumOperands();
+  for (unsigned i = 1; i < NumArgs; ++i) {
+    MachineOperand &Op = MI->getOperand(i);
+    if (!(Op.isReg() && Op.isImplicit()))
+      MIB.addOperand(Op);
+  }
+  if (MI->hasOneMemOperand())
+    MIB->setMemRefs(MI->memoperands_begin(), MI->memoperands_end());
+
+  BuildMI(*BB, MI, dl,
+    TII->get(TargetOpcode::COPY), MI->getOperand(0).getReg())
+    .addReg(X86::XMM0);
+
+  MI->eraseFromParent();
+  return BB;
+}
+
+// FIXME: Custom handling because TableGen doesn't support multiple implicit
+// defs in an instruction pattern
+static MachineBasicBlock *EmitPCMPSTRI(MachineInstr *MI, MachineBasicBlock *BB,
+                                       const TargetInstrInfo *TII) {
   unsigned Opc;
-  if (!Subtarget->hasAVX()) {
-    if (memArg)
-      Opc = numArgs == 3 ? X86::PCMPISTRM128rm : X86::PCMPESTRM128rm;
-    else
-      Opc = numArgs == 3 ? X86::PCMPISTRM128rr : X86::PCMPESTRM128rr;
-  } else {
-    if (memArg)
-      Opc = numArgs == 3 ? X86::VPCMPISTRM128rm : X86::VPCMPESTRM128rm;
-    else
-      Opc = numArgs == 3 ? X86::VPCMPISTRM128rr : X86::VPCMPESTRM128rr;
+  switch (MI->getOpcode()) {
+  default: llvm_unreachable("illegal opcode!");
+  case X86::PCMPISTRIREG:  Opc = X86::PCMPISTRIrr;  break;
+  case X86::VPCMPISTRIREG: Opc = X86::VPCMPISTRIrr; break;
+  case X86::PCMPISTRIMEM:  Opc = X86::PCMPISTRIrm;  break;
+  case X86::VPCMPISTRIMEM: Opc = X86::VPCMPISTRIrm; break;
+  case X86::PCMPESTRIREG:  Opc = X86::PCMPESTRIrr;  break;
+  case X86::VPCMPESTRIREG: Opc = X86::VPCMPESTRIrr; break;
+  case X86::PCMPESTRIMEM:  Opc = X86::PCMPESTRIrm;  break;
+  case X86::VPCMPESTRIMEM: Opc = X86::VPCMPESTRIrm; break;
   }
 
+  DebugLoc dl = MI->getDebugLoc();
   MachineInstrBuilder MIB = BuildMI(*BB, MI, dl, TII->get(Opc));
-  for (unsigned i = 0; i < numArgs; ++i) {
-    MachineOperand &Op = MI->getOperand(i+1);
+
+  unsigned NumArgs = MI->getNumOperands(); // remove the results
+  for (unsigned i = 1; i < NumArgs; ++i) {
+    MachineOperand &Op = MI->getOperand(i);
     if (!(Op.isReg() && Op.isImplicit()))
       MIB.addOperand(Op);
   }
+  if (MI->hasOneMemOperand())
+    MIB->setMemRefs(MI->memoperands_begin(), MI->memoperands_end());
+
   BuildMI(*BB, MI, dl,
     TII->get(TargetOpcode::COPY), MI->getOperand(0).getReg())
-    .addReg(X86::XMM0);
+    .addReg(X86::ECX);
 
   MI->eraseFromParent();
   return BB;
 }
 
-MachineBasicBlock *
-X86TargetLowering::EmitMonitor(MachineInstr *MI, MachineBasicBlock *BB) const {
+static MachineBasicBlock * EmitMonitor(MachineInstr *MI, MachineBasicBlock *BB,
+                                       const TargetInstrInfo *TII,
+                                       const X86Subtarget* Subtarget) {
   DebugLoc dl = MI->getDebugLoc();
-  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
 
   // Address into RAX/EAX, other two args into ECX, EDX.
   unsigned MemOpc = Subtarget->is64Bit() ? X86::LEA64r : X86::LEA32r;
@@ -12717,51 +13553,248 @@ X86TargetLowering::EmitLoweredTLSCall(MachineInstr *MI,
   DebugLoc DL = MI->getDebugLoc();
   MachineFunction *F = BB->getParent();
 
-  assert(Subtarget->isTargetDarwin() && "Darwin only instr emitted?");
-  assert(MI->getOperand(3).isGlobal() && "This should be a global");
+  assert(Subtarget->isTargetDarwin() && "Darwin only instr emitted?");
+  assert(MI->getOperand(3).isGlobal() && "This should be a global");
+
+  // Get a register mask for the lowered call.
+  // FIXME: The 32-bit calls have non-standard calling conventions. Use a
+  // proper register mask.
+  const uint32_t *RegMask =
+    getTargetMachine().getRegisterInfo()->getCallPreservedMask(CallingConv::C);
+  if (Subtarget->is64Bit()) {
+    MachineInstrBuilder MIB = BuildMI(*BB, MI, DL,
+                                      TII->get(X86::MOV64rm), X86::RDI)
+    .addReg(X86::RIP)
+    .addImm(0).addReg(0)
+    .addGlobalAddress(MI->getOperand(3).getGlobal(), 0,
+                      MI->getOperand(3).getTargetFlags())
+    .addReg(0);
+    MIB = BuildMI(*BB, MI, DL, TII->get(X86::CALL64m));
+    addDirectMem(MIB, X86::RDI);
+    MIB.addReg(X86::RAX, RegState::ImplicitDefine).addRegMask(RegMask);
+  } else if (getTargetMachine().getRelocationModel() != Reloc::PIC_) {
+    MachineInstrBuilder MIB = BuildMI(*BB, MI, DL,
+                                      TII->get(X86::MOV32rm), X86::EAX)
+    .addReg(0)
+    .addImm(0).addReg(0)
+    .addGlobalAddress(MI->getOperand(3).getGlobal(), 0,
+                      MI->getOperand(3).getTargetFlags())
+    .addReg(0);
+    MIB = BuildMI(*BB, MI, DL, TII->get(X86::CALL32m));
+    addDirectMem(MIB, X86::EAX);
+    MIB.addReg(X86::EAX, RegState::ImplicitDefine).addRegMask(RegMask);
+  } else {
+    MachineInstrBuilder MIB = BuildMI(*BB, MI, DL,
+                                      TII->get(X86::MOV32rm), X86::EAX)
+    .addReg(TII->getGlobalBaseReg(F))
+    .addImm(0).addReg(0)
+    .addGlobalAddress(MI->getOperand(3).getGlobal(), 0,
+                      MI->getOperand(3).getTargetFlags())
+    .addReg(0);
+    MIB = BuildMI(*BB, MI, DL, TII->get(X86::CALL32m));
+    addDirectMem(MIB, X86::EAX);
+    MIB.addReg(X86::EAX, RegState::ImplicitDefine).addRegMask(RegMask);
+  }
+
+  MI->eraseFromParent(); // The pseudo instruction is gone now.
+  return BB;
+}
+
+MachineBasicBlock *
+X86TargetLowering::emitEHSjLjSetJmp(MachineInstr *MI,
+                                    MachineBasicBlock *MBB) const {
+  DebugLoc DL = MI->getDebugLoc();
+  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
+
+  MachineFunction *MF = MBB->getParent();
+  MachineRegisterInfo &MRI = MF->getRegInfo();
+
+  const BasicBlock *BB = MBB->getBasicBlock();
+  MachineFunction::iterator I = MBB;
+  ++I;
+
+  // Memory Reference
+  MachineInstr::mmo_iterator MMOBegin = MI->memoperands_begin();
+  MachineInstr::mmo_iterator MMOEnd = MI->memoperands_end();
+
+  unsigned DstReg;
+  unsigned MemOpndSlot = 0;
+
+  unsigned CurOp = 0;
+
+  DstReg = MI->getOperand(CurOp++).getReg();
+  const TargetRegisterClass *RC = MRI.getRegClass(DstReg);
+  assert(RC->hasType(MVT::i32) && "Invalid destination!");
+  unsigned mainDstReg = MRI.createVirtualRegister(RC);
+  unsigned restoreDstReg = MRI.createVirtualRegister(RC);
+
+  MemOpndSlot = CurOp;
+
+  MVT PVT = getPointerTy();
+  assert((PVT == MVT::i64 || PVT == MVT::i32) &&
+         "Invalid Pointer Size!");
+
+  // For v = setjmp(buf), we generate
+  //
+  // thisMBB:
+  //  buf[LabelOffset] = restoreMBB
+  //  SjLjSetup restoreMBB
+  //
+  // mainMBB:
+  //  v_main = 0
+  //
+  // sinkMBB:
+  //  v = phi(main, restore)
+  //
+  // restoreMBB:
+  //  v_restore = 1
+
+  MachineBasicBlock *thisMBB = MBB;
+  MachineBasicBlock *mainMBB = MF->CreateMachineBasicBlock(BB);
+  MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB);
+  MachineBasicBlock *restoreMBB = MF->CreateMachineBasicBlock(BB);
+  MF->insert(I, mainMBB);
+  MF->insert(I, sinkMBB);
+  MF->push_back(restoreMBB);
+
+  MachineInstrBuilder MIB;
+
+  // Transfer the remainder of BB and its successor edges to sinkMBB.
+  sinkMBB->splice(sinkMBB->begin(), MBB,
+                  llvm::next(MachineBasicBlock::iterator(MI)), MBB->end());
+  sinkMBB->transferSuccessorsAndUpdatePHIs(MBB);
+
+  // thisMBB:
+  unsigned PtrStoreOpc = 0;
+  unsigned LabelReg = 0;
+  const int64_t LabelOffset = 1 * PVT.getStoreSize();
+  Reloc::Model RM = getTargetMachine().getRelocationModel();
+  bool UseImmLabel = (getTargetMachine().getCodeModel() == CodeModel::Small) &&
+                     (RM == Reloc::Static || RM == Reloc::DynamicNoPIC);
+
+  // Prepare IP either in reg or imm.
+  if (!UseImmLabel) {
+    PtrStoreOpc = (PVT == MVT::i64) ? X86::MOV64mr : X86::MOV32mr;
+    const TargetRegisterClass *PtrRC = getRegClassFor(PVT);
+    LabelReg = MRI.createVirtualRegister(PtrRC);
+    if (Subtarget->is64Bit()) {
+      MIB = BuildMI(*thisMBB, MI, DL, TII->get(X86::LEA64r), LabelReg)
+              .addReg(X86::RIP)
+              .addImm(0)
+              .addReg(0)
+              .addMBB(restoreMBB)
+              .addReg(0);
+    } else {
+      const X86InstrInfo *XII = static_cast<const X86InstrInfo*>(TII);
+      MIB = BuildMI(*thisMBB, MI, DL, TII->get(X86::LEA32r), LabelReg)
+              .addReg(XII->getGlobalBaseReg(MF))
+              .addImm(0)
+              .addReg(0)
+              .addMBB(restoreMBB, Subtarget->ClassifyBlockAddressReference())
+              .addReg(0);
+    }
+  } else
+    PtrStoreOpc = (PVT == MVT::i64) ? X86::MOV64mi32 : X86::MOV32mi;
+  // Store IP
+  MIB = BuildMI(*thisMBB, MI, DL, TII->get(PtrStoreOpc));
+  for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
+    if (i == X86::AddrDisp)
+      MIB.addDisp(MI->getOperand(MemOpndSlot + i), LabelOffset);
+    else
+      MIB.addOperand(MI->getOperand(MemOpndSlot + i));
+  }
+  if (!UseImmLabel)
+    MIB.addReg(LabelReg);
+  else
+    MIB.addMBB(restoreMBB);
+  MIB.setMemRefs(MMOBegin, MMOEnd);
+  // Setup
+  MIB = BuildMI(*thisMBB, MI, DL, TII->get(X86::EH_SjLj_Setup))
+          .addMBB(restoreMBB);
+  MIB.addRegMask(RegInfo->getNoPreservedMask());
+  thisMBB->addSuccessor(mainMBB);
+  thisMBB->addSuccessor(restoreMBB);
+
+  // mainMBB:
+  //  EAX = 0
+  BuildMI(mainMBB, DL, TII->get(X86::MOV32r0), mainDstReg);
+  mainMBB->addSuccessor(sinkMBB);
+
+  // sinkMBB:
+  BuildMI(*sinkMBB, sinkMBB->begin(), DL,
+          TII->get(X86::PHI), DstReg)
+    .addReg(mainDstReg).addMBB(mainMBB)
+    .addReg(restoreDstReg).addMBB(restoreMBB);
+
+  // restoreMBB:
+  BuildMI(restoreMBB, DL, TII->get(X86::MOV32ri), restoreDstReg).addImm(1);
+  BuildMI(restoreMBB, DL, TII->get(X86::JMP_4)).addMBB(sinkMBB);
+  restoreMBB->addSuccessor(sinkMBB);
+
+  MI->eraseFromParent();
+  return sinkMBB;
+}
+
+MachineBasicBlock *
+X86TargetLowering::emitEHSjLjLongJmp(MachineInstr *MI,
+                                     MachineBasicBlock *MBB) const {
+  DebugLoc DL = MI->getDebugLoc();
+  const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
+
+  MachineFunction *MF = MBB->getParent();
+  MachineRegisterInfo &MRI = MF->getRegInfo();
 
-  // Get a register mask for the lowered call.
-  // FIXME: The 32-bit calls have non-standard calling conventions. Use a
-  // proper register mask.
-  const uint32_t *RegMask =
-    getTargetMachine().getRegisterInfo()->getCallPreservedMask(CallingConv::C);
-  if (Subtarget->is64Bit()) {
-    MachineInstrBuilder MIB = BuildMI(*BB, MI, DL,
-                                      TII->get(X86::MOV64rm), X86::RDI)
-    .addReg(X86::RIP)
-    .addImm(0).addReg(0)
-    .addGlobalAddress(MI->getOperand(3).getGlobal(), 0,
-                      MI->getOperand(3).getTargetFlags())
-    .addReg(0);
-    MIB = BuildMI(*BB, MI, DL, TII->get(X86::CALL64m));
-    addDirectMem(MIB, X86::RDI);
-    MIB.addReg(X86::RAX, RegState::ImplicitDefine).addRegMask(RegMask);
-  } else if (getTargetMachine().getRelocationModel() != Reloc::PIC_) {
-    MachineInstrBuilder MIB = BuildMI(*BB, MI, DL,
-                                      TII->get(X86::MOV32rm), X86::EAX)
-    .addReg(0)
-    .addImm(0).addReg(0)
-    .addGlobalAddress(MI->getOperand(3).getGlobal(), 0,
-                      MI->getOperand(3).getTargetFlags())
-    .addReg(0);
-    MIB = BuildMI(*BB, MI, DL, TII->get(X86::CALL32m));
-    addDirectMem(MIB, X86::EAX);
-    MIB.addReg(X86::EAX, RegState::ImplicitDefine).addRegMask(RegMask);
-  } else {
-    MachineInstrBuilder MIB = BuildMI(*BB, MI, DL,
-                                      TII->get(X86::MOV32rm), X86::EAX)
-    .addReg(TII->getGlobalBaseReg(F))
-    .addImm(0).addReg(0)
-    .addGlobalAddress(MI->getOperand(3).getGlobal(), 0,
-                      MI->getOperand(3).getTargetFlags())
-    .addReg(0);
-    MIB = BuildMI(*BB, MI, DL, TII->get(X86::CALL32m));
-    addDirectMem(MIB, X86::EAX);
-    MIB.addReg(X86::EAX, RegState::ImplicitDefine).addRegMask(RegMask);
+  // Memory Reference
+  MachineInstr::mmo_iterator MMOBegin = MI->memoperands_begin();
+  MachineInstr::mmo_iterator MMOEnd = MI->memoperands_end();
+
+  MVT PVT = getPointerTy();
+  assert((PVT == MVT::i64 || PVT == MVT::i32) &&
+         "Invalid Pointer Size!");
+
+  const TargetRegisterClass *RC =
+    (PVT == MVT::i64) ? &X86::GR64RegClass : &X86::GR32RegClass;
+  unsigned Tmp = MRI.createVirtualRegister(RC);
+  // Since FP is only updated here but NOT referenced, it's treated as GPR.
+  unsigned FP = (PVT == MVT::i64) ? X86::RBP : X86::EBP;
+  unsigned SP = RegInfo->getStackRegister();
+
+  MachineInstrBuilder MIB;
+
+  const int64_t LabelOffset = 1 * PVT.getStoreSize();
+  const int64_t SPOffset = 2 * PVT.getStoreSize();
+
+  unsigned PtrLoadOpc = (PVT == MVT::i64) ? X86::MOV64rm : X86::MOV32rm;
+  unsigned IJmpOpc = (PVT == MVT::i64) ? X86::JMP64r : X86::JMP32r;
+
+  // Reload FP
+  MIB = BuildMI(*MBB, MI, DL, TII->get(PtrLoadOpc), FP);
+  for (unsigned i = 0; i < X86::AddrNumOperands; ++i)
+    MIB.addOperand(MI->getOperand(i));
+  MIB.setMemRefs(MMOBegin, MMOEnd);
+  // Reload IP
+  MIB = BuildMI(*MBB, MI, DL, TII->get(PtrLoadOpc), Tmp);
+  for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
+    if (i == X86::AddrDisp)
+      MIB.addDisp(MI->getOperand(i), LabelOffset);
+    else
+      MIB.addOperand(MI->getOperand(i));
+  }
+  MIB.setMemRefs(MMOBegin, MMOEnd);
+  // Reload SP
+  MIB = BuildMI(*MBB, MI, DL, TII->get(PtrLoadOpc), SP);
+  for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
+    if (i == X86::AddrDisp)
+      MIB.addDisp(MI->getOperand(i), SPOffset);
+    else
+      MIB.addOperand(MI->getOperand(i));
   }
+  MIB.setMemRefs(MMOBegin, MMOEnd);
+  // Jump
+  BuildMI(*MBB, MI, DL, TII->get(IJmpOpc)).addReg(Tmp);
 
-  MI->eraseFromParent(); // The pseudo instruction is gone now.
-  return BB;
+  MI->eraseFromParent();
+  return MBB;
 }
 
 MachineBasicBlock *
@@ -12893,198 +13926,101 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI,
   case X86::PCMPESTRM128REG:
   case X86::VPCMPESTRM128REG:
   case X86::PCMPESTRM128MEM:
-  case X86::VPCMPESTRM128MEM: {
-    unsigned NumArgs;
-    bool MemArg;
-    switch (MI->getOpcode()) {
-    default: llvm_unreachable("illegal opcode!");
-    case X86::PCMPISTRM128REG:
-    case X86::VPCMPISTRM128REG:
-      NumArgs = 3; MemArg = false; break;
-    case X86::PCMPISTRM128MEM:
-    case X86::VPCMPISTRM128MEM:
-      NumArgs = 3; MemArg = true; break;
-    case X86::PCMPESTRM128REG:
-    case X86::VPCMPESTRM128REG:
-      NumArgs = 5; MemArg = false; break;
-    case X86::PCMPESTRM128MEM:
-    case X86::VPCMPESTRM128MEM:
-      NumArgs = 5; MemArg = true; break;
-    }
-    return EmitPCMP(MI, BB, NumArgs, MemArg);
-  }
-
-    // Thread synchronization.
+  case X86::VPCMPESTRM128MEM:
+    assert(Subtarget->hasSSE42() &&
+           "Target must have SSE4.2 or AVX features enabled");
+    return EmitPCMPSTRM(MI, BB, getTargetMachine().getInstrInfo());
+
+  // String/text processing lowering.
+  case X86::PCMPISTRIREG:
+  case X86::VPCMPISTRIREG:
+  case X86::PCMPISTRIMEM:
+  case X86::VPCMPISTRIMEM:
+  case X86::PCMPESTRIREG:
+  case X86::VPCMPESTRIREG:
+  case X86::PCMPESTRIMEM:
+  case X86::VPCMPESTRIMEM:
+    assert(Subtarget->hasSSE42() &&
+           "Target must have SSE4.2 or AVX features enabled");
+    return EmitPCMPSTRI(MI, BB, getTargetMachine().getInstrInfo());
+
+  // Thread synchronization.
   case X86::MONITOR:
-    return EmitMonitor(MI, BB);
+    return EmitMonitor(MI, BB, getTargetMachine().getInstrInfo(), Subtarget);
 
-    // Atomic Lowering.
-  case X86::ATOMAND32:
-    return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND32rr,
-                                               X86::AND32ri, X86::MOV32rm,
-                                               X86::LCMPXCHG32,
-                                               X86::NOT32r, X86::EAX,
-                                               &X86::GR32RegClass);
-  case X86::ATOMOR32:
-    return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR32rr,
-                                               X86::OR32ri, X86::MOV32rm,
-                                               X86::LCMPXCHG32,
-                                               X86::NOT32r, X86::EAX,
-                                               &X86::GR32RegClass);
-  case X86::ATOMXOR32:
-    return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR32rr,
-                                               X86::XOR32ri, X86::MOV32rm,
-                                               X86::LCMPXCHG32,
-                                               X86::NOT32r, X86::EAX,
-                                               &X86::GR32RegClass);
-  case X86::ATOMNAND32:
-    return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND32rr,
-                                               X86::AND32ri, X86::MOV32rm,
-                                               X86::LCMPXCHG32,
-                                               X86::NOT32r, X86::EAX,
-                                               &X86::GR32RegClass, true);
-  case X86::ATOMMIN32:
-    return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVL32rr);
-  case X86::ATOMMAX32:
-    return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVG32rr);
-  case X86::ATOMUMIN32:
-    return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVB32rr);
-  case X86::ATOMUMAX32:
-    return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVA32rr);
+  // xbegin
+  case X86::XBEGIN:
+    return EmitXBegin(MI, BB, getTargetMachine().getInstrInfo());
 
+  // Atomic Lowering.
+  case X86::ATOMAND8:
   case X86::ATOMAND16:
-    return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND16rr,
-                                               X86::AND16ri, X86::MOV16rm,
-                                               X86::LCMPXCHG16,
-                                               X86::NOT16r, X86::AX,
-                                               &X86::GR16RegClass);
+  case X86::ATOMAND32:
+  case X86::ATOMAND64:
+    // Fall through
+  case X86::ATOMOR8:
   case X86::ATOMOR16:
-    return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR16rr,
-                                               X86::OR16ri, X86::MOV16rm,
-                                               X86::LCMPXCHG16,
-                                               X86::NOT16r, X86::AX,
-                                               &X86::GR16RegClass);
+  case X86::ATOMOR32:
+  case X86::ATOMOR64:
+    // Fall through
   case X86::ATOMXOR16:
-    return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR16rr,
-                                               X86::XOR16ri, X86::MOV16rm,
-                                               X86::LCMPXCHG16,
-                                               X86::NOT16r, X86::AX,
-                                               &X86::GR16RegClass);
-  case X86::ATOMNAND16:
-    return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND16rr,
-                                               X86::AND16ri, X86::MOV16rm,
-                                               X86::LCMPXCHG16,
-                                               X86::NOT16r, X86::AX,
-                                               &X86::GR16RegClass, true);
-  case X86::ATOMMIN16:
-    return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVL16rr);
-  case X86::ATOMMAX16:
-    return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVG16rr);
-  case X86::ATOMUMIN16:
-    return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVB16rr);
-  case X86::ATOMUMAX16:
-    return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVA16rr);
-
-  case X86::ATOMAND8:
-    return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND8rr,
-                                               X86::AND8ri, X86::MOV8rm,
-                                               X86::LCMPXCHG8,
-                                               X86::NOT8r, X86::AL,
-                                               &X86::GR8RegClass);
-  case X86::ATOMOR8:
-    return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR8rr,
-                                               X86::OR8ri, X86::MOV8rm,
-                                               X86::LCMPXCHG8,
-                                               X86::NOT8r, X86::AL,
-                                               &X86::GR8RegClass);
   case X86::ATOMXOR8:
-    return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR8rr,
-                                               X86::XOR8ri, X86::MOV8rm,
-                                               X86::LCMPXCHG8,
-                                               X86::NOT8r, X86::AL,
-                                               &X86::GR8RegClass);
-  case X86::ATOMNAND8:
-    return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND8rr,
-                                               X86::AND8ri, X86::MOV8rm,
-                                               X86::LCMPXCHG8,
-                                               X86::NOT8r, X86::AL,
-                                               &X86::GR8RegClass, true);
-  // FIXME: There are no CMOV8 instructions; MIN/MAX need some other way.
-  // This group is for 64-bit host.
-  case X86::ATOMAND64:
-    return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND64rr,
-                                               X86::AND64ri32, X86::MOV64rm,
-                                               X86::LCMPXCHG64,
-                                               X86::NOT64r, X86::RAX,
-                                               &X86::GR64RegClass);
-  case X86::ATOMOR64:
-    return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::OR64rr,
-                                               X86::OR64ri32, X86::MOV64rm,
-                                               X86::LCMPXCHG64,
-                                               X86::NOT64r, X86::RAX,
-                                               &X86::GR64RegClass);
+  case X86::ATOMXOR32:
   case X86::ATOMXOR64:
-    return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::XOR64rr,
-                                               X86::XOR64ri32, X86::MOV64rm,
-                                               X86::LCMPXCHG64,
-                                               X86::NOT64r, X86::RAX,
-                                               &X86::GR64RegClass);
+    // Fall through
+  case X86::ATOMNAND8:
+  case X86::ATOMNAND16:
+  case X86::ATOMNAND32:
   case X86::ATOMNAND64:
-    return EmitAtomicBitwiseWithCustomInserter(MI, BB, X86::AND64rr,
-                                               X86::AND64ri32, X86::MOV64rm,
-                                               X86::LCMPXCHG64,
-                                               X86::NOT64r, X86::RAX,
-                                               &X86::GR64RegClass, true);
-  case X86::ATOMMIN64:
-    return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVL64rr);
+    // Fall through
+  case X86::ATOMMAX8:
+  case X86::ATOMMAX16:
+  case X86::ATOMMAX32:
   case X86::ATOMMAX64:
-    return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVG64rr);
-  case X86::ATOMUMIN64:
-    return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVB64rr);
+    // Fall through
+  case X86::ATOMMIN8:
+  case X86::ATOMMIN16:
+  case X86::ATOMMIN32:
+  case X86::ATOMMIN64:
+    // Fall through
+  case X86::ATOMUMAX8:
+  case X86::ATOMUMAX16:
+  case X86::ATOMUMAX32:
   case X86::ATOMUMAX64:
-    return EmitAtomicMinMaxWithCustomInserter(MI, BB, X86::CMOVA64rr);
+    // Fall through
+  case X86::ATOMUMIN8:
+  case X86::ATOMUMIN16:
+  case X86::ATOMUMIN32:
+  case X86::ATOMUMIN64:
+    return EmitAtomicLoadArith(MI, BB);
 
   // This group does 64-bit operations on a 32-bit host.
   case X86::ATOMAND6432:
-    return EmitAtomicBit6432WithCustomInserter(MI, BB,
-                                               X86::AND32rr, X86::AND32rr,
-                                               X86::AND32ri, X86::AND32ri,
-                                               false);
   case X86::ATOMOR6432:
-    return EmitAtomicBit6432WithCustomInserter(MI, BB,
-                                               X86::OR32rr, X86::OR32rr,
-                                               X86::OR32ri, X86::OR32ri,
-                                               false);
   case X86::ATOMXOR6432:
-    return EmitAtomicBit6432WithCustomInserter(MI, BB,
-                                               X86::XOR32rr, X86::XOR32rr,
-                                               X86::XOR32ri, X86::XOR32ri,
-                                               false);
   case X86::ATOMNAND6432:
-    return EmitAtomicBit6432WithCustomInserter(MI, BB,
-                                               X86::AND32rr, X86::AND32rr,
-                                               X86::AND32ri, X86::AND32ri,
-                                               true);
   case X86::ATOMADD6432:
-    return EmitAtomicBit6432WithCustomInserter(MI, BB,
-                                               X86::ADD32rr, X86::ADC32rr,
-                                               X86::ADD32ri, X86::ADC32ri,
-                                               false);
   case X86::ATOMSUB6432:
-    return EmitAtomicBit6432WithCustomInserter(MI, BB,
-                                               X86::SUB32rr, X86::SBB32rr,
-                                               X86::SUB32ri, X86::SBB32ri,
-                                               false);
+  case X86::ATOMMAX6432:
+  case X86::ATOMMIN6432:
+  case X86::ATOMUMAX6432:
+  case X86::ATOMUMIN6432:
   case X86::ATOMSWAP6432:
-    return EmitAtomicBit6432WithCustomInserter(MI, BB,
-                                               X86::MOV32rr, X86::MOV32rr,
-                                               X86::MOV32ri, X86::MOV32ri,
-                                               false);
+    return EmitAtomicLoadArith6432(MI, BB);
+
   case X86::VASTART_SAVE_XMM_REGS:
     return EmitVAStartSaveXMMRegsWithCustomInserter(MI, BB);
 
   case X86::VAARG_64:
     return EmitVAARG64WithCustomInserter(MI, BB);
+
+  case X86::EH_SjLj_SetJmp32:
+  case X86::EH_SjLj_SetJmp64:
+    return emitEHSjLjSetJmp(MI, BB);
+
+  case X86::EH_SjLj_LongJmp32:
+  case X86::EH_SjLj_LongJmp64:
+    return emitEHSjLjLongJmp(MI, BB);
   }
 }
 
@@ -13331,12 +14267,12 @@ static SDValue PerformShuffleCombine(SDNode *N, SelectionDAG &DAG,
 }
 
 
-/// DCI, PerformTruncateCombine - Converts truncate operation to
+/// PerformTruncateCombine - Converts truncate operation to
 /// a sequence of vector shuffle operations.
 /// It is possible when we truncate 256-bit vector to 128-bit vector
-
-SDValue X86TargetLowering::PerformTruncateCombine(SDNode *N, SelectionDAG &DAG,
-                                                  DAGCombinerInfo &DCI) const {
+static SDValue PerformTruncateCombine(SDNode *N, SelectionDAG &DAG,
+                                      TargetLowering::DAGCombinerInfo &DCI,
+                                      const X86Subtarget *Subtarget)  {
   if (!DCI.isBeforeLegalizeOps())
     return SDValue();
 
@@ -13528,7 +14464,7 @@ static SDValue XFormVExtractWithShuffleIntoLoad(SDNode *N, SelectionDAG &DAG,
     // alignment is valid.
     unsigned Align = LN0->getAlignment();
     const TargetLowering &TLI = DAG.getTargetLoweringInfo();
-    unsigned NewAlign = TLI.getTargetData()->
+    unsigned NewAlign = TLI.getDataLayout()->
       getABITypeAlignment(VT.getTypeForEVT(*DAG.getContext()));
 
     if (NewAlign > Align || !TLI.isOperationLegalOrCustom(ISD::LOAD, VT))
@@ -13559,6 +14495,14 @@ static SDValue PerformEXTRACT_VECTOR_ELTCombine(SDNode *N, SelectionDAG &DAG,
     return NewOp;
 
   SDValue InputVector = N->getOperand(0);
+  // Detect whether we are trying to convert from mmx to i32 and the bitcast
+  // from mmx to v2i32 has a single usage.
+  if (InputVector.getNode()->getOpcode() == llvm::ISD::BITCAST &&
+      InputVector.getNode()->getOperand(0).getValueType() == MVT::x86mmx &&
+      InputVector.hasOneUse() && N->getValueType(0) == MVT::i32)
+    return DAG.getNode(X86ISD::MMX_MOVD2W, InputVector.getDebugLoc(),
+                       N->getValueType(0),
+                       InputVector.getNode()->getOperand(0));
 
   // Only operate on vectors of 4 elements, where the alternative shuffling
   // gets to be more expensive.
@@ -13959,7 +14903,7 @@ static SDValue PerformSELECTCombine(SDNode *N, SelectionDAG &DAG,
 //
 // where Op could be BRCOND or CMOV.
 //
-static SDValue BoolTestSetCCCombine(SDValue Cmp, X86::CondCode &CC) {
+static SDValue checkBoolTestSetCCCombine(SDValue Cmp, X86::CondCode &CC) {
   // Quit if not CMP and SUB with its value result used.
   if (Cmp.getOpcode() != X86ISD::CMP &&
       (Cmp.getOpcode() != X86ISD::SUB || Cmp.getNode()->hasAnyUseOfValue(0)))
@@ -13995,40 +14939,55 @@ static SDValue BoolTestSetCCCombine(SDValue Cmp, X86::CondCode &CC) {
   if (SetCC.getOpcode() == ISD::ZERO_EXTEND)
     SetCC = SetCC.getOperand(0);
 
-  // Quit if not SETCC.
-  // FIXME: So far we only handle the boolean value generated from SETCC. If
-  // there is other ways to generate boolean values, we need handle them here
-  // as well.
-  if (SetCC.getOpcode() != X86ISD::SETCC)
-    return SDValue();
-
-  // Set the condition code or opposite one if necessary.
-  CC = X86::CondCode(SetCC.getConstantOperandVal(0));
-  if (needOppositeCond)
-    CC = X86::GetOppositeBranchCondition(CC);
-
-  return SetCC.getOperand(1);
-}
-
-static bool IsValidFCMOVCondition(X86::CondCode CC) {
-  switch (CC) {
-  default:
-    return false;
-  case X86::COND_B:
-  case X86::COND_BE:
-  case X86::COND_E:
-  case X86::COND_P:
-  case X86::COND_AE:
-  case X86::COND_A:
-  case X86::COND_NE:
-  case X86::COND_NP:
-    return true;
+  switch (SetCC.getOpcode()) {
+  case X86ISD::SETCC:
+    // Set the condition code or opposite one if necessary.
+    CC = X86::CondCode(SetCC.getConstantOperandVal(0));
+    if (needOppositeCond)
+      CC = X86::GetOppositeBranchCondition(CC);
+    return SetCC.getOperand(1);
+  case X86ISD::CMOV: {
+    // Check whether false/true value has canonical one, i.e. 0 or 1.
+    ConstantSDNode *FVal = dyn_cast<ConstantSDNode>(SetCC.getOperand(0));
+    ConstantSDNode *TVal = dyn_cast<ConstantSDNode>(SetCC.getOperand(1));
+    // Quit if true value is not a constant.
+    if (!TVal)
+      return SDValue();
+    // Quit if false value is not a constant.
+    if (!FVal) {
+      // A special case for rdrand, where 0 is set if false cond is found.
+      SDValue Op = SetCC.getOperand(0);
+      if (Op.getOpcode() != X86ISD::RDRAND)
+        return SDValue();
+    }
+    // Quit if false value is not the constant 0 or 1.
+    bool FValIsFalse = true;
+    if (FVal && FVal->getZExtValue() != 0) {
+      if (FVal->getZExtValue() != 1)
+        return SDValue();
+      // If FVal is 1, opposite cond is needed.
+      needOppositeCond = !needOppositeCond;
+      FValIsFalse = false;
+    }
+    // Quit if TVal is not the constant opposite of FVal.
+    if (FValIsFalse && TVal->getZExtValue() != 1)
+      return SDValue();
+    if (!FValIsFalse && TVal->getZExtValue() != 0)
+      return SDValue();
+    CC = X86::CondCode(SetCC.getConstantOperandVal(2));
+    if (needOppositeCond)
+      CC = X86::GetOppositeBranchCondition(CC);
+    return SetCC.getOperand(3);
   }
+  }
+
+  return SDValue();
 }
 
 /// Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL]
 static SDValue PerformCMOVCombine(SDNode *N, SelectionDAG &DAG,
-                                  TargetLowering::DAGCombinerInfo &DCI) {
+                                  TargetLowering::DAGCombinerInfo &DCI,
+                                  const X86Subtarget *Subtarget) {
   DebugLoc DL = N->getDebugLoc();
 
   // If the flag operand isn't dead, don't touch this CMOV.
@@ -14053,10 +15012,10 @@ static SDValue PerformCMOVCombine(SDNode *N, SelectionDAG &DAG,
 
   SDValue Flags;
 
-  Flags = BoolTestSetCCCombine(Cond, CC);
+  Flags = checkBoolTestSetCCCombine(Cond, CC);
   if (Flags.getNode() &&
       // Extra check as FCMOV only supports a subset of X86 cond.
-      (FalseOp.getValueType() != MVT::f80 || IsValidFCMOVCondition(CC))) {
+      (FalseOp.getValueType() != MVT::f80 || hasFPCMov(CC))) {
     SDValue Ops[] = { FalseOp, TrueOp,
                       DAG.getConstant(CC, MVT::i8), Flags };
     return DAG.getNode(X86ISD::CMOV, DL, N->getVTList(),
@@ -14073,6 +15032,7 @@ static SDValue PerformCMOVCombine(SDNode *N, SelectionDAG &DAG,
       if (TrueC->getAPIntValue().ult(FalseC->getAPIntValue())) {
         CC = X86::GetOppositeBranchCondition(CC);
         std::swap(TrueC, FalseC);
+        std::swap(TrueOp, FalseOp);
       }
 
       // Optimize C ? 8 : 0 -> zext(setcc(C)) << 3.  Likewise for any pow2/0.
@@ -14155,6 +15115,46 @@ static SDValue PerformCMOVCombine(SDNode *N, SelectionDAG &DAG,
       }
     }
   }
+
+  // Handle these cases:
+  //   (select (x != c), e, c) -> select (x != c), e, x),
+  //   (select (x == c), c, e) -> select (x == c), x, e)
+  // where the c is an integer constant, and the "select" is the combination
+  // of CMOV and CMP.
+  //
+  // The rationale for this change is that the conditional-move from a constant
+  // needs two instructions, however, conditional-move from a register needs
+  // only one instruction.
+  //
+  // CAVEAT: By replacing a constant with a symbolic value, it may obscure
+  //  some instruction-combining opportunities. This opt needs to be
+  //  postponed as late as possible.
+  //
+  if (!DCI.isBeforeLegalize() && !DCI.isBeforeLegalizeOps()) {
+    // the DCI.xxxx conditions are provided to postpone the optimization as
+    // late as possible.
+
+    ConstantSDNode *CmpAgainst = 0;
+    if ((Cond.getOpcode() == X86ISD::CMP || Cond.getOpcode() == X86ISD::SUB) &&
+        (CmpAgainst = dyn_cast<ConstantSDNode>(Cond.getOperand(1))) &&
+        dyn_cast<ConstantSDNode>(Cond.getOperand(0)) == 0) {
+
+      if (CC == X86::COND_NE &&
+          CmpAgainst == dyn_cast<ConstantSDNode>(FalseOp)) {
+        CC = X86::GetOppositeBranchCondition(CC);
+        std::swap(TrueOp, FalseOp);
+      }
+
+      if (CC == X86::COND_E &&
+          CmpAgainst == dyn_cast<ConstantSDNode>(TrueOp)) {
+        SDValue Ops[] = { FalseOp, Cond.getOperand(0),
+                          DAG.getConstant(CC, MVT::i8), Cond };
+        return DAG.getNode(X86ISD::CMOV, DL, N->getVTList (), Ops,
+                           array_lengthof(Ops));
+      }
+    }
+  }
+
   return SDValue();
 }
 
@@ -14811,11 +15811,11 @@ static SDValue PerformLOADCombine(SDNode *N, SelectionDAG &DAG,
   ISD::LoadExtType Ext = Ld->getExtensionType();
 
   // If this is a vector EXT Load then attempt to optimize it using a
-  // shuffle. We need SSE4 for the shuffles.
+  // shuffle. We need SSSE3 shuffles.
   // TODO: It is possible to support ZExt by zeroing the undef values
   // during the shuffle phase or after the shuffle.
   if (RegVT.isVector() && RegVT.isInteger() &&
-      Ext == ISD::EXTLOAD && Subtarget->hasSSE41()) {
+      Ext == ISD::EXTLOAD && Subtarget->hasSSSE3()) {
     assert(MemVT != RegVT && "Cannot extend to the same type");
     assert(MemVT.isVector() && "Must load a vector from memory");
 
@@ -15041,7 +16041,8 @@ static SDValue PerformSTORECombine(SDNode *N, SelectionDAG &DAG,
     return SDValue();
 
   const Function *F = DAG.getMachineFunction().getFunction();
-  bool NoImplicitFloatOps = F->hasFnAttr(Attribute::NoImplicitFloat);
+  bool NoImplicitFloatOps = F->getFnAttributes().
+    hasAttribute(Attributes::NoImplicitFloat);
   bool F64IsLegal = !DAG.getTarget().Options.UseSoftFloat && !NoImplicitFloatOps
                      && Subtarget->hasSSE2();
   if ((VT.isVector() ||
@@ -15313,6 +16314,29 @@ static SDValue PerformFORCombine(SDNode *N, SelectionDAG &DAG) {
   return SDValue();
 }
 
+/// PerformFMinFMaxCombine - Do target-specific dag combines on X86ISD::FMIN and
+/// X86ISD::FMAX nodes.
+static SDValue PerformFMinFMaxCombine(SDNode *N, SelectionDAG &DAG) {
+  assert(N->getOpcode() == X86ISD::FMIN || N->getOpcode() == X86ISD::FMAX);
+
+  // Only perform optimizations if UnsafeMath is used.
+  if (!DAG.getTarget().Options.UnsafeFPMath)
+    return SDValue();
+
+  // If we run in unsafe-math mode, then convert the FMAX and FMIN nodes
+  // into FMINC and FMAXC, which are Commutative operations.
+  unsigned NewOp = 0;
+  switch (N->getOpcode()) {
+    default: llvm_unreachable("unknown opcode");
+    case X86ISD::FMIN:  NewOp = X86ISD::FMINC; break;
+    case X86ISD::FMAX:  NewOp = X86ISD::FMAXC; break;
+  }
+
+  return DAG.getNode(NewOp, N->getDebugLoc(), N->getValueType(0),
+                     N->getOperand(0), N->getOperand(1));
+}
+
+
 /// PerformFANDCombine - Do target-specific dag combines on X86ISD::FAND nodes.
 static SDValue PerformFANDCombine(SDNode *N, SelectionDAG &DAG) {
   // FAND(0.0, x) -> 0.0
@@ -15418,8 +16442,13 @@ static SDValue PerformFMACombine(SDNode *N, SelectionDAG &DAG,
   DebugLoc dl = N->getDebugLoc();
   EVT VT = N->getValueType(0);
 
+  // Let legalize expand this if it isn't a legal type yet.
+  if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
+    return SDValue();
+
   EVT ScalarVT = VT.getScalarType();
-  if ((ScalarVT != MVT::f32 && ScalarVT != MVT::f64) || !Subtarget->hasFMA())
+  if ((ScalarVT != MVT::f32 && ScalarVT != MVT::f64) ||
+      (!Subtarget->hasFMA() && !Subtarget->hasFMA4()))
     return SDValue();
 
   SDValue A = N->getOperand(0);
@@ -15441,9 +16470,10 @@ static SDValue PerformFMACombine(SDNode *N, SelectionDAG &DAG,
 
   unsigned Opcode;
   if (!NegMul)
-    Opcode = (!NegC)? X86ISD::FMADD : X86ISD::FMSUB;
+    Opcode = (!NegC) ? X86ISD::FMADD : X86ISD::FMSUB;
   else
-    Opcode = (!NegC)? X86ISD::FNMADD : X86ISD::FNMSUB;
+    Opcode = (!NegC) ? X86ISD::FNMADD : X86ISD::FNMSUB;
+
   return DAG.getNode(Opcode, dl, VT, A, B, C);
 }
 
@@ -15540,24 +16570,51 @@ static SDValue PerformISDSETCCCombine(SDNode *N, SelectionDAG &DAG) {
   return SDValue();
 }
 
+// Helper function of PerformSETCCCombine. It is to materialize "setb reg" 
+// as "sbb reg,reg", since it can be extended without zext and produces 
+// an all-ones bit which is more useful than 0/1 in some cases.
+static SDValue MaterializeSETB(DebugLoc DL, SDValue EFLAGS, SelectionDAG &DAG) {
+  return DAG.getNode(ISD::AND, DL, MVT::i8,
+                     DAG.getNode(X86ISD::SETCC_CARRY, DL, MVT::i8,
+                                 DAG.getConstant(X86::COND_B, MVT::i8), EFLAGS),
+                     DAG.getConstant(1, MVT::i8));
+}
+
 // Optimize  RES = X86ISD::SETCC CONDCODE, EFLAG_INPUT
-static SDValue PerformSETCCCombine(SDNode *N, SelectionDAG &DAG) {
+static SDValue PerformSETCCCombine(SDNode *N, SelectionDAG &DAG,
+                                   TargetLowering::DAGCombinerInfo &DCI,
+                                   const X86Subtarget *Subtarget) {
   DebugLoc DL = N->getDebugLoc();
   X86::CondCode CC = X86::CondCode(N->getConstantOperandVal(0));
   SDValue EFLAGS = N->getOperand(1);
 
+  if (CC == X86::COND_A) {
+    // Try to convert COND_A into COND_B in an attempt to facilitate 
+    // materializing "setb reg".
+    //
+    // Do not flip "e > c", where "c" is a constant, because Cmp instruction
+    // cannot take an immediate as its first operand.
+    //
+    if (EFLAGS.getOpcode() == X86ISD::SUB && EFLAGS.hasOneUse() && 
+        EFLAGS.getValueType().isInteger() &&
+        !isa<ConstantSDNode>(EFLAGS.getOperand(1))) {
+      SDValue NewSub = DAG.getNode(X86ISD::SUB, EFLAGS.getDebugLoc(),
+                                   EFLAGS.getNode()->getVTList(),
+                                   EFLAGS.getOperand(1), EFLAGS.getOperand(0));
+      SDValue NewEFLAGS = SDValue(NewSub.getNode(), EFLAGS.getResNo());
+      return MaterializeSETB(DL, NewEFLAGS, DAG);
+    }
+  }
+
   // Materialize "setb reg" as "sbb reg,reg", since it can be extended without
   // a zext and produces an all-ones bit which is more useful than 0/1 in some
   // cases.
   if (CC == X86::COND_B)
-    return DAG.getNode(ISD::AND, DL, MVT::i8,
-                       DAG.getNode(X86ISD::SETCC_CARRY, DL, MVT::i8,
-                                   DAG.getConstant(CC, MVT::i8), EFLAGS),
-                       DAG.getConstant(1, MVT::i8));
+    return MaterializeSETB(DL, EFLAGS, DAG);
 
   SDValue Flags;
 
-  Flags = BoolTestSetCCCombine(EFLAGS, CC);
+  Flags = checkBoolTestSetCCCombine(EFLAGS, CC);
   if (Flags.getNode()) {
     SDValue Cond = DAG.getConstant(CC, MVT::i8);
     return DAG.getNode(X86ISD::SETCC, DL, N->getVTList(), Cond, Flags);
@@ -15579,7 +16636,7 @@ static SDValue PerformBrCondCombine(SDNode *N, SelectionDAG &DAG,
 
   SDValue Flags;
 
-  Flags = BoolTestSetCCCombine(EFLAGS, CC);
+  Flags = checkBoolTestSetCCCombine(EFLAGS, CC);
   if (Flags.getNode()) {
     SDValue Cond = DAG.getConstant(CC, MVT::i8);
     return DAG.getNode(X86ISD::BRCOND, DL, N->getVTList(), Chain, Dest, Cond,
@@ -15589,23 +16646,6 @@ static SDValue PerformBrCondCombine(SDNode *N, SelectionDAG &DAG,
   return SDValue();
 }
 
-static SDValue PerformUINT_TO_FPCombine(SDNode *N, SelectionDAG &DAG) {
-  SDValue Op0 = N->getOperand(0);
-  EVT InVT = Op0->getValueType(0);
-
-  // UINT_TO_FP(v4i8) -> SINT_TO_FP(ZEXT(v4i8 to v4i32))
-  if (InVT == MVT::v8i8 || InVT == MVT::v4i8) {
-    DebugLoc dl = N->getDebugLoc();
-    MVT DstVT = InVT == MVT::v4i8 ? MVT::v4i32 : MVT::v8i32;
-    SDValue P = DAG.getNode(ISD::ZERO_EXTEND, dl, DstVT, Op0);
-    // Notice that we use SINT_TO_FP because we know that the high bits
-    // are zero and SINT_TO_FP is better supported by the hardware.
-    return DAG.getNode(ISD::SINT_TO_FP, dl, N->getValueType(0), P);
-  }
-
-  return SDValue();
-}
-
 static SDValue PerformSINT_TO_FPCombine(SDNode *N, SelectionDAG &DAG,
                                         const X86TargetLowering *XTLI) {
   SDValue Op0 = N->getOperand(0);
@@ -15637,20 +16677,6 @@ static SDValue PerformSINT_TO_FPCombine(SDNode *N, SelectionDAG &DAG,
   return SDValue();
 }
 
-static SDValue PerformFP_TO_SINTCombine(SDNode *N, SelectionDAG &DAG) {
-  EVT VT = N->getValueType(0);
-
-  // v4i8 = FP_TO_SINT() -> v4i8 = TRUNCATE (V4i32 = FP_TO_SINT()
-  if (VT == MVT::v8i8 || VT == MVT::v4i8) {
-    DebugLoc dl = N->getDebugLoc();
-    MVT DstVT = VT == MVT::v4i8 ? MVT::v4i32 : MVT::v8i32;
-    SDValue I = DAG.getNode(ISD::FP_TO_SINT, dl, DstVT, N->getOperand(0));
-    return DAG.getNode(ISD::TRUNCATE, dl, VT, I);
-  }
-
-  return SDValue();
-}
-
 // Optimize RES, EFLAGS = X86ISD::ADC LHS, RHS, EFLAGS
 static SDValue PerformADCCombine(SDNode *N, SelectionDAG &DAG,
                                  X86TargetLowering::DAGCombinerInfo &DCI) {
@@ -15765,6 +16791,21 @@ static SDValue PerformSubCombine(SDNode *N, SelectionDAG &DAG,
   return OptimizeConditionalInDecrement(N, DAG);
 }
 
+/// performVZEXTCombine - Performs build vector combines
+static SDValue performVZEXTCombine(SDNode *N, SelectionDAG &DAG,
+                                        TargetLowering::DAGCombinerInfo &DCI,
+                                        const X86Subtarget *Subtarget) {
+  // (vzext (bitcast (vzext (x)) -> (vzext x)
+  SDValue In = N->getOperand(0);
+  while (In.getOpcode() == ISD::BITCAST)
+    In = In.getOperand(0);
+
+  if (In.getOpcode() != X86ISD::VZEXT)
+    return SDValue();
+
+  return DAG.getNode(X86ISD::VZEXT, N->getDebugLoc(), N->getValueType(0), In.getOperand(0));
+}
+
 SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
                                              DAGCombinerInfo &DCI) const {
   SelectionDAG &DAG = DCI.DAG;
@@ -15774,7 +16815,7 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
     return PerformEXTRACT_VECTOR_ELTCombine(N, DAG, DCI);
   case ISD::VSELECT:
   case ISD::SELECT:         return PerformSELECTCombine(N, DAG, DCI, Subtarget);
-  case X86ISD::CMOV:        return PerformCMOVCombine(N, DAG, DCI);
+  case X86ISD::CMOV:        return PerformCMOVCombine(N, DAG, DCI, Subtarget);
   case ISD::ADD:            return PerformAddCombine(N, DAG, Subtarget);
   case ISD::SUB:            return PerformSubCombine(N, DAG, Subtarget);
   case X86ISD::ADC:         return PerformADCCombine(N, DAG, DCI);
@@ -15787,23 +16828,24 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
   case ISD::XOR:            return PerformXorCombine(N, DAG, DCI, Subtarget);
   case ISD::LOAD:           return PerformLOADCombine(N, DAG, DCI, Subtarget);
   case ISD::STORE:          return PerformSTORECombine(N, DAG, Subtarget);
-  case ISD::UINT_TO_FP:     return PerformUINT_TO_FPCombine(N, DAG);
   case ISD::SINT_TO_FP:     return PerformSINT_TO_FPCombine(N, DAG, this);
-  case ISD::FP_TO_SINT:     return PerformFP_TO_SINTCombine(N, DAG);
   case ISD::FADD:           return PerformFADDCombine(N, DAG, Subtarget);
   case ISD::FSUB:           return PerformFSUBCombine(N, DAG, Subtarget);
   case X86ISD::FXOR:
   case X86ISD::FOR:         return PerformFORCombine(N, DAG);
+  case X86ISD::FMIN:
+  case X86ISD::FMAX:        return PerformFMinFMaxCombine(N, DAG);
   case X86ISD::FAND:        return PerformFANDCombine(N, DAG);
   case X86ISD::BT:          return PerformBTCombine(N, DAG, DCI);
   case X86ISD::VZEXT_MOVL:  return PerformVZEXT_MOVLCombine(N, DAG);
   case ISD::ANY_EXTEND:
   case ISD::ZERO_EXTEND:    return PerformZExtCombine(N, DAG, DCI, Subtarget);
   case ISD::SIGN_EXTEND:    return PerformSExtCombine(N, DAG, DCI, Subtarget);
-  case ISD::TRUNCATE:       return PerformTruncateCombine(N, DAG, DCI);
+  case ISD::TRUNCATE:       return PerformTruncateCombine(N, DAG,DCI,Subtarget);
   case ISD::SETCC:          return PerformISDSETCCCombine(N, DAG);
-  case X86ISD::SETCC:       return PerformSETCCCombine(N, DAG);
+  case X86ISD::SETCC:       return PerformSETCCCombine(N, DAG, DCI, Subtarget);
   case X86ISD::BRCOND:      return PerformBrCondCombine(N, DAG, DCI, Subtarget);
+  case X86ISD::VZEXT:       return performVZEXTCombine(N, DAG, DCI, Subtarget);
   case X86ISD::SHUFP:       // Handle all target specific shuffles
   case X86ISD::PALIGN:
   case X86ISD::UNPCKH:
@@ -16231,7 +17273,7 @@ void X86TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
     return;
   case 'K':
     if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
-      if ((int8_t)C->getSExtValue() == C->getSExtValue()) {
+      if (isInt<8>(C->getSExtValue())) {
         Result = DAG.getTargetConstant(C->getZExtValue(), Op.getValueType());
         break;
       }
@@ -16556,3 +17598,207 @@ X86TargetLowering::getRegForInlineAsmConstraint(const std::string &Constraint,
 
   return Res;
 }
+
+//===----------------------------------------------------------------------===//
+//
+// X86 cost model.
+//
+//===----------------------------------------------------------------------===//
+
+struct X86CostTblEntry {
+  int ISD;
+  MVT Type;
+  unsigned Cost;
+};
+
+static int
+FindInTable(const X86CostTblEntry *Tbl, unsigned len, int ISD, MVT Ty) {
+  for (unsigned int i = 0; i < len; ++i)
+    if (Tbl[i].ISD == ISD && Tbl[i].Type == Ty)
+      return i;
+
+  // Could not find an entry.
+  return -1;
+}
+
+struct X86TypeConversionCostTblEntry {
+  int ISD;
+  MVT Dst;
+  MVT Src;
+  unsigned Cost;
+};
+
+static int
+FindInConvertTable(const X86TypeConversionCostTblEntry *Tbl, unsigned len,
+                   int ISD, MVT Dst, MVT Src) {
+  for (unsigned int i = 0; i < len; ++i)
+    if (Tbl[i].ISD == ISD && Tbl[i].Src == Src && Tbl[i].Dst == Dst)
+      return i;
+
+  // Could not find an entry.
+  return -1;
+}
+
+unsigned
+X86VectorTargetTransformInfo::getArithmeticInstrCost(unsigned Opcode,
+                                                     Type *Ty) const {
+  // Legalize the type.
+  std::pair<unsigned, MVT> LT = getTypeLegalizationCost(Ty);
+
+  int ISD = InstructionOpcodeToISD(Opcode);
+  assert(ISD && "Invalid opcode");
+
+  const X86Subtarget &ST = TLI->getTargetMachine().getSubtarget<X86Subtarget>();
+
+  static const X86CostTblEntry AVX1CostTable[] = {
+    // We don't have to scalarize unsupported ops. We can issue two half-sized
+    // operations and we only need to extract the upper YMM half.
+    // Two ops + 1 extract + 1 insert = 4.
+    { ISD::MUL,     MVT::v8i32,    4 },
+    { ISD::SUB,     MVT::v8i32,    4 },
+    { ISD::ADD,     MVT::v8i32,    4 },
+    { ISD::MUL,     MVT::v4i64,    4 },
+    { ISD::SUB,     MVT::v4i64,    4 },
+    { ISD::ADD,     MVT::v4i64,    4 },
+    };
+
+  // Look for AVX1 lowering tricks.
+  if (ST.hasAVX()) {
+    int Idx = FindInTable(AVX1CostTable, array_lengthof(AVX1CostTable), ISD,
+                          LT.second);
+    if (Idx != -1)
+      return LT.first * AVX1CostTable[Idx].Cost;
+  }
+  // Fallback to the default implementation.
+  return VectorTargetTransformImpl::getArithmeticInstrCost(Opcode, Ty);
+}
+
+unsigned
+X86VectorTargetTransformInfo::getVectorInstrCost(unsigned Opcode, Type *Val,
+                                                 unsigned Index) const {
+  assert(Val->isVectorTy() && "This must be a vector type");
+
+  if (Index != -1U) {
+    // Legalize the type.
+    std::pair<unsigned, MVT> LT = getTypeLegalizationCost(Val);
+
+    // This type is legalized to a scalar type.
+    if (!LT.second.isVector())
+      return 0;
+
+    // The type may be split. Normalize the index to the new type.
+    unsigned Width = LT.second.getVectorNumElements();
+    Index = Index % Width;
+
+    // Floating point scalars are already located in index #0.
+    if (Val->getScalarType()->isFloatingPointTy() && Index == 0)
+      return 0;
+  }
+
+  return VectorTargetTransformImpl::getVectorInstrCost(Opcode, Val, Index);
+}
+
+unsigned X86VectorTargetTransformInfo::getCmpSelInstrCost(unsigned Opcode,
+                                                          Type *ValTy,
+                                                          Type *CondTy) const {
+  // Legalize the type.
+  std::pair<unsigned, MVT> LT = getTypeLegalizationCost(ValTy);
+
+  MVT MTy = LT.second;
+
+  int ISD = InstructionOpcodeToISD(Opcode);
+  assert(ISD && "Invalid opcode");
+
+  const X86Subtarget &ST =
+  TLI->getTargetMachine().getSubtarget<X86Subtarget>();
+
+  static const X86CostTblEntry SSE42CostTbl[] = {
+    { ISD::SETCC,   MVT::v2f64,   1 },
+    { ISD::SETCC,   MVT::v4f32,   1 },
+    { ISD::SETCC,   MVT::v2i64,   1 },
+    { ISD::SETCC,   MVT::v4i32,   1 },
+    { ISD::SETCC,   MVT::v8i16,   1 },
+    { ISD::SETCC,   MVT::v16i8,   1 },
+  };
+
+  static const X86CostTblEntry AVX1CostTbl[] = {
+    { ISD::SETCC,   MVT::v4f64,   1 },
+    { ISD::SETCC,   MVT::v8f32,   1 },
+    // AVX1 does not support 8-wide integer compare.
+    { ISD::SETCC,   MVT::v4i64,   4 },
+    { ISD::SETCC,   MVT::v8i32,   4 },
+    { ISD::SETCC,   MVT::v16i16,  4 },
+    { ISD::SETCC,   MVT::v32i8,   4 },
+  };
+
+  static const X86CostTblEntry AVX2CostTbl[] = {
+    { ISD::SETCC,   MVT::v4i64,   1 },
+    { ISD::SETCC,   MVT::v8i32,   1 },
+    { ISD::SETCC,   MVT::v16i16,  1 },
+    { ISD::SETCC,   MVT::v32i8,   1 },
+  };
+
+  if (ST.hasSSE42()) {
+    int Idx = FindInTable(SSE42CostTbl, array_lengthof(SSE42CostTbl), ISD, MTy);
+    if (Idx != -1)
+      return LT.first * SSE42CostTbl[Idx].Cost;
+  }
+
+  if (ST.hasAVX()) {
+    int Idx = FindInTable(AVX1CostTbl, array_lengthof(AVX1CostTbl), ISD, MTy);
+    if (Idx != -1)
+      return LT.first * AVX1CostTbl[Idx].Cost;
+  }
+
+  if (ST.hasAVX2()) {
+    int Idx = FindInTable(AVX2CostTbl, array_lengthof(AVX2CostTbl), ISD, MTy);
+    if (Idx != -1)
+      return LT.first * AVX2CostTbl[Idx].Cost;
+  }
+
+  return VectorTargetTransformImpl::getCmpSelInstrCost(Opcode, ValTy, CondTy);
+}
+
+unsigned X86VectorTargetTransformInfo::getCastInstrCost(unsigned Opcode,
+                                                        Type *Dst,
+                                                        Type *Src) const {
+  int ISD = InstructionOpcodeToISD(Opcode);
+  assert(ISD && "Invalid opcode");
+
+  EVT SrcTy = TLI->getValueType(Src);
+  EVT DstTy = TLI->getValueType(Dst);
+
+  if (!SrcTy.isSimple() || !DstTy.isSimple())
+    return VectorTargetTransformImpl::getCastInstrCost(Opcode, Dst, Src);
+
+  const X86Subtarget &ST = TLI->getTargetMachine().getSubtarget<X86Subtarget>();
+
+  static const X86TypeConversionCostTblEntry AVXConversionTbl[] = {
+    { ISD::SIGN_EXTEND, MVT::v8i32, MVT::v8i16, 1 },
+    { ISD::ZERO_EXTEND, MVT::v8i32, MVT::v8i16, 1 },
+    { ISD::SIGN_EXTEND, MVT::v4i64, MVT::v4i32, 1 },
+    { ISD::ZERO_EXTEND, MVT::v4i64, MVT::v4i32, 1 },
+    { ISD::TRUNCATE,    MVT::v4i32, MVT::v4i64, 1 },
+    { ISD::TRUNCATE,    MVT::v8i16, MVT::v8i32, 1 },
+    { ISD::SINT_TO_FP,  MVT::v8f32, MVT::v8i8,  1 },
+    { ISD::SINT_TO_FP,  MVT::v4f32, MVT::v4i8,  1 },
+    { ISD::UINT_TO_FP,  MVT::v8f32, MVT::v8i8,  1 },
+    { ISD::UINT_TO_FP,  MVT::v4f32, MVT::v4i8,  1 },
+    { ISD::FP_TO_SINT,  MVT::v8i8,  MVT::v8f32, 1 },
+    { ISD::FP_TO_SINT,  MVT::v4i8,  MVT::v4f32, 1 },
+    { ISD::ZERO_EXTEND, MVT::v8i32, MVT::v8i1,  6 },
+    { ISD::SIGN_EXTEND, MVT::v8i32, MVT::v8i1,  9 },
+    { ISD::TRUNCATE,    MVT::v8i32, MVT::v8i64, 3 },
+  };
+
+  if (ST.hasAVX()) {
+    int Idx = FindInConvertTable(AVXConversionTbl,
+                                 array_lengthof(AVXConversionTbl),
+                                 ISD, DstTy.getSimpleVT(), SrcTy.getSimpleVT());
+    if (Idx != -1)
+      return AVXConversionTbl[Idx].Cost;
+  }
+
+  return VectorTargetTransformImpl::getCastInstrCost(Opcode, Dst, Src);
+}
+
diff --git a/lib/Target/X86/X86ISelLowering.h b/lib/Target/X86/X86ISelLowering.h
index 896d067..465c603 100644
--- a/lib/Target/X86/X86ISelLowering.h
+++ b/lib/Target/X86/X86ISelLowering.h
@@ -19,6 +19,7 @@
 #include "X86RegisterInfo.h"
 #include "X86MachineFunctionInfo.h"
 #include "llvm/Target/TargetLowering.h"
+#include "llvm/Target/TargetTransformImpl.h"
 #include "llvm/Target/TargetOptions.h"
 #include "llvm/CodeGen/FastISel.h"
 #include "llvm/CodeGen/SelectionDAG.h"
@@ -142,6 +143,10 @@ namespace llvm {
       /// mnemonic, so do I; blame Intel.
       MOVDQ2Q,
 
+      /// MMX_MOVD2W - Copies a 32-bit value from the low word of a MMX
+      /// vector to a GPR.
+      MMX_MOVD2W,
+
       /// PEXTRB - Extract an 8-bit value from a vector and zero extend it to
       /// i32, corresponds to X86::PEXTRB.
       PEXTRB,
@@ -195,6 +200,9 @@ namespace llvm {
       ///
       FMAX, FMIN,
 
+      /// FMAXC, FMINC - Commutative FMIN and FMAX.
+      FMAXC, FMINC,
+
       /// FRSQRT, FRCP - Floating point reciprocal-sqrt and reciprocal
       /// approximation.  Note that these typically require refinement
       /// in order to obtain suitable precision.
@@ -214,6 +222,12 @@ namespace llvm {
       // EH_RETURN - Exception Handling helpers.
       EH_RETURN,
 
+      // EH_SJLJ_SETJMP - SjLj exception handling setjmp.
+      EH_SJLJ_SETJMP,
+
+      // EH_SJLJ_LONGJMP - SjLj exception handling longjmp.
+      EH_SJLJ_LONGJMP,
+
       /// TC_RETURN - Tail call return.
       ///   operand #0 chain
       ///   operand #1 callee (register or absolute)
@@ -227,9 +241,18 @@ namespace llvm {
       // VSEXT_MOVL - Vector move low and sign extend.
       VSEXT_MOVL,
 
+      // VZEXT - Vector integer zero-extend.
+      VZEXT,
+
+      // VSEXT - Vector integer signed-extend.
+      VSEXT,
+
       // VFPEXT - Vector FP extend.
       VFPEXT,
 
+      // VFPROUND - Vector FP round.
+      VFPROUND,
+
       // VSHL, VSRL - 128-bit vector logical left / right shift
       VSHLDQ, VSRLDQ,
 
@@ -345,6 +368,10 @@ namespace llvm {
       ATOMXOR64_DAG,
       ATOMAND64_DAG,
       ATOMNAND64_DAG,
+      ATOMMAX64_DAG,
+      ATOMMIN64_DAG,
+      ATOMUMAX64_DAG,
+      ATOMUMIN64_DAG,
       ATOMSWAP64_DAG,
 
       // LCMPXCHG_DAG, LCMPXCHG8_DAG, LCMPXCHG16_DAG - Compare and swap.
@@ -458,10 +485,6 @@ namespace llvm {
     getPICJumpTableRelocBaseExpr(const MachineFunction *MF,
                                  unsigned JTI, MCContext &Ctx) const;
 
-    /// getStackPtrReg - Return the stack pointer register we are using: either
-    /// ESP or RSP.
-    unsigned getStackPtrReg() const { return X86StackPtr; }
-
     /// getByValTypeAlignment - Return the desired alignment for ByVal aggregate
     /// function arguments in the caller parameter area. For X86, aggregates
     /// that contains are placed at 16-byte boundaries while the rest are at
@@ -694,10 +717,7 @@ namespace llvm {
     /// make the right decision when generating code for different targets.
     const X86Subtarget *Subtarget;
     const X86RegisterInfo *RegInfo;
-    const TargetData *TD;
-
-    /// X86StackPtr - X86 physical register used as stack ptr.
-    unsigned X86StackPtr;
+    const DataLayout *TD;
 
     /// X86ScalarSSEf32, X86ScalarSSEf64 - Select between SSE or x87
     /// floating point ops.
@@ -741,6 +761,7 @@ namespace llvm {
                                            bool isVarArg,
                                            bool isCalleeStructRet,
                                            bool isCallerStructRet,
+                                           Type *RetTy,
                                     const SmallVectorImpl<ISD::OutputArg> &Outs,
                                     const SmallVectorImpl<SDValue> &OutVals,
                                     const SmallVectorImpl<ISD::InputArg> &Ins,
@@ -760,15 +781,11 @@ namespace llvm {
     SDValue LowerAsSplatVectorLoad(SDValue SrcOp, EVT VT, DebugLoc dl,
                                    SelectionDAG &DAG) const;
     SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
-    SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerEXTRACT_VECTOR_ELT_SSE4(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerINSERT_VECTOR_ELT_SSE4(SDValue Op, SelectionDAG &DAG) const;
-    SDValue LowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) const;
-    SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const;
-    SDValue LowerINSERT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerGlobalAddress(const GlobalValue *GV, DebugLoc dl,
@@ -782,12 +799,15 @@ namespace llvm {
     SDValue LowerUINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerUINT_TO_FP_i64(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerUINT_TO_FP_i32(SDValue Op, SelectionDAG &DAG) const;
+    SDValue lowerUINT_TO_FP_vec(SDValue Op, SelectionDAG &DAG) const;
+    SDValue lowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const;
+    SDValue lowerZERO_EXTEND(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerFP_TO_SINT(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerFP_TO_UINT(SDValue Op, SelectionDAG &DAG) const;
+    SDValue lowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerFABS(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerFNEG(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const;
-    SDValue LowerFGETSIGN(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerToBT(SDValue And, ISD::CondCode CC,
                       DebugLoc dl, SelectionDAG &DAG) const;
     SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const;
@@ -799,39 +819,26 @@ namespace llvm {
     SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerVAARG(SDValue Op, SelectionDAG &DAG) const;
-    SDValue LowerVACOPY(SDValue Op, SelectionDAG &DAG) const;
-    SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
-    SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerFRAME_TO_ARGS_OFFSET(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const;
+    SDValue lowerEH_SJLJ_SETJMP(SDValue Op, SelectionDAG &DAG) const;
+    SDValue lowerEH_SJLJ_LONGJMP(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerINIT_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;
-    SDValue LowerADJUST_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerFLT_ROUNDS_(SDValue Op, SelectionDAG &DAG) const;
-    SDValue LowerCTLZ(SDValue Op, SelectionDAG &DAG) const;
-    SDValue LowerCTLZ_ZERO_UNDEF(SDValue Op, SelectionDAG &DAG) const;
-    SDValue LowerCTTZ(SDValue Op, SelectionDAG &DAG) const;
-    SDValue LowerADD(SDValue Op, SelectionDAG &DAG) const;
-    SDValue LowerSUB(SDValue Op, SelectionDAG &DAG) const;
-    SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerShift(SDValue Op, SelectionDAG &DAG) const;
-    SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) const;
 
-    SDValue LowerCMP_SWAP(SDValue Op, SelectionDAG &DAG) const;
-    SDValue LowerLOAD_SUB(SDValue Op, SelectionDAG &DAG) const;
-    SDValue LowerREADCYCLECOUNTER(SDValue Op, SelectionDAG &DAG) const;
-    SDValue LowerMEMBARRIER(SDValue Op, SelectionDAG &DAG) const;
-    SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG) const;
     SDValue LowerSIGN_EXTEND_INREG(SDValue Op, SelectionDAG &DAG) const;
-    SDValue PerformTruncateCombine(SDNode* N, SelectionDAG &DAG, DAGCombinerInfo &DCI) const;
 
-    // Utility functions to help LowerVECTOR_SHUFFLE
-    SDValue LowerVECTOR_SHUFFLEv8i16(SDValue Op, SelectionDAG &DAG) const;
-    SDValue LowerVectorBroadcast(SDValue &Op, SelectionDAG &DAG) const;
+    // Utility functions to help LowerVECTOR_SHUFFLE & LowerBUILD_VECTOR
+    SDValue LowerVectorBroadcast(SDValue Op, SelectionDAG &DAG) const;
     SDValue NormalizeVectorShuffle(SDValue Op, SelectionDAG &DAG) const;
+    SDValue buildFromShuffleMostly(SDValue Op, SelectionDAG &DAG) const;
+
+    SDValue LowerVectorAllZeroTest(SDValue Op, SelectionDAG &DAG) const;
 
-    SDValue LowerVectorFpExtend(SDValue &Op, SelectionDAG &DAG) const;
+    SDValue lowerVectorIntExtend(SDValue Op, SelectionDAG &DAG) const;
 
     virtual SDValue
       LowerFormalArguments(SDValue Chain,
@@ -864,51 +871,17 @@ namespace llvm {
                    const SmallVectorImpl<ISD::OutputArg> &Outs,
                    LLVMContext &Context) const;
 
-    /// Utility function to emit string processing sse4.2 instructions
-    /// that return in xmm0.
-    /// This takes the instruction to expand, the associated machine basic
-    /// block, the number of args, and whether or not the second arg is
-    /// in memory or not.
-    MachineBasicBlock *EmitPCMP(MachineInstr *BInstr, MachineBasicBlock *BB,
-                                unsigned argNum, bool inMem) const;
-
-    /// Utility functions to emit monitor and mwait instructions. These
-    /// need to make sure that the arguments to the intrinsic are in the
-    /// correct registers.
-    MachineBasicBlock *EmitMonitor(MachineInstr *MI,
-                                   MachineBasicBlock *BB) const;
-    MachineBasicBlock *EmitMwait(MachineInstr *MI, MachineBasicBlock *BB) const;
-
-    /// Utility function to emit atomic bitwise operations (and, or, xor).
-    /// It takes the bitwise instruction to expand, the associated machine basic
-    /// block, and the associated X86 opcodes for reg/reg and reg/imm.
-    MachineBasicBlock *EmitAtomicBitwiseWithCustomInserter(
-                                                    MachineInstr *BInstr,
-                                                    MachineBasicBlock *BB,
-                                                    unsigned regOpc,
-                                                    unsigned immOpc,
-                                                    unsigned loadOpc,
-                                                    unsigned cxchgOpc,
-                                                    unsigned notOpc,
-                                                    unsigned EAXreg,
-                                              const TargetRegisterClass *RC,
-                                                    bool Invert = false) const;
-
-    MachineBasicBlock *EmitAtomicBit6432WithCustomInserter(
-                                                    MachineInstr *BInstr,
-                                                    MachineBasicBlock *BB,
-                                                    unsigned regOpcL,
-                                                    unsigned regOpcH,
-                                                    unsigned immOpcL,
-                                                    unsigned immOpcH,
-                                                    bool Invert = false) const;
-
-    /// Utility function to emit atomic min and max.  It takes the min/max
-    /// instruction to expand, the associated basic block, and the associated
-    /// cmov opcode for moving the min or max value.
-    MachineBasicBlock *EmitAtomicMinMaxWithCustomInserter(MachineInstr *BInstr,
-                                                          MachineBasicBlock *BB,
-                                                        unsigned cmovOpc) const;
+    /// Utility function to emit atomic-load-arith operations (and, or, xor,
+    /// nand, max, min, umax, umin). It takes the corresponding instruction to
+    /// expand, the associated machine basic block, and the associated X86
+    /// opcodes for reg/reg.
+    MachineBasicBlock *EmitAtomicLoadArith(MachineInstr *MI,
+                                           MachineBasicBlock *MBB) const;
+
+    /// Utility function to emit atomic-load-arith operations (and, or, xor,
+    /// nand, add, sub, swap) for 64-bit operands on 32-bit target.
+    MachineBasicBlock *EmitAtomicLoadArith6432(MachineInstr *MI,
+                                               MachineBasicBlock *MBB) const;
 
     // Utility function to emit the low-level va_arg code for X86-64.
     MachineBasicBlock *EmitVAARG64WithCustomInserter(
@@ -936,6 +909,12 @@ namespace llvm {
     MachineBasicBlock *emitLoweredTLSAddr(MachineInstr *MI,
                                           MachineBasicBlock *BB) const;
 
+    MachineBasicBlock *emitEHSjLjSetJmp(MachineInstr *MI,
+                                        MachineBasicBlock *MBB) const;
+
+    MachineBasicBlock *emitEHSjLjLongJmp(MachineInstr *MI,
+                                         MachineBasicBlock *MBB) const;
+
     /// Emit nodes that will be selected as "test Op0,Op0", or something
     /// equivalent, for use with the given x86 condition code.
     SDValue EmitTest(SDValue Op0, unsigned X86CC, SelectionDAG &DAG) const;
@@ -953,6 +932,23 @@ namespace llvm {
     FastISel *createFastISel(FunctionLoweringInfo &funcInfo,
                              const TargetLibraryInfo *libInfo);
   }
+
+  class X86VectorTargetTransformInfo : public VectorTargetTransformImpl {
+  public:
+    explicit X86VectorTargetTransformInfo(const TargetLowering *TL) :
+    VectorTargetTransformImpl(TL) {}
+
+    virtual unsigned getArithmeticInstrCost(unsigned Opcode, Type *Ty) const;
+
+    virtual unsigned getVectorInstrCost(unsigned Opcode, Type *Val,
+                                        unsigned Index) const;
+
+    unsigned getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
+                                Type *CondTy) const;
+
+    virtual unsigned getCastInstrCost(unsigned Opcode, Type *Dst,
+                                      Type *Src) const;
+  };
 }
 
 #endif    // X86ISELLOWERING_H
diff --git a/lib/Target/X86/X86InstrCompiler.td b/lib/Target/X86/X86InstrCompiler.td
index 99c2b8f..9e6f279 100644
--- a/lib/Target/X86/X86InstrCompiler.td
+++ b/lib/Target/X86/X86InstrCompiler.td
@@ -165,6 +165,33 @@ def EH_RETURN64   : I<0xC3, RawFrm, (outs), (ins GR64:$addr),
 
 }
 
+let hasSideEffects = 1, isBarrier = 1, isCodeGenOnly = 1,
+    usesCustomInserter = 1 in {
+  def EH_SjLj_SetJmp32  : I<0, Pseudo, (outs GR32:$dst), (ins i32mem:$buf),
+                            "#EH_SJLJ_SETJMP32",
+                            [(set GR32:$dst, (X86eh_sjlj_setjmp addr:$buf))]>,
+                          Requires<[In32BitMode]>;
+  def EH_SjLj_SetJmp64  : I<0, Pseudo, (outs GR32:$dst), (ins i64mem:$buf),
+                            "#EH_SJLJ_SETJMP64",
+                            [(set GR32:$dst, (X86eh_sjlj_setjmp addr:$buf))]>,
+                          Requires<[In64BitMode]>;
+  let isTerminator = 1 in {
+  def EH_SjLj_LongJmp32 : I<0, Pseudo, (outs), (ins i32mem:$buf),
+                            "#EH_SJLJ_LONGJMP32",
+                            [(X86eh_sjlj_longjmp addr:$buf)]>,
+                          Requires<[In32BitMode]>;
+  def EH_SjLj_LongJmp64 : I<0, Pseudo, (outs), (ins i64mem:$buf),
+                            "#EH_SJLJ_LONGJMP64",
+                            [(X86eh_sjlj_longjmp addr:$buf)]>,
+                          Requires<[In64BitMode]>;
+  }
+}
+
+let isBranch = 1, isTerminator = 1, isCodeGenOnly = 1 in {
+  def EH_SjLj_Setup : I<0, Pseudo, (outs), (ins brtarget:$dst),
+                        "#EH_SjLj_Setup\t$dst", []>;
+}
+
 //===----------------------------------------------------------------------===//
 // Pseudo instructions used by segmented stacks.
 //
@@ -230,25 +257,18 @@ def MOV64ri64i32 : Ii32<0xB8, AddRegFrm, (outs GR64:$dst), (ins i64i32imm:$src),
                         IIC_ALU_NONMEM>;
 
 // Use sbb to materialize carry bit.
-let Uses = [EFLAGS], Defs = [EFLAGS], isCodeGenOnly = 1 in {
+let Uses = [EFLAGS], Defs = [EFLAGS], isPseudo = 1 in {
 // FIXME: These are pseudo ops that should be replaced with Pat<> patterns.
 // However, Pat<> can't replicate the destination reg into the inputs of the
 // result.
-// FIXME: Change these to have encoding Pseudo when X86MCCodeEmitter replaces
-// X86CodeEmitter.
-def SETB_C8r : I<0x18, MRMInitReg, (outs GR8:$dst), (ins), "",
-                 [(set GR8:$dst, (X86setcc_c X86_COND_B, EFLAGS))],
-                 IIC_ALU_NONMEM>;
-def SETB_C16r : I<0x19, MRMInitReg, (outs GR16:$dst), (ins), "",
-                 [(set GR16:$dst, (X86setcc_c X86_COND_B, EFLAGS))],
-                 IIC_ALU_NONMEM>,
-                OpSize;
-def SETB_C32r : I<0x19, MRMInitReg, (outs GR32:$dst), (ins), "",
-                 [(set GR32:$dst, (X86setcc_c X86_COND_B, EFLAGS))],
-                 IIC_ALU_NONMEM>;
-def SETB_C64r : RI<0x19, MRMInitReg, (outs GR64:$dst), (ins), "",
-                 [(set GR64:$dst, (X86setcc_c X86_COND_B, EFLAGS))],
-                 IIC_ALU_NONMEM>;
+def SETB_C8r : I<0, Pseudo, (outs GR8:$dst), (ins), "",
+                 [(set GR8:$dst, (X86setcc_c X86_COND_B, EFLAGS))]>;
+def SETB_C16r : I<0, Pseudo, (outs GR16:$dst), (ins), "",
+                 [(set GR16:$dst, (X86setcc_c X86_COND_B, EFLAGS))]>;
+def SETB_C32r : I<0, Pseudo, (outs GR32:$dst), (ins), "",
+                 [(set GR32:$dst, (X86setcc_c X86_COND_B, EFLAGS))]>;
+def SETB_C64r : I<0, Pseudo, (outs GR64:$dst), (ins), "",
+                 [(set GR64:$dst, (X86setcc_c X86_COND_B, EFLAGS))]>;
 } // isCodeGenOnly
 
 
@@ -453,6 +473,11 @@ def CMOV_GR16 : I<0, Pseudo,
                     "#CMOV_GR16* PSEUDO!",
                     [(set GR16:$dst,
                       (X86cmov GR16:$src1, GR16:$src2, imm:$cond, EFLAGS))]>;
+} // Predicates = [NoCMov]
+
+// fcmov doesn't handle all possible EFLAGS, provide a fallback if there is no
+// SSE1.
+let Predicates = [FPStackf32] in
 def CMOV_RFP32 : I<0, Pseudo,
                     (outs RFP32:$dst),
                     (ins RFP32:$src1, RFP32:$src2, i8imm:$cond),
@@ -460,6 +485,9 @@ def CMOV_RFP32 : I<0, Pseudo,
                     [(set RFP32:$dst,
                       (X86cmov RFP32:$src1, RFP32:$src2, imm:$cond,
                                                   EFLAGS))]>;
+// fcmov doesn't handle all possible EFLAGS, provide a fallback if there is no
+// SSE2.
+let Predicates = [FPStackf64] in
 def CMOV_RFP64 : I<0, Pseudo,
                     (outs RFP64:$dst),
                     (ins RFP64:$src1, RFP64:$src2, i8imm:$cond),
@@ -474,7 +502,6 @@ def CMOV_RFP80 : I<0, Pseudo,
                     [(set RFP80:$dst,
                       (X86cmov RFP80:$src1, RFP80:$src2, imm:$cond,
                                                   EFLAGS))]>;
-} // Predicates = [NoCMov]
 } // UsesCustomInserter = 1, Uses = [EFLAGS]
 
 
@@ -482,130 +509,74 @@ def CMOV_RFP80 : I<0, Pseudo,
 // Atomic Instruction Pseudo Instructions
 //===----------------------------------------------------------------------===//
 
-// Atomic exchange, and, or, xor
-let Constraints = "$val = $dst", Defs = [EFLAGS],
-                  usesCustomInserter = 1 in {
-
-def ATOMAND8 : I<0, Pseudo, (outs GR8:$dst),(ins i8mem:$ptr, GR8:$val),
-               "#ATOMAND8 PSEUDO!",
-               [(set GR8:$dst, (atomic_load_and_8 addr:$ptr, GR8:$val))]>;
-def ATOMOR8 : I<0, Pseudo, (outs GR8:$dst),(ins i8mem:$ptr, GR8:$val),
-               "#ATOMOR8 PSEUDO!",
-               [(set GR8:$dst, (atomic_load_or_8 addr:$ptr, GR8:$val))]>;
-def ATOMXOR8 : I<0, Pseudo,(outs GR8:$dst),(ins i8mem:$ptr, GR8:$val),
-               "#ATOMXOR8 PSEUDO!",
-               [(set GR8:$dst, (atomic_load_xor_8 addr:$ptr, GR8:$val))]>;
-def ATOMNAND8 : I<0, Pseudo,(outs GR8:$dst),(ins i8mem:$ptr, GR8:$val),
-               "#ATOMNAND8 PSEUDO!",
-               [(set GR8:$dst, (atomic_load_nand_8 addr:$ptr, GR8:$val))]>;
-
-def ATOMAND16 : I<0, Pseudo, (outs GR16:$dst),(ins i16mem:$ptr, GR16:$val),
-               "#ATOMAND16 PSEUDO!",
-               [(set GR16:$dst, (atomic_load_and_16 addr:$ptr, GR16:$val))]>;
-def ATOMOR16 : I<0, Pseudo, (outs GR16:$dst),(ins i16mem:$ptr, GR16:$val),
-               "#ATOMOR16 PSEUDO!",
-               [(set GR16:$dst, (atomic_load_or_16 addr:$ptr, GR16:$val))]>;
-def ATOMXOR16 : I<0, Pseudo,(outs GR16:$dst),(ins i16mem:$ptr, GR16:$val),
-               "#ATOMXOR16 PSEUDO!",
-               [(set GR16:$dst, (atomic_load_xor_16 addr:$ptr, GR16:$val))]>;
-def ATOMNAND16 : I<0, Pseudo,(outs GR16:$dst),(ins i16mem:$ptr, GR16:$val),
-               "#ATOMNAND16 PSEUDO!",
-               [(set GR16:$dst, (atomic_load_nand_16 addr:$ptr, GR16:$val))]>;
-def ATOMMIN16: I<0, Pseudo, (outs GR16:$dst), (ins i16mem:$ptr, GR16:$val),
-               "#ATOMMIN16 PSEUDO!",
-               [(set GR16:$dst, (atomic_load_min_16 addr:$ptr, GR16:$val))]>;
-def ATOMMAX16: I<0, Pseudo, (outs GR16:$dst),(ins i16mem:$ptr, GR16:$val),
-               "#ATOMMAX16 PSEUDO!",
-               [(set GR16:$dst, (atomic_load_max_16 addr:$ptr, GR16:$val))]>;
-def ATOMUMIN16: I<0, Pseudo, (outs GR16:$dst),(ins i16mem:$ptr, GR16:$val),
-               "#ATOMUMIN16 PSEUDO!",
-               [(set GR16:$dst, (atomic_load_umin_16 addr:$ptr, GR16:$val))]>;
-def ATOMUMAX16: I<0, Pseudo, (outs GR16:$dst),(ins i16mem:$ptr, GR16:$val),
-               "#ATOMUMAX16 PSEUDO!",
-               [(set GR16:$dst, (atomic_load_umax_16 addr:$ptr, GR16:$val))]>;
-
-
-def ATOMAND32 : I<0, Pseudo, (outs GR32:$dst),(ins i32mem:$ptr, GR32:$val),
-               "#ATOMAND32 PSEUDO!",
-               [(set GR32:$dst, (atomic_load_and_32 addr:$ptr, GR32:$val))]>;
-def ATOMOR32 : I<0, Pseudo, (outs GR32:$dst),(ins i32mem:$ptr, GR32:$val),
-               "#ATOMOR32 PSEUDO!",
-               [(set GR32:$dst, (atomic_load_or_32 addr:$ptr, GR32:$val))]>;
-def ATOMXOR32 : I<0, Pseudo,(outs GR32:$dst),(ins i32mem:$ptr, GR32:$val),
-               "#ATOMXOR32 PSEUDO!",
-               [(set GR32:$dst, (atomic_load_xor_32 addr:$ptr, GR32:$val))]>;
-def ATOMNAND32 : I<0, Pseudo,(outs GR32:$dst),(ins i32mem:$ptr, GR32:$val),
-               "#ATOMNAND32 PSEUDO!",
-               [(set GR32:$dst, (atomic_load_nand_32 addr:$ptr, GR32:$val))]>;
-def ATOMMIN32: I<0, Pseudo, (outs GR32:$dst), (ins i32mem:$ptr, GR32:$val),
-               "#ATOMMIN32 PSEUDO!",
-               [(set GR32:$dst, (atomic_load_min_32 addr:$ptr, GR32:$val))]>;
-def ATOMMAX32: I<0, Pseudo, (outs GR32:$dst),(ins i32mem:$ptr, GR32:$val),
-               "#ATOMMAX32 PSEUDO!",
-               [(set GR32:$dst, (atomic_load_max_32 addr:$ptr, GR32:$val))]>;
-def ATOMUMIN32: I<0, Pseudo, (outs GR32:$dst),(ins i32mem:$ptr, GR32:$val),
-               "#ATOMUMIN32 PSEUDO!",
-               [(set GR32:$dst, (atomic_load_umin_32 addr:$ptr, GR32:$val))]>;
-def ATOMUMAX32: I<0, Pseudo, (outs GR32:$dst),(ins i32mem:$ptr, GR32:$val),
-               "#ATOMUMAX32 PSEUDO!",
-               [(set GR32:$dst, (atomic_load_umax_32 addr:$ptr, GR32:$val))]>;
-
-
-
-def ATOMAND64 : I<0, Pseudo, (outs GR64:$dst),(ins i64mem:$ptr, GR64:$val),
-               "#ATOMAND64 PSEUDO!",
-               [(set GR64:$dst, (atomic_load_and_64 addr:$ptr, GR64:$val))]>;
-def ATOMOR64 : I<0, Pseudo, (outs GR64:$dst),(ins i64mem:$ptr, GR64:$val),
-               "#ATOMOR64 PSEUDO!",
-               [(set GR64:$dst, (atomic_load_or_64 addr:$ptr, GR64:$val))]>;
-def ATOMXOR64 : I<0, Pseudo,(outs GR64:$dst),(ins i64mem:$ptr, GR64:$val),
-               "#ATOMXOR64 PSEUDO!",
-               [(set GR64:$dst, (atomic_load_xor_64 addr:$ptr, GR64:$val))]>;
-def ATOMNAND64 : I<0, Pseudo,(outs GR64:$dst),(ins i64mem:$ptr, GR64:$val),
-               "#ATOMNAND64 PSEUDO!",
-               [(set GR64:$dst, (atomic_load_nand_64 addr:$ptr, GR64:$val))]>;
-def ATOMMIN64: I<0, Pseudo, (outs GR64:$dst), (ins i64mem:$ptr, GR64:$val),
-               "#ATOMMIN64 PSEUDO!",
-               [(set GR64:$dst, (atomic_load_min_64 addr:$ptr, GR64:$val))]>;
-def ATOMMAX64: I<0, Pseudo, (outs GR64:$dst),(ins i64mem:$ptr, GR64:$val),
-               "#ATOMMAX64 PSEUDO!",
-               [(set GR64:$dst, (atomic_load_max_64 addr:$ptr, GR64:$val))]>;
-def ATOMUMIN64: I<0, Pseudo, (outs GR64:$dst),(ins i64mem:$ptr, GR64:$val),
-               "#ATOMUMIN64 PSEUDO!",
-               [(set GR64:$dst, (atomic_load_umin_64 addr:$ptr, GR64:$val))]>;
-def ATOMUMAX64: I<0, Pseudo, (outs GR64:$dst),(ins i64mem:$ptr, GR64:$val),
-               "#ATOMUMAX64 PSEUDO!",
-               [(set GR64:$dst, (atomic_load_umax_64 addr:$ptr, GR64:$val))]>;
+// Pseudo atomic instructions
+
+multiclass PSEUDO_ATOMIC_LOAD_BINOP<string mnemonic> {
+  let usesCustomInserter = 1, mayLoad = 1, mayStore = 1 in {
+    def #NAME#8  : I<0, Pseudo, (outs GR8:$dst),
+                     (ins i8mem:$ptr, GR8:$val),
+                     !strconcat(mnemonic, "8 PSEUDO!"), []>;
+    def #NAME#16 : I<0, Pseudo,(outs GR16:$dst),
+                     (ins i16mem:$ptr, GR16:$val),
+                     !strconcat(mnemonic, "16 PSEUDO!"), []>;
+    def #NAME#32 : I<0, Pseudo, (outs GR32:$dst),
+                     (ins i32mem:$ptr, GR32:$val),
+                     !strconcat(mnemonic, "32 PSEUDO!"), []>;
+    def #NAME#64 : I<0, Pseudo, (outs GR64:$dst),
+                     (ins i64mem:$ptr, GR64:$val),
+                     !strconcat(mnemonic, "64 PSEUDO!"), []>;
+  }
+}
+
+multiclass PSEUDO_ATOMIC_LOAD_BINOP_PATS<string name, string frag> {
+  def : Pat<(!cast<PatFrag>(frag # "_8") addr:$ptr, GR8:$val),
+            (!cast<Instruction>(name # "8") addr:$ptr, GR8:$val)>;
+  def : Pat<(!cast<PatFrag>(frag # "_16") addr:$ptr, GR16:$val),
+            (!cast<Instruction>(name # "16") addr:$ptr, GR16:$val)>;
+  def : Pat<(!cast<PatFrag>(frag # "_32") addr:$ptr, GR32:$val),
+            (!cast<Instruction>(name # "32") addr:$ptr, GR32:$val)>;
+  def : Pat<(!cast<PatFrag>(frag # "_64") addr:$ptr, GR64:$val),
+            (!cast<Instruction>(name # "64") addr:$ptr, GR64:$val)>;
 }
 
-let Constraints = "$val1 = $dst1, $val2 = $dst2",
-                  Defs = [EFLAGS, EAX, EBX, ECX, EDX],
-                  Uses = [EAX, EBX, ECX, EDX],
-                  mayLoad = 1, mayStore = 1,
-                  usesCustomInserter = 1 in {
-def ATOMAND6432 : I<0, Pseudo, (outs GR32:$dst1, GR32:$dst2),
-                               (ins i64mem:$ptr, GR32:$val1, GR32:$val2),
-               "#ATOMAND6432 PSEUDO!", []>;
-def ATOMOR6432 : I<0, Pseudo, (outs GR32:$dst1, GR32:$dst2),
-                               (ins i64mem:$ptr, GR32:$val1, GR32:$val2),
-               "#ATOMOR6432 PSEUDO!", []>;
-def ATOMXOR6432 : I<0, Pseudo, (outs GR32:$dst1, GR32:$dst2),
-                               (ins i64mem:$ptr, GR32:$val1, GR32:$val2),
-               "#ATOMXOR6432 PSEUDO!", []>;
-def ATOMNAND6432 : I<0, Pseudo, (outs GR32:$dst1, GR32:$dst2),
-                               (ins i64mem:$ptr, GR32:$val1, GR32:$val2),
-               "#ATOMNAND6432 PSEUDO!", []>;
-def ATOMADD6432 : I<0, Pseudo, (outs GR32:$dst1, GR32:$dst2),
-                               (ins i64mem:$ptr, GR32:$val1, GR32:$val2),
-               "#ATOMADD6432 PSEUDO!", []>;
-def ATOMSUB6432 : I<0, Pseudo, (outs GR32:$dst1, GR32:$dst2),
-                               (ins i64mem:$ptr, GR32:$val1, GR32:$val2),
-               "#ATOMSUB6432 PSEUDO!", []>;
-def ATOMSWAP6432 : I<0, Pseudo, (outs GR32:$dst1, GR32:$dst2),
-                               (ins i64mem:$ptr, GR32:$val1, GR32:$val2),
-               "#ATOMSWAP6432 PSEUDO!", []>;
+// Atomic exchange, and, or, xor
+defm ATOMAND  : PSEUDO_ATOMIC_LOAD_BINOP<"#ATOMAND">;
+defm ATOMOR   : PSEUDO_ATOMIC_LOAD_BINOP<"#ATOMOR">;
+defm ATOMXOR  : PSEUDO_ATOMIC_LOAD_BINOP<"#ATOMXOR">;
+defm ATOMNAND : PSEUDO_ATOMIC_LOAD_BINOP<"#ATOMNAND">;
+defm ATOMMAX  : PSEUDO_ATOMIC_LOAD_BINOP<"#ATOMMAX">;
+defm ATOMMIN  : PSEUDO_ATOMIC_LOAD_BINOP<"#ATOMMIN">;
+defm ATOMUMAX : PSEUDO_ATOMIC_LOAD_BINOP<"#ATOMUMAX">;
+defm ATOMUMIN : PSEUDO_ATOMIC_LOAD_BINOP<"#ATOMUMIN">;
+
+defm : PSEUDO_ATOMIC_LOAD_BINOP_PATS<"ATOMAND",  "atomic_load_and">;
+defm : PSEUDO_ATOMIC_LOAD_BINOP_PATS<"ATOMOR",   "atomic_load_or">;
+defm : PSEUDO_ATOMIC_LOAD_BINOP_PATS<"ATOMXOR",  "atomic_load_xor">;
+defm : PSEUDO_ATOMIC_LOAD_BINOP_PATS<"ATOMNAND", "atomic_load_nand">;
+defm : PSEUDO_ATOMIC_LOAD_BINOP_PATS<"ATOMMAX",  "atomic_load_max">;
+defm : PSEUDO_ATOMIC_LOAD_BINOP_PATS<"ATOMMIN",  "atomic_load_min">;
+defm : PSEUDO_ATOMIC_LOAD_BINOP_PATS<"ATOMUMAX", "atomic_load_umax">;
+defm : PSEUDO_ATOMIC_LOAD_BINOP_PATS<"ATOMUMIN", "atomic_load_umin">;
+
+multiclass PSEUDO_ATOMIC_LOAD_BINOP6432<string mnemonic> {
+  let usesCustomInserter = 1, mayLoad = 1, mayStore = 1 in
+    def #NAME#6432 : I<0, Pseudo, (outs GR32:$dst1, GR32:$dst2),
+                       (ins i64mem:$ptr, GR32:$val1, GR32:$val2),
+                       !strconcat(mnemonic, "6432 PSEUDO!"), []>;
 }
 
+defm ATOMAND  : PSEUDO_ATOMIC_LOAD_BINOP6432<"#ATOMAND">;
+defm ATOMOR   : PSEUDO_ATOMIC_LOAD_BINOP6432<"#ATOMOR">;
+defm ATOMXOR  : PSEUDO_ATOMIC_LOAD_BINOP6432<"#ATOMXOR">;
+defm ATOMNAND : PSEUDO_ATOMIC_LOAD_BINOP6432<"#ATOMNAND">;
+defm ATOMADD  : PSEUDO_ATOMIC_LOAD_BINOP6432<"#ATOMADD">;
+defm ATOMSUB  : PSEUDO_ATOMIC_LOAD_BINOP6432<"#ATOMSUB">;
+defm ATOMMAX  : PSEUDO_ATOMIC_LOAD_BINOP6432<"#ATOMMAX">;
+defm ATOMMIN  : PSEUDO_ATOMIC_LOAD_BINOP6432<"#ATOMMIN">;
+defm ATOMUMAX : PSEUDO_ATOMIC_LOAD_BINOP6432<"#ATOMUMAX">;
+defm ATOMUMIN : PSEUDO_ATOMIC_LOAD_BINOP6432<"#ATOMUMIN">;
+defm ATOMSWAP : PSEUDO_ATOMIC_LOAD_BINOP6432<"#ATOMSWAP">;
+
 //===----------------------------------------------------------------------===//
 // Normal-Instructions-With-Lock-Prefix Pseudo Instructions
 //===----------------------------------------------------------------------===//
@@ -617,7 +588,6 @@ def ATOMSWAP6432 : I<0, Pseudo, (outs GR32:$dst1, GR32:$dst2),
 // TODO: Get this to fold the constant into the instruction.
 let isCodeGenOnly = 1, Defs = [EFLAGS] in
 def OR32mrLocked  : I<0x09, MRMDestMem, (outs), (ins i32mem:$dst, GR32:$zero),
-                      "lock\n\t"
                       "or{l}\t{$zero, $dst|$dst, $zero}",
                       [], IIC_ALU_MEM>, Requires<[In32BitMode]>, LOCK;
 
@@ -637,72 +607,72 @@ let Defs = [EFLAGS], mayLoad = 1, mayStore = 1, isCodeGenOnly = 1 in {
 def #NAME#8mr : I<{RegOpc{7}, RegOpc{6}, RegOpc{5}, RegOpc{4},
                    RegOpc{3}, RegOpc{2}, RegOpc{1}, 0 },
                    MRMDestMem, (outs), (ins i8mem:$dst, GR8:$src2),
-                   !strconcat("lock\n\t", mnemonic, "{b}\t",
+                   !strconcat(mnemonic, "{b}\t",
                               "{$src2, $dst|$dst, $src2}"),
                    [], IIC_ALU_NONMEM>, LOCK;
 def #NAME#16mr : I<{RegOpc{7}, RegOpc{6}, RegOpc{5}, RegOpc{4},
                     RegOpc{3}, RegOpc{2}, RegOpc{1}, 1 },
                     MRMDestMem, (outs), (ins i16mem:$dst, GR16:$src2),
-                    !strconcat("lock\n\t", mnemonic, "{w}\t",
+                    !strconcat(mnemonic, "{w}\t",
                                "{$src2, $dst|$dst, $src2}"),
                     [], IIC_ALU_NONMEM>, OpSize, LOCK;
 def #NAME#32mr : I<{RegOpc{7}, RegOpc{6}, RegOpc{5}, RegOpc{4},
                     RegOpc{3}, RegOpc{2}, RegOpc{1}, 1 },
                     MRMDestMem, (outs), (ins i32mem:$dst, GR32:$src2),
-                    !strconcat("lock\n\t", mnemonic, "{l}\t",
+                    !strconcat(mnemonic, "{l}\t",
                                "{$src2, $dst|$dst, $src2}"),
                     [], IIC_ALU_NONMEM>, LOCK;
 def #NAME#64mr : RI<{RegOpc{7}, RegOpc{6}, RegOpc{5}, RegOpc{4},
                      RegOpc{3}, RegOpc{2}, RegOpc{1}, 1 },
                      MRMDestMem, (outs), (ins i64mem:$dst, GR64:$src2),
-                     !strconcat("lock\n\t", mnemonic, "{q}\t",
+                     !strconcat(mnemonic, "{q}\t",
                                 "{$src2, $dst|$dst, $src2}"),
                      [], IIC_ALU_NONMEM>, LOCK;
 
 def #NAME#8mi : Ii8<{ImmOpc{7}, ImmOpc{6}, ImmOpc{5}, ImmOpc{4},
                      ImmOpc{3}, ImmOpc{2}, ImmOpc{1}, 0 },
                      ImmMod, (outs), (ins i8mem :$dst, i8imm :$src2),
-                     !strconcat("lock\n\t", mnemonic, "{b}\t",
+                     !strconcat(mnemonic, "{b}\t",
                                 "{$src2, $dst|$dst, $src2}"),
                      [], IIC_ALU_MEM>, LOCK;
 
 def #NAME#16mi : Ii16<{ImmOpc{7}, ImmOpc{6}, ImmOpc{5}, ImmOpc{4},
                        ImmOpc{3}, ImmOpc{2}, ImmOpc{1}, 1 },
                        ImmMod, (outs), (ins i16mem :$dst, i16imm :$src2),
-                       !strconcat("lock\n\t", mnemonic, "{w}\t",
+                       !strconcat(mnemonic, "{w}\t",
                                   "{$src2, $dst|$dst, $src2}"),
-                       [], IIC_ALU_MEM>, LOCK;
+                       [], IIC_ALU_MEM>, OpSize, LOCK;
 
 def #NAME#32mi : Ii32<{ImmOpc{7}, ImmOpc{6}, ImmOpc{5}, ImmOpc{4},
                        ImmOpc{3}, ImmOpc{2}, ImmOpc{1}, 1 },
                        ImmMod, (outs), (ins i32mem :$dst, i32imm :$src2),
-                       !strconcat("lock\n\t", mnemonic, "{l}\t",
+                       !strconcat(mnemonic, "{l}\t",
                                   "{$src2, $dst|$dst, $src2}"),
                        [], IIC_ALU_MEM>, LOCK;
 
 def #NAME#64mi32 : RIi32<{ImmOpc{7}, ImmOpc{6}, ImmOpc{5}, ImmOpc{4},
                           ImmOpc{3}, ImmOpc{2}, ImmOpc{1}, 1 },
                           ImmMod, (outs), (ins i64mem :$dst, i64i32imm :$src2),
-                          !strconcat("lock\n\t", mnemonic, "{q}\t",
+                          !strconcat(mnemonic, "{q}\t",
                                      "{$src2, $dst|$dst, $src2}"),
                           [], IIC_ALU_MEM>, LOCK;
 
 def #NAME#16mi8 : Ii8<{ImmOpc8{7}, ImmOpc8{6}, ImmOpc8{5}, ImmOpc8{4},
                        ImmOpc8{3}, ImmOpc8{2}, ImmOpc8{1}, 1 },
                        ImmMod, (outs), (ins i16mem :$dst, i16i8imm :$src2),
-                       !strconcat("lock\n\t", mnemonic, "{w}\t",
+                       !strconcat(mnemonic, "{w}\t",
                                   "{$src2, $dst|$dst, $src2}"),
-                       [], IIC_ALU_MEM>, LOCK;
+                       [], IIC_ALU_MEM>, OpSize, LOCK;
 def #NAME#32mi8 : Ii8<{ImmOpc8{7}, ImmOpc8{6}, ImmOpc8{5}, ImmOpc8{4},
                        ImmOpc8{3}, ImmOpc8{2}, ImmOpc8{1}, 1 },
                        ImmMod, (outs), (ins i32mem :$dst, i32i8imm :$src2),
-                       !strconcat("lock\n\t", mnemonic, "{l}\t",
+                       !strconcat(mnemonic, "{l}\t",
                                   "{$src2, $dst|$dst, $src2}"),
                        [], IIC_ALU_MEM>, LOCK;
 def #NAME#64mi8 : RIi8<{ImmOpc8{7}, ImmOpc8{6}, ImmOpc8{5}, ImmOpc8{4},
                         ImmOpc8{3}, ImmOpc8{2}, ImmOpc8{1}, 1 },
                         ImmMod, (outs), (ins i64mem :$dst, i64i8imm :$src2),
-                        !strconcat("lock\n\t", mnemonic, "{q}\t",
+                        !strconcat(mnemonic, "{q}\t",
                                    "{$src2, $dst|$dst, $src2}"),
                         [], IIC_ALU_MEM>, LOCK;
 
@@ -717,107 +687,117 @@ defm LOCK_AND : LOCK_ArithBinOp<0x20, 0x80, 0x83, MRM4m, "and">;
 defm LOCK_XOR : LOCK_ArithBinOp<0x30, 0x80, 0x83, MRM6m, "xor">;
 
 // Optimized codegen when the non-memory output is not used.
+multiclass LOCK_ArithUnOp<bits<8> Opc8, bits<8> Opc, Format Form,
+                          string mnemonic> {
 let Defs = [EFLAGS], mayLoad = 1, mayStore = 1, isCodeGenOnly = 1 in {
 
-def LOCK_INC8m  : I<0xFE, MRM0m, (outs), (ins i8mem :$dst),
-                    "lock\n\t"
-                    "inc{b}\t$dst", [], IIC_UNARY_MEM>, LOCK;
-def LOCK_INC16m : I<0xFF, MRM0m, (outs), (ins i16mem:$dst),
-                    "lock\n\t"
-                    "inc{w}\t$dst", [], IIC_UNARY_MEM>, OpSize, LOCK;
-def LOCK_INC32m : I<0xFF, MRM0m, (outs), (ins i32mem:$dst),
-                    "lock\n\t"
-                    "inc{l}\t$dst", [], IIC_UNARY_MEM>, LOCK;
-def LOCK_INC64m : RI<0xFF, MRM0m, (outs), (ins i64mem:$dst),
-                     "lock\n\t"
-                     "inc{q}\t$dst", [], IIC_UNARY_MEM>, LOCK;
-
-def LOCK_DEC8m  : I<0xFE, MRM1m, (outs), (ins i8mem :$dst),
-                    "lock\n\t"
-                    "dec{b}\t$dst", [], IIC_UNARY_MEM>, LOCK;
-def LOCK_DEC16m : I<0xFF, MRM1m, (outs), (ins i16mem:$dst),
-                    "lock\n\t"
-                    "dec{w}\t$dst", [], IIC_UNARY_MEM>, OpSize, LOCK;
-def LOCK_DEC32m : I<0xFF, MRM1m, (outs), (ins i32mem:$dst),
-                    "lock\n\t"
-                    "dec{l}\t$dst", [], IIC_UNARY_MEM>, LOCK;
-def LOCK_DEC64m : RI<0xFF, MRM1m, (outs), (ins i64mem:$dst),
-                      "lock\n\t"
-                      "dec{q}\t$dst", [], IIC_UNARY_MEM>, LOCK;
+def #NAME#8m  : I<Opc8, Form, (outs), (ins i8mem :$dst),
+                  !strconcat(mnemonic, "{b}\t$dst"),
+                  [], IIC_UNARY_MEM>, LOCK;
+def #NAME#16m : I<Opc, Form, (outs), (ins i16mem:$dst),
+                  !strconcat(mnemonic, "{w}\t$dst"),
+                  [], IIC_UNARY_MEM>, OpSize, LOCK;
+def #NAME#32m : I<Opc, Form, (outs), (ins i32mem:$dst),
+                  !strconcat(mnemonic, "{l}\t$dst"),
+                  [], IIC_UNARY_MEM>, LOCK;
+def #NAME#64m : RI<Opc, Form, (outs), (ins i64mem:$dst),
+                   !strconcat(mnemonic, "{q}\t$dst"),
+                   [], IIC_UNARY_MEM>, LOCK;
+}
 }
 
-// Atomic compare and swap.
-let Defs = [EAX, EDX, EFLAGS], Uses = [EAX, EBX, ECX, EDX],
-    isCodeGenOnly = 1 in
-def LCMPXCHG8B : I<0xC7, MRM1m, (outs), (ins i64mem:$ptr),
-               "lock\n\t"
-               "cmpxchg8b\t$ptr",
-               [(X86cas8 addr:$ptr)], IIC_CMPX_LOCK_8B>, TB, LOCK;
+defm LOCK_INC    : LOCK_ArithUnOp<0xFE, 0xFF, MRM0m, "inc">;
+defm LOCK_DEC    : LOCK_ArithUnOp<0xFE, 0xFF, MRM1m, "dec">;
 
-let Defs = [RAX, RDX, EFLAGS], Uses = [RAX, RBX, RCX, RDX],
-    isCodeGenOnly = 1 in
-def LCMPXCHG16B : RI<0xC7, MRM1m, (outs), (ins i128mem:$ptr),
-                    "lock\n\t"
-                    "cmpxchg16b\t$ptr",
-                    [(X86cas16 addr:$ptr)], IIC_CMPX_LOCK_16B>, TB, LOCK,
-                    Requires<[HasCmpxchg16b]>;
-
-let Defs = [AL, EFLAGS], Uses = [AL], isCodeGenOnly = 1 in {
-def LCMPXCHG8 : I<0xB0, MRMDestMem, (outs), (ins i8mem:$ptr, GR8:$swap),
-               "lock\n\t"
-               "cmpxchg{b}\t{$swap, $ptr|$ptr, $swap}",
-               [(X86cas addr:$ptr, GR8:$swap, 1)], IIC_CMPX_LOCK_8>, TB, LOCK;
+// Atomic compare and swap.
+multiclass LCMPXCHG_UnOp<bits<8> Opc, Format Form, string mnemonic,
+                         SDPatternOperator frag, X86MemOperand x86memop,
+                         InstrItinClass itin> {
+let isCodeGenOnly = 1 in {
+  def #NAME# : I<Opc, Form, (outs), (ins x86memop:$ptr),
+                 !strconcat(mnemonic, "\t$ptr"),
+                 [(frag addr:$ptr)], itin>, TB, LOCK;
+}
 }
 
-let Defs = [AX, EFLAGS], Uses = [AX], isCodeGenOnly = 1 in {
-def LCMPXCHG16 : I<0xB1, MRMDestMem, (outs), (ins i16mem:$ptr, GR16:$swap),
-               "lock\n\t"
-               "cmpxchg{w}\t{$swap, $ptr|$ptr, $swap}",
-               [(X86cas addr:$ptr, GR16:$swap, 2)], IIC_CMPX_LOCK>, TB, OpSize, LOCK;
+multiclass LCMPXCHG_BinOp<bits<8> Opc8, bits<8> Opc, Format Form,
+                          string mnemonic, SDPatternOperator frag,
+                          InstrItinClass itin8, InstrItinClass itin> {
+let isCodeGenOnly = 1 in {
+  let Defs = [AL, EFLAGS], Uses = [AL] in
+  def #NAME#8  : I<Opc8, Form, (outs), (ins i8mem:$ptr, GR8:$swap),
+                   !strconcat(mnemonic, "{b}\t{$swap, $ptr|$ptr, $swap}"),
+                   [(frag addr:$ptr, GR8:$swap, 1)], itin8>, TB, LOCK;
+  let Defs = [AX, EFLAGS], Uses = [AX] in
+  def #NAME#16 : I<Opc, Form, (outs), (ins i16mem:$ptr, GR16:$swap),
+                   !strconcat(mnemonic, "{w}\t{$swap, $ptr|$ptr, $swap}"),
+                   [(frag addr:$ptr, GR16:$swap, 2)], itin>, TB, OpSize, LOCK;
+  let Defs = [EAX, EFLAGS], Uses = [EAX] in
+  def #NAME#32 : I<Opc, Form, (outs), (ins i32mem:$ptr, GR32:$swap),
+                   !strconcat(mnemonic, "{l}\t{$swap, $ptr|$ptr, $swap}"),
+                   [(frag addr:$ptr, GR32:$swap, 4)], itin>, TB, LOCK;
+  let Defs = [RAX, EFLAGS], Uses = [RAX] in
+  def #NAME#64 : RI<Opc, Form, (outs), (ins i64mem:$ptr, GR64:$swap),
+                    !strconcat(mnemonic, "{q}\t{$swap, $ptr|$ptr, $swap}"),
+                    [(frag addr:$ptr, GR64:$swap, 8)], itin>, TB, LOCK;
+}
 }
 
-let Defs = [EAX, EFLAGS], Uses = [EAX], isCodeGenOnly = 1 in {
-def LCMPXCHG32 : I<0xB1, MRMDestMem, (outs), (ins i32mem:$ptr, GR32:$swap),
-               "lock\n\t"
-               "cmpxchg{l}\t{$swap, $ptr|$ptr, $swap}",
-               [(X86cas addr:$ptr, GR32:$swap, 4)], IIC_CMPX_LOCK>, TB, LOCK;
+let Defs = [EAX, EDX, EFLAGS], Uses = [EAX, EBX, ECX, EDX] in {
+defm LCMPXCHG8B : LCMPXCHG_UnOp<0xC7, MRM1m, "cmpxchg8b",
+                                X86cas8, i64mem,
+                                IIC_CMPX_LOCK_8B>;
 }
 
-let Defs = [RAX, EFLAGS], Uses = [RAX], isCodeGenOnly = 1 in {
-def LCMPXCHG64 : RI<0xB1, MRMDestMem, (outs), (ins i64mem:$ptr, GR64:$swap),
-               "lock\n\t"
-               "cmpxchg{q}\t{$swap, $ptr|$ptr, $swap}",
-               [(X86cas addr:$ptr, GR64:$swap, 8)], IIC_CMPX_LOCK>, TB, LOCK;
+let Defs = [RAX, RDX, EFLAGS], Uses = [RAX, RBX, RCX, RDX],
+    Predicates = [HasCmpxchg16b] in {
+defm LCMPXCHG16B : LCMPXCHG_UnOp<0xC7, MRM1m, "cmpxchg16b",
+                                 X86cas16, i128mem,
+                                 IIC_CMPX_LOCK_16B>, REX_W;
 }
 
+defm LCMPXCHG : LCMPXCHG_BinOp<0xB0, 0xB1, MRMDestMem, "cmpxchg",
+                               X86cas, IIC_CMPX_LOCK_8, IIC_CMPX_LOCK>;
+
 // Atomic exchange and add
-let Constraints = "$val = $dst", Defs = [EFLAGS], isCodeGenOnly = 1 in {
-def LXADD8  : I<0xC0, MRMSrcMem, (outs GR8:$dst), (ins GR8:$val, i8mem:$ptr),
-               "lock\n\t"
-               "xadd{b}\t{$val, $ptr|$ptr, $val}",
-               [(set GR8:$dst, (atomic_load_add_8 addr:$ptr, GR8:$val))],
-                IIC_XADD_LOCK_MEM8>,
-                TB, LOCK;
-def LXADD16 : I<0xC1, MRMSrcMem, (outs GR16:$dst), (ins GR16:$val, i16mem:$ptr),
-               "lock\n\t"
-               "xadd{w}\t{$val, $ptr|$ptr, $val}",
-               [(set GR16:$dst, (atomic_load_add_16 addr:$ptr, GR16:$val))],
-                IIC_XADD_LOCK_MEM>,
-                TB, OpSize, LOCK;
-def LXADD32 : I<0xC1, MRMSrcMem, (outs GR32:$dst), (ins GR32:$val, i32mem:$ptr),
-               "lock\n\t"
-               "xadd{l}\t{$val, $ptr|$ptr, $val}",
-               [(set GR32:$dst, (atomic_load_add_32 addr:$ptr, GR32:$val))],
-                IIC_XADD_LOCK_MEM>,
-                TB, LOCK;
-def LXADD64 : RI<0xC1, MRMSrcMem, (outs GR64:$dst), (ins GR64:$val,i64mem:$ptr),
-               "lock\n\t"
-               "xadd{q}\t{$val, $ptr|$ptr, $val}",
-               [(set GR64:$dst, (atomic_load_add_64 addr:$ptr, GR64:$val))],
-                IIC_XADD_LOCK_MEM>,
-                TB, LOCK;
+multiclass ATOMIC_LOAD_BINOP<bits<8> opc8, bits<8> opc, string mnemonic,
+                             string frag,
+                             InstrItinClass itin8, InstrItinClass itin> {
+  let Constraints = "$val = $dst", Defs = [EFLAGS], isCodeGenOnly = 1 in {
+    def #NAME#8  : I<opc8, MRMSrcMem, (outs GR8:$dst),
+                     (ins GR8:$val, i8mem:$ptr),
+                     !strconcat(mnemonic, "{b}\t{$val, $ptr|$ptr, $val}"),
+                     [(set GR8:$dst,
+                           (!cast<PatFrag>(frag # "_8") addr:$ptr, GR8:$val))],
+                     itin8>;
+    def #NAME#16 : I<opc, MRMSrcMem, (outs GR16:$dst),
+                     (ins GR16:$val, i16mem:$ptr),
+                     !strconcat(mnemonic, "{w}\t{$val, $ptr|$ptr, $val}"),
+                     [(set
+                        GR16:$dst,
+                        (!cast<PatFrag>(frag # "_16") addr:$ptr, GR16:$val))],
+                     itin>, OpSize;
+    def #NAME#32 : I<opc, MRMSrcMem, (outs GR32:$dst),
+                     (ins GR32:$val, i32mem:$ptr),
+                     !strconcat(mnemonic, "{l}\t{$val, $ptr|$ptr, $val}"),
+                     [(set
+                        GR32:$dst,
+                        (!cast<PatFrag>(frag # "_32") addr:$ptr, GR32:$val))],
+                     itin>;
+    def #NAME#64 : RI<opc, MRMSrcMem, (outs GR64:$dst),
+                      (ins GR64:$val, i64mem:$ptr),
+                      !strconcat(mnemonic, "{q}\t{$val, $ptr|$ptr, $val}"),
+                      [(set
+                         GR64:$dst,
+                         (!cast<PatFrag>(frag # "_64") addr:$ptr, GR64:$val))],
+                      itin>;
+  }
 }
 
+defm LXADD : ATOMIC_LOAD_BINOP<0xc0, 0xc1, "xadd", "atomic_load_add",
+                               IIC_XADD_LOCK_MEM8, IIC_XADD_LOCK_MEM>,
+             TB, LOCK;
+
 def ACQUIRE_MOV8rm  : I<0, Pseudo, (outs GR8 :$dst), (ins i8mem :$src),
                       "#ACQUIRE_MOV PSEUDO!",
                       [(set GR8:$dst,  (atomic_load_8  addr:$src))]>;
@@ -1017,7 +997,24 @@ def : Pat<(X86call (i64 tglobaladdr:$dst)),
 def : Pat<(X86call (i64 texternalsym:$dst)),
           (CALL64pcrel32 texternalsym:$dst)>;
 
-// tailcall stuff
+// Tailcall stuff. The TCRETURN instructions execute after the epilog, so they
+// can never use callee-saved registers. That is the purpose of the GR64_TC
+// register classes.
+//
+// The only volatile register that is never used by the calling convention is
+// %r11. This happens when calling a vararg function with 6 arguments.
+//
+// Match an X86tcret that uses less than 7 volatile registers.
+def X86tcret_6regs : PatFrag<(ops node:$ptr, node:$off),
+                             (X86tcret node:$ptr, node:$off), [{
+  // X86tcret args: (*chain, ptr, imm, regs..., glue)
+  unsigned NumRegs = 0;
+  for (unsigned i = 3, e = N->getNumOperands(); i != e; ++i)
+    if (isa<RegisterSDNode>(N->getOperand(i)) && ++NumRegs > 6)
+      return false;
+  return true;
+}]>;
+
 def : Pat<(X86tcret ptr_rc_tailcall:$dst, imm:$off),
           (TCRETURNri ptr_rc_tailcall:$dst, imm:$off)>,
           Requires<[In32BitMode]>;
@@ -1041,7 +1038,9 @@ def : Pat<(X86tcret ptr_rc_tailcall:$dst, imm:$off),
           (TCRETURNri64 ptr_rc_tailcall:$dst, imm:$off)>,
           Requires<[In64BitMode]>;
 
-def : Pat<(X86tcret (load addr:$dst), imm:$off),
+// Don't fold loads into X86tcret requiring more than 6 regs.
+// There wouldn't be enough scratch registers for base+index.
+def : Pat<(X86tcret_6regs (load addr:$dst), imm:$off),
           (TCRETURNmi64 addr:$dst, imm:$off)>,
           Requires<[In64BitMode]>;
 
diff --git a/lib/Target/X86/X86InstrControl.td b/lib/Target/X86/X86InstrControl.td
index b0c27c8..bfe9541 100644
--- a/lib/Target/X86/X86InstrControl.td
+++ b/lib/Target/X86/X86InstrControl.td
@@ -16,15 +16,18 @@
 //
 
 // Return instructions.
+//
+// The X86retflag return instructions are variadic because we may add ST0 and
+// ST1 arguments when returning values on the x87 stack.
 let isTerminator = 1, isReturn = 1, isBarrier = 1,
     hasCtrlDep = 1, FPForm = SpecialFP in {
-  def RET    : I   <0xC3, RawFrm, (outs), (ins),
+  def RET    : I   <0xC3, RawFrm, (outs), (ins variable_ops),
                     "ret",
                     [(X86retflag 0)], IIC_RET>;
   def RETW   : I   <0xC3, RawFrm, (outs), (ins),
                     "ret{w}",
                     [], IIC_RET>, OpSize;
-  def RETI   : Ii16<0xC2, RawFrm, (outs), (ins i16imm:$amt),
+  def RETI   : Ii16<0xC2, RawFrm, (outs), (ins i16imm:$amt, variable_ops),
                     "ret\t$amt",
                     [(X86retflag timm:$amt)], IIC_RET_IMM>;
   def RETIW  : Ii16<0xC2, RawFrm, (outs), (ins i16imm:$amt),
diff --git a/lib/Target/X86/X86InstrFMA.td b/lib/Target/X86/X86InstrFMA.td
index 265b4bb..959d91a 100644
--- a/lib/Target/X86/X86InstrFMA.td
+++ b/lib/Target/X86/X86InstrFMA.td
@@ -16,243 +16,180 @@
 //===----------------------------------------------------------------------===//
 
 let Constraints = "$src1 = $dst" in {
-multiclass fma3p_rm<bits<8> opc, string OpcodeStr> {
-let neverHasSideEffects = 1 in {
-  def r : FMA3<opc, MRMSrcReg, (outs VR128:$dst),
-               (ins VR128:$src1, VR128:$src2, VR128:$src3),
-               !strconcat(OpcodeStr, 
-                          "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), []>;
-  let mayLoad = 1 in
-  def m : FMA3<opc, MRMSrcMem, (outs VR128:$dst),
-               (ins VR128:$src1, VR128:$src2, f128mem:$src3),
-               !strconcat(OpcodeStr, 
-                          "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), []>;
-  def rY : FMA3<opc, MRMSrcReg, (outs VR256:$dst),
-                (ins VR256:$src1, VR256:$src2, VR256:$src3),
-                !strconcat(OpcodeStr, 
-                           "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), []>;
-  let mayLoad = 1 in
-  def mY : FMA3<opc, MRMSrcMem, (outs VR256:$dst),
-                (ins VR256:$src1, VR256:$src2, f256mem:$src3),
-                !strconcat(OpcodeStr, 
-                           "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), []>;
-} // neverHasSideEffects = 1
-}
-
-// Intrinsic for 213 pattern
-multiclass fma3p_rm_int<bits<8> opc, string OpcodeStr,
-                        PatFrag MemFrag128, PatFrag MemFrag256,
-                        Intrinsic Int128, Intrinsic Int256, SDNode Op213, 
-			                  ValueType OpVT128, ValueType OpVT256> {
-  def r_Int : FMA3<opc, MRMSrcReg, (outs VR128:$dst),
-                   (ins VR128:$src1, VR128:$src2, VR128:$src3),
-                   !strconcat(OpcodeStr,
-                              "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
-                   [(set VR128:$dst, (Int128 VR128:$src2, VR128:$src1,
-                                      VR128:$src3))]>;
-
+multiclass fma3p_rm<bits<8> opc, string OpcodeStr,
+                    PatFrag MemFrag128, PatFrag MemFrag256,
+                    ValueType OpVT128, ValueType OpVT256,
+                    SDPatternOperator Op = null_frag> {
+  let isCommutable = 1 in
   def r     : FMA3<opc, MRMSrcReg, (outs VR128:$dst),
                    (ins VR128:$src1, VR128:$src2, VR128:$src3),
                    !strconcat(OpcodeStr,
                               "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
-                   [(set VR128:$dst, (OpVT128 (Op213 VR128:$src2, 
+                   [(set VR128:$dst, (OpVT128 (Op VR128:$src2,
                                                VR128:$src1, VR128:$src3)))]>;
 
-  def m_Int : FMA3<opc, MRMSrcMem, (outs VR128:$dst),
-                   (ins VR128:$src1, VR128:$src2, f128mem:$src3),
-                   !strconcat(OpcodeStr,
-                              "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
-                   [(set VR128:$dst, (Int128 VR128:$src2, VR128:$src1,
-                                      (MemFrag128 addr:$src3)))]>;
-
+  let mayLoad = 1 in
   def m     : FMA3<opc, MRMSrcMem, (outs VR128:$dst),
                    (ins VR128:$src1, VR128:$src2, f128mem:$src3),
-                   !strconcat(OpcodeStr, 
+                   !strconcat(OpcodeStr,
                               "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
-                   [(set VR128:$dst, (OpVT128 (Op213 VR128:$src2, VR128:$src1,
+                   [(set VR128:$dst, (OpVT128 (Op VR128:$src2, VR128:$src1,
                                                (MemFrag128 addr:$src3))))]>;
 
-
-  def rY_Int : FMA3<opc, MRMSrcReg, (outs VR256:$dst),
-                    (ins VR256:$src1, VR256:$src2, VR256:$src3),
-                    !strconcat(OpcodeStr, 
-                               "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
-                    [(set VR256:$dst, (Int256 VR256:$src2, VR256:$src1, 
-                                       VR256:$src3))]>;
-
+  let isCommutable = 1 in
   def rY    : FMA3<opc, MRMSrcReg, (outs VR256:$dst),
                    (ins VR256:$src1, VR256:$src2, VR256:$src3),
-                   !strconcat(OpcodeStr, 
+                   !strconcat(OpcodeStr,
                               "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
-                   [(set VR256:$dst, (OpVT256 (Op213 VR256:$src2, VR256:$src1,
-                                               VR256:$src3)))]>;
-
-  def mY_Int : FMA3<opc, MRMSrcMem, (outs VR256:$dst),
-                    (ins VR256:$src1, VR256:$src2, f256mem:$src3),
-                    !strconcat(OpcodeStr, 
-                               "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
-                    [(set VR256:$dst, (Int256 VR256:$src2, VR256:$src1, 
-                                       (MemFrag256 addr:$src3)))]>;
+                   [(set VR256:$dst, (OpVT256 (Op VR256:$src2, VR256:$src1,
+                                               VR256:$src3)))]>, VEX_L;
 
+  let mayLoad = 1 in
   def mY    : FMA3<opc, MRMSrcMem, (outs VR256:$dst),
                    (ins VR256:$src1, VR256:$src2, f256mem:$src3),
-                   !strconcat(OpcodeStr, 
+                   !strconcat(OpcodeStr,
                               "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
                    [(set VR256:$dst,
-                     (OpVT256 (Op213 VR256:$src2, VR256:$src1, 
-                               (MemFrag256 addr:$src3))))]>;
+                     (OpVT256 (Op VR256:$src2, VR256:$src1,
+                               (MemFrag256 addr:$src3))))]>, VEX_L;
 }
 } // Constraints = "$src1 = $dst"
 
 multiclass fma3p_forms<bits<8> opc132, bits<8> opc213, bits<8> opc231,
                        string OpcodeStr, string PackTy,
                        PatFrag MemFrag128, PatFrag MemFrag256,
-                       Intrinsic Int128, Intrinsic Int256, SDNode Op,
-                       ValueType OpTy128, ValueType OpTy256> {
-  defm r213 : fma3p_rm_int <opc213, !strconcat(OpcodeStr,
-                            !strconcat("213", PackTy)), MemFrag128, MemFrag256,
-                            Int128, Int256, Op, OpTy128, OpTy256>;
-  defm r132 : fma3p_rm <opc132, 
-                        !strconcat(OpcodeStr, !strconcat("132", PackTy))>;
-  defm r231 : fma3p_rm <opc231, 
-                        !strconcat(OpcodeStr, !strconcat("231", PackTy))>;
+                       SDNode Op, ValueType OpTy128, ValueType OpTy256> {
+  defm r213 : fma3p_rm<opc213,
+                       !strconcat(OpcodeStr, !strconcat("213", PackTy)),
+                       MemFrag128, MemFrag256, OpTy128, OpTy256, Op>;
+let neverHasSideEffects = 1 in {
+  defm r132 : fma3p_rm<opc132,
+                       !strconcat(OpcodeStr, !strconcat("132", PackTy)),
+                       MemFrag128, MemFrag256, OpTy128, OpTy256>;
+  defm r231 : fma3p_rm<opc231,
+                       !strconcat(OpcodeStr, !strconcat("231", PackTy)),
+                       MemFrag128, MemFrag256, OpTy128, OpTy256>;
+} // neverHasSideEffects = 1
 }
 
 // Fused Multiply-Add
 let ExeDomain = SSEPackedSingle in {
   defm VFMADDPS    : fma3p_forms<0x98, 0xA8, 0xB8, "vfmadd", "ps", memopv4f32,
-                                 memopv8f32, int_x86_fma_vfmadd_ps,
-                                 int_x86_fma_vfmadd_ps_256, X86Fmadd,
-                                 v4f32, v8f32>;
+                                 memopv8f32, X86Fmadd, v4f32, v8f32>;
   defm VFMSUBPS    : fma3p_forms<0x9A, 0xAA, 0xBA, "vfmsub", "ps", memopv4f32,
-                                 memopv8f32, int_x86_fma_vfmsub_ps, 
-                                 int_x86_fma_vfmsub_ps_256, X86Fmsub,
-                                 v4f32, v8f32>;
+                                 memopv8f32, X86Fmsub, v4f32, v8f32>;
   defm VFMADDSUBPS : fma3p_forms<0x96, 0xA6, 0xB6, "vfmaddsub", "ps",
-                                 memopv4f32, memopv8f32, 
-                                 int_x86_fma_vfmaddsub_ps,
-                                 int_x86_fma_vfmaddsub_ps_256, X86Fmaddsub, 
+                                 memopv4f32, memopv8f32, X86Fmaddsub,
                                  v4f32, v8f32>;
   defm VFMSUBADDPS : fma3p_forms<0x97, 0xA7, 0xB7, "vfmsubadd", "ps",
-                                 memopv4f32, memopv8f32, 
-                                 int_x86_fma_vfmsubadd_ps,
-                                 int_x86_fma_vfmaddsub_ps_256, X86Fmsubadd,
+                                 memopv4f32, memopv8f32, X86Fmsubadd,
                                  v4f32, v8f32>;
 }
 
 let ExeDomain = SSEPackedDouble in {
   defm VFMADDPD    : fma3p_forms<0x98, 0xA8, 0xB8, "vfmadd", "pd", memopv2f64,
-                                 memopv4f64, int_x86_fma_vfmadd_pd,
-                                 int_x86_fma_vfmadd_pd_256, X86Fmadd, v2f64,
-                                 v4f64>, VEX_W;
+                                 memopv4f64, X86Fmadd, v2f64, v4f64>, VEX_W;
   defm VFMSUBPD    : fma3p_forms<0x9A, 0xAA, 0xBA, "vfmsub", "pd", memopv2f64,
-                                 memopv4f64, int_x86_fma_vfmsub_pd,
-                                 int_x86_fma_vfmsub_pd_256, X86Fmsub, v2f64,
-                                 v4f64>, VEX_W;
+                                 memopv4f64, X86Fmsub, v2f64, v4f64>, VEX_W;
   defm VFMADDSUBPD : fma3p_forms<0x96, 0xA6, 0xB6, "vfmaddsub", "pd",
-                                 memopv2f64, memopv4f64,
-                                 int_x86_fma_vfmaddsub_pd,
-                                 int_x86_fma_vfmaddsub_pd_256, X86Fmaddsub,
+                                 memopv2f64, memopv4f64, X86Fmaddsub,
                                  v2f64, v4f64>, VEX_W;
   defm VFMSUBADDPD : fma3p_forms<0x97, 0xA7, 0xB7, "vfmsubadd", "pd",
-                                 memopv2f64, memopv4f64, 
-                                 int_x86_fma_vfmsubadd_pd,
-                                 int_x86_fma_vfmsubadd_pd_256, X86Fmsubadd,
+                                 memopv2f64, memopv4f64, X86Fmsubadd,
                                  v2f64, v4f64>, VEX_W;
 }
 
 // Fused Negative Multiply-Add
 let ExeDomain = SSEPackedSingle in {
   defm VFNMADDPS : fma3p_forms<0x9C, 0xAC, 0xBC, "vfnmadd", "ps",  memopv4f32,
-                               memopv8f32, int_x86_fma_vfnmadd_ps,
-                               int_x86_fma_vfnmadd_ps_256, X86Fnmadd, v4f32,
-                               v8f32>;
+                               memopv8f32, X86Fnmadd, v4f32, v8f32>;
   defm VFNMSUBPS : fma3p_forms<0x9E, 0xAE, 0xBE, "vfnmsub", "ps",  memopv4f32,
-                               memopv8f32, int_x86_fma_vfnmsub_ps,
-                               int_x86_fma_vfnmsub_ps_256, X86Fnmsub, v4f32,
-                               v8f32>;
+                               memopv8f32, X86Fnmsub, v4f32, v8f32>;
 }
 let ExeDomain = SSEPackedDouble in {
   defm VFNMADDPD : fma3p_forms<0x9C, 0xAC, 0xBC, "vfnmadd", "pd", memopv2f64,
-                               memopv4f64, int_x86_fma_vfnmadd_pd,
-                               int_x86_fma_vfnmadd_pd_256, X86Fnmadd, v2f64,
-                               v4f64>, VEX_W;
+                               memopv4f64, X86Fnmadd, v2f64, v4f64>, VEX_W;
   defm VFNMSUBPD : fma3p_forms<0x9E, 0xAE, 0xBE, "vfnmsub", "pd",
-                               memopv2f64,
-                               memopv4f64, int_x86_fma_vfnmsub_pd,
-                               int_x86_fma_vfnmsub_pd_256, X86Fnmsub, v2f64,
+                               memopv2f64, memopv4f64, X86Fnmsub, v2f64,
                                v4f64>, VEX_W;
 }
 
 let Constraints = "$src1 = $dst" in {
 multiclass fma3s_rm<bits<8> opc, string OpcodeStr, X86MemOperand x86memop,
-                    RegisterClass RC> {
-let neverHasSideEffects = 1 in {
-  def r : FMA3<opc, MRMSrcReg, (outs RC:$dst),
-               (ins RC:$src1, RC:$src2, RC:$src3),
-               !strconcat(OpcodeStr, 
-                          "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), []>;
+                    RegisterClass RC, ValueType OpVT, PatFrag mem_frag,
+                    SDPatternOperator OpNode = null_frag> {
+  let isCommutable = 1 in
+  def r     : FMA3<opc, MRMSrcReg, (outs RC:$dst),
+                   (ins RC:$src1, RC:$src2, RC:$src3),
+                   !strconcat(OpcodeStr,
+                              "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
+                   [(set RC:$dst,
+                     (OpVT (OpNode RC:$src2, RC:$src1, RC:$src3)))]>;
   let mayLoad = 1 in
-  def m : FMA3<opc, MRMSrcMem, (outs RC:$dst),
-               (ins RC:$src1, RC:$src2, x86memop:$src3),
-               !strconcat(OpcodeStr, 
-                          "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), []>;
-} // neverHasSideEffects = 1
+  def m     : FMA3<opc, MRMSrcMem, (outs RC:$dst),
+                   (ins RC:$src1, RC:$src2, x86memop:$src3),
+                   !strconcat(OpcodeStr,
+                              "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
+                   [(set RC:$dst,
+                     (OpVT (OpNode RC:$src2, RC:$src1,
+                            (mem_frag addr:$src3))))]>;
 }
 
 multiclass fma3s_rm_int<bits<8> opc, string OpcodeStr, Operand memop,
-                        ComplexPattern mem_cpat, Intrinsic IntId, 
-                        RegisterClass RC, SDNode OpNode, ValueType OpVT> {
+                        ComplexPattern mem_cpat, Intrinsic IntId,
+                        RegisterClass RC> {
+  let isCommutable = 1 in
   def r_Int : FMA3<opc, MRMSrcReg, (outs VR128:$dst),
                    (ins VR128:$src1, VR128:$src2, VR128:$src3),
-                   !strconcat(OpcodeStr, 
-                              "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 
-                   [(set VR128:$dst, (IntId VR128:$src2, VR128:$src1, 
+                   !strconcat(OpcodeStr,
+                              "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
+                   [(set VR128:$dst, (IntId VR128:$src2, VR128:$src1,
                      VR128:$src3))]>;
   def m_Int : FMA3<opc, MRMSrcMem, (outs VR128:$dst),
                    (ins VR128:$src1, VR128:$src2, memop:$src3),
-                   !strconcat(OpcodeStr, 
+                   !strconcat(OpcodeStr,
                               "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
                    [(set VR128:$dst,
                      (IntId VR128:$src2, VR128:$src1, mem_cpat:$src3))]>;
-  def r     : FMA3<opc, MRMSrcReg, (outs RC:$dst),
-                   (ins RC:$src1, RC:$src2, RC:$src3),
-                   !strconcat(OpcodeStr, 
-                              "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
-                   [(set RC:$dst,
-		                 (OpVT (OpNode RC:$src2, RC:$src1, RC:$src3)))]>;
-  let mayLoad = 1 in
-  def m     : FMA3<opc, MRMSrcMem, (outs RC:$dst),
-                   (ins RC:$src1, RC:$src2, memop:$src3),
-                   !strconcat(OpcodeStr, 
-                              "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), []>;
 }
 } // Constraints = "$src1 = $dst"
 
 multiclass fma3s_forms<bits<8> opc132, bits<8> opc213, bits<8> opc231,
-                       string OpStr, Intrinsic IntF32, Intrinsic IntF64, 
-                       SDNode OpNode> {
-  defm SSr132 : fma3s_rm<opc132, !strconcat(OpStr, "132ss"), f32mem, FR32>;
-  defm SSr231 : fma3s_rm<opc231, !strconcat(OpStr, "231ss"), f32mem, FR32>;
-  defm SDr132 : fma3s_rm<opc132, !strconcat(OpStr, "132sd"), f64mem, FR64>,
-                        VEX_W;
-  defm SDr231 : fma3s_rm<opc231, !strconcat(OpStr, "231sd"), f64mem, FR64>,
-                        VEX_W;
-  defm SSr213 : fma3s_rm_int <opc213, !strconcat(OpStr, "213ss"), ssmem,
-                              sse_load_f32, IntF32, FR32, OpNode, f32>;
-  defm SDr213 : fma3s_rm_int <opc213, !strconcat(OpStr, "213sd"), sdmem,
-                              sse_load_f64, IntF64, FR64, OpNode, f64>, VEX_W;
+                       string OpStr, string PackTy, Intrinsic Int,
+                       SDNode OpNode, RegisterClass RC, ValueType OpVT,
+                       X86MemOperand x86memop, Operand memop, PatFrag mem_frag,
+                       ComplexPattern mem_cpat> {
+let neverHasSideEffects = 1 in {
+  defm r132 : fma3s_rm<opc132, !strconcat(OpStr, !strconcat("132", PackTy)),
+                       x86memop, RC, OpVT, mem_frag>;
+  defm r231 : fma3s_rm<opc231, !strconcat(OpStr, !strconcat("231", PackTy)),
+                       x86memop, RC, OpVT, mem_frag>;
+}
+
+defm r213 : fma3s_rm<opc213, !strconcat(OpStr, !strconcat("213", PackTy)),
+                     x86memop, RC, OpVT, mem_frag, OpNode>,
+            fma3s_rm_int<opc213, !strconcat(OpStr, !strconcat("213", PackTy)),
+                         memop, mem_cpat, Int, RC>;
+}
+
+multiclass fma3s<bits<8> opc132, bits<8> opc213, bits<8> opc231,
+                 string OpStr, Intrinsic IntF32, Intrinsic IntF64,
+                 SDNode OpNode> {
+  defm SS : fma3s_forms<opc132, opc213, opc231, OpStr, "ss", IntF32, OpNode,
+                        FR32, f32, f32mem, ssmem, loadf32, sse_load_f32>;
+  defm SD : fma3s_forms<opc132, opc213, opc231, OpStr, "sd", IntF64, OpNode,
+                        FR64, f64, f64mem, sdmem, loadf64, sse_load_f64>, VEX_W;
 }
 
-defm VFMADD : fma3s_forms<0x99, 0xA9, 0xB9, "vfmadd", int_x86_fma_vfmadd_ss,
-                          int_x86_fma_vfmadd_sd, X86Fmadd>, VEX_LIG;
-defm VFMSUB : fma3s_forms<0x9B, 0xAB, 0xBB, "vfmsub", int_x86_fma_vfmsub_ss,
-                          int_x86_fma_vfmsub_sd, X86Fmsub>, VEX_LIG;
+defm VFMADD : fma3s<0x99, 0xA9, 0xB9, "vfmadd", int_x86_fma_vfmadd_ss,
+                    int_x86_fma_vfmadd_sd, X86Fmadd>, VEX_LIG;
+defm VFMSUB : fma3s<0x9B, 0xAB, 0xBB, "vfmsub", int_x86_fma_vfmsub_ss,
+                    int_x86_fma_vfmsub_sd, X86Fmsub>, VEX_LIG;
 
-defm VFNMADD : fma3s_forms<0x9D, 0xAD, 0xBD, "vfnmadd", int_x86_fma_vfnmadd_ss,
-                           int_x86_fma_vfnmadd_sd, X86Fnmadd>, VEX_LIG;
-defm VFNMSUB : fma3s_forms<0x9F, 0xAF, 0xBF, "vfnmsub", int_x86_fma_vfnmsub_ss,
-                           int_x86_fma_vfnmsub_sd, X86Fnmsub>, VEX_LIG;
+defm VFNMADD : fma3s<0x9D, 0xAD, 0xBD, "vfnmadd", int_x86_fma_vfnmadd_ss,
+                     int_x86_fma_vfnmadd_sd, X86Fnmadd>, VEX_LIG;
+defm VFNMSUB : fma3s<0x9F, 0xAF, 0xBF, "vfnmsub", int_x86_fma_vfnmsub_ss,
+                     int_x86_fma_vfnmsub_sd, X86Fnmsub>, VEX_LIG;
 
 
 //===----------------------------------------------------------------------===//
@@ -260,73 +197,102 @@ defm VFNMSUB : fma3s_forms<0x9F, 0xAF, 0xBF, "vfnmsub", int_x86_fma_vfnmsub_ss,
 //===----------------------------------------------------------------------===//
 
 
-multiclass fma4s<bits<8> opc, string OpcodeStr, Operand memop,
-                 ComplexPattern mem_cpat, Intrinsic Int> {
-  def rr : FMA4<opc, MRMSrcReg, (outs VR128:$dst),
-           (ins VR128:$src1, VR128:$src2, VR128:$src3),
+multiclass fma4s<bits<8> opc, string OpcodeStr, RegisterClass RC,
+                 X86MemOperand x86memop, ValueType OpVT, SDNode OpNode,
+                 PatFrag mem_frag> {
+  let isCommutable = 1 in
+  def rr : FMA4<opc, MRMSrcReg, (outs RC:$dst),
+           (ins RC:$src1, RC:$src2, RC:$src3),
            !strconcat(OpcodeStr,
            "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
-           [(set VR128:$dst,
-             (Int VR128:$src1, VR128:$src2, VR128:$src3))]>, VEX_W, MemOp4;
-  def rm : FMA4<opc, MRMSrcMem, (outs VR128:$dst),
-           (ins VR128:$src1, VR128:$src2, memop:$src3),
+           [(set RC:$dst,
+             (OpVT (OpNode RC:$src1, RC:$src2, RC:$src3)))]>, VEX_W, MemOp4;
+  def rm : FMA4<opc, MRMSrcMem, (outs RC:$dst),
+           (ins RC:$src1, RC:$src2, x86memop:$src3),
            !strconcat(OpcodeStr,
            "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
-           [(set VR128:$dst,
-             (Int VR128:$src1, VR128:$src2, mem_cpat:$src3))]>, VEX_W, MemOp4;
-  def mr : FMA4<opc, MRMSrcMem, (outs VR128:$dst),
-           (ins VR128:$src1, memop:$src2, VR128:$src3),
+           [(set RC:$dst, (OpNode RC:$src1, RC:$src2,
+                           (mem_frag addr:$src3)))]>, VEX_W, MemOp4;
+  def mr : FMA4<opc, MRMSrcMem, (outs RC:$dst),
+           (ins RC:$src1, x86memop:$src2, RC:$src3),
            !strconcat(OpcodeStr,
            "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
-           [(set VR128:$dst,
-             (Int VR128:$src1, mem_cpat:$src2, VR128:$src3))]>;
+           [(set RC:$dst,
+             (OpNode RC:$src1, (mem_frag addr:$src2), RC:$src3))]>;
 // For disassembler
 let isCodeGenOnly = 1 in
-  def rr_REV : FMA4<opc, MRMSrcReg, (outs VR128:$dst),
-               (ins VR128:$src1, VR128:$src2, VR128:$src3),
+  def rr_REV : FMA4<opc, MRMSrcReg, (outs RC:$dst),
+               (ins RC:$src1, RC:$src2, RC:$src3),
                !strconcat(OpcodeStr,
                "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), []>;
 }
 
-multiclass fma4p<bits<8> opc, string OpcodeStr,
-                 Intrinsic Int128, Intrinsic Int256,
+multiclass fma4s_int<bits<8> opc, string OpcodeStr, Operand memop,
+                     ComplexPattern mem_cpat, Intrinsic Int> {
+  let isCommutable = 1 in
+  def rr_Int : FMA4<opc, MRMSrcReg, (outs VR128:$dst),
+               (ins VR128:$src1, VR128:$src2, VR128:$src3),
+               !strconcat(OpcodeStr,
+               "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
+               [(set VR128:$dst,
+                 (Int VR128:$src1, VR128:$src2, VR128:$src3))]>, VEX_W, MemOp4;
+  def rm_Int : FMA4<opc, MRMSrcMem, (outs VR128:$dst),
+               (ins VR128:$src1, VR128:$src2, memop:$src3),
+               !strconcat(OpcodeStr,
+               "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
+               [(set VR128:$dst, (Int VR128:$src1, VR128:$src2,
+                                  mem_cpat:$src3))]>, VEX_W, MemOp4;
+  def mr_Int : FMA4<opc, MRMSrcMem, (outs VR128:$dst),
+               (ins VR128:$src1, memop:$src2, VR128:$src3),
+               !strconcat(OpcodeStr,
+               "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
+               [(set VR128:$dst,
+                 (Int VR128:$src1, mem_cpat:$src2, VR128:$src3))]>;
+}
+
+multiclass fma4p<bits<8> opc, string OpcodeStr, SDNode OpNode,
+                 ValueType OpVT128, ValueType OpVT256,
                  PatFrag ld_frag128, PatFrag ld_frag256> {
+  let isCommutable = 1 in
   def rr : FMA4<opc, MRMSrcReg, (outs VR128:$dst),
            (ins VR128:$src1, VR128:$src2, VR128:$src3),
            !strconcat(OpcodeStr,
            "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
            [(set VR128:$dst,
-             (Int128 VR128:$src1, VR128:$src2, VR128:$src3))]>, VEX_W, MemOp4;
+             (OpVT128 (OpNode VR128:$src1, VR128:$src2, VR128:$src3)))]>,
+           VEX_W, MemOp4;
   def rm : FMA4<opc, MRMSrcMem, (outs VR128:$dst),
            (ins VR128:$src1, VR128:$src2, f128mem:$src3),
            !strconcat(OpcodeStr,
            "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
-           [(set VR128:$dst, (Int128 VR128:$src1, VR128:$src2,
+           [(set VR128:$dst, (OpNode VR128:$src1, VR128:$src2,
                               (ld_frag128 addr:$src3)))]>, VEX_W, MemOp4;
   def mr : FMA4<opc, MRMSrcMem, (outs VR128:$dst),
            (ins VR128:$src1, f128mem:$src2, VR128:$src3),
            !strconcat(OpcodeStr,
            "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
            [(set VR128:$dst,
-             (Int128 VR128:$src1, (ld_frag128 addr:$src2), VR128:$src3))]>;
+             (OpNode VR128:$src1, (ld_frag128 addr:$src2), VR128:$src3))]>;
+  let isCommutable = 1 in
   def rrY : FMA4<opc, MRMSrcReg, (outs VR256:$dst),
            (ins VR256:$src1, VR256:$src2, VR256:$src3),
            !strconcat(OpcodeStr,
            "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
            [(set VR256:$dst,
-             (Int256 VR256:$src1, VR256:$src2, VR256:$src3))]>, VEX_W, MemOp4;
+             (OpVT256 (OpNode VR256:$src1, VR256:$src2, VR256:$src3)))]>,
+           VEX_W, MemOp4, VEX_L;
   def rmY : FMA4<opc, MRMSrcMem, (outs VR256:$dst),
            (ins VR256:$src1, VR256:$src2, f256mem:$src3),
            !strconcat(OpcodeStr,
            "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
-           [(set VR256:$dst, (Int256 VR256:$src1, VR256:$src2,
-                              (ld_frag256 addr:$src3)))]>, VEX_W, MemOp4;
+           [(set VR256:$dst, (OpNode VR256:$src1, VR256:$src2,
+                              (ld_frag256 addr:$src3)))]>, VEX_W, MemOp4, VEX_L;
   def mrY : FMA4<opc, MRMSrcMem, (outs VR256:$dst),
            (ins VR256:$src1, f256mem:$src2, VR256:$src3),
            !strconcat(OpcodeStr,
            "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
-           [(set VR256:$dst,
-             (Int256 VR256:$src1, (ld_frag256 addr:$src2), VR256:$src3))]>;
+           [(set VR256:$dst, (OpNode VR256:$src1,
+                              (ld_frag256 addr:$src2), VR256:$src3))]>, VEX_L;
 // For disassembler
 let isCodeGenOnly = 1 in {
   def rr_REV : FMA4<opc, MRMSrcReg, (outs VR128:$dst),
@@ -336,51 +302,65 @@ let isCodeGenOnly = 1 in {
   def rrY_REV : FMA4<opc, MRMSrcReg, (outs VR256:$dst),
                 (ins VR256:$src1, VR256:$src2, VR256:$src3),
                 !strconcat(OpcodeStr,
-                "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), []>;
+                "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), []>,
+                VEX_L;
 } // isCodeGenOnly = 1
 }
 
 let Predicates = [HasFMA4] in {
 
-defm VFMADDSS4    : fma4s<0x6A, "vfmaddss", ssmem, sse_load_f32,
-                          int_x86_fma_vfmadd_ss>;
-defm VFMADDSD4    : fma4s<0x6B, "vfmaddsd", sdmem, sse_load_f64,
-                          int_x86_fma_vfmadd_sd>;
-defm VFMADDPS4    : fma4p<0x68, "vfmaddps", int_x86_fma_vfmadd_ps,
-                          int_x86_fma_vfmadd_ps_256, memopv4f32, memopv8f32>;
-defm VFMADDPD4    : fma4p<0x69, "vfmaddpd", int_x86_fma_vfmadd_pd,
-                          int_x86_fma_vfmadd_pd_256, memopv2f64, memopv4f64>;
-defm VFMSUBSS4    : fma4s<0x6E, "vfmsubss", ssmem, sse_load_f32,
-                          int_x86_fma_vfmsub_ss>;
-defm VFMSUBSD4    : fma4s<0x6F, "vfmsubsd", sdmem, sse_load_f64,
-                          int_x86_fma_vfmsub_sd>;
-defm VFMSUBPS4    : fma4p<0x6C, "vfmsubps", int_x86_fma_vfmsub_ps,
-                          int_x86_fma_vfmsub_ps_256, memopv4f32, memopv8f32>;
-defm VFMSUBPD4    : fma4p<0x6D, "vfmsubpd", int_x86_fma_vfmsub_pd,
-                          int_x86_fma_vfmsub_pd_256, memopv2f64, memopv4f64>;
-defm VFNMADDSS4   : fma4s<0x7A, "vfnmaddss", ssmem, sse_load_f32,
-                          int_x86_fma_vfnmadd_ss>;
-defm VFNMADDSD4   : fma4s<0x7B, "vfnmaddsd", sdmem, sse_load_f64,
-                          int_x86_fma_vfnmadd_sd>;
-defm VFNMADDPS4   : fma4p<0x78, "vfnmaddps", int_x86_fma_vfnmadd_ps,
-                          int_x86_fma_vfnmadd_ps_256, memopv4f32, memopv8f32>;
-defm VFNMADDPD4   : fma4p<0x79, "vfnmaddpd", int_x86_fma_vfnmadd_pd,
-                          int_x86_fma_vfnmadd_pd_256, memopv2f64, memopv4f64>;
-defm VFNMSUBSS4   : fma4s<0x7E, "vfnmsubss", ssmem, sse_load_f32,
-                          int_x86_fma_vfnmsub_ss>;
-defm VFNMSUBSD4   : fma4s<0x7F, "vfnmsubsd", sdmem, sse_load_f64,
-                          int_x86_fma_vfnmsub_sd>;
-defm VFNMSUBPS4   : fma4p<0x7C, "vfnmsubps", int_x86_fma_vfnmsub_ps,
-                          int_x86_fma_vfnmsub_ps_256, memopv4f32, memopv8f32>;
-defm VFNMSUBPD4   : fma4p<0x7D, "vfnmsubpd", int_x86_fma_vfnmsub_pd,
-                          int_x86_fma_vfnmsub_pd_256, memopv2f64, memopv4f64>;
-defm VFMADDSUBPS4 : fma4p<0x5C, "vfmaddsubps", int_x86_fma_vfmaddsub_ps,
-                          int_x86_fma_vfmaddsub_ps_256, memopv4f32, memopv8f32>;
-defm VFMADDSUBPD4 : fma4p<0x5D, "vfmaddsubpd", int_x86_fma_vfmaddsub_pd,
-                          int_x86_fma_vfmaddsub_pd_256, memopv2f64, memopv4f64>;
-defm VFMSUBADDPS4 : fma4p<0x5E, "vfmsubaddps", int_x86_fma_vfmsubadd_ps,
-                          int_x86_fma_vfmsubadd_ps_256, memopv4f32, memopv8f32>;
-defm VFMSUBADDPD4 : fma4p<0x5F, "vfmsubaddpd", int_x86_fma_vfmsubadd_pd,
-                          int_x86_fma_vfmsubadd_pd_256, memopv2f64, memopv4f64>;
+defm VFMADDSS4  : fma4s<0x6A, "vfmaddss", FR32, f32mem, f32, X86Fmadd, loadf32>,
+                  fma4s_int<0x6A, "vfmaddss", ssmem, sse_load_f32,
+                            int_x86_fma_vfmadd_ss>;
+defm VFMADDSD4  : fma4s<0x6B, "vfmaddsd", FR64, f64mem, f64, X86Fmadd, loadf64>,
+                  fma4s_int<0x6B, "vfmaddsd", sdmem, sse_load_f64,
+                            int_x86_fma_vfmadd_sd>;
+defm VFMSUBSS4  : fma4s<0x6E, "vfmsubss", FR32, f32mem, f32, X86Fmsub, loadf32>,
+                  fma4s_int<0x6E, "vfmsubss", ssmem, sse_load_f32,
+                            int_x86_fma_vfmsub_ss>;
+defm VFMSUBSD4  : fma4s<0x6F, "vfmsubsd", FR64, f64mem, f64, X86Fmsub, loadf64>,
+                  fma4s_int<0x6F, "vfmsubsd", sdmem, sse_load_f64,
+                            int_x86_fma_vfmsub_sd>;
+defm VFNMADDSS4 : fma4s<0x7A, "vfnmaddss", FR32, f32mem, f32,
+                        X86Fnmadd, loadf32>,
+                  fma4s_int<0x7A, "vfnmaddss", ssmem, sse_load_f32,
+                            int_x86_fma_vfnmadd_ss>;
+defm VFNMADDSD4 : fma4s<0x7B, "vfnmaddsd", FR64, f64mem, f64,
+                        X86Fnmadd, loadf64>,
+                  fma4s_int<0x7B, "vfnmaddsd", sdmem, sse_load_f64,
+                            int_x86_fma_vfnmadd_sd>;
+defm VFNMSUBSS4 : fma4s<0x7E, "vfnmsubss", FR32, f32mem, f32,
+                        X86Fnmsub, loadf32>,
+                  fma4s_int<0x7E, "vfnmsubss", ssmem, sse_load_f32,
+                            int_x86_fma_vfnmsub_ss>;
+defm VFNMSUBSD4 : fma4s<0x7F, "vfnmsubsd", FR64, f64mem, f64,
+                        X86Fnmsub, loadf64>,
+                  fma4s_int<0x7F, "vfnmsubsd", sdmem, sse_load_f64,
+                            int_x86_fma_vfnmsub_sd>;
+
+defm VFMADDPS4    : fma4p<0x68, "vfmaddps", X86Fmadd, v4f32, v8f32,
+                          memopv4f32, memopv8f32>;
+defm VFMADDPD4    : fma4p<0x69, "vfmaddpd", X86Fmadd, v2f64, v4f64,
+                          memopv2f64, memopv4f64>;
+defm VFMSUBPS4    : fma4p<0x6C, "vfmsubps", X86Fmsub, v4f32, v8f32,
+                          memopv4f32, memopv8f32>;
+defm VFMSUBPD4    : fma4p<0x6D, "vfmsubpd", X86Fmsub, v2f64, v4f64,
+                          memopv2f64, memopv4f64>;
+defm VFNMADDPS4   : fma4p<0x78, "vfnmaddps", X86Fnmadd, v4f32, v8f32,
+                          memopv4f32, memopv8f32>;
+defm VFNMADDPD4   : fma4p<0x79, "vfnmaddpd", X86Fnmadd, v2f64, v4f64,
+                          memopv2f64, memopv4f64>;
+defm VFNMSUBPS4   : fma4p<0x7C, "vfnmsubps", X86Fnmsub, v4f32, v8f32,
+                          memopv4f32, memopv8f32>;
+defm VFNMSUBPD4   : fma4p<0x7D, "vfnmsubpd", X86Fnmsub, v2f64, v4f64,
+                          memopv2f64, memopv4f64>;
+defm VFMADDSUBPS4 : fma4p<0x5C, "vfmaddsubps", X86Fmaddsub, v4f32, v8f32,
+                          memopv4f32, memopv8f32>;
+defm VFMADDSUBPD4 : fma4p<0x5D, "vfmaddsubpd", X86Fmaddsub, v2f64, v4f64,
+                          memopv2f64, memopv4f64>;
+defm VFMSUBADDPS4 : fma4p<0x5E, "vfmsubaddps", X86Fmsubadd, v4f32, v8f32,
+                          memopv4f32, memopv8f32>;
+defm VFMSUBADDPD4 : fma4p<0x5F, "vfmsubaddpd", X86Fmsubadd, v2f64, v4f64,
+                          memopv2f64, memopv4f64>;
 } // HasFMA4
 
diff --git a/lib/Target/X86/X86InstrFormats.td b/lib/Target/X86/X86InstrFormats.td
index 81b4f81..268e9fc 100644
--- a/lib/Target/X86/X86InstrFormats.td
+++ b/lib/Target/X86/X86InstrFormats.td
@@ -44,14 +44,15 @@ def RawFrmImm16 : Format<44>;
 def MRM_D0 : Format<45>;
 def MRM_D1 : Format<46>;
 def MRM_D4 : Format<47>;
-def MRM_D8 : Format<48>;
-def MRM_D9 : Format<49>;
-def MRM_DA : Format<50>;
-def MRM_DB : Format<51>;
-def MRM_DC : Format<52>;
-def MRM_DD : Format<53>;
-def MRM_DE : Format<54>;
-def MRM_DF : Format<55>;
+def MRM_D5 : Format<48>;
+def MRM_D8 : Format<49>;
+def MRM_D9 : Format<50>;
+def MRM_DA : Format<51>;
+def MRM_DB : Format<52>;
+def MRM_DC : Format<53>;
+def MRM_DD : Format<54>;
+def MRM_DE : Format<55>;
+def MRM_DF : Format<56>;
 
 // ImmType - This specifies the immediate type used by an instruction. This is
 // part of the ad-hoc solution used to emit machine instruction encodings by our
@@ -287,12 +288,14 @@ class Iseg32 <bits<8> o, Format f, dag outs, dag ins, string asm,
   let CodeSize = 3;
 }
 
+def __xs : XS;
+
 // SI - SSE 1 & 2 scalar instructions
 class SI<bits<8> o, Format F, dag outs, dag ins, string asm,
          list<dag> pattern, InstrItinClass itin = IIC_DEFAULT>
       : I<o, F, outs, ins, asm, pattern, itin> {
   let Predicates = !if(hasVEXPrefix /* VEX */, [HasAVX],
-            !if(!eq(Prefix, 12 /* XS */), [HasSSE1], [HasSSE2]));
+            !if(!eq(Prefix, __xs.Prefix), [UseSSE1], [UseSSE2]));
 
   // AVX instructions have a 'v' prefix in the mnemonic
   let AsmString = !if(hasVEXPrefix, !strconcat("v", asm), asm);
@@ -303,7 +306,7 @@ class SIi8<bits<8> o, Format F, dag outs, dag ins, string asm,
            list<dag> pattern, InstrItinClass itin = IIC_DEFAULT>
       : Ii8<o, F, outs, ins, asm, pattern, itin> {
   let Predicates = !if(hasVEXPrefix /* VEX */, [HasAVX],
-            !if(!eq(Prefix, 12 /* XS */), [HasSSE1], [HasSSE2]));
+            !if(!eq(Prefix, __xs.Prefix), [UseSSE1], [UseSSE2]));
 
   // AVX instructions have a 'v' prefix in the mnemonic
   let AsmString = !if(hasVEXPrefix, !strconcat("v", asm), asm);
@@ -314,18 +317,25 @@ class PI<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern,
          InstrItinClass itin, Domain d>
       : I<o, F, outs, ins, asm, pattern, itin, d> {
   let Predicates = !if(hasVEXPrefix /* VEX */, [HasAVX],
-        !if(hasOpSizePrefix /* OpSize */, [HasSSE2], [HasSSE1]));
+        !if(hasOpSizePrefix /* OpSize */, [UseSSE2], [UseSSE1]));
 
   // AVX instructions have a 'v' prefix in the mnemonic
   let AsmString = !if(hasVEXPrefix, !strconcat("v", asm), asm);
 }
 
+// MMXPI - SSE 1 & 2 packed instructions with MMX operands
+class MMXPI<bits<8> o, Format F, dag outs, dag ins, string asm, list<dag> pattern,
+            InstrItinClass itin, Domain d>
+      : I<o, F, outs, ins, asm, pattern, itin, d> {
+  let Predicates = !if(hasOpSizePrefix /* OpSize */, [HasSSE2], [HasSSE1]);
+}
+
 // PIi8 - SSE 1 & 2 packed instructions with immediate
 class PIi8<bits<8> o, Format F, dag outs, dag ins, string asm,
            list<dag> pattern, InstrItinClass itin, Domain d>
       : Ii8<o, F, outs, ins, asm, pattern, itin, d> {
   let Predicates = !if(hasVEX_4VPrefix /* VEX */, [HasAVX],
-        !if(hasOpSizePrefix /* OpSize */, [HasSSE2], [HasSSE1]));
+        !if(hasOpSizePrefix /* OpSize */, [UseSSE2], [UseSSE1]));
 
   // AVX instructions have a 'v' prefix in the mnemonic
   let AsmString = !if(hasVEX_4VPrefix, !strconcat("v", asm), asm);
@@ -341,18 +351,18 @@ class PIi8<bits<8> o, Format F, dag outs, dag ins, string asm,
 
 class SSI<bits<8> o, Format F, dag outs, dag ins, string asm,
           list<dag> pattern, InstrItinClass itin = IIC_DEFAULT>
-      : I<o, F, outs, ins, asm, pattern, itin>, XS, Requires<[HasSSE1]>;
+      : I<o, F, outs, ins, asm, pattern, itin>, XS, Requires<[UseSSE1]>;
 class SSIi8<bits<8> o, Format F, dag outs, dag ins, string asm,
             list<dag> pattern, InstrItinClass itin = IIC_DEFAULT>
-      : Ii8<o, F, outs, ins, asm, pattern, itin>, XS, Requires<[HasSSE1]>;
+      : Ii8<o, F, outs, ins, asm, pattern, itin>, XS, Requires<[UseSSE1]>;
 class PSI<bits<8> o, Format F, dag outs, dag ins, string asm,
           list<dag> pattern, InstrItinClass itin = IIC_DEFAULT>
       : I<o, F, outs, ins, asm, pattern, itin, SSEPackedSingle>, TB,
-        Requires<[HasSSE1]>;
+        Requires<[UseSSE1]>;
 class PSIi8<bits<8> o, Format F, dag outs, dag ins, string asm,
             list<dag> pattern, InstrItinClass itin = IIC_DEFAULT>
       : Ii8<o, F, outs, ins, asm, pattern, itin, SSEPackedSingle>, TB,
-        Requires<[HasSSE1]>;
+        Requires<[UseSSE1]>;
 class VSSI<bits<8> o, Format F, dag outs, dag ins, string asm,
            list<dag> pattern, InstrItinClass itin = IIC_DEFAULT>
       : I<o, F, outs, ins, !strconcat("v", asm), pattern, itin>, XS,
@@ -372,27 +382,31 @@ class VPSI<bits<8> o, Format F, dag outs, dag ins, string asm,
 //   PDIi8  - SSE2 instructions with ImmT == Imm8 and TB and OpSize prefixes.
 //   VSDI   - SSE2 instructions with XD prefix in AVX form.
 //   VPDI   - SSE2 instructions with TB and OpSize prefixes in AVX form.
+//   MMXSDIi8  - SSE2 instructions with ImmT == Imm8 and XD prefix as well as
+//               MMX operands.
+//   MMXSSDIi8 - SSE2 instructions with ImmT == Imm8 and XS prefix as well as
+//               MMX operands.
 
 class SDI<bits<8> o, Format F, dag outs, dag ins, string asm,
           list<dag> pattern, InstrItinClass itin = IIC_DEFAULT>
-      : I<o, F, outs, ins, asm, pattern, itin>, XD, Requires<[HasSSE2]>;
+      : I<o, F, outs, ins, asm, pattern, itin>, XD, Requires<[UseSSE2]>;
 class SDIi8<bits<8> o, Format F, dag outs, dag ins, string asm,
             list<dag> pattern, InstrItinClass itin = IIC_DEFAULT>
-      : Ii8<o, F, outs, ins, asm, pattern, itin>, XD, Requires<[HasSSE2]>;
+      : Ii8<o, F, outs, ins, asm, pattern, itin>, XD, Requires<[UseSSE2]>;
 class S2SI<bits<8> o, Format F, dag outs, dag ins, string asm,
            list<dag> pattern, InstrItinClass itin = IIC_DEFAULT>
-      : I<o, F, outs, ins, asm, pattern, itin>, XS, Requires<[HasSSE2]>;
+      : I<o, F, outs, ins, asm, pattern, itin>, XS, Requires<[UseSSE2]>;
 class S2SIi8<bits<8> o, Format F, dag outs, dag ins, string asm,
              list<dag> pattern, InstrItinClass itin = IIC_DEFAULT>
-      : Ii8<o, F, outs, ins, asm, pattern>, XS, Requires<[HasSSE2]>;
+      : Ii8<o, F, outs, ins, asm, pattern>, XS, Requires<[UseSSE2]>;
 class PDI<bits<8> o, Format F, dag outs, dag ins, string asm,
           list<dag> pattern, InstrItinClass itin = IIC_DEFAULT>
       : I<o, F, outs, ins, asm, pattern, itin, SSEPackedDouble>, TB, OpSize,
-        Requires<[HasSSE2]>;
+        Requires<[UseSSE2]>;
 class PDIi8<bits<8> o, Format F, dag outs, dag ins, string asm,
             list<dag> pattern, InstrItinClass itin = IIC_DEFAULT>
       : Ii8<o, F, outs, ins, asm, pattern, itin, SSEPackedDouble>, TB, OpSize,
-        Requires<[HasSSE2]>;
+        Requires<[UseSSE2]>;
 class VSDI<bits<8> o, Format F, dag outs, dag ins, string asm,
            list<dag> pattern, InstrItinClass itin = IIC_DEFAULT>
       : I<o, F, outs, ins, !strconcat("v", asm), pattern, itin>, XD,
@@ -405,6 +419,12 @@ class VPDI<bits<8> o, Format F, dag outs, dag ins, string asm,
            list<dag> pattern, InstrItinClass itin = IIC_DEFAULT>
       : I<o, F, outs, ins, !strconcat("v", asm), pattern, itin, SSEPackedDouble>, TB,
         OpSize, Requires<[HasAVX]>;
+class MMXSDIi8<bits<8> o, Format F, dag outs, dag ins, string asm,
+               list<dag> pattern, InstrItinClass itin = IIC_DEFAULT>
+      : Ii8<o, F, outs, ins, asm, pattern, itin>, XD, Requires<[HasSSE2]>;
+class MMXS2SIi8<bits<8> o, Format F, dag outs, dag ins, string asm,
+                list<dag> pattern, InstrItinClass itin = IIC_DEFAULT>
+      : Ii8<o, F, outs, ins, asm, pattern>, XS, Requires<[HasSSE2]>;
 
 // SSE3 Instruction Templates:
 // 
@@ -415,21 +435,23 @@ class VPDI<bits<8> o, Format F, dag outs, dag ins, string asm,
 class S3SI<bits<8> o, Format F, dag outs, dag ins, string asm, 
            list<dag> pattern, InstrItinClass itin = IIC_DEFAULT>
       : I<o, F, outs, ins, asm, pattern, itin, SSEPackedSingle>, XS,
-        Requires<[HasSSE3]>;
+        Requires<[UseSSE3]>;
 class S3DI<bits<8> o, Format F, dag outs, dag ins, string asm, 
            list<dag> pattern, InstrItinClass itin = IIC_DEFAULT>
       : I<o, F, outs, ins, asm, pattern, itin, SSEPackedDouble>, XD,
-        Requires<[HasSSE3]>;
+        Requires<[UseSSE3]>;
 class S3I<bits<8> o, Format F, dag outs, dag ins, string asm,
           list<dag> pattern, InstrItinClass itin = IIC_DEFAULT>
       : I<o, F, outs, ins, asm, pattern, itin, SSEPackedDouble>, TB, OpSize,
-        Requires<[HasSSE3]>;
+        Requires<[UseSSE3]>;
 
 
 // SSSE3 Instruction Templates:
 // 
 //   SS38I - SSSE3 instructions with T8 prefix.
 //   SS3AI - SSSE3 instructions with TA prefix.
+//   MMXSS38I - SSSE3 instructions with T8 prefix and MMX operands.
+//   MMXSS3AI - SSSE3 instructions with TA prefix and MMX operands.
 //
 // Note: SSSE3 instructions have 64-bit and 128-bit versions. The 64-bit version
 // uses the MMX registers. The 64-bit versions are grouped with the MMX
@@ -438,10 +460,18 @@ class S3I<bits<8> o, Format F, dag outs, dag ins, string asm,
 class SS38I<bits<8> o, Format F, dag outs, dag ins, string asm,
             list<dag> pattern, InstrItinClass itin = IIC_DEFAULT>
       : I<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, T8,
-        Requires<[HasSSSE3]>;
+        Requires<[UseSSSE3]>;
 class SS3AI<bits<8> o, Format F, dag outs, dag ins, string asm,
             list<dag> pattern, InstrItinClass itin = IIC_DEFAULT>
       : Ii8<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, TA,
+        Requires<[UseSSSE3]>;
+class MMXSS38I<bits<8> o, Format F, dag outs, dag ins, string asm,
+               list<dag> pattern, InstrItinClass itin = IIC_DEFAULT>
+      : I<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, T8,
+        Requires<[HasSSSE3]>;
+class MMXSS3AI<bits<8> o, Format F, dag outs, dag ins, string asm,
+               list<dag> pattern, InstrItinClass itin = IIC_DEFAULT>
+      : Ii8<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, TA,
         Requires<[HasSSSE3]>;
 
 // SSE4.1 Instruction Templates:
@@ -452,11 +482,11 @@ class SS3AI<bits<8> o, Format F, dag outs, dag ins, string asm,
 class SS48I<bits<8> o, Format F, dag outs, dag ins, string asm,
             list<dag> pattern, InstrItinClass itin = IIC_DEFAULT>
       : I<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, T8,
-        Requires<[HasSSE41]>;
+        Requires<[UseSSE41]>;
 class SS4AIi8<bits<8> o, Format F, dag outs, dag ins, string asm,
             list<dag> pattern, InstrItinClass itin = IIC_DEFAULT>
       : Ii8<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, TA,
-        Requires<[HasSSE41]>;
+        Requires<[UseSSE41]>;
 
 // SSE4.2 Instruction Templates:
 // 
@@ -464,9 +494,10 @@ class SS4AIi8<bits<8> o, Format F, dag outs, dag ins, string asm,
 class SS428I<bits<8> o, Format F, dag outs, dag ins, string asm,
              list<dag> pattern, InstrItinClass itin = IIC_DEFAULT>
       : I<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, T8,
-        Requires<[HasSSE42]>;
+        Requires<[UseSSE42]>;
 
 //   SS42FI - SSE 4.2 instructions with T8XD prefix.
+// NOTE: 'HasSSE42' is used as SS42FI is only used for CRC32 insns.
 class SS42FI<bits<8> o, Format F, dag outs, dag ins, string asm,
              list<dag> pattern, InstrItinClass itin = IIC_DEFAULT>
       : I<o, F, outs, ins, asm, pattern, itin>, T8XD, Requires<[HasSSE42]>;
@@ -475,7 +506,7 @@ class SS42FI<bits<8> o, Format F, dag outs, dag ins, string asm,
 class SS42AI<bits<8> o, Format F, dag outs, dag ins, string asm,
              list<dag> pattern, InstrItinClass itin = IIC_DEFAULT>
       : Ii8<o, F, outs, ins, asm, pattern, itin, SSEPackedInt>, TA,
-        Requires<[HasSSE42]>;
+        Requires<[UseSSE42]>;
 
 // AVX Instruction Templates:
 //   Instructions introduced in AVX (no SSE equivalent forms)
diff --git a/lib/Target/X86/X86InstrFragmentsSIMD.td b/lib/Target/X86/X86InstrFragmentsSIMD.td
index 1db68c8..73ba001 100644
--- a/lib/Target/X86/X86InstrFragmentsSIMD.td
+++ b/lib/Target/X86/X86InstrFragmentsSIMD.td
@@ -29,6 +29,13 @@ def SDTX86VFCMP : SDTypeProfile<1, 3, [SDTCisInt<0>, SDTCisSameAs<1, 2>,
 
 def X86fmin    : SDNode<"X86ISD::FMIN",      SDTFPBinOp>;
 def X86fmax    : SDNode<"X86ISD::FMAX",      SDTFPBinOp>;
+
+// Commutative and Associative FMIN and FMAX.
+def X86fminc    : SDNode<"X86ISD::FMINC", SDTFPBinOp,
+    [SDNPCommutative, SDNPAssociative]>;
+def X86fmaxc    : SDNode<"X86ISD::FMAXC", SDTFPBinOp,
+    [SDNPCommutative, SDNPAssociative]>;
+
 def X86fand    : SDNode<"X86ISD::FAND",      SDTFPBinOp,
                         [SDNPCommutative, SDNPAssociative]>;
 def X86for     : SDNode<"X86ISD::FOR",       SDTFPBinOp,
@@ -73,18 +80,30 @@ def X86vzmovl  : SDNode<"X86ISD::VZEXT_MOVL",
                  SDTypeProfile<1, 1, [SDTCisSameAs<0,1>]>>;
 
 def X86vzmovly  : SDNode<"X86ISD::VZEXT_MOVL",
-                 SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>, 
+                 SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>,
                                       SDTCisOpSmallerThanOp<1, 0> ]>>;
 
 def X86vsmovl  : SDNode<"X86ISD::VSEXT_MOVL",
-                 SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisInt<1>, SDTCisInt<0>]>>;
+                 SDTypeProfile<1, 1,
+                 [SDTCisVec<0>, SDTCisInt<1>, SDTCisInt<0>]>>;
 
 def X86vzload  : SDNode<"X86ISD::VZEXT_LOAD", SDTLoad,
                         [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
 
+def X86vzext   : SDNode<"X86ISD::VZEXT",
+                         SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>,
+                                              SDTCisInt<0>, SDTCisInt<1>]>>;
+
+def X86vsext   : SDNode<"X86ISD::VSEXT",
+                         SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>,
+                                              SDTCisInt<0>, SDTCisInt<1>]>>;
+
 def X86vfpext  : SDNode<"X86ISD::VFPEXT",
                         SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>,
                                              SDTCisFP<0>, SDTCisFP<1>]>>;
+def X86vfpround: SDNode<"X86ISD::VFPROUND",
+                        SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>,
+                                             SDTCisFP<0>, SDTCisFP<1>]>>;
 
 def X86vshldq  : SDNode<"X86ISD::VSHLDQ",    SDTIntShiftOp>;
 def X86vshrdq  : SDNode<"X86ISD::VSRLDQ",    SDTIntShiftOp>;
@@ -175,8 +194,8 @@ def X86Fmadd     : SDNode<"X86ISD::FMADD",     SDTFma>;
 def X86Fnmadd    : SDNode<"X86ISD::FNMADD",    SDTFma>;
 def X86Fmsub     : SDNode<"X86ISD::FMSUB",     SDTFma>;
 def X86Fnmsub    : SDNode<"X86ISD::FNMSUB",    SDTFma>;
-def X86Fmaddsub  : SDNode<"X86ISD::FMSUBADD",  SDTFma>;
-def X86Fmsubadd  : SDNode<"X86ISD::FMADDSUB",  SDTFma>;
+def X86Fmaddsub  : SDNode<"X86ISD::FMADDSUB",  SDTFma>;
+def X86Fmsubadd  : SDNode<"X86ISD::FMSUBADD",  SDTFma>;
 
 def SDT_PCMPISTRI : SDTypeProfile<2, 3, [SDTCisVT<0, i32>, SDTCisVT<1, i32>,
                                          SDTCisVT<2, v16i8>, SDTCisVT<3, v16i8>,
@@ -232,6 +251,10 @@ def loadv8f32    : PatFrag<(ops node:$ptr), (v8f32 (load node:$ptr))>;
 def loadv4f64    : PatFrag<(ops node:$ptr), (v4f64 (load node:$ptr))>;
 def loadv4i64    : PatFrag<(ops node:$ptr), (v4i64 (load node:$ptr))>;
 
+// 128-/256-bit extload pattern fragments
+def extloadv2f32 : PatFrag<(ops node:$ptr), (v2f64 (extloadvf32 node:$ptr))>;
+def extloadv4f32 : PatFrag<(ops node:$ptr), (v4f64 (extloadvf32 node:$ptr))>;
+
 // Like 'store', but always requires 128-bit vector alignment.
 def alignedstore : PatFrag<(ops node:$val, node:$ptr),
                            (store node:$val, node:$ptr), [{
diff --git a/lib/Target/X86/X86InstrInfo.cpp b/lib/Target/X86/X86InstrInfo.cpp
index cca04e5..5a99ff0 100644
--- a/lib/Target/X86/X86InstrInfo.cpp
+++ b/lib/Target/X86/X86InstrInfo.cpp
@@ -561,6 +561,16 @@ X86InstrInfo::X86InstrInfo(X86TargetMachine &tm)
     { X86::VSQRTPSYr_Int,   X86::VSQRTPSYm_Int,       TB_ALIGN_32 },
     { X86::VBROADCASTSSYrr, X86::VBROADCASTSSYrm,     TB_NO_REVERSE },
     { X86::VBROADCASTSDYrr, X86::VBROADCASTSDYrm,     TB_NO_REVERSE },
+
+    // BMI/BMI2 foldable instructions
+    { X86::RORX32ri,        X86::RORX32mi,            0 },
+    { X86::RORX64ri,        X86::RORX64mi,            0 },
+    { X86::SARX32rr,        X86::SARX32rm,            0 },
+    { X86::SARX64rr,        X86::SARX64rm,            0 },
+    { X86::SHRX32rr,        X86::SHRX32rm,            0 },
+    { X86::SHRX64rr,        X86::SHRX64rm,            0 },
+    { X86::SHLX32rr,        X86::SHLX32rm,            0 },
+    { X86::SHLX64rr,        X86::SHLX64rm,            0 },
   };
 
   for (unsigned i = 0, e = array_lengthof(OpTbl1); i != e; ++i) {
@@ -1110,6 +1120,44 @@ X86InstrInfo::X86InstrInfo(X86TargetMachine &tm)
     { X86::VPUNPCKLWDYrr,     X86::VPUNPCKLWDYrm,      TB_ALIGN_32 },
     { X86::VPXORYrr,          X86::VPXORYrm,           TB_ALIGN_32 },
     // FIXME: add AVX 256-bit foldable instructions
+
+    // FMA4 foldable patterns
+    { X86::VFMADDSS4rr,       X86::VFMADDSS4mr,        0           },
+    { X86::VFMADDSD4rr,       X86::VFMADDSD4mr,        0           },
+    { X86::VFMADDPS4rr,       X86::VFMADDPS4mr,        TB_ALIGN_16 },
+    { X86::VFMADDPD4rr,       X86::VFMADDPD4mr,        TB_ALIGN_16 },
+    { X86::VFMADDPS4rrY,      X86::VFMADDPS4mrY,       TB_ALIGN_32 },
+    { X86::VFMADDPD4rrY,      X86::VFMADDPD4mrY,       TB_ALIGN_32 },
+    { X86::VFNMADDSS4rr,      X86::VFNMADDSS4mr,       0           },
+    { X86::VFNMADDSD4rr,      X86::VFNMADDSD4mr,       0           },
+    { X86::VFNMADDPS4rr,      X86::VFNMADDPS4mr,       TB_ALIGN_16 },
+    { X86::VFNMADDPD4rr,      X86::VFNMADDPD4mr,       TB_ALIGN_16 },
+    { X86::VFNMADDPS4rrY,     X86::VFNMADDPS4mrY,      TB_ALIGN_32 },
+    { X86::VFNMADDPD4rrY,     X86::VFNMADDPD4mrY,      TB_ALIGN_32 },
+    { X86::VFMSUBSS4rr,       X86::VFMSUBSS4mr,        0           },
+    { X86::VFMSUBSD4rr,       X86::VFMSUBSD4mr,        0           },
+    { X86::VFMSUBPS4rr,       X86::VFMSUBPS4mr,        TB_ALIGN_16 },
+    { X86::VFMSUBPD4rr,       X86::VFMSUBPD4mr,        TB_ALIGN_16 },
+    { X86::VFMSUBPS4rrY,      X86::VFMSUBPS4mrY,       TB_ALIGN_32 },
+    { X86::VFMSUBPD4rrY,      X86::VFMSUBPD4mrY,       TB_ALIGN_32 },
+    { X86::VFNMSUBSS4rr,      X86::VFNMSUBSS4mr,       0           },
+    { X86::VFNMSUBSD4rr,      X86::VFNMSUBSD4mr,       0           },
+    { X86::VFNMSUBPS4rr,      X86::VFNMSUBPS4mr,       TB_ALIGN_16 },
+    { X86::VFNMSUBPD4rr,      X86::VFNMSUBPD4mr,       TB_ALIGN_16 },
+    { X86::VFNMSUBPS4rrY,     X86::VFNMSUBPS4mrY,      TB_ALIGN_32 },
+    { X86::VFNMSUBPD4rrY,     X86::VFNMSUBPD4mrY,      TB_ALIGN_32 },
+    { X86::VFMADDSUBPS4rr,    X86::VFMADDSUBPS4mr,     TB_ALIGN_16 },
+    { X86::VFMADDSUBPD4rr,    X86::VFMADDSUBPD4mr,     TB_ALIGN_16 },
+    { X86::VFMADDSUBPS4rrY,   X86::VFMADDSUBPS4mrY,    TB_ALIGN_32 },
+    { X86::VFMADDSUBPD4rrY,   X86::VFMADDSUBPD4mrY,    TB_ALIGN_32 },
+    { X86::VFMSUBADDPS4rr,    X86::VFMSUBADDPS4mr,     TB_ALIGN_16 },
+    { X86::VFMSUBADDPD4rr,    X86::VFMSUBADDPD4mr,     TB_ALIGN_16 },
+    { X86::VFMSUBADDPS4rrY,   X86::VFMSUBADDPS4mrY,    TB_ALIGN_32 },
+    { X86::VFMSUBADDPD4rrY,   X86::VFMSUBADDPD4mrY,    TB_ALIGN_32 },
+
+    // BMI/BMI2 foldable instructions
+    { X86::MULX32rr,          X86::MULX32rm,            0 },
+    { X86::MULX64rr,          X86::MULX64rm,            0 },
   };
 
   for (unsigned i = 0, e = array_lengthof(OpTbl2); i != e; ++i) {
@@ -1145,10 +1193,6 @@ X86InstrInfo::X86InstrInfo(X86TargetMachine &tm)
     { X86::VFMADDPDr132rY,        X86::VFMADDPDr132mY,        TB_ALIGN_32 },
     { X86::VFMADDPSr213rY,        X86::VFMADDPSr213mY,        TB_ALIGN_32 },
     { X86::VFMADDPDr213rY,        X86::VFMADDPDr213mY,        TB_ALIGN_32 },
-    { X86::VFMADDPSr213r_Int,     X86::VFMADDPSr213m_Int,     TB_ALIGN_16 },
-    { X86::VFMADDPDr213r_Int,     X86::VFMADDPDr213m_Int,     TB_ALIGN_16 },
-    { X86::VFMADDPSr213rY_Int,    X86::VFMADDPSr213mY_Int,    TB_ALIGN_32 },
-    { X86::VFMADDPDr213rY_Int,    X86::VFMADDPDr213mY_Int,    TB_ALIGN_32 },
 
     { X86::VFNMADDSSr231r,        X86::VFNMADDSSr231m,        0 },
     { X86::VFNMADDSDr231r,        X86::VFNMADDSDr231m,        0 },
@@ -1171,10 +1215,6 @@ X86InstrInfo::X86InstrInfo(X86TargetMachine &tm)
     { X86::VFNMADDPDr132rY,       X86::VFNMADDPDr132mY,       TB_ALIGN_32 },
     { X86::VFNMADDPSr213rY,       X86::VFNMADDPSr213mY,       TB_ALIGN_32 },
     { X86::VFNMADDPDr213rY,       X86::VFNMADDPDr213mY,       TB_ALIGN_32 },
-    { X86::VFNMADDPSr213r_Int,    X86::VFNMADDPSr213m_Int,    TB_ALIGN_16 },
-    { X86::VFNMADDPDr213r_Int,    X86::VFNMADDPDr213m_Int,    TB_ALIGN_16 },
-    { X86::VFNMADDPSr213rY_Int,   X86::VFNMADDPSr213mY_Int,   TB_ALIGN_32 },
-    { X86::VFNMADDPDr213rY_Int,   X86::VFNMADDPDr213mY_Int,   TB_ALIGN_32 },
 
     { X86::VFMSUBSSr231r,         X86::VFMSUBSSr231m,         0 },
     { X86::VFMSUBSDr231r,         X86::VFMSUBSDr231m,         0 },
@@ -1197,10 +1237,6 @@ X86InstrInfo::X86InstrInfo(X86TargetMachine &tm)
     { X86::VFMSUBPDr132rY,        X86::VFMSUBPDr132mY,        TB_ALIGN_32 },
     { X86::VFMSUBPSr213rY,        X86::VFMSUBPSr213mY,        TB_ALIGN_32 },
     { X86::VFMSUBPDr213rY,        X86::VFMSUBPDr213mY,        TB_ALIGN_32 },
-    { X86::VFMSUBPSr213r_Int,     X86::VFMSUBPSr213m_Int,     TB_ALIGN_16 },
-    { X86::VFMSUBPDr213r_Int,     X86::VFMSUBPDr213m_Int,     TB_ALIGN_16 },
-    { X86::VFMSUBPSr213rY_Int,    X86::VFMSUBPSr213mY_Int,    TB_ALIGN_32 },
-    { X86::VFMSUBPDr213rY_Int,    X86::VFMSUBPDr213mY_Int,    TB_ALIGN_32 },
 
     { X86::VFNMSUBSSr231r,        X86::VFNMSUBSSr231m,        0 },
     { X86::VFNMSUBSDr231r,        X86::VFNMSUBSDr231m,        0 },
@@ -1223,10 +1259,6 @@ X86InstrInfo::X86InstrInfo(X86TargetMachine &tm)
     { X86::VFNMSUBPDr132rY,       X86::VFNMSUBPDr132mY,       TB_ALIGN_32 },
     { X86::VFNMSUBPSr213rY,       X86::VFNMSUBPSr213mY,       TB_ALIGN_32 },
     { X86::VFNMSUBPDr213rY,       X86::VFNMSUBPDr213mY,       TB_ALIGN_32 },
-    { X86::VFNMSUBPSr213r_Int,    X86::VFNMSUBPSr213m_Int,    TB_ALIGN_16 },
-    { X86::VFNMSUBPDr213r_Int,    X86::VFNMSUBPDr213m_Int,    TB_ALIGN_16 },
-    { X86::VFNMSUBPSr213rY_Int,   X86::VFNMSUBPSr213mY_Int,   TB_ALIGN_32 },
-    { X86::VFNMSUBPDr213rY_Int,   X86::VFNMSUBPDr213mY_Int,   TB_ALIGN_32 },
 
     { X86::VFMADDSUBPSr231r,      X86::VFMADDSUBPSr231m,      TB_ALIGN_16 },
     { X86::VFMADDSUBPDr231r,      X86::VFMADDSUBPDr231m,      TB_ALIGN_16 },
@@ -1240,10 +1272,6 @@ X86InstrInfo::X86InstrInfo(X86TargetMachine &tm)
     { X86::VFMADDSUBPDr132rY,     X86::VFMADDSUBPDr132mY,     TB_ALIGN_32 },
     { X86::VFMADDSUBPSr213rY,     X86::VFMADDSUBPSr213mY,     TB_ALIGN_32 },
     { X86::VFMADDSUBPDr213rY,     X86::VFMADDSUBPDr213mY,     TB_ALIGN_32 },
-    { X86::VFMADDSUBPSr213r_Int,  X86::VFMADDSUBPSr213m_Int,  TB_ALIGN_16 },
-    { X86::VFMADDSUBPDr213r_Int,  X86::VFMADDSUBPDr213m_Int,  TB_ALIGN_16 },
-    { X86::VFMADDSUBPSr213rY_Int, X86::VFMADDSUBPSr213mY_Int, TB_ALIGN_32 },
-    { X86::VFMADDSUBPDr213rY_Int, X86::VFMADDSUBPDr213mY_Int, TB_ALIGN_32 },
 
     { X86::VFMSUBADDPSr231r,      X86::VFMSUBADDPSr231m,      TB_ALIGN_16 },
     { X86::VFMSUBADDPDr231r,      X86::VFMSUBADDPDr231m,      TB_ALIGN_16 },
@@ -1257,10 +1285,40 @@ X86InstrInfo::X86InstrInfo(X86TargetMachine &tm)
     { X86::VFMSUBADDPDr132rY,     X86::VFMSUBADDPDr132mY,     TB_ALIGN_32 },
     { X86::VFMSUBADDPSr213rY,     X86::VFMSUBADDPSr213mY,     TB_ALIGN_32 },
     { X86::VFMSUBADDPDr213rY,     X86::VFMSUBADDPDr213mY,     TB_ALIGN_32 },
-    { X86::VFMSUBADDPSr213r_Int,  X86::VFMSUBADDPSr213m_Int,  TB_ALIGN_16 },
-    { X86::VFMSUBADDPDr213r_Int,  X86::VFMSUBADDPDr213m_Int,  TB_ALIGN_16 },
-    { X86::VFMSUBADDPSr213rY_Int, X86::VFMSUBADDPSr213mY_Int, TB_ALIGN_32 },
-    { X86::VFMSUBADDPDr213rY_Int, X86::VFMSUBADDPDr213mY_Int, TB_ALIGN_32 },
+
+    // FMA4 foldable patterns
+    { X86::VFMADDSS4rr,           X86::VFMADDSS4rm,           0           },
+    { X86::VFMADDSD4rr,           X86::VFMADDSD4rm,           0           },
+    { X86::VFMADDPS4rr,           X86::VFMADDPS4rm,           TB_ALIGN_16 },
+    { X86::VFMADDPD4rr,           X86::VFMADDPD4rm,           TB_ALIGN_16 },
+    { X86::VFMADDPS4rrY,          X86::VFMADDPS4rmY,          TB_ALIGN_32 },
+    { X86::VFMADDPD4rrY,          X86::VFMADDPD4rmY,          TB_ALIGN_32 },
+    { X86::VFNMADDSS4rr,          X86::VFNMADDSS4rm,          0           },
+    { X86::VFNMADDSD4rr,          X86::VFNMADDSD4rm,          0           },
+    { X86::VFNMADDPS4rr,          X86::VFNMADDPS4rm,          TB_ALIGN_16 },
+    { X86::VFNMADDPD4rr,          X86::VFNMADDPD4rm,          TB_ALIGN_16 },
+    { X86::VFNMADDPS4rrY,         X86::VFNMADDPS4rmY,         TB_ALIGN_32 },
+    { X86::VFNMADDPD4rrY,         X86::VFNMADDPD4rmY,         TB_ALIGN_32 },
+    { X86::VFMSUBSS4rr,           X86::VFMSUBSS4rm,           0           },
+    { X86::VFMSUBSD4rr,           X86::VFMSUBSD4rm,           0           },
+    { X86::VFMSUBPS4rr,           X86::VFMSUBPS4rm,           TB_ALIGN_16 },
+    { X86::VFMSUBPD4rr,           X86::VFMSUBPD4rm,           TB_ALIGN_16 },
+    { X86::VFMSUBPS4rrY,          X86::VFMSUBPS4rmY,          TB_ALIGN_32 },
+    { X86::VFMSUBPD4rrY,          X86::VFMSUBPD4rmY,          TB_ALIGN_32 },
+    { X86::VFNMSUBSS4rr,          X86::VFNMSUBSS4rm,          0           },
+    { X86::VFNMSUBSD4rr,          X86::VFNMSUBSD4rm,          0           },
+    { X86::VFNMSUBPS4rr,          X86::VFNMSUBPS4rm,          TB_ALIGN_16 },
+    { X86::VFNMSUBPD4rr,          X86::VFNMSUBPD4rm,          TB_ALIGN_16 },
+    { X86::VFNMSUBPS4rrY,         X86::VFNMSUBPS4rmY,         TB_ALIGN_32 },
+    { X86::VFNMSUBPD4rrY,         X86::VFNMSUBPD4rmY,         TB_ALIGN_32 },
+    { X86::VFMADDSUBPS4rr,        X86::VFMADDSUBPS4rm,        TB_ALIGN_16 },
+    { X86::VFMADDSUBPD4rr,        X86::VFMADDSUBPD4rm,        TB_ALIGN_16 },
+    { X86::VFMADDSUBPS4rrY,       X86::VFMADDSUBPS4rmY,       TB_ALIGN_32 },
+    { X86::VFMADDSUBPD4rrY,       X86::VFMADDSUBPD4rmY,       TB_ALIGN_32 },
+    { X86::VFMSUBADDPS4rr,        X86::VFMSUBADDPS4rm,        TB_ALIGN_16 },
+    { X86::VFMSUBADDPD4rr,        X86::VFMSUBADDPD4rm,        TB_ALIGN_16 },
+    { X86::VFMSUBADDPS4rrY,       X86::VFMSUBADDPS4rmY,       TB_ALIGN_32 },
+    { X86::VFMSUBADDPD4rrY,       X86::VFMSUBADDPD4rmY,       TB_ALIGN_32 },
   };
 
   for (unsigned i = 0, e = array_lengthof(OpTbl3); i != e; ++i) {
@@ -1318,8 +1376,7 @@ X86InstrInfo::isCoalescableExtInstr(const MachineInstr &MI,
     SrcReg = MI.getOperand(1).getReg();
     DstReg = MI.getOperand(0).getReg();
     switch (MI.getOpcode()) {
-    default:
-      llvm_unreachable(0);
+    default: llvm_unreachable("Unreachable!");
     case X86::MOVSX16rr8:
     case X86::MOVZX16rr8:
     case X86::MOVSX32rr8:
@@ -1483,69 +1540,69 @@ X86InstrInfo::isReallyTriviallyReMaterializable(const MachineInstr *MI,
                                                 AliasAnalysis *AA) const {
   switch (MI->getOpcode()) {
   default: break;
-    case X86::MOV8rm:
-    case X86::MOV16rm:
-    case X86::MOV32rm:
-    case X86::MOV64rm:
-    case X86::LD_Fp64m:
-    case X86::MOVSSrm:
-    case X86::MOVSDrm:
-    case X86::MOVAPSrm:
-    case X86::MOVUPSrm:
-    case X86::MOVAPDrm:
-    case X86::MOVDQArm:
-    case X86::VMOVSSrm:
-    case X86::VMOVSDrm:
-    case X86::VMOVAPSrm:
-    case X86::VMOVUPSrm:
-    case X86::VMOVAPDrm:
-    case X86::VMOVDQArm:
-    case X86::VMOVAPSYrm:
-    case X86::VMOVUPSYrm:
-    case X86::VMOVAPDYrm:
-    case X86::VMOVDQAYrm:
-    case X86::MMX_MOVD64rm:
-    case X86::MMX_MOVQ64rm:
-    case X86::FsVMOVAPSrm:
-    case X86::FsVMOVAPDrm:
-    case X86::FsMOVAPSrm:
-    case X86::FsMOVAPDrm: {
-      // Loads from constant pools are trivially rematerializable.
-      if (MI->getOperand(1).isReg() &&
-          MI->getOperand(2).isImm() &&
-          MI->getOperand(3).isReg() && MI->getOperand(3).getReg() == 0 &&
-          MI->isInvariantLoad(AA)) {
-        unsigned BaseReg = MI->getOperand(1).getReg();
-        if (BaseReg == 0 || BaseReg == X86::RIP)
-          return true;
-        // Allow re-materialization of PIC load.
-        if (!ReMatPICStubLoad && MI->getOperand(4).isGlobal())
-          return false;
-        const MachineFunction &MF = *MI->getParent()->getParent();
-        const MachineRegisterInfo &MRI = MF.getRegInfo();
-        return regIsPICBase(BaseReg, MRI);
-      }
-      return false;
+  case X86::MOV8rm:
+  case X86::MOV16rm:
+  case X86::MOV32rm:
+  case X86::MOV64rm:
+  case X86::LD_Fp64m:
+  case X86::MOVSSrm:
+  case X86::MOVSDrm:
+  case X86::MOVAPSrm:
+  case X86::MOVUPSrm:
+  case X86::MOVAPDrm:
+  case X86::MOVDQArm:
+  case X86::VMOVSSrm:
+  case X86::VMOVSDrm:
+  case X86::VMOVAPSrm:
+  case X86::VMOVUPSrm:
+  case X86::VMOVAPDrm:
+  case X86::VMOVDQArm:
+  case X86::VMOVAPSYrm:
+  case X86::VMOVUPSYrm:
+  case X86::VMOVAPDYrm:
+  case X86::VMOVDQAYrm:
+  case X86::MMX_MOVD64rm:
+  case X86::MMX_MOVQ64rm:
+  case X86::FsVMOVAPSrm:
+  case X86::FsVMOVAPDrm:
+  case X86::FsMOVAPSrm:
+  case X86::FsMOVAPDrm: {
+    // Loads from constant pools are trivially rematerializable.
+    if (MI->getOperand(1).isReg() &&
+        MI->getOperand(2).isImm() &&
+        MI->getOperand(3).isReg() && MI->getOperand(3).getReg() == 0 &&
+        MI->isInvariantLoad(AA)) {
+      unsigned BaseReg = MI->getOperand(1).getReg();
+      if (BaseReg == 0 || BaseReg == X86::RIP)
+        return true;
+      // Allow re-materialization of PIC load.
+      if (!ReMatPICStubLoad && MI->getOperand(4).isGlobal())
+        return false;
+      const MachineFunction &MF = *MI->getParent()->getParent();
+      const MachineRegisterInfo &MRI = MF.getRegInfo();
+      return regIsPICBase(BaseReg, MRI);
     }
+    return false;
+  }
 
-     case X86::LEA32r:
-     case X86::LEA64r: {
-       if (MI->getOperand(2).isImm() &&
-           MI->getOperand(3).isReg() && MI->getOperand(3).getReg() == 0 &&
-           !MI->getOperand(4).isReg()) {
-         // lea fi#, lea GV, etc. are all rematerializable.
-         if (!MI->getOperand(1).isReg())
-           return true;
-         unsigned BaseReg = MI->getOperand(1).getReg();
-         if (BaseReg == 0)
-           return true;
-         // Allow re-materialization of lea PICBase + x.
-         const MachineFunction &MF = *MI->getParent()->getParent();
-         const MachineRegisterInfo &MRI = MF.getRegInfo();
-         return regIsPICBase(BaseReg, MRI);
-       }
-       return false;
-     }
+  case X86::LEA32r:
+  case X86::LEA64r: {
+    if (MI->getOperand(2).isImm() &&
+        MI->getOperand(3).isReg() && MI->getOperand(3).getReg() == 0 &&
+        !MI->getOperand(4).isReg()) {
+      // lea fi#, lea GV, etc. are all rematerializable.
+      if (!MI->getOperand(1).isReg())
+        return true;
+      unsigned BaseReg = MI->getOperand(1).getReg();
+      if (BaseReg == 0)
+        return true;
+      // Allow re-materialization of lea PICBase + x.
+      const MachineFunction &MF = *MI->getParent()->getParent();
+      const MachineRegisterInfo &MRI = MF.getRegInfo();
+      return regIsPICBase(BaseReg, MRI);
+    }
+    return false;
+  }
   }
 
   // All other instructions marked M_REMATERIALIZABLE are always trivially
@@ -1654,7 +1711,7 @@ void X86InstrInfo::reMaterialize(MachineBasicBlock &MBB,
   case X86::MOV64r0: {
     if (!isSafeToClobberEFLAGS(MBB, I)) {
       switch (Opc) {
-      default: break;
+      default: llvm_unreachable("Unreachable!");
       case X86::MOV8r0:  Opc = X86::MOV8ri;  break;
       case X86::MOV16r0: Opc = X86::MOV16ri; break;
       case X86::MOV32r0: Opc = X86::MOV32ri; break;
@@ -1727,8 +1784,7 @@ X86InstrInfo::convertToThreeAddressWithLEA(unsigned MIOpc,
   MachineInstrBuilder MIB = BuildMI(*MFI, MBBI, MI->getDebugLoc(),
                                     get(Opc), leaOutReg);
   switch (MIOpc) {
-  default:
-    llvm_unreachable(0);
+  default: llvm_unreachable("Unreachable!");
   case X86::SHL16ri: {
     unsigned ShAmt = MI->getOperand(2).getImm();
     MIB.addReg(0).addImm(1 << ShAmt)
@@ -1812,10 +1868,8 @@ X86InstrInfo::convertToThreeAddress(MachineFunction::iterator &MFI,
   MachineInstr *MI = MBBI;
   MachineFunction &MF = *MI->getParent()->getParent();
   // All instructions input are two-addr instructions.  Get the known operands.
-  unsigned Dest = MI->getOperand(0).getReg();
-  unsigned Src = MI->getOperand(1).getReg();
-  bool isDead = MI->getOperand(0).isDead();
-  bool isKill = MI->getOperand(1).isKill();
+  const MachineOperand &Dest = MI->getOperand(0);
+  const MachineOperand &Src = MI->getOperand(1);
 
   MachineInstr *NewMI = NULL;
   // FIXME: 16-bit LEA's are really slow on Athlons, but not bad on P4's.  When
@@ -1833,11 +1887,9 @@ X86InstrInfo::convertToThreeAddress(MachineFunction::iterator &MFI,
     unsigned B = MI->getOperand(1).getReg();
     unsigned C = MI->getOperand(2).getReg();
     if (B != C) return 0;
-    unsigned A = MI->getOperand(0).getReg();
     unsigned M = MI->getOperand(3).getImm();
     NewMI = BuildMI(MF, MI->getDebugLoc(), get(X86::PSHUFDri))
-      .addReg(A, RegState::Define | getDeadRegState(isDead))
-      .addReg(B, getKillRegState(isKill)).addImm(M);
+      .addOperand(Dest).addOperand(Src).addImm(M);
     break;
   }
   case X86::SHUFPDrri: {
@@ -1847,15 +1899,13 @@ X86InstrInfo::convertToThreeAddress(MachineFunction::iterator &MFI,
     unsigned B = MI->getOperand(1).getReg();
     unsigned C = MI->getOperand(2).getReg();
     if (B != C) return 0;
-    unsigned A = MI->getOperand(0).getReg();
     unsigned M = MI->getOperand(3).getImm();
 
     // Convert to PSHUFD mask.
     M = ((M & 1) << 1) | ((M & 1) << 3) | ((M & 2) << 4) | ((M & 2) << 6)| 0x44;
 
     NewMI = BuildMI(MF, MI->getDebugLoc(), get(X86::PSHUFDri))
-      .addReg(A, RegState::Define | getDeadRegState(isDead))
-      .addReg(B, getKillRegState(isKill)).addImm(M);
+      .addOperand(Dest).addOperand(Src).addImm(M);
     break;
   }
   case X86::SHL64ri: {
@@ -1866,15 +1916,14 @@ X86InstrInfo::convertToThreeAddress(MachineFunction::iterator &MFI,
     if (ShAmt == 0 || ShAmt >= 4) return 0;
 
     // LEA can't handle RSP.
-    if (TargetRegisterInfo::isVirtualRegister(Src) &&
-        !MF.getRegInfo().constrainRegClass(Src, &X86::GR64_NOSPRegClass))
+    if (TargetRegisterInfo::isVirtualRegister(Src.getReg()) &&
+        !MF.getRegInfo().constrainRegClass(Src.getReg(),
+                                           &X86::GR64_NOSPRegClass))
       return 0;
 
     NewMI = BuildMI(MF, MI->getDebugLoc(), get(X86::LEA64r))
-      .addReg(Dest, RegState::Define | getDeadRegState(isDead))
-      .addReg(0).addImm(1 << ShAmt)
-      .addReg(Src, getKillRegState(isKill))
-      .addImm(0).addReg(0);
+      .addOperand(Dest)
+      .addReg(0).addImm(1 << ShAmt).addOperand(Src).addImm(0).addReg(0);
     break;
   }
   case X86::SHL32ri: {
@@ -1885,15 +1934,15 @@ X86InstrInfo::convertToThreeAddress(MachineFunction::iterator &MFI,
     if (ShAmt == 0 || ShAmt >= 4) return 0;
 
     // LEA can't handle ESP.
-    if (TargetRegisterInfo::isVirtualRegister(Src) &&
-        !MF.getRegInfo().constrainRegClass(Src, &X86::GR32_NOSPRegClass))
+    if (TargetRegisterInfo::isVirtualRegister(Src.getReg()) &&
+        !MF.getRegInfo().constrainRegClass(Src.getReg(),
+                                           &X86::GR32_NOSPRegClass))
       return 0;
 
     unsigned Opc = is64Bit ? X86::LEA64_32r : X86::LEA32r;
     NewMI = BuildMI(MF, MI->getDebugLoc(), get(Opc))
-      .addReg(Dest, RegState::Define | getDeadRegState(isDead))
-      .addReg(0).addImm(1 << ShAmt)
-      .addReg(Src, getKillRegState(isKill)).addImm(0).addReg(0);
+      .addOperand(Dest)
+      .addReg(0).addImm(1 << ShAmt).addOperand(Src).addImm(0).addReg(0);
     break;
   }
   case X86::SHL16ri: {
@@ -1906,10 +1955,8 @@ X86InstrInfo::convertToThreeAddress(MachineFunction::iterator &MFI,
     if (DisableLEA16)
       return is64Bit ? convertToThreeAddressWithLEA(MIOpc, MFI, MBBI, LV) : 0;
     NewMI = BuildMI(MF, MI->getDebugLoc(), get(X86::LEA16r))
-      .addReg(Dest, RegState::Define | getDeadRegState(isDead))
-      .addReg(0).addImm(1 << ShAmt)
-      .addReg(Src, getKillRegState(isKill))
-      .addImm(0).addReg(0);
+      .addOperand(Dest)
+      .addReg(0).addImm(1 << ShAmt).addOperand(Src).addImm(0).addReg(0);
     break;
   }
   default: {
@@ -1932,14 +1979,12 @@ X86InstrInfo::convertToThreeAddress(MachineFunction::iterator &MFI,
         (const TargetRegisterClass*)&X86::GR32_NOSPRegClass;
 
       // LEA can't handle RSP.
-      if (TargetRegisterInfo::isVirtualRegister(Src) &&
-          !MF.getRegInfo().constrainRegClass(Src, RC))
+      if (TargetRegisterInfo::isVirtualRegister(Src.getReg()) &&
+          !MF.getRegInfo().constrainRegClass(Src.getReg(), RC))
         return 0;
 
-      NewMI = addRegOffset(BuildMI(MF, MI->getDebugLoc(), get(Opc))
-                              .addReg(Dest, RegState::Define |
-                                      getDeadRegState(isDead)),
-                              Src, isKill, 1);
+      NewMI = addOffset(BuildMI(MF, MI->getDebugLoc(), get(Opc))
+                        .addOperand(Dest).addOperand(Src), 1);
       break;
     }
     case X86::INC16r:
@@ -1947,10 +1992,8 @@ X86InstrInfo::convertToThreeAddress(MachineFunction::iterator &MFI,
       if (DisableLEA16)
         return is64Bit ? convertToThreeAddressWithLEA(MIOpc, MFI, MBBI, LV) : 0;
       assert(MI->getNumOperands() >= 2 && "Unknown inc instruction!");
-      NewMI = addRegOffset(BuildMI(MF, MI->getDebugLoc(), get(X86::LEA16r))
-                           .addReg(Dest, RegState::Define |
-                                   getDeadRegState(isDead)),
-                           Src, isKill, 1);
+      NewMI = addOffset(BuildMI(MF, MI->getDebugLoc(), get(X86::LEA16r))
+                        .addOperand(Dest).addOperand(Src), 1);
       break;
     case X86::DEC64r:
     case X86::DEC32r:
@@ -1962,14 +2005,12 @@ X86InstrInfo::convertToThreeAddress(MachineFunction::iterator &MFI,
         (const TargetRegisterClass*)&X86::GR64_NOSPRegClass :
         (const TargetRegisterClass*)&X86::GR32_NOSPRegClass;
       // LEA can't handle RSP.
-      if (TargetRegisterInfo::isVirtualRegister(Src) &&
-          !MF.getRegInfo().constrainRegClass(Src, RC))
+      if (TargetRegisterInfo::isVirtualRegister(Src.getReg()) &&
+          !MF.getRegInfo().constrainRegClass(Src.getReg(), RC))
         return 0;
 
-      NewMI = addRegOffset(BuildMI(MF, MI->getDebugLoc(), get(Opc))
-                              .addReg(Dest, RegState::Define |
-                                      getDeadRegState(isDead)),
-                              Src, isKill, -1);
+      NewMI = addOffset(BuildMI(MF, MI->getDebugLoc(), get(Opc))
+                        .addOperand(Dest).addOperand(Src), -1);
       break;
     }
     case X86::DEC16r:
@@ -1977,10 +2018,8 @@ X86InstrInfo::convertToThreeAddress(MachineFunction::iterator &MFI,
       if (DisableLEA16)
         return is64Bit ? convertToThreeAddressWithLEA(MIOpc, MFI, MBBI, LV) : 0;
       assert(MI->getNumOperands() >= 2 && "Unknown dec instruction!");
-      NewMI = addRegOffset(BuildMI(MF, MI->getDebugLoc(), get(X86::LEA16r))
-                           .addReg(Dest, RegState::Define |
-                                   getDeadRegState(isDead)),
-                           Src, isKill, -1);
+      NewMI = addOffset(BuildMI(MF, MI->getDebugLoc(), get(X86::LEA16r))
+                        .addOperand(Dest).addOperand(Src), -1);
       break;
     case X86::ADD64rr:
     case X86::ADD64rr_DB:
@@ -2007,9 +2046,8 @@ X86InstrInfo::convertToThreeAddress(MachineFunction::iterator &MFI,
         return 0;
 
       NewMI = addRegReg(BuildMI(MF, MI->getDebugLoc(), get(Opc))
-                        .addReg(Dest, RegState::Define |
-                                getDeadRegState(isDead)),
-                        Src, isKill, Src2, isKill2);
+                        .addOperand(Dest),
+                        Src.getReg(), Src.isKill(), Src2, isKill2);
 
       // Preserve undefness of the operands.
       bool isUndef = MI->getOperand(1).isUndef();
@@ -2029,9 +2067,15 @@ X86InstrInfo::convertToThreeAddress(MachineFunction::iterator &MFI,
       unsigned Src2 = MI->getOperand(2).getReg();
       bool isKill2 = MI->getOperand(2).isKill();
       NewMI = addRegReg(BuildMI(MF, MI->getDebugLoc(), get(X86::LEA16r))
-                        .addReg(Dest, RegState::Define |
-                                getDeadRegState(isDead)),
-                        Src, isKill, Src2, isKill2);
+                        .addOperand(Dest),
+                        Src.getReg(), Src.isKill(), Src2, isKill2);
+
+      // Preserve undefness of the operands.
+      bool isUndef = MI->getOperand(1).isUndef();
+      bool isUndef2 = MI->getOperand(2).isUndef();
+      NewMI->getOperand(1).setIsUndef(isUndef);
+      NewMI->getOperand(3).setIsUndef(isUndef2);
+
       if (LV && isKill2)
         LV->replaceKillInstruction(Src2, MI, NewMI);
       break;
@@ -2041,10 +2085,9 @@ X86InstrInfo::convertToThreeAddress(MachineFunction::iterator &MFI,
     case X86::ADD64ri32_DB:
     case X86::ADD64ri8_DB:
       assert(MI->getNumOperands() >= 3 && "Unknown add instruction!");
-      NewMI = addRegOffset(BuildMI(MF, MI->getDebugLoc(), get(X86::LEA64r))
-                              .addReg(Dest, RegState::Define |
-                                      getDeadRegState(isDead)),
-                              Src, isKill, MI->getOperand(2).getImm());
+      NewMI = addOffset(BuildMI(MF, MI->getDebugLoc(), get(X86::LEA64r))
+                        .addOperand(Dest).addOperand(Src),
+                        MI->getOperand(2).getImm());
       break;
     case X86::ADD32ri:
     case X86::ADD32ri8:
@@ -2052,10 +2095,9 @@ X86InstrInfo::convertToThreeAddress(MachineFunction::iterator &MFI,
     case X86::ADD32ri8_DB: {
       assert(MI->getNumOperands() >= 3 && "Unknown add instruction!");
       unsigned Opc = is64Bit ? X86::LEA64_32r : X86::LEA32r;
-      NewMI = addRegOffset(BuildMI(MF, MI->getDebugLoc(), get(Opc))
-                              .addReg(Dest, RegState::Define |
-                                      getDeadRegState(isDead)),
-                                Src, isKill, MI->getOperand(2).getImm());
+      NewMI = addOffset(BuildMI(MF, MI->getDebugLoc(), get(Opc))
+                        .addOperand(Dest).addOperand(Src),
+                        MI->getOperand(2).getImm());
       break;
     }
     case X86::ADD16ri:
@@ -2065,10 +2107,9 @@ X86InstrInfo::convertToThreeAddress(MachineFunction::iterator &MFI,
       if (DisableLEA16)
         return is64Bit ? convertToThreeAddressWithLEA(MIOpc, MFI, MBBI, LV) : 0;
       assert(MI->getNumOperands() >= 3 && "Unknown add instruction!");
-      NewMI = addRegOffset(BuildMI(MF, MI->getDebugLoc(), get(X86::LEA16r))
-                              .addReg(Dest, RegState::Define |
-                                      getDeadRegState(isDead)),
-                              Src, isKill, MI->getOperand(2).getImm());
+      NewMI = addOffset(BuildMI(MF, MI->getDebugLoc(), get(X86::LEA16r))
+                        .addOperand(Dest).addOperand(Src),
+                        MI->getOperand(2).getImm());
       break;
     }
   }
@@ -2077,10 +2118,10 @@ X86InstrInfo::convertToThreeAddress(MachineFunction::iterator &MFI,
   if (!NewMI) return 0;
 
   if (LV) {  // Update live variables
-    if (isKill)
-      LV->replaceKillInstruction(Src, MI, NewMI);
-    if (isDead)
-      LV->replaceKillInstruction(Dest, MI, NewMI);
+    if (Src.isKill())
+      LV->replaceKillInstruction(Src.getReg(), MI, NewMI);
+    if (Dest.isDead())
+      LV->replaceKillInstruction(Dest.getReg(), MI, NewMI);
   }
 
   MFI->insert(MBBI, NewMI);          // Insert the new inst
@@ -2120,57 +2161,25 @@ X86InstrInfo::commuteInstruction(MachineInstr *MI, bool NewMI) const {
     MI->getOperand(3).setImm(Size-Amt);
     return TargetInstrInfoImpl::commuteInstruction(MI, NewMI);
   }
-  case X86::CMOVB16rr:
-  case X86::CMOVB32rr:
-  case X86::CMOVB64rr:
-  case X86::CMOVAE16rr:
-  case X86::CMOVAE32rr:
-  case X86::CMOVAE64rr:
-  case X86::CMOVE16rr:
-  case X86::CMOVE32rr:
-  case X86::CMOVE64rr:
-  case X86::CMOVNE16rr:
-  case X86::CMOVNE32rr:
-  case X86::CMOVNE64rr:
-  case X86::CMOVBE16rr:
-  case X86::CMOVBE32rr:
-  case X86::CMOVBE64rr:
-  case X86::CMOVA16rr:
-  case X86::CMOVA32rr:
-  case X86::CMOVA64rr:
-  case X86::CMOVL16rr:
-  case X86::CMOVL32rr:
-  case X86::CMOVL64rr:
-  case X86::CMOVGE16rr:
-  case X86::CMOVGE32rr:
-  case X86::CMOVGE64rr:
-  case X86::CMOVLE16rr:
-  case X86::CMOVLE32rr:
-  case X86::CMOVLE64rr:
-  case X86::CMOVG16rr:
-  case X86::CMOVG32rr:
-  case X86::CMOVG64rr:
-  case X86::CMOVS16rr:
-  case X86::CMOVS32rr:
-  case X86::CMOVS64rr:
-  case X86::CMOVNS16rr:
-  case X86::CMOVNS32rr:
-  case X86::CMOVNS64rr:
-  case X86::CMOVP16rr:
-  case X86::CMOVP32rr:
-  case X86::CMOVP64rr:
-  case X86::CMOVNP16rr:
-  case X86::CMOVNP32rr:
-  case X86::CMOVNP64rr:
-  case X86::CMOVO16rr:
-  case X86::CMOVO32rr:
-  case X86::CMOVO64rr:
-  case X86::CMOVNO16rr:
-  case X86::CMOVNO32rr:
-  case X86::CMOVNO64rr: {
-    unsigned Opc = 0;
+  case X86::CMOVB16rr:  case X86::CMOVB32rr:  case X86::CMOVB64rr:
+  case X86::CMOVAE16rr: case X86::CMOVAE32rr: case X86::CMOVAE64rr:
+  case X86::CMOVE16rr:  case X86::CMOVE32rr:  case X86::CMOVE64rr:
+  case X86::CMOVNE16rr: case X86::CMOVNE32rr: case X86::CMOVNE64rr:
+  case X86::CMOVBE16rr: case X86::CMOVBE32rr: case X86::CMOVBE64rr:
+  case X86::CMOVA16rr:  case X86::CMOVA32rr:  case X86::CMOVA64rr:
+  case X86::CMOVL16rr:  case X86::CMOVL32rr:  case X86::CMOVL64rr:
+  case X86::CMOVGE16rr: case X86::CMOVGE32rr: case X86::CMOVGE64rr:
+  case X86::CMOVLE16rr: case X86::CMOVLE32rr: case X86::CMOVLE64rr:
+  case X86::CMOVG16rr:  case X86::CMOVG32rr:  case X86::CMOVG64rr:
+  case X86::CMOVS16rr:  case X86::CMOVS32rr:  case X86::CMOVS64rr:
+  case X86::CMOVNS16rr: case X86::CMOVNS32rr: case X86::CMOVNS64rr:
+  case X86::CMOVP16rr:  case X86::CMOVP32rr:  case X86::CMOVP64rr:
+  case X86::CMOVNP16rr: case X86::CMOVNP32rr: case X86::CMOVNP64rr:
+  case X86::CMOVO16rr:  case X86::CMOVO32rr:  case X86::CMOVO64rr:
+  case X86::CMOVNO16rr: case X86::CMOVNO32rr: case X86::CMOVNO64rr: {
+    unsigned Opc;
     switch (MI->getOpcode()) {
-    default: break;
+    default: llvm_unreachable("Unreachable!");
     case X86::CMOVB16rr:  Opc = X86::CMOVAE16rr; break;
     case X86::CMOVB32rr:  Opc = X86::CMOVAE32rr; break;
     case X86::CMOVB64rr:  Opc = X86::CMOVAE64rr; break;
@@ -2279,7 +2288,7 @@ static X86::CondCode getCondFromSETOpc(unsigned Opc) {
 }
 
 /// getCondFromCmovOpc - return condition code of a CMov opcode.
-static X86::CondCode getCondFromCMovOpc(unsigned Opc) {
+X86::CondCode X86::getCondFromCMovOpc(unsigned Opc) {
   switch (Opc) {
   default: return X86::COND_INVALID;
   case X86::CMOVA16rm:  case X86::CMOVA16rr:  case X86::CMOVA32rm:
@@ -2402,7 +2411,7 @@ static X86::CondCode getSwappedCondition(X86::CondCode CC) {
 /// whether it has memory operand.
 static unsigned getSETFromCond(X86::CondCode CC,
                                bool HasMemoryOperand) {
-  static const unsigned Opc[16][2] = {
+  static const uint16_t Opc[16][2] = {
     { X86::SETAr,  X86::SETAm  },
     { X86::SETAEr, X86::SETAEm },
     { X86::SETBr,  X86::SETBm  },
@@ -2429,7 +2438,7 @@ static unsigned getSETFromCond(X86::CondCode CC,
 /// register size in bytes, and operand type.
 static unsigned getCMovFromCond(X86::CondCode CC, unsigned RegBytes,
                                 bool HasMemoryOperand) {
-  static const unsigned Opc[32][3] = {
+  static const uint16_t Opc[32][3] = {
     { X86::CMOVA16rr,  X86::CMOVA32rr,  X86::CMOVA64rr  },
     { X86::CMOVAE16rr, X86::CMOVAE32rr, X86::CMOVAE64rr },
     { X86::CMOVB16rr,  X86::CMOVB32rr,  X86::CMOVB64rr  },
@@ -2762,19 +2771,18 @@ static unsigned CopyToFromAsymmetricReg(unsigned DestReg, unsigned SrcReg,
   // SrcReg(GR64)  -> DestReg(VR64)
 
   if (X86::GR64RegClass.contains(DestReg)) {
-    if (X86::VR128RegClass.contains(SrcReg)) {
+    if (X86::VR128RegClass.contains(SrcReg))
       // Copy from a VR128 register to a GR64 register.
       return HasAVX ? X86::VMOVPQIto64rr : X86::MOVPQIto64rr;
-    } else if (X86::VR64RegClass.contains(SrcReg)) {
+    if (X86::VR64RegClass.contains(SrcReg))
       // Copy from a VR64 register to a GR64 register.
       return X86::MOVSDto64rr;
-    }
   } else if (X86::GR64RegClass.contains(SrcReg)) {
     // Copy from a GR64 register to a VR128 register.
     if (X86::VR128RegClass.contains(DestReg))
       return HasAVX ? X86::VMOV64toPQIrr : X86::MOV64toPQIrr;
     // Copy from a GR64 register to a VR64 register.
-    else if (X86::VR64RegClass.contains(DestReg))
+    if (X86::VR64RegClass.contains(DestReg))
       return X86::MOV64toSDrr;
   }
 
@@ -2782,12 +2790,12 @@ static unsigned CopyToFromAsymmetricReg(unsigned DestReg, unsigned SrcReg,
   // SrcReg(GR32) -> DestReg(FR32)
 
   if (X86::GR32RegClass.contains(DestReg) && X86::FR32RegClass.contains(SrcReg))
-      // Copy from a FR32 register to a GR32 register.
-      return HasAVX ? X86::VMOVSS2DIrr : X86::MOVSS2DIrr;
+    // Copy from a FR32 register to a GR32 register.
+    return HasAVX ? X86::VMOVSS2DIrr : X86::MOVSS2DIrr;
 
   if (X86::FR32RegClass.contains(DestReg) && X86::GR32RegClass.contains(SrcReg))
-      // Copy from a GR32 register to a FR32 register.
-      return HasAVX ? X86::VMOVDI2SSrr : X86::MOVDI2SSrr;
+    // Copy from a GR32 register to a FR32 register.
+    return HasAVX ? X86::VMOVDI2SSrr : X86::MOVDI2SSrr;
 
   return 0;
 }
@@ -2798,7 +2806,7 @@ void X86InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
                                bool KillSrc) const {
   // First deal with the normal symmetric copies.
   bool HasAVX = TM.getSubtarget<X86Subtarget>().hasAVX();
-  unsigned Opc = 0;
+  unsigned Opc;
   if (X86::GR64RegClass.contains(DestReg, SrcReg))
     Opc = X86::MOV64rr;
   else if (X86::GR32RegClass.contains(DestReg, SrcReg))
@@ -2837,7 +2845,8 @@ void X86InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
       BuildMI(MBB, MI, DL, get(X86::PUSHF64));
       BuildMI(MBB, MI, DL, get(X86::POP64r), DestReg);
       return;
-    } else if (X86::GR32RegClass.contains(DestReg)) {
+    }
+    if (X86::GR32RegClass.contains(DestReg)) {
       BuildMI(MBB, MI, DL, get(X86::PUSHF32));
       BuildMI(MBB, MI, DL, get(X86::POP32r), DestReg);
       return;
@@ -2849,7 +2858,8 @@ void X86InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
         .addReg(SrcReg, getKillRegState(KillSrc));
       BuildMI(MBB, MI, DL, get(X86::POPF64));
       return;
-    } else if (X86::GR32RegClass.contains(SrcReg)) {
+    }
+    if (X86::GR32RegClass.contains(SrcReg)) {
       BuildMI(MBB, MI, DL, get(X86::PUSH32r))
         .addReg(SrcReg, getKillRegState(KillSrc));
       BuildMI(MBB, MI, DL, get(X86::POPF32));
@@ -3139,11 +3149,19 @@ inline static bool isDefConvertible(MachineInstr *MI) {
   case X86::SUB8ri:    case X86::SUB64rr:  case X86::SUB32rr:
   case X86::SUB16rr:   case X86::SUB8rr:   case X86::SUB64rm:
   case X86::SUB32rm:   case X86::SUB16rm:  case X86::SUB8rm:
+  case X86::DEC64r:  case X86::DEC32r:  case X86::DEC16r: case X86::DEC8r:
+  case X86::DEC64m:  case X86::DEC32m:  case X86::DEC16m: case X86::DEC8m:
+  case X86::DEC64_32r: case X86::DEC64_16r:
+  case X86::DEC64_32m: case X86::DEC64_16m:
   case X86::ADD64ri32: case X86::ADD64ri8: case X86::ADD32ri:
   case X86::ADD32ri8:  case X86::ADD16ri:  case X86::ADD16ri8:
   case X86::ADD8ri:    case X86::ADD64rr:  case X86::ADD32rr:
   case X86::ADD16rr:   case X86::ADD8rr:   case X86::ADD64rm:
   case X86::ADD32rm:   case X86::ADD16rm:  case X86::ADD8rm:
+  case X86::INC64r:  case X86::INC32r:  case X86::INC16r: case X86::INC8r:
+  case X86::INC64m:  case X86::INC32m:  case X86::INC16m: case X86::INC8m:
+  case X86::INC64_32r: case X86::INC64_16r:
+  case X86::INC64_32m: case X86::INC64_16m:
   case X86::AND64ri32: case X86::AND64ri8: case X86::AND32ri:
   case X86::AND32ri8:  case X86::AND16ri:  case X86::AND16ri8:
   case X86::AND8ri:    case X86::AND64rr:  case X86::AND32rr:
@@ -3193,7 +3211,7 @@ optimizeCompareInstr(MachineInstr *CmpInstr, unsigned SrcReg, unsigned SrcReg2,
       return false;
     // There is no use of the destination register, we can replace SUB with CMP.
     switch (CmpInstr->getOpcode()) {
-    default: llvm_unreachable(0);
+    default: llvm_unreachable("Unreachable!");
     case X86::SUB64rm:   NewOpcode = X86::CMP64rm;   break;
     case X86::SUB32rm:   NewOpcode = X86::CMP32rm;   break;
     case X86::SUB16rm:   NewOpcode = X86::CMP16rm;   break;
@@ -3318,7 +3336,7 @@ optimizeCompareInstr(MachineInstr *CmpInstr, unsigned SrcReg, unsigned SrcReg2,
         if (OldCC != X86::COND_INVALID)
           OpcIsSET = true;
         else
-          OldCC = getCondFromCMovOpc(Instr.getOpcode());
+          OldCC = X86::getCondFromCMovOpc(Instr.getOpcode());
       }
       if (OldCC == X86::COND_INVALID) return false;
     }
@@ -3383,12 +3401,14 @@ optimizeCompareInstr(MachineInstr *CmpInstr, unsigned SrcReg, unsigned SrcReg2,
     Sub->getParent()->insert(MachineBasicBlock::iterator(Sub), Movr0Inst);
   }
 
-  // Make sure Sub instruction defines EFLAGS.
+  // Make sure Sub instruction defines EFLAGS and mark the def live.
+  unsigned LastOperand = Sub->getNumOperands() - 1;
   assert(Sub->getNumOperands() >= 2 &&
-         Sub->getOperand(Sub->getNumOperands()-1).isReg() &&
-         Sub->getOperand(Sub->getNumOperands()-1).getReg() == X86::EFLAGS &&
+         Sub->getOperand(LastOperand).isReg() &&
+         Sub->getOperand(LastOperand).getReg() == X86::EFLAGS &&
          "EFLAGS should be the last operand of SUB, ADD, OR, XOR, AND");
-  Sub->getOperand(Sub->getNumOperands()-1).setIsDef(true);
+  Sub->getOperand(LastOperand).setIsDef(true);
+  Sub->getOperand(LastOperand).setIsDead(false);
   CmpInstr->eraseFromParent();
 
   // Modify the condition code of instructions in OpsToUpdate.
@@ -3497,10 +3517,25 @@ static bool Expand2AddrUndef(MachineInstr *MI, const MCInstrDesc &Desc) {
 bool X86InstrInfo::expandPostRAPseudo(MachineBasicBlock::iterator MI) const {
   bool HasAVX = TM.getSubtarget<X86Subtarget>().hasAVX();
   switch (MI->getOpcode()) {
+  case X86::SETB_C8r:
+    return Expand2AddrUndef(MI, get(X86::SBB8rr));
+  case X86::SETB_C16r:
+    return Expand2AddrUndef(MI, get(X86::SBB16rr));
+  case X86::SETB_C32r:
+    return Expand2AddrUndef(MI, get(X86::SBB32rr));
+  case X86::SETB_C64r:
+    return Expand2AddrUndef(MI, get(X86::SBB64rr));
   case X86::V_SET0:
   case X86::FsFLD0SS:
   case X86::FsFLD0SD:
     return Expand2AddrUndef(MI, get(HasAVX ? X86::VXORPSrr : X86::XORPSrr));
+  case X86::AVX_SET0:
+    assert(HasAVX && "AVX not supported");
+    return Expand2AddrUndef(MI, get(X86::VXORPSYrr));
+  case X86::V_SETALLONES:
+    return Expand2AddrUndef(MI, get(HasAVX ? X86::VPCMPEQDrr : X86::PCMPEQDrr));
+  case X86::AVX2_SETALLONES:
+    return Expand2AddrUndef(MI, get(X86::VPCMPEQDYrr));
   case X86::TEST8ri_NOREX:
     MI->setDesc(get(X86::TEST8ri));
     return true;
@@ -3614,14 +3649,16 @@ X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF,
     OpcodeTablePtr = &RegOp2MemOpTable2Addr;
     isTwoAddrFold = true;
   } else if (i == 0) { // If operand 0
-    if (MI->getOpcode() == X86::MOV64r0)
-      NewMI = MakeM0Inst(*this, X86::MOV64mi32, MOs, MI);
-    else if (MI->getOpcode() == X86::MOV32r0)
-      NewMI = MakeM0Inst(*this, X86::MOV32mi, MOs, MI);
-    else if (MI->getOpcode() == X86::MOV16r0)
-      NewMI = MakeM0Inst(*this, X86::MOV16mi, MOs, MI);
-    else if (MI->getOpcode() == X86::MOV8r0)
-      NewMI = MakeM0Inst(*this, X86::MOV8mi, MOs, MI);
+    unsigned Opc = 0;
+    switch (MI->getOpcode()) {
+    default: break;
+    case X86::MOV64r0: Opc = X86::MOV64mi32; break;
+    case X86::MOV32r0: Opc = X86::MOV32mi;   break;
+    case X86::MOV16r0: Opc = X86::MOV16mi;   break;
+    case X86::MOV8r0:  Opc = X86::MOV8mi;    break;
+    }
+    if (Opc)
+       NewMI = MakeM0Inst(*this, Opc, MOs, MI);
     if (NewMI)
       return NewMI;
 
@@ -3799,7 +3836,8 @@ MachineInstr* X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF,
 
   // Unless optimizing for size, don't fold to avoid partial
   // register update stalls
-  if (!MF.getFunction()->hasFnAttr(Attribute::OptimizeForSize) &&
+  if (!MF.getFunction()->getFnAttributes().
+        hasAttribute(Attributes::OptimizeForSize) &&
       hasPartialRegUpdate(MI->getOpcode()))
     return 0;
 
@@ -3840,7 +3878,8 @@ MachineInstr* X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF,
 
   // Unless optimizing for size, don't fold to avoid partial
   // register update stalls
-  if (!MF.getFunction()->hasFnAttr(Attribute::OptimizeForSize) &&
+  if (!MF.getFunction()->getFnAttributes().
+        hasAttribute(Attributes::OptimizeForSize) &&
       hasPartialRegUpdate(MI->getOpcode()))
     return 0;
 
@@ -3850,15 +3889,12 @@ MachineInstr* X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF,
     Alignment = (*LoadMI->memoperands_begin())->getAlignment();
   else
     switch (LoadMI->getOpcode()) {
-    case X86::AVX_SET0PSY:
-    case X86::AVX_SET0PDY:
     case X86::AVX2_SETALLONES:
-    case X86::AVX2_SET0:
+    case X86::AVX_SET0:
       Alignment = 32;
       break;
     case X86::V_SET0:
     case X86::V_SETALLONES:
-    case X86::AVX_SETALLONES:
       Alignment = 16;
       break;
     case X86::FsFLD0SD:
@@ -3894,11 +3930,8 @@ MachineInstr* X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF,
   switch (LoadMI->getOpcode()) {
   case X86::V_SET0:
   case X86::V_SETALLONES:
-  case X86::AVX_SET0PSY:
-  case X86::AVX_SET0PDY:
-  case X86::AVX_SETALLONES:
   case X86::AVX2_SETALLONES:
-  case X86::AVX2_SET0:
+  case X86::AVX_SET0:
   case X86::FsFLD0SD:
   case X86::FsFLD0SS: {
     // Folding a V_SET0 or V_SETALLONES as a load, to ease register pressure.
@@ -3930,15 +3963,12 @@ MachineInstr* X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF,
       Ty = Type::getFloatTy(MF.getFunction()->getContext());
     else if (Opc == X86::FsFLD0SD)
       Ty = Type::getDoubleTy(MF.getFunction()->getContext());
-    else if (Opc == X86::AVX_SET0PSY || Opc == X86::AVX_SET0PDY)
-      Ty = VectorType::get(Type::getFloatTy(MF.getFunction()->getContext()), 8);
-    else if (Opc == X86::AVX2_SETALLONES || Opc == X86::AVX2_SET0)
+    else if (Opc == X86::AVX2_SETALLONES || Opc == X86::AVX_SET0)
       Ty = VectorType::get(Type::getInt32Ty(MF.getFunction()->getContext()), 8);
     else
       Ty = VectorType::get(Type::getInt32Ty(MF.getFunction()->getContext()), 4);
 
-    bool IsAllOnes = (Opc == X86::V_SETALLONES || Opc == X86::AVX_SETALLONES ||
-                      Opc == X86::AVX2_SETALLONES);
+    bool IsAllOnes = (Opc == X86::V_SETALLONES || Opc == X86::AVX2_SETALLONES);
     const Constant *C = IsAllOnes ? Constant::getAllOnesValue(Ty) :
                                     Constant::getNullValue(Ty);
     unsigned CPI = MCP.getConstantPoolIndex(C, Alignment);
@@ -4013,6 +4043,8 @@ bool X86InstrInfo::canFoldMemoryOperand(const MachineInstr *MI,
     OpcodeTablePtr = &RegOp2MemOpTable1;
   } else if (OpNum == 2) {
     OpcodeTablePtr = &RegOp2MemOpTable2;
+  } else if (OpNum == 3) {
+    OpcodeTablePtr = &RegOp2MemOpTable3;
   }
 
   if (OpcodeTablePtr && OpcodeTablePtr->count(Opc))
@@ -4102,7 +4134,6 @@ bool X86InstrInfo::unfoldMemoryOperand(MachineFunction &MF, MachineInstr *MI,
                getUndefRegState(MO.isUndef()));
   }
   // Change CMP32ri r, 0 back to TEST32rr r, r, etc.
-  unsigned NewOpc = 0;
   switch (DataMI->getOpcode()) {
   default: break;
   case X86::CMP64ri32:
@@ -4115,8 +4146,9 @@ bool X86InstrInfo::unfoldMemoryOperand(MachineFunction &MF, MachineInstr *MI,
     MachineOperand &MO0 = DataMI->getOperand(0);
     MachineOperand &MO1 = DataMI->getOperand(1);
     if (MO1.getImm() == 0) {
+      unsigned NewOpc;
       switch (DataMI->getOpcode()) {
-      default: break;
+      default: llvm_unreachable("Unreachable!");
       case X86::CMP64ri8:
       case X86::CMP64ri32: NewOpc = X86::TEST64rr; break;
       case X86::CMP32ri8:
diff --git a/lib/Target/X86/X86InstrInfo.h b/lib/Target/X86/X86InstrInfo.h
index b6f69af..260f054 100644
--- a/lib/Target/X86/X86InstrInfo.h
+++ b/lib/Target/X86/X86InstrInfo.h
@@ -61,6 +61,9 @@ namespace X86 {
   // Turn condition code into conditional branch opcode.
   unsigned GetCondBranchFromCond(CondCode CC);
 
+  // Turn CMov opcode into condition code.
+  CondCode getCondFromCMovOpc(unsigned Opc);
+
   /// GetOppositeBranchCondition - Return the inverse of the specified cond,
   /// e.g. turning COND_E to COND_NE.
   CondCode GetOppositeBranchCondition(X86::CondCode CC);
diff --git a/lib/Target/X86/X86InstrInfo.td b/lib/Target/X86/X86InstrInfo.td
index d293156..650fa95 100644
--- a/lib/Target/X86/X86InstrInfo.td
+++ b/lib/Target/X86/X86InstrInfo.td
@@ -114,7 +114,7 @@ def SDT_X86TCRET : SDTypeProfile<0, 2, [SDTCisPtrTy<0>, SDTCisVT<1, i32>]>;
 def SDT_X86MEMBARRIER : SDTypeProfile<0, 0, []>;
 
 def X86MemBarrier : SDNode<"X86ISD::MEMBARRIER", SDT_X86MEMBARRIER,
-                            [SDNPHasChain]>;
+                            [SDNPHasChain,SDNPSideEffect]>;
 def X86MFence : SDNode<"X86ISD::MFENCE", SDT_X86MEMBARRIER,
                         [SDNPHasChain]>;
 def X86SFence : SDNode<"X86ISD::SFENCE", SDT_X86MEMBARRIER,
@@ -216,6 +216,14 @@ def X86tlsbaseaddr : SDNode<"X86ISD::TLSBASEADDR", SDT_X86TLSBASEADDR,
 def X86ehret : SDNode<"X86ISD::EH_RETURN", SDT_X86EHRET,
                         [SDNPHasChain]>;
 
+def X86eh_sjlj_setjmp  : SDNode<"X86ISD::EH_SJLJ_SETJMP",
+                                SDTypeProfile<1, 1, [SDTCisInt<0>,
+                                                     SDTCisPtrTy<1>]>,
+                                [SDNPHasChain, SDNPSideEffect]>;
+def X86eh_sjlj_longjmp : SDNode<"X86ISD::EH_SJLJ_LONGJMP",
+                                SDTypeProfile<0, 1, [SDTCisPtrTy<0>]>,
+                                [SDNPHasChain, SDNPSideEffect]>;
+
 def X86tcret : SDNode<"X86ISD::TC_RETURN", SDT_X86TCRET,
                         [SDNPHasChain,  SDNPOptInGlue, SDNPVariadic]>;
 
@@ -397,7 +405,7 @@ def i64mem_TC : Operand<i64> {
 
 let OperandType = "OPERAND_PCREL",
     ParserMatchClass = X86AbsMemAsmOperand,
-    PrintMethod = "print_pcrel_imm" in {
+    PrintMethod = "printPCRelImm" in {
 def i32imm_pcrel : Operand<i32>;
 def i16imm_pcrel : Operand<i16>;
 
@@ -418,7 +426,7 @@ def SSECC : Operand<i8> {
 }
 
 def AVXCC : Operand<i8> {
-  let PrintMethod = "printSSECC";
+  let PrintMethod = "printAVXCC";
   let OperandType = "OPERAND_IMMEDIATE";
 }
 
@@ -499,7 +507,7 @@ def i64i32imm  : Operand<i64> {
 // 64-bits but only 32 bits are significant, and those bits are treated as being
 // pc relative.
 def i64i32imm_pcrel : Operand<i64> {
-  let PrintMethod = "print_pcrel_imm";
+  let PrintMethod = "printPCRelImm";
   let ParserMatchClass = X86AbsMemAsmOperand;
   let OperandType = "OPERAND_PCREL";
 }
@@ -552,14 +560,21 @@ def HasMMX       : Predicate<"Subtarget->hasMMX()">;
 def Has3DNow     : Predicate<"Subtarget->has3DNow()">;
 def Has3DNowA    : Predicate<"Subtarget->has3DNowA()">;
 def HasSSE1      : Predicate<"Subtarget->hasSSE1()">;
+def UseSSE1      : Predicate<"Subtarget->hasSSE1() && !Subtarget->hasAVX()">;
 def HasSSE2      : Predicate<"Subtarget->hasSSE2()">;
+def UseSSE2      : Predicate<"Subtarget->hasSSE2() && !Subtarget->hasAVX()">;
 def HasSSE3      : Predicate<"Subtarget->hasSSE3()">;
+def UseSSE3      : Predicate<"Subtarget->hasSSE3() && !Subtarget->hasAVX()">;
 def HasSSSE3     : Predicate<"Subtarget->hasSSSE3()">;
+def UseSSSE3     : Predicate<"Subtarget->hasSSSE3() && !Subtarget->hasAVX()">;
 def HasSSE41     : Predicate<"Subtarget->hasSSE41()">;
+def UseSSE41     : Predicate<"Subtarget->hasSSE41() && !Subtarget->hasAVX()">;
 def HasSSE42     : Predicate<"Subtarget->hasSSE42()">;
+def UseSSE42     : Predicate<"Subtarget->hasSSE42() && !Subtarget->hasAVX()">;
 def HasSSE4A     : Predicate<"Subtarget->hasSSE4A()">;
 def HasAVX       : Predicate<"Subtarget->hasAVX()">;
 def HasAVX2      : Predicate<"Subtarget->hasAVX2()">;
+def HasAVX1Only  : Predicate<"Subtarget->hasAVX() && !Subtarget->hasAVX2()">;
 
 def HasPOPCNT    : Predicate<"Subtarget->hasPOPCNT()">;
 def HasAES       : Predicate<"Subtarget->hasAES()">;
@@ -574,6 +589,7 @@ def HasFSGSBase  : Predicate<"Subtarget->hasFSGSBase()">;
 def HasLZCNT     : Predicate<"Subtarget->hasLZCNT()">;
 def HasBMI       : Predicate<"Subtarget->hasBMI()">;
 def HasBMI2      : Predicate<"Subtarget->hasBMI2()">;
+def HasRTM       : Predicate<"Subtarget->hasRTM()">;
 def FPStackf32   : Predicate<"!Subtarget->hasSSE1()">;
 def FPStackf64   : Predicate<"!Subtarget->hasSSE2()">;
 def HasCmpxchg16b: Predicate<"Subtarget->hasCmpxchg16b()">;
@@ -1259,28 +1275,46 @@ def BTS64mi8 : RIi8<0xBA, MRM5m, (outs), (ins i64mem:$src1, i64i8imm:$src2),
 // Atomic support
 //
 
-
 // Atomic swap. These are just normal xchg instructions. But since a memory
 // operand is referenced, the atomicity is ensured.
+multiclass ATOMIC_SWAP<bits<8> opc8, bits<8> opc, string mnemonic, string frag,
+                       InstrItinClass itin> {
+  let Constraints = "$val = $dst" in {
+    def #NAME#8rm  : I<opc8, MRMSrcMem, (outs GR8:$dst),
+                       (ins GR8:$val, i8mem:$ptr),
+                       !strconcat(mnemonic, "{b}\t{$val, $ptr|$ptr, $val}"),
+                       [(set
+                          GR8:$dst,
+                          (!cast<PatFrag>(frag # "_8") addr:$ptr, GR8:$val))],
+                       itin>;
+    def #NAME#16rm : I<opc, MRMSrcMem, (outs GR16:$dst),
+                       (ins GR16:$val, i16mem:$ptr),
+                       !strconcat(mnemonic, "{w}\t{$val, $ptr|$ptr, $val}"),
+                       [(set
+                          GR16:$dst,
+                          (!cast<PatFrag>(frag # "_16") addr:$ptr, GR16:$val))],
+                       itin>, OpSize;
+    def #NAME#32rm : I<opc, MRMSrcMem, (outs GR32:$dst),
+                       (ins GR32:$val, i32mem:$ptr),
+                       !strconcat(mnemonic, "{l}\t{$val, $ptr|$ptr, $val}"),
+                       [(set
+                          GR32:$dst,
+                          (!cast<PatFrag>(frag # "_32") addr:$ptr, GR32:$val))],
+                       itin>;
+    def #NAME#64rm : RI<opc, MRMSrcMem, (outs GR64:$dst),
+                        (ins GR64:$val, i64mem:$ptr),
+                        !strconcat(mnemonic, "{q}\t{$val, $ptr|$ptr, $val}"),
+                        [(set
+                          GR64:$dst,
+                          (!cast<PatFrag>(frag # "_64") addr:$ptr, GR64:$val))],
+                        itin>;
+  }
+}
+
+defm XCHG    : ATOMIC_SWAP<0x86, 0x87, "xchg", "atomic_swap", IIC_XCHG_MEM>;
+
+// Swap between registers.
 let Constraints = "$val = $dst" in {
-def XCHG8rm  : I<0x86, MRMSrcMem, (outs GR8:$dst), (ins GR8:$val, i8mem:$ptr),
-               "xchg{b}\t{$val, $ptr|$ptr, $val}",
-               [(set GR8:$dst, (atomic_swap_8 addr:$ptr, GR8:$val))],
-               IIC_XCHG_MEM>;
-def XCHG16rm : I<0x87, MRMSrcMem, (outs GR16:$dst),(ins GR16:$val, i16mem:$ptr),
-               "xchg{w}\t{$val, $ptr|$ptr, $val}",
-               [(set GR16:$dst, (atomic_swap_16 addr:$ptr, GR16:$val))],
-               IIC_XCHG_MEM>,
-                OpSize;
-def XCHG32rm : I<0x87, MRMSrcMem, (outs GR32:$dst),(ins GR32:$val, i32mem:$ptr),
-               "xchg{l}\t{$val, $ptr|$ptr, $val}",
-               [(set GR32:$dst, (atomic_swap_32 addr:$ptr, GR32:$val))],
-               IIC_XCHG_MEM>;
-def XCHG64rm : RI<0x87, MRMSrcMem, (outs GR64:$dst),(ins GR64:$val,i64mem:$ptr),
-                  "xchg{q}\t{$val, $ptr|$ptr, $val}",
-                  [(set GR64:$dst, (atomic_swap_64 addr:$ptr, GR64:$val))],
-                  IIC_XCHG_MEM>;
-
 def XCHG8rr : I<0x86, MRMSrcReg, (outs GR8:$dst), (ins GR8:$val, GR8:$src),
                 "xchg{b}\t{$val, $src|$src, $val}", [], IIC_XCHG_REG>;
 def XCHG16rr : I<0x87, MRMSrcReg, (outs GR16:$dst), (ins GR16:$val, GR16:$src),
@@ -1291,6 +1325,7 @@ def XCHG64rr : RI<0x87, MRMSrcReg, (outs GR64:$dst), (ins GR64:$val,GR64:$src),
                   "xchg{q}\t{$val, $src|$src, $val}", [], IIC_XCHG_REG>;
 }
 
+// Swap between EAX and other registers.
 def XCHG16ar : I<0x90, AddRegFrm, (outs), (ins GR16:$src),
                   "xchg{w}\t{$src, %ax|AX, $src}", [], IIC_XCHG_REG>, OpSize;
 def XCHG32ar : I<0x90, AddRegFrm, (outs), (ins GR32:$src),
@@ -1672,6 +1707,8 @@ include "X86Instr3DNow.td"
 include "X86InstrVMX.td"
 include "X86InstrSVM.td"
 
+include "X86InstrTSX.td"
+
 // System instructions.
 include "X86InstrSystem.td"
 
diff --git a/lib/Target/X86/X86InstrMMX.td b/lib/Target/X86/X86InstrMMX.td
index c8f40bb..127af6f 100644
--- a/lib/Target/X86/X86InstrMMX.td
+++ b/lib/Target/X86/X86InstrMMX.td
@@ -118,11 +118,11 @@ let Constraints = "$src1 = $dst" in {
 /// Unary MMX instructions requiring SSSE3.
 multiclass SS3I_unop_rm_int_mm<bits<8> opc, string OpcodeStr,
                                Intrinsic IntId64, OpndItins itins> {
-  def rr64 : SS38I<opc, MRMSrcReg, (outs VR64:$dst), (ins VR64:$src),
+  def rr64 : MMXSS38I<opc, MRMSrcReg, (outs VR64:$dst), (ins VR64:$src),
                    !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
                    [(set VR64:$dst, (IntId64 VR64:$src))], itins.rr>;
 
-  def rm64 : SS38I<opc, MRMSrcMem, (outs VR64:$dst), (ins i64mem:$src),
+  def rm64 : MMXSS38I<opc, MRMSrcMem, (outs VR64:$dst), (ins i64mem:$src),
                    !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
                    [(set VR64:$dst,
                      (IntId64 (bitconvert (memopmmx addr:$src))))],
@@ -134,11 +134,11 @@ let ImmT = NoImm, Constraints = "$src1 = $dst" in {
 multiclass SS3I_binop_rm_int_mm<bits<8> opc, string OpcodeStr,
                              Intrinsic IntId64, OpndItins itins> {
   let isCommutable = 0 in
-  def rr64 : SS38I<opc, MRMSrcReg, (outs VR64:$dst),
+  def rr64 : MMXSS38I<opc, MRMSrcReg, (outs VR64:$dst),
        (ins VR64:$src1, VR64:$src2),
         !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),
        [(set VR64:$dst, (IntId64 VR64:$src1, VR64:$src2))], itins.rr>;
-  def rm64 : SS38I<opc, MRMSrcMem, (outs VR64:$dst),
+  def rm64 : MMXSS38I<opc, MRMSrcMem, (outs VR64:$dst),
        (ins VR64:$src1, i64mem:$src2),
         !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"),
        [(set VR64:$dst,
@@ -149,11 +149,11 @@ multiclass SS3I_binop_rm_int_mm<bits<8> opc, string OpcodeStr,
 
 /// PALIGN MMX instructions (require SSSE3).
 multiclass ssse3_palign_mm<string asm, Intrinsic IntId> {
-  def R64irr  : SS3AI<0x0F, MRMSrcReg, (outs VR64:$dst),
+  def R64irr  : MMXSS3AI<0x0F, MRMSrcReg, (outs VR64:$dst),
       (ins VR64:$src1, VR64:$src2, i8imm:$src3),
       !strconcat(asm, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 
       [(set VR64:$dst, (IntId VR64:$src1, VR64:$src2, (i8 imm:$src3)))]>;
-  def R64irm  : SS3AI<0x0F, MRMSrcMem, (outs VR64:$dst),
+  def R64irm  : MMXSS3AI<0x0F, MRMSrcMem, (outs VR64:$dst),
       (ins VR64:$src1, i64mem:$src2, i8imm:$src3),
       !strconcat(asm, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
       [(set VR64:$dst, (IntId VR64:$src1,
@@ -163,12 +163,10 @@ multiclass ssse3_palign_mm<string asm, Intrinsic IntId> {
 multiclass sse12_cvt_pint<bits<8> opc, RegisterClass SrcRC, RegisterClass DstRC,
                          Intrinsic Int, X86MemOperand x86memop, PatFrag ld_frag,
                          string asm, OpndItins itins, Domain d> {
-  def irr : PI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src), asm,
-                        [(set DstRC:$dst, (Int SrcRC:$src))], 
-                        itins.rr, d>;
-  def irm : PI<opc, MRMSrcMem, (outs DstRC:$dst), (ins x86memop:$src), asm,
-                        [(set DstRC:$dst, (Int (ld_frag addr:$src)))], 
-                        itins.rm, d>;
+  def irr : MMXPI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src), asm,
+                  [(set DstRC:$dst, (Int SrcRC:$src))], itins.rr, d>;
+  def irm : MMXPI<opc, MRMSrcMem, (outs DstRC:$dst), (ins x86memop:$src), asm,
+                  [(set DstRC:$dst, (Int (ld_frag addr:$src)))], itins.rm, d>;
 }
 
 multiclass sse12_cvt_pint_3addr<bits<8> opc, RegisterClass SrcRC,
@@ -209,8 +207,14 @@ def MMX_MOVD64rm : MMXI<0x6E, MRMSrcMem, (outs VR64:$dst), (ins i32mem:$src),
 let mayStore = 1 in
 def MMX_MOVD64mr : MMXI<0x7E, MRMDestMem, (outs), (ins i32mem:$dst, VR64:$src),
                         "movd\t{$src, $dst|$dst, $src}", [], IIC_MMX_MOV_MM_RM>;
-def MMX_MOVD64grr : MMXI<0x7E, MRMDestReg, (outs), (ins GR32:$dst, VR64:$src),
-                        "movd\t{$src, $dst|$dst, $src}", [], IIC_MMX_MOV_REG_MM>;
+
+// Low word of MMX to GPR.
+def MMX_X86movd2w : SDNode<"X86ISD::MMX_MOVD2W", SDTypeProfile<1, 1,
+                            [SDTCisVT<0, i32>, SDTCisVT<1, x86mmx>]>>;
+def MMX_MOVD64grr : MMXI<0x7E, MRMDestReg, (outs GR32:$dst), (ins VR64:$src),
+                         "movd\t{$src, $dst|$dst, $src}",
+                         [(set GR32:$dst,
+                          (MMX_X86movd2w (x86mmx VR64:$src)))], IIC_MMX_MOV_REG_MM>;
 
 let neverHasSideEffects = 1 in
 def MMX_MOVD64to64rr : MMXRI<0x6E, MRMSrcReg, (outs VR64:$dst), (ins GR64:$src),
@@ -243,29 +247,30 @@ def MMX_MOVQ64mr : MMXI<0x7F, MRMDestMem, (outs), (ins i64mem:$dst, VR64:$src),
                         [(store (x86mmx VR64:$src), addr:$dst)],
                         IIC_MMX_MOVQ_RM>;
 
-def MMX_MOVDQ2Qrr : SDIi8<0xD6, MRMSrcReg, (outs VR64:$dst),
-                          (ins VR128:$src), "movdq2q\t{$src, $dst|$dst, $src}",
-                          [(set VR64:$dst,
-                            (x86mmx (bitconvert
-                            (i64 (vector_extract (v2i64 VR128:$src),
-                                  (iPTR 0))))))],
-                          IIC_MMX_MOVQ_RR>;
-
-def MMX_MOVQ2DQrr : S2SIi8<0xD6, MRMSrcReg, (outs VR128:$dst),
-                            (ins VR64:$src), "movq2dq\t{$src, $dst|$dst, $src}",
-          [(set VR128:$dst,
-            (v2i64 (scalar_to_vector
-                              (i64 (bitconvert (x86mmx VR64:$src))))))],
-                           IIC_MMX_MOVQ_RR>;
+def MMX_MOVDQ2Qrr : MMXSDIi8<0xD6, MRMSrcReg, (outs VR64:$dst),
+                             (ins VR128:$src), "movdq2q\t{$src, $dst|$dst, $src}",
+                             [(set VR64:$dst,
+                               (x86mmx (bitconvert
+                               (i64 (vector_extract (v2i64 VR128:$src),
+                                     (iPTR 0))))))],
+                             IIC_MMX_MOVQ_RR>;
+
+def MMX_MOVQ2DQrr : MMXS2SIi8<0xD6, MRMSrcReg, (outs VR128:$dst),
+                              (ins VR64:$src), "movq2dq\t{$src, $dst|$dst, $src}",
+                              [(set VR128:$dst,
+                                (v2i64
+                                  (scalar_to_vector
+                                    (i64 (bitconvert (x86mmx VR64:$src))))))],
+                              IIC_MMX_MOVQ_RR>;
 
 let neverHasSideEffects = 1 in
-def MMX_MOVQ2FR64rr: S2SIi8<0xD6, MRMSrcReg, (outs FR64:$dst),
-                       (ins VR64:$src), "movq2dq\t{$src, $dst|$dst, $src}", [],
-                       IIC_MMX_MOVQ_RR>;
+def MMX_MOVQ2FR64rr: MMXS2SIi8<0xD6, MRMSrcReg, (outs FR64:$dst),
+                               (ins VR64:$src), "movq2dq\t{$src, $dst|$dst, $src}",
+                               [], IIC_MMX_MOVQ_RR>;
 
-def MMX_MOVFR642Qrr: SDIi8<0xD6, MRMSrcReg, (outs VR64:$dst),
-                       (ins FR64:$src), "movdq2q\t{$src, $dst|$dst, $src}", [],
-                       IIC_MMX_MOVQ_RR>;
+def MMX_MOVFR642Qrr: MMXSDIi8<0xD6, MRMSrcReg, (outs VR64:$dst),
+                              (ins FR64:$src), "movdq2q\t{$src, $dst|$dst, $src}",
+                              [], IIC_MMX_MOVQ_RR>;
 
 def MMX_MOVNTQmr  : MMXI<0xE7, MRMDestMem, (outs), (ins i64mem:$dst, VR64:$src),
                          "movntq\t{$src, $dst|$dst, $src}",
@@ -577,6 +582,7 @@ def MMX_MASKMOVQ64: MMXI64<0xF7, MRMSrcReg, (outs), (ins VR64:$src, VR64:$mask),
                            IIC_MMX_MASKMOV>;
 
 // 64-bit bit convert.
+let Predicates = [HasSSE2] in {
 def : Pat<(x86mmx (bitconvert (i64 GR64:$src))),
           (MMX_MOVD64to64rr GR64:$src)>;
 def : Pat<(i64 (bitconvert (x86mmx VR64:$src))),
@@ -585,5 +591,6 @@ def : Pat<(f64 (bitconvert (x86mmx VR64:$src))),
           (MMX_MOVQ2FR64rr VR64:$src)>;
 def : Pat<(x86mmx (bitconvert (f64 FR64:$src))),
           (MMX_MOVFR642Qrr FR64:$src)>;
+}
 
 
diff --git a/lib/Target/X86/X86InstrSSE.td b/lib/Target/X86/X86InstrSSE.td
index 20dc81e..6f48d7e 100644
--- a/lib/Target/X86/X86InstrSSE.td
+++ b/lib/Target/X86/X86InstrSSE.td
@@ -251,35 +251,37 @@ def : Pat<(f64 (vector_extract (v2f64 VR128:$src), (iPTR 0))),
 
 // A 128-bit subvector extract from the first 256-bit vector position
 // is a subregister copy that needs no instruction.
-def : Pat<(v4i32 (extract_subvector (v8i32 VR256:$src), (i32 0))),
+def : Pat<(v4i32 (extract_subvector (v8i32 VR256:$src), (iPTR 0))),
           (v4i32 (EXTRACT_SUBREG (v8i32 VR256:$src), sub_xmm))>;
-def : Pat<(v4f32 (extract_subvector (v8f32 VR256:$src), (i32 0))),
+def : Pat<(v4f32 (extract_subvector (v8f32 VR256:$src), (iPTR 0))),
           (v4f32 (EXTRACT_SUBREG (v8f32 VR256:$src), sub_xmm))>;
 
-def : Pat<(v2i64 (extract_subvector (v4i64 VR256:$src), (i32 0))),
+def : Pat<(v2i64 (extract_subvector (v4i64 VR256:$src), (iPTR 0))),
           (v2i64 (EXTRACT_SUBREG (v4i64 VR256:$src), sub_xmm))>;
-def : Pat<(v2f64 (extract_subvector (v4f64 VR256:$src), (i32 0))),
+def : Pat<(v2f64 (extract_subvector (v4f64 VR256:$src), (iPTR 0))),
           (v2f64 (EXTRACT_SUBREG (v4f64 VR256:$src), sub_xmm))>;
 
-def : Pat<(v8i16 (extract_subvector (v16i16 VR256:$src), (i32 0))),
+def : Pat<(v8i16 (extract_subvector (v16i16 VR256:$src), (iPTR 0))),
           (v8i16 (EXTRACT_SUBREG (v16i16 VR256:$src), sub_xmm))>;
-def : Pat<(v16i8 (extract_subvector (v32i8 VR256:$src), (i32 0))),
+def : Pat<(v16i8 (extract_subvector (v32i8 VR256:$src), (iPTR 0))),
           (v16i8 (EXTRACT_SUBREG (v32i8 VR256:$src), sub_xmm))>;
 
 // A 128-bit subvector insert to the first 256-bit vector position
 // is a subregister copy that needs no instruction.
-def : Pat<(insert_subvector undef, (v2i64 VR128:$src), (i32 0)),
+let AddedComplexity = 25 in { // to give priority over vinsertf128rm
+def : Pat<(insert_subvector undef, (v2i64 VR128:$src), (iPTR 0)),
           (INSERT_SUBREG (v4i64 (IMPLICIT_DEF)), VR128:$src, sub_xmm)>;
-def : Pat<(insert_subvector undef, (v2f64 VR128:$src), (i32 0)),
+def : Pat<(insert_subvector undef, (v2f64 VR128:$src), (iPTR 0)),
           (INSERT_SUBREG (v4f64 (IMPLICIT_DEF)), VR128:$src, sub_xmm)>;
-def : Pat<(insert_subvector undef, (v4i32 VR128:$src), (i32 0)),
+def : Pat<(insert_subvector undef, (v4i32 VR128:$src), (iPTR 0)),
           (INSERT_SUBREG (v8i32 (IMPLICIT_DEF)), VR128:$src, sub_xmm)>;
-def : Pat<(insert_subvector undef, (v4f32 VR128:$src), (i32 0)),
+def : Pat<(insert_subvector undef, (v4f32 VR128:$src), (iPTR 0)),
           (INSERT_SUBREG (v8f32 (IMPLICIT_DEF)), VR128:$src, sub_xmm)>;
-def : Pat<(insert_subvector undef, (v8i16 VR128:$src), (i32 0)),
+def : Pat<(insert_subvector undef, (v8i16 VR128:$src), (iPTR 0)),
           (INSERT_SUBREG (v16i16 (IMPLICIT_DEF)), VR128:$src, sub_xmm)>;
-def : Pat<(insert_subvector undef, (v16i8 VR128:$src), (i32 0)),
+def : Pat<(insert_subvector undef, (v16i8 VR128:$src), (iPTR 0)),
           (INSERT_SUBREG (v32i8 (IMPLICIT_DEF)), VR128:$src, sub_xmm)>;
+}
 
 // Implicitly promote a 32-bit scalar to a vector.
 def : Pat<(v4f32 (scalar_to_vector FR32:$src)),
@@ -362,7 +364,7 @@ let Predicates = [HasAVX] in {
   def : Pat<(v16i16 (bitconvert (v32i8 VR256:$src))),  (v16i16 VR256:$src)>;
 }
 
-// Alias instructions that map fld0 to pxor for sse.
+// Alias instructions that map fld0 to xorps for sse or vxorps for avx.
 // This is expanded by ExpandPostRAPseudos.
 let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1,
     isPseudo = 1 in {
@@ -382,11 +384,11 @@ let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1,
 // We set canFoldAsLoad because this can be converted to a constant-pool
 // load of an all-zeros value if folding it would be beneficial.
 let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1,
-    isPseudo = 1, neverHasSideEffects = 1 in {
-def V_SET0 : I<0, Pseudo, (outs VR128:$dst), (ins), "", []>;
+    isPseudo = 1 in {
+def V_SET0 : I<0, Pseudo, (outs VR128:$dst), (ins), "",
+               [(set VR128:$dst, (v4f32 immAllZerosV))]>;
 }
 
-def : Pat<(v4f32 immAllZerosV), (V_SET0)>;
 def : Pat<(v2f64 immAllZerosV), (V_SET0)>;
 def : Pat<(v4i32 immAllZerosV), (V_SET0)>;
 def : Pat<(v2i64 immAllZerosV), (V_SET0)>;
@@ -394,35 +396,29 @@ def : Pat<(v8i16 immAllZerosV), (V_SET0)>;
 def : Pat<(v16i8 immAllZerosV), (V_SET0)>;
 
 
-// The same as done above but for AVX.  The 256-bit ISA does not support PI,
+// The same as done above but for AVX.  The 256-bit AVX1 ISA doesn't support PI,
 // and doesn't need it because on sandy bridge the register is set to zero
 // at the rename stage without using any execution unit, so SET0PSY
 // and SET0PDY can be used for vector int instructions without penalty
-// FIXME: Change encoding to pseudo! This is blocked right now by the x86
-// JIT implementatioan, it does not expand the instructions below like
-// X86MCInstLower does.
 let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1,
-    isCodeGenOnly = 1 in {
-let Predicates = [HasAVX] in {
-def AVX_SET0PSY : PSI<0x57, MRMInitReg, (outs VR256:$dst), (ins), "",
-                   [(set VR256:$dst, (v8f32 immAllZerosV))]>, VEX_4V;
-def AVX_SET0PDY : PDI<0x57, MRMInitReg, (outs VR256:$dst), (ins), "",
-                   [(set VR256:$dst, (v4f64 immAllZerosV))]>, VEX_4V;
-}
-let Predicates = [HasAVX2], neverHasSideEffects = 1 in
-def AVX2_SET0   : PDI<0xef, MRMInitReg, (outs VR256:$dst), (ins), "",
-                   []>, VEX_4V;
+    isPseudo = 1, Predicates = [HasAVX] in {
+def AVX_SET0 : I<0, Pseudo, (outs VR256:$dst), (ins), "",
+                 [(set VR256:$dst, (v8f32 immAllZerosV))]>;
 }
 
-let Predicates = [HasAVX2], AddedComplexity = 5 in {
-  def : Pat<(v4i64 immAllZerosV), (AVX2_SET0)>;
-  def : Pat<(v8i32 immAllZerosV), (AVX2_SET0)>;
-  def : Pat<(v16i16 immAllZerosV), (AVX2_SET0)>;
-  def : Pat<(v32i8 immAllZerosV), (AVX2_SET0)>;
+let Predicates = [HasAVX] in
+  def : Pat<(v4f64 immAllZerosV), (AVX_SET0)>;
+
+let Predicates = [HasAVX2] in {
+  def : Pat<(v4i64 immAllZerosV), (AVX_SET0)>;
+  def : Pat<(v8i32 immAllZerosV), (AVX_SET0)>;
+  def : Pat<(v16i16 immAllZerosV), (AVX_SET0)>;
+  def : Pat<(v32i8 immAllZerosV), (AVX_SET0)>;
 }
 
-// AVX has no support for 256-bit integer instructions, but since the 128-bit
+// AVX1 has no support for 256-bit integer instructions, but since the 128-bit
 // VPXOR instruction writes zero to its upper part, it's safe build zeros.
+let Predicates = [HasAVX1Only] in {
 def : Pat<(v32i8 immAllZerosV), (SUBREG_TO_REG (i8 0), (V_SET0), sub_xmm)>;
 def : Pat<(bc_v32i8 (v8f32 immAllZerosV)),
           (SUBREG_TO_REG (i8 0), (V_SET0), sub_xmm)>;
@@ -438,22 +434,17 @@ def : Pat<(bc_v8i32 (v8f32 immAllZerosV)),
 def : Pat<(v4i64 immAllZerosV), (SUBREG_TO_REG (i64 0), (V_SET0), sub_xmm)>;
 def : Pat<(bc_v4i64 (v8f32 immAllZerosV)),
           (SUBREG_TO_REG (i64 0), (V_SET0), sub_xmm)>;
+}
 
 // We set canFoldAsLoad because this can be converted to a constant-pool
 // load of an all-ones value if folding it would be beneficial.
-// FIXME: Change encoding to pseudo! This is blocked right now by the x86
-// JIT implementation, it does not expand the instructions below like
-// X86MCInstLower does.
 let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1,
-    isCodeGenOnly = 1, ExeDomain = SSEPackedInt in {
-  let Predicates = [HasAVX] in
-  def AVX_SETALLONES : PDI<0x76, MRMInitReg, (outs VR128:$dst), (ins), "",
-                         [(set VR128:$dst, (v4i32 immAllOnesV))]>, VEX_4V;
-  def V_SETALLONES : PDI<0x76, MRMInitReg, (outs VR128:$dst), (ins), "",
-                         [(set VR128:$dst, (v4i32 immAllOnesV))]>;
+    isPseudo = 1 in {
+  def V_SETALLONES : I<0, Pseudo, (outs VR128:$dst), (ins), "",
+                       [(set VR128:$dst, (v4i32 immAllOnesV))]>;
   let Predicates = [HasAVX2] in
-  def AVX2_SETALLONES : PDI<0x76, MRMInitReg, (outs VR256:$dst), (ins), "",
-                          [(set VR256:$dst, (v8i32 immAllOnesV))]>, VEX_4V;
+  def AVX2_SETALLONES : I<0, Pseudo, (outs VR256:$dst), (ins), "",
+                          [(set VR256:$dst, (v8i32 immAllOnesV))]>;
 }
 
 
@@ -605,27 +596,27 @@ let Predicates = [HasAVX] in {
   // Represent the same patterns above but in the form they appear for
   // 256-bit types
   def : Pat<(v8i32 (X86vzmovl (insert_subvector undef,
-                   (v4i32 (scalar_to_vector (loadi32 addr:$src))), (i32 0)))),
+                   (v4i32 (scalar_to_vector (loadi32 addr:$src))), (iPTR 0)))),
             (SUBREG_TO_REG (i32 0), (VMOVSSrm addr:$src), sub_xmm)>;
   def : Pat<(v8f32 (X86vzmovl (insert_subvector undef,
-                   (v4f32 (scalar_to_vector (loadf32 addr:$src))), (i32 0)))),
+                   (v4f32 (scalar_to_vector (loadf32 addr:$src))), (iPTR 0)))),
             (SUBREG_TO_REG (i32 0), (VMOVSSrm addr:$src), sub_xmm)>;
   def : Pat<(v4f64 (X86vzmovl (insert_subvector undef,
-                   (v2f64 (scalar_to_vector (loadf64 addr:$src))), (i32 0)))),
+                   (v2f64 (scalar_to_vector (loadf64 addr:$src))), (iPTR 0)))),
             (SUBREG_TO_REG (i32 0), (VMOVSDrm addr:$src), sub_xmm)>;
   }
   def : Pat<(v8f32 (X86vzmovl (insert_subvector undef,
-                   (v4f32 (scalar_to_vector FR32:$src)), (i32 0)))),
+                   (v4f32 (scalar_to_vector FR32:$src)), (iPTR 0)))),
             (SUBREG_TO_REG (i32 0),
                            (v4f32 (VMOVSSrr (v4f32 (V_SET0)), FR32:$src)),
                            sub_xmm)>;
   def : Pat<(v4f64 (X86vzmovl (insert_subvector undef,
-                   (v2f64 (scalar_to_vector FR64:$src)), (i32 0)))),
+                   (v2f64 (scalar_to_vector FR64:$src)), (iPTR 0)))),
             (SUBREG_TO_REG (i64 0),
                            (v2f64 (VMOVSDrr (v2f64 (V_SET0)), FR64:$src)),
                            sub_xmm)>;
   def : Pat<(v4i64 (X86vzmovl (insert_subvector undef,
-                   (v2i64 (scalar_to_vector (loadi64 addr:$src))), (i32 0)))),
+                   (v2i64 (scalar_to_vector (loadi64 addr:$src))), (iPTR 0)))),
             (SUBREG_TO_REG (i64 0), (VMOVSDrm addr:$src), sub_xmm)>;
 
   // Move low f64 and clear high bits.
@@ -704,7 +695,7 @@ let Predicates = [HasAVX] in {
             (VMOVSDrr VR128:$src1, (COPY_TO_REGCLASS VR128:$src2, FR64))>;
 }
 
-let Predicates = [HasSSE1] in {
+let Predicates = [UseSSE1] in {
   let AddedComplexity = 15 in {
   // Move scalar to XMM zero-extended, zeroing a VR128 then do a
   // MOVSS to the lower bits.
@@ -738,7 +729,7 @@ let Predicates = [HasSSE1] in {
             (MOVSSrr VR128:$src1, (COPY_TO_REGCLASS VR128:$src2, FR32))>;
 }
 
-let Predicates = [HasSSE2] in {
+let Predicates = [UseSSE2] in {
   let AddedComplexity = 15 in {
   // Move scalar to XMM zero-extended, zeroing a VR128 then do a
   // MOVSD to the lower bits.
@@ -822,16 +813,16 @@ defm VMOVUPD : sse12_mov_packed<0x10, VR128, f128mem, loadv2f64,
 
 defm VMOVAPSY : sse12_mov_packed<0x28, VR256, f256mem, alignedloadv8f32,
                               "movaps", SSEPackedSingle, SSE_MOVA_ITINS>,
-                              TB, VEX;
+                              TB, VEX, VEX_L;
 defm VMOVAPDY : sse12_mov_packed<0x28, VR256, f256mem, alignedloadv4f64,
                               "movapd", SSEPackedDouble, SSE_MOVA_ITINS>,
-                              TB, OpSize, VEX;
+                              TB, OpSize, VEX, VEX_L;
 defm VMOVUPSY : sse12_mov_packed<0x10, VR256, f256mem, loadv8f32,
                               "movups", SSEPackedSingle, SSE_MOVU_ITINS>,
-                              TB, VEX;
+                              TB, VEX, VEX_L;
 defm VMOVUPDY : sse12_mov_packed<0x10, VR256, f256mem, loadv4f64,
                               "movupd", SSEPackedDouble, SSE_MOVU_ITINS, 0>,
-                              TB, OpSize, VEX;
+                              TB, OpSize, VEX, VEX_L;
 defm MOVAPS : sse12_mov_packed<0x28, VR128, f128mem, alignedloadv4f32,
                               "movaps", SSEPackedSingle, SSE_MOVA_ITINS>,
                               TB;
@@ -864,19 +855,19 @@ def VMOVUPDmr : VPDI<0x11, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src),
 def VMOVAPSYmr : VPSI<0x29, MRMDestMem, (outs), (ins f256mem:$dst, VR256:$src),
                    "movaps\t{$src, $dst|$dst, $src}",
                    [(alignedstore256 (v8f32 VR256:$src), addr:$dst)],
-                   IIC_SSE_MOVA_P_MR>, VEX;
+                   IIC_SSE_MOVA_P_MR>, VEX, VEX_L;
 def VMOVAPDYmr : VPDI<0x29, MRMDestMem, (outs), (ins f256mem:$dst, VR256:$src),
                    "movapd\t{$src, $dst|$dst, $src}",
                    [(alignedstore256 (v4f64 VR256:$src), addr:$dst)],
-                   IIC_SSE_MOVA_P_MR>, VEX;
+                   IIC_SSE_MOVA_P_MR>, VEX, VEX_L;
 def VMOVUPSYmr : VPSI<0x11, MRMDestMem, (outs), (ins f256mem:$dst, VR256:$src),
                    "movups\t{$src, $dst|$dst, $src}",
                    [(store (v8f32 VR256:$src), addr:$dst)],
-                   IIC_SSE_MOVU_P_MR>, VEX;
+                   IIC_SSE_MOVU_P_MR>, VEX, VEX_L;
 def VMOVUPDYmr : VPDI<0x11, MRMDestMem, (outs), (ins f256mem:$dst, VR256:$src),
                    "movupd\t{$src, $dst|$dst, $src}",
                    [(store (v4f64 VR256:$src), addr:$dst)],
-                   IIC_SSE_MOVU_P_MR>, VEX;
+                   IIC_SSE_MOVU_P_MR>, VEX, VEX_L;
 
 // For disassembler
 let isCodeGenOnly = 1 in {
@@ -899,33 +890,33 @@ let isCodeGenOnly = 1 in {
   def VMOVAPSYrr_REV : VPSI<0x29, MRMDestReg, (outs VR256:$dst),
                             (ins VR256:$src),
                             "movaps\t{$src, $dst|$dst, $src}", [],
-                            IIC_SSE_MOVA_P_RR>, VEX;
+                            IIC_SSE_MOVA_P_RR>, VEX, VEX_L;
   def VMOVAPDYrr_REV : VPDI<0x29, MRMDestReg, (outs VR256:$dst),
                             (ins VR256:$src),
                             "movapd\t{$src, $dst|$dst, $src}", [],
-                            IIC_SSE_MOVA_P_RR>, VEX;
+                            IIC_SSE_MOVA_P_RR>, VEX, VEX_L;
   def VMOVUPSYrr_REV : VPSI<0x11, MRMDestReg, (outs VR256:$dst),
                             (ins VR256:$src),
                             "movups\t{$src, $dst|$dst, $src}", [],
-                            IIC_SSE_MOVU_P_RR>, VEX;
+                            IIC_SSE_MOVU_P_RR>, VEX, VEX_L;
   def VMOVUPDYrr_REV : VPDI<0x11, MRMDestReg, (outs VR256:$dst),
                             (ins VR256:$src),
                             "movupd\t{$src, $dst|$dst, $src}", [],
-                            IIC_SSE_MOVU_P_RR>, VEX;
+                            IIC_SSE_MOVU_P_RR>, VEX, VEX_L;
 }
 
 let Predicates = [HasAVX] in {
 def : Pat<(v8i32 (X86vzmovl
-                        (insert_subvector undef, (v4i32 VR128:$src), (i32 0)))),
+                  (insert_subvector undef, (v4i32 VR128:$src), (iPTR 0)))),
           (SUBREG_TO_REG (i32 0), (VMOVAPSrr VR128:$src), sub_xmm)>;
 def : Pat<(v4i64 (X86vzmovl
-                        (insert_subvector undef, (v2i64 VR128:$src), (i32 0)))),
+                  (insert_subvector undef, (v2i64 VR128:$src), (iPTR 0)))),
           (SUBREG_TO_REG (i32 0), (VMOVAPSrr VR128:$src), sub_xmm)>;
 def : Pat<(v8f32 (X86vzmovl
-                        (insert_subvector undef, (v4f32 VR128:$src), (i32 0)))),
+                  (insert_subvector undef, (v4f32 VR128:$src), (iPTR 0)))),
           (SUBREG_TO_REG (i32 0), (VMOVAPSrr VR128:$src), sub_xmm)>;
 def : Pat<(v4f64 (X86vzmovl
-                        (insert_subvector undef, (v2f64 VR128:$src), (i32 0)))),
+                  (insert_subvector undef, (v2f64 VR128:$src), (iPTR 0)))),
           (SUBREG_TO_REG (i32 0), (VMOVAPSrr VR128:$src), sub_xmm)>;
 }
 
@@ -975,10 +966,10 @@ let Predicates = [HasAVX] in {
             (VMOVUPDmr addr:$dst, VR128:$src)>;
 }
 
-let Predicates = [HasSSE1] in
+let Predicates = [UseSSE1] in
   def : Pat<(int_x86_sse_storeu_ps addr:$dst, VR128:$src),
             (MOVUPSmr addr:$dst, VR128:$src)>;
-let Predicates = [HasSSE2] in
+let Predicates = [UseSSE2] in
   def : Pat<(int_x86_sse2_storeu_pd addr:$dst, VR128:$src),
             (MOVUPDmr addr:$dst, VR128:$src)>;
 
@@ -1028,12 +1019,52 @@ let Predicates = [HasAVX] in {
             (VMOVUPSYmr addr:$dst, VR256:$src)>;
   def : Pat<(store (v32i8 VR256:$src), addr:$dst),
             (VMOVUPSYmr addr:$dst, VR256:$src)>;
+
+  // Special patterns for storing subvector extracts of lower 128-bits
+  // Its cheaper to just use VMOVAPS/VMOVUPS instead of VEXTRACTF128mr
+  def : Pat<(alignedstore (v2f64 (extract_subvector
+                                  (v4f64 VR256:$src), (iPTR 0))), addr:$dst),
+            (VMOVAPDmr addr:$dst, (v2f64 (EXTRACT_SUBREG VR256:$src,sub_xmm)))>;
+  def : Pat<(alignedstore (v4f32 (extract_subvector
+                                  (v8f32 VR256:$src), (iPTR 0))), addr:$dst),
+            (VMOVAPSmr addr:$dst, (v4f32 (EXTRACT_SUBREG VR256:$src,sub_xmm)))>;
+  def : Pat<(alignedstore (v2i64 (extract_subvector
+                                  (v4i64 VR256:$src), (iPTR 0))), addr:$dst),
+            (VMOVAPDmr addr:$dst, (v2i64 (EXTRACT_SUBREG VR256:$src,sub_xmm)))>;
+  def : Pat<(alignedstore (v4i32 (extract_subvector
+                                  (v8i32 VR256:$src), (iPTR 0))), addr:$dst),
+            (VMOVAPSmr addr:$dst, (v4i32 (EXTRACT_SUBREG VR256:$src,sub_xmm)))>;
+  def : Pat<(alignedstore (v8i16 (extract_subvector
+                                  (v16i16 VR256:$src), (iPTR 0))), addr:$dst),
+            (VMOVAPSmr addr:$dst, (v8i16 (EXTRACT_SUBREG VR256:$src,sub_xmm)))>;
+  def : Pat<(alignedstore (v16i8 (extract_subvector
+                                  (v32i8 VR256:$src), (iPTR 0))), addr:$dst),
+            (VMOVAPSmr addr:$dst, (v16i8 (EXTRACT_SUBREG VR256:$src,sub_xmm)))>;
+
+  def : Pat<(store (v2f64 (extract_subvector
+                           (v4f64 VR256:$src), (iPTR 0))), addr:$dst),
+            (VMOVUPDmr addr:$dst, (v2f64 (EXTRACT_SUBREG VR256:$src,sub_xmm)))>;
+  def : Pat<(store (v4f32 (extract_subvector
+                           (v8f32 VR256:$src), (iPTR 0))), addr:$dst),
+            (VMOVUPSmr addr:$dst, (v4f32 (EXTRACT_SUBREG VR256:$src,sub_xmm)))>;
+  def : Pat<(store (v2i64 (extract_subvector
+                           (v4i64 VR256:$src), (iPTR 0))), addr:$dst),
+            (VMOVUPDmr addr:$dst, (v2i64 (EXTRACT_SUBREG VR256:$src,sub_xmm)))>;
+  def : Pat<(store (v4i32 (extract_subvector
+                           (v8i32 VR256:$src), (iPTR 0))), addr:$dst),
+            (VMOVUPSmr addr:$dst, (v4i32 (EXTRACT_SUBREG VR256:$src,sub_xmm)))>;
+  def : Pat<(store (v8i16 (extract_subvector
+                           (v16i16 VR256:$src), (iPTR 0))), addr:$dst),
+            (VMOVAPSmr addr:$dst, (v8i16 (EXTRACT_SUBREG VR256:$src,sub_xmm)))>;
+  def : Pat<(store (v16i8 (extract_subvector
+                           (v32i8 VR256:$src), (iPTR 0))), addr:$dst),
+            (VMOVUPSmr addr:$dst, (v16i8 (EXTRACT_SUBREG VR256:$src,sub_xmm)))>;
 }
 
 // Use movaps / movups for SSE integer load / store (one byte shorter).
 // The instructions selected below are then converted to MOVDQA/MOVDQU
 // during the SSE domain pass.
-let Predicates = [HasSSE1] in {
+let Predicates = [UseSSE1] in {
   def : Pat<(alignedloadv2i64 addr:$src),
             (MOVAPSrm addr:$src)>;
   def : Pat<(loadv2i64 addr:$src),
@@ -1180,7 +1211,7 @@ let Predicates = [HasAVX] in {
             (VMOVLPDmr addr:$src1, VR128:$src2)>;
 }
 
-let Predicates = [HasSSE1] in {
+let Predicates = [UseSSE1] in {
   // (store (vector_shuffle (load addr), v2, <4, 5, 2, 3>), addr) using MOVLPS
   def : Pat<(store (i64 (vector_extract (bc_v2i64 (v4f32 VR128:$src2)),
                                  (iPTR 0))), addr:$src1),
@@ -1205,7 +1236,7 @@ let Predicates = [HasSSE1] in {
             (MOVLPSmr addr:$src1, VR128:$src2)>;
 }
 
-let Predicates = [HasSSE2] in {
+let Predicates = [UseSSE2] in {
   // Shuffle with MOVLPD
   def : Pat<(v2f64 (X86Movlpd VR128:$src1, (load addr:$src2))),
             (MOVLPDrm VR128:$src1, addr:$src2)>;
@@ -1271,7 +1302,7 @@ let Predicates = [HasAVX] in {
             (VMOVHPSrm VR128:$src1, addr:$src2)>;
 
   // FIXME: Instead of X86Unpckl, there should be a X86Movlhpd here, the problem
-  // is during lowering, where it's not possible to recognize the load fold 
+  // is during lowering, where it's not possible to recognize the load fold
   // cause it has two uses through a bitcast. One use disappears at isel time
   // and the fold opportunity reappears.
   def : Pat<(v2f64 (X86Unpckl VR128:$src1,
@@ -1279,7 +1310,7 @@ let Predicates = [HasAVX] in {
             (VMOVHPDrm VR128:$src1, addr:$src2)>;
 }
 
-let Predicates = [HasSSE1] in {
+let Predicates = [UseSSE1] in {
   // MOVHPS patterns
   def : Pat<(X86Movlhps VR128:$src1,
                  (bc_v4f32 (v2i64 (scalar_to_vector (loadi64 addr:$src2))))),
@@ -1289,9 +1320,9 @@ let Predicates = [HasSSE1] in {
             (MOVHPSrm VR128:$src1, addr:$src2)>;
 }
 
-let Predicates = [HasSSE2] in {
+let Predicates = [UseSSE2] in {
   // FIXME: Instead of X86Unpckl, there should be a X86Movlhpd here, the problem
-  // is during lowering, where it's not possible to recognize the load fold 
+  // is during lowering, where it's not possible to recognize the load fold
   // cause it has two uses through a bitcast. One use disappears at isel time
   // and the fold opportunity reappears.
   def : Pat<(v2f64 (X86Unpckl VR128:$src1,
@@ -1346,7 +1377,7 @@ let Predicates = [HasAVX] in {
             (VMOVHLPSrr VR128:$src1, VR128:$src2)>;
 }
 
-let Predicates = [HasSSE1] in {
+let Predicates = [UseSSE1] in {
   // MOVLHPS patterns
   def : Pat<(v4i32 (X86Movlhps VR128:$src1, VR128:$src2)),
             (MOVLHPSrr VR128:$src1, VR128:$src2)>;
@@ -1456,7 +1487,7 @@ def : InstAlias<"vcvtsi2sd{l}\t{$src, $src1, $dst|$dst, $src1, $src}",
 def : InstAlias<"vcvtsi2sd{l}\t{$src, $src1, $dst|$dst, $src1, $src}",
                 (VCVTSI2SDrm FR64:$dst, FR64:$src1, i32mem:$src)>;
 
-let Predicates = [HasAVX], AddedComplexity = 1 in {
+let Predicates = [HasAVX] in {
   def : Pat<(f32 (sint_to_fp (loadi32 addr:$src))),
             (VCVTSI2SSrm (f32 (IMPLICIT_DEF)), addr:$src)>;
   def : Pat<(f32 (sint_to_fp (loadi64 addr:$src))),
@@ -1628,12 +1659,12 @@ defm VCVTDQ2PS   : sse12_cvt_p<0x5B, VR128, VR128, i128mem,
 defm VCVTDQ2PSY  : sse12_cvt_p<0x5B, VR256, VR256, i256mem,
                                "vcvtdq2ps\t{$src, $dst|$dst, $src}",
                                SSEPackedSingle, SSE_CVT_PS>,
-                               TB, VEX, Requires<[HasAVX]>;
+                               TB, VEX, VEX_L, Requires<[HasAVX]>;
 
 defm CVTDQ2PS : sse12_cvt_p<0x5B, VR128, VR128, i128mem,
                             "cvtdq2ps\t{$src, $dst|$dst, $src}",
                             SSEPackedSingle, SSE_CVT_PS>,
-                            TB, Requires<[HasSSE2]>;
+                            TB, Requires<[UseSSE2]>;
 
 /// SSE 2 Only
 
@@ -1663,7 +1694,7 @@ def CVTSD2SSrm  : I<0x5A, MRMSrcMem, (outs FR32:$dst), (ins f64mem:$src),
                       [(set FR32:$dst, (fround (loadf64 addr:$src)))],
                       IIC_SSE_CVT_Scalar_RM>,
                       XD,
-                  Requires<[HasSSE2, OptForSize]>;
+                  Requires<[UseSSE2, OptForSize]>;
 
 def Int_VCVTSD2SSrr: I<0x5A, MRMSrcReg,
                        (outs VR128:$dst), (ins VR128:$src1, VR128:$src2),
@@ -1684,13 +1715,13 @@ def Int_CVTSD2SSrr: I<0x5A, MRMSrcReg,
                        "cvtsd2ss\t{$src2, $src1, $dst|$dst, $src1, $src2}",
                        [(set VR128:$dst,
                          (int_x86_sse2_cvtsd2ss VR128:$src1, VR128:$src2))],
-                       IIC_SSE_CVT_Scalar_RR>, XD, Requires<[HasSSE2]>;
+                       IIC_SSE_CVT_Scalar_RR>, XD, Requires<[UseSSE2]>;
 def Int_CVTSD2SSrm: I<0x5A, MRMSrcReg,
                        (outs VR128:$dst), (ins VR128:$src1, sdmem:$src2),
                        "cvtsd2ss\t{$src2, $src1, $dst|$dst, $src1, $src2}",
                        [(set VR128:$dst, (int_x86_sse2_cvtsd2ss
                                           VR128:$src1, sse_load_f64:$src2))],
-                       IIC_SSE_CVT_Scalar_RM>, XD, Requires<[HasSSE2]>;
+                       IIC_SSE_CVT_Scalar_RM>, XD, Requires<[UseSSE2]>;
 }
 
 // Convert scalar single to scalar double
@@ -1709,30 +1740,28 @@ def VCVTSS2SDrm : I<0x5A, MRMSrcMem, (outs FR64:$dst),
                     XS, VEX_4V, VEX_LIG, Requires<[HasAVX, OptForSize]>;
 }
 
-let AddedComplexity = 1 in { // give AVX priority
-  def : Pat<(f64 (fextend FR32:$src)),
-            (VCVTSS2SDrr FR32:$src, FR32:$src)>, Requires<[HasAVX]>;
-  def : Pat<(fextend (loadf32 addr:$src)),
-            (VCVTSS2SDrm (f32 (IMPLICIT_DEF)), addr:$src)>, Requires<[HasAVX]>;
+def : Pat<(f64 (fextend FR32:$src)),
+    (VCVTSS2SDrr FR32:$src, FR32:$src)>, Requires<[HasAVX]>;
+def : Pat<(fextend (loadf32 addr:$src)),
+    (VCVTSS2SDrm (f32 (IMPLICIT_DEF)), addr:$src)>, Requires<[HasAVX]>;
 
-  def : Pat<(extloadf32 addr:$src),
-            (VCVTSS2SDrm (f32 (IMPLICIT_DEF)), addr:$src)>,
-            Requires<[HasAVX, OptForSize]>;
-  def : Pat<(extloadf32 addr:$src),
-            (VCVTSS2SDrr (f32 (IMPLICIT_DEF)), (VMOVSSrm addr:$src))>,
-            Requires<[HasAVX, OptForSpeed]>;
-} // AddedComplexity = 1
+def : Pat<(extloadf32 addr:$src),
+    (VCVTSS2SDrm (f32 (IMPLICIT_DEF)), addr:$src)>,
+    Requires<[HasAVX, OptForSize]>;
+def : Pat<(extloadf32 addr:$src),
+    (VCVTSS2SDrr (f32 (IMPLICIT_DEF)), (VMOVSSrm addr:$src))>,
+    Requires<[HasAVX, OptForSpeed]>;
 
 def CVTSS2SDrr : I<0x5A, MRMSrcReg, (outs FR64:$dst), (ins FR32:$src),
                    "cvtss2sd\t{$src, $dst|$dst, $src}",
                    [(set FR64:$dst, (fextend FR32:$src))],
                    IIC_SSE_CVT_Scalar_RR>, XS,
-                 Requires<[HasSSE2]>;
+                 Requires<[UseSSE2]>;
 def CVTSS2SDrm : I<0x5A, MRMSrcMem, (outs FR64:$dst), (ins f32mem:$src),
                    "cvtss2sd\t{$src, $dst|$dst, $src}",
                    [(set FR64:$dst, (extloadf32 addr:$src))],
                    IIC_SSE_CVT_Scalar_RM>, XS,
-                 Requires<[HasSSE2, OptForSize]>;
+                 Requires<[UseSSE2, OptForSize]>;
 
 // extload f32 -> f64.  This matches load+fextend because we have a hack in
 // the isel (PreprocessForFPConvert) that can introduce loads after dag
@@ -1740,9 +1769,9 @@ def CVTSS2SDrm : I<0x5A, MRMSrcMem, (outs FR64:$dst), (ins f32mem:$src),
 // Since these loads aren't folded into the fextend, we have to match it
 // explicitly here.
 def : Pat<(fextend (loadf32 addr:$src)),
-          (CVTSS2SDrm addr:$src)>, Requires<[HasSSE2]>;
+          (CVTSS2SDrm addr:$src)>, Requires<[UseSSE2]>;
 def : Pat<(extloadf32 addr:$src),
-          (CVTSS2SDrr (MOVSSrm addr:$src))>, Requires<[HasSSE2, OptForSpeed]>;
+          (CVTSS2SDrr (MOVSSrm addr:$src))>, Requires<[UseSSE2, OptForSpeed]>;
 
 def Int_VCVTSS2SDrr: I<0x5A, MRMSrcReg,
                       (outs VR128:$dst), (ins VR128:$src1, VR128:$src2),
@@ -1762,13 +1791,13 @@ def Int_CVTSS2SDrr: I<0x5A, MRMSrcReg,
                     "cvtss2sd\t{$src2, $dst|$dst, $src2}",
                     [(set VR128:$dst,
                       (int_x86_sse2_cvtss2sd VR128:$src1, VR128:$src2))],
-                    IIC_SSE_CVT_Scalar_RR>, XS, Requires<[HasSSE2]>;
+                    IIC_SSE_CVT_Scalar_RR>, XS, Requires<[UseSSE2]>;
 def Int_CVTSS2SDrm: I<0x5A, MRMSrcMem,
                       (outs VR128:$dst), (ins VR128:$src1, ssmem:$src2),
                     "cvtss2sd\t{$src2, $dst|$dst, $src2}",
                     [(set VR128:$dst,
                       (int_x86_sse2_cvtss2sd VR128:$src1, sse_load_f32:$src2))],
-                    IIC_SSE_CVT_Scalar_RM>, XS, Requires<[HasSSE2]>;
+                    IIC_SSE_CVT_Scalar_RM>, XS, Requires<[UseSSE2]>;
 }
 
 // Convert packed single/double fp to doubleword
@@ -1785,12 +1814,12 @@ def VCVTPS2DQYrr : VPDI<0x5B, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src),
                         "cvtps2dq\t{$src, $dst|$dst, $src}",
                         [(set VR256:$dst,
                           (int_x86_avx_cvt_ps2dq_256 VR256:$src))],
-                        IIC_SSE_CVT_PS_RR>, VEX;
+                        IIC_SSE_CVT_PS_RR>, VEX, VEX_L;
 def VCVTPS2DQYrm : VPDI<0x5B, MRMSrcMem, (outs VR256:$dst), (ins f256mem:$src),
                         "cvtps2dq\t{$src, $dst|$dst, $src}",
                         [(set VR256:$dst,
                           (int_x86_avx_cvt_ps2dq_256 (memopv8f32 addr:$src)))],
-                        IIC_SSE_CVT_PS_RM>, VEX;
+                        IIC_SSE_CVT_PS_RM>, VEX, VEX_L;
 def CVTPS2DQrr : PDI<0x5B, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
                      "cvtps2dq\t{$src, $dst|$dst, $src}",
                      [(set VR128:$dst, (int_x86_sse2_cvtps2dq VR128:$src))],
@@ -1824,7 +1853,7 @@ def VCVTPD2DQXrm : SDI<0xE6, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src),
 def VCVTPD2DQYrr : SDI<0xE6, MRMSrcReg, (outs VR128:$dst), (ins VR256:$src),
                        "vcvtpd2dq{y}\t{$src, $dst|$dst, $src}",
                        [(set VR128:$dst,
-                         (int_x86_avx_cvt_pd2dq_256 VR256:$src))]>, VEX;
+                         (int_x86_avx_cvt_pd2dq_256 VR256:$src))]>, VEX, VEX_L;
 def VCVTPD2DQYrm : SDI<0xE6, MRMSrcMem, (outs VR128:$dst), (ins f256mem:$src),
                        "vcvtpd2dq{y}\t{$src, $dst|$dst, $src}",
                        [(set VR128:$dst,
@@ -1860,12 +1889,12 @@ def VCVTTPS2DQYrr : VS2SI<0x5B, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src),
                           "cvttps2dq\t{$src, $dst|$dst, $src}",
                           [(set VR256:$dst,
                             (int_x86_avx_cvtt_ps2dq_256 VR256:$src))],
-                          IIC_SSE_CVT_PS_RR>, VEX;
+                          IIC_SSE_CVT_PS_RR>, VEX, VEX_L;
 def VCVTTPS2DQYrm : VS2SI<0x5B, MRMSrcMem, (outs VR256:$dst), (ins f256mem:$src),
                           "cvttps2dq\t{$src, $dst|$dst, $src}",
                           [(set VR256:$dst, (int_x86_avx_cvtt_ps2dq_256
                                              (memopv8f32 addr:$src)))],
-                          IIC_SSE_CVT_PS_RM>, VEX;
+                          IIC_SSE_CVT_PS_RM>, VEX, VEX_L;
 
 def CVTTPS2DQrr : S2SI<0x5B, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
                        "cvttps2dq\t{$src, $dst|$dst, $src}",
@@ -1904,7 +1933,7 @@ let Predicates = [HasAVX] in {
             (VCVTTPS2DQYrm addr:$src)>;
 }
 
-let Predicates = [HasSSE2] in {
+let Predicates = [UseSSE2] in {
   def : Pat<(v4f32 (sint_to_fp (v4i32 VR128:$src))),
             (CVTDQ2PSrr VR128:$src)>;
   def : Pat<(v4f32 (sint_to_fp (bc_v4i32 (memopv2i64 addr:$src)))),
@@ -1945,7 +1974,7 @@ def VCVTTPD2DQYrr : VPDI<0xE6, MRMSrcReg, (outs VR128:$dst), (ins VR256:$src),
                          "cvttpd2dq{y}\t{$src, $dst|$dst, $src}",
                          [(set VR128:$dst,
                            (int_x86_avx_cvtt_pd2dq_256 VR256:$src))],
-                         IIC_SSE_CVT_PD_RR>, VEX;
+                         IIC_SSE_CVT_PD_RR>, VEX, VEX_L;
 def VCVTTPD2DQYrm : VPDI<0xE6, MRMSrcMem, (outs VR128:$dst), (ins f256mem:$src),
                          "cvttpd2dq{y}\t{$src, $dst|$dst, $src}",
                          [(set VR128:$dst,
@@ -1978,31 +2007,31 @@ def VCVTPS2PDrr : I<0x5A, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
                      "vcvtps2pd\t{$src, $dst|$dst, $src}",
                      [(set VR128:$dst, (int_x86_sse2_cvtps2pd VR128:$src))],
                      IIC_SSE_CVT_PD_RR>, TB, VEX;
-let neverHasSideEffects = 1, mayLoad = 1 in
 def VCVTPS2PDrm : I<0x5A, MRMSrcMem, (outs VR128:$dst), (ins f64mem:$src),
-                     "vcvtps2pd\t{$src, $dst|$dst, $src}", [],
-                     IIC_SSE_CVT_PD_RM>, TB, VEX;
+                    "vcvtps2pd\t{$src, $dst|$dst, $src}",
+                    [(set VR128:$dst, (v2f64 (extloadv2f32 addr:$src)))],
+                    IIC_SSE_CVT_PD_RM>, TB, VEX;
 def VCVTPS2PDYrr : I<0x5A, MRMSrcReg, (outs VR256:$dst), (ins VR128:$src),
                      "vcvtps2pd\t{$src, $dst|$dst, $src}",
                      [(set VR256:$dst,
                        (int_x86_avx_cvt_ps2_pd_256 VR128:$src))],
-                     IIC_SSE_CVT_PD_RR>, TB, VEX;
+                     IIC_SSE_CVT_PD_RR>, TB, VEX, VEX_L;
 def VCVTPS2PDYrm : I<0x5A, MRMSrcMem, (outs VR256:$dst), (ins f128mem:$src),
                      "vcvtps2pd\t{$src, $dst|$dst, $src}",
                      [(set VR256:$dst,
                        (int_x86_avx_cvt_ps2_pd_256 (memopv4f32 addr:$src)))],
-                     IIC_SSE_CVT_PD_RM>, TB, VEX;
+                     IIC_SSE_CVT_PD_RM>, TB, VEX, VEX_L;
 }
 
-let Predicates = [HasSSE2] in {
+let Predicates = [UseSSE2] in {
 def CVTPS2PDrr : I<0x5A, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
                        "cvtps2pd\t{$src, $dst|$dst, $src}",
                        [(set VR128:$dst, (int_x86_sse2_cvtps2pd VR128:$src))],
                        IIC_SSE_CVT_PD_RR>, TB;
-let neverHasSideEffects = 1, mayLoad = 1 in
 def CVTPS2PDrm : I<0x5A, MRMSrcMem, (outs VR128:$dst), (ins f64mem:$src),
-                       "cvtps2pd\t{$src, $dst|$dst, $src}", [],
-                       IIC_SSE_CVT_PD_RM>, TB;
+                   "cvtps2pd\t{$src, $dst|$dst, $src}",
+                   [(set VR128:$dst, (v2f64 (extloadv2f32 addr:$src)))],
+                   IIC_SSE_CVT_PD_RM>, TB;
 }
 
 // Convert Packed DW Integers to Packed Double FP
@@ -2019,11 +2048,11 @@ def VCVTDQ2PDYrm  : S2SI<0xE6, MRMSrcMem, (outs VR256:$dst), (ins i128mem:$src),
                      "vcvtdq2pd\t{$src, $dst|$dst, $src}",
                      [(set VR256:$dst,
                        (int_x86_avx_cvtdq2_pd_256
-                        (bitconvert (memopv2i64 addr:$src))))]>, VEX;
+                        (bitconvert (memopv2i64 addr:$src))))]>, VEX, VEX_L;
 def VCVTDQ2PDYrr  : S2SI<0xE6, MRMSrcReg, (outs VR256:$dst), (ins VR128:$src),
                      "vcvtdq2pd\t{$src, $dst|$dst, $src}",
                      [(set VR256:$dst,
-                       (int_x86_avx_cvtdq2_pd_256 VR128:$src))]>, VEX;
+                       (int_x86_avx_cvtdq2_pd_256 VR128:$src))]>, VEX, VEX_L;
 }
 
 let neverHasSideEffects = 1, mayLoad = 1 in
@@ -2066,7 +2095,7 @@ def VCVTPD2PSYrr : VPDI<0x5A, MRMSrcReg, (outs VR128:$dst), (ins VR256:$src),
                         "cvtpd2ps{y}\t{$src, $dst|$dst, $src}",
                         [(set VR128:$dst,
                           (int_x86_avx_cvt_pd2_ps_256 VR256:$src))],
-                        IIC_SSE_CVT_PD_RR>, VEX;
+                        IIC_SSE_CVT_PD_RR>, VEX, VEX_L;
 def VCVTPD2PSYrm : VPDI<0x5A, MRMSrcMem, (outs VR128:$dst), (ins f256mem:$src),
                         "cvtpd2ps{y}\t{$src, $dst|$dst, $src}",
                         [(set VR128:$dst,
@@ -2096,6 +2125,10 @@ let Predicates = [HasAVX] in {
             (VCVTDQ2PSYrm addr:$src)>;
 
   // Match fround and fextend for 128/256-bit conversions
+  def : Pat<(v4f32 (X86vfpround (v2f64 VR128:$src))),
+            (VCVTPD2PSrr VR128:$src)>;
+  def : Pat<(v4f32 (X86vfpround (memopv2f64 addr:$src))),
+            (VCVTPD2PSXrm addr:$src)>;
   def : Pat<(v4f32 (fround (v4f64 VR256:$src))),
             (VCVTPD2PSYrr VR256:$src)>;
   def : Pat<(v4f32 (fround (loadv4f64 addr:$src))),
@@ -2105,12 +2138,17 @@ let Predicates = [HasAVX] in {
             (VCVTPS2PDrr VR128:$src)>;
   def : Pat<(v4f64 (fextend (v4f32 VR128:$src))),
             (VCVTPS2PDYrr VR128:$src)>;
-  def : Pat<(v4f64 (fextend (loadv4f32 addr:$src))),
+  def : Pat<(v4f64 (extloadv4f32 addr:$src)),
             (VCVTPS2PDYrm addr:$src)>;
 }
 
-let Predicates = [HasSSE2] in {
-  // Match fextend for 128 conversions
+let Predicates = [UseSSE2] in {
+  // Match fround and fextend for 128 conversions
+  def : Pat<(v4f32 (X86vfpround (v2f64 VR128:$src))),
+            (CVTPD2PSrr VR128:$src)>;
+  def : Pat<(v4f32 (X86vfpround (memopv2f64 addr:$src))),
+            (CVTPD2PSrm addr:$src)>;
+
   def : Pat<(v2f64 (X86vfpext (v4f32 VR128:$src))),
             (CVTPS2PDrr VR128:$src)>;
 }
@@ -2121,7 +2159,7 @@ let Predicates = [HasSSE2] in {
 
 // sse12_cmp_scalar - sse 1 & 2 compare scalar instructions
 multiclass sse12_cmp_scalar<RegisterClass RC, X86MemOperand x86memop,
-                            Operand CC, SDNode OpNode, ValueType VT, 
+                            Operand CC, SDNode OpNode, ValueType VT,
                             PatFrag ld_frag, string asm, string asm_alt,
                             OpndItins itins> {
   def rr : SIi8<0xC2, MRMSrcReg,
@@ -2267,7 +2305,7 @@ let Defs = [EFLAGS] in {
 
 // sse12_cmp_packed - sse 1 & 2 compare packed instructions
 multiclass sse12_cmp_packed<RegisterClass RC, X86MemOperand x86memop,
-                            Operand CC, Intrinsic Int, string asm, 
+                            Operand CC, Intrinsic Int, string asm,
                             string asm_alt, Domain d> {
   def rri : PIi8<0xC2, MRMSrcReg,
              (outs RC:$dst), (ins RC:$src1, RC:$src2, CC:$cc), asm,
@@ -2300,11 +2338,11 @@ defm VCMPPD : sse12_cmp_packed<VR128, f128mem, AVXCC, int_x86_sse2_cmp_pd,
 defm VCMPPSY : sse12_cmp_packed<VR256, f256mem, AVXCC, int_x86_avx_cmp_ps_256,
                "cmp${cc}ps\t{$src2, $src1, $dst|$dst, $src1, $src2}",
                "cmpps\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}",
-               SSEPackedSingle>, TB, VEX_4V;
+               SSEPackedSingle>, TB, VEX_4V, VEX_L;
 defm VCMPPDY : sse12_cmp_packed<VR256, f256mem, AVXCC, int_x86_avx_cmp_pd_256,
                "cmp${cc}pd\t{$src2, $src1, $dst|$dst, $src1, $src2}",
                "cmppd\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}",
-               SSEPackedDouble>, TB, OpSize, VEX_4V;
+               SSEPackedDouble>, TB, OpSize, VEX_4V, VEX_L;
 let Constraints = "$src1 = $dst" in {
   defm CMPPS : sse12_cmp_packed<VR128, f128mem, SSECC, int_x86_sse_cmp_ps,
                  "cmp${cc}ps\t{$src2, $dst|$dst, $src2}",
@@ -2336,14 +2374,14 @@ def : Pat<(v4i64 (X86cmpp (v4f64 VR256:$src1), (memop addr:$src2), imm:$cc)),
           (VCMPPDYrmi VR256:$src1, addr:$src2, imm:$cc)>;
 }
 
-let Predicates = [HasSSE1] in {
+let Predicates = [UseSSE1] in {
 def : Pat<(v4i32 (X86cmpp (v4f32 VR128:$src1), VR128:$src2, imm:$cc)),
           (CMPPSrri (v4f32 VR128:$src1), (v4f32 VR128:$src2), imm:$cc)>;
 def : Pat<(v4i32 (X86cmpp (v4f32 VR128:$src1), (memop addr:$src2), imm:$cc)),
           (CMPPSrmi (v4f32 VR128:$src1), addr:$src2, imm:$cc)>;
 }
 
-let Predicates = [HasSSE2] in {
+let Predicates = [UseSSE2] in {
 def : Pat<(v2i64 (X86cmpp (v2f64 VR128:$src1), VR128:$src2, imm:$cc)),
           (CMPPDrri VR128:$src1, VR128:$src2, imm:$cc)>;
 def : Pat<(v2i64 (X86cmpp (v2f64 VR128:$src1), (memop addr:$src2), imm:$cc)),
@@ -2374,13 +2412,13 @@ defm VSHUFPS  : sse12_shuffle<VR128, f128mem, v4f32,
            memopv4f32, SSEPackedSingle>, TB, VEX_4V;
 defm VSHUFPSY : sse12_shuffle<VR256, f256mem, v8f32,
            "shufps\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
-           memopv8f32, SSEPackedSingle>, TB, VEX_4V;
+           memopv8f32, SSEPackedSingle>, TB, VEX_4V, VEX_L;
 defm VSHUFPD  : sse12_shuffle<VR128, f128mem, v2f64,
            "shufpd\t{$src3, $src2, $src1, $dst|$dst, $src2, $src2, $src3}",
            memopv2f64, SSEPackedDouble>, TB, OpSize, VEX_4V;
 defm VSHUFPDY : sse12_shuffle<VR256, f256mem, v4f64,
            "shufpd\t{$src3, $src2, $src1, $dst|$dst, $src2, $src2, $src3}",
-           memopv4f64, SSEPackedDouble>, TB, OpSize, VEX_4V;
+           memopv4f64, SSEPackedDouble>, TB, OpSize, VEX_4V, VEX_L;
 
 let Constraints = "$src1 = $dst" in {
   defm SHUFPS : sse12_shuffle<VR128, f128mem, v4f32,
@@ -2420,7 +2458,7 @@ let Predicates = [HasAVX] in {
             (VSHUFPDYrmi VR256:$src1, addr:$src2, imm:$imm)>;
 }
 
-let Predicates = [HasSSE1] in {
+let Predicates = [UseSSE1] in {
   def : Pat<(v4i32 (X86Shufp VR128:$src1,
                        (bc_v4i32 (memopv2i64 addr:$src2)), (i8 imm:$imm))),
             (SHUFPSrmi VR128:$src1, addr:$src2, imm:$imm)>;
@@ -2428,7 +2466,7 @@ let Predicates = [HasSSE1] in {
             (SHUFPSrri VR128:$src1, VR128:$src2, imm:$imm)>;
 }
 
-let Predicates = [HasSSE2] in {
+let Predicates = [UseSSE2] in {
   // Generic SHUFPD patterns
   def : Pat<(v2i64 (X86Shufp VR128:$src1,
                        (memopv2i64 addr:$src2), (i8 imm:$imm))),
@@ -2474,16 +2512,16 @@ defm VUNPCKLPD: sse12_unpack_interleave<0x14, X86Unpckl, v2f64, memopv2f64,
 
 defm VUNPCKHPSY: sse12_unpack_interleave<0x15, X86Unpckh, v8f32, memopv8f32,
       VR256, f256mem, "unpckhps\t{$src2, $src1, $dst|$dst, $src1, $src2}",
-                     SSEPackedSingle>, TB, VEX_4V;
+                     SSEPackedSingle>, TB, VEX_4V, VEX_L;
 defm VUNPCKHPDY: sse12_unpack_interleave<0x15, X86Unpckh, v4f64, memopv4f64,
       VR256, f256mem, "unpckhpd\t{$src2, $src1, $dst|$dst, $src1, $src2}",
-                     SSEPackedDouble>, TB, OpSize, VEX_4V;
+                     SSEPackedDouble>, TB, OpSize, VEX_4V, VEX_L;
 defm VUNPCKLPSY: sse12_unpack_interleave<0x14, X86Unpckl, v8f32, memopv8f32,
       VR256, f256mem, "unpcklps\t{$src2, $src1, $dst|$dst, $src1, $src2}",
-                     SSEPackedSingle>, TB, VEX_4V;
+                     SSEPackedSingle>, TB, VEX_4V, VEX_L;
 defm VUNPCKLPDY: sse12_unpack_interleave<0x14, X86Unpckl, v4f64, memopv4f64,
       VR256, f256mem, "unpcklpd\t{$src2, $src1, $dst|$dst, $src1, $src2}",
-                     SSEPackedDouble>, TB, OpSize, VEX_4V;
+                     SSEPackedDouble>, TB, OpSize, VEX_4V, VEX_L;
 
 let Constraints = "$src1 = $dst" in {
   defm UNPCKHPS: sse12_unpack_interleave<0x15, X86Unpckh, v4f32, memopv4f32,
@@ -2500,7 +2538,27 @@ let Constraints = "$src1 = $dst" in {
                        SSEPackedDouble>, TB, OpSize;
 } // Constraints = "$src1 = $dst"
 
-let Predicates = [HasAVX], AddedComplexity = 1 in {
+let Predicates = [HasAVX1Only] in {
+  def : Pat<(v8i32 (X86Unpckl VR256:$src1, (bc_v8i32 (memopv4i64 addr:$src2)))),
+            (VUNPCKLPSYrm VR256:$src1, addr:$src2)>;
+  def : Pat<(v8i32 (X86Unpckl VR256:$src1, VR256:$src2)),
+            (VUNPCKLPSYrr VR256:$src1, VR256:$src2)>;
+  def : Pat<(v8i32 (X86Unpckh VR256:$src1, (bc_v8i32 (memopv4i64 addr:$src2)))),
+            (VUNPCKHPSYrm VR256:$src1, addr:$src2)>;
+  def : Pat<(v8i32 (X86Unpckh VR256:$src1, VR256:$src2)),
+            (VUNPCKHPSYrr VR256:$src1, VR256:$src2)>;
+
+  def : Pat<(v4i64 (X86Unpckl VR256:$src1, (memopv4i64 addr:$src2))),
+            (VUNPCKLPDYrm VR256:$src1, addr:$src2)>;
+  def : Pat<(v4i64 (X86Unpckl VR256:$src1, VR256:$src2)),
+            (VUNPCKLPDYrr VR256:$src1, VR256:$src2)>;
+  def : Pat<(v4i64 (X86Unpckh VR256:$src1, (memopv4i64 addr:$src2))),
+            (VUNPCKHPDYrm VR256:$src1, addr:$src2)>;
+  def : Pat<(v4i64 (X86Unpckh VR256:$src1, VR256:$src2)),
+            (VUNPCKHPDYrr VR256:$src1, VR256:$src2)>;
+}
+
+let Predicates = [HasAVX] in {
   // FIXME: Instead of X86Movddup, there should be a X86Unpckl here, the
   // problem is during lowering, where it's not possible to recognize the load
   // fold cause it has two uses through a bitcast. One use disappears at isel
@@ -2509,7 +2567,7 @@ let Predicates = [HasAVX], AddedComplexity = 1 in {
             (VUNPCKLPDrr VR128:$src, VR128:$src)>;
 }
 
-let Predicates = [HasSSE2] in {
+let Predicates = [UseSSE2] in {
   // FIXME: Instead of X86Movddup, there should be a X86Unpckl here, the
   // problem is during lowering, where it's not possible to recognize the load
   // fold cause it has two uses through a bitcast. One use disappears at isel
@@ -2540,10 +2598,11 @@ let Predicates = [HasAVX] in {
                                         "movmskpd", SSEPackedDouble>, TB,
                                         OpSize, VEX;
   defm VMOVMSKPSY : sse12_extr_sign_mask<VR256, int_x86_avx_movmsk_ps_256,
-                                        "movmskps", SSEPackedSingle>, TB, VEX;
+                                        "movmskps", SSEPackedSingle>, TB,
+                                        VEX, VEX_L;
   defm VMOVMSKPDY : sse12_extr_sign_mask<VR256, int_x86_avx_movmsk_pd_256,
                                         "movmskpd", SSEPackedDouble>, TB,
-                                        OpSize, VEX;
+                                        OpSize, VEX, VEX_L;
 
   def : Pat<(i32 (X86fgetsign FR32:$src)),
             (VMOVMSKPSrr32 (COPY_TO_REGCLASS FR32:$src, VR128))>;
@@ -2564,11 +2623,11 @@ let Predicates = [HasAVX] in {
              OpSize, VEX;
   def VMOVMSKPSYr64r : PI<0x50, MRMSrcReg, (outs GR64:$dst), (ins VR256:$src),
              "movmskps\t{$src, $dst|$dst, $src}", [], IIC_SSE_MOVMSK,
-             SSEPackedSingle>, TB, VEX;
+             SSEPackedSingle>, TB, VEX, VEX_L;
   def VMOVMSKPDYr64r : PI<0x50, MRMSrcReg, (outs GR64:$dst), (ins VR256:$src),
              "movmskpd\t{$src, $dst|$dst, $src}", [], IIC_SSE_MOVMSK,
              SSEPackedDouble>, TB,
-             OpSize, VEX;
+             OpSize, VEX, VEX_L;
 }
 
 defm MOVMSKPS : sse12_extr_sign_mask<VR128, int_x86_sse_movmsk_ps, "movmskps",
@@ -2578,16 +2637,16 @@ defm MOVMSKPD : sse12_extr_sign_mask<VR128, int_x86_sse2_movmsk_pd, "movmskpd",
 
 def : Pat<(i32 (X86fgetsign FR32:$src)),
           (MOVMSKPSrr32 (COPY_TO_REGCLASS FR32:$src, VR128))>,
-      Requires<[HasSSE1]>;
+      Requires<[UseSSE1]>;
 def : Pat<(i64 (X86fgetsign FR32:$src)),
           (MOVMSKPSrr64 (COPY_TO_REGCLASS FR32:$src, VR128))>,
-      Requires<[HasSSE1]>;
+      Requires<[UseSSE1]>;
 def : Pat<(i32 (X86fgetsign FR64:$src)),
           (MOVMSKPDrr32 (COPY_TO_REGCLASS FR64:$src, VR128))>,
-      Requires<[HasSSE2]>;
+      Requires<[UseSSE2]>;
 def : Pat<(i64 (X86fgetsign FR64:$src)),
           (MOVMSKPDrr64 (COPY_TO_REGCLASS FR64:$src, VR128))>,
-      Requires<[HasSSE2]>;
+      Requires<[UseSSE2]>;
 
 //===---------------------------------------------------------------------===//
 // SSE2 - Packed Integer Logical Instructions
@@ -2646,13 +2705,13 @@ defm PANDN : PDI_binop_rm<0xDF, "pandn", X86andnp, v2i64, VR128, memopv2i64,
 
 let Predicates = [HasAVX2] in {
 defm VPANDY : PDI_binop_rm<0xDB, "vpand", and, v4i64, VR256, memopv4i64,
-                           i256mem, SSE_BIT_ITINS_P, 1, 0>, VEX_4V;
+                           i256mem, SSE_BIT_ITINS_P, 1, 0>, VEX_4V, VEX_L;
 defm VPORY  : PDI_binop_rm<0xEB, "vpor", or, v4i64, VR256, memopv4i64,
-                           i256mem, SSE_BIT_ITINS_P, 1, 0>, VEX_4V;
+                           i256mem, SSE_BIT_ITINS_P, 1, 0>, VEX_4V, VEX_L;
 defm VPXORY : PDI_binop_rm<0xEF, "vpxor", xor, v4i64, VR256, memopv4i64,
-                           i256mem, SSE_BIT_ITINS_P, 1, 0>, VEX_4V;
+                           i256mem, SSE_BIT_ITINS_P, 1, 0>, VEX_4V, VEX_L;
 defm VPANDNY : PDI_binop_rm<0xDF, "vpandn", X86andnp, v4i64, VR256, memopv4i64,
-                            i256mem, SSE_BIT_ITINS_P, 0, 0>, VEX_4V;
+                            i256mem, SSE_BIT_ITINS_P, 0, 0>, VEX_4V, VEX_L;
 }
 
 //===----------------------------------------------------------------------===//
@@ -2683,14 +2742,12 @@ multiclass sse12_fp_alias_pack_logical<bits<8> opc, string OpcodeStr,
 }
 
 // Alias bitwise logical operations using SSE logical ops on packed FP values.
-let mayLoad = 0 in {
-  defm FsAND  : sse12_fp_alias_pack_logical<0x54, "and", X86fand,
-                SSE_BIT_ITINS_P>;
-  defm FsOR   : sse12_fp_alias_pack_logical<0x56, "or", X86for,
-                SSE_BIT_ITINS_P>;
-  defm FsXOR  : sse12_fp_alias_pack_logical<0x57, "xor", X86fxor,
-                SSE_BIT_ITINS_P>;
-}
+defm FsAND  : sse12_fp_alias_pack_logical<0x54, "and", X86fand,
+              SSE_BIT_ITINS_P>;
+defm FsOR   : sse12_fp_alias_pack_logical<0x56, "or", X86for,
+              SSE_BIT_ITINS_P>;
+defm FsXOR  : sse12_fp_alias_pack_logical<0x57, "xor", X86fxor,
+              SSE_BIT_ITINS_P>;
 
 let neverHasSideEffects = 1, Pattern = []<dag>, isCommutable = 0 in
   defm FsANDN : sse12_fp_alias_pack_logical<0x55, "andn", undef,
@@ -2740,7 +2797,7 @@ multiclass sse12_fp_packed_logical_y<bits<8> opc, string OpcodeStr,
           !strconcat(OpcodeStr, "ps"), f256mem,
           [(set VR256:$dst, (v4i64 (OpNode VR256:$src1, VR256:$src2)))],
           [(set VR256:$dst, (OpNode (bc_v4i64 (v8f32 VR256:$src1)),
-                                    (memopv4i64 addr:$src2)))], 0>, TB, VEX_4V;
+                             (memopv4i64 addr:$src2)))], 0>, TB, VEX_4V, VEX_L;
 
     defm PDY : sse12_fp_packed_logical_rm<opc, VR256, SSEPackedDouble,
           !strconcat(OpcodeStr, "pd"), f256mem,
@@ -2748,7 +2805,7 @@ multiclass sse12_fp_packed_logical_y<bits<8> opc, string OpcodeStr,
                                     (bc_v4i64 (v4f64 VR256:$src2))))],
           [(set VR256:$dst, (OpNode (bc_v4i64 (v4f64 VR256:$src1)),
                                     (memopv4i64 addr:$src2)))], 0>,
-                                    TB, OpSize, VEX_4V;
+                                    TB, OpSize, VEX_4V, VEX_L;
 }
 
 // AVX 256-bit packed logical ops forms
@@ -2794,27 +2851,23 @@ multiclass basic_sse12_fp_binop_s<bits<8> opc, string OpcodeStr, SDNode OpNode,
 multiclass basic_sse12_fp_binop_p<bits<8> opc, string OpcodeStr, SDNode OpNode,
                                    SizeItins itins,
                                    bit Is2Addr = 1> {
-  let mayLoad = 0 in {
   defm PS : sse12_fp_packed<opc, !strconcat(OpcodeStr, "ps"), OpNode, VR128,
               v4f32, f128mem, memopv4f32, SSEPackedSingle, itins.s, Is2Addr>,
               TB;
   defm PD : sse12_fp_packed<opc, !strconcat(OpcodeStr, "pd"), OpNode, VR128,
               v2f64, f128mem, memopv2f64, SSEPackedDouble, itins.d, Is2Addr>,
               TB, OpSize;
-  }
 }
 
 multiclass basic_sse12_fp_binop_p_y<bits<8> opc, string OpcodeStr,
                                     SDNode OpNode,
                                     SizeItins itins> {
-  let mayLoad = 0 in {
-    defm PSY : sse12_fp_packed<opc, !strconcat(OpcodeStr, "ps"), OpNode, VR256,
+  defm PSY : sse12_fp_packed<opc, !strconcat(OpcodeStr, "ps"), OpNode, VR256,
                 v8f32, f256mem, memopv8f32, SSEPackedSingle, itins.s, 0>,
-                TB;
-    defm PDY : sse12_fp_packed<opc, !strconcat(OpcodeStr, "pd"), OpNode, VR256,
+                TB, VEX_L;
+  defm PDY : sse12_fp_packed<opc, !strconcat(OpcodeStr, "pd"), OpNode, VR256,
                 v4f64, f256mem, memopv4f64, SSEPackedDouble, itins.d, 0>,
-                TB, OpSize;
-  }
+                TB, OpSize, VEX_L;
 }
 
 multiclass basic_sse12_fp_binop_s_int<bits<8> opc, string OpcodeStr,
@@ -2846,11 +2899,11 @@ multiclass basic_sse12_fp_binop_p_y_int<bits<8> opc, string OpcodeStr,
                                         SizeItins itins> {
   defm PSY : sse12_fp_packed_int<opc, OpcodeStr, VR256,
      !strconcat(OpcodeStr, "ps"), "avx", "_ps_256", f256mem, memopv8f32,
-      SSEPackedSingle, itins.s, 0>, TB;
+      SSEPackedSingle, itins.s, 0>, TB, VEX_L;
 
   defm PDY : sse12_fp_packed_int<opc, OpcodeStr, VR256,
      !strconcat(OpcodeStr, "pd"), "avx", "_pd_256", f256mem, memopv4f64,
-      SSEPackedDouble, itins.d, 0>, TB, OpSize;
+      SSEPackedDouble, itins.d, 0>, TB, OpSize, VEX_L;
 }
 
 // Binary Arithmetic instructions
@@ -2872,7 +2925,8 @@ let isCommutable = 0 in {
               basic_sse12_fp_binop_s_int<0x5C, "sub", SSE_ALU_ITINS_S, 0>,
                 VEX_4V, VEX_LIG;
   defm VSUB : basic_sse12_fp_binop_p<0x5C, "sub", fsub, SSE_ALU_ITINS_P, 0>,
-              basic_sse12_fp_binop_p_y<0x5C, "sub", fsub, SSE_ALU_ITINS_P>, VEX_4V;
+              basic_sse12_fp_binop_p_y<0x5C, "sub", fsub, SSE_ALU_ITINS_P>,
+                VEX_4V;
   defm VDIV : basic_sse12_fp_binop_s<0x5E, "div", fdiv, SSE_DIV_ITINS_S, 0>,
               basic_sse12_fp_binop_s_int<0x5E, "div", SSE_DIV_ITINS_S, 0>,
                 VEX_4V, VEX_LIG;
@@ -2923,6 +2977,23 @@ let Constraints = "$src1 = $dst" in {
   }
 }
 
+let isCodeGenOnly = 1 in {
+  defm VMAXC: basic_sse12_fp_binop_s<0x5F, "max", X86fmaxc, SSE_ALU_ITINS_S, 0>,
+       VEX_4V, VEX_LIG;
+  defm VMAXC: basic_sse12_fp_binop_p<0x5F, "max", X86fmaxc, SSE_ALU_ITINS_P, 0>,
+       basic_sse12_fp_binop_p_y<0x5F, "max", X86fmaxc, SSE_ALU_ITINS_P>, VEX_4V;
+  defm VMINC: basic_sse12_fp_binop_s<0x5D, "min", X86fminc, SSE_ALU_ITINS_S, 0>,
+       VEX_4V, VEX_LIG;
+  defm VMINC: basic_sse12_fp_binop_p<0x5D, "min", X86fminc, SSE_ALU_ITINS_P, 0>,
+       basic_sse12_fp_binop_p_y<0x5D, "min", X86fminc, SSE_ALU_ITINS_P>, VEX_4V;
+  let Constraints = "$src1 = $dst" in {
+    defm MAXC: basic_sse12_fp_binop_s<0x5F, "max", X86fmaxc, SSE_ALU_ITINS_S>,
+         basic_sse12_fp_binop_p<0x5F, "max", X86fmaxc, SSE_ALU_ITINS_P>;
+    defm MINC: basic_sse12_fp_binop_s<0x5D, "min", X86fminc, SSE_ALU_ITINS_S>,
+         basic_sse12_fp_binop_p<0x5D, "min", X86fminc, SSE_ALU_ITINS_P>;
+  }
+}
+
 /// Unop Arithmetic
 /// In addition, we also have a special variant of the scalar form here to
 /// represent the associated intrinsic operation.  This form is unlike the
@@ -2960,7 +3031,7 @@ multiclass sse1_fp_unop_s<bits<8> opc, string OpcodeStr,
   def SSm : I<opc, MRMSrcMem, (outs FR32:$dst), (ins f32mem:$src),
                 !strconcat(OpcodeStr, "ss\t{$src, $dst|$dst, $src}"),
                 [(set FR32:$dst, (OpNode (load addr:$src)))], itins.rm>, XS,
-            Requires<[HasSSE1, OptForSize]>;
+            Requires<[UseSSE1, OptForSize]>;
   def SSr_Int : SSI<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
                     !strconcat(OpcodeStr, "ss\t{$src, $dst|$dst, $src}"),
                     [(set VR128:$dst, (F32Int VR128:$src))], itins.rr>;
@@ -2974,7 +3045,7 @@ multiclass sse1_fp_unop_s_avx<bits<8> opc, string OpcodeStr> {
   def SSr : SSI<opc, MRMSrcReg, (outs FR32:$dst), (ins FR32:$src1, FR32:$src2),
                 !strconcat(OpcodeStr,
                            "ss\t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>;
-  let mayLoad = 1 in
+  let mayLoad = 1 in {
   def SSm : SSI<opc, MRMSrcMem, (outs FR32:$dst), (ins FR32:$src1,f32mem:$src2),
                 !strconcat(OpcodeStr,
                            "ss\t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>;
@@ -2982,6 +3053,7 @@ multiclass sse1_fp_unop_s_avx<bits<8> opc, string OpcodeStr> {
                 (ins VR128:$src1, ssmem:$src2),
                 !strconcat(OpcodeStr,
                            "ss\t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>;
+  }
 }
 
 /// sse1_fp_unop_p - SSE1 unops in packed form.
@@ -3001,11 +3073,11 @@ multiclass sse1_fp_unop_p_y<bits<8> opc, string OpcodeStr, SDNode OpNode,
   def PSYr : PSI<opc, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src),
               !strconcat(OpcodeStr, "ps\t{$src, $dst|$dst, $src}"),
               [(set VR256:$dst, (v8f32 (OpNode VR256:$src)))],
-              itins.rr>;
+              itins.rr>, VEX_L;
   def PSYm : PSI<opc, MRMSrcMem, (outs VR256:$dst), (ins f256mem:$src),
                 !strconcat(OpcodeStr, "ps\t{$src, $dst|$dst, $src}"),
                 [(set VR256:$dst, (OpNode (memopv8f32 addr:$src)))],
-                itins.rm>;
+                itins.rm>, VEX_L;
 }
 
 /// sse1_fp_unop_p_int - SSE1 intrinsics unops in packed forms.
@@ -3027,11 +3099,11 @@ multiclass sse1_fp_unop_p_y_int<bits<8> opc, string OpcodeStr,
   def PSYr_Int : PSI<opc, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src),
                     !strconcat(OpcodeStr, "ps\t{$src, $dst|$dst, $src}"),
                     [(set VR256:$dst, (V4F32Int VR256:$src))],
-                    itins.rr>;
+                    itins.rr>, VEX_L;
   def PSYm_Int : PSI<opc, MRMSrcMem, (outs VR256:$dst), (ins f256mem:$src),
                     !strconcat(OpcodeStr, "ps\t{$src, $dst|$dst, $src}"),
                     [(set VR256:$dst, (V4F32Int (memopv8f32 addr:$src)))],
-                    itins.rm>;
+                    itins.rm>, VEX_L;
 }
 
 /// sse2_fp_unop_s - SSE2 unops in scalar form.
@@ -3044,7 +3116,7 @@ multiclass sse2_fp_unop_s<bits<8> opc, string OpcodeStr,
   def SDm : I<opc, MRMSrcMem, (outs FR64:$dst), (ins f64mem:$src),
                 !strconcat(OpcodeStr, "sd\t{$src, $dst|$dst, $src}"),
                 [(set FR64:$dst, (OpNode (load addr:$src)))], itins.rm>, XD,
-            Requires<[HasSSE2, OptForSize]>;
+            Requires<[UseSSE2, OptForSize]>;
   def SDr_Int : SDI<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
                     !strconcat(OpcodeStr, "sd\t{$src, $dst|$dst, $src}"),
                     [(set VR128:$dst, (F64Int VR128:$src))], itins.rr>;
@@ -3054,20 +3126,20 @@ multiclass sse2_fp_unop_s<bits<8> opc, string OpcodeStr,
 }
 
 /// sse2_fp_unop_s_avx - AVX SSE2 unops in scalar form.
+let hasSideEffects = 0 in
 multiclass sse2_fp_unop_s_avx<bits<8> opc, string OpcodeStr> {
-  let neverHasSideEffects = 1 in {
   def SDr : SDI<opc, MRMSrcReg, (outs FR64:$dst), (ins FR64:$src1, FR64:$src2),
                !strconcat(OpcodeStr,
                           "sd\t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>;
-  let mayLoad = 1 in
+  let mayLoad = 1 in {
   def SDm : SDI<opc, MRMSrcMem, (outs FR64:$dst), (ins FR64:$src1,f64mem:$src2),
                !strconcat(OpcodeStr,
                           "sd\t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>;
-  }
   def SDm_Int : SDI<opc, MRMSrcMem, (outs VR128:$dst),
                (ins VR128:$src1, sdmem:$src2),
                !strconcat(OpcodeStr,
                           "sd\t{$src2, $src1, $dst|$dst, $src1, $src2}"), []>;
+  }
 }
 
 /// sse2_fp_unop_p - SSE2 unops in vector forms.
@@ -3087,11 +3159,11 @@ multiclass sse2_fp_unop_p_y<bits<8> opc, string OpcodeStr, SDNode OpNode,
   def PDYr : PDI<opc, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src),
               !strconcat(OpcodeStr, "pd\t{$src, $dst|$dst, $src}"),
               [(set VR256:$dst, (v4f64 (OpNode VR256:$src)))],
-              itins.rr>;
+              itins.rr>, VEX_L;
   def PDYm : PDI<opc, MRMSrcMem, (outs VR256:$dst), (ins f256mem:$src),
                 !strconcat(OpcodeStr, "pd\t{$src, $dst|$dst, $src}"),
                 [(set VR256:$dst, (OpNode (memopv4f64 addr:$src)))],
-                itins.rm>;
+                itins.rm>, VEX_L;
 }
 
 /// sse2_fp_unop_p_int - SSE2 intrinsic unops in vector forms.
@@ -3113,11 +3185,11 @@ multiclass sse2_fp_unop_p_y_int<bits<8> opc, string OpcodeStr,
   def PDYr_Int : PDI<opc, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src),
                     !strconcat(OpcodeStr, "pd\t{$src, $dst|$dst, $src}"),
                     [(set VR256:$dst, (V2F64Int VR256:$src))],
-                    itins.rr>;
+                    itins.rr>, VEX_L;
   def PDYm_Int : PDI<opc, MRMSrcMem, (outs VR256:$dst), (ins f256mem:$src),
                     !strconcat(OpcodeStr, "pd\t{$src, $dst|$dst, $src}"),
                     [(set VR256:$dst, (V2F64Int (memopv4f64 addr:$src)))],
-                    itins.rm>;
+                    itins.rm>, VEX_L;
 }
 
 let Predicates = [HasAVX] in {
@@ -3158,7 +3230,6 @@ let Predicates = [HasAVX] in {
                                     SSE_RCPP>, VEX;
 }
 
-let AddedComplexity = 1 in {
 def : Pat<(f32 (fsqrt FR32:$src)),
           (VSQRTSSr (f32 (IMPLICIT_DEF)), FR32:$src)>, Requires<[HasAVX]>;
 def : Pat<(f32 (fsqrt (load addr:$src))),
@@ -3181,9 +3252,8 @@ def : Pat<(f32 (X86frcp FR32:$src)),
 def : Pat<(f32 (X86frcp (load addr:$src))),
           (VRCPSSm (f32 (IMPLICIT_DEF)), addr:$src)>,
           Requires<[HasAVX, OptForSize]>;
-}
 
-let Predicates = [HasAVX], AddedComplexity = 1 in {
+let Predicates = [HasAVX] in {
   def : Pat<(int_x86_sse_sqrt_ss VR128:$src),
             (COPY_TO_REGCLASS (VSQRTSSr (f32 (IMPLICIT_DEF)),
                                         (COPY_TO_REGCLASS VR128:$src, FR32)),
@@ -3223,17 +3293,52 @@ defm SQRT  : sse1_fp_unop_s<0x51, "sqrt",  fsqrt, int_x86_sse_sqrt_ss,
              sse2_fp_unop_p<0x51, "sqrt",  fsqrt, SSE_SQRTS>,
              sse2_fp_unop_p_int<0x51, "sqrt", int_x86_sse2_sqrt_pd, SSE_SQRTS>;
 
+/// sse1_fp_unop_s_rw - SSE1 unops where vector form has a read-write operand.
+multiclass sse1_fp_unop_rw<bits<8> opc, string OpcodeStr, SDNode OpNode,
+                               Intrinsic F32Int, OpndItins itins> {
+  def SSr : SSI<opc, MRMSrcReg, (outs FR32:$dst), (ins FR32:$src),
+                !strconcat(OpcodeStr, "ss\t{$src, $dst|$dst, $src}"),
+                [(set FR32:$dst, (OpNode FR32:$src))]>;
+  // For scalar unary operations, fold a load into the operation
+  // only in OptForSize mode. It eliminates an instruction, but it also
+  // eliminates a whole-register clobber (the load), so it introduces a
+  // partial register update condition.
+  def SSm : I<opc, MRMSrcMem, (outs FR32:$dst), (ins f32mem:$src),
+                !strconcat(OpcodeStr, "ss\t{$src, $dst|$dst, $src}"),
+                [(set FR32:$dst, (OpNode (load addr:$src)))], itins.rm>, XS,
+            Requires<[UseSSE1, OptForSize]>;
+  let Constraints = "$src1 = $dst" in {
+    def SSr_Int : SSI<opc, MRMSrcReg, (outs VR128:$dst),
+                      (ins VR128:$src1, VR128:$src2),
+                      !strconcat(OpcodeStr, "ss\t{$src2, $dst|$dst, $src2}"),
+                      [], itins.rr>;
+    def SSm_Int : SSI<opc, MRMSrcMem, (outs VR128:$dst),
+                      (ins VR128:$src1, ssmem:$src2),
+                      !strconcat(OpcodeStr, "ss\t{$src2, $dst|$dst, $src2}"),
+                      [], itins.rm>;
+  }
+}
+
 // Reciprocal approximations. Note that these typically require refinement
 // in order to obtain suitable precision.
-defm RSQRT : sse1_fp_unop_s<0x52, "rsqrt", X86frsqrt, int_x86_sse_rsqrt_ss,
-                            SSE_SQRTS>,
+defm RSQRT : sse1_fp_unop_rw<0x52, "rsqrt", X86frsqrt, int_x86_sse_rsqrt_ss,
+                             SSE_SQRTS>,
              sse1_fp_unop_p<0x52, "rsqrt", X86frsqrt, SSE_SQRTS>,
              sse1_fp_unop_p_int<0x52, "rsqrt", int_x86_sse_rsqrt_ps,
                             SSE_SQRTS>;
-defm RCP   : sse1_fp_unop_s<0x53, "rcp", X86frcp, int_x86_sse_rcp_ss,
-                            SSE_RCPS>,
+let Predicates = [UseSSE1] in {
+  def : Pat<(int_x86_sse_rsqrt_ss VR128:$src),
+            (RSQRTSSr_Int VR128:$src, VR128:$src)>;
+}
+
+defm RCP   : sse1_fp_unop_rw<0x53, "rcp", X86frcp, int_x86_sse_rcp_ss,
+                             SSE_RCPS>,
              sse1_fp_unop_p<0x53, "rcp", X86frcp, SSE_RCPS>,
              sse1_fp_unop_p_int<0x53, "rcp", int_x86_sse_rcp_ps, SSE_RCPS>;
+let Predicates = [UseSSE1] in {
+  def : Pat<(int_x86_sse_rcp_ss VR128:$src),
+            (RCPSSr_Int VR128:$src, VR128:$src)>;
+}
 
 // There is no f64 version of the reciprocal approximation instructions.
 
@@ -3271,20 +3376,20 @@ let AddedComplexity = 400 in { // Prefer non-temporal versions
                        "movntps\t{$src, $dst|$dst, $src}",
                        [(alignednontemporalstore (v8f32 VR256:$src),
                                                  addr:$dst)],
-                                                 IIC_SSE_MOVNT>, VEX;
+                                                 IIC_SSE_MOVNT>, VEX, VEX_L;
   def VMOVNTPDYmr : VPDI<0x2B, MRMDestMem, (outs),
                        (ins f256mem:$dst, VR256:$src),
                        "movntpd\t{$src, $dst|$dst, $src}",
                        [(alignednontemporalstore (v4f64 VR256:$src),
                                                  addr:$dst)],
-                                                 IIC_SSE_MOVNT>, VEX;
+                                                 IIC_SSE_MOVNT>, VEX, VEX_L;
   let ExeDomain = SSEPackedInt in
   def VMOVNTDQYmr : VPDI<0xE7, MRMDestMem, (outs),
                       (ins f256mem:$dst, VR256:$src),
                       "movntdq\t{$src, $dst|$dst, $src}",
                       [(alignednontemporalstore (v4i64 VR256:$src),
                                                 addr:$dst)],
-                                                IIC_SSE_MOVNT>, VEX;
+                                                IIC_SSE_MOVNT>, VEX, VEX_L;
 }
 
 let AddedComplexity = 400 in { // Prefer non-temporal versions
@@ -3304,7 +3409,7 @@ def MOVNTDQmr : PDI<0xE7, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src),
                     IIC_SSE_MOVNT>;
 
 def : Pat<(alignednontemporalstore (v2i64 VR128:$src), addr:$dst),
-          (MOVNTDQmr addr:$dst, VR128:$src)>, Requires<[HasSSE2]>;
+          (MOVNTDQmr addr:$dst, VR128:$src)>, Requires<[UseSSE2]>;
 
 // There is no AVX form for instructions below this point
 def MOVNTImr : I<0xC3, MRMDestMem, (outs), (ins i32mem:$dst, GR32:$src),
@@ -3393,14 +3498,14 @@ def VMOVDQArr  : VPDI<0x6F, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
                     VEX;
 def VMOVDQAYrr : VPDI<0x6F, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src),
                     "movdqa\t{$src, $dst|$dst, $src}", [], IIC_SSE_MOVA_P_RR>,
-                    VEX;
+                    VEX, VEX_L;
 }
 def VMOVDQUrr  : VSSI<0x6F, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
                     "movdqu\t{$src, $dst|$dst, $src}", [], IIC_SSE_MOVU_P_RR>,
                     VEX;
 def VMOVDQUYrr : VSSI<0x6F, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src),
                     "movdqu\t{$src, $dst|$dst, $src}", [], IIC_SSE_MOVU_P_RR>,
-                    VEX;
+                    VEX, VEX_L;
 
 // For Disassembler
 let isCodeGenOnly = 1 in {
@@ -3410,16 +3515,14 @@ def VMOVDQArr_REV  : VPDI<0x7F, MRMDestReg, (outs VR128:$dst), (ins VR128:$src),
                         VEX;
 def VMOVDQAYrr_REV : VPDI<0x7F, MRMDestReg, (outs VR256:$dst), (ins VR256:$src),
                         "movdqa\t{$src, $dst|$dst, $src}", [],
-                        IIC_SSE_MOVA_P_RR>,
-                        VEX;
+                        IIC_SSE_MOVA_P_RR>, VEX, VEX_L;
 def VMOVDQUrr_REV  : VSSI<0x7F, MRMDestReg, (outs VR128:$dst), (ins VR128:$src),
                         "movdqu\t{$src, $dst|$dst, $src}", [],
                         IIC_SSE_MOVU_P_RR>,
                         VEX;
 def VMOVDQUYrr_REV : VSSI<0x7F, MRMDestReg, (outs VR256:$dst), (ins VR256:$src),
                         "movdqu\t{$src, $dst|$dst, $src}", [],
-                        IIC_SSE_MOVU_P_RR>,
-                        VEX;
+                        IIC_SSE_MOVU_P_RR>, VEX, VEX_L;
 }
 
 let canFoldAsLoad = 1, mayLoad = 1 in {
@@ -3428,14 +3531,14 @@ def VMOVDQArm  : VPDI<0x6F, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src),
                    VEX;
 def VMOVDQAYrm : VPDI<0x6F, MRMSrcMem, (outs VR256:$dst), (ins i256mem:$src),
                    "movdqa\t{$src, $dst|$dst, $src}", [], IIC_SSE_MOVA_P_RM>,
-                   VEX;
+                   VEX, VEX_L;
 let Predicates = [HasAVX] in {
   def VMOVDQUrm  : I<0x6F, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src),
                     "vmovdqu\t{$src, $dst|$dst, $src}",[], IIC_SSE_MOVU_P_RM>,
                     XS, VEX;
   def VMOVDQUYrm : I<0x6F, MRMSrcMem, (outs VR256:$dst), (ins i256mem:$src),
                     "vmovdqu\t{$src, $dst|$dst, $src}",[], IIC_SSE_MOVU_P_RM>,
-                    XS, VEX;
+                    XS, VEX, VEX_L;
 }
 }
 
@@ -3447,14 +3550,14 @@ def VMOVDQAmr  : VPDI<0x7F, MRMDestMem, (outs),
 def VMOVDQAYmr : VPDI<0x7F, MRMDestMem, (outs),
                      (ins i256mem:$dst, VR256:$src),
                      "movdqa\t{$src, $dst|$dst, $src}", [], IIC_SSE_MOVA_P_MR>,
-                     VEX;
+                     VEX, VEX_L;
 let Predicates = [HasAVX] in {
 def VMOVDQUmr  : I<0x7F, MRMDestMem, (outs), (ins i128mem:$dst, VR128:$src),
                   "vmovdqu\t{$src, $dst|$dst, $src}",[], IIC_SSE_MOVU_P_MR>,
                   XS, VEX;
 def VMOVDQUYmr : I<0x7F, MRMDestMem, (outs), (ins i256mem:$dst, VR256:$src),
                   "vmovdqu\t{$src, $dst|$dst, $src}",[], IIC_SSE_MOVU_P_MR>,
-                  XS, VEX;
+                  XS, VEX, VEX_L;
 }
 }
 
@@ -3464,7 +3567,7 @@ def MOVDQArr : PDI<0x6F, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
 
 def MOVDQUrr :   I<0x6F, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
                    "movdqu\t{$src, $dst|$dst, $src}",
-                   [], IIC_SSE_MOVU_P_RR>, XS, Requires<[HasSSE2]>;
+                   [], IIC_SSE_MOVU_P_RR>, XS, Requires<[UseSSE2]>;
 
 // For Disassembler
 let isCodeGenOnly = 1 in {
@@ -3474,7 +3577,7 @@ def MOVDQArr_REV : PDI<0x7F, MRMDestReg, (outs VR128:$dst), (ins VR128:$src),
 
 def MOVDQUrr_REV :   I<0x7F, MRMDestReg, (outs VR128:$dst), (ins VR128:$src),
                        "movdqu\t{$src, $dst|$dst, $src}",
-                       [], IIC_SSE_MOVU_P_RR>, XS, Requires<[HasSSE2]>;
+                       [], IIC_SSE_MOVU_P_RR>, XS, Requires<[UseSSE2]>;
 }
 
 let canFoldAsLoad = 1, mayLoad = 1 in {
@@ -3486,7 +3589,7 @@ def MOVDQUrm :   I<0x6F, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src),
                    "movdqu\t{$src, $dst|$dst, $src}",
                    [/*(set VR128:$dst, (loadv2i64 addr:$src))*/],
                    IIC_SSE_MOVU_P_RM>,
-                 XS, Requires<[HasSSE2]>;
+                 XS, Requires<[UseSSE2]>;
 }
 
 let mayStore = 1 in {
@@ -3498,7 +3601,7 @@ def MOVDQUmr :   I<0x7F, MRMDestMem, (outs), (ins i128mem:$dst, VR128:$src),
                    "movdqu\t{$src, $dst|$dst, $src}",
                    [/*(store (v2i64 VR128:$src), addr:$dst)*/],
                    IIC_SSE_MOVU_P_MR>,
-                 XS, Requires<[HasSSE2]>;
+                 XS, Requires<[UseSSE2]>;
 }
 
 // Intrinsic forms of MOVDQU load and store
@@ -3512,7 +3615,7 @@ def MOVDQUmr_Int :   I<0x7F, MRMDestMem, (outs), (ins i128mem:$dst, VR128:$src),
                        "movdqu\t{$src, $dst|$dst, $src}",
                        [(int_x86_sse2_storeu_dq addr:$dst, VR128:$src)],
                        IIC_SSE_MOVU_P_MR>,
-                     XS, Requires<[HasSSE2]>;
+                     XS, Requires<[UseSSE2]>;
 
 } // ExeDomain = SSEPackedInt
 
@@ -3690,82 +3793,82 @@ defm VPSADBW  : PDI_binop_rm_int<0xF6, "vpsadbw", int_x86_sse2_psad_bw,
 
 let Predicates = [HasAVX2] in {
 defm VPADDBY  : PDI_binop_rm<0xFC, "vpaddb", add, v32i8, VR256, memopv4i64,
-                             i256mem, SSE_INTALU_ITINS_P, 1, 0>, VEX_4V;
+                             i256mem, SSE_INTALU_ITINS_P, 1, 0>, VEX_4V, VEX_L;
 defm VPADDWY  : PDI_binop_rm<0xFD, "vpaddw", add, v16i16, VR256, memopv4i64,
-                             i256mem, SSE_INTALU_ITINS_P, 1, 0>, VEX_4V;
+                             i256mem, SSE_INTALU_ITINS_P, 1, 0>, VEX_4V, VEX_L;
 defm VPADDDY  : PDI_binop_rm<0xFE, "vpaddd", add, v8i32, VR256, memopv4i64,
-                             i256mem, SSE_INTALU_ITINS_P, 1, 0>, VEX_4V;
+                             i256mem, SSE_INTALU_ITINS_P, 1, 0>, VEX_4V, VEX_L;
 defm VPADDQY  : PDI_binop_rm<0xD4, "vpaddq", add, v4i64, VR256, memopv4i64,
-                             i256mem, SSE_INTALUQ_ITINS_P, 1, 0>, VEX_4V;
+                             i256mem, SSE_INTALUQ_ITINS_P, 1, 0>, VEX_4V, VEX_L;
 defm VPMULLWY : PDI_binop_rm<0xD5, "vpmullw", mul, v16i16, VR256, memopv4i64,
-                             i256mem, SSE_INTMUL_ITINS_P, 1, 0>, VEX_4V;
+                             i256mem, SSE_INTMUL_ITINS_P, 1, 0>, VEX_4V, VEX_L;
 defm VPSUBBY  : PDI_binop_rm<0xF8, "vpsubb", sub, v32i8, VR256, memopv4i64,
-                             i256mem, SSE_INTALU_ITINS_P, 0, 0>, VEX_4V;
+                             i256mem, SSE_INTALU_ITINS_P, 0, 0>, VEX_4V, VEX_L;
 defm VPSUBWY  : PDI_binop_rm<0xF9, "vpsubw", sub, v16i16,VR256, memopv4i64,
-                             i256mem, SSE_INTALU_ITINS_P, 0, 0>, VEX_4V;
+                             i256mem, SSE_INTALU_ITINS_P, 0, 0>, VEX_4V, VEX_L;
 defm VPSUBDY  : PDI_binop_rm<0xFA, "vpsubd", sub, v8i32, VR256, memopv4i64,
-                             i256mem, SSE_INTALU_ITINS_P, 0, 0>, VEX_4V;
+                             i256mem, SSE_INTALU_ITINS_P, 0, 0>, VEX_4V, VEX_L;
 defm VPSUBQY  : PDI_binop_rm<0xFB, "vpsubq", sub, v4i64, VR256, memopv4i64,
-                             i256mem, SSE_INTALUQ_ITINS_P, 0, 0>, VEX_4V;
+                             i256mem, SSE_INTALUQ_ITINS_P, 0, 0>, VEX_4V, VEX_L;
 defm VPMULUDQY : PDI_binop_rm2<0xF4, "vpmuludq", X86pmuludq, v4i64, v8i32,
                                VR256, memopv4i64, i256mem,
-                               SSE_INTMUL_ITINS_P, 1, 0>, VEX_4V;
+                               SSE_INTMUL_ITINS_P, 1, 0>, VEX_4V, VEX_L;
 
 // Intrinsic forms
 defm VPSUBSBY  : PDI_binop_rm_int<0xE8, "vpsubsb" , int_x86_avx2_psubs_b,
                                   VR256, memopv4i64, i256mem,
-                                  SSE_INTALU_ITINS_P, 0, 0>, VEX_4V;
+                                  SSE_INTALU_ITINS_P, 0, 0>, VEX_4V, VEX_L;
 defm VPSUBSWY  : PDI_binop_rm_int<0xE9, "vpsubsw" , int_x86_avx2_psubs_w,
                                   VR256, memopv4i64, i256mem,
-                                  SSE_INTALU_ITINS_P, 0, 0>, VEX_4V;
+                                  SSE_INTALU_ITINS_P, 0, 0>, VEX_4V, VEX_L;
 defm VPSUBUSBY : PDI_binop_rm_int<0xD8, "vpsubusb", int_x86_avx2_psubus_b,
                                   VR256, memopv4i64, i256mem,
-                                  SSE_INTALU_ITINS_P, 0, 0>, VEX_4V;
+                                  SSE_INTALU_ITINS_P, 0, 0>, VEX_4V, VEX_L;
 defm VPSUBUSWY : PDI_binop_rm_int<0xD9, "vpsubusw", int_x86_avx2_psubus_w,
                                   VR256, memopv4i64, i256mem,
-                                  SSE_INTALU_ITINS_P, 0, 0>, VEX_4V;
+                                  SSE_INTALU_ITINS_P, 0, 0>, VEX_4V, VEX_L;
 defm VPADDSBY  : PDI_binop_rm_int<0xEC, "vpaddsb" , int_x86_avx2_padds_b,
                                   VR256, memopv4i64, i256mem,
-                                  SSE_INTALU_ITINS_P, 1, 0>, VEX_4V;
+                                  SSE_INTALU_ITINS_P, 1, 0>, VEX_4V, VEX_L;
 defm VPADDSWY  : PDI_binop_rm_int<0xED, "vpaddsw" , int_x86_avx2_padds_w,
                                   VR256, memopv4i64, i256mem,
-                                  SSE_INTALU_ITINS_P, 1, 0>, VEX_4V;
+                                  SSE_INTALU_ITINS_P, 1, 0>, VEX_4V, VEX_L;
 defm VPADDUSBY : PDI_binop_rm_int<0xDC, "vpaddusb", int_x86_avx2_paddus_b,
                                   VR256, memopv4i64, i256mem,
-                                  SSE_INTALU_ITINS_P, 1, 0>, VEX_4V;
+                                  SSE_INTALU_ITINS_P, 1, 0>, VEX_4V, VEX_L;
 defm VPADDUSWY : PDI_binop_rm_int<0xDD, "vpaddusw", int_x86_avx2_paddus_w,
                                   VR256, memopv4i64, i256mem,
-                                  SSE_INTALU_ITINS_P, 1, 0>, VEX_4V;
+                                  SSE_INTALU_ITINS_P, 1, 0>, VEX_4V, VEX_L;
 defm VPMULHUWY : PDI_binop_rm_int<0xE4, "vpmulhuw", int_x86_avx2_pmulhu_w,
                                   VR256, memopv4i64, i256mem,
-                                  SSE_INTMUL_ITINS_P, 1, 0>, VEX_4V;
+                                  SSE_INTMUL_ITINS_P, 1, 0>, VEX_4V, VEX_L;
 defm VPMULHWY  : PDI_binop_rm_int<0xE5, "vpmulhw" , int_x86_avx2_pmulh_w,
                                   VR256, memopv4i64, i256mem,
-                                  SSE_INTMUL_ITINS_P, 1, 0>, VEX_4V;
+                                  SSE_INTMUL_ITINS_P, 1, 0>, VEX_4V, VEX_L;
 defm VPMADDWDY : PDI_binop_rm_int<0xF5, "vpmaddwd", int_x86_avx2_pmadd_wd,
                                   VR256, memopv4i64, i256mem,
-                                  SSE_PMADD, 1, 0>, VEX_4V;
+                                  SSE_PMADD, 1, 0>, VEX_4V, VEX_L;
 defm VPAVGBY   : PDI_binop_rm_int<0xE0, "vpavgb", int_x86_avx2_pavg_b,
                                   VR256, memopv4i64, i256mem,
-                                  SSE_INTALU_ITINS_P, 1, 0>, VEX_4V;
+                                  SSE_INTALU_ITINS_P, 1, 0>, VEX_4V, VEX_L;
 defm VPAVGWY   : PDI_binop_rm_int<0xE3, "vpavgw", int_x86_avx2_pavg_w,
                                   VR256, memopv4i64, i256mem,
-                                  SSE_INTALU_ITINS_P, 1, 0>, VEX_4V;
+                                  SSE_INTALU_ITINS_P, 1, 0>, VEX_4V, VEX_L;
 defm VPMINUBY  : PDI_binop_rm_int<0xDA, "vpminub", int_x86_avx2_pminu_b,
                                   VR256, memopv4i64, i256mem,
-                                  SSE_INTALU_ITINS_P, 1, 0>, VEX_4V;
+                                  SSE_INTALU_ITINS_P, 1, 0>, VEX_4V, VEX_L;
 defm VPMINSWY  : PDI_binop_rm_int<0xEA, "vpminsw", int_x86_avx2_pmins_w,
                                   VR256, memopv4i64, i256mem,
-                                  SSE_INTALU_ITINS_P, 1, 0>, VEX_4V;
+                                  SSE_INTALU_ITINS_P, 1, 0>, VEX_4V, VEX_L;
 defm VPMAXUBY  : PDI_binop_rm_int<0xDE, "vpmaxub", int_x86_avx2_pmaxu_b,
                                   VR256, memopv4i64, i256mem,
-                                  SSE_INTALU_ITINS_P, 1, 0>, VEX_4V;
+                                  SSE_INTALU_ITINS_P, 1, 0>, VEX_4V, VEX_L;
 defm VPMAXSWY  : PDI_binop_rm_int<0xEE, "vpmaxsw", int_x86_avx2_pmaxs_w,
                                   VR256, memopv4i64, i256mem,
-                                  SSE_INTALU_ITINS_P, 1, 0>, VEX_4V;
+                                  SSE_INTALU_ITINS_P, 1, 0>, VEX_4V, VEX_L;
 defm VPSADBWY  : PDI_binop_rm_int<0xF6, "vpsadbw", int_x86_avx2_psad_bw,
                                   VR256, memopv4i64, i256mem,
-                                  SSE_INTALU_ITINS_P, 1, 0>, VEX_4V;
+                                  SSE_INTALU_ITINS_P, 1, 0>, VEX_4V, VEX_L;
 }
 
 let Constraints = "$src1 = $dst" in {
@@ -3901,30 +4004,30 @@ let ExeDomain = SSEPackedInt in {
 let Predicates = [HasAVX2] in {
 defm VPSLLWY : PDI_binop_rmi<0xF1, 0x71, MRM6r, "vpsllw", X86vshl, X86vshli,
                              VR256, v16i16, v8i16, bc_v8i16,
-                             SSE_INTSHIFT_ITINS_P, 0>, VEX_4V;
+                             SSE_INTSHIFT_ITINS_P, 0>, VEX_4V, VEX_L;
 defm VPSLLDY : PDI_binop_rmi<0xF2, 0x72, MRM6r, "vpslld", X86vshl, X86vshli,
                              VR256, v8i32, v4i32, bc_v4i32,
-                             SSE_INTSHIFT_ITINS_P, 0>, VEX_4V;
+                             SSE_INTSHIFT_ITINS_P, 0>, VEX_4V, VEX_L;
 defm VPSLLQY : PDI_binop_rmi<0xF3, 0x73, MRM6r, "vpsllq", X86vshl, X86vshli,
                              VR256, v4i64, v2i64, bc_v2i64,
-                             SSE_INTSHIFT_ITINS_P, 0>, VEX_4V;
+                             SSE_INTSHIFT_ITINS_P, 0>, VEX_4V, VEX_L;
 
 defm VPSRLWY : PDI_binop_rmi<0xD1, 0x71, MRM2r, "vpsrlw", X86vsrl, X86vsrli,
                              VR256, v16i16, v8i16, bc_v8i16,
-                             SSE_INTSHIFT_ITINS_P, 0>, VEX_4V;
+                             SSE_INTSHIFT_ITINS_P, 0>, VEX_4V, VEX_L;
 defm VPSRLDY : PDI_binop_rmi<0xD2, 0x72, MRM2r, "vpsrld", X86vsrl, X86vsrli,
                              VR256, v8i32, v4i32, bc_v4i32,
-                             SSE_INTSHIFT_ITINS_P, 0>, VEX_4V;
+                             SSE_INTSHIFT_ITINS_P, 0>, VEX_4V, VEX_L;
 defm VPSRLQY : PDI_binop_rmi<0xD3, 0x73, MRM2r, "vpsrlq", X86vsrl, X86vsrli,
                              VR256, v4i64, v2i64, bc_v2i64,
-                             SSE_INTSHIFT_ITINS_P, 0>, VEX_4V;
+                             SSE_INTSHIFT_ITINS_P, 0>, VEX_4V, VEX_L;
 
 defm VPSRAWY : PDI_binop_rmi<0xE1, 0x71, MRM4r, "vpsraw", X86vsra, X86vsrai,
                              VR256, v16i16, v8i16, bc_v8i16,
-                             SSE_INTSHIFT_ITINS_P, 0>, VEX_4V;
+                             SSE_INTSHIFT_ITINS_P, 0>, VEX_4V, VEX_L;
 defm VPSRADY : PDI_binop_rmi<0xE2, 0x72, MRM4r, "vpsrad", X86vsra, X86vsrai,
                              VR256, v8i32, v4i32, bc_v4i32,
-                             SSE_INTSHIFT_ITINS_P, 0>, VEX_4V;
+                             SSE_INTSHIFT_ITINS_P, 0>, VEX_4V, VEX_L;
 
 let ExeDomain = SSEPackedInt in {
   // 256-bit logical shifts.
@@ -3933,13 +4036,13 @@ let ExeDomain = SSEPackedInt in {
                     "vpslldq\t{$src2, $src1, $dst|$dst, $src1, $src2}",
                     [(set VR256:$dst,
                       (int_x86_avx2_psll_dq_bs VR256:$src1, imm:$src2))]>,
-                    VEX_4V;
+                    VEX_4V, VEX_L;
   def VPSRLDQYri : PDIi8<0x73, MRM3r,
                     (outs VR256:$dst), (ins VR256:$src1, i32i8imm:$src2),
                     "vpsrldq\t{$src2, $src1, $dst|$dst, $src1, $src2}",
                     [(set VR256:$dst,
                       (int_x86_avx2_psrl_dq_bs VR256:$src1, imm:$src2))]>,
-                    VEX_4V;
+                    VEX_4V, VEX_L;
   // PSRADQYri doesn't exist in SSE[1-3].
 }
 } // Predicates = [HasAVX2]
@@ -4010,7 +4113,7 @@ let Predicates = [HasAVX2] in {
             (VPSRLDQYri VR256:$src1, (BYTE_imm imm:$src2))>;
 }
 
-let Predicates = [HasSSE2] in {
+let Predicates = [UseSSE2] in {
   def : Pat<(int_x86_sse2_psll_dq VR128:$src1, imm:$src2),
             (PSLLDQri VR128:$src1, (BYTE_imm imm:$src2))>;
   def : Pat<(int_x86_sse2_psrl_dq VR128:$src1, imm:$src2),
@@ -4053,22 +4156,22 @@ let Predicates = [HasAVX] in {
 let Predicates = [HasAVX2] in {
   defm VPCMPEQBY : PDI_binop_rm<0x74, "vpcmpeqb", X86pcmpeq, v32i8,
                                 VR256, memopv4i64, i256mem,
-                                SSE_INTALU_ITINS_P, 1, 0>, VEX_4V;
+                                SSE_INTALU_ITINS_P, 1, 0>, VEX_4V, VEX_L;
   defm VPCMPEQWY : PDI_binop_rm<0x75, "vpcmpeqw", X86pcmpeq, v16i16,
                                 VR256, memopv4i64, i256mem,
-                                SSE_INTALU_ITINS_P, 1, 0>, VEX_4V;
+                                SSE_INTALU_ITINS_P, 1, 0>, VEX_4V, VEX_L;
   defm VPCMPEQDY : PDI_binop_rm<0x76, "vpcmpeqd", X86pcmpeq, v8i32,
                                 VR256, memopv4i64, i256mem,
-                                SSE_INTALU_ITINS_P, 1, 0>, VEX_4V;
+                                SSE_INTALU_ITINS_P, 1, 0>, VEX_4V, VEX_L;
   defm VPCMPGTBY : PDI_binop_rm<0x64, "vpcmpgtb", X86pcmpgt, v32i8,
                                 VR256, memopv4i64, i256mem,
-                                SSE_INTALU_ITINS_P, 0, 0>, VEX_4V;
+                                SSE_INTALU_ITINS_P, 0, 0>, VEX_4V, VEX_L;
   defm VPCMPGTWY : PDI_binop_rm<0x65, "vpcmpgtw", X86pcmpgt, v16i16,
                                 VR256, memopv4i64, i256mem,
-                                SSE_INTALU_ITINS_P, 0, 0>, VEX_4V;
+                                SSE_INTALU_ITINS_P, 0, 0>, VEX_4V, VEX_L;
   defm VPCMPGTDY : PDI_binop_rm<0x66, "vpcmpgtd", X86pcmpgt, v8i32,
                                 VR256, memopv4i64, i256mem,
-                                SSE_INTALU_ITINS_P, 0, 0>, VEX_4V;
+                                SSE_INTALU_ITINS_P, 0, 0>, VEX_4V, VEX_L;
 }
 
 let Constraints = "$src1 = $dst" in {
@@ -4111,13 +4214,13 @@ defm VPACKUSWB : PDI_binop_rm_int<0x67, "vpackuswb", int_x86_sse2_packuswb_128,
 let Predicates = [HasAVX2] in {
 defm VPACKSSWBY : PDI_binop_rm_int<0x63, "vpacksswb", int_x86_avx2_packsswb,
                                    VR256, memopv4i64, i256mem,
-                                   SSE_INTALU_ITINS_P, 0, 0>, VEX_4V;
+                                   SSE_INTALU_ITINS_P, 0, 0>, VEX_4V, VEX_L;
 defm VPACKSSDWY : PDI_binop_rm_int<0x6B, "vpackssdw", int_x86_avx2_packssdw,
                                    VR256, memopv4i64, i256mem,
-                                   SSE_INTALU_ITINS_P, 0, 0>, VEX_4V;
+                                   SSE_INTALU_ITINS_P, 0, 0>, VEX_4V, VEX_L;
 defm VPACKUSWBY : PDI_binop_rm_int<0x67, "vpackuswb", int_x86_avx2_packuswb,
                                    VR256, memopv4i64, i256mem,
-                                   SSE_INTALU_ITINS_P, 0, 0>, VEX_4V;
+                                   SSE_INTALU_ITINS_P, 0, 0>, VEX_4V, VEX_L;
 }
 
 let Constraints = "$src1 = $dst" in {
@@ -4187,12 +4290,15 @@ let Predicates = [HasAVX] in {
 }
 
 let Predicates = [HasAVX2] in {
-  defm VPSHUFD : sse2_pshuffle_y<"vpshufd", v8i32, X86PShufd>, TB, OpSize, VEX;
-  defm VPSHUFHW : sse2_pshuffle_y<"vpshufhw", v16i16, X86PShufhw>, XS, VEX;
-  defm VPSHUFLW : sse2_pshuffle_y<"vpshuflw", v16i16, X86PShuflw>, XD, VEX;
+  defm VPSHUFD : sse2_pshuffle_y<"vpshufd", v8i32, X86PShufd>,
+                                TB, OpSize, VEX,VEX_L;
+  defm VPSHUFHW : sse2_pshuffle_y<"vpshufhw", v16i16, X86PShufhw>,
+                                  XS, VEX, VEX_L;
+  defm VPSHUFLW : sse2_pshuffle_y<"vpshuflw", v16i16, X86PShuflw>,
+                                  XD, VEX, VEX_L;
 }
 
-let Predicates = [HasSSE2] in {
+let Predicates = [UseSSE2] in {
  let AddedComplexity = 5 in
   defm PSHUFD : sse2_pshuffle<"pshufd", v4i32, X86PShufd>, TB, OpSize;
 
@@ -4268,22 +4374,22 @@ let Predicates = [HasAVX] in {
 
 let Predicates = [HasAVX2] in {
   defm VPUNPCKLBW  : sse2_unpack_y<0x60, "vpunpcklbw", v32i8, X86Unpckl,
-                                   bc_v32i8>, VEX_4V;
+                                   bc_v32i8>, VEX_4V, VEX_L;
   defm VPUNPCKLWD  : sse2_unpack_y<0x61, "vpunpcklwd", v16i16, X86Unpckl,
-                                   bc_v16i16>, VEX_4V;
+                                   bc_v16i16>, VEX_4V, VEX_L;
   defm VPUNPCKLDQ  : sse2_unpack_y<0x62, "vpunpckldq", v8i32, X86Unpckl,
-                                   bc_v8i32>, VEX_4V;
+                                   bc_v8i32>, VEX_4V, VEX_L;
   defm VPUNPCKLQDQ : sse2_unpack_y<0x6C, "vpunpcklqdq", v4i64, X86Unpckl,
-                                   bc_v4i64>, VEX_4V;
+                                   bc_v4i64>, VEX_4V, VEX_L;
 
   defm VPUNPCKHBW  : sse2_unpack_y<0x68, "vpunpckhbw", v32i8, X86Unpckh,
-                                   bc_v32i8>, VEX_4V;
+                                   bc_v32i8>, VEX_4V, VEX_L;
   defm VPUNPCKHWD  : sse2_unpack_y<0x69, "vpunpckhwd", v16i16, X86Unpckh,
-                                   bc_v16i16>, VEX_4V;
+                                   bc_v16i16>, VEX_4V, VEX_L;
   defm VPUNPCKHDQ  : sse2_unpack_y<0x6A, "vpunpckhdq", v8i32, X86Unpckh,
-                                   bc_v8i32>, VEX_4V;
+                                   bc_v8i32>, VEX_4V, VEX_L;
   defm VPUNPCKHQDQ : sse2_unpack_y<0x6D, "vpunpckhqdq", v4i64, X86Unpckh,
-                                   bc_v4i64>, VEX_4V;
+                                   bc_v4i64>, VEX_4V, VEX_L;
 }
 
 let Constraints = "$src1 = $dst" in {
@@ -4307,28 +4413,6 @@ let Constraints = "$src1 = $dst" in {
 }
 } // ExeDomain = SSEPackedInt
 
-// Patterns for using AVX1 instructions with integer vectors
-// Here to give AVX2 priority
-let Predicates = [HasAVX] in {
-  def : Pat<(v8i32 (X86Unpckl VR256:$src1, (bc_v8i32 (memopv4i64 addr:$src2)))),
-            (VUNPCKLPSYrm VR256:$src1, addr:$src2)>;
-  def : Pat<(v8i32 (X86Unpckl VR256:$src1, VR256:$src2)),
-            (VUNPCKLPSYrr VR256:$src1, VR256:$src2)>;
-  def : Pat<(v8i32 (X86Unpckh VR256:$src1, (bc_v8i32 (memopv4i64 addr:$src2)))),
-            (VUNPCKHPSYrm VR256:$src1, addr:$src2)>;
-  def : Pat<(v8i32 (X86Unpckh VR256:$src1, VR256:$src2)),
-            (VUNPCKHPSYrr VR256:$src1, VR256:$src2)>;
-
-  def : Pat<(v4i64 (X86Unpckl VR256:$src1, (memopv4i64 addr:$src2))),
-            (VUNPCKLPDYrm VR256:$src1, addr:$src2)>;
-  def : Pat<(v4i64 (X86Unpckl VR256:$src1, VR256:$src2)),
-            (VUNPCKLPDYrr VR256:$src1, VR256:$src2)>;
-  def : Pat<(v4i64 (X86Unpckh VR256:$src1, (memopv4i64 addr:$src2))),
-            (VUNPCKHPDYrm VR256:$src1, addr:$src2)>;
-  def : Pat<(v4i64 (X86Unpckh VR256:$src1, VR256:$src2)),
-            (VUNPCKHPDYrr VR256:$src1, VR256:$src2)>;
-}
-
 //===---------------------------------------------------------------------===//
 // SSE2 - Packed Integer Extract and Insert
 //===---------------------------------------------------------------------===//
@@ -4377,7 +4461,7 @@ let Predicates = [HasAVX] in {
 }
 
 let Constraints = "$src1 = $dst" in
-  defm PINSRW : sse2_pinsrw, TB, OpSize, Requires<[HasSSE2]>;
+  defm PINSRW : sse2_pinsrw, TB, OpSize, Requires<[UseSSE2]>;
 
 } // ExeDomain = SSEPackedInt
 
@@ -4397,9 +4481,9 @@ def VPMOVMSKBr64r : VPDI<0xD7, MRMSrcReg, (outs GR64:$dst), (ins VR128:$src),
 let Predicates = [HasAVX2] in {
 def VPMOVMSKBYrr  : VPDI<0xD7, MRMSrcReg, (outs GR32:$dst), (ins VR256:$src),
            "pmovmskb\t{$src, $dst|$dst, $src}",
-           [(set GR32:$dst, (int_x86_avx2_pmovmskb VR256:$src))]>, VEX;
+           [(set GR32:$dst, (int_x86_avx2_pmovmskb VR256:$src))]>, VEX, VEX_L;
 def VPMOVMSKBYr64r : VPDI<0xD7, MRMSrcReg, (outs GR64:$dst), (ins VR256:$src),
-           "pmovmskb\t{$src, $dst|$dst, $src}", []>, VEX;
+           "pmovmskb\t{$src, $dst|$dst, $src}", []>, VEX, VEX_L;
 }
 
 def PMOVMSKBrr : PDI<0xD7, MRMSrcReg, (outs GR32:$dst), (ins VR128:$src),
@@ -4538,7 +4622,7 @@ def MOVPDI2DImr  : PDI<0x7E, MRMDestMem, (outs), (ins i32mem:$dst, VR128:$src),
 // Move Packed Doubleword Int first element to Doubleword Int
 //
 def VMOVPQIto64rr : I<0x7E, MRMDestReg, (outs GR64:$dst), (ins VR128:$src),
-                          "mov{d|q}\t{$src, $dst|$dst, $src}",
+                          "vmov{d|q}\t{$src, $dst|$dst, $src}",
                           [(set GR64:$dst, (vector_extract (v2i64 VR128:$src),
                                                            (iPTR 0)))],
                                                            IIC_SSE_MOVD_ToGP>,
@@ -4654,14 +4738,14 @@ let Predicates = [HasAVX] in {
   }
   // Use regular 128-bit instructions to match 256-bit scalar_to_vec+zext.
   def : Pat<(v8i32 (X86vzmovl (insert_subvector undef,
-                                (v4i32 (scalar_to_vector GR32:$src)),(i32 0)))),
+                               (v4i32 (scalar_to_vector GR32:$src)),(iPTR 0)))),
             (SUBREG_TO_REG (i32 0), (VMOVZDI2PDIrr GR32:$src), sub_xmm)>;
   def : Pat<(v4i64 (X86vzmovl (insert_subvector undef,
-                                (v2i64 (scalar_to_vector GR64:$src)),(i32 0)))),
+                               (v2i64 (scalar_to_vector GR64:$src)),(iPTR 0)))),
             (SUBREG_TO_REG (i64 0), (VMOVZQI2PQIrr GR64:$src), sub_xmm)>;
 }
 
-let Predicates = [HasSSE2], AddedComplexity = 20 in {
+let Predicates = [UseSSE2], AddedComplexity = 20 in {
   def : Pat<(v4i32 (X86vzmovl (bc_v4i32 (loadv4f32 addr:$src)))),
             (MOVZDI2PDIrm addr:$src)>;
   def : Pat<(v4i32 (X86vzmovl (bc_v4i32 (loadv2i64 addr:$src)))),
@@ -4701,7 +4785,7 @@ def MOVQI2PQIrm : I<0x7E, MRMSrcMem, (outs VR128:$dst), (ins i64mem:$src),
                     [(set VR128:$dst,
                       (v2i64 (scalar_to_vector (loadi64 addr:$src))))],
                       IIC_SSE_MOVDQ>, XS,
-                    Requires<[HasSSE2]>; // SSE2 instruction with XS Prefix
+                    Requires<[UseSSE2]>; // SSE2 instruction with XS Prefix
 
 //===---------------------------------------------------------------------===//
 // Move Packed Quadword Int to Quadword Int
@@ -4744,7 +4828,7 @@ def MOVZQI2PQIrm : I<0x7E, MRMSrcMem, (outs VR128:$dst), (ins i64mem:$src),
                        (v2i64 (X86vzmovl (v2i64 (scalar_to_vector
                                                  (loadi64 addr:$src))))))],
                                                  IIC_SSE_MOVDQ>,
-                     XS, Requires<[HasSSE2]>;
+                     XS, Requires<[UseSSE2]>;
 
 let Predicates = [HasAVX], AddedComplexity = 20 in {
   def : Pat<(v2i64 (X86vzmovl (loadv2i64 addr:$src))),
@@ -4755,7 +4839,7 @@ let Predicates = [HasAVX], AddedComplexity = 20 in {
             (VMOVZQI2PQIrm addr:$src)>;
 }
 
-let Predicates = [HasSSE2], AddedComplexity = 20 in {
+let Predicates = [UseSSE2], AddedComplexity = 20 in {
   def : Pat<(v2i64 (X86vzmovl (loadv2i64 addr:$src))),
             (MOVZQI2PQIrm addr:$src)>;
   def : Pat<(v2i64 (X86vzmovl (bc_v2i64 (loadv4f32 addr:$src)))),
@@ -4785,7 +4869,7 @@ def MOVZPQILo2PQIrr : I<0x7E, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src),
                         "movq\t{$src, $dst|$dst, $src}",
                     [(set VR128:$dst, (v2i64 (X86vzmovl (v2i64 VR128:$src))))],
                     IIC_SSE_MOVQ_RR>,
-                      XS, Requires<[HasSSE2]>;
+                      XS, Requires<[UseSSE2]>;
 
 let AddedComplexity = 20 in
 def VMOVZPQILo2PQIrm : I<0x7E, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src),
@@ -4800,7 +4884,7 @@ def MOVZPQILo2PQIrm : I<0x7E, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src),
                     [(set VR128:$dst, (v2i64 (X86vzmovl
                                              (loadv2i64 addr:$src))))],
                                              IIC_SSE_MOVDQ>,
-                      XS, Requires<[HasSSE2]>;
+                      XS, Requires<[UseSSE2]>;
 }
 
 let AddedComplexity = 20 in {
@@ -4810,7 +4894,7 @@ let AddedComplexity = 20 in {
     def : Pat<(v2f64 (X86vzmovl (v2f64 VR128:$src))),
               (VMOVZPQILo2PQIrr VR128:$src)>;
   }
-  let Predicates = [HasSSE2] in {
+  let Predicates = [UseSSE2] in {
     def : Pat<(v2i64 (X86vzmovl (loadv2i64 addr:$src))),
               (MOVZPQILo2PQIrm addr:$src)>;
     def : Pat<(v2f64 (X86vzmovl (v2f64 VR128:$src))),
@@ -4862,9 +4946,9 @@ let Predicates = [HasAVX] in {
   defm VMOVSLDUP  : sse3_replicate_sfp<0x12, X86Movsldup, "vmovsldup",
                                        v4f32, VR128, memopv4f32, f128mem>, VEX;
   defm VMOVSHDUPY : sse3_replicate_sfp<0x16, X86Movshdup, "vmovshdup",
-                                       v8f32, VR256, memopv8f32, f256mem>, VEX;
+                                 v8f32, VR256, memopv8f32, f256mem>, VEX, VEX_L;
   defm VMOVSLDUPY : sse3_replicate_sfp<0x12, X86Movsldup, "vmovsldup",
-                                       v8f32, VR256, memopv8f32, f256mem>, VEX;
+                                 v8f32, VR256, memopv8f32, f256mem>, VEX, VEX_L;
 }
 defm MOVSHDUP : sse3_replicate_sfp<0x16, X86Movshdup, "movshdup", v4f32, VR128,
                                    memopv4f32, f128mem>;
@@ -4890,7 +4974,7 @@ let Predicates = [HasAVX] in {
             (VMOVSLDUPYrm addr:$src)>;
 }
 
-let Predicates = [HasSSE3] in {
+let Predicates = [UseSSE3] in {
   def : Pat<(v4i32 (X86Movshdup VR128:$src)),
             (MOVSHDUPrr VR128:$src)>;
   def : Pat<(v4i32 (X86Movshdup (bc_v4i32 (memopv2i64 addr:$src)))),
@@ -4932,7 +5016,7 @@ def rm  : S3DI<0x12, MRMSrcMem, (outs VR256:$dst), (ins f256mem:$src),
 
 let Predicates = [HasAVX] in {
   defm VMOVDDUP  : sse3_replicate_dfp<"vmovddup">, VEX;
-  defm VMOVDDUPY : sse3_replicate_dfp_y<"vmovddup">, VEX;
+  defm VMOVDDUPY : sse3_replicate_dfp_y<"vmovddup">, VEX, VEX_L;
 }
 
 defm MOVDDUP : sse3_replicate_dfp<"movddup">;
@@ -4959,7 +5043,7 @@ let Predicates = [HasAVX] in {
             (VMOVDDUPYrr VR256:$src)>;
 }
 
-let Predicates = [HasSSE3] in {
+let Predicates = [UseSSE3] in {
   def : Pat<(X86Movddup (memopv2f64 addr:$src)),
             (MOVDDUPrm addr:$src)>;
   def : Pat<(X86Movddup (bc_v2f64 (memopv4f32 addr:$src))),
@@ -4981,7 +5065,8 @@ let Predicates = [HasAVX] in {
                    [(set VR128:$dst, (int_x86_sse3_ldu_dq addr:$src))]>, VEX;
   def VLDDQUYrm : S3DI<0xF0, MRMSrcMem, (outs VR256:$dst), (ins i256mem:$src),
                    "vlddqu\t{$src, $dst|$dst, $src}",
-                   [(set VR256:$dst, (int_x86_avx_ldu_dq_256 addr:$src))]>, VEX;
+                   [(set VR256:$dst, (int_x86_avx_ldu_dq_256 addr:$src))]>,
+                   VEX, VEX_L;
 }
 def LDDQUrm : S3DI<0xF0, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src),
                    "lddqu\t{$src, $dst|$dst, $src}",
@@ -5014,16 +5099,16 @@ let Predicates = [HasAVX] in {
     defm VADDSUBPS : sse3_addsub<int_x86_sse3_addsub_ps, "vaddsubps", VR128,
                                  f128mem, SSE_ALU_F32P, 0>, TB, XD, VEX_4V;
     defm VADDSUBPSY : sse3_addsub<int_x86_avx_addsub_ps_256, "vaddsubps", VR256,
-                                 f256mem, SSE_ALU_F32P, 0>, TB, XD, VEX_4V;
+                               f256mem, SSE_ALU_F32P, 0>, TB, XD, VEX_4V, VEX_L;
   }
   let ExeDomain = SSEPackedDouble in {
     defm VADDSUBPD : sse3_addsub<int_x86_sse3_addsub_pd, "vaddsubpd", VR128,
                                  f128mem, SSE_ALU_F64P, 0>, TB, OpSize, VEX_4V;
     defm VADDSUBPDY : sse3_addsub<int_x86_avx_addsub_pd_256, "vaddsubpd", VR256,
-                                 f256mem, SSE_ALU_F64P, 0>, TB, OpSize, VEX_4V;
+                           f256mem, SSE_ALU_F64P, 0>, TB, OpSize, VEX_4V, VEX_L;
   }
 }
-let Constraints = "$src1 = $dst", Predicates = [HasSSE3] in {
+let Constraints = "$src1 = $dst", Predicates = [UseSSE3] in {
   let ExeDomain = SSEPackedSingle in
   defm ADDSUBPS : sse3_addsub<int_x86_sse3_addsub_ps, "addsubps", VR128,
                               f128mem, SSE_ALU_F32P>, TB, XD;
@@ -5075,9 +5160,9 @@ let Predicates = [HasAVX] in {
     defm VHSUBPS  : S3D_Int<0x7D, "vhsubps", v4f32, VR128, f128mem,
                             X86fhsub, 0>, VEX_4V;
     defm VHADDPSY : S3D_Int<0x7C, "vhaddps", v8f32, VR256, f256mem,
-                            X86fhadd, 0>, VEX_4V;
+                            X86fhadd, 0>, VEX_4V, VEX_L;
     defm VHSUBPSY : S3D_Int<0x7D, "vhsubps", v8f32, VR256, f256mem,
-                            X86fhsub, 0>, VEX_4V;
+                            X86fhsub, 0>, VEX_4V, VEX_L;
   }
   let ExeDomain = SSEPackedDouble in {
     defm VHADDPD  : S3_Int <0x7C, "vhaddpd", v2f64, VR128, f128mem,
@@ -5085,9 +5170,9 @@ let Predicates = [HasAVX] in {
     defm VHSUBPD  : S3_Int <0x7D, "vhsubpd", v2f64, VR128, f128mem,
                             X86fhsub, 0>, VEX_4V;
     defm VHADDPDY : S3_Int <0x7C, "vhaddpd", v4f64, VR256, f256mem,
-                            X86fhadd, 0>, VEX_4V;
+                            X86fhadd, 0>, VEX_4V, VEX_L;
     defm VHSUBPDY : S3_Int <0x7D, "vhsubpd", v4f64, VR256, f256mem,
-                            X86fhsub, 0>, VEX_4V;
+                            X86fhsub, 0>, VEX_4V, VEX_L;
   }
 }
 
@@ -5153,11 +5238,11 @@ let Predicates = [HasAVX] in {
 
 let Predicates = [HasAVX2] in {
   defm VPABSB  : SS3I_unop_rm_int_y<0x1C, "vpabsb",
-                                    int_x86_avx2_pabs_b>, VEX;
+                                    int_x86_avx2_pabs_b>, VEX, VEX_L;
   defm VPABSW  : SS3I_unop_rm_int_y<0x1D, "vpabsw",
-                                    int_x86_avx2_pabs_w>, VEX;
+                                    int_x86_avx2_pabs_w>, VEX, VEX_L;
   defm VPABSD  : SS3I_unop_rm_int_y<0x1E, "vpabsd",
-                                    int_x86_avx2_pabs_d>, VEX;
+                                    int_x86_avx2_pabs_d>, VEX, VEX_L;
 }
 
 defm PABSB : SS3I_unop_rm_int<0x1C, "pabsb",
@@ -5296,37 +5381,37 @@ let ImmT = NoImm, Predicates = [HasAVX2] in {
 let isCommutable = 0 in {
   defm VPHADDWY   : SS3I_binop_rm<0x01, "vphaddw", X86hadd, v16i16, VR256,
                                   memopv4i64, i256mem,
-                                  SSE_PHADDSUBW, 0>, VEX_4V;
+                                  SSE_PHADDSUBW, 0>, VEX_4V, VEX_L;
   defm VPHADDDY   : SS3I_binop_rm<0x02, "vphaddd", X86hadd, v8i32, VR256,
                                   memopv4i64, i256mem,
-                                  SSE_PHADDSUBW, 0>, VEX_4V;
+                                  SSE_PHADDSUBW, 0>, VEX_4V, VEX_L;
   defm VPHSUBWY   : SS3I_binop_rm<0x05, "vphsubw", X86hsub, v16i16, VR256,
                                   memopv4i64, i256mem,
-                                  SSE_PHADDSUBW, 0>, VEX_4V;
+                                  SSE_PHADDSUBW, 0>, VEX_4V, VEX_L;
   defm VPHSUBDY   : SS3I_binop_rm<0x06, "vphsubd", X86hsub, v8i32, VR256,
                                   memopv4i64, i256mem,
-                                  SSE_PHADDSUBW, 0>, VEX_4V;
+                                  SSE_PHADDSUBW, 0>, VEX_4V, VEX_L;
   defm VPSIGNBY   : SS3I_binop_rm<0x08, "vpsignb", X86psign, v32i8, VR256,
                                   memopv4i64, i256mem,
-                                  SSE_PHADDSUBW, 0>, VEX_4V;
+                                  SSE_PHADDSUBW, 0>, VEX_4V, VEX_L;
   defm VPSIGNWY   : SS3I_binop_rm<0x09, "vpsignw", X86psign, v16i16, VR256,
                                   memopv4i64, i256mem,
-                                  SSE_PHADDSUBW, 0>, VEX_4V;
+                                  SSE_PHADDSUBW, 0>, VEX_4V, VEX_L;
   defm VPSIGNDY   : SS3I_binop_rm<0x0A, "vpsignd", X86psign, v8i32, VR256,
                                   memopv4i64, i256mem,
-                                  SSE_PHADDSUBW, 0>, VEX_4V;
+                                  SSE_PHADDSUBW, 0>, VEX_4V, VEX_L;
   defm VPSHUFBY   : SS3I_binop_rm<0x00, "vpshufb", X86pshufb, v32i8, VR256,
                                   memopv4i64, i256mem,
-                                  SSE_PHADDSUBW, 0>, VEX_4V;
+                                  SSE_PHADDSUBW, 0>, VEX_4V, VEX_L;
   defm VPHADDSW   : SS3I_binop_rm_int_y<0x03, "vphaddsw",
-                                        int_x86_avx2_phadd_sw>, VEX_4V;
+                                        int_x86_avx2_phadd_sw>, VEX_4V, VEX_L;
   defm VPHSUBSW   : SS3I_binop_rm_int_y<0x07, "vphsubsw",
-                                        int_x86_avx2_phsub_sw>, VEX_4V;
+                                        int_x86_avx2_phsub_sw>, VEX_4V, VEX_L;
   defm VPMADDUBSW : SS3I_binop_rm_int_y<0x04, "vpmaddubsw",
-                                        int_x86_avx2_pmadd_ub_sw>, VEX_4V;
+                                       int_x86_avx2_pmadd_ub_sw>, VEX_4V, VEX_L;
 }
 defm VPMULHRSW    : SS3I_binop_rm_int_y<0x0B, "vpmulhrsw",
-                                        int_x86_avx2_pmul_hr_sw>, VEX_4V;
+                                        int_x86_avx2_pmul_hr_sw>, VEX_4V, VEX_L;
 }
 
 // None of these have i8 immediate fields.
@@ -5405,8 +5490,8 @@ multiclass ssse3_palign_y<string asm, bit Is2Addr = 1> {
 let Predicates = [HasAVX] in
   defm VPALIGN : ssse3_palign<"vpalignr", 0>, VEX_4V;
 let Predicates = [HasAVX2] in
-  defm VPALIGN : ssse3_palign_y<"vpalignr", 0>, VEX_4V;
-let Constraints = "$src1 = $dst", Predicates = [HasSSSE3] in
+  defm VPALIGN : ssse3_palign_y<"vpalignr", 0>, VEX_4V, VEX_L;
+let Constraints = "$src1 = $dst", Predicates = [UseSSSE3] in
   defm PALIGN : ssse3_palign<"palignr">;
 
 let Predicates = [HasAVX2] in {
@@ -5431,7 +5516,7 @@ def : Pat<(v16i8 (X86PAlign VR128:$src1, VR128:$src2, (i8 imm:$imm))),
           (VPALIGNR128rr VR128:$src2, VR128:$src1, imm:$imm)>;
 }
 
-let Predicates = [HasSSSE3] in {
+let Predicates = [UseSSSE3] in {
 def : Pat<(v4i32 (X86PAlign VR128:$src1, VR128:$src2, (i8 imm:$imm))),
           (PALIGNR128rr VR128:$src2, VR128:$src1, imm:$imm)>;
 def : Pat<(v4f32 (X86PAlign VR128:$src1, VR128:$src2, (i8 imm:$imm))),
@@ -5512,17 +5597,17 @@ defm VPMOVZXDQ : SS41I_binop_rm_int8<0x35, "vpmovzxdq", int_x86_sse41_pmovzxdq>,
 
 let Predicates = [HasAVX2] in {
 defm VPMOVSXBW : SS41I_binop_rm_int16_y<0x20, "vpmovsxbw",
-                                        int_x86_avx2_pmovsxbw>, VEX;
+                                        int_x86_avx2_pmovsxbw>, VEX, VEX_L;
 defm VPMOVSXWD : SS41I_binop_rm_int16_y<0x23, "vpmovsxwd",
-                                        int_x86_avx2_pmovsxwd>, VEX;
+                                        int_x86_avx2_pmovsxwd>, VEX, VEX_L;
 defm VPMOVSXDQ : SS41I_binop_rm_int16_y<0x25, "vpmovsxdq",
-                                        int_x86_avx2_pmovsxdq>, VEX;
+                                        int_x86_avx2_pmovsxdq>, VEX, VEX_L;
 defm VPMOVZXBW : SS41I_binop_rm_int16_y<0x30, "vpmovzxbw",
-                                        int_x86_avx2_pmovzxbw>, VEX;
+                                        int_x86_avx2_pmovzxbw>, VEX, VEX_L;
 defm VPMOVZXWD : SS41I_binop_rm_int16_y<0x33, "vpmovzxwd",
-                                        int_x86_avx2_pmovzxwd>, VEX;
+                                        int_x86_avx2_pmovzxwd>, VEX, VEX_L;
 defm VPMOVZXDQ : SS41I_binop_rm_int16_y<0x35, "vpmovzxdq",
-                                        int_x86_avx2_pmovzxdq>, VEX;
+                                        int_x86_avx2_pmovzxdq>, VEX, VEX_L;
 }
 
 defm PMOVSXBW   : SS41I_binop_rm_int8<0x20, "pmovsxbw", int_x86_sse41_pmovsxbw>;
@@ -5538,64 +5623,88 @@ let Predicates = [HasAVX] in {
             (VPMOVSXBWrm addr:$src)>;
   def : Pat<(int_x86_sse41_pmovsxbw (vzload_v2i64 addr:$src)),
             (VPMOVSXBWrm addr:$src)>;
+  def : Pat<(int_x86_sse41_pmovsxbw (bc_v16i8 (loadv2i64 addr:$src))),
+            (VPMOVSXBWrm addr:$src)>;
 
   def : Pat<(int_x86_sse41_pmovsxwd (vzmovl_v2i64 addr:$src)),
             (VPMOVSXWDrm addr:$src)>;
   def : Pat<(int_x86_sse41_pmovsxwd (vzload_v2i64 addr:$src)),
             (VPMOVSXWDrm addr:$src)>;
+  def : Pat<(int_x86_sse41_pmovsxwd (bc_v8i16 (loadv2i64 addr:$src))),
+            (VPMOVSXWDrm addr:$src)>;
 
   def : Pat<(int_x86_sse41_pmovsxdq (vzmovl_v2i64 addr:$src)),
             (VPMOVSXDQrm addr:$src)>;
   def : Pat<(int_x86_sse41_pmovsxdq (vzload_v2i64 addr:$src)),
             (VPMOVSXDQrm addr:$src)>;
+  def : Pat<(int_x86_sse41_pmovsxdq (bc_v4i32 (loadv2i64 addr:$src))),
+            (VPMOVSXDQrm addr:$src)>;
 
   def : Pat<(int_x86_sse41_pmovzxbw (vzmovl_v2i64 addr:$src)),
             (VPMOVZXBWrm addr:$src)>;
   def : Pat<(int_x86_sse41_pmovzxbw (vzload_v2i64 addr:$src)),
             (VPMOVZXBWrm addr:$src)>;
+  def : Pat<(int_x86_sse41_pmovzxbw (bc_v16i8 (loadv2i64 addr:$src))),
+            (VPMOVZXBWrm addr:$src)>;
 
   def : Pat<(int_x86_sse41_pmovzxwd (vzmovl_v2i64 addr:$src)),
             (VPMOVZXWDrm addr:$src)>;
   def : Pat<(int_x86_sse41_pmovzxwd (vzload_v2i64 addr:$src)),
             (VPMOVZXWDrm addr:$src)>;
+  def : Pat<(int_x86_sse41_pmovzxwd (bc_v8i16 (loadv2i64 addr:$src))),
+            (VPMOVZXWDrm addr:$src)>;
 
   def : Pat<(int_x86_sse41_pmovzxdq (vzmovl_v2i64 addr:$src)),
             (VPMOVZXDQrm addr:$src)>;
   def : Pat<(int_x86_sse41_pmovzxdq (vzload_v2i64 addr:$src)),
             (VPMOVZXDQrm addr:$src)>;
+  def : Pat<(int_x86_sse41_pmovzxdq (bc_v4i32 (loadv2i64 addr:$src))),
+            (VPMOVZXDQrm addr:$src)>;
 }
 
-let Predicates = [HasSSE41] in {
+let Predicates = [UseSSE41] in {
   // Common patterns involving scalar load.
   def : Pat<(int_x86_sse41_pmovsxbw (vzmovl_v2i64 addr:$src)),
             (PMOVSXBWrm addr:$src)>;
   def : Pat<(int_x86_sse41_pmovsxbw (vzload_v2i64 addr:$src)),
             (PMOVSXBWrm addr:$src)>;
+  def : Pat<(int_x86_sse41_pmovsxbw (bc_v16i8 (loadv2i64 addr:$src))),
+            (PMOVSXBWrm addr:$src)>;
 
   def : Pat<(int_x86_sse41_pmovsxwd (vzmovl_v2i64 addr:$src)),
             (PMOVSXWDrm addr:$src)>;
   def : Pat<(int_x86_sse41_pmovsxwd (vzload_v2i64 addr:$src)),
             (PMOVSXWDrm addr:$src)>;
+  def : Pat<(int_x86_sse41_pmovsxwd (bc_v8i16 (loadv2i64 addr:$src))),
+            (PMOVSXWDrm addr:$src)>;
 
   def : Pat<(int_x86_sse41_pmovsxdq (vzmovl_v2i64 addr:$src)),
             (PMOVSXDQrm addr:$src)>;
   def : Pat<(int_x86_sse41_pmovsxdq (vzload_v2i64 addr:$src)),
             (PMOVSXDQrm addr:$src)>;
+  def : Pat<(int_x86_sse41_pmovsxdq (bc_v4i32 (loadv2i64 addr:$src))),
+            (PMOVSXDQrm addr:$src)>;
 
   def : Pat<(int_x86_sse41_pmovzxbw (vzmovl_v2i64 addr:$src)),
             (PMOVZXBWrm addr:$src)>;
   def : Pat<(int_x86_sse41_pmovzxbw (vzload_v2i64 addr:$src)),
             (PMOVZXBWrm addr:$src)>;
+  def : Pat<(int_x86_sse41_pmovzxbw (bc_v16i8 (loadv2i64 addr:$src))),
+            (PMOVZXBWrm addr:$src)>;
 
   def : Pat<(int_x86_sse41_pmovzxwd (vzmovl_v2i64 addr:$src)),
             (PMOVZXWDrm addr:$src)>;
   def : Pat<(int_x86_sse41_pmovzxwd (vzload_v2i64 addr:$src)),
             (PMOVZXWDrm addr:$src)>;
+  def : Pat<(int_x86_sse41_pmovzxwd (bc_v8i16 (loadv2i64 addr:$src))),
+            (PMOVZXWDrm addr:$src)>;
 
   def : Pat<(int_x86_sse41_pmovzxdq (vzmovl_v2i64 addr:$src)),
             (PMOVZXDQrm addr:$src)>;
   def : Pat<(int_x86_sse41_pmovzxdq (vzload_v2i64 addr:$src)),
             (PMOVZXDQrm addr:$src)>;
+  def : Pat<(int_x86_sse41_pmovzxdq (bc_v4i32 (loadv2i64 addr:$src))),
+            (PMOVZXDQrm addr:$src)>;
 }
 
 let Predicates = [HasAVX2] in {
@@ -5615,7 +5724,7 @@ let Predicates = [HasAVX] in {
   def : Pat<(v4i32 (X86vsmovl (v8i16 VR128:$src))), (VPMOVSXWDrr VR128:$src)>;
 }
 
-let Predicates = [HasSSE41] in {
+let Predicates = [UseSSE41] in {
   def : Pat<(v2i64 (X86vsmovl (v4i32 VR128:$src))), (PMOVSXDQrr VR128:$src)>;
   def : Pat<(v4i32 (X86vsmovl (v8i16 VR128:$src))), (PMOVSXWDrr VR128:$src)>;
 }
@@ -5659,13 +5768,13 @@ defm VPMOVZXWQ : SS41I_binop_rm_int4<0x34, "vpmovzxwq", int_x86_sse41_pmovzxwq>,
 
 let Predicates = [HasAVX2] in {
 defm VPMOVSXBD : SS41I_binop_rm_int8_y<0x21, "vpmovsxbd",
-                                       int_x86_avx2_pmovsxbd>, VEX;
+                                       int_x86_avx2_pmovsxbd>, VEX, VEX_L;
 defm VPMOVSXWQ : SS41I_binop_rm_int8_y<0x24, "vpmovsxwq",
-                                       int_x86_avx2_pmovsxwq>, VEX;
+                                       int_x86_avx2_pmovsxwq>, VEX, VEX_L;
 defm VPMOVZXBD : SS41I_binop_rm_int8_y<0x31, "vpmovzxbd",
-                                       int_x86_avx2_pmovzxbd>, VEX;
+                                       int_x86_avx2_pmovzxbd>, VEX, VEX_L;
 defm VPMOVZXWQ : SS41I_binop_rm_int8_y<0x34, "vpmovzxwq",
-                                       int_x86_avx2_pmovzxwq>, VEX;
+                                       int_x86_avx2_pmovzxwq>, VEX, VEX_L;
 }
 
 defm PMOVSXBD   : SS41I_binop_rm_int4<0x21, "pmovsxbd", int_x86_sse41_pmovsxbd>;
@@ -5686,7 +5795,7 @@ let Predicates = [HasAVX] in {
             (VPMOVZXWQrm addr:$src)>;
 }
 
-let Predicates = [HasSSE41] in {
+let Predicates = [UseSSE41] in {
   // Common patterns involving scalar load
   def : Pat<(int_x86_sse41_pmovsxbd (vzmovl_v4i32 addr:$src)),
             (PMOVSXBDrm addr:$src)>;
@@ -5734,9 +5843,9 @@ defm VPMOVZXBQ : SS41I_binop_rm_int2<0x32, "vpmovzxbq", int_x86_sse41_pmovzxbq>,
 }
 let Predicates = [HasAVX2] in {
 defm VPMOVSXBQ : SS41I_binop_rm_int4_y<0x22, "vpmovsxbq",
-                                       int_x86_avx2_pmovsxbq>, VEX;
+                                       int_x86_avx2_pmovsxbq>, VEX, VEX_L;
 defm VPMOVZXBQ : SS41I_binop_rm_int4_y<0x32, "vpmovzxbq",
-                                       int_x86_avx2_pmovzxbq>, VEX;
+                                       int_x86_avx2_pmovzxbq>, VEX, VEX_L;
 }
 defm PMOVSXBQ   : SS41I_binop_rm_int2<0x22, "pmovsxbq", int_x86_sse41_pmovsxbq>;
 defm PMOVZXBQ   : SS41I_binop_rm_int2<0x32, "pmovzxbq", int_x86_sse41_pmovzxbq>;
@@ -5754,7 +5863,7 @@ let Predicates = [HasAVX] in {
             (VPMOVZXBQrm addr:$src)>;
 }
 
-let Predicates = [HasSSE41] in {
+let Predicates = [UseSSE41] in {
   // Common patterns involving scalar load
   def : Pat<(int_x86_sse41_pmovsxbq
               (bitconvert (v4i32 (X86vzmovl
@@ -5767,6 +5876,100 @@ let Predicates = [HasSSE41] in {
             (PMOVZXBQrm addr:$src)>;
 }
 
+let Predicates = [HasAVX2] in {
+  def : Pat<(v16i16 (X86vzext (v16i8 VR128:$src))), (VPMOVZXBWYrr VR128:$src)>;
+  def : Pat<(v8i32  (X86vzext (v16i8 VR128:$src))), (VPMOVZXBDYrr VR128:$src)>;
+  def : Pat<(v4i64  (X86vzext (v16i8 VR128:$src))), (VPMOVZXBQYrr VR128:$src)>;
+
+  def : Pat<(v8i32  (X86vzext (v8i16 VR128:$src))), (VPMOVZXWDYrr VR128:$src)>;
+  def : Pat<(v4i64  (X86vzext (v8i16 VR128:$src))), (VPMOVZXWQYrr VR128:$src)>;
+
+  def : Pat<(v4i64  (X86vzext (v4i32 VR128:$src))), (VPMOVZXDQYrr VR128:$src)>;
+
+  def : Pat<(v16i16 (X86vzext (v32i8 VR256:$src))),
+            (VPMOVZXBWYrr (EXTRACT_SUBREG VR256:$src, sub_xmm))>;
+  def : Pat<(v8i32 (X86vzext (v32i8 VR256:$src))),
+            (VPMOVZXBDYrr (EXTRACT_SUBREG VR256:$src, sub_xmm))>;
+  def : Pat<(v4i64 (X86vzext (v32i8 VR256:$src))),
+            (VPMOVZXBQYrr (EXTRACT_SUBREG VR256:$src, sub_xmm))>;
+
+  def : Pat<(v8i32 (X86vzext (v16i16 VR256:$src))),
+            (VPMOVZXWDYrr (EXTRACT_SUBREG VR256:$src, sub_xmm))>;
+  def : Pat<(v4i64 (X86vzext (v16i16 VR256:$src))),
+            (VPMOVZXWQYrr (EXTRACT_SUBREG VR256:$src, sub_xmm))>;
+
+  def : Pat<(v4i64 (X86vzext (v8i32 VR256:$src))),
+            (VPMOVZXDQYrr (EXTRACT_SUBREG VR256:$src, sub_xmm))>;
+}
+
+let Predicates = [HasAVX] in {
+  def : Pat<(v8i16 (X86vzext (v16i8 VR128:$src))), (VPMOVZXBWrr VR128:$src)>;
+  def : Pat<(v4i32 (X86vzext (v16i8 VR128:$src))), (VPMOVZXBDrr VR128:$src)>;
+  def : Pat<(v2i64 (X86vzext (v16i8 VR128:$src))), (VPMOVZXBQrr VR128:$src)>;
+
+  def : Pat<(v4i32 (X86vzext (v8i16 VR128:$src))), (VPMOVZXWDrr VR128:$src)>;
+  def : Pat<(v2i64 (X86vzext (v8i16 VR128:$src))), (VPMOVZXWQrr VR128:$src)>;
+
+  def : Pat<(v2i64 (X86vzext (v4i32 VR128:$src))), (VPMOVZXDQrr VR128:$src)>;
+
+  def : Pat<(v8i16 (X86vzext (v16i8 (bitconvert (v2i64 (scalar_to_vector (loadi64 addr:$src))))))),
+            (VPMOVZXBWrm addr:$src)>;
+  def : Pat<(v8i16 (X86vzext (v16i8 (bitconvert (v2f64 (scalar_to_vector (loadf64 addr:$src))))))),
+            (VPMOVZXBWrm addr:$src)>;
+  def : Pat<(v4i32 (X86vzext (v16i8 (bitconvert (v4i32 (scalar_to_vector (loadi32 addr:$src))))))),
+            (VPMOVZXBDrm addr:$src)>;
+  def : Pat<(v2i64 (X86vzext (v16i8 (bitconvert (v4i32 (scalar_to_vector (loadi16_anyext addr:$src))))))),
+            (VPMOVZXBQrm addr:$src)>;
+
+  def : Pat<(v4i32 (X86vzext (v8i16 (bitconvert (v2i64 (scalar_to_vector (loadi64 addr:$src))))))),
+            (VPMOVZXWDrm addr:$src)>;
+  def : Pat<(v4i32 (X86vzext (v8i16 (bitconvert (v2f64 (scalar_to_vector (loadf64 addr:$src))))))),
+            (VPMOVZXWDrm addr:$src)>;
+  def : Pat<(v2i64 (X86vzext (v8i16 (bitconvert (v4i32 (scalar_to_vector (loadi32 addr:$src))))))),
+            (VPMOVZXWQrm addr:$src)>;
+
+  def : Pat<(v2i64 (X86vzext (v4i32 (bitconvert (v2i64 (scalar_to_vector (loadi64 addr:$src))))))),
+            (VPMOVZXDQrm addr:$src)>;
+  def : Pat<(v2i64 (X86vzext (v4i32 (bitconvert (v2f64 (scalar_to_vector (loadf64 addr:$src))))))),
+            (VPMOVZXDQrm addr:$src)>;
+  def : Pat<(v2i64 (X86vzext (v4i32 (bitconvert (v2i64 (X86vzload addr:$src)))))),
+            (VPMOVZXDQrm addr:$src)>;
+}
+
+let Predicates = [UseSSE41] in {
+  def : Pat<(v8i16 (X86vzext (v16i8 VR128:$src))), (PMOVZXBWrr VR128:$src)>;
+  def : Pat<(v4i32 (X86vzext (v16i8 VR128:$src))), (PMOVZXBDrr VR128:$src)>;
+  def : Pat<(v2i64 (X86vzext (v16i8 VR128:$src))), (PMOVZXBQrr VR128:$src)>;
+
+  def : Pat<(v4i32 (X86vzext (v8i16 VR128:$src))), (PMOVZXWDrr VR128:$src)>;
+  def : Pat<(v2i64 (X86vzext (v8i16 VR128:$src))), (PMOVZXWQrr VR128:$src)>;
+
+  def : Pat<(v2i64 (X86vzext (v4i32 VR128:$src))), (PMOVZXDQrr VR128:$src)>;
+
+  def : Pat<(v8i16 (X86vzext (v16i8 (bitconvert (v2i64 (scalar_to_vector (loadi64 addr:$src))))))),
+            (PMOVZXBWrm addr:$src)>;
+  def : Pat<(v8i16 (X86vzext (v16i8 (bitconvert (v2f64 (scalar_to_vector (loadf64 addr:$src))))))),
+            (PMOVZXBWrm addr:$src)>;
+  def : Pat<(v4i32 (X86vzext (v16i8 (bitconvert (v4i32 (scalar_to_vector (loadi32 addr:$src))))))),
+            (PMOVZXBDrm addr:$src)>;
+  def : Pat<(v2i64 (X86vzext (v16i8 (bitconvert (v4i32 (scalar_to_vector (loadi16_anyext addr:$src))))))),
+            (PMOVZXBQrm addr:$src)>;
+
+  def : Pat<(v4i32 (X86vzext (v8i16 (bitconvert (v2i64 (scalar_to_vector (loadi64 addr:$src))))))),
+            (PMOVZXWDrm addr:$src)>;
+  def : Pat<(v4i32 (X86vzext (v8i16 (bitconvert (v2f64 (scalar_to_vector (loadf64 addr:$src))))))),
+            (PMOVZXWDrm addr:$src)>;
+  def : Pat<(v2i64 (X86vzext (v8i16 (bitconvert (v4i32 (scalar_to_vector (loadi32 addr:$src))))))),
+            (PMOVZXWQrm addr:$src)>;
+
+  def : Pat<(v2i64 (X86vzext (v4i32 (bitconvert (v2i64 (scalar_to_vector (loadi64 addr:$src))))))),
+            (PMOVZXDQrm addr:$src)>;
+  def : Pat<(v2i64 (X86vzext (v4i32 (bitconvert (v2f64 (scalar_to_vector (loadf64 addr:$src))))))),
+            (PMOVZXDQrm addr:$src)>;
+  def : Pat<(v2i64 (X86vzext (v4i32 (bitconvert (v2i64 (X86vzload addr:$src)))))),
+            (PMOVZXDQrm addr:$src)>;
+}
+
 //===----------------------------------------------------------------------===//
 // SSE4.1 - Extract Instructions
 //===----------------------------------------------------------------------===//
@@ -5900,7 +6103,7 @@ def : Pat<(store (f32 (bitconvert (extractelt (bc_v4i32 (v4f32 VR128:$src1)),
                                               imm:$src2))),
                  addr:$dst),
           (EXTRACTPSmr addr:$dst, VR128:$src1, imm:$src2)>,
-          Requires<[HasSSE41]>;
+          Requires<[UseSSE41]>;
 
 //===----------------------------------------------------------------------===//
 // SSE4.1 - Insert Instructions
@@ -6147,7 +6350,7 @@ let Predicates = [HasAVX] in {
   defm VROUNDY : sse41_fp_unop_rm<0x08, 0x09, "vround", f256mem, VR256,
                                   memopv8f32, memopv4f64,
                                   int_x86_avx_round_ps_256,
-                                  int_x86_avx_round_pd_256>, VEX;
+                                  int_x86_avx_round_pd_256>, VEX, VEX_L;
   defm VROUND  : sse41_fp_binop_rm<0x0A, 0x0B, "vround",
                                   int_x86_sse41_round_ss,
                                   int_x86_sse41_round_sd, 0>, VEX_4V, VEX_LIG;
@@ -6172,6 +6375,15 @@ let Predicates = [HasAVX] in {
             (VROUNDSSr (f32 (IMPLICIT_DEF)), FR32:$src, (i32 0x3))>;
   def : Pat<(f64 (ftrunc FR64:$src)),
             (VROUNDSDr (f64 (IMPLICIT_DEF)), FR64:$src, (i32 0x3))>;
+
+  def : Pat<(v4f32 (ffloor VR128:$src)),
+            (VROUNDPSr VR128:$src, (i32 0x1))>;
+  def : Pat<(v2f64 (ffloor VR128:$src)),
+            (VROUNDPDr VR128:$src, (i32 0x1))>;
+  def : Pat<(v8f32 (ffloor VR256:$src)),
+            (VROUNDYPSr VR256:$src, (i32 0x1))>;
+  def : Pat<(v4f64 (ffloor VR256:$src)),
+            (VROUNDYPDr VR256:$src, (i32 0x1))>;
 }
 
 defm ROUND  : sse41_fp_unop_rm<0x08, 0x09, "round", f128mem, VR128,
@@ -6181,26 +6393,33 @@ let Constraints = "$src1 = $dst" in
 defm ROUND  : sse41_fp_binop_rm<0x0A, 0x0B, "round",
                                int_x86_sse41_round_ss, int_x86_sse41_round_sd>;
 
-def : Pat<(ffloor FR32:$src),
-          (ROUNDSSr (f32 (IMPLICIT_DEF)), FR32:$src, (i32 0x1))>;
-def : Pat<(f64 (ffloor FR64:$src)),
-          (ROUNDSDr (f64 (IMPLICIT_DEF)), FR64:$src, (i32 0x1))>;
-def : Pat<(f32 (fnearbyint FR32:$src)),
-          (ROUNDSSr (f32 (IMPLICIT_DEF)), FR32:$src, (i32 0xC))>;
-def : Pat<(f64 (fnearbyint FR64:$src)),
-          (ROUNDSDr (f64 (IMPLICIT_DEF)), FR64:$src, (i32 0xC))>;
-def : Pat<(f32 (fceil FR32:$src)),
-          (ROUNDSSr (f32 (IMPLICIT_DEF)), FR32:$src, (i32 0x2))>;
-def : Pat<(f64 (fceil FR64:$src)),
-          (ROUNDSDr (f64 (IMPLICIT_DEF)), FR64:$src, (i32 0x2))>;
-def : Pat<(f32 (frint FR32:$src)),
-          (ROUNDSSr (f32 (IMPLICIT_DEF)), FR32:$src, (i32 0x4))>;
-def : Pat<(f64 (frint FR64:$src)),
-          (ROUNDSDr (f64 (IMPLICIT_DEF)), FR64:$src, (i32 0x4))>;
-def : Pat<(f32 (ftrunc FR32:$src)),
-          (ROUNDSSr (f32 (IMPLICIT_DEF)), FR32:$src, (i32 0x3))>;
-def : Pat<(f64 (ftrunc FR64:$src)),
-          (ROUNDSDr (f64 (IMPLICIT_DEF)), FR64:$src, (i32 0x3))>;
+let Predicates = [UseSSE41] in {
+  def : Pat<(ffloor FR32:$src),
+            (ROUNDSSr (f32 (IMPLICIT_DEF)), FR32:$src, (i32 0x1))>;
+  def : Pat<(f64 (ffloor FR64:$src)),
+            (ROUNDSDr (f64 (IMPLICIT_DEF)), FR64:$src, (i32 0x1))>;
+  def : Pat<(f32 (fnearbyint FR32:$src)),
+            (ROUNDSSr (f32 (IMPLICIT_DEF)), FR32:$src, (i32 0xC))>;
+  def : Pat<(f64 (fnearbyint FR64:$src)),
+            (ROUNDSDr (f64 (IMPLICIT_DEF)), FR64:$src, (i32 0xC))>;
+  def : Pat<(f32 (fceil FR32:$src)),
+            (ROUNDSSr (f32 (IMPLICIT_DEF)), FR32:$src, (i32 0x2))>;
+  def : Pat<(f64 (fceil FR64:$src)),
+            (ROUNDSDr (f64 (IMPLICIT_DEF)), FR64:$src, (i32 0x2))>;
+  def : Pat<(f32 (frint FR32:$src)),
+            (ROUNDSSr (f32 (IMPLICIT_DEF)), FR32:$src, (i32 0x4))>;
+  def : Pat<(f64 (frint FR64:$src)),
+            (ROUNDSDr (f64 (IMPLICIT_DEF)), FR64:$src, (i32 0x4))>;
+  def : Pat<(f32 (ftrunc FR32:$src)),
+            (ROUNDSSr (f32 (IMPLICIT_DEF)), FR32:$src, (i32 0x3))>;
+  def : Pat<(f64 (ftrunc FR64:$src)),
+            (ROUNDSDr (f64 (IMPLICIT_DEF)), FR64:$src, (i32 0x3))>;
+
+  def : Pat<(v4f32 (ffloor VR128:$src)),
+            (ROUNDPSr VR128:$src, (i32 0x1))>;
+  def : Pat<(v2f64 (ffloor VR128:$src)),
+            (ROUNDPDr VR128:$src, (i32 0x1))>;
+}
 
 //===----------------------------------------------------------------------===//
 // SSE4.1 - Packed Bit Test
@@ -6221,11 +6440,11 @@ def VPTESTrm  : SS48I<0x17, MRMSrcMem, (outs), (ins VR128:$src1, f128mem:$src2),
 def VPTESTYrr : SS48I<0x17, MRMSrcReg, (outs), (ins VR256:$src1, VR256:$src2),
                 "vptest\t{$src2, $src1|$src1, $src2}",
                 [(set EFLAGS, (X86ptest VR256:$src1, (v4i64 VR256:$src2)))]>,
-                OpSize, VEX;
+                OpSize, VEX, VEX_L;
 def VPTESTYrm : SS48I<0x17, MRMSrcMem, (outs), (ins VR256:$src1, i256mem:$src2),
                 "vptest\t{$src2, $src1|$src1, $src2}",
                 [(set EFLAGS,(X86ptest VR256:$src1, (memopv4i64 addr:$src2)))]>,
-                OpSize, VEX;
+                OpSize, VEX, VEX_L;
 }
 
 let Defs = [EFLAGS] in {
@@ -6254,11 +6473,13 @@ multiclass avx_bittest<bits<8> opc, string OpcodeStr, RegisterClass RC,
 let Defs = [EFLAGS], Predicates = [HasAVX] in {
 let ExeDomain = SSEPackedSingle in {
 defm VTESTPS  : avx_bittest<0x0E, "vtestps", VR128, f128mem, memopv4f32, v4f32>;
-defm VTESTPSY : avx_bittest<0x0E, "vtestps", VR256, f256mem, memopv8f32, v8f32>;
+defm VTESTPSY : avx_bittest<0x0E, "vtestps", VR256, f256mem, memopv8f32, v8f32>,
+                            VEX_L;
 }
 let ExeDomain = SSEPackedDouble in {
 defm VTESTPD  : avx_bittest<0x0F, "vtestpd", VR128, f128mem, memopv2f64, v2f64>;
-defm VTESTPDY : avx_bittest<0x0F, "vtestpd", VR256, f256mem, memopv4f64, v4f64>;
+defm VTESTPDY : avx_bittest<0x0F, "vtestpd", VR256, f256mem, memopv4f64, v4f64>,
+                            VEX_L;
 }
 }
 
@@ -6338,7 +6559,7 @@ multiclass SS41I_binop_rm_int<bits<8> opc, string OpcodeStr,
           (bitconvert (memopv2i64 addr:$src2))))]>, OpSize;
 }
 
-/// SS41I_binop_rm_int - Simple SSE 4.1 binary operator
+/// SS41I_binop_rm_int_y - Simple SSE 4.1 binary operator
 multiclass SS41I_binop_rm_int_y<bits<8> opc, string OpcodeStr,
                                 Intrinsic IntId256> {
   let isCommutable = 1 in
@@ -6381,25 +6602,25 @@ let Predicates = [HasAVX] in {
 let Predicates = [HasAVX2] in {
   let isCommutable = 0 in
   defm VPACKUSDW : SS41I_binop_rm_int_y<0x2B, "vpackusdw",
-                                        int_x86_avx2_packusdw>, VEX_4V;
+                                        int_x86_avx2_packusdw>, VEX_4V, VEX_L;
   defm VPMINSB   : SS41I_binop_rm_int_y<0x38, "vpminsb",
-                                        int_x86_avx2_pmins_b>, VEX_4V;
+                                        int_x86_avx2_pmins_b>, VEX_4V, VEX_L;
   defm VPMINSD   : SS41I_binop_rm_int_y<0x39, "vpminsd",
-                                        int_x86_avx2_pmins_d>, VEX_4V;
+                                        int_x86_avx2_pmins_d>, VEX_4V, VEX_L;
   defm VPMINUD   : SS41I_binop_rm_int_y<0x3B, "vpminud",
-                                        int_x86_avx2_pminu_d>, VEX_4V;
+                                        int_x86_avx2_pminu_d>, VEX_4V, VEX_L;
   defm VPMINUW   : SS41I_binop_rm_int_y<0x3A, "vpminuw",
-                                        int_x86_avx2_pminu_w>, VEX_4V;
+                                        int_x86_avx2_pminu_w>, VEX_4V, VEX_L;
   defm VPMAXSB   : SS41I_binop_rm_int_y<0x3C, "vpmaxsb",
-                                        int_x86_avx2_pmaxs_b>, VEX_4V;
+                                        int_x86_avx2_pmaxs_b>, VEX_4V, VEX_L;
   defm VPMAXSD   : SS41I_binop_rm_int_y<0x3D, "vpmaxsd",
-                                        int_x86_avx2_pmaxs_d>, VEX_4V;
+                                        int_x86_avx2_pmaxs_d>, VEX_4V, VEX_L;
   defm VPMAXUD   : SS41I_binop_rm_int_y<0x3F, "vpmaxud",
-                                        int_x86_avx2_pmaxu_d>, VEX_4V;
+                                        int_x86_avx2_pmaxu_d>, VEX_4V, VEX_L;
   defm VPMAXUW   : SS41I_binop_rm_int_y<0x3E, "vpmaxuw",
-                                        int_x86_avx2_pmaxu_w>, VEX_4V;
+                                        int_x86_avx2_pmaxu_w>, VEX_4V, VEX_L;
   defm VPMULDQ   : SS41I_binop_rm_int_y<0x28, "vpmuldq",
-                                        int_x86_avx2_pmul_dq>, VEX_4V;
+                                        int_x86_avx2_pmul_dq>, VEX_4V, VEX_L;
 }
 
 let Constraints = "$src1 = $dst" in {
@@ -6445,9 +6666,9 @@ let Predicates = [HasAVX] in {
 }
 let Predicates = [HasAVX2] in {
   defm VPMULLDY  : SS48I_binop_rm<0x40, "vpmulld", mul, v8i32, VR256,
-                                  memopv4i64, i256mem, 0>, VEX_4V;
+                                  memopv4i64, i256mem, 0>, VEX_4V, VEX_L;
   defm VPCMPEQQY : SS48I_binop_rm<0x29, "vpcmpeqq", X86pcmpeq, v4i64, VR256,
-                                  memopv4i64, i256mem, 0>, VEX_4V;
+                                  memopv4i64, i256mem, 0>, VEX_4V, VEX_L;
 }
 
 let Constraints = "$src1 = $dst" in {
@@ -6490,13 +6711,15 @@ let Predicates = [HasAVX] in {
     defm VBLENDPS : SS41I_binop_rmi_int<0x0C, "vblendps", int_x86_sse41_blendps,
                                         VR128, memopv4f32, f128mem, 0>, VEX_4V;
     defm VBLENDPSY : SS41I_binop_rmi_int<0x0C, "vblendps",
-              int_x86_avx_blend_ps_256, VR256, memopv8f32, f256mem, 0>, VEX_4V;
+                                    int_x86_avx_blend_ps_256, VR256, memopv8f32,
+                                    f256mem, 0>, VEX_4V, VEX_L;
     }
     let ExeDomain = SSEPackedDouble in {
     defm VBLENDPD : SS41I_binop_rmi_int<0x0D, "vblendpd", int_x86_sse41_blendpd,
                                         VR128, memopv2f64, f128mem, 0>, VEX_4V;
     defm VBLENDPDY : SS41I_binop_rmi_int<0x0D, "vblendpd",
-              int_x86_avx_blend_pd_256, VR256, memopv4f64, f256mem, 0>, VEX_4V;
+                                     int_x86_avx_blend_pd_256,VR256, memopv4f64,
+                                     f256mem, 0>, VEX_4V, VEX_L;
     }
   defm VPBLENDW : SS41I_binop_rmi_int<0x0E, "vpblendw", int_x86_sse41_pblendw,
                                       VR128, memopv2i64, i128mem, 0>, VEX_4V;
@@ -6511,15 +6734,15 @@ let Predicates = [HasAVX] in {
                                    VR128, memopv2f64, f128mem, 0>, VEX_4V;
   let ExeDomain = SSEPackedSingle in
   defm VDPPSY : SS41I_binop_rmi_int<0x40, "vdpps", int_x86_avx_dp_ps_256,
-                                   VR256, memopv8f32, i256mem, 0>, VEX_4V;
+                                  VR256, memopv8f32, i256mem, 0>, VEX_4V, VEX_L;
 }
 
 let Predicates = [HasAVX2] in {
   let isCommutable = 0 in {
   defm VPBLENDWY : SS41I_binop_rmi_int<0x0E, "vpblendw", int_x86_avx2_pblendw,
-                                       VR256, memopv4i64, i256mem, 0>, VEX_4V;
+                                  VR256, memopv4i64, i256mem, 0>, VEX_4V, VEX_L;
   defm VMPSADBWY : SS41I_binop_rmi_int<0x42, "vmpsadbw", int_x86_avx2_mpsadbw,
-                                       VR256, memopv4i64, i256mem, 0>, VEX_4V;
+                                  VR256, memopv4i64, i256mem, 0>, VEX_4V, VEX_L;
   }
 }
 
@@ -6570,13 +6793,13 @@ let ExeDomain = SSEPackedDouble in {
 defm VBLENDVPD  : SS41I_quaternary_int_avx<0x4B, "vblendvpd", VR128, f128mem,
                                            memopv2f64, int_x86_sse41_blendvpd>;
 defm VBLENDVPDY : SS41I_quaternary_int_avx<0x4B, "vblendvpd", VR256, f256mem,
-                                         memopv4f64, int_x86_avx_blendv_pd_256>;
+                                  memopv4f64, int_x86_avx_blendv_pd_256>, VEX_L;
 } // ExeDomain = SSEPackedDouble
 let ExeDomain = SSEPackedSingle in {
 defm VBLENDVPS  : SS41I_quaternary_int_avx<0x4A, "vblendvps", VR128, f128mem,
                                            memopv4f32, int_x86_sse41_blendvps>;
 defm VBLENDVPSY : SS41I_quaternary_int_avx<0x4A, "vblendvps", VR256, f256mem,
-                                         memopv8f32, int_x86_avx_blendv_ps_256>;
+                                  memopv8f32, int_x86_avx_blendv_ps_256>, VEX_L;
 } // ExeDomain = SSEPackedSingle
 defm VPBLENDVB  : SS41I_quaternary_int_avx<0x4C, "vpblendvb", VR128, i128mem,
                                            memopv2i64, int_x86_sse41_pblendvb>;
@@ -6584,7 +6807,7 @@ defm VPBLENDVB  : SS41I_quaternary_int_avx<0x4C, "vpblendvb", VR128, i128mem,
 
 let Predicates = [HasAVX2] in {
 defm VPBLENDVBY : SS41I_quaternary_int_avx<0x4C, "vpblendvb", VR256, i256mem,
-                                           memopv4i64, int_x86_avx2_pblendvb>;
+                                      memopv4i64, int_x86_avx2_pblendvb>, VEX_L;
 }
 
 let Predicates = [HasAVX] in {
@@ -6687,7 +6910,7 @@ def : InstAlias<"pblendvb\t{%xmm0, $src2, $dst|$dst, $src2, %xmm0}",
 def : InstAlias<"pblendvb\t{%xmm0, $src2, $dst|$dst, $src2, %xmm0}",
                 (PBLENDVBrm0 VR128:$dst, i128mem:$src2)>;
 
-let Predicates = [HasSSE41] in {
+let Predicates = [UseSSE41] in {
   def : Pat<(v16i8 (vselect (v16i8 XMM0), (v16i8 VR128:$src1),
                             (v16i8 VR128:$src2))),
             (PBLENDVBrr0 VR128:$src2, VR128:$src1)>;
@@ -6725,7 +6948,7 @@ let Predicates = [HasAVX2] in
 def VMOVNTDQAYrm : SS48I<0x2A, MRMSrcMem, (outs VR256:$dst), (ins i256mem:$src),
                          "vmovntdqa\t{$src, $dst|$dst, $src}",
                          [(set VR256:$dst, (int_x86_avx2_movntdqa addr:$src))]>,
-                         OpSize, VEX;
+                         OpSize, VEX, VEX_L;
 def MOVNTDQArm : SS48I<0x2A, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src),
                        "movntdqa\t{$src, $dst|$dst, $src}",
                        [(set VR128:$dst, (int_x86_sse41_movntdqa addr:$src))]>,
@@ -6761,7 +6984,7 @@ let Predicates = [HasAVX] in
 
 let Predicates = [HasAVX2] in
   defm VPCMPGTQY : SS42I_binop_rm<0x37, "vpcmpgtq", X86pcmpgt, v4i64, VR256,
-                                  memopv4i64, i256mem, 0>, VEX_4V;
+                                  memopv4i64, i256mem, 0>, VEX_4V, VEX_L;
 
 let Constraints = "$src1 = $dst" in
   defm PCMPGTQ : SS42I_binop_rm<0x37, "pcmpgtq", X86pcmpgt, v2i64, VR128,
@@ -6779,34 +7002,31 @@ multiclass pseudo_pcmpistrm<string asm> {
                                                   imm:$src3))]>;
   def MEM : PseudoI<(outs VR128:$dst),
                     (ins VR128:$src1, i128mem:$src2, i8imm:$src3),
-    [(set VR128:$dst, (int_x86_sse42_pcmpistrm128
-                       VR128:$src1, (load addr:$src2), imm:$src3))]>;
+    [(set VR128:$dst, (int_x86_sse42_pcmpistrm128 VR128:$src1,
+                       (bc_v16i8 (memopv2i64 addr:$src2)), imm:$src3))]>;
 }
 
 let Defs = [EFLAGS], usesCustomInserter = 1 in {
-  let AddedComplexity = 1 in
-    defm VPCMPISTRM128 : pseudo_pcmpistrm<"#VPCMPISTRM128">, Requires<[HasAVX]>;
-  defm PCMPISTRM128 : pseudo_pcmpistrm<"#PCMPISTRM128">, Requires<[HasSSE42]>;
+  defm VPCMPISTRM128 : pseudo_pcmpistrm<"#VPCMPISTRM128">, Requires<[HasAVX]>;
+  defm PCMPISTRM128 : pseudo_pcmpistrm<"#PCMPISTRM128">, Requires<[UseSSE42]>;
 }
 
-let Defs = [XMM0, EFLAGS], neverHasSideEffects = 1, Predicates = [HasAVX] in {
-  def VPCMPISTRM128rr : SS42AI<0x62, MRMSrcReg, (outs),
-      (ins VR128:$src1, VR128:$src2, i8imm:$src3),
-      "vpcmpistrm\t{$src3, $src2, $src1|$src1, $src2, $src3}", []>, OpSize, VEX;
+multiclass pcmpistrm_SS42AI<string asm> {
+  def rr : SS42AI<0x62, MRMSrcReg, (outs),
+    (ins VR128:$src1, VR128:$src2, i8imm:$src3),
+    !strconcat(asm, "\t{$src3, $src2, $src1|$src1, $src2, $src3}"),
+    []>, OpSize;
   let mayLoad = 1 in
-  def VPCMPISTRM128rm : SS42AI<0x62, MRMSrcMem, (outs),
-      (ins VR128:$src1, i128mem:$src2, i8imm:$src3),
-      "vpcmpistrm\t{$src3, $src2, $src1|$src1, $src2, $src3}", []>, OpSize, VEX;
+  def rm :SS42AI<0x62, MRMSrcMem, (outs),
+    (ins VR128:$src1, i128mem:$src2, i8imm:$src3),
+    !strconcat(asm, "\t{$src3, $src2, $src1|$src1, $src2, $src3}"),
+    []>, OpSize;
 }
 
 let Defs = [XMM0, EFLAGS], neverHasSideEffects = 1 in {
-  def PCMPISTRM128rr : SS42AI<0x62, MRMSrcReg, (outs),
-      (ins VR128:$src1, VR128:$src2, i8imm:$src3),
-      "pcmpistrm\t{$src3, $src2, $src1|$src1, $src2, $src3}", []>, OpSize;
-  let mayLoad = 1 in
-  def PCMPISTRM128rm : SS42AI<0x62, MRMSrcMem, (outs),
-      (ins VR128:$src1, i128mem:$src2, i8imm:$src3),
-      "pcmpistrm\t{$src3, $src2, $src1|$src1, $src2, $src3}", []>, OpSize;
+  let Predicates = [HasAVX] in
+  defm VPCMPISTRM128 : pcmpistrm_SS42AI<"vpcmpistrm">, VEX;
+  defm PCMPISTRM128  : pcmpistrm_SS42AI<"pcmpistrm"> ;
 }
 
 // Packed Compare Explicit Length Strings, Return Mask
@@ -6817,74 +7037,103 @@ multiclass pseudo_pcmpestrm<string asm> {
                        VR128:$src1, EAX, VR128:$src3, EDX, imm:$src5))]>;
   def MEM : PseudoI<(outs VR128:$dst),
                     (ins VR128:$src1, i128mem:$src3, i8imm:$src5),
-    [(set VR128:$dst, (int_x86_sse42_pcmpestrm128
-                       VR128:$src1, EAX, (load addr:$src3), EDX, imm:$src5))]>;
+    [(set VR128:$dst, (int_x86_sse42_pcmpestrm128 VR128:$src1, EAX,
+                       (bc_v16i8 (memopv2i64 addr:$src3)), EDX, imm:$src5))]>;
 }
 
 let Defs = [EFLAGS], Uses = [EAX, EDX], usesCustomInserter = 1 in {
-  let AddedComplexity = 1 in
-    defm VPCMPESTRM128 : pseudo_pcmpestrm<"#VPCMPESTRM128">, Requires<[HasAVX]>;
-  defm PCMPESTRM128 : pseudo_pcmpestrm<"#PCMPESTRM128">, Requires<[HasSSE42]>;
+  defm VPCMPESTRM128 : pseudo_pcmpestrm<"#VPCMPESTRM128">, Requires<[HasAVX]>;
+  defm PCMPESTRM128 : pseudo_pcmpestrm<"#PCMPESTRM128">, Requires<[UseSSE42]>;
 }
 
-let Predicates = [HasAVX],
-    Defs = [XMM0, EFLAGS], Uses = [EAX, EDX], neverHasSideEffects = 1 in {
-  def VPCMPESTRM128rr : SS42AI<0x60, MRMSrcReg, (outs),
-      (ins VR128:$src1, VR128:$src3, i8imm:$src5),
-      "vpcmpestrm\t{$src5, $src3, $src1|$src1, $src3, $src5}", []>, OpSize, VEX;
+multiclass SS42AI_pcmpestrm<string asm> {
+  def rr : SS42AI<0x60, MRMSrcReg, (outs),
+    (ins VR128:$src1, VR128:$src3, i8imm:$src5),
+    !strconcat(asm, "\t{$src5, $src3, $src1|$src1, $src3, $src5}"),
+    []>, OpSize;
   let mayLoad = 1 in
-  def VPCMPESTRM128rm : SS42AI<0x60, MRMSrcMem, (outs),
-      (ins VR128:$src1, i128mem:$src3, i8imm:$src5),
-      "vpcmpestrm\t{$src5, $src3, $src1|$src1, $src3, $src5}", []>, OpSize, VEX;
+  def rm : SS42AI<0x60, MRMSrcMem, (outs),
+    (ins VR128:$src1, i128mem:$src3, i8imm:$src5),
+    !strconcat(asm, "\t{$src5, $src3, $src1|$src1, $src3, $src5}"),
+    []>, OpSize;
 }
 
 let Defs = [XMM0, EFLAGS], Uses = [EAX, EDX], neverHasSideEffects = 1 in {
-  def PCMPESTRM128rr : SS42AI<0x60, MRMSrcReg, (outs),
-      (ins VR128:$src1, VR128:$src3, i8imm:$src5),
-      "pcmpestrm\t{$src5, $src3, $src1|$src1, $src3, $src5}", []>, OpSize;
-  let mayLoad = 1 in
-  def PCMPESTRM128rm : SS42AI<0x60, MRMSrcMem, (outs),
-      (ins VR128:$src1, i128mem:$src3, i8imm:$src5),
-      "pcmpestrm\t{$src5, $src3, $src1|$src1, $src3, $src5}", []>, OpSize;
+  let Predicates = [HasAVX] in
+  defm VPCMPESTRM128 : SS42AI_pcmpestrm<"vpcmpestrm">, VEX;
+  defm PCMPESTRM128 :  SS42AI_pcmpestrm<"pcmpestrm">;
 }
 
 // Packed Compare Implicit Length Strings, Return Index
-let Defs = [ECX, EFLAGS], neverHasSideEffects = 1 in {
-  multiclass SS42AI_pcmpistri<string asm> {
-    def rr : SS42AI<0x63, MRMSrcReg, (outs),
-      (ins VR128:$src1, VR128:$src2, i8imm:$src3),
-      !strconcat(asm, "\t{$src3, $src2, $src1|$src1, $src2, $src3}"),
-      []>, OpSize;
-    let mayLoad = 1 in
-    def rm : SS42AI<0x63, MRMSrcMem, (outs),
-      (ins VR128:$src1, i128mem:$src2, i8imm:$src3),
-      !strconcat(asm, "\t{$src3, $src2, $src1|$src1, $src2, $src3}"),
-      []>, OpSize;
-  }
+multiclass pseudo_pcmpistri<string asm> {
+  def REG : PseudoI<(outs GR32:$dst),
+                    (ins VR128:$src1, VR128:$src2, i8imm:$src3),
+    [(set GR32:$dst, EFLAGS,
+      (X86pcmpistri VR128:$src1, VR128:$src2, imm:$src3))]>;
+  def MEM : PseudoI<(outs GR32:$dst),
+                    (ins VR128:$src1, i128mem:$src2, i8imm:$src3),
+    [(set GR32:$dst, EFLAGS, (X86pcmpistri VR128:$src1,
+                              (bc_v16i8 (memopv2i64 addr:$src2)), imm:$src3))]>;
 }
 
-let Predicates = [HasAVX] in
-defm VPCMPISTRI  : SS42AI_pcmpistri<"vpcmpistri">, VEX;
-defm PCMPISTRI   : SS42AI_pcmpistri<"pcmpistri">;
+let Defs = [EFLAGS], usesCustomInserter = 1 in {
+  defm VPCMPISTRI : pseudo_pcmpistri<"#VPCMPISTRI">, Requires<[HasAVX]>;
+  defm PCMPISTRI  : pseudo_pcmpistri<"#PCMPISTRI">, Requires<[UseSSE42]>;
+}
+
+multiclass SS42AI_pcmpistri<string asm> {
+  def rr : SS42AI<0x63, MRMSrcReg, (outs),
+    (ins VR128:$src1, VR128:$src2, i8imm:$src3),
+    !strconcat(asm, "\t{$src3, $src2, $src1|$src1, $src2, $src3}"),
+    []>, OpSize;
+  let mayLoad = 1 in
+  def rm : SS42AI<0x63, MRMSrcMem, (outs),
+    (ins VR128:$src1, i128mem:$src2, i8imm:$src3),
+    !strconcat(asm, "\t{$src3, $src2, $src1|$src1, $src2, $src3}"),
+    []>, OpSize;
+}
+
+let Defs = [ECX, EFLAGS], neverHasSideEffects = 1 in {
+  let Predicates = [HasAVX] in
+  defm VPCMPISTRI : SS42AI_pcmpistri<"vpcmpistri">, VEX;
+  defm PCMPISTRI  : SS42AI_pcmpistri<"pcmpistri">;
+}
 
 // Packed Compare Explicit Length Strings, Return Index
-let Defs = [ECX, EFLAGS], Uses = [EAX, EDX], neverHasSideEffects = 1 in {
-  multiclass SS42AI_pcmpestri<string asm> {
-    def rr : SS42AI<0x61, MRMSrcReg, (outs),
-      (ins VR128:$src1, VR128:$src3, i8imm:$src5),
-      !strconcat(asm, "\t{$src5, $src3, $src1|$src1, $src3, $src5}"),
-      []>, OpSize;
-    let mayLoad = 1 in
-    def rm : SS42AI<0x61, MRMSrcMem, (outs),
-      (ins VR128:$src1, i128mem:$src3, i8imm:$src5),
-      !strconcat(asm, "\t{$src5, $src3, $src1|$src1, $src3, $src5}"),
-      []>, OpSize;
-  }
+multiclass pseudo_pcmpestri<string asm> {
+  def REG : PseudoI<(outs GR32:$dst),
+                    (ins VR128:$src1, VR128:$src3, i8imm:$src5),
+    [(set GR32:$dst, EFLAGS,
+      (X86pcmpestri VR128:$src1, EAX, VR128:$src3, EDX, imm:$src5))]>;
+  def MEM : PseudoI<(outs GR32:$dst),
+                    (ins VR128:$src1, i128mem:$src3, i8imm:$src5),
+    [(set GR32:$dst, EFLAGS,
+      (X86pcmpestri VR128:$src1, EAX, (bc_v16i8 (memopv2i64 addr:$src3)), EDX,
+       imm:$src5))]>;
 }
 
-let Predicates = [HasAVX] in
-defm VPCMPESTRI  : SS42AI_pcmpestri<"vpcmpestri">, VEX;
-defm PCMPESTRI   : SS42AI_pcmpestri<"pcmpestri">;
+let Defs = [EFLAGS], Uses = [EAX, EDX], usesCustomInserter = 1 in {
+  defm VPCMPESTRI : pseudo_pcmpestri<"#VPCMPESTRI">, Requires<[HasAVX]>;
+  defm PCMPESTRI  : pseudo_pcmpestri<"#PCMPESTRI">, Requires<[UseSSE42]>;
+}
+
+multiclass SS42AI_pcmpestri<string asm> {
+  def rr : SS42AI<0x61, MRMSrcReg, (outs),
+    (ins VR128:$src1, VR128:$src3, i8imm:$src5),
+    !strconcat(asm, "\t{$src5, $src3, $src1|$src1, $src3, $src5}"),
+    []>, OpSize;
+  let mayLoad = 1 in
+  def rm : SS42AI<0x61, MRMSrcMem, (outs),
+    (ins VR128:$src1, i128mem:$src3, i8imm:$src5),
+    !strconcat(asm, "\t{$src5, $src3, $src1|$src1, $src3, $src5}"),
+    []>, OpSize;
+}
+
+let Defs = [ECX, EFLAGS], Uses = [EAX, EDX], neverHasSideEffects = 1 in {
+  let Predicates = [HasAVX] in
+  defm VPCMPESTRI : SS42AI_pcmpestri<"vpcmpestri">, VEX;
+  defm PCMPESTRI  : SS42AI_pcmpestri<"pcmpestri">;
+}
 
 //===----------------------------------------------------------------------===//
 // SSE4.2 - CRC Instructions
@@ -7175,27 +7424,27 @@ let ExeDomain = SSEPackedSingle in {
   def VBROADCASTSSrm  : avx_broadcast<0x18, "vbroadcastss", VR128, f32mem,
                                       int_x86_avx_vbroadcast_ss>;
   def VBROADCASTSSYrm : avx_broadcast<0x18, "vbroadcastss", VR256, f32mem,
-                                      int_x86_avx_vbroadcast_ss_256>;
+                                      int_x86_avx_vbroadcast_ss_256>, VEX_L;
 }
 let ExeDomain = SSEPackedDouble in
 def VBROADCASTSDYrm  : avx_broadcast<0x19, "vbroadcastsd", VR256, f64mem,
-                                    int_x86_avx_vbroadcast_sd_256>;
+                                    int_x86_avx_vbroadcast_sd_256>, VEX_L;
 def VBROADCASTF128 : avx_broadcast<0x1A, "vbroadcastf128", VR256, f128mem,
-                                   int_x86_avx_vbroadcastf128_pd_256>;
+                                   int_x86_avx_vbroadcastf128_pd_256>, VEX_L;
 
 let ExeDomain = SSEPackedSingle in {
   def VBROADCASTSSrr  : avx2_broadcast_reg<0x18, "vbroadcastss", VR128,
                                            int_x86_avx2_vbroadcast_ss_ps>;
   def VBROADCASTSSYrr : avx2_broadcast_reg<0x18, "vbroadcastss", VR256,
-                                           int_x86_avx2_vbroadcast_ss_ps_256>;
+                                      int_x86_avx2_vbroadcast_ss_ps_256>, VEX_L;
 }
 let ExeDomain = SSEPackedDouble in
 def VBROADCASTSDYrr  : avx2_broadcast_reg<0x19, "vbroadcastsd", VR256,
-                                          int_x86_avx2_vbroadcast_sd_pd_256>;
+                                      int_x86_avx2_vbroadcast_sd_pd_256>, VEX_L;
 
 let Predicates = [HasAVX2] in
 def VBROADCASTI128 : avx_broadcast<0x5A, "vbroadcasti128", VR256, i128mem,
-                                   int_x86_avx2_vbroadcasti128>;
+                                   int_x86_avx2_vbroadcasti128>, VEX_L;
 
 let Predicates = [HasAVX] in
 def : Pat<(int_x86_avx_vbroadcastf128_ps_256 addr:$src),
@@ -7209,50 +7458,69 @@ let neverHasSideEffects = 1, ExeDomain = SSEPackedSingle in {
 def VINSERTF128rr : AVXAIi8<0x18, MRMSrcReg, (outs VR256:$dst),
           (ins VR256:$src1, VR128:$src2, i8imm:$src3),
           "vinsertf128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
-          []>, VEX_4V;
+          []>, VEX_4V, VEX_L;
 let mayLoad = 1 in
 def VINSERTF128rm : AVXAIi8<0x18, MRMSrcMem, (outs VR256:$dst),
           (ins VR256:$src1, f128mem:$src2, i8imm:$src3),
           "vinsertf128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
-          []>, VEX_4V;
+          []>, VEX_4V, VEX_L;
 }
 
 let Predicates = [HasAVX] in {
 def : Pat<(vinsertf128_insert:$ins (v8f32 VR256:$src1), (v4f32 VR128:$src2),
-                                   (i32 imm)),
+                                   (iPTR imm)),
           (VINSERTF128rr VR256:$src1, VR128:$src2,
                          (INSERT_get_vinsertf128_imm VR256:$ins))>;
 def : Pat<(vinsertf128_insert:$ins (v4f64 VR256:$src1), (v2f64 VR128:$src2),
-                                   (i32 imm)),
+                                   (iPTR imm)),
           (VINSERTF128rr VR256:$src1, VR128:$src2,
                          (INSERT_get_vinsertf128_imm VR256:$ins))>;
+
+def : Pat<(vinsertf128_insert:$ins (v8f32 VR256:$src1), (memopv4f32 addr:$src2),
+                                   (iPTR imm)),
+          (VINSERTF128rm VR256:$src1, addr:$src2,
+                         (INSERT_get_vinsertf128_imm VR256:$ins))>;
+def : Pat<(vinsertf128_insert:$ins (v4f64 VR256:$src1), (memopv2f64 addr:$src2),
+                                   (iPTR imm)),
+          (VINSERTF128rm VR256:$src1, addr:$src2,
+                         (INSERT_get_vinsertf128_imm VR256:$ins))>;
+}
+
+let Predicates = [HasAVX1Only] in {
 def : Pat<(vinsertf128_insert:$ins (v4i64 VR256:$src1), (v2i64 VR128:$src2),
-                                   (i32 imm)),
+                                   (iPTR imm)),
           (VINSERTF128rr VR256:$src1, VR128:$src2,
                          (INSERT_get_vinsertf128_imm VR256:$ins))>;
 def : Pat<(vinsertf128_insert:$ins (v8i32 VR256:$src1), (v4i32 VR128:$src2),
-                                   (i32 imm)),
+                                   (iPTR imm)),
           (VINSERTF128rr VR256:$src1, VR128:$src2,
                          (INSERT_get_vinsertf128_imm VR256:$ins))>;
 def : Pat<(vinsertf128_insert:$ins (v32i8 VR256:$src1), (v16i8 VR128:$src2),
-                                   (i32 imm)),
+                                   (iPTR imm)),
           (VINSERTF128rr VR256:$src1, VR128:$src2,
                          (INSERT_get_vinsertf128_imm VR256:$ins))>;
 def : Pat<(vinsertf128_insert:$ins (v16i16 VR256:$src1), (v8i16 VR128:$src2),
-                                   (i32 imm)),
+                                   (iPTR imm)),
           (VINSERTF128rr VR256:$src1, VR128:$src2,
                          (INSERT_get_vinsertf128_imm VR256:$ins))>;
 
-def : Pat<(vinsertf128_insert:$ins (v8f32 VR256:$src1), (loadv4f32 addr:$src2),
-                                   (i32 imm)),
+def : Pat<(vinsertf128_insert:$ins (v4i64 VR256:$src1), (memopv2i64 addr:$src2),
+                                   (iPTR imm)),
           (VINSERTF128rm VR256:$src1, addr:$src2,
                          (INSERT_get_vinsertf128_imm VR256:$ins))>;
-def : Pat<(vinsertf128_insert:$ins (v4f64 VR256:$src1), (loadv2f64 addr:$src2),
-                                   (i32 imm)),
+def : Pat<(vinsertf128_insert:$ins (v8i32 VR256:$src1),
+                                   (bc_v4i32 (memopv2i64 addr:$src2)),
+                                   (iPTR imm)),
           (VINSERTF128rm VR256:$src1, addr:$src2,
                          (INSERT_get_vinsertf128_imm VR256:$ins))>;
-def : Pat<(vinsertf128_insert:$ins (v4i64 VR256:$src1), (loadv2i64 addr:$src2),
-                                   (i32 imm)),
+def : Pat<(vinsertf128_insert:$ins (v32i8 VR256:$src1),
+                                   (bc_v16i8 (memopv2i64 addr:$src2)),
+                                   (iPTR imm)),
+          (VINSERTF128rm VR256:$src1, addr:$src2,
+                         (INSERT_get_vinsertf128_imm VR256:$ins))>;
+def : Pat<(vinsertf128_insert:$ins (v16i16 VR256:$src1),
+                                   (bc_v8i16 (memopv2i64 addr:$src2)),
+                                   (iPTR imm)),
           (VINSERTF128rm VR256:$src1, addr:$src2,
                          (INSERT_get_vinsertf128_imm VR256:$ins))>;
 }
@@ -7264,64 +7532,69 @@ let neverHasSideEffects = 1, ExeDomain = SSEPackedSingle in {
 def VEXTRACTF128rr : AVXAIi8<0x19, MRMDestReg, (outs VR128:$dst),
           (ins VR256:$src1, i8imm:$src2),
           "vextractf128\t{$src2, $src1, $dst|$dst, $src1, $src2}",
-          []>, VEX;
+          []>, VEX, VEX_L;
 let mayStore = 1 in
 def VEXTRACTF128mr : AVXAIi8<0x19, MRMDestMem, (outs),
           (ins f128mem:$dst, VR256:$src1, i8imm:$src2),
           "vextractf128\t{$src2, $src1, $dst|$dst, $src1, $src2}",
-          []>, VEX;
-}
-
-// Extract and store.
-let Predicates = [HasAVX] in {
-  def : Pat<(alignedstore (int_x86_avx_vextractf128_ps_256 VR256:$src1, imm:$src2), addr:$dst),
-          (VEXTRACTF128mr addr:$dst, VR256:$src1, imm:$src2)>;
-  def : Pat<(alignedstore (int_x86_avx_vextractf128_pd_256 VR256:$src1, imm:$src2), addr:$dst),
-          (VEXTRACTF128mr addr:$dst, VR256:$src1, imm:$src2)>;
-  def : Pat<(alignedstore (int_x86_avx_vextractf128_si_256 VR256:$src1, imm:$src2), addr:$dst),
-          (VEXTRACTF128mr addr:$dst, VR256:$src1, imm:$src2)>;
-
-  def : Pat<(int_x86_sse_storeu_ps addr:$dst, (int_x86_avx_vextractf128_ps_256 VR256:$src1, imm:$src2)),
-          (VEXTRACTF128mr addr:$dst, VR256:$src1, imm:$src2)>;
-  def : Pat<(int_x86_sse2_storeu_pd addr:$dst, (int_x86_avx_vextractf128_pd_256 VR256:$src1, imm:$src2)),
-          (VEXTRACTF128mr addr:$dst, VR256:$src1, imm:$src2)>;
-  def : Pat<(int_x86_sse2_storeu_dq addr:$dst, (bc_v16i8 (int_x86_avx_vextractf128_si_256 VR256:$src1, imm:$src2))),
-          (VEXTRACTF128mr addr:$dst, VR256:$src1, imm:$src2)>;
+          []>, VEX, VEX_L;
 }
 
 // AVX1 patterns
 let Predicates = [HasAVX] in {
-def : Pat<(int_x86_avx_vextractf128_pd_256 VR256:$src1, imm:$src2),
-          (VEXTRACTF128rr VR256:$src1, imm:$src2)>;
-def : Pat<(int_x86_avx_vextractf128_ps_256 VR256:$src1, imm:$src2),
-          (VEXTRACTF128rr VR256:$src1, imm:$src2)>;
-def : Pat<(int_x86_avx_vextractf128_si_256 VR256:$src1, imm:$src2),
-          (VEXTRACTF128rr VR256:$src1, imm:$src2)>;
-
-def : Pat<(vextractf128_extract:$ext VR256:$src1, (i32 imm)),
+def : Pat<(vextractf128_extract:$ext VR256:$src1, (iPTR imm)),
           (v4f32 (VEXTRACTF128rr
                     (v8f32 VR256:$src1),
                     (EXTRACT_get_vextractf128_imm VR128:$ext)))>;
-def : Pat<(vextractf128_extract:$ext VR256:$src1, (i32 imm)),
+def : Pat<(vextractf128_extract:$ext VR256:$src1, (iPTR imm)),
           (v2f64 (VEXTRACTF128rr
                     (v4f64 VR256:$src1),
                     (EXTRACT_get_vextractf128_imm VR128:$ext)))>;
-def : Pat<(vextractf128_extract:$ext VR256:$src1, (i32 imm)),
+
+def : Pat<(alignedstore (v4f32 (vextractf128_extract:$ext (v8f32 VR256:$src1),
+                                (iPTR imm))), addr:$dst),
+          (VEXTRACTF128mr addr:$dst, VR256:$src1,
+           (EXTRACT_get_vextractf128_imm VR128:$ext))>;
+def : Pat<(alignedstore (v2f64 (vextractf128_extract:$ext (v4f64 VR256:$src1),
+                                (iPTR imm))), addr:$dst),
+          (VEXTRACTF128mr addr:$dst, VR256:$src1,
+           (EXTRACT_get_vextractf128_imm VR128:$ext))>;
+}
+
+let Predicates = [HasAVX1Only] in {
+def : Pat<(vextractf128_extract:$ext VR256:$src1, (iPTR imm)),
           (v2i64 (VEXTRACTF128rr
-                    (v4i64 VR256:$src1),
-                    (EXTRACT_get_vextractf128_imm VR128:$ext)))>;
-def : Pat<(vextractf128_extract:$ext VR256:$src1, (i32 imm)),
+                  (v4i64 VR256:$src1),
+                  (EXTRACT_get_vextractf128_imm VR128:$ext)))>;
+def : Pat<(vextractf128_extract:$ext VR256:$src1, (iPTR imm)),
           (v4i32 (VEXTRACTF128rr
-                    (v8i32 VR256:$src1),
-                    (EXTRACT_get_vextractf128_imm VR128:$ext)))>;
-def : Pat<(vextractf128_extract:$ext VR256:$src1, (i32 imm)),
+                  (v8i32 VR256:$src1),
+                  (EXTRACT_get_vextractf128_imm VR128:$ext)))>;
+def : Pat<(vextractf128_extract:$ext VR256:$src1, (iPTR imm)),
           (v8i16 (VEXTRACTF128rr
-                    (v16i16 VR256:$src1),
-                    (EXTRACT_get_vextractf128_imm VR128:$ext)))>;
-def : Pat<(vextractf128_extract:$ext VR256:$src1, (i32 imm)),
+                  (v16i16 VR256:$src1),
+                  (EXTRACT_get_vextractf128_imm VR128:$ext)))>;
+def : Pat<(vextractf128_extract:$ext VR256:$src1, (iPTR imm)),
           (v16i8 (VEXTRACTF128rr
-                    (v32i8 VR256:$src1),
-                    (EXTRACT_get_vextractf128_imm VR128:$ext)))>;
+                  (v32i8 VR256:$src1),
+                  (EXTRACT_get_vextractf128_imm VR128:$ext)))>;
+
+def : Pat<(alignedstore (v2i64 (vextractf128_extract:$ext (v4i64 VR256:$src1),
+                                (iPTR imm))), addr:$dst),
+          (VEXTRACTF128mr addr:$dst, VR256:$src1,
+           (EXTRACT_get_vextractf128_imm VR128:$ext))>;
+def : Pat<(alignedstore (v4i32 (vextractf128_extract:$ext (v8i32 VR256:$src1),
+                                (iPTR imm))), addr:$dst),
+          (VEXTRACTF128mr addr:$dst, VR256:$src1,
+           (EXTRACT_get_vextractf128_imm VR128:$ext))>;
+def : Pat<(alignedstore (v8i16 (vextractf128_extract:$ext (v16i16 VR256:$src1),
+                                (iPTR imm))), addr:$dst),
+          (VEXTRACTF128mr addr:$dst, VR256:$src1,
+           (EXTRACT_get_vextractf128_imm VR128:$ext))>;
+def : Pat<(alignedstore (v16i8 (vextractf128_extract:$ext (v32i8 VR256:$src1),
+                                (iPTR imm))), addr:$dst),
+          (VEXTRACTF128mr addr:$dst, VR256:$src1,
+           (EXTRACT_get_vextractf128_imm VR128:$ext))>;
 }
 
 //===----------------------------------------------------------------------===//
@@ -7339,7 +7612,7 @@ multiclass avx_movmask_rm<bits<8> opc_rm, bits<8> opc_mr, string OpcodeStr,
              (ins VR256:$src1, f256mem:$src2),
              !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
              [(set VR256:$dst, (IntLd256 addr:$src2, VR256:$src1))]>,
-             VEX_4V;
+             VEX_4V, VEX_L;
   def mr  : AVX8I<opc_mr, MRMDestMem, (outs),
              (ins f128mem:$dst, VR128:$src1, VR128:$src2),
              !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
@@ -7347,7 +7620,7 @@ multiclass avx_movmask_rm<bits<8> opc_rm, bits<8> opc_mr, string OpcodeStr,
   def Ymr : AVX8I<opc_mr, MRMDestMem, (outs),
              (ins f256mem:$dst, VR256:$src1, VR256:$src2),
              !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
-             [(IntSt256 addr:$dst, VR256:$src1, VR256:$src2)]>, VEX_4V;
+             [(IntSt256 addr:$dst, VR256:$src1, VR256:$src2)]>, VEX_4V, VEX_L;
 }
 
 let ExeDomain = SSEPackedSingle in
@@ -7395,13 +7668,13 @@ let ExeDomain = SSEPackedSingle in {
   defm VPERMILPS  : avx_permil<0x0C, 0x04, "vpermilps", VR128, f128mem, i128mem,
                                memopv2i64, int_x86_avx_vpermilvar_ps, v4f32>;
   defm VPERMILPSY : avx_permil<0x0C, 0x04, "vpermilps", VR256, f256mem, i256mem,
-                              memopv4i64, int_x86_avx_vpermilvar_ps_256, v8f32>;
+                       memopv4i64, int_x86_avx_vpermilvar_ps_256, v8f32>, VEX_L;
 }
 let ExeDomain = SSEPackedDouble in {
   defm VPERMILPD  : avx_permil<0x0D, 0x05, "vpermilpd", VR128, f128mem, i128mem,
                                memopv2i64, int_x86_avx_vpermilvar_pd, v2f64>;
   defm VPERMILPDY : avx_permil<0x0D, 0x05, "vpermilpd", VR256, f256mem, i256mem,
-                              memopv4i64, int_x86_avx_vpermilvar_pd_256, v4f64>;
+                       memopv4i64, int_x86_avx_vpermilvar_pd_256, v4f64>, VEX_L;
 }
 
 let Predicates = [HasAVX] in {
@@ -7429,38 +7702,38 @@ def VPERM2F128rr : AVXAIi8<0x06, MRMSrcReg, (outs VR256:$dst),
           (ins VR256:$src1, VR256:$src2, i8imm:$src3),
           "vperm2f128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
           [(set VR256:$dst, (v8f32 (X86VPerm2x128 VR256:$src1, VR256:$src2,
-                              (i8 imm:$src3))))]>, VEX_4V;
+                              (i8 imm:$src3))))]>, VEX_4V, VEX_L;
 def VPERM2F128rm : AVXAIi8<0x06, MRMSrcMem, (outs VR256:$dst),
           (ins VR256:$src1, f256mem:$src2, i8imm:$src3),
           "vperm2f128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
           [(set VR256:$dst, (X86VPerm2x128 VR256:$src1, (memopv8f32 addr:$src2),
-                             (i8 imm:$src3)))]>, VEX_4V;
+                             (i8 imm:$src3)))]>, VEX_4V, VEX_L;
 }
 
 let Predicates = [HasAVX] in {
+def : Pat<(v4f64 (X86VPerm2x128 VR256:$src1, VR256:$src2, (i8 imm:$imm))),
+          (VPERM2F128rr VR256:$src1, VR256:$src2, imm:$imm)>;
+def : Pat<(v4f64 (X86VPerm2x128 VR256:$src1,
+                  (memopv4f64 addr:$src2), (i8 imm:$imm))),
+          (VPERM2F128rm VR256:$src1, addr:$src2, imm:$imm)>;
+}
+
+let Predicates = [HasAVX1Only] in {
 def : Pat<(v8i32 (X86VPerm2x128 VR256:$src1, VR256:$src2, (i8 imm:$imm))),
           (VPERM2F128rr VR256:$src1, VR256:$src2, imm:$imm)>;
 def : Pat<(v4i64 (X86VPerm2x128 VR256:$src1, VR256:$src2, (i8 imm:$imm))),
           (VPERM2F128rr VR256:$src1, VR256:$src2, imm:$imm)>;
-def : Pat<(v4f64 (X86VPerm2x128 VR256:$src1, VR256:$src2, (i8 imm:$imm))),
-          (VPERM2F128rr VR256:$src1, VR256:$src2, imm:$imm)>;
 def : Pat<(v32i8 (X86VPerm2x128 VR256:$src1, VR256:$src2, (i8 imm:$imm))),
           (VPERM2F128rr VR256:$src1, VR256:$src2, imm:$imm)>;
 def : Pat<(v16i16 (X86VPerm2x128 VR256:$src1, VR256:$src2, (i8 imm:$imm))),
           (VPERM2F128rr VR256:$src1, VR256:$src2, imm:$imm)>;
 
-def : Pat<(v8f32 (X86VPerm2x128 VR256:$src1,
-                  (memopv8f32 addr:$src2), (i8 imm:$imm))),
-          (VPERM2F128rm VR256:$src1, addr:$src2, imm:$imm)>;
 def : Pat<(v8i32 (X86VPerm2x128 VR256:$src1,
                   (bc_v8i32 (memopv4i64 addr:$src2)), (i8 imm:$imm))),
           (VPERM2F128rm VR256:$src1, addr:$src2, imm:$imm)>;
 def : Pat<(v4i64 (X86VPerm2x128 VR256:$src1,
                   (memopv4i64 addr:$src2), (i8 imm:$imm))),
           (VPERM2F128rm VR256:$src1, addr:$src2, imm:$imm)>;
-def : Pat<(v4f64 (X86VPerm2x128 VR256:$src1,
-                  (memopv4f64 addr:$src2), (i8 imm:$imm))),
-          (VPERM2F128rm VR256:$src1, addr:$src2, imm:$imm)>;
 def : Pat<(v32i8 (X86VPerm2x128 VR256:$src1,
                   (bc_v32i8 (memopv4i64 addr:$src2)), (i8 imm:$imm))),
           (VPERM2F128rm VR256:$src1, addr:$src2, imm:$imm)>;
@@ -7511,9 +7784,9 @@ multiclass f16c_ps2ph<RegisterClass RC, X86MemOperand x86memop, Intrinsic Int> {
 
 let Predicates = [HasAVX, HasF16C] in {
   defm VCVTPH2PS  : f16c_ph2ps<VR128, f64mem, int_x86_vcvtph2ps_128>;
-  defm VCVTPH2PSY : f16c_ph2ps<VR256, f128mem, int_x86_vcvtph2ps_256>;
+  defm VCVTPH2PSY : f16c_ph2ps<VR256, f128mem, int_x86_vcvtph2ps_256>, VEX_L;
   defm VCVTPS2PH  : f16c_ps2ph<VR128, f64mem, int_x86_vcvtps2ph_128>;
-  defm VCVTPS2PHY : f16c_ps2ph<VR256, f128mem, int_x86_vcvtps2ph_256>;
+  defm VCVTPS2PHY : f16c_ps2ph<VR256, f128mem, int_x86_vcvtps2ph_256>, VEX_L;
 }
 
 //===----------------------------------------------------------------------===//
@@ -7545,7 +7818,7 @@ let isCommutable = 0 in {
 defm VPBLENDD : AVX2_binop_rmi_int<0x02, "vpblendd", int_x86_avx2_pblendd_128,
                                    VR128, memopv2i64, i128mem>;
 defm VPBLENDDY : AVX2_binop_rmi_int<0x02, "vpblendd", int_x86_avx2_pblendd_256,
-                                    VR256, memopv4i64, i256mem>;
+                                    VR256, memopv4i64, i256mem>, VEX_L;
 }
 
 //===----------------------------------------------------------------------===//
@@ -7564,11 +7837,12 @@ multiclass avx2_broadcast<bits<8> opc, string OpcodeStr,
                     (Int128 (scalar_to_vector (ld_frag addr:$src))))]>, VEX;
   def Yrr : AVX28I<opc, MRMSrcReg, (outs VR256:$dst), (ins VR128:$src),
                    !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
-                   [(set VR256:$dst, (Int256 VR128:$src))]>, VEX;
+                   [(set VR256:$dst, (Int256 VR128:$src))]>, VEX, VEX_L;
   def Yrm : AVX28I<opc, MRMSrcMem, (outs VR256:$dst), (ins x86memop:$src),
                    !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
                    [(set VR256:$dst,
-                    (Int256 (scalar_to_vector (ld_frag addr:$src))))]>, VEX;
+                    (Int256 (scalar_to_vector (ld_frag addr:$src))))]>,
+                   VEX, VEX_L;
 }
 
 defm VPBROADCASTB  : avx2_broadcast<0x78, "vpbroadcastb", i8mem, loadi8,
@@ -7647,19 +7921,22 @@ let Predicates = [HasAVX2] in {
 }
 
 // AVX1 broadcast patterns
-let Predicates = [HasAVX] in {
+let Predicates = [HasAVX1Only] in {
 def : Pat<(v8i32 (X86VBroadcast (loadi32 addr:$src))),
           (VBROADCASTSSYrm addr:$src)>;
 def : Pat<(v4i64 (X86VBroadcast (loadi64 addr:$src))),
           (VBROADCASTSDYrm addr:$src)>;
+def : Pat<(v4i32 (X86VBroadcast (loadi32 addr:$src))),
+          (VBROADCASTSSrm addr:$src)>;
+}
+
+let Predicates = [HasAVX] in {
 def : Pat<(v8f32 (X86VBroadcast (loadf32 addr:$src))),
           (VBROADCASTSSYrm addr:$src)>;
 def : Pat<(v4f64 (X86VBroadcast (loadf64 addr:$src))),
           (VBROADCASTSDYrm addr:$src)>;
 def : Pat<(v4f32 (X86VBroadcast (loadf32 addr:$src))),
           (VBROADCASTSSrm addr:$src)>;
-def : Pat<(v4i32 (X86VBroadcast (loadi32 addr:$src))),
-          (VBROADCASTSSrm addr:$src)>;
 
   // Provide fallback in case the load node that is used in the patterns above
   // is used by additional users, which prevents the pattern selection.
@@ -7700,7 +7977,8 @@ multiclass avx2_perm<bits<8> opc, string OpcodeStr, PatFrag mem_frag,
                    !strconcat(OpcodeStr,
                        "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
                    [(set VR256:$dst,
-                     (OpVT (X86VPermv VR256:$src1, VR256:$src2)))]>, VEX_4V;
+                     (OpVT (X86VPermv VR256:$src1, VR256:$src2)))]>,
+                   VEX_4V, VEX_L;
   def Yrm : AVX28I<opc, MRMSrcMem, (outs VR256:$dst),
                    (ins VR256:$src1, i256mem:$src2),
                    !strconcat(OpcodeStr,
@@ -7708,7 +7986,7 @@ multiclass avx2_perm<bits<8> opc, string OpcodeStr, PatFrag mem_frag,
                    [(set VR256:$dst,
                      (OpVT (X86VPermv VR256:$src1,
                             (bitconvert (mem_frag addr:$src2)))))]>,
-                   VEX_4V;
+                   VEX_4V, VEX_L;
 }
 
 defm VPERMD : avx2_perm<0x36, "vpermd", memopv4i64, v8i32>;
@@ -7722,14 +8000,15 @@ multiclass avx2_perm_imm<bits<8> opc, string OpcodeStr, PatFrag mem_frag,
                      !strconcat(OpcodeStr,
                          "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
                      [(set VR256:$dst,
-                       (OpVT (X86VPermi VR256:$src1, (i8 imm:$src2))))]>, VEX;
+                       (OpVT (X86VPermi VR256:$src1, (i8 imm:$src2))))]>,
+                     VEX, VEX_L;
   def Ymi : AVX2AIi8<opc, MRMSrcMem, (outs VR256:$dst),
                      (ins i256mem:$src1, i8imm:$src2),
                      !strconcat(OpcodeStr,
                          "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
                      [(set VR256:$dst,
                        (OpVT (X86VPermi (mem_frag addr:$src1),
-                              (i8 imm:$src2))))]>, VEX;
+                              (i8 imm:$src2))))]>, VEX, VEX_L;
 }
 
 defm VPERMQ : avx2_perm_imm<0x00, "vpermq", memopv4i64, v4i64>, VEX_W;
@@ -7739,20 +8018,18 @@ defm VPERMPD : avx2_perm_imm<0x01, "vpermpd", memopv4f64, v4f64>, VEX_W;
 //===----------------------------------------------------------------------===//
 // VPERM2I128 - Permute Floating-Point Values in 128-bit chunks
 //
-let AddedComplexity = 1 in {
 def VPERM2I128rr : AVX2AIi8<0x46, MRMSrcReg, (outs VR256:$dst),
           (ins VR256:$src1, VR256:$src2, i8imm:$src3),
           "vperm2i128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
           [(set VR256:$dst, (v4i64 (X86VPerm2x128 VR256:$src1, VR256:$src2,
-                            (i8 imm:$src3))))]>, VEX_4V;
+                            (i8 imm:$src3))))]>, VEX_4V, VEX_L;
 def VPERM2I128rm : AVX2AIi8<0x46, MRMSrcMem, (outs VR256:$dst),
           (ins VR256:$src1, f256mem:$src2, i8imm:$src3),
           "vperm2i128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
           [(set VR256:$dst, (X86VPerm2x128 VR256:$src1, (memopv4i64 addr:$src2),
-                             (i8 imm:$src3)))]>, VEX_4V;
-}
+                             (i8 imm:$src3)))]>, VEX_4V, VEX_L;
 
-let Predicates = [HasAVX2], AddedComplexity = 1 in {
+let Predicates = [HasAVX2] in {
 def : Pat<(v8i32 (X86VPerm2x128 VR256:$src1, VR256:$src2, (i8 imm:$imm))),
           (VPERM2I128rr VR256:$src1, VR256:$src2, imm:$imm)>;
 def : Pat<(v32i8 (X86VPerm2x128 VR256:$src1, VR256:$src2, (i8 imm:$imm))),
@@ -7779,31 +8056,51 @@ let neverHasSideEffects = 1 in {
 def VINSERTI128rr : AVX2AIi8<0x38, MRMSrcReg, (outs VR256:$dst),
           (ins VR256:$src1, VR128:$src2, i8imm:$src3),
           "vinserti128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
-          []>, VEX_4V;
+          []>, VEX_4V, VEX_L;
 let mayLoad = 1 in
 def VINSERTI128rm : AVX2AIi8<0x38, MRMSrcMem, (outs VR256:$dst),
           (ins VR256:$src1, i128mem:$src2, i8imm:$src3),
           "vinserti128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
-          []>, VEX_4V;
+          []>, VEX_4V, VEX_L;
 }
 
-let Predicates = [HasAVX2], AddedComplexity = 1 in {
+let Predicates = [HasAVX2] in {
 def : Pat<(vinsertf128_insert:$ins (v4i64 VR256:$src1), (v2i64 VR128:$src2),
-                                   (i32 imm)),
+                                   (iPTR imm)),
           (VINSERTI128rr VR256:$src1, VR128:$src2,
                          (INSERT_get_vinsertf128_imm VR256:$ins))>;
 def : Pat<(vinsertf128_insert:$ins (v8i32 VR256:$src1), (v4i32 VR128:$src2),
-                                   (i32 imm)),
+                                   (iPTR imm)),
           (VINSERTI128rr VR256:$src1, VR128:$src2,
                          (INSERT_get_vinsertf128_imm VR256:$ins))>;
 def : Pat<(vinsertf128_insert:$ins (v32i8 VR256:$src1), (v16i8 VR128:$src2),
-                                   (i32 imm)),
+                                   (iPTR imm)),
           (VINSERTI128rr VR256:$src1, VR128:$src2,
                          (INSERT_get_vinsertf128_imm VR256:$ins))>;
 def : Pat<(vinsertf128_insert:$ins (v16i16 VR256:$src1), (v8i16 VR128:$src2),
-                                   (i32 imm)),
+                                   (iPTR imm)),
           (VINSERTI128rr VR256:$src1, VR128:$src2,
                          (INSERT_get_vinsertf128_imm VR256:$ins))>;
+
+def : Pat<(vinsertf128_insert:$ins (v4i64 VR256:$src1), (memopv2i64 addr:$src2),
+                                   (iPTR imm)),
+          (VINSERTI128rm VR256:$src1, addr:$src2,
+                         (INSERT_get_vinsertf128_imm VR256:$ins))>;
+def : Pat<(vinsertf128_insert:$ins (v8i32 VR256:$src1),
+                                   (bc_v4i32 (memopv2i64 addr:$src2)),
+                                   (iPTR imm)),
+          (VINSERTI128rm VR256:$src1, addr:$src2,
+                         (INSERT_get_vinsertf128_imm VR256:$ins))>;
+def : Pat<(vinsertf128_insert:$ins (v32i8 VR256:$src1),
+                                   (bc_v16i8 (memopv2i64 addr:$src2)),
+                                   (iPTR imm)),
+          (VINSERTI128rm VR256:$src1, addr:$src2,
+                         (INSERT_get_vinsertf128_imm VR256:$ins))>;
+def : Pat<(vinsertf128_insert:$ins (v16i16 VR256:$src1),
+                                   (bc_v8i16 (memopv2i64 addr:$src2)),
+                                   (iPTR imm)),
+          (VINSERTI128rm VR256:$src1, addr:$src2,
+                         (INSERT_get_vinsertf128_imm VR256:$ins))>;
 }
 
 //===----------------------------------------------------------------------===//
@@ -7814,29 +8111,47 @@ def VEXTRACTI128rr : AVX2AIi8<0x39, MRMDestReg, (outs VR128:$dst),
           "vextracti128\t{$src2, $src1, $dst|$dst, $src1, $src2}",
           [(set VR128:$dst,
             (int_x86_avx2_vextracti128 VR256:$src1, imm:$src2))]>,
-          VEX;
+          VEX, VEX_L;
 let neverHasSideEffects = 1, mayStore = 1 in
 def VEXTRACTI128mr : AVX2AIi8<0x39, MRMDestMem, (outs),
           (ins i128mem:$dst, VR256:$src1, i8imm:$src2),
-          "vextracti128\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>, VEX;
+          "vextracti128\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>,
+          VEX, VEX_L;
 
-let Predicates = [HasAVX2], AddedComplexity = 1 in {
-def : Pat<(vextractf128_extract:$ext VR256:$src1, (i32 imm)),
+let Predicates = [HasAVX2] in {
+def : Pat<(vextractf128_extract:$ext VR256:$src1, (iPTR imm)),
           (v2i64 (VEXTRACTI128rr
                     (v4i64 VR256:$src1),
                     (EXTRACT_get_vextractf128_imm VR128:$ext)))>;
-def : Pat<(vextractf128_extract:$ext VR256:$src1, (i32 imm)),
+def : Pat<(vextractf128_extract:$ext VR256:$src1, (iPTR imm)),
           (v4i32 (VEXTRACTI128rr
                     (v8i32 VR256:$src1),
                     (EXTRACT_get_vextractf128_imm VR128:$ext)))>;
-def : Pat<(vextractf128_extract:$ext VR256:$src1, (i32 imm)),
+def : Pat<(vextractf128_extract:$ext VR256:$src1, (iPTR imm)),
           (v8i16 (VEXTRACTI128rr
                     (v16i16 VR256:$src1),
                     (EXTRACT_get_vextractf128_imm VR128:$ext)))>;
-def : Pat<(vextractf128_extract:$ext VR256:$src1, (i32 imm)),
+def : Pat<(vextractf128_extract:$ext VR256:$src1, (iPTR imm)),
           (v16i8 (VEXTRACTI128rr
                     (v32i8 VR256:$src1),
                     (EXTRACT_get_vextractf128_imm VR128:$ext)))>;
+
+def : Pat<(alignedstore (v2i64 (vextractf128_extract:$ext (v4i64 VR256:$src1),
+                                (iPTR imm))), addr:$dst),
+          (VEXTRACTI128mr addr:$dst, VR256:$src1,
+           (EXTRACT_get_vextractf128_imm VR128:$ext))>;
+def : Pat<(alignedstore (v4i32 (vextractf128_extract:$ext (v8i32 VR256:$src1),
+                                (iPTR imm))), addr:$dst),
+          (VEXTRACTI128mr addr:$dst, VR256:$src1,
+           (EXTRACT_get_vextractf128_imm VR128:$ext))>;
+def : Pat<(alignedstore (v8i16 (vextractf128_extract:$ext (v16i16 VR256:$src1),
+                                (iPTR imm))), addr:$dst),
+          (VEXTRACTI128mr addr:$dst, VR256:$src1,
+           (EXTRACT_get_vextractf128_imm VR128:$ext))>;
+def : Pat<(alignedstore (v16i8 (vextractf128_extract:$ext (v32i8 VR256:$src1),
+                                (iPTR imm))), addr:$dst),
+          (VEXTRACTI128mr addr:$dst, VR256:$src1,
+           (EXTRACT_get_vextractf128_imm VR128:$ext))>;
 }
 
 //===----------------------------------------------------------------------===//
@@ -7852,7 +8167,8 @@ multiclass avx2_pmovmask<string OpcodeStr,
   def Yrm : AVX28I<0x8c, MRMSrcMem, (outs VR256:$dst),
              (ins VR256:$src1, i256mem:$src2),
              !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
-             [(set VR256:$dst, (IntLd256 addr:$src2, VR256:$src1))]>, VEX_4V;
+             [(set VR256:$dst, (IntLd256 addr:$src2, VR256:$src1))]>,
+             VEX_4V, VEX_L;
   def mr  : AVX28I<0x8e, MRMDestMem, (outs),
              (ins i128mem:$dst, VR128:$src1, VR128:$src2),
              !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
@@ -7860,7 +8176,7 @@ multiclass avx2_pmovmask<string OpcodeStr,
   def Ymr : AVX28I<0x8e, MRMDestMem, (outs),
              (ins i256mem:$dst, VR256:$src1, VR256:$src2),
              !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
-             [(IntSt256 addr:$dst, VR256:$src1, VR256:$src2)]>, VEX_4V;
+             [(IntSt256 addr:$dst, VR256:$src1, VR256:$src2)]>, VEX_4V, VEX_L;
 }
 
 defm VPMASKMOVD : avx2_pmovmask<"vpmaskmovd",
@@ -7898,14 +8214,14 @@ multiclass avx2_var_shift<bits<8> opc, string OpcodeStr, SDNode OpNode,
              !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
              [(set VR256:$dst,
                (vt256 (OpNode VR256:$src1, (vt256 VR256:$src2))))]>,
-             VEX_4V;
+             VEX_4V, VEX_L;
   def Yrm : AVX28I<opc, MRMSrcMem, (outs VR256:$dst),
              (ins VR256:$src1, i256mem:$src2),
              !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
              [(set VR256:$dst,
                (vt256 (OpNode VR256:$src1,
                        (vt256 (bitconvert (memopv4i64 addr:$src2))))))]>,
-             VEX_4V;
+             VEX_4V, VEX_L;
 }
 
 defm VPSLLVD : avx2_var_shift<0x47, "vpsllvd", shl, v4i32, v8i32>;
diff --git a/lib/Target/X86/X86InstrShiftRotate.td b/lib/Target/X86/X86InstrShiftRotate.td
index bdeb63f..893488c 100644
--- a/lib/Target/X86/X86InstrShiftRotate.td
+++ b/lib/Target/X86/X86InstrShiftRotate.td
@@ -839,6 +839,16 @@ def SHRD64mri8 : RIi8<0xAC, MRMDestMem,
 
 } // Defs = [EFLAGS]
 
+def ROT32L2R_imm8  : SDNodeXForm<imm, [{
+  // Convert a ROTL shamt to a ROTR shamt on 32-bit integer.
+  return getI8Imm(32 - N->getZExtValue());
+}]>;
+
+def ROT64L2R_imm8  : SDNodeXForm<imm, [{
+  // Convert a ROTL shamt to a ROTR shamt on 64-bit integer.
+  return getI8Imm(64 - N->getZExtValue());
+}]>;
+
 multiclass bmi_rotate<string asm, RegisterClass RC, X86MemOperand x86memop> {
 let neverHasSideEffects = 1 in {
   def ri : Ii8<0xF0, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, i8imm:$src2),
@@ -873,4 +883,72 @@ let Predicates = [HasBMI2] in {
   defm SHRX64 : bmi_shift<"shrx{q}", GR64, i64mem>, T8XD, VEX_W;
   defm SHLX32 : bmi_shift<"shlx{l}", GR32, i32mem>, T8, OpSize;
   defm SHLX64 : bmi_shift<"shlx{q}", GR64, i64mem>, T8, OpSize, VEX_W;
+
+  // Prefer RORX which is non-destructive and doesn't update EFLAGS.
+  let AddedComplexity = 10 in {
+    def : Pat<(rotl GR32:$src, (i8 imm:$shamt)),
+              (RORX32ri GR32:$src, (ROT32L2R_imm8 imm:$shamt))>;
+    def : Pat<(rotl GR64:$src, (i8 imm:$shamt)),
+              (RORX64ri GR64:$src, (ROT64L2R_imm8 imm:$shamt))>;
+  }
+
+  def : Pat<(rotl (loadi32 addr:$src), (i8 imm:$shamt)),
+            (RORX32mi addr:$src, (ROT32L2R_imm8 imm:$shamt))>;
+  def : Pat<(rotl (loadi64 addr:$src), (i8 imm:$shamt)),
+            (RORX64mi addr:$src, (ROT64L2R_imm8 imm:$shamt))>;
+
+  // Prefer SARX/SHRX/SHLX over SAR/SHR/SHL with variable shift BUT not
+  // immedidate shift, i.e. the following code is considered better
+  //
+  //  mov %edi, %esi
+  //  shl $imm, %esi
+  //  ... %edi, ...
+  //
+  // than
+  //
+  //  movb $imm, %sil
+  //  shlx %sil, %edi, %esi
+  //  ... %edi, ...
+  //
+  let AddedComplexity = 1 in {
+    def : Pat<(sra GR32:$src1, GR8:$src2),
+              (SARX32rr GR32:$src1,
+                        (INSERT_SUBREG
+                          (i32 (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>;
+    def : Pat<(sra GR64:$src1, GR8:$src2),
+              (SARX64rr GR64:$src1,
+                        (INSERT_SUBREG
+                          (i64 (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>;
+
+    def : Pat<(srl GR32:$src1, GR8:$src2),
+              (SHRX32rr GR32:$src1,
+                        (INSERT_SUBREG
+                          (i32 (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>;
+    def : Pat<(srl GR64:$src1, GR8:$src2),
+              (SHRX64rr GR64:$src1,
+                        (INSERT_SUBREG
+                          (i64 (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>;
+
+    def : Pat<(shl GR32:$src1, GR8:$src2),
+              (SHLX32rr GR32:$src1,
+                        (INSERT_SUBREG
+                          (i32 (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>;
+    def : Pat<(shl GR64:$src1, GR8:$src2),
+              (SHLX64rr GR64:$src1,
+                        (INSERT_SUBREG
+                          (i64 (IMPLICIT_DEF)), GR8:$src2, sub_8bit))>;
+  }
+
+  // Patterns on SARXrm/SHRXrm/SHLXrm are explicitly omitted to favor
+  //
+  //  mov (%ecx), %esi
+  //  shl $imm, $esi
+  //
+  // over
+  //
+  //  movb $imm %al
+  //  shlx %al, (%ecx), %esi
+  //
+  // As SARXrr/SHRXrr/SHLXrr is favored on variable shift, the peephole
+  // optimization will fold them into SARXrm/SHRXrm/SHLXrm if possible.
 }
diff --git a/lib/Target/X86/X86InstrTSX.td b/lib/Target/X86/X86InstrTSX.td
new file mode 100644
index 0000000..ad55058
--- /dev/null
+++ b/lib/Target/X86/X86InstrTSX.td
@@ -0,0 +1,32 @@
+//===-- X86InstrVMX.td - TSX Instruction Set Extension -----*- tablegen -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file describes the instructions that make up the Intel TSX instruction
+// set.
+//
+//===----------------------------------------------------------------------===//
+
+//===----------------------------------------------------------------------===//
+// TSX instructions
+
+let usesCustomInserter = 1 in
+def XBEGIN : I<0, Pseudo, (outs GR32:$dst), (ins),
+               "# XBEGIN", [(set GR32:$dst, (int_x86_xbegin))]>,
+             Requires<[HasRTM]>;
+
+let isBranch = 1, isTerminator = 1, Defs = [EAX] in
+def XBEGIN_4 : Ii32PCRel<0xc7, MRM_F8, (outs), (ins brtarget:$dst),
+                         "xbegin\t$dst", []>;
+
+def XEND : I<0x01, MRM_D5, (outs), (ins),
+             "xend", [(int_x86_xend)]>, TB, Requires<[HasRTM]>;
+
+def XABORT : Ii8<0xc6, MRM_F8, (outs), (ins i8imm:$imm),
+                 "xabort\t$imm",
+                 [(int_x86_xabort imm:$imm)]>, Requires<[HasRTM]>;
diff --git a/lib/Target/X86/X86InstrXOP.td b/lib/Target/X86/X86InstrXOP.td
index 8ec2c68..2aa08fa 100644
--- a/lib/Target/X86/X86InstrXOP.td
+++ b/lib/Target/X86/X86InstrXOP.td
@@ -75,10 +75,10 @@ multiclass xop2op256<bits<8> opc, string OpcodeStr, Intrinsic Int,
                      PatFrag memop> {
   def rrY : IXOP<opc, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src),
            !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
-           [(set VR256:$dst, (Int VR256:$src))]>, VEX;
+           [(set VR256:$dst, (Int VR256:$src))]>, VEX, VEX_L;
   def rmY : IXOP<opc, MRMSrcMem, (outs VR256:$dst), (ins f256mem:$src),
            !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
-           [(set VR256:$dst, (Int (bitconvert (memop addr:$src))))]>, VEX;
+           [(set VR256:$dst, (Int (bitconvert (memop addr:$src))))]>, VEX, VEX_L;
 }
 
 let isAsmParserOnly = 1 in {
@@ -238,7 +238,7 @@ multiclass xop4op256<bits<8> opc, string OpcodeStr, Intrinsic Int> {
            !strconcat(OpcodeStr,
            "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"),
            [(set VR256:$dst, (Int VR256:$src1, VR256:$src2, VR256:$src3))]>,
-           VEX_4V, VEX_I8IMM;
+           VEX_4V, VEX_I8IMM, VEX_L;
   def rmY : IXOPi8<opc, MRMSrcMem, (outs VR256:$dst),
            (ins VR256:$src1, VR256:$src2, i256mem:$src3),
            !strconcat(OpcodeStr,
@@ -246,7 +246,7 @@ multiclass xop4op256<bits<8> opc, string OpcodeStr, Intrinsic Int> {
            [(set VR256:$dst,
              (Int VR256:$src1, VR256:$src2,
               (bitconvert (memopv4i64 addr:$src3))))]>,
-           VEX_4V, VEX_I8IMM, VEX_W, MemOp4;
+           VEX_4V, VEX_I8IMM, VEX_W, MemOp4, VEX_L;
   def mrY : IXOPi8<opc, MRMSrcMem, (outs VR256:$dst),
            (ins VR256:$src1, f256mem:$src2, VR256:$src3),
            !strconcat(OpcodeStr,
@@ -254,7 +254,7 @@ multiclass xop4op256<bits<8> opc, string OpcodeStr, Intrinsic Int> {
            [(set VR256:$dst,
              (Int VR256:$src1, (bitconvert (memopv4i64 addr:$src2)),
               VR256:$src3))]>,
-           VEX_4V, VEX_I8IMM;
+           VEX_4V, VEX_I8IMM, VEX_L;
 }
 
 let isAsmParserOnly = 1 in {
@@ -287,20 +287,21 @@ multiclass xop5op<bits<8> opc, string OpcodeStr, Intrinsic Int128,
         !strconcat(OpcodeStr,
         "\t{$src4, $src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3, $src4}"),
         [(set VR256:$dst,
-          (Int256 VR256:$src1, VR256:$src2, VR256:$src3, imm:$src4))]>;
+          (Int256 VR256:$src1, VR256:$src2, VR256:$src3, imm:$src4))]>, VEX_L;
   def rmY : IXOP5<opc, MRMSrcMem, (outs VR256:$dst),
         (ins VR256:$src1, VR256:$src2, f256mem:$src3, i8imm:$src4),
         !strconcat(OpcodeStr,
         "\t{$src4, $src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3, $src4}"),
         [(set VR256:$dst,
           (Int256 VR256:$src1, VR256:$src2, (ld_256 addr:$src3), imm:$src4))]>,
-        VEX_W, MemOp4;
+        VEX_W, MemOp4, VEX_L;
   def mrY : IXOP5<opc, MRMSrcMem, (outs VR256:$dst),
         (ins VR256:$src1, f256mem:$src2, VR256:$src3, i8imm:$src4),
         !strconcat(OpcodeStr,
         "\t{$src4, $src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3, $src4}"),
         [(set VR256:$dst,
-           (Int256 VR256:$src1, (ld_256 addr:$src2), VR256:$src3, imm:$src4))]>;
+           (Int256 VR256:$src1, (ld_256 addr:$src2), VR256:$src3, imm:$src4))]>,
+        VEX_L;
 }
 
 defm VPERMIL2PD : xop5op<0x49, "vpermil2pd", int_x86_xop_vpermil2pd,
diff --git a/lib/Target/X86/X86JITInfo.cpp b/lib/Target/X86/X86JITInfo.cpp
index 0168d12..764aa5d 100644
--- a/lib/Target/X86/X86JITInfo.cpp
+++ b/lib/Target/X86/X86JITInfo.cpp
@@ -532,6 +532,15 @@ uintptr_t X86JITInfo::getPICJumpTableEntry(uintptr_t BB, uintptr_t Entry) {
 #endif
 }
 
+template<typename T> static void addUnaligned(void *Pos, T Delta) {
+  T Value;
+  std::memcpy(reinterpret_cast<char*>(&Value), reinterpret_cast<char*>(Pos),
+              sizeof(T));
+  Value += Delta;
+  std::memcpy(reinterpret_cast<char*>(Pos), reinterpret_cast<char*>(&Value),
+              sizeof(T));
+}
+
 /// relocate - Before the JIT can run a block of code that has been emitted,
 /// it must rewrite the code to contain the actual addresses of any
 /// referenced global symbols.
@@ -545,24 +554,24 @@ void X86JITInfo::relocate(void *Function, MachineRelocation *MR,
       // PC relative relocation, add the relocated value to the value already in
       // memory, after we adjust it for where the PC is.
       ResultPtr = ResultPtr -(intptr_t)RelocPos - 4 - MR->getConstantVal();
-      *((unsigned*)RelocPos) += (unsigned)ResultPtr;
+      addUnaligned<unsigned>(RelocPos, ResultPtr);
       break;
     }
     case X86::reloc_picrel_word: {
       // PIC base relative relocation, add the relocated value to the value
       // already in memory, after we adjust it for where the PIC base is.
       ResultPtr = ResultPtr - ((intptr_t)Function + MR->getConstantVal());
-      *((unsigned*)RelocPos) += (unsigned)ResultPtr;
+      addUnaligned<unsigned>(RelocPos, ResultPtr);
       break;
     }
     case X86::reloc_absolute_word:
     case X86::reloc_absolute_word_sext:
       // Absolute relocation, just add the relocated value to the value already
       // in memory.
-      *((unsigned*)RelocPos) += (unsigned)ResultPtr;
+      addUnaligned<unsigned>(RelocPos, ResultPtr);
       break;
     case X86::reloc_absolute_dword:
-      *((intptr_t*)RelocPos) += ResultPtr;
+      addUnaligned<intptr_t>(RelocPos, ResultPtr);
       break;
     }
   }
diff --git a/lib/Target/X86/X86MCInstLower.cpp b/lib/Target/X86/X86MCInstLower.cpp
index 9c0ce4e..cfd68f7 100644
--- a/lib/Target/X86/X86MCInstLower.cpp
+++ b/lib/Target/X86/X86MCInstLower.cpp
@@ -12,7 +12,6 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "X86MCInstLower.h"
 #include "X86AsmPrinter.h"
 #include "X86COFFMachineModuleInfo.h"
 #include "InstPrinter/X86ATTInstPrinter.h"
@@ -29,6 +28,31 @@
 #include "llvm/ADT/SmallString.h"
 using namespace llvm;
 
+namespace {
+
+/// X86MCInstLower - This class is used to lower an MachineInstr into an MCInst.
+class X86MCInstLower {
+  MCContext &Ctx;
+  Mangler *Mang;
+  const MachineFunction &MF;
+  const TargetMachine &TM;
+  const MCAsmInfo &MAI;
+  X86AsmPrinter &AsmPrinter;
+public:
+  X86MCInstLower(Mangler *mang, const MachineFunction &MF,
+                 X86AsmPrinter &asmprinter);
+
+  void Lower(const MachineInstr *MI, MCInst &OutMI) const;
+
+  MCSymbol *GetSymbolFromOperand(const MachineOperand &MO) const;
+  MCOperand LowerSymbolOperand(const MachineOperand &MO, MCSymbol *Sym) const;
+
+private:
+  MachineModuleInfoMachO &getMachOMMI() const;
+};
+
+} // end anonymous namespace
+
 X86MCInstLower::X86MCInstLower(Mangler *mang, const MachineFunction &mf,
                                X86AsmPrinter &asmprinter)
 : Ctx(mf.getContext()), Mang(mang), MF(mf), TM(mf.getTarget()),
@@ -43,15 +67,11 @@ MachineModuleInfoMachO &X86MCInstLower::getMachOMMI() const {
 /// operand to an MCSymbol.
 MCSymbol *X86MCInstLower::
 GetSymbolFromOperand(const MachineOperand &MO) const {
-  assert((MO.isGlobal() || MO.isSymbol()) && "Isn't a symbol reference");
+  assert((MO.isGlobal() || MO.isSymbol() || MO.isMBB()) && "Isn't a symbol reference");
 
   SmallString<128> Name;
 
-  if (!MO.isGlobal()) {
-    assert(MO.isSymbol());
-    Name += MAI.getGlobalPrefix();
-    Name += MO.getSymbolName();
-  } else {
+  if (MO.isGlobal()) {
     const GlobalValue *GV = MO.getGlobal();
     bool isImplicitlyPrivate = false;
     if (MO.getTargetFlags() == X86II::MO_DARWIN_STUB ||
@@ -61,6 +81,11 @@ GetSymbolFromOperand(const MachineOperand &MO) const {
       isImplicitlyPrivate = true;
 
     Mang->getNameWithPrefix(Name, GV, isImplicitlyPrivate);
+  } else if (MO.isSymbol()) {
+    Name += MAI.getGlobalPrefix();
+    Name += MO.getSymbolName();
+  } else if (MO.isMBB()) {
+    Name += MO.getMBB()->getSymbol()->getName();
   }
 
   // If the target flags on the operand changes the name of the symbol, do that
@@ -191,7 +216,7 @@ MCOperand X86MCInstLower::LowerSymbolOperand(const MachineOperand &MO,
   if (Expr == 0)
     Expr = MCSymbolRefExpr::Create(Sym, RefKind, Ctx);
 
-  if (!MO.isJTI() && MO.getOffset())
+  if (!MO.isJTI() && !MO.isMBB() && MO.getOffset())
     Expr = MCBinaryExpr::CreateAdd(Expr,
                                    MCConstantExpr::Create(MO.getOffset(), Ctx),
                                    Ctx);
@@ -324,9 +349,6 @@ void X86MCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const {
       MCOp = MCOperand::CreateImm(MO.getImm());
       break;
     case MachineOperand::MO_MachineBasicBlock:
-      MCOp = MCOperand::CreateExpr(MCSymbolRefExpr::Create(
-                       MO.getMBB()->getSymbol(), Ctx));
-      break;
     case MachineOperand::MO_GlobalAddress:
     case MachineOperand::MO_ExternalSymbol:
       MCOp = LowerSymbolOperand(MO, GetSymbolFromOperand(MO));
@@ -371,18 +393,8 @@ ReSimplify:
   case X86::MOVZX64rm8:   LowerSubReg32_Op0(OutMI, X86::MOVZX32rm8); break;
   case X86::MOVZX64rr16:  LowerSubReg32_Op0(OutMI, X86::MOVZX32rr16); break;
   case X86::MOVZX64rm16:  LowerSubReg32_Op0(OutMI, X86::MOVZX32rm16); break;
-  case X86::SETB_C8r:     LowerUnaryToTwoAddr(OutMI, X86::SBB8rr); break;
-  case X86::SETB_C16r:    LowerUnaryToTwoAddr(OutMI, X86::SBB16rr); break;
-  case X86::SETB_C32r:    LowerUnaryToTwoAddr(OutMI, X86::SBB32rr); break;
-  case X86::SETB_C64r:    LowerUnaryToTwoAddr(OutMI, X86::SBB64rr); break;
   case X86::MOV8r0:       LowerUnaryToTwoAddr(OutMI, X86::XOR8rr); break;
   case X86::MOV32r0:      LowerUnaryToTwoAddr(OutMI, X86::XOR32rr); break;
-  case X86::V_SETALLONES:  LowerUnaryToTwoAddr(OutMI, X86::PCMPEQDrr); break;
-  case X86::AVX_SET0PSY:   LowerUnaryToTwoAddr(OutMI, X86::VXORPSYrr); break;
-  case X86::AVX_SET0PDY:   LowerUnaryToTwoAddr(OutMI, X86::VXORPDYrr); break;
-  case X86::AVX_SETALLONES:  LowerUnaryToTwoAddr(OutMI, X86::VPCMPEQDrr); break;
-  case X86::AVX2_SETALLONES: LowerUnaryToTwoAddr(OutMI, X86::VPCMPEQDYrr);break;
-  case X86::AVX2_SET0:     LowerUnaryToTwoAddr(OutMI, X86::VPXORYrr); break;
 
   case X86::MOV16r0:
     LowerSubReg32_Op0(OutMI, X86::MOV32r0);   // MOV16r0 -> MOV32r0
diff --git a/lib/Target/X86/X86MCInstLower.h b/lib/Target/X86/X86MCInstLower.h
deleted file mode 100644
index b4d4cfd..0000000
--- a/lib/Target/X86/X86MCInstLower.h
+++ /dev/null
@@ -1,52 +0,0 @@
-//===-- X86MCInstLower.h - Lower MachineInstr to MCInst ---------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef X86_MCINSTLOWER_H
-#define X86_MCINSTLOWER_H
-
-#include "llvm/Support/Compiler.h"
-
-namespace llvm {
-  class MCAsmInfo;
-  class MCContext;
-  class MCInst;
-  class MCOperand;
-  class MCSymbol;
-  class MachineInstr;
-  class MachineFunction;
-  class MachineModuleInfoMachO;
-  class MachineOperand;
-  class Mangler;
-  class TargetMachine;
-  class X86AsmPrinter;
-
-/// X86MCInstLower - This class is used to lower an MachineInstr into an MCInst.
-class LLVM_LIBRARY_VISIBILITY X86MCInstLower {
-  MCContext &Ctx;
-  Mangler *Mang;
-  const MachineFunction &MF;
-  const TargetMachine &TM;
-  const MCAsmInfo &MAI;
-  X86AsmPrinter &AsmPrinter;
-public:
-  X86MCInstLower(Mangler *mang, const MachineFunction &MF,
-                 X86AsmPrinter &asmprinter);
-
-  void Lower(const MachineInstr *MI, MCInst &OutMI) const;
-
-  MCSymbol *GetSymbolFromOperand(const MachineOperand &MO) const;
-  MCOperand LowerSymbolOperand(const MachineOperand &MO, MCSymbol *Sym) const;
-
-private:
-  MachineModuleInfoMachO &getMachOMMI() const;
-};
-
-}
-
-#endif
diff --git a/lib/Target/X86/X86RegisterInfo.cpp b/lib/Target/X86/X86RegisterInfo.cpp
index 877b8f6..73ac747 100644
--- a/lib/Target/X86/X86RegisterInfo.cpp
+++ b/lib/Target/X86/X86RegisterInfo.cpp
@@ -106,23 +106,7 @@ X86RegisterInfo::trackLivenessAfterRegAlloc(const MachineFunction &MF) const {
 
 int
 X86RegisterInfo::getSEHRegNum(unsigned i) const {
-  int reg = X86_MC::getX86RegNum(i);
-  switch (i) {
-  case X86::R8:  case X86::R8D:  case X86::R8W:  case X86::R8B:
-  case X86::R9:  case X86::R9D:  case X86::R9W:  case X86::R9B:
-  case X86::R10: case X86::R10D: case X86::R10W: case X86::R10B:
-  case X86::R11: case X86::R11D: case X86::R11W: case X86::R11B:
-  case X86::R12: case X86::R12D: case X86::R12W: case X86::R12B:
-  case X86::R13: case X86::R13D: case X86::R13W: case X86::R13B:
-  case X86::R14: case X86::R14D: case X86::R14W: case X86::R14B:
-  case X86::R15: case X86::R15D: case X86::R15W: case X86::R15B:
-  case X86::XMM8: case X86::XMM9: case X86::XMM10: case X86::XMM11:
-  case X86::XMM12: case X86::XMM13: case X86::XMM14: case X86::XMM15:
-  case X86::YMM8: case X86::YMM9: case X86::YMM10: case X86::YMM11:
-  case X86::YMM12: case X86::YMM13: case X86::YMM14: case X86::YMM15:
-    reg += 8;
-  }
-  return reg;
+  return getEncodingValue(i);
 }
 
 const TargetRegisterClass *
@@ -245,15 +229,26 @@ const uint16_t *
 X86RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
   bool callsEHReturn = false;
   bool ghcCall = false;
+  bool oclBiCall = false;
+  bool HasAVX = TM.getSubtarget<X86Subtarget>().hasAVX();
 
   if (MF) {
     callsEHReturn = MF->getMMI().callsEHReturn();
     const Function *F = MF->getFunction();
     ghcCall = (F ? F->getCallingConv() == CallingConv::GHC : false);
+    oclBiCall = (F ? F->getCallingConv() == CallingConv::Intel_OCL_BI : false);
   }
 
   if (ghcCall)
     return CSR_NoRegs_SaveList;
+  if (oclBiCall) {
+    if (HasAVX && IsWin64)
+        return CSR_Win64_Intel_OCL_BI_AVX_SaveList;
+    if (HasAVX && Is64Bit)
+        return CSR_64_Intel_OCL_BI_AVX_SaveList;
+    if (!HasAVX && !IsWin64 && Is64Bit)
+        return CSR_64_Intel_OCL_BI_SaveList;
+  }
   if (Is64Bit) {
     if (IsWin64)
       return CSR_Win64_SaveList;
@@ -268,6 +263,16 @@ X86RegisterInfo::getCalleeSavedRegs(const MachineFunction *MF) const {
 
 const uint32_t*
 X86RegisterInfo::getCallPreservedMask(CallingConv::ID CC) const {
+  bool HasAVX = TM.getSubtarget<X86Subtarget>().hasAVX();
+
+  if (CC == CallingConv::Intel_OCL_BI) {
+    if (IsWin64 && HasAVX)
+      return CSR_Win64_Intel_OCL_BI_AVX_RegMask;
+    if (Is64Bit && HasAVX)
+      return CSR_64_Intel_OCL_BI_AVX_RegMask;
+    if (!HasAVX && !IsWin64 && Is64Bit)
+      return CSR_64_Intel_OCL_BI_RegMask;
+  }
   if (CC == CallingConv::GHC)
     return CSR_NoRegs_RegMask;
   if (!Is64Bit)
@@ -277,6 +282,11 @@ X86RegisterInfo::getCallPreservedMask(CallingConv::ID CC) const {
   return CSR_64_RegMask;
 }
 
+const uint32_t*
+X86RegisterInfo::getNoPreservedMask() const {
+  return CSR_NoRegs_RegMask;
+}
+
 BitVector X86RegisterInfo::getReservedRegs(const MachineFunction &MF) const {
   BitVector Reserved(getNumRegs());
   const TargetFrameLowering *TFI = MF.getTarget().getFrameLowering();
@@ -398,8 +408,9 @@ bool X86RegisterInfo::needsStackRealignment(const MachineFunction &MF) const {
   const MachineFrameInfo *MFI = MF.getFrameInfo();
   const Function *F = MF.getFunction();
   unsigned StackAlign = TM.getFrameLowering()->getStackAlignment();
-  bool requiresRealignment = ((MFI->getMaxAlignment() > StackAlign) ||
-                               F->hasFnAttr(Attribute::StackAlignment));
+  bool requiresRealignment =
+    ((MFI->getMaxAlignment() > StackAlign) ||
+     F->getFnAttributes().hasAttribute(Attributes::StackAlignment));
 
   // If we've requested that we force align the stack do so now.
   if (ForceStackAlign)
@@ -522,7 +533,7 @@ eliminateCallFramePseudoInstr(MachineFunction &MF, MachineBasicBlock &MBB,
 
 void
 X86RegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
-                                     int SPAdj, RegScavenger *RS) const{
+                                     int SPAdj, RegScavenger *RS) const {
   assert(SPAdj == 0 && "Unexpected");
 
   unsigned i = 0;
@@ -590,9 +601,10 @@ unsigned X86RegisterInfo::getEHHandlerRegister() const {
 }
 
 namespace llvm {
-unsigned getX86SubSuperRegister(unsigned Reg, EVT VT, bool High) {
-  switch (VT.getSimpleVT().SimpleTy) {
-  default: return Reg;
+unsigned getX86SubSuperRegister(unsigned Reg, MVT::SimpleValueType VT,
+                                bool High) {
+  switch (VT) {
+  default: llvm_unreachable("Unexpected VT");
   case MVT::i8:
     if (High) {
       switch (Reg) {
@@ -608,7 +620,7 @@ unsigned getX86SubSuperRegister(unsigned Reg, EVT VT, bool High) {
       }
     } else {
       switch (Reg) {
-      default: return 0;
+      default: llvm_unreachable("Unexpected register");
       case X86::AH: case X86::AL: case X86::AX: case X86::EAX: case X86::RAX:
         return X86::AL;
       case X86::DH: case X86::DL: case X86::DX: case X86::EDX: case X86::RDX:
@@ -645,7 +657,7 @@ unsigned getX86SubSuperRegister(unsigned Reg, EVT VT, bool High) {
     }
   case MVT::i16:
     switch (Reg) {
-    default: return Reg;
+    default: llvm_unreachable("Unexpected register");
     case X86::AH: case X86::AL: case X86::AX: case X86::EAX: case X86::RAX:
       return X86::AX;
     case X86::DH: case X86::DL: case X86::DX: case X86::EDX: case X86::RDX:
@@ -681,7 +693,7 @@ unsigned getX86SubSuperRegister(unsigned Reg, EVT VT, bool High) {
     }
   case MVT::i32:
     switch (Reg) {
-    default: return Reg;
+    default: llvm_unreachable("Unexpected register");
     case X86::AH: case X86::AL: case X86::AX: case X86::EAX: case X86::RAX:
       return X86::EAX;
     case X86::DH: case X86::DL: case X86::DX: case X86::EDX: case X86::RDX:
@@ -733,7 +745,7 @@ unsigned getX86SubSuperRegister(unsigned Reg, EVT VT, bool High) {
       }
     }
     switch (Reg) {
-    default: return Reg;
+    default: llvm_unreachable("Unexpected register");
     case X86::AH: case X86::AL: case X86::AX: case X86::EAX: case X86::RAX:
       return X86::RAX;
     case X86::DH: case X86::DL: case X86::DX: case X86::EDX: case X86::RDX:
diff --git a/lib/Target/X86/X86RegisterInfo.h b/lib/Target/X86/X86RegisterInfo.h
index 1bc32cb..7932ede 100644
--- a/lib/Target/X86/X86RegisterInfo.h
+++ b/lib/Target/X86/X86RegisterInfo.h
@@ -58,10 +58,6 @@ private:
 public:
   X86RegisterInfo(X86TargetMachine &tm, const TargetInstrInfo &tii);
 
-  /// getX86RegNum - Returns the native X86 register number for the given LLVM
-  /// register identifier.
-  static unsigned getX86RegNum(unsigned RegNo);
-
   // FIXME: This should be tablegen'd like getDwarfRegNum is
   int getSEHRegNum(unsigned i) const;
 
@@ -104,6 +100,7 @@ public:
   /// callee-save registers on this target.
   const uint16_t *getCalleeSavedRegs(const MachineFunction* MF = 0) const;
   const uint32_t *getCallPreservedMask(CallingConv::ID) const;
+  const uint32_t *getNoPreservedMask() const;
 
   /// getReservedRegs - Returns a bitset indexed by physical register number
   /// indicating if a register is a special register that has particular uses and
@@ -141,8 +138,8 @@ public:
 
 // getX86SubSuperRegister - X86 utility function. It returns the sub or super
 // register of a specific X86 register.
-// e.g. getX86SubSuperRegister(X86::EAX, EVT::i16) return X86:AX
-unsigned getX86SubSuperRegister(unsigned, EVT, bool High=false);
+// e.g. getX86SubSuperRegister(X86::EAX, MVT::i16) return X86:AX
+unsigned getX86SubSuperRegister(unsigned, MVT::SimpleValueType, bool High=false);
 
 } // End llvm namespace
 
diff --git a/lib/Target/X86/X86RegisterInfo.td b/lib/Target/X86/X86RegisterInfo.td
index edc7184..be6282a 100644
--- a/lib/Target/X86/X86RegisterInfo.td
+++ b/lib/Target/X86/X86RegisterInfo.td
@@ -13,258 +13,264 @@
 //
 //===----------------------------------------------------------------------===//
 
-//===----------------------------------------------------------------------===//
-//  Register definitions...
-//
-let Namespace = "X86" in {
+class X86Reg<string n, bits<16> Enc, list<Register> subregs = []> : Register<n> {
+  let Namespace = "X86";
+  let HWEncoding = Enc;
+  let SubRegs = subregs;
+}
 
-  // Subregister indices.
+// Subregister indices.
+let Namespace = "X86" in {
   def sub_8bit    : SubRegIndex;
   def sub_8bit_hi : SubRegIndex;
   def sub_16bit   : SubRegIndex;
   def sub_32bit   : SubRegIndex;
-  def sub_xmm : SubRegIndex;
-
-
-  // In the register alias definitions below, we define which registers alias
-  // which others.  We only specify which registers the small registers alias,
-  // because the register file generator is smart enough to figure out that
-  // AL aliases AX if we tell it that AX aliased AL (for example).
-
-  // Dwarf numbering is different for 32-bit and 64-bit, and there are
-  // variations by target as well. Currently the first entry is for X86-64,
-  // second - for EH on X86-32/Darwin and third is 'generic' one (X86-32/Linux
-  // and debug information on X86-32/Darwin)
-
-  // 8-bit registers
-  // Low registers
-  def AL : Register<"al">;
-  def DL : Register<"dl">;
-  def CL : Register<"cl">;
-  def BL : Register<"bl">;
-
-  // X86-64 only, requires REX.
-  let CostPerUse = 1 in {
-  def SIL : Register<"sil">;
-  def DIL : Register<"dil">;
-  def BPL : Register<"bpl">;
-  def SPL : Register<"spl">;
-  def R8B  : Register<"r8b">;
-  def R9B  : Register<"r9b">;
-  def R10B : Register<"r10b">;
-  def R11B : Register<"r11b">;
-  def R12B : Register<"r12b">;
-  def R13B : Register<"r13b">;
-  def R14B : Register<"r14b">;
-  def R15B : Register<"r15b">;
-  }
-
-  // High registers. On x86-64, these cannot be used in any instruction
-  // with a REX prefix.
-  def AH : Register<"ah">;
-  def DH : Register<"dh">;
-  def CH : Register<"ch">;
-  def BH : Register<"bh">;
-
-  // 16-bit registers
-  let SubRegIndices = [sub_8bit, sub_8bit_hi], CoveredBySubRegs = 1 in {
-  def AX : RegisterWithSubRegs<"ax", [AL,AH]>;
-  def DX : RegisterWithSubRegs<"dx", [DL,DH]>;
-  def CX : RegisterWithSubRegs<"cx", [CL,CH]>;
-  def BX : RegisterWithSubRegs<"bx", [BL,BH]>;
-  }
-  let SubRegIndices = [sub_8bit] in {
-  def SI : RegisterWithSubRegs<"si", [SIL]>;
-  def DI : RegisterWithSubRegs<"di", [DIL]>;
-  def BP : RegisterWithSubRegs<"bp", [BPL]>;
-  def SP : RegisterWithSubRegs<"sp", [SPL]>;
-  }
-  def IP : Register<"ip">;
-
-  // X86-64 only, requires REX.
-  let SubRegIndices = [sub_8bit], CostPerUse = 1 in {
-  def R8W  : RegisterWithSubRegs<"r8w", [R8B]>;
-  def R9W  : RegisterWithSubRegs<"r9w", [R9B]>;
-  def R10W : RegisterWithSubRegs<"r10w", [R10B]>;
-  def R11W : RegisterWithSubRegs<"r11w", [R11B]>;
-  def R12W : RegisterWithSubRegs<"r12w", [R12B]>;
-  def R13W : RegisterWithSubRegs<"r13w", [R13B]>;
-  def R14W : RegisterWithSubRegs<"r14w", [R14B]>;
-  def R15W : RegisterWithSubRegs<"r15w", [R15B]>;
-  }
-  // 32-bit registers
-  let SubRegIndices = [sub_16bit] in {
-  def EAX : RegisterWithSubRegs<"eax", [AX]>, DwarfRegNum<[-2, 0, 0]>;
-  def EDX : RegisterWithSubRegs<"edx", [DX]>, DwarfRegNum<[-2, 2, 2]>;
-  def ECX : RegisterWithSubRegs<"ecx", [CX]>, DwarfRegNum<[-2, 1, 1]>;
-  def EBX : RegisterWithSubRegs<"ebx", [BX]>, DwarfRegNum<[-2, 3, 3]>;
-  def ESI : RegisterWithSubRegs<"esi", [SI]>, DwarfRegNum<[-2, 6, 6]>;
-  def EDI : RegisterWithSubRegs<"edi", [DI]>, DwarfRegNum<[-2, 7, 7]>;
-  def EBP : RegisterWithSubRegs<"ebp", [BP]>, DwarfRegNum<[-2, 4, 5]>;
-  def ESP : RegisterWithSubRegs<"esp", [SP]>, DwarfRegNum<[-2, 5, 4]>;
-  def EIP : RegisterWithSubRegs<"eip", [IP]>, DwarfRegNum<[-2, 8, 8]>;
-
-  // X86-64 only, requires REX
-  let CostPerUse = 1 in {
-  def R8D  : RegisterWithSubRegs<"r8d", [R8W]>;
-  def R9D  : RegisterWithSubRegs<"r9d", [R9W]>;
-  def R10D : RegisterWithSubRegs<"r10d", [R10W]>;
-  def R11D : RegisterWithSubRegs<"r11d", [R11W]>;
-  def R12D : RegisterWithSubRegs<"r12d", [R12W]>;
-  def R13D : RegisterWithSubRegs<"r13d", [R13W]>;
-  def R14D : RegisterWithSubRegs<"r14d", [R14W]>;
-  def R15D : RegisterWithSubRegs<"r15d", [R15W]>;
-  }}
-
-  // 64-bit registers, X86-64 only
-  let SubRegIndices = [sub_32bit] in {
-  def RAX : RegisterWithSubRegs<"rax", [EAX]>, DwarfRegNum<[0, -2, -2]>;
-  def RDX : RegisterWithSubRegs<"rdx", [EDX]>, DwarfRegNum<[1, -2, -2]>;
-  def RCX : RegisterWithSubRegs<"rcx", [ECX]>, DwarfRegNum<[2, -2, -2]>;
-  def RBX : RegisterWithSubRegs<"rbx", [EBX]>, DwarfRegNum<[3, -2, -2]>;
-  def RSI : RegisterWithSubRegs<"rsi", [ESI]>, DwarfRegNum<[4, -2, -2]>;
-  def RDI : RegisterWithSubRegs<"rdi", [EDI]>, DwarfRegNum<[5, -2, -2]>;
-  def RBP : RegisterWithSubRegs<"rbp", [EBP]>, DwarfRegNum<[6, -2, -2]>;
-  def RSP : RegisterWithSubRegs<"rsp", [ESP]>, DwarfRegNum<[7, -2, -2]>;
-
-  // These also require REX.
-  let CostPerUse = 1 in {
-  def R8  : RegisterWithSubRegs<"r8", [R8D]>, DwarfRegNum<[8, -2, -2]>;
-  def R9  : RegisterWithSubRegs<"r9", [R9D]>, DwarfRegNum<[9, -2, -2]>;
-  def R10 : RegisterWithSubRegs<"r10", [R10D]>, DwarfRegNum<[10, -2, -2]>;
-  def R11 : RegisterWithSubRegs<"r11", [R11D]>, DwarfRegNum<[11, -2, -2]>;
-  def R12 : RegisterWithSubRegs<"r12", [R12D]>, DwarfRegNum<[12, -2, -2]>;
-  def R13 : RegisterWithSubRegs<"r13", [R13D]>, DwarfRegNum<[13, -2, -2]>;
-  def R14 : RegisterWithSubRegs<"r14", [R14D]>, DwarfRegNum<[14, -2, -2]>;
-  def R15 : RegisterWithSubRegs<"r15", [R15D]>, DwarfRegNum<[15, -2, -2]>;
-  def RIP : RegisterWithSubRegs<"rip", [EIP]>,  DwarfRegNum<[16, -2, -2]>;
-  }}
-
-  // MMX Registers. These are actually aliased to ST0 .. ST7
-  def MM0 : Register<"mm0">, DwarfRegNum<[41, 29, 29]>;
-  def MM1 : Register<"mm1">, DwarfRegNum<[42, 30, 30]>;
-  def MM2 : Register<"mm2">, DwarfRegNum<[43, 31, 31]>;
-  def MM3 : Register<"mm3">, DwarfRegNum<[44, 32, 32]>;
-  def MM4 : Register<"mm4">, DwarfRegNum<[45, 33, 33]>;
-  def MM5 : Register<"mm5">, DwarfRegNum<[46, 34, 34]>;
-  def MM6 : Register<"mm6">, DwarfRegNum<[47, 35, 35]>;
-  def MM7 : Register<"mm7">, DwarfRegNum<[48, 36, 36]>;
-
-  // Pseudo Floating Point registers
-  def FP0 : Register<"fp0">;
-  def FP1 : Register<"fp1">;
-  def FP2 : Register<"fp2">;
-  def FP3 : Register<"fp3">;
-  def FP4 : Register<"fp4">;
-  def FP5 : Register<"fp5">;
-  def FP6 : Register<"fp6">;
-
-  // XMM Registers, used by the various SSE instruction set extensions.
-  def XMM0: Register<"xmm0">, DwarfRegNum<[17, 21, 21]>;
-  def XMM1: Register<"xmm1">, DwarfRegNum<[18, 22, 22]>;
-  def XMM2: Register<"xmm2">, DwarfRegNum<[19, 23, 23]>;
-  def XMM3: Register<"xmm3">, DwarfRegNum<[20, 24, 24]>;
-  def XMM4: Register<"xmm4">, DwarfRegNum<[21, 25, 25]>;
-  def XMM5: Register<"xmm5">, DwarfRegNum<[22, 26, 26]>;
-  def XMM6: Register<"xmm6">, DwarfRegNum<[23, 27, 27]>;
-  def XMM7: Register<"xmm7">, DwarfRegNum<[24, 28, 28]>;
-
-  // X86-64 only
-  let CostPerUse = 1 in {
-  def XMM8:  Register<"xmm8">,  DwarfRegNum<[25, -2, -2]>;
-  def XMM9:  Register<"xmm9">,  DwarfRegNum<[26, -2, -2]>;
-  def XMM10: Register<"xmm10">, DwarfRegNum<[27, -2, -2]>;
-  def XMM11: Register<"xmm11">, DwarfRegNum<[28, -2, -2]>;
-  def XMM12: Register<"xmm12">, DwarfRegNum<[29, -2, -2]>;
-  def XMM13: Register<"xmm13">, DwarfRegNum<[30, -2, -2]>;
-  def XMM14: Register<"xmm14">, DwarfRegNum<[31, -2, -2]>;
-  def XMM15: Register<"xmm15">, DwarfRegNum<[32, -2, -2]>;
-  } // CostPerUse
-
-  // YMM Registers, used by AVX instructions
-  let SubRegIndices = [sub_xmm] in {
-  def YMM0: RegisterWithSubRegs<"ymm0", [XMM0]>, DwarfRegAlias<XMM0>;
-  def YMM1: RegisterWithSubRegs<"ymm1", [XMM1]>, DwarfRegAlias<XMM1>;
-  def YMM2: RegisterWithSubRegs<"ymm2", [XMM2]>, DwarfRegAlias<XMM2>;
-  def YMM3: RegisterWithSubRegs<"ymm3", [XMM3]>, DwarfRegAlias<XMM3>;
-  def YMM4: RegisterWithSubRegs<"ymm4", [XMM4]>, DwarfRegAlias<XMM4>;
-  def YMM5: RegisterWithSubRegs<"ymm5", [XMM5]>, DwarfRegAlias<XMM5>;
-  def YMM6: RegisterWithSubRegs<"ymm6", [XMM6]>, DwarfRegAlias<XMM6>;
-  def YMM7: RegisterWithSubRegs<"ymm7", [XMM7]>, DwarfRegAlias<XMM7>;
-  def YMM8:  RegisterWithSubRegs<"ymm8", [XMM8]>, DwarfRegAlias<XMM8>;
-  def YMM9:  RegisterWithSubRegs<"ymm9", [XMM9]>, DwarfRegAlias<XMM9>;
-  def YMM10: RegisterWithSubRegs<"ymm10", [XMM10]>, DwarfRegAlias<XMM10>;
-  def YMM11: RegisterWithSubRegs<"ymm11", [XMM11]>, DwarfRegAlias<XMM11>;
-  def YMM12: RegisterWithSubRegs<"ymm12", [XMM12]>, DwarfRegAlias<XMM12>;
-  def YMM13: RegisterWithSubRegs<"ymm13", [XMM13]>, DwarfRegAlias<XMM13>;
-  def YMM14: RegisterWithSubRegs<"ymm14", [XMM14]>, DwarfRegAlias<XMM14>;
-  def YMM15: RegisterWithSubRegs<"ymm15", [XMM15]>, DwarfRegAlias<XMM15>;
-  }
-
-  class STRegister<string Name, list<Register> A> : Register<Name> {
-    let Aliases = A;
-  }
-
-  // Floating point stack registers. These don't map one-to-one to the FP
-  // pseudo registers, but we still mark them as aliasing FP registers. That
-  // way both kinds can be live without exceeding the stack depth. ST registers
-  // are only live around inline assembly.
-  def ST0 : STRegister<"st(0)", []>, DwarfRegNum<[33, 12, 11]>;
-  def ST1 : STRegister<"st(1)", [FP6]>, DwarfRegNum<[34, 13, 12]>;
-  def ST2 : STRegister<"st(2)", [FP5]>, DwarfRegNum<[35, 14, 13]>;
-  def ST3 : STRegister<"st(3)", [FP4]>, DwarfRegNum<[36, 15, 14]>;
-  def ST4 : STRegister<"st(4)", [FP3]>, DwarfRegNum<[37, 16, 15]>;
-  def ST5 : STRegister<"st(5)", [FP2]>, DwarfRegNum<[38, 17, 16]>;
-  def ST6 : STRegister<"st(6)", [FP1]>, DwarfRegNum<[39, 18, 17]>;
-  def ST7 : STRegister<"st(7)", [FP0]>, DwarfRegNum<[40, 19, 18]>;
-
-  // Floating-point status word
-  def FPSW : Register<"fpsw">;
-
-  // Status flags register
-  def EFLAGS : Register<"flags">;
-
-  // Segment registers
-  def CS : Register<"cs">;
-  def DS : Register<"ds">;
-  def SS : Register<"ss">;
-  def ES : Register<"es">;
-  def FS : Register<"fs">;
-  def GS : Register<"gs">;
-
-  // Debug registers
-  def DR0 : Register<"dr0">;
-  def DR1 : Register<"dr1">;
-  def DR2 : Register<"dr2">;
-  def DR3 : Register<"dr3">;
-  def DR4 : Register<"dr4">;
-  def DR5 : Register<"dr5">;
-  def DR6 : Register<"dr6">;
-  def DR7 : Register<"dr7">;
-
-  // Control registers
-  def CR0 : Register<"cr0">;
-  def CR1 : Register<"cr1">;
-  def CR2 : Register<"cr2">;
-  def CR3 : Register<"cr3">;
-  def CR4 : Register<"cr4">;
-  def CR5 : Register<"cr5">;
-  def CR6 : Register<"cr6">;
-  def CR7 : Register<"cr7">;
-  def CR8 : Register<"cr8">;
-  def CR9 : Register<"cr9">;
-  def CR10 : Register<"cr10">;
-  def CR11 : Register<"cr11">;
-  def CR12 : Register<"cr12">;
-  def CR13 : Register<"cr13">;
-  def CR14 : Register<"cr14">;
-  def CR15 : Register<"cr15">;
-
-  // Pseudo index registers
-  def EIZ : Register<"eiz">;
-  def RIZ : Register<"riz">;
+  def sub_xmm     : SubRegIndex;
 }
 
+//===----------------------------------------------------------------------===//
+//  Register definitions...
+//
+
+// In the register alias definitions below, we define which registers alias
+// which others.  We only specify which registers the small registers alias,
+// because the register file generator is smart enough to figure out that
+// AL aliases AX if we tell it that AX aliased AL (for example).
+
+// Dwarf numbering is different for 32-bit and 64-bit, and there are
+// variations by target as well. Currently the first entry is for X86-64,
+// second - for EH on X86-32/Darwin and third is 'generic' one (X86-32/Linux
+// and debug information on X86-32/Darwin)
+
+// 8-bit registers
+// Low registers
+def AL : X86Reg<"al", 0>;
+def DL : X86Reg<"dl", 2>;
+def CL : X86Reg<"cl", 1>;
+def BL : X86Reg<"bl", 3>;
+
+// High registers. On x86-64, these cannot be used in any instruction
+// with a REX prefix.
+def AH : X86Reg<"ah", 4>;
+def DH : X86Reg<"dh", 6>;
+def CH : X86Reg<"ch", 5>;
+def BH : X86Reg<"bh", 7>;
+
+// X86-64 only, requires REX.
+let CostPerUse = 1 in {
+def SIL  : X86Reg<"sil",   6>;
+def DIL  : X86Reg<"dil",   7>;
+def BPL  : X86Reg<"bpl",   5>;
+def SPL  : X86Reg<"spl",   4>;
+def R8B  : X86Reg<"r8b",   8>;
+def R9B  : X86Reg<"r9b",   9>;
+def R10B : X86Reg<"r10b", 10>;
+def R11B : X86Reg<"r11b", 11>;
+def R12B : X86Reg<"r12b", 12>;
+def R13B : X86Reg<"r13b", 13>;
+def R14B : X86Reg<"r14b", 14>;
+def R15B : X86Reg<"r15b", 15>;
+}
+
+// 16-bit registers
+let SubRegIndices = [sub_8bit, sub_8bit_hi], CoveredBySubRegs = 1 in {
+def AX : X86Reg<"ax", 0, [AL,AH]>;
+def DX : X86Reg<"dx", 2, [DL,DH]>;
+def CX : X86Reg<"cx", 1, [CL,CH]>;
+def BX : X86Reg<"bx", 3, [BL,BH]>;
+}
+let SubRegIndices = [sub_8bit] in {
+def SI : X86Reg<"si", 6, [SIL]>;
+def DI : X86Reg<"di", 7, [DIL]>;
+def BP : X86Reg<"bp", 5, [BPL]>;
+def SP : X86Reg<"sp", 4, [SPL]>;
+}
+def IP : X86Reg<"ip", 0>;
+
+// X86-64 only, requires REX.
+let SubRegIndices = [sub_8bit], CostPerUse = 1 in {
+def R8W  : X86Reg<"r8w",   8, [R8B]>;
+def R9W  : X86Reg<"r9w",   9, [R9B]>;
+def R10W : X86Reg<"r10w", 10, [R10B]>;
+def R11W : X86Reg<"r11w", 11, [R11B]>;
+def R12W : X86Reg<"r12w", 12, [R12B]>;
+def R13W : X86Reg<"r13w", 13, [R13B]>;
+def R14W : X86Reg<"r14w", 14, [R14B]>;
+def R15W : X86Reg<"r15w", 15, [R15B]>;
+}
+
+// 32-bit registers
+let SubRegIndices = [sub_16bit] in {
+def EAX : X86Reg<"eax", 0, [AX]>, DwarfRegNum<[-2, 0, 0]>;
+def EDX : X86Reg<"edx", 2, [DX]>, DwarfRegNum<[-2, 2, 2]>;
+def ECX : X86Reg<"ecx", 1, [CX]>, DwarfRegNum<[-2, 1, 1]>;
+def EBX : X86Reg<"ebx", 3, [BX]>, DwarfRegNum<[-2, 3, 3]>;
+def ESI : X86Reg<"esi", 6, [SI]>, DwarfRegNum<[-2, 6, 6]>;
+def EDI : X86Reg<"edi", 7, [DI]>, DwarfRegNum<[-2, 7, 7]>;
+def EBP : X86Reg<"ebp", 5, [BP]>, DwarfRegNum<[-2, 4, 5]>;
+def ESP : X86Reg<"esp", 4, [SP]>, DwarfRegNum<[-2, 5, 4]>;
+def EIP : X86Reg<"eip", 0, [IP]>, DwarfRegNum<[-2, 8, 8]>;
+
+// X86-64 only, requires REX
+let CostPerUse = 1 in {
+def R8D  : X86Reg<"r8d",   8, [R8W]>;
+def R9D  : X86Reg<"r9d",   9, [R9W]>;
+def R10D : X86Reg<"r10d", 10, [R10W]>;
+def R11D : X86Reg<"r11d", 11, [R11W]>;
+def R12D : X86Reg<"r12d", 12, [R12W]>;
+def R13D : X86Reg<"r13d", 13, [R13W]>;
+def R14D : X86Reg<"r14d", 14, [R14W]>;
+def R15D : X86Reg<"r15d", 15, [R15W]>;
+}}
+
+// 64-bit registers, X86-64 only
+let SubRegIndices = [sub_32bit] in {
+def RAX : X86Reg<"rax", 0, [EAX]>, DwarfRegNum<[0, -2, -2]>;
+def RDX : X86Reg<"rdx", 2, [EDX]>, DwarfRegNum<[1, -2, -2]>;
+def RCX : X86Reg<"rcx", 1, [ECX]>, DwarfRegNum<[2, -2, -2]>;
+def RBX : X86Reg<"rbx", 3, [EBX]>, DwarfRegNum<[3, -2, -2]>;
+def RSI : X86Reg<"rsi", 6, [ESI]>, DwarfRegNum<[4, -2, -2]>;
+def RDI : X86Reg<"rdi", 7, [EDI]>, DwarfRegNum<[5, -2, -2]>;
+def RBP : X86Reg<"rbp", 5, [EBP]>, DwarfRegNum<[6, -2, -2]>;
+def RSP : X86Reg<"rsp", 4, [ESP]>, DwarfRegNum<[7, -2, -2]>;
+
+// These also require REX.
+let CostPerUse = 1 in {
+def R8  : X86Reg<"r8",   8, [R8D]>,  DwarfRegNum<[ 8, -2, -2]>;
+def R9  : X86Reg<"r9",   9, [R9D]>,  DwarfRegNum<[ 9, -2, -2]>;
+def R10 : X86Reg<"r10", 10, [R10D]>, DwarfRegNum<[10, -2, -2]>;
+def R11 : X86Reg<"r11", 11, [R11D]>, DwarfRegNum<[11, -2, -2]>;
+def R12 : X86Reg<"r12", 12, [R12D]>, DwarfRegNum<[12, -2, -2]>;
+def R13 : X86Reg<"r13", 13, [R13D]>, DwarfRegNum<[13, -2, -2]>;
+def R14 : X86Reg<"r14", 14, [R14D]>, DwarfRegNum<[14, -2, -2]>;
+def R15 : X86Reg<"r15", 15, [R15D]>, DwarfRegNum<[15, -2, -2]>;
+def RIP : X86Reg<"rip",  0, [EIP]>,  DwarfRegNum<[16, -2, -2]>;
+}}
+
+// MMX Registers. These are actually aliased to ST0 .. ST7
+def MM0 : X86Reg<"mm0", 0>, DwarfRegNum<[41, 29, 29]>;
+def MM1 : X86Reg<"mm1", 1>, DwarfRegNum<[42, 30, 30]>;
+def MM2 : X86Reg<"mm2", 2>, DwarfRegNum<[43, 31, 31]>;
+def MM3 : X86Reg<"mm3", 3>, DwarfRegNum<[44, 32, 32]>;
+def MM4 : X86Reg<"mm4", 4>, DwarfRegNum<[45, 33, 33]>;
+def MM5 : X86Reg<"mm5", 5>, DwarfRegNum<[46, 34, 34]>;
+def MM6 : X86Reg<"mm6", 6>, DwarfRegNum<[47, 35, 35]>;
+def MM7 : X86Reg<"mm7", 7>, DwarfRegNum<[48, 36, 36]>;
+
+// Pseudo Floating Point registers
+def FP0 : X86Reg<"fp0", 0>;
+def FP1 : X86Reg<"fp1", 0>;
+def FP2 : X86Reg<"fp2", 0>;
+def FP3 : X86Reg<"fp3", 0>;
+def FP4 : X86Reg<"fp4", 0>;
+def FP5 : X86Reg<"fp5", 0>;
+def FP6 : X86Reg<"fp6", 0>;
+
+// XMM Registers, used by the various SSE instruction set extensions.
+def XMM0: X86Reg<"xmm0", 0>, DwarfRegNum<[17, 21, 21]>;
+def XMM1: X86Reg<"xmm1", 1>, DwarfRegNum<[18, 22, 22]>;
+def XMM2: X86Reg<"xmm2", 2>, DwarfRegNum<[19, 23, 23]>;
+def XMM3: X86Reg<"xmm3", 3>, DwarfRegNum<[20, 24, 24]>;
+def XMM4: X86Reg<"xmm4", 4>, DwarfRegNum<[21, 25, 25]>;
+def XMM5: X86Reg<"xmm5", 5>, DwarfRegNum<[22, 26, 26]>;
+def XMM6: X86Reg<"xmm6", 6>, DwarfRegNum<[23, 27, 27]>;
+def XMM7: X86Reg<"xmm7", 7>, DwarfRegNum<[24, 28, 28]>;
+
+// X86-64 only
+let CostPerUse = 1 in {
+def XMM8:  X86Reg<"xmm8",   8>, DwarfRegNum<[25, -2, -2]>;
+def XMM9:  X86Reg<"xmm9",   9>, DwarfRegNum<[26, -2, -2]>;
+def XMM10: X86Reg<"xmm10", 10>, DwarfRegNum<[27, -2, -2]>;
+def XMM11: X86Reg<"xmm11", 11>, DwarfRegNum<[28, -2, -2]>;
+def XMM12: X86Reg<"xmm12", 12>, DwarfRegNum<[29, -2, -2]>;
+def XMM13: X86Reg<"xmm13", 13>, DwarfRegNum<[30, -2, -2]>;
+def XMM14: X86Reg<"xmm14", 14>, DwarfRegNum<[31, -2, -2]>;
+def XMM15: X86Reg<"xmm15", 15>, DwarfRegNum<[32, -2, -2]>;
+} // CostPerUse
+
+// YMM Registers, used by AVX instructions
+let SubRegIndices = [sub_xmm] in {
+def YMM0:  X86Reg<"ymm0",   0, [XMM0]>,  DwarfRegAlias<XMM0>;
+def YMM1:  X86Reg<"ymm1",   1, [XMM1]>,  DwarfRegAlias<XMM1>;
+def YMM2:  X86Reg<"ymm2",   2, [XMM2]>,  DwarfRegAlias<XMM2>;
+def YMM3:  X86Reg<"ymm3",   3, [XMM3]>,  DwarfRegAlias<XMM3>;
+def YMM4:  X86Reg<"ymm4",   4, [XMM4]>,  DwarfRegAlias<XMM4>;
+def YMM5:  X86Reg<"ymm5",   5, [XMM5]>,  DwarfRegAlias<XMM5>;
+def YMM6:  X86Reg<"ymm6",   6, [XMM6]>,  DwarfRegAlias<XMM6>;
+def YMM7:  X86Reg<"ymm7",   7, [XMM7]>,  DwarfRegAlias<XMM7>;
+def YMM8:  X86Reg<"ymm8",   8, [XMM8]>,  DwarfRegAlias<XMM8>;
+def YMM9:  X86Reg<"ymm9",   9, [XMM9]>,  DwarfRegAlias<XMM9>;
+def YMM10: X86Reg<"ymm10", 10, [XMM10]>, DwarfRegAlias<XMM10>;
+def YMM11: X86Reg<"ymm11", 11, [XMM11]>, DwarfRegAlias<XMM11>;
+def YMM12: X86Reg<"ymm12", 12, [XMM12]>, DwarfRegAlias<XMM12>;
+def YMM13: X86Reg<"ymm13", 13, [XMM13]>, DwarfRegAlias<XMM13>;
+def YMM14: X86Reg<"ymm14", 14, [XMM14]>, DwarfRegAlias<XMM14>;
+def YMM15: X86Reg<"ymm15", 15, [XMM15]>, DwarfRegAlias<XMM15>;
+}
+
+class STRegister<string n, bits<16> Enc, list<Register> A> : X86Reg<n, Enc> {
+  let Aliases = A;
+}
+
+// Floating point stack registers. These don't map one-to-one to the FP
+// pseudo registers, but we still mark them as aliasing FP registers. That
+// way both kinds can be live without exceeding the stack depth. ST registers
+// are only live around inline assembly.
+def ST0 : STRegister<"st(0)", 0, []>,    DwarfRegNum<[33, 12, 11]>;
+def ST1 : STRegister<"st(1)", 1, [FP6]>, DwarfRegNum<[34, 13, 12]>;
+def ST2 : STRegister<"st(2)", 2, [FP5]>, DwarfRegNum<[35, 14, 13]>;
+def ST3 : STRegister<"st(3)", 3, [FP4]>, DwarfRegNum<[36, 15, 14]>;
+def ST4 : STRegister<"st(4)", 4, [FP3]>, DwarfRegNum<[37, 16, 15]>;
+def ST5 : STRegister<"st(5)", 5, [FP2]>, DwarfRegNum<[38, 17, 16]>;
+def ST6 : STRegister<"st(6)", 6, [FP1]>, DwarfRegNum<[39, 18, 17]>;
+def ST7 : STRegister<"st(7)", 7, [FP0]>, DwarfRegNum<[40, 19, 18]>;
+
+// Floating-point status word
+def FPSW : X86Reg<"fpsw", 0>;
+
+// Status flags register
+def EFLAGS : X86Reg<"flags", 0>;
+
+// Segment registers
+def CS : X86Reg<"cs", 1>;
+def DS : X86Reg<"ds", 3>;
+def SS : X86Reg<"ss", 2>;
+def ES : X86Reg<"es", 0>;
+def FS : X86Reg<"fs", 4>;
+def GS : X86Reg<"gs", 5>;
+
+// Debug registers
+def DR0 : X86Reg<"dr0", 0>;
+def DR1 : X86Reg<"dr1", 1>;
+def DR2 : X86Reg<"dr2", 2>;
+def DR3 : X86Reg<"dr3", 3>;
+def DR4 : X86Reg<"dr4", 4>;
+def DR5 : X86Reg<"dr5", 5>;
+def DR6 : X86Reg<"dr6", 6>;
+def DR7 : X86Reg<"dr7", 7>;
+
+// Control registers
+def CR0  : X86Reg<"cr0",   0>;
+def CR1  : X86Reg<"cr1",   1>;
+def CR2  : X86Reg<"cr2",   2>;
+def CR3  : X86Reg<"cr3",   3>;
+def CR4  : X86Reg<"cr4",   4>;
+def CR5  : X86Reg<"cr5",   5>;
+def CR6  : X86Reg<"cr6",   6>;
+def CR7  : X86Reg<"cr7",   7>;
+def CR8  : X86Reg<"cr8",   8>;
+def CR9  : X86Reg<"cr9",   9>;
+def CR10 : X86Reg<"cr10", 10>;
+def CR11 : X86Reg<"cr11", 11>;
+def CR12 : X86Reg<"cr12", 12>;
+def CR13 : X86Reg<"cr13", 13>;
+def CR14 : X86Reg<"cr14", 14>;
+def CR15 : X86Reg<"cr15", 15>;
+
+// Pseudo index registers
+def EIZ : X86Reg<"eiz", 4>;
+def RIZ : X86Reg<"riz", 4>;
+
 
 //===----------------------------------------------------------------------===//
 // Register Class Definitions... now that we have all of the pieces, define the
diff --git a/lib/Target/X86/X86SelectionDAGInfo.cpp b/lib/Target/X86/X86SelectionDAGInfo.cpp
index 00edcbc..723e50c 100644
--- a/lib/Target/X86/X86SelectionDAGInfo.cpp
+++ b/lib/Target/X86/X86SelectionDAGInfo.cpp
@@ -54,7 +54,7 @@ X86SelectionDAGInfo::EmitTargetCodeForMemset(SelectionDAG &DAG, DebugLoc dl,
     if (const char *bzeroEntry =  V &&
         V->isNullValue() ? Subtarget->getBZeroEntry() : 0) {
       EVT IntPtr = TLI.getPointerTy();
-      Type *IntPtrTy = getTargetData()->getIntPtrType(*DAG.getContext());
+      Type *IntPtrTy = getDataLayout()->getIntPtrType(*DAG.getContext());
       TargetLowering::ArgListTy Args;
       TargetLowering::ArgListEntry Entry;
       Entry.Node = Dst;
diff --git a/lib/Target/X86/X86Subtarget.cpp b/lib/Target/X86/X86Subtarget.cpp
index 9087852..d1ed680 100644
--- a/lib/Target/X86/X86Subtarget.cpp
+++ b/lib/Target/X86/X86Subtarget.cpp
@@ -163,17 +163,6 @@ bool X86Subtarget::IsLegalToCallImmediateAddr(const TargetMachine &TM) const {
   return isTargetELF() || TM.getRelocationModel() == Reloc::Static;
 }
 
-/// getSpecialAddressLatency - For targets where it is beneficial to
-/// backschedule instructions that compute addresses, return a value
-/// indicating the number of scheduling cycles of backscheduling that
-/// should be attempted.
-unsigned X86Subtarget::getSpecialAddressLatency() const {
-  // For x86 out-of-order targets, back-schedule address computations so
-  // that loads and stores aren't blocked.
-  // This value was chosen arbitrarily.
-  return 200;
-}
-
 void X86Subtarget::AutoDetectSubtargetFeatures() {
   unsigned EAX = 0, EBX = 0, ECX = 0, EDX = 0;
   unsigned MaxLevel;
@@ -313,6 +302,10 @@ void X86Subtarget::AutoDetectSubtargetFeatures() {
         HasBMI2 = true;
         ToggleFeature(X86::FeatureBMI2);
       }
+      if (IsIntel && ((EBX >> 11) & 0x1)) {
+        HasRTM = true;
+        ToggleFeature(X86::FeatureRTM);
+      }
     }
   }
 }
@@ -341,11 +334,13 @@ X86Subtarget::X86Subtarget(const std::string &TT, const std::string &CPU,
   , HasLZCNT(false)
   , HasBMI(false)
   , HasBMI2(false)
+  , HasRTM(false)
   , IsBTMemSlow(false)
   , IsUAMemFast(false)
   , HasVectorUAMem(false)
   , HasCmpxchg16b(false)
   , UseLeaForSP(false)
+  , HasSlowDivide(false)
   , PostRAScheduler(false)
   , stackAlignment(4)
   // FIXME: this is a known good value for Yonah. How about others?
@@ -400,6 +395,10 @@ X86Subtarget::X86Subtarget(const std::string &TT, const std::string &CPU,
     }
   }
 
+  // CPUName may have been set by the CPU detection code. Make sure the
+  // new MCSchedModel is used.
+  InitMCProcessorInfo(CPUName, FS);
+
   if (X86ProcFamily == IntelAtom)
     PostRAScheduler = true;
 
@@ -416,12 +415,12 @@ X86Subtarget::X86Subtarget(const std::string &TT, const std::string &CPU,
   assert((!In64BitMode || HasX86_64) &&
          "64-bit code requested on a subtarget that doesn't support it!");
 
-  // Stack alignment is 16 bytes on Darwin, FreeBSD, Linux and Solaris (both
+  // Stack alignment is 16 bytes on Darwin, Linux and Solaris (both
   // 32 and 64 bit) and for all 64-bit targets.
   if (StackAlignOverride)
     stackAlignment = StackAlignOverride;
-  else if (isTargetDarwin() || isTargetFreeBSD() || isTargetLinux() ||
-           isTargetSolaris() || In64BitMode)
+  else if (isTargetDarwin() || isTargetLinux() || isTargetSolaris() ||
+           In64BitMode)
     stackAlignment = 16;
 }
 
diff --git a/lib/Target/X86/X86Subtarget.h b/lib/Target/X86/X86Subtarget.h
index 6841c5b..8bf4cc7 100644
--- a/lib/Target/X86/X86Subtarget.h
+++ b/lib/Target/X86/X86Subtarget.h
@@ -118,6 +118,9 @@ protected:
   /// HasBMI2 - Processor has BMI2 instructions.
   bool HasBMI2;
 
+  /// HasRTM - Processor has RTM instructions.
+  bool HasRTM;
+
   /// IsBTMemSlow - True if BT (bit test) of memory instructions are slow.
   bool IsBTMemSlow;
 
@@ -136,6 +139,10 @@ protected:
   /// the stack pointer. This is an optimization for Intel Atom processors.
   bool UseLeaForSP;
 
+  /// HasSlowDivide - True if smaller divides are significantly faster than
+  /// full divides and should be used when possible.
+  bool HasSlowDivide;
+
   /// PostRAScheduler - True if using post-register-allocation scheduler.
   bool PostRAScheduler;
 
@@ -205,7 +212,8 @@ public:
   bool hasAES() const { return HasAES; }
   bool hasPCLMUL() const { return HasPCLMUL; }
   bool hasFMA() const { return HasFMA; }
-  bool hasFMA4() const { return HasFMA4; }
+  // FIXME: Favor FMA when both are enabled. Is this the right thing to do?
+  bool hasFMA4() const { return HasFMA4 && !HasFMA; }
   bool hasXOP() const { return HasXOP; }
   bool hasMOVBE() const { return HasMOVBE; }
   bool hasRDRAND() const { return HasRDRAND; }
@@ -214,11 +222,13 @@ public:
   bool hasLZCNT() const { return HasLZCNT; }
   bool hasBMI() const { return HasBMI; }
   bool hasBMI2() const { return HasBMI2; }
+  bool hasRTM() const { return HasRTM; }
   bool isBTMemSlow() const { return IsBTMemSlow; }
   bool isUnalignedMemAccessFast() const { return IsUAMemFast; }
   bool hasVectorUAMem() const { return HasVectorUAMem; }
   bool hasCmpxchg16b() const { return HasCmpxchg16b; }
   bool useLeaForSP() const { return UseLeaForSP; }
+  bool hasSlowDivide() const { return HasSlowDivide; }
 
   bool isAtom() const { return X86ProcFamily == IntelAtom; }
 
@@ -231,10 +241,10 @@ public:
   bool isTargetSolaris() const {
     return TargetTriple.getOS() == Triple::Solaris;
   }
-
-  // ELF is a reasonably sane default and the only other X86 targets we
-  // support are Darwin and Windows. Just use "not those".
-  bool isTargetELF() const { return TargetTriple.isOSBinFormatELF(); }
+  bool isTargetELF() const {
+    return (TargetTriple.getEnvironment() == Triple::ELF ||
+            TargetTriple.isOSBinFormatELF());
+  }
   bool isTargetLinux() const { return TargetTriple.getOS() == Triple::Linux; }
   bool isTargetNaCl() const {
     return TargetTriple.getOS() == Triple::NativeClient;
@@ -245,7 +255,10 @@ public:
   bool isTargetMingw() const { return TargetTriple.getOS() == Triple::MinGW32; }
   bool isTargetCygwin() const { return TargetTriple.getOS() == Triple::Cygwin; }
   bool isTargetCygMing() const { return TargetTriple.isOSCygMing(); }
-  bool isTargetCOFF() const { return TargetTriple.isOSBinFormatCOFF(); }
+  bool isTargetCOFF() const {
+    return (TargetTriple.getEnvironment() != Triple::ELF &&
+            TargetTriple.isOSBinFormatCOFF());
+  }
   bool isTargetEnvMacho() const { return TargetTriple.isEnvironmentMachO(); }
 
   bool isTargetWin64() const {
@@ -296,12 +309,6 @@ public:
   /// returns null.
   const char *getBZeroEntry() const;
 
-  /// getSpecialAddressLatency - For targets where it is beneficial to
-  /// backschedule instructions that compute addresses, return a value
-  /// indicating the number of scheduling cycles of backscheduling that
-  /// should be attempted.
-  unsigned getSpecialAddressLatency() const;
-
   /// enablePostRAScheduler - run for Atom optimization.
   bool enablePostRAScheduler(CodeGenOpt::Level OptLevel,
                              TargetSubtargetInfo::AntiDepBreakMode& Mode,
diff --git a/lib/Target/X86/X86TargetMachine.cpp b/lib/Target/X86/X86TargetMachine.cpp
index b7ba568..158f9dc 100644
--- a/lib/Target/X86/X86TargetMachine.cpp
+++ b/lib/Target/X86/X86TargetMachine.cpp
@@ -36,7 +36,7 @@ X86_32TargetMachine::X86_32TargetMachine(const Target &T, StringRef TT,
                                          Reloc::Model RM, CodeModel::Model CM,
                                          CodeGenOpt::Level OL)
   : X86TargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, false),
-    DataLayout(getSubtargetImpl()->isTargetDarwin() ?
+    DL(getSubtargetImpl()->isTargetDarwin() ?
                "e-p:32:32-f64:32:64-i64:32:64-f80:128:128-f128:128:128-"
                "n8:16:32-S128" :
                (getSubtargetImpl()->isTargetCygMing() ||
@@ -48,7 +48,8 @@ X86_32TargetMachine::X86_32TargetMachine(const Target &T, StringRef TT,
     InstrInfo(*this),
     TSInfo(*this),
     TLInfo(*this),
-    JITInfo(*this) {
+    JITInfo(*this),
+    STTI(&TLInfo), VTTI(&TLInfo) {
 }
 
 void X86_64TargetMachine::anchor() { }
@@ -59,12 +60,13 @@ X86_64TargetMachine::X86_64TargetMachine(const Target &T, StringRef TT,
                                          Reloc::Model RM, CodeModel::Model CM,
                                          CodeGenOpt::Level OL)
   : X86TargetMachine(T, TT, CPU, FS, Options, RM, CM, OL, true),
-    DataLayout("e-p:64:64-s:64-f64:64:64-i64:64:64-f80:128:128-f128:128:128-"
+    DL("e-p:64:64-s:64-f64:64:64-i64:64:64-f80:128:128-f128:128:128-"
                "n8:16:32:64-S128"),
     InstrInfo(*this),
     TSInfo(*this),
     TLInfo(*this),
-    JITInfo(*this) {
+    JITInfo(*this),
+    STTI(&TLInfo), VTTI(&TLInfo){
 }
 
 /// X86TargetMachine ctor - Create an X86 target.
@@ -78,7 +80,6 @@ X86TargetMachine::X86TargetMachine(const Target &T, StringRef TT,
   : LLVMTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL),
     Subtarget(TT, CPU, FS, Options.StackAlignmentOverride, is64Bit),
     FrameLowering(*this, Subtarget),
-    ELFWriterInfo(is64Bit, true),
     InstrItins(Subtarget.getInstrItineraryData()){
   // Determine the PICStyle based on the target selected.
   if (getRelocationModel() == Reloc::Static) {
@@ -113,6 +114,12 @@ UseVZeroUpper("x86-use-vzeroupper",
   cl::desc("Minimize AVX to SSE transition penalty"),
   cl::init(true));
 
+// Temporary option to control early if-conversion for x86 while adding machine
+// models.
+static cl::opt<bool>
+X86EarlyIfConv("x86-early-ifcvt",
+	       cl::desc("Enable early if-conversion on X86"));
+
 //===----------------------------------------------------------------------===//
 // Pass Pipeline Configuration
 //===----------------------------------------------------------------------===//
@@ -142,7 +149,7 @@ public:
 TargetPassConfig *X86TargetMachine::createPassConfig(PassManagerBase &PM) {
   X86PassConfig *PC = new X86PassConfig(this, PM);
 
-  if (Subtarget.hasCMov())
+  if (X86EarlyIfConv && Subtarget.hasCMov())
     PC->enablePass(&EarlyIfConverterID);
 
   return PC;
diff --git a/lib/Target/X86/X86TargetMachine.h b/lib/Target/X86/X86TargetMachine.h
index 8e935af..12311a1 100644
--- a/lib/Target/X86/X86TargetMachine.h
+++ b/lib/Target/X86/X86TargetMachine.h
@@ -15,7 +15,6 @@
 #define X86TARGETMACHINE_H
 
 #include "X86.h"
-#include "X86ELFWriterInfo.h"
 #include "X86InstrInfo.h"
 #include "X86ISelLowering.h"
 #include "X86FrameLowering.h"
@@ -23,8 +22,9 @@
 #include "X86SelectionDAGInfo.h"
 #include "X86Subtarget.h"
 #include "llvm/Target/TargetMachine.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetFrameLowering.h"
+#include "llvm/Target/TargetTransformImpl.h"
 
 namespace llvm {
 
@@ -33,7 +33,6 @@ class StringRef;
 class X86TargetMachine : public LLVMTargetMachine {
   X86Subtarget       Subtarget;
   X86FrameLowering   FrameLowering;
-  X86ELFWriterInfo   ELFWriterInfo;
   InstrItineraryData InstrItins;
 
 public:
@@ -62,9 +61,6 @@ public:
   virtual const X86RegisterInfo  *getRegisterInfo() const {
     return &getInstrInfo()->getRegisterInfo();
   }
-  virtual const X86ELFWriterInfo *getELFWriterInfo() const {
-    return Subtarget.isTargetELF() ? &ELFWriterInfo : 0;
-  }
   virtual const InstrItineraryData *getInstrItineraryData() const {
     return &InstrItins;
   }
@@ -80,17 +76,19 @@ public:
 ///
 class X86_32TargetMachine : public X86TargetMachine {
   virtual void anchor();
-  const TargetData  DataLayout; // Calculates type size & alignment
+  const DataLayout  DL; // Calculates type size & alignment
   X86InstrInfo      InstrInfo;
   X86SelectionDAGInfo TSInfo;
   X86TargetLowering TLInfo;
   X86JITInfo        JITInfo;
+  ScalarTargetTransformImpl STTI;
+  X86VectorTargetTransformInfo VTTI;
 public:
   X86_32TargetMachine(const Target &T, StringRef TT,
                       StringRef CPU, StringRef FS, const TargetOptions &Options,
                       Reloc::Model RM, CodeModel::Model CM,
                       CodeGenOpt::Level OL);
-  virtual const TargetData *getTargetData() const { return &DataLayout; }
+  virtual const DataLayout *getDataLayout() const { return &DL; }
   virtual const X86TargetLowering *getTargetLowering() const {
     return &TLInfo;
   }
@@ -103,23 +101,31 @@ public:
   virtual       X86JITInfo       *getJITInfo()         {
     return &JITInfo;
   }
+  virtual const ScalarTargetTransformInfo *getScalarTargetTransformInfo()const {
+    return &STTI;
+  }
+  virtual const VectorTargetTransformInfo *getVectorTargetTransformInfo()const {
+    return &VTTI;
+  }
 };
 
 /// X86_64TargetMachine - X86 64-bit target machine.
 ///
 class X86_64TargetMachine : public X86TargetMachine {
   virtual void anchor();
-  const TargetData  DataLayout; // Calculates type size & alignment
+  const DataLayout  DL; // Calculates type size & alignment
   X86InstrInfo      InstrInfo;
   X86SelectionDAGInfo TSInfo;
   X86TargetLowering TLInfo;
   X86JITInfo        JITInfo;
+  ScalarTargetTransformImpl STTI;
+  X86VectorTargetTransformInfo VTTI;
 public:
   X86_64TargetMachine(const Target &T, StringRef TT,
                       StringRef CPU, StringRef FS, const TargetOptions &Options,
                       Reloc::Model RM, CodeModel::Model CM,
                       CodeGenOpt::Level OL);
-  virtual const TargetData *getTargetData() const { return &DataLayout; }
+  virtual const DataLayout *getDataLayout() const { return &DL; }
   virtual const X86TargetLowering *getTargetLowering() const {
     return &TLInfo;
   }
@@ -132,6 +138,12 @@ public:
   virtual       X86JITInfo       *getJITInfo()         {
     return &JITInfo;
   }
+  virtual const ScalarTargetTransformInfo *getScalarTargetTransformInfo()const {
+    return &STTI;
+  }
+  virtual const VectorTargetTransformInfo *getVectorTargetTransformInfo()const {
+    return &VTTI;
+  }
 };
 
 } // End llvm namespace
diff --git a/lib/Target/X86/X86VZeroUpper.cpp b/lib/Target/X86/X86VZeroUpper.cpp
index 80b75dc..c4a5887 100644
--- a/lib/Target/X86/X86VZeroUpper.cpp
+++ b/lib/Target/X86/X86VZeroUpper.cpp
@@ -42,7 +42,6 @@ namespace {
 
   private:
     const TargetInstrInfo *TII; // Machine instruction info.
-    MachineBasicBlock *MBB;     // Current basic block
 
     // Any YMM register live-in to this function?
     bool FnHasLiveInYmm;
@@ -84,7 +83,7 @@ namespace {
     //  2) All states must be clean for the result to be clean
     //  3) If none above and one unknown, the result state is also unknown
     //
-    unsigned computeState(unsigned PrevState, unsigned CurState) {
+    static unsigned computeState(unsigned PrevState, unsigned CurState) {
       if (PrevState == ST_INIT)
         return CurState;
 
@@ -122,7 +121,7 @@ static bool checkFnHasLiveInYmm(MachineRegisterInfo &MRI) {
 }
 
 static bool hasYmmReg(MachineInstr *MI) {
-  for (int i = 0, e = MI->getNumOperands(); i != e; ++i) {
+  for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
     const MachineOperand &MO = MI->getOperand(i);
     if (!MO.isReg())
       continue;
@@ -148,7 +147,7 @@ bool VZeroUpperInserter::runOnMachineFunction(MachineFunction &MF) {
   const TargetRegisterClass *RC = &X86::VR256RegClass;
   for (TargetRegisterClass::iterator i = RC->begin(), e = RC->end();
        i != e; i++) {
-    if (MRI.isPhysRegUsed(*i)) {
+    if (!MRI.reg_nodbg_empty(*i)) {
       YMMUsed = true;
       break;
     }
@@ -189,7 +188,6 @@ bool VZeroUpperInserter::processBasicBlock(MachineFunction &MF,
                                            MachineBasicBlock &BB) {
   bool Changed = false;
   unsigned BBNum = BB.getNumber();
-  MBB = &BB;
 
   // Don't process already solved BBs
   if (BBSolved[BBNum])
@@ -207,7 +205,7 @@ bool VZeroUpperInserter::processBasicBlock(MachineFunction &MF,
 
   // The entry MBB for the function may set the initial state to dirty if
   // the function receives any YMM incoming arguments
-  if (MBB == MF.begin()) {
+  if (&BB == MF.begin()) {
     EntryState = ST_CLEAN;
     if (FnHasLiveInYmm)
       EntryState = ST_DIRTY;
@@ -253,7 +251,7 @@ bool VZeroUpperInserter::processBasicBlock(MachineFunction &MF,
       // When unknown, only compute the information within the block to have
       // it available in the exit if possible, but don't change the block.
       if (EntryState != ST_UNKNOWN) {
-        BuildMI(*MBB, I, dl, TII->get(X86::VZEROUPPER));
+        BuildMI(BB, I, dl, TII->get(X86::VZEROUPPER));
         ++NumVZU;
       }
 
diff --git a/lib/Target/XCore/XCoreAsmPrinter.cpp b/lib/Target/XCore/XCoreAsmPrinter.cpp
index c76866f..caae562 100644
--- a/lib/Target/XCore/XCoreAsmPrinter.cpp
+++ b/lib/Target/XCore/XCoreAsmPrinter.cpp
@@ -31,7 +31,7 @@
 #include "llvm/MC/MCStreamer.h"
 #include "llvm/MC/MCSymbol.h"
 #include "llvm/Target/Mangler.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetLoweringObjectFile.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringExtras.h"
@@ -112,7 +112,7 @@ void XCoreAsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) {
       EmitSpecialLLVMGlobal(GV))
     return;
 
-  const TargetData *TD = TM.getTargetData();
+  const DataLayout *TD = TM.getDataLayout();
   OutStreamer.SwitchSection(getObjFileLowering().SectionForGlobal(GV, Mang,TM));
 
   
diff --git a/lib/Target/XCore/XCoreFrameLowering.cpp b/lib/Target/XCore/XCoreFrameLowering.cpp
index a4e5647..e18d973 100644
--- a/lib/Target/XCore/XCoreFrameLowering.cpp
+++ b/lib/Target/XCore/XCoreFrameLowering.cpp
@@ -23,7 +23,7 @@
 #include "llvm/CodeGen/MachineModuleInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/RegisterScavenging.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetOptions.h"
 #include "llvm/Support/ErrorHandling.h"
 
@@ -98,12 +98,13 @@ void XCoreFrameLowering::emitPrologue(MachineFunction &MF) const {
   DebugLoc dl = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc();
 
   bool FP = hasFP(MF);
-  bool Nested = MF.getFunction()->
-                getAttributes().hasAttrSomewhere(Attribute::Nest);
+  const AttrListPtr &PAL = MF.getFunction()->getAttributes();
 
-  if (Nested) {
-    loadFromStack(MBB, MBBI, XCore::R11, 0, dl, TII);
-  }
+  for (unsigned I = 0, E = PAL.getNumAttrs(); I != E; ++I)
+    if (PAL.getAttributesAtIndex(I).hasAttribute(Attributes::Nest)) {
+      loadFromStack(MBB, MBBI, XCore::R11, 0, dl, TII);
+      break;
+    }
 
   // Work out frame sizes.
   int FrameSize = MFI->getStackSize();
diff --git a/lib/Target/XCore/XCoreISelLowering.cpp b/lib/Target/XCore/XCoreISelLowering.cpp
index 8643ffc..9e7816e 100644
--- a/lib/Target/XCore/XCoreISelLowering.cpp
+++ b/lib/Target/XCore/XCoreISelLowering.cpp
@@ -285,7 +285,7 @@ LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const
     llvm_unreachable(0);
   }
   SDValue base = getGlobalAddressWrapper(GA, GV, DAG);
-  const TargetData *TD = TM.getTargetData();
+  const DataLayout *TD = TM.getDataLayout();
   unsigned Size = TD->getTypeAllocSize(Ty);
   SDValue offset = DAG.getNode(ISD::MUL, dl, MVT::i32, BuildGetId(DAG, dl),
                        DAG.getConstant(Size, MVT::i32));
@@ -298,7 +298,7 @@ LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const
   DebugLoc DL = Op.getDebugLoc();
 
   const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress();
-  SDValue Result = DAG.getBlockAddress(BA, getPointerTy(), /*isTarget=*/true);
+  SDValue Result = DAG.getTargetBlockAddress(BA, getPointerTy());
 
   return DAG.getNode(XCoreISD::PCRelativeWrapper, DL, getPointerTy(), Result);
 }
@@ -405,7 +405,7 @@ LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
   if (allowsUnalignedMemoryAccesses(LD->getMemoryVT()))
     return SDValue();
 
-  unsigned ABIAlignment = getTargetData()->
+  unsigned ABIAlignment = getDataLayout()->
     getABITypeAlignment(LD->getMemoryVT().getTypeForEVT(*DAG.getContext()));
   // Leave aligned load alone.
   if (LD->getAlignment() >= ABIAlignment)
@@ -477,7 +477,7 @@ LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
   }
 
   // Lower to a call to __misaligned_load(BasePtr).
-  Type *IntPtrTy = getTargetData()->getIntPtrType(*DAG.getContext());
+  Type *IntPtrTy = getDataLayout()->getIntPtrType(*DAG.getContext());
   TargetLowering::ArgListTy Args;
   TargetLowering::ArgListEntry Entry;
 
@@ -507,7 +507,7 @@ LowerSTORE(SDValue Op, SelectionDAG &DAG) const
   if (allowsUnalignedMemoryAccesses(ST->getMemoryVT())) {
     return SDValue();
   }
-  unsigned ABIAlignment = getTargetData()->
+  unsigned ABIAlignment = getDataLayout()->
     getABITypeAlignment(ST->getMemoryVT().getTypeForEVT(*DAG.getContext()));
   // Leave aligned store alone.
   if (ST->getAlignment() >= ABIAlignment) {
@@ -536,7 +536,7 @@ LowerSTORE(SDValue Op, SelectionDAG &DAG) const
   }
 
   // Lower to a call to __misaligned_store(BasePtr, Value).
-  Type *IntPtrTy = getTargetData()->getIntPtrType(*DAG.getContext());
+  Type *IntPtrTy = getDataLayout()->getIntPtrType(*DAG.getContext());
   TargetLowering::ArgListTy Args;
   TargetLowering::ArgListEntry Entry;
 
@@ -1499,7 +1499,7 @@ SDValue XCoreTargetLowering::PerformDAGCombine(SDNode *N,
     if (StoreBits % 8) {
       break;
     }
-    unsigned ABIAlignment = getTargetData()->getABITypeAlignment(
+    unsigned ABIAlignment = getDataLayout()->getABITypeAlignment(
         ST->getMemoryVT().getTypeForEVT(*DCI.DAG.getContext()));
     unsigned Alignment = ST->getAlignment();
     if (Alignment >= ABIAlignment) {
@@ -1570,7 +1570,7 @@ XCoreTargetLowering::isLegalAddressingMode(const AddrMode &AM,
   if (Ty->getTypeID() == Type::VoidTyID)
     return AM.Scale == 0 && isImmUs(AM.BaseOffs) && isImmUs4(AM.BaseOffs);
 
-  const TargetData *TD = TM.getTargetData();
+  const DataLayout *TD = TM.getDataLayout();
   unsigned Size = TD->getTypeAllocSize(Ty);
   if (AM.BaseGV) {
     return Size >= 4 && !AM.HasBaseReg && AM.Scale == 0 &&
diff --git a/lib/Target/XCore/XCoreInstrInfo.td b/lib/Target/XCore/XCoreInstrInfo.td
index ae646a2..3e7666b 100644
--- a/lib/Target/XCore/XCoreInstrInfo.td
+++ b/lib/Target/XCore/XCoreInstrInfo.td
@@ -33,7 +33,7 @@ def XCoreBranchLink     : SDNode<"XCoreISD::BL",SDT_XCoreBranchLink,
                              SDNPVariadic]>;
 
 def XCoreRetsp       : SDNode<"XCoreISD::RETSP", SDTBrind,
-                         [SDNPHasChain, SDNPOptInGlue]>;
+                         [SDNPHasChain, SDNPOptInGlue, SDNPMayLoad]>;
 
 def SDT_XCoreBR_JT    : SDTypeProfile<0, 2,
                                       [SDTCisVT<0, i32>, SDTCisVT<1, i32>]>;
@@ -58,7 +58,7 @@ def cprelwrapper : SDNode<"XCoreISD::CPRelativeWrapper", SDT_XCoreAddress,
 
 def SDT_XCoreStwsp    : SDTypeProfile<0, 2, [SDTCisInt<1>]>;
 def XCoreStwsp        : SDNode<"XCoreISD::STWSP", SDT_XCoreStwsp,
-                               [SDNPHasChain]>;
+                               [SDNPHasChain, SDNPMayStore]>;
 
 // These are target-independent nodes, but have target-specific formats.
 def SDT_XCoreCallSeqStart : SDCallSeqStart<[ SDTCisVT<0, i32> ]>;
diff --git a/lib/Target/XCore/XCoreRegisterInfo.cpp b/lib/Target/XCore/XCoreRegisterInfo.cpp
index cdd0a08..be5855a 100644
--- a/lib/Target/XCore/XCoreRegisterInfo.cpp
+++ b/lib/Target/XCore/XCoreRegisterInfo.cpp
@@ -176,7 +176,7 @@ XCoreRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator II,
 
   #ifndef NDEBUG
   DEBUG(errs() << "\nFunction         : " 
-        << MF.getFunction()->getName() << "\n");
+        << MF.getName() << "\n");
   DEBUG(errs() << "<--------->\n");
   DEBUG(MI.print(errs()));
   DEBUG(errs() << "FrameIndex         : " << FrameIndex << "\n");
diff --git a/lib/Target/XCore/XCoreTargetMachine.cpp b/lib/Target/XCore/XCoreTargetMachine.cpp
index 11ec86b..d5a932c 100644
--- a/lib/Target/XCore/XCoreTargetMachine.cpp
+++ b/lib/Target/XCore/XCoreTargetMachine.cpp
@@ -27,12 +27,12 @@ XCoreTargetMachine::XCoreTargetMachine(const Target &T, StringRef TT,
                                        CodeGenOpt::Level OL)
   : LLVMTargetMachine(T, TT, CPU, FS, Options, RM, CM, OL),
     Subtarget(TT, CPU, FS),
-    DataLayout("e-p:32:32:32-a0:0:32-f32:32:32-f64:32:32-i1:8:32-i8:8:32-"
+    DL("e-p:32:32:32-a0:0:32-f32:32:32-f64:32:32-i1:8:32-i8:8:32-"
                "i16:16:32-i32:32:32-i64:32:32-n32"),
     InstrInfo(),
     FrameLowering(Subtarget),
     TLInfo(*this),
-    TSInfo(*this) {
+    TSInfo(*this), STTI(&TLInfo), VTTI(&TLInfo) {
 }
 
 namespace {
diff --git a/lib/Target/XCore/XCoreTargetMachine.h b/lib/Target/XCore/XCoreTargetMachine.h
index 2546681..c60c6a3 100644
--- a/lib/Target/XCore/XCoreTargetMachine.h
+++ b/lib/Target/XCore/XCoreTargetMachine.h
@@ -20,17 +20,20 @@
 #include "XCoreISelLowering.h"
 #include "XCoreSelectionDAGInfo.h"
 #include "llvm/Target/TargetMachine.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/Target/TargetTransformImpl.h"
+#include "llvm/DataLayout.h"
 
 namespace llvm {
 
 class XCoreTargetMachine : public LLVMTargetMachine {
   XCoreSubtarget Subtarget;
-  const TargetData DataLayout;       // Calculates type size & alignment
+  const DataLayout DL;       // Calculates type size & alignment
   XCoreInstrInfo InstrInfo;
   XCoreFrameLowering FrameLowering;
   XCoreTargetLowering TLInfo;
   XCoreSelectionDAGInfo TSInfo;
+  ScalarTargetTransformImpl STTI;
+  VectorTargetTransformImpl VTTI;
 public:
   XCoreTargetMachine(const Target &T, StringRef TT,
                      StringRef CPU, StringRef FS, const TargetOptions &Options,
@@ -53,7 +56,13 @@ public:
   virtual const TargetRegisterInfo *getRegisterInfo() const {
     return &InstrInfo.getRegisterInfo();
   }
-  virtual const TargetData       *getTargetData() const { return &DataLayout; }
+  virtual const ScalarTargetTransformInfo *getScalarTargetTransformInfo()const {
+    return &STTI;
+  }
+  virtual const VectorTargetTransformInfo *getVectorTargetTransformInfo()const {
+    return &VTTI;
+  }
+  virtual const DataLayout       *getDataLayout() const { return &DL; }
 
   // Pass Pipeline Configuration
   virtual TargetPassConfig *createPassConfig(PassManagerBase &PM);
diff --git a/lib/Transforms/IPO/ArgumentPromotion.cpp b/lib/Transforms/IPO/ArgumentPromotion.cpp
index b94dd69..be48b20 100644
--- a/lib/Transforms/IPO/ArgumentPromotion.cpp
+++ b/lib/Transforms/IPO/ArgumentPromotion.cpp
@@ -153,7 +153,8 @@ CallGraphNode *ArgPromotion::PromoteArguments(CallGraphNode *CGN) {
   SmallPtrSet<Argument*, 8> ArgsToPromote;
   SmallPtrSet<Argument*, 8> ByValArgsToTransform;
   for (unsigned i = 0; i != PointerArgs.size(); ++i) {
-    bool isByVal = F->paramHasAttr(PointerArgs[i].second+1, Attribute::ByVal);
+    bool isByVal=F->getParamAttributes(PointerArgs[i].second+1).
+      hasAttribute(Attributes::ByVal);
     Argument *PtrArg = PointerArgs[i].first;
     Type *AgTy = cast<PointerType>(PtrArg->getType())->getElementType();
 
@@ -517,8 +518,10 @@ CallGraphNode *ArgPromotion::DoPromotion(Function *F,
   const AttrListPtr &PAL = F->getAttributes();
 
   // Add any return attributes.
-  if (Attributes attrs = PAL.getRetAttributes())
-    AttributesVec.push_back(AttributeWithIndex::get(0, attrs));
+  Attributes attrs = PAL.getRetAttributes();
+  if (attrs.hasAttributes())
+    AttributesVec.push_back(AttributeWithIndex::get(AttrListPtr::ReturnIndex,
+                                                    attrs));
 
   // First, determine the new argument list
   unsigned ArgIndex = 1;
@@ -534,7 +537,8 @@ CallGraphNode *ArgPromotion::DoPromotion(Function *F,
     } else if (!ArgsToPromote.count(I)) {
       // Unchanged argument
       Params.push_back(I->getType());
-      if (Attributes attrs = PAL.getParamAttributes(ArgIndex))
+      Attributes attrs = PAL.getParamAttributes(ArgIndex);
+      if (attrs.hasAttributes())
         AttributesVec.push_back(AttributeWithIndex::get(Params.size(), attrs));
     } else if (I->use_empty()) {
       // Dead argument (which are always marked as promotable)
@@ -587,19 +591,13 @@ CallGraphNode *ArgPromotion::DoPromotion(Function *F,
   }
 
   // Add any function attributes.
-  if (Attributes attrs = PAL.getFnAttributes())
-    AttributesVec.push_back(AttributeWithIndex::get(~0, attrs));
+  attrs = PAL.getFnAttributes();
+  if (attrs.hasAttributes())
+    AttributesVec.push_back(AttributeWithIndex::get(AttrListPtr::FunctionIndex,
+                                                    attrs));
 
   Type *RetTy = FTy->getReturnType();
 
-  // Work around LLVM bug PR56: the CWriter cannot emit varargs functions which
-  // have zero fixed arguments.
-  bool ExtraArgHack = false;
-  if (Params.empty() && FTy->isVarArg()) {
-    ExtraArgHack = true;
-    Params.push_back(Type::getInt32Ty(F->getContext()));
-  }
-
   // Construct the new function type using the new arguments.
   FunctionType *NFTy = FunctionType::get(RetTy, Params, FTy->isVarArg());
 
@@ -613,7 +611,7 @@ CallGraphNode *ArgPromotion::DoPromotion(Function *F,
   
   // Recompute the parameter attributes list based on the new arguments for
   // the function.
-  NF->setAttributes(AttrListPtr::get(AttributesVec));
+  NF->setAttributes(AttrListPtr::get(F->getContext(), AttributesVec));
   AttributesVec.clear();
 
   F->getParent()->getFunctionList().insert(F, NF);
@@ -641,8 +639,10 @@ CallGraphNode *ArgPromotion::DoPromotion(Function *F,
     const AttrListPtr &CallPAL = CS.getAttributes();
 
     // Add any return attributes.
-    if (Attributes attrs = CallPAL.getRetAttributes())
-      AttributesVec.push_back(AttributeWithIndex::get(0, attrs));
+    Attributes attrs = CallPAL.getRetAttributes();
+    if (attrs.hasAttributes())
+      AttributesVec.push_back(AttributeWithIndex::get(AttrListPtr::ReturnIndex,
+                                                      attrs));
 
     // Loop over the operands, inserting GEP and loads in the caller as
     // appropriate.
@@ -653,7 +653,8 @@ CallGraphNode *ArgPromotion::DoPromotion(Function *F,
       if (!ArgsToPromote.count(I) && !ByValArgsToTransform.count(I)) {
         Args.push_back(*AI);          // Unmodified argument
 
-        if (Attributes Attrs = CallPAL.getParamAttributes(ArgIndex))
+        Attributes Attrs = CallPAL.getParamAttributes(ArgIndex);
+        if (Attrs.hasAttributes())
           AttributesVec.push_back(AttributeWithIndex::get(Args.size(), Attrs));
 
       } else if (ByValArgsToTransform.count(I)) {
@@ -711,30 +712,32 @@ CallGraphNode *ArgPromotion::DoPromotion(Function *F,
         }
       }
 
-    if (ExtraArgHack)
-      Args.push_back(Constant::getNullValue(Type::getInt32Ty(F->getContext())));
-
     // Push any varargs arguments on the list.
     for (; AI != CS.arg_end(); ++AI, ++ArgIndex) {
       Args.push_back(*AI);
-      if (Attributes Attrs = CallPAL.getParamAttributes(ArgIndex))
+      Attributes Attrs = CallPAL.getParamAttributes(ArgIndex);
+      if (Attrs.hasAttributes())
         AttributesVec.push_back(AttributeWithIndex::get(Args.size(), Attrs));
     }
 
     // Add any function attributes.
-    if (Attributes attrs = CallPAL.getFnAttributes())
-      AttributesVec.push_back(AttributeWithIndex::get(~0, attrs));
+    attrs = CallPAL.getFnAttributes();
+    if (attrs.hasAttributes())
+      AttributesVec.push_back(AttributeWithIndex::get(AttrListPtr::FunctionIndex,
+                                                      attrs));
 
     Instruction *New;
     if (InvokeInst *II = dyn_cast<InvokeInst>(Call)) {
       New = InvokeInst::Create(NF, II->getNormalDest(), II->getUnwindDest(),
                                Args, "", Call);
       cast<InvokeInst>(New)->setCallingConv(CS.getCallingConv());
-      cast<InvokeInst>(New)->setAttributes(AttrListPtr::get(AttributesVec));
+      cast<InvokeInst>(New)->setAttributes(AttrListPtr::get(II->getContext(),
+                                                            AttributesVec));
     } else {
       New = CallInst::Create(NF, Args, "", Call);
       cast<CallInst>(New)->setCallingConv(CS.getCallingConv());
-      cast<CallInst>(New)->setAttributes(AttrListPtr::get(AttributesVec));
+      cast<CallInst>(New)->setAttributes(AttrListPtr::get(New->getContext(),
+                                                          AttributesVec));
       if (cast<CallInst>(Call)->isTailCall())
         cast<CallInst>(New)->setTailCall();
     }
@@ -870,16 +873,9 @@ CallGraphNode *ArgPromotion::DoPromotion(Function *F,
     }
 
     // Increment I2 past all of the arguments added for this promoted pointer.
-    for (unsigned i = 0, e = ArgIndices.size(); i != e; ++i)
-      ++I2;
+    std::advance(I2, ArgIndices.size());
   }
 
-  // Notify the alias analysis implementation that we inserted a new argument.
-  if (ExtraArgHack)
-    AA.copyValue(Constant::getNullValue(Type::getInt32Ty(F->getContext())), 
-                 NF->arg_begin());
-
-
   // Tell the alias analysis that the old function is about to disappear.
   AA.replaceWithNewValue(F, NF);
 
diff --git a/lib/Transforms/IPO/BarrierNoopPass.cpp b/lib/Transforms/IPO/BarrierNoopPass.cpp
new file mode 100644
index 0000000..2e32240
--- /dev/null
+++ b/lib/Transforms/IPO/BarrierNoopPass.cpp
@@ -0,0 +1,47 @@
+//===- BarrierNoopPass.cpp - A barrier pass for the pass manager ----------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// NOTE: DO NOT USE THIS IF AVOIDABLE
+//
+// This pass is a nonce pass intended to allow manipulation of the implicitly
+// nesting pass manager. For example, it can be used to cause a CGSCC pass
+// manager to be closed prior to running a new collection of function passes.
+//
+// FIXME: This is a huge HACK. This should be removed when the pass manager's
+// nesting is made explicit instead of implicit.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Pass.h"
+#include "llvm/Transforms/IPO.h"
+using namespace llvm;
+
+namespace {
+/// \brief A nonce module pass used to place a barrier in a pass manager.
+///
+/// There is no mechanism for ending a CGSCC pass manager once one is started.
+/// This prevents extension points from having clear deterministic ordering
+/// when they are phrased as non-module passes.
+class BarrierNoop : public ModulePass {
+public:
+  static char ID; // Pass identification.
+
+  BarrierNoop() : ModulePass(ID) {
+    initializeBarrierNoopPass(*PassRegistry::getPassRegistry());
+  }
+
+  bool runOnModule(Module &M) { return false; }
+};
+}
+
+ModulePass *llvm::createBarrierNoopPass() { return new BarrierNoop(); }
+
+char BarrierNoop::ID = 0;
+INITIALIZE_PASS(BarrierNoop, "barrier", "A No-Op Barrier Pass",
+                false, false)
diff --git a/lib/Transforms/IPO/CMakeLists.txt b/lib/Transforms/IPO/CMakeLists.txt
index 3f6b1de..90c1c33 100644
--- a/lib/Transforms/IPO/CMakeLists.txt
+++ b/lib/Transforms/IPO/CMakeLists.txt
@@ -1,5 +1,6 @@
 add_llvm_library(LLVMipo
   ArgumentPromotion.cpp
+  BarrierNoopPass.cpp
   ConstantMerge.cpp
   DeadArgumentElimination.cpp
   ExtractGV.cpp
diff --git a/lib/Transforms/IPO/ConstantMerge.cpp b/lib/Transforms/IPO/ConstantMerge.cpp
index d8fae8a..e2f0126 100644
--- a/lib/Transforms/IPO/ConstantMerge.cpp
+++ b/lib/Transforms/IPO/ConstantMerge.cpp
@@ -23,7 +23,7 @@
 #include "llvm/DerivedTypes.h"
 #include "llvm/Module.h"
 #include "llvm/Pass.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/PointerIntPair.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -50,7 +50,7 @@ namespace {
     // alignment to a concrete value.
     unsigned getAlignment(GlobalVariable *GV) const;
 
-    const TargetData *TD;
+    const DataLayout *TD;
   };
 }
 
@@ -98,7 +98,7 @@ unsigned ConstantMerge::getAlignment(GlobalVariable *GV) const {
 }
 
 bool ConstantMerge::runOnModule(Module &M) {
-  TD = getAnalysisIfAvailable<TargetData>();
+  TD = getAnalysisIfAvailable<DataLayout>();
 
   // Find all the globals that are marked "used".  These cannot be merged.
   SmallPtrSet<const GlobalValue*, 8> UsedGlobals;
@@ -107,7 +107,7 @@ bool ConstantMerge::runOnModule(Module &M) {
   
   // Map unique <constants, has-unknown-alignment> pairs to globals.  We don't
   // want to merge globals of unknown alignment with those of explicit
-  // alignment.  If we have TargetData, we always know the alignment.
+  // alignment.  If we have DataLayout, we always know the alignment.
   DenseMap<PointerIntPair<Constant*, 1, bool>, GlobalVariable*> CMap;
 
   // Replacements - This vector contains a list of replacements to perform.
diff --git a/lib/Transforms/IPO/DeadArgumentElimination.cpp b/lib/Transforms/IPO/DeadArgumentElimination.cpp
index fd23a93..4cfd0b2 100644
--- a/lib/Transforms/IPO/DeadArgumentElimination.cpp
+++ b/lib/Transforms/IPO/DeadArgumentElimination.cpp
@@ -21,7 +21,9 @@
 #include "llvm/Transforms/IPO.h"
 #include "llvm/CallingConv.h"
 #include "llvm/Constant.h"
+#include "llvm/DebugInfo.h"
 #include "llvm/DerivedTypes.h"
+#include "llvm/DIBuilder.h"
 #include "llvm/Instructions.h"
 #include "llvm/IntrinsicInst.h"
 #include "llvm/LLVMContext.h"
@@ -30,6 +32,7 @@
 #include "llvm/Support/CallSite.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/StringExtras.h"
@@ -121,6 +124,15 @@ namespace {
 
     typedef SmallVector<RetOrArg, 5> UseVector;
 
+    // Map each LLVM function to corresponding metadata with debug info. If
+    // the function is replaced with another one, we should patch the pointer
+    // to LLVM function in metadata.
+    // As the code generation for module is finished (and DIBuilder is
+    // finalized) we assume that subprogram descriptors won't be changed, and
+    // they are stored in map for short duration anyway.
+    typedef DenseMap<Function*, DISubprogram> FunctionDIMap;
+    FunctionDIMap FunctionDIs;
+
   protected:
     // DAH uses this to specify a different ID.
     explicit DAE(char &ID) : ModulePass(ID) {}
@@ -141,6 +153,7 @@ namespace {
                        unsigned RetValNum = 0);
     Liveness SurveyUses(const Value *V, UseVector &MaybeLiveUses);
 
+    void CollectFunctionDIs(Module &M);
     void SurveyFunction(const Function &F);
     void MarkValue(const RetOrArg &RA, Liveness L,
                    const UseVector &MaybeLiveUses);
@@ -180,6 +193,33 @@ INITIALIZE_PASS(DAH, "deadarghaX0r",
 ModulePass *llvm::createDeadArgEliminationPass() { return new DAE(); }
 ModulePass *llvm::createDeadArgHackingPass() { return new DAH(); }
 
+/// CollectFunctionDIs - Map each function in the module to its debug info
+/// descriptor.
+void DAE::CollectFunctionDIs(Module &M) {
+  FunctionDIs.clear();
+
+  for (Module::named_metadata_iterator I = M.named_metadata_begin(),
+       E = M.named_metadata_end(); I != E; ++I) {
+    NamedMDNode &NMD = *I;
+    for (unsigned MDIndex = 0, MDNum = NMD.getNumOperands();
+         MDIndex < MDNum; ++MDIndex) {
+      MDNode *Node = NMD.getOperand(MDIndex);
+      if (!DIDescriptor(Node).isCompileUnit())
+        continue;
+      DICompileUnit CU(Node);
+      const DIArray &SPs = CU.getSubprograms();
+      for (unsigned SPIndex = 0, SPNum = SPs.getNumElements();
+           SPIndex < SPNum; ++SPIndex) {
+        DISubprogram SP(SPs.getElement(SPIndex));
+        if (!SP.Verify())
+          continue;
+        if (Function *F = SP.getFunction())
+          FunctionDIs[F] = SP;
+      }
+    }
+  }
+}
+
 /// DeleteDeadVarargs - If this is an function that takes a ... list, and if
 /// llvm.vastart is never called, the varargs list is dead for the function.
 bool DAE::DeleteDeadVarargs(Function &Fn) {
@@ -236,9 +276,11 @@ bool DAE::DeleteDeadVarargs(Function &Fn) {
       SmallVector<AttributeWithIndex, 8> AttributesVec;
       for (unsigned i = 0; PAL.getSlot(i).Index <= NumArgs; ++i)
         AttributesVec.push_back(PAL.getSlot(i));
-      if (Attributes FnAttrs = PAL.getFnAttributes())
-        AttributesVec.push_back(AttributeWithIndex::get(~0, FnAttrs));
-      PAL = AttrListPtr::get(AttributesVec);
+      Attributes FnAttrs = PAL.getFnAttributes();
+      if (FnAttrs.hasAttributes())
+        AttributesVec.push_back(AttributeWithIndex::get(AttrListPtr::FunctionIndex,
+                                                        FnAttrs));
+      PAL = AttrListPtr::get(Fn.getContext(), AttributesVec);
     }
 
     Instruction *New;
@@ -284,6 +326,11 @@ bool DAE::DeleteDeadVarargs(Function &Fn) {
     I2->takeName(I);
   }
 
+  // Patch the pointer to LLVM function in debug info descriptor.
+  FunctionDIMap::iterator DI = FunctionDIs.find(&Fn);
+  if (DI != FunctionDIs.end())
+    DI->second.replaceFunction(NF);
+
   // Finally, nuke the old function.
   Fn.eraseFromParent();
   return true;
@@ -717,13 +764,17 @@ bool DAE::RemoveDeadStuffFromFunction(Function *F) {
   // here. Currently, this should not be possible, but special handling might be
   // required when new return value attributes are added.
   if (NRetTy->isVoidTy())
-    RAttrs &= ~Attribute::typeIncompatible(NRetTy);
+    RAttrs =
+      Attributes::get(NRetTy->getContext(), AttrBuilder(RAttrs).
+                      removeAttributes(Attributes::typeIncompatible(NRetTy)));
   else
-    assert((RAttrs & Attribute::typeIncompatible(NRetTy)) == 0
-           && "Return attributes no longer compatible?");
+    assert(!AttrBuilder(RAttrs).
+             hasAttributes(Attributes::typeIncompatible(NRetTy)) &&
+           "Return attributes no longer compatible?");
 
-  if (RAttrs)
-    AttributesVec.push_back(AttributeWithIndex::get(0, RAttrs));
+  if (RAttrs.hasAttributes())
+    AttributesVec.push_back(AttributeWithIndex::get(AttrListPtr::ReturnIndex,
+                                                    RAttrs));
 
   // Remember which arguments are still alive.
   SmallVector<bool, 10> ArgAlive(FTy->getNumParams(), false);
@@ -740,7 +791,8 @@ bool DAE::RemoveDeadStuffFromFunction(Function *F) {
 
       // Get the original parameter attributes (skipping the first one, that is
       // for the return value.
-      if (Attributes Attrs = PAL.getParamAttributes(i + 1))
+      Attributes Attrs = PAL.getParamAttributes(i + 1);
+      if (Attrs.hasAttributes())
         AttributesVec.push_back(AttributeWithIndex::get(Params.size(), Attrs));
     } else {
       ++NumArgumentsEliminated;
@@ -749,11 +801,12 @@ bool DAE::RemoveDeadStuffFromFunction(Function *F) {
     }
   }
 
-  if (FnAttrs != Attribute::None)
-    AttributesVec.push_back(AttributeWithIndex::get(~0, FnAttrs));
+  if (FnAttrs.hasAttributes())
+    AttributesVec.push_back(AttributeWithIndex::get(AttrListPtr::FunctionIndex,
+                                                    FnAttrs));
 
   // Reconstruct the AttributesList based on the vector we constructed.
-  AttrListPtr NewPAL = AttrListPtr::get(AttributesVec);
+  AttrListPtr NewPAL = AttrListPtr::get(F->getContext(), AttributesVec);
 
   // Create the new function type based on the recomputed parameters.
   FunctionType *NFTy = FunctionType::get(NRetTy, Params, FTy->isVarArg());
@@ -786,9 +839,12 @@ bool DAE::RemoveDeadStuffFromFunction(Function *F) {
     Attributes RAttrs = CallPAL.getRetAttributes();
     Attributes FnAttrs = CallPAL.getFnAttributes();
     // Adjust in case the function was changed to return void.
-    RAttrs &= ~Attribute::typeIncompatible(NF->getReturnType());
-    if (RAttrs)
-      AttributesVec.push_back(AttributeWithIndex::get(0, RAttrs));
+    RAttrs =
+      Attributes::get(NF->getContext(), AttrBuilder(RAttrs).
+           removeAttributes(Attributes::typeIncompatible(NF->getReturnType())));
+    if (RAttrs.hasAttributes())
+      AttributesVec.push_back(AttributeWithIndex::get(AttrListPtr::ReturnIndex,
+                                                      RAttrs));
 
     // Declare these outside of the loops, so we can reuse them for the second
     // loop, which loops the varargs.
@@ -800,22 +856,25 @@ bool DAE::RemoveDeadStuffFromFunction(Function *F) {
       if (ArgAlive[i]) {
         Args.push_back(*I);
         // Get original parameter attributes, but skip return attributes.
-        if (Attributes Attrs = CallPAL.getParamAttributes(i + 1))
+        Attributes Attrs = CallPAL.getParamAttributes(i + 1);
+        if (Attrs.hasAttributes())
           AttributesVec.push_back(AttributeWithIndex::get(Args.size(), Attrs));
       }
 
     // Push any varargs arguments on the list. Don't forget their attributes.
     for (CallSite::arg_iterator E = CS.arg_end(); I != E; ++I, ++i) {
       Args.push_back(*I);
-      if (Attributes Attrs = CallPAL.getParamAttributes(i + 1))
+      Attributes Attrs = CallPAL.getParamAttributes(i + 1);
+      if (Attrs.hasAttributes())
         AttributesVec.push_back(AttributeWithIndex::get(Args.size(), Attrs));
     }
 
-    if (FnAttrs != Attribute::None)
-      AttributesVec.push_back(AttributeWithIndex::get(~0, FnAttrs));
+    if (FnAttrs.hasAttributes())
+      AttributesVec.push_back(AttributeWithIndex::get(AttrListPtr::FunctionIndex,
+                                                      FnAttrs));
 
     // Reconstruct the AttributesList based on the vector we constructed.
-    AttrListPtr NewCallPAL = AttrListPtr::get(AttributesVec);
+    AttrListPtr NewCallPAL = AttrListPtr::get(F->getContext(), AttributesVec);
 
     Instruction *New;
     if (InvokeInst *II = dyn_cast<InvokeInst>(Call)) {
@@ -952,6 +1011,11 @@ bool DAE::RemoveDeadStuffFromFunction(Function *F) {
         BB->getInstList().erase(RI);
       }
 
+  // Patch the pointer to LLVM function in debug info descriptor.
+  FunctionDIMap::iterator DI = FunctionDIs.find(F);
+  if (DI != FunctionDIs.end())
+    DI->second.replaceFunction(NF);
+
   // Now that the old function is dead, delete it.
   F->eraseFromParent();
 
@@ -961,6 +1025,9 @@ bool DAE::RemoveDeadStuffFromFunction(Function *F) {
 bool DAE::runOnModule(Module &M) {
   bool Changed = false;
 
+  // Collect debug info descriptors for functions.
+  CollectFunctionDIs(M);
+
   // First pass: Do a simple check to see if any functions can have their "..."
   // removed.  We can do this if they never call va_start.  This loop cannot be
   // fused with the next loop, because deleting a function invalidates
diff --git a/lib/Transforms/IPO/ExtractGV.cpp b/lib/Transforms/IPO/ExtractGV.cpp
index 4c7f0ed..6716deb 100644
--- a/lib/Transforms/IPO/ExtractGV.cpp
+++ b/lib/Transforms/IPO/ExtractGV.cpp
@@ -51,32 +51,75 @@ namespace {
       // Visit the GlobalVariables.
       for (Module::global_iterator I = M.global_begin(), E = M.global_end();
            I != E; ++I) {
-        if (deleteStuff == (bool)Named.count(I) && !I->isDeclaration()) {
-          I->setInitializer(0);
-        } else {
+        bool Delete =
+          deleteStuff == (bool)Named.count(I) && !I->isDeclaration();
+        if (!Delete) {
           if (I->hasAvailableExternallyLinkage())
             continue;
           if (I->getName() == "llvm.global_ctors")
             continue;
         }
 
-        if (I->hasLocalLinkage())
+        bool Local = I->hasLocalLinkage();
+        if (Local)
           I->setVisibility(GlobalValue::HiddenVisibility);
-        I->setLinkage(GlobalValue::ExternalLinkage);
+
+        if (Local || Delete)
+          I->setLinkage(GlobalValue::ExternalLinkage);
+
+        if (Delete)
+          I->setInitializer(0);
       }
 
       // Visit the Functions.
       for (Module::iterator I = M.begin(), E = M.end(); I != E; ++I) {
-        if (deleteStuff == (bool)Named.count(I) && !I->isDeclaration()) {
-          I->deleteBody();
-        } else {
+        bool Delete =
+          deleteStuff == (bool)Named.count(I) && !I->isDeclaration();
+        if (!Delete) {
           if (I->hasAvailableExternallyLinkage())
             continue;
         }
 
-        if (I->hasLocalLinkage())
+        bool Local = I->hasLocalLinkage();
+        if (Local)
           I->setVisibility(GlobalValue::HiddenVisibility);
-        I->setLinkage(GlobalValue::ExternalLinkage);
+
+        if (Local || Delete)
+          I->setLinkage(GlobalValue::ExternalLinkage);
+
+        if (Delete)
+          I->deleteBody();
+      }
+
+      // Visit the Aliases.
+      for (Module::alias_iterator I = M.alias_begin(), E = M.alias_end();
+           I != E;) {
+        Module::alias_iterator CurI = I;
+        ++I;
+
+        if (CurI->hasLocalLinkage()) {
+          CurI->setVisibility(GlobalValue::HiddenVisibility);
+          CurI->setLinkage(GlobalValue::ExternalLinkage);
+        }
+
+        if (deleteStuff == (bool)Named.count(CurI)) {
+          Type *Ty =  CurI->getType()->getElementType();
+
+          CurI->removeFromParent();
+          llvm::Value *Declaration;
+          if (FunctionType *FTy = dyn_cast<FunctionType>(Ty)) {
+            Declaration = Function::Create(FTy, GlobalValue::ExternalLinkage,
+                                           CurI->getName(), &M);
+
+          } else {
+            Declaration =
+              new GlobalVariable(M, Ty, false, GlobalValue::ExternalLinkage,
+                                 0, CurI->getName());
+
+          }
+          CurI->replaceAllUsesWith(Declaration);
+          delete CurI;
+        }
       }
 
       return true;
diff --git a/lib/Transforms/IPO/FunctionAttrs.cpp b/lib/Transforms/IPO/FunctionAttrs.cpp
index f3f6228..18409f7 100644
--- a/lib/Transforms/IPO/FunctionAttrs.cpp
+++ b/lib/Transforms/IPO/FunctionAttrs.cpp
@@ -28,9 +28,9 @@
 #include "llvm/Analysis/CallGraph.h"
 #include "llvm/Analysis/CaptureTracking.h"
 #include "llvm/ADT/SCCIterator.h"
+#include "llvm/ADT/SetVector.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/Statistic.h"
-#include "llvm/ADT/UniqueVector.h"
 #include "llvm/Support/InstIterator.h"
 using namespace llvm;
 
@@ -212,10 +212,17 @@ bool FunctionAttrs::AddReadAttrs(const CallGraphSCC &SCC) {
     MadeChange = true;
 
     // Clear out any existing attributes.
-    F->removeAttribute(~0, Attribute::ReadOnly | Attribute::ReadNone);
+    AttrBuilder B;
+    B.addAttribute(Attributes::ReadOnly)
+      .addAttribute(Attributes::ReadNone);
+    F->removeAttribute(AttrListPtr::FunctionIndex,
+                       Attributes::get(F->getContext(), B));
 
     // Add in the new attribute.
-    F->addAttribute(~0, ReadsMemory? Attribute::ReadOnly : Attribute::ReadNone);
+    B.clear();
+    B.addAttribute(ReadsMemory ? Attributes::ReadOnly : Attributes::ReadNone);
+    F->addAttribute(AttrListPtr::FunctionIndex,
+                    Attributes::get(F->getContext(), B));
 
     if (ReadsMemory)
       ++NumReadOnly;
@@ -276,8 +283,6 @@ namespace {
 
     void tooManyUses() { Captured = true; }
 
-    bool shouldExplore(Use *U) { return true; }
-
     bool captured(Use *U) {
       CallSite CS(U->getUser());
       if (!CS.getInstruction()) { Captured = true; return true; }
@@ -352,6 +357,9 @@ bool FunctionAttrs::AddNoCaptureAttrs(const CallGraphSCC &SCC) {
 
   ArgumentGraph AG;
 
+  AttrBuilder B;
+  B.addAttribute(Attributes::NoCapture);
+
   // Check each function in turn, determining which pointer arguments are not
   // captured.
   for (CallGraphSCC::iterator I = SCC.begin(), E = SCC.end(); I != E; ++I) {
@@ -373,7 +381,7 @@ bool FunctionAttrs::AddNoCaptureAttrs(const CallGraphSCC &SCC) {
       for (Function::arg_iterator A = F->arg_begin(), E = F->arg_end();
            A != E; ++A) {
         if (A->getType()->isPointerTy() && !A->hasNoCaptureAttr()) {
-          A->addAttr(Attribute::NoCapture);
+          A->addAttr(Attributes::get(F->getContext(), B));
           ++NumNoCapture;
           Changed = true;
         }
@@ -388,7 +396,7 @@ bool FunctionAttrs::AddNoCaptureAttrs(const CallGraphSCC &SCC) {
         if (!Tracker.Captured) {
           if (Tracker.Uses.empty()) {
             // If it's trivially not captured, mark it nocapture now.
-            A->addAttr(Attribute::NoCapture);
+            A->addAttr(Attributes::get(F->getContext(), B));
             ++NumNoCapture;
             Changed = true;
           } else {
@@ -421,7 +429,9 @@ bool FunctionAttrs::AddNoCaptureAttrs(const CallGraphSCC &SCC) {
       // eg. "void f(int* x) { if (...) f(x); }"
       if (ArgumentSCC[0]->Uses.size() == 1 &&
           ArgumentSCC[0]->Uses[0] == ArgumentSCC[0]) {
-        ArgumentSCC[0]->Definition->addAttr(Attribute::NoCapture);
+        ArgumentSCC[0]->
+          Definition->
+          addAttr(Attributes::get(ArgumentSCC[0]->Definition->getContext(), B));
         ++NumNoCapture;
         Changed = true;
       }
@@ -463,7 +473,7 @@ bool FunctionAttrs::AddNoCaptureAttrs(const CallGraphSCC &SCC) {
 
     for (unsigned i = 0, e = ArgumentSCC.size(); i != e; ++i) {
       Argument *A = ArgumentSCC[i]->Definition;
-      A->addAttr(Attribute::NoCapture);
+      A->addAttr(Attributes::get(A->getContext(), B));
       ++NumNoCapture;
       Changed = true;
     }
@@ -476,13 +486,13 @@ bool FunctionAttrs::AddNoCaptureAttrs(const CallGraphSCC &SCC) {
 /// or a pointer that doesn't alias any other pointer visible to the caller.
 bool FunctionAttrs::IsFunctionMallocLike(Function *F,
                               SmallPtrSet<Function*, 8> &SCCNodes) const {
-  UniqueVector<Value *> FlowsToReturn;
+  SmallSetVector<Value *, 8> FlowsToReturn;
   for (Function::iterator I = F->begin(), E = F->end(); I != E; ++I)
     if (ReturnInst *Ret = dyn_cast<ReturnInst>(I->getTerminator()))
       FlowsToReturn.insert(Ret->getReturnValue());
 
   for (unsigned i = 0; i != FlowsToReturn.size(); ++i) {
-    Value *RetVal = FlowsToReturn[i+1];   // UniqueVector[0] is reserved.
+    Value *RetVal = FlowsToReturn[i];
 
     if (Constant *C = dyn_cast<Constant>(RetVal)) {
       if (!C->isNullValue() && !isa<UndefValue>(C))
@@ -520,7 +530,7 @@ bool FunctionAttrs::IsFunctionMallocLike(Function *F,
         case Instruction::Call:
         case Instruction::Invoke: {
           CallSite CS(RVI);
-          if (CS.paramHasAttr(0, Attribute::NoAlias))
+          if (CS.paramHasAttr(0, Attributes::NoAlias))
             break;
           if (CS.getCalledFunction() &&
               SCCNodes.count(CS.getCalledFunction()))
diff --git a/lib/Transforms/IPO/GlobalOpt.cpp b/lib/Transforms/IPO/GlobalOpt.cpp
index 6d950d2..591278f 100644
--- a/lib/Transforms/IPO/GlobalOpt.cpp
+++ b/lib/Transforms/IPO/GlobalOpt.cpp
@@ -25,7 +25,7 @@
 #include "llvm/Pass.h"
 #include "llvm/Analysis/ConstantFolding.h"
 #include "llvm/Analysis/MemoryBuiltins.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetLibraryInfo.h"
 #include "llvm/Support/CallSite.h"
 #include "llvm/Support/Debug.h"
@@ -83,7 +83,7 @@ namespace {
                                const GlobalStatus &GS);
     bool OptimizeEmptyGlobalCXXDtors(Function *CXAAtExitFn);
 
-    TargetData *TD;
+    DataLayout *TD;
     TargetLibraryInfo *TLI;
   };
 }
@@ -225,6 +225,7 @@ static bool AnalyzeGlobal(const Value *V, GlobalStatus &GS,
 
         // Don't hack on volatile stores.
         if (SI->isVolatile()) return true;
+
         GS.Ordering = StrongerOrdering(GS.Ordering, SI->getOrdering());
 
         // If this is a direct store to the global (i.e., the global is a scalar
@@ -234,6 +235,14 @@ static bool AnalyzeGlobal(const Value *V, GlobalStatus &GS,
           if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(
                                                            SI->getOperand(1))) {
             Value *StoredVal = SI->getOperand(0);
+
+            if (Constant *C = dyn_cast<Constant>(StoredVal)) {
+              if (C->isThreadDependent()) {
+                // The stored value changes between threads; don't track it.
+                return true;
+              }
+            }
+
             if (StoredVal == GV->getInitializer()) {
               if (GS.StoredType < GlobalStatus::isInitializerStored)
                 GS.StoredType = GlobalStatus::isInitializerStored;
@@ -346,7 +355,7 @@ static bool isLeakCheckerRoot(GlobalVariable *GV) {
 /// Given a value that is stored to a global but never read, determine whether
 /// it's safe to remove the store and the chain of computation that feeds the
 /// store.
-static bool IsSafeComputationToRemove(Value *V) {
+static bool IsSafeComputationToRemove(Value *V, const TargetLibraryInfo *TLI) {
   do {
     if (isa<Constant>(V))
       return true;
@@ -355,7 +364,7 @@ static bool IsSafeComputationToRemove(Value *V) {
     if (isa<LoadInst>(V) || isa<InvokeInst>(V) || isa<Argument>(V) ||
         isa<GlobalValue>(V))
       return false;
-    if (isAllocationFn(V))
+    if (isAllocationFn(V, TLI))
       return true;
 
     Instruction *I = cast<Instruction>(V);
@@ -376,7 +385,8 @@ static bool IsSafeComputationToRemove(Value *V) {
 /// of the global and clean up any that obviously don't assign the global a
 /// value that isn't dynamically allocated.
 ///
-static bool CleanupPointerRootUsers(GlobalVariable *GV) {
+static bool CleanupPointerRootUsers(GlobalVariable *GV,
+                                    const TargetLibraryInfo *TLI) {
   // A brief explanation of leak checkers.  The goal is to find bugs where
   // pointers are forgotten, causing an accumulating growth in memory
   // usage over time.  The common strategy for leak checkers is to whitelist the
@@ -432,18 +442,18 @@ static bool CleanupPointerRootUsers(GlobalVariable *GV) {
         C->destroyConstant();
         // This could have invalidated UI, start over from scratch.
         Dead.clear();
-        CleanupPointerRootUsers(GV);
+        CleanupPointerRootUsers(GV, TLI);
         return true;
       }
     }
   }
 
   for (int i = 0, e = Dead.size(); i != e; ++i) {
-    if (IsSafeComputationToRemove(Dead[i].first)) {
+    if (IsSafeComputationToRemove(Dead[i].first, TLI)) {
       Dead[i].second->eraseFromParent();
       Instruction *I = Dead[i].first;
       do {
-	if (isAllocationFn(I))
+	if (isAllocationFn(I, TLI))
 	  break;
         Instruction *J = dyn_cast<Instruction>(I->getOperand(0));
         if (!J)
@@ -463,7 +473,7 @@ static bool CleanupPointerRootUsers(GlobalVariable *GV) {
 /// quick scan over the use list to clean up the easy and obvious cruft.  This
 /// returns true if it made a change.
 static bool CleanupConstantGlobalUsers(Value *V, Constant *Init,
-                                       TargetData *TD, TargetLibraryInfo *TLI) {
+                                       DataLayout *TD, TargetLibraryInfo *TLI) {
   bool Changed = false;
   for (Value::use_iterator UI = V->use_begin(), E = V->use_end(); UI != E;) {
     User *U = *UI++;
@@ -655,7 +665,7 @@ static bool GlobalUsersSafeToSRA(GlobalValue *GV) {
 /// behavior of the program in a more fine-grained way.  We have determined that
 /// this transformation is safe already.  We return the first global variable we
 /// insert so that the caller can reprocess it.
-static GlobalVariable *SRAGlobal(GlobalVariable *GV, const TargetData &TD) {
+static GlobalVariable *SRAGlobal(GlobalVariable *GV, const DataLayout &TD) {
   // Make sure this global only has simple uses that we can SRA.
   if (!GlobalUsersSafeToSRA(GV))
     return 0;
@@ -931,7 +941,7 @@ static bool OptimizeAwayTrappingUsesOfValue(Value *V, Constant *NewV) {
 /// if the loaded value is dynamically null, then we know that they cannot be
 /// reachable with a null optimize away the load.
 static bool OptimizeAwayTrappingUsesOfLoads(GlobalVariable *GV, Constant *LV,
-                                            TargetData *TD,
+                                            DataLayout *TD,
                                             TargetLibraryInfo *TLI) {
   bool Changed = false;
 
@@ -961,7 +971,9 @@ static bool OptimizeAwayTrappingUsesOfLoads(GlobalVariable *GV, Constant *LV,
       // If we get here we could have other crazy uses that are transitively
       // loaded.
       assert((isa<PHINode>(GlobalUser) || isa<SelectInst>(GlobalUser) ||
-              isa<ConstantExpr>(GlobalUser) || isa<CmpInst>(GlobalUser)) &&
+              isa<ConstantExpr>(GlobalUser) || isa<CmpInst>(GlobalUser) ||
+              isa<BitCastInst>(GlobalUser) ||
+              isa<GetElementPtrInst>(GlobalUser)) &&
              "Only expect load and stores!");
     }
   }
@@ -975,7 +987,7 @@ static bool OptimizeAwayTrappingUsesOfLoads(GlobalVariable *GV, Constant *LV,
   // nor is the global.
   if (AllNonStoreUsesGone) {
     if (isLeakCheckerRoot(GV)) {
-      Changed |= CleanupPointerRootUsers(GV);
+      Changed |= CleanupPointerRootUsers(GV, TLI);
     } else {
       Changed = true;
       CleanupConstantGlobalUsers(GV, 0, TD, TLI);
@@ -993,7 +1005,7 @@ static bool OptimizeAwayTrappingUsesOfLoads(GlobalVariable *GV, Constant *LV,
 /// ConstantPropUsersOf - Walk the use list of V, constant folding all of the
 /// instructions that are foldable.
 static void ConstantPropUsersOf(Value *V,
-                                TargetData *TD, TargetLibraryInfo *TLI) {
+                                DataLayout *TD, TargetLibraryInfo *TLI) {
   for (Value::use_iterator UI = V->use_begin(), E = V->use_end(); UI != E; )
     if (Instruction *I = dyn_cast<Instruction>(*UI++))
       if (Constant *NewC = ConstantFoldInstruction(I, TD, TLI)) {
@@ -1016,7 +1028,7 @@ static GlobalVariable *OptimizeGlobalAddressOfMalloc(GlobalVariable *GV,
                                                      CallInst *CI,
                                                      Type *AllocTy,
                                                      ConstantInt *NElements,
-                                                     TargetData *TD,
+                                                     DataLayout *TD,
                                                      TargetLibraryInfo *TLI) {
   DEBUG(errs() << "PROMOTING GLOBAL: " << *GV << "  CALL = " << *CI << '\n');
 
@@ -1465,9 +1477,10 @@ static void RewriteUsesOfLoadForHeapSRoA(LoadInst *Load,
 /// PerformHeapAllocSRoA - CI is an allocation of an array of structures.  Break
 /// it up into multiple allocations of arrays of the fields.
 static GlobalVariable *PerformHeapAllocSRoA(GlobalVariable *GV, CallInst *CI,
-                                            Value *NElems, TargetData *TD) {
+                                            Value *NElems, DataLayout *TD,
+                                            const TargetLibraryInfo *TLI) {
   DEBUG(dbgs() << "SROA HEAP ALLOC: " << *GV << "  MALLOC = " << *CI << '\n');
-  Type *MAT = getMallocAllocatedType(CI);
+  Type *MAT = getMallocAllocatedType(CI, TLI);
   StructType *STy = cast<StructType>(MAT);
 
   // There is guaranteed to be at least one use of the malloc (storing
@@ -1656,7 +1669,7 @@ static bool TryToOptimizeStoreOfMallocToGlobal(GlobalVariable *GV,
                                                Type *AllocTy,
                                                AtomicOrdering Ordering,
                                                Module::global_iterator &GVI,
-                                               TargetData *TD,
+                                               DataLayout *TD,
                                                TargetLibraryInfo *TLI) {
   if (!TD)
     return false;
@@ -1688,7 +1701,7 @@ static bool TryToOptimizeStoreOfMallocToGlobal(GlobalVariable *GV,
   // This eliminates dynamic allocation, avoids an indirection accessing the
   // data, and exposes the resultant global to further GlobalOpt.
   // We cannot optimize the malloc if we cannot determine malloc array size.
-  Value *NElems = getMallocArraySize(CI, TD, true);
+  Value *NElems = getMallocArraySize(CI, TD, TLI, true);
   if (!NElems)
     return false;
 
@@ -1725,7 +1738,7 @@ static bool TryToOptimizeStoreOfMallocToGlobal(GlobalVariable *GV,
 
     // If this is a fixed size array, transform the Malloc to be an alloc of
     // structs.  malloc [100 x struct],1 -> malloc struct, 100
-    if (ArrayType *AT = dyn_cast<ArrayType>(getMallocAllocatedType(CI))) {
+    if (ArrayType *AT = dyn_cast<ArrayType>(getMallocAllocatedType(CI, TLI))) {
       Type *IntPtrTy = TD->getIntPtrType(CI->getContext());
       unsigned TypeSize = TD->getStructLayout(AllocSTy)->getSizeInBytes();
       Value *AllocSize = ConstantInt::get(IntPtrTy, TypeSize);
@@ -1742,7 +1755,8 @@ static bool TryToOptimizeStoreOfMallocToGlobal(GlobalVariable *GV,
         CI = cast<CallInst>(Malloc);
     }
 
-    GVI = PerformHeapAllocSRoA(GV, CI, getMallocArraySize(CI, TD, true), TD);
+    GVI = PerformHeapAllocSRoA(GV, CI, getMallocArraySize(CI, TD, TLI, true),
+                               TD, TLI);
     return true;
   }
 
@@ -1754,7 +1768,7 @@ static bool TryToOptimizeStoreOfMallocToGlobal(GlobalVariable *GV,
 static bool OptimizeOnceStoredGlobal(GlobalVariable *GV, Value *StoredOnceVal,
                                      AtomicOrdering Ordering,
                                      Module::global_iterator &GVI,
-                                     TargetData *TD, TargetLibraryInfo *TLI) {
+                                     DataLayout *TD, TargetLibraryInfo *TLI) {
   // Ignore no-op GEPs and bitcasts.
   StoredOnceVal = StoredOnceVal->stripPointerCasts();
 
@@ -1771,8 +1785,8 @@ static bool OptimizeOnceStoredGlobal(GlobalVariable *GV, Value *StoredOnceVal,
       // Optimize away any trapping uses of the loaded value.
       if (OptimizeAwayTrappingUsesOfLoads(GV, SOVC, TD, TLI))
         return true;
-    } else if (CallInst *CI = extractMallocCall(StoredOnceVal)) {
-      Type *MallocType = getMallocAllocatedType(CI);
+    } else if (CallInst *CI = extractMallocCall(StoredOnceVal, TLI)) {
+      Type *MallocType = getMallocAllocatedType(CI, TLI);
       if (MallocType &&
           TryToOptimizeStoreOfMallocToGlobal(GV, CI, MallocType, Ordering, GVI,
                                              TD, TLI))
@@ -1964,7 +1978,7 @@ bool GlobalOpt::ProcessInternalGlobal(GlobalVariable *GV,
     bool Changed;
     if (isLeakCheckerRoot(GV)) {
       // Delete any constant stores to the global.
-      Changed = CleanupPointerRootUsers(GV);
+      Changed = CleanupPointerRootUsers(GV, TLI);
     } else {
       // Delete any stores we can find to the global.  We may not be able to
       // make it completely dead though.
@@ -1997,7 +2011,7 @@ bool GlobalOpt::ProcessInternalGlobal(GlobalVariable *GV,
     ++NumMarked;
     return true;
   } else if (!GV->getInitializer()->getType()->isSingleValueType()) {
-    if (TargetData *TD = getAnalysisIfAvailable<TargetData>())
+    if (DataLayout *TD = getAnalysisIfAvailable<DataLayout>())
       if (GlobalVariable *FirstNewGV = SRAGlobal(GV, *TD)) {
         GVI = FirstNewGV;  // Don't skip the newly produced globals!
         return true;
@@ -2056,25 +2070,26 @@ static void ChangeCalleesToFastCall(Function *F) {
   }
 }
 
-static AttrListPtr StripNest(const AttrListPtr &Attrs) {
+static AttrListPtr StripNest(LLVMContext &C, const AttrListPtr &Attrs) {
   for (unsigned i = 0, e = Attrs.getNumSlots(); i != e; ++i) {
-    if ((Attrs.getSlot(i).Attrs & Attribute::Nest) == 0)
+    if (!Attrs.getSlot(i).Attrs.hasAttribute(Attributes::Nest))
       continue;
 
     // There can be only one.
-    return Attrs.removeAttr(Attrs.getSlot(i).Index, Attribute::Nest);
+    return Attrs.removeAttr(C, Attrs.getSlot(i).Index,
+                            Attributes::get(C, Attributes::Nest));
   }
 
   return Attrs;
 }
 
 static void RemoveNestAttribute(Function *F) {
-  F->setAttributes(StripNest(F->getAttributes()));
+  F->setAttributes(StripNest(F->getContext(), F->getAttributes()));
   for (Value::use_iterator UI = F->use_begin(), E = F->use_end(); UI != E;++UI){
     if (isa<BlockAddress>(*UI))
       continue;
     CallSite User(cast<Instruction>(*UI));
-    User.setAttributes(StripNest(User.getAttributes()));
+    User.setAttributes(StripNest(F->getContext(), User.getAttributes()));
   }
 }
 
@@ -2103,7 +2118,7 @@ bool GlobalOpt::OptimizeFunctions(Module &M) {
         Changed = true;
       }
 
-      if (F->getAttributes().hasAttrSomewhere(Attribute::Nest) &&
+      if (F->getAttributes().hasAttrSomewhere(Attributes::Nest) &&
           !F->hasAddressTaken()) {
         // The function is not used by a trampoline intrinsic, so it is safe
         // to remove the 'nest' attribute.
@@ -2251,7 +2266,7 @@ static GlobalVariable *InstallGlobalCtors(GlobalVariable *GCL,
 static inline bool 
 isSimpleEnoughValueToCommit(Constant *C,
                             SmallPtrSet<Constant*, 8> &SimpleConstants,
-                            const TargetData *TD);
+                            const DataLayout *TD);
 
 
 /// isSimpleEnoughValueToCommit - Return true if the specified constant can be
@@ -2264,7 +2279,7 @@ isSimpleEnoughValueToCommit(Constant *C,
 /// time.
 static bool isSimpleEnoughValueToCommitHelper(Constant *C,
                                    SmallPtrSet<Constant*, 8> &SimpleConstants,
-                                   const TargetData *TD) {
+                                   const DataLayout *TD) {
   // Simple integer, undef, constant aggregate zero, global addresses, etc are
   // all supported.
   if (C->getNumOperands() == 0 || isa<BlockAddress>(C) ||
@@ -2319,7 +2334,7 @@ static bool isSimpleEnoughValueToCommitHelper(Constant *C,
 static inline bool 
 isSimpleEnoughValueToCommit(Constant *C,
                             SmallPtrSet<Constant*, 8> &SimpleConstants,
-                            const TargetData *TD) {
+                            const DataLayout *TD) {
   // If we already checked this constant, we win.
   if (!SimpleConstants.insert(C)) return true;
   // Check the constant.
@@ -2450,7 +2465,7 @@ namespace {
 /// Once an evaluation call fails, the evaluation object should not be reused.
 class Evaluator {
 public:
-  Evaluator(const TargetData *TD, const TargetLibraryInfo *TLI)
+  Evaluator(const DataLayout *TD, const TargetLibraryInfo *TLI)
     : TD(TD), TLI(TLI) {
     ValueStack.push_back(new DenseMap<Value*, Constant*>);
   }
@@ -2531,7 +2546,7 @@ private:
   /// simple enough to live in a static initializer of a global.
   SmallPtrSet<Constant*, 8> SimpleConstants;
 
-  const TargetData *TD;
+  const DataLayout *TD;
   const TargetLibraryInfo *TLI;
 };
 
@@ -2869,7 +2884,7 @@ bool Evaluator::EvaluateFunction(Function *F, Constant *&RetVal,
 
 /// EvaluateStaticConstructor - Evaluate static constructors in the function, if
 /// we can.  Return true if we can, false otherwise.
-static bool EvaluateStaticConstructor(Function *F, const TargetData *TD,
+static bool EvaluateStaticConstructor(Function *F, const DataLayout *TD,
                                       const TargetLibraryInfo *TLI) {
   // Call the function.
   Evaluator Eval(TD, TLI);
@@ -3110,7 +3125,7 @@ bool GlobalOpt::OptimizeEmptyGlobalCXXDtors(Function *CXAAtExitFn) {
 bool GlobalOpt::runOnModule(Module &M) {
   bool Changed = false;
 
-  TD = getAnalysisIfAvailable<TargetData>();
+  TD = getAnalysisIfAvailable<DataLayout>();
   TLI = &getAnalysis<TargetLibraryInfo>();
 
   // Try to find the llvm.globalctors list.
diff --git a/lib/Transforms/IPO/IPO.cpp b/lib/Transforms/IPO/IPO.cpp
index 6233922..5d563d8 100644
--- a/lib/Transforms/IPO/IPO.cpp
+++ b/lib/Transforms/IPO/IPO.cpp
@@ -1,4 +1,4 @@
-//===-- Scalar.cpp --------------------------------------------------------===//
+//===-- IPO.cpp -----------------------------------------------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -95,7 +95,10 @@ void LLVMAddIPSCCPPass(LLVMPassManagerRef PM) {
 }
 
 void LLVMAddInternalizePass(LLVMPassManagerRef PM, unsigned AllButMain) {
-  unwrap(PM)->add(createInternalizePass(AllButMain != 0));
+  std::vector<const char *> Export;
+  if (AllButMain)
+    Export.push_back("main");
+  unwrap(PM)->add(createInternalizePass(Export));
 }
 
 void LLVMAddStripDeadPrototypesPass(LLVMPassManagerRef PM) {
diff --git a/lib/Transforms/IPO/InlineAlways.cpp b/lib/Transforms/IPO/InlineAlways.cpp
index 664ddf6..b1c36c1 100644
--- a/lib/Transforms/IPO/InlineAlways.cpp
+++ b/lib/Transforms/IPO/InlineAlways.cpp
@@ -23,7 +23,7 @@
 #include "llvm/Support/CallSite.h"
 #include "llvm/Transforms/IPO.h"
 #include "llvm/Transforms/IPO/InlinerPass.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/ADT/SmallPtrSet.h"
 
 using namespace llvm;
@@ -65,7 +65,7 @@ Pass *llvm::createAlwaysInlinerPass(bool InsertLifetime) {
 
 /// \brief Minimal filter to detect invalid constructs for inlining.
 static bool isInlineViable(Function &F) {
-  bool ReturnsTwice = F.hasFnAttr(Attribute::ReturnsTwice);
+  bool ReturnsTwice =F.getFnAttributes().hasAttribute(Attributes::ReturnsTwice);
   for (Function::iterator BI = F.begin(), BE = F.end(); BI != BE; ++BI) {
     // Disallow inlining of functions which contain an indirect branch.
     if (isa<IndirectBrInst>(BI->getTerminator()))
@@ -114,7 +114,7 @@ InlineCost AlwaysInliner::getInlineCost(CallSite CS) {
   if (Callee->isDeclaration()) return InlineCost::getNever();
 
   // Return never for anything not marked as always inline.
-  if (!Callee->hasFnAttr(Attribute::AlwaysInline))
+  if (!Callee->getFnAttributes().hasAttribute(Attributes::AlwaysInline))
     return InlineCost::getNever();
 
   // Do some minimal analysis to preclude non-viable functions.
diff --git a/lib/Transforms/IPO/InlineSimple.cpp b/lib/Transforms/IPO/InlineSimple.cpp
index 50038d8..bf0b1f9 100644
--- a/lib/Transforms/IPO/InlineSimple.cpp
+++ b/lib/Transforms/IPO/InlineSimple.cpp
@@ -22,7 +22,7 @@
 #include "llvm/Support/CallSite.h"
 #include "llvm/Transforms/IPO.h"
 #include "llvm/Transforms/IPO/InlinerPass.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 
 using namespace llvm;
 
@@ -62,7 +62,7 @@ Pass *llvm::createFunctionInliningPass(int Threshold) {
 // doInitialization - Initializes the vector of functions that have been
 // annotated with the noinline attribute.
 bool SimpleInliner::doInitialization(CallGraph &CG) {
-  CA.setTargetData(getAnalysisIfAvailable<TargetData>());
+  CA.setDataLayout(getAnalysisIfAvailable<DataLayout>());
   return false;
 }
 
diff --git a/lib/Transforms/IPO/Inliner.cpp b/lib/Transforms/IPO/Inliner.cpp
index 712888a..abcb25f 100644
--- a/lib/Transforms/IPO/Inliner.cpp
+++ b/lib/Transforms/IPO/Inliner.cpp
@@ -19,7 +19,8 @@
 #include "llvm/IntrinsicInst.h"
 #include "llvm/Analysis/CallGraph.h"
 #include "llvm/Analysis/InlineCost.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
+#include "llvm/Target/TargetLibraryInfo.h"
 #include "llvm/Transforms/IPO/InlinerPass.h"
 #include "llvm/Transforms/Utils/Cloning.h"
 #include "llvm/Transforms/Utils/Local.h"
@@ -92,11 +93,11 @@ static bool InlineCallIfPossible(CallSite CS, InlineFunctionInfo &IFI,
 
   // If the inlined function had a higher stack protection level than the
   // calling function, then bump up the caller's stack protection level.
-  if (Callee->hasFnAttr(Attribute::StackProtectReq))
-    Caller->addFnAttr(Attribute::StackProtectReq);
-  else if (Callee->hasFnAttr(Attribute::StackProtect) &&
-           !Caller->hasFnAttr(Attribute::StackProtectReq))
-    Caller->addFnAttr(Attribute::StackProtect);
+  if (Callee->getFnAttributes().hasAttribute(Attributes::StackProtectReq))
+    Caller->addFnAttr(Attributes::StackProtectReq);
+  else if (Callee->getFnAttributes().hasAttribute(Attributes::StackProtect) &&
+           !Caller->getFnAttributes().hasAttribute(Attributes::StackProtectReq))
+    Caller->addFnAttr(Attributes::StackProtect);
 
   // Look at all of the allocas that we inlined through this call site.  If we
   // have already inlined other allocas through other calls into this function,
@@ -208,14 +209,15 @@ unsigned Inliner::getInlineThreshold(CallSite CS) const {
   // would decrease the threshold.
   Function *Caller = CS.getCaller();
   bool OptSize = Caller && !Caller->isDeclaration() &&
-    Caller->hasFnAttr(Attribute::OptimizeForSize);
-  if (!(InlineLimit.getNumOccurrences() > 0) && OptSize && OptSizeThreshold < thres)
+    Caller->getFnAttributes().hasAttribute(Attributes::OptimizeForSize);
+  if (!(InlineLimit.getNumOccurrences() > 0) && OptSize &&
+      OptSizeThreshold < thres)
     thres = OptSizeThreshold;
 
   // Listen to the inlinehint attribute when it would increase the threshold.
   Function *Callee = CS.getCalledFunction();
   bool InlineHint = Callee && !Callee->isDeclaration() &&
-    Callee->hasFnAttr(Attribute::InlineHint);
+    Callee->getFnAttributes().hasAttribute(Attributes::InlineHint);
   if (InlineHint && HintThreshold > thres)
     thres = HintThreshold;
 
@@ -338,7 +340,8 @@ static bool InlineHistoryIncludes(Function *F, int InlineHistoryID,
 
 bool Inliner::runOnSCC(CallGraphSCC &SCC) {
   CallGraph &CG = getAnalysis<CallGraph>();
-  const TargetData *TD = getAnalysisIfAvailable<TargetData>();
+  const DataLayout *TD = getAnalysisIfAvailable<DataLayout>();
+  const TargetLibraryInfo *TLI = getAnalysisIfAvailable<TargetLibraryInfo>();
 
   SmallPtrSet<Function*, 8> SCCFunctions;
   DEBUG(dbgs() << "Inliner visiting SCC:");
@@ -417,7 +420,7 @@ bool Inliner::runOnSCC(CallGraphSCC &SCC) {
       // just delete the call instead of trying to inline it, regardless of
       // size.  This happens because IPSCCP propagates the result out of the
       // call and then we're left with the dead call.
-      if (isInstructionTriviallyDead(CS.getInstruction())) {
+      if (isInstructionTriviallyDead(CS.getInstruction(), TLI)) {
         DEBUG(dbgs() << "    -> Deleting dead call: "
                      << *CS.getInstruction() << "\n");
         // Update the call graph by deleting the edge from Callee to Caller.
@@ -530,7 +533,8 @@ bool Inliner::removeDeadFunctions(CallGraph &CG, bool AlwaysInlineOnly) {
     // Handle the case when this function is called and we only want to care
     // about always-inline functions. This is a bit of a hack to share code
     // between here and the InlineAlways pass.
-    if (AlwaysInlineOnly && !F->hasFnAttr(Attribute::AlwaysInline))
+    if (AlwaysInlineOnly &&
+        !F->getFnAttributes().hasAttribute(Attributes::AlwaysInline))
       continue;
 
     // If the only remaining users of the function are dead constants, remove
diff --git a/lib/Transforms/IPO/Internalize.cpp b/lib/Transforms/IPO/Internalize.cpp
index fb5869e..aa629cc 100644
--- a/lib/Transforms/IPO/Internalize.cpp
+++ b/lib/Transforms/IPO/Internalize.cpp
@@ -7,9 +7,9 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This pass loops over all of the functions in the input module, looking for a
-// main function.  If a main function is found, all other functions and all
-// global variables with initializers are marked as internal.
+// This pass loops over all of the functions and variables in the input module.
+// If the function or variable is not in the list of external names given to
+// the pass it is marked as internal.
 //
 //===----------------------------------------------------------------------===//
 
@@ -45,12 +45,9 @@ APIList("internalize-public-api-list", cl::value_desc("list"),
 namespace {
   class InternalizePass : public ModulePass {
     std::set<std::string> ExternalNames;
-    /// If no api symbols were specified and a main function is defined,
-    /// assume the main function is the only API
-    bool AllButMain;
   public:
     static char ID; // Pass identification, replacement for typeid
-    explicit InternalizePass(bool AllButMain = true);
+    explicit InternalizePass();
     explicit InternalizePass(const std::vector <const char *>& exportList);
     void LoadFile(const char *Filename);
     virtual bool runOnModule(Module &M);
@@ -66,8 +63,8 @@ char InternalizePass::ID = 0;
 INITIALIZE_PASS(InternalizePass, "internalize",
                 "Internalize Global Symbols", false, false)
 
-InternalizePass::InternalizePass(bool AllButMain)
-  : ModulePass(ID), AllButMain(AllButMain){
+InternalizePass::InternalizePass()
+  : ModulePass(ID) {
   initializeInternalizePassPass(*PassRegistry::getPassRegistry());
   if (!APIFile.empty())           // If a filename is specified, use it.
     LoadFile(APIFile.c_str());
@@ -76,7 +73,7 @@ InternalizePass::InternalizePass(bool AllButMain)
 }
 
 InternalizePass::InternalizePass(const std::vector<const char *>&exportList)
-  : ModulePass(ID), AllButMain(false){
+  : ModulePass(ID){
   initializeInternalizePassPass(*PassRegistry::getPassRegistry());
   for(std::vector<const char *>::const_iterator itr = exportList.begin();
         itr != exportList.end(); itr++) {
@@ -103,23 +100,6 @@ void InternalizePass::LoadFile(const char *Filename) {
 bool InternalizePass::runOnModule(Module &M) {
   CallGraph *CG = getAnalysisIfAvailable<CallGraph>();
   CallGraphNode *ExternalNode = CG ? CG->getExternalCallingNode() : 0;
-  
-  if (ExternalNames.empty()) {
-    // Return if we're not in 'all but main' mode and have no external api
-    if (!AllButMain)
-      return false;
-    // If no list or file of symbols was specified, check to see if there is a
-    // "main" symbol defined in the module.  If so, use it, otherwise do not
-    // internalize the module, it must be a library or something.
-    //
-    Function *MainFunc = M.getFunction("main");
-    if (MainFunc == 0 || MainFunc->isDeclaration())
-      return false;  // No main found, must be a library...
-
-    // Preserve main, internalize all else.
-    ExternalNames.insert(MainFunc->getName());
-  }
-
   bool Changed = false;
 
   // Never internalize functions which code-gen might insert.
@@ -189,8 +169,8 @@ bool InternalizePass::runOnModule(Module &M) {
   return Changed;
 }
 
-ModulePass *llvm::createInternalizePass(bool AllButMain) {
-  return new InternalizePass(AllButMain);
+ModulePass *llvm::createInternalizePass() {
+  return new InternalizePass();
 }
 
 ModulePass *llvm::createInternalizePass(const std::vector <const char *> &el) {
diff --git a/lib/Transforms/IPO/MergeFunctions.cpp b/lib/Transforms/IPO/MergeFunctions.cpp
index 9f70f66..44283dd 100644
--- a/lib/Transforms/IPO/MergeFunctions.cpp
+++ b/lib/Transforms/IPO/MergeFunctions.cpp
@@ -63,7 +63,7 @@
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/ValueHandle.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include <vector>
 using namespace llvm;
 
@@ -92,19 +92,19 @@ static unsigned profileFunction(const Function *F) {
 namespace {
 
 /// ComparableFunction - A struct that pairs together functions with a
-/// TargetData so that we can keep them together as elements in the DenseSet.
+/// DataLayout so that we can keep them together as elements in the DenseSet.
 class ComparableFunction {
 public:
   static const ComparableFunction EmptyKey;
   static const ComparableFunction TombstoneKey;
-  static TargetData * const LookupOnly;
+  static DataLayout * const LookupOnly;
 
-  ComparableFunction(Function *Func, TargetData *TD)
+  ComparableFunction(Function *Func, DataLayout *TD)
     : Func(Func), Hash(profileFunction(Func)), TD(TD) {}
 
   Function *getFunc() const { return Func; }
   unsigned getHash() const { return Hash; }
-  TargetData *getTD() const { return TD; }
+  DataLayout *getTD() const { return TD; }
 
   // Drops AssertingVH reference to the function. Outside of debug mode, this
   // does nothing.
@@ -120,13 +120,13 @@ private:
 
   AssertingVH<Function> Func;
   unsigned Hash;
-  TargetData *TD;
+  DataLayout *TD;
 };
 
 const ComparableFunction ComparableFunction::EmptyKey = ComparableFunction(0);
 const ComparableFunction ComparableFunction::TombstoneKey =
     ComparableFunction(1);
-TargetData *const ComparableFunction::LookupOnly = (TargetData*)(-1);
+DataLayout *const ComparableFunction::LookupOnly = (DataLayout*)(-1);
 
 }
 
@@ -150,12 +150,12 @@ namespace llvm {
 namespace {
 
 /// FunctionComparator - Compares two functions to determine whether or not
-/// they will generate machine code with the same behaviour. TargetData is
+/// they will generate machine code with the same behaviour. DataLayout is
 /// used if available. The comparator always fails conservatively (erring on the
 /// side of claiming that two functions are different).
 class FunctionComparator {
 public:
-  FunctionComparator(const TargetData *TD, const Function *F1,
+  FunctionComparator(const DataLayout *TD, const Function *F1,
                      const Function *F2)
     : F1(F1), F2(F2), TD(TD) {}
 
@@ -190,7 +190,7 @@ private:
   // The two functions undergoing comparison.
   const Function *F1, *F2;
 
-  const TargetData *TD;
+  const DataLayout *TD;
 
   DenseMap<const Value *, const Value *> id_map;
   DenseSet<const Value *> seen_values;
@@ -591,8 +591,8 @@ private:
   /// to modify it.
   FnSetType FnSet;
 
-  /// TargetData for more accurate GEP comparisons. May be NULL.
-  TargetData *TD;
+  /// DataLayout for more accurate GEP comparisons. May be NULL.
+  DataLayout *TD;
 
   /// Whether or not the target supports global aliases.
   bool HasGlobalAliases;
@@ -609,7 +609,7 @@ ModulePass *llvm::createMergeFunctionsPass() {
 
 bool MergeFunctions::runOnModule(Module &M) {
   bool Changed = false;
-  TD = getAnalysisIfAvailable<TargetData>();
+  TD = getAnalysisIfAvailable<DataLayout>();
 
   for (Module::iterator I = M.begin(), E = M.end(); I != E; ++I) {
     if (!I->isDeclaration() && !I->hasAvailableExternallyLinkage())
diff --git a/lib/Transforms/IPO/PassManagerBuilder.cpp b/lib/Transforms/IPO/PassManagerBuilder.cpp
index 43b4ab5..05253fc 100644
--- a/lib/Transforms/IPO/PassManagerBuilder.cpp
+++ b/lib/Transforms/IPO/PassManagerBuilder.cpp
@@ -33,13 +33,21 @@
 using namespace llvm;
 
 static cl::opt<bool>
-RunVectorization("vectorize", cl::desc("Run vectorization passes"));
+RunLoopVectorization("vectorize-loops",
+                     cl::desc("Run the Loop vectorization passes"));
+
+static cl::opt<bool>
+RunBBVectorization("vectorize", cl::desc("Run the BB vectorization passes"));
 
 static cl::opt<bool>
 UseGVNAfterVectorization("use-gvn-after-vectorization",
   cl::init(false), cl::Hidden,
   cl::desc("Run GVN instead of Early CSE after vectorization passes"));
 
+static cl::opt<bool> UseNewSROA("use-new-sroa",
+  cl::init(true), cl::Hidden,
+  cl::desc("Enable the new, experimental SROA pass"));
+
 PassManagerBuilder::PassManagerBuilder() {
     OptLevel = 2;
     SizeLevel = 0;
@@ -48,7 +56,8 @@ PassManagerBuilder::PassManagerBuilder() {
     DisableSimplifyLibCalls = false;
     DisableUnitAtATime = false;
     DisableUnrollLoops = false;
-    Vectorize = RunVectorization;
+    Vectorize = RunBBVectorization;
+    LoopVectorize = RunLoopVectorization;
 }
 
 PassManagerBuilder::~PassManagerBuilder() {
@@ -100,7 +109,10 @@ void PassManagerBuilder::populateFunctionPassManager(FunctionPassManager &FPM) {
   addInitialAliasAnalysisPasses(FPM);
 
   FPM.add(createCFGSimplificationPass());
-  FPM.add(createScalarReplAggregatesPass());
+  if (UseNewSROA)
+    FPM.add(createSROAPass());
+  else
+    FPM.add(createScalarReplAggregatesPass());
   FPM.add(createEarlyCSEPass());
   FPM.add(createLowerExpectIntrinsicPass());
 }
@@ -112,6 +124,14 @@ void PassManagerBuilder::populateModulePassManager(PassManagerBase &MPM) {
       MPM.add(Inliner);
       Inliner = 0;
     }
+
+    // FIXME: This is a HACK! The inliner pass above implicitly creates a CGSCC
+    // pass manager, but we don't want to add extensions into that pass manager.
+    // To prevent this we must insert a no-op module pass to reset the pass
+    // manager to get the same behavior as EP_OptimizerLast in non-O0 builds.
+    if (!GlobalExtensions->empty() || !Extensions.empty())
+      MPM.add(createBarrierNoopPass());
+
     addExtensionsToPM(EP_EnabledOnOptLevel0, MPM);
     return;
   }
@@ -147,7 +167,10 @@ void PassManagerBuilder::populateModulePassManager(PassManagerBase &MPM) {
 
   // Start of function pass.
   // Break up aggregate allocas, using SSAUpdater.
-  MPM.add(createScalarReplAggregatesPass(-1, false));
+  if (UseNewSROA)
+    MPM.add(createSROAPass(/*RequiresDomTree*/ false));
+  else
+    MPM.add(createScalarReplAggregatesPass(-1, false));
   MPM.add(createEarlyCSEPass());              // Catch trivial redundancies
   if (!DisableSimplifyLibCalls)
     MPM.add(createSimplifyLibCallsPass());    // Library Call Optimizations
@@ -166,6 +189,12 @@ void PassManagerBuilder::populateModulePassManager(PassManagerBase &MPM) {
   MPM.add(createIndVarSimplifyPass());        // Canonicalize indvars
   MPM.add(createLoopIdiomPass());             // Recognize idioms like memset.
   MPM.add(createLoopDeletionPass());          // Delete dead loops
+
+  if (LoopVectorize) {
+    MPM.add(createLoopVectorizePass());
+    MPM.add(createLICMPass());
+  }
+
   if (!DisableUnrollLoops)
     MPM.add(createLoopUnrollPass());          // Unroll small loops
   addExtensionsToPM(EP_LoopOptimizerEnd, MPM);
@@ -201,13 +230,12 @@ void PassManagerBuilder::populateModulePassManager(PassManagerBase &MPM) {
     // FIXME: We shouldn't bother with this anymore.
     MPM.add(createStripDeadPrototypesPass()); // Get rid of dead prototypes
 
-    // GlobalOpt already deletes dead functions and globals, at -O3 try a
+    // GlobalOpt already deletes dead functions and globals, at -O2 try a
     // late pass of GlobalDCE.  It is capable of deleting dead cycles.
-    if (OptLevel > 2)
+    if (OptLevel > 1) {
       MPM.add(createGlobalDCEPass());         // Remove dead fns and globals.
-
-    if (OptLevel > 1)
       MPM.add(createConstantMergePass());     // Merge dup global constants
+    }
   }
   addExtensionsToPM(EP_OptimizerLast, MPM);
 }
@@ -222,8 +250,11 @@ void PassManagerBuilder::populateLTOPassManager(PassManagerBase &PM,
   // Now that composite has been compiled, scan through the module, looking
   // for a main function.  If main is defined, mark all other functions
   // internal.
-  if (Internalize)
-    PM.add(createInternalizePass(true));
+  if (Internalize) {
+    std::vector<const char*> E;
+    E.push_back("main");
+    PM.add(createInternalizePass(E));
+  }
 
   // Propagate constants at call sites into the functions they call.  This
   // opens opportunities for globalopt (and inlining) by substituting function
@@ -265,7 +296,10 @@ void PassManagerBuilder::populateLTOPassManager(PassManagerBase &PM,
   PM.add(createInstructionCombiningPass());
   PM.add(createJumpThreadingPass());
   // Break up allocas
-  PM.add(createScalarReplAggregatesPass());
+  if (UseNewSROA)
+    PM.add(createSROAPass());
+  else
+    PM.add(createScalarReplAggregatesPass());
 
   // Run a few AA driven optimizations here and now, to cleanup the code.
   PM.add(createFunctionAttrsPass()); // Add nocapture.
diff --git a/lib/Transforms/IPO/PruneEH.cpp b/lib/Transforms/IPO/PruneEH.cpp
index c8cc8fd..fb4ecbf 100644
--- a/lib/Transforms/IPO/PruneEH.cpp
+++ b/lib/Transforms/IPO/PruneEH.cpp
@@ -137,16 +137,18 @@ bool PruneEH::runOnSCC(CallGraphSCC &SCC) {
   // If the SCC doesn't unwind or doesn't throw, note this fact.
   if (!SCCMightUnwind || !SCCMightReturn)
     for (CallGraphSCC::iterator I = SCC.begin(), E = SCC.end(); I != E; ++I) {
-      Attributes NewAttributes = Attribute::None;
+      AttrBuilder NewAttributes;
 
       if (!SCCMightUnwind)
-        NewAttributes |= Attribute::NoUnwind;
+        NewAttributes.addAttribute(Attributes::NoUnwind);
       if (!SCCMightReturn)
-        NewAttributes |= Attribute::NoReturn;
+        NewAttributes.addAttribute(Attributes::NoReturn);
 
       Function *F = (*I)->getFunction();
       const AttrListPtr &PAL = F->getAttributes();
-      const AttrListPtr &NPAL = PAL.addAttr(~0, NewAttributes);
+      const AttrListPtr &NPAL = PAL.addAttr(F->getContext(), ~0,
+                                            Attributes::get(F->getContext(),
+                                                            NewAttributes));
       if (PAL != NPAL) {
         MadeChange = true;
         F->setAttributes(NPAL);
diff --git a/lib/Transforms/InstCombine/InstCombine.h b/lib/Transforms/InstCombine/InstCombine.h
index 0d5ef90..7467eca 100644
--- a/lib/Transforms/InstCombine/InstCombine.h
+++ b/lib/Transforms/InstCombine/InstCombine.h
@@ -18,10 +18,11 @@
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/Support/InstVisitor.h"
 #include "llvm/Support/TargetFolder.h"
+#include "llvm/Transforms/Utils/SimplifyLibCalls.h"
 
 namespace llvm {
   class CallSite;
-  class TargetData;
+  class DataLayout;
   class TargetLibraryInfo;
   class DbgDeclareInst;
   class MemIntrinsic;
@@ -71,9 +72,10 @@ public:
 class LLVM_LIBRARY_VISIBILITY InstCombiner
                              : public FunctionPass,
                                public InstVisitor<InstCombiner, Instruction*> {
-  TargetData *TD;
+  DataLayout *TD;
   TargetLibraryInfo *TLI;
   bool MadeIRChange;
+  LibCallSimplifier *Simplifier;
 public:
   /// Worklist - All of the instructions that need to be simplified.
   InstCombineWorklist Worklist;
@@ -95,7 +97,7 @@ public:
 
   virtual void getAnalysisUsage(AnalysisUsage &AU) const;
 
-  TargetData *getTargetData() const { return TD; }
+  DataLayout *getDataLayout() const { return TD; }
 
   TargetLibraryInfo *getTargetLibraryInfo() const { return TLI; }
 
@@ -218,7 +220,7 @@ private:
                           Type *Ty);
 
   Instruction *visitCallSite(CallSite CS);
-  Instruction *tryOptimizeCall(CallInst *CI, const TargetData *TD);
+  Instruction *tryOptimizeCall(CallInst *CI, const DataLayout *TD);
   bool transformConstExprCastCall(CallSite CS);
   Instruction *transformCallThroughTrampoline(CallSite CS,
                                               IntrinsicInst *Tramp);
@@ -365,6 +367,10 @@ private:
 
 
   Value *EvaluateInDifferentType(Value *V, Type *Ty, bool isSigned);
+
+  /// Descale - Return a value X such that Val = X * Scale, or null if none.  If
+  /// the multiplication is known not to overflow then NoSignedWrap is set.
+  Value *Descale(Value *Val, APInt Scale, bool &NoSignedWrap);
 };
 
       
diff --git a/lib/Transforms/InstCombine/InstCombineAddSub.cpp b/lib/Transforms/InstCombine/InstCombineAddSub.cpp
index 99b62f8..d8257e6 100644
--- a/lib/Transforms/InstCombine/InstCombineAddSub.cpp
+++ b/lib/Transforms/InstCombine/InstCombineAddSub.cpp
@@ -13,7 +13,7 @@
 
 #include "InstCombine.h"
 #include "llvm/Analysis/InstructionSimplify.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Support/GetElementPtrTypeIterator.h"
 #include "llvm/Support/PatternMatch.h"
 using namespace llvm;
diff --git a/lib/Transforms/InstCombine/InstCombineCalls.cpp b/lib/Transforms/InstCombine/InstCombineCalls.cpp
index cbe1ca4..48f2704 100644
--- a/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -13,7 +13,7 @@
 
 #include "InstCombine.h"
 #include "llvm/Support/CallSite.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Analysis/MemoryBuiltins.h"
 #include "llvm/Transforms/Utils/BuildLibCalls.h"
 #include "llvm/Transforms/Utils/Local.h"
@@ -29,6 +29,26 @@ static Type *getPromotedType(Type *Ty) {
   return Ty;
 }
 
+/// reduceToSingleValueType - Given an aggregate type which ultimately holds a
+/// single scalar element, like {{{type}}} or [1 x type], return type.
+static Type *reduceToSingleValueType(Type *T) {
+  while (!T->isSingleValueType()) {
+    if (StructType *STy = dyn_cast<StructType>(T)) {
+      if (STy->getNumElements() == 1)
+        T = STy->getElementType(0);
+      else
+        break;
+    } else if (ArrayType *ATy = dyn_cast<ArrayType>(T)) {
+      if (ATy->getNumElements() == 1)
+        T = ATy->getElementType();
+      else
+        break;
+    } else
+      break;
+  }
+
+  return T;
+}
 
 Instruction *InstCombiner::SimplifyMemTransfer(MemIntrinsic *MI) {
   unsigned DstAlign = getKnownAlignment(MI->getArgOperand(0), TD);
@@ -74,35 +94,37 @@ Instruction *InstCombiner::SimplifyMemTransfer(MemIntrinsic *MI) {
   // dest address will be promotable.  See if we can find a better type than the
   // integer datatype.
   Value *StrippedDest = MI->getArgOperand(0)->stripPointerCasts();
+  MDNode *CopyMD = 0;
   if (StrippedDest != MI->getArgOperand(0)) {
     Type *SrcETy = cast<PointerType>(StrippedDest->getType())
                                     ->getElementType();
     if (TD && SrcETy->isSized() && TD->getTypeStoreSize(SrcETy) == Size) {
       // The SrcETy might be something like {{{double}}} or [1 x double].  Rip
       // down through these levels if so.
-      while (!SrcETy->isSingleValueType()) {
-        if (StructType *STy = dyn_cast<StructType>(SrcETy)) {
-          if (STy->getNumElements() == 1)
-            SrcETy = STy->getElementType(0);
-          else
-            break;
-        } else if (ArrayType *ATy = dyn_cast<ArrayType>(SrcETy)) {
-          if (ATy->getNumElements() == 1)
-            SrcETy = ATy->getElementType();
-          else
-            break;
-        } else
-          break;
-      }
+      SrcETy = reduceToSingleValueType(SrcETy);
 
       if (SrcETy->isSingleValueType()) {
         NewSrcPtrTy = PointerType::get(SrcETy, SrcAddrSp);
         NewDstPtrTy = PointerType::get(SrcETy, DstAddrSp);
+
+        // If the memcpy has metadata describing the members, see if we can
+        // get the TBAA tag describing our copy.
+        if (MDNode *M = MI->getMetadata(LLVMContext::MD_tbaa_struct)) {
+          if (M->getNumOperands() == 3 &&
+              M->getOperand(0) &&
+              isa<ConstantInt>(M->getOperand(0)) &&
+              cast<ConstantInt>(M->getOperand(0))->isNullValue() &&
+              M->getOperand(1) &&
+              isa<ConstantInt>(M->getOperand(1)) &&
+              cast<ConstantInt>(M->getOperand(1))->getValue() == Size &&
+              M->getOperand(2) &&
+              isa<MDNode>(M->getOperand(2)))
+            CopyMD = cast<MDNode>(M->getOperand(2));
+        }
       }
     }
   }
 
-
   // If the memcpy/memmove provides better alignment info than we can
   // infer, use it.
   SrcAlign = std::max(SrcAlign, CopyAlign);
@@ -112,8 +134,12 @@ Instruction *InstCombiner::SimplifyMemTransfer(MemIntrinsic *MI) {
   Value *Dest = Builder->CreateBitCast(MI->getArgOperand(0), NewDstPtrTy);
   LoadInst *L = Builder->CreateLoad(Src, MI->isVolatile());
   L->setAlignment(SrcAlign);
+  if (CopyMD)
+    L->setMetadata(LLVMContext::MD_tbaa, CopyMD);
   StoreInst *S = Builder->CreateStore(L, Dest, MI->isVolatile());
   S->setAlignment(DstAlign);
+  if (CopyMD)
+    S->setMetadata(LLVMContext::MD_tbaa, CopyMD);
 
   // Set the size of the copy to 0, it will be deleted on the next iteration.
   MI->setArgOperand(2, Constant::getNullValue(MemOpLength->getType()));
@@ -168,7 +194,7 @@ Instruction *InstCombiner::SimplifyMemSet(MemSetInst *MI) {
 /// the heavy lifting.
 ///
 Instruction *InstCombiner::visitCallInst(CallInst &CI) {
-  if (isFreeCall(&CI))
+  if (isFreeCall(&CI, TLI))
     return visitFree(CI);
 
   // If the caller function is nounwind, mark the call as nounwind, even if the
@@ -243,7 +269,7 @@ Instruction *InstCombiner::visitCallInst(CallInst &CI) {
   default: break;
   case Intrinsic::objectsize: {
     uint64_t Size;
-    if (getObjectSize(II->getArgOperand(0), Size, TD))
+    if (getObjectSize(II->getArgOperand(0), Size, TD, TLI))
       return ReplaceInstUsesWith(CI, ConstantInt::get(CI.getType(), Size));
     return 0;
   }
@@ -731,7 +757,7 @@ Instruction *InstCombiner::visitInvokeInst(InvokeInst &II) {
 /// passed through the varargs area, we can eliminate the use of the cast.
 static bool isSafeToEliminateVarargsCast(const CallSite CS,
                                          const CastInst * const CI,
-                                         const TargetData * const TD,
+                                         const DataLayout * const TD,
                                          const int ix) {
   if (!CI->isLosslessCast())
     return false;
@@ -752,49 +778,17 @@ static bool isSafeToEliminateVarargsCast(const CallSite CS,
   return true;
 }
 
-namespace {
-class InstCombineFortifiedLibCalls : public SimplifyFortifiedLibCalls {
-  InstCombiner *IC;
-protected:
-  void replaceCall(Value *With) {
-    NewInstruction = IC->ReplaceInstUsesWith(*CI, With);
-  }
-  bool isFoldable(unsigned SizeCIOp, unsigned SizeArgOp, bool isString) const {
-    if (CI->getArgOperand(SizeCIOp) == CI->getArgOperand(SizeArgOp))
-      return true;
-    if (ConstantInt *SizeCI =
-                           dyn_cast<ConstantInt>(CI->getArgOperand(SizeCIOp))) {
-      if (SizeCI->isAllOnesValue())
-        return true;
-      if (isString) {
-        uint64_t Len = GetStringLength(CI->getArgOperand(SizeArgOp));
-        // If the length is 0 we don't know how long it is and so we can't
-        // remove the check.
-        if (Len == 0) return false;
-        return SizeCI->getZExtValue() >= Len;
-      }
-      if (ConstantInt *Arg = dyn_cast<ConstantInt>(
-                                                  CI->getArgOperand(SizeArgOp)))
-        return SizeCI->getZExtValue() >= Arg->getZExtValue();
-    }
-    return false;
-  }
-public:
-  InstCombineFortifiedLibCalls(InstCombiner *IC) : IC(IC), NewInstruction(0) { }
-  Instruction *NewInstruction;
-};
-} // end anonymous namespace
-
 // Try to fold some different type of calls here.
 // Currently we're only working with the checking functions, memcpy_chk,
 // mempcpy_chk, memmove_chk, memset_chk, strcpy_chk, stpcpy_chk, strncpy_chk,
 // strcat_chk and strncat_chk.
-Instruction *InstCombiner::tryOptimizeCall(CallInst *CI, const TargetData *TD) {
+Instruction *InstCombiner::tryOptimizeCall(CallInst *CI, const DataLayout *TD) {
   if (CI->getCalledFunction() == 0) return 0;
 
-  InstCombineFortifiedLibCalls Simplifier(this);
-  Simplifier.fold(CI, TD, TLI);
-  return Simplifier.NewInstruction;
+  if (Value *With = Simplifier->optimizeCall(CI))
+    return ReplaceInstUsesWith(*CI, With);
+
+  return 0;
 }
 
 static IntrinsicInst *FindInitTrampolineFromAlloca(Value *TrampMem) {
@@ -877,7 +871,7 @@ static IntrinsicInst *FindInitTrampoline(Value *Callee) {
 // visitCallSite - Improvements for call and invoke instructions.
 //
 Instruction *InstCombiner::visitCallSite(CallSite CS) {
-  if (isAllocLikeFn(CS.getInstruction()))
+  if (isAllocLikeFn(CS.getInstruction(), TLI))
     return visitAllocSite(*CS.getInstruction());
 
   bool Changed = false;
@@ -961,7 +955,7 @@ Instruction *InstCombiner::visitCallSite(CallSite CS) {
     Changed = true;
   }
 
-  // Try to optimize the call if possible, we require TargetData for most of
+  // Try to optimize the call if possible, we require DataLayout for most of
   // this.  None of these calls are seen as possibly dead so go ahead and
   // delete the instruction now.
   if (CallInst *CI = dyn_cast<CallInst>(CS.getInstruction())) {
@@ -1013,8 +1007,8 @@ bool InstCombiner::transformConstExprCastCall(CallSite CS) {
       return false;   // Cannot transform this return value.
 
     if (!CallerPAL.isEmpty() && !Caller->use_empty()) {
-      Attributes RAttrs = CallerPAL.getRetAttributes();
-      if (RAttrs & Attribute::typeIncompatible(NewRetTy))
+      AttrBuilder RAttrs = CallerPAL.getRetAttributes();
+      if (RAttrs.hasAttributes(Attributes::typeIncompatible(NewRetTy)))
         return false;   // Attribute not compatible with transformed value.
     }
 
@@ -1044,12 +1038,13 @@ bool InstCombiner::transformConstExprCastCall(CallSite CS) {
       return false;   // Cannot transform this parameter value.
 
     Attributes Attrs = CallerPAL.getParamAttributes(i + 1);
-    if (Attrs & Attribute::typeIncompatible(ParamTy))
+    if (AttrBuilder(Attrs).
+          hasAttributes(Attributes::typeIncompatible(ParamTy)))
       return false;   // Attribute not compatible with transformed value.
 
     // If the parameter is passed as a byval argument, then we have to have a
     // sized type and the sized type has to have the same size as the old type.
-    if (ParamTy != ActTy && (Attrs & Attribute::ByVal)) {
+    if (ParamTy != ActTy && Attrs.hasAttribute(Attributes::ByVal)) {
       PointerType *ParamPTy = dyn_cast<PointerType>(ParamTy);
       if (ParamPTy == 0 || !ParamPTy->getElementType()->isSized() || TD == 0)
         return false;
@@ -1101,7 +1096,7 @@ bool InstCombiner::transformConstExprCastCall(CallSite CS) {
       if (CallerPAL.getSlot(i - 1).Index <= FT->getNumParams())
         break;
       Attributes PAttrs = CallerPAL.getSlot(i - 1).Attrs;
-      if (PAttrs & Attribute::VarArgsIncompatible)
+      if (PAttrs.hasIncompatibleWithVarArgsAttrs())
         return false;
     }
 
@@ -1114,15 +1109,17 @@ bool InstCombiner::transformConstExprCastCall(CallSite CS) {
   attrVec.reserve(NumCommonArgs);
 
   // Get any return attributes.
-  Attributes RAttrs = CallerPAL.getRetAttributes();
+  AttrBuilder RAttrs = CallerPAL.getRetAttributes();
 
   // If the return value is not being used, the type may not be compatible
   // with the existing attributes.  Wipe out any problematic attributes.
-  RAttrs &= ~Attribute::typeIncompatible(NewRetTy);
+  RAttrs.removeAttributes(Attributes::typeIncompatible(NewRetTy));
 
   // Add the new return attributes.
-  if (RAttrs)
-    attrVec.push_back(AttributeWithIndex::get(0, RAttrs));
+  if (RAttrs.hasAttributes())
+    attrVec.push_back(
+      AttributeWithIndex::get(AttrListPtr::ReturnIndex,
+                              Attributes::get(FT->getContext(), RAttrs)));
 
   AI = CS.arg_begin();
   for (unsigned i = 0; i != NumCommonArgs; ++i, ++AI) {
@@ -1136,7 +1133,8 @@ bool InstCombiner::transformConstExprCastCall(CallSite CS) {
     }
 
     // Add any parameter attributes.
-    if (Attributes PAttrs = CallerPAL.getParamAttributes(i + 1))
+    Attributes PAttrs = CallerPAL.getParamAttributes(i + 1);
+    if (PAttrs.hasAttributes())
       attrVec.push_back(AttributeWithIndex::get(i + 1, PAttrs));
   }
 
@@ -1164,19 +1162,23 @@ bool InstCombiner::transformConstExprCastCall(CallSite CS) {
         }
 
         // Add any parameter attributes.
-        if (Attributes PAttrs = CallerPAL.getParamAttributes(i + 1))
+        Attributes PAttrs = CallerPAL.getParamAttributes(i + 1);
+        if (PAttrs.hasAttributes())
           attrVec.push_back(AttributeWithIndex::get(i + 1, PAttrs));
       }
     }
   }
 
-  if (Attributes FnAttrs =  CallerPAL.getFnAttributes())
-    attrVec.push_back(AttributeWithIndex::get(~0, FnAttrs));
+  Attributes FnAttrs = CallerPAL.getFnAttributes();
+  if (FnAttrs.hasAttributes())
+    attrVec.push_back(AttributeWithIndex::get(AttrListPtr::FunctionIndex,
+                                              FnAttrs));
 
   if (NewRetTy->isVoidTy())
     Caller->setName("");   // Void type should not have a name.
 
-  const AttrListPtr &NewCallerPAL = AttrListPtr::get(attrVec);
+  const AttrListPtr &NewCallerPAL = AttrListPtr::get(Callee->getContext(),
+                                                     attrVec);
 
   Instruction *NC;
   if (InvokeInst *II = dyn_cast<InvokeInst>(Caller)) {
@@ -1240,8 +1242,9 @@ InstCombiner::transformCallThroughTrampoline(CallSite CS,
 
   // If the call already has the 'nest' attribute somewhere then give up -
   // otherwise 'nest' would occur twice after splicing in the chain.
-  if (Attrs.hasAttrSomewhere(Attribute::Nest))
-    return 0;
+  for (unsigned I = 0, E = Attrs.getNumAttrs(); I != E; ++I)
+    if (Attrs.getAttributesAtIndex(I).hasAttribute(Attributes::Nest))
+      return 0;
 
   assert(Tramp &&
          "transformCallThroughTrampoline called with incorrect CallSite.");
@@ -1254,12 +1257,12 @@ InstCombiner::transformCallThroughTrampoline(CallSite CS,
   if (!NestAttrs.isEmpty()) {
     unsigned NestIdx = 1;
     Type *NestTy = 0;
-    Attributes NestAttr = Attribute::None;
+    Attributes NestAttr;
 
     // Look for a parameter marked with the 'nest' attribute.
     for (FunctionType::param_iterator I = NestFTy->param_begin(),
          E = NestFTy->param_end(); I != E; ++NestIdx, ++I)
-      if (NestAttrs.paramHasAttr(NestIdx, Attribute::Nest)) {
+      if (NestAttrs.getParamAttributes(NestIdx).hasAttribute(Attributes::Nest)){
         // Record the parameter type and any other attributes.
         NestTy = *I;
         NestAttr = NestAttrs.getParamAttributes(NestIdx);
@@ -1278,8 +1281,10 @@ InstCombiner::transformCallThroughTrampoline(CallSite CS,
       // mean appending it.  Likewise for attributes.
 
       // Add any result attributes.
-      if (Attributes Attr = Attrs.getRetAttributes())
-        NewAttrs.push_back(AttributeWithIndex::get(0, Attr));
+      Attributes Attr = Attrs.getRetAttributes();
+      if (Attr.hasAttributes())
+        NewAttrs.push_back(AttributeWithIndex::get(AttrListPtr::ReturnIndex,
+                                                   Attr));
 
       {
         unsigned Idx = 1;
@@ -1299,7 +1304,8 @@ InstCombiner::transformCallThroughTrampoline(CallSite CS,
 
           // Add the original argument and attributes.
           NewArgs.push_back(*I);
-          if (Attributes Attr = Attrs.getParamAttributes(Idx))
+          Attr = Attrs.getParamAttributes(Idx);
+          if (Attr.hasAttributes())
             NewAttrs.push_back
               (AttributeWithIndex::get(Idx + (Idx >= NestIdx), Attr));
 
@@ -1308,8 +1314,10 @@ InstCombiner::transformCallThroughTrampoline(CallSite CS,
       }
 
       // Add any function attributes.
-      if (Attributes Attr = Attrs.getFnAttributes())
-        NewAttrs.push_back(AttributeWithIndex::get(~0, Attr));
+      Attr = Attrs.getFnAttributes();
+      if (Attr.hasAttributes())
+        NewAttrs.push_back(AttributeWithIndex::get(AttrListPtr::FunctionIndex,
+                                                   Attr));
 
       // The trampoline may have been bitcast to a bogus type (FTy).
       // Handle this by synthesizing a new function type, equal to FTy
@@ -1348,7 +1356,7 @@ InstCombiner::transformCallThroughTrampoline(CallSite CS,
         NestF->getType() == PointerType::getUnqual(NewFTy) ?
         NestF : ConstantExpr::getBitCast(NestF,
                                          PointerType::getUnqual(NewFTy));
-      const AttrListPtr &NewPAL = AttrListPtr::get(NewAttrs);
+      const AttrListPtr &NewPAL = AttrListPtr::get(FTy->getContext(), NewAttrs);
 
       Instruction *NewCaller;
       if (InvokeInst *II = dyn_cast<InvokeInst>(Caller)) {
diff --git a/lib/Transforms/InstCombine/InstCombineCasts.cpp b/lib/Transforms/InstCombine/InstCombineCasts.cpp
index 555b442..bb59db8 100644
--- a/lib/Transforms/InstCombine/InstCombineCasts.cpp
+++ b/lib/Transforms/InstCombine/InstCombineCasts.cpp
@@ -13,7 +13,7 @@
 
 #include "InstCombine.h"
 #include "llvm/Analysis/ConstantFolding.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetLibraryInfo.h"
 #include "llvm/Support/PatternMatch.h"
 using namespace llvm;
@@ -78,7 +78,7 @@ static Value *DecomposeSimpleLinearExpr(Value *Val, unsigned &Scale,
 /// try to eliminate the cast by moving the type information into the alloc.
 Instruction *InstCombiner::PromoteCastOfAllocation(BitCastInst &CI,
                                                    AllocaInst &AI) {
-  // This requires TargetData to get the alloca alignment and size information.
+  // This requires DataLayout to get the alloca alignment and size information.
   if (!TD) return 0;
 
   PointerType *PTy = cast<PointerType>(CI.getType());
@@ -229,7 +229,7 @@ isEliminableCastPair(
   const CastInst *CI, ///< The first cast instruction
   unsigned opcode,       ///< The opcode of the second cast instruction
   Type *DstTy,     ///< The target type for the second cast instruction
-  TargetData *TD         ///< The target data for pointer size
+  DataLayout *TD         ///< The target data for pointer size
 ) {
 
   Type *SrcTy = CI->getOperand(0)->getType();   // A from above
@@ -238,17 +238,20 @@ isEliminableCastPair(
   // Get the opcodes of the two Cast instructions
   Instruction::CastOps firstOp = Instruction::CastOps(CI->getOpcode());
   Instruction::CastOps secondOp = Instruction::CastOps(opcode);
-
+  Type *SrcIntPtrTy = TD && SrcTy->isPtrOrPtrVectorTy() ?
+    TD->getIntPtrType(SrcTy) : 0;
+  Type *MidIntPtrTy = TD && MidTy->isPtrOrPtrVectorTy() ?
+    TD->getIntPtrType(MidTy) : 0;
+  Type *DstIntPtrTy = TD && DstTy->isPtrOrPtrVectorTy() ?
+    TD->getIntPtrType(DstTy) : 0;
   unsigned Res = CastInst::isEliminableCastPair(firstOp, secondOp, SrcTy, MidTy,
-                                                DstTy,
-                                  TD ? TD->getIntPtrType(CI->getContext()) : 0);
-  
+                                                DstTy, SrcIntPtrTy, MidIntPtrTy,
+                                                DstIntPtrTy);
+
   // We don't want to form an inttoptr or ptrtoint that converts to an integer
   // type that differs from the pointer size.
-  if ((Res == Instruction::IntToPtr &&
-          (!TD || SrcTy != TD->getIntPtrType(CI->getContext()))) ||
-      (Res == Instruction::PtrToInt &&
-          (!TD || DstTy != TD->getIntPtrType(CI->getContext()))))
+  if ((Res == Instruction::IntToPtr && SrcTy != DstIntPtrTy) ||
+      (Res == Instruction::PtrToInt && DstTy != SrcIntPtrTy))
     Res = 0;
   
   return Instruction::CastOps(Res);
diff --git a/lib/Transforms/InstCombine/InstCombineCompares.cpp b/lib/Transforms/InstCombine/InstCombineCompares.cpp
index bdd310e..7c3f8fe 100644
--- a/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ b/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -16,7 +16,8 @@
 #include "llvm/Analysis/ConstantFolding.h"
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/MemoryBuiltins.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
+#include "llvm/Target/TargetLibraryInfo.h"
 #include "llvm/Support/ConstantRange.h"
 #include "llvm/Support/GetElementPtrTypeIterator.h"
 #include "llvm/Support/PatternMatch.h"
@@ -473,7 +474,7 @@ FoldCmpLoadFromIndexedGlobal(GetElementPtrInst *GEP, GlobalVariable *GV,
 /// If we can't emit an optimized form for this expression, this returns null.
 ///
 static Value *EvaluateGEPOffsetExpression(User *GEP, InstCombiner &IC) {
-  TargetData &TD = *IC.getTargetData();
+  DataLayout &TD = *IC.getDataLayout();
   gep_type_iterator GTI = gep_type_begin(GEP);
 
   // Check to see if this gep only has a single variable index.  If so, and if
@@ -2355,8 +2356,25 @@ Instruction *InstCombiner::visitICmpInst(ICmpInst &I) {
         // Try not to increase register pressure.
         BO0->hasOneUse() && BO1->hasOneUse()) {
       // Determine Y and Z in the form icmp (X+Y), (X+Z).
-      Value *Y = (A == C || A == D) ? B : A;
-      Value *Z = (C == A || C == B) ? D : C;
+      Value *Y, *Z;
+      if (A == C) {
+        // C + B == C + D  ->  B == D
+        Y = B;
+        Z = D;
+      } else if (A == D) {
+        // D + B == C + D  ->  B == C
+        Y = B;
+        Z = C;
+      } else if (B == C) {
+        // A + C == C + D  ->  A == D
+        Y = A;
+        Z = D;
+      } else {
+        assert(B == D);
+        // A + D == C + D  ->  A == C
+        Y = A;
+        Z = C;
+      }
       return new ICmpInst(Pred, Y, Z);
     }
 
@@ -2894,10 +2912,6 @@ Instruction *InstCombiner::visitFCmpInst(FCmpInst &I) {
         if (!RHSF)
           break;
 
-        // We can't convert a PPC double double.
-        if (RHSF->getType()->isPPC_FP128Ty())
-          break;
-
         const fltSemantics *Sem;
         // FIXME: This shouldn't be here.
         if (LHSExt->getSrcTy()->isHalfTy())
@@ -2910,6 +2924,8 @@ Instruction *InstCombiner::visitFCmpInst(FCmpInst &I) {
           Sem = &APFloat::IEEEquad;
         else if (LHSExt->getSrcTy()->isX86_FP80Ty())
           Sem = &APFloat::x87DoubleExtended;
+        else if (LHSExt->getSrcTy()->isPPC_FP128Ty())
+          Sem = &APFloat::PPCDoubleDouble;
         else
           break;
 
@@ -2985,6 +3001,44 @@ Instruction *InstCombiner::visitFCmpInst(FCmpInst &I) {
                 return Res;
         }
         break;
+      case Instruction::Call: {
+        CallInst *CI = cast<CallInst>(LHSI);
+        LibFunc::Func Func;
+        // Various optimization for fabs compared with zero.
+        if (RHSC->isNullValue() && CI->getCalledFunction() &&
+            TLI->getLibFunc(CI->getCalledFunction()->getName(), Func) &&
+            TLI->has(Func)) {
+          if (Func == LibFunc::fabs || Func == LibFunc::fabsf ||
+              Func == LibFunc::fabsl) {
+            switch (I.getPredicate()) {
+            default: break;
+            // fabs(x) < 0 --> false
+            case FCmpInst::FCMP_OLT:
+              return ReplaceInstUsesWith(I, Builder->getFalse());
+            // fabs(x) > 0 --> x != 0
+            case FCmpInst::FCMP_OGT:
+              return new FCmpInst(FCmpInst::FCMP_ONE, CI->getArgOperand(0),
+                                  RHSC);
+            // fabs(x) <= 0 --> x == 0
+            case FCmpInst::FCMP_OLE:
+              return new FCmpInst(FCmpInst::FCMP_OEQ, CI->getArgOperand(0),
+                                  RHSC);
+            // fabs(x) >= 0 --> !isnan(x)
+            case FCmpInst::FCMP_OGE:
+              return new FCmpInst(FCmpInst::FCMP_ORD, CI->getArgOperand(0),
+                                  RHSC);
+            // fabs(x) == 0 --> x == 0
+            // fabs(x) != 0 --> x != 0
+            case FCmpInst::FCMP_OEQ:
+            case FCmpInst::FCMP_UEQ:
+            case FCmpInst::FCMP_ONE:
+            case FCmpInst::FCMP_UNE:
+              return new FCmpInst(I.getPredicate(), CI->getArgOperand(0),
+                                  RHSC);
+            }
+          }
+        }
+      }
       }
   }
 
diff --git a/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp b/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
index c485844..4d106fc 100644
--- a/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
+++ b/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp
@@ -14,13 +14,161 @@
 #include "InstCombine.h"
 #include "llvm/IntrinsicInst.h"
 #include "llvm/Analysis/Loads.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/Transforms/Utils/Local.h"
 #include "llvm/ADT/Statistic.h"
 using namespace llvm;
 
-STATISTIC(NumDeadStore, "Number of dead stores eliminated");
+STATISTIC(NumDeadStore,    "Number of dead stores eliminated");
+STATISTIC(NumGlobalCopies, "Number of allocas copied from constant global");
+
+/// pointsToConstantGlobal - Return true if V (possibly indirectly) points to
+/// some part of a constant global variable.  This intentionally only accepts
+/// constant expressions because we can't rewrite arbitrary instructions.
+static bool pointsToConstantGlobal(Value *V) {
+  if (GlobalVariable *GV = dyn_cast<GlobalVariable>(V))
+    return GV->isConstant();
+  if (ConstantExpr *CE = dyn_cast<ConstantExpr>(V))
+    if (CE->getOpcode() == Instruction::BitCast ||
+        CE->getOpcode() == Instruction::GetElementPtr)
+      return pointsToConstantGlobal(CE->getOperand(0));
+  return false;
+}
+
+/// isOnlyCopiedFromConstantGlobal - Recursively walk the uses of a (derived)
+/// pointer to an alloca.  Ignore any reads of the pointer, return false if we
+/// see any stores or other unknown uses.  If we see pointer arithmetic, keep
+/// track of whether it moves the pointer (with IsOffset) but otherwise traverse
+/// the uses.  If we see a memcpy/memmove that targets an unoffseted pointer to
+/// the alloca, and if the source pointer is a pointer to a constant global, we
+/// can optimize this.
+static bool
+isOnlyCopiedFromConstantGlobal(Value *V, MemTransferInst *&TheCopy,
+                               SmallVectorImpl<Instruction *> &ToDelete,
+                               bool IsOffset = false) {
+  // We track lifetime intrinsics as we encounter them.  If we decide to go
+  // ahead and replace the value with the global, this lets the caller quickly
+  // eliminate the markers.
+
+  for (Value::use_iterator UI = V->use_begin(), E = V->use_end(); UI!=E; ++UI) {
+    User *U = cast<Instruction>(*UI);
+
+    if (LoadInst *LI = dyn_cast<LoadInst>(U)) {
+      // Ignore non-volatile loads, they are always ok.
+      if (!LI->isSimple()) return false;
+      continue;
+    }
+
+    if (BitCastInst *BCI = dyn_cast<BitCastInst>(U)) {
+      // If uses of the bitcast are ok, we are ok.
+      if (!isOnlyCopiedFromConstantGlobal(BCI, TheCopy, ToDelete, IsOffset))
+        return false;
+      continue;
+    }
+    if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(U)) {
+      // If the GEP has all zero indices, it doesn't offset the pointer.  If it
+      // doesn't, it does.
+      if (!isOnlyCopiedFromConstantGlobal(GEP, TheCopy, ToDelete,
+                                          IsOffset || !GEP->hasAllZeroIndices()))
+        return false;
+      continue;
+    }
+
+    if (CallSite CS = U) {
+      // If this is the function being called then we treat it like a load and
+      // ignore it.
+      if (CS.isCallee(UI))
+        continue;
+
+      // If this is a readonly/readnone call site, then we know it is just a
+      // load (but one that potentially returns the value itself), so we can
+      // ignore it if we know that the value isn't captured.
+      unsigned ArgNo = CS.getArgumentNo(UI);
+      if (CS.onlyReadsMemory() &&
+          (CS.getInstruction()->use_empty() || CS.doesNotCapture(ArgNo)))
+        continue;
+
+      // If this is being passed as a byval argument, the caller is making a
+      // copy, so it is only a read of the alloca.
+      if (CS.isByValArgument(ArgNo))
+        continue;
+    }
+
+    // Lifetime intrinsics can be handled by the caller.
+    if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(U)) {
+      if (II->getIntrinsicID() == Intrinsic::lifetime_start ||
+          II->getIntrinsicID() == Intrinsic::lifetime_end) {
+        assert(II->use_empty() && "Lifetime markers have no result to use!");
+        ToDelete.push_back(II);
+        continue;
+      }
+    }
+
+    // If this is isn't our memcpy/memmove, reject it as something we can't
+    // handle.
+    MemTransferInst *MI = dyn_cast<MemTransferInst>(U);
+    if (MI == 0)
+      return false;
+
+    // If the transfer is using the alloca as a source of the transfer, then
+    // ignore it since it is a load (unless the transfer is volatile).
+    if (UI.getOperandNo() == 1) {
+      if (MI->isVolatile()) return false;
+      continue;
+    }
+
+    // If we already have seen a copy, reject the second one.
+    if (TheCopy) return false;
+
+    // If the pointer has been offset from the start of the alloca, we can't
+    // safely handle this.
+    if (IsOffset) return false;
+
+    // If the memintrinsic isn't using the alloca as the dest, reject it.
+    if (UI.getOperandNo() != 0) return false;
+
+    // If the source of the memcpy/move is not a constant global, reject it.
+    if (!pointsToConstantGlobal(MI->getSource()))
+      return false;
+
+    // Otherwise, the transform is safe.  Remember the copy instruction.
+    TheCopy = MI;
+  }
+  return true;
+}
+
+/// isOnlyCopiedFromConstantGlobal - Return true if the specified alloca is only
+/// modified by a copy from a constant global.  If we can prove this, we can
+/// replace any uses of the alloca with uses of the global directly.
+static MemTransferInst *
+isOnlyCopiedFromConstantGlobal(AllocaInst *AI,
+                               SmallVectorImpl<Instruction *> &ToDelete) {
+  MemTransferInst *TheCopy = 0;
+  if (isOnlyCopiedFromConstantGlobal(AI, TheCopy, ToDelete))
+    return TheCopy;
+  return 0;
+}
+
+/// getPointeeAlignment - Compute the minimum alignment of the value pointed
+/// to by the given pointer.
+static unsigned getPointeeAlignment(Value *V, const DataLayout &TD) {
+  if (ConstantExpr *CE = dyn_cast<ConstantExpr>(V))
+    if (CE->getOpcode() == Instruction::BitCast ||
+        (CE->getOpcode() == Instruction::GetElementPtr &&
+         cast<GEPOperator>(CE)->hasAllZeroIndices()))
+      return getPointeeAlignment(CE->getOperand(0), TD);
+
+  if (GlobalVariable *GV = dyn_cast<GlobalVariable>(V))
+    if (!GV->isDeclaration())
+      return TD.getPreferredAlignment(GV);
+
+  if (PointerType *PT = dyn_cast<PointerType>(V->getType()))
+    if (PT->getElementType()->isSized())
+      return TD.getABITypeAlignment(PT->getElementType());
+
+  return 0;
+}
 
 Instruction *InstCombiner::visitAllocaInst(AllocaInst &AI) {
   // Ensure that the alloca array size argument has type intptr_t, so that
@@ -99,12 +247,16 @@ Instruction *InstCombiner::visitAllocaInst(AllocaInst &AI) {
           return &AI;
         }
 
+        // If the alignment of the entry block alloca is 0 (unspecified),
+        // assign it the preferred alignment.
+        if (EntryAI->getAlignment() == 0)
+          EntryAI->setAlignment(
+            TD->getPrefTypeAlignment(EntryAI->getAllocatedType()));
         // Replace this zero-sized alloca with the one at the start of the entry
         // block after ensuring that the address will be aligned enough for both
         // types.
-        unsigned MaxAlign =
-          std::max(TD->getPrefTypeAlignment(EntryAI->getAllocatedType()),
-                   TD->getPrefTypeAlignment(AI.getAllocatedType()));
+        unsigned MaxAlign = std::max(EntryAI->getAlignment(),
+                                     AI.getAlignment());
         EntryAI->setAlignment(MaxAlign);
         if (AI.getType() != EntryAI->getType())
           return new BitCastInst(EntryAI, AI.getType());
@@ -113,6 +265,31 @@ Instruction *InstCombiner::visitAllocaInst(AllocaInst &AI) {
     }
   }
 
+  if (TD) {
+    // Check to see if this allocation is only modified by a memcpy/memmove from
+    // a constant global whose alignment is equal to or exceeds that of the
+    // allocation.  If this is the case, we can change all users to use
+    // the constant global instead.  This is commonly produced by the CFE by
+    // constructs like "void foo() { int A[] = {1,2,3,4,5,6,7,8,9...}; }" if 'A'
+    // is only subsequently read.
+    SmallVector<Instruction *, 4> ToDelete;
+    if (MemTransferInst *Copy = isOnlyCopiedFromConstantGlobal(&AI, ToDelete)) {
+      if (AI.getAlignment() <= getPointeeAlignment(Copy->getSource(), *TD)) {
+        DEBUG(dbgs() << "Found alloca equal to global: " << AI << '\n');
+        DEBUG(dbgs() << "  memcpy = " << *Copy << '\n');
+        for (unsigned i = 0, e = ToDelete.size(); i != e; ++i)
+          EraseInstFromFunction(*ToDelete[i]);
+        Constant *TheSrc = cast<Constant>(Copy->getSource());
+        Instruction *NewI
+          = ReplaceInstUsesWith(AI, ConstantExpr::getBitCast(TheSrc,
+                                                             AI.getType()));
+        EraseInstFromFunction(*Copy);
+        ++NumGlobalCopies;
+        return NewI;
+      }
+    }
+  }
+
   // At last, use the generic allocation site handler to aggressively remove
   // unused allocas.
   return visitAllocSite(AI);
@@ -121,7 +298,7 @@ Instruction *InstCombiner::visitAllocaInst(AllocaInst &AI) {
 
 /// InstCombineLoadCast - Fold 'load (cast P)' -> cast (load P)' when possible.
 static Instruction *InstCombineLoadCast(InstCombiner &IC, LoadInst &LI,
-                                        const TargetData *TD) {
+                                        const DataLayout *TD) {
   User *CI = cast<User>(LI.getOperand(0));
   Value *CastOp = CI->getOperand(0);
 
@@ -151,14 +328,14 @@ static Instruction *InstCombineLoadCast(InstCombiner &IC, LoadInst &LI,
             SrcPTy = SrcTy->getElementType();
           }
 
-      if (IC.getTargetData() &&
+      if (IC.getDataLayout() &&
           (SrcPTy->isIntegerTy() || SrcPTy->isPointerTy() || 
             SrcPTy->isVectorTy()) &&
           // Do not allow turning this into a load of an integer, which is then
           // casted to a pointer, this pessimizes pointer analysis a lot.
           (SrcPTy->isPointerTy() == LI.getType()->isPointerTy()) &&
-          IC.getTargetData()->getTypeSizeInBits(SrcPTy) ==
-               IC.getTargetData()->getTypeSizeInBits(DestPTy)) {
+          IC.getDataLayout()->getTypeSizeInBits(SrcPTy) ==
+               IC.getDataLayout()->getTypeSizeInBits(DestPTy)) {
 
         // Okay, we are casting from one integer or pointer type to another of
         // the same size.  Instead of casting the pointer before the load, cast
@@ -336,11 +513,11 @@ static Instruction *InstCombineStoreToCast(InstCombiner &IC, StoreInst &SI) {
   
   // If the pointers point into different address spaces or if they point to
   // values with different sizes, we can't do the transformation.
-  if (!IC.getTargetData() ||
+  if (!IC.getDataLayout() ||
       SrcTy->getAddressSpace() != 
         cast<PointerType>(CI->getType())->getAddressSpace() ||
-      IC.getTargetData()->getTypeSizeInBits(SrcPTy) !=
-      IC.getTargetData()->getTypeSizeInBits(DestPTy))
+      IC.getDataLayout()->getTypeSizeInBits(SrcPTy) !=
+      IC.getDataLayout()->getTypeSizeInBits(DestPTy))
     return 0;
 
   // Okay, we are casting from one integer or pointer type to another of
diff --git a/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp b/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
index 35a0bbb..cefe45e 100644
--- a/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
+++ b/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
@@ -37,7 +37,7 @@ static Value *simplifyValueKnownNonZero(Value *V, InstCombiner &IC) {
   if (match(V, m_LShr(m_OneUse(m_Shl(m_Value(PowerOf2), m_Value(A))),
                       m_Value(B))) &&
       // The "1" can be any value known to be a power of 2.
-      isPowerOfTwo(PowerOf2, IC.getTargetData())) {
+      isPowerOfTwo(PowerOf2, IC.getDataLayout())) {
     A = IC.Builder->CreateSub(A, B);
     return IC.Builder->CreateShl(PowerOf2, A);
   }
@@ -46,7 +46,7 @@ static Value *simplifyValueKnownNonZero(Value *V, InstCombiner &IC) {
   // inexact.  Similarly for <<.
   if (BinaryOperator *I = dyn_cast<BinaryOperator>(V))
     if (I->isLogicalShift() &&
-        isPowerOfTwo(I->getOperand(0), IC.getTargetData())) {
+        isPowerOfTwo(I->getOperand(0), IC.getDataLayout())) {
       // We know that this is an exact/nuw shift and that the input is a
       // non-zero context as well.
       if (Value *V2 = simplifyValueKnownNonZero(I->getOperand(0), IC)) {
@@ -462,12 +462,23 @@ Instruction *InstCombiner::visitUDiv(BinaryOperator &I) {
     }
   }
 
+  // (x lshr C1) udiv C2 --> x udiv (C2 << C1)
+  if (ConstantInt *C2 = dyn_cast<ConstantInt>(Op1)) {
+    Value *X;
+    ConstantInt *C1;
+    if (match(Op0, m_LShr(m_Value(X), m_ConstantInt(C1)))) {
+      APInt NC = C2->getValue().shl(C1->getLimitedValue(C1->getBitWidth()-1));
+      return BinaryOperator::CreateUDiv(X, Builder->getInt(NC));
+    }
+  }
+
   // X udiv (C1 << N), where C1 is "1<<C2"  -->  X >> (N+C2)
   { const APInt *CI; Value *N;
     if (match(Op1, m_Shl(m_Power2(CI), m_Value(N))) ||
         match(Op1, m_ZExt(m_Shl(m_Power2(CI), m_Value(N))))) {
       if (*CI != 1)
-        N = Builder->CreateAdd(N, ConstantInt::get(I.getType(),CI->logBase2()));
+        N = Builder->CreateAdd(N,
+                               ConstantInt::get(N->getType(), CI->logBase2()));
       if (ZExtInst *Z = dyn_cast<ZExtInst>(Op1))
         N = Builder->CreateZExt(N, Z->getDestTy());
       if (I.isExact())
diff --git a/lib/Transforms/InstCombine/InstCombinePHI.cpp b/lib/Transforms/InstCombine/InstCombinePHI.cpp
index 664546c..de9c77e 100644
--- a/lib/Transforms/InstCombine/InstCombinePHI.cpp
+++ b/lib/Transforms/InstCombine/InstCombinePHI.cpp
@@ -13,7 +13,7 @@
 
 #include "InstCombine.h"
 #include "llvm/Analysis/InstructionSimplify.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/STLExtras.h"
 using namespace llvm;
diff --git a/lib/Transforms/InstCombine/InstCombineSelect.cpp b/lib/Transforms/InstCombine/InstCombineSelect.cpp
index 291e800..a2d4c88 100644
--- a/lib/Transforms/InstCombine/InstCombineSelect.cpp
+++ b/lib/Transforms/InstCombine/InstCombineSelect.cpp
@@ -287,7 +287,7 @@ Instruction *InstCombiner::FoldSelectIntoOp(SelectInst &SI, Value *TrueVal,
 /// SimplifyWithOpReplaced - See if V simplifies when its operand Op is
 /// replaced with RepOp.
 static Value *SimplifyWithOpReplaced(Value *V, Value *Op, Value *RepOp,
-                                     const TargetData *TD,
+                                     const DataLayout *TD,
                                      const TargetLibraryInfo *TLI) {
   // Trivial replacement.
   if (V == Op)
@@ -333,6 +333,10 @@ static Value *SimplifyWithOpReplaced(Value *V, Value *Op, Value *RepOp,
 
     // All operands were constants, fold it.
     if (ConstOps.size() == I->getNumOperands()) {
+      if (CmpInst *C = dyn_cast<CmpInst>(I))
+        return ConstantFoldCompareInstOperands(C->getPredicate(), ConstOps[0],
+                                               ConstOps[1], TD, TLI);
+
       if (LoadInst *LI = dyn_cast<LoadInst>(I))
         if (!LI->isVolatile())
           return ConstantFoldLoadFromConstPtr(ConstOps[0], TD);
@@ -903,7 +907,7 @@ Instruction *InstCombiner::visitSelectInst(SelectInst &SI) {
     return &SI;
   }
 
-  if (VectorType* VecTy = dyn_cast<VectorType>(SI.getType())) {
+  if (VectorType *VecTy = dyn_cast<VectorType>(SI.getType())) {
     unsigned VWidth = VecTy->getNumElements();
     APInt UndefElts(VWidth, 0);
     APInt AllOnesEltMask(APInt::getAllOnesValue(VWidth));
@@ -912,6 +916,28 @@ Instruction *InstCombiner::visitSelectInst(SelectInst &SI) {
         return ReplaceInstUsesWith(SI, V);
       return &SI;
     }
+
+    if (ConstantVector *CV = dyn_cast<ConstantVector>(CondVal)) {
+      // Form a shufflevector instruction.
+      SmallVector<Constant *, 8> Mask(VWidth);
+      Type *Int32Ty = Type::getInt32Ty(CV->getContext());
+      for (unsigned i = 0; i != VWidth; ++i) {
+        Constant *Elem = cast<Constant>(CV->getOperand(i));
+        if (ConstantInt *E = dyn_cast<ConstantInt>(Elem))
+          Mask[i] = ConstantInt::get(Int32Ty, i + (E->isZero() ? VWidth : 0));
+        else if (isa<UndefValue>(Elem))
+          Mask[i] = UndefValue::get(Int32Ty);
+        else
+          return 0;
+      }
+      Constant *MaskVal = ConstantVector::get(Mask);
+      Value *V = Builder->CreateShuffleVector(TrueVal, FalseVal, MaskVal);
+      return ReplaceInstUsesWith(SI, V);
+    }
+
+    if (isa<ConstantAggregateZero>(CondVal)) {
+      return ReplaceInstUsesWith(SI, FalseVal);
+    }
   }
 
   return 0;
diff --git a/lib/Transforms/InstCombine/InstCombineShifts.cpp b/lib/Transforms/InstCombine/InstCombineShifts.cpp
index 4bb2403..57021f1 100644
--- a/lib/Transforms/InstCombine/InstCombineShifts.cpp
+++ b/lib/Transforms/InstCombine/InstCombineShifts.cpp
@@ -190,7 +190,7 @@ static Value *GetShiftedValue(Value *V, unsigned NumBits, bool isLeftShift,
       V = IC.Builder->CreateLShr(C, NumBits);
     // If we got a constantexpr back, try to simplify it with TD info.
     if (ConstantExpr *CE = dyn_cast<ConstantExpr>(V))
-      V = ConstantFoldConstantExpression(CE, IC.getTargetData(),
+      V = ConstantFoldConstantExpression(CE, IC.getDataLayout(),
                                          IC.getTargetLibraryInfo());
     return V;
   }
diff --git a/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp b/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
index 54be8ed..602b203 100644
--- a/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
+++ b/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
@@ -14,7 +14,7 @@
 
 
 #include "InstCombine.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/IntrinsicInst.h"
 
 using namespace llvm;
diff --git a/lib/Transforms/InstCombine/InstCombineVectorOps.cpp b/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
index cf60f0f..dd7ea14 100644
--- a/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
+++ b/lib/Transforms/InstCombine/InstCombineVectorOps.cpp
@@ -636,8 +636,11 @@ Instruction *InstCombiner::visitShuffleVectorInst(ShuffleVectorInst &SVI) {
 
       // If LHS's width is changed, shift the mask value accordingly.
       // If newRHS == NULL, i.e. LHSOp0 == RHSOp0, we want to remap any
-      // references to RHSOp0 to LHSOp0, so we don't need to shift the mask.
-      if (eltMask >= 0 && newRHS != NULL)
+      // references from RHSOp0 to LHSOp0, so we don't need to shift the mask.
+      // If newRHS == newLHS, we want to remap any references from newRHS to
+      // newLHS so that we can properly identify splats that may occur due to
+      // obfuscation accross the two vectors.
+      if (eltMask >= 0 && newRHS != NULL && newLHS != newRHS)
         eltMask += newLHSWidth;
     }
 
diff --git a/lib/Transforms/InstCombine/InstCombineWorklist.h b/lib/Transforms/InstCombine/InstCombineWorklist.h
index 99a02fc..ea654ae 100644
--- a/lib/Transforms/InstCombine/InstCombineWorklist.h
+++ b/lib/Transforms/InstCombine/InstCombineWorklist.h
@@ -26,8 +26,8 @@ class LLVM_LIBRARY_VISIBILITY InstCombineWorklist {
   SmallVector<Instruction*, 256> Worklist;
   DenseMap<Instruction*, unsigned> WorklistMap;
   
-  void operator=(const InstCombineWorklist&RHS);   // DO NOT IMPLEMENT
-  InstCombineWorklist(const InstCombineWorklist&); // DO NOT IMPLEMENT
+  void operator=(const InstCombineWorklist&RHS) LLVM_DELETED_FUNCTION;
+  InstCombineWorklist(const InstCombineWorklist&) LLVM_DELETED_FUNCTION;
 public:
   InstCombineWorklist() {}
   
diff --git a/lib/Transforms/InstCombine/InstructionCombining.cpp b/lib/Transforms/InstCombine/InstructionCombining.cpp
index 68ecd51..9a46f25 100644
--- a/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -40,7 +40,7 @@
 #include "llvm/Analysis/ConstantFolding.h"
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/MemoryBuiltins.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetLibraryInfo.h"
 #include "llvm/Transforms/Utils/Local.h"
 #include "llvm/Support/CFG.h"
@@ -88,7 +88,7 @@ void InstCombiner::getAnalysisUsage(AnalysisUsage &AU) const {
 
 
 Value *InstCombiner::EmitGEPOffset(User *GEP) {
-  return llvm::EmitGEPOffset(Builder, *getTargetData(), GEP);
+  return llvm::EmitGEPOffset(Builder, *getDataLayout(), GEP);
 }
 
 /// ShouldChangeType - Return true if it is desirable to convert a computation
@@ -805,6 +805,244 @@ static bool shouldMergeGEPs(GEPOperator &GEP, GEPOperator &Src) {
   return true;
 }
 
+/// Descale - Return a value X such that Val = X * Scale, or null if none.  If
+/// the multiplication is known not to overflow then NoSignedWrap is set.
+Value *InstCombiner::Descale(Value *Val, APInt Scale, bool &NoSignedWrap) {
+  assert(isa<IntegerType>(Val->getType()) && "Can only descale integers!");
+  assert(cast<IntegerType>(Val->getType())->getBitWidth() ==
+         Scale.getBitWidth() && "Scale not compatible with value!");
+
+  // If Val is zero or Scale is one then Val = Val * Scale.
+  if (match(Val, m_Zero()) || Scale == 1) {
+    NoSignedWrap = true;
+    return Val;
+  }
+
+  // If Scale is zero then it does not divide Val.
+  if (Scale.isMinValue())
+    return 0;
+
+  // Look through chains of multiplications, searching for a constant that is
+  // divisible by Scale.  For example, descaling X*(Y*(Z*4)) by a factor of 4
+  // will find the constant factor 4 and produce X*(Y*Z).  Descaling X*(Y*8) by
+  // a factor of 4 will produce X*(Y*2).  The principle of operation is to bore
+  // down from Val:
+  //
+  //     Val = M1 * X          ||   Analysis starts here and works down
+  //      M1 = M2 * Y          ||   Doesn't descend into terms with more
+  //      M2 =  Z * 4          \/   than one use
+  //
+  // Then to modify a term at the bottom:
+  //
+  //     Val = M1 * X
+  //      M1 =  Z * Y          ||   Replaced M2 with Z
+  //
+  // Then to work back up correcting nsw flags.
+
+  // Op - the term we are currently analyzing.  Starts at Val then drills down.
+  // Replaced with its descaled value before exiting from the drill down loop.
+  Value *Op = Val;
+
+  // Parent - initially null, but after drilling down notes where Op came from.
+  // In the example above, Parent is (Val, 0) when Op is M1, because M1 is the
+  // 0'th operand of Val.
+  std::pair<Instruction*, unsigned> Parent;
+
+  // RequireNoSignedWrap - Set if the transform requires a descaling at deeper
+  // levels that doesn't overflow.
+  bool RequireNoSignedWrap = false;
+
+  // logScale - log base 2 of the scale.  Negative if not a power of 2.
+  int32_t logScale = Scale.exactLogBase2();
+
+  for (;; Op = Parent.first->getOperand(Parent.second)) { // Drill down
+
+    if (ConstantInt *CI = dyn_cast<ConstantInt>(Op)) {
+      // If Op is a constant divisible by Scale then descale to the quotient.
+      APInt Quotient(Scale), Remainder(Scale); // Init ensures right bitwidth.
+      APInt::sdivrem(CI->getValue(), Scale, Quotient, Remainder);
+      if (!Remainder.isMinValue())
+        // Not divisible by Scale.
+        return 0;
+      // Replace with the quotient in the parent.
+      Op = ConstantInt::get(CI->getType(), Quotient);
+      NoSignedWrap = true;
+      break;
+    }
+
+    if (BinaryOperator *BO = dyn_cast<BinaryOperator>(Op)) {
+
+      if (BO->getOpcode() == Instruction::Mul) {
+        // Multiplication.
+        NoSignedWrap = BO->hasNoSignedWrap();
+        if (RequireNoSignedWrap && !NoSignedWrap)
+          return 0;
+
+        // There are three cases for multiplication: multiplication by exactly
+        // the scale, multiplication by a constant different to the scale, and
+        // multiplication by something else.
+        Value *LHS = BO->getOperand(0);
+        Value *RHS = BO->getOperand(1);
+
+        if (ConstantInt *CI = dyn_cast<ConstantInt>(RHS)) {
+          // Multiplication by a constant.
+          if (CI->getValue() == Scale) {
+            // Multiplication by exactly the scale, replace the multiplication
+            // by its left-hand side in the parent.
+            Op = LHS;
+            break;
+          }
+
+          // Otherwise drill down into the constant.
+          if (!Op->hasOneUse())
+            return 0;
+
+          Parent = std::make_pair(BO, 1);
+          continue;
+        }
+
+        // Multiplication by something else. Drill down into the left-hand side
+        // since that's where the reassociate pass puts the good stuff.
+        if (!Op->hasOneUse())
+          return 0;
+
+        Parent = std::make_pair(BO, 0);
+        continue;
+      }
+
+      if (logScale > 0 && BO->getOpcode() == Instruction::Shl &&
+          isa<ConstantInt>(BO->getOperand(1))) {
+        // Multiplication by a power of 2.
+        NoSignedWrap = BO->hasNoSignedWrap();
+        if (RequireNoSignedWrap && !NoSignedWrap)
+          return 0;
+
+        Value *LHS = BO->getOperand(0);
+        int32_t Amt = cast<ConstantInt>(BO->getOperand(1))->
+          getLimitedValue(Scale.getBitWidth());
+        // Op = LHS << Amt.
+
+        if (Amt == logScale) {
+          // Multiplication by exactly the scale, replace the multiplication
+          // by its left-hand side in the parent.
+          Op = LHS;
+          break;
+        }
+        if (Amt < logScale || !Op->hasOneUse())
+          return 0;
+
+        // Multiplication by more than the scale.  Reduce the multiplying amount
+        // by the scale in the parent.
+        Parent = std::make_pair(BO, 1);
+        Op = ConstantInt::get(BO->getType(), Amt - logScale);
+        break;
+      }
+    }
+
+    if (!Op->hasOneUse())
+      return 0;
+
+    if (CastInst *Cast = dyn_cast<CastInst>(Op)) {
+      if (Cast->getOpcode() == Instruction::SExt) {
+        // Op is sign-extended from a smaller type, descale in the smaller type.
+        unsigned SmallSize = Cast->getSrcTy()->getPrimitiveSizeInBits();
+        APInt SmallScale = Scale.trunc(SmallSize);
+        // Suppose Op = sext X, and we descale X as Y * SmallScale.  We want to
+        // descale Op as (sext Y) * Scale.  In order to have
+        //   sext (Y * SmallScale) = (sext Y) * Scale
+        // some conditions need to hold however: SmallScale must sign-extend to
+        // Scale and the multiplication Y * SmallScale should not overflow.
+        if (SmallScale.sext(Scale.getBitWidth()) != Scale)
+          // SmallScale does not sign-extend to Scale.
+          return 0;
+        assert(SmallScale.exactLogBase2() == logScale);
+        // Require that Y * SmallScale must not overflow.
+        RequireNoSignedWrap = true;
+
+        // Drill down through the cast.
+        Parent = std::make_pair(Cast, 0);
+        Scale = SmallScale;
+        continue;
+      }
+
+      if (Cast->getOpcode() == Instruction::Trunc) {
+        // Op is truncated from a larger type, descale in the larger type.
+        // Suppose Op = trunc X, and we descale X as Y * sext Scale.  Then
+        //   trunc (Y * sext Scale) = (trunc Y) * Scale
+        // always holds.  However (trunc Y) * Scale may overflow even if
+        // trunc (Y * sext Scale) does not, so nsw flags need to be cleared
+        // from this point up in the expression (see later).
+        if (RequireNoSignedWrap)
+          return 0;
+
+        // Drill down through the cast.
+        unsigned LargeSize = Cast->getSrcTy()->getPrimitiveSizeInBits();
+        Parent = std::make_pair(Cast, 0);
+        Scale = Scale.sext(LargeSize);
+        if (logScale + 1 == (int32_t)Cast->getType()->getPrimitiveSizeInBits())
+          logScale = -1;
+        assert(Scale.exactLogBase2() == logScale);
+        continue;
+      }
+    }
+
+    // Unsupported expression, bail out.
+    return 0;
+  }
+
+  // We know that we can successfully descale, so from here on we can safely
+  // modify the IR.  Op holds the descaled version of the deepest term in the
+  // expression.  NoSignedWrap is 'true' if multiplying Op by Scale is known
+  // not to overflow.
+
+  if (!Parent.first)
+    // The expression only had one term.
+    return Op;
+
+  // Rewrite the parent using the descaled version of its operand.
+  assert(Parent.first->hasOneUse() && "Drilled down when more than one use!");
+  assert(Op != Parent.first->getOperand(Parent.second) &&
+         "Descaling was a no-op?");
+  Parent.first->setOperand(Parent.second, Op);
+  Worklist.Add(Parent.first);
+
+  // Now work back up the expression correcting nsw flags.  The logic is based
+  // on the following observation: if X * Y is known not to overflow as a signed
+  // multiplication, and Y is replaced by a value Z with smaller absolute value,
+  // then X * Z will not overflow as a signed multiplication either.  As we work
+  // our way up, having NoSignedWrap 'true' means that the descaled value at the
+  // current level has strictly smaller absolute value than the original.
+  Instruction *Ancestor = Parent.first;
+  do {
+    if (BinaryOperator *BO = dyn_cast<BinaryOperator>(Ancestor)) {
+      // If the multiplication wasn't nsw then we can't say anything about the
+      // value of the descaled multiplication, and we have to clear nsw flags
+      // from this point on up.
+      bool OpNoSignedWrap = BO->hasNoSignedWrap();
+      NoSignedWrap &= OpNoSignedWrap;
+      if (NoSignedWrap != OpNoSignedWrap) {
+        BO->setHasNoSignedWrap(NoSignedWrap);
+        Worklist.Add(Ancestor);
+      }
+    } else if (Ancestor->getOpcode() == Instruction::Trunc) {
+      // The fact that the descaled input to the trunc has smaller absolute
+      // value than the original input doesn't tell us anything useful about
+      // the absolute values of the truncations.
+      NoSignedWrap = false;
+    }
+    assert((Ancestor->getOpcode() != Instruction::SExt || NoSignedWrap) &&
+           "Failed to keep proper track of nsw flags while drilling down?");
+
+    if (Ancestor == Val)
+      // Got to the top, all done!
+      return Val;
+
+    // Move up one level in the expression.
+    assert(Ancestor->hasOneUse() && "Drilled down when more than one use!");
+    Ancestor = Ancestor->use_back();
+  } while (1);
+}
+
 Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) {
   SmallVector<Value*, 8> Ops(GEP.op_begin(), GEP.op_end());
 
@@ -817,7 +1055,7 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) {
   // by multiples of a zero size type with zero.
   if (TD) {
     bool MadeChange = false;
-    Type *IntPtrTy = TD->getIntPtrType(GEP.getContext());
+    Type *IntPtrTy = TD->getIntPtrType(GEP.getPointerOperandType());
 
     gep_type_iterator GTI = gep_type_begin(GEP);
     for (User::op_iterator I = GEP.op_begin() + 1, E = GEP.op_end();
@@ -836,7 +1074,7 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) {
         }
 
       Type *IndexTy = (*I)->getType();
-      if (IndexTy != IntPtrTy && !IndexTy->isVectorTy()) {
+      if (IndexTy != IntPtrTy) {
         // If we are using a wider index than needed for this platform, shrink
         // it to what we need.  If narrower, sign-extend it to what we need.
         // This explicit cast can make subsequent optimizations more obvious.
@@ -855,7 +1093,7 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) {
     if (!shouldMergeGEPs(*cast<GEPOperator>(&GEP), *Src))
       return 0;
 
-    // Note that if our source is a gep chain itself that we wait for that
+    // Note that if our source is a gep chain itself then we wait for that
     // chain to be resolved before we perform this transformation.  This
     // avoids us creating a TON of code in some cases.
     if (GEPOperator *SrcGEP =
@@ -987,63 +1225,74 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) {
       }
 
       // Transform things like:
+      // %V = mul i64 %N, 4
+      // %t = getelementptr i8* bitcast (i32* %arr to i8*), i32 %V
+      // into:  %t1 = getelementptr i32* %arr, i32 %N; bitcast
+      if (TD && ResElTy->isSized() && SrcElTy->isSized()) {
+        // Check that changing the type amounts to dividing the index by a scale
+        // factor.
+        uint64_t ResSize = TD->getTypeAllocSize(ResElTy);
+        uint64_t SrcSize = TD->getTypeAllocSize(SrcElTy);
+        if (ResSize && SrcSize % ResSize == 0) {
+          Value *Idx = GEP.getOperand(1);
+          unsigned BitWidth = Idx->getType()->getPrimitiveSizeInBits();
+          uint64_t Scale = SrcSize / ResSize;
+
+          // Earlier transforms ensure that the index has type IntPtrType, which
+          // considerably simplifies the logic by eliminating implicit casts.
+          assert(Idx->getType() == TD->getIntPtrType(GEP.getContext()) &&
+                 "Index not cast to pointer width?");
+
+          bool NSW;
+          if (Value *NewIdx = Descale(Idx, APInt(BitWidth, Scale), NSW)) {
+            // Successfully decomposed Idx as NewIdx * Scale, form a new GEP.
+            // If the multiplication NewIdx * Scale may overflow then the new
+            // GEP may not be "inbounds".
+            Value *NewGEP = GEP.isInBounds() && NSW ?
+              Builder->CreateInBoundsGEP(StrippedPtr, NewIdx, GEP.getName()) :
+              Builder->CreateGEP(StrippedPtr, NewIdx, GEP.getName());
+            // The NewGEP must be pointer typed, so must the old one -> BitCast
+            return new BitCastInst(NewGEP, GEP.getType());
+          }
+        }
+      }
+
+      // Similarly, transform things like:
       // getelementptr i8* bitcast ([100 x double]* X to i8*), i32 %tmp
       //   (where tmp = 8*tmp2) into:
       // getelementptr [100 x double]* %arr, i32 0, i32 %tmp2; bitcast
-
-      if (TD && SrcElTy->isArrayTy() && ResElTy->isIntegerTy(8)) {
+      if (TD && ResElTy->isSized() && SrcElTy->isSized() &&
+          SrcElTy->isArrayTy()) {
+        // Check that changing to the array element type amounts to dividing the
+        // index by a scale factor.
+        uint64_t ResSize = TD->getTypeAllocSize(ResElTy);
         uint64_t ArrayEltSize =
-            TD->getTypeAllocSize(cast<ArrayType>(SrcElTy)->getElementType());
-
-        // Check to see if "tmp" is a scale by a multiple of ArrayEltSize.  We
-        // allow either a mul, shift, or constant here.
-        Value *NewIdx = 0;
-        ConstantInt *Scale = 0;
-        if (ArrayEltSize == 1) {
-          NewIdx = GEP.getOperand(1);
-          Scale = ConstantInt::get(cast<IntegerType>(NewIdx->getType()), 1);
-        } else if (ConstantInt *CI = dyn_cast<ConstantInt>(GEP.getOperand(1))) {
-          NewIdx = ConstantInt::get(CI->getType(), 1);
-          Scale = CI;
-        } else if (Instruction *Inst =dyn_cast<Instruction>(GEP.getOperand(1))){
-          if (Inst->getOpcode() == Instruction::Shl &&
-              isa<ConstantInt>(Inst->getOperand(1))) {
-            ConstantInt *ShAmt = cast<ConstantInt>(Inst->getOperand(1));
-            uint32_t ShAmtVal = ShAmt->getLimitedValue(64);
-            Scale = ConstantInt::get(cast<IntegerType>(Inst->getType()),
-                                     1ULL << ShAmtVal);
-            NewIdx = Inst->getOperand(0);
-          } else if (Inst->getOpcode() == Instruction::Mul &&
-                     isa<ConstantInt>(Inst->getOperand(1))) {
-            Scale = cast<ConstantInt>(Inst->getOperand(1));
-            NewIdx = Inst->getOperand(0);
+          TD->getTypeAllocSize(cast<ArrayType>(SrcElTy)->getElementType());
+        if (ResSize && ArrayEltSize % ResSize == 0) {
+          Value *Idx = GEP.getOperand(1);
+          unsigned BitWidth = Idx->getType()->getPrimitiveSizeInBits();
+          uint64_t Scale = ArrayEltSize / ResSize;
+
+          // Earlier transforms ensure that the index has type IntPtrType, which
+          // considerably simplifies the logic by eliminating implicit casts.
+          assert(Idx->getType() == TD->getIntPtrType(GEP.getContext()) &&
+                 "Index not cast to pointer width?");
+
+          bool NSW;
+          if (Value *NewIdx = Descale(Idx, APInt(BitWidth, Scale), NSW)) {
+            // Successfully decomposed Idx as NewIdx * Scale, form a new GEP.
+            // If the multiplication NewIdx * Scale may overflow then the new
+            // GEP may not be "inbounds".
+            Value *Off[2];
+            Off[0] = Constant::getNullValue(Type::getInt32Ty(GEP.getContext()));
+            Off[1] = NewIdx;
+            Value *NewGEP = GEP.isInBounds() && NSW ?
+              Builder->CreateInBoundsGEP(StrippedPtr, Off, GEP.getName()) :
+              Builder->CreateGEP(StrippedPtr, Off, GEP.getName());
+            // The NewGEP must be pointer typed, so must the old one -> BitCast
+            return new BitCastInst(NewGEP, GEP.getType());
           }
         }
-
-        // If the index will be to exactly the right offset with the scale taken
-        // out, perform the transformation. Note, we don't know whether Scale is
-        // signed or not. We'll use unsigned version of division/modulo
-        // operation after making sure Scale doesn't have the sign bit set.
-        if (ArrayEltSize && Scale && Scale->getSExtValue() >= 0LL &&
-            Scale->getZExtValue() % ArrayEltSize == 0) {
-          Scale = ConstantInt::get(Scale->getType(),
-                                   Scale->getZExtValue() / ArrayEltSize);
-          if (Scale->getZExtValue() != 1) {
-            Constant *C = ConstantExpr::getIntegerCast(Scale, NewIdx->getType(),
-                                                       false /*ZExt*/);
-            NewIdx = Builder->CreateMul(NewIdx, C, "idxscale");
-          }
-
-          // Insert the new GEP instruction.
-          Value *Idx[2];
-          Idx[0] = Constant::getNullValue(Type::getInt32Ty(GEP.getContext()));
-          Idx[1] = NewIdx;
-          Value *NewGEP = GEP.isInBounds() ?
-            Builder->CreateInBoundsGEP(StrippedPtr, Idx, GEP.getName()):
-            Builder->CreateGEP(StrippedPtr, Idx, GEP.getName());
-          // The NewGEP must be pointer typed, so must the old one -> BitCast
-          return new BitCastInst(NewGEP, GEP.getType());
-        }
       }
     }
   }
@@ -1068,7 +1317,7 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) {
         // If the bitcast is of an allocation, and the allocation will be
         // converted to match the type of the cast, don't touch this.
         if (isa<AllocaInst>(BCI->getOperand(0)) ||
-            isAllocationFn(BCI->getOperand(0))) {
+            isAllocationFn(BCI->getOperand(0), TLI)) {
           // See if the bitcast simplifies, if so, don't nuke this GEP yet.
           if (Instruction *I = visitBitCast(*BCI)) {
             if (I != BCI) {
@@ -1107,7 +1356,8 @@ Instruction *InstCombiner::visitGetElementPtrInst(GetElementPtrInst &GEP) {
 
 
 static bool
-isAllocSiteRemovable(Instruction *AI, SmallVectorImpl<WeakVH> &Users) {
+isAllocSiteRemovable(Instruction *AI, SmallVectorImpl<WeakVH> &Users,
+                     const TargetLibraryInfo *TLI) {
   SmallVector<Instruction*, 4> Worklist;
   Worklist.push_back(AI);
 
@@ -1163,7 +1413,7 @@ isAllocSiteRemovable(Instruction *AI, SmallVectorImpl<WeakVH> &Users) {
           }
         }
 
-        if (isFreeCall(I)) {
+        if (isFreeCall(I, TLI)) {
           Users.push_back(I);
           continue;
         }
@@ -1188,7 +1438,7 @@ Instruction *InstCombiner::visitAllocSite(Instruction &MI) {
   // to null and free calls, delete the calls and replace the comparisons with
   // true or false as appropriate.
   SmallVector<WeakVH, 64> Users;
-  if (isAllocSiteRemovable(&MI, Users)) {
+  if (isAllocSiteRemovable(&MI, Users, TLI)) {
     for (unsigned i = 0, e = Users.size(); i != e; ++i) {
       Instruction *I = cast_or_null<Instruction>(&*Users[i]);
       if (!I) continue;
@@ -1853,7 +2103,7 @@ static bool TryToSinkInstruction(Instruction *I, BasicBlock *DestBlock) {
 static bool AddReachableCodeToWorklist(BasicBlock *BB,
                                        SmallPtrSet<BasicBlock*, 64> &Visited,
                                        InstCombiner &IC,
-                                       const TargetData *TD,
+                                       const DataLayout *TD,
                                        const TargetLibraryInfo *TLI) {
   bool MadeIRChange = false;
   SmallVector<BasicBlock*, 256> Worklist;
@@ -1872,7 +2122,7 @@ static bool AddReachableCodeToWorklist(BasicBlock *BB,
       Instruction *Inst = BBI++;
 
       // DCE instruction if trivially dead.
-      if (isInstructionTriviallyDead(Inst)) {
+      if (isInstructionTriviallyDead(Inst, TLI)) {
         ++NumDeadInst;
         DEBUG(errs() << "IC: DCE: " << *Inst << '\n');
         Inst->eraseFromParent();
@@ -2002,7 +2252,7 @@ bool InstCombiner::DoOneIteration(Function &F, unsigned Iteration) {
     if (I == 0) continue;  // skip null values.
 
     // Check to see if we can DCE the instruction.
-    if (isInstructionTriviallyDead(I)) {
+    if (isInstructionTriviallyDead(I, TLI)) {
       DEBUG(errs() << "IC: DCE: " << *I << '\n');
       EraseInstFromFunction(*I);
       ++NumDeadInst;
@@ -2102,7 +2352,7 @@ bool InstCombiner::DoOneIteration(Function &F, unsigned Iteration) {
 
         // If the instruction was modified, it's possible that it is now dead.
         // if so, remove it.
-        if (isInstructionTriviallyDead(I)) {
+        if (isInstructionTriviallyDead(I, TLI)) {
           EraseInstFromFunction(*I);
         } else {
           Worklist.Add(I);
@@ -2117,9 +2367,27 @@ bool InstCombiner::DoOneIteration(Function &F, unsigned Iteration) {
   return MadeIRChange;
 }
 
+namespace {
+class InstCombinerLibCallSimplifier : public LibCallSimplifier {
+  InstCombiner *IC;
+public:
+  InstCombinerLibCallSimplifier(const DataLayout *TD,
+                                const TargetLibraryInfo *TLI,
+                                InstCombiner *IC)
+    : LibCallSimplifier(TD, TLI) {
+    this->IC = IC;
+  }
+
+  /// replaceAllUsesWith - override so that instruction replacement
+  /// can be defined in terms of the instruction combiner framework.
+  virtual void replaceAllUsesWith(Instruction *I, Value *With) const {
+    IC->ReplaceInstUsesWith(*I, With);
+  }
+};
+}
 
 bool InstCombiner::runOnFunction(Function &F) {
-  TD = getAnalysisIfAvailable<TargetData>();
+  TD = getAnalysisIfAvailable<DataLayout>();
   TLI = &getAnalysis<TargetLibraryInfo>();
 
   /// Builder - This is an IRBuilder that automatically inserts new
@@ -2129,6 +2397,9 @@ bool InstCombiner::runOnFunction(Function &F) {
                InstCombineIRInserter(Worklist));
   Builder = &TheBuilder;
 
+  InstCombinerLibCallSimplifier TheSimplifier(TD, TLI, this);
+  Simplifier = &TheSimplifier;
+
   bool EverMadeChange = false;
 
   // Lower dbg.declare intrinsics otherwise their value may be clobbered
diff --git a/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/lib/Transforms/Instrumentation/AddressSanitizer.cpp
index 17b83ce..b7be462 100644
--- a/lib/Transforms/Instrumentation/AddressSanitizer.cpp
+++ b/lib/Transforms/Instrumentation/AddressSanitizer.cpp
@@ -15,7 +15,7 @@
 
 #define DEBUG_TYPE "asan"
 
-#include "FunctionBlackList.h"
+#include "BlackList.h"
 #include "llvm/Function.h"
 #include "llvm/IRBuilder.h"
 #include "llvm/InlineAsm.h"
@@ -35,7 +35,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/system_error.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Transforms/Instrumentation.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
@@ -61,6 +61,8 @@ static const int   kAsanCtorAndCtorPriority = 1;
 static const char *kAsanReportErrorTemplate = "__asan_report_";
 static const char *kAsanRegisterGlobalsName = "__asan_register_globals";
 static const char *kAsanUnregisterGlobalsName = "__asan_unregister_globals";
+static const char *kAsanPoisonGlobalsName = "__asan_before_dynamic_init";
+static const char *kAsanUnpoisonGlobalsName = "__asan_after_dynamic_init";
 static const char *kAsanInitName = "__asan_init";
 static const char *kAsanHandleNoReturnName = "__asan_handle_no_return";
 static const char *kAsanMappingOffsetName = "__asan_mapping_offset";
@@ -106,6 +108,8 @@ static cl::opt<bool> ClUseAfterReturn("asan-use-after-return",
 // This flag may need to be replaced with -f[no]asan-globals.
 static cl::opt<bool> ClGlobals("asan-globals",
        cl::desc("Handle global objects"), cl::Hidden, cl::init(true));
+static cl::opt<bool> ClInitializers("asan-initialization-order",
+       cl::desc("Handle C++ initializer order"), cl::Hidden, cl::init(false));
 static cl::opt<bool> ClMemIntrin("asan-memintrin",
        cl::desc("Handle memset/memcpy/memmove"), cl::Hidden, cl::init(true));
 // This flag may need to be replaced with -fasan-blacklist.
@@ -144,41 +148,33 @@ static cl::opt<int> ClDebugMax("asan-debug-max", cl::desc("Debug man inst"),
                                cl::Hidden, cl::init(-1));
 
 namespace {
-
-/// An object of this type is created while instrumenting every function.
-struct AsanFunctionContext {
-  AsanFunctionContext(Function &Function) : F(Function) { }
-
-  Function &F;
-};
-
 /// AddressSanitizer: instrument the code in module to find memory bugs.
-struct AddressSanitizer : public ModulePass {
+struct AddressSanitizer : public FunctionPass {
   AddressSanitizer();
   virtual const char *getPassName() const;
-  void instrumentMop(AsanFunctionContext &AFC, Instruction *I);
-  void instrumentAddress(AsanFunctionContext &AFC,
-                         Instruction *OrigIns, IRBuilder<> &IRB,
+  void instrumentMop(Instruction *I);
+  void instrumentAddress(Instruction *OrigIns, IRBuilder<> &IRB,
                          Value *Addr, uint32_t TypeSize, bool IsWrite);
   Value *createSlowPathCmp(IRBuilder<> &IRB, Value *AddrLong,
                            Value *ShadowValue, uint32_t TypeSize);
   Instruction *generateCrashCode(Instruction *InsertBefore, Value *Addr,
                                  bool IsWrite, size_t AccessSizeIndex);
-  bool instrumentMemIntrinsic(AsanFunctionContext &AFC, MemIntrinsic *MI);
-  void instrumentMemIntrinsicParam(AsanFunctionContext &AFC,
-                                   Instruction *OrigIns, Value *Addr,
+  bool instrumentMemIntrinsic(MemIntrinsic *MI);
+  void instrumentMemIntrinsicParam(Instruction *OrigIns, Value *Addr,
                                    Value *Size,
                                    Instruction *InsertBefore, bool IsWrite);
   Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
-  bool handleFunction(Module &M, Function &F);
+  bool runOnFunction(Function &F);
+  void createInitializerPoisonCalls(Module &M,
+                                    Value *FirstAddr, Value *LastAddr);
   bool maybeInsertAsanInitAtFunctionEntry(Function &F);
-  bool poisonStackInFunction(Module &M, Function &F);
-  virtual bool runOnModule(Module &M);
+  bool poisonStackInFunction(Function &F);
+  virtual bool doInitialization(Module &M);
+  virtual bool doFinalization(Module &M);
   bool insertGlobalRedzones(Module &M);
   static char ID;  // Pass identification, replacement for typeid
 
  private:
-
   uint64_t getAllocaSizeInBytes(AllocaInst *AI) {
     Type *Ty = AI->getAllocatedType();
     uint64_t SizeInBytes = TD->getTypeAllocSize(Ty);
@@ -194,12 +190,15 @@ struct AddressSanitizer : public ModulePass {
   }
 
   Function *checkInterfaceFunction(Constant *FuncOrBitcast);
+  bool ShouldInstrumentGlobal(GlobalVariable *G);
   void PoisonStack(const ArrayRef<AllocaInst*> &AllocaVec, IRBuilder<> IRB,
                    Value *ShadowBase, bool DoPoison);
   bool LooksLikeCodeInBug11395(Instruction *I);
+  void FindDynamicInitializers(Module &M);
+  bool HasDynamicInitializer(GlobalVariable *G);
 
   LLVMContext *C;
-  TargetData *TD;
+  DataLayout *TD;
   uint64_t MappingOffset;
   int MappingScale;
   size_t RedzoneSize;
@@ -208,11 +207,15 @@ struct AddressSanitizer : public ModulePass {
   Type *IntptrPtrTy;
   Function *AsanCtorFunction;
   Function *AsanInitFunction;
+  Function *AsanStackMallocFunc, *AsanStackFreeFunc;
+  Function *AsanHandleNoReturnFunc;
   Instruction *CtorInsertBefore;
-  OwningPtr<FunctionBlackList> BL;
+  OwningPtr<BlackList> BL;
   // This array is indexed by AccessIsWrite and log2(AccessSize).
   Function *AsanErrorCallback[2][kNumberOfAccessSizes];
   InlineAsm *EmptyAsm;
+  SmallSet<GlobalValue*, 32> DynamicallyInitializedGlobals;
+  SmallSet<GlobalValue*, 32> GlobalsCreatedByAsan;
 };
 
 }  // namespace
@@ -221,8 +224,8 @@ char AddressSanitizer::ID = 0;
 INITIALIZE_PASS(AddressSanitizer, "asan",
     "AddressSanitizer: detects use-after-free and out-of-bounds bugs.",
     false, false)
-AddressSanitizer::AddressSanitizer() : ModulePass(ID) { }
-ModulePass *llvm::createAddressSanitizerPass() {
+AddressSanitizer::AddressSanitizer() : FunctionPass(ID) { }
+FunctionPass *llvm::createAddressSanitizerPass() {
   return new AddressSanitizer();
 }
 
@@ -243,38 +246,6 @@ static GlobalVariable *createPrivateGlobalForString(Module &M, StringRef Str) {
                             GlobalValue::PrivateLinkage, StrConst, "");
 }
 
-// Split the basic block and insert an if-then code.
-// Before:
-//   Head
-//   Cmp
-//   Tail
-// After:
-//   Head
-//   if (Cmp)
-//     ThenBlock
-//   Tail
-//
-// ThenBlock block is created and its terminator is returned.
-// If Unreachable, ThenBlock is terminated with UnreachableInst, otherwise
-// it is terminated with BranchInst to Tail.
-static TerminatorInst *splitBlockAndInsertIfThen(Value *Cmp, bool Unreachable) {
-  Instruction *SplitBefore = cast<Instruction>(Cmp)->getNextNode();
-  BasicBlock *Head = SplitBefore->getParent();
-  BasicBlock *Tail = Head->splitBasicBlock(SplitBefore);
-  TerminatorInst *HeadOldTerm = Head->getTerminator();
-  LLVMContext &C = Head->getParent()->getParent()->getContext();
-  BasicBlock *ThenBlock = BasicBlock::Create(C, "", Head->getParent(), Tail);
-  TerminatorInst *CheckTerm;
-  if (Unreachable)
-    CheckTerm = new UnreachableInst(C, ThenBlock);
-  else
-    CheckTerm = BranchInst::Create(Tail, ThenBlock);
-  BranchInst *HeadNewTerm =
-    BranchInst::Create(/*ifTrue*/ThenBlock, /*ifFalse*/Tail, Cmp);
-  ReplaceInstWithInst(HeadOldTerm, HeadNewTerm);
-  return CheckTerm;
-}
-
 Value *AddressSanitizer::memToShadow(Value *Shadow, IRBuilder<> &IRB) {
   // Shadow >> scale
   Shadow = IRB.CreateLShr(Shadow, MappingScale);
@@ -286,12 +257,12 @@ Value *AddressSanitizer::memToShadow(Value *Shadow, IRBuilder<> &IRB) {
 }
 
 void AddressSanitizer::instrumentMemIntrinsicParam(
-    AsanFunctionContext &AFC, Instruction *OrigIns,
+    Instruction *OrigIns,
     Value *Addr, Value *Size, Instruction *InsertBefore, bool IsWrite) {
   // Check the first byte.
   {
     IRBuilder<> IRB(InsertBefore);
-    instrumentAddress(AFC, OrigIns, IRB, Addr, 8, IsWrite);
+    instrumentAddress(OrigIns, IRB, Addr, 8, IsWrite);
   }
   // Check the last byte.
   {
@@ -301,13 +272,12 @@ void AddressSanitizer::instrumentMemIntrinsicParam(
     SizeMinusOne = IRB.CreateIntCast(SizeMinusOne, IntptrTy, false);
     Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
     Value *AddrPlusSizeMinisOne = IRB.CreateAdd(AddrLong, SizeMinusOne);
-    instrumentAddress(AFC, OrigIns, IRB, AddrPlusSizeMinisOne, 8, IsWrite);
+    instrumentAddress(OrigIns, IRB, AddrPlusSizeMinisOne, 8, IsWrite);
   }
 }
 
 // Instrument memset/memmove/memcpy
-bool AddressSanitizer::instrumentMemIntrinsic(AsanFunctionContext &AFC,
-                                              MemIntrinsic *MI) {
+bool AddressSanitizer::instrumentMemIntrinsic(MemIntrinsic *MI) {
   Value *Dst = MI->getDest();
   MemTransferInst *MemTran = dyn_cast<MemTransferInst>(MI);
   Value *Src = MemTran ? MemTran->getSource() : 0;
@@ -323,12 +293,12 @@ bool AddressSanitizer::instrumentMemIntrinsic(AsanFunctionContext &AFC,
 
     Value *Cmp = IRB.CreateICmpNE(Length,
                                   Constant::getNullValue(Length->getType()));
-    InsertBefore = splitBlockAndInsertIfThen(Cmp, false);
+    InsertBefore = SplitBlockAndInsertIfThen(cast<Instruction>(Cmp), false);
   }
 
-  instrumentMemIntrinsicParam(AFC, MI, Dst, Length, InsertBefore, true);
+  instrumentMemIntrinsicParam(MI, Dst, Length, InsertBefore, true);
   if (Src)
-    instrumentMemIntrinsicParam(AFC, MI, Src, Length, InsertBefore, false);
+    instrumentMemIntrinsicParam(MI, Src, Length, InsertBefore, false);
   return true;
 }
 
@@ -358,14 +328,50 @@ static Value *isInterestingMemoryAccess(Instruction *I, bool *IsWrite) {
   return NULL;
 }
 
-void AddressSanitizer::instrumentMop(AsanFunctionContext &AFC, Instruction *I) {
-  bool IsWrite;
+void AddressSanitizer::FindDynamicInitializers(Module& M) {
+  // Clang generates metadata identifying all dynamically initialized globals.
+  NamedMDNode *DynamicGlobals =
+      M.getNamedMetadata("llvm.asan.dynamically_initialized_globals");
+  if (!DynamicGlobals)
+    return;
+  for (int i = 0, n = DynamicGlobals->getNumOperands(); i < n; ++i) {
+    MDNode *MDN = DynamicGlobals->getOperand(i);
+    assert(MDN->getNumOperands() == 1);
+    Value *VG = MDN->getOperand(0);
+    // The optimizer may optimize away a global entirely, in which case we
+    // cannot instrument access to it.
+    if (!VG)
+      continue;
+
+    GlobalVariable *G = cast<GlobalVariable>(VG);
+    DynamicallyInitializedGlobals.insert(G);
+  }
+}
+// Returns true if a global variable is initialized dynamically in this TU.
+bool AddressSanitizer::HasDynamicInitializer(GlobalVariable *G) {
+  return DynamicallyInitializedGlobals.count(G);
+}
+
+void AddressSanitizer::instrumentMop(Instruction *I) {
+  bool IsWrite = false;
   Value *Addr = isInterestingMemoryAccess(I, &IsWrite);
   assert(Addr);
-  if (ClOpt && ClOptGlobals && isa<GlobalVariable>(Addr)) {
-    // We are accessing a global scalar variable. Nothing to catch here.
-    return;
+  if (ClOpt && ClOptGlobals) {
+    if (GlobalVariable *G = dyn_cast<GlobalVariable>(Addr)) {
+      // If initialization order checking is disabled, a simple access to a
+      // dynamically initialized global is always valid.
+      if (!ClInitializers)
+        return;
+      // If a global variable does not have dynamic initialization we don't
+      // have to instrument it.  However, if a global has external linkage, we
+      // assume it has dynamic initialization, as it may have an initializer
+      // in a different TU.
+      if (G->getLinkage() != GlobalVariable::ExternalLinkage &&
+          !HasDynamicInitializer(G))
+        return;
+    }
   }
+
   Type *OrigPtrTy = Addr->getType();
   Type *OrigTy = cast<PointerType>(OrigPtrTy)->getElementType();
 
@@ -379,7 +385,7 @@ void AddressSanitizer::instrumentMop(AsanFunctionContext &AFC, Instruction *I) {
   }
 
   IRBuilder<> IRB(I);
-  instrumentAddress(AFC, I, IRB, Addr, TypeSize, IsWrite);
+  instrumentAddress(I, IRB, Addr, TypeSize, IsWrite);
 }
 
 // Validate the result of Module::getOrInsertFunction called for an interface
@@ -424,8 +430,7 @@ Value *AddressSanitizer::createSlowPathCmp(IRBuilder<> &IRB, Value *AddrLong,
   return IRB.CreateICmpSGE(LastAccessedByte, ShadowValue);
 }
 
-void AddressSanitizer::instrumentAddress(AsanFunctionContext &AFC,
-                                         Instruction *OrigIns,
+void AddressSanitizer::instrumentAddress(Instruction *OrigIns,
                                          IRBuilder<> &IRB, Value *Addr,
                                          uint32_t TypeSize, bool IsWrite) {
   Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
@@ -444,17 +449,19 @@ void AddressSanitizer::instrumentAddress(AsanFunctionContext &AFC,
   TerminatorInst *CrashTerm = 0;
 
   if (ClAlwaysSlowPath || (TypeSize < 8 * Granularity)) {
-    TerminatorInst *CheckTerm = splitBlockAndInsertIfThen(Cmp, false);
+    TerminatorInst *CheckTerm =
+        SplitBlockAndInsertIfThen(cast<Instruction>(Cmp), false);
     assert(dyn_cast<BranchInst>(CheckTerm)->isUnconditional());
     BasicBlock *NextBB = CheckTerm->getSuccessor(0);
     IRB.SetInsertPoint(CheckTerm);
     Value *Cmp2 = createSlowPathCmp(IRB, AddrLong, ShadowValue, TypeSize);
-    BasicBlock *CrashBlock = BasicBlock::Create(*C, "", &AFC.F, NextBB);
+    BasicBlock *CrashBlock =
+        BasicBlock::Create(*C, "", NextBB->getParent(), NextBB);
     CrashTerm = new UnreachableInst(*C, CrashBlock);
     BranchInst *NewTerm = BranchInst::Create(CrashBlock, NextBB, Cmp2);
     ReplaceInstWithInst(CheckTerm, NewTerm);
   } else {
-    CrashTerm = splitBlockAndInsertIfThen(Cmp, true);
+    CrashTerm = SplitBlockAndInsertIfThen(cast<Instruction>(Cmp), true);
   }
 
   Instruction *Crash =
@@ -462,68 +469,108 @@ void AddressSanitizer::instrumentAddress(AsanFunctionContext &AFC,
   Crash->setDebugLoc(OrigIns->getDebugLoc());
 }
 
+void AddressSanitizer::createInitializerPoisonCalls(Module &M,
+                                                    Value *FirstAddr,
+                                                    Value *LastAddr) {
+  // We do all of our poisoning and unpoisoning within _GLOBAL__I_a.
+  Function *GlobalInit = M.getFunction("_GLOBAL__I_a");
+  // If that function is not present, this TU contains no globals, or they have
+  // all been optimized away
+  if (!GlobalInit)
+    return;
+
+  // Set up the arguments to our poison/unpoison functions.
+  IRBuilder<> IRB(GlobalInit->begin()->getFirstInsertionPt());
+
+  // Declare our poisoning and unpoisoning functions.
+  Function *AsanPoisonGlobals = checkInterfaceFunction(M.getOrInsertFunction(
+      kAsanPoisonGlobalsName, IRB.getVoidTy(), IntptrTy, IntptrTy, NULL));
+  AsanPoisonGlobals->setLinkage(Function::ExternalLinkage);
+  Function *AsanUnpoisonGlobals = checkInterfaceFunction(M.getOrInsertFunction(
+      kAsanUnpoisonGlobalsName, IRB.getVoidTy(), NULL));
+  AsanUnpoisonGlobals->setLinkage(Function::ExternalLinkage);
+
+  // Add a call to poison all external globals before the given function starts.
+  IRB.CreateCall2(AsanPoisonGlobals, FirstAddr, LastAddr);
+
+  // Add calls to unpoison all globals before each return instruction.
+  for (Function::iterator I = GlobalInit->begin(), E = GlobalInit->end();
+      I != E; ++I) {
+    if (ReturnInst *RI = dyn_cast<ReturnInst>(I->getTerminator())) {
+      CallInst::Create(AsanUnpoisonGlobals, "", RI);
+    }
+  }
+}
+
+bool AddressSanitizer::ShouldInstrumentGlobal(GlobalVariable *G) {
+  Type *Ty = cast<PointerType>(G->getType())->getElementType();
+  DEBUG(dbgs() << "GLOBAL: " << *G << "\n");
+
+  if (BL->isIn(*G)) return false;
+  if (!Ty->isSized()) return false;
+  if (!G->hasInitializer()) return false;
+  if (GlobalsCreatedByAsan.count(G)) return false;  // Our own global.
+  // Touch only those globals that will not be defined in other modules.
+  // Don't handle ODR type linkages since other modules may be built w/o asan.
+  if (G->getLinkage() != GlobalVariable::ExternalLinkage &&
+      G->getLinkage() != GlobalVariable::PrivateLinkage &&
+      G->getLinkage() != GlobalVariable::InternalLinkage)
+    return false;
+  // Two problems with thread-locals:
+  //   - The address of the main thread's copy can't be computed at link-time.
+  //   - Need to poison all copies, not just the main thread's one.
+  if (G->isThreadLocal())
+    return false;
+  // For now, just ignore this Alloca if the alignment is large.
+  if (G->getAlignment() > RedzoneSize) return false;
+
+  // Ignore all the globals with the names starting with "\01L_OBJC_".
+  // Many of those are put into the .cstring section. The linker compresses
+  // that section by removing the spare \0s after the string terminator, so
+  // our redzones get broken.
+  if ((G->getName().find("\01L_OBJC_") == 0) ||
+      (G->getName().find("\01l_OBJC_") == 0)) {
+    DEBUG(dbgs() << "Ignoring \\01L_OBJC_* global: " << *G);
+    return false;
+  }
+
+  if (G->hasSection()) {
+    StringRef Section(G->getSection());
+    // Ignore the globals from the __OBJC section. The ObjC runtime assumes
+    // those conform to /usr/lib/objc/runtime.h, so we can't add redzones to
+    // them.
+    if ((Section.find("__OBJC,") == 0) ||
+        (Section.find("__DATA, __objc_") == 0)) {
+      DEBUG(dbgs() << "Ignoring ObjC runtime global: " << *G);
+      return false;
+    }
+    // See http://code.google.com/p/address-sanitizer/issues/detail?id=32
+    // Constant CFString instances are compiled in the following way:
+    //  -- the string buffer is emitted into
+    //     __TEXT,__cstring,cstring_literals
+    //  -- the constant NSConstantString structure referencing that buffer
+    //     is placed into __DATA,__cfstring
+    // Therefore there's no point in placing redzones into __DATA,__cfstring.
+    // Moreover, it causes the linker to crash on OS X 10.7
+    if (Section.find("__DATA,__cfstring") == 0) {
+      DEBUG(dbgs() << "Ignoring CFString: " << *G);
+      return false;
+    }
+  }
+
+  return true;
+}
+
 // This function replaces all global variables with new variables that have
 // trailing redzones. It also creates a function that poisons
 // redzones and inserts this function into llvm.global_ctors.
 bool AddressSanitizer::insertGlobalRedzones(Module &M) {
   SmallVector<GlobalVariable *, 16> GlobalsToChange;
 
-  for (Module::GlobalListType::iterator G = M.getGlobalList().begin(),
-       E = M.getGlobalList().end(); G != E; ++G) {
-    Type *Ty = cast<PointerType>(G->getType())->getElementType();
-    DEBUG(dbgs() << "GLOBAL: " << *G);
-
-    if (!Ty->isSized()) continue;
-    if (!G->hasInitializer()) continue;
-    // Touch only those globals that will not be defined in other modules.
-    // Don't handle ODR type linkages since other modules may be built w/o asan.
-    if (G->getLinkage() != GlobalVariable::ExternalLinkage &&
-        G->getLinkage() != GlobalVariable::PrivateLinkage &&
-        G->getLinkage() != GlobalVariable::InternalLinkage)
-      continue;
-    // Two problems with thread-locals:
-    //   - The address of the main thread's copy can't be computed at link-time.
-    //   - Need to poison all copies, not just the main thread's one.
-    if (G->isThreadLocal())
-      continue;
-    // For now, just ignore this Alloca if the alignment is large.
-    if (G->getAlignment() > RedzoneSize) continue;
-
-    // Ignore all the globals with the names starting with "\01L_OBJC_".
-    // Many of those are put into the .cstring section. The linker compresses
-    // that section by removing the spare \0s after the string terminator, so
-    // our redzones get broken.
-    if ((G->getName().find("\01L_OBJC_") == 0) ||
-        (G->getName().find("\01l_OBJC_") == 0)) {
-      DEBUG(dbgs() << "Ignoring \\01L_OBJC_* global: " << *G);
-      continue;
-    }
-
-    if (G->hasSection()) {
-      StringRef Section(G->getSection());
-      // Ignore the globals from the __OBJC section. The ObjC runtime assumes
-      // those conform to /usr/lib/objc/runtime.h, so we can't add redzones to
-      // them.
-      if ((Section.find("__OBJC,") == 0) ||
-          (Section.find("__DATA, __objc_") == 0)) {
-        DEBUG(dbgs() << "Ignoring ObjC runtime global: " << *G);
-        continue;
-      }
-      // See http://code.google.com/p/address-sanitizer/issues/detail?id=32
-      // Constant CFString instances are compiled in the following way:
-      //  -- the string buffer is emitted into
-      //     __TEXT,__cstring,cstring_literals
-      //  -- the constant NSConstantString structure referencing that buffer
-      //     is placed into __DATA,__cfstring
-      // Therefore there's no point in placing redzones into __DATA,__cfstring.
-      // Moreover, it causes the linker to crash on OS X 10.7
-      if (Section.find("__DATA,__cfstring") == 0) {
-        DEBUG(dbgs() << "Ignoring CFString: " << *G);
-        continue;
-      }
-    }
-
-    GlobalsToChange.push_back(G);
+  for (Module::GlobalListType::iterator G = M.global_begin(),
+       E = M.global_end(); G != E; ++G) {
+    if (ShouldInstrumentGlobal(G))
+      GlobalsToChange.push_back(G);
   }
 
   size_t n = GlobalsToChange.size();
@@ -534,13 +581,22 @@ bool AddressSanitizer::insertGlobalRedzones(Module &M) {
   //   size_t size;
   //   size_t size_with_redzone;
   //   const char *name;
+  //   size_t has_dynamic_init;
   // We initialize an array of such structures and pass it to a run-time call.
   StructType *GlobalStructTy = StructType::get(IntptrTy, IntptrTy,
-                                               IntptrTy, IntptrTy, NULL);
-  SmallVector<Constant *, 16> Initializers(n);
+                                               IntptrTy, IntptrTy,
+                                               IntptrTy, NULL);
+  SmallVector<Constant *, 16> Initializers(n), DynamicInit;
 
   IRBuilder<> IRB(CtorInsertBefore);
 
+  if (ClInitializers)
+    FindDynamicInitializers(M);
+
+  // The addresses of the first and last dynamically initialized globals in
+  // this TU.  Used in initialization order checking.
+  Value *FirstDynamic = 0, *LastDynamic = 0;
+
   for (size_t i = 0; i < n; i++) {
     GlobalVariable *G = GlobalsToChange[i];
     PointerType *PtrTy = cast<PointerType>(G->getType());
@@ -549,6 +605,10 @@ bool AddressSanitizer::insertGlobalRedzones(Module &M) {
     uint64_t RightRedzoneSize = RedzoneSize +
         (RedzoneSize - (SizeInBytes % RedzoneSize));
     Type *RightRedZoneTy = ArrayType::get(IRB.getInt8Ty(), RightRedzoneSize);
+    // Determine whether this global should be poisoned in initialization.
+    bool GlobalHasDynamicInitializer = HasDynamicInitializer(G);
+    // Don't check initialization order if this global is blacklisted.
+    GlobalHasDynamicInitializer &= !BL->isInInit(*G);
 
     StructType *NewTy = StructType::get(Ty, RightRedZoneTy, NULL);
     Constant *NewInitializer = ConstantStruct::get(
@@ -583,8 +643,17 @@ bool AddressSanitizer::insertGlobalRedzones(Module &M) {
         ConstantInt::get(IntptrTy, SizeInBytes),
         ConstantInt::get(IntptrTy, SizeInBytes + RightRedzoneSize),
         ConstantExpr::getPointerCast(Name, IntptrTy),
+        ConstantInt::get(IntptrTy, GlobalHasDynamicInitializer),
         NULL);
-    DEBUG(dbgs() << "NEW GLOBAL:\n" << *NewGlobal);
+
+    // Populate the first and last globals declared in this TU.
+    if (ClInitializers && GlobalHasDynamicInitializer) {
+      LastDynamic = ConstantExpr::getPointerCast(NewGlobal, IntptrTy);
+      if (FirstDynamic == 0)
+        FirstDynamic = LastDynamic;
+    }
+
+    DEBUG(dbgs() << "NEW GLOBAL: " << *NewGlobal << "\n");
   }
 
   ArrayType *ArrayOfGlobalStructTy = ArrayType::get(GlobalStructTy, n);
@@ -592,8 +661,13 @@ bool AddressSanitizer::insertGlobalRedzones(Module &M) {
       M, ArrayOfGlobalStructTy, false, GlobalVariable::PrivateLinkage,
       ConstantArray::get(ArrayOfGlobalStructTy, Initializers), "");
 
+  // Create calls for poisoning before initializers run and unpoisoning after.
+  if (ClInitializers && FirstDynamic && LastDynamic)
+    createInitializerPoisonCalls(M, FirstDynamic, LastDynamic);
+
   Function *AsanRegisterGlobals = checkInterfaceFunction(M.getOrInsertFunction(
-      kAsanRegisterGlobalsName, IRB.getVoidTy(), IntptrTy, IntptrTy, NULL));
+      kAsanRegisterGlobalsName, IRB.getVoidTy(),
+      IntptrTy, IntptrTy, NULL));
   AsanRegisterGlobals->setLinkage(Function::ExternalLinkage);
 
   IRB.CreateCall2(AsanRegisterGlobals,
@@ -623,12 +697,13 @@ bool AddressSanitizer::insertGlobalRedzones(Module &M) {
 }
 
 // virtual
-bool AddressSanitizer::runOnModule(Module &M) {
+bool AddressSanitizer::doInitialization(Module &M) {
   // Initialize the private fields. No one has accessed them before.
-  TD = getAnalysisIfAvailable<TargetData>();
+  TD = getAnalysisIfAvailable<DataLayout>();
+
   if (!TD)
     return false;
-  BL.reset(new FunctionBlackList(ClBlackListFile));
+  BL.reset(new BlackList(ClBlackListFile));
 
   C = &(M.getContext());
   LongSize = TD->getPointerSizeInBits();
@@ -656,17 +731,27 @@ bool AddressSanitizer::runOnModule(Module &M) {
       std::string FunctionName = std::string(kAsanReportErrorTemplate) +
           (AccessIsWrite ? "store" : "load") + itostr(1 << AccessSizeIndex);
       // If we are merging crash callbacks, they have two parameters.
-      AsanErrorCallback[AccessIsWrite][AccessSizeIndex] = cast<Function>(
-          M.getOrInsertFunction(FunctionName, IRB.getVoidTy(), IntptrTy, NULL));
+      AsanErrorCallback[AccessIsWrite][AccessSizeIndex] =
+          checkInterfaceFunction(M.getOrInsertFunction(
+              FunctionName, IRB.getVoidTy(), IntptrTy, NULL));
     }
   }
+
+  AsanStackMallocFunc = checkInterfaceFunction(M.getOrInsertFunction(
+      kAsanStackMallocName, IntptrTy, IntptrTy, IntptrTy, NULL));
+  AsanStackFreeFunc = checkInterfaceFunction(M.getOrInsertFunction(
+      kAsanStackFreeName, IRB.getVoidTy(),
+      IntptrTy, IntptrTy, IntptrTy, NULL));
+  AsanHandleNoReturnFunc = checkInterfaceFunction(M.getOrInsertFunction(
+      kAsanHandleNoReturnName, IRB.getVoidTy(), NULL));
+
   // We insert an empty inline asm after __asan_report* to avoid callback merge.
   EmptyAsm = InlineAsm::get(FunctionType::get(IRB.getVoidTy(), false),
                             StringRef(""), StringRef(""),
                             /*hasSideEffects=*/true);
 
   llvm::Triple targetTriple(M.getTargetTriple());
-  bool isAndroid = targetTriple.getEnvironment() == llvm::Triple::ANDROIDEABI;
+  bool isAndroid = targetTriple.getEnvironment() == llvm::Triple::Android;
 
   MappingOffset = isAndroid ? kDefaultShadowOffsetAndroid :
     (LongSize == 32 ? kDefaultShadowOffset32 : kDefaultShadowOffset64);
@@ -686,10 +771,6 @@ bool AddressSanitizer::runOnModule(Module &M) {
   // For scales 6 and 7, the redzone has to be 64 and 128 bytes respectively.
   RedzoneSize = std::max(32, (int)(1 << MappingScale));
 
-  bool Res = false;
-
-  if (ClGlobals)
-    Res |= insertGlobalRedzones(M);
 
   if (ClMappingOffsetLog >= 0) {
     // Tell the run-time the current values of mapping offset and scale.
@@ -709,17 +790,20 @@ bool AddressSanitizer::runOnModule(Module &M) {
     IRB.CreateLoad(asan_mapping_scale, true);
   }
 
-
-  for (Module::iterator F = M.begin(), E = M.end(); F != E; ++F) {
-    if (F->isDeclaration()) continue;
-    Res |= handleFunction(M, *F);
-  }
-
   appendToGlobalCtors(M, AsanCtorFunction, kAsanCtorAndCtorPriority);
 
-  return Res;
+  return true;
+}
+
+bool AddressSanitizer::doFinalization(Module &M) {
+  // We transform the globals at the very end so that the optimization analysis
+  // works on the original globals.
+  if (ClGlobals)
+    return insertGlobalRedzones(M);
+  return false;
 }
 
+
 bool AddressSanitizer::maybeInsertAsanInitAtFunctionEntry(Function &F) {
   // For each NSObject descendant having a +load method, this method is invoked
   // by the ObjC runtime before any of the static constructors is called.
@@ -736,19 +820,22 @@ bool AddressSanitizer::maybeInsertAsanInitAtFunctionEntry(Function &F) {
   return false;
 }
 
-bool AddressSanitizer::handleFunction(Module &M, Function &F) {
+bool AddressSanitizer::runOnFunction(Function &F) {
   if (BL->isIn(F)) return false;
   if (&F == AsanCtorFunction) return false;
+  DEBUG(dbgs() << "ASAN instrumenting:\n" << F << "\n");
 
   // If needed, insert __asan_init before checking for AddressSafety attr.
   maybeInsertAsanInitAtFunctionEntry(F);
 
-  if (!F.hasFnAttr(Attribute::AddressSafety)) return false;
+  if (!F.getFnAttributes().hasAttribute(Attributes::AddressSafety))
+    return false;
 
   if (!ClDebugFunc.empty() && ClDebugFunc != F.getName())
     return false;
-  // We want to instrument every address only once per basic block
-  // (unless there are calls between uses).
+
+  // We want to instrument every address only once per basic block (unless there
+  // are calls between uses).
   SmallSet<Value*, 16> TempsToInstrument;
   SmallVector<Instruction*, 16> ToInstrument;
   SmallVector<Instruction*, 8> NoReturnCalls;
@@ -786,8 +873,6 @@ bool AddressSanitizer::handleFunction(Module &M, Function &F) {
     }
   }
 
-  AsanFunctionContext AFC(F);
-
   // Instrument.
   int NumInstrumented = 0;
   for (size_t i = 0, n = ToInstrument.size(); i != n; i++) {
@@ -795,25 +880,23 @@ bool AddressSanitizer::handleFunction(Module &M, Function &F) {
     if (ClDebugMin < 0 || ClDebugMax < 0 ||
         (NumInstrumented >= ClDebugMin && NumInstrumented <= ClDebugMax)) {
       if (isInterestingMemoryAccess(Inst, &IsWrite))
-        instrumentMop(AFC, Inst);
+        instrumentMop(Inst);
       else
-        instrumentMemIntrinsic(AFC, cast<MemIntrinsic>(Inst));
+        instrumentMemIntrinsic(cast<MemIntrinsic>(Inst));
     }
     NumInstrumented++;
   }
 
-  DEBUG(dbgs() << F);
-
-  bool ChangedStack = poisonStackInFunction(M, F);
+  bool ChangedStack = poisonStackInFunction(F);
 
   // We must unpoison the stack before every NoReturn call (throw, _exit, etc).
   // See e.g. http://code.google.com/p/address-sanitizer/issues/detail?id=37
   for (size_t i = 0, n = NoReturnCalls.size(); i != n; i++) {
     Instruction *CI = NoReturnCalls[i];
     IRBuilder<> IRB(CI);
-    IRB.CreateCall(M.getOrInsertFunction(kAsanHandleNoReturnName,
-                                         IRB.getVoidTy(), NULL));
+    IRB.CreateCall(AsanHandleNoReturnFunc);
   }
+  DEBUG(dbgs() << "ASAN done instrumenting:\n" << F << "\n");
 
   return NumInstrumented > 0 || ChangedStack || !NoReturnCalls.empty();
 }
@@ -926,7 +1009,7 @@ bool AddressSanitizer::LooksLikeCodeInBug11395(Instruction *I) {
 // compiler hoists the load of the shadow value somewhere too high.
 // This causes asan to report a non-existing bug on 453.povray.
 // It sounds like an LLVM bug.
-bool AddressSanitizer::poisonStackInFunction(Module &M, Function &F) {
+bool AddressSanitizer::poisonStackInFunction(Function &F) {
   if (!ClStack) return false;
   SmallVector<AllocaInst*, 16> AllocaVec;
   SmallVector<Instruction*, 8> RetVec;
@@ -976,8 +1059,6 @@ bool AddressSanitizer::poisonStackInFunction(Module &M, Function &F) {
   Value *LocalStackBase = OrigStackBase;
 
   if (DoStackMalloc) {
-    Value *AsanStackMallocFunc = M.getOrInsertFunction(
-        kAsanStackMallocName, IntptrTy, IntptrTy, IntptrTy, NULL);
     LocalStackBase = IRB.CreateCall2(AsanStackMallocFunc,
         ConstantInt::get(IntptrTy, LocalStackSize), OrigStackBase);
   }
@@ -1012,22 +1093,16 @@ bool AddressSanitizer::poisonStackInFunction(Module &M, Function &F) {
   Value *BasePlus1 = IRB.CreateAdd(LocalStackBase,
                                    ConstantInt::get(IntptrTy, LongSize/8));
   BasePlus1 = IRB.CreateIntToPtr(BasePlus1, IntptrPtrTy);
-  Value *Description = IRB.CreatePointerCast(
-      createPrivateGlobalForString(M, StackDescription.str()),
-      IntptrTy);
+  GlobalVariable *StackDescriptionGlobal =
+      createPrivateGlobalForString(*F.getParent(), StackDescription.str());
+  GlobalsCreatedByAsan.insert(StackDescriptionGlobal);
+  Value *Description = IRB.CreatePointerCast(StackDescriptionGlobal, IntptrTy);
   IRB.CreateStore(Description, BasePlus1);
 
   // Poison the stack redzones at the entry.
   Value *ShadowBase = memToShadow(LocalStackBase, IRB);
   PoisonStack(ArrayRef<AllocaInst*>(AllocaVec), IRB, ShadowBase, true);
 
-  Value *AsanStackFreeFunc = NULL;
-  if (DoStackMalloc) {
-    AsanStackFreeFunc = M.getOrInsertFunction(
-        kAsanStackFreeName, IRB.getVoidTy(),
-        IntptrTy, IntptrTy, IntptrTy, NULL);
-  }
-
   // Unpoison the stack before all ret instructions.
   for (size_t i = 0, n = RetVec.size(); i < n; i++) {
     Instruction *Ret = RetVec[i];
@@ -1046,6 +1121,10 @@ bool AddressSanitizer::poisonStackInFunction(Module &M, Function &F) {
     }
   }
 
+  // We are done. Remove the old unused alloca instructions.
+  for (size_t i = 0, n = AllocaVec.size(); i < n; i++)
+    AllocaVec[i]->eraseFromParent();
+
   if (ClDebugStack) {
     DEBUG(dbgs() << F);
   }
diff --git a/lib/Transforms/Instrumentation/BlackList.cpp b/lib/Transforms/Instrumentation/BlackList.cpp
new file mode 100644
index 0000000..ef34b8a
--- /dev/null
+++ b/lib/Transforms/Instrumentation/BlackList.cpp
@@ -0,0 +1,105 @@
+//===-- BlackList.cpp - blacklist for sanitizers --------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This is a utility class for instrumentation passes (like AddressSanitizer
+// or ThreadSanitizer) to avoid instrumenting some functions or global
+// variables based on a user-supplied blacklist.
+//
+//===----------------------------------------------------------------------===//
+
+#include <utility>
+#include <string>
+
+#include "BlackList.h"
+#include "llvm/ADT/OwningPtr.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/Function.h"
+#include "llvm/GlobalVariable.h"
+#include "llvm/Module.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Regex.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/system_error.h"
+
+namespace llvm {
+
+BlackList::BlackList(const StringRef Path) {
+  // Validate and open blacklist file.
+  if (!Path.size()) return;
+  OwningPtr<MemoryBuffer> File;
+  if (error_code EC = MemoryBuffer::getFile(Path, File)) {
+    report_fatal_error("Can't open blacklist file: " + Path + ": " +
+                       EC.message());
+  }
+
+  // Iterate through each line in the blacklist file.
+  SmallVector<StringRef, 16> Lines;
+  SplitString(File.take()->getBuffer(), Lines, "\n\r");
+  StringMap<std::string> Regexps;
+  for (SmallVector<StringRef, 16>::iterator I = Lines.begin(), E = Lines.end();
+       I != E; ++I) {
+    // Ignore empty lines and lines starting with "#"
+    if (I->empty() || I->startswith("#"))
+      continue;
+    // Get our prefix and unparsed regexp.
+    std::pair<StringRef, StringRef> SplitLine = I->split(":");
+    StringRef Prefix = SplitLine.first;
+    std::string Regexp = SplitLine.second;
+
+    // Replace * with .*
+    for (size_t pos = 0; (pos = Regexp.find("*", pos)) != std::string::npos;
+         pos += strlen(".*")) {
+      Regexp.replace(pos, strlen("*"), ".*");
+    }
+
+    // Check that the regexp is valid.
+    Regex CheckRE(Regexp);
+    std::string Error;
+    if (!CheckRE.isValid(Error)) {
+      report_fatal_error("malformed blacklist regex: " + SplitLine.second +
+          ": " + Error);
+    }
+
+    // Add this regexp into the proper group by its prefix.
+    if (Regexps[Prefix].size())
+      Regexps[Prefix] += "|";
+    Regexps[Prefix] += Regexp;
+  }
+
+  // Iterate through each of the prefixes, and create Regexs for them.
+  for (StringMap<std::string>::iterator I = Regexps.begin(), E = Regexps.end();
+       I != E; ++I) {
+    Entries[I->getKey()] = new Regex(I->getValue());
+  }
+}
+
+bool BlackList::isIn(const Function &F) {
+  return isIn(*F.getParent()) || inSection("fun", F.getName());
+}
+
+bool BlackList::isIn(const GlobalVariable &G) {
+  return isIn(*G.getParent()) || inSection("global", G.getName());
+}
+
+bool BlackList::isIn(const Module &M) {
+  return inSection("src", M.getModuleIdentifier());
+}
+
+bool BlackList::isInInit(const GlobalVariable &G) {
+  return isIn(*G.getParent()) || inSection("global-init", G.getName());
+}
+
+bool BlackList::inSection(const StringRef Section,
+                                  const StringRef Query) {
+  Regex *FunctionRegex = Entries[Section];
+  return FunctionRegex ? FunctionRegex->match(Query) : false;
+}
+
+}  // namespace llvm
diff --git a/lib/Transforms/Instrumentation/BlackList.h b/lib/Transforms/Instrumentation/BlackList.h
new file mode 100644
index 0000000..f3c05a5
--- /dev/null
+++ b/lib/Transforms/Instrumentation/BlackList.h
@@ -0,0 +1,57 @@
+//===-- BlackList.h - blacklist for sanitizers ------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//===----------------------------------------------------------------------===//
+//
+// This is a utility class for instrumentation passes (like AddressSanitizer
+// or ThreadSanitizer) to avoid instrumenting some functions or global
+// variables based on a user-supplied blacklist.
+//
+// The blacklist disables instrumentation of various functions and global
+// variables.  Each line contains a prefix, followed by a wild card expression.
+// Empty lines and lines starting with "#" are ignored.
+// ---
+// # Blacklisted items:
+// fun:*_ZN4base6subtle*
+// global:*global_with_bad_access_or_initialization*
+// global-init:*global_with_initialization_issues*
+// src:file_with_tricky_code.cc
+// ---
+// Note that the wild card is in fact an llvm::Regex, but * is automatically
+// replaced with .*
+// This is similar to the "ignore" feature of ThreadSanitizer.
+// http://code.google.com/p/data-race-test/wiki/ThreadSanitizerIgnores
+//
+//===----------------------------------------------------------------------===//
+//
+
+#include "llvm/ADT/StringMap.h"
+
+namespace llvm {
+class Function;
+class GlobalVariable;
+class Module;
+class Regex;
+class StringRef;
+
+class BlackList {
+ public:
+  BlackList(const StringRef Path);
+  // Returns whether either this function or it's source file are blacklisted.
+  bool isIn(const Function &F);
+  // Returns whether either this global or it's source file are blacklisted.
+  bool isIn(const GlobalVariable &G);
+  // Returns whether this module is blacklisted by filename.
+  bool isIn(const Module &M);
+  // Returns whether a global should be excluded from initialization checking.
+  bool isInInit(const GlobalVariable &G);
+ private:
+  StringMap<Regex*> Entries;
+
+  bool inSection(const StringRef Section, const StringRef Query);
+};
+
+}  // namespace llvm
diff --git a/lib/Transforms/Instrumentation/BoundsChecking.cpp b/lib/Transforms/Instrumentation/BoundsChecking.cpp
index 09e0f14..7810b1b 100644
--- a/lib/Transforms/Instrumentation/BoundsChecking.cpp
+++ b/lib/Transforms/Instrumentation/BoundsChecking.cpp
@@ -23,7 +23,8 @@
 #include "llvm/Support/InstIterator.h"
 #include "llvm/Support/TargetFolder.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
+#include "llvm/Target/TargetLibraryInfo.h"
 #include "llvm/Transforms/Instrumentation.h"
 using namespace llvm;
 
@@ -47,11 +48,13 @@ namespace {
     virtual bool runOnFunction(Function &F);
 
     virtual void getAnalysisUsage(AnalysisUsage &AU) const {
-      AU.addRequired<TargetData>();
+      AU.addRequired<DataLayout>();
+      AU.addRequired<TargetLibraryInfo>();
     }
 
   private:
-    const TargetData *TD;
+    const DataLayout *TD;
+    const TargetLibraryInfo *TLI;
     ObjectSizeOffsetEvaluator *ObjSizeEval;
     BuilderTy *Builder;
     Instruction *Inst;
@@ -140,7 +143,7 @@ bool BoundsChecking::instrument(Value *Ptr, Value *InstVal) {
   Value *Offset = SizeOffset.second;
   ConstantInt *SizeCI = dyn_cast<ConstantInt>(Size);
 
-  IntegerType *IntTy = TD->getIntPtrType(Inst->getContext());
+  Type *IntTy = TD->getIntPtrType(Ptr->getType());
   Value *NeededSizeVal = ConstantInt::get(IntTy, NeededSize);
 
   // three checks are required to ensure safety:
@@ -165,12 +168,13 @@ bool BoundsChecking::instrument(Value *Ptr, Value *InstVal) {
 }
 
 bool BoundsChecking::runOnFunction(Function &F) {
-  TD = &getAnalysis<TargetData>();
+  TD = &getAnalysis<DataLayout>();
+  TLI = &getAnalysis<TargetLibraryInfo>();
 
   TrapBB = 0;
   BuilderTy TheBuilder(F.getContext(), TargetFolder(TD));
   Builder = &TheBuilder;
-  ObjectSizeOffsetEvaluator TheObjSizeEval(TD, F.getContext());
+  ObjectSizeOffsetEvaluator TheObjSizeEval(TD, TLI, F.getContext());
   ObjSizeEval = &TheObjSizeEval;
 
   // check HANDLE_MEMORY_INST in include/llvm/Instruction.def for memory
diff --git a/lib/Transforms/Instrumentation/CMakeLists.txt b/lib/Transforms/Instrumentation/CMakeLists.txt
index 00de882..058f68c 100644
--- a/lib/Transforms/Instrumentation/CMakeLists.txt
+++ b/lib/Transforms/Instrumentation/CMakeLists.txt
@@ -1,8 +1,8 @@
 add_llvm_library(LLVMInstrumentation
   AddressSanitizer.cpp
+  BlackList.cpp
   BoundsChecking.cpp
   EdgeProfiling.cpp
-  FunctionBlackList.cpp
   GCOVProfiling.cpp
   Instrumentation.cpp
   OptimalEdgeProfiling.cpp
diff --git a/lib/Transforms/Instrumentation/FunctionBlackList.cpp b/lib/Transforms/Instrumentation/FunctionBlackList.cpp
deleted file mode 100644
index 188ea4d..0000000
--- a/lib/Transforms/Instrumentation/FunctionBlackList.cpp
+++ /dev/null
@@ -1,79 +0,0 @@
-//===-- FunctionBlackList.cpp - blacklist of functions --------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This is a utility class for instrumentation passes (like AddressSanitizer 
-// or ThreadSanitizer) to avoid instrumenting some functions based on
-// user-supplied blacklist.
-//
-//===----------------------------------------------------------------------===//
-
-#include "FunctionBlackList.h"
-#include "llvm/ADT/OwningPtr.h"
-#include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/StringExtras.h"
-#include "llvm/Function.h"
-#include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/Regex.h"
-#include "llvm/Support/raw_ostream.h"
-#include "llvm/Support/system_error.h"
-
-namespace llvm {
-
-FunctionBlackList::FunctionBlackList(const std::string &Path) {
-  Functions = NULL;
-  const char *kFunPrefix = "fun:";
-  if (!Path.size()) return;
-  std::string Fun;
-
-  OwningPtr<MemoryBuffer> File;
-  if (error_code EC = MemoryBuffer::getFile(Path.c_str(), File)) {
-    report_fatal_error("Can't open blacklist file " + Path + ": " +
-                       EC.message());
-  }
-  MemoryBuffer *Buff = File.take();
-  const char *Data = Buff->getBufferStart();
-  size_t DataLen = Buff->getBufferSize();
-  SmallVector<StringRef, 16> Lines;
-  SplitString(StringRef(Data, DataLen), Lines, "\n\r");
-  for (size_t i = 0, numLines = Lines.size(); i < numLines; i++) {
-    if (Lines[i].startswith(kFunPrefix)) {
-      std::string ThisFunc = Lines[i].substr(strlen(kFunPrefix));
-      std::string ThisFuncRE;
-      // add ThisFunc replacing * with .*
-      for (size_t j = 0, n = ThisFunc.size(); j < n; j++) {
-        if (ThisFunc[j] == '*')
-          ThisFuncRE += '.';
-        ThisFuncRE += ThisFunc[j];
-      }
-      // Check that the regexp is valid.
-      Regex CheckRE(ThisFuncRE);
-      std::string Error;
-      if (!CheckRE.isValid(Error))
-        report_fatal_error("malformed blacklist regex: " + ThisFunc +
-                           ": " + Error);
-      // Append to the final regexp.
-      if (Fun.size())
-        Fun += "|";
-      Fun += ThisFuncRE;
-    }
-  }
-  if (Fun.size()) {
-    Functions = new Regex(Fun);
-  }
-}
-
-bool FunctionBlackList::isIn(const Function &F) {
-  if (Functions) {
-    bool Res = Functions->match(F.getName());
-    return Res;
-  }
-  return false;
-}
-
-}  // namespace llvm
diff --git a/lib/Transforms/Instrumentation/FunctionBlackList.h b/lib/Transforms/Instrumentation/FunctionBlackList.h
deleted file mode 100644
index c1239b9..0000000
--- a/lib/Transforms/Instrumentation/FunctionBlackList.h
+++ /dev/null
@@ -1,37 +0,0 @@
-//===-- FunctionBlackList.cpp - blacklist of functions ----------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//===----------------------------------------------------------------------===//
-//
-// This is a utility class for instrumentation passes (like AddressSanitizer
-// or ThreadSanitizer) to avoid instrumenting some functions based on
-// user-supplied blacklist.
-//
-//===----------------------------------------------------------------------===//
-//
-
-#include <string>
-
-namespace llvm {
-class Function;
-class Regex;
-
-// Blacklisted functions are not instrumented.
-// The blacklist file contains one or more lines like this:
-// ---
-// fun:FunctionWildCard
-// ---
-// This is similar to the "ignore" feature of ThreadSanitizer.
-// http://code.google.com/p/data-race-test/wiki/ThreadSanitizerIgnores
-class FunctionBlackList {
- public:
-  FunctionBlackList(const std::string &Path);
-  bool isIn(const Function &F);
- private:
-  Regex *Functions;
-};
-
-}  // namespace llvm
diff --git a/lib/Transforms/Instrumentation/GCOVProfiling.cpp b/lib/Transforms/Instrumentation/GCOVProfiling.cpp
index 264a6a6..e9192e5 100644
--- a/lib/Transforms/Instrumentation/GCOVProfiling.cpp
+++ b/lib/Transforms/Instrumentation/GCOVProfiling.cpp
@@ -88,11 +88,11 @@ namespace {
 
     // Add the function to write out all our counters to the global destructor
     // list.
-    void insertCounterWriteout(SmallVector<std::pair<GlobalVariable *,
-                                                     MDNode *>, 8> &);
+    void insertCounterWriteout(ArrayRef<std::pair<GlobalVariable*, MDNode*> >);
     void insertIndirectCounterIncrement();
+    void insertFlush(ArrayRef<std::pair<GlobalVariable*, MDNode*> >);
 
-    std::string mangleName(DICompileUnit CU, std::string NewStem);
+    std::string mangleName(DICompileUnit CU, const char *NewStem);
 
     bool EmitNotes;
     bool EmitData;
@@ -329,7 +329,7 @@ namespace {
   };
 }
 
-std::string GCOVProfiler::mangleName(DICompileUnit CU, std::string NewStem) {
+std::string GCOVProfiler::mangleName(DICompileUnit CU, const char *NewStem) {
   if (NamedMDNode *GCov = M->getNamedMetadata("llvm.gcov")) {
     for (int i = 0, e = GCov->getNumOperands(); i != e; ++i) {
       MDNode *N = GCov->getOperand(i);
@@ -519,6 +519,7 @@ bool GCOVProfiler::emitProfileArcs() {
     }
 
     insertCounterWriteout(CountersBySP);
+    insertFlush(CountersBySP);
   }
 
   if (InsertIndCounterIncrCode)
@@ -630,14 +631,15 @@ GlobalVariable *GCOVProfiler::getEdgeStateValue() {
 }
 
 void GCOVProfiler::insertCounterWriteout(
-    SmallVector<std::pair<GlobalVariable *, MDNode *>, 8> &CountersBySP) {
-  FunctionType *WriteoutFTy =
-      FunctionType::get(Type::getVoidTy(*Ctx), false);
-  Function *WriteoutF = Function::Create(WriteoutFTy,
-                                         GlobalValue::InternalLinkage,
-                                         "__llvm_gcov_writeout", M);
+    ArrayRef<std::pair<GlobalVariable *, MDNode *> > CountersBySP) {
+  FunctionType *WriteoutFTy = FunctionType::get(Type::getVoidTy(*Ctx), false);
+  Function *WriteoutF = M->getFunction("__llvm_gcov_writeout");
+  if (!WriteoutF)
+    WriteoutF = Function::Create(WriteoutFTy, GlobalValue::InternalLinkage,
+                                 "__llvm_gcov_writeout", M);
   WriteoutF->setUnnamedAddr(true);
-  BasicBlock *BB = BasicBlock::Create(*Ctx, "", WriteoutF);
+
+  BasicBlock *BB = BasicBlock::Create(*Ctx, "entry", WriteoutF);
   IRBuilder<> Builder(BB);
 
   Constant *StartFile = getStartFileFunc();
@@ -648,11 +650,11 @@ void GCOVProfiler::insertCounterWriteout(
   NamedMDNode *CU_Nodes = M->getNamedMetadata("llvm.dbg.cu");
   if (CU_Nodes) {
     for (unsigned i = 0, e = CU_Nodes->getNumOperands(); i != e; ++i) {
-      DICompileUnit compile_unit(CU_Nodes->getOperand(i));
-      std::string FilenameGcda = mangleName(compile_unit, "gcda");
+      DICompileUnit CU(CU_Nodes->getOperand(i));
+      std::string FilenameGcda = mangleName(CU, "gcda");
       Builder.CreateCall(StartFile,
                          Builder.CreateGlobalStringPtr(FilenameGcda));
-      for (SmallVector<std::pair<GlobalVariable *, MDNode *>, 8>::iterator
+      for (ArrayRef<std::pair<GlobalVariable *, MDNode *> >::iterator
              I = CountersBySP.begin(), E = CountersBySP.end();
            I != E; ++I) {
         DISubprogram SP(I->second);
@@ -680,7 +682,7 @@ void GCOVProfiler::insertCounterWriteout(
                                  "__llvm_gcov_init", M);
   F->setUnnamedAddr(true);
   F->setLinkage(GlobalValue::InternalLinkage);
-  F->addFnAttr(Attribute::NoInline);
+  F->addFnAttr(Attributes::NoInline);
 
   BB = BasicBlock::Create(*Ctx, "entry", F);
   Builder.SetInsertPoint(BB);
@@ -699,7 +701,7 @@ void GCOVProfiler::insertIndirectCounterIncrement() {
     cast<Function>(GCOVProfiler::getIncrementIndirectCounterFunc());
   Fn->setUnnamedAddr(true);
   Fn->setLinkage(GlobalValue::InternalLinkage);
-  Fn->addFnAttr(Attribute::NoInline);
+  Fn->addFnAttr(Attributes::NoInline);
 
   Type *Int32Ty = Type::getInt32Ty(*Ctx);
   Type *Int64Ty = Type::getInt64Ty(*Ctx);
@@ -745,3 +747,42 @@ void GCOVProfiler::insertIndirectCounterIncrement() {
   Builder.SetInsertPoint(Exit);
   Builder.CreateRetVoid();
 }
+
+void GCOVProfiler::
+insertFlush(ArrayRef<std::pair<GlobalVariable*, MDNode*> > CountersBySP) {
+  FunctionType *FTy = FunctionType::get(Type::getVoidTy(*Ctx), false);
+  Function *FlushF = M->getFunction("__gcov_flush");
+  if (!FlushF)
+    FlushF = Function::Create(FTy, GlobalValue::InternalLinkage,
+                              "__gcov_flush", M);
+  else
+    FlushF->setLinkage(GlobalValue::InternalLinkage);
+  FlushF->setUnnamedAddr(true);
+
+  BasicBlock *Entry = BasicBlock::Create(*Ctx, "entry", FlushF);
+
+  // Write out the current counters.
+  Constant *WriteoutF = M->getFunction("__llvm_gcov_writeout");
+  assert(WriteoutF && "Need to create the writeout function first!");
+
+  IRBuilder<> Builder(Entry);
+  Builder.CreateCall(WriteoutF);
+
+  // Zero out the counters.
+  for (ArrayRef<std::pair<GlobalVariable *, MDNode *> >::iterator
+         I = CountersBySP.begin(), E = CountersBySP.end();
+       I != E; ++I) {
+    GlobalVariable *GV = I->first;
+    Constant *Null = Constant::getNullValue(GV->getType()->getElementType());
+    Builder.CreateStore(Null, GV);
+  }
+
+  Type *RetTy = FlushF->getReturnType();
+  if (RetTy == Type::getVoidTy(*Ctx))
+    Builder.CreateRetVoid();
+  else if (RetTy->isIntegerTy())
+    // Used if __gcov_flush was implicitly declared.
+    Builder.CreateRet(ConstantInt::get(RetTy, 0));
+  else
+    report_fatal_error("invalid return type for __gcov_flush");
+}
diff --git a/lib/Transforms/Instrumentation/MaximumSpanningTree.h b/lib/Transforms/Instrumentation/MaximumSpanningTree.h
index f76c77e..a4bb5a6 100644
--- a/lib/Transforms/Instrumentation/MaximumSpanningTree.h
+++ b/lib/Transforms/Instrumentation/MaximumSpanningTree.h
@@ -26,30 +26,6 @@ namespace llvm {
   /// The type parameter T determines the type of the nodes of the graph.
   template <typename T>
   class MaximumSpanningTree {
-
-    // A comparing class for comparing weighted edges.
-    template <typename CT>
-    struct EdgeWeightCompare {
-      bool operator()(typename MaximumSpanningTree<CT>::EdgeWeight X, 
-                      typename MaximumSpanningTree<CT>::EdgeWeight Y) const {
-        if (X.second > Y.second) return true;
-        if (X.second < Y.second) return false;
-        if (const BasicBlock *BBX = dyn_cast<BasicBlock>(X.first.first)) {
-          if (const BasicBlock *BBY = dyn_cast<BasicBlock>(Y.first.first)) {
-            if (BBX->size() > BBY->size()) return true;
-            if (BBX->size() < BBY->size()) return false;
-          }
-        }
-        if (const BasicBlock *BBX = dyn_cast<BasicBlock>(X.first.second)) {
-          if (const BasicBlock *BBY = dyn_cast<BasicBlock>(Y.first.second)) {
-            if (BBX->size() > BBY->size()) return true;
-            if (BBX->size() < BBY->size()) return false;
-          }
-        }
-        return false;
-      }
-    };
-
   public:
     typedef std::pair<const T*, const T*> Edge;
     typedef std::pair<Edge, double> EdgeWeight;
@@ -59,6 +35,33 @@ namespace llvm {
 
     MaxSpanTree MST;
 
+  private:
+    // A comparing class for comparing weighted edges.
+    struct EdgeWeightCompare {
+      static bool getBlockSize(const T *X) {
+        const BasicBlock *BB = dyn_cast_or_null<BasicBlock>(X);
+        return BB ? BB->size() : 0;
+      }
+
+      bool operator()(EdgeWeight X, EdgeWeight Y) const {
+        if (X.second > Y.second) return true;
+        if (X.second < Y.second) return false;
+
+        // Equal edge weights: break ties by comparing block sizes.
+        size_t XSizeA = getBlockSize(X.first.first);
+        size_t YSizeA = getBlockSize(Y.first.first);
+        if (XSizeA > YSizeA) return true;
+        if (XSizeA < YSizeA) return false;
+
+        size_t XSizeB = getBlockSize(X.first.second);
+        size_t YSizeB = getBlockSize(Y.first.second);
+        if (XSizeB > YSizeB) return true;
+        if (XSizeB < YSizeB) return false;
+
+        return false;
+      }
+    };
+
   public:
     static char ID; // Class identification, replacement for typeinfo
 
@@ -66,7 +69,7 @@ namespace llvm {
     /// spanning tree.
     MaximumSpanningTree(EdgeWeights &EdgeVector) {
 
-      std::stable_sort(EdgeVector.begin(), EdgeVector.end(), EdgeWeightCompare<T>());
+      std::stable_sort(EdgeVector.begin(), EdgeVector.end(), EdgeWeightCompare());
 
       // Create spanning tree, Forest contains a special data structure
       // that makes checking if two nodes are already in a common (sub-)tree
diff --git a/lib/Transforms/Instrumentation/ThreadSanitizer.cpp b/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
index dc0fa71..9e10fc4 100644
--- a/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
+++ b/lib/Transforms/Instrumentation/ThreadSanitizer.cpp
@@ -21,7 +21,7 @@
 
 #define DEBUG_TYPE "tsan"
 
-#include "FunctionBlackList.h"
+#include "BlackList.h"
 #include "llvm/Function.h"
 #include "llvm/IRBuilder.h"
 #include "llvm/Intrinsics.h"
@@ -38,7 +38,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Transforms/Instrumentation.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/Transforms/Utils/ModuleUtils.h"
@@ -47,10 +47,19 @@ using namespace llvm;
 
 static cl::opt<std::string>  ClBlackListFile("tsan-blacklist",
        cl::desc("Blacklist file"), cl::Hidden);
+static cl::opt<bool>  ClInstrumentMemoryAccesses(
+    "tsan-instrument-memory-accesses", cl::init(true),
+    cl::desc("Instrument memory accesses"), cl::Hidden);
+static cl::opt<bool>  ClInstrumentFuncEntryExit(
+    "tsan-instrument-func-entry-exit", cl::init(true),
+    cl::desc("Instrument function entry and exit"), cl::Hidden);
+static cl::opt<bool>  ClInstrumentAtomics(
+    "tsan-instrument-atomics", cl::init(true),
+    cl::desc("Instrument atomics"), cl::Hidden);
 
 STATISTIC(NumInstrumentedReads, "Number of instrumented reads");
 STATISTIC(NumInstrumentedWrites, "Number of instrumented writes");
-STATISTIC(NumOmittedReadsBeforeWrite, 
+STATISTIC(NumOmittedReadsBeforeWrite,
           "Number of reads ignored due to following writes");
 STATISTIC(NumAccessesWithBadSize, "Number of accesses with bad size");
 STATISTIC(NumInstrumentedVtableWrites, "Number of vtable ptr writes");
@@ -76,8 +85,8 @@ struct ThreadSanitizer : public FunctionPass {
   bool addrPointsToConstantData(Value *Addr);
   int getMemoryAccessFuncIndex(Value *Addr);
 
-  TargetData *TD;
-  OwningPtr<FunctionBlackList> BL;
+  DataLayout *TD;
+  OwningPtr<BlackList> BL;
   IntegerType *OrdTy;
   // Callbacks to run-time library are computed in doInitialization.
   Function *TsanFuncEntry;
@@ -88,6 +97,10 @@ struct ThreadSanitizer : public FunctionPass {
   Function *TsanWrite[kNumberOfAccessSizes];
   Function *TsanAtomicLoad[kNumberOfAccessSizes];
   Function *TsanAtomicStore[kNumberOfAccessSizes];
+  Function *TsanAtomicRMW[AtomicRMWInst::LAST_BINOP + 1][kNumberOfAccessSizes];
+  Function *TsanAtomicCAS[kNumberOfAccessSizes];
+  Function *TsanAtomicThreadFence;
+  Function *TsanAtomicSignalFence;
   Function *TsanVptrUpdate;
 };
 }  // namespace
@@ -118,10 +131,10 @@ static Function *checkInterfaceFunction(Constant *FuncOrBitcast) {
 }
 
 bool ThreadSanitizer::doInitialization(Module &M) {
-  TD = getAnalysisIfAvailable<TargetData>();
+  TD = getAnalysisIfAvailable<DataLayout>();
   if (!TD)
     return false;
-  BL.reset(new FunctionBlackList(ClBlackListFile));
+  BL.reset(new BlackList(ClBlackListFile));
 
   // Always insert a call to __tsan_init into the module's CTORs.
   IRBuilder<> IRB(M.getContext());
@@ -158,10 +171,42 @@ bool ThreadSanitizer::doInitialization(Module &M) {
     TsanAtomicStore[i] = checkInterfaceFunction(M.getOrInsertFunction(
         AtomicStoreName, IRB.getVoidTy(), PtrTy, Ty, OrdTy,
         NULL));
+
+    for (int op = AtomicRMWInst::FIRST_BINOP;
+        op <= AtomicRMWInst::LAST_BINOP; ++op) {
+      TsanAtomicRMW[op][i] = NULL;
+      const char *NamePart = NULL;
+      if (op == AtomicRMWInst::Xchg)
+        NamePart = "_exchange";
+      else if (op == AtomicRMWInst::Add)
+        NamePart = "_fetch_add";
+      else if (op == AtomicRMWInst::Sub)
+        NamePart = "_fetch_sub";
+      else if (op == AtomicRMWInst::And)
+        NamePart = "_fetch_and";
+      else if (op == AtomicRMWInst::Or)
+        NamePart = "_fetch_or";
+      else if (op == AtomicRMWInst::Xor)
+        NamePart = "_fetch_xor";
+      else
+        continue;
+      SmallString<32> RMWName("__tsan_atomic" + itostr(BitSize) + NamePart);
+      TsanAtomicRMW[op][i] = checkInterfaceFunction(M.getOrInsertFunction(
+          RMWName, Ty, PtrTy, Ty, OrdTy, NULL));
+    }
+
+    SmallString<32> AtomicCASName("__tsan_atomic" + itostr(BitSize) +
+                                  "_compare_exchange_val");
+    TsanAtomicCAS[i] = checkInterfaceFunction(M.getOrInsertFunction(
+        AtomicCASName, Ty, PtrTy, Ty, Ty, OrdTy, NULL));
   }
   TsanVptrUpdate = checkInterfaceFunction(M.getOrInsertFunction(
       "__tsan_vptr_update", IRB.getVoidTy(), IRB.getInt8PtrTy(),
       IRB.getInt8PtrTy(), NULL));
+  TsanAtomicThreadFence = checkInterfaceFunction(M.getOrInsertFunction(
+      "__tsan_atomic_thread_fence", IRB.getVoidTy(), OrdTy, NULL));
+  TsanAtomicSignalFence = checkInterfaceFunction(M.getOrInsertFunction(
+      "__tsan_atomic_signal_fence", IRB.getVoidTy(), OrdTy, NULL));
   return true;
 }
 
@@ -186,7 +231,7 @@ bool ThreadSanitizer::addrPointsToConstantData(Value *Addr) {
       NumOmittedReadsFromConstantGlobals++;
       return true;
     }
-  } else if(LoadInst *L = dyn_cast<LoadInst>(Addr)) {
+  } else if (LoadInst *L = dyn_cast<LoadInst>(Addr)) {
     if (isVtableAccess(L)) {
       // Reads from a vtable pointer can not race with any writes.
       NumOmittedReadsFromVtable++;
@@ -244,8 +289,8 @@ static bool isAtomic(Instruction *I) {
     return true;
   if (isa<AtomicCmpXchgInst>(I))
     return true;
-  if (FenceInst *FI = dyn_cast<FenceInst>(I))
-    return FI->getSynchScope() == CrossThread;
+  if (isa<FenceInst>(I))
+    return true;
   return false;
 }
 
@@ -284,17 +329,19 @@ bool ThreadSanitizer::runOnFunction(Function &F) {
   // (e.g. variables that do not escape, etc).
 
   // Instrument memory accesses.
-  for (size_t i = 0, n = AllLoadsAndStores.size(); i < n; ++i) {
-    Res |= instrumentLoadOrStore(AllLoadsAndStores[i]);
-  }
+  if (ClInstrumentMemoryAccesses)
+    for (size_t i = 0, n = AllLoadsAndStores.size(); i < n; ++i) {
+      Res |= instrumentLoadOrStore(AllLoadsAndStores[i]);
+    }
 
   // Instrument atomic memory accesses.
-  for (size_t i = 0, n = AtomicAccesses.size(); i < n; ++i) {
-    Res |= instrumentAtomic(AtomicAccesses[i]);
-  }
+  if (ClInstrumentAtomics)
+    for (size_t i = 0, n = AtomicAccesses.size(); i < n; ++i) {
+      Res |= instrumentAtomic(AtomicAccesses[i]);
+    }
 
   // Instrument function entry/exit points if there were instrumented accesses.
-  if (Res || HasCalls) {
+  if ((Res || HasCalls) && ClInstrumentFuncEntryExit) {
     IRBuilder<> IRB(F.getEntryBlock().getFirstNonPHI());
     Value *ReturnAddress = IRB.CreateCall(
         Intrinsic::getDeclaration(F.getParent(), Intrinsic::returnaddress),
@@ -343,12 +390,12 @@ static ConstantInt *createOrdering(IRBuilder<> *IRB, AtomicOrdering ord) {
   switch (ord) {
     case NotAtomic:              assert(false);
     case Unordered:              // Fall-through.
-    case Monotonic:              v = 1 << 0; break;
- // case Consume:                v = 1 << 1; break;  // Not specified yet.
-    case Acquire:                v = 1 << 2; break;
-    case Release:                v = 1 << 3; break;
-    case AcquireRelease:         v = 1 << 4; break;
-    case SequentiallyConsistent: v = 1 << 5; break;
+    case Monotonic:              v = 0; break;
+ // case Consume:                v = 1; break;  // Not specified yet.
+    case Acquire:                v = 2; break;
+    case Release:                v = 3; break;
+    case AcquireRelease:         v = 4; break;
+    case SequentiallyConsistent: v = 5; break;
   }
   return IRB->getInt32(v);
 }
@@ -385,12 +432,44 @@ bool ThreadSanitizer::instrumentAtomic(Instruction *I) {
     CallInst *C = CallInst::Create(TsanAtomicStore[Idx],
                                    ArrayRef<Value*>(Args));
     ReplaceInstWithInst(I, C);
-  } else if (isa<AtomicRMWInst>(I)) {
-    // FIXME: Not yet supported.
-  } else if (isa<AtomicCmpXchgInst>(I)) {
-    // FIXME: Not yet supported.
-  } else if (isa<FenceInst>(I)) {
-    // FIXME: Not yet supported.
+  } else if (AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(I)) {
+    Value *Addr = RMWI->getPointerOperand();
+    int Idx = getMemoryAccessFuncIndex(Addr);
+    if (Idx < 0)
+      return false;
+    Function *F = TsanAtomicRMW[RMWI->getOperation()][Idx];
+    if (F == NULL)
+      return false;
+    const size_t ByteSize = 1 << Idx;
+    const size_t BitSize = ByteSize * 8;
+    Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize);
+    Type *PtrTy = Ty->getPointerTo();
+    Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy),
+                     IRB.CreateIntCast(RMWI->getValOperand(), Ty, false),
+                     createOrdering(&IRB, RMWI->getOrdering())};
+    CallInst *C = CallInst::Create(F, ArrayRef<Value*>(Args));
+    ReplaceInstWithInst(I, C);
+  } else if (AtomicCmpXchgInst *CASI = dyn_cast<AtomicCmpXchgInst>(I)) {
+    Value *Addr = CASI->getPointerOperand();
+    int Idx = getMemoryAccessFuncIndex(Addr);
+    if (Idx < 0)
+      return false;
+    const size_t ByteSize = 1 << Idx;
+    const size_t BitSize = ByteSize * 8;
+    Type *Ty = Type::getIntNTy(IRB.getContext(), BitSize);
+    Type *PtrTy = Ty->getPointerTo();
+    Value *Args[] = {IRB.CreatePointerCast(Addr, PtrTy),
+                     IRB.CreateIntCast(CASI->getCompareOperand(), Ty, false),
+                     IRB.CreateIntCast(CASI->getNewValOperand(), Ty, false),
+                     createOrdering(&IRB, CASI->getOrdering())};
+    CallInst *C = CallInst::Create(TsanAtomicCAS[Idx], ArrayRef<Value*>(Args));
+    ReplaceInstWithInst(I, C);
+  } else if (FenceInst *FI = dyn_cast<FenceInst>(I)) {
+    Value *Args[] = {createOrdering(&IRB, FI->getOrdering())};
+    Function *F = FI->getSynchScope() == SingleThread ?
+        TsanAtomicSignalFence : TsanAtomicThreadFence;
+    CallInst *C = CallInst::Create(F, ArrayRef<Value*>(Args));
+    ReplaceInstWithInst(I, C);
   }
   return true;
 }
diff --git a/lib/Transforms/Scalar/CMakeLists.txt b/lib/Transforms/Scalar/CMakeLists.txt
index a01e066..b3fc6e3 100644
--- a/lib/Transforms/Scalar/CMakeLists.txt
+++ b/lib/Transforms/Scalar/CMakeLists.txt
@@ -25,6 +25,7 @@ add_llvm_library(LLVMScalarOpts
   Reassociate.cpp
   Reg2Mem.cpp
   SCCP.cpp
+  SROA.cpp
   Scalar.cpp
   ScalarReplAggregates.cpp
   SimplifyCFGPass.cpp
diff --git a/lib/Transforms/Scalar/CodeGenPrepare.cpp b/lib/Transforms/Scalar/CodeGenPrepare.cpp
index a3c426a..123ed0f 100644
--- a/lib/Transforms/Scalar/CodeGenPrepare.cpp
+++ b/lib/Transforms/Scalar/CodeGenPrepare.cpp
@@ -27,6 +27,7 @@
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/Dominators.h"
+#include "llvm/Analysis/DominatorInternals.h"
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/ProfileInfo.h"
 #include "llvm/Assembly/Writer.h"
@@ -37,12 +38,13 @@
 #include "llvm/Support/PatternMatch.h"
 #include "llvm/Support/ValueHandle.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetLibraryInfo.h"
 #include "llvm/Target/TargetLowering.h"
 #include "llvm/Transforms/Utils/AddrModeMatcher.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/Transforms/Utils/BuildLibCalls.h"
+#include "llvm/Transforms/Utils/BypassSlowDivision.h"
 #include "llvm/Transforms/Utils/Local.h"
 using namespace llvm;
 using namespace llvm::PatternMatch;
@@ -146,9 +148,18 @@ bool CodeGenPrepare::runOnFunction(Function &F) {
   TLInfo = &getAnalysis<TargetLibraryInfo>();
   DT = getAnalysisIfAvailable<DominatorTree>();
   PFI = getAnalysisIfAvailable<ProfileInfo>();
-  OptSize = F.hasFnAttr(Attribute::OptimizeForSize);
+  OptSize = F.getFnAttributes().hasAttribute(Attributes::OptimizeForSize);
+
+  /// This optimization identifies DIV instructions that can be
+  /// profitably bypassed and carried out with a shorter, faster divide.
+  if (TLI && TLI->isSlowDivBypassed()) {
+    const DenseMap<unsigned int, unsigned int> &BypassWidths =
+       TLI->getBypassSlowDivWidths();
+    for (Function::iterator I = F.begin(); I != F.end(); I++)
+      EverMadeChange |= bypassSlowDivision(F, I, BypassWidths);
+  }
 
-  // First pass, eliminate blocks that contain only PHI nodes and an
+  // Eliminate blocks that contain only PHI nodes and an
   // unconditional branch.
   EverMadeChange |= EliminateMostlyEmptyBlocks(F);
 
@@ -160,7 +171,7 @@ bool CodeGenPrepare::runOnFunction(Function &F) {
   bool MadeChange = true;
   while (MadeChange) {
     MadeChange = false;
-    for (Function::iterator I = F.begin(), E = F.end(); I != E; ) {
+    for (Function::iterator I = F.begin(); I != F.end(); ) {
       BasicBlock *BB = I++;
       MadeChange |= OptimizeBlock(*BB);
     }
@@ -215,11 +226,13 @@ bool CodeGenPrepare::EliminateFallThrough(Function &F) {
     // edge, just collapse it.
     BasicBlock *SinglePred = BB->getSinglePredecessor();
 
-    if (!SinglePred || SinglePred == BB) continue;
+    // Don't merge if BB's address is taken.
+    if (!SinglePred || SinglePred == BB || BB->hasAddressTaken()) continue;
 
     BranchInst *Term = dyn_cast<BranchInst>(SinglePred->getTerminator());
     if (Term && !Term->isConditional()) {
       Changed = true;
+      DEBUG(dbgs() << "To merge:\n"<< *SinglePred << "\n\n\n");
       // Remember if SinglePred was the entry block of the function.
       // If so, we will need to move BB back to the entry position.
       bool isEntry = SinglePred == &SinglePred->getParent()->getEntryBlock();
@@ -230,7 +243,6 @@ bool CodeGenPrepare::EliminateFallThrough(Function &F) {
 
       // We have erased a block. Update the iterator.
       I = BB;
-      DEBUG(dbgs() << "Merged:\n"<< *SinglePred << "\n\n\n");
     }
   }
   return Changed;
@@ -610,7 +622,7 @@ bool CodeGenPrepare::OptimizeCallInst(CallInst *CI) {
     // happens.
     WeakVH IterHandle(CurInstIterator);
 
-    replaceAndRecursivelySimplify(CI, RetVal, TLI ? TLI->getTargetData() : 0,
+    replaceAndRecursivelySimplify(CI, RetVal, TLI ? TLI->getDataLayout() : 0,
                                   TLInfo, ModifiedDT ? 0 : DT);
 
     // If the iterator instruction was recursively deleted, start over at the
@@ -634,8 +646,8 @@ bool CodeGenPrepare::OptimizeCallInst(CallInst *CI) {
   // From here on out we're working with named functions.
   if (CI->getCalledFunction() == 0) return false;
 
-  // We'll need TargetData from here on out.
-  const TargetData *TD = TLI ? TLI->getTargetData() : 0;
+  // We'll need DataLayout from here on out.
+  const DataLayout *TD = TLI ? TLI->getDataLayout() : 0;
   if (!TD) return false;
 
   // Lower all default uses of _chk calls.  This is very similar
@@ -649,6 +661,7 @@ bool CodeGenPrepare::OptimizeCallInst(CallInst *CI) {
 /// DupRetToEnableTailCallOpts - Look for opportunities to duplicate return
 /// instructions to the predecessor to enable tail call optimizations. The
 /// case it is currently looking for is:
+/// @code
 /// bb0:
 ///   %tmp0 = tail call i32 @f0()
 ///   br label %return
@@ -661,9 +674,11 @@ bool CodeGenPrepare::OptimizeCallInst(CallInst *CI) {
 /// return:
 ///   %retval = phi i32 [ %tmp0, %bb0 ], [ %tmp1, %bb1 ], [ %tmp2, %bb2 ]
 ///   ret i32 %retval
+/// @endcode
 ///
 /// =>
 ///
+/// @code
 /// bb0:
 ///   %tmp0 = tail call i32 @f0()
 ///   ret i32 %tmp0
@@ -673,7 +688,7 @@ bool CodeGenPrepare::OptimizeCallInst(CallInst *CI) {
 /// bb2:
 ///   %tmp2 = tail call i32 @f2()
 ///   ret i32 %tmp2
-///
+/// @endcode
 bool CodeGenPrepare::DupRetToEnableTailCallOpts(ReturnInst *RI) {
   if (!TLI)
     return false;
@@ -699,7 +714,8 @@ bool CodeGenPrepare::DupRetToEnableTailCallOpts(ReturnInst *RI) {
   // See llvm::isInTailCallPosition().
   const Function *F = BB->getParent();
   Attributes CallerRetAttr = F->getAttributes().getRetAttributes();
-  if ((CallerRetAttr & Attribute::ZExt) || (CallerRetAttr & Attribute::SExt))
+  if (CallerRetAttr.hasAttribute(Attributes::ZExt) ||
+      CallerRetAttr.hasAttribute(Attributes::SExt))
     return false;
 
   // Make sure there are no instructions between the PHI and return, or that the
@@ -757,7 +773,10 @@ bool CodeGenPrepare::DupRetToEnableTailCallOpts(ReturnInst *RI) {
     // Conservatively require the attributes of the call to match those of the
     // return. Ignore noalias because it doesn't affect the call sequence.
     Attributes CalleeRetAttr = CS.getAttributes().getRetAttributes();
-    if ((CalleeRetAttr ^ CallerRetAttr) & ~Attribute::NoAlias)
+    if (AttrBuilder(CalleeRetAttr).
+          removeAttribute(Attributes::NoAlias) !=
+        AttrBuilder(CallerRetAttr).
+          removeAttribute(Attributes::NoAlias))
       continue;
 
     // Make sure the call instruction is followed by an unconditional branch to
@@ -774,7 +793,7 @@ bool CodeGenPrepare::DupRetToEnableTailCallOpts(ReturnInst *RI) {
   }
 
   // If we eliminated all predecessors of the block, delete the block now.
-  if (Changed && pred_begin(BB) == pred_end(BB))
+  if (Changed && !BB->hasAddressTaken() && pred_begin(BB) == pred_end(BB))
     BB->eraseFromParent();
 
   return Changed;
@@ -914,7 +933,7 @@ bool CodeGenPrepare::OptimizeMemoryInst(Instruction *MemoryInst, Value *Addr,
     DEBUG(dbgs() << "CGP: SINKING nonlocal addrmode: " << AddrMode << " for "
                  << *MemoryInst);
     Type *IntPtrTy =
-          TLI->getTargetData()->getIntPtrType(AccessTy->getContext());
+          TLI->getDataLayout()->getIntPtrType(AccessTy->getContext());
 
     Value *Result = 0;
 
@@ -988,7 +1007,7 @@ bool CodeGenPrepare::OptimizeMemoryInst(Instruction *MemoryInst, Value *Addr,
     WeakVH IterHandle(CurInstIterator);
     BasicBlock *BB = CurInstIterator->getParent();
 
-    RecursivelyDeleteTriviallyDeadInstructions(Repl);
+    RecursivelyDeleteTriviallyDeadInstructions(Repl, TLInfo);
 
     if (IterHandle != CurInstIterator) {
       // If the iterator instruction was recursively deleted, start over at the
@@ -1174,17 +1193,32 @@ static bool isFormingBranchFromSelectProfitable(SelectInst *SI) {
 }
 
 
+/// If we have a SelectInst that will likely profit from branch prediction,
+/// turn it into a branch.
 bool CodeGenPrepare::OptimizeSelectInst(SelectInst *SI) {
-  // If we have a SelectInst that will likely profit from branch prediction,
-  // turn it into a branch.
-  if (DisableSelectToBranch || OptSize || !TLI ||
-      !TLI->isPredictableSelectExpensive())
-    return false;
+  bool VectorCond = !SI->getCondition()->getType()->isIntegerTy(1);
 
-  if (!SI->getCondition()->getType()->isIntegerTy(1) ||
-      !isFormingBranchFromSelectProfitable(SI))
+  // Can we convert the 'select' to CF ?
+  if (DisableSelectToBranch || OptSize || !TLI || VectorCond)
     return false;
 
+  TargetLowering::SelectSupportKind SelectKind;
+  if (VectorCond)
+    SelectKind = TargetLowering::VectorMaskSelect;
+  else if (SI->getType()->isVectorTy())
+    SelectKind = TargetLowering::ScalarCondVectorVal;
+  else
+    SelectKind = TargetLowering::ScalarValSelect;
+
+  // Do we have efficient codegen support for this kind of 'selects' ?
+  if (TLI->isSelectSupported(SelectKind)) {
+    // We have efficient codegen support for the select instruction.
+    // Check if it is profitable to keep this 'select'.
+    if (!TLI->isPredictableSelectExpensive() ||
+        !isFormingBranchFromSelectProfitable(SI))
+      return false;
+  }
+
   ModifiedDT = true;
 
   // First, we split the block containing the select into 2 blocks.
@@ -1302,7 +1336,7 @@ bool CodeGenPrepare::OptimizeBlock(BasicBlock &BB) {
   bool MadeChange = false;
 
   CurInstIterator = BB.begin();
-  for (BasicBlock::iterator E = BB.end(); CurInstIterator != E; )
+  while (CurInstIterator != BB.end())
     MadeChange |= OptimizeInst(CurInstIterator++);
 
   return MadeChange;
diff --git a/lib/Transforms/Scalar/ConstantProp.cpp b/lib/Transforms/Scalar/ConstantProp.cpp
index 5430f62..369720b 100644
--- a/lib/Transforms/Scalar/ConstantProp.cpp
+++ b/lib/Transforms/Scalar/ConstantProp.cpp
@@ -24,7 +24,7 @@
 #include "llvm/Constant.h"
 #include "llvm/Instruction.h"
 #include "llvm/Pass.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetLibraryInfo.h"
 #include "llvm/Support/InstIterator.h"
 #include "llvm/ADT/Statistic.h"
@@ -67,7 +67,7 @@ bool ConstantPropagation::runOnFunction(Function &F) {
       WorkList.insert(&*i);
   }
   bool Changed = false;
-  TargetData *TD = getAnalysisIfAvailable<TargetData>();
+  DataLayout *TD = getAnalysisIfAvailable<DataLayout>();
   TargetLibraryInfo *TLI = &getAnalysis<TargetLibraryInfo>();
 
   while (!WorkList.empty()) {
diff --git a/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp b/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
index 9b0aadb..3ec6f3d 100644
--- a/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
+++ b/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
@@ -235,6 +235,11 @@ bool CorrelatedValuePropagation::processSwitch(SwitchInst *SI) {
       // This case never fires - remove it.
       CI.getCaseSuccessor()->removePredecessor(BB);
       SI->removeCase(CI); // Does not invalidate the iterator.
+
+      // The condition can be modified by removePredecessor's PHI simplification
+      // logic.
+      Cond = SI->getCondition();
+
       ++NumDeadCases;
       Changed = true;
     } else if (State == LazyValueInfo::True) {
diff --git a/lib/Transforms/Scalar/DCE.cpp b/lib/Transforms/Scalar/DCE.cpp
index 8dbcc23..a2e074f 100644
--- a/lib/Transforms/Scalar/DCE.cpp
+++ b/lib/Transforms/Scalar/DCE.cpp
@@ -22,6 +22,7 @@
 #include "llvm/Instruction.h"
 #include "llvm/Pass.h"
 #include "llvm/Support/InstIterator.h"
+#include "llvm/Target/TargetLibraryInfo.h"
 #include "llvm/ADT/Statistic.h"
 using namespace llvm;
 
@@ -38,10 +39,11 @@ namespace {
       initializeDeadInstEliminationPass(*PassRegistry::getPassRegistry());
     }
     virtual bool runOnBasicBlock(BasicBlock &BB) {
+      TargetLibraryInfo *TLI = getAnalysisIfAvailable<TargetLibraryInfo>();
       bool Changed = false;
       for (BasicBlock::iterator DI = BB.begin(); DI != BB.end(); ) {
         Instruction *Inst = DI++;
-        if (isInstructionTriviallyDead(Inst)) {
+        if (isInstructionTriviallyDead(Inst, TLI)) {
           Inst->eraseFromParent();
           Changed = true;
           ++DIEEliminated;
@@ -87,6 +89,8 @@ char DCE::ID = 0;
 INITIALIZE_PASS(DCE, "dce", "Dead Code Elimination", false, false)
 
 bool DCE::runOnFunction(Function &F) {
+  TargetLibraryInfo *TLI = getAnalysisIfAvailable<TargetLibraryInfo>();
+
   // Start out with all of the instructions in the worklist...
   std::vector<Instruction*> WorkList;
   for (inst_iterator i = inst_begin(F), e = inst_end(F); i != e; ++i)
@@ -101,7 +105,7 @@ bool DCE::runOnFunction(Function &F) {
     Instruction *I = WorkList.back();
     WorkList.pop_back();
 
-    if (isInstructionTriviallyDead(I)) {       // If the instruction is dead.
+    if (isInstructionTriviallyDead(I, TLI)) { // If the instruction is dead.
       // Loop over all of the values that the instruction uses, if there are
       // instructions being used, add them to the worklist, because they might
       // go dead after this one is removed.
@@ -114,13 +118,8 @@ bool DCE::runOnFunction(Function &F) {
       I->eraseFromParent();
 
       // Remove the instruction from the worklist if it still exists in it.
-      for (std::vector<Instruction*>::iterator WI = WorkList.begin();
-           WI != WorkList.end(); ) {
-        if (*WI == I)
-          WI = WorkList.erase(WI);
-        else
-          ++WI;
-      }
+      WorkList.erase(std::remove(WorkList.begin(), WorkList.end(), I),
+                     WorkList.end());
 
       MadeChange = true;
       ++DCEEliminated;
diff --git a/lib/Transforms/Scalar/DeadStoreElimination.cpp b/lib/Transforms/Scalar/DeadStoreElimination.cpp
index 8b1283f..736cc05 100644
--- a/lib/Transforms/Scalar/DeadStoreElimination.cpp
+++ b/lib/Transforms/Scalar/DeadStoreElimination.cpp
@@ -29,7 +29,8 @@
 #include "llvm/Analysis/MemoryBuiltins.h"
 #include "llvm/Analysis/MemoryDependenceAnalysis.h"
 #include "llvm/Analysis/ValueTracking.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
+#include "llvm/Target/TargetLibraryInfo.h"
 #include "llvm/Transforms/Utils/Local.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/ADT/SetVector.h"
@@ -45,6 +46,7 @@ namespace {
     AliasAnalysis *AA;
     MemoryDependenceAnalysis *MD;
     DominatorTree *DT;
+    const TargetLibraryInfo *TLI;
 
     static char ID; // Pass identification, replacement for typeid
     DSE() : FunctionPass(ID), AA(0), MD(0), DT(0) {
@@ -55,6 +57,7 @@ namespace {
       AA = &getAnalysis<AliasAnalysis>();
       MD = &getAnalysis<MemoryDependenceAnalysis>();
       DT = &getAnalysis<DominatorTree>();
+      TLI = AA->getTargetLibraryInfo();
 
       bool Changed = false;
       for (Function::iterator I = F.begin(), E = F.end(); I != E; ++I)
@@ -106,6 +109,7 @@ FunctionPass *llvm::createDeadStoreEliminationPass() { return new DSE(); }
 ///
 static void DeleteDeadInstruction(Instruction *I,
                                   MemoryDependenceAnalysis &MD,
+                                  const TargetLibraryInfo *TLI,
                                   SmallSetVector<Value*, 16> *ValueSet = 0) {
   SmallVector<Instruction*, 32> NowDeadInsts;
 
@@ -130,7 +134,7 @@ static void DeleteDeadInstruction(Instruction *I,
       if (!Op->use_empty()) continue;
 
       if (Instruction *OpI = dyn_cast<Instruction>(Op))
-        if (isInstructionTriviallyDead(OpI))
+        if (isInstructionTriviallyDead(OpI, TLI))
           NowDeadInsts.push_back(OpI);
     }
 
@@ -143,7 +147,7 @@ static void DeleteDeadInstruction(Instruction *I,
 
 /// hasMemoryWrite - Does this instruction write some memory?  This only returns
 /// true for things that we can analyze with other helpers below.
-static bool hasMemoryWrite(Instruction *I) {
+static bool hasMemoryWrite(Instruction *I, const TargetLibraryInfo *TLI) {
   if (isa<StoreInst>(I))
     return true;
   if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
@@ -158,6 +162,26 @@ static bool hasMemoryWrite(Instruction *I) {
       return true;
     }
   }
+  if (CallSite CS = I) {
+    if (Function *F = CS.getCalledFunction()) {
+      if (TLI && TLI->has(LibFunc::strcpy) &&
+          F->getName() == TLI->getName(LibFunc::strcpy)) {
+        return true;
+      }
+      if (TLI && TLI->has(LibFunc::strncpy) &&
+          F->getName() == TLI->getName(LibFunc::strncpy)) {
+        return true;
+      }
+      if (TLI && TLI->has(LibFunc::strcat) &&
+          F->getName() == TLI->getName(LibFunc::strcat)) {
+        return true;
+      }
+      if (TLI && TLI->has(LibFunc::strncat) &&
+          F->getName() == TLI->getName(LibFunc::strncat)) {
+        return true;
+      }
+    }
+  }
   return false;
 }
 
@@ -175,7 +199,7 @@ getLocForWrite(Instruction *Inst, AliasAnalysis &AA) {
     // If we don't have target data around, an unknown size in Location means
     // that we should use the size of the pointee type.  This isn't valid for
     // memset/memcpy, which writes more than an i8.
-    if (Loc.Size == AliasAnalysis::UnknownSize && AA.getTargetData() == 0)
+    if (Loc.Size == AliasAnalysis::UnknownSize && AA.getDataLayout() == 0)
       return AliasAnalysis::Location();
     return Loc;
   }
@@ -189,7 +213,7 @@ getLocForWrite(Instruction *Inst, AliasAnalysis &AA) {
     // If we don't have target data around, an unknown size in Location means
     // that we should use the size of the pointee type.  This isn't valid for
     // init.trampoline, which writes more than an i8.
-    if (AA.getTargetData() == 0) return AliasAnalysis::Location();
+    if (AA.getDataLayout() == 0) return AliasAnalysis::Location();
 
     // FIXME: We don't know the size of the trampoline, so we can't really
     // handle it here.
@@ -205,7 +229,8 @@ getLocForWrite(Instruction *Inst, AliasAnalysis &AA) {
 /// instruction if any.
 static AliasAnalysis::Location
 getLocForRead(Instruction *Inst, AliasAnalysis &AA) {
-  assert(hasMemoryWrite(Inst) && "Unknown instruction case");
+  assert(hasMemoryWrite(Inst, AA.getTargetLibraryInfo()) &&
+         "Unknown instruction case");
 
   // The only instructions that both read and write are the mem transfer
   // instructions (memcpy/memmove).
@@ -222,23 +247,29 @@ static bool isRemovable(Instruction *I) {
   if (StoreInst *SI = dyn_cast<StoreInst>(I))
     return SI->isUnordered();
 
-  IntrinsicInst *II = cast<IntrinsicInst>(I);
-  switch (II->getIntrinsicID()) {
-  default: llvm_unreachable("doesn't pass 'hasMemoryWrite' predicate");
-  case Intrinsic::lifetime_end:
-    // Never remove dead lifetime_end's, e.g. because it is followed by a
-    // free.
-    return false;
-  case Intrinsic::init_trampoline:
-    // Always safe to remove init_trampoline.
-    return true;
+  if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
+    switch (II->getIntrinsicID()) {
+    default: llvm_unreachable("doesn't pass 'hasMemoryWrite' predicate");
+    case Intrinsic::lifetime_end:
+      // Never remove dead lifetime_end's, e.g. because it is followed by a
+      // free.
+      return false;
+    case Intrinsic::init_trampoline:
+      // Always safe to remove init_trampoline.
+      return true;
 
-  case Intrinsic::memset:
-  case Intrinsic::memmove:
-  case Intrinsic::memcpy:
-    // Don't remove volatile memory intrinsics.
-    return !cast<MemIntrinsic>(II)->isVolatile();
+    case Intrinsic::memset:
+    case Intrinsic::memmove:
+    case Intrinsic::memcpy:
+      // Don't remove volatile memory intrinsics.
+      return !cast<MemIntrinsic>(II)->isVolatile();
+    }
   }
+
+  if (CallSite CS = I)
+    return CS.getInstruction()->use_empty();
+
+  return false;
 }
 
 
@@ -249,14 +280,19 @@ static bool isShortenable(Instruction *I) {
   if (isa<StoreInst>(I))
     return false;
 
-  IntrinsicInst *II = cast<IntrinsicInst>(I);
-  switch (II->getIntrinsicID()) {
-    default: return false;
-    case Intrinsic::memset:
-    case Intrinsic::memcpy:
-      // Do shorten memory intrinsics.
-      return true;
+  if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
+    switch (II->getIntrinsicID()) {
+      default: return false;
+      case Intrinsic::memset:
+      case Intrinsic::memcpy:
+        // Do shorten memory intrinsics.
+        return true;
+    }
   }
+
+  // Don't shorten libcalls calls for now.
+
+  return false;
 }
 
 /// getStoredPointerOperand - Return the pointer that is being written to.
@@ -266,17 +302,23 @@ static Value *getStoredPointerOperand(Instruction *I) {
   if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(I))
     return MI->getDest();
 
-  IntrinsicInst *II = cast<IntrinsicInst>(I);
-  switch (II->getIntrinsicID()) {
-  default: llvm_unreachable("Unexpected intrinsic!");
-  case Intrinsic::init_trampoline:
-    return II->getArgOperand(0);
+  if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
+    switch (II->getIntrinsicID()) {
+    default: llvm_unreachable("Unexpected intrinsic!");
+    case Intrinsic::init_trampoline:
+      return II->getArgOperand(0);
+    }
   }
+
+  CallSite CS = I;
+  // All the supported functions so far happen to have dest as their first
+  // argument.
+  return CS.getArgument(0);
 }
 
 static uint64_t getPointerSize(const Value *V, AliasAnalysis &AA) {
   uint64_t Size;
-  if (getObjectSize(V, Size, AA.getTargetData()))
+  if (getObjectSize(V, Size, AA.getDataLayout(), AA.getTargetLibraryInfo()))
     return Size;
   return AliasAnalysis::UnknownSize;
 }
@@ -309,10 +351,10 @@ static OverwriteResult isOverwrite(const AliasAnalysis::Location &Later,
     // comparison.
     if (Later.Size == AliasAnalysis::UnknownSize ||
         Earlier.Size == AliasAnalysis::UnknownSize) {
-      // If we have no TargetData information around, then the size of the store
+      // If we have no DataLayout information around, then the size of the store
       // is inferrable from the pointee type.  If they are the same type, then
       // we know that the store is safe.
-      if (AA.getTargetData() == 0 &&
+      if (AA.getDataLayout() == 0 &&
           Later.Ptr->getType() == Earlier.Ptr->getType())
         return OverwriteComplete;
 
@@ -328,13 +370,13 @@ static OverwriteResult isOverwrite(const AliasAnalysis::Location &Later,
   // larger than the earlier one.
   if (Later.Size == AliasAnalysis::UnknownSize ||
       Earlier.Size == AliasAnalysis::UnknownSize ||
-      AA.getTargetData() == 0)
+      AA.getDataLayout() == 0)
     return OverwriteUnknown;
 
   // Check to see if the later store is to the entire object (either a global,
   // an alloca, or a byval argument).  If so, then it clearly overwrites any
   // other store to the same object.
-  const TargetData &TD = *AA.getTargetData();
+  const DataLayout &TD = *AA.getDataLayout();
 
   const Value *UO1 = GetUnderlyingObject(P1, &TD),
               *UO2 = GetUnderlyingObject(P2, &TD);
@@ -454,13 +496,13 @@ bool DSE::runOnBasicBlock(BasicBlock &BB) {
     Instruction *Inst = BBI++;
 
     // Handle 'free' calls specially.
-    if (CallInst *F = isFreeCall(Inst)) {
+    if (CallInst *F = isFreeCall(Inst, TLI)) {
       MadeChange |= HandleFree(F);
       continue;
     }
 
     // If we find something that writes memory, get its memory dependence.
-    if (!hasMemoryWrite(Inst))
+    if (!hasMemoryWrite(Inst, TLI))
       continue;
 
     MemDepResult InstDep = MD->getDependency(Inst);
@@ -483,7 +525,7 @@ bool DSE::runOnBasicBlock(BasicBlock &BB) {
           // in case we need it.
           WeakVH NextInst(BBI);
 
-          DeleteDeadInstruction(SI, *MD);
+          DeleteDeadInstruction(SI, *MD, TLI);
 
           if (NextInst == 0)  // Next instruction deleted.
             BBI = BB.begin();
@@ -530,7 +572,7 @@ bool DSE::runOnBasicBlock(BasicBlock &BB) {
                 << *DepWrite << "\n  KILLER: " << *Inst << '\n');
 
           // Delete the store and now-dead instructions that feed it.
-          DeleteDeadInstruction(DepWrite, *MD);
+          DeleteDeadInstruction(DepWrite, *MD, TLI);
           ++NumFastStores;
           MadeChange = true;
 
@@ -627,7 +669,7 @@ bool DSE::HandleFree(CallInst *F) {
     MemDepResult Dep = MD->getPointerDependencyFrom(Loc, false, InstPt, BB);
     while (Dep.isDef() || Dep.isClobber()) {
       Instruction *Dependency = Dep.getInst();
-      if (!hasMemoryWrite(Dependency) || !isRemovable(Dependency))
+      if (!hasMemoryWrite(Dependency, TLI) || !isRemovable(Dependency))
         break;
 
       Value *DepPointer =
@@ -640,7 +682,7 @@ bool DSE::HandleFree(CallInst *F) {
       Instruction *Next = llvm::next(BasicBlock::iterator(Dependency));
 
       // DCE instructions only used to calculate that store
-      DeleteDeadInstruction(Dependency, *MD);
+      DeleteDeadInstruction(Dependency, *MD, TLI);
       ++NumFastStores;
       MadeChange = true;
 
@@ -659,6 +701,22 @@ bool DSE::HandleFree(CallInst *F) {
   return MadeChange;
 }
 
+namespace {
+  struct CouldRef {
+    typedef Value *argument_type;
+    const CallSite CS;
+    AliasAnalysis *AA;
+
+    bool operator()(Value *I) {
+      // See if the call site touches the value.
+      AliasAnalysis::ModRefResult A =
+        AA->getModRefInfo(CS, I, getPointerSize(I, *AA));
+
+      return A == AliasAnalysis::ModRef || A == AliasAnalysis::Ref;
+    }
+  };
+}
+
 /// handleEndBlock - Remove dead stores to stack-allocated locations in the
 /// function end block.  Ex:
 /// %A = alloca i32
@@ -680,7 +738,7 @@ bool DSE::handleEndBlock(BasicBlock &BB) {
 
     // Okay, so these are dead heap objects, but if the pointer never escapes
     // then it's leaked by this function anyways.
-    else if (isAllocLikeFn(I) && !PointerMayBeCaptured(I, true, true))
+    else if (isAllocLikeFn(I, TLI) && !PointerMayBeCaptured(I, true, true))
       DeadStackObjects.insert(I);
   }
 
@@ -696,7 +754,7 @@ bool DSE::handleEndBlock(BasicBlock &BB) {
     --BBI;
 
     // If we find a store, check to see if it points into a dead stack value.
-    if (hasMemoryWrite(BBI) && isRemovable(BBI)) {
+    if (hasMemoryWrite(BBI, TLI) && isRemovable(BBI)) {
       // See through pointer-to-pointer bitcasts
       SmallVector<Value *, 4> Pointers;
       GetUnderlyingObjects(getStoredPointerOperand(BBI), Pointers);
@@ -724,7 +782,7 @@ bool DSE::handleEndBlock(BasicBlock &BB) {
               dbgs() << '\n');
 
         // DCE instructions only used to calculate that store.
-        DeleteDeadInstruction(Dead, *MD, &DeadStackObjects);
+        DeleteDeadInstruction(Dead, *MD, TLI, &DeadStackObjects);
         ++NumFastStores;
         MadeChange = true;
         continue;
@@ -732,9 +790,9 @@ bool DSE::handleEndBlock(BasicBlock &BB) {
     }
 
     // Remove any dead non-memory-mutating instructions.
-    if (isInstructionTriviallyDead(BBI)) {
+    if (isInstructionTriviallyDead(BBI, TLI)) {
       Instruction *Inst = BBI++;
-      DeleteDeadInstruction(Inst, *MD, &DeadStackObjects);
+      DeleteDeadInstruction(Inst, *MD, TLI, &DeadStackObjects);
       ++NumFastOther;
       MadeChange = true;
       continue;
@@ -750,7 +808,7 @@ bool DSE::handleEndBlock(BasicBlock &BB) {
     if (CallSite CS = cast<Value>(BBI)) {
       // Remove allocation function calls from the list of dead stack objects; 
       // there can't be any references before the definition.
-      if (isAllocLikeFn(BBI))
+      if (isAllocLikeFn(BBI, TLI))
         DeadStackObjects.remove(BBI);
 
       // If this call does not access memory, it can't be loading any of our
@@ -760,20 +818,8 @@ bool DSE::handleEndBlock(BasicBlock &BB) {
 
       // If the call might load from any of our allocas, then any store above
       // the call is live.
-      SmallVector<Value*, 8> LiveAllocas;
-      for (SmallSetVector<Value*, 16>::iterator I = DeadStackObjects.begin(),
-           E = DeadStackObjects.end(); I != E; ++I) {
-        // See if the call site touches it.
-        AliasAnalysis::ModRefResult A =
-          AA->getModRefInfo(CS, *I, getPointerSize(*I, *AA));
-
-        if (A == AliasAnalysis::ModRef || A == AliasAnalysis::Ref)
-          LiveAllocas.push_back(*I);
-      }
-
-      for (SmallVector<Value*, 8>::iterator I = LiveAllocas.begin(),
-           E = LiveAllocas.end(); I != E; ++I)
-        DeadStackObjects.remove(*I);
+      CouldRef Pred = { CS, AA };
+      DeadStackObjects.remove_if(Pred);
 
       // If all of the allocas were clobbered by the call then we're not going
       // to find anything else to process.
@@ -816,6 +862,20 @@ bool DSE::handleEndBlock(BasicBlock &BB) {
   return MadeChange;
 }
 
+namespace {
+  struct CouldAlias {
+    typedef Value *argument_type;
+    const AliasAnalysis::Location &LoadedLoc;
+    AliasAnalysis *AA;
+
+    bool operator()(Value *I) {
+      // See if the loaded location could alias the stack location.
+      AliasAnalysis::Location StackLoc(I, getPointerSize(I, *AA));
+      return !AA->isNoAlias(StackLoc, LoadedLoc);
+    }
+  };
+}
+
 /// RemoveAccessedObjects - Check to see if the specified location may alias any
 /// of the stack objects in the DeadStackObjects set.  If so, they become live
 /// because the location is being loaded.
@@ -834,16 +894,7 @@ void DSE::RemoveAccessedObjects(const AliasAnalysis::Location &LoadedLoc,
     return;
   }
 
-  SmallVector<Value*, 16> NowLive;
-  for (SmallSetVector<Value*, 16>::iterator I = DeadStackObjects.begin(),
-       E = DeadStackObjects.end(); I != E; ++I) {
-    // See if the loaded location could alias the stack location.
-    AliasAnalysis::Location StackLoc(*I, getPointerSize(*I, *AA));
-    if (!AA->isNoAlias(StackLoc, LoadedLoc))
-      NowLive.push_back(*I);
-  }
-
-  for (SmallVector<Value*, 16>::iterator I = NowLive.begin(), E = NowLive.end();
-       I != E; ++I)
-    DeadStackObjects.remove(*I);
+  // Remove objects that could alias LoadedLoc.
+  CouldAlias Pred = { LoadedLoc, AA };
+  DeadStackObjects.remove_if(Pred);
 }
diff --git a/lib/Transforms/Scalar/EarlyCSE.cpp b/lib/Transforms/Scalar/EarlyCSE.cpp
index 9759549..101009d 100644
--- a/lib/Transforms/Scalar/EarlyCSE.cpp
+++ b/lib/Transforms/Scalar/EarlyCSE.cpp
@@ -18,11 +18,12 @@
 #include "llvm/Pass.h"
 #include "llvm/Analysis/Dominators.h"
 #include "llvm/Analysis/InstructionSimplify.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetLibraryInfo.h"
 #include "llvm/Transforms/Utils/Local.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/RecyclingAllocator.h"
+#include "llvm/ADT/Hashing.h"
 #include "llvm/ADT/ScopedHashTable.h"
 #include "llvm/ADT/Statistic.h"
 #include <deque>
@@ -90,35 +91,56 @@ template<> struct DenseMapInfo<SimpleValue> {
 
 unsigned DenseMapInfo<SimpleValue>::getHashValue(SimpleValue Val) {
   Instruction *Inst = Val.Inst;
-
   // Hash in all of the operands as pointers.
-  unsigned Res = 0;
-  for (unsigned i = 0, e = Inst->getNumOperands(); i != e; ++i)
-    Res ^= getHash(Inst->getOperand(i)) << (i & 0xF);
+  if (BinaryOperator* BinOp = dyn_cast<BinaryOperator>(Inst)) {
+    Value *LHS = BinOp->getOperand(0);
+    Value *RHS = BinOp->getOperand(1);
+    if (BinOp->isCommutative() && BinOp->getOperand(0) > BinOp->getOperand(1))
+      std::swap(LHS, RHS);
+
+    if (isa<OverflowingBinaryOperator>(BinOp)) {
+      // Hash the overflow behavior
+      unsigned Overflow =
+        BinOp->hasNoSignedWrap()   * OverflowingBinaryOperator::NoSignedWrap |
+        BinOp->hasNoUnsignedWrap() * OverflowingBinaryOperator::NoUnsignedWrap;
+      return hash_combine(BinOp->getOpcode(), Overflow, LHS, RHS);
+    }
 
-  if (CastInst *CI = dyn_cast<CastInst>(Inst))
-    Res ^= getHash(CI->getType());
-  else if (CmpInst *CI = dyn_cast<CmpInst>(Inst))
-    Res ^= CI->getPredicate();
-  else if (const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(Inst)) {
-    for (ExtractValueInst::idx_iterator I = EVI->idx_begin(),
-         E = EVI->idx_end(); I != E; ++I)
-      Res ^= *I;
-  } else if (const InsertValueInst *IVI = dyn_cast<InsertValueInst>(Inst)) {
-    for (InsertValueInst::idx_iterator I = IVI->idx_begin(),
-         E = IVI->idx_end(); I != E; ++I)
-      Res ^= *I;
-  } else {
-    // nothing extra to hash in.
-    assert((isa<CallInst>(Inst) ||
-            isa<BinaryOperator>(Inst) || isa<GetElementPtrInst>(Inst) ||
-            isa<SelectInst>(Inst) || isa<ExtractElementInst>(Inst) ||
-            isa<InsertElementInst>(Inst) || isa<ShuffleVectorInst>(Inst)) &&
-           "Invalid/unknown instruction");
+    return hash_combine(BinOp->getOpcode(), LHS, RHS);
   }
 
+  if (CmpInst *CI = dyn_cast<CmpInst>(Inst)) {
+    Value *LHS = CI->getOperand(0);
+    Value *RHS = CI->getOperand(1);
+    CmpInst::Predicate Pred = CI->getPredicate();
+    if (Inst->getOperand(0) > Inst->getOperand(1)) {
+      std::swap(LHS, RHS);
+      Pred = CI->getSwappedPredicate();
+    }
+    return hash_combine(Inst->getOpcode(), Pred, LHS, RHS);
+  }
+
+  if (CastInst *CI = dyn_cast<CastInst>(Inst))
+    return hash_combine(CI->getOpcode(), CI->getType(), CI->getOperand(0));
+
+  if (const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(Inst))
+    return hash_combine(EVI->getOpcode(), EVI->getOperand(0),
+                        hash_combine_range(EVI->idx_begin(), EVI->idx_end()));
+
+  if (const InsertValueInst *IVI = dyn_cast<InsertValueInst>(Inst))
+    return hash_combine(IVI->getOpcode(), IVI->getOperand(0),
+                        IVI->getOperand(1),
+                        hash_combine_range(IVI->idx_begin(), IVI->idx_end()));
+
+  assert((isa<CallInst>(Inst) || isa<BinaryOperator>(Inst) ||
+          isa<GetElementPtrInst>(Inst) || isa<SelectInst>(Inst) ||
+          isa<ExtractElementInst>(Inst) || isa<InsertElementInst>(Inst) ||
+          isa<ShuffleVectorInst>(Inst)) && "Invalid/unknown instruction");
+
   // Mix in the opcode.
-  return (Res << 1) ^ Inst->getOpcode();
+  return hash_combine(Inst->getOpcode(),
+                      hash_combine_range(Inst->value_op_begin(),
+                                         Inst->value_op_end()));
 }
 
 bool DenseMapInfo<SimpleValue>::isEqual(SimpleValue LHS, SimpleValue RHS) {
@@ -128,7 +150,41 @@ bool DenseMapInfo<SimpleValue>::isEqual(SimpleValue LHS, SimpleValue RHS) {
     return LHSI == RHSI;
 
   if (LHSI->getOpcode() != RHSI->getOpcode()) return false;
-  return LHSI->isIdenticalTo(RHSI);
+  if (LHSI->isIdenticalTo(RHSI)) return true;
+
+  // If we're not strictly identical, we still might be a commutable instruction
+  if (BinaryOperator *LHSBinOp = dyn_cast<BinaryOperator>(LHSI)) {
+    if (!LHSBinOp->isCommutative())
+      return false;
+
+    assert(isa<BinaryOperator>(RHSI)
+           && "same opcode, but different instruction type?");
+    BinaryOperator *RHSBinOp = cast<BinaryOperator>(RHSI);
+
+    // Check overflow attributes
+    if (isa<OverflowingBinaryOperator>(LHSBinOp)) {
+      assert(isa<OverflowingBinaryOperator>(RHSBinOp)
+             && "same opcode, but different operator type?");
+      if (LHSBinOp->hasNoUnsignedWrap() != RHSBinOp->hasNoUnsignedWrap() ||
+          LHSBinOp->hasNoSignedWrap() != RHSBinOp->hasNoSignedWrap())
+        return false;
+    }
+
+    // Commuted equality
+    return LHSBinOp->getOperand(0) == RHSBinOp->getOperand(1) &&
+      LHSBinOp->getOperand(1) == RHSBinOp->getOperand(0);
+  }
+  if (CmpInst *LHSCmp = dyn_cast<CmpInst>(LHSI)) {
+    assert(isa<CmpInst>(RHSI)
+           && "same opcode, but different instruction type?");
+    CmpInst *RHSCmp = cast<CmpInst>(RHSI);
+    // Commuted equality
+    return LHSCmp->getOperand(0) == RHSCmp->getOperand(1) &&
+      LHSCmp->getOperand(1) == RHSCmp->getOperand(0) &&
+      LHSCmp->getSwappedPredicate() == RHSCmp->getPredicate();
+  }
+
+  return false;
 }
 
 //===----------------------------------------------------------------------===//
@@ -216,7 +272,7 @@ namespace {
 /// cases.
 class EarlyCSE : public FunctionPass {
 public:
-  const TargetData *TD;
+  const DataLayout *TD;
   const TargetLibraryInfo *TLI;
   DominatorTree *DT;
   typedef RecyclingAllocator<BumpPtrAllocator,
@@ -274,7 +330,8 @@ private:
         CallScope(*availableCalls) {}
 
    private:
-    NodeScope(const NodeScope&); // DO NOT IMPLEMENT
+    NodeScope(const NodeScope&) LLVM_DELETED_FUNCTION;
+    void operator=(const NodeScope&) LLVM_DELETED_FUNCTION;
 
     ScopedHTType::ScopeTy Scope;
     LoadHTType::ScopeTy LoadScope;
@@ -313,7 +370,8 @@ private:
     void process() { Processed = true; }
 
    private:
-    StackNode(const StackNode&); // DO NOT IMPLEMENT
+    StackNode(const StackNode&) LLVM_DELETED_FUNCTION;
+    void operator=(const StackNode&) LLVM_DELETED_FUNCTION;
 
     // Members.
     unsigned CurrentGeneration;
@@ -374,7 +432,7 @@ bool EarlyCSE::processNode(DomTreeNode *Node) {
     Instruction *Inst = I++;
 
     // Dead instructions should just be removed.
-    if (isInstructionTriviallyDead(Inst)) {
+    if (isInstructionTriviallyDead(Inst, TLI)) {
       DEBUG(dbgs() << "EarlyCSE DCE: " << *Inst << '\n');
       Inst->eraseFromParent();
       Changed = true;
@@ -506,7 +564,7 @@ bool EarlyCSE::processNode(DomTreeNode *Node) {
 bool EarlyCSE::runOnFunction(Function &F) {
   std::deque<StackNode *> nodesToProcess;
 
-  TD = getAnalysisIfAvailable<TargetData>();
+  TD = getAnalysisIfAvailable<DataLayout>();
   TLI = &getAnalysis<TargetLibraryInfo>();
   DT = &getAnalysis<DominatorTree>();
 
diff --git a/lib/Transforms/Scalar/GVN.cpp b/lib/Transforms/Scalar/GVN.cpp
index 4822fd0..f003e06 100644
--- a/lib/Transforms/Scalar/GVN.cpp
+++ b/lib/Transforms/Scalar/GVN.cpp
@@ -41,7 +41,7 @@
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/PatternMatch.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetLibraryInfo.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/Transforms/Utils/SSAUpdater.h"
@@ -271,16 +271,16 @@ void ValueTable::add(Value *V, uint32_t num) {
   valueNumbering.insert(std::make_pair(V, num));
 }
 
-uint32_t ValueTable::lookup_or_add_call(CallInst* C) {
+uint32_t ValueTable::lookup_or_add_call(CallInst *C) {
   if (AA->doesNotAccessMemory(C)) {
     Expression exp = create_expression(C);
-    uint32_t& e = expressionNumbering[exp];
+    uint32_t &e = expressionNumbering[exp];
     if (!e) e = nextValueNumber++;
     valueNumbering[C] = e;
     return e;
   } else if (AA->onlyReadsMemory(C)) {
     Expression exp = create_expression(C);
-    uint32_t& e = expressionNumbering[exp];
+    uint32_t &e = expressionNumbering[exp];
     if (!e) {
       e = nextValueNumber++;
       valueNumbering[C] = e;
@@ -413,7 +413,7 @@ uint32_t ValueTable::lookup_or_add(Value *V) {
     case Instruction::LShr:
     case Instruction::AShr:
     case Instruction::And:
-    case Instruction::Or :
+    case Instruction::Or:
     case Instruction::Xor:
     case Instruction::ICmp:
     case Instruction::FCmp:
@@ -503,7 +503,7 @@ namespace {
     bool NoLoads;
     MemoryDependenceAnalysis *MD;
     DominatorTree *DT;
-    const TargetData *TD;
+    const DataLayout *TD;
     const TargetLibraryInfo *TLI;
 
     ValueTable VN;
@@ -535,7 +535,7 @@ namespace {
       InstrsToErase.push_back(I);
     }
 
-    const TargetData *getTargetData() const { return TD; }
+    const DataLayout *getDataLayout() const { return TD; }
     DominatorTree &getDominatorTree() const { return *DT; }
     AliasAnalysis *getAliasAnalysis() const { return VN.getAliasAnalysis(); }
     MemoryDependenceAnalysis &getMemDep() const { return *MD; }
@@ -632,6 +632,7 @@ INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfo)
 INITIALIZE_AG_DEPENDENCY(AliasAnalysis)
 INITIALIZE_PASS_END(GVN, "gvn", "Global Value Numbering", false, false)
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void GVN::dump(DenseMap<uint32_t, Value*>& d) {
   errs() << "{\n";
   for (DenseMap<uint32_t, Value*>::iterator I = d.begin(),
@@ -641,6 +642,7 @@ void GVN::dump(DenseMap<uint32_t, Value*>& d) {
   }
   errs() << "}\n";
 }
+#endif
 
 /// IsValueFullyAvailableInBlock - Return true if we can prove that the value
 /// we're analyzing is fully available in the specified block.  As we go, keep
@@ -728,7 +730,7 @@ SpeculationFailure:
 /// CoerceAvailableValueToLoadType will succeed.
 static bool CanCoerceMustAliasedValueToLoad(Value *StoredVal,
                                             Type *LoadTy,
-                                            const TargetData &TD) {
+                                            const DataLayout &TD) {
   // If the loaded or stored value is an first class array or struct, don't try
   // to transform them.  We need to be able to bitcast to integer.
   if (LoadTy->isStructTy() || LoadTy->isArrayTy() ||
@@ -744,7 +746,6 @@ static bool CanCoerceMustAliasedValueToLoad(Value *StoredVal,
   return true;
 }
 
-
 /// CoerceAvailableValueToLoadType - If we saw a store of a value to memory, and
 /// then a load from a must-aliased pointer of a different type, try to coerce
 /// the stored value.  LoadedTy is the type of the load we want to replace and
@@ -754,7 +755,7 @@ static bool CanCoerceMustAliasedValueToLoad(Value *StoredVal,
 static Value *CoerceAvailableValueToLoadType(Value *StoredVal,
                                              Type *LoadedTy,
                                              Instruction *InsertPt,
-                                             const TargetData &TD) {
+                                             const DataLayout &TD) {
   if (!CanCoerceMustAliasedValueToLoad(StoredVal, LoadedTy, TD))
     return 0;
 
@@ -767,24 +768,25 @@ static Value *CoerceAvailableValueToLoadType(Value *StoredVal,
   // If the store and reload are the same size, we can always reuse it.
   if (StoreSize == LoadSize) {
     // Pointer to Pointer -> use bitcast.
-    if (StoredValTy->isPointerTy() && LoadedTy->isPointerTy())
+    if (StoredValTy->getScalarType()->isPointerTy() &&
+        LoadedTy->getScalarType()->isPointerTy())
       return new BitCastInst(StoredVal, LoadedTy, "", InsertPt);
 
     // Convert source pointers to integers, which can be bitcast.
-    if (StoredValTy->isPointerTy()) {
-      StoredValTy = TD.getIntPtrType(StoredValTy->getContext());
+    if (StoredValTy->getScalarType()->isPointerTy()) {
+      StoredValTy = TD.getIntPtrType(StoredValTy);
       StoredVal = new PtrToIntInst(StoredVal, StoredValTy, "", InsertPt);
     }
 
     Type *TypeToCastTo = LoadedTy;
-    if (TypeToCastTo->isPointerTy())
-      TypeToCastTo = TD.getIntPtrType(StoredValTy->getContext());
+    if (TypeToCastTo->getScalarType()->isPointerTy())
+      TypeToCastTo = TD.getIntPtrType(TypeToCastTo);
 
     if (StoredValTy != TypeToCastTo)
       StoredVal = new BitCastInst(StoredVal, TypeToCastTo, "", InsertPt);
 
     // Cast to pointer if the load needs a pointer type.
-    if (LoadedTy->isPointerTy())
+    if (LoadedTy->getScalarType()->isPointerTy())
       StoredVal = new IntToPtrInst(StoredVal, LoadedTy, "", InsertPt);
 
     return StoredVal;
@@ -796,8 +798,8 @@ static Value *CoerceAvailableValueToLoadType(Value *StoredVal,
   assert(StoreSize >= LoadSize && "CanCoerceMustAliasedValueToLoad fail");
 
   // Convert source pointers to integers, which can be manipulated.
-  if (StoredValTy->isPointerTy()) {
-    StoredValTy = TD.getIntPtrType(StoredValTy->getContext());
+  if (StoredValTy->getScalarType()->isPointerTy()) {
+    StoredValTy = TD.getIntPtrType(StoredValTy);
     StoredVal = new PtrToIntInst(StoredVal, StoredValTy, "", InsertPt);
   }
 
@@ -822,7 +824,7 @@ static Value *CoerceAvailableValueToLoadType(Value *StoredVal,
     return StoredVal;
 
   // If the result is a pointer, inttoptr.
-  if (LoadedTy->isPointerTy())
+  if (LoadedTy->getScalarType()->isPointerTy())
     return new IntToPtrInst(StoredVal, LoadedTy, "inttoptr", InsertPt);
 
   // Otherwise, bitcast.
@@ -840,7 +842,7 @@ static Value *CoerceAvailableValueToLoadType(Value *StoredVal,
 static int AnalyzeLoadFromClobberingWrite(Type *LoadTy, Value *LoadPtr,
                                           Value *WritePtr,
                                           uint64_t WriteSizeInBits,
-                                          const TargetData &TD) {
+                                          const DataLayout &TD) {
   // If the loaded or stored value is a first class array or struct, don't try
   // to transform them.  We need to be able to bitcast to integer.
   if (LoadTy->isStructTy() || LoadTy->isArrayTy())
@@ -913,7 +915,7 @@ static int AnalyzeLoadFromClobberingWrite(Type *LoadTy, Value *LoadPtr,
 /// memdep query of a load that ends up being a clobbering store.
 static int AnalyzeLoadFromClobberingStore(Type *LoadTy, Value *LoadPtr,
                                           StoreInst *DepSI,
-                                          const TargetData &TD) {
+                                          const DataLayout &TD) {
   // Cannot handle reading from store of first-class aggregate yet.
   if (DepSI->getValueOperand()->getType()->isStructTy() ||
       DepSI->getValueOperand()->getType()->isArrayTy())
@@ -929,7 +931,7 @@ static int AnalyzeLoadFromClobberingStore(Type *LoadTy, Value *LoadPtr,
 /// memdep query of a load that ends up being clobbered by another load.  See if
 /// the other load can feed into the second load.
 static int AnalyzeLoadFromClobberingLoad(Type *LoadTy, Value *LoadPtr,
-                                         LoadInst *DepLI, const TargetData &TD){
+                                         LoadInst *DepLI, const DataLayout &TD){
   // Cannot handle reading from store of first-class aggregate yet.
   if (DepLI->getType()->isStructTy() || DepLI->getType()->isArrayTy())
     return -1;
@@ -957,7 +959,7 @@ static int AnalyzeLoadFromClobberingLoad(Type *LoadTy, Value *LoadPtr,
 
 static int AnalyzeLoadFromClobberingMemInst(Type *LoadTy, Value *LoadPtr,
                                             MemIntrinsic *MI,
-                                            const TargetData &TD) {
+                                            const DataLayout &TD) {
   // If the mem operation is a non-constant size, we can't handle it.
   ConstantInt *SizeCst = dyn_cast<ConstantInt>(MI->getLength());
   if (SizeCst == 0) return -1;
@@ -1007,7 +1009,7 @@ static int AnalyzeLoadFromClobberingMemInst(Type *LoadTy, Value *LoadPtr,
 /// before we give up.
 static Value *GetStoreValueForLoad(Value *SrcVal, unsigned Offset,
                                    Type *LoadTy,
-                                   Instruction *InsertPt, const TargetData &TD){
+                                   Instruction *InsertPt, const DataLayout &TD){
   LLVMContext &Ctx = SrcVal->getType()->getContext();
 
   uint64_t StoreSize = (TD.getTypeSizeInBits(SrcVal->getType()) + 7) / 8;
@@ -1017,8 +1019,9 @@ static Value *GetStoreValueForLoad(Value *SrcVal, unsigned Offset,
 
   // Compute which bits of the stored value are being used by the load.  Convert
   // to an integer type to start with.
-  if (SrcVal->getType()->isPointerTy())
-    SrcVal = Builder.CreatePtrToInt(SrcVal, TD.getIntPtrType(Ctx));
+  if (SrcVal->getType()->getScalarType()->isPointerTy())
+    SrcVal = Builder.CreatePtrToInt(SrcVal,
+        TD.getIntPtrType(SrcVal->getType()));
   if (!SrcVal->getType()->isIntegerTy())
     SrcVal = Builder.CreateBitCast(SrcVal, IntegerType::get(Ctx, StoreSize*8));
 
@@ -1046,7 +1049,7 @@ static Value *GetStoreValueForLoad(Value *SrcVal, unsigned Offset,
 static Value *GetLoadValueForLoad(LoadInst *SrcVal, unsigned Offset,
                                   Type *LoadTy, Instruction *InsertPt,
                                   GVN &gvn) {
-  const TargetData &TD = *gvn.getTargetData();
+  const DataLayout &TD = *gvn.getDataLayout();
   // If Offset+LoadTy exceeds the size of SrcVal, then we must be wanting to
   // widen SrcVal out to a larger load.
   unsigned SrcValSize = TD.getTypeStoreSize(SrcVal->getType());
@@ -1105,7 +1108,7 @@ static Value *GetLoadValueForLoad(LoadInst *SrcVal, unsigned Offset,
 /// memdep query of a load that ends up being a clobbering mem intrinsic.
 static Value *GetMemInstValueForLoad(MemIntrinsic *SrcInst, unsigned Offset,
                                      Type *LoadTy, Instruction *InsertPt,
-                                     const TargetData &TD){
+                                     const DataLayout &TD){
   LLVMContext &Ctx = LoadTy->getContext();
   uint64_t LoadSize = TD.getTypeSizeInBits(LoadTy)/8;
 
@@ -1229,7 +1232,7 @@ struct AvailableValueInBlock {
     if (isSimpleValue()) {
       Res = getSimpleValue();
       if (Res->getType() != LoadTy) {
-        const TargetData *TD = gvn.getTargetData();
+        const DataLayout *TD = gvn.getDataLayout();
         assert(TD && "Need target data to handle type mismatch case");
         Res = GetStoreValueForLoad(Res, Offset, LoadTy, BB->getTerminator(),
                                    *TD);
@@ -1251,7 +1254,7 @@ struct AvailableValueInBlock {
                      << *Res << '\n' << "\n\n\n");
       }
     } else {
-      const TargetData *TD = gvn.getTargetData();
+      const DataLayout *TD = gvn.getDataLayout();
       assert(TD && "Need target data to handle type mismatch case");
       Res = GetMemInstValueForLoad(getMemIntrinValue(), Offset,
                                    LoadTy, BB->getTerminator(), *TD);
@@ -1299,7 +1302,7 @@ static Value *ConstructSSAForLoadSet(LoadInst *LI,
   Value *V = SSAUpdate.GetValueInMiddleOfBlock(LI->getParent());
 
   // If new PHI nodes were created, notify alias analysis.
-  if (V->getType()->isPointerTy()) {
+  if (V->getType()->getScalarType()->isPointerTy()) {
     AliasAnalysis *AA = gvn.getAliasAnalysis();
 
     for (unsigned i = 0, e = NewPHIs.size(); i != e; ++i)
@@ -1436,7 +1439,7 @@ bool GVN::processNonLocalLoad(LoadInst *LI) {
     Instruction *DepInst = DepInfo.getInst();
 
     // Loading the allocation -> undef.
-    if (isa<AllocaInst>(DepInst) || isMallocLikeFn(DepInst) ||
+    if (isa<AllocaInst>(DepInst) || isMallocLikeFn(DepInst, TLI) ||
         // Loading immediately after lifetime begin -> undef.
         isLifetimeStart(DepInst)) {
       ValuesPerBlock.push_back(AvailableValueInBlock::get(DepBB,
@@ -1496,7 +1499,7 @@ bool GVN::processNonLocalLoad(LoadInst *LI) {
 
     if (isa<PHINode>(V))
       V->takeName(LI);
-    if (V->getType()->isPointerTy())
+    if (V->getType()->getScalarType()->isPointerTy())
       MD->invalidateCachedPointerInfo(V);
     markInstructionForDeletion(LI);
     ++NumGVNLoad;
@@ -1728,7 +1731,7 @@ bool GVN::processNonLocalLoad(LoadInst *LI) {
   LI->replaceAllUsesWith(V);
   if (isa<PHINode>(V))
     V->takeName(LI);
-  if (V->getType()->isPointerTy())
+  if (V->getType()->getScalarType()->isPointerTy())
     MD->invalidateCachedPointerInfo(V);
   markInstructionForDeletion(LI);
   ++NumPRELoad;
@@ -1855,7 +1858,7 @@ bool GVN::processLoad(LoadInst *L) {
 
       // Replace the load!
       L->replaceAllUsesWith(AvailVal);
-      if (AvailVal->getType()->isPointerTy())
+      if (AvailVal->getType()->getScalarType()->isPointerTy())
         MD->invalidateCachedPointerInfo(AvailVal);
       markInstructionForDeletion(L);
       ++NumGVNLoad;
@@ -1912,7 +1915,7 @@ bool GVN::processLoad(LoadInst *L) {
 
     // Remove it!
     L->replaceAllUsesWith(StoredVal);
-    if (StoredVal->getType()->isPointerTy())
+    if (StoredVal->getType()->getScalarType()->isPointerTy())
       MD->invalidateCachedPointerInfo(StoredVal);
     markInstructionForDeletion(L);
     ++NumGVNLoad;
@@ -1941,7 +1944,7 @@ bool GVN::processLoad(LoadInst *L) {
 
     // Remove it!
     patchAndReplaceAllUsesWith(AvailableVal, L);
-    if (DepLI->getType()->isPointerTy())
+    if (DepLI->getType()->getScalarType()->isPointerTy())
       MD->invalidateCachedPointerInfo(DepLI);
     markInstructionForDeletion(L);
     ++NumGVNLoad;
@@ -1951,7 +1954,7 @@ bool GVN::processLoad(LoadInst *L) {
   // If this load really doesn't depend on anything, then we must be loading an
   // undef value.  This can happen when loading for a fresh allocation with no
   // intervening stores, for example.
-  if (isa<AllocaInst>(DepInst) || isMallocLikeFn(DepInst)) {
+  if (isa<AllocaInst>(DepInst) || isMallocLikeFn(DepInst, TLI)) {
     L->replaceAllUsesWith(UndefValue::get(L->getType()));
     markInstructionForDeletion(L);
     ++NumGVNLoad;
@@ -2182,7 +2185,7 @@ bool GVN::processInstruction(Instruction *I) {
   // "%z = and i32 %x, %y" becomes "%z = and i32 %x, %x" which we now simplify.
   if (Value *V = SimplifyInstruction(I, TD, TLI, DT)) {
     I->replaceAllUsesWith(V);
-    if (MD && V->getType()->isPointerTy())
+    if (MD && V->getType()->getScalarType()->isPointerTy())
       MD->invalidateCachedPointerInfo(V);
     markInstructionForDeletion(I);
     ++NumGVNSimpl;
@@ -2231,12 +2234,20 @@ bool GVN::processInstruction(Instruction *I) {
     Value *SwitchCond = SI->getCondition();
     BasicBlock *Parent = SI->getParent();
     bool Changed = false;
+
+    // Remember how many outgoing edges there are to every successor.
+    SmallDenseMap<BasicBlock *, unsigned, 16> SwitchEdges;
+    for (unsigned i = 0, n = SI->getNumSuccessors(); i != n; ++i)
+      ++SwitchEdges[SI->getSuccessor(i)];
+
     for (SwitchInst::CaseIt i = SI->case_begin(), e = SI->case_end();
          i != e; ++i) {
       BasicBlock *Dst = i.getCaseSuccessor();
-      BasicBlockEdge E(Parent, Dst);
-      if (E.isSingleEdge())
+      // If there is only a single edge, propagate the case value into it.
+      if (SwitchEdges.lookup(Dst) == 1) {
+        BasicBlockEdge E(Parent, Dst);
         Changed |= propagateEquality(SwitchCond, i.getCaseValue(), E);
+      }
     }
     return Changed;
   }
@@ -2274,7 +2285,7 @@ bool GVN::processInstruction(Instruction *I) {
 
   // Remove it!
   patchAndReplaceAllUsesWith(repl, I);
-  if (MD && repl->getType()->isPointerTy())
+  if (MD && repl->getType()->getScalarType()->isPointerTy())
     MD->invalidateCachedPointerInfo(repl);
   markInstructionForDeletion(I);
   return true;
@@ -2285,7 +2296,7 @@ bool GVN::runOnFunction(Function& F) {
   if (!NoLoads)
     MD = &getAnalysis<MemoryDependenceAnalysis>();
   DT = &getAnalysis<DominatorTree>();
-  TD = getAnalysisIfAvailable<TargetData>();
+  TD = getAnalysisIfAvailable<DataLayout>();
   TLI = &getAnalysis<TargetLibraryInfo>();
   VN.setAliasAnalysis(&getAnalysis<AliasAnalysis>());
   VN.setMemDep(MD);
@@ -2522,7 +2533,7 @@ bool GVN::performPRE(Function &F) {
       addToLeaderTable(ValNo, Phi, CurrentBlock);
       Phi->setDebugLoc(CurInst->getDebugLoc());
       CurInst->replaceAllUsesWith(Phi);
-      if (Phi->getType()->isPointerTy()) {
+      if (Phi->getType()->getScalarType()->isPointerTy()) {
         // Because we have added a PHI-use of the pointer value, it has now
         // "escaped" from alias analysis' perspective.  We need to inform
         // AA of this.
diff --git a/lib/Transforms/Scalar/GlobalMerge.cpp b/lib/Transforms/Scalar/GlobalMerge.cpp
index b36a3cb..6301aad 100644
--- a/lib/Transforms/Scalar/GlobalMerge.cpp
+++ b/lib/Transforms/Scalar/GlobalMerge.cpp
@@ -62,7 +62,7 @@
 #include "llvm/Intrinsics.h"
 #include "llvm/Module.h"
 #include "llvm/Pass.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetLowering.h"
 #include "llvm/Target/TargetLoweringObjectFile.h"
 #include "llvm/ADT/Statistic.h"
@@ -98,9 +98,9 @@ namespace {
     }
 
     struct GlobalCmp {
-      const TargetData *TD;
+      const DataLayout *TD;
 
-      GlobalCmp(const TargetData *td) : TD(td) { }
+      GlobalCmp(const DataLayout *td) : TD(td) { }
 
       bool operator()(const GlobalVariable *GV1, const GlobalVariable *GV2) {
         Type *Ty1 = cast<PointerType>(GV1->getType())->getElementType();
@@ -119,7 +119,7 @@ INITIALIZE_PASS(GlobalMerge, "global-merge",
 
 bool GlobalMerge::doMerge(SmallVectorImpl<GlobalVariable*> &Globals,
                              Module &M, bool isConst) const {
-  const TargetData *TD = TLI->getTargetData();
+  const DataLayout *TD = TLI->getDataLayout();
 
   // FIXME: Infer the maximum possible offset depending on the actual users
   // (these max offsets are different for the users inside Thumb or ARM
@@ -170,7 +170,7 @@ bool GlobalMerge::doMerge(SmallVectorImpl<GlobalVariable*> &Globals,
 
 bool GlobalMerge::doInitialization(Module &M) {
   SmallVector<GlobalVariable*, 16> Globals, ConstGlobals, BSSGlobals;
-  const TargetData *TD = TLI->getTargetData();
+  const DataLayout *TD = TLI->getDataLayout();
   unsigned MaxOffset = TLI->getMaximalGlobalOffset();
   bool Changed = false;
 
diff --git a/lib/Transforms/Scalar/IndVarSimplify.cpp b/lib/Transforms/Scalar/IndVarSimplify.cpp
index 37f8bdf..310fd61 100644
--- a/lib/Transforms/Scalar/IndVarSimplify.cpp
+++ b/lib/Transforms/Scalar/IndVarSimplify.cpp
@@ -43,7 +43,8 @@
 #include "llvm/Transforms/Utils/Local.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/Transforms/Utils/SimplifyIndVar.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
+#include "llvm/Target/TargetLibraryInfo.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
@@ -67,7 +68,8 @@ namespace {
     LoopInfo        *LI;
     ScalarEvolution *SE;
     DominatorTree   *DT;
-    TargetData      *TD;
+    DataLayout      *TD;
+    TargetLibraryInfo *TLI;
 
     SmallVector<WeakVH, 16> DeadInsts;
     bool Changed;
@@ -218,8 +220,6 @@ static Instruction *getInsertPointForUses(Instruction *User, Value *Def,
 /// ConvertToSInt - Convert APF to an integer, if possible.
 static bool ConvertToSInt(const APFloat &APF, int64_t &IntVal) {
   bool isExact = false;
-  if (&APF.getSemantics() == &APFloat::PPCDoubleDouble)
-    return false;
   // See if we can convert this to an int64_t
   uint64_t UIntVal;
   if (APF.convertToInteger(&UIntVal, 64, true, APFloat::rmTowardZero,
@@ -414,11 +414,11 @@ void IndVarSimplify::HandleFloatingPointIV(Loop *L, PHINode *PN) {
   // new comparison.
   NewCompare->takeName(Compare);
   Compare->replaceAllUsesWith(NewCompare);
-  RecursivelyDeleteTriviallyDeadInstructions(Compare);
+  RecursivelyDeleteTriviallyDeadInstructions(Compare, TLI);
 
   // Delete the old floating point increment.
   Incr->replaceAllUsesWith(UndefValue::get(Incr->getType()));
-  RecursivelyDeleteTriviallyDeadInstructions(Incr);
+  RecursivelyDeleteTriviallyDeadInstructions(Incr, TLI);
 
   // If the FP induction variable still has uses, this is because something else
   // in the loop uses its value.  In order to canonicalize the induction
@@ -431,7 +431,7 @@ void IndVarSimplify::HandleFloatingPointIV(Loop *L, PHINode *PN) {
     Value *Conv = new SIToFPInst(NewPHI, PN->getType(), "indvar.conv",
                                  PN->getParent()->getFirstInsertionPt());
     PN->replaceAllUsesWith(Conv);
-    RecursivelyDeleteTriviallyDeadInstructions(PN);
+    RecursivelyDeleteTriviallyDeadInstructions(PN, TLI);
   }
   Changed = true;
 }
@@ -549,15 +549,17 @@ void IndVarSimplify::RewriteLoopExitValues(Loop *L, SCEVExpander &Rewriter) {
 
         PN->setIncomingValue(i, ExitVal);
 
-        // If this instruction is dead now, delete it.
-        RecursivelyDeleteTriviallyDeadInstructions(Inst);
+        // If this instruction is dead now, delete it. Don't do it now to avoid
+        // invalidating iterators.
+        if (isInstructionTriviallyDead(Inst, TLI))
+          DeadInsts.push_back(Inst);
 
         if (NumPreds == 1) {
           // Completely replace a single-pred PHI. This is safe, because the
           // NewVal won't be variant in the loop, so we don't need an LCSSA phi
           // node anymore.
           PN->replaceAllUsesWith(ExitVal);
-          RecursivelyDeleteTriviallyDeadInstructions(PN);
+          PN->eraseFromParent();
         }
       }
       if (NumPreds != 1) {
@@ -595,13 +597,13 @@ namespace {
 
   class WideIVVisitor : public IVVisitor {
     ScalarEvolution *SE;
-    const TargetData *TD;
+    const DataLayout *TD;
 
   public:
     WideIVInfo WI;
 
     WideIVVisitor(PHINode *NarrowIV, ScalarEvolution *SCEV,
-                  const TargetData *TData) :
+                  const DataLayout *TData) :
       SE(SCEV), TD(TData) { WI.NarrowIV = NarrowIV; }
 
     // Implement the interface used by simplifyUsersOfIV.
@@ -1259,8 +1261,13 @@ static bool needsLFTR(Loop *L, DominatorTree *DT) {
   if (!Phi)
     return true;
 
+  // Do LFTR if PHI node is defined in the loop, but is *not* a counter.
+  int Idx = Phi->getBasicBlockIndex(L->getLoopLatch());
+  if (Idx < 0)
+    return true;
+
   // Do LFTR if the exit condition's IV is *not* a simple counter.
-  Value *IncV = Phi->getIncomingValueForBlock(L->getLoopLatch());
+  Value *IncV = Phi->getIncomingValue(Idx);
   return Phi != getLoopPhiForCounter(IncV, L, DT);
 }
 
@@ -1339,7 +1346,7 @@ static bool AlmostDeadIV(PHINode *Phi, BasicBlock *LatchBlock, Value *Cond) {
 /// could at least handle constant BECounts.
 static PHINode *
 FindLoopCounter(Loop *L, const SCEV *BECount,
-                ScalarEvolution *SE, DominatorTree *DT, const TargetData *TD) {
+                ScalarEvolution *SE, DominatorTree *DT, const DataLayout *TD) {
   uint64_t BCWidth = SE->getTypeSizeInBits(BECount->getType());
 
   Value *Cond =
@@ -1696,7 +1703,8 @@ bool IndVarSimplify::runOnLoop(Loop *L, LPPassManager &LPM) {
   LI = &getAnalysis<LoopInfo>();
   SE = &getAnalysis<ScalarEvolution>();
   DT = &getAnalysis<DominatorTree>();
-  TD = getAnalysisIfAvailable<TargetData>();
+  TD = getAnalysisIfAvailable<DataLayout>();
+  TLI = getAnalysisIfAvailable<TargetLibraryInfo>();
 
   DeadInsts.clear();
   Changed = false;
@@ -1763,7 +1771,7 @@ bool IndVarSimplify::runOnLoop(Loop *L, LPPassManager &LPM) {
   while (!DeadInsts.empty())
     if (Instruction *Inst =
           dyn_cast_or_null<Instruction>(&*DeadInsts.pop_back_val()))
-      RecursivelyDeleteTriviallyDeadInstructions(Inst);
+      RecursivelyDeleteTriviallyDeadInstructions(Inst, TLI);
 
   // The Rewriter may not be used from this point on.
 
@@ -1772,7 +1780,7 @@ bool IndVarSimplify::runOnLoop(Loop *L, LPPassManager &LPM) {
   SinkUnusedInvariants(L);
 
   // Clean up dead instructions.
-  Changed |= DeleteDeadPHIs(L->getHeader());
+  Changed |= DeleteDeadPHIs(L->getHeader(), TLI);
   // Check a post-condition.
   assert(L->isLCSSAForm(*DT) &&
          "Indvars did not leave the loop in lcssa form!");
diff --git a/lib/Transforms/Scalar/JumpThreading.cpp b/lib/Transforms/Scalar/JumpThreading.cpp
index dd42c59..e7ffa09 100644
--- a/lib/Transforms/Scalar/JumpThreading.cpp
+++ b/lib/Transforms/Scalar/JumpThreading.cpp
@@ -23,7 +23,7 @@
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/Transforms/Utils/Local.h"
 #include "llvm/Transforms/Utils/SSAUpdater.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetLibraryInfo.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
@@ -75,7 +75,7 @@ namespace {
   /// revectored to the false side of the second if.
   ///
   class JumpThreading : public FunctionPass {
-    TargetData *TD;
+    DataLayout *TD;
     TargetLibraryInfo *TLI;
     LazyValueInfo *LVI;
 #ifdef NDEBUG
@@ -147,7 +147,7 @@ FunctionPass *llvm::createJumpThreadingPass() { return new JumpThreading(); }
 ///
 bool JumpThreading::runOnFunction(Function &F) {
   DEBUG(dbgs() << "Jump threading on function '" << F.getName() << "'\n");
-  TD = getAnalysisIfAvailable<TargetData>();
+  TD = getAnalysisIfAvailable<DataLayout>();
   TLI = &getAnalysis<TargetLibraryInfo>();
   LVI = &getAnalysis<LazyValueInfo>();
 
@@ -1455,7 +1455,7 @@ bool JumpThreading::ThreadEdge(BasicBlock *BB,
   // At this point, the IR is fully up to date and consistent.  Do a quick scan
   // over the new instructions and zap any that are constants or dead.  This
   // frequently happens because of phi translation.
-  SimplifyInstructionsInBlock(NewBB, TD);
+  SimplifyInstructionsInBlock(NewBB, TD, TLI);
 
   // Threaded an edge!
   ++NumThreads;
diff --git a/lib/Transforms/Scalar/LICM.cpp b/lib/Transforms/Scalar/LICM.cpp
index 0192e92..4818437 100644
--- a/lib/Transforms/Scalar/LICM.cpp
+++ b/lib/Transforms/Scalar/LICM.cpp
@@ -46,7 +46,7 @@
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/Transforms/Utils/Local.h"
 #include "llvm/Transforms/Utils/SSAUpdater.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetLibraryInfo.h"
 #include "llvm/Support/CFG.h"
 #include "llvm/Support/CommandLine.h"
@@ -100,7 +100,7 @@ namespace {
     LoopInfo      *LI;       // Current LoopInfo
     DominatorTree *DT;       // Dominator Tree for the current Loop.
 
-    TargetData *TD;          // TargetData for constant folding.
+    DataLayout *TD;          // DataLayout for constant folding.
     TargetLibraryInfo *TLI;  // TargetLibraryInfo for constant folding.
 
     // State that is updated as we process loops.
@@ -108,6 +108,9 @@ namespace {
     BasicBlock *Preheader;   // The preheader block of the current loop...
     Loop *CurLoop;           // The current loop we are working on...
     AliasSetTracker *CurAST; // AliasSet information for the current loop...
+    bool MayThrow;           // The current loop contains an instruction which
+                             // may throw, thus preventing code motion of
+                             // instructions with side effects.
     DenseMap<Loop*, AliasSetTracker*> LoopToAliasSetMap;
 
     /// cloneBasicBlockAnalysis - Simple Analysis hook. Clone alias set info.
@@ -204,7 +207,7 @@ bool LICM::runOnLoop(Loop *L, LPPassManager &LPM) {
   AA = &getAnalysis<AliasAnalysis>();
   DT = &getAnalysis<DominatorTree>();
 
-  TD = getAnalysisIfAvailable<TargetData>();
+  TD = getAnalysisIfAvailable<DataLayout>();
   TLI = &getAnalysis<TargetLibraryInfo>();
 
   CurAST = new AliasSetTracker(*AA);
@@ -240,6 +243,15 @@ bool LICM::runOnLoop(Loop *L, LPPassManager &LPM) {
       CurAST->add(*BB);                 // Incorporate the specified basic block
   }
 
+  MayThrow = false;
+  // TODO: We've already searched for instructions which may throw in subloops.
+  // We may want to reuse this information.
+  for (Loop::block_iterator BB = L->block_begin(), BBE = L->block_end();
+       (BB != BBE) && !MayThrow ; ++BB)
+    for (BasicBlock::iterator I = (*BB)->begin(), E = (*BB)->end();
+         (I != E) && !MayThrow; ++I)
+      MayThrow |= I->mayThrow();
+
   // We want to visit all of the instructions in this loop... that are not parts
   // of our subloops (they have already had their invariants hoisted out of
   // their loop, into this loop, so there is no need to process the BODIES of
@@ -307,7 +319,7 @@ void LICM::SinkRegion(DomTreeNode *N) {
 
     // If the instruction is dead, we would try to sink it because it isn't used
     // in the loop, instead, just delete it.
-    if (isInstructionTriviallyDead(&I)) {
+    if (isInstructionTriviallyDead(&I, TLI)) {
       DEBUG(dbgs() << "LICM deleting dead inst: " << I << '\n');
       ++II;
       CurAST->deleteValue(&I);
@@ -418,17 +430,22 @@ bool LICM::canSinkOrHoistInst(Instruction &I) {
       if (!FoundMod) return true;
     }
 
-    // FIXME: This should use mod/ref information to see if we can hoist or sink
-    // the call.
+    // FIXME: This should use mod/ref information to see if we can hoist or
+    // sink the call.
 
     return false;
   }
 
-  // Otherwise these instructions are hoistable/sinkable
-  return isa<BinaryOperator>(I) || isa<CastInst>(I) ||
-         isa<SelectInst>(I) || isa<GetElementPtrInst>(I) || isa<CmpInst>(I) ||
-         isa<InsertElementInst>(I) || isa<ExtractElementInst>(I) ||
-         isa<ShuffleVectorInst>(I);
+  // Only these instructions are hoistable/sinkable.
+  bool HoistableKind = (isa<BinaryOperator>(I) || isa<CastInst>(I) ||
+                            isa<SelectInst>(I) || isa<GetElementPtrInst>(I) ||
+                            isa<CmpInst>(I)    || isa<InsertElementInst>(I) ||
+                            isa<ExtractElementInst>(I) ||
+                            isa<ShuffleVectorInst>(I));
+  if (!HoistableKind)
+      return false;
+
+  return isSafeToExecuteUnconditionally(I);
 }
 
 /// isNotUsedInLoop - Return true if the only users of this instruction are
@@ -604,6 +621,12 @@ bool LICM::isSafeToExecuteUnconditionally(Instruction &Inst) {
 }
 
 bool LICM::isGuaranteedToExecute(Instruction &Inst) {
+
+  // Somewhere in this loop there is an instruction which may throw and make us
+  // exit the loop.
+  if (MayThrow)
+    return false;
+
   // Otherwise we have to check to make sure that the instruction dominates all
   // of the exit blocks.  If it doesn't, then there is a path out of the loop
   // which does not execute this instruction, so we can't hoist it.
diff --git a/lib/Transforms/Scalar/LoopIdiomRecognize.cpp b/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
index ac1082c..a44e798 100644
--- a/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
+++ b/lib/Transforms/Scalar/LoopIdiomRecognize.cpp
@@ -54,7 +54,7 @@
 #include "llvm/Analysis/ValueTracking.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetLibraryInfo.h"
 #include "llvm/Transforms/Utils/Local.h"
 using namespace llvm;
@@ -65,7 +65,7 @@ STATISTIC(NumMemCpy, "Number of memcpy's formed from loop load+stores");
 namespace {
   class LoopIdiomRecognize : public LoopPass {
     Loop *CurLoop;
-    const TargetData *TD;
+    const DataLayout *TD;
     DominatorTree *DT;
     ScalarEvolution *SE;
     TargetLibraryInfo *TLI;
@@ -132,7 +132,8 @@ Pass *llvm::createLoopIdiomPass() { return new LoopIdiomRecognize(); }
 /// and zero out all the operands of this instruction.  If any of them become
 /// dead, delete them and the computation tree that feeds them.
 ///
-static void deleteDeadInstruction(Instruction *I, ScalarEvolution &SE) {
+static void deleteDeadInstruction(Instruction *I, ScalarEvolution &SE,
+                                  const TargetLibraryInfo *TLI) {
   SmallVector<Instruction*, 32> NowDeadInsts;
 
   NowDeadInsts.push_back(I);
@@ -153,7 +154,7 @@ static void deleteDeadInstruction(Instruction *I, ScalarEvolution &SE) {
       if (!Op->use_empty()) continue;
 
       if (Instruction *OpI = dyn_cast<Instruction>(Op))
-        if (isInstructionTriviallyDead(OpI))
+        if (isInstructionTriviallyDead(OpI, TLI))
           NowDeadInsts.push_back(OpI);
     }
 
@@ -164,15 +165,21 @@ static void deleteDeadInstruction(Instruction *I, ScalarEvolution &SE) {
 
 /// deleteIfDeadInstruction - If the specified value is a dead instruction,
 /// delete it and any recursively used instructions.
-static void deleteIfDeadInstruction(Value *V, ScalarEvolution &SE) {
+static void deleteIfDeadInstruction(Value *V, ScalarEvolution &SE,
+                                    const TargetLibraryInfo *TLI) {
   if (Instruction *I = dyn_cast<Instruction>(V))
-    if (isInstructionTriviallyDead(I))
-      deleteDeadInstruction(I, SE);
+    if (isInstructionTriviallyDead(I, TLI))
+      deleteDeadInstruction(I, SE, TLI);
 }
 
 bool LoopIdiomRecognize::runOnLoop(Loop *L, LPPassManager &LPM) {
   CurLoop = L;
 
+  // If the loop could not be converted to canonical form, it must have an
+  // indirectbr in it, just give up.
+  if (!L->getLoopPreheader())
+    return false;
+
   // Disable loop idiom recognition if the function's name is a common idiom.
   StringRef Name = L->getHeader()->getParent()->getName();
   if (Name == "memset" || Name == "memcpy")
@@ -192,7 +199,7 @@ bool LoopIdiomRecognize::runOnLoop(Loop *L, LPPassManager &LPM) {
       return false;
 
   // We require target data for now.
-  TD = getAnalysisIfAvailable<TargetData>();
+  TD = getAnalysisIfAvailable<DataLayout>();
   if (TD == 0) return false;
 
   DT = &getAnalysis<DominatorTree>();
@@ -401,7 +408,7 @@ static bool mayLoopAccessLocation(Value *Ptr,AliasAnalysis::ModRefResult Access,
 ///
 /// Note that we don't ever attempt to use memset_pattern8 or 4, because these
 /// just replicate their input array and then pass on to memset_pattern16.
-static Constant *getMemSetPatternValue(Value *V, const TargetData &TD) {
+static Constant *getMemSetPatternValue(Value *V, const DataLayout &TD) {
   // If the value isn't a constant, we can't promote it to being in a constant
   // array.  We could theoretically do a store to an alloca or something, but
   // that doesn't seem worthwhile.
@@ -490,7 +497,7 @@ processLoopStridedStore(Value *DestPtr, unsigned StoreSize,
                             StoreSize, getAnalysis<AliasAnalysis>(), TheStore)){
     Expander.clear();
     // If we generated new code for the base pointer, clean up.
-    deleteIfDeadInstruction(BasePtr, *SE);
+    deleteIfDeadInstruction(BasePtr, *SE, TLI);
     return false;
   }
 
@@ -538,7 +545,7 @@ processLoopStridedStore(Value *DestPtr, unsigned StoreSize,
 
   // Okay, the memset has been formed.  Zap the original store and anything that
   // feeds into it.
-  deleteDeadInstruction(TheStore, *SE);
+  deleteDeadInstruction(TheStore, *SE, TLI);
   ++NumMemSet;
   return true;
 }
@@ -579,7 +586,7 @@ processLoopStoreOfLoopLoad(StoreInst *SI, unsigned StoreSize,
                             getAnalysis<AliasAnalysis>(), SI)) {
     Expander.clear();
     // If we generated new code for the base pointer, clean up.
-    deleteIfDeadInstruction(StoreBasePtr, *SE);
+    deleteIfDeadInstruction(StoreBasePtr, *SE, TLI);
     return false;
   }
 
@@ -594,8 +601,8 @@ processLoopStoreOfLoopLoad(StoreInst *SI, unsigned StoreSize,
                             StoreSize, getAnalysis<AliasAnalysis>(), SI)) {
     Expander.clear();
     // If we generated new code for the base pointer, clean up.
-    deleteIfDeadInstruction(LoadBasePtr, *SE);
-    deleteIfDeadInstruction(StoreBasePtr, *SE);
+    deleteIfDeadInstruction(LoadBasePtr, *SE, TLI);
+    deleteIfDeadInstruction(StoreBasePtr, *SE, TLI);
     return false;
   }
 
@@ -628,7 +635,7 @@ processLoopStoreOfLoopLoad(StoreInst *SI, unsigned StoreSize,
 
   // Okay, the memset has been formed.  Zap the original store and anything that
   // feeds into it.
-  deleteDeadInstruction(SI, *SE);
+  deleteDeadInstruction(SI, *SE, TLI);
   ++NumMemCpy;
   return true;
 }
diff --git a/lib/Transforms/Scalar/LoopInstSimplify.cpp b/lib/Transforms/Scalar/LoopInstSimplify.cpp
index 982400c..558f62e 100644
--- a/lib/Transforms/Scalar/LoopInstSimplify.cpp
+++ b/lib/Transforms/Scalar/LoopInstSimplify.cpp
@@ -18,7 +18,7 @@
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/LoopPass.h"
 #include "llvm/Support/Debug.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetLibraryInfo.h"
 #include "llvm/Transforms/Scalar.h"
 #include "llvm/Transforms/Utils/Local.h"
@@ -66,7 +66,7 @@ Pass *llvm::createLoopInstSimplifyPass() {
 bool LoopInstSimplify::runOnLoop(Loop *L, LPPassManager &LPM) {
   DominatorTree *DT = getAnalysisIfAvailable<DominatorTree>();
   LoopInfo *LI = &getAnalysis<LoopInfo>();
-  const TargetData *TD = getAnalysisIfAvailable<TargetData>();
+  const DataLayout *TD = getAnalysisIfAvailable<DataLayout>();
   const TargetLibraryInfo *TLI = &getAnalysis<TargetLibraryInfo>();
 
   SmallVector<BasicBlock*, 8> ExitBlocks;
@@ -120,7 +120,7 @@ bool LoopInstSimplify::runOnLoop(Loop *L, LPPassManager &LPM) {
             ++NumSimplified;
           }
         }
-        LocalChanged |= RecursivelyDeleteTriviallyDeadInstructions(I);
+        LocalChanged |= RecursivelyDeleteTriviallyDeadInstructions(I, TLI);
 
         if (IsSubloopHeader && !isa<PHINode>(I))
           break;
diff --git a/lib/Transforms/Scalar/LoopRotation.cpp b/lib/Transforms/Scalar/LoopRotation.cpp
index 7eeb152..abe07aa 100644
--- a/lib/Transforms/Scalar/LoopRotation.cpp
+++ b/lib/Transforms/Scalar/LoopRotation.cpp
@@ -24,6 +24,7 @@
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/Transforms/Utils/SSAUpdater.h"
 #include "llvm/Transforms/Utils/ValueMapper.h"
+#include "llvm/Support/CFG.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/ADT/Statistic.h"
 using namespace llvm;
@@ -256,6 +257,7 @@ bool LoopRotate::rotateLoop(Loop *L) {
     return false;
 
   BasicBlock *OrigHeader = L->getHeader();
+  BasicBlock *OrigLatch = L->getLoopLatch();
 
   BranchInst *BI = dyn_cast<BranchInst>(OrigHeader->getTerminator());
   if (BI == 0 || BI->isUnconditional())
@@ -267,13 +269,9 @@ bool LoopRotate::rotateLoop(Loop *L) {
   if (!L->isLoopExiting(OrigHeader))
     return false;
 
-  // Updating PHInodes in loops with multiple exits adds complexity.
-  // Keep it simple, and restrict loop rotation to loops with one exit only.
-  // In future, lift this restriction and support for multiple exits if
-  // required.
-  SmallVector<BasicBlock*, 8> ExitBlocks;
-  L->getExitBlocks(ExitBlocks);
-  if (ExitBlocks.size() > 1)
+  // If the loop latch already contains a branch that leaves the loop then the
+  // loop is already rotated.
+  if (OrigLatch == 0 || L->isLoopExiting(OrigLatch))
     return false;
 
   // Check size of original header and reject loop if it is very big.
@@ -286,11 +284,10 @@ bool LoopRotate::rotateLoop(Loop *L) {
 
   // Now, this loop is suitable for rotation.
   BasicBlock *OrigPreheader = L->getLoopPreheader();
-  BasicBlock *OrigLatch = L->getLoopLatch();
 
   // If the loop could not be converted to canonical form, it must have an
   // indirectbr in it, just give up.
-  if (OrigPreheader == 0 || OrigLatch == 0)
+  if (OrigPreheader == 0)
     return false;
 
   // Anything ScalarEvolution may know about this loop or the PHI nodes
@@ -298,6 +295,8 @@ bool LoopRotate::rotateLoop(Loop *L) {
   if (ScalarEvolution *SE = getAnalysisIfAvailable<ScalarEvolution>())
     SE->forgetLoop(L);
 
+  DEBUG(dbgs() << "LoopRotation: rotating "; L->dump());
+
   // Find new Loop header. NewHeader is a Header's one and only successor
   // that is inside loop.  Header's other successor is outside the
   // loop.  Otherwise loop is not suitable for rotation.
@@ -408,10 +407,19 @@ bool LoopRotate::rotateLoop(Loop *L) {
     // Update DominatorTree to reflect the CFG change we just made.  Then split
     // edges as necessary to preserve LoopSimplify form.
     if (DominatorTree *DT = getAnalysisIfAvailable<DominatorTree>()) {
-      // Since OrigPreheader now has the conditional branch to Exit block, it is
-      // the dominator of Exit.
-      DT->changeImmediateDominator(Exit, OrigPreheader);
-      DT->changeImmediateDominator(NewHeader, OrigPreheader);
+      // Everything that was dominated by the old loop header is now dominated
+      // by the original loop preheader. Conceptually the header was merged
+      // into the preheader, even though we reuse the actual block as a new
+      // loop latch.
+      DomTreeNode *OrigHeaderNode = DT->getNode(OrigHeader);
+      SmallVector<DomTreeNode *, 8> HeaderChildren(OrigHeaderNode->begin(),
+                                                   OrigHeaderNode->end());
+      DomTreeNode *OrigPreheaderNode = DT->getNode(OrigPreheader);
+      for (unsigned I = 0, E = HeaderChildren.size(); I != E; ++I)
+        DT->changeImmediateDominator(HeaderChildren[I], OrigPreheaderNode);
+
+      assert(DT->getNode(Exit)->getIDom() == OrigPreheaderNode);
+      assert(DT->getNode(NewHeader)->getIDom() == OrigPreheaderNode);
 
       // Update OrigHeader to be dominated by the new header block.
       DT->changeImmediateDominator(OrigHeader, OrigLatch);
@@ -440,6 +448,35 @@ bool LoopRotate::rotateLoop(Loop *L) {
       // Update OrigHeader to be dominated by the new header block.
       DT->changeImmediateDominator(NewHeader, OrigPreheader);
       DT->changeImmediateDominator(OrigHeader, OrigLatch);
+
+      // Brute force incremental dominator tree update. Call
+      // findNearestCommonDominator on all CFG predecessors of each child of the
+      // original header.
+      DomTreeNode *OrigHeaderNode = DT->getNode(OrigHeader);
+      SmallVector<DomTreeNode *, 8> HeaderChildren(OrigHeaderNode->begin(),
+                                                   OrigHeaderNode->end());
+      bool Changed;
+      do {
+        Changed = false;
+        for (unsigned I = 0, E = HeaderChildren.size(); I != E; ++I) {
+          DomTreeNode *Node = HeaderChildren[I];
+          BasicBlock *BB = Node->getBlock();
+
+          pred_iterator PI = pred_begin(BB);
+          BasicBlock *NearestDom = *PI;
+          for (pred_iterator PE = pred_end(BB); PI != PE; ++PI)
+            NearestDom = DT->findNearestCommonDominator(NearestDom, *PI);
+
+          // Remember if this changes the DomTree.
+          if (Node->getIDom()->getBlock() != NearestDom) {
+            DT->changeImmediateDominator(BB, NearestDom);
+            Changed = true;
+          }
+        }
+
+      // If the dominator changed, this may have an effect on other
+      // predecessors, continue until we reach a fixpoint.
+      } while (Changed);
     }
   }
 
@@ -452,6 +489,8 @@ bool LoopRotate::rotateLoop(Loop *L) {
   // emitted code isn't too gross in this common case.
   MergeBlockIntoPredecessor(OrigHeader, this);
 
+  DEBUG(dbgs() << "LoopRotation: into "; L->dump());
+
   ++NumRotated;
   return true;
 }
diff --git a/lib/Transforms/Scalar/LoopStrengthReduce.cpp b/lib/Transforms/Scalar/LoopStrengthReduce.cpp
index b14a713..958348d 100644
--- a/lib/Transforms/Scalar/LoopStrengthReduce.cpp
+++ b/lib/Transforms/Scalar/LoopStrengthReduce.cpp
@@ -54,7 +54,7 @@
 //===----------------------------------------------------------------------===//
 
 #define DEBUG_TYPE "loop-reduce"
-#include "llvm/Transforms/Scalar.h"
+#include "llvm/AddressingMode.h"
 #include "llvm/Constants.h"
 #include "llvm/Instructions.h"
 #include "llvm/IntrinsicInst.h"
@@ -64,6 +64,7 @@
 #include "llvm/Analysis/LoopPass.h"
 #include "llvm/Analysis/ScalarEvolutionExpander.h"
 #include "llvm/Assembly/Writer.h"
+#include "llvm/Transforms/Scalar.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include "llvm/Transforms/Utils/Local.h"
 #include "llvm/ADT/SmallBitVector.h"
@@ -121,9 +122,11 @@ void RegSortData::print(raw_ostream &OS) const {
   OS << "[NumUses=" << UsedByIndices.count() << ']';
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void RegSortData::dump() const {
   print(errs()); errs() << '\n';
 }
+#endif
 
 namespace {
 
@@ -223,7 +226,7 @@ namespace {
 struct Formula {
   /// AM - This is used to represent complex addressing, as well as other kinds
   /// of interesting uses.
-  TargetLowering::AddrMode AM;
+  AddrMode AM;
 
   /// BaseRegs - The list of "base" registers for this use. When this is
   /// non-empty, AM.HasBaseReg should be set to true.
@@ -414,9 +417,11 @@ void Formula::print(raw_ostream &OS) const {
   }
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void Formula::dump() const {
   print(errs()); errs() << '\n';
 }
+#endif
 
 /// isAddRecSExtable - Return true if the given addrec can be sign-extended
 /// without changing its value.
@@ -738,7 +743,8 @@ DeleteTriviallyDeadInstructions(SmallVectorImpl<WeakVH> &DeadInsts) {
   bool Changed = false;
 
   while (!DeadInsts.empty()) {
-    Instruction *I = dyn_cast_or_null<Instruction>(&*DeadInsts.pop_back_val());
+    Value *V = DeadInsts.pop_back_val();
+    Instruction *I = dyn_cast_or_null<Instruction>(V);
 
     if (I == 0 || !isInstructionTriviallyDead(I))
       continue;
@@ -973,9 +979,11 @@ void Cost::print(raw_ostream &OS) const {
     OS << ", plus " << SetupCost << " setup cost";
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void Cost::dump() const {
   print(errs()); errs() << '\n';
 }
+#endif
 
 namespace {
 
@@ -1059,9 +1067,11 @@ void LSRFixup::print(raw_ostream &OS) const {
     OS << ", Offset=" << Offset;
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void LSRFixup::dump() const {
   print(errs()); errs() << '\n';
 }
+#endif
 
 namespace {
 
@@ -1251,14 +1261,16 @@ void LSRUse::print(raw_ostream &OS) const {
     OS << ", widest fixup type: " << *WidestFixupType;
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void LSRUse::dump() const {
   print(errs()); errs() << '\n';
 }
+#endif
 
 /// isLegalUse - Test whether the use described by AM is "legal", meaning it can
 /// be completely folded into the user instruction at isel time. This includes
 /// address-mode folding and special icmp tricks.
-static bool isLegalUse(const TargetLowering::AddrMode &AM,
+static bool isLegalUse(const AddrMode &AM,
                        LSRUse::KindType Kind, Type *AccessTy,
                        const TargetLowering *TLI) {
   switch (Kind) {
@@ -1315,7 +1327,7 @@ static bool isLegalUse(const TargetLowering::AddrMode &AM,
   llvm_unreachable("Invalid LSRUse Kind!");
 }
 
-static bool isLegalUse(TargetLowering::AddrMode AM,
+static bool isLegalUse(AddrMode AM,
                        int64_t MinOffset, int64_t MaxOffset,
                        LSRUse::KindType Kind, Type *AccessTy,
                        const TargetLowering *TLI) {
@@ -1346,7 +1358,7 @@ static bool isAlwaysFoldable(int64_t BaseOffs,
 
   // Conservatively, create an address with an immediate and a
   // base and a scale.
-  TargetLowering::AddrMode AM;
+  AddrMode AM;
   AM.BaseOffs = BaseOffs;
   AM.BaseGV = BaseGV;
   AM.HasBaseReg = HasBaseReg;
@@ -1384,7 +1396,7 @@ static bool isAlwaysFoldable(const SCEV *S,
 
   // Conservatively, create an address with an immediate and a
   // base and a scale.
-  TargetLowering::AddrMode AM;
+  AddrMode AM;
   AM.BaseOffs = BaseOffs;
   AM.BaseGV = BaseGV;
   AM.HasBaseReg = HasBaseReg;
@@ -2009,7 +2021,7 @@ LSRInstance::OptimizeLoopTermCond() {
               goto decline_post_inc;
             // Check for possible scaled-address reuse.
             Type *AccessTy = getAccessType(UI->getUser());
-            TargetLowering::AddrMode AM;
+            AddrMode AM;
             AM.Scale = C->getSExtValue();
             if (TLI->isLegalAddressingMode(AM, AccessTy))
               goto decline_post_inc;
@@ -3435,9 +3447,11 @@ void WorkItem::print(raw_ostream &OS) const {
      << " , add offset " << Imm;
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void WorkItem::dump() const {
   print(errs()); errs() << '\n';
 }
+#endif
 
 /// GenerateCrossUseConstantOffsets - Look for registers which are a constant
 /// distance apart and try to form reuse opportunities between them.
@@ -4451,17 +4465,21 @@ void LSRInstance::RewriteForPHI(PHINode *PN,
             SplitLandingPadPredecessors(Parent, BB, "", "", P, NewBBs);
             NewBB = NewBBs[0];
           }
-
-          // If PN is outside of the loop and BB is in the loop, we want to
-          // move the block to be immediately before the PHI block, not
-          // immediately after BB.
-          if (L->contains(BB) && !L->contains(PN))
-            NewBB->moveBefore(PN->getParent());
-
-          // Splitting the edge can reduce the number of PHI entries we have.
-          e = PN->getNumIncomingValues();
-          BB = NewBB;
-          i = PN->getBasicBlockIndex(BB);
+          // If NewBB==NULL, then SplitCriticalEdge refused to split because all
+          // phi predecessors are identical. The simple thing to do is skip
+          // splitting in this case rather than complicate the API.
+          if (NewBB) {
+            // If PN is outside of the loop and BB is in the loop, we want to
+            // move the block to be immediately before the PHI block, not
+            // immediately after BB.
+            if (L->contains(BB) && !L->contains(PN))
+              NewBB->moveBefore(PN->getParent());
+
+            // Splitting the edge can reduce the number of PHI entries we have.
+            e = PN->getNumIncomingValues();
+            BB = NewBB;
+            i = PN->getBasicBlockIndex(BB);
+          }
         }
       }
 
@@ -4730,9 +4748,11 @@ void LSRInstance::print(raw_ostream &OS) const {
   print_uses(OS);
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void LSRInstance::dump() const {
   print(errs()); errs() << '\n';
 }
+#endif
 
 namespace {
 
diff --git a/lib/Transforms/Scalar/LoopUnrollPass.cpp b/lib/Transforms/Scalar/LoopUnrollPass.cpp
index 09a186f..0d781ac 100644
--- a/lib/Transforms/Scalar/LoopUnrollPass.cpp
+++ b/lib/Transforms/Scalar/LoopUnrollPass.cpp
@@ -22,7 +22,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Utils/UnrollLoop.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include <climits>
 
 using namespace llvm;
@@ -113,7 +113,7 @@ Pass *llvm::createLoopUnrollPass(int Threshold, int Count, int AllowPartial) {
 
 /// ApproximateLoopSize - Approximate the size of the loop.
 static unsigned ApproximateLoopSize(const Loop *L, unsigned &NumCalls,
-                                    const TargetData *TD) {
+                                    const DataLayout *TD) {
   CodeMetrics Metrics;
   for (Loop::block_iterator I = L->block_begin(), E = L->block_end();
        I != E; ++I)
@@ -145,7 +145,8 @@ bool LoopUnroll::runOnLoop(Loop *L, LPPassManager &LPM) {
   // not user specified.
   unsigned Threshold = CurrentThreshold;
   if (!UserThreshold &&
-      Header->getParent()->hasFnAttr(Attribute::OptimizeForSize))
+      Header->getParent()->getFnAttributes().
+        hasAttribute(Attributes::OptimizeForSize))
     Threshold = OptSizeUnrollThreshold;
 
   // Find trip count and trip multiple if count is not available
@@ -178,7 +179,7 @@ bool LoopUnroll::runOnLoop(Loop *L, LPPassManager &LPM) {
 
   // Enforce the threshold.
   if (Threshold != NoThreshold) {
-    const TargetData *TD = getAnalysisIfAvailable<TargetData>();
+    const DataLayout *TD = getAnalysisIfAvailable<DataLayout>();
     unsigned NumInlineCandidates;
     unsigned LoopSize = ApproximateLoopSize(L, NumInlineCandidates, TD);
     DEBUG(dbgs() << "  Loop Size = " << LoopSize << "\n");
diff --git a/lib/Transforms/Scalar/LoopUnswitch.cpp b/lib/Transforms/Scalar/LoopUnswitch.cpp
index 58f7739..047b43e 100644
--- a/lib/Transforms/Scalar/LoopUnswitch.cpp
+++ b/lib/Transforms/Scalar/LoopUnswitch.cpp
@@ -638,7 +638,8 @@ bool LoopUnswitch::UnswitchIfProfitable(Value *LoopCond, Constant *Val) {
   // Check to see if it would be profitable to unswitch current loop.
 
   // Do not do non-trivial unswitch while optimizing for size.
-  if (OptimizeForSize || F->hasFnAttr(Attribute::OptimizeForSize))
+  if (OptimizeForSize ||
+      F->getFnAttributes().hasAttribute(Attributes::OptimizeForSize))
     return false;
 
   UnswitchNontrivialCondition(LoopCond, Val, currentLoop);
@@ -906,13 +907,9 @@ void LoopUnswitch::UnswitchNontrivialCondition(Value *LIC, Constant *Val,
 /// specified.
 static void RemoveFromWorklist(Instruction *I,
                                std::vector<Instruction*> &Worklist) {
-  std::vector<Instruction*>::iterator WI = std::find(Worklist.begin(),
-                                                     Worklist.end(), I);
-  while (WI != Worklist.end()) {
-    unsigned Offset = WI-Worklist.begin();
-    Worklist.erase(WI);
-    WI = std::find(Worklist.begin()+Offset, Worklist.end(), I);
-  }
+
+  Worklist.erase(std::remove(Worklist.begin(), Worklist.end(), I),
+                 Worklist.end());
 }
 
 /// ReplaceUsesOfWith - When we find that I really equals V, remove I from the
diff --git a/lib/Transforms/Scalar/MemCpyOptimizer.cpp b/lib/Transforms/Scalar/MemCpyOptimizer.cpp
index 2a5ee33..517657cf 100644
--- a/lib/Transforms/Scalar/MemCpyOptimizer.cpp
+++ b/lib/Transforms/Scalar/MemCpyOptimizer.cpp
@@ -27,7 +27,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/GetElementPtrTypeIterator.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetLibraryInfo.h"
 #include "llvm/Transforms/Utils/Local.h"
 #include <list>
@@ -38,8 +38,8 @@ STATISTIC(NumMemSetInfer, "Number of memsets inferred");
 STATISTIC(NumMoveToCpy,   "Number of memmoves converted to memcpy");
 STATISTIC(NumCpyToSet,    "Number of memcpys converted to memset");
 
-static int64_t GetOffsetFromIndex(const GetElementPtrInst *GEP, unsigned Idx,
-                                  bool &VariableIdxFound, const TargetData &TD){
+static int64_t GetOffsetFromIndex(const GEPOperator *GEP, unsigned Idx,
+                                  bool &VariableIdxFound, const DataLayout &TD){
   // Skip over the first indices.
   gep_type_iterator GTI = gep_type_begin(GEP);
   for (unsigned i = 1; i != Idx; ++i, ++GTI)
@@ -72,11 +72,11 @@ static int64_t GetOffsetFromIndex(const GetElementPtrInst *GEP, unsigned Idx,
 /// constant offset, and return that constant offset.  For example, Ptr1 might
 /// be &A[42], and Ptr2 might be &A[40].  In this case offset would be -8.
 static bool IsPointerOffset(Value *Ptr1, Value *Ptr2, int64_t &Offset,
-                            const TargetData &TD) {
+                            const DataLayout &TD) {
   Ptr1 = Ptr1->stripPointerCasts();
   Ptr2 = Ptr2->stripPointerCasts();
-  GetElementPtrInst *GEP1 = dyn_cast<GetElementPtrInst>(Ptr1);
-  GetElementPtrInst *GEP2 = dyn_cast<GetElementPtrInst>(Ptr2);
+  GEPOperator *GEP1 = dyn_cast<GEPOperator>(Ptr1);
+  GEPOperator *GEP2 = dyn_cast<GEPOperator>(Ptr2);
 
   bool VariableIdxFound = false;
 
@@ -141,12 +141,12 @@ struct MemsetRange {
   /// TheStores - The actual stores that make up this range.
   SmallVector<Instruction*, 16> TheStores;
 
-  bool isProfitableToUseMemset(const TargetData &TD) const;
+  bool isProfitableToUseMemset(const DataLayout &TD) const;
 
 };
 } // end anon namespace
 
-bool MemsetRange::isProfitableToUseMemset(const TargetData &TD) const {
+bool MemsetRange::isProfitableToUseMemset(const DataLayout &TD) const {
   // If we found more than 4 stores to merge or 16 bytes, use memset.
   if (TheStores.size() >= 4 || End-Start >= 16) return true;
 
@@ -192,9 +192,9 @@ class MemsetRanges {
   /// because each element is relatively large and expensive to copy.
   std::list<MemsetRange> Ranges;
   typedef std::list<MemsetRange>::iterator range_iterator;
-  const TargetData &TD;
+  const DataLayout &TD;
 public:
-  MemsetRanges(const TargetData &td) : TD(td) {}
+  MemsetRanges(const DataLayout &td) : TD(td) {}
 
   typedef std::list<MemsetRange>::const_iterator const_iterator;
   const_iterator begin() const { return Ranges.begin(); }
@@ -302,7 +302,7 @@ namespace {
   class MemCpyOpt : public FunctionPass {
     MemoryDependenceAnalysis *MD;
     TargetLibraryInfo *TLI;
-    const TargetData *TD;
+    const DataLayout *TD;
   public:
     static char ID; // Pass identification, replacement for typeid
     MemCpyOpt() : FunctionPass(ID) {
@@ -332,7 +332,7 @@ namespace {
     bool processMemCpy(MemCpyInst *M);
     bool processMemMove(MemMoveInst *M);
     bool performCallSlotOptzn(Instruction *cpy, Value *cpyDst, Value *cpySrc,
-                              uint64_t cpyLen, CallInst *C);
+                              uint64_t cpyLen, unsigned cpyAlign, CallInst *C);
     bool processMemCpyMemCpyDependence(MemCpyInst *M, MemCpyInst *MDep,
                                        uint64_t MSize);
     bool processByValArgument(CallSite CS, unsigned ArgNo);
@@ -509,10 +509,18 @@ bool MemCpyOpt::processStore(StoreInst *SI, BasicBlock::iterator &BBI) {
       }
 
       if (C) {
+        unsigned storeAlign = SI->getAlignment();
+        if (!storeAlign)
+          storeAlign = TD->getABITypeAlignment(SI->getOperand(0)->getType());
+        unsigned loadAlign = LI->getAlignment();
+        if (!loadAlign)
+          loadAlign = TD->getABITypeAlignment(LI->getType());
+
         bool changed = performCallSlotOptzn(LI,
                         SI->getPointerOperand()->stripPointerCasts(),
                         LI->getPointerOperand()->stripPointerCasts(),
-                        TD->getTypeStoreSize(SI->getOperand(0)->getType()), C);
+                        TD->getTypeStoreSize(SI->getOperand(0)->getType()),
+                        std::min(storeAlign, loadAlign), C);
         if (changed) {
           MD->removeInstruction(SI);
           SI->eraseFromParent();
@@ -559,7 +567,8 @@ bool MemCpyOpt::processMemSet(MemSetInst *MSI, BasicBlock::iterator &BBI) {
 /// the call write its result directly into the destination of the memcpy.
 bool MemCpyOpt::performCallSlotOptzn(Instruction *cpy,
                                      Value *cpyDest, Value *cpySrc,
-                                     uint64_t cpyLen, CallInst *C) {
+                                     uint64_t cpyLen, unsigned cpyAlign,
+                                     CallInst *C) {
   // The general transformation to keep in mind is
   //
   //   call @func(..., src, ...)
@@ -625,6 +634,16 @@ bool MemCpyOpt::performCallSlotOptzn(Instruction *cpy,
     return false;
   }
 
+  // Check that dest points to memory that is at least as aligned as src.
+  unsigned srcAlign = srcAlloca->getAlignment();
+  if (!srcAlign)
+    srcAlign = TD->getABITypeAlignment(srcAlloca->getAllocatedType());
+  bool isDestSufficientlyAligned = srcAlign <= cpyAlign;
+  // If dest is not aligned enough and we can't increase its alignment then
+  // bail out.
+  if (!isDestSufficientlyAligned && !isa<AllocaInst>(cpyDest))
+    return false;
+
   // Check that src is not accessed except via the call and the memcpy.  This
   // guarantees that it holds only undefined values when passed in (so the final
   // memcpy can be dropped), that it is not read or written between the call and
@@ -673,20 +692,26 @@ bool MemCpyOpt::performCallSlotOptzn(Instruction *cpy,
   bool changedArgument = false;
   for (unsigned i = 0; i < CS.arg_size(); ++i)
     if (CS.getArgument(i)->stripPointerCasts() == cpySrc) {
-      if (cpySrc->getType() != cpyDest->getType())
-        cpyDest = CastInst::CreatePointerCast(cpyDest, cpySrc->getType(),
-                                              cpyDest->getName(), C);
+      Value *Dest = cpySrc->getType() == cpyDest->getType() ?  cpyDest
+        : CastInst::CreatePointerCast(cpyDest, cpySrc->getType(),
+                                      cpyDest->getName(), C);
       changedArgument = true;
-      if (CS.getArgument(i)->getType() == cpyDest->getType())
-        CS.setArgument(i, cpyDest);
+      if (CS.getArgument(i)->getType() == Dest->getType())
+        CS.setArgument(i, Dest);
       else
-        CS.setArgument(i, CastInst::CreatePointerCast(cpyDest,
-                          CS.getArgument(i)->getType(), cpyDest->getName(), C));
+        CS.setArgument(i, CastInst::CreatePointerCast(Dest,
+                          CS.getArgument(i)->getType(), Dest->getName(), C));
     }
 
   if (!changedArgument)
     return false;
 
+  // If the destination wasn't sufficiently aligned then increase its alignment.
+  if (!isDestSufficientlyAligned) {
+    assert(isa<AllocaInst>(cpyDest) && "Can only increase alloca alignment!");
+    cast<AllocaInst>(cpyDest)->setAlignment(srcAlign);
+  }
+
   // Drop any cached information about the call, because we may have changed
   // its dependence information by changing its parameter.
   MD->removeInstruction(C);
@@ -813,7 +838,8 @@ bool MemCpyOpt::processMemCpy(MemCpyInst *M) {
   if (DepInfo.isClobber()) {
     if (CallInst *C = dyn_cast<CallInst>(DepInfo.getInst())) {
       if (performCallSlotOptzn(M, M->getDest(), M->getSource(),
-                               CopySize->getZExtValue(), C)) {
+                               CopySize->getZExtValue(), M->getAlignment(),
+                               C)) {
         MD->removeInstruction(M);
         M->eraseFromParent();
         return true;
@@ -974,7 +1000,7 @@ bool MemCpyOpt::iterateOnFunction(Function &F) {
 bool MemCpyOpt::runOnFunction(Function &F) {
   bool MadeChange = false;
   MD = &getAnalysis<MemoryDependenceAnalysis>();
-  TD = getAnalysisIfAvailable<TargetData>();
+  TD = getAnalysisIfAvailable<DataLayout>();
   TLI = &getAnalysis<TargetLibraryInfo>();
 
   // If we don't have at least memset and memcpy, there is little point of doing
diff --git a/lib/Transforms/Scalar/ObjCARC.cpp b/lib/Transforms/Scalar/ObjCARC.cpp
index 3222f20..dfdf505 100644
--- a/lib/Transforms/Scalar/ObjCARC.cpp
+++ b/lib/Transforms/Scalar/ObjCARC.cpp
@@ -29,6 +29,7 @@
 //===----------------------------------------------------------------------===//
 
 #define DEBUG_TYPE "objc-arc"
+#include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/ADT/DenseMap.h"
 using namespace llvm;
@@ -1120,9 +1121,8 @@ namespace {
     bool relatedSelect(const SelectInst *A, const Value *B);
     bool relatedPHI(const PHINode *A, const Value *B);
 
-    // Do not implement.
-    void operator=(const ProvenanceAnalysis &);
-    ProvenanceAnalysis(const ProvenanceAnalysis &);
+    void operator=(const ProvenanceAnalysis &) LLVM_DELETED_FUNCTION;
+    ProvenanceAnalysis(const ProvenanceAnalysis &) LLVM_DELETED_FUNCTION;
 
   public:
     ProvenanceAnalysis() {}
@@ -1236,16 +1236,19 @@ bool ProvenanceAnalysis::relatedCheck(const Value *A, const Value *B) {
 
   // An ObjC-Identified object can't alias a load if it is never locally stored.
   if (AIsIdentified) {
+    // Check for an obvious escape.
+    if (isa<LoadInst>(B))
+      return isStoredObjCPointer(A);
     if (BIsIdentified) {
-      // If both pointers have provenance, they can be directly compared.
-      if (A != B)
-        return false;
-    } else {
-      if (isa<LoadInst>(B))
-        return isStoredObjCPointer(A);
+      // Check for an obvious escape.
+      if (isa<LoadInst>(A))
+        return isStoredObjCPointer(B);
+      // Both pointers are identified and escapes aren't an evident problem.
+      return false;
     }
-  } else {
-    if (BIsIdentified && isa<LoadInst>(A))
+  } else if (BIsIdentified) {
+    // Check for an obvious escape.
+    if (isa<LoadInst>(A))
       return isStoredObjCPointer(B);
   }
 
@@ -1381,9 +1384,6 @@ namespace {
   /// PtrState - This class summarizes several per-pointer runtime properties
   /// which are propogated through the flow graph.
   class PtrState {
-    /// NestCount - The known minimum level of retain+release nesting.
-    unsigned NestCount;
-
     /// KnownPositiveRefCount - True if the reference count is known to
     /// be incremented.
     bool KnownPositiveRefCount;
@@ -1401,7 +1401,7 @@ namespace {
     /// TODO: Encapsulate this better.
     RRInfo RRI;
 
-    PtrState() : NestCount(0), KnownPositiveRefCount(false), Partial(false),
+    PtrState() : KnownPositiveRefCount(false), Partial(false),
                  Seq(S_None) {}
 
     void SetKnownPositiveRefCount() {
@@ -1416,18 +1416,6 @@ namespace {
       return KnownPositiveRefCount;
     }
 
-    void IncrementNestCount() {
-      if (NestCount != UINT_MAX) ++NestCount;
-    }
-
-    void DecrementNestCount() {
-      if (NestCount != 0) --NestCount;
-    }
-
-    bool IsKnownNested() const {
-      return NestCount > 0;
-    }
-
     void SetSeq(Sequence NewSeq) {
       Seq = NewSeq;
     }
@@ -1454,7 +1442,6 @@ void
 PtrState::Merge(const PtrState &Other, bool TopDown) {
   Seq = MergeSeqs(Seq, Other.Seq, TopDown);
   KnownPositiveRefCount = KnownPositiveRefCount && Other.KnownPositiveRefCount;
-  NestCount = std::min(NestCount, Other.NestCount);
 
   // We can't merge a plain objc_retain with an objc_retainBlock.
   if (RRI.IsRetainBlock != Other.RRI.IsRetainBlock)
@@ -1610,6 +1597,12 @@ void BBState::MergePred(const BBState &Other) {
   // loop backedge. Loop backedges are special.
   TopDownPathCount += Other.TopDownPathCount;
 
+  // Check for overflow. If we have overflow, fall back to conservative behavior.
+  if (TopDownPathCount < Other.TopDownPathCount) {
+    clearTopDownPointers();
+    return;
+  }
+
   // For each entry in the other set, if our set has an entry with the same key,
   // merge the entries. Otherwise, copy the entry and merge it with an empty
   // entry.
@@ -1635,6 +1628,12 @@ void BBState::MergeSucc(const BBState &Other) {
   // loop backedge. Loop backedges are special.
   BottomUpPathCount += Other.BottomUpPathCount;
 
+  // Check for overflow. If we have overflow, fall back to conservative behavior.
+  if (BottomUpPathCount < Other.BottomUpPathCount) {
+    clearBottomUpPointers();
+    return;
+  }
+
   // For each entry in the other set, if our set has an entry with the
   // same key, merge the entries. Otherwise, copy the entry and merge
   // it with an empty entry.
@@ -1789,7 +1788,9 @@ Constant *ObjCARCOpt::getRetainRVCallee(Module *M) {
     Type *I8X = PointerType::getUnqual(Type::getInt8Ty(C));
     Type *Params[] = { I8X };
     FunctionType *FTy = FunctionType::get(I8X, Params, /*isVarArg=*/false);
-    AttrListPtr Attributes = AttrListPtr().addAttr(~0u, Attribute::NoUnwind);
+    AttrListPtr Attributes =
+      AttrListPtr().addAttr(M->getContext(), AttrListPtr::FunctionIndex,
+                            Attributes::get(C, Attributes::NoUnwind));
     RetainRVCallee =
       M->getOrInsertFunction("objc_retainAutoreleasedReturnValue", FTy,
                              Attributes);
@@ -1803,7 +1804,9 @@ Constant *ObjCARCOpt::getAutoreleaseRVCallee(Module *M) {
     Type *I8X = PointerType::getUnqual(Type::getInt8Ty(C));
     Type *Params[] = { I8X };
     FunctionType *FTy = FunctionType::get(I8X, Params, /*isVarArg=*/false);
-    AttrListPtr Attributes = AttrListPtr().addAttr(~0u, Attribute::NoUnwind);
+    AttrListPtr Attributes =
+      AttrListPtr().addAttr(M->getContext(), AttrListPtr::FunctionIndex,
+                            Attributes::get(C, Attributes::NoUnwind));
     AutoreleaseRVCallee =
       M->getOrInsertFunction("objc_autoreleaseReturnValue", FTy,
                              Attributes);
@@ -1815,7 +1818,9 @@ Constant *ObjCARCOpt::getReleaseCallee(Module *M) {
   if (!ReleaseCallee) {
     LLVMContext &C = M->getContext();
     Type *Params[] = { PointerType::getUnqual(Type::getInt8Ty(C)) };
-    AttrListPtr Attributes = AttrListPtr().addAttr(~0u, Attribute::NoUnwind);
+    AttrListPtr Attributes =
+      AttrListPtr().addAttr(M->getContext(), AttrListPtr::FunctionIndex,
+                            Attributes::get(C, Attributes::NoUnwind));
     ReleaseCallee =
       M->getOrInsertFunction(
         "objc_release",
@@ -1829,7 +1834,9 @@ Constant *ObjCARCOpt::getRetainCallee(Module *M) {
   if (!RetainCallee) {
     LLVMContext &C = M->getContext();
     Type *Params[] = { PointerType::getUnqual(Type::getInt8Ty(C)) };
-    AttrListPtr Attributes = AttrListPtr().addAttr(~0u, Attribute::NoUnwind);
+    AttrListPtr Attributes =
+      AttrListPtr().addAttr(M->getContext(), AttrListPtr::FunctionIndex,
+                            Attributes::get(C, Attributes::NoUnwind));
     RetainCallee =
       M->getOrInsertFunction(
         "objc_retain",
@@ -1858,7 +1865,9 @@ Constant *ObjCARCOpt::getAutoreleaseCallee(Module *M) {
   if (!AutoreleaseCallee) {
     LLVMContext &C = M->getContext();
     Type *Params[] = { PointerType::getUnqual(Type::getInt8Ty(C)) };
-    AttrListPtr Attributes = AttrListPtr().addAttr(~0u, Attribute::NoUnwind);
+    AttrListPtr Attributes =
+      AttrListPtr().addAttr(M->getContext(), AttrListPtr::FunctionIndex,
+                            Attributes::get(C, Attributes::NoUnwind));
     AutoreleaseCallee =
       M->getOrInsertFunction(
         "objc_autorelease",
@@ -1868,6 +1877,26 @@ Constant *ObjCARCOpt::getAutoreleaseCallee(Module *M) {
   return AutoreleaseCallee;
 }
 
+/// IsPotentialUse - Test whether the given value is possible a
+/// reference-counted pointer, including tests which utilize AliasAnalysis.
+static bool IsPotentialUse(const Value *Op, AliasAnalysis &AA) {
+  // First make the rudimentary check.
+  if (!IsPotentialUse(Op))
+    return false;
+
+  // Objects in constant memory are not reference-counted.
+  if (AA.pointsToConstantMemory(Op))
+    return false;
+
+  // Pointers in constant memory are not pointing to reference-counted objects.
+  if (const LoadInst *LI = dyn_cast<LoadInst>(Op))
+    if (AA.pointsToConstantMemory(LI->getPointerOperand()))
+      return false;
+
+  // Otherwise assume the worst.
+  return true;
+}
+
 /// CanAlterRefCount - Test whether the given instruction can result in a
 /// reference count modification (positive or negative) for the pointer's
 /// object.
@@ -1894,7 +1923,7 @@ CanAlterRefCount(const Instruction *Inst, const Value *Ptr,
     for (ImmutableCallSite::arg_iterator I = CS.arg_begin(), E = CS.arg_end();
          I != E; ++I) {
       const Value *Op = *I;
-      if (IsPotentialUse(Op) && PA.related(Ptr, Op))
+      if (IsPotentialUse(Op, *PA.getAA()) && PA.related(Ptr, Op))
         return true;
     }
     return false;
@@ -1919,14 +1948,14 @@ CanUse(const Instruction *Inst, const Value *Ptr, ProvenanceAnalysis &PA,
     // Comparing a pointer with null, or any other constant, isn't really a use,
     // because we don't care what the pointer points to, or about the values
     // of any other dynamic reference-counted pointers.
-    if (!IsPotentialUse(ICI->getOperand(1)))
+    if (!IsPotentialUse(ICI->getOperand(1), *PA.getAA()))
       return false;
   } else if (ImmutableCallSite CS = static_cast<const Value *>(Inst)) {
     // For calls, just check the arguments (and not the callee operand).
     for (ImmutableCallSite::arg_iterator OI = CS.arg_begin(),
          OE = CS.arg_end(); OI != OE; ++OI) {
       const Value *Op = *OI;
-      if (IsPotentialUse(Op) && PA.related(Ptr, Op))
+      if (IsPotentialUse(Op, *PA.getAA()) && PA.related(Ptr, Op))
         return true;
     }
     return false;
@@ -1936,14 +1965,14 @@ CanUse(const Instruction *Inst, const Value *Ptr, ProvenanceAnalysis &PA,
     const Value *Op = GetUnderlyingObjCPtr(SI->getPointerOperand());
     // If we can't tell what the underlying object was, assume there is a
     // dependence.
-    return IsPotentialUse(Op) && PA.related(Op, Ptr);
+    return IsPotentialUse(Op, *PA.getAA()) && PA.related(Op, Ptr);
   }
 
   // Check each operand for a match.
   for (User::const_op_iterator OI = Inst->op_begin(), OE = Inst->op_end();
        OI != OE; ++OI) {
     const Value *Op = *OI;
-    if (IsPotentialUse(Op) && PA.related(Ptr, Op))
+    if (IsPotentialUse(Op, *PA.getAA()) && PA.related(Ptr, Op))
       return true;
   }
   return false;
@@ -2612,11 +2641,11 @@ ObjCARCOpt::VisitInstructionBottomUp(Instruction *Inst,
     MDNode *ReleaseMetadata = Inst->getMetadata(ImpreciseReleaseMDKind);
     S.ResetSequenceProgress(ReleaseMetadata ? S_MovableRelease : S_Release);
     S.RRI.ReleaseMetadata = ReleaseMetadata;
-    S.RRI.KnownSafe = S.IsKnownNested() || S.IsKnownIncremented();
+    S.RRI.KnownSafe = S.IsKnownIncremented();
     S.RRI.IsTailCallRelease = cast<CallInst>(Inst)->isTailCall();
     S.RRI.Calls.insert(Inst);
 
-    S.IncrementNestCount();
+    S.SetKnownPositiveRefCount();
     break;
   }
   case IC_RetainBlock:
@@ -2631,7 +2660,6 @@ ObjCARCOpt::VisitInstructionBottomUp(Instruction *Inst,
 
     PtrState &S = MyStates.getPtrBottomUpState(Arg);
     S.SetKnownPositiveRefCount();
-    S.DecrementNestCount();
 
     switch (S.GetSeq()) {
     case S_Stop:
@@ -2747,8 +2775,9 @@ ObjCARCOpt::VisitBottomUp(BasicBlock *BB,
 
   // Merge the states from each successor to compute the initial state
   // for the current block.
-  for (BBState::edge_iterator SI(MyStates.succ_begin()),
-       SE(MyStates.succ_end()); SI != SE; ++SI) {
+  BBState::edge_iterator SI(MyStates.succ_begin()),
+                         SE(MyStates.succ_end());
+  if (SI != SE) {
     const BasicBlock *Succ = *SI;
     DenseMap<const BasicBlock *, BBState>::iterator I = BBStates.find(Succ);
     assert(I != BBStates.end());
@@ -2760,7 +2789,6 @@ ObjCARCOpt::VisitBottomUp(BasicBlock *BB,
       assert(I != BBStates.end());
       MyStates.MergeSucc(I->second);
     }
-    break;
   }
 
   // Visit all the instructions, bottom-up.
@@ -2823,12 +2851,11 @@ ObjCARCOpt::VisitInstructionTopDown(Instruction *Inst,
 
       S.ResetSequenceProgress(S_Retain);
       S.RRI.IsRetainBlock = Class == IC_RetainBlock;
-      // Don't check S.IsKnownIncremented() here because it's not sufficient.
-      S.RRI.KnownSafe = S.IsKnownNested();
+      S.RRI.KnownSafe = S.IsKnownIncremented();
       S.RRI.Calls.insert(Inst);
     }
 
-    S.IncrementNestCount();
+    S.SetKnownPositiveRefCount();
 
     // A retain can be a potential use; procede to the generic checking
     // code below.
@@ -2838,7 +2865,7 @@ ObjCARCOpt::VisitInstructionTopDown(Instruction *Inst,
     Arg = GetObjCArg(Inst);
 
     PtrState &S = MyStates.getPtrTopDownState(Arg);
-    S.DecrementNestCount();
+    S.ClearRefCount();
 
     switch (S.GetSeq()) {
     case S_Retain:
@@ -2935,8 +2962,9 @@ ObjCARCOpt::VisitTopDown(BasicBlock *BB,
 
   // Merge the states from each predecessor to compute the initial state
   // for the current block.
-  for (BBState::edge_iterator PI(MyStates.pred_begin()),
-       PE(MyStates.pred_end()); PI != PE; ++PI) {
+  BBState::edge_iterator PI(MyStates.pred_begin()),
+                         PE(MyStates.pred_end());
+  if (PI != PE) {
     const BasicBlock *Pred = *PI;
     DenseMap<const BasicBlock *, BBState>::iterator I = BBStates.find(Pred);
     assert(I != BBStates.end());
@@ -2948,7 +2976,6 @@ ObjCARCOpt::VisitTopDown(BasicBlock *BB,
       assert(I != BBStates.end());
       MyStates.MergePred(I->second);
     }
-    break;
   }
 
   // Visit all the instructions, top-down.
@@ -3532,19 +3559,19 @@ bool ObjCARCOpt::OptimizeSequences(Function &F) {
 }
 
 /// OptimizeReturns - Look for this pattern:
-///
+/// \code
 ///    %call = call i8* @something(...)
 ///    %2 = call i8* @objc_retain(i8* %call)
 ///    %3 = call i8* @objc_autorelease(i8* %2)
 ///    ret i8* %3
-///
+/// \endcode
 /// And delete the retain and autorelease.
 ///
 /// Otherwise if it's just this:
-///
+/// \code
 ///    %3 = call i8* @objc_autorelease(i8* %2)
 ///    ret i8* %3
-///
+/// \endcode
 /// convert the autorelease to autoreleaseRV.
 void ObjCARCOpt::OptimizeReturns(Function &F) {
   if (!F.getReturnType()->isPointerTy())
@@ -3814,8 +3841,9 @@ Constant *ObjCARCContract::getStoreStrongCallee(Module *M) {
     Type *Params[] = { I8XX, I8X };
 
     AttrListPtr Attributes = AttrListPtr()
-      .addAttr(~0u, Attribute::NoUnwind)
-      .addAttr(1, Attribute::NoCapture);
+      .addAttr(M->getContext(), AttrListPtr::FunctionIndex,
+               Attributes::get(C, Attributes::NoUnwind))
+      .addAttr(M->getContext(), 1, Attributes::get(C, Attributes::NoCapture));
 
     StoreStrongCallee =
       M->getOrInsertFunction(
@@ -3832,7 +3860,9 @@ Constant *ObjCARCContract::getRetainAutoreleaseCallee(Module *M) {
     Type *I8X = PointerType::getUnqual(Type::getInt8Ty(C));
     Type *Params[] = { I8X };
     FunctionType *FTy = FunctionType::get(I8X, Params, /*isVarArg=*/false);
-    AttrListPtr Attributes = AttrListPtr().addAttr(~0u, Attribute::NoUnwind);
+    AttrListPtr Attributes =
+      AttrListPtr().addAttr(M->getContext(), AttrListPtr::FunctionIndex,
+                            Attributes::get(C, Attributes::NoUnwind));
     RetainAutoreleaseCallee =
       M->getOrInsertFunction("objc_retainAutorelease", FTy, Attributes);
   }
@@ -3845,7 +3875,9 @@ Constant *ObjCARCContract::getRetainAutoreleaseRVCallee(Module *M) {
     Type *I8X = PointerType::getUnqual(Type::getInt8Ty(C));
     Type *Params[] = { I8X };
     FunctionType *FTy = FunctionType::get(I8X, Params, /*isVarArg=*/false);
-    AttrListPtr Attributes = AttrListPtr().addAttr(~0u, Attribute::NoUnwind);
+    AttrListPtr Attributes =
+      AttrListPtr().addAttr(M->getContext(), AttrListPtr::FunctionIndex,
+                            Attributes::get(C, Attributes::NoUnwind));
     RetainAutoreleaseRVCallee =
       M->getOrInsertFunction("objc_retainAutoreleaseReturnValue", FTy,
                              Attributes);
diff --git a/lib/Transforms/Scalar/Reassociate.cpp b/lib/Transforms/Scalar/Reassociate.cpp
index 09687d8..7a40797 100644
--- a/lib/Transforms/Scalar/Reassociate.cpp
+++ b/lib/Transforms/Scalar/Reassociate.cpp
@@ -339,36 +339,6 @@ static void IncorporateWeight(APInt &LHS, const APInt &RHS, unsigned Opcode) {
   }
 }
 
-/// EvaluateRepeatedConstant - Compute C op C op ... op C where the constant C
-/// is repeated Weight times.
-static Constant *EvaluateRepeatedConstant(unsigned Opcode, Constant *C,
-                                          APInt Weight) {
-  // For addition the result can be efficiently computed as the product of the
-  // constant and the weight.
-  if (Opcode == Instruction::Add)
-    return ConstantExpr::getMul(C, ConstantInt::get(C->getContext(), Weight));
-
-  // The weight might be huge, so compute by repeated squaring to ensure that
-  // compile time is proportional to the logarithm of the weight.
-  Constant *Result = 0;
-  Constant *Power = C; // Successively C, C op C, (C op C) op (C op C) etc.
-  // Visit the bits in Weight.
-  while (Weight != 0) {
-    // If the current bit in Weight is non-zero do Result = Result op Power.
-    if (Weight[0])
-      Result = Result ? ConstantExpr::get(Opcode, Result, Power) : Power;
-    // Move on to the next bit if any more are non-zero.
-    Weight = Weight.lshr(1);
-    if (Weight.isMinValue())
-      break;
-    // Square the power.
-    Power = ConstantExpr::get(Opcode, Power, Power);
-  }
-
-  assert(Result && "Only positive weights supported!");
-  return Result;
-}
-
 typedef std::pair<Value*, APInt> RepeatedValue;
 
 /// LinearizeExprTree - Given an associative binary expression, return the leaf
@@ -382,9 +352,7 @@ typedef std::pair<Value*, APInt> RepeatedValue;
 /// op
 ///   (Ops[N].first op Ops[N].first op ... Ops[N].first)  <- Ops[N].second times
 ///
-/// Note that the values Ops[0].first, ..., Ops[N].first are all distinct, and
-/// they are all non-constant except possibly for the last one, which if it is
-/// constant will have weight one (Ops[N].second === 1).
+/// Note that the values Ops[0].first, ..., Ops[N].first are all distinct.
 ///
 /// This routine may modify the function, in which case it returns 'true'.  The
 /// changes it makes may well be destructive, changing the value computed by 'I'
@@ -604,7 +572,6 @@ static bool LinearizeExprTree(BinaryOperator *I,
 
   // The leaves, repeated according to their weights, represent the linearized
   // form of the expression.
-  Constant *Cst = 0; // Accumulate constants here.
   for (unsigned i = 0, e = LeafOrder.size(); i != e; ++i) {
     Value *V = LeafOrder[i];
     LeafMap::iterator It = Leaves.find(V);
@@ -618,31 +585,14 @@ static bool LinearizeExprTree(BinaryOperator *I,
       continue;
     // Ensure the leaf is only output once.
     It->second = 0;
-    // Glob all constants together into Cst.
-    if (Constant *C = dyn_cast<Constant>(V)) {
-      C = EvaluateRepeatedConstant(Opcode, C, Weight);
-      Cst = Cst ? ConstantExpr::get(Opcode, Cst, C) : C;
-      continue;
-    }
-    // Add non-constant
     Ops.push_back(std::make_pair(V, Weight));
   }
 
-  // Add any constants back into Ops, all globbed together and reduced to having
-  // weight 1 for the convenience of users.
-  Constant *Identity = ConstantExpr::getBinOpIdentity(Opcode, I->getType());
-  if (Cst && Cst != Identity) {
-    // If combining multiple constants resulted in the absorber then the entire
-    // expression must evaluate to the absorber.
-    if (Cst == Absorber)
-      Ops.clear();
-    Ops.push_back(std::make_pair(Cst, APInt(Bitwidth, 1)));
-  }
-
   // For nilpotent operations or addition there may be no operands, for example
   // because the expression was "X xor X" or consisted of 2^Bitwidth additions:
   // in both cases the weight reduces to 0 causing the value to be skipped.
   if (Ops.empty()) {
+    Constant *Identity = ConstantExpr::getBinOpIdentity(Opcode, I->getType());
     assert(Identity && "Associative operation without identity!");
     Ops.push_back(std::make_pair(Identity, APInt(Bitwidth, 1)));
   }
@@ -656,8 +606,8 @@ void Reassociate::RewriteExprTree(BinaryOperator *I,
                                   SmallVectorImpl<ValueEntry> &Ops) {
   assert(Ops.size() > 1 && "Single values should be used directly!");
 
-  // Since our optimizations never increase the number of operations, the new
-  // expression can always be written by reusing the existing binary operators
+  // Since our optimizations should never increase the number of operations, the
+  // new expression can usually be written reusing the existing binary operators
   // from the original expression tree, without creating any new instructions,
   // though the rewritten expression may have a completely different topology.
   // We take care to not change anything if the new expression will be the same
@@ -671,6 +621,20 @@ void Reassociate::RewriteExprTree(BinaryOperator *I,
   unsigned Opcode = I->getOpcode();
   BinaryOperator *Op = I;
 
+  /// NotRewritable - The operands being written will be the leaves of the new
+  /// expression and must not be used as inner nodes (via NodesToRewrite) by
+  /// mistake.  Inner nodes are always reassociable, and usually leaves are not
+  /// (if they were they would have been incorporated into the expression and so
+  /// would not be leaves), so most of the time there is no danger of this.  But
+  /// in rare cases a leaf may become reassociable if an optimization kills uses
+  /// of it, or it may momentarily become reassociable during rewriting (below)
+  /// due it being removed as an operand of one of its uses.  Ensure that misuse
+  /// of leaf nodes as inner nodes cannot occur by remembering all of the future
+  /// leaves and refusing to reuse any of them as inner nodes.
+  SmallPtrSet<Value*, 8> NotRewritable;
+  for (unsigned i = 0, e = Ops.size(); i != e; ++i)
+    NotRewritable.insert(Ops[i].Op);
+
   // ExpressionChanged - Non-null if the rewritten expression differs from the
   // original in some non-trivial way, requiring the clearing of optional flags.
   // Flags are cleared from the operator in ExpressionChanged up to I inclusive.
@@ -703,12 +667,14 @@ void Reassociate::RewriteExprTree(BinaryOperator *I,
       // the old operands with the new ones.
       DEBUG(dbgs() << "RA: " << *Op << '\n');
       if (NewLHS != OldLHS) {
-        if (BinaryOperator *BO = isReassociableOp(OldLHS, Opcode))
+        BinaryOperator *BO = isReassociableOp(OldLHS, Opcode);
+        if (BO && !NotRewritable.count(BO))
           NodesToRewrite.push_back(BO);
         Op->setOperand(0, NewLHS);
       }
       if (NewRHS != OldRHS) {
-        if (BinaryOperator *BO = isReassociableOp(OldRHS, Opcode))
+        BinaryOperator *BO = isReassociableOp(OldRHS, Opcode);
+        if (BO && !NotRewritable.count(BO))
           NodesToRewrite.push_back(BO);
         Op->setOperand(1, NewRHS);
       }
@@ -732,7 +698,8 @@ void Reassociate::RewriteExprTree(BinaryOperator *I,
         Op->swapOperands();
       } else {
         // Overwrite with the new right-hand side.
-        if (BinaryOperator *BO = isReassociableOp(Op->getOperand(1), Opcode))
+        BinaryOperator *BO = isReassociableOp(Op->getOperand(1), Opcode);
+        if (BO && !NotRewritable.count(BO))
           NodesToRewrite.push_back(BO);
         Op->setOperand(1, NewRHS);
         ExpressionChanged = Op;
@@ -745,7 +712,8 @@ void Reassociate::RewriteExprTree(BinaryOperator *I,
     // Now deal with the left-hand side.  If this is already an operation node
     // from the original expression then just rewrite the rest of the expression
     // into it.
-    if (BinaryOperator *BO = isReassociableOp(Op->getOperand(0), Opcode)) {
+    BinaryOperator *BO = isReassociableOp(Op->getOperand(0), Opcode);
+    if (BO && !NotRewritable.count(BO)) {
       Op = BO;
       continue;
     }
@@ -1446,9 +1414,26 @@ Value *Reassociate::OptimizeExpression(BinaryOperator *I,
                                        SmallVectorImpl<ValueEntry> &Ops) {
   // Now that we have the linearized expression tree, try to optimize it.
   // Start by folding any constants that we found.
-  if (Ops.size() == 1) return Ops[0].Op;
-
+  Constant *Cst = 0;
   unsigned Opcode = I->getOpcode();
+  while (!Ops.empty() && isa<Constant>(Ops.back().Op)) {
+    Constant *C = cast<Constant>(Ops.pop_back_val().Op);
+    Cst = Cst ? ConstantExpr::get(Opcode, C, Cst) : C;
+  }
+  // If there was nothing but constants then we are done.
+  if (Ops.empty())
+    return Cst;
+
+  // Put the combined constant back at the end of the operand list, except if
+  // there is no point.  For example, an add of 0 gets dropped here, while a
+  // multiplication by zero turns the whole expression into zero.
+  if (Cst && Cst != ConstantExpr::getBinOpIdentity(Opcode, I->getType())) {
+    if (Cst == ConstantExpr::getBinOpAbsorber(Opcode, I->getType()))
+      return Cst;
+    Ops.push_back(ValueEntry(0, Cst));
+  }
+
+  if (Ops.size() == 1) return Ops[0].Op;
 
   // Handle destructive annihilation due to identities between elements in the
   // argument list here.
diff --git a/lib/Transforms/Scalar/SCCP.cpp b/lib/Transforms/Scalar/SCCP.cpp
index 2c39aab..686520e 100644
--- a/lib/Transforms/Scalar/SCCP.cpp
+++ b/lib/Transforms/Scalar/SCCP.cpp
@@ -26,7 +26,7 @@
 #include "llvm/Pass.h"
 #include "llvm/Analysis/ConstantFolding.h"
 #include "llvm/Transforms/Utils/Local.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetLibraryInfo.h"
 #include "llvm/Support/CallSite.h"
 #include "llvm/Support/Debug.h"
@@ -153,7 +153,7 @@ namespace {
 /// Constant Propagation.
 ///
 class SCCPSolver : public InstVisitor<SCCPSolver> {
-  const TargetData *TD;
+  const DataLayout *TD;
   const TargetLibraryInfo *TLI;
   SmallPtrSet<BasicBlock*, 8> BBExecutable; // The BBs that are executable.
   DenseMap<Value*, LatticeVal> ValueState;  // The state each value is in.
@@ -205,7 +205,7 @@ class SCCPSolver : public InstVisitor<SCCPSolver> {
   typedef std::pair<BasicBlock*, BasicBlock*> Edge;
   DenseSet<Edge> KnownFeasibleEdges;
 public:
-  SCCPSolver(const TargetData *td, const TargetLibraryInfo *tli)
+  SCCPSolver(const DataLayout *td, const TargetLibraryInfo *tli)
     : TD(td), TLI(tli) {}
 
   /// MarkBlockExecutable - This method can be used by clients to mark all of
@@ -1564,7 +1564,7 @@ static void DeleteInstructionInBlock(BasicBlock *BB) {
 //
 bool SCCP::runOnFunction(Function &F) {
   DEBUG(dbgs() << "SCCP on function '" << F.getName() << "'\n");
-  const TargetData *TD = getAnalysisIfAvailable<TargetData>();
+  const DataLayout *TD = getAnalysisIfAvailable<DataLayout>();
   const TargetLibraryInfo *TLI = &getAnalysis<TargetLibraryInfo>();
   SCCPSolver Solver(TD, TLI);
 
@@ -1693,7 +1693,7 @@ static bool AddressIsTaken(const GlobalValue *GV) {
 }
 
 bool IPSCCP::runOnModule(Module &M) {
-  const TargetData *TD = getAnalysisIfAvailable<TargetData>();
+  const DataLayout *TD = getAnalysisIfAvailable<DataLayout>();
   const TargetLibraryInfo *TLI = &getAnalysis<TargetLibraryInfo>();
   SCCPSolver Solver(TD, TLI);
 
diff --git a/lib/Transforms/Scalar/SROA.cpp b/lib/Transforms/Scalar/SROA.cpp
new file mode 100644
index 0000000..ccc2f7a
--- /dev/null
+++ b/lib/Transforms/Scalar/SROA.cpp
@@ -0,0 +1,3697 @@
+//===- SROA.cpp - Scalar Replacement Of Aggregates ------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+/// \file
+/// This transformation implements the well known scalar replacement of
+/// aggregates transformation. It tries to identify promotable elements of an
+/// aggregate alloca, and promote them to registers. It will also try to
+/// convert uses of an element (or set of elements) of an alloca into a vector
+/// or bitfield-style integer scalar if appropriate.
+///
+/// It works to do this with minimal slicing of the alloca so that regions
+/// which are merely transferred in and out of external memory remain unchanged
+/// and are not decomposed to scalar code.
+///
+/// Because this also performs alloca promotion, it can be thought of as also
+/// serving the purpose of SSA formation. The algorithm iterates on the
+/// function until all opportunities for promotion have been realized.
+///
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "sroa"
+#include "llvm/Transforms/Scalar.h"
+#include "llvm/Constants.h"
+#include "llvm/DIBuilder.h"
+#include "llvm/DebugInfo.h"
+#include "llvm/DerivedTypes.h"
+#include "llvm/Function.h"
+#include "llvm/IRBuilder.h"
+#include "llvm/Instructions.h"
+#include "llvm/IntrinsicInst.h"
+#include "llvm/LLVMContext.h"
+#include "llvm/Module.h"
+#include "llvm/Operator.h"
+#include "llvm/Pass.h"
+#include "llvm/ADT/SetVector.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/Analysis/Dominators.h"
+#include "llvm/Analysis/Loads.h"
+#include "llvm/Analysis/ValueTracking.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/GetElementPtrTypeIterator.h"
+#include "llvm/Support/InstVisitor.h"
+#include "llvm/Support/MathExtras.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/DataLayout.h"
+#include "llvm/Transforms/Utils/Local.h"
+#include "llvm/Transforms/Utils/PromoteMemToReg.h"
+#include "llvm/Transforms/Utils/SSAUpdater.h"
+using namespace llvm;
+
+STATISTIC(NumAllocasAnalyzed, "Number of allocas analyzed for replacement");
+STATISTIC(NumNewAllocas,      "Number of new, smaller allocas introduced");
+STATISTIC(NumPromoted,        "Number of allocas promoted to SSA values");
+STATISTIC(NumLoadsSpeculated, "Number of loads speculated to allow promotion");
+STATISTIC(NumDeleted,         "Number of instructions deleted");
+STATISTIC(NumVectorized,      "Number of vectorized aggregates");
+
+/// Hidden option to force the pass to not use DomTree and mem2reg, instead
+/// forming SSA values through the SSAUpdater infrastructure.
+static cl::opt<bool>
+ForceSSAUpdater("force-ssa-updater", cl::init(false), cl::Hidden);
+
+namespace {
+/// \brief Alloca partitioning representation.
+///
+/// This class represents a partitioning of an alloca into slices, and
+/// information about the nature of uses of each slice of the alloca. The goal
+/// is that this information is sufficient to decide if and how to split the
+/// alloca apart and replace slices with scalars. It is also intended that this
+/// structure can capture the relevant information needed both to decide about
+/// and to enact these transformations.
+class AllocaPartitioning {
+public:
+  /// \brief A common base class for representing a half-open byte range.
+  struct ByteRange {
+    /// \brief The beginning offset of the range.
+    uint64_t BeginOffset;
+
+    /// \brief The ending offset, not included in the range.
+    uint64_t EndOffset;
+
+    ByteRange() : BeginOffset(), EndOffset() {}
+    ByteRange(uint64_t BeginOffset, uint64_t EndOffset)
+        : BeginOffset(BeginOffset), EndOffset(EndOffset) {}
+
+    /// \brief Support for ordering ranges.
+    ///
+    /// This provides an ordering over ranges such that start offsets are
+    /// always increasing, and within equal start offsets, the end offsets are
+    /// decreasing. Thus the spanning range comes first in a cluster with the
+    /// same start position.
+    bool operator<(const ByteRange &RHS) const {
+      if (BeginOffset < RHS.BeginOffset) return true;
+      if (BeginOffset > RHS.BeginOffset) return false;
+      if (EndOffset > RHS.EndOffset) return true;
+      return false;
+    }
+
+    /// \brief Support comparison with a single offset to allow binary searches.
+    friend bool operator<(const ByteRange &LHS, uint64_t RHSOffset) {
+      return LHS.BeginOffset < RHSOffset;
+    }
+
+    friend LLVM_ATTRIBUTE_UNUSED bool operator<(uint64_t LHSOffset,
+                                                const ByteRange &RHS) {
+      return LHSOffset < RHS.BeginOffset;
+    }
+
+    bool operator==(const ByteRange &RHS) const {
+      return BeginOffset == RHS.BeginOffset && EndOffset == RHS.EndOffset;
+    }
+    bool operator!=(const ByteRange &RHS) const { return !operator==(RHS); }
+  };
+
+  /// \brief A partition of an alloca.
+  ///
+  /// This structure represents a contiguous partition of the alloca. These are
+  /// formed by examining the uses of the alloca. During formation, they may
+  /// overlap but once an AllocaPartitioning is built, the Partitions within it
+  /// are all disjoint.
+  struct Partition : public ByteRange {
+    /// \brief Whether this partition is splittable into smaller partitions.
+    ///
+    /// We flag partitions as splittable when they are formed entirely due to
+    /// accesses by trivially splittable operations such as memset and memcpy.
+    bool IsSplittable;
+
+    /// \brief Test whether a partition has been marked as dead.
+    bool isDead() const {
+      if (BeginOffset == UINT64_MAX) {
+        assert(EndOffset == UINT64_MAX);
+        return true;
+      }
+      return false;
+    }
+
+    /// \brief Kill a partition.
+    /// This is accomplished by setting both its beginning and end offset to
+    /// the maximum possible value.
+    void kill() {
+      assert(!isDead() && "He's Dead, Jim!");
+      BeginOffset = EndOffset = UINT64_MAX;
+    }
+
+    Partition() : ByteRange(), IsSplittable() {}
+    Partition(uint64_t BeginOffset, uint64_t EndOffset, bool IsSplittable)
+        : ByteRange(BeginOffset, EndOffset), IsSplittable(IsSplittable) {}
+  };
+
+  /// \brief A particular use of a partition of the alloca.
+  ///
+  /// This structure is used to associate uses of a partition with it. They
+  /// mark the range of bytes which are referenced by a particular instruction,
+  /// and includes a handle to the user itself and the pointer value in use.
+  /// The bounds of these uses are determined by intersecting the bounds of the
+  /// memory use itself with a particular partition. As a consequence there is
+  /// intentionally overlap between various uses of the same partition.
+  struct PartitionUse : public ByteRange {
+    /// \brief The use in question. Provides access to both user and used value.
+    ///
+    /// Note that this may be null if the partition use is *dead*, that is, it
+    /// should be ignored.
+    Use *U;
+
+    PartitionUse() : ByteRange(), U() {}
+    PartitionUse(uint64_t BeginOffset, uint64_t EndOffset, Use *U)
+        : ByteRange(BeginOffset, EndOffset), U(U) {}
+  };
+
+  /// \brief Construct a partitioning of a particular alloca.
+  ///
+  /// Construction does most of the work for partitioning the alloca. This
+  /// performs the necessary walks of users and builds a partitioning from it.
+  AllocaPartitioning(const DataLayout &TD, AllocaInst &AI);
+
+  /// \brief Test whether a pointer to the allocation escapes our analysis.
+  ///
+  /// If this is true, the partitioning is never fully built and should be
+  /// ignored.
+  bool isEscaped() const { return PointerEscapingInstr; }
+
+  /// \brief Support for iterating over the partitions.
+  /// @{
+  typedef SmallVectorImpl<Partition>::iterator iterator;
+  iterator begin() { return Partitions.begin(); }
+  iterator end() { return Partitions.end(); }
+
+  typedef SmallVectorImpl<Partition>::const_iterator const_iterator;
+  const_iterator begin() const { return Partitions.begin(); }
+  const_iterator end() const { return Partitions.end(); }
+  /// @}
+
+  /// \brief Support for iterating over and manipulating a particular
+  /// partition's uses.
+  ///
+  /// The iteration support provided for uses is more limited, but also
+  /// includes some manipulation routines to support rewriting the uses of
+  /// partitions during SROA.
+  /// @{
+  typedef SmallVectorImpl<PartitionUse>::iterator use_iterator;
+  use_iterator use_begin(unsigned Idx) { return Uses[Idx].begin(); }
+  use_iterator use_begin(const_iterator I) { return Uses[I - begin()].begin(); }
+  use_iterator use_end(unsigned Idx) { return Uses[Idx].end(); }
+  use_iterator use_end(const_iterator I) { return Uses[I - begin()].end(); }
+
+  typedef SmallVectorImpl<PartitionUse>::const_iterator const_use_iterator;
+  const_use_iterator use_begin(unsigned Idx) const { return Uses[Idx].begin(); }
+  const_use_iterator use_begin(const_iterator I) const {
+    return Uses[I - begin()].begin();
+  }
+  const_use_iterator use_end(unsigned Idx) const { return Uses[Idx].end(); }
+  const_use_iterator use_end(const_iterator I) const {
+    return Uses[I - begin()].end();
+  }
+
+  unsigned use_size(unsigned Idx) const { return Uses[Idx].size(); }
+  unsigned use_size(const_iterator I) const { return Uses[I - begin()].size(); }
+  const PartitionUse &getUse(unsigned PIdx, unsigned UIdx) const {
+    return Uses[PIdx][UIdx];
+  }
+  const PartitionUse &getUse(const_iterator I, unsigned UIdx) const {
+    return Uses[I - begin()][UIdx];
+  }
+
+  void use_push_back(unsigned Idx, const PartitionUse &PU) {
+    Uses[Idx].push_back(PU);
+  }
+  void use_push_back(const_iterator I, const PartitionUse &PU) {
+    Uses[I - begin()].push_back(PU);
+  }
+  /// @}
+
+  /// \brief Allow iterating the dead users for this alloca.
+  ///
+  /// These are instructions which will never actually use the alloca as they
+  /// are outside the allocated range. They are safe to replace with undef and
+  /// delete.
+  /// @{
+  typedef SmallVectorImpl<Instruction *>::const_iterator dead_user_iterator;
+  dead_user_iterator dead_user_begin() const { return DeadUsers.begin(); }
+  dead_user_iterator dead_user_end() const { return DeadUsers.end(); }
+  /// @}
+
+  /// \brief Allow iterating the dead expressions referring to this alloca.
+  ///
+  /// These are operands which have cannot actually be used to refer to the
+  /// alloca as they are outside its range and the user doesn't correct for
+  /// that. These mostly consist of PHI node inputs and the like which we just
+  /// need to replace with undef.
+  /// @{
+  typedef SmallVectorImpl<Use *>::const_iterator dead_op_iterator;
+  dead_op_iterator dead_op_begin() const { return DeadOperands.begin(); }
+  dead_op_iterator dead_op_end() const { return DeadOperands.end(); }
+  /// @}
+
+  /// \brief MemTransferInst auxiliary data.
+  /// This struct provides some auxiliary data about memory transfer
+  /// intrinsics such as memcpy and memmove. These intrinsics can use two
+  /// different ranges within the same alloca, and provide other challenges to
+  /// correctly represent. We stash extra data to help us untangle this
+  /// after the partitioning is complete.
+  struct MemTransferOffsets {
+    /// The destination begin and end offsets when the destination is within
+    /// this alloca. If the end offset is zero the destination is not within
+    /// this alloca.
+    uint64_t DestBegin, DestEnd;
+
+    /// The source begin and end offsets when the source is within this alloca.
+    /// If the end offset is zero, the source is not within this alloca.
+    uint64_t SourceBegin, SourceEnd;
+
+    /// Flag for whether an alloca is splittable.
+    bool IsSplittable;
+  };
+  MemTransferOffsets getMemTransferOffsets(MemTransferInst &II) const {
+    return MemTransferInstData.lookup(&II);
+  }
+
+  /// \brief Map from a PHI or select operand back to a partition.
+  ///
+  /// When manipulating PHI nodes or selects, they can use more than one
+  /// partition of an alloca. We store a special mapping to allow finding the
+  /// partition referenced by each of these operands, if any.
+  iterator findPartitionForPHIOrSelectOperand(Use *U) {
+    SmallDenseMap<Use *, std::pair<unsigned, unsigned> >::const_iterator MapIt
+      = PHIOrSelectOpMap.find(U);
+    if (MapIt == PHIOrSelectOpMap.end())
+      return end();
+
+    return begin() + MapIt->second.first;
+  }
+
+  /// \brief Map from a PHI or select operand back to the specific use of
+  /// a partition.
+  ///
+  /// Similar to mapping these operands back to the partitions, this maps
+  /// directly to the use structure of that partition.
+  use_iterator findPartitionUseForPHIOrSelectOperand(Use *U) {
+    SmallDenseMap<Use *, std::pair<unsigned, unsigned> >::const_iterator MapIt
+      = PHIOrSelectOpMap.find(U);
+    assert(MapIt != PHIOrSelectOpMap.end());
+    return Uses[MapIt->second.first].begin() + MapIt->second.second;
+  }
+
+  /// \brief Compute a common type among the uses of a particular partition.
+  ///
+  /// This routines walks all of the uses of a particular partition and tries
+  /// to find a common type between them. Untyped operations such as memset and
+  /// memcpy are ignored.
+  Type *getCommonType(iterator I) const;
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+  void print(raw_ostream &OS, const_iterator I, StringRef Indent = "  ") const;
+  void printUsers(raw_ostream &OS, const_iterator I,
+                  StringRef Indent = "  ") const;
+  void print(raw_ostream &OS) const;
+  void LLVM_ATTRIBUTE_NOINLINE LLVM_ATTRIBUTE_USED dump(const_iterator I) const;
+  void LLVM_ATTRIBUTE_NOINLINE LLVM_ATTRIBUTE_USED dump() const;
+#endif
+
+private:
+  template <typename DerivedT, typename RetT = void> class BuilderBase;
+  class PartitionBuilder;
+  friend class AllocaPartitioning::PartitionBuilder;
+  class UseBuilder;
+  friend class AllocaPartitioning::UseBuilder;
+
+#ifndef NDEBUG
+  /// \brief Handle to alloca instruction to simplify method interfaces.
+  AllocaInst &AI;
+#endif
+
+  /// \brief The instruction responsible for this alloca having no partitioning.
+  ///
+  /// When an instruction (potentially) escapes the pointer to the alloca, we
+  /// store a pointer to that here and abort trying to partition the alloca.
+  /// This will be null if the alloca is partitioned successfully.
+  Instruction *PointerEscapingInstr;
+
+  /// \brief The partitions of the alloca.
+  ///
+  /// We store a vector of the partitions over the alloca here. This vector is
+  /// sorted by increasing begin offset, and then by decreasing end offset. See
+  /// the Partition inner class for more details. Initially (during
+  /// construction) there are overlaps, but we form a disjoint sequence of
+  /// partitions while finishing construction and a fully constructed object is
+  /// expected to always have this as a disjoint space.
+  SmallVector<Partition, 8> Partitions;
+
+  /// \brief The uses of the partitions.
+  ///
+  /// This is essentially a mapping from each partition to a list of uses of
+  /// that partition. The mapping is done with a Uses vector that has the exact
+  /// same number of entries as the partition vector. Each entry is itself
+  /// a vector of the uses.
+  SmallVector<SmallVector<PartitionUse, 2>, 8> Uses;
+
+  /// \brief Instructions which will become dead if we rewrite the alloca.
+  ///
+  /// Note that these are not separated by partition. This is because we expect
+  /// a partitioned alloca to be completely rewritten or not rewritten at all.
+  /// If rewritten, all these instructions can simply be removed and replaced
+  /// with undef as they come from outside of the allocated space.
+  SmallVector<Instruction *, 8> DeadUsers;
+
+  /// \brief Operands which will become dead if we rewrite the alloca.
+  ///
+  /// These are operands that in their particular use can be replaced with
+  /// undef when we rewrite the alloca. These show up in out-of-bounds inputs
+  /// to PHI nodes and the like. They aren't entirely dead (there might be
+  /// a GEP back into the bounds using it elsewhere) and nor is the PHI, but we
+  /// want to swap this particular input for undef to simplify the use lists of
+  /// the alloca.
+  SmallVector<Use *, 8> DeadOperands;
+
+  /// \brief The underlying storage for auxiliary memcpy and memset info.
+  SmallDenseMap<MemTransferInst *, MemTransferOffsets, 4> MemTransferInstData;
+
+  /// \brief A side datastructure used when building up the partitions and uses.
+  ///
+  /// This mapping is only really used during the initial building of the
+  /// partitioning so that we can retain information about PHI and select nodes
+  /// processed.
+  SmallDenseMap<Instruction *, std::pair<uint64_t, bool> > PHIOrSelectSizes;
+
+  /// \brief Auxiliary information for particular PHI or select operands.
+  SmallDenseMap<Use *, std::pair<unsigned, unsigned>, 4> PHIOrSelectOpMap;
+
+  /// \brief A utility routine called from the constructor.
+  ///
+  /// This does what it says on the tin. It is the key of the alloca partition
+  /// splitting and merging. After it is called we have the desired disjoint
+  /// collection of partitions.
+  void splitAndMergePartitions();
+};
+}
+
+template <typename DerivedT, typename RetT>
+class AllocaPartitioning::BuilderBase
+    : public InstVisitor<DerivedT, RetT> {
+public:
+  BuilderBase(const DataLayout &TD, AllocaInst &AI, AllocaPartitioning &P)
+      : TD(TD),
+        AllocSize(TD.getTypeAllocSize(AI.getAllocatedType())),
+        P(P) {
+    enqueueUsers(AI, 0);
+  }
+
+protected:
+  const DataLayout &TD;
+  const uint64_t AllocSize;
+  AllocaPartitioning &P;
+
+  SmallPtrSet<Use *, 8> VisitedUses;
+
+  struct OffsetUse {
+    Use *U;
+    int64_t Offset;
+  };
+  SmallVector<OffsetUse, 8> Queue;
+
+  // The active offset and use while visiting.
+  Use *U;
+  int64_t Offset;
+
+  void enqueueUsers(Instruction &I, int64_t UserOffset) {
+    for (Value::use_iterator UI = I.use_begin(), UE = I.use_end();
+         UI != UE; ++UI) {
+      if (VisitedUses.insert(&UI.getUse())) {
+        OffsetUse OU = { &UI.getUse(), UserOffset };
+        Queue.push_back(OU);
+      }
+    }
+  }
+
+  bool computeConstantGEPOffset(GetElementPtrInst &GEPI, int64_t &GEPOffset) {
+    GEPOffset = Offset;
+    for (gep_type_iterator GTI = gep_type_begin(GEPI), GTE = gep_type_end(GEPI);
+         GTI != GTE; ++GTI) {
+      ConstantInt *OpC = dyn_cast<ConstantInt>(GTI.getOperand());
+      if (!OpC)
+        return false;
+      if (OpC->isZero())
+        continue;
+
+      // Handle a struct index, which adds its field offset to the pointer.
+      if (StructType *STy = dyn_cast<StructType>(*GTI)) {
+        unsigned ElementIdx = OpC->getZExtValue();
+        const StructLayout *SL = TD.getStructLayout(STy);
+        uint64_t ElementOffset = SL->getElementOffset(ElementIdx);
+        // Check that we can continue to model this GEP in a signed 64-bit offset.
+        if (ElementOffset > INT64_MAX ||
+            (GEPOffset >= 0 &&
+             ((uint64_t)GEPOffset + ElementOffset) > INT64_MAX)) {
+          DEBUG(dbgs() << "WARNING: Encountered a cumulative offset exceeding "
+                       << "what can be represented in an int64_t!\n"
+                       << "  alloca: " << P.AI << "\n");
+          return false;
+        }
+        if (GEPOffset < 0)
+          GEPOffset = ElementOffset + (uint64_t)-GEPOffset;
+        else
+          GEPOffset += ElementOffset;
+        continue;
+      }
+
+      APInt Index = OpC->getValue().sextOrTrunc(TD.getPointerSizeInBits());
+      Index *= APInt(Index.getBitWidth(),
+                     TD.getTypeAllocSize(GTI.getIndexedType()));
+      Index += APInt(Index.getBitWidth(), (uint64_t)GEPOffset,
+                     /*isSigned*/true);
+      // Check if the result can be stored in our int64_t offset.
+      if (!Index.isSignedIntN(sizeof(GEPOffset) * 8)) {
+        DEBUG(dbgs() << "WARNING: Encountered a cumulative offset exceeding "
+                     << "what can be represented in an int64_t!\n"
+                     << "  alloca: " << P.AI << "\n");
+        return false;
+      }
+
+      GEPOffset = Index.getSExtValue();
+    }
+    return true;
+  }
+
+  Value *foldSelectInst(SelectInst &SI) {
+    // If the condition being selected on is a constant or the same value is
+    // being selected between, fold the select. Yes this does (rarely) happen
+    // early on.
+    if (ConstantInt *CI = dyn_cast<ConstantInt>(SI.getCondition()))
+      return SI.getOperand(1+CI->isZero());
+    if (SI.getOperand(1) == SI.getOperand(2)) {
+      assert(*U == SI.getOperand(1));
+      return SI.getOperand(1);
+    }
+    return 0;
+  }
+};
+
+/// \brief Builder for the alloca partitioning.
+///
+/// This class builds an alloca partitioning by recursively visiting the uses
+/// of an alloca and splitting the partitions for each load and store at each
+/// offset.
+class AllocaPartitioning::PartitionBuilder
+    : public BuilderBase<PartitionBuilder, bool> {
+  friend class InstVisitor<PartitionBuilder, bool>;
+
+  SmallDenseMap<Instruction *, unsigned> MemTransferPartitionMap;
+
+public:
+  PartitionBuilder(const DataLayout &TD, AllocaInst &AI, AllocaPartitioning &P)
+      : BuilderBase<PartitionBuilder, bool>(TD, AI, P) {}
+
+  /// \brief Run the builder over the allocation.
+  bool operator()() {
+    // Note that we have to re-evaluate size on each trip through the loop as
+    // the queue grows at the tail.
+    for (unsigned Idx = 0; Idx < Queue.size(); ++Idx) {
+      U = Queue[Idx].U;
+      Offset = Queue[Idx].Offset;
+      if (!visit(cast<Instruction>(U->getUser())))
+        return false;
+    }
+    return true;
+  }
+
+private:
+  bool markAsEscaping(Instruction &I) {
+    P.PointerEscapingInstr = &I;
+    return false;
+  }
+
+  void insertUse(Instruction &I, int64_t Offset, uint64_t Size,
+                 bool IsSplittable = false) {
+    // Completely skip uses which have a zero size or don't overlap the
+    // allocation.
+    if (Size == 0 ||
+        (Offset >= 0 && (uint64_t)Offset >= AllocSize) ||
+        (Offset < 0 && (uint64_t)-Offset >= Size)) {
+      DEBUG(dbgs() << "WARNING: Ignoring " << Size << " byte use @" << Offset
+                   << " which starts past the end of the " << AllocSize
+                   << " byte alloca:\n"
+                   << "    alloca: " << P.AI << "\n"
+                   << "       use: " << I << "\n");
+      return;
+    }
+
+    // Clamp the start to the beginning of the allocation.
+    if (Offset < 0) {
+      DEBUG(dbgs() << "WARNING: Clamping a " << Size << " byte use @" << Offset
+                   << " to start at the beginning of the alloca:\n"
+                   << "    alloca: " << P.AI << "\n"
+                   << "       use: " << I << "\n");
+      Size -= (uint64_t)-Offset;
+      Offset = 0;
+    }
+
+    uint64_t BeginOffset = Offset, EndOffset = BeginOffset + Size;
+
+    // Clamp the end offset to the end of the allocation. Note that this is
+    // formulated to handle even the case where "BeginOffset + Size" overflows.
+    // NOTE! This may appear superficially to be something we could ignore
+    // entirely, but that is not so! There may be PHI-node uses where some
+    // instructions are dead but not others. We can't completely ignore the
+    // PHI node, and so have to record at least the information here.
+    assert(AllocSize >= BeginOffset); // Established above.
+    if (Size > AllocSize - BeginOffset) {
+      DEBUG(dbgs() << "WARNING: Clamping a " << Size << " byte use @" << Offset
+                   << " to remain within the " << AllocSize << " byte alloca:\n"
+                   << "    alloca: " << P.AI << "\n"
+                   << "       use: " << I << "\n");
+      EndOffset = AllocSize;
+    }
+
+    Partition New(BeginOffset, EndOffset, IsSplittable);
+    P.Partitions.push_back(New);
+  }
+
+  bool handleLoadOrStore(Type *Ty, Instruction &I, int64_t Offset,
+                         bool IsVolatile) {
+    uint64_t Size = TD.getTypeStoreSize(Ty);
+
+    // If this memory access can be shown to *statically* extend outside the
+    // bounds of of the allocation, it's behavior is undefined, so simply
+    // ignore it. Note that this is more strict than the generic clamping
+    // behavior of insertUse. We also try to handle cases which might run the
+    // risk of overflow.
+    // FIXME: We should instead consider the pointer to have escaped if this
+    // function is being instrumented for addressing bugs or race conditions.
+    if (Offset < 0 || (uint64_t)Offset >= AllocSize ||
+        Size > (AllocSize - (uint64_t)Offset)) {
+      DEBUG(dbgs() << "WARNING: Ignoring " << Size << " byte "
+                   << (isa<LoadInst>(I) ? "load" : "store") << " @" << Offset
+                   << " which extends past the end of the " << AllocSize
+                   << " byte alloca:\n"
+                   << "    alloca: " << P.AI << "\n"
+                   << "       use: " << I << "\n");
+      return true;
+    }
+
+    // We allow splitting of loads and stores where the type is an integer type
+    // and which cover the entire alloca. Such integer loads and stores
+    // often require decomposition into fine grained loads and stores.
+    bool IsSplittable = false;
+    if (IntegerType *ITy = dyn_cast<IntegerType>(Ty))
+      IsSplittable = !IsVolatile && ITy->getBitWidth() == AllocSize*8;
+
+    insertUse(I, Offset, Size, IsSplittable);
+    return true;
+  }
+
+  bool visitBitCastInst(BitCastInst &BC) {
+    enqueueUsers(BC, Offset);
+    return true;
+  }
+
+  bool visitGetElementPtrInst(GetElementPtrInst &GEPI) {
+    int64_t GEPOffset;
+    if (!computeConstantGEPOffset(GEPI, GEPOffset))
+      return markAsEscaping(GEPI);
+
+    enqueueUsers(GEPI, GEPOffset);
+    return true;
+  }
+
+  bool visitLoadInst(LoadInst &LI) {
+    assert((!LI.isSimple() || LI.getType()->isSingleValueType()) &&
+           "All simple FCA loads should have been pre-split");
+    return handleLoadOrStore(LI.getType(), LI, Offset, LI.isVolatile());
+  }
+
+  bool visitStoreInst(StoreInst &SI) {
+    Value *ValOp = SI.getValueOperand();
+    if (ValOp == *U)
+      return markAsEscaping(SI);
+
+    assert((!SI.isSimple() || ValOp->getType()->isSingleValueType()) &&
+           "All simple FCA stores should have been pre-split");
+    return handleLoadOrStore(ValOp->getType(), SI, Offset, SI.isVolatile());
+  }
+
+
+  bool visitMemSetInst(MemSetInst &II) {
+    assert(II.getRawDest() == *U && "Pointer use is not the destination?");
+    ConstantInt *Length = dyn_cast<ConstantInt>(II.getLength());
+    uint64_t Size = Length ? Length->getZExtValue() : AllocSize - Offset;
+    insertUse(II, Offset, Size, Length);
+    return true;
+  }
+
+  bool visitMemTransferInst(MemTransferInst &II) {
+    ConstantInt *Length = dyn_cast<ConstantInt>(II.getLength());
+    uint64_t Size = Length ? Length->getZExtValue() : AllocSize - Offset;
+    if (!Size)
+      // Zero-length mem transfer intrinsics can be ignored entirely.
+      return true;
+
+    MemTransferOffsets &Offsets = P.MemTransferInstData[&II];
+
+    // Only intrinsics with a constant length can be split.
+    Offsets.IsSplittable = Length;
+
+    if (*U == II.getRawDest()) {
+      Offsets.DestBegin = Offset;
+      Offsets.DestEnd = Offset + Size;
+    }
+    if (*U == II.getRawSource()) {
+      Offsets.SourceBegin = Offset;
+      Offsets.SourceEnd = Offset + Size;
+    }
+
+    // If we have set up end offsets for both the source and the destination,
+    // we have found both sides of this transfer pointing at the same alloca.
+    bool SeenBothEnds = Offsets.SourceEnd && Offsets.DestEnd;
+    if (SeenBothEnds && II.getRawDest() != II.getRawSource()) {
+      unsigned PrevIdx = MemTransferPartitionMap[&II];
+
+      // Check if the begin offsets match and this is a non-volatile transfer.
+      // In that case, we can completely elide the transfer.
+      if (!II.isVolatile() && Offsets.SourceBegin == Offsets.DestBegin) {
+        P.Partitions[PrevIdx].kill();
+        return true;
+      }
+
+      // Otherwise we have an offset transfer within the same alloca. We can't
+      // split those.
+      P.Partitions[PrevIdx].IsSplittable = Offsets.IsSplittable = false;
+    } else if (SeenBothEnds) {
+      // Handle the case where this exact use provides both ends of the
+      // operation.
+      assert(II.getRawDest() == II.getRawSource());
+
+      // For non-volatile transfers this is a no-op.
+      if (!II.isVolatile())
+        return true;
+
+      // Otherwise just suppress splitting.
+      Offsets.IsSplittable = false;
+    }
+
+
+    // Insert the use now that we've fixed up the splittable nature.
+    insertUse(II, Offset, Size, Offsets.IsSplittable);
+
+    // Setup the mapping from intrinsic to partition of we've not seen both
+    // ends of this transfer.
+    if (!SeenBothEnds) {
+      unsigned NewIdx = P.Partitions.size() - 1;
+      bool Inserted
+        = MemTransferPartitionMap.insert(std::make_pair(&II, NewIdx)).second;
+      assert(Inserted &&
+             "Already have intrinsic in map but haven't seen both ends");
+      (void)Inserted;
+    }
+
+    return true;
+  }
+
+  // Disable SRoA for any intrinsics except for lifetime invariants.
+  // FIXME: What about debug instrinsics? This matches old behavior, but
+  // doesn't make sense.
+  bool visitIntrinsicInst(IntrinsicInst &II) {
+    if (II.getIntrinsicID() == Intrinsic::lifetime_start ||
+        II.getIntrinsicID() == Intrinsic::lifetime_end) {
+      ConstantInt *Length = cast<ConstantInt>(II.getArgOperand(0));
+      uint64_t Size = std::min(AllocSize - Offset, Length->getLimitedValue());
+      insertUse(II, Offset, Size, true);
+      return true;
+    }
+
+    return markAsEscaping(II);
+  }
+
+  Instruction *hasUnsafePHIOrSelectUse(Instruction *Root, uint64_t &Size) {
+    // We consider any PHI or select that results in a direct load or store of
+    // the same offset to be a viable use for partitioning purposes. These uses
+    // are considered unsplittable and the size is the maximum loaded or stored
+    // size.
+    SmallPtrSet<Instruction *, 4> Visited;
+    SmallVector<std::pair<Instruction *, Instruction *>, 4> Uses;
+    Visited.insert(Root);
+    Uses.push_back(std::make_pair(cast<Instruction>(*U), Root));
+    // If there are no loads or stores, the access is dead. We mark that as
+    // a size zero access.
+    Size = 0;
+    do {
+      Instruction *I, *UsedI;
+      llvm::tie(UsedI, I) = Uses.pop_back_val();
+
+      if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
+        Size = std::max(Size, TD.getTypeStoreSize(LI->getType()));
+        continue;
+      }
+      if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
+        Value *Op = SI->getOperand(0);
+        if (Op == UsedI)
+          return SI;
+        Size = std::max(Size, TD.getTypeStoreSize(Op->getType()));
+        continue;
+      }
+
+      if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(I)) {
+        if (!GEP->hasAllZeroIndices())
+          return GEP;
+      } else if (!isa<BitCastInst>(I) && !isa<PHINode>(I) &&
+                 !isa<SelectInst>(I)) {
+        return I;
+      }
+
+      for (Value::use_iterator UI = I->use_begin(), UE = I->use_end(); UI != UE;
+           ++UI)
+        if (Visited.insert(cast<Instruction>(*UI)))
+          Uses.push_back(std::make_pair(I, cast<Instruction>(*UI)));
+    } while (!Uses.empty());
+
+    return 0;
+  }
+
+  bool visitPHINode(PHINode &PN) {
+    // See if we already have computed info on this node.
+    std::pair<uint64_t, bool> &PHIInfo = P.PHIOrSelectSizes[&PN];
+    if (PHIInfo.first) {
+      PHIInfo.second = true;
+      insertUse(PN, Offset, PHIInfo.first);
+      return true;
+    }
+
+    // Check for an unsafe use of the PHI node.
+    if (Instruction *EscapingI = hasUnsafePHIOrSelectUse(&PN, PHIInfo.first))
+      return markAsEscaping(*EscapingI);
+
+    insertUse(PN, Offset, PHIInfo.first);
+    return true;
+  }
+
+  bool visitSelectInst(SelectInst &SI) {
+    if (Value *Result = foldSelectInst(SI)) {
+      if (Result == *U)
+        // If the result of the constant fold will be the pointer, recurse
+        // through the select as if we had RAUW'ed it.
+        enqueueUsers(SI, Offset);
+
+      return true;
+    }
+
+    // See if we already have computed info on this node.
+    std::pair<uint64_t, bool> &SelectInfo = P.PHIOrSelectSizes[&SI];
+    if (SelectInfo.first) {
+      SelectInfo.second = true;
+      insertUse(SI, Offset, SelectInfo.first);
+      return true;
+    }
+
+    // Check for an unsafe use of the PHI node.
+    if (Instruction *EscapingI = hasUnsafePHIOrSelectUse(&SI, SelectInfo.first))
+      return markAsEscaping(*EscapingI);
+
+    insertUse(SI, Offset, SelectInfo.first);
+    return true;
+  }
+
+  /// \brief Disable SROA entirely if there are unhandled users of the alloca.
+  bool visitInstruction(Instruction &I) { return markAsEscaping(I); }
+};
+
+
+/// \brief Use adder for the alloca partitioning.
+///
+/// This class adds the uses of an alloca to all of the partitions which they
+/// use. For splittable partitions, this can end up doing essentially a linear
+/// walk of the partitions, but the number of steps remains bounded by the
+/// total result instruction size:
+/// - The number of partitions is a result of the number unsplittable
+///   instructions using the alloca.
+/// - The number of users of each partition is at worst the total number of
+///   splittable instructions using the alloca.
+/// Thus we will produce N * M instructions in the end, where N are the number
+/// of unsplittable uses and M are the number of splittable. This visitor does
+/// the exact same number of updates to the partitioning.
+///
+/// In the more common case, this visitor will leverage the fact that the
+/// partition space is pre-sorted, and do a logarithmic search for the
+/// partition needed, making the total visit a classical ((N + M) * log(N))
+/// complexity operation.
+class AllocaPartitioning::UseBuilder : public BuilderBase<UseBuilder> {
+  friend class InstVisitor<UseBuilder>;
+
+  /// \brief Set to de-duplicate dead instructions found in the use walk.
+  SmallPtrSet<Instruction *, 4> VisitedDeadInsts;
+
+public:
+  UseBuilder(const DataLayout &TD, AllocaInst &AI, AllocaPartitioning &P)
+      : BuilderBase<UseBuilder>(TD, AI, P) {}
+
+  /// \brief Run the builder over the allocation.
+  void operator()() {
+    // Note that we have to re-evaluate size on each trip through the loop as
+    // the queue grows at the tail.
+    for (unsigned Idx = 0; Idx < Queue.size(); ++Idx) {
+      U = Queue[Idx].U;
+      Offset = Queue[Idx].Offset;
+      this->visit(cast<Instruction>(U->getUser()));
+    }
+  }
+
+private:
+  void markAsDead(Instruction &I) {
+    if (VisitedDeadInsts.insert(&I))
+      P.DeadUsers.push_back(&I);
+  }
+
+  void insertUse(Instruction &User, int64_t Offset, uint64_t Size) {
+    // If the use has a zero size or extends outside of the allocation, record
+    // it as a dead use for elimination later.
+    if (Size == 0 || (uint64_t)Offset >= AllocSize ||
+        (Offset < 0 && (uint64_t)-Offset >= Size))
+      return markAsDead(User);
+
+    // Clamp the start to the beginning of the allocation.
+    if (Offset < 0) {
+      Size -= (uint64_t)-Offset;
+      Offset = 0;
+    }
+
+    uint64_t BeginOffset = Offset, EndOffset = BeginOffset + Size;
+
+    // Clamp the end offset to the end of the allocation. Note that this is
+    // formulated to handle even the case where "BeginOffset + Size" overflows.
+    assert(AllocSize >= BeginOffset); // Established above.
+    if (Size > AllocSize - BeginOffset)
+      EndOffset = AllocSize;
+
+    // NB: This only works if we have zero overlapping partitions.
+    iterator B = std::lower_bound(P.begin(), P.end(), BeginOffset);
+    if (B != P.begin() && llvm::prior(B)->EndOffset > BeginOffset)
+      B = llvm::prior(B);
+    for (iterator I = B, E = P.end(); I != E && I->BeginOffset < EndOffset;
+         ++I) {
+      PartitionUse NewPU(std::max(I->BeginOffset, BeginOffset),
+                         std::min(I->EndOffset, EndOffset), U);
+      P.use_push_back(I, NewPU);
+      if (isa<PHINode>(U->getUser()) || isa<SelectInst>(U->getUser()))
+        P.PHIOrSelectOpMap[U]
+          = std::make_pair(I - P.begin(), P.Uses[I - P.begin()].size() - 1);
+    }
+  }
+
+  void handleLoadOrStore(Type *Ty, Instruction &I, int64_t Offset) {
+    uint64_t Size = TD.getTypeStoreSize(Ty);
+
+    // If this memory access can be shown to *statically* extend outside the
+    // bounds of of the allocation, it's behavior is undefined, so simply
+    // ignore it. Note that this is more strict than the generic clamping
+    // behavior of insertUse.
+    if (Offset < 0 || (uint64_t)Offset >= AllocSize ||
+        Size > (AllocSize - (uint64_t)Offset))
+      return markAsDead(I);
+
+    insertUse(I, Offset, Size);
+  }
+
+  void visitBitCastInst(BitCastInst &BC) {
+    if (BC.use_empty())
+      return markAsDead(BC);
+
+    enqueueUsers(BC, Offset);
+  }
+
+  void visitGetElementPtrInst(GetElementPtrInst &GEPI) {
+    if (GEPI.use_empty())
+      return markAsDead(GEPI);
+
+    int64_t GEPOffset;
+    if (!computeConstantGEPOffset(GEPI, GEPOffset))
+      llvm_unreachable("Unable to compute constant offset for use");
+
+    enqueueUsers(GEPI, GEPOffset);
+  }
+
+  void visitLoadInst(LoadInst &LI) {
+    handleLoadOrStore(LI.getType(), LI, Offset);
+  }
+
+  void visitStoreInst(StoreInst &SI) {
+    handleLoadOrStore(SI.getOperand(0)->getType(), SI, Offset);
+  }
+
+  void visitMemSetInst(MemSetInst &II) {
+    ConstantInt *Length = dyn_cast<ConstantInt>(II.getLength());
+    uint64_t Size = Length ? Length->getZExtValue() : AllocSize - Offset;
+    insertUse(II, Offset, Size);
+  }
+
+  void visitMemTransferInst(MemTransferInst &II) {
+    ConstantInt *Length = dyn_cast<ConstantInt>(II.getLength());
+    uint64_t Size = Length ? Length->getZExtValue() : AllocSize - Offset;
+    if (!Size)
+      return markAsDead(II);
+
+    MemTransferOffsets &Offsets = P.MemTransferInstData[&II];
+    if (!II.isVolatile() && Offsets.DestEnd && Offsets.SourceEnd &&
+        Offsets.DestBegin == Offsets.SourceBegin)
+      return markAsDead(II); // Skip identity transfers without side-effects.
+
+    insertUse(II, Offset, Size);
+  }
+
+  void visitIntrinsicInst(IntrinsicInst &II) {
+    assert(II.getIntrinsicID() == Intrinsic::lifetime_start ||
+           II.getIntrinsicID() == Intrinsic::lifetime_end);
+
+    ConstantInt *Length = cast<ConstantInt>(II.getArgOperand(0));
+    insertUse(II, Offset,
+              std::min(AllocSize - Offset, Length->getLimitedValue()));
+  }
+
+  void insertPHIOrSelect(Instruction &User, uint64_t Offset) {
+    uint64_t Size = P.PHIOrSelectSizes.lookup(&User).first;
+
+    // For PHI and select operands outside the alloca, we can't nuke the entire
+    // phi or select -- the other side might still be relevant, so we special
+    // case them here and use a separate structure to track the operands
+    // themselves which should be replaced with undef.
+    if (Offset >= AllocSize) {
+      P.DeadOperands.push_back(U);
+      return;
+    }
+
+    insertUse(User, Offset, Size);
+  }
+  void visitPHINode(PHINode &PN) {
+    if (PN.use_empty())
+      return markAsDead(PN);
+
+    insertPHIOrSelect(PN, Offset);
+  }
+  void visitSelectInst(SelectInst &SI) {
+    if (SI.use_empty())
+      return markAsDead(SI);
+
+    if (Value *Result = foldSelectInst(SI)) {
+      if (Result == *U)
+        // If the result of the constant fold will be the pointer, recurse
+        // through the select as if we had RAUW'ed it.
+        enqueueUsers(SI, Offset);
+      else
+        // Otherwise the operand to the select is dead, and we can replace it
+        // with undef.
+        P.DeadOperands.push_back(U);
+
+      return;
+    }
+
+    insertPHIOrSelect(SI, Offset);
+  }
+
+  /// \brief Unreachable, we've already visited the alloca once.
+  void visitInstruction(Instruction &I) {
+    llvm_unreachable("Unhandled instruction in use builder.");
+  }
+};
+
+void AllocaPartitioning::splitAndMergePartitions() {
+  size_t NumDeadPartitions = 0;
+
+  // Track the range of splittable partitions that we pass when accumulating
+  // overlapping unsplittable partitions.
+  uint64_t SplitEndOffset = 0ull;
+
+  Partition New(0ull, 0ull, false);
+
+  for (unsigned i = 0, j = i, e = Partitions.size(); i != e; i = j) {
+    ++j;
+
+    if (!Partitions[i].IsSplittable || New.BeginOffset == New.EndOffset) {
+      assert(New.BeginOffset == New.EndOffset);
+      New = Partitions[i];
+    } else {
+      assert(New.IsSplittable);
+      New.EndOffset = std::max(New.EndOffset, Partitions[i].EndOffset);
+    }
+    assert(New.BeginOffset != New.EndOffset);
+
+    // Scan the overlapping partitions.
+    while (j != e && New.EndOffset > Partitions[j].BeginOffset) {
+      // If the new partition we are forming is splittable, stop at the first
+      // unsplittable partition.
+      if (New.IsSplittable && !Partitions[j].IsSplittable)
+        break;
+
+      // Grow the new partition to include any equally splittable range. 'j' is
+      // always equally splittable when New is splittable, but when New is not
+      // splittable, we may subsume some (or part of some) splitable partition
+      // without growing the new one.
+      if (New.IsSplittable == Partitions[j].IsSplittable) {
+        New.EndOffset = std::max(New.EndOffset, Partitions[j].EndOffset);
+      } else {
+        assert(!New.IsSplittable);
+        assert(Partitions[j].IsSplittable);
+        SplitEndOffset = std::max(SplitEndOffset, Partitions[j].EndOffset);
+      }
+
+      Partitions[j].kill();
+      ++NumDeadPartitions;
+      ++j;
+    }
+
+    // If the new partition is splittable, chop off the end as soon as the
+    // unsplittable subsequent partition starts and ensure we eventually cover
+    // the splittable area.
+    if (j != e && New.IsSplittable) {
+      SplitEndOffset = std::max(SplitEndOffset, New.EndOffset);
+      New.EndOffset = std::min(New.EndOffset, Partitions[j].BeginOffset);
+    }
+
+    // Add the new partition if it differs from the original one and is
+    // non-empty. We can end up with an empty partition here if it was
+    // splittable but there is an unsplittable one that starts at the same
+    // offset.
+    if (New != Partitions[i]) {
+      if (New.BeginOffset != New.EndOffset)
+        Partitions.push_back(New);
+      // Mark the old one for removal.
+      Partitions[i].kill();
+      ++NumDeadPartitions;
+    }
+
+    New.BeginOffset = New.EndOffset;
+    if (!New.IsSplittable) {
+      New.EndOffset = std::max(New.EndOffset, SplitEndOffset);
+      if (j != e && !Partitions[j].IsSplittable)
+        New.EndOffset = std::min(New.EndOffset, Partitions[j].BeginOffset);
+      New.IsSplittable = true;
+      // If there is a trailing splittable partition which won't be fused into
+      // the next splittable partition go ahead and add it onto the partitions
+      // list.
+      if (New.BeginOffset < New.EndOffset &&
+          (j == e || !Partitions[j].IsSplittable ||
+           New.EndOffset < Partitions[j].BeginOffset)) {
+        Partitions.push_back(New);
+        New.BeginOffset = New.EndOffset = 0ull;
+      }
+    }
+  }
+
+  // Re-sort the partitions now that they have been split and merged into
+  // disjoint set of partitions. Also remove any of the dead partitions we've
+  // replaced in the process.
+  std::sort(Partitions.begin(), Partitions.end());
+  if (NumDeadPartitions) {
+    assert(Partitions.back().isDead());
+    assert((ptrdiff_t)NumDeadPartitions ==
+           std::count(Partitions.begin(), Partitions.end(), Partitions.back()));
+  }
+  Partitions.erase(Partitions.end() - NumDeadPartitions, Partitions.end());
+}
+
+AllocaPartitioning::AllocaPartitioning(const DataLayout &TD, AllocaInst &AI)
+    :
+#ifndef NDEBUG
+      AI(AI),
+#endif
+      PointerEscapingInstr(0) {
+  PartitionBuilder PB(TD, AI, *this);
+  if (!PB())
+    return;
+
+  // Sort the uses. This arranges for the offsets to be in ascending order,
+  // and the sizes to be in descending order.
+  std::sort(Partitions.begin(), Partitions.end());
+
+  // Remove any partitions from the back which are marked as dead.
+  while (!Partitions.empty() && Partitions.back().isDead())
+    Partitions.pop_back();
+
+  if (Partitions.size() > 1) {
+    // Intersect splittability for all partitions with equal offsets and sizes.
+    // Then remove all but the first so that we have a sequence of non-equal but
+    // potentially overlapping partitions.
+    for (iterator I = Partitions.begin(), J = I, E = Partitions.end(); I != E;
+         I = J) {
+      ++J;
+      while (J != E && *I == *J) {
+        I->IsSplittable &= J->IsSplittable;
+        ++J;
+      }
+    }
+    Partitions.erase(std::unique(Partitions.begin(), Partitions.end()),
+                     Partitions.end());
+
+    // Split splittable and merge unsplittable partitions into a disjoint set
+    // of partitions over the used space of the allocation.
+    splitAndMergePartitions();
+  }
+
+  // Now build up the user lists for each of these disjoint partitions by
+  // re-walking the recursive users of the alloca.
+  Uses.resize(Partitions.size());
+  UseBuilder UB(TD, AI, *this);
+  UB();
+}
+
+Type *AllocaPartitioning::getCommonType(iterator I) const {
+  Type *Ty = 0;
+  for (const_use_iterator UI = use_begin(I), UE = use_end(I); UI != UE; ++UI) {
+    if (!UI->U)
+      continue; // Skip dead uses.
+    if (isa<IntrinsicInst>(*UI->U->getUser()))
+      continue;
+    if (UI->BeginOffset != I->BeginOffset || UI->EndOffset != I->EndOffset)
+      continue;
+
+    Type *UserTy = 0;
+    if (LoadInst *LI = dyn_cast<LoadInst>(UI->U->getUser())) {
+      UserTy = LI->getType();
+    } else if (StoreInst *SI = dyn_cast<StoreInst>(UI->U->getUser())) {
+      UserTy = SI->getValueOperand()->getType();
+    } else {
+      return 0; // Bail if we have weird uses.
+    }
+
+    if (IntegerType *ITy = dyn_cast<IntegerType>(UserTy)) {
+      // If the type is larger than the partition, skip it. We only encounter
+      // this for split integer operations where we want to use the type of the
+      // entity causing the split.
+      if (ITy->getBitWidth() > (I->EndOffset - I->BeginOffset)*8)
+        continue;
+
+      // If we have found an integer type use covering the alloca, use that
+      // regardless of the other types, as integers are often used for a "bucket
+      // of bits" type.
+      return ITy;
+    }
+
+    if (Ty && Ty != UserTy)
+      return 0;
+
+    Ty = UserTy;
+  }
+  return Ty;
+}
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+
+void AllocaPartitioning::print(raw_ostream &OS, const_iterator I,
+                               StringRef Indent) const {
+  OS << Indent << "partition #" << (I - begin())
+     << " [" << I->BeginOffset << "," << I->EndOffset << ")"
+     << (I->IsSplittable ? " (splittable)" : "")
+     << (Uses[I - begin()].empty() ? " (zero uses)" : "")
+     << "\n";
+}
+
+void AllocaPartitioning::printUsers(raw_ostream &OS, const_iterator I,
+                                    StringRef Indent) const {
+  for (const_use_iterator UI = use_begin(I), UE = use_end(I);
+       UI != UE; ++UI) {
+    if (!UI->U)
+      continue; // Skip dead uses.
+    OS << Indent << "  [" << UI->BeginOffset << "," << UI->EndOffset << ") "
+       << "used by: " << *UI->U->getUser() << "\n";
+    if (MemTransferInst *II = dyn_cast<MemTransferInst>(UI->U->getUser())) {
+      const MemTransferOffsets &MTO = MemTransferInstData.lookup(II);
+      bool IsDest;
+      if (!MTO.IsSplittable)
+        IsDest = UI->BeginOffset == MTO.DestBegin;
+      else
+        IsDest = MTO.DestBegin != 0u;
+      OS << Indent << "    (original " << (IsDest ? "dest" : "source") << ": "
+         << "[" << (IsDest ? MTO.DestBegin : MTO.SourceBegin)
+         << "," << (IsDest ? MTO.DestEnd : MTO.SourceEnd) << ")\n";
+    }
+  }
+}
+
+void AllocaPartitioning::print(raw_ostream &OS) const {
+  if (PointerEscapingInstr) {
+    OS << "No partitioning for alloca: " << AI << "\n"
+       << "  A pointer to this alloca escaped by:\n"
+       << "  " << *PointerEscapingInstr << "\n";
+    return;
+  }
+
+  OS << "Partitioning of alloca: " << AI << "\n";
+  unsigned Num = 0;
+  for (const_iterator I = begin(), E = end(); I != E; ++I, ++Num) {
+    print(OS, I);
+    printUsers(OS, I);
+  }
+}
+
+void AllocaPartitioning::dump(const_iterator I) const { print(dbgs(), I); }
+void AllocaPartitioning::dump() const { print(dbgs()); }
+
+#endif // !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+
+
+namespace {
+/// \brief Implementation of LoadAndStorePromoter for promoting allocas.
+///
+/// This subclass of LoadAndStorePromoter adds overrides to handle promoting
+/// the loads and stores of an alloca instruction, as well as updating its
+/// debug information. This is used when a domtree is unavailable and thus
+/// mem2reg in its full form can't be used to handle promotion of allocas to
+/// scalar values.
+class AllocaPromoter : public LoadAndStorePromoter {
+  AllocaInst &AI;
+  DIBuilder &DIB;
+
+  SmallVector<DbgDeclareInst *, 4> DDIs;
+  SmallVector<DbgValueInst *, 4> DVIs;
+
+public:
+  AllocaPromoter(const SmallVectorImpl<Instruction*> &Insts, SSAUpdater &S,
+                 AllocaInst &AI, DIBuilder &DIB)
+    : LoadAndStorePromoter(Insts, S), AI(AI), DIB(DIB) {}
+
+  void run(const SmallVectorImpl<Instruction*> &Insts) {
+    // Remember which alloca we're promoting (for isInstInList).
+    if (MDNode *DebugNode = MDNode::getIfExists(AI.getContext(), &AI)) {
+      for (Value::use_iterator UI = DebugNode->use_begin(),
+                               UE = DebugNode->use_end();
+           UI != UE; ++UI)
+        if (DbgDeclareInst *DDI = dyn_cast<DbgDeclareInst>(*UI))
+          DDIs.push_back(DDI);
+        else if (DbgValueInst *DVI = dyn_cast<DbgValueInst>(*UI))
+          DVIs.push_back(DVI);
+    }
+
+    LoadAndStorePromoter::run(Insts);
+    AI.eraseFromParent();
+    while (!DDIs.empty())
+      DDIs.pop_back_val()->eraseFromParent();
+    while (!DVIs.empty())
+      DVIs.pop_back_val()->eraseFromParent();
+  }
+
+  virtual bool isInstInList(Instruction *I,
+                            const SmallVectorImpl<Instruction*> &Insts) const {
+    if (LoadInst *LI = dyn_cast<LoadInst>(I))
+      return LI->getOperand(0) == &AI;
+    return cast<StoreInst>(I)->getPointerOperand() == &AI;
+  }
+
+  virtual void updateDebugInfo(Instruction *Inst) const {
+    for (SmallVector<DbgDeclareInst *, 4>::const_iterator I = DDIs.begin(),
+           E = DDIs.end(); I != E; ++I) {
+      DbgDeclareInst *DDI = *I;
+      if (StoreInst *SI = dyn_cast<StoreInst>(Inst))
+        ConvertDebugDeclareToDebugValue(DDI, SI, DIB);
+      else if (LoadInst *LI = dyn_cast<LoadInst>(Inst))
+        ConvertDebugDeclareToDebugValue(DDI, LI, DIB);
+    }
+    for (SmallVector<DbgValueInst *, 4>::const_iterator I = DVIs.begin(),
+           E = DVIs.end(); I != E; ++I) {
+      DbgValueInst *DVI = *I;
+      Value *Arg = NULL;
+      if (StoreInst *SI = dyn_cast<StoreInst>(Inst)) {
+        // If an argument is zero extended then use argument directly. The ZExt
+        // may be zapped by an optimization pass in future.
+        if (ZExtInst *ZExt = dyn_cast<ZExtInst>(SI->getOperand(0)))
+          Arg = dyn_cast<Argument>(ZExt->getOperand(0));
+        if (SExtInst *SExt = dyn_cast<SExtInst>(SI->getOperand(0)))
+          Arg = dyn_cast<Argument>(SExt->getOperand(0));
+        if (!Arg)
+          Arg = SI->getOperand(0);
+      } else if (LoadInst *LI = dyn_cast<LoadInst>(Inst)) {
+        Arg = LI->getOperand(0);
+      } else {
+        continue;
+      }
+      Instruction *DbgVal =
+        DIB.insertDbgValueIntrinsic(Arg, 0, DIVariable(DVI->getVariable()),
+                                     Inst);
+      DbgVal->setDebugLoc(DVI->getDebugLoc());
+    }
+  }
+};
+} // end anon namespace
+
+
+namespace {
+/// \brief An optimization pass providing Scalar Replacement of Aggregates.
+///
+/// This pass takes allocations which can be completely analyzed (that is, they
+/// don't escape) and tries to turn them into scalar SSA values. There are
+/// a few steps to this process.
+///
+/// 1) It takes allocations of aggregates and analyzes the ways in which they
+///    are used to try to split them into smaller allocations, ideally of
+///    a single scalar data type. It will split up memcpy and memset accesses
+///    as necessary and try to isolate invidual scalar accesses.
+/// 2) It will transform accesses into forms which are suitable for SSA value
+///    promotion. This can be replacing a memset with a scalar store of an
+///    integer value, or it can involve speculating operations on a PHI or
+///    select to be a PHI or select of the results.
+/// 3) Finally, this will try to detect a pattern of accesses which map cleanly
+///    onto insert and extract operations on a vector value, and convert them to
+///    this form. By doing so, it will enable promotion of vector aggregates to
+///    SSA vector values.
+class SROA : public FunctionPass {
+  const bool RequiresDomTree;
+
+  LLVMContext *C;
+  const DataLayout *TD;
+  DominatorTree *DT;
+
+  /// \brief Worklist of alloca instructions to simplify.
+  ///
+  /// Each alloca in the function is added to this. Each new alloca formed gets
+  /// added to it as well to recursively simplify unless that alloca can be
+  /// directly promoted. Finally, each time we rewrite a use of an alloca other
+  /// the one being actively rewritten, we add it back onto the list if not
+  /// already present to ensure it is re-visited.
+  SetVector<AllocaInst *, SmallVector<AllocaInst *, 16> > Worklist;
+
+  /// \brief A collection of instructions to delete.
+  /// We try to batch deletions to simplify code and make things a bit more
+  /// efficient.
+  SetVector<Instruction *, SmallVector<Instruction *, 8> > DeadInsts;
+
+  /// \brief Post-promotion worklist.
+  ///
+  /// Sometimes we discover an alloca which has a high probability of becoming
+  /// viable for SROA after a round of promotion takes place. In those cases,
+  /// the alloca is enqueued here for re-processing.
+  ///
+  /// Note that we have to be very careful to clear allocas out of this list in
+  /// the event they are deleted.
+  SetVector<AllocaInst *, SmallVector<AllocaInst *, 16> > PostPromotionWorklist;
+
+  /// \brief A collection of alloca instructions we can directly promote.
+  std::vector<AllocaInst *> PromotableAllocas;
+
+public:
+  SROA(bool RequiresDomTree = true)
+      : FunctionPass(ID), RequiresDomTree(RequiresDomTree),
+        C(0), TD(0), DT(0) {
+    initializeSROAPass(*PassRegistry::getPassRegistry());
+  }
+  bool runOnFunction(Function &F);
+  void getAnalysisUsage(AnalysisUsage &AU) const;
+
+  const char *getPassName() const { return "SROA"; }
+  static char ID;
+
+private:
+  friend class PHIOrSelectSpeculator;
+  friend class AllocaPartitionRewriter;
+  friend class AllocaPartitionVectorRewriter;
+
+  bool rewriteAllocaPartition(AllocaInst &AI,
+                              AllocaPartitioning &P,
+                              AllocaPartitioning::iterator PI);
+  bool splitAlloca(AllocaInst &AI, AllocaPartitioning &P);
+  bool runOnAlloca(AllocaInst &AI);
+  void deleteDeadInstructions(SmallPtrSet<AllocaInst *, 4> &DeletedAllocas);
+  bool promoteAllocas(Function &F);
+};
+}
+
+char SROA::ID = 0;
+
+FunctionPass *llvm::createSROAPass(bool RequiresDomTree) {
+  return new SROA(RequiresDomTree);
+}
+
+INITIALIZE_PASS_BEGIN(SROA, "sroa", "Scalar Replacement Of Aggregates",
+                      false, false)
+INITIALIZE_PASS_DEPENDENCY(DominatorTree)
+INITIALIZE_PASS_END(SROA, "sroa", "Scalar Replacement Of Aggregates",
+                    false, false)
+
+namespace {
+/// \brief Visitor to speculate PHIs and Selects where possible.
+class PHIOrSelectSpeculator : public InstVisitor<PHIOrSelectSpeculator> {
+  // Befriend the base class so it can delegate to private visit methods.
+  friend class llvm::InstVisitor<PHIOrSelectSpeculator>;
+
+  const DataLayout &TD;
+  AllocaPartitioning &P;
+  SROA &Pass;
+
+public:
+  PHIOrSelectSpeculator(const DataLayout &TD, AllocaPartitioning &P, SROA &Pass)
+    : TD(TD), P(P), Pass(Pass) {}
+
+  /// \brief Visit the users of an alloca partition and rewrite them.
+  void visitUsers(AllocaPartitioning::const_iterator PI) {
+    // Note that we need to use an index here as the underlying vector of uses
+    // may be grown during speculation. However, we never need to re-visit the
+    // new uses, and so we can use the initial size bound.
+    for (unsigned Idx = 0, Size = P.use_size(PI); Idx != Size; ++Idx) {
+      const AllocaPartitioning::PartitionUse &PU = P.getUse(PI, Idx);
+      if (!PU.U)
+        continue; // Skip dead use.
+
+      visit(cast<Instruction>(PU.U->getUser()));
+    }
+  }
+
+private:
+  // By default, skip this instruction.
+  void visitInstruction(Instruction &I) {}
+
+  /// PHI instructions that use an alloca and are subsequently loaded can be
+  /// rewritten to load both input pointers in the pred blocks and then PHI the
+  /// results, allowing the load of the alloca to be promoted.
+  /// From this:
+  ///   %P2 = phi [i32* %Alloca, i32* %Other]
+  ///   %V = load i32* %P2
+  /// to:
+  ///   %V1 = load i32* %Alloca      -> will be mem2reg'd
+  ///   ...
+  ///   %V2 = load i32* %Other
+  ///   ...
+  ///   %V = phi [i32 %V1, i32 %V2]
+  ///
+  /// We can do this to a select if its only uses are loads and if the operands
+  /// to the select can be loaded unconditionally.
+  ///
+  /// FIXME: This should be hoisted into a generic utility, likely in
+  /// Transforms/Util/Local.h
+  bool isSafePHIToSpeculate(PHINode &PN, SmallVectorImpl<LoadInst *> &Loads) {
+    // For now, we can only do this promotion if the load is in the same block
+    // as the PHI, and if there are no stores between the phi and load.
+    // TODO: Allow recursive phi users.
+    // TODO: Allow stores.
+    BasicBlock *BB = PN.getParent();
+    unsigned MaxAlign = 0;
+    for (Value::use_iterator UI = PN.use_begin(), UE = PN.use_end();
+         UI != UE; ++UI) {
+      LoadInst *LI = dyn_cast<LoadInst>(*UI);
+      if (LI == 0 || !LI->isSimple()) return false;
+
+      // For now we only allow loads in the same block as the PHI.  This is
+      // a common case that happens when instcombine merges two loads through
+      // a PHI.
+      if (LI->getParent() != BB) return false;
+
+      // Ensure that there are no instructions between the PHI and the load that
+      // could store.
+      for (BasicBlock::iterator BBI = &PN; &*BBI != LI; ++BBI)
+        if (BBI->mayWriteToMemory())
+          return false;
+
+      MaxAlign = std::max(MaxAlign, LI->getAlignment());
+      Loads.push_back(LI);
+    }
+
+    // We can only transform this if it is safe to push the loads into the
+    // predecessor blocks. The only thing to watch out for is that we can't put
+    // a possibly trapping load in the predecessor if it is a critical edge.
+    for (unsigned Idx = 0, Num = PN.getNumIncomingValues(); Idx != Num;
+         ++Idx) {
+      TerminatorInst *TI = PN.getIncomingBlock(Idx)->getTerminator();
+      Value *InVal = PN.getIncomingValue(Idx);
+
+      // If the value is produced by the terminator of the predecessor (an
+      // invoke) or it has side-effects, there is no valid place to put a load
+      // in the predecessor.
+      if (TI == InVal || TI->mayHaveSideEffects())
+        return false;
+
+      // If the predecessor has a single successor, then the edge isn't
+      // critical.
+      if (TI->getNumSuccessors() == 1)
+        continue;
+
+      // If this pointer is always safe to load, or if we can prove that there
+      // is already a load in the block, then we can move the load to the pred
+      // block.
+      if (InVal->isDereferenceablePointer() ||
+          isSafeToLoadUnconditionally(InVal, TI, MaxAlign, &TD))
+        continue;
+
+      return false;
+    }
+
+    return true;
+  }
+
+  void visitPHINode(PHINode &PN) {
+    DEBUG(dbgs() << "    original: " << PN << "\n");
+
+    SmallVector<LoadInst *, 4> Loads;
+    if (!isSafePHIToSpeculate(PN, Loads))
+      return;
+
+    assert(!Loads.empty());
+
+    Type *LoadTy = cast<PointerType>(PN.getType())->getElementType();
+    IRBuilder<> PHIBuilder(&PN);
+    PHINode *NewPN = PHIBuilder.CreatePHI(LoadTy, PN.getNumIncomingValues(),
+                                          PN.getName() + ".sroa.speculated");
+
+    // Get the TBAA tag and alignment to use from one of the loads.  It doesn't
+    // matter which one we get and if any differ, it doesn't matter.
+    LoadInst *SomeLoad = cast<LoadInst>(Loads.back());
+    MDNode *TBAATag = SomeLoad->getMetadata(LLVMContext::MD_tbaa);
+    unsigned Align = SomeLoad->getAlignment();
+
+    // Rewrite all loads of the PN to use the new PHI.
+    do {
+      LoadInst *LI = Loads.pop_back_val();
+      LI->replaceAllUsesWith(NewPN);
+      Pass.DeadInsts.insert(LI);
+    } while (!Loads.empty());
+
+    // Inject loads into all of the pred blocks.
+    for (unsigned Idx = 0, Num = PN.getNumIncomingValues(); Idx != Num; ++Idx) {
+      BasicBlock *Pred = PN.getIncomingBlock(Idx);
+      TerminatorInst *TI = Pred->getTerminator();
+      Use *InUse = &PN.getOperandUse(PN.getOperandNumForIncomingValue(Idx));
+      Value *InVal = PN.getIncomingValue(Idx);
+      IRBuilder<> PredBuilder(TI);
+
+      LoadInst *Load
+        = PredBuilder.CreateLoad(InVal, (PN.getName() + ".sroa.speculate.load." +
+                                         Pred->getName()));
+      ++NumLoadsSpeculated;
+      Load->setAlignment(Align);
+      if (TBAATag)
+        Load->setMetadata(LLVMContext::MD_tbaa, TBAATag);
+      NewPN->addIncoming(Load, Pred);
+
+      Instruction *Ptr = dyn_cast<Instruction>(InVal);
+      if (!Ptr)
+        // No uses to rewrite.
+        continue;
+
+      // Try to lookup and rewrite any partition uses corresponding to this phi
+      // input.
+      AllocaPartitioning::iterator PI
+        = P.findPartitionForPHIOrSelectOperand(InUse);
+      if (PI == P.end())
+        continue;
+
+      // Replace the Use in the PartitionUse for this operand with the Use
+      // inside the load.
+      AllocaPartitioning::use_iterator UI
+        = P.findPartitionUseForPHIOrSelectOperand(InUse);
+      assert(isa<PHINode>(*UI->U->getUser()));
+      UI->U = &Load->getOperandUse(Load->getPointerOperandIndex());
+    }
+    DEBUG(dbgs() << "          speculated to: " << *NewPN << "\n");
+  }
+
+  /// Select instructions that use an alloca and are subsequently loaded can be
+  /// rewritten to load both input pointers and then select between the result,
+  /// allowing the load of the alloca to be promoted.
+  /// From this:
+  ///   %P2 = select i1 %cond, i32* %Alloca, i32* %Other
+  ///   %V = load i32* %P2
+  /// to:
+  ///   %V1 = load i32* %Alloca      -> will be mem2reg'd
+  ///   %V2 = load i32* %Other
+  ///   %V = select i1 %cond, i32 %V1, i32 %V2
+  ///
+  /// We can do this to a select if its only uses are loads and if the operand
+  /// to the select can be loaded unconditionally.
+  bool isSafeSelectToSpeculate(SelectInst &SI,
+                               SmallVectorImpl<LoadInst *> &Loads) {
+    Value *TValue = SI.getTrueValue();
+    Value *FValue = SI.getFalseValue();
+    bool TDerefable = TValue->isDereferenceablePointer();
+    bool FDerefable = FValue->isDereferenceablePointer();
+
+    for (Value::use_iterator UI = SI.use_begin(), UE = SI.use_end();
+         UI != UE; ++UI) {
+      LoadInst *LI = dyn_cast<LoadInst>(*UI);
+      if (LI == 0 || !LI->isSimple()) return false;
+
+      // Both operands to the select need to be dereferencable, either
+      // absolutely (e.g. allocas) or at this point because we can see other
+      // accesses to it.
+      if (!TDerefable && !isSafeToLoadUnconditionally(TValue, LI,
+                                                      LI->getAlignment(), &TD))
+        return false;
+      if (!FDerefable && !isSafeToLoadUnconditionally(FValue, LI,
+                                                      LI->getAlignment(), &TD))
+        return false;
+      Loads.push_back(LI);
+    }
+
+    return true;
+  }
+
+  void visitSelectInst(SelectInst &SI) {
+    DEBUG(dbgs() << "    original: " << SI << "\n");
+    IRBuilder<> IRB(&SI);
+
+    // If the select isn't safe to speculate, just use simple logic to emit it.
+    SmallVector<LoadInst *, 4> Loads;
+    if (!isSafeSelectToSpeculate(SI, Loads))
+      return;
+
+    Use *Ops[2] = { &SI.getOperandUse(1), &SI.getOperandUse(2) };
+    AllocaPartitioning::iterator PIs[2];
+    AllocaPartitioning::PartitionUse PUs[2];
+    for (unsigned i = 0, e = 2; i != e; ++i) {
+      PIs[i] = P.findPartitionForPHIOrSelectOperand(Ops[i]);
+      if (PIs[i] != P.end()) {
+        // If the pointer is within the partitioning, remove the select from
+        // its uses. We'll add in the new loads below.
+        AllocaPartitioning::use_iterator UI
+          = P.findPartitionUseForPHIOrSelectOperand(Ops[i]);
+        PUs[i] = *UI;
+        // Clear out the use here so that the offsets into the use list remain
+        // stable but this use is ignored when rewriting.
+        UI->U = 0;
+      }
+    }
+
+    Value *TV = SI.getTrueValue();
+    Value *FV = SI.getFalseValue();
+    // Replace the loads of the select with a select of two loads.
+    while (!Loads.empty()) {
+      LoadInst *LI = Loads.pop_back_val();
+
+      IRB.SetInsertPoint(LI);
+      LoadInst *TL =
+        IRB.CreateLoad(TV, LI->getName() + ".sroa.speculate.load.true");
+      LoadInst *FL =
+        IRB.CreateLoad(FV, LI->getName() + ".sroa.speculate.load.false");
+      NumLoadsSpeculated += 2;
+
+      // Transfer alignment and TBAA info if present.
+      TL->setAlignment(LI->getAlignment());
+      FL->setAlignment(LI->getAlignment());
+      if (MDNode *Tag = LI->getMetadata(LLVMContext::MD_tbaa)) {
+        TL->setMetadata(LLVMContext::MD_tbaa, Tag);
+        FL->setMetadata(LLVMContext::MD_tbaa, Tag);
+      }
+
+      Value *V = IRB.CreateSelect(SI.getCondition(), TL, FL,
+                                  LI->getName() + ".sroa.speculated");
+
+      LoadInst *Loads[2] = { TL, FL };
+      for (unsigned i = 0, e = 2; i != e; ++i) {
+        if (PIs[i] != P.end()) {
+          Use *LoadUse = &Loads[i]->getOperandUse(0);
+          assert(PUs[i].U->get() == LoadUse->get());
+          PUs[i].U = LoadUse;
+          P.use_push_back(PIs[i], PUs[i]);
+        }
+      }
+
+      DEBUG(dbgs() << "          speculated to: " << *V << "\n");
+      LI->replaceAllUsesWith(V);
+      Pass.DeadInsts.insert(LI);
+    }
+  }
+};
+}
+
+/// \brief Accumulate the constant offsets in a GEP into a single APInt offset.
+///
+/// If the provided GEP is all-constant, the total byte offset formed by the
+/// GEP is computed and Offset is set to it. If the GEP has any non-constant
+/// operands, the function returns false and the value of Offset is unmodified.
+static bool accumulateGEPOffsets(const DataLayout &TD, GEPOperator &GEP,
+                                 APInt &Offset) {
+  APInt GEPOffset(Offset.getBitWidth(), 0);
+  for (gep_type_iterator GTI = gep_type_begin(GEP), GTE = gep_type_end(GEP);
+       GTI != GTE; ++GTI) {
+    ConstantInt *OpC = dyn_cast<ConstantInt>(GTI.getOperand());
+    if (!OpC)
+      return false;
+    if (OpC->isZero()) continue;
+
+    // Handle a struct index, which adds its field offset to the pointer.
+    if (StructType *STy = dyn_cast<StructType>(*GTI)) {
+      unsigned ElementIdx = OpC->getZExtValue();
+      const StructLayout *SL = TD.getStructLayout(STy);
+      GEPOffset += APInt(Offset.getBitWidth(),
+                         SL->getElementOffset(ElementIdx));
+      continue;
+    }
+
+    APInt TypeSize(Offset.getBitWidth(),
+                   TD.getTypeAllocSize(GTI.getIndexedType()));
+    if (VectorType *VTy = dyn_cast<VectorType>(*GTI)) {
+      assert((VTy->getScalarSizeInBits() % 8) == 0 &&
+             "vector element size is not a multiple of 8, cannot GEP over it");
+      TypeSize = VTy->getScalarSizeInBits() / 8;
+    }
+
+    GEPOffset += OpC->getValue().sextOrTrunc(Offset.getBitWidth()) * TypeSize;
+  }
+  Offset = GEPOffset;
+  return true;
+}
+
+/// \brief Build a GEP out of a base pointer and indices.
+///
+/// This will return the BasePtr if that is valid, or build a new GEP
+/// instruction using the IRBuilder if GEP-ing is needed.
+static Value *buildGEP(IRBuilder<> &IRB, Value *BasePtr,
+                       SmallVectorImpl<Value *> &Indices,
+                       const Twine &Prefix) {
+  if (Indices.empty())
+    return BasePtr;
+
+  // A single zero index is a no-op, so check for this and avoid building a GEP
+  // in that case.
+  if (Indices.size() == 1 && cast<ConstantInt>(Indices.back())->isZero())
+    return BasePtr;
+
+  return IRB.CreateInBoundsGEP(BasePtr, Indices, Prefix + ".idx");
+}
+
+/// \brief Get a natural GEP off of the BasePtr walking through Ty toward
+/// TargetTy without changing the offset of the pointer.
+///
+/// This routine assumes we've already established a properly offset GEP with
+/// Indices, and arrived at the Ty type. The goal is to continue to GEP with
+/// zero-indices down through type layers until we find one the same as
+/// TargetTy. If we can't find one with the same type, we at least try to use
+/// one with the same size. If none of that works, we just produce the GEP as
+/// indicated by Indices to have the correct offset.
+static Value *getNaturalGEPWithType(IRBuilder<> &IRB, const DataLayout &TD,
+                                    Value *BasePtr, Type *Ty, Type *TargetTy,
+                                    SmallVectorImpl<Value *> &Indices,
+                                    const Twine &Prefix) {
+  if (Ty == TargetTy)
+    return buildGEP(IRB, BasePtr, Indices, Prefix);
+
+  // See if we can descend into a struct and locate a field with the correct
+  // type.
+  unsigned NumLayers = 0;
+  Type *ElementTy = Ty;
+  do {
+    if (ElementTy->isPointerTy())
+      break;
+    if (SequentialType *SeqTy = dyn_cast<SequentialType>(ElementTy)) {
+      ElementTy = SeqTy->getElementType();
+      // Note that we use the default address space as this index is over an
+      // array or a vector, not a pointer.
+      Indices.push_back(IRB.getInt(APInt(TD.getPointerSizeInBits(0), 0)));
+    } else if (StructType *STy = dyn_cast<StructType>(ElementTy)) {
+      if (STy->element_begin() == STy->element_end())
+        break; // Nothing left to descend into.
+      ElementTy = *STy->element_begin();
+      Indices.push_back(IRB.getInt32(0));
+    } else {
+      break;
+    }
+    ++NumLayers;
+  } while (ElementTy != TargetTy);
+  if (ElementTy != TargetTy)
+    Indices.erase(Indices.end() - NumLayers, Indices.end());
+
+  return buildGEP(IRB, BasePtr, Indices, Prefix);
+}
+
+/// \brief Recursively compute indices for a natural GEP.
+///
+/// This is the recursive step for getNaturalGEPWithOffset that walks down the
+/// element types adding appropriate indices for the GEP.
+static Value *getNaturalGEPRecursively(IRBuilder<> &IRB, const DataLayout &TD,
+                                       Value *Ptr, Type *Ty, APInt &Offset,
+                                       Type *TargetTy,
+                                       SmallVectorImpl<Value *> &Indices,
+                                       const Twine &Prefix) {
+  if (Offset == 0)
+    return getNaturalGEPWithType(IRB, TD, Ptr, Ty, TargetTy, Indices, Prefix);
+
+  // We can't recurse through pointer types.
+  if (Ty->isPointerTy())
+    return 0;
+
+  // We try to analyze GEPs over vectors here, but note that these GEPs are
+  // extremely poorly defined currently. The long-term goal is to remove GEPing
+  // over a vector from the IR completely.
+  if (VectorType *VecTy = dyn_cast<VectorType>(Ty)) {
+    unsigned ElementSizeInBits = VecTy->getScalarSizeInBits();
+    if (ElementSizeInBits % 8)
+      return 0; // GEPs over non-multiple of 8 size vector elements are invalid.
+    APInt ElementSize(Offset.getBitWidth(), ElementSizeInBits / 8);
+    APInt NumSkippedElements = Offset.sdiv(ElementSize);
+    if (NumSkippedElements.ugt(VecTy->getNumElements()))
+      return 0;
+    Offset -= NumSkippedElements * ElementSize;
+    Indices.push_back(IRB.getInt(NumSkippedElements));
+    return getNaturalGEPRecursively(IRB, TD, Ptr, VecTy->getElementType(),
+                                    Offset, TargetTy, Indices, Prefix);
+  }
+
+  if (ArrayType *ArrTy = dyn_cast<ArrayType>(Ty)) {
+    Type *ElementTy = ArrTy->getElementType();
+    APInt ElementSize(Offset.getBitWidth(), TD.getTypeAllocSize(ElementTy));
+    APInt NumSkippedElements = Offset.sdiv(ElementSize);
+    if (NumSkippedElements.ugt(ArrTy->getNumElements()))
+      return 0;
+
+    Offset -= NumSkippedElements * ElementSize;
+    Indices.push_back(IRB.getInt(NumSkippedElements));
+    return getNaturalGEPRecursively(IRB, TD, Ptr, ElementTy, Offset, TargetTy,
+                                    Indices, Prefix);
+  }
+
+  StructType *STy = dyn_cast<StructType>(Ty);
+  if (!STy)
+    return 0;
+
+  const StructLayout *SL = TD.getStructLayout(STy);
+  uint64_t StructOffset = Offset.getZExtValue();
+  if (StructOffset >= SL->getSizeInBytes())
+    return 0;
+  unsigned Index = SL->getElementContainingOffset(StructOffset);
+  Offset -= APInt(Offset.getBitWidth(), SL->getElementOffset(Index));
+  Type *ElementTy = STy->getElementType(Index);
+  if (Offset.uge(TD.getTypeAllocSize(ElementTy)))
+    return 0; // The offset points into alignment padding.
+
+  Indices.push_back(IRB.getInt32(Index));
+  return getNaturalGEPRecursively(IRB, TD, Ptr, ElementTy, Offset, TargetTy,
+                                  Indices, Prefix);
+}
+
+/// \brief Get a natural GEP from a base pointer to a particular offset and
+/// resulting in a particular type.
+///
+/// The goal is to produce a "natural" looking GEP that works with the existing
+/// composite types to arrive at the appropriate offset and element type for
+/// a pointer. TargetTy is the element type the returned GEP should point-to if
+/// possible. We recurse by decreasing Offset, adding the appropriate index to
+/// Indices, and setting Ty to the result subtype.
+///
+/// If no natural GEP can be constructed, this function returns null.
+static Value *getNaturalGEPWithOffset(IRBuilder<> &IRB, const DataLayout &TD,
+                                      Value *Ptr, APInt Offset, Type *TargetTy,
+                                      SmallVectorImpl<Value *> &Indices,
+                                      const Twine &Prefix) {
+  PointerType *Ty = cast<PointerType>(Ptr->getType());
+
+  // Don't consider any GEPs through an i8* as natural unless the TargetTy is
+  // an i8.
+  if (Ty == IRB.getInt8PtrTy() && TargetTy->isIntegerTy(8))
+    return 0;
+
+  Type *ElementTy = Ty->getElementType();
+  if (!ElementTy->isSized())
+    return 0; // We can't GEP through an unsized element.
+  APInt ElementSize(Offset.getBitWidth(), TD.getTypeAllocSize(ElementTy));
+  if (ElementSize == 0)
+    return 0; // Zero-length arrays can't help us build a natural GEP.
+  APInt NumSkippedElements = Offset.sdiv(ElementSize);
+
+  Offset -= NumSkippedElements * ElementSize;
+  Indices.push_back(IRB.getInt(NumSkippedElements));
+  return getNaturalGEPRecursively(IRB, TD, Ptr, ElementTy, Offset, TargetTy,
+                                  Indices, Prefix);
+}
+
+/// \brief Compute an adjusted pointer from Ptr by Offset bytes where the
+/// resulting pointer has PointerTy.
+///
+/// This tries very hard to compute a "natural" GEP which arrives at the offset
+/// and produces the pointer type desired. Where it cannot, it will try to use
+/// the natural GEP to arrive at the offset and bitcast to the type. Where that
+/// fails, it will try to use an existing i8* and GEP to the byte offset and
+/// bitcast to the type.
+///
+/// The strategy for finding the more natural GEPs is to peel off layers of the
+/// pointer, walking back through bit casts and GEPs, searching for a base
+/// pointer from which we can compute a natural GEP with the desired
+/// properities. The algorithm tries to fold as many constant indices into
+/// a single GEP as possible, thus making each GEP more independent of the
+/// surrounding code.
+static Value *getAdjustedPtr(IRBuilder<> &IRB, const DataLayout &TD,
+                             Value *Ptr, APInt Offset, Type *PointerTy,
+                             const Twine &Prefix) {
+  // Even though we don't look through PHI nodes, we could be called on an
+  // instruction in an unreachable block, which may be on a cycle.
+  SmallPtrSet<Value *, 4> Visited;
+  Visited.insert(Ptr);
+  SmallVector<Value *, 4> Indices;
+
+  // We may end up computing an offset pointer that has the wrong type. If we
+  // never are able to compute one directly that has the correct type, we'll
+  // fall back to it, so keep it around here.
+  Value *OffsetPtr = 0;
+
+  // Remember any i8 pointer we come across to re-use if we need to do a raw
+  // byte offset.
+  Value *Int8Ptr = 0;
+  APInt Int8PtrOffset(Offset.getBitWidth(), 0);
+
+  Type *TargetTy = PointerTy->getPointerElementType();
+
+  do {
+    // First fold any existing GEPs into the offset.
+    while (GEPOperator *GEP = dyn_cast<GEPOperator>(Ptr)) {
+      APInt GEPOffset(Offset.getBitWidth(), 0);
+      if (!accumulateGEPOffsets(TD, *GEP, GEPOffset))
+        break;
+      Offset += GEPOffset;
+      Ptr = GEP->getPointerOperand();
+      if (!Visited.insert(Ptr))
+        break;
+    }
+
+    // See if we can perform a natural GEP here.
+    Indices.clear();
+    if (Value *P = getNaturalGEPWithOffset(IRB, TD, Ptr, Offset, TargetTy,
+                                           Indices, Prefix)) {
+      if (P->getType() == PointerTy) {
+        // Zap any offset pointer that we ended up computing in previous rounds.
+        if (OffsetPtr && OffsetPtr->use_empty())
+          if (Instruction *I = dyn_cast<Instruction>(OffsetPtr))
+            I->eraseFromParent();
+        return P;
+      }
+      if (!OffsetPtr) {
+        OffsetPtr = P;
+      }
+    }
+
+    // Stash this pointer if we've found an i8*.
+    if (Ptr->getType()->isIntegerTy(8)) {
+      Int8Ptr = Ptr;
+      Int8PtrOffset = Offset;
+    }
+
+    // Peel off a layer of the pointer and update the offset appropriately.
+    if (Operator::getOpcode(Ptr) == Instruction::BitCast) {
+      Ptr = cast<Operator>(Ptr)->getOperand(0);
+    } else if (GlobalAlias *GA = dyn_cast<GlobalAlias>(Ptr)) {
+      if (GA->mayBeOverridden())
+        break;
+      Ptr = GA->getAliasee();
+    } else {
+      break;
+    }
+    assert(Ptr->getType()->isPointerTy() && "Unexpected operand type!");
+  } while (Visited.insert(Ptr));
+
+  if (!OffsetPtr) {
+    if (!Int8Ptr) {
+      Int8Ptr = IRB.CreateBitCast(Ptr, IRB.getInt8PtrTy(),
+                                  Prefix + ".raw_cast");
+      Int8PtrOffset = Offset;
+    }
+
+    OffsetPtr = Int8PtrOffset == 0 ? Int8Ptr :
+      IRB.CreateInBoundsGEP(Int8Ptr, IRB.getInt(Int8PtrOffset),
+                            Prefix + ".raw_idx");
+  }
+  Ptr = OffsetPtr;
+
+  // On the off chance we were targeting i8*, guard the bitcast here.
+  if (Ptr->getType() != PointerTy)
+    Ptr = IRB.CreateBitCast(Ptr, PointerTy, Prefix + ".cast");
+
+  return Ptr;
+}
+
+/// \brief Test whether we can convert a value from the old to the new type.
+///
+/// This predicate should be used to guard calls to convertValue in order to
+/// ensure that we only try to convert viable values. The strategy is that we
+/// will peel off single element struct and array wrappings to get to an
+/// underlying value, and convert that value.
+static bool canConvertValue(const DataLayout &DL, Type *OldTy, Type *NewTy) {
+  if (OldTy == NewTy)
+    return true;
+  if (DL.getTypeSizeInBits(NewTy) != DL.getTypeSizeInBits(OldTy))
+    return false;
+  if (!NewTy->isSingleValueType() || !OldTy->isSingleValueType())
+    return false;
+
+  if (NewTy->isPointerTy() || OldTy->isPointerTy()) {
+    if (NewTy->isPointerTy() && OldTy->isPointerTy())
+      return true;
+    if (NewTy->isIntegerTy() || OldTy->isIntegerTy())
+      return true;
+    return false;
+  }
+
+  return true;
+}
+
+/// \brief Generic routine to convert an SSA value to a value of a different
+/// type.
+///
+/// This will try various different casting techniques, such as bitcasts,
+/// inttoptr, and ptrtoint casts. Use the \c canConvertValue predicate to test
+/// two types for viability with this routine.
+static Value *convertValue(const DataLayout &DL, IRBuilder<> &IRB, Value *V,
+                           Type *Ty) {
+  assert(canConvertValue(DL, V->getType(), Ty) &&
+         "Value not convertable to type");
+  if (V->getType() == Ty)
+    return V;
+  if (V->getType()->isIntegerTy() && Ty->isPointerTy())
+    return IRB.CreateIntToPtr(V, Ty);
+  if (V->getType()->isPointerTy() && Ty->isIntegerTy())
+    return IRB.CreatePtrToInt(V, Ty);
+
+  return IRB.CreateBitCast(V, Ty);
+}
+
+/// \brief Test whether the given alloca partition can be promoted to a vector.
+///
+/// This is a quick test to check whether we can rewrite a particular alloca
+/// partition (and its newly formed alloca) into a vector alloca with only
+/// whole-vector loads and stores such that it could be promoted to a vector
+/// SSA value. We only can ensure this for a limited set of operations, and we
+/// don't want to do the rewrites unless we are confident that the result will
+/// be promotable, so we have an early test here.
+static bool isVectorPromotionViable(const DataLayout &TD,
+                                    Type *AllocaTy,
+                                    AllocaPartitioning &P,
+                                    uint64_t PartitionBeginOffset,
+                                    uint64_t PartitionEndOffset,
+                                    AllocaPartitioning::const_use_iterator I,
+                                    AllocaPartitioning::const_use_iterator E) {
+  VectorType *Ty = dyn_cast<VectorType>(AllocaTy);
+  if (!Ty)
+    return false;
+
+  uint64_t VecSize = TD.getTypeSizeInBits(Ty);
+  uint64_t ElementSize = Ty->getScalarSizeInBits();
+
+  // While the definition of LLVM vectors is bitpacked, we don't support sizes
+  // that aren't byte sized.
+  if (ElementSize % 8)
+    return false;
+  assert((VecSize % 8) == 0 && "vector size not a multiple of element size?");
+  VecSize /= 8;
+  ElementSize /= 8;
+
+  for (; I != E; ++I) {
+    if (!I->U)
+      continue; // Skip dead use.
+
+    uint64_t BeginOffset = I->BeginOffset - PartitionBeginOffset;
+    uint64_t BeginIndex = BeginOffset / ElementSize;
+    if (BeginIndex * ElementSize != BeginOffset ||
+        BeginIndex >= Ty->getNumElements())
+      return false;
+    uint64_t EndOffset = I->EndOffset - PartitionBeginOffset;
+    uint64_t EndIndex = EndOffset / ElementSize;
+    if (EndIndex * ElementSize != EndOffset ||
+        EndIndex > Ty->getNumElements())
+      return false;
+
+    // FIXME: We should build shuffle vector instructions to handle
+    // non-element-sized accesses.
+    if ((EndOffset - BeginOffset) != ElementSize &&
+        (EndOffset - BeginOffset) != VecSize)
+      return false;
+
+    if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(I->U->getUser())) {
+      if (MI->isVolatile())
+        return false;
+      if (MemTransferInst *MTI = dyn_cast<MemTransferInst>(I->U->getUser())) {
+        const AllocaPartitioning::MemTransferOffsets &MTO
+          = P.getMemTransferOffsets(*MTI);
+        if (!MTO.IsSplittable)
+          return false;
+      }
+    } else if (I->U->get()->getType()->getPointerElementType()->isStructTy()) {
+      // Disable vector promotion when there are loads or stores of an FCA.
+      return false;
+    } else if (LoadInst *LI = dyn_cast<LoadInst>(I->U->getUser())) {
+      if (LI->isVolatile())
+        return false;
+    } else if (StoreInst *SI = dyn_cast<StoreInst>(I->U->getUser())) {
+      if (SI->isVolatile())
+        return false;
+    } else {
+      return false;
+    }
+  }
+  return true;
+}
+
+/// \brief Test whether the given alloca partition's integer operations can be
+/// widened to promotable ones.
+///
+/// This is a quick test to check whether we can rewrite the integer loads and
+/// stores to a particular alloca into wider loads and stores and be able to
+/// promote the resulting alloca.
+static bool isIntegerWideningViable(const DataLayout &TD,
+                                    Type *AllocaTy,
+                                    uint64_t AllocBeginOffset,
+                                    AllocaPartitioning &P,
+                                    AllocaPartitioning::const_use_iterator I,
+                                    AllocaPartitioning::const_use_iterator E) {
+  uint64_t SizeInBits = TD.getTypeSizeInBits(AllocaTy);
+
+  // Don't try to handle allocas with bit-padding.
+  if (SizeInBits != TD.getTypeStoreSizeInBits(AllocaTy))
+    return false;
+
+  // We need to ensure that an integer type with the appropriate bitwidth can
+  // be converted to the alloca type, whatever that is. We don't want to force
+  // the alloca itself to have an integer type if there is a more suitable one.
+  Type *IntTy = Type::getIntNTy(AllocaTy->getContext(), SizeInBits);
+  if (!canConvertValue(TD, AllocaTy, IntTy) ||
+      !canConvertValue(TD, IntTy, AllocaTy))
+    return false;
+
+  uint64_t Size = TD.getTypeStoreSize(AllocaTy);
+
+  // Check the uses to ensure the uses are (likely) promoteable integer uses.
+  // Also ensure that the alloca has a covering load or store. We don't want
+  // to widen the integer operotains only to fail to promote due to some other
+  // unsplittable entry (which we may make splittable later).
+  bool WholeAllocaOp = false;
+  for (; I != E; ++I) {
+    if (!I->U)
+      continue; // Skip dead use.
+
+    uint64_t RelBegin = I->BeginOffset - AllocBeginOffset;
+    uint64_t RelEnd = I->EndOffset - AllocBeginOffset;
+
+    // We can't reasonably handle cases where the load or store extends past
+    // the end of the aloca's type and into its padding.
+    if (RelEnd > Size)
+      return false;
+
+    if (LoadInst *LI = dyn_cast<LoadInst>(I->U->getUser())) {
+      if (LI->isVolatile())
+        return false;
+      if (RelBegin == 0 && RelEnd == Size)
+        WholeAllocaOp = true;
+      if (IntegerType *ITy = dyn_cast<IntegerType>(LI->getType())) {
+        if (ITy->getBitWidth() < TD.getTypeStoreSize(ITy))
+          return false;
+        continue;
+      }
+      // Non-integer loads need to be convertible from the alloca type so that
+      // they are promotable.
+      if (RelBegin != 0 || RelEnd != Size ||
+          !canConvertValue(TD, AllocaTy, LI->getType()))
+        return false;
+    } else if (StoreInst *SI = dyn_cast<StoreInst>(I->U->getUser())) {
+      Type *ValueTy = SI->getValueOperand()->getType();
+      if (SI->isVolatile())
+        return false;
+      if (RelBegin == 0 && RelEnd == Size)
+        WholeAllocaOp = true;
+      if (IntegerType *ITy = dyn_cast<IntegerType>(ValueTy)) {
+        if (ITy->getBitWidth() < TD.getTypeStoreSize(ITy))
+          return false;
+        continue;
+      }
+      // Non-integer stores need to be convertible to the alloca type so that
+      // they are promotable.
+      if (RelBegin != 0 || RelEnd != Size ||
+          !canConvertValue(TD, ValueTy, AllocaTy))
+        return false;
+    } else if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(I->U->getUser())) {
+      if (MI->isVolatile())
+        return false;
+      if (MemTransferInst *MTI = dyn_cast<MemTransferInst>(I->U->getUser())) {
+        const AllocaPartitioning::MemTransferOffsets &MTO
+          = P.getMemTransferOffsets(*MTI);
+        if (!MTO.IsSplittable)
+          return false;
+      }
+    } else if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I->U->getUser())) {
+      if (II->getIntrinsicID() != Intrinsic::lifetime_start &&
+          II->getIntrinsicID() != Intrinsic::lifetime_end)
+        return false;
+    } else {
+      return false;
+    }
+  }
+  return WholeAllocaOp;
+}
+
+static Value *extractInteger(const DataLayout &DL, IRBuilder<> &IRB, Value *V,
+                             IntegerType *Ty, uint64_t Offset,
+                             const Twine &Name) {
+  DEBUG(dbgs() << "       start: " << *V << "\n");
+  IntegerType *IntTy = cast<IntegerType>(V->getType());
+  assert(DL.getTypeStoreSize(Ty) + Offset <= DL.getTypeStoreSize(IntTy) &&
+         "Element extends past full value");
+  uint64_t ShAmt = 8*Offset;
+  if (DL.isBigEndian())
+    ShAmt = 8*(DL.getTypeStoreSize(IntTy) - DL.getTypeStoreSize(Ty) - Offset);
+  if (ShAmt) {
+    V = IRB.CreateLShr(V, ShAmt, Name + ".shift");
+    DEBUG(dbgs() << "     shifted: " << *V << "\n");
+  }
+  assert(Ty->getBitWidth() <= IntTy->getBitWidth() &&
+         "Cannot extract to a larger integer!");
+  if (Ty != IntTy) {
+    V = IRB.CreateTrunc(V, Ty, Name + ".trunc");
+    DEBUG(dbgs() << "     trunced: " << *V << "\n");
+  }
+  return V;
+}
+
+static Value *insertInteger(const DataLayout &DL, IRBuilder<> &IRB, Value *Old,
+                            Value *V, uint64_t Offset, const Twine &Name) {
+  IntegerType *IntTy = cast<IntegerType>(Old->getType());
+  IntegerType *Ty = cast<IntegerType>(V->getType());
+  assert(Ty->getBitWidth() <= IntTy->getBitWidth() &&
+         "Cannot insert a larger integer!");
+  DEBUG(dbgs() << "       start: " << *V << "\n");
+  if (Ty != IntTy) {
+    V = IRB.CreateZExt(V, IntTy, Name + ".ext");
+    DEBUG(dbgs() << "    extended: " << *V << "\n");
+  }
+  assert(DL.getTypeStoreSize(Ty) + Offset <= DL.getTypeStoreSize(IntTy) &&
+         "Element store outside of alloca store");
+  uint64_t ShAmt = 8*Offset;
+  if (DL.isBigEndian())
+    ShAmt = 8*(DL.getTypeStoreSize(IntTy) - DL.getTypeStoreSize(Ty) - Offset);
+  if (ShAmt) {
+    V = IRB.CreateShl(V, ShAmt, Name + ".shift");
+    DEBUG(dbgs() << "     shifted: " << *V << "\n");
+  }
+
+  if (ShAmt || Ty->getBitWidth() < IntTy->getBitWidth()) {
+    APInt Mask = ~Ty->getMask().zext(IntTy->getBitWidth()).shl(ShAmt);
+    Old = IRB.CreateAnd(Old, Mask, Name + ".mask");
+    DEBUG(dbgs() << "      masked: " << *Old << "\n");
+    V = IRB.CreateOr(Old, V, Name + ".insert");
+    DEBUG(dbgs() << "    inserted: " << *V << "\n");
+  }
+  return V;
+}
+
+namespace {
+/// \brief Visitor to rewrite instructions using a partition of an alloca to
+/// use a new alloca.
+///
+/// Also implements the rewriting to vector-based accesses when the partition
+/// passes the isVectorPromotionViable predicate. Most of the rewriting logic
+/// lives here.
+class AllocaPartitionRewriter : public InstVisitor<AllocaPartitionRewriter,
+                                                   bool> {
+  // Befriend the base class so it can delegate to private visit methods.
+  friend class llvm::InstVisitor<AllocaPartitionRewriter, bool>;
+
+  const DataLayout &TD;
+  AllocaPartitioning &P;
+  SROA &Pass;
+  AllocaInst &OldAI, &NewAI;
+  const uint64_t NewAllocaBeginOffset, NewAllocaEndOffset;
+  Type *NewAllocaTy;
+
+  // If we are rewriting an alloca partition which can be written as pure
+  // vector operations, we stash extra information here. When VecTy is
+  // non-null, we have some strict guarantees about the rewriten alloca:
+  //   - The new alloca is exactly the size of the vector type here.
+  //   - The accesses all either map to the entire vector or to a single
+  //     element.
+  //   - The set of accessing instructions is only one of those handled above
+  //     in isVectorPromotionViable. Generally these are the same access kinds
+  //     which are promotable via mem2reg.
+  VectorType *VecTy;
+  Type *ElementTy;
+  uint64_t ElementSize;
+
+  // This is a convenience and flag variable that will be null unless the new
+  // alloca's integer operations should be widened to this integer type due to
+  // passing isIntegerWideningViable above. If it is non-null, the desired
+  // integer type will be stored here for easy access during rewriting.
+  IntegerType *IntTy;
+
+  // The offset of the partition user currently being rewritten.
+  uint64_t BeginOffset, EndOffset;
+  Use *OldUse;
+  Instruction *OldPtr;
+
+  // The name prefix to use when rewriting instructions for this alloca.
+  std::string NamePrefix;
+
+public:
+  AllocaPartitionRewriter(const DataLayout &TD, AllocaPartitioning &P,
+                          AllocaPartitioning::iterator PI,
+                          SROA &Pass, AllocaInst &OldAI, AllocaInst &NewAI,
+                          uint64_t NewBeginOffset, uint64_t NewEndOffset)
+    : TD(TD), P(P), Pass(Pass),
+      OldAI(OldAI), NewAI(NewAI),
+      NewAllocaBeginOffset(NewBeginOffset),
+      NewAllocaEndOffset(NewEndOffset),
+      NewAllocaTy(NewAI.getAllocatedType()),
+      VecTy(), ElementTy(), ElementSize(), IntTy(),
+      BeginOffset(), EndOffset() {
+  }
+
+  /// \brief Visit the users of the alloca partition and rewrite them.
+  bool visitUsers(AllocaPartitioning::const_use_iterator I,
+                  AllocaPartitioning::const_use_iterator E) {
+    if (isVectorPromotionViable(TD, NewAI.getAllocatedType(), P,
+                                NewAllocaBeginOffset, NewAllocaEndOffset,
+                                I, E)) {
+      ++NumVectorized;
+      VecTy = cast<VectorType>(NewAI.getAllocatedType());
+      ElementTy = VecTy->getElementType();
+      assert((VecTy->getScalarSizeInBits() % 8) == 0 &&
+             "Only multiple-of-8 sized vector elements are viable");
+      ElementSize = VecTy->getScalarSizeInBits() / 8;
+    } else if (isIntegerWideningViable(TD, NewAI.getAllocatedType(),
+                                       NewAllocaBeginOffset, P, I, E)) {
+      IntTy = Type::getIntNTy(NewAI.getContext(),
+                              TD.getTypeSizeInBits(NewAI.getAllocatedType()));
+    }
+    bool CanSROA = true;
+    for (; I != E; ++I) {
+      if (!I->U)
+        continue; // Skip dead uses.
+      BeginOffset = I->BeginOffset;
+      EndOffset = I->EndOffset;
+      OldUse = I->U;
+      OldPtr = cast<Instruction>(I->U->get());
+      NamePrefix = (Twine(NewAI.getName()) + "." + Twine(BeginOffset)).str();
+      CanSROA &= visit(cast<Instruction>(I->U->getUser()));
+    }
+    if (VecTy) {
+      assert(CanSROA);
+      VecTy = 0;
+      ElementTy = 0;
+      ElementSize = 0;
+    }
+    if (IntTy) {
+      assert(CanSROA);
+      IntTy = 0;
+    }
+    return CanSROA;
+  }
+
+private:
+  // Every instruction which can end up as a user must have a rewrite rule.
+  bool visitInstruction(Instruction &I) {
+    DEBUG(dbgs() << "    !!!! Cannot rewrite: " << I << "\n");
+    llvm_unreachable("No rewrite rule for this instruction!");
+  }
+
+  Twine getName(const Twine &Suffix) {
+    return NamePrefix + Suffix;
+  }
+
+  Value *getAdjustedAllocaPtr(IRBuilder<> &IRB, Type *PointerTy) {
+    assert(BeginOffset >= NewAllocaBeginOffset);
+    APInt Offset(TD.getPointerSizeInBits(), BeginOffset - NewAllocaBeginOffset);
+    return getAdjustedPtr(IRB, TD, &NewAI, Offset, PointerTy, getName(""));
+  }
+
+  /// \brief Compute suitable alignment to access an offset into the new alloca.
+  unsigned getOffsetAlign(uint64_t Offset) {
+    unsigned NewAIAlign = NewAI.getAlignment();
+    if (!NewAIAlign)
+      NewAIAlign = TD.getABITypeAlignment(NewAI.getAllocatedType());
+    return MinAlign(NewAIAlign, Offset);
+  }
+
+  /// \brief Compute suitable alignment to access this partition of the new
+  /// alloca.
+  unsigned getPartitionAlign() {
+    return getOffsetAlign(BeginOffset - NewAllocaBeginOffset);
+  }
+
+  /// \brief Compute suitable alignment to access a type at an offset of the
+  /// new alloca.
+  ///
+  /// \returns zero if the type's ABI alignment is a suitable alignment,
+  /// otherwise returns the maximal suitable alignment.
+  unsigned getOffsetTypeAlign(Type *Ty, uint64_t Offset) {
+    unsigned Align = getOffsetAlign(Offset);
+    return Align == TD.getABITypeAlignment(Ty) ? 0 : Align;
+  }
+
+  /// \brief Compute suitable alignment to access a type at the beginning of
+  /// this partition of the new alloca.
+  ///
+  /// See \c getOffsetTypeAlign for details; this routine delegates to it.
+  unsigned getPartitionTypeAlign(Type *Ty) {
+    return getOffsetTypeAlign(Ty, BeginOffset - NewAllocaBeginOffset);
+  }
+
+  ConstantInt *getIndex(IRBuilder<> &IRB, uint64_t Offset) {
+    assert(VecTy && "Can only call getIndex when rewriting a vector");
+    uint64_t RelOffset = Offset - NewAllocaBeginOffset;
+    assert(RelOffset / ElementSize < UINT32_MAX && "Index out of bounds");
+    uint32_t Index = RelOffset / ElementSize;
+    assert(Index * ElementSize == RelOffset);
+    return IRB.getInt32(Index);
+  }
+
+  void deleteIfTriviallyDead(Value *V) {
+    Instruction *I = cast<Instruction>(V);
+    if (isInstructionTriviallyDead(I))
+      Pass.DeadInsts.insert(I);
+  }
+
+  Value *rewriteVectorizedLoadInst(IRBuilder<> &IRB, LoadInst &LI, Value *OldOp) {
+    Value *V = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(),
+                                     getName(".load"));
+    if (LI.getType() == VecTy->getElementType() ||
+        BeginOffset > NewAllocaBeginOffset || EndOffset < NewAllocaEndOffset) {
+      V = IRB.CreateExtractElement(V, getIndex(IRB, BeginOffset),
+                                   getName(".extract"));
+    }
+    return V;
+  }
+
+  Value *rewriteIntegerLoad(IRBuilder<> &IRB, LoadInst &LI) {
+    assert(IntTy && "We cannot insert an integer to the alloca");
+    assert(!LI.isVolatile());
+    Value *V = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(),
+                                     getName(".load"));
+    V = convertValue(TD, IRB, V, IntTy);
+    assert(BeginOffset >= NewAllocaBeginOffset && "Out of bounds offset");
+    uint64_t Offset = BeginOffset - NewAllocaBeginOffset;
+    if (Offset > 0 || EndOffset < NewAllocaEndOffset)
+      V = extractInteger(TD, IRB, V, cast<IntegerType>(LI.getType()), Offset,
+                         getName(".extract"));
+    return V;
+  }
+
+  bool visitLoadInst(LoadInst &LI) {
+    DEBUG(dbgs() << "    original: " << LI << "\n");
+    Value *OldOp = LI.getOperand(0);
+    assert(OldOp == OldPtr);
+    IRBuilder<> IRB(&LI);
+
+    uint64_t Size = EndOffset - BeginOffset;
+    bool IsSplitIntLoad = Size < TD.getTypeStoreSize(LI.getType());
+
+    // If this memory access can be shown to *statically* extend outside the
+    // bounds of the original allocation it's behavior is undefined. Rather
+    // than trying to transform it, just replace it with undef.
+    // FIXME: We should do something more clever for functions being
+    // instrumented by asan.
+    // FIXME: Eventually, once ASan and friends can flush out bugs here, this
+    // should be transformed to a load of null making it unreachable.
+    uint64_t OldAllocSize = TD.getTypeAllocSize(OldAI.getAllocatedType());
+    if (TD.getTypeStoreSize(LI.getType()) > OldAllocSize) {
+      LI.replaceAllUsesWith(UndefValue::get(LI.getType()));
+      Pass.DeadInsts.insert(&LI);
+      deleteIfTriviallyDead(OldOp);
+      DEBUG(dbgs() << "          to: undef!!\n");
+      return true;
+    }
+
+    Type *TargetTy = IsSplitIntLoad ? Type::getIntNTy(LI.getContext(), Size * 8)
+                                    : LI.getType();
+    bool IsPtrAdjusted = false;
+    Value *V;
+    if (VecTy) {
+      V = rewriteVectorizedLoadInst(IRB, LI, OldOp);
+    } else if (IntTy && LI.getType()->isIntegerTy()) {
+      V = rewriteIntegerLoad(IRB, LI);
+    } else if (BeginOffset == NewAllocaBeginOffset &&
+               canConvertValue(TD, NewAllocaTy, LI.getType())) {
+      V = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(),
+                                LI.isVolatile(), getName(".load"));
+    } else {
+      Type *LTy = TargetTy->getPointerTo();
+      V = IRB.CreateAlignedLoad(getAdjustedAllocaPtr(IRB, LTy),
+                                getPartitionTypeAlign(TargetTy),
+                                LI.isVolatile(), getName(".load"));
+      IsPtrAdjusted = true;
+    }
+    V = convertValue(TD, IRB, V, TargetTy);
+
+    if (IsSplitIntLoad) {
+      assert(!LI.isVolatile());
+      assert(LI.getType()->isIntegerTy() &&
+             "Only integer type loads and stores are split");
+      assert(LI.getType()->getIntegerBitWidth() ==
+             TD.getTypeStoreSizeInBits(LI.getType()) &&
+             "Non-byte-multiple bit width");
+      assert(LI.getType()->getIntegerBitWidth() ==
+             TD.getTypeAllocSizeInBits(OldAI.getAllocatedType()) &&
+             "Only alloca-wide loads can be split and recomposed");
+      // Move the insertion point just past the load so that we can refer to it.
+      IRB.SetInsertPoint(llvm::next(BasicBlock::iterator(&LI)));
+      // Create a placeholder value with the same type as LI to use as the
+      // basis for the new value. This allows us to replace the uses of LI with
+      // the computed value, and then replace the placeholder with LI, leaving
+      // LI only used for this computation.
+      Value *Placeholder
+        = new LoadInst(UndefValue::get(LI.getType()->getPointerTo()));
+      V = insertInteger(TD, IRB, Placeholder, V, BeginOffset,
+                        getName(".insert"));
+      LI.replaceAllUsesWith(V);
+      Placeholder->replaceAllUsesWith(&LI);
+      delete Placeholder;
+    } else {
+      LI.replaceAllUsesWith(V);
+    }
+
+    Pass.DeadInsts.insert(&LI);
+    deleteIfTriviallyDead(OldOp);
+    DEBUG(dbgs() << "          to: " << *V << "\n");
+    return !LI.isVolatile() && !IsPtrAdjusted;
+  }
+
+  bool rewriteVectorizedStoreInst(IRBuilder<> &IRB, Value *V,
+                                  StoreInst &SI, Value *OldOp) {
+    if (V->getType() == ElementTy ||
+        BeginOffset > NewAllocaBeginOffset || EndOffset < NewAllocaEndOffset) {
+      if (V->getType() != ElementTy)
+        V = convertValue(TD, IRB, V, ElementTy);
+      LoadInst *LI = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(),
+                                           getName(".load"));
+      V = IRB.CreateInsertElement(LI, V, getIndex(IRB, BeginOffset),
+                                  getName(".insert"));
+    } else if (V->getType() != VecTy) {
+      V = convertValue(TD, IRB, V, VecTy);
+    }
+    StoreInst *Store = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlignment());
+    Pass.DeadInsts.insert(&SI);
+
+    (void)Store;
+    DEBUG(dbgs() << "          to: " << *Store << "\n");
+    return true;
+  }
+
+  bool rewriteIntegerStore(IRBuilder<> &IRB, Value *V, StoreInst &SI) {
+    assert(IntTy && "We cannot extract an integer from the alloca");
+    assert(!SI.isVolatile());
+    if (TD.getTypeSizeInBits(V->getType()) != IntTy->getBitWidth()) {
+      Value *Old = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(),
+                                         getName(".oldload"));
+      Old = convertValue(TD, IRB, Old, IntTy);
+      assert(BeginOffset >= NewAllocaBeginOffset && "Out of bounds offset");
+      uint64_t Offset = BeginOffset - NewAllocaBeginOffset;
+      V = insertInteger(TD, IRB, Old, SI.getValueOperand(), Offset,
+                        getName(".insert"));
+    }
+    V = convertValue(TD, IRB, V, NewAllocaTy);
+    StoreInst *Store = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlignment());
+    Pass.DeadInsts.insert(&SI);
+    (void)Store;
+    DEBUG(dbgs() << "          to: " << *Store << "\n");
+    return true;
+  }
+
+  bool visitStoreInst(StoreInst &SI) {
+    DEBUG(dbgs() << "    original: " << SI << "\n");
+    Value *OldOp = SI.getOperand(1);
+    assert(OldOp == OldPtr);
+    IRBuilder<> IRB(&SI);
+
+    Value *V = SI.getValueOperand();
+
+    // Strip all inbounds GEPs and pointer casts to try to dig out any root
+    // alloca that should be re-examined after promoting this alloca.
+    if (V->getType()->isPointerTy())
+      if (AllocaInst *AI = dyn_cast<AllocaInst>(V->stripInBoundsOffsets()))
+        Pass.PostPromotionWorklist.insert(AI);
+
+    uint64_t Size = EndOffset - BeginOffset;
+    if (Size < TD.getTypeStoreSize(V->getType())) {
+      assert(!SI.isVolatile());
+      assert(V->getType()->isIntegerTy() &&
+             "Only integer type loads and stores are split");
+      assert(V->getType()->getIntegerBitWidth() ==
+             TD.getTypeStoreSizeInBits(V->getType()) &&
+             "Non-byte-multiple bit width");
+      assert(V->getType()->getIntegerBitWidth() ==
+             TD.getTypeSizeInBits(OldAI.getAllocatedType()) &&
+             "Only alloca-wide stores can be split and recomposed");
+      IntegerType *NarrowTy = Type::getIntNTy(SI.getContext(), Size * 8);
+      V = extractInteger(TD, IRB, V, NarrowTy, BeginOffset,
+                         getName(".extract"));
+    }
+
+    if (VecTy)
+      return rewriteVectorizedStoreInst(IRB, V, SI, OldOp);
+    if (IntTy && V->getType()->isIntegerTy())
+      return rewriteIntegerStore(IRB, V, SI);
+
+    StoreInst *NewSI;
+    if (BeginOffset == NewAllocaBeginOffset &&
+        canConvertValue(TD, V->getType(), NewAllocaTy)) {
+      V = convertValue(TD, IRB, V, NewAllocaTy);
+      NewSI = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlignment(),
+                                     SI.isVolatile());
+    } else {
+      Value *NewPtr = getAdjustedAllocaPtr(IRB, V->getType()->getPointerTo());
+      NewSI = IRB.CreateAlignedStore(V, NewPtr,
+                                     getPartitionTypeAlign(V->getType()),
+                                     SI.isVolatile());
+    }
+    (void)NewSI;
+    Pass.DeadInsts.insert(&SI);
+    deleteIfTriviallyDead(OldOp);
+
+    DEBUG(dbgs() << "          to: " << *NewSI << "\n");
+    return NewSI->getPointerOperand() == &NewAI && !SI.isVolatile();
+  }
+
+  bool visitMemSetInst(MemSetInst &II) {
+    DEBUG(dbgs() << "    original: " << II << "\n");
+    IRBuilder<> IRB(&II);
+    assert(II.getRawDest() == OldPtr);
+
+    // If the memset has a variable size, it cannot be split, just adjust the
+    // pointer to the new alloca.
+    if (!isa<Constant>(II.getLength())) {
+      II.setDest(getAdjustedAllocaPtr(IRB, II.getRawDest()->getType()));
+      Type *CstTy = II.getAlignmentCst()->getType();
+      II.setAlignment(ConstantInt::get(CstTy, getPartitionAlign()));
+
+      deleteIfTriviallyDead(OldPtr);
+      return false;
+    }
+
+    // Record this instruction for deletion.
+    Pass.DeadInsts.insert(&II);
+
+    Type *AllocaTy = NewAI.getAllocatedType();
+    Type *ScalarTy = AllocaTy->getScalarType();
+
+    // If this doesn't map cleanly onto the alloca type, and that type isn't
+    // a single value type, just emit a memset.
+    if (!VecTy && !IntTy &&
+        (BeginOffset != NewAllocaBeginOffset ||
+         EndOffset != NewAllocaEndOffset ||
+         !AllocaTy->isSingleValueType() ||
+         !TD.isLegalInteger(TD.getTypeSizeInBits(ScalarTy)))) {
+      Type *SizeTy = II.getLength()->getType();
+      Constant *Size = ConstantInt::get(SizeTy, EndOffset - BeginOffset);
+      CallInst *New
+        = IRB.CreateMemSet(getAdjustedAllocaPtr(IRB,
+                                                II.getRawDest()->getType()),
+                           II.getValue(), Size, getPartitionAlign(),
+                           II.isVolatile());
+      (void)New;
+      DEBUG(dbgs() << "          to: " << *New << "\n");
+      return false;
+    }
+
+    // If we can represent this as a simple value, we have to build the actual
+    // value to store, which requires expanding the byte present in memset to
+    // a sensible representation for the alloca type. This is essentially
+    // splatting the byte to a sufficiently wide integer, bitcasting to the
+    // desired scalar type, and splatting it across any desired vector type.
+    uint64_t Size = EndOffset - BeginOffset;
+    Value *V = II.getValue();
+    IntegerType *VTy = cast<IntegerType>(V->getType());
+    Type *SplatIntTy = Type::getIntNTy(VTy->getContext(), Size*8);
+    if (Size*8 > VTy->getBitWidth())
+      V = IRB.CreateMul(IRB.CreateZExt(V, SplatIntTy, getName(".zext")),
+                        ConstantExpr::getUDiv(
+                          Constant::getAllOnesValue(SplatIntTy),
+                          ConstantExpr::getZExt(
+                            Constant::getAllOnesValue(V->getType()),
+                            SplatIntTy)),
+                        getName(".isplat"));
+
+    // If this is an element-wide memset of a vectorizable alloca, insert it.
+    if (VecTy && (BeginOffset > NewAllocaBeginOffset ||
+                  EndOffset < NewAllocaEndOffset)) {
+      if (V->getType() != ScalarTy)
+        V = convertValue(TD, IRB, V, ScalarTy);
+      StoreInst *Store = IRB.CreateAlignedStore(
+        IRB.CreateInsertElement(IRB.CreateAlignedLoad(&NewAI,
+                                                      NewAI.getAlignment(),
+                                                      getName(".load")),
+                                V, getIndex(IRB, BeginOffset),
+                                getName(".insert")),
+        &NewAI, NewAI.getAlignment());
+      (void)Store;
+      DEBUG(dbgs() << "          to: " << *Store << "\n");
+      return true;
+    }
+
+    // If this is a memset on an alloca where we can widen stores, insert the
+    // set integer.
+    if (IntTy && (BeginOffset > NewAllocaBeginOffset ||
+                  EndOffset < NewAllocaEndOffset)) {
+      assert(!II.isVolatile());
+      Value *Old = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(),
+                                         getName(".oldload"));
+      Old = convertValue(TD, IRB, Old, IntTy);
+      assert(BeginOffset >= NewAllocaBeginOffset && "Out of bounds offset");
+      uint64_t Offset = BeginOffset - NewAllocaBeginOffset;
+      V = insertInteger(TD, IRB, Old, V, Offset, getName(".insert"));
+    }
+
+    if (V->getType() != AllocaTy)
+      V = convertValue(TD, IRB, V, AllocaTy);
+
+    Value *New = IRB.CreateAlignedStore(V, &NewAI, NewAI.getAlignment(),
+                                        II.isVolatile());
+    (void)New;
+    DEBUG(dbgs() << "          to: " << *New << "\n");
+    return !II.isVolatile();
+  }
+
+  bool visitMemTransferInst(MemTransferInst &II) {
+    // Rewriting of memory transfer instructions can be a bit tricky. We break
+    // them into two categories: split intrinsics and unsplit intrinsics.
+
+    DEBUG(dbgs() << "    original: " << II << "\n");
+    IRBuilder<> IRB(&II);
+
+    assert(II.getRawSource() == OldPtr || II.getRawDest() == OldPtr);
+    bool IsDest = II.getRawDest() == OldPtr;
+
+    const AllocaPartitioning::MemTransferOffsets &MTO
+      = P.getMemTransferOffsets(II);
+
+    // Compute the relative offset within the transfer.
+    unsigned IntPtrWidth = TD.getPointerSizeInBits();
+    APInt RelOffset(IntPtrWidth, BeginOffset - (IsDest ? MTO.DestBegin
+                                                       : MTO.SourceBegin));
+
+    unsigned Align = II.getAlignment();
+    if (Align > 1)
+      Align = MinAlign(RelOffset.zextOrTrunc(64).getZExtValue(),
+                       MinAlign(II.getAlignment(), getPartitionAlign()));
+
+    // For unsplit intrinsics, we simply modify the source and destination
+    // pointers in place. This isn't just an optimization, it is a matter of
+    // correctness. With unsplit intrinsics we may be dealing with transfers
+    // within a single alloca before SROA ran, or with transfers that have
+    // a variable length. We may also be dealing with memmove instead of
+    // memcpy, and so simply updating the pointers is the necessary for us to
+    // update both source and dest of a single call.
+    if (!MTO.IsSplittable) {
+      Value *OldOp = IsDest ? II.getRawDest() : II.getRawSource();
+      if (IsDest)
+        II.setDest(getAdjustedAllocaPtr(IRB, II.getRawDest()->getType()));
+      else
+        II.setSource(getAdjustedAllocaPtr(IRB, II.getRawSource()->getType()));
+
+      Type *CstTy = II.getAlignmentCst()->getType();
+      II.setAlignment(ConstantInt::get(CstTy, Align));
+
+      DEBUG(dbgs() << "          to: " << II << "\n");
+      deleteIfTriviallyDead(OldOp);
+      return false;
+    }
+    // For split transfer intrinsics we have an incredibly useful assurance:
+    // the source and destination do not reside within the same alloca, and at
+    // least one of them does not escape. This means that we can replace
+    // memmove with memcpy, and we don't need to worry about all manner of
+    // downsides to splitting and transforming the operations.
+
+    // If this doesn't map cleanly onto the alloca type, and that type isn't
+    // a single value type, just emit a memcpy.
+    bool EmitMemCpy
+      = !VecTy && !IntTy && (BeginOffset != NewAllocaBeginOffset ||
+                             EndOffset != NewAllocaEndOffset ||
+                             !NewAI.getAllocatedType()->isSingleValueType());
+
+    // If we're just going to emit a memcpy, the alloca hasn't changed, and the
+    // size hasn't been shrunk based on analysis of the viable range, this is
+    // a no-op.
+    if (EmitMemCpy && &OldAI == &NewAI) {
+      uint64_t OrigBegin = IsDest ? MTO.DestBegin : MTO.SourceBegin;
+      uint64_t OrigEnd = IsDest ? MTO.DestEnd : MTO.SourceEnd;
+      // Ensure the start lines up.
+      assert(BeginOffset == OrigBegin);
+      (void)OrigBegin;
+
+      // Rewrite the size as needed.
+      if (EndOffset != OrigEnd)
+        II.setLength(ConstantInt::get(II.getLength()->getType(),
+                                      EndOffset - BeginOffset));
+      return false;
+    }
+    // Record this instruction for deletion.
+    Pass.DeadInsts.insert(&II);
+
+    bool IsWholeAlloca = BeginOffset == NewAllocaBeginOffset &&
+                         EndOffset == NewAllocaEndOffset;
+    bool IsVectorElement = VecTy && !IsWholeAlloca;
+    uint64_t Size = EndOffset - BeginOffset;
+    IntegerType *SubIntTy
+      = IntTy ? Type::getIntNTy(IntTy->getContext(), Size*8) : 0;
+
+    Type *OtherPtrTy = IsDest ? II.getRawSource()->getType()
+                              : II.getRawDest()->getType();
+    if (!EmitMemCpy) {
+      if (IsVectorElement)
+        OtherPtrTy = VecTy->getElementType()->getPointerTo();
+      else if (IntTy && !IsWholeAlloca)
+        OtherPtrTy = SubIntTy->getPointerTo();
+      else
+        OtherPtrTy = NewAI.getType();
+    }
+
+    // Compute the other pointer, folding as much as possible to produce
+    // a single, simple GEP in most cases.
+    Value *OtherPtr = IsDest ? II.getRawSource() : II.getRawDest();
+    OtherPtr = getAdjustedPtr(IRB, TD, OtherPtr, RelOffset, OtherPtrTy,
+                              getName("." + OtherPtr->getName()));
+
+    // Strip all inbounds GEPs and pointer casts to try to dig out any root
+    // alloca that should be re-examined after rewriting this instruction.
+    if (AllocaInst *AI
+          = dyn_cast<AllocaInst>(OtherPtr->stripInBoundsOffsets()))
+      Pass.Worklist.insert(AI);
+
+    if (EmitMemCpy) {
+      Value *OurPtr
+        = getAdjustedAllocaPtr(IRB, IsDest ? II.getRawDest()->getType()
+                                           : II.getRawSource()->getType());
+      Type *SizeTy = II.getLength()->getType();
+      Constant *Size = ConstantInt::get(SizeTy, EndOffset - BeginOffset);
+
+      CallInst *New = IRB.CreateMemCpy(IsDest ? OurPtr : OtherPtr,
+                                       IsDest ? OtherPtr : OurPtr,
+                                       Size, Align, II.isVolatile());
+      (void)New;
+      DEBUG(dbgs() << "          to: " << *New << "\n");
+      return false;
+    }
+
+    // Note that we clamp the alignment to 1 here as a 0 alignment for a memcpy
+    // is equivalent to 1, but that isn't true if we end up rewriting this as
+    // a load or store.
+    if (!Align)
+      Align = 1;
+
+    Value *SrcPtr = OtherPtr;
+    Value *DstPtr = &NewAI;
+    if (!IsDest)
+      std::swap(SrcPtr, DstPtr);
+
+    Value *Src;
+    if (IsVectorElement && !IsDest) {
+      // We have to extract rather than load.
+      Src = IRB.CreateExtractElement(
+        IRB.CreateAlignedLoad(SrcPtr, Align, getName(".copyload")),
+        getIndex(IRB, BeginOffset),
+        getName(".copyextract"));
+    } else if (IntTy && !IsWholeAlloca && !IsDest) {
+      Src = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(),
+                                  getName(".load"));
+      Src = convertValue(TD, IRB, Src, IntTy);
+      assert(BeginOffset >= NewAllocaBeginOffset && "Out of bounds offset");
+      uint64_t Offset = BeginOffset - NewAllocaBeginOffset;
+      Src = extractInteger(TD, IRB, Src, SubIntTy, Offset, getName(".extract"));
+    } else {
+      Src = IRB.CreateAlignedLoad(SrcPtr, Align, II.isVolatile(),
+                                  getName(".copyload"));
+    }
+
+    if (IntTy && !IsWholeAlloca && IsDest) {
+      Value *Old = IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(),
+                                         getName(".oldload"));
+      Old = convertValue(TD, IRB, Old, IntTy);
+      assert(BeginOffset >= NewAllocaBeginOffset && "Out of bounds offset");
+      uint64_t Offset = BeginOffset - NewAllocaBeginOffset;
+      Src = insertInteger(TD, IRB, Old, Src, Offset, getName(".insert"));
+      Src = convertValue(TD, IRB, Src, NewAllocaTy);
+    }
+
+    if (IsVectorElement && IsDest) {
+      // We have to insert into a loaded copy before storing.
+      Src = IRB.CreateInsertElement(
+        IRB.CreateAlignedLoad(&NewAI, NewAI.getAlignment(), getName(".load")),
+        Src, getIndex(IRB, BeginOffset),
+        getName(".insert"));
+    }
+
+    StoreInst *Store = cast<StoreInst>(
+      IRB.CreateAlignedStore(Src, DstPtr, Align, II.isVolatile()));
+    (void)Store;
+    DEBUG(dbgs() << "          to: " << *Store << "\n");
+    return !II.isVolatile();
+  }
+
+  bool visitIntrinsicInst(IntrinsicInst &II) {
+    assert(II.getIntrinsicID() == Intrinsic::lifetime_start ||
+           II.getIntrinsicID() == Intrinsic::lifetime_end);
+    DEBUG(dbgs() << "    original: " << II << "\n");
+    IRBuilder<> IRB(&II);
+    assert(II.getArgOperand(1) == OldPtr);
+
+    // Record this instruction for deletion.
+    Pass.DeadInsts.insert(&II);
+
+    ConstantInt *Size
+      = ConstantInt::get(cast<IntegerType>(II.getArgOperand(0)->getType()),
+                         EndOffset - BeginOffset);
+    Value *Ptr = getAdjustedAllocaPtr(IRB, II.getArgOperand(1)->getType());
+    Value *New;
+    if (II.getIntrinsicID() == Intrinsic::lifetime_start)
+      New = IRB.CreateLifetimeStart(Ptr, Size);
+    else
+      New = IRB.CreateLifetimeEnd(Ptr, Size);
+
+    DEBUG(dbgs() << "          to: " << *New << "\n");
+    return true;
+  }
+
+  bool visitPHINode(PHINode &PN) {
+    DEBUG(dbgs() << "    original: " << PN << "\n");
+
+    // We would like to compute a new pointer in only one place, but have it be
+    // as local as possible to the PHI. To do that, we re-use the location of
+    // the old pointer, which necessarily must be in the right position to
+    // dominate the PHI.
+    IRBuilder<> PtrBuilder(cast<Instruction>(OldPtr));
+
+    Value *NewPtr = getAdjustedAllocaPtr(PtrBuilder, OldPtr->getType());
+    // Replace the operands which were using the old pointer.
+    std::replace(PN.op_begin(), PN.op_end(), cast<Value>(OldPtr), NewPtr);
+
+    DEBUG(dbgs() << "          to: " << PN << "\n");
+    deleteIfTriviallyDead(OldPtr);
+    return false;
+  }
+
+  bool visitSelectInst(SelectInst &SI) {
+    DEBUG(dbgs() << "    original: " << SI << "\n");
+    IRBuilder<> IRB(&SI);
+
+    // Find the operand we need to rewrite here.
+    bool IsTrueVal = SI.getTrueValue() == OldPtr;
+    if (IsTrueVal)
+      assert(SI.getFalseValue() != OldPtr && "Pointer is both operands!");
+    else
+      assert(SI.getFalseValue() == OldPtr && "Pointer isn't an operand!");
+
+    Value *NewPtr = getAdjustedAllocaPtr(IRB, OldPtr->getType());
+    SI.setOperand(IsTrueVal ? 1 : 2, NewPtr);
+    DEBUG(dbgs() << "          to: " << SI << "\n");
+    deleteIfTriviallyDead(OldPtr);
+    return false;
+  }
+
+};
+}
+
+namespace {
+/// \brief Visitor to rewrite aggregate loads and stores as scalar.
+///
+/// This pass aggressively rewrites all aggregate loads and stores on
+/// a particular pointer (or any pointer derived from it which we can identify)
+/// with scalar loads and stores.
+class AggLoadStoreRewriter : public InstVisitor<AggLoadStoreRewriter, bool> {
+  // Befriend the base class so it can delegate to private visit methods.
+  friend class llvm::InstVisitor<AggLoadStoreRewriter, bool>;
+
+  const DataLayout &TD;
+
+  /// Queue of pointer uses to analyze and potentially rewrite.
+  SmallVector<Use *, 8> Queue;
+
+  /// Set to prevent us from cycling with phi nodes and loops.
+  SmallPtrSet<User *, 8> Visited;
+
+  /// The current pointer use being rewritten. This is used to dig up the used
+  /// value (as opposed to the user).
+  Use *U;
+
+public:
+  AggLoadStoreRewriter(const DataLayout &TD) : TD(TD) {}
+
+  /// Rewrite loads and stores through a pointer and all pointers derived from
+  /// it.
+  bool rewrite(Instruction &I) {
+    DEBUG(dbgs() << "  Rewriting FCA loads and stores...\n");
+    enqueueUsers(I);
+    bool Changed = false;
+    while (!Queue.empty()) {
+      U = Queue.pop_back_val();
+      Changed |= visit(cast<Instruction>(U->getUser()));
+    }
+    return Changed;
+  }
+
+private:
+  /// Enqueue all the users of the given instruction for further processing.
+  /// This uses a set to de-duplicate users.
+  void enqueueUsers(Instruction &I) {
+    for (Value::use_iterator UI = I.use_begin(), UE = I.use_end(); UI != UE;
+         ++UI)
+      if (Visited.insert(*UI))
+        Queue.push_back(&UI.getUse());
+  }
+
+  // Conservative default is to not rewrite anything.
+  bool visitInstruction(Instruction &I) { return false; }
+
+  /// \brief Generic recursive split emission class.
+  template <typename Derived>
+  class OpSplitter {
+  protected:
+    /// The builder used to form new instructions.
+    IRBuilder<> IRB;
+    /// The indices which to be used with insert- or extractvalue to select the
+    /// appropriate value within the aggregate.
+    SmallVector<unsigned, 4> Indices;
+    /// The indices to a GEP instruction which will move Ptr to the correct slot
+    /// within the aggregate.
+    SmallVector<Value *, 4> GEPIndices;
+    /// The base pointer of the original op, used as a base for GEPing the
+    /// split operations.
+    Value *Ptr;
+
+    /// Initialize the splitter with an insertion point, Ptr and start with a
+    /// single zero GEP index.
+    OpSplitter(Instruction *InsertionPoint, Value *Ptr)
+      : IRB(InsertionPoint), GEPIndices(1, IRB.getInt32(0)), Ptr(Ptr) {}
+
+  public:
+    /// \brief Generic recursive split emission routine.
+    ///
+    /// This method recursively splits an aggregate op (load or store) into
+    /// scalar or vector ops. It splits recursively until it hits a single value
+    /// and emits that single value operation via the template argument.
+    ///
+    /// The logic of this routine relies on GEPs and insertvalue and
+    /// extractvalue all operating with the same fundamental index list, merely
+    /// formatted differently (GEPs need actual values).
+    ///
+    /// \param Ty  The type being split recursively into smaller ops.
+    /// \param Agg The aggregate value being built up or stored, depending on
+    /// whether this is splitting a load or a store respectively.
+    void emitSplitOps(Type *Ty, Value *&Agg, const Twine &Name) {
+      if (Ty->isSingleValueType())
+        return static_cast<Derived *>(this)->emitFunc(Ty, Agg, Name);
+
+      if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
+        unsigned OldSize = Indices.size();
+        (void)OldSize;
+        for (unsigned Idx = 0, Size = ATy->getNumElements(); Idx != Size;
+             ++Idx) {
+          assert(Indices.size() == OldSize && "Did not return to the old size");
+          Indices.push_back(Idx);
+          GEPIndices.push_back(IRB.getInt32(Idx));
+          emitSplitOps(ATy->getElementType(), Agg, Name + "." + Twine(Idx));
+          GEPIndices.pop_back();
+          Indices.pop_back();
+        }
+        return;
+      }
+
+      if (StructType *STy = dyn_cast<StructType>(Ty)) {
+        unsigned OldSize = Indices.size();
+        (void)OldSize;
+        for (unsigned Idx = 0, Size = STy->getNumElements(); Idx != Size;
+             ++Idx) {
+          assert(Indices.size() == OldSize && "Did not return to the old size");
+          Indices.push_back(Idx);
+          GEPIndices.push_back(IRB.getInt32(Idx));
+          emitSplitOps(STy->getElementType(Idx), Agg, Name + "." + Twine(Idx));
+          GEPIndices.pop_back();
+          Indices.pop_back();
+        }
+        return;
+      }
+
+      llvm_unreachable("Only arrays and structs are aggregate loadable types");
+    }
+  };
+
+  struct LoadOpSplitter : public OpSplitter<LoadOpSplitter> {
+    LoadOpSplitter(Instruction *InsertionPoint, Value *Ptr)
+      : OpSplitter<LoadOpSplitter>(InsertionPoint, Ptr) {}
+
+    /// Emit a leaf load of a single value. This is called at the leaves of the
+    /// recursive emission to actually load values.
+    void emitFunc(Type *Ty, Value *&Agg, const Twine &Name) {
+      assert(Ty->isSingleValueType());
+      // Load the single value and insert it using the indices.
+      Value *Load = IRB.CreateLoad(IRB.CreateInBoundsGEP(Ptr, GEPIndices,
+                                                         Name + ".gep"),
+                                   Name + ".load");
+      Agg = IRB.CreateInsertValue(Agg, Load, Indices, Name + ".insert");
+      DEBUG(dbgs() << "          to: " << *Load << "\n");
+    }
+  };
+
+  bool visitLoadInst(LoadInst &LI) {
+    assert(LI.getPointerOperand() == *U);
+    if (!LI.isSimple() || LI.getType()->isSingleValueType())
+      return false;
+
+    // We have an aggregate being loaded, split it apart.
+    DEBUG(dbgs() << "    original: " << LI << "\n");
+    LoadOpSplitter Splitter(&LI, *U);
+    Value *V = UndefValue::get(LI.getType());
+    Splitter.emitSplitOps(LI.getType(), V, LI.getName() + ".fca");
+    LI.replaceAllUsesWith(V);
+    LI.eraseFromParent();
+    return true;
+  }
+
+  struct StoreOpSplitter : public OpSplitter<StoreOpSplitter> {
+    StoreOpSplitter(Instruction *InsertionPoint, Value *Ptr)
+      : OpSplitter<StoreOpSplitter>(InsertionPoint, Ptr) {}
+
+    /// Emit a leaf store of a single value. This is called at the leaves of the
+    /// recursive emission to actually produce stores.
+    void emitFunc(Type *Ty, Value *&Agg, const Twine &Name) {
+      assert(Ty->isSingleValueType());
+      // Extract the single value and store it using the indices.
+      Value *Store = IRB.CreateStore(
+        IRB.CreateExtractValue(Agg, Indices, Name + ".extract"),
+        IRB.CreateInBoundsGEP(Ptr, GEPIndices, Name + ".gep"));
+      (void)Store;
+      DEBUG(dbgs() << "          to: " << *Store << "\n");
+    }
+  };
+
+  bool visitStoreInst(StoreInst &SI) {
+    if (!SI.isSimple() || SI.getPointerOperand() != *U)
+      return false;
+    Value *V = SI.getValueOperand();
+    if (V->getType()->isSingleValueType())
+      return false;
+
+    // We have an aggregate being stored, split it apart.
+    DEBUG(dbgs() << "    original: " << SI << "\n");
+    StoreOpSplitter Splitter(&SI, *U);
+    Splitter.emitSplitOps(V->getType(), V, V->getName() + ".fca");
+    SI.eraseFromParent();
+    return true;
+  }
+
+  bool visitBitCastInst(BitCastInst &BC) {
+    enqueueUsers(BC);
+    return false;
+  }
+
+  bool visitGetElementPtrInst(GetElementPtrInst &GEPI) {
+    enqueueUsers(GEPI);
+    return false;
+  }
+
+  bool visitPHINode(PHINode &PN) {
+    enqueueUsers(PN);
+    return false;
+  }
+
+  bool visitSelectInst(SelectInst &SI) {
+    enqueueUsers(SI);
+    return false;
+  }
+};
+}
+
+/// \brief Strip aggregate type wrapping.
+///
+/// This removes no-op aggregate types wrapping an underlying type. It will
+/// strip as many layers of types as it can without changing either the type
+/// size or the allocated size.
+static Type *stripAggregateTypeWrapping(const DataLayout &DL, Type *Ty) {
+  if (Ty->isSingleValueType())
+    return Ty;
+
+  uint64_t AllocSize = DL.getTypeAllocSize(Ty);
+  uint64_t TypeSize = DL.getTypeSizeInBits(Ty);
+
+  Type *InnerTy;
+  if (ArrayType *ArrTy = dyn_cast<ArrayType>(Ty)) {
+    InnerTy = ArrTy->getElementType();
+  } else if (StructType *STy = dyn_cast<StructType>(Ty)) {
+    const StructLayout *SL = DL.getStructLayout(STy);
+    unsigned Index = SL->getElementContainingOffset(0);
+    InnerTy = STy->getElementType(Index);
+  } else {
+    return Ty;
+  }
+
+  if (AllocSize > DL.getTypeAllocSize(InnerTy) ||
+      TypeSize > DL.getTypeSizeInBits(InnerTy))
+    return Ty;
+
+  return stripAggregateTypeWrapping(DL, InnerTy);
+}
+
+/// \brief Try to find a partition of the aggregate type passed in for a given
+/// offset and size.
+///
+/// This recurses through the aggregate type and tries to compute a subtype
+/// based on the offset and size. When the offset and size span a sub-section
+/// of an array, it will even compute a new array type for that sub-section,
+/// and the same for structs.
+///
+/// Note that this routine is very strict and tries to find a partition of the
+/// type which produces the *exact* right offset and size. It is not forgiving
+/// when the size or offset cause either end of type-based partition to be off.
+/// Also, this is a best-effort routine. It is reasonable to give up and not
+/// return a type if necessary.
+static Type *getTypePartition(const DataLayout &TD, Type *Ty,
+                              uint64_t Offset, uint64_t Size) {
+  if (Offset == 0 && TD.getTypeAllocSize(Ty) == Size)
+    return stripAggregateTypeWrapping(TD, Ty);
+  if (Offset > TD.getTypeAllocSize(Ty) ||
+      (TD.getTypeAllocSize(Ty) - Offset) < Size)
+    return 0;
+
+  if (SequentialType *SeqTy = dyn_cast<SequentialType>(Ty)) {
+    // We can't partition pointers...
+    if (SeqTy->isPointerTy())
+      return 0;
+
+    Type *ElementTy = SeqTy->getElementType();
+    uint64_t ElementSize = TD.getTypeAllocSize(ElementTy);
+    uint64_t NumSkippedElements = Offset / ElementSize;
+    if (ArrayType *ArrTy = dyn_cast<ArrayType>(SeqTy))
+      if (NumSkippedElements >= ArrTy->getNumElements())
+        return 0;
+    if (VectorType *VecTy = dyn_cast<VectorType>(SeqTy))
+      if (NumSkippedElements >= VecTy->getNumElements())
+        return 0;
+    Offset -= NumSkippedElements * ElementSize;
+
+    // First check if we need to recurse.
+    if (Offset > 0 || Size < ElementSize) {
+      // Bail if the partition ends in a different array element.
+      if ((Offset + Size) > ElementSize)
+        return 0;
+      // Recurse through the element type trying to peel off offset bytes.
+      return getTypePartition(TD, ElementTy, Offset, Size);
+    }
+    assert(Offset == 0);
+
+    if (Size == ElementSize)
+      return stripAggregateTypeWrapping(TD, ElementTy);
+    assert(Size > ElementSize);
+    uint64_t NumElements = Size / ElementSize;
+    if (NumElements * ElementSize != Size)
+      return 0;
+    return ArrayType::get(ElementTy, NumElements);
+  }
+
+  StructType *STy = dyn_cast<StructType>(Ty);
+  if (!STy)
+    return 0;
+
+  const StructLayout *SL = TD.getStructLayout(STy);
+  if (Offset >= SL->getSizeInBytes())
+    return 0;
+  uint64_t EndOffset = Offset + Size;
+  if (EndOffset > SL->getSizeInBytes())
+    return 0;
+
+  unsigned Index = SL->getElementContainingOffset(Offset);
+  Offset -= SL->getElementOffset(Index);
+
+  Type *ElementTy = STy->getElementType(Index);
+  uint64_t ElementSize = TD.getTypeAllocSize(ElementTy);
+  if (Offset >= ElementSize)
+    return 0; // The offset points into alignment padding.
+
+  // See if any partition must be contained by the element.
+  if (Offset > 0 || Size < ElementSize) {
+    if ((Offset + Size) > ElementSize)
+      return 0;
+    return getTypePartition(TD, ElementTy, Offset, Size);
+  }
+  assert(Offset == 0);
+
+  if (Size == ElementSize)
+    return stripAggregateTypeWrapping(TD, ElementTy);
+
+  StructType::element_iterator EI = STy->element_begin() + Index,
+                               EE = STy->element_end();
+  if (EndOffset < SL->getSizeInBytes()) {
+    unsigned EndIndex = SL->getElementContainingOffset(EndOffset);
+    if (Index == EndIndex)
+      return 0; // Within a single element and its padding.
+
+    // Don't try to form "natural" types if the elements don't line up with the
+    // expected size.
+    // FIXME: We could potentially recurse down through the last element in the
+    // sub-struct to find a natural end point.
+    if (SL->getElementOffset(EndIndex) != EndOffset)
+      return 0;
+
+    assert(Index < EndIndex);
+    EE = STy->element_begin() + EndIndex;
+  }
+
+  // Try to build up a sub-structure.
+  StructType *SubTy = StructType::get(STy->getContext(), makeArrayRef(EI, EE),
+                                      STy->isPacked());
+  const StructLayout *SubSL = TD.getStructLayout(SubTy);
+  if (Size != SubSL->getSizeInBytes())
+    return 0; // The sub-struct doesn't have quite the size needed.
+
+  return SubTy;
+}
+
+/// \brief Rewrite an alloca partition's users.
+///
+/// This routine drives both of the rewriting goals of the SROA pass. It tries
+/// to rewrite uses of an alloca partition to be conducive for SSA value
+/// promotion. If the partition needs a new, more refined alloca, this will
+/// build that new alloca, preserving as much type information as possible, and
+/// rewrite the uses of the old alloca to point at the new one and have the
+/// appropriate new offsets. It also evaluates how successful the rewrite was
+/// at enabling promotion and if it was successful queues the alloca to be
+/// promoted.
+bool SROA::rewriteAllocaPartition(AllocaInst &AI,
+                                  AllocaPartitioning &P,
+                                  AllocaPartitioning::iterator PI) {
+  uint64_t AllocaSize = PI->EndOffset - PI->BeginOffset;
+  bool IsLive = false;
+  for (AllocaPartitioning::use_iterator UI = P.use_begin(PI),
+                                        UE = P.use_end(PI);
+       UI != UE && !IsLive; ++UI)
+    if (UI->U)
+      IsLive = true;
+  if (!IsLive)
+    return false; // No live uses left of this partition.
+
+  DEBUG(dbgs() << "Speculating PHIs and selects in partition "
+               << "[" << PI->BeginOffset << "," << PI->EndOffset << ")\n");
+
+  PHIOrSelectSpeculator Speculator(*TD, P, *this);
+  DEBUG(dbgs() << "  speculating ");
+  DEBUG(P.print(dbgs(), PI, ""));
+  Speculator.visitUsers(PI);
+
+  // Try to compute a friendly type for this partition of the alloca. This
+  // won't always succeed, in which case we fall back to a legal integer type
+  // or an i8 array of an appropriate size.
+  Type *AllocaTy = 0;
+  if (Type *PartitionTy = P.getCommonType(PI))
+    if (TD->getTypeAllocSize(PartitionTy) >= AllocaSize)
+      AllocaTy = PartitionTy;
+  if (!AllocaTy)
+    if (Type *PartitionTy = getTypePartition(*TD, AI.getAllocatedType(),
+                                             PI->BeginOffset, AllocaSize))
+      AllocaTy = PartitionTy;
+  if ((!AllocaTy ||
+       (AllocaTy->isArrayTy() &&
+        AllocaTy->getArrayElementType()->isIntegerTy())) &&
+      TD->isLegalInteger(AllocaSize * 8))
+    AllocaTy = Type::getIntNTy(*C, AllocaSize * 8);
+  if (!AllocaTy)
+    AllocaTy = ArrayType::get(Type::getInt8Ty(*C), AllocaSize);
+  assert(TD->getTypeAllocSize(AllocaTy) >= AllocaSize);
+
+  // Check for the case where we're going to rewrite to a new alloca of the
+  // exact same type as the original, and with the same access offsets. In that
+  // case, re-use the existing alloca, but still run through the rewriter to
+  // performe phi and select speculation.
+  AllocaInst *NewAI;
+  if (AllocaTy == AI.getAllocatedType()) {
+    assert(PI->BeginOffset == 0 &&
+           "Non-zero begin offset but same alloca type");
+    assert(PI == P.begin() && "Begin offset is zero on later partition");
+    NewAI = &AI;
+  } else {
+    unsigned Alignment = AI.getAlignment();
+    if (!Alignment) {
+      // The minimum alignment which users can rely on when the explicit
+      // alignment is omitted or zero is that required by the ABI for this
+      // type.
+      Alignment = TD->getABITypeAlignment(AI.getAllocatedType());
+    }
+    Alignment = MinAlign(Alignment, PI->BeginOffset);
+    // If we will get at least this much alignment from the type alone, leave
+    // the alloca's alignment unconstrained.
+    if (Alignment <= TD->getABITypeAlignment(AllocaTy))
+      Alignment = 0;
+    NewAI = new AllocaInst(AllocaTy, 0, Alignment,
+                           AI.getName() + ".sroa." + Twine(PI - P.begin()),
+                           &AI);
+    ++NumNewAllocas;
+  }
+
+  DEBUG(dbgs() << "Rewriting alloca partition "
+               << "[" << PI->BeginOffset << "," << PI->EndOffset << ") to: "
+               << *NewAI << "\n");
+
+  // Track the high watermark of the post-promotion worklist. We will reset it
+  // to this point if the alloca is not in fact scheduled for promotion.
+  unsigned PPWOldSize = PostPromotionWorklist.size();
+
+  AllocaPartitionRewriter Rewriter(*TD, P, PI, *this, AI, *NewAI,
+                                   PI->BeginOffset, PI->EndOffset);
+  DEBUG(dbgs() << "  rewriting ");
+  DEBUG(P.print(dbgs(), PI, ""));
+  bool Promotable = Rewriter.visitUsers(P.use_begin(PI), P.use_end(PI));
+  if (Promotable) {
+    DEBUG(dbgs() << "  and queuing for promotion\n");
+    PromotableAllocas.push_back(NewAI);
+  } else if (NewAI != &AI) {
+    // If we can't promote the alloca, iterate on it to check for new
+    // refinements exposed by splitting the current alloca. Don't iterate on an
+    // alloca which didn't actually change and didn't get promoted.
+    Worklist.insert(NewAI);
+  }
+
+  // Drop any post-promotion work items if promotion didn't happen.
+  if (!Promotable)
+    while (PostPromotionWorklist.size() > PPWOldSize)
+      PostPromotionWorklist.pop_back();
+
+  return true;
+}
+
+/// \brief Walks the partitioning of an alloca rewriting uses of each partition.
+bool SROA::splitAlloca(AllocaInst &AI, AllocaPartitioning &P) {
+  bool Changed = false;
+  for (AllocaPartitioning::iterator PI = P.begin(), PE = P.end(); PI != PE;
+       ++PI)
+    Changed |= rewriteAllocaPartition(AI, P, PI);
+
+  return Changed;
+}
+
+/// \brief Analyze an alloca for SROA.
+///
+/// This analyzes the alloca to ensure we can reason about it, builds
+/// a partitioning of the alloca, and then hands it off to be split and
+/// rewritten as needed.
+bool SROA::runOnAlloca(AllocaInst &AI) {
+  DEBUG(dbgs() << "SROA alloca: " << AI << "\n");
+  ++NumAllocasAnalyzed;
+
+  // Special case dead allocas, as they're trivial.
+  if (AI.use_empty()) {
+    AI.eraseFromParent();
+    return true;
+  }
+
+  // Skip alloca forms that this analysis can't handle.
+  if (AI.isArrayAllocation() || !AI.getAllocatedType()->isSized() ||
+      TD->getTypeAllocSize(AI.getAllocatedType()) == 0)
+    return false;
+
+  bool Changed = false;
+
+  // First, split any FCA loads and stores touching this alloca to promote
+  // better splitting and promotion opportunities.
+  AggLoadStoreRewriter AggRewriter(*TD);
+  Changed |= AggRewriter.rewrite(AI);
+
+  // Build the partition set using a recursive instruction-visiting builder.
+  AllocaPartitioning P(*TD, AI);
+  DEBUG(P.print(dbgs()));
+  if (P.isEscaped())
+    return Changed;
+
+  // Delete all the dead users of this alloca before splitting and rewriting it.
+  for (AllocaPartitioning::dead_user_iterator DI = P.dead_user_begin(),
+                                              DE = P.dead_user_end();
+       DI != DE; ++DI) {
+    Changed = true;
+    (*DI)->replaceAllUsesWith(UndefValue::get((*DI)->getType()));
+    DeadInsts.insert(*DI);
+  }
+  for (AllocaPartitioning::dead_op_iterator DO = P.dead_op_begin(),
+                                            DE = P.dead_op_end();
+       DO != DE; ++DO) {
+    Value *OldV = **DO;
+    // Clobber the use with an undef value.
+    **DO = UndefValue::get(OldV->getType());
+    if (Instruction *OldI = dyn_cast<Instruction>(OldV))
+      if (isInstructionTriviallyDead(OldI)) {
+        Changed = true;
+        DeadInsts.insert(OldI);
+      }
+  }
+
+  // No partitions to split. Leave the dead alloca for a later pass to clean up.
+  if (P.begin() == P.end())
+    return Changed;
+
+  return splitAlloca(AI, P) || Changed;
+}
+
+/// \brief Delete the dead instructions accumulated in this run.
+///
+/// Recursively deletes the dead instructions we've accumulated. This is done
+/// at the very end to maximize locality of the recursive delete and to
+/// minimize the problems of invalidated instruction pointers as such pointers
+/// are used heavily in the intermediate stages of the algorithm.
+///
+/// We also record the alloca instructions deleted here so that they aren't
+/// subsequently handed to mem2reg to promote.
+void SROA::deleteDeadInstructions(SmallPtrSet<AllocaInst*, 4> &DeletedAllocas) {
+  while (!DeadInsts.empty()) {
+    Instruction *I = DeadInsts.pop_back_val();
+    DEBUG(dbgs() << "Deleting dead instruction: " << *I << "\n");
+
+    I->replaceAllUsesWith(UndefValue::get(I->getType()));
+
+    for (User::op_iterator OI = I->op_begin(), E = I->op_end(); OI != E; ++OI)
+      if (Instruction *U = dyn_cast<Instruction>(*OI)) {
+        // Zero out the operand and see if it becomes trivially dead.
+        *OI = 0;
+        if (isInstructionTriviallyDead(U))
+          DeadInsts.insert(U);
+      }
+
+    if (AllocaInst *AI = dyn_cast<AllocaInst>(I))
+      DeletedAllocas.insert(AI);
+
+    ++NumDeleted;
+    I->eraseFromParent();
+  }
+}
+
+/// \brief Promote the allocas, using the best available technique.
+///
+/// This attempts to promote whatever allocas have been identified as viable in
+/// the PromotableAllocas list. If that list is empty, there is nothing to do.
+/// If there is a domtree available, we attempt to promote using the full power
+/// of mem2reg. Otherwise, we build and use the AllocaPromoter above which is
+/// based on the SSAUpdater utilities. This function returns whether any
+/// promotion occured.
+bool SROA::promoteAllocas(Function &F) {
+  if (PromotableAllocas.empty())
+    return false;
+
+  NumPromoted += PromotableAllocas.size();
+
+  if (DT && !ForceSSAUpdater) {
+    DEBUG(dbgs() << "Promoting allocas with mem2reg...\n");
+    PromoteMemToReg(PromotableAllocas, *DT);
+    PromotableAllocas.clear();
+    return true;
+  }
+
+  DEBUG(dbgs() << "Promoting allocas with SSAUpdater...\n");
+  SSAUpdater SSA;
+  DIBuilder DIB(*F.getParent());
+  SmallVector<Instruction*, 64> Insts;
+
+  for (unsigned Idx = 0, Size = PromotableAllocas.size(); Idx != Size; ++Idx) {
+    AllocaInst *AI = PromotableAllocas[Idx];
+    for (Value::use_iterator UI = AI->use_begin(), UE = AI->use_end();
+         UI != UE;) {
+      Instruction *I = cast<Instruction>(*UI++);
+      // FIXME: Currently the SSAUpdater infrastructure doesn't reason about
+      // lifetime intrinsics and so we strip them (and the bitcasts+GEPs
+      // leading to them) here. Eventually it should use them to optimize the
+      // scalar values produced.
+      if (isa<BitCastInst>(I) || isa<GetElementPtrInst>(I)) {
+        assert(onlyUsedByLifetimeMarkers(I) &&
+               "Found a bitcast used outside of a lifetime marker.");
+        while (!I->use_empty())
+          cast<Instruction>(*I->use_begin())->eraseFromParent();
+        I->eraseFromParent();
+        continue;
+      }
+      if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
+        assert(II->getIntrinsicID() == Intrinsic::lifetime_start ||
+               II->getIntrinsicID() == Intrinsic::lifetime_end);
+        II->eraseFromParent();
+        continue;
+      }
+
+      Insts.push_back(I);
+    }
+    AllocaPromoter(Insts, SSA, *AI, DIB).run(Insts);
+    Insts.clear();
+  }
+
+  PromotableAllocas.clear();
+  return true;
+}
+
+namespace {
+  /// \brief A predicate to test whether an alloca belongs to a set.
+  class IsAllocaInSet {
+    typedef SmallPtrSet<AllocaInst *, 4> SetType;
+    const SetType &Set;
+
+  public:
+    typedef AllocaInst *argument_type;
+
+    IsAllocaInSet(const SetType &Set) : Set(Set) {}
+    bool operator()(AllocaInst *AI) const { return Set.count(AI); }
+  };
+}
+
+bool SROA::runOnFunction(Function &F) {
+  DEBUG(dbgs() << "SROA function: " << F.getName() << "\n");
+  C = &F.getContext();
+  TD = getAnalysisIfAvailable<DataLayout>();
+  if (!TD) {
+    DEBUG(dbgs() << "  Skipping SROA -- no target data!\n");
+    return false;
+  }
+  DT = getAnalysisIfAvailable<DominatorTree>();
+
+  BasicBlock &EntryBB = F.getEntryBlock();
+  for (BasicBlock::iterator I = EntryBB.begin(), E = llvm::prior(EntryBB.end());
+       I != E; ++I)
+    if (AllocaInst *AI = dyn_cast<AllocaInst>(I))
+      Worklist.insert(AI);
+
+  bool Changed = false;
+  // A set of deleted alloca instruction pointers which should be removed from
+  // the list of promotable allocas.
+  SmallPtrSet<AllocaInst *, 4> DeletedAllocas;
+
+  do {
+    while (!Worklist.empty()) {
+      Changed |= runOnAlloca(*Worklist.pop_back_val());
+      deleteDeadInstructions(DeletedAllocas);
+
+      // Remove the deleted allocas from various lists so that we don't try to
+      // continue processing them.
+      if (!DeletedAllocas.empty()) {
+        Worklist.remove_if(IsAllocaInSet(DeletedAllocas));
+        PostPromotionWorklist.remove_if(IsAllocaInSet(DeletedAllocas));
+        PromotableAllocas.erase(std::remove_if(PromotableAllocas.begin(),
+                                               PromotableAllocas.end(),
+                                               IsAllocaInSet(DeletedAllocas)),
+                                PromotableAllocas.end());
+        DeletedAllocas.clear();
+      }
+    }
+
+    Changed |= promoteAllocas(F);
+
+    Worklist = PostPromotionWorklist;
+    PostPromotionWorklist.clear();
+  } while (!Worklist.empty());
+
+  return Changed;
+}
+
+void SROA::getAnalysisUsage(AnalysisUsage &AU) const {
+  if (RequiresDomTree)
+    AU.addRequired<DominatorTree>();
+  AU.setPreservesCFG();
+}
diff --git a/lib/Transforms/Scalar/Scalar.cpp b/lib/Transforms/Scalar/Scalar.cpp
index 48318c8..39630fd 100644
--- a/lib/Transforms/Scalar/Scalar.cpp
+++ b/lib/Transforms/Scalar/Scalar.cpp
@@ -19,7 +19,7 @@
 #include "llvm/PassManager.h"
 #include "llvm/Analysis/Passes.h"
 #include "llvm/Analysis/Verifier.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Transforms/Scalar.h"
 
 using namespace llvm;
@@ -59,6 +59,7 @@ void llvm::initializeScalarOpts(PassRegistry &Registry) {
   initializeRegToMemPass(Registry);
   initializeSCCPPass(Registry);
   initializeIPSCCPPass(Registry);
+  initializeSROAPass(Registry);
   initializeSROA_DTPass(Registry);
   initializeSROA_SSAUpPass(Registry);
   initializeCFGSimplifyPassPass(Registry);
diff --git a/lib/Transforms/Scalar/ScalarReplAggregates.cpp b/lib/Transforms/Scalar/ScalarReplAggregates.cpp
index 6637126..a46d09c 100644
--- a/lib/Transforms/Scalar/ScalarReplAggregates.cpp
+++ b/lib/Transforms/Scalar/ScalarReplAggregates.cpp
@@ -46,7 +46,7 @@
 #include "llvm/Support/GetElementPtrTypeIterator.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Transforms/Utils/Local.h"
 #include "llvm/Transforms/Utils/PromoteMemToReg.h"
 #include "llvm/Transforms/Utils/SSAUpdater.h"
@@ -56,7 +56,6 @@ STATISTIC(NumReplaced,  "Number of allocas broken up");
 STATISTIC(NumPromoted,  "Number of allocas promoted");
 STATISTIC(NumAdjusted,  "Number of scalar allocas adjusted to allow promotion");
 STATISTIC(NumConverted, "Number of aggregates converted to scalar");
-STATISTIC(NumGlobals,   "Number of allocas copied from constant global");
 
 namespace {
   struct SROA : public FunctionPass {
@@ -88,7 +87,7 @@ namespace {
 
   private:
     bool HasDomTree;
-    TargetData *TD;
+    DataLayout *TD;
 
     /// DeadInsts - Keep track of instructions we have made dead, so that
     /// we can remove them after we are done working.
@@ -183,9 +182,6 @@ namespace {
     void RewriteLoadUserOfWholeAlloca(LoadInst *LI, AllocaInst *AI,
                                       SmallVector<AllocaInst*, 32> &NewElts);
     bool ShouldAttemptScalarRepl(AllocaInst *AI);
-
-    static MemTransferInst *isOnlyCopiedFromConstantGlobal(
-        AllocaInst *AI, SmallVector<Instruction*, 4> &ToDelete);
   };
 
   // SROA_DT - SROA that uses DominatorTree.
@@ -262,7 +258,7 @@ namespace {
 class ConvertToScalarInfo {
   /// AllocaSize - The size of the alloca being considered in bytes.
   unsigned AllocaSize;
-  const TargetData &TD;
+  const DataLayout &TD;
   unsigned ScalarLoadThreshold;
 
   /// IsNotTrivial - This is set to true if there is some access to the object
@@ -305,7 +301,7 @@ class ConvertToScalarInfo {
   bool HadDynamicAccess;
 
 public:
-  explicit ConvertToScalarInfo(unsigned Size, const TargetData &td,
+  explicit ConvertToScalarInfo(unsigned Size, const DataLayout &td,
                                unsigned SLT)
     : AllocaSize(Size), TD(td), ScalarLoadThreshold(SLT), IsNotTrivial(false),
     ScalarKind(Unknown), VectorTy(0), HadNonMemTransferAccess(false),
@@ -1024,11 +1020,11 @@ ConvertScalar_InsertValue(Value *SV, Value *Old,
 
 
 bool SROA::runOnFunction(Function &F) {
-  TD = getAnalysisIfAvailable<TargetData>();
+  TD = getAnalysisIfAvailable<DataLayout>();
 
   bool Changed = performPromotion(F);
 
-  // FIXME: ScalarRepl currently depends on TargetData more than it
+  // FIXME: ScalarRepl currently depends on DataLayout more than it
   // theoretically needs to. It should be refactored in order to support
   // target-independent IR. Until this is done, just skip the actual
   // scalar-replacement portion of this pass.
@@ -1138,7 +1134,7 @@ public:
 ///
 /// We can do this to a select if its only uses are loads and if the operand to
 /// the select can be loaded unconditionally.
-static bool isSafeSelectToSpeculate(SelectInst *SI, const TargetData *TD) {
+static bool isSafeSelectToSpeculate(SelectInst *SI, const DataLayout *TD) {
   bool TDerefable = SI->getTrueValue()->isDereferenceablePointer();
   bool FDerefable = SI->getFalseValue()->isDereferenceablePointer();
 
@@ -1176,7 +1172,7 @@ static bool isSafeSelectToSpeculate(SelectInst *SI, const TargetData *TD) {
 ///
 /// We can do this to a select if its only uses are loads and if the operand to
 /// the select can be loaded unconditionally.
-static bool isSafePHIToSpeculate(PHINode *PN, const TargetData *TD) {
+static bool isSafePHIToSpeculate(PHINode *PN, const DataLayout *TD) {
   // For now, we can only do this promotion if the load is in the same block as
   // the PHI, and if there are no stores between the phi and load.
   // TODO: Allow recursive phi users.
@@ -1240,7 +1236,7 @@ static bool isSafePHIToSpeculate(PHINode *PN, const TargetData *TD) {
 /// direct (non-volatile) loads and stores to it.  If the alloca is close but
 /// not quite there, this will transform the code to allow promotion.  As such,
 /// it is a non-pure predicate.
-static bool tryToMakeAllocaBePromotable(AllocaInst *AI, const TargetData *TD) {
+static bool tryToMakeAllocaBePromotable(AllocaInst *AI, const DataLayout *TD) {
   SetVector<Instruction*, SmallVector<Instruction*, 4>,
             SmallPtrSet<Instruction*, 4> > InstsToRewrite;
 
@@ -1465,26 +1461,6 @@ bool SROA::ShouldAttemptScalarRepl(AllocaInst *AI) {
   return false;
 }
 
-/// getPointeeAlignment - Compute the minimum alignment of the value pointed
-/// to by the given pointer.
-static unsigned getPointeeAlignment(Value *V, const TargetData &TD) {
-  if (ConstantExpr *CE = dyn_cast<ConstantExpr>(V))
-    if (CE->getOpcode() == Instruction::BitCast ||
-        (CE->getOpcode() == Instruction::GetElementPtr &&
-         cast<GEPOperator>(CE)->hasAllZeroIndices()))
-      return getPointeeAlignment(CE->getOperand(0), TD);
-
-  if (GlobalVariable *GV = dyn_cast<GlobalVariable>(V))
-    if (!GV->isDeclaration())
-      return TD.getPreferredAlignment(GV);
-
-  if (PointerType *PT = dyn_cast<PointerType>(V->getType()))
-    return TD.getABITypeAlignment(PT->getElementType());
-
-  return 0;
-}
-
-
 // performScalarRepl - This algorithm is a simple worklist driven algorithm,
 // which runs on all of the alloca instructions in the function, removing them
 // if they are only used by getelementptr instructions.
@@ -1516,29 +1492,6 @@ bool SROA::performScalarRepl(Function &F) {
     if (AI->isArrayAllocation() || !AI->getAllocatedType()->isSized())
       continue;
 
-    // Check to see if this allocation is only modified by a memcpy/memmove from
-    // a constant global whose alignment is equal to or exceeds that of the
-    // allocation.  If this is the case, we can change all users to use
-    // the constant global instead.  This is commonly produced by the CFE by
-    // constructs like "void foo() { int A[] = {1,2,3,4,5,6,7,8,9...}; }" if 'A'
-    // is only subsequently read.
-    SmallVector<Instruction *, 4> ToDelete;
-    if (MemTransferInst *Copy = isOnlyCopiedFromConstantGlobal(AI, ToDelete)) {
-      if (AI->getAlignment() <= getPointeeAlignment(Copy->getSource(), *TD)) {
-        DEBUG(dbgs() << "Found alloca equal to global: " << *AI << '\n');
-        DEBUG(dbgs() << "  memcpy = " << *Copy << '\n');
-        for (unsigned i = 0, e = ToDelete.size(); i != e; ++i)
-          ToDelete[i]->eraseFromParent();
-        Constant *TheSrc = cast<Constant>(Copy->getSource());
-        AI->replaceAllUsesWith(ConstantExpr::getBitCast(TheSrc, AI->getType()));
-        Copy->eraseFromParent();  // Don't mutate the global.
-        AI->eraseFromParent();
-        ++NumGlobals;
-        Changed = true;
-        continue;
-      }
-    }
-
     // Check to see if we can perform the core SROA transformation.  We cannot
     // transform the allocation instruction if it is an array allocation
     // (allocations OF arrays are ok though), and an allocation of a scalar
@@ -2584,7 +2537,7 @@ void SROA::RewriteLoadUserOfWholeAlloca(LoadInst *LI, AllocaInst *AI,
 /// HasPadding - Return true if the specified type has any structure or
 /// alignment padding in between the elements that would be split apart
 /// by SROA; return false otherwise.
-static bool HasPadding(Type *Ty, const TargetData &TD) {
+static bool HasPadding(Type *Ty, const DataLayout &TD) {
   if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
     Ty = ATy->getElementType();
     return TD.getTypeSizeInBits(Ty) != TD.getTypeAllocSizeInBits(Ty);
@@ -2656,134 +2609,3 @@ bool SROA::isSafeAllocaToScalarRepl(AllocaInst *AI) {
 
   return true;
 }
-
-
-
-/// PointsToConstantGlobal - Return true if V (possibly indirectly) points to
-/// some part of a constant global variable.  This intentionally only accepts
-/// constant expressions because we don't can't rewrite arbitrary instructions.
-static bool PointsToConstantGlobal(Value *V) {
-  if (GlobalVariable *GV = dyn_cast<GlobalVariable>(V))
-    return GV->isConstant();
-  if (ConstantExpr *CE = dyn_cast<ConstantExpr>(V))
-    if (CE->getOpcode() == Instruction::BitCast ||
-        CE->getOpcode() == Instruction::GetElementPtr)
-      return PointsToConstantGlobal(CE->getOperand(0));
-  return false;
-}
-
-/// isOnlyCopiedFromConstantGlobal - Recursively walk the uses of a (derived)
-/// pointer to an alloca.  Ignore any reads of the pointer, return false if we
-/// see any stores or other unknown uses.  If we see pointer arithmetic, keep
-/// track of whether it moves the pointer (with isOffset) but otherwise traverse
-/// the uses.  If we see a memcpy/memmove that targets an unoffseted pointer to
-/// the alloca, and if the source pointer is a pointer to a constant global, we
-/// can optimize this.
-static bool
-isOnlyCopiedFromConstantGlobal(Value *V, MemTransferInst *&TheCopy,
-                               bool isOffset,
-                               SmallVector<Instruction *, 4> &LifetimeMarkers) {
-  // We track lifetime intrinsics as we encounter them.  If we decide to go
-  // ahead and replace the value with the global, this lets the caller quickly
-  // eliminate the markers.
-
-  for (Value::use_iterator UI = V->use_begin(), E = V->use_end(); UI!=E; ++UI) {
-    User *U = cast<Instruction>(*UI);
-
-    if (LoadInst *LI = dyn_cast<LoadInst>(U)) {
-      // Ignore non-volatile loads, they are always ok.
-      if (!LI->isSimple()) return false;
-      continue;
-    }
-
-    if (BitCastInst *BCI = dyn_cast<BitCastInst>(U)) {
-      // If uses of the bitcast are ok, we are ok.
-      if (!isOnlyCopiedFromConstantGlobal(BCI, TheCopy, isOffset,
-                                          LifetimeMarkers))
-        return false;
-      continue;
-    }
-    if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(U)) {
-      // If the GEP has all zero indices, it doesn't offset the pointer.  If it
-      // doesn't, it does.
-      if (!isOnlyCopiedFromConstantGlobal(GEP, TheCopy,
-                                          isOffset || !GEP->hasAllZeroIndices(),
-                                          LifetimeMarkers))
-        return false;
-      continue;
-    }
-
-    if (CallSite CS = U) {
-      // If this is the function being called then we treat it like a load and
-      // ignore it.
-      if (CS.isCallee(UI))
-        continue;
-
-      // If this is a readonly/readnone call site, then we know it is just a
-      // load (but one that potentially returns the value itself), so we can
-      // ignore it if we know that the value isn't captured.
-      unsigned ArgNo = CS.getArgumentNo(UI);
-      if (CS.onlyReadsMemory() &&
-          (CS.getInstruction()->use_empty() || CS.doesNotCapture(ArgNo)))
-        continue;
-
-      // If this is being passed as a byval argument, the caller is making a
-      // copy, so it is only a read of the alloca.
-      if (CS.isByValArgument(ArgNo))
-        continue;
-    }
-
-    // Lifetime intrinsics can be handled by the caller.
-    if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(U)) {
-      if (II->getIntrinsicID() == Intrinsic::lifetime_start ||
-          II->getIntrinsicID() == Intrinsic::lifetime_end) {
-        assert(II->use_empty() && "Lifetime markers have no result to use!");
-        LifetimeMarkers.push_back(II);
-        continue;
-      }
-    }
-
-    // If this is isn't our memcpy/memmove, reject it as something we can't
-    // handle.
-    MemTransferInst *MI = dyn_cast<MemTransferInst>(U);
-    if (MI == 0)
-      return false;
-
-    // If the transfer is using the alloca as a source of the transfer, then
-    // ignore it since it is a load (unless the transfer is volatile).
-    if (UI.getOperandNo() == 1) {
-      if (MI->isVolatile()) return false;
-      continue;
-    }
-
-    // If we already have seen a copy, reject the second one.
-    if (TheCopy) return false;
-
-    // If the pointer has been offset from the start of the alloca, we can't
-    // safely handle this.
-    if (isOffset) return false;
-
-    // If the memintrinsic isn't using the alloca as the dest, reject it.
-    if (UI.getOperandNo() != 0) return false;
-
-    // If the source of the memcpy/move is not a constant global, reject it.
-    if (!PointsToConstantGlobal(MI->getSource()))
-      return false;
-
-    // Otherwise, the transform is safe.  Remember the copy instruction.
-    TheCopy = MI;
-  }
-  return true;
-}
-
-/// isOnlyCopiedFromConstantGlobal - Return true if the specified alloca is only
-/// modified by a copy from a constant global.  If we can prove this, we can
-/// replace any uses of the alloca with uses of the global directly.
-MemTransferInst *
-SROA::isOnlyCopiedFromConstantGlobal(AllocaInst *AI,
-                                     SmallVector<Instruction*, 4> &ToDelete) {
-  MemTransferInst *TheCopy = 0;
-  if (::isOnlyCopiedFromConstantGlobal(AI, TheCopy, false, ToDelete))
-    return TheCopy;
-  return 0;
-}
diff --git a/lib/Transforms/Scalar/SimplifyCFGPass.cpp b/lib/Transforms/Scalar/SimplifyCFGPass.cpp
index d13e4ab..9f24bb6 100644
--- a/lib/Transforms/Scalar/SimplifyCFGPass.cpp
+++ b/lib/Transforms/Scalar/SimplifyCFGPass.cpp
@@ -31,10 +31,11 @@
 #include "llvm/Attributes.h"
 #include "llvm/Support/CFG.h"
 #include "llvm/Pass.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/TargetTransformInfo.h"
 using namespace llvm;
 
 STATISTIC(NumSimpl, "Number of blocks simplified");
@@ -59,9 +60,9 @@ FunctionPass *llvm::createCFGSimplificationPass() {
   return new CFGSimplifyPass();
 }
 
-/// ChangeToUnreachable - Insert an unreachable instruction before the specified
+/// changeToUnreachable - Insert an unreachable instruction before the specified
 /// instruction, making it and the rest of the code in the block dead.
-static void ChangeToUnreachable(Instruction *I, bool UseLLVMTrap) {
+static void changeToUnreachable(Instruction *I, bool UseLLVMTrap) {
   BasicBlock *BB = I->getParent();
   // Loop over all of the successors, removing BB's entry from any PHI
   // nodes.
@@ -87,8 +88,8 @@ static void ChangeToUnreachable(Instruction *I, bool UseLLVMTrap) {
   }
 }
 
-/// ChangeToCall - Convert the specified invoke into a normal call.
-static void ChangeToCall(InvokeInst *II) {
+/// changeToCall - Convert the specified invoke into a normal call.
+static void changeToCall(InvokeInst *II) {
   SmallVector<Value*, 8> Args(II->op_begin(), II->op_end() - 3);
   CallInst *NewCall = CallInst::Create(II->getCalledValue(), Args, "", II);
   NewCall->takeName(II);
@@ -105,7 +106,7 @@ static void ChangeToCall(InvokeInst *II) {
   II->eraseFromParent();
 }
 
-static bool MarkAliveBlocks(BasicBlock *BB,
+static bool markAliveBlocks(BasicBlock *BB,
                             SmallPtrSet<BasicBlock*, 128> &Reachable) {
 
   SmallVector<BasicBlock*, 128> Worklist;
@@ -129,7 +130,7 @@ static bool MarkAliveBlocks(BasicBlock *BB,
           ++BBI;
           if (!isa<UnreachableInst>(BBI)) {
             // Don't insert a call to llvm.trap right before the unreachable.
-            ChangeToUnreachable(BBI, false);
+            changeToUnreachable(BBI, false);
             Changed = true;
           }
           break;
@@ -148,7 +149,7 @@ static bool MarkAliveBlocks(BasicBlock *BB,
         if (isa<UndefValue>(Ptr) ||
             (isa<ConstantPointerNull>(Ptr) &&
              SI->getPointerAddressSpace() == 0)) {
-          ChangeToUnreachable(SI, true);
+          changeToUnreachable(SI, true);
           Changed = true;
           break;
         }
@@ -159,7 +160,7 @@ static bool MarkAliveBlocks(BasicBlock *BB,
     if (InvokeInst *II = dyn_cast<InvokeInst>(BB->getTerminator())) {
       Value *Callee = II->getCalledValue();
       if (isa<ConstantPointerNull>(Callee) || isa<UndefValue>(Callee)) {
-        ChangeToUnreachable(II, true);
+        changeToUnreachable(II, true);
         Changed = true;
       } else if (II->doesNotThrow()) {
         if (II->use_empty() && II->onlyReadsMemory()) {
@@ -168,7 +169,7 @@ static bool MarkAliveBlocks(BasicBlock *BB,
           II->getUnwindDest()->removePredecessor(II->getParent());
           II->eraseFromParent();
         } else
-          ChangeToCall(II);
+          changeToCall(II);
         Changed = true;
       }
     }
@@ -180,12 +181,12 @@ static bool MarkAliveBlocks(BasicBlock *BB,
   return Changed;
 }
 
-/// RemoveUnreachableBlocksFromFn - Remove blocks that are not reachable, even
+/// removeUnreachableBlocksFromFn - Remove blocks that are not reachable, even
 /// if they are in a dead cycle.  Return true if a change was made, false
 /// otherwise.
-static bool RemoveUnreachableBlocksFromFn(Function &F) {
+static bool removeUnreachableBlocksFromFn(Function &F) {
   SmallPtrSet<BasicBlock*, 128> Reachable;
-  bool Changed = MarkAliveBlocks(F.begin(), Reachable);
+  bool Changed = markAliveBlocks(F.begin(), Reachable);
 
   // If there are unreachable blocks in the CFG...
   if (Reachable.size() == F.size())
@@ -215,9 +216,9 @@ static bool RemoveUnreachableBlocksFromFn(Function &F) {
   return true;
 }
 
-/// MergeEmptyReturnBlocks - If we have more than one empty (other than phi
+/// mergeEmptyReturnBlocks - If we have more than one empty (other than phi
 /// node) return blocks, merge them together to promote recursive block merging.
-static bool MergeEmptyReturnBlocks(Function &F) {
+static bool mergeEmptyReturnBlocks(Function &F) {
   bool Changed = false;
 
   BasicBlock *RetBlock = 0;
@@ -291,9 +292,10 @@ static bool MergeEmptyReturnBlocks(Function &F) {
   return Changed;
 }
 
-/// IterativeSimplifyCFG - Call SimplifyCFG on all the blocks in the function,
+/// iterativelySimplifyCFG - Call SimplifyCFG on all the blocks in the function,
 /// iterating until no more changes are made.
-static bool IterativeSimplifyCFG(Function &F, const TargetData *TD) {
+static bool iterativelySimplifyCFG(Function &F, const DataLayout *TD,
+                                   const TargetTransformInfo *TTI) {
   bool Changed = false;
   bool LocalChange = true;
   while (LocalChange) {
@@ -302,7 +304,7 @@ static bool IterativeSimplifyCFG(Function &F, const TargetData *TD) {
     // Loop over all of the basic blocks and remove them if they are unneeded...
     //
     for (Function::iterator BBIt = F.begin(); BBIt != F.end(); ) {
-      if (SimplifyCFG(BBIt++, TD)) {
+      if (SimplifyCFG(BBIt++, TD, TTI)) {
         LocalChange = true;
         ++NumSimpl;
       }
@@ -316,25 +318,27 @@ static bool IterativeSimplifyCFG(Function &F, const TargetData *TD) {
 // simplify the CFG.
 //
 bool CFGSimplifyPass::runOnFunction(Function &F) {
-  const TargetData *TD = getAnalysisIfAvailable<TargetData>();
-  bool EverChanged = RemoveUnreachableBlocksFromFn(F);
-  EverChanged |= MergeEmptyReturnBlocks(F);
-  EverChanged |= IterativeSimplifyCFG(F, TD);
+  const DataLayout *TD = getAnalysisIfAvailable<DataLayout>();
+  const TargetTransformInfo *TTI =
+      getAnalysisIfAvailable<TargetTransformInfo>();
+  bool EverChanged = removeUnreachableBlocksFromFn(F);
+  EverChanged |= mergeEmptyReturnBlocks(F);
+  EverChanged |= iterativelySimplifyCFG(F, TD, TTI);
 
   // If neither pass changed anything, we're done.
   if (!EverChanged) return false;
 
-  // IterativeSimplifyCFG can (rarely) make some loops dead.  If this happens,
-  // RemoveUnreachableBlocksFromFn is needed to nuke them, which means we should
+  // iterativelySimplifyCFG can (rarely) make some loops dead.  If this happens,
+  // removeUnreachableBlocksFromFn is needed to nuke them, which means we should
   // iterate between the two optimizations.  We structure the code like this to
-  // avoid reruning IterativeSimplifyCFG if the second pass of
-  // RemoveUnreachableBlocksFromFn doesn't do anything.
-  if (!RemoveUnreachableBlocksFromFn(F))
+  // avoid reruning iterativelySimplifyCFG if the second pass of
+  // removeUnreachableBlocksFromFn doesn't do anything.
+  if (!removeUnreachableBlocksFromFn(F))
     return true;
 
   do {
-    EverChanged = IterativeSimplifyCFG(F, TD);
-    EverChanged |= RemoveUnreachableBlocksFromFn(F);
+    EverChanged = iterativelySimplifyCFG(F, TD, TTI);
+    EverChanged |= removeUnreachableBlocksFromFn(F);
   } while (EverChanged);
 
   return true;
diff --git a/lib/Transforms/Scalar/SimplifyLibCalls.cpp b/lib/Transforms/Scalar/SimplifyLibCalls.cpp
index f110320..17d07cd 100644
--- a/lib/Transforms/Scalar/SimplifyLibCalls.cpp
+++ b/lib/Transforms/Scalar/SimplifyLibCalls.cpp
@@ -28,9 +28,10 @@
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/Analysis/ValueTracking.h"
+#include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetLibraryInfo.h"
 #include "llvm/Config/config.h"            // FIXME: Shouldn't depend on host!
 using namespace llvm;
@@ -38,6 +39,10 @@ using namespace llvm;
 STATISTIC(NumSimplified, "Number of library calls simplified");
 STATISTIC(NumAnnotated, "Number of attributes added to library functions");
 
+static cl::opt<bool> UnsafeFPShrink("enable-double-float-shrink", cl::Hidden,
+                                   cl::init(false),
+                                   cl::desc("Enable unsafe double to float "
+                                            "shrinking for math lib calls"));
 //===----------------------------------------------------------------------===//
 // Optimizer Base Class
 //===----------------------------------------------------------------------===//
@@ -48,7 +53,7 @@ namespace {
 class LibCallOptimization {
 protected:
   Function *Caller;
-  const TargetData *TD;
+  const DataLayout *TD;
   const TargetLibraryInfo *TLI;
   LLVMContext* Context;
 public:
@@ -63,7 +68,7 @@ public:
   virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B)
     =0;
 
-  Value *OptimizeCall(CallInst *CI, const TargetData *TD,
+  Value *OptimizeCall(CallInst *CI, const DataLayout *TD,
                       const TargetLibraryInfo *TLI, IRBuilder<> &B) {
     Caller = CI->getParent()->getParent();
     this->TD = TD;
@@ -85,22 +90,6 @@ public:
 // Helper Functions
 //===----------------------------------------------------------------------===//
 
-/// IsOnlyUsedInZeroEqualityComparison - Return true if it only matters that the
-/// value is equal or not-equal to zero.
-static bool IsOnlyUsedInZeroEqualityComparison(Value *V) {
-  for (Value::use_iterator UI = V->use_begin(), E = V->use_end();
-       UI != E; ++UI) {
-    if (ICmpInst *IC = dyn_cast<ICmpInst>(*UI))
-      if (IC->isEquality())
-        if (Constant *C = dyn_cast<Constant>(IC->getOperand(1)))
-          if (C->isNullValue())
-            continue;
-    // Unknown instruction.
-    return false;
-  }
-  return true;
-}
-
 static bool CallHasFloatingPointArgument(const CallInst *CI) {
   for (CallInst::const_op_iterator it = CI->op_begin(), e = CI->op_end();
        it != e; ++it) {
@@ -110,799 +99,62 @@ static bool CallHasFloatingPointArgument(const CallInst *CI) {
   return false;
 }
 
-/// IsOnlyUsedInEqualityComparison - Return true if it is only used in equality
-/// comparisons with With.
-static bool IsOnlyUsedInEqualityComparison(Value *V, Value *With) {
-  for (Value::use_iterator UI = V->use_begin(), E = V->use_end();
-       UI != E; ++UI) {
-    if (ICmpInst *IC = dyn_cast<ICmpInst>(*UI))
-      if (IC->isEquality() && IC->getOperand(1) == With)
-        continue;
-    // Unknown instruction.
-    return false;
-  }
-  return true;
-}
-
+namespace {
 //===----------------------------------------------------------------------===//
-// String and Memory LibCall Optimizations
+// Math Library Optimizations
 //===----------------------------------------------------------------------===//
 
 //===---------------------------------------===//
-// 'strcat' Optimizations
-namespace {
-struct StrCatOpt : public LibCallOptimization {
-  virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
-    // Verify the "strcat" function prototype.
-    FunctionType *FT = Callee->getFunctionType();
-    if (FT->getNumParams() != 2 ||
-        FT->getReturnType() != B.getInt8PtrTy() ||
-        FT->getParamType(0) != FT->getReturnType() ||
-        FT->getParamType(1) != FT->getReturnType())
-      return 0;
-
-    // Extract some information from the instruction
-    Value *Dst = CI->getArgOperand(0);
-    Value *Src = CI->getArgOperand(1);
-
-    // See if we can get the length of the input string.
-    uint64_t Len = GetStringLength(Src);
-    if (Len == 0) return 0;
-    --Len;  // Unbias length.
-
-    // Handle the simple, do-nothing case: strcat(x, "") -> x
-    if (Len == 0)
-      return Dst;
-
-    // These optimizations require TargetData.
-    if (!TD) return 0;
-
-    return EmitStrLenMemCpy(Src, Dst, Len, B);
-  }
-
-  Value *EmitStrLenMemCpy(Value *Src, Value *Dst, uint64_t Len, IRBuilder<> &B) {
-    // We need to find the end of the destination string.  That's where the
-    // memory is to be moved to. We just generate a call to strlen.
-    Value *DstLen = EmitStrLen(Dst, B, TD, TLI);
-    if (!DstLen)
-      return 0;
-
-    // Now that we have the destination's length, we must index into the
-    // destination's pointer to get the actual memcpy destination (end of
-    // the string .. we're concatenating).
-    Value *CpyDst = B.CreateGEP(Dst, DstLen, "endptr");
-
-    // We have enough information to now generate the memcpy call to do the
-    // concatenation for us.  Make a memcpy to copy the nul byte with align = 1.
-    B.CreateMemCpy(CpyDst, Src,
-                   ConstantInt::get(TD->getIntPtrType(*Context), Len + 1), 1);
-    return Dst;
-  }
-};
-
-//===---------------------------------------===//
-// 'strncat' Optimizations
-
-struct StrNCatOpt : public StrCatOpt {
-  virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
-    // Verify the "strncat" function prototype.
-    FunctionType *FT = Callee->getFunctionType();
-    if (FT->getNumParams() != 3 ||
-        FT->getReturnType() != B.getInt8PtrTy() ||
-        FT->getParamType(0) != FT->getReturnType() ||
-        FT->getParamType(1) != FT->getReturnType() ||
-        !FT->getParamType(2)->isIntegerTy())
-      return 0;
-
-    // Extract some information from the instruction
-    Value *Dst = CI->getArgOperand(0);
-    Value *Src = CI->getArgOperand(1);
-    uint64_t Len;
-
-    // We don't do anything if length is not constant
-    if (ConstantInt *LengthArg = dyn_cast<ConstantInt>(CI->getArgOperand(2)))
-      Len = LengthArg->getZExtValue();
-    else
-      return 0;
-
-    // See if we can get the length of the input string.
-    uint64_t SrcLen = GetStringLength(Src);
-    if (SrcLen == 0) return 0;
-    --SrcLen;  // Unbias length.
-
-    // Handle the simple, do-nothing cases:
-    // strncat(x, "", c) -> x
-    // strncat(x,  c, 0) -> x
-    if (SrcLen == 0 || Len == 0) return Dst;
-
-    // These optimizations require TargetData.
-    if (!TD) return 0;
-
-    // We don't optimize this case
-    if (Len < SrcLen) return 0;
-
-    // strncat(x, s, c) -> strcat(x, s)
-    // s is constant so the strcat can be optimized further
-    return EmitStrLenMemCpy(Src, Dst, SrcLen, B);
-  }
-};
-
-//===---------------------------------------===//
-// 'strchr' Optimizations
-
-struct StrChrOpt : public LibCallOptimization {
-  virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
-    // Verify the "strchr" function prototype.
-    FunctionType *FT = Callee->getFunctionType();
-    if (FT->getNumParams() != 2 ||
-        FT->getReturnType() != B.getInt8PtrTy() ||
-        FT->getParamType(0) != FT->getReturnType() ||
-        !FT->getParamType(1)->isIntegerTy(32))
-      return 0;
-
-    Value *SrcStr = CI->getArgOperand(0);
-
-    // If the second operand is non-constant, see if we can compute the length
-    // of the input string and turn this into memchr.
-    ConstantInt *CharC = dyn_cast<ConstantInt>(CI->getArgOperand(1));
-    if (CharC == 0) {
-      // These optimizations require TargetData.
-      if (!TD) return 0;
-
-      uint64_t Len = GetStringLength(SrcStr);
-      if (Len == 0 || !FT->getParamType(1)->isIntegerTy(32))// memchr needs i32.
-        return 0;
-
-      return EmitMemChr(SrcStr, CI->getArgOperand(1), // include nul.
-                        ConstantInt::get(TD->getIntPtrType(*Context), Len),
-                        B, TD, TLI);
-    }
-
-    // Otherwise, the character is a constant, see if the first argument is
-    // a string literal.  If so, we can constant fold.
-    StringRef Str;
-    if (!getConstantStringInfo(SrcStr, Str))
-      return 0;
-
-    // Compute the offset, make sure to handle the case when we're searching for
-    // zero (a weird way to spell strlen).
-    size_t I = CharC->getSExtValue() == 0 ?
-        Str.size() : Str.find(CharC->getSExtValue());
-    if (I == StringRef::npos) // Didn't find the char.  strchr returns null.
-      return Constant::getNullValue(CI->getType());
-
-    // strchr(s+n,c)  -> gep(s+n+i,c)
-    return B.CreateGEP(SrcStr, B.getInt64(I), "strchr");
-  }
-};
-
-//===---------------------------------------===//
-// 'strrchr' Optimizations
-
-struct StrRChrOpt : public LibCallOptimization {
-  virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
-    // Verify the "strrchr" function prototype.
-    FunctionType *FT = Callee->getFunctionType();
-    if (FT->getNumParams() != 2 ||
-        FT->getReturnType() != B.getInt8PtrTy() ||
-        FT->getParamType(0) != FT->getReturnType() ||
-        !FT->getParamType(1)->isIntegerTy(32))
-      return 0;
-
-    Value *SrcStr = CI->getArgOperand(0);
-    ConstantInt *CharC = dyn_cast<ConstantInt>(CI->getArgOperand(1));
-
-    // Cannot fold anything if we're not looking for a constant.
-    if (!CharC)
-      return 0;
-
-    StringRef Str;
-    if (!getConstantStringInfo(SrcStr, Str)) {
-      // strrchr(s, 0) -> strchr(s, 0)
-      if (TD && CharC->isZero())
-        return EmitStrChr(SrcStr, '\0', B, TD, TLI);
-      return 0;
-    }
-
-    // Compute the offset.
-    size_t I = CharC->getSExtValue() == 0 ?
-        Str.size() : Str.rfind(CharC->getSExtValue());
-    if (I == StringRef::npos) // Didn't find the char. Return null.
-      return Constant::getNullValue(CI->getType());
-
-    // strrchr(s+n,c) -> gep(s+n+i,c)
-    return B.CreateGEP(SrcStr, B.getInt64(I), "strrchr");
-  }
-};
-
-//===---------------------------------------===//
-// 'strcmp' Optimizations
-
-struct StrCmpOpt : public LibCallOptimization {
-  virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
-    // Verify the "strcmp" function prototype.
-    FunctionType *FT = Callee->getFunctionType();
-    if (FT->getNumParams() != 2 ||
-        !FT->getReturnType()->isIntegerTy(32) ||
-        FT->getParamType(0) != FT->getParamType(1) ||
-        FT->getParamType(0) != B.getInt8PtrTy())
-      return 0;
-
-    Value *Str1P = CI->getArgOperand(0), *Str2P = CI->getArgOperand(1);
-    if (Str1P == Str2P)      // strcmp(x,x)  -> 0
-      return ConstantInt::get(CI->getType(), 0);
-
-    StringRef Str1, Str2;
-    bool HasStr1 = getConstantStringInfo(Str1P, Str1);
-    bool HasStr2 = getConstantStringInfo(Str2P, Str2);
-
-    // strcmp(x, y)  -> cnst  (if both x and y are constant strings)
-    if (HasStr1 && HasStr2)
-      return ConstantInt::get(CI->getType(), Str1.compare(Str2));
-
-    if (HasStr1 && Str1.empty()) // strcmp("", x) -> -*x
-      return B.CreateNeg(B.CreateZExt(B.CreateLoad(Str2P, "strcmpload"),
-                                      CI->getType()));
-
-    if (HasStr2 && Str2.empty()) // strcmp(x,"") -> *x
-      return B.CreateZExt(B.CreateLoad(Str1P, "strcmpload"), CI->getType());
-
-    // strcmp(P, "x") -> memcmp(P, "x", 2)
-    uint64_t Len1 = GetStringLength(Str1P);
-    uint64_t Len2 = GetStringLength(Str2P);
-    if (Len1 && Len2) {
-      // These optimizations require TargetData.
-      if (!TD) return 0;
-
-      return EmitMemCmp(Str1P, Str2P,
-                        ConstantInt::get(TD->getIntPtrType(*Context),
-                        std::min(Len1, Len2)), B, TD, TLI);
-    }
-
-    return 0;
-  }
-};
-
-//===---------------------------------------===//
-// 'strncmp' Optimizations
-
-struct StrNCmpOpt : public LibCallOptimization {
-  virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
-    // Verify the "strncmp" function prototype.
-    FunctionType *FT = Callee->getFunctionType();
-    if (FT->getNumParams() != 3 ||
-        !FT->getReturnType()->isIntegerTy(32) ||
-        FT->getParamType(0) != FT->getParamType(1) ||
-        FT->getParamType(0) != B.getInt8PtrTy() ||
-        !FT->getParamType(2)->isIntegerTy())
-      return 0;
-
-    Value *Str1P = CI->getArgOperand(0), *Str2P = CI->getArgOperand(1);
-    if (Str1P == Str2P)      // strncmp(x,x,n)  -> 0
-      return ConstantInt::get(CI->getType(), 0);
-
-    // Get the length argument if it is constant.
-    uint64_t Length;
-    if (ConstantInt *LengthArg = dyn_cast<ConstantInt>(CI->getArgOperand(2)))
-      Length = LengthArg->getZExtValue();
-    else
-      return 0;
-
-    if (Length == 0) // strncmp(x,y,0)   -> 0
-      return ConstantInt::get(CI->getType(), 0);
-
-    if (TD && Length == 1) // strncmp(x,y,1) -> memcmp(x,y,1)
-      return EmitMemCmp(Str1P, Str2P, CI->getArgOperand(2), B, TD, TLI);
-
-    StringRef Str1, Str2;
-    bool HasStr1 = getConstantStringInfo(Str1P, Str1);
-    bool HasStr2 = getConstantStringInfo(Str2P, Str2);
-
-    // strncmp(x, y)  -> cnst  (if both x and y are constant strings)
-    if (HasStr1 && HasStr2) {
-      StringRef SubStr1 = Str1.substr(0, Length);
-      StringRef SubStr2 = Str2.substr(0, Length);
-      return ConstantInt::get(CI->getType(), SubStr1.compare(SubStr2));
-    }
-
-    if (HasStr1 && Str1.empty())  // strncmp("", x, n) -> -*x
-      return B.CreateNeg(B.CreateZExt(B.CreateLoad(Str2P, "strcmpload"),
-                                      CI->getType()));
-
-    if (HasStr2 && Str2.empty())  // strncmp(x, "", n) -> *x
-      return B.CreateZExt(B.CreateLoad(Str1P, "strcmpload"), CI->getType());
-
-    return 0;
-  }
-};
-
-
-//===---------------------------------------===//
-// 'strcpy' Optimizations
-
-struct StrCpyOpt : public LibCallOptimization {
-  bool OptChkCall;  // True if it's optimizing a __strcpy_chk libcall.
-
-  StrCpyOpt(bool c) : OptChkCall(c) {}
-
-  virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
-    // Verify the "strcpy" function prototype.
-    unsigned NumParams = OptChkCall ? 3 : 2;
-    FunctionType *FT = Callee->getFunctionType();
-    if (FT->getNumParams() != NumParams ||
-        FT->getReturnType() != FT->getParamType(0) ||
-        FT->getParamType(0) != FT->getParamType(1) ||
-        FT->getParamType(0) != B.getInt8PtrTy())
-      return 0;
-
-    Value *Dst = CI->getArgOperand(0), *Src = CI->getArgOperand(1);
-    if (Dst == Src)      // strcpy(x,x)  -> x
-      return Src;
-
-    // These optimizations require TargetData.
-    if (!TD) return 0;
-
-    // See if we can get the length of the input string.
-    uint64_t Len = GetStringLength(Src);
-    if (Len == 0) return 0;
-
-    // We have enough information to now generate the memcpy call to do the
-    // concatenation for us.  Make a memcpy to copy the nul byte with align = 1.
-    if (!OptChkCall ||
-        !EmitMemCpyChk(Dst, Src,
-                       ConstantInt::get(TD->getIntPtrType(*Context), Len),
-                       CI->getArgOperand(2), B, TD, TLI))
-      B.CreateMemCpy(Dst, Src,
-                     ConstantInt::get(TD->getIntPtrType(*Context), Len), 1);
-    return Dst;
-  }
-};
-
-//===---------------------------------------===//
-// 'stpcpy' Optimizations
-
-struct StpCpyOpt: public LibCallOptimization {
-  bool OptChkCall;  // True if it's optimizing a __stpcpy_chk libcall.
-
-  StpCpyOpt(bool c) : OptChkCall(c) {}
-
-  virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
-    // Verify the "stpcpy" function prototype.
-    unsigned NumParams = OptChkCall ? 3 : 2;
-    FunctionType *FT = Callee->getFunctionType();
-    if (FT->getNumParams() != NumParams ||
-        FT->getReturnType() != FT->getParamType(0) ||
-        FT->getParamType(0) != FT->getParamType(1) ||
-        FT->getParamType(0) != B.getInt8PtrTy())
-      return 0;
-
-    // These optimizations require TargetData.
-    if (!TD) return 0;
-
-    Value *Dst = CI->getArgOperand(0), *Src = CI->getArgOperand(1);
-    if (Dst == Src) {  // stpcpy(x,x)  -> x+strlen(x)
-      Value *StrLen = EmitStrLen(Src, B, TD, TLI);
-      return StrLen ? B.CreateInBoundsGEP(Dst, StrLen) : 0;
-    }
-
-    // See if we can get the length of the input string.
-    uint64_t Len = GetStringLength(Src);
-    if (Len == 0) return 0;
-
-    Value *LenV = ConstantInt::get(TD->getIntPtrType(*Context), Len);
-    Value *DstEnd = B.CreateGEP(Dst,
-                                ConstantInt::get(TD->getIntPtrType(*Context),
-                                                 Len - 1));
-
-    // We have enough information to now generate the memcpy call to do the
-    // copy for us.  Make a memcpy to copy the nul byte with align = 1.
-    if (!OptChkCall || !EmitMemCpyChk(Dst, Src, LenV, CI->getArgOperand(2), B,
-                                      TD, TLI))
-      B.CreateMemCpy(Dst, Src, LenV, 1);
-    return DstEnd;
-  }
-};
-
-//===---------------------------------------===//
-// 'strncpy' Optimizations
-
-struct StrNCpyOpt : public LibCallOptimization {
-  virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
-    FunctionType *FT = Callee->getFunctionType();
-    if (FT->getNumParams() != 3 || FT->getReturnType() != FT->getParamType(0) ||
-        FT->getParamType(0) != FT->getParamType(1) ||
-        FT->getParamType(0) != B.getInt8PtrTy() ||
-        !FT->getParamType(2)->isIntegerTy())
-      return 0;
-
-    Value *Dst = CI->getArgOperand(0);
-    Value *Src = CI->getArgOperand(1);
-    Value *LenOp = CI->getArgOperand(2);
-
-    // See if we can get the length of the input string.
-    uint64_t SrcLen = GetStringLength(Src);
-    if (SrcLen == 0) return 0;
-    --SrcLen;
-
-    if (SrcLen == 0) {
-      // strncpy(x, "", y) -> memset(x, '\0', y, 1)
-      B.CreateMemSet(Dst, B.getInt8('\0'), LenOp, 1);
-      return Dst;
-    }
-
-    uint64_t Len;
-    if (ConstantInt *LengthArg = dyn_cast<ConstantInt>(LenOp))
-      Len = LengthArg->getZExtValue();
-    else
-      return 0;
-
-    if (Len == 0) return Dst; // strncpy(x, y, 0) -> x
-
-    // These optimizations require TargetData.
-    if (!TD) return 0;
-
-    // Let strncpy handle the zero padding
-    if (Len > SrcLen+1) return 0;
-
-    // strncpy(x, s, c) -> memcpy(x, s, c, 1) [s and c are constant]
-    B.CreateMemCpy(Dst, Src,
-                   ConstantInt::get(TD->getIntPtrType(*Context), Len), 1);
-
-    return Dst;
-  }
-};
-
-//===---------------------------------------===//
-// 'strlen' Optimizations
-
-struct StrLenOpt : public LibCallOptimization {
-  virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
-    FunctionType *FT = Callee->getFunctionType();
-    if (FT->getNumParams() != 1 ||
-        FT->getParamType(0) != B.getInt8PtrTy() ||
-        !FT->getReturnType()->isIntegerTy())
-      return 0;
-
-    Value *Src = CI->getArgOperand(0);
-
-    // Constant folding: strlen("xyz") -> 3
-    if (uint64_t Len = GetStringLength(Src))
-      return ConstantInt::get(CI->getType(), Len-1);
-
-    // strlen(x) != 0 --> *x != 0
-    // strlen(x) == 0 --> *x == 0
-    if (IsOnlyUsedInZeroEqualityComparison(CI))
-      return B.CreateZExt(B.CreateLoad(Src, "strlenfirst"), CI->getType());
-    return 0;
-  }
-};
-
-
-//===---------------------------------------===//
-// 'strpbrk' Optimizations
-
-struct StrPBrkOpt : public LibCallOptimization {
-  virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
-    FunctionType *FT = Callee->getFunctionType();
-    if (FT->getNumParams() != 2 ||
-        FT->getParamType(0) != B.getInt8PtrTy() ||
-        FT->getParamType(1) != FT->getParamType(0) ||
-        FT->getReturnType() != FT->getParamType(0))
-      return 0;
-
-    StringRef S1, S2;
-    bool HasS1 = getConstantStringInfo(CI->getArgOperand(0), S1);
-    bool HasS2 = getConstantStringInfo(CI->getArgOperand(1), S2);
-
-    // strpbrk(s, "") -> NULL
-    // strpbrk("", s) -> NULL
-    if ((HasS1 && S1.empty()) || (HasS2 && S2.empty()))
-      return Constant::getNullValue(CI->getType());
-
-    // Constant folding.
-    if (HasS1 && HasS2) {
-      size_t I = S1.find_first_of(S2);
-      if (I == std::string::npos) // No match.
-        return Constant::getNullValue(CI->getType());
-
-      return B.CreateGEP(CI->getArgOperand(0), B.getInt64(I), "strpbrk");
-    }
-
-    // strpbrk(s, "a") -> strchr(s, 'a')
-    if (TD && HasS2 && S2.size() == 1)
-      return EmitStrChr(CI->getArgOperand(0), S2[0], B, TD, TLI);
-
-    return 0;
-  }
-};
-
-//===---------------------------------------===//
-// 'strto*' Optimizations.  This handles strtol, strtod, strtof, strtoul, etc.
-
-struct StrToOpt : public LibCallOptimization {
-  virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
-    FunctionType *FT = Callee->getFunctionType();
-    if ((FT->getNumParams() != 2 && FT->getNumParams() != 3) ||
-        !FT->getParamType(0)->isPointerTy() ||
-        !FT->getParamType(1)->isPointerTy())
-      return 0;
-
-    Value *EndPtr = CI->getArgOperand(1);
-    if (isa<ConstantPointerNull>(EndPtr)) {
-      // With a null EndPtr, this function won't capture the main argument.
-      // It would be readonly too, except that it still may write to errno.
-      CI->addAttribute(1, Attribute::NoCapture);
-    }
-
-    return 0;
-  }
-};
-
-//===---------------------------------------===//
-// 'strspn' Optimizations
-
-struct StrSpnOpt : public LibCallOptimization {
-  virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
-    FunctionType *FT = Callee->getFunctionType();
-    if (FT->getNumParams() != 2 ||
-        FT->getParamType(0) != B.getInt8PtrTy() ||
-        FT->getParamType(1) != FT->getParamType(0) ||
-        !FT->getReturnType()->isIntegerTy())
-      return 0;
-
-    StringRef S1, S2;
-    bool HasS1 = getConstantStringInfo(CI->getArgOperand(0), S1);
-    bool HasS2 = getConstantStringInfo(CI->getArgOperand(1), S2);
-
-    // strspn(s, "") -> 0
-    // strspn("", s) -> 0
-    if ((HasS1 && S1.empty()) || (HasS2 && S2.empty()))
-      return Constant::getNullValue(CI->getType());
-
-    // Constant folding.
-    if (HasS1 && HasS2) {
-      size_t Pos = S1.find_first_not_of(S2);
-      if (Pos == StringRef::npos) Pos = S1.size();
-      return ConstantInt::get(CI->getType(), Pos);
-    }
-
-    return 0;
-  }
-};
-
-//===---------------------------------------===//
-// 'strcspn' Optimizations
-
-struct StrCSpnOpt : public LibCallOptimization {
-  virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
-    FunctionType *FT = Callee->getFunctionType();
-    if (FT->getNumParams() != 2 ||
-        FT->getParamType(0) != B.getInt8PtrTy() ||
-        FT->getParamType(1) != FT->getParamType(0) ||
-        !FT->getReturnType()->isIntegerTy())
-      return 0;
-
-    StringRef S1, S2;
-    bool HasS1 = getConstantStringInfo(CI->getArgOperand(0), S1);
-    bool HasS2 = getConstantStringInfo(CI->getArgOperand(1), S2);
-
-    // strcspn("", s) -> 0
-    if (HasS1 && S1.empty())
-      return Constant::getNullValue(CI->getType());
-
-    // Constant folding.
-    if (HasS1 && HasS2) {
-      size_t Pos = S1.find_first_of(S2);
-      if (Pos == StringRef::npos) Pos = S1.size();
-      return ConstantInt::get(CI->getType(), Pos);
-    }
-
-    // strcspn(s, "") -> strlen(s)
-    if (TD && HasS2 && S2.empty())
-      return EmitStrLen(CI->getArgOperand(0), B, TD, TLI);
-
-    return 0;
-  }
-};
-
-//===---------------------------------------===//
-// 'strstr' Optimizations
+// Double -> Float Shrinking Optimizations for Unary Functions like 'floor'
 
-struct StrStrOpt : public LibCallOptimization {
+struct UnaryDoubleFPOpt : public LibCallOptimization {
+  bool CheckRetType;
+  UnaryDoubleFPOpt(bool CheckReturnType): CheckRetType(CheckReturnType) {}
   virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
     FunctionType *FT = Callee->getFunctionType();
-    if (FT->getNumParams() != 2 ||
-        !FT->getParamType(0)->isPointerTy() ||
-        !FT->getParamType(1)->isPointerTy() ||
-        !FT->getReturnType()->isPointerTy())
+    if (FT->getNumParams() != 1 || !FT->getReturnType()->isDoubleTy() ||
+        !FT->getParamType(0)->isDoubleTy())
       return 0;
 
-    // fold strstr(x, x) -> x.
-    if (CI->getArgOperand(0) == CI->getArgOperand(1))
-      return B.CreateBitCast(CI->getArgOperand(0), CI->getType());
-
-    // fold strstr(a, b) == a -> strncmp(a, b, strlen(b)) == 0
-    if (TD && IsOnlyUsedInEqualityComparison(CI, CI->getArgOperand(0))) {
-      Value *StrLen = EmitStrLen(CI->getArgOperand(1), B, TD, TLI);
-      if (!StrLen)
-        return 0;
-      Value *StrNCmp = EmitStrNCmp(CI->getArgOperand(0), CI->getArgOperand(1),
-                                   StrLen, B, TD, TLI);
-      if (!StrNCmp)
-        return 0;
-      for (Value::use_iterator UI = CI->use_begin(), UE = CI->use_end();
-           UI != UE; ) {
-        ICmpInst *Old = cast<ICmpInst>(*UI++);
-        Value *Cmp = B.CreateICmp(Old->getPredicate(), StrNCmp,
-                                  ConstantInt::getNullValue(StrNCmp->getType()),
-                                  "cmp");
-        Old->replaceAllUsesWith(Cmp);
-        Old->eraseFromParent();
+    if (CheckRetType) {
+      // Check if all the uses for function like 'sin' are converted to float.
+      for (Value::use_iterator UseI = CI->use_begin(); UseI != CI->use_end();
+          ++UseI) {
+        FPTruncInst *Cast = dyn_cast<FPTruncInst>(*UseI);
+        if (Cast == 0 || !Cast->getType()->isFloatTy())
+          return 0;
       }
-      return CI;
-    }
-
-    // See if either input string is a constant string.
-    StringRef SearchStr, ToFindStr;
-    bool HasStr1 = getConstantStringInfo(CI->getArgOperand(0), SearchStr);
-    bool HasStr2 = getConstantStringInfo(CI->getArgOperand(1), ToFindStr);
-
-    // fold strstr(x, "") -> x.
-    if (HasStr2 && ToFindStr.empty())
-      return B.CreateBitCast(CI->getArgOperand(0), CI->getType());
-
-    // If both strings are known, constant fold it.
-    if (HasStr1 && HasStr2) {
-      std::string::size_type Offset = SearchStr.find(ToFindStr);
-
-      if (Offset == StringRef::npos) // strstr("foo", "bar") -> null
-        return Constant::getNullValue(CI->getType());
-
-      // strstr("abcd", "bc") -> gep((char*)"abcd", 1)
-      Value *Result = CastToCStr(CI->getArgOperand(0), B);
-      Result = B.CreateConstInBoundsGEP1_64(Result, Offset, "strstr");
-      return B.CreateBitCast(Result, CI->getType());
-    }
-
-    // fold strstr(x, "y") -> strchr(x, 'y').
-    if (HasStr2 && ToFindStr.size() == 1) {
-      Value *StrChr= EmitStrChr(CI->getArgOperand(0), ToFindStr[0], B, TD, TLI);
-      return StrChr ? B.CreateBitCast(StrChr, CI->getType()) : 0;
     }
-    return 0;
-  }
-};
-
-
-//===---------------------------------------===//
-// 'memcmp' Optimizations
-
-struct MemCmpOpt : public LibCallOptimization {
-  virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
-    FunctionType *FT = Callee->getFunctionType();
-    if (FT->getNumParams() != 3 || !FT->getParamType(0)->isPointerTy() ||
-        !FT->getParamType(1)->isPointerTy() ||
-        !FT->getReturnType()->isIntegerTy(32))
-      return 0;
 
-    Value *LHS = CI->getArgOperand(0), *RHS = CI->getArgOperand(1);
-
-    if (LHS == RHS)  // memcmp(s,s,x) -> 0
-      return Constant::getNullValue(CI->getType());
-
-    // Make sure we have a constant length.
-    ConstantInt *LenC = dyn_cast<ConstantInt>(CI->getArgOperand(2));
-    if (!LenC) return 0;
-    uint64_t Len = LenC->getZExtValue();
-
-    if (Len == 0) // memcmp(s1,s2,0) -> 0
-      return Constant::getNullValue(CI->getType());
-
-    // memcmp(S1,S2,1) -> *(unsigned char*)LHS - *(unsigned char*)RHS
-    if (Len == 1) {
-      Value *LHSV = B.CreateZExt(B.CreateLoad(CastToCStr(LHS, B), "lhsc"),
-                                 CI->getType(), "lhsv");
-      Value *RHSV = B.CreateZExt(B.CreateLoad(CastToCStr(RHS, B), "rhsc"),
-                                 CI->getType(), "rhsv");
-      return B.CreateSub(LHSV, RHSV, "chardiff");
-    }
-
-    // Constant folding: memcmp(x, y, l) -> cnst (all arguments are constant)
-    StringRef LHSStr, RHSStr;
-    if (getConstantStringInfo(LHS, LHSStr) &&
-        getConstantStringInfo(RHS, RHSStr)) {
-      // Make sure we're not reading out-of-bounds memory.
-      if (Len > LHSStr.size() || Len > RHSStr.size())
-        return 0;
-      uint64_t Ret = memcmp(LHSStr.data(), RHSStr.data(), Len);
-      return ConstantInt::get(CI->getType(), Ret);
-    }
-
-    return 0;
-  }
-};
-
-//===---------------------------------------===//
-// 'memcpy' Optimizations
-
-struct MemCpyOpt : public LibCallOptimization {
-  virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
-    // These optimizations require TargetData.
-    if (!TD) return 0;
-
-    FunctionType *FT = Callee->getFunctionType();
-    if (FT->getNumParams() != 3 || FT->getReturnType() != FT->getParamType(0) ||
-        !FT->getParamType(0)->isPointerTy() ||
-        !FT->getParamType(1)->isPointerTy() ||
-        FT->getParamType(2) != TD->getIntPtrType(*Context))
-      return 0;
-
-    // memcpy(x, y, n) -> llvm.memcpy(x, y, n, 1)
-    B.CreateMemCpy(CI->getArgOperand(0), CI->getArgOperand(1),
-                   CI->getArgOperand(2), 1);
-    return CI->getArgOperand(0);
-  }
-};
-
-//===---------------------------------------===//
-// 'memmove' Optimizations
-
-struct MemMoveOpt : public LibCallOptimization {
-  virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
-    // These optimizations require TargetData.
-    if (!TD) return 0;
-
-    FunctionType *FT = Callee->getFunctionType();
-    if (FT->getNumParams() != 3 || FT->getReturnType() != FT->getParamType(0) ||
-        !FT->getParamType(0)->isPointerTy() ||
-        !FT->getParamType(1)->isPointerTy() ||
-        FT->getParamType(2) != TD->getIntPtrType(*Context))
-      return 0;
-
-    // memmove(x, y, n) -> llvm.memmove(x, y, n, 1)
-    B.CreateMemMove(CI->getArgOperand(0), CI->getArgOperand(1),
-                    CI->getArgOperand(2), 1);
-    return CI->getArgOperand(0);
-  }
-};
-
-//===---------------------------------------===//
-// 'memset' Optimizations
-
-struct MemSetOpt : public LibCallOptimization {
-  virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
-    // These optimizations require TargetData.
-    if (!TD) return 0;
-
-    FunctionType *FT = Callee->getFunctionType();
-    if (FT->getNumParams() != 3 || FT->getReturnType() != FT->getParamType(0) ||
-        !FT->getParamType(0)->isPointerTy() ||
-        !FT->getParamType(1)->isIntegerTy() ||
-        FT->getParamType(2) != TD->getIntPtrType(*Context))
+    // If this is something like 'floor((double)floatval)', convert to floorf.
+    FPExtInst *Cast = dyn_cast<FPExtInst>(CI->getArgOperand(0));
+    if (Cast == 0 || !Cast->getOperand(0)->getType()->isFloatTy())
       return 0;
 
-    // memset(p, v, n) -> llvm.memset(p, v, n, 1)
-    Value *Val = B.CreateIntCast(CI->getArgOperand(1), B.getInt8Ty(), false);
-    B.CreateMemSet(CI->getArgOperand(0), Val, CI->getArgOperand(2), 1);
-    return CI->getArgOperand(0);
+    // floor((double)floatval) -> (double)floorf(floatval)
+    Value *V = Cast->getOperand(0);
+    V = EmitUnaryFloatFnCall(V, Callee->getName(), B, Callee->getAttributes());
+    return B.CreateFPExt(V, B.getDoubleTy());
   }
 };
 
-//===----------------------------------------------------------------------===//
-// Math Library Optimizations
-//===----------------------------------------------------------------------===//
-
 //===---------------------------------------===//
 // 'cos*' Optimizations
-
 struct CosOpt : public LibCallOptimization {
   virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
+    Value *Ret = NULL;
+    if (UnsafeFPShrink && Callee->getName() == "cos" &&
+        TLI->has(LibFunc::cosf)) {
+      UnaryDoubleFPOpt UnsafeUnaryDoubleFP(true);
+      Ret = UnsafeUnaryDoubleFP.CallOptimizer(Callee, CI, B);
+    }
+
     FunctionType *FT = Callee->getFunctionType();
     // Just make sure this has 1 argument of FP type, which matches the
     // result type.
     if (FT->getNumParams() != 1 || FT->getReturnType() != FT->getParamType(0) ||
         !FT->getParamType(0)->isFloatingPointTy())
-      return 0;
+      return Ret;
 
     // cos(-x) -> cos(x)
     Value *Op1 = CI->getArgOperand(0);
@@ -910,7 +162,7 @@ struct CosOpt : public LibCallOptimization {
       BinaryOperator *BinExpr = cast<BinaryOperator>(Op1);
       return B.CreateCall(Callee, BinExpr->getOperand(1), "cos");
     }
-    return 0;
+    return Ret;
   }
 };
 
@@ -919,13 +171,20 @@ struct CosOpt : public LibCallOptimization {
 
 struct PowOpt : public LibCallOptimization {
   virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
+    Value *Ret = NULL;
+    if (UnsafeFPShrink && Callee->getName() == "pow" &&
+        TLI->has(LibFunc::powf)) {
+      UnaryDoubleFPOpt UnsafeUnaryDoubleFP(true);
+      Ret = UnsafeUnaryDoubleFP.CallOptimizer(Callee, CI, B);
+    }
+
     FunctionType *FT = Callee->getFunctionType();
     // Just make sure this has 2 arguments of the same FP type, which match the
     // result type.
     if (FT->getNumParams() != 2 || FT->getReturnType() != FT->getParamType(0) ||
         FT->getParamType(0) != FT->getParamType(1) ||
         !FT->getParamType(0)->isFloatingPointTy())
-      return 0;
+      return Ret;
 
     Value *Op1 = CI->getArgOperand(0), *Op2 = CI->getArgOperand(1);
     if (ConstantFP *Op1C = dyn_cast<ConstantFP>(Op1)) {
@@ -936,7 +195,7 @@ struct PowOpt : public LibCallOptimization {
     }
 
     ConstantFP *Op2C = dyn_cast<ConstantFP>(Op2);
-    if (Op2C == 0) return 0;
+    if (Op2C == 0) return Ret;
 
     if (Op2C->getValueAPF().isZero())  // pow(x, 0.0) -> 1.0
       return ConstantFP::get(CI->getType(), 1.0);
@@ -974,12 +233,19 @@ struct PowOpt : public LibCallOptimization {
 
 struct Exp2Opt : public LibCallOptimization {
   virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
+    Value *Ret = NULL;
+    if (UnsafeFPShrink && Callee->getName() == "exp2" &&
+        TLI->has(LibFunc::exp2)) {
+      UnaryDoubleFPOpt UnsafeUnaryDoubleFP(true);
+      Ret = UnsafeUnaryDoubleFP.CallOptimizer(Callee, CI, B);
+    }
+
     FunctionType *FT = Callee->getFunctionType();
     // Just make sure this has 1 argument of FP type, which matches the
     // result type.
     if (FT->getNumParams() != 1 || FT->getReturnType() != FT->getParamType(0) ||
         !FT->getParamType(0)->isFloatingPointTy())
-      return 0;
+      return Ret;
 
     Value *Op = CI->getArgOperand(0);
     // Turn exp2(sitofp(x)) -> ldexp(1.0, sext(x))  if sizeof(x) <= 32
@@ -1016,29 +282,7 @@ struct Exp2Opt : public LibCallOptimization {
 
       return CI;
     }
-    return 0;
-  }
-};
-
-//===---------------------------------------===//
-// Double -> Float Shrinking Optimizations for Unary Functions like 'floor'
-
-struct UnaryDoubleFPOpt : public LibCallOptimization {
-  virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
-    FunctionType *FT = Callee->getFunctionType();
-    if (FT->getNumParams() != 1 || !FT->getReturnType()->isDoubleTy() ||
-        !FT->getParamType(0)->isDoubleTy())
-      return 0;
-
-    // If this is something like 'floor((double)floatval)', convert to floorf.
-    FPExtInst *Cast = dyn_cast<FPExtInst>(CI->getArgOperand(0));
-    if (Cast == 0 || !Cast->getOperand(0)->getType()->isFloatTy())
-      return 0;
-
-    // floor((double)floatval) -> (double)floorf(floatval)
-    Value *V = Cast->getOperand(0);
-    V = EmitUnaryFloatFnCall(V, Callee->getName(), B, Callee->getAttributes());
-    return B.CreateFPExt(V, B.getDoubleTy());
+    return Ret;
   }
 };
 
@@ -1063,8 +307,8 @@ struct FFSOpt : public LibCallOptimization {
 
     // Constant fold.
     if (ConstantInt *CI = dyn_cast<ConstantInt>(Op)) {
-      if (CI->getValue() == 0)  // ffs(0) -> 0.
-        return Constant::getNullValue(CI->getType());
+      if (CI->isZero()) // ffs(0) -> 0.
+        return B.getInt32(0);
       // ffs(c) -> cttz(c)+1
       return B.getInt32(CI->getValue().countTrailingZeros() + 1);
     }
@@ -1267,7 +511,7 @@ struct SPrintFOpt : public LibCallOptimization {
         if (FormatStr[i] == '%')
           return 0; // we found a format specifier, bail out.
 
-      // These optimizations require TargetData.
+      // These optimizations require DataLayout.
       if (!TD) return 0;
 
       // sprintf(str, fmt) -> llvm.memcpy(str, fmt, strlen(fmt)+1, 1)
@@ -1297,7 +541,7 @@ struct SPrintFOpt : public LibCallOptimization {
     }
 
     if (FormatStr[1] == 's') {
-      // These optimizations require TargetData.
+      // These optimizations require DataLayout.
       if (!TD) return 0;
 
       // sprintf(dest, "%s", str) -> llvm.memcpy(dest, str, strlen(str)+1, 1)
@@ -1385,7 +629,7 @@ struct FWriteOpt : public LibCallOptimization {
 
 struct FPutsOpt : public LibCallOptimization {
   virtual Value *CallOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
-    // These optimizations require TargetData.
+    // These optimizations require DataLayout.
     if (!TD) return 0;
 
     // Require two pointers.  Also, we can't optimize if return value is used.
@@ -1422,7 +666,7 @@ struct FPrintFOpt : public LibCallOptimization {
         if (FormatStr[i] == '%')  // Could handle %% -> % if we cared.
           return 0; // We found a format specifier.
 
-      // These optimizations require TargetData.
+      // These optimizations require DataLayout.
       if (!TD) return 0;
 
       Value *NewCI = EmitFWrite(CI->getArgOperand(1),
@@ -1524,17 +768,9 @@ namespace {
     TargetLibraryInfo *TLI;
 
     StringMap<LibCallOptimization*> Optimizations;
-    // String and Memory LibCall Optimizations
-    StrCatOpt StrCat; StrNCatOpt StrNCat; StrChrOpt StrChr; StrRChrOpt StrRChr;
-    StrCmpOpt StrCmp; StrNCmpOpt StrNCmp;
-    StrCpyOpt StrCpy; StrCpyOpt StrCpyChk;
-    StpCpyOpt StpCpy; StpCpyOpt StpCpyChk;
-    StrNCpyOpt StrNCpy;
-    StrLenOpt StrLen; StrPBrkOpt StrPBrk;
-    StrToOpt StrTo; StrSpnOpt StrSpn; StrCSpnOpt StrCSpn; StrStrOpt StrStr;
-    MemCmpOpt MemCmp; MemCpyOpt MemCpy; MemMoveOpt MemMove; MemSetOpt MemSet;
     // Math Library Optimizations
-    CosOpt Cos; PowOpt Pow; Exp2Opt Exp2; UnaryDoubleFPOpt UnaryDoubleFP;
+    CosOpt Cos; PowOpt Pow; Exp2Opt Exp2;
+    UnaryDoubleFPOpt UnaryDoubleFP, UnsafeUnaryDoubleFP;
     // Integer Optimizations
     FFSOpt FFS; AbsOpt Abs; IsDigitOpt IsDigit; IsAsciiOpt IsAscii;
     ToAsciiOpt ToAscii;
@@ -1546,11 +782,13 @@ namespace {
     bool Modified;  // This is only used by doInitialization.
   public:
     static char ID; // Pass identification
-    SimplifyLibCalls() : FunctionPass(ID), StrCpy(false), StrCpyChk(true),
-                         StpCpy(false), StpCpyChk(true) {
+    SimplifyLibCalls() : FunctionPass(ID), UnaryDoubleFP(false),
+                         UnsafeUnaryDoubleFP(true) {
       initializeSimplifyLibCallsPass(*PassRegistry::getPassRegistry());
     }
     void AddOpt(LibFunc::Func F, LibCallOptimization* Opt);
+    void AddOpt(LibFunc::Func F1, LibFunc::Func F2, LibCallOptimization* Opt);
+
     void InitOptimizations();
     bool runOnFunction(Function &F);
 
@@ -1586,40 +824,15 @@ void SimplifyLibCalls::AddOpt(LibFunc::Func F, LibCallOptimization* Opt) {
     Optimizations[TLI->getName(F)] = Opt;
 }
 
+void SimplifyLibCalls::AddOpt(LibFunc::Func F1, LibFunc::Func F2,
+                              LibCallOptimization* Opt) {
+  if (TLI->has(F1) && TLI->has(F2))
+    Optimizations[TLI->getName(F1)] = Opt;
+}
+
 /// Optimizations - Populate the Optimizations map with all the optimizations
 /// we know.
 void SimplifyLibCalls::InitOptimizations() {
-  // String and Memory LibCall Optimizations
-  Optimizations["strcat"] = &StrCat;
-  Optimizations["strncat"] = &StrNCat;
-  Optimizations["strchr"] = &StrChr;
-  Optimizations["strrchr"] = &StrRChr;
-  Optimizations["strcmp"] = &StrCmp;
-  Optimizations["strncmp"] = &StrNCmp;
-  Optimizations["strcpy"] = &StrCpy;
-  Optimizations["strncpy"] = &StrNCpy;
-  Optimizations["stpcpy"] = &StpCpy;
-  Optimizations["strlen"] = &StrLen;
-  Optimizations["strpbrk"] = &StrPBrk;
-  Optimizations["strtol"] = &StrTo;
-  Optimizations["strtod"] = &StrTo;
-  Optimizations["strtof"] = &StrTo;
-  Optimizations["strtoul"] = &StrTo;
-  Optimizations["strtoll"] = &StrTo;
-  Optimizations["strtold"] = &StrTo;
-  Optimizations["strtoull"] = &StrTo;
-  Optimizations["strspn"] = &StrSpn;
-  Optimizations["strcspn"] = &StrCSpn;
-  Optimizations["strstr"] = &StrStr;
-  Optimizations["memcmp"] = &MemCmp;
-  AddOpt(LibFunc::memcpy, &MemCpy);
-  Optimizations["memmove"] = &MemMove;
-  AddOpt(LibFunc::memset, &MemSet);
-
-  // _chk variants of String and Memory LibCall Optimizations.
-  Optimizations["__strcpy_chk"] = &StrCpyChk;
-  Optimizations["__stpcpy_chk"] = &StpCpyChk;
-
   // Math Library Optimizations
   Optimizations["cosf"] = &Cos;
   Optimizations["cos"] = &Cos;
@@ -1641,16 +854,37 @@ void SimplifyLibCalls::InitOptimizations() {
   Optimizations["llvm.exp2.f64"] = &Exp2;
   Optimizations["llvm.exp2.f32"] = &Exp2;
 
-  if (TLI->has(LibFunc::floor) && TLI->has(LibFunc::floorf))
-    Optimizations["floor"] = &UnaryDoubleFP;
-  if (TLI->has(LibFunc::ceil) && TLI->has(LibFunc::ceilf))
-    Optimizations["ceil"] = &UnaryDoubleFP;
-  if (TLI->has(LibFunc::round) && TLI->has(LibFunc::roundf))
-    Optimizations["round"] = &UnaryDoubleFP;
-  if (TLI->has(LibFunc::rint) && TLI->has(LibFunc::rintf))
-    Optimizations["rint"] = &UnaryDoubleFP;
-  if (TLI->has(LibFunc::nearbyint) && TLI->has(LibFunc::nearbyintf))
-    Optimizations["nearbyint"] = &UnaryDoubleFP;
+  AddOpt(LibFunc::ceil, LibFunc::ceilf, &UnaryDoubleFP);
+  AddOpt(LibFunc::fabs, LibFunc::fabsf, &UnaryDoubleFP);
+  AddOpt(LibFunc::floor, LibFunc::floorf, &UnaryDoubleFP);
+  AddOpt(LibFunc::rint, LibFunc::rintf, &UnaryDoubleFP);
+  AddOpt(LibFunc::round, LibFunc::roundf, &UnaryDoubleFP);
+  AddOpt(LibFunc::nearbyint, LibFunc::nearbyintf, &UnaryDoubleFP);
+  AddOpt(LibFunc::trunc, LibFunc::truncf, &UnaryDoubleFP);
+
+  if(UnsafeFPShrink) {
+    AddOpt(LibFunc::acos, LibFunc::acosf, &UnsafeUnaryDoubleFP);
+    AddOpt(LibFunc::acosh, LibFunc::acoshf, &UnsafeUnaryDoubleFP);
+    AddOpt(LibFunc::asin, LibFunc::asinf, &UnsafeUnaryDoubleFP);
+    AddOpt(LibFunc::asinh, LibFunc::asinhf, &UnsafeUnaryDoubleFP);
+    AddOpt(LibFunc::atan, LibFunc::atanf, &UnsafeUnaryDoubleFP);
+    AddOpt(LibFunc::atanh, LibFunc::atanhf, &UnsafeUnaryDoubleFP);
+    AddOpt(LibFunc::cbrt, LibFunc::cbrtf, &UnsafeUnaryDoubleFP);
+    AddOpt(LibFunc::cosh, LibFunc::coshf, &UnsafeUnaryDoubleFP);
+    AddOpt(LibFunc::exp, LibFunc::expf, &UnsafeUnaryDoubleFP);
+    AddOpt(LibFunc::exp10, LibFunc::exp10f, &UnsafeUnaryDoubleFP);
+    AddOpt(LibFunc::expm1, LibFunc::expm1f, &UnsafeUnaryDoubleFP);
+    AddOpt(LibFunc::log, LibFunc::logf, &UnsafeUnaryDoubleFP);
+    AddOpt(LibFunc::log10, LibFunc::log10f, &UnsafeUnaryDoubleFP);
+    AddOpt(LibFunc::log1p, LibFunc::log1pf, &UnsafeUnaryDoubleFP);
+    AddOpt(LibFunc::log2, LibFunc::log2f, &UnsafeUnaryDoubleFP);
+    AddOpt(LibFunc::logb, LibFunc::logbf, &UnsafeUnaryDoubleFP);
+    AddOpt(LibFunc::sin, LibFunc::sinf, &UnsafeUnaryDoubleFP);
+    AddOpt(LibFunc::sinh, LibFunc::sinhf, &UnsafeUnaryDoubleFP);
+    AddOpt(LibFunc::sqrt, LibFunc::sqrtf, &UnsafeUnaryDoubleFP);
+    AddOpt(LibFunc::tan, LibFunc::tanf, &UnsafeUnaryDoubleFP);
+    AddOpt(LibFunc::tanh, LibFunc::tanhf, &UnsafeUnaryDoubleFP);
+  }
 
   // Integer Optimizations
   Optimizations["ffs"] = &FFS;
@@ -1681,7 +915,7 @@ bool SimplifyLibCalls::runOnFunction(Function &F) {
   if (Optimizations.empty())
     InitOptimizations();
 
-  const TargetData *TD = getAnalysisIfAvailable<TargetData>();
+  const DataLayout *TD = getAnalysisIfAvailable<DataLayout>();
 
   IRBuilder<> Builder(F.getContext());
 
diff --git a/lib/Transforms/Utils/AddrModeMatcher.cpp b/lib/Transforms/Utils/AddrModeMatcher.cpp
index d831452..6815e41 100644
--- a/lib/Transforms/Utils/AddrModeMatcher.cpp
+++ b/lib/Transforms/Utils/AddrModeMatcher.cpp
@@ -16,7 +16,7 @@
 #include "llvm/GlobalValue.h"
 #include "llvm/Instruction.h"
 #include "llvm/Assembly/Writer.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/GetElementPtrTypeIterator.h"
 #include "llvm/Support/PatternMatch.h"
@@ -55,10 +55,12 @@ void ExtAddrMode::print(raw_ostream &OS) const {
   OS << ']';
 }
 
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
 void ExtAddrMode::dump() const {
   print(dbgs());
   dbgs() << '\n';
 }
+#endif
 
 
 /// MatchScaledValue - Try adding ScaleReg*Scale to the current addressing mode.
@@ -219,7 +221,7 @@ bool AddressingModeMatcher::MatchOperationAddr(User *AddrInst, unsigned Opcode,
     unsigned VariableScale = 0;
     
     int64_t ConstantOffset = 0;
-    const TargetData *TD = TLI.getTargetData();
+    const DataLayout *TD = TLI.getDataLayout();
     gep_type_iterator GTI = gep_type_begin(AddrInst);
     for (unsigned i = 1, e = AddrInst->getNumOperands(); i != e; ++i, ++GTI) {
       if (StructType *STy = dyn_cast<StructType>(*GTI)) {
diff --git a/lib/Transforms/Utils/BasicBlockUtils.cpp b/lib/Transforms/Utils/BasicBlockUtils.cpp
index 2679b93..9fea113 100644
--- a/lib/Transforms/Utils/BasicBlockUtils.cpp
+++ b/lib/Transforms/Utils/BasicBlockUtils.cpp
@@ -22,7 +22,7 @@
 #include "llvm/Analysis/Dominators.h"
 #include "llvm/Analysis/LoopInfo.h"
 #include "llvm/Analysis/MemoryDependenceAnalysis.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Transforms/Utils/Local.h"
 #include "llvm/Transforms/Scalar.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -94,7 +94,7 @@ void llvm::FoldSingleEntryPHINodes(BasicBlock *BB, Pass *P) {
 /// is dead. Also recursively delete any operands that become dead as
 /// a result. This includes tracing the def-use list from the PHI to see if
 /// it is ultimately unused or if it reaches an unused cycle.
-bool llvm::DeleteDeadPHIs(BasicBlock *BB) {
+bool llvm::DeleteDeadPHIs(BasicBlock *BB, const TargetLibraryInfo *TLI) {
   // Recursively deleting a PHI may cause multiple PHIs to be deleted
   // or RAUW'd undef, so use an array of WeakVH for the PHIs to delete.
   SmallVector<WeakVH, 8> PHIs;
@@ -105,7 +105,7 @@ bool llvm::DeleteDeadPHIs(BasicBlock *BB) {
   bool Changed = false;
   for (unsigned i = 0, e = PHIs.size(); i != e; ++i)
     if (PHINode *PN = dyn_cast_or_null<PHINode>(PHIs[i].operator Value*()))
-      Changed |= RecursivelyDeleteDeadPHINode(PN);
+      Changed |= RecursivelyDeleteDeadPHINode(PN, TLI);
 
   return Changed;
 }
@@ -687,3 +687,42 @@ ReturnInst *llvm::FoldReturnIntoUncondBranch(ReturnInst *RI, BasicBlock *BB,
   return cast<ReturnInst>(NewRet);
 }
 
+/// SplitBlockAndInsertIfThen - Split the containing block at the
+/// specified instruction - everything before and including Cmp stays
+/// in the old basic block, and everything after Cmp is moved to a
+/// new block. The two blocks are connected by a conditional branch
+/// (with value of Cmp being the condition).
+/// Before:
+///   Head
+///   Cmp
+///   Tail
+/// After:
+///   Head
+///   Cmp
+///   if (Cmp)
+///     ThenBlock
+///   Tail
+///
+/// If Unreachable is true, then ThenBlock ends with
+/// UnreachableInst, otherwise it branches to Tail.
+/// Returns the NewBasicBlock's terminator.
+
+TerminatorInst *llvm::SplitBlockAndInsertIfThen(Instruction *Cmp,
+    bool Unreachable, MDNode *BranchWeights) {
+  Instruction *SplitBefore = Cmp->getNextNode();
+  BasicBlock *Head = SplitBefore->getParent();
+  BasicBlock *Tail = Head->splitBasicBlock(SplitBefore);
+  TerminatorInst *HeadOldTerm = Head->getTerminator();
+  LLVMContext &C = Head->getContext();
+  BasicBlock *ThenBlock = BasicBlock::Create(C, "", Head->getParent(), Tail);
+  TerminatorInst *CheckTerm;
+  if (Unreachable)
+    CheckTerm = new UnreachableInst(C, ThenBlock);
+  else
+    CheckTerm = BranchInst::Create(Tail, ThenBlock);
+  BranchInst *HeadNewTerm =
+    BranchInst::Create(/*ifTrue*/ThenBlock, /*ifFalse*/Tail, Cmp);
+  HeadNewTerm->setMetadata(LLVMContext::MD_prof, BranchWeights);
+  ReplaceInstWithInst(HeadOldTerm, HeadNewTerm);
+  return CheckTerm;
+}
diff --git a/lib/Transforms/Utils/BuildLibCalls.cpp b/lib/Transforms/Utils/BuildLibCalls.cpp
index e13fd71..74b2ee1 100644
--- a/lib/Transforms/Utils/BuildLibCalls.cpp
+++ b/lib/Transforms/Utils/BuildLibCalls.cpp
@@ -22,7 +22,7 @@
 #include "llvm/Module.h"
 #include "llvm/Type.h"
 #include "llvm/ADT/SmallString.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetLibraryInfo.h"
 
 using namespace llvm;
@@ -34,19 +34,22 @@ Value *llvm::CastToCStr(Value *V, IRBuilder<> &B) {
 
 /// EmitStrLen - Emit a call to the strlen function to the builder, for the
 /// specified pointer.  This always returns an integer value of size intptr_t.
-Value *llvm::EmitStrLen(Value *Ptr, IRBuilder<> &B, const TargetData *TD,
+Value *llvm::EmitStrLen(Value *Ptr, IRBuilder<> &B, const DataLayout *TD,
                         const TargetLibraryInfo *TLI) {
   if (!TLI->has(LibFunc::strlen))
     return 0;
 
   Module *M = B.GetInsertBlock()->getParent()->getParent();
   AttributeWithIndex AWI[2];
-  AWI[0] = AttributeWithIndex::get(1, Attribute::NoCapture);
-  AWI[1] = AttributeWithIndex::get(~0u, Attribute::ReadOnly |
-                                   Attribute::NoUnwind);
+  AWI[0] = AttributeWithIndex::get(M->getContext(), 1, Attributes::NoCapture);
+  Attributes::AttrVal AVs[2] = { Attributes::ReadOnly, Attributes::NoUnwind };
+  AWI[1] = AttributeWithIndex::get(M->getContext(), AttrListPtr::FunctionIndex,
+                                   ArrayRef<Attributes::AttrVal>(AVs, 2));
 
   LLVMContext &Context = B.GetInsertBlock()->getContext();
-  Constant *StrLen = M->getOrInsertFunction("strlen", AttrListPtr::get(AWI),
+  Constant *StrLen = M->getOrInsertFunction("strlen",
+                                            AttrListPtr::get(M->getContext(),
+                                                             AWI),
                                             TD->getIntPtrType(Context),
                                             B.getInt8PtrTy(),
                                             NULL);
@@ -61,18 +64,21 @@ Value *llvm::EmitStrLen(Value *Ptr, IRBuilder<> &B, const TargetData *TD,
 /// specified pointer.  Ptr is required to be some pointer type, MaxLen must
 /// be of size_t type, and the return value has 'intptr_t' type.
 Value *llvm::EmitStrNLen(Value *Ptr, Value *MaxLen, IRBuilder<> &B,
-                         const TargetData *TD, const TargetLibraryInfo *TLI) {
+                         const DataLayout *TD, const TargetLibraryInfo *TLI) {
   if (!TLI->has(LibFunc::strnlen))
     return 0;
 
   Module *M = B.GetInsertBlock()->getParent()->getParent();
   AttributeWithIndex AWI[2];
-  AWI[0] = AttributeWithIndex::get(1, Attribute::NoCapture);
-  AWI[1] = AttributeWithIndex::get(~0u, Attribute::ReadOnly |
-                                   Attribute::NoUnwind);
+  AWI[0] = AttributeWithIndex::get(M->getContext(), 1, Attributes::NoCapture);
+  Attributes::AttrVal AVs[2] = { Attributes::ReadOnly, Attributes::NoUnwind };
+  AWI[1] = AttributeWithIndex::get(M->getContext(), AttrListPtr::FunctionIndex,
+                                   ArrayRef<Attributes::AttrVal>(AVs, 2));
 
   LLVMContext &Context = B.GetInsertBlock()->getContext();
-  Constant *StrNLen = M->getOrInsertFunction("strnlen", AttrListPtr::get(AWI),
+  Constant *StrNLen = M->getOrInsertFunction("strnlen",
+                                             AttrListPtr::get(M->getContext(),
+                                                              AWI),
                                              TD->getIntPtrType(Context),
                                              B.getInt8PtrTy(),
                                              TD->getIntPtrType(Context),
@@ -88,17 +94,21 @@ Value *llvm::EmitStrNLen(Value *Ptr, Value *MaxLen, IRBuilder<> &B,
 /// specified pointer and character.  Ptr is required to be some pointer type,
 /// and the return value has 'i8*' type.
 Value *llvm::EmitStrChr(Value *Ptr, char C, IRBuilder<> &B,
-                        const TargetData *TD, const TargetLibraryInfo *TLI) {
+                        const DataLayout *TD, const TargetLibraryInfo *TLI) {
   if (!TLI->has(LibFunc::strchr))
     return 0;
 
   Module *M = B.GetInsertBlock()->getParent()->getParent();
+  Attributes::AttrVal AVs[2] = { Attributes::ReadOnly, Attributes::NoUnwind };
   AttributeWithIndex AWI =
-    AttributeWithIndex::get(~0u, Attribute::ReadOnly | Attribute::NoUnwind);
+    AttributeWithIndex::get(M->getContext(), AttrListPtr::FunctionIndex,
+                            ArrayRef<Attributes::AttrVal>(AVs, 2));
 
   Type *I8Ptr = B.getInt8PtrTy();
   Type *I32Ty = B.getInt32Ty();
-  Constant *StrChr = M->getOrInsertFunction("strchr", AttrListPtr::get(AWI),
+  Constant *StrChr = M->getOrInsertFunction("strchr",
+                                            AttrListPtr::get(M->getContext(),
+                                                             AWI),
                                             I8Ptr, I8Ptr, I32Ty, NULL);
   CallInst *CI = B.CreateCall2(StrChr, CastToCStr(Ptr, B),
                                ConstantInt::get(I32Ty, C), "strchr");
@@ -109,20 +119,23 @@ Value *llvm::EmitStrChr(Value *Ptr, char C, IRBuilder<> &B,
 
 /// EmitStrNCmp - Emit a call to the strncmp function to the builder.
 Value *llvm::EmitStrNCmp(Value *Ptr1, Value *Ptr2, Value *Len,
-                         IRBuilder<> &B, const TargetData *TD,
+                         IRBuilder<> &B, const DataLayout *TD,
                          const TargetLibraryInfo *TLI) {
   if (!TLI->has(LibFunc::strncmp))
     return 0;
 
   Module *M = B.GetInsertBlock()->getParent()->getParent();
   AttributeWithIndex AWI[3];
-  AWI[0] = AttributeWithIndex::get(1, Attribute::NoCapture);
-  AWI[1] = AttributeWithIndex::get(2, Attribute::NoCapture);
-  AWI[2] = AttributeWithIndex::get(~0u, Attribute::ReadOnly |
-                                   Attribute::NoUnwind);
+  AWI[0] = AttributeWithIndex::get(M->getContext(), 1, Attributes::NoCapture);
+  AWI[1] = AttributeWithIndex::get(M->getContext(), 2, Attributes::NoCapture);
+  Attributes::AttrVal AVs[2] = { Attributes::ReadOnly, Attributes::NoUnwind };
+  AWI[2] = AttributeWithIndex::get(M->getContext(), AttrListPtr::FunctionIndex,
+                                   ArrayRef<Attributes::AttrVal>(AVs, 2));
 
   LLVMContext &Context = B.GetInsertBlock()->getContext();
-  Value *StrNCmp = M->getOrInsertFunction("strncmp", AttrListPtr::get(AWI),
+  Value *StrNCmp = M->getOrInsertFunction("strncmp",
+                                          AttrListPtr::get(M->getContext(),
+                                                           AWI),
                                           B.getInt32Ty(),
                                           B.getInt8PtrTy(),
                                           B.getInt8PtrTy(),
@@ -139,17 +152,19 @@ Value *llvm::EmitStrNCmp(Value *Ptr1, Value *Ptr2, Value *Len,
 /// EmitStrCpy - Emit a call to the strcpy function to the builder, for the
 /// specified pointer arguments.
 Value *llvm::EmitStrCpy(Value *Dst, Value *Src, IRBuilder<> &B,
-                        const TargetData *TD, const TargetLibraryInfo *TLI,
+                        const DataLayout *TD, const TargetLibraryInfo *TLI,
                         StringRef Name) {
   if (!TLI->has(LibFunc::strcpy))
     return 0;
 
   Module *M = B.GetInsertBlock()->getParent()->getParent();
   AttributeWithIndex AWI[2];
-  AWI[0] = AttributeWithIndex::get(2, Attribute::NoCapture);
-  AWI[1] = AttributeWithIndex::get(~0u, Attribute::NoUnwind);
+  AWI[0] = AttributeWithIndex::get(M->getContext(), 2, Attributes::NoCapture);
+  AWI[1] = AttributeWithIndex::get(M->getContext(), AttrListPtr::FunctionIndex,
+                                   Attributes::NoUnwind);
   Type *I8Ptr = B.getInt8PtrTy();
-  Value *StrCpy = M->getOrInsertFunction(Name, AttrListPtr::get(AWI),
+  Value *StrCpy = M->getOrInsertFunction(Name,
+                                         AttrListPtr::get(M->getContext(), AWI),
                                          I8Ptr, I8Ptr, I8Ptr, NULL);
   CallInst *CI = B.CreateCall2(StrCpy, CastToCStr(Dst, B), CastToCStr(Src, B),
                                Name);
@@ -161,17 +176,20 @@ Value *llvm::EmitStrCpy(Value *Dst, Value *Src, IRBuilder<> &B,
 /// EmitStrNCpy - Emit a call to the strncpy function to the builder, for the
 /// specified pointer arguments.
 Value *llvm::EmitStrNCpy(Value *Dst, Value *Src, Value *Len,
-                         IRBuilder<> &B, const TargetData *TD,
+                         IRBuilder<> &B, const DataLayout *TD,
                          const TargetLibraryInfo *TLI, StringRef Name) {
   if (!TLI->has(LibFunc::strncpy))
     return 0;
 
   Module *M = B.GetInsertBlock()->getParent()->getParent();
   AttributeWithIndex AWI[2];
-  AWI[0] = AttributeWithIndex::get(2, Attribute::NoCapture);
-  AWI[1] = AttributeWithIndex::get(~0u, Attribute::NoUnwind);
+  AWI[0] = AttributeWithIndex::get(M->getContext(), 2, Attributes::NoCapture);
+  AWI[1] = AttributeWithIndex::get(M->getContext(), AttrListPtr::FunctionIndex,
+                                   Attributes::NoUnwind);
   Type *I8Ptr = B.getInt8PtrTy();
-  Value *StrNCpy = M->getOrInsertFunction(Name, AttrListPtr::get(AWI),
+  Value *StrNCpy = M->getOrInsertFunction(Name,
+                                          AttrListPtr::get(M->getContext(),
+                                                           AWI),
                                           I8Ptr, I8Ptr, I8Ptr,
                                           Len->getType(), NULL);
   CallInst *CI = B.CreateCall3(StrNCpy, CastToCStr(Dst, B), CastToCStr(Src, B),
@@ -185,17 +203,18 @@ Value *llvm::EmitStrNCpy(Value *Dst, Value *Src, Value *Len,
 /// This expects that the Len and ObjSize have type 'intptr_t' and Dst/Src
 /// are pointers.
 Value *llvm::EmitMemCpyChk(Value *Dst, Value *Src, Value *Len, Value *ObjSize,
-                           IRBuilder<> &B, const TargetData *TD,
+                           IRBuilder<> &B, const DataLayout *TD,
                            const TargetLibraryInfo *TLI) {
   if (!TLI->has(LibFunc::memcpy_chk))
     return 0;
 
   Module *M = B.GetInsertBlock()->getParent()->getParent();
   AttributeWithIndex AWI;
-  AWI = AttributeWithIndex::get(~0u, Attribute::NoUnwind);
+  AWI = AttributeWithIndex::get(M->getContext(), AttrListPtr::FunctionIndex,
+                                Attributes::NoUnwind);
   LLVMContext &Context = B.GetInsertBlock()->getContext();
   Value *MemCpy = M->getOrInsertFunction("__memcpy_chk",
-                                         AttrListPtr::get(AWI),
+                                         AttrListPtr::get(M->getContext(), AWI),
                                          B.getInt8PtrTy(),
                                          B.getInt8PtrTy(),
                                          B.getInt8PtrTy(),
@@ -212,16 +231,19 @@ Value *llvm::EmitMemCpyChk(Value *Dst, Value *Src, Value *Len, Value *ObjSize,
 /// EmitMemChr - Emit a call to the memchr function.  This assumes that Ptr is
 /// a pointer, Val is an i32 value, and Len is an 'intptr_t' value.
 Value *llvm::EmitMemChr(Value *Ptr, Value *Val,
-                        Value *Len, IRBuilder<> &B, const TargetData *TD,
+                        Value *Len, IRBuilder<> &B, const DataLayout *TD,
                         const TargetLibraryInfo *TLI) {
   if (!TLI->has(LibFunc::memchr))
     return 0;
 
   Module *M = B.GetInsertBlock()->getParent()->getParent();
   AttributeWithIndex AWI;
-  AWI = AttributeWithIndex::get(~0u, Attribute::ReadOnly | Attribute::NoUnwind);
+  Attributes::AttrVal AVs[2] = { Attributes::ReadOnly, Attributes::NoUnwind };
+  AWI = AttributeWithIndex::get(M->getContext(), AttrListPtr::FunctionIndex,
+                                ArrayRef<Attributes::AttrVal>(AVs, 2));
   LLVMContext &Context = B.GetInsertBlock()->getContext();
-  Value *MemChr = M->getOrInsertFunction("memchr", AttrListPtr::get(AWI),
+  Value *MemChr = M->getOrInsertFunction("memchr",
+                                         AttrListPtr::get(M->getContext(), AWI),
                                          B.getInt8PtrTy(),
                                          B.getInt8PtrTy(),
                                          B.getInt32Ty(),
@@ -237,20 +259,22 @@ Value *llvm::EmitMemChr(Value *Ptr, Value *Val,
 
 /// EmitMemCmp - Emit a call to the memcmp function.
 Value *llvm::EmitMemCmp(Value *Ptr1, Value *Ptr2,
-                        Value *Len, IRBuilder<> &B, const TargetData *TD,
+                        Value *Len, IRBuilder<> &B, const DataLayout *TD,
                         const TargetLibraryInfo *TLI) {
   if (!TLI->has(LibFunc::memcmp))
     return 0;
 
   Module *M = B.GetInsertBlock()->getParent()->getParent();
   AttributeWithIndex AWI[3];
-  AWI[0] = AttributeWithIndex::get(1, Attribute::NoCapture);
-  AWI[1] = AttributeWithIndex::get(2, Attribute::NoCapture);
-  AWI[2] = AttributeWithIndex::get(~0u, Attribute::ReadOnly |
-                                   Attribute::NoUnwind);
+  AWI[0] = AttributeWithIndex::get(M->getContext(), 1, Attributes::NoCapture);
+  AWI[1] = AttributeWithIndex::get(M->getContext(), 2, Attributes::NoCapture);
+  Attributes::AttrVal AVs[2] = { Attributes::ReadOnly, Attributes::NoUnwind };
+  AWI[2] = AttributeWithIndex::get(M->getContext(), AttrListPtr::FunctionIndex,
+                                   ArrayRef<Attributes::AttrVal>(AVs, 2));
 
   LLVMContext &Context = B.GetInsertBlock()->getContext();
-  Value *MemCmp = M->getOrInsertFunction("memcmp", AttrListPtr::get(AWI),
+  Value *MemCmp = M->getOrInsertFunction("memcmp",
+                                         AttrListPtr::get(M->getContext(), AWI),
                                          B.getInt32Ty(),
                                          B.getInt8PtrTy(),
                                          B.getInt8PtrTy(),
@@ -294,7 +318,7 @@ Value *llvm::EmitUnaryFloatFnCall(Value *Op, StringRef Name, IRBuilder<> &B,
 
 /// EmitPutChar - Emit a call to the putchar function.  This assumes that Char
 /// is an integer.
-Value *llvm::EmitPutChar(Value *Char, IRBuilder<> &B, const TargetData *TD,
+Value *llvm::EmitPutChar(Value *Char, IRBuilder<> &B, const DataLayout *TD,
                          const TargetLibraryInfo *TLI) {
   if (!TLI->has(LibFunc::putchar))
     return 0;
@@ -316,17 +340,19 @@ Value *llvm::EmitPutChar(Value *Char, IRBuilder<> &B, const TargetData *TD,
 
 /// EmitPutS - Emit a call to the puts function.  This assumes that Str is
 /// some pointer.
-Value *llvm::EmitPutS(Value *Str, IRBuilder<> &B, const TargetData *TD,
+Value *llvm::EmitPutS(Value *Str, IRBuilder<> &B, const DataLayout *TD,
                       const TargetLibraryInfo *TLI) {
   if (!TLI->has(LibFunc::puts))
     return 0;
 
   Module *M = B.GetInsertBlock()->getParent()->getParent();
   AttributeWithIndex AWI[2];
-  AWI[0] = AttributeWithIndex::get(1, Attribute::NoCapture);
-  AWI[1] = AttributeWithIndex::get(~0u, Attribute::NoUnwind);
+  AWI[0] = AttributeWithIndex::get(M->getContext(), 1, Attributes::NoCapture);
+  AWI[1] = AttributeWithIndex::get(M->getContext(), AttrListPtr::FunctionIndex,
+                                   Attributes::NoUnwind);
 
-  Value *PutS = M->getOrInsertFunction("puts", AttrListPtr::get(AWI),
+  Value *PutS = M->getOrInsertFunction("puts",
+                                       AttrListPtr::get(M->getContext(), AWI),
                                        B.getInt32Ty(),
                                        B.getInt8PtrTy(),
                                        NULL);
@@ -339,17 +365,19 @@ Value *llvm::EmitPutS(Value *Str, IRBuilder<> &B, const TargetData *TD,
 /// EmitFPutC - Emit a call to the fputc function.  This assumes that Char is
 /// an integer and File is a pointer to FILE.
 Value *llvm::EmitFPutC(Value *Char, Value *File, IRBuilder<> &B,
-                       const TargetData *TD, const TargetLibraryInfo *TLI) {
+                       const DataLayout *TD, const TargetLibraryInfo *TLI) {
   if (!TLI->has(LibFunc::fputc))
     return 0;
 
   Module *M = B.GetInsertBlock()->getParent()->getParent();
   AttributeWithIndex AWI[2];
-  AWI[0] = AttributeWithIndex::get(2, Attribute::NoCapture);
-  AWI[1] = AttributeWithIndex::get(~0u, Attribute::NoUnwind);
+  AWI[0] = AttributeWithIndex::get(M->getContext(), 2, Attributes::NoCapture);
+  AWI[1] = AttributeWithIndex::get(M->getContext(), AttrListPtr::FunctionIndex,
+                                   Attributes::NoUnwind);
   Constant *F;
   if (File->getType()->isPointerTy())
-    F = M->getOrInsertFunction("fputc", AttrListPtr::get(AWI),
+    F = M->getOrInsertFunction("fputc",
+                               AttrListPtr::get(M->getContext(), AWI),
                                B.getInt32Ty(),
                                B.getInt32Ty(), File->getType(),
                                NULL);
@@ -370,19 +398,21 @@ Value *llvm::EmitFPutC(Value *Char, Value *File, IRBuilder<> &B,
 /// EmitFPutS - Emit a call to the puts function.  Str is required to be a
 /// pointer and File is a pointer to FILE.
 Value *llvm::EmitFPutS(Value *Str, Value *File, IRBuilder<> &B,
-                       const TargetData *TD, const TargetLibraryInfo *TLI) {
+                       const DataLayout *TD, const TargetLibraryInfo *TLI) {
   if (!TLI->has(LibFunc::fputs))
     return 0;
 
   Module *M = B.GetInsertBlock()->getParent()->getParent();
   AttributeWithIndex AWI[3];
-  AWI[0] = AttributeWithIndex::get(1, Attribute::NoCapture);
-  AWI[1] = AttributeWithIndex::get(2, Attribute::NoCapture);
-  AWI[2] = AttributeWithIndex::get(~0u, Attribute::NoUnwind);
+  AWI[0] = AttributeWithIndex::get(M->getContext(), 1, Attributes::NoCapture);
+  AWI[1] = AttributeWithIndex::get(M->getContext(), 2, Attributes::NoCapture);
+  AWI[2] = AttributeWithIndex::get(M->getContext(), AttrListPtr::FunctionIndex,
+                                   Attributes::NoUnwind);
   StringRef FPutsName = TLI->getName(LibFunc::fputs);
   Constant *F;
   if (File->getType()->isPointerTy())
-    F = M->getOrInsertFunction(FPutsName, AttrListPtr::get(AWI),
+    F = M->getOrInsertFunction(FPutsName,
+                               AttrListPtr::get(M->getContext(), AWI),
                                B.getInt32Ty(),
                                B.getInt8PtrTy(),
                                File->getType(), NULL);
@@ -400,21 +430,23 @@ Value *llvm::EmitFPutS(Value *Str, Value *File, IRBuilder<> &B,
 /// EmitFWrite - Emit a call to the fwrite function.  This assumes that Ptr is
 /// a pointer, Size is an 'intptr_t', and File is a pointer to FILE.
 Value *llvm::EmitFWrite(Value *Ptr, Value *Size, Value *File,
-                        IRBuilder<> &B, const TargetData *TD,
+                        IRBuilder<> &B, const DataLayout *TD,
                         const TargetLibraryInfo *TLI) {
   if (!TLI->has(LibFunc::fwrite))
     return 0;
 
   Module *M = B.GetInsertBlock()->getParent()->getParent();
   AttributeWithIndex AWI[3];
-  AWI[0] = AttributeWithIndex::get(1, Attribute::NoCapture);
-  AWI[1] = AttributeWithIndex::get(4, Attribute::NoCapture);
-  AWI[2] = AttributeWithIndex::get(~0u, Attribute::NoUnwind);
+  AWI[0] = AttributeWithIndex::get(M->getContext(), 1, Attributes::NoCapture);
+  AWI[1] = AttributeWithIndex::get(M->getContext(), 4, Attributes::NoCapture);
+  AWI[2] = AttributeWithIndex::get(M->getContext(), AttrListPtr::FunctionIndex,
+                                   Attributes::NoUnwind);
   LLVMContext &Context = B.GetInsertBlock()->getContext();
   StringRef FWriteName = TLI->getName(LibFunc::fwrite);
   Constant *F;
   if (File->getType()->isPointerTy())
-    F = M->getOrInsertFunction(FWriteName, AttrListPtr::get(AWI),
+    F = M->getOrInsertFunction(FWriteName,
+                               AttrListPtr::get(M->getContext(), AWI),
                                TD->getIntPtrType(Context),
                                B.getInt8PtrTy(),
                                TD->getIntPtrType(Context),
@@ -436,9 +468,9 @@ Value *llvm::EmitFWrite(Value *Ptr, Value *Size, Value *File,
 
 SimplifyFortifiedLibCalls::~SimplifyFortifiedLibCalls() { }
 
-bool SimplifyFortifiedLibCalls::fold(CallInst *CI, const TargetData *TD,
+bool SimplifyFortifiedLibCalls::fold(CallInst *CI, const DataLayout *TD,
                                      const TargetLibraryInfo *TLI) {
-  // We really need TargetData for later.
+  // We really need DataLayout for later.
   if (!TD) return false;
   
   this->CI = CI;
diff --git a/lib/Transforms/Utils/BypassSlowDivision.cpp b/lib/Transforms/Utils/BypassSlowDivision.cpp
new file mode 100644
index 0000000..bee2f7b
--- /dev/null
+++ b/lib/Transforms/Utils/BypassSlowDivision.cpp
@@ -0,0 +1,262 @@
+//===-- BypassSlowDivision.cpp - Bypass slow division ---------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains an optimization for div and rem on architectures that
+// execute short instructions significantly faster than longer instructions.
+// For example, on Intel Atom 32-bit divides are slow enough that during
+// runtime it is profitable to check the value of the operands, and if they are
+// positive and less than 256 use an unsigned 8-bit divide.
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "bypass-slow-division"
+#include "llvm/Instructions.h"
+#include "llvm/Function.h"
+#include "llvm/IRBuilder.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/Transforms/Utils/BypassSlowDivision.h"
+
+using namespace llvm;
+
+namespace {
+  struct DivOpInfo {
+    bool SignedOp;
+    Value *Dividend;
+    Value *Divisor;
+
+    DivOpInfo(bool InSignedOp, Value *InDividend, Value *InDivisor)
+      : SignedOp(InSignedOp), Dividend(InDividend), Divisor(InDivisor) {}
+  };
+
+  struct DivPhiNodes {
+    PHINode *Quotient;
+    PHINode *Remainder;
+
+    DivPhiNodes(PHINode *InQuotient, PHINode *InRemainder)
+      : Quotient(InQuotient), Remainder(InRemainder) {}
+  };
+}
+
+namespace llvm {
+  template<>
+  struct DenseMapInfo<DivOpInfo> {
+    static bool isEqual(const DivOpInfo &Val1, const DivOpInfo &Val2) {
+      return Val1.SignedOp == Val2.SignedOp &&
+             Val1.Dividend == Val2.Dividend &&
+             Val1.Divisor == Val2.Divisor;
+    }
+
+    static DivOpInfo getEmptyKey() {
+      return DivOpInfo(false, 0, 0);
+    }
+
+    static DivOpInfo getTombstoneKey() {
+      return DivOpInfo(true, 0, 0);
+    }
+
+    static unsigned getHashValue(const DivOpInfo &Val) {
+      return (unsigned)(reinterpret_cast<uintptr_t>(Val.Dividend) ^
+                        reinterpret_cast<uintptr_t>(Val.Divisor)) ^
+                        (unsigned)Val.SignedOp;
+    }
+  };
+
+  typedef DenseMap<DivOpInfo, DivPhiNodes> DivCacheTy;
+}
+
+// insertFastDiv - Substitutes the div/rem instruction with code that checks the
+// value of the operands and uses a shorter-faster div/rem instruction when
+// possible and the longer-slower div/rem instruction otherwise.
+static bool insertFastDiv(Function &F,
+                          Function::iterator &I,
+                          BasicBlock::iterator &J,
+                          IntegerType *BypassType,
+                          bool UseDivOp,
+                          bool UseSignedOp,
+                          DivCacheTy &PerBBDivCache) {
+  // Get instruction operands
+  Instruction *Instr = J;
+  Value *Dividend = Instr->getOperand(0);
+  Value *Divisor = Instr->getOperand(1);
+
+  if (isa<ConstantInt>(Divisor) ||
+      (isa<ConstantInt>(Dividend) && isa<ConstantInt>(Divisor))) {
+    // Operations with immediate values should have
+    // been solved and replaced during compile time.
+    return false;
+  }
+
+  // Basic Block is split before divide
+  BasicBlock *MainBB = I;
+  BasicBlock *SuccessorBB = I->splitBasicBlock(J);
+  ++I; //advance iterator I to successorBB
+
+  // Add new basic block for slow divide operation
+  BasicBlock *SlowBB = BasicBlock::Create(F.getContext(), "",
+                                          MainBB->getParent(), SuccessorBB);
+  SlowBB->moveBefore(SuccessorBB);
+  IRBuilder<> SlowBuilder(SlowBB, SlowBB->begin());
+  Value *SlowQuotientV;
+  Value *SlowRemainderV;
+  if (UseSignedOp) {
+    SlowQuotientV = SlowBuilder.CreateSDiv(Dividend, Divisor);
+    SlowRemainderV = SlowBuilder.CreateSRem(Dividend, Divisor);
+  } else {
+    SlowQuotientV = SlowBuilder.CreateUDiv(Dividend, Divisor);
+    SlowRemainderV = SlowBuilder.CreateURem(Dividend, Divisor);
+  }
+  SlowBuilder.CreateBr(SuccessorBB);
+
+  // Add new basic block for fast divide operation
+  BasicBlock *FastBB = BasicBlock::Create(F.getContext(), "",
+                                          MainBB->getParent(), SuccessorBB);
+  FastBB->moveBefore(SlowBB);
+  IRBuilder<> FastBuilder(FastBB, FastBB->begin());
+  Value *ShortDivisorV = FastBuilder.CreateCast(Instruction::Trunc, Divisor,
+                                                BypassType);
+  Value *ShortDividendV = FastBuilder.CreateCast(Instruction::Trunc, Dividend,
+                                                 BypassType);
+
+  // udiv/urem because optimization only handles positive numbers
+  Value *ShortQuotientV = FastBuilder.CreateExactUDiv(ShortDividendV,
+                                                      ShortDivisorV);
+  Value *ShortRemainderV = FastBuilder.CreateURem(ShortDividendV,
+                                                  ShortDivisorV);
+  Value *FastQuotientV = FastBuilder.CreateCast(Instruction::ZExt,
+                                                ShortQuotientV,
+                                                Dividend->getType());
+  Value *FastRemainderV = FastBuilder.CreateCast(Instruction::ZExt,
+                                                 ShortRemainderV,
+                                                 Dividend->getType());
+  FastBuilder.CreateBr(SuccessorBB);
+
+  // Phi nodes for result of div and rem
+  IRBuilder<> SuccessorBuilder(SuccessorBB, SuccessorBB->begin());
+  PHINode *QuoPhi = SuccessorBuilder.CreatePHI(Instr->getType(), 2);
+  QuoPhi->addIncoming(SlowQuotientV, SlowBB);
+  QuoPhi->addIncoming(FastQuotientV, FastBB);
+  PHINode *RemPhi = SuccessorBuilder.CreatePHI(Instr->getType(), 2);
+  RemPhi->addIncoming(SlowRemainderV, SlowBB);
+  RemPhi->addIncoming(FastRemainderV, FastBB);
+
+  // Replace Instr with appropriate phi node
+  if (UseDivOp)
+    Instr->replaceAllUsesWith(QuoPhi);
+  else
+    Instr->replaceAllUsesWith(RemPhi);
+  Instr->eraseFromParent();
+
+  // Combine operands into a single value with OR for value testing below
+  MainBB->getInstList().back().eraseFromParent();
+  IRBuilder<> MainBuilder(MainBB, MainBB->end());
+  Value *OrV = MainBuilder.CreateOr(Dividend, Divisor);
+
+  // BitMask is inverted to check if the operands are
+  // larger than the bypass type
+  uint64_t BitMask = ~BypassType->getBitMask();
+  Value *AndV = MainBuilder.CreateAnd(OrV, BitMask);
+
+  // Compare operand values and branch
+  Value *ZeroV = MainBuilder.getInt32(0);
+  Value *CmpV = MainBuilder.CreateICmpEQ(AndV, ZeroV);
+  MainBuilder.CreateCondBr(CmpV, FastBB, SlowBB);
+
+  // point iterator J at first instruction of successorBB
+  J = I->begin();
+
+  // Cache phi nodes to be used later in place of other instances
+  // of div or rem with the same sign, dividend, and divisor
+  DivOpInfo Key(UseSignedOp, Dividend, Divisor);
+  DivPhiNodes Value(QuoPhi, RemPhi);
+  PerBBDivCache.insert(std::pair<DivOpInfo, DivPhiNodes>(Key, Value));
+  return true;
+}
+
+// reuseOrInsertFastDiv - Reuses previously computed dividend or remainder if
+// operands and operation are identical. Otherwise call insertFastDiv to perform
+// the optimization and cache the resulting dividend and remainder.
+static bool reuseOrInsertFastDiv(Function &F,
+                                 Function::iterator &I,
+                                 BasicBlock::iterator &J,
+                                 IntegerType *BypassType,
+                                 bool UseDivOp,
+                                 bool UseSignedOp,
+                                 DivCacheTy &PerBBDivCache) {
+  // Get instruction operands
+  Instruction *Instr = J;
+  DivOpInfo Key(UseSignedOp, Instr->getOperand(0), Instr->getOperand(1));
+  DivCacheTy::iterator CacheI = PerBBDivCache.find(Key);
+
+  if (CacheI == PerBBDivCache.end()) {
+    // If previous instance does not exist, insert fast div
+    return insertFastDiv(F, I, J, BypassType, UseDivOp, UseSignedOp,
+                         PerBBDivCache);
+  }
+
+  // Replace operation value with previously generated phi node
+  DivPhiNodes &Value = CacheI->second;
+  if (UseDivOp) {
+    // Replace all uses of div instruction with quotient phi node
+    J->replaceAllUsesWith(Value.Quotient);
+  } else {
+    // Replace all uses of rem instruction with remainder phi node
+    J->replaceAllUsesWith(Value.Remainder);
+  }
+
+  // Advance to next operation
+  ++J;
+
+  // Remove redundant operation
+  Instr->eraseFromParent();
+  return true;
+}
+
+// bypassSlowDivision - This optimization identifies DIV instructions that can
+// be profitably bypassed and carried out with a shorter, faster divide.
+bool llvm::bypassSlowDivision(Function &F,
+                              Function::iterator &I,
+                              const DenseMap<unsigned int, unsigned int> &BypassWidths) {
+  DivCacheTy DivCache;
+
+  bool MadeChange = false;
+  for (BasicBlock::iterator J = I->begin(); J != I->end(); J++) {
+
+    // Get instruction details
+    unsigned Opcode = J->getOpcode();
+    bool UseDivOp = Opcode == Instruction::SDiv || Opcode == Instruction::UDiv;
+    bool UseRemOp = Opcode == Instruction::SRem || Opcode == Instruction::URem;
+    bool UseSignedOp = Opcode == Instruction::SDiv ||
+                       Opcode == Instruction::SRem;
+
+    // Only optimize div or rem ops
+    if (!UseDivOp && !UseRemOp)
+      continue;
+
+    // Skip division on vector types, only optimize integer instructions
+    if (!J->getType()->isIntegerTy())
+      continue;
+
+    // Get bitwidth of div/rem instruction
+    IntegerType *T = cast<IntegerType>(J->getType());
+    int bitwidth = T->getBitWidth();
+
+    // Continue if bitwidth is not bypassed
+    DenseMap<unsigned int, unsigned int>::const_iterator BI = BypassWidths.find(bitwidth);
+    if (BI == BypassWidths.end())
+      continue;
+
+    // Get type for div/rem instruction with bypass bitwidth
+    IntegerType *BT = IntegerType::get(J->getContext(), BI->second);
+
+    MadeChange |= reuseOrInsertFastDiv(F, I, J, BT, UseDivOp,
+                                       UseSignedOp, DivCache);
+  }
+
+  return MadeChange;
+}
diff --git a/lib/Transforms/Utils/CMakeLists.txt b/lib/Transforms/Utils/CMakeLists.txt
index 4ff31ca..620209b 100644
--- a/lib/Transforms/Utils/CMakeLists.txt
+++ b/lib/Transforms/Utils/CMakeLists.txt
@@ -3,6 +3,7 @@ add_llvm_library(LLVMTransformUtils
   BasicBlockUtils.cpp
   BreakCriticalEdges.cpp
   BuildLibCalls.cpp
+  BypassSlowDivision.cpp
   CloneFunction.cpp
   CloneModule.cpp
   CmpInstAnalysis.cpp
@@ -10,6 +11,7 @@ add_llvm_library(LLVMTransformUtils
   DemoteRegToStack.cpp
   InlineFunction.cpp
   InstructionNamer.cpp
+  IntegerDivision.cpp
   LCSSA.cpp
   Local.cpp
   LoopSimplify.cpp
@@ -19,12 +21,14 @@ add_llvm_library(LLVMTransformUtils
   LowerInvoke.cpp
   LowerSwitch.cpp
   Mem2Reg.cpp
+  MetaRenamer.cpp
   ModuleUtils.cpp
   PromoteMemoryToRegister.cpp
   SSAUpdater.cpp
   SimplifyCFG.cpp
   SimplifyIndVar.cpp
   SimplifyInstructions.cpp
+  SimplifyLibCalls.cpp
   UnifyFunctionExitNodes.cpp
   Utils.cpp
   ValueMapper.cpp
diff --git a/lib/Transforms/Utils/CloneFunction.cpp b/lib/Transforms/Utils/CloneFunction.cpp
index 99237b8..7ba9f6d 100644
--- a/lib/Transforms/Utils/CloneFunction.cpp
+++ b/lib/Transforms/Utils/CloneFunction.cpp
@@ -98,10 +98,14 @@ void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc,
         Anew->addAttr( OldFunc->getAttributes()
                        .getParamAttributes(I->getArgNo() + 1));
     NewFunc->setAttributes(NewFunc->getAttributes()
-                           .addAttr(0, OldFunc->getAttributes()
+                           .addAttr(NewFunc->getContext(),
+                                    AttrListPtr::ReturnIndex,
+                                    OldFunc->getAttributes()
                                      .getRetAttributes()));
     NewFunc->setAttributes(NewFunc->getAttributes()
-                           .addAttr(~0, OldFunc->getAttributes()
+                           .addAttr(NewFunc->getContext(),
+                                    AttrListPtr::FunctionIndex,
+                                    OldFunc->getAttributes()
                                      .getFnAttributes()));
 
   }
@@ -202,14 +206,14 @@ namespace {
     bool ModuleLevelChanges;
     const char *NameSuffix;
     ClonedCodeInfo *CodeInfo;
-    const TargetData *TD;
+    const DataLayout *TD;
   public:
     PruningFunctionCloner(Function *newFunc, const Function *oldFunc,
                           ValueToValueMapTy &valueMap,
                           bool moduleLevelChanges,
                           const char *nameSuffix, 
                           ClonedCodeInfo *codeInfo,
-                          const TargetData *td)
+                          const DataLayout *td)
     : NewFunc(newFunc), OldFunc(oldFunc),
       VMap(valueMap), ModuleLevelChanges(moduleLevelChanges),
       NameSuffix(nameSuffix), CodeInfo(codeInfo), TD(td) {
@@ -365,7 +369,7 @@ void llvm::CloneAndPruneFunctionInto(Function *NewFunc, const Function *OldFunc,
                                      SmallVectorImpl<ReturnInst*> &Returns,
                                      const char *NameSuffix, 
                                      ClonedCodeInfo *CodeInfo,
-                                     const TargetData *TD,
+                                     const DataLayout *TD,
                                      Instruction *TheCall) {
   assert(NameSuffix && "NameSuffix cannot be null!");
   
diff --git a/lib/Transforms/Utils/CodeExtractor.cpp b/lib/Transforms/Utils/CodeExtractor.cpp
index c545cd6..281714f 100644
--- a/lib/Transforms/Utils/CodeExtractor.cpp
+++ b/lib/Transforms/Utils/CodeExtractor.cpp
@@ -346,7 +346,7 @@ Function *CodeExtractor::constructFunction(const ValueSet &inputs,
                                            header->getName(), M);
   // If the old function is no-throw, so is the new one.
   if (oldFunction->doesNotThrow())
-    newFunction->setDoesNotThrow(true);
+    newFunction->setDoesNotThrow();
   
   newFunction->getBasicBlockList().push_back(newRootNode);
 
diff --git a/lib/Transforms/Utils/InlineFunction.cpp b/lib/Transforms/Utils/InlineFunction.cpp
index 89e89e7..009847f 100644
--- a/lib/Transforms/Utils/InlineFunction.cpp
+++ b/lib/Transforms/Utils/InlineFunction.cpp
@@ -27,7 +27,7 @@
 #include "llvm/Analysis/CallGraph.h"
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Support/CallSite.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Transforms/Utils/Local.h"
 using namespace llvm;
 
@@ -357,7 +357,7 @@ static Value *HandleByValArgument(Value *Arg, Instruction *TheCall,
 
   Type *VoidPtrTy = Type::getInt8PtrTy(Context);
   
-  // Create the alloca.  If we have TargetData, use nice alignment.
+  // Create the alloca.  If we have DataLayout, use nice alignment.
   unsigned Align = 1;
   if (IFI.TD)
     Align = IFI.TD->getPrefTypeAlignment(AggTy);
diff --git a/lib/Transforms/Utils/IntegerDivision.cpp b/lib/Transforms/Utils/IntegerDivision.cpp
new file mode 100644
index 0000000..55227e2
--- /dev/null
+++ b/lib/Transforms/Utils/IntegerDivision.cpp
@@ -0,0 +1,420 @@
+//===-- IntegerDivision.cpp - Expand integer division ---------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains an implementation of 32bit scalar integer division for
+// targets that don't have native support. It's largely derived from
+// compiler-rt's implementation of __udivsi3, but hand-tuned to reduce the
+// amount of control flow
+//
+//===----------------------------------------------------------------------===//
+
+#define DEBUG_TYPE "integer-division"
+#include "llvm/Function.h"
+#include "llvm/Instructions.h"
+#include "llvm/Intrinsics.h"
+#include "llvm/IRBuilder.h"
+#include "llvm/Transforms/Utils/IntegerDivision.h"
+
+using namespace llvm;
+
+/// Generate code to compute the remainder of two signed integers. Returns the
+/// remainder, which will have the sign of the dividend. Builder's insert point
+/// should be pointing where the caller wants code generated, e.g. at the srem
+/// instruction. This will generate a urem in the process, and Builder's insert
+/// point will be pointing at the uren (if present, i.e. not folded), ready to
+/// be expanded if the user wishes
+static Value *generateSignedRemainderCode(Value *Dividend, Value *Divisor,
+                                          IRBuilder<> &Builder) {
+  ConstantInt *ThirtyOne = Builder.getInt32(31);
+
+  // ;   %dividend_sgn = ashr i32 %dividend, 31
+  // ;   %divisor_sgn  = ashr i32 %divisor, 31
+  // ;   %dvd_xor      = xor i32 %dividend, %dividend_sgn
+  // ;   %dvs_xor      = xor i32 %divisor, %divisor_sgn
+  // ;   %u_dividend   = sub i32 %dvd_xor, %dividend_sgn
+  // ;   %u_divisor    = sub i32 %dvs_xor, %divisor_sgn
+  // ;   %urem         = urem i32 %dividend, %divisor
+  // ;   %xored        = xor i32 %urem, %dividend_sgn
+  // ;   %srem         = sub i32 %xored, %dividend_sgn
+  Value *DividendSign = Builder.CreateAShr(Dividend, ThirtyOne);
+  Value *DivisorSign  = Builder.CreateAShr(Divisor, ThirtyOne);
+  Value *DvdXor       = Builder.CreateXor(Dividend, DividendSign);
+  Value *DvsXor       = Builder.CreateXor(Divisor, DivisorSign);
+  Value *UDividend    = Builder.CreateSub(DvdXor, DividendSign);
+  Value *UDivisor     = Builder.CreateSub(DvsXor, DivisorSign);
+  Value *URem         = Builder.CreateURem(UDividend, UDivisor);
+  Value *Xored        = Builder.CreateXor(URem, DividendSign);
+  Value *SRem         = Builder.CreateSub(Xored, DividendSign);
+
+  if (Instruction *URemInst = dyn_cast<Instruction>(URem))
+    Builder.SetInsertPoint(URemInst);
+
+  return SRem;
+}
+
+
+/// Generate code to compute the remainder of two unsigned integers. Returns the
+/// remainder. Builder's insert point should be pointing where the caller wants
+/// code generated, e.g. at the urem instruction. This will generate a udiv in
+/// the process, and Builder's insert point will be pointing at the udiv (if
+/// present, i.e. not folded), ready to be expanded if the user wishes
+static Value *generatedUnsignedRemainderCode(Value *Dividend, Value *Divisor,
+                                             IRBuilder<> &Builder) {
+  // Remainder = Dividend - Quotient*Divisor
+
+  // ;   %quotient  = udiv i32 %dividend, %divisor
+  // ;   %product   = mul i32 %divisor, %quotient
+  // ;   %remainder = sub i32 %dividend, %product
+  Value *Quotient  = Builder.CreateUDiv(Dividend, Divisor);
+  Value *Product   = Builder.CreateMul(Divisor, Quotient);
+  Value *Remainder = Builder.CreateSub(Dividend, Product);
+
+  if (Instruction *UDiv = dyn_cast<Instruction>(Quotient))
+    Builder.SetInsertPoint(UDiv);
+
+  return Remainder;
+}
+
+/// Generate code to divide two signed integers. Returns the quotient, rounded
+/// towards 0. Builder's insert point should be pointing where the caller wants
+/// code generated, e.g. at the sdiv instruction. This will generate a udiv in
+/// the process, and Builder's insert point will be pointing at the udiv (if
+/// present, i.e. not folded), ready to be expanded if the user wishes.
+static Value *generateSignedDivisionCode(Value *Dividend, Value *Divisor,
+                                         IRBuilder<> &Builder) {
+  // Implementation taken from compiler-rt's __divsi3
+
+  ConstantInt *ThirtyOne = Builder.getInt32(31);
+
+  // ;   %tmp    = ashr i32 %dividend, 31
+  // ;   %tmp1   = ashr i32 %divisor, 31
+  // ;   %tmp2   = xor i32 %tmp, %dividend
+  // ;   %u_dvnd = sub nsw i32 %tmp2, %tmp
+  // ;   %tmp3   = xor i32 %tmp1, %divisor
+  // ;   %u_dvsr = sub nsw i32 %tmp3, %tmp1
+  // ;   %q_sgn  = xor i32 %tmp1, %tmp
+  // ;   %q_mag  = udiv i32 %u_dvnd, %u_dvsr
+  // ;   %tmp4   = xor i32 %q_mag, %q_sgn
+  // ;   %q      = sub i32 %tmp4, %q_sgn
+  Value *Tmp    = Builder.CreateAShr(Dividend, ThirtyOne);
+  Value *Tmp1   = Builder.CreateAShr(Divisor, ThirtyOne);
+  Value *Tmp2   = Builder.CreateXor(Tmp, Dividend);
+  Value *U_Dvnd = Builder.CreateSub(Tmp2, Tmp);
+  Value *Tmp3   = Builder.CreateXor(Tmp1, Divisor);
+  Value *U_Dvsr = Builder.CreateSub(Tmp3, Tmp1);
+  Value *Q_Sgn  = Builder.CreateXor(Tmp1, Tmp);
+  Value *Q_Mag  = Builder.CreateUDiv(U_Dvnd, U_Dvsr);
+  Value *Tmp4   = Builder.CreateXor(Q_Mag, Q_Sgn);
+  Value *Q      = Builder.CreateSub(Tmp4, Q_Sgn);
+
+  if (Instruction *UDiv = dyn_cast<Instruction>(Q_Mag))
+    Builder.SetInsertPoint(UDiv);
+
+  return Q;
+}
+
+/// Generates code to divide two unsigned scalar 32-bit integers. Returns the
+/// quotient, rounded towards 0. Builder's insert point should be pointing where
+/// the caller wants code generated, e.g. at the udiv instruction.
+static Value *generateUnsignedDivisionCode(Value *Dividend, Value *Divisor,
+                                           IRBuilder<> &Builder) {
+  // The basic algorithm can be found in the compiler-rt project's
+  // implementation of __udivsi3.c. Here, we do a lower-level IR based approach
+  // that's been hand-tuned to lessen the amount of control flow involved.
+
+  // Some helper values
+  IntegerType *I32Ty = Builder.getInt32Ty();
+
+  ConstantInt *Zero      = Builder.getInt32(0);
+  ConstantInt *One       = Builder.getInt32(1);
+  ConstantInt *ThirtyOne = Builder.getInt32(31);
+  ConstantInt *NegOne    = ConstantInt::getSigned(I32Ty, -1);
+  ConstantInt *True      = Builder.getTrue();
+
+  BasicBlock *IBB = Builder.GetInsertBlock();
+  Function *F = IBB->getParent();
+  Function *CTLZi32 = Intrinsic::getDeclaration(F->getParent(), Intrinsic::ctlz,
+                                                I32Ty);
+
+  // Our CFG is going to look like:
+  // +---------------------+
+  // | special-cases       |
+  // |   ...               |
+  // +---------------------+
+  //  |       |
+  //  |   +----------+
+  //  |   |  bb1     |
+  //  |   |  ...     |
+  //  |   +----------+
+  //  |    |      |
+  //  |    |  +------------+
+  //  |    |  |  preheader |
+  //  |    |  |  ...       |
+  //  |    |  +------------+
+  //  |    |      |
+  //  |    |      |      +---+
+  //  |    |      |      |   |
+  //  |    |  +------------+ |
+  //  |    |  |  do-while  | |
+  //  |    |  |  ...       | |
+  //  |    |  +------------+ |
+  //  |    |      |      |   |
+  //  |   +-----------+  +---+
+  //  |   | loop-exit |
+  //  |   |  ...      |
+  //  |   +-----------+
+  //  |     |
+  // +-------+
+  // | ...   |
+  // | end   |
+  // +-------+
+  BasicBlock *SpecialCases = Builder.GetInsertBlock();
+  SpecialCases->setName(Twine(SpecialCases->getName(), "_udiv-special-cases"));
+  BasicBlock *End = SpecialCases->splitBasicBlock(Builder.GetInsertPoint(),
+                                                  "udiv-end");
+  BasicBlock *LoopExit  = BasicBlock::Create(Builder.getContext(),
+                                             "udiv-loop-exit", F, End);
+  BasicBlock *DoWhile   = BasicBlock::Create(Builder.getContext(),
+                                             "udiv-do-while", F, End);
+  BasicBlock *Preheader = BasicBlock::Create(Builder.getContext(),
+                                             "udiv-preheader", F, End);
+  BasicBlock *BB1       = BasicBlock::Create(Builder.getContext(),
+                                             "udiv-bb1", F, End);
+
+  // We'll be overwriting the terminator to insert our extra blocks
+  SpecialCases->getTerminator()->eraseFromParent();
+
+  // First off, check for special cases: dividend or divisor is zero, divisor
+  // is greater than dividend, and divisor is 1.
+  // ; special-cases:
+  // ;   %ret0_1      = icmp eq i32 %divisor, 0
+  // ;   %ret0_2      = icmp eq i32 %dividend, 0
+  // ;   %ret0_3      = or i1 %ret0_1, %ret0_2
+  // ;   %tmp0        = tail call i32 @llvm.ctlz.i32(i32 %divisor, i1 true)
+  // ;   %tmp1        = tail call i32 @llvm.ctlz.i32(i32 %dividend, i1 true)
+  // ;   %sr          = sub nsw i32 %tmp0, %tmp1
+  // ;   %ret0_4      = icmp ugt i32 %sr, 31
+  // ;   %ret0        = or i1 %ret0_3, %ret0_4
+  // ;   %retDividend = icmp eq i32 %sr, 31
+  // ;   %retVal      = select i1 %ret0, i32 0, i32 %dividend
+  // ;   %earlyRet    = or i1 %ret0, %retDividend
+  // ;   br i1 %earlyRet, label %end, label %bb1
+  Builder.SetInsertPoint(SpecialCases);
+  Value *Ret0_1      = Builder.CreateICmpEQ(Divisor, Zero);
+  Value *Ret0_2      = Builder.CreateICmpEQ(Dividend, Zero);
+  Value *Ret0_3      = Builder.CreateOr(Ret0_1, Ret0_2);
+  Value *Tmp0        = Builder.CreateCall2(CTLZi32, Divisor, True);
+  Value *Tmp1        = Builder.CreateCall2(CTLZi32, Dividend, True);
+  Value *SR          = Builder.CreateSub(Tmp0, Tmp1);
+  Value *Ret0_4      = Builder.CreateICmpUGT(SR, ThirtyOne);
+  Value *Ret0        = Builder.CreateOr(Ret0_3, Ret0_4);
+  Value *RetDividend = Builder.CreateICmpEQ(SR, ThirtyOne);
+  Value *RetVal      = Builder.CreateSelect(Ret0, Zero, Dividend);
+  Value *EarlyRet    = Builder.CreateOr(Ret0, RetDividend);
+  Builder.CreateCondBr(EarlyRet, End, BB1);
+
+  // ; bb1:                                             ; preds = %special-cases
+  // ;   %sr_1     = add i32 %sr, 1
+  // ;   %tmp2     = sub i32 31, %sr
+  // ;   %q        = shl i32 %dividend, %tmp2
+  // ;   %skipLoop = icmp eq i32 %sr_1, 0
+  // ;   br i1 %skipLoop, label %loop-exit, label %preheader
+  Builder.SetInsertPoint(BB1);
+  Value *SR_1     = Builder.CreateAdd(SR, One);
+  Value *Tmp2     = Builder.CreateSub(ThirtyOne, SR);
+  Value *Q        = Builder.CreateShl(Dividend, Tmp2);
+  Value *SkipLoop = Builder.CreateICmpEQ(SR_1, Zero);
+  Builder.CreateCondBr(SkipLoop, LoopExit, Preheader);
+
+  // ; preheader:                                           ; preds = %bb1
+  // ;   %tmp3 = lshr i32 %dividend, %sr_1
+  // ;   %tmp4 = add i32 %divisor, -1
+  // ;   br label %do-while
+  Builder.SetInsertPoint(Preheader);
+  Value *Tmp3 = Builder.CreateLShr(Dividend, SR_1);
+  Value *Tmp4 = Builder.CreateAdd(Divisor, NegOne);
+  Builder.CreateBr(DoWhile);
+
+  // ; do-while:                                 ; preds = %do-while, %preheader
+  // ;   %carry_1 = phi i32 [ 0, %preheader ], [ %carry, %do-while ]
+  // ;   %sr_3    = phi i32 [ %sr_1, %preheader ], [ %sr_2, %do-while ]
+  // ;   %r_1     = phi i32 [ %tmp3, %preheader ], [ %r, %do-while ]
+  // ;   %q_2     = phi i32 [ %q, %preheader ], [ %q_1, %do-while ]
+  // ;   %tmp5  = shl i32 %r_1, 1
+  // ;   %tmp6  = lshr i32 %q_2, 31
+  // ;   %tmp7  = or i32 %tmp5, %tmp6
+  // ;   %tmp8  = shl i32 %q_2, 1
+  // ;   %q_1   = or i32 %carry_1, %tmp8
+  // ;   %tmp9  = sub i32 %tmp4, %tmp7
+  // ;   %tmp10 = ashr i32 %tmp9, 31
+  // ;   %carry = and i32 %tmp10, 1
+  // ;   %tmp11 = and i32 %tmp10, %divisor
+  // ;   %r     = sub i32 %tmp7, %tmp11
+  // ;   %sr_2  = add i32 %sr_3, -1
+  // ;   %tmp12 = icmp eq i32 %sr_2, 0
+  // ;   br i1 %tmp12, label %loop-exit, label %do-while
+  Builder.SetInsertPoint(DoWhile);
+  PHINode *Carry_1 = Builder.CreatePHI(I32Ty, 2);
+  PHINode *SR_3    = Builder.CreatePHI(I32Ty, 2);
+  PHINode *R_1     = Builder.CreatePHI(I32Ty, 2);
+  PHINode *Q_2     = Builder.CreatePHI(I32Ty, 2);
+  Value *Tmp5  = Builder.CreateShl(R_1, One);
+  Value *Tmp6  = Builder.CreateLShr(Q_2, ThirtyOne);
+  Value *Tmp7  = Builder.CreateOr(Tmp5, Tmp6);
+  Value *Tmp8  = Builder.CreateShl(Q_2, One);
+  Value *Q_1   = Builder.CreateOr(Carry_1, Tmp8);
+  Value *Tmp9  = Builder.CreateSub(Tmp4, Tmp7);
+  Value *Tmp10 = Builder.CreateAShr(Tmp9, 31);
+  Value *Carry = Builder.CreateAnd(Tmp10, One);
+  Value *Tmp11 = Builder.CreateAnd(Tmp10, Divisor);
+  Value *R     = Builder.CreateSub(Tmp7, Tmp11);
+  Value *SR_2  = Builder.CreateAdd(SR_3, NegOne);
+  Value *Tmp12 = Builder.CreateICmpEQ(SR_2, Zero);
+  Builder.CreateCondBr(Tmp12, LoopExit, DoWhile);
+
+  // ; loop-exit:                                      ; preds = %do-while, %bb1
+  // ;   %carry_2 = phi i32 [ 0, %bb1 ], [ %carry, %do-while ]
+  // ;   %q_3     = phi i32 [ %q, %bb1 ], [ %q_1, %do-while ]
+  // ;   %tmp13 = shl i32 %q_3, 1
+  // ;   %q_4   = or i32 %carry_2, %tmp13
+  // ;   br label %end
+  Builder.SetInsertPoint(LoopExit);
+  PHINode *Carry_2 = Builder.CreatePHI(I32Ty, 2);
+  PHINode *Q_3     = Builder.CreatePHI(I32Ty, 2);
+  Value *Tmp13 = Builder.CreateShl(Q_3, One);
+  Value *Q_4   = Builder.CreateOr(Carry_2, Tmp13);
+  Builder.CreateBr(End);
+
+  // ; end:                                 ; preds = %loop-exit, %special-cases
+  // ;   %q_5 = phi i32 [ %q_4, %loop-exit ], [ %retVal, %special-cases ]
+  // ;   ret i32 %q_5
+  Builder.SetInsertPoint(End, End->begin());
+  PHINode *Q_5 = Builder.CreatePHI(I32Ty, 2);
+
+  // Populate the Phis, since all values have now been created. Our Phis were:
+  // ;   %carry_1 = phi i32 [ 0, %preheader ], [ %carry, %do-while ]
+  Carry_1->addIncoming(Zero, Preheader);
+  Carry_1->addIncoming(Carry, DoWhile);
+  // ;   %sr_3 = phi i32 [ %sr_1, %preheader ], [ %sr_2, %do-while ]
+  SR_3->addIncoming(SR_1, Preheader);
+  SR_3->addIncoming(SR_2, DoWhile);
+  // ;   %r_1 = phi i32 [ %tmp3, %preheader ], [ %r, %do-while ]
+  R_1->addIncoming(Tmp3, Preheader);
+  R_1->addIncoming(R, DoWhile);
+  // ;   %q_2 = phi i32 [ %q, %preheader ], [ %q_1, %do-while ]
+  Q_2->addIncoming(Q, Preheader);
+  Q_2->addIncoming(Q_1, DoWhile);
+  // ;   %carry_2 = phi i32 [ 0, %bb1 ], [ %carry, %do-while ]
+  Carry_2->addIncoming(Zero, BB1);
+  Carry_2->addIncoming(Carry, DoWhile);
+  // ;   %q_3 = phi i32 [ %q, %bb1 ], [ %q_1, %do-while ]
+  Q_3->addIncoming(Q, BB1);
+  Q_3->addIncoming(Q_1, DoWhile);
+  // ;   %q_5 = phi i32 [ %q_4, %loop-exit ], [ %retVal, %special-cases ]
+  Q_5->addIncoming(Q_4, LoopExit);
+  Q_5->addIncoming(RetVal, SpecialCases);
+
+  return Q_5;
+}
+
+/// Generate code to calculate the remainder of two integers, replacing Rem with
+/// the generated code. This currently generates code using the udiv expansion,
+/// but future work includes generating more specialized code, e.g. when more
+/// information about the operands are known. Currently only implements 32bit
+/// scalar division (due to udiv's limitation), but future work is removing this
+/// limitation.
+///
+/// @brief Replace Rem with generated code.
+bool llvm::expandRemainder(BinaryOperator *Rem) {
+  assert((Rem->getOpcode() == Instruction::SRem ||
+          Rem->getOpcode() == Instruction::URem) &&
+         "Trying to expand remainder from a non-remainder function");
+
+  IRBuilder<> Builder(Rem);
+
+  // First prepare the sign if it's a signed remainder
+  if (Rem->getOpcode() == Instruction::SRem) {
+    Value *Remainder = generateSignedRemainderCode(Rem->getOperand(0),
+                                                   Rem->getOperand(1), Builder);
+
+    Rem->replaceAllUsesWith(Remainder);
+    Rem->dropAllReferences();
+    Rem->eraseFromParent();
+
+    // If we didn't actually generate a udiv instruction, we're done
+    BinaryOperator *BO = dyn_cast<BinaryOperator>(Builder.GetInsertPoint());
+    if (!BO || BO->getOpcode() != Instruction::URem)
+      return true;
+
+    Rem = BO;
+  }
+
+  Value *Remainder = generatedUnsignedRemainderCode(Rem->getOperand(0),
+                                                    Rem->getOperand(1),
+                                                    Builder);
+
+  Rem->replaceAllUsesWith(Remainder);
+  Rem->dropAllReferences();
+  Rem->eraseFromParent();
+
+  // Expand the udiv
+  if (BinaryOperator *UDiv = dyn_cast<BinaryOperator>(Builder.GetInsertPoint())) {
+    assert(UDiv->getOpcode() == Instruction::UDiv && "Non-udiv in expansion?");
+    expandDivision(UDiv);
+  }
+
+  return true;
+}
+
+
+/// Generate code to divide two integers, replacing Div with the generated
+/// code. This currently generates code similarly to compiler-rt's
+/// implementations, but future work includes generating more specialized code
+/// when more information about the operands are known. Currently only
+/// implements 32bit scalar division, but future work is removing this
+/// limitation.
+///
+/// @brief Replace Div with generated code.
+bool llvm::expandDivision(BinaryOperator *Div) {
+  assert((Div->getOpcode() == Instruction::SDiv ||
+          Div->getOpcode() == Instruction::UDiv) &&
+         "Trying to expand division from a non-division function");
+
+  IRBuilder<> Builder(Div);
+
+  if (Div->getType()->isVectorTy())
+    llvm_unreachable("Div over vectors not supported");
+
+  // First prepare the sign if it's a signed division
+  if (Div->getOpcode() == Instruction::SDiv) {
+    // Lower the code to unsigned division, and reset Div to point to the udiv.
+    Value *Quotient = generateSignedDivisionCode(Div->getOperand(0),
+                                                 Div->getOperand(1), Builder);
+    Div->replaceAllUsesWith(Quotient);
+    Div->dropAllReferences();
+    Div->eraseFromParent();
+
+    // If we didn't actually generate a udiv instruction, we're done
+    BinaryOperator *BO = dyn_cast<BinaryOperator>(Builder.GetInsertPoint());
+    if (!BO || BO->getOpcode() != Instruction::UDiv)
+      return true;
+
+    Div = BO;
+  }
+
+  // Insert the unsigned division code
+  Value *Quotient = generateUnsignedDivisionCode(Div->getOperand(0),
+                                                 Div->getOperand(1),
+                                                 Builder);
+  Div->replaceAllUsesWith(Quotient);
+  Div->dropAllReferences();
+  Div->eraseFromParent();
+
+  return true;
+}
diff --git a/lib/Transforms/Utils/LCSSA.cpp b/lib/Transforms/Utils/LCSSA.cpp
index b654111..5e05c83 100644
--- a/lib/Transforms/Utils/LCSSA.cpp
+++ b/lib/Transforms/Utils/LCSSA.cpp
@@ -53,6 +53,8 @@ namespace {
 
     // Cached analysis information for the current function.
     DominatorTree *DT;
+    LoopInfo *LI;
+    ScalarEvolution *SE;
     std::vector<BasicBlock*> LoopBlocks;
     PredIteratorCache PredCache;
     Loop *L;
@@ -117,6 +119,8 @@ bool LCSSA::runOnLoop(Loop *TheLoop, LPPassManager &LPM) {
   L = TheLoop;
   
   DT = &getAnalysis<DominatorTree>();
+  LI = &getAnalysis<LoopInfo>();
+  SE = getAnalysisIfAvailable<ScalarEvolution>();
 
   // Get the set of exiting blocks.
   SmallVector<BasicBlock*, 8> ExitBlocks;
@@ -156,6 +160,12 @@ bool LCSSA::runOnLoop(Loop *TheLoop, LPPassManager &LPM) {
       MadeChange |= ProcessInstruction(I, ExitBlocks);
     }
   }
+
+  // If we modified the code, remove any caches about the loop from SCEV to
+  // avoid dangling entries.
+  // FIXME: This is a big hammer, can we clear the cache more selectively?
+  if (SE && MadeChange)
+    SE->forgetLoop(L);
   
   assert(L->isLCSSAForm(*DT));
   PredCache.clear();
@@ -245,7 +255,7 @@ bool LCSSA::ProcessInstruction(Instruction *Inst,
     // Remember that this phi makes the value alive in this block.
     SSAUpdate.AddAvailableValue(ExitBB, PN);
   }
-  
+
   // Rewrite all uses outside the loop in terms of the new PHIs we just
   // inserted.
   for (unsigned i = 0, e = UsesToRewrite.size(); i != e; ++i) {
@@ -260,6 +270,9 @@ bool LCSSA::ProcessInstruction(Instruction *Inst,
 
     if (isa<PHINode>(UserBB->begin()) &&
         isExitBlock(UserBB, ExitBlocks)) {
+      // Tell the VHs that the uses changed. This updates SCEV's caches.
+      if (UsesToRewrite[i]->get()->hasValueHandle())
+        ValueHandleBase::ValueIsRAUWd(*UsesToRewrite[i], UserBB->begin());
       UsesToRewrite[i]->set(UserBB->begin());
       continue;
     }
diff --git a/lib/Transforms/Utils/Local.cpp b/lib/Transforms/Utils/Local.cpp
index bed7d72..a954d82 100644
--- a/lib/Transforms/Utils/Local.cpp
+++ b/lib/Transforms/Utils/Local.cpp
@@ -23,6 +23,7 @@
 #include "llvm/Instructions.h"
 #include "llvm/IntrinsicInst.h"
 #include "llvm/Intrinsics.h"
+#include "llvm/MDBuilder.h"
 #include "llvm/Metadata.h"
 #include "llvm/Operator.h"
 #include "llvm/ADT/DenseMap.h"
@@ -38,7 +39,7 @@
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/ValueHandle.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 using namespace llvm;
 
 //===----------------------------------------------------------------------===//
@@ -52,7 +53,8 @@ using namespace llvm;
 /// Also calls RecursivelyDeleteTriviallyDeadInstructions() on any branch/switch
 /// conditions and indirectbr addresses this might make dead if
 /// DeleteDeadConditions is true.
-bool llvm::ConstantFoldTerminator(BasicBlock *BB, bool DeleteDeadConditions) {
+bool llvm::ConstantFoldTerminator(BasicBlock *BB, bool DeleteDeadConditions,
+                                  const TargetLibraryInfo *TLI) {
   TerminatorInst *T = BB->getTerminator();
   IRBuilder<> Builder(T);
 
@@ -96,7 +98,7 @@ bool llvm::ConstantFoldTerminator(BasicBlock *BB, bool DeleteDeadConditions) {
       Value *Cond = BI->getCondition();
       BI->eraseFromParent();
       if (DeleteDeadConditions)
-        RecursivelyDeleteTriviallyDeadInstructions(Cond);
+        RecursivelyDeleteTriviallyDeadInstructions(Cond, TLI);
       return true;
     }
     return false;
@@ -121,6 +123,27 @@ bool llvm::ConstantFoldTerminator(BasicBlock *BB, bool DeleteDeadConditions) {
       // Check to see if this branch is going to the same place as the default
       // dest.  If so, eliminate it as an explicit compare.
       if (i.getCaseSuccessor() == DefaultDest) {
+        MDNode* MD = SI->getMetadata(LLVMContext::MD_prof);
+        // MD should have 2 + NumCases operands.
+        if (MD && MD->getNumOperands() == 2 + SI->getNumCases()) {
+          // Collect branch weights into a vector.
+          SmallVector<uint32_t, 8> Weights;
+          for (unsigned MD_i = 1, MD_e = MD->getNumOperands(); MD_i < MD_e;
+               ++MD_i) {
+            ConstantInt* CI = dyn_cast<ConstantInt>(MD->getOperand(MD_i));
+            assert(CI);
+            Weights.push_back(CI->getValue().getZExtValue());
+          }
+          // Merge weight of this case to the default weight.
+          unsigned idx = i.getCaseIndex();
+          Weights[0] += Weights[idx+1];
+          // Remove weight for this case.
+          std::swap(Weights[idx+1], Weights.back());
+          Weights.pop_back();
+          SI->setMetadata(LLVMContext::MD_prof,
+                          MDBuilder(BB->getContext()).
+                          createBranchWeights(Weights));
+        }
         // Remove this entry.
         DefaultDest->removePredecessor(SI->getParent());
         SI->removeCase(i);
@@ -161,7 +184,7 @@ bool llvm::ConstantFoldTerminator(BasicBlock *BB, bool DeleteDeadConditions) {
       Value *Cond = SI->getCondition();
       SI->eraseFromParent();
       if (DeleteDeadConditions)
-        RecursivelyDeleteTriviallyDeadInstructions(Cond);
+        RecursivelyDeleteTriviallyDeadInstructions(Cond, TLI);
       return true;
     }
     
@@ -177,8 +200,20 @@ bool llvm::ConstantFoldTerminator(BasicBlock *BB, bool DeleteDeadConditions) {
             "cond");
 
         // Insert the new branch.
-        Builder.CreateCondBr(Cond, FirstCase.getCaseSuccessor(),
-                             SI->getDefaultDest());
+        BranchInst *NewBr = Builder.CreateCondBr(Cond,
+                                FirstCase.getCaseSuccessor(),
+                                SI->getDefaultDest());
+        MDNode* MD = SI->getMetadata(LLVMContext::MD_prof);
+        if (MD && MD->getNumOperands() == 3) {
+          ConstantInt *SICase = dyn_cast<ConstantInt>(MD->getOperand(2));
+          ConstantInt *SIDef = dyn_cast<ConstantInt>(MD->getOperand(1));
+          assert(SICase && SIDef);
+          // The TrueWeight should be the weight for the single case of SI.
+          NewBr->setMetadata(LLVMContext::MD_prof,
+                 MDBuilder(BB->getContext()).
+                 createBranchWeights(SICase->getValue().getZExtValue(),
+                                     SIDef->getValue().getZExtValue()));
+        }
 
         // Delete the old switch.
         SI->eraseFromParent();
@@ -205,7 +240,7 @@ bool llvm::ConstantFoldTerminator(BasicBlock *BB, bool DeleteDeadConditions) {
       Value *Address = IBI->getAddress();
       IBI->eraseFromParent();
       if (DeleteDeadConditions)
-        RecursivelyDeleteTriviallyDeadInstructions(Address);
+        RecursivelyDeleteTriviallyDeadInstructions(Address, TLI);
       
       // If we didn't find our destination in the IBI successor list, then we
       // have undefined behavior.  Replace the unconditional branch with an
@@ -230,7 +265,8 @@ bool llvm::ConstantFoldTerminator(BasicBlock *BB, bool DeleteDeadConditions) {
 /// isInstructionTriviallyDead - Return true if the result produced by the
 /// instruction is not used, and the instruction has no side effects.
 ///
-bool llvm::isInstructionTriviallyDead(Instruction *I) {
+bool llvm::isInstructionTriviallyDead(Instruction *I,
+                                      const TargetLibraryInfo *TLI) {
   if (!I->use_empty() || isa<TerminatorInst>(I)) return false;
 
   // We don't want the landingpad instruction removed by anything this general.
@@ -265,9 +301,9 @@ bool llvm::isInstructionTriviallyDead(Instruction *I) {
       return isa<UndefValue>(II->getArgOperand(1));
   }
 
-  if (isAllocLikeFn(I)) return true;
+  if (isAllocLikeFn(I, TLI)) return true;
 
-  if (CallInst *CI = isFreeCall(I))
+  if (CallInst *CI = isFreeCall(I, TLI))
     if (Constant *C = dyn_cast<Constant>(CI->getArgOperand(0)))
       return C->isNullValue() || isa<UndefValue>(C);
 
@@ -278,9 +314,11 @@ bool llvm::isInstructionTriviallyDead(Instruction *I) {
 /// trivially dead instruction, delete it.  If that makes any of its operands
 /// trivially dead, delete them too, recursively.  Return true if any
 /// instructions were deleted.
-bool llvm::RecursivelyDeleteTriviallyDeadInstructions(Value *V) {
+bool
+llvm::RecursivelyDeleteTriviallyDeadInstructions(Value *V,
+                                                 const TargetLibraryInfo *TLI) {
   Instruction *I = dyn_cast<Instruction>(V);
-  if (!I || !I->use_empty() || !isInstructionTriviallyDead(I))
+  if (!I || !I->use_empty() || !isInstructionTriviallyDead(I, TLI))
     return false;
   
   SmallVector<Instruction*, 16> DeadInsts;
@@ -301,7 +339,7 @@ bool llvm::RecursivelyDeleteTriviallyDeadInstructions(Value *V) {
       // operand, and if it is 'trivially' dead, delete it in a future loop
       // iteration.
       if (Instruction *OpI = dyn_cast<Instruction>(OpV))
-        if (isInstructionTriviallyDead(OpI))
+        if (isInstructionTriviallyDead(OpI, TLI))
           DeadInsts.push_back(OpI);
     }
     
@@ -334,19 +372,20 @@ static bool areAllUsesEqual(Instruction *I) {
 /// either forms a cycle or is terminated by a trivially dead instruction,
 /// delete it.  If that makes any of its operands trivially dead, delete them
 /// too, recursively.  Return true if a change was made.
-bool llvm::RecursivelyDeleteDeadPHINode(PHINode *PN) {
+bool llvm::RecursivelyDeleteDeadPHINode(PHINode *PN,
+                                        const TargetLibraryInfo *TLI) {
   SmallPtrSet<Instruction*, 4> Visited;
   for (Instruction *I = PN; areAllUsesEqual(I) && !I->mayHaveSideEffects();
        I = cast<Instruction>(*I->use_begin())) {
     if (I->use_empty())
-      return RecursivelyDeleteTriviallyDeadInstructions(I);
+      return RecursivelyDeleteTriviallyDeadInstructions(I, TLI);
 
     // If we find an instruction more than once, we're on a cycle that
     // won't prove fruitful.
     if (!Visited.insert(I)) {
       // Break the cycle and delete the instruction and its operands.
       I->replaceAllUsesWith(UndefValue::get(I->getType()));
-      (void)RecursivelyDeleteTriviallyDeadInstructions(I);
+      (void)RecursivelyDeleteTriviallyDeadInstructions(I, TLI);
       return true;
     }
   }
@@ -358,7 +397,8 @@ bool llvm::RecursivelyDeleteDeadPHINode(PHINode *PN) {
 ///
 /// This returns true if it changed the code, note that it can delete
 /// instructions in other blocks as well in this block.
-bool llvm::SimplifyInstructionsInBlock(BasicBlock *BB, const TargetData *TD) {
+bool llvm::SimplifyInstructionsInBlock(BasicBlock *BB, const DataLayout *TD,
+                                       const TargetLibraryInfo *TLI) {
   bool MadeChange = false;
 
 #ifndef NDEBUG
@@ -381,7 +421,7 @@ bool llvm::SimplifyInstructionsInBlock(BasicBlock *BB, const TargetData *TD) {
       continue;
     }
 
-    MadeChange |= RecursivelyDeleteTriviallyDeadInstructions(Inst);
+    MadeChange |= RecursivelyDeleteTriviallyDeadInstructions(Inst, TLI);
     if (BIHandle != BI)
       BI = BB->begin();
   }
@@ -405,7 +445,7 @@ bool llvm::SimplifyInstructionsInBlock(BasicBlock *BB, const TargetData *TD) {
 /// .. and delete the predecessor corresponding to the '1', this will attempt to
 /// recursively fold the and to 0.
 void llvm::RemovePredecessorAndSimplify(BasicBlock *BB, BasicBlock *Pred,
-                                        TargetData *TD) {
+                                        DataLayout *TD) {
   // This only adjusts blocks with PHI nodes.
   if (!isa<PHINode>(BB->begin()))
     return;
@@ -720,7 +760,7 @@ bool llvm::EliminateDuplicatePHINodes(BasicBlock *BB) {
 /// their preferred alignment from the beginning.
 ///
 static unsigned enforceKnownAlignment(Value *V, unsigned Align,
-                                      unsigned PrefAlign, const TargetData *TD) {
+                                      unsigned PrefAlign, const DataLayout *TD) {
   V = V->stripPointerCasts();
 
   if (AllocaInst *AI = dyn_cast<AllocaInst>(V)) {
@@ -763,7 +803,7 @@ static unsigned enforceKnownAlignment(Value *V, unsigned Align,
 /// and it is more than the alignment of the ultimate object, see if we can
 /// increase the alignment of the ultimate object, making this check succeed.
 unsigned llvm::getOrEnforceKnownAlignment(Value *V, unsigned PrefAlign,
-                                          const TargetData *TD) {
+                                          const DataLayout *TD) {
   assert(V->getType()->isPointerTy() &&
          "getOrEnforceKnownAlignment expects a pointer!");
   unsigned BitWidth = TD ? TD->getPointerSizeInBits() : 64;
diff --git a/lib/Transforms/Utils/LoopSimplify.cpp b/lib/Transforms/Utils/LoopSimplify.cpp
index 0bc185d..9d9e201 100644
--- a/lib/Transforms/Utils/LoopSimplify.cpp
+++ b/lib/Transforms/Utils/LoopSimplify.cpp
@@ -46,6 +46,7 @@
 #include "llvm/LLVMContext.h"
 #include "llvm/Type.h"
 #include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/Analysis/DependenceAnalysis.h"
 #include "llvm/Analysis/Dominators.h"
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/LoopPass.h"
@@ -89,6 +90,7 @@ namespace {
 
       AU.addPreserved<AliasAnalysis>();
       AU.addPreserved<ScalarEvolution>();
+      AU.addPreserved<DependenceAnalysis>();
       AU.addPreservedID(BreakCriticalEdgesID);  // No critical edges added.
     }
 
@@ -194,6 +196,11 @@ ReprocessLoop:
 
           BI->setCondition(ConstantInt::get(Cond->getType(),
                                             !L->contains(BI->getSuccessor(0))));
+
+          // This may make the loop analyzable, force SCEV recomputation.
+          if (SE)
+            SE->forgetLoop(L);
+
           Changed = true;
         }
       }
diff --git a/lib/Transforms/Utils/MetaRenamer.cpp b/lib/Transforms/Utils/MetaRenamer.cpp
new file mode 100644
index 0000000..233bc12
--- /dev/null
+++ b/lib/Transforms/Utils/MetaRenamer.cpp
@@ -0,0 +1,132 @@
+//===- MetaRenamer.cpp - Rename everything with metasyntatic names --------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass renames everything with metasyntatic names. The intent is to use
+// this pass after bugpoint reduction to conceal the nature of the original
+// program.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/Transforms/IPO.h"
+#include "llvm/DerivedTypes.h"
+#include "llvm/Function.h"
+#include "llvm/Module.h"
+#include "llvm/Pass.h"
+#include "llvm/Type.h"
+#include "llvm/TypeFinder.h"
+
+using namespace llvm;
+
+namespace {
+
+  // This PRNG is from the ISO C spec. It is intentionally simple and
+  // unsuitable for cryptographic use. We're just looking for enough
+  // variety to surprise and delight users.
+  struct PRNG {
+    unsigned long next;
+
+    void srand(unsigned int seed) {
+      next = seed;
+    }
+
+    int rand(void) {
+      next = next * 1103515245 + 12345;
+      return (unsigned int)(next / 65536) % 32768;
+    }
+  };
+
+  struct MetaRenamer : public ModulePass {
+    static char ID; // Pass identification, replacement for typeid
+    MetaRenamer() : ModulePass(ID) {
+      initializeMetaRenamerPass(*PassRegistry::getPassRegistry());
+    }
+
+    void getAnalysisUsage(AnalysisUsage &AU) const {
+      AU.setPreservesAll();
+    }
+
+    bool runOnModule(Module &M) {
+      static const char *metaNames[] = {
+        // See http://en.wikipedia.org/wiki/Metasyntactic_variable
+        "foo", "bar", "baz", "quux", "barney", "snork", "zot", "blam", "hoge",
+        "wibble", "wobble", "widget", "wombat", "ham", "eggs", "pluto", "spam"
+      };
+
+      // Seed our PRNG with simple additive sum of ModuleID. We're looking to
+      // simply avoid always having the same function names, and we need to
+      // remain deterministic.
+      unsigned int randSeed = 0;
+      for (std::string::const_iterator I = M.getModuleIdentifier().begin(),
+           E = M.getModuleIdentifier().end(); I != E; ++I)
+        randSeed += *I;
+
+      PRNG prng;
+      prng.srand(randSeed);
+
+      // Rename all aliases
+      for (Module::alias_iterator AI = M.alias_begin(), AE = M.alias_end();
+           AI != AE; ++AI)
+        AI->setName("alias");
+
+      // Rename all global variables
+      for (Module::global_iterator GI = M.global_begin(), GE = M.global_end();
+           GI != GE; ++GI)
+        GI->setName("global");
+
+      // Rename all struct types
+      TypeFinder StructTypes;
+      StructTypes.run(M, true);
+      for (unsigned i = 0, e = StructTypes.size(); i != e; ++i) {
+        StructType *STy = StructTypes[i];
+        if (STy->isLiteral() || STy->getName().empty()) continue;
+
+        SmallString<128> NameStorage;
+        STy->setName((Twine("struct.") + metaNames[prng.rand() %
+                     array_lengthof(metaNames)]).toStringRef(NameStorage));
+      }
+
+      // Rename all functions
+      for (Module::iterator FI = M.begin(), FE = M.end();
+           FI != FE; ++FI) {
+        FI->setName(metaNames[prng.rand() % array_lengthof(metaNames)]);
+        runOnFunction(*FI);
+      }
+      return true;
+    }
+
+    bool runOnFunction(Function &F) {
+      for (Function::arg_iterator AI = F.arg_begin(), AE = F.arg_end();
+           AI != AE; ++AI)
+        if (!AI->getType()->isVoidTy())
+          AI->setName("arg");
+
+      for (Function::iterator BB = F.begin(), E = F.end(); BB != E; ++BB) {
+        BB->setName("bb");
+
+        for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; ++I)
+          if (!I->getType()->isVoidTy())
+            I->setName("tmp");
+      }
+      return true;
+    }
+  };
+}
+
+char MetaRenamer::ID = 0;
+INITIALIZE_PASS(MetaRenamer, "metarenamer", 
+                "Assign new names to everything", false, false)
+//===----------------------------------------------------------------------===//
+//
+// MetaRenamer - Rename everything with metasyntactic names.
+//
+ModulePass *llvm::createMetaRenamerPass() {
+  return new MetaRenamer();
+}
diff --git a/lib/Transforms/Utils/PromoteMemoryToRegister.cpp b/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
index dd5e20e..558de9d 100644
--- a/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
+++ b/lib/Transforms/Utils/PromoteMemoryToRegister.cpp
@@ -212,9 +212,13 @@ namespace {
     ///
     DenseMap<AllocaInst*, unsigned>  AllocaLookup;
 
-    /// NewPhiNodes - The PhiNodes we're adding.
+    /// NewPhiNodes - The PhiNodes we're adding.  That map is used to simplify
+    /// some Phi nodes as we iterate over it, so it should have deterministic
+    /// iterators.  We could use a MapVector, but since we already maintain a
+    /// map from BasicBlock* to a stable numbering (BBNumbers), the DenseMap is
+    /// more efficient (also supports removal).
     ///
-    DenseMap<std::pair<BasicBlock*, unsigned>, PHINode*> NewPhiNodes;
+    DenseMap<std::pair<unsigned, unsigned>, PHINode*> NewPhiNodes;
     
     /// PhiToAllocaMap - For each PHI node, keep track of which entry in Allocas
     /// it corresponds to.
@@ -588,7 +592,11 @@ void PromoteMem2Reg::run() {
   while (EliminatedAPHI) {
     EliminatedAPHI = false;
     
-    for (DenseMap<std::pair<BasicBlock*, unsigned>, PHINode*>::iterator I =
+    // Iterating over NewPhiNodes is deterministic, so it is safe to try to
+    // simplify and RAUW them as we go.  If it was not, we could add uses to
+    // the values we replace with in a non deterministic order, thus creating
+    // non deterministic def->use chains.
+    for (DenseMap<std::pair<unsigned, unsigned>, PHINode*>::iterator I =
            NewPhiNodes.begin(), E = NewPhiNodes.end(); I != E;) {
       PHINode *PN = I->second;
 
@@ -612,7 +620,7 @@ void PromoteMem2Reg::run() {
   // have incoming values for all predecessors.  Loop over all PHI nodes we have
   // created, inserting undef values if they are missing any incoming values.
   //
-  for (DenseMap<std::pair<BasicBlock*, unsigned>, PHINode*>::iterator I =
+  for (DenseMap<std::pair<unsigned, unsigned>, PHINode*>::iterator I =
          NewPhiNodes.begin(), E = NewPhiNodes.end(); I != E; ++I) {
     // We want to do this once per basic block.  As such, only process a block
     // when we find the PHI that is the first entry in the block.
@@ -992,7 +1000,7 @@ void PromoteMem2Reg::PromoteSingleBlockAlloca(AllocaInst *AI, AllocaInfo &Info,
 bool PromoteMem2Reg::QueuePhiNode(BasicBlock *BB, unsigned AllocaNo,
                                   unsigned &Version) {
   // Look up the basic-block in question.
-  PHINode *&PN = NewPhiNodes[std::make_pair(BB, AllocaNo)];
+  PHINode *&PN = NewPhiNodes[std::make_pair(BBNumbers[BB], AllocaNo)];
 
   // If the BB already has a phi node added for the i'th alloca then we're done!
   if (PN) return false;
diff --git a/lib/Transforms/Utils/SSAUpdater.cpp b/lib/Transforms/Utils/SSAUpdater.cpp
index e568a61..72d4199 100644
--- a/lib/Transforms/Utils/SSAUpdater.cpp
+++ b/lib/Transforms/Utils/SSAUpdater.cpp
@@ -39,7 +39,7 @@ SSAUpdater::SSAUpdater(SmallVectorImpl<PHINode*> *NewPHI)
   : AV(0), ProtoType(0), ProtoName(), InsertedPHIs(NewPHI) {}
 
 SSAUpdater::~SSAUpdater() {
-  delete &getAvailableVals(AV);
+  delete static_cast<AvailableValsTy*>(AV);
 }
 
 /// Initialize - Reset this object to get ready for a new set of SSA
diff --git a/lib/Transforms/Utils/SimplifyCFG.cpp b/lib/Transforms/Utils/SimplifyCFG.cpp
index 518df7c..c767da6 100644
--- a/lib/Transforms/Utils/SimplifyCFG.cpp
+++ b/lib/Transforms/Utils/SimplifyCFG.cpp
@@ -14,6 +14,7 @@
 #define DEBUG_TYPE "simplifycfg"
 #include "llvm/Transforms/Utils/Local.h"
 #include "llvm/Constants.h"
+#include "llvm/DataLayout.h"
 #include "llvm/DerivedTypes.h"
 #include "llvm/GlobalVariable.h"
 #include "llvm/IRBuilder.h"
@@ -22,6 +23,7 @@
 #include "llvm/LLVMContext.h"
 #include "llvm/MDBuilder.h"
 #include "llvm/Metadata.h"
+#include "llvm/Module.h"
 #include "llvm/Operator.h"
 #include "llvm/Type.h"
 #include "llvm/ADT/DenseMap.h"
@@ -38,7 +40,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/NoFolder.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/TargetTransformInfo.h"
 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
 #include <algorithm>
 #include <set>
@@ -53,6 +55,13 @@ static cl::opt<bool>
 DupRet("simplifycfg-dup-ret", cl::Hidden, cl::init(false),
        cl::desc("Duplicate return instructions into unconditional branches"));
 
+static cl::opt<bool>
+SinkCommon("simplifycfg-sink-common", cl::Hidden, cl::init(true),
+       cl::desc("Sink common instructions down to the end block"));
+
+STATISTIC(NumBitMaps, "Number of switch instructions turned into bitmaps");
+STATISTIC(NumLookupTables, "Number of switch instructions turned into lookup tables");
+STATISTIC(NumSinkCommons, "Number of common instructions sunk down to the end block");
 STATISTIC(NumSpeculations, "Number of speculative executed instructions");
 
 namespace {
@@ -68,10 +77,13 @@ namespace {
       // Comparing pointers is ok as we only rely on the order for uniquing.
       return Value < RHS.Value;
     }
+
+    bool operator==(BasicBlock *RHSDest) const { return Dest == RHSDest; }
   };
 
 class SimplifyCFGOpt {
-  const TargetData *const TD;
+  const DataLayout *const TD;
+  const TargetTransformInfo *const TTI;
 
   Value *isValueEqualityComparison(TerminatorInst *TI);
   BasicBlock *GetValueEqualityComparisonCases(TerminatorInst *TI,
@@ -91,7 +103,8 @@ class SimplifyCFGOpt {
   bool SimplifyCondBranch(BranchInst *BI, IRBuilder <>&Builder);
 
 public:
-  explicit SimplifyCFGOpt(const TargetData *td) : TD(td) {}
+  SimplifyCFGOpt(const DataLayout *td, const TargetTransformInfo *tti)
+      : TD(td), TTI(tti) {}
   bool run(BasicBlock *BB);
 };
 }
@@ -101,14 +114,14 @@ public:
 ///
 static bool SafeToMergeTerminators(TerminatorInst *SI1, TerminatorInst *SI2) {
   if (SI1 == SI2) return false;  // Can't merge with self!
-  
+
   // It is not safe to merge these two switch instructions if they have a common
   // successor, and if that successor has a PHI node, and if *that* PHI node has
   // conflicting incoming values from the two switch blocks.
   BasicBlock *SI1BB = SI1->getParent();
   BasicBlock *SI2BB = SI2->getParent();
   SmallPtrSet<BasicBlock*, 16> SI1Succs(succ_begin(SI1BB), succ_end(SI1BB));
-  
+
   for (succ_iterator I = succ_begin(SI2BB), E = succ_end(SI2BB); I != E; ++I)
     if (SI1Succs.count(*I))
       for (BasicBlock::iterator BBI = (*I)->begin();
@@ -118,7 +131,7 @@ static bool SafeToMergeTerminators(TerminatorInst *SI1, TerminatorInst *SI2) {
             PN->getIncomingValueForBlock(SI2BB))
           return false;
       }
-        
+
   return true;
 }
 
@@ -135,7 +148,7 @@ static bool isProfitableToFoldUnconditional(BranchInst *SI1,
   assert(SI1->isUnconditional() && SI2->isConditional());
 
   // We fold the unconditional branch if we can easily update all PHI nodes in
-  // common successors: 
+  // common successors:
   // 1> We have a constant incoming value for the conditional branch;
   // 2> We have "Cond" as the incoming value for the unconditional branch;
   // 3> SI2->getCondition() and Cond have same operands.
@@ -170,7 +183,7 @@ static bool isProfitableToFoldUnconditional(BranchInst *SI1,
 static void AddPredecessorToBlock(BasicBlock *Succ, BasicBlock *NewPred,
                                   BasicBlock *ExistPred) {
   if (!isa<PHINode>(Succ->begin())) return; // Quick exit if nothing to do
-  
+
   PHINode *PN;
   for (BasicBlock::iterator I = Succ->begin();
        (PN = dyn_cast<PHINode>(I)); ++I)
@@ -222,7 +235,7 @@ static Value *GetIfCondition(BasicBlock *BB, BasicBlock *&IfTrue,
     // doesn't dominate BB.
     if (Pred2->getSinglePredecessor() == 0)
       return 0;
-    
+
     // If we found a conditional branch predecessor, make sure that it branches
     // to BB and Pred2Br.  If it doesn't, this isn't an "if statement".
     if (Pred1Br->getSuccessor(0) == BB &&
@@ -252,7 +265,7 @@ static Value *GetIfCondition(BasicBlock *BB, BasicBlock *&IfTrue,
   // Otherwise, if this is a conditional branch, then we can use it!
   BranchInst *BI = dyn_cast<BranchInst>(CommonPred->getTerminator());
   if (BI == 0) return 0;
-  
+
   assert(BI->isConditional() && "Two successors but not conditional?");
   if (BI->getSuccessor(0) == Pred1) {
     IfTrue = Pred1;
@@ -345,7 +358,7 @@ static bool DominatesMergePoint(Value *V, BasicBlock *BB,
   // If we aren't allowing aggressive promotion anymore, then don't consider
   // instructions in the 'if region'.
   if (AggressiveInsts == 0) return false;
-  
+
   // If we have seen this instruction before, don't count it again.
   if (AggressiveInsts->count(I)) return true;
 
@@ -374,7 +387,7 @@ static bool DominatesMergePoint(Value *V, BasicBlock *BB,
 
 /// GetConstantInt - Extract ConstantInt from value, looking through IntToPtr
 /// and PointerNullValue. Return NULL if value is not a constant int.
-static ConstantInt *GetConstantInt(Value *V, const TargetData *TD) {
+static ConstantInt *GetConstantInt(Value *V, const DataLayout *TD) {
   // Normal constant int.
   ConstantInt *CI = dyn_cast<ConstantInt>(V);
   if (CI || !TD || !isa<Constant>(V) || !V->getType()->isPointerTy())
@@ -382,7 +395,7 @@ static ConstantInt *GetConstantInt(Value *V, const TargetData *TD) {
 
   // This is some kind of pointer constant. Turn it into a pointer-sized
   // ConstantInt if possible.
-  IntegerType *PtrTy = TD->getIntPtrType(V->getContext());
+  IntegerType *PtrTy = cast<IntegerType>(TD->getIntPtrType(V->getType()));
 
   // Null pointer means 0, see SelectionDAGBuilder::getValue(const Value*).
   if (isa<ConstantPointerNull>(V))
@@ -408,10 +421,10 @@ static ConstantInt *GetConstantInt(Value *V, const TargetData *TD) {
 /// Values vector.
 static Value *
 GatherConstantCompares(Value *V, std::vector<ConstantInt*> &Vals, Value *&Extra,
-                       const TargetData *TD, bool isEQ, unsigned &UsedICmps) {
+                       const DataLayout *TD, bool isEQ, unsigned &UsedICmps) {
   Instruction *I = dyn_cast<Instruction>(V);
   if (I == 0) return 0;
-  
+
   // If this is an icmp against a constant, handle this as one of the cases.
   if (ICmpInst *ICI = dyn_cast<ICmpInst>(I)) {
     if (ConstantInt *C = GetConstantInt(I->getOperand(1), TD)) {
@@ -420,21 +433,21 @@ GatherConstantCompares(Value *V, std::vector<ConstantInt*> &Vals, Value *&Extra,
         Vals.push_back(C);
         return I->getOperand(0);
       }
-      
+
       // If we have "x ult 3" comparison, for example, then we can add 0,1,2 to
       // the set.
       ConstantRange Span =
         ConstantRange::makeICmpRegion(ICI->getPredicate(), C->getValue());
-      
+
       // If this is an and/!= check then we want to optimize "x ugt 2" into
       // x != 0 && x != 1.
       if (!isEQ)
         Span = Span.inverse();
-      
+
       // If there are a ton of values, we don't want to make a ginormous switch.
       if (Span.getSetSize().ugt(8) || Span.isEmptySet())
         return 0;
-      
+
       for (APInt Tmp = Span.getLower(); Tmp != Span.getUpper(); ++Tmp)
         Vals.push_back(ConstantInt::get(V->getContext(), Tmp));
       UsedICmps++;
@@ -442,11 +455,11 @@ GatherConstantCompares(Value *V, std::vector<ConstantInt*> &Vals, Value *&Extra,
     }
     return 0;
   }
-  
+
   // Otherwise, we can only handle an | or &, depending on isEQ.
   if (I->getOpcode() != (isEQ ? Instruction::Or : Instruction::And))
     return 0;
-  
+
   unsigned NumValsBeforeLHS = Vals.size();
   unsigned UsedICmpsBeforeLHS = UsedICmps;
   if (Value *LHS = GatherConstantCompares(I->getOperand(0), Vals, Extra, TD,
@@ -467,12 +480,12 @@ GatherConstantCompares(Value *V, std::vector<ConstantInt*> &Vals, Value *&Extra,
       Extra = I->getOperand(1);
       return LHS;
     }
-    
+
     Vals.resize(NumValsBeforeLHS);
     UsedICmps = UsedICmpsBeforeLHS;
     return 0;
   }
-  
+
   // If the LHS can't be folded in, but Extra is available and RHS can, try to
   // use LHS as Extra.
   if (Extra == 0 || Extra == I->getOperand(0)) {
@@ -484,7 +497,7 @@ GatherConstantCompares(Value *V, std::vector<ConstantInt*> &Vals, Value *&Extra,
     assert(Vals.size() == NumValsBeforeLHS);
     Extra = OldExtra;
   }
-  
+
   return 0;
 }
 
@@ -556,11 +569,7 @@ GetValueEqualityComparisonCases(TerminatorInst *TI,
 /// in the list that match the specified block.
 static void EliminateBlockCases(BasicBlock *BB,
                               std::vector<ValueEqualityComparisonCase> &Cases) {
-  for (unsigned i = 0, e = Cases.size(); i != e; ++i)
-    if (Cases[i].Dest == BB) {
-      Cases.erase(Cases.begin()+i);
-      --i; --e;
-    }
+  Cases.erase(std::remove(Cases.begin(), Cases.end(), BB), Cases.end());
 }
 
 /// ValuesOverlap - Return true if there are any keys in C1 that exist in C2 as
@@ -615,6 +624,9 @@ SimplifyEqualityComparisonWithOnlyPredecessor(TerminatorInst *TI,
   assert(ThisVal && "This isn't a value comparison!!");
   if (ThisVal != PredVal) return false;  // Different predicates.
 
+  // TODO: Preserve branch weight metadata, similarly to how
+  // FoldValueComparisonIntoPredecessors preserves it.
+
   // Find out information about when control will move from Pred to TI's block.
   std::vector<ValueEqualityComparisonCase> PredCases;
   BasicBlock *PredDef = GetValueEqualityComparisonCases(Pred->getTerminator(),
@@ -634,7 +646,7 @@ SimplifyEqualityComparisonWithOnlyPredecessor(TerminatorInst *TI,
     // can simplify TI.
     if (!ValuesOverlap(PredCases, ThisCases))
       return false;
-    
+
     if (isa<BranchInst>(TI)) {
       // Okay, one of the successors of this condbr is dead.  Convert it to a
       // uncond br.
@@ -652,7 +664,7 @@ SimplifyEqualityComparisonWithOnlyPredecessor(TerminatorInst *TI,
       EraseTerminatorInstAndDCECond(TI);
       return true;
     }
-      
+
     SwitchInst *SI = cast<SwitchInst>(TI);
     // Okay, TI has cases that are statically dead, prune them away.
     SmallPtrSet<Constant*, 16> DeadCases;
@@ -662,18 +674,37 @@ SimplifyEqualityComparisonWithOnlyPredecessor(TerminatorInst *TI,
     DEBUG(dbgs() << "Threading pred instr: " << *Pred->getTerminator()
                  << "Through successor TI: " << *TI);
 
+    // Collect branch weights into a vector.
+    SmallVector<uint32_t, 8> Weights;
+    MDNode* MD = SI->getMetadata(LLVMContext::MD_prof);
+    bool HasWeight = MD && (MD->getNumOperands() == 2 + SI->getNumCases());
+    if (HasWeight)
+      for (unsigned MD_i = 1, MD_e = MD->getNumOperands(); MD_i < MD_e;
+           ++MD_i) {
+        ConstantInt* CI = dyn_cast<ConstantInt>(MD->getOperand(MD_i));
+        assert(CI);
+        Weights.push_back(CI->getValue().getZExtValue());
+      }
     for (SwitchInst::CaseIt i = SI->case_end(), e = SI->case_begin(); i != e;) {
       --i;
       if (DeadCases.count(i.getCaseValue())) {
+        if (HasWeight) {
+          std::swap(Weights[i.getCaseIndex()+1], Weights.back());
+          Weights.pop_back();
+        }
         i.getCaseSuccessor()->removePredecessor(TI->getParent());
         SI->removeCase(i);
       }
     }
+    if (HasWeight && Weights.size() >= 2)
+      SI->setMetadata(LLVMContext::MD_prof,
+                      MDBuilder(SI->getParent()->getContext()).
+                      createBranchWeights(Weights));
 
     DEBUG(dbgs() << "Leaving: " << *TI << "\n");
     return true;
   }
-  
+
   // Otherwise, TI's block must correspond to some matched value.  Find out
   // which value (or set of values) this is.
   ConstantInt *TIV = 0;
@@ -729,8 +760,8 @@ namespace {
 }
 
 static int ConstantIntSortPredicate(const void *P1, const void *P2) {
-  const ConstantInt *LHS = *(const ConstantInt**)P1;
-  const ConstantInt *RHS = *(const ConstantInt**)P2;
+  const ConstantInt *LHS = *(const ConstantInt*const*)P1;
+  const ConstantInt *RHS = *(const ConstantInt*const*)P2;
   if (LHS->getValue().ult(RHS->getValue()))
     return 1;
   if (LHS->getValue() == RHS->getValue())
@@ -738,6 +769,56 @@ static int ConstantIntSortPredicate(const void *P1, const void *P2) {
   return -1;
 }
 
+static inline bool HasBranchWeights(const Instruction* I) {
+  MDNode* ProfMD = I->getMetadata(LLVMContext::MD_prof);
+  if (ProfMD && ProfMD->getOperand(0))
+    if (MDString* MDS = dyn_cast<MDString>(ProfMD->getOperand(0)))
+      return MDS->getString().equals("branch_weights");
+
+  return false;
+}
+
+/// Get Weights of a given TerminatorInst, the default weight is at the front
+/// of the vector. If TI is a conditional eq, we need to swap the branch-weight
+/// metadata.
+static void GetBranchWeights(TerminatorInst *TI,
+                             SmallVectorImpl<uint64_t> &Weights) {
+  MDNode* MD = TI->getMetadata(LLVMContext::MD_prof);
+  assert(MD);
+  for (unsigned i = 1, e = MD->getNumOperands(); i < e; ++i) {
+    ConstantInt* CI = dyn_cast<ConstantInt>(MD->getOperand(i));
+    assert(CI);
+    Weights.push_back(CI->getValue().getZExtValue());
+  }
+
+  // If TI is a conditional eq, the default case is the false case,
+  // and the corresponding branch-weight data is at index 2. We swap the
+  // default weight to be the first entry.
+  if (BranchInst* BI = dyn_cast<BranchInst>(TI)) {
+    assert(Weights.size() == 2);
+    ICmpInst *ICI = cast<ICmpInst>(BI->getCondition());
+    if (ICI->getPredicate() == ICmpInst::ICMP_EQ)
+      std::swap(Weights.front(), Weights.back());
+  }
+}
+
+/// Sees if any of the weights are too big for a uint32_t, and halves all the
+/// weights if any are.
+static void FitWeights(MutableArrayRef<uint64_t> Weights) {
+  bool Halve = false;
+  for (unsigned i = 0; i < Weights.size(); ++i)
+    if (Weights[i] > UINT_MAX) {
+      Halve = true;
+      break;
+    }
+
+  if (! Halve)
+    return;
+
+  for (unsigned i = 0; i < Weights.size(); ++i)
+    Weights[i] /= 2;
+}
+
 /// FoldValueComparisonIntoPredecessors - The specified terminator is a value
 /// equality comparison instruction (either a switch or a branch on "X == c").
 /// See if any of the predecessors of the terminator block are value comparisons
@@ -770,6 +851,31 @@ bool SimplifyCFGOpt::FoldValueComparisonIntoPredecessors(TerminatorInst *TI,
       // build.
       SmallVector<BasicBlock*, 8> NewSuccessors;
 
+      // Update the branch weight metadata along the way
+      SmallVector<uint64_t, 8> Weights;
+      bool PredHasWeights = HasBranchWeights(PTI);
+      bool SuccHasWeights = HasBranchWeights(TI);
+
+      if (PredHasWeights) {
+        GetBranchWeights(PTI, Weights);
+        // branch-weight metadata is inconsistant here.
+        if (Weights.size() != 1 + PredCases.size())
+          PredHasWeights = SuccHasWeights = false;
+      } else if (SuccHasWeights)
+        // If there are no predecessor weights but there are successor weights,
+        // populate Weights with 1, which will later be scaled to the sum of
+        // successor's weights
+        Weights.assign(1 + PredCases.size(), 1);
+
+      SmallVector<uint64_t, 8> SuccWeights;
+      if (SuccHasWeights) {
+        GetBranchWeights(TI, SuccWeights);
+        // branch-weight metadata is inconsistant here.
+        if (SuccWeights.size() != 1 + BBCases.size())
+          PredHasWeights = SuccHasWeights = false;
+      } else if (PredHasWeights)
+        SuccWeights.assign(1 + BBCases.size(), 1);
+
       if (PredDefault == BB) {
         // If this is the default destination from PTI, only the edges in TI
         // that don't occur in PTI, or that branch to BB will be activated.
@@ -780,6 +886,14 @@ bool SimplifyCFGOpt::FoldValueComparisonIntoPredecessors(TerminatorInst *TI,
           else {
             // The default destination is BB, we don't need explicit targets.
             std::swap(PredCases[i], PredCases.back());
+
+            if (PredHasWeights || SuccHasWeights) {
+              // Increase weight for the default case.
+              Weights[0] += Weights[i+1];
+              std::swap(Weights[i+1], Weights.back());
+              Weights.pop_back();
+            }
+
             PredCases.pop_back();
             --i; --e;
           }
@@ -790,21 +904,47 @@ bool SimplifyCFGOpt::FoldValueComparisonIntoPredecessors(TerminatorInst *TI,
           PredDefault = BBDefault;
           NewSuccessors.push_back(BBDefault);
         }
+
+        unsigned CasesFromPred = Weights.size();
+        uint64_t ValidTotalSuccWeight = 0;
         for (unsigned i = 0, e = BBCases.size(); i != e; ++i)
           if (!PTIHandled.count(BBCases[i].Value) &&
               BBCases[i].Dest != BBDefault) {
             PredCases.push_back(BBCases[i]);
             NewSuccessors.push_back(BBCases[i].Dest);
+            if (SuccHasWeights || PredHasWeights) {
+              // The default weight is at index 0, so weight for the ith case
+              // should be at index i+1. Scale the cases from successor by
+              // PredDefaultWeight (Weights[0]).
+              Weights.push_back(Weights[0] * SuccWeights[i+1]);
+              ValidTotalSuccWeight += SuccWeights[i+1];
+            }
           }
 
+        if (SuccHasWeights || PredHasWeights) {
+          ValidTotalSuccWeight += SuccWeights[0];
+          // Scale the cases from predecessor by ValidTotalSuccWeight.
+          for (unsigned i = 1; i < CasesFromPred; ++i)
+            Weights[i] *= ValidTotalSuccWeight;
+          // Scale the default weight by SuccDefaultWeight (SuccWeights[0]).
+          Weights[0] *= SuccWeights[0];
+        }
       } else {
         // If this is not the default destination from PSI, only the edges
         // in SI that occur in PSI with a destination of BB will be
         // activated.
         std::set<ConstantInt*, ConstantIntOrdering> PTIHandled;
+        std::map<ConstantInt*, uint64_t> WeightsForHandled;
         for (unsigned i = 0, e = PredCases.size(); i != e; ++i)
           if (PredCases[i].Dest == BB) {
             PTIHandled.insert(PredCases[i].Value);
+
+            if (PredHasWeights || SuccHasWeights) {
+              WeightsForHandled[PredCases[i].Value] = Weights[i+1];
+              std::swap(Weights[i+1], Weights.back());
+              Weights.pop_back();
+            }
+
             std::swap(PredCases[i], PredCases.back());
             PredCases.pop_back();
             --i; --e;
@@ -815,6 +955,8 @@ bool SimplifyCFGOpt::FoldValueComparisonIntoPredecessors(TerminatorInst *TI,
         for (unsigned i = 0, e = BBCases.size(); i != e; ++i)
           if (PTIHandled.count(BBCases[i].Value)) {
             // If this is one we are capable of getting...
+            if (PredHasWeights || SuccHasWeights)
+              Weights.push_back(WeightsForHandled[BBCases[i].Value]);
             PredCases.push_back(BBCases[i]);
             NewSuccessors.push_back(BBCases[i].Dest);
             PTIHandled.erase(BBCases[i].Value);// This constant is taken care of
@@ -822,9 +964,11 @@ bool SimplifyCFGOpt::FoldValueComparisonIntoPredecessors(TerminatorInst *TI,
 
         // If there are any constants vectored to BB that TI doesn't handle,
         // they must go to the default destination of TI.
-        for (std::set<ConstantInt*, ConstantIntOrdering>::iterator I = 
+        for (std::set<ConstantInt*, ConstantIntOrdering>::iterator I =
                                     PTIHandled.begin(),
                E = PTIHandled.end(); I != E; ++I) {
+          if (PredHasWeights || SuccHasWeights) 
+            Weights.push_back(WeightsForHandled[*I]); 
           PredCases.push_back(ValueEqualityComparisonCase(*I, BBDefault));
           NewSuccessors.push_back(BBDefault);
         }
@@ -839,7 +983,7 @@ bool SimplifyCFGOpt::FoldValueComparisonIntoPredecessors(TerminatorInst *TI,
       Builder.SetInsertPoint(PTI);
       // Convert pointer to int before we switch.
       if (CV->getType()->isPointerTy()) {
-        assert(TD && "Cannot switch on pointer without TargetData");
+        assert(TD && "Cannot switch on pointer without DataLayout");
         CV = Builder.CreatePtrToInt(CV, TD->getIntPtrType(CV->getContext()),
                                     "magicptr");
       }
@@ -851,6 +995,17 @@ bool SimplifyCFGOpt::FoldValueComparisonIntoPredecessors(TerminatorInst *TI,
       for (unsigned i = 0, e = PredCases.size(); i != e; ++i)
         NewSI->addCase(PredCases[i].Value, PredCases[i].Dest);
 
+      if (PredHasWeights || SuccHasWeights) {
+        // Halve the weights if any of them cannot fit in an uint32_t
+        FitWeights(Weights);
+
+        SmallVector<uint32_t, 8> MDWeights(Weights.begin(), Weights.end());
+
+        NewSI->setMetadata(LLVMContext::MD_prof,
+                           MDBuilder(BB->getContext()).
+                           createBranchWeights(MDWeights));
+      }
+
       EraseTerminatorInstAndDCECond(PTI);
 
       // Okay, last check.  If BB is still a successor of PSI, then we must
@@ -984,11 +1139,11 @@ HoistTerminator:
       Value *BB1V = PN->getIncomingValueForBlock(BB1);
       Value *BB2V = PN->getIncomingValueForBlock(BB2);
       if (BB1V == BB2V) continue;
-      
+
       // These values do not agree.  Insert a select instruction before NT
       // that determines the right value.
       SelectInst *&SI = InsertedSelects[std::make_pair(BB1V, BB2V)];
-      if (SI == 0) 
+      if (SI == 0)
         SI = cast<SelectInst>
           (Builder.CreateSelect(BI->getCondition(), BB1V, BB2V,
                                 BB1V->getName()+"."+BB2V->getName()));
@@ -1008,6 +1163,175 @@ HoistTerminator:
   return true;
 }
 
+/// SinkThenElseCodeToEnd - Given an unconditional branch that goes to BBEnd,
+/// check whether BBEnd has only two predecessors and the other predecessor
+/// ends with an unconditional branch. If it is true, sink any common code
+/// in the two predecessors to BBEnd.
+static bool SinkThenElseCodeToEnd(BranchInst *BI1) {
+  assert(BI1->isUnconditional());
+  BasicBlock *BB1 = BI1->getParent();
+  BasicBlock *BBEnd = BI1->getSuccessor(0);
+
+  // Check that BBEnd has two predecessors and the other predecessor ends with
+  // an unconditional branch.
+  pred_iterator PI = pred_begin(BBEnd), PE = pred_end(BBEnd);
+  BasicBlock *Pred0 = *PI++;
+  if (PI == PE) // Only one predecessor.
+    return false;
+  BasicBlock *Pred1 = *PI++;
+  if (PI != PE) // More than two predecessors.
+    return false;
+  BasicBlock *BB2 = (Pred0 == BB1) ? Pred1 : Pred0;
+  BranchInst *BI2 = dyn_cast<BranchInst>(BB2->getTerminator());
+  if (!BI2 || !BI2->isUnconditional())
+    return false;
+
+  // Gather the PHI nodes in BBEnd.
+  std::map<Value*, std::pair<Value*, PHINode*> > MapValueFromBB1ToBB2;
+  Instruction *FirstNonPhiInBBEnd = 0;
+  for (BasicBlock::iterator I = BBEnd->begin(), E = BBEnd->end();
+       I != E; ++I) {
+    if (PHINode *PN = dyn_cast<PHINode>(I)) {
+      Value *BB1V = PN->getIncomingValueForBlock(BB1);
+      Value *BB2V = PN->getIncomingValueForBlock(BB2); 
+      MapValueFromBB1ToBB2[BB1V] = std::make_pair(BB2V, PN);
+    } else {
+      FirstNonPhiInBBEnd = &*I;
+      break;
+    }
+  }
+  if (!FirstNonPhiInBBEnd)
+    return false;
+  
+
+  // This does very trivial matching, with limited scanning, to find identical
+  // instructions in the two blocks.  We scan backward for obviously identical
+  // instructions in an identical order.
+  BasicBlock::InstListType::reverse_iterator RI1 = BB1->getInstList().rbegin(),
+      RE1 = BB1->getInstList().rend(), RI2 = BB2->getInstList().rbegin(),
+      RE2 = BB2->getInstList().rend();
+  // Skip debug info.
+  while (RI1 != RE1 && isa<DbgInfoIntrinsic>(&*RI1)) ++RI1;
+  if (RI1 == RE1)
+    return false;
+  while (RI2 != RE2 && isa<DbgInfoIntrinsic>(&*RI2)) ++RI2;
+  if (RI2 == RE2)
+    return false;
+  // Skip the unconditional branches.
+  ++RI1;
+  ++RI2;
+
+  bool Changed = false;
+  while (RI1 != RE1 && RI2 != RE2) {
+    // Skip debug info.
+    while (RI1 != RE1 && isa<DbgInfoIntrinsic>(&*RI1)) ++RI1;
+    if (RI1 == RE1)
+      return Changed;
+    while (RI2 != RE2 && isa<DbgInfoIntrinsic>(&*RI2)) ++RI2;
+    if (RI2 == RE2)
+      return Changed;
+
+    Instruction *I1 = &*RI1, *I2 = &*RI2;
+    // I1 and I2 should have a single use in the same PHI node, and they
+    // perform the same operation.
+    // Cannot move control-flow-involving, volatile loads, vaarg, etc.
+    if (isa<PHINode>(I1) || isa<PHINode>(I2) ||
+        isa<TerminatorInst>(I1) || isa<TerminatorInst>(I2) ||
+        isa<LandingPadInst>(I1) || isa<LandingPadInst>(I2) ||
+        isa<AllocaInst>(I1) || isa<AllocaInst>(I2) ||
+        I1->mayHaveSideEffects() || I2->mayHaveSideEffects() ||
+        I1->mayReadOrWriteMemory() || I2->mayReadOrWriteMemory() ||
+        !I1->hasOneUse() || !I2->hasOneUse() ||
+        MapValueFromBB1ToBB2.find(I1) == MapValueFromBB1ToBB2.end() ||
+        MapValueFromBB1ToBB2[I1].first != I2)
+      return Changed;
+
+    // Check whether we should swap the operands of ICmpInst.
+    ICmpInst *ICmp1 = dyn_cast<ICmpInst>(I1), *ICmp2 = dyn_cast<ICmpInst>(I2);
+    bool SwapOpnds = false;
+    if (ICmp1 && ICmp2 &&
+        ICmp1->getOperand(0) != ICmp2->getOperand(0) &&
+        ICmp1->getOperand(1) != ICmp2->getOperand(1) &&
+        (ICmp1->getOperand(0) == ICmp2->getOperand(1) ||
+         ICmp1->getOperand(1) == ICmp2->getOperand(0))) {
+      ICmp2->swapOperands();
+      SwapOpnds = true;
+    }
+    if (!I1->isSameOperationAs(I2)) {
+      if (SwapOpnds)
+        ICmp2->swapOperands();
+      return Changed;
+    }
+
+    // The operands should be either the same or they need to be generated
+    // with a PHI node after sinking. We only handle the case where there is
+    // a single pair of different operands.
+    Value *DifferentOp1 = 0, *DifferentOp2 = 0;
+    unsigned Op1Idx = 0;
+    for (unsigned I = 0, E = I1->getNumOperands(); I != E; ++I) {
+      if (I1->getOperand(I) == I2->getOperand(I))
+        continue;
+      // Early exit if we have more-than one pair of different operands or
+      // the different operand is already in MapValueFromBB1ToBB2.
+      // Early exit if we need a PHI node to replace a constant.
+      if (DifferentOp1 ||
+          MapValueFromBB1ToBB2.find(I1->getOperand(I)) !=
+          MapValueFromBB1ToBB2.end() ||
+          isa<Constant>(I1->getOperand(I)) ||
+          isa<Constant>(I2->getOperand(I))) {
+        // If we can't sink the instructions, undo the swapping.
+        if (SwapOpnds)
+          ICmp2->swapOperands();
+        return Changed;
+      }
+      DifferentOp1 = I1->getOperand(I);
+      Op1Idx = I;
+      DifferentOp2 = I2->getOperand(I);
+    }
+
+    // We insert the pair of different operands to MapValueFromBB1ToBB2 and
+    // remove (I1, I2) from MapValueFromBB1ToBB2.
+    if (DifferentOp1) {
+      PHINode *NewPN = PHINode::Create(DifferentOp1->getType(), 2,
+                                       DifferentOp1->getName() + ".sink",
+                                       BBEnd->begin());
+      MapValueFromBB1ToBB2[DifferentOp1] = std::make_pair(DifferentOp2, NewPN);
+      // I1 should use NewPN instead of DifferentOp1.
+      I1->setOperand(Op1Idx, NewPN);
+      NewPN->addIncoming(DifferentOp1, BB1);
+      NewPN->addIncoming(DifferentOp2, BB2);
+      DEBUG(dbgs() << "Create PHI node " << *NewPN << "\n";);
+    }
+    PHINode *OldPN = MapValueFromBB1ToBB2[I1].second;
+    MapValueFromBB1ToBB2.erase(I1);
+
+    DEBUG(dbgs() << "SINK common instructions " << *I1 << "\n";);
+    DEBUG(dbgs() << "                         " << *I2 << "\n";);
+    // We need to update RE1 and RE2 if we are going to sink the first
+    // instruction in the basic block down.
+    bool UpdateRE1 = (I1 == BB1->begin()), UpdateRE2 = (I2 == BB2->begin());
+    // Sink the instruction.
+    BBEnd->getInstList().splice(FirstNonPhiInBBEnd, BB1->getInstList(), I1);
+    if (!OldPN->use_empty())
+      OldPN->replaceAllUsesWith(I1);
+    OldPN->eraseFromParent();
+
+    if (!I2->use_empty())
+      I2->replaceAllUsesWith(I1);
+    I1->intersectOptionalDataWith(I2);
+    I2->eraseFromParent();
+
+    if (UpdateRE1)
+      RE1 = BB1->getInstList().rend();
+    if (UpdateRE2)
+      RE2 = BB2->getInstList().rend();
+    FirstNonPhiInBBEnd = I1;
+    NumSinkCommons++;
+    Changed = true;
+  }
+  return Changed;
+}
+
 /// SpeculativelyExecuteBB - Given a conditional branch that goes to BB1
 /// and an BB2 and the only successor of BB1 is BB2, hoist simple code
 /// (for now, restricted to a single instruction that's side effect free) from
@@ -1056,7 +1380,7 @@ static bool SpeculativelyExecuteBB(BranchInst *BI, BasicBlock *BB1) {
     // Do not hoist the instruction if any of its operands are defined but not
     // used in this BB. The transformation will prevent the operand from
     // being sunk into the use block.
-    for (User::op_iterator i = HInst->op_begin(), e = HInst->op_end(); 
+    for (User::op_iterator i = HInst->op_begin(), e = HInst->op_end();
          i != e; ++i) {
       Instruction *OpI = dyn_cast<Instruction>(*i);
       if (OpI && OpI->getParent() == BIParent &&
@@ -1112,7 +1436,7 @@ static bool SpeculativelyExecuteBB(BranchInst *BI, BasicBlock *BB1) {
   // as well.
   if (PHIs.empty())
     return false;
-  
+
   // If we get here, we can hoist the instruction and if-convert.
   DEBUG(dbgs() << "SPECULATIVELY EXECUTING BB" << *BB1 << "\n";);
 
@@ -1162,13 +1486,13 @@ static bool SpeculativelyExecuteBB(BranchInst *BI, BasicBlock *BB1) {
 static bool BlockIsSimpleEnoughToThreadThrough(BasicBlock *BB) {
   BranchInst *BI = cast<BranchInst>(BB->getTerminator());
   unsigned Size = 0;
-  
+
   for (BasicBlock::iterator BBI = BB->begin(); &*BBI != BI; ++BBI) {
     if (isa<DbgInfoIntrinsic>(BBI))
       continue;
     if (Size > 10) return false;  // Don't clone large BB's.
     ++Size;
-    
+
     // We can only support instructions that do not define values that are
     // live outside of the current basic block.
     for (Value::use_iterator UI = BBI->use_begin(), E = BBI->use_end();
@@ -1176,7 +1500,7 @@ static bool BlockIsSimpleEnoughToThreadThrough(BasicBlock *BB) {
       Instruction *U = cast<Instruction>(*UI);
       if (U->getParent() != BB || isa<PHINode>(U)) return false;
     }
-    
+
     // Looks ok, continue checking.
   }
 
@@ -1187,38 +1511,38 @@ static bool BlockIsSimpleEnoughToThreadThrough(BasicBlock *BB) {
 /// that is defined in the same block as the branch and if any PHI entries are
 /// constants, thread edges corresponding to that entry to be branches to their
 /// ultimate destination.
-static bool FoldCondBranchOnPHI(BranchInst *BI, const TargetData *TD) {
+static bool FoldCondBranchOnPHI(BranchInst *BI, const DataLayout *TD) {
   BasicBlock *BB = BI->getParent();
   PHINode *PN = dyn_cast<PHINode>(BI->getCondition());
   // NOTE: we currently cannot transform this case if the PHI node is used
   // outside of the block.
   if (!PN || PN->getParent() != BB || !PN->hasOneUse())
     return false;
-  
+
   // Degenerate case of a single entry PHI.
   if (PN->getNumIncomingValues() == 1) {
     FoldSingleEntryPHINodes(PN->getParent());
-    return true;    
+    return true;
   }
 
   // Now we know that this block has multiple preds and two succs.
   if (!BlockIsSimpleEnoughToThreadThrough(BB)) return false;
-  
+
   // Okay, this is a simple enough basic block.  See if any phi values are
   // constants.
   for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
     ConstantInt *CB = dyn_cast<ConstantInt>(PN->getIncomingValue(i));
     if (CB == 0 || !CB->getType()->isIntegerTy(1)) continue;
-    
+
     // Okay, we now know that all edges from PredBB should be revectored to
     // branch to RealDest.
     BasicBlock *PredBB = PN->getIncomingBlock(i);
     BasicBlock *RealDest = BI->getSuccessor(!CB->getZExtValue());
-    
+
     if (RealDest == BB) continue;  // Skip self loops.
     // Skip if the predecessor's terminator is an indirect branch.
     if (isa<IndirectBrInst>(PredBB->getTerminator())) continue;
-    
+
     // The dest block might have PHI nodes, other predecessors and other
     // difficult cases.  Instead of being smart about this, just insert a new
     // block that jumps to the destination block, effectively splitting
@@ -1227,7 +1551,7 @@ static bool FoldCondBranchOnPHI(BranchInst *BI, const TargetData *TD) {
                                             RealDest->getName()+".critedge",
                                             RealDest->getParent(), RealDest);
     BranchInst::Create(RealDest, EdgeBB);
-    
+
     // Update PHI nodes.
     AddPredecessorToBlock(RealDest, EdgeBB, BB);
 
@@ -1244,7 +1568,7 @@ static bool FoldCondBranchOnPHI(BranchInst *BI, const TargetData *TD) {
       // Clone the instruction.
       Instruction *N = BBI->clone();
       if (BBI->hasName()) N->setName(BBI->getName()+".c");
-      
+
       // Update operands due to translation.
       for (User::op_iterator i = N->op_begin(), e = N->op_end();
            i != e; ++i) {
@@ -1252,7 +1576,7 @@ static bool FoldCondBranchOnPHI(BranchInst *BI, const TargetData *TD) {
         if (PI != TranslateMap.end())
           *i = PI->second;
       }
-      
+
       // Check for trivial simplification.
       if (Value *V = SimplifyInstruction(N, TD)) {
         TranslateMap[BBI] = V;
@@ -1283,7 +1607,7 @@ static bool FoldCondBranchOnPHI(BranchInst *BI, const TargetData *TD) {
 
 /// FoldTwoEntryPHINode - Given a BB that starts with the specified two-entry
 /// PHI node, see if we can eliminate it.
-static bool FoldTwoEntryPHINode(PHINode *PN, const TargetData *TD) {
+static bool FoldTwoEntryPHINode(PHINode *PN, const DataLayout *TD) {
   // Ok, this is a two entry PHI node.  Check to see if this is a simple "if
   // statement", which has a very simple dominance structure.  Basically, we
   // are trying to find the condition that is being branched on, which
@@ -1297,7 +1621,7 @@ static bool FoldTwoEntryPHINode(PHINode *PN, const TargetData *TD) {
       // Don't bother if the branch will be constant folded trivially.
       isa<ConstantInt>(IfCond))
     return false;
-  
+
   // Okay, we found that we can merge this two-entry phi node into a select.
   // Doing so would require us to fold *all* two entry phi nodes in this block.
   // At some point this becomes non-profitable (particularly if the target
@@ -1307,14 +1631,14 @@ static bool FoldTwoEntryPHINode(PHINode *PN, const TargetData *TD) {
   for (BasicBlock::iterator I = BB->begin(); isa<PHINode>(I); ++NumPhis, ++I)
     if (NumPhis > 2)
       return false;
-  
+
   // Loop over the PHI's seeing if we can promote them all to select
   // instructions.  While we are at it, keep track of the instructions
   // that need to be moved to the dominating block.
   SmallPtrSet<Instruction*, 4> AggressiveInsts;
   unsigned MaxCostVal0 = PHINodeFoldingThreshold,
            MaxCostVal1 = PHINodeFoldingThreshold;
-  
+
   for (BasicBlock::iterator II = BB->begin(); isa<PHINode>(II);) {
     PHINode *PN = cast<PHINode>(II++);
     if (Value *V = SimplifyInstruction(PN, TD)) {
@@ -1322,19 +1646,19 @@ static bool FoldTwoEntryPHINode(PHINode *PN, const TargetData *TD) {
       PN->eraseFromParent();
       continue;
     }
-    
+
     if (!DominatesMergePoint(PN->getIncomingValue(0), BB, &AggressiveInsts,
                              MaxCostVal0) ||
         !DominatesMergePoint(PN->getIncomingValue(1), BB, &AggressiveInsts,
                              MaxCostVal1))
       return false;
   }
-  
+
   // If we folded the first phi, PN dangles at this point.  Refresh it.  If
   // we ran out of PHIs then we simplified them all.
   PN = dyn_cast<PHINode>(BB->begin());
   if (PN == 0) return true;
-  
+
   // Don't fold i1 branches on PHIs which contain binary operators.  These can
   // often be turned into switches and other things.
   if (PN->getType()->isIntegerTy(1) &&
@@ -1342,7 +1666,7 @@ static bool FoldTwoEntryPHINode(PHINode *PN, const TargetData *TD) {
        isa<BinaryOperator>(PN->getIncomingValue(1)) ||
        isa<BinaryOperator>(IfCond)))
     return false;
-  
+
   // If we all PHI nodes are promotable, check to make sure that all
   // instructions in the predecessor blocks can be promoted as well.  If
   // not, we won't be able to get rid of the control flow, so it's not
@@ -1362,7 +1686,7 @@ static bool FoldTwoEntryPHINode(PHINode *PN, const TargetData *TD) {
         return false;
       }
   }
-    
+
   if (cast<BranchInst>(IfBlock2->getTerminator())->isConditional()) {
     IfBlock2 = 0;
   } else {
@@ -1375,15 +1699,15 @@ static bool FoldTwoEntryPHINode(PHINode *PN, const TargetData *TD) {
         return false;
       }
   }
-  
+
   DEBUG(dbgs() << "FOUND IF CONDITION!  " << *IfCond << "  T: "
                << IfTrue->getName() << "  F: " << IfFalse->getName() << "\n");
-      
+
   // If we can still promote the PHI nodes after this gauntlet of tests,
   // do all of the PHI's now.
   Instruction *InsertPt = DomBlock->getTerminator();
   IRBuilder<true, NoFolder> Builder(InsertPt);
-  
+
   // Move all 'aggressive' instructions, which are defined in the
   // conditional parts of the if's up to the dominating block.
   if (IfBlock1)
@@ -1394,19 +1718,19 @@ static bool FoldTwoEntryPHINode(PHINode *PN, const TargetData *TD) {
     DomBlock->getInstList().splice(InsertPt,
                                    IfBlock2->getInstList(), IfBlock2->begin(),
                                    IfBlock2->getTerminator());
-  
+
   while (PHINode *PN = dyn_cast<PHINode>(BB->begin())) {
     // Change the PHI node into a select instruction.
     Value *TrueVal  = PN->getIncomingValue(PN->getIncomingBlock(0) == IfFalse);
     Value *FalseVal = PN->getIncomingValue(PN->getIncomingBlock(0) == IfTrue);
-    
-    SelectInst *NV = 
+
+    SelectInst *NV =
       cast<SelectInst>(Builder.CreateSelect(IfCond, TrueVal, FalseVal, ""));
     PN->replaceAllUsesWith(NV);
     NV->takeName(PN);
     PN->eraseFromParent();
   }
-  
+
   // At this point, IfBlock1 and IfBlock2 are both empty, so our if statement
   // has been flattened.  Change DomBlock to jump directly to our new block to
   // avoid other simplifycfg's kicking in on the diamond.
@@ -1420,14 +1744,14 @@ static bool FoldTwoEntryPHINode(PHINode *PN, const TargetData *TD) {
 /// SimplifyCondBranchToTwoReturns - If we found a conditional branch that goes
 /// to two returning blocks, try to merge them together into one return,
 /// introducing a select if the return values disagree.
-static bool SimplifyCondBranchToTwoReturns(BranchInst *BI, 
+static bool SimplifyCondBranchToTwoReturns(BranchInst *BI,
                                            IRBuilder<> &Builder) {
   assert(BI->isConditional() && "Must be a conditional branch");
   BasicBlock *TrueSucc = BI->getSuccessor(0);
   BasicBlock *FalseSucc = BI->getSuccessor(1);
   ReturnInst *TrueRet = cast<ReturnInst>(TrueSucc->getTerminator());
   ReturnInst *FalseRet = cast<ReturnInst>(FalseSucc->getTerminator());
-  
+
   // Check to ensure both blocks are empty (just a return) or optionally empty
   // with PHI nodes.  If there are other instructions, merging would cause extra
   // computation on one path or the other.
@@ -1447,12 +1771,12 @@ static bool SimplifyCondBranchToTwoReturns(BranchInst *BI,
     EraseTerminatorInstAndDCECond(BI);
     return true;
   }
-    
+
   // Otherwise, figure out what the true and false return values are
   // so we can insert a new select instruction.
   Value *TrueValue = TrueRet->getReturnValue();
   Value *FalseValue = FalseRet->getReturnValue();
-  
+
   // Unwrap any PHI nodes in the return blocks.
   if (PHINode *TVPN = dyn_cast_or_null<PHINode>(TrueValue))
     if (TVPN->getParent() == TrueSucc)
@@ -1460,7 +1784,7 @@ static bool SimplifyCondBranchToTwoReturns(BranchInst *BI,
   if (PHINode *FVPN = dyn_cast_or_null<PHINode>(FalseValue))
     if (FVPN->getParent() == FalseSucc)
       FalseValue = FVPN->getIncomingValueForBlock(BI->getParent());
-  
+
   // In order for this transformation to be safe, we must be able to
   // unconditionally execute both operands to the return.  This is
   // normally the case, but we could have a potentially-trapping
@@ -1472,12 +1796,12 @@ static bool SimplifyCondBranchToTwoReturns(BranchInst *BI,
   if (ConstantExpr *FCV = dyn_cast_or_null<ConstantExpr>(FalseValue))
     if (FCV->canTrap())
       return false;
-  
+
   // Okay, we collected all the mapped values and checked them for sanity, and
   // defined to really do this transformation.  First, update the CFG.
   TrueSucc->removePredecessor(BI->getParent());
   FalseSucc->removePredecessor(BI->getParent());
-  
+
   // Insert select instructions where needed.
   Value *BrCond = BI->getCondition();
   if (TrueValue) {
@@ -1491,15 +1815,15 @@ static bool SimplifyCondBranchToTwoReturns(BranchInst *BI,
     }
   }
 
-  Value *RI = !TrueValue ? 
+  Value *RI = !TrueValue ?
     Builder.CreateRetVoid() : Builder.CreateRet(TrueValue);
 
   (void) RI;
-      
+
   DEBUG(dbgs() << "\nCHANGING BRANCH TO TWO RETURNS INTO SELECT:"
                << "\n  " << *BI << "NewRet = " << *RI
                << "TRUEBLOCK: " << *TrueSucc << "FALSEBLOCK: "<< *FalseSucc);
-      
+
   EraseTerminatorInstAndDCECond(BI);
 
   return true;
@@ -1510,7 +1834,7 @@ static bool SimplifyCondBranchToTwoReturns(BranchInst *BI,
 /// parameters and return true, or returns false if no or invalid metadata was
 /// found.
 static bool ExtractBranchMetadata(BranchInst *BI,
-                                  APInt &ProbTrue, APInt &ProbFalse) {
+                                  uint64_t &ProbTrue, uint64_t &ProbFalse) {
   assert(BI->isConditional() &&
          "Looking for probabilities on unconditional branch?");
   MDNode *ProfileData = BI->getMetadata(LLVMContext::MD_prof);
@@ -1518,35 +1842,11 @@ static bool ExtractBranchMetadata(BranchInst *BI,
   ConstantInt *CITrue = dyn_cast<ConstantInt>(ProfileData->getOperand(1));
   ConstantInt *CIFalse = dyn_cast<ConstantInt>(ProfileData->getOperand(2));
   if (!CITrue || !CIFalse) return false;
-  ProbTrue = CITrue->getValue();
-  ProbFalse = CIFalse->getValue();
-  assert(ProbTrue.getBitWidth() == 32 && ProbFalse.getBitWidth() == 32 &&
-         "Branch probability metadata must be 32-bit integers");
+  ProbTrue = CITrue->getValue().getZExtValue();
+  ProbFalse = CIFalse->getValue().getZExtValue();
   return true;
 }
 
-/// MultiplyAndLosePrecision - Multiplies A and B, then returns the result. In
-/// the event of overflow, logically-shifts all four inputs right until the
-/// multiply fits.
-static APInt MultiplyAndLosePrecision(APInt &A, APInt &B, APInt &C, APInt &D,
-                                      unsigned &BitsLost) {
-  BitsLost = 0;
-  bool Overflow = false;
-  APInt Result = A.umul_ov(B, Overflow);
-  if (Overflow) {
-    APInt MaxB = APInt::getMaxValue(A.getBitWidth()).udiv(A);
-    do {
-      B = B.lshr(1);
-      ++BitsLost;
-    } while (B.ugt(MaxB));
-    A = A.lshr(BitsLost);
-    C = C.lshr(BitsLost);
-    D = D.lshr(BitsLost);
-    Result = A * B;
-  }
-  return Result;
-}
-
 /// checkCSEInPredecessor - Return true if the given instruction is available
 /// in its predecessor block. If yes, the instruction will be removed.
 ///
@@ -1600,7 +1900,7 @@ bool llvm::FoldBranchToCommonDest(BranchInst *BI) {
     if (Cond == 0)
       return false;
   }
-     
+
   if (Cond == 0 || (!isa<CmpInst>(Cond) && !isa<BinaryOperator>(Cond)) ||
     Cond->getParent() != BB || !Cond->hasOneUse())
   return false;
@@ -1623,7 +1923,7 @@ bool llvm::FoldBranchToCommonDest(BranchInst *BI) {
       isSafeToSpeculativelyExecute(FrontIt)) {
     BonusInst = &*FrontIt;
     ++FrontIt;
-    
+
     // Ignore dbg intrinsics.
     while (isa<DbgInfoIntrinsic>(FrontIt)) ++FrontIt;
   }
@@ -1631,13 +1931,13 @@ bool llvm::FoldBranchToCommonDest(BranchInst *BI) {
   // Only a single bonus inst is allowed.
   if (&*FrontIt != Cond)
     return false;
-  
+
   // Make sure the instruction after the condition is the cond branch.
   BasicBlock::iterator CondIt = Cond; ++CondIt;
 
   // Ingore dbg intrinsics.
   while (isa<DbgInfoIntrinsic>(CondIt)) ++CondIt;
-  
+
   if (&*CondIt != BI)
     return false;
 
@@ -1649,7 +1949,7 @@ bool llvm::FoldBranchToCommonDest(BranchInst *BI) {
   if (ConstantExpr *CE = dyn_cast<ConstantExpr>(Cond->getOperand(1)))
     if (CE->canTrap())
       return false;
-  
+
   // Finally, don't infinitely unroll conditional loops.
   BasicBlock *TrueDest  = BI->getSuccessor(0);
   BasicBlock *FalseDest = (BI->isConditional()) ? BI->getSuccessor(1) : 0;
@@ -1659,22 +1959,22 @@ bool llvm::FoldBranchToCommonDest(BranchInst *BI) {
   for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI) {
     BasicBlock *PredBlock = *PI;
     BranchInst *PBI = dyn_cast<BranchInst>(PredBlock->getTerminator());
-    
+
     // Check that we have two conditional branches.  If there is a PHI node in
     // the common successor, verify that the same value flows in from both
     // blocks.
     SmallVector<PHINode*, 4> PHIs;
     if (PBI == 0 || PBI->isUnconditional() ||
-        (BI->isConditional() && 
+        (BI->isConditional() &&
          !SafeToMergeTerminators(BI, PBI)) ||
         (!BI->isConditional() &&
          !isProfitableToFoldUnconditional(BI, PBI, Cond, PHIs)))
       continue;
-    
+
     // Determine if the two branches share a common destination.
-    Instruction::BinaryOps Opc;
+    Instruction::BinaryOps Opc = Instruction::BinaryOpsEnd;
     bool InvertPredCond = false;
-    
+
     if (BI->isConditional()) {
       if (PBI->getSuccessor(0) == TrueDest)
         Opc = Instruction::Or;
@@ -1693,7 +1993,7 @@ bool llvm::FoldBranchToCommonDest(BranchInst *BI) {
 
     // Ensure that any values used in the bonus instruction are also used
     // by the terminator of the predecessor.  This means that those values
-    // must already have been resolved, so we won't be inhibiting the 
+    // must already have been resolved, so we won't be inhibiting the
     // out-of-order core by speculating them earlier.
     if (BonusInst) {
       // Collect the values used by the bonus inst
@@ -1707,47 +2007,47 @@ bool llvm::FoldBranchToCommonDest(BranchInst *BI) {
 
       SmallVector<std::pair<Value*, unsigned>, 4> Worklist;
       Worklist.push_back(std::make_pair(PBI->getOperand(0), 0));
-      
+
       // Walk up to four levels back up the use-def chain of the predecessor's
       // terminator to see if all those values were used.  The choice of four
       // levels is arbitrary, to provide a compile-time-cost bound.
       while (!Worklist.empty()) {
         std::pair<Value*, unsigned> Pair = Worklist.back();
         Worklist.pop_back();
-        
+
         if (Pair.second >= 4) continue;
         UsedValues.erase(Pair.first);
         if (UsedValues.empty()) break;
-        
+
         if (Instruction *I = dyn_cast<Instruction>(Pair.first)) {
           for (Instruction::op_iterator OI = I->op_begin(), OE = I->op_end();
                OI != OE; ++OI)
             Worklist.push_back(std::make_pair(OI->get(), Pair.second+1));
-        }       
+        }
       }
-      
+
       if (!UsedValues.empty()) return false;
     }
 
     DEBUG(dbgs() << "FOLDING BRANCH TO COMMON DEST:\n" << *PBI << *BB);
-    IRBuilder<> Builder(PBI);    
+    IRBuilder<> Builder(PBI);
 
     // If we need to invert the condition in the pred block to match, do so now.
     if (InvertPredCond) {
       Value *NewCond = PBI->getCondition();
-      
+
       if (NewCond->hasOneUse() && isa<CmpInst>(NewCond)) {
         CmpInst *CI = cast<CmpInst>(NewCond);
         CI->setPredicate(CI->getInversePredicate());
       } else {
-        NewCond = Builder.CreateNot(NewCond, 
+        NewCond = Builder.CreateNot(NewCond,
                                     PBI->getCondition()->getName()+".not");
       }
-      
+
       PBI->setCondition(NewCond);
       PBI->swapSuccessors();
     }
-    
+
     // If we have a bonus inst, clone it into the predecessor block.
     Instruction *NewBonus = 0;
     if (BonusInst) {
@@ -1756,7 +2056,7 @@ bool llvm::FoldBranchToCommonDest(BranchInst *BI) {
       NewBonus->takeName(BonusInst);
       BonusInst->setName(BonusInst->getName()+".old");
     }
-    
+
     // Clone Cond into the predecessor basic block, and or/and the
     // two conditions together.
     Instruction *New = Cond->clone();
@@ -1764,21 +2064,60 @@ bool llvm::FoldBranchToCommonDest(BranchInst *BI) {
     PredBlock->getInstList().insert(PBI, New);
     New->takeName(Cond);
     Cond->setName(New->getName()+".old");
-    
+
     if (BI->isConditional()) {
-      Instruction *NewCond = 
+      Instruction *NewCond =
         cast<Instruction>(Builder.CreateBinOp(Opc, PBI->getCondition(),
                                             New, "or.cond"));
       PBI->setCondition(NewCond);
 
+      uint64_t PredTrueWeight, PredFalseWeight, SuccTrueWeight, SuccFalseWeight;
+      bool PredHasWeights = ExtractBranchMetadata(PBI, PredTrueWeight,
+                                                  PredFalseWeight);
+      bool SuccHasWeights = ExtractBranchMetadata(BI, SuccTrueWeight,
+                                                  SuccFalseWeight);
+      SmallVector<uint64_t, 8> NewWeights;
+
       if (PBI->getSuccessor(0) == BB) {
+        if (PredHasWeights && SuccHasWeights) {
+          // PBI: br i1 %x, BB, FalseDest
+          // BI:  br i1 %y, TrueDest, FalseDest
+          //TrueWeight is TrueWeight for PBI * TrueWeight for BI.
+          NewWeights.push_back(PredTrueWeight * SuccTrueWeight);
+          //FalseWeight is FalseWeight for PBI * TotalWeight for BI +
+          //               TrueWeight for PBI * FalseWeight for BI.
+          // We assume that total weights of a BranchInst can fit into 32 bits.
+          // Therefore, we will not have overflow using 64-bit arithmetic.
+          NewWeights.push_back(PredFalseWeight * (SuccFalseWeight +
+               SuccTrueWeight) + PredTrueWeight * SuccFalseWeight);
+        }
         AddPredecessorToBlock(TrueDest, PredBlock, BB);
         PBI->setSuccessor(0, TrueDest);
       }
       if (PBI->getSuccessor(1) == BB) {
+        if (PredHasWeights && SuccHasWeights) {
+          // PBI: br i1 %x, TrueDest, BB
+          // BI:  br i1 %y, TrueDest, FalseDest
+          //TrueWeight is TrueWeight for PBI * TotalWeight for BI +
+          //              FalseWeight for PBI * TrueWeight for BI.
+          NewWeights.push_back(PredTrueWeight * (SuccFalseWeight +
+              SuccTrueWeight) + PredFalseWeight * SuccTrueWeight);
+          //FalseWeight is FalseWeight for PBI * FalseWeight for BI.
+          NewWeights.push_back(PredFalseWeight * SuccFalseWeight);
+        }
         AddPredecessorToBlock(FalseDest, PredBlock, BB);
         PBI->setSuccessor(1, FalseDest);
       }
+      if (NewWeights.size() == 2) {
+        // Halve the weights if any of them cannot fit in an uint32_t
+        FitWeights(NewWeights);
+
+        SmallVector<uint32_t, 8> MDWeights(NewWeights.begin(),NewWeights.end());
+        PBI->setMetadata(LLVMContext::MD_prof,
+                         MDBuilder(BI->getContext()).
+                         createBranchWeights(MDWeights));
+      } else
+        PBI->setMetadata(LLVMContext::MD_prof, NULL);
     } else {
       // Update PHI nodes in the common successors.
       for (unsigned i = 0, e = PHIs.size(); i != e; ++i) {
@@ -1806,7 +2145,7 @@ bool llvm::FoldBranchToCommonDest(BranchInst *BI) {
           // Create (PBI_Cond and BI_Value) or (!PBI_Cond and PBI_C)
           // PBI_C is true: (PBI_Cond and BI_Value) or (!PBI_Cond)
           //       is false: PBI_Cond and BI_Value
-          MergedCond = 
+          MergedCond =
             cast<Instruction>(Builder.CreateBinOp(Instruction::And,
                                 PBI->getCondition(), New,
                                 "and.cond"));
@@ -1814,7 +2153,7 @@ bool llvm::FoldBranchToCommonDest(BranchInst *BI) {
             Instruction *NotCond =
               cast<Instruction>(Builder.CreateNot(PBI->getCondition(),
                                   "not.cond"));
-            MergedCond = 
+            MergedCond =
               cast<Instruction>(Builder.CreateBinOp(Instruction::Or,
                                   NotCond, MergedCond,
                                   "or.cond"));
@@ -1833,95 +2172,11 @@ bool llvm::FoldBranchToCommonDest(BranchInst *BI) {
     // TODO: If BB is reachable from all paths through PredBlock, then we
     // could replace PBI's branch probabilities with BI's.
 
-    // Merge probability data into PredBlock's branch.
-    APInt A, B, C, D;
-    if (PBI->isConditional() && BI->isConditional() &&
-        ExtractBranchMetadata(PBI, C, D) && ExtractBranchMetadata(BI, A, B)) {
-      // Given IR which does:
-      //   bbA:
-      //     br i1 %x, label %bbB, label %bbC
-      //   bbB:
-      //     br i1 %y, label %bbD, label %bbC
-      // Let's call the probability that we take the edge from %bbA to %bbB
-      // 'a', from %bbA to %bbC, 'b', from %bbB to %bbD 'c' and from %bbB to
-      // %bbC probability 'd'.
-      //
-      // We transform the IR into:
-      //   bbA:
-      //     br i1 %z, label %bbD, label %bbC
-      // where the probability of going to %bbD is (a*c) and going to bbC is
-      // (b+a*d).
-      //
-      // Probabilities aren't stored as ratios directly. Using branch weights,
-      // we get:
-      // (a*c)% = A*C, (b+(a*d))% = A*D+B*C+B*D.
-
-      // In the event of overflow, we want to drop the LSB of the input
-      // probabilities.
-      unsigned BitsLost;
-
-      // Ignore overflow result on ProbTrue.
-      APInt ProbTrue = MultiplyAndLosePrecision(A, C, B, D, BitsLost);
-
-      APInt Tmp1 = MultiplyAndLosePrecision(B, D, A, C, BitsLost);
-      if (BitsLost) {
-        ProbTrue = ProbTrue.lshr(BitsLost*2);
-      }
-
-      APInt Tmp2 = MultiplyAndLosePrecision(A, D, C, B, BitsLost);
-      if (BitsLost) {
-        ProbTrue = ProbTrue.lshr(BitsLost*2);
-        Tmp1 = Tmp1.lshr(BitsLost*2);
-      }
-
-      APInt Tmp3 = MultiplyAndLosePrecision(B, C, A, D, BitsLost);
-      if (BitsLost) {
-        ProbTrue = ProbTrue.lshr(BitsLost*2);
-        Tmp1 = Tmp1.lshr(BitsLost*2);
-        Tmp2 = Tmp2.lshr(BitsLost*2);
-      }
-
-      bool Overflow1 = false, Overflow2 = false;
-      APInt Tmp4 = Tmp2.uadd_ov(Tmp3, Overflow1);
-      APInt ProbFalse = Tmp4.uadd_ov(Tmp1, Overflow2);
-
-      if (Overflow1 || Overflow2) {
-        ProbTrue = ProbTrue.lshr(1);
-        Tmp1 = Tmp1.lshr(1);
-        Tmp2 = Tmp2.lshr(1);
-        Tmp3 = Tmp3.lshr(1);
-        Tmp4 = Tmp2 + Tmp3;
-        ProbFalse = Tmp4 + Tmp1;
-      }
-
-      // The sum of branch weights must fit in 32-bits.
-      if (ProbTrue.isNegative() && ProbFalse.isNegative()) {
-        ProbTrue = ProbTrue.lshr(1);
-        ProbFalse = ProbFalse.lshr(1);
-      }
-
-      if (ProbTrue != ProbFalse) {
-        // Normalize the result.
-        APInt GCD = APIntOps::GreatestCommonDivisor(ProbTrue, ProbFalse);
-        ProbTrue = ProbTrue.udiv(GCD);
-        ProbFalse = ProbFalse.udiv(GCD);
-
-        MDBuilder MDB(BI->getContext());
-        MDNode *N = MDB.createBranchWeights(ProbTrue.getZExtValue(),
-                                            ProbFalse.getZExtValue());
-        PBI->setMetadata(LLVMContext::MD_prof, N);
-      } else {
-        PBI->setMetadata(LLVMContext::MD_prof, NULL);
-      }
-    } else {
-      PBI->setMetadata(LLVMContext::MD_prof, NULL);
-    }
-
     // Copy any debug value intrinsics into the end of PredBlock.
     for (BasicBlock::iterator I = BB->begin(), E = BB->end(); I != E; ++I)
       if (isa<DbgInfoIntrinsic>(*I))
         I->clone()->insertBefore(PBI);
-      
+
     return true;
   }
   return false;
@@ -1936,7 +2191,7 @@ static bool SimplifyCondBranchToCondBranch(BranchInst *PBI, BranchInst *BI) {
   BasicBlock *BB = BI->getParent();
 
   // If this block ends with a branch instruction, and if there is a
-  // predecessor that ends on a branch of the same condition, make 
+  // predecessor that ends on a branch of the same condition, make
   // this conditional branch redundant.
   if (PBI->getCondition() == BI->getCondition() &&
       PBI->getSuccessor(0) != PBI->getSuccessor(1)) {
@@ -1945,11 +2200,11 @@ static bool SimplifyCondBranchToCondBranch(BranchInst *PBI, BranchInst *BI) {
     if (BB->getSinglePredecessor()) {
       // Turn this into a branch on constant.
       bool CondIsTrue = PBI->getSuccessor(0) == BB;
-      BI->setCondition(ConstantInt::get(Type::getInt1Ty(BB->getContext()), 
+      BI->setCondition(ConstantInt::get(Type::getInt1Ty(BB->getContext()),
                                         CondIsTrue));
       return true;  // Nuke the branch on constant.
     }
-    
+
     // Otherwise, if there are multiple predecessors, insert a PHI that merges
     // in the constant and simplify the block result.  Subsequent passes of
     // simplifycfg will thread the block.
@@ -1969,18 +2224,18 @@ static bool SimplifyCondBranchToCondBranch(BranchInst *PBI, BranchInst *BI) {
             PBI->getCondition() == BI->getCondition() &&
             PBI->getSuccessor(0) != PBI->getSuccessor(1)) {
           bool CondIsTrue = PBI->getSuccessor(0) == BB;
-          NewPN->addIncoming(ConstantInt::get(Type::getInt1Ty(BB->getContext()), 
+          NewPN->addIncoming(ConstantInt::get(Type::getInt1Ty(BB->getContext()),
                                               CondIsTrue), P);
         } else {
           NewPN->addIncoming(BI->getCondition(), P);
         }
       }
-      
+
       BI->setCondition(NewPN);
       return true;
     }
   }
-  
+
   // If this is a conditional branch in an empty block, and if any
   // predecessors is a conditional branch to one of our destinations,
   // fold the conditions into logical ops and one cond br.
@@ -1991,11 +2246,11 @@ static bool SimplifyCondBranchToCondBranch(BranchInst *PBI, BranchInst *BI) {
   if (&*BBI != BI)
     return false;
 
-  
+
   if (ConstantExpr *CE = dyn_cast<ConstantExpr>(BI->getCondition()))
     if (CE->canTrap())
       return false;
-  
+
   int PBIOp, BIOp;
   if (PBI->getSuccessor(0) == BI->getSuccessor(0))
     PBIOp = BIOp = 0;
@@ -2007,31 +2262,31 @@ static bool SimplifyCondBranchToCondBranch(BranchInst *PBI, BranchInst *BI) {
     PBIOp = BIOp = 1;
   else
     return false;
-    
+
   // Check to make sure that the other destination of this branch
   // isn't BB itself.  If so, this is an infinite loop that will
   // keep getting unwound.
   if (PBI->getSuccessor(PBIOp) == BB)
     return false;
-    
-  // Do not perform this transformation if it would require 
+
+  // Do not perform this transformation if it would require
   // insertion of a large number of select instructions. For targets
   // without predication/cmovs, this is a big pessimization.
   BasicBlock *CommonDest = PBI->getSuccessor(PBIOp);
-      
+
   unsigned NumPhis = 0;
   for (BasicBlock::iterator II = CommonDest->begin();
        isa<PHINode>(II); ++II, ++NumPhis)
     if (NumPhis > 2) // Disable this xform.
       return false;
-    
+
   // Finally, if everything is ok, fold the branches to logical ops.
   BasicBlock *OtherDest  = BI->getSuccessor(BIOp ^ 1);
-  
+
   DEBUG(dbgs() << "FOLDING BRs:" << *PBI->getParent()
                << "AND: " << *BI->getParent());
-  
-  
+
+
   // If OtherDest *is* BB, then BB is a basic block with a single conditional
   // branch in it, where one edge (OtherDest) goes back to itself but the other
   // exits.  We don't *know* that the program avoids the infinite loop
@@ -2046,13 +2301,13 @@ static bool SimplifyCondBranchToCondBranch(BranchInst *PBI, BranchInst *BI) {
                                                   "infloop", BB->getParent());
     BranchInst::Create(InfLoopBlock, InfLoopBlock);
     OtherDest = InfLoopBlock;
-  }  
-  
+  }
+
   DEBUG(dbgs() << *PBI->getParent()->getParent());
 
   // BI may have other predecessors.  Because of this, we leave
   // it alone, but modify PBI.
-  
+
   // Make sure we get to CommonDest on True&True directions.
   Value *PBICond = PBI->getCondition();
   IRBuilder<true, NoFolder> Builder(PBI);
@@ -2065,16 +2320,43 @@ static bool SimplifyCondBranchToCondBranch(BranchInst *PBI, BranchInst *BI) {
 
   // Merge the conditions.
   Value *Cond = Builder.CreateOr(PBICond, BICond, "brmerge");
-  
+
   // Modify PBI to branch on the new condition to the new dests.
   PBI->setCondition(Cond);
   PBI->setSuccessor(0, CommonDest);
   PBI->setSuccessor(1, OtherDest);
-  
+
+  // Update branch weight for PBI.
+  uint64_t PredTrueWeight, PredFalseWeight, SuccTrueWeight, SuccFalseWeight;
+  bool PredHasWeights = ExtractBranchMetadata(PBI, PredTrueWeight,
+                                              PredFalseWeight);
+  bool SuccHasWeights = ExtractBranchMetadata(BI, SuccTrueWeight,
+                                              SuccFalseWeight);
+  if (PredHasWeights && SuccHasWeights) {
+    uint64_t PredCommon = PBIOp ? PredFalseWeight : PredTrueWeight;
+    uint64_t PredOther = PBIOp ?PredTrueWeight : PredFalseWeight;
+    uint64_t SuccCommon = BIOp ? SuccFalseWeight : SuccTrueWeight;
+    uint64_t SuccOther = BIOp ? SuccTrueWeight : SuccFalseWeight;
+    // The weight to CommonDest should be PredCommon * SuccTotal +
+    //                                    PredOther * SuccCommon.
+    // The weight to OtherDest should be PredOther * SuccOther.
+    SmallVector<uint64_t, 2> NewWeights;
+    NewWeights.push_back(PredCommon * (SuccCommon + SuccOther) +
+                         PredOther * SuccCommon);
+    NewWeights.push_back(PredOther * SuccOther);
+    // Halve the weights if any of them cannot fit in an uint32_t
+    FitWeights(NewWeights);
+
+    SmallVector<uint32_t, 2> MDWeights(NewWeights.begin(),NewWeights.end());
+    PBI->setMetadata(LLVMContext::MD_prof,
+                     MDBuilder(BI->getContext()).
+                     createBranchWeights(MDWeights));
+  }
+
   // OtherDest may have phi nodes.  If so, add an entry from PBI's
   // block that are identical to the entries for BI's block.
   AddPredecessorToBlock(OtherDest, PBI->getParent(), BB);
-  
+
   // We know that the CommonDest already had an edge from PBI to
   // it.  If it has PHIs though, the PHIs may have different
   // entries for BB and PBI's BB.  If so, insert a select to make
@@ -2092,10 +2374,10 @@ static bool SimplifyCondBranchToCondBranch(BranchInst *PBI, BranchInst *BI) {
       PN->setIncomingValue(PBBIdx, NV);
     }
   }
-  
+
   DEBUG(dbgs() << "INTO: " << *PBI->getParent());
   DEBUG(dbgs() << *PBI->getParent()->getParent());
-  
+
   // This basic block is probably dead.  We know it has at least
   // one fewer predecessor.
   return true;
@@ -2107,7 +2389,9 @@ static bool SimplifyCondBranchToCondBranch(BranchInst *PBI, BranchInst *BI) {
 // Also makes sure not to introduce new successors by assuming that edges to
 // non-successor TrueBBs and FalseBBs aren't reachable.
 static bool SimplifyTerminatorOnSelect(TerminatorInst *OldTerm, Value *Cond,
-                                       BasicBlock *TrueBB, BasicBlock *FalseBB){
+                                       BasicBlock *TrueBB, BasicBlock *FalseBB,
+                                       uint32_t TrueWeight,
+                                       uint32_t FalseWeight){
   // Remove any superfluous successor edges from the CFG.
   // First, figure out which successors to preserve.
   // If TrueBB and FalseBB are equal, only try to preserve one copy of that
@@ -2136,10 +2420,15 @@ static bool SimplifyTerminatorOnSelect(TerminatorInst *OldTerm, Value *Cond,
       // We were only looking for one successor, and it was present.
       // Create an unconditional branch to it.
       Builder.CreateBr(TrueBB);
-    else
+    else {
       // We found both of the successors we were looking for.
       // Create a conditional branch sharing the condition of the select.
-      Builder.CreateCondBr(Cond, TrueBB, FalseBB);
+      BranchInst *NewBI = Builder.CreateCondBr(Cond, TrueBB, FalseBB);
+      if (TrueWeight != FalseWeight)
+        NewBI->setMetadata(LLVMContext::MD_prof,
+                           MDBuilder(OldTerm->getContext()).
+                           createBranchWeights(TrueWeight, FalseWeight));
+    }
   } else if (KeepEdge1 && (KeepEdge2 || TrueBB == FalseBB)) {
     // Neither of the selected blocks were successors, so this
     // terminator must be unreachable.
@@ -2176,8 +2465,23 @@ static bool SimplifySwitchOnSelect(SwitchInst *SI, SelectInst *Select) {
   BasicBlock *TrueBB = SI->findCaseValue(TrueVal).getCaseSuccessor();
   BasicBlock *FalseBB = SI->findCaseValue(FalseVal).getCaseSuccessor();
 
+  // Get weight for TrueBB and FalseBB.
+  uint32_t TrueWeight = 0, FalseWeight = 0;
+  SmallVector<uint64_t, 8> Weights;
+  bool HasWeights = HasBranchWeights(SI);
+  if (HasWeights) {
+    GetBranchWeights(SI, Weights);
+    if (Weights.size() == 1 + SI->getNumCases()) {
+      TrueWeight = (uint32_t)Weights[SI->findCaseValue(TrueVal).
+                                     getSuccessorIndex()];
+      FalseWeight = (uint32_t)Weights[SI->findCaseValue(FalseVal).
+                                      getSuccessorIndex()];
+    }
+  }
+
   // Perform the actual simplification.
-  return SimplifyTerminatorOnSelect(SI, Condition, TrueBB, FalseBB);
+  return SimplifyTerminatorOnSelect(SI, Condition, TrueBB, FalseBB,
+                                    TrueWeight, FalseWeight);
 }
 
 // SimplifyIndirectBrOnSelect - Replaces
@@ -2197,7 +2501,8 @@ static bool SimplifyIndirectBrOnSelect(IndirectBrInst *IBI, SelectInst *SI) {
   BasicBlock *FalseBB = FBA->getBasicBlock();
 
   // Perform the actual simplification.
-  return SimplifyTerminatorOnSelect(IBI, SI->getCondition(), TrueBB, FalseBB);
+  return SimplifyTerminatorOnSelect(IBI, SI->getCondition(), TrueBB, FalseBB,
+                                    0, 0);
 }
 
 /// TryToSimplifyUncondBranchWithICmpInIt - This is called when we find an icmp
@@ -2214,11 +2519,11 @@ static bool SimplifyIndirectBrOnSelect(IndirectBrInst *IBI, SelectInst *SI) {
 ///   br label %end
 /// end:
 ///   ... = phi i1 [ true, %entry ], [ %tmp, %DEFAULT ], [ true, %entry ]
-/// 
+///
 /// We prefer to split the edge to 'end' so that there is a true/false entry to
 /// the PHI, merging the third icmp into the switch.
 static bool TryToSimplifyUncondBranchWithICmpInIt(ICmpInst *ICI,
-                                                  const TargetData *TD,
+                                                  const DataLayout *TD,
                                                   IRBuilder<> &Builder) {
   BasicBlock *BB = ICI->getParent();
 
@@ -2228,17 +2533,17 @@ static bool TryToSimplifyUncondBranchWithICmpInIt(ICmpInst *ICI,
 
   Value *V = ICI->getOperand(0);
   ConstantInt *Cst = cast<ConstantInt>(ICI->getOperand(1));
-  
+
   // The pattern we're looking for is where our only predecessor is a switch on
   // 'V' and this block is the default case for the switch.  In this case we can
   // fold the compared value into the switch to simplify things.
   BasicBlock *Pred = BB->getSinglePredecessor();
   if (Pred == 0 || !isa<SwitchInst>(Pred->getTerminator())) return false;
-  
+
   SwitchInst *SI = cast<SwitchInst>(Pred->getTerminator());
   if (SI->getCondition() != V)
     return false;
-  
+
   // If BB is reachable on a non-default case, then we simply know the value of
   // V in this block.  Substitute it and constant fold the icmp instruction
   // away.
@@ -2246,7 +2551,7 @@ static bool TryToSimplifyUncondBranchWithICmpInIt(ICmpInst *ICI,
     ConstantInt *VVal = SI->findCaseDest(BB);
     assert(VVal && "Should have a unique destination value");
     ICI->setOperand(0, VVal);
-    
+
     if (Value *V = SimplifyInstruction(ICI, TD)) {
       ICI->replaceAllUsesWith(V);
       ICI->eraseFromParent();
@@ -2254,7 +2559,7 @@ static bool TryToSimplifyUncondBranchWithICmpInIt(ICmpInst *ICI,
     // BB is now empty, so it is likely to simplify away.
     return SimplifyCFG(BB) | true;
   }
-  
+
   // Ok, the block is reachable from the default dest.  If the constant we're
   // comparing exists in one of the other edges, then we can constant fold ICI
   // and zap it.
@@ -2264,13 +2569,13 @@ static bool TryToSimplifyUncondBranchWithICmpInIt(ICmpInst *ICI,
       V = ConstantInt::getFalse(BB->getContext());
     else
       V = ConstantInt::getTrue(BB->getContext());
-    
+
     ICI->replaceAllUsesWith(V);
     ICI->eraseFromParent();
     // BB is now empty, so it is likely to simplify away.
     return SimplifyCFG(BB) | true;
   }
-  
+
   // The use of the icmp has to be in the 'end' block, by the only PHI node in
   // the block.
   BasicBlock *SuccBlock = BB->getTerminator()->getSuccessor(0);
@@ -2296,8 +2601,23 @@ static bool TryToSimplifyUncondBranchWithICmpInIt(ICmpInst *ICI,
   // the switch to the merge point on the compared value.
   BasicBlock *NewBB = BasicBlock::Create(BB->getContext(), "switch.edge",
                                          BB->getParent(), BB);
+  SmallVector<uint64_t, 8> Weights;
+  bool HasWeights = HasBranchWeights(SI);
+  if (HasWeights) {
+    GetBranchWeights(SI, Weights);
+    if (Weights.size() == 1 + SI->getNumCases()) {
+      // Split weight for default case to case for "Cst".
+      Weights[0] = (Weights[0]+1) >> 1;
+      Weights.push_back(Weights[0]);
+
+      SmallVector<uint32_t, 8> MDWeights(Weights.begin(), Weights.end());
+      SI->setMetadata(LLVMContext::MD_prof,
+                      MDBuilder(SI->getContext()).
+                      createBranchWeights(MDWeights));
+    }
+  }
   SI->addCase(Cst, NewBB);
-  
+
   // NewBB branches to the phi block, add the uncond branch and the phi entry.
   Builder.SetInsertPoint(NewBB);
   Builder.SetCurrentDebugLocation(SI->getDebugLoc());
@@ -2309,12 +2629,12 @@ static bool TryToSimplifyUncondBranchWithICmpInIt(ICmpInst *ICI,
 /// SimplifyBranchOnICmpChain - The specified branch is a conditional branch.
 /// Check to see if it is branching on an or/and chain of icmp instructions, and
 /// fold it into a switch instruction if so.
-static bool SimplifyBranchOnICmpChain(BranchInst *BI, const TargetData *TD,
+static bool SimplifyBranchOnICmpChain(BranchInst *BI, const DataLayout *TD,
                                       IRBuilder<> &Builder) {
   Instruction *Cond = dyn_cast<Instruction>(BI->getCondition());
   if (Cond == 0) return false;
-  
-  
+
+
   // Change br (X == 0 | X == 1), T, F into a switch instruction.
   // If this is a bunch of seteq's or'd together, or if it's a bunch of
   // 'setne's and'ed together, collect them.
@@ -2323,7 +2643,7 @@ static bool SimplifyBranchOnICmpChain(BranchInst *BI, const TargetData *TD,
   bool TrueWhenEqual = true;
   Value *ExtraCase = 0;
   unsigned UsedICmps = 0;
-  
+
   if (Cond->getOpcode() == Instruction::Or) {
     CompVal = GatherConstantCompares(Cond, Values, ExtraCase, TD, true,
                                      UsedICmps);
@@ -2332,7 +2652,7 @@ static bool SimplifyBranchOnICmpChain(BranchInst *BI, const TargetData *TD,
                                      UsedICmps);
     TrueWhenEqual = false;
   }
-  
+
   // If we didn't have a multiply compared value, fail.
   if (CompVal == 0) return false;
 
@@ -2344,21 +2664,24 @@ static bool SimplifyBranchOnICmpChain(BranchInst *BI, const TargetData *TD,
   // instruction can't handle, remove them now.
   array_pod_sort(Values.begin(), Values.end(), ConstantIntSortPredicate);
   Values.erase(std::unique(Values.begin(), Values.end()), Values.end());
-  
+
   // If Extra was used, we require at least two switch values to do the
   // transformation.  A switch with one value is just an cond branch.
   if (ExtraCase && Values.size() < 2) return false;
-  
+
+  // TODO: Preserve branch weight metadata, similarly to how
+  // FoldValueComparisonIntoPredecessors preserves it.
+
   // Figure out which block is which destination.
   BasicBlock *DefaultBB = BI->getSuccessor(1);
   BasicBlock *EdgeBB    = BI->getSuccessor(0);
   if (!TrueWhenEqual) std::swap(DefaultBB, EdgeBB);
-  
+
   BasicBlock *BB = BI->getParent();
-  
+
   DEBUG(dbgs() << "Converting 'icmp' chain with " << Values.size()
                << " cases into SWITCH.  BB is:\n" << *BB);
-  
+
   // If there are any extra values that couldn't be folded into the switch
   // then we evaluate them with an explicit branch first.  Split the block
   // right before the condbr to handle it.
@@ -2372,13 +2695,13 @@ static bool SimplifyBranchOnICmpChain(BranchInst *BI, const TargetData *TD,
       Builder.CreateCondBr(ExtraCase, EdgeBB, NewBB);
     else
       Builder.CreateCondBr(ExtraCase, NewBB, EdgeBB);
-      
+
     OldTI->eraseFromParent();
-    
+
     // If there are PHI nodes in EdgeBB, then we need to add a new entry to them
     // for the edge we just added.
     AddPredecessorToBlock(EdgeBB, BB, NewBB);
-    
+
     DEBUG(dbgs() << "  ** 'icmp' chain unhandled condition: " << *ExtraCase
           << "\nEXTRABB = " << *BB);
     BB = NewBB;
@@ -2387,19 +2710,19 @@ static bool SimplifyBranchOnICmpChain(BranchInst *BI, const TargetData *TD,
   Builder.SetInsertPoint(BI);
   // Convert pointer to int before we switch.
   if (CompVal->getType()->isPointerTy()) {
-    assert(TD && "Cannot switch on pointer without TargetData");
+    assert(TD && "Cannot switch on pointer without DataLayout");
     CompVal = Builder.CreatePtrToInt(CompVal,
                                      TD->getIntPtrType(CompVal->getContext()),
                                      "magicptr");
   }
-  
+
   // Create the new switch instruction now.
   SwitchInst *New = Builder.CreateSwitch(CompVal, DefaultBB, Values.size());
 
   // Add all of the 'cases' to the switch instruction.
   for (unsigned i = 0, e = Values.size(); i != e; ++i)
     New->addCase(Values[i], EdgeBB);
-  
+
   // We added edges from PI to the EdgeBB.  As such, if there were any
   // PHI nodes in EdgeBB, they need entries to be added corresponding to
   // the number of edges added.
@@ -2410,10 +2733,10 @@ static bool SimplifyBranchOnICmpChain(BranchInst *BI, const TargetData *TD,
     for (unsigned i = 0, e = Values.size()-1; i != e; ++i)
       PN->addIncoming(InVal, BB);
   }
-  
+
   // Erase the old branch instruction.
   EraseTerminatorInstAndDCECond(BI);
-  
+
   DEBUG(dbgs() << "  ** 'icmp' chain result is:\n" << *BB << '\n');
   return true;
 }
@@ -2467,7 +2790,7 @@ bool SimplifyCFGOpt::SimplifyResume(ResumeInst *RI, IRBuilder<> &Builder) {
 bool SimplifyCFGOpt::SimplifyReturn(ReturnInst *RI, IRBuilder<> &Builder) {
   BasicBlock *BB = RI->getParent();
   if (!BB->getFirstNonPHIOrDbg()->isTerminator()) return false;
-  
+
   // Find predecessors that end with branches.
   SmallVector<BasicBlock*, 8> UncondBranchPreds;
   SmallVector<BranchInst*, 8> CondBranchPreds;
@@ -2481,7 +2804,7 @@ bool SimplifyCFGOpt::SimplifyReturn(ReturnInst *RI, IRBuilder<> &Builder) {
         CondBranchPreds.push_back(BI);
     }
   }
-  
+
   // If we found some, do the transformation!
   if (!UncondBranchPreds.empty() && DupRet) {
     while (!UncondBranchPreds.empty()) {
@@ -2490,21 +2813,21 @@ bool SimplifyCFGOpt::SimplifyReturn(ReturnInst *RI, IRBuilder<> &Builder) {
             << "INTO UNCOND BRANCH PRED: " << *Pred);
       (void)FoldReturnIntoUncondBranch(RI, BB, Pred);
     }
-    
+
     // If we eliminated all predecessors of the block, delete the block now.
     if (pred_begin(BB) == pred_end(BB))
       // We know there are no successors, so just nuke the block.
       BB->eraseFromParent();
-    
+
     return true;
   }
-  
+
   // Check out all of the conditional branches going to this return
   // instruction.  If any of them just select between returns, change the
   // branch itself into a select/return pair.
   while (!CondBranchPreds.empty()) {
     BranchInst *BI = CondBranchPreds.pop_back_val();
-    
+
     // Check to see if the non-BB successor is also a return block.
     if (isa<ReturnInst>(BI->getSuccessor(0)->getTerminator()) &&
         isa<ReturnInst>(BI->getSuccessor(1)->getTerminator()) &&
@@ -2516,9 +2839,9 @@ bool SimplifyCFGOpt::SimplifyReturn(ReturnInst *RI, IRBuilder<> &Builder) {
 
 bool SimplifyCFGOpt::SimplifyUnreachable(UnreachableInst *UI) {
   BasicBlock *BB = UI->getParent();
-  
+
   bool Changed = false;
-  
+
   // If there are any instructions immediately before the unreachable that can
   // be removed, do so.
   while (UI != BB->begin()) {
@@ -2558,11 +2881,11 @@ bool SimplifyCFGOpt::SimplifyUnreachable(UnreachableInst *UI) {
     BBI->eraseFromParent();
     Changed = true;
   }
-  
+
   // If the unreachable instruction is the first in the block, take a gander
   // at all of the predecessors of this instruction, and simplify them.
   if (&BB->front() != UI) return Changed;
-  
+
   SmallVector<BasicBlock*, 8> Preds(pred_begin(BB), pred_end(BB));
   for (unsigned i = 0, e = Preds.size(); i != e; ++i) {
     TerminatorInst *TI = Preds[i]->getTerminator();
@@ -2615,7 +2938,7 @@ bool SimplifyCFGOpt::SimplifyUnreachable(UnreachableInst *UI) {
         BasicBlock *MaxBlock = 0;
         for (std::map<BasicBlock*, std::pair<unsigned, unsigned> >::iterator
              I = Popularity.begin(), E = Popularity.end(); I != E; ++I) {
-          if (I->second.first > MaxPop || 
+          if (I->second.first > MaxPop ||
               (I->second.first == MaxPop && MaxIndex > I->second.second)) {
             MaxPop = I->second.first;
             MaxIndex = I->second.second;
@@ -2627,13 +2950,13 @@ bool SimplifyCFGOpt::SimplifyUnreachable(UnreachableInst *UI) {
           // edges to it.
           SI->setDefaultDest(MaxBlock);
           Changed = true;
-          
+
           // If MaxBlock has phinodes in it, remove MaxPop-1 entries from
           // it.
           if (isa<PHINode>(MaxBlock->begin()))
             for (unsigned i = 0; i != MaxPop-1; ++i)
               MaxBlock->removePredecessor(SI->getParent());
-          
+
           for (SwitchInst::CaseIt i = SI->case_begin(), e = SI->case_end();
                i != e; ++i)
             if (i.getCaseSuccessor() == MaxBlock) {
@@ -2648,7 +2971,7 @@ bool SimplifyCFGOpt::SimplifyUnreachable(UnreachableInst *UI) {
         // place to note that the call does not throw though.
         BranchInst *BI = Builder.CreateBr(II->getNormalDest());
         II->removeFromParent();   // Take out of symbol table
-        
+
         // Insert the call now...
         SmallVector<Value*, 8> Args(II->op_begin(), II->op_end()-3);
         Builder.SetInsertPoint(BI);
@@ -2663,7 +2986,7 @@ bool SimplifyCFGOpt::SimplifyUnreachable(UnreachableInst *UI) {
       }
     }
   }
-  
+
   // If this block is now dead, remove it.
   if (pred_begin(BB) == pred_end(BB) &&
       BB != &BB->getParent()->getEntryBlock()) {
@@ -2706,9 +3029,28 @@ static bool TurnSwitchRangeIntoICmp(SwitchInst *SI, IRBuilder<> &Builder) {
   if (!Offset->isNullValue())
     Sub = Builder.CreateAdd(Sub, Offset, Sub->getName()+".off");
   Value *Cmp = Builder.CreateICmpULT(Sub, NumCases, "switch");
-  Builder.CreateCondBr(
+  BranchInst *NewBI = Builder.CreateCondBr(
       Cmp, SI->case_begin().getCaseSuccessor(), SI->getDefaultDest());
 
+  // Update weight for the newly-created conditional branch.
+  SmallVector<uint64_t, 8> Weights;
+  bool HasWeights = HasBranchWeights(SI);
+  if (HasWeights) {
+    GetBranchWeights(SI, Weights);
+    if (Weights.size() == 1 + SI->getNumCases()) {
+      // Combine all weights for the cases to be the true weight of NewBI.
+      // We assume that the sum of all weights for a Terminator can fit into 32
+      // bits.
+      uint32_t NewTrueWeight = 0;
+      for (unsigned I = 1, E = Weights.size(); I != E; ++I)
+        NewTrueWeight += (uint32_t)Weights[I];
+      NewBI->setMetadata(LLVMContext::MD_prof,
+                         MDBuilder(SI->getContext()).
+                         createBranchWeights(NewTrueWeight,
+                                             (uint32_t)Weights[0]));
+    }
+  }
+
   // Prune obsolete incoming values off the successor's PHI nodes.
   for (BasicBlock::iterator BBI = SI->case_begin().getCaseSuccessor()->begin();
        isa<PHINode>(BBI); ++BBI) {
@@ -2739,15 +3081,33 @@ static bool EliminateDeadSwitchCases(SwitchInst *SI) {
     }
   }
 
+  SmallVector<uint64_t, 8> Weights;
+  bool HasWeight = HasBranchWeights(SI);
+  if (HasWeight) {
+    GetBranchWeights(SI, Weights);
+    HasWeight = (Weights.size() == 1 + SI->getNumCases());
+  }
+
   // Remove dead cases from the switch.
   for (unsigned I = 0, E = DeadCases.size(); I != E; ++I) {
     SwitchInst::CaseIt Case = SI->findCaseValue(DeadCases[I]);
     assert(Case != SI->case_default() &&
            "Case was not found. Probably mistake in DeadCases forming.");
+    if (HasWeight) {
+      std::swap(Weights[Case.getCaseIndex()+1], Weights.back());
+      Weights.pop_back();
+    }
+
     // Prune unused values from PHI nodes.
     Case.getCaseSuccessor()->removePredecessor(SI->getParent());
     SI->removeCase(Case);
   }
+  if (HasWeight) {
+    SmallVector<uint32_t, 8> MDWeights(Weights.begin(), Weights.end());
+    SI->setMetadata(LLVMContext::MD_prof,
+                    MDBuilder(SI->getParent()->getContext()).
+                    createBranchWeights(MDWeights));
+  }
 
   return !DeadCases.empty();
 }
@@ -2823,33 +3183,512 @@ static bool ForwardSwitchConditionToPHI(SwitchInst *SI) {
   return Changed;
 }
 
-bool SimplifyCFGOpt::SimplifySwitch(SwitchInst *SI, IRBuilder<> &Builder) {
-  // If this switch is too complex to want to look at, ignore it.
-  if (!isValueEqualityComparison(SI))
+/// ValidLookupTableConstant - Return true if the backend will be able to handle
+/// initializing an array of constants like C.
+static bool ValidLookupTableConstant(Constant *C) {
+  if (ConstantExpr *CE = dyn_cast<ConstantExpr>(C))
+    return CE->isGEPWithNoNotionalOverIndexing();
+
+  return isa<ConstantFP>(C) ||
+      isa<ConstantInt>(C) ||
+      isa<ConstantPointerNull>(C) ||
+      isa<GlobalValue>(C) ||
+      isa<UndefValue>(C);
+}
+
+/// LookupConstant - If V is a Constant, return it. Otherwise, try to look up
+/// its constant value in ConstantPool, returning 0 if it's not there.
+static Constant *LookupConstant(Value *V,
+                         const SmallDenseMap<Value*, Constant*>& ConstantPool) {
+  if (Constant *C = dyn_cast<Constant>(V))
+    return C;
+  return ConstantPool.lookup(V);
+}
+
+/// ConstantFold - Try to fold instruction I into a constant. This works for
+/// simple instructions such as binary operations where both operands are
+/// constant or can be replaced by constants from the ConstantPool. Returns the
+/// resulting constant on success, 0 otherwise.
+static Constant *ConstantFold(Instruction *I,
+                         const SmallDenseMap<Value*, Constant*>& ConstantPool) {
+  if (BinaryOperator *BO = dyn_cast<BinaryOperator>(I)) {
+    Constant *A = LookupConstant(BO->getOperand(0), ConstantPool);
+    if (!A)
+      return 0;
+    Constant *B = LookupConstant(BO->getOperand(1), ConstantPool);
+    if (!B)
+      return 0;
+    return ConstantExpr::get(BO->getOpcode(), A, B);
+  }
+
+  if (CmpInst *Cmp = dyn_cast<CmpInst>(I)) {
+    Constant *A = LookupConstant(I->getOperand(0), ConstantPool);
+    if (!A)
+      return 0;
+    Constant *B = LookupConstant(I->getOperand(1), ConstantPool);
+    if (!B)
+      return 0;
+    return ConstantExpr::getCompare(Cmp->getPredicate(), A, B);
+  }
+
+  if (SelectInst *Select = dyn_cast<SelectInst>(I)) {
+    Constant *A = LookupConstant(Select->getCondition(), ConstantPool);
+    if (!A)
+      return 0;
+    if (A->isAllOnesValue())
+      return LookupConstant(Select->getTrueValue(), ConstantPool);
+    if (A->isNullValue())
+      return LookupConstant(Select->getFalseValue(), ConstantPool);
+    return 0;
+  }
+
+  if (CastInst *Cast = dyn_cast<CastInst>(I)) {
+    Constant *A = LookupConstant(I->getOperand(0), ConstantPool);
+    if (!A)
+      return 0;
+    return ConstantExpr::getCast(Cast->getOpcode(), A, Cast->getDestTy());
+  }
+
+  return 0;
+}
+
+/// GetCaseResults - Try to determine the resulting constant values in phi nodes
+/// at the common destination basic block, *CommonDest, for one of the case
+/// destionations CaseDest corresponding to value CaseVal (0 for the default
+/// case), of a switch instruction SI.
+static bool GetCaseResults(SwitchInst *SI,
+                           ConstantInt *CaseVal,
+                           BasicBlock *CaseDest,
+                           BasicBlock **CommonDest,
+                           SmallVector<std::pair<PHINode*,Constant*>, 4> &Res) {
+  // The block from which we enter the common destination.
+  BasicBlock *Pred = SI->getParent();
+
+  // If CaseDest is empty except for some side-effect free instructions through
+  // which we can constant-propagate the CaseVal, continue to its successor.
+  SmallDenseMap<Value*, Constant*> ConstantPool;
+  ConstantPool.insert(std::make_pair(SI->getCondition(), CaseVal));
+  for (BasicBlock::iterator I = CaseDest->begin(), E = CaseDest->end(); I != E;
+       ++I) {
+    if (TerminatorInst *T = dyn_cast<TerminatorInst>(I)) {
+      // If the terminator is a simple branch, continue to the next block.
+      if (T->getNumSuccessors() != 1)
+        return false;
+      Pred = CaseDest;
+      CaseDest = T->getSuccessor(0);
+    } else if (isa<DbgInfoIntrinsic>(I)) {
+      // Skip debug intrinsic.
+      continue;
+    } else if (Constant *C = ConstantFold(I, ConstantPool)) {
+      // Instruction is side-effect free and constant.
+      ConstantPool.insert(std::make_pair(I, C));
+    } else {
+      break;
+    }
+  }
+
+  // If we did not have a CommonDest before, use the current one.
+  if (!*CommonDest)
+    *CommonDest = CaseDest;
+  // If the destination isn't the common one, abort.
+  if (CaseDest != *CommonDest)
+    return false;
+
+  // Get the values for this case from phi nodes in the destination block.
+  BasicBlock::iterator I = (*CommonDest)->begin();
+  while (PHINode *PHI = dyn_cast<PHINode>(I++)) {
+    int Idx = PHI->getBasicBlockIndex(Pred);
+    if (Idx == -1)
+      continue;
+
+    Constant *ConstVal = LookupConstant(PHI->getIncomingValue(Idx),
+                                        ConstantPool);
+    if (!ConstVal)
+      return false;
+
+    // Note: If the constant comes from constant-propagating the case value
+    // through the CaseDest basic block, it will be safe to remove the
+    // instructions in that block. They cannot be used (except in the phi nodes
+    // we visit) outside CaseDest, because that block does not dominate its
+    // successor. If it did, we would not be in this phi node.
+
+    // Be conservative about which kinds of constants we support.
+    if (!ValidLookupTableConstant(ConstVal))
+      return false;
+
+    Res.push_back(std::make_pair(PHI, ConstVal));
+  }
+
+  return true;
+}
+
+namespace {
+  /// SwitchLookupTable - This class represents a lookup table that can be used
+  /// to replace a switch.
+  class SwitchLookupTable {
+  public:
+    /// SwitchLookupTable - Create a lookup table to use as a switch replacement
+    /// with the contents of Values, using DefaultValue to fill any holes in the
+    /// table.
+    SwitchLookupTable(Module &M,
+                      uint64_t TableSize,
+                      ConstantInt *Offset,
+               const SmallVector<std::pair<ConstantInt*, Constant*>, 4>& Values,
+                      Constant *DefaultValue,
+                      const DataLayout *TD);
+
+    /// BuildLookup - Build instructions with Builder to retrieve the value at
+    /// the position given by Index in the lookup table.
+    Value *BuildLookup(Value *Index, IRBuilder<> &Builder);
+
+    /// WouldFitInRegister - Return true if a table with TableSize elements of
+    /// type ElementType would fit in a target-legal register.
+    static bool WouldFitInRegister(const DataLayout *TD,
+                                   uint64_t TableSize,
+                                   const Type *ElementType);
+
+  private:
+    // Depending on the contents of the table, it can be represented in
+    // different ways.
+    enum {
+      // For tables where each element contains the same value, we just have to
+      // store that single value and return it for each lookup.
+      SingleValueKind,
+
+      // For small tables with integer elements, we can pack them into a bitmap
+      // that fits into a target-legal register. Values are retrieved by
+      // shift and mask operations.
+      BitMapKind,
+
+      // The table is stored as an array of values. Values are retrieved by load
+      // instructions from the table.
+      ArrayKind
+    } Kind;
+
+    // For SingleValueKind, this is the single value.
+    Constant *SingleValue;
+
+    // For BitMapKind, this is the bitmap.
+    ConstantInt *BitMap;
+    IntegerType *BitMapElementTy;
+
+    // For ArrayKind, this is the array.
+    GlobalVariable *Array;
+  };
+}
+
+SwitchLookupTable::SwitchLookupTable(Module &M,
+                                     uint64_t TableSize,
+                                     ConstantInt *Offset,
+               const SmallVector<std::pair<ConstantInt*, Constant*>, 4>& Values,
+                                     Constant *DefaultValue,
+                                     const DataLayout *TD) {
+  assert(Values.size() && "Can't build lookup table without values!");
+  assert(TableSize >= Values.size() && "Can't fit values in table!");
+
+  // If all values in the table are equal, this is that value.
+  SingleValue = Values.begin()->second;
+
+  // Build up the table contents.
+  SmallVector<Constant*, 64> TableContents(TableSize);
+  for (size_t I = 0, E = Values.size(); I != E; ++I) {
+    ConstantInt *CaseVal = Values[I].first;
+    Constant *CaseRes = Values[I].second;
+    assert(CaseRes->getType() == DefaultValue->getType());
+
+    uint64_t Idx = (CaseVal->getValue() - Offset->getValue())
+                   .getLimitedValue();
+    TableContents[Idx] = CaseRes;
+
+    if (CaseRes != SingleValue)
+      SingleValue = 0;
+  }
+
+  // Fill in any holes in the table with the default result.
+  if (Values.size() < TableSize) {
+    for (uint64_t I = 0; I < TableSize; ++I) {
+      if (!TableContents[I])
+        TableContents[I] = DefaultValue;
+    }
+
+    if (DefaultValue != SingleValue)
+      SingleValue = 0;
+  }
+
+  // If each element in the table contains the same value, we only need to store
+  // that single value.
+  if (SingleValue) {
+    Kind = SingleValueKind;
+    return;
+  }
+
+  // If the type is integer and the table fits in a register, build a bitmap.
+  if (WouldFitInRegister(TD, TableSize, DefaultValue->getType())) {
+    IntegerType *IT = cast<IntegerType>(DefaultValue->getType());
+    APInt TableInt(TableSize * IT->getBitWidth(), 0);
+    for (uint64_t I = TableSize; I > 0; --I) {
+      TableInt <<= IT->getBitWidth();
+      // Insert values into the bitmap. Undef values are set to zero.
+      if (!isa<UndefValue>(TableContents[I - 1])) {
+        ConstantInt *Val = cast<ConstantInt>(TableContents[I - 1]);
+        TableInt |= Val->getValue().zext(TableInt.getBitWidth());
+      }
+    }
+    BitMap = ConstantInt::get(M.getContext(), TableInt);
+    BitMapElementTy = IT;
+    Kind = BitMapKind;
+    ++NumBitMaps;
+    return;
+  }
+
+  // Store the table in an array.
+  ArrayType *ArrayTy = ArrayType::get(DefaultValue->getType(), TableSize);
+  Constant *Initializer = ConstantArray::get(ArrayTy, TableContents);
+
+  Array = new GlobalVariable(M, ArrayTy, /*constant=*/ true,
+                             GlobalVariable::PrivateLinkage,
+                             Initializer,
+                             "switch.table");
+  Array->setUnnamedAddr(true);
+  Kind = ArrayKind;
+}
+
+Value *SwitchLookupTable::BuildLookup(Value *Index, IRBuilder<> &Builder) {
+  switch (Kind) {
+    case SingleValueKind:
+      return SingleValue;
+    case BitMapKind: {
+      // Type of the bitmap (e.g. i59).
+      IntegerType *MapTy = BitMap->getType();
+
+      // Cast Index to the same type as the bitmap.
+      // Note: The Index is <= the number of elements in the table, so
+      // truncating it to the width of the bitmask is safe.
+      Value *ShiftAmt = Builder.CreateZExtOrTrunc(Index, MapTy, "switch.cast");
+
+      // Multiply the shift amount by the element width.
+      ShiftAmt = Builder.CreateMul(ShiftAmt,
+                      ConstantInt::get(MapTy, BitMapElementTy->getBitWidth()),
+                                   "switch.shiftamt");
+
+      // Shift down.
+      Value *DownShifted = Builder.CreateLShr(BitMap, ShiftAmt,
+                                              "switch.downshift");
+      // Mask off.
+      return Builder.CreateTrunc(DownShifted, BitMapElementTy,
+                                 "switch.masked");
+    }
+    case ArrayKind: {
+      Value *GEPIndices[] = { Builder.getInt32(0), Index };
+      Value *GEP = Builder.CreateInBoundsGEP(Array, GEPIndices,
+                                             "switch.gep");
+      return Builder.CreateLoad(GEP, "switch.load");
+    }
+  }
+  llvm_unreachable("Unknown lookup table kind!");
+}
+
+bool SwitchLookupTable::WouldFitInRegister(const DataLayout *TD,
+                                           uint64_t TableSize,
+                                           const Type *ElementType) {
+  if (!TD)
+    return false;
+  const IntegerType *IT = dyn_cast<IntegerType>(ElementType);
+  if (!IT)
+    return false;
+  // FIXME: If the type is wider than it needs to be, e.g. i8 but all values
+  // are <= 15, we could try to narrow the type.
+
+  // Avoid overflow, fitsInLegalInteger uses unsigned int for the width.
+  if (TableSize >= UINT_MAX/IT->getBitWidth())
+    return false;
+  return TD->fitsInLegalInteger(TableSize * IT->getBitWidth());
+}
+
+/// ShouldBuildLookupTable - Determine whether a lookup table should be built
+/// for this switch, based on the number of caes, size of the table and the
+/// types of the results.
+static bool ShouldBuildLookupTable(SwitchInst *SI,
+                                   uint64_t TableSize,
+                                   const DataLayout *TD,
+                            const SmallDenseMap<PHINode*, Type*>& ResultTypes) {
+  // The table density should be at least 40%. This is the same criterion as for
+  // jump tables, see SelectionDAGBuilder::handleJTSwitchCase.
+  // FIXME: Find the best cut-off.
+  if (SI->getNumCases() > TableSize || TableSize >= UINT64_MAX / 10)
+    return false; // TableSize overflowed, or mul below might overflow.
+  if (SI->getNumCases() * 10 >= TableSize * 4)
+    return true;
+
+  // If each table would fit in a register, we should build it anyway.
+  for (SmallDenseMap<PHINode*, Type*>::const_iterator I = ResultTypes.begin(),
+       E = ResultTypes.end(); I != E; ++I) {
+    if (!SwitchLookupTable::WouldFitInRegister(TD, TableSize, I->second))
+      return false;
+  }
+  return true;
+}
+
+/// SwitchToLookupTable - If the switch is only used to initialize one or more
+/// phi nodes in a common successor block with different constant values,
+/// replace the switch with lookup tables.
+static bool SwitchToLookupTable(SwitchInst *SI,
+                                IRBuilder<> &Builder,
+                                const DataLayout* TD,
+                                const TargetTransformInfo *TTI) {
+  assert(SI->getNumCases() > 1 && "Degenerate switch?");
+
+  // Only build lookup table when we have a target that supports it.
+  if (!TTI || !TTI->getScalarTargetTransformInfo() ||
+      !TTI->getScalarTargetTransformInfo()->shouldBuildLookupTables())
     return false;
 
+  // FIXME: If the switch is too sparse for a lookup table, perhaps we could
+  // split off a dense part and build a lookup table for that.
+
+  // FIXME: This creates arrays of GEPs to constant strings, which means each
+  // GEP needs a runtime relocation in PIC code. We should just build one big
+  // string and lookup indices into that.
+
+  // Ignore the switch if the number of cases is too small.
+  // This is similar to the check when building jump tables in
+  // SelectionDAGBuilder::handleJTSwitchCase.
+  // FIXME: Determine the best cut-off.
+  if (SI->getNumCases() < 4)
+    return false;
+
+  // Figure out the corresponding result for each case value and phi node in the
+  // common destination, as well as the the min and max case values.
+  assert(SI->case_begin() != SI->case_end());
+  SwitchInst::CaseIt CI = SI->case_begin();
+  ConstantInt *MinCaseVal = CI.getCaseValue();
+  ConstantInt *MaxCaseVal = CI.getCaseValue();
+
+  BasicBlock *CommonDest = 0;
+  typedef SmallVector<std::pair<ConstantInt*, Constant*>, 4> ResultListTy;
+  SmallDenseMap<PHINode*, ResultListTy> ResultLists;
+  SmallDenseMap<PHINode*, Constant*> DefaultResults;
+  SmallDenseMap<PHINode*, Type*> ResultTypes;
+  SmallVector<PHINode*, 4> PHIs;
+
+  for (SwitchInst::CaseIt E = SI->case_end(); CI != E; ++CI) {
+    ConstantInt *CaseVal = CI.getCaseValue();
+    if (CaseVal->getValue().slt(MinCaseVal->getValue()))
+      MinCaseVal = CaseVal;
+    if (CaseVal->getValue().sgt(MaxCaseVal->getValue()))
+      MaxCaseVal = CaseVal;
+
+    // Resulting value at phi nodes for this case value.
+    typedef SmallVector<std::pair<PHINode*, Constant*>, 4> ResultsTy;
+    ResultsTy Results;
+    if (!GetCaseResults(SI, CaseVal, CI.getCaseSuccessor(), &CommonDest,
+                        Results))
+      return false;
+
+    // Append the result from this case to the list for each phi.
+    for (ResultsTy::iterator I = Results.begin(), E = Results.end(); I!=E; ++I) {
+      if (!ResultLists.count(I->first))
+        PHIs.push_back(I->first);
+      ResultLists[I->first].push_back(std::make_pair(CaseVal, I->second));
+    }
+  }
+
+  // Get the resulting values for the default case.
+  SmallVector<std::pair<PHINode*, Constant*>, 4> DefaultResultsList;
+  if (!GetCaseResults(SI, 0, SI->getDefaultDest(), &CommonDest,
+                      DefaultResultsList))
+    return false;
+  for (size_t I = 0, E = DefaultResultsList.size(); I != E; ++I) {
+    PHINode *PHI = DefaultResultsList[I].first;
+    Constant *Result = DefaultResultsList[I].second;
+    DefaultResults[PHI] = Result;
+    ResultTypes[PHI] = Result->getType();
+  }
+
+  APInt RangeSpread = MaxCaseVal->getValue() - MinCaseVal->getValue();
+  uint64_t TableSize = RangeSpread.getLimitedValue() + 1;
+  if (!ShouldBuildLookupTable(SI, TableSize, TD, ResultTypes))
+    return false;
+
+  // Create the BB that does the lookups.
+  Module &Mod = *CommonDest->getParent()->getParent();
+  BasicBlock *LookupBB = BasicBlock::Create(Mod.getContext(),
+                                            "switch.lookup",
+                                            CommonDest->getParent(),
+                                            CommonDest);
+
+  // Check whether the condition value is within the case range, and branch to
+  // the new BB.
+  Builder.SetInsertPoint(SI);
+  Value *TableIndex = Builder.CreateSub(SI->getCondition(), MinCaseVal,
+                                        "switch.tableidx");
+  Value *Cmp = Builder.CreateICmpULT(TableIndex, ConstantInt::get(
+      MinCaseVal->getType(), TableSize));
+  Builder.CreateCondBr(Cmp, LookupBB, SI->getDefaultDest());
+
+  // Populate the BB that does the lookups.
+  Builder.SetInsertPoint(LookupBB);
+  bool ReturnedEarly = false;
+  for (size_t I = 0, E = PHIs.size(); I != E; ++I) {
+    PHINode *PHI = PHIs[I];
+
+    SwitchLookupTable Table(Mod, TableSize, MinCaseVal, ResultLists[PHI],
+                            DefaultResults[PHI], TD);
+
+    Value *Result = Table.BuildLookup(TableIndex, Builder);
+
+    // If the result is used to return immediately from the function, we want to
+    // do that right here.
+    if (PHI->hasOneUse() && isa<ReturnInst>(*PHI->use_begin()) &&
+        *PHI->use_begin() == CommonDest->getFirstNonPHIOrDbg()) {
+      Builder.CreateRet(Result);
+      ReturnedEarly = true;
+      break;
+    }
+
+    PHI->addIncoming(Result, LookupBB);
+  }
+
+  if (!ReturnedEarly)
+    Builder.CreateBr(CommonDest);
+
+  // Remove the switch.
+  for (unsigned i = 0; i < SI->getNumSuccessors(); ++i) {
+    BasicBlock *Succ = SI->getSuccessor(i);
+    if (Succ == SI->getDefaultDest()) continue;
+    Succ->removePredecessor(SI->getParent());
+  }
+  SI->eraseFromParent();
+
+  ++NumLookupTables;
+  return true;
+}
+
+bool SimplifyCFGOpt::SimplifySwitch(SwitchInst *SI, IRBuilder<> &Builder) {
   BasicBlock *BB = SI->getParent();
 
-  // If we only have one predecessor, and if it is a branch on this value,
-  // see if that predecessor totally determines the outcome of this switch.
-  if (BasicBlock *OnlyPred = BB->getSinglePredecessor())
-    if (SimplifyEqualityComparisonWithOnlyPredecessor(SI, OnlyPred, Builder))
-      return SimplifyCFG(BB) | true;
+  if (isValueEqualityComparison(SI)) {
+    // If we only have one predecessor, and if it is a branch on this value,
+    // see if that predecessor totally determines the outcome of this switch.
+    if (BasicBlock *OnlyPred = BB->getSinglePredecessor())
+      if (SimplifyEqualityComparisonWithOnlyPredecessor(SI, OnlyPred, Builder))
+        return SimplifyCFG(BB) | true;
 
-  Value *Cond = SI->getCondition();
-  if (SelectInst *Select = dyn_cast<SelectInst>(Cond))
-    if (SimplifySwitchOnSelect(SI, Select))
-      return SimplifyCFG(BB) | true;
+    Value *Cond = SI->getCondition();
+    if (SelectInst *Select = dyn_cast<SelectInst>(Cond))
+      if (SimplifySwitchOnSelect(SI, Select))
+        return SimplifyCFG(BB) | true;
 
-  // If the block only contains the switch, see if we can fold the block
-  // away into any preds.
-  BasicBlock::iterator BBI = BB->begin();
-  // Ignore dbg intrinsics.
-  while (isa<DbgInfoIntrinsic>(BBI))
-    ++BBI;
-  if (SI == &*BBI)
-    if (FoldValueComparisonIntoPredecessors(SI, Builder))
-      return SimplifyCFG(BB) | true;
+    // If the block only contains the switch, see if we can fold the block
+    // away into any preds.
+    BasicBlock::iterator BBI = BB->begin();
+    // Ignore dbg intrinsics.
+    while (isa<DbgInfoIntrinsic>(BBI))
+      ++BBI;
+    if (SI == &*BBI)
+      if (FoldValueComparisonIntoPredecessors(SI, Builder))
+        return SimplifyCFG(BB) | true;
+  }
 
   // Try to transform the switch into an icmp and a branch.
   if (TurnSwitchRangeIntoICmp(SI, Builder))
@@ -2862,13 +3701,16 @@ bool SimplifyCFGOpt::SimplifySwitch(SwitchInst *SI, IRBuilder<> &Builder) {
   if (ForwardSwitchConditionToPHI(SI))
     return SimplifyCFG(BB) | true;
 
+  if (SwitchToLookupTable(SI, Builder, TD, TTI))
+    return SimplifyCFG(BB) | true;
+
   return false;
 }
 
 bool SimplifyCFGOpt::SimplifyIndirectBr(IndirectBrInst *IBI) {
   BasicBlock *BB = IBI->getParent();
   bool Changed = false;
-  
+
   // Eliminate redundant destinations.
   SmallPtrSet<Value *, 8> Succs;
   for (unsigned i = 0, e = IBI->getNumDestinations(); i != e; ++i) {
@@ -2879,7 +3721,7 @@ bool SimplifyCFGOpt::SimplifyIndirectBr(IndirectBrInst *IBI) {
       --i; --e;
       Changed = true;
     }
-  } 
+  }
 
   if (IBI->getNumDestinations() == 0) {
     // If the indirectbr has no successors, change it to unreachable.
@@ -2887,14 +3729,14 @@ bool SimplifyCFGOpt::SimplifyIndirectBr(IndirectBrInst *IBI) {
     EraseTerminatorInstAndDCECond(IBI);
     return true;
   }
-  
+
   if (IBI->getNumDestinations() == 1) {
     // If the indirectbr has one successor, change it to a direct branch.
     BranchInst::Create(IBI->getDestination(0), IBI);
     EraseTerminatorInstAndDCECond(IBI);
     return true;
   }
-  
+
   if (SelectInst *SI = dyn_cast<SelectInst>(IBI->getAddress())) {
     if (SimplifyIndirectBrOnSelect(IBI, SI))
       return SimplifyCFG(BB) | true;
@@ -2904,13 +3746,16 @@ bool SimplifyCFGOpt::SimplifyIndirectBr(IndirectBrInst *IBI) {
 
 bool SimplifyCFGOpt::SimplifyUncondBranch(BranchInst *BI, IRBuilder<> &Builder){
   BasicBlock *BB = BI->getParent();
-  
+
+  if (SinkCommon && SinkThenElseCodeToEnd(BI))
+    return true;
+
   // If the Terminator is the only non-phi instruction, simplify the block.
   BasicBlock::iterator I = BB->getFirstNonPHIOrDbgOrLifetime();
   if (I->isTerminator() && BB != &BB->getParent()->getEntryBlock() &&
       TryToSimplifyUncondBranchFromEmptyBlock(BB))
     return true;
-  
+
   // If the only instruction in the block is a seteq/setne comparison
   // against a constant, try to simplify the block.
   if (ICmpInst *ICI = dyn_cast<ICmpInst>(I))
@@ -2921,7 +3766,7 @@ bool SimplifyCFGOpt::SimplifyUncondBranch(BranchInst *BI, IRBuilder<> &Builder){
           TryToSimplifyUncondBranchWithICmpInIt(ICI, TD, Builder))
         return true;
     }
-  
+
   // If this basic block is ONLY a compare and a branch, and if a predecessor
   // branches to us and our successor, fold the comparison into the
   // predecessor and use logical operations to update the incoming value
@@ -2934,7 +3779,7 @@ bool SimplifyCFGOpt::SimplifyUncondBranch(BranchInst *BI, IRBuilder<> &Builder){
 
 bool SimplifyCFGOpt::SimplifyCondBranch(BranchInst *BI, IRBuilder<> &Builder) {
   BasicBlock *BB = BI->getParent();
-  
+
   // Conditional branch
   if (isValueEqualityComparison(BI)) {
     // If we only have one predecessor, and if it is a branch on this value,
@@ -2943,7 +3788,7 @@ bool SimplifyCFGOpt::SimplifyCondBranch(BranchInst *BI, IRBuilder<> &Builder) {
     if (BasicBlock *OnlyPred = BB->getSinglePredecessor())
       if (SimplifyEqualityComparisonWithOnlyPredecessor(BI, OnlyPred, Builder))
         return SimplifyCFG(BB) | true;
-    
+
     // This block must be empty, except for the setcond inst, if it exists.
     // Ignore dbg intrinsics.
     BasicBlock::iterator I = BB->begin();
@@ -2962,17 +3807,17 @@ bool SimplifyCFGOpt::SimplifyCondBranch(BranchInst *BI, IRBuilder<> &Builder) {
         return SimplifyCFG(BB) | true;
     }
   }
-  
+
   // Try to turn "br (X == 0 | X == 1), T, F" into a switch instruction.
   if (SimplifyBranchOnICmpChain(BI, TD, Builder))
     return true;
-  
+
   // If this basic block is ONLY a compare and a branch, and if a predecessor
   // branches to us and one of our successors, fold the comparison into the
   // predecessor and use logical operations to pick the right destination.
   if (FoldBranchToCommonDest(BI))
     return SimplifyCFG(BB) | true;
-  
+
   // We have a conditional branch to two blocks that are only reachable
   // from BI.  We know that the condbr dominates the two blocks, so see if
   // there is any identical code in the "then" and "else" blocks.  If so, we
@@ -2999,14 +3844,14 @@ bool SimplifyCFGOpt::SimplifyCondBranch(BranchInst *BI, IRBuilder<> &Builder) {
       if (SpeculativelyExecuteBB(BI, BI->getSuccessor(1)))
         return SimplifyCFG(BB) | true;
   }
-  
+
   // If this is a branch on a phi node in the current block, thread control
   // through this block if any PHI node entries are constants.
   if (PHINode *PN = dyn_cast<PHINode>(BI->getCondition()))
     if (PN->getParent() == BI->getParent())
       if (FoldCondBranchOnPHI(BI, TD))
         return SimplifyCFG(BB) | true;
-  
+
   // Scan predecessor blocks for conditional branches.
   for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI)
     if (BranchInst *PBI = dyn_cast<BranchInst>((*PI)->getTerminator()))
@@ -3023,11 +3868,12 @@ static bool passingValueIsAlwaysUndefined(Value *V, Instruction *I) {
   if (!C)
     return false;
 
-  if (!I->hasOneUse()) // Only look at single-use instructions, for compile time
+  if (I->use_empty())
     return false;
 
   if (C->isNullValue()) {
-    Instruction *Use = I->use_back();
+    // Only look at the first use, avoid hurting compile time with long uselists
+    User *Use = *I->use_begin();
 
     // Now make sure that there are no instructions in between that can alter
     // control flow (eg. calls)
@@ -3114,7 +3960,7 @@ bool SimplifyCFGOpt::run(BasicBlock *BB) {
   //
   if (MergeBlockIntoPredecessor(BB))
     return true;
-  
+
   IRBuilder<> Builder(BB);
 
   // If there is a trivial two-entry PHI node in this basic block, and we can
@@ -3152,6 +3998,7 @@ bool SimplifyCFGOpt::run(BasicBlock *BB) {
 /// eliminates unreachable basic blocks, and does other "peephole" optimization
 /// of the CFG.  It returns true if a modification was made.
 ///
-bool llvm::SimplifyCFG(BasicBlock *BB, const TargetData *TD) {
-  return SimplifyCFGOpt(TD).run(BB);
+bool llvm::SimplifyCFG(BasicBlock *BB, const DataLayout *TD,
+                       const TargetTransformInfo *TTI) {
+  return SimplifyCFGOpt(TD, TTI).run(BB);
 }
diff --git a/lib/Transforms/Utils/SimplifyIndVar.cpp b/lib/Transforms/Utils/SimplifyIndVar.cpp
index 5d673f1..110f3808 100644
--- a/lib/Transforms/Utils/SimplifyIndVar.cpp
+++ b/lib/Transforms/Utils/SimplifyIndVar.cpp
@@ -24,7 +24,7 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Utils/SimplifyIndVar.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 
@@ -44,7 +44,7 @@ namespace {
     Loop             *L;
     LoopInfo         *LI;
     ScalarEvolution  *SE;
-    const TargetData *TD; // May be NULL
+    const DataLayout *TD; // May be NULL
 
     SmallVectorImpl<WeakVH> &DeadInsts;
 
@@ -56,7 +56,7 @@ namespace {
       L(Loop),
       LI(LPM->getAnalysisIfAvailable<LoopInfo>()),
       SE(SE),
-      TD(LPM->getAnalysisIfAvailable<TargetData>()),
+      TD(LPM->getAnalysisIfAvailable<DataLayout>()),
       DeadInsts(Dead),
       Changed(false) {
       assert(LI && "IV simplification requires LoopInfo");
diff --git a/lib/Transforms/Utils/SimplifyInstructions.cpp b/lib/Transforms/Utils/SimplifyInstructions.cpp
index 81eb9e0..65353dc 100644
--- a/lib/Transforms/Utils/SimplifyInstructions.cpp
+++ b/lib/Transforms/Utils/SimplifyInstructions.cpp
@@ -23,7 +23,7 @@
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/Dominators.h"
 #include "llvm/Analysis/InstructionSimplify.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetLibraryInfo.h"
 #include "llvm/Transforms/Scalar.h"
 #include "llvm/Transforms/Utils/Local.h"
@@ -46,7 +46,7 @@ namespace {
     /// runOnFunction - Remove instructions that simplify.
     bool runOnFunction(Function &F) {
       const DominatorTree *DT = getAnalysisIfAvailable<DominatorTree>();
-      const TargetData *TD = getAnalysisIfAvailable<TargetData>();
+      const DataLayout *TD = getAnalysisIfAvailable<DataLayout>();
       const TargetLibraryInfo *TLI = &getAnalysis<TargetLibraryInfo>();
       SmallPtrSet<const Instruction*, 8> S1, S2, *ToSimplify = &S1, *Next = &S2;
       bool Changed = false;
@@ -72,7 +72,7 @@ namespace {
                 ++NumSimplified;
                 Changed = true;
               }
-            Changed |= RecursivelyDeleteTriviallyDeadInstructions(I);
+            Changed |= RecursivelyDeleteTriviallyDeadInstructions(I, TLI);
           }
 
         // Place the list of instructions to simplify on the next loop iteration
diff --git a/lib/Transforms/Utils/SimplifyLibCalls.cpp b/lib/Transforms/Utils/SimplifyLibCalls.cpp
new file mode 100644
index 0000000..c3ea638
--- /dev/null
+++ b/lib/Transforms/Utils/SimplifyLibCalls.cpp
@@ -0,0 +1,1149 @@
+//===------ SimplifyLibCalls.cpp - Library calls simplifier ---------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This is a utility pass used for testing the InstructionSimplify analysis.
+// The analysis is applied to every instruction, and if it simplifies then the
+// instruction is replaced by the simplification.  If you are looking for a pass
+// that performs serious instruction folding, use the instcombine pass instead.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Utils/SimplifyLibCalls.h"
+#include "llvm/DataLayout.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/Analysis/ValueTracking.h"
+#include "llvm/Function.h"
+#include "llvm/IRBuilder.h"
+#include "llvm/LLVMContext.h"
+#include "llvm/Target/TargetLibraryInfo.h"
+#include "llvm/Transforms/Utils/BuildLibCalls.h"
+
+using namespace llvm;
+
+/// This class is the abstract base class for the set of optimizations that
+/// corresponds to one library call.
+namespace {
+class LibCallOptimization {
+protected:
+  Function *Caller;
+  const DataLayout *TD;
+  const TargetLibraryInfo *TLI;
+  const LibCallSimplifier *LCS;
+  LLVMContext* Context;
+public:
+  LibCallOptimization() { }
+  virtual ~LibCallOptimization() {}
+
+  /// callOptimizer - This pure virtual method is implemented by base classes to
+  /// do various optimizations.  If this returns null then no transformation was
+  /// performed.  If it returns CI, then it transformed the call and CI is to be
+  /// deleted.  If it returns something else, replace CI with the new value and
+  /// delete CI.
+  virtual Value *callOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B)
+    =0;
+
+  Value *optimizeCall(CallInst *CI, const DataLayout *TD,
+                      const TargetLibraryInfo *TLI,
+                      const LibCallSimplifier *LCS, IRBuilder<> &B) {
+    Caller = CI->getParent()->getParent();
+    this->TD = TD;
+    this->TLI = TLI;
+    this->LCS = LCS;
+    if (CI->getCalledFunction())
+      Context = &CI->getCalledFunction()->getContext();
+
+    // We never change the calling convention.
+    if (CI->getCallingConv() != llvm::CallingConv::C)
+      return NULL;
+
+    return callOptimizer(CI->getCalledFunction(), CI, B);
+  }
+};
+
+//===----------------------------------------------------------------------===//
+// Helper Functions
+//===----------------------------------------------------------------------===//
+
+/// isOnlyUsedInZeroEqualityComparison - Return true if it only matters that the
+/// value is equal or not-equal to zero.
+static bool isOnlyUsedInZeroEqualityComparison(Value *V) {
+  for (Value::use_iterator UI = V->use_begin(), E = V->use_end();
+       UI != E; ++UI) {
+    if (ICmpInst *IC = dyn_cast<ICmpInst>(*UI))
+      if (IC->isEquality())
+        if (Constant *C = dyn_cast<Constant>(IC->getOperand(1)))
+          if (C->isNullValue())
+            continue;
+    // Unknown instruction.
+    return false;
+  }
+  return true;
+}
+
+/// isOnlyUsedInEqualityComparison - Return true if it is only used in equality
+/// comparisons with With.
+static bool isOnlyUsedInEqualityComparison(Value *V, Value *With) {
+  for (Value::use_iterator UI = V->use_begin(), E = V->use_end();
+       UI != E; ++UI) {
+    if (ICmpInst *IC = dyn_cast<ICmpInst>(*UI))
+      if (IC->isEquality() && IC->getOperand(1) == With)
+        continue;
+    // Unknown instruction.
+    return false;
+  }
+  return true;
+}
+
+//===----------------------------------------------------------------------===//
+// Fortified Library Call Optimizations
+//===----------------------------------------------------------------------===//
+
+struct FortifiedLibCallOptimization : public LibCallOptimization {
+protected:
+  virtual bool isFoldable(unsigned SizeCIOp, unsigned SizeArgOp,
+			  bool isString) const = 0;
+};
+
+struct InstFortifiedLibCallOptimization : public FortifiedLibCallOptimization {
+  CallInst *CI;
+
+  bool isFoldable(unsigned SizeCIOp, unsigned SizeArgOp, bool isString) const {
+    if (CI->getArgOperand(SizeCIOp) == CI->getArgOperand(SizeArgOp))
+      return true;
+    if (ConstantInt *SizeCI =
+                           dyn_cast<ConstantInt>(CI->getArgOperand(SizeCIOp))) {
+      if (SizeCI->isAllOnesValue())
+        return true;
+      if (isString) {
+        uint64_t Len = GetStringLength(CI->getArgOperand(SizeArgOp));
+        // If the length is 0 we don't know how long it is and so we can't
+        // remove the check.
+        if (Len == 0) return false;
+        return SizeCI->getZExtValue() >= Len;
+      }
+      if (ConstantInt *Arg = dyn_cast<ConstantInt>(
+                                                  CI->getArgOperand(SizeArgOp)))
+        return SizeCI->getZExtValue() >= Arg->getZExtValue();
+    }
+    return false;
+  }
+};
+
+struct MemCpyChkOpt : public InstFortifiedLibCallOptimization {
+  virtual Value *callOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
+    this->CI = CI;
+    FunctionType *FT = Callee->getFunctionType();
+    LLVMContext &Context = CI->getParent()->getContext();
+
+    // Check if this has the right signature.
+    if (FT->getNumParams() != 4 || FT->getReturnType() != FT->getParamType(0) ||
+        !FT->getParamType(0)->isPointerTy() ||
+        !FT->getParamType(1)->isPointerTy() ||
+        FT->getParamType(2) != TD->getIntPtrType(Context) ||
+        FT->getParamType(3) != TD->getIntPtrType(Context))
+      return 0;
+
+    if (isFoldable(3, 2, false)) {
+      B.CreateMemCpy(CI->getArgOperand(0), CI->getArgOperand(1),
+                     CI->getArgOperand(2), 1);
+      return CI->getArgOperand(0);
+    }
+    return 0;
+  }
+};
+
+struct MemMoveChkOpt : public InstFortifiedLibCallOptimization {
+  virtual Value *callOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
+    this->CI = CI;
+    FunctionType *FT = Callee->getFunctionType();
+    LLVMContext &Context = CI->getParent()->getContext();
+
+    // Check if this has the right signature.
+    if (FT->getNumParams() != 4 || FT->getReturnType() != FT->getParamType(0) ||
+        !FT->getParamType(0)->isPointerTy() ||
+        !FT->getParamType(1)->isPointerTy() ||
+        FT->getParamType(2) != TD->getIntPtrType(Context) ||
+        FT->getParamType(3) != TD->getIntPtrType(Context))
+      return 0;
+
+    if (isFoldable(3, 2, false)) {
+      B.CreateMemMove(CI->getArgOperand(0), CI->getArgOperand(1),
+                      CI->getArgOperand(2), 1);
+      return CI->getArgOperand(0);
+    }
+    return 0;
+  }
+};
+
+struct MemSetChkOpt : public InstFortifiedLibCallOptimization {
+  virtual Value *callOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
+    this->CI = CI;
+    FunctionType *FT = Callee->getFunctionType();
+    LLVMContext &Context = CI->getParent()->getContext();
+
+    // Check if this has the right signature.
+    if (FT->getNumParams() != 4 || FT->getReturnType() != FT->getParamType(0) ||
+        !FT->getParamType(0)->isPointerTy() ||
+        !FT->getParamType(1)->isIntegerTy() ||
+        FT->getParamType(2) != TD->getIntPtrType(Context) ||
+        FT->getParamType(3) != TD->getIntPtrType(Context))
+      return 0;
+
+    if (isFoldable(3, 2, false)) {
+      Value *Val = B.CreateIntCast(CI->getArgOperand(1), B.getInt8Ty(),
+                                   false);
+      B.CreateMemSet(CI->getArgOperand(0), Val, CI->getArgOperand(2), 1);
+      return CI->getArgOperand(0);
+    }
+    return 0;
+  }
+};
+
+struct StrCpyChkOpt : public InstFortifiedLibCallOptimization {
+  virtual Value *callOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
+    this->CI = CI;
+    StringRef Name = Callee->getName();
+    FunctionType *FT = Callee->getFunctionType();
+    LLVMContext &Context = CI->getParent()->getContext();
+
+    // Check if this has the right signature.
+    if (FT->getNumParams() != 3 ||
+        FT->getReturnType() != FT->getParamType(0) ||
+        FT->getParamType(0) != FT->getParamType(1) ||
+        FT->getParamType(0) != Type::getInt8PtrTy(Context) ||
+        FT->getParamType(2) != TD->getIntPtrType(Context))
+      return 0;
+
+    Value *Dst = CI->getArgOperand(0), *Src = CI->getArgOperand(1);
+    if (Dst == Src)      // __strcpy_chk(x,x)  -> x
+      return Src;
+
+    // If a) we don't have any length information, or b) we know this will
+    // fit then just lower to a plain strcpy. Otherwise we'll keep our
+    // strcpy_chk call which may fail at runtime if the size is too long.
+    // TODO: It might be nice to get a maximum length out of the possible
+    // string lengths for varying.
+    if (isFoldable(2, 1, true)) {
+      Value *Ret = EmitStrCpy(Dst, Src, B, TD, TLI, Name.substr(2, 6));
+      return Ret;
+    } else {
+      // Maybe we can stil fold __strcpy_chk to __memcpy_chk.
+      uint64_t Len = GetStringLength(Src);
+      if (Len == 0) return 0;
+
+      // This optimization require DataLayout.
+      if (!TD) return 0;
+
+      Value *Ret =
+	EmitMemCpyChk(Dst, Src,
+                      ConstantInt::get(TD->getIntPtrType(Context), Len),
+                      CI->getArgOperand(2), B, TD, TLI);
+      return Ret;
+    }
+    return 0;
+  }
+};
+
+struct StpCpyChkOpt : public InstFortifiedLibCallOptimization {
+  virtual Value *callOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
+    this->CI = CI;
+    StringRef Name = Callee->getName();
+    FunctionType *FT = Callee->getFunctionType();
+    LLVMContext &Context = CI->getParent()->getContext();
+
+    // Check if this has the right signature.
+    if (FT->getNumParams() != 3 ||
+        FT->getReturnType() != FT->getParamType(0) ||
+        FT->getParamType(0) != FT->getParamType(1) ||
+        FT->getParamType(0) != Type::getInt8PtrTy(Context) ||
+        FT->getParamType(2) != TD->getIntPtrType(FT->getParamType(0)))
+      return 0;
+
+    Value *Dst = CI->getArgOperand(0), *Src = CI->getArgOperand(1);
+    if (Dst == Src) {  // stpcpy(x,x)  -> x+strlen(x)
+      Value *StrLen = EmitStrLen(Src, B, TD, TLI);
+      return StrLen ? B.CreateInBoundsGEP(Dst, StrLen) : 0;
+    }
+
+    // If a) we don't have any length information, or b) we know this will
+    // fit then just lower to a plain stpcpy. Otherwise we'll keep our
+    // stpcpy_chk call which may fail at runtime if the size is too long.
+    // TODO: It might be nice to get a maximum length out of the possible
+    // string lengths for varying.
+    if (isFoldable(2, 1, true)) {
+      Value *Ret = EmitStrCpy(Dst, Src, B, TD, TLI, Name.substr(2, 6));
+      return Ret;
+    } else {
+      // Maybe we can stil fold __stpcpy_chk to __memcpy_chk.
+      uint64_t Len = GetStringLength(Src);
+      if (Len == 0) return 0;
+
+      // This optimization require DataLayout.
+      if (!TD) return 0;
+
+      Type *PT = FT->getParamType(0);
+      Value *LenV = ConstantInt::get(TD->getIntPtrType(PT), Len);
+      Value *DstEnd = B.CreateGEP(Dst,
+                                  ConstantInt::get(TD->getIntPtrType(PT),
+                                                   Len - 1));
+      if (!EmitMemCpyChk(Dst, Src, LenV, CI->getArgOperand(2), B, TD, TLI))
+        return 0;
+      return DstEnd;
+    }
+    return 0;
+  }
+};
+
+struct StrNCpyChkOpt : public InstFortifiedLibCallOptimization {
+  virtual Value *callOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
+    this->CI = CI;
+    StringRef Name = Callee->getName();
+    FunctionType *FT = Callee->getFunctionType();
+    LLVMContext &Context = CI->getParent()->getContext();
+
+    // Check if this has the right signature.
+    if (FT->getNumParams() != 4 || FT->getReturnType() != FT->getParamType(0) ||
+        FT->getParamType(0) != FT->getParamType(1) ||
+        FT->getParamType(0) != Type::getInt8PtrTy(Context) ||
+        !FT->getParamType(2)->isIntegerTy() ||
+        FT->getParamType(3) != TD->getIntPtrType(Context))
+      return 0;
+
+    if (isFoldable(3, 2, false)) {
+      Value *Ret = EmitStrNCpy(CI->getArgOperand(0), CI->getArgOperand(1),
+                               CI->getArgOperand(2), B, TD, TLI,
+                               Name.substr(2, 7));
+      return Ret;
+    }
+    return 0;
+  }
+};
+
+//===----------------------------------------------------------------------===//
+// String and Memory Library Call Optimizations
+//===----------------------------------------------------------------------===//
+
+struct StrCatOpt : public LibCallOptimization {
+  virtual Value *callOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
+    // Verify the "strcat" function prototype.
+    FunctionType *FT = Callee->getFunctionType();
+    if (FT->getNumParams() != 2 ||
+        FT->getReturnType() != B.getInt8PtrTy() ||
+        FT->getParamType(0) != FT->getReturnType() ||
+        FT->getParamType(1) != FT->getReturnType())
+      return 0;
+
+    // Extract some information from the instruction
+    Value *Dst = CI->getArgOperand(0);
+    Value *Src = CI->getArgOperand(1);
+
+    // See if we can get the length of the input string.
+    uint64_t Len = GetStringLength(Src);
+    if (Len == 0) return 0;
+    --Len;  // Unbias length.
+
+    // Handle the simple, do-nothing case: strcat(x, "") -> x
+    if (Len == 0)
+      return Dst;
+
+    // These optimizations require DataLayout.
+    if (!TD) return 0;
+
+    return emitStrLenMemCpy(Src, Dst, Len, B);
+  }
+
+  Value *emitStrLenMemCpy(Value *Src, Value *Dst, uint64_t Len,
+                          IRBuilder<> &B) {
+    // We need to find the end of the destination string.  That's where the
+    // memory is to be moved to. We just generate a call to strlen.
+    Value *DstLen = EmitStrLen(Dst, B, TD, TLI);
+    if (!DstLen)
+      return 0;
+
+    // Now that we have the destination's length, we must index into the
+    // destination's pointer to get the actual memcpy destination (end of
+    // the string .. we're concatenating).
+    Value *CpyDst = B.CreateGEP(Dst, DstLen, "endptr");
+
+    // We have enough information to now generate the memcpy call to do the
+    // concatenation for us.  Make a memcpy to copy the nul byte with align = 1.
+    B.CreateMemCpy(CpyDst, Src,
+                   ConstantInt::get(TD->getIntPtrType(*Context), Len + 1), 1);
+    return Dst;
+  }
+};
+
+struct StrNCatOpt : public StrCatOpt {
+  virtual Value *callOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
+    // Verify the "strncat" function prototype.
+    FunctionType *FT = Callee->getFunctionType();
+    if (FT->getNumParams() != 3 ||
+        FT->getReturnType() != B.getInt8PtrTy() ||
+        FT->getParamType(0) != FT->getReturnType() ||
+        FT->getParamType(1) != FT->getReturnType() ||
+        !FT->getParamType(2)->isIntegerTy())
+      return 0;
+
+    // Extract some information from the instruction
+    Value *Dst = CI->getArgOperand(0);
+    Value *Src = CI->getArgOperand(1);
+    uint64_t Len;
+
+    // We don't do anything if length is not constant
+    if (ConstantInt *LengthArg = dyn_cast<ConstantInt>(CI->getArgOperand(2)))
+      Len = LengthArg->getZExtValue();
+    else
+      return 0;
+
+    // See if we can get the length of the input string.
+    uint64_t SrcLen = GetStringLength(Src);
+    if (SrcLen == 0) return 0;
+    --SrcLen;  // Unbias length.
+
+    // Handle the simple, do-nothing cases:
+    // strncat(x, "", c) -> x
+    // strncat(x,  c, 0) -> x
+    if (SrcLen == 0 || Len == 0) return Dst;
+
+    // These optimizations require DataLayout.
+    if (!TD) return 0;
+
+    // We don't optimize this case
+    if (Len < SrcLen) return 0;
+
+    // strncat(x, s, c) -> strcat(x, s)
+    // s is constant so the strcat can be optimized further
+    return emitStrLenMemCpy(Src, Dst, SrcLen, B);
+  }
+};
+
+struct StrChrOpt : public LibCallOptimization {
+  virtual Value *callOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
+    // Verify the "strchr" function prototype.
+    FunctionType *FT = Callee->getFunctionType();
+    if (FT->getNumParams() != 2 ||
+        FT->getReturnType() != B.getInt8PtrTy() ||
+        FT->getParamType(0) != FT->getReturnType() ||
+        !FT->getParamType(1)->isIntegerTy(32))
+      return 0;
+
+    Value *SrcStr = CI->getArgOperand(0);
+
+    // If the second operand is non-constant, see if we can compute the length
+    // of the input string and turn this into memchr.
+    ConstantInt *CharC = dyn_cast<ConstantInt>(CI->getArgOperand(1));
+    if (CharC == 0) {
+      // These optimizations require DataLayout.
+      if (!TD) return 0;
+
+      uint64_t Len = GetStringLength(SrcStr);
+      if (Len == 0 || !FT->getParamType(1)->isIntegerTy(32))// memchr needs i32.
+        return 0;
+
+      return EmitMemChr(SrcStr, CI->getArgOperand(1), // include nul.
+                        ConstantInt::get(TD->getIntPtrType(*Context), Len),
+                        B, TD, TLI);
+    }
+
+    // Otherwise, the character is a constant, see if the first argument is
+    // a string literal.  If so, we can constant fold.
+    StringRef Str;
+    if (!getConstantStringInfo(SrcStr, Str))
+      return 0;
+
+    // Compute the offset, make sure to handle the case when we're searching for
+    // zero (a weird way to spell strlen).
+    size_t I = CharC->getSExtValue() == 0 ?
+        Str.size() : Str.find(CharC->getSExtValue());
+    if (I == StringRef::npos) // Didn't find the char.  strchr returns null.
+      return Constant::getNullValue(CI->getType());
+
+    // strchr(s+n,c)  -> gep(s+n+i,c)
+    return B.CreateGEP(SrcStr, B.getInt64(I), "strchr");
+  }
+};
+
+struct StrRChrOpt : public LibCallOptimization {
+  virtual Value *callOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
+    // Verify the "strrchr" function prototype.
+    FunctionType *FT = Callee->getFunctionType();
+    if (FT->getNumParams() != 2 ||
+        FT->getReturnType() != B.getInt8PtrTy() ||
+        FT->getParamType(0) != FT->getReturnType() ||
+        !FT->getParamType(1)->isIntegerTy(32))
+      return 0;
+
+    Value *SrcStr = CI->getArgOperand(0);
+    ConstantInt *CharC = dyn_cast<ConstantInt>(CI->getArgOperand(1));
+
+    // Cannot fold anything if we're not looking for a constant.
+    if (!CharC)
+      return 0;
+
+    StringRef Str;
+    if (!getConstantStringInfo(SrcStr, Str)) {
+      // strrchr(s, 0) -> strchr(s, 0)
+      if (TD && CharC->isZero())
+        return EmitStrChr(SrcStr, '\0', B, TD, TLI);
+      return 0;
+    }
+
+    // Compute the offset.
+    size_t I = CharC->getSExtValue() == 0 ?
+        Str.size() : Str.rfind(CharC->getSExtValue());
+    if (I == StringRef::npos) // Didn't find the char. Return null.
+      return Constant::getNullValue(CI->getType());
+
+    // strrchr(s+n,c) -> gep(s+n+i,c)
+    return B.CreateGEP(SrcStr, B.getInt64(I), "strrchr");
+  }
+};
+
+struct StrCmpOpt : public LibCallOptimization {
+  virtual Value *callOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
+    // Verify the "strcmp" function prototype.
+    FunctionType *FT = Callee->getFunctionType();
+    if (FT->getNumParams() != 2 ||
+        !FT->getReturnType()->isIntegerTy(32) ||
+        FT->getParamType(0) != FT->getParamType(1) ||
+        FT->getParamType(0) != B.getInt8PtrTy())
+      return 0;
+
+    Value *Str1P = CI->getArgOperand(0), *Str2P = CI->getArgOperand(1);
+    if (Str1P == Str2P)      // strcmp(x,x)  -> 0
+      return ConstantInt::get(CI->getType(), 0);
+
+    StringRef Str1, Str2;
+    bool HasStr1 = getConstantStringInfo(Str1P, Str1);
+    bool HasStr2 = getConstantStringInfo(Str2P, Str2);
+
+    // strcmp(x, y)  -> cnst  (if both x and y are constant strings)
+    if (HasStr1 && HasStr2)
+      return ConstantInt::get(CI->getType(), Str1.compare(Str2));
+
+    if (HasStr1 && Str1.empty()) // strcmp("", x) -> -*x
+      return B.CreateNeg(B.CreateZExt(B.CreateLoad(Str2P, "strcmpload"),
+                                      CI->getType()));
+
+    if (HasStr2 && Str2.empty()) // strcmp(x,"") -> *x
+      return B.CreateZExt(B.CreateLoad(Str1P, "strcmpload"), CI->getType());
+
+    // strcmp(P, "x") -> memcmp(P, "x", 2)
+    uint64_t Len1 = GetStringLength(Str1P);
+    uint64_t Len2 = GetStringLength(Str2P);
+    if (Len1 && Len2) {
+      // These optimizations require DataLayout.
+      if (!TD) return 0;
+
+      return EmitMemCmp(Str1P, Str2P,
+                        ConstantInt::get(TD->getIntPtrType(*Context),
+                        std::min(Len1, Len2)), B, TD, TLI);
+    }
+
+    return 0;
+  }
+};
+
+struct StrNCmpOpt : public LibCallOptimization {
+  virtual Value *callOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
+    // Verify the "strncmp" function prototype.
+    FunctionType *FT = Callee->getFunctionType();
+    if (FT->getNumParams() != 3 ||
+        !FT->getReturnType()->isIntegerTy(32) ||
+        FT->getParamType(0) != FT->getParamType(1) ||
+        FT->getParamType(0) != B.getInt8PtrTy() ||
+        !FT->getParamType(2)->isIntegerTy())
+      return 0;
+
+    Value *Str1P = CI->getArgOperand(0), *Str2P = CI->getArgOperand(1);
+    if (Str1P == Str2P)      // strncmp(x,x,n)  -> 0
+      return ConstantInt::get(CI->getType(), 0);
+
+    // Get the length argument if it is constant.
+    uint64_t Length;
+    if (ConstantInt *LengthArg = dyn_cast<ConstantInt>(CI->getArgOperand(2)))
+      Length = LengthArg->getZExtValue();
+    else
+      return 0;
+
+    if (Length == 0) // strncmp(x,y,0)   -> 0
+      return ConstantInt::get(CI->getType(), 0);
+
+    if (TD && Length == 1) // strncmp(x,y,1) -> memcmp(x,y,1)
+      return EmitMemCmp(Str1P, Str2P, CI->getArgOperand(2), B, TD, TLI);
+
+    StringRef Str1, Str2;
+    bool HasStr1 = getConstantStringInfo(Str1P, Str1);
+    bool HasStr2 = getConstantStringInfo(Str2P, Str2);
+
+    // strncmp(x, y)  -> cnst  (if both x and y are constant strings)
+    if (HasStr1 && HasStr2) {
+      StringRef SubStr1 = Str1.substr(0, Length);
+      StringRef SubStr2 = Str2.substr(0, Length);
+      return ConstantInt::get(CI->getType(), SubStr1.compare(SubStr2));
+    }
+
+    if (HasStr1 && Str1.empty())  // strncmp("", x, n) -> -*x
+      return B.CreateNeg(B.CreateZExt(B.CreateLoad(Str2P, "strcmpload"),
+                                      CI->getType()));
+
+    if (HasStr2 && Str2.empty())  // strncmp(x, "", n) -> *x
+      return B.CreateZExt(B.CreateLoad(Str1P, "strcmpload"), CI->getType());
+
+    return 0;
+  }
+};
+
+struct StrCpyOpt : public LibCallOptimization {
+  virtual Value *callOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
+    // Verify the "strcpy" function prototype.
+    FunctionType *FT = Callee->getFunctionType();
+    if (FT->getNumParams() != 2 ||
+        FT->getReturnType() != FT->getParamType(0) ||
+        FT->getParamType(0) != FT->getParamType(1) ||
+        FT->getParamType(0) != B.getInt8PtrTy())
+      return 0;
+
+    Value *Dst = CI->getArgOperand(0), *Src = CI->getArgOperand(1);
+    if (Dst == Src)      // strcpy(x,x)  -> x
+      return Src;
+
+    // These optimizations require DataLayout.
+    if (!TD) return 0;
+
+    // See if we can get the length of the input string.
+    uint64_t Len = GetStringLength(Src);
+    if (Len == 0) return 0;
+
+    // We have enough information to now generate the memcpy call to do the
+    // copy for us.  Make a memcpy to copy the nul byte with align = 1.
+    B.CreateMemCpy(Dst, Src,
+		   ConstantInt::get(TD->getIntPtrType(*Context), Len), 1);
+    return Dst;
+  }
+};
+
+struct StpCpyOpt: public LibCallOptimization {
+  virtual Value *callOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
+    // Verify the "stpcpy" function prototype.
+    FunctionType *FT = Callee->getFunctionType();
+    if (FT->getNumParams() != 2 ||
+        FT->getReturnType() != FT->getParamType(0) ||
+        FT->getParamType(0) != FT->getParamType(1) ||
+        FT->getParamType(0) != B.getInt8PtrTy())
+      return 0;
+
+    // These optimizations require DataLayout.
+    if (!TD) return 0;
+
+    Value *Dst = CI->getArgOperand(0), *Src = CI->getArgOperand(1);
+    if (Dst == Src) {  // stpcpy(x,x)  -> x+strlen(x)
+      Value *StrLen = EmitStrLen(Src, B, TD, TLI);
+      return StrLen ? B.CreateInBoundsGEP(Dst, StrLen) : 0;
+    }
+
+    // See if we can get the length of the input string.
+    uint64_t Len = GetStringLength(Src);
+    if (Len == 0) return 0;
+
+    Type *PT = FT->getParamType(0);
+    Value *LenV = ConstantInt::get(TD->getIntPtrType(PT), Len);
+    Value *DstEnd = B.CreateGEP(Dst,
+                                ConstantInt::get(TD->getIntPtrType(PT),
+                                                 Len - 1));
+
+    // We have enough information to now generate the memcpy call to do the
+    // copy for us.  Make a memcpy to copy the nul byte with align = 1.
+    B.CreateMemCpy(Dst, Src, LenV, 1);
+    return DstEnd;
+  }
+};
+
+struct StrNCpyOpt : public LibCallOptimization {
+  virtual Value *callOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
+    FunctionType *FT = Callee->getFunctionType();
+    if (FT->getNumParams() != 3 || FT->getReturnType() != FT->getParamType(0) ||
+        FT->getParamType(0) != FT->getParamType(1) ||
+        FT->getParamType(0) != B.getInt8PtrTy() ||
+        !FT->getParamType(2)->isIntegerTy())
+      return 0;
+
+    Value *Dst = CI->getArgOperand(0);
+    Value *Src = CI->getArgOperand(1);
+    Value *LenOp = CI->getArgOperand(2);
+
+    // See if we can get the length of the input string.
+    uint64_t SrcLen = GetStringLength(Src);
+    if (SrcLen == 0) return 0;
+    --SrcLen;
+
+    if (SrcLen == 0) {
+      // strncpy(x, "", y) -> memset(x, '\0', y, 1)
+      B.CreateMemSet(Dst, B.getInt8('\0'), LenOp, 1);
+      return Dst;
+    }
+
+    uint64_t Len;
+    if (ConstantInt *LengthArg = dyn_cast<ConstantInt>(LenOp))
+      Len = LengthArg->getZExtValue();
+    else
+      return 0;
+
+    if (Len == 0) return Dst; // strncpy(x, y, 0) -> x
+
+    // These optimizations require DataLayout.
+    if (!TD) return 0;
+
+    // Let strncpy handle the zero padding
+    if (Len > SrcLen+1) return 0;
+
+    Type *PT = FT->getParamType(0);
+    // strncpy(x, s, c) -> memcpy(x, s, c, 1) [s and c are constant]
+    B.CreateMemCpy(Dst, Src,
+                   ConstantInt::get(TD->getIntPtrType(PT), Len), 1);
+
+    return Dst;
+  }
+};
+
+struct StrLenOpt : public LibCallOptimization {
+  virtual Value *callOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
+    FunctionType *FT = Callee->getFunctionType();
+    if (FT->getNumParams() != 1 ||
+        FT->getParamType(0) != B.getInt8PtrTy() ||
+        !FT->getReturnType()->isIntegerTy())
+      return 0;
+
+    Value *Src = CI->getArgOperand(0);
+
+    // Constant folding: strlen("xyz") -> 3
+    if (uint64_t Len = GetStringLength(Src))
+      return ConstantInt::get(CI->getType(), Len-1);
+
+    // strlen(x) != 0 --> *x != 0
+    // strlen(x) == 0 --> *x == 0
+    if (isOnlyUsedInZeroEqualityComparison(CI))
+      return B.CreateZExt(B.CreateLoad(Src, "strlenfirst"), CI->getType());
+    return 0;
+  }
+};
+
+struct StrPBrkOpt : public LibCallOptimization {
+  virtual Value *callOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
+    FunctionType *FT = Callee->getFunctionType();
+    if (FT->getNumParams() != 2 ||
+        FT->getParamType(0) != B.getInt8PtrTy() ||
+        FT->getParamType(1) != FT->getParamType(0) ||
+        FT->getReturnType() != FT->getParamType(0))
+      return 0;
+
+    StringRef S1, S2;
+    bool HasS1 = getConstantStringInfo(CI->getArgOperand(0), S1);
+    bool HasS2 = getConstantStringInfo(CI->getArgOperand(1), S2);
+
+    // strpbrk(s, "") -> NULL
+    // strpbrk("", s) -> NULL
+    if ((HasS1 && S1.empty()) || (HasS2 && S2.empty()))
+      return Constant::getNullValue(CI->getType());
+
+    // Constant folding.
+    if (HasS1 && HasS2) {
+      size_t I = S1.find_first_of(S2);
+      if (I == std::string::npos) // No match.
+        return Constant::getNullValue(CI->getType());
+
+      return B.CreateGEP(CI->getArgOperand(0), B.getInt64(I), "strpbrk");
+    }
+
+    // strpbrk(s, "a") -> strchr(s, 'a')
+    if (TD && HasS2 && S2.size() == 1)
+      return EmitStrChr(CI->getArgOperand(0), S2[0], B, TD, TLI);
+
+    return 0;
+  }
+};
+
+struct StrToOpt : public LibCallOptimization {
+  virtual Value *callOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
+    FunctionType *FT = Callee->getFunctionType();
+    if ((FT->getNumParams() != 2 && FT->getNumParams() != 3) ||
+        !FT->getParamType(0)->isPointerTy() ||
+        !FT->getParamType(1)->isPointerTy())
+      return 0;
+
+    Value *EndPtr = CI->getArgOperand(1);
+    if (isa<ConstantPointerNull>(EndPtr)) {
+      // With a null EndPtr, this function won't capture the main argument.
+      // It would be readonly too, except that it still may write to errno.
+      CI->addAttribute(1, Attributes::get(Callee->getContext(),
+                                          Attributes::NoCapture));
+    }
+
+    return 0;
+  }
+};
+
+struct StrSpnOpt : public LibCallOptimization {
+  virtual Value *callOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
+    FunctionType *FT = Callee->getFunctionType();
+    if (FT->getNumParams() != 2 ||
+        FT->getParamType(0) != B.getInt8PtrTy() ||
+        FT->getParamType(1) != FT->getParamType(0) ||
+        !FT->getReturnType()->isIntegerTy())
+      return 0;
+
+    StringRef S1, S2;
+    bool HasS1 = getConstantStringInfo(CI->getArgOperand(0), S1);
+    bool HasS2 = getConstantStringInfo(CI->getArgOperand(1), S2);
+
+    // strspn(s, "") -> 0
+    // strspn("", s) -> 0
+    if ((HasS1 && S1.empty()) || (HasS2 && S2.empty()))
+      return Constant::getNullValue(CI->getType());
+
+    // Constant folding.
+    if (HasS1 && HasS2) {
+      size_t Pos = S1.find_first_not_of(S2);
+      if (Pos == StringRef::npos) Pos = S1.size();
+      return ConstantInt::get(CI->getType(), Pos);
+    }
+
+    return 0;
+  }
+};
+
+struct StrCSpnOpt : public LibCallOptimization {
+  virtual Value *callOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
+    FunctionType *FT = Callee->getFunctionType();
+    if (FT->getNumParams() != 2 ||
+        FT->getParamType(0) != B.getInt8PtrTy() ||
+        FT->getParamType(1) != FT->getParamType(0) ||
+        !FT->getReturnType()->isIntegerTy())
+      return 0;
+
+    StringRef S1, S2;
+    bool HasS1 = getConstantStringInfo(CI->getArgOperand(0), S1);
+    bool HasS2 = getConstantStringInfo(CI->getArgOperand(1), S2);
+
+    // strcspn("", s) -> 0
+    if (HasS1 && S1.empty())
+      return Constant::getNullValue(CI->getType());
+
+    // Constant folding.
+    if (HasS1 && HasS2) {
+      size_t Pos = S1.find_first_of(S2);
+      if (Pos == StringRef::npos) Pos = S1.size();
+      return ConstantInt::get(CI->getType(), Pos);
+    }
+
+    // strcspn(s, "") -> strlen(s)
+    if (TD && HasS2 && S2.empty())
+      return EmitStrLen(CI->getArgOperand(0), B, TD, TLI);
+
+    return 0;
+  }
+};
+
+struct StrStrOpt : public LibCallOptimization {
+  virtual Value *callOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
+    FunctionType *FT = Callee->getFunctionType();
+    if (FT->getNumParams() != 2 ||
+        !FT->getParamType(0)->isPointerTy() ||
+        !FT->getParamType(1)->isPointerTy() ||
+        !FT->getReturnType()->isPointerTy())
+      return 0;
+
+    // fold strstr(x, x) -> x.
+    if (CI->getArgOperand(0) == CI->getArgOperand(1))
+      return B.CreateBitCast(CI->getArgOperand(0), CI->getType());
+
+    // fold strstr(a, b) == a -> strncmp(a, b, strlen(b)) == 0
+    if (TD && isOnlyUsedInEqualityComparison(CI, CI->getArgOperand(0))) {
+      Value *StrLen = EmitStrLen(CI->getArgOperand(1), B, TD, TLI);
+      if (!StrLen)
+        return 0;
+      Value *StrNCmp = EmitStrNCmp(CI->getArgOperand(0), CI->getArgOperand(1),
+                                   StrLen, B, TD, TLI);
+      if (!StrNCmp)
+        return 0;
+      for (Value::use_iterator UI = CI->use_begin(), UE = CI->use_end();
+           UI != UE; ) {
+        ICmpInst *Old = cast<ICmpInst>(*UI++);
+        Value *Cmp = B.CreateICmp(Old->getPredicate(), StrNCmp,
+                                  ConstantInt::getNullValue(StrNCmp->getType()),
+                                  "cmp");
+        LCS->replaceAllUsesWith(Old, Cmp);
+      }
+      return CI;
+    }
+
+    // See if either input string is a constant string.
+    StringRef SearchStr, ToFindStr;
+    bool HasStr1 = getConstantStringInfo(CI->getArgOperand(0), SearchStr);
+    bool HasStr2 = getConstantStringInfo(CI->getArgOperand(1), ToFindStr);
+
+    // fold strstr(x, "") -> x.
+    if (HasStr2 && ToFindStr.empty())
+      return B.CreateBitCast(CI->getArgOperand(0), CI->getType());
+
+    // If both strings are known, constant fold it.
+    if (HasStr1 && HasStr2) {
+      std::string::size_type Offset = SearchStr.find(ToFindStr);
+
+      if (Offset == StringRef::npos) // strstr("foo", "bar") -> null
+        return Constant::getNullValue(CI->getType());
+
+      // strstr("abcd", "bc") -> gep((char*)"abcd", 1)
+      Value *Result = CastToCStr(CI->getArgOperand(0), B);
+      Result = B.CreateConstInBoundsGEP1_64(Result, Offset, "strstr");
+      return B.CreateBitCast(Result, CI->getType());
+    }
+
+    // fold strstr(x, "y") -> strchr(x, 'y').
+    if (HasStr2 && ToFindStr.size() == 1) {
+      Value *StrChr= EmitStrChr(CI->getArgOperand(0), ToFindStr[0], B, TD, TLI);
+      return StrChr ? B.CreateBitCast(StrChr, CI->getType()) : 0;
+    }
+    return 0;
+  }
+};
+
+struct MemCmpOpt : public LibCallOptimization {
+  virtual Value *callOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
+    FunctionType *FT = Callee->getFunctionType();
+    if (FT->getNumParams() != 3 || !FT->getParamType(0)->isPointerTy() ||
+        !FT->getParamType(1)->isPointerTy() ||
+        !FT->getReturnType()->isIntegerTy(32))
+      return 0;
+
+    Value *LHS = CI->getArgOperand(0), *RHS = CI->getArgOperand(1);
+
+    if (LHS == RHS)  // memcmp(s,s,x) -> 0
+      return Constant::getNullValue(CI->getType());
+
+    // Make sure we have a constant length.
+    ConstantInt *LenC = dyn_cast<ConstantInt>(CI->getArgOperand(2));
+    if (!LenC) return 0;
+    uint64_t Len = LenC->getZExtValue();
+
+    if (Len == 0) // memcmp(s1,s2,0) -> 0
+      return Constant::getNullValue(CI->getType());
+
+    // memcmp(S1,S2,1) -> *(unsigned char*)LHS - *(unsigned char*)RHS
+    if (Len == 1) {
+      Value *LHSV = B.CreateZExt(B.CreateLoad(CastToCStr(LHS, B), "lhsc"),
+                                 CI->getType(), "lhsv");
+      Value *RHSV = B.CreateZExt(B.CreateLoad(CastToCStr(RHS, B), "rhsc"),
+                                 CI->getType(), "rhsv");
+      return B.CreateSub(LHSV, RHSV, "chardiff");
+    }
+
+    // Constant folding: memcmp(x, y, l) -> cnst (all arguments are constant)
+    StringRef LHSStr, RHSStr;
+    if (getConstantStringInfo(LHS, LHSStr) &&
+        getConstantStringInfo(RHS, RHSStr)) {
+      // Make sure we're not reading out-of-bounds memory.
+      if (Len > LHSStr.size() || Len > RHSStr.size())
+        return 0;
+      uint64_t Ret = memcmp(LHSStr.data(), RHSStr.data(), Len);
+      return ConstantInt::get(CI->getType(), Ret);
+    }
+
+    return 0;
+  }
+};
+
+struct MemCpyOpt : public LibCallOptimization {
+  virtual Value *callOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
+    // These optimizations require DataLayout.
+    if (!TD) return 0;
+
+    FunctionType *FT = Callee->getFunctionType();
+    if (FT->getNumParams() != 3 || FT->getReturnType() != FT->getParamType(0) ||
+        !FT->getParamType(0)->isPointerTy() ||
+        !FT->getParamType(1)->isPointerTy() ||
+        FT->getParamType(2) != TD->getIntPtrType(*Context))
+      return 0;
+
+    // memcpy(x, y, n) -> llvm.memcpy(x, y, n, 1)
+    B.CreateMemCpy(CI->getArgOperand(0), CI->getArgOperand(1),
+                   CI->getArgOperand(2), 1);
+    return CI->getArgOperand(0);
+  }
+};
+
+struct MemMoveOpt : public LibCallOptimization {
+  virtual Value *callOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
+    // These optimizations require DataLayout.
+    if (!TD) return 0;
+
+    FunctionType *FT = Callee->getFunctionType();
+    if (FT->getNumParams() != 3 || FT->getReturnType() != FT->getParamType(0) ||
+        !FT->getParamType(0)->isPointerTy() ||
+        !FT->getParamType(1)->isPointerTy() ||
+        FT->getParamType(2) != TD->getIntPtrType(*Context))
+      return 0;
+
+    // memmove(x, y, n) -> llvm.memmove(x, y, n, 1)
+    B.CreateMemMove(CI->getArgOperand(0), CI->getArgOperand(1),
+                    CI->getArgOperand(2), 1);
+    return CI->getArgOperand(0);
+  }
+};
+
+struct MemSetOpt : public LibCallOptimization {
+  virtual Value *callOptimizer(Function *Callee, CallInst *CI, IRBuilder<> &B) {
+    // These optimizations require DataLayout.
+    if (!TD) return 0;
+
+    FunctionType *FT = Callee->getFunctionType();
+    if (FT->getNumParams() != 3 || FT->getReturnType() != FT->getParamType(0) ||
+        !FT->getParamType(0)->isPointerTy() ||
+        !FT->getParamType(1)->isIntegerTy() ||
+        FT->getParamType(2) != TD->getIntPtrType(*Context))
+      return 0;
+
+    // memset(p, v, n) -> llvm.memset(p, v, n, 1)
+    Value *Val = B.CreateIntCast(CI->getArgOperand(1), B.getInt8Ty(), false);
+    B.CreateMemSet(CI->getArgOperand(0), Val, CI->getArgOperand(2), 1);
+    return CI->getArgOperand(0);
+  }
+};
+
+} // End anonymous namespace.
+
+namespace llvm {
+
+class LibCallSimplifierImpl {
+  const DataLayout *TD;
+  const TargetLibraryInfo *TLI;
+  const LibCallSimplifier *LCS;
+  StringMap<LibCallOptimization*> Optimizations;
+
+  // Fortified library call optimizations.
+  MemCpyChkOpt MemCpyChk;
+  MemMoveChkOpt MemMoveChk;
+  MemSetChkOpt MemSetChk;
+  StrCpyChkOpt StrCpyChk;
+  StpCpyChkOpt StpCpyChk;
+  StrNCpyChkOpt StrNCpyChk;
+
+  // String library call optimizations.
+  StrCatOpt StrCat;
+  StrNCatOpt StrNCat;
+  StrChrOpt StrChr;
+  StrRChrOpt StrRChr;
+  StrCmpOpt StrCmp;
+  StrNCmpOpt StrNCmp;
+  StrCpyOpt StrCpy;
+  StpCpyOpt StpCpy;
+  StrNCpyOpt StrNCpy;
+  StrLenOpt StrLen;
+  StrPBrkOpt StrPBrk;
+  StrToOpt StrTo;
+  StrSpnOpt StrSpn;
+  StrCSpnOpt StrCSpn;
+  StrStrOpt StrStr;
+
+  // Memory library call optimizations.
+  MemCmpOpt MemCmp;
+  MemCpyOpt MemCpy;
+  MemMoveOpt MemMove;
+  MemSetOpt MemSet;
+
+  void initOptimizations();
+  void addOpt(LibFunc::Func F, LibCallOptimization* Opt);
+public:
+  LibCallSimplifierImpl(const DataLayout *TD, const TargetLibraryInfo *TLI,
+                        const LibCallSimplifier *LCS) {
+    this->TD = TD;
+    this->TLI = TLI;
+    this->LCS = LCS;
+  }
+
+  Value *optimizeCall(CallInst *CI);
+};
+
+void LibCallSimplifierImpl::initOptimizations() {
+  // Fortified library call optimizations.
+  Optimizations["__memcpy_chk"] = &MemCpyChk;
+  Optimizations["__memmove_chk"] = &MemMoveChk;
+  Optimizations["__memset_chk"] = &MemSetChk;
+  Optimizations["__strcpy_chk"] = &StrCpyChk;
+  Optimizations["__stpcpy_chk"] = &StpCpyChk;
+  Optimizations["__strncpy_chk"] = &StrNCpyChk;
+  Optimizations["__stpncpy_chk"] = &StrNCpyChk;
+
+  // String library call optimizations.
+  addOpt(LibFunc::strcat, &StrCat);
+  addOpt(LibFunc::strncat, &StrNCat);
+  addOpt(LibFunc::strchr, &StrChr);
+  addOpt(LibFunc::strrchr, &StrRChr);
+  addOpt(LibFunc::strcmp, &StrCmp);
+  addOpt(LibFunc::strncmp, &StrNCmp);
+  addOpt(LibFunc::strcpy, &StrCpy);
+  addOpt(LibFunc::stpcpy, &StpCpy);
+  addOpt(LibFunc::strncpy, &StrNCpy);
+  addOpt(LibFunc::strlen, &StrLen);
+  addOpt(LibFunc::strpbrk, &StrPBrk);
+  addOpt(LibFunc::strtol, &StrTo);
+  addOpt(LibFunc::strtod, &StrTo);
+  addOpt(LibFunc::strtof, &StrTo);
+  addOpt(LibFunc::strtoul, &StrTo);
+  addOpt(LibFunc::strtoll, &StrTo);
+  addOpt(LibFunc::strtold, &StrTo);
+  addOpt(LibFunc::strtoull, &StrTo);
+  addOpt(LibFunc::strspn, &StrSpn);
+  addOpt(LibFunc::strcspn, &StrCSpn);
+  addOpt(LibFunc::strstr, &StrStr);
+
+  // Memory library call optimizations.
+  addOpt(LibFunc::memcmp, &MemCmp);
+  addOpt(LibFunc::memcpy, &MemCpy);
+  addOpt(LibFunc::memmove, &MemMove);
+  addOpt(LibFunc::memset, &MemSet);
+}
+
+Value *LibCallSimplifierImpl::optimizeCall(CallInst *CI) {
+  if (Optimizations.empty())
+    initOptimizations();
+
+  Function *Callee = CI->getCalledFunction();
+  LibCallOptimization *LCO = Optimizations.lookup(Callee->getName());
+  if (LCO) {
+    IRBuilder<> Builder(CI);
+    return LCO->optimizeCall(CI, TD, TLI, LCS, Builder);
+  }
+  return 0;
+}
+
+void LibCallSimplifierImpl::addOpt(LibFunc::Func F, LibCallOptimization* Opt) {
+  if (TLI->has(F))
+    Optimizations[TLI->getName(F)] = Opt;
+}
+
+LibCallSimplifier::LibCallSimplifier(const DataLayout *TD,
+                                     const TargetLibraryInfo *TLI) {
+  Impl = new LibCallSimplifierImpl(TD, TLI, this);
+}
+
+LibCallSimplifier::~LibCallSimplifier() {
+  delete Impl;
+}
+
+Value *LibCallSimplifier::optimizeCall(CallInst *CI) {
+  return Impl->optimizeCall(CI);
+}
+
+void LibCallSimplifier::replaceAllUsesWith(Instruction *I, Value *With) const {
+  I->replaceAllUsesWith(With);
+  I->eraseFromParent();
+}
+
+}
diff --git a/lib/Transforms/Utils/Utils.cpp b/lib/Transforms/Utils/Utils.cpp
index 24e8c8f..5812d46 100644
--- a/lib/Transforms/Utils/Utils.cpp
+++ b/lib/Transforms/Utils/Utils.cpp
@@ -29,6 +29,7 @@ void llvm::initializeTransformUtils(PassRegistry &Registry) {
   initializePromotePassPass(Registry);
   initializeUnifyFunctionExitNodesPass(Registry);
   initializeInstSimplifierPass(Registry);
+  initializeMetaRenamerPass(Registry);
 }
 
 /// LLVMInitializeTransformUtils - C binding for initializeTransformUtilsPasses.
diff --git a/lib/Transforms/Utils/ValueMapper.cpp b/lib/Transforms/Utils/ValueMapper.cpp
index fc2538d..a30b093 100644
--- a/lib/Transforms/Utils/ValueMapper.cpp
+++ b/lib/Transforms/Utils/ValueMapper.cpp
@@ -21,7 +21,7 @@
 using namespace llvm;
 
 // Out of line method to get vtable etc for class.
-void ValueMapTypeRemapper::Anchor() {}
+void ValueMapTypeRemapper::anchor() {}
 
 Value *llvm::MapValue(const Value *V, ValueToValueMapTy &VM, RemapFlags Flags,
                       ValueMapTypeRemapper *TypeMapper) {
diff --git a/lib/Transforms/Vectorize/BBVectorize.cpp b/lib/Transforms/Vectorize/BBVectorize.cpp
index 62d23cb..f7be3e3 100644
--- a/lib/Transforms/Vectorize/BBVectorize.cpp
+++ b/lib/Transforms/Vectorize/BBVectorize.cpp
@@ -28,12 +28,14 @@
 #include "llvm/Type.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Analysis/AliasAnalysis.h"
 #include "llvm/Analysis/AliasSetTracker.h"
+#include "llvm/Analysis/Dominators.h"
 #include "llvm/Analysis/ScalarEvolution.h"
 #include "llvm/Analysis/ScalarEvolutionExpressions.h"
 #include "llvm/Analysis/ValueTracking.h"
@@ -41,17 +43,27 @@
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/ValueHandle.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
+#include "llvm/TargetTransformInfo.h"
 #include "llvm/Transforms/Utils/Local.h"
 #include "llvm/Transforms/Vectorize.h"
 #include <algorithm>
 #include <map>
 using namespace llvm;
 
+static cl::opt<bool>
+IgnoreTargetInfo("bb-vectorize-ignore-target-info",  cl::init(false),
+  cl::Hidden, cl::desc("Ignore target information"));
+
 static cl::opt<unsigned>
 ReqChainDepth("bb-vectorize-req-chain-depth", cl::init(6), cl::Hidden,
   cl::desc("The required chain depth for vectorization"));
 
+static cl::opt<bool>
+UseChainDepthWithTI("bb-vectorize-use-chain-depth",  cl::init(false),
+  cl::Hidden, cl::desc("Use the chain depth requirement with"
+                       " target information"));
+
 static cl::opt<unsigned>
 SearchLimit("bb-vectorize-search-limit", cl::init(400), cl::Hidden,
   cl::desc("The maximum search distance for instruction pairs"));
@@ -93,8 +105,9 @@ static cl::opt<bool>
 NoFloats("bb-vectorize-no-floats", cl::init(false), cl::Hidden,
   cl::desc("Don't try to vectorize floating-point values"));
 
+// FIXME: This should default to false once pointer vector support works.
 static cl::opt<bool>
-NoPointers("bb-vectorize-no-pointers", cl::init(false), cl::Hidden,
+NoPointers("bb-vectorize-no-pointers", cl::init(/*false*/ true), cl::Hidden,
   cl::desc("Don't try to vectorize pointer values"));
 
 static cl::opt<bool>
@@ -159,6 +172,12 @@ DebugCycleCheck("bb-vectorize-debug-cycle-check",
   cl::init(false), cl::Hidden,
   cl::desc("When debugging is enabled, output information on the"
            " cycle-checking process"));
+
+static cl::opt<bool>
+PrintAfterEveryPair("bb-vectorize-debug-print-after-every-pair",
+  cl::init(false), cl::Hidden,
+  cl::desc("When debugging is enabled, dump the basic block after"
+           " every pair is fused"));
 #endif
 
 STATISTIC(NumFusedOps, "Number of operations fused by bb-vectorize");
@@ -177,13 +196,19 @@ namespace {
     BBVectorize(Pass *P, const VectorizeConfig &C)
       : BasicBlockPass(ID), Config(C) {
       AA = &P->getAnalysis<AliasAnalysis>();
+      DT = &P->getAnalysis<DominatorTree>();
       SE = &P->getAnalysis<ScalarEvolution>();
-      TD = P->getAnalysisIfAvailable<TargetData>();
+      TD = P->getAnalysisIfAvailable<DataLayout>();
+      TTI = IgnoreTargetInfo ? 0 :
+        P->getAnalysisIfAvailable<TargetTransformInfo>();
+      VTTI = TTI ? TTI->getVectorTargetTransformInfo() : 0;
     }
 
     typedef std::pair<Value *, Value *> ValuePair;
+    typedef std::pair<ValuePair, int> ValuePairWithCost;
     typedef std::pair<ValuePair, size_t> ValuePairWithDepth;
     typedef std::pair<ValuePair, ValuePair> VPPair; // A ValuePair pair
+    typedef std::pair<VPPair, unsigned> VPPairWithType;
     typedef std::pair<std::multimap<Value *, Value *>::iterator,
               std::multimap<Value *, Value *>::iterator> VPIteratorPair;
     typedef std::pair<std::multimap<ValuePair, ValuePair>::iterator,
@@ -191,8 +216,11 @@ namespace {
                 VPPIteratorPair;
 
     AliasAnalysis *AA;
+    DominatorTree *DT;
     ScalarEvolution *SE;
-    TargetData *TD;
+    DataLayout *TD;
+    TargetTransformInfo *TTI;
+    const VectorTargetTransformInfo *VTTI;
 
     // FIXME: const correct?
 
@@ -201,11 +229,23 @@ namespace {
     bool getCandidatePairs(BasicBlock &BB,
                        BasicBlock::iterator &Start,
                        std::multimap<Value *, Value *> &CandidatePairs,
+                       DenseSet<ValuePair> &FixedOrderPairs,
+                       DenseMap<ValuePair, int> &CandidatePairCostSavings,
                        std::vector<Value *> &PairableInsts, bool NonPow2Len);
 
+    // FIXME: The current implementation does not account for pairs that
+    // are connected in multiple ways. For example:
+    //   C1 = A1 / A2; C2 = A2 / A1 (which may be both direct and a swap)
+    enum PairConnectionType {
+      PairConnectionDirect,
+      PairConnectionSwap,
+      PairConnectionSplat
+    };
+
     void computeConnectedPairs(std::multimap<Value *, Value *> &CandidatePairs,
                        std::vector<Value *> &PairableInsts,
-                       std::multimap<ValuePair, ValuePair> &ConnectedPairs);
+                       std::multimap<ValuePair, ValuePair> &ConnectedPairs,
+                       DenseMap<VPPair, unsigned> &PairConnectionTypes);
 
     void buildDepMap(BasicBlock &BB,
                        std::multimap<Value *, Value *> &CandidatePairs,
@@ -213,19 +253,29 @@ namespace {
                        DenseSet<ValuePair> &PairableInstUsers);
 
     void choosePairs(std::multimap<Value *, Value *> &CandidatePairs,
+                        DenseMap<ValuePair, int> &CandidatePairCostSavings,
                         std::vector<Value *> &PairableInsts,
+                        DenseSet<ValuePair> &FixedOrderPairs,
+                        DenseMap<VPPair, unsigned> &PairConnectionTypes,
                         std::multimap<ValuePair, ValuePair> &ConnectedPairs,
+                        std::multimap<ValuePair, ValuePair> &ConnectedPairDeps,
                         DenseSet<ValuePair> &PairableInstUsers,
                         DenseMap<Value *, Value *>& ChosenPairs);
 
     void fuseChosenPairs(BasicBlock &BB,
                      std::vector<Value *> &PairableInsts,
-                     DenseMap<Value *, Value *>& ChosenPairs);
+                     DenseMap<Value *, Value *>& ChosenPairs,
+                     DenseSet<ValuePair> &FixedOrderPairs,
+                     DenseMap<VPPair, unsigned> &PairConnectionTypes,
+                     std::multimap<ValuePair, ValuePair> &ConnectedPairs,
+                     std::multimap<ValuePair, ValuePair> &ConnectedPairDeps);
+
 
     bool isInstVectorizable(Instruction *I, bool &IsSimpleLoadStore);
 
     bool areInstsCompatible(Instruction *I, Instruction *J,
-                       bool IsSimpleLoadStore, bool NonPow2Len);
+                       bool IsSimpleLoadStore, bool NonPow2Len,
+                       int &CostSavings, int &FixedOrder);
 
     bool trackUsesOfI(DenseSet<Value *> &Users,
                       AliasSetTracker &WriteSet, Instruction *I,
@@ -236,6 +286,7 @@ namespace {
                       std::multimap<Value *, Value *> &CandidatePairs,
                       std::vector<Value *> &PairableInsts,
                       std::multimap<ValuePair, ValuePair> &ConnectedPairs,
+                      DenseMap<VPPair, unsigned> &PairConnectionTypes,
                       ValuePair P);
 
     bool pairsConflict(ValuePair P, ValuePair Q,
@@ -267,17 +318,21 @@ namespace {
 
     void findBestTreeFor(
                       std::multimap<Value *, Value *> &CandidatePairs,
+                      DenseMap<ValuePair, int> &CandidatePairCostSavings,
                       std::vector<Value *> &PairableInsts,
+                      DenseSet<ValuePair> &FixedOrderPairs,
+                      DenseMap<VPPair, unsigned> &PairConnectionTypes,
                       std::multimap<ValuePair, ValuePair> &ConnectedPairs,
+                      std::multimap<ValuePair, ValuePair> &ConnectedPairDeps,
                       DenseSet<ValuePair> &PairableInstUsers,
                       std::multimap<ValuePair, ValuePair> &PairableInstUserMap,
                       DenseMap<Value *, Value *> &ChosenPairs,
                       DenseSet<ValuePair> &BestTree, size_t &BestMaxDepth,
-                      size_t &BestEffSize, VPIteratorPair ChoiceRange,
+                      int &BestEffSize, VPIteratorPair ChoiceRange,
                       bool UseCycleCheck);
 
     Value *getReplacementPointerInput(LLVMContext& Context, Instruction *I,
-                     Instruction *J, unsigned o, bool FlipMemInputs);
+                     Instruction *J, unsigned o);
 
     void fillNewShuffleMask(LLVMContext& Context, Instruction *J,
                      unsigned MaskOffset, unsigned NumInElem,
@@ -289,20 +344,20 @@ namespace {
 
     bool expandIEChain(LLVMContext& Context, Instruction *I, Instruction *J,
                        unsigned o, Value *&LOp, unsigned numElemL,
-                       Type *ArgTypeL, Type *ArgTypeR,
+                       Type *ArgTypeL, Type *ArgTypeR, bool IBeforeJ,
                        unsigned IdxOff = 0);
 
     Value *getReplacementInput(LLVMContext& Context, Instruction *I,
-                     Instruction *J, unsigned o, bool FlipMemInputs);
+                     Instruction *J, unsigned o, bool IBeforeJ);
 
     void getReplacementInputsForPair(LLVMContext& Context, Instruction *I,
                      Instruction *J, SmallVector<Value *, 3> &ReplacedOperands,
-                     bool FlipMemInputs);
+                     bool IBeforeJ);
 
     void replaceOutputsOfPair(LLVMContext& Context, Instruction *I,
                      Instruction *J, Instruction *K,
                      Instruction *&InsertionPt, Instruction *&K1,
-                     Instruction *&K2, bool FlipMemInputs);
+                     Instruction *&K2);
 
     void collectPairLoadMoveSet(BasicBlock &BB,
                      DenseMap<Value *, Value *> &ChosenPairs,
@@ -314,10 +369,6 @@ namespace {
                      DenseMap<Value *, Value *> &ChosenPairs,
                      std::multimap<Value *, Value *> &LoadMoveSet);
 
-    void collectPtrInfo(std::vector<Value *> &PairableInsts,
-                        DenseMap<Value *, Value *> &ChosenPairs,
-                        DenseSet<Value *> &LowPtrInsts);
-
     bool canMoveUsesOfIAfterJ(BasicBlock &BB,
                      std::multimap<Value *, Value *> &LoadMoveSet,
                      Instruction *I, Instruction *J);
@@ -330,13 +381,22 @@ namespace {
     void combineMetadata(Instruction *K, const Instruction *J);
 
     bool vectorizeBB(BasicBlock &BB) {
+      if (!DT->isReachableFromEntry(&BB)) {
+        DEBUG(dbgs() << "BBV: skipping unreachable " << BB.getName() <<
+              " in " << BB.getParent()->getName() << "\n");
+        return false;
+      }
+
+      DEBUG(if (VTTI) dbgs() << "BBV: using target information\n");
+
       bool changed = false;
       // Iterate a sufficient number of times to merge types of size 1 bit,
       // then 2 bits, then 4, etc. up to half of the target vector width of the
       // target vector register.
       unsigned n = 1;
       for (unsigned v = 2;
-           v <= Config.VectorBits && (!Config.MaxIter || n <= Config.MaxIter);
+           (VTTI || v <= Config.VectorBits) &&
+           (!Config.MaxIter || n <= Config.MaxIter);
            v *= 2, ++n) {
         DEBUG(dbgs() << "BBV: fusing loop #" << n <<
               " for " << BB.getName() << " in " <<
@@ -363,8 +423,12 @@ namespace {
 
     virtual bool runOnBasicBlock(BasicBlock &BB) {
       AA = &getAnalysis<AliasAnalysis>();
+      DT = &getAnalysis<DominatorTree>();
       SE = &getAnalysis<ScalarEvolution>();
-      TD = getAnalysisIfAvailable<TargetData>();
+      TD = getAnalysisIfAvailable<DataLayout>();
+      TTI = IgnoreTargetInfo ? 0 :
+        getAnalysisIfAvailable<TargetTransformInfo>();
+      VTTI = TTI ? TTI->getVectorTargetTransformInfo() : 0;
 
       return vectorizeBB(BB);
     }
@@ -372,8 +436,10 @@ namespace {
     virtual void getAnalysisUsage(AnalysisUsage &AU) const {
       BasicBlockPass::getAnalysisUsage(AU);
       AU.addRequired<AliasAnalysis>();
+      AU.addRequired<DominatorTree>();
       AU.addRequired<ScalarEvolution>();
       AU.addPreserved<AliasAnalysis>();
+      AU.addPreserved<DominatorTree>();
       AU.addPreserved<ScalarEvolution>();
       AU.setPreservesCFG();
     }
@@ -415,6 +481,14 @@ namespace {
         T2 = cast<CastInst>(I)->getSrcTy();
       else
         T2 = T1;
+
+      if (SelectInst *SI = dyn_cast<SelectInst>(I)) {
+        T2 = SI->getCondition()->getType();
+      } else if (ShuffleVectorInst *SI = dyn_cast<ShuffleVectorInst>(I)) {
+        T2 = SI->getOperand(0)->getType();
+      } else if (CmpInst *CI = dyn_cast<CmpInst>(I)) {
+        T2 = CI->getOperand(0)->getType();
+      }
     }
 
     // Returns the weight associated with the provided value. A chain of
@@ -446,6 +520,62 @@ namespace {
       return 1;
     }
 
+    // Returns the cost of the provided instruction using VTTI.
+    // This does not handle loads and stores.
+    unsigned getInstrCost(unsigned Opcode, Type *T1, Type *T2) {
+      switch (Opcode) {
+      default: break;
+      case Instruction::GetElementPtr:
+        // We mark this instruction as zero-cost because scalar GEPs are usually
+        // lowered to the intruction addressing mode. At the moment we don't
+        // generate vector GEPs.
+        return 0;
+      case Instruction::Br:
+        return VTTI->getCFInstrCost(Opcode);
+      case Instruction::PHI:
+        return 0;
+      case Instruction::Add:
+      case Instruction::FAdd:
+      case Instruction::Sub:
+      case Instruction::FSub:
+      case Instruction::Mul:
+      case Instruction::FMul:
+      case Instruction::UDiv:
+      case Instruction::SDiv:
+      case Instruction::FDiv:
+      case Instruction::URem:
+      case Instruction::SRem:
+      case Instruction::FRem:
+      case Instruction::Shl:
+      case Instruction::LShr:
+      case Instruction::AShr:
+      case Instruction::And:
+      case Instruction::Or:
+      case Instruction::Xor:
+        return VTTI->getArithmeticInstrCost(Opcode, T1);
+      case Instruction::Select:
+      case Instruction::ICmp:
+      case Instruction::FCmp:
+        return VTTI->getCmpSelInstrCost(Opcode, T1, T2);
+      case Instruction::ZExt:
+      case Instruction::SExt:
+      case Instruction::FPToUI:
+      case Instruction::FPToSI:
+      case Instruction::FPExt:
+      case Instruction::PtrToInt:
+      case Instruction::IntToPtr:
+      case Instruction::SIToFP:
+      case Instruction::UIToFP:
+      case Instruction::Trunc:
+      case Instruction::FPTrunc:
+      case Instruction::BitCast:
+      case Instruction::ShuffleVector:
+        return VTTI->getCastInstrCost(Opcode, T1, T2);
+      }
+
+      return 1;
+    }
+
     // This determines the relative offset of two loads or stores, returning
     // true if the offset could be determined to be some constant value.
     // For example, if OffsetInElmts == 1, then J accesses the memory directly
@@ -453,20 +583,30 @@ namespace {
     // directly after J.
     bool getPairPtrInfo(Instruction *I, Instruction *J,
         Value *&IPtr, Value *&JPtr, unsigned &IAlignment, unsigned &JAlignment,
-        int64_t &OffsetInElmts) {
+        unsigned &IAddressSpace, unsigned &JAddressSpace,
+        int64_t &OffsetInElmts, bool ComputeOffset = true) {
       OffsetInElmts = 0;
-      if (isa<LoadInst>(I)) {
-        IPtr = cast<LoadInst>(I)->getPointerOperand();
-        JPtr = cast<LoadInst>(J)->getPointerOperand();
-        IAlignment = cast<LoadInst>(I)->getAlignment();
-        JAlignment = cast<LoadInst>(J)->getAlignment();
+      if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
+        LoadInst *LJ = cast<LoadInst>(J);
+        IPtr = LI->getPointerOperand();
+        JPtr = LJ->getPointerOperand();
+        IAlignment = LI->getAlignment();
+        JAlignment = LJ->getAlignment();
+        IAddressSpace = LI->getPointerAddressSpace();
+        JAddressSpace = LJ->getPointerAddressSpace();
       } else {
-        IPtr = cast<StoreInst>(I)->getPointerOperand();
-        JPtr = cast<StoreInst>(J)->getPointerOperand();
-        IAlignment = cast<StoreInst>(I)->getAlignment();
-        JAlignment = cast<StoreInst>(J)->getAlignment();
+        StoreInst *SI = cast<StoreInst>(I), *SJ = cast<StoreInst>(J);
+        IPtr = SI->getPointerOperand();
+        JPtr = SJ->getPointerOperand();
+        IAlignment = SI->getAlignment();
+        JAlignment = SJ->getAlignment();
+        IAddressSpace = SI->getPointerAddressSpace();
+        JAddressSpace = SJ->getPointerAddressSpace();
       }
 
+      if (!ComputeOffset)
+        return true;
+
       const SCEV *IPtrSCEV = SE->getSCEV(IPtr);
       const SCEV *JPtrSCEV = SE->getSCEV(JPtr);
 
@@ -536,6 +676,19 @@ namespace {
 
       return false;
     }
+
+    bool isPureIEChain(InsertElementInst *IE) {
+      InsertElementInst *IENext = IE;
+      do {
+        if (!isa<UndefValue>(IENext->getOperand(0)) &&
+            !isa<InsertElementInst>(IENext->getOperand(0))) {
+          return false;
+        }
+      } while ((IENext =
+                 dyn_cast<InsertElementInst>(IENext->getOperand(0))));
+
+      return true;
+    }
   };
 
   // This function implements one vectorization iteration on the provided
@@ -546,11 +699,18 @@ namespace {
 
     std::vector<Value *> AllPairableInsts;
     DenseMap<Value *, Value *> AllChosenPairs;
+    DenseSet<ValuePair> AllFixedOrderPairs;
+    DenseMap<VPPair, unsigned> AllPairConnectionTypes;
+    std::multimap<ValuePair, ValuePair> AllConnectedPairs, AllConnectedPairDeps;
 
     do {
       std::vector<Value *> PairableInsts;
       std::multimap<Value *, Value *> CandidatePairs;
+      DenseSet<ValuePair> FixedOrderPairs;
+      DenseMap<ValuePair, int> CandidatePairCostSavings;
       ShouldContinue = getCandidatePairs(BB, Start, CandidatePairs,
+                                         FixedOrderPairs,
+                                         CandidatePairCostSavings,
                                          PairableInsts, NonPow2Len);
       if (PairableInsts.empty()) continue;
 
@@ -563,10 +723,18 @@ namespace {
       // Note that it only matters that both members of the second pair use some
       // element of the first pair (to allow for splatting).
 
-      std::multimap<ValuePair, ValuePair> ConnectedPairs;
-      computeConnectedPairs(CandidatePairs, PairableInsts, ConnectedPairs);
+      std::multimap<ValuePair, ValuePair> ConnectedPairs, ConnectedPairDeps;
+      DenseMap<VPPair, unsigned> PairConnectionTypes;
+      computeConnectedPairs(CandidatePairs, PairableInsts, ConnectedPairs,
+                            PairConnectionTypes);
       if (ConnectedPairs.empty()) continue;
 
+      for (std::multimap<ValuePair, ValuePair>::iterator
+           I = ConnectedPairs.begin(), IE = ConnectedPairs.end();
+           I != IE; ++I) {
+        ConnectedPairDeps.insert(VPPair(I->second, I->first));
+      }
+
       // Build the pairable-instruction dependency map
       DenseSet<ValuePair> PairableInstUsers;
       buildDepMap(BB, CandidatePairs, PairableInsts, PairableInstUsers);
@@ -578,13 +746,48 @@ namespace {
       // variables.
 
       DenseMap<Value *, Value *> ChosenPairs;
-      choosePairs(CandidatePairs, PairableInsts, ConnectedPairs,
+      choosePairs(CandidatePairs, CandidatePairCostSavings,
+        PairableInsts, FixedOrderPairs, PairConnectionTypes,
+        ConnectedPairs, ConnectedPairDeps,
         PairableInstUsers, ChosenPairs);
 
       if (ChosenPairs.empty()) continue;
       AllPairableInsts.insert(AllPairableInsts.end(), PairableInsts.begin(),
                               PairableInsts.end());
       AllChosenPairs.insert(ChosenPairs.begin(), ChosenPairs.end());
+
+      // Only for the chosen pairs, propagate information on fixed-order pairs,
+      // pair connections, and their types to the data structures used by the
+      // pair fusion procedures.
+      for (DenseMap<Value *, Value *>::iterator I = ChosenPairs.begin(),
+           IE = ChosenPairs.end(); I != IE; ++I) {
+        if (FixedOrderPairs.count(*I))
+          AllFixedOrderPairs.insert(*I);
+        else if (FixedOrderPairs.count(ValuePair(I->second, I->first)))
+          AllFixedOrderPairs.insert(ValuePair(I->second, I->first));
+
+        for (DenseMap<Value *, Value *>::iterator J = ChosenPairs.begin();
+             J != IE; ++J) {
+          DenseMap<VPPair, unsigned>::iterator K =
+            PairConnectionTypes.find(VPPair(*I, *J));
+          if (K != PairConnectionTypes.end()) {
+            AllPairConnectionTypes.insert(*K);
+          } else {
+            K = PairConnectionTypes.find(VPPair(*J, *I));
+            if (K != PairConnectionTypes.end())
+              AllPairConnectionTypes.insert(*K);
+          }
+        }
+      }
+
+      for (std::multimap<ValuePair, ValuePair>::iterator
+           I = ConnectedPairs.begin(), IE = ConnectedPairs.end();
+           I != IE; ++I) {
+        if (AllPairConnectionTypes.count(*I)) {
+          AllConnectedPairs.insert(*I);
+          AllConnectedPairDeps.insert(VPPair(I->second, I->first));
+        }
+      }
     } while (ShouldContinue);
 
     if (AllChosenPairs.empty()) return false;
@@ -597,11 +800,13 @@ namespace {
     // replaced with a vector_extract on the result.  Subsequent optimization
     // passes should coalesce the build/extract combinations.
 
-    fuseChosenPairs(BB, AllPairableInsts, AllChosenPairs);
+    fuseChosenPairs(BB, AllPairableInsts, AllChosenPairs, AllFixedOrderPairs,
+                    AllPairConnectionTypes,
+                    AllConnectedPairs, AllConnectedPairDeps);
 
     // It is important to cleanup here so that future iterations of this
     // function have less work to do.
-    (void) SimplifyInstructionsInBlock(&BB, TD);
+    (void) SimplifyInstructionsInBlock(&BB, TD, AA->getTargetLibraryInfo());
     return true;
   }
 
@@ -667,15 +872,22 @@ namespace {
         !(VectorType::isValidElementType(T2) || T2->isVectorTy()))
       return false;
 
-    if (T1->getScalarSizeInBits() == 1 && T2->getScalarSizeInBits() == 1) {
+    if (T1->getScalarSizeInBits() == 1) {
       if (!Config.VectorizeBools)
         return false;
     } else {
-      if (!Config.VectorizeInts
-          && (T1->isIntOrIntVectorTy() || T2->isIntOrIntVectorTy()))
+      if (!Config.VectorizeInts && T1->isIntOrIntVectorTy())
         return false;
     }
-  
+
+    if (T2->getScalarSizeInBits() == 1) {
+      if (!Config.VectorizeBools)
+        return false;
+    } else {
+      if (!Config.VectorizeInts && T2->isIntOrIntVectorTy())
+        return false;
+    }
+
     if (!Config.VectorizeFloats
         && (T1->isFPOrFPVectorTy() || T2->isFPOrFPVectorTy()))
       return false;
@@ -691,8 +903,8 @@ namespace {
          T2->getScalarType()->isPointerTy()))
       return false;
 
-    if (T1->getPrimitiveSizeInBits() >= Config.VectorBits ||
-        T2->getPrimitiveSizeInBits() >= Config.VectorBits)
+    if (!VTTI && (T1->getPrimitiveSizeInBits() >= Config.VectorBits ||
+                  T2->getPrimitiveSizeInBits() >= Config.VectorBits))
       return false;
 
     return true;
@@ -703,10 +915,14 @@ namespace {
   // that I has already been determined to be vectorizable and that J is not
   // in the use tree of I.
   bool BBVectorize::areInstsCompatible(Instruction *I, Instruction *J,
-                       bool IsSimpleLoadStore, bool NonPow2Len) {
+                       bool IsSimpleLoadStore, bool NonPow2Len,
+                       int &CostSavings, int &FixedOrder) {
     DEBUG(if (DebugInstructionExamination) dbgs() << "BBV: looking at " << *I <<
                      " <-> " << *J << "\n");
 
+    CostSavings = 0;
+    FixedOrder = 0;
+
     // Loads and stores can be merged if they have different alignments,
     // but are otherwise the same.
     if (!J->isSameOperationAs(I, Instruction::CompareIgnoringAlignment |
@@ -719,38 +935,84 @@ namespace {
     unsigned MaxTypeBits = std::max(
       IT1->getPrimitiveSizeInBits() + JT1->getPrimitiveSizeInBits(),
       IT2->getPrimitiveSizeInBits() + JT2->getPrimitiveSizeInBits());
-    if (MaxTypeBits > Config.VectorBits)
+    if (!VTTI && MaxTypeBits > Config.VectorBits)
       return false;
 
     // FIXME: handle addsub-type operations!
 
     if (IsSimpleLoadStore) {
       Value *IPtr, *JPtr;
-      unsigned IAlignment, JAlignment;
+      unsigned IAlignment, JAlignment, IAddressSpace, JAddressSpace;
       int64_t OffsetInElmts = 0;
       if (getPairPtrInfo(I, J, IPtr, JPtr, IAlignment, JAlignment,
+            IAddressSpace, JAddressSpace,
             OffsetInElmts) && abs64(OffsetInElmts) == 1) {
-        if (Config.AlignedOnly) {
-          Type *aTypeI = isa<StoreInst>(I) ?
-            cast<StoreInst>(I)->getValueOperand()->getType() : I->getType();
-          Type *aTypeJ = isa<StoreInst>(J) ?
-            cast<StoreInst>(J)->getValueOperand()->getType() : J->getType();
+        FixedOrder = (int) OffsetInElmts;
+        unsigned BottomAlignment = IAlignment;
+        if (OffsetInElmts < 0) BottomAlignment = JAlignment;
+
+        Type *aTypeI = isa<StoreInst>(I) ?
+          cast<StoreInst>(I)->getValueOperand()->getType() : I->getType();
+        Type *aTypeJ = isa<StoreInst>(J) ?
+          cast<StoreInst>(J)->getValueOperand()->getType() : J->getType();
+        Type *VType = getVecTypeForPair(aTypeI, aTypeJ);
 
+        if (Config.AlignedOnly) {
           // An aligned load or store is possible only if the instruction
           // with the lower offset has an alignment suitable for the
           // vector type.
 
-          unsigned BottomAlignment = IAlignment;
-          if (OffsetInElmts < 0) BottomAlignment = JAlignment;
-
-          Type *VType = getVecTypeForPair(aTypeI, aTypeJ);
           unsigned VecAlignment = TD->getPrefTypeAlignment(VType);
           if (BottomAlignment < VecAlignment)
             return false;
         }
+
+        if (VTTI) {
+          unsigned ICost = VTTI->getMemoryOpCost(I->getOpcode(), I->getType(),
+                                                 IAlignment, IAddressSpace);
+          unsigned JCost = VTTI->getMemoryOpCost(J->getOpcode(), J->getType(),
+                                                 JAlignment, JAddressSpace);
+          unsigned VCost = VTTI->getMemoryOpCost(I->getOpcode(), VType,
+                                                 BottomAlignment,
+                                                 IAddressSpace);
+          if (VCost > ICost + JCost)
+            return false;
+
+          // We don't want to fuse to a type that will be split, even
+          // if the two input types will also be split and there is no other
+          // associated cost.
+          unsigned VParts = VTTI->getNumberOfParts(VType);
+          if (VParts > 1)
+            return false;
+          else if (!VParts && VCost == ICost + JCost)
+            return false;
+
+          CostSavings = ICost + JCost - VCost;
+        }
       } else {
         return false;
       }
+    } else if (VTTI) {
+      unsigned ICost = getInstrCost(I->getOpcode(), IT1, IT2);
+      unsigned JCost = getInstrCost(J->getOpcode(), JT1, JT2);
+      Type *VT1 = getVecTypeForPair(IT1, JT1),
+           *VT2 = getVecTypeForPair(IT2, JT2);
+      unsigned VCost = getInstrCost(I->getOpcode(), VT1, VT2);
+
+      if (VCost > ICost + JCost)
+        return false;
+
+      // We don't want to fuse to a type that will be split, even
+      // if the two input types will also be split and there is no other
+      // associated cost.
+      unsigned VParts1 = VTTI->getNumberOfParts(VT1),
+               VParts2 = VTTI->getNumberOfParts(VT2);
+      if (VParts1 > 1 || VParts2 > 1)
+        return false;
+      else if ((!VParts1 || !VParts2) && VCost == ICost + JCost)
+        return false;
+
+      CostSavings = ICost + JCost - VCost;
     }
 
     // The powi intrinsic is special because only the first argument is
@@ -833,6 +1095,8 @@ namespace {
   bool BBVectorize::getCandidatePairs(BasicBlock &BB,
                        BasicBlock::iterator &Start,
                        std::multimap<Value *, Value *> &CandidatePairs,
+                       DenseSet<ValuePair> &FixedOrderPairs,
+                       DenseMap<ValuePair, int> &CandidatePairCostSavings,
                        std::vector<Value *> &PairableInsts, bool NonPow2Len) {
     BasicBlock::iterator E = BB.end();
     if (Start == E) return false;
@@ -869,7 +1133,9 @@ namespace {
 
         // J does not use I, and comes before the first use of I, so it can be
         // merged with I if the instructions are compatible.
-        if (!areInstsCompatible(I, J, IsSimpleLoadStore, NonPow2Len)) continue;
+        int CostSavings, FixedOrder;
+        if (!areInstsCompatible(I, J, IsSimpleLoadStore, NonPow2Len,
+            CostSavings, FixedOrder)) continue;
 
         // J is a candidate for merging with I.
         if (!PairableInsts.size() ||
@@ -878,6 +1144,14 @@ namespace {
         }
 
         CandidatePairs.insert(ValuePair(I, J));
+        if (VTTI)
+          CandidatePairCostSavings.insert(ValuePairWithCost(ValuePair(I, J),
+                                                            CostSavings));
+
+        if (FixedOrder == 1)
+          FixedOrderPairs.insert(ValuePair(I, J));
+        else if (FixedOrder == -1)
+          FixedOrderPairs.insert(ValuePair(J, I));
 
         // The next call to this function must start after the last instruction
         // selected during this invocation.
@@ -887,7 +1161,8 @@ namespace {
         }
 
         DEBUG(if (DebugCandidateSelection) dbgs() << "BBV: candidate pair "
-                     << *I << " <-> " << *J << "\n");
+                     << *I << " <-> " << *J << " (cost savings: " <<
+                     CostSavings << ")\n");
 
         // If we have already found too many pairs, break here and this function
         // will be called again starting after the last instruction selected
@@ -915,6 +1190,7 @@ namespace {
                       std::multimap<Value *, Value *> &CandidatePairs,
                       std::vector<Value *> &PairableInsts,
                       std::multimap<ValuePair, ValuePair> &ConnectedPairs,
+                      DenseMap<VPPair, unsigned> &PairConnectionTypes,
                       ValuePair P) {
     StoreInst *SI, *SJ;
 
@@ -946,12 +1222,18 @@ namespace {
         VPIteratorPair JPairRange = CandidatePairs.equal_range(*J);
 
         // Look for <I, J>:
-        if (isSecondInIteratorPair<Value*>(*J, IPairRange))
-          ConnectedPairs.insert(VPPair(P, ValuePair(*I, *J)));
+        if (isSecondInIteratorPair<Value*>(*J, IPairRange)) {
+          VPPair VP(P, ValuePair(*I, *J));
+          ConnectedPairs.insert(VP);
+          PairConnectionTypes.insert(VPPairWithType(VP, PairConnectionDirect));
+        }
 
         // Look for <J, I>:
-        if (isSecondInIteratorPair<Value*>(*I, JPairRange))
-          ConnectedPairs.insert(VPPair(P, ValuePair(*J, *I)));
+        if (isSecondInIteratorPair<Value*>(*I, JPairRange)) {
+          VPPair VP(P, ValuePair(*J, *I));
+          ConnectedPairs.insert(VP);
+          PairConnectionTypes.insert(VPPairWithType(VP, PairConnectionSwap));
+        }
       }
 
       if (Config.SplatBreaksChain) continue;
@@ -962,8 +1244,11 @@ namespace {
             P.first == SJ->getPointerOperand())
           continue;
 
-        if (isSecondInIteratorPair<Value*>(*J, IPairRange))
-          ConnectedPairs.insert(VPPair(P, ValuePair(*I, *J)));
+        if (isSecondInIteratorPair<Value*>(*J, IPairRange)) {
+          VPPair VP(P, ValuePair(*I, *J));
+          ConnectedPairs.insert(VP);
+          PairConnectionTypes.insert(VPPairWithType(VP, PairConnectionSplat));
+        }
       }
     }
 
@@ -985,8 +1270,11 @@ namespace {
             P.second == SJ->getPointerOperand())
           continue;
 
-        if (isSecondInIteratorPair<Value*>(*J, IPairRange))
-          ConnectedPairs.insert(VPPair(P, ValuePair(*I, *J)));
+        if (isSecondInIteratorPair<Value*>(*J, IPairRange)) {
+          VPPair VP(P, ValuePair(*I, *J));
+          ConnectedPairs.insert(VP);
+          PairConnectionTypes.insert(VPPairWithType(VP, PairConnectionSplat));
+        }
       }
     }
   }
@@ -997,7 +1285,8 @@ namespace {
   void BBVectorize::computeConnectedPairs(
                       std::multimap<Value *, Value *> &CandidatePairs,
                       std::vector<Value *> &PairableInsts,
-                      std::multimap<ValuePair, ValuePair> &ConnectedPairs) {
+                      std::multimap<ValuePair, ValuePair> &ConnectedPairs,
+                      DenseMap<VPPair, unsigned> &PairConnectionTypes) {
 
     for (std::vector<Value *>::iterator PI = PairableInsts.begin(),
          PE = PairableInsts.end(); PI != PE; ++PI) {
@@ -1006,7 +1295,7 @@ namespace {
       for (std::multimap<Value *, Value *>::iterator P = choiceRange.first;
            P != choiceRange.second; ++P)
         computePairsConnectedTo(CandidatePairs, PairableInsts,
-                                ConnectedPairs, *P);
+                                ConnectedPairs, PairConnectionTypes, *P);
     }
 
     DEBUG(dbgs() << "BBV: found " << ConnectedPairs.size()
@@ -1196,7 +1485,7 @@ namespace {
       PrunedTree.insert(QTop.first);
 
       // Visit each child, pruning as necessary...
-      DenseMap<ValuePair, size_t> BestChildren;
+      SmallVector<ValuePairWithDepth, 8> BestChildren;
       VPPIteratorPair QTopRange = ConnectedPairs.equal_range(QTop.first);
       for (std::multimap<ValuePair, ValuePair>::iterator K = QTopRange.first;
            K != QTopRange.second; ++K) {
@@ -1228,7 +1517,7 @@ namespace {
         DenseSet<ValuePair> CurrentPairs;
 
         bool CanAdd = true;
-        for (DenseMap<ValuePair, size_t>::iterator C2
+        for (SmallVector<ValuePairWithDepth, 8>::iterator C2
               = BestChildren.begin(), E2 = BestChildren.end();
              C2 != E2; ++C2) {
           if (C2->first.first == C->first.first ||
@@ -1313,22 +1602,22 @@ namespace {
         // to an already-selected child. Check for this here, and if a
         // conflict is found, then remove the previously-selected child
         // before adding this one in its place.
-        for (DenseMap<ValuePair, size_t>::iterator C2
+        for (SmallVector<ValuePairWithDepth, 8>::iterator C2
               = BestChildren.begin(); C2 != BestChildren.end();) {
           if (C2->first.first == C->first.first ||
               C2->first.first == C->first.second ||
               C2->first.second == C->first.first ||
               C2->first.second == C->first.second ||
               pairsConflict(C2->first, C->first, PairableInstUsers))
-            BestChildren.erase(C2++);
+            C2 = BestChildren.erase(C2);
           else
             ++C2;
         }
 
-        BestChildren.insert(ValuePairWithDepth(C->first, C->second));
+        BestChildren.push_back(ValuePairWithDepth(C->first, C->second));
       }
 
-      for (DenseMap<ValuePair, size_t>::iterator C
+      for (SmallVector<ValuePairWithDepth, 8>::iterator C
             = BestChildren.begin(), E2 = BestChildren.end();
            C != E2; ++C) {
         size_t DepthF = getDepthFactor(C->first.first);
@@ -1341,13 +1630,17 @@ namespace {
   // pairs, given the choice of root pairs as an iterator range.
   void BBVectorize::findBestTreeFor(
                       std::multimap<Value *, Value *> &CandidatePairs,
+                      DenseMap<ValuePair, int> &CandidatePairCostSavings,
                       std::vector<Value *> &PairableInsts,
+                      DenseSet<ValuePair> &FixedOrderPairs,
+                      DenseMap<VPPair, unsigned> &PairConnectionTypes,
                       std::multimap<ValuePair, ValuePair> &ConnectedPairs,
+                      std::multimap<ValuePair, ValuePair> &ConnectedPairDeps,
                       DenseSet<ValuePair> &PairableInstUsers,
                       std::multimap<ValuePair, ValuePair> &PairableInstUserMap,
                       DenseMap<Value *, Value *> &ChosenPairs,
                       DenseSet<ValuePair> &BestTree, size_t &BestMaxDepth,
-                      size_t &BestEffSize, VPIteratorPair ChoiceRange,
+                      int &BestEffSize, VPIteratorPair ChoiceRange,
                       bool UseCycleCheck) {
     for (std::multimap<Value *, Value *>::iterator J = ChoiceRange.first;
          J != ChoiceRange.second; ++J) {
@@ -1397,17 +1690,289 @@ namespace {
                    PairableInstUsers, PairableInstUserMap, ChosenPairs, Tree,
                    PrunedTree, *J, UseCycleCheck);
 
-      size_t EffSize = 0;
-      for (DenseSet<ValuePair>::iterator S = PrunedTree.begin(),
-           E = PrunedTree.end(); S != E; ++S)
-        EffSize += getDepthFactor(S->first);
+      int EffSize = 0;
+      if (VTTI) {
+        DenseSet<Value *> PrunedTreeInstrs;
+        for (DenseSet<ValuePair>::iterator S = PrunedTree.begin(),
+             E = PrunedTree.end(); S != E; ++S) {
+          PrunedTreeInstrs.insert(S->first);
+          PrunedTreeInstrs.insert(S->second);
+        }
+
+        // The set of pairs that have already contributed to the total cost.
+        DenseSet<ValuePair> IncomingPairs;
+
+        // If the cost model were perfect, this might not be necessary; but we
+        // need to make sure that we don't get stuck vectorizing our own
+        // shuffle chains.
+        bool HasNontrivialInsts = false;
+
+        // The node weights represent the cost savings associated with
+        // fusing the pair of instructions.
+        for (DenseSet<ValuePair>::iterator S = PrunedTree.begin(),
+             E = PrunedTree.end(); S != E; ++S) {
+          if (!isa<ShuffleVectorInst>(S->first) &&
+              !isa<InsertElementInst>(S->first) &&
+              !isa<ExtractElementInst>(S->first))
+            HasNontrivialInsts = true;
+
+          bool FlipOrder = false;
+
+          if (getDepthFactor(S->first)) {
+            int ESContrib = CandidatePairCostSavings.find(*S)->second;
+            DEBUG(if (DebugPairSelection) dbgs() << "\tweight {"
+                   << *S->first << " <-> " << *S->second << "} = " <<
+                   ESContrib << "\n");
+            EffSize += ESContrib;
+          }
+
+          // The edge weights contribute in a negative sense: they represent
+          // the cost of shuffles.
+          VPPIteratorPair IP = ConnectedPairDeps.equal_range(*S);
+          if (IP.first != ConnectedPairDeps.end()) {
+            unsigned NumDepsDirect = 0, NumDepsSwap = 0;
+            for (std::multimap<ValuePair, ValuePair>::iterator Q = IP.first;
+                 Q != IP.second; ++Q) {
+              if (!PrunedTree.count(Q->second))
+                continue;
+              DenseMap<VPPair, unsigned>::iterator R =
+                PairConnectionTypes.find(VPPair(Q->second, Q->first));
+              assert(R != PairConnectionTypes.end() &&
+                     "Cannot find pair connection type");
+              if (R->second == PairConnectionDirect)
+                ++NumDepsDirect;
+              else if (R->second == PairConnectionSwap)
+                ++NumDepsSwap;
+            }
+
+            // If there are more swaps than direct connections, then
+            // the pair order will be flipped during fusion. So the real
+            // number of swaps is the minimum number.
+            FlipOrder = !FixedOrderPairs.count(*S) &&
+              ((NumDepsSwap > NumDepsDirect) ||
+                FixedOrderPairs.count(ValuePair(S->second, S->first)));
+
+            for (std::multimap<ValuePair, ValuePair>::iterator Q = IP.first;
+                 Q != IP.second; ++Q) {
+              if (!PrunedTree.count(Q->second))
+                continue;
+              DenseMap<VPPair, unsigned>::iterator R =
+                PairConnectionTypes.find(VPPair(Q->second, Q->first));
+              assert(R != PairConnectionTypes.end() &&
+                     "Cannot find pair connection type");
+              Type *Ty1 = Q->second.first->getType(),
+                   *Ty2 = Q->second.second->getType();
+              Type *VTy = getVecTypeForPair(Ty1, Ty2);
+              if ((R->second == PairConnectionDirect && FlipOrder) ||
+                  (R->second == PairConnectionSwap && !FlipOrder)  ||
+                  R->second == PairConnectionSplat) {
+                int ESContrib = (int) getInstrCost(Instruction::ShuffleVector,
+                                                   VTy, VTy);
+                DEBUG(if (DebugPairSelection) dbgs() << "\tcost {" <<
+                  *Q->second.first << " <-> " << *Q->second.second <<
+                    "} -> {" <<
+                  *S->first << " <-> " << *S->second << "} = " <<
+                   ESContrib << "\n");
+                EffSize -= ESContrib;
+              }
+            }
+          }
+
+          // Compute the cost of outgoing edges. We assume that edges outgoing
+          // to shuffles, inserts or extracts can be merged, and so contribute
+          // no additional cost.
+          if (!S->first->getType()->isVoidTy()) {
+            Type *Ty1 = S->first->getType(),
+                 *Ty2 = S->second->getType();
+            Type *VTy = getVecTypeForPair(Ty1, Ty2);
+
+            bool NeedsExtraction = false;
+            for (Value::use_iterator I = S->first->use_begin(),
+                 IE = S->first->use_end(); I != IE; ++I) {
+              if (ShuffleVectorInst *SI = dyn_cast<ShuffleVectorInst>(*I)) {
+                // Shuffle can be folded if it has no other input
+                if (isa<UndefValue>(SI->getOperand(1)))
+                  continue;
+              }
+              if (isa<ExtractElementInst>(*I))
+                continue;
+              if (PrunedTreeInstrs.count(*I))
+                continue;
+              NeedsExtraction = true;
+              break;
+            }
+
+            if (NeedsExtraction) {
+              int ESContrib;
+              if (Ty1->isVectorTy())
+                ESContrib = (int) getInstrCost(Instruction::ShuffleVector,
+                                               Ty1, VTy);
+              else
+                ESContrib = (int) VTTI->getVectorInstrCost(
+                                    Instruction::ExtractElement, VTy, 0);
+
+              DEBUG(if (DebugPairSelection) dbgs() << "\tcost {" <<
+                *S->first << "} = " << ESContrib << "\n");
+              EffSize -= ESContrib;
+            }
+
+            NeedsExtraction = false;
+            for (Value::use_iterator I = S->second->use_begin(),
+                 IE = S->second->use_end(); I != IE; ++I) {
+              if (ShuffleVectorInst *SI = dyn_cast<ShuffleVectorInst>(*I)) {
+                // Shuffle can be folded if it has no other input
+                if (isa<UndefValue>(SI->getOperand(1)))
+                  continue;
+              }
+              if (isa<ExtractElementInst>(*I))
+                continue;
+              if (PrunedTreeInstrs.count(*I))
+                continue;
+              NeedsExtraction = true;
+              break;
+            }
+
+            if (NeedsExtraction) {
+              int ESContrib;
+              if (Ty2->isVectorTy())
+                ESContrib = (int) getInstrCost(Instruction::ShuffleVector,
+                                               Ty2, VTy);
+              else
+                ESContrib = (int) VTTI->getVectorInstrCost(
+                                    Instruction::ExtractElement, VTy, 1);
+              DEBUG(if (DebugPairSelection) dbgs() << "\tcost {" <<
+                *S->second << "} = " << ESContrib << "\n");
+              EffSize -= ESContrib;
+            }
+          }
+
+          // Compute the cost of incoming edges.
+          if (!isa<LoadInst>(S->first) && !isa<StoreInst>(S->first)) {
+            Instruction *S1 = cast<Instruction>(S->first),
+                        *S2 = cast<Instruction>(S->second);
+            for (unsigned o = 0; o < S1->getNumOperands(); ++o) {
+              Value *O1 = S1->getOperand(o), *O2 = S2->getOperand(o);
+
+              // Combining constants into vector constants (or small vector
+              // constants into larger ones are assumed free).
+              if (isa<Constant>(O1) && isa<Constant>(O2))
+                continue;
+
+              if (FlipOrder)
+                std::swap(O1, O2);
+
+              ValuePair VP  = ValuePair(O1, O2);
+              ValuePair VPR = ValuePair(O2, O1);
+
+              // Internal edges are not handled here.
+              if (PrunedTree.count(VP) || PrunedTree.count(VPR))
+                continue;
+
+              Type *Ty1 = O1->getType(),
+                   *Ty2 = O2->getType();
+              Type *VTy = getVecTypeForPair(Ty1, Ty2);
+
+              // Combining vector operations of the same type is also assumed
+              // folded with other operations.
+              if (Ty1 == Ty2) {
+                // If both are insert elements, then both can be widened.
+                InsertElementInst *IEO1 = dyn_cast<InsertElementInst>(O1),
+                                  *IEO2 = dyn_cast<InsertElementInst>(O2);
+                if (IEO1 && IEO2 && isPureIEChain(IEO1) && isPureIEChain(IEO2))
+                  continue;
+                // If both are extract elements, and both have the same input
+                // type, then they can be replaced with a shuffle
+                ExtractElementInst *EIO1 = dyn_cast<ExtractElementInst>(O1),
+                                   *EIO2 = dyn_cast<ExtractElementInst>(O2);
+                if (EIO1 && EIO2 &&
+                    EIO1->getOperand(0)->getType() ==
+                      EIO2->getOperand(0)->getType())
+                  continue;
+                // If both are a shuffle with equal operand types and only two
+                // unqiue operands, then they can be replaced with a single
+                // shuffle
+                ShuffleVectorInst *SIO1 = dyn_cast<ShuffleVectorInst>(O1),
+                                  *SIO2 = dyn_cast<ShuffleVectorInst>(O2);
+                if (SIO1 && SIO2 &&
+                    SIO1->getOperand(0)->getType() ==
+                      SIO2->getOperand(0)->getType()) {
+                  SmallSet<Value *, 4> SIOps;
+                  SIOps.insert(SIO1->getOperand(0));
+                  SIOps.insert(SIO1->getOperand(1));
+                  SIOps.insert(SIO2->getOperand(0));
+                  SIOps.insert(SIO2->getOperand(1));
+                  if (SIOps.size() <= 2)
+                    continue;
+                }
+              }
+
+              int ESContrib;
+              // This pair has already been formed.
+              if (IncomingPairs.count(VP)) {
+                continue;
+              } else if (IncomingPairs.count(VPR)) {
+                ESContrib = (int) getInstrCost(Instruction::ShuffleVector,
+                                               VTy, VTy);
+              } else if (!Ty1->isVectorTy() && !Ty2->isVectorTy()) {
+                ESContrib = (int) VTTI->getVectorInstrCost(
+                                    Instruction::InsertElement, VTy, 0);
+                ESContrib += (int) VTTI->getVectorInstrCost(
+                                     Instruction::InsertElement, VTy, 1);
+              } else if (!Ty1->isVectorTy()) {
+                // O1 needs to be inserted into a vector of size O2, and then
+                // both need to be shuffled together.
+                ESContrib = (int) VTTI->getVectorInstrCost(
+                                    Instruction::InsertElement, Ty2, 0);
+                ESContrib += (int) getInstrCost(Instruction::ShuffleVector,
+                                                VTy, Ty2);
+              } else if (!Ty2->isVectorTy()) {
+                // O2 needs to be inserted into a vector of size O1, and then
+                // both need to be shuffled together.
+                ESContrib = (int) VTTI->getVectorInstrCost(
+                                    Instruction::InsertElement, Ty1, 0);
+                ESContrib += (int) getInstrCost(Instruction::ShuffleVector,
+                                                VTy, Ty1);
+              } else {
+                Type *TyBig = Ty1, *TySmall = Ty2;
+                if (Ty2->getVectorNumElements() > Ty1->getVectorNumElements())
+                  std::swap(TyBig, TySmall);
+
+                ESContrib = (int) getInstrCost(Instruction::ShuffleVector,
+                                               VTy, TyBig);
+                if (TyBig != TySmall)
+                  ESContrib += (int) getInstrCost(Instruction::ShuffleVector,
+                                                  TyBig, TySmall);
+              }
+
+              DEBUG(if (DebugPairSelection) dbgs() << "\tcost {"
+                     << *O1 << " <-> " << *O2 << "} = " <<
+                     ESContrib << "\n");
+              EffSize -= ESContrib;
+              IncomingPairs.insert(VP);
+            }
+          }
+        }
+
+        if (!HasNontrivialInsts) {
+          DEBUG(if (DebugPairSelection) dbgs() <<
+                "\tNo non-trivial instructions in tree;"
+                " override to zero effective size\n");
+          EffSize = 0;
+        }
+      } else {
+        for (DenseSet<ValuePair>::iterator S = PrunedTree.begin(),
+             E = PrunedTree.end(); S != E; ++S)
+          EffSize += (int) getDepthFactor(S->first);
+      }
 
       DEBUG(if (DebugPairSelection)
              dbgs() << "BBV: found pruned Tree for pair {"
              << *J->first << " <-> " << *J->second << "} of depth " <<
              MaxDepth << " and size " << PrunedTree.size() <<
             " (effective size: " << EffSize << ")\n");
-      if (MaxDepth >= Config.ReqChainDepth && EffSize > BestEffSize) {
+      if (((VTTI && !UseChainDepthWithTI) ||
+            MaxDepth >= Config.ReqChainDepth) &&
+          EffSize > 0 && EffSize > BestEffSize) {
         BestMaxDepth = MaxDepth;
         BestEffSize = EffSize;
         BestTree = PrunedTree;
@@ -1419,8 +1984,12 @@ namespace {
   // that will be fused into vector instructions.
   void BBVectorize::choosePairs(
                       std::multimap<Value *, Value *> &CandidatePairs,
+                      DenseMap<ValuePair, int> &CandidatePairCostSavings,
                       std::vector<Value *> &PairableInsts,
+                      DenseSet<ValuePair> &FixedOrderPairs,
+                      DenseMap<VPPair, unsigned> &PairConnectionTypes,
                       std::multimap<ValuePair, ValuePair> &ConnectedPairs,
+                      std::multimap<ValuePair, ValuePair> &ConnectedPairDeps,
                       DenseSet<ValuePair> &PairableInstUsers,
                       DenseMap<Value *, Value *>& ChosenPairs) {
     bool UseCycleCheck =
@@ -1435,9 +2004,12 @@ namespace {
       VPIteratorPair ChoiceRange = CandidatePairs.equal_range(*I);
 
       // The best pair to choose and its tree:
-      size_t BestMaxDepth = 0, BestEffSize = 0;
+      size_t BestMaxDepth = 0;
+      int BestEffSize = 0;
       DenseSet<ValuePair> BestTree;
-      findBestTreeFor(CandidatePairs, PairableInsts, ConnectedPairs,
+      findBestTreeFor(CandidatePairs, CandidatePairCostSavings,
+                      PairableInsts, FixedOrderPairs, PairConnectionTypes,
+                      ConnectedPairs, ConnectedPairDeps,
                       PairableInstUsers, PairableInstUserMap, ChosenPairs,
                       BestTree, BestMaxDepth, BestEffSize, ChoiceRange,
                       UseCycleCheck);
@@ -1490,24 +2062,19 @@ namespace {
   // Returns the value that is to be used as the pointer input to the vector
   // instruction that fuses I with J.
   Value *BBVectorize::getReplacementPointerInput(LLVMContext& Context,
-                     Instruction *I, Instruction *J, unsigned o,
-                     bool FlipMemInputs) {
+                     Instruction *I, Instruction *J, unsigned o) {
     Value *IPtr, *JPtr;
-    unsigned IAlignment, JAlignment;
+    unsigned IAlignment, JAlignment, IAddressSpace, JAddressSpace;
     int64_t OffsetInElmts;
 
-    // Note: the analysis might fail here, that is why FlipMemInputs has
+    // Note: the analysis might fail here, that is why the pair order has
     // been precomputed (OffsetInElmts must be unused here).
     (void) getPairPtrInfo(I, J, IPtr, JPtr, IAlignment, JAlignment,
-                          OffsetInElmts);
+                          IAddressSpace, JAddressSpace,
+                          OffsetInElmts, false);
 
     // The pointer value is taken to be the one with the lowest offset.
-    Value *VPtr;
-    if (!FlipMemInputs) {
-      VPtr = IPtr;
-    } else {
-      VPtr = JPtr;
-    }
+    Value *VPtr = IPtr;
 
     Type *ArgTypeI = cast<PointerType>(IPtr->getType())->getElementType();
     Type *ArgTypeJ = cast<PointerType>(JPtr->getType())->getElementType();
@@ -1515,7 +2082,7 @@ namespace {
     Type *VArgPtrType = PointerType::get(VArgType,
       cast<PointerType>(IPtr->getType())->getAddressSpace());
     return new BitCastInst(VPtr, VArgPtrType, getReplacementName(I, true, o),
-                        /* insert before */ FlipMemInputs ? J : I);
+                        /* insert before */ I);
   }
 
   void BBVectorize::fillNewShuffleMask(LLVMContext& Context, Instruction *J,
@@ -1585,23 +2152,12 @@ namespace {
                                   Instruction *J, unsigned o, Value *&LOp,
                                   unsigned numElemL,
                                   Type *ArgTypeL, Type *ArgTypeH,
-                                  unsigned IdxOff) {
+                                  bool IBeforeJ, unsigned IdxOff) {
     bool ExpandedIEChain = false;
     if (InsertElementInst *LIE = dyn_cast<InsertElementInst>(LOp)) {
       // If we have a pure insertelement chain, then this can be rewritten
       // into a chain that directly builds the larger type.
-      bool PureChain = true;
-      InsertElementInst *LIENext = LIE;
-      do {
-        if (!isa<UndefValue>(LIENext->getOperand(0)) &&
-            !isa<InsertElementInst>(LIENext->getOperand(0))) {
-          PureChain = false;
-          break;
-        }
-      } while ((LIENext =
-                 dyn_cast<InsertElementInst>(LIENext->getOperand(0))));
-
-      if (PureChain) {
+      if (isPureIEChain(LIE)) {
         SmallVector<Value *, 8> VectElemts(numElemL,
           UndefValue::get(ArgTypeL->getScalarType()));
         InsertElementInst *LIENext = LIE;
@@ -1619,8 +2175,9 @@ namespace {
           LIENext = InsertElementInst::Create(LIEPrev, VectElemts[i],
                              ConstantInt::get(Type::getInt32Ty(Context),
                                               i + IdxOff),
-                             getReplacementName(I, true, o, i+1));
-          LIENext->insertBefore(J);
+                             getReplacementName(IBeforeJ ? I : J,
+                                                true, o, i+1));
+          LIENext->insertBefore(IBeforeJ ? J : I);
           LIEPrev = LIENext;
         }
 
@@ -1635,7 +2192,7 @@ namespace {
   // Returns the value to be used as the specified operand of the vector
   // instruction that fuses I with J.
   Value *BBVectorize::getReplacementInput(LLVMContext& Context, Instruction *I,
-                     Instruction *J, unsigned o, bool FlipMemInputs) {
+                     Instruction *J, unsigned o, bool IBeforeJ) {
     Value *CV0 = ConstantInt::get(Type::getInt32Ty(Context), 0);
     Value *CV1 = ConstantInt::get(Type::getInt32Ty(Context), 1);
 
@@ -1646,12 +2203,6 @@ namespace {
 
     Instruction *L = I, *H = J;
     Type *ArgTypeL = ArgTypeI, *ArgTypeH = ArgTypeJ;
-    if (FlipMemInputs) {
-      L = J;
-      H = I;
-      ArgTypeL = ArgTypeJ;
-      ArgTypeH = ArgTypeI;
-    }
 
     unsigned numElemL;
     if (ArgTypeL->isVectorTy())
@@ -1804,8 +2355,9 @@ namespace {
           Instruction *S =
             new ShuffleVectorInst(I1, UndefValue::get(I1T),
                                   ConstantVector::get(Mask),
-                                  getReplacementName(I, true, o));
-          S->insertBefore(J);
+                                  getReplacementName(IBeforeJ ? I : J,
+                                                     true, o));
+          S->insertBefore(IBeforeJ ? J : I);
           return S;
         }
 
@@ -1826,8 +2378,9 @@ namespace {
           Instruction *NewI1 =
             new ShuffleVectorInst(I1, UndefValue::get(I1T),
                                   ConstantVector::get(Mask),
-                                  getReplacementName(I, true, o, 1));
-          NewI1->insertBefore(J);
+                                  getReplacementName(IBeforeJ ? I : J,
+                                                     true, o, 1));
+          NewI1->insertBefore(IBeforeJ ? J : I);
           I1 = NewI1;
           I1T = I2T;
           I1Elem = I2Elem;
@@ -1842,8 +2395,9 @@ namespace {
           Instruction *NewI2 =
             new ShuffleVectorInst(I2, UndefValue::get(I2T),
                                   ConstantVector::get(Mask),
-                                  getReplacementName(I, true, o, 1));
-          NewI2->insertBefore(J);
+                                  getReplacementName(IBeforeJ ? I : J,
+                                                     true, o, 1));
+          NewI2->insertBefore(IBeforeJ ? J : I);
           I2 = NewI2;
           I2T = I1T;
           I2Elem = I1Elem;
@@ -1863,8 +2417,8 @@ namespace {
 
         Instruction *NewOp =
           new ShuffleVectorInst(I1, I2, ConstantVector::get(Mask),
-                                getReplacementName(I, true, o));
-        NewOp->insertBefore(J);
+                                getReplacementName(IBeforeJ ? I : J, true, o));
+        NewOp->insertBefore(IBeforeJ ? J : I);
         return NewOp;
       }
     }
@@ -1872,17 +2426,17 @@ namespace {
     Type *ArgType = ArgTypeL;
     if (numElemL < numElemH) {
       if (numElemL == 1 && expandIEChain(Context, I, J, o, HOp, numElemH,
-                                         ArgTypeL, VArgType, 1)) {
+                                         ArgTypeL, VArgType, IBeforeJ, 1)) {
         // This is another short-circuit case: we're combining a scalar into
         // a vector that is formed by an IE chain. We've just expanded the IE
         // chain, now insert the scalar and we're done.
 
         Instruction *S = InsertElementInst::Create(HOp, LOp, CV0,
-                                               getReplacementName(I, true, o));
-        S->insertBefore(J);
+                           getReplacementName(IBeforeJ ? I : J, true, o));
+        S->insertBefore(IBeforeJ ? J : I);
         return S;
       } else if (!expandIEChain(Context, I, J, o, LOp, numElemL, ArgTypeL,
-                                ArgTypeH)) {
+                                ArgTypeH, IBeforeJ)) {
         // The two vector inputs to the shuffle must be the same length,
         // so extend the smaller vector to be the same length as the larger one.
         Instruction *NLOp;
@@ -1897,29 +2451,32 @@ namespace {
     
           NLOp = new ShuffleVectorInst(LOp, UndefValue::get(ArgTypeL),
                                        ConstantVector::get(Mask),
-                                       getReplacementName(I, true, o, 1));
+                                       getReplacementName(IBeforeJ ? I : J,
+                                                          true, o, 1));
         } else {
           NLOp = InsertElementInst::Create(UndefValue::get(ArgTypeH), LOp, CV0,
-                                           getReplacementName(I, true, o, 1));
+                                           getReplacementName(IBeforeJ ? I : J,
+                                                              true, o, 1));
         }
   
-        NLOp->insertBefore(J);
+        NLOp->insertBefore(IBeforeJ ? J : I);
         LOp = NLOp;
       }
 
       ArgType = ArgTypeH;
     } else if (numElemL > numElemH) {
       if (numElemH == 1 && expandIEChain(Context, I, J, o, LOp, numElemL,
-                                         ArgTypeH, VArgType)) {
+                                         ArgTypeH, VArgType, IBeforeJ)) {
         Instruction *S =
           InsertElementInst::Create(LOp, HOp, 
                                     ConstantInt::get(Type::getInt32Ty(Context),
                                                      numElemL),
-                                    getReplacementName(I, true, o));
-        S->insertBefore(J);
+                                    getReplacementName(IBeforeJ ? I : J,
+                                                       true, o));
+        S->insertBefore(IBeforeJ ? J : I);
         return S;
       } else if (!expandIEChain(Context, I, J, o, HOp, numElemH, ArgTypeH,
-                                ArgTypeL)) {
+                                ArgTypeL, IBeforeJ)) {
         Instruction *NHOp;
         if (numElemH > 1) {
           std::vector<Constant *> Mask(numElemL);
@@ -1931,13 +2488,15 @@ namespace {
     
           NHOp = new ShuffleVectorInst(HOp, UndefValue::get(ArgTypeH),
                                        ConstantVector::get(Mask),
-                                       getReplacementName(I, true, o, 1));
+                                       getReplacementName(IBeforeJ ? I : J,
+                                                          true, o, 1));
         } else {
           NHOp = InsertElementInst::Create(UndefValue::get(ArgTypeL), HOp, CV0,
-                                           getReplacementName(I, true, o, 1));
+                                           getReplacementName(IBeforeJ ? I : J,
+                                                              true, o, 1));
         }
   
-        NHOp->insertBefore(J);
+        NHOp->insertBefore(IBeforeJ ? J : I);
         HOp = NHOp;
       }
     }
@@ -1955,19 +2514,21 @@ namespace {
       }
 
       Instruction *BV = new ShuffleVectorInst(LOp, HOp,
-                                              ConstantVector::get(Mask),
-                                              getReplacementName(I, true, o));
-      BV->insertBefore(J);
+                          ConstantVector::get(Mask),
+                          getReplacementName(IBeforeJ ? I : J, true, o));
+      BV->insertBefore(IBeforeJ ? J : I);
       return BV;
     }
 
     Instruction *BV1 = InsertElementInst::Create(
                                           UndefValue::get(VArgType), LOp, CV0,
-                                          getReplacementName(I, true, o, 1));
-    BV1->insertBefore(I);
+                                          getReplacementName(IBeforeJ ? I : J,
+                                                             true, o, 1));
+    BV1->insertBefore(IBeforeJ ? J : I);
     Instruction *BV2 = InsertElementInst::Create(BV1, HOp, CV1,
-                                          getReplacementName(I, true, o, 2));
-    BV2->insertBefore(J);
+                                          getReplacementName(IBeforeJ ? I : J,
+                                                             true, o, 2));
+    BV2->insertBefore(IBeforeJ ? J : I);
     return BV2;
   }
 
@@ -1976,7 +2537,7 @@ namespace {
   void BBVectorize::getReplacementInputsForPair(LLVMContext& Context,
                      Instruction *I, Instruction *J,
                      SmallVector<Value *, 3> &ReplacedOperands,
-                     bool FlipMemInputs) {
+                     bool IBeforeJ) {
     unsigned NumOperands = I->getNumOperands();
 
     for (unsigned p = 0, o = NumOperands-1; p < NumOperands; ++p, --o) {
@@ -1985,8 +2546,7 @@ namespace {
 
       if (isa<LoadInst>(I) || (o == 1 && isa<StoreInst>(I))) {
         // This is the pointer for a load/store instruction.
-        ReplacedOperands[o] = getReplacementPointerInput(Context, I, J, o,
-                                FlipMemInputs);
+        ReplacedOperands[o] = getReplacementPointerInput(Context, I, J, o);
         continue;
       } else if (isa<CallInst>(I)) {
         Function *F = cast<CallInst>(I)->getCalledFunction();
@@ -2014,8 +2574,7 @@ namespace {
         continue;
       }
 
-      ReplacedOperands[o] =
-        getReplacementInput(Context, I, J, o, FlipMemInputs);
+      ReplacedOperands[o] = getReplacementInput(Context, I, J, o, IBeforeJ);
     }
   }
 
@@ -2026,8 +2585,7 @@ namespace {
   void BBVectorize::replaceOutputsOfPair(LLVMContext& Context, Instruction *I,
                      Instruction *J, Instruction *K,
                      Instruction *&InsertionPt,
-                     Instruction *&K1, Instruction *&K2,
-                     bool FlipMemInputs) {
+                     Instruction *&K1, Instruction *&K2) {
     if (isa<StoreInst>(I)) {
       AA->replaceWithNewValue(I, K);
       AA->replaceWithNewValue(J, K);
@@ -2057,13 +2615,11 @@ namespace {
         }
 
         K1 = new ShuffleVectorInst(K, UndefValue::get(VType),
-                                   ConstantVector::get(
-                                     FlipMemInputs ? Mask2 : Mask1),
+                                   ConstantVector::get( Mask1),
                                    getReplacementName(K, false, 1));
       } else {
         Value *CV0 = ConstantInt::get(Type::getInt32Ty(Context), 0);
-        Value *CV1 = ConstantInt::get(Type::getInt32Ty(Context), numElem-1);
-        K1 = ExtractElementInst::Create(K, FlipMemInputs ? CV1 : CV0,
+        K1 = ExtractElementInst::Create(K, CV0,
                                           getReplacementName(K, false, 1));
       }
 
@@ -2075,13 +2631,11 @@ namespace {
         }
 
         K2 = new ShuffleVectorInst(K, UndefValue::get(VType),
-                                   ConstantVector::get(
-                                     FlipMemInputs ? Mask1 : Mask2),
+                                   ConstantVector::get( Mask2),
                                    getReplacementName(K, false, 2));
       } else {
-        Value *CV0 = ConstantInt::get(Type::getInt32Ty(Context), 0);
         Value *CV1 = ConstantInt::get(Type::getInt32Ty(Context), numElem-1);
-        K2 = ExtractElementInst::Create(K, FlipMemInputs ? CV0 : CV1,
+        K2 = ExtractElementInst::Create(K, CV1,
                                           getReplacementName(K, false, 2));
       }
 
@@ -2181,36 +2735,6 @@ namespace {
     }
   }
 
-  // As with the aliasing information, SCEV can also change because of
-  // vectorization. This information is used to compute relative pointer
-  // offsets; the necessary information will be cached here prior to
-  // fusion.
-  void BBVectorize::collectPtrInfo(std::vector<Value *> &PairableInsts,
-                                   DenseMap<Value *, Value *> &ChosenPairs,
-                                   DenseSet<Value *> &LowPtrInsts) {
-    for (std::vector<Value *>::iterator PI = PairableInsts.begin(),
-      PIE = PairableInsts.end(); PI != PIE; ++PI) {
-      DenseMap<Value *, Value *>::iterator P = ChosenPairs.find(*PI);
-      if (P == ChosenPairs.end()) continue;
-
-      Instruction *I = cast<Instruction>(P->first);
-      Instruction *J = cast<Instruction>(P->second);
-
-      if (!isa<LoadInst>(I) && !isa<StoreInst>(I))
-        continue;
-
-      Value *IPtr, *JPtr;
-      unsigned IAlignment, JAlignment;
-      int64_t OffsetInElmts;
-      if (!getPairPtrInfo(I, J, IPtr, JPtr, IAlignment, JAlignment,
-                          OffsetInElmts) || abs64(OffsetInElmts) != 1)
-        llvm_unreachable("Pre-fusion pointer analysis failed");
-
-      Value *LowPI = (OffsetInElmts > 0) ? I : J;
-      LowPtrInsts.insert(LowPI);
-    }
-  }
-
   // When the first instruction in each pair is cloned, it will inherit its
   // parent's metadata. This metadata must be combined with that of the other
   // instruction in a safe way.
@@ -2244,27 +2768,27 @@ namespace {
   // second member).
   void BBVectorize::fuseChosenPairs(BasicBlock &BB,
                      std::vector<Value *> &PairableInsts,
-                     DenseMap<Value *, Value *> &ChosenPairs) {
+                     DenseMap<Value *, Value *> &ChosenPairs,
+                     DenseSet<ValuePair> &FixedOrderPairs,
+                     DenseMap<VPPair, unsigned> &PairConnectionTypes,
+                     std::multimap<ValuePair, ValuePair> &ConnectedPairs,
+                     std::multimap<ValuePair, ValuePair> &ConnectedPairDeps) {
     LLVMContext& Context = BB.getContext();
 
     // During the vectorization process, the order of the pairs to be fused
     // could be flipped. So we'll add each pair, flipped, into the ChosenPairs
     // list. After a pair is fused, the flipped pair is removed from the list.
-    std::vector<ValuePair> FlippedPairs;
-    FlippedPairs.reserve(ChosenPairs.size());
+    DenseSet<ValuePair> FlippedPairs;
     for (DenseMap<Value *, Value *>::iterator P = ChosenPairs.begin(),
          E = ChosenPairs.end(); P != E; ++P)
-      FlippedPairs.push_back(ValuePair(P->second, P->first));
-    for (std::vector<ValuePair>::iterator P = FlippedPairs.begin(),
+      FlippedPairs.insert(ValuePair(P->second, P->first));
+    for (DenseSet<ValuePair>::iterator P = FlippedPairs.begin(),
          E = FlippedPairs.end(); P != E; ++P)
       ChosenPairs.insert(*P);
 
     std::multimap<Value *, Value *> LoadMoveSet;
     collectLoadMoveSet(BB, PairableInsts, ChosenPairs, LoadMoveSet);
 
-    DenseSet<Value *> LowPtrInsts;
-    collectPtrInfo(PairableInsts, ChosenPairs, LowPtrInsts);
-
     DEBUG(dbgs() << "BBV: initial: \n" << BB << "\n");
 
     for (BasicBlock::iterator PI = BB.getFirstInsertionPt(); PI != BB.end();) {
@@ -2304,44 +2828,92 @@ namespace {
         continue;
       }
 
-      bool FlipMemInputs = false;
-      if (isa<LoadInst>(I) || isa<StoreInst>(I))
-        FlipMemInputs = (LowPtrInsts.find(I) == LowPtrInsts.end());
+      // If the pair must have the other order, then flip it.
+      bool FlipPairOrder = FixedOrderPairs.count(ValuePair(J, I));
+      if (!FlipPairOrder && !FixedOrderPairs.count(ValuePair(I, J))) {
+        // This pair does not have a fixed order, and so we might want to
+        // flip it if that will yield fewer shuffles. We count the number
+        // of dependencies connected via swaps, and those directly connected,
+        // and flip the order if the number of swaps is greater.
+        bool OrigOrder = true;
+        VPPIteratorPair IP = ConnectedPairDeps.equal_range(ValuePair(I, J));
+        if (IP.first == ConnectedPairDeps.end()) {
+          IP = ConnectedPairDeps.equal_range(ValuePair(J, I));
+          OrigOrder = false;
+        }
 
+        if (IP.first != ConnectedPairDeps.end()) {
+          unsigned NumDepsDirect = 0, NumDepsSwap = 0;
+          for (std::multimap<ValuePair, ValuePair>::iterator Q = IP.first;
+               Q != IP.second; ++Q) {
+            DenseMap<VPPair, unsigned>::iterator R =
+              PairConnectionTypes.find(VPPair(Q->second, Q->first));
+            assert(R != PairConnectionTypes.end() &&
+                   "Cannot find pair connection type");
+            if (R->second == PairConnectionDirect)
+              ++NumDepsDirect;
+            else if (R->second == PairConnectionSwap)
+              ++NumDepsSwap;
+          }
+
+          if (!OrigOrder)
+            std::swap(NumDepsDirect, NumDepsSwap);
+
+          if (NumDepsSwap > NumDepsDirect) {
+            FlipPairOrder = true;
+            DEBUG(dbgs() << "BBV: reordering pair: " << *I <<
+                            " <-> " << *J << "\n");
+          }
+        }
+      }
+
+      Instruction *L = I, *H = J;
+      if (FlipPairOrder)
+        std::swap(H, L);
+
+      // If the pair being fused uses the opposite order from that in the pair
+      // connection map, then we need to flip the types.
+      VPPIteratorPair IP = ConnectedPairs.equal_range(ValuePair(H, L));
+      for (std::multimap<ValuePair, ValuePair>::iterator Q = IP.first;
+           Q != IP.second; ++Q) {
+        DenseMap<VPPair, unsigned>::iterator R = PairConnectionTypes.find(*Q);
+        assert(R != PairConnectionTypes.end() &&
+               "Cannot find pair connection type");
+        if (R->second == PairConnectionDirect)
+          R->second = PairConnectionSwap;
+        else if (R->second == PairConnectionSwap)
+          R->second = PairConnectionDirect;
+      }
+
+      bool LBeforeH = !FlipPairOrder;
       unsigned NumOperands = I->getNumOperands();
       SmallVector<Value *, 3> ReplacedOperands(NumOperands);
-      getReplacementInputsForPair(Context, I, J, ReplacedOperands,
-        FlipMemInputs);
+      getReplacementInputsForPair(Context, L, H, ReplacedOperands,
+                                  LBeforeH);
 
       // Make a copy of the original operation, change its type to the vector
       // type and replace its operands with the vector operands.
-      Instruction *K = I->clone();
-      if (I->hasName()) K->takeName(I);
+      Instruction *K = L->clone();
+      if (L->hasName())
+        K->takeName(L);
+      else if (H->hasName())
+        K->takeName(H);
 
       if (!isa<StoreInst>(K))
-        K->mutateType(getVecTypeForPair(I->getType(), J->getType()));
+        K->mutateType(getVecTypeForPair(L->getType(), H->getType()));
 
-      combineMetadata(K, J);
+      combineMetadata(K, H);
+      K->intersectOptionalDataWith(H);
 
       for (unsigned o = 0; o < NumOperands; ++o)
         K->setOperand(o, ReplacedOperands[o]);
 
-      // If we've flipped the memory inputs, make sure that we take the correct
-      // alignment.
-      if (FlipMemInputs) {
-        if (isa<StoreInst>(K))
-          cast<StoreInst>(K)->setAlignment(cast<StoreInst>(J)->getAlignment());
-        else
-          cast<LoadInst>(K)->setAlignment(cast<LoadInst>(J)->getAlignment());
-      }
-
       K->insertAfter(J);
 
       // Instruction insertion point:
       Instruction *InsertionPt = K;
       Instruction *K1 = 0, *K2 = 0;
-      replaceOutputsOfPair(Context, I, J, K, InsertionPt, K1, K2,
-        FlipMemInputs);
+      replaceOutputsOfPair(Context, L, H, K, InsertionPt, K1, K2);
 
       // The use tree of the first original instruction must be moved to after
       // the location of the second instruction. The entire use tree of the
@@ -2351,10 +2923,10 @@ namespace {
       moveUsesOfIAfterJ(BB, LoadMoveSet, InsertionPt, I, J);
 
       if (!isa<StoreInst>(I)) {
-        I->replaceAllUsesWith(K1);
-        J->replaceAllUsesWith(K2);
-        AA->replaceWithNewValue(I, K1);
-        AA->replaceWithNewValue(J, K2);
+        L->replaceAllUsesWith(K1);
+        H->replaceAllUsesWith(K2);
+        AA->replaceWithNewValue(L, K1);
+        AA->replaceWithNewValue(H, K2);
       }
 
       // Instructions that may read from memory may be in the load move set.
@@ -2387,6 +2959,9 @@ namespace {
       SE->forgetValue(J);
       I->eraseFromParent();
       J->eraseFromParent();
+
+      DEBUG(if (PrintAfterEveryPair) dbgs() << "BBV: block is now: \n" <<
+                                               BB << "\n");
     }
 
     DEBUG(dbgs() << "BBV: final: \n" << BB << "\n");
@@ -2397,6 +2972,7 @@ char BBVectorize::ID = 0;
 static const char bb_vectorize_name[] = "Basic-Block Vectorization";
 INITIALIZE_PASS_BEGIN(BBVectorize, BBV_NAME, bb_vectorize_name, false, false)
 INITIALIZE_AG_DEPENDENCY(AliasAnalysis)
+INITIALIZE_PASS_DEPENDENCY(DominatorTree)
 INITIALIZE_PASS_DEPENDENCY(ScalarEvolution)
 INITIALIZE_PASS_END(BBVectorize, BBV_NAME, bb_vectorize_name, false, false)
 
diff --git a/lib/Transforms/Vectorize/CMakeLists.txt b/lib/Transforms/Vectorize/CMakeLists.txt
index 06cf1e4..e64034a 100644
--- a/lib/Transforms/Vectorize/CMakeLists.txt
+++ b/lib/Transforms/Vectorize/CMakeLists.txt
@@ -1,6 +1,7 @@
 add_llvm_library(LLVMVectorize
   BBVectorize.cpp
   Vectorize.cpp
+  LoopVectorize.cpp
   )
 
 add_dependencies(LLVMVectorize intrinsics_gen)
diff --git a/lib/Transforms/Vectorize/LoopVectorize.cpp b/lib/Transforms/Vectorize/LoopVectorize.cpp
new file mode 100644
index 0000000..a7ef248
--- /dev/null
+++ b/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -0,0 +1,1941 @@
+//===- LoopVectorize.cpp - A Loop Vectorizer ------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This is the LLVM loop vectorizer. This pass modifies 'vectorizable' loops
+// and generates target-independent LLVM-IR. Legalization of the IR is done
+// in the codegen. However, the vectorizes uses (will use) the codegen
+// interfaces to generate IR that is likely to result in an optimal binary.
+//
+// The loop vectorizer combines consecutive loop iteration into a single
+// 'wide' iteration. After this transformation the index is incremented
+// by the SIMD vector width, and not by one.
+//
+// This pass has three parts:
+// 1. The main loop pass that drives the different parts.
+// 2. LoopVectorizationLegality - A unit that checks for the legality
+//    of the vectorization.
+// 3. SingleBlockLoopVectorizer - A unit that performs the actual
+//    widening of instructions.
+// 4. LoopVectorizationCostModel - A unit that checks for the profitability
+//    of vectorization. It decides on the optimal vector width, which
+//    can be one, if vectorization is not profitable.
+//===----------------------------------------------------------------------===//
+//
+// The reduction-variable vectorization is based on the paper:
+//  D. Nuzman and R. Henderson. Multi-platform Auto-vectorization.
+//
+// Variable uniformity checks are inspired by:
+// Karrenberg, R. and Hack, S. Whole Function Vectorization.
+//
+// Other ideas/concepts are from:
+//  A. Zaks and D. Nuzman. Autovectorization in GCC-two years later.
+//
+//===----------------------------------------------------------------------===//
+#define LV_NAME "loop-vectorize"
+#define DEBUG_TYPE LV_NAME
+#include "llvm/Constants.h"
+#include "llvm/DerivedTypes.h"
+#include "llvm/Instructions.h"
+#include "llvm/LLVMContext.h"
+#include "llvm/Pass.h"
+#include "llvm/Analysis/LoopPass.h"
+#include "llvm/Value.h"
+#include "llvm/Function.h"
+#include "llvm/Analysis/Verifier.h"
+#include "llvm/Module.h"
+#include "llvm/Type.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/Analysis/AliasSetTracker.h"
+#include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/Analysis/Dominators.h"
+#include "llvm/Analysis/ScalarEvolutionExpressions.h"
+#include "llvm/Analysis/ScalarEvolutionExpander.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/ValueTracking.h"
+#include "llvm/Transforms/Scalar.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/TargetTransformInfo.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/DataLayout.h"
+#include "llvm/Transforms/Utils/Local.h"
+#include <algorithm>
+using namespace llvm;
+
+static cl::opt<unsigned>
+VectorizationFactor("force-vector-width", cl::init(0), cl::Hidden,
+          cl::desc("Set the default vectorization width. Zero is autoselect."));
+
+/// We don't vectorize loops with a known constant trip count below this number.
+const unsigned TinyTripCountThreshold = 16;
+
+/// When performing a runtime memory check, do not check more than this
+/// number of pointers. Notice that the check is quadratic!
+const unsigned RuntimeMemoryCheckThreshold = 2;
+
+namespace {
+
+// Forward declarations.
+class LoopVectorizationLegality;
+class LoopVectorizationCostModel;
+
+/// SingleBlockLoopVectorizer vectorizes loops which contain only one basic
+/// block to a specified vectorization factor (VF).
+/// This class performs the widening of scalars into vectors, or multiple
+/// scalars. This class also implements the following features:
+/// * It inserts an epilogue loop for handling loops that don't have iteration
+///   counts that are known to be a multiple of the vectorization factor.
+/// * It handles the code generation for reduction variables.
+/// * Scalarization (implementation using scalars) of un-vectorizable
+///   instructions.
+/// SingleBlockLoopVectorizer does not perform any vectorization-legality
+/// checks, and relies on the caller to check for the different legality
+/// aspects. The SingleBlockLoopVectorizer relies on the
+/// LoopVectorizationLegality class to provide information about the induction
+/// and reduction variables that were found to a given vectorization factor.
+class SingleBlockLoopVectorizer {
+public:
+  /// Ctor.
+  SingleBlockLoopVectorizer(Loop *Orig, ScalarEvolution *Se, LoopInfo *Li,
+                            DominatorTree *dt, LPPassManager *Lpm,
+                            unsigned VecWidth):
+  OrigLoop(Orig), SE(Se), LI(Li), DT(dt), LPM(Lpm), VF(VecWidth),
+  Builder(Se->getContext()), Induction(0), OldInduction(0) { }
+
+  // Perform the actual loop widening (vectorization).
+  void vectorize(LoopVectorizationLegality *Legal) {
+    ///Create a new empty loop. Unlink the old loop and connect the new one.
+    createEmptyLoop(Legal);
+    /// Widen each instruction in the old loop to a new one in the new loop.
+    /// Use the Legality module to find the induction and reduction variables.
+    vectorizeLoop(Legal);
+    // Register the new loop and update the analysis passes.
+    updateAnalysis();
+ }
+
+private:
+  /// Create an empty loop, based on the loop ranges of the old loop.
+  void createEmptyLoop(LoopVectorizationLegality *Legal);
+  /// Copy and widen the instructions from the old loop.
+  void vectorizeLoop(LoopVectorizationLegality *Legal);
+  /// Insert the new loop to the loop hierarchy and pass manager
+  /// and update the analysis passes.
+  void updateAnalysis();
+
+  /// This instruction is un-vectorizable. Implement it as a sequence
+  /// of scalars.
+  void scalarizeInstruction(Instruction *Instr);
+
+  /// Create a broadcast instruction. This method generates a broadcast
+  /// instruction (shuffle) for loop invariant values and for the induction
+  /// value. If this is the induction variable then we extend it to N, N+1, ...
+  /// this is needed because each iteration in the loop corresponds to a SIMD
+  /// element.
+  Value *getBroadcastInstrs(Value *V);
+
+  /// This is a helper function used by getBroadcastInstrs. It adds 0, 1, 2 ..
+  /// for each element in the vector. Starting from zero.
+  Value *getConsecutiveVector(Value* Val);
+
+  /// When we go over instructions in the basic block we rely on previous
+  /// values within the current basic block or on loop invariant values.
+  /// When we widen (vectorize) values we place them in the map. If the values
+  /// are not within the map, they have to be loop invariant, so we simply
+  /// broadcast them into a vector.
+  Value *getVectorValue(Value *V);
+
+  /// Get a uniform vector of constant integers. We use this to get
+  /// vectors of ones and zeros for the reduction code.
+  Constant* getUniformVector(unsigned Val, Type* ScalarTy);
+
+  typedef DenseMap<Value*, Value*> ValueMap;
+
+  /// The original loop.
+  Loop *OrigLoop;
+  // Scev analysis to use.
+  ScalarEvolution *SE;
+  // Loop Info.
+  LoopInfo *LI;
+  // Dominator Tree.
+  DominatorTree *DT;
+  // Loop Pass Manager;
+  LPPassManager *LPM;
+  // The vectorization factor to use.
+  unsigned VF;
+
+  // The builder that we use
+  IRBuilder<> Builder;
+
+  // --- Vectorization state ---
+
+  /// The vector-loop preheader.
+  BasicBlock *LoopVectorPreHeader;
+  /// The scalar-loop preheader.
+  BasicBlock *LoopScalarPreHeader;
+  /// Middle Block between the vector and the scalar.
+  BasicBlock *LoopMiddleBlock;
+  ///The ExitBlock of the scalar loop.
+  BasicBlock *LoopExitBlock;
+  ///The vector loop body.
+  BasicBlock *LoopVectorBody;
+  ///The scalar loop body.
+  BasicBlock *LoopScalarBody;
+  ///The first bypass block.
+  BasicBlock *LoopBypassBlock;
+
+  /// The new Induction variable which was added to the new block.
+  PHINode *Induction;
+  /// The induction variable of the old basic block.
+  PHINode *OldInduction;
+  // Maps scalars to widened vectors.
+  ValueMap WidenMap;
+};
+
+/// LoopVectorizationLegality checks if it is legal to vectorize a loop, and
+/// to what vectorization factor.
+/// This class does not look at the profitability of vectorization, only the
+/// legality. This class has two main kinds of checks:
+/// * Memory checks - The code in canVectorizeMemory checks if vectorization
+///   will change the order of memory accesses in a way that will change the
+///   correctness of the program.
+/// * Scalars checks - The code in canVectorizeBlock checks for a number
+///   of different conditions, such as the availability of a single induction
+///   variable, that all types are supported and vectorize-able, etc.
+/// This code reflects the capabilities of SingleBlockLoopVectorizer.
+/// This class is also used by SingleBlockLoopVectorizer for identifying
+/// induction variable and the different reduction variables.
+class LoopVectorizationLegality {
+public:
+  LoopVectorizationLegality(Loop *Lp, ScalarEvolution *Se, DataLayout *Dl):
+  TheLoop(Lp), SE(Se), DL(Dl), Induction(0) { }
+
+  /// This represents the kinds of reductions that we support.
+  enum ReductionKind {
+    NoReduction, /// Not a reduction.
+    IntegerAdd,  /// Sum of numbers.
+    IntegerMult, /// Product of numbers.
+    IntegerOr,   /// Bitwise or logical OR of numbers.
+    IntegerAnd,  /// Bitwise or logical AND of numbers.
+    IntegerXor   /// Bitwise or logical XOR of numbers.
+  };
+
+  /// This POD struct holds information about reduction variables.
+  struct ReductionDescriptor {
+    // Default C'tor
+    ReductionDescriptor():
+    StartValue(0), LoopExitInstr(0), Kind(NoReduction) {}
+
+    // C'tor.
+    ReductionDescriptor(Value *Start, Instruction *Exit, ReductionKind K):
+    StartValue(Start), LoopExitInstr(Exit), Kind(K) {}
+
+    // The starting value of the reduction.
+    // It does not have to be zero!
+    Value *StartValue;
+    // The instruction who's value is used outside the loop.
+    Instruction *LoopExitInstr;
+    // The kind of the reduction.
+    ReductionKind Kind;
+  };
+
+  // This POD struct holds information about the memory runtime legality
+  // check that a group of pointers do not overlap.
+  struct RuntimePointerCheck {
+    /// This flag indicates if we need to add the runtime check.
+    bool Need;
+    /// Holds the pointers that we need to check.
+    SmallVector<Value*, 2> Pointers;
+  };
+
+  /// ReductionList contains the reduction descriptors for all
+  /// of the reductions that were found in the loop.
+  typedef DenseMap<PHINode*, ReductionDescriptor> ReductionList;
+
+  /// Returns true if it is legal to vectorize this loop.
+  /// This does not mean that it is profitable to vectorize this
+  /// loop, only that it is legal to do so.
+  bool canVectorize();
+
+  /// Returns the Induction variable.
+  PHINode *getInduction() {return Induction;}
+
+  /// Returns the reduction variables found in the loop.
+  ReductionList *getReductionVars() { return &Reductions; }
+
+  /// Check if the pointer returned by this GEP is consecutive
+  /// when the index is vectorized. This happens when the last
+  /// index of the GEP is consecutive, like the induction variable.
+  /// This check allows us to vectorize A[idx] into a wide load/store.
+  bool isConsecutiveGep(Value *Ptr);
+
+  /// Returns true if the value V is uniform within the loop.
+  bool isUniform(Value *V);
+
+  /// Returns true if this instruction will remain scalar after vectorization.
+  bool isUniformAfterVectorization(Instruction* I) {return Uniforms.count(I);}
+
+  /// Returns the information that we collected about runtime memory check.
+  RuntimePointerCheck *getRuntimePointerCheck() {return &PtrRtCheck; }
+private:
+  /// Check if a single basic block loop is vectorizable.
+  /// At this point we know that this is a loop with a constant trip count
+  /// and we only need to check individual instructions.
+  bool canVectorizeBlock(BasicBlock &BB);
+
+  /// When we vectorize loops we may change the order in which
+  /// we read and write from memory. This method checks if it is
+  /// legal to vectorize the code, considering only memory constrains.
+  /// Returns true if BB is vectorizable
+  bool canVectorizeMemory(BasicBlock &BB);
+
+  /// Returns True, if 'Phi' is the kind of reduction variable for type
+  /// 'Kind'. If this is a reduction variable, it adds it to ReductionList.
+  bool AddReductionVar(PHINode *Phi, ReductionKind Kind);
+  /// Returns true if the instruction I can be a reduction variable of type
+  /// 'Kind'.
+  bool isReductionInstr(Instruction *I, ReductionKind Kind);
+  /// Returns True, if 'Phi' is an induction variable.
+  bool isInductionVariable(PHINode *Phi);
+  /// Return true if can compute the address bounds of Ptr within the loop.
+  bool hasComputableBounds(Value *Ptr);
+
+  /// The loop that we evaluate.
+  Loop *TheLoop;
+  /// Scev analysis.
+  ScalarEvolution *SE;
+  /// DataLayout analysis.
+  DataLayout *DL;
+
+  //  ---  vectorization state --- //
+
+  /// Holds the induction variable.
+  PHINode *Induction;
+  /// Holds the reduction variables.
+  ReductionList Reductions;
+  /// Allowed outside users. This holds the reduction
+  /// vars which can be accessed from outside the loop.
+  SmallPtrSet<Value*, 4> AllowedExit;
+  /// This set holds the variables which are known to be uniform after
+  /// vectorization.
+  SmallPtrSet<Instruction*, 4> Uniforms;
+  /// We need to check that all of the pointers in this list are disjoint
+  /// at runtime.
+  RuntimePointerCheck PtrRtCheck;
+};
+
+/// LoopVectorizationCostModel - estimates the expected speedups due to
+/// vectorization.
+/// In many cases vectorization is not profitable. This can happen because
+/// of a number of reasons. In this class we mainly attempt to predict
+/// the expected speedup/slowdowns due to the supported instruction set.
+/// We use the VectorTargetTransformInfo to query the different backends
+/// for the cost of different operations.
+class LoopVectorizationCostModel {
+public:
+  /// C'tor.
+  LoopVectorizationCostModel(Loop *Lp, ScalarEvolution *Se,
+                             LoopVectorizationLegality *Leg,
+                             const VectorTargetTransformInfo *Vtti):
+  TheLoop(Lp), SE(Se), Legal(Leg), VTTI(Vtti) { }
+
+  /// Returns the most profitable vectorization factor for the loop that is
+  /// smaller or equal to the VF argument. This method checks every power
+  /// of two up to VF.
+  unsigned findBestVectorizationFactor(unsigned VF = 8);
+
+private:
+  /// Returns the expected execution cost. The unit of the cost does
+  /// not matter because we use the 'cost' units to compare different
+  /// vector widths. The cost that is returned is *not* normalized by
+  /// the factor width.
+  unsigned expectedCost(unsigned VF);
+
+  /// Returns the execution time cost of an instruction for a given vector
+  /// width. Vector width of one means scalar.
+  unsigned getInstructionCost(Instruction *I, unsigned VF);
+
+  /// A helper function for converting Scalar types to vector types.
+  /// If the incoming type is void, we return void. If the VF is 1, we return
+  /// the scalar type.
+  static Type* ToVectorTy(Type *Scalar, unsigned VF);
+
+  /// The loop that we evaluate.
+  Loop *TheLoop;
+  /// Scev analysis.
+  ScalarEvolution *SE;
+
+  /// Vectorization legality.
+  LoopVectorizationLegality *Legal;
+  /// Vector target information.
+  const VectorTargetTransformInfo *VTTI;
+};
+
+struct LoopVectorize : public LoopPass {
+  static char ID; // Pass identification, replacement for typeid
+
+  LoopVectorize() : LoopPass(ID) {
+    initializeLoopVectorizePass(*PassRegistry::getPassRegistry());
+  }
+
+  ScalarEvolution *SE;
+  DataLayout *DL;
+  LoopInfo *LI;
+  TargetTransformInfo *TTI;
+  DominatorTree *DT;
+
+  virtual bool runOnLoop(Loop *L, LPPassManager &LPM) {
+    // We only vectorize innermost loops.
+    if (!L->empty())
+      return false;
+
+    SE = &getAnalysis<ScalarEvolution>();
+    DL = getAnalysisIfAvailable<DataLayout>();
+    LI = &getAnalysis<LoopInfo>();
+    TTI = getAnalysisIfAvailable<TargetTransformInfo>();
+    DT = &getAnalysis<DominatorTree>();
+
+    DEBUG(dbgs() << "LV: Checking a loop in \"" <<
+          L->getHeader()->getParent()->getName() << "\"\n");
+
+    // Check if it is legal to vectorize the loop.
+    LoopVectorizationLegality LVL(L, SE, DL);
+    if (!LVL.canVectorize()) {
+      DEBUG(dbgs() << "LV: Not vectorizing.\n");
+      return false;
+    }
+
+    // Select the preffered vectorization factor.
+    unsigned VF = 1;
+    if (VectorizationFactor == 0) {
+      const VectorTargetTransformInfo *VTTI = 0;
+      if (TTI)
+        VTTI = TTI->getVectorTargetTransformInfo();
+      // Use the cost model.
+      LoopVectorizationCostModel CM(L, SE, &LVL, VTTI);
+      VF = CM.findBestVectorizationFactor();
+
+      if (VF == 1) {
+        DEBUG(dbgs() << "LV: Vectorization is possible but not beneficial.\n");
+        return false;
+      }
+
+    } else {
+      // Use the user command flag.
+      VF = VectorizationFactor;
+    }
+
+    DEBUG(dbgs() << "LV: Found a vectorizable loop ("<< VF << ") in "<<
+          L->getHeader()->getParent()->getParent()->getModuleIdentifier()<<
+          "\n");
+
+    // If we decided that it is *legal* to vectorizer the loop then do it.
+    SingleBlockLoopVectorizer LB(L, SE, LI, DT, &LPM, VF);
+    LB.vectorize(&LVL);
+
+    DEBUG(verifyFunction(*L->getHeader()->getParent()));
+    return true;
+  }
+
+  virtual void getAnalysisUsage(AnalysisUsage &AU) const {
+    LoopPass::getAnalysisUsage(AU);
+    AU.addRequiredID(LoopSimplifyID);
+    AU.addRequiredID(LCSSAID);
+    AU.addRequired<LoopInfo>();
+    AU.addRequired<ScalarEvolution>();
+    AU.addRequired<DominatorTree>();
+    AU.addPreserved<LoopInfo>();
+    AU.addPreserved<DominatorTree>();
+  }
+
+};
+
+Value *SingleBlockLoopVectorizer::getBroadcastInstrs(Value *V) {
+  // Instructions that access the old induction variable
+  // actually want to get the new one.
+  if (V == OldInduction)
+    V = Induction;
+  // Create the types.
+  LLVMContext &C = V->getContext();
+  Type *VTy = VectorType::get(V->getType(), VF);
+  Type *I32 = IntegerType::getInt32Ty(C);
+  Constant *Zero = ConstantInt::get(I32, 0);
+  Value *Zeros = ConstantAggregateZero::get(VectorType::get(I32, VF));
+  Value *UndefVal = UndefValue::get(VTy);
+  // Insert the value into a new vector.
+  Value *SingleElem = Builder.CreateInsertElement(UndefVal, V, Zero);
+  // Broadcast the scalar into all locations in the vector.
+  Value *Shuf = Builder.CreateShuffleVector(SingleElem, UndefVal, Zeros,
+                                             "broadcast");
+  // We are accessing the induction variable. Make sure to promote the
+  // index for each consecutive SIMD lane. This adds 0,1,2 ... to all lanes.
+  if (V == Induction)
+    return getConsecutiveVector(Shuf);
+  return Shuf;
+}
+
+Value *SingleBlockLoopVectorizer::getConsecutiveVector(Value* Val) {
+  assert(Val->getType()->isVectorTy() && "Must be a vector");
+  assert(Val->getType()->getScalarType()->isIntegerTy() &&
+         "Elem must be an integer");
+  // Create the types.
+  Type *ITy = Val->getType()->getScalarType();
+  VectorType *Ty = cast<VectorType>(Val->getType());
+  unsigned VLen = Ty->getNumElements();
+  SmallVector<Constant*, 8> Indices;
+
+  // Create a vector of consecutive numbers from zero to VF.
+  for (unsigned i = 0; i < VLen; ++i)
+    Indices.push_back(ConstantInt::get(ITy, i));
+
+  // Add the consecutive indices to the vector value.
+  Constant *Cv = ConstantVector::get(Indices);
+  assert(Cv->getType() == Val->getType() && "Invalid consecutive vec");
+  return Builder.CreateAdd(Val, Cv, "induction");
+}
+
+bool LoopVectorizationLegality::isConsecutiveGep(Value *Ptr) {
+  GetElementPtrInst *Gep = dyn_cast_or_null<GetElementPtrInst>(Ptr);
+  if (!Gep)
+    return false;
+
+  unsigned NumOperands = Gep->getNumOperands();
+  Value *LastIndex = Gep->getOperand(NumOperands - 1);
+
+  // Check that all of the gep indices are uniform except for the last.
+  for (unsigned i = 0; i < NumOperands - 1; ++i)
+    if (!SE->isLoopInvariant(SE->getSCEV(Gep->getOperand(i)), TheLoop))
+      return false;
+
+  // We can emit wide load/stores only of the last index is the induction
+  // variable.
+  const SCEV *Last = SE->getSCEV(LastIndex);
+  if (const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Last)) {
+    const SCEV *Step = AR->getStepRecurrence(*SE);
+
+    // The memory is consecutive because the last index is consecutive
+    // and all other indices are loop invariant.
+    if (Step->isOne())
+      return true;
+  }
+
+  return false;
+}
+
+bool LoopVectorizationLegality::isUniform(Value *V) {
+  return (SE->isLoopInvariant(SE->getSCEV(V), TheLoop));
+}
+
+Value *SingleBlockLoopVectorizer::getVectorValue(Value *V) {
+  assert(!V->getType()->isVectorTy() && "Can't widen a vector");
+  // If we saved a vectorized copy of V, use it.
+  Value *&MapEntry = WidenMap[V];
+  if (MapEntry)
+    return MapEntry;
+
+  // Broadcast V and save the value for future uses.
+  Value *B = getBroadcastInstrs(V);
+  MapEntry = B;
+  return B;
+}
+
+Constant*
+SingleBlockLoopVectorizer::getUniformVector(unsigned Val, Type* ScalarTy) {
+  SmallVector<Constant*, 8> Indices;
+  // Create a vector of consecutive numbers from zero to VF.
+  for (unsigned i = 0; i < VF; ++i)
+    Indices.push_back(ConstantInt::get(ScalarTy, Val, true));
+
+  // Add the consecutive indices to the vector value.
+  return ConstantVector::get(Indices);
+}
+
+void SingleBlockLoopVectorizer::scalarizeInstruction(Instruction *Instr) {
+  assert(!Instr->getType()->isAggregateType() && "Can't handle vectors");
+  // Holds vector parameters or scalars, in case of uniform vals.
+  SmallVector<Value*, 8> Params;
+
+  // Find all of the vectorized parameters.
+  for (unsigned op = 0, e = Instr->getNumOperands(); op != e; ++op) {
+    Value *SrcOp = Instr->getOperand(op);
+
+    // If we are accessing the old induction variable, use the new one.
+    if (SrcOp == OldInduction) {
+      Params.push_back(getBroadcastInstrs(Induction));
+      continue;
+    }
+
+    // Try using previously calculated values.
+    Instruction *SrcInst = dyn_cast<Instruction>(SrcOp);
+
+    // If the src is an instruction that appeared earlier in the basic block
+    // then it should already be vectorized.
+    if (SrcInst && SrcInst->getParent() == Instr->getParent()) {
+      assert(WidenMap.count(SrcInst) && "Source operand is unavailable");
+      // The parameter is a vector value from earlier.
+      Params.push_back(WidenMap[SrcInst]);
+    } else {
+      // The parameter is a scalar from outside the loop. Maybe even a constant.
+      Params.push_back(SrcOp);
+    }
+  }
+
+  assert(Params.size() == Instr->getNumOperands() &&
+         "Invalid number of operands");
+
+  // Does this instruction return a value ?
+  bool IsVoidRetTy = Instr->getType()->isVoidTy();
+  Value *VecResults = 0;
+
+  // If we have a return value, create an empty vector. We place the scalarized
+  // instructions in this vector.
+  if (!IsVoidRetTy)
+    VecResults = UndefValue::get(VectorType::get(Instr->getType(), VF));
+
+  // For each scalar that we create:
+  for (unsigned i = 0; i < VF; ++i) {
+    Instruction *Cloned = Instr->clone();
+    if (!IsVoidRetTy)
+      Cloned->setName(Instr->getName() + ".cloned");
+    // Replace the operands of the cloned instrucions with extracted scalars.
+    for (unsigned op = 0, e = Instr->getNumOperands(); op != e; ++op) {
+      Value *Op = Params[op];
+      // Param is a vector. Need to extract the right lane.
+      if (Op->getType()->isVectorTy())
+        Op = Builder.CreateExtractElement(Op, Builder.getInt32(i));
+      Cloned->setOperand(op, Op);
+    }
+
+    // Place the cloned scalar in the new loop.
+    Builder.Insert(Cloned);
+
+    // If the original scalar returns a value we need to place it in a vector
+    // so that future users will be able to use it.
+    if (!IsVoidRetTy)
+      VecResults = Builder.CreateInsertElement(VecResults, Cloned,
+                                               Builder.getInt32(i));
+  }
+
+  if (!IsVoidRetTy)
+    WidenMap[Instr] = VecResults;
+}
+
+void
+SingleBlockLoopVectorizer::createEmptyLoop(LoopVectorizationLegality *Legal) {
+  /*
+   In this function we generate a new loop. The new loop will contain
+   the vectorized instructions while the old loop will continue to run the
+   scalar remainder.
+
+    [ ] <-- vector loop bypass.
+  /  |
+ /   v
+|   [ ]     <-- vector pre header.
+|    |
+|    v
+|   [  ] \
+|   [  ]_|   <-- vector loop.
+|    |
+ \   v
+   >[ ]   <--- middle-block.
+  /  |
+ /   v
+|   [ ]     <--- new preheader.
+|    |
+|    v
+|   [ ] \
+|   [ ]_|   <-- old scalar loop to handle remainder.
+ \   |
+  \  v
+   >[ ]     <-- exit block.
+   ...
+   */
+
+  OldInduction = Legal->getInduction();
+  assert(OldInduction && "We must have a single phi node.");
+  Type *IdxTy = OldInduction->getType();
+
+  // Find the loop boundaries.
+  const SCEV *ExitCount = SE->getExitCount(OrigLoop, OrigLoop->getHeader());
+  assert(ExitCount != SE->getCouldNotCompute() && "Invalid loop count");
+
+  // Get the total trip count from the count by adding 1.
+  ExitCount = SE->getAddExpr(ExitCount,
+                             SE->getConstant(ExitCount->getType(), 1));
+  // We may need to extend the index in case there is a type mismatch.
+  // We know that the count starts at zero and does not overflow.
+  // We are using Zext because it should be less expensive.
+  if (ExitCount->getType() != IdxTy)
+    ExitCount = SE->getZeroExtendExpr(ExitCount, IdxTy);
+
+  // This is the original scalar-loop preheader.
+  BasicBlock *BypassBlock = OrigLoop->getLoopPreheader();
+  BasicBlock *ExitBlock = OrigLoop->getExitBlock();
+  assert(ExitBlock && "Must have an exit block");
+
+  // The loop index does not have to start at Zero. It starts with this value.
+  Value *StartIdx = OldInduction->getIncomingValueForBlock(BypassBlock);
+
+  assert(OrigLoop->getNumBlocks() == 1 && "Invalid loop");
+  assert(BypassBlock && "Invalid loop structure");
+
+  BasicBlock *VectorPH =
+      BypassBlock->splitBasicBlock(BypassBlock->getTerminator(), "vector.ph");
+  BasicBlock *VecBody = VectorPH->splitBasicBlock(VectorPH->getTerminator(),
+                                                 "vector.body");
+
+  BasicBlock *MiddleBlock = VecBody->splitBasicBlock(VecBody->getTerminator(),
+                                                  "middle.block");
+  BasicBlock *ScalarPH =
+    MiddleBlock->splitBasicBlock(MiddleBlock->getTerminator(),
+                                 "scalar.preheader");
+  // Find the induction variable.
+  BasicBlock *OldBasicBlock = OrigLoop->getHeader();
+
+  // Use this IR builder to create the loop instructions (Phi, Br, Cmp)
+  // inside the loop.
+  Builder.SetInsertPoint(VecBody->getFirstInsertionPt());
+
+  // Generate the induction variable.
+  Induction = Builder.CreatePHI(IdxTy, 2, "index");
+  Constant *Step = ConstantInt::get(IdxTy, VF);
+
+  // Expand the trip count and place the new instructions in the preheader.
+  // Notice that the pre-header does not change, only the loop body.
+  SCEVExpander Exp(*SE, "induction");
+  Instruction *Loc = BypassBlock->getTerminator();
+
+  // Count holds the overall loop count (N).
+  Value *Count = Exp.expandCodeFor(ExitCount, Induction->getType(), Loc);
+
+  // Add the start index to the loop count to get the new end index.
+  Value *IdxEnd = BinaryOperator::CreateAdd(Count, StartIdx, "end.idx", Loc);
+
+  // Now we need to generate the expression for N - (N % VF), which is
+  // the part that the vectorized body will execute.
+  Constant *CIVF = ConstantInt::get(IdxTy, VF);
+  Value *R = BinaryOperator::CreateURem(Count, CIVF, "n.mod.vf", Loc);
+  Value *CountRoundDown = BinaryOperator::CreateSub(Count, R, "n.vec", Loc);
+  Value *IdxEndRoundDown = BinaryOperator::CreateAdd(CountRoundDown, StartIdx,
+                                                     "end.idx.rnd.down", Loc);
+
+  // Now, compare the new count to zero. If it is zero, jump to the scalar part.
+  Value *Cmp = CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_EQ,
+                               IdxEndRoundDown,
+                               StartIdx,
+                               "cmp.zero", Loc);
+
+  LoopVectorizationLegality::RuntimePointerCheck *PtrRtCheck =
+    Legal->getRuntimePointerCheck();
+  Value *MemoryRuntimeCheck = 0;
+  if (PtrRtCheck->Need) {
+    unsigned NumPointers = PtrRtCheck->Pointers.size();
+    SmallVector<Value* , 2> Starts;
+    SmallVector<Value* , 2> Ends;
+
+    // Use this type for pointer arithmetic.
+    Type* PtrArithTy = PtrRtCheck->Pointers[0]->getType();
+
+    for (unsigned i=0; i < NumPointers; ++i) {
+      Value *Ptr = PtrRtCheck->Pointers[i];
+      const SCEV *Sc = SE->getSCEV(Ptr);
+
+      if (SE->isLoopInvariant(Sc, OrigLoop)) {
+        DEBUG(dbgs() << "LV1: Adding RT check for a loop invariant ptr:" <<
+              *Ptr <<"\n");
+        Starts.push_back(Ptr);
+        Ends.push_back(Ptr);
+      } else {
+        DEBUG(dbgs() << "LV: Adding RT check for range:" << *Ptr <<"\n");
+        const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(Sc);
+        Value *Start = Exp.expandCodeFor(AR->getStart(), PtrArithTy, Loc);
+        const SCEV *Ex = SE->getExitCount(OrigLoop, OrigLoop->getHeader());
+        const SCEV *ScEnd = AR->evaluateAtIteration(Ex, *SE);
+        assert(!isa<SCEVCouldNotCompute>(ScEnd) && "Invalid scev range.");
+        Value *End = Exp.expandCodeFor(ScEnd, PtrArithTy, Loc);
+        Starts.push_back(Start);
+        Ends.push_back(End);
+      }
+    }
+
+    for (unsigned i=0; i < NumPointers; ++i) {
+      for (unsigned j=i+1; j < NumPointers; ++j) {
+        Value *Cmp0 = CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_ULE,
+                                      Starts[0], Ends[1], "bound0", Loc);
+        Value *Cmp1 = CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_ULE,
+                                      Starts[1], Ends[0], "bound1", Loc);
+        Value *IsConflict = BinaryOperator::Create(Instruction::And, Cmp0, Cmp1,
+                                                    "found.conflict", Loc);
+        if (MemoryRuntimeCheck) {
+          MemoryRuntimeCheck = BinaryOperator::Create(Instruction::Or,
+                                                      MemoryRuntimeCheck,
+                                                      IsConflict,
+                                                      "conflict.rdx", Loc);
+        } else {
+          MemoryRuntimeCheck = IsConflict;
+        }
+      }
+    }
+  }// end of need-runtime-check code.
+
+  // If we are using memory runtime checks, include them in.
+  if (MemoryRuntimeCheck) {
+    Cmp = BinaryOperator::Create(Instruction::Or, Cmp, MemoryRuntimeCheck,
+                                 "CntOrMem", Loc);
+  }
+
+  BranchInst::Create(MiddleBlock, VectorPH, Cmp, Loc);
+  // Remove the old terminator.
+  Loc->eraseFromParent();
+
+  // We are going to resume the execution of the scalar loop.
+  // This PHI decides on what number to start. If we come from the
+  // vector loop then we need to start with the end index minus the
+  // index modulo VF. If we come from a bypass edge then we need to start
+  // from the real start.
+  PHINode* ResumeIndex = PHINode::Create(IdxTy, 2, "resume.idx",
+                                         MiddleBlock->getTerminator());
+  ResumeIndex->addIncoming(StartIdx, BypassBlock);
+  ResumeIndex->addIncoming(IdxEndRoundDown, VecBody);
+
+  // Add a check in the middle block to see if we have completed
+  // all of the iterations in the first vector loop.
+  // If (N - N%VF) == N, then we *don't* need to run the remainder.
+  Value *CmpN = CmpInst::Create(Instruction::ICmp, CmpInst::ICMP_EQ, IdxEnd,
+                                ResumeIndex, "cmp.n",
+                                MiddleBlock->getTerminator());
+
+  BranchInst::Create(ExitBlock, ScalarPH, CmpN, MiddleBlock->getTerminator());
+  // Remove the old terminator.
+  MiddleBlock->getTerminator()->eraseFromParent();
+
+  // Create i+1 and fill the PHINode.
+  Value *NextIdx = Builder.CreateAdd(Induction, Step, "index.next");
+  Induction->addIncoming(StartIdx, VectorPH);
+  Induction->addIncoming(NextIdx, VecBody);
+  // Create the compare.
+  Value *ICmp = Builder.CreateICmpEQ(NextIdx, IdxEndRoundDown);
+  Builder.CreateCondBr(ICmp, MiddleBlock, VecBody);
+
+  // Now we have two terminators. Remove the old one from the block.
+  VecBody->getTerminator()->eraseFromParent();
+
+  // Fix the scalar body iteration count.
+  unsigned BlockIdx = OldInduction->getBasicBlockIndex(ScalarPH);
+  OldInduction->setIncomingValue(BlockIdx, ResumeIndex);
+
+  // Get ready to start creating new instructions into the vectorized body.
+  Builder.SetInsertPoint(VecBody->getFirstInsertionPt());
+
+  // Register the new loop.
+  Loop* Lp = new Loop();
+  LPM->insertLoop(Lp, OrigLoop->getParentLoop());
+
+  Lp->addBasicBlockToLoop(VecBody, LI->getBase());
+
+  Loop *ParentLoop = OrigLoop->getParentLoop();
+  if (ParentLoop) {
+    ParentLoop->addBasicBlockToLoop(ScalarPH, LI->getBase());
+    ParentLoop->addBasicBlockToLoop(VectorPH, LI->getBase());
+    ParentLoop->addBasicBlockToLoop(MiddleBlock, LI->getBase());
+  }
+
+  // Save the state.
+  LoopVectorPreHeader = VectorPH;
+  LoopScalarPreHeader = ScalarPH;
+  LoopMiddleBlock = MiddleBlock;
+  LoopExitBlock = ExitBlock;
+  LoopVectorBody = VecBody;
+  LoopScalarBody = OldBasicBlock;
+  LoopBypassBlock = BypassBlock;
+}
+
+/// This function returns the identity element (or neutral element) for
+/// the operation K.
+static unsigned
+getReductionIdentity(LoopVectorizationLegality::ReductionKind K) {
+  switch (K) {
+  case LoopVectorizationLegality::IntegerXor:
+  case LoopVectorizationLegality::IntegerAdd:
+  case LoopVectorizationLegality::IntegerOr:
+    // Adding, Xoring, Oring zero to a number does not change it.
+    return 0;
+  case LoopVectorizationLegality::IntegerMult:
+    // Multiplying a number by 1 does not change it.
+    return 1;
+  case LoopVectorizationLegality::IntegerAnd:
+    // AND-ing a number with an all-1 value does not change it.
+    return -1;
+  default:
+    llvm_unreachable("Unknown reduction kind");
+  }
+}
+
+void
+SingleBlockLoopVectorizer::vectorizeLoop(LoopVectorizationLegality *Legal) {
+  //===------------------------------------------------===//
+  //
+  // Notice: any optimization or new instruction that go
+  // into the code below should be also be implemented in
+  // the cost-model.
+  //
+  //===------------------------------------------------===//
+  typedef SmallVector<PHINode*, 4> PhiVector;
+  BasicBlock &BB = *OrigLoop->getHeader();
+  Constant *Zero = ConstantInt::get(
+    IntegerType::getInt32Ty(BB.getContext()), 0);
+
+  // In order to support reduction variables we need to be able to vectorize
+  // Phi nodes. Phi nodes have cycles, so we need to vectorize them in two
+  // steages. First, we create a new vector PHI node with no incoming edges.
+  // We use this value when we vectorize all of the instructions that use the
+  // PHI. Next, after all of the instructions in the block are complete we
+  // add the new incoming edges to the PHI. At this point all of the
+  // instructions in the basic block are vectorized, so we can use them to
+  // construct the PHI.
+  PhiVector PHIsToFix;
+
+  // For each instruction in the old loop.
+  for (BasicBlock::iterator it = BB.begin(), e = BB.end(); it != e; ++it) {
+    Instruction *Inst = it;
+
+    switch (Inst->getOpcode()) {
+      case Instruction::Br:
+        // Nothing to do for PHIs and BR, since we already took care of the
+        // loop control flow instructions.
+        continue;
+      case Instruction::PHI:{
+        PHINode* P = cast<PHINode>(Inst);
+        // Special handling for the induction var.
+        if (OldInduction == Inst)
+          continue;
+        // This is phase one of vectorizing PHIs.
+        // This has to be a reduction variable.
+        assert(Legal->getReductionVars()->count(P) && "Not a Reduction");
+        Type *VecTy = VectorType::get(Inst->getType(), VF);
+        WidenMap[Inst] = Builder.CreatePHI(VecTy, 2, "vec.phi");
+        PHIsToFix.push_back(P);
+        continue;
+      }
+      case Instruction::Add:
+      case Instruction::FAdd:
+      case Instruction::Sub:
+      case Instruction::FSub:
+      case Instruction::Mul:
+      case Instruction::FMul:
+      case Instruction::UDiv:
+      case Instruction::SDiv:
+      case Instruction::FDiv:
+      case Instruction::URem:
+      case Instruction::SRem:
+      case Instruction::FRem:
+      case Instruction::Shl:
+      case Instruction::LShr:
+      case Instruction::AShr:
+      case Instruction::And:
+      case Instruction::Or:
+      case Instruction::Xor: {
+        // Just widen binops.
+        BinaryOperator *BinOp = dyn_cast<BinaryOperator>(Inst);
+        Value *A = getVectorValue(Inst->getOperand(0));
+        Value *B = getVectorValue(Inst->getOperand(1));
+
+        // Use this vector value for all users of the original instruction.
+        Value *V = Builder.CreateBinOp(BinOp->getOpcode(), A, B);
+        WidenMap[Inst] = V;
+
+        // Update the NSW, NUW and Exact flags.
+        BinaryOperator *VecOp = cast<BinaryOperator>(V);
+        if (isa<OverflowingBinaryOperator>(BinOp)) {
+          VecOp->setHasNoSignedWrap(BinOp->hasNoSignedWrap());
+          VecOp->setHasNoUnsignedWrap(BinOp->hasNoUnsignedWrap());
+        }
+        if (isa<PossiblyExactOperator>(VecOp))
+          VecOp->setIsExact(BinOp->isExact());
+        break;
+      }
+      case Instruction::Select: {
+        // Widen selects.
+        // If the selector is loop invariant we can create a select
+        // instruction with a scalar condition. Otherwise, use vector-select.
+        Value *Cond = Inst->getOperand(0);
+        bool InvariantCond = SE->isLoopInvariant(SE->getSCEV(Cond), OrigLoop);
+
+        // The condition can be loop invariant  but still defined inside the
+        // loop. This means that we can't just use the original 'cond' value.
+        // We have to take the 'vectorized' value and pick the first lane.
+        // Instcombine will make this a no-op.
+        Cond = getVectorValue(Cond);
+        if (InvariantCond)
+          Cond = Builder.CreateExtractElement(Cond, Builder.getInt32(0));
+
+        Value *Op0 = getVectorValue(Inst->getOperand(1));
+        Value *Op1 = getVectorValue(Inst->getOperand(2));
+        WidenMap[Inst] = Builder.CreateSelect(Cond, Op0, Op1);
+        break;
+      }
+
+      case Instruction::ICmp:
+      case Instruction::FCmp: {
+        // Widen compares. Generate vector compares.
+        bool FCmp = (Inst->getOpcode() == Instruction::FCmp);
+        CmpInst *Cmp = dyn_cast<CmpInst>(Inst);
+        Value *A = getVectorValue(Inst->getOperand(0));
+        Value *B = getVectorValue(Inst->getOperand(1));
+        if (FCmp)
+          WidenMap[Inst] = Builder.CreateFCmp(Cmp->getPredicate(), A, B);
+        else
+          WidenMap[Inst] = Builder.CreateICmp(Cmp->getPredicate(), A, B);
+        break;
+      }
+
+      case Instruction::Store: {
+        // Attempt to issue a wide store.
+        StoreInst *SI = dyn_cast<StoreInst>(Inst);
+        Type *StTy = VectorType::get(SI->getValueOperand()->getType(), VF);
+        Value *Ptr = SI->getPointerOperand();
+        unsigned Alignment = SI->getAlignment();
+
+        assert(!Legal->isUniform(Ptr) &&
+               "We do not allow storing to uniform addresses");
+
+        GetElementPtrInst *Gep = dyn_cast<GetElementPtrInst>(Ptr);
+
+        // This store does not use GEPs.
+        if (!Legal->isConsecutiveGep(Gep)) {
+          scalarizeInstruction(Inst);
+          break;
+        }
+
+        // The last index does not have to be the induction. It can be
+        // consecutive and be a function of the index. For example A[I+1];
+        unsigned NumOperands = Gep->getNumOperands();
+        Value *LastIndex = getVectorValue(Gep->getOperand(NumOperands - 1));
+        LastIndex = Builder.CreateExtractElement(LastIndex, Zero);
+
+        // Create the new GEP with the new induction variable.
+        GetElementPtrInst *Gep2 = cast<GetElementPtrInst>(Gep->clone());
+        Gep2->setOperand(NumOperands - 1, LastIndex);
+        Ptr = Builder.Insert(Gep2);
+        Ptr = Builder.CreateBitCast(Ptr, StTy->getPointerTo());
+        Value *Val = getVectorValue(SI->getValueOperand());
+        Builder.CreateStore(Val, Ptr)->setAlignment(Alignment);
+        break;
+      }
+      case Instruction::Load: {
+        // Attempt to issue a wide load.
+        LoadInst *LI = dyn_cast<LoadInst>(Inst);
+        Type *RetTy = VectorType::get(LI->getType(), VF);
+        Value *Ptr = LI->getPointerOperand();
+        unsigned Alignment = LI->getAlignment();
+        GetElementPtrInst *Gep = dyn_cast<GetElementPtrInst>(Ptr);
+
+        // If we don't have a gep, or that the pointer is loop invariant,
+        // scalarize the load.
+        if (!Gep || Legal->isUniform(Gep) || !Legal->isConsecutiveGep(Gep)) {
+          scalarizeInstruction(Inst);
+          break;
+        }
+
+        // The last index does not have to be the induction. It can be
+        // consecutive and be a function of the index. For example A[I+1];
+        unsigned NumOperands = Gep->getNumOperands();
+        Value *LastIndex = getVectorValue(Gep->getOperand(NumOperands -1));
+        LastIndex = Builder.CreateExtractElement(LastIndex, Zero);
+
+        // Create the new GEP with the new induction variable.
+        GetElementPtrInst *Gep2 = cast<GetElementPtrInst>(Gep->clone());
+        Gep2->setOperand(NumOperands - 1, LastIndex);
+        Ptr = Builder.Insert(Gep2);
+        Ptr = Builder.CreateBitCast(Ptr, RetTy->getPointerTo());
+        LI = Builder.CreateLoad(Ptr);
+        LI->setAlignment(Alignment);
+        // Use this vector value for all users of the load.
+        WidenMap[Inst] = LI;
+        break;
+      }
+      case Instruction::ZExt:
+      case Instruction::SExt:
+      case Instruction::FPToUI:
+      case Instruction::FPToSI:
+      case Instruction::FPExt:
+      case Instruction::PtrToInt:
+      case Instruction::IntToPtr:
+      case Instruction::SIToFP:
+      case Instruction::UIToFP:
+      case Instruction::Trunc:
+      case Instruction::FPTrunc:
+      case Instruction::BitCast: {
+        /// Vectorize bitcasts.
+        CastInst *CI = dyn_cast<CastInst>(Inst);
+        Value *A = getVectorValue(Inst->getOperand(0));
+        Type *DestTy = VectorType::get(CI->getType()->getScalarType(), VF);
+        WidenMap[Inst] = Builder.CreateCast(CI->getOpcode(), A, DestTy);
+        break;
+      }
+
+      default:
+        /// All other instructions are unsupported. Scalarize them.
+        scalarizeInstruction(Inst);
+        break;
+    }// end of switch.
+  }// end of for_each instr.
+
+  // At this point every instruction in the original loop is widended to
+  // a vector form. We are almost done. Now, we need to fix the PHI nodes
+  // that we vectorized. The PHI nodes are currently empty because we did
+  // not want to introduce cycles. Notice that the remaining PHI nodes
+  // that we need to fix are reduction variables.
+
+  // Create the 'reduced' values for each of the induction vars.
+  // The reduced values are the vector values that we scalarize and combine
+  // after the loop is finished.
+  for (PhiVector::iterator it = PHIsToFix.begin(), e = PHIsToFix.end();
+       it != e; ++it) {
+    PHINode *RdxPhi = *it;
+    PHINode *VecRdxPhi = dyn_cast<PHINode>(WidenMap[RdxPhi]);
+    assert(RdxPhi && "Unable to recover vectorized PHI");
+
+    // Find the reduction variable descriptor.
+    assert(Legal->getReductionVars()->count(RdxPhi) &&
+           "Unable to find the reduction variable");
+    LoopVectorizationLegality::ReductionDescriptor RdxDesc =
+      (*Legal->getReductionVars())[RdxPhi];
+
+    // We need to generate a reduction vector from the incoming scalar.
+    // To do so, we need to generate the 'identity' vector and overide
+    // one of the elements with the incoming scalar reduction. We need
+    // to do it in the vector-loop preheader.
+    Builder.SetInsertPoint(LoopBypassBlock->getTerminator());
+
+    // This is the vector-clone of the value that leaves the loop.
+    Value *VectorExit = getVectorValue(RdxDesc.LoopExitInstr);
+    Type *VecTy = VectorExit->getType();
+
+    // Find the reduction identity variable. Zero for addition, or, xor,
+    // one for multiplication, -1 for And.
+    Constant *Identity = getUniformVector(getReductionIdentity(RdxDesc.Kind),
+                                          VecTy->getScalarType());
+
+    // This vector is the Identity vector where the first element is the
+    // incoming scalar reduction.
+    Value *VectorStart = Builder.CreateInsertElement(Identity,
+                                                    RdxDesc.StartValue, Zero);
+
+
+    // Fix the vector-loop phi.
+    // We created the induction variable so we know that the
+    // preheader is the first entry.
+    BasicBlock *VecPreheader = Induction->getIncomingBlock(0);
+
+    // Reductions do not have to start at zero. They can start with
+    // any loop invariant values.
+    VecRdxPhi->addIncoming(VectorStart, VecPreheader);
+    unsigned SelfEdgeIdx = (RdxPhi)->getBasicBlockIndex(LoopScalarBody);
+    Value *Val = getVectorValue(RdxPhi->getIncomingValue(SelfEdgeIdx));
+    VecRdxPhi->addIncoming(Val, LoopVectorBody);
+
+    // Before each round, move the insertion point right between
+    // the PHIs and the values we are going to write.
+    // This allows us to write both PHINodes and the extractelement
+    // instructions.
+    Builder.SetInsertPoint(LoopMiddleBlock->getFirstInsertionPt());
+
+    // This PHINode contains the vectorized reduction variable, or
+    // the initial value vector, if we bypass the vector loop.
+    PHINode *NewPhi = Builder.CreatePHI(VecTy, 2, "rdx.vec.exit.phi");
+    NewPhi->addIncoming(VectorStart, LoopBypassBlock);
+    NewPhi->addIncoming(getVectorValue(RdxDesc.LoopExitInstr), LoopVectorBody);
+
+    // Extract the first scalar.
+    Value *Scalar0 =
+      Builder.CreateExtractElement(NewPhi, Builder.getInt32(0));
+    // Extract and reduce the remaining vector elements.
+    for (unsigned i=1; i < VF; ++i) {
+      Value *Scalar1 =
+        Builder.CreateExtractElement(NewPhi, Builder.getInt32(i));
+      switch (RdxDesc.Kind) {
+        case LoopVectorizationLegality::IntegerAdd:
+          Scalar0 = Builder.CreateAdd(Scalar0, Scalar1);
+          break;
+        case LoopVectorizationLegality::IntegerMult:
+          Scalar0 = Builder.CreateMul(Scalar0, Scalar1);
+          break;
+        case LoopVectorizationLegality::IntegerOr:
+          Scalar0 = Builder.CreateOr(Scalar0, Scalar1);
+          break;
+        case LoopVectorizationLegality::IntegerAnd:
+          Scalar0 = Builder.CreateAnd(Scalar0, Scalar1);
+          break;
+        case LoopVectorizationLegality::IntegerXor:
+          Scalar0 = Builder.CreateXor(Scalar0, Scalar1);
+          break;
+        default:
+          llvm_unreachable("Unknown reduction operation");
+      }
+    }
+
+    // Now, we need to fix the users of the reduction variable
+    // inside and outside of the scalar remainder loop.
+    // We know that the loop is in LCSSA form. We need to update the
+    // PHI nodes in the exit blocks.
+    for (BasicBlock::iterator LEI = LoopExitBlock->begin(),
+         LEE = LoopExitBlock->end(); LEI != LEE; ++LEI) {
+      PHINode *LCSSAPhi = dyn_cast<PHINode>(LEI);
+      if (!LCSSAPhi) continue;
+
+      // All PHINodes need to have a single entry edge, or two if
+      // we already fixed them.
+      assert(LCSSAPhi->getNumIncomingValues() < 3 && "Invalid LCSSA PHI");
+
+      // We found our reduction value exit-PHI. Update it with the
+      // incoming bypass edge.
+      if (LCSSAPhi->getIncomingValue(0) == RdxDesc.LoopExitInstr) {
+        // Add an edge coming from the bypass.
+        LCSSAPhi->addIncoming(Scalar0, LoopMiddleBlock);
+        break;
+      }
+    }// end of the LCSSA phi scan.
+
+    // Fix the scalar loop reduction variable with the incoming reduction sum
+    // from the vector body and from the backedge value.
+    int IncomingEdgeBlockIdx = (RdxPhi)->getBasicBlockIndex(LoopScalarBody);
+    int SelfEdgeBlockIdx = (IncomingEdgeBlockIdx ? 0 : 1); // The other block.
+    (RdxPhi)->setIncomingValue(SelfEdgeBlockIdx, Scalar0);
+    (RdxPhi)->setIncomingValue(IncomingEdgeBlockIdx, RdxDesc.LoopExitInstr);
+  }// end of for each redux variable.
+}
+
+void SingleBlockLoopVectorizer::updateAnalysis() {
+  // The original basic block.
+  SE->forgetLoop(OrigLoop);
+
+  // Update the dominator tree information.
+  assert(DT->properlyDominates(LoopBypassBlock, LoopExitBlock) &&
+         "Entry does not dominate exit.");
+
+  DT->addNewBlock(LoopVectorPreHeader, LoopBypassBlock);
+  DT->addNewBlock(LoopVectorBody, LoopVectorPreHeader);
+  DT->addNewBlock(LoopMiddleBlock, LoopBypassBlock);
+  DT->addNewBlock(LoopScalarPreHeader, LoopMiddleBlock);
+  DT->changeImmediateDominator(LoopScalarBody, LoopScalarPreHeader);
+  DT->changeImmediateDominator(LoopExitBlock, LoopMiddleBlock);
+
+  DEBUG(DT->verifyAnalysis());
+}
+
+bool LoopVectorizationLegality::canVectorize() {
+  if (!TheLoop->getLoopPreheader()) {
+    assert(false && "No preheader!!");
+    DEBUG(dbgs() << "LV: Loop not normalized." << "\n");
+    return  false;
+  }
+
+  // We can only vectorize single basic block loops.
+  unsigned NumBlocks = TheLoop->getNumBlocks();
+  if (NumBlocks != 1) {
+    DEBUG(dbgs() << "LV: Too many blocks:" << NumBlocks << "\n");
+    return false;
+  }
+
+  // We need to have a loop header.
+  BasicBlock *BB = TheLoop->getHeader();
+  DEBUG(dbgs() << "LV: Found a loop: " << BB->getName() << "\n");
+
+  // ScalarEvolution needs to be able to find the exit count.
+  const SCEV *ExitCount = SE->getExitCount(TheLoop, BB);
+  if (ExitCount == SE->getCouldNotCompute()) {
+    DEBUG(dbgs() << "LV: SCEV could not compute the loop exit count.\n");
+    return false;
+  }
+
+  // Do not loop-vectorize loops with a tiny trip count.
+  unsigned TC = SE->getSmallConstantTripCount(TheLoop, BB);
+  if (TC > 0u && TC < TinyTripCountThreshold) {
+    DEBUG(dbgs() << "LV: Found a loop with a very small trip count. " <<
+          "This loop is not worth vectorizing.\n");
+    return false;
+  }
+
+  // Go over each instruction and look at memory deps.
+  if (!canVectorizeBlock(*BB)) {
+    DEBUG(dbgs() << "LV: Can't vectorize this loop header\n");
+    return false;
+  }
+
+  DEBUG(dbgs() << "LV: We can vectorize this loop" <<
+        (PtrRtCheck.Need ? " (with a runtime bound check)" : "")
+        <<"!\n");
+
+  // Okay! We can vectorize. At this point we don't have any other mem analysis
+  // which may limit our maximum vectorization factor, so just return true with
+  // no restrictions.
+  return true;
+}
+
+bool LoopVectorizationLegality::canVectorizeBlock(BasicBlock &BB) {
+  // Scan the instructions in the block and look for hazards.
+  for (BasicBlock::iterator it = BB.begin(), e = BB.end(); it != e; ++it) {
+    Instruction *I = it;
+
+    PHINode *Phi = dyn_cast<PHINode>(I);
+    if (Phi) {
+      // This should not happen because the loop should be normalized.
+      if (Phi->getNumIncomingValues() != 2) {
+        DEBUG(dbgs() << "LV: Found an invalid PHI.\n");
+        return false;
+      }
+      // We only look at integer phi nodes.
+      if (!Phi->getType()->isIntegerTy()) {
+        DEBUG(dbgs() << "LV: Found an non-int PHI.\n");
+        return false;
+      }
+
+      if (isInductionVariable(Phi)) {
+        if (Induction) {
+          DEBUG(dbgs() << "LV: Found too many inductions."<< *Phi <<"\n");
+          return false;
+        }
+        DEBUG(dbgs() << "LV: Found the induction PHI."<< *Phi <<"\n");
+        Induction = Phi;
+        continue;
+      }
+      if (AddReductionVar(Phi, IntegerAdd)) {
+        DEBUG(dbgs() << "LV: Found an ADD reduction PHI."<< *Phi <<"\n");
+        continue;
+      }
+      if (AddReductionVar(Phi, IntegerMult)) {
+        DEBUG(dbgs() << "LV: Found a MUL reduction PHI."<< *Phi <<"\n");
+        continue;
+      }
+      if (AddReductionVar(Phi, IntegerOr)) {
+        DEBUG(dbgs() << "LV: Found an OR reduction PHI."<< *Phi <<"\n");
+        continue;
+      }
+      if (AddReductionVar(Phi, IntegerAnd)) {
+        DEBUG(dbgs() << "LV: Found an AND reduction PHI."<< *Phi <<"\n");
+        continue;
+      }
+      if (AddReductionVar(Phi, IntegerXor)) {
+        DEBUG(dbgs() << "LV: Found a XOR reduction PHI."<< *Phi <<"\n");
+        continue;
+      }
+
+      DEBUG(dbgs() << "LV: Found an unidentified PHI."<< *Phi <<"\n");
+      return false;
+    }// end of PHI handling
+
+    // We still don't handle functions.
+    CallInst *CI = dyn_cast<CallInst>(I);
+    if (CI) {
+      DEBUG(dbgs() << "LV: Found a call site.\n");
+      return false;
+    }
+
+    // We do not re-vectorize vectors.
+    if (!VectorType::isValidElementType(I->getType()) &&
+        !I->getType()->isVoidTy()) {
+      DEBUG(dbgs() << "LV: Found unvectorizable type." << "\n");
+      return false;
+    }
+
+    // Reduction instructions are allowed to have exit users.
+    // All other instructions must not have external users.
+    if (!AllowedExit.count(I))
+      //Check that all of the users of the loop are inside the BB.
+      for (Value::use_iterator it = I->use_begin(), e = I->use_end();
+           it != e; ++it) {
+        Instruction *U = cast<Instruction>(*it);
+        // This user may be a reduction exit value.
+        BasicBlock *Parent = U->getParent();
+        if (Parent != &BB) {
+          DEBUG(dbgs() << "LV: Found an outside user for : "<< *U << "\n");
+          return false;
+        }
+    }
+  } // next instr.
+
+  if (!Induction) {
+      DEBUG(dbgs() << "LV: Did not find an induction var.\n");
+      return false;
+  }
+
+  // Don't vectorize if the memory dependencies do not allow vectorization.
+  if (!canVectorizeMemory(BB))
+    return false;
+
+  // We now know that the loop is vectorizable!
+  // Collect variables that will remain uniform after vectorization.
+  std::vector<Value*> Worklist;
+
+  // Start with the conditional branch and walk up the block.
+  Worklist.push_back(BB.getTerminator()->getOperand(0));
+
+  while (Worklist.size()) {
+    Instruction *I = dyn_cast<Instruction>(Worklist.back());
+    Worklist.pop_back();
+    // Look at instructions inside this block.
+    if (!I) continue;
+    if (I->getParent() != &BB) continue;
+
+    // Stop when reaching PHI nodes.
+    if (isa<PHINode>(I)) {
+      assert(I == Induction && "Found a uniform PHI that is not the induction");
+      break;
+    }
+
+    // This is a known uniform.
+    Uniforms.insert(I);
+
+    // Insert all operands.
+    for (int i=0, Op = I->getNumOperands(); i < Op; ++i) {
+      Worklist.push_back(I->getOperand(i));
+    }
+  }
+
+  return true;
+}
+
+bool LoopVectorizationLegality::canVectorizeMemory(BasicBlock &BB) {
+  typedef SmallVector<Value*, 16> ValueVector;
+  typedef SmallPtrSet<Value*, 16> ValueSet;
+  // Holds the Load and Store *instructions*.
+  ValueVector Loads;
+  ValueVector Stores;
+  PtrRtCheck.Pointers.clear();
+  PtrRtCheck.Need = false;
+
+  // Scan the BB and collect legal loads and stores.
+  for (BasicBlock::iterator it = BB.begin(), e = BB.end(); it != e; ++it) {
+    Instruction *I = it;
+
+    // If this is a load, save it. If this instruction can read from memory
+    // but is not a load, then we quit. Notice that we don't handle function
+    // calls that read or write.
+    if (I->mayReadFromMemory()) {
+      LoadInst *Ld = dyn_cast<LoadInst>(I);
+      if (!Ld) return false;
+      if (!Ld->isSimple()) {
+        DEBUG(dbgs() << "LV: Found a non-simple load.\n");
+        return false;
+      }
+      Loads.push_back(Ld);
+      continue;
+    }
+
+    // Save store instructions. Abort if other instructions write to memory.
+    if (I->mayWriteToMemory()) {
+      StoreInst *St = dyn_cast<StoreInst>(I);
+      if (!St) return false;
+      if (!St->isSimple()) {
+        DEBUG(dbgs() << "LV: Found a non-simple store.\n");
+        return false;
+      }
+      Stores.push_back(St);
+    }
+  } // next instr.
+
+  // Now we have two lists that hold the loads and the stores.
+  // Next, we find the pointers that they use.
+
+  // Check if we see any stores. If there are no stores, then we don't
+  // care if the pointers are *restrict*.
+  if (!Stores.size()) {
+        DEBUG(dbgs() << "LV: Found a read-only loop!\n");
+        return true;
+  }
+
+  // Holds the read and read-write *pointers* that we find.
+  ValueVector Reads;
+  ValueVector ReadWrites;
+
+  // Holds the analyzed pointers. We don't want to call GetUnderlyingObjects
+  // multiple times on the same object. If the ptr is accessed twice, once
+  // for read and once for write, it will only appear once (on the write
+  // list). This is okay, since we are going to check for conflicts between
+  // writes and between reads and writes, but not between reads and reads.
+  ValueSet Seen;
+
+  ValueVector::iterator I, IE;
+  for (I = Stores.begin(), IE = Stores.end(); I != IE; ++I) {
+    StoreInst *ST = dyn_cast<StoreInst>(*I);
+    assert(ST && "Bad StoreInst");
+    Value* Ptr = ST->getPointerOperand();
+
+    if (isUniform(Ptr)) {
+      DEBUG(dbgs() << "LV: We don't allow storing to uniform addresses\n");
+      return false;
+    }
+
+    // If we did *not* see this pointer before, insert it to
+    // the read-write list. At this phase it is only a 'write' list.
+    if (Seen.insert(Ptr))
+      ReadWrites.push_back(Ptr);
+  }
+
+  for (I = Loads.begin(), IE = Loads.end(); I != IE; ++I) {
+    LoadInst *LD = dyn_cast<LoadInst>(*I);
+    assert(LD && "Bad LoadInst");
+    Value* Ptr = LD->getPointerOperand();
+    // If we did *not* see this pointer before, insert it to the
+    // read list. If we *did* see it before, then it is already in
+    // the read-write list. This allows us to vectorize expressions
+    // such as A[i] += x;  Because the address of A[i] is a read-write
+    // pointer. This only works if the index of A[i] is consecutive.
+    // If the address of i is unknown (for example A[B[i]]) then we may
+    // read a few words, modify, and write a few words, and some of the
+    // words may be written to the same address.
+    if (Seen.insert(Ptr) || !isConsecutiveGep(Ptr))
+      Reads.push_back(Ptr);
+  }
+
+  // If we write (or read-write) to a single destination and there are no
+  // other reads in this loop then is it safe to vectorize.
+  if (ReadWrites.size() == 1 && Reads.size() == 0) {
+    DEBUG(dbgs() << "LV: Found a write-only loop!\n");
+    return true;
+  }
+
+  // Find pointers with computable bounds. We are going to use this information
+  // to place a runtime bound check.
+  bool RT = true;
+  for (I = ReadWrites.begin(), IE = ReadWrites.end(); I != IE; ++I)
+    if (hasComputableBounds(*I)) {
+      PtrRtCheck.Pointers.push_back(*I);
+      DEBUG(dbgs() << "LV: Found a runtime check ptr:" << **I <<"\n");
+    } else {
+      RT = false;
+      break;
+    }
+  for (I = Reads.begin(), IE = Reads.end(); I != IE; ++I)
+    if (hasComputableBounds(*I)) {
+      PtrRtCheck.Pointers.push_back(*I);
+      DEBUG(dbgs() << "LV: Found a runtime check ptr:" << **I <<"\n");
+    } else {
+      RT = false;
+      break;
+    }
+
+  // Check that we did not collect too many pointers or found a
+  // unsizeable pointer.
+  if (!RT || PtrRtCheck.Pointers.size() > RuntimeMemoryCheckThreshold) {
+    PtrRtCheck.Pointers.clear();
+    RT = false;
+  }
+
+  PtrRtCheck.Need = RT;
+
+  if (RT) {
+    DEBUG(dbgs() << "LV: We can perform a memory runtime check if needed.\n");
+  }
+
+  // Now that the pointers are in two lists (Reads and ReadWrites), we
+  // can check that there are no conflicts between each of the writes and
+  // between the writes to the reads.
+  ValueSet WriteObjects;
+  ValueVector TempObjects;
+
+  // Check that the read-writes do not conflict with other read-write
+  // pointers.
+  for (I = ReadWrites.begin(), IE = ReadWrites.end(); I != IE; ++I) {
+    GetUnderlyingObjects(*I, TempObjects, DL);
+    for (ValueVector::iterator it=TempObjects.begin(), e=TempObjects.end();
+         it != e; ++it) {
+      if (!isIdentifiedObject(*it)) {
+        DEBUG(dbgs() << "LV: Found an unidentified write ptr:"<< **it <<"\n");
+        return RT;
+      }
+      if (!WriteObjects.insert(*it)) {
+        DEBUG(dbgs() << "LV: Found a possible write-write reorder:"
+              << **it <<"\n");
+        return RT;
+      }
+    }
+    TempObjects.clear();
+  }
+
+  /// Check that the reads don't conflict with the read-writes.
+  for (I = Reads.begin(), IE = Reads.end(); I != IE; ++I) {
+    GetUnderlyingObjects(*I, TempObjects, DL);
+    for (ValueVector::iterator it=TempObjects.begin(), e=TempObjects.end();
+         it != e; ++it) {
+      if (!isIdentifiedObject(*it)) {
+        DEBUG(dbgs() << "LV: Found an unidentified read ptr:"<< **it <<"\n");
+        return RT;
+      }
+      if (WriteObjects.count(*it)) {
+        DEBUG(dbgs() << "LV: Found a possible read/write reorder:"
+              << **it <<"\n");
+        return RT;
+      }
+    }
+    TempObjects.clear();
+  }
+
+  // It is safe to vectorize and we don't need any runtime checks.
+  DEBUG(dbgs() << "LV: We don't need a runtime memory check.\n");
+  PtrRtCheck.Pointers.clear();
+  PtrRtCheck.Need = false;
+  return true;
+}
+
+bool LoopVectorizationLegality::AddReductionVar(PHINode *Phi,
+                                                ReductionKind Kind) {
+  if (Phi->getNumIncomingValues() != 2)
+    return false;
+
+  // Find the possible incoming reduction variable.
+  BasicBlock *BB = Phi->getParent();
+  int SelfEdgeIdx = Phi->getBasicBlockIndex(BB);
+  int InEdgeBlockIdx = (SelfEdgeIdx ? 0 : 1); // The other entry.
+  Value *RdxStart = Phi->getIncomingValue(InEdgeBlockIdx);
+
+  // ExitInstruction is the single value which is used outside the loop.
+  // We only allow for a single reduction value to be used outside the loop.
+  // This includes users of the reduction, variables (which form a cycle
+  // which ends in the phi node).
+  Instruction *ExitInstruction = 0;
+
+  // Iter is our iterator. We start with the PHI node and scan for all of the
+  // users of this instruction. All users must be instructions which can be
+  // used as reduction variables (such as ADD). We may have a single
+  // out-of-block user. They cycle must end with the original PHI.
+  // Also, we can't have multiple block-local users.
+  Instruction *Iter = Phi;
+  while (true) {
+    // Any reduction instr must be of one of the allowed kinds.
+    if (!isReductionInstr(Iter, Kind))
+      return false;
+
+    // Did we found a user inside this block ?
+    bool FoundInBlockUser = false;
+    // Did we reach the initial PHI node ?
+    bool FoundStartPHI = false;
+
+    // If the instruction has no users then this is a broken
+    // chain and can't be a reduction variable.
+    if (Iter->use_empty())
+      return false;
+
+    // For each of the *users* of iter.
+    for (Value::use_iterator it = Iter->use_begin(), e = Iter->use_end();
+         it != e; ++it) {
+      Instruction *U = cast<Instruction>(*it);
+      // We already know that the PHI is a user.
+      if (U == Phi) {
+        FoundStartPHI = true;
+        continue;
+      }
+      // Check if we found the exit user.
+      BasicBlock *Parent = U->getParent();
+      if (Parent != BB) {
+        // We must have a single exit instruction.
+        if (ExitInstruction != 0)
+          return false;
+        ExitInstruction = Iter;
+      }
+      // We can't have multiple inside users.
+      if (FoundInBlockUser)
+        return false;
+      FoundInBlockUser = true;
+      Iter = U;
+    }
+
+    // We found a reduction var if we have reached the original
+    // phi node and we only have a single instruction with out-of-loop
+    // users.
+   if (FoundStartPHI && ExitInstruction) {
+     // This instruction is allowed to have out-of-loop users.
+     AllowedExit.insert(ExitInstruction);
+
+     // Save the description of this reduction variable.
+     ReductionDescriptor RD(RdxStart, ExitInstruction, Kind);
+     Reductions[Phi] = RD;
+     return true;
+   }
+  }
+}
+
+bool
+LoopVectorizationLegality::isReductionInstr(Instruction *I,
+                                            ReductionKind Kind) {
+    switch (I->getOpcode()) {
+    default:
+      return false;
+    case Instruction::PHI:
+      // possibly.
+      return true;
+    case Instruction::Add:
+    case Instruction::Sub:
+      return Kind == IntegerAdd;
+    case Instruction::Mul:
+    case Instruction::UDiv:
+    case Instruction::SDiv:
+      return Kind == IntegerMult;
+    case Instruction::And:
+      return Kind == IntegerAnd;
+    case Instruction::Or:
+      return Kind == IntegerOr;
+    case Instruction::Xor:
+      return Kind == IntegerXor;
+    }
+}
+
+bool LoopVectorizationLegality::isInductionVariable(PHINode *Phi) {
+  // Check that the PHI is consecutive and starts at zero.
+  const SCEV *PhiScev = SE->getSCEV(Phi);
+  const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(PhiScev);
+  if (!AR) {
+    DEBUG(dbgs() << "LV: PHI is not a poly recurrence.\n");
+    return false;
+  }
+  const SCEV *Step = AR->getStepRecurrence(*SE);
+
+  if (!Step->isOne()) {
+    DEBUG(dbgs() << "LV: PHI stride does not equal one.\n");
+    return false;
+  }
+  return true;
+}
+
+bool LoopVectorizationLegality::hasComputableBounds(Value *Ptr) {
+  const SCEV *PhiScev = SE->getSCEV(Ptr);
+  const SCEVAddRecExpr *AR = dyn_cast<SCEVAddRecExpr>(PhiScev);
+  if (!AR)
+    return false;
+
+  return AR->isAffine();
+}
+
+unsigned
+LoopVectorizationCostModel::findBestVectorizationFactor(unsigned VF) {
+  if (!VTTI) {
+    DEBUG(dbgs() << "LV: No vector target information. Not vectorizing. \n");
+    return 1;
+  }
+
+  float Cost = expectedCost(1);
+  unsigned Width = 1;
+  DEBUG(dbgs() << "LV: Scalar loop costs: "<< (int)Cost << ".\n");
+  for (unsigned i=2; i <= VF; i*=2) {
+    // Notice that the vector loop needs to be executed less times, so
+    // we need to divide the cost of the vector loops by the width of
+    // the vector elements.
+    float VectorCost = expectedCost(i) / (float)i;
+    DEBUG(dbgs() << "LV: Vector loop of width "<< i << " costs: " <<
+          (int)VectorCost << ".\n");
+    if (VectorCost < Cost) {
+      Cost = VectorCost;
+      Width = i;
+    }
+  }
+
+  DEBUG(dbgs() << "LV: Selecting VF = : "<< Width << ".\n");
+  return Width;
+}
+
+unsigned LoopVectorizationCostModel::expectedCost(unsigned VF) {
+  // We can only estimate the cost of single basic block loops.
+  assert(1 == TheLoop->getNumBlocks() && "Too many blocks in loop");
+
+  BasicBlock *BB = TheLoop->getHeader();
+  unsigned Cost = 0;
+
+  // For each instruction in the old loop.
+  for (BasicBlock::iterator it = BB->begin(), e = BB->end(); it != e; ++it) {
+    Instruction *Inst = it;
+    unsigned C = getInstructionCost(Inst, VF);
+    Cost += C;
+    DEBUG(dbgs() << "LV: Found an estimated cost of "<< C <<" for VF "<< VF <<
+          " For instruction: "<< *Inst << "\n");
+  }
+
+  return Cost;
+}
+
+unsigned
+LoopVectorizationCostModel::getInstructionCost(Instruction *I, unsigned VF) {
+  assert(VTTI && "Invalid vector target transformation info");
+
+  // If we know that this instruction will remain uniform, check the cost of
+  // the scalar version.
+  if (Legal->isUniformAfterVectorization(I))
+    VF = 1;
+
+  Type *RetTy = I->getType();
+  Type *VectorTy = ToVectorTy(RetTy, VF);
+
+
+  // TODO: We need to estimate the cost of intrinsic calls.
+  switch (I->getOpcode()) {
+    case Instruction::GetElementPtr:
+      // We mark this instruction as zero-cost because scalar GEPs are usually
+      // lowered to the intruction addressing mode. At the moment we don't
+      // generate vector geps.
+      return 0;
+    case Instruction::Br: {
+      return VTTI->getCFInstrCost(I->getOpcode());
+    }
+    case Instruction::PHI:
+      return 0;
+    case Instruction::Add:
+    case Instruction::FAdd:
+    case Instruction::Sub:
+    case Instruction::FSub:
+    case Instruction::Mul:
+    case Instruction::FMul:
+    case Instruction::UDiv:
+    case Instruction::SDiv:
+    case Instruction::FDiv:
+    case Instruction::URem:
+    case Instruction::SRem:
+    case Instruction::FRem:
+    case Instruction::Shl:
+    case Instruction::LShr:
+    case Instruction::AShr:
+    case Instruction::And:
+    case Instruction::Or:
+    case Instruction::Xor: {
+      return VTTI->getArithmeticInstrCost(I->getOpcode(), VectorTy);
+    }
+    case Instruction::Select: {
+      SelectInst *SI = cast<SelectInst>(I);
+      const SCEV *CondSCEV = SE->getSCEV(SI->getCondition());
+      bool ScalarCond = (SE->isLoopInvariant(CondSCEV, TheLoop));
+      Type *CondTy = SI->getCondition()->getType();
+      if (ScalarCond)
+        CondTy = VectorType::get(CondTy, VF);
+
+      return VTTI->getCmpSelInstrCost(I->getOpcode(), VectorTy, CondTy);
+    }
+    case Instruction::ICmp:
+    case Instruction::FCmp: {
+      Type *ValTy = I->getOperand(0)->getType();
+      VectorTy = ToVectorTy(ValTy, VF);
+      return VTTI->getCmpSelInstrCost(I->getOpcode(), VectorTy);
+    }
+    case Instruction::Store: {
+      StoreInst *SI = cast<StoreInst>(I);
+      Type *ValTy = SI->getValueOperand()->getType();
+      VectorTy = ToVectorTy(ValTy, VF);
+
+      if (VF == 1)
+        return VTTI->getMemoryOpCost(I->getOpcode(), ValTy,
+                              SI->getAlignment(), SI->getPointerAddressSpace());
+
+      // Scalarized stores.
+      if (!Legal->isConsecutiveGep(SI->getPointerOperand())) {
+        unsigned Cost = 0;
+        unsigned ExtCost = VTTI->getInstrCost(Instruction::ExtractElement,
+                                              ValTy);
+        // The cost of extracting from the value vector.
+        Cost += VF * (ExtCost);
+        // The cost of the scalar stores.
+        Cost += VF * VTTI->getMemoryOpCost(I->getOpcode(),
+                                           ValTy->getScalarType(),
+                                           SI->getAlignment(),
+                                           SI->getPointerAddressSpace());
+        return Cost;
+      }
+
+      // Wide stores.
+      return VTTI->getMemoryOpCost(I->getOpcode(), VectorTy, SI->getAlignment(),
+                                   SI->getPointerAddressSpace());
+    }
+    case Instruction::Load: {
+      LoadInst *LI = cast<LoadInst>(I);
+
+      if (VF == 1)
+        return VTTI->getMemoryOpCost(I->getOpcode(), RetTy,
+                                     LI->getAlignment(),
+                                     LI->getPointerAddressSpace());
+
+      // Scalarized loads.
+      if (!Legal->isConsecutiveGep(LI->getPointerOperand())) {
+        unsigned Cost = 0;
+        unsigned InCost = VTTI->getInstrCost(Instruction::InsertElement, RetTy);
+        // The cost of inserting the loaded value into the result vector.
+        Cost += VF * (InCost);
+        // The cost of the scalar stores.
+        Cost += VF * VTTI->getMemoryOpCost(I->getOpcode(),
+                                           RetTy->getScalarType(),
+                                           LI->getAlignment(),
+                                           LI->getPointerAddressSpace());
+        return Cost;
+      }
+
+      // Wide loads.
+      return VTTI->getMemoryOpCost(I->getOpcode(), VectorTy, LI->getAlignment(),
+                                   LI->getPointerAddressSpace());
+    }
+    case Instruction::ZExt:
+    case Instruction::SExt:
+    case Instruction::FPToUI:
+    case Instruction::FPToSI:
+    case Instruction::FPExt:
+    case Instruction::PtrToInt:
+    case Instruction::IntToPtr:
+    case Instruction::SIToFP:
+    case Instruction::UIToFP:
+    case Instruction::Trunc:
+    case Instruction::FPTrunc:
+    case Instruction::BitCast: {
+      Type *SrcVecTy = ToVectorTy(I->getOperand(0)->getType(), VF);
+      return VTTI->getCastInstrCost(I->getOpcode(), VectorTy, SrcVecTy);
+    }
+    default: {
+      // We are scalarizing the instruction. Return the cost of the scalar
+      // instruction, plus the cost of insert and extract into vector
+      // elements, times the vector width.
+      unsigned Cost = 0;
+
+      bool IsVoid = RetTy->isVoidTy();
+
+      unsigned InsCost = (IsVoid ? 0 :
+                          VTTI->getInstrCost(Instruction::InsertElement,
+                                             VectorTy));
+
+      unsigned ExtCost = VTTI->getInstrCost(Instruction::ExtractElement,
+                                            VectorTy);
+
+      // The cost of inserting the results plus extracting each one of the
+      // operands.
+      Cost += VF * (InsCost + ExtCost * I->getNumOperands());
+
+      // The cost of executing VF copies of the scalar instruction.
+      Cost += VF * VTTI->getInstrCost(I->getOpcode(), RetTy);
+      return Cost;
+    }
+  }// end of switch.
+}
+
+Type* LoopVectorizationCostModel::ToVectorTy(Type *Scalar, unsigned VF) {
+  if (Scalar->isVoidTy() || VF == 1)
+    return Scalar;
+  return VectorType::get(Scalar, VF);
+}
+
+} // namespace
+
+char LoopVectorize::ID = 0;
+static const char lv_name[] = "Loop Vectorization";
+INITIALIZE_PASS_BEGIN(LoopVectorize, LV_NAME, lv_name, false, false)
+INITIALIZE_AG_DEPENDENCY(AliasAnalysis)
+INITIALIZE_PASS_DEPENDENCY(ScalarEvolution)
+INITIALIZE_PASS_DEPENDENCY(LoopSimplify)
+INITIALIZE_PASS_END(LoopVectorize, LV_NAME, lv_name, false, false)
+
+namespace llvm {
+  Pass *createLoopVectorizePass() {
+    return new LoopVectorize();
+  }
+}
+
diff --git a/lib/Transforms/Vectorize/Vectorize.cpp b/lib/Transforms/Vectorize/Vectorize.cpp
index 1ef6002..d26973a 100644
--- a/lib/Transforms/Vectorize/Vectorize.cpp
+++ b/lib/Transforms/Vectorize/Vectorize.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This file implements common infrastructure for libLLVMVectorizeOpts.a, which 
+// This file implements common infrastructure for libLLVMVectorizeOpts.a, which
 // implements several vectorization transformations over the LLVM intermediate
 // representation, including the C bindings for that library.
 //
@@ -23,10 +23,11 @@
 
 using namespace llvm;
 
-/// initializeVectorizationPasses - Initialize all passes linked into the 
+/// initializeVectorizationPasses - Initialize all passes linked into the
 /// Vectorization library.
 void llvm::initializeVectorization(PassRegistry &Registry) {
   initializeBBVectorizePass(Registry);
+  initializeLoopVectorizePass(Registry);
 }
 
 void LLVMInitializeVectorization(LLVMPassRegistryRef R) {
@@ -37,3 +38,6 @@ void LLVMAddBBVectorizePass(LLVMPassManagerRef PM) {
   unwrap(PM)->add(createBBVectorizePass());
 }
 
+void LLVMAddLoopVectorizePass(LLVMPassManagerRef PM) {
+  unwrap(PM)->add(createLoopVectorizePass());
+}
diff --git a/lib/VMCore/AsmWriter.cpp b/lib/VMCore/AsmWriter.cpp
index 7ef1131..b72c17f 100644
--- a/lib/VMCore/AsmWriter.cpp
+++ b/lib/VMCore/AsmWriter.cpp
@@ -66,6 +66,25 @@ static const Module *getModuleFromVal(const Value *V) {
   return 0;
 }
 
+static void PrintCallingConv(unsigned cc, raw_ostream &Out)
+{
+  switch (cc) {
+    case CallingConv::Fast:         Out << "fastcc"; break;
+    case CallingConv::Cold:         Out << "coldcc"; break;
+    case CallingConv::X86_StdCall:  Out << "x86_stdcallcc"; break;
+    case CallingConv::X86_FastCall: Out << "x86_fastcallcc"; break;
+    case CallingConv::X86_ThisCall: Out << "x86_thiscallcc"; break;
+    case CallingConv::Intel_OCL_BI: Out << "intel_ocl_bicc"; break;
+    case CallingConv::ARM_APCS:     Out << "arm_apcscc"; break;
+    case CallingConv::ARM_AAPCS:    Out << "arm_aapcscc"; break;
+    case CallingConv::ARM_AAPCS_VFP:Out << "arm_aapcs_vfpcc"; break;
+    case CallingConv::MSP430_INTR:  Out << "msp430_intrcc"; break;
+    case CallingConv::PTX_Kernel:   Out << "ptx_kernel"; break;
+    case CallingConv::PTX_Device:   Out << "ptx_device"; break;
+    default:                        Out << "cc" << cc; break;
+  }
+}
+ 
 // PrintEscapedString - Print each character of the specified string, escaping
 // it if it is not printable or if it is an escape char.
 static void PrintEscapedString(StringRef Name, raw_ostream &Out) {
@@ -141,8 +160,8 @@ static void PrintLLVMName(raw_ostream &OS, const Value *V) {
 /// TypePrinting - Type printing machinery.
 namespace {
 class TypePrinting {
-  TypePrinting(const TypePrinting &);   // DO NOT IMPLEMENT
-  void operator=(const TypePrinting&);  // DO NOT IMPLEMENT
+  TypePrinting(const TypePrinting &) LLVM_DELETED_FUNCTION;
+  void operator=(const TypePrinting&) LLVM_DELETED_FUNCTION;
 public:
 
   /// NamedTypes - The named types that are used by the current module.
@@ -380,8 +399,8 @@ private:
   /// Add all of the functions arguments, basic blocks, and instructions.
   void processFunction();
 
-  SlotTracker(const SlotTracker &);  // DO NOT IMPLEMENT
-  void operator=(const SlotTracker &);  // DO NOT IMPLEMENT
+  SlotTracker(const SlotTracker &) LLVM_DELETED_FUNCTION;
+  void operator=(const SlotTracker &) LLVM_DELETED_FUNCTION;
 };
 
 }  // end anonymous namespace
@@ -1029,6 +1048,9 @@ static void WriteAsOperandInternal(raw_ostream &Out, const Value *V,
       Out << "sideeffect ";
     if (IA->isAlignStack())
       Out << "alignstack ";
+    // We don't emit the AD_ATT dialect as it's the assumed default.
+    if (IA->getDialect() == InlineAsm::AD_Intel)
+      Out << "inteldialect ";
     Out << '"';
     PrintEscapedString(IA->getAsmString(), Out);
     Out << "\", \"";
@@ -1222,8 +1244,8 @@ void AssemblyWriter::writeParamOperand(const Value *Operand,
   // Print the type
   TypePrinter.print(Operand->getType(), Out);
   // Print parameter attributes list
-  if (Attrs != Attribute::None)
-    Out << ' ' << Attribute::getAsString(Attrs);
+  if (Attrs.hasAttributes())
+    Out << ' ' << Attrs.getAsString();
   Out << ' ';
   // Print the operand
   WriteAsOperandInternal(Out, Operand, &TypePrinter, &Machine, TheModule);
@@ -1285,8 +1307,9 @@ void AssemblyWriter::printModule(const Module *M) {
   // Output all globals.
   if (!M->global_empty()) Out << '\n';
   for (Module::const_global_iterator I = M->global_begin(), E = M->global_end();
-       I != E; ++I)
-    printGlobal(I);
+       I != E; ++I) {
+    printGlobal(I); Out << '\n';
+  }
 
   // Output all aliases.
   if (!M->alias_empty()) Out << "\n";
@@ -1353,12 +1376,12 @@ static void PrintLinkage(GlobalValue::LinkageTypes LT,
   case GlobalValue::LinkerPrivateWeakLinkage:
     Out << "linker_private_weak ";
     break;
-  case GlobalValue::LinkerPrivateWeakDefAutoLinkage:
-    Out << "linker_private_weak_def_auto ";
-    break;
   case GlobalValue::InternalLinkage:      Out << "internal ";       break;
   case GlobalValue::LinkOnceAnyLinkage:   Out << "linkonce ";       break;
   case GlobalValue::LinkOnceODRLinkage:   Out << "linkonce_odr ";   break;
+  case GlobalValue::LinkOnceODRAutoHideLinkage:
+    Out << "linkonce_odr_auto_hide ";
+    break;
   case GlobalValue::WeakAnyLinkage:       Out << "weak ";           break;
   case GlobalValue::WeakODRLinkage:       Out << "weak_odr ";       break;
   case GlobalValue::CommonLinkage:        Out << "common ";         break;
@@ -1436,7 +1459,6 @@ void AssemblyWriter::printGlobal(const GlobalVariable *GV) {
     Out << ", align " << GV->getAlignment();
 
   printInfoComment(*GV);
-  Out << '\n';
 }
 
 void AssemblyWriter::printAlias(const GlobalAlias *GA) {
@@ -1527,27 +1549,16 @@ void AssemblyWriter::printFunction(const Function *F) {
   PrintVisibility(F->getVisibility(), Out);
 
   // Print the calling convention.
-  switch (F->getCallingConv()) {
-  case CallingConv::C: break;   // default
-  case CallingConv::Fast:         Out << "fastcc "; break;
-  case CallingConv::Cold:         Out << "coldcc "; break;
-  case CallingConv::X86_StdCall:  Out << "x86_stdcallcc "; break;
-  case CallingConv::X86_FastCall: Out << "x86_fastcallcc "; break;
-  case CallingConv::X86_ThisCall: Out << "x86_thiscallcc "; break;
-  case CallingConv::ARM_APCS:     Out << "arm_apcscc "; break;
-  case CallingConv::ARM_AAPCS:    Out << "arm_aapcscc "; break;
-  case CallingConv::ARM_AAPCS_VFP:Out << "arm_aapcs_vfpcc "; break;
-  case CallingConv::MSP430_INTR:  Out << "msp430_intrcc "; break;
-  case CallingConv::PTX_Kernel:   Out << "ptx_kernel "; break;
-  case CallingConv::PTX_Device:   Out << "ptx_device "; break;
-  default: Out << "cc" << F->getCallingConv() << " "; break;
+  if (F->getCallingConv() != CallingConv::C) {
+    PrintCallingConv(F->getCallingConv(), Out);
+    Out << " ";
   }
 
   FunctionType *FT = F->getFunctionType();
   const AttrListPtr &Attrs = F->getAttributes();
   Attributes RetAttrs = Attrs.getRetAttributes();
-  if (RetAttrs != Attribute::None)
-    Out <<  Attribute::getAsString(Attrs.getRetAttributes()) << ' ';
+  if (RetAttrs.hasAttributes())
+    Out <<  Attrs.getRetAttributes().getAsString() << ' ';
   TypePrinter.print(F->getReturnType(), Out);
   Out << ' ';
   WriteAsOperandInternal(Out, F, &TypePrinter, &Machine, F->getParent());
@@ -1576,8 +1587,8 @@ void AssemblyWriter::printFunction(const Function *F) {
       TypePrinter.print(FT->getParamType(i), Out);
 
       Attributes ArgAttrs = Attrs.getParamAttributes(i+1);
-      if (ArgAttrs != Attribute::None)
-        Out << ' ' << Attribute::getAsString(ArgAttrs);
+      if (ArgAttrs.hasAttributes())
+        Out << ' ' << ArgAttrs.getAsString();
     }
   }
 
@@ -1590,8 +1601,8 @@ void AssemblyWriter::printFunction(const Function *F) {
   if (F->hasUnnamedAddr())
     Out << " unnamed_addr";
   Attributes FnAttrs = Attrs.getFnAttributes();
-  if (FnAttrs != Attribute::None)
-    Out << ' ' << Attribute::getAsString(Attrs.getFnAttributes());
+  if (FnAttrs.hasAttributes())
+    Out << ' ' << Attrs.getFnAttributes().getAsString();
   if (F->hasSection()) {
     Out << " section \"";
     PrintEscapedString(F->getSection(), Out);
@@ -1624,8 +1635,8 @@ void AssemblyWriter::printArgument(const Argument *Arg,
   TypePrinter.print(Arg->getType(), Out);
 
   // Output parameter attributes list
-  if (Attrs != Attribute::None)
-    Out << ' ' << Attribute::getAsString(Attrs);
+  if (Attrs.hasAttributes())
+    Out << ' ' << Attrs.getAsString();
 
   // Output name, if available...
   if (Arg->hasName()) {
@@ -1828,20 +1839,9 @@ void AssemblyWriter::printInstruction(const Instruction &I) {
     Out << " void";
   } else if (const CallInst *CI = dyn_cast<CallInst>(&I)) {
     // Print the calling convention being used.
-    switch (CI->getCallingConv()) {
-    case CallingConv::C: break;   // default
-    case CallingConv::Fast:  Out << " fastcc"; break;
-    case CallingConv::Cold:  Out << " coldcc"; break;
-    case CallingConv::X86_StdCall:  Out << " x86_stdcallcc"; break;
-    case CallingConv::X86_FastCall: Out << " x86_fastcallcc"; break;
-    case CallingConv::X86_ThisCall: Out << " x86_thiscallcc"; break;
-    case CallingConv::ARM_APCS:     Out << " arm_apcscc "; break;
-    case CallingConv::ARM_AAPCS:    Out << " arm_aapcscc "; break;
-    case CallingConv::ARM_AAPCS_VFP:Out << " arm_aapcs_vfpcc "; break;
-    case CallingConv::MSP430_INTR:  Out << " msp430_intrcc "; break;
-    case CallingConv::PTX_Kernel:   Out << " ptx_kernel"; break;
-    case CallingConv::PTX_Device:   Out << " ptx_device"; break;
-    default: Out << " cc" << CI->getCallingConv(); break;
+    if (CI->getCallingConv() != CallingConv::C) {
+      Out << " ";
+      PrintCallingConv(CI->getCallingConv(), Out);
     }
 
     Operand = CI->getCalledValue();
@@ -1850,8 +1850,8 @@ void AssemblyWriter::printInstruction(const Instruction &I) {
     Type *RetTy = FTy->getReturnType();
     const AttrListPtr &PAL = CI->getAttributes();
 
-    if (PAL.getRetAttributes() != Attribute::None)
-      Out << ' ' << Attribute::getAsString(PAL.getRetAttributes());
+    if (PAL.getRetAttributes().hasAttributes())
+      Out << ' ' << PAL.getRetAttributes().getAsString();
 
     // If possible, print out the short form of the call instruction.  We can
     // only do this if the first argument is a pointer to a nonvararg function,
@@ -1874,8 +1874,8 @@ void AssemblyWriter::printInstruction(const Instruction &I) {
       writeParamOperand(CI->getArgOperand(op), PAL.getParamAttributes(op + 1));
     }
     Out << ')';
-    if (PAL.getFnAttributes() != Attribute::None)
-      Out << ' ' << Attribute::getAsString(PAL.getFnAttributes());
+    if (PAL.getFnAttributes().hasAttributes())
+      Out << ' ' << PAL.getFnAttributes().getAsString();
   } else if (const InvokeInst *II = dyn_cast<InvokeInst>(&I)) {
     Operand = II->getCalledValue();
     PointerType *PTy = cast<PointerType>(Operand->getType());
@@ -1884,24 +1884,13 @@ void AssemblyWriter::printInstruction(const Instruction &I) {
     const AttrListPtr &PAL = II->getAttributes();
 
     // Print the calling convention being used.
-    switch (II->getCallingConv()) {
-    case CallingConv::C: break;   // default
-    case CallingConv::Fast:  Out << " fastcc"; break;
-    case CallingConv::Cold:  Out << " coldcc"; break;
-    case CallingConv::X86_StdCall:  Out << " x86_stdcallcc"; break;
-    case CallingConv::X86_FastCall: Out << " x86_fastcallcc"; break;
-    case CallingConv::X86_ThisCall: Out << " x86_thiscallcc"; break;
-    case CallingConv::ARM_APCS:     Out << " arm_apcscc "; break;
-    case CallingConv::ARM_AAPCS:    Out << " arm_aapcscc "; break;
-    case CallingConv::ARM_AAPCS_VFP:Out << " arm_aapcs_vfpcc "; break;
-    case CallingConv::MSP430_INTR:  Out << " msp430_intrcc "; break;
-    case CallingConv::PTX_Kernel:   Out << " ptx_kernel"; break;
-    case CallingConv::PTX_Device:   Out << " ptx_device"; break;
-    default: Out << " cc" << II->getCallingConv(); break;
+    if (II->getCallingConv() != CallingConv::C) {
+      Out << " ";
+      PrintCallingConv(II->getCallingConv(), Out);
     }
 
-    if (PAL.getRetAttributes() != Attribute::None)
-      Out << ' ' << Attribute::getAsString(PAL.getRetAttributes());
+    if (PAL.getRetAttributes().hasAttributes())
+      Out << ' ' << PAL.getRetAttributes().getAsString();
 
     // If possible, print out the short form of the invoke instruction. We can
     // only do this if the first argument is a pointer to a nonvararg function,
@@ -1925,8 +1914,8 @@ void AssemblyWriter::printInstruction(const Instruction &I) {
     }
 
     Out << ')';
-    if (PAL.getFnAttributes() != Attribute::None)
-      Out << ' ' << Attribute::getAsString(PAL.getFnAttributes());
+    if (PAL.getFnAttributes().hasAttributes())
+      Out << ' ' << PAL.getFnAttributes().getAsString();
 
     Out << "\n          to ";
     writeOperand(II->getNormalDest(), true);
diff --git a/lib/VMCore/Attributes.cpp b/lib/VMCore/Attributes.cpp
index c8219eb..f1268e6 100644
--- a/lib/VMCore/Attributes.cpp
+++ b/lib/VMCore/Attributes.cpp
@@ -7,11 +7,14 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This file implements the AttributesList class and Attribute utilities.
+// This file implements the Attributes, AttributeImpl, AttrBuilder,
+// AttributeListImpl, and AttrListPtr classes.
 //
 //===----------------------------------------------------------------------===//
 
 #include "llvm/Attributes.h"
+#include "AttributesImpl.h"
+#include "LLVMContextImpl.h"
 #include "llvm/Type.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/FoldingSet.h"
@@ -23,215 +26,382 @@
 using namespace llvm;
 
 //===----------------------------------------------------------------------===//
-// Attribute Function Definitions
+// Attributes Implementation
 //===----------------------------------------------------------------------===//
 
-std::string Attribute::getAsString(Attributes Attrs) {
+Attributes Attributes::get(LLVMContext &Context, ArrayRef<AttrVal> Vals) {
+  AttrBuilder B;
+  for (ArrayRef<AttrVal>::iterator I = Vals.begin(), E = Vals.end();
+       I != E; ++I)
+    B.addAttribute(*I);
+  return Attributes::get(Context, B);
+}
+
+Attributes Attributes::get(LLVMContext &Context, AttrBuilder &B) {
+  // If there are no attributes, return an empty Attributes class.
+  if (!B.hasAttributes())
+    return Attributes();
+
+  // Otherwise, build a key to look up the existing attributes.
+  LLVMContextImpl *pImpl = Context.pImpl;
+  FoldingSetNodeID ID;
+  ID.AddInteger(B.Raw());
+
+  void *InsertPoint;
+  AttributesImpl *PA = pImpl->AttrsSet.FindNodeOrInsertPos(ID, InsertPoint);
+
+  if (!PA) {
+    // If we didn't find any existing attributes of the same shape then create a
+    // new one and insert it.
+    PA = new AttributesImpl(B.Raw());
+    pImpl->AttrsSet.InsertNode(PA, InsertPoint);
+  }
+
+  // Return the AttributesList that we found or created.
+  return Attributes(PA);
+}
+
+bool Attributes::hasAttribute(AttrVal Val) const {
+  return Attrs && Attrs->hasAttribute(Val);
+}
+
+bool Attributes::hasAttributes() const {
+  return Attrs && Attrs->hasAttributes();
+}
+
+bool Attributes::hasAttributes(const Attributes &A) const {
+  return Attrs && Attrs->hasAttributes(A);
+}
+
+/// This returns the alignment field of an attribute as a byte alignment value.
+unsigned Attributes::getAlignment() const {
+  if (!hasAttribute(Attributes::Alignment))
+    return 0;
+  return 1U << ((Attrs->getAlignment() >> 16) - 1);
+}
+
+/// This returns the stack alignment field of an attribute as a byte alignment
+/// value.
+unsigned Attributes::getStackAlignment() const {
+  if (!hasAttribute(Attributes::StackAlignment))
+    return 0;
+  return 1U << ((Attrs->getStackAlignment() >> 26) - 1);
+}
+
+uint64_t Attributes::Raw() const {
+  return Attrs ? Attrs->Raw() : 0;
+}
+
+Attributes Attributes::typeIncompatible(Type *Ty) {
+  AttrBuilder Incompatible;
+
+  if (!Ty->isIntegerTy())
+    // Attributes that only apply to integers.
+    Incompatible.addAttribute(Attributes::SExt)
+      .addAttribute(Attributes::ZExt);
+
+  if (!Ty->isPointerTy())
+    // Attributes that only apply to pointers.
+    Incompatible.addAttribute(Attributes::ByVal)
+      .addAttribute(Attributes::Nest)
+      .addAttribute(Attributes::NoAlias)
+      .addAttribute(Attributes::NoCapture)
+      .addAttribute(Attributes::StructRet);
+
+  return Attributes::get(Ty->getContext(), Incompatible);
+}
+
+/// encodeLLVMAttributesForBitcode - This returns an integer containing an
+/// encoding of all the LLVM attributes found in the given attribute bitset.
+/// Any change to this encoding is a breaking change to bitcode compatibility.
+uint64_t Attributes::encodeLLVMAttributesForBitcode(Attributes Attrs) {
+  // FIXME: It doesn't make sense to store the alignment information as an
+  // expanded out value, we should store it as a log2 value.  However, we can't
+  // just change that here without breaking bitcode compatibility.  If this ever
+  // becomes a problem in practice, we should introduce new tag numbers in the
+  // bitcode file and have those tags use a more efficiently encoded alignment
+  // field.
+
+  // Store the alignment in the bitcode as a 16-bit raw value instead of a 5-bit
+  // log2 encoded value. Shift the bits above the alignment up by 11 bits.
+  uint64_t EncodedAttrs = Attrs.Raw() & 0xffff;
+  if (Attrs.hasAttribute(Attributes::Alignment))
+    EncodedAttrs |= Attrs.getAlignment() << 16;
+  EncodedAttrs |= (Attrs.Raw() & (0xffffULL << 21)) << 11;
+  return EncodedAttrs;
+}
+
+/// decodeLLVMAttributesForBitcode - This returns an attribute bitset containing
+/// the LLVM attributes that have been decoded from the given integer.  This
+/// function must stay in sync with 'encodeLLVMAttributesForBitcode'.
+Attributes Attributes::decodeLLVMAttributesForBitcode(LLVMContext &C,
+                                                      uint64_t EncodedAttrs) {
+  // The alignment is stored as a 16-bit raw value from bits 31--16.  We shift
+  // the bits above 31 down by 11 bits.
+  unsigned Alignment = (EncodedAttrs & (0xffffULL << 16)) >> 16;
+  assert((!Alignment || isPowerOf2_32(Alignment)) &&
+         "Alignment must be a power of two.");
+
+  AttrBuilder B(EncodedAttrs & 0xffff);
+  if (Alignment)
+    B.addAlignmentAttr(Alignment);
+  B.addRawValue((EncodedAttrs & (0xffffULL << 32)) >> 11);
+  return Attributes::get(C, B);
+}
+
+std::string Attributes::getAsString() const {
   std::string Result;
-  if (Attrs & Attribute::ZExt)
+  if (hasAttribute(Attributes::ZExt))
     Result += "zeroext ";
-  if (Attrs & Attribute::SExt)
+  if (hasAttribute(Attributes::SExt))
     Result += "signext ";
-  if (Attrs & Attribute::NoReturn)
+  if (hasAttribute(Attributes::NoReturn))
     Result += "noreturn ";
-  if (Attrs & Attribute::NoUnwind)
+  if (hasAttribute(Attributes::NoUnwind))
     Result += "nounwind ";
-  if (Attrs & Attribute::UWTable)
+  if (hasAttribute(Attributes::UWTable))
     Result += "uwtable ";
-  if (Attrs & Attribute::ReturnsTwice)
+  if (hasAttribute(Attributes::ReturnsTwice))
     Result += "returns_twice ";
-  if (Attrs & Attribute::InReg)
+  if (hasAttribute(Attributes::InReg))
     Result += "inreg ";
-  if (Attrs & Attribute::NoAlias)
+  if (hasAttribute(Attributes::NoAlias))
     Result += "noalias ";
-  if (Attrs & Attribute::NoCapture)
+  if (hasAttribute(Attributes::NoCapture))
     Result += "nocapture ";
-  if (Attrs & Attribute::StructRet)
+  if (hasAttribute(Attributes::StructRet))
     Result += "sret ";
-  if (Attrs & Attribute::ByVal)
+  if (hasAttribute(Attributes::ByVal))
     Result += "byval ";
-  if (Attrs & Attribute::Nest)
+  if (hasAttribute(Attributes::Nest))
     Result += "nest ";
-  if (Attrs & Attribute::ReadNone)
+  if (hasAttribute(Attributes::ReadNone))
     Result += "readnone ";
-  if (Attrs & Attribute::ReadOnly)
+  if (hasAttribute(Attributes::ReadOnly))
     Result += "readonly ";
-  if (Attrs & Attribute::OptimizeForSize)
+  if (hasAttribute(Attributes::OptimizeForSize))
     Result += "optsize ";
-  if (Attrs & Attribute::NoInline)
+  if (hasAttribute(Attributes::NoInline))
     Result += "noinline ";
-  if (Attrs & Attribute::InlineHint)
+  if (hasAttribute(Attributes::InlineHint))
     Result += "inlinehint ";
-  if (Attrs & Attribute::AlwaysInline)
+  if (hasAttribute(Attributes::AlwaysInline))
     Result += "alwaysinline ";
-  if (Attrs & Attribute::StackProtect)
+  if (hasAttribute(Attributes::StackProtect))
     Result += "ssp ";
-  if (Attrs & Attribute::StackProtectReq)
+  if (hasAttribute(Attributes::StackProtectReq))
     Result += "sspreq ";
-  if (Attrs & Attribute::NoRedZone)
+  if (hasAttribute(Attributes::NoRedZone))
     Result += "noredzone ";
-  if (Attrs & Attribute::NoImplicitFloat)
+  if (hasAttribute(Attributes::NoImplicitFloat))
     Result += "noimplicitfloat ";
-  if (Attrs & Attribute::Naked)
+  if (hasAttribute(Attributes::Naked))
     Result += "naked ";
-  if (Attrs & Attribute::NonLazyBind)
+  if (hasAttribute(Attributes::NonLazyBind))
     Result += "nonlazybind ";
-  if (Attrs & Attribute::AddressSafety)
+  if (hasAttribute(Attributes::AddressSafety))
     Result += "address_safety ";
-  if (Attrs & Attribute::StackAlignment) {
+  if (hasAttribute(Attributes::MinSize))
+    Result += "minsize ";
+  if (hasAttribute(Attributes::StackAlignment)) {
     Result += "alignstack(";
-    Result += utostr(Attribute::getStackAlignmentFromAttrs(Attrs));
+    Result += utostr(getStackAlignment());
     Result += ") ";
   }
-  if (Attrs & Attribute::Alignment) {
+  if (hasAttribute(Attributes::Alignment)) {
     Result += "align ";
-    Result += utostr(Attribute::getAlignmentFromAttrs(Attrs));
+    Result += utostr(getAlignment());
     Result += " ";
   }
-  if (Attrs & Attribute::IANSDialect)
-    Result += "ia_nsdialect ";
-
   // Trim the trailing space.
   assert(!Result.empty() && "Unknown attribute!");
   Result.erase(Result.end()-1);
   return Result;
 }
 
-Attributes Attribute::typeIncompatible(Type *Ty) {
-  Attributes Incompatible = None;
-  
-  if (!Ty->isIntegerTy())
-    // Attributes that only apply to integers.
-    Incompatible |= SExt | ZExt;
-  
-  if (!Ty->isPointerTy())
-    // Attributes that only apply to pointers.
-    Incompatible |= ByVal | Nest | NoAlias | StructRet | NoCapture;
-  
-  return Incompatible;
+//===----------------------------------------------------------------------===//
+// AttrBuilder Implementation
+//===----------------------------------------------------------------------===//
+
+AttrBuilder &AttrBuilder::addAttribute(Attributes::AttrVal Val){
+  Bits |= AttributesImpl::getAttrMask(Val);
+  return *this;
+}
+
+AttrBuilder &AttrBuilder::addRawValue(uint64_t Val) {
+  Bits |= Val;
+  return *this;
+}
+
+AttrBuilder &AttrBuilder::addAlignmentAttr(unsigned Align) {
+  if (Align == 0) return *this;
+  assert(isPowerOf2_32(Align) && "Alignment must be a power of two.");
+  assert(Align <= 0x40000000 && "Alignment too large.");
+  Bits |= (Log2_32(Align) + 1) << 16;
+  return *this;
+}
+AttrBuilder &AttrBuilder::addStackAlignmentAttr(unsigned Align){
+  // Default alignment, allow the target to define how to align it.
+  if (Align == 0) return *this;
+  assert(isPowerOf2_32(Align) && "Alignment must be a power of two.");
+  assert(Align <= 0x100 && "Alignment too large.");
+  Bits |= (Log2_32(Align) + 1) << 26;
+  return *this;
+}
+
+AttrBuilder &AttrBuilder::removeAttribute(Attributes::AttrVal Val) {
+  Bits &= ~AttributesImpl::getAttrMask(Val);
+  return *this;
+}
+
+AttrBuilder &AttrBuilder::addAttributes(const Attributes &A) {
+  Bits |= A.Raw();
+  return *this;
+}
+
+AttrBuilder &AttrBuilder::removeAttributes(const Attributes &A){
+  Bits &= ~A.Raw();
+  return *this;
+}
+
+bool AttrBuilder::hasAttribute(Attributes::AttrVal A) const {
+  return Bits & AttributesImpl::getAttrMask(A);
+}
+
+bool AttrBuilder::hasAttributes() const {
+  return Bits != 0;
+}
+bool AttrBuilder::hasAttributes(const Attributes &A) const {
+  return Bits & A.Raw();
+}
+bool AttrBuilder::hasAlignmentAttr() const {
+  return Bits & AttributesImpl::getAttrMask(Attributes::Alignment);
+}
+
+uint64_t AttrBuilder::getAlignment() const {
+  if (!hasAlignmentAttr())
+    return 0;
+  return 1U <<
+    (((Bits & AttributesImpl::getAttrMask(Attributes::Alignment)) >> 16) - 1);
+}
+
+uint64_t AttrBuilder::getStackAlignment() const {
+  if (!hasAlignmentAttr())
+    return 0;
+  return 1U <<
+    (((Bits & AttributesImpl::getAttrMask(Attributes::StackAlignment))>>26)-1);
 }
 
 //===----------------------------------------------------------------------===//
-// AttributeListImpl Definition
+// AttributeImpl Definition
 //===----------------------------------------------------------------------===//
 
-namespace llvm {
-  class AttributeListImpl;
+uint64_t AttributesImpl::getAttrMask(uint64_t Val) {
+  switch (Val) {
+  case Attributes::None:            return 0;
+  case Attributes::ZExt:            return 1 << 0;
+  case Attributes::SExt:            return 1 << 1;
+  case Attributes::NoReturn:        return 1 << 2;
+  case Attributes::InReg:           return 1 << 3;
+  case Attributes::StructRet:       return 1 << 4;
+  case Attributes::NoUnwind:        return 1 << 5;
+  case Attributes::NoAlias:         return 1 << 6;
+  case Attributes::ByVal:           return 1 << 7;
+  case Attributes::Nest:            return 1 << 8;
+  case Attributes::ReadNone:        return 1 << 9;
+  case Attributes::ReadOnly:        return 1 << 10;
+  case Attributes::NoInline:        return 1 << 11;
+  case Attributes::AlwaysInline:    return 1 << 12;
+  case Attributes::OptimizeForSize: return 1 << 13;
+  case Attributes::StackProtect:    return 1 << 14;
+  case Attributes::StackProtectReq: return 1 << 15;
+  case Attributes::Alignment:       return 31 << 16;
+  case Attributes::NoCapture:       return 1 << 21;
+  case Attributes::NoRedZone:       return 1 << 22;
+  case Attributes::NoImplicitFloat: return 1 << 23;
+  case Attributes::Naked:           return 1 << 24;
+  case Attributes::InlineHint:      return 1 << 25;
+  case Attributes::StackAlignment:  return 7 << 26;
+  case Attributes::ReturnsTwice:    return 1 << 29;
+  case Attributes::UWTable:         return 1 << 30;
+  case Attributes::NonLazyBind:     return 1U << 31;
+  case Attributes::AddressSafety:   return 1ULL << 32;
+  case Attributes::MinSize:         return 1ULL << 33;
+  }
+  llvm_unreachable("Unsupported attribute type");
 }
 
-static ManagedStatic<FoldingSet<AttributeListImpl> > AttributesLists;
+bool AttributesImpl::hasAttribute(uint64_t A) const {
+  return (Bits & getAttrMask(A)) != 0;
+}
 
-namespace llvm {
-static ManagedStatic<sys::SmartMutex<true> > ALMutex;
+bool AttributesImpl::hasAttributes() const {
+  return Bits != 0;
+}
 
-class AttributeListImpl : public FoldingSetNode {
-  sys::cas_flag RefCount;
-  
-  // AttributesList is uniqued, these should not be publicly available.
-  void operator=(const AttributeListImpl &); // Do not implement
-  AttributeListImpl(const AttributeListImpl &); // Do not implement
-  ~AttributeListImpl();                        // Private implementation
-public:
-  SmallVector<AttributeWithIndex, 4> Attrs;
-  
-  AttributeListImpl(ArrayRef<AttributeWithIndex> attrs)
-    : Attrs(attrs.begin(), attrs.end()) {
-    RefCount = 0;
-  }
-  
-  void AddRef() {
-    sys::SmartScopedLock<true> Lock(*ALMutex);
-    ++RefCount;
-  }
-  void DropRef() {
-    sys::SmartScopedLock<true> Lock(*ALMutex);
-    if (!AttributesLists.isConstructed())
-      return;
-    sys::cas_flag new_val = --RefCount;
-    if (new_val == 0)
-      delete this;
-  }
-  
-  void Profile(FoldingSetNodeID &ID) const {
-    Profile(ID, Attrs);
-  }
-  static void Profile(FoldingSetNodeID &ID, ArrayRef<AttributeWithIndex> Attrs){
-    for (unsigned i = 0, e = Attrs.size(); i != e; ++i) {
-      ID.AddInteger(Attrs[i].Attrs.Raw());
-      ID.AddInteger(Attrs[i].Index);
-    }
-  }
-};
+bool AttributesImpl::hasAttributes(const Attributes &A) const {
+  return Bits & A.Raw();        // FIXME: Raw() won't work here in the future.
 }
 
-AttributeListImpl::~AttributeListImpl() {
-  // NOTE: Lock must be acquired by caller.
-  AttributesLists->RemoveNode(this);
+uint64_t AttributesImpl::getAlignment() const {
+  return Bits & getAttrMask(Attributes::Alignment);
 }
 
+uint64_t AttributesImpl::getStackAlignment() const {
+  return Bits & getAttrMask(Attributes::StackAlignment);
+}
 
-AttrListPtr AttrListPtr::get(ArrayRef<AttributeWithIndex> Attrs) {
+//===----------------------------------------------------------------------===//
+// AttributeListImpl Definition
+//===----------------------------------------------------------------------===//
+
+AttrListPtr AttrListPtr::get(LLVMContext &C,
+                             ArrayRef<AttributeWithIndex> Attrs) {
   // If there are no attributes then return a null AttributesList pointer.
   if (Attrs.empty())
     return AttrListPtr();
-  
+
 #ifndef NDEBUG
   for (unsigned i = 0, e = Attrs.size(); i != e; ++i) {
-    assert(Attrs[i].Attrs != Attribute::None && 
+    assert(Attrs[i].Attrs.hasAttributes() &&
            "Pointless attribute!");
     assert((!i || Attrs[i-1].Index < Attrs[i].Index) &&
            "Misordered AttributesList!");
   }
 #endif
-  
+
   // Otherwise, build a key to look up the existing attributes.
+  LLVMContextImpl *pImpl = C.pImpl;
   FoldingSetNodeID ID;
   AttributeListImpl::Profile(ID, Attrs);
-  void *InsertPos;
-  
-  sys::SmartScopedLock<true> Lock(*ALMutex);
-  
-  AttributeListImpl *PAL =
-    AttributesLists->FindNodeOrInsertPos(ID, InsertPos);
-  
+
+  void *InsertPoint;
+  AttributeListImpl *PA = pImpl->AttrsLists.FindNodeOrInsertPos(ID,
+                                                                InsertPoint);
+
   // If we didn't find any existing attributes of the same shape then
   // create a new one and insert it.
-  if (!PAL) {
-    PAL = new AttributeListImpl(Attrs);
-    AttributesLists->InsertNode(PAL, InsertPos);
+  if (!PA) {
+    PA = new AttributeListImpl(Attrs);
+    pImpl->AttrsLists.InsertNode(PA, InsertPoint);
   }
-  
+
   // Return the AttributesList that we found or created.
-  return AttrListPtr(PAL);
+  return AttrListPtr(PA);
 }
 
-
 //===----------------------------------------------------------------------===//
 // AttrListPtr Method Implementations
 //===----------------------------------------------------------------------===//
 
-AttrListPtr::AttrListPtr(AttributeListImpl *LI) : AttrList(LI) {
-  if (LI) LI->AddRef();
-}
-
-AttrListPtr::AttrListPtr(const AttrListPtr &P) : AttrList(P.AttrList) {
-  if (AttrList) AttrList->AddRef();  
-}
-
 const AttrListPtr &AttrListPtr::operator=(const AttrListPtr &RHS) {
-  sys::SmartScopedLock<true> Lock(*ALMutex);
   if (AttrList == RHS.AttrList) return *this;
-  if (AttrList) AttrList->DropRef();
+
   AttrList = RHS.AttrList;
-  if (AttrList) AttrList->AddRef();
   return *this;
 }
 
-AttrListPtr::~AttrListPtr() {
-  if (AttrList) AttrList->DropRef();
-}
-
-/// getNumSlots - Return the number of slots used in this attribute list. 
+/// getNumSlots - Return the number of slots used in this attribute list.
 /// This is the number of arguments that have an attribute set on them
 /// (including the function itself).
 unsigned AttrListPtr::getNumSlots() const {
@@ -245,48 +415,60 @@ const AttributeWithIndex &AttrListPtr::getSlot(unsigned Slot) const {
   return AttrList->Attrs[Slot];
 }
 
-
-/// getAttributes - The attributes for the specified index are
-/// returned.  Attributes for the result are denoted with Idx = 0.
-/// Function notes are denoted with idx = ~0.
+/// getAttributes - The attributes for the specified index are returned.
+/// Attributes for the result are denoted with Idx = 0.  Function notes are
+/// denoted with idx = ~0.
 Attributes AttrListPtr::getAttributes(unsigned Idx) const {
-  if (AttrList == 0) return Attribute::None;
-  
+  if (AttrList == 0) return Attributes();
+
   const SmallVector<AttributeWithIndex, 4> &Attrs = AttrList->Attrs;
   for (unsigned i = 0, e = Attrs.size(); i != e && Attrs[i].Index <= Idx; ++i)
     if (Attrs[i].Index == Idx)
       return Attrs[i].Attrs;
-  return Attribute::None;
+
+  return Attributes();
 }
 
 /// hasAttrSomewhere - Return true if the specified attribute is set for at
 /// least one parameter or for the return value.
-bool AttrListPtr::hasAttrSomewhere(Attributes Attr) const {
+bool AttrListPtr::hasAttrSomewhere(Attributes::AttrVal Attr) const {
   if (AttrList == 0) return false;
-  
+
   const SmallVector<AttributeWithIndex, 4> &Attrs = AttrList->Attrs;
   for (unsigned i = 0, e = Attrs.size(); i != e; ++i)
-    if (Attrs[i].Attrs & Attr)
+    if (Attrs[i].Attrs.hasAttribute(Attr))
       return true;
+
   return false;
 }
 
+unsigned AttrListPtr::getNumAttrs() const {
+  return AttrList ? AttrList->Attrs.size() : 0;
+}
+
+Attributes &AttrListPtr::getAttributesAtIndex(unsigned i) const {
+  assert(AttrList && "Trying to get an attribute from an empty list!");
+  assert(i < AttrList->Attrs.size() && "Index out of range!");
+  return AttrList->Attrs[i].Attrs;
+}
 
-AttrListPtr AttrListPtr::addAttr(unsigned Idx, Attributes Attrs) const {
+AttrListPtr AttrListPtr::addAttr(LLVMContext &C, unsigned Idx,
+                                 Attributes Attrs) const {
   Attributes OldAttrs = getAttributes(Idx);
 #ifndef NDEBUG
   // FIXME it is not obvious how this should work for alignment.
   // For now, say we can't change a known alignment.
-  Attributes OldAlign = OldAttrs & Attribute::Alignment;
-  Attributes NewAlign = Attrs & Attribute::Alignment;
+  unsigned OldAlign = OldAttrs.getAlignment();
+  unsigned NewAlign = Attrs.getAlignment();
   assert((!OldAlign || !NewAlign || OldAlign == NewAlign) &&
          "Attempt to change alignment!");
 #endif
-  
-  Attributes NewAttrs = OldAttrs | Attrs;
-  if (NewAttrs == OldAttrs)
+
+  AttrBuilder NewAttrs =
+    AttrBuilder(OldAttrs).addAttributes(Attrs);
+  if (NewAttrs == AttrBuilder(OldAttrs))
     return *this;
-  
+
   SmallVector<AttributeWithIndex, 8> NewAttrList;
   if (AttrList == 0)
     NewAttrList.push_back(AttributeWithIndex::get(Idx, Attrs));
@@ -299,61 +481,67 @@ AttrListPtr AttrListPtr::addAttr(unsigned Idx, Attributes Attrs) const {
 
     // If there are attributes already at this index, merge them in.
     if (i != e && OldAttrList[i].Index == Idx) {
-      Attrs |= OldAttrList[i].Attrs;
+      Attrs =
+        Attributes::get(C, AttrBuilder(Attrs).
+                        addAttributes(OldAttrList[i].Attrs));
       ++i;
     }
-    
+
     NewAttrList.push_back(AttributeWithIndex::get(Idx, Attrs));
-    
+
     // Copy attributes for arguments after this one.
-    NewAttrList.insert(NewAttrList.end(), 
+    NewAttrList.insert(NewAttrList.end(),
                        OldAttrList.begin()+i, OldAttrList.end());
   }
-  
-  return get(NewAttrList);
+
+  return get(C, NewAttrList);
 }
 
-AttrListPtr AttrListPtr::removeAttr(unsigned Idx, Attributes Attrs) const {
+AttrListPtr AttrListPtr::removeAttr(LLVMContext &C, unsigned Idx,
+                                    Attributes Attrs) const {
 #ifndef NDEBUG
   // FIXME it is not obvious how this should work for alignment.
   // For now, say we can't pass in alignment, which no current use does.
-  assert(!(Attrs & Attribute::Alignment) && "Attempt to exclude alignment!");
+  assert(!Attrs.hasAttribute(Attributes::Alignment) &&
+         "Attempt to exclude alignment!");
 #endif
   if (AttrList == 0) return AttrListPtr();
-  
+
   Attributes OldAttrs = getAttributes(Idx);
-  Attributes NewAttrs = OldAttrs & ~Attrs;
-  if (NewAttrs == OldAttrs)
+  AttrBuilder NewAttrs =
+    AttrBuilder(OldAttrs).removeAttributes(Attrs);
+  if (NewAttrs == AttrBuilder(OldAttrs))
     return *this;
 
   SmallVector<AttributeWithIndex, 8> NewAttrList;
   const SmallVector<AttributeWithIndex, 4> &OldAttrList = AttrList->Attrs;
   unsigned i = 0, e = OldAttrList.size();
-  
+
   // Copy attributes for arguments before this one.
   for (; i != e && OldAttrList[i].Index < Idx; ++i)
     NewAttrList.push_back(OldAttrList[i]);
-  
+
   // If there are attributes already at this index, merge them in.
   assert(OldAttrList[i].Index == Idx && "Attribute isn't set?");
-  Attrs = OldAttrList[i].Attrs & ~Attrs;
+  Attrs = Attributes::get(C, AttrBuilder(OldAttrList[i].Attrs).
+                          removeAttributes(Attrs));
   ++i;
-  if (Attrs)  // If any attributes left for this parameter, add them.
+  if (Attrs.hasAttributes()) // If any attributes left for this param, add them.
     NewAttrList.push_back(AttributeWithIndex::get(Idx, Attrs));
-  
+
   // Copy attributes for arguments after this one.
-  NewAttrList.insert(NewAttrList.end(), 
+  NewAttrList.insert(NewAttrList.end(),
                      OldAttrList.begin()+i, OldAttrList.end());
-  
-  return get(NewAttrList);
+
+  return get(C, NewAttrList);
 }
 
 void AttrListPtr::dump() const {
   dbgs() << "PAL[ ";
   for (unsigned i = 0; i < getNumSlots(); ++i) {
     const AttributeWithIndex &PAWI = getSlot(i);
-    dbgs() << "{" << PAWI.Index << "," << PAWI.Attrs << "} ";
+    dbgs() << "{" << PAWI.Index << "," << PAWI.Attrs.getAsString() << "} ";
   }
-  
+
   dbgs() << "]\n";
 }
diff --git a/lib/VMCore/AttributesImpl.h b/lib/VMCore/AttributesImpl.h
new file mode 100644
index 0000000..5c107e1
--- /dev/null
+++ b/lib/VMCore/AttributesImpl.h
@@ -0,0 +1,71 @@
+//===-- AttributesImpl.h - Attributes Internals -----------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines various helper methods and classes used by LLVMContextImpl
+// for creating and managing attributes.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ATTRIBUTESIMPL_H
+#define LLVM_ATTRIBUTESIMPL_H
+
+#include "llvm/Attributes.h"
+#include "llvm/ADT/FoldingSet.h"
+
+namespace llvm {
+
+class AttributesImpl : public FoldingSetNode {
+  uint64_t Bits;                // FIXME: We will be expanding this.
+public:
+  AttributesImpl(uint64_t bits) : Bits(bits) {}
+
+  bool hasAttribute(uint64_t A) const;
+
+  bool hasAttributes() const;
+  bool hasAttributes(const Attributes &A) const;
+
+  uint64_t getAlignment() const;
+  uint64_t getStackAlignment() const;
+
+  uint64_t Raw() const { return Bits; } // FIXME: Remove.
+
+  static uint64_t getAttrMask(uint64_t Val);
+
+  void Profile(FoldingSetNodeID &ID) const {
+    Profile(ID, Bits);
+  }
+  static void Profile(FoldingSetNodeID &ID, uint64_t Bits) {
+    ID.AddInteger(Bits);
+  }
+};
+
+class AttributeListImpl : public FoldingSetNode {
+  // AttributesList is uniqued, these should not be publicly available.
+  void operator=(const AttributeListImpl &) LLVM_DELETED_FUNCTION;
+  AttributeListImpl(const AttributeListImpl &) LLVM_DELETED_FUNCTION;
+public:
+  SmallVector<AttributeWithIndex, 4> Attrs;
+
+  AttributeListImpl(ArrayRef<AttributeWithIndex> attrs)
+    : Attrs(attrs.begin(), attrs.end()) {}
+
+  void Profile(FoldingSetNodeID &ID) const {
+    Profile(ID, Attrs);
+  }
+  static void Profile(FoldingSetNodeID &ID, ArrayRef<AttributeWithIndex> Attrs){
+    for (unsigned i = 0, e = Attrs.size(); i != e; ++i) {
+      ID.AddInteger(Attrs[i].Attrs.Raw());
+      ID.AddInteger(Attrs[i].Index);
+    }
+  }
+};
+
+} // end llvm namespace
+
+#endif
diff --git a/lib/VMCore/AutoUpgrade.cpp b/lib/VMCore/AutoUpgrade.cpp
index 094ca75..5fff460 100644
--- a/lib/VMCore/AutoUpgrade.cpp
+++ b/lib/VMCore/AutoUpgrade.cpp
@@ -148,7 +148,8 @@ bool llvm::UpgradeIntrinsicFunction(Function *F, Function *&NewFn) {
   if (NewFn)
     F = NewFn;
   if (unsigned id = F->getIntrinsicID())
-    F->setAttributes(Intrinsic::getAttributes((Intrinsic::ID)id));
+    F->setAttributes(Intrinsic::getAttributes(F->getContext(),
+                                              (Intrinsic::ID)id));
   return Upgraded;
 }
 
diff --git a/lib/VMCore/CMakeLists.txt b/lib/VMCore/CMakeLists.txt
index 6a20be6..06eab0e 100644
--- a/lib/VMCore/CMakeLists.txt
+++ b/lib/VMCore/CMakeLists.txt
@@ -1,5 +1,3 @@
-set(LLVM_REQUIRES_RTTI 1)
-
 add_llvm_library(LLVMCore
   AsmWriter.cpp
   Attributes.cpp
@@ -8,6 +6,7 @@ add_llvm_library(LLVMCore
   ConstantFold.cpp
   Constants.cpp
   Core.cpp
+  DataLayout.cpp
   DebugInfo.cpp
   DebugLoc.cpp
   DIBuilder.cpp
@@ -32,6 +31,7 @@ add_llvm_library(LLVMCore
   PrintModulePass.cpp
   Type.cpp
   TypeFinder.cpp
+  TargetTransformInfo.cpp
   Use.cpp
   User.cpp
   Value.cpp
@@ -42,7 +42,7 @@ add_llvm_library(LLVMCore
 
 # Workaround: It takes over 20 minutes to compile with msvc10.
 # FIXME: Suppressing optimizations to core libraries would not be good thing.
-if( MSVC_VERSION EQUAL 1600 )
+if( MSVC_VERSION LESS 1700 )
 set_property(
   SOURCE Function.cpp
   PROPERTY COMPILE_FLAGS "/Og-"
diff --git a/lib/VMCore/ConstantFold.cpp b/lib/VMCore/ConstantFold.cpp
index 8e82876..fe3edac 100644
--- a/lib/VMCore/ConstantFold.cpp
+++ b/lib/VMCore/ConstantFold.cpp
@@ -12,7 +12,7 @@
 // ConstantExpr::get* methods to automatically fold constants when possible.
 //
 // The current constant folding implementation is implemented in two pieces: the
-// pieces that don't need TargetData, and the pieces that do. This is to avoid
+// pieces that don't need DataLayout, and the pieces that do. This is to avoid
 // a dependence in VMCore on Target.
 //
 //===----------------------------------------------------------------------===//
@@ -87,9 +87,13 @@ foldConstantCastPair(
   Instruction::CastOps firstOp = Instruction::CastOps(Op->getOpcode());
   Instruction::CastOps secondOp = Instruction::CastOps(opc);
 
+  // Assume that pointers are never more than 64 bits wide.
+  IntegerType *FakeIntPtrTy = Type::getInt64Ty(DstTy->getContext());
+
   // Let CastInst::isEliminableCastPair do the heavy lifting.
   return CastInst::isEliminableCastPair(firstOp, secondOp, SrcTy, MidTy, DstTy,
-                                        Type::getInt64Ty(DstTy->getContext()));
+                                        FakeIntPtrTy, FakeIntPtrTy,
+                                        FakeIntPtrTy);
 }
 
 static Constant *FoldBitCast(Constant *V, Type *DestTy) {
@@ -514,10 +518,6 @@ Constant *llvm::ConstantFoldCastInstruction(unsigned opc, Constant *V,
     return UndefValue::get(DestTy);
   }
 
-  // No compile-time operations on this type yet.
-  if (V->getType()->isPPC_FP128Ty() || DestTy->isPPC_FP128Ty())
-    return 0;
-
   if (V->isNullValue() && !DestTy->isX86_MMXTy())
     return Constant::getNullValue(DestTy);
 
@@ -576,6 +576,7 @@ Constant *llvm::ConstantFoldCastInstruction(unsigned opc, Constant *V,
                   DestTy->isDoubleTy() ? APFloat::IEEEdouble :
                   DestTy->isX86_FP80Ty() ? APFloat::x87DoubleExtended :
                   DestTy->isFP128Ty() ? APFloat::IEEEquad :
+                  DestTy->isPPC_FP128Ty() ? APFloat::PPCDoubleDouble :
                   APFloat::Bogus,
                   APFloat::rmNearestTiesToEven, &ignored);
       return ConstantFP::get(V->getContext(), Val);
@@ -646,7 +647,8 @@ Constant *llvm::ConstantFoldCastInstruction(unsigned opc, Constant *V,
   case Instruction::SIToFP:
     if (ConstantInt *CI = dyn_cast<ConstantInt>(V)) {
       APInt api = CI->getValue();
-      APFloat apf(APInt::getNullValue(DestTy->getPrimitiveSizeInBits()), true);
+      APFloat apf(APInt::getNullValue(DestTy->getPrimitiveSizeInBits()),
+                  !DestTy->isPPC_FP128Ty() /* isEEEE */);
       (void)apf.convertFromAPInt(api, 
                                  opc==Instruction::SIToFP,
                                  APFloat::rmNearestTiesToEven);
@@ -867,10 +869,6 @@ Constant *llvm::ConstantFoldInsertValueInstruction(Constant *Agg,
 
 Constant *llvm::ConstantFoldBinaryInstruction(unsigned Opcode,
                                               Constant *C1, Constant *C2) {
-  // No compile-time operations on this type yet.
-  if (C1->getType()->isPPC_FP128Ty())
-    return 0;
-
   // Handle UndefValue up front.
   if (isa<UndefValue>(C1) || isa<UndefValue>(C2)) {
     switch (Opcode) {
@@ -1273,10 +1271,6 @@ static FCmpInst::Predicate evaluateFCmpRelation(Constant *V1, Constant *V2) {
   assert(V1->getType() == V2->getType() &&
          "Cannot compare values of different types!");
 
-  // No compile-time operations on this type yet.
-  if (V1->getType()->isPPC_FP128Ty())
-    return FCmpInst::BAD_FCMP_PREDICATE;
-
   // Handle degenerate case quickly
   if (V1 == V2) return FCmpInst::FCMP_OEQ;
 
@@ -1602,10 +1596,6 @@ Constant *llvm::ConstantFoldCompareInstruction(unsigned short pred,
     return ConstantInt::get(ResultTy, CmpInst::isTrueWhenEqual(pred));
   }
 
-  // No compile-time operations on this type yet.
-  if (C1->getType()->isPPC_FP128Ty())
-    return 0;
-
   // icmp eq/ne(null,GV) -> false/true
   if (C1->isNullValue()) {
     if (const GlobalValue *GV = dyn_cast<GlobalValue>(C2))
diff --git a/lib/VMCore/Constants.cpp b/lib/VMCore/Constants.cpp
index a4e21e1..edd6a73 100644
--- a/lib/VMCore/Constants.cpp
+++ b/lib/VMCore/Constants.cpp
@@ -245,6 +245,33 @@ bool Constant::canTrap() const {
   }
 }
 
+/// isThreadDependent - Return true if the value can vary between threads.
+bool Constant::isThreadDependent() const {
+  SmallPtrSet<const Constant*, 64> Visited;
+  SmallVector<const Constant*, 64> WorkList;
+  WorkList.push_back(this);
+  Visited.insert(this);
+
+  while (!WorkList.empty()) {
+    const Constant *C = WorkList.pop_back_val();
+
+    if (const GlobalVariable *GV = dyn_cast<GlobalVariable>(C)) {
+      if (GV->isThreadLocal())
+        return true;
+    }
+
+    for (unsigned I = 0, E = C->getNumOperands(); I != E; ++I) {
+      const Constant *D = dyn_cast<Constant>(C->getOperand(I));
+      if (!D)
+        continue;
+      if (Visited.insert(D))
+        WorkList.push_back(D);
+    }
+  }
+
+  return false;
+}
+
 /// isConstantUsed - Return true if the constant has users other than constant
 /// exprs and other dangling things.
 bool Constant::isConstantUsed() const {
diff --git a/lib/VMCore/ConstantsContext.h b/lib/VMCore/ConstantsContext.h
index 8903a8f..996eb12 100644
--- a/lib/VMCore/ConstantsContext.h
+++ b/lib/VMCore/ConstantsContext.h
@@ -33,7 +33,7 @@ struct ConstantTraits;
 /// behind the scenes to implement unary constant exprs.
 class UnaryConstantExpr : public ConstantExpr {
   virtual void anchor();
-  void *operator new(size_t, unsigned);  // DO NOT IMPLEMENT
+  void *operator new(size_t, unsigned) LLVM_DELETED_FUNCTION;
 public:
   // allocate space for exactly one operand
   void *operator new(size_t s) {
@@ -50,7 +50,7 @@ public:
 /// behind the scenes to implement binary constant exprs.
 class BinaryConstantExpr : public ConstantExpr {
   virtual void anchor();
-  void *operator new(size_t, unsigned);  // DO NOT IMPLEMENT
+  void *operator new(size_t, unsigned) LLVM_DELETED_FUNCTION;
 public:
   // allocate space for exactly two operands
   void *operator new(size_t s) {
@@ -71,7 +71,7 @@ public:
 /// behind the scenes to implement select constant exprs.
 class SelectConstantExpr : public ConstantExpr {
   virtual void anchor();
-  void *operator new(size_t, unsigned);  // DO NOT IMPLEMENT
+  void *operator new(size_t, unsigned) LLVM_DELETED_FUNCTION;
 public:
   // allocate space for exactly three operands
   void *operator new(size_t s) {
@@ -92,7 +92,7 @@ public:
 /// extractelement constant exprs.
 class ExtractElementConstantExpr : public ConstantExpr {
   virtual void anchor();
-  void *operator new(size_t, unsigned);  // DO NOT IMPLEMENT
+  void *operator new(size_t, unsigned) LLVM_DELETED_FUNCTION;
 public:
   // allocate space for exactly two operands
   void *operator new(size_t s) {
@@ -113,7 +113,7 @@ public:
 /// insertelement constant exprs.
 class InsertElementConstantExpr : public ConstantExpr {
   virtual void anchor();
-  void *operator new(size_t, unsigned);  // DO NOT IMPLEMENT
+  void *operator new(size_t, unsigned) LLVM_DELETED_FUNCTION;
 public:
   // allocate space for exactly three operands
   void *operator new(size_t s) {
@@ -135,7 +135,7 @@ public:
 /// shufflevector constant exprs.
 class ShuffleVectorConstantExpr : public ConstantExpr {
   virtual void anchor();
-  void *operator new(size_t, unsigned);  // DO NOT IMPLEMENT
+  void *operator new(size_t, unsigned) LLVM_DELETED_FUNCTION;
 public:
   // allocate space for exactly three operands
   void *operator new(size_t s) {
@@ -160,7 +160,7 @@ public:
 /// extractvalue constant exprs.
 class ExtractValueConstantExpr : public ConstantExpr {
   virtual void anchor();
-  void *operator new(size_t, unsigned);  // DO NOT IMPLEMENT
+  void *operator new(size_t, unsigned) LLVM_DELETED_FUNCTION;
 public:
   // allocate space for exactly one operand
   void *operator new(size_t s) {
@@ -186,7 +186,7 @@ public:
 /// insertvalue constant exprs.
 class InsertValueConstantExpr : public ConstantExpr {
   virtual void anchor();
-  void *operator new(size_t, unsigned);  // DO NOT IMPLEMENT
+  void *operator new(size_t, unsigned) LLVM_DELETED_FUNCTION;
 public:
   // allocate space for exactly one operand
   void *operator new(size_t s) {
@@ -234,7 +234,7 @@ public:
 // needed in order to store the predicate value for these instructions.
 class CompareConstantExpr : public ConstantExpr {
   virtual void anchor();
-  void *operator new(size_t, unsigned);  // DO NOT IMPLEMENT
+  void *operator new(size_t, unsigned) LLVM_DELETED_FUNCTION;
 public:
   // allocate space for exactly two operands
   void *operator new(size_t s) {
@@ -352,18 +352,21 @@ struct ExprMapKeyType {
 struct InlineAsmKeyType {
   InlineAsmKeyType(StringRef AsmString,
                    StringRef Constraints, bool hasSideEffects,
-                   bool isAlignStack)
+                   bool isAlignStack, InlineAsm::AsmDialect asmDialect)
     : asm_string(AsmString), constraints(Constraints),
-      has_side_effects(hasSideEffects), is_align_stack(isAlignStack) {}
+      has_side_effects(hasSideEffects), is_align_stack(isAlignStack),
+      asm_dialect(asmDialect) {}
   std::string asm_string;
   std::string constraints;
   bool has_side_effects;
   bool is_align_stack;
+  InlineAsm::AsmDialect asm_dialect;
   bool operator==(const InlineAsmKeyType& that) const {
     return this->asm_string == that.asm_string &&
            this->constraints == that.constraints &&
            this->has_side_effects == that.has_side_effects &&
-           this->is_align_stack == that.is_align_stack;
+           this->is_align_stack == that.is_align_stack &&
+           this->asm_dialect == that.asm_dialect;
   }
   bool operator<(const InlineAsmKeyType& that) const {
     if (this->asm_string != that.asm_string)
@@ -374,6 +377,8 @@ struct InlineAsmKeyType {
       return this->has_side_effects < that.has_side_effects;
     if (this->is_align_stack != that.is_align_stack)
       return this->is_align_stack < that.is_align_stack;
+    if (this->asm_dialect != that.asm_dialect)
+      return this->asm_dialect < that.asm_dialect;
     return false;
   }
 
@@ -490,7 +495,8 @@ template<>
 struct ConstantCreator<InlineAsm, PointerType, InlineAsmKeyType> {
   static InlineAsm *create(PointerType *Ty, const InlineAsmKeyType &Key) {
     return new InlineAsm(Ty, Key.asm_string, Key.constraints,
-                         Key.has_side_effects, Key.is_align_stack);
+                         Key.has_side_effects, Key.is_align_stack,
+                         Key.asm_dialect);
   }
 };
 
@@ -499,7 +505,8 @@ struct ConstantKeyData<InlineAsm> {
   typedef InlineAsmKeyType ValType;
   static ValType getValType(InlineAsm *Asm) {
     return InlineAsmKeyType(Asm->getAsmString(), Asm->getConstraintString(),
-                            Asm->hasSideEffects(), Asm->isAlignStack());
+                            Asm->hasSideEffects(), Asm->isAlignStack(),
+                            Asm->getDialect());
   }
 };
 
diff --git a/lib/VMCore/Core.cpp b/lib/VMCore/Core.cpp
index 972db3c..847bc13 100644
--- a/lib/VMCore/Core.cpp
+++ b/lib/VMCore/Core.cpp
@@ -568,6 +568,19 @@ const char *LLVMGetMDString(LLVMValueRef V, unsigned* Len) {
   return 0;
 }
 
+unsigned LLVMGetMDNodeNumOperands(LLVMValueRef V)
+{
+  return cast<MDNode>(unwrap(V))->getNumOperands();
+}
+
+void LLVMGetMDNodeOperands(LLVMValueRef V, LLVMValueRef *Dest)
+{
+  const MDNode *N = cast<MDNode>(unwrap(V));
+  const unsigned numOperands = N->getNumOperands();
+  for (unsigned i = 0; i < numOperands; i++)
+    Dest[i] = wrap(N->getOperand(i));
+}
+
 unsigned LLVMGetNamedMetadataNumOperands(LLVMModuleRef M, const char* name)
 {
   if (NamedMDNode *N = unwrap(M)->getNamedMetadata(name)) {
@@ -1084,6 +1097,8 @@ LLVMLinkage LLVMGetLinkage(LLVMValueRef Global) {
     return LLVMLinkOnceAnyLinkage;
   case GlobalValue::LinkOnceODRLinkage:
     return LLVMLinkOnceODRLinkage;
+  case GlobalValue::LinkOnceODRAutoHideLinkage:
+    return LLVMLinkOnceODRAutoHideLinkage;
   case GlobalValue::WeakAnyLinkage:
     return LLVMWeakAnyLinkage;
   case GlobalValue::WeakODRLinkage:
@@ -1098,8 +1113,6 @@ LLVMLinkage LLVMGetLinkage(LLVMValueRef Global) {
     return LLVMLinkerPrivateLinkage;
   case GlobalValue::LinkerPrivateWeakLinkage:
     return LLVMLinkerPrivateWeakLinkage;
-  case GlobalValue::LinkerPrivateWeakDefAutoLinkage:
-    return LLVMLinkerPrivateWeakDefAutoLinkage;
   case GlobalValue::DLLImportLinkage:
     return LLVMDLLImportLinkage;
   case GlobalValue::DLLExportLinkage:
@@ -1129,6 +1142,9 @@ void LLVMSetLinkage(LLVMValueRef Global, LLVMLinkage Linkage) {
   case LLVMLinkOnceODRLinkage:
     GV->setLinkage(GlobalValue::LinkOnceODRLinkage);
     break;
+  case LLVMLinkOnceODRAutoHideLinkage:
+    GV->setLinkage(GlobalValue::LinkOnceODRAutoHideLinkage);
+    break;
   case LLVMWeakAnyLinkage:
     GV->setLinkage(GlobalValue::WeakAnyLinkage);
     break;
@@ -1150,9 +1166,6 @@ void LLVMSetLinkage(LLVMValueRef Global, LLVMLinkage Linkage) {
   case LLVMLinkerPrivateWeakLinkage:
     GV->setLinkage(GlobalValue::LinkerPrivateWeakLinkage);
     break;
-  case LLVMLinkerPrivateWeakDefAutoLinkage:
-    GV->setLinkage(GlobalValue::LinkerPrivateWeakDefAutoLinkage);
-    break;
   case LLVMDLLImportLinkage:
     GV->setLinkage(GlobalValue::DLLImportLinkage);
     break;
@@ -1368,14 +1381,20 @@ void LLVMSetGC(LLVMValueRef Fn, const char *GC) {
 void LLVMAddFunctionAttr(LLVMValueRef Fn, LLVMAttribute PA) {
   Function *Func = unwrap<Function>(Fn);
   const AttrListPtr PAL = Func->getAttributes();
-  const AttrListPtr PALnew = PAL.addAttr(~0U, Attributes(PA));
+  AttrBuilder B(PA);
+  const AttrListPtr PALnew =
+    PAL.addAttr(Func->getContext(), AttrListPtr::FunctionIndex,
+                Attributes::get(Func->getContext(), B));
   Func->setAttributes(PALnew);
 }
 
 void LLVMRemoveFunctionAttr(LLVMValueRef Fn, LLVMAttribute PA) {
   Function *Func = unwrap<Function>(Fn);
   const AttrListPtr PAL = Func->getAttributes();
-  const AttrListPtr PALnew = PAL.removeAttr(~0U, Attributes(PA));
+  AttrBuilder B(PA);
+  const AttrListPtr PALnew =
+    PAL.removeAttr(Func->getContext(), AttrListPtr::FunctionIndex,
+                   Attributes::get(Func->getContext(), B));
   Func->setAttributes(PALnew);
 }
 
@@ -1445,11 +1464,15 @@ LLVMValueRef LLVMGetPreviousParam(LLVMValueRef Arg) {
 }
 
 void LLVMAddAttribute(LLVMValueRef Arg, LLVMAttribute PA) {
-  unwrap<Argument>(Arg)->addAttr(Attributes(PA));
+  Argument *A = unwrap<Argument>(Arg);
+  AttrBuilder B(PA);
+  A->addAttr(Attributes::get(A->getContext(), B));
 }
 
 void LLVMRemoveAttribute(LLVMValueRef Arg, LLVMAttribute PA) {
-  unwrap<Argument>(Arg)->removeAttr(Attributes(PA));
+  Argument *A = unwrap<Argument>(Arg);
+  AttrBuilder B(PA);
+  A->removeAttr(Attributes::get(A->getContext(), B));
 }
 
 LLVMAttribute LLVMGetAttribute(LLVMValueRef Arg) {
@@ -1461,8 +1484,10 @@ LLVMAttribute LLVMGetAttribute(LLVMValueRef Arg) {
   
 
 void LLVMSetParamAlignment(LLVMValueRef Arg, unsigned align) {
-  unwrap<Argument>(Arg)->addAttr(
-          Attribute::constructAlignmentFromInt(align));
+  AttrBuilder B;
+  B.addAlignmentAttr(align);
+  unwrap<Argument>(Arg)->addAttr(Attributes::
+                                 get(unwrap<Argument>(Arg)->getContext(), B));
 }
 
 /*--.. Operations on basic blocks ..........................................--*/
@@ -1651,23 +1676,28 @@ void LLVMSetInstructionCallConv(LLVMValueRef Instr, unsigned CC) {
 void LLVMAddInstrAttribute(LLVMValueRef Instr, unsigned index, 
                            LLVMAttribute PA) {
   CallSite Call = CallSite(unwrap<Instruction>(Instr));
+  AttrBuilder B(PA);
   Call.setAttributes(
-    Call.getAttributes().addAttr(index, Attributes(PA)));
+    Call.getAttributes().addAttr(Call->getContext(), index,
+                                 Attributes::get(Call->getContext(), B)));
 }
 
 void LLVMRemoveInstrAttribute(LLVMValueRef Instr, unsigned index, 
                               LLVMAttribute PA) {
   CallSite Call = CallSite(unwrap<Instruction>(Instr));
+  AttrBuilder B(PA);
   Call.setAttributes(
-    Call.getAttributes().removeAttr(index, Attributes(PA)));
+    Call.getAttributes().removeAttr(Call->getContext(), index,
+                                    Attributes::get(Call->getContext(), B)));
 }
 
 void LLVMSetInstrParamAlignment(LLVMValueRef Instr, unsigned index, 
                                 unsigned align) {
   CallSite Call = CallSite(unwrap<Instruction>(Instr));
-  Call.setAttributes(
-    Call.getAttributes().addAttr(index, 
-        Attribute::constructAlignmentFromInt(align)));
+  AttrBuilder B;
+  B.addAlignmentAttr(align);
+  Call.setAttributes(Call.getAttributes().addAttr(Call->getContext(), index,
+                                       Attributes::get(Call->getContext(), B)));
 }
 
 /*--.. Operations on call instructions (only) ..............................--*/
diff --git a/lib/VMCore/DIBuilder.cpp b/lib/VMCore/DIBuilder.cpp
index f5894e9..152b825 100644
--- a/lib/VMCore/DIBuilder.cpp
+++ b/lib/VMCore/DIBuilder.cpp
@@ -492,7 +492,8 @@ DIType DIBuilder::createStructType(DIDescriptor Context, StringRef Name,
     NULL,
     Elements,
     ConstantInt::get(Type::getInt32Ty(VMContext), RunTimeLang),
-    Constant::getNullValue(Type::getInt32Ty(VMContext))
+    ConstantInt::get(Type::getInt32Ty(VMContext), 0),
+    ConstantInt::get(Type::getInt32Ty(VMContext), 0),
   };
   return DIType(MDNode::get(VMContext, Elts));
 }
@@ -550,7 +551,7 @@ DIType DIBuilder::createEnumerationType(DIDescriptor Scope, StringRef Name,
                                         uint64_t SizeInBits,
                                         uint64_t AlignInBits,
                                         DIArray Elements,
-                                        DIType ClassType, unsigned Flags) {
+                                        DIType ClassType) {
   // TAG_enumeration_type is encoded in DICompositeType format.
   Value *Elts[] = {
     GetTagConstant(VMContext, dwarf::DW_TAG_enumeration_type),
@@ -561,7 +562,7 @@ DIType DIBuilder::createEnumerationType(DIDescriptor Scope, StringRef Name,
     ConstantInt::get(Type::getInt64Ty(VMContext), SizeInBits),
     ConstantInt::get(Type::getInt64Ty(VMContext), AlignInBits),
     ConstantInt::get(Type::getInt32Ty(VMContext), 0),
-    ConstantInt::get(Type::getInt32Ty(VMContext), Flags),
+    ConstantInt::get(Type::getInt32Ty(VMContext), 0),
     ClassType,
     Elements,
     ConstantInt::get(Type::getInt32Ty(VMContext), 0),
@@ -640,6 +641,30 @@ DIType DIBuilder::createArtificialType(DIType Ty) {
   return DIType(MDNode::get(VMContext, Elts));
 }
 
+/// createArtificialType - Create a new DIType with "artificial" flag set.
+DIType DIBuilder::createObjectPointerType(DIType Ty) {
+  if (Ty.isObjectPointer())
+    return Ty;
+
+  SmallVector<Value *, 9> Elts;
+  MDNode *N = Ty;
+  assert (N && "Unexpected input DIType!");
+  for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
+    if (Value *V = N->getOperand(i))
+      Elts.push_back(V);
+    else
+      Elts.push_back(Constant::getNullValue(Type::getInt32Ty(VMContext)));
+  }
+
+  unsigned CurFlags = Ty.getFlags();
+  CurFlags = CurFlags | (DIType::FlagObjectPointer | DIType::FlagArtificial);
+
+  // Flags are stored at this slot.
+  Elts[8] = ConstantInt::get(Type::getInt32Ty(VMContext), CurFlags);
+
+  return DIType(MDNode::get(VMContext, Elts));
+}
+
 /// retainType - Retain DIType in a module even if it is not referenced
 /// through debug info anchors.
 void DIBuilder::retainType(DIType T) {
@@ -682,7 +707,9 @@ DIType DIBuilder::createTemporaryType(DIFile F) {
 /// can be RAUW'd if the full type is seen.
 DIType DIBuilder::createForwardDecl(unsigned Tag, StringRef Name,
                                     DIDescriptor Scope, DIFile F,
-                                    unsigned Line, unsigned RuntimeLang) {
+                                    unsigned Line, unsigned RuntimeLang,
+                                    uint64_t SizeInBits,
+                                    uint64_t AlignInBits) {
   // Create a temporary MDNode.
   Value *Elts[] = {
     GetTagConstant(VMContext, Tag),
@@ -690,9 +717,8 @@ DIType DIBuilder::createForwardDecl(unsigned Tag, StringRef Name,
     MDString::get(VMContext, Name),
     F,
     ConstantInt::get(Type::getInt32Ty(VMContext), Line),
-    // To ease transition include sizes etc of 0.
-    ConstantInt::get(Type::getInt32Ty(VMContext), 0),
-    ConstantInt::get(Type::getInt32Ty(VMContext), 0),
+    ConstantInt::get(Type::getInt64Ty(VMContext), SizeInBits),
+    ConstantInt::get(Type::getInt64Ty(VMContext), AlignInBits),
     ConstantInt::get(Type::getInt32Ty(VMContext), 0),
     ConstantInt::get(Type::getInt32Ty(VMContext),
                      DIDescriptor::FlagFwdDecl),
diff --git a/lib/VMCore/DataLayout.cpp b/lib/VMCore/DataLayout.cpp
new file mode 100644
index 0000000..19cf0f5
--- /dev/null
+++ b/lib/VMCore/DataLayout.cpp
@@ -0,0 +1,749 @@
+//===-- DataLayout.cpp - Data size & alignment routines --------------------==//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines layout properties related to datatype size/offset/alignment
+// information.
+//
+// This structure should be created once, filled in if the defaults are not
+// correct and then passed around by const&.  None of the members functions
+// require modification to the object.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/DataLayout.h"
+#include "llvm/Constants.h"
+#include "llvm/DerivedTypes.h"
+#include "llvm/Module.h"
+#include "llvm/Support/GetElementPtrTypeIterator.h"
+#include "llvm/Support/MathExtras.h"
+#include "llvm/Support/ManagedStatic.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/Mutex.h"
+#include "llvm/ADT/DenseMap.h"
+#include <algorithm>
+#include <cstdlib>
+using namespace llvm;
+
+// Handle the Pass registration stuff necessary to use DataLayout's.
+
+// Register the default SparcV9 implementation...
+INITIALIZE_PASS(DataLayout, "datalayout", "Data Layout", false, true)
+char DataLayout::ID = 0;
+
+//===----------------------------------------------------------------------===//
+// Support for StructLayout
+//===----------------------------------------------------------------------===//
+
+StructLayout::StructLayout(StructType *ST, const DataLayout &TD) {
+  assert(!ST->isOpaque() && "Cannot get layout of opaque structs");
+  StructAlignment = 0;
+  StructSize = 0;
+  NumElements = ST->getNumElements();
+
+  // Loop over each of the elements, placing them in memory.
+  for (unsigned i = 0, e = NumElements; i != e; ++i) {
+    Type *Ty = ST->getElementType(i);
+    unsigned TyAlign = ST->isPacked() ? 1 : TD.getABITypeAlignment(Ty);
+
+    // Add padding if necessary to align the data element properly.
+    if ((StructSize & (TyAlign-1)) != 0)
+      StructSize = DataLayout::RoundUpAlignment(StructSize, TyAlign);
+
+    // Keep track of maximum alignment constraint.
+    StructAlignment = std::max(TyAlign, StructAlignment);
+
+    MemberOffsets[i] = StructSize;
+    StructSize += TD.getTypeAllocSize(Ty); // Consume space for this data item
+  }
+
+  // Empty structures have alignment of 1 byte.
+  if (StructAlignment == 0) StructAlignment = 1;
+
+  // Add padding to the end of the struct so that it could be put in an array
+  // and all array elements would be aligned correctly.
+  if ((StructSize & (StructAlignment-1)) != 0)
+    StructSize = DataLayout::RoundUpAlignment(StructSize, StructAlignment);
+}
+
+
+/// getElementContainingOffset - Given a valid offset into the structure,
+/// return the structure index that contains it.
+unsigned StructLayout::getElementContainingOffset(uint64_t Offset) const {
+  const uint64_t *SI =
+    std::upper_bound(&MemberOffsets[0], &MemberOffsets[NumElements], Offset);
+  assert(SI != &MemberOffsets[0] && "Offset not in structure type!");
+  --SI;
+  assert(*SI <= Offset && "upper_bound didn't work");
+  assert((SI == &MemberOffsets[0] || *(SI-1) <= Offset) &&
+         (SI+1 == &MemberOffsets[NumElements] || *(SI+1) > Offset) &&
+         "Upper bound didn't work!");
+
+  // Multiple fields can have the same offset if any of them are zero sized.
+  // For example, in { i32, [0 x i32], i32 }, searching for offset 4 will stop
+  // at the i32 element, because it is the last element at that offset.  This is
+  // the right one to return, because anything after it will have a higher
+  // offset, implying that this element is non-empty.
+  return SI-&MemberOffsets[0];
+}
+
+//===----------------------------------------------------------------------===//
+// LayoutAlignElem, LayoutAlign support
+//===----------------------------------------------------------------------===//
+
+LayoutAlignElem
+LayoutAlignElem::get(AlignTypeEnum align_type, unsigned abi_align,
+                     unsigned pref_align, uint32_t bit_width) {
+  assert(abi_align <= pref_align && "Preferred alignment worse than ABI!");
+  LayoutAlignElem retval;
+  retval.AlignType = align_type;
+  retval.ABIAlign = abi_align;
+  retval.PrefAlign = pref_align;
+  retval.TypeBitWidth = bit_width;
+  return retval;
+}
+
+bool
+LayoutAlignElem::operator==(const LayoutAlignElem &rhs) const {
+  return (AlignType == rhs.AlignType
+          && ABIAlign == rhs.ABIAlign
+          && PrefAlign == rhs.PrefAlign
+          && TypeBitWidth == rhs.TypeBitWidth);
+}
+
+const LayoutAlignElem
+DataLayout::InvalidAlignmentElem =
+            LayoutAlignElem::get((AlignTypeEnum) -1, 0, 0, 0);
+
+//===----------------------------------------------------------------------===//
+// PointerAlignElem, PointerAlign support
+//===----------------------------------------------------------------------===//
+
+PointerAlignElem
+PointerAlignElem::get(uint32_t addr_space, unsigned abi_align,
+                     unsigned pref_align, uint32_t bit_width) {
+  assert(abi_align <= pref_align && "Preferred alignment worse than ABI!");
+  PointerAlignElem retval;
+  retval.AddressSpace = addr_space;
+  retval.ABIAlign = abi_align;
+  retval.PrefAlign = pref_align;
+  retval.TypeBitWidth = bit_width;
+  return retval;
+}
+
+bool
+PointerAlignElem::operator==(const PointerAlignElem &rhs) const {
+  return (ABIAlign == rhs.ABIAlign
+          && AddressSpace == rhs.AddressSpace
+          && PrefAlign == rhs.PrefAlign
+          && TypeBitWidth == rhs.TypeBitWidth);
+}
+
+const PointerAlignElem
+DataLayout::InvalidPointerElem = PointerAlignElem::get(~0U, 0U, 0U, 0U);
+
+//===----------------------------------------------------------------------===//
+//                       DataLayout Class Implementation
+//===----------------------------------------------------------------------===//
+
+/// getInt - Get an integer ignoring errors.
+static int getInt(StringRef R) {
+  int Result = 0;
+  R.getAsInteger(10, Result);
+  return Result;
+}
+
+void DataLayout::init() {
+  initializeDataLayoutPass(*PassRegistry::getPassRegistry());
+
+  LayoutMap = 0;
+  LittleEndian = false;
+  StackNaturalAlign = 0;
+
+  // Default alignments
+  setAlignment(INTEGER_ALIGN,   1,  1, 1);   // i1
+  setAlignment(INTEGER_ALIGN,   1,  1, 8);   // i8
+  setAlignment(INTEGER_ALIGN,   2,  2, 16);  // i16
+  setAlignment(INTEGER_ALIGN,   4,  4, 32);  // i32
+  setAlignment(INTEGER_ALIGN,   4,  8, 64);  // i64
+  setAlignment(FLOAT_ALIGN,     2,  2, 16);  // half
+  setAlignment(FLOAT_ALIGN,     4,  4, 32);  // float
+  setAlignment(FLOAT_ALIGN,     8,  8, 64);  // double
+  setAlignment(FLOAT_ALIGN,    16, 16, 128); // ppcf128, quad, ...
+  setAlignment(VECTOR_ALIGN,    8,  8, 64);  // v2i32, v1i64, ...
+  setAlignment(VECTOR_ALIGN,   16, 16, 128); // v16i8, v8i16, v4i32, ...
+  setAlignment(AGGREGATE_ALIGN, 0,  8,  0);  // struct
+  setPointerAlignment(0, 8, 8, 8);
+}
+
+std::string DataLayout::parseSpecifier(StringRef Desc, DataLayout *td) {
+
+  if (td)
+    td->init();
+
+  while (!Desc.empty()) {
+    std::pair<StringRef, StringRef> Split = Desc.split('-');
+    StringRef Token = Split.first;
+    Desc = Split.second;
+
+    if (Token.empty())
+      continue;
+
+    Split = Token.split(':');
+    StringRef Specifier = Split.first;
+    Token = Split.second;
+
+    assert(!Specifier.empty() && "Can't be empty here");
+
+    switch (Specifier[0]) {
+    case 'E':
+      if (td)
+        td->LittleEndian = false;
+      break;
+    case 'e':
+      if (td)
+        td->LittleEndian = true;
+      break;
+    case 'p': {
+      int AddrSpace = 0;
+      if (Specifier.size() > 1) {
+        AddrSpace = getInt(Specifier.substr(1));
+        if (AddrSpace < 0 || AddrSpace > (1 << 24))
+          return "Invalid address space, must be a positive 24bit integer";
+      }
+      Split = Token.split(':');
+      int PointerMemSizeBits = getInt(Split.first);
+      if (PointerMemSizeBits < 0 || PointerMemSizeBits % 8 != 0)
+        return "invalid pointer size, must be a positive 8-bit multiple";
+
+      // Pointer ABI alignment.
+      Split = Split.second.split(':');
+      int PointerABIAlignBits = getInt(Split.first);
+      if (PointerABIAlignBits < 0 || PointerABIAlignBits % 8 != 0) {
+        return "invalid pointer ABI alignment, "
+               "must be a positive 8-bit multiple";
+      }
+
+      // Pointer preferred alignment.
+      Split = Split.second.split(':');
+      int PointerPrefAlignBits = getInt(Split.first);
+      if (PointerPrefAlignBits < 0 || PointerPrefAlignBits % 8 != 0) {
+        return "invalid pointer preferred alignment, "
+               "must be a positive 8-bit multiple";
+      }
+
+      if (PointerPrefAlignBits == 0)
+        PointerPrefAlignBits = PointerABIAlignBits;
+      if (td)
+        td->setPointerAlignment(AddrSpace, PointerABIAlignBits/8,
+            PointerPrefAlignBits/8, PointerMemSizeBits/8);
+      break;
+    }
+    case 'i':
+    case 'v':
+    case 'f':
+    case 'a':
+    case 's': {
+      AlignTypeEnum AlignType;
+      char field = Specifier[0];
+      switch (field) {
+      default:
+      case 'i': AlignType = INTEGER_ALIGN; break;
+      case 'v': AlignType = VECTOR_ALIGN; break;
+      case 'f': AlignType = FLOAT_ALIGN; break;
+      case 'a': AlignType = AGGREGATE_ALIGN; break;
+      case 's': AlignType = STACK_ALIGN; break;
+      }
+      int Size = getInt(Specifier.substr(1));
+      if (Size < 0) {
+        return std::string("invalid ") + field + "-size field, "
+               "must be positive";
+      }
+
+      Split = Token.split(':');
+      int ABIAlignBits = getInt(Split.first);
+      if (ABIAlignBits < 0 || ABIAlignBits % 8 != 0) {
+        return std::string("invalid ") + field +"-abi-alignment field, "
+               "must be a positive 8-bit multiple";
+      }
+      unsigned ABIAlign = ABIAlignBits / 8;
+
+      Split = Split.second.split(':');
+
+      int PrefAlignBits = getInt(Split.first);
+      if (PrefAlignBits < 0 || PrefAlignBits % 8 != 0) {
+        return std::string("invalid ") + field +"-preferred-alignment field, "
+               "must be a positive 8-bit multiple";
+      }
+      unsigned PrefAlign = PrefAlignBits / 8;
+      if (PrefAlign == 0)
+        PrefAlign = ABIAlign;
+
+      if (td)
+        td->setAlignment(AlignType, ABIAlign, PrefAlign, Size);
+      break;
+    }
+    case 'n':  // Native integer types.
+      Specifier = Specifier.substr(1);
+      do {
+        int Width = getInt(Specifier);
+        if (Width <= 0) {
+          return std::string("invalid native integer size \'") +
+            Specifier.str() + "\', must be a positive integer.";
+        }
+        if (td && Width != 0)
+          td->LegalIntWidths.push_back(Width);
+        Split = Token.split(':');
+        Specifier = Split.first;
+        Token = Split.second;
+      } while (!Specifier.empty() || !Token.empty());
+      break;
+    case 'S': { // Stack natural alignment.
+      int StackNaturalAlignBits = getInt(Specifier.substr(1));
+      if (StackNaturalAlignBits < 0 || StackNaturalAlignBits % 8 != 0) {
+        return "invalid natural stack alignment (S-field), "
+               "must be a positive 8-bit multiple";
+      }
+      if (td)
+        td->StackNaturalAlign = StackNaturalAlignBits / 8;
+      break;
+    }
+    default:
+      break;
+    }
+  }
+
+  return "";
+}
+
+/// Default ctor.
+///
+/// @note This has to exist, because this is a pass, but it should never be
+/// used.
+DataLayout::DataLayout() : ImmutablePass(ID) {
+  report_fatal_error("Bad DataLayout ctor used.  "
+                    "Tool did not specify a DataLayout to use?");
+}
+
+DataLayout::DataLayout(const Module *M)
+  : ImmutablePass(ID) {
+  std::string errMsg = parseSpecifier(M->getDataLayout(), this);
+  assert(errMsg == "" && "Module M has malformed data layout string.");
+  (void)errMsg;
+}
+
+void
+DataLayout::setAlignment(AlignTypeEnum align_type, unsigned abi_align,
+                         unsigned pref_align, uint32_t bit_width) {
+  assert(abi_align <= pref_align && "Preferred alignment worse than ABI!");
+  assert(pref_align < (1 << 16) && "Alignment doesn't fit in bitfield");
+  assert(bit_width < (1 << 24) && "Bit width doesn't fit in bitfield");
+  for (unsigned i = 0, e = Alignments.size(); i != e; ++i) {
+    if (Alignments[i].AlignType == (unsigned)align_type &&
+        Alignments[i].TypeBitWidth == bit_width) {
+      // Update the abi, preferred alignments.
+      Alignments[i].ABIAlign = abi_align;
+      Alignments[i].PrefAlign = pref_align;
+      return;
+    }
+  }
+
+  Alignments.push_back(LayoutAlignElem::get(align_type, abi_align,
+                                            pref_align, bit_width));
+}
+
+void
+DataLayout::setPointerAlignment(uint32_t addr_space, unsigned abi_align,
+                         unsigned pref_align, uint32_t bit_width) {
+  assert(abi_align <= pref_align && "Preferred alignment worse than ABI!");
+  DenseMap<unsigned,PointerAlignElem>::iterator val = Pointers.find(addr_space);
+  if (val == Pointers.end()) {
+    Pointers[addr_space] = PointerAlignElem::get(addr_space,
+          abi_align, pref_align, bit_width);
+  } else {
+    val->second.ABIAlign = abi_align;
+    val->second.PrefAlign = pref_align;
+    val->second.TypeBitWidth = bit_width;
+  }
+}
+
+/// getAlignmentInfo - Return the alignment (either ABI if ABIInfo = true or
+/// preferred if ABIInfo = false) the layout wants for the specified datatype.
+unsigned DataLayout::getAlignmentInfo(AlignTypeEnum AlignType,
+                                      uint32_t BitWidth, bool ABIInfo,
+                                      Type *Ty) const {
+  // Check to see if we have an exact match and remember the best match we see.
+  int BestMatchIdx = -1;
+  int LargestInt = -1;
+  for (unsigned i = 0, e = Alignments.size(); i != e; ++i) {
+    if (Alignments[i].AlignType == (unsigned)AlignType &&
+        Alignments[i].TypeBitWidth == BitWidth)
+      return ABIInfo ? Alignments[i].ABIAlign : Alignments[i].PrefAlign;
+
+    // The best match so far depends on what we're looking for.
+     if (AlignType == INTEGER_ALIGN &&
+         Alignments[i].AlignType == INTEGER_ALIGN) {
+      // The "best match" for integers is the smallest size that is larger than
+      // the BitWidth requested.
+      if (Alignments[i].TypeBitWidth > BitWidth && (BestMatchIdx == -1 ||
+           Alignments[i].TypeBitWidth < Alignments[BestMatchIdx].TypeBitWidth))
+        BestMatchIdx = i;
+      // However, if there isn't one that's larger, then we must use the
+      // largest one we have (see below)
+      if (LargestInt == -1 ||
+          Alignments[i].TypeBitWidth > Alignments[LargestInt].TypeBitWidth)
+        LargestInt = i;
+    }
+  }
+
+  // Okay, we didn't find an exact solution.  Fall back here depending on what
+  // is being looked for.
+  if (BestMatchIdx == -1) {
+    // If we didn't find an integer alignment, fall back on most conservative.
+    if (AlignType == INTEGER_ALIGN) {
+      BestMatchIdx = LargestInt;
+    } else {
+      assert(AlignType == VECTOR_ALIGN && "Unknown alignment type!");
+
+      // By default, use natural alignment for vector types. This is consistent
+      // with what clang and llvm-gcc do.
+      unsigned Align = getTypeAllocSize(cast<VectorType>(Ty)->getElementType());
+      Align *= cast<VectorType>(Ty)->getNumElements();
+      // If the alignment is not a power of 2, round up to the next power of 2.
+      // This happens for non-power-of-2 length vectors.
+      if (Align & (Align-1))
+        Align = NextPowerOf2(Align);
+      return Align;
+    }
+  }
+
+  // Since we got a "best match" index, just return it.
+  return ABIInfo ? Alignments[BestMatchIdx].ABIAlign
+                 : Alignments[BestMatchIdx].PrefAlign;
+}
+
+namespace {
+
+class StructLayoutMap {
+  typedef DenseMap<StructType*, StructLayout*> LayoutInfoTy;
+  LayoutInfoTy LayoutInfo;
+
+public:
+  virtual ~StructLayoutMap() {
+    // Remove any layouts.
+    for (LayoutInfoTy::iterator I = LayoutInfo.begin(), E = LayoutInfo.end();
+         I != E; ++I) {
+      StructLayout *Value = I->second;
+      Value->~StructLayout();
+      free(Value);
+    }
+  }
+
+  StructLayout *&operator[](StructType *STy) {
+    return LayoutInfo[STy];
+  }
+
+  // for debugging...
+  virtual void dump() const {}
+};
+
+} // end anonymous namespace
+
+DataLayout::~DataLayout() {
+  delete static_cast<StructLayoutMap*>(LayoutMap);
+}
+
+const StructLayout *DataLayout::getStructLayout(StructType *Ty) const {
+  if (!LayoutMap)
+    LayoutMap = new StructLayoutMap();
+
+  StructLayoutMap *STM = static_cast<StructLayoutMap*>(LayoutMap);
+  StructLayout *&SL = (*STM)[Ty];
+  if (SL) return SL;
+
+  // Otherwise, create the struct layout.  Because it is variable length, we
+  // malloc it, then use placement new.
+  int NumElts = Ty->getNumElements();
+  StructLayout *L =
+    (StructLayout *)malloc(sizeof(StructLayout)+(NumElts-1) * sizeof(uint64_t));
+
+  // Set SL before calling StructLayout's ctor.  The ctor could cause other
+  // entries to be added to TheMap, invalidating our reference.
+  SL = L;
+
+  new (L) StructLayout(Ty, *this);
+
+  return L;
+}
+
+std::string DataLayout::getStringRepresentation() const {
+  std::string Result;
+  raw_string_ostream OS(Result);
+
+  OS << (LittleEndian ? "e" : "E");
+  SmallVector<unsigned, 8> addrSpaces;
+  // Lets get all of the known address spaces and sort them
+  // into increasing order so that we can emit the string
+  // in a cleaner format.
+  for (DenseMap<unsigned, PointerAlignElem>::const_iterator
+      pib = Pointers.begin(), pie = Pointers.end();
+      pib != pie; ++pib) {
+    addrSpaces.push_back(pib->first);
+  }
+  std::sort(addrSpaces.begin(), addrSpaces.end());
+  for (SmallVector<unsigned, 8>::iterator asb = addrSpaces.begin(),
+      ase = addrSpaces.end(); asb != ase; ++asb) {
+    const PointerAlignElem &PI = Pointers.find(*asb)->second;
+    OS << "-p";
+    if (PI.AddressSpace) {
+      OS << PI.AddressSpace;
+    }
+     OS << ":" << PI.TypeBitWidth*8 << ':' << PI.ABIAlign*8
+        << ':' << PI.PrefAlign*8;
+  }
+  OS << "-S" << StackNaturalAlign*8;
+
+  for (unsigned i = 0, e = Alignments.size(); i != e; ++i) {
+    const LayoutAlignElem &AI = Alignments[i];
+    OS << '-' << (char)AI.AlignType << AI.TypeBitWidth << ':'
+       << AI.ABIAlign*8 << ':' << AI.PrefAlign*8;
+  }
+
+  if (!LegalIntWidths.empty()) {
+    OS << "-n" << (unsigned)LegalIntWidths[0];
+
+    for (unsigned i = 1, e = LegalIntWidths.size(); i != e; ++i)
+      OS << ':' << (unsigned)LegalIntWidths[i];
+  }
+  return OS.str();
+}
+
+
+uint64_t DataLayout::getTypeSizeInBits(Type *Ty) const {
+  assert(Ty->isSized() && "Cannot getTypeInfo() on a type that is unsized!");
+  switch (Ty->getTypeID()) {
+  case Type::LabelTyID:
+    return getPointerSizeInBits(0);
+  case Type::PointerTyID: {
+    unsigned AS = dyn_cast<PointerType>(Ty)->getAddressSpace();
+    return getPointerSizeInBits(AS);
+    }
+  case Type::ArrayTyID: {
+    ArrayType *ATy = cast<ArrayType>(Ty);
+    return getTypeAllocSizeInBits(ATy->getElementType())*ATy->getNumElements();
+  }
+  case Type::StructTyID:
+    // Get the layout annotation... which is lazily created on demand.
+    return getStructLayout(cast<StructType>(Ty))->getSizeInBits();
+  case Type::IntegerTyID:
+    return cast<IntegerType>(Ty)->getBitWidth();
+  case Type::VoidTyID:
+    return 8;
+  case Type::HalfTyID:
+    return 16;
+  case Type::FloatTyID:
+    return 32;
+  case Type::DoubleTyID:
+  case Type::X86_MMXTyID:
+    return 64;
+  case Type::PPC_FP128TyID:
+  case Type::FP128TyID:
+    return 128;
+  // In memory objects this is always aligned to a higher boundary, but
+  // only 80 bits contain information.
+  case Type::X86_FP80TyID:
+    return 80;
+  case Type::VectorTyID: {
+    VectorType *VTy = cast<VectorType>(Ty);
+    return VTy->getNumElements()*getTypeSizeInBits(VTy->getElementType());
+  }
+  default:
+    llvm_unreachable("DataLayout::getTypeSizeInBits(): Unsupported type");
+  }
+}
+
+/*!
+  \param abi_or_pref Flag that determines which alignment is returned. true
+  returns the ABI alignment, false returns the preferred alignment.
+  \param Ty The underlying type for which alignment is determined.
+
+  Get the ABI (\a abi_or_pref == true) or preferred alignment (\a abi_or_pref
+  == false) for the requested type \a Ty.
+ */
+unsigned DataLayout::getAlignment(Type *Ty, bool abi_or_pref) const {
+  int AlignType = -1;
+
+  assert(Ty->isSized() && "Cannot getTypeInfo() on a type that is unsized!");
+  switch (Ty->getTypeID()) {
+  // Early escape for the non-numeric types.
+  case Type::LabelTyID:
+    return (abi_or_pref
+            ? getPointerABIAlignment(0)
+            : getPointerPrefAlignment(0));
+  case Type::PointerTyID: {
+    unsigned AS = dyn_cast<PointerType>(Ty)->getAddressSpace();
+    return (abi_or_pref
+            ? getPointerABIAlignment(AS)
+            : getPointerPrefAlignment(AS));
+    }
+  case Type::ArrayTyID:
+    return getAlignment(cast<ArrayType>(Ty)->getElementType(), abi_or_pref);
+
+  case Type::StructTyID: {
+    // Packed structure types always have an ABI alignment of one.
+    if (cast<StructType>(Ty)->isPacked() && abi_or_pref)
+      return 1;
+
+    // Get the layout annotation... which is lazily created on demand.
+    const StructLayout *Layout = getStructLayout(cast<StructType>(Ty));
+    unsigned Align = getAlignmentInfo(AGGREGATE_ALIGN, 0, abi_or_pref, Ty);
+    return std::max(Align, Layout->getAlignment());
+  }
+  case Type::IntegerTyID:
+  case Type::VoidTyID:
+    AlignType = INTEGER_ALIGN;
+    break;
+  case Type::HalfTyID:
+  case Type::FloatTyID:
+  case Type::DoubleTyID:
+  // PPC_FP128TyID and FP128TyID have different data contents, but the
+  // same size and alignment, so they look the same here.
+  case Type::PPC_FP128TyID:
+  case Type::FP128TyID:
+  case Type::X86_FP80TyID:
+    AlignType = FLOAT_ALIGN;
+    break;
+  case Type::X86_MMXTyID:
+  case Type::VectorTyID:
+    AlignType = VECTOR_ALIGN;
+    break;
+  default:
+    llvm_unreachable("Bad type for getAlignment!!!");
+  }
+
+  return getAlignmentInfo((AlignTypeEnum)AlignType, getTypeSizeInBits(Ty),
+                          abi_or_pref, Ty);
+}
+
+unsigned DataLayout::getABITypeAlignment(Type *Ty) const {
+  return getAlignment(Ty, true);
+}
+
+/// getABIIntegerTypeAlignment - Return the minimum ABI-required alignment for
+/// an integer type of the specified bitwidth.
+unsigned DataLayout::getABIIntegerTypeAlignment(unsigned BitWidth) const {
+  return getAlignmentInfo(INTEGER_ALIGN, BitWidth, true, 0);
+}
+
+
+unsigned DataLayout::getCallFrameTypeAlignment(Type *Ty) const {
+  for (unsigned i = 0, e = Alignments.size(); i != e; ++i)
+    if (Alignments[i].AlignType == STACK_ALIGN)
+      return Alignments[i].ABIAlign;
+
+  return getABITypeAlignment(Ty);
+}
+
+unsigned DataLayout::getPrefTypeAlignment(Type *Ty) const {
+  return getAlignment(Ty, false);
+}
+
+unsigned DataLayout::getPreferredTypeAlignmentShift(Type *Ty) const {
+  unsigned Align = getPrefTypeAlignment(Ty);
+  assert(!(Align & (Align-1)) && "Alignment is not a power of two!");
+  return Log2_32(Align);
+}
+
+/// getIntPtrType - Return an integer type with size at least as big as that
+/// of a pointer in the given address space.
+IntegerType *DataLayout::getIntPtrType(LLVMContext &C,
+                                       unsigned AddressSpace) const {
+  return IntegerType::get(C, getPointerSizeInBits(AddressSpace));
+}
+
+/// getIntPtrType - Return an integer (vector of integer) type with size at
+/// least as big as that of a pointer of the given pointer (vector of pointer)
+/// type.
+Type *DataLayout::getIntPtrType(Type *Ty) const {
+  assert(Ty->isPtrOrPtrVectorTy() &&
+         "Expected a pointer or pointer vector type.");
+  unsigned NumBits = getTypeSizeInBits(Ty->getScalarType());
+  IntegerType *IntTy = IntegerType::get(Ty->getContext(), NumBits);
+  if (VectorType *VecTy = dyn_cast<VectorType>(Ty))
+    return VectorType::get(IntTy, VecTy->getNumElements());
+  return IntTy;
+}
+
+uint64_t DataLayout::getIndexedOffset(Type *ptrTy,
+                                      ArrayRef<Value *> Indices) const {
+  Type *Ty = ptrTy;
+  assert(Ty->isPointerTy() && "Illegal argument for getIndexedOffset()");
+  uint64_t Result = 0;
+
+  generic_gep_type_iterator<Value* const*>
+    TI = gep_type_begin(ptrTy, Indices);
+  for (unsigned CurIDX = 0, EndIDX = Indices.size(); CurIDX != EndIDX;
+       ++CurIDX, ++TI) {
+    if (StructType *STy = dyn_cast<StructType>(*TI)) {
+      assert(Indices[CurIDX]->getType() ==
+             Type::getInt32Ty(ptrTy->getContext()) &&
+             "Illegal struct idx");
+      unsigned FieldNo = cast<ConstantInt>(Indices[CurIDX])->getZExtValue();
+
+      // Get structure layout information...
+      const StructLayout *Layout = getStructLayout(STy);
+
+      // Add in the offset, as calculated by the structure layout info...
+      Result += Layout->getElementOffset(FieldNo);
+
+      // Update Ty to refer to current element
+      Ty = STy->getElementType(FieldNo);
+    } else {
+      // Update Ty to refer to current element
+      Ty = cast<SequentialType>(Ty)->getElementType();
+
+      // Get the array index and the size of each array element.
+      if (int64_t arrayIdx = cast<ConstantInt>(Indices[CurIDX])->getSExtValue())
+        Result += (uint64_t)arrayIdx * getTypeAllocSize(Ty);
+    }
+  }
+
+  return Result;
+}
+
+/// getPreferredAlignment - Return the preferred alignment of the specified
+/// global.  This includes an explicitly requested alignment (if the global
+/// has one).
+unsigned DataLayout::getPreferredAlignment(const GlobalVariable *GV) const {
+  Type *ElemType = GV->getType()->getElementType();
+  unsigned Alignment = getPrefTypeAlignment(ElemType);
+  unsigned GVAlignment = GV->getAlignment();
+  if (GVAlignment >= Alignment) {
+    Alignment = GVAlignment;
+  } else if (GVAlignment != 0) {
+    Alignment = std::max(GVAlignment, getABITypeAlignment(ElemType));
+  }
+
+  if (GV->hasInitializer() && GVAlignment == 0) {
+    if (Alignment < 16) {
+      // If the global is not external, see if it is large.  If so, give it a
+      // larger alignment.
+      if (getTypeSizeInBits(ElemType) > 128)
+        Alignment = 16;    // 16-byte alignment.
+    }
+  }
+  return Alignment;
+}
+
+/// getPreferredAlignmentLog - Return the preferred alignment of the
+/// specified global, returned in log form.  This includes an explicitly
+/// requested alignment (if the global has one).
+unsigned DataLayout::getPreferredAlignmentLog(const GlobalVariable *GV) const {
+  return Log2_32(getPreferredAlignment(GV));
+}
diff --git a/lib/VMCore/DebugInfo.cpp b/lib/VMCore/DebugInfo.cpp
index c8f8f7d..3029ce2 100644
--- a/lib/VMCore/DebugInfo.cpp
+++ b/lib/VMCore/DebugInfo.cpp
@@ -111,6 +111,16 @@ Function *DIDescriptor::getFunctionField(unsigned Elt) const {
   return 0;
 }
 
+void DIDescriptor::replaceFunctionField(unsigned Elt, Function *F) {
+  if (DbgNode == 0)
+    return;
+
+  if (Elt < DbgNode->getNumOperands()) {
+    MDNode *Node = const_cast<MDNode*>(DbgNode);
+    Node->replaceOperandWith(Elt, F);
+  }
+}
+
 unsigned DIVariable::getNumAddrElements() const {
   if (getVersion() <= LLVMDebugVersion8)
     return DbgNode->getNumOperands()-6;
diff --git a/lib/VMCore/Dominators.cpp b/lib/VMCore/Dominators.cpp
index 60bdeac..77b2403 100644
--- a/lib/VMCore/Dominators.cpp
+++ b/lib/VMCore/Dominators.cpp
@@ -161,6 +161,11 @@ bool DominatorTree::dominates(const Instruction *Def,
 
 bool DominatorTree::dominates(const BasicBlockEdge &BBE,
                               const BasicBlock *UseBB) const {
+  // Assert that we have a single edge. We could handle them by simply
+  // returning false, but since isSingleEdge is linear on the number of
+  // edges, the callers can normally handle them more efficiently.
+  assert(BBE.isSingleEdge());
+
   // If the BB the edge ends in doesn't dominate the use BB, then the
   // edge also doesn't.
   const BasicBlock *Start = BBE.getStart();
@@ -207,6 +212,11 @@ bool DominatorTree::dominates(const BasicBlockEdge &BBE,
 
 bool DominatorTree::dominates(const BasicBlockEdge &BBE,
                               const Use &U) const {
+  // Assert that we have a single edge. We could handle them by simply
+  // returning false, but since isSingleEdge is linear on the number of
+  // edges, the callers can normally handle them more efficiently.
+  assert(BBE.isSingleEdge());
+
   Instruction *UserInst = cast<Instruction>(U.getUser());
   // A PHI in the end of the edge is dominated by it.
   PHINode *PN = dyn_cast<PHINode>(UserInst);
diff --git a/lib/VMCore/Function.cpp b/lib/VMCore/Function.cpp
index 2e0b316..9c4f2d9 100644
--- a/lib/VMCore/Function.cpp
+++ b/lib/VMCore/Function.cpp
@@ -78,7 +78,8 @@ unsigned Argument::getArgNo() const {
 /// in its containing function.
 bool Argument::hasByValAttr() const {
   if (!getType()->isPointerTy()) return false;
-  return getParent()->paramHasAttr(getArgNo()+1, Attribute::ByVal);
+  return getParent()->getParamAttributes(getArgNo()+1).
+    hasAttribute(Attributes::ByVal);
 }
 
 unsigned Argument::getParamAlignment() const {
@@ -91,21 +92,24 @@ unsigned Argument::getParamAlignment() const {
 /// it in its containing function.
 bool Argument::hasNestAttr() const {
   if (!getType()->isPointerTy()) return false;
-  return getParent()->paramHasAttr(getArgNo()+1, Attribute::Nest);
+  return getParent()->getParamAttributes(getArgNo()+1).
+    hasAttribute(Attributes::Nest);
 }
 
 /// hasNoAliasAttr - Return true if this argument has the noalias attribute on
 /// it in its containing function.
 bool Argument::hasNoAliasAttr() const {
   if (!getType()->isPointerTy()) return false;
-  return getParent()->paramHasAttr(getArgNo()+1, Attribute::NoAlias);
+  return getParent()->getParamAttributes(getArgNo()+1).
+    hasAttribute(Attributes::NoAlias);
 }
 
 /// hasNoCaptureAttr - Return true if this argument has the nocapture attribute
 /// on it in its containing function.
 bool Argument::hasNoCaptureAttr() const {
   if (!getType()->isPointerTy()) return false;
-  return getParent()->paramHasAttr(getArgNo()+1, Attribute::NoCapture);
+  return getParent()->getParamAttributes(getArgNo()+1).
+    hasAttribute(Attributes::NoCapture);
 }
 
 /// hasSRetAttr - Return true if this argument has the sret attribute on
@@ -114,7 +118,8 @@ bool Argument::hasStructRetAttr() const {
   if (!getType()->isPointerTy()) return false;
   if (this != getParent()->arg_begin())
     return false; // StructRet param must be first param
-  return getParent()->paramHasAttr(1, Attribute::StructRet);
+  return getParent()->getParamAttributes(1).
+    hasAttribute(Attributes::StructRet);
 }
 
 /// addAttr - Add a Attribute to an argument
@@ -180,7 +185,7 @@ Function::Function(FunctionType *Ty, LinkageTypes Linkage,
 
   // Ensure intrinsics have the right parameter attributes.
   if (unsigned IID = getIntrinsicID())
-    setAttributes(Intrinsic::getAttributes(Intrinsic::ID(IID)));
+    setAttributes(Intrinsic::getAttributes(getContext(), Intrinsic::ID(IID)));
 
 }
 
@@ -244,13 +249,13 @@ void Function::dropAllReferences() {
 
 void Function::addAttribute(unsigned i, Attributes attr) {
   AttrListPtr PAL = getAttributes();
-  PAL = PAL.addAttr(i, attr);
+  PAL = PAL.addAttr(getContext(), i, attr);
   setAttributes(PAL);
 }
 
 void Function::removeAttribute(unsigned i, Attributes attr) {
   AttrListPtr PAL = getAttributes();
-  PAL = PAL.removeAttr(i, attr);
+  PAL = PAL.removeAttr(getContext(), i, attr);
   setAttributes(PAL);
 }
 
diff --git a/lib/VMCore/GCOV.cpp b/lib/VMCore/GCOV.cpp
index 003a5d4..ea2f0a6 100644
--- a/lib/VMCore/GCOV.cpp
+++ b/lib/VMCore/GCOV.cpp
@@ -28,19 +28,19 @@ GCOVFile::~GCOVFile() {
 }
 
 /// isGCDAFile - Return true if Format identifies a .gcda file.
-static bool isGCDAFile(GCOVFormat Format) {
-  return Format == GCDA_402 || Format == GCDA_404;
+static bool isGCDAFile(GCOV::GCOVFormat Format) {
+  return Format == GCOV::GCDA_402 || Format == GCOV::GCDA_404;
 }
 
 /// isGCNOFile - Return true if Format identifies a .gcno file.
-static bool isGCNOFile(GCOVFormat Format) {
-  return Format == GCNO_402 || Format == GCNO_404;
+static bool isGCNOFile(GCOV::GCOVFormat Format) {
+  return Format == GCOV::GCNO_402 || Format == GCOV::GCNO_404;
 }
 
 /// read - Read GCOV buffer.
 bool GCOVFile::read(GCOVBuffer &Buffer) {
-  GCOVFormat Format = Buffer.readGCOVFormat();
-  if (Format == InvalidGCOV)
+  GCOV::GCOVFormat Format = Buffer.readGCOVFormat();
+  if (Format == GCOV::InvalidGCOV)
     return false;
 
   unsigned i = 0;
@@ -48,7 +48,7 @@ bool GCOVFile::read(GCOVBuffer &Buffer) {
     GCOVFunction *GFun = NULL;
     if (isGCDAFile(Format)) {
       // Use existing function while reading .gcda file.
-      assert (i < Functions.size() && ".gcda data does not match .gcno data");
+      assert(i < Functions.size() && ".gcda data does not match .gcno data");
       GFun = Functions[i];
     } else if (isGCNOFile(Format)){
       GFun = new GCOVFunction();
@@ -87,21 +87,21 @@ GCOVFunction::~GCOVFunction() {
 
 /// read - Read a aunction from the buffer. Return false if buffer cursor
 /// does not point to a function tag.
-bool GCOVFunction::read(GCOVBuffer &Buff, GCOVFormat Format) {
+bool GCOVFunction::read(GCOVBuffer &Buff, GCOV::GCOVFormat Format) {
   if (!Buff.readFunctionTag())
     return false;
 
   Buff.readInt(); // Function header length
   Ident = Buff.readInt(); 
   Buff.readInt(); // Checksum #1
-  if (Format != GCNO_402)
+  if (Format != GCOV::GCNO_402)
     Buff.readInt(); // Checksum #2
 
   Name = Buff.readString();
-  if (Format == GCNO_402 || Format == GCNO_404)
+  if (Format == GCOV::GCNO_402 || Format == GCOV::GCNO_404)
     Filename = Buff.readString();
 
-  if (Format == GCDA_402 || Format == GCDA_404) {
+  if (Format == GCOV::GCDA_402 || Format == GCOV::GCDA_404) {
     Buff.readArcTag();
     uint32_t Count = Buff.readInt() / 2;
     for (unsigned i = 0, e = Count; i != e; ++i) {
@@ -113,7 +113,9 @@ bool GCOVFunction::read(GCOVBuffer &Buff, GCOVFormat Format) {
   LineNumber = Buff.readInt();
 
   // read blocks.
-  assert (Buff.readBlockTag() && "Block Tag not found!");
+  bool BlockTagFound = Buff.readBlockTag();
+  (void)BlockTagFound;
+  assert(BlockTagFound && "Block Tag not found!");
   uint32_t BlockCount = Buff.readInt();
   for (int i = 0, e = BlockCount; i != e; ++i) {
     Buff.readInt(); // Block flags;
@@ -124,7 +126,7 @@ bool GCOVFunction::read(GCOVBuffer &Buff, GCOVFormat Format) {
   while (Buff.readEdgeTag()) {
     uint32_t EdgeCount = (Buff.readInt() - 1) / 2;
     uint32_t BlockNo = Buff.readInt();
-    assert (BlockNo < BlockCount && "Unexpected Block number!");
+    assert(BlockNo < BlockCount && "Unexpected Block number!");
     for (int i = 0, e = EdgeCount; i != e; ++i) {
       Blocks[BlockNo]->addEdge(Buff.readInt());
       Buff.readInt(); // Edge flag
@@ -136,7 +138,7 @@ bool GCOVFunction::read(GCOVBuffer &Buff, GCOVFormat Format) {
     uint32_t LineTableLength = Buff.readInt();
     uint32_t Size = Buff.getCursor() + LineTableLength*4;
     uint32_t BlockNo = Buff.readInt();
-    assert (BlockNo < BlockCount && "Unexpected Block number!");
+    assert(BlockNo < BlockCount && "Unexpected Block number!");
     GCOVBlock *Block = Blocks[BlockNo];
     Buff.readInt(); // flag
     while (Buff.getCursor() != (Size - 4)) {
diff --git a/lib/VMCore/IRBuilder.cpp b/lib/VMCore/IRBuilder.cpp
index 5c4e6d9..04f08fe 100644
--- a/lib/VMCore/IRBuilder.cpp
+++ b/lib/VMCore/IRBuilder.cpp
@@ -80,7 +80,7 @@ CreateMemSet(Value *Ptr, Value *Val, Value *Size, unsigned Align,
 
 CallInst *IRBuilderBase::
 CreateMemCpy(Value *Dst, Value *Src, Value *Size, unsigned Align,
-             bool isVolatile, MDNode *TBAATag) {
+             bool isVolatile, MDNode *TBAATag, MDNode *TBAAStructTag) {
   Dst = getCastedInt8PtrValue(Dst);
   Src = getCastedInt8PtrValue(Src);
 
@@ -94,6 +94,10 @@ CreateMemCpy(Value *Dst, Value *Src, Value *Size, unsigned Align,
   // Set the TBAA info if present.
   if (TBAATag)
     CI->setMetadata(LLVMContext::MD_tbaa, TBAATag);
+
+  // Set the TBAA Struct info if present.
+  if (TBAAStructTag)
+    CI->setMetadata(LLVMContext::MD_tbaa_struct, TBAAStructTag);
   
   return CI;  
 }
diff --git a/lib/VMCore/InlineAsm.cpp b/lib/VMCore/InlineAsm.cpp
index 736e370..2e636aa 100644
--- a/lib/VMCore/InlineAsm.cpp
+++ b/lib/VMCore/InlineAsm.cpp
@@ -27,19 +27,20 @@ InlineAsm::~InlineAsm() {
 
 InlineAsm *InlineAsm::get(FunctionType *Ty, StringRef AsmString,
                           StringRef Constraints, bool hasSideEffects,
-                          bool isAlignStack) {
-  InlineAsmKeyType Key(AsmString, Constraints, hasSideEffects, isAlignStack);
+                          bool isAlignStack, AsmDialect asmDialect) {
+  InlineAsmKeyType Key(AsmString, Constraints, hasSideEffects, isAlignStack,
+                       asmDialect);
   LLVMContextImpl *pImpl = Ty->getContext().pImpl;
   return pImpl->InlineAsms.getOrCreate(PointerType::getUnqual(Ty), Key);
 }
 
 InlineAsm::InlineAsm(PointerType *Ty, const std::string &asmString,
                      const std::string &constraints, bool hasSideEffects,
-                     bool isAlignStack)
+                     bool isAlignStack, AsmDialect asmDialect)
   : Value(Ty, Value::InlineAsmVal),
-    AsmString(asmString), 
-    Constraints(constraints), HasSideEffects(hasSideEffects), 
-    IsAlignStack(isAlignStack) {
+    AsmString(asmString), Constraints(constraints),
+    HasSideEffects(hasSideEffects), IsAlignStack(isAlignStack),
+    Dialect(asmDialect) {
 
   // Do various checks on the constraint string and type.
   assert(Verify(getFunctionType(), constraints) &&
diff --git a/lib/VMCore/Instructions.cpp b/lib/VMCore/Instructions.cpp
index 9af98e8..94bd2a1 100644
--- a/lib/VMCore/Instructions.cpp
+++ b/lib/VMCore/Instructions.cpp
@@ -332,21 +332,30 @@ CallInst::CallInst(const CallInst &CI)
 
 void CallInst::addAttribute(unsigned i, Attributes attr) {
   AttrListPtr PAL = getAttributes();
-  PAL = PAL.addAttr(i, attr);
+  PAL = PAL.addAttr(getContext(), i, attr);
   setAttributes(PAL);
 }
 
 void CallInst::removeAttribute(unsigned i, Attributes attr) {
   AttrListPtr PAL = getAttributes();
-  PAL = PAL.removeAttr(i, attr);
+  PAL = PAL.removeAttr(getContext(), i, attr);
   setAttributes(PAL);
 }
 
-bool CallInst::paramHasAttr(unsigned i, Attributes attr) const {
-  if (AttributeList.paramHasAttr(i, attr))
+bool CallInst::hasFnAttr(Attributes::AttrVal A) const {
+  if (AttributeList.getParamAttributes(AttrListPtr::FunctionIndex)
+      .hasAttribute(A))
     return true;
   if (const Function *F = getCalledFunction())
-    return F->paramHasAttr(i, attr);
+    return F->getParamAttributes(AttrListPtr::FunctionIndex).hasAttribute(A);
+  return false;
+}
+
+bool CallInst::paramHasAttr(unsigned i, Attributes::AttrVal A) const {
+  if (AttributeList.getParamAttributes(i).hasAttribute(A))
+    return true;
+  if (const Function *F = getCalledFunction())
+    return F->getParamAttributes(i).hasAttribute(A);
   return false;
 }
 
@@ -562,23 +571,32 @@ void InvokeInst::setSuccessorV(unsigned idx, BasicBlock *B) {
   return setSuccessor(idx, B);
 }
 
-bool InvokeInst::paramHasAttr(unsigned i, Attributes attr) const {
-  if (AttributeList.paramHasAttr(i, attr))
+bool InvokeInst::hasFnAttr(Attributes::AttrVal A) const {
+  if (AttributeList.getParamAttributes(AttrListPtr::FunctionIndex).
+      hasAttribute(A))
     return true;
   if (const Function *F = getCalledFunction())
-    return F->paramHasAttr(i, attr);
+    return F->getParamAttributes(AttrListPtr::FunctionIndex).hasAttribute(A);
+  return false;
+}
+
+bool InvokeInst::paramHasAttr(unsigned i, Attributes::AttrVal A) const {
+  if (AttributeList.getParamAttributes(i).hasAttribute(A))
+    return true;
+  if (const Function *F = getCalledFunction())
+    return F->getParamAttributes(i).hasAttribute(A);
   return false;
 }
 
 void InvokeInst::addAttribute(unsigned i, Attributes attr) {
   AttrListPtr PAL = getAttributes();
-  PAL = PAL.addAttr(i, attr);
+  PAL = PAL.addAttr(getContext(), i, attr);
   setAttributes(PAL);
 }
 
 void InvokeInst::removeAttribute(unsigned i, Attributes attr) {
   AttrListPtr PAL = getAttributes();
-  PAL = PAL.removeAttr(i, attr);
+  PAL = PAL.removeAttr(getContext(), i, attr);
   setAttributes(PAL);
 }
 
@@ -1381,18 +1399,6 @@ Type *GetElementPtrInst::getIndexedType(Type *Ptr, ArrayRef<uint64_t> IdxList) {
   return getIndexedTypeInternal(Ptr, IdxList);
 }
 
-unsigned GetElementPtrInst::getAddressSpace(Value *Ptr) {
-  Type *Ty = Ptr->getType();
-
-  if (VectorType *VTy = dyn_cast<VectorType>(Ty))
-    Ty = VTy->getElementType();
-
-  if (PointerType *PTy = dyn_cast<PointerType>(Ty))
-    return PTy->getAddressSpace();
-
-  llvm_unreachable("Invalid GEP pointer type");
-}
-
 /// hasAllZeroIndices - Return true if all of the indices of this GEP are
 /// zeros.  If so, the result pointer and the first operand have the same
 /// value, just potentially different types.
@@ -2112,7 +2118,8 @@ bool CastInst::isNoopCast(Type *IntPtrTy) const {
 /// If no such cast is permited, the function returns 0.
 unsigned CastInst::isEliminableCastPair(
   Instruction::CastOps firstOp, Instruction::CastOps secondOp,
-  Type *SrcTy, Type *MidTy, Type *DstTy, Type *IntPtrTy) {
+  Type *SrcTy, Type *MidTy, Type *DstTy, Type *SrcIntPtrTy, Type *MidIntPtrTy,
+  Type *DstIntPtrTy) {
   // Define the 144 possibilities for these two cast instructions. The values
   // in this matrix determine what to do in a given situation and select the
   // case in the switch below.  The rows correspond to firstOp, the columns 
@@ -2215,9 +2222,9 @@ unsigned CastInst::isEliminableCastPair(
       return 0;
     case 7: { 
       // ptrtoint, inttoptr -> bitcast (ptr -> ptr) if int size is >= ptr size
-      if (!IntPtrTy)
+      if (!SrcIntPtrTy || DstIntPtrTy != SrcIntPtrTy)
         return 0;
-      unsigned PtrSize = IntPtrTy->getScalarSizeInBits();
+      unsigned PtrSize = SrcIntPtrTy->getScalarSizeInBits();
       unsigned MidSize = MidTy->getScalarSizeInBits();
       if (MidSize >= PtrSize)
         return Instruction::BitCast;
@@ -2256,9 +2263,9 @@ unsigned CastInst::isEliminableCastPair(
       return 0;
     case 13: {
       // inttoptr, ptrtoint -> bitcast if SrcSize<=PtrSize and SrcSize==DstSize
-      if (!IntPtrTy)
+      if (!MidIntPtrTy)
         return 0;
-      unsigned PtrSize = IntPtrTy->getScalarSizeInBits();
+      unsigned PtrSize = MidIntPtrTy->getScalarSizeInBits();
       unsigned SrcSize = SrcTy->getScalarSizeInBits();
       unsigned DstSize = DstTy->getScalarSizeInBits();
       if (SrcSize <= PtrSize && SrcSize == DstSize)
@@ -2836,7 +2843,7 @@ BitCastInst::BitCastInst(
 //                               CmpInst Classes
 //===----------------------------------------------------------------------===//
 
-void CmpInst::Anchor() const {}
+void CmpInst::anchor() {}
 
 CmpInst::CmpInst(Type *ty, OtherOps op, unsigned short predicate,
                  Value *LHS, Value *RHS, const Twine &Name,
diff --git a/lib/VMCore/LLVMContext.cpp b/lib/VMCore/LLVMContext.cpp
index f07f0b3..2446ec9 100644
--- a/lib/VMCore/LLVMContext.cpp
+++ b/lib/VMCore/LLVMContext.cpp
@@ -53,6 +53,11 @@ LLVMContext::LLVMContext() : pImpl(new LLVMContextImpl(*this)) {
   unsigned RangeID = getMDKindID("range");
   assert(RangeID == MD_range && "range kind id drifted");
   (void)RangeID;
+
+  // Create the 'tbaa.struct' metadata kind.
+  unsigned TBAAStructID = getMDKindID("tbaa.struct");
+  assert(TBAAStructID == MD_tbaa_struct && "tbaa.struct kind id drifted");
+  (void)TBAAStructID;
 }
 LLVMContext::~LLVMContext() { delete pImpl; }
 
diff --git a/lib/VMCore/LLVMContextImpl.cpp b/lib/VMCore/LLVMContextImpl.cpp
index 6279bb8..d35d284 100644
--- a/lib/VMCore/LLVMContextImpl.cpp
+++ b/lib/VMCore/LLVMContextImpl.cpp
@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "LLVMContextImpl.h"
+#include "llvm/Attributes.h"
 #include "llvm/Module.h"
 #include "llvm/ADT/STLExtras.h"
 #include <algorithm>
@@ -93,7 +94,21 @@ LLVMContextImpl::~LLVMContextImpl() {
        E = CDSConstants.end(); I != E; ++I)
     delete I->second;
   CDSConstants.clear();
-  
+
+  // Destroy attributes.
+  for (FoldingSetIterator<AttributesImpl> I = AttrsSet.begin(),
+         E = AttrsSet.end(); I != E; ) {
+    FoldingSetIterator<AttributesImpl> Elem = I++;
+    delete &*Elem;
+  }
+
+  // Destroy attribute lists.
+  for (FoldingSetIterator<AttributeListImpl> I = AttrsLists.begin(),
+         E = AttrsLists.end(); I != E; ) {
+    FoldingSetIterator<AttributeListImpl> Elem = I++;
+    delete &*Elem;
+  }
+
   // Destroy MDNodes.  ~MDNode can move and remove nodes between the MDNodeSet
   // and the NonUniquedMDNodes sets, so copy the values out first.
   SmallVector<MDNode*, 8> MDNodes;
@@ -107,6 +122,7 @@ LLVMContextImpl::~LLVMContextImpl() {
     (*I)->destroy();
   assert(MDNodeSet.empty() && NonUniquedMDNodes.empty() &&
          "Destroying all MDNodes didn't empty the Context's sets.");
+
   // Destroy MDStrings.
   DeleteContainerSeconds(MDStringCache);
 }
diff --git a/lib/VMCore/LLVMContextImpl.h b/lib/VMCore/LLVMContextImpl.h
index 2252028..90cf424 100644
--- a/lib/VMCore/LLVMContextImpl.h
+++ b/lib/VMCore/LLVMContextImpl.h
@@ -16,6 +16,7 @@
 #define LLVM_LLVMCONTEXT_IMPL_H
 
 #include "llvm/LLVMContext.h"
+#include "AttributesImpl.h"
 #include "ConstantsContext.h"
 #include "LeaksContext.h"
 #include "llvm/Constants.h"
@@ -253,10 +254,14 @@ public:
   typedef DenseMap<DenseMapAPFloatKeyInfo::KeyTy, ConstantFP*, 
                          DenseMapAPFloatKeyInfo> FPMapTy;
   FPMapTy FPConstants;
-  
+
+  FoldingSet<AttributesImpl> AttrsSet;
+  FoldingSet<AttributeListImpl> AttrsLists;
+
   StringMap<Value*> MDStringCache;
-  
+
   FoldingSet<MDNode> MDNodeSet;
+
   // MDNodes may be uniqued or not uniqued.  When they're not uniqued, they
   // aren't in the MDNodeSet, but they're still shared between objects, so no
   // one object can destroy them.  This set allows us to at least destroy them
diff --git a/lib/VMCore/Makefile b/lib/VMCore/Makefile
index 2b9b0f2..8b98651 100644
--- a/lib/VMCore/Makefile
+++ b/lib/VMCore/Makefile
@@ -9,7 +9,6 @@
 LEVEL = ../..
 LIBRARYNAME = LLVMCore
 BUILD_ARCHIVE = 1
-REQUIRES_RTTI = 1
 
 BUILT_SOURCES = $(PROJ_OBJ_ROOT)/include/llvm/Intrinsics.gen
 
diff --git a/lib/VMCore/PassManager.cpp b/lib/VMCore/PassManager.cpp
index 4530c04..53f1149 100644
--- a/lib/VMCore/PassManager.cpp
+++ b/lib/VMCore/PassManager.cpp
@@ -1189,7 +1189,7 @@ void PMDataManager::dumpAnalysisUsage(StringRef Msg, const Pass *P,
   assert(PassDebugging >= Details);
   if (Set.empty())
     return;
-  dbgs() << (void*)P << std::string(getDepth()*2+3, ' ') << Msg << " Analyses:";
+  dbgs() << (const void*)P << std::string(getDepth()*2+3, ' ') << Msg << " Analyses:";
   for (unsigned i = 0; i != Set.size(); ++i) {
     if (i) dbgs() << ',';
     const PassInfo *PInf = PassRegistry::getPassRegistry()->getPassInfo(Set[i]);
diff --git a/lib/VMCore/TargetTransformInfo.cpp b/lib/VMCore/TargetTransformInfo.cpp
new file mode 100644
index 0000000..e91c29c
--- /dev/null
+++ b/lib/VMCore/TargetTransformInfo.cpp
@@ -0,0 +1,31 @@
+//===- llvm/VMCore/TargetTransformInfo.cpp ----------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/TargetTransformInfo.h"
+#include "llvm/Support/ErrorHandling.h"
+
+using namespace llvm;
+
+/// Default ctor.
+///
+/// @note This has to exist, because this is a pass, but it should never be
+/// used.
+TargetTransformInfo::TargetTransformInfo() : ImmutablePass(ID) {
+  /// You are seeing this error because your pass required the TTI
+  /// using a call to "getAnalysis<TargetTransformInfo>()", and you did
+  /// not initialize a machine target which can provide the TTI.
+  /// You should use "getAnalysisIfAvailable<TargetTransformInfo>()" instead.
+  report_fatal_error("Bad TargetTransformInfo ctor used.  "
+                     "Tool did not specify a TargetTransformInfo to use?");
+}
+
+INITIALIZE_PASS(TargetTransformInfo, "targettransforminfo",
+                "Target Transform Info", false, true)
+char TargetTransformInfo::ID = 0;
+
diff --git a/lib/VMCore/Type.cpp b/lib/VMCore/Type.cpp
index 5e9a00f..1656ab2 100644
--- a/lib/VMCore/Type.cpp
+++ b/lib/VMCore/Type.cpp
@@ -47,35 +47,17 @@ Type *Type::getScalarType() {
   return this;
 }
 
+const Type *Type::getScalarType() const {
+  if (const VectorType *VTy = dyn_cast<VectorType>(this))
+    return VTy->getElementType();
+  return this;
+}
+
 /// isIntegerTy - Return true if this is an IntegerType of the specified width.
 bool Type::isIntegerTy(unsigned Bitwidth) const {
   return isIntegerTy() && cast<IntegerType>(this)->getBitWidth() == Bitwidth;
 }
 
-/// isIntOrIntVectorTy - Return true if this is an integer type or a vector of
-/// integer types.
-///
-bool Type::isIntOrIntVectorTy() const {
-  if (isIntegerTy())
-    return true;
-  if (getTypeID() != Type::VectorTyID) return false;
-  
-  return cast<VectorType>(this)->getElementType()->isIntegerTy();
-}
-
-/// isFPOrFPVectorTy - Return true if this is a FP type or a vector of FP types.
-///
-bool Type::isFPOrFPVectorTy() const {
-  if (getTypeID() == Type::HalfTyID || getTypeID() == Type::FloatTyID ||
-      getTypeID() == Type::DoubleTyID ||
-      getTypeID() == Type::FP128TyID || getTypeID() == Type::X86_FP80TyID || 
-      getTypeID() == Type::PPC_FP128TyID)
-    return true;
-  if (getTypeID() != Type::VectorTyID) return false;
-  
-  return cast<VectorType>(this)->getElementType()->isFloatingPointTy();
-}
-
 // canLosslesslyBitCastTo - Return true if this type can be converted to
 // 'Ty' without any reinterpretation of bits.  For example, i8* to i32*.
 //
@@ -220,8 +202,6 @@ Type *Type::getStructElementType(unsigned N) const {
   return cast<StructType>(this)->getElementType(N);
 }
 
-
-
 Type *Type::getSequentialElementType() const {
   return cast<SequentialType>(this)->getElementType();
 }
@@ -235,12 +215,10 @@ unsigned Type::getVectorNumElements() const {
 }
 
 unsigned Type::getPointerAddressSpace() const {
-  return cast<PointerType>(this)->getAddressSpace();
+  return cast<PointerType>(getScalarType())->getAddressSpace();
 }
 
 
-
-
 //===----------------------------------------------------------------------===//
 //                          Primitive 'Type' data
 //===----------------------------------------------------------------------===//
@@ -400,12 +378,10 @@ FunctionType *FunctionType::get(Type *ReturnType,
   return FT;
 }
 
-
 FunctionType *FunctionType::get(Type *Result, bool isVarArg) {
   return get(Result, ArrayRef<Type *>(), isVarArg);
 }
 
-
 /// isValidReturnType - Return true if the specified type is valid as a return
 /// type.
 bool FunctionType::isValidReturnType(Type *RetTy) {
@@ -553,7 +529,6 @@ StructType *StructType::create(LLVMContext &Context) {
   return create(Context, StringRef());
 }
 
-
 StructType *StructType::create(ArrayRef<Type*> Elements, StringRef Name,
                                bool isPacked) {
   assert(!Elements.empty() &&
@@ -637,7 +612,6 @@ bool StructType::isLayoutIdentical(StructType *Other) const {
   return std::equal(element_begin(), element_end(), Other->element_begin());
 }
 
-
 /// getTypeByName - Return the type with the specified name, or null if there
 /// is none by that name.
 StructType *Module::getTypeByName(StringRef Name) const {
@@ -700,7 +674,6 @@ ArrayType::ArrayType(Type *ElType, uint64_t NumEl)
   NumElements = NumEl;
 }
 
-
 ArrayType *ArrayType::get(Type *elementType, uint64_t NumElements) {
   Type *ElementType = const_cast<Type*>(elementType);
   assert(isValidElementType(ElementType) && "Invalid type for array element!");
diff --git a/lib/VMCore/User.cpp b/lib/VMCore/User.cpp
index 5f35ce4..e847ce6 100644
--- a/lib/VMCore/User.cpp
+++ b/lib/VMCore/User.cpp
@@ -10,6 +10,7 @@
 #include "llvm/Constant.h"
 #include "llvm/GlobalValue.h"
 #include "llvm/User.h"
+#include "llvm/Operator.h"
 
 namespace llvm {
 
@@ -78,4 +79,12 @@ void User::operator delete(void *Usr) {
   ::operator delete(Storage);
 }
 
+//===----------------------------------------------------------------------===//
+//                             Operator Class
+//===----------------------------------------------------------------------===//
+
+Operator::~Operator() {
+  llvm_unreachable("should never destroy an Operator");
+}
+
 } // End llvm namespace
diff --git a/lib/VMCore/Value.cpp b/lib/VMCore/Value.cpp
index d871108..8d0720d 100644
--- a/lib/VMCore/Value.cpp
+++ b/lib/VMCore/Value.cpp
@@ -394,7 +394,7 @@ static bool isDereferenceablePointer(const Value *V,
   // It's also not always safe to follow a bitcast, for example:
   //   bitcast i8* (alloca i8) to i32*
   // would result in a 4-byte load from a 1-byte alloca. Some cases could
-  // be handled using TargetData to check sizes and alignments though.
+  // be handled using DataLayout to check sizes and alignments though.
 
   // These are obviously ok.
   if (isa<AllocaInst>(V)) return true;
diff --git a/lib/VMCore/ValueTypes.cpp b/lib/VMCore/ValueTypes.cpp
index d1ca953..2ee9f0f 100644
--- a/lib/VMCore/ValueTypes.cpp
+++ b/lib/VMCore/ValueTypes.cpp
@@ -55,24 +55,32 @@ bool EVT::isExtendedVector() const {
   return LLVMTy->isVectorTy();
 }
 
+bool EVT::isExtended16BitVector() const {
+  return isExtendedVector() && getExtendedSizeInBits() == 16;
+}
+
+bool EVT::isExtended32BitVector() const {
+  return isExtendedVector() && getExtendedSizeInBits() == 32;
+}
+
 bool EVT::isExtended64BitVector() const {
-  return isExtendedVector() && getSizeInBits() == 64;
+  return isExtendedVector() && getExtendedSizeInBits() == 64;
 }
 
 bool EVT::isExtended128BitVector() const {
-  return isExtendedVector() && getSizeInBits() == 128;
+  return isExtendedVector() && getExtendedSizeInBits() == 128;
 }
 
 bool EVT::isExtended256BitVector() const {
-  return isExtendedVector() && getSizeInBits() == 256;
+  return isExtendedVector() && getExtendedSizeInBits() == 256;
 }
 
 bool EVT::isExtended512BitVector() const {
-  return isExtendedVector() && getSizeInBits() == 512;
+  return isExtendedVector() && getExtendedSizeInBits() == 512;
 }
 
 bool EVT::isExtended1024BitVector() const {
-  return isExtendedVector() && getSizeInBits() == 1024;
+  return isExtendedVector() && getExtendedSizeInBits() == 1024;
 }
 
 EVT EVT::getExtendedVectorElementType() const {
@@ -120,15 +128,21 @@ std::string EVT::getEVTString() const {
   case MVT::Other:   return "ch";
   case MVT::Glue:    return "glue";
   case MVT::x86mmx:  return "x86mmx";
+  case MVT::v2i1:    return "v2i1";
+  case MVT::v4i1:    return "v4i1";
+  case MVT::v8i1:    return "v8i1";
+  case MVT::v16i1:   return "v16i1";
   case MVT::v2i8:    return "v2i8";
   case MVT::v4i8:    return "v4i8";
   case MVT::v8i8:    return "v8i8";
   case MVT::v16i8:   return "v16i8";
   case MVT::v32i8:   return "v32i8";
+  case MVT::v1i16:   return "v1i16";
   case MVT::v2i16:   return "v2i16";
   case MVT::v4i16:   return "v4i16";
   case MVT::v8i16:   return "v8i16";
   case MVT::v16i16:  return "v16i16";
+  case MVT::v1i32:   return "v1i32";
   case MVT::v2i32:   return "v2i32";
   case MVT::v4i32:   return "v4i32";
   case MVT::v8i32:   return "v8i32";
@@ -171,15 +185,21 @@ Type *EVT::getTypeForEVT(LLVMContext &Context) const {
   case MVT::f128:    return Type::getFP128Ty(Context);
   case MVT::ppcf128: return Type::getPPC_FP128Ty(Context);
   case MVT::x86mmx:  return Type::getX86_MMXTy(Context);
+  case MVT::v2i1:    return VectorType::get(Type::getInt1Ty(Context), 2);
+  case MVT::v4i1:    return VectorType::get(Type::getInt1Ty(Context), 4);
+  case MVT::v8i1:    return VectorType::get(Type::getInt1Ty(Context), 8);
+  case MVT::v16i1:   return VectorType::get(Type::getInt1Ty(Context), 16);
   case MVT::v2i8:    return VectorType::get(Type::getInt8Ty(Context), 2);
   case MVT::v4i8:    return VectorType::get(Type::getInt8Ty(Context), 4);
   case MVT::v8i8:    return VectorType::get(Type::getInt8Ty(Context), 8);
   case MVT::v16i8:   return VectorType::get(Type::getInt8Ty(Context), 16);
   case MVT::v32i8:   return VectorType::get(Type::getInt8Ty(Context), 32);
+  case MVT::v1i16:   return VectorType::get(Type::getInt16Ty(Context), 1);
   case MVT::v2i16:   return VectorType::get(Type::getInt16Ty(Context), 2);
   case MVT::v4i16:   return VectorType::get(Type::getInt16Ty(Context), 4);
   case MVT::v8i16:   return VectorType::get(Type::getInt16Ty(Context), 8);
   case MVT::v16i16:  return VectorType::get(Type::getInt16Ty(Context), 16);
+  case MVT::v1i32:   return VectorType::get(Type::getInt32Ty(Context), 1);
   case MVT::v2i32:   return VectorType::get(Type::getInt32Ty(Context), 2);
   case MVT::v4i32:   return VectorType::get(Type::getInt32Ty(Context), 4);
   case MVT::v8i32:   return VectorType::get(Type::getInt32Ty(Context), 8);
diff --git a/lib/VMCore/Verifier.cpp b/lib/VMCore/Verifier.cpp
index 38914b3..eb40b09 100644
--- a/lib/VMCore/Verifier.cpp
+++ b/lib/VMCore/Verifier.cpp
@@ -400,8 +400,8 @@ void Verifier::visitGlobalValue(GlobalValue &GV) {
             "Only global arrays can have appending linkage!", GVar);
   }
 
-  Assert1(!GV.hasLinkerPrivateWeakDefAutoLinkage() || GV.hasDefaultVisibility(),
-          "linker_private_weak_def_auto can only have default visibility!",
+  Assert1(!GV.hasLinkOnceODRAutoHideLinkage() || GV.hasDefaultVisibility(),
+          "linkonce_odr_auto_hide can only have default visibility!",
           &GV);
 }
 
@@ -526,40 +526,60 @@ void Verifier::visitMDNode(MDNode &MD, Function *F) {
 // value of the specified type.  The value V is printed in error messages.
 void Verifier::VerifyParameterAttrs(Attributes Attrs, Type *Ty,
                                     bool isReturnValue, const Value *V) {
-  if (Attrs == Attribute::None)
+  if (!Attrs.hasAttributes())
     return;
 
-  Attributes FnCheckAttr = Attrs & Attribute::FunctionOnly;
-  Assert1(!FnCheckAttr, "Attribute " + Attribute::getAsString(FnCheckAttr) +
-          " only applies to the function!", V);
-
-  if (isReturnValue) {
-    Attributes RetI = Attrs & Attribute::ParameterOnly;
-    Assert1(!RetI, "Attribute " + Attribute::getAsString(RetI) +
-            " does not apply to return values!", V);
-  }
-
-  for (unsigned i = 0;
-       i < array_lengthof(Attribute::MutuallyIncompatible); ++i) {
-    Attributes MutI = Attrs & Attribute::MutuallyIncompatible[i];
-    Assert1(MutI.isEmptyOrSingleton(), "Attributes " +
-            Attribute::getAsString(MutI) + " are incompatible!", V);
-  }
-
-  Attributes TypeI = Attrs & Attribute::typeIncompatible(Ty);
-  Assert1(!TypeI, "Wrong type for attribute " +
-          Attribute::getAsString(TypeI), V);
-
-  Attributes ByValI = Attrs & Attribute::ByVal;
-  if (PointerType *PTy = dyn_cast<PointerType>(Ty)) {
-    Assert1(!ByValI || PTy->getElementType()->isSized(),
-            "Attribute " + Attribute::getAsString(ByValI) +
-            " does not support unsized types!", V);
-  } else {
-    Assert1(!ByValI,
-            "Attribute " + Attribute::getAsString(ByValI) +
-            " only applies to parameters with pointer type!", V);
-  }
+  Assert1(!Attrs.hasFunctionOnlyAttrs(),
+          "Some attributes in '" + Attrs.getAsString() +
+          "' only apply to functions!", V);
+
+  if (isReturnValue)
+    Assert1(!Attrs.hasParameterOnlyAttrs(),
+            "Attributes 'byval', 'nest', 'sret', and 'nocapture' "
+            "do not apply to return values!", V);
+
+  // Check for mutually incompatible attributes.
+  Assert1(!((Attrs.hasAttribute(Attributes::ByVal) &&
+             Attrs.hasAttribute(Attributes::Nest)) ||
+            (Attrs.hasAttribute(Attributes::ByVal) &&
+             Attrs.hasAttribute(Attributes::StructRet)) ||
+            (Attrs.hasAttribute(Attributes::Nest) &&
+             Attrs.hasAttribute(Attributes::StructRet))), "Attributes "
+          "'byval, nest, and sret' are incompatible!", V);
+
+  Assert1(!((Attrs.hasAttribute(Attributes::ByVal) &&
+             Attrs.hasAttribute(Attributes::Nest)) ||
+            (Attrs.hasAttribute(Attributes::ByVal) &&
+             Attrs.hasAttribute(Attributes::InReg)) ||
+            (Attrs.hasAttribute(Attributes::Nest) &&
+             Attrs.hasAttribute(Attributes::InReg))), "Attributes "
+          "'byval, nest, and inreg' are incompatible!", V);
+
+  Assert1(!(Attrs.hasAttribute(Attributes::ZExt) &&
+            Attrs.hasAttribute(Attributes::SExt)), "Attributes "
+          "'zeroext and signext' are incompatible!", V);
+
+  Assert1(!(Attrs.hasAttribute(Attributes::ReadNone) &&
+            Attrs.hasAttribute(Attributes::ReadOnly)), "Attributes "
+          "'readnone and readonly' are incompatible!", V);
+
+  Assert1(!(Attrs.hasAttribute(Attributes::NoInline) &&
+            Attrs.hasAttribute(Attributes::AlwaysInline)), "Attributes "
+          "'noinline and alwaysinline' are incompatible!", V);
+
+  Assert1(!AttrBuilder(Attrs).
+            hasAttributes(Attributes::typeIncompatible(Ty)),
+          "Wrong types for attribute: " +
+          Attributes::typeIncompatible(Ty).getAsString(), V);
+
+  if (PointerType *PTy = dyn_cast<PointerType>(Ty))
+    Assert1(!Attrs.hasAttribute(Attributes::ByVal) ||
+            PTy->getElementType()->isSized(),
+            "Attribute 'byval' does not support unsized types!", V);
+  else
+    Assert1(!Attrs.hasAttribute(Attributes::ByVal),
+            "Attribute 'byval' only applies to parameters with pointer type!",
+            V);
 }
 
 // VerifyFunctionAttrs - Check parameter attributes against a function type.
@@ -585,26 +605,50 @@ void Verifier::VerifyFunctionAttrs(FunctionType *FT,
 
     VerifyParameterAttrs(Attr.Attrs, Ty, Attr.Index == 0, V);
 
-    if (Attr.Attrs & Attribute::Nest) {
+    if (Attr.Attrs.hasAttribute(Attributes::Nest)) {
       Assert1(!SawNest, "More than one parameter has attribute nest!", V);
       SawNest = true;
     }
 
-    if (Attr.Attrs & Attribute::StructRet)
+    if (Attr.Attrs.hasAttribute(Attributes::StructRet))
       Assert1(Attr.Index == 1, "Attribute sret not on first parameter!", V);
   }
 
   Attributes FAttrs = Attrs.getFnAttributes();
-  Attributes NotFn = FAttrs & (~Attribute::FunctionOnly);
-  Assert1(!NotFn, "Attribute " + Attribute::getAsString(NotFn) +
-          " does not apply to the function!", V);
-
-  for (unsigned i = 0;
-       i < array_lengthof(Attribute::MutuallyIncompatible); ++i) {
-    Attributes MutI = FAttrs & Attribute::MutuallyIncompatible[i];
-    Assert1(MutI.isEmptyOrSingleton(), "Attributes " +
-            Attribute::getAsString(MutI) + " are incompatible!", V);
-  }
+  AttrBuilder NotFn(FAttrs);
+  NotFn.removeFunctionOnlyAttrs();
+  Assert1(!NotFn.hasAttributes(), "Attributes '" +
+          Attributes::get(V->getContext(), NotFn).getAsString() +
+          "' do not apply to the function!", V);
+
+  // Check for mutually incompatible attributes.
+  Assert1(!((FAttrs.hasAttribute(Attributes::ByVal) &&
+             FAttrs.hasAttribute(Attributes::Nest)) ||
+            (FAttrs.hasAttribute(Attributes::ByVal) &&
+             FAttrs.hasAttribute(Attributes::StructRet)) ||
+            (FAttrs.hasAttribute(Attributes::Nest) &&
+             FAttrs.hasAttribute(Attributes::StructRet))), "Attributes "
+          "'byval, nest, and sret' are incompatible!", V);
+
+  Assert1(!((FAttrs.hasAttribute(Attributes::ByVal) &&
+             FAttrs.hasAttribute(Attributes::Nest)) ||
+            (FAttrs.hasAttribute(Attributes::ByVal) &&
+             FAttrs.hasAttribute(Attributes::InReg)) ||
+            (FAttrs.hasAttribute(Attributes::Nest) &&
+             FAttrs.hasAttribute(Attributes::InReg))), "Attributes "
+          "'byval, nest, and inreg' are incompatible!", V);
+
+  Assert1(!(FAttrs.hasAttribute(Attributes::ZExt) &&
+            FAttrs.hasAttribute(Attributes::SExt)), "Attributes "
+          "'zeroext and signext' are incompatible!", V);
+
+  Assert1(!(FAttrs.hasAttribute(Attributes::ReadNone) &&
+            FAttrs.hasAttribute(Attributes::ReadOnly)), "Attributes "
+          "'readnone and readonly' are incompatible!", V);
+
+  Assert1(!(FAttrs.hasAttribute(Attributes::NoInline) &&
+            FAttrs.hasAttribute(Attributes::AlwaysInline)), "Attributes "
+          "'noinline and alwaysinline' are incompatible!", V);
 }
 
 static bool VerifyAttributeCount(const AttrListPtr &Attrs, unsigned Params) {
@@ -661,6 +705,7 @@ void Verifier::visitFunction(Function &F) {
   case CallingConv::Cold:
   case CallingConv::X86_FastCall:
   case CallingConv::X86_ThisCall:
+  case CallingConv::Intel_OCL_BI:
   case CallingConv::PTX_Kernel:
   case CallingConv::PTX_Device:
     Assert1(!F.isVarArg(),
@@ -1170,9 +1215,8 @@ void Verifier::VerifyCallSite(CallSite CS) {
 
       VerifyParameterAttrs(Attr, CS.getArgument(Idx-1)->getType(), false, I);
 
-      Attributes VArgI = Attr & Attribute::VarArgsIncompatible;
-      Assert1(!VArgI, "Attribute " + Attribute::getAsString(VArgI) +
-              " cannot be used for vararg call arguments!", I);
+      Assert1(!Attr.hasIncompatibleWithVarArgsAttrs(),
+              "Attribute 'sret' cannot be used for vararg call arguments!", I);
     }
 
   // Verify that there's no metadata unless it's a direct call to an intrinsic.
@@ -1378,6 +1422,15 @@ void Verifier::visitLoadInst(LoadInst &LI) {
             "Load cannot have Release ordering", &LI);
     Assert1(LI.getAlignment() != 0,
             "Atomic load must specify explicit alignment", &LI);
+    if (!ElTy->isPointerTy()) {
+      Assert2(ElTy->isIntegerTy(),
+              "atomic store operand must have integer type!",
+              &LI, ElTy);
+      unsigned Size = ElTy->getPrimitiveSizeInBits();
+      Assert2(Size >= 8 && !(Size & (Size - 1)),
+              "atomic store operand must be power-of-two byte-sized integer",
+              &LI, ElTy);
+    }
   } else {
     Assert1(LI.getSynchScope() == CrossThread,
             "Non-atomic load cannot have SynchronizationScope specified", &LI);
@@ -1444,6 +1497,15 @@ void Verifier::visitStoreInst(StoreInst &SI) {
             "Store cannot have Acquire ordering", &SI);
     Assert1(SI.getAlignment() != 0,
             "Atomic store must specify explicit alignment", &SI);
+    if (!ElTy->isPointerTy()) {
+      Assert2(ElTy->isIntegerTy(),
+              "atomic store operand must have integer type!",
+              &SI, ElTy);
+      unsigned Size = ElTy->getPrimitiveSizeInBits();
+      Assert2(Size >= 8 && !(Size & (Size - 1)),
+              "atomic store operand must be power-of-two byte-sized integer",
+              &SI, ElTy);
+    }
   } else {
     Assert1(SI.getSynchScope() == CrossThread,
             "Non-atomic store cannot have SynchronizationScope specified", &SI);
@@ -1471,6 +1533,13 @@ void Verifier::visitAtomicCmpXchgInst(AtomicCmpXchgInst &CXI) {
   PointerType *PTy = dyn_cast<PointerType>(CXI.getOperand(0)->getType());
   Assert1(PTy, "First cmpxchg operand must be a pointer.", &CXI);
   Type *ElTy = PTy->getElementType();
+  Assert2(ElTy->isIntegerTy(),
+          "cmpxchg operand must have integer type!",
+          &CXI, ElTy);
+  unsigned Size = ElTy->getPrimitiveSizeInBits();
+  Assert2(Size >= 8 && !(Size & (Size - 1)),
+          "cmpxchg operand must be power-of-two byte-sized integer",
+          &CXI, ElTy);
   Assert2(ElTy == CXI.getOperand(1)->getType(),
           "Expected value type does not match pointer operand type!",
           &CXI, ElTy);
@@ -1488,6 +1557,13 @@ void Verifier::visitAtomicRMWInst(AtomicRMWInst &RMWI) {
   PointerType *PTy = dyn_cast<PointerType>(RMWI.getOperand(0)->getType());
   Assert1(PTy, "First atomicrmw operand must be a pointer.", &RMWI);
   Type *ElTy = PTy->getElementType();
+  Assert2(ElTy->isIntegerTy(),
+          "atomicrmw operand must have integer type!",
+          &RMWI, ElTy);
+  unsigned Size = ElTy->getPrimitiveSizeInBits();
+  Assert2(Size >= 8 && !(Size & (Size - 1)),
+          "atomicrmw operand must be power-of-two byte-sized integer",
+          &RMWI, ElTy);
   Assert2(ElTy == RMWI.getOperand(1)->getType(),
           "Argument value type does not match pointer operand type!",
           &RMWI, ElTy);
@@ -1575,6 +1651,13 @@ void Verifier::visitLandingPadInst(LandingPadInst &LPI) {
 
 void Verifier::verifyDominatesUse(Instruction &I, unsigned i) {
   Instruction *Op = cast<Instruction>(I.getOperand(i));
+  // If the we have an invalid invoke, don't try to compute the dominance.
+  // We already reject it in the invoke specific checks and the dominance
+  // computation doesn't handle multiple edges.
+  if (InvokeInst *II = dyn_cast<InvokeInst>(Op)) {
+    if (II->getNormalDest() == II->getUnwindDest())
+      return;
+  }
 
   const Use &U = I.getOperandUse(i);
   Assert2(InstsInThisBlock.count(Op) || DT->dominates(Op, U),
-- 
cgit v1.1


From 056abd2059c65a3e908193aeae16fad98017437c Mon Sep 17 00:00:00 2001
From: dim <dim@FreeBSD.org>
Date: Sun, 2 Dec 2012 13:20:44 +0000
Subject: Vendor import of clang release_32 branch r168974 (effectively, 3.2
 RC2): http://llvm.org/svn/llvm-project/cfe/branches/release_32@168974

---
 lib/ARCMigrate/ARCMT.cpp                           |   75 +-
 lib/ARCMigrate/CMakeLists.txt                      |    3 +-
 lib/ARCMigrate/FileRemapper.cpp                    |    2 +-
 lib/ARCMigrate/Internals.h                         |    2 +
 lib/ARCMigrate/ObjCMT.cpp                          |    2 +-
 lib/ARCMigrate/Transforms.cpp                      |    4 +-
 lib/AST/ASTConsumer.cpp                            |    5 +
 lib/AST/ASTContext.cpp                             |  325 +-
 lib/AST/ASTDiagnostic.cpp                          |  277 +-
 lib/AST/ASTImporter.cpp                            |  478 +-
 lib/AST/CMakeLists.txt                             |    3 +
 lib/AST/CXXInheritance.cpp                         |   32 +-
 lib/AST/Comment.cpp                                |   93 +-
 lib/AST/CommentBriefParser.cpp                     |   51 +-
 lib/AST/CommentCommandTraits.cpp                   |  141 +-
 lib/AST/CommentDumper.cpp                          |   48 +-
 lib/AST/CommentLexer.cpp                           |   62 +-
 lib/AST/CommentParser.cpp                          |   72 +-
 lib/AST/CommentSema.cpp                            |  314 +-
 lib/AST/Decl.cpp                                   |   88 +-
 lib/AST/DeclBase.cpp                               |    2 +-
 lib/AST/DeclCXX.cpp                                |   30 +-
 lib/AST/DeclObjC.cpp                               |  269 +-
 lib/AST/DeclPrinter.cpp                            |   18 +-
 lib/AST/DeclTemplate.cpp                           |   86 +-
 lib/AST/DumpXML.cpp                                |   23 +-
 lib/AST/Expr.cpp                                   |  299 +-
 lib/AST/ExprCXX.cpp                                |  121 +-
 lib/AST/ExprClassification.cpp                     |    1 +
 lib/AST/ExprConstant.cpp                           |   61 +-
 lib/AST/ItaniumMangle.cpp                          |  133 +-
 lib/AST/MicrosoftMangle.cpp                        |  242 +-
 lib/AST/NSAPI.cpp                                  |    1 +
 lib/AST/ParentMap.cpp                              |   69 +-
 lib/AST/RawCommentList.cpp                         |   69 +-
 lib/AST/RecordLayoutBuilder.cpp                    |   84 +-
 lib/AST/Stmt.cpp                                   |  167 +-
 lib/AST/StmtDumper.cpp                             |  115 +-
 lib/AST/StmtPrinter.cpp                            |   68 +-
 lib/AST/StmtProfile.cpp                            |   16 +-
 lib/AST/TemplateBase.cpp                           |   63 +-
 lib/AST/Type.cpp                                   |   55 +-
 lib/AST/TypeLoc.cpp                                |   55 +-
 lib/AST/TypePrinter.cpp                            |   14 +-
 lib/AST/VTableBuilder.cpp                          |    3 +
 lib/ASTMatchers/ASTMatchFinder.cpp                 |  411 +-
 lib/ASTMatchers/ASTMatchersInternal.cpp            |   66 +-
 lib/Analysis/AnalysisDeclContext.cpp               |   41 +-
 lib/Analysis/BodyFarm.cpp                          |  374 +
 lib/Analysis/BodyFarm.h                            |   43 +
 lib/Analysis/CFG.cpp                               |   83 +-
 lib/Analysis/CMakeLists.txt                        |    4 +-
 lib/Analysis/FormatString.cpp                      |   57 +-
 lib/Analysis/ObjCNoReturn.cpp                      |   67 +
 lib/Analysis/PrintfFormatString.cpp                |   43 +-
 lib/Analysis/ReachableCode.cpp                     |    4 +-
 lib/Analysis/ScanfFormatString.cpp                 |   36 +-
 lib/Analysis/ThreadSafety.cpp                      |  340 +-
 lib/Analysis/UninitializedValues.cpp               |   64 +-
 lib/Basic/ConvertUTF.c                             |   24 +-
 lib/Basic/ConvertUTFWrapper.cpp                    |   18 +-
 lib/Basic/Diagnostic.cpp                           |   25 +-
 lib/Basic/DiagnosticIDs.cpp                        |    5 +-
 lib/Basic/FileManager.cpp                          |    7 +
 lib/Basic/IdentifierTable.cpp                      |    4 +-
 lib/Basic/Module.cpp                               |   11 +-
 lib/Basic/SourceLocation.cpp                       |    7 +
 lib/Basic/SourceManager.cpp                        |  133 +-
 lib/Basic/TargetInfo.cpp                           |    8 +-
 lib/Basic/Targets.cpp                              |  406 +-
 lib/Basic/Version.cpp                              |    2 +-
 lib/CodeGen/ABIInfo.h                              |   38 +-
 lib/CodeGen/BackendUtil.cpp                        |  109 +-
 lib/CodeGen/CGBlocks.cpp                           |  138 +-
 lib/CodeGen/CGBlocks.h                             |   26 +-
 lib/CodeGen/CGBuiltin.cpp                          |  430 +-
 lib/CodeGen/CGCXXABI.cpp                           |    2 +-
 lib/CodeGen/CGCXXABI.h                             |   12 +
 lib/CodeGen/CGCall.cpp                             |  223 +-
 lib/CodeGen/CGClass.cpp                            |   31 +-
 lib/CodeGen/CGDebugInfo.cpp                        |  257 +-
 lib/CodeGen/CGDebugInfo.h                          |    3 +
 lib/CodeGen/CGDecl.cpp                             |   53 +-
 lib/CodeGen/CGDeclCXX.cpp                          |   77 +-
 lib/CodeGen/CGException.cpp                        |   17 +-
 lib/CodeGen/CGExpr.cpp                             |  568 +-
 lib/CodeGen/CGExprAgg.cpp                          |   32 +-
 lib/CodeGen/CGExprCXX.cpp                          |   71 +-
 lib/CodeGen/CGExprComplex.cpp                      |    5 +-
 lib/CodeGen/CGExprConstant.cpp                     |   41 +-
 lib/CodeGen/CGExprScalar.cpp                       |  482 +-
 lib/CodeGen/CGObjC.cpp                             |   86 +-
 lib/CodeGen/CGObjCGNU.cpp                          |  156 +-
 lib/CodeGen/CGObjCMac.cpp                          |  798 +-
 lib/CodeGen/CGObjCRuntime.cpp                      |    7 +
 lib/CodeGen/CGObjCRuntime.h                        |    8 +
 lib/CodeGen/CGRTTI.cpp                             |   17 +-
 lib/CodeGen/CGRecordLayout.h                       |    4 +-
 lib/CodeGen/CGRecordLayoutBuilder.cpp              |   26 +-
 lib/CodeGen/CGStmt.cpp                             |  128 +-
 lib/CodeGen/CGVTables.cpp                          |   60 +-
 lib/CodeGen/CodeGenAction.cpp                      |    6 +-
 lib/CodeGen/CodeGenFunction.cpp                    |   66 +-
 lib/CodeGen/CodeGenFunction.h                      |  133 +-
 lib/CodeGen/CodeGenModule.cpp                      |  227 +-
 lib/CodeGen/CodeGenModule.h                        |   36 +-
 lib/CodeGen/CodeGenTBAA.cpp                        |   57 +
 lib/CodeGen/CodeGenTBAA.h                          |   15 +
 lib/CodeGen/CodeGenTypes.cpp                       |    4 +-
 lib/CodeGen/CodeGenTypes.h                         |    6 +-
 lib/CodeGen/ItaniumCXXABI.cpp                      |   58 +-
 lib/CodeGen/MicrosoftCXXABI.cpp                    |   65 +-
 lib/CodeGen/ModuleBuilder.cpp                      |    6 +-
 lib/CodeGen/TargetInfo.cpp                         |  752 +-
 lib/Driver/Arg.cpp                                 |   34 +-
 lib/Driver/ArgList.cpp                             |   41 +-
 lib/Driver/CC1AsOptions.cpp                        |   14 +-
 lib/Driver/Compilation.cpp                         |  100 +
 lib/Driver/Driver.cpp                              |  248 +-
 lib/Driver/DriverOptions.cpp                       |   14 +-
 lib/Driver/OptTable.cpp                            |  158 +-
 lib/Driver/Option.cpp                              |  333 +-
 lib/Driver/SanitizerArgs.h                         |  106 +
 lib/Driver/ToolChain.cpp                           |   61 +-
 lib/Driver/ToolChains.cpp                          |  481 +-
 lib/Driver/ToolChains.h                            |   43 +-
 lib/Driver/Tools.cpp                               |  980 ++-
 lib/Driver/Tools.h                                 |    5 +-
 lib/Driver/Types.cpp                               |   14 -
 lib/Driver/WindowsToolChain.cpp                    |   14 +-
 lib/Edit/RewriteObjCFoundationAPI.cpp              |    1 +
 lib/Frontend/ASTConsumers.cpp                      |    5 +-
 lib/Frontend/ASTMerge.cpp                          |    5 +-
 lib/Frontend/ASTUnit.cpp                           |  306 +-
 lib/Frontend/ChainedDiagnosticConsumer.cpp         |    2 +-
 lib/Frontend/ChainedIncludesSource.cpp             |   22 +-
 lib/Frontend/CompilerInstance.cpp                  |  139 +-
 lib/Frontend/CompilerInvocation.cpp                | 1261 +---
 lib/Frontend/CreateInvocationFromCommandLine.cpp   |   11 +-
 lib/Frontend/DependencyFile.cpp                    |   10 +-
 lib/Frontend/DependencyGraph.cpp                   |   10 +-
 lib/Frontend/DiagnosticRenderer.cpp                |   94 +-
 lib/Frontend/FrontendAction.cpp                    |   76 +-
 lib/Frontend/FrontendActions.cpp                   |  133 +-
 lib/Frontend/InitHeaderSearch.cpp                  |    6 +-
 lib/Frontend/InitPreprocessor.cpp                  |   86 +-
 lib/Frontend/LogDiagnosticPrinter.cpp              |    7 +-
 lib/Frontend/PrintPreprocessedOutput.cpp           |    8 +-
 lib/Frontend/SerializedDiagnosticPrinter.cpp       |  232 +-
 lib/Frontend/TextDiagnostic.cpp                    |  203 +-
 lib/Frontend/TextDiagnosticPrinter.cpp             |   10 +-
 lib/Frontend/VerifyDiagnosticConsumer.cpp          |  213 +-
 lib/Frontend/Warnings.cpp                          |    5 +-
 lib/FrontendTool/CMakeLists.txt                    |    3 +-
 lib/FrontendTool/ExecuteCompilerInvocation.cpp     |    9 +-
 lib/Headers/CMakeLists.txt                         |    4 +
 lib/Headers/__wmmintrin_aes.h                      |   67 +
 lib/Headers/__wmmintrin_pclmul.h                   |   34 +
 lib/Headers/altivec.h                              |   26 +-
 lib/Headers/bmi2intrin.h                           |   19 +
 lib/Headers/cpuid.h                                |    2 +-
 lib/Headers/f16cintrin.h                           |   58 +
 lib/Headers/immintrin.h                            |    4 +
 lib/Headers/module.map                             |   37 +
 lib/Headers/rtmintrin.h                            |   49 +
 lib/Headers/unwind.h                               |    2 +-
 lib/Headers/wmmintrin.h                            |   41 +-
 lib/Headers/x86intrin.h                            |    4 +
 lib/Headers/xmmintrin.h                            |    9 +-
 lib/Lex/HeaderMap.cpp                              |    2 +-
 lib/Lex/HeaderSearch.cpp                           |   47 +-
 lib/Lex/Lexer.cpp                                  |   75 +-
 lib/Lex/LiteralSupport.cpp                         |  263 +-
 lib/Lex/MacroArgs.cpp                              |    2 +-
 lib/Lex/MacroInfo.cpp                              |   90 +-
 lib/Lex/ModuleMap.cpp                              |  524 +-
 lib/Lex/PPDirectives.cpp                           |   56 +-
 lib/Lex/PPExpressions.cpp                          |   20 +-
 lib/Lex/PPLexerChange.cpp                          |    6 +-
 lib/Lex/PPMacroExpansion.cpp                       |  333 +-
 lib/Lex/PTHLexer.cpp                               |   11 +-
 lib/Lex/Pragma.cpp                                 |   22 +-
 lib/Lex/PreprocessingRecord.cpp                    |   56 +-
 lib/Lex/Preprocessor.cpp                           |   66 +-
 lib/Lex/TokenLexer.cpp                             |   88 +-
 lib/Parse/ParseAST.cpp                             |    1 -
 lib/Parse/ParseCXXInlineMethods.cpp                |    6 +-
 lib/Parse/ParseDecl.cpp                            |  224 +-
 lib/Parse/ParseDeclCXX.cpp                         |  190 +-
 lib/Parse/ParseExpr.cpp                            |  179 +-
 lib/Parse/ParseExprCXX.cpp                         |  203 +-
 lib/Parse/ParseInit.cpp                            |   11 +-
 lib/Parse/ParseObjc.cpp                            |   67 +-
 lib/Parse/ParsePragma.cpp                          |  248 +-
 lib/Parse/ParsePragma.h                            |   37 +-
 lib/Parse/ParseStmt.cpp                            |  297 +-
 lib/Parse/ParseTemplate.cpp                        |   10 +-
 lib/Parse/ParseTentative.cpp                       |  131 +-
 lib/Parse/Parser.cpp                               |  272 +-
 lib/Parse/RAIIObjectsForParser.h                   |   10 +-
 lib/Rewrite/CMakeLists.txt                         |   35 +-
 lib/Rewrite/Core/CMakeLists.txt                    |   24 +
 lib/Rewrite/Core/DeltaTree.cpp                     |  464 ++
 lib/Rewrite/Core/HTMLRewrite.cpp                   |  584 ++
 lib/Rewrite/Core/Makefile                          |   18 +
 lib/Rewrite/Core/RewriteRope.cpp                   |  807 +++
 lib/Rewrite/Core/Rewriter.cpp                      |  486 ++
 lib/Rewrite/Core/TokenRewriter.cpp                 |   99 +
 lib/Rewrite/DeltaTree.cpp                          |  467 --
 lib/Rewrite/FixItRewriter.cpp                      |  205 -
 lib/Rewrite/Frontend/CMakeLists.txt                |   28 +
 lib/Rewrite/Frontend/FixItRewriter.cpp             |  205 +
 lib/Rewrite/Frontend/FrontendActions.cpp           |  192 +
 lib/Rewrite/Frontend/HTMLPrint.cpp                 |   94 +
 lib/Rewrite/Frontend/InclusionRewriter.cpp         |  363 +
 lib/Rewrite/Frontend/Makefile                      |   18 +
 lib/Rewrite/Frontend/RewriteMacros.cpp             |  217 +
 lib/Rewrite/Frontend/RewriteModernObjC.cpp         | 7607 ++++++++++++++++++++
 lib/Rewrite/Frontend/RewriteObjC.cpp               | 6029 ++++++++++++++++
 lib/Rewrite/Frontend/RewriteTest.cpp               |   39 +
 lib/Rewrite/FrontendActions.cpp                    |  192 -
 lib/Rewrite/HTMLPrint.cpp                          |   94 -
 lib/Rewrite/HTMLRewrite.cpp                        |  583 --
 lib/Rewrite/InclusionRewriter.cpp                  |  361 -
 lib/Rewrite/Makefile                               |   10 +-
 lib/Rewrite/RewriteMacros.cpp                      |  217 -
 lib/Rewrite/RewriteModernObjC.cpp                  | 7543 -------------------
 lib/Rewrite/RewriteObjC.cpp                        | 6035 ----------------
 lib/Rewrite/RewriteRope.cpp                        |  811 ---
 lib/Rewrite/RewriteTest.cpp                        |   39 -
 lib/Rewrite/Rewriter.cpp                           |  486 --
 lib/Rewrite/TokenRewriter.cpp                      |   99 -
 lib/Sema/AnalysisBasedWarnings.cpp                 |  290 +-
 lib/Sema/CMakeLists.txt                            |    3 +
 lib/Sema/CodeCompleteConsumer.cpp                  |    7 +-
 lib/Sema/DeclSpec.cpp                              |   38 +-
 lib/Sema/DelayedDiagnostic.cpp                     |    2 +
 lib/Sema/IdentifierResolver.cpp                    |   10 +-
 lib/Sema/JumpDiagnostics.cpp                       |  138 +-
 lib/Sema/MultiplexExternalSemaSource.cpp           |  271 +
 lib/Sema/ScopeInfo.cpp                             |  189 +
 lib/Sema/Sema.cpp                                  |   64 +-
 lib/Sema/SemaAccess.cpp                            |   17 +-
 lib/Sema/SemaAttr.cpp                              |   23 +-
 lib/Sema/SemaCXXScopeSpec.cpp                      |    3 +-
 lib/Sema/SemaCast.cpp                              |   68 +-
 lib/Sema/SemaChecking.cpp                          |  706 +-
 lib/Sema/SemaCodeComplete.cpp                      |   86 +-
 lib/Sema/SemaDecl.cpp                              |  840 ++-
 lib/Sema/SemaDeclAttr.cpp                          |  135 +-
 lib/Sema/SemaDeclCXX.cpp                           |  687 +-
 lib/Sema/SemaDeclObjC.cpp                          |  145 +-
 lib/Sema/SemaExceptionSpec.cpp                     |   64 +-
 lib/Sema/SemaExpr.cpp                              |  475 +-
 lib/Sema/SemaExprCXX.cpp                           |  163 +-
 lib/Sema/SemaExprMember.cpp                        |   34 +-
 lib/Sema/SemaExprObjC.cpp                          |  124 +-
 lib/Sema/SemaInit.cpp                              |  120 +-
 lib/Sema/SemaLambda.cpp                            |   31 +-
 lib/Sema/SemaLookup.cpp                            |   43 +-
 lib/Sema/SemaObjCProperty.cpp                      |  227 +-
 lib/Sema/SemaOverload.cpp                          |  430 +-
 lib/Sema/SemaPseudoObject.cpp                      |   87 +-
 lib/Sema/SemaStmt.cpp                              |  930 +--
 lib/Sema/SemaStmtAsm.cpp                           |  661 ++
 lib/Sema/SemaStmtAttr.cpp                          |    9 +-
 lib/Sema/SemaTemplate.cpp                          |  370 +-
 lib/Sema/SemaTemplateDeduction.cpp                 |  160 +-
 lib/Sema/SemaTemplateInstantiate.cpp               |  126 +-
 lib/Sema/SemaTemplateInstantiateDecl.cpp           |  172 +-
 lib/Sema/SemaTemplateVariadic.cpp                  |    1 +
 lib/Sema/SemaType.cpp                              |  176 +-
 lib/Sema/TreeTransform.h                           |  360 +-
 lib/Serialization/ASTCommon.cpp                    |    3 +
 lib/Serialization/ASTReader.cpp                    | 2337 +++---
 lib/Serialization/ASTReaderDecl.cpp                |  151 +-
 lib/Serialization/ASTReaderStmt.cpp                |   35 +-
 lib/Serialization/ASTWriter.cpp                    |  781 +-
 lib/Serialization/ASTWriterDecl.cpp                |   58 +-
 lib/Serialization/ASTWriterStmt.cpp                |   28 +-
 lib/Serialization/GeneratePCH.cpp                  |   13 +-
 lib/Serialization/Module.cpp                       |   18 +-
 lib/Serialization/ModuleManager.cpp                |   42 +-
 .../Checkers/AdjustedReturnValueChecker.cpp        |   92 -
 lib/StaticAnalyzer/Checkers/ArrayBoundChecker.cpp  |    2 +-
 .../Checkers/ArrayBoundCheckerV2.cpp               |    2 +-
 lib/StaticAnalyzer/Checkers/AttrNonNullChecker.cpp |    4 +-
 .../Checkers/BasicObjCFoundationChecks.cpp         |  109 +-
 .../Checkers/BoolAssignmentChecker.cpp             |    2 +-
 .../Checkers/BuiltinFunctionChecker.cpp            |    2 +-
 lib/StaticAnalyzer/Checkers/CMakeLists.txt         |    6 +-
 lib/StaticAnalyzer/Checkers/CStringChecker.cpp     |   77 +-
 .../Checkers/CStringSyntaxChecker.cpp              |    9 +-
 .../Checkers/CallAndMessageChecker.cpp             |   22 +-
 lib/StaticAnalyzer/Checkers/CastSizeChecker.cpp    |    2 +-
 .../Checkers/CastToStructChecker.cpp               |    2 +-
 lib/StaticAnalyzer/Checkers/CheckObjCDealloc.cpp   |    2 +-
 .../Checkers/CheckSecuritySyntaxOnly.cpp           |    1 +
 .../Checkers/CheckerDocumentation.cpp              |   59 +-
 lib/StaticAnalyzer/Checkers/Checkers.td            |   83 +-
 lib/StaticAnalyzer/Checkers/ChrootChecker.cpp      |    2 +-
 lib/StaticAnalyzer/Checkers/DeadStoresChecker.cpp  |   72 +-
 lib/StaticAnalyzer/Checkers/DebugCheckers.cpp      |   35 +
 lib/StaticAnalyzer/Checkers/DereferenceChecker.cpp |   60 +-
 .../Checkers/DirectIvarAssignment.cpp              |  180 +
 lib/StaticAnalyzer/Checkers/DivZeroChecker.cpp     |   10 +-
 .../Checkers/DynamicTypePropagation.cpp            |    9 +-
 .../Checkers/ExprInspectionChecker.cpp             |    4 +-
 .../Checkers/FixedAddressChecker.cpp               |    2 +-
 .../Checkers/GenericTaintChecker.cpp               |   12 +-
 .../Checkers/IdempotentOperationChecker.cpp        |    2 +-
 .../Checkers/IvarInvalidationChecker.cpp           |  550 ++
 .../Checkers/MacOSKeychainAPIChecker.cpp           |   41 +-
 lib/StaticAnalyzer/Checkers/MacOSXAPIChecker.cpp   |   16 +-
 lib/StaticAnalyzer/Checkers/MallocChecker.cpp      |  259 +-
 .../Checkers/MallocSizeofChecker.cpp               |   87 +-
 .../Checkers/NSAutoreleasePoolChecker.cpp          |    2 +-
 lib/StaticAnalyzer/Checkers/NSErrorChecker.cpp     |   20 +-
 lib/StaticAnalyzer/Checkers/OSAtomicChecker.cpp    |  218 -
 lib/StaticAnalyzer/Checkers/ObjCAtSyncChecker.cpp  |    9 +-
 .../Checkers/ObjCContainersASTChecker.cpp          |   21 +-
 .../Checkers/ObjCContainersChecker.cpp             |   14 +-
 .../Checkers/ObjCMissingSuperCallChecker.cpp       |  203 +
 .../Checkers/ObjCSelfInitChecker.cpp               |   94 +-
 .../Checkers/PointerArithChecker.cpp               |    2 +-
 lib/StaticAnalyzer/Checkers/PointerSubChecker.cpp  |    2 +-
 lib/StaticAnalyzer/Checkers/PthreadLockChecker.cpp |   16 +-
 lib/StaticAnalyzer/Checkers/RetainCountChecker.cpp |  356 +-
 .../Checkers/ReturnPointerRangeChecker.cpp         |    2 +-
 lib/StaticAnalyzer/Checkers/ReturnUndefChecker.cpp |   19 +-
 .../Checkers/SimpleStreamChecker.cpp               |  348 +
 .../Checkers/StackAddrEscapeChecker.cpp            |   23 +-
 lib/StaticAnalyzer/Checkers/StreamChecker.cpp      |   46 +-
 lib/StaticAnalyzer/Checkers/TaintTesterChecker.cpp |    2 +-
 lib/StaticAnalyzer/Checkers/UndefBranchChecker.cpp |    5 +-
 .../Checkers/UndefCapturedBlockVarChecker.cpp      |    2 +-
 lib/StaticAnalyzer/Checkers/UndefResultChecker.cpp |    7 +-
 .../Checkers/UndefinedArraySubscriptChecker.cpp    |    4 +-
 .../Checkers/UndefinedAssignmentChecker.cpp        |    5 +-
 lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp     |   18 +-
 lib/StaticAnalyzer/Checkers/VLASizeChecker.cpp     |    4 +-
 lib/StaticAnalyzer/Core/AnalysisManager.cpp        |   36 +-
 lib/StaticAnalyzer/Core/AnalyzerOptions.cpp        |  138 +
 lib/StaticAnalyzer/Core/BasicConstraintManager.cpp |  446 --
 lib/StaticAnalyzer/Core/BasicValueFactory.cpp      |    6 +-
 lib/StaticAnalyzer/Core/BugReporter.cpp            |  517 +-
 lib/StaticAnalyzer/Core/BugReporterVisitors.cpp    |  602 +-
 lib/StaticAnalyzer/Core/CMakeLists.txt             |    7 +-
 lib/StaticAnalyzer/Core/CallEvent.cpp              |  166 +-
 lib/StaticAnalyzer/Core/CheckerContext.cpp         |   27 +-
 lib/StaticAnalyzer/Core/CheckerManager.cpp         |   54 +-
 lib/StaticAnalyzer/Core/ConstraintManager.cpp      |   39 +
 lib/StaticAnalyzer/Core/CoreEngine.cpp             |   15 +-
 lib/StaticAnalyzer/Core/Environment.cpp            |  204 +-
 lib/StaticAnalyzer/Core/ExplodedGraph.cpp          |  155 +-
 lib/StaticAnalyzer/Core/ExprEngine.cpp             |  414 +-
 lib/StaticAnalyzer/Core/ExprEngineC.cpp            |  133 +-
 lib/StaticAnalyzer/Core/ExprEngineCXX.cpp          |   79 +-
 .../Core/ExprEngineCallAndReturn.cpp               |  351 +-
 lib/StaticAnalyzer/Core/ExprEngineObjC.cpp         |   82 +-
 lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp        |   15 +-
 lib/StaticAnalyzer/Core/MemRegion.cpp              |   34 +-
 lib/StaticAnalyzer/Core/PathDiagnostic.cpp         |  306 +-
 lib/StaticAnalyzer/Core/PlistDiagnostics.cpp       |   47 +-
 lib/StaticAnalyzer/Core/ProgramState.cpp           |   68 +-
 lib/StaticAnalyzer/Core/RangeConstraintManager.cpp |   66 +-
 lib/StaticAnalyzer/Core/RegionStore.cpp            |  308 +-
 lib/StaticAnalyzer/Core/SValBuilder.cpp            |   37 +-
 lib/StaticAnalyzer/Core/SVals.cpp                  |    3 +-
 .../Core/SimpleConstraintManager.cpp               |   18 +-
 lib/StaticAnalyzer/Core/SimpleConstraintManager.h  |    4 +-
 lib/StaticAnalyzer/Core/SimpleSValBuilder.cpp      |   25 +-
 lib/StaticAnalyzer/Core/Store.cpp                  |   86 +
 lib/StaticAnalyzer/Core/SymbolManager.cpp          |  109 +-
 lib/StaticAnalyzer/Core/TextPathDiagnostics.cpp    |    1 -
 lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp   |  154 +-
 lib/StaticAnalyzer/Frontend/AnalysisConsumer.h     |    4 +-
 lib/StaticAnalyzer/Frontend/CMakeLists.txt         |    3 +-
 .../Frontend/CheckerRegistration.cpp               |    2 +-
 lib/Tooling/CMakeLists.txt                         |    7 +-
 lib/Tooling/CommandLineClangTool.cpp               |   80 -
 lib/Tooling/CommonOptionsParser.cpp                |   79 +
 lib/Tooling/CompilationDatabase.cpp                |  297 +-
 lib/Tooling/CustomCompilationDatabase.h            |   42 -
 lib/Tooling/FileMatchTrie.cpp                      |  188 +
 lib/Tooling/JSONCompilationDatabase.cpp            |  303 +
 lib/Tooling/Refactoring.cpp                        |   11 +-
 lib/Tooling/Tooling.cpp                            |   30 +-
 388 files changed, 44874 insertions(+), 31751 deletions(-)
 create mode 100644 lib/Analysis/BodyFarm.cpp
 create mode 100644 lib/Analysis/BodyFarm.h
 create mode 100644 lib/Analysis/ObjCNoReturn.cpp
 create mode 100644 lib/Driver/SanitizerArgs.h
 create mode 100644 lib/Headers/__wmmintrin_aes.h
 create mode 100644 lib/Headers/__wmmintrin_pclmul.h
 create mode 100644 lib/Headers/f16cintrin.h
 create mode 100644 lib/Headers/rtmintrin.h
 create mode 100644 lib/Rewrite/Core/CMakeLists.txt
 create mode 100644 lib/Rewrite/Core/DeltaTree.cpp
 create mode 100644 lib/Rewrite/Core/HTMLRewrite.cpp
 create mode 100644 lib/Rewrite/Core/Makefile
 create mode 100644 lib/Rewrite/Core/RewriteRope.cpp
 create mode 100644 lib/Rewrite/Core/Rewriter.cpp
 create mode 100644 lib/Rewrite/Core/TokenRewriter.cpp
 delete mode 100644 lib/Rewrite/DeltaTree.cpp
 delete mode 100644 lib/Rewrite/FixItRewriter.cpp
 create mode 100644 lib/Rewrite/Frontend/CMakeLists.txt
 create mode 100644 lib/Rewrite/Frontend/FixItRewriter.cpp
 create mode 100644 lib/Rewrite/Frontend/FrontendActions.cpp
 create mode 100644 lib/Rewrite/Frontend/HTMLPrint.cpp
 create mode 100644 lib/Rewrite/Frontend/InclusionRewriter.cpp
 create mode 100644 lib/Rewrite/Frontend/Makefile
 create mode 100644 lib/Rewrite/Frontend/RewriteMacros.cpp
 create mode 100644 lib/Rewrite/Frontend/RewriteModernObjC.cpp
 create mode 100644 lib/Rewrite/Frontend/RewriteObjC.cpp
 create mode 100644 lib/Rewrite/Frontend/RewriteTest.cpp
 delete mode 100644 lib/Rewrite/FrontendActions.cpp
 delete mode 100644 lib/Rewrite/HTMLPrint.cpp
 delete mode 100644 lib/Rewrite/HTMLRewrite.cpp
 delete mode 100644 lib/Rewrite/InclusionRewriter.cpp
 delete mode 100644 lib/Rewrite/RewriteMacros.cpp
 delete mode 100644 lib/Rewrite/RewriteModernObjC.cpp
 delete mode 100644 lib/Rewrite/RewriteObjC.cpp
 delete mode 100644 lib/Rewrite/RewriteRope.cpp
 delete mode 100644 lib/Rewrite/RewriteTest.cpp
 delete mode 100644 lib/Rewrite/Rewriter.cpp
 delete mode 100644 lib/Rewrite/TokenRewriter.cpp
 create mode 100644 lib/Sema/MultiplexExternalSemaSource.cpp
 create mode 100644 lib/Sema/ScopeInfo.cpp
 create mode 100644 lib/Sema/SemaStmtAsm.cpp
 delete mode 100644 lib/StaticAnalyzer/Checkers/AdjustedReturnValueChecker.cpp
 create mode 100644 lib/StaticAnalyzer/Checkers/DirectIvarAssignment.cpp
 create mode 100644 lib/StaticAnalyzer/Checkers/IvarInvalidationChecker.cpp
 delete mode 100644 lib/StaticAnalyzer/Checkers/OSAtomicChecker.cpp
 create mode 100644 lib/StaticAnalyzer/Checkers/ObjCMissingSuperCallChecker.cpp
 create mode 100644 lib/StaticAnalyzer/Checkers/SimpleStreamChecker.cpp
 create mode 100644 lib/StaticAnalyzer/Core/AnalyzerOptions.cpp
 delete mode 100644 lib/StaticAnalyzer/Core/BasicConstraintManager.cpp
 create mode 100644 lib/StaticAnalyzer/Core/ConstraintManager.cpp
 delete mode 100644 lib/Tooling/CommandLineClangTool.cpp
 create mode 100644 lib/Tooling/CommonOptionsParser.cpp
 delete mode 100644 lib/Tooling/CustomCompilationDatabase.h
 create mode 100644 lib/Tooling/FileMatchTrie.cpp
 create mode 100644 lib/Tooling/JSONCompilationDatabase.cpp

(limited to 'lib')

diff --git a/lib/ARCMigrate/ARCMT.cpp b/lib/ARCMigrate/ARCMT.cpp
index f291dec..b57d996 100644
--- a/lib/ARCMigrate/ARCMT.cpp
+++ b/lib/ARCMigrate/ARCMT.cpp
@@ -14,7 +14,7 @@
 #include "clang/Frontend/TextDiagnosticPrinter.h"
 #include "clang/Frontend/Utils.h"
 #include "clang/AST/ASTConsumer.h"
-#include "clang/Rewrite/Rewriter.h"
+#include "clang/Rewrite/Core/Rewriter.h"
 #include "clang/Sema/SemaDiagnostic.h"
 #include "clang/Basic/DiagnosticCategories.h"
 #include "clang/Lex/Preprocessor.h"
@@ -42,7 +42,7 @@ bool CapturedDiagList::clearDiagnostic(ArrayRef<unsigned> IDs,
       while (I != List.end() && I->getLevel() == DiagnosticsEngine::Note)
         ++I;
       // Clear the diagnostic and any notes following it.
-      List.erase(eraseS, I);
+      I = List.erase(eraseS, I);
       continue;
     }
 
@@ -147,54 +147,10 @@ public:
 
 } // end anonymous namespace
 
-static inline StringRef SimulatorVersionDefineName() {
-  return "__IPHONE_OS_VERSION_MIN_REQUIRED=";
-}
-
-/// \brief Parse the simulator version define:
-/// __IPHONE_OS_VERSION_MIN_REQUIRED=([0-9])([0-9][0-9])([0-9][0-9])
-// and return the grouped values as integers, e.g:
-//   __IPHONE_OS_VERSION_MIN_REQUIRED=40201
-// will return Major=4, Minor=2, Micro=1.
-static bool GetVersionFromSimulatorDefine(StringRef define,
-                                          unsigned &Major, unsigned &Minor,
-                                          unsigned &Micro) {
-  assert(define.startswith(SimulatorVersionDefineName()));
-  StringRef name, version;
-  llvm::tie(name, version) = define.split('=');
-  if (version.empty())
-    return false;
-  std::string verstr = version.str();
-  char *end;
-  unsigned num = (unsigned) strtol(verstr.c_str(), &end, 10);
-  if (*end != '\0')
-    return false;
-  Major = num / 10000;
-  num = num % 10000;
-  Minor = num / 100;
-  Micro = num % 100;
-  return true;
-}
-
 static bool HasARCRuntime(CompilerInvocation &origCI) {
   // This duplicates some functionality from Darwin::AddDeploymentTarget
   // but this function is well defined, so keep it decoupled from the driver
   // and avoid unrelated complications.
-
-  for (unsigned i = 0, e = origCI.getPreprocessorOpts().Macros.size();
-         i != e; ++i) {
-    StringRef define = origCI.getPreprocessorOpts().Macros[i].first;
-    bool isUndef = origCI.getPreprocessorOpts().Macros[i].second;
-    if (isUndef)
-      continue;
-    if (!define.startswith(SimulatorVersionDefineName()))
-      continue;
-    unsigned Major = 0, Minor = 0, Micro = 0;
-    if (GetVersionFromSimulatorDefine(define, Major, Minor, Micro) &&
-        Major < 10 && Minor < 100 && Micro < 100)
-      return Major >= 5;
-  }
-
   llvm::Triple triple(origCI.getTargetOpts().Triple);
 
   if (triple.getOS() == llvm::Triple::IOS)
@@ -237,18 +193,19 @@ createInvocationForMigration(CompilerInvocation &origCI) {
   WarnOpts.push_back("error=arc-unsafe-retained-assign");
   CInvok->getDiagnosticOpts().Warnings = llvm_move(WarnOpts);
 
-  CInvok->getLangOpts()->ObjCRuntimeHasWeak = HasARCRuntime(origCI);
+  CInvok->getLangOpts()->ObjCARCWeak = HasARCRuntime(origCI);
 
   return CInvok.take();
 }
 
 static void emitPremigrationErrors(const CapturedDiagList &arcDiags,
-                                   const DiagnosticOptions &diagOpts,
+                                   DiagnosticOptions *diagOpts,
                                    Preprocessor &PP) {
   TextDiagnosticPrinter printer(llvm::errs(), diagOpts);
   IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
   IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
-      new DiagnosticsEngine(DiagID, &printer, /*ShouldOwnClient=*/false));
+      new DiagnosticsEngine(DiagID, diagOpts, &printer,
+                            /*ShouldOwnClient=*/false));
   Diags->setSourceManager(&PP.getSourceManager());
   
   printer.BeginSourceFile(PP.getLangOpts(), &PP);
@@ -286,7 +243,8 @@ bool arcmt::checkForManualIssues(CompilerInvocation &origCI,
   assert(DiagClient);
   IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
   IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
-      new DiagnosticsEngine(DiagID, DiagClient, /*ShouldOwnClient=*/false));
+      new DiagnosticsEngine(DiagID, &origCI.getDiagnosticOpts(),
+                            DiagClient, /*ShouldOwnClient=*/false));
 
   // Filter of all diagnostics.
   CaptureDiagnosticConsumer errRec(*Diags, *DiagClient, capturedDiags);
@@ -314,7 +272,7 @@ bool arcmt::checkForManualIssues(CompilerInvocation &origCI,
   }
 
   if (emitPremigrationARCErrors)
-    emitPremigrationErrors(capturedDiags, origCI.getDiagnosticOpts(),
+    emitPremigrationErrors(capturedDiags, &origCI.getDiagnosticOpts(),
                            Unit->getPreprocessor());
   if (!plistOut.empty()) {
     SmallVector<StoredDiagnostic, 8> arcDiags;
@@ -395,7 +353,8 @@ static bool applyTransforms(CompilerInvocation &origCI,
 
   IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
   IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
-      new DiagnosticsEngine(DiagID, DiagClient, /*ShouldOwnClient=*/false));
+      new DiagnosticsEngine(DiagID, &origCI.getDiagnosticOpts(),
+                            DiagClient, /*ShouldOwnClient=*/false));
 
   if (outputDir.empty()) {
     origCI.getLangOpts()->ObjCAutoRefCount = true;
@@ -434,7 +393,8 @@ bool arcmt::getFileRemappings(std::vector<std::pair<std::string,std::string> > &
 
   IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
   IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
-      new DiagnosticsEngine(DiagID, DiagClient, /*ShouldOwnClient=*/false));
+      new DiagnosticsEngine(DiagID, new DiagnosticOptions,
+                            DiagClient, /*ShouldOwnClient=*/false));
 
   FileRemapper remapper;
   bool err = remapper.initFromDisk(outputDir, *Diags,
@@ -458,7 +418,8 @@ bool arcmt::getFileRemappingsFromFileList(
 
   llvm::IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
   llvm::IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
-      new DiagnosticsEngine(DiagID, DiagClient, /*ShouldOwnClient=*/false));
+      new DiagnosticsEngine(DiagID, new DiagnosticOptions,
+                            DiagClient, /*ShouldOwnClient=*/false));
 
   for (ArrayRef<StringRef>::iterator
          I = remapFiles.begin(), E = remapFiles.end(); I != E; ++I) {
@@ -574,7 +535,8 @@ MigrationProcess::MigrationProcess(const CompilerInvocation &CI,
   if (!outputDir.empty()) {
     IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
     IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
-      new DiagnosticsEngine(DiagID, DiagClient, /*ShouldOwnClient=*/false));
+      new DiagnosticsEngine(DiagID, &CI.getDiagnosticOpts(),
+                            DiagClient, /*ShouldOwnClient=*/false));
     Remapper.initFromDisk(outputDir, *Diags, /*ignoreIfFilesChanges=*/true);
   }
 }
@@ -593,7 +555,8 @@ bool MigrationProcess::applyTransform(TransformFn trans,
   assert(DiagClient);
   IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
   IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
-      new DiagnosticsEngine(DiagID, DiagClient, /*ShouldOwnClient=*/false));
+      new DiagnosticsEngine(DiagID, new DiagnosticOptions,
+                            DiagClient, /*ShouldOwnClient=*/false));
 
   // Filter of all diagnostics.
   CaptureDiagnosticConsumer errRec(*Diags, *DiagClient, capturedDiags);
diff --git a/lib/ARCMigrate/CMakeLists.txt b/lib/ARCMigrate/CMakeLists.txt
index f602fc8..731bcb4 100644
--- a/lib/ARCMigrate/CMakeLists.txt
+++ b/lib/ARCMigrate/CMakeLists.txt
@@ -37,5 +37,6 @@ target_link_libraries(clangARCMigrate
   clangAST
   clangParse
   clangFrontend
-  clangRewrite
+  clangRewriteCore
+  clangRewriteFrontend
   )
diff --git a/lib/ARCMigrate/FileRemapper.cpp b/lib/ARCMigrate/FileRemapper.cpp
index e9b49b3..28ca9a5 100644
--- a/lib/ARCMigrate/FileRemapper.cpp
+++ b/lib/ARCMigrate/FileRemapper.cpp
@@ -8,7 +8,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/ARCMigrate/FileRemapper.h"
-#include "clang/Frontend/PreprocessorOptions.h"
+#include "clang/Lex/PreprocessorOptions.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/Diagnostic.h"
 #include "llvm/Support/MemoryBuffer.h"
diff --git a/lib/ARCMigrate/Internals.h b/lib/ARCMigrate/Internals.h
index 935fc9b..1966a98 100644
--- a/lib/ARCMigrate/Internals.h
+++ b/lib/ARCMigrate/Internals.h
@@ -11,8 +11,10 @@
 #define LLVM_CLANG_LIB_ARCMIGRATE_INTERNALS_H
 
 #include "clang/ARCMigrate/ARCMT.h"
+#include "clang/Basic/Diagnostic.h"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/Optional.h"
+#include <list>
 
 namespace clang {
   class Sema;
diff --git a/lib/ARCMigrate/ObjCMT.cpp b/lib/ARCMigrate/ObjCMT.cpp
index 0098f97..dfe14e2 100644
--- a/lib/ARCMigrate/ObjCMT.cpp
+++ b/lib/ARCMigrate/ObjCMT.cpp
@@ -18,7 +18,7 @@
 #include "clang/Edit/EditedSource.h"
 #include "clang/Edit/Commit.h"
 #include "clang/Edit/EditsReceiver.h"
-#include "clang/Rewrite/Rewriter.h"
+#include "clang/Rewrite/Core/Rewriter.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Basic/FileManager.h"
 #include "llvm/ADT/SmallString.h"
diff --git a/lib/ARCMigrate/Transforms.cpp b/lib/ARCMigrate/Transforms.cpp
index 1175c36..805a67d 100644
--- a/lib/ARCMigrate/Transforms.cpp
+++ b/lib/ARCMigrate/Transforms.cpp
@@ -40,7 +40,7 @@ bool MigrationPass::CFBridgingFunctionsDefined() {
 
 bool trans::canApplyWeak(ASTContext &Ctx, QualType type,
                          bool AllowOnUnknownClass) {
-  if (!Ctx.getLangOpts().ObjCRuntimeHasWeak)
+  if (!Ctx.getLangOpts().ObjCARCWeak)
     return false;
 
   QualType T = type;
@@ -59,7 +59,7 @@ bool trans::canApplyWeak(ASTContext &Ctx, QualType type,
       return false; // id/NSObject is not safe for weak.
     if (!AllowOnUnknownClass && !Class->hasDefinition())
       return false; // forward classes are not verifiable, therefore not safe.
-    if (Class->isArcWeakrefUnavailable())
+    if (Class && Class->isArcWeakrefUnavailable())
       return false;
   }
 
diff --git a/lib/AST/ASTConsumer.cpp b/lib/AST/ASTConsumer.cpp
index 1672bc8..a4e17c0 100644
--- a/lib/AST/ASTConsumer.cpp
+++ b/lib/AST/ASTConsumer.cpp
@@ -13,6 +13,7 @@
 
 #include "clang/AST/ASTConsumer.h"
 #include "clang/AST/DeclGroup.h"
+#include "clang/AST/Decl.h"
 using namespace clang;
 
 bool ASTConsumer::HandleTopLevelDecl(DeclGroupRef D) {
@@ -24,3 +25,7 @@ void ASTConsumer::HandleInterestingDecl(DeclGroupRef D) {
 }
 
 void ASTConsumer::HandleTopLevelDeclInObjCContainer(DeclGroupRef D) {}
+
+void ASTConsumer::HandleImplicitImportDecl(ImportDecl *D) {
+  HandleTopLevelDecl(DeclGroupRef(D));
+}
diff --git a/lib/AST/ASTContext.cpp b/lib/AST/ASTContext.cpp
index c021323..74c68ae 100644
--- a/lib/AST/ASTContext.cpp
+++ b/lib/AST/ASTContext.cpp
@@ -24,6 +24,7 @@
 #include "clang/AST/ASTMutationListener.h"
 #include "clang/AST/RecordLayout.h"
 #include "clang/AST/Mangle.h"
+#include "clang/AST/Comment.h"
 #include "clang/Basic/Builtins.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/TargetInfo.h"
@@ -72,6 +73,22 @@ RawComment *ASTContext::getRawCommentForDeclNoCache(const Decl *D) const {
       return NULL;
   }
 
+  if (const VarDecl *VD = dyn_cast<VarDecl>(D)) {
+    if (VD->isStaticDataMember() &&
+        VD->getTemplateSpecializationKind() == TSK_ImplicitInstantiation)
+      return NULL;
+  }
+
+  if (const CXXRecordDecl *CRD = dyn_cast<CXXRecordDecl>(D)) {
+    if (CRD->getTemplateSpecializationKind() == TSK_ImplicitInstantiation)
+      return NULL;
+  }
+
+  if (const EnumDecl *ED = dyn_cast<EnumDecl>(D)) {
+    if (ED->getTemplateSpecializationKind() == TSK_ImplicitInstantiation)
+      return NULL;
+  }
+
   // TODO: handle comments for function parameters properly.
   if (isa<ParmVarDecl>(D))
     return NULL;
@@ -196,15 +213,72 @@ RawComment *ASTContext::getRawCommentForDeclNoCache(const Decl *D) const {
 namespace {
 /// If we have a 'templated' declaration for a template, adjust 'D' to
 /// refer to the actual template.
+/// If we have an implicit instantiation, adjust 'D' to refer to template.
 const Decl *adjustDeclToTemplate(const Decl *D) {
   if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
+    // Is this function declaration part of a function template?
     if (const FunctionTemplateDecl *FTD = FD->getDescribedFunctionTemplate())
-      D = FTD;
-  } else if (const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(D)) {
-    if (const ClassTemplateDecl *CTD = RD->getDescribedClassTemplate())
-      D = CTD;
+      return FTD;
+
+    // Nothing to do if function is not an implicit instantiation.
+    if (FD->getTemplateSpecializationKind() != TSK_ImplicitInstantiation)
+      return D;
+
+    // Function is an implicit instantiation of a function template?
+    if (const FunctionTemplateDecl *FTD = FD->getPrimaryTemplate())
+      return FTD;
+
+    // Function is instantiated from a member definition of a class template?
+    if (const FunctionDecl *MemberDecl =
+            FD->getInstantiatedFromMemberFunction())
+      return MemberDecl;
+
+    return D;
   }
-  // FIXME: Alias templates?
+  if (const VarDecl *VD = dyn_cast<VarDecl>(D)) {
+    // Static data member is instantiated from a member definition of a class
+    // template?
+    if (VD->isStaticDataMember())
+      if (const VarDecl *MemberDecl = VD->getInstantiatedFromStaticDataMember())
+        return MemberDecl;
+
+    return D;
+  }
+  if (const CXXRecordDecl *CRD = dyn_cast<CXXRecordDecl>(D)) {
+    // Is this class declaration part of a class template?
+    if (const ClassTemplateDecl *CTD = CRD->getDescribedClassTemplate())
+      return CTD;
+
+    // Class is an implicit instantiation of a class template or partial
+    // specialization?
+    if (const ClassTemplateSpecializationDecl *CTSD =
+            dyn_cast<ClassTemplateSpecializationDecl>(CRD)) {
+      if (CTSD->getSpecializationKind() != TSK_ImplicitInstantiation)
+        return D;
+      llvm::PointerUnion<ClassTemplateDecl *,
+                         ClassTemplatePartialSpecializationDecl *>
+          PU = CTSD->getSpecializedTemplateOrPartial();
+      return PU.is<ClassTemplateDecl*>() ?
+          static_cast<const Decl*>(PU.get<ClassTemplateDecl *>()) :
+          static_cast<const Decl*>(
+              PU.get<ClassTemplatePartialSpecializationDecl *>());
+    }
+
+    // Class is instantiated from a member definition of a class template?
+    if (const MemberSpecializationInfo *Info =
+                   CRD->getMemberSpecializationInfo())
+      return Info->getInstantiatedFrom();
+
+    return D;
+  }
+  if (const EnumDecl *ED = dyn_cast<EnumDecl>(D)) {
+    // Enum is instantiated from a member definition of a class template?
+    if (const EnumDecl *MemberDecl = ED->getInstantiatedFromMemberEnum())
+      return MemberDecl;
+
+    return D;
+  }
+  // FIXME: Adjust alias templates?
   return D;
 }
 } // unnamed namespace
@@ -282,23 +356,83 @@ const RawComment *ASTContext::getRawCommentForAnyRedecl(
   return RC;
 }
 
-comments::FullComment *ASTContext::getCommentForDecl(const Decl *D) const {
+static void addRedeclaredMethods(const ObjCMethodDecl *ObjCMethod,
+                   SmallVectorImpl<const NamedDecl *> &Redeclared) {
+  const DeclContext *DC = ObjCMethod->getDeclContext();
+  if (const ObjCImplDecl *IMD = dyn_cast<ObjCImplDecl>(DC)) {
+    const ObjCInterfaceDecl *ID = IMD->getClassInterface();
+    if (!ID)
+      return;
+    // Add redeclared method here.
+    for (const ObjCCategoryDecl *ClsExtDecl = ID->getFirstClassExtension();
+         ClsExtDecl; ClsExtDecl = ClsExtDecl->getNextClassExtension()) {
+      if (ObjCMethodDecl *RedeclaredMethod =
+            ClsExtDecl->getMethod(ObjCMethod->getSelector(),
+                                  ObjCMethod->isInstanceMethod()))
+        Redeclared.push_back(RedeclaredMethod);
+    }
+  }
+}
+
+comments::FullComment *ASTContext::cloneFullComment(comments::FullComment *FC,
+                                                    const Decl *D) const {
+  comments::DeclInfo *ThisDeclInfo = new (*this) comments::DeclInfo;
+  ThisDeclInfo->CommentDecl = D;
+  ThisDeclInfo->IsFilled = false;
+  ThisDeclInfo->fill();
+  ThisDeclInfo->CommentDecl = FC->getDecl();
+  comments::FullComment *CFC =
+    new (*this) comments::FullComment(FC->getBlocks(),
+                                      ThisDeclInfo);
+  return CFC;
+  
+}
+
+comments::FullComment *ASTContext::getCommentForDecl(
+                                              const Decl *D,
+                                              const Preprocessor *PP) const {
   D = adjustDeclToTemplate(D);
+  
   const Decl *Canonical = D->getCanonicalDecl();
   llvm::DenseMap<const Decl *, comments::FullComment *>::iterator Pos =
       ParsedComments.find(Canonical);
-  if (Pos != ParsedComments.end())
+  
+  if (Pos != ParsedComments.end()) {
+    if (Canonical != D) {
+      comments::FullComment *FC = Pos->second;
+      comments::FullComment *CFC = cloneFullComment(FC, D);
+      return CFC;
+    }
     return Pos->second;
-
+  }
+  
   const Decl *OriginalDecl;
+  
   const RawComment *RC = getRawCommentForAnyRedecl(D, &OriginalDecl);
-  if (!RC)
+  if (!RC) {
+    if (isa<ObjCMethodDecl>(D) || isa<FunctionDecl>(D)) {
+      SmallVector<const NamedDecl*, 8> Overridden;
+      if (const ObjCMethodDecl *OMD = dyn_cast<ObjCMethodDecl>(D))
+        addRedeclaredMethods(OMD, Overridden);
+      getOverriddenMethods(dyn_cast<NamedDecl>(D), Overridden);
+      for (unsigned i = 0, e = Overridden.size(); i < e; i++) {
+        if (comments::FullComment *FC = getCommentForDecl(Overridden[i], PP)) {
+          comments::FullComment *CFC = cloneFullComment(FC, D);
+          return CFC;
+        }
+      }
+    }
     return NULL;
-
+  }
+  
+  // If the RawComment was attached to other redeclaration of this Decl, we
+  // should parse the comment in context of that other Decl.  This is important
+  // because comments can contain references to parameter names which can be
+  // different across redeclarations.
   if (D != OriginalDecl)
-    return getCommentForDecl(OriginalDecl);
+    return getCommentForDecl(OriginalDecl, PP);
 
-  comments::FullComment *FC = RC->parse(*this, D);
+  comments::FullComment *FC = RC->parse(*this, PP, D);
   ParsedComments[Canonical] = FC;
   return FC;
 }
@@ -481,6 +615,7 @@ ASTContext::ASTContext(LangOptions& LOpts, SourceManager &SM,
     Int128Decl(0), UInt128Decl(0),
     BuiltinVaListDecl(0),
     ObjCIdDecl(0), ObjCSelDecl(0), ObjCClassDecl(0), ObjCProtocolClassDecl(0),
+    BOOLDecl(0),
     CFConstantStringTypeDecl(0), ObjCInstanceTypeDecl(0),
     FILEDecl(0), 
     jmp_bufDecl(0), sigjmp_bufDecl(0), ucontext_tDecl(0),
@@ -495,6 +630,7 @@ ASTContext::ASTContext(LangOptions& LOpts, SourceManager &SM,
     DeclarationNames(*this),
     ExternalSource(0), Listener(0),
     Comments(SM), CommentsLoaded(false),
+    CommentCommandTraits(BumpAlloc),
     LastSDM(0, 0),
     UniqueBlockByRefTypeID(0) 
 {
@@ -683,12 +819,12 @@ void ASTContext::InitBuiltinTypes(const TargetInfo &Target) {
   InitBuiltinType(Int128Ty,            BuiltinType::Int128);
   InitBuiltinType(UnsignedInt128Ty,    BuiltinType::UInt128);
 
-  if (LangOpts.CPlusPlus) { // C++ 3.9.1p5
+  if (LangOpts.CPlusPlus && LangOpts.WChar) { // C++ 3.9.1p5
     if (TargetInfo::isTypeSigned(Target.getWCharType()))
       InitBuiltinType(WCharTy,           BuiltinType::WChar_S);
     else  // -fshort-wchar makes wchar_t be unsigned.
       InitBuiltinType(WCharTy,           BuiltinType::WChar_U);
-  } else // C99
+  } else // C99 (or C++ using -fno-wchar)
     WCharTy = getFromTargetType(Target.getWCharType());
 
   WIntTy = getFromTargetType(Target.getWIntType());
@@ -725,6 +861,9 @@ void ASTContext::InitBuiltinTypes(const TargetInfo &Target) {
   // Placeholder type for unbridged ARC casts.
   InitBuiltinType(ARCUnbridgedCastTy,  BuiltinType::ARCUnbridgedCast);
 
+  // Placeholder type for builtin functions.
+  InitBuiltinType(BuiltinFnTy,  BuiltinType::BuiltinFn);
+
   // C99 6.2.5p11.
   FloatComplexTy      = getComplexType(FloatTy);
   DoubleComplexTy     = getComplexType(DoubleTy);
@@ -909,7 +1048,7 @@ bool ASTContext::BitfieldFollowsNonBitfield(const FieldDecl *FD,
 ASTContext::overridden_cxx_method_iterator
 ASTContext::overridden_methods_begin(const CXXMethodDecl *Method) const {
   llvm::DenseMap<const CXXMethodDecl *, CXXMethodVector>::const_iterator Pos
-    = OverriddenMethods.find(Method);
+    = OverriddenMethods.find(Method->getCanonicalDecl());
   if (Pos == OverriddenMethods.end())
     return 0;
 
@@ -919,7 +1058,7 @@ ASTContext::overridden_methods_begin(const CXXMethodDecl *Method) const {
 ASTContext::overridden_cxx_method_iterator
 ASTContext::overridden_methods_end(const CXXMethodDecl *Method) const {
   llvm::DenseMap<const CXXMethodDecl *, CXXMethodVector>::const_iterator Pos
-    = OverriddenMethods.find(Method);
+    = OverriddenMethods.find(Method->getCanonicalDecl());
   if (Pos == OverriddenMethods.end())
     return 0;
 
@@ -929,7 +1068,7 @@ ASTContext::overridden_methods_end(const CXXMethodDecl *Method) const {
 unsigned
 ASTContext::overridden_methods_size(const CXXMethodDecl *Method) const {
   llvm::DenseMap<const CXXMethodDecl *, CXXMethodVector>::const_iterator Pos
-    = OverriddenMethods.find(Method);
+    = OverriddenMethods.find(Method->getCanonicalDecl());
   if (Pos == OverriddenMethods.end())
     return 0;
 
@@ -938,9 +1077,30 @@ ASTContext::overridden_methods_size(const CXXMethodDecl *Method) const {
 
 void ASTContext::addOverriddenMethod(const CXXMethodDecl *Method, 
                                      const CXXMethodDecl *Overridden) {
+  assert(Method->isCanonicalDecl() && Overridden->isCanonicalDecl());
   OverriddenMethods[Method].push_back(Overridden);
 }
 
+void ASTContext::getOverriddenMethods(
+                      const NamedDecl *D,
+                      SmallVectorImpl<const NamedDecl *> &Overridden) const {
+  assert(D);
+
+  if (const CXXMethodDecl *CXXMethod = dyn_cast<CXXMethodDecl>(D)) {
+    Overridden.append(CXXMethod->begin_overridden_methods(),
+                      CXXMethod->end_overridden_methods());
+    return;
+  }
+
+  const ObjCMethodDecl *Method = dyn_cast<ObjCMethodDecl>(D);
+  if (!Method)
+    return;
+
+  SmallVector<const ObjCMethodDecl *, 8> OverDecls;
+  Method->getOverriddenMethods(OverDecls);
+  Overridden.append(OverDecls.begin(), OverDecls.end());
+}
+
 void ASTContext::addedLocalImportDecl(ImportDecl *Import) {
   assert(!Import->NextLocalImport && "Import declaration already in the chain");
   assert(!Import->isFromASTFile() && "Non-local import declaration");
@@ -1062,6 +1222,27 @@ CharUnits ASTContext::getDeclAlign(const Decl *D, bool RefAsPointee) const {
   return toCharUnitsFromBits(Align);
 }
 
+// getTypeInfoDataSizeInChars - Return the size of a type, in
+// chars. If the type is a record, its data size is returned.  This is
+// the size of the memcpy that's performed when assigning this type
+// using a trivial copy/move assignment operator.
+std::pair<CharUnits, CharUnits>
+ASTContext::getTypeInfoDataSizeInChars(QualType T) const {
+  std::pair<CharUnits, CharUnits> sizeAndAlign = getTypeInfoInChars(T);
+
+  // In C++, objects can sometimes be allocated into the tail padding
+  // of a base-class subobject.  We decide whether that's possible
+  // during class layout, so here we can just trust the layout results.
+  if (getLangOpts().CPlusPlus) {
+    if (const RecordType *RT = T->getAs<RecordType>()) {
+      const ASTRecordLayout &layout = getASTRecordLayout(RT->getDecl());
+      sizeAndAlign.first = layout.getDataSize();
+    }
+  }
+
+  return sizeAndAlign;
+}
+
 std::pair<CharUnits, CharUnits>
 ASTContext::getTypeInfoInChars(const Type *T) const {
   std::pair<uint64_t, unsigned> Info = getTypeInfo(T);
@@ -3353,6 +3534,12 @@ QualType ASTContext::getPointerDiffType() const {
   return getFromTargetType(Target->getPtrDiffType(0));
 }
 
+/// \brief Return the unique type for "pid_t" defined in
+/// <sys/types.h>. We need this to compute the correct type for vfork().
+QualType ASTContext::getProcessIDType() const {
+  return getFromTargetType(Target->getProcessIDType());
+}
+
 //===----------------------------------------------------------------------===//
 //                              Type Operators
 //===----------------------------------------------------------------------===//
@@ -3581,11 +3768,14 @@ ASTContext::getCanonicalTemplateArgument(const TemplateArgument &Arg) const {
       return Arg;
 
     case TemplateArgument::Declaration: {
-      if (Decl *D = Arg.getAsDecl())
-          return TemplateArgument(D->getCanonicalDecl());
-      return TemplateArgument((Decl*)0);
+      ValueDecl *D = cast<ValueDecl>(Arg.getAsDecl()->getCanonicalDecl());
+      return TemplateArgument(D, Arg.isDeclForReferenceParam());
     }
 
+    case TemplateArgument::NullPtr:
+      return TemplateArgument(getCanonicalType(Arg.getNullPtrType()),
+                              /*isNullPtr*/true);
+
     case TemplateArgument::Template:
       return TemplateArgument(getCanonicalTemplateName(Arg.getAsTemplate()));
 
@@ -4297,7 +4487,13 @@ std::string ASTContext::getObjCEncodingForBlock(const BlockExpr *Expr) const {
   QualType BlockTy =
       Expr->getType()->getAs<BlockPointerType>()->getPointeeType();
   // Encode result type.
-  getObjCEncodingForType(BlockTy->getAs<FunctionType>()->getResultType(), S);
+  if (getLangOpts().EncodeExtendedBlockSig)
+    getObjCEncodingForMethodParameter(Decl::OBJC_TQ_None,
+                            BlockTy->getAs<FunctionType>()->getResultType(),
+                            S, true /*Extended*/);
+  else
+    getObjCEncodingForType(BlockTy->getAs<FunctionType>()->getResultType(),
+                           S);
   // Compute size of all parameters.
   // Start with computing size of a pointer in number of bytes.
   // FIXME: There might(should) be a better way of doing this computation!
@@ -4332,7 +4528,11 @@ std::string ASTContext::getObjCEncodingForBlock(const BlockExpr *Expr) const {
         PType = PVDecl->getType();
     } else if (PType->isFunctionType())
       PType = PVDecl->getType();
-    getObjCEncodingForType(PType, S);
+    if (getLangOpts().EncodeExtendedBlockSig)
+      getObjCEncodingForMethodParameter(Decl::OBJC_TQ_None, PType,
+                                      S, true /*Extended*/);
+    else
+      getObjCEncodingForType(PType, S);
     S += charUnitsToString(ParmOffset);
     ParmOffset += getObjCEncodingTypeSize(PType);
   }
@@ -5393,6 +5593,65 @@ static TypedefDecl *CreatePNaClABIBuiltinVaListDecl(const ASTContext *Context) {
   return VaListTypedefDecl;
 }
 
+static TypedefDecl *
+CreateAAPCSABIBuiltinVaListDecl(const ASTContext *Context) {
+  RecordDecl *VaListDecl;
+  if (Context->getLangOpts().CPlusPlus) {
+    // namespace std { struct __va_list {
+    NamespaceDecl *NS;
+    NS = NamespaceDecl::Create(const_cast<ASTContext &>(*Context),
+                               Context->getTranslationUnitDecl(),
+                               /*Inline*/false, SourceLocation(),
+                               SourceLocation(), &Context->Idents.get("std"),
+                               /*PrevDecl*/0);
+
+    VaListDecl = CXXRecordDecl::Create(*Context, TTK_Struct,
+                                       Context->getTranslationUnitDecl(),
+                                       SourceLocation(), SourceLocation(),
+                                       &Context->Idents.get("__va_list"));
+
+    VaListDecl->setDeclContext(NS);
+
+  } else {
+    // struct __va_list {
+    VaListDecl = CreateRecordDecl(*Context, TTK_Struct,
+                                  Context->getTranslationUnitDecl(),
+                                  &Context->Idents.get("__va_list"));
+  }
+
+  VaListDecl->startDefinition();
+
+  // void * __ap;
+  FieldDecl *Field = FieldDecl::Create(const_cast<ASTContext &>(*Context),
+                                       VaListDecl,
+                                       SourceLocation(),
+                                       SourceLocation(),
+                                       &Context->Idents.get("__ap"),
+                                       Context->getPointerType(Context->VoidTy),
+                                       /*TInfo=*/0,
+                                       /*BitWidth=*/0,
+                                       /*Mutable=*/false,
+                                       ICIS_NoInit);
+  Field->setAccess(AS_public);
+  VaListDecl->addDecl(Field);
+
+  // };
+  VaListDecl->completeDefinition();
+
+  // typedef struct __va_list __builtin_va_list;
+  TypeSourceInfo *TInfo
+    = Context->getTrivialTypeSourceInfo(Context->getRecordType(VaListDecl));
+
+  TypedefDecl *VaListTypeDecl
+    = TypedefDecl::Create(const_cast<ASTContext &>(*Context),
+                          Context->getTranslationUnitDecl(),
+                          SourceLocation(), SourceLocation(),
+                          &Context->Idents.get("__builtin_va_list"),
+                          TInfo);
+
+  return VaListTypeDecl;
+}
+
 static TypedefDecl *CreateVaListDecl(const ASTContext *Context,
                                      TargetInfo::BuiltinVaListKind Kind) {
   switch (Kind) {
@@ -5406,6 +5665,8 @@ static TypedefDecl *CreateVaListDecl(const ASTContext *Context,
     return CreateX86_64ABIBuiltinVaListDecl(Context);
   case TargetInfo::PNaClABIBuiltinVaList:
     return CreatePNaClABIBuiltinVaListDecl(Context);
+  case TargetInfo::AAPCSABIBuiltinVaList:
+    return CreateAAPCSABIBuiltinVaListDecl(Context);
   }
 
   llvm_unreachable("Unhandled __builtin_va_list type kind");
@@ -5702,7 +5963,7 @@ ASTContext::ProtocolCompatibleWithProtocol(ObjCProtocolDecl *lProto,
   return false;
 }
 
-/// QualifiedIdConformsQualifiedId - compare id<p,...> with id<p1,...>
+/// QualifiedIdConformsQualifiedId - compare id<pr,...> with id<pr1,...>
 /// return true if lhs's protocols conform to rhs's protocol; false
 /// otherwise.
 bool ASTContext::QualifiedIdConformsQualifiedId(QualType lhs, QualType rhs) {
@@ -5711,8 +5972,8 @@ bool ASTContext::QualifiedIdConformsQualifiedId(QualType lhs, QualType rhs) {
   return false;
 }
 
-/// ObjCQualifiedClassTypesAreCompatible - compare  Class<p,...> and
-/// Class<p1, ...>.
+/// ObjCQualifiedClassTypesAreCompatible - compare  Class<pr,...> and
+/// Class<pr1, ...>.
 bool ASTContext::ObjCQualifiedClassTypesAreCompatible(QualType lhs, 
                                                       QualType rhs) {
   const ObjCObjectPointerType *lhsQID = lhs->getAs<ObjCObjectPointerType>();
@@ -6315,10 +6576,13 @@ QualType ASTContext::mergeFunctionTypes(QualType lhs, QualType rhs,
     for (unsigned i = 0; i < proto_nargs; ++i) {
       QualType argTy = proto->getArgType(i);
       
-      // Look at the promotion type of enum types, since that is the type used
+      // Look at the converted type of enum types, since that is the type used
       // to pass enum values.
-      if (const EnumType *Enum = argTy->getAs<EnumType>())
-        argTy = Enum->getDecl()->getPromotionType();
+      if (const EnumType *Enum = argTy->getAs<EnumType>()) {
+        argTy = Enum->getDecl()->getIntegerType();
+        if (argTy.isNull())
+          return QualType();
+      }
       
       if (argTy->isPromotableIntegerType() ||
           getCanonicalType(argTy).getUnqualifiedType() == FloatTy)
@@ -6725,7 +6989,7 @@ unsigned ASTContext::getIntWidth(QualType T) const {
   return (unsigned)getTypeSize(T);
 }
 
-QualType ASTContext::getCorrespondingUnsignedType(QualType T) {
+QualType ASTContext::getCorrespondingUnsignedType(QualType T) const {
   assert(T->hasSignedIntegerRepresentation() && "Unexpected type");
   
   // Turn <4 x signed int> -> <4 x unsigned int>
@@ -6959,6 +7223,9 @@ static QualType DecodeTypeFromStr(const char *&Str, const ASTContext &Context,
       return QualType();
     }
     break;
+  case 'p':
+    Type = Context.getProcessIDType();
+    break;
   }
 
   // If there are modifiers and if we're allowed to parse them, go for it.
diff --git a/lib/AST/ASTDiagnostic.cpp b/lib/AST/ASTDiagnostic.cpp
index a605f1a..0b9c524 100644
--- a/lib/AST/ASTDiagnostic.cpp
+++ b/lib/AST/ASTDiagnostic.cpp
@@ -430,6 +430,15 @@ class TemplateDiff {
       /// arguments or the type arguments that are templates.
       TemplateDecl *FromTD, *ToTD;
 
+      /// FromQual, ToQual - Qualifiers for template types.
+      Qualifiers FromQual, ToQual;
+
+      /// FromInt, ToInt - APSInt's for integral arguments.
+      llvm::APSInt FromInt, ToInt;
+
+      /// IsValidFromInt, IsValidToInt - Whether the APSInt's are valid.
+      bool IsValidFromInt, IsValidToInt;
+
       /// FromDefault, ToDefault - Whether the argument is a default argument.
       bool FromDefault, ToDefault;
 
@@ -480,6 +489,21 @@ class TemplateDiff {
       FlatTree[CurrentNode].ToExpr = ToExpr;
     }
 
+    /// SetNode - Set FromInt and ToInt of the current node.
+    void SetNode(llvm::APSInt FromInt, llvm::APSInt ToInt,
+                 bool IsValidFromInt, bool IsValidToInt) {
+      FlatTree[CurrentNode].FromInt = FromInt;
+      FlatTree[CurrentNode].ToInt = ToInt;
+      FlatTree[CurrentNode].IsValidFromInt = IsValidFromInt;
+      FlatTree[CurrentNode].IsValidToInt = IsValidToInt;
+    }
+
+    /// SetNode - Set FromQual and ToQual of the current node.
+    void SetNode(Qualifiers FromQual, Qualifiers ToQual) {
+      FlatTree[CurrentNode].FromQual = FromQual;
+      FlatTree[CurrentNode].ToQual = ToQual;
+    }
+
     /// SetSame - Sets the same flag of the current node.
     void SetSame(bool Same) {
       FlatTree[CurrentNode].Same = Same;
@@ -557,6 +581,12 @@ class TemplateDiff {
              (FlatTree[ReadNode].FromTD || FlatTree[ReadNode].ToTD);
     }
 
+    /// NodeIsAPSInt - Returns true if the arugments are stored in APSInt's.
+    bool NodeIsAPSInt() {
+      return FlatTree[ReadNode].IsValidFromInt ||
+             FlatTree[ReadNode].IsValidToInt;
+    }
+
     /// GetNode - Gets the FromType and ToType.
     void GetNode(QualType &FromType, QualType &ToType) {
       FromType = FlatTree[ReadNode].FromType;
@@ -575,6 +605,21 @@ class TemplateDiff {
       ToTD = FlatTree[ReadNode].ToTD;
     }
 
+    /// GetNode - Gets the FromInt and ToInt.
+    void GetNode(llvm::APSInt &FromInt, llvm::APSInt &ToInt,
+                 bool &IsValidFromInt, bool &IsValidToInt) {
+      FromInt = FlatTree[ReadNode].FromInt;
+      ToInt = FlatTree[ReadNode].ToInt;
+      IsValidFromInt = FlatTree[ReadNode].IsValidFromInt;
+      IsValidToInt = FlatTree[ReadNode].IsValidToInt;
+    }
+
+    /// GetNode - Gets the FromQual and ToQual.
+    void GetNode(Qualifiers &FromQual, Qualifiers &ToQual) {
+      FromQual = FlatTree[ReadNode].FromQual;
+      ToQual = FlatTree[ReadNode].ToQual;
+    }
+
     /// NodeIsSame - Returns true the arguments are the same.
     bool NodeIsSame() {
       return FlatTree[ReadNode].Same;
@@ -778,18 +823,21 @@ class TemplateDiff {
           if (Context.hasSameType(FromType, ToType)) {
             Tree.SetSame(true);
           } else {
+            Qualifiers FromQual = FromType.getQualifiers(),
+                       ToQual = ToType.getQualifiers();
             const TemplateSpecializationType *FromArgTST =
                 GetTemplateSpecializationType(Context, FromType);
             const TemplateSpecializationType *ToArgTST =
                 GetTemplateSpecializationType(Context, ToType);
 
-            if (FromArgTST && ToArgTST) {
-              bool SameTemplate = hasSameTemplate(FromArgTST, ToArgTST);
-              if (SameTemplate) {
-                Tree.SetNode(FromArgTST->getTemplateName().getAsTemplateDecl(),
-                             ToArgTST->getTemplateName().getAsTemplateDecl());
-                DiffTemplate(FromArgTST, ToArgTST);
-              }
+            if (FromArgTST && ToArgTST &&
+                hasSameTemplate(FromArgTST, ToArgTST)) {
+              FromQual -= QualType(FromArgTST, 0).getQualifiers();
+              ToQual -= QualType(ToArgTST, 0).getQualifiers();
+              Tree.SetNode(FromArgTST->getTemplateName().getAsTemplateDecl(),
+                           ToArgTST->getTemplateName().getAsTemplateDecl());
+              Tree.SetNode(FromQual, ToQual);
+              DiffTemplate(FromArgTST, ToArgTST);
             }
           }
         }
@@ -799,12 +847,41 @@ class TemplateDiff {
       if (NonTypeTemplateParmDecl *DefaultNTTPD =
               dyn_cast<NonTypeTemplateParmDecl>(ParamND)) {
         Expr *FromExpr, *ToExpr;
-        GetExpr(FromIter, DefaultNTTPD, FromExpr);
-        GetExpr(ToIter, DefaultNTTPD, ToExpr);
-        Tree.SetNode(FromExpr, ToExpr);
-        Tree.SetSame(IsEqualExpr(Context, FromExpr, ToExpr));
-        Tree.SetDefault(FromIter.isEnd() && FromExpr,
-                        ToIter.isEnd() && ToExpr);
+        llvm::APSInt FromInt, ToInt;
+        bool HasFromInt = !FromIter.isEnd() &&
+                          FromIter->getKind() == TemplateArgument::Integral;
+        bool HasToInt = !ToIter.isEnd() &&
+                        ToIter->getKind() == TemplateArgument::Integral;
+        //bool IsValidFromInt = false, IsValidToInt = false;
+        if (HasFromInt)
+          FromInt = FromIter->getAsIntegral();
+        else
+          GetExpr(FromIter, DefaultNTTPD, FromExpr);
+
+        if (HasToInt)
+          ToInt = ToIter->getAsIntegral();
+        else
+          GetExpr(ToIter, DefaultNTTPD, ToExpr);
+
+        if (!HasFromInt && !HasToInt) {
+          Tree.SetNode(FromExpr, ToExpr);
+          Tree.SetSame(IsEqualExpr(Context, FromExpr, ToExpr));
+          Tree.SetDefault(FromIter.isEnd() && FromExpr,
+                          ToIter.isEnd() && ToExpr);
+        } else {
+          if (!HasFromInt && FromExpr) {
+            FromInt = FromExpr->EvaluateKnownConstInt(Context);
+            HasFromInt = true;
+          }
+          if (!HasToInt && ToExpr) {
+            ToInt = ToExpr->EvaluateKnownConstInt(Context);
+            HasToInt = true;
+          }
+          Tree.SetNode(FromInt, ToInt, HasFromInt, HasToInt);
+          Tree.SetSame(llvm::APSInt::isSameValue(FromInt, ToInt));
+          Tree.SetDefault(FromIter.isEnd() && HasFromInt,
+                          ToIter.isEnd() && HasToInt);
+        }
       }
 
       // Handle Templates
@@ -824,6 +901,26 @@ class TemplateDiff {
     }
   }
 
+  /// makeTemplateList - Dump every template alias into the vector.
+  static void makeTemplateList(
+      SmallVector<const TemplateSpecializationType*, 1> &TemplateList,
+      const TemplateSpecializationType *TST) {
+    while (TST) {
+      TemplateList.push_back(TST);
+      if (!TST->isTypeAlias())
+        return;
+      TST = TST->getAliasedType()->getAs<TemplateSpecializationType>();
+    }
+  }
+
+  /// hasSameBaseTemplate - Returns true when the base templates are the same,
+  /// even if the template arguments are not.
+  static bool hasSameBaseTemplate(const TemplateSpecializationType *FromTST,
+                                  const TemplateSpecializationType *ToTST) {
+    return FromTST->getTemplateName().getAsTemplateDecl()->getIdentifier() ==
+           ToTST->getTemplateName().getAsTemplateDecl()->getIdentifier();
+  }
+
   /// hasSameTemplate - Returns true if both types are specialized from the
   /// same template declaration.  If they come from different template aliases,
   /// do a parallel ascension search to determine the highest template alias in
@@ -831,49 +928,29 @@ class TemplateDiff {
   static bool hasSameTemplate(const TemplateSpecializationType *&FromTST,
                               const TemplateSpecializationType *&ToTST) {
     // Check the top templates if they are the same.
-    if (FromTST->getTemplateName().getAsTemplateDecl()->getIdentifier() ==
-        ToTST->getTemplateName().getAsTemplateDecl()->getIdentifier())
+    if (hasSameBaseTemplate(FromTST, ToTST))
       return true;
 
     // Create vectors of template aliases.
     SmallVector<const TemplateSpecializationType*, 1> FromTemplateList,
                                                       ToTemplateList;
 
-    const TemplateSpecializationType *TempToTST = ToTST, *TempFromTST = FromTST;
-    FromTemplateList.push_back(FromTST);
-    ToTemplateList.push_back(ToTST);
-
-    // Dump every template alias into the vectors.
-    while (TempFromTST->isTypeAlias()) {
-      TempFromTST =
-          TempFromTST->getAliasedType()->getAs<TemplateSpecializationType>();
-      if (!TempFromTST)
-        break;
-      FromTemplateList.push_back(TempFromTST);
-    }
-    while (TempToTST->isTypeAlias()) {
-      TempToTST =
-          TempToTST->getAliasedType()->getAs<TemplateSpecializationType>();
-      if (!TempToTST)
-        break;
-      ToTemplateList.push_back(TempToTST);
-    }
+    makeTemplateList(FromTemplateList, FromTST);
+    makeTemplateList(ToTemplateList, ToTST);
 
     SmallVector<const TemplateSpecializationType*, 1>::reverse_iterator
         FromIter = FromTemplateList.rbegin(), FromEnd = FromTemplateList.rend(),
         ToIter = ToTemplateList.rbegin(), ToEnd = ToTemplateList.rend();
 
     // Check if the lowest template types are the same.  If not, return.
-    if ((*FromIter)->getTemplateName().getAsTemplateDecl()->getIdentifier() !=
-        (*ToIter)->getTemplateName().getAsTemplateDecl()->getIdentifier())
+    if (!hasSameBaseTemplate(*FromIter, *ToIter))
       return false;
 
     // Begin searching up the template aliases.  The bottom most template
     // matches so move up until one pair does not match.  Use the template
     // right before that one.
     for (; FromIter != FromEnd && ToIter != ToEnd; ++FromIter, ++ToIter) {
-      if ((*FromIter)->getTemplateName().getAsTemplateDecl()->getIdentifier() !=
-          (*ToIter)->getTemplateName().getAsTemplateDecl()->getIdentifier())
+      if (!hasSameBaseTemplate(*FromIter, *ToIter))
         break;
     }
 
@@ -923,7 +1000,9 @@ class TemplateDiff {
     bool isVariadic = DefaultTTPD->isParameterPack();
 
     TemplateArgument TA = DefaultTTPD->getDefaultArgument().getArgument();
-    TemplateDecl *DefaultTD = TA.getAsTemplate().getAsTemplateDecl();
+    TemplateDecl *DefaultTD = 0;
+    if (TA.getKind() != TemplateArgument::Null)
+      DefaultTD = TA.getAsTemplate().getAsTemplateDecl();
 
     if (!Iter.isEnd())
       ArgDecl = Iter->getAsTemplate().getAsTemplateDecl();
@@ -1018,6 +1097,15 @@ class TemplateDiff {
                               Tree.ToDefault(), Tree.NodeIsSame());
         return;
       }
+
+      if (Tree.NodeIsAPSInt()) {
+        llvm::APSInt FromInt, ToInt;
+        bool IsValidFromInt, IsValidToInt;
+        Tree.GetNode(FromInt, ToInt, IsValidFromInt, IsValidToInt);
+        PrintAPSInt(FromInt, ToInt, IsValidFromInt, IsValidToInt,
+                    Tree.FromDefault(), Tree.ToDefault(), Tree.NodeIsSame());
+        return;
+      }
       llvm_unreachable("Unable to deduce template difference.");
     }
 
@@ -1027,6 +1115,10 @@ class TemplateDiff {
 
     assert(Tree.HasChildren() && "Template difference not found in diff tree.");
 
+    Qualifiers FromQual, ToQual;
+    Tree.GetNode(FromQual, ToQual);
+    PrintQualifiers(FromQual, ToQual);
+
     OS << FromTD->getNameAsString() << '<'; 
     Tree.MoveToChild();
     unsigned NumElideArgs = 0;
@@ -1088,6 +1180,17 @@ class TemplateDiff {
       return;
     }
 
+    if (!FromType.isNull() && !ToType.isNull() &&
+        FromType.getLocalUnqualifiedType() ==
+        ToType.getLocalUnqualifiedType()) {
+      Qualifiers FromQual = FromType.getLocalQualifiers(),
+                 ToQual = ToType.getLocalQualifiers(),
+                 CommonQual;
+      PrintQualifiers(FromQual, ToQual);
+      FromType.getLocalUnqualifiedType().print(OS, Policy);
+      return;
+    }
+
     std::string FromTypeStr = FromType.isNull() ? "(no argument)"
                                                 : FromType.getAsString();
     std::string ToTypeStr = ToType.isNull() ? "(no argument)"
@@ -1177,6 +1280,34 @@ class TemplateDiff {
     }
   }
 
+  /// PrintAPSInt - Handles printing of integral arguments, highlighting
+  /// argument differences.
+  void PrintAPSInt(llvm::APSInt FromInt, llvm::APSInt ToInt,
+                   bool IsValidFromInt, bool IsValidToInt, bool FromDefault,
+                   bool ToDefault, bool Same) {
+    assert((IsValidFromInt || IsValidToInt) &&
+           "Only one integral argument may be missing.");
+
+    if (Same) {
+      OS << FromInt.toString(10);
+    } else if (!PrintTree) {
+      OS << (FromDefault ? "(default) " : "");
+      Bold();
+      OS << (IsValidFromInt ? FromInt.toString(10) : "(no argument)");
+      Unbold();
+    } else {
+      OS << (FromDefault ? "[(default) " : "[");
+      Bold();
+      OS << (IsValidFromInt ? FromInt.toString(10) : "(no argument)");
+      Unbold();
+      OS << " != " << (ToDefault ? "(default) " : "");
+      Bold();
+      OS << (IsValidToInt ? ToInt.toString(10) : "(no argument)");
+      Unbold();
+      OS << ']';
+    }
+  }
+
   // Prints the appropriate placeholder for elided template arguments.
   void PrintElideArgs(unsigned NumElideArgs, unsigned Indent) {
     if (PrintTree) {
@@ -1191,6 +1322,68 @@ class TemplateDiff {
       OS << "[" << NumElideArgs << " * ...]";
   }
 
+  // Prints and highlights differences in Qualifiers.
+  void PrintQualifiers(Qualifiers FromQual, Qualifiers ToQual) {
+    // Both types have no qualifiers
+    if (FromQual.empty() && ToQual.empty())
+      return;
+
+    // Both types have same qualifiers
+    if (FromQual == ToQual) {
+      PrintQualifier(FromQual, /*ApplyBold*/false);
+      return;
+    }
+
+    // Find common qualifiers and strip them from FromQual and ToQual.
+    Qualifiers CommonQual = Qualifiers::removeCommonQualifiers(FromQual,
+                                                               ToQual);
+
+    // The qualifiers are printed before the template name.
+    // Inline printing:
+    // The common qualifiers are printed.  Then, qualifiers only in this type
+    // are printed and highlighted.  Finally, qualifiers only in the other
+    // type are printed and highlighted inside parentheses after "missing".
+    // Tree printing:
+    // Qualifiers are printed next to each other, inside brackets, and
+    // separated by "!=".  The printing order is:
+    // common qualifiers, highlighted from qualifiers, "!=",
+    // common qualifiers, highlighted to qualifiers
+    if (PrintTree) {
+      OS << "[";
+      if (CommonQual.empty() && FromQual.empty()) {
+        Bold();
+        OS << "(no qualifiers) ";
+        Unbold();
+      } else {
+        PrintQualifier(CommonQual, /*ApplyBold*/false);
+        PrintQualifier(FromQual, /*ApplyBold*/true);
+      }
+      OS << "!= ";
+      if (CommonQual.empty() && ToQual.empty()) {
+        Bold();
+        OS << "(no qualifiers)";
+        Unbold();
+      } else {
+        PrintQualifier(CommonQual, /*ApplyBold*/false,
+                       /*appendSpaceIfNonEmpty*/!ToQual.empty());
+        PrintQualifier(ToQual, /*ApplyBold*/true,
+                       /*appendSpaceIfNonEmpty*/false);
+      }
+      OS << "] ";
+    } else {
+      PrintQualifier(CommonQual, /*ApplyBold*/false);
+      PrintQualifier(FromQual, /*ApplyBold*/true);
+    }
+  }
+
+  void PrintQualifier(Qualifiers Q, bool ApplyBold,
+                      bool AppendSpaceIfNonEmpty = true) {
+    if (Q.empty()) return;
+    if (ApplyBold) Bold();
+    Q.print(OS, Policy, AppendSpaceIfNonEmpty);
+    if (ApplyBold) Unbold();
+  }
+
 public:
 
   TemplateDiff(ASTContext &Context, QualType FromType, QualType ToType,
@@ -1210,6 +1403,9 @@ public:
 
   /// DiffTemplate - Start the template type diffing.
   void DiffTemplate() {
+    Qualifiers FromQual = FromType.getQualifiers(),
+               ToQual = ToType.getQualifiers();
+
     const TemplateSpecializationType *FromOrigTST =
         GetTemplateSpecializationType(Context, FromType);
     const TemplateSpecializationType *ToOrigTST =
@@ -1224,7 +1420,10 @@ public:
       return;
     }
 
+    FromQual -= QualType(FromOrigTST, 0).getQualifiers();
+    ToQual -= QualType(ToOrigTST, 0).getQualifiers();
     Tree.SetNode(FromType, ToType);
+    Tree.SetNode(FromQual, ToQual);
 
     // Same base template, but different arguments.
     Tree.SetNode(FromOrigTST->getTemplateName().getAsTemplateDecl(),
diff --git a/lib/AST/ASTImporter.cpp b/lib/AST/ASTImporter.cpp
index 3e952ac..0d4f303 100644
--- a/lib/AST/ASTImporter.cpp
+++ b/lib/AST/ASTImporter.cpp
@@ -228,16 +228,12 @@ namespace {
     
   public:
     DiagnosticBuilder Diag1(SourceLocation Loc, unsigned DiagID) {
-      if (!Complain)
-        return DiagnosticBuilder::getEmpty();
-
+      assert(Complain && "Not allowed to complain");
       return C1.getDiagnostics().Report(Loc, DiagID);
     }
 
     DiagnosticBuilder Diag2(SourceLocation Loc, unsigned DiagID) {
-      if (!Complain)
-        return DiagnosticBuilder::getEmpty();
-
+      assert(Complain && "Not allowed to complain");
       return C2.getDiagnostics().Report(Loc, DiagID);
     }
   };
@@ -288,7 +284,7 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
       
   case TemplateArgument::Type:
     return Context.IsStructurallyEquivalent(Arg1.getAsType(), Arg2.getAsType());
-      
+
   case TemplateArgument::Integral:
     if (!Context.IsStructurallyEquivalent(Arg1.getIntegralType(), 
                                           Arg2.getIntegralType()))
@@ -297,10 +293,11 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
     return llvm::APSInt::isSameValue(Arg1.getAsIntegral(), Arg2.getAsIntegral());
       
   case TemplateArgument::Declaration:
-    if (!Arg1.getAsDecl() || !Arg2.getAsDecl())
-      return !Arg1.getAsDecl() && !Arg2.getAsDecl();
     return Context.IsStructurallyEquivalent(Arg1.getAsDecl(), Arg2.getAsDecl());
-      
+
+  case TemplateArgument::NullPtr:
+    return true; // FIXME: Is this correct?
+
   case TemplateArgument::Template:
     return IsStructurallyEquivalent(Context, 
                                     Arg1.getAsTemplate(), 
@@ -822,33 +819,47 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
 static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
                                      FieldDecl *Field1, FieldDecl *Field2) {
   RecordDecl *Owner2 = cast<RecordDecl>(Field2->getDeclContext());
-  
-  if (!IsStructurallyEquivalent(Context, 
+
+  // For anonymous structs/unions, match up the anonymous struct/union type
+  // declarations directly, so that we don't go off searching for anonymous
+  // types
+  if (Field1->isAnonymousStructOrUnion() &&
+      Field2->isAnonymousStructOrUnion()) {
+    RecordDecl *D1 = Field1->getType()->castAs<RecordType>()->getDecl();
+    RecordDecl *D2 = Field2->getType()->castAs<RecordType>()->getDecl();
+    return IsStructurallyEquivalent(Context, D1, D2);
+  }
+
+  if (!IsStructurallyEquivalent(Context,
                                 Field1->getType(), Field2->getType())) {
-    Context.Diag2(Owner2->getLocation(), diag::warn_odr_tag_type_inconsistent)
-    << Context.C2.getTypeDeclType(Owner2);
-    Context.Diag2(Field2->getLocation(), diag::note_odr_field)
-    << Field2->getDeclName() << Field2->getType();
-    Context.Diag1(Field1->getLocation(), diag::note_odr_field)
-    << Field1->getDeclName() << Field1->getType();
+    if (Context.Complain) {
+      Context.Diag2(Owner2->getLocation(), diag::warn_odr_tag_type_inconsistent)
+        << Context.C2.getTypeDeclType(Owner2);
+      Context.Diag2(Field2->getLocation(), diag::note_odr_field)
+        << Field2->getDeclName() << Field2->getType();
+      Context.Diag1(Field1->getLocation(), diag::note_odr_field)
+        << Field1->getDeclName() << Field1->getType();
+    }
     return false;
   }
   
   if (Field1->isBitField() != Field2->isBitField()) {
-    Context.Diag2(Owner2->getLocation(), diag::warn_odr_tag_type_inconsistent)
-    << Context.C2.getTypeDeclType(Owner2);
-    if (Field1->isBitField()) {
-      Context.Diag1(Field1->getLocation(), diag::note_odr_bit_field)
-      << Field1->getDeclName() << Field1->getType()
-      << Field1->getBitWidthValue(Context.C1);
-      Context.Diag2(Field2->getLocation(), diag::note_odr_not_bit_field)
-      << Field2->getDeclName();
-    } else {
-      Context.Diag2(Field2->getLocation(), diag::note_odr_bit_field)
-      << Field2->getDeclName() << Field2->getType()
-      << Field2->getBitWidthValue(Context.C2);
-      Context.Diag1(Field1->getLocation(), diag::note_odr_not_bit_field)
-      << Field1->getDeclName();
+    if (Context.Complain) {
+      Context.Diag2(Owner2->getLocation(), diag::warn_odr_tag_type_inconsistent)
+        << Context.C2.getTypeDeclType(Owner2);
+      if (Field1->isBitField()) {
+        Context.Diag1(Field1->getLocation(), diag::note_odr_bit_field)
+        << Field1->getDeclName() << Field1->getType()
+        << Field1->getBitWidthValue(Context.C1);
+        Context.Diag2(Field2->getLocation(), diag::note_odr_not_bit_field)
+        << Field2->getDeclName();
+      } else {
+        Context.Diag2(Field2->getLocation(), diag::note_odr_bit_field)
+        << Field2->getDeclName() << Field2->getType()
+        << Field2->getBitWidthValue(Context.C2);
+        Context.Diag1(Field1->getLocation(), diag::note_odr_not_bit_field)
+        << Field1->getDeclName();
+      }
     }
     return false;
   }
@@ -859,12 +870,14 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
     unsigned Bits2 = Field2->getBitWidthValue(Context.C2);
     
     if (Bits1 != Bits2) {
-      Context.Diag2(Owner2->getLocation(), diag::warn_odr_tag_type_inconsistent)
-      << Context.C2.getTypeDeclType(Owner2);
-      Context.Diag2(Field2->getLocation(), diag::note_odr_bit_field)
-      << Field2->getDeclName() << Field2->getType() << Bits2;
-      Context.Diag1(Field1->getLocation(), diag::note_odr_bit_field)
-      << Field1->getDeclName() << Field1->getType() << Bits1;
+      if (Context.Complain) {
+        Context.Diag2(Owner2->getLocation(), diag::warn_odr_tag_type_inconsistent)
+          << Context.C2.getTypeDeclType(Owner2);
+        Context.Diag2(Field2->getLocation(), diag::note_odr_bit_field)
+          << Field2->getDeclName() << Field2->getType() << Bits2;
+        Context.Diag1(Field1->getLocation(), diag::note_odr_bit_field)
+          << Field1->getDeclName() << Field1->getType() << Bits1;
+      }
       return false;
     }
   }
@@ -872,17 +885,65 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
   return true;
 }
 
+/// \brief Find the index of the given anonymous struct/union within its
+/// context.
+///
+/// \returns Returns the index of this anonymous struct/union in its context,
+/// including the next assigned index (if none of them match). Returns an
+/// empty option if the context is not a record, i.e.. if the anonymous
+/// struct/union is at namespace or block scope.
+static llvm::Optional<unsigned>
+findAnonymousStructOrUnionIndex(RecordDecl *Anon) {
+  ASTContext &Context = Anon->getASTContext();
+  QualType AnonTy = Context.getRecordType(Anon);
+
+  RecordDecl *Owner = dyn_cast<RecordDecl>(Anon->getDeclContext());
+  if (!Owner)
+    return llvm::Optional<unsigned>();
+
+  unsigned Index = 0;
+  for (DeclContext::decl_iterator D = Owner->noload_decls_begin(),
+                               DEnd = Owner->noload_decls_end();
+       D != DEnd; ++D) {
+    FieldDecl *F = dyn_cast<FieldDecl>(*D);
+    if (!F || !F->isAnonymousStructOrUnion())
+      continue;
+
+    if (Context.hasSameType(F->getType(), AnonTy))
+      break;
+
+    ++Index;
+  }
+
+  return Index;
+}
+
 /// \brief Determine structural equivalence of two records.
 static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
                                      RecordDecl *D1, RecordDecl *D2) {
   if (D1->isUnion() != D2->isUnion()) {
-    Context.Diag2(D2->getLocation(), diag::warn_odr_tag_type_inconsistent)
-      << Context.C2.getTypeDeclType(D2);
-    Context.Diag1(D1->getLocation(), diag::note_odr_tag_kind_here)
-      << D1->getDeclName() << (unsigned)D1->getTagKind();
+    if (Context.Complain) {
+      Context.Diag2(D2->getLocation(), diag::warn_odr_tag_type_inconsistent)
+        << Context.C2.getTypeDeclType(D2);
+      Context.Diag1(D1->getLocation(), diag::note_odr_tag_kind_here)
+        << D1->getDeclName() << (unsigned)D1->getTagKind();
+    }
     return false;
   }
-  
+
+  if (D1->isAnonymousStructOrUnion() && D2->isAnonymousStructOrUnion()) {
+    // If both anonymous structs/unions are in a record context, make sure
+    // they occur in the same location in the context records.
+    if (llvm::Optional<unsigned> Index1
+          = findAnonymousStructOrUnionIndex(D1)) {
+      if (llvm::Optional<unsigned> Index2
+            = findAnonymousStructOrUnionIndex(D2)) {
+        if (*Index1 != *Index2)
+          return false;
+      }
+    }
+  }
+
   // If both declarations are class template specializations, we know
   // the ODR applies, so check the template and template arguments.
   ClassTemplateSpecializationDecl *Spec1
@@ -920,12 +981,14 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
   if (CXXRecordDecl *D1CXX = dyn_cast<CXXRecordDecl>(D1)) {
     if (CXXRecordDecl *D2CXX = dyn_cast<CXXRecordDecl>(D2)) {
       if (D1CXX->getNumBases() != D2CXX->getNumBases()) {
-        Context.Diag2(D2->getLocation(), diag::warn_odr_tag_type_inconsistent)
-          << Context.C2.getTypeDeclType(D2);
-        Context.Diag2(D2->getLocation(), diag::note_odr_number_of_bases)
-          << D2CXX->getNumBases();
-        Context.Diag1(D1->getLocation(), diag::note_odr_number_of_bases)
-          << D1CXX->getNumBases();
+        if (Context.Complain) {
+          Context.Diag2(D2->getLocation(), diag::warn_odr_tag_type_inconsistent)
+            << Context.C2.getTypeDeclType(D2);
+          Context.Diag2(D2->getLocation(), diag::note_odr_number_of_bases)
+            << D2CXX->getNumBases();
+          Context.Diag1(D1->getLocation(), diag::note_odr_number_of_bases)
+            << D1CXX->getNumBases();
+        }
         return false;
       }
       
@@ -937,38 +1000,44 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
            ++Base1, ++Base2) {        
         if (!IsStructurallyEquivalent(Context, 
                                       Base1->getType(), Base2->getType())) {
-          Context.Diag2(D2->getLocation(), diag::warn_odr_tag_type_inconsistent)
-            << Context.C2.getTypeDeclType(D2);
-          Context.Diag2(Base2->getLocStart(), diag::note_odr_base)
-            << Base2->getType()
-            << Base2->getSourceRange();
-          Context.Diag1(Base1->getLocStart(), diag::note_odr_base)
-            << Base1->getType()
-            << Base1->getSourceRange();
+          if (Context.Complain) {
+            Context.Diag2(D2->getLocation(), diag::warn_odr_tag_type_inconsistent)
+              << Context.C2.getTypeDeclType(D2);
+            Context.Diag2(Base2->getLocStart(), diag::note_odr_base)
+              << Base2->getType()
+              << Base2->getSourceRange();
+            Context.Diag1(Base1->getLocStart(), diag::note_odr_base)
+              << Base1->getType()
+              << Base1->getSourceRange();
+          }
           return false;
         }
         
         // Check virtual vs. non-virtual inheritance mismatch.
         if (Base1->isVirtual() != Base2->isVirtual()) {
-          Context.Diag2(D2->getLocation(), diag::warn_odr_tag_type_inconsistent)
-            << Context.C2.getTypeDeclType(D2);
-          Context.Diag2(Base2->getLocStart(),
-                        diag::note_odr_virtual_base)
-            << Base2->isVirtual() << Base2->getSourceRange();
-          Context.Diag1(Base1->getLocStart(), diag::note_odr_base)
-            << Base1->isVirtual()
-            << Base1->getSourceRange();
+          if (Context.Complain) {
+            Context.Diag2(D2->getLocation(), diag::warn_odr_tag_type_inconsistent)
+              << Context.C2.getTypeDeclType(D2);
+            Context.Diag2(Base2->getLocStart(),
+                          diag::note_odr_virtual_base)
+              << Base2->isVirtual() << Base2->getSourceRange();
+            Context.Diag1(Base1->getLocStart(), diag::note_odr_base)
+              << Base1->isVirtual()
+              << Base1->getSourceRange();
+          }
           return false;
         }
       }
     } else if (D1CXX->getNumBases() > 0) {
-      Context.Diag2(D2->getLocation(), diag::warn_odr_tag_type_inconsistent)
-        << Context.C2.getTypeDeclType(D2);
-      const CXXBaseSpecifier *Base1 = D1CXX->bases_begin();
-      Context.Diag1(Base1->getLocStart(), diag::note_odr_base)
-        << Base1->getType()
-        << Base1->getSourceRange();
-      Context.Diag2(D2->getLocation(), diag::note_odr_missing_base);
+      if (Context.Complain) {
+        Context.Diag2(D2->getLocation(), diag::warn_odr_tag_type_inconsistent)
+          << Context.C2.getTypeDeclType(D2);
+        const CXXBaseSpecifier *Base1 = D1CXX->bases_begin();
+        Context.Diag1(Base1->getLocStart(), diag::note_odr_base)
+          << Base1->getType()
+          << Base1->getSourceRange();
+        Context.Diag2(D2->getLocation(), diag::note_odr_missing_base);
+      }
       return false;
     }
   }
@@ -981,11 +1050,13 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
        Field1 != Field1End;
        ++Field1, ++Field2) {
     if (Field2 == Field2End) {
-      Context.Diag2(D2->getLocation(), diag::warn_odr_tag_type_inconsistent)
-        << Context.C2.getTypeDeclType(D2);
-      Context.Diag1(Field1->getLocation(), diag::note_odr_field)
-        << Field1->getDeclName() << Field1->getType();
-      Context.Diag2(D2->getLocation(), diag::note_odr_missing_field);
+      if (Context.Complain) {
+        Context.Diag2(D2->getLocation(), diag::warn_odr_tag_type_inconsistent)
+          << Context.C2.getTypeDeclType(D2);
+        Context.Diag1(Field1->getLocation(), diag::note_odr_field)
+          << Field1->getDeclName() << Field1->getType();
+        Context.Diag2(D2->getLocation(), diag::note_odr_missing_field);
+      }
       return false;
     }
     
@@ -994,11 +1065,13 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
   }
   
   if (Field2 != Field2End) {
-    Context.Diag2(D2->getLocation(), diag::warn_odr_tag_type_inconsistent)
-      << Context.C2.getTypeDeclType(D2);
-    Context.Diag2(Field2->getLocation(), diag::note_odr_field)
-      << Field2->getDeclName() << Field2->getType();
-    Context.Diag1(D1->getLocation(), diag::note_odr_missing_field);
+    if (Context.Complain) {
+      Context.Diag2(D2->getLocation(), diag::warn_odr_tag_type_inconsistent)
+        << Context.C2.getTypeDeclType(D2);
+      Context.Diag2(Field2->getLocation(), diag::note_odr_field)
+        << Field2->getDeclName() << Field2->getType();
+      Context.Diag1(D1->getLocation(), diag::note_odr_missing_field);
+    }
     return false;
   }
   
@@ -1014,12 +1087,14 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
                                   EC1End = D1->enumerator_end();
        EC1 != EC1End; ++EC1, ++EC2) {
     if (EC2 == EC2End) {
-      Context.Diag2(D2->getLocation(), diag::warn_odr_tag_type_inconsistent)
-        << Context.C2.getTypeDeclType(D2);
-      Context.Diag1(EC1->getLocation(), diag::note_odr_enumerator)
-        << EC1->getDeclName() 
-        << EC1->getInitVal().toString(10);
-      Context.Diag2(D2->getLocation(), diag::note_odr_missing_enumerator);
+      if (Context.Complain) {
+        Context.Diag2(D2->getLocation(), diag::warn_odr_tag_type_inconsistent)
+          << Context.C2.getTypeDeclType(D2);
+        Context.Diag1(EC1->getLocation(), diag::note_odr_enumerator)
+          << EC1->getDeclName() 
+          << EC1->getInitVal().toString(10);
+        Context.Diag2(D2->getLocation(), diag::note_odr_missing_enumerator);
+      }
       return false;
     }
     
@@ -1027,25 +1102,29 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
     llvm::APSInt Val2 = EC2->getInitVal();
     if (!llvm::APSInt::isSameValue(Val1, Val2) || 
         !IsStructurallyEquivalent(EC1->getIdentifier(), EC2->getIdentifier())) {
+      if (Context.Complain) {
+        Context.Diag2(D2->getLocation(), diag::warn_odr_tag_type_inconsistent)
+          << Context.C2.getTypeDeclType(D2);
+        Context.Diag2(EC2->getLocation(), diag::note_odr_enumerator)
+          << EC2->getDeclName() 
+          << EC2->getInitVal().toString(10);
+        Context.Diag1(EC1->getLocation(), diag::note_odr_enumerator)
+          << EC1->getDeclName() 
+          << EC1->getInitVal().toString(10);
+      }
+      return false;
+    }
+  }
+  
+  if (EC2 != EC2End) {
+    if (Context.Complain) {
       Context.Diag2(D2->getLocation(), diag::warn_odr_tag_type_inconsistent)
         << Context.C2.getTypeDeclType(D2);
       Context.Diag2(EC2->getLocation(), diag::note_odr_enumerator)
         << EC2->getDeclName() 
         << EC2->getInitVal().toString(10);
-      Context.Diag1(EC1->getLocation(), diag::note_odr_enumerator)
-        << EC1->getDeclName() 
-        << EC1->getInitVal().toString(10);
-      return false;
+      Context.Diag1(D1->getLocation(), diag::note_odr_missing_enumerator);
     }
-  }
-  
-  if (EC2 != EC2End) {
-    Context.Diag2(D2->getLocation(), diag::warn_odr_tag_type_inconsistent)
-      << Context.C2.getTypeDeclType(D2);
-    Context.Diag2(EC2->getLocation(), diag::note_odr_enumerator)
-      << EC2->getDeclName() 
-      << EC2->getInitVal().toString(10);
-    Context.Diag1(D1->getLocation(), diag::note_odr_missing_enumerator);
     return false;
   }
   
@@ -1056,20 +1135,24 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
                                      TemplateParameterList *Params1,
                                      TemplateParameterList *Params2) {
   if (Params1->size() != Params2->size()) {
-    Context.Diag2(Params2->getTemplateLoc(), 
-                  diag::err_odr_different_num_template_parameters)
-      << Params1->size() << Params2->size();
-    Context.Diag1(Params1->getTemplateLoc(), 
-                  diag::note_odr_template_parameter_list);
+    if (Context.Complain) {
+      Context.Diag2(Params2->getTemplateLoc(), 
+                    diag::err_odr_different_num_template_parameters)
+        << Params1->size() << Params2->size();
+      Context.Diag1(Params1->getTemplateLoc(), 
+                    diag::note_odr_template_parameter_list);
+    }
     return false;
   }
   
   for (unsigned I = 0, N = Params1->size(); I != N; ++I) {
     if (Params1->getParam(I)->getKind() != Params2->getParam(I)->getKind()) {
-      Context.Diag2(Params2->getParam(I)->getLocation(), 
-                    diag::err_odr_different_template_parameter_kind);
-      Context.Diag1(Params1->getParam(I)->getLocation(),
-                    diag::note_odr_template_parameter_here);
+      if (Context.Complain) {
+        Context.Diag2(Params2->getParam(I)->getLocation(), 
+                      diag::err_odr_different_template_parameter_kind);
+        Context.Diag1(Params1->getParam(I)->getLocation(),
+                      diag::note_odr_template_parameter_here);
+      }
       return false;
     }
     
@@ -1087,10 +1170,12 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
                                      TemplateTypeParmDecl *D1,
                                      TemplateTypeParmDecl *D2) {
   if (D1->isParameterPack() != D2->isParameterPack()) {
-    Context.Diag2(D2->getLocation(), diag::err_odr_parameter_pack_non_pack)
-      << D2->isParameterPack();
-    Context.Diag1(D1->getLocation(), diag::note_odr_parameter_pack_non_pack)
-      << D1->isParameterPack();
+    if (Context.Complain) {
+      Context.Diag2(D2->getLocation(), diag::err_odr_parameter_pack_non_pack)
+        << D2->isParameterPack();
+      Context.Diag1(D1->getLocation(), diag::note_odr_parameter_pack_non_pack)
+        << D1->isParameterPack();
+    }
     return false;
   }
   
@@ -1100,24 +1185,25 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
 static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
                                      NonTypeTemplateParmDecl *D1,
                                      NonTypeTemplateParmDecl *D2) {
-  // FIXME: Enable once we have variadic templates.
-#if 0
   if (D1->isParameterPack() != D2->isParameterPack()) {
-    Context.Diag2(D2->getLocation(), diag::err_odr_parameter_pack_non_pack)
-      << D2->isParameterPack();
-    Context.Diag1(D1->getLocation(), diag::note_odr_parameter_pack_non_pack)
-      << D1->isParameterPack();
+    if (Context.Complain) {
+      Context.Diag2(D2->getLocation(), diag::err_odr_parameter_pack_non_pack)
+        << D2->isParameterPack();
+      Context.Diag1(D1->getLocation(), diag::note_odr_parameter_pack_non_pack)
+        << D1->isParameterPack();
+    }
     return false;
   }
-#endif
   
   // Check types.
   if (!Context.IsStructurallyEquivalent(D1->getType(), D2->getType())) {
-    Context.Diag2(D2->getLocation(), 
-                  diag::err_odr_non_type_parameter_type_inconsistent)
-      << D2->getType() << D1->getType();
-    Context.Diag1(D1->getLocation(), diag::note_odr_value_here)
-      << D1->getType();
+    if (Context.Complain) {
+      Context.Diag2(D2->getLocation(),
+                    diag::err_odr_non_type_parameter_type_inconsistent)
+        << D2->getType() << D1->getType();
+      Context.Diag1(D1->getLocation(), diag::note_odr_value_here)
+        << D1->getType();
+    }
     return false;
   }
   
@@ -1127,17 +1213,16 @@ static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
 static bool IsStructurallyEquivalent(StructuralEquivalenceContext &Context,
                                      TemplateTemplateParmDecl *D1,
                                      TemplateTemplateParmDecl *D2) {
-  // FIXME: Enable once we have variadic templates.
-#if 0
   if (D1->isParameterPack() != D2->isParameterPack()) {
-    Context.Diag2(D2->getLocation(), diag::err_odr_parameter_pack_non_pack)
-    << D2->isParameterPack();
-    Context.Diag1(D1->getLocation(), diag::note_odr_parameter_pack_non_pack)
-    << D1->isParameterPack();
+    if (Context.Complain) {
+      Context.Diag2(D2->getLocation(), diag::err_odr_parameter_pack_non_pack)
+        << D2->isParameterPack();
+      Context.Diag1(D1->getLocation(), diag::note_odr_parameter_pack_non_pack)
+        << D1->isParameterPack();
+    }
     return false;
   }
-#endif
-  
+
   // Check template parameter lists.
   return IsStructurallyEquivalent(Context, D1->getTemplateParameters(),
                                   D2->getTemplateParameters());
@@ -1509,11 +1594,26 @@ QualType ASTNodeImporter::VisitFunctionProtoType(const FunctionProtoType *T) {
     ExceptionTypes.push_back(ExceptionType);
   }
 
-  FunctionProtoType::ExtProtoInfo EPI = T->getExtProtoInfo();
-  EPI.Exceptions = ExceptionTypes.data();
-       
+  FunctionProtoType::ExtProtoInfo FromEPI = T->getExtProtoInfo();
+  FunctionProtoType::ExtProtoInfo ToEPI;
+
+  ToEPI.ExtInfo = FromEPI.ExtInfo;
+  ToEPI.Variadic = FromEPI.Variadic;
+  ToEPI.HasTrailingReturn = FromEPI.HasTrailingReturn;
+  ToEPI.TypeQuals = FromEPI.TypeQuals;
+  ToEPI.RefQualifier = FromEPI.RefQualifier;
+  ToEPI.NumExceptions = ExceptionTypes.size();
+  ToEPI.Exceptions = ExceptionTypes.data();
+  ToEPI.ConsumedArguments = FromEPI.ConsumedArguments;
+  ToEPI.ExceptionSpecType = FromEPI.ExceptionSpecType;
+  ToEPI.NoexceptExpr = Importer.Import(FromEPI.NoexceptExpr);
+  ToEPI.ExceptionSpecDecl = cast_or_null<FunctionDecl>(
+                                Importer.Import(FromEPI.ExceptionSpecDecl));
+  ToEPI.ExceptionSpecTemplate = cast_or_null<FunctionDecl>(
+                                Importer.Import(FromEPI.ExceptionSpecTemplate));
+
   return Importer.getToContext().getFunctionType(ToResultType, ArgTypes.data(),
-                                                 ArgTypes.size(), EPI);
+                                                 ArgTypes.size(), ToEPI);
 }
 
 QualType ASTNodeImporter::VisitParenType(const ParenType *T) {
@@ -1961,11 +2061,20 @@ ASTNodeImporter::ImportTemplateArgument(const TemplateArgument &From) {
     return TemplateArgument(From, ToType);
   }
 
-  case TemplateArgument::Declaration:
-    if (Decl *To = Importer.Import(From.getAsDecl()))
-      return TemplateArgument(To);
+  case TemplateArgument::Declaration: {
+    ValueDecl *FromD = From.getAsDecl();
+    if (ValueDecl *To = cast_or_null<ValueDecl>(Importer.Import(FromD)))
+      return TemplateArgument(To, From.isDeclForReferenceParam());
     return TemplateArgument();
-      
+  }
+
+  case TemplateArgument::NullPtr: {
+    QualType ToType = Importer.Import(From.getNullPtrType());
+    if (ToType.isNull())
+      return TemplateArgument();
+    return TemplateArgument(ToType, /*isNullPtr*/true);
+  }
+
   case TemplateArgument::Template: {
     TemplateName ToTemplate = Importer.Import(From.getAsTemplate());
     if (ToTemplate.isNull())
@@ -2318,6 +2427,20 @@ Decl *ASTNodeImporter::VisitRecordDecl(RecordDecl *D) {
       }
       
       if (RecordDecl *FoundRecord = dyn_cast<RecordDecl>(Found)) {
+        if (D->isAnonymousStructOrUnion() && 
+            FoundRecord->isAnonymousStructOrUnion()) {
+          // If both anonymous structs/unions are in a record context, make sure
+          // they occur in the same location in the context records.
+          if (llvm::Optional<unsigned> Index1
+              = findAnonymousStructOrUnionIndex(D)) {
+            if (llvm::Optional<unsigned> Index2
+                = findAnonymousStructOrUnionIndex(FoundRecord)) {
+              if (*Index1 != *Index2)
+                continue;
+            }
+          }
+        }
+
         if (RecordDecl *FoundDef = FoundRecord->getDefinition()) {
           if ((SearchName && !D->isCompleteDefinition())
               || (D->isCompleteDefinition() &&
@@ -2491,8 +2614,30 @@ Decl *ASTNodeImporter::VisitFunctionDecl(FunctionDecl *D) {
   // Import additional name location/type info.
   ImportDeclarationNameLoc(D->getNameInfo(), NameInfo);
 
+  QualType FromTy = D->getType();
+  bool usedDifferentExceptionSpec = false;
+
+  if (const FunctionProtoType *
+        FromFPT = D->getType()->getAs<FunctionProtoType>()) {
+    FunctionProtoType::ExtProtoInfo FromEPI = FromFPT->getExtProtoInfo();
+    // FunctionProtoType::ExtProtoInfo's ExceptionSpecDecl can point to the
+    // FunctionDecl that we are importing the FunctionProtoType for.
+    // To avoid an infinite recursion when importing, create the FunctionDecl
+    // with a simplified function type and update it afterwards.
+    if (FromEPI.ExceptionSpecDecl || FromEPI.ExceptionSpecTemplate ||
+        FromEPI.NoexceptExpr) {
+      FunctionProtoType::ExtProtoInfo DefaultEPI;
+      FromTy = Importer.getFromContext().getFunctionType(
+                            FromFPT->getResultType(),
+                            FromFPT->arg_type_begin(),
+                            FromFPT->arg_type_end() - FromFPT->arg_type_begin(),
+                            DefaultEPI);
+      usedDifferentExceptionSpec = true;
+    }
+  }
+
   // Import the type.
-  QualType T = Importer.Import(D->getType());
+  QualType T = Importer.Import(FromTy);
   if (T.isNull())
     return 0;
   
@@ -2572,6 +2717,14 @@ Decl *ASTNodeImporter::VisitFunctionDecl(FunctionDecl *D) {
   }
   ToFunction->setParams(Parameters);
 
+  if (usedDifferentExceptionSpec) {
+    // Update FunctionProtoType::ExtProtoInfo.
+    QualType T = Importer.Import(D->getType());
+    if (T.isNull())
+      return 0;
+    ToFunction->setType(T);
+  }
+
   // FIXME: Other bits to merge?
 
   // Add this function to the lexical context.
@@ -2596,6 +2749,25 @@ Decl *ASTNodeImporter::VisitCXXConversionDecl(CXXConversionDecl *D) {
   return VisitCXXMethodDecl(D);
 }
 
+static unsigned getFieldIndex(Decl *F) {
+  RecordDecl *Owner = dyn_cast<RecordDecl>(F->getDeclContext());
+  if (!Owner)
+    return 0;
+
+  unsigned Index = 1;
+  for (DeclContext::decl_iterator D = Owner->noload_decls_begin(),
+                               DEnd = Owner->noload_decls_end();
+       D != DEnd; ++D) {
+    if (*D == F)
+      return Index;
+
+    if (isa<FieldDecl>(*D) || isa<IndirectFieldDecl>(*D))
+      ++Index;
+  }
+
+  return Index;
+}
+
 Decl *ASTNodeImporter::VisitFieldDecl(FieldDecl *D) {
   // Import the major distinguishing characteristics of a variable.
   DeclContext *DC, *LexicalDC;
@@ -2609,6 +2781,10 @@ Decl *ASTNodeImporter::VisitFieldDecl(FieldDecl *D) {
   DC->localUncachedLookup(Name, FoundDecls);
   for (unsigned I = 0, N = FoundDecls.size(); I != N; ++I) {
     if (FieldDecl *FoundField = dyn_cast<FieldDecl>(FoundDecls[I])) {
+      // For anonymous fields, match up by index.
+      if (!Name && getFieldIndex(D) != getFieldIndex(FoundField))
+        continue;
+
       if (Importer.IsStructurallyEquivalent(D->getType(), 
                                             FoundField->getType())) {
         Importer.Imported(D, FoundField);
@@ -2642,6 +2818,7 @@ Decl *ASTNodeImporter::VisitFieldDecl(FieldDecl *D) {
   ToField->setLexicalDeclContext(LexicalDC);
   if (ToField->hasInClassInitializer())
     ToField->setInClassInitializer(D->getInClassInitializer());
+  ToField->setImplicit(D->isImplicit());
   Importer.Imported(D, ToField);
   LexicalDC->addDeclInternal(ToField);
   return ToField;
@@ -2661,6 +2838,10 @@ Decl *ASTNodeImporter::VisitIndirectFieldDecl(IndirectFieldDecl *D) {
   for (unsigned I = 0, N = FoundDecls.size(); I != N; ++I) {
     if (IndirectFieldDecl *FoundField 
                                 = dyn_cast<IndirectFieldDecl>(FoundDecls[I])) {
+      // For anonymous indirect fields, match up by index.
+      if (!Name && getFieldIndex(D) != getFieldIndex(FoundField))
+        continue;
+
       if (Importer.IsStructurallyEquivalent(D->getType(), 
                                             FoundField->getType(),
                                             Name)) {
@@ -3025,7 +3206,7 @@ Decl *ASTNodeImporter::VisitObjCMethodDecl(ObjCMethodDecl *D) {
                              ResultTy, ResultTInfo, DC,
                              D->isInstanceMethod(),
                              D->isVariadic(),
-                             D->isSynthesized(),
+                             D->isPropertyAccessor(),
                              D->isImplicit(),
                              D->isDefined(),
                              D->getImplementationControl(),
@@ -4027,7 +4208,8 @@ Expr *ASTNodeImporter::VisitBinaryOperator(BinaryOperator *E) {
   return new (Importer.getToContext()) BinaryOperator(LHS, RHS, E->getOpcode(),
                                                       T, E->getValueKind(),
                                                       E->getObjectKind(),
-                                          Importer.Import(E->getOperatorLoc()));
+                                           Importer.Import(E->getOperatorLoc()),
+                                                      E->isFPContractable());
 }
 
 Expr *ASTNodeImporter::VisitCompoundAssignOperator(CompoundAssignOperator *E) {
@@ -4056,7 +4238,8 @@ Expr *ASTNodeImporter::VisitCompoundAssignOperator(CompoundAssignOperator *E) {
                                                T, E->getValueKind(),
                                                E->getObjectKind(),
                                                CompLHSType, CompResultType,
-                                          Importer.Import(E->getOperatorLoc()));
+                                           Importer.Import(E->getOperatorLoc()),
+                                               E->isFPContractable());
 }
 
 static bool ImportCastPath(CastExpr *E, CXXCastPath &Path) {
@@ -4479,7 +4662,8 @@ FileID ASTImporter::Import(FileID FromID) {
     llvm::MemoryBuffer *ToBuf
       = llvm::MemoryBuffer::getMemBufferCopy(FromBuf->getBuffer(),
                                              FromBuf->getBufferIdentifier());
-    ToID = ToSM.createFileIDForMemBuffer(ToBuf);
+    ToID = ToSM.createFileIDForMemBuffer(ToBuf,
+                                    FromSLoc.getFile().getFileCharacteristic());
   }
   
   
diff --git a/lib/AST/CMakeLists.txt b/lib/AST/CMakeLists.txt
index bcc96f9..d20d77e 100644
--- a/lib/AST/CMakeLists.txt
+++ b/lib/AST/CMakeLists.txt
@@ -64,7 +64,10 @@ add_dependencies(clangAST
   ClangAttrClasses
   ClangAttrList
   ClangAttrImpl
+  ClangCommentCommandInfo
   ClangCommentNodes
+  ClangCommentHTMLTags
+  ClangCommentHTMLTagsProperties
   ClangDeclNodes
   ClangDiagnosticAST
   ClangDiagnosticComment
diff --git a/lib/AST/CXXInheritance.cpp b/lib/AST/CXXInheritance.cpp
index cf3913b..213b214 100644
--- a/lib/AST/CXXInheritance.cpp
+++ b/lib/AST/CXXInheritance.cpp
@@ -14,6 +14,7 @@
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/RecordLayout.h"
 #include "clang/AST/DeclCXX.h"
+#include "llvm/ADT/SetVector.h"
 #include <algorithm>
 #include <set>
 
@@ -25,13 +26,9 @@ void CXXBasePaths::ComputeDeclsFound() {
   assert(NumDeclsFound == 0 && !DeclsFound &&
          "Already computed the set of declarations");
 
-  SmallVector<NamedDecl *, 8> Decls;
+  llvm::SetVector<NamedDecl *, SmallVector<NamedDecl *, 8> > Decls;
   for (paths_iterator Path = begin(), PathEnd = end(); Path != PathEnd; ++Path)
-    Decls.push_back(*Path->Decls.first);
-
-  // Eliminate duplicated decls.
-  llvm::array_pod_sort(Decls.begin(), Decls.end());
-  Decls.erase(std::unique(Decls.begin(), Decls.end()), Decls.end());
+    Decls.insert(*Path->Decls.first);
 
   NumDeclsFound = Decls.size();
   DeclsFound = new NamedDecl * [NumDeclsFound];
@@ -258,7 +255,7 @@ bool CXXBasePaths::lookupInBases(ASTContext &Context,
       }
     } else if (VisitBase) {
       CXXRecordDecl *BaseRecord
-        = cast<CXXRecordDecl>(BaseSpec->getType()->getAs<RecordType>()
+        = cast<CXXRecordDecl>(BaseSpec->getType()->castAs<RecordType>()
                                 ->getDecl());
       if (lookupInBases(Context, BaseRecord, BaseMatches, UserData)) {
         // C++ [class.member.lookup]p2:
@@ -365,8 +362,8 @@ bool CXXRecordDecl::FindBaseClass(const CXXBaseSpecifier *Specifier,
                                   void *BaseRecord) {
   assert(((Decl *)BaseRecord)->getCanonicalDecl() == BaseRecord &&
          "User data for FindBaseClass is not canonical!");
-  return Specifier->getType()->getAs<RecordType>()->getDecl()
-           ->getCanonicalDecl() == BaseRecord;
+  return Specifier->getType()->castAs<RecordType>()->getDecl()
+            ->getCanonicalDecl() == BaseRecord;
 }
 
 bool CXXRecordDecl::FindVirtualBaseClass(const CXXBaseSpecifier *Specifier, 
@@ -375,14 +372,15 @@ bool CXXRecordDecl::FindVirtualBaseClass(const CXXBaseSpecifier *Specifier,
   assert(((Decl *)BaseRecord)->getCanonicalDecl() == BaseRecord &&
          "User data for FindBaseClass is not canonical!");
   return Specifier->isVirtual() &&
-         Specifier->getType()->getAs<RecordType>()->getDecl()
-           ->getCanonicalDecl() == BaseRecord;
+         Specifier->getType()->castAs<RecordType>()->getDecl()
+            ->getCanonicalDecl() == BaseRecord;
 }
 
 bool CXXRecordDecl::FindTagMember(const CXXBaseSpecifier *Specifier, 
                                   CXXBasePath &Path,
                                   void *Name) {
-  RecordDecl *BaseRecord = Specifier->getType()->getAs<RecordType>()->getDecl();
+  RecordDecl *BaseRecord =
+    Specifier->getType()->castAs<RecordType>()->getDecl();
 
   DeclarationName N = DeclarationName::getFromOpaquePtr(Name);
   for (Path.Decls = BaseRecord->lookup(N);
@@ -398,7 +396,8 @@ bool CXXRecordDecl::FindTagMember(const CXXBaseSpecifier *Specifier,
 bool CXXRecordDecl::FindOrdinaryMember(const CXXBaseSpecifier *Specifier, 
                                        CXXBasePath &Path,
                                        void *Name) {
-  RecordDecl *BaseRecord = Specifier->getType()->getAs<RecordType>()->getDecl();
+  RecordDecl *BaseRecord =
+    Specifier->getType()->castAs<RecordType>()->getDecl();
   
   const unsigned IDNS = IDNS_Ordinary | IDNS_Tag | IDNS_Member;
   DeclarationName N = DeclarationName::getFromOpaquePtr(Name);
@@ -416,7 +415,8 @@ bool CXXRecordDecl::
 FindNestedNameSpecifierMember(const CXXBaseSpecifier *Specifier, 
                               CXXBasePath &Path,
                               void *Name) {
-  RecordDecl *BaseRecord = Specifier->getType()->getAs<RecordType>()->getDecl();
+  RecordDecl *BaseRecord =
+    Specifier->getType()->castAs<RecordType>()->getDecl();
   
   DeclarationName N = DeclarationName::getFromOpaquePtr(Name);
   for (Path.Decls = BaseRecord->lookup(N);
@@ -694,7 +694,7 @@ AddIndirectPrimaryBases(const CXXRecordDecl *RD, ASTContext &Context,
            "Cannot get indirect primary bases for class with dependent bases.");
 
     const CXXRecordDecl *BaseDecl =
-      cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
+      cast<CXXRecordDecl>(I->getType()->castAs<RecordType>()->getDecl());
 
     // Only bases with virtual bases participate in computing the
     // indirect primary virtual base classes.
@@ -717,7 +717,7 @@ CXXRecordDecl::getIndirectPrimaryBases(CXXIndirectPrimaryBaseSet& Bases) const {
            "Cannot get indirect primary bases for class with dependent bases.");
 
     const CXXRecordDecl *BaseDecl =
-      cast<CXXRecordDecl>(I->getType()->getAs<RecordType>()->getDecl());
+      cast<CXXRecordDecl>(I->getType()->castAs<RecordType>()->getDecl());
 
     // Only bases with virtual bases participate in computing the
     // indirect primary virtual base classes.
diff --git a/lib/AST/Comment.cpp b/lib/AST/Comment.cpp
index 8a711f0..361f8ac 100644
--- a/lib/AST/Comment.cpp
+++ b/lib/AST/Comment.cpp
@@ -7,6 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "clang/AST/ASTContext.h"
 #include "clang/AST/Comment.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclObjC.h"
@@ -37,11 +38,12 @@ void Comment::dump() const {
   // in CommentDumper.cpp, that object file would be removed by linker because
   // none of its functions are referenced by other object files, despite the
   // LLVM_ATTRIBUTE_USED.
-  dump(llvm::errs(), NULL);
+  dump(llvm::errs(), NULL, NULL);
 }
 
-void Comment::dump(SourceManager &SM) const {
-  dump(llvm::errs(), &SM);
+void Comment::dump(const ASTContext &Context) const {
+  dump(llvm::errs(), &Context.getCommentCommandTraits(),
+       &Context.getSourceManager());
 }
 
 namespace {
@@ -149,13 +151,14 @@ void DeclInfo::fill() {
   ParamVars = ArrayRef<const ParmVarDecl *>();
   TemplateParameters = NULL;
 
-  if (!ThisDecl) {
+  if (!CommentDecl) {
     // If there is no declaration, the defaults is our only guess.
     IsFilled = true;
     return;
   }
-
-  Decl::Kind K = ThisDecl->getKind();
+  CurrentDecl = CommentDecl;
+  
+  Decl::Kind K = CommentDecl->getKind();
   switch (K) {
   default:
     // Defaults are should be good for declarations we don't handle explicitly.
@@ -165,7 +168,7 @@ void DeclInfo::fill() {
   case Decl::CXXConstructor:
   case Decl::CXXDestructor:
   case Decl::CXXConversion: {
-    const FunctionDecl *FD = cast<FunctionDecl>(ThisDecl);
+    const FunctionDecl *FD = cast<FunctionDecl>(CommentDecl);
     Kind = FunctionKind;
     ParamVars = ArrayRef<const ParmVarDecl *>(FD->param_begin(),
                                               FD->getNumParams());
@@ -179,14 +182,14 @@ void DeclInfo::fill() {
 
     if (K == Decl::CXXMethod || K == Decl::CXXConstructor ||
         K == Decl::CXXDestructor || K == Decl::CXXConversion) {
-      const CXXMethodDecl *MD = cast<CXXMethodDecl>(ThisDecl);
+      const CXXMethodDecl *MD = cast<CXXMethodDecl>(CommentDecl);
       IsInstanceMethod = MD->isInstance();
       IsClassMethod = !IsInstanceMethod;
     }
     break;
   }
   case Decl::ObjCMethod: {
-    const ObjCMethodDecl *MD = cast<ObjCMethodDecl>(ThisDecl);
+    const ObjCMethodDecl *MD = cast<ObjCMethodDecl>(CommentDecl);
     Kind = FunctionKind;
     ParamVars = ArrayRef<const ParmVarDecl *>(MD->param_begin(),
                                               MD->param_size());
@@ -197,7 +200,7 @@ void DeclInfo::fill() {
     break;
   }
   case Decl::FunctionTemplate: {
-    const FunctionTemplateDecl *FTD = cast<FunctionTemplateDecl>(ThisDecl);
+    const FunctionTemplateDecl *FTD = cast<FunctionTemplateDecl>(CommentDecl);
     Kind = FunctionKind;
     TemplateKind = Template;
     const FunctionDecl *FD = FTD->getTemplatedDecl();
@@ -208,7 +211,7 @@ void DeclInfo::fill() {
     break;
   }
   case Decl::ClassTemplate: {
-    const ClassTemplateDecl *CTD = cast<ClassTemplateDecl>(ThisDecl);
+    const ClassTemplateDecl *CTD = cast<ClassTemplateDecl>(CommentDecl);
     Kind = ClassKind;
     TemplateKind = Template;
     TemplateParameters = CTD->getTemplateParameters();
@@ -216,7 +219,7 @@ void DeclInfo::fill() {
   }
   case Decl::ClassTemplatePartialSpecialization: {
     const ClassTemplatePartialSpecializationDecl *CTPSD =
-        cast<ClassTemplatePartialSpecializationDecl>(ThisDecl);
+        cast<ClassTemplatePartialSpecializationDecl>(CommentDecl);
     Kind = ClassKind;
     TemplateKind = TemplatePartialSpecialization;
     TemplateParameters = CTPSD->getTemplateParameters();
@@ -240,12 +243,55 @@ void DeclInfo::fill() {
   case Decl::Namespace:
     Kind = NamespaceKind;
     break;
-  case Decl::Typedef:
+  case Decl::Typedef: {
+    Kind = TypedefKind;
+    // If this is a typedef to something we consider a function, extract
+    // arguments and return type.
+    const TypedefDecl *TD = cast<TypedefDecl>(CommentDecl);
+    const TypeSourceInfo *TSI = TD->getTypeSourceInfo();
+    if (!TSI)
+      break;
+    TypeLoc TL = TSI->getTypeLoc().getUnqualifiedLoc();
+    while (true) {
+      TL = TL.IgnoreParens();
+      // Look through qualified types.
+      if (QualifiedTypeLoc *QualifiedTL = dyn_cast<QualifiedTypeLoc>(&TL)) {
+        TL = QualifiedTL->getUnqualifiedLoc();
+        continue;
+      }
+      // Look through pointer types.
+      if (PointerTypeLoc *PointerTL = dyn_cast<PointerTypeLoc>(&TL)) {
+        TL = PointerTL->getPointeeLoc().getUnqualifiedLoc();
+        continue;
+      }
+      if (BlockPointerTypeLoc *BlockPointerTL =
+              dyn_cast<BlockPointerTypeLoc>(&TL)) {
+        TL = BlockPointerTL->getPointeeLoc().getUnqualifiedLoc();
+        continue;
+      }
+      if (MemberPointerTypeLoc *MemberPointerTL =
+              dyn_cast<MemberPointerTypeLoc>(&TL)) {
+        TL = MemberPointerTL->getPointeeLoc().getUnqualifiedLoc();
+        continue;
+      }
+      // Is this a typedef for a function type?
+      if (FunctionTypeLoc *FTL = dyn_cast<FunctionTypeLoc>(&TL)) {
+        Kind = FunctionKind;
+        ArrayRef<ParmVarDecl *> Params = FTL->getParams();
+        ParamVars = ArrayRef<const ParmVarDecl *>(Params.data(),
+                                                  Params.size());
+        ResultType = FTL->getResultLoc().getType();
+        break;
+      }
+      break;
+    }
+    break;
+  }
   case Decl::TypeAlias:
     Kind = TypedefKind;
     break;
   case Decl::TypeAliasTemplate: {
-    const TypeAliasTemplateDecl *TAT = cast<TypeAliasTemplateDecl>(ThisDecl);
+    const TypeAliasTemplateDecl *TAT = cast<TypeAliasTemplateDecl>(CommentDecl);
     Kind = TypedefKind;
     TemplateKind = Template;
     TemplateParameters = TAT->getTemplateParameters();
@@ -259,6 +305,25 @@ void DeclInfo::fill() {
   IsFilled = true;
 }
 
+StringRef ParamCommandComment::getParamName(const FullComment *FC) const {
+  assert(isParamIndexValid());
+  return FC->getThisDeclInfo()->ParamVars[getParamIndex()]->getName();
+}
+
+StringRef TParamCommandComment::getParamName(const FullComment *FC) const {
+  assert(isPositionValid());
+  const TemplateParameterList *TPL = FC->getThisDeclInfo()->TemplateParameters;
+  for (unsigned i = 0, e = getDepth(); i != e; ++i) {
+    if (i == e-1)
+      return TPL->getParam(getIndex(i))->getName();
+    const NamedDecl *Param = TPL->getParam(getIndex(i));
+    if (const TemplateTemplateParmDecl *TTP =
+          dyn_cast<TemplateTemplateParmDecl>(Param))
+      TPL = TTP->getTemplateParameters();
+  }
+  return "";
+}
+
 } // end namespace comments
 } // end namespace clang
 
diff --git a/lib/AST/CommentBriefParser.cpp b/lib/AST/CommentBriefParser.cpp
index 0aebc1e..95daa7e 100644
--- a/lib/AST/CommentBriefParser.cpp
+++ b/lib/AST/CommentBriefParser.cpp
@@ -15,6 +15,11 @@ namespace clang {
 namespace comments {
 
 namespace {
+inline bool isWhitespace(char C) {
+  return C == ' ' || C == '\n' || C == '\r' ||
+         C == '\t' || C == '\f' || C == '\v';
+}
+
 /// Convert all whitespace into spaces, remove leading and trailing spaces,
 /// compress multiple spaces into one.
 void cleanupBrief(std::string &S) {
@@ -23,8 +28,7 @@ void cleanupBrief(std::string &S) {
   for (std::string::iterator I = S.begin(), E = S.end();
        I != E; ++I) {
     const char C = *I;
-    if (C == ' ' || C == '\n' || C == '\r' ||
-        C == '\t' || C == '\v' || C == '\f') {
+    if (isWhitespace(C)) {
       if (!PrevWasSpace) {
         *O++ = ' ';
         PrevWasSpace = true;
@@ -40,6 +44,15 @@ void cleanupBrief(std::string &S) {
 
   S.resize(O - S.begin());
 }
+
+bool isWhitespace(StringRef Text) {
+  for (StringRef::const_iterator I = Text.begin(), E = Text.end();
+       I != E; ++I) {
+    if (!isWhitespace(*I))
+      return false;
+  }
+  return true;
+}
 } // unnamed namespace
 
 BriefParser::BriefParser(Lexer &L, const CommandTraits &Traits) :
@@ -66,19 +79,23 @@ std::string BriefParser::Parse() {
     }
 
     if (Tok.is(tok::command)) {
-      StringRef Name = Tok.getCommandName();
-      if (Traits.isBriefCommand(Name)) {
+      const CommandInfo *Info = Traits.getCommandInfo(Tok.getCommandID());
+      if (Info->IsBriefCommand) {
         FirstParagraphOrBrief.clear();
         InBrief = true;
         ConsumeToken();
         continue;
       }
-      if (Traits.isReturnsCommand(Name)) {
+      if (Info->IsReturnsCommand) {
         InReturns = true;
+        InBrief = false;
+        InFirstParagraph = false;
         ReturnsParagraph += "Returns ";
+        ConsumeToken();
+        continue;
       }
       // Block commands implicitly start a new paragraph.
-      if (Traits.isBlockCommand(Name)) {
+      if (Info->IsBlockCommand) {
         // We found an implicit paragraph end.
         InFirstParagraph = false;
         if (InBrief)
@@ -93,13 +110,29 @@ std::string BriefParser::Parse() {
         ReturnsParagraph += ' ';
       ConsumeToken();
 
+      // If the next token is a whitespace only text, ignore it.  Thus we allow
+      // two paragraphs to be separated by line that has only whitespace in it.
+      //
+      // We don't need to add a space to the parsed text because we just added
+      // a space for the newline.
+      if (Tok.is(tok::text)) {
+        if (isWhitespace(Tok.getText()))
+          ConsumeToken();
+      }
+
       if (Tok.is(tok::newline)) {
         ConsumeToken();
-        // We found a paragraph end.
-        InFirstParagraph = false;
-        InReturns = false;
+        // We found a paragraph end.  This ends the brief description if
+        // \\brief command or its equivalent was explicitly used.
+        // Stop scanning text because an explicit \\brief paragraph is the
+        // preffered one.
         if (InBrief)
           break;
+        // End first paragraph if we found some non-whitespace text.
+        if (InFirstParagraph && !isWhitespace(FirstParagraphOrBrief))
+          InFirstParagraph = false;
+        // End the \\returns paragraph because we found the paragraph end.
+        InReturns = false;
       }
       continue;
     }
diff --git a/lib/AST/CommentCommandTraits.cpp b/lib/AST/CommentCommandTraits.cpp
index dc7a0bd..e7e40fd 100644
--- a/lib/AST/CommentCommandTraits.cpp
+++ b/lib/AST/CommentCommandTraits.cpp
@@ -8,125 +8,64 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/AST/CommentCommandTraits.h"
-#include "llvm/ADT/StringSwitch.h"
+#include "llvm/ADT/STLExtras.h"
 
 namespace clang {
 namespace comments {
 
-// TODO: tablegen
+#include "clang/AST/CommentCommandInfo.inc"
 
-bool CommandTraits::isVerbatimBlockCommand(StringRef StartName,
-                                           StringRef &EndName) const {
-  const char *Result = llvm::StringSwitch<const char *>(StartName)
-    .Case("code", "endcode")
-    .Case("verbatim", "endverbatim")
-    .Case("htmlonly", "endhtmlonly")
-    .Case("latexonly", "endlatexonly")
-    .Case("xmlonly", "endxmlonly")
-    .Case("manonly", "endmanonly")
-    .Case("rtfonly", "endrtfonly")
+CommandTraits::CommandTraits(llvm::BumpPtrAllocator &Allocator) :
+    NextID(llvm::array_lengthof(Commands)), Allocator(Allocator)
+{ }
 
-    .Case("dot", "enddot")
-    .Case("msc", "endmsc")
-
-    .Case("f$", "f$") // Inline LaTeX formula
-    .Case("f[", "f]") // Displayed LaTeX formula
-    .Case("f{", "f}") // LaTeX environment
-
-    .Default(NULL);
-
-  if (Result) {
-    EndName = Result;
-    return true;
-  }
-
-  for (VerbatimBlockCommandVector::const_iterator
-           I = VerbatimBlockCommands.begin(),
-           E = VerbatimBlockCommands.end();
-       I != E; ++I)
-    if (I->StartName == StartName) {
-      EndName = I->EndName;
-      return true;
-    }
-
-  return false;
+const CommandInfo *CommandTraits::getCommandInfoOrNULL(StringRef Name) const {
+  if (const CommandInfo *Info = getBuiltinCommandInfo(Name))
+    return Info;
+  return getRegisteredCommandInfo(Name);
 }
 
-bool CommandTraits::isVerbatimLineCommand(StringRef Name) const {
-  bool Result = isDeclarationCommand(Name) || llvm::StringSwitch<bool>(Name)
-  .Case("defgroup", true)
-  .Case("ingroup", true)
-  .Case("addtogroup", true)
-  .Case("weakgroup", true)
-  .Case("name", true)
-
-  .Case("section", true)
-  .Case("subsection", true)
-  .Case("subsubsection", true)
-  .Case("paragraph", true)
-
-  .Case("mainpage", true)
-  .Case("subpage", true)
-  .Case("ref", true)
+const CommandInfo *CommandTraits::getCommandInfo(unsigned CommandID) const {
+  if (const CommandInfo *Info = getBuiltinCommandInfo(CommandID))
+    return Info;
+  return getRegisteredCommandInfo(CommandID);
+}
 
-  .Default(false);
+const CommandInfo *CommandTraits::registerUnknownCommand(StringRef CommandName) {
+  char *Name = Allocator.Allocate<char>(CommandName.size() + 1);
+  memcpy(Name, CommandName.data(), CommandName.size());
+  Name[CommandName.size()] = '\0';
 
-  if (Result)
-    return true;
+  // Value-initialize (=zero-initialize in this case) a new CommandInfo.
+  CommandInfo *Info = new (Allocator) CommandInfo();
+  Info->Name = Name;
+  Info->ID = NextID++;
+  Info->IsUnknownCommand = true;
 
-  for (VerbatimLineCommandVector::const_iterator
-           I = VerbatimLineCommands.begin(),
-           E = VerbatimLineCommands.end();
-       I != E; ++I)
-    if (I->Name == Name)
-      return true;
+  RegisteredCommands.push_back(Info);
 
-  return false;
+  return Info;
 }
 
-bool CommandTraits::isDeclarationCommand(StringRef Name) const {
-  return llvm::StringSwitch<bool>(Name)
-      // Doxygen commands.
-      .Case("fn", true)
-      .Case("var", true)
-      .Case("property", true)
-      .Case("typedef", true)
-
-      .Case("overload", true)
-
-      // HeaderDoc commands.
-      .Case("class", true)
-      .Case("interface", true)
-      .Case("protocol", true)
-      .Case("category", true)
-      .Case("template", true)
-      .Case("function", true)
-      .Case("method", true)
-      .Case("callback", true)
-      .Case("var", true)
-      .Case("const", true)
-      .Case("constant", true)
-      .Case("property", true)
-      .Case("struct", true)
-      .Case("union", true)
-      .Case("typedef", true)
-      .Case("enum", true)
-
-      .Default(false);
+const CommandInfo *CommandTraits::getBuiltinCommandInfo(
+                                                  unsigned CommandID) {
+  if (CommandID < llvm::array_lengthof(Commands))
+    return &Commands[CommandID];
+  return NULL;
 }
 
-void CommandTraits::addVerbatimBlockCommand(StringRef StartName,
-                                            StringRef EndName) {
-  VerbatimBlockCommand VBC;
-  VBC.StartName = StartName;
-  VBC.EndName = EndName;
-  VerbatimBlockCommands.push_back(VBC);
+const CommandInfo *CommandTraits::getRegisteredCommandInfo(
+                                                  StringRef Name) const {
+  for (unsigned i = 0, e = RegisteredCommands.size(); i != e; ++i) {
+    if (RegisteredCommands[i]->Name == Name)
+      return RegisteredCommands[i];
+  }
+  return NULL;
 }
 
-void CommandTraits::addVerbatimLineCommand(StringRef Name) {
-  VerbatimLineCommand VLC;
-  VLC.Name = Name;
-  VerbatimLineCommands.push_back(VLC);
+const CommandInfo *CommandTraits::getRegisteredCommandInfo(
+                                                  unsigned CommandID) const {
+  return RegisteredCommands[CommandID - llvm::array_lengthof(Commands)];
 }
 
 } // end namespace comments
diff --git a/lib/AST/CommentDumper.cpp b/lib/AST/CommentDumper.cpp
index dffc823..19d24b2 100644
--- a/lib/AST/CommentDumper.cpp
+++ b/lib/AST/CommentDumper.cpp
@@ -16,12 +16,20 @@ namespace comments {
 namespace {
 class CommentDumper: public comments::ConstCommentVisitor<CommentDumper> {
   raw_ostream &OS;
-  SourceManager *SM;
+  const CommandTraits *Traits;
+  const SourceManager *SM;
+
+  /// The \c FullComment parent of the comment being dumped.
+  const FullComment *FC;
+
   unsigned IndentLevel;
 
 public:
-  CommentDumper(raw_ostream &OS, SourceManager *SM) :
-      OS(OS), SM(SM), IndentLevel(0)
+  CommentDumper(raw_ostream &OS,
+                const CommandTraits *Traits,
+                const SourceManager *SM,
+                const FullComment *FC) :
+      OS(OS), Traits(Traits), SM(SM), FC(FC), IndentLevel(0)
   { }
 
   void dumpIndent() const {
@@ -56,6 +64,15 @@ public:
   void visitVerbatimLineComment(const VerbatimLineComment *C);
 
   void visitFullComment(const FullComment *C);
+
+  const char *getCommandName(unsigned CommandID) {
+    if (Traits)
+      return Traits->getCommandInfo(CommandID)->Name;
+    const CommandInfo *Info = CommandTraits::getBuiltinCommandInfo(CommandID);
+    if (Info)
+      return Info->Name;
+    return "<not a builtin command>";
+  }
 };
 
 void CommentDumper::dumpSourceRange(const Comment *C) {
@@ -107,7 +124,7 @@ void CommentDumper::visitTextComment(const TextComment *C) {
 void CommentDumper::visitInlineCommandComment(const InlineCommandComment *C) {
   dumpComment(C);
 
-  OS << " Name=\"" << C->getCommandName() << "\"";
+  OS << " Name=\"" << getCommandName(C->getCommandID()) << "\"";
   switch (C->getRenderKind()) {
   case InlineCommandComment::RenderNormal:
     OS << " RenderNormal";
@@ -155,7 +172,7 @@ void CommentDumper::visitParagraphComment(const ParagraphComment *C) {
 void CommentDumper::visitBlockCommandComment(const BlockCommandComment *C) {
   dumpComment(C);
 
-  OS << " Name=\"" << C->getCommandName() << "\"";
+  OS << " Name=\"" << getCommandName(C->getCommandID()) << "\"";
   for (unsigned i = 0, e = C->getNumArgs(); i != e; ++i)
     OS << " Arg[" << i << "]=\"" << C->getArgText(i) << "\"";
 }
@@ -170,8 +187,12 @@ void CommentDumper::visitParamCommandComment(const ParamCommandComment *C) {
   else
     OS << " implicitly";
 
-  if (C->hasParamName())
-    OS << " Param=\"" << C->getParamName() << "\"";
+  if (C->hasParamName()) {
+    if (C->isParamIndexValid())
+      OS << " Param=\"" << C->getParamName(FC) << "\"";
+    else
+      OS << " Param=\"" << C->getParamNameAsWritten() << "\"";
+  }
 
   if (C->isParamIndexValid())
     OS << " ParamIndex=" << C->getParamIndex();
@@ -181,7 +202,10 @@ void CommentDumper::visitTParamCommandComment(const TParamCommandComment *C) {
   dumpComment(C);
 
   if (C->hasParamName()) {
-    OS << " Param=\"" << C->getParamName() << "\"";
+    if (C->isPositionValid())
+      OS << " Param=\"" << C->getParamName(FC) << "\"";
+    else
+      OS << " Param=\"" << C->getParamNameAsWritten() << "\"";
   }
 
   if (C->isPositionValid()) {
@@ -198,7 +222,7 @@ void CommentDumper::visitTParamCommandComment(const TParamCommandComment *C) {
 void CommentDumper::visitVerbatimBlockComment(const VerbatimBlockComment *C) {
   dumpComment(C);
 
-  OS << " Name=\"" << C->getCommandName() << "\""
+  OS << " Name=\"" << getCommandName(C->getCommandID()) << "\""
         " CloseName=\"" << C->getCloseName() << "\"";
 }
 
@@ -220,8 +244,10 @@ void CommentDumper::visitFullComment(const FullComment *C) {
 
 } // unnamed namespace
 
-void Comment::dump(llvm::raw_ostream &OS, SourceManager *SM) const {
-  CommentDumper D(llvm::errs(), SM);
+void Comment::dump(llvm::raw_ostream &OS, const CommandTraits *Traits,
+                   const SourceManager *SM) const {
+  const FullComment *FC = dyn_cast<FullComment>(this);
+  CommentDumper D(llvm::errs(), Traits, SM, FC);
   D.dumpSubtree(this);
   llvm::errs() << '\n';
 }
diff --git a/lib/AST/CommentLexer.cpp b/lib/AST/CommentLexer.cpp
index b6516ec..31a09f7 100644
--- a/lib/AST/CommentLexer.cpp
+++ b/lib/AST/CommentLexer.cpp
@@ -28,6 +28,9 @@ bool isHTMLHexCharacterReferenceCharacter(char C) {
          (C >= 'a' && C <= 'f') ||
          (C >= 'A' && C <= 'F');
 }
+
+#include "clang/AST/CommentHTMLTags.inc"
+
 } // unnamed namespace
 
 StringRef Lexer::resolveHTMLNamedCharacterReference(StringRef Name) const {
@@ -223,6 +226,11 @@ bool isWhitespace(const char *BufferPtr, const char *BufferEnd) {
   return skipWhitespace(BufferPtr, BufferEnd) == BufferEnd;
 }
 
+bool isCommandNameStartCharacter(char C) {
+  return (C >= 'a' && C <= 'z') ||
+         (C >= 'A' && C <= 'Z');
+}
+
 bool isCommandNameCharacter(char C) {
   return (C >= 'a' && C <= 'z') ||
          (C >= 'A' && C <= 'Z') ||
@@ -337,7 +345,7 @@ void Lexer::lexCommentText(Token &T) {
         }
 
         // Don't make zero-length commands.
-        if (!isCommandNameCharacter(*TokenPtr)) {
+        if (!isCommandNameStartCharacter(*TokenPtr)) {
           formTextToken(T, TokenPtr);
           return;
         }
@@ -356,18 +364,23 @@ void Lexer::lexCommentText(Token &T) {
         }
 
         const StringRef CommandName(BufferPtr + 1, Length);
-        StringRef EndName;
 
-        if (Traits.isVerbatimBlockCommand(CommandName, EndName)) {
-          setupAndLexVerbatimBlock(T, TokenPtr, *BufferPtr, EndName);
+        const CommandInfo *Info = Traits.getCommandInfoOrNULL(CommandName);
+        if (!Info) {
+          formTokenWithChars(T, TokenPtr, tok::unknown_command);
+          T.setUnknownCommandName(CommandName);
+          return;
+        }
+        if (Info->IsVerbatimBlockCommand) {
+          setupAndLexVerbatimBlock(T, TokenPtr, *BufferPtr, Info);
           return;
         }
-        if (Traits.isVerbatimLineCommand(CommandName)) {
-          setupAndLexVerbatimLine(T, TokenPtr);
+        if (Info->IsVerbatimLineCommand) {
+          setupAndLexVerbatimLine(T, TokenPtr, Info);
           return;
         }
         formTokenWithChars(T, TokenPtr, tok::command);
-        T.setCommandName(CommandName);
+        T.setCommandID(Info->getID());
         return;
       }
 
@@ -420,14 +433,15 @@ void Lexer::lexCommentText(Token &T) {
 
 void Lexer::setupAndLexVerbatimBlock(Token &T,
                                      const char *TextBegin,
-                                     char Marker, StringRef EndName) {
+                                     char Marker, const CommandInfo *Info) {
+  assert(Info->IsVerbatimBlockCommand);
+
   VerbatimBlockEndCommandName.clear();
   VerbatimBlockEndCommandName.append(Marker == '\\' ? "\\" : "@");
-  VerbatimBlockEndCommandName.append(EndName);
+  VerbatimBlockEndCommandName.append(Info->EndCommandName);
 
-  StringRef Name(BufferPtr + 1, TextBegin - (BufferPtr + 1));
   formTokenWithChars(T, TextBegin, tok::verbatim_block_begin);
-  T.setVerbatimBlockName(Name);
+  T.setVerbatimBlockID(Info->getID());
 
   // If there is a newline following the verbatim opening command, skip the
   // newline so that we don't create an tok::verbatim_block_line with empty
@@ -468,7 +482,7 @@ again:
     const char *End = BufferPtr + VerbatimBlockEndCommandName.size();
     StringRef Name(BufferPtr + 1, End - (BufferPtr + 1));
     formTokenWithChars(T, End, tok::verbatim_block_end);
-    T.setVerbatimBlockName(Name);
+    T.setVerbatimBlockID(Traits.getCommandInfo(Name)->getID());
     State = LS_Normal;
     return;
   } else {
@@ -498,10 +512,11 @@ void Lexer::lexVerbatimBlockBody(Token &T) {
   lexVerbatimBlockFirstLine(T);
 }
 
-void Lexer::setupAndLexVerbatimLine(Token &T, const char *TextBegin) {
-  const StringRef Name(BufferPtr + 1, TextBegin - BufferPtr - 1);
+void Lexer::setupAndLexVerbatimLine(Token &T, const char *TextBegin,
+                                    const CommandInfo *Info) {
+  assert(Info->IsVerbatimLineCommand);
   formTokenWithChars(T, TextBegin, tok::verbatim_line_name);
-  T.setVerbatimLineName(Name);
+  T.setVerbatimLineID(Info->getID());
 
   State = LS_VerbatimLineText;
 }
@@ -585,8 +600,12 @@ void Lexer::setupAndLexHTMLStartTag(Token &T) {
   assert(BufferPtr[0] == '<' &&
          isHTMLIdentifierStartingCharacter(BufferPtr[1]));
   const char *TagNameEnd = skipHTMLIdentifier(BufferPtr + 2, CommentEnd);
-
   StringRef Name(BufferPtr + 1, TagNameEnd - (BufferPtr + 1));
+  if (!isHTMLTagName(Name)) {
+    formTextToken(T, TagNameEnd);
+    return;
+  }
+
   formTokenWithChars(T, TagNameEnd, tok::html_start_tag);
   T.setHTMLTagStartName(Name);
 
@@ -665,11 +684,16 @@ void Lexer::setupAndLexHTMLEndTag(Token &T) {
 
   const char *TagNameBegin = skipWhitespace(BufferPtr + 2, CommentEnd);
   const char *TagNameEnd = skipHTMLIdentifier(TagNameBegin, CommentEnd);
+  StringRef Name(TagNameBegin, TagNameEnd - TagNameBegin);
+  if (!isHTMLTagName(Name)) {
+    formTextToken(T, TagNameEnd);
+    return;
+  }
 
   const char *End = skipWhitespace(TagNameEnd, CommentEnd);
 
   formTokenWithChars(T, End, tok::html_end_tag);
-  T.setHTMLTagEndName(StringRef(TagNameBegin, TagNameEnd - TagNameBegin));
+  T.setHTMLTagEndName(Name);
 
   if (BufferPtr != CommentEnd && *BufferPtr == '>')
     State = LS_HTMLEndTag;
@@ -683,11 +707,11 @@ void Lexer::lexHTMLEndTag(Token &T) {
 }
 
 Lexer::Lexer(llvm::BumpPtrAllocator &Allocator, const CommandTraits &Traits,
-             SourceLocation FileLoc, const CommentOptions &CommOpts,
+             SourceLocation FileLoc,
              const char *BufferStart, const char *BufferEnd):
     Allocator(Allocator), Traits(Traits),
     BufferStart(BufferStart), BufferEnd(BufferEnd),
-    FileLoc(FileLoc), CommOpts(CommOpts), BufferPtr(BufferStart),
+    FileLoc(FileLoc), BufferPtr(BufferStart),
     CommentState(LCS_BeforeComment), State(LS_Normal) {
 }
 
diff --git a/lib/AST/CommentParser.cpp b/lib/AST/CommentParser.cpp
index 43abf6a..d0a8474 100644
--- a/lib/AST/CommentParser.cpp
+++ b/lib/AST/CommentParser.cpp
@@ -132,8 +132,8 @@ class TextTokenRetokenizer {
     Result.setKind(tok::text);
     Result.setLength(TokLength);
 #ifndef NDEBUG
-    Result.TextPtr1 = "<UNSET>";
-    Result.TextLen1 = 7;
+    Result.TextPtr = "<UNSET>";
+    Result.IntVal = 7;
 #endif
     Result.setText(Text);
   }
@@ -312,26 +312,26 @@ BlockCommandComment *Parser::parseBlockCommand() {
   BlockCommandComment *BC;
   bool IsParam = false;
   bool IsTParam = false;
-  unsigned NumArgs = 0;
-  if (Traits.isParamCommand(Tok.getCommandName())) {
+  const CommandInfo *Info = Traits.getCommandInfo(Tok.getCommandID());
+  if (Info->IsParamCommand) {
     IsParam = true;
     PC = S.actOnParamCommandStart(Tok.getLocation(),
                                   Tok.getEndLocation(),
-                                  Tok.getCommandName());
-  } if (Traits.isTParamCommand(Tok.getCommandName())) {
+                                  Tok.getCommandID());
+  } if (Info->IsTParamCommand) {
     IsTParam = true;
     TPC = S.actOnTParamCommandStart(Tok.getLocation(),
                                     Tok.getEndLocation(),
-                                    Tok.getCommandName());
+                                    Tok.getCommandID());
   } else {
-    NumArgs = Traits.getBlockCommandNumArgs(Tok.getCommandName());
     BC = S.actOnBlockCommandStart(Tok.getLocation(),
                                   Tok.getEndLocation(),
-                                  Tok.getCommandName());
+                                  Tok.getCommandID());
   }
   consumeToken();
 
-  if (Tok.is(tok::command) && Traits.isBlockCommand(Tok.getCommandName())) {
+  if (Tok.is(tok::command) &&
+      Traits.getCommandInfo(Tok.getCommandID())->IsBlockCommand) {
     // Block command ahead.  We can't nest block commands, so pretend that this
     // command has an empty argument.
     ParagraphComment *Paragraph = S.actOnParagraphComment(
@@ -348,7 +348,7 @@ BlockCommandComment *Parser::parseBlockCommand() {
     }
   }
 
-  if (IsParam || IsTParam || NumArgs > 0) {
+  if (IsParam || IsTParam || Info->NumArgs > 0) {
     // In order to parse command arguments we need to retokenize a few
     // following text tokens.
     TextTokenRetokenizer Retokenizer(Allocator, *this);
@@ -358,7 +358,7 @@ BlockCommandComment *Parser::parseBlockCommand() {
     else if (IsTParam)
       parseTParamCommandArgs(TPC, Retokenizer);
     else
-      parseBlockCommandArgs(BC, Retokenizer, NumArgs);
+      parseBlockCommandArgs(BC, Retokenizer, Info->NumArgs);
 
     Retokenizer.putBackLeftoverTokens();
   }
@@ -394,14 +394,14 @@ InlineCommandComment *Parser::parseInlineCommand() {
   if (ArgTokValid) {
     IC = S.actOnInlineCommand(CommandTok.getLocation(),
                               CommandTok.getEndLocation(),
-                              CommandTok.getCommandName(),
+                              CommandTok.getCommandID(),
                               ArgTok.getLocation(),
                               ArgTok.getEndLocation(),
                               ArgTok.getText());
   } else {
     IC = S.actOnInlineCommand(CommandTok.getLocation(),
                               CommandTok.getEndLocation(),
-                              CommandTok.getCommandName());
+                              CommandTok.getCommandID());
   }
 
   Retokenizer.putBackLeftoverTokens();
@@ -540,23 +540,39 @@ BlockContentComment *Parser::parseParagraphOrBlockCommand() {
       assert(Content.size() != 0);
       break; // Block content or EOF ahead, finish this parapgaph.
 
-    case tok::command:
-      if (Traits.isBlockCommand(Tok.getCommandName())) {
+    case tok::unknown_command:
+      Content.push_back(S.actOnUnknownCommand(Tok.getLocation(),
+                                              Tok.getEndLocation(),
+                                              Tok.getUnknownCommandName()));
+      consumeToken();
+      continue;
+
+    case tok::command: {
+      const CommandInfo *Info = Traits.getCommandInfo(Tok.getCommandID());
+      if (Info->IsBlockCommand) {
         if (Content.size() == 0)
           return parseBlockCommand();
         break; // Block command ahead, finish this parapgaph.
       }
-      if (Traits.isInlineCommand(Tok.getCommandName())) {
-        Content.push_back(parseInlineCommand());
+      if (Info->IsVerbatimBlockEndCommand) {
+        Diag(Tok.getLocation(),
+             diag::warn_verbatim_block_end_without_start)
+          << Info->Name
+          << SourceRange(Tok.getLocation(), Tok.getEndLocation());
+        consumeToken();
         continue;
       }
-
-      // Not a block command, not an inline command ==> an unknown command.
-      Content.push_back(S.actOnUnknownCommand(Tok.getLocation(),
-                                              Tok.getEndLocation(),
-                                              Tok.getCommandName()));
-      consumeToken();
+      if (Info->IsUnknownCommand) {
+        Content.push_back(S.actOnUnknownCommand(Tok.getLocation(),
+                                                Tok.getEndLocation(),
+                                                Info->getID()));
+        consumeToken();
+        continue;
+      }
+      assert(Info->IsInlineCommand);
+      Content.push_back(parseInlineCommand());
       continue;
+    }
 
     case tok::newline: {
       consumeToken();
@@ -606,7 +622,7 @@ VerbatimBlockComment *Parser::parseVerbatimBlock() {
 
   VerbatimBlockComment *VB =
       S.actOnVerbatimBlockStart(Tok.getLocation(),
-                                Tok.getVerbatimBlockName());
+                                Tok.getVerbatimBlockID());
   consumeToken();
 
   // Don't create an empty line if verbatim opening command is followed
@@ -634,8 +650,9 @@ VerbatimBlockComment *Parser::parseVerbatimBlock() {
   }
 
   if (Tok.is(tok::verbatim_block_end)) {
+    const CommandInfo *Info = Traits.getCommandInfo(Tok.getVerbatimBlockID());
     S.actOnVerbatimBlockFinish(VB, Tok.getLocation(),
-                               Tok.getVerbatimBlockName(),
+                               Info->Name,
                                S.copyArray(llvm::makeArrayRef(Lines)));
     consumeToken();
   } else {
@@ -666,7 +683,7 @@ VerbatimLineComment *Parser::parseVerbatimLine() {
   }
 
   VerbatimLineComment *VL = S.actOnVerbatimLine(NameTok.getLocation(),
-                                                NameTok.getVerbatimLineName(),
+                                                NameTok.getVerbatimLineID(),
                                                 TextBegin,
                                                 Text);
   consumeToken();
@@ -676,6 +693,7 @@ VerbatimLineComment *Parser::parseVerbatimLine() {
 BlockContentComment *Parser::parseBlockContent() {
   switch (Tok.getKind()) {
   case tok::text:
+  case tok::unknown_command:
   case tok::command:
   case tok::html_start_tag:
   case tok::html_end_tag:
diff --git a/lib/AST/CommentSema.cpp b/lib/AST/CommentSema.cpp
index c39ee57..08ecb3a 100644
--- a/lib/AST/CommentSema.cpp
+++ b/lib/AST/CommentSema.cpp
@@ -13,15 +13,22 @@
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclTemplate.h"
 #include "clang/Basic/SourceManager.h"
+#include "clang/Lex/Preprocessor.h"
 #include "llvm/ADT/StringSwitch.h"
+#include "llvm/ADT/SmallString.h"
 
 namespace clang {
 namespace comments {
 
+namespace {
+#include "clang/AST/CommentHTMLTagsProperties.inc"
+} // unnamed namespace
+
 Sema::Sema(llvm::BumpPtrAllocator &Allocator, const SourceManager &SourceMgr,
-           DiagnosticsEngine &Diags, const CommandTraits &Traits) :
+           DiagnosticsEngine &Diags, CommandTraits &Traits,
+           const Preprocessor *PP) :
     Allocator(Allocator), SourceMgr(SourceMgr), Diags(Diags), Traits(Traits),
-    ThisDeclInfo(NULL), BriefCommand(NULL), ReturnsCommand(NULL) {
+    PP(PP), ThisDeclInfo(NULL), BriefCommand(NULL), ReturnsCommand(NULL) {
 }
 
 void Sema::setDecl(const Decl *D) {
@@ -29,7 +36,7 @@ void Sema::setDecl(const Decl *D) {
     return;
 
   ThisDeclInfo = new (Allocator) DeclInfo;
-  ThisDeclInfo->ThisDecl = D;
+  ThisDeclInfo->CommentDecl = D;
   ThisDeclInfo->IsFilled = false;
 }
 
@@ -40,8 +47,8 @@ ParagraphComment *Sema::actOnParagraphComment(
 
 BlockCommandComment *Sema::actOnBlockCommandStart(SourceLocation LocBegin,
                                                   SourceLocation LocEnd,
-                                                  StringRef Name) {
-  return new (Allocator) BlockCommandComment(LocBegin, LocEnd, Name);
+                                                  unsigned CommandID) {
+  return new (Allocator) BlockCommandComment(LocBegin, LocEnd, CommandID);
 }
 
 void Sema::actOnBlockCommandArgs(BlockCommandComment *Command,
@@ -55,18 +62,19 @@ void Sema::actOnBlockCommandFinish(BlockCommandComment *Command,
   checkBlockCommandEmptyParagraph(Command);
   checkBlockCommandDuplicate(Command);
   checkReturnsCommand(Command);
+  checkDeprecatedCommand(Command);
 }
 
 ParamCommandComment *Sema::actOnParamCommandStart(SourceLocation LocBegin,
                                                   SourceLocation LocEnd,
-                                                  StringRef Name) {
+                                                  unsigned CommandID) {
   ParamCommandComment *Command =
-      new (Allocator) ParamCommandComment(LocBegin, LocEnd, Name);
+      new (Allocator) ParamCommandComment(LocBegin, LocEnd, CommandID);
 
   if (!isFunctionDecl())
     Diag(Command->getLocation(),
          diag::warn_doc_param_not_attached_to_a_function_decl)
-      << Command->getCommandNameRange();
+      << Command->getCommandNameRange(Traits);
 
   return Command;
 }
@@ -142,56 +150,6 @@ void Sema::actOnParamCommandParamNameArg(ParamCommandComment *Command,
                                                      ArgLocEnd),
                                          Arg);
   Command->setArgs(llvm::makeArrayRef(A, 1));
-
-  if (!isFunctionDecl()) {
-    // We already warned that this \\param is not attached to a function decl.
-    return;
-  }
-
-  ArrayRef<const ParmVarDecl *> ParamVars = getParamVars();
-
-  // Check that referenced parameter name is in the function decl.
-  const unsigned ResolvedParamIndex = resolveParmVarReference(Arg, ParamVars);
-  if (ResolvedParamIndex != ParamCommandComment::InvalidParamIndex) {
-    Command->setParamIndex(ResolvedParamIndex);
-    if (ParamVarDocs[ResolvedParamIndex]) {
-      SourceRange ArgRange(ArgLocBegin, ArgLocEnd);
-      Diag(ArgLocBegin, diag::warn_doc_param_duplicate)
-        << Arg << ArgRange;
-      ParamCommandComment *PrevCommand = ParamVarDocs[ResolvedParamIndex];
-      Diag(PrevCommand->getLocation(), diag::note_doc_param_previous)
-        << PrevCommand->getParamNameRange();
-    }
-    ParamVarDocs[ResolvedParamIndex] = Command;
-    return;
-  }
-
-  SourceRange ArgRange(ArgLocBegin, ArgLocEnd);
-  Diag(ArgLocBegin, diag::warn_doc_param_not_found)
-    << Arg << ArgRange;
-
-  // No parameters -- can't suggest a correction.
-  if (ParamVars.size() == 0)
-    return;
-
-  unsigned CorrectedParamIndex = ParamCommandComment::InvalidParamIndex;
-  if (ParamVars.size() == 1) {
-    // If function has only one parameter then only that parameter
-    // can be documented.
-    CorrectedParamIndex = 0;
-  } else {
-    // Do typo correction.
-    CorrectedParamIndex = correctTypoInParmVarReference(Arg, ParamVars);
-  }
-  if (CorrectedParamIndex != ParamCommandComment::InvalidParamIndex) {
-    const ParmVarDecl *CorrectedPVD = ParamVars[CorrectedParamIndex];
-    if (const IdentifierInfo *CorrectedII = CorrectedPVD->getIdentifier())
-      Diag(ArgLocBegin, diag::note_doc_param_name_suggestion)
-        << CorrectedII->getName()
-        << FixItHint::CreateReplacement(ArgRange, CorrectedII->getName());
-  }
-
-  return;
 }
 
 void Sema::actOnParamCommandFinish(ParamCommandComment *Command,
@@ -202,14 +160,14 @@ void Sema::actOnParamCommandFinish(ParamCommandComment *Command,
 
 TParamCommandComment *Sema::actOnTParamCommandStart(SourceLocation LocBegin,
                                                     SourceLocation LocEnd,
-                                                    StringRef Name) {
+                                                    unsigned CommandID) {
   TParamCommandComment *Command =
-      new (Allocator) TParamCommandComment(LocBegin, LocEnd, Name);
+      new (Allocator) TParamCommandComment(LocBegin, LocEnd, CommandID);
 
   if (!isTemplateOrSpecialization())
     Diag(Command->getLocation(),
          diag::warn_doc_tparam_not_attached_to_a_template_decl)
-      << Command->getCommandNameRange();
+      << Command->getCommandNameRange(Traits);
 
   return Command;
 }
@@ -285,19 +243,20 @@ void Sema::actOnTParamCommandFinish(TParamCommandComment *Command,
 
 InlineCommandComment *Sema::actOnInlineCommand(SourceLocation CommandLocBegin,
                                                SourceLocation CommandLocEnd,
-                                               StringRef CommandName) {
+                                               unsigned CommandID) {
   ArrayRef<InlineCommandComment::Argument> Args;
+  StringRef CommandName = Traits.getCommandInfo(CommandID)->Name;
   return new (Allocator) InlineCommandComment(
                                   CommandLocBegin,
                                   CommandLocEnd,
-                                  CommandName,
+                                  CommandID,
                                   getInlineCommandRenderKind(CommandName),
                                   Args);
 }
 
 InlineCommandComment *Sema::actOnInlineCommand(SourceLocation CommandLocBegin,
                                                SourceLocation CommandLocEnd,
-                                               StringRef CommandName,
+                                               unsigned CommandID,
                                                SourceLocation ArgLocBegin,
                                                SourceLocation ArgLocEnd,
                                                StringRef Arg) {
@@ -305,21 +264,29 @@ InlineCommandComment *Sema::actOnInlineCommand(SourceLocation CommandLocBegin,
   Argument *A = new (Allocator) Argument(SourceRange(ArgLocBegin,
                                                      ArgLocEnd),
                                          Arg);
+  StringRef CommandName = Traits.getCommandInfo(CommandID)->Name;
 
   return new (Allocator) InlineCommandComment(
                                   CommandLocBegin,
                                   CommandLocEnd,
-                                  CommandName,
+                                  CommandID,
                                   getInlineCommandRenderKind(CommandName),
                                   llvm::makeArrayRef(A, 1));
 }
 
 InlineContentComment *Sema::actOnUnknownCommand(SourceLocation LocBegin,
                                                 SourceLocation LocEnd,
-                                                StringRef Name) {
+                                                StringRef CommandName) {
+  unsigned CommandID = Traits.registerUnknownCommand(CommandName)->getID();
+  return actOnUnknownCommand(LocBegin, LocEnd, CommandID);
+}
+
+InlineContentComment *Sema::actOnUnknownCommand(SourceLocation LocBegin,
+                                                SourceLocation LocEnd,
+                                                unsigned CommandID) {
   ArrayRef<InlineCommandComment::Argument> Args;
   return new (Allocator) InlineCommandComment(
-                                  LocBegin, LocEnd, Name,
+                                  LocBegin, LocEnd, CommandID,
                                   InlineCommandComment::RenderNormal,
                                   Args);
 }
@@ -331,11 +298,12 @@ TextComment *Sema::actOnText(SourceLocation LocBegin,
 }
 
 VerbatimBlockComment *Sema::actOnVerbatimBlockStart(SourceLocation Loc,
-                                                    StringRef Name) {
+                                                    unsigned CommandID) {
+  StringRef CommandName = Traits.getCommandInfo(CommandID)->Name;
   return new (Allocator) VerbatimBlockComment(
                                   Loc,
-                                  Loc.getLocWithOffset(1 + Name.size()),
-                                  Name);
+                                  Loc.getLocWithOffset(1 + CommandName.size()),
+                                  CommandID);
 }
 
 VerbatimBlockLineComment *Sema::actOnVerbatimBlockLine(SourceLocation Loc,
@@ -353,13 +321,13 @@ void Sema::actOnVerbatimBlockFinish(
 }
 
 VerbatimLineComment *Sema::actOnVerbatimLine(SourceLocation LocBegin,
-                                             StringRef Name,
+                                             unsigned CommandID,
                                              SourceLocation TextBegin,
                                              StringRef Text) {
   return new (Allocator) VerbatimLineComment(
                               LocBegin,
                               TextBegin.getLocWithOffset(Text.size()),
-                              Name,
+                              CommandID,
                               TextBegin,
                               Text);
 }
@@ -445,30 +413,35 @@ HTMLEndTagComment *Sema::actOnHTMLEndTag(SourceLocation LocBegin,
 
 FullComment *Sema::actOnFullComment(
                               ArrayRef<BlockContentComment *> Blocks) {
-  return new (Allocator) FullComment(Blocks, ThisDeclInfo);
+  FullComment *FC = new (Allocator) FullComment(Blocks, ThisDeclInfo);
+  resolveParamCommandIndexes(FC);
+  return FC;
 }
 
 void Sema::checkBlockCommandEmptyParagraph(BlockCommandComment *Command) {
+  if (Traits.getCommandInfo(Command->getCommandID())->IsEmptyParagraphAllowed)
+    return;
+
   ParagraphComment *Paragraph = Command->getParagraph();
   if (Paragraph->isWhitespace()) {
     SourceLocation DiagLoc;
     if (Command->getNumArgs() > 0)
       DiagLoc = Command->getArgRange(Command->getNumArgs() - 1).getEnd();
     if (!DiagLoc.isValid())
-      DiagLoc = Command->getCommandNameRange().getEnd();
+      DiagLoc = Command->getCommandNameRange(Traits).getEnd();
     Diag(DiagLoc, diag::warn_doc_block_command_empty_paragraph)
-      << Command->getCommandName()
+      << Command->getCommandName(Traits)
       << Command->getSourceRange();
   }
 }
 
 void Sema::checkReturnsCommand(const BlockCommandComment *Command) {
-  if (!Traits.isReturnsCommand(Command->getCommandName()))
+  if (!Traits.getCommandInfo(Command->getCommandID())->IsReturnsCommand)
     return;
   if (isFunctionDecl()) {
     if (ThisDeclInfo->ResultType->isVoidType()) {
       unsigned DiagKind;
-      switch (ThisDeclInfo->ThisDecl->getKind()) {
+      switch (ThisDeclInfo->CommentDecl->getKind()) {
       default:
         if (ThisDeclInfo->IsObjCMethod)
           DiagKind = 3;
@@ -484,7 +457,7 @@ void Sema::checkReturnsCommand(const BlockCommandComment *Command) {
       }
       Diag(Command->getLocation(),
            diag::warn_doc_returns_attached_to_a_void_function)
-        << Command->getCommandName()
+        << Command->getCommandName(Traits)
         << DiagKind
         << Command->getSourceRange();
     }
@@ -492,20 +465,20 @@ void Sema::checkReturnsCommand(const BlockCommandComment *Command) {
   }
   Diag(Command->getLocation(),
        diag::warn_doc_returns_not_attached_to_a_function_decl)
-    << Command->getCommandName()
+    << Command->getCommandName(Traits)
     << Command->getSourceRange();
 }
 
 void Sema::checkBlockCommandDuplicate(const BlockCommandComment *Command) {
-  StringRef Name = Command->getCommandName();
+  const CommandInfo *Info = Traits.getCommandInfo(Command->getCommandID());
   const BlockCommandComment *PrevCommand = NULL;
-  if (Traits.isBriefCommand(Name)) {
+  if (Info->IsBriefCommand) {
     if (!BriefCommand) {
       BriefCommand = Command;
       return;
     }
     PrevCommand = BriefCommand;
-  } else if (Traits.isReturnsCommand(Name)) {
+  } else if (Info->IsReturnsCommand) {
     if (!ReturnsCommand) {
       ReturnsCommand = Command;
       return;
@@ -515,18 +488,153 @@ void Sema::checkBlockCommandDuplicate(const BlockCommandComment *Command) {
     // We don't want to check this command for duplicates.
     return;
   }
+  StringRef CommandName = Command->getCommandName(Traits);
+  StringRef PrevCommandName = PrevCommand->getCommandName(Traits);
   Diag(Command->getLocation(), diag::warn_doc_block_command_duplicate)
-      << Name
+      << CommandName
       << Command->getSourceRange();
-  if (Name == PrevCommand->getCommandName())
+  if (CommandName == PrevCommandName)
     Diag(PrevCommand->getLocation(), diag::note_doc_block_command_previous)
-        << PrevCommand->getCommandName()
-        << Command->getSourceRange();
+        << PrevCommandName
+        << PrevCommand->getSourceRange();
   else
     Diag(PrevCommand->getLocation(),
          diag::note_doc_block_command_previous_alias)
-        << PrevCommand->getCommandName()
-        << Name;
+        << PrevCommandName
+        << CommandName;
+}
+
+void Sema::checkDeprecatedCommand(const BlockCommandComment *Command) {
+  if (!Traits.getCommandInfo(Command->getCommandID())->IsDeprecatedCommand)
+    return;
+
+  const Decl *D = ThisDeclInfo->CommentDecl;
+  if (!D)
+    return;
+
+  if (D->hasAttr<DeprecatedAttr>() ||
+      D->hasAttr<AvailabilityAttr>() ||
+      D->hasAttr<UnavailableAttr>())
+    return;
+
+  Diag(Command->getLocation(),
+       diag::warn_doc_deprecated_not_sync)
+    << Command->getSourceRange();
+
+  // Try to emit a fixit with a deprecation attribute.
+  if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
+    // Don't emit a Fix-It for non-member function definitions.  GCC does not
+    // accept attributes on them.
+    const DeclContext *Ctx = FD->getDeclContext();
+    if ((!Ctx || !Ctx->isRecord()) &&
+        FD->doesThisDeclarationHaveABody())
+      return;
+
+    StringRef AttributeSpelling = "__attribute__((deprecated))";
+    if (PP) {
+      TokenValue Tokens[] = {
+        tok::kw___attribute, tok::l_paren, tok::l_paren,
+        PP->getIdentifierInfo("deprecated"),
+        tok::r_paren, tok::r_paren
+      };
+      StringRef MacroName = PP->getLastMacroWithSpelling(FD->getLocation(),
+                                                         Tokens);
+      if (!MacroName.empty())
+        AttributeSpelling = MacroName;
+    }
+
+    SmallString<64> TextToInsert(" ");
+    TextToInsert += AttributeSpelling;
+    Diag(FD->getLocEnd(),
+         diag::note_add_deprecation_attr)
+      << FixItHint::CreateInsertion(FD->getLocEnd().getLocWithOffset(1),
+                                    TextToInsert);
+  }
+}
+
+void Sema::resolveParamCommandIndexes(const FullComment *FC) {
+  if (!isFunctionDecl()) {
+    // We already warned that \\param commands are not attached to a function
+    // decl.
+    return;
+  }
+
+  llvm::SmallVector<ParamCommandComment *, 8> UnresolvedParamCommands;
+
+  // Comment AST nodes that correspond to \c ParamVars for which we have
+  // found a \\param command or NULL if no documentation was found so far.
+  llvm::SmallVector<ParamCommandComment *, 8> ParamVarDocs;
+
+  ArrayRef<const ParmVarDecl *> ParamVars = getParamVars();
+  ParamVarDocs.resize(ParamVars.size(), NULL);
+
+  // First pass over all \\param commands: resolve all parameter names.
+  for (Comment::child_iterator I = FC->child_begin(), E = FC->child_end();
+       I != E; ++I) {
+    ParamCommandComment *PCC = dyn_cast<ParamCommandComment>(*I);
+    if (!PCC || !PCC->hasParamName())
+      continue;
+    StringRef ParamName = PCC->getParamNameAsWritten();
+
+    // Check that referenced parameter name is in the function decl.
+    const unsigned ResolvedParamIndex = resolveParmVarReference(ParamName,
+                                                                ParamVars);
+    if (ResolvedParamIndex == ParamCommandComment::InvalidParamIndex) {
+      UnresolvedParamCommands.push_back(PCC);
+      continue;
+    }
+    PCC->setParamIndex(ResolvedParamIndex);
+    if (ParamVarDocs[ResolvedParamIndex]) {
+      SourceRange ArgRange = PCC->getParamNameRange();
+      Diag(ArgRange.getBegin(), diag::warn_doc_param_duplicate)
+        << ParamName << ArgRange;
+      ParamCommandComment *PrevCommand = ParamVarDocs[ResolvedParamIndex];
+      Diag(PrevCommand->getLocation(), diag::note_doc_param_previous)
+        << PrevCommand->getParamNameRange();
+    }
+    ParamVarDocs[ResolvedParamIndex] = PCC;
+  }
+
+  // Find parameter declarations that have no corresponding \\param.
+  llvm::SmallVector<const ParmVarDecl *, 8> OrphanedParamDecls;
+  for (unsigned i = 0, e = ParamVarDocs.size(); i != e; ++i) {
+    if (!ParamVarDocs[i])
+      OrphanedParamDecls.push_back(ParamVars[i]);
+  }
+
+  // Second pass over unresolved \\param commands: do typo correction.
+  // Suggest corrections from a set of parameter declarations that have no
+  // corresponding \\param.
+  for (unsigned i = 0, e = UnresolvedParamCommands.size(); i != e; ++i) {
+    const ParamCommandComment *PCC = UnresolvedParamCommands[i];
+
+    SourceRange ArgRange = PCC->getParamNameRange();
+    StringRef ParamName = PCC->getParamNameAsWritten();
+    Diag(ArgRange.getBegin(), diag::warn_doc_param_not_found)
+      << ParamName << ArgRange;
+
+    // All parameters documented -- can't suggest a correction.
+    if (OrphanedParamDecls.size() == 0)
+      continue;
+
+    unsigned CorrectedParamIndex = ParamCommandComment::InvalidParamIndex;
+    if (OrphanedParamDecls.size() == 1) {
+      // If one parameter is not documented then that parameter is the only
+      // possible suggestion.
+      CorrectedParamIndex = 0;
+    } else {
+      // Do typo correction.
+      CorrectedParamIndex = correctTypoInParmVarReference(ParamName,
+                                                          OrphanedParamDecls);
+    }
+    if (CorrectedParamIndex != ParamCommandComment::InvalidParamIndex) {
+      const ParmVarDecl *CorrectedPVD = OrphanedParamDecls[CorrectedParamIndex];
+      if (const IdentifierInfo *CorrectedII = CorrectedPVD->getIdentifier())
+        Diag(ArgRange.getBegin(), diag::note_doc_param_name_suggestion)
+          << CorrectedII->getName()
+          << FixItHint::CreateReplacement(ArgRange, CorrectedII->getName());
+    }
+  }
 }
 
 bool Sema::isFunctionDecl() {
@@ -553,7 +661,6 @@ ArrayRef<const ParmVarDecl *> Sema::getParamVars() {
 
 void Sema::inspectThisDecl() {
   ThisDeclInfo->fill();
-  ParamVarDocs.resize(ThisDeclInfo->ParamVars.size(), NULL);
 }
 
 unsigned Sema::resolveParmVarReference(StringRef Name,
@@ -629,7 +736,7 @@ unsigned Sema::correctTypoInParmVarReference(
   if (Corrector.getBestDecl())
     return Corrector.getBestDeclIndex();
   else
-    return ParamCommandComment::InvalidParamIndex;;
+    return ParamCommandComment::InvalidParamIndex;
 }
 
 namespace {
@@ -700,7 +807,7 @@ StringRef Sema::correctTypoInTParamReference(
 
 InlineCommandComment::RenderKind
 Sema::getInlineCommandRenderKind(StringRef Name) const {
-  assert(Traits.isInlineCommand(Name));
+  assert(Traits.getCommandInfo(Name)->IsInlineCommand);
 
   return llvm::StringSwitch<InlineCommandComment::RenderKind>(Name)
       .Case("b", InlineCommandComment::RenderBold)
@@ -709,31 +816,6 @@ Sema::getInlineCommandRenderKind(StringRef Name) const {
       .Default(InlineCommandComment::RenderNormal);
 }
 
-bool Sema::isHTMLEndTagOptional(StringRef Name) {
-  return llvm::StringSwitch<bool>(Name)
-      .Case("p", true)
-      .Case("li", true)
-      .Case("dt", true)
-      .Case("dd", true)
-      .Case("tr", true)
-      .Case("th", true)
-      .Case("td", true)
-      .Case("thead", true)
-      .Case("tfoot", true)
-      .Case("tbody", true)
-      .Case("colgroup", true)
-      .Default(false);
-}
-
-bool Sema::isHTMLEndTagForbidden(StringRef Name) {
-  return llvm::StringSwitch<bool>(Name)
-      .Case("br", true)
-      .Case("hr", true)
-      .Case("img", true)
-      .Case("col", true)
-      .Default(false);
-}
-
 } // end namespace comments
 } // end namespace clang
 
diff --git a/lib/AST/Decl.cpp b/lib/AST/Decl.cpp
index d5b0be3..7b13755 100644
--- a/lib/AST/Decl.cpp
+++ b/lib/AST/Decl.cpp
@@ -126,12 +126,12 @@ static LinkageInfo getLVForTemplateArgumentList(const TemplateArgument *Args,
       break;
 
     case TemplateArgument::Declaration:
-      // The decl can validly be null as the representation of nullptr
-      // arguments, valid only in C++0x.
-      if (Decl *D = Args[I].getAsDecl()) {
-        if (NamedDecl *ND = dyn_cast<NamedDecl>(D))
-          LV.mergeWithMin(getLVForDecl(ND, OnlyTemplate));
-      }
+      if (NamedDecl *ND = dyn_cast<NamedDecl>(Args[I].getAsDecl()))
+        LV.mergeWithMin(getLVForDecl(ND, OnlyTemplate));
+      break;
+
+    case TemplateArgument::NullPtr:
+      LV.mergeWithMin(getLVForType(Args[I].getNullPtrType()));
       break;
 
     case TemplateArgument::Template:
@@ -193,7 +193,7 @@ static bool useInlineVisibilityHidden(const NamedDecl *D) {
   // anyway.
   return TSK != TSK_ExplicitInstantiationDeclaration &&
     TSK != TSK_ExplicitInstantiationDefinition &&
-    FD->hasBody(Def) && Def->isInlined();
+    FD->hasBody(Def) && Def->isInlined() && !Def->hasAttr<GNUInlineAttr>();
 }
 
 static LinkageInfo getLVForNamespaceScopeDecl(const NamedDecl *D,
@@ -213,12 +213,12 @@ static LinkageInfo getLVForNamespaceScopeDecl(const NamedDecl *D,
     if (Var->getStorageClass() == SC_Static)
       return LinkageInfo::internal();
 
-    // - an object or reference that is explicitly declared const
-    //   and neither explicitly declared extern nor previously
-    //   declared to have external linkage; or
-    // (there is no equivalent in C99)
+    // - a non-volatile object or reference that is explicitly declared const
+    //   or constexpr and neither explicitly declared extern nor previously
+    //   declared to have external linkage; or (there is no equivalent in C99)
     if (Context.getLangOpts().CPlusPlus &&
-        Var->getType().isConstant(Context) && 
+        Var->getType().isConstQualified() && 
+        !Var->getType().isVolatileQualified() &&
         Var->getStorageClass() != SC_Extern &&
         Var->getStorageClass() != SC_PrivateExtern) {
       bool FoundExtern = false;
@@ -236,8 +236,8 @@ static LinkageInfo getLVForNamespaceScopeDecl(const NamedDecl *D,
       for (; PrevVar; PrevVar = PrevVar->getPreviousDecl())
         if (PrevVar->getStorageClass() == SC_PrivateExtern)
           break;
-        if (PrevVar)
-          return PrevVar->getLinkageAndVisibility();
+      if (PrevVar)
+        return PrevVar->getLinkageAndVisibility();
     }
   } else if (isa<FunctionDecl>(D) || isa<FunctionTemplateDecl>(D)) {
     // C++ [temp]p4:
@@ -341,25 +341,9 @@ static LinkageInfo getLVForNamespaceScopeDecl(const NamedDecl *D,
     if (Var->getStorageClass() == SC_PrivateExtern)
       LV.mergeVisibility(HiddenVisibility, true);
 
-    if (!Context.getLangOpts().CPlusPlus &&
-        (Var->getStorageClass() == SC_Extern ||
-         Var->getStorageClass() == SC_PrivateExtern)) {
-
-      // C99 6.2.2p4:
-      //   For an identifier declared with the storage-class specifier
-      //   extern in a scope in which a prior declaration of that
-      //   identifier is visible, if the prior declaration specifies
-      //   internal or external linkage, the linkage of the identifier
-      //   at the later declaration is the same as the linkage
-      //   specified at the prior declaration. If no prior declaration
-      //   is visible, or if the prior declaration specifies no
-      //   linkage, then the identifier has external linkage.
-      if (const VarDecl *PrevVar = Var->getPreviousDecl()) {
-        LinkageInfo PrevLV = getLVForDecl(PrevVar, OnlyTemplate);
-        if (PrevLV.linkage()) LV.setLinkage(PrevLV.linkage());
-        LV.mergeVisibility(PrevLV);
-      }
-    }
+    // Note that Sema::MergeVarDecl already takes care of implementing
+    // C99 6.2.2p4 and propagating the visibility attribute, so we don't have
+    // to do it here.
 
   //     - a function, unless it has internal linkage; or
   } else if (const FunctionDecl *Function = dyn_cast<FunctionDecl>(D)) {
@@ -841,13 +825,10 @@ static LinkageInfo getLVForDecl(const NamedDecl *D, bool OnlyTemplate) {
           if (llvm::Optional<Visibility> Vis = Var->getExplicitVisibility())
             LV.mergeVisibility(*Vis, true);
         }
-        
-        if (const VarDecl *Prev = Var->getPreviousDecl()) {
-          LinkageInfo PrevLV = getLVForDecl(Prev, OnlyTemplate);
-          if (PrevLV.linkage()) LV.setLinkage(PrevLV.linkage());
-          LV.mergeVisibility(PrevLV);
-        }
 
+        // Note that Sema::MergeVarDecl already takes care of implementing
+        // C99 6.2.2p4 and propagating the visibility attribute, so we don't
+        // have to do it here.
         return LV;
       }
   }
@@ -903,7 +884,7 @@ std::string NamedDecl::getQualifiedNameAsString(const PrintingPolicy &P) const {
     } else if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(*I)) {
       const FunctionProtoType *FT = 0;
       if (FD->hasWrittenPrototype())
-        FT = dyn_cast<FunctionProtoType>(FD->getType()->getAs<FunctionType>());
+        FT = dyn_cast<FunctionProtoType>(FD->getType()->castAs<FunctionType>());
 
       OS << *FD << '(';
       if (FT) {
@@ -1204,8 +1185,11 @@ void VarDecl::setStorageClass(StorageClass SC) {
 }
 
 SourceRange VarDecl::getSourceRange() const {
-  if (getInit())
-    return SourceRange(getOuterLocStart(), getInit()->getLocEnd());
+  if (const Expr *Init = getInit()) {
+    SourceLocation InitEnd = Init->getLocEnd();
+    if (InitEnd.isValid())
+      return SourceRange(getOuterLocStart(), InitEnd);
+  }
   return DeclaratorDecl::getSourceRange();
 }
 
@@ -1859,7 +1843,7 @@ unsigned FunctionDecl::getBuiltinID() const {
 /// based on its FunctionType.  This is the length of the ParamInfo array
 /// after it has been created.
 unsigned FunctionDecl::getNumParams() const {
-  const FunctionType *FT = getType()->getAs<FunctionType>();
+  const FunctionType *FT = getType()->castAs<FunctionType>();
   if (isa<FunctionNoProtoType>(FT))
     return 0;
   return cast<FunctionProtoType>(FT)->getNumArgs();
@@ -2514,7 +2498,7 @@ unsigned FieldDecl::getFieldIndex() const {
   unsigned Index = 0;
   const RecordDecl *RD = getParent();
   const FieldDecl *LastFD = 0;
-  bool IsMsStruct = RD->hasAttr<MsStructAttr>();
+  bool IsMsStruct = RD->isMsStruct(getASTContext());
 
   for (RecordDecl::field_iterator I = RD->field_begin(), E = RD->field_end();
        I != E; ++I, ++Index) {
@@ -2762,6 +2746,17 @@ void RecordDecl::completeDefinition() {
   TagDecl::completeDefinition();
 }
 
+/// isMsStruct - Get whether or not this record uses ms_struct layout.
+/// This which can be turned on with an attribute, pragma, or the
+/// -mms-bitfields command-line option.
+bool RecordDecl::isMsStruct(const ASTContext &C) const {
+  return hasAttr<MsStructAttr>() || C.getLangOpts().MSBitfields == 1;
+}
+
+static bool isFieldOrIndirectField(Decl::Kind K) {
+  return FieldDecl::classofKind(K) || IndirectFieldDecl::classofKind(K);
+}
+
 void RecordDecl::LoadFieldsFromExternalStorage() const {
   ExternalASTSource *Source = getASTContext().getExternalSource();
   assert(hasExternalLexicalStorage() && Source && "No external storage?");
@@ -2771,7 +2766,8 @@ void RecordDecl::LoadFieldsFromExternalStorage() const {
 
   SmallVector<Decl*, 64> Decls;
   LoadedFieldsFromExternalStorage = true;  
-  switch (Source->FindExternalLexicalDeclsBy<FieldDecl>(this, Decls)) {
+  switch (Source->FindExternalLexicalDecls(this, isFieldOrIndirectField,
+                                           Decls)) {
   case ELR_Success:
     break;
     
@@ -2783,7 +2779,7 @@ void RecordDecl::LoadFieldsFromExternalStorage() const {
 #ifndef NDEBUG
   // Check that all decls we got were FieldDecls.
   for (unsigned i=0, e=Decls.size(); i != e; ++i)
-    assert(isa<FieldDecl>(Decls[i]));
+    assert(isa<FieldDecl>(Decls[i]) || isa<IndirectFieldDecl>(Decls[i]));
 #endif
 
   if (Decls.empty())
diff --git a/lib/AST/DeclBase.cpp b/lib/AST/DeclBase.cpp
index f9ce46d..4400d50 100644
--- a/lib/AST/DeclBase.cpp
+++ b/lib/AST/DeclBase.cpp
@@ -961,7 +961,7 @@ DeclContext::lookup_result
 ExternalASTSource::SetExternalVisibleDeclsForName(const DeclContext *DC,
                                                   DeclarationName Name,
                                                   ArrayRef<NamedDecl*> Decls) {
-  ASTContext &Context = DC->getParentASTContext();;
+  ASTContext &Context = DC->getParentASTContext();
 
   StoredDeclsMap *Map;
   if (!(Map = DC->LookupPtr.getPointer()))
diff --git a/lib/AST/DeclCXX.cpp b/lib/AST/DeclCXX.cpp
index 2f21e4c..82e630a 100644
--- a/lib/AST/DeclCXX.cpp
+++ b/lib/AST/DeclCXX.cpp
@@ -90,11 +90,12 @@ CXXRecordDecl *CXXRecordDecl::Create(const ASTContext &C, TagKind TK,
 }
 
 CXXRecordDecl *CXXRecordDecl::CreateLambda(const ASTContext &C, DeclContext *DC,
-                                           SourceLocation Loc, bool Dependent) {
+                                           TypeSourceInfo *Info, SourceLocation Loc,
+                                           bool Dependent) {
   CXXRecordDecl* R = new (C) CXXRecordDecl(CXXRecord, TTK_Class, DC, Loc, Loc,
                                            0, 0);
   R->IsBeingDefined = true;
-  R->DefinitionData = new (C) struct LambdaDefinitionData(R, Dependent);
+  R->DefinitionData = new (C) struct LambdaDefinitionData(R, Info, Dependent);
   C.getTypeDeclType(R, /*PrevDecl=*/0);
   return R;
 }
@@ -239,10 +240,13 @@ CXXRecordDecl::setBases(CXXBaseSpecifier const * const *Bases,
       //    -- the constructor selected to copy/move each direct base class
       //       subobject is trivial, and
       // FIXME: C++0x: We need to only consider the selected constructor
-      // instead of all of them.
+      // instead of all of them. For now, we treat a move constructor as being
+      // non-trivial if it calls anything other than a trivial move constructor.
       if (!BaseClassDecl->hasTrivialCopyConstructor())
         data().HasTrivialCopyConstructor = false;
-      if (!BaseClassDecl->hasTrivialMoveConstructor())
+      if (!BaseClassDecl->hasTrivialMoveConstructor() ||
+          !(BaseClassDecl->hasDeclaredMoveConstructor() ||
+            BaseClassDecl->needsImplicitMoveConstructor()))
         data().HasTrivialMoveConstructor = false;
 
       // C++0x [class.copy]p27:
@@ -254,7 +258,9 @@ CXXRecordDecl::setBases(CXXBaseSpecifier const * const *Bases,
       // of all of them.
       if (!BaseClassDecl->hasTrivialCopyAssignment())
         data().HasTrivialCopyAssignment = false;
-      if (!BaseClassDecl->hasTrivialMoveAssignment())
+      if (!BaseClassDecl->hasTrivialMoveAssignment() ||
+          !(BaseClassDecl->hasDeclaredMoveAssignment() ||
+            BaseClassDecl->needsImplicitMoveAssignment()))
         data().HasTrivialMoveAssignment = false;
 
       // C++11 [class.ctor]p6:
@@ -466,7 +472,8 @@ void CXXRecordDecl::addedMember(Decl *D) {
   if (!D->isImplicit() &&
       !isa<FieldDecl>(D) &&
       !isa<IndirectFieldDecl>(D) &&
-      (!isa<TagDecl>(D) || cast<TagDecl>(D)->getTagKind() == TTK_Class))
+      (!isa<TagDecl>(D) || cast<TagDecl>(D)->getTagKind() == TTK_Class ||
+        cast<TagDecl>(D)->getTagKind() == TTK_Interface))
     data().HasOnlyCMembers = false;
 
   // Ignore friends and invalid declarations.
@@ -828,7 +835,9 @@ NotASpecialMember:;
         // FIXME: C++0x: We don't correctly model 'selected' constructors.
         if (!FieldRec->hasTrivialCopyConstructor())
           data().HasTrivialCopyConstructor = false;
-        if (!FieldRec->hasTrivialMoveConstructor())
+        if (!FieldRec->hasTrivialMoveConstructor() ||
+            !(FieldRec->hasDeclaredMoveConstructor() ||
+              FieldRec->needsImplicitMoveConstructor()))
           data().HasTrivialMoveConstructor = false;
 
         // C++0x [class.copy]p27:
@@ -840,7 +849,9 @@ NotASpecialMember:;
         // FIXME: C++0x: We don't correctly model 'selected' operators.
         if (!FieldRec->hasTrivialCopyAssignment())
           data().HasTrivialCopyAssignment = false;
-        if (!FieldRec->hasTrivialMoveAssignment())
+        if (!FieldRec->hasTrivialMoveAssignment() ||
+            !(FieldRec->hasDeclaredMoveAssignment() ||
+              FieldRec->needsImplicitMoveAssignment()))
           data().HasTrivialMoveAssignment = false;
 
         if (!FieldRec->hasTrivialDestructor())
@@ -936,7 +947,8 @@ NotASpecialMember:;
 }
 
 bool CXXRecordDecl::isCLike() const {
-  if (getTagKind() == TTK_Class || !TemplateOrInstantiation.isNull())
+  if (getTagKind() == TTK_Class || getTagKind() == TTK_Interface ||
+      !TemplateOrInstantiation.isNull())
     return false;
   if (!hasDefinition())
     return true;
diff --git a/lib/AST/DeclObjC.cpp b/lib/AST/DeclObjC.cpp
index 4d48ad8..65a9878 100644
--- a/lib/AST/DeclObjC.cpp
+++ b/lib/AST/DeclObjC.cpp
@@ -16,6 +16,7 @@
 #include "clang/AST/Stmt.h"
 #include "clang/AST/ASTMutationListener.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallString.h"
 using namespace clang;
 
 //===----------------------------------------------------------------------===//
@@ -93,6 +94,16 @@ ObjCPropertyDecl::findPropertyDecl(const DeclContext *DC,
   return 0;
 }
 
+IdentifierInfo *
+ObjCPropertyDecl::getDefaultSynthIvarName(ASTContext &Ctx) const {
+  SmallString<128> ivarName;
+  {
+    llvm::raw_svector_ostream os(ivarName);
+    os << '_' << getIdentifier()->getName();
+  }
+  return &Ctx.Idents.get(ivarName.str());
+}
+
 /// FindPropertyDeclaration - Finds declaration of the property given its name
 /// in 'PropertyId' and returns it. It returns 0, if not found.
 ObjCPropertyDecl *
@@ -179,6 +190,21 @@ ObjCInterfaceDecl::FindPropertyVisibleInPrimaryClass(
   return 0;
 }
 
+void ObjCInterfaceDecl::collectPropertiesToImplement(PropertyMap &PM) const {
+  for (ObjCContainerDecl::prop_iterator P = prop_begin(),
+      E = prop_end(); P != E; ++P) {
+    ObjCPropertyDecl *Prop = *P;
+    PM[Prop->getIdentifier()] = Prop;
+  }
+  for (ObjCInterfaceDecl::all_protocol_iterator
+      PI = all_referenced_protocol_begin(),
+      E = all_referenced_protocol_end(); PI != E; ++PI)
+    (*PI)->collectPropertiesToImplement(PM);
+  // Note, the properties declared only in class extensions are still copied
+  // into the main @interface's property list, and therefore we don't
+  // explicitly, have to search class extension properties.
+}
+
 void ObjCInterfaceDecl::mergeClassExtensionProtocolList(
                               ObjCProtocolDecl *const* ExtList, unsigned ExtNum,
                               ASTContext &C)
@@ -414,16 +440,15 @@ ObjCMethodDecl *ObjCMethodDecl::Create(ASTContext &C,
                                        DeclContext *contextDecl,
                                        bool isInstance,
                                        bool isVariadic,
-                                       bool isSynthesized,
+                                       bool isPropertyAccessor,
                                        bool isImplicitlyDeclared,
                                        bool isDefined,
                                        ImplementationControl impControl,
                                        bool HasRelatedResultType) {
   return new (C) ObjCMethodDecl(beginLoc, endLoc,
                                 SelInfo, T, ResultTInfo, contextDecl,
-                                isInstance,
-                                isVariadic, isSynthesized, isImplicitlyDeclared,
-                                isDefined,
+                                isInstance, isVariadic, isPropertyAccessor,
+                                isImplicitlyDeclared, isDefined,
                                 impControl,
                                 HasRelatedResultType);
 }
@@ -434,6 +459,10 @@ ObjCMethodDecl *ObjCMethodDecl::CreateDeserialized(ASTContext &C, unsigned ID) {
                                   Selector(), QualType(), 0, 0);
 }
 
+Stmt *ObjCMethodDecl::getBody() const {
+  return Body.get(getASTContext().getExternalSource());
+}
+
 void ObjCMethodDecl::setAsRedeclaration(const ObjCMethodDecl *PrevMethod) {
   assert(PrevMethod);
   getASTContext().setObjCMethodRedeclaration(PrevMethod, this);
@@ -707,6 +736,224 @@ ObjCInterfaceDecl *ObjCMethodDecl::getClassInterface() {
   llvm_unreachable("unknown method context");
 }
 
+static void CollectOverriddenMethodsRecurse(const ObjCContainerDecl *Container,
+                                            const ObjCMethodDecl *Method,
+                               SmallVectorImpl<const ObjCMethodDecl *> &Methods,
+                                            bool MovedToSuper) {
+  if (!Container)
+    return;
+
+  // In categories look for overriden methods from protocols. A method from
+  // category is not "overriden" since it is considered as the "same" method
+  // (same USR) as the one from the interface.
+  if (const ObjCCategoryDecl *
+        Category = dyn_cast<ObjCCategoryDecl>(Container)) {
+    // Check whether we have a matching method at this category but only if we
+    // are at the super class level.
+    if (MovedToSuper)
+      if (ObjCMethodDecl *
+            Overridden = Container->getMethod(Method->getSelector(),
+                                              Method->isInstanceMethod()))
+        if (Method != Overridden) {
+          // We found an override at this category; there is no need to look
+          // into its protocols.
+          Methods.push_back(Overridden);
+          return;
+        }
+
+    for (ObjCCategoryDecl::protocol_iterator P = Category->protocol_begin(),
+                                          PEnd = Category->protocol_end();
+         P != PEnd; ++P)
+      CollectOverriddenMethodsRecurse(*P, Method, Methods, MovedToSuper);
+    return;
+  }
+
+  // Check whether we have a matching method at this level.
+  if (const ObjCMethodDecl *
+        Overridden = Container->getMethod(Method->getSelector(),
+                                                    Method->isInstanceMethod()))
+    if (Method != Overridden) {
+      // We found an override at this level; there is no need to look
+      // into other protocols or categories.
+      Methods.push_back(Overridden);
+      return;
+    }
+
+  if (const ObjCProtocolDecl *Protocol = dyn_cast<ObjCProtocolDecl>(Container)){
+    for (ObjCProtocolDecl::protocol_iterator P = Protocol->protocol_begin(),
+                                          PEnd = Protocol->protocol_end();
+         P != PEnd; ++P)
+      CollectOverriddenMethodsRecurse(*P, Method, Methods, MovedToSuper);
+  }
+
+  if (const ObjCInterfaceDecl *
+        Interface = dyn_cast<ObjCInterfaceDecl>(Container)) {
+    for (ObjCInterfaceDecl::protocol_iterator P = Interface->protocol_begin(),
+                                           PEnd = Interface->protocol_end();
+         P != PEnd; ++P)
+      CollectOverriddenMethodsRecurse(*P, Method, Methods, MovedToSuper);
+
+    for (const ObjCCategoryDecl *Category = Interface->getCategoryList();
+         Category; Category = Category->getNextClassCategory())
+      CollectOverriddenMethodsRecurse(Category, Method, Methods,
+                                      MovedToSuper);
+
+    if (const ObjCInterfaceDecl *Super = Interface->getSuperClass())
+      return CollectOverriddenMethodsRecurse(Super, Method, Methods,
+                                             /*MovedToSuper=*/true);
+  }
+}
+
+static inline void CollectOverriddenMethods(const ObjCContainerDecl *Container,
+                                            const ObjCMethodDecl *Method,
+                             SmallVectorImpl<const ObjCMethodDecl *> &Methods) {
+  CollectOverriddenMethodsRecurse(Container, Method, Methods,
+                                  /*MovedToSuper=*/false);
+}
+
+static void collectOverriddenMethodsSlow(const ObjCMethodDecl *Method,
+                          SmallVectorImpl<const ObjCMethodDecl *> &overridden) {
+  assert(Method->isOverriding());
+
+  if (const ObjCProtocolDecl *
+        ProtD = dyn_cast<ObjCProtocolDecl>(Method->getDeclContext())) {
+    CollectOverriddenMethods(ProtD, Method, overridden);
+
+  } else if (const ObjCImplDecl *
+               IMD = dyn_cast<ObjCImplDecl>(Method->getDeclContext())) {
+    const ObjCInterfaceDecl *ID = IMD->getClassInterface();
+    if (!ID)
+      return;
+    // Start searching for overridden methods using the method from the
+    // interface as starting point.
+    if (const ObjCMethodDecl *IFaceMeth = ID->getMethod(Method->getSelector(),
+                                                  Method->isInstanceMethod()))
+      Method = IFaceMeth;
+    CollectOverriddenMethods(ID, Method, overridden);
+
+  } else if (const ObjCCategoryDecl *
+               CatD = dyn_cast<ObjCCategoryDecl>(Method->getDeclContext())) {
+    const ObjCInterfaceDecl *ID = CatD->getClassInterface();
+    if (!ID)
+      return;
+    // Start searching for overridden methods using the method from the
+    // interface as starting point.
+    if (const ObjCMethodDecl *IFaceMeth = ID->getMethod(Method->getSelector(),
+                                                  Method->isInstanceMethod()))
+      Method = IFaceMeth;
+    CollectOverriddenMethods(ID, Method, overridden);
+
+  } else {
+    CollectOverriddenMethods(
+                  dyn_cast_or_null<ObjCContainerDecl>(Method->getDeclContext()),
+                  Method, overridden);
+  }
+}
+
+static void collectOnCategoriesAfterLocation(SourceLocation Loc,
+                                             const ObjCInterfaceDecl *Class,
+                                             SourceManager &SM,
+                                             const ObjCMethodDecl *Method,
+                             SmallVectorImpl<const ObjCMethodDecl *> &Methods) {
+  if (!Class)
+    return;
+
+  for (const ObjCCategoryDecl *Category = Class->getCategoryList();
+       Category; Category = Category->getNextClassCategory())
+    if (SM.isBeforeInTranslationUnit(Loc, Category->getLocation()))
+      CollectOverriddenMethodsRecurse(Category, Method, Methods, true);
+
+  collectOnCategoriesAfterLocation(Loc, Class->getSuperClass(), SM,
+                                   Method, Methods);
+}
+
+/// \brief Faster collection that is enabled when ObjCMethodDecl::isOverriding()
+/// returns false.
+/// You'd think that in that case there are no overrides but categories can
+/// "introduce" new overridden methods that are missed by Sema because the
+/// overrides lookup that it does for methods, inside implementations, will
+/// stop at the interface level (if there is a method there) and not look
+/// further in super classes.
+/// Methods in an implementation can overide methods in super class's category
+/// but not in current class's category. But, such methods
+static void collectOverriddenMethodsFast(SourceManager &SM,
+                                         const ObjCMethodDecl *Method,
+                             SmallVectorImpl<const ObjCMethodDecl *> &Methods) {
+  assert(!Method->isOverriding());
+
+  const ObjCContainerDecl *
+    ContD = cast<ObjCContainerDecl>(Method->getDeclContext());
+  if (isa<ObjCInterfaceDecl>(ContD) || isa<ObjCProtocolDecl>(ContD))
+    return;
+  const ObjCInterfaceDecl *Class = Method->getClassInterface();
+  if (!Class)
+    return;
+
+  collectOnCategoriesAfterLocation(Class->getLocation(), Class->getSuperClass(),
+                                   SM, Method, Methods);
+}
+
+void ObjCMethodDecl::getOverriddenMethods(
+                    SmallVectorImpl<const ObjCMethodDecl *> &Overridden) const {
+  const ObjCMethodDecl *Method = this;
+
+  if (Method->isRedeclaration()) {
+    Method = cast<ObjCContainerDecl>(Method->getDeclContext())->
+                   getMethod(Method->getSelector(), Method->isInstanceMethod());
+  }
+
+  if (!Method->isOverriding()) {
+    collectOverriddenMethodsFast(getASTContext().getSourceManager(),
+                                 Method, Overridden);
+  } else {
+    collectOverriddenMethodsSlow(Method, Overridden);
+    assert(!Overridden.empty() &&
+           "ObjCMethodDecl's overriding bit is not as expected");
+  }
+}
+
+const ObjCPropertyDecl *
+ObjCMethodDecl::findPropertyDecl(bool CheckOverrides) const {
+  Selector Sel = getSelector();
+  unsigned NumArgs = Sel.getNumArgs();
+  if (NumArgs > 1)
+    return 0;
+
+  if (!isInstanceMethod() || getMethodFamily() != OMF_None)
+    return 0;
+  
+  if (isPropertyAccessor()) {
+    const ObjCContainerDecl *Container = cast<ObjCContainerDecl>(getParent());
+    bool IsGetter = (NumArgs == 0);
+
+    for (ObjCContainerDecl::prop_iterator I = Container->prop_begin(),
+                                          E = Container->prop_end();
+         I != E; ++I) {
+      Selector NextSel = IsGetter ? (*I)->getGetterName()
+                                  : (*I)->getSetterName();
+      if (NextSel == Sel)
+        return *I;
+    }
+
+    llvm_unreachable("Marked as a property accessor but no property found!");
+  }
+
+  if (!CheckOverrides)
+    return 0;
+
+  typedef SmallVector<const ObjCMethodDecl *, 8> OverridesTy;
+  OverridesTy Overrides;
+  getOverriddenMethods(Overrides);
+  for (OverridesTy::const_iterator I = Overrides.begin(), E = Overrides.end();
+       I != E; ++I) {
+    if (const ObjCPropertyDecl *Prop = (*I)->findPropertyDecl(false))
+      return Prop;
+  }
+
+  return 0;
+
+}
+
 //===----------------------------------------------------------------------===//
 // ObjCInterfaceDecl
 //===----------------------------------------------------------------------===//
@@ -1081,6 +1328,20 @@ void ObjCProtocolDecl::startDefinition() {
     RD->Data = this->Data;
 }
 
+void ObjCProtocolDecl::collectPropertiesToImplement(PropertyMap &PM) const {
+  for (ObjCProtocolDecl::prop_iterator P = prop_begin(),
+      E = prop_end(); P != E; ++P) {
+    ObjCPropertyDecl *Prop = *P;
+    // Insert into PM if not there already.
+    PM.insert(std::make_pair(Prop->getIdentifier(), Prop));
+  }
+  // Scan through protocol's protocols.
+  for (ObjCProtocolDecl::protocol_iterator PI = protocol_begin(),
+      E = protocol_end(); PI != E; ++PI)
+    (*PI)->collectPropertiesToImplement(PM);
+}
+
+
 //===----------------------------------------------------------------------===//
 // ObjCCategoryDecl
 //===----------------------------------------------------------------------===//
diff --git a/lib/AST/DeclPrinter.cpp b/lib/AST/DeclPrinter.cpp
index 7f47604..386ad66 100644
--- a/lib/AST/DeclPrinter.cpp
+++ b/lib/AST/DeclPrinter.cpp
@@ -105,6 +105,8 @@ static QualType GetBaseType(QualType T) {
       break;
     else if (const PointerType* PTy = BaseType->getAs<PointerType>())
       BaseType = PTy->getPointeeType();
+    else if (const BlockPointerType *BPy = BaseType->getAs<BlockPointerType>())
+      BaseType = BPy->getPointeeType();
     else if (const ArrayType* ATy = dyn_cast<ArrayType>(BaseType))
       BaseType = ATy->getElementType();
     else if (const FunctionType* FTy = BaseType->getAs<FunctionType>())
@@ -189,6 +191,9 @@ raw_ostream& DeclPrinter::Indent(unsigned Indentation) {
 }
 
 void DeclPrinter::prettyPrintAttributes(Decl *D) {
+  if (Policy.SuppressAttributes)
+    return;
+  
   if (D->hasAttrs()) {
     AttrVec &Attrs = D->getAttrs();
     for (AttrVec::const_iterator i=Attrs.begin(), e=Attrs.end(); i!=e; ++i) {
@@ -220,6 +225,9 @@ void DeclPrinter::Print(AccessSpecifier AS) {
 //----------------------------------------------------------------------------
 
 void DeclPrinter::VisitDeclContext(DeclContext *DC, bool Indent) {
+  if (Policy.TerseOutput)
+    return;
+
   if (Indent)
     Indentation += Policy.Indentation;
 
@@ -459,7 +467,7 @@ void DeclPrinter::VisitFunctionDecl(FunctionDecl *D) {
           if (I)
             Proto += ", ";
 
-          Proto += FT->getExceptionType(I).getAsString(SubPolicy);;
+          Proto += FT->getExceptionType(I).getAsString(SubPolicy);
         }
       Proto += ")";
     } else if (FT && isNoexceptExceptionSpec(FT->getExceptionSpecType())) {
@@ -550,7 +558,7 @@ void DeclPrinter::VisitFunctionDecl(FunctionDecl *D) {
     Out << " = 0";
   else if (D->isDeletedAsWritten())
     Out << " = delete";
-  else if (D->doesThisDeclarationHaveABody()) {
+  else if (D->doesThisDeclarationHaveABody() && !Policy.TerseOutput) {
     if (!D->hasPrototype() && D->getNumParams()) {
       // This is a K&R function definition, so we need to print the
       // parameters.
@@ -621,13 +629,13 @@ void DeclPrinter::VisitVarDecl(VarDecl *D) {
       ImplicitInit = D->getInitStyle() == VarDecl::CallInit &&
           Construct->getNumArgs() == 0 && !Construct->isListInitialization();
     if (!ImplicitInit) {
-      if (D->getInitStyle() == VarDecl::CallInit)
+      if ((D->getInitStyle() == VarDecl::CallInit) && !isa<ParenListExpr>(Init))
         Out << "(";
       else if (D->getInitStyle() == VarDecl::CInit) {
         Out << " = ";
       }
       Init->printPretty(Out, 0, Policy, Indentation);
-      if (D->getInitStyle() == VarDecl::CallInit)
+      if ((D->getInitStyle() == VarDecl::CallInit) && !isa<ParenListExpr>(Init))
         Out << ")";
     }
   }
@@ -869,7 +877,7 @@ void DeclPrinter::VisitObjCMethodDecl(ObjCMethodDecl *OMD) {
   if (OMD->isVariadic())
       Out << ", ...";
 
-  if (OMD->getBody()) {
+  if (OMD->getBody() && !Policy.TerseOutput) {
     Out << ' ';
     OMD->getBody()->printPretty(Out, 0, Policy);
     Out << '\n';
diff --git a/lib/AST/DeclTemplate.cpp b/lib/AST/DeclTemplate.cpp
index a7e8999..a70983f 100644
--- a/lib/AST/DeclTemplate.cpp
+++ b/lib/AST/DeclTemplate.cpp
@@ -32,9 +32,25 @@ TemplateParameterList::TemplateParameterList(SourceLocation TemplateLoc,
                                              NamedDecl **Params, unsigned NumParams,
                                              SourceLocation RAngleLoc)
   : TemplateLoc(TemplateLoc), LAngleLoc(LAngleLoc), RAngleLoc(RAngleLoc),
-    NumParams(NumParams) {
-  for (unsigned Idx = 0; Idx < NumParams; ++Idx)
-    begin()[Idx] = Params[Idx];
+    NumParams(NumParams), ContainsUnexpandedParameterPack(false) {
+  assert(this->NumParams == NumParams && "Too many template parameters");
+  for (unsigned Idx = 0; Idx < NumParams; ++Idx) {
+    NamedDecl *P = Params[Idx];
+    begin()[Idx] = P;
+
+    if (!P->isTemplateParameterPack()) {
+      if (NonTypeTemplateParmDecl *NTTP = dyn_cast<NonTypeTemplateParmDecl>(P))
+        if (NTTP->getType()->containsUnexpandedParameterPack())
+          ContainsUnexpandedParameterPack = true;
+
+      if (TemplateTemplateParmDecl *TTP = dyn_cast<TemplateTemplateParmDecl>(P))
+        if (TTP->getTemplateParameters()->containsUnexpandedParameterPack())
+          ContainsUnexpandedParameterPack = true;
+
+      // FIXME: If a default argument contains an unexpanded parameter pack, the
+      // template parameter list does too.
+    }
+  }
 }
 
 TemplateParameterList *
@@ -577,6 +593,19 @@ SourceLocation NonTypeTemplateParmDecl::getDefaultArgumentLoc() const {
 
 void TemplateTemplateParmDecl::anchor() { }
 
+TemplateTemplateParmDecl::TemplateTemplateParmDecl(
+    DeclContext *DC, SourceLocation L, unsigned D, unsigned P,
+    IdentifierInfo *Id, TemplateParameterList *Params,
+    unsigned NumExpansions, TemplateParameterList * const *Expansions)
+  : TemplateDecl(TemplateTemplateParm, DC, L, Id, Params),
+    TemplateParmPosition(D, P), DefaultArgument(),
+    DefaultArgumentWasInherited(false), ParameterPack(true),
+    ExpandedParameterPack(true), NumExpandedParams(NumExpansions) {
+  if (Expansions)
+    std::memcpy(reinterpret_cast<void*>(this + 1), Expansions,
+                sizeof(TemplateParameterList*) * NumExpandedParams);
+}
+
 TemplateTemplateParmDecl *
 TemplateTemplateParmDecl::Create(const ASTContext &C, DeclContext *DC,
                                  SourceLocation L, unsigned D, unsigned P,
@@ -587,12 +616,35 @@ TemplateTemplateParmDecl::Create(const ASTContext &C, DeclContext *DC,
 }
 
 TemplateTemplateParmDecl *
+TemplateTemplateParmDecl::Create(const ASTContext &C, DeclContext *DC,
+                                 SourceLocation L, unsigned D, unsigned P,
+                                 IdentifierInfo *Id,
+                                 TemplateParameterList *Params,
+                            llvm::ArrayRef<TemplateParameterList*> Expansions) {
+  void *Mem = C.Allocate(sizeof(TemplateTemplateParmDecl) +
+                         sizeof(TemplateParameterList*) * Expansions.size());
+  return new (Mem) TemplateTemplateParmDecl(DC, L, D, P, Id, Params,
+                                            Expansions.size(),
+                                            Expansions.data());
+}
+
+TemplateTemplateParmDecl *
 TemplateTemplateParmDecl::CreateDeserialized(ASTContext &C, unsigned ID) {
   void *Mem = AllocateDeserializedDecl(C, ID, sizeof(TemplateTemplateParmDecl));
   return new (Mem) TemplateTemplateParmDecl(0, SourceLocation(), 0, 0, false,
                                             0, 0);
 }
 
+TemplateTemplateParmDecl *
+TemplateTemplateParmDecl::CreateDeserialized(ASTContext &C, unsigned ID,
+                                             unsigned NumExpansions) {
+  unsigned Size = sizeof(TemplateTemplateParmDecl) +
+                  sizeof(TemplateParameterList*) * NumExpansions;
+  void *Mem = AllocateDeserializedDecl(C, ID, Size);
+  return new (Mem) TemplateTemplateParmDecl(0, SourceLocation(), 0, 0, 0, 0,
+                                            NumExpansions, 0);
+}
+
 //===----------------------------------------------------------------------===//
 // TemplateArgumentList Implementation
 //===----------------------------------------------------------------------===//
@@ -712,13 +764,27 @@ ClassTemplateSpecializationDecl::getSpecializedTemplate() const {
 SourceRange
 ClassTemplateSpecializationDecl::getSourceRange() const {
   if (ExplicitInfo) {
-    SourceLocation Begin = getExternLoc();
-    if (Begin.isInvalid())
-      Begin = getTemplateKeywordLoc();
-    SourceLocation End = getRBraceLoc();
-    if (End.isInvalid())
-      End = getTypeAsWritten()->getTypeLoc().getEndLoc();
-    return SourceRange(Begin, End);
+    SourceLocation Begin = getTemplateKeywordLoc();
+    if (Begin.isValid()) {
+      // Here we have an explicit (partial) specialization or instantiation.
+      assert(getSpecializationKind() == TSK_ExplicitSpecialization ||
+             getSpecializationKind() == TSK_ExplicitInstantiationDeclaration ||
+             getSpecializationKind() == TSK_ExplicitInstantiationDefinition);
+      if (getExternLoc().isValid())
+        Begin = getExternLoc();
+      SourceLocation End = getRBraceLoc();
+      if (End.isInvalid())
+        End = getTypeAsWritten()->getTypeLoc().getEndLoc();
+      return SourceRange(Begin, End);
+    }
+    // An implicit instantiation of a class template partial specialization
+    // uses ExplicitInfo to record the TypeAsWritten, but the source
+    // locations should be retrieved from the instantiation pattern.
+    typedef ClassTemplatePartialSpecializationDecl CTPSDecl;
+    CTPSDecl *ctpsd = const_cast<CTPSDecl*>(cast<CTPSDecl>(this));
+    CTPSDecl *inst_from = ctpsd->getInstantiatedFromMember();
+    assert(inst_from != 0);
+    return inst_from->getSourceRange();
   }
   else {
     // No explicit info available.
diff --git a/lib/AST/DumpXML.cpp b/lib/AST/DumpXML.cpp
index 84f3fc4..5f43fbc 100644
--- a/lib/AST/DumpXML.cpp
+++ b/lib/AST/DumpXML.cpp
@@ -1,4 +1,4 @@
-//===--- DumpXML.cpp - Detailed XML dumping ---------------------*- C++ -*-===//
+//===--- DumpXML.cpp - Detailed XML dumping -------------------------------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -64,6 +64,8 @@ template <class Impl> struct XMLDeclVisitor {
   static_cast<Impl*>(this)->NAME(static_cast<CLASS*>(D))
 
   void dispatch(Decl *D) {
+    if (D->isUsed())
+      static_cast<Impl*>(this)->set("used", "1");
     switch (D->getKind()) {
 #define DECL(DERIVED, BASE) \
       case Decl::DERIVED: \
@@ -316,12 +318,12 @@ struct XMLDumper : public XMLDeclVisitor<XMLDumper>,
     }
     case TemplateArgument::Template:
     case TemplateArgument::TemplateExpansion:
+    case TemplateArgument::NullPtr:
       // FIXME: Implement!
       break;
         
     case TemplateArgument::Declaration: {
-      if (Decl *D = A.getAsDecl())
-        visitDeclRef(D);
+      visitDeclRef(A.getAsDecl());
       break;
     }
     case TemplateArgument::Integral: {
@@ -841,7 +843,7 @@ struct XMLDumper : public XMLDeclVisitor<XMLDumper>,
 
     setFlag("instance", D->isInstanceMethod());
     setFlag("variadic", D->isVariadic());
-    setFlag("synthesized", D->isSynthesized());
+    setFlag("property_accessor", D->isPropertyAccessor());
     setFlag("defined", D->isDefined());
     setFlag("related_result_type", D->hasRelatedResultType());
   }
@@ -920,6 +922,7 @@ struct XMLDumper : public XMLDeclVisitor<XMLDumper>,
     case CC_X86Pascal: return set("cc", "x86_pascal");
     case CC_AAPCS: return set("cc", "aapcs");
     case CC_AAPCS_VFP: return set("cc", "aapcs_vfp");
+    case CC_PnaclCall: return set("cc", "pnaclcall");
     }
   }
 
@@ -974,6 +977,16 @@ struct XMLDumper : public XMLDeclVisitor<XMLDumper>,
     setFlag("const", T->isConst());
     setFlag("volatile", T->isVolatile());
     setFlag("restrict", T->isRestrict());
+    switch (T->getExceptionSpecType()) {
+    case EST_None: break;
+    case EST_DynamicNone: set("exception_spec", "throw()"); break;
+    case EST_Dynamic: set("exception_spec", "throw(T)"); break;
+    case EST_MSAny: set("exception_spec", "throw(...)"); break;
+    case EST_BasicNoexcept: set("exception_spec", "noexcept"); break;
+    case EST_ComputedNoexcept: set("exception_spec", "noexcept(expr)"); break;
+    case EST_Unevaluated: set("exception_spec", "unevaluated"); break;
+    case EST_Uninstantiated: set("exception_spec", "uninstantiated"); break;
+    }
   }
   void visitFunctionProtoTypeChildren(FunctionProtoType *T) {
     push("parameters");
@@ -1023,7 +1036,7 @@ struct XMLDumper : public XMLDeclVisitor<XMLDumper>,
 }
 
 void Decl::dumpXML() const {
-  dump(llvm::errs());
+  dumpXML(llvm::errs());
 }
 
 void Decl::dumpXML(raw_ostream &out) const {
diff --git a/lib/AST/Expr.cpp b/lib/AST/Expr.cpp
index 24361ef..f3a2e05 100644
--- a/lib/AST/Expr.cpp
+++ b/lib/AST/Expr.cpp
@@ -48,6 +48,75 @@ const CXXRecordDecl *Expr::getBestDynamicClassType() const {
   return cast<CXXRecordDecl>(D);
 }
 
+const Expr *
+Expr::skipRValueSubobjectAdjustments(
+                     SmallVectorImpl<SubobjectAdjustment> &Adjustments) const  {
+  const Expr *E = this;
+  while (true) {
+    E = E->IgnoreParens();
+
+    if (const CastExpr *CE = dyn_cast<CastExpr>(E)) {
+      if ((CE->getCastKind() == CK_DerivedToBase ||
+           CE->getCastKind() == CK_UncheckedDerivedToBase) &&
+          E->getType()->isRecordType()) {
+        E = CE->getSubExpr();
+        CXXRecordDecl *Derived
+          = cast<CXXRecordDecl>(E->getType()->getAs<RecordType>()->getDecl());
+        Adjustments.push_back(SubobjectAdjustment(CE, Derived));
+        continue;
+      }
+
+      if (CE->getCastKind() == CK_NoOp) {
+        E = CE->getSubExpr();
+        continue;
+      }
+    } else if (const MemberExpr *ME = dyn_cast<MemberExpr>(E)) {
+      if (!ME->isArrow() && ME->getBase()->isRValue()) {
+        assert(ME->getBase()->getType()->isRecordType());
+        if (FieldDecl *Field = dyn_cast<FieldDecl>(ME->getMemberDecl())) {
+          E = ME->getBase();
+          Adjustments.push_back(SubobjectAdjustment(Field));
+          continue;
+        }
+      }
+    } else if (const BinaryOperator *BO = dyn_cast<BinaryOperator>(E)) {
+      if (BO->isPtrMemOp()) {
+        assert(BO->getRHS()->isRValue());
+        E = BO->getLHS();
+        const MemberPointerType *MPT =
+          BO->getRHS()->getType()->getAs<MemberPointerType>();
+        Adjustments.push_back(SubobjectAdjustment(MPT, BO->getRHS()));
+      }
+    }
+
+    // Nothing changed.
+    break;
+  }
+  return E;
+}
+
+const Expr *
+Expr::findMaterializedTemporary(const MaterializeTemporaryExpr *&MTE) const {
+  const Expr *E = this;
+  // Look through single-element init lists that claim to be lvalues. They're
+  // just syntactic wrappers in this case.
+  if (const InitListExpr *ILE = dyn_cast<InitListExpr>(E)) {
+    if (ILE->getNumInits() == 1 && ILE->isGLValue())
+      E = ILE->getInit(0);
+  }
+
+  // Look through expressions for materialized temporaries (for now).
+  if (const MaterializeTemporaryExpr *M
+      = dyn_cast<MaterializeTemporaryExpr>(E)) {
+    MTE = M;
+    E = M->GetTemporaryExpr();
+  }
+
+  if (const CXXDefaultArgExpr *DAE = dyn_cast<CXXDefaultArgExpr>(E))
+    E = DAE->getExpr();
+  return E;
+}
+
 /// isKnownToHaveBooleanValue - Return true if this is an integer expression
 /// that is known to return 0 or 1.  This happens for _Bool/bool expressions
 /// but also int expressions which are produced by things like comparisons in
@@ -784,19 +853,19 @@ void StringLiteral::setString(ASTContext &C, StringRef Str,
   switch(CharByteWidth) {
     case 1: {
       char *AStrData = new (C) char[Length];
-      std::memcpy(AStrData,Str.data(),Str.size());
+      std::memcpy(AStrData,Str.data(),Length*sizeof(*AStrData));
       StrData.asChar = AStrData;
       break;
     }
     case 2: {
       uint16_t *AStrData = new (C) uint16_t[Length];
-      std::memcpy(AStrData,Str.data(),Str.size());
+      std::memcpy(AStrData,Str.data(),Length*sizeof(*AStrData));
       StrData.asUInt16 = AStrData;
       break;
     }
     case 4: {
       uint32_t *AStrData = new (C) uint32_t[Length];
-      std::memcpy(AStrData,Str.data(),Str.size());
+      std::memcpy(AStrData,Str.data(),Length*sizeof(*AStrData));
       StrData.asUInt32 = AStrData;
       break;
     }
@@ -869,7 +938,7 @@ getLocationOfByte(unsigned ByteNo, const SourceManager &SM,
 
 /// getOpcodeStr - Turn an Opcode enum value into the punctuation char it
 /// corresponds to, e.g. "sizeof" or "[pre]++".
-const char *UnaryOperator::getOpcodeStr(Opcode Op) {
+StringRef UnaryOperator::getOpcodeStr(Opcode Op) {
   switch (Op) {
   case UO_PostInc: return "++";
   case UO_PostDec: return "--";
@@ -923,18 +992,18 @@ OverloadedOperatorKind UnaryOperator::getOverloadedOperator(Opcode Opc) {
 //===----------------------------------------------------------------------===//
 
 CallExpr::CallExpr(ASTContext& C, StmtClass SC, Expr *fn, unsigned NumPreArgs,
-                   Expr **args, unsigned numargs, QualType t, ExprValueKind VK,
+                   ArrayRef<Expr*> args, QualType t, ExprValueKind VK,
                    SourceLocation rparenloc)
   : Expr(SC, t, VK, OK_Ordinary,
          fn->isTypeDependent(),
          fn->isValueDependent(),
          fn->isInstantiationDependent(),
          fn->containsUnexpandedParameterPack()),
-    NumArgs(numargs) {
+    NumArgs(args.size()) {
 
-  SubExprs = new (C) Stmt*[numargs+PREARGS_START+NumPreArgs];
+  SubExprs = new (C) Stmt*[args.size()+PREARGS_START+NumPreArgs];
   SubExprs[FN] = fn;
-  for (unsigned i = 0; i != numargs; ++i) {
+  for (unsigned i = 0; i != args.size(); ++i) {
     if (args[i]->isTypeDependent())
       ExprBits.TypeDependent = true;
     if (args[i]->isValueDependent())
@@ -951,18 +1020,18 @@ CallExpr::CallExpr(ASTContext& C, StmtClass SC, Expr *fn, unsigned NumPreArgs,
   RParenLoc = rparenloc;
 }
 
-CallExpr::CallExpr(ASTContext& C, Expr *fn, Expr **args, unsigned numargs,
+CallExpr::CallExpr(ASTContext& C, Expr *fn, ArrayRef<Expr*> args,
                    QualType t, ExprValueKind VK, SourceLocation rparenloc)
   : Expr(CallExprClass, t, VK, OK_Ordinary,
          fn->isTypeDependent(),
          fn->isValueDependent(),
          fn->isInstantiationDependent(),
          fn->containsUnexpandedParameterPack()),
-    NumArgs(numargs) {
+    NumArgs(args.size()) {
 
-  SubExprs = new (C) Stmt*[numargs+PREARGS_START];
+  SubExprs = new (C) Stmt*[args.size()+PREARGS_START];
   SubExprs[FN] = fn;
-  for (unsigned i = 0; i != numargs; ++i) {
+  for (unsigned i = 0; i != args.size(); ++i) {
     if (args[i]->isTypeDependent())
       ExprBits.TypeDependent = true;
     if (args[i]->isValueDependent())
@@ -1123,15 +1192,15 @@ SourceLocation CallExpr::getLocEnd() const {
 OffsetOfExpr *OffsetOfExpr::Create(ASTContext &C, QualType type, 
                                    SourceLocation OperatorLoc,
                                    TypeSourceInfo *tsi, 
-                                   OffsetOfNode* compsPtr, unsigned numComps, 
-                                   Expr** exprsPtr, unsigned numExprs,
+                                   ArrayRef<OffsetOfNode> comps,
+                                   ArrayRef<Expr*> exprs,
                                    SourceLocation RParenLoc) {
   void *Mem = C.Allocate(sizeof(OffsetOfExpr) +
-                         sizeof(OffsetOfNode) * numComps + 
-                         sizeof(Expr*) * numExprs);
+                         sizeof(OffsetOfNode) * comps.size() +
+                         sizeof(Expr*) * exprs.size());
 
-  return new (Mem) OffsetOfExpr(C, type, OperatorLoc, tsi, compsPtr, numComps,
-                                exprsPtr, numExprs, RParenLoc);
+  return new (Mem) OffsetOfExpr(C, type, OperatorLoc, tsi, comps, exprs,
+                                RParenLoc);
 }
 
 OffsetOfExpr *OffsetOfExpr::CreateEmpty(ASTContext &C,
@@ -1144,8 +1213,7 @@ OffsetOfExpr *OffsetOfExpr::CreateEmpty(ASTContext &C,
 
 OffsetOfExpr::OffsetOfExpr(ASTContext &C, QualType type, 
                            SourceLocation OperatorLoc, TypeSourceInfo *tsi,
-                           OffsetOfNode* compsPtr, unsigned numComps, 
-                           Expr** exprsPtr, unsigned numExprs,
+                           ArrayRef<OffsetOfNode> comps, ArrayRef<Expr*> exprs,
                            SourceLocation RParenLoc)
   : Expr(OffsetOfExprClass, type, VK_RValue, OK_Ordinary,
          /*TypeDependent=*/false, 
@@ -1153,19 +1221,19 @@ OffsetOfExpr::OffsetOfExpr(ASTContext &C, QualType type,
          tsi->getType()->isInstantiationDependentType(),
          tsi->getType()->containsUnexpandedParameterPack()),
     OperatorLoc(OperatorLoc), RParenLoc(RParenLoc), TSInfo(tsi), 
-    NumComps(numComps), NumExprs(numExprs) 
+    NumComps(comps.size()), NumExprs(exprs.size())
 {
-  for(unsigned i = 0; i < numComps; ++i) {
-    setComponent(i, compsPtr[i]);
+  for (unsigned i = 0; i != comps.size(); ++i) {
+    setComponent(i, comps[i]);
   }
   
-  for(unsigned i = 0; i < numExprs; ++i) {
-    if (exprsPtr[i]->isTypeDependent() || exprsPtr[i]->isValueDependent())
+  for (unsigned i = 0; i != exprs.size(); ++i) {
+    if (exprs[i]->isTypeDependent() || exprs[i]->isValueDependent())
       ExprBits.ValueDependent = true;
-    if (exprsPtr[i]->containsUnexpandedParameterPack())
+    if (exprs[i]->containsUnexpandedParameterPack())
       ExprBits.ContainsUnexpandedParameterPack = true;
 
-    setIndexExpr(i, exprsPtr[i]);
+    setIndexExpr(i, exprs[i]);
   }
 }
 
@@ -1259,9 +1327,12 @@ SourceLocation MemberExpr::getLocStart() const {
   return MemberLoc;
 }
 SourceLocation MemberExpr::getLocEnd() const {
+  SourceLocation EndLoc = getMemberNameInfo().getEndLoc();
   if (hasExplicitTemplateArgs())
-    return getRAngleLoc();
-  return getMemberNameInfo().getEndLoc();
+    EndLoc = getRAngleLoc();
+  else if (EndLoc.isInvalid())
+    EndLoc = getBase()->getLocEnd();
+  return EndLoc;
 }
 
 void CastExpr::CheckCastConsistency() const {
@@ -1311,12 +1382,16 @@ void CastExpr::CheckCastConsistency() const {
     assert(getType()->isBlockPointerType());
     assert(getSubExpr()->getType()->isBlockPointerType());
     goto CheckNoBasePath;
-      
+
+  case CK_FunctionToPointerDecay:
+    assert(getType()->isPointerType());
+    assert(getSubExpr()->getType()->isFunctionType());
+    goto CheckNoBasePath;
+
   // These should not have an inheritance path.
   case CK_Dynamic:
   case CK_ToUnion:
   case CK_ArrayToPointerDecay:
-  case CK_FunctionToPointerDecay:
   case CK_NullToMemberPointer:
   case CK_NullToPointer:
   case CK_ConstructorConversion:
@@ -1357,6 +1432,7 @@ void CastExpr::CheckCastConsistency() const {
   case CK_IntegralComplexToBoolean:
   case CK_LValueBitCast:            // -> bool&
   case CK_UserDefinedConversion:    // operator bool()
+  case CK_BuiltinFnToFnPtr:
   CheckNoBasePath:
     assert(path_empty() && "Cast kind should not have a base path!");
     break;
@@ -1469,6 +1545,8 @@ const char *CastExpr::getCastKindName() const {
     return "NonAtomicToAtomic";
   case CK_CopyAndAutoreleaseBlockObject:
     return "CopyAndAutoreleaseBlockObject";
+  case CK_BuiltinFnToFnPtr:
+    return "BuiltinFnToFnPtr";
   }
 
   llvm_unreachable("Unhandled cast kind!");
@@ -1564,7 +1642,7 @@ CStyleCastExpr *CStyleCastExpr::CreateEmpty(ASTContext &C, unsigned PathSize) {
 
 /// getOpcodeStr - Turn an Opcode enum value into the punctuation char it
 /// corresponds to, e.g. "<<=".
-const char *BinaryOperator::getOpcodeStr(Opcode Op) {
+StringRef BinaryOperator::getOpcodeStr(Opcode Op) {
   switch (Op) {
   case BO_PtrMemD:   return ".*";
   case BO_PtrMemI:   return "->*";
@@ -1666,16 +1744,15 @@ OverloadedOperatorKind BinaryOperator::getOverloadedOperator(Opcode Opc) {
 }
 
 InitListExpr::InitListExpr(ASTContext &C, SourceLocation lbraceloc,
-                           Expr **initExprs, unsigned numInits,
-                           SourceLocation rbraceloc)
+                           ArrayRef<Expr*> initExprs, SourceLocation rbraceloc)
   : Expr(InitListExprClass, QualType(), VK_RValue, OK_Ordinary, false, false,
          false, false),
-    InitExprs(C, numInits),
-    LBraceLoc(lbraceloc), RBraceLoc(rbraceloc), SyntacticForm(0)
+    InitExprs(C, initExprs.size()),
+    LBraceLoc(lbraceloc), RBraceLoc(rbraceloc), AltForm(0, true)
 {
   sawArrayRangeDesignator(false);
   setInitializesStdInitializerList(false);
-  for (unsigned I = 0; I != numInits; ++I) {
+  for (unsigned I = 0; I != initExprs.size(); ++I) {
     if (initExprs[I]->isTypeDependent())
       ExprBits.TypeDependent = true;
     if (initExprs[I]->isValueDependent())
@@ -1686,7 +1763,7 @@ InitListExpr::InitListExpr(ASTContext &C, SourceLocation lbraceloc,
       ExprBits.ContainsUnexpandedParameterPack = true;
   }
       
-  InitExprs.insert(C, InitExprs.end(), initExprs, initExprs+numInits);
+  InitExprs.insert(C, InitExprs.end(), initExprs.begin(), initExprs.end());
 }
 
 void InitListExpr::reserveInits(ASTContext &C, unsigned NumInits) {
@@ -1723,15 +1800,15 @@ void InitListExpr::setArrayFiller(Expr *filler) {
 bool InitListExpr::isStringLiteralInit() const {
   if (getNumInits() != 1)
     return false;
-  const ConstantArrayType *CAT = dyn_cast<ConstantArrayType>(getType());
-  if (!CAT || !CAT->getElementType()->isIntegerType())
+  const ArrayType *AT = getType()->getAsArrayTypeUnsafe();
+  if (!AT || !AT->getElementType()->isIntegerType())
     return false;
-  const Expr *Init = getInit(0)->IgnoreParenImpCasts();
+  const Expr *Init = getInit(0)->IgnoreParens();
   return isa<StringLiteral>(Init) || isa<ObjCEncodeExpr>(Init);
 }
 
 SourceRange InitListExpr::getSourceRange() const {
-  if (SyntacticForm)
+  if (InitListExpr *SyntacticForm = getSyntacticForm())
     return SyntacticForm->getSourceRange();
   SourceLocation Beg = LBraceLoc, End = RBraceLoc;
   if (Beg.isInvalid()) {
@@ -1945,6 +2022,11 @@ bool Expr::isUnusedResultAWarning(const Expr *&WarnE, SourceLocation &Loc,
     return false;
   }
 
+  // If we don't know precisely what we're looking at, let's not warn.
+  case UnresolvedLookupExprClass:
+  case CXXUnresolvedConstructExprClass:
+    return false;
+
   case CXXTemporaryObjectExprClass:
   case CXXConstructExprClass:
     return false;
@@ -2014,6 +2096,7 @@ bool Expr::isUnusedResultAWarning(const Expr *&WarnE, SourceLocation &Loc,
     R1 = getSourceRange();
     return true;
   }
+  case CXXFunctionalCastExprClass:
   case CStyleCastExprClass: {
     // Ignore an explicit cast to void unless the operand is a non-trivial
     // volatile lvalue.
@@ -2032,6 +2115,10 @@ bool Expr::isUnusedResultAWarning(const Expr *&WarnE, SourceLocation &Loc,
       return false;
     }
 
+    // Ignore casts within macro expansions.
+    if (getExprLoc().isMacroID())
+      return CE->getSubExpr()->isUnusedResultAWarning(WarnE, Loc, R1, R2, Ctx);
+
     // If this is a cast to a constructor conversion, check the operand.
     // Otherwise, the result of the cast is unused.
     if (CE->getCastKind() == CK_ConstructorConversion)
@@ -2641,6 +2728,7 @@ bool Expr::HasSideEffects(const ASTContext &Ctx) const {
   case UnresolvedMemberExprClass:
   case PackExpansionExprClass:
   case SubstNonTypeTemplateParmPackExprClass:
+  case FunctionParmPackExprClass:
     llvm_unreachable("shouldn't see dependent / unresolved nodes here");
 
   case DeclRefExprClass:
@@ -2995,6 +3083,24 @@ const ObjCPropertyRefExpr *Expr::getObjCProperty() const {
   return cast<ObjCPropertyRefExpr>(E);
 }
 
+bool Expr::isObjCSelfExpr() const {
+  const Expr *E = IgnoreParenImpCasts();
+
+  const DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(E);
+  if (!DRE)
+    return false;
+
+  const ImplicitParamDecl *Param = dyn_cast<ImplicitParamDecl>(DRE->getDecl());
+  if (!Param)
+    return false;
+
+  const ObjCMethodDecl *M = dyn_cast<ObjCMethodDecl>(Param->getDeclContext());
+  if (!M)
+    return false;
+
+  return M->getSelfDecl() == Param;
+}
+
 FieldDecl *Expr::getBitField() {
   Expr *E = this->IgnoreParens();
 
@@ -3339,33 +3445,29 @@ Selector ObjCMessageExpr::getSelector() const {
   return Selector(SelectorOrMethod); 
 }
 
-ObjCInterfaceDecl *ObjCMessageExpr::getReceiverInterface() const {
+QualType ObjCMessageExpr::getReceiverType() const {
   switch (getReceiverKind()) {
   case Instance:
-    if (const ObjCObjectPointerType *Ptr
-          = getInstanceReceiver()->getType()->getAs<ObjCObjectPointerType>())
-      return Ptr->getInterfaceDecl();
-    break;
-
+    return getInstanceReceiver()->getType();
   case Class:
-    if (const ObjCObjectType *Ty
-          = getClassReceiver()->getAs<ObjCObjectType>())
-      return Ty->getInterface();
-    break;
-
+    return getClassReceiver();
   case SuperInstance:
-    if (const ObjCObjectPointerType *Ptr
-          = getSuperType()->getAs<ObjCObjectPointerType>())
-      return Ptr->getInterfaceDecl();
-    break;
-
   case SuperClass:
-    if (const ObjCObjectType *Iface
-          = getSuperType()->getAs<ObjCObjectType>())
-      return Iface->getInterface();
-    break;
+    return getSuperType();
   }
 
+  llvm_unreachable("unexpected receiver kind");
+}
+
+ObjCInterfaceDecl *ObjCMessageExpr::getReceiverInterface() const {
+  QualType T = getReceiverType();
+
+  if (const ObjCObjectPointerType *Ptr = T->getAs<ObjCObjectPointerType>())
+    return Ptr->getInterfaceDecl();
+
+  if (const ObjCObjectType *Ty = T->getAs<ObjCObjectType>())
+    return Ty->getInterface();
+
   return 0;
 }
 
@@ -3386,17 +3488,17 @@ bool ChooseExpr::isConditionTrue(const ASTContext &C) const {
   return getCond()->EvaluateKnownConstInt(C) != 0;
 }
 
-ShuffleVectorExpr::ShuffleVectorExpr(ASTContext &C, Expr **args, unsigned nexpr,
+ShuffleVectorExpr::ShuffleVectorExpr(ASTContext &C, ArrayRef<Expr*> args,
                                      QualType Type, SourceLocation BLoc,
                                      SourceLocation RP) 
    : Expr(ShuffleVectorExprClass, Type, VK_RValue, OK_Ordinary,
           Type->isDependentType(), Type->isDependentType(),
           Type->isInstantiationDependentType(),
           Type->containsUnexpandedParameterPack()),
-     BuiltinLoc(BLoc), RParenLoc(RP), NumExprs(nexpr) 
+     BuiltinLoc(BLoc), RParenLoc(RP), NumExprs(args.size())
 {
-  SubExprs = new (C) Stmt*[nexpr];
-  for (unsigned i = 0; i < nexpr; i++) {
+  SubExprs = new (C) Stmt*[args.size()];
+  for (unsigned i = 0; i != args.size(); i++) {
     if (args[i]->isTypeDependent())
       ExprBits.TypeDependent = true;
     if (args[i]->isValueDependent())
@@ -3421,8 +3523,9 @@ void ShuffleVectorExpr::setExprs(ASTContext &C, Expr ** Exprs,
 
 GenericSelectionExpr::GenericSelectionExpr(ASTContext &Context,
                                SourceLocation GenericLoc, Expr *ControllingExpr,
-                               TypeSourceInfo **AssocTypes, Expr **AssocExprs,
-                               unsigned NumAssocs, SourceLocation DefaultLoc,
+                               ArrayRef<TypeSourceInfo*> AssocTypes,
+                               ArrayRef<Expr*> AssocExprs,
+                               SourceLocation DefaultLoc,
                                SourceLocation RParenLoc,
                                bool ContainsUnexpandedParameterPack,
                                unsigned ResultIndex)
@@ -3434,19 +3537,21 @@ GenericSelectionExpr::GenericSelectionExpr(ASTContext &Context,
          AssocExprs[ResultIndex]->isValueDependent(),
          AssocExprs[ResultIndex]->isInstantiationDependent(),
          ContainsUnexpandedParameterPack),
-    AssocTypes(new (Context) TypeSourceInfo*[NumAssocs]),
-    SubExprs(new (Context) Stmt*[END_EXPR+NumAssocs]), NumAssocs(NumAssocs),
-    ResultIndex(ResultIndex), GenericLoc(GenericLoc), DefaultLoc(DefaultLoc),
-    RParenLoc(RParenLoc) {
+    AssocTypes(new (Context) TypeSourceInfo*[AssocTypes.size()]),
+    SubExprs(new (Context) Stmt*[END_EXPR+AssocExprs.size()]),
+    NumAssocs(AssocExprs.size()), ResultIndex(ResultIndex),
+    GenericLoc(GenericLoc), DefaultLoc(DefaultLoc), RParenLoc(RParenLoc) {
   SubExprs[CONTROLLING] = ControllingExpr;
-  std::copy(AssocTypes, AssocTypes+NumAssocs, this->AssocTypes);
-  std::copy(AssocExprs, AssocExprs+NumAssocs, SubExprs+END_EXPR);
+  assert(AssocTypes.size() == AssocExprs.size());
+  std::copy(AssocTypes.begin(), AssocTypes.end(), this->AssocTypes);
+  std::copy(AssocExprs.begin(), AssocExprs.end(), SubExprs+END_EXPR);
 }
 
 GenericSelectionExpr::GenericSelectionExpr(ASTContext &Context,
                                SourceLocation GenericLoc, Expr *ControllingExpr,
-                               TypeSourceInfo **AssocTypes, Expr **AssocExprs,
-                               unsigned NumAssocs, SourceLocation DefaultLoc,
+                               ArrayRef<TypeSourceInfo*> AssocTypes,
+                               ArrayRef<Expr*> AssocExprs,
+                               SourceLocation DefaultLoc,
                                SourceLocation RParenLoc,
                                bool ContainsUnexpandedParameterPack)
   : Expr(GenericSelectionExprClass,
@@ -3457,13 +3562,14 @@ GenericSelectionExpr::GenericSelectionExpr(ASTContext &Context,
          /*isValueDependent=*/true,
          /*isInstantiationDependent=*/true,
          ContainsUnexpandedParameterPack),
-    AssocTypes(new (Context) TypeSourceInfo*[NumAssocs]),
-    SubExprs(new (Context) Stmt*[END_EXPR+NumAssocs]), NumAssocs(NumAssocs),
-    ResultIndex(-1U), GenericLoc(GenericLoc), DefaultLoc(DefaultLoc),
-    RParenLoc(RParenLoc) {
+    AssocTypes(new (Context) TypeSourceInfo*[AssocTypes.size()]),
+    SubExprs(new (Context) Stmt*[END_EXPR+AssocExprs.size()]),
+    NumAssocs(AssocExprs.size()), ResultIndex(-1U), GenericLoc(GenericLoc),
+    DefaultLoc(DefaultLoc), RParenLoc(RParenLoc) {
   SubExprs[CONTROLLING] = ControllingExpr;
-  std::copy(AssocTypes, AssocTypes+NumAssocs, this->AssocTypes);
-  std::copy(AssocExprs, AssocExprs+NumAssocs, SubExprs+END_EXPR);
+  assert(AssocTypes.size() == AssocExprs.size());
+  std::copy(AssocTypes.begin(), AssocTypes.end(), this->AssocTypes);
+  std::copy(AssocExprs.begin(), AssocExprs.end(), SubExprs+END_EXPR);
 }
 
 //===----------------------------------------------------------------------===//
@@ -3483,8 +3589,7 @@ DesignatedInitExpr::DesignatedInitExpr(ASTContext &C, QualType Ty,
                                        const Designator *Designators,
                                        SourceLocation EqualOrColonLoc,
                                        bool GNUSyntax,
-                                       Expr **IndexExprs,
-                                       unsigned NumIndexExprs,
+                                       ArrayRef<Expr*> IndexExprs,
                                        Expr *Init)
   : Expr(DesignatedInitExprClass, Ty,
          Init->getValueKind(), Init->getObjectKind(),
@@ -3492,7 +3597,7 @@ DesignatedInitExpr::DesignatedInitExpr(ASTContext &C, QualType Ty,
          Init->isInstantiationDependent(),
          Init->containsUnexpandedParameterPack()),
     EqualOrColonLoc(EqualOrColonLoc), GNUSyntax(GNUSyntax),
-    NumDesignators(NumDesignators), NumSubExprs(NumIndexExprs + 1) {
+    NumDesignators(NumDesignators), NumSubExprs(IndexExprs.size() + 1) {
   this->Designators = new (C) Designator[NumDesignators];
 
   // Record the initializer itself.
@@ -3542,20 +3647,20 @@ DesignatedInitExpr::DesignatedInitExpr(ASTContext &C, QualType Ty,
     }
   }
 
-  assert(IndexIdx == NumIndexExprs && "Wrong number of index expressions");
+  assert(IndexIdx == IndexExprs.size() && "Wrong number of index expressions");
 }
 
 DesignatedInitExpr *
 DesignatedInitExpr::Create(ASTContext &C, Designator *Designators,
                            unsigned NumDesignators,
-                           Expr **IndexExprs, unsigned NumIndexExprs,
+                           ArrayRef<Expr*> IndexExprs,
                            SourceLocation ColonOrEqualLoc,
                            bool UsesColonSyntax, Expr *Init) {
   void *Mem = C.Allocate(sizeof(DesignatedInitExpr) +
-                         sizeof(Stmt *) * (NumIndexExprs + 1), 8);
+                         sizeof(Stmt *) * (IndexExprs.size() + 1), 8);
   return new (Mem) DesignatedInitExpr(C, C.VoidTy, NumDesignators, Designators,
                                       ColonOrEqualLoc, UsesColonSyntax,
-                                      IndexExprs, NumIndexExprs, Init);
+                                      IndexExprs, Init);
 }
 
 DesignatedInitExpr *DesignatedInitExpr::CreateEmpty(ASTContext &C,
@@ -3651,13 +3756,13 @@ void DesignatedInitExpr::ExpandDesignator(ASTContext &C, unsigned Idx,
 }
 
 ParenListExpr::ParenListExpr(ASTContext& C, SourceLocation lparenloc,
-                             Expr **exprs, unsigned nexprs,
+                             ArrayRef<Expr*> exprs,
                              SourceLocation rparenloc)
   : Expr(ParenListExprClass, QualType(), VK_RValue, OK_Ordinary,
          false, false, false, false),
-    NumExprs(nexprs), LParenLoc(lparenloc), RParenLoc(rparenloc) {
-  Exprs = new (C) Stmt*[nexprs];
-  for (unsigned i = 0; i != nexprs; ++i) {
+    NumExprs(exprs.size()), LParenLoc(lparenloc), RParenLoc(rparenloc) {
+  Exprs = new (C) Stmt*[exprs.size()];
+  for (unsigned i = 0; i != exprs.size(); ++i) {
     if (exprs[i]->isTypeDependent())
       ExprBits.TypeDependent = true;
     if (exprs[i]->isValueDependent())
@@ -3902,14 +4007,14 @@ ObjCSubscriptRefExpr *ObjCSubscriptRefExpr::Create(ASTContext &C,
                                         getMethod, setMethod, RB);
 }
 
-AtomicExpr::AtomicExpr(SourceLocation BLoc, Expr **args, unsigned nexpr,
+AtomicExpr::AtomicExpr(SourceLocation BLoc, ArrayRef<Expr*> args,
                        QualType t, AtomicOp op, SourceLocation RP)
   : Expr(AtomicExprClass, t, VK_RValue, OK_Ordinary,
          false, false, false, false),
-    NumSubExprs(nexpr), BuiltinLoc(BLoc), RParenLoc(RP), Op(op)
+    NumSubExprs(args.size()), BuiltinLoc(BLoc), RParenLoc(RP), Op(op)
 {
-  assert(nexpr == getNumSubExprs(op) && "wrong number of subexpressions");
-  for (unsigned i = 0; i < nexpr; i++) {
+  assert(args.size() == getNumSubExprs(op) && "wrong number of subexpressions");
+  for (unsigned i = 0; i != args.size(); i++) {
     if (args[i]->isTypeDependent())
       ExprBits.TypeDependent = true;
     if (args[i]->isValueDependent())
diff --git a/lib/AST/ExprCXX.cpp b/lib/AST/ExprCXX.cpp
index 3fa49e0..55722a2 100644
--- a/lib/AST/ExprCXX.cpp
+++ b/lib/AST/ExprCXX.cpp
@@ -51,6 +51,26 @@ QualType CXXUuidofExpr::getTypeOperand() const {
                                                         .getUnqualifiedType();
 }
 
+// static
+UuidAttr *CXXUuidofExpr::GetUuidAttrOfType(QualType QT) {
+  // Optionally remove one level of pointer, reference or array indirection.
+  const Type *Ty = QT.getTypePtr();
+  if (QT->isPointerType() || QT->isReferenceType())
+    Ty = QT->getPointeeType().getTypePtr();
+  else if (QT->isArrayType())
+    Ty = cast<ArrayType>(QT)->getElementType().getTypePtr();
+
+  // Loop all record redeclaration looking for an uuid attribute.
+  CXXRecordDecl *RD = Ty->getAsCXXRecordDecl();
+  for (CXXRecordDecl::redecl_iterator I = RD->redecls_begin(),
+       E = RD->redecls_end(); I != E; ++I) {
+    if (UuidAttr *Uuid = I->getAttr<UuidAttr>())
+      return Uuid;
+  }
+
+  return 0;
+}
+
 // CXXScalarValueInitExpr
 SourceRange CXXScalarValueInitExpr::getSourceRange() const {
   SourceLocation Start = RParenLoc;
@@ -63,24 +83,24 @@ SourceRange CXXScalarValueInitExpr::getSourceRange() const {
 CXXNewExpr::CXXNewExpr(ASTContext &C, bool globalNew, FunctionDecl *operatorNew,
                        FunctionDecl *operatorDelete,
                        bool usualArrayDeleteWantsSize,
-                       Expr **placementArgs, unsigned numPlaceArgs,
+                       ArrayRef<Expr*> placementArgs,
                        SourceRange typeIdParens, Expr *arraySize,
                        InitializationStyle initializationStyle,
                        Expr *initializer, QualType ty,
                        TypeSourceInfo *allocatedTypeInfo,
-                       SourceLocation startLoc, SourceRange directInitRange)
+                       SourceRange Range, SourceRange directInitRange)
   : Expr(CXXNewExprClass, ty, VK_RValue, OK_Ordinary,
          ty->isDependentType(), ty->isDependentType(),
          ty->isInstantiationDependentType(),
          ty->containsUnexpandedParameterPack()),
     SubExprs(0), OperatorNew(operatorNew), OperatorDelete(operatorDelete),
     AllocatedTypeInfo(allocatedTypeInfo), TypeIdParens(typeIdParens),
-    StartLoc(startLoc), DirectInitRange(directInitRange),
+    Range(Range), DirectInitRange(directInitRange),
     GlobalNew(globalNew), UsualArrayDeleteWantsSize(usualArrayDeleteWantsSize) {
   assert((initializer != 0 || initializationStyle == NoInit) &&
          "Only NoInit can have no initializer.");
   StoredInitializationStyle = initializer ? initializationStyle + 1 : 0;
-  AllocateArgsArray(C, arraySize != 0, numPlaceArgs, initializer != 0);
+  AllocateArgsArray(C, arraySize != 0, placementArgs.size(), initializer != 0);
   unsigned i = 0;
   if (Array) {
     if (arraySize->isInstantiationDependent())
@@ -102,7 +122,7 @@ CXXNewExpr::CXXNewExpr(ASTContext &C, bool globalNew, FunctionDecl *operatorNew,
     SubExprs[i++] = initializer;
   }
 
-  for (unsigned j = 0; j < NumPlacementArgs; ++j) {
+  for (unsigned j = 0; j != placementArgs.size(); ++j) {
     if (placementArgs[j]->isInstantiationDependent())
       ExprBits.InstantiationDependent = true;
     if (placementArgs[j]->containsUnexpandedParameterPack())
@@ -110,6 +130,14 @@ CXXNewExpr::CXXNewExpr(ASTContext &C, bool globalNew, FunctionDecl *operatorNew,
 
     SubExprs[i++] = placementArgs[j];
   }
+
+  switch (getInitializationStyle()) {
+  case CallInit:
+    this->Range.setEnd(DirectInitRange.getEnd()); break;
+  case ListInit:
+    this->Range.setEnd(getInitializer()->getSourceRange().getEnd()); break;
+  default: break;
+  }
 }
 
 void CXXNewExpr::AllocateArgsArray(ASTContext &C, bool isArray,
@@ -127,18 +155,6 @@ bool CXXNewExpr::shouldNullCheckAllocation(ASTContext &Ctx) const {
     castAs<FunctionProtoType>()->isNothrow(Ctx);
 }
 
-SourceLocation CXXNewExpr::getEndLoc() const {
-  switch (getInitializationStyle()) {
-  case NoInit:
-    return AllocatedTypeInfo->getTypeLoc().getEndLoc();
-  case CallInit:
-    return DirectInitRange.getEnd();
-  case ListInit:
-    return getInitializer()->getSourceRange().getEnd();
-  }
-  llvm_unreachable("bogus initialization style");
-}
-
 // CXXDeleteExpr
 QualType CXXDeleteExpr::getDestroyedType() const {
   const Expr *Arg = getArgument();
@@ -227,7 +243,7 @@ UnresolvedLookupExpr::Create(ASTContext &C,
   return new (Mem) UnresolvedLookupExpr(C, NamingClass, QualifierLoc,
                                         TemplateKWLoc, NameInfo,
                                         ADL, /*Overload*/ true, Args,
-                                        Begin, End, /*StdIsAssociated=*/false);
+                                        Begin, End);
 }
 
 UnresolvedLookupExpr *
@@ -697,15 +713,14 @@ CXXBindTemporaryExpr *CXXBindTemporaryExpr::Create(ASTContext &C,
 CXXTemporaryObjectExpr::CXXTemporaryObjectExpr(ASTContext &C,
                                                CXXConstructorDecl *Cons,
                                                TypeSourceInfo *Type,
-                                               Expr **Args,
-                                               unsigned NumArgs,
+                                               ArrayRef<Expr*> Args,
                                                SourceRange parenRange,
                                                bool HadMultipleCandidates,
                                                bool ZeroInitialization)
   : CXXConstructExpr(C, CXXTemporaryObjectExprClass, 
                      Type->getType().getNonReferenceType(), 
                      Type->getTypeLoc().getBeginLoc(),
-                     Cons, false, Args, NumArgs,
+                     Cons, false, Args,
                      HadMultipleCandidates, /*FIXME*/false, ZeroInitialization,
                      CXXConstructExpr::CK_Complete, parenRange),
     Type(Type) {
@@ -719,14 +734,14 @@ SourceRange CXXTemporaryObjectExpr::getSourceRange() const {
 CXXConstructExpr *CXXConstructExpr::Create(ASTContext &C, QualType T,
                                            SourceLocation Loc,
                                            CXXConstructorDecl *D, bool Elidable,
-                                           Expr **Args, unsigned NumArgs,
+                                           ArrayRef<Expr*> Args,
                                            bool HadMultipleCandidates,
                                            bool ListInitialization,
                                            bool ZeroInitialization,
                                            ConstructionKind ConstructKind,
                                            SourceRange ParenRange) {
   return new (C) CXXConstructExpr(C, CXXConstructExprClass, T, Loc, D, 
-                                  Elidable, Args, NumArgs,
+                                  Elidable, Args,
                                   HadMultipleCandidates, ListInitialization,
                                   ZeroInitialization, ConstructKind,
                                   ParenRange);
@@ -735,7 +750,7 @@ CXXConstructExpr *CXXConstructExpr::Create(ASTContext &C, QualType T,
 CXXConstructExpr::CXXConstructExpr(ASTContext &C, StmtClass SC, QualType T,
                                    SourceLocation Loc,
                                    CXXConstructorDecl *D, bool elidable,
-                                   Expr **args, unsigned numargs,
+                                   ArrayRef<Expr*> args,
                                    bool HadMultipleCandidates,
                                    bool ListInitialization,
                                    bool ZeroInitialization,
@@ -745,16 +760,16 @@ CXXConstructExpr::CXXConstructExpr(ASTContext &C, StmtClass SC, QualType T,
          T->isDependentType(), T->isDependentType(),
          T->isInstantiationDependentType(),
          T->containsUnexpandedParameterPack()),
-    Constructor(D), Loc(Loc), ParenRange(ParenRange),  NumArgs(numargs),
+    Constructor(D), Loc(Loc), ParenRange(ParenRange),  NumArgs(args.size()),
     Elidable(elidable), HadMultipleCandidates(HadMultipleCandidates),
     ListInitialization(ListInitialization),
     ZeroInitialization(ZeroInitialization),
     ConstructKind(ConstructKind), Args(0)
 {
   if (NumArgs) {
-    Args = new (C) Stmt*[NumArgs];
+    Args = new (C) Stmt*[args.size()];
     
-    for (unsigned i = 0; i != NumArgs; ++i) {
+    for (unsigned i = 0; i != args.size(); ++i) {
       assert(args[i] && "NULL argument in CXXConstructExpr");
 
       if (args[i]->isValueDependent())
@@ -877,9 +892,12 @@ LambdaExpr *LambdaExpr::Create(ASTContext &Context,
   QualType T = Context.getTypeDeclType(Class);
 
   unsigned Size = sizeof(LambdaExpr) + sizeof(Stmt *) * (Captures.size() + 1);
-  if (!ArrayIndexVars.empty())
-    Size += sizeof(VarDecl *) * ArrayIndexVars.size()
-          + sizeof(unsigned) * (Captures.size() + 1);
+  if (!ArrayIndexVars.empty()) {
+    Size += sizeof(unsigned) * (Captures.size() + 1);
+    // Realign for following VarDecl array.
+    Size = llvm::RoundUpToAlignment(Size, llvm::alignOf<VarDecl*>());
+    Size += sizeof(VarDecl *) * ArrayIndexVars.size();
+  }
   void *Mem = Context.Allocate(Size);
   return new (Mem) LambdaExpr(T, IntroducerRange, CaptureDefault, 
                               Captures, ExplicitParams, ExplicitResultType,
@@ -997,8 +1015,7 @@ ExprWithCleanups *ExprWithCleanups::Create(ASTContext &C, EmptyShell empty,
 
 CXXUnresolvedConstructExpr::CXXUnresolvedConstructExpr(TypeSourceInfo *Type,
                                                  SourceLocation LParenLoc,
-                                                 Expr **Args,
-                                                 unsigned NumArgs,
+                                                 ArrayRef<Expr*> Args,
                                                  SourceLocation RParenLoc)
   : Expr(CXXUnresolvedConstructExprClass, 
          Type->getType().getNonReferenceType(),
@@ -1011,9 +1028,9 @@ CXXUnresolvedConstructExpr::CXXUnresolvedConstructExpr(TypeSourceInfo *Type,
     Type(Type),
     LParenLoc(LParenLoc),
     RParenLoc(RParenLoc),
-    NumArgs(NumArgs) {
+    NumArgs(Args.size()) {
   Stmt **StoredArgs = reinterpret_cast<Stmt **>(this + 1);
-  for (unsigned I = 0; I != NumArgs; ++I) {
+  for (unsigned I = 0; I != Args.size(); ++I) {
     if (Args[I]->containsUnexpandedParameterPack())
       ExprBits.ContainsUnexpandedParameterPack = true;
 
@@ -1025,13 +1042,11 @@ CXXUnresolvedConstructExpr *
 CXXUnresolvedConstructExpr::Create(ASTContext &C,
                                    TypeSourceInfo *Type,
                                    SourceLocation LParenLoc,
-                                   Expr **Args,
-                                   unsigned NumArgs,
+                                   ArrayRef<Expr*> Args,
                                    SourceLocation RParenLoc) {
   void *Mem = C.Allocate(sizeof(CXXUnresolvedConstructExpr) +
-                         sizeof(Expr *) * NumArgs);
-  return new (Mem) CXXUnresolvedConstructExpr(Type, LParenLoc,
-                                              Args, NumArgs, RParenLoc);
+                         sizeof(Expr *) * Args.size());
+  return new (Mem) CXXUnresolvedConstructExpr(Type, LParenLoc, Args, RParenLoc);
 }
 
 CXXUnresolvedConstructExpr *
@@ -1300,6 +1315,34 @@ TemplateArgument SubstNonTypeTemplateParmPackExpr::getArgumentPack() const {
   return TemplateArgument(Arguments, NumArguments);
 }
 
+FunctionParmPackExpr::FunctionParmPackExpr(QualType T, ParmVarDecl *ParamPack,
+                                           SourceLocation NameLoc,
+                                           unsigned NumParams,
+                                           Decl * const *Params)
+  : Expr(FunctionParmPackExprClass, T, VK_LValue, OK_Ordinary,
+         true, true, true, true),
+    ParamPack(ParamPack), NameLoc(NameLoc), NumParameters(NumParams) {
+  if (Params)
+    std::uninitialized_copy(Params, Params + NumParams,
+                            reinterpret_cast<Decl**>(this+1));
+}
+
+FunctionParmPackExpr *
+FunctionParmPackExpr::Create(ASTContext &Context, QualType T,
+                             ParmVarDecl *ParamPack, SourceLocation NameLoc,
+                             llvm::ArrayRef<Decl*> Params) {
+  return new (Context.Allocate(sizeof(FunctionParmPackExpr) +
+                               sizeof(ParmVarDecl*) * Params.size()))
+    FunctionParmPackExpr(T, ParamPack, NameLoc, Params.size(), Params.data());
+}
+
+FunctionParmPackExpr *
+FunctionParmPackExpr::CreateEmpty(ASTContext &Context, unsigned NumParams) {
+  return new (Context.Allocate(sizeof(FunctionParmPackExpr) +
+                               sizeof(ParmVarDecl*) * NumParams))
+    FunctionParmPackExpr(QualType(), 0, SourceLocation(), 0, 0);
+}
+
 TypeTraitExpr::TypeTraitExpr(QualType T, SourceLocation Loc, TypeTrait Kind,
                              ArrayRef<TypeSourceInfo *> Args,
                              SourceLocation RParenLoc,
diff --git a/lib/AST/ExprClassification.cpp b/lib/AST/ExprClassification.cpp
index f16d70b..24ec6bb 100644
--- a/lib/AST/ExprClassification.cpp
+++ b/lib/AST/ExprClassification.cpp
@@ -134,6 +134,7 @@ static Cl::Kinds ClassifyInternal(ASTContext &Ctx, const Expr *E) {
     // ObjC instance variables are lvalues
     // FIXME: ObjC++0x might have different rules
   case Expr::ObjCIvarRefExprClass:
+  case Expr::FunctionParmPackExprClass:
     return Cl::CL_LValue;
 
     // C99 6.5.2.5p5 says that compound literals are lvalues.
diff --git a/lib/AST/ExprConstant.cpp b/lib/AST/ExprConstant.cpp
index 06c41a2..6e0b5fc 100644
--- a/lib/AST/ExprConstant.cpp
+++ b/lib/AST/ExprConstant.cpp
@@ -953,7 +953,7 @@ static void NoteLValueLocation(EvalInfo &Info, APValue::LValueBase Base) {
   if (VD)
     Info.Note(VD->getLocation(), diag::note_declared_at);
   else
-    Info.Note(Base.dyn_cast<const Expr*>()->getExprLoc(),
+    Info.Note(Base.get<const Expr*>()->getExprLoc(),
               diag::note_constexpr_temporary_here);
 }
 
@@ -987,6 +987,14 @@ static bool CheckLValueConstantExpression(EvalInfo &Info, SourceLocation Loc,
           LVal.getLValueCallIndex() == 0) &&
          "have call index for global lvalue");
 
+  // Check if this is a thread-local variable.
+  if (const ValueDecl *VD = Base.dyn_cast<const ValueDecl*>()) {
+    if (const VarDecl *Var = dyn_cast<const VarDecl>(VD)) {
+      if (Var->isThreadSpecified())
+        return false;
+    }
+  }
+
   // Allow address constant expressions to be past-the-end pointers. This is
   // an extension: the standard requires them to point to an object.
   if (!IsReferenceType)
@@ -2586,7 +2594,7 @@ public:
     const FieldDecl *FD = dyn_cast<FieldDecl>(E->getMemberDecl());
     if (!FD) return Error(E);
     assert(!FD->getType()->isReferenceType() && "prvalue reference?");
-    assert(BaseTy->getAs<RecordType>()->getDecl()->getCanonicalDecl() ==
+    assert(BaseTy->castAs<RecordType>()->getDecl()->getCanonicalDecl() ==
            FD->getParent()->getCanonicalDecl() && "record / field mismatch");
 
     SubobjectDesignator Designator(BaseTy);
@@ -2665,7 +2673,7 @@ public:
     if (E->isArrow()) {
       if (!EvaluatePointer(E->getBase(), Result, this->Info))
         return false;
-      BaseTy = E->getBase()->getType()->getAs<PointerType>()->getPointeeType();
+      BaseTy = E->getBase()->getType()->castAs<PointerType>()->getPointeeType();
     } else if (E->getBase()->isRValue()) {
       assert(E->getBase()->getType()->isRecordType());
       if (!EvaluateTemporary(E->getBase(), Result, this->Info))
@@ -2878,19 +2886,13 @@ LValueExprEvaluator::VisitCompoundLiteralExpr(const CompoundLiteralExpr *E) {
 }
 
 bool LValueExprEvaluator::VisitCXXTypeidExpr(const CXXTypeidExpr *E) {
-  if (E->isTypeOperand())
+  if (!E->isPotentiallyEvaluated())
     return Success(E);
-  CXXRecordDecl *RD = E->getExprOperand()->getType()->getAsCXXRecordDecl();
-  // FIXME: The standard says "a typeid expression whose operand is of a
-  // polymorphic class type" is not a constant expression, but it probably
-  // means "a typeid expression whose operand is potentially evaluated".
-  if (RD && RD->isPolymorphic()) {
-    Info.Diag(E, diag::note_constexpr_typeid_polymorphic)
-      << E->getExprOperand()->getType()
-      << E->getExprOperand()->getSourceRange();
-    return false;
-  }
-  return Success(E);
+
+  Info.Diag(E, diag::note_constexpr_typeid_polymorphic)
+    << E->getExprOperand()->getType()
+    << E->getExprOperand()->getSourceRange();
+  return false;
 }
 
 bool LValueExprEvaluator::VisitCXXUuidofExpr(const CXXUuidofExpr *E) {
@@ -3036,7 +3038,7 @@ bool PointerExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) {
   if (E->getOpcode() == BO_Sub)
     AdditionalOffset = -AdditionalOffset;
 
-  QualType Pointee = PExp->getType()->getAs<PointerType>()->getPointeeType();
+  QualType Pointee = PExp->getType()->castAs<PointerType>()->getPointeeType();
   return HandleLValueArrayAdjustment(Info, E, Result, Pointee,
                                      AdditionalOffset);
 }
@@ -4288,6 +4290,16 @@ bool IntExprEvaluator::VisitCallExpr(const CallExpr *E) {
     return Error(E);
   }
 
+  case Builtin::BI__builtin_bswap16:
+  case Builtin::BI__builtin_bswap32:
+  case Builtin::BI__builtin_bswap64: {
+    APSInt Val;
+    if (!EvaluateInteger(E->getArg(0), Val, Info))
+      return false;
+
+    return Success(Val.byteSwap(), E);
+  }
+
   case Builtin::BI__builtin_classify_type:
     return Success(EvaluateBuiltinClassifyType(E), E);
 
@@ -4902,7 +4914,7 @@ bool IntExprEvaluator::VisitBinaryOperator(const BinaryOperator *E) {
           if (!LHSValue.Offset.isZero() || !RHSValue.Offset.isZero())
             return false;
           const Expr *LHSExpr = LHSValue.Base.dyn_cast<const Expr*>();
-          const Expr *RHSExpr = LHSValue.Base.dyn_cast<const Expr*>();
+          const Expr *RHSExpr = RHSValue.Base.dyn_cast<const Expr*>();
           if (!LHSExpr || !RHSExpr)
             return false;
           const AddrLabelExpr *LHSAddrExpr = dyn_cast<AddrLabelExpr>(LHSExpr);
@@ -5176,7 +5188,7 @@ bool IntExprEvaluator::VisitUnaryExprOrTypeTraitExpr(
     QualType Ty = E->getTypeOfArgument();
 
     if (Ty->isVectorType()) {
-      unsigned n = Ty->getAs<VectorType>()->getNumElements();
+      unsigned n = Ty->castAs<VectorType>()->getNumElements();
 
       // The vec_step built-in functions that take a 3-component
       // vector return 4. (OpenCL 1.1 spec 6.11.12)
@@ -5349,6 +5361,7 @@ bool IntExprEvaluator::VisitCastExpr(const CastExpr *E) {
   case CK_IntegralRealToComplex:
   case CK_IntegralComplexCast:
   case CK_IntegralComplexToFloatingComplex:
+  case CK_BuiltinFnToFnPtr:
     llvm_unreachable("invalid cast kind for integral value");
 
   case CK_BitCast:
@@ -5753,7 +5766,7 @@ static bool EvaluateComplex(const Expr *E, ComplexValue &Result,
 }
 
 bool ComplexExprEvaluator::ZeroInitialization(const Expr *E) {
-  QualType ElemTy = E->getType()->getAs<ComplexType>()->getElementType();
+  QualType ElemTy = E->getType()->castAs<ComplexType>()->getElementType();
   if (ElemTy->isRealFloatingType()) {
     Result.makeComplexFloat();
     APFloat Zero = APFloat::getZero(Info.Ctx.getFloatTypeSemantics(ElemTy));
@@ -5835,6 +5848,7 @@ bool ComplexExprEvaluator::VisitCastExpr(const CastExpr *E) {
   case CK_ARCReclaimReturnedObject:
   case CK_ARCExtendBlockObject:
   case CK_CopyAndAutoreleaseBlockObject:
+  case CK_BuiltinFnToFnPtr:
     llvm_unreachable("invalid cast kind for complex value");
 
   case CK_LValueToRValue:
@@ -5911,9 +5925,9 @@ bool ComplexExprEvaluator::VisitCastExpr(const CastExpr *E) {
     if (!Visit(E->getSubExpr()))
       return false;
 
-    QualType To = E->getType()->getAs<ComplexType>()->getElementType();
+    QualType To = E->getType()->castAs<ComplexType>()->getElementType();
     QualType From
-      = E->getSubExpr()->getType()->getAs<ComplexType>()->getElementType();
+      = E->getSubExpr()->getType()->castAs<ComplexType>()->getElementType();
     Result.makeComplexFloat();
     return HandleIntToFloatCast(Info, E, From, Result.IntReal,
                                 To, Result.FloatReal) &&
@@ -6177,11 +6191,9 @@ static bool Evaluate(APValue &Result, EvalInfo &Info, const Expr *E) {
       return false;
     Result = Info.CurrentCall->Temporaries[E];
   } else if (E->getType()->isVoidType()) {
-    if (Info.getLangOpts().CPlusPlus0x)
+    if (!Info.getLangOpts().CPlusPlus0x)
       Info.CCEDiag(E, diag::note_constexpr_nonliteral)
         << E->getType();
-    else
-      Info.CCEDiag(E, diag::note_invalid_subexpr_in_const_expr);
     if (!EvaluateVoid(E, Info))
       return false;
   } else if (Info.getLangOpts().CPlusPlus0x) {
@@ -6470,6 +6482,7 @@ static ICEDiag CheckICE(const Expr* E, ASTContext &Ctx) {
   case Expr::OpaqueValueExprClass:
   case Expr::PackExpansionExprClass:
   case Expr::SubstNonTypeTemplateParmPackExprClass:
+  case Expr::FunctionParmPackExprClass:
   case Expr::AsTypeExprClass:
   case Expr::ObjCIndirectCopyRestoreExprClass:
   case Expr::MaterializeTemporaryExprClass:
diff --git a/lib/AST/ItaniumMangle.cpp b/lib/AST/ItaniumMangle.cpp
index 7c7a5e5..851944a 100644
--- a/lib/AST/ItaniumMangle.cpp
+++ b/lib/AST/ItaniumMangle.cpp
@@ -343,17 +343,10 @@ private:
   void mangleCXXDtorType(CXXDtorType T);
 
   void mangleTemplateArgs(const ASTTemplateArgumentListInfo &TemplateArgs);
-  void mangleTemplateArgs(TemplateName Template,
-                          const TemplateArgument *TemplateArgs,
-                          unsigned NumTemplateArgs);  
-  void mangleTemplateArgs(const TemplateParameterList &PL,
-                          const TemplateArgument *TemplateArgs,
+  void mangleTemplateArgs(const TemplateArgument *TemplateArgs,
                           unsigned NumTemplateArgs);
-  void mangleTemplateArgs(const TemplateParameterList &PL,
-                          const TemplateArgumentList &AL);
-  void mangleTemplateArg(const NamedDecl *P, TemplateArgument A);
-  void mangleUnresolvedTemplateArgs(const TemplateArgument *args,
-                                    unsigned numArgs);
+  void mangleTemplateArgs(const TemplateArgumentList &AL);
+  void mangleTemplateArg(TemplateArgument A);
 
   void mangleTemplateParameter(unsigned Index);
 
@@ -570,8 +563,7 @@ void CXXNameMangler::mangleName(const NamedDecl *ND) {
     const TemplateArgumentList *TemplateArgs = 0;
     if (const TemplateDecl *TD = isTemplate(ND, TemplateArgs)) {
       mangleUnscopedTemplateName(TD);
-      TemplateParameterList *TemplateParameters = TD->getTemplateParameters();
-      mangleTemplateArgs(*TemplateParameters, *TemplateArgs);
+      mangleTemplateArgs(*TemplateArgs);
       return;
     }
 
@@ -593,8 +585,7 @@ void CXXNameMangler::mangleName(const TemplateDecl *TD,
 
   if (DC->isTranslationUnit() || isStdNamespace(DC)) {
     mangleUnscopedTemplateName(TD);
-    TemplateParameterList *TemplateParameters = TD->getTemplateParameters();
-    mangleTemplateArgs(*TemplateParameters, TemplateArgs, NumTemplateArgs);
+    mangleTemplateArgs(TemplateArgs, NumTemplateArgs);
   } else {
     mangleNestedName(TD, TemplateArgs, NumTemplateArgs);
   }
@@ -693,9 +684,10 @@ void CXXNameMangler::mangleFloat(const llvm::APFloat &f) {
 void CXXNameMangler::mangleNumber(const llvm::APSInt &Value) {
   if (Value.isSigned() && Value.isNegative()) {
     Out << 'n';
-    Value.abs().print(Out, true);
-  } else
-    Value.print(Out, Value.isSigned());
+    Value.abs().print(Out, /*signed*/ false);
+  } else {
+    Value.print(Out, /*signed*/ false);
+  }
 }
 
 void CXXNameMangler::mangleNumber(int64_t Number) {
@@ -737,8 +729,7 @@ void CXXNameMangler::manglePrefix(QualType type) {
       // FIXME: GCC does not appear to mangle the template arguments when
       // the template in question is a dependent template name. Should we
       // emulate that badness?
-      mangleTemplateArgs(TST->getTemplateName(), TST->getArgs(),
-                         TST->getNumArgs());
+      mangleTemplateArgs(TST->getArgs(), TST->getNumArgs());
       addSubstitution(QualType(TST, 0));
     }
   } else if (const DependentTemplateSpecializationType *DTST
@@ -751,7 +742,7 @@ void CXXNameMangler::manglePrefix(QualType type) {
     // FIXME: GCC does not appear to mangle the template arguments when
     // the template in question is a dependent template name. Should we
     // emulate that badness?
-    mangleTemplateArgs(Template, DTST->getArgs(), DTST->getNumArgs());
+    mangleTemplateArgs(DTST->getArgs(), DTST->getNumArgs());
   } else {
     // We use the QualType mangle type variant here because it handles
     // substitutions.
@@ -942,7 +933,7 @@ void CXXNameMangler::mangleUnresolvedPrefix(NestedNameSpecifier *qualifier,
       }
       }
 
-      mangleUnresolvedTemplateArgs(tst->getArgs(), tst->getNumArgs());
+      mangleTemplateArgs(tst->getArgs(), tst->getNumArgs());
       break;
     }
 
@@ -959,7 +950,7 @@ void CXXNameMangler::mangleUnresolvedPrefix(NestedNameSpecifier *qualifier,
       const DependentTemplateSpecializationType *tst
         = cast<DependentTemplateSpecializationType>(type);
       mangleSourceName(tst->getIdentifier());
-      mangleUnresolvedTemplateArgs(tst->getArgs(), tst->getNumArgs());
+      mangleTemplateArgs(tst->getArgs(), tst->getNumArgs());
       break;
     }
     }
@@ -1228,8 +1219,7 @@ void CXXNameMangler::mangleNestedName(const NamedDecl *ND,
   const TemplateArgumentList *TemplateArgs = 0;
   if (const TemplateDecl *TD = isTemplate(ND, TemplateArgs)) {
     mangleTemplatePrefix(TD);
-    TemplateParameterList *TemplateParameters = TD->getTemplateParameters();
-    mangleTemplateArgs(*TemplateParameters, *TemplateArgs);
+    mangleTemplateArgs(*TemplateArgs);
   }
   else {
     manglePrefix(DC, NoFunction);
@@ -1246,8 +1236,7 @@ void CXXNameMangler::mangleNestedName(const TemplateDecl *TD,
   Out << 'N';
 
   mangleTemplatePrefix(TD);
-  TemplateParameterList *TemplateParameters = TD->getTemplateParameters();
-  mangleTemplateArgs(*TemplateParameters, TemplateArgs, NumTemplateArgs);
+  mangleTemplateArgs(TemplateArgs, NumTemplateArgs);
 
   Out << 'E';
 }
@@ -1341,11 +1330,8 @@ void CXXNameMangler::mangleLambda(const CXXRecordDecl *Lambda) {
   }
 
   Out << "Ul";
-  DeclarationName Name
-    = getASTContext().DeclarationNames.getCXXOperatorName(OO_Call);
-  const FunctionProtoType *Proto
-    = cast<CXXMethodDecl>(*Lambda->lookup(Name).first)->getType()->
-        getAs<FunctionProtoType>();
+  const FunctionProtoType *Proto = Lambda->getLambdaTypeInfo()->getType()->
+                                   getAs<FunctionProtoType>();
   mangleBareFunctionType(Proto, /*MangleReturnType=*/false);        
   Out << "E";
   
@@ -1423,8 +1409,7 @@ void CXXNameMangler::manglePrefix(const DeclContext *DC, bool NoFunction) {
   const TemplateArgumentList *TemplateArgs = 0;
   if (const TemplateDecl *TD = isTemplate(ND, TemplateArgs)) {
     mangleTemplatePrefix(TD);
-    TemplateParameterList *TemplateParameters = TD->getTemplateParameters();
-    mangleTemplateArgs(*TemplateParameters, *TemplateArgs);
+    mangleTemplateArgs(*TemplateArgs);
   }
   else if(NoFunction && (isa<FunctionDecl>(ND) || isa<ObjCMethodDecl>(ND)))
     return;
@@ -2110,6 +2095,7 @@ void CXXNameMangler::mangleNeonVectorType(const VectorType *T) {
 //                         ::= Dv [<dimension expression>] _ <element type>
 // <extended element type> ::= <element type>
 //                         ::= p # AltiVec vector pixel
+//                         ::= b # Altivec vector bool
 void CXXNameMangler::mangleType(const VectorType *T) {
   if ((T->getVectorKind() == VectorType::NeonVector ||
        T->getVectorKind() == VectorType::NeonPolyVector)) {
@@ -2174,7 +2160,7 @@ void CXXNameMangler::mangleType(const TemplateSpecializationType *T) {
     // FIXME: GCC does not appear to mangle the template arguments when
     // the template in question is a dependent template name. Should we
     // emulate that badness?
-    mangleTemplateArgs(T->getTemplateName(), T->getArgs(), T->getNumArgs());
+    mangleTemplateArgs(T->getArgs(), T->getNumArgs());
     addSubstitution(QualType(T, 0));
   }
 }
@@ -2200,7 +2186,7 @@ void CXXNameMangler::mangleType(const DependentTemplateSpecializationType *T) {
   // FIXME: GCC does not appear to mangle the template arguments when
   // the template in question is a dependent template name. Should we
   // emulate that badness?
-  mangleTemplateArgs(Prefix, T->getArgs(), T->getNumArgs());    
+  mangleTemplateArgs(T->getArgs(), T->getNumArgs());    
   Out << 'E';
 }
 
@@ -2414,7 +2400,6 @@ recurse:
   case Expr::ExpressionTraitExprClass:
   case Expr::VAArgExprClass:
   case Expr::CXXUuidofExprClass:
-  case Expr::CXXNoexceptExprClass:
   case Expr::CUDAKernelCallExprClass:
   case Expr::AsTypeExprClass:
   case Expr::PseudoObjectExprClass:
@@ -2606,6 +2591,11 @@ recurse:
     Out <<"_E";
     break;
 
+  case Expr::CXXNoexceptExprClass:
+    Out << "nx";
+    mangleExpression(cast<CXXNoexceptExpr>(E)->getOperand());
+    break;
+
   case Expr::UnaryExprOrTypeTraitExprClass: {
     const UnaryExprOrTypeTraitExpr *SAE = cast<UnaryExprOrTypeTraitExpr>(E);
     
@@ -2808,7 +2798,15 @@ recurse:
     // };
     Out << "_SUBSTPACK_";
     break;
-      
+
+  case Expr::FunctionParmPackExprClass: {
+    // FIXME: not clear how to mangle this!
+    const FunctionParmPackExpr *FPPE = cast<FunctionParmPackExpr>(E);
+    Out << "v110_SUBSTPACK";
+    mangleFunctionParam(FPPE->getParameterPack());
+    break;
+  }
+
   case Expr::DependentScopeDeclRefExprClass: {
     const DependentScopeDeclRefExpr *DRE = cast<DependentScopeDeclRefExpr>(E);
     mangleUnresolvedName(DRE->getQualifier(), 0, DRE->getDeclName(), Arity);
@@ -3043,50 +3041,28 @@ void CXXNameMangler::mangleTemplateArgs(
   // <template-args> ::= I <template-arg>+ E
   Out << 'I';
   for (unsigned i = 0, e = TemplateArgs.NumTemplateArgs; i != e; ++i)
-    mangleTemplateArg(0, TemplateArgs.getTemplateArgs()[i].getArgument());
-  Out << 'E';
-}
-
-void CXXNameMangler::mangleTemplateArgs(TemplateName Template,
-                                        const TemplateArgument *TemplateArgs,
-                                        unsigned NumTemplateArgs) {
-  if (TemplateDecl *TD = Template.getAsTemplateDecl())
-    return mangleTemplateArgs(*TD->getTemplateParameters(), TemplateArgs,
-                              NumTemplateArgs);
-  
-  mangleUnresolvedTemplateArgs(TemplateArgs, NumTemplateArgs);
-}
-
-void CXXNameMangler::mangleUnresolvedTemplateArgs(const TemplateArgument *args,
-                                                  unsigned numArgs) {
-  // <template-args> ::= I <template-arg>+ E
-  Out << 'I';
-  for (unsigned i = 0; i != numArgs; ++i)
-    mangleTemplateArg(0, args[i]);
+    mangleTemplateArg(TemplateArgs.getTemplateArgs()[i].getArgument());
   Out << 'E';
 }
 
-void CXXNameMangler::mangleTemplateArgs(const TemplateParameterList &PL,
-                                        const TemplateArgumentList &AL) {
+void CXXNameMangler::mangleTemplateArgs(const TemplateArgumentList &AL) {
   // <template-args> ::= I <template-arg>+ E
   Out << 'I';
   for (unsigned i = 0, e = AL.size(); i != e; ++i)
-    mangleTemplateArg(PL.getParam(i), AL[i]);
+    mangleTemplateArg(AL[i]);
   Out << 'E';
 }
 
-void CXXNameMangler::mangleTemplateArgs(const TemplateParameterList &PL,
-                                        const TemplateArgument *TemplateArgs,
+void CXXNameMangler::mangleTemplateArgs(const TemplateArgument *TemplateArgs,
                                         unsigned NumTemplateArgs) {
   // <template-args> ::= I <template-arg>+ E
   Out << 'I';
   for (unsigned i = 0; i != NumTemplateArgs; ++i)
-    mangleTemplateArg(PL.getParam(i), TemplateArgs[i]);
+    mangleTemplateArg(TemplateArgs[i]);
   Out << 'E';
 }
 
-void CXXNameMangler::mangleTemplateArg(const NamedDecl *P,
-                                       TemplateArgument A) {
+void CXXNameMangler::mangleTemplateArg(TemplateArgument A) {
   // <template-arg> ::= <type>              # type or template
   //                ::= X <expression> E    # expression
   //                ::= <expr-primary>      # simple expressions
@@ -3135,25 +3111,12 @@ void CXXNameMangler::mangleTemplateArg(const NamedDecl *P,
     mangleIntegerLiteral(A.getIntegralType(), A.getAsIntegral());
     break;
   case TemplateArgument::Declaration: {
-    assert(P && "Missing template parameter for declaration argument");
     //  <expr-primary> ::= L <mangled-name> E # external name
-    //  <expr-primary> ::= L <type> 0 E
     // Clang produces AST's where pointer-to-member-function expressions
     // and pointer-to-function expressions are represented as a declaration not
     // an expression. We compensate for it here to produce the correct mangling.
-    const NonTypeTemplateParmDecl *Parameter = cast<NonTypeTemplateParmDecl>(P);
-
-    // Handle NULL pointer arguments.
-    if (!A.getAsDecl()) {
-      Out << "L";
-      mangleType(Parameter->getType());
-      Out << "0E";
-      break;
-    }
-    
-
-    NamedDecl *D = cast<NamedDecl>(A.getAsDecl());
-    bool compensateMangling = !Parameter->getType()->isReferenceType();
+    ValueDecl *D = A.getAsDecl();
+    bool compensateMangling = !A.isDeclForReferenceParam();
     if (compensateMangling) {
       Out << 'X';
       mangleOperatorName(OO_Amp, 1);
@@ -3176,14 +3139,20 @@ void CXXNameMangler::mangleTemplateArg(const NamedDecl *P,
 
     break;
   }
-      
+  case TemplateArgument::NullPtr: {
+    //  <expr-primary> ::= L <type> 0 E
+    Out << 'L';
+    mangleType(A.getNullPtrType());
+    Out << "0E";
+    break;
+  }
   case TemplateArgument::Pack: {
     // Note: proposal by Mike Herrick on 12/20/10
     Out << 'J';
     for (TemplateArgument::pack_iterator PA = A.pack_begin(), 
                                       PAEnd = A.pack_end();
          PA != PAEnd; ++PA)
-      mangleTemplateArg(P, *PA);
+      mangleTemplateArg(*PA);
     Out << 'E';
   }
   }
diff --git a/lib/AST/MicrosoftMangle.cpp b/lib/AST/MicrosoftMangle.cpp
index e2cee7f..5d5b83d 100644
--- a/lib/AST/MicrosoftMangle.cpp
+++ b/lib/AST/MicrosoftMangle.cpp
@@ -20,6 +20,7 @@
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/ExprCXX.h"
 #include "clang/Basic/ABI.h"
+#include "clang/Basic/DiagnosticOptions.h"
 
 #include <map>
 
@@ -56,7 +57,7 @@ public:
   void mangleVariableEncoding(const VarDecl *VD);
   void mangleNumber(int64_t Number);
   void mangleNumber(const llvm::APSInt &Value);
-  void mangleType(QualType T, SourceRange Range);
+  void mangleType(QualType T, SourceRange Range, bool MangleQualifiers = true);
 
 private:
   void disableBackReferences() { UseNameBackReferences = false; }
@@ -68,6 +69,7 @@ private:
   void manglePostfix(const DeclContext *DC, bool NoFunction=false);
   void mangleOperatorName(OverloadedOperatorKind OO, SourceLocation Loc);
   void mangleQualifiers(Qualifiers Quals, bool IsMember);
+  void manglePointerQualifiers(Qualifiers Quals);
 
   void mangleUnscopedTemplateName(const TemplateDecl *ND);
   void mangleTemplateInstantiationName(const TemplateDecl *TD,
@@ -75,7 +77,7 @@ private:
   void mangleObjCMethodName(const ObjCMethodDecl *MD);
   void mangleLocalName(const FunctionDecl *FD);
 
-  void mangleTypeRepeated(QualType T, SourceRange Range);
+  void mangleArgumentType(QualType T, SourceRange Range);
 
   // Declare manglers for every type class.
 #define ABSTRACT_TYPE(CLASS, PARENT)
@@ -95,6 +97,7 @@ private:
   void mangleFunctionClass(const FunctionDecl *FD);
   void mangleCallingConvention(const FunctionType *T, bool IsInstMethod = false);
   void mangleIntegerLiteral(QualType T, const llvm::APSInt &Number);
+  void mangleExpression(const Expr *E);
   void mangleThrowSpecification(const FunctionProtoType *T);
 
   void mangleTemplateArgs(
@@ -266,18 +269,18 @@ void MicrosoftCXXNameMangler::mangleVariableEncoding(const VarDecl *VD) {
     Out << '4';
   // Now mangle the type.
   // <variable-type> ::= <type> <cvr-qualifiers>
-  //                 ::= <type> A # pointers, references, arrays
+  //                 ::= <type> <pointee-cvr-qualifiers> # pointers, references
   // Pointers and references are odd. The type of 'int * const foo;' gets
   // mangled as 'QAHA' instead of 'PAHB', for example.
   TypeLoc TL = VD->getTypeSourceInfo()->getTypeLoc();
   QualType Ty = TL.getType();
   if (Ty->isPointerType() || Ty->isReferenceType()) {
     mangleType(Ty, TL.getSourceRange());
-    Out << 'A';
+    mangleQualifiers(Ty->getPointeeType().getQualifiers(), false);
   } else if (const ArrayType *AT = getASTContext().getAsArrayType(Ty)) {
     // Global arrays are funny, too.
     mangleType(AT, true);
-    Out << 'A';
+    mangleQualifiers(Ty.getQualifiers(), false);
   } else {
     mangleType(Ty.getLocalUnqualifiedType(), TL.getSourceRange());
     mangleQualifiers(Ty.getLocalQualifiers(), false);
@@ -304,39 +307,23 @@ void MicrosoftCXXNameMangler::mangleName(const NamedDecl *ND) {
 }
 
 void MicrosoftCXXNameMangler::mangleNumber(int64_t Number) {
-  // <number> ::= [?] <decimal digit> # 1 <= Number <= 10
-  //          ::= [?] <hex digit>+ @ # 0 or > 9; A = 0, B = 1, etc...
-  //          ::= [?] @ # 0 (alternate mangling, not emitted by VC)
-  if (Number < 0) {
-    Out << '?';
-    Number = -Number;
-  }
-  // There's a special shorter mangling for 0, but Microsoft
-  // chose not to use it. Instead, 0 gets mangled as "A@". Oh well...
-  if (Number >= 1 && Number <= 10)
-    Out << Number-1;
-  else {
-    // We have to build up the encoding in reverse order, so it will come
-    // out right when we write it out.
-    char Encoding[16];
-    char *EndPtr = Encoding+sizeof(Encoding);
-    char *CurPtr = EndPtr;
-    do {
-      *--CurPtr = 'A' + (Number % 16);
-      Number /= 16;
-    } while (Number);
-    Out.write(CurPtr, EndPtr-CurPtr);
-    Out << '@';
-  }
+  llvm::APSInt APSNumber(/*BitWidth=*/64, /*isUnsigned=*/false);
+  APSNumber = Number;
+  mangleNumber(APSNumber);
 }
 
 void MicrosoftCXXNameMangler::mangleNumber(const llvm::APSInt &Value) {
+  // <number> ::= [?] <decimal digit> # 1 <= Number <= 10
+  //          ::= [?] <hex digit>+ @ # 0 or > 9; A = 0, B = 1, etc...
+  //          ::= [?] @ # 0 (alternate mangling, not emitted by VC)
   if (Value.isSigned() && Value.isNegative()) {
     Out << '?';
     mangleNumber(llvm::APSInt(Value.abs()));
     return;
   }
   llvm::APSInt Temp(Value);
+  // There's a special shorter mangling for 0, but Microsoft
+  // chose not to use it. Instead, 0 gets mangled as "A@". Oh well...
   if (Value.uge(1) && Value.ule(10)) {
     --Temp;
     Temp.print(Out, false);
@@ -348,10 +335,10 @@ void MicrosoftCXXNameMangler::mangleNumber(const llvm::APSInt &Value) {
     char *CurPtr = EndPtr;
     llvm::APSInt NibbleMask(Value.getBitWidth(), Value.isUnsigned());
     NibbleMask = 0xf;
-    for (int i = 0, e = Value.getActiveBits() / 4; i != e; ++i) {
+    do {
       *--CurPtr = 'A' + Temp.And(NibbleMask).getLimitedValue(0xf);
       Temp = Temp.lshr(4);
-    }
+    } while (Temp != 0);
     Out.write(CurPtr, EndPtr-CurPtr);
     Out << '@';
   }
@@ -386,7 +373,7 @@ isTemplate(const NamedDecl *ND,
       dyn_cast<ClassTemplateSpecializationDecl>(ND)) {
     TypeSourceInfo *TSI = Spec->getTypeAsWritten();
     if (TSI) {
-      TemplateSpecializationTypeLoc &TSTL =
+      TemplateSpecializationTypeLoc TSTL =
         cast<TemplateSpecializationTypeLoc>(TSI->getTypeLoc());
       TemplateArgumentListInfo LI(TSTL.getLAngleLoc(), TSTL.getRAngleLoc());
       for (unsigned i = 0, e = TSTL.getNumArgs(); i != e; ++i)
@@ -784,6 +771,23 @@ MicrosoftCXXNameMangler::mangleIntegerLiteral(QualType T,
 }
 
 void
+MicrosoftCXXNameMangler::mangleExpression(const Expr *E) {
+  // See if this is a constant expression.
+  llvm::APSInt Value;
+  if (E->isIntegerConstantExpr(Value, Context.getASTContext())) {
+    mangleIntegerLiteral(E->getType(), Value);
+    return;
+  }
+
+  // As bad as this diagnostic is, it's better than crashing.
+  DiagnosticsEngine &Diags = Context.getDiags();
+  unsigned DiagID = Diags.getCustomDiagID(DiagnosticsEngine::Error,
+                                   "cannot yet mangle expression type %0");
+  Diags.Report(E->getExprLoc(), DiagID)
+    << E->getStmtClassName() << E->getSourceRange();
+}
+
+void
 MicrosoftCXXNameMangler::mangleTemplateArgs(
                      const SmallVectorImpl<TemplateArgumentLoc> &TemplateArgs) {
   // <template-args> ::= {<type> | <integer-literal>}+ @
@@ -800,21 +804,19 @@ MicrosoftCXXNameMangler::mangleTemplateArgs(
     case TemplateArgument::Integral:
       mangleIntegerLiteral(TA.getIntegralType(), TA.getAsIntegral());
       break;
-    case TemplateArgument::Expression: {
-      // See if this is a constant expression.
-      Expr *TAE = TA.getAsExpr();
-      llvm::APSInt Value;
-      if (TAE->isIntegerConstantExpr(Value, Context.getASTContext())) {
-        mangleIntegerLiteral(TAE->getType(), Value);
-        break;
-      }
-      /* fallthrough */
-    } default: {
+    case TemplateArgument::Expression:
+      mangleExpression(TA.getAsExpr());
+      break;
+    case TemplateArgument::Template:
+    case TemplateArgument::TemplateExpansion:
+    case TemplateArgument::Declaration:
+    case TemplateArgument::NullPtr:
+    case TemplateArgument::Pack: {
       // Issue a diagnostic.
       DiagnosticsEngine &Diags = Context.getDiags();
       unsigned DiagID = Diags.getCustomDiagID(DiagnosticsEngine::Error,
-        "cannot mangle this %select{ERROR|ERROR|pointer/reference|ERROR|"
-        "template|template pack expansion|expression|parameter pack}0 "
+        "cannot mangle this %select{ERROR|ERROR|pointer/reference|nullptr|"
+        "integral|template|template pack expansion|ERROR|parameter pack}0 "
         "template argument yet");
       Diags.Report(TAL.getLocation(), DiagID)
         << TA.getKind()
@@ -879,43 +881,60 @@ void MicrosoftCXXNameMangler::mangleQualifiers(Qualifiers Quals,
   //         ::= 3 # ?
   //         ::= 4 # ?
   //         ::= 5 # not really based
+  bool HasConst = Quals.hasConst(),
+       HasVolatile = Quals.hasVolatile();
   if (!IsMember) {
-    if (!Quals.hasVolatile()) {
-      if (!Quals.hasConst())
-        Out << 'A';
-      else
-        Out << 'B';
+    if (HasConst && HasVolatile) {
+      Out << 'D';
+    } else if (HasVolatile) {
+      Out << 'C';
+    } else if (HasConst) {
+      Out << 'B';
     } else {
-      if (!Quals.hasConst())
-        Out << 'C';
-      else
-        Out << 'D';
+      Out << 'A';
     }
   } else {
-    if (!Quals.hasVolatile()) {
-      if (!Quals.hasConst())
-        Out << 'Q';
-      else
-        Out << 'R';
+    if (HasConst && HasVolatile) {
+      Out << 'T';
+    } else if (HasVolatile) {
+      Out << 'S';
+    } else if (HasConst) {
+      Out << 'R';
     } else {
-      if (!Quals.hasConst())
-        Out << 'S';
-      else
-        Out << 'T';
+      Out << 'Q';
     }
   }
 
   // FIXME: For now, just drop all extension qualifiers on the floor.
 }
 
-void MicrosoftCXXNameMangler::mangleTypeRepeated(QualType T, SourceRange Range) {
+void MicrosoftCXXNameMangler::manglePointerQualifiers(Qualifiers Quals) {
+  // <pointer-cvr-qualifiers> ::= P  # no qualifiers
+  //                          ::= Q  # const
+  //                          ::= R  # volatile
+  //                          ::= S  # const volatile
+  bool HasConst = Quals.hasConst(),
+       HasVolatile = Quals.hasVolatile();
+  if (HasConst && HasVolatile) {
+    Out << 'S';
+  } else if (HasVolatile) {
+    Out << 'R';
+  } else if (HasConst) {
+    Out << 'Q';
+  } else {
+    Out << 'P';
+  }
+}
+
+void MicrosoftCXXNameMangler::mangleArgumentType(QualType T,
+                                                 SourceRange Range) {
   void *TypePtr = getASTContext().getCanonicalType(T).getAsOpaquePtr();
   ArgBackRefMap::iterator Found = TypeBackReferences.find(TypePtr);
 
   if (Found == TypeBackReferences.end()) {
     size_t OutSizeBefore = Out.GetNumBytesInBuffer();
 
-    mangleType(T,Range);
+    mangleType(T, Range, false);
 
     // See if it's worth creating a back reference.
     // Only types longer than 1 character are considered
@@ -930,38 +949,30 @@ void MicrosoftCXXNameMangler::mangleTypeRepeated(QualType T, SourceRange Range)
   }
 }
 
-void MicrosoftCXXNameMangler::mangleType(QualType T, SourceRange Range) {
+void MicrosoftCXXNameMangler::mangleType(QualType T, SourceRange Range,
+                                         bool MangleQualifiers) {
   // Only operate on the canonical type!
   T = getASTContext().getCanonicalType(T);
-  
+
   Qualifiers Quals = T.getLocalQualifiers();
-  if (Quals) {
-    // We have to mangle these now, while we still have enough information.
-    // <pointer-cvr-qualifiers> ::= P  # pointer
-    //                          ::= Q  # const pointer
-    //                          ::= R  # volatile pointer
-    //                          ::= S  # const volatile pointer
-    if (T->isAnyPointerType() || T->isMemberPointerType() ||
-        T->isBlockPointerType()) {
-      if (!Quals.hasVolatile())
-        Out << 'Q';
-      else {
-        if (!Quals.hasConst())
-          Out << 'R';
-        else
-          Out << 'S';
-      }
-    } else
-      // Just emit qualifiers like normal.
-      // NB: When we mangle a pointer/reference type, and the pointee
-      // type has no qualifiers, the lack of qualifier gets mangled
-      // in there.
-      mangleQualifiers(Quals, false);
-  } else if (T->isAnyPointerType() || T->isMemberPointerType() ||
-             T->isBlockPointerType()) {
-    Out << 'P';
+  // We have to mangle these now, while we still have enough information.
+  if (T->isAnyPointerType() || T->isMemberPointerType() ||
+      T->isBlockPointerType()) {
+    manglePointerQualifiers(Quals);
+  } else if (Quals && MangleQualifiers) {
+    mangleQualifiers(Quals, false);
   }
-  switch (T->getTypeClass()) {
+
+  SplitQualType split = T.split();
+  const Type *ty = split.Ty;
+
+  // If we're mangling a qualified array type, push the qualifiers to
+  // the element type.
+  if (split.Quals && isa<ArrayType>(T)) {
+    ty = Context.getASTContext().getAsArrayType(T);
+  }
+
+  switch (ty->getTypeClass()) {
 #define ABSTRACT_TYPE(CLASS, PARENT)
 #define NON_CANONICAL_TYPE(CLASS, PARENT) \
   case Type::CLASS: \
@@ -969,7 +980,7 @@ void MicrosoftCXXNameMangler::mangleType(QualType T, SourceRange Range) {
     return;
 #define TYPE(CLASS, PARENT) \
   case Type::CLASS: \
-    mangleType(static_cast<const CLASS##Type*>(T.getTypePtr()), Range); \
+    mangleType(cast<CLASS##Type>(ty), Range); \
     break;
 #include "clang/AST/TypeNodes.def"
 #undef ABSTRACT_TYPE
@@ -1059,6 +1070,8 @@ void MicrosoftCXXNameMangler::mangleType(const FunctionProtoType *T,
                                          SourceRange) {
   // Structors only appear in decls, so at this point we know it's not a
   // structor type.
+  // FIXME: This may not be lambda-friendly.
+  Out << "$$A6";
   mangleType(T, NULL, false, false);
 }
 void MicrosoftCXXNameMangler::mangleType(const FunctionNoProtoType *T,
@@ -1117,14 +1130,14 @@ void MicrosoftCXXNameMangler::mangleType(const FunctionType *T,
              ParmEnd = D->param_end(); Parm != ParmEnd; ++Parm) {
         TypeSourceInfo *TSI = (*Parm)->getTypeSourceInfo();
         QualType Type = TSI ? TSI->getType() : (*Parm)->getType();
-        mangleTypeRepeated(Type, (*Parm)->getSourceRange());
+        mangleArgumentType(Type, (*Parm)->getSourceRange());
       }
     } else {
       // Happens for function pointer type arguments for example.
       for (FunctionProtoType::arg_type_iterator Arg = Proto->arg_type_begin(),
            ArgEnd = Proto->arg_type_end();
            Arg != ArgEnd; ++Arg)
-        mangleTypeRepeated(*Arg, SourceRange());
+        mangleArgumentType(*Arg, SourceRange());
     }
     // <builtin-type>      ::= Z  # ellipsis
     if (Proto->isVariadic())
@@ -1214,7 +1227,7 @@ void MicrosoftCXXNameMangler::mangleCallingConvention(const FunctionType *T,
   if (CC == CC_Default) {
     if (IsInstMethod) {
       const FunctionProtoType *FPT =
-        T->getCanonicalTypeUnqualified().getAs<FunctionProtoType>();
+        T->getCanonicalTypeUnqualified().castAs<FunctionProtoType>();
       bool isVariadic = FPT->isVariadic();
       CC = getASTContext().getDefaultCXXMethodCallConv(isVariadic);
     } else {
@@ -1260,10 +1273,10 @@ void MicrosoftCXXNameMangler::mangleType(const UnresolvedUsingType *T,
 // <class-type>  ::= V <name>
 // <enum-type>   ::= W <size> <name>
 void MicrosoftCXXNameMangler::mangleType(const EnumType *T, SourceRange) {
-  mangleType(static_cast<const TagType*>(T));
+  mangleType(cast<TagType>(T));
 }
 void MicrosoftCXXNameMangler::mangleType(const RecordType *T, SourceRange) {
-  mangleType(static_cast<const TagType*>(T));
+  mangleType(cast<TagType>(T));
 }
 void MicrosoftCXXNameMangler::mangleType(const TagType *T) {
   switch (T->getDecl()->getTagKind()) {
@@ -1271,6 +1284,7 @@ void MicrosoftCXXNameMangler::mangleType(const TagType *T) {
       Out << 'T';
       break;
     case TTK_Struct:
+    case TTK_Interface:
       Out << 'U';
       break;
     case TTK_Class:
@@ -1286,37 +1300,39 @@ void MicrosoftCXXNameMangler::mangleType(const TagType *T) {
 }
 
 // <type>       ::= <array-type>
-// <array-type> ::= P <cvr-qualifiers> [Y <dimension-count> <dimension>+]
-//                                                  <element-type> # as global
+// <array-type> ::= <pointer-cvr-qualifiers> <cvr-qualifiers>
+//                  [Y <dimension-count> <dimension>+]
+//                  <element-type> # as global
 //              ::= Q <cvr-qualifiers> [Y <dimension-count> <dimension>+]
-//                                                  <element-type> # as param
+//                  <element-type> # as param
 // It's supposed to be the other way around, but for some strange reason, it
 // isn't. Today this behavior is retained for the sole purpose of backwards
 // compatibility.
 void MicrosoftCXXNameMangler::mangleType(const ArrayType *T, bool IsGlobal) {
   // This isn't a recursive mangling, so now we have to do it all in this
   // one call.
-  if (IsGlobal)
-    Out << 'P';
-  else
+  if (IsGlobal) {
+    manglePointerQualifiers(T->getElementType().getQualifiers());
+  } else {
     Out << 'Q';
+  }
   mangleExtraDimensions(T->getElementType());
 }
 void MicrosoftCXXNameMangler::mangleType(const ConstantArrayType *T,
                                          SourceRange) {
-  mangleType(static_cast<const ArrayType *>(T), false);
+  mangleType(cast<ArrayType>(T), false);
 }
 void MicrosoftCXXNameMangler::mangleType(const VariableArrayType *T,
                                          SourceRange) {
-  mangleType(static_cast<const ArrayType *>(T), false);
+  mangleType(cast<ArrayType>(T), false);
 }
 void MicrosoftCXXNameMangler::mangleType(const DependentSizedArrayType *T,
                                          SourceRange) {
-  mangleType(static_cast<const ArrayType *>(T), false);
+  mangleType(cast<ArrayType>(T), false);
 }
 void MicrosoftCXXNameMangler::mangleType(const IncompleteArrayType *T,
                                          SourceRange) {
-  mangleType(static_cast<const ArrayType *>(T), false);
+  mangleType(cast<ArrayType>(T), false);
 }
 void MicrosoftCXXNameMangler::mangleExtraDimensions(QualType ElementTy) {
   SmallVector<llvm::APInt, 3> Dimensions;
@@ -1409,10 +1425,8 @@ void MicrosoftCXXNameMangler::mangleType(const PointerType *T,
     Out << '6';
     mangleType(FT, NULL, false, false);
   } else {
-    if (!PointeeTy.hasQualifiers())
-      // Lack of qualifiers is mangled as 'A'.
-      Out << 'A';
-    mangleType(PointeeTy, Range);
+    mangleQualifiers(PointeeTy.getQualifiers(), false);
+    mangleType(PointeeTy, Range, false);
   }
 }
 void MicrosoftCXXNameMangler::mangleType(const ObjCObjectPointerType *T,
@@ -1497,7 +1511,9 @@ void MicrosoftCXXNameMangler::mangleType(const ObjCObjectType *T,
 void MicrosoftCXXNameMangler::mangleType(const BlockPointerType *T,
                                          SourceRange Range) {
   Out << "_E";
-  mangleType(T->getPointeeType(), Range);
+
+  QualType pointee = T->getPointeeType();
+  mangleType(pointee->castAs<FunctionProtoType>(), NULL, false, false);
 }
 
 void MicrosoftCXXNameMangler::mangleType(const InjectedClassNameType *T,
diff --git a/lib/AST/NSAPI.cpp b/lib/AST/NSAPI.cpp
index 39077d1..0837509 100644
--- a/lib/AST/NSAPI.cpp
+++ b/lib/AST/NSAPI.cpp
@@ -344,6 +344,7 @@ NSAPI::getNSNumberFactoryMethodKind(QualType T) const {
   case BuiltinType::ARCUnbridgedCast:
   case BuiltinType::Half:
   case BuiltinType::PseudoObject:
+  case BuiltinType::BuiltinFn:
     break;
   }
   
diff --git a/lib/AST/ParentMap.cpp b/lib/AST/ParentMap.cpp
index fa87afd..1135928 100644
--- a/lib/AST/ParentMap.cpp
+++ b/lib/AST/ParentMap.cpp
@@ -20,22 +20,63 @@ using namespace clang;
 
 typedef llvm::DenseMap<Stmt*, Stmt*> MapTy;
 
-static void BuildParentMap(MapTy& M, Stmt* S) {
-  for (Stmt::child_range I = S->children(); I; ++I)
-    if (*I) {
-      // Prefer the first time we see this statement in the traversal.
-      // This is important for PseudoObjectExprs.
-      Stmt *&Parent = M[*I];
-      if (!Parent) {
-        Parent = S;
-        BuildParentMap(M, *I);
+enum OpaqueValueMode {
+  OV_Transparent,
+  OV_Opaque
+};
+
+static void BuildParentMap(MapTy& M, Stmt* S,
+                           OpaqueValueMode OVMode = OV_Transparent) {
+
+  switch (S->getStmtClass()) {
+  case Stmt::PseudoObjectExprClass: {
+    assert(OVMode == OV_Transparent && "Should not appear alongside OVEs");
+    PseudoObjectExpr *POE = cast<PseudoObjectExpr>(S);
+
+    M[POE->getSyntacticForm()] = S;
+    BuildParentMap(M, POE->getSyntacticForm(), OV_Transparent);
+
+    for (PseudoObjectExpr::semantics_iterator I = POE->semantics_begin(),
+                                              E = POE->semantics_end();
+         I != E; ++I) {
+      M[*I] = S;
+      BuildParentMap(M, *I, OV_Opaque);
+    }
+    break;
+  }
+  case Stmt::BinaryConditionalOperatorClass: {
+    assert(OVMode == OV_Transparent && "Should not appear alongside OVEs");
+    BinaryConditionalOperator *BCO = cast<BinaryConditionalOperator>(S);
+
+    M[BCO->getCommon()] = S;
+    BuildParentMap(M, BCO->getCommon(), OV_Transparent);
+
+    M[BCO->getCond()] = S;
+    BuildParentMap(M, BCO->getCond(), OV_Opaque);
+
+    M[BCO->getTrueExpr()] = S;
+    BuildParentMap(M, BCO->getTrueExpr(), OV_Opaque);
+
+    M[BCO->getFalseExpr()] = S;
+    BuildParentMap(M, BCO->getFalseExpr(), OV_Transparent);
+
+    break;
+  }
+  case Stmt::OpaqueValueExprClass:
+    if (OVMode == OV_Transparent) {
+      OpaqueValueExpr *OVE = cast<OpaqueValueExpr>(S);
+      M[OVE->getSourceExpr()] = S;
+      BuildParentMap(M, OVE->getSourceExpr(), OV_Transparent);
+    }
+    break;
+  default:
+    for (Stmt::child_range I = S->children(); I; ++I) {
+      if (*I) {
+        M[*I] = S;
+        BuildParentMap(M, *I, OVMode);
       }
     }
-  
-  // Also include the source expr tree of an OpaqueValueExpr in the map.
-  if (const OpaqueValueExpr *OVE = dyn_cast<OpaqueValueExpr>(S)) {
-    M[OVE->getSourceExpr()] = S;
-    BuildParentMap(M, OVE->getSourceExpr());
+    break;
   }
 }
 
diff --git a/lib/AST/RawCommentList.cpp b/lib/AST/RawCommentList.cpp
index a5a3287..80b6272 100644
--- a/lib/AST/RawCommentList.cpp
+++ b/lib/AST/RawCommentList.cpp
@@ -143,11 +143,10 @@ const char *RawComment::extractBriefText(const ASTContext &Context) const {
   // a separate allocator for all temporary stuff.
   llvm::BumpPtrAllocator Allocator;
 
-  comments::CommandTraits Traits;
-  comments::Lexer L(Allocator, Traits,
-                    Range.getBegin(), comments::CommentOptions(),
+  comments::Lexer L(Allocator, Context.getCommentCommandTraits(),
+                    Range.getBegin(),
                     RawText.begin(), RawText.end());
-  comments::BriefParser P(L, Traits);
+  comments::BriefParser P(L, Context.getCommentCommandTraits());
 
   const std::string Result = P.Parse();
   const unsigned BriefTextLength = Result.size();
@@ -160,19 +159,22 @@ const char *RawComment::extractBriefText(const ASTContext &Context) const {
 }
 
 comments::FullComment *RawComment::parse(const ASTContext &Context,
+                                         const Preprocessor *PP,
                                          const Decl *D) const {
   // Make sure that RawText is valid.
   getRawText(Context.getSourceManager());
 
-  comments::CommandTraits Traits;
-  comments::Lexer L(Context.getAllocator(), Traits,
-                    getSourceRange().getBegin(), comments::CommentOptions(),
+  comments::Lexer L(Context.getAllocator(), Context.getCommentCommandTraits(),
+                    getSourceRange().getBegin(),
                     RawText.begin(), RawText.end());
   comments::Sema S(Context.getAllocator(), Context.getSourceManager(),
-                   Context.getDiagnostics(), Traits);
+                   Context.getDiagnostics(),
+                   Context.getCommentCommandTraits(),
+                   PP);
   S.setDecl(D);
   comments::Parser P(L, S, Context.getAllocator(), Context.getSourceManager(),
-                     Context.getDiagnostics(), Traits);
+                     Context.getDiagnostics(),
+                     Context.getCommentCommandTraits());
 
   return P.parseFullComment();
 }
@@ -182,26 +184,22 @@ bool containsOnlyWhitespace(StringRef Str) {
   return Str.find_first_not_of(" \t\f\v\r\n") == StringRef::npos;
 }
 
-bool onlyWhitespaceBetweenComments(SourceManager &SM,
-                                   const RawComment &C1, const RawComment &C2) {
-  std::pair<FileID, unsigned> C1EndLocInfo = SM.getDecomposedLoc(
-                                                C1.getSourceRange().getEnd());
-  std::pair<FileID, unsigned> C2BeginLocInfo = SM.getDecomposedLoc(
-                                              C2.getSourceRange().getBegin());
+bool onlyWhitespaceBetween(SourceManager &SM,
+                           SourceLocation Loc1, SourceLocation Loc2) {
+  std::pair<FileID, unsigned> Loc1Info = SM.getDecomposedLoc(Loc1);
+  std::pair<FileID, unsigned> Loc2Info = SM.getDecomposedLoc(Loc2);
 
-  // Question does not make sense if comments are located in different files.
-  if (C1EndLocInfo.first != C2BeginLocInfo.first)
+  // Question does not make sense if locations are in different files.
+  if (Loc1Info.first != Loc2Info.first)
     return false;
 
   bool Invalid = false;
-  const char *Buffer = SM.getBufferData(C1EndLocInfo.first, &Invalid).data();
+  const char *Buffer = SM.getBufferData(Loc1Info.first, &Invalid).data();
   if (Invalid)
     return false;
 
-  StringRef TextBetweenComments(Buffer + C1EndLocInfo.second,
-                                C2BeginLocInfo.second - C1EndLocInfo.second);
-
-  return containsOnlyWhitespace(TextBetweenComments);
+  StringRef Text(Buffer + Loc1Info.second, Loc2Info.second - Loc1Info.second);
+  return containsOnlyWhitespace(Text);
 }
 } // unnamed namespace
 
@@ -221,11 +219,13 @@ void RawCommentList::addComment(const RawComment &RC,
   }
 
   if (OnlyWhitespaceSeen) {
-    if (!onlyWhitespaceBetweenComments(SourceMgr, LastComment, RC))
+    if (!onlyWhitespaceBetween(SourceMgr,
+                               PrevCommentEndLoc,
+                               RC.getSourceRange().getBegin()))
       OnlyWhitespaceSeen = false;
   }
 
-  LastComment = RC;
+  PrevCommentEndLoc = RC.getSourceRange().getEnd();
 
   // Ordinary comments are not interesting for us.
   if (RC.isOrdinary())
@@ -244,15 +244,20 @@ void RawCommentList::addComment(const RawComment &RC,
 
   // Merge comments only if there is only whitespace between them.
   // Can't merge trailing and non-trailing comments.
-  // Merge trailing comments if they are on same or consecutive lines.
+  // Merge comments if they are on same or consecutive lines.
+  bool Merged = false;
   if (OnlyWhitespaceSeen &&
-      (C1.isTrailingComment() == C2.isTrailingComment()) &&
-      (!C1.isTrailingComment() ||
-       C1.getEndLine(SourceMgr) + 1 >= C2.getBeginLine(SourceMgr))) {
-    SourceRange MergedRange(C1.getSourceRange().getBegin(),
-                            C2.getSourceRange().getEnd());
-    *Comments.back() = RawComment(SourceMgr, MergedRange, true);
-  } else
+      (C1.isTrailingComment() == C2.isTrailingComment())) {
+    unsigned C1EndLine = C1.getEndLine(SourceMgr);
+    unsigned C2BeginLine = C2.getBeginLine(SourceMgr);
+    if (C1EndLine + 1 == C2BeginLine || C1EndLine == C2BeginLine) {
+      SourceRange MergedRange(C1.getSourceRange().getBegin(),
+                              C2.getSourceRange().getEnd());
+      *Comments.back() = RawComment(SourceMgr, MergedRange, true);
+      Merged = true;
+    }
+  }
+  if (!Merged)
     Comments.push_back(new (Allocator) RawComment(RC));
 
   OnlyWhitespaceSeen = true;
diff --git a/lib/AST/RecordLayoutBuilder.cpp b/lib/AST/RecordLayoutBuilder.cpp
index d5df63f..4dfffc4 100644
--- a/lib/AST/RecordLayoutBuilder.cpp
+++ b/lib/AST/RecordLayoutBuilder.cpp
@@ -789,8 +789,8 @@ protected:
   void setDataSize(CharUnits NewSize) { DataSize = Context.toBits(NewSize); }
   void setDataSize(uint64_t NewSize) { DataSize = NewSize; }
 
-  RecordLayoutBuilder(const RecordLayoutBuilder&);   // DO NOT IMPLEMENT
-  void operator=(const RecordLayoutBuilder&); // DO NOT IMPLEMENT
+  RecordLayoutBuilder(const RecordLayoutBuilder &) LLVM_DELETED_FUNCTION;
+  void operator=(const RecordLayoutBuilder &) LLVM_DELETED_FUNCTION;
 public:
   static const CXXMethodDecl *ComputeKeyFunction(const CXXRecordDecl *RD);
 };
@@ -1557,6 +1557,13 @@ CharUnits RecordLayoutBuilder::LayoutBase(const BaseSubobjectInfo *Base) {
     bool Allowed = EmptySubobjects->CanPlaceBaseAtOffset(Base, Offset);
     (void)Allowed;
     assert(Allowed && "Base subobject externally placed at overlapping offset");
+
+    if (InferAlignment && Offset < getDataSize().RoundUpToAlignment(BaseAlign)){
+      // The externally-supplied base offset is before the base offset we
+      // computed. Assume that the structure is packed.
+      Alignment = CharUnits::One();
+      InferAlignment = false;
+    }
   }
   
   if (!Base->Class->isEmpty()) {
@@ -1574,12 +1581,12 @@ CharUnits RecordLayoutBuilder::LayoutBase(const BaseSubobjectInfo *Base) {
 }
 
 void RecordLayoutBuilder::InitializeLayout(const Decl *D) {
-  if (const RecordDecl *RD = dyn_cast<RecordDecl>(D))
+  if (const RecordDecl *RD = dyn_cast<RecordDecl>(D)) {
     IsUnion = RD->isUnion();
+    IsMsStruct = RD->isMsStruct(Context);
+  }
 
-  Packed = D->hasAttr<PackedAttr>();
-  
-  IsMsStruct = D->hasAttr<MsStructAttr>();
+  Packed = D->hasAttr<PackedAttr>();  
 
   // Honor the default struct packing maximum alignment flag.
   if (unsigned DefaultMaxFieldAlignment = Context.getLangOpts().PackStruct) {
@@ -1616,7 +1623,6 @@ void RecordLayoutBuilder::InitializeLayout(const Decl *D) {
       if (ExternalLayout) {
         if (ExternalAlign > 0) {
           Alignment = Context.toCharUnitsFromBits(ExternalAlign);
-          UnpackedAlignment = Alignment;
         } else {
           // The external source didn't have alignment information; infer it.
           InferAlignment = true;
@@ -2085,7 +2091,7 @@ void RecordLayoutBuilder::LayoutField(const FieldDecl *D) {
       ZeroLengthBitfield = 0;
     }
 
-    if (Context.getLangOpts().MSBitfields || IsMsStruct) {
+    if (IsMsStruct) {
       // If MS bitfield layout is required, figure out what type is being
       // laid out and align the field to the width of that type.
       
@@ -2166,11 +2172,6 @@ void RecordLayoutBuilder::LayoutField(const FieldDecl *D) {
 }
 
 void RecordLayoutBuilder::FinishLayout(const NamedDecl *D) {
-  if (ExternalLayout) {
-    setSize(ExternalSize);
-    return;
-  }
-  
   // In C++, records cannot be of size 0.
   if (Context.getLangOpts().CPlusPlus && getSizeInBits() == 0) {
     if (const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(D)) {
@@ -2184,20 +2185,37 @@ void RecordLayoutBuilder::FinishLayout(const NamedDecl *D) {
       setSize(CharUnits::One());
   }
 
+  // Finally, round the size of the record up to the alignment of the
+  // record itself.
+  uint64_t UnpaddedSize = getSizeInBits() - UnfilledBitsInLastByte;
+  uint64_t UnpackedSizeInBits =
+  llvm::RoundUpToAlignment(getSizeInBits(),
+                           Context.toBits(UnpackedAlignment));
+  CharUnits UnpackedSize = Context.toCharUnitsFromBits(UnpackedSizeInBits);
+  uint64_t RoundedSize
+    = llvm::RoundUpToAlignment(getSizeInBits(), Context.toBits(Alignment));
+
+  if (ExternalLayout) {
+    // If we're inferring alignment, and the external size is smaller than
+    // our size after we've rounded up to alignment, conservatively set the
+    // alignment to 1.
+    if (InferAlignment && ExternalSize < RoundedSize) {
+      Alignment = CharUnits::One();
+      InferAlignment = false;
+    }
+    setSize(ExternalSize);
+    return;
+  }
+
+
   // MSVC doesn't round up to the alignment of the record with virtual bases.
   if (const CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(D)) {
     if (isMicrosoftCXXABI() && RD->getNumVBases())
       return;
   }
 
-  // Finally, round the size of the record up to the alignment of the
-  // record itself.
-  uint64_t UnpaddedSize = getSizeInBits() - UnfilledBitsInLastByte;
-  uint64_t UnpackedSizeInBits = 
-    llvm::RoundUpToAlignment(getSizeInBits(), 
-                             Context.toBits(UnpackedAlignment));
-  CharUnits UnpackedSize = Context.toCharUnitsFromBits(UnpackedSizeInBits);
-  setSize(llvm::RoundUpToAlignment(getSizeInBits(), Context.toBits(Alignment)));
+  // Set the size to the final size.
+  setSize(RoundedSize);
 
   unsigned CharBitNum = Context.getTargetInfo().getCharWidth();
   if (const RecordDecl *RD = dyn_cast<RecordDecl>(D)) {
@@ -2255,7 +2273,7 @@ RecordLayoutBuilder::updateExternalFieldOffset(const FieldDecl *Field,
   if (InferAlignment && ExternalFieldOffset < ComputedOffset) {
     // The externally-supplied field offset is before the field offset we
     // computed. Assume that the structure is packed.
-    Alignment = CharUnits::fromQuantity(1);
+    Alignment = CharUnits::One();
     InferAlignment = false;
   }
   
@@ -2263,6 +2281,20 @@ RecordLayoutBuilder::updateExternalFieldOffset(const FieldDecl *Field,
   return ExternalFieldOffset;
 }
 
+/// \brief Get diagnostic %select index for tag kind for
+/// field padding diagnostic message.
+/// WARNING: Indexes apply to particular diagnostics only!
+///
+/// \returns diagnostic %select index.
+static unsigned getPaddingDiagFromTagKind(TagTypeKind Tag) {
+  switch (Tag) {
+  case TTK_Struct: return 0;
+  case TTK_Interface: return 1;
+  case TTK_Class: return 2;
+  default: llvm_unreachable("Invalid tag kind for field padding diagnostic!");
+  }
+}
+
 void RecordLayoutBuilder::CheckFieldPadding(uint64_t Offset,
                                             uint64_t UnpaddedOffset,
                                             uint64_t UnpackedOffset,
@@ -2291,14 +2323,14 @@ void RecordLayoutBuilder::CheckFieldPadding(uint64_t Offset,
     }
     if (D->getIdentifier())
       Diag(D->getLocation(), diag::warn_padded_struct_field)
-          << (D->getParent()->isStruct() ? 0 : 1) // struct|class
+          << getPaddingDiagFromTagKind(D->getParent()->getTagKind())
           << Context.getTypeDeclType(D->getParent())
           << PadSize
           << (InBits ? 1 : 0) /*(byte|bit)*/ << (PadSize > 1) // plural or not
           << D->getIdentifier();
     else
       Diag(D->getLocation(), diag::warn_padded_struct_anon_field)
-          << (D->getParent()->isStruct() ? 0 : 1) // struct|class
+          << getPaddingDiagFromTagKind(D->getParent()->getTagKind())
           << Context.getTypeDeclType(D->getParent())
           << PadSize
           << (InBits ? 1 : 0) /*(byte|bit)*/ << (PadSize > 1); // plural or not
@@ -2508,8 +2540,8 @@ ASTContext::getObjCLayout(const ObjCInterfaceDecl *D,
   assert(D && D->isThisDeclarationADefinition() && "Invalid interface decl!");
 
   // Look up this layout, if already laid out, return what we have.
-  ObjCContainerDecl *Key =
-    Impl ? (ObjCContainerDecl*) Impl : (ObjCContainerDecl*) D;
+  const ObjCContainerDecl *Key =
+    Impl ? (const ObjCContainerDecl*) Impl : (const ObjCContainerDecl*) D;
   if (const ASTRecordLayout *Entry = ObjCLayouts[Key])
     return *Entry;
 
diff --git a/lib/AST/Stmt.cpp b/lib/AST/Stmt.cpp
index 77452c9..eafcf92 100644
--- a/lib/AST/Stmt.cpp
+++ b/lib/AST/Stmt.cpp
@@ -20,6 +20,7 @@
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/ASTDiagnostic.h"
 #include "clang/Basic/TargetInfo.h"
+#include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/raw_ostream.h"
 using namespace clang;
 
@@ -320,15 +321,52 @@ bool Stmt::hasImplicitControlFlow() const {
   }
 }
 
-Expr *AsmStmt::getOutputExpr(unsigned i) {
-  return cast<Expr>(Exprs[i]);
+std::string AsmStmt::generateAsmString(ASTContext &C) const {
+  if (const GCCAsmStmt *gccAsmStmt = dyn_cast<GCCAsmStmt>(this))
+    return gccAsmStmt->generateAsmString(C);
+  if (const MSAsmStmt *msAsmStmt = dyn_cast<MSAsmStmt>(this))
+    return msAsmStmt->generateAsmString(C);
+  llvm_unreachable("unknown asm statement kind!");
 }
 
-/// getOutputConstraint - Return the constraint string for the specified
-/// output operand.  All output constraints are known to be non-empty (either
-/// '=' or '+').
 StringRef AsmStmt::getOutputConstraint(unsigned i) const {
-  return getOutputConstraintLiteral(i)->getString();
+  if (const GCCAsmStmt *gccAsmStmt = dyn_cast<GCCAsmStmt>(this))
+    return gccAsmStmt->getOutputConstraint(i);
+  if (const MSAsmStmt *msAsmStmt = dyn_cast<MSAsmStmt>(this))
+    return msAsmStmt->getOutputConstraint(i);
+  llvm_unreachable("unknown asm statement kind!");
+}
+
+const Expr *AsmStmt::getOutputExpr(unsigned i) const {
+  if (const GCCAsmStmt *gccAsmStmt = dyn_cast<GCCAsmStmt>(this))
+    return gccAsmStmt->getOutputExpr(i);
+  if (const MSAsmStmt *msAsmStmt = dyn_cast<MSAsmStmt>(this))
+    return msAsmStmt->getOutputExpr(i);
+  llvm_unreachable("unknown asm statement kind!");
+}
+
+StringRef AsmStmt::getInputConstraint(unsigned i) const {
+  if (const GCCAsmStmt *gccAsmStmt = dyn_cast<GCCAsmStmt>(this))
+    return gccAsmStmt->getInputConstraint(i);
+  if (const MSAsmStmt *msAsmStmt = dyn_cast<MSAsmStmt>(this))
+    return msAsmStmt->getInputConstraint(i);
+  llvm_unreachable("unknown asm statement kind!");
+}
+
+const Expr *AsmStmt::getInputExpr(unsigned i) const {
+  if (const GCCAsmStmt *gccAsmStmt = dyn_cast<GCCAsmStmt>(this))
+    return gccAsmStmt->getInputExpr(i);
+  if (const MSAsmStmt *msAsmStmt = dyn_cast<MSAsmStmt>(this))
+    return msAsmStmt->getInputExpr(i);
+  llvm_unreachable("unknown asm statement kind!");
+}
+
+StringRef AsmStmt::getClobber(unsigned i) const {
+  if (const GCCAsmStmt *gccAsmStmt = dyn_cast<GCCAsmStmt>(this))
+    return gccAsmStmt->getClobber(i);
+  if (const MSAsmStmt *msAsmStmt = dyn_cast<MSAsmStmt>(this))
+    return msAsmStmt->getClobber(i);
+  llvm_unreachable("unknown asm statement kind!");
 }
 
 /// getNumPlusOperands - Return the number of output operands that have a "+"
@@ -341,22 +379,35 @@ unsigned AsmStmt::getNumPlusOperands() const {
   return Res;
 }
 
-Expr *AsmStmt::getInputExpr(unsigned i) {
+StringRef GCCAsmStmt::getClobber(unsigned i) const {
+  return getClobberStringLiteral(i)->getString();
+}
+
+Expr *GCCAsmStmt::getOutputExpr(unsigned i) {
+  return cast<Expr>(Exprs[i]);
+}
+
+/// getOutputConstraint - Return the constraint string for the specified
+/// output operand.  All output constraints are known to be non-empty (either
+/// '=' or '+').
+StringRef GCCAsmStmt::getOutputConstraint(unsigned i) const {
+  return getOutputConstraintLiteral(i)->getString();
+}
+
+Expr *GCCAsmStmt::getInputExpr(unsigned i) {
   return cast<Expr>(Exprs[i + NumOutputs]);
 }
-void AsmStmt::setInputExpr(unsigned i, Expr *E) {
+void GCCAsmStmt::setInputExpr(unsigned i, Expr *E) {
   Exprs[i + NumOutputs] = E;
 }
 
-
 /// getInputConstraint - Return the specified input constraint.  Unlike output
 /// constraints, these can be empty.
-StringRef AsmStmt::getInputConstraint(unsigned i) const {
+StringRef GCCAsmStmt::getInputConstraint(unsigned i) const {
   return getInputConstraintLiteral(i)->getString();
 }
 
-
-void AsmStmt::setOutputsAndInputsAndClobbers(ASTContext &C,
+void GCCAsmStmt::setOutputsAndInputsAndClobbers(ASTContext &C,
                                              IdentifierInfo **Names,
                                              StringLiteral **Constraints,
                                              Stmt **Exprs,
@@ -390,7 +441,7 @@ void AsmStmt::setOutputsAndInputsAndClobbers(ASTContext &C,
 /// getNamedOperand - Given a symbolic operand reference like %[foo],
 /// translate this into a numeric value needed to reference the same operand.
 /// This returns -1 if the operand name is invalid.
-int AsmStmt::getNamedOperand(StringRef SymbolicName) const {
+int GCCAsmStmt::getNamedOperand(StringRef SymbolicName) const {
   unsigned NumPlusOperands = 0;
 
   // Check if this is an output operand.
@@ -410,7 +461,7 @@ int AsmStmt::getNamedOperand(StringRef SymbolicName) const {
 /// AnalyzeAsmString - Analyze the asm string of the current asm, decomposing
 /// it into pieces.  If the asm string is erroneous, emit errors and return
 /// true, otherwise return false.
-unsigned AsmStmt::AnalyzeAsmString(SmallVectorImpl<AsmStringPiece>&Pieces,
+unsigned GCCAsmStmt::AnalyzeAsmString(SmallVectorImpl<AsmStringPiece>&Pieces,
                                    ASTContext &C, unsigned &DiagOffs) const {
   StringRef Str = getAsmString()->getString();
   const char *StrStart = Str.begin();
@@ -548,6 +599,44 @@ unsigned AsmStmt::AnalyzeAsmString(SmallVectorImpl<AsmStringPiece>&Pieces,
   }
 }
 
+/// Assemble final IR asm string (GCC-style).
+std::string GCCAsmStmt::generateAsmString(ASTContext &C) const {
+  // Analyze the asm string to decompose it into its pieces.  We know that Sema
+  // has already done this, so it is guaranteed to be successful.
+  SmallVector<GCCAsmStmt::AsmStringPiece, 4> Pieces;
+  unsigned DiagOffs;
+  AnalyzeAsmString(Pieces, C, DiagOffs);
+
+  std::string AsmString;
+  for (unsigned i = 0, e = Pieces.size(); i != e; ++i) {
+    if (Pieces[i].isString())
+      AsmString += Pieces[i].getString();
+    else if (Pieces[i].getModifier() == '\0')
+      AsmString += '$' + llvm::utostr(Pieces[i].getOperandNo());
+    else
+      AsmString += "${" + llvm::utostr(Pieces[i].getOperandNo()) + ':' +
+                   Pieces[i].getModifier() + '}';
+  }
+  return AsmString;
+}
+
+/// Assemble final IR asm string (MS-style).
+std::string MSAsmStmt::generateAsmString(ASTContext &C) const {
+  // FIXME: This needs to be translated into the IR string representation.
+  return AsmStr;
+}
+
+Expr *MSAsmStmt::getOutputExpr(unsigned i) {
+  return cast<Expr>(Exprs[i]);
+}
+
+Expr *MSAsmStmt::getInputExpr(unsigned i) {
+  return cast<Expr>(Exprs[i + NumOutputs]);
+}
+void MSAsmStmt::setInputExpr(unsigned i, Expr *E) {
+  Exprs[i + NumOutputs] = E;
+}
+
 QualType CXXCatchStmt::getCaughtType() const {
   if (ExceptionDecl)
     return ExceptionDecl->getType();
@@ -558,15 +647,14 @@ QualType CXXCatchStmt::getCaughtType() const {
 // Constructors
 //===----------------------------------------------------------------------===//
 
-AsmStmt::AsmStmt(ASTContext &C, SourceLocation asmloc, bool issimple,
-                 bool isvolatile, bool msasm,
-                 unsigned numoutputs, unsigned numinputs,
-                 IdentifierInfo **names, StringLiteral **constraints,
-                 Expr **exprs, StringLiteral *asmstr, unsigned numclobbers,
-                 StringLiteral **clobbers, SourceLocation rparenloc)
-  : Stmt(AsmStmtClass), AsmLoc(asmloc), RParenLoc(rparenloc), AsmStr(asmstr)
-  , IsSimple(issimple), IsVolatile(isvolatile), MSAsm(msasm)
-  , NumOutputs(numoutputs), NumInputs(numinputs), NumClobbers(numclobbers) {
+GCCAsmStmt::GCCAsmStmt(ASTContext &C, SourceLocation asmloc, bool issimple,
+                       bool isvolatile, unsigned numoutputs, unsigned numinputs,
+                       IdentifierInfo **names, StringLiteral **constraints,
+                       Expr **exprs, StringLiteral *asmstr,
+                       unsigned numclobbers, StringLiteral **clobbers,
+                       SourceLocation rparenloc)
+  : AsmStmt(GCCAsmStmtClass, asmloc, issimple, isvolatile, numoutputs,
+            numinputs, numclobbers), RParenLoc(rparenloc), AsmStr(asmstr) {
 
   unsigned NumExprs = NumOutputs + NumInputs;
 
@@ -585,26 +673,37 @@ AsmStmt::AsmStmt(ASTContext &C, SourceLocation asmloc, bool issimple,
 
 MSAsmStmt::MSAsmStmt(ASTContext &C, SourceLocation asmloc,
                      SourceLocation lbraceloc, bool issimple, bool isvolatile,
-                     ArrayRef<Token> asmtoks, ArrayRef<IdentifierInfo*> inputs,
-                     ArrayRef<IdentifierInfo*> outputs, StringRef asmstr,
-                     ArrayRef<StringRef> clobbers, SourceLocation endloc)
-  : Stmt(MSAsmStmtClass), AsmLoc(asmloc), LBraceLoc(lbraceloc), EndLoc(endloc),
-    AsmStr(asmstr.str()), IsSimple(issimple), IsVolatile(isvolatile),
-    NumAsmToks(asmtoks.size()), NumInputs(inputs.size()),
-    NumOutputs(outputs.size()), NumClobbers(clobbers.size()) {
+                     ArrayRef<Token> asmtoks, unsigned numoutputs,
+                     unsigned numinputs, ArrayRef<IdentifierInfo*> names,
+                     ArrayRef<StringRef> constraints, ArrayRef<Expr*> exprs,
+                     StringRef asmstr, ArrayRef<StringRef> clobbers,
+                     SourceLocation endloc)
+  : AsmStmt(MSAsmStmtClass, asmloc, issimple, isvolatile, numoutputs,
+            numinputs, clobbers.size()), LBraceLoc(lbraceloc),
+            EndLoc(endloc), AsmStr(asmstr.str()), NumAsmToks(asmtoks.size()) {
 
   unsigned NumExprs = NumOutputs + NumInputs;
 
   Names = new (C) IdentifierInfo*[NumExprs];
-  for (unsigned i = 0, e = NumOutputs; i != e; ++i)
-    Names[i] = outputs[i];
-  for (unsigned i = NumOutputs, e = NumExprs; i != e; ++i)
-    Names[i] = inputs[i];
+  for (unsigned i = 0, e = NumExprs; i != e; ++i)
+    Names[i] = names[i];
+
+  Exprs = new (C) Stmt*[NumExprs];
+  for (unsigned i = 0, e = NumExprs; i != e; ++i)
+    Exprs[i] = exprs[i];
 
   AsmToks = new (C) Token[NumAsmToks];
   for (unsigned i = 0, e = NumAsmToks; i != e; ++i)
     AsmToks[i] = asmtoks[i];
 
+  Constraints = new (C) StringRef[NumExprs];
+  for (unsigned i = 0, e = NumExprs; i != e; ++i) {
+    size_t size = constraints[i].size();
+    char *dest = new (C) char[size];
+    std::strncpy(dest, constraints[i].data(), size); 
+    Constraints[i] = StringRef(dest, size);
+  }
+
   Clobbers = new (C) StringRef[NumClobbers];
   for (unsigned i = 0, e = NumClobbers; i != e; ++i) {
     // FIXME: Avoid the allocation/copy if at all possible.
diff --git a/lib/AST/StmtDumper.cpp b/lib/AST/StmtDumper.cpp
index 962e352..fbc990f 100644
--- a/lib/AST/StmtDumper.cpp
+++ b/lib/AST/StmtDumper.cpp
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This file implements the Stmt::dump/Stmt::print methods, which dump out the
+// This file implements the Stmt::dump method, which dumps out the
 // AST in a form that exposes type details and other fields.
 //
 //===----------------------------------------------------------------------===//
@@ -30,6 +30,7 @@ namespace  {
     SourceManager *SM;
     raw_ostream &OS;
     unsigned IndentLevel;
+    bool IsFirstLine;
 
     /// MaxDepth - When doing a normal dump (not dumpAll) we only want to dump
     /// the first few levels of an AST.  This keeps track of how many ast levels
@@ -41,46 +42,64 @@ namespace  {
     const char *LastLocFilename;
     unsigned LastLocLine;
 
+    class IndentScope {
+      StmtDumper &Dumper;
+    public:
+      IndentScope(StmtDumper &Dumper) : Dumper(Dumper) {
+        Dumper.indent();
+      }
+      ~IndentScope() {
+        Dumper.unindent();
+      }
+    };
+
   public:
     StmtDumper(SourceManager *sm, raw_ostream &os, unsigned maxDepth)
-      : SM(sm), OS(os), IndentLevel(0-1), MaxDepth(maxDepth) {
+      : SM(sm), OS(os), IndentLevel(0), IsFirstLine(true), MaxDepth(maxDepth) {
       LastLocFilename = "";
       LastLocLine = ~0U;
     }
 
+    ~StmtDumper() {
+      OS << "\n";
+    }
+
     void DumpSubTree(Stmt *S) {
       // Prune the recursion if not using dump all.
       if (MaxDepth == 0) return;
 
-      ++IndentLevel;
-      if (S) {
-        if (DeclStmt* DS = dyn_cast<DeclStmt>(S))
-          VisitDeclStmt(DS);
-        else {
-          Visit(S);
-
-          // Print out children.
-          Stmt::child_range CI = S->children();
-          if (CI) {
-            while (CI) {
-              OS << '\n';
-              DumpSubTree(*CI++);
-            }
-          }
-        }
-        OS << ')';
-      } else {
-        Indent();
+      IndentScope Indent(*this);
+
+      if (!S) {
         OS << "<<<NULL>>>";
+        return;
+      }
+
+      if (DeclStmt* DS = dyn_cast<DeclStmt>(S)) {
+        VisitDeclStmt(DS);
+        return;
       }
-      --IndentLevel;
+
+      Visit(S);
+      for (Stmt::child_range CI = S->children(); CI; CI++)
+        DumpSubTree(*CI);
     }
 
     void DumpDeclarator(Decl *D);
 
-    void Indent() const {
-      for (int i = 0, e = IndentLevel; i < e; ++i)
-        OS << "  ";
+    void indent() {
+      if (IsFirstLine)
+        IsFirstLine = false;
+      else
+        OS << "\n";
+      OS.indent(IndentLevel * 2);
+      OS << "(";
+      IndentLevel++;
+    }
+
+    void unindent() {
+      OS << ")";
+      IndentLevel--;
     }
 
     void DumpType(QualType T) {
@@ -96,9 +115,8 @@ namespace  {
     }
     void DumpDeclRef(Decl *node);
     void DumpStmt(const Stmt *Node) {
-      Indent();
-      OS << "(" << Node->getStmtClassName()
-         << " " << (void*)Node;
+      OS << Node->getStmtClassName()
+         << " " << (const void*)Node;
       DumpSourceRange(Node);
     }
     void DumpValueKind(ExprValueKind K) {
@@ -262,7 +280,7 @@ void StmtDumper::DumpDeclarator(Decl *D) {
     // If this is a vardecl with an initializer, emit it.
     if (VarDecl *V = dyn_cast<VarDecl>(VD)) {
       if (V->getInit()) {
-        OS << " =\n";
+        OS << " =";
         DumpSubTree(V->getInit());
       }
     }
@@ -294,9 +312,9 @@ void StmtDumper::DumpDeclarator(Decl *D) {
   } else if (LabelDecl *LD = dyn_cast<LabelDecl>(D)) {
     OS << "label " << *LD;
   } else if (StaticAssertDecl *SAD = dyn_cast<StaticAssertDecl>(D)) {
-    OS << "\"static_assert(\n";
+    OS << "\"static_assert(";
     DumpSubTree(SAD->getAssertExpr());
-    OS << ",\n";
+    OS << ",";
     DumpSubTree(SAD->getMessage());
     OS << ");\"";
   } else {
@@ -306,17 +324,12 @@ void StmtDumper::DumpDeclarator(Decl *D) {
 
 void StmtDumper::VisitDeclStmt(DeclStmt *Node) {
   DumpStmt(Node);
-  OS << "\n";
   for (DeclStmt::decl_iterator DI = Node->decl_begin(), DE = Node->decl_end();
        DI != DE; ++DI) {
+    IndentScope Indent(*this);
     Decl* D = *DI;
-    ++IndentLevel;
-    Indent();
     OS << (void*) D << " ";
     DumpDeclarator(D);
-    if (DI+1 != DE)
-      OS << "\n";
-    --IndentLevel;
   }
 }
 
@@ -503,35 +516,29 @@ void StmtDumper::VisitBlockExpr(BlockExpr *Node) {
   BlockDecl *block = Node->getBlockDecl();
   OS << " decl=" << block;
 
-  IndentLevel++;
   if (block->capturesCXXThis()) {
-    OS << '\n'; Indent(); OS << "(capture this)";
+    IndentScope Indent(*this);
+    OS << "capture this";
   }
   for (BlockDecl::capture_iterator
          i = block->capture_begin(), e = block->capture_end(); i != e; ++i) {
-    OS << '\n';
-    Indent();
-    OS << "(capture ";
+    IndentScope Indent(*this);
+    OS << "capture ";
     if (i->isByRef()) OS << "byref ";
     if (i->isNested()) OS << "nested ";
     if (i->getVariable())
       DumpDeclRef(i->getVariable());
     if (i->hasCopyExpr()) DumpSubTree(i->getCopyExpr());
-    OS << ")";
   }
-  IndentLevel--;
 
-  OS << '\n';
   DumpSubTree(block->getBody());
 }
 
 void StmtDumper::VisitOpaqueValueExpr(OpaqueValueExpr *Node) {
   DumpExpr(Node);
 
-  if (Expr *Source = Node->getSourceExpr()) {
-    OS << '\n';
+  if (Expr *Source = Node->getSourceExpr())
     DumpSubTree(Source);
-  }
 }
 
 // GNU extensions.
@@ -589,15 +596,11 @@ void StmtDumper::VisitCXXBindTemporaryExpr(CXXBindTemporaryExpr *Node) {
 
 void StmtDumper::VisitExprWithCleanups(ExprWithCleanups *Node) {
   DumpExpr(Node);
-  ++IndentLevel;
   for (unsigned i = 0, e = Node->getNumObjects(); i != e; ++i) {
-    OS << "\n";
-    Indent();
-    OS << "(cleanup ";
+    IndentScope Indent(*this);
+    OS << "cleanup ";
     DumpDeclRef(Node->getObject(i));
-    OS << ")";
   }
-  --IndentLevel;
 }
 
 void StmtDumper::DumpCXXTemporary(CXXTemporary *Temporary) {
@@ -734,7 +737,6 @@ void Stmt::dump(SourceManager &SM) const {
 void Stmt::dump(raw_ostream &OS, SourceManager &SM) const {
   StmtDumper P(&SM, OS, 4);
   P.DumpSubTree(const_cast<Stmt*>(this));
-  OS << "\n";
 }
 
 /// dump - This does a local dump of the specified AST fragment.  It dumps the
@@ -743,19 +745,16 @@ void Stmt::dump(raw_ostream &OS, SourceManager &SM) const {
 void Stmt::dump() const {
   StmtDumper P(0, llvm::errs(), 4);
   P.DumpSubTree(const_cast<Stmt*>(this));
-  llvm::errs() << "\n";
 }
 
 /// dumpAll - This does a dump of the specified AST fragment and all subtrees.
 void Stmt::dumpAll(SourceManager &SM) const {
   StmtDumper P(&SM, llvm::errs(), ~0U);
   P.DumpSubTree(const_cast<Stmt*>(this));
-  llvm::errs() << "\n";
 }
 
 /// dumpAll - This does a dump of the specified AST fragment and all subtrees.
 void Stmt::dumpAll() const {
   StmtDumper P(0, llvm::errs(), ~0U);
   P.DumpSubTree(const_cast<Stmt*>(this));
-  llvm::errs() << "\n";
 }
diff --git a/lib/AST/StmtPrinter.cpp b/lib/AST/StmtPrinter.cpp
index c0960ce..57eb1a9 100644
--- a/lib/AST/StmtPrinter.cpp
+++ b/lib/AST/StmtPrinter.cpp
@@ -61,7 +61,7 @@ namespace  {
 
     void PrintRawCompoundStmt(CompoundStmt *S);
     void PrintRawDecl(Decl *D);
-    void PrintRawDeclStmt(DeclStmt *S);
+    void PrintRawDeclStmt(const DeclStmt *S);
     void PrintRawIfStmt(IfStmt *If);
     void PrintRawCXXCatchStmt(CXXCatchStmt *Catch);
     void PrintCallArgs(CallExpr *E);
@@ -121,8 +121,8 @@ void StmtPrinter::PrintRawDecl(Decl *D) {
   D->print(OS, Policy, IndentLevel);
 }
 
-void StmtPrinter::PrintRawDeclStmt(DeclStmt *S) {
-  DeclStmt::decl_iterator Begin = S->decl_begin(), End = S->decl_end();
+void StmtPrinter::PrintRawDeclStmt(const DeclStmt *S) {
+  DeclStmt::const_decl_iterator Begin = S->decl_begin(), End = S->decl_end();
   SmallVector<Decl*, 2> Decls;
   for ( ; Begin != End; ++Begin)
     Decls.push_back(*Begin);
@@ -187,7 +187,10 @@ void StmtPrinter::VisitAttributedStmt(AttributedStmt *Node) {
 
 void StmtPrinter::PrintRawIfStmt(IfStmt *If) {
   OS << "if (";
-  PrintExpr(If->getCond());
+  if (const DeclStmt *DS = If->getConditionVariableDeclStmt())
+    PrintRawDeclStmt(DS);
+  else
+    PrintExpr(If->getCond());
   OS << ')';
 
   if (CompoundStmt *CS = dyn_cast<CompoundStmt>(If->getThen())) {
@@ -224,7 +227,10 @@ void StmtPrinter::VisitIfStmt(IfStmt *If) {
 
 void StmtPrinter::VisitSwitchStmt(SwitchStmt *Node) {
   Indent() << "switch (";
-  PrintExpr(Node->getCond());
+  if (const DeclStmt *DS = Node->getConditionVariableDeclStmt())
+    PrintRawDeclStmt(DS);
+  else
+    PrintExpr(Node->getCond());
   OS << ")";
 
   // Pretty print compoundstmt bodies (very common).
@@ -240,7 +246,10 @@ void StmtPrinter::VisitSwitchStmt(SwitchStmt *Node) {
 
 void StmtPrinter::VisitWhileStmt(WhileStmt *Node) {
   Indent() << "while (";
-  PrintExpr(Node->getCond());
+  if (const DeclStmt *DS = Node->getConditionVariableDeclStmt())
+    PrintRawDeclStmt(DS);
+  else
+    PrintExpr(Node->getCond());
   OS << ")\n";
   PrintStmt(Node->getBody());
 }
@@ -366,7 +375,7 @@ void StmtPrinter::VisitReturnStmt(ReturnStmt *Node) {
 }
 
 
-void StmtPrinter::VisitAsmStmt(AsmStmt *Node) {
+void StmtPrinter::VisitGCCAsmStmt(GCCAsmStmt *Node) {
   Indent() << "asm ";
 
   if (Node->isVolatile())
@@ -422,7 +431,7 @@ void StmtPrinter::VisitAsmStmt(AsmStmt *Node) {
     if (i != 0)
       OS << ", ";
 
-    VisitStringLiteral(Node->getClobber(i));
+    VisitStringLiteral(Node->getClobberStringLiteral(i));
   }
 
   OS << ");\n";
@@ -734,10 +743,30 @@ void StmtPrinter::VisitIntegerLiteral(IntegerLiteral *Node) {
   case BuiltinType::UInt128:   OS << "Ui128"; break;
   }
 }
-void StmtPrinter::VisitFloatingLiteral(FloatingLiteral *Node) {
+
+static void PrintFloatingLiteral(raw_ostream &OS, FloatingLiteral *Node,
+                                 bool PrintSuffix) {
   SmallString<16> Str;
   Node->getValue().toString(Str);
   OS << Str;
+  if (Str.find_first_not_of("-0123456789") == StringRef::npos)
+    OS << '.'; // Trailing dot in order to separate from ints.
+
+  if (!PrintSuffix)
+    return;
+
+  // Emit suffixes.  Float literals are always a builtin float type.
+  switch (Node->getType()->getAs<BuiltinType>()->getKind()) {
+  default: llvm_unreachable("Unexpected type for float literal!");
+  case BuiltinType::Half:       break; // FIXME: suffix?
+  case BuiltinType::Double:     break; // no suffix.
+  case BuiltinType::Float:      OS << 'F'; break;
+  case BuiltinType::LongDouble: OS << 'L'; break;
+  }
+}
+
+void StmtPrinter::VisitFloatingLiteral(FloatingLiteral *Node) {
+  PrintFloatingLiteral(OS, Node, /*PrintSuffix=*/true);
 }
 
 void StmtPrinter::VisitImaginaryLiteral(ImaginaryLiteral *Node) {
@@ -907,7 +936,7 @@ void StmtPrinter::VisitExtVectorElementExpr(ExtVectorElementExpr *Node) {
   OS << Node->getAccessor().getName();
 }
 void StmtPrinter::VisitCStyleCastExpr(CStyleCastExpr *Node) {
-  OS << "(" << Node->getType().getAsString(Policy) << ")";
+  OS << "(" << Node->getTypeAsWritten().getAsString(Policy) << ")";
   PrintExpr(Node->getSubExpr());
 }
 void StmtPrinter::VisitCompoundLiteralExpr(CompoundLiteralExpr *Node) {
@@ -1110,6 +1139,8 @@ void StmtPrinter::VisitCXXOperatorCallExpr(CXXOperatorCallExpr *Node) {
       PrintExpr(Node->getArg(0));
       OS << ' ' << OpStrings[Kind];
     }
+  } else if (Kind == OO_Arrow) {
+    PrintExpr(Node->getArg(0));
   } else if (Kind == OO_Call) {
     PrintExpr(Node->getArg(0));
     OS << '(';
@@ -1217,7 +1248,12 @@ void StmtPrinter::VisitUserDefinedLiteral(UserDefinedLiteral *Node) {
     OS << Int->getValue().toString(10, /*isSigned*/false);
     break;
   }
-  case UserDefinedLiteral::LOK_Floating:
+  case UserDefinedLiteral::LOK_Floating: {
+    // Print floating literal without suffix.
+    FloatingLiteral *Float = cast<FloatingLiteral>(Node->getCookedLiteral());
+    PrintFloatingLiteral(OS, Float, /*PrintSuffix=*/false);
+    break;
+  }
   case UserDefinedLiteral::LOK_String:
   case UserDefinedLiteral::LOK_Character:
     PrintExpr(Node->getCookedLiteral());
@@ -1379,10 +1415,12 @@ void StmtPrinter::VisitCXXNewExpr(CXXNewExpr *E) {
     OS << "::";
   OS << "new ";
   unsigned NumPlace = E->getNumPlacementArgs();
-  if (NumPlace > 0) {
+  if (NumPlace > 0 && !isa<CXXDefaultArgExpr>(E->getPlacementArg(0))) {
     OS << "(";
     PrintExpr(E->getPlacementArg(0));
     for (unsigned i = 1; i < NumPlace; ++i) {
+      if (isa<CXXDefaultArgExpr>(E->getPlacementArg(i)))
+        break;
       OS << ", ";
       PrintExpr(E->getPlacementArg(i));
     }
@@ -1429,6 +1467,7 @@ void StmtPrinter::VisitCXXPseudoDestructorExpr(CXXPseudoDestructorExpr *E) {
     OS << '.';
   if (E->getQualifier())
     E->getQualifier()->print(OS, Policy);
+  OS << "~";
 
   std::string TypeS;
   if (IdentifierInfo *II = E->getDestroyedTypeIdentifier())
@@ -1531,6 +1570,7 @@ static const char *getTypeTraitName(UnaryTypeTrait UTT) {
   case UTT_IsFunction:              return "__is_function";
   case UTT_IsFundamental:           return "__is_fundamental";
   case UTT_IsIntegral:              return "__is_integral";
+  case UTT_IsInterfaceClass:        return "__is_interface_class";
   case UTT_IsLiteral:               return "__is_literal";
   case UTT_IsLvalueReference:       return "__is_lvalue_reference";
   case UTT_IsMemberFunctionPointer: return "__is_member_function_pointer";
@@ -1647,6 +1687,10 @@ void StmtPrinter::VisitSubstNonTypeTemplateParmExpr(
   Visit(Node->getReplacement());
 }
 
+void StmtPrinter::VisitFunctionParmPackExpr(FunctionParmPackExpr *E) {
+  OS << *E->getParameterPack();
+}
+
 void StmtPrinter::VisitMaterializeTemporaryExpr(MaterializeTemporaryExpr *Node){
   PrintExpr(Node->GetTemporaryExpr());
 }
diff --git a/lib/AST/StmtProfile.cpp b/lib/AST/StmtProfile.cpp
index 2168b64..bfd3132 100644
--- a/lib/AST/StmtProfile.cpp
+++ b/lib/AST/StmtProfile.cpp
@@ -158,7 +158,7 @@ void StmtProfiler::VisitReturnStmt(const ReturnStmt *S) {
   VisitStmt(S);
 }
 
-void StmtProfiler::VisitAsmStmt(const AsmStmt *S) {
+void StmtProfiler::VisitGCCAsmStmt(const GCCAsmStmt *S) {
   VisitStmt(S);
   ID.AddBoolean(S->isVolatile());
   ID.AddBoolean(S->isSimple());
@@ -175,7 +175,7 @@ void StmtProfiler::VisitAsmStmt(const AsmStmt *S) {
   }
   ID.AddInteger(S->getNumClobbers());
   for (unsigned I = 0, N = S->getNumClobbers(); I != N; ++I)
-    VisitStringLiteral(S->getClobber(I));
+    VisitStringLiteral(S->getClobberStringLiteral(I));
 }
 
 void StmtProfiler::VisitMSAsmStmt(const MSAsmStmt *S) {
@@ -973,6 +973,14 @@ void StmtProfiler::VisitSubstNonTypeTemplateParmExpr(
   Visit(E->getReplacement());
 }
 
+void StmtProfiler::VisitFunctionParmPackExpr(const FunctionParmPackExpr *S) {
+  VisitExpr(S);
+  VisitDecl(S->getParameterPack());
+  ID.AddInteger(S->getNumExpansions());
+  for (FunctionParmPackExpr::iterator I = S->begin(), E = S->end(); I != E; ++I)
+    VisitDecl(*I);
+}
+
 void StmtProfiler::VisitMaterializeTemporaryExpr(
                                            const MaterializeTemporaryExpr *S) {
   VisitExpr(S);
@@ -1165,6 +1173,10 @@ void StmtProfiler::VisitTemplateArgument(const TemplateArgument &Arg) {
     VisitDecl(Arg.getAsDecl());
     break;
 
+  case TemplateArgument::NullPtr:
+    VisitType(Arg.getNullPtrType());
+    break;
+
   case TemplateArgument::Integral:
     Arg.getAsIntegral().Profile(ID);
     VisitType(Arg.getIntegralType());
diff --git a/lib/AST/TemplateBase.cpp b/lib/AST/TemplateBase.cpp
index 95ff4ed..e9ee385 100644
--- a/lib/AST/TemplateBase.cpp
+++ b/lib/AST/TemplateBase.cpp
@@ -77,7 +77,7 @@ TemplateArgument TemplateArgument::CreatePackCopy(ASTContext &Context,
                                                   const TemplateArgument *Args,
                                                   unsigned NumArgs) {
   if (NumArgs == 0)
-    return TemplateArgument(0, 0);
+    return getEmptyPack();
   
   TemplateArgument *Storage = new (Context) TemplateArgument [NumArgs];
   std::copy(Args, Args + NumArgs, Storage);
@@ -99,12 +99,11 @@ bool TemplateArgument::isDependent() const {
     return true;
 
   case Declaration:
-    if (Decl *D = getAsDecl()) {
-      if (DeclContext *DC = dyn_cast<DeclContext>(D))
-        return DC->isDependentContext();
-      return D->getDeclContext()->isDependentContext();
-    }
-      
+    if (DeclContext *DC = dyn_cast<DeclContext>(getAsDecl()))
+      return DC->isDependentContext();
+    return getAsDecl()->getDeclContext()->isDependentContext();
+
+  case NullPtr:
     return false;
 
   case Integral:
@@ -141,11 +140,11 @@ bool TemplateArgument::isInstantiationDependent() const {
     return true;
     
   case Declaration:
-    if (Decl *D = getAsDecl()) {
-      if (DeclContext *DC = dyn_cast<DeclContext>(D))
-        return DC->isDependentContext();
-      return D->getDeclContext()->isDependentContext();
-    }
+    if (DeclContext *DC = dyn_cast<DeclContext>(getAsDecl()))
+      return DC->isDependentContext();
+    return getAsDecl()->getDeclContext()->isDependentContext();
+
+  case NullPtr:
     return false;
       
   case Integral:
@@ -174,6 +173,7 @@ bool TemplateArgument::isPackExpansion() const {
   case Integral:
   case Pack:    
   case Template:
+  case NullPtr:
     return false;
       
   case TemplateExpansion:
@@ -195,6 +195,7 @@ bool TemplateArgument::containsUnexpandedParameterPack() const {
   case Declaration:
   case Integral:
   case TemplateExpansion:
+  case NullPtr:
     break;
 
   case Type:
@@ -286,12 +287,16 @@ bool TemplateArgument::structurallyEquals(const TemplateArgument &Other) const {
   switch (getKind()) {
   case Null:
   case Type:
-  case Declaration:
   case Expression:      
   case Template:
   case TemplateExpansion:
+  case NullPtr:
     return TypeOrValue == Other.TypeOrValue;
 
+  case Declaration:
+    return getAsDecl() == Other.getAsDecl() && 
+           isDeclForReferenceParam() && Other.isDeclForReferenceParam();
+
   case Integral:
     return getIntegralType() == Other.getIntegralType() &&
            getAsIntegral() == Other.getAsIntegral();
@@ -319,12 +324,13 @@ TemplateArgument TemplateArgument::getPackExpansionPattern() const {
     
   case TemplateExpansion:
     return TemplateArgument(getAsTemplateOrTemplatePattern());
-    
+
   case Declaration:
   case Integral:
   case Pack:
   case Null:
   case Template:
+  case NullPtr:
     return TemplateArgument();
   }
 
@@ -348,18 +354,20 @@ void TemplateArgument::print(const PrintingPolicy &Policy,
   }
     
   case Declaration: {
-    if (NamedDecl *ND = dyn_cast_or_null<NamedDecl>(getAsDecl())) {
-      if (ND->getDeclName()) {
-        Out << *ND;
-      } else {
-        Out << "<anonymous>";
-      }
+    NamedDecl *ND = cast<NamedDecl>(getAsDecl());
+    if (ND->getDeclName()) {
+      // FIXME: distinguish between pointer and reference args?
+      Out << *ND;
     } else {
-      Out << "nullptr";
+      Out << "<anonymous>";
     }
     break;
   }
-    
+
+  case NullPtr:
+    Out << "nullptr";
+    break;
+
   case Template:
     getAsTemplate().print(Out, Policy);
     break;
@@ -411,6 +419,9 @@ SourceRange TemplateArgumentLoc::getSourceRange() const {
   case TemplateArgument::Declaration:
     return getSourceDeclExpression()->getSourceRange();
 
+  case TemplateArgument::NullPtr:
+    return getSourceNullPtrExpression()->getSourceRange();
+
   case TemplateArgument::Type:
     if (TypeSourceInfo *TSI = getTypeSourceInfo())
       return TSI->getTypeLoc().getSourceRange();
@@ -430,6 +441,8 @@ SourceRange TemplateArgumentLoc::getSourceRange() const {
     return SourceRange(getTemplateNameLoc(), getTemplateEllipsisLoc());
 
   case TemplateArgument::Integral:
+    return getSourceIntegralExpression()->getSourceRange();
+
   case TemplateArgument::Pack:
   case TemplateArgument::Null:
     return SourceRange();
@@ -490,6 +503,7 @@ TemplateArgumentLoc::getPackExpansionPattern(SourceLocation &Ellipsis,
                                getTemplateNameLoc());
     
   case TemplateArgument::Declaration:
+  case TemplateArgument::NullPtr:
   case TemplateArgument::Template:
   case TemplateArgument::Integral:
   case TemplateArgument::Pack:
@@ -512,8 +526,9 @@ const DiagnosticBuilder &clang::operator<<(const DiagnosticBuilder &DB,
     return DB << Arg.getAsType();
       
   case TemplateArgument::Declaration:
-    if (Decl *D = Arg.getAsDecl())
-      return DB << D;
+    return DB << Arg.getAsDecl();
+
+  case TemplateArgument::NullPtr:
     return DB << "nullptr";
       
   case TemplateArgument::Integral:
diff --git a/lib/AST/Type.cpp b/lib/AST/Type.cpp
index abefae4..580ec50 100644
--- a/lib/AST/Type.cpp
+++ b/lib/AST/Type.cpp
@@ -288,18 +288,17 @@ QualType QualType::IgnoreParens(QualType T) {
   return T;
 }
 
-/// \brief This will check for a TypedefType by removing any existing sugar
-/// until it reaches a TypedefType or a non-sugared type.
-template <> const TypedefType *Type::getAs() const {
-  const Type *Cur = this;
-
+/// \brief This will check for a T (which should be a Type which can act as
+/// sugar, such as a TypedefType) by removing any existing sugar until it
+/// reaches a T or a non-sugared type.
+template<typename T> static const T *getAsSugar(const Type *Cur) {
   while (true) {
-    if (const TypedefType *TDT = dyn_cast<TypedefType>(Cur))
-      return TDT;
+    if (const T *Sugar = dyn_cast<T>(Cur))
+      return Sugar;
     switch (Cur->getTypeClass()) {
 #define ABSTRACT_TYPE(Class, Parent)
 #define TYPE(Class, Parent) \
-    case Class: { \
+    case Type::Class: { \
       const Class##Type *Ty = cast<Class##Type>(Cur); \
       if (!Ty->isSugared()) return 0; \
       Cur = Ty->desugar().getTypePtr(); \
@@ -310,6 +309,14 @@ template <> const TypedefType *Type::getAs() const {
   }
 }
 
+template <> const TypedefType *Type::getAs() const {
+  return getAsSugar<TypedefType>(this);
+}
+
+template <> const TemplateSpecializationType *Type::getAs() const {
+  return getAsSugar<TemplateSpecializationType>(this);
+}
+
 /// getUnqualifiedDesugaredType - Pull any qualifiers and syntactic
 /// sugar off the given type.  This should produce an object of the
 /// same dynamic type as the canonical type.
@@ -357,9 +364,15 @@ bool Type::isStructureType() const {
     return RT->getDecl()->isStruct();
   return false;
 }
+bool Type::isInterfaceType() const {
+  if (const RecordType *RT = getAs<RecordType>())
+    return RT->getDecl()->isInterface();
+  return false;
+}
 bool Type::isStructureOrClassType() const {
   if (const RecordType *RT = getAs<RecordType>())
-    return RT->getDecl()->isStruct() || RT->getDecl()->isClass();
+    return RT->getDecl()->isStruct() || RT->getDecl()->isClass() ||
+      RT->getDecl()->isInterface();
   return false;
 }
 bool Type::isVoidPointerType() const {
@@ -499,10 +512,18 @@ const ObjCObjectPointerType *Type::getAsObjCInterfacePointerType() const {
   return 0;
 }
 
-const CXXRecordDecl *Type::getCXXRecordDeclForPointerType() const {
+const CXXRecordDecl *Type::getPointeeCXXRecordDecl() const {
+  QualType PointeeType;
   if (const PointerType *PT = getAs<PointerType>())
-    if (const RecordType *RT = PT->getPointeeType()->getAs<RecordType>())
-      return dyn_cast<CXXRecordDecl>(RT->getDecl());
+    PointeeType = PT->getPointeeType();
+  else if (const ReferenceType *RT = getAs<ReferenceType>())
+    PointeeType = RT->getPointeeType();
+  else
+    return 0;
+
+  if (const RecordType *RT = PointeeType->getAs<RecordType>())
+    return dyn_cast<CXXRecordDecl>(RT->getDecl());
+
   return 0;
 }
 
@@ -1205,8 +1226,6 @@ bool QualType::isCXX11PODType(ASTContext &Context) const {
       return false;
 
     case Qualifiers::OCL_None:
-      if (ty->isObjCLifetimeType())
-        return false;
       break;
     }        
   }
@@ -1317,6 +1336,7 @@ TypeWithKeyword::getKeywordForTypeSpec(unsigned TypeSpec) {
   case TST_typename: return ETK_Typename;
   case TST_class: return ETK_Class;
   case TST_struct: return ETK_Struct;
+  case TST_interface: return ETK_Interface;
   case TST_union: return ETK_Union;
   case TST_enum: return ETK_Enum;
   }
@@ -1327,6 +1347,7 @@ TypeWithKeyword::getTagTypeKindForTypeSpec(unsigned TypeSpec) {
   switch(TypeSpec) {
   case TST_class: return TTK_Class;
   case TST_struct: return TTK_Struct;
+  case TST_interface: return TTK_Interface;
   case TST_union: return TTK_Union;
   case TST_enum: return TTK_Enum;
   }
@@ -1339,6 +1360,7 @@ TypeWithKeyword::getKeywordForTagTypeKind(TagTypeKind Kind) {
   switch (Kind) {
   case TTK_Class: return ETK_Class;
   case TTK_Struct: return ETK_Struct;
+  case TTK_Interface: return ETK_Interface;
   case TTK_Union: return ETK_Union;
   case TTK_Enum: return ETK_Enum;
   }
@@ -1350,6 +1372,7 @@ TypeWithKeyword::getTagTypeKindForKeyword(ElaboratedTypeKeyword Keyword) {
   switch (Keyword) {
   case ETK_Class: return TTK_Class;
   case ETK_Struct: return TTK_Struct;
+  case ETK_Interface: return TTK_Interface;
   case ETK_Union: return TTK_Union;
   case ETK_Enum: return TTK_Enum;
   case ETK_None: // Fall through.
@@ -1367,6 +1390,7 @@ TypeWithKeyword::KeywordIsTagTypeKind(ElaboratedTypeKeyword Keyword) {
     return false;
   case ETK_Class:
   case ETK_Struct:
+  case ETK_Interface:
   case ETK_Union:
   case ETK_Enum:
     return true;
@@ -1381,6 +1405,7 @@ TypeWithKeyword::getKeywordName(ElaboratedTypeKeyword Keyword) {
   case ETK_Typename: return "typename";
   case ETK_Class:  return "class";
   case ETK_Struct: return "struct";
+  case ETK_Interface: return "__interface";
   case ETK_Union:  return "union";
   case ETK_Enum:   return "enum";
   }
@@ -1480,6 +1505,7 @@ StringRef BuiltinType::getName(const PrintingPolicy &Policy) const {
   case Dependent:         return "<dependent type>";
   case UnknownAny:        return "<unknown type>";
   case ARCUnbridgedCast:  return "<ARC unbridged cast type>";
+  case BuiltinFn:         return "<builtin fn type>";
   case ObjCId:            return "id";
   case ObjCClass:         return "Class";
   case ObjCSel:           return "SEL";
@@ -1516,6 +1542,7 @@ StringRef FunctionType::getNameForCallConv(CallingConv CC) {
   case CC_X86Pascal: return "pascal";
   case CC_AAPCS: return "aapcs";
   case CC_AAPCS_VFP: return "aapcs-vfp";
+  case CC_PnaclCall: return "pnaclcall";
   }
 
   llvm_unreachable("Invalid calling convention.");
diff --git a/lib/AST/TypeLoc.cpp b/lib/AST/TypeLoc.cpp
index c7bb7da..58c4cbd 100644
--- a/lib/AST/TypeLoc.cpp
+++ b/lib/AST/TypeLoc.cpp
@@ -98,23 +98,38 @@ void TypeLoc::initializeImpl(ASTContext &Context, TypeLoc TL,
 
 SourceLocation TypeLoc::getBeginLoc() const {
   TypeLoc Cur = *this;
+  TypeLoc LeftMost = Cur;
   while (true) {
     switch (Cur.getTypeLocClass()) {
-    // FIXME: Currently QualifiedTypeLoc does not have a source range
-    // case Qualified:
     case Elaborated:
-    case DependentName:
-    case DependentTemplateSpecialization:
+      LeftMost = Cur;
       break;
+    case FunctionProto:
+      if (cast<FunctionProtoTypeLoc>(&Cur)->getTypePtr()->hasTrailingReturn()) {
+        LeftMost = Cur;
+        break;
+      }
+      /* Fall through */
+    case FunctionNoProto:
+    case ConstantArray:
+    case DependentSizedArray:
+    case IncompleteArray:
+    case VariableArray:
+      // FIXME: Currently QualifiedTypeLoc does not have a source range
+    case Qualified:
+      Cur = Cur.getNextTypeLoc();
+      continue;
     default:
-      TypeLoc Next = Cur.getNextTypeLoc();
-      if (Next.isNull()) break;
-      Cur = Next;
+      if (!Cur.getLocalSourceRange().getBegin().isInvalid())
+        LeftMost = Cur;
+      Cur = Cur.getNextTypeLoc();
+      if (Cur.isNull())
+        break;
       continue;
-    }
+    } // switch
     break;
-  }
-  return Cur.getLocalSourceRange().getBegin();
+  } // while
+  return LeftMost.getLocalSourceRange().getBegin();
 }
 
 SourceLocation TypeLoc::getEndLoc() const {
@@ -131,10 +146,15 @@ SourceLocation TypeLoc::getEndLoc() const {
     case DependentSizedArray:
     case IncompleteArray:
     case VariableArray:
-    case FunctionProto:
     case FunctionNoProto:
       Last = Cur;
       break;
+    case FunctionProto:
+      if (cast<FunctionProtoTypeLoc>(&Cur)->getTypePtr()->hasTrailingReturn())
+        Last = TypeLoc();
+      else
+        Last = Cur;
+      break;
     case Pointer:
     case BlockPointer:
     case MemberPointer:
@@ -241,6 +261,7 @@ TypeSpecifierType BuiltinTypeLoc::getWrittenTypeSpec() const {
   case BuiltinType::ObjCId:
   case BuiltinType::ObjCClass:
   case BuiltinType::ObjCSel:
+  case BuiltinType::BuiltinFn:
     return TST_unspecified;
   }
 
@@ -300,7 +321,9 @@ void TemplateSpecializationTypeLoc::initializeArgLocs(ASTContext &Context,
     case TemplateArgument::Null: 
     case TemplateArgument::Declaration:
     case TemplateArgument::Integral:
-    case TemplateArgument::Pack:
+    case TemplateArgument::NullPtr:
+      llvm_unreachable("Impossible TemplateArgument");
+
     case TemplateArgument::Expression:
       ArgInfos[i] = TemplateArgumentLocInfo(Args[i].getAsExpr());
       break;
@@ -310,7 +333,7 @@ void TemplateSpecializationTypeLoc::initializeArgLocs(ASTContext &Context,
                           Context.getTrivialTypeSourceInfo(Args[i].getAsType(), 
                                                            Loc));
       break;
-        
+
     case TemplateArgument::Template:
     case TemplateArgument::TemplateExpansion: {
       NestedNameSpecifierLocBuilder Builder;
@@ -327,7 +350,11 @@ void TemplateSpecializationTypeLoc::initializeArgLocs(ASTContext &Context,
                                             ? SourceLocation()
                                             : Loc);
       break;
-    }        
+    }
+
+    case TemplateArgument::Pack:
+      ArgInfos[i] = TemplateArgumentLocInfo();
+      break;
     }
   }
 }
diff --git a/lib/AST/TypePrinter.cpp b/lib/AST/TypePrinter.cpp
index c42117c..90b2ca9 100644
--- a/lib/AST/TypePrinter.cpp
+++ b/lib/AST/TypePrinter.cpp
@@ -141,9 +141,6 @@ void TypePrinter::print(const Type *T, Qualifiers Quals, raw_ostream &OS,
     OS << "NULL TYPE";
     return;
   }
-  
-  if (Policy.SuppressSpecifiers && T->isSpecifierType())
-    return;
 
   SaveAndRestore<bool> PHVal(HasEmptyPlaceHolder, PlaceHolder.empty());
 
@@ -556,7 +553,8 @@ void TypePrinter::printExtVectorAfter(const ExtVectorType *T, raw_ostream &OS) {
 
 void 
 FunctionProtoType::printExceptionSpecification(raw_ostream &OS, 
-                                               PrintingPolicy Policy) const {
+                                               const PrintingPolicy &Policy)
+                                                                         const {
   
   if (hasDynamicExceptionSpec()) {
     OS << " throw(";
@@ -646,6 +644,9 @@ void TypePrinter::printFunctionProtoAfter(const FunctionProtoType *T,
   case CC_AAPCS_VFP:
     OS << " __attribute__((pcs(\"aapcs-vfp\")))";
     break;
+  case CC_PnaclCall:
+    OS << " __attribute__((pnaclcall))";
+    break;
   }
   if (Info.getNoReturn())
     OS << " __attribute__((noreturn))";
@@ -798,6 +799,7 @@ void TypePrinter::printAtomicAfter(const AtomicType *T, raw_ostream &OS) { }
 /// Appends the given scope to the end of a string.
 void TypePrinter::AppendScope(DeclContext *DC, raw_ostream &OS) {
   if (DC->isTranslationUnit()) return;
+  if (DC->isFunctionOrMethod()) return;
   AppendScope(DC->getParent(), OS);
 
   if (NamespaceDecl *NS = dyn_cast<NamespaceDecl>(DC)) {
@@ -1165,6 +1167,7 @@ void TypePrinter::printAttributedAfter(const AttributedType *T,
    OS << ')';
    break;
   }
+  case AttributedType::attr_pnaclcall: OS << "pnaclcall"; break;
   }
   OS << "))";
 }
@@ -1343,7 +1346,8 @@ PrintTemplateArgumentList(raw_ostream &OS,
 
 void 
 FunctionProtoType::printExceptionSpecification(std::string &S, 
-                                               PrintingPolicy Policy) const {
+                                               const PrintingPolicy &Policy)
+                                                                         const {
   
   if (hasDynamicExceptionSpec()) {
     S += " throw(";
diff --git a/lib/AST/VTableBuilder.cpp b/lib/AST/VTableBuilder.cpp
index 104530f..33dad40 100644
--- a/lib/AST/VTableBuilder.cpp
+++ b/lib/AST/VTableBuilder.cpp
@@ -1891,6 +1891,9 @@ void VTableBuilder::dumpLayout(raw_ostream& Out) {
       if (MD->isPure())
         Out << " [pure]";
 
+      if (MD->isDeleted())
+        Out << " [deleted]";
+
       ThunkInfo Thunk = VTableThunks.lookup(I);
       if (!Thunk.isEmpty()) {
         // If this function pointer has a return adjustment, dump it.
diff --git a/lib/ASTMatchers/ASTMatchFinder.cpp b/lib/ASTMatchers/ASTMatchFinder.cpp
index 085049d..8ecb26e 100644
--- a/lib/ASTMatchers/ASTMatchFinder.cpp
+++ b/lib/ASTMatchers/ASTMatchFinder.cpp
@@ -27,10 +27,75 @@ namespace ast_matchers {
 namespace internal {
 namespace {
 
+typedef MatchFinder::MatchCallback MatchCallback;
+
+/// \brief A \c RecursiveASTVisitor that builds a map from nodes to their
+/// parents as defined by the \c RecursiveASTVisitor.
+///
+/// Note that the relationship described here is purely in terms of AST
+/// traversal - there are other relationships (for example declaration context)
+/// in the AST that are better modeled by special matchers.
+///
+/// FIXME: Currently only builds up the map using \c Stmt and \c Decl nodes.
+class ParentMapASTVisitor : public RecursiveASTVisitor<ParentMapASTVisitor> {
+public:
+  /// \brief Maps from a node to its parent.
+  typedef llvm::DenseMap<const void*, ast_type_traits::DynTypedNode> ParentMap;
+
+  /// \brief Builds and returns the translation unit's parent map.
+  ///
+  ///  The caller takes ownership of the returned \c ParentMap.
+  static ParentMap *buildMap(TranslationUnitDecl &TU) {
+    ParentMapASTVisitor Visitor(new ParentMap);
+    Visitor.TraverseDecl(&TU);
+    return Visitor.Parents;
+  }
+
+private:
+  typedef RecursiveASTVisitor<ParentMapASTVisitor> VisitorBase;
+
+  ParentMapASTVisitor(ParentMap *Parents) : Parents(Parents) {}
+
+  bool shouldVisitTemplateInstantiations() const { return true; }
+  bool shouldVisitImplicitCode() const { return true; }
+
+  template <typename T>
+  bool TraverseNode(T *Node, bool (VisitorBase::*traverse)(T*)) {
+    if (Node == NULL)
+      return true;
+    if (ParentStack.size() > 0)
+      (*Parents)[Node] = ParentStack.back();
+    ParentStack.push_back(ast_type_traits::DynTypedNode::create(*Node));
+    bool Result = (this->*traverse)(Node);
+    ParentStack.pop_back();
+    return Result;
+  }
+
+  bool TraverseDecl(Decl *DeclNode) {
+    return TraverseNode(DeclNode, &VisitorBase::TraverseDecl);
+  }
+
+  bool TraverseStmt(Stmt *StmtNode) {
+    return TraverseNode(StmtNode, &VisitorBase::TraverseStmt);
+  }
+
+  ParentMap *Parents;
+  llvm::SmallVector<ast_type_traits::DynTypedNode, 16> ParentStack;
+
+  friend class RecursiveASTVisitor<ParentMapASTVisitor>;
+};
+
 // We use memoization to avoid running the same matcher on the same
 // AST node twice.  This pair is the key for looking up match
 // result.  It consists of an ID of the MatcherInterface (for
 // identifying the matcher) and a pointer to the AST node.
+//
+// We currently only memoize on nodes whose pointers identify the
+// nodes (\c Stmt and \c Decl, but not \c QualType or \c TypeLoc).
+// For \c QualType and \c TypeLoc it is possible to implement
+// generation of keys for each type.
+// FIXME: Benchmark whether memoization of non-pointer typed nodes
+// provides enough benefit for the additional amount of code.
 typedef std::pair<uint64_t, const void*> UntypedMatchInput;
 
 // Used to store the result of a match and possibly bound nodes.
@@ -50,16 +115,16 @@ public:
   // descendants of a traversed node. max_depth is the maximum depth
   // to traverse: use 1 for matching the children and INT_MAX for
   // matching the descendants.
-  MatchChildASTVisitor(const UntypedBaseMatcher *BaseMatcher,
+  MatchChildASTVisitor(const DynTypedMatcher *Matcher,
                        ASTMatchFinder *Finder,
                        BoundNodesTreeBuilder *Builder,
                        int MaxDepth,
                        ASTMatchFinder::TraversalKind Traversal,
                        ASTMatchFinder::BindKind Bind)
-      : BaseMatcher(BaseMatcher),
+      : Matcher(Matcher),
         Finder(Finder),
         Builder(Builder),
-        CurrentDepth(-1),
+        CurrentDepth(0),
         MaxDepth(MaxDepth),
         Traversal(Traversal),
         Bind(Bind),
@@ -76,10 +141,23 @@ public:
   //     Traverse*(c) for each child c of 'node'.
   //   - Traverse*(c) in turn calls Traverse(c), completing the
   //     recursion.
-  template <typename T>
-  bool findMatch(const T &Node) {
+  bool findMatch(const ast_type_traits::DynTypedNode &DynNode) {
     reset();
-    traverse(Node);
+    if (const Decl *D = DynNode.get<Decl>())
+      traverse(*D);
+    else if (const Stmt *S = DynNode.get<Stmt>())
+      traverse(*S);
+    else if (const NestedNameSpecifier *NNS =
+             DynNode.get<NestedNameSpecifier>())
+      traverse(*NNS);
+    else if (const NestedNameSpecifierLoc *NNSLoc =
+             DynNode.get<NestedNameSpecifierLoc>())
+      traverse(*NNSLoc);
+    else if (const QualType *Q = DynNode.get<QualType>())
+      traverse(*Q);
+    else if (const TypeLoc *T = DynNode.get<TypeLoc>())
+      traverse(*T);
+    // FIXME: Add other base types after adding tests.
     return Matches;
   }
 
@@ -87,9 +165,11 @@ public:
   // They are public only to allow CRTP to work. They are *not *part
   // of the public API of this class.
   bool TraverseDecl(Decl *DeclNode) {
+    ScopedIncrement ScopedDepth(&CurrentDepth);
     return (DeclNode == NULL) || traverse(*DeclNode);
   }
   bool TraverseStmt(Stmt *StmtNode) {
+    ScopedIncrement ScopedDepth(&CurrentDepth);
     const Stmt *StmtToTraverse = StmtNode;
     if (Traversal ==
         ASTMatchFinder::TK_IgnoreImplicitCastsAndParentheses) {
@@ -100,9 +180,39 @@ public:
     }
     return (StmtToTraverse == NULL) || traverse(*StmtToTraverse);
   }
+  // We assume that the QualType and the contained type are on the same
+  // hierarchy level. Thus, we try to match either of them.
   bool TraverseType(QualType TypeNode) {
+    ScopedIncrement ScopedDepth(&CurrentDepth);
+    // Match the Type.
+    if (!match(*TypeNode))
+      return false;
+    // The QualType is matched inside traverse.
     return traverse(TypeNode);
   }
+  // We assume that the TypeLoc, contained QualType and contained Type all are
+  // on the same hierarchy level. Thus, we try to match all of them.
+  bool TraverseTypeLoc(TypeLoc TypeLocNode) {
+    ScopedIncrement ScopedDepth(&CurrentDepth);
+    // Match the Type.
+    if (!match(*TypeLocNode.getType()))
+      return false;
+    // Match the QualType.
+    if (!match(TypeLocNode.getType()))
+      return false;
+    // The TypeLoc is matched inside traverse.
+    return traverse(TypeLocNode);
+  }
+  bool TraverseNestedNameSpecifier(NestedNameSpecifier *NNS) {
+    ScopedIncrement ScopedDepth(&CurrentDepth);
+    return (NNS == NULL) || traverse(*NNS);
+  }
+  bool TraverseNestedNameSpecifierLoc(NestedNameSpecifierLoc NNS) {
+    ScopedIncrement ScopedDepth(&CurrentDepth);
+    if (!match(*NNS.getNestedNameSpecifier()))
+      return false;
+    return !NNS || traverse(NNS);
+  }
 
   bool shouldVisitTemplateInstantiations() const { return true; }
   bool shouldVisitImplicitCode() const { return true; }
@@ -120,7 +230,7 @@ private:
   // Resets the state of this object.
   void reset() {
     Matches = false;
-    CurrentDepth = -1;
+    CurrentDepth = 0;
   }
 
   // Forwards the call to the corresponding Traverse*() method in the
@@ -134,49 +244,57 @@ private:
   bool baseTraverse(QualType TypeNode) {
     return VisitorBase::TraverseType(TypeNode);
   }
+  bool baseTraverse(TypeLoc TypeLocNode) {
+    return VisitorBase::TraverseTypeLoc(TypeLocNode);
+  }
+  bool baseTraverse(const NestedNameSpecifier &NNS) {
+    return VisitorBase::TraverseNestedNameSpecifier(
+        const_cast<NestedNameSpecifier*>(&NNS));
+  }
+  bool baseTraverse(NestedNameSpecifierLoc NNS) {
+    return VisitorBase::TraverseNestedNameSpecifierLoc(NNS);
+  }
 
-  // Traverses the subtree rooted at 'node'; returns true if the
-  // traversal should continue after this function returns; also sets
-  // matched_ to true if a match is found during the traversal.
+  // Sets 'Matched' to true if 'Matcher' matches 'Node' and:
+  //   0 < CurrentDepth <= MaxDepth.
+  //
+  // Returns 'true' if traversal should continue after this function
+  // returns, i.e. if no match is found or 'Bind' is 'BK_All'.
   template <typename T>
-  bool traverse(const T &Node) {
-    TOOLING_COMPILE_ASSERT(IsBaseType<T>::value,
-                           traverse_can_only_be_instantiated_with_base_type);
-    ScopedIncrement ScopedDepth(&CurrentDepth);
-    if (CurrentDepth == 0) {
-      // We don't want to match the root node, so just recurse.
-      return baseTraverse(Node);
+  bool match(const T &Node) {
+    if (CurrentDepth == 0 || CurrentDepth > MaxDepth) {
+      return true;
     }
     if (Bind != ASTMatchFinder::BK_All) {
-      if (BaseMatcher->matches(Node, Finder, Builder)) {
+      if (Matcher->matches(ast_type_traits::DynTypedNode::create(Node),
+                           Finder, Builder)) {
         Matches = true;
         return false;  // Abort as soon as a match is found.
       }
-      if (CurrentDepth < MaxDepth) {
-        // The current node doesn't match, and we haven't reached the
-        // maximum depth yet, so recurse.
-        return baseTraverse(Node);
-      }
-      // The current node doesn't match, and we have reached the
-      // maximum depth, so don't recurse (but continue the traversal
-      // such that other nodes at the current level can be visited).
-      return true;
     } else {
       BoundNodesTreeBuilder RecursiveBuilder;
-      if (BaseMatcher->matches(Node, Finder, &RecursiveBuilder)) {
+      if (Matcher->matches(ast_type_traits::DynTypedNode::create(Node),
+                           Finder, &RecursiveBuilder)) {
         // After the first match the matcher succeeds.
         Matches = true;
         Builder->addMatch(RecursiveBuilder.build());
       }
-      if (CurrentDepth < MaxDepth) {
-        baseTraverse(Node);
-      }
-      // In kBindAll mode we always search for more matches.
-      return true;
     }
+    return true;
   }
 
-  const UntypedBaseMatcher *const BaseMatcher;
+  // Traverses the subtree rooted at 'Node'; returns true if the
+  // traversal should continue after this function returns.
+  template <typename T>
+  bool traverse(const T &Node) {
+    TOOLING_COMPILE_ASSERT(IsBaseType<T>::value,
+                           traverse_can_only_be_instantiated_with_base_type);
+    if (!match(Node))
+      return false;
+    return baseTraverse(Node);
+  }
+
+  const DynTypedMatcher *const Matcher;
   ASTMatchFinder *const Finder;
   BoundNodesTreeBuilder *const Builder;
   int CurrentDepth;
@@ -191,12 +309,21 @@ private:
 class MatchASTVisitor : public RecursiveASTVisitor<MatchASTVisitor>,
                         public ASTMatchFinder {
 public:
-  MatchASTVisitor(std::vector< std::pair<const UntypedBaseMatcher*,
-                               MatchFinder::MatchCallback*> > *Triggers)
-     : Triggers(Triggers),
+  MatchASTVisitor(std::vector<std::pair<const internal::DynTypedMatcher*,
+                                        MatchCallback*> > *MatcherCallbackPairs)
+     : MatcherCallbackPairs(MatcherCallbackPairs),
        ActiveASTContext(NULL) {
   }
 
+  void onStartOfTranslationUnit() {
+    for (std::vector<std::pair<const internal::DynTypedMatcher*,
+                               MatchCallback*> >::const_iterator
+             I = MatcherCallbackPairs->begin(), E = MatcherCallbackPairs->end();
+         I != E; ++I) {
+      I->second->onStartOfTranslationUnit();
+    }
+  }
+
   void set_active_ast_context(ASTContext *NewActiveASTContext) {
     ActiveASTContext = NewActiveASTContext;
   }
@@ -207,7 +334,7 @@ public:
   bool VisitTypedefDecl(TypedefDecl *DeclNode) {
     // When we see 'typedef A B', we add name 'B' to the set of names
     // A's canonical type maps to.  This is necessary for implementing
-    // IsDerivedFrom(x) properly, where x can be the name of the base
+    // isDerivedFrom(x) properly, where x can be the name of the base
     // class or any of its aliases.
     //
     // In general, the is-alias-of (as defined by typedefs) relation
@@ -228,7 +355,7 @@ public:
     //      `- E
     //
     // It is wrong to assume that the relation is a chain.  A correct
-    // implementation of IsDerivedFrom() needs to recognize that B and
+    // implementation of isDerivedFrom() needs to recognize that B and
     // E are aliases, even though neither is a typedef of the other.
     // Therefore, we cannot simply walk through one typedef chain to
     // find out whether the type name matches.
@@ -243,23 +370,27 @@ public:
   bool TraverseStmt(Stmt *StmtNode);
   bool TraverseType(QualType TypeNode);
   bool TraverseTypeLoc(TypeLoc TypeNode);
+  bool TraverseNestedNameSpecifier(NestedNameSpecifier *NNS);
+  bool TraverseNestedNameSpecifierLoc(NestedNameSpecifierLoc NNS);
 
   // Matches children or descendants of 'Node' with 'BaseMatcher'.
-  template <typename T>
-  bool memoizedMatchesRecursively(const T &Node,
-                                  const UntypedBaseMatcher &BaseMatcher,
+  bool memoizedMatchesRecursively(const ast_type_traits::DynTypedNode &Node,
+                                  const DynTypedMatcher &Matcher,
                                   BoundNodesTreeBuilder *Builder, int MaxDepth,
                                   TraversalKind Traversal, BindKind Bind) {
-    TOOLING_COMPILE_ASSERT((llvm::is_same<T, Decl>::value) ||
-                           (llvm::is_same<T, Stmt>::value),
-                           type_does_not_support_memoization);
-    const UntypedMatchInput input(BaseMatcher.getID(), &Node);
+    const UntypedMatchInput input(Matcher.getID(), Node.getMemoizationData());
+
+    // For AST-nodes that don't have an identity, we can't memoize.
+    if (!input.second)
+      return matchesRecursively(Node, Matcher, Builder, MaxDepth, Traversal,
+                                Bind);
+
     std::pair<MemoizationMap::iterator, bool> InsertResult
       = ResultCache.insert(std::make_pair(input, MemoizedMatchResult()));
     if (InsertResult.second) {
       BoundNodesTreeBuilder DescendantBoundNodesBuilder;
       InsertResult.first->second.ResultOfMatch =
-        matchesRecursively(Node, BaseMatcher, &DescendantBoundNodesBuilder,
+        matchesRecursively(Node, Matcher, &DescendantBoundNodesBuilder,
                            MaxDepth, Traversal, Bind);
       InsertResult.first->second.Nodes =
         DescendantBoundNodesBuilder.build();
@@ -269,12 +400,12 @@ public:
   }
 
   // Matches children or descendants of 'Node' with 'BaseMatcher'.
-  template <typename T>
-  bool matchesRecursively(const T &Node, const UntypedBaseMatcher &BaseMatcher,
+  bool matchesRecursively(const ast_type_traits::DynTypedNode &Node,
+                          const DynTypedMatcher &Matcher,
                           BoundNodesTreeBuilder *Builder, int MaxDepth,
                           TraversalKind Traversal, BindKind Bind) {
     MatchChildASTVisitor Visitor(
-      &BaseMatcher, this, Builder, MaxDepth, Traversal, Bind);
+      &Matcher, this, Builder, MaxDepth, Traversal, Bind);
     return Visitor.findMatch(Node);
   }
 
@@ -282,38 +413,54 @@ public:
                                   const Matcher<NamedDecl> &Base,
                                   BoundNodesTreeBuilder *Builder);
 
-  // Implements ASTMatchFinder::MatchesChildOf.
-  virtual bool matchesChildOf(const Decl &DeclNode,
-                              const UntypedBaseMatcher &BaseMatcher,
+  // Implements ASTMatchFinder::matchesChildOf.
+  virtual bool matchesChildOf(const ast_type_traits::DynTypedNode &Node,
+                              const DynTypedMatcher &Matcher,
                               BoundNodesTreeBuilder *Builder,
                               TraversalKind Traversal,
                               BindKind Bind) {
-    return matchesRecursively(DeclNode, BaseMatcher, Builder, 1, Traversal,
+    return matchesRecursively(Node, Matcher, Builder, 1, Traversal,
                               Bind);
   }
-  virtual bool matchesChildOf(const Stmt &StmtNode,
-                              const UntypedBaseMatcher &BaseMatcher,
-                              BoundNodesTreeBuilder *Builder,
-                              TraversalKind Traversal,
-                              BindKind Bind) {
-    return matchesRecursively(StmtNode, BaseMatcher, Builder, 1, Traversal,
-                              Bind);
-  }
-
-  // Implements ASTMatchFinder::MatchesDescendantOf.
-  virtual bool matchesDescendantOf(const Decl &DeclNode,
-                                   const UntypedBaseMatcher &BaseMatcher,
+  // Implements ASTMatchFinder::matchesDescendantOf.
+  virtual bool matchesDescendantOf(const ast_type_traits::DynTypedNode &Node,
+                                   const DynTypedMatcher &Matcher,
                                    BoundNodesTreeBuilder *Builder,
                                    BindKind Bind) {
-    return memoizedMatchesRecursively(DeclNode, BaseMatcher, Builder, INT_MAX,
+    return memoizedMatchesRecursively(Node, Matcher, Builder, INT_MAX,
                                       TK_AsIs, Bind);
   }
-  virtual bool matchesDescendantOf(const Stmt &StmtNode,
-                                   const UntypedBaseMatcher &BaseMatcher,
-                                   BoundNodesTreeBuilder *Builder,
-                                   BindKind Bind) {
-    return memoizedMatchesRecursively(StmtNode, BaseMatcher, Builder, INT_MAX,
-                                      TK_AsIs, Bind);
+  // Implements ASTMatchFinder::matchesAncestorOf.
+  virtual bool matchesAncestorOf(const ast_type_traits::DynTypedNode &Node,
+                                 const DynTypedMatcher &Matcher,
+                                 BoundNodesTreeBuilder *Builder,
+                                 AncestorMatchMode MatchMode) {
+    if (!Parents) {
+      // We always need to run over the whole translation unit, as
+      // \c hasAncestor can escape any subtree.
+      Parents.reset(ParentMapASTVisitor::buildMap(
+        *ActiveASTContext->getTranslationUnitDecl()));
+    }
+    ast_type_traits::DynTypedNode Ancestor = Node;
+    while (Ancestor.get<TranslationUnitDecl>() !=
+           ActiveASTContext->getTranslationUnitDecl()) {
+      assert(Ancestor.getMemoizationData() &&
+             "Invariant broken: only nodes that support memoization may be "
+             "used in the parent map.");
+      ParentMapASTVisitor::ParentMap::const_iterator I =
+        Parents->find(Ancestor.getMemoizationData());
+      if (I == Parents->end()) {
+        assert(false &&
+               "Found node that is not in the parent map.");
+        return false;
+      }
+      Ancestor = I->second;
+      if (Matcher.matches(Ancestor, this, Builder))
+        return true;
+      if (MatchMode == ASTMatchFinder::AMM_ParentOnly)
+        return false;
+    }
+    return false;
   }
 
   bool shouldVisitTemplateInstantiations() const { return true; }
@@ -358,21 +505,22 @@ private:
   // result callback for every node that matches.
   template <typename T>
   void match(const T &node) {
-    for (std::vector< std::pair<const UntypedBaseMatcher*,
-                      MatchFinder::MatchCallback*> >::const_iterator
-             It = Triggers->begin(), End = Triggers->end();
-         It != End; ++It) {
+    for (std::vector<std::pair<const internal::DynTypedMatcher*,
+                               MatchCallback*> >::const_iterator
+             I = MatcherCallbackPairs->begin(), E = MatcherCallbackPairs->end();
+         I != E; ++I) {
       BoundNodesTreeBuilder Builder;
-      if (It->first->matches(node, this, &Builder)) {
+      if (I->first->matches(ast_type_traits::DynTypedNode::create(node),
+                            this, &Builder)) {
         BoundNodesTree BoundNodes = Builder.build();
-        MatchVisitor Visitor(ActiveASTContext, It->second);
+        MatchVisitor Visitor(ActiveASTContext, I->second);
         BoundNodes.visitMatches(&Visitor);
       }
     }
   }
 
-  std::vector< std::pair<const UntypedBaseMatcher*,
-               MatchFinder::MatchCallback*> > *const Triggers;
+  std::vector<std::pair<const internal::DynTypedMatcher*,
+                        MatchCallback*> > *const MatcherCallbackPairs;
   ASTContext *ActiveASTContext;
 
   // Maps a canonical type to its TypedefDecls.
@@ -381,16 +529,16 @@ private:
   // Maps (matcher, node) -> the match result for memoization.
   typedef llvm::DenseMap<UntypedMatchInput, MemoizedMatchResult> MemoizationMap;
   MemoizationMap ResultCache;
+
+  llvm::OwningPtr<ParentMapASTVisitor::ParentMap> Parents;
 };
 
 // Returns true if the given class is directly or indirectly derived
-// from a base type with the given name.  A class is considered to be
-// also derived from itself.
+// from a base type with the given name.  A class is not considered to be
+// derived from itself.
 bool MatchASTVisitor::classIsDerivedFrom(const CXXRecordDecl *Declaration,
                                          const Matcher<NamedDecl> &Base,
                                          BoundNodesTreeBuilder *Builder) {
-  if (Base.matches(*Declaration, this, Builder))
-    return true;
   if (!Declaration->hasDefinition())
     return false;
   typedef CXXRecordDecl::base_class_const_iterator BaseIterator;
@@ -403,6 +551,7 @@ bool MatchASTVisitor::classIsDerivedFrom(const CXXRecordDecl *Declaration,
 
     // Type::getAs<...>() drills through typedefs.
     if (TypeNode->getAs<DependentNameType>() != NULL ||
+        TypeNode->getAs<DependentTemplateSpecializationType>() != NULL ||
         TypeNode->getAs<TemplateTypeParmType>() != NULL)
       // Dependent names and template TypeNode parameters will be matched when
       // the template is instantiated.
@@ -439,6 +588,8 @@ bool MatchASTVisitor::classIsDerivedFrom(const CXXRecordDecl *Declaration,
     }
     assert(ClassDecl != NULL);
     assert(ClassDecl != Declaration);
+    if (Base.matches(*ClassDecl, this, Builder))
+      return true;
     if (classIsDerivedFrom(ClassDecl, Base, Builder))
       return true;
   }
@@ -466,19 +617,40 @@ bool MatchASTVisitor::TraverseType(QualType TypeNode) {
   return RecursiveASTVisitor<MatchASTVisitor>::TraverseType(TypeNode);
 }
 
-bool MatchASTVisitor::TraverseTypeLoc(TypeLoc TypeLoc) {
-  match(TypeLoc.getType());
-  return RecursiveASTVisitor<MatchASTVisitor>::
-      TraverseTypeLoc(TypeLoc);
+bool MatchASTVisitor::TraverseTypeLoc(TypeLoc TypeLocNode) {
+  // The RecursiveASTVisitor only visits types if they're not within TypeLocs.
+  // We still want to find those types via matchers, so we match them here. Note
+  // that the TypeLocs are structurally a shadow-hierarchy to the expressed
+  // type, so we visit all involved parts of a compound type when matching on
+  // each TypeLoc.
+  match(TypeLocNode);
+  match(TypeLocNode.getType());
+  return RecursiveASTVisitor<MatchASTVisitor>::TraverseTypeLoc(TypeLocNode);
+}
+
+bool MatchASTVisitor::TraverseNestedNameSpecifier(NestedNameSpecifier *NNS) {
+  match(*NNS);
+  return RecursiveASTVisitor<MatchASTVisitor>::TraverseNestedNameSpecifier(NNS);
+}
+
+bool MatchASTVisitor::TraverseNestedNameSpecifierLoc(
+    NestedNameSpecifierLoc NNS) {
+  match(NNS);
+  // We only match the nested name specifier here (as opposed to traversing it)
+  // because the traversal is already done in the parallel "Loc"-hierarchy.
+  match(*NNS.getNestedNameSpecifier());
+  return
+      RecursiveASTVisitor<MatchASTVisitor>::TraverseNestedNameSpecifierLoc(NNS);
 }
 
 class MatchASTConsumer : public ASTConsumer {
 public:
-  MatchASTConsumer(std::vector< std::pair<const UntypedBaseMatcher*,
-                                MatchFinder::MatchCallback*> > *Triggers,
-                   MatchFinder::ParsingDoneTestCallback *ParsingDone)
-      : Visitor(Triggers),
-        ParsingDone(ParsingDone) {}
+  MatchASTConsumer(
+    std::vector<std::pair<const internal::DynTypedMatcher*,
+                          MatchCallback*> > *MatcherCallbackPairs,
+    MatchFinder::ParsingDoneTestCallback *ParsingDone)
+    : Visitor(MatcherCallbackPairs),
+      ParsingDone(ParsingDone) {}
 
 private:
   virtual void HandleTranslationUnit(ASTContext &Context) {
@@ -486,6 +658,7 @@ private:
       ParsingDone->run();
     }
     Visitor.set_active_ast_context(&Context);
+    Visitor.onStartOfTranslationUnit();
     Visitor.TraverseDecl(Context.getTranslationUnitDecl());
     Visitor.set_active_ast_context(NULL);
   }
@@ -508,9 +681,9 @@ MatchFinder::ParsingDoneTestCallback::~ParsingDoneTestCallback() {}
 MatchFinder::MatchFinder() : ParsingDone(NULL) {}
 
 MatchFinder::~MatchFinder() {
-  for (std::vector< std::pair<const internal::UntypedBaseMatcher*,
-                    MatchFinder::MatchCallback*> >::const_iterator
-           It = Triggers.begin(), End = Triggers.end();
+  for (std::vector<std::pair<const internal::DynTypedMatcher*,
+                             MatchCallback*> >::const_iterator
+           It = MatcherCallbackPairs.begin(), End = MatcherCallbackPairs.end();
        It != End; ++It) {
     delete It->first;
   }
@@ -518,24 +691,54 @@ MatchFinder::~MatchFinder() {
 
 void MatchFinder::addMatcher(const DeclarationMatcher &NodeMatch,
                              MatchCallback *Action) {
-  Triggers.push_back(std::make_pair(
-    new internal::TypedBaseMatcher<Decl>(NodeMatch), Action));
+  MatcherCallbackPairs.push_back(std::make_pair(
+    new internal::Matcher<Decl>(NodeMatch), Action));
 }
 
 void MatchFinder::addMatcher(const TypeMatcher &NodeMatch,
                              MatchCallback *Action) {
-  Triggers.push_back(std::make_pair(
-    new internal::TypedBaseMatcher<QualType>(NodeMatch), Action));
+  MatcherCallbackPairs.push_back(std::make_pair(
+    new internal::Matcher<QualType>(NodeMatch), Action));
 }
 
 void MatchFinder::addMatcher(const StatementMatcher &NodeMatch,
                              MatchCallback *Action) {
-  Triggers.push_back(std::make_pair(
-    new internal::TypedBaseMatcher<Stmt>(NodeMatch), Action));
+  MatcherCallbackPairs.push_back(std::make_pair(
+    new internal::Matcher<Stmt>(NodeMatch), Action));
+}
+
+void MatchFinder::addMatcher(const NestedNameSpecifierMatcher &NodeMatch,
+                             MatchCallback *Action) {
+  MatcherCallbackPairs.push_back(std::make_pair(
+    new NestedNameSpecifierMatcher(NodeMatch), Action));
+}
+
+void MatchFinder::addMatcher(const NestedNameSpecifierLocMatcher &NodeMatch,
+                             MatchCallback *Action) {
+  MatcherCallbackPairs.push_back(std::make_pair(
+    new NestedNameSpecifierLocMatcher(NodeMatch), Action));
+}
+
+void MatchFinder::addMatcher(const TypeLocMatcher &NodeMatch,
+                             MatchCallback *Action) {
+  MatcherCallbackPairs.push_back(std::make_pair(
+    new TypeLocMatcher(NodeMatch), Action));
 }
 
 ASTConsumer *MatchFinder::newASTConsumer() {
-  return new internal::MatchASTConsumer(&Triggers, ParsingDone);
+  return new internal::MatchASTConsumer(&MatcherCallbackPairs, ParsingDone);
+}
+
+void MatchFinder::findAll(const Decl &Node, ASTContext &Context) {
+  internal::MatchASTVisitor Visitor(&MatcherCallbackPairs);
+  Visitor.set_active_ast_context(&Context);
+  Visitor.TraverseDecl(const_cast<Decl*>(&Node));
+}
+
+void MatchFinder::findAll(const Stmt &Node, ASTContext &Context) {
+  internal::MatchASTVisitor Visitor(&MatcherCallbackPairs);
+  Visitor.set_active_ast_context(&Context);
+  Visitor.TraverseStmt(const_cast<Stmt*>(&Node));
 }
 
 void MatchFinder::registerTestCallbackAfterParsing(
diff --git a/lib/ASTMatchers/ASTMatchersInternal.cpp b/lib/ASTMatchers/ASTMatchersInternal.cpp
index 69c5190..408195d 100644
--- a/lib/ASTMatchers/ASTMatchersInternal.cpp
+++ b/lib/ASTMatchers/ASTMatchersInternal.cpp
@@ -18,18 +18,29 @@ namespace clang {
 namespace ast_matchers {
 namespace internal {
 
+void BoundNodesMap::copyTo(BoundNodesTreeBuilder *Builder) const {
+  for (IDToNodeMap::const_iterator It = NodeMap.begin();
+       It != NodeMap.end();
+       ++It) {
+    Builder->setBinding(It->first, It->second);
+  }
+}
+
+void BoundNodesMap::copyTo(BoundNodesMap *Other) const {
+  copy(NodeMap.begin(), NodeMap.end(),
+       inserter(Other->NodeMap, Other->NodeMap.begin()));
+}
+
 BoundNodesTree::BoundNodesTree() {}
 
 BoundNodesTree::BoundNodesTree(
-  const std::map<std::string, const Decl*>& DeclBindings,
-  const std::map<std::string, const Stmt*>& StmtBindings,
+  const BoundNodesMap& Bindings,
   const std::vector<BoundNodesTree> RecursiveBindings)
-  : DeclBindings(DeclBindings), StmtBindings(StmtBindings),
+  : Bindings(Bindings),
     RecursiveBindings(RecursiveBindings) {}
 
 void BoundNodesTree::copyTo(BoundNodesTreeBuilder* Builder) const {
-  copyBindingsTo(DeclBindings, Builder);
-  copyBindingsTo(StmtBindings, Builder);
+  Bindings.copyTo(Builder);
   for (std::vector<BoundNodesTree>::const_iterator
          I = RecursiveBindings.begin(),
          E = RecursiveBindings.end();
@@ -38,63 +49,34 @@ void BoundNodesTree::copyTo(BoundNodesTreeBuilder* Builder) const {
   }
 }
 
-template <typename T>
-void BoundNodesTree::copyBindingsTo(
-    const T& Bindings, BoundNodesTreeBuilder* Builder) const {
-  for (typename T::const_iterator I = Bindings.begin(),
-                                  E = Bindings.end();
-       I != E; ++I) {
-    Builder->setBinding(I->first, I->second);
-  }
-}
-
 void BoundNodesTree::visitMatches(Visitor* ResultVisitor) {
-  std::map<std::string, const Decl*> AggregatedDeclBindings;
-  std::map<std::string, const Stmt*> AggregatedStmtBindings;
-  visitMatchesRecursively(ResultVisitor, AggregatedDeclBindings,
-                          AggregatedStmtBindings);
+  BoundNodesMap AggregatedBindings;
+  visitMatchesRecursively(ResultVisitor, AggregatedBindings);
 }
 
 void BoundNodesTree::
 visitMatchesRecursively(Visitor* ResultVisitor,
-                        std::map<std::string, const Decl*>
-                          AggregatedDeclBindings,
-                        std::map<std::string, const Stmt*>
-                          AggregatedStmtBindings) {
-  copy(DeclBindings.begin(), DeclBindings.end(),
-       inserter(AggregatedDeclBindings, AggregatedDeclBindings.begin()));
-  copy(StmtBindings.begin(), StmtBindings.end(),
-       inserter(AggregatedStmtBindings, AggregatedStmtBindings.begin()));
+                        const BoundNodesMap& AggregatedBindings) {
+  BoundNodesMap CombinedBindings(AggregatedBindings);
+  Bindings.copyTo(&CombinedBindings);
   if (RecursiveBindings.empty()) {
-    ResultVisitor->visitMatch(BoundNodes(AggregatedDeclBindings,
-                                         AggregatedStmtBindings));
+    ResultVisitor->visitMatch(BoundNodes(CombinedBindings));
   } else {
     for (unsigned I = 0; I < RecursiveBindings.size(); ++I) {
       RecursiveBindings[I].visitMatchesRecursively(ResultVisitor,
-                                                   AggregatedDeclBindings,
-                                                   AggregatedStmtBindings);
+                                                   CombinedBindings);
     }
   }
 }
 
 BoundNodesTreeBuilder::BoundNodesTreeBuilder() {}
 
-void BoundNodesTreeBuilder::setBinding(const std::string &Id,
-                                       const Decl *Node) {
-  DeclBindings[Id] = Node;
-}
-
-void BoundNodesTreeBuilder::setBinding(const std::string &Id,
-                                       const Stmt *Node) {
-  StmtBindings[Id] = Node;
-}
-
 void BoundNodesTreeBuilder::addMatch(const BoundNodesTree& Bindings) {
   RecursiveBindings.push_back(Bindings);
 }
 
 BoundNodesTree BoundNodesTreeBuilder::build() const {
-  return BoundNodesTree(DeclBindings, StmtBindings, RecursiveBindings);
+  return BoundNodesTree(Bindings, RecursiveBindings);
 }
 
 } // end namespace internal
diff --git a/lib/Analysis/AnalysisDeclContext.cpp b/lib/Analysis/AnalysisDeclContext.cpp
index 7de7f39..e7df0a8 100644
--- a/lib/Analysis/AnalysisDeclContext.cpp
+++ b/lib/Analysis/AnalysisDeclContext.cpp
@@ -29,13 +29,15 @@
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/Support/ErrorHandling.h"
 
+#include "BodyFarm.h"
+
 using namespace clang;
 
 typedef llvm::DenseMap<const void *, ManagedAnalysis *> ManagedAnalysisMap;
 
 AnalysisDeclContext::AnalysisDeclContext(AnalysisDeclContextManager *Mgr,
-                                 const Decl *d,
-                                 const CFG::BuildOptions &buildOptions)
+                                         const Decl *d,
+                                         const CFG::BuildOptions &buildOptions)
   : Manager(Mgr),
     D(d),
     cfgBuildOptions(buildOptions),
@@ -49,7 +51,7 @@ AnalysisDeclContext::AnalysisDeclContext(AnalysisDeclContextManager *Mgr,
 }
 
 AnalysisDeclContext::AnalysisDeclContext(AnalysisDeclContextManager *Mgr,
-                                 const Decl *d)
+                                         const Decl *d)
 : Manager(Mgr),
   D(d),
   forcedBlkExprs(0),
@@ -62,11 +64,16 @@ AnalysisDeclContext::AnalysisDeclContext(AnalysisDeclContextManager *Mgr,
 }
 
 AnalysisDeclContextManager::AnalysisDeclContextManager(bool useUnoptimizedCFG,
-                                               bool addImplicitDtors,
-                                               bool addInitializers) {
+                                                       bool addImplicitDtors,
+                                                       bool addInitializers,
+                                                       bool addTemporaryDtors,
+                                                       bool synthesizeBodies)
+  : SynthesizeBodies(synthesizeBodies)
+{
   cfgBuildOptions.PruneTriviallyFalseEdges = !useUnoptimizedCFG;
   cfgBuildOptions.AddImplicitDtors = addImplicitDtors;
   cfgBuildOptions.AddInitializers = addInitializers;
+  cfgBuildOptions.AddTemporaryDtors = addTemporaryDtors;
 }
 
 void AnalysisDeclContextManager::clear() {
@@ -75,9 +82,18 @@ void AnalysisDeclContextManager::clear() {
   Contexts.clear();
 }
 
+static BodyFarm &getBodyFarm(ASTContext &C) {
+  static BodyFarm *BF = new BodyFarm(C);
+  return *BF;
+}
+
 Stmt *AnalysisDeclContext::getBody() const {
-  if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D))
-    return FD->getBody();
+  if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
+    Stmt *Body = FD->getBody();
+    if (!Body && Manager && Manager->synthesizeBodies())
+      return getBodyFarm(getASTContext()).getBody(FD);
+    return Body;
+  }
   else if (const ObjCMethodDecl *MD = dyn_cast<ObjCMethodDecl>(D))
     return MD->getBody();
   else if (const BlockDecl *BD = dyn_cast<BlockDecl>(D))
@@ -201,6 +217,13 @@ PseudoConstantAnalysis *AnalysisDeclContext::getPseudoConstantAnalysis() {
 }
 
 AnalysisDeclContext *AnalysisDeclContextManager::getContext(const Decl *D) {
+  if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
+    // Calling 'hasBody' replaces 'FD' in place with the FunctionDecl
+    // that has the body.
+    FD->hasBody(FD);
+    D = FD;
+  }
+
   AnalysisDeclContext *&AC = Contexts[D];
   if (!AC)
     AC = new AnalysisDeclContext(this, D, cfgBuildOptions);
@@ -332,6 +355,10 @@ const StackFrameContext *LocationContext::getCurrentStackFrame() const {
   return NULL;
 }
 
+bool LocationContext::inTopFrame() const {
+  return getCurrentStackFrame()->inTopFrame();
+}
+
 bool LocationContext::isParentOf(const LocationContext *LC) const {
   do {
     const LocationContext *Parent = LC->getParent();
diff --git a/lib/Analysis/BodyFarm.cpp b/lib/Analysis/BodyFarm.cpp
new file mode 100644
index 0000000..794ff9c
--- /dev/null
+++ b/lib/Analysis/BodyFarm.cpp
@@ -0,0 +1,374 @@
+//== BodyFarm.cpp  - Factory for conjuring up fake bodies ----------*- C++ -*-//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// BodyFarm is a factory for creating faux implementations for functions/methods
+// for analysis purposes.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/StringSwitch.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/Expr.h"
+#include "clang/AST/Decl.h"
+#include "clang/AST/ExprObjC.h"
+#include "BodyFarm.h"
+
+using namespace clang;
+
+//===----------------------------------------------------------------------===//
+// Helper creation functions for constructing faux ASTs.
+//===----------------------------------------------------------------------===//
+
+static bool isDispatchBlock(QualType Ty) {
+  // Is it a block pointer?
+  const BlockPointerType *BPT = Ty->getAs<BlockPointerType>();
+  if (!BPT)
+    return false;
+
+  // Check if the block pointer type takes no arguments and
+  // returns void.
+  const FunctionProtoType *FT =
+  BPT->getPointeeType()->getAs<FunctionProtoType>();
+  if (!FT || !FT->getResultType()->isVoidType()  ||
+      FT->getNumArgs() != 0)
+    return false;
+
+  return true;
+}
+
+namespace {
+class ASTMaker {
+public:
+  ASTMaker(ASTContext &C) : C(C) {}
+  
+  /// Create a new BinaryOperator representing a simple assignment.
+  BinaryOperator *makeAssignment(const Expr *LHS, const Expr *RHS, QualType Ty);
+  
+  /// Create a new BinaryOperator representing a comparison.
+  BinaryOperator *makeComparison(const Expr *LHS, const Expr *RHS,
+                                 BinaryOperator::Opcode Op);
+  
+  /// Create a new compound stmt using the provided statements.
+  CompoundStmt *makeCompound(ArrayRef<Stmt*>);
+  
+  /// Create a new DeclRefExpr for the referenced variable.
+  DeclRefExpr *makeDeclRefExpr(const VarDecl *D);
+  
+  /// Create a new UnaryOperator representing a dereference.
+  UnaryOperator *makeDereference(const Expr *Arg, QualType Ty);
+  
+  /// Create an implicit cast for an integer conversion.
+  Expr *makeIntegralCast(const Expr *Arg, QualType Ty);
+  
+  /// Create an implicit cast to a builtin boolean type.
+  ImplicitCastExpr *makeIntegralCastToBoolean(const Expr *Arg);
+  
+  // Create an implicit cast for lvalue-to-rvaluate conversions.
+  ImplicitCastExpr *makeLvalueToRvalue(const Expr *Arg, QualType Ty);
+  
+  /// Create an Objective-C bool literal.
+  ObjCBoolLiteralExpr *makeObjCBool(bool Val);
+  
+  /// Create a Return statement.
+  ReturnStmt *makeReturn(const Expr *RetVal);
+  
+private:
+  ASTContext &C;
+};
+}
+
+BinaryOperator *ASTMaker::makeAssignment(const Expr *LHS, const Expr *RHS,
+                                         QualType Ty) {
+ return new (C) BinaryOperator(const_cast<Expr*>(LHS), const_cast<Expr*>(RHS),
+                               BO_Assign, Ty, VK_RValue,
+                               OK_Ordinary, SourceLocation(), false);
+}
+
+BinaryOperator *ASTMaker::makeComparison(const Expr *LHS, const Expr *RHS,
+                                         BinaryOperator::Opcode Op) {
+  assert(BinaryOperator::isLogicalOp(Op) ||
+         BinaryOperator::isComparisonOp(Op));
+  return new (C) BinaryOperator(const_cast<Expr*>(LHS),
+                                const_cast<Expr*>(RHS),
+                                Op,
+                                C.getLogicalOperationType(),
+                                VK_RValue,
+                                OK_Ordinary, SourceLocation(), false);
+}
+
+CompoundStmt *ASTMaker::makeCompound(ArrayRef<Stmt *> Stmts) {
+  return new (C) CompoundStmt(C, const_cast<Stmt**>(Stmts.data()),
+                              Stmts.size(),
+                              SourceLocation(), SourceLocation());
+}
+
+DeclRefExpr *ASTMaker::makeDeclRefExpr(const VarDecl *D) {
+  DeclRefExpr *DR =
+    DeclRefExpr::Create(/* Ctx = */ C,
+                        /* QualifierLoc = */ NestedNameSpecifierLoc(),
+                        /* TemplateKWLoc = */ SourceLocation(),
+                        /* D = */ const_cast<VarDecl*>(D),
+                        /* isEnclosingLocal = */ false,
+                        /* NameLoc = */ SourceLocation(),
+                        /* T = */ D->getType(),
+                        /* VK = */ VK_LValue);
+  return DR;
+}
+
+UnaryOperator *ASTMaker::makeDereference(const Expr *Arg, QualType Ty) {
+  return new (C) UnaryOperator(const_cast<Expr*>(Arg), UO_Deref, Ty,
+                               VK_LValue, OK_Ordinary, SourceLocation());
+}
+
+ImplicitCastExpr *ASTMaker::makeLvalueToRvalue(const Expr *Arg, QualType Ty) {
+  return ImplicitCastExpr::Create(C, Ty, CK_LValueToRValue,
+                                  const_cast<Expr*>(Arg), 0, VK_RValue);
+}
+
+Expr *ASTMaker::makeIntegralCast(const Expr *Arg, QualType Ty) {
+  if (Arg->getType() == Ty)
+    return const_cast<Expr*>(Arg);
+  
+  return ImplicitCastExpr::Create(C, Ty, CK_IntegralCast,
+                                  const_cast<Expr*>(Arg), 0, VK_RValue);
+}
+
+ImplicitCastExpr *ASTMaker::makeIntegralCastToBoolean(const Expr *Arg) {
+  return ImplicitCastExpr::Create(C, C.BoolTy, CK_IntegralToBoolean,
+                                  const_cast<Expr*>(Arg), 0, VK_RValue);
+}
+
+ObjCBoolLiteralExpr *ASTMaker::makeObjCBool(bool Val) {
+  QualType Ty = C.getBOOLDecl() ? C.getBOOLType() : C.ObjCBuiltinBoolTy;
+  return new (C) ObjCBoolLiteralExpr(Val, Ty, SourceLocation());
+}
+
+ReturnStmt *ASTMaker::makeReturn(const Expr *RetVal) {
+  return new (C) ReturnStmt(SourceLocation(), const_cast<Expr*>(RetVal), 0);
+}
+
+//===----------------------------------------------------------------------===//
+// Creation functions for faux ASTs.
+//===----------------------------------------------------------------------===//
+
+typedef Stmt *(*FunctionFarmer)(ASTContext &C, const FunctionDecl *D);
+
+/// Create a fake body for dispatch_once.
+static Stmt *create_dispatch_once(ASTContext &C, const FunctionDecl *D) {
+  // Check if we have at least two parameters.
+  if (D->param_size() != 2)
+    return 0;
+
+  // Check if the first parameter is a pointer to integer type.
+  const ParmVarDecl *Predicate = D->getParamDecl(0);
+  QualType PredicateQPtrTy = Predicate->getType();
+  const PointerType *PredicatePtrTy = PredicateQPtrTy->getAs<PointerType>();
+  if (!PredicatePtrTy)
+    return 0;
+  QualType PredicateTy = PredicatePtrTy->getPointeeType();
+  if (!PredicateTy->isIntegerType())
+    return 0;
+  
+  // Check if the second parameter is the proper block type.
+  const ParmVarDecl *Block = D->getParamDecl(1);
+  QualType Ty = Block->getType();
+  if (!isDispatchBlock(Ty))
+    return 0;
+  
+  // Everything checks out.  Create a fakse body that checks the predicate,
+  // sets it, and calls the block.  Basically, an AST dump of:
+  //
+  // void dispatch_once(dispatch_once_t *predicate, dispatch_block_t block) {
+  //  if (!*predicate) {
+  //    *predicate = 1;
+  //    block();
+  //  }
+  // }
+  
+  ASTMaker M(C);
+  
+  // (1) Create the call.
+  DeclRefExpr *DR = M.makeDeclRefExpr(Block);
+  ImplicitCastExpr *ICE = M.makeLvalueToRvalue(DR, Ty);
+  CallExpr *CE = new (C) CallExpr(C, ICE, ArrayRef<Expr*>(), C.VoidTy,
+                                  VK_RValue, SourceLocation());
+
+  // (2) Create the assignment to the predicate.
+  IntegerLiteral *IL =
+    IntegerLiteral::Create(C, llvm::APInt(C.getTypeSize(C.IntTy), (uint64_t) 1),
+                           C.IntTy, SourceLocation());
+  BinaryOperator *B =
+    M.makeAssignment(
+       M.makeDereference(
+          M.makeLvalueToRvalue(
+            M.makeDeclRefExpr(Predicate), PredicateQPtrTy),
+            PredicateTy),
+       M.makeIntegralCast(IL, PredicateTy),
+       PredicateTy);
+  
+  // (3) Create the compound statement.
+  Stmt *Stmts[2];
+  Stmts[0] = B;
+  Stmts[1] = CE;
+  CompoundStmt *CS = M.makeCompound(ArrayRef<Stmt*>(Stmts, 2));
+  
+  // (4) Create the 'if' condition.
+  ImplicitCastExpr *LValToRval =
+    M.makeLvalueToRvalue(
+      M.makeDereference(
+        M.makeLvalueToRvalue(
+          M.makeDeclRefExpr(Predicate),
+          PredicateQPtrTy),
+        PredicateTy),
+    PredicateTy);
+  
+  UnaryOperator *UO = new (C) UnaryOperator(LValToRval, UO_LNot, C.IntTy,
+                                           VK_RValue, OK_Ordinary,
+                                           SourceLocation());
+  
+  // (5) Create the 'if' statement.
+  IfStmt *If = new (C) IfStmt(C, SourceLocation(), 0, UO, CS);
+  return If;
+}
+
+/// Create a fake body for dispatch_sync.
+static Stmt *create_dispatch_sync(ASTContext &C, const FunctionDecl *D) {
+  // Check if we have at least two parameters.
+  if (D->param_size() != 2)
+    return 0;
+  
+  // Check if the second parameter is a block.
+  const ParmVarDecl *PV = D->getParamDecl(1);
+  QualType Ty = PV->getType();
+  if (!isDispatchBlock(Ty))
+    return 0;
+  
+  // Everything checks out.  Create a fake body that just calls the block.
+  // This is basically just an AST dump of:
+  //
+  // void dispatch_sync(dispatch_queue_t queue, void (^block)(void)) {
+  //   block();
+  // }
+  //  
+  ASTMaker M(C);
+  DeclRefExpr *DR = M.makeDeclRefExpr(PV);
+  ImplicitCastExpr *ICE = M.makeLvalueToRvalue(DR, Ty);
+  CallExpr *CE = new (C) CallExpr(C, ICE, ArrayRef<Expr*>(), C.VoidTy,
+                                  VK_RValue, SourceLocation());
+  return CE;
+}
+
+static Stmt *create_OSAtomicCompareAndSwap(ASTContext &C, const FunctionDecl *D)
+{
+  // There are exactly 3 arguments.
+  if (D->param_size() != 3)
+    return 0;
+  
+  // Body for:
+  //   if (oldValue == *theValue) {
+  //    *theValue = newValue;
+  //    return YES;
+  //   }
+  //   else return NO;
+  
+  QualType ResultTy = D->getResultType();
+  bool isBoolean = ResultTy->isBooleanType();
+  if (!isBoolean && !ResultTy->isIntegralType(C))
+    return 0;
+  
+  const ParmVarDecl *OldValue = D->getParamDecl(0);
+  QualType OldValueTy = OldValue->getType();
+
+  const ParmVarDecl *NewValue = D->getParamDecl(1);
+  QualType NewValueTy = NewValue->getType();
+  
+  assert(OldValueTy == NewValueTy);
+  
+  const ParmVarDecl *TheValue = D->getParamDecl(2);
+  QualType TheValueTy = TheValue->getType();
+  const PointerType *PT = TheValueTy->getAs<PointerType>();
+  if (!PT)
+    return 0;
+  QualType PointeeTy = PT->getPointeeType();
+  
+  ASTMaker M(C);
+  // Construct the comparison.
+  Expr *Comparison =
+    M.makeComparison(
+      M.makeLvalueToRvalue(M.makeDeclRefExpr(OldValue), OldValueTy),
+      M.makeLvalueToRvalue(
+        M.makeDereference(
+          M.makeLvalueToRvalue(M.makeDeclRefExpr(TheValue), TheValueTy),
+          PointeeTy),
+        PointeeTy),
+      BO_EQ);
+
+  // Construct the body of the IfStmt.
+  Stmt *Stmts[2];
+  Stmts[0] =
+    M.makeAssignment(
+      M.makeDereference(
+        M.makeLvalueToRvalue(M.makeDeclRefExpr(TheValue), TheValueTy),
+        PointeeTy),
+      M.makeLvalueToRvalue(M.makeDeclRefExpr(NewValue), NewValueTy),
+      NewValueTy);
+  
+  Expr *BoolVal = M.makeObjCBool(true);
+  Expr *RetVal = isBoolean ? M.makeIntegralCastToBoolean(BoolVal)
+                           : M.makeIntegralCast(BoolVal, ResultTy);
+  Stmts[1] = M.makeReturn(RetVal);
+  CompoundStmt *Body = M.makeCompound(ArrayRef<Stmt*>(Stmts, 2));
+  
+  // Construct the else clause.
+  BoolVal = M.makeObjCBool(false);
+  RetVal = isBoolean ? M.makeIntegralCastToBoolean(BoolVal)
+                     : M.makeIntegralCast(BoolVal, ResultTy);
+  Stmt *Else = M.makeReturn(RetVal);
+  
+  /// Construct the If.
+  Stmt *If =
+    new (C) IfStmt(C, SourceLocation(), 0, Comparison, Body,
+                   SourceLocation(), Else);
+  
+  return If;  
+}
+
+Stmt *BodyFarm::getBody(const FunctionDecl *D) {
+  D = D->getCanonicalDecl();
+  
+  llvm::Optional<Stmt *> &Val = Bodies[D];
+  if (Val.hasValue())
+    return Val.getValue();
+  
+  Val = 0;
+  
+  if (D->getIdentifier() == 0)
+    return 0;
+
+  StringRef Name = D->getName();
+  if (Name.empty())
+    return 0;
+
+  FunctionFarmer FF;
+
+  if (Name.startswith("OSAtomicCompareAndSwap") ||
+      Name.startswith("objc_atomicCompareAndSwap")) {
+    FF = create_OSAtomicCompareAndSwap;
+  }
+  else {
+    FF = llvm::StringSwitch<FunctionFarmer>(Name)
+          .Case("dispatch_sync", create_dispatch_sync)
+          .Case("dispatch_once", create_dispatch_once)
+        .Default(NULL);
+  }
+  
+  if (FF) { Val = FF(C, D); }
+  return Val.getValue();
+}
+
diff --git a/lib/Analysis/BodyFarm.h b/lib/Analysis/BodyFarm.h
new file mode 100644
index 0000000..d503cc1
--- /dev/null
+++ b/lib/Analysis/BodyFarm.h
@@ -0,0 +1,43 @@
+//== BodyFarm.h - Factory for conjuring up fake bodies -------------*- C++ -*-//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// BodyFarm is a factory for creating faux implementations for functions/methods
+// for analysis purposes.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_ANALYSIS_BODYFARM_H
+#define LLVM_CLANG_ANALYSIS_BODYFARM_H
+
+#include "llvm/ADT/Optional.h"
+#include "llvm/ADT/DenseMap.h"
+
+namespace clang {
+
+class ASTContext;
+class Decl;
+class FunctionDecl;
+class Stmt;
+  
+class BodyFarm {
+public:
+  BodyFarm(ASTContext &C) : C(C) {}
+  
+  /// Factory method for creating bodies for ordinary functions.
+  Stmt *getBody(const FunctionDecl *D);
+  
+private:
+  typedef llvm::DenseMap<const Decl *, llvm::Optional<Stmt *> > BodyMap;
+
+  ASTContext &C;
+  BodyMap Bodies;
+};
+}
+
+#endif
diff --git a/lib/Analysis/CFG.cpp b/lib/Analysis/CFG.cpp
index 05c5385..315e543 100644
--- a/lib/Analysis/CFG.cpp
+++ b/lib/Analysis/CFG.cpp
@@ -467,6 +467,30 @@ private:
         CachedBoolEvals[S] = Result; // update or insert
         return Result;
       }
+      else {
+        switch (Bop->getOpcode()) {
+          default: break;
+          // For 'x & 0' and 'x * 0', we can determine that
+          // the value is always false.
+          case BO_Mul:
+          case BO_And: {
+            // If either operand is zero, we know the value
+            // must be false.
+            llvm::APSInt IntVal;
+            if (Bop->getLHS()->EvaluateAsInt(IntVal, *Context)) {
+              if (IntVal.getBoolValue() == false) {
+                return TryResult(false);
+              }
+            }
+            if (Bop->getRHS()->EvaluateAsInt(IntVal, *Context)) {
+              if (IntVal.getBoolValue() == false) {
+                return TryResult(false);
+              }
+            }
+          }
+          break;
+        }
+      }
     }
 
     return evaluateAsBooleanConditionNoCache(S);
@@ -682,7 +706,7 @@ CFGBlock *CFGBuilder::addInitializer(CXXCtorInitializer *I) {
       IsReference = FD->getType()->isReferenceType();
     HasTemporaries = isa<ExprWithCleanups>(Init);
 
-    if (BuildOpts.AddImplicitDtors && HasTemporaries) {
+    if (BuildOpts.AddTemporaryDtors && HasTemporaries) {
       // Generate destructors for temporaries in initialization expression.
       VisitForTemporaryDtors(cast<ExprWithCleanups>(Init)->getSubExpr(),
           IsReference);
@@ -1022,6 +1046,14 @@ CFGBlock *CFGBuilder::Visit(Stmt * S, AddStmtChoice asc) {
     case Stmt::ExprWithCleanupsClass:
       return VisitExprWithCleanups(cast<ExprWithCleanups>(S), asc);
 
+    case Stmt::CXXDefaultArgExprClass:
+      // FIXME: The expression inside a CXXDefaultArgExpr is owned by the
+      // called function's declaration, not by the caller. If we simply add
+      // this expression to the CFG, we could end up with the same Expr
+      // appearing multiple times.
+      // PR13385 / <rdar://problem/12156507>
+      return VisitStmt(S, asc);
+
     case Stmt::CXXBindTemporaryExprClass:
       return VisitCXXBindTemporaryExpr(cast<CXXBindTemporaryExpr>(S), asc);
 
@@ -1585,7 +1617,7 @@ CFGBlock *CFGBuilder::VisitDeclSubExpr(DeclStmt *DS) {
     IsReference = VD->getType()->isReferenceType();
     HasTemporaries = isa<ExprWithCleanups>(Init);
 
-    if (BuildOpts.AddImplicitDtors && HasTemporaries) {
+    if (BuildOpts.AddTemporaryDtors && HasTemporaries) {
       // Generate destructors for temporaries in initialization expression.
       VisitForTemporaryDtors(cast<ExprWithCleanups>(Init)->getSubExpr(),
           IsReference);
@@ -1616,8 +1648,10 @@ CFGBlock *CFGBuilder::VisitDeclSubExpr(DeclStmt *DS) {
 
   // If the type of VD is a VLA, then we must process its size expressions.
   for (const VariableArrayType* VA = FindVA(VD->getType().getTypePtr());
-       VA != 0; VA = FindVA(VA->getElementType().getTypePtr()))
-    Block = addStmt(VA->getSizeExpr());
+       VA != 0; VA = FindVA(VA->getElementType().getTypePtr())) {
+    if (CFGBlock *newBlock = addStmt(VA->getSizeExpr()))
+      LastBlock = newBlock;
+  }
 
   // Remove variable from local scope.
   if (ScopePos && VD == *ScopePos)
@@ -1735,7 +1769,7 @@ CFGBlock *CFGBuilder::VisitIfStmt(IfStmt *I) {
   // Add the condition as the last statement in the new block.  This may create
   // new blocks as the condition may contain control-flow.  Any newly created
   // blocks will be pointed to be "Block".
-  Block = addStmt(I->getCond());
+  CFGBlock *LastBlock = addStmt(I->getCond());
 
   // Finally, if the IfStmt contains a condition variable, add both the IfStmt
   // and the condition variable initialization to the CFG.
@@ -1743,11 +1777,11 @@ CFGBlock *CFGBuilder::VisitIfStmt(IfStmt *I) {
     if (Expr *Init = VD->getInit()) {
       autoCreateBlock();
       appendStmt(Block, I->getConditionVariableDeclStmt());
-      addStmt(Init);
+      LastBlock = addStmt(Init);
     }
   }
 
-  return Block;
+  return LastBlock;
 }
 
 
@@ -2254,7 +2288,7 @@ CFGBlock *CFGBuilder::VisitWhileStmt(WhileStmt *W) {
       }
 
     // The default case when not handling logical operators.
-    EntryConditionBlock = ExitConditionBlock = createBlock(false);
+    ExitConditionBlock = createBlock(false);
     ExitConditionBlock->setTerminator(W);
 
     // Now add the actual condition to the condition block.
@@ -2579,7 +2613,7 @@ CFGBlock *CFGBuilder::VisitSwitchStmt(SwitchStmt *Terminator) {
   // Add the terminator and condition in the switch block.
   SwitchTerminatedBlock->setTerminator(Terminator);
   Block = SwitchTerminatedBlock;
-  Block = addStmt(Terminator->getCond());
+  CFGBlock *LastBlock = addStmt(Terminator->getCond());
 
   // Finally, if the SwitchStmt contains a condition variable, add both the
   // SwitchStmt and the condition variable initialization to the CFG.
@@ -2587,11 +2621,11 @@ CFGBlock *CFGBuilder::VisitSwitchStmt(SwitchStmt *Terminator) {
     if (Expr *Init = VD->getInit()) {
       autoCreateBlock();
       appendStmt(Block, Terminator->getConditionVariableDeclStmt());
-      addStmt(Init);
+      LastBlock = addStmt(Init);
     }
   }
 
-  return Block;
+  return LastBlock;
 }
   
 static bool shouldAddCase(bool &switchExclusivelyCovered,
@@ -2775,8 +2809,7 @@ CFGBlock *CFGBuilder::VisitCXXTryStmt(CXXTryStmt *Terminator) {
 
   assert(Terminator->getTryBlock() && "try must contain a non-NULL body");
   Block = NULL;
-  Block = addStmt(Terminator->getTryBlock());
-  return Block;
+  return addStmt(Terminator->getTryBlock());
 }
 
 CFGBlock *CFGBuilder::VisitCXXCatchStmt(CXXCatchStmt *CS) {
@@ -2917,15 +2950,15 @@ CFGBlock *CFGBuilder::VisitCXXForRangeStmt(CXXForRangeStmt *S) {
     addLocalScopeAndDtors(S->getLoopVarStmt());
 
     // Populate a new block to contain the loop body and loop variable.
-    Block = addStmt(S->getBody());
+    addStmt(S->getBody());
     if (badCFG)
       return 0;
-    Block = addStmt(S->getLoopVarStmt());
+    CFGBlock *LoopVarStmtBlock = addStmt(S->getLoopVarStmt());
     if (badCFG)
       return 0;
     
     // This new body block is a successor to our condition block.
-    addSuccessor(ConditionBlock, KnownVal.isFalse() ? 0 : Block);
+    addSuccessor(ConditionBlock, KnownVal.isFalse() ? 0 : LoopVarStmtBlock);
   }
 
   // Link up the condition block with the code that follows the loop (the
@@ -2940,7 +2973,7 @@ CFGBlock *CFGBuilder::VisitCXXForRangeStmt(CXXForRangeStmt *S) {
 
 CFGBlock *CFGBuilder::VisitExprWithCleanups(ExprWithCleanups *E,
     AddStmtChoice asc) {
-  if (BuildOpts.AddImplicitDtors) {
+  if (BuildOpts.AddTemporaryDtors) {
     // If adding implicit destructors visit the full expression for adding
     // destructors of temporaries.
     VisitForTemporaryDtors(E->getSubExpr());
@@ -3020,6 +3053,8 @@ CFGBlock *CFGBuilder::VisitIndirectGotoStmt(IndirectGotoStmt *I) {
 }
 
 CFGBlock *CFGBuilder::VisitForTemporaryDtors(Stmt *E, bool BindToTemporary) {
+  assert(BuildOpts.AddImplicitDtors && BuildOpts.AddTemporaryDtors);
+
 tryAgain:
   if (!E) {
     badCFG = true;
@@ -3449,12 +3484,12 @@ class StmtPrinterHelper : public PrinterHelper  {
   StmtMapTy StmtMap;
   DeclMapTy DeclMap;
   signed currentBlock;
-  unsigned currentStmt;
+  unsigned currStmt;
   const LangOptions &LangOpts;
 public:
 
   StmtPrinterHelper(const CFG* cfg, const LangOptions &LO)
-    : currentBlock(0), currentStmt(0), LangOpts(LO)
+    : currentBlock(0), currStmt(0), LangOpts(LO)
   {
     for (CFG::const_iterator I = cfg->begin(), E = cfg->end(); I != E; ++I ) {
       unsigned j = 1;
@@ -3515,7 +3550,7 @@ public:
 
   const LangOptions &getLangOpts() const { return LangOpts; }
   void setBlockID(signed i) { currentBlock = i; }
-  void setStmtID(unsigned i) { currentStmt = i; }
+  void setStmtID(unsigned i) { currStmt = i; }
 
   virtual bool handledStmt(Stmt *S, raw_ostream &OS) {
     StmtMapTy::iterator I = StmtMap.find(S);
@@ -3524,7 +3559,7 @@ public:
       return false;
 
     if (currentBlock >= 0 && I->second.first == (unsigned) currentBlock
-                          && I->second.second == currentStmt) {
+                          && I->second.second == currStmt) {
       return false;
     }
 
@@ -3539,7 +3574,7 @@ public:
       return false;
 
     if (currentBlock >= 0 && I->second.first == (unsigned) currentBlock
-                          && I->second.second == currentStmt) {
+                          && I->second.second == currStmt) {
       return false;
     }
 
@@ -3831,8 +3866,8 @@ static void print_block(raw_ostream &OS, const CFG* cfg,
 
     if (Helper) Helper->setBlockID(-1);
 
-    CFGBlockTerminatorPrint TPrinter(OS, Helper,
-                                     PrintingPolicy(Helper->getLangOpts()));
+    PrintingPolicy PP(Helper ? Helper->getLangOpts() : LangOptions());
+    CFGBlockTerminatorPrint TPrinter(OS, Helper, PP);
     TPrinter.Visit(const_cast<Stmt*>(B.getTerminator().getStmt()));
     OS << '\n';
     
diff --git a/lib/Analysis/CMakeLists.txt b/lib/Analysis/CMakeLists.txt
index d57e481..ca16666 100644
--- a/lib/Analysis/CMakeLists.txt
+++ b/lib/Analysis/CMakeLists.txt
@@ -1,13 +1,15 @@
 add_clang_library(clangAnalysis
   AnalysisDeclContext.cpp
-  CallGraph.cpp
+  BodyFarm.cpp
   CFG.cpp
   CFGReachabilityAnalysis.cpp
   CFGStmtMap.cpp
+  CallGraph.cpp
   CocoaConventions.cpp
   Dominators.cpp
   FormatString.cpp
   LiveVariables.cpp
+  ObjCNoReturn.cpp
   PostOrderCFGView.cpp
   PrintfFormatString.cpp
   ProgramPoint.cpp
diff --git a/lib/Analysis/FormatString.cpp b/lib/Analysis/FormatString.cpp
index e7ea486..73063b5 100644
--- a/lib/Analysis/FormatString.cpp
+++ b/lib/Analysis/FormatString.cpp
@@ -14,6 +14,7 @@
 
 #include "FormatStringParsing.h"
 #include "clang/Basic/LangOptions.h"
+#include "clang/Basic/TargetInfo.h"
 
 using clang::analyze_format_string::ArgType;
 using clang::analyze_format_string::FormatStringHandler;
@@ -489,9 +490,12 @@ analyze_format_string::LengthModifier::toString() const {
 const char *ConversionSpecifier::toString() const {
   switch (kind) {
   case dArg: return "d";
+  case DArg: return "D";
   case iArg: return "i";
   case oArg: return "o";
+  case OArg: return "O";
   case uArg: return "u";
+  case UArg: return "U";
   case xArg: return "x";
   case XArg: return "X";
   case fArg: return "f";
@@ -523,6 +527,29 @@ const char *ConversionSpecifier::toString() const {
   return NULL;
 }
 
+llvm::Optional<ConversionSpecifier>
+ConversionSpecifier::getStandardSpecifier() const {
+  ConversionSpecifier::Kind NewKind;
+  
+  switch (getKind()) {
+  default:
+    return llvm::Optional<ConversionSpecifier>();
+  case DArg:
+    NewKind = dArg;
+    break;
+  case UArg:
+    NewKind = uArg;
+    break;
+  case OArg:
+    NewKind = oArg;
+    break;
+  }
+
+  ConversionSpecifier FixedCS(*this);
+  FixedCS.setKind(NewKind);
+  return FixedCS;
+}
+
 //===----------------------------------------------------------------------===//
 // Methods on OptionalAmount.
 //===----------------------------------------------------------------------===//
@@ -548,7 +575,7 @@ void OptionalAmount::toString(raw_ostream &os) const {
   }
 }
 
-bool FormatSpecifier::hasValidLengthModifier() const {
+bool FormatSpecifier::hasValidLengthModifier(const TargetInfo &Target) const {
   switch (LM.getKind()) {
     case LengthModifier::None:
       return true;
@@ -563,9 +590,12 @@ bool FormatSpecifier::hasValidLengthModifier() const {
     case LengthModifier::AsPtrDiff:
       switch (CS.getKind()) {
         case ConversionSpecifier::dArg:
+        case ConversionSpecifier::DArg:
         case ConversionSpecifier::iArg:
         case ConversionSpecifier::oArg:
+        case ConversionSpecifier::OArg:
         case ConversionSpecifier::uArg:
+        case ConversionSpecifier::UArg:
         case ConversionSpecifier::xArg:
         case ConversionSpecifier::XArg:
         case ConversionSpecifier::nArg:
@@ -578,9 +608,12 @@ bool FormatSpecifier::hasValidLengthModifier() const {
     case LengthModifier::AsLong:
       switch (CS.getKind()) {
         case ConversionSpecifier::dArg:
+        case ConversionSpecifier::DArg:
         case ConversionSpecifier::iArg:
         case ConversionSpecifier::oArg:
+        case ConversionSpecifier::OArg:
         case ConversionSpecifier::uArg:
+        case ConversionSpecifier::UArg:
         case ConversionSpecifier::xArg:
         case ConversionSpecifier::XArg:
         case ConversionSpecifier::aArg:
@@ -611,14 +644,15 @@ bool FormatSpecifier::hasValidLengthModifier() const {
         case ConversionSpecifier::gArg:
         case ConversionSpecifier::GArg:
           return true;
-        // GNU extension.
+        // GNU libc extension.
         case ConversionSpecifier::dArg:
         case ConversionSpecifier::iArg:
         case ConversionSpecifier::oArg:
         case ConversionSpecifier::uArg:
         case ConversionSpecifier::xArg:
         case ConversionSpecifier::XArg:
-          return true;
+          return !Target.getTriple().isOSDarwin() &&
+                 !Target.getTriple().isOSWindows();
         default:
           return false;
       }
@@ -697,6 +731,9 @@ bool FormatSpecifier::hasStandardConversionSpecifier(const LangOptions &LangOpt)
       return LangOpt.ObjC1 || LangOpt.ObjC2;
     case ConversionSpecifier::InvalidSpecifier:
     case ConversionSpecifier::PrintErrno:
+    case ConversionSpecifier::DArg:
+    case ConversionSpecifier::OArg:
+    case ConversionSpecifier::UArg:
       return false;
   }
   llvm_unreachable("Invalid ConversionSpecifier Kind!");
@@ -719,6 +756,20 @@ bool FormatSpecifier::hasStandardLengthConversionCombination() const {
   return true;
 }
 
+llvm::Optional<LengthModifier>
+FormatSpecifier::getCorrectedLengthModifier() const {
+  if (CS.isAnyIntArg() || CS.getKind() == ConversionSpecifier::nArg) {
+    if (LM.getKind() == LengthModifier::AsLongDouble ||
+        LM.getKind() == LengthModifier::AsQuad) {
+      LengthModifier FixedLM(LM);
+      FixedLM.setKind(LengthModifier::AsLongLong);
+      return FixedLM;
+    }
+  }
+
+  return llvm::Optional<LengthModifier>();
+}
+
 bool FormatSpecifier::namedTypeToLengthModifier(QualType QT,
                                                 LengthModifier &LM) {
   assert(isa<TypedefType>(QT) && "Expected a TypedefType");
diff --git a/lib/Analysis/ObjCNoReturn.cpp b/lib/Analysis/ObjCNoReturn.cpp
new file mode 100644
index 0000000..52d844b
--- /dev/null
+++ b/lib/Analysis/ObjCNoReturn.cpp
@@ -0,0 +1,67 @@
+//= ObjCNoReturn.cpp - Handling of Cocoa APIs known not to return --*- C++ -*---
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements special handling of recognizing ObjC API hooks that
+// do not return but aren't marked as such in API headers.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/ExprObjC.h"
+#include "clang/Analysis/DomainSpecific/ObjCNoReturn.h"
+
+using namespace clang;
+
+static bool isSubclass(const ObjCInterfaceDecl *Class, IdentifierInfo *II) {
+  if (!Class)
+    return false;
+  if (Class->getIdentifier() == II)
+    return true;
+  return isSubclass(Class->getSuperClass(), II);
+}
+
+ObjCNoReturn::ObjCNoReturn(ASTContext &C)
+  : RaiseSel(GetNullarySelector("raise", C)),
+    NSExceptionII(&C.Idents.get("NSException"))
+{
+  // Generate selectors.
+  SmallVector<IdentifierInfo*, 3> II;
+  
+  // raise:format:
+  II.push_back(&C.Idents.get("raise"));
+  II.push_back(&C.Idents.get("format"));
+  NSExceptionInstanceRaiseSelectors[0] =
+    C.Selectors.getSelector(II.size(), &II[0]);
+    
+  // raise:format:arguments:
+  II.push_back(&C.Idents.get("arguments"));
+  NSExceptionInstanceRaiseSelectors[1] =
+    C.Selectors.getSelector(II.size(), &II[0]);
+}
+
+
+bool ObjCNoReturn::isImplicitNoReturn(const ObjCMessageExpr *ME) {
+  Selector S = ME->getSelector();
+  
+  if (ME->isInstanceMessage()) {
+    // Check for the "raise" message.
+    return S == RaiseSel;
+  }
+
+  if (const ObjCInterfaceDecl *ID = ME->getReceiverInterface()) {
+    if (isSubclass(ID, NSExceptionII)) {
+      for (unsigned i = 0; i < NUM_RAISE_SELECTORS; ++i) {
+        if (S == NSExceptionInstanceRaiseSelectors[i])
+          return true;
+      }
+    }
+  }
+  
+  return false;
+}
diff --git a/lib/Analysis/PrintfFormatString.cpp b/lib/Analysis/PrintfFormatString.cpp
index 9e4c0fe..2fa5a88 100644
--- a/lib/Analysis/PrintfFormatString.cpp
+++ b/lib/Analysis/PrintfFormatString.cpp
@@ -13,6 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Analysis/Analyses/FormatString.h"
+#include "clang/Basic/TargetInfo.h"
 #include "FormatStringParsing.h"
 
 using clang::analyze_format_string::ArgType;
@@ -52,7 +53,8 @@ static PrintfSpecifierResult ParsePrintfSpecifier(FormatStringHandler &H,
                                                   const char *&Beg,
                                                   const char *E,
                                                   unsigned &argIndex,
-                                                  const LangOptions &LO) {
+                                                  const LangOptions &LO,
+                                                  const TargetInfo &Target) {
 
   using namespace clang::analyze_format_string;
   using namespace clang::analyze_printf;
@@ -196,6 +198,19 @@ static PrintfSpecifierResult ParsePrintfSpecifier(FormatStringHandler &H,
     case '@': k = ConversionSpecifier::ObjCObjArg; break;
     // Glibc specific.
     case 'm': k = ConversionSpecifier::PrintErrno; break;
+    // Apple-specific
+    case 'D':
+      if (Target.getTriple().isOSDarwin())
+        k = ConversionSpecifier::DArg;
+      break;
+    case 'O':
+      if (Target.getTriple().isOSDarwin())
+        k = ConversionSpecifier::OArg;
+      break;
+    case 'U':
+      if (Target.getTriple().isOSDarwin())
+        k = ConversionSpecifier::UArg;
+      break;
   }
   PrintfConversionSpecifier CS(conversionPosition, k);
   FS.setConversionSpecifier(CS);
@@ -212,18 +227,19 @@ static PrintfSpecifierResult ParsePrintfSpecifier(FormatStringHandler &H,
 bool clang::analyze_format_string::ParsePrintfString(FormatStringHandler &H,
                                                      const char *I,
                                                      const char *E,
-                                                     const LangOptions &LO) {
+                                                     const LangOptions &LO,
+                                                     const TargetInfo &Target) {
 
   unsigned argIndex = 0;
 
   // Keep looking for a format specifier until we have exhausted the string.
   while (I != E) {
     const PrintfSpecifierResult &FSR = ParsePrintfSpecifier(H, I, E, argIndex,
-                                                            LO);
+                                                            LO, Target);
     // Did a fail-stop error of any kind occur when parsing the specifier?
     // If so, don't do any more processing.
     if (FSR.shouldStop())
-      return true;;
+      return true;
     // Did we exhaust the string or encounter an error that
     // we can recover from?
     if (!FSR.hasValue())
@@ -482,9 +498,11 @@ bool PrintfSpecifier::fixType(QualType QT, const LangOptions &LangOpt,
     namedTypeToLengthModifier(QT, LM);
 
   // If fixing the length modifier was enough, we are done.
-  const analyze_printf::ArgType &ATR = getArgType(Ctx, IsObjCLiteral);
-  if (hasValidLengthModifier() && ATR.isValid() && ATR.matchesType(Ctx, QT))
-    return true;
+  if (hasValidLengthModifier(Ctx.getTargetInfo())) {
+    const analyze_printf::ArgType &ATR = getArgType(Ctx, IsObjCLiteral);
+    if (ATR.isValid() && ATR.matchesType(Ctx, QT))
+      return true;
+  }
 
   // Set conversion specifier and disable any flags which do not apply to it.
   // Let typedefs to char fall through to int, as %c is silly for uint8_t.
@@ -549,6 +567,7 @@ bool PrintfSpecifier::hasValidPlusPrefix() const {
   // The plus prefix only makes sense for signed conversions
   switch (CS.getKind()) {
   case ConversionSpecifier::dArg:
+  case ConversionSpecifier::DArg:
   case ConversionSpecifier::iArg:
   case ConversionSpecifier::fArg:
   case ConversionSpecifier::FArg:
@@ -572,6 +591,7 @@ bool PrintfSpecifier::hasValidAlternativeForm() const {
   // Alternate form flag only valid with the oxXaAeEfFgG conversions
   switch (CS.getKind()) {
   case ConversionSpecifier::oArg:
+  case ConversionSpecifier::OArg:
   case ConversionSpecifier::xArg:
   case ConversionSpecifier::XArg:
   case ConversionSpecifier::aArg:
@@ -596,9 +616,12 @@ bool PrintfSpecifier::hasValidLeadingZeros() const {
   // Leading zeroes flag only valid with the diouxXaAeEfFgG conversions
   switch (CS.getKind()) {
   case ConversionSpecifier::dArg:
+  case ConversionSpecifier::DArg:
   case ConversionSpecifier::iArg:
   case ConversionSpecifier::oArg:
+  case ConversionSpecifier::OArg:
   case ConversionSpecifier::uArg:
+  case ConversionSpecifier::UArg:
   case ConversionSpecifier::xArg:
   case ConversionSpecifier::XArg:
   case ConversionSpecifier::aArg:
@@ -623,6 +646,7 @@ bool PrintfSpecifier::hasValidSpacePrefix() const {
   // The space prefix only makes sense for signed conversions
   switch (CS.getKind()) {
   case ConversionSpecifier::dArg:
+  case ConversionSpecifier::DArg:
   case ConversionSpecifier::iArg:
   case ConversionSpecifier::fArg:
   case ConversionSpecifier::FArg:
@@ -659,8 +683,10 @@ bool PrintfSpecifier::hasValidThousandsGroupingPrefix() const {
 
   switch (CS.getKind()) {
     case ConversionSpecifier::dArg:
+    case ConversionSpecifier::DArg:
     case ConversionSpecifier::iArg:
     case ConversionSpecifier::uArg:
+    case ConversionSpecifier::UArg:
     case ConversionSpecifier::fArg:
     case ConversionSpecifier::FArg:
     case ConversionSpecifier::gArg:
@@ -678,9 +704,12 @@ bool PrintfSpecifier::hasValidPrecision() const {
   // Precision is only valid with the diouxXaAeEfFgGs conversions
   switch (CS.getKind()) {
   case ConversionSpecifier::dArg:
+  case ConversionSpecifier::DArg:
   case ConversionSpecifier::iArg:
   case ConversionSpecifier::oArg:
+  case ConversionSpecifier::OArg:
   case ConversionSpecifier::uArg:
+  case ConversionSpecifier::UArg:
   case ConversionSpecifier::xArg:
   case ConversionSpecifier::XArg:
   case ConversionSpecifier::aArg:
diff --git a/lib/Analysis/ReachableCode.cpp b/lib/Analysis/ReachableCode.cpp
index bb63e2c..11f2ebe 100644
--- a/lib/Analysis/ReachableCode.cpp
+++ b/lib/Analysis/ReachableCode.cpp
@@ -112,8 +112,8 @@ const Stmt *DeadCodeScan::findDeadCode(const clang::CFGBlock *Block) {
 
 static int SrcCmp(const void *p1, const void *p2) {
   return
-    ((std::pair<const CFGBlock *, const Stmt *>*) p2)->second->getLocStart() <
-    ((std::pair<const CFGBlock *, const Stmt *>*) p1)->second->getLocStart();
+    ((const std::pair<const CFGBlock *, const Stmt *>*) p2)->second->getLocStart() <
+    ((const std::pair<const CFGBlock *, const Stmt *>*) p1)->second->getLocStart();
 }
 
 unsigned DeadCodeScan::scanBackwards(const clang::CFGBlock *Start,
diff --git a/lib/Analysis/ScanfFormatString.cpp b/lib/Analysis/ScanfFormatString.cpp
index 2942400..574e56a 100644
--- a/lib/Analysis/ScanfFormatString.cpp
+++ b/lib/Analysis/ScanfFormatString.cpp
@@ -13,6 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Analysis/Analyses/FormatString.h"
+#include "clang/Basic/TargetInfo.h"
 #include "FormatStringParsing.h"
 
 using clang::analyze_format_string::ArgType;
@@ -67,7 +68,8 @@ static ScanfSpecifierResult ParseScanfSpecifier(FormatStringHandler &H,
                                                 const char *&Beg,
                                                 const char *E,
                                                 unsigned &argIndex,
-                                                const LangOptions &LO) {
+                                                const LangOptions &LO,
+                                                const TargetInfo &Target) {
   
   using namespace clang::analyze_scanf;
   const char *I = Beg;
@@ -172,6 +174,20 @@ static ScanfSpecifierResult ParseScanfSpecifier(FormatStringHandler &H,
     case 'o': k = ConversionSpecifier::oArg; break;
     case 's': k = ConversionSpecifier::sArg; break;
     case 'p': k = ConversionSpecifier::pArg; break;
+    // Apple extensions
+      // Apple-specific
+    case 'D':
+      if (Target.getTriple().isOSDarwin())
+        k = ConversionSpecifier::DArg;
+      break;
+    case 'O':
+      if (Target.getTriple().isOSDarwin())
+        k = ConversionSpecifier::OArg;
+      break;
+    case 'U':
+      if (Target.getTriple().isOSDarwin())
+        k = ConversionSpecifier::UArg;
+      break;
   }
   ScanfConversionSpecifier CS(conversionPosition, k);
   if (k == ScanfConversionSpecifier::ScanListArg) {
@@ -202,6 +218,7 @@ ArgType ScanfSpecifier::getArgType(ASTContext &Ctx) const {
   switch(CS.getKind()) {
     // Signed int.
     case ConversionSpecifier::dArg:
+    case ConversionSpecifier::DArg:
     case ConversionSpecifier::iArg:
       switch (LM.getKind()) {
         case LengthModifier::None:
@@ -233,7 +250,9 @@ ArgType ScanfSpecifier::getArgType(ASTContext &Ctx) const {
 
     // Unsigned int.
     case ConversionSpecifier::oArg:
+    case ConversionSpecifier::OArg:
     case ConversionSpecifier::uArg:
+    case ConversionSpecifier::UArg:
     case ConversionSpecifier::xArg:
     case ConversionSpecifier::XArg:
       switch (LM.getKind()) {
@@ -430,9 +449,11 @@ bool ScanfSpecifier::fixType(QualType QT, const LangOptions &LangOpt,
     namedTypeToLengthModifier(PT, LM);
 
   // If fixing the length modifier was enough, we are done.
-  const analyze_scanf::ArgType &AT = getArgType(Ctx);
-  if (hasValidLengthModifier() && AT.isValid() && AT.matchesType(Ctx, QT))
-    return true;
+  if (hasValidLengthModifier(Ctx.getTargetInfo())) {
+    const analyze_scanf::ArgType &AT = getArgType(Ctx);
+    if (AT.isValid() && AT.matchesType(Ctx, QT))
+      return true;
+  }
 
   // Figure out the conversion specifier.
   if (PT->isRealFloatingType())
@@ -463,18 +484,19 @@ void ScanfSpecifier::toString(raw_ostream &os) const {
 bool clang::analyze_format_string::ParseScanfString(FormatStringHandler &H,
                                                     const char *I,
                                                     const char *E,
-                                                    const LangOptions &LO) {
+                                                    const LangOptions &LO,
+                                                    const TargetInfo &Target) {
   
   unsigned argIndex = 0;
   
   // Keep looking for a format specifier until we have exhausted the string.
   while (I != E) {
     const ScanfSpecifierResult &FSR = ParseScanfSpecifier(H, I, E, argIndex,
-                                                          LO);
+                                                          LO, Target);
     // Did a fail-stop error of any kind occur when parsing the specifier?
     // If so, don't do any more processing.
     if (FSR.shouldStop())
-      return true;;
+      return true;
       // Did we exhaust the string or encounter an error that
       // we can recover from?
     if (!FSR.hasValue())
diff --git a/lib/Analysis/ThreadSafety.cpp b/lib/Analysis/ThreadSafety.cpp
index 5954682..c7f1f62 100644
--- a/lib/Analysis/ThreadSafety.cpp
+++ b/lib/Analysis/ThreadSafety.cpp
@@ -70,27 +70,28 @@ namespace {
 class SExpr {
 private:
   enum ExprOp {
-    EOP_Nop,      //< No-op
-    EOP_Wildcard, //< Matches anything.
-    EOP_This,     //< This keyword.
-    EOP_NVar,     //< Named variable.
-    EOP_LVar,     //< Local variable.
-    EOP_Dot,      //< Field access
-    EOP_Call,     //< Function call
-    EOP_MCall,    //< Method call
-    EOP_Index,    //< Array index
-    EOP_Unary,    //< Unary operation
-    EOP_Binary,   //< Binary operation
-    EOP_Unknown   //< Catchall for everything else
+    EOP_Nop,       ///< No-op
+    EOP_Wildcard,  ///< Matches anything.
+    EOP_Universal, ///< Universal lock.
+    EOP_This,      ///< This keyword.
+    EOP_NVar,      ///< Named variable.
+    EOP_LVar,      ///< Local variable.
+    EOP_Dot,       ///< Field access
+    EOP_Call,      ///< Function call
+    EOP_MCall,     ///< Method call
+    EOP_Index,     ///< Array index
+    EOP_Unary,     ///< Unary operation
+    EOP_Binary,    ///< Binary operation
+    EOP_Unknown    ///< Catchall for everything else
   };
 
 
   class SExprNode {
    private:
-    unsigned char  Op;     //< Opcode of the root node
-    unsigned char  Flags;  //< Additional opcode-specific data
-    unsigned short Sz;     //< Number of child nodes
-    const void*    Data;   //< Additional opcode-specific data
+    unsigned char  Op;     ///< Opcode of the root node
+    unsigned char  Flags;  ///< Additional opcode-specific data
+    unsigned short Sz;     ///< Number of child nodes
+    const void*    Data;   ///< Additional opcode-specific data
 
    public:
     SExprNode(ExprOp O, unsigned F, const void* D)
@@ -118,18 +119,19 @@ private:
 
     unsigned arity() const {
       switch (Op) {
-        case EOP_Nop:      return 0;
-        case EOP_Wildcard: return 0;
-        case EOP_NVar:     return 0;
-        case EOP_LVar:     return 0;
-        case EOP_This:     return 0;
-        case EOP_Dot:      return 1;
-        case EOP_Call:     return Flags+1;  // First arg is function.
-        case EOP_MCall:    return Flags+1;  // First arg is implicit obj.
-        case EOP_Index:    return 2;
-        case EOP_Unary:    return 1;
-        case EOP_Binary:   return 2;
-        case EOP_Unknown:  return Flags;
+        case EOP_Nop:       return 0;
+        case EOP_Wildcard:  return 0;
+        case EOP_Universal: return 0;
+        case EOP_NVar:      return 0;
+        case EOP_LVar:      return 0;
+        case EOP_This:      return 0;
+        case EOP_Dot:       return 1;
+        case EOP_Call:      return Flags+1;  // First arg is function.
+        case EOP_MCall:     return Flags+1;  // First arg is implicit obj.
+        case EOP_Index:     return 2;
+        case EOP_Unary:     return 1;
+        case EOP_Binary:    return 2;
+        case EOP_Unknown:   return Flags;
       }
       return 0;
     }
@@ -194,6 +196,11 @@ private:
     return NodeVec.size()-1;
   }
 
+  unsigned makeUniversal() {
+    NodeVec.push_back(SExprNode(EOP_Universal, 0, 0));
+    return NodeVec.size()-1;
+  }
+
   unsigned makeNamedVar(const NamedDecl *D) {
     NodeVec.push_back(SExprNode(EOP_NVar, 0, D));
     return NodeVec.size()-1;
@@ -219,8 +226,21 @@ private:
     return NodeVec.size()-1;
   }
 
-  unsigned makeMCall(unsigned NumArgs, const NamedDecl *D) {
-    NodeVec.push_back(SExprNode(EOP_MCall, NumArgs, D));
+  // Grab the very first declaration of virtual method D
+  const CXXMethodDecl* getFirstVirtualDecl(const CXXMethodDecl *D) {
+    while (true) {
+      D = D->getCanonicalDecl();
+      CXXMethodDecl::method_iterator I = D->begin_overridden_methods(),
+                                     E = D->end_overridden_methods();
+      if (I == E)
+        return D;  // Method does not override anything
+      D = *I;      // FIXME: this does not work with multiple inheritance.
+    }
+    return 0;
+  }
+
+  unsigned makeMCall(unsigned NumArgs, const CXXMethodDecl *D) {
+    NodeVec.push_back(SExprNode(EOP_MCall, NumArgs, getFirstVirtualDecl(D)));
     return NodeVec.size()-1;
   }
 
@@ -300,8 +320,9 @@ private:
     } else if (CXXMemberCallExpr *CMCE = dyn_cast<CXXMemberCallExpr>(Exp)) {
       // When calling a function with a lock_returned attribute, replace
       // the function call with the expression in lock_returned.
-      if (LockReturnedAttr* At =
-            CMCE->getMethodDecl()->getAttr<LockReturnedAttr>()) {
+      CXXMethodDecl* MD =
+        cast<CXXMethodDecl>(CMCE->getMethodDecl()->getMostRecentDecl());
+      if (LockReturnedAttr* At = MD->getAttr<LockReturnedAttr>()) {
         CallingContext LRCallCtx(CMCE->getMethodDecl());
         LRCallCtx.SelfArg = CMCE->getImplicitObjectArgument();
         LRCallCtx.SelfArrow =
@@ -320,8 +341,7 @@ private:
         return buildSExpr(CMCE->getImplicitObjectArgument(), CallCtx, NDeref);
       }
       unsigned NumCallArgs = CMCE->getNumArgs();
-      unsigned Root =
-        makeMCall(NumCallArgs, CMCE->getMethodDecl()->getCanonicalDecl());
+      unsigned Root = makeMCall(NumCallArgs, CMCE->getMethodDecl());
       unsigned Sz = buildSExpr(CMCE->getImplicitObjectArgument(), CallCtx);
       Expr** CallArgs = CMCE->getArgs();
       for (unsigned i = 0; i < NumCallArgs; ++i) {
@@ -330,8 +350,9 @@ private:
       NodeVec[Root].setSize(Sz + 1);
       return Sz + 1;
     } else if (CallExpr *CE = dyn_cast<CallExpr>(Exp)) {
-      if (LockReturnedAttr* At =
-            CE->getDirectCallee()->getAttr<LockReturnedAttr>()) {
+      FunctionDecl* FD =
+        cast<FunctionDecl>(CE->getDirectCallee()->getMostRecentDecl());
+      if (LockReturnedAttr* At = FD->getAttr<LockReturnedAttr>()) {
         CallingContext LRCallCtx(CE->getDirectCallee());
         LRCallCtx.NumArgs = CE->getNumArgs();
         LRCallCtx.FunArgs = CE->getArgs();
@@ -442,9 +463,23 @@ private:
   /// \param DeclExp An expression involving the Decl on which the attribute
   ///        occurs.
   /// \param D  The declaration to which the lock/unlock attribute is attached.
-  void buildSExprFromExpr(Expr *MutexExp, Expr *DeclExp, const NamedDecl *D) {
+  void buildSExprFromExpr(Expr *MutexExp, Expr *DeclExp, const NamedDecl *D,
+                          VarDecl *SelfDecl = 0) {
     CallingContext CallCtx(D);
 
+    if (MutexExp) {
+      if (StringLiteral* SLit = dyn_cast<StringLiteral>(MutexExp)) {
+        if (SLit->getString() == StringRef("*"))
+          // The "*" expr is a universal lock, which essentially turns off
+          // checks until it is removed from the lockset.
+          makeUniversal();
+        else
+          // Ignore other string literals for now.
+          makeNop();
+        return;
+      }
+    }
+
     // If we are processing a raw attribute expression, with no substitutions.
     if (DeclExp == 0) {
       buildSExpr(MutexExp, 0);
@@ -465,7 +500,7 @@ private:
       CallCtx.NumArgs = CE->getNumArgs();
       CallCtx.FunArgs = CE->getArgs();
     } else if (CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(DeclExp)) {
-      CallCtx.SelfArg = 0;  // FIXME -- get the parent from DeclStmt
+      CallCtx.SelfArg = 0;  // Will be set below
       CallCtx.NumArgs = CE->getNumArgs();
       CallCtx.FunArgs = CE->getArgs();
     } else if (D && isa<CXXDestructorDecl>(D)) {
@@ -473,14 +508,26 @@ private:
       CallCtx.SelfArg = DeclExp;
     }
 
-    // If the attribute has no arguments, then assume the argument is "this".
-    if (MutexExp == 0) {
-      buildSExpr(CallCtx.SelfArg, 0);
+    // Hack to handle constructors, where self cannot be recovered from
+    // the expression.
+    if (SelfDecl && !CallCtx.SelfArg) {
+      DeclRefExpr SelfDRE(SelfDecl, false, SelfDecl->getType(), VK_LValue,
+                          SelfDecl->getLocation());
+      CallCtx.SelfArg = &SelfDRE;
+
+      // If the attribute has no arguments, then assume the argument is "this".
+      if (MutexExp == 0)
+        buildSExpr(CallCtx.SelfArg, 0);
+      else  // For most attributes.
+        buildSExpr(MutexExp, &CallCtx);
       return;
     }
 
-    // For most attributes.
-    buildSExpr(MutexExp, &CallCtx);
+    // If the attribute has no arguments, then assume the argument is "this".
+    if (MutexExp == 0)
+      buildSExpr(CallCtx.SelfArg, 0);
+    else  // For most attributes.
+      buildSExpr(MutexExp, &CallCtx);
   }
 
   /// \brief Get index of next sibling of node i.
@@ -496,8 +543,9 @@ public:
   ///        occurs.
   /// \param D  The declaration to which the lock/unlock attribute is attached.
   /// Caller must check isValid() after construction.
-  SExpr(Expr* MutexExp, Expr *DeclExp, const NamedDecl* D) {
-    buildSExprFromExpr(MutexExp, DeclExp, D);
+  SExpr(Expr* MutexExp, Expr *DeclExp, const NamedDecl* D,
+        VarDecl *SelfDecl=0) {
+    buildSExprFromExpr(MutexExp, DeclExp, D, SelfDecl);
   }
 
   /// Return true if this is a valid decl sequence.
@@ -506,6 +554,17 @@ public:
     return !NodeVec.empty();
   }
 
+  bool shouldIgnore() const {
+    // Nop is a mutex that we have decided to deliberately ignore.
+    assert(NodeVec.size() > 0 && "Invalid Mutex");
+    return NodeVec[0].kind() == EOP_Nop;
+  }
+
+  bool isUniversal() const {
+    assert(NodeVec.size() > 0 && "Invalid Mutex");
+    return NodeVec[0].kind() == EOP_Universal;
+  }
+
   /// Issue a warning about an invalid lock expression
   static void warnInvalidLock(ThreadSafetyHandler &Handler, Expr* MutexExp,
                               Expr *DeclExp, const NamedDecl* D) {
@@ -528,7 +587,9 @@ public:
 
   bool matches(const SExpr &Other, unsigned i = 0, unsigned j = 0) const {
     if (NodeVec[i].matches(Other.NodeVec[j])) {
-      unsigned n = NodeVec[i].arity();
+      unsigned ni = NodeVec[i].arity();
+      unsigned nj = Other.NodeVec[j].arity();
+      unsigned n = (ni < nj) ? ni : nj;
       bool Result = true;
       unsigned ci = i+1;  // first child of i
       unsigned cj = j+1;  // first child of j
@@ -541,6 +602,15 @@ public:
     return false;
   }
 
+  // A partial match between a.mu and b.mu returns true a and b have the same
+  // type (and thus mu refers to the same mutex declaration), regardless of
+  // whether a and b are different objects or not.
+  bool partiallyMatches(const SExpr &Other) const {
+    if (NodeVec[0].kind() == EOP_Dot)
+      return NodeVec[0].matches(Other.NodeVec[0]);
+    return false;
+  }
+
   /// \brief Pretty print a lock expression for use in error messages.
   std::string toString(unsigned i = 0) const {
     assert(isValid());
@@ -553,6 +623,8 @@ public:
         return "_";
       case EOP_Wildcard:
         return "(?)";
+      case EOP_Universal:
+        return "*";
       case EOP_This:
         return "this";
       case EOP_NVar:
@@ -695,6 +767,10 @@ struct LockData {
     ID.AddInteger(AcquireLoc.getRawEncoding());
     ID.AddInteger(LKind);
   }
+
+  bool isAtLeast(LockKind LK) {
+    return (LK == LK_Shared) || (LKind == LK_Exclusive);
+  }
 };
 
 
@@ -780,9 +856,28 @@ public:
     return false;
   }
 
-  LockData* findLock(FactManager& FM, const SExpr& M) const {
+  LockData* findLock(FactManager &FM, const SExpr &M) const {
+    for (const_iterator I = begin(), E = end(); I != E; ++I) {
+      const SExpr &Exp = FM[*I].MutID;
+      if (Exp.matches(M))
+        return &FM[*I].LDat;
+    }
+    return 0;
+  }
+
+  LockData* findLockUniv(FactManager &FM, const SExpr &M) const {
+    for (const_iterator I = begin(), E = end(); I != E; ++I) {
+      const SExpr &Exp = FM[*I].MutID;
+      if (Exp.matches(M) || Exp.isUniversal())
+        return &FM[*I].LDat;
+    }
+    return 0;
+  }
+
+  FactEntry* findPartialMatch(FactManager &FM, const SExpr &M) const {
     for (const_iterator I=begin(), E=end(); I != E; ++I) {
-      if (FM[*I].MutID.matches(M)) return &FM[*I].LDat;
+      const SExpr& Exp = FM[*I].MutID;
+      if (Exp.partiallyMatches(M)) return &FM[*I];
     }
     return 0;
   }
@@ -811,6 +906,7 @@ struct CFGBlockInfo {
   SourceLocation EntryLoc;      // Location of first statement in block
   SourceLocation ExitLoc;       // Location of last statement in block.
   unsigned EntryIndex;          // Used to replay contexts later
+  bool Reachable;               // Is this block reachable?
 
   const FactSet &getSet(CFGBlockSide Side) const {
     return Side == CBS_Entry ? EntrySet : ExitSet;
@@ -821,7 +917,7 @@ struct CFGBlockInfo {
 
 private:
   CFGBlockInfo(LocalVarContext EmptyCtx)
-    : EntryContext(EmptyCtx), ExitContext(EmptyCtx)
+    : EntryContext(EmptyCtx), ExitContext(EmptyCtx), Reachable(false)
   { }
 
 public:
@@ -939,7 +1035,7 @@ public:
       return;
     }
     Dec->printName(llvm::errs());
-    llvm::errs() << "." << i << " " << ((void*) Dec);
+    llvm::errs() << "." << i << " " << ((const void*) Dec);
   }
 
   /// Dumps an ASCII representation of the variable map to llvm::errs()
@@ -1339,7 +1435,7 @@ public:
 
   template <typename AttrType>
   void getMutexIDs(MutexIDList &Mtxs, AttrType *Attr, Expr *Exp,
-                   const NamedDecl *D);
+                   const NamedDecl *D, VarDecl *SelfDecl=0);
 
   template <class AttrType>
   void getMutexIDs(MutexIDList &Mtxs, AttrType *Attr, Expr *Exp,
@@ -1376,6 +1472,9 @@ void ThreadSafetyAnalyzer::addLock(FactSet &FSet, const SExpr &Mutex,
                                    const LockData &LDat) {
   // FIXME: deal with acquired before/after annotations.
   // FIXME: Don't always warn when we have support for reentrant locks.
+  if (Mutex.shouldIgnore())
+    return;
+
   if (FSet.findLock(FactMan, Mutex)) {
     Handler.handleDoubleLock(Mutex.toString(), LDat.AcquireLoc);
   } else {
@@ -1385,12 +1484,15 @@ void ThreadSafetyAnalyzer::addLock(FactSet &FSet, const SExpr &Mutex,
 
 
 /// \brief Remove a lock from the lockset, warning if the lock is not there.
-/// \param LockExp The lock expression corresponding to the lock to be removed
+/// \param Mutex The lock expression corresponding to the lock to be removed
 /// \param UnlockLoc The source location of the unlock (only used in error msg)
 void ThreadSafetyAnalyzer::removeLock(FactSet &FSet,
                                       const SExpr &Mutex,
                                       SourceLocation UnlockLoc,
                                       bool FullyRemove) {
+  if (Mutex.shouldIgnore())
+    return;
+
   const LockData *LDat = FSet.findLock(FactMan, Mutex);
   if (!LDat) {
     Handler.handleUnmatchedUnlock(Mutex.toString(), UnlockLoc);
@@ -1423,12 +1525,13 @@ void ThreadSafetyAnalyzer::removeLock(FactSet &FSet,
 /// and push them onto Mtxs, discarding any duplicates.
 template <typename AttrType>
 void ThreadSafetyAnalyzer::getMutexIDs(MutexIDList &Mtxs, AttrType *Attr,
-                                       Expr *Exp, const NamedDecl *D) {
+                                       Expr *Exp, const NamedDecl *D,
+                                       VarDecl *SelfDecl) {
   typedef typename AttrType::args_iterator iterator_type;
 
   if (Attr->args_size() == 0) {
     // The mutex held is the "this" object.
-    SExpr Mu(0, Exp, D);
+    SExpr Mu(0, Exp, D, SelfDecl);
     if (!Mu.isValid())
       SExpr::warnInvalidLock(Handler, 0, Exp, D);
     else
@@ -1437,7 +1540,7 @@ void ThreadSafetyAnalyzer::getMutexIDs(MutexIDList &Mtxs, AttrType *Attr,
   }
 
   for (iterator_type I=Attr->args_begin(), E=Attr->args_end(); I != E; ++I) {
-    SExpr Mu(*I, Exp, D);
+    SExpr Mu(*I, Exp, D, SelfDecl);
     if (!Mu.isValid())
       SExpr::warnInvalidLock(Handler, *I, Exp, D);
     else
@@ -1512,6 +1615,9 @@ const CallExpr* ThreadSafetyAnalyzer::getTrylockCallExpr(const Stmt *Cond,
   else if (const ImplicitCastExpr *CE = dyn_cast<ImplicitCastExpr>(Cond)) {
     return getTrylockCallExpr(CE->getSubExpr(), C, Negate);
   }
+  else if (const ExprWithCleanups* EWC = dyn_cast<ExprWithCleanups>(Cond)) {
+    return getTrylockCallExpr(EWC->getSubExpr(), C, Negate);
+  }
   else if (const DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(Cond)) {
     const Expr *E = LocalVarMap.lookupExpr(DRE->getDecl(), C);
     return getTrylockCallExpr(E, C, Negate);
@@ -1591,7 +1697,7 @@ void ThreadSafetyAnalyzer::getEdgeLockset(FactSet& Result,
       case attr::SharedTrylockFunction: {
         SharedTrylockFunctionAttr *A =
           cast<SharedTrylockFunctionAttr>(Attr);
-        getMutexIDs(ExclusiveLocksToAdd, A, Exp, FunDecl,
+        getMutexIDs(SharedLocksToAdd, A, Exp, FunDecl,
                     PredBlock, CurrBlock, A->getSuccessValue(), Negate);
         break;
       }
@@ -1631,39 +1737,12 @@ class BuildLockset : public StmtVisitor<BuildLockset> {
 
   void warnIfMutexNotHeld(const NamedDecl *D, Expr *Exp, AccessKind AK,
                           Expr *MutexExp, ProtectedOperationKind POK);
+  void warnIfMutexHeld(const NamedDecl *D, Expr *Exp, Expr *MutexExp);
 
   void checkAccess(Expr *Exp, AccessKind AK);
   void checkDereference(Expr *Exp, AccessKind AK);
   void handleCall(Expr *Exp, const NamedDecl *D, VarDecl *VD = 0);
 
-  /// \brief Returns true if the lockset contains a lock, regardless of whether
-  /// the lock is held exclusively or shared.
-  bool locksetContains(const SExpr &Mu) const {
-    return FSet.findLock(Analyzer->FactMan, Mu);
-  }
-
-  /// \brief Returns true if the lockset contains a lock with the passed in
-  /// locktype.
-  bool locksetContains(const SExpr &Mu, LockKind KindRequested) const {
-    const LockData *LockHeld = FSet.findLock(Analyzer->FactMan, Mu);
-    return (LockHeld && KindRequested == LockHeld->LKind);
-  }
-
-  /// \brief Returns true if the lockset contains a lock with at least the
-  /// passed in locktype. So for example, if we pass in LK_Shared, this function
-  /// returns true if the lock is held LK_Shared or LK_Exclusive. If we pass in
-  /// LK_Exclusive, this function returns true if the lock is held LK_Exclusive.
-  bool locksetContainsAtLeast(const SExpr &Lock,
-                              LockKind KindRequested) const {
-    switch (KindRequested) {
-      case LK_Shared:
-        return locksetContains(Lock);
-      case LK_Exclusive:
-        return locksetContains(Lock, KindRequested);
-    }
-    llvm_unreachable("Unknown LockKind");
-  }
-
 public:
   BuildLockset(ThreadSafetyAnalyzer *Anlzr, CFGBlockInfo &Info)
     : StmtVisitor<BuildLockset>(),
@@ -1701,13 +1780,57 @@ void BuildLockset::warnIfMutexNotHeld(const NamedDecl *D, Expr *Exp,
   LockKind LK = getLockKindFromAccessKind(AK);
 
   SExpr Mutex(MutexExp, Exp, D);
-  if (!Mutex.isValid())
+  if (!Mutex.isValid()) {
     SExpr::warnInvalidLock(Analyzer->Handler, MutexExp, Exp, D);
-  else if (!locksetContainsAtLeast(Mutex, LK))
+    return;
+  } else if (Mutex.shouldIgnore()) {
+    return;
+  }
+
+  LockData* LDat = FSet.findLockUniv(Analyzer->FactMan, Mutex);
+  bool NoError = true;
+  if (!LDat) {
+    // No exact match found.  Look for a partial match.
+    FactEntry* FEntry = FSet.findPartialMatch(Analyzer->FactMan, Mutex);
+    if (FEntry) {
+      // Warn that there's no precise match.
+      LDat = &FEntry->LDat;
+      std::string PartMatchStr = FEntry->MutID.toString();
+      StringRef   PartMatchName(PartMatchStr);
+      Analyzer->Handler.handleMutexNotHeld(D, POK, Mutex.toString(), LK,
+                                           Exp->getExprLoc(), &PartMatchName);
+    } else {
+      // Warn that there's no match at all.
+      Analyzer->Handler.handleMutexNotHeld(D, POK, Mutex.toString(), LK,
+                                           Exp->getExprLoc());
+    }
+    NoError = false;
+  }
+  // Make sure the mutex we found is the right kind.
+  if (NoError && LDat && !LDat->isAtLeast(LK))
     Analyzer->Handler.handleMutexNotHeld(D, POK, Mutex.toString(), LK,
                                          Exp->getExprLoc());
 }
 
+/// \brief Warn if the LSet contains the given lock.
+void BuildLockset::warnIfMutexHeld(const NamedDecl *D, Expr* Exp,
+                                   Expr *MutexExp) {
+  SExpr Mutex(MutexExp, Exp, D);
+  if (!Mutex.isValid()) {
+    SExpr::warnInvalidLock(Analyzer->Handler, MutexExp, Exp, D);
+    return;
+  }
+
+  LockData* LDat = FSet.findLock(Analyzer->FactMan, Mutex);
+  if (LDat) {
+    std::string DeclName = D->getNameAsString();
+    StringRef   DeclNameSR (DeclName);
+    Analyzer->Handler.handleFunExcludesLock(DeclNameSR, Mutex.toString(),
+                                            Exp->getExprLoc());
+  }
+}
+
+
 /// \brief This method identifies variable dereferences and checks pt_guarded_by
 /// and pt_guarded_var annotations. Note that we only check these annotations
 /// at the time a pointer is dereferenced.
@@ -1776,7 +1899,7 @@ void BuildLockset::handleCall(Expr *Exp, const NamedDecl *D, VarDecl *VD) {
       // to our lockset with kind exclusive.
       case attr::ExclusiveLockFunction: {
         ExclusiveLockFunctionAttr *A = cast<ExclusiveLockFunctionAttr>(At);
-        Analyzer->getMutexIDs(ExclusiveLocksToAdd, A, Exp, D);
+        Analyzer->getMutexIDs(ExclusiveLocksToAdd, A, Exp, D, VD);
         break;
       }
 
@@ -1784,7 +1907,7 @@ void BuildLockset::handleCall(Expr *Exp, const NamedDecl *D, VarDecl *VD) {
       // to our lockset with kind shared.
       case attr::SharedLockFunction: {
         SharedLockFunctionAttr *A = cast<SharedLockFunctionAttr>(At);
-        Analyzer->getMutexIDs(SharedLocksToAdd, A, Exp, D);
+        Analyzer->getMutexIDs(SharedLocksToAdd, A, Exp, D, VD);
         break;
       }
 
@@ -1792,7 +1915,7 @@ void BuildLockset::handleCall(Expr *Exp, const NamedDecl *D, VarDecl *VD) {
       // mutexes from the lockset, and flag a warning if they are not there.
       case attr::UnlockFunction: {
         UnlockFunctionAttr *A = cast<UnlockFunctionAttr>(At);
-        Analyzer->getMutexIDs(LocksToRemove, A, Exp, D);
+        Analyzer->getMutexIDs(LocksToRemove, A, Exp, D, VD);
         break;
       }
 
@@ -1816,15 +1939,10 @@ void BuildLockset::handleCall(Expr *Exp, const NamedDecl *D, VarDecl *VD) {
 
       case attr::LocksExcluded: {
         LocksExcludedAttr *A = cast<LocksExcludedAttr>(At);
+
         for (LocksExcludedAttr::args_iterator I = A->args_begin(),
             E = A->args_end(); I != E; ++I) {
-          SExpr Mutex(*I, Exp, D);
-          if (!Mutex.isValid())
-            SExpr::warnInvalidLock(Analyzer->Handler, *I, Exp, D);
-          else if (locksetContains(Mutex))
-            Analyzer->Handler.handleFunExcludesLock(D->getName(),
-                                                    Mutex.toString(),
-                                                    Exp->getExprLoc());
+          warnIfMutexHeld(D, Exp, *I);
         }
         break;
       }
@@ -1973,8 +2091,8 @@ void BuildLockset::VisitDeclStmt(DeclStmt *S) {
 /// are the same. In the event of a difference, we use the intersection of these
 /// two locksets at the start of D.
 ///
-/// \param LSet1 The first lockset.
-/// \param LSet2 The second lockset.
+/// \param FSet1 The first lockset.
+/// \param FSet2 The second lockset.
 /// \param JoinLoc The location of the join point for error reporting
 /// \param LEK1 The error message to report if a mutex is missing from LSet1
 /// \param LEK2 The error message to report if a mutex is missing from Lset2
@@ -2012,7 +2130,7 @@ void ThreadSafetyAnalyzer::intersectAndWarn(FactSet &FSet1,
                                             JoinLoc, LEK1);
         }
       }
-      else if (!LDat2.Managed)
+      else if (!LDat2.Managed && !FSet2Mutex.isUniversal())
         Handler.handleMutexHeldEndOfScope(FSet2Mutex.toString(),
                                           LDat2.AcquireLoc,
                                           JoinLoc, LEK1);
@@ -2035,7 +2153,7 @@ void ThreadSafetyAnalyzer::intersectAndWarn(FactSet &FSet1,
                                             JoinLoc, LEK1);
         }
       }
-      else if (!LDat1.Managed)
+      else if (!LDat1.Managed && !FSet1Mutex.isUniversal())
         Handler.handleMutexHeldEndOfScope(FSet1Mutex.toString(),
                                           LDat1.AcquireLoc,
                                           JoinLoc, LEK2);
@@ -2081,6 +2199,9 @@ void ThreadSafetyAnalyzer::runAnalysis(AnalysisDeclContext &AC) {
   PostOrderCFGView *SortedGraph = AC.getAnalysis<PostOrderCFGView>();
   PostOrderCFGView::CFGBlockSet VisitedBlocks(CFGraph);
 
+  // Mark entry block as reachable
+  BlockInfo[CFGraph->getEntry().getBlockID()].Reachable = true;
+
   // Compute SSA names for local variables
   LocalVarMap.traverseCFG(CFGraph, SortedGraph, BlockInfo);
 
@@ -2168,10 +2289,16 @@ void ThreadSafetyAnalyzer::runAnalysis(AnalysisDeclContext &AC) {
       if (*PI == 0 || !VisitedBlocks.alreadySet(*PI))
         continue;
 
+      int PrevBlockID = (*PI)->getBlockID();
+      CFGBlockInfo *PrevBlockInfo = &BlockInfo[PrevBlockID];
+
       // Ignore edges from blocks that can't return.
-      if ((*PI)->hasNoReturnElement())
+      if ((*PI)->hasNoReturnElement() || !PrevBlockInfo->Reachable)
         continue;
 
+      // Okay, we can reach this block from the entry.
+      CurrBlockInfo->Reachable = true;
+
       // If the previous block ended in a 'continue' or 'break' statement, then
       // a difference in locksets is probably due to a bug in that block, rather
       // than in some other predecessor. In that case, keep the other
@@ -2183,8 +2310,7 @@ void ThreadSafetyAnalyzer::runAnalysis(AnalysisDeclContext &AC) {
         }
       }
 
-      int PrevBlockID = (*PI)->getBlockID();
-      CFGBlockInfo *PrevBlockInfo = &BlockInfo[PrevBlockID];
+
       FactSet PrevLockset;
       getEdgeLockset(PrevLockset, PrevBlockInfo->ExitSet, *PI, CurrBlock);
 
@@ -2198,6 +2324,10 @@ void ThreadSafetyAnalyzer::runAnalysis(AnalysisDeclContext &AC) {
       }
     }
 
+    // Skip rest of block if it's not reachable.
+    if (!CurrBlockInfo->Reachable)
+      continue;
+
     // Process continue and break blocks. Assume that the lockset for the
     // resulting block is unaffected by any discrepancies in them.
     for (unsigned SpecialI = 0, SpecialN = SpecialBlocks.size();
@@ -2287,6 +2417,10 @@ void ThreadSafetyAnalyzer::runAnalysis(AnalysisDeclContext &AC) {
   CFGBlockInfo *Initial = &BlockInfo[CFGraph->getEntry().getBlockID()];
   CFGBlockInfo *Final   = &BlockInfo[CFGraph->getExit().getBlockID()];
 
+  // Skip the final check if the exit block is unreachable.
+  if (!Final->Reachable)
+    return;
+
   // FIXME: Should we call this function for all blocks which exit the function?
   intersectAndWarn(Initial->EntrySet, Final->ExitSet,
                    Final->ExitLoc,
diff --git a/lib/Analysis/UninitializedValues.cpp b/lib/Analysis/UninitializedValues.cpp
index 858be45..b2e27ca 100644
--- a/lib/Analysis/UninitializedValues.cpp
+++ b/lib/Analysis/UninitializedValues.cpp
@@ -13,6 +13,7 @@
 
 #include <utility>
 #include "llvm/ADT/Optional.h"
+#include "llvm/ADT/SmallBitVector.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/PackedVector.h"
 #include "llvm/ADT/DenseMap.h"
@@ -22,6 +23,7 @@
 #include "clang/Analysis/AnalysisContext.h"
 #include "clang/Analysis/Visitors/CFGRecStmtDeclVisitor.h"
 #include "clang/Analysis/Analyses/UninitializedValues.h"
+#include "clang/Analysis/DomainSpecific/ObjCNoReturn.h"
 #include "llvm/Support/SaveAndRestore.h"
 
 using namespace clang;
@@ -97,22 +99,21 @@ static bool isAlwaysUninit(const Value v) {
 
 namespace {
 
-typedef llvm::PackedVector<Value, 2> ValueVector;
+typedef llvm::PackedVector<Value, 2, llvm::SmallBitVector> ValueVector;
 
 class CFGBlockValues {
   const CFG &cfg;
-  std::vector<ValueVector*> vals;
+  SmallVector<ValueVector, 8> vals;
   ValueVector scratch;
   DeclToIndex declToIndex;
 public:
   CFGBlockValues(const CFG &cfg);
-  ~CFGBlockValues();
 
   unsigned getNumEntries() const { return declToIndex.size(); }
   
   void computeSetOfDeclarations(const DeclContext &dc);  
   ValueVector &getValueVector(const CFGBlock *block) {
-    return *vals[block->getBlockID()];
+    return vals[block->getBlockID()];
   }
 
   void setAllScratchValues(Value V);
@@ -138,12 +139,6 @@ public:
 
 CFGBlockValues::CFGBlockValues(const CFG &c) : cfg(c), vals(0) {}
 
-CFGBlockValues::~CFGBlockValues() {
-  for (std::vector<ValueVector*>::iterator I = vals.begin(), E = vals.end();
-       I != E; ++I)
-    delete *I;
-}
-
 void CFGBlockValues::computeSetOfDeclarations(const DeclContext &dc) {
   declToIndex.computeMap(dc);
   unsigned decls = declToIndex.size();
@@ -153,7 +148,7 @@ void CFGBlockValues::computeSetOfDeclarations(const DeclContext &dc) {
     return;
   vals.resize(n);
   for (unsigned i = 0; i < n; ++i)
-    vals[i] = new ValueVector(decls);
+    vals[i].resize(decls);
 }
 
 #if DEBUG_LOGGING
@@ -412,6 +407,7 @@ class TransferFunctions : public StmtVisitor<TransferFunctions> {
   const CFGBlock *block;
   AnalysisDeclContext &ac;
   const ClassifyRefs &classification;
+  ObjCNoReturn objCNoRet;
   UninitVariablesHandler *handler;
 
 public:
@@ -420,16 +416,18 @@ public:
                     const ClassifyRefs &classification,
                     UninitVariablesHandler *handler)
     : vals(vals), cfg(cfg), block(block), ac(ac),
-      classification(classification), handler(handler) {}
+      classification(classification), objCNoRet(ac.getASTContext()),
+      handler(handler) {}
 
   void reportUse(const Expr *ex, const VarDecl *vd);
 
-  void VisitObjCForCollectionStmt(ObjCForCollectionStmt *FS);
+  void VisitBinaryOperator(BinaryOperator *bo);
   void VisitBlockExpr(BlockExpr *be);
   void VisitCallExpr(CallExpr *ce);
-  void VisitDeclStmt(DeclStmt *ds);
   void VisitDeclRefExpr(DeclRefExpr *dr);
-  void VisitBinaryOperator(BinaryOperator *bo);
+  void VisitDeclStmt(DeclStmt *ds);
+  void VisitObjCForCollectionStmt(ObjCForCollectionStmt *FS);
+  void VisitObjCMessageExpr(ObjCMessageExpr *ME);
 
   bool isTrackedVar(const VarDecl *vd) {
     return ::isTrackedVar(vd, cast<DeclContext>(ac.getDecl()));
@@ -605,14 +603,26 @@ void TransferFunctions::VisitBlockExpr(BlockExpr *be) {
 }
 
 void TransferFunctions::VisitCallExpr(CallExpr *ce) {
-  // After a call to a function like setjmp or vfork, any variable which is
-  // initialized anywhere within this function may now be initialized. For now,
-  // just assume such a call initializes all variables.
-  // FIXME: Only mark variables as initialized if they have an initializer which
-  // is reachable from here.
-  Decl *Callee = ce->getCalleeDecl();
-  if (Callee && Callee->hasAttr<ReturnsTwiceAttr>())
-    vals.setAllScratchValues(Initialized);
+  if (Decl *Callee = ce->getCalleeDecl()) {
+    if (Callee->hasAttr<ReturnsTwiceAttr>()) {
+      // After a call to a function like setjmp or vfork, any variable which is
+      // initialized anywhere within this function may now be initialized. For
+      // now, just assume such a call initializes all variables.  FIXME: Only
+      // mark variables as initialized if they have an initializer which is
+      // reachable from here.
+      vals.setAllScratchValues(Initialized);
+    }
+    else if (Callee->hasAttr<AnalyzerNoReturnAttr>()) {
+      // Functions labeled like "analyzer_noreturn" are often used to denote
+      // "panic" functions that in special debug situations can still return,
+      // but for the most part should not be treated as returning.  This is a
+      // useful annotation borrowed from the static analyzer that is useful for
+      // suppressing branch-specific false positives when we call one of these
+      // functions but keep pretending the path continues (when in reality the
+      // user doesn't care).
+      vals.setAllScratchValues(Unknown);
+    }
+  }
 }
 
 void TransferFunctions::VisitDeclRefExpr(DeclRefExpr *dr) {
@@ -677,6 +687,14 @@ void TransferFunctions::VisitDeclStmt(DeclStmt *DS) {
   }
 }
 
+void TransferFunctions::VisitObjCMessageExpr(ObjCMessageExpr *ME) {
+  // If the Objective-C message expression is an implicit no-return that
+  // is not modeled in the CFG, set the tracked dataflow values to Unknown.
+  if (objCNoRet.isImplicitNoReturn(ME)) {
+    vals.setAllScratchValues(Unknown);
+  }
+}
+
 //------------------------------------------------------------------------====//
 // High-level "driver" logic for uninitialized values analysis.
 //====------------------------------------------------------------------------//
diff --git a/lib/Basic/ConvertUTF.c b/lib/Basic/ConvertUTF.c
index 4793b25..d16965d 100644
--- a/lib/Basic/ConvertUTF.c
+++ b/lib/Basic/ConvertUTF.c
@@ -111,7 +111,6 @@ static const UTF8 firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC
  * into an inline function.
  */
 
-#ifdef CLANG_NEEDS_THESE_ONE_DAY
 
 /* --------------------------------------------------------------------- */
 
@@ -285,7 +284,6 @@ ConversionResult ConvertUTF16toUTF8 (
     *targetStart = target;
     return result;
 }
-#endif
 
 /* --------------------------------------------------------------------- */
 
@@ -361,7 +359,7 @@ static Boolean isLegalUTF8(const UTF8 *source, int length) {
         /* Everything else falls through when "true"... */
     case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
     case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
-    case 2: if ((a = (*--srcptr)) > 0xBF) return false;
+    case 2: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false;
 
         switch (*source) {
             /* no fall-through in this inner switch */
@@ -395,15 +393,25 @@ Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd) {
 /* --------------------------------------------------------------------- */
 
 /*
+ * Exported function to return the total number of bytes in a codepoint
+ * represented in UTF-8, given the value of the first byte.
+ */
+unsigned getNumBytesForUTF8(UTF8 first) {
+  return trailingBytesForUTF8[first] + 1;
+}
+
+/* --------------------------------------------------------------------- */
+
+/*
  * Exported function to return whether a UTF-8 string is legal or not.
  * This is not used here; it's just exported.
  */
-Boolean isLegalUTF8String(const UTF8 *source, const UTF8 *sourceEnd) {
-    while (source != sourceEnd) {
-        int length = trailingBytesForUTF8[*source] + 1;
-        if (length > sourceEnd - source || !isLegalUTF8(source, length))
+Boolean isLegalUTF8String(const UTF8 **source, const UTF8 *sourceEnd) {
+    while (*source != sourceEnd) {
+        int length = trailingBytesForUTF8[**source] + 1;
+        if (length > sourceEnd - *source || !isLegalUTF8(*source, length))
             return false;
-        source += length;
+        *source += length;
     }
     return true;
 }
diff --git a/lib/Basic/ConvertUTFWrapper.cpp b/lib/Basic/ConvertUTFWrapper.cpp
index a1b3f7f..6be3828 100644
--- a/lib/Basic/ConvertUTFWrapper.cpp
+++ b/lib/Basic/ConvertUTFWrapper.cpp
@@ -13,16 +13,19 @@
 namespace clang {
 
 bool ConvertUTF8toWide(unsigned WideCharWidth, llvm::StringRef Source,
-                       char *&ResultPtr) {
+                       char *&ResultPtr, const UTF8 *&ErrorPtr) {
   assert(WideCharWidth == 1 || WideCharWidth == 2 || WideCharWidth == 4);
   ConversionResult result = conversionOK;
   // Copy the character span over.
   if (WideCharWidth == 1) {
-    if (!isLegalUTF8String(reinterpret_cast<const UTF8*>(Source.begin()),
-                           reinterpret_cast<const UTF8*>(Source.end())))
+    const UTF8 *Pos = reinterpret_cast<const UTF8*>(Source.begin());
+    if (!isLegalUTF8String(&Pos, reinterpret_cast<const UTF8*>(Source.end()))) {
       result = sourceIllegal;
-    memcpy(ResultPtr, Source.data(), Source.size());
-    ResultPtr += Source.size();
+      ErrorPtr = Pos;
+    } else {
+      memcpy(ResultPtr, Source.data(), Source.size());
+      ResultPtr += Source.size();
+    }
   } else if (WideCharWidth == 2) {
     const UTF8 *sourceStart = (const UTF8*)Source.data();
     // FIXME: Make the type of the result buffer correct instead of
@@ -34,6 +37,8 @@ bool ConvertUTF8toWide(unsigned WideCharWidth, llvm::StringRef Source,
         &targetStart, targetStart + 2*Source.size(), flags);
     if (result == conversionOK)
       ResultPtr = reinterpret_cast<char*>(targetStart);
+    else
+      ErrorPtr = sourceStart;
   } else if (WideCharWidth == 4) {
     const UTF8 *sourceStart = (const UTF8*)Source.data();
     // FIXME: Make the type of the result buffer correct instead of
@@ -45,6 +50,8 @@ bool ConvertUTF8toWide(unsigned WideCharWidth, llvm::StringRef Source,
         &targetStart, targetStart + 4*Source.size(), flags);
     if (result == conversionOK)
       ResultPtr = reinterpret_cast<char*>(targetStart);
+    else
+      ErrorPtr = sourceStart;
   }
   assert((result != targetExhausted)
          && "ConvertUTF8toUTFXX exhausted target buffer");
@@ -67,4 +74,3 @@ bool ConvertCodePointToUTF8(unsigned Source, char *&ResultPtr) {
 }
 
 } // end namespace clang
-
diff --git a/lib/Basic/Diagnostic.cpp b/lib/Basic/Diagnostic.cpp
index 8065b2d..854c4c5 100644
--- a/lib/Basic/Diagnostic.cpp
+++ b/lib/Basic/Diagnostic.cpp
@@ -12,9 +12,11 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Basic/Diagnostic.h"
+#include "clang/Basic/DiagnosticOptions.h"
 #include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/PartialDiagnostic.h"
 #include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringExtras.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/CrashRecoveryContext.h"
 #include <cctype>
@@ -36,9 +38,10 @@ static void DummyArgToStringFn(DiagnosticsEngine::ArgumentKind AK, intptr_t QT,
 
 DiagnosticsEngine::DiagnosticsEngine(
                        const IntrusiveRefCntPtr<DiagnosticIDs> &diags,
+                       DiagnosticOptions *DiagOpts,       
                        DiagnosticConsumer *client, bool ShouldOwnClient)
-  : Diags(diags), Client(client), OwnsDiagClient(ShouldOwnClient),
-    SourceMgr(0) {
+  : Diags(diags), DiagOpts(DiagOpts), Client(client),
+    OwnsDiagClient(ShouldOwnClient), SourceMgr(0) {
   ArgToStringFn = DummyArgToStringFn;
   ArgToStringCookie = 0;
 
@@ -515,23 +518,7 @@ static void HandleOrdinalModifier(unsigned ValNo,
 
   // We could use text forms for the first N ordinals, but the numeric
   // forms are actually nicer in diagnostics because they stand out.
-  Out << ValNo;
-
-  // It is critically important that we do this perfectly for
-  // user-written sequences with over 100 elements.
-  switch (ValNo % 100) {
-  case 11:
-  case 12:
-  case 13:
-    Out << "th"; return;
-  default:
-    switch (ValNo % 10) {
-    case 1: Out << "st"; return;
-    case 2: Out << "nd"; return;
-    case 3: Out << "rd"; return;
-    default: Out << "th"; return;
-    }
-  }
+  Out << ValNo << llvm::getOrdinalSuffix(ValNo);
 }
 
 
diff --git a/lib/Basic/DiagnosticIDs.cpp b/lib/Basic/DiagnosticIDs.cpp
index ca96fd2..ed97643 100644
--- a/lib/Basic/DiagnosticIDs.cpp
+++ b/lib/Basic/DiagnosticIDs.cpp
@@ -628,9 +628,9 @@ bool DiagnosticIDs::ProcessDiag(DiagnosticsEngine &Diag) const {
   if (DiagLevel >= DiagnosticIDs::Error) {
     if (isUnrecoverable(DiagID))
       Diag.UnrecoverableErrorOccurred = true;
-    
+
+    Diag.ErrorOccurred = true;
     if (Diag.Client->IncludeInDiagnosticCounts()) {
-      Diag.ErrorOccurred = true;
       ++Diag.NumErrors;
     }
 
@@ -686,4 +686,3 @@ bool DiagnosticIDs::isARCDiagnostic(unsigned DiagID) {
   unsigned cat = getCategoryNumberForDiag(DiagID);
   return DiagnosticIDs::getCategoryNameFromID(cat).startswith("ARC ");
 }
-
diff --git a/lib/Basic/FileManager.cpp b/lib/Basic/FileManager.cpp
index c6b894c..a816969 100644
--- a/lib/Basic/FileManager.cpp
+++ b/lib/Basic/FileManager.cpp
@@ -36,6 +36,9 @@
 #include <sys/uio.h>
 #else
 #include <io.h>
+#ifndef S_ISFIFO
+#define S_ISFIFO(x) (0)
+#endif
 #endif
 using namespace clang;
 
@@ -57,6 +60,10 @@ FileEntry::~FileEntry() {
   if (FD != -1) ::close(FD);
 }
 
+bool FileEntry::isNamedPipe() const {
+  return S_ISFIFO(FileMode);
+}
+
 //===----------------------------------------------------------------------===//
 // Windows.
 //===----------------------------------------------------------------------===//
diff --git a/lib/Basic/IdentifierTable.cpp b/lib/Basic/IdentifierTable.cpp
index 4869ae1..1965bf9 100644
--- a/lib/Basic/IdentifierTable.cpp
+++ b/lib/Basic/IdentifierTable.cpp
@@ -17,7 +17,6 @@
 #include "llvm/ADT/FoldingSet.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallString.h"
-#include "llvm/ADT/StringSwitch.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/ErrorHandling.h"
 #include <cctype>
@@ -33,6 +32,7 @@ IdentifierInfo::IdentifierInfo() {
   TokenID = tok::identifier;
   ObjCOrBuiltinID = 0;
   HasMacro = false;
+  HadMacro = false;
   IsExtension = false;
   IsCXX11CompatKeyword = false;
   IsPoisoned = false;
@@ -105,6 +105,7 @@ namespace {
     KEYC11 = 0x400,
     KEYARC = 0x800,
     KEYNOMS = 0x01000,
+    WCHARSUPPORT = 0x02000,
     KEYALL = (0xffff & ~KEYNOMS) // Because KEYNOMS is used to exclude.
   };
 }
@@ -129,6 +130,7 @@ static void AddKeyword(StringRef Keyword,
   else if (LangOpts.MicrosoftExt && (Flags & KEYMS)) AddResult = 1;
   else if (LangOpts.Borland && (Flags & KEYBORLAND)) AddResult = 1;
   else if (LangOpts.Bool && (Flags & BOOLSUPPORT)) AddResult = 2;
+  else if (LangOpts.WChar && (Flags & WCHARSUPPORT)) AddResult = 2;
   else if (LangOpts.AltiVec && (Flags & KEYALTIVEC)) AddResult = 2;
   else if (LangOpts.OpenCL && (Flags & KEYOPENCL)) AddResult = 2;
   else if (!LangOpts.CPlusPlus && (Flags & KEYNOCXX)) AddResult = 2;
diff --git a/lib/Basic/Module.cpp b/lib/Basic/Module.cpp
index 6348840..76c7f8b 100644
--- a/lib/Basic/Module.cpp
+++ b/lib/Basic/Module.cpp
@@ -23,8 +23,8 @@ using namespace clang;
 
 Module::Module(StringRef Name, SourceLocation DefinitionLoc, Module *Parent, 
                bool IsFramework, bool IsExplicit)
-  : Name(Name), DefinitionLoc(DefinitionLoc), Parent(Parent), 
-    Umbrella(), IsAvailable(true), IsFromModuleFile(false), 
+  : Name(Name), DefinitionLoc(DefinitionLoc), Parent(Parent),
+    Umbrella(), ASTFile(0), IsAvailable(true), IsFromModuleFile(false),
     IsFramework(IsFramework), IsExplicit(IsExplicit), IsSystem(false),
     InferSubmodules(false), InferExplicitSubmodules(false), 
     InferExportWildcard(false), NameVisibility(Hidden) 
@@ -219,6 +219,13 @@ void Module::print(llvm::raw_ostream &OS, unsigned Indent) const {
     OS.write_escaped(Headers[I]->getName());
     OS << "\"\n";
   }
+
+  for (unsigned I = 0, N = ExcludedHeaders.size(); I != N; ++I) {
+    OS.indent(Indent + 2);
+    OS << "exclude header \"";
+    OS.write_escaped(ExcludedHeaders[I]->getName());
+    OS << "\"\n";
+  }
   
   for (submodule_const_iterator MI = submodule_begin(), MIEnd = submodule_end();
        MI != MIEnd; ++MI)
diff --git a/lib/Basic/SourceLocation.cpp b/lib/Basic/SourceLocation.cpp
index bb5a10a..0d62f7b 100644
--- a/lib/Basic/SourceLocation.cpp
+++ b/lib/Basic/SourceLocation.cpp
@@ -61,6 +61,13 @@ void SourceLocation::print(raw_ostream &OS, const SourceManager &SM)const{
   OS << '>';
 }
 
+std::string SourceLocation::printToString(const SourceManager &SM) const {
+  std::string S;
+  llvm::raw_string_ostream OS(S);
+  print(OS, SM);
+  return S;
+}
+
 void SourceLocation::dump(const SourceManager &SM) const {
   print(llvm::errs(), SM);
 }
diff --git a/lib/Basic/SourceManager.cpp b/lib/Basic/SourceManager.cpp
index 9ec2474..cd0284a 100644
--- a/lib/Basic/SourceManager.cpp
+++ b/lib/Basic/SourceManager.cpp
@@ -1029,6 +1029,17 @@ unsigned SourceManager::getColumnNumber(FileID FID, unsigned FilePos,
     return 1;
   }
 
+  // See if we just calculated the line number for this FilePos and can use
+  // that to lookup the start of the line instead of searching for it.
+  if (LastLineNoFileIDQuery == FID &&
+      LastLineNoContentCache->SourceLineCache != 0) {
+    unsigned *SourceLineCache = LastLineNoContentCache->SourceLineCache;
+    unsigned LineStart = SourceLineCache[LastLineNoResult - 1];
+    unsigned LineEnd = SourceLineCache[LastLineNoResult];
+    if (FilePos >= LineStart && FilePos < LineEnd)
+      return FilePos - LineStart + 1;
+  }
+
   const char *Buf = MemBuf->getBufferStart();
   unsigned LineStart = FilePos;
   while (LineStart && Buf[LineStart-1] != '\n' && Buf[LineStart-1] != '\r')
@@ -1112,7 +1123,7 @@ static void ComputeLineNumbers(DiagnosticsEngine &Diag, ContentCache *FI,
 
     // Scan 16 byte chunks for '\r' and '\n'. Ignore '\0'.
     while (NextBuf+16 <= End) {
-      __m128i Chunk = *(__m128i*)NextBuf;
+      const __m128i Chunk = *(const __m128i*)NextBuf;
       __m128i Cmp = _mm_or_si128(_mm_cmpeq_epi8(Chunk, CRs),
                                  _mm_cmpeq_epi8(Chunk, LFs));
       unsigned Mask = _mm_movemask_epi8(Cmp);
@@ -1577,6 +1588,7 @@ FileID SourceManager::translateFile(const FileEntry *SourceFile) const {
     }      
   }
   
+  (void) SourceFile;
   return FirstFID;
 }
 
@@ -1693,46 +1705,91 @@ void SourceManager::computeMacroArgsCache(MacroArgsMap *&CachePtr,
     if (!ExpInfo.isMacroArgExpansion())
       continue;
 
-    SourceLocation SpellLoc = ExpInfo.getSpellingLoc();
-    while (!SpellLoc.isFileID()) {
-      std::pair<FileID, unsigned> LocInfo = getDecomposedLoc(SpellLoc);
-      const ExpansionInfo &Info = getSLocEntry(LocInfo.first).getExpansion();
-      if (!Info.isMacroArgExpansion())
-        break;
-      SpellLoc = Info.getSpellingLoc().getLocWithOffset(LocInfo.second);
+    associateFileChunkWithMacroArgExp(MacroArgsCache, FID,
+                                 ExpInfo.getSpellingLoc(),
+                                 SourceLocation::getMacroLoc(Entry.getOffset()),
+                                 getFileIDSize(FileID::get(ID)));
+  }
+}
+
+void SourceManager::associateFileChunkWithMacroArgExp(
+                                         MacroArgsMap &MacroArgsCache,
+                                         FileID FID,
+                                         SourceLocation SpellLoc,
+                                         SourceLocation ExpansionLoc,
+                                         unsigned ExpansionLength) const {
+  if (!SpellLoc.isFileID()) {
+    unsigned SpellBeginOffs = SpellLoc.getOffset();
+    unsigned SpellEndOffs = SpellBeginOffs + ExpansionLength;
+
+    // The spelling range for this macro argument expansion can span multiple
+    // consecutive FileID entries. Go through each entry contained in the
+    // spelling range and if one is itself a macro argument expansion, recurse
+    // and associate the file chunk that it represents.
+
+    FileID SpellFID; // Current FileID in the spelling range.
+    unsigned SpellRelativeOffs;
+    llvm::tie(SpellFID, SpellRelativeOffs) = getDecomposedLoc(SpellLoc);
+    while (1) {
+      const SLocEntry &Entry = getSLocEntry(SpellFID);
+      unsigned SpellFIDBeginOffs = Entry.getOffset();
+      unsigned SpellFIDSize = getFileIDSize(SpellFID);
+      unsigned SpellFIDEndOffs = SpellFIDBeginOffs + SpellFIDSize;
+      const ExpansionInfo &Info = Entry.getExpansion();
+      if (Info.isMacroArgExpansion()) {
+        unsigned CurrSpellLength;
+        if (SpellFIDEndOffs < SpellEndOffs)
+          CurrSpellLength = SpellFIDSize - SpellRelativeOffs;
+        else
+          CurrSpellLength = ExpansionLength;
+        associateFileChunkWithMacroArgExp(MacroArgsCache, FID,
+                      Info.getSpellingLoc().getLocWithOffset(SpellRelativeOffs),
+                      ExpansionLoc, CurrSpellLength);
+      }
+
+      if (SpellFIDEndOffs >= SpellEndOffs)
+        return; // we covered all FileID entries in the spelling range.
+
+      // Move to the next FileID entry in the spelling range.
+      unsigned advance = SpellFIDSize - SpellRelativeOffs + 1;
+      ExpansionLoc = ExpansionLoc.getLocWithOffset(advance);
+      ExpansionLength -= advance;
+      ++SpellFID.ID;
+      SpellRelativeOffs = 0;
     }
-    if (!SpellLoc.isFileID())
-      continue;
-    
-    unsigned BeginOffs;
-    if (!isInFileID(SpellLoc, FID, &BeginOffs))
-      continue;
 
-    unsigned EndOffs = BeginOffs + getFileIDSize(FileID::get(ID));
-
-    // Add a new chunk for this macro argument. A previous macro argument chunk
-    // may have been lexed again, so e.g. if the map is
-    //     0   -> SourceLocation()
-    //     100 -> Expanded loc #1
-    //     110 -> SourceLocation()
-    // and we found a new macro FileID that lexed from offet 105 with length 3,
-    // the new map will be:
-    //     0   -> SourceLocation()
-    //     100 -> Expanded loc #1
-    //     105 -> Expanded loc #2
-    //     108 -> Expanded loc #1
-    //     110 -> SourceLocation()
-    //
-    // Since re-lexed macro chunks will always be the same size or less of
-    // previous chunks, we only need to find where the ending of the new macro
-    // chunk is mapped to and update the map with new begin/end mappings.
-
-    MacroArgsMap::iterator I = MacroArgsCache.upper_bound(EndOffs);
-    --I;
-    SourceLocation EndOffsMappedLoc = I->second;
-    MacroArgsCache[BeginOffs] = SourceLocation::getMacroLoc(Entry.getOffset());
-    MacroArgsCache[EndOffs] = EndOffsMappedLoc;
   }
+
+  assert(SpellLoc.isFileID());
+
+  unsigned BeginOffs;
+  if (!isInFileID(SpellLoc, FID, &BeginOffs))
+    return;
+
+  unsigned EndOffs = BeginOffs + ExpansionLength;
+
+  // Add a new chunk for this macro argument. A previous macro argument chunk
+  // may have been lexed again, so e.g. if the map is
+  //     0   -> SourceLocation()
+  //     100 -> Expanded loc #1
+  //     110 -> SourceLocation()
+  // and we found a new macro FileID that lexed from offet 105 with length 3,
+  // the new map will be:
+  //     0   -> SourceLocation()
+  //     100 -> Expanded loc #1
+  //     105 -> Expanded loc #2
+  //     108 -> Expanded loc #1
+  //     110 -> SourceLocation()
+  //
+  // Since re-lexed macro chunks will always be the same size or less of
+  // previous chunks, we only need to find where the ending of the new macro
+  // chunk is mapped to and update the map with new begin/end mappings.
+
+  MacroArgsMap::iterator I = MacroArgsCache.upper_bound(EndOffs);
+  --I;
+  SourceLocation EndOffsMappedLoc = I->second;
+  MacroArgsCache[BeginOffs] = ExpansionLoc;
+  MacroArgsCache[EndOffs] = EndOffsMappedLoc;
 }
 
 /// \brief If \arg Loc points inside a function macro argument, the returned
diff --git a/lib/Basic/TargetInfo.cpp b/lib/Basic/TargetInfo.cpp
index db5941a..83d4e2b 100644
--- a/lib/Basic/TargetInfo.cpp
+++ b/lib/Basic/TargetInfo.cpp
@@ -24,7 +24,8 @@ using namespace clang;
 static const LangAS::Map DefaultAddrSpaceMap = { 0 };
 
 // TargetInfo Constructor.
-TargetInfo::TargetInfo(const std::string &T) : Triple(T) {
+TargetInfo::TargetInfo(const std::string &T) : TargetOpts(), Triple(T)
+{
   // Set defaults.  Defaults are set for a 32-bit RISC platform, like PPC or
   // SPARC.  These should be overridden by concrete targets as needed.
   BigEndian = true;
@@ -59,6 +60,7 @@ TargetInfo::TargetInfo(const std::string &T) : Triple(T) {
   Char32Type = UnsignedInt;
   Int64Type = SignedLongLong;
   SigAtomicType = SignedInt;
+  ProcessIDType = SignedInt;
   UseSignedCharForObjCBool = true;
   UseBitFieldTypeAlignment = true;
   UseZeroLengthBitfieldAlignment = false;
@@ -363,6 +365,8 @@ bool TargetInfo::validateOutputConstraint(ConstraintInfo &Info) const {
       break;
     case '?': // Disparage slightly code.
     case '!': // Disparage severely.
+    case '#': // Ignore as constraint.
+    case '*': // Ignore for choosing register preferences.
       break;  // Pass them.
     }
 
@@ -482,6 +486,8 @@ bool TargetInfo::validateInputConstraint(ConstraintInfo *OutputConstraints,
       break;
     case '?': // Disparage slightly code.
     case '!': // Disparage severely.
+    case '#': // Ignore as constraint.
+    case '*': // Ignore for choosing register preferences.
       break;  // Pass them.
     }
 
diff --git a/lib/Basic/Targets.cpp b/lib/Basic/Targets.cpp
index 1d495f1..f36ef82 100644
--- a/lib/Basic/Targets.cpp
+++ b/lib/Basic/Targets.cpp
@@ -92,6 +92,9 @@ static void getDarwinDefines(MacroBuilder &Builder, const LangOptions &Opts,
   Builder.defineMacro("__APPLE__");
   Builder.defineMacro("__MACH__");
   Builder.defineMacro("OBJC_NEW_PROPERTIES");
+  // AddressSanitizer doesn't play well with source fortification, which is on
+  // by default on Darwin.
+  if (Opts.SanitizeAddress) Builder.defineMacro("_FORTIFY_SOURCE", "0");
 
   if (!Opts.ObjCAutoRefCount) {
     // __weak is always defined, for use in blocks and with objc pointers.
@@ -316,7 +319,7 @@ protected:
     DefineStd(Builder, "linux", Opts);
     Builder.defineMacro("__gnu_linux__");
     Builder.defineMacro("__ELF__");
-    if (Triple.getEnvironment() == llvm::Triple::ANDROIDEABI)
+    if (Triple.getEnvironment() == llvm::Triple::Android)
       Builder.defineMacro("__ANDROID__", "1");
     if (Opts.POSIXThreads)
       Builder.defineMacro("_REENTRANT");
@@ -587,6 +590,48 @@ public:
     : OSTargetInfo<Target>(triple) {}
 };
 
+template <typename Target>
+class NaClTargetInfo : public OSTargetInfo<Target> {
+ protected:
+  virtual void getOSDefines(const LangOptions &Opts, const llvm::Triple &Triple,
+                            MacroBuilder &Builder) const {
+    if (Opts.POSIXThreads)
+      Builder.defineMacro("_REENTRANT");
+    if (Opts.CPlusPlus)
+      Builder.defineMacro("_GNU_SOURCE");
+
+    DefineStd(Builder, "unix", Opts);
+    Builder.defineMacro("__ELF__");
+    Builder.defineMacro("__native_client__");
+  }
+ public:
+  NaClTargetInfo(const std::string &triple)
+    : OSTargetInfo<Target>(triple) {
+    this->UserLabelPrefix = "";
+    this->LongAlign = 32;
+    this->LongWidth = 32;
+    this->PointerAlign = 32;
+    this->PointerWidth = 32;
+    this->IntMaxType = TargetInfo::SignedLongLong;
+    this->UIntMaxType = TargetInfo::UnsignedLongLong;
+    this->Int64Type = TargetInfo::SignedLongLong;
+    this->DoubleAlign = 64;
+    this->LongDoubleWidth = 64;
+    this->LongDoubleAlign = 64;
+    this->SizeType = TargetInfo::UnsignedInt;
+    this->PtrDiffType = TargetInfo::SignedInt;
+    this->IntPtrType = TargetInfo::SignedInt;
+    this->RegParmMax = 2;
+    this->LongDoubleFormat = &llvm::APFloat::IEEEdouble;
+    this->DescriptionString = "e-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-"
+                              "f32:32:32-f64:64:64-p:32:32:32-v128:32:32";
+  }
+  virtual typename Target::CallingConvCheckResult checkCallingConvention(
+      CallingConv CC) const {
+    return CC == CC_PnaclCall ? Target::CCCR_OK :
+        Target::checkCallingConvention(CC);
+  }
+};
 } // end anonymous namespace.
 
 //===----------------------------------------------------------------------===//
@@ -641,6 +686,8 @@ public:
       .Case("970", true)
       .Case("g5", true)
       .Case("a2", true)
+      .Case("e500mc", true)
+      .Case("e5500", true)
       .Case("pwr6", true)
       .Case("pwr7", true)
       .Case("ppc", true)
@@ -990,6 +1037,9 @@ public:
       LongDoubleWidth = LongDoubleAlign = 64;
       LongDoubleFormat = &llvm::APFloat::IEEEdouble;
     }
+
+    // PPC32 supports atomics up to 4 bytes.
+    MaxAtomicPromoteWidth = MaxAtomicInlineWidth = 32;
   }
 
   virtual BuiltinVaListKind getBuiltinVaListKind() const {
@@ -1007,13 +1057,20 @@ public:
     IntMaxType = SignedLong;
     UIntMaxType = UnsignedLong;
     Int64Type = SignedLong;
-    DescriptionString = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-"
-                        "i64:64:64-f32:32:32-f64:64:64-v128:128:128-n32:64";
 
     if (getTriple().getOS() == llvm::Triple::FreeBSD) {
       LongDoubleWidth = LongDoubleAlign = 64;
       LongDoubleFormat = &llvm::APFloat::IEEEdouble;
-    }
+      DescriptionString = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-"
+                          "i64:64:64-f32:32:32-f64:64:64-f128:64:64-"
+                          "v128:128:128-n32:64";
+    } else
+      DescriptionString = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-"
+                          "i64:64:64-f32:32:32-f64:64:64-f128:128:128-"
+                          "v128:128:128-n32:64";
+
+    // PPC64 supports atomics up to 8 bytes.
+    MaxAtomicPromoteWidth = MaxAtomicInlineWidth = 64;
   }
   virtual BuiltinVaListKind getBuiltinVaListKind() const {
     return TargetInfo::CharPtrBuiltinVaList;
@@ -1047,6 +1104,8 @@ public:
     : DarwinTargetInfo<PPC64TargetInfo>(triple) {
     HasAlignMac68kSupport = true;
     SuitableAlign = 128;
+    DescriptionString = "E-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-"
+                        "i64:64:64-f32:32:32-f64:64:64-v128:128:128-n32:64";
   }
 };
 } // end anonymous namespace.
@@ -1172,6 +1231,71 @@ namespace {
 }
 
 namespace {
+
+static const unsigned R600AddrSpaceMap[] = {
+  1,    // opencl_global
+  3,    // opencl_local
+  2,    // opencl_constant
+  1,    // cuda_device
+  2,    // cuda_constant
+  3     // cuda_shared
+};
+
+class R600TargetInfo : public TargetInfo {
+public:
+  R600TargetInfo(const std::string& triple) : TargetInfo(triple) {
+    DescriptionString =
+          "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16"
+          "-i32:32:32-i64:64:64-f32:32:32-f64:64:64-f80:32:32"
+          "-v16:16:16-v24:32:32-v32:32:32-v48:64:64-v64:64:64"
+          "-v96:128:128-v128:128:128-v192:256:256-v256:256:256"
+          "-v512:512:512-v1024:1024:1024-v2048:2048:2048"
+          "-n8:16:32:64";
+    AddrSpaceMap = &R600AddrSpaceMap;
+  }
+
+  virtual const char * getClobbers() const {
+    return "";
+  }
+
+  virtual void getGCCRegNames(const char * const *&Names,
+                              unsigned &numNames) const  {
+    Names = NULL;
+    numNames = 0;
+  }
+
+  virtual void getGCCRegAliases(const GCCRegAlias *&Aliases,
+                                unsigned &NumAliases) const {
+    Aliases = NULL;
+    NumAliases = 0;
+  }
+
+  virtual bool validateAsmConstraint(const char *&Name,
+                                     TargetInfo::ConstraintInfo &info) const {
+    return true;
+  }
+
+  virtual void getTargetBuiltins(const Builtin::Info *&Records,
+                                 unsigned &NumRecords) const {
+    Records = NULL;
+    NumRecords = 0;
+  }
+
+
+  virtual void getTargetDefines(const LangOptions &Opts,
+                                MacroBuilder &Builder) const {
+    Builder.defineMacro("__R600__");
+  }
+
+  virtual BuiltinVaListKind getBuiltinVaListKind() const {
+    return TargetInfo::CharPtrBuiltinVaList;
+  }
+
+};
+
+} // end anonymous namespace
+
+namespace {
 // MBlaze abstract base class
 class MBlazeTargetInfo : public TargetInfo {
   static const char * const GCCRegNames[];
@@ -1351,10 +1475,12 @@ class X86TargetInfo : public TargetInfo {
   bool HasBMI;
   bool HasBMI2;
   bool HasPOPCNT;
+  bool HasRTM;
   bool HasSSE4a;
   bool HasFMA4;
   bool HasFMA;
   bool HasXOP;
+  bool HasF16C;
 
   /// \brief Enumeration of all of the X86 CPUs supported by Clang.
   ///
@@ -1500,8 +1626,9 @@ public:
   X86TargetInfo(const std::string& triple)
     : TargetInfo(triple), SSELevel(NoSSE), MMX3DNowLevel(NoMMX3DNow),
       HasAES(false), HasPCLMUL(false), HasLZCNT(false), HasRDRND(false),
-      HasBMI(false), HasBMI2(false), HasPOPCNT(false), HasSSE4a(false),
-      HasFMA4(false), HasFMA(false), HasXOP(false), CPU(CK_Generic) {
+      HasBMI(false), HasBMI2(false), HasPOPCNT(false), HasRTM(false),
+      HasSSE4a(false), HasFMA4(false), HasFMA(false), HasXOP(false),
+      HasF16C(false), CPU(CK_Generic) {
     BigEndian = false;
     LongDoubleFormat = &llvm::APFloat::x87DoubleExtended;
   }
@@ -1544,9 +1671,10 @@ public:
   virtual bool hasFeature(StringRef Feature) const;
   virtual void HandleTargetFeatures(std::vector<std::string> &Features);
   virtual const char* getABI() const {
-    if (PointerWidth == 64 && SSELevel >= AVX)
+    if (getTriple().getArch() == llvm::Triple::x86_64 && SSELevel >= AVX)
       return "avx";
-    else if (PointerWidth == 32 && MMX3DNowLevel == NoMMX3DNow)
+    else if (getTriple().getArch() == llvm::Triple::x86 &&
+             MMX3DNowLevel == NoMMX3DNow)
       return "no-mmx";
     return "";
   }
@@ -1640,7 +1768,7 @@ public:
     case CK_AthlonMP:
     case CK_Geode:
       // Only accept certain architectures when compiling in 32-bit mode.
-      if (PointerWidth != 32)
+      if (getTriple().getArch() != llvm::Triple::x86)
         return false;
 
       // Fallthrough
@@ -1668,6 +1796,19 @@ public:
     }
     llvm_unreachable("Unhandled CPU kind");
   }
+
+  virtual CallingConvCheckResult checkCallingConvention(CallingConv CC) const {
+    // We accept all non-ARM calling conventions
+    return (CC == CC_X86ThisCall ||
+            CC == CC_X86FastCall ||
+            CC == CC_X86StdCall || 
+            CC == CC_C || 
+            CC == CC_X86Pascal) ? CCCR_OK : CCCR_Warning;
+  }
+
+  virtual CallingConv getDefaultCallingConv() const {
+    return CC_C;
+  }
 };
 
 void X86TargetInfo::getDefaultFeatures(llvm::StringMap<bool> &Features) const {
@@ -1691,14 +1832,16 @@ void X86TargetInfo::getDefaultFeatures(llvm::StringMap<bool> &Features) const {
   Features["bmi"] = false;
   Features["bmi2"] = false;
   Features["popcnt"] = false;
+  Features["rtm"] = false;
   Features["fma4"] = false;
   Features["fma"] = false;
   Features["xop"] = false;
+  Features["f16c"] = false;
 
   // FIXME: This *really* should not be here.
 
   // X86_64 always has SSE2.
-  if (PointerWidth == 64)
+  if (getTriple().getArch() == llvm::Triple::x86_64)
     Features["sse2"] = Features["sse"] = Features["mmx"] = true;
 
   switch (CPU) {
@@ -1770,6 +1913,7 @@ void X86TargetInfo::getDefaultFeatures(llvm::StringMap<bool> &Features) const {
     setFeatureEnabled(Features, "rdrnd", true);
     setFeatureEnabled(Features, "bmi", true);
     setFeatureEnabled(Features, "bmi2", true);
+    setFeatureEnabled(Features, "rtm", true);
     setFeatureEnabled(Features, "fma", true);
     break;
   case CK_K6:
@@ -1904,6 +2048,10 @@ bool X86TargetInfo::setFeatureEnabled(llvm::StringMap<bool> &Features,
       Features["bmi2"] = true;
     else if (Name == "popcnt")
       Features["popcnt"] = true;
+    else if (Name == "f16c")
+      Features["f16c"] = true;
+    else if (Name == "rtm")
+      Features["rtm"] = true;
   } else {
     if (Name == "mmx")
       Features["mmx"] = Features["3dnow"] = Features["3dnowa"] = false;
@@ -1964,6 +2112,10 @@ bool X86TargetInfo::setFeatureEnabled(llvm::StringMap<bool> &Features,
       Features["fma4"] = Features["xop"] = false;
     else if (Name == "xop")
       Features["xop"] = false;
+    else if (Name == "f16c")
+      Features["f16c"] = false;
+    else if (Name == "rtm")
+      Features["rtm"] = false;
   }
 
   return true;
@@ -2015,6 +2167,11 @@ void X86TargetInfo::HandleTargetFeatures(std::vector<std::string> &Features) {
       continue;
     }
 
+    if (Feature == "rtm") {
+      HasRTM = true;
+      continue;
+    }
+
     if (Feature == "sse4a") {
       HasSSE4a = true;
       continue;
@@ -2035,6 +2192,11 @@ void X86TargetInfo::HandleTargetFeatures(std::vector<std::string> &Features) {
       continue;
     }
 
+    if (Feature == "f16c") {
+      HasF16C = true;
+      continue;
+    }
+
     assert(Features[i][0] == '+' && "Invalid target feature!");
     X86SSEEnum Level = llvm::StringSwitch<X86SSEEnum>(Feature)
       .Case("avx2", AVX2)
@@ -2071,7 +2233,7 @@ void X86TargetInfo::HandleTargetFeatures(std::vector<std::string> &Features) {
 void X86TargetInfo::getTargetDefines(const LangOptions &Opts,
                                      MacroBuilder &Builder) const {
   // Target identification.
-  if (PointerWidth == 64) {
+  if (getTriple().getArch() == llvm::Triple::x86_64) {
     Builder.defineMacro("__amd64__");
     Builder.defineMacro("__amd64");
     Builder.defineMacro("__x86_64");
@@ -2231,6 +2393,9 @@ void X86TargetInfo::getTargetDefines(const LangOptions &Opts,
   if (HasPOPCNT)
     Builder.defineMacro("__POPCNT__");
 
+  if (HasRTM)
+    Builder.defineMacro("__RTM__");
+
   if (HasSSE4a)
     Builder.defineMacro("__SSE4A__");
 
@@ -2243,6 +2408,9 @@ void X86TargetInfo::getTargetDefines(const LangOptions &Opts,
   if (HasXOP)
     Builder.defineMacro("__XOP__");
 
+  if (HasF16C)
+    Builder.defineMacro("__F16C__");
+
   // Each case falls through to the previous one here.
   switch (SSELevel) {
   case AVX2:
@@ -2267,7 +2435,7 @@ void X86TargetInfo::getTargetDefines(const LangOptions &Opts,
     break;
   }
 
-  if (Opts.MicrosoftExt && PointerWidth == 32) {
+  if (Opts.MicrosoftExt && getTriple().getArch() == llvm::Triple::x86) {
     switch (SSELevel) {
     case AVX2:
     case AVX:
@@ -2315,6 +2483,7 @@ bool X86TargetInfo::hasFeature(StringRef Feature) const {
       .Case("mmx", MMX3DNowLevel >= MMX)
       .Case("pclmul", HasPCLMUL)
       .Case("popcnt", HasPOPCNT)
+      .Case("rtm", HasRTM)
       .Case("sse", SSELevel >= SSE1)
       .Case("sse2", SSELevel >= SSE2)
       .Case("sse3", SSELevel >= SSE3)
@@ -2323,9 +2492,10 @@ bool X86TargetInfo::hasFeature(StringRef Feature) const {
       .Case("sse42", SSELevel >= SSE42)
       .Case("sse4a", HasSSE4a)
       .Case("x86", true)
-      .Case("x86_32", PointerWidth == 32)
-      .Case("x86_64", PointerWidth == 64)
+      .Case("x86_32", getTriple().getArch() == llvm::Triple::x86)
+      .Case("x86_64", getTriple().getArch() == llvm::Triple::x86_64)
       .Case("xop", HasXOP)
+      .Case("f16c", HasF16C)
       .Default(false);
 }
 
@@ -2595,7 +2765,9 @@ public:
     SizeType = UnsignedLong;
     IntPtrType = SignedLong;
     PtrDiffType = SignedLong;
+    ProcessIDType = SignedLong;
     this->UserLabelPrefix = "";
+    this->TLSSupported = false;
   }
   virtual void getTargetDefines(const LangOptions &Opts,
                                 MacroBuilder &Builder) const {
@@ -2703,6 +2875,15 @@ public:
     if (RegNo == 1) return 1;
     return -1;
   }
+
+  virtual CallingConvCheckResult checkCallingConvention(CallingConv CC) const {
+    return TargetInfo::checkCallingConvention(CC);
+  }
+
+  virtual CallingConv getDefaultCallingConv() const {
+    return CC_Default;
+  }
+
 };
 } // end anonymous namespace
 
@@ -2820,14 +3001,14 @@ namespace {
 class ARMTargetInfo : public TargetInfo {
   // Possible FPU choices.
   enum FPUMode {
-    NoFPU,
-    VFP2FPU,
-    VFP3FPU,
-    NeonFPU
+    VFP2FPU = (1 << 0),
+    VFP3FPU = (1 << 1),
+    VFP4FPU = (1 << 2),
+    NeonFPU = (1 << 3)
   };
 
   static bool FPUModeIsVFP(FPUMode Mode) {
-    return Mode >= VFP2FPU && Mode <= NeonFPU;
+    return Mode & (VFP2FPU | VFP3FPU | VFP4FPU | NeonFPU);
   }
 
   static const TargetInfo::GCCRegAlias GCCRegAliases[];
@@ -2835,8 +3016,9 @@ class ARMTargetInfo : public TargetInfo {
 
   std::string ABI, CPU;
 
-  unsigned FPU : 3;
+  unsigned FPU : 4;
 
+  unsigned IsAAPCS : 1;
   unsigned IsThumb : 1;
 
   // Initialized via features.
@@ -2847,7 +3029,7 @@ class ARMTargetInfo : public TargetInfo {
 
 public:
   ARMTargetInfo(const std::string &TripleStr)
-    : TargetInfo(TripleStr), ABI("aapcs-linux"), CPU("arm1136j-s")
+    : TargetInfo(TripleStr), ABI("aapcs-linux"), CPU("arm1136j-s"), IsAAPCS(true)
   {
     BigEndian = false;
     SizeType = UnsignedInt;
@@ -2910,6 +3092,8 @@ public:
       /// gcc.
       ZeroLengthBitfieldBoundary = 32;
 
+      IsAAPCS = false;
+
       if (IsThumb) {
         // Thumb1 add sp, #imm requires the immediate value be multiple of 4,
         // so set preferred for small types to 32.
@@ -2923,10 +3107,11 @@ public:
       }
 
       // FIXME: Override "preferred align" for double and long long.
-    } else if (Name == "aapcs") {
+    } else if (Name == "aapcs" || Name == "aapcs-vfp") {
+      IsAAPCS = true;
       // FIXME: Enumerated types are variable width in straight AAPCS.
     } else if (Name == "aapcs-linux") {
-      ;
+      IsAAPCS = true;
     } else
       return false;
 
@@ -2936,16 +3121,21 @@ public:
   void getDefaultFeatures(llvm::StringMap<bool> &Features) const {
     if (CPU == "arm1136jf-s" || CPU == "arm1176jzf-s" || CPU == "mpcore")
       Features["vfp2"] = true;
-    else if (CPU == "cortex-a8" || CPU == "cortex-a9")
+    else if (CPU == "cortex-a8" || CPU == "cortex-a15" ||
+             CPU == "cortex-a9" || CPU == "cortex-a9-mp")
+      Features["neon"] = true;
+    else if (CPU == "swift") {
+      Features["vfp4"] = true;
       Features["neon"] = true;
+    }
   }
 
   virtual bool setFeatureEnabled(llvm::StringMap<bool> &Features,
                                  StringRef Name,
                                  bool Enabled) const {
     if (Name == "soft-float" || Name == "soft-float-abi" ||
-        Name == "vfp2" || Name == "vfp3" || Name == "neon" || Name == "d16" ||
-        Name == "neonfp") {
+        Name == "vfp2" || Name == "vfp3" || Name == "vfp4" || Name == "neon" ||
+        Name == "d16" || Name == "neonfp") {
       Features[Name] = Enabled;
     } else
       return false;
@@ -2954,7 +3144,7 @@ public:
   }
 
   virtual void HandleTargetFeatures(std::vector<std::string> &Features) {
-    FPU = NoFPU;
+    FPU = 0;
     SoftFloat = SoftFloatABI = false;
     for (unsigned i = 0, e = Features.size(); i != e; ++i) {
       if (Features[i] == "+soft-float")
@@ -2962,11 +3152,13 @@ public:
       else if (Features[i] == "+soft-float-abi")
         SoftFloatABI = true;
       else if (Features[i] == "+vfp2")
-        FPU = VFP2FPU;
+        FPU |= VFP2FPU;
       else if (Features[i] == "+vfp3")
-        FPU = VFP3FPU;
+        FPU |= VFP3FPU;
+      else if (Features[i] == "+vfp4")
+        FPU |= VFP4FPU;
       else if (Features[i] == "+neon")
-        FPU = NeonFPU;
+        FPU |= NeonFPU;
     }
 
     // Remove front-end specific options which the backend handles differently.
@@ -2988,6 +3180,7 @@ public:
               StringRef(getCPUDefineSuffix(CPU)).startswith("7"))    
         .Default(false);
   }
+  // FIXME: Should we actually have some table instead of these switches?
   static const char *getCPUDefineSuffix(StringRef Name) {
     return llvm::StringSwitch<const char*>(Name)
       .Cases("arm8", "arm810", "4")
@@ -3004,12 +3197,19 @@ public:
       .Cases("arm1176jz-s", "arm1176jzf-s", "6ZK")
       .Cases("arm1136jf-s", "mpcorenovfp", "mpcore", "6K")
       .Cases("arm1156t2-s", "arm1156t2f-s", "6T2")
-      .Cases("cortex-a8", "cortex-a9", "7A")
-      .Case("cortex-m3", "7M")
-      .Case("cortex-m4", "7M")
+      .Cases("cortex-a8", "cortex-a9", "cortex-a15", "7A")
+      .Case("cortex-a9-mp", "7F")
+      .Case("swift", "7S")
+      .Cases("cortex-m3", "cortex-m4", "7M")
       .Case("cortex-m0", "6M")
       .Default(0);
   }
+  static const char *getCPUProfile(StringRef Name) {
+    return llvm::StringSwitch<const char*>(Name)
+      .Cases("cortex-a8", "cortex-a9", "A")
+      .Cases("cortex-m3", "cortex-m4", "cortex-m0", "M")
+      .Default("");
+  }
   virtual bool setCPU(const std::string &Name) {
     if (!getCPUDefineSuffix(Name))
       return false;
@@ -3030,7 +3230,11 @@ public:
 
     StringRef CPUArch = getCPUDefineSuffix(CPU);
     Builder.defineMacro("__ARM_ARCH_" + CPUArch + "__");
-
+    Builder.defineMacro("__ARM_ARCH", CPUArch.substr(0, 1));
+    StringRef CPUProfile = getCPUProfile(CPU);
+    if (!CPUProfile.empty())
+      Builder.defineMacro("__ARM_ARCH_PROFILE", CPUProfile);
+    
     // Subtarget options.
 
     // FIXME: It's more complicated than this and we don't really support
@@ -3038,8 +3242,15 @@ public:
     if ('5' <= CPUArch[0] && CPUArch[0] <= '7')
       Builder.defineMacro("__THUMB_INTERWORK__");
 
-    if (ABI == "aapcs" || ABI == "aapcs-linux")
-      Builder.defineMacro("__ARM_EABI__");
+    if (ABI == "aapcs" || ABI == "aapcs-linux" || ABI == "aapcs-vfp") {
+      // M-class CPUs on Darwin follow AAPCS, but not EABI.
+      if (!(getTriple().isOSDarwin() && CPUProfile == "M"))
+        Builder.defineMacro("__ARM_EABI__");
+      Builder.defineMacro("__ARM_PCS", "1");
+
+      if ((!SoftFloat && !SoftFloatABI) || ABI == "aapcs-vfp")
+        Builder.defineMacro("__ARM_PCS_VFP", "1");
+    }
 
     if (SoftFloat)
       Builder.defineMacro("__SOFTFP__");
@@ -3058,14 +3269,21 @@ public:
     // Note, this is always on in gcc, even though it doesn't make sense.
     Builder.defineMacro("__APCS_32__");
 
-    if (FPUModeIsVFP((FPUMode) FPU))
+    if (FPUModeIsVFP((FPUMode) FPU)) {
       Builder.defineMacro("__VFP_FP__");
-
+      if (FPU & VFP2FPU)
+        Builder.defineMacro("__ARM_VFPV2__");
+      if (FPU & VFP3FPU)
+        Builder.defineMacro("__ARM_VFPV3__");
+      if (FPU & VFP4FPU)
+        Builder.defineMacro("__ARM_VFPV4__");
+    }
+    
     // This only gets set when Neon instructions are actually available, unlike
     // the VFP define, hence the soft float and arch check. This is subtly
     // different from gcc, we follow the intent which was that it should be set
     // when Neon instructions are actually available.
-    if (FPU == NeonFPU && !SoftFloat && IsARMv7)
+    if ((FPU & NeonFPU) && !SoftFloat && IsARMv7)
       Builder.defineMacro("__ARM_NEON__");
   }
   virtual void getTargetBuiltins(const Builtin::Info *&Records,
@@ -3075,7 +3293,7 @@ public:
   }
   virtual bool isCLZForZeroUndef() const { return false; }
   virtual BuiltinVaListKind getBuiltinVaListKind() const {
-    return TargetInfo::VoidPtrBuiltinVaList;
+    return IsAAPCS ? AAPCSABIBuiltinVaList : TargetInfo::VoidPtrBuiltinVaList;
   }
   virtual void getGCCRegNames(const char * const *&Names,
                               unsigned &NumNames) const;
@@ -3127,10 +3345,38 @@ public:
     }
     return R;
   }
+  virtual bool validateConstraintModifier(StringRef Constraint,
+                                          const char Modifier,
+                                          unsigned Size) const {
+    // Strip off constraint modifiers.
+    while (Constraint[0] == '=' ||
+           Constraint[0] == '+' ||
+           Constraint[0] == '&')
+      Constraint = Constraint.substr(1);
+
+    switch (Constraint[0]) {
+    default: break;
+    case 'r': {
+      switch (Modifier) {
+      default:
+        return Size == 32;
+      case 'q':
+        // A register of size 32 cannot fit a vector type.
+        return false;
+      }
+    }
+    }
+
+    return true;
+  }
   virtual const char *getClobbers() const {
     // FIXME: Is this really right?
     return "";
   }
+
+  virtual CallingConvCheckResult checkCallingConvention(CallingConv CC) const {
+    return (CC == CC_AAPCS || CC == CC_AAPCS_VFP) ? CCCR_OK : CCCR_Warning;
+  }
 };
 
 const char * const ARMTargetInfo::GCCRegNames[] = {
@@ -3701,8 +3947,12 @@ public:
     Features[CPU] = true;
   }
 
-  virtual void getArchDefines(const LangOptions &Opts,
-                              MacroBuilder &Builder) const {
+  virtual void getTargetDefines(const LangOptions &Opts,
+                                MacroBuilder &Builder) const {
+    DefineStd(Builder, "mips", Opts);
+    Builder.defineMacro("_mips");
+    Builder.defineMacro("__REGISTER_PREFIX__", "");
+
     switch (FloatABI) {
     case HardFloat:
       Builder.defineMacro("__mips_hard_float", Twine(1));
@@ -3736,10 +3986,11 @@ public:
     Builder.defineMacro("_MIPS_SZPTR", Twine(getPointerWidth(0)));
     Builder.defineMacro("_MIPS_SZINT", Twine(getIntWidth()));
     Builder.defineMacro("_MIPS_SZLONG", Twine(getLongWidth()));
+
+    Builder.defineMacro("_MIPS_ARCH", "\"" + CPU + "\"");
+    Builder.defineMacro("_MIPS_ARCH_" + StringRef(CPU).upper());
   }
 
-  virtual void getTargetDefines(const LangOptions &Opts,
-                                MacroBuilder &Builder) const = 0;
   virtual void getTargetBuiltins(const Builtin::Info *&Records,
                                  unsigned &NumRecords) const {
     Records = BuiltinInfo;
@@ -3859,9 +4110,9 @@ public:
     } else
       return false;
   }
-  virtual void getArchDefines(const LangOptions &Opts,
-                              MacroBuilder &Builder) const {
-    MipsTargetInfoBase::getArchDefines(Opts, Builder);
+  virtual void getTargetDefines(const LangOptions &Opts,
+                                MacroBuilder &Builder) const {
+    MipsTargetInfoBase::getTargetDefines(Opts, Builder);
 
     if (ABI == "o32") {
       Builder.defineMacro("__mips_o32");
@@ -3921,12 +4172,9 @@ public:
   }
   virtual void getTargetDefines(const LangOptions &Opts,
                                 MacroBuilder &Builder) const {
-    DefineStd(Builder, "mips", Opts);
-    Builder.defineMacro("_mips");
     DefineStd(Builder, "MIPSEB", Opts);
     Builder.defineMacro("_MIPSEB");
-    Builder.defineMacro("__REGISTER_PREFIX__", "");
-    getArchDefines(Opts, Builder);
+    Mips32TargetInfoBase::getTargetDefines(Opts, Builder);
   }
 };
 
@@ -3939,12 +4187,9 @@ public:
   }
   virtual void getTargetDefines(const LangOptions &Opts,
                                 MacroBuilder &Builder) const {
-    DefineStd(Builder, "mips", Opts);
-    Builder.defineMacro("_mips");
     DefineStd(Builder, "MIPSEL", Opts);
     Builder.defineMacro("_MIPSEL");
-    Builder.defineMacro("__REGISTER_PREFIX__", "");
-    getArchDefines(Opts, Builder);
+    Mips32TargetInfoBase::getTargetDefines(Opts, Builder);
   }
 };
 
@@ -3974,9 +4219,12 @@ public:
 
     return true;
   }
-  virtual void getArchDefines(const LangOptions &Opts,
-                              MacroBuilder &Builder) const {
-    MipsTargetInfoBase::getArchDefines(Opts, Builder);
+  virtual void getTargetDefines(const LangOptions &Opts,
+                                MacroBuilder &Builder) const {
+    MipsTargetInfoBase::getTargetDefines(Opts, Builder);
+
+    Builder.defineMacro("__mips64");
+    Builder.defineMacro("__mips64__");
 
     if (ABI == "n32") {
       Builder.defineMacro("__mips_n32");
@@ -4048,12 +4296,9 @@ public:
   }
   virtual void getTargetDefines(const LangOptions &Opts,
                                 MacroBuilder &Builder) const {
-    DefineStd(Builder, "mips", Opts);
-    Builder.defineMacro("_mips");
     DefineStd(Builder, "MIPSEB", Opts);
     Builder.defineMacro("_MIPSEB");
-    Builder.defineMacro("__REGISTER_PREFIX__", "");
-    getArchDefines(Opts, Builder);
+    Mips64TargetInfoBase::getTargetDefines(Opts, Builder);
   }
 };
 
@@ -4075,12 +4320,9 @@ public:
   }
   virtual void getTargetDefines(const LangOptions &Opts,
                                 MacroBuilder &Builder) const {
-    DefineStd(Builder, "mips", Opts);
-    Builder.defineMacro("_mips");
     DefineStd(Builder, "MIPSEL", Opts);
     Builder.defineMacro("_MIPSEL");
-    Builder.defineMacro("__REGISTER_PREFIX__", "");
-    getArchDefines(Opts, Builder);
+    Mips64TargetInfoBase::getTargetDefines(Opts, Builder);
   }
 };
 } // end anonymous namespace.
@@ -4118,15 +4360,7 @@ public:
   }
   virtual void getTargetDefines(const LangOptions &Opts,
                                 MacroBuilder &Builder) const {
-    DefineStd(Builder, "unix", Opts);
-    Builder.defineMacro("__ELF__");
-    if (Opts.POSIXThreads)
-      Builder.defineMacro("_REENTRANT");
-    if (Opts.CPlusPlus)
-      Builder.defineMacro("_GNU_SOURCE");
-
     Builder.defineMacro("__LITTLE_ENDIAN__");
-    Builder.defineMacro("__native_client__");
     getArchDefines(Opts, Builder);
   }
   virtual bool hasFeature(StringRef Feature) const {
@@ -4199,6 +4433,8 @@ static TargetInfo *AllocateTarget(const std::string &T) {
       return new BitrigTargetInfo<ARMTargetInfo>(T);
     case llvm::Triple::RTEMS:
       return new RTEMSTargetInfo<ARMTargetInfo>(T);
+    case llvm::Triple::NativeClient:
+      return new NaClTargetInfo<ARMTargetInfo>(T);
     default:
       return new ARMTargetInfo(T);
     }
@@ -4269,7 +4505,7 @@ static TargetInfo *AllocateTarget(const std::string &T) {
   case llvm::Triple::le32:
     switch (os) {
       case llvm::Triple::NativeClient:
-        return new PNaClTargetInfo(T);
+        return new NaClTargetInfo<PNaClTargetInfo>(T);
       default:
         return NULL;
     }
@@ -4316,6 +4552,9 @@ static TargetInfo *AllocateTarget(const std::string &T) {
   case llvm::Triple::mblaze:
     return new MBlazeTargetInfo(T);
 
+  case llvm::Triple::r600:
+    return new R600TargetInfo(T);
+
   case llvm::Triple::sparc:
     switch (os) {
     case llvm::Triple::Linux:
@@ -4374,6 +4613,8 @@ static TargetInfo *AllocateTarget(const std::string &T) {
       return new HaikuX86_32TargetInfo(T);
     case llvm::Triple::RTEMS:
       return new RTEMSX86_32TargetInfo(T);
+    case llvm::Triple::NativeClient:
+      return new NaClTargetInfo<X86_32TargetInfo>(T);
     default:
       return new X86_32TargetInfo(T);
     }
@@ -4403,6 +4644,8 @@ static TargetInfo *AllocateTarget(const std::string &T) {
       return new MinGWX86_64TargetInfo(T);
     case llvm::Triple::Win32:   // This is what Triple.h supports now.
       return new VisualStudioWindowsX86_64TargetInfo(T);
+    case llvm::Triple::NativeClient:
+      return new NaClTargetInfo<X86_64TargetInfo>(T);
     default:
       return new X86_64TargetInfo(T);
     }
@@ -4421,6 +4664,7 @@ TargetInfo *TargetInfo::CreateTargetInfo(DiagnosticsEngine &Diags,
     Diags.Report(diag::err_target_unknown_triple) << Triple.str();
     return 0;
   }
+  Target->setTargetOpts(Opts);
 
   // Set the target CPU if specified.
   if (!Opts.CPU.empty() && !Target->setCPU(Opts.CPU)) {
@@ -4447,8 +4691,10 @@ TargetInfo *TargetInfo::CreateTargetInfo(DiagnosticsEngine &Diags,
 
   // Apply the user specified deltas.
   // First the enables.
-  for (std::vector<std::string>::const_iterator it = Opts.Features.begin(),
-         ie = Opts.Features.end(); it != ie; ++it) {
+  for (std::vector<std::string>::const_iterator 
+         it = Opts.FeaturesAsWritten.begin(),
+         ie = Opts.FeaturesAsWritten.end(); 
+       it != ie; ++it) {
     const char *Name = it->c_str();
 
     if (Name[0] != '+')
@@ -4462,8 +4708,10 @@ TargetInfo *TargetInfo::CreateTargetInfo(DiagnosticsEngine &Diags,
   }
 
   // Then the disables.
-  for (std::vector<std::string>::const_iterator it = Opts.Features.begin(),
-         ie = Opts.Features.end(); it != ie; ++it) {
+  for (std::vector<std::string>::const_iterator 
+         it = Opts.FeaturesAsWritten.begin(),
+         ie = Opts.FeaturesAsWritten.end(); 
+       it != ie; ++it) {
     const char *Name = it->c_str();
 
     if (Name[0] == '+')
diff --git a/lib/Basic/Version.cpp b/lib/Basic/Version.cpp
index 9daa30a..286e96e 100644
--- a/lib/Basic/Version.cpp
+++ b/lib/Basic/Version.cpp
@@ -32,7 +32,7 @@ std::string getClangRepositoryPath() {
 
   // If the SVN_REPOSITORY is empty, try to use the SVN keyword. This helps us
   // pick up a tag in an SVN export, for example.
-  static StringRef SVNRepository("$URL: http://llvm.org/svn/llvm-project/cfe/trunk/lib/Basic/Version.cpp $");
+  static StringRef SVNRepository("$URL: http://llvm.org/svn/llvm-project/cfe/branches/release_32/lib/Basic/Version.cpp $");
   if (URL.empty()) {
     URL = SVNRepository.slice(SVNRepository.find(':'),
                               SVNRepository.find("/lib/Basic"));
diff --git a/lib/CodeGen/ABIInfo.h b/lib/CodeGen/ABIInfo.h
index 86f5380..da6d035 100644
--- a/lib/CodeGen/ABIInfo.h
+++ b/lib/CodeGen/ABIInfo.h
@@ -16,7 +16,7 @@
 namespace llvm {
   class Value;
   class LLVMContext;
-  class TargetData;
+  class DataLayout;
 }
 
 namespace clang {
@@ -70,46 +70,52 @@ namespace clang {
   private:
     Kind TheKind;
     llvm::Type *TypeData;
-    llvm::Type *PaddingType; // Currently allowed only for Direct.
+    llvm::Type *PaddingType;
     unsigned UIntData;
     bool BoolData0;
     bool BoolData1;
     bool InReg;
+    bool PaddingInReg;
 
     ABIArgInfo(Kind K, llvm::Type *TD, unsigned UI, bool B0, bool B1, bool IR,
-               llvm::Type* P)
+               bool PIR, llvm::Type* P)
       : TheKind(K), TypeData(TD), PaddingType(P), UIntData(UI), BoolData0(B0),
-        BoolData1(B1), InReg(IR) {}
+        BoolData1(B1), InReg(IR), PaddingInReg(PIR) {}
 
   public:
     ABIArgInfo() : TheKind(Direct), TypeData(0), UIntData(0) {}
 
     static ABIArgInfo getDirect(llvm::Type *T = 0, unsigned Offset = 0,
                                 llvm::Type *Padding = 0) {
-      return ABIArgInfo(Direct, T, Offset, false, false, false, Padding);
+      return ABIArgInfo(Direct, T, Offset, false, false, false, false, Padding);
     }
-    static ABIArgInfo getDirectInReg(llvm::Type *T) {
-      return ABIArgInfo(Direct, T, 0, false, false, true, 0);
+    static ABIArgInfo getDirectInReg(llvm::Type *T = 0) {
+      return ABIArgInfo(Direct, T, 0, false, false, true, false, 0);
     }
     static ABIArgInfo getExtend(llvm::Type *T = 0) {
-      return ABIArgInfo(Extend, T, 0, false, false, false, 0);
+      return ABIArgInfo(Extend, T, 0, false, false, false, false, 0);
     }
     static ABIArgInfo getExtendInReg(llvm::Type *T = 0) {
-      return ABIArgInfo(Extend, T, 0, false, false, true, 0);
+      return ABIArgInfo(Extend, T, 0, false, false, true, false, 0);
     }
     static ABIArgInfo getIgnore() {
-      return ABIArgInfo(Ignore, 0, 0, false, false, false, 0);
+      return ABIArgInfo(Ignore, 0, 0, false, false, false, false, 0);
     }
     static ABIArgInfo getIndirect(unsigned Alignment, bool ByVal = true
                                   , bool Realign = false) {
-      return ABIArgInfo(Indirect, 0, Alignment, ByVal, Realign, false, 0);
+      return ABIArgInfo(Indirect, 0, Alignment, ByVal, Realign, false, false, 0);
     }
     static ABIArgInfo getIndirectInReg(unsigned Alignment, bool ByVal = true
                                   , bool Realign = false) {
-      return ABIArgInfo(Indirect, 0, Alignment, ByVal, Realign, true, 0);
+      return ABIArgInfo(Indirect, 0, Alignment, ByVal, Realign, true, false, 0);
     }
     static ABIArgInfo getExpand() {
-      return ABIArgInfo(Expand, 0, 0, false, false, false, 0);
+      return ABIArgInfo(Expand, 0, 0, false, false, false, false, 0);
+    }
+    static ABIArgInfo getExpandWithPadding(bool PaddingInReg,
+                                           llvm::Type *Padding) {
+     return ABIArgInfo(Expand, 0, 0, false, false, false, PaddingInReg,
+                       Padding);
     }
 
     Kind getKind() const { return TheKind; }
@@ -133,6 +139,10 @@ namespace clang {
       return PaddingType;
     }
 
+    bool getPaddingInReg() const {
+      return PaddingInReg;
+    }
+
     llvm::Type *getCoerceToType() const {
       assert(canHaveCoerceToType() && "Invalid kind!");
       return TypeData;
@@ -178,7 +188,7 @@ namespace clang {
 
     ASTContext &getContext() const;
     llvm::LLVMContext &getVMContext() const;
-    const llvm::TargetData &getTargetData() const;
+    const llvm::DataLayout &getDataLayout() const;
 
     virtual void computeInfo(CodeGen::CGFunctionInfo &FI) const = 0;
 
diff --git a/lib/CodeGen/BackendUtil.cpp b/lib/CodeGen/BackendUtil.cpp
index 0a1915b..62f87c9 100644
--- a/lib/CodeGen/BackendUtil.cpp
+++ b/lib/CodeGen/BackendUtil.cpp
@@ -27,7 +27,7 @@
 #include "llvm/Support/TargetRegistry.h"
 #include "llvm/Support/Timer.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Target/TargetLibraryInfo.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Target/TargetOptions.h"
@@ -54,36 +54,67 @@ class EmitAssemblyHelper {
   mutable FunctionPassManager *PerFunctionPasses;
 
 private:
-  PassManager *getCodeGenPasses() const {
+  PassManager *getCodeGenPasses(TargetMachine *TM) const {
     if (!CodeGenPasses) {
       CodeGenPasses = new PassManager();
-      CodeGenPasses->add(new TargetData(TheModule));
+      CodeGenPasses->add(new DataLayout(TheModule));
+      // Add TargetTransformInfo.
+      if (TM) {
+        TargetTransformInfo *TTI =
+        new TargetTransformInfo(TM->getScalarTargetTransformInfo(),
+                                TM->getVectorTargetTransformInfo());
+        CodeGenPasses->add(TTI);
+      }
     }
     return CodeGenPasses;
   }
 
-  PassManager *getPerModulePasses() const {
+  PassManager *getPerModulePasses(TargetMachine *TM) const {
     if (!PerModulePasses) {
       PerModulePasses = new PassManager();
-      PerModulePasses->add(new TargetData(TheModule));
+      PerModulePasses->add(new DataLayout(TheModule));
+      if (TM) {
+        TargetTransformInfo *TTI =
+        new TargetTransformInfo(TM->getScalarTargetTransformInfo(),
+                                TM->getVectorTargetTransformInfo());
+        PerModulePasses->add(TTI);
+      }
     }
     return PerModulePasses;
   }
 
-  FunctionPassManager *getPerFunctionPasses() const {
+  FunctionPassManager *getPerFunctionPasses(TargetMachine *TM) const {
     if (!PerFunctionPasses) {
       PerFunctionPasses = new FunctionPassManager(TheModule);
-      PerFunctionPasses->add(new TargetData(TheModule));
+      PerFunctionPasses->add(new DataLayout(TheModule));
+      if (TM) {
+        TargetTransformInfo *TTI =
+        new TargetTransformInfo(TM->getScalarTargetTransformInfo(),
+                                TM->getVectorTargetTransformInfo());
+        PerFunctionPasses->add(TTI);
+      }
     }
     return PerFunctionPasses;
   }
 
-  void CreatePasses();
+
+  void CreatePasses(TargetMachine *TM);
+
+  /// CreateTargetMachine - Generates the TargetMachine.
+  /// Returns Null if it is unable to create the target machine.
+  /// Some of our clang tests specify triples which are not built
+  /// into clang. This is okay because these tests check the generated
+  /// IR, and they require DataLayout which depends on the triple.
+  /// In this case, we allow this method to fail and not report an error.
+  /// When MustCreateTM is used, we print an error if we are unable to load
+  /// the requested target.
+  TargetMachine *CreateTargetMachine(bool MustCreateTM);
 
   /// AddEmitPasses - Add passes necessary to emit assembly or LLVM IR.
   ///
   /// \return True on success.
-  bool AddEmitPasses(BackendAction Action, formatted_raw_ostream &OS);
+  bool AddEmitPasses(BackendAction Action, formatted_raw_ostream &OS,
+                     TargetMachine *TM);
 
 public:
   EmitAssemblyHelper(DiagnosticsEngine &_Diags,
@@ -137,9 +168,9 @@ static void addThreadSanitizerPass(const PassManagerBuilder &Builder,
   PM.add(createThreadSanitizerPass());
 }
 
-void EmitAssemblyHelper::CreatePasses() {
+void EmitAssemblyHelper::CreatePasses(TargetMachine *TM) {
   unsigned OptLevel = CodeGenOpts.OptimizationLevel;
-  CodeGenOptions::InliningMethod Inlining = CodeGenOpts.Inlining;
+  CodeGenOptions::InliningMethod Inlining = CodeGenOpts.getInlining();
 
   // Handle disabling of LLVM optimization, where we want to preserve the
   // internal module before any optimization.
@@ -174,14 +205,14 @@ void EmitAssemblyHelper::CreatePasses() {
                            addBoundsCheckingPass);
   }
 
-  if (LangOpts.AddressSanitizer) {
-    PMBuilder.addExtension(PassManagerBuilder::EP_ScalarOptimizerLate,
+  if (LangOpts.SanitizeAddress) {
+    PMBuilder.addExtension(PassManagerBuilder::EP_OptimizerLast,
                            addAddressSanitizerPass);
     PMBuilder.addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0,
                            addAddressSanitizerPass);
   }
 
-  if (LangOpts.ThreadSanitizer) {
+  if (LangOpts.SanitizeThread) {
     PMBuilder.addExtension(PassManagerBuilder::EP_OptimizerLast,
                            addThreadSanitizerPass);
     PMBuilder.addExtension(PassManagerBuilder::EP_EnabledOnOptLevel0,
@@ -218,38 +249,36 @@ void EmitAssemblyHelper::CreatePasses() {
     break;
   }
 
- 
   // Set up the per-function pass manager.
-  FunctionPassManager *FPM = getPerFunctionPasses();
+  FunctionPassManager *FPM = getPerFunctionPasses(TM);
   if (CodeGenOpts.VerifyModule)
     FPM->add(createVerifierPass());
   PMBuilder.populateFunctionPassManager(*FPM);
 
   // Set up the per-module pass manager.
-  PassManager *MPM = getPerModulePasses();
+  PassManager *MPM = getPerModulePasses(TM);
 
   if (CodeGenOpts.EmitGcovArcs || CodeGenOpts.EmitGcovNotes) {
     MPM->add(createGCOVProfilerPass(CodeGenOpts.EmitGcovNotes,
                                     CodeGenOpts.EmitGcovArcs,
                                     TargetTriple.isMacOSX()));
 
-    if (CodeGenOpts.DebugInfo == CodeGenOptions::NoDebugInfo)
+    if (CodeGenOpts.getDebugInfo() == CodeGenOptions::NoDebugInfo)
       MPM->add(createStripSymbolsPass(true));
   }
-  
-  
+
   PMBuilder.populateModulePassManager(*MPM);
 }
 
-bool EmitAssemblyHelper::AddEmitPasses(BackendAction Action,
-                                       formatted_raw_ostream &OS) {
+TargetMachine *EmitAssemblyHelper::CreateTargetMachine(bool MustCreateTM) {
   // Create the TargetMachine for generating code.
   std::string Error;
   std::string Triple = TheModule->getTargetTriple();
   const llvm::Target *TheTarget = TargetRegistry::lookupTarget(Triple, Error);
   if (!TheTarget) {
-    Diags.Report(diag::err_fe_unable_to_create_target) << Error;
-    return false;
+    if (MustCreateTM)
+      Diags.Report(diag::err_fe_unable_to_create_target) << Error;
+    return 0;
   }
 
   // FIXME: Expose these capabilities via actual APIs!!!! Aside from just
@@ -361,7 +390,7 @@ bool EmitAssemblyHelper::AddEmitPasses(BackendAction Action,
     break;
   case LangOptions::FPC_Fast:
     Options.AllowFPOpFusion = llvm::FPOpFusion::Fast;
-    break;              
+    break;
   }
 
   Options.LessPreciseFPMADOption = CodeGenOpts.LessPreciseFPMAD;
@@ -375,6 +404,7 @@ bool EmitAssemblyHelper::AddEmitPasses(BackendAction Action,
   Options.DisableTailCalls = CodeGenOpts.DisableTailCalls;
   Options.TrapFuncName = CodeGenOpts.TrapFuncName;
   Options.PositionIndependentExecutable = LangOpts.PIELevel != 0;
+  Options.SSPBufferSize = CodeGenOpts.SSPBufferSize;
 
   TargetMachine *TM = TheTarget->createTargetMachine(Triple, TargetOpts.CPU,
                                                      FeaturesStr, Options,
@@ -391,15 +421,27 @@ bool EmitAssemblyHelper::AddEmitPasses(BackendAction Action,
   if (CodeGenOpts.NoExecStack)
     TM->setMCNoExecStack(true);
 
+  return TM;
+}
+
+bool EmitAssemblyHelper::AddEmitPasses(BackendAction Action,
+                                       formatted_raw_ostream &OS,
+                                       TargetMachine *TM) {
+
   // Create the code generator passes.
-  PassManager *PM = getCodeGenPasses();
+  PassManager *PM = getCodeGenPasses(TM);
 
   // Add LibraryInfo.
-  TargetLibraryInfo *TLI = new TargetLibraryInfo();
+  llvm::Triple TargetTriple(TheModule->getTargetTriple());
+  TargetLibraryInfo *TLI = new TargetLibraryInfo(TargetTriple);
   if (!CodeGenOpts.SimplifyLibCalls)
     TLI->disableAllFunctions();
   PM->add(TLI);
 
+  // Add TargetTransformInfo.
+  PM->add(new TargetTransformInfo(TM->getScalarTargetTransformInfo(),
+                                  TM->getVectorTargetTransformInfo()));
+
   // Normal mode, emit a .s or .o file by running the code generator. Note,
   // this also adds codegenerator level optimization passes.
   TargetMachine::CodeGenFileType CGFT = TargetMachine::CGFT_AssemblyFile;
@@ -430,23 +472,28 @@ void EmitAssemblyHelper::EmitAssembly(BackendAction Action, raw_ostream *OS) {
   TimeRegion Region(llvm::TimePassesIsEnabled ? &CodeGenerationTime : 0);
   llvm::formatted_raw_ostream FormattedOS;
 
-  CreatePasses();
+  bool UsesCodeGen = (Action != Backend_EmitNothing &&
+                      Action != Backend_EmitBC &&
+                      Action != Backend_EmitLL);
+  TargetMachine *TM = CreateTargetMachine(UsesCodeGen);
+  CreatePasses(TM);
+
   switch (Action) {
   case Backend_EmitNothing:
     break;
 
   case Backend_EmitBC:
-    getPerModulePasses()->add(createBitcodeWriterPass(*OS));
+    getPerModulePasses(TM)->add(createBitcodeWriterPass(*OS));
     break;
 
   case Backend_EmitLL:
     FormattedOS.setStream(*OS, formatted_raw_ostream::PRESERVE_STREAM);
-    getPerModulePasses()->add(createPrintModulePass(&FormattedOS));
+    getPerModulePasses(TM)->add(createPrintModulePass(&FormattedOS));
     break;
 
   default:
     FormattedOS.setStream(*OS, formatted_raw_ostream::PRESERVE_STREAM);
-    if (!AddEmitPasses(Action, FormattedOS))
+    if (!AddEmitPasses(Action, FormattedOS, TM))
       return;
   }
 
diff --git a/lib/CodeGen/CGBlocks.cpp b/lib/CodeGen/CGBlocks.cpp
index 37ef4af..6742f36 100644
--- a/lib/CodeGen/CGBlocks.cpp
+++ b/lib/CodeGen/CGBlocks.cpp
@@ -19,7 +19,7 @@
 #include "clang/AST/DeclObjC.h"
 #include "llvm/Module.h"
 #include "llvm/ADT/SmallSet.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include <algorithm>
 
 using namespace clang;
@@ -27,7 +27,8 @@ using namespace CodeGen;
 
 CGBlockInfo::CGBlockInfo(const BlockDecl *block, StringRef name)
   : Name(name), CXXThisIndex(0), CanBeGlobal(false), NeedsCopyDispose(false),
-    HasCXXObject(false), UsesStret(false), StructureType(0), Block(block),
+    HasCXXObject(false), UsesStret(false), HasCapturedVariableLayout(false),
+    StructureType(0), Block(block),
     DominatingIP(0) {
     
   // Skip asm prefix, if any.  'name' is usually taken directly from
@@ -56,7 +57,18 @@ static llvm::Constant *buildDisposeHelper(CodeGenModule &CGM,
   return CodeGenFunction(CGM).GenerateDestroyHelperFunction(blockInfo);
 }
 
-/// Build the block descriptor constant for a block.
+/// buildBlockDescriptor - Build the block descriptor meta-data for a block.
+/// buildBlockDescriptor is accessed from 5th field of the Block_literal
+/// meta-data and contains stationary information about the block literal.
+/// Its definition will have 4 (or optinally 6) words.
+/// struct Block_descriptor {
+///   unsigned long reserved;
+///   unsigned long size;  // size of Block_literal metadata in bytes.
+///   void *copy_func_helper_decl;  // optional copy helper.
+///   void *destroy_func_decl; // optioanl destructor helper.
+///   void *block_method_encoding_address;//@encode for block literal signature.
+///   void *block_layout_info; // encoding of captured block variables.
+/// };
 static llvm::Constant *buildBlockDescriptor(CodeGenModule &CGM,
                                             const CGBlockInfo &blockInfo) {
   ASTContext &C = CGM.getContext();
@@ -92,8 +104,12 @@ static llvm::Constant *buildBlockDescriptor(CodeGenModule &CGM,
                           CGM.GetAddrOfConstantCString(typeAtEncoding), i8p));
   
   // GC layout.
-  if (C.getLangOpts().ObjC1)
-    elements.push_back(CGM.getObjCRuntime().BuildGCBlockLayout(CGM, blockInfo));
+  if (C.getLangOpts().ObjC1) {
+    if (CGM.getLangOpts().getGC() != LangOptions::NonGC)
+      elements.push_back(CGM.getObjCRuntime().BuildGCBlockLayout(CGM, blockInfo));
+    else
+      elements.push_back(CGM.getObjCRuntime().BuildRCBlockLayout(CGM, blockInfo));
+  }
   else
     elements.push_back(llvm::Constant::getNullValue(i8p));
 
@@ -293,7 +309,10 @@ static void computeBlockInfo(CodeGenModule &CGM, CodeGenFunction *CGF,
     info.CanBeGlobal = true;
     return;
   }
-
+  else if (C.getLangOpts().ObjC1 &&
+           CGM.getLangOpts().getGC() == LangOptions::NonGC)
+    info.HasCapturedVariableLayout = true;
+  
   // Collect the layout chunks.
   SmallVector<BlockLayoutChunk, 16> layout;
   layout.reserve(block->capturesCXXThis() +
@@ -652,6 +671,7 @@ llvm::Value *CodeGenFunction::EmitBlockLiteral(const CGBlockInfo &blockInfo) {
 
   // Compute the initial on-stack block flags.
   BlockFlags flags = BLOCK_HAS_SIGNATURE;
+  if (blockInfo.HasCapturedVariableLayout) flags |= BLOCK_HAS_EXTENDED_LAYOUT;
   if (blockInfo.NeedsCopyDispose) flags |= BLOCK_HAS_COPY_DISPOSE;
   if (blockInfo.HasCXXObject) flags |= BLOCK_HAS_CXX_OBJ;
   if (blockInfo.UsesStret) flags |= BLOCK_USE_STRET;
@@ -1001,8 +1021,7 @@ CodeGenFunction::GenerateBlockFunction(GlobalDecl GD,
   const BlockDecl *blockDecl = blockInfo.getBlockDecl();
 
   // Check if we should generate debug info for this block function.
-  if (CGM.getModuleDebugInfo())
-    DebugInfo = CGM.getModuleDebugInfo();
+  maybeInitializeDebugInfo();
   CurGD = GD;
   
   BlockInfo = &blockInfo;
@@ -1135,7 +1154,8 @@ CodeGenFunction::GenerateBlockFunction(GlobalDecl GD,
       const VarDecl *variable = ci->getVariable();
       DI->EmitLocation(Builder, variable->getLocation());
 
-      if (CGM.getCodeGenOpts().DebugInfo >= CodeGenOptions::LimitedDebugInfo) {
+      if (CGM.getCodeGenOpts().getDebugInfo()
+            >= CodeGenOptions::LimitedDebugInfo) {
         const CGBlockInfo::Capture &capture = blockInfo.getCapture(variable);
         if (capture.isConstant()) {
           DI->EmitDeclareOfAutoVariable(variable, LocalDeclMap[variable],
@@ -1207,8 +1227,7 @@ CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) {
     = &CGM.getContext().Idents.get("__copy_helper_block_");
 
   // Check if we should generate debug info for this block helper function.
-  if (CGM.getModuleDebugInfo())
-    DebugInfo = CGM.getModuleDebugInfo();
+  maybeInitializeDebugInfo();
 
   FunctionDecl *FD = FunctionDecl::Create(C,
                                           C.getTranslationUnitDecl(),
@@ -1243,7 +1262,8 @@ CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) {
     const Expr *copyExpr = ci->getCopyExpr();
     BlockFieldFlags flags;
 
-    bool isARCWeakCapture = false;
+    bool useARCWeakCopy = false;
+    bool useARCStrongCopy = false;
 
     if (copyExpr) {
       assert(!ci->isByRef());
@@ -1256,21 +1276,35 @@ CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) {
 
     } else if (type->isObjCRetainableType()) {
       flags = BLOCK_FIELD_IS_OBJECT;
-      if (type->isBlockPointerType())
+      bool isBlockPointer = type->isBlockPointerType();
+      if (isBlockPointer)
         flags = BLOCK_FIELD_IS_BLOCK;
 
       // Special rules for ARC captures:
       if (getLangOpts().ObjCAutoRefCount) {
         Qualifiers qs = type.getQualifiers();
 
-        // Don't generate special copy logic for a captured object
-        // unless it's __strong or __weak.
-        if (!qs.hasStrongOrWeakObjCLifetime())
+        // We need to register __weak direct captures with the runtime.
+        if (qs.getObjCLifetime() == Qualifiers::OCL_Weak) {
+          useARCWeakCopy = true;
+
+        // We need to retain the copied value for __strong direct captures.
+        } else if (qs.getObjCLifetime() == Qualifiers::OCL_Strong) {
+          // If it's a block pointer, we have to copy the block and
+          // assign that to the destination pointer, so we might as
+          // well use _Block_object_assign.  Otherwise we can avoid that.
+          if (!isBlockPointer)
+            useARCStrongCopy = true;
+
+        // Otherwise the memcpy is fine.
+        } else {
           continue;
+        }
 
-        // Support __weak direct captures.
-        if (qs.getObjCLifetime() == Qualifiers::OCL_Weak)
-          isARCWeakCapture = true;
+      // Non-ARC captures of retainable pointers are strong and
+      // therefore require a call to _Block_object_assign.
+      } else {
+        // fall through
       }
     } else {
       continue;
@@ -1283,14 +1317,36 @@ CodeGenFunction::GenerateCopyHelperFunction(const CGBlockInfo &blockInfo) {
     // If there's an explicit copy expression, we do that.
     if (copyExpr) {
       EmitSynthesizedCXXCopyCtor(dstField, srcField, copyExpr);
-    } else if (isARCWeakCapture) {
+    } else if (useARCWeakCopy) {
       EmitARCCopyWeak(dstField, srcField);
     } else {
       llvm::Value *srcValue = Builder.CreateLoad(srcField, "blockcopy.src");
-      srcValue = Builder.CreateBitCast(srcValue, VoidPtrTy);
-      llvm::Value *dstAddr = Builder.CreateBitCast(dstField, VoidPtrTy);
-      Builder.CreateCall3(CGM.getBlockObjectAssign(), dstAddr, srcValue,
-                          llvm::ConstantInt::get(Int32Ty, flags.getBitMask()));
+      if (useARCStrongCopy) {
+        // At -O0, store null into the destination field (so that the
+        // storeStrong doesn't over-release) and then call storeStrong.
+        // This is a workaround to not having an initStrong call.
+        if (CGM.getCodeGenOpts().OptimizationLevel == 0) {
+          llvm::PointerType *ty = cast<llvm::PointerType>(srcValue->getType());
+          llvm::Value *null = llvm::ConstantPointerNull::get(ty);
+          Builder.CreateStore(null, dstField);
+          EmitARCStoreStrongCall(dstField, srcValue, true);
+
+        // With optimization enabled, take advantage of the fact that
+        // the blocks runtime guarantees a memcpy of the block data, and
+        // just emit a retain of the src field.
+        } else {
+          EmitARCRetainNonBlock(srcValue);
+
+          // We don't need this anymore, so kill it.  It's not quite
+          // worth the annoyance to avoid creating it in the first place.
+          cast<llvm::Instruction>(dstField)->eraseFromParent();
+        }
+      } else {
+        srcValue = Builder.CreateBitCast(srcValue, VoidPtrTy);
+        llvm::Value *dstAddr = Builder.CreateBitCast(dstField, VoidPtrTy);
+        Builder.CreateCall3(CGM.getBlockObjectAssign(), dstAddr, srcValue,
+                            llvm::ConstantInt::get(Int32Ty, flags.getBitMask()));
+      }
     }
   }
 
@@ -1321,8 +1377,7 @@ CodeGenFunction::GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo) {
                            "__destroy_helper_block_", &CGM.getModule());
 
   // Check if we should generate debug info for this block destroy function.
-  if (CGM.getModuleDebugInfo())
-    DebugInfo = CGM.getModuleDebugInfo();
+  maybeInitializeDebugInfo();
 
   IdentifierInfo *II
     = &CGM.getContext().Idents.get("__destroy_helper_block_");
@@ -1356,7 +1411,8 @@ CodeGenFunction::GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo) {
     BlockFieldFlags flags;
     const CXXDestructorDecl *dtor = 0;
 
-    bool isARCWeakCapture = false;
+    bool useARCWeakDestroy = false;
+    bool useARCStrongDestroy = false;
 
     if (ci->isByRef()) {
       flags = BLOCK_FIELD_IS_BYREF;
@@ -1382,7 +1438,11 @@ CodeGenFunction::GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo) {
 
         // Support __weak direct captures.
         if (qs.getObjCLifetime() == Qualifiers::OCL_Weak)
-          isARCWeakCapture = true;
+          useARCWeakDestroy = true;
+
+        // Tools really want us to use objc_storeStrong here.
+        else
+          useARCStrongDestroy = true;
       }
     } else {
       continue;
@@ -1396,9 +1456,13 @@ CodeGenFunction::GenerateDestroyHelperFunction(const CGBlockInfo &blockInfo) {
       PushDestructorCleanup(dtor, srcField);
 
     // If this is a __weak capture, emit the release directly.
-    } else if (isARCWeakCapture) {
+    } else if (useARCWeakDestroy) {
       EmitARCDestroyWeak(srcField);
 
+    // Destroy strong objects with a call if requested.
+    } else if (useARCStrongDestroy) {
+      EmitARCDestroyStrong(srcField, /*precise*/ false);
+
     // Otherwise we call _Block_object_dispose.  It wouldn't be too
     // hard to just emit this as a cleanup if we wanted to make sure
     // that things were done in reverse.
@@ -1497,10 +1561,7 @@ public:
   }
 
   void emitDispose(CodeGenFunction &CGF, llvm::Value *field) {
-    llvm::LoadInst *value = CGF.Builder.CreateLoad(field);
-    value->setAlignment(Alignment.getQuantity());
-
-    CGF.EmitARCRelease(value, /*precise*/ false);
+    CGF.EmitARCDestroyStrong(field, /*precise*/ false);
   }
 
   void profileImpl(llvm::FoldingSetNodeID &id) const {
@@ -1530,10 +1591,7 @@ public:
   }
 
   void emitDispose(CodeGenFunction &CGF, llvm::Value *field) {
-    llvm::LoadInst *value = CGF.Builder.CreateLoad(field);
-    value->setAlignment(Alignment.getQuantity());
-
-    CGF.EmitARCRelease(value, /*precise*/ false);
+    CGF.EmitARCDestroyStrong(field, /*precise*/ false);
   }
 
   void profileImpl(llvm::FoldingSetNodeID &id) const {
@@ -1612,6 +1670,8 @@ generateByrefCopyHelper(CodeGenFunction &CGF,
                                           SC_None,
                                           false, false);
 
+  // Initialize debug info if necessary.
+  CGF.maybeInitializeDebugInfo();
   CGF.StartFunction(FD, R, Fn, FI, args, SourceLocation());
 
   if (byrefInfo.needsCopy()) {
@@ -1682,6 +1742,8 @@ generateByrefDisposeHelper(CodeGenFunction &CGF,
                                           SC_Static,
                                           SC_None,
                                           false, false);
+  // Initialize debug info if necessary.
+  CGF.maybeInitializeDebugInfo();
   CGF.StartFunction(FD, R, Fn, FI, args, SourceLocation());
 
   if (byrefInfo.needsDispose()) {
@@ -1879,7 +1941,7 @@ llvm::Type *CodeGenFunction::BuildByRefType(const VarDecl *D) {
     
     // And either 2 or 4 pointers.
     CurrentOffsetInBytes += (HasCopyAndDispose ? 4 : 2) *
-      CGM.getTargetData().getTypeAllocSize(Int8PtrTy);
+      CGM.getDataLayout().getTypeAllocSize(Int8PtrTy);
     
     // Align the offset.
     unsigned AlignedOffsetInBytes = 
diff --git a/lib/CodeGen/CGBlocks.h b/lib/CodeGen/CGBlocks.h
index 095cfdb..f85701a 100644
--- a/lib/CodeGen/CGBlocks.h
+++ b/lib/CodeGen/CGBlocks.h
@@ -33,7 +33,7 @@ namespace llvm {
   class Constant;
   class Function;
   class GlobalValue;
-  class TargetData;
+  class DataLayout;
   class FunctionType;
   class PointerType;
   class Value;
@@ -47,12 +47,24 @@ namespace CodeGen {
 class CodeGenModule;
 class CGBlockInfo;
 
-enum BlockFlag_t {
+// Flags stored in __block variables.
+enum BlockByrefFlags {
+  BLOCK_BYREF_HAS_COPY_DISPOSE         = (1   << 25), // compiler
+  BLOCK_BYREF_LAYOUT_MASK              = (0xF << 28), // compiler
+  BLOCK_BYREF_LAYOUT_EXTENDED          = (1   << 28),
+  BLOCK_BYREF_LAYOUT_NON_OBJECT        = (2   << 28),
+  BLOCK_BYREF_LAYOUT_STRONG            = (3   << 28),
+  BLOCK_BYREF_LAYOUT_WEAK              = (4   << 28),
+  BLOCK_BYREF_LAYOUT_UNRETAINED        = (5   << 28)
+};
+
+enum BlockLiteralFlags {
   BLOCK_HAS_COPY_DISPOSE =  (1 << 25),
   BLOCK_HAS_CXX_OBJ =       (1 << 26),
   BLOCK_IS_GLOBAL =         (1 << 28),
   BLOCK_USE_STRET =         (1 << 29),
-  BLOCK_HAS_SIGNATURE  =    (1 << 30)
+  BLOCK_HAS_SIGNATURE  =    (1 << 30),
+  BLOCK_HAS_EXTENDED_LAYOUT = (1 << 31)
 };
 class BlockFlags {
   uint32_t flags;
@@ -60,7 +72,7 @@ class BlockFlags {
   BlockFlags(uint32_t flags) : flags(flags) {}
 public:
   BlockFlags() : flags(0) {}
-  BlockFlags(BlockFlag_t flag) : flags(flag) {}
+  BlockFlags(BlockLiteralFlags flag) : flags(flag) {}
 
   uint32_t getBitMask() const { return flags; }
   bool empty() const { return flags == 0; }
@@ -76,7 +88,7 @@ public:
     return (l.flags & r.flags);
   }
 };
-inline BlockFlags operator|(BlockFlag_t l, BlockFlag_t r) {
+inline BlockFlags operator|(BlockLiteralFlags l, BlockLiteralFlags r) {
   return BlockFlags(l) | BlockFlags(r);
 }
 
@@ -182,6 +194,10 @@ public:
   /// UsesStret : True if the block uses an stret return.  Mutable
   /// because it gets set later in the block-creation process.
   mutable bool UsesStret : 1;
+  
+  /// HasCapturedVariableLayout : True if block has captured variables
+  /// and their layout meta-data has been generated.
+  bool HasCapturedVariableLayout : 1;
 
   /// The mapping of allocated indexes within the block.
   llvm::DenseMap<const VarDecl*, Capture> Captures;  
diff --git a/lib/CodeGen/CGBuiltin.cpp b/lib/CodeGen/CGBuiltin.cpp
index 59ed313..e8c05d3 100644
--- a/lib/CodeGen/CGBuiltin.cpp
+++ b/lib/CodeGen/CGBuiltin.cpp
@@ -20,7 +20,7 @@
 #include "clang/AST/Decl.h"
 #include "clang/Basic/TargetBuiltins.h"
 #include "llvm/Intrinsics.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 
 using namespace clang;
 using namespace CodeGen;
@@ -86,8 +86,7 @@ static RValue EmitBinaryAtomic(CodeGenFunction &CGF,
   assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
 
   llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0));
-  unsigned AddrSpace =
-    cast<llvm::PointerType>(DestPtr->getType())->getAddressSpace();
+  unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
 
   llvm::IntegerType *IntType =
     llvm::IntegerType::get(CGF.getLLVMContext(),
@@ -121,8 +120,7 @@ static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF,
   assert(CGF.getContext().hasSameUnqualifiedType(T, E->getArg(1)->getType()));
 
   llvm::Value *DestPtr = CGF.EmitScalarExpr(E->getArg(0));
-  unsigned AddrSpace =
-    cast<llvm::PointerType>(DestPtr->getType())->getAddressSpace();
+  unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
 
   llvm::IntegerType *IntType =
     llvm::IntegerType::get(CGF.getLLVMContext(),
@@ -148,7 +146,7 @@ static RValue EmitBinaryAtomicPost(CodeGenFunction &CGF,
 static Value *EmitFAbs(CodeGenFunction &CGF, Value *V, QualType ValTy) {
   const BuiltinType *ValTyP = ValTy->getAs<BuiltinType>();
   assert(ValTyP && "isn't scalar fp type!");
-  
+
   StringRef FnName;
   switch (ValTyP->getKind()) {
   default: llvm_unreachable("Isn't a scalar fp type!");
@@ -156,7 +154,7 @@ static Value *EmitFAbs(CodeGenFunction &CGF, Value *V, QualType ValTy) {
   case BuiltinType::Double:     FnName = "fabs"; break;
   case BuiltinType::LongDouble: FnName = "fabsl"; break;
   }
-  
+
   // The prototype is something that takes and returns whatever V's type is.
   llvm::FunctionType *FT = llvm::FunctionType::get(V->getType(), V->getType(),
                                                    false);
@@ -214,7 +212,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     return RValue::get(Builder.CreateCall2(CGM.getIntrinsic(Intrinsic::vacopy),
                                            DstPtr, SrcPtr));
   }
-  case Builtin::BI__builtin_abs: 
+  case Builtin::BI__builtin_abs:
   case Builtin::BI__builtin_labs:
   case Builtin::BI__builtin_llabs: {
     Value *ArgValue = EmitScalarExpr(E->getArg(0));
@@ -229,18 +227,18 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
 
     return RValue::get(Result);
   }
-  
+
   case Builtin::BI__builtin_conj:
   case Builtin::BI__builtin_conjf:
   case Builtin::BI__builtin_conjl: {
     ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
     Value *Real = ComplexVal.first;
     Value *Imag = ComplexVal.second;
-    Value *Zero = 
-      Imag->getType()->isFPOrFPVectorTy() 
+    Value *Zero =
+      Imag->getType()->isFPOrFPVectorTy()
         ? llvm::ConstantFP::getZeroValueForNegation(Imag->getType())
         : llvm::Constant::getNullValue(Imag->getType());
-    
+
     Imag = Builder.CreateFSub(Zero, Imag, "sub");
     return RValue::getComplex(std::make_pair(Real, Imag));
   }
@@ -250,14 +248,14 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
     return RValue::get(ComplexVal.first);
   }
-      
+
   case Builtin::BI__builtin_cimag:
   case Builtin::BI__builtin_cimagf:
   case Builtin::BI__builtin_cimagl: {
     ComplexPairTy ComplexVal = EmitComplexExpr(E->getArg(0));
     return RValue::get(ComplexVal.second);
   }
-      
+
   case Builtin::BI__builtin_ctzs:
   case Builtin::BI__builtin_ctz:
   case Builtin::BI__builtin_ctzl:
@@ -356,6 +354,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
                                         "expval");
     return RValue::get(Result);
   }
+  case Builtin::BI__builtin_bswap16:
   case Builtin::BI__builtin_bswap32:
   case Builtin::BI__builtin_bswap64: {
     Value *ArgValue = EmitScalarExpr(E->getArg(0));
@@ -371,15 +370,15 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     // We pass this builtin onto the optimizer so that it can
     // figure out the object size in more complex cases.
     llvm::Type *ResType = ConvertType(E->getType());
-    
+
     // LLVM only supports 0 and 2, make sure that we pass along that
     // as a boolean.
     Value *Ty = EmitScalarExpr(E->getArg(1));
     ConstantInt *CI = dyn_cast<ConstantInt>(Ty);
     assert(CI);
     uint64_t val = CI->getZExtValue();
-    CI = ConstantInt::get(Builder.getInt1Ty(), (val & 0x2) >> 1);    
-    
+    CI = ConstantInt::get(Builder.getInt1Ty(), (val & 0x2) >> 1);
+
     Value *F = CGM.getIntrinsic(Intrinsic::objectsize, ResType);
     return RValue::get(Builder.CreateCall2(F, EmitScalarExpr(E->getArg(0)),CI));
   }
@@ -402,9 +401,15 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     Value *F = CGM.getIntrinsic(Intrinsic::trap);
     return RValue::get(Builder.CreateCall(F));
   }
+  case Builtin::BI__debugbreak: {
+    Value *F = CGM.getIntrinsic(Intrinsic::debugtrap);
+    return RValue::get(Builder.CreateCall(F));
+  }
   case Builtin::BI__builtin_unreachable: {
-    if (CatchUndefined)
-      EmitBranch(getTrapBB());
+    if (getLangOpts().SanitizeUnreachable)
+      EmitCheck(Builder.getFalse(), "builtin_unreachable",
+                EmitCheckSourceLocation(E->getExprLoc()),
+                llvm::ArrayRef<llvm::Value *>());
     else
       Builder.CreateUnreachable();
 
@@ -413,7 +418,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
 
     return RValue::get(0);
   }
-      
+
   case Builtin::BI__builtin_powi:
   case Builtin::BI__builtin_powif:
   case Builtin::BI__builtin_powil: {
@@ -464,16 +469,16 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     V = Builder.CreateFCmpUNO(V, V, "cmp");
     return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType())));
   }
-  
+
   case Builtin::BI__builtin_isinf: {
     // isinf(x) --> fabs(x) == infinity
     Value *V = EmitScalarExpr(E->getArg(0));
     V = EmitFAbs(*this, V, E->getArg(0)->getType());
-    
+
     V = Builder.CreateFCmpOEQ(V, ConstantFP::getInfinity(V->getType()),"isinf");
     return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType())));
   }
-      
+
   // TODO: BI__builtin_isinf_sign
   //   isinf_sign(x) -> isinf(x) ? (signbit(x) ? -1 : 1) : 0
 
@@ -499,11 +504,11 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     // isfinite(x) --> x == x && fabs(x) != infinity;
     Value *V = EmitScalarExpr(E->getArg(0));
     Value *Eq = Builder.CreateFCmpOEQ(V, V, "iseq");
-    
+
     Value *Abs = EmitFAbs(*this, V, E->getArg(0)->getType());
     Value *IsNotInf =
       Builder.CreateFCmpUNE(Abs, ConstantFP::getInfinity(V->getType()),"isinf");
-    
+
     V = Builder.CreateAnd(Eq, IsNotInf, "and");
     return RValue::get(Builder.CreateZExt(V, ConvertType(E->getType())));
   }
@@ -565,7 +570,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     Builder.SetInsertPoint(End);
     return RValue::get(Result);
   }
-      
+
   case Builtin::BIalloca:
   case Builtin::BI__builtin_alloca: {
     Value *Size = EmitScalarExpr(E->getArg(0));
@@ -573,85 +578,90 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
   }
   case Builtin::BIbzero:
   case Builtin::BI__builtin_bzero: {
-    Value *Address = EmitScalarExpr(E->getArg(0));
+    std::pair<llvm::Value*, unsigned> Dest =
+        EmitPointerWithAlignment(E->getArg(0));
     Value *SizeVal = EmitScalarExpr(E->getArg(1));
-    unsigned Align = GetPointeeAlignment(E->getArg(0));
-    Builder.CreateMemSet(Address, Builder.getInt8(0), SizeVal, Align, false);
-    return RValue::get(Address);
+    Builder.CreateMemSet(Dest.first, Builder.getInt8(0), SizeVal,
+                         Dest.second, false);
+    return RValue::get(Dest.first);
   }
   case Builtin::BImemcpy:
   case Builtin::BI__builtin_memcpy: {
-    Value *Address = EmitScalarExpr(E->getArg(0));
-    Value *SrcAddr = EmitScalarExpr(E->getArg(1));
+    std::pair<llvm::Value*, unsigned> Dest =
+        EmitPointerWithAlignment(E->getArg(0));
+    std::pair<llvm::Value*, unsigned> Src =
+        EmitPointerWithAlignment(E->getArg(1));
     Value *SizeVal = EmitScalarExpr(E->getArg(2));
-    unsigned Align = std::min(GetPointeeAlignment(E->getArg(0)),
-                              GetPointeeAlignment(E->getArg(1)));
-    Builder.CreateMemCpy(Address, SrcAddr, SizeVal, Align, false);
-    return RValue::get(Address);
+    unsigned Align = std::min(Dest.second, Src.second);
+    Builder.CreateMemCpy(Dest.first, Src.first, SizeVal, Align, false);
+    return RValue::get(Dest.first);
   }
-      
+
   case Builtin::BI__builtin___memcpy_chk: {
-    // fold __builtin_memcpy_chk(x, y, cst1, cst2) to memset iff cst1<=cst2.
+    // fold __builtin_memcpy_chk(x, y, cst1, cst2) to memcpy iff cst1<=cst2.
     llvm::APSInt Size, DstSize;
     if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) ||
         !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext()))
       break;
     if (Size.ugt(DstSize))
       break;
-    Value *Dest = EmitScalarExpr(E->getArg(0));
-    Value *Src = EmitScalarExpr(E->getArg(1));
+    std::pair<llvm::Value*, unsigned> Dest =
+        EmitPointerWithAlignment(E->getArg(0));
+    std::pair<llvm::Value*, unsigned> Src =
+        EmitPointerWithAlignment(E->getArg(1));
     Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
-    unsigned Align = std::min(GetPointeeAlignment(E->getArg(0)),
-                              GetPointeeAlignment(E->getArg(1)));
-    Builder.CreateMemCpy(Dest, Src, SizeVal, Align, false);
-    return RValue::get(Dest);
+    unsigned Align = std::min(Dest.second, Src.second);
+    Builder.CreateMemCpy(Dest.first, Src.first, SizeVal, Align, false);
+    return RValue::get(Dest.first);
   }
-      
+
   case Builtin::BI__builtin_objc_memmove_collectable: {
     Value *Address = EmitScalarExpr(E->getArg(0));
     Value *SrcAddr = EmitScalarExpr(E->getArg(1));
     Value *SizeVal = EmitScalarExpr(E->getArg(2));
-    CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this, 
+    CGM.getObjCRuntime().EmitGCMemmoveCollectable(*this,
                                                   Address, SrcAddr, SizeVal);
     return RValue::get(Address);
   }
 
   case Builtin::BI__builtin___memmove_chk: {
-    // fold __builtin_memmove_chk(x, y, cst1, cst2) to memset iff cst1<=cst2.
+    // fold __builtin_memmove_chk(x, y, cst1, cst2) to memmove iff cst1<=cst2.
     llvm::APSInt Size, DstSize;
     if (!E->getArg(2)->EvaluateAsInt(Size, CGM.getContext()) ||
         !E->getArg(3)->EvaluateAsInt(DstSize, CGM.getContext()))
       break;
     if (Size.ugt(DstSize))
       break;
-    Value *Dest = EmitScalarExpr(E->getArg(0));
-    Value *Src = EmitScalarExpr(E->getArg(1));
+    std::pair<llvm::Value*, unsigned> Dest =
+        EmitPointerWithAlignment(E->getArg(0));
+    std::pair<llvm::Value*, unsigned> Src =
+        EmitPointerWithAlignment(E->getArg(1));
     Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
-    unsigned Align = std::min(GetPointeeAlignment(E->getArg(0)),
-                              GetPointeeAlignment(E->getArg(1)));
-    Builder.CreateMemMove(Dest, Src, SizeVal, Align, false);
-    return RValue::get(Dest);
+    unsigned Align = std::min(Dest.second, Src.second);
+    Builder.CreateMemMove(Dest.first, Src.first, SizeVal, Align, false);
+    return RValue::get(Dest.first);
   }
 
   case Builtin::BImemmove:
   case Builtin::BI__builtin_memmove: {
-    Value *Address = EmitScalarExpr(E->getArg(0));
-    Value *SrcAddr = EmitScalarExpr(E->getArg(1));
+    std::pair<llvm::Value*, unsigned> Dest =
+        EmitPointerWithAlignment(E->getArg(0));
+    std::pair<llvm::Value*, unsigned> Src =
+        EmitPointerWithAlignment(E->getArg(1));
     Value *SizeVal = EmitScalarExpr(E->getArg(2));
-    unsigned Align = std::min(GetPointeeAlignment(E->getArg(0)),
-                              GetPointeeAlignment(E->getArg(1)));
-    Builder.CreateMemMove(Address, SrcAddr, SizeVal, Align, false);
-    return RValue::get(Address);
+    unsigned Align = std::min(Dest.second, Src.second);
+    Builder.CreateMemMove(Dest.first, Src.first, SizeVal, Align, false);
+    return RValue::get(Dest.first);
   }
   case Builtin::BImemset:
   case Builtin::BI__builtin_memset: {
-    Value *Address = EmitScalarExpr(E->getArg(0));
+    std::pair<llvm::Value*, unsigned> Dest =
+        EmitPointerWithAlignment(E->getArg(0));
     Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)),
                                          Builder.getInt8Ty());
     Value *SizeVal = EmitScalarExpr(E->getArg(2));
-    unsigned Align = GetPointeeAlignment(E->getArg(0));
-    Builder.CreateMemSet(Address, ByteVal, SizeVal, Align, false);
-    return RValue::get(Address);
+    Builder.CreateMemSet(Dest.first, ByteVal, SizeVal, Dest.second, false);
+    return RValue::get(Dest.first);
   }
   case Builtin::BI__builtin___memset_chk: {
     // fold __builtin_memset_chk(x, y, cst1, cst2) to memset iff cst1<=cst2.
@@ -661,14 +671,13 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
       break;
     if (Size.ugt(DstSize))
       break;
-    Value *Address = EmitScalarExpr(E->getArg(0));
+    std::pair<llvm::Value*, unsigned> Dest =
+        EmitPointerWithAlignment(E->getArg(0));
     Value *ByteVal = Builder.CreateTrunc(EmitScalarExpr(E->getArg(1)),
                                          Builder.getInt8Ty());
     Value *SizeVal = llvm::ConstantInt::get(Builder.getContext(), Size);
-    unsigned Align = GetPointeeAlignment(E->getArg(0));
-    Builder.CreateMemSet(Address, ByteVal, SizeVal, Align, false);
-    
-    return RValue::get(Address);
+    Builder.CreateMemSet(Dest.first, ByteVal, SizeVal, Dest.second, false);
+    return RValue::get(Dest.first);
   }
   case Builtin::BI__builtin_dwarf_cfa: {
     // The offset in bytes from the first argument to the CFA.
@@ -682,7 +691,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     int32_t Offset = 0;
 
     Value *F = CGM.getIntrinsic(Intrinsic::eh_dwarf_cfa);
-    return RValue::get(Builder.CreateCall(F, 
+    return RValue::get(Builder.CreateCall(F,
                                       llvm::ConstantInt::get(Int32Ty, Offset)));
   }
   case Builtin::BI__builtin_return_address: {
@@ -907,9 +916,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
   case Builtin::BI__sync_val_compare_and_swap_16: {
     QualType T = E->getType();
     llvm::Value *DestPtr = EmitScalarExpr(E->getArg(0));
-    unsigned AddrSpace =
-      cast<llvm::PointerType>(DestPtr->getType())->getAddressSpace();
-    
+    unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
+
     llvm::IntegerType *IntType =
       llvm::IntegerType::get(getLLVMContext(),
                              getContext().getTypeSize(T));
@@ -935,9 +943,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
   case Builtin::BI__sync_bool_compare_and_swap_16: {
     QualType T = E->getArg(1)->getType();
     llvm::Value *DestPtr = EmitScalarExpr(E->getArg(0));
-    unsigned AddrSpace =
-      cast<llvm::PointerType>(DestPtr->getType())->getAddressSpace();
-    
+    unsigned AddrSpace = DestPtr->getType()->getPointerAddressSpace();
+
     llvm::IntegerType *IntType =
       llvm::IntegerType::get(getLLVMContext(),
                              getContext().getTypeSize(T));
@@ -982,7 +989,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     llvm::Type *ITy = llvm::IntegerType::get(getLLVMContext(),
                                              StoreSize.getQuantity() * 8);
     Ptr = Builder.CreateBitCast(Ptr, ITy->getPointerTo());
-    llvm::StoreInst *Store = 
+    llvm::StoreInst *Store =
       Builder.CreateStore(llvm::Constant::getNullValue(ITy), Ptr);
     Store->setAlignment(StoreSize.getQuantity());
     Store->setAtomic(llvm::Release);
@@ -993,7 +1000,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     // We assume this is supposed to correspond to a C++0x-style
     // sequentially-consistent fence (i.e. this is only usable for
     // synchonization, not device I/O or anything like that). This intrinsic
-    // is really badly designed in the sense that in theory, there isn't 
+    // is really badly designed in the sense that in theory, there isn't
     // any way to safely use it... but in practice, it mostly works
     // to use it with non-atomic loads and stores to get acquire/release
     // semantics.
@@ -1033,8 +1040,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
         PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified();
 
     Value *Ptr = EmitScalarExpr(E->getArg(0));
-    unsigned AddrSpace =
-        cast<llvm::PointerType>(Ptr->getType())->getAddressSpace();
+    unsigned AddrSpace = Ptr->getType()->getPointerAddressSpace();
     Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace));
     Value *NewVal = Builder.getInt8(1);
     Value *Order = EmitScalarExpr(E->getArg(1));
@@ -1120,8 +1126,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
         PtrTy->castAs<PointerType>()->getPointeeType().isVolatileQualified();
 
     Value *Ptr = EmitScalarExpr(E->getArg(0));
-    unsigned AddrSpace =
-        cast<llvm::PointerType>(Ptr->getType())->getAddressSpace();
+    unsigned AddrSpace = Ptr->getType()->getPointerAddressSpace();
     Ptr = Builder.CreateBitCast(Ptr, Int8Ty->getPointerTo(AddrSpace));
     Value *NewVal = Builder.getInt8(0);
     Value *Order = EmitScalarExpr(E->getArg(1));
@@ -1310,6 +1315,8 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     llvm::StringRef Str = cast<StringLiteral>(AnnotationStrExpr)->getString();
     return RValue::get(EmitAnnotationCall(F, AnnVal, Str, E->getExprLoc()));
   }
+  case Builtin::BI__noop:
+    return RValue::get(0);
   }
 
   // If this is an alias for a lib function (e.g. __builtin_sin), emit
@@ -1318,7 +1325,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
   if (getContext().BuiltinInfo.isLibFunction(BuiltinID))
     return emitLibraryCall(*this, FD, E,
                            CGM.getBuiltinLibFunction(FD, BuiltinID));
-  
+
   // If this is a predefined lib function (e.g. malloc), emit the call
   // using exactly the normal call path.
   if (getContext().BuiltinInfo.isPredefinedLibFunction(BuiltinID))
@@ -1350,7 +1357,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
       if ((ICEArguments & (1 << i)) == 0) {
         ArgValue = EmitScalarExpr(E->getArg(i));
       } else {
-        // If this is required to be a constant, constant fold it so that we 
+        // If this is required to be a constant, constant fold it so that we
         // know that the generated intrinsic gets a ConstantInt.
         llvm::APSInt Result;
         bool IsConst = E->getArg(i)->isIntegerConstantExpr(Result,getContext());
@@ -1375,7 +1382,7 @@ RValue CodeGenFunction::EmitBuiltinExpr(const FunctionDecl *FD,
     QualType BuiltinRetType = E->getType();
 
     llvm::Type *RetTy = VoidTy;
-    if (!BuiltinRetType->isVoidType()) 
+    if (!BuiltinRetType->isVoidType())
       RetTy = ConvertType(BuiltinRetType);
 
     if (RetTy != V->getType()) {
@@ -1457,10 +1464,10 @@ Value *CodeGenFunction::EmitNeonCall(Function *F, SmallVectorImpl<Value*> &Ops,
   return Builder.CreateCall(F, Ops, name);
 }
 
-Value *CodeGenFunction::EmitNeonShiftVector(Value *V, llvm::Type *Ty, 
+Value *CodeGenFunction::EmitNeonShiftVector(Value *V, llvm::Type *Ty,
                                             bool neg) {
   int SV = cast<ConstantInt>(V)->getSExtValue();
-  
+
   llvm::VectorType *VTy = cast<llvm::VectorType>(Ty);
   llvm::Constant *C = ConstantInt::get(VTy->getElementType(), neg ? -SV : SV);
   return llvm::ConstantVector::getSplat(VTy->getNumElements(), C);
@@ -1469,34 +1476,56 @@ Value *CodeGenFunction::EmitNeonShiftVector(Value *V, llvm::Type *Ty,
 /// GetPointeeAlignment - Given an expression with a pointer type, find the
 /// alignment of the type referenced by the pointer.  Skip over implicit
 /// casts.
-unsigned CodeGenFunction::GetPointeeAlignment(const Expr *Addr) {
-  unsigned Align = 1;
-  // Check if the type is a pointer.  The implicit cast operand might not be.
-  while (Addr->getType()->isPointerType()) {
-    QualType PtTy = Addr->getType()->getPointeeType();
-    
-    // Can't get alignment of incomplete types.
-    if (!PtTy->isIncompleteType()) {
-      unsigned NewA = getContext().getTypeAlignInChars(PtTy).getQuantity();
-      if (NewA > Align)
-        Align = NewA;
+std::pair<llvm::Value*, unsigned>
+CodeGenFunction::EmitPointerWithAlignment(const Expr *Addr) {
+  assert(Addr->getType()->isPointerType());
+  Addr = Addr->IgnoreParens();
+  if (const ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(Addr)) {
+    if ((ICE->getCastKind() == CK_BitCast || ICE->getCastKind() == CK_NoOp) &&
+        ICE->getSubExpr()->getType()->isPointerType()) {
+      std::pair<llvm::Value*, unsigned> Ptr =
+          EmitPointerWithAlignment(ICE->getSubExpr());
+      Ptr.first = Builder.CreateBitCast(Ptr.first,
+                                        ConvertType(Addr->getType()));
+      return Ptr;
+    } else if (ICE->getCastKind() == CK_ArrayToPointerDecay) {
+      LValue LV = EmitLValue(ICE->getSubExpr());
+      unsigned Align = LV.getAlignment().getQuantity();
+      if (!Align) {
+        // FIXME: Once LValues are fixed to always set alignment,
+        // zap this code.
+        QualType PtTy = ICE->getSubExpr()->getType();
+        if (!PtTy->isIncompleteType())
+          Align = getContext().getTypeAlignInChars(PtTy).getQuantity();
+        else
+          Align = 1;
+      }
+      return std::make_pair(LV.getAddress(), Align);
     }
-
-    // If the address is an implicit cast, repeat with the cast operand.
-    if (const ImplicitCastExpr *CastAddr = dyn_cast<ImplicitCastExpr>(Addr)) {
-      Addr = CastAddr->getSubExpr();
-      continue;
+  }
+  if (const UnaryOperator *UO = dyn_cast<UnaryOperator>(Addr)) {
+    if (UO->getOpcode() == UO_AddrOf) {
+      LValue LV = EmitLValue(UO->getSubExpr());
+      unsigned Align = LV.getAlignment().getQuantity();
+      if (!Align) {
+        // FIXME: Once LValues are fixed to always set alignment,
+        // zap this code.
+        QualType PtTy = UO->getSubExpr()->getType();
+        if (!PtTy->isIncompleteType())
+          Align = getContext().getTypeAlignInChars(PtTy).getQuantity();
+        else
+          Align = 1;
+      }
+      return std::make_pair(LV.getAddress(), Align);
     }
-    break;
   }
-  return Align;
-}
 
-/// GetPointeeAlignmentValue - Given an expression with a pointer type, find
-/// the alignment of the type referenced by the pointer.  Skip over implicit
-/// casts.  Return the alignment as an llvm::Value.
-Value *CodeGenFunction::GetPointeeAlignmentValue(const Expr *Addr) {
-  return llvm::ConstantInt::get(Int32Ty, GetPointeeAlignment(Addr));
+  unsigned Align = 1;
+  QualType PtTy = Addr->getType()->getPointeeType();
+  if (!PtTy->isIncompleteType())
+    Align = getContext().getTypeAlignInChars(PtTy).getQuantity();
+
+  return std::make_pair(EmitScalarExpr(Addr), Align);
 }
 
 Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
@@ -1549,8 +1578,69 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
   }
 
   SmallVector<Value*, 4> Ops;
-  for (unsigned i = 0, e = E->getNumArgs() - 1; i != e; i++)
+  llvm::Value *Align = 0;
+  for (unsigned i = 0, e = E->getNumArgs() - 1; i != e; i++) {
+    if (i == 0) {
+      switch (BuiltinID) {
+      case ARM::BI__builtin_neon_vld1_v:
+      case ARM::BI__builtin_neon_vld1q_v:
+      case ARM::BI__builtin_neon_vld1q_lane_v:
+      case ARM::BI__builtin_neon_vld1_lane_v:
+      case ARM::BI__builtin_neon_vld1_dup_v:
+      case ARM::BI__builtin_neon_vld1q_dup_v:
+      case ARM::BI__builtin_neon_vst1_v:
+      case ARM::BI__builtin_neon_vst1q_v:
+      case ARM::BI__builtin_neon_vst1q_lane_v:
+      case ARM::BI__builtin_neon_vst1_lane_v:
+      case ARM::BI__builtin_neon_vst2_v:
+      case ARM::BI__builtin_neon_vst2q_v:
+      case ARM::BI__builtin_neon_vst2_lane_v:
+      case ARM::BI__builtin_neon_vst2q_lane_v:
+      case ARM::BI__builtin_neon_vst3_v:
+      case ARM::BI__builtin_neon_vst3q_v:
+      case ARM::BI__builtin_neon_vst3_lane_v:
+      case ARM::BI__builtin_neon_vst3q_lane_v:
+      case ARM::BI__builtin_neon_vst4_v:
+      case ARM::BI__builtin_neon_vst4q_v:
+      case ARM::BI__builtin_neon_vst4_lane_v:
+      case ARM::BI__builtin_neon_vst4q_lane_v:
+        // Get the alignment for the argument in addition to the value;
+        // we'll use it later.
+        std::pair<llvm::Value*, unsigned> Src =
+            EmitPointerWithAlignment(E->getArg(0));
+        Ops.push_back(Src.first);
+        Align = Builder.getInt32(Src.second);
+        continue;
+      }
+    }
+    if (i == 1) {
+      switch (BuiltinID) {
+      case ARM::BI__builtin_neon_vld2_v:
+      case ARM::BI__builtin_neon_vld2q_v:
+      case ARM::BI__builtin_neon_vld3_v:
+      case ARM::BI__builtin_neon_vld3q_v:
+      case ARM::BI__builtin_neon_vld4_v:
+      case ARM::BI__builtin_neon_vld4q_v:
+      case ARM::BI__builtin_neon_vld2_lane_v:
+      case ARM::BI__builtin_neon_vld2q_lane_v:
+      case ARM::BI__builtin_neon_vld3_lane_v:
+      case ARM::BI__builtin_neon_vld3q_lane_v:
+      case ARM::BI__builtin_neon_vld4_lane_v:
+      case ARM::BI__builtin_neon_vld4q_lane_v:
+      case ARM::BI__builtin_neon_vld2_dup_v:
+      case ARM::BI__builtin_neon_vld3_dup_v:
+      case ARM::BI__builtin_neon_vld4_dup_v:
+        // Get the alignment for the argument in addition to the value;
+        // we'll use it later.
+        std::pair<llvm::Value*, unsigned> Src =
+            EmitPointerWithAlignment(E->getArg(1));
+        Ops.push_back(Src.first);
+        Align = Builder.getInt32(Src.second);
+        continue;
+      }
+    }
     Ops.push_back(EmitScalarExpr(E->getArg(i)));
+  }
 
   // vget_lane and vset_lane are not overloaded and do not have an extra
   // argument that specifies the vector type.
@@ -1596,7 +1686,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
       Ty = FloatTy;
     else
       Ty = DoubleTy;
-    
+
     // Determine whether this is an unsigned conversion or not.
     bool usgn = Result.getZExtValue() == 1;
     unsigned Int = usgn ? Intrinsic::arm_vcvtru : Intrinsic::arm_vcvtr;
@@ -1605,7 +1695,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     Function *F = CGM.getIntrinsic(Int, Ty);
     return Builder.CreateCall(F, Ops, "vcvtr");
   }
-  
+
   // Determine the type of this overloaded NEON intrinsic.
   NeonTypeFlags Type(Result.getZExtValue());
   bool usgn = Type.isUnsigned();
@@ -1620,6 +1710,10 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
   unsigned Int;
   switch (BuiltinID) {
   default: return 0;
+  case ARM::BI__builtin_neon_vbsl_v:
+  case ARM::BI__builtin_neon_vbslq_v:
+    return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vbsl, Ty),
+                        Ops, "vbsl");
   case ARM::BI__builtin_neon_vabd_v:
   case ARM::BI__builtin_neon_vabdq_v:
     Int = usgn ? Intrinsic::arm_neon_vabdu : Intrinsic::arm_neon_vabds;
@@ -1690,7 +1784,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
   case ARM::BI__builtin_neon_vcvtq_f32_v:
     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
     Ty = GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float32, false, quad));
-    return usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt") 
+    return usgn ? Builder.CreateUIToFP(Ops[0], Ty, "vcvt")
                 : Builder.CreateSIToFP(Ops[0], Ty, "vcvt");
   case ARM::BI__builtin_neon_vcvt_s32_v:
   case ARM::BI__builtin_neon_vcvt_u32_v:
@@ -1699,7 +1793,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     llvm::Type *FloatTy =
       GetNeonType(this, NeonTypeFlags(NeonTypeFlags::Float32, false, quad));
     Ops[0] = Builder.CreateBitCast(Ops[0], FloatTy);
-    return usgn ? Builder.CreateFPToUI(Ops[0], Ty, "vcvt") 
+    return usgn ? Builder.CreateFPToUI(Ops[0], Ty, "vcvt")
                 : Builder.CreateFPToSI(Ops[0], Ty, "vcvt");
   }
   case ARM::BI__builtin_neon_vcvt_n_f32_v:
@@ -1730,7 +1824,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     SmallVector<Constant*, 16> Indices;
     for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
       Indices.push_back(ConstantInt::get(Int32Ty, i+CV));
-    
+
     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
     Value *SV = llvm::ConstantVector::get(Indices);
@@ -1746,7 +1840,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vhsub");
   case ARM::BI__builtin_neon_vld1_v:
   case ARM::BI__builtin_neon_vld1q_v:
-    Ops.push_back(GetPointeeAlignmentValue(E->getArg(0)));
+    Ops.push_back(Align);
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vld1, Ty),
                         Ops, "vld1");
   case ARM::BI__builtin_neon_vld1q_lane_v:
@@ -1761,8 +1855,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
       // Load the value as a one-element vector.
       Ty = llvm::VectorType::get(VTy->getElementType(), 1);
       Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld1, Ty);
-      Value *Ld = Builder.CreateCall2(F, Ops[0],
-                                      GetPointeeAlignmentValue(E->getArg(0)));
+      Value *Ld = Builder.CreateCall2(F, Ops[0], Align);
       // Combine them.
       SmallVector<Constant*, 2> Indices;
       Indices.push_back(ConstantInt::get(Int32Ty, 1-Lane));
@@ -1776,7 +1869,6 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     Ty = llvm::PointerType::getUnqual(VTy->getElementType());
     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
     LoadInst *Ld = Builder.CreateLoad(Ops[0]);
-    Value *Align = GetPointeeAlignmentValue(E->getArg(0));
     Ld->setAlignment(cast<ConstantInt>(Align)->getZExtValue());
     return Builder.CreateInsertElement(Ops[1], Ld, Ops[2], "vld1_lane");
   }
@@ -1786,7 +1878,6 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     Ty = llvm::PointerType::getUnqual(VTy->getElementType());
     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
     LoadInst *Ld = Builder.CreateLoad(Ops[0]);
-    Value *Align = GetPointeeAlignmentValue(E->getArg(0));
     Ld->setAlignment(cast<ConstantInt>(Align)->getZExtValue());
     llvm::Constant *CI = ConstantInt::get(Int32Ty, 0);
     Ops[0] = Builder.CreateInsertElement(V, Ld, CI);
@@ -1795,7 +1886,6 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
   case ARM::BI__builtin_neon_vld2_v:
   case ARM::BI__builtin_neon_vld2q_v: {
     Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld2, Ty);
-    Value *Align = GetPointeeAlignmentValue(E->getArg(1));
     Ops[1] = Builder.CreateCall2(F, Ops[1], Align, "vld2");
     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
@@ -1804,7 +1894,6 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
   case ARM::BI__builtin_neon_vld3_v:
   case ARM::BI__builtin_neon_vld3q_v: {
     Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld3, Ty);
-    Value *Align = GetPointeeAlignmentValue(E->getArg(1));
     Ops[1] = Builder.CreateCall2(F, Ops[1], Align, "vld3");
     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
@@ -1813,7 +1902,6 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
   case ARM::BI__builtin_neon_vld4_v:
   case ARM::BI__builtin_neon_vld4q_v: {
     Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld4, Ty);
-    Value *Align = GetPointeeAlignmentValue(E->getArg(1));
     Ops[1] = Builder.CreateCall2(F, Ops[1], Align, "vld4");
     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
@@ -1824,7 +1912,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     Function *F = CGM.getIntrinsic(Intrinsic::arm_neon_vld2lane, Ty);
     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
     Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
-    Ops.push_back(GetPointeeAlignmentValue(E->getArg(1)));
+    Ops.push_back(Align);
     Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld2_lane");
     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
@@ -1836,7 +1924,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
     Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
     Ops[4] = Builder.CreateBitCast(Ops[4], Ty);
-    Ops.push_back(GetPointeeAlignmentValue(E->getArg(1)));
+    Ops.push_back(Align);
     Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld3_lane");
     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
@@ -1849,7 +1937,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     Ops[3] = Builder.CreateBitCast(Ops[3], Ty);
     Ops[4] = Builder.CreateBitCast(Ops[4], Ty);
     Ops[5] = Builder.CreateBitCast(Ops[5], Ty);
-    Ops.push_back(GetPointeeAlignmentValue(E->getArg(1)));
+    Ops.push_back(Align);
     Ops[1] = Builder.CreateCall(F, makeArrayRef(Ops).slice(1), "vld3_lane");
     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
@@ -1861,47 +1949,46 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     // Handle 64-bit elements as a special-case.  There is no "dup" needed.
     if (VTy->getElementType()->getPrimitiveSizeInBits() == 64) {
       switch (BuiltinID) {
-      case ARM::BI__builtin_neon_vld2_dup_v: 
-        Int = Intrinsic::arm_neon_vld2; 
+      case ARM::BI__builtin_neon_vld2_dup_v:
+        Int = Intrinsic::arm_neon_vld2;
         break;
       case ARM::BI__builtin_neon_vld3_dup_v:
-        Int = Intrinsic::arm_neon_vld3; 
+        Int = Intrinsic::arm_neon_vld3;
         break;
       case ARM::BI__builtin_neon_vld4_dup_v:
-        Int = Intrinsic::arm_neon_vld4; 
+        Int = Intrinsic::arm_neon_vld4;
         break;
       default: llvm_unreachable("unknown vld_dup intrinsic?");
       }
       Function *F = CGM.getIntrinsic(Int, Ty);
-      Value *Align = GetPointeeAlignmentValue(E->getArg(1));
       Ops[1] = Builder.CreateCall2(F, Ops[1], Align, "vld_dup");
       Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
       Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
       return Builder.CreateStore(Ops[1], Ops[0]);
     }
     switch (BuiltinID) {
-    case ARM::BI__builtin_neon_vld2_dup_v: 
-      Int = Intrinsic::arm_neon_vld2lane; 
+    case ARM::BI__builtin_neon_vld2_dup_v:
+      Int = Intrinsic::arm_neon_vld2lane;
       break;
     case ARM::BI__builtin_neon_vld3_dup_v:
-      Int = Intrinsic::arm_neon_vld3lane; 
+      Int = Intrinsic::arm_neon_vld3lane;
       break;
     case ARM::BI__builtin_neon_vld4_dup_v:
-      Int = Intrinsic::arm_neon_vld4lane; 
+      Int = Intrinsic::arm_neon_vld4lane;
       break;
     default: llvm_unreachable("unknown vld_dup intrinsic?");
     }
     Function *F = CGM.getIntrinsic(Int, Ty);
     llvm::StructType *STy = cast<llvm::StructType>(F->getReturnType());
-    
+
     SmallVector<Value*, 6> Args;
     Args.push_back(Ops[1]);
     Args.append(STy->getNumElements(), UndefValue::get(Ty));
 
     llvm::Constant *CI = ConstantInt::get(Int32Ty, 0);
     Args.push_back(CI);
-    Args.push_back(GetPointeeAlignmentValue(E->getArg(1)));
-    
+    Args.push_back(Align);
+
     Ops[1] = Builder.CreateCall(F, Args, "vld_dup");
     // splat lane 0 to all elts in each vector of the result.
     for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
@@ -1944,6 +2031,14 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     Int = usgn ? Intrinsic::arm_neon_vmullu : Intrinsic::arm_neon_vmulls;
     Int = Type.isPoly() ? (unsigned)Intrinsic::arm_neon_vmullp : Int;
     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vmull");
+  case ARM::BI__builtin_neon_vfma_v:
+  case ARM::BI__builtin_neon_vfmaq_v: {
+    Value *F = CGM.getIntrinsic(Intrinsic::fma, Ty);
+    Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
+    Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
+    Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
+    return Builder.CreateCall3(F, Ops[0], Ops[1], Ops[2]);
+  }
   case ARM::BI__builtin_neon_vpadal_v:
   case ARM::BI__builtin_neon_vpadalq_v: {
     Int = usgn ? Intrinsic::arm_neon_vpadalu : Intrinsic::arm_neon_vpadals;
@@ -2016,7 +2111,8 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     Int = usgn ? Intrinsic::arm_neon_vqrshiftu : Intrinsic::arm_neon_vqrshifts;
     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshl");
   case ARM::BI__builtin_neon_vqrshrn_n_v:
-    Int = usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns;
+    Int =
+      usgn ? Intrinsic::arm_neon_vqrshiftnu : Intrinsic::arm_neon_vqrshiftns;
     return EmitNeonCall(CGM.getIntrinsic(Int, Ty), Ops, "vqrshrn_n",
                         1, true);
   case ARM::BI__builtin_neon_vqrshrun_n_v:
@@ -2086,7 +2182,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
     Ops[2] = EmitNeonShiftVector(Ops[2], Ty, true);
     Int = usgn ? Intrinsic::arm_neon_vrshiftu : Intrinsic::arm_neon_vrshifts;
-    Ops[1] = Builder.CreateCall2(CGM.getIntrinsic(Int, Ty), Ops[1], Ops[2]); 
+    Ops[1] = Builder.CreateCall2(CGM.getIntrinsic(Int, Ty), Ops[1], Ops[2]);
     return Builder.CreateAdd(Ops[0], Ops[1], "vrsra_n");
   case ARM::BI__builtin_neon_vrsubhn_v:
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vrsubhn, Ty),
@@ -2101,7 +2197,8 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
   case ARM::BI__builtin_neon_vshl_n_v:
   case ARM::BI__builtin_neon_vshlq_n_v:
     Ops[1] = EmitNeonShiftVector(Ops[1], Ty, false);
-    return Builder.CreateShl(Builder.CreateBitCast(Ops[0],Ty), Ops[1], "vshl_n");
+    return Builder.CreateShl(Builder.CreateBitCast(Ops[0],Ty), Ops[1],
+                             "vshl_n");
   case ARM::BI__builtin_neon_vshrn_n_v:
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vshiftn, Ty),
                         Ops, "vshrn_n", 1, true);
@@ -2133,7 +2230,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     return Builder.CreateAdd(Ops[0], Ops[1]);
   case ARM::BI__builtin_neon_vst1_v:
   case ARM::BI__builtin_neon_vst1q_v:
-    Ops.push_back(GetPointeeAlignmentValue(E->getArg(0)));
+    Ops.push_back(Align);
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst1, Ty),
                         Ops, "");
   case ARM::BI__builtin_neon_vst1q_lane_v:
@@ -2143,7 +2240,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
       Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
       Value *SV = llvm::ConstantVector::get(cast<llvm::Constant>(Ops[2]));
       Ops[1] = Builder.CreateShuffleVector(Ops[1], Ops[1], SV);
-      Ops[2] = GetPointeeAlignmentValue(E->getArg(0));
+      Ops[2] = Align;
       return Builder.CreateCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst1,
                                                  Ops[1]->getType()), Ops);
     }
@@ -2154,38 +2251,37 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     Ty = llvm::PointerType::getUnqual(Ops[1]->getType());
     StoreInst *St = Builder.CreateStore(Ops[1],
                                         Builder.CreateBitCast(Ops[0], Ty));
-    Value *Align = GetPointeeAlignmentValue(E->getArg(0));
     St->setAlignment(cast<ConstantInt>(Align)->getZExtValue());
     return St;
   }
   case ARM::BI__builtin_neon_vst2_v:
   case ARM::BI__builtin_neon_vst2q_v:
-    Ops.push_back(GetPointeeAlignmentValue(E->getArg(0)));
+    Ops.push_back(Align);
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst2, Ty),
                         Ops, "");
   case ARM::BI__builtin_neon_vst2_lane_v:
   case ARM::BI__builtin_neon_vst2q_lane_v:
-    Ops.push_back(GetPointeeAlignmentValue(E->getArg(0)));
+    Ops.push_back(Align);
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst2lane, Ty),
                         Ops, "");
   case ARM::BI__builtin_neon_vst3_v:
   case ARM::BI__builtin_neon_vst3q_v:
-    Ops.push_back(GetPointeeAlignmentValue(E->getArg(0)));
+    Ops.push_back(Align);
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst3, Ty),
                         Ops, "");
   case ARM::BI__builtin_neon_vst3_lane_v:
   case ARM::BI__builtin_neon_vst3q_lane_v:
-    Ops.push_back(GetPointeeAlignmentValue(E->getArg(0)));
+    Ops.push_back(Align);
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst3lane, Ty),
                         Ops, "");
   case ARM::BI__builtin_neon_vst4_v:
   case ARM::BI__builtin_neon_vst4q_v:
-    Ops.push_back(GetPointeeAlignmentValue(E->getArg(0)));
+    Ops.push_back(Align);
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst4, Ty),
                         Ops, "");
   case ARM::BI__builtin_neon_vst4_lane_v:
   case ARM::BI__builtin_neon_vst4q_lane_v:
-    Ops.push_back(GetPointeeAlignmentValue(E->getArg(0)));
+    Ops.push_back(Align);
     return EmitNeonCall(CGM.getIntrinsic(Intrinsic::arm_neon_vst4lane, Ty),
                         Ops, "");
   case ARM::BI__builtin_neon_vsubhn_v:
@@ -2220,7 +2316,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     Ops[0] = Builder.CreateBitCast(Ops[0], Ty);
     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
     Ops[0] = Builder.CreateAnd(Ops[0], Ops[1]);
-    Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0], 
+    Ops[0] = Builder.CreateICmp(ICmpInst::ICMP_NE, Ops[0],
                                 ConstantAggregateZero::get(Ty));
     return Builder.CreateSExt(Ops[0], Ty, "vtst");
   }
@@ -2250,7 +2346,7 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
     Value *SV = 0;
-    
+
     for (unsigned vi = 0; vi != 2; ++vi) {
       SmallVector<Constant*, 16> Indices;
       for (unsigned i = 0, e = VTy->getNumElements(); i != e; ++i)
@@ -2263,13 +2359,13 @@ Value *CodeGenFunction::EmitARMBuiltinExpr(unsigned BuiltinID,
     }
     return SV;
   }
-  case ARM::BI__builtin_neon_vzip_v: 
+  case ARM::BI__builtin_neon_vzip_v:
   case ARM::BI__builtin_neon_vzipq_v: {
     Ops[0] = Builder.CreateBitCast(Ops[0], llvm::PointerType::getUnqual(Ty));
     Ops[1] = Builder.CreateBitCast(Ops[1], Ty);
     Ops[2] = Builder.CreateBitCast(Ops[2], Ty);
     Value *SV = 0;
-    
+
     for (unsigned vi = 0; vi != 2; ++vi) {
       SmallVector<Constant*, 16> Indices;
       for (unsigned i = 0, e = VTy->getNumElements(); i != e; i += 2) {
@@ -2382,62 +2478,62 @@ Value *CodeGenFunction::EmitX86BuiltinExpr(unsigned BuiltinID,
   }
   case X86::BI__builtin_ia32_palignr: {
     unsigned shiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
-    
+
     // If palignr is shifting the pair of input vectors less than 9 bytes,
     // emit a shuffle instruction.
     if (shiftVal <= 8) {
       SmallVector<llvm::Constant*, 8> Indices;
       for (unsigned i = 0; i != 8; ++i)
         Indices.push_back(llvm::ConstantInt::get(Int32Ty, shiftVal + i));
-      
+
       Value* SV = llvm::ConstantVector::get(Indices);
       return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr");
     }
-    
+
     // If palignr is shifting the pair of input vectors more than 8 but less
     // than 16 bytes, emit a logical right shift of the destination.
     if (shiftVal < 16) {
       // MMX has these as 1 x i64 vectors for some odd optimization reasons.
       llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 1);
-      
+
       Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast");
       Ops[1] = llvm::ConstantInt::get(VecTy, (shiftVal-8) * 8);
-      
+
       // create i32 constant
       llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_mmx_psrl_q);
       return Builder.CreateCall(F, makeArrayRef(&Ops[0], 2), "palignr");
     }
-    
+
     // If palignr is shifting the pair of vectors more than 16 bytes, emit zero.
     return llvm::Constant::getNullValue(ConvertType(E->getType()));
   }
   case X86::BI__builtin_ia32_palignr128: {
     unsigned shiftVal = cast<llvm::ConstantInt>(Ops[2])->getZExtValue();
-    
+
     // If palignr is shifting the pair of input vectors less than 17 bytes,
     // emit a shuffle instruction.
     if (shiftVal <= 16) {
       SmallVector<llvm::Constant*, 16> Indices;
       for (unsigned i = 0; i != 16; ++i)
         Indices.push_back(llvm::ConstantInt::get(Int32Ty, shiftVal + i));
-      
+
       Value* SV = llvm::ConstantVector::get(Indices);
       return Builder.CreateShuffleVector(Ops[1], Ops[0], SV, "palignr");
     }
-    
+
     // If palignr is shifting the pair of input vectors more than 16 but less
     // than 32 bytes, emit a logical right shift of the destination.
     if (shiftVal < 32) {
       llvm::Type *VecTy = llvm::VectorType::get(Int64Ty, 2);
-      
+
       Ops[0] = Builder.CreateBitCast(Ops[0], VecTy, "cast");
       Ops[1] = llvm::ConstantInt::get(Int32Ty, (shiftVal-16) * 8);
-      
+
       // create i32 constant
       llvm::Function *F = CGM.getIntrinsic(Intrinsic::x86_sse2_psrl_dq);
       return Builder.CreateCall(F, makeArrayRef(&Ops[0], 2), "palignr");
     }
-    
+
     // If palignr is shifting the pair of vectors more than 32 bytes, emit zero.
     return llvm::Constant::getNullValue(ConvertType(E->getType()));
   }
diff --git a/lib/CodeGen/CGCXXABI.cpp b/lib/CodeGen/CGCXXABI.cpp
index aba5d75..91795b9 100644
--- a/lib/CodeGen/CGCXXABI.cpp
+++ b/lib/CodeGen/CGCXXABI.cpp
@@ -189,7 +189,7 @@ void CGCXXABI::ReadArrayCookie(CodeGenFunction &CGF, llvm::Value *ptr,
                                llvm::Value *&numElements,
                                llvm::Value *&allocPtr, CharUnits &cookieSize) {
   // Derive a char* in the same address space as the pointer.
-  unsigned AS = cast<llvm::PointerType>(ptr->getType())->getAddressSpace();
+  unsigned AS = ptr->getType()->getPointerAddressSpace();
   llvm::Type *charPtrTy = CGF.Int8Ty->getPointerTo(AS);
   ptr = CGF.Builder.CreateBitCast(ptr, charPtrTy);
 
diff --git a/lib/CodeGen/CGCXXABI.h b/lib/CodeGen/CGCXXABI.h
index a0dcdfd..570aeb0 100644
--- a/lib/CodeGen/CGCXXABI.h
+++ b/lib/CodeGen/CGCXXABI.h
@@ -154,6 +154,15 @@ protected:
   llvm::Constant *getMemberPointerAdjustment(const CastExpr *E);
 
 public:
+  /// Adjust the given non-null pointer to an object of polymorphic
+  /// type to point to the complete object.
+  ///
+  /// The IR type of the result should be a pointer but is otherwise
+  /// irrelevant.
+  virtual llvm::Value *adjustToCompleteObject(CodeGenFunction &CGF,
+                                              llvm::Value *ptr,
+                                              QualType type) = 0;
+
   /// Build the signature of the given constructor variant by adding
   /// any required parameters.  For convenience, ResTy has been
   /// initialized to 'void', and ArgTys has been initialized with the
@@ -196,6 +205,9 @@ public:
   /// Gets the pure virtual member call function.
   virtual StringRef GetPureVirtualCallName() = 0;
 
+  /// Gets the deleted virtual member call name.
+  virtual StringRef GetDeletedVirtualCallName() = 0;
+
   /**************************** Array cookies ******************************/
 
   /// Returns the extra size required in order to store the array
diff --git a/lib/CodeGen/CGCall.cpp b/lib/CodeGen/CGCall.cpp
index 7d2b9d3..2d1d152 100644
--- a/lib/CodeGen/CGCall.cpp
+++ b/lib/CodeGen/CGCall.cpp
@@ -25,7 +25,7 @@
 #include "clang/Frontend/CodeGenOptions.h"
 #include "llvm/Attributes.h"
 #include "llvm/Support/CallSite.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/InlineAsm.h"
 #include "llvm/Transforms/Utils/Local.h"
 using namespace clang;
@@ -148,6 +148,9 @@ static CallingConv getCallingConventionForDecl(const Decl *D) {
   if (PcsAttr *PCS = D->getAttr<PcsAttr>())
     return (PCS->getPCS() == PcsAttr::AAPCS ? CC_AAPCS : CC_AAPCS_VFP);
 
+  if (D->hasAttr<PnaclCallAttr>())
+    return CC_PnaclCall;
+
   return CC_C;
 }
 
@@ -588,9 +591,9 @@ EnterStructPointerForCoercedAccess(llvm::Value *SrcPtr,
   // If the first elt is at least as large as what we're looking for, or if the
   // first element is the same size as the whole struct, we can enter it.
   uint64_t FirstEltSize =
-    CGF.CGM.getTargetData().getTypeAllocSize(FirstElt);
+    CGF.CGM.getDataLayout().getTypeAllocSize(FirstElt);
   if (FirstEltSize < DstSize &&
-      FirstEltSize < CGF.CGM.getTargetData().getTypeAllocSize(SrcSTy))
+      FirstEltSize < CGF.CGM.getDataLayout().getTypeAllocSize(SrcSTy))
     return SrcPtr;
 
   // GEP into the first element.
@@ -653,14 +656,14 @@ static llvm::Value *CreateCoercedLoad(llvm::Value *SrcPtr,
   if (SrcTy == Ty)
     return CGF.Builder.CreateLoad(SrcPtr);
 
-  uint64_t DstSize = CGF.CGM.getTargetData().getTypeAllocSize(Ty);
+  uint64_t DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(Ty);
 
   if (llvm::StructType *SrcSTy = dyn_cast<llvm::StructType>(SrcTy)) {
     SrcPtr = EnterStructPointerForCoercedAccess(SrcPtr, SrcSTy, DstSize, CGF);
     SrcTy = cast<llvm::PointerType>(SrcPtr->getType())->getElementType();
   }
 
-  uint64_t SrcSize = CGF.CGM.getTargetData().getTypeAllocSize(SrcTy);
+  uint64_t SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy);
 
   // If the source and destination are integer or pointer types, just do an
   // extension or truncation to the desired type.
@@ -740,7 +743,7 @@ static void CreateCoercedStore(llvm::Value *Src,
     return;
   }
 
-  uint64_t SrcSize = CGF.CGM.getTargetData().getTypeAllocSize(SrcTy);
+  uint64_t SrcSize = CGF.CGM.getDataLayout().getTypeAllocSize(SrcTy);
 
   if (llvm::StructType *DstSTy = dyn_cast<llvm::StructType>(DstTy)) {
     DstPtr = EnterStructPointerForCoercedAccess(DstPtr, DstSTy, SrcSize, CGF);
@@ -756,7 +759,7 @@ static void CreateCoercedStore(llvm::Value *Src,
     return;
   }
 
-  uint64_t DstSize = CGF.CGM.getTargetData().getTypeAllocSize(DstTy);
+  uint64_t DstSize = CGF.CGM.getDataLayout().getTypeAllocSize(DstTy);
 
   // If store is legal, just bitcast the src pointer.
   if (SrcSize <= DstSize) {
@@ -864,6 +867,10 @@ CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI) {
          ie = FI.arg_end(); it != ie; ++it) {
     const ABIArgInfo &argAI = it->info;
 
+    // Insert a padding type to ensure proper alignment.
+    if (llvm::Type *PaddingType = argAI.getPaddingType())
+      argTypes.push_back(PaddingType);
+
     switch (argAI.getKind()) {
     case ABIArgInfo::Ignore:
       break;
@@ -877,9 +884,6 @@ CodeGenTypes::GetFunctionType(const CGFunctionInfo &FI) {
 
     case ABIArgInfo::Extend:
     case ABIArgInfo::Direct: {
-      // Insert a padding type to ensure proper alignment.
-      if (llvm::Type *PaddingType = argAI.getPaddingType())
-        argTypes.push_back(PaddingType);
       // If the coerce-to type is a first class aggregate, flatten it.  Either
       // way is semantically identical, but fast-isel and the optimizer
       // generally likes scalar values better than FCAs.
@@ -924,50 +928,52 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI,
                                            const Decl *TargetDecl,
                                            AttributeListType &PAL,
                                            unsigned &CallingConv) {
-  llvm::Attributes FuncAttrs;
-  llvm::Attributes RetAttrs;
+  llvm::AttrBuilder FuncAttrs;
+  llvm::AttrBuilder RetAttrs;
 
   CallingConv = FI.getEffectiveCallingConvention();
 
   if (FI.isNoReturn())
-    FuncAttrs |= llvm::Attribute::NoReturn;
+    FuncAttrs.addAttribute(llvm::Attributes::NoReturn);
 
   // FIXME: handle sseregparm someday...
   if (TargetDecl) {
     if (TargetDecl->hasAttr<ReturnsTwiceAttr>())
-      FuncAttrs |= llvm::Attribute::ReturnsTwice;
+      FuncAttrs.addAttribute(llvm::Attributes::ReturnsTwice);
     if (TargetDecl->hasAttr<NoThrowAttr>())
-      FuncAttrs |= llvm::Attribute::NoUnwind;
+      FuncAttrs.addAttribute(llvm::Attributes::NoUnwind);
     else if (const FunctionDecl *Fn = dyn_cast<FunctionDecl>(TargetDecl)) {
       const FunctionProtoType *FPT = Fn->getType()->getAs<FunctionProtoType>();
       if (FPT && FPT->isNothrow(getContext()))
-        FuncAttrs |= llvm::Attribute::NoUnwind;
+        FuncAttrs.addAttribute(llvm::Attributes::NoUnwind);
     }
 
     if (TargetDecl->hasAttr<NoReturnAttr>())
-      FuncAttrs |= llvm::Attribute::NoReturn;
+      FuncAttrs.addAttribute(llvm::Attributes::NoReturn);
 
     if (TargetDecl->hasAttr<ReturnsTwiceAttr>())
-      FuncAttrs |= llvm::Attribute::ReturnsTwice;
+      FuncAttrs.addAttribute(llvm::Attributes::ReturnsTwice);
 
     // 'const' and 'pure' attribute functions are also nounwind.
     if (TargetDecl->hasAttr<ConstAttr>()) {
-      FuncAttrs |= llvm::Attribute::ReadNone;
-      FuncAttrs |= llvm::Attribute::NoUnwind;
+      FuncAttrs.addAttribute(llvm::Attributes::ReadNone);
+      FuncAttrs.addAttribute(llvm::Attributes::NoUnwind);
     } else if (TargetDecl->hasAttr<PureAttr>()) {
-      FuncAttrs |= llvm::Attribute::ReadOnly;
-      FuncAttrs |= llvm::Attribute::NoUnwind;
+      FuncAttrs.addAttribute(llvm::Attributes::ReadOnly);
+      FuncAttrs.addAttribute(llvm::Attributes::NoUnwind);
     }
     if (TargetDecl->hasAttr<MallocAttr>())
-      RetAttrs |= llvm::Attribute::NoAlias;
+      RetAttrs.addAttribute(llvm::Attributes::NoAlias);
   }
 
   if (CodeGenOpts.OptimizeSize)
-    FuncAttrs |= llvm::Attribute::OptimizeForSize;
+    FuncAttrs.addAttribute(llvm::Attributes::OptimizeForSize);
+  if (CodeGenOpts.OptimizeSize == 2)
+    FuncAttrs.addAttribute(llvm::Attributes::MinSize);
   if (CodeGenOpts.DisableRedZone)
-    FuncAttrs |= llvm::Attribute::NoRedZone;
+    FuncAttrs.addAttribute(llvm::Attributes::NoRedZone);
   if (CodeGenOpts.NoImplicitFloat)
-    FuncAttrs |= llvm::Attribute::NoImplicitFloat;
+    FuncAttrs.addAttribute(llvm::Attributes::NoImplicitFloat);
 
   QualType RetTy = FI.getReturnType();
   unsigned Index = 1;
@@ -975,24 +981,28 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI,
   switch (RetAI.getKind()) {
   case ABIArgInfo::Extend:
    if (RetTy->hasSignedIntegerRepresentation())
-     RetAttrs |= llvm::Attribute::SExt;
+     RetAttrs.addAttribute(llvm::Attributes::SExt);
    else if (RetTy->hasUnsignedIntegerRepresentation())
-     RetAttrs |= llvm::Attribute::ZExt;
+     RetAttrs.addAttribute(llvm::Attributes::ZExt);
     break;
   case ABIArgInfo::Direct:
   case ABIArgInfo::Ignore:
     break;
 
   case ABIArgInfo::Indirect: {
-    llvm::Attributes SRETAttrs = llvm::Attribute::StructRet;
+    llvm::AttrBuilder SRETAttrs;
+    SRETAttrs.addAttribute(llvm::Attributes::StructRet);
     if (RetAI.getInReg())
-      SRETAttrs |= llvm::Attribute::InReg;
-    PAL.push_back(llvm::AttributeWithIndex::get(Index, SRETAttrs));
+      SRETAttrs.addAttribute(llvm::Attributes::InReg);
+    PAL.push_back(llvm::
+                  AttributeWithIndex::get(Index,
+                                         llvm::Attributes::get(getLLVMContext(),
+                                                               SRETAttrs)));
 
     ++Index;
     // sret disables readnone and readonly
-    FuncAttrs &= ~(llvm::Attribute::ReadOnly |
-                   llvm::Attribute::ReadNone);
+    FuncAttrs.removeAttribute(llvm::Attributes::ReadOnly)
+      .removeAttribute(llvm::Attributes::ReadNone);
     break;
   }
 
@@ -1000,14 +1010,29 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI,
     llvm_unreachable("Invalid ABI kind for return argument");
   }
 
-  if (RetAttrs)
-    PAL.push_back(llvm::AttributeWithIndex::get(0, RetAttrs));
+  if (RetAttrs.hasAttributes())
+    PAL.push_back(llvm::
+                  AttributeWithIndex::get(llvm::AttrListPtr::ReturnIndex,
+                                         llvm::Attributes::get(getLLVMContext(),
+                                                               RetAttrs)));
 
   for (CGFunctionInfo::const_arg_iterator it = FI.arg_begin(),
          ie = FI.arg_end(); it != ie; ++it) {
     QualType ParamType = it->type;
     const ABIArgInfo &AI = it->info;
-    llvm::Attributes Attrs;
+    llvm::AttrBuilder Attrs;
+
+    if (AI.getPaddingType()) {
+      if (AI.getPaddingInReg()) {
+        llvm::AttrBuilder PadAttrs;
+        PadAttrs.addAttribute(llvm::Attributes::InReg);
+
+        llvm::Attributes A =llvm::Attributes::get(getLLVMContext(), PadAttrs);
+        PAL.push_back(llvm::AttributeWithIndex::get(Index, A));
+      }
+      // Increment Index if there is padding.
+      ++Index;
+    }
 
     // 'restrict' -> 'noalias' is done in EmitFunctionProlog when we
     // have the corresponding parameter variable.  It doesn't make
@@ -1015,38 +1040,40 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI,
     switch (AI.getKind()) {
     case ABIArgInfo::Extend:
       if (ParamType->isSignedIntegerOrEnumerationType())
-        Attrs |= llvm::Attribute::SExt;
+        Attrs.addAttribute(llvm::Attributes::SExt);
       else if (ParamType->isUnsignedIntegerOrEnumerationType())
-        Attrs |= llvm::Attribute::ZExt;
+        Attrs.addAttribute(llvm::Attributes::ZExt);
       // FALL THROUGH
     case ABIArgInfo::Direct:
       if (AI.getInReg())
-          Attrs |= llvm::Attribute::InReg;
+        Attrs.addAttribute(llvm::Attributes::InReg);
 
       // FIXME: handle sseregparm someday...
 
-      // Increment Index if there is padding.
-      Index += (AI.getPaddingType() != 0);
-
       if (llvm::StructType *STy =
           dyn_cast<llvm::StructType>(AI.getCoerceToType())) {
         unsigned Extra = STy->getNumElements()-1;  // 1 will be added below.
-        if (Attrs != llvm::Attribute::None)
+        if (Attrs.hasAttributes())
           for (unsigned I = 0; I < Extra; ++I)
-            PAL.push_back(llvm::AttributeWithIndex::get(Index + I, Attrs));
+            PAL.push_back(llvm::AttributeWithIndex::get(Index + I,
+                                         llvm::Attributes::get(getLLVMContext(),
+                                                               Attrs)));
         Index += Extra;
       }
       break;
 
     case ABIArgInfo::Indirect:
+      if (AI.getInReg())
+        Attrs.addAttribute(llvm::Attributes::InReg);
+
       if (AI.getIndirectByVal())
-        Attrs |= llvm::Attribute::ByVal;
+        Attrs.addAttribute(llvm::Attributes::ByVal);
+
+      Attrs.addAlignmentAttr(AI.getIndirectAlign());
 
-      Attrs |=
-        llvm::Attribute::constructAlignmentFromInt(AI.getIndirectAlign());
       // byval disables readnone and readonly.
-      FuncAttrs &= ~(llvm::Attribute::ReadOnly |
-                     llvm::Attribute::ReadNone);
+      FuncAttrs.removeAttribute(llvm::Attributes::ReadOnly)
+        .removeAttribute(llvm::Attributes::ReadNone);
       break;
 
     case ABIArgInfo::Ignore:
@@ -1064,12 +1091,17 @@ void CodeGenModule::ConstructAttributeList(const CGFunctionInfo &FI,
     }
     }
 
-    if (Attrs)
-      PAL.push_back(llvm::AttributeWithIndex::get(Index, Attrs));
+    if (Attrs.hasAttributes())
+      PAL.push_back(llvm::AttributeWithIndex::get(Index,
+                                         llvm::Attributes::get(getLLVMContext(),
+                                                               Attrs)));
     ++Index;
   }
-  if (FuncAttrs)
-    PAL.push_back(llvm::AttributeWithIndex::get(~0, FuncAttrs));
+  if (FuncAttrs.hasAttributes())
+    PAL.push_back(llvm::
+                  AttributeWithIndex::get(llvm::AttrListPtr::FunctionIndex,
+                                         llvm::Attributes::get(getLLVMContext(),
+                                                               FuncAttrs)));
 }
 
 /// An argument came in as a promoted argument; demote it back to its
@@ -1117,7 +1149,8 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
   // Name the struct return argument.
   if (CGM.ReturnTypeUsesSRet(FI)) {
     AI->setName("agg.result");
-    AI->addAttr(llvm::Attribute::NoAlias);
+    AI->addAttr(llvm::Attributes::get(getLLVMContext(),
+                                      llvm::Attributes::NoAlias));
     ++AI;
   }
 
@@ -1134,6 +1167,10 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
     bool isPromoted =
       isa<ParmVarDecl>(Arg) && cast<ParmVarDecl>(Arg)->isKNRPromoted();
 
+    // Skip the dummy padding argument.
+    if (ArgI.getPaddingType())
+      ++AI;
+
     switch (ArgI.getKind()) {
     case ABIArgInfo::Indirect: {
       llvm::Value *V = AI;
@@ -1175,9 +1212,6 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
 
     case ABIArgInfo::Extend:
     case ABIArgInfo::Direct: {
-      // Skip the dummy padding argument.
-      if (ArgI.getPaddingType())
-        ++AI;
 
       // If we have the trivial case, handle it with no muss and fuss.
       if (!isa<llvm::StructType>(ArgI.getCoerceToType()) &&
@@ -1187,7 +1221,8 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
         llvm::Value *V = AI;
 
         if (Arg->getType().isRestrictQualified())
-          AI->addAttr(llvm::Attribute::NoAlias);
+          AI->addAttr(llvm::Attributes::get(getLLVMContext(),
+                                            llvm::Attributes::NoAlias));
 
         // Ensure the argument is the correct type.
         if (V->getType() != ArgI.getCoerceToType())
@@ -1205,7 +1240,7 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
       // The alignment we need to use is the max of the requested alignment for
       // the argument plus the alignment required by our access code below.
       unsigned AlignmentToUse =
-        CGM.getTargetData().getABITypeAlignment(ArgI.getCoerceToType());
+        CGM.getDataLayout().getABITypeAlignment(ArgI.getCoerceToType());
       AlignmentToUse = std::max(AlignmentToUse,
                         (unsigned)getContext().getDeclAlign(Arg).getQuantity());
 
@@ -1226,10 +1261,10 @@ void CodeGenFunction::EmitFunctionProlog(const CGFunctionInfo &FI,
       // and the optimizer generally likes scalar values better than FCAs.
       llvm::StructType *STy = dyn_cast<llvm::StructType>(ArgI.getCoerceToType());
       if (STy && STy->getNumElements() > 1) {
-        uint64_t SrcSize = CGM.getTargetData().getTypeAllocSize(STy);
+        uint64_t SrcSize = CGM.getDataLayout().getTypeAllocSize(STy);
         llvm::Type *DstTy =
           cast<llvm::PointerType>(Ptr->getType())->getElementType();
-        uint64_t DstSize = CGM.getTargetData().getTypeAllocSize(DstTy);
+        uint64_t DstSize = CGM.getDataLayout().getTypeAllocSize(DstTy);
 
         if (SrcSize <= DstSize) {
           Ptr = Builder.CreateBitCast(Ptr, llvm::PointerType::getUnqual(STy));
@@ -1363,12 +1398,23 @@ static llvm::Value *tryEmitFusedAutoreleaseOfResult(CodeGenFunction &CGF,
                                           .objc_retainAutoreleasedReturnValue) {
     doRetainAutorelease = false;
 
-    // Look for an inline asm immediately preceding the call and kill it, too.
-    llvm::Instruction *prev = call->getPrevNode();
-    if (llvm::CallInst *asmCall = dyn_cast_or_null<llvm::CallInst>(prev))
-      if (asmCall->getCalledValue()
-            == CGF.CGM.getARCEntrypoints().retainAutoreleasedReturnValueMarker)
-        insnsToKill.push_back(prev);
+    // If we emitted an assembly marker for this call (and the
+    // ARCEntrypoints field should have been set if so), go looking
+    // for that call.  If we can't find it, we can't do this
+    // optimization.  But it should always be the immediately previous
+    // instruction, unless we needed bitcasts around the call.
+    if (CGF.CGM.getARCEntrypoints().retainAutoreleasedReturnValueMarker) {
+      llvm::Instruction *prev = call->getPrevNode();
+      assert(prev);
+      if (isa<llvm::BitCastInst>(prev)) {
+        prev = prev->getPrevNode();
+        assert(prev);
+      }
+      assert(isa<llvm::CallInst>(prev));
+      assert(cast<llvm::CallInst>(prev)->getCalledValue() ==
+               CGF.CGM.getARCEntrypoints().retainAutoreleasedReturnValueMarker);
+      insnsToKill.push_back(prev);
+    }
   } else {
     return 0;
   }
@@ -1755,7 +1801,7 @@ void CodeGenFunction::EmitCallArg(CallArgList &args, const Expr *E,
                                   QualType type) {
   if (const ObjCIndirectCopyRestoreExpr *CRE
         = dyn_cast<ObjCIndirectCopyRestoreExpr>(E)) {
-    assert(getContext().getLangOpts().ObjCAutoRefCount);
+    assert(getLangOpts().ObjCAutoRefCount);
     assert(getContext().hasSameType(E->getType(), type));
     return emitWritebackArg(*this, args, CRE);
   }
@@ -1943,6 +1989,13 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
 
     unsigned TypeAlign =
       getContext().getTypeAlignInChars(I->Ty).getQuantity();
+
+    // Insert a padding argument to ensure proper alignment.
+    if (llvm::Type *PaddingType = ArgInfo.getPaddingType()) {
+      Args.push_back(llvm::UndefValue::get(PaddingType));
+      ++IRArgNo;
+    }
+
     switch (ArgInfo.getKind()) {
     case ABIArgInfo::Indirect: {
       if (RV.isScalar() || RV.isComplex()) {
@@ -1969,7 +2022,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
         //    we cannot force it to be sufficiently aligned.
         llvm::Value *Addr = RV.getAggregateAddr();
         unsigned Align = ArgInfo.getIndirectAlign();
-        const llvm::TargetData *TD = &CGM.getTargetData();
+        const llvm::DataLayout *TD = &CGM.getDataLayout();
         if ((!ArgInfo.getIndirectByVal() && I->NeedsCopy) ||
             (ArgInfo.getIndirectByVal() && TypeAlign < Align &&
              llvm::getOrEnforceKnownAlignment(Addr, Align, TD) < Align)) {
@@ -1998,12 +2051,6 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
 
     case ABIArgInfo::Extend:
     case ABIArgInfo::Direct: {
-      // Insert a padding argument to ensure proper alignment.
-      if (llvm::Type *PaddingType = ArgInfo.getPaddingType()) {
-        Args.push_back(llvm::UndefValue::get(PaddingType));
-        ++IRArgNo;
-      }
-
       if (!isa<llvm::StructType>(ArgInfo.getCoerceToType()) &&
           ArgInfo.getCoerceToType() == ConvertType(info_it->type) &&
           ArgInfo.getDirectOffset() == 0) {
@@ -2049,8 +2096,25 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
       // and the optimizer generally likes scalar values better than FCAs.
       if (llvm::StructType *STy =
             dyn_cast<llvm::StructType>(ArgInfo.getCoerceToType())) {
-        SrcPtr = Builder.CreateBitCast(SrcPtr,
-                                       llvm::PointerType::getUnqual(STy));
+        llvm::Type *SrcTy =
+          cast<llvm::PointerType>(SrcPtr->getType())->getElementType();
+        uint64_t SrcSize = CGM.getDataLayout().getTypeAllocSize(SrcTy);
+        uint64_t DstSize = CGM.getDataLayout().getTypeAllocSize(STy);
+
+        // If the source type is smaller than the destination type of the
+        // coerce-to logic, copy the source value into a temp alloca the size
+        // of the destination type to allow loading all of it. The bits past
+        // the source value are left undef.
+        if (SrcSize < DstSize) {
+          llvm::AllocaInst *TempAlloca
+            = CreateTempAlloca(STy, SrcPtr->getName() + ".coerce");
+          Builder.CreateMemCpy(TempAlloca, SrcPtr, SrcSize, 0);
+          SrcPtr = TempAlloca;
+        } else {
+          SrcPtr = Builder.CreateBitCast(SrcPtr,
+                                         llvm::PointerType::getUnqual(STy));
+        }
+
         for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
           llvm::Value *EltPtr = Builder.CreateConstGEP2_32(SrcPtr, 0, i);
           llvm::LoadInst *LI = Builder.CreateLoad(EltPtr);
@@ -2113,10 +2177,11 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
   unsigned CallingConv;
   CodeGen::AttributeListType AttributeList;
   CGM.ConstructAttributeList(CallInfo, TargetDecl, AttributeList, CallingConv);
-  llvm::AttrListPtr Attrs = llvm::AttrListPtr::get(AttributeList);
+  llvm::AttrListPtr Attrs = llvm::AttrListPtr::get(getLLVMContext(),
+                                                   AttributeList);
 
   llvm::BasicBlock *InvokeDest = 0;
-  if (!(Attrs.getFnAttributes() & llvm::Attribute::NoUnwind))
+  if (!Attrs.getFnAttributes().hasAttribute(llvm::Attributes::NoUnwind))
     InvokeDest = getInvokeDest();
 
   llvm::CallSite CS;
diff --git a/lib/CodeGen/CGClass.cpp b/lib/CodeGen/CGClass.cpp
index e37fa3a..b2225e4 100644
--- a/lib/CodeGen/CGClass.cpp
+++ b/lib/CodeGen/CGClass.cpp
@@ -542,12 +542,6 @@ namespace {
   };
 }
 
-static bool hasTrivialCopyOrMoveConstructor(const CXXRecordDecl *Record,
-                                            bool Moving) {
-  return Moving ? Record->hasTrivialMoveConstructor() :
-                  Record->hasTrivialCopyConstructor();
-}
-
 static void EmitMemberInitializer(CodeGenFunction &CGF,
                                   const CXXRecordDecl *ClassDecl,
                                   CXXCtorInitializer *MemberInit,
@@ -588,12 +582,11 @@ static void EmitMemberInitializer(CodeGenFunction &CGF,
   if (Array && Constructor->isImplicitlyDefined() &&
       Constructor->isCopyOrMoveConstructor()) {
     QualType BaseElementTy = CGF.getContext().getBaseElementType(Array);
-    const CXXRecordDecl *Record = BaseElementTy->getAsCXXRecordDecl();
+    CXXConstructExpr *CE = dyn_cast<CXXConstructExpr>(MemberInit->getInit());
     if (BaseElementTy.isPODType(CGF.getContext()) ||
-        (Record && hasTrivialCopyOrMoveConstructor(Record,
-                       Constructor->isMoveConstructor()))) {
-      // Find the source pointer. We knows it's the last argument because
-      // we know we're in a copy constructor.
+        (CE && CE->getConstructor()->isTrivial())) {
+      // Find the source pointer. We know it's the last argument because
+      // we know we're in an implicit copy constructor.
       unsigned SrcArgIndex = Args.size() - 1;
       llvm::Value *SrcPtr
         = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(Args[SrcArgIndex]));
@@ -952,8 +945,8 @@ void CodeGenFunction::EmitDestructorBody(FunctionArgList &Args) {
     }
     // -fapple-kext must inline any call to this dtor into
     // the caller's body.
-    if (getContext().getLangOpts().AppleKext)
-      CurFn->addFnAttr(llvm::Attribute::AlwaysInline);
+    if (getLangOpts().AppleKext)
+      CurFn->addFnAttr(llvm::Attributes::AlwaysInline);
     break;
   }
 
@@ -1238,7 +1231,7 @@ CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
 
   CGDebugInfo *DI = getDebugInfo();
   if (DI &&
-      CGM.getCodeGenOpts().DebugInfo == CodeGenOptions::LimitedDebugInfo) {
+      CGM.getCodeGenOpts().getDebugInfo() == CodeGenOptions::LimitedDebugInfo) {
     // If debug info for this class has not been emitted then this is the
     // right time to do so.
     const CXXRecordDecl *Parent = D->getParent();
@@ -1268,7 +1261,9 @@ CodeGenFunction::EmitCXXConstructorCall(const CXXConstructorDecl *D,
   llvm::Value *VTT = GetVTTParameter(*this, GlobalDecl(D, Type), ForVirtualBase);
   llvm::Value *Callee = CGM.GetAddrOfCXXConstructor(D, Type);
 
-  EmitCXXMemberCall(D, Callee, ReturnValueSlot(), This, VTT, ArgBeg, ArgEnd);
+  // FIXME: Provide a source location here.
+  EmitCXXMemberCall(D, SourceLocation(), Callee, ReturnValueSlot(), This,
+                    VTT, ArgBeg, ArgEnd);
 }
 
 void
@@ -1413,14 +1408,16 @@ void CodeGenFunction::EmitCXXDestructorCall(const CXXDestructorDecl *DD,
   llvm::Value *VTT = GetVTTParameter(*this, GlobalDecl(DD, Type), 
                                      ForVirtualBase);
   llvm::Value *Callee = 0;
-  if (getContext().getLangOpts().AppleKext)
+  if (getLangOpts().AppleKext)
     Callee = BuildAppleKextVirtualDestructorCall(DD, Type, 
                                                  DD->getParent());
     
   if (!Callee)
     Callee = CGM.GetAddrOfCXXDestructor(DD, Type);
   
-  EmitCXXMemberCall(DD, Callee, ReturnValueSlot(), This, VTT, 0, 0);
+  // FIXME: Provide a source location here.
+  EmitCXXMemberCall(DD, SourceLocation(), Callee, ReturnValueSlot(), This,
+                    VTT, 0, 0);
 }
 
 namespace {
diff --git a/lib/CodeGen/CGDebugInfo.cpp b/lib/CodeGen/CGDebugInfo.cpp
index fd1c7a3..80fa09b 100644
--- a/lib/CodeGen/CGDebugInfo.cpp
+++ b/lib/CodeGen/CGDebugInfo.cpp
@@ -15,6 +15,7 @@
 #include "CodeGenFunction.h"
 #include "CodeGenModule.h"
 #include "CGBlocks.h"
+#include "CGObjCRuntime.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/DeclFriend.h"
 #include "clang/AST/DeclObjC.h"
@@ -34,7 +35,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/Dwarf.h"
 #include "llvm/Support/FileSystem.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 using namespace clang;
 using namespace clang::CodeGen;
 
@@ -157,7 +158,7 @@ StringRef CGDebugInfo::getObjCMethodName(const ObjCMethodDecl *OMD) {
       OS << OID->getName();
   } else if (const ObjCCategoryImplDecl *OCD = 
              dyn_cast<const ObjCCategoryImplDecl>(DC)){
-      OS << ((NamedDecl *)OCD)->getIdentifier()->getNameStart() << '(' <<
+      OS << ((const NamedDecl *)OCD)->getIdentifier()->getNameStart() << '(' <<
           OCD->getIdentifier()->getNameStart() << ')';
   }
   OS << ' ' << OMD->getSelector().getAsString() << ']';
@@ -254,9 +255,13 @@ unsigned CGDebugInfo::getLineNumber(SourceLocation Loc) {
   return PLoc.isValid()? PLoc.getLine() : 0;
 }
 
-/// getColumnNumber - Get column number for the location. If location is 
-/// invalid then use current location.
+/// getColumnNumber - Get column number for the location.
 unsigned CGDebugInfo::getColumnNumber(SourceLocation Loc) {
+  // We may not want column information at all.
+  if (!CGM.getCodeGenOpts().DebugColumnInfo)
+    return 0;
+
+  // If the location is invalid then use the current column.
   if (Loc.isInvalid() && CurLoc.isInvalid())
     return 0;
   SourceManager &SM = CGM.getContext().getSourceManager();
@@ -347,44 +352,60 @@ llvm::DIType CGDebugInfo::CreateType(const BuiltinType *BT) {
     llvm_unreachable("Unexpected builtin type");
   case BuiltinType::NullPtr:
     return DBuilder.
-      createNullPtrType(BT->getName(CGM.getContext().getLangOpts()));
+      createNullPtrType(BT->getName(CGM.getLangOpts()));
   case BuiltinType::Void:
     return llvm::DIType();
   case BuiltinType::ObjCClass:
-    return DBuilder.createForwardDecl(llvm::dwarf::DW_TAG_structure_type,
-                                      "objc_class", TheCU,
-                                      getOrCreateMainFile(), 0);
+    if (ClassTy.Verify())
+      return ClassTy;
+    ClassTy = DBuilder.createForwardDecl(llvm::dwarf::DW_TAG_structure_type,
+                                         "objc_class", TheCU,
+                                         getOrCreateMainFile(), 0);
+    return ClassTy;
   case BuiltinType::ObjCId: {
     // typedef struct objc_class *Class;
     // typedef struct objc_object {
     //  Class isa;
     // } *id;
 
-    // TODO: Cache these two types to avoid duplicates.
-    llvm::DIType OCTy =
-      DBuilder.createForwardDecl(llvm::dwarf::DW_TAG_structure_type,
-                                 "objc_class", TheCU, getOrCreateMainFile(), 0);
+    if (ObjTy.Verify())
+      return ObjTy;
+
+    if (!ClassTy.Verify())
+      ClassTy = DBuilder.createForwardDecl(llvm::dwarf::DW_TAG_structure_type,
+                                           "objc_class", TheCU,
+                                           getOrCreateMainFile(), 0);
+
     unsigned Size = CGM.getContext().getTypeSize(CGM.getContext().VoidPtrTy);
     
-    llvm::DIType ISATy = DBuilder.createPointerType(OCTy, Size);
+    llvm::DIType ISATy = DBuilder.createPointerType(ClassTy, Size);
+
+    llvm::DIType FwdTy =  DBuilder.createStructType(TheCU, "objc_object", 
+                                                    getOrCreateMainFile(),
+                                                    0, 0, 0, 0,
+                                                    llvm::DIArray());
 
-    SmallVector<llvm::Value *, 16> EltTys;
+    llvm::TrackingVH<llvm::MDNode> ObjNode(FwdTy);
+    SmallVector<llvm::Value *, 1> EltTys;
     llvm::DIType FieldTy = 
-      DBuilder.createMemberType(getOrCreateMainFile(), "isa",
+      DBuilder.createMemberType(llvm::DIDescriptor(ObjNode), "isa",
                                 getOrCreateMainFile(), 0, Size,
                                 0, 0, 0, ISATy);
     EltTys.push_back(FieldTy);
     llvm::DIArray Elements = DBuilder.getOrCreateArray(EltTys);
-    
-    return DBuilder.createStructType(TheCU, "objc_object", 
-                                     getOrCreateMainFile(),
-                                     0, 0, 0, 0, Elements);
+
+    ObjNode->replaceOperandWith(10, Elements);
+    ObjTy = llvm::DIType(ObjNode);
+    return ObjTy;
   }
   case BuiltinType::ObjCSel: {
-    return
+    if (SelTy.Verify())
+      return SelTy;
+    SelTy =
       DBuilder.createForwardDecl(llvm::dwarf::DW_TAG_structure_type,
                                  "objc_selector", TheCU, getOrCreateMainFile(),
                                  0);
+    return SelTy;
   }
   case BuiltinType::UChar:
   case BuiltinType::Char_U: Encoding = llvm::dwarf::DW_ATE_unsigned_char; break;
@@ -417,7 +438,7 @@ llvm::DIType CGDebugInfo::CreateType(const BuiltinType *BT) {
   case BuiltinType::ULong:     BTName = "long unsigned int"; break;
   case BuiltinType::ULongLong: BTName = "long long unsigned int"; break;
   default:
-    BTName = BT->getName(CGM.getContext().getLangOpts());
+    BTName = BT->getName(CGM.getLangOpts());
     break;
   }
   // Bit size, align and offset of the type.
@@ -498,21 +519,17 @@ llvm::DIType CGDebugInfo::createRecordFwdDecl(const RecordDecl *RD,
                                               llvm::DIDescriptor Ctx) {
   llvm::DIFile DefUnit = getOrCreateFile(RD->getLocation());
   unsigned Line = getLineNumber(RD->getLocation());
-  StringRef RDName = RD->getName();
+  StringRef RDName = getClassName(RD);
 
-  // Get the tag.
-  const CXXRecordDecl *CXXDecl = dyn_cast<CXXRecordDecl>(RD);
   unsigned Tag = 0;
-  if (CXXDecl) {
-    RDName = getClassName(RD);
-    Tag = llvm::dwarf::DW_TAG_class_type;
-  }
-  else if (RD->isStruct())
+  if (RD->isStruct() || RD->isInterface())
     Tag = llvm::dwarf::DW_TAG_structure_type;
   else if (RD->isUnion())
     Tag = llvm::dwarf::DW_TAG_union_type;
-  else
-    llvm_unreachable("Unknown RecordDecl type!");
+  else {
+    assert(RD->isClass());
+    Tag = llvm::dwarf::DW_TAG_class_type;
+  }
 
   // Create the type.
   return DBuilder.createForwardDecl(Tag, RDName, Ctx, DefUnit, Line);
@@ -550,7 +567,7 @@ llvm::DIDescriptor CGDebugInfo::createContextChain(const Decl *Context) {
 /// then emit record's fwd if debug info size reduction is enabled.
 llvm::DIType CGDebugInfo::CreatePointeeType(QualType PointeeTy,
                                             llvm::DIFile Unit) {
-  if (CGM.getCodeGenOpts().DebugInfo != CodeGenOptions::LimitedDebugInfo)
+  if (CGM.getCodeGenOpts().getDebugInfo() != CodeGenOptions::LimitedDebugInfo)
     return getOrCreateType(PointeeTy, Unit);
 
   // Limit debug info for the pointee type.
@@ -777,8 +794,6 @@ CollectRecordFields(const RecordDecl *record, llvm::DIFile tunit,
     for (CXXRecordDecl::capture_const_iterator I = CXXDecl->captures_begin(),
            E = CXXDecl->captures_end(); I != E; ++I, ++Field, ++fieldno) {
       const LambdaExpr::Capture C = *I;
-      // TODO: Need to handle 'this' in some way by probably renaming the
-      // this of the lambda class and having a field member of 'this'.
       if (C.capturesVariable()) {
         VarDecl *V = C.getCapturedVar();
         llvm::DIFile VUnit = getOrCreateFile(C.getLocation());
@@ -793,10 +808,24 @@ CollectRecordFields(const RecordDecl *record, llvm::DIFile tunit,
                             Field->getAccess(), layout.getFieldOffset(fieldno),
                             VUnit, RecordTy);
         elements.push_back(fieldType);
+      } else {
+        // TODO: Need to handle 'this' in some way by probably renaming the
+        // this of the lambda class and having a field member of 'this' or
+        // by using AT_object_pointer for the function and having that be
+        // used as 'this' for semantic references.
+        assert(C.capturesThis() && "Field that isn't captured and isn't this?");
+        FieldDecl *f = *Field;
+        llvm::DIFile VUnit = getOrCreateFile(f->getLocation());
+        QualType type = f->getType();
+        llvm::DIType fieldType
+          = createFieldType("this", type, 0, f->getLocation(), f->getAccess(),
+                            layout.getFieldOffset(fieldNo), VUnit, RecordTy);
+
+        elements.push_back(fieldType);
       }
     }
   } else {
-    bool IsMsStruct = record->hasAttr<MsStructAttr>();
+    bool IsMsStruct = record->isMsStruct(CGM.getContext());
     const FieldDecl *LastFD = 0;
     for (RecordDecl::field_iterator I = record->field_begin(),
            E = record->field_end();
@@ -875,12 +904,12 @@ CGDebugInfo::getOrCreateMethodType(const CXXMethodDecl *Method,
       // TODO: This and the artificial type below are misleading, the
       // types aren't artificial the argument is, but the current
       // metadata doesn't represent that.
-      ThisPtrType = DBuilder.createArtificialType(ThisPtrType);
+      ThisPtrType = DBuilder.createObjectPointerType(ThisPtrType);
       Elts.push_back(ThisPtrType);
     } else {
       llvm::DIType ThisPtrType = getOrCreateType(ThisPtr, Unit);
       TypeCache[ThisPtr.getAsOpaquePtr()] = ThisPtrType;
-      ThisPtrType = DBuilder.createArtificialType(ThisPtrType);
+      ThisPtrType = DBuilder.createObjectPointerType(ThisPtrType);
       Elts.push_back(ThisPtrType);
     }
   }
@@ -995,12 +1024,8 @@ CollectCXXMemberFunctions(const CXXRecordDecl *RD, llvm::DIFile Unit,
     if (D->isImplicit() && !D->isUsed())
       continue;
 
-    if (const CXXMethodDecl *Method = dyn_cast<CXXMethodDecl>(D)) {
-      // Only emit debug information for user provided functions, we're
-      // unlikely to want info for artificial functions.
-      if (Method->isUserProvided())
-        EltTys.push_back(CreateCXXMemberFunction(Method, Unit, RecordTy));
-    }
+    if (const CXXMethodDecl *Method = dyn_cast<CXXMethodDecl>(D))
+      EltTys.push_back(CreateCXXMemberFunction(Method, Unit, RecordTy));
     else if (FunctionTemplateDecl *FTD = dyn_cast<FunctionTemplateDecl>(D))
       for (FunctionTemplateDecl::spec_iterator SI = FTD->spec_begin(),
              SE = FTD->spec_end(); SI != SE; ++SI)
@@ -1182,7 +1207,7 @@ CollectVTableInfo(const CXXRecordDecl *RD, llvm::DIFile Unit,
 /// getOrCreateRecordType - Emit record type's standalone debug info. 
 llvm::DIType CGDebugInfo::getOrCreateRecordType(QualType RTy, 
                                                 SourceLocation Loc) {
-  assert(CGM.getCodeGenOpts().DebugInfo >= CodeGenOptions::LimitedDebugInfo);
+  assert(CGM.getCodeGenOpts().getDebugInfo() >= CodeGenOptions::LimitedDebugInfo);
   llvm::DIType T = getOrCreateType(RTy, getOrCreateFile(Loc));
   return T;
 }
@@ -1191,7 +1216,7 @@ llvm::DIType CGDebugInfo::getOrCreateRecordType(QualType RTy,
 /// debug info.
 llvm::DIType CGDebugInfo::getOrCreateInterfaceType(QualType D,
 						   SourceLocation Loc) {
-  assert(CGM.getCodeGenOpts().DebugInfo >= CodeGenOptions::LimitedDebugInfo);
+  assert(CGM.getCodeGenOpts().getDebugInfo() >= CodeGenOptions::LimitedDebugInfo);
   llvm::DIType T = getOrCreateType(D, getOrCreateFile(Loc));
   DBuilder.retainType(T);
   return T;
@@ -1388,12 +1413,21 @@ llvm::DIType CGDebugInfo::CreateType(const ObjCInterfaceType *Ty,
       FieldAlign = CGM.getContext().getTypeAlign(FType);
     }
 
-    // We can't know the offset of our ivar in the structure if we're using
-    // the non-fragile abi and the debugger should ignore the value anyways.
-    // Call it the FieldNo+1 due to how debuggers use the information,
-    // e.g. negating the value when it needs a lookup in the dynamic table.
-    uint64_t FieldOffset = CGM.getLangOpts().ObjCRuntime.isNonFragile()
-                             ? FieldNo+1 : RL.getFieldOffset(FieldNo);
+    uint64_t FieldOffset;
+    if (CGM.getLangOpts().ObjCRuntime.isNonFragile()) {
+      // We don't know the runtime offset of an ivar if we're using the
+      // non-fragile ABI.  For bitfields, use the bit offset into the first
+      // byte of storage of the bitfield.  For other fields, use zero.
+      if (Field->isBitField()) {
+        FieldOffset = CGM.getObjCRuntime().ComputeBitfieldBitOffset(
+            CGM, ID, Field);
+        FieldOffset %= CGM.getContext().getCharWidth();
+      } else {
+        FieldOffset = 0;
+      }
+    } else {
+      FieldOffset = RL.getFieldOffset(FieldNo);
+    }
 
     unsigned Flags = 0;
     if (Field->getAccessControl() == ObjCIvarDecl::Protected)
@@ -1570,9 +1604,29 @@ llvm::DIType CGDebugInfo::CreateType(const AtomicType *Ty,
 
 /// CreateEnumType - get enumeration type.
 llvm::DIType CGDebugInfo::CreateEnumType(const EnumDecl *ED) {
-  SmallVector<llvm::Value *, 16> Enumerators;
+  uint64_t Size = 0;
+  uint64_t Align = 0;
+  if (!ED->getTypeForDecl()->isIncompleteType()) {
+    Size = CGM.getContext().getTypeSize(ED->getTypeForDecl());
+    Align = CGM.getContext().getTypeAlign(ED->getTypeForDecl());
+  }
+
+  // If this is just a forward declaration, construct an appropriately
+  // marked node and just return it.
+  if (!ED->getDefinition()) {
+    llvm::DIDescriptor EDContext;
+    EDContext = getContextDescriptor(cast<Decl>(ED->getDeclContext()));
+    llvm::DIFile DefUnit = getOrCreateFile(ED->getLocation());
+    unsigned Line = getLineNumber(ED->getLocation());
+    StringRef EDName = ED->getName();
+    return DBuilder.createForwardDecl(llvm::dwarf::DW_TAG_enumeration_type,
+                                      EDName, EDContext, DefUnit, Line, 0,
+                                      Size, Align);
+  }
 
   // Create DIEnumerator elements for each enumerator.
+  SmallVector<llvm::Value *, 16> Enumerators;
+  ED = ED->getDefinition();
   for (EnumDecl::enumerator_iterator
          Enum = ED->enumerator_begin(), EnumEnd = ED->enumerator_end();
        Enum != EnumEnd; ++Enum) {
@@ -1586,21 +1640,14 @@ llvm::DIType CGDebugInfo::CreateEnumType(const EnumDecl *ED) {
 
   llvm::DIFile DefUnit = getOrCreateFile(ED->getLocation());
   unsigned Line = getLineNumber(ED->getLocation());
-  uint64_t Size = 0;
-  uint64_t Align = 0;
-  if (!ED->getTypeForDecl()->isIncompleteType()) {
-    Size = CGM.getContext().getTypeSize(ED->getTypeForDecl());
-    Align = CGM.getContext().getTypeAlign(ED->getTypeForDecl());
-  }
   llvm::DIDescriptor EnumContext = 
     getContextDescriptor(cast<Decl>(ED->getDeclContext()));
   llvm::DIType ClassTy = ED->isScopedUsingClassTag() ?
     getOrCreateType(ED->getIntegerType(), DefUnit) : llvm::DIType();
-  unsigned Flags = !ED->isCompleteDefinition() ? llvm::DIDescriptor::FlagFwdDecl : 0;
   llvm::DIType DbgTy = 
     DBuilder.createEnumerationType(EnumContext, ED->getName(), DefUnit, Line,
                                    Size, Align, EltArray,
-                                   ClassTy, Flags);
+                                   ClassTy);
   return DbgTy;
 }
 
@@ -1838,10 +1885,10 @@ llvm::DIType CGDebugInfo::CreateLimitedType(const RecordType *Ty) {
   // Get overall information about the record type for the debug info.
   llvm::DIFile DefUnit = getOrCreateFile(RD->getLocation());
   unsigned Line = getLineNumber(RD->getLocation());
-  StringRef RDName = RD->getName();
+  StringRef RDName = getClassName(RD);
 
   llvm::DIDescriptor RDContext;
-  if (CGM.getCodeGenOpts().DebugInfo == CodeGenOptions::LimitedDebugInfo)
+  if (CGM.getCodeGenOpts().getDebugInfo() == CodeGenOptions::LimitedDebugInfo)
     RDContext = createContextChain(cast<Decl>(RD->getDeclContext()));
   else
     RDContext = getContextDescriptor(cast<Decl>(RD->getDeclContext()));
@@ -1859,9 +1906,7 @@ llvm::DIType CGDebugInfo::CreateLimitedType(const RecordType *Ty) {
   if (RD->isUnion())
     RealDecl = DBuilder.createUnionType(RDContext, RDName, DefUnit, Line,
 					Size, Align, 0, llvm::DIArray());
-  else if (CXXDecl) {
-    RDName = getClassName(RD);
-    
+  else if (RD->isClass()) {
     // FIXME: This could be a struct type giving a default visibility different
     // than C++ class type, but needs llvm metadata changes first.
     RealDecl = DBuilder.createClassType(RDContext, RDName, DefUnit, Line,
@@ -1969,7 +2014,7 @@ llvm::DISubprogram CGDebugInfo::getFunctionDeclaration(const Decl *D) {
 
 // getOrCreateFunctionType - Construct DIType. If it is a c++ method, include
 // implicit parameter "this".
-llvm::DIType CGDebugInfo::getOrCreateFunctionType(const Decl * D,
+llvm::DIType CGDebugInfo::getOrCreateFunctionType(const Decl *D,
                                                   QualType FnType,
                                                   llvm::DIFile F) {
 
@@ -1982,9 +2027,11 @@ llvm::DIType CGDebugInfo::getOrCreateFunctionType(const Decl * D,
     // First element is always return type. For 'void' functions it is NULL.
     Elts.push_back(getOrCreateType(OMethod->getResultType(), F));
     // "self" pointer is always first argument.
-    Elts.push_back(getOrCreateType(OMethod->getSelfDecl()->getType(), F));
-    // "cmd" pointer is always second argument.
-    Elts.push_back(getOrCreateType(OMethod->getCmdDecl()->getType(), F));
+    llvm::DIType SelfTy = getOrCreateType(OMethod->getSelfDecl()->getType(), F);
+    Elts.push_back(DBuilder.createObjectPointerType(SelfTy));
+    // "_cmd" pointer is always second argument.
+    llvm::DIType CmdTy = getOrCreateType(OMethod->getCmdDecl()->getType(), F);
+    Elts.push_back(DBuilder.createArtificialType(CmdTy));
     // Get rest of the arguments.
     for (ObjCMethodDecl::param_const_iterator PI = OMethod->param_begin(), 
            PE = OMethod->param_end(); PI != PE; ++PI)
@@ -2007,14 +2054,22 @@ void CGDebugInfo::EmitFunctionStart(GlobalDecl GD, QualType FnType,
   FnBeginRegionCount.push_back(LexicalBlockStack.size());
 
   const Decl *D = GD.getDecl();
+  // Function may lack declaration in source code if it is created by Clang
+  // CodeGen (examples: _GLOBAL__I_a, __cxx_global_array_dtor, thunk).
+  bool HasDecl = (D != 0);
   // Use the location of the declaration.
-  SourceLocation Loc = D->getLocation();
-  
+  SourceLocation Loc;
+  if (HasDecl)
+    Loc = D->getLocation();
+
   unsigned Flags = 0;
   llvm::DIFile Unit = getOrCreateFile(Loc);
   llvm::DIDescriptor FDContext(Unit);
   llvm::DIArray TParamsArray;
-  if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
+  if (!HasDecl) {
+    // Use llvm function name.
+    Name = Fn->getName();
+  } else if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
     // If there is a DISubprogram for this function available then use it.
     llvm::DenseMap<const FunctionDecl *, llvm::WeakVH>::iterator
       FI = SPCache.find(FD->getCanonicalDecl());
@@ -2035,10 +2090,10 @@ void CGDebugInfo::EmitFunctionStart(GlobalDecl GD, QualType FnType,
       Flags |= llvm::DIDescriptor::FlagPrototyped;
     }
     if (LinkageName == Name ||
-        CGM.getCodeGenOpts().DebugInfo <= CodeGenOptions::DebugLineTablesOnly)
+        CGM.getCodeGenOpts().getDebugInfo() <= CodeGenOptions::DebugLineTablesOnly)
       LinkageName = StringRef();
 
-    if (CGM.getCodeGenOpts().DebugInfo >= CodeGenOptions::LimitedDebugInfo) {
+    if (CGM.getCodeGenOpts().getDebugInfo() >= CodeGenOptions::LimitedDebugInfo) {
       if (const NamespaceDecl *NSDecl =
           dyn_cast_or_null<NamespaceDecl>(FD->getDeclContext()))
         FDContext = getOrCreateNameSpace(NSDecl);
@@ -2061,12 +2116,13 @@ void CGDebugInfo::EmitFunctionStart(GlobalDecl GD, QualType FnType,
     Name = Name.substr(1);
 
   unsigned LineNo = getLineNumber(Loc);
-  if (D->isImplicit())
+  if (!HasDecl || D->isImplicit())
     Flags |= llvm::DIDescriptor::FlagArtificial;
 
   llvm::DIType DIFnType;
   llvm::DISubprogram SPDecl;
-  if (CGM.getCodeGenOpts().DebugInfo >= CodeGenOptions::LimitedDebugInfo) {
+  if (HasDecl &&
+      CGM.getCodeGenOpts().getDebugInfo() >= CodeGenOptions::LimitedDebugInfo) {
     DIFnType = getOrCreateFunctionType(D, FnType, Unit);
     SPDecl = getFunctionDeclaration(D);
   } else {
@@ -2089,7 +2145,8 @@ void CGDebugInfo::EmitFunctionStart(GlobalDecl GD, QualType FnType,
   // Push function on region stack.
   llvm::MDNode *SPN = SP;
   LexicalBlockStack.push_back(SPN);
-  RegionMap[D] = llvm::WeakVH(SP);
+  if (HasDecl)
+    RegionMap[D] = llvm::WeakVH(SP);
 }
 
 /// EmitLocation - Emit metadata to indicate a change in line/column
@@ -2242,7 +2299,7 @@ llvm::DIType CGDebugInfo::EmitTypeForVarWithBlocksAttr(const ValueDecl *VD,
 void CGDebugInfo::EmitDeclare(const VarDecl *VD, unsigned Tag,
                               llvm::Value *Storage, 
                               unsigned ArgNo, CGBuilderTy &Builder) {
-  assert(CGM.getCodeGenOpts().DebugInfo >= CodeGenOptions::LimitedDebugInfo);
+  assert(CGM.getCodeGenOpts().getDebugInfo() >= CodeGenOptions::LimitedDebugInfo);
   assert(!LexicalBlockStack.empty() && "Region stack mismatch, stack empty!");
 
   llvm::DIFile Unit = getOrCreateFile(VD->getLocation());
@@ -2253,7 +2310,7 @@ void CGDebugInfo::EmitDeclare(const VarDecl *VD, unsigned Tag,
   else 
     Ty = getOrCreateType(VD->getType(), Unit);
 
-  // If there is not any debug info for type then do not emit debug info
+  // If there is no debug info for this type then do not emit debug info
   // for this variable.
   if (!Ty)
     return;
@@ -2279,8 +2336,16 @@ void CGDebugInfo::EmitDeclare(const VarDecl *VD, unsigned Tag,
   unsigned Flags = 0;
   if (VD->isImplicit())
     Flags |= llvm::DIDescriptor::FlagArtificial;
+  // If this is the first argument and it is implicit then
+  // give it an object pointer flag.
+  // FIXME: There has to be a better way to do this, but for static
+  // functions there won't be an implicit param at arg1 and
+  // otherwise it is 'self' or 'this'.
+  if (isa<ImplicitParamDecl>(VD) && ArgNo == 1)
+    Flags |= llvm::DIDescriptor::FlagObjectPointer;
+
   llvm::MDNode *Scope = LexicalBlockStack.back();
-    
+
   StringRef Name = VD->getName();
   if (!Name.empty()) {
     if (VD->hasAttr<BlocksAttr>()) {
@@ -2376,14 +2441,15 @@ void CGDebugInfo::EmitDeclare(const VarDecl *VD, unsigned Tag,
 void CGDebugInfo::EmitDeclareOfAutoVariable(const VarDecl *VD,
                                             llvm::Value *Storage,
                                             CGBuilderTy &Builder) {
-  assert(CGM.getCodeGenOpts().DebugInfo >= CodeGenOptions::LimitedDebugInfo);
+  assert(CGM.getCodeGenOpts().getDebugInfo() >= CodeGenOptions::LimitedDebugInfo);
   EmitDeclare(VD, llvm::dwarf::DW_TAG_auto_variable, Storage, 0, Builder);
 }
 
-void CGDebugInfo::EmitDeclareOfBlockDeclRefVariable(
-  const VarDecl *VD, llvm::Value *Storage, CGBuilderTy &Builder,
-  const CGBlockInfo &blockInfo) {
-  assert(CGM.getCodeGenOpts().DebugInfo >= CodeGenOptions::LimitedDebugInfo);
+void CGDebugInfo::EmitDeclareOfBlockDeclRefVariable(const VarDecl *VD,
+                                                    llvm::Value *Storage,
+                                                    CGBuilderTy &Builder,
+                                                 const CGBlockInfo &blockInfo) {
+  assert(CGM.getCodeGenOpts().getDebugInfo() >= CodeGenOptions::LimitedDebugInfo);
   assert(!LexicalBlockStack.empty() && "Region stack mismatch, stack empty!");
   
   if (Builder.GetInsertBlock() == 0)
@@ -2399,11 +2465,16 @@ void CGDebugInfo::EmitDeclareOfBlockDeclRefVariable(
   else 
     Ty = getOrCreateType(VD->getType(), Unit);
 
+  // Self is passed along as an implicit non-arg variable in a
+  // block. Mark it as the object pointer.
+  if (isa<ImplicitParamDecl>(VD) && VD->getName() == "self")
+    Ty = DBuilder.createObjectPointerType(Ty);
+
   // Get location information.
   unsigned Line = getLineNumber(VD->getLocation());
   unsigned Column = getColumnNumber(VD->getLocation());
 
-  const llvm::TargetData &target = CGM.getTargetData();
+  const llvm::DataLayout &target = CGM.getDataLayout();
 
   CharUnits offset = CharUnits::fromQuantity(
     target.getStructLayout(blockInfo.StructureType)
@@ -2418,7 +2489,7 @@ void CGDebugInfo::EmitDeclareOfBlockDeclRefVariable(
     addr.push_back(llvm::ConstantInt::get(Int64Ty, llvm::DIBuilder::OpPlus));
     // offset of __forwarding field
     offset = CGM.getContext()
-                .toCharUnitsFromBits(target.getPointerSizeInBits());
+                .toCharUnitsFromBits(target.getPointerSizeInBits(0));
     addr.push_back(llvm::ConstantInt::get(Int64Ty, offset.getQuantity()));
     addr.push_back(llvm::ConstantInt::get(Int64Ty, llvm::DIBuilder::OpDeref));
     addr.push_back(llvm::ConstantInt::get(Int64Ty, llvm::DIBuilder::OpPlus));
@@ -2444,7 +2515,7 @@ void CGDebugInfo::EmitDeclareOfBlockDeclRefVariable(
 void CGDebugInfo::EmitDeclareOfArgVariable(const VarDecl *VD, llvm::Value *AI,
                                            unsigned ArgNo,
                                            CGBuilderTy &Builder) {
-  assert(CGM.getCodeGenOpts().DebugInfo >= CodeGenOptions::LimitedDebugInfo);
+  assert(CGM.getCodeGenOpts().getDebugInfo() >= CodeGenOptions::LimitedDebugInfo);
   EmitDeclare(VD, llvm::dwarf::DW_TAG_arg_variable, AI, ArgNo, Builder);
 }
 
@@ -2461,7 +2532,7 @@ namespace {
 void CGDebugInfo::EmitDeclareOfBlockLiteralArgVariable(const CGBlockInfo &block,
                                                        llvm::Value *addr,
                                                        CGBuilderTy &Builder) {
-  assert(CGM.getCodeGenOpts().DebugInfo >= CodeGenOptions::LimitedDebugInfo);
+  assert(CGM.getCodeGenOpts().getDebugInfo() >= CodeGenOptions::LimitedDebugInfo);
   ASTContext &C = CGM.getContext();
   const BlockDecl *blockDecl = block.getBlockDecl();
 
@@ -2475,7 +2546,7 @@ void CGDebugInfo::EmitDeclareOfBlockLiteralArgVariable(const CGBlockInfo &block,
   getContextDescriptor(cast<Decl>(blockDecl->getDeclContext()));
 
   const llvm::StructLayout *blockLayout =
-    CGM.getTargetData().getStructLayout(block.StructureType);
+    CGM.getDataLayout().getStructLayout(block.StructureType);
 
   SmallVector<llvm::Value*, 16> fields;
   fields.push_back(createFieldType("__isa", C.VoidPtrTy, 0, loc, AS_public,
@@ -2606,7 +2677,7 @@ void CGDebugInfo::EmitDeclareOfBlockLiteralArgVariable(const CGBlockInfo &block,
 /// EmitGlobalVariable - Emit information about a global variable.
 void CGDebugInfo::EmitGlobalVariable(llvm::GlobalVariable *Var,
                                      const VarDecl *D) {
-  assert(CGM.getCodeGenOpts().DebugInfo >= CodeGenOptions::LimitedDebugInfo);
+  assert(CGM.getCodeGenOpts().getDebugInfo() >= CodeGenOptions::LimitedDebugInfo);
   // Create global variable debug descriptor.
   llvm::DIFile Unit = getOrCreateFile(D->getLocation());
   unsigned LineNo = getLineNumber(D->getLocation());
@@ -2640,7 +2711,7 @@ void CGDebugInfo::EmitGlobalVariable(llvm::GlobalVariable *Var,
 /// EmitGlobalVariable - Emit information about an objective-c interface.
 void CGDebugInfo::EmitGlobalVariable(llvm::GlobalVariable *Var,
                                      ObjCInterfaceDecl *ID) {
-  assert(CGM.getCodeGenOpts().DebugInfo >= CodeGenOptions::LimitedDebugInfo);
+  assert(CGM.getCodeGenOpts().getDebugInfo() >= CodeGenOptions::LimitedDebugInfo);
   // Create global variable debug descriptor.
   llvm::DIFile Unit = getOrCreateFile(ID->getLocation());
   unsigned LineNo = getLineNumber(ID->getLocation());
@@ -2666,7 +2737,7 @@ void CGDebugInfo::EmitGlobalVariable(llvm::GlobalVariable *Var,
 /// EmitGlobalVariable - Emit global variable's debug info.
 void CGDebugInfo::EmitGlobalVariable(const ValueDecl *VD, 
                                      llvm::Constant *Init) {
-  assert(CGM.getCodeGenOpts().DebugInfo >= CodeGenOptions::LimitedDebugInfo);
+  assert(CGM.getCodeGenOpts().getDebugInfo() >= CodeGenOptions::LimitedDebugInfo);
   // Create the descriptor for the variable.
   llvm::DIFile Unit = getOrCreateFile(VD->getLocation());
   StringRef Name = VD->getName();
diff --git a/lib/CodeGen/CGDebugInfo.h b/lib/CodeGen/CGDebugInfo.h
index 44cc49a..2e88a73 100644
--- a/lib/CodeGen/CGDebugInfo.h
+++ b/lib/CodeGen/CGDebugInfo.h
@@ -50,6 +50,9 @@ class CGDebugInfo {
   llvm::DICompileUnit TheCU;
   SourceLocation CurLoc, PrevLoc;
   llvm::DIType VTablePtrType;
+  llvm::DIType ClassTy;
+  llvm::DIType ObjTy;
+  llvm::DIType SelTy;
   
   /// TypeCache - Cache of previously constructed Types.
   llvm::DenseMap<void *, llvm::WeakVH> TypeCache;
diff --git a/lib/CodeGen/CGDecl.cpp b/lib/CodeGen/CGDecl.cpp
index be6638e..8870587 100644
--- a/lib/CodeGen/CGDecl.cpp
+++ b/lib/CodeGen/CGDecl.cpp
@@ -24,7 +24,7 @@
 #include "clang/Frontend/CodeGenOptions.h"
 #include "llvm/GlobalVariable.h"
 #include "llvm/Intrinsics.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Type.h"
 using namespace clang;
 using namespace CodeGen;
@@ -121,7 +121,7 @@ void CodeGenFunction::EmitVarDecl(const VarDecl &D) {
     // uniqued.  We can't do this in C, though, because there's no
     // standard way to agree on which variables are the same (i.e.
     // there's no mangling).
-    if (getContext().getLangOpts().CPlusPlus)
+    if (getLangOpts().CPlusPlus)
       if (llvm::GlobalValue::isWeakForLinker(CurFn->getLinkage()))
         Linkage = CurFn->getLinkage();
 
@@ -141,7 +141,7 @@ void CodeGenFunction::EmitVarDecl(const VarDecl &D) {
 static std::string GetStaticDeclName(CodeGenFunction &CGF, const VarDecl &D,
                                      const char *Separator) {
   CodeGenModule &CGM = CGF.CGM;
-  if (CGF.getContext().getLangOpts().CPlusPlus) {
+  if (CGF.getLangOpts().CPlusPlus) {
     StringRef Name = CGM.getMangledName(&D);
     return Name.str();
   }
@@ -184,12 +184,14 @@ CodeGenFunction::CreateStaticVarDecl(const VarDecl &D,
     Name = GetStaticDeclName(*this, D, Separator);
 
   llvm::Type *LTy = CGM.getTypes().ConvertTypeForMem(Ty);
+  unsigned AddrSpace =
+   CGM.GetGlobalVarAddressSpace(&D, CGM.getContext().getTargetAddressSpace(Ty));
   llvm::GlobalVariable *GV =
     new llvm::GlobalVariable(CGM.getModule(), LTy,
                              Ty.isConstant(getContext()), Linkage,
                              CGM.EmitNullConstant(D.getType()), Name, 0,
                              llvm::GlobalVariable::NotThreadLocal,
-                             CGM.getContext().getTargetAddressSpace(Ty));
+                             AddrSpace);
   GV->setAlignment(getContext().getDeclAlign(&D).getQuantity());
   if (Linkage != llvm::GlobalValue::InternalLinkage)
     GV->setVisibility(CurFn->getVisibility());
@@ -220,7 +222,7 @@ CodeGenFunction::AddInitializerToStaticVarDecl(const VarDecl &D,
   // If constant emission failed, then this should be a C++ static
   // initializer.
   if (!Init) {
-    if (!getContext().getLangOpts().CPlusPlus)
+    if (!getLangOpts().CPlusPlus)
       CGM.ErrorUnsupported(D.getInit(), "constant l-value expression");
     else if (Builder.GetInsertBlock()) {
       // Since we have a static initializer, this global variable can't
@@ -331,7 +333,7 @@ void CodeGenFunction::EmitStaticVarDecl(const VarDecl &D,
   // Emit global variable debug descriptor for static vars.
   CGDebugInfo *DI = getDebugInfo();
   if (DI &&
-      CGM.getCodeGenOpts().DebugInfo >= CodeGenOptions::LimitedDebugInfo) {
+      CGM.getCodeGenOpts().getDebugInfo() >= CodeGenOptions::LimitedDebugInfo) {
     DI->setLocation(D.getLocation());
     DI->EmitGlobalVariable(var, &D);
   }
@@ -704,9 +706,8 @@ static bool canEmitInitWithFewStoresAfterMemset(llvm::Constant *Init,
 /// stores that would be required.
 static void emitStoresForInitAfterMemset(llvm::Constant *Init, llvm::Value *Loc,
                                          bool isVolatile, CGBuilderTy &Builder) {
-  // Zero doesn't require a store.
-  if (Init->isNullValue() || isa<llvm::UndefValue>(Init))
-    return;
+  assert(!Init->isNullValue() && !isa<llvm::UndefValue>(Init) &&
+         "called emitStoresForInitAfterMemset for zero or undef value.");
 
   if (isa<llvm::ConstantInt>(Init) || isa<llvm::ConstantFP>(Init) ||
       isa<llvm::ConstantVector>(Init) || isa<llvm::BlockAddress>(Init) ||
@@ -719,10 +720,11 @@ static void emitStoresForInitAfterMemset(llvm::Constant *Init, llvm::Value *Loc,
         dyn_cast<llvm::ConstantDataSequential>(Init)) {
     for (unsigned i = 0, e = CDS->getNumElements(); i != e; ++i) {
       llvm::Constant *Elt = CDS->getElementAsConstant(i);
-      
-      // Get a pointer to the element and emit it.
-      emitStoresForInitAfterMemset(Elt, Builder.CreateConstGEP2_32(Loc, 0, i),
-                                   isVolatile, Builder);
+
+      // If necessary, get a pointer to the element and emit it.
+      if (!Elt->isNullValue() && !isa<llvm::UndefValue>(Elt))
+        emitStoresForInitAfterMemset(Elt, Builder.CreateConstGEP2_32(Loc, 0, i),
+                                     isVolatile, Builder);
     }
     return;
   }
@@ -732,9 +734,11 @@ static void emitStoresForInitAfterMemset(llvm::Constant *Init, llvm::Value *Loc,
 
   for (unsigned i = 0, e = Init->getNumOperands(); i != e; ++i) {
     llvm::Constant *Elt = cast<llvm::Constant>(Init->getOperand(i));
-    // Get a pointer to the element and emit it.
-    emitStoresForInitAfterMemset(Elt, Builder.CreateConstGEP2_32(Loc, 0, i),
-                                 isVolatile, Builder);
+
+    // If necessary, get a pointer to the element and emit it.
+    if (!Elt->isNullValue() && !isa<llvm::UndefValue>(Elt))
+      emitStoresForInitAfterMemset(Elt, Builder.CreateConstGEP2_32(Loc, 0, i),
+                                   isVolatile, Builder);
   }
 }
 
@@ -791,7 +795,7 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) {
   llvm::Value *DeclPtr;
   if (Ty->isConstantSizeType()) {
     if (!Target.useGlobalsForAutomaticVariables()) {
-      bool NRVO = getContext().getLangOpts().ElideConstructors &&
+      bool NRVO = getLangOpts().ElideConstructors &&
                   D.isNRVOVariable();
 
       // If this value is a POD array or struct with a statically
@@ -910,7 +914,8 @@ CodeGenFunction::EmitAutoVarAlloca(const VarDecl &D) {
   // Emit debug info for local var declaration.
   if (HaveInsertPoint())
     if (CGDebugInfo *DI = getDebugInfo()) {
-      if (CGM.getCodeGenOpts().DebugInfo >= CodeGenOptions::LimitedDebugInfo) {
+      if (CGM.getCodeGenOpts().getDebugInfo()
+            >= CodeGenOptions::LimitedDebugInfo) {
         DI->setLocation(D.getLocation());
         if (Target.useGlobalsForAutomaticVariables()) {
           DI->EmitGlobalVariable(static_cast<llvm::GlobalVariable *>(DeclPtr),
@@ -1056,10 +1061,11 @@ void CodeGenFunction::EmitAutoVarInit(const AutoVarEmission &emission) {
   // If the initializer is all or mostly zeros, codegen with memset then do
   // a few stores afterward.
   if (shouldUseMemSetPlusStoresToInitialize(constant,
-                CGM.getTargetData().getTypeAllocSize(constant->getType()))) {
+                CGM.getDataLayout().getTypeAllocSize(constant->getType()))) {
     Builder.CreateMemSet(Loc, llvm::ConstantInt::get(Int8Ty, 0), SizeVal,
                          alignment.getQuantity(), isVolatile);
-    if (!constant->isNullValue()) {
+    // Zero and undef don't require a stores.
+    if (!constant->isNullValue() && !isa<llvm::UndefValue>(constant)) {
       Loc = Builder.CreateBitCast(Loc, constant->getType()->getPointerTo());
       emitStoresForInitAfterMemset(constant, Loc, isVolatile, Builder);
     }
@@ -1493,8 +1499,8 @@ void CodeGenFunction::EmitParmDecl(const VarDecl &D, llvm::Value *Arg,
       LocalDeclMap[&D] = Arg;
 
       if (CGDebugInfo *DI = getDebugInfo()) {
-        if (CGM.getCodeGenOpts().DebugInfo >=
-            CodeGenOptions::LimitedDebugInfo) {
+        if (CGM.getCodeGenOpts().getDebugInfo()
+              >= CodeGenOptions::LimitedDebugInfo) {
           DI->setLocation(D.getLocation());
           DI->EmitDeclareOfBlockLiteralArgVariable(*BlockInfo, Arg, Builder);
         }
@@ -1576,7 +1582,8 @@ void CodeGenFunction::EmitParmDecl(const VarDecl &D, llvm::Value *Arg,
 
   // Emit debug info for param declaration.
   if (CGDebugInfo *DI = getDebugInfo()) {
-    if (CGM.getCodeGenOpts().DebugInfo >= CodeGenOptions::LimitedDebugInfo) {
+    if (CGM.getCodeGenOpts().getDebugInfo()
+          >= CodeGenOptions::LimitedDebugInfo) {
       DI->EmitDeclareOfArgVariable(&D, DeclPtr, ArgNo, Builder);
     }
   }
diff --git a/lib/CodeGen/CGDeclCXX.cpp b/lib/CodeGen/CGDeclCXX.cpp
index 492b95a..65be3c1 100644
--- a/lib/CodeGen/CGDeclCXX.cpp
+++ b/lib/CodeGen/CGDeclCXX.cpp
@@ -16,6 +16,7 @@
 #include "CGCXXABI.h"
 #include "clang/Frontend/CodeGenOptions.h"
 #include "llvm/Intrinsics.h"
+#include "llvm/ADT/StringExtras.h"
 
 using namespace clang;
 using namespace CodeGen;
@@ -163,6 +164,9 @@ static llvm::Constant *createAtExitStub(CodeGenModule &CGM,
 
   CodeGenFunction CGF(CGM);
 
+  // Initialize debug info if needed.
+  CGF.maybeInitializeDebugInfo();
+
   CGF.StartFunction(GlobalDecl(), CGM.getContext().VoidTy, fn,
                     CGM.getTypes().arrangeNullaryFunction(),
                     FunctionArgList(), SourceLocation());
@@ -218,7 +222,7 @@ CreateGlobalInitOrDestructFunction(CodeGenModule &CGM,
   llvm::Function *Fn =
     llvm::Function::Create(FTy, llvm::GlobalValue::InternalLinkage,
                            Name, &CGM.getModule());
-  if (!CGM.getContext().getLangOpts().AppleKext) {
+  if (!CGM.getLangOpts().AppleKext) {
     // Set the section if needed.
     if (const char *Section = 
           CGM.getContext().getTargetInfo().getStaticInitSectionSpecifier())
@@ -228,8 +232,8 @@ CreateGlobalInitOrDestructFunction(CodeGenModule &CGM,
   if (!CGM.getLangOpts().Exceptions)
     Fn->setDoesNotThrow();
 
-  if (CGM.getLangOpts().AddressSanitizer)
-    Fn->addFnAttr(llvm::Attribute::AddressSafety);
+  if (CGM.getLangOpts().SanitizeAddress)
+    Fn->addFnAttr(llvm::Attributes::AddressSafety);
 
   return Fn;
 }
@@ -252,8 +256,7 @@ CodeGenModule::EmitCXXGlobalVarDeclInitFunc(const VarDecl *D,
     OrderGlobalInits Key(order, PrioritizedCXXGlobalInits.size());
     PrioritizedCXXGlobalInits.push_back(std::make_pair(Key, Fn));
     DelayedCXXInitPosition.erase(D);
-  }
-  else {
+  }  else {
     llvm::DenseMap<const Decl *, unsigned>::iterator I =
       DelayedCXXInitPosition.find(D);
     if (I == DelayedCXXInitPosition.end()) {
@@ -276,28 +279,50 @@ CodeGenModule::EmitCXXGlobalInitFunc() {
 
   llvm::FunctionType *FTy = llvm::FunctionType::get(VoidTy, false);
 
-  // Create our global initialization function.
-  llvm::Function *Fn = 
-    CreateGlobalInitOrDestructFunction(*this, FTy, "_GLOBAL__I_a");
 
+  // Create our global initialization function.
   if (!PrioritizedCXXGlobalInits.empty()) {
     SmallVector<llvm::Constant*, 8> LocalCXXGlobalInits;
     llvm::array_pod_sort(PrioritizedCXXGlobalInits.begin(), 
-                         PrioritizedCXXGlobalInits.end()); 
-    for (unsigned i = 0; i < PrioritizedCXXGlobalInits.size(); i++) {
-      llvm::Function *Fn = PrioritizedCXXGlobalInits[i].second;
-      LocalCXXGlobalInits.push_back(Fn);
-    }
-    LocalCXXGlobalInits.append(CXXGlobalInits.begin(), CXXGlobalInits.end());
-    CodeGenFunction(*this).GenerateCXXGlobalInitFunc(Fn,
+                         PrioritizedCXXGlobalInits.end());
+    // Iterate over "chunks" of ctors with same priority and emit each chunk
+    // into separate function. Note - everything is sorted first by priority,
+    // second - by lex order, so we emit ctor functions in proper order.
+    for (SmallVectorImpl<GlobalInitData >::iterator
+           I = PrioritizedCXXGlobalInits.begin(),
+           E = PrioritizedCXXGlobalInits.end(); I != E; ) {
+      SmallVectorImpl<GlobalInitData >::iterator
+        PrioE = std::upper_bound(I + 1, E, *I, GlobalInitPriorityCmp());
+
+      LocalCXXGlobalInits.clear();
+      unsigned Priority = I->first.priority;
+      // Compute the function suffix from priority. Prepend with zeroes to make
+      // sure the function names are also ordered as priorities.
+      std::string PrioritySuffix = llvm::utostr(Priority);
+      // Priority is always <= 65535 (enforced by sema)..
+      PrioritySuffix = std::string(6-PrioritySuffix.size(), '0')+PrioritySuffix;
+      llvm::Function *Fn = 
+        CreateGlobalInitOrDestructFunction(*this, FTy,
+                                           "_GLOBAL__I_" + PrioritySuffix);
+      
+      for (; I < PrioE; ++I)
+        LocalCXXGlobalInits.push_back(I->second);
+
+      CodeGenFunction(*this).GenerateCXXGlobalInitFunc(Fn,
                                                     &LocalCXXGlobalInits[0],
                                                     LocalCXXGlobalInits.size());
+      AddGlobalCtor(Fn, Priority);
+    }
   }
-  else
-    CodeGenFunction(*this).GenerateCXXGlobalInitFunc(Fn,
-                                                     &CXXGlobalInits[0],
-                                                     CXXGlobalInits.size());
+  
+  llvm::Function *Fn = 
+    CreateGlobalInitOrDestructFunction(*this, FTy, "_GLOBAL__I_a");
+
+  CodeGenFunction(*this).GenerateCXXGlobalInitFunc(Fn,
+                                                   &CXXGlobalInits[0],
+                                                   CXXGlobalInits.size());
   AddGlobalCtor(Fn);
+
   CXXGlobalInits.clear();
   PrioritizedCXXGlobalInits.clear();
 }
@@ -321,8 +346,9 @@ void CodeGenFunction::GenerateCXXGlobalVarDeclInitFunc(llvm::Function *Fn,
                                                        const VarDecl *D,
                                                  llvm::GlobalVariable *Addr,
                                                        bool PerformInit) {
-  if (CGM.getModuleDebugInfo() && !D->hasAttr<NoDebugAttr>())
-    DebugInfo = CGM.getModuleDebugInfo();
+  // Check if we need to emit debug info for variable initializer.
+  if (!D->hasAttr<NoDebugAttr>())
+    maybeInitializeDebugInfo();
 
   StartFunction(GlobalDecl(D), getContext().VoidTy, Fn,
                 getTypes().arrangeNullaryFunction(),
@@ -344,6 +370,9 @@ void CodeGenFunction::GenerateCXXGlobalVarDeclInitFunc(llvm::Function *Fn,
 void CodeGenFunction::GenerateCXXGlobalInitFunc(llvm::Function *Fn,
                                                 llvm::Constant **Decls,
                                                 unsigned NumDecls) {
+  // Initialize debug info if needed.
+  maybeInitializeDebugInfo();
+
   StartFunction(GlobalDecl(), getContext().VoidTy, Fn,
                 getTypes().arrangeNullaryFunction(),
                 FunctionArgList(), SourceLocation());
@@ -369,6 +398,9 @@ void CodeGenFunction::GenerateCXXGlobalInitFunc(llvm::Function *Fn,
 void CodeGenFunction::GenerateCXXGlobalDtorsFunc(llvm::Function *Fn,
                   const std::vector<std::pair<llvm::WeakVH, llvm::Constant*> >
                                                 &DtorsAndObjects) {
+  // Initialize debug info if needed.
+  maybeInitializeDebugInfo();
+
   StartFunction(GlobalDecl(), getContext().VoidTy, Fn,
                 getTypes().arrangeNullaryFunction(),
                 FunctionArgList(), SourceLocation());
@@ -405,6 +437,9 @@ CodeGenFunction::generateDestroyHelper(llvm::Constant *addr,
   llvm::Function *fn = 
     CreateGlobalInitOrDestructFunction(CGM, FTy, "__cxx_global_array_dtor");
 
+  // Initialize debug info if needed.
+  maybeInitializeDebugInfo();
+
   StartFunction(GlobalDecl(), getContext().VoidTy, fn, FI, args,
                 SourceLocation());
 
diff --git a/lib/CodeGen/CGException.cpp b/lib/CodeGen/CGException.cpp
index ba9c296..86dee5a 100644
--- a/lib/CodeGen/CGException.cpp
+++ b/lib/CodeGen/CGException.cpp
@@ -307,14 +307,15 @@ static bool PersonalityHasOnlyCXXUses(llvm::Constant *Fn) {
 /// aggressive about only using the ObjC++ personality in a function
 /// when it really needs it.
 void CodeGenModule::SimplifyPersonality() {
-  // For now, this is really a Darwin-specific operation.
-  if (!Context.getTargetInfo().getTriple().isOSDarwin())
-    return;
-
   // If we're not in ObjC++ -fexceptions, there's nothing to do.
   if (!LangOpts.CPlusPlus || !LangOpts.ObjC1 || !LangOpts.Exceptions)
     return;
 
+  // Both the problem this endeavors to fix and the way the logic
+  // above works is specific to the NeXT runtime.
+  if (!LangOpts.ObjCRuntime.isNeXTFamily())
+    return;
+
   const EHPersonality &ObjCXX = EHPersonality::get(LangOpts);
   const EHPersonality &CXX = getCXXPersonality(LangOpts);
   if (&ObjCXX == &CXX)
@@ -534,7 +535,7 @@ static void emitFilterDispatchBlock(CodeGenFunction &CGF,
     llvm::Value *zero = CGF.Builder.getInt32(0);
     llvm::Value *failsFilter =
       CGF.Builder.CreateICmpSLT(selector, zero, "ehspec.fails");
-    CGF.Builder.CreateCondBr(failsFilter, unexpectedBB, CGF.getEHResumeBlock());
+    CGF.Builder.CreateCondBr(failsFilter, unexpectedBB, CGF.getEHResumeBlock(false));
 
     CGF.EmitBlock(unexpectedBB);
   }
@@ -614,7 +615,7 @@ CodeGenFunction::getEHDispatchBlock(EHScopeStack::stable_iterator si) {
   // The dispatch block for the end of the scope chain is a block that
   // just resumes unwinding.
   if (si == EHStack.stable_end())
-    return getEHResumeBlock();
+    return getEHResumeBlock(true);
 
   // Otherwise, we should look at the actual scope.
   EHScope &scope = *EHStack.find(si);
@@ -1546,7 +1547,7 @@ llvm::BasicBlock *CodeGenFunction::getTerminateHandler() {
   return TerminateHandler;
 }
 
-llvm::BasicBlock *CodeGenFunction::getEHResumeBlock() {
+llvm::BasicBlock *CodeGenFunction::getEHResumeBlock(bool isCleanup) {
   if (EHResumeBlock) return EHResumeBlock;
 
   CGBuilderTy::InsertPoint SavedIP = Builder.saveIP();
@@ -1560,7 +1561,7 @@ llvm::BasicBlock *CodeGenFunction::getEHResumeBlock() {
   // This can always be a call because we necessarily didn't find
   // anything on the EH stack which needs our help.
   const char *RethrowName = Personality.CatchallRethrowFn;
-  if (RethrowName != 0) {
+  if (RethrowName != 0 && !isCleanup) {
     Builder.CreateCall(getCatchallRethrowFn(*this, RethrowName),
                        getExceptionFromSlot())
       ->setDoesNotReturn();
diff --git a/lib/CodeGen/CGExpr.cpp b/lib/CodeGen/CGExpr.cpp
index 1fe4c18..63cc5b5 100644
--- a/lib/CodeGen/CGExpr.cpp
+++ b/lib/CodeGen/CGExpr.cpp
@@ -26,7 +26,8 @@
 #include "llvm/Intrinsics.h"
 #include "llvm/LLVMContext.h"
 #include "llvm/MDBuilder.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
+#include "llvm/ADT/Hashing.h"
 using namespace clang;
 using namespace CodeGen;
 
@@ -156,50 +157,6 @@ void CodeGenFunction::EmitAnyExprToMem(const Expr *E,
   }
 }
 
-namespace {
-/// \brief An adjustment to be made to the temporary created when emitting a
-/// reference binding, which accesses a particular subobject of that temporary.
-  struct SubobjectAdjustment {
-    enum {
-      DerivedToBaseAdjustment,
-      FieldAdjustment,
-      MemberPointerAdjustment
-    } Kind;
-
-    union {
-      struct {
-        const CastExpr *BasePath;
-        const CXXRecordDecl *DerivedClass;
-      } DerivedToBase;
-
-      FieldDecl *Field;
-
-      struct {
-        const MemberPointerType *MPT;
-        llvm::Value *Ptr;
-      } Ptr;
-    };
-
-    SubobjectAdjustment(const CastExpr *BasePath,
-                        const CXXRecordDecl *DerivedClass)
-      : Kind(DerivedToBaseAdjustment) {
-      DerivedToBase.BasePath = BasePath;
-      DerivedToBase.DerivedClass = DerivedClass;
-    }
-
-    SubobjectAdjustment(FieldDecl *Field)
-      : Kind(FieldAdjustment) {
-      this->Field = Field;
-    }
-
-    SubobjectAdjustment(const MemberPointerType *MPT, llvm::Value *Ptr)
-      : Kind(MemberPointerAdjustment) {
-      this->Ptr.MPT = MPT;
-      this->Ptr.Ptr = Ptr;
-    }
-  };
-}
-
 static llvm::Value *
 CreateReferenceTemporary(CodeGenFunction &CGF, QualType Type,
                          const NamedDecl *InitializedDecl) {
@@ -232,32 +189,18 @@ EmitExprForReferenceBinding(CodeGenFunction &CGF, const Expr *E,
                             const CXXDestructorDecl *&ReferenceTemporaryDtor,
                             QualType &ObjCARCReferenceLifetimeType,
                             const NamedDecl *InitializedDecl) {
-  // Look through single-element init lists that claim to be lvalues. They're
-  // just syntactic wrappers in this case.
-  if (const InitListExpr *ILE = dyn_cast<InitListExpr>(E)) {
-    if (ILE->getNumInits() == 1 && ILE->isGLValue())
-      E = ILE->getInit(0);
-  }
-
-  // Look through expressions for materialized temporaries (for now).
-  if (const MaterializeTemporaryExpr *M 
-                                      = dyn_cast<MaterializeTemporaryExpr>(E)) {
-    // Objective-C++ ARC:
-    //   If we are binding a reference to a temporary that has ownership, we 
-    //   need to perform retain/release operations on the temporary.
-    if (CGF.getContext().getLangOpts().ObjCAutoRefCount &&        
-        E->getType()->isObjCLifetimeType() &&
-        (E->getType().getObjCLifetime() == Qualifiers::OCL_Strong ||
-         E->getType().getObjCLifetime() == Qualifiers::OCL_Weak ||
-         E->getType().getObjCLifetime() == Qualifiers::OCL_Autoreleasing))
-      ObjCARCReferenceLifetimeType = E->getType();
-    
-    E = M->GetTemporaryExpr();
-  }
+  const MaterializeTemporaryExpr *M = NULL;
+  E = E->findMaterializedTemporary(M);
+  // Objective-C++ ARC:
+  //   If we are binding a reference to a temporary that has ownership, we
+  //   need to perform retain/release operations on the temporary.
+  if (M && CGF.getLangOpts().ObjCAutoRefCount &&
+      M->getType()->isObjCLifetimeType() &&
+      (M->getType().getObjCLifetime() == Qualifiers::OCL_Strong ||
+       M->getType().getObjCLifetime() == Qualifiers::OCL_Weak ||
+       M->getType().getObjCLifetime() == Qualifiers::OCL_Autoreleasing))
+    ObjCARCReferenceLifetimeType = M->getType();
 
-  if (const CXXDefaultArgExpr *DAE = dyn_cast<CXXDefaultArgExpr>(E))
-    E = DAE->getExpr();
-  
   if (const ExprWithCleanups *EWC = dyn_cast<ExprWithCleanups>(E)) {
     CGF.enterFullExpression(EWC);
     CodeGenFunction::RunCleanupsScope Scope(CGF);
@@ -335,54 +278,13 @@ EmitExprForReferenceBinding(CodeGenFunction &CGF, const Expr *E,
       
       return ReferenceTemporary;
     }
-    
-    SmallVector<SubobjectAdjustment, 2> Adjustments;
-    while (true) {
-      E = E->IgnoreParens();
-
-      if (const CastExpr *CE = dyn_cast<CastExpr>(E)) {
-        if ((CE->getCastKind() == CK_DerivedToBase ||
-             CE->getCastKind() == CK_UncheckedDerivedToBase) &&
-            E->getType()->isRecordType()) {
-          E = CE->getSubExpr();
-          CXXRecordDecl *Derived 
-            = cast<CXXRecordDecl>(E->getType()->getAs<RecordType>()->getDecl());
-          Adjustments.push_back(SubobjectAdjustment(CE, Derived));
-          continue;
-        }
-
-        if (CE->getCastKind() == CK_NoOp) {
-          E = CE->getSubExpr();
-          continue;
-        }
-      } else if (const MemberExpr *ME = dyn_cast<MemberExpr>(E)) {
-        if (!ME->isArrow() && ME->getBase()->isRValue()) {
-          assert(ME->getBase()->getType()->isRecordType());
-          if (FieldDecl *Field = dyn_cast<FieldDecl>(ME->getMemberDecl())) {
-            E = ME->getBase();
-            Adjustments.push_back(SubobjectAdjustment(Field));
-            continue;
-          }
-        }
-      } else if (const BinaryOperator *BO = dyn_cast<BinaryOperator>(E)) {
-        if (BO->isPtrMemOp()) {
-          assert(BO->getLHS()->isRValue());
-          E = BO->getLHS();
-          const MemberPointerType *MPT =
-              BO->getRHS()->getType()->getAs<MemberPointerType>();
-          llvm::Value *Ptr = CGF.EmitScalarExpr(BO->getRHS());
-          Adjustments.push_back(SubobjectAdjustment(MPT, Ptr));
-        }
-      }
 
-      if (const OpaqueValueExpr *opaque = dyn_cast<OpaqueValueExpr>(E))
-        if (opaque->getType()->isRecordType())
-          return CGF.EmitOpaqueValueLValue(opaque).getAddress();
+    SmallVector<SubobjectAdjustment, 2> Adjustments;
+    E = E->skipRValueSubobjectAdjustments(Adjustments);
+    if (const OpaqueValueExpr *opaque = dyn_cast<OpaqueValueExpr>(E))
+      if (opaque->getType()->isRecordType())
+        return CGF.EmitOpaqueValueLValue(opaque).getAddress();
 
-      // Nothing changed.
-      break;
-    }
-    
     // Create a reference temporary if necessary.
     AggValueSlot AggSlot = AggValueSlot::ignored();
     if (CGF.hasAggregateLLVMType(E->getType()) &&
@@ -446,8 +348,9 @@ EmitExprForReferenceBinding(CodeGenFunction &CGF, const Expr *E,
         }
 
         case SubobjectAdjustment::MemberPointerAdjustment: {
+          llvm::Value *Ptr = CGF.EmitScalarExpr(Adjustment.Ptr.RHS);
           Object = CGF.CGM.getCXXABI().EmitMemberDataPointerAddress(
-                        CGF, Object, Adjustment.Ptr.Ptr, Adjustment.Ptr.MPT);
+                        CGF, Object, Ptr, Adjustment.Ptr.MPT);
           break;
         }
         }
@@ -486,6 +389,15 @@ CodeGenFunction::EmitReferenceBindingToExpr(const Expr *E,
                                                    ReferenceTemporaryDtor,
                                                    ObjCARCReferenceLifetimeType,
                                                    InitializedDecl);
+  if (SanitizePerformTypeCheck && !E->getType()->isFunctionType()) {
+    // C++11 [dcl.ref]p5 (as amended by core issue 453):
+    //   If a glvalue to which a reference is directly bound designates neither
+    //   an existing object or function of an appropriate type nor a region of
+    //   storage of suitable size and alignment to contain an object of the
+    //   reference's type, the behavior is undefined.
+    QualType Ty = E->getType();
+    EmitTypeCheck(TCK_ReferenceBinding, E->getExprLoc(), Value, Ty);
+  }
   if (!ReferenceTemporaryDtor && ObjCARCReferenceLifetimeType.isNull())
     return RValue::get(Value);
   
@@ -549,22 +461,133 @@ unsigned CodeGenFunction::getAccessedFieldNo(unsigned Idx,
       ->getZExtValue();
 }
 
-void CodeGenFunction::EmitCheck(llvm::Value *Address, unsigned Size) {
-  if (!CatchUndefined)
+/// Emit the hash_16_bytes function from include/llvm/ADT/Hashing.h.
+static llvm::Value *emitHash16Bytes(CGBuilderTy &Builder, llvm::Value *Low,
+                                    llvm::Value *High) {
+  llvm::Value *KMul = Builder.getInt64(0x9ddfea08eb382d69ULL);
+  llvm::Value *K47 = Builder.getInt64(47);
+  llvm::Value *A0 = Builder.CreateMul(Builder.CreateXor(Low, High), KMul);
+  llvm::Value *A1 = Builder.CreateXor(Builder.CreateLShr(A0, K47), A0);
+  llvm::Value *B0 = Builder.CreateMul(Builder.CreateXor(High, A1), KMul);
+  llvm::Value *B1 = Builder.CreateXor(Builder.CreateLShr(B0, K47), B0);
+  return Builder.CreateMul(B1, KMul);
+}
+
+void CodeGenFunction::EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc,
+                                    llvm::Value *Address,
+                                    QualType Ty, CharUnits Alignment) {
+  if (!SanitizePerformTypeCheck)
     return;
 
-  // This needs to be to the standard address space.
-  Address = Builder.CreateBitCast(Address, Int8PtrTy);
+  // Don't check pointers outside the default address space. The null check
+  // isn't correct, the object-size check isn't supported by LLVM, and we can't
+  // communicate the addresses to the runtime handler for the vptr check.
+  if (Address->getType()->getPointerAddressSpace())
+    return;
 
-  llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::objectsize, IntPtrTy);
+  llvm::Value *Cond = 0;
 
-  llvm::Value *Min = Builder.getFalse();
-  llvm::Value *C = Builder.CreateCall2(F, Address, Min);
-  llvm::BasicBlock *Cont = createBasicBlock();
-  Builder.CreateCondBr(Builder.CreateICmpUGE(C,
-                                        llvm::ConstantInt::get(IntPtrTy, Size)),
-                       Cont, getTrapBB());
-  EmitBlock(Cont);
+  if (getLangOpts().SanitizeNull) {
+    // The glvalue must not be an empty glvalue.
+    Cond = Builder.CreateICmpNE(
+        Address, llvm::Constant::getNullValue(Address->getType()));
+  }
+
+  if (getLangOpts().SanitizeObjectSize && !Ty->isIncompleteType()) {
+    uint64_t Size = getContext().getTypeSizeInChars(Ty).getQuantity();
+
+    // The glvalue must refer to a large enough storage region.
+    // FIXME: If Address Sanitizer is enabled, insert dynamic instrumentation
+    //        to check this.
+    llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::objectsize, IntPtrTy);
+    llvm::Value *Min = Builder.getFalse();
+    llvm::Value *CastAddr = Builder.CreateBitCast(Address, Int8PtrTy);
+    llvm::Value *LargeEnough =
+        Builder.CreateICmpUGE(Builder.CreateCall2(F, CastAddr, Min),
+                              llvm::ConstantInt::get(IntPtrTy, Size));
+    Cond = Cond ? Builder.CreateAnd(Cond, LargeEnough) : LargeEnough;
+  }
+
+  uint64_t AlignVal = 0;
+
+  if (getLangOpts().SanitizeAlignment) {
+    AlignVal = Alignment.getQuantity();
+    if (!Ty->isIncompleteType() && !AlignVal)
+      AlignVal = getContext().getTypeAlignInChars(Ty).getQuantity();
+
+    // The glvalue must be suitably aligned.
+    if (AlignVal) {
+      llvm::Value *Align =
+          Builder.CreateAnd(Builder.CreatePtrToInt(Address, IntPtrTy),
+                            llvm::ConstantInt::get(IntPtrTy, AlignVal - 1));
+      llvm::Value *Aligned =
+        Builder.CreateICmpEQ(Align, llvm::ConstantInt::get(IntPtrTy, 0));
+      Cond = Cond ? Builder.CreateAnd(Cond, Aligned) : Aligned;
+    }
+  }
+
+  if (Cond) {
+    llvm::Constant *StaticData[] = {
+      EmitCheckSourceLocation(Loc),
+      EmitCheckTypeDescriptor(Ty),
+      llvm::ConstantInt::get(SizeTy, AlignVal),
+      llvm::ConstantInt::get(Int8Ty, TCK)
+    };
+    EmitCheck(Cond, "type_mismatch", StaticData, Address);
+  }
+
+  // If possible, check that the vptr indicates that there is a subobject of
+  // type Ty at offset zero within this object.
+  CXXRecordDecl *RD = Ty->getAsCXXRecordDecl();
+  if (getLangOpts().SanitizeVptr && TCK != TCK_ConstructorCall &&
+      RD && RD->hasDefinition() && RD->isDynamicClass()) {
+    // Compute a hash of the mangled name of the type.
+    //
+    // FIXME: This is not guaranteed to be deterministic! Move to a
+    //        fingerprinting mechanism once LLVM provides one. For the time
+    //        being the implementation happens to be deterministic.
+    llvm::SmallString<64> MangledName;
+    llvm::raw_svector_ostream Out(MangledName);
+    CGM.getCXXABI().getMangleContext().mangleCXXRTTI(Ty.getUnqualifiedType(),
+                                                     Out);
+    llvm::hash_code TypeHash = hash_value(Out.str());
+
+    // Load the vptr, and compute hash_16_bytes(TypeHash, vptr).
+    llvm::Value *Low = llvm::ConstantInt::get(Int64Ty, TypeHash);
+    llvm::Type *VPtrTy = llvm::PointerType::get(IntPtrTy, 0);
+    llvm::Value *VPtrAddr = Builder.CreateBitCast(Address, VPtrTy);
+    llvm::Value *VPtrVal = Builder.CreateLoad(VPtrAddr);
+    llvm::Value *High = Builder.CreateZExt(VPtrVal, Int64Ty);
+
+    llvm::Value *Hash = emitHash16Bytes(Builder, Low, High);
+    Hash = Builder.CreateTrunc(Hash, IntPtrTy);
+
+    // Look the hash up in our cache.
+    const int CacheSize = 128;
+    llvm::Type *HashTable = llvm::ArrayType::get(IntPtrTy, CacheSize);
+    llvm::Value *Cache = CGM.CreateRuntimeVariable(HashTable,
+                                                   "__ubsan_vptr_type_cache");
+    llvm::Value *Slot = Builder.CreateAnd(Hash,
+                                          llvm::ConstantInt::get(IntPtrTy,
+                                                                 CacheSize-1));
+    llvm::Value *Indices[] = { Builder.getInt32(0), Slot };
+    llvm::Value *CacheVal =
+      Builder.CreateLoad(Builder.CreateInBoundsGEP(Cache, Indices));
+
+    // If the hash isn't in the cache, call a runtime handler to perform the
+    // hard work of checking whether the vptr is for an object of the right
+    // type. This will either fill in the cache and return, or produce a
+    // diagnostic.
+    llvm::Constant *StaticData[] = {
+      EmitCheckSourceLocation(Loc),
+      EmitCheckTypeDescriptor(Ty),
+      CGM.GetAddrOfRTTIDescriptor(Ty.getUnqualifiedType()),
+      llvm::ConstantInt::get(Int8Ty, TCK)
+    };
+    llvm::Value *DynamicData[] = { Address, Hash };
+    EmitCheck(Builder.CreateICmpEQ(CacheVal, Hash),
+              "dynamic_type_cache_miss", StaticData, DynamicData, true);
+  }
 }
 
 
@@ -641,11 +664,11 @@ LValue CodeGenFunction::EmitUnsupportedLValue(const Expr *E,
   return MakeAddrLValue(llvm::UndefValue::get(Ty), E->getType());
 }
 
-LValue CodeGenFunction::EmitCheckedLValue(const Expr *E) {
+LValue CodeGenFunction::EmitCheckedLValue(const Expr *E, TypeCheckKind TCK) {
   LValue LV = EmitLValue(E);
   if (!isa<DeclRefExpr>(E) && !LV.isBitField() && LV.isSimple())
-    EmitCheck(LV.getAddress(), 
-              getContext().getTypeSizeInChars(E->getType()).getQuantity());
+    EmitTypeCheck(TCK, E->getExprLoc(), LV.getAddress(),
+                  E->getType(), LV.getAlignment());
   return LV;
 }
 
@@ -672,7 +695,7 @@ LValue CodeGenFunction::EmitLValue(const Expr *E) {
     llvm_unreachable("cannot emit a property reference directly");
 
   case Expr::ObjCSelectorExprClass:
-  return EmitObjCSelectorLValue(cast<ObjCSelectorExpr>(E));
+    return EmitObjCSelectorLValue(cast<ObjCSelectorExpr>(E));
   case Expr::ObjCIsaExprClass:
     return EmitObjCIsaExpr(cast<ObjCIsaExpr>(E));
   case Expr::BinaryOperatorClass:
@@ -709,6 +732,8 @@ LValue CodeGenFunction::EmitLValue(const Expr *E) {
     return EmitCXXConstructLValue(cast<CXXConstructExpr>(E));
   case Expr::CXXBindTemporaryExprClass:
     return EmitCXXBindTemporaryLValue(cast<CXXBindTemporaryExpr>(E));
+  case Expr::CXXUuidofExprClass:
+    return EmitCXXUuidofLValue(cast<CXXUuidofExpr>(E));
   case Expr::LambdaExprClass:
     return EmitLambdaLValue(cast<LambdaExpr>(E));
 
@@ -1124,7 +1149,7 @@ RValue CodeGenFunction::EmitLoadOfBitfieldLValue(LValue LV) {
 
   // Get the output type.
   llvm::Type *ResLTy = ConvertType(LV.getType());
-  unsigned ResSizeInBits = CGM.getTargetData().getTypeSizeInBits(ResLTy);
+  unsigned ResSizeInBits = CGM.getDataLayout().getTypeSizeInBits(ResLTy);
 
   // Compute the result as an OR of all of the individual component accesses.
   llvm::Value *Res = 0;
@@ -1322,7 +1347,7 @@ void CodeGenFunction::EmitStoreThroughBitfieldLValue(RValue Src, LValue Dst,
 
   // Get the output type.
   llvm::Type *ResLTy = ConvertTypeForMem(Dst.getType());
-  unsigned ResSizeInBits = CGM.getTargetData().getTypeSizeInBits(ResLTy);
+  unsigned ResSizeInBits = CGM.getDataLayout().getTypeSizeInBits(ResLTy);
 
   // Get the source value, truncated to the width of the bit-field.
   llvm::Value *SrcVal = Src.getScalarVal();
@@ -1645,6 +1670,21 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
   CharUnits Alignment = getContext().getDeclAlign(ND);
   QualType T = E->getType();
 
+  // A DeclRefExpr for a reference initialized by a constant expression can
+  // appear without being odr-used. Directly emit the constant initializer.
+  if (const VarDecl *VD = dyn_cast<VarDecl>(ND)) {
+    const Expr *Init = VD->getAnyInitializer(VD);
+    if (Init && !isa<ParmVarDecl>(VD) && VD->getType()->isReferenceType() &&
+        VD->isUsableInConstantExpressions(getContext()) &&
+        VD->checkInitIsICE()) {
+      llvm::Constant *Val =
+        CGM.EmitConstantValue(*VD->evaluateValue(), VD->getType(), this);
+      assert(Val && "failed to emit reference constant expression");
+      // FIXME: Eventually we will want to emit vector element references.
+      return MakeAddrLValue(Val, T, Alignment);
+    }
+  }
+
   // FIXME: We should be able to assert this for FunctionDecls as well!
   // FIXME: We should be able to assert this for all DeclRefExprs, not just
   // those with a valid source location.
@@ -1655,7 +1695,7 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
   if (ND->hasAttr<WeakRefAttr>()) {
     const ValueDecl *VD = cast<ValueDecl>(ND);
     llvm::Constant *Aliasee = CGM.GetWeakRefReference(VD);
-    return MakeAddrLValue(Aliasee, E->getType(), Alignment);
+    return MakeAddrLValue(Aliasee, T, Alignment);
   }
 
   if (const VarDecl *VD = dyn_cast<VarDecl>(ND)) {
@@ -1683,9 +1723,8 @@ LValue CodeGenFunction::EmitDeclRefLValue(const DeclRefExpr *E) {
       }
 
       assert(isa<BlockDecl>(CurCodeDecl) && E->refersToEnclosingLocal());
-      CharUnits alignment = getContext().getDeclAlign(VD);
       return MakeAddrLValue(GetAddrOfBlockDecl(VD, isBlockVariable),
-                            E->getType(), alignment);
+                            T, Alignment);
     }
 
     assert(V && "DeclRefExpr not entered in LocalDeclMap?");
@@ -1736,8 +1775,8 @@ LValue CodeGenFunction::EmitUnaryOpLValue(const UnaryOperator *E) {
     // of a pointer to object; as in void foo (__weak id *param); *param = 0;
     // But, we continue to generate __strong write barrier on indirect write
     // into a pointer to object.
-    if (getContext().getLangOpts().ObjC1 &&
-        getContext().getLangOpts().getGC() != LangOptions::NonGC &&
+    if (getLangOpts().ObjC1 &&
+        getLangOpts().getGC() != LangOptions::NonGC &&
         LV.isObjCWeak())
       LV.setNonGC(!E->isOBJCGCCandidate(getContext()));
     return LV;
@@ -1815,8 +1854,9 @@ GetAddrOfConstantWideString(StringRef Str,
 static void ConvertUTF8ToWideString(unsigned CharByteWidth, StringRef Source,
                                     SmallString<32>& Target) {
   Target.resize(CharByteWidth * (Source.size() + 1));
-  char* ResultPtr = &Target[0];
-  bool success = ConvertUTF8toWide(CharByteWidth, Source, ResultPtr);
+  char *ResultPtr = &Target[0];
+  const UTF8 *ErrorPtr;
+  bool success = ConvertUTF8toWide(CharByteWidth, Source, ResultPtr, ErrorPtr);
   (void)success;
   assert(success);
   Target.resize(ResultPtr - &Target[0]);
@@ -1888,33 +1928,167 @@ LValue CodeGenFunction::EmitPredefinedLValue(const PredefinedExpr *E) {
   }
 }
 
-llvm::BasicBlock *CodeGenFunction::getTrapBB() {
-  const CodeGenOptions &GCO = CGM.getCodeGenOpts();
+/// Emit a type description suitable for use by a runtime sanitizer library. The
+/// format of a type descriptor is
+///
+/// \code
+///   { i16 TypeKind, i16 TypeInfo }
+/// \endcode
+///
+/// followed by an array of i8 containing the type name. TypeKind is 0 for an
+/// integer, 1 for a floating point value, and -1 for anything else.
+llvm::Constant *CodeGenFunction::EmitCheckTypeDescriptor(QualType T) {
+  // FIXME: Only emit each type's descriptor once.
+  uint16_t TypeKind = -1;
+  uint16_t TypeInfo = 0;
+
+  if (T->isIntegerType()) {
+    TypeKind = 0;
+    TypeInfo = (llvm::Log2_32(getContext().getTypeSize(T)) << 1) |
+               T->isSignedIntegerType();
+  } else if (T->isFloatingType()) {
+    TypeKind = 1;
+    TypeInfo = getContext().getTypeSize(T);
+  }
+
+  // Format the type name as if for a diagnostic, including quotes and
+  // optionally an 'aka'.
+  llvm::SmallString<32> Buffer;
+  CGM.getDiags().ConvertArgToString(DiagnosticsEngine::ak_qualtype,
+                                    (intptr_t)T.getAsOpaquePtr(),
+                                    0, 0, 0, 0, 0, 0, Buffer,
+                                    ArrayRef<intptr_t>());
+
+  llvm::Constant *Components[] = {
+    Builder.getInt16(TypeKind), Builder.getInt16(TypeInfo),
+    llvm::ConstantDataArray::getString(getLLVMContext(), Buffer)
+  };
+  llvm::Constant *Descriptor = llvm::ConstantStruct::getAnon(Components);
+
+  llvm::GlobalVariable *GV =
+    new llvm::GlobalVariable(CGM.getModule(), Descriptor->getType(),
+                             /*isConstant=*/true,
+                             llvm::GlobalVariable::PrivateLinkage,
+                             Descriptor);
+  GV->setUnnamedAddr(true);
+  return GV;
+}
 
-  // If we are not optimzing, don't collapse all calls to trap in the function
-  // to the same call, that way, in the debugger they can see which operation
-  // did in fact fail.  If we are optimizing, we collapse all calls to trap down
-  // to just one per function to save on codesize.
-  if (GCO.OptimizationLevel && TrapBB)
-    return TrapBB;
+llvm::Value *CodeGenFunction::EmitCheckValue(llvm::Value *V) {
+  llvm::Type *TargetTy = IntPtrTy;
+
+  // Integers which fit in intptr_t are zero-extended and passed directly.
+  if (V->getType()->isIntegerTy() &&
+      V->getType()->getIntegerBitWidth() <= TargetTy->getIntegerBitWidth())
+    return Builder.CreateZExt(V, TargetTy);
+
+  // Pointers are passed directly, everything else is passed by address.
+  if (!V->getType()->isPointerTy()) {
+    llvm::Value *Ptr = Builder.CreateAlloca(V->getType());
+    Builder.CreateStore(V, Ptr);
+    V = Ptr;
+  }
+  return Builder.CreatePtrToInt(V, TargetTy);
+}
+
+/// \brief Emit a representation of a SourceLocation for passing to a handler
+/// in a sanitizer runtime library. The format for this data is:
+/// \code
+///   struct SourceLocation {
+///     const char *Filename;
+///     int32_t Line, Column;
+///   };
+/// \endcode
+/// For an invalid SourceLocation, the Filename pointer is null.
+llvm::Constant *CodeGenFunction::EmitCheckSourceLocation(SourceLocation Loc) {
+  PresumedLoc PLoc = getContext().getSourceManager().getPresumedLoc(Loc);
+
+  llvm::Constant *Data[] = {
+    // FIXME: Only emit each file name once.
+    PLoc.isValid() ? cast<llvm::Constant>(
+                       Builder.CreateGlobalStringPtr(PLoc.getFilename()))
+                   : llvm::Constant::getNullValue(Int8PtrTy),
+    Builder.getInt32(PLoc.getLine()),
+    Builder.getInt32(PLoc.getColumn())
+  };
 
-  llvm::BasicBlock *Cont = 0;
-  if (HaveInsertPoint()) {
-    Cont = createBasicBlock("cont");
-    EmitBranch(Cont);
+  return llvm::ConstantStruct::getAnon(Data);
+}
+
+void CodeGenFunction::EmitCheck(llvm::Value *Checked, StringRef CheckName,
+                                llvm::ArrayRef<llvm::Constant *> StaticArgs,
+                                llvm::ArrayRef<llvm::Value *> DynamicArgs,
+                                bool Recoverable) {
+  llvm::BasicBlock *Cont = createBasicBlock("cont");
+
+  llvm::BasicBlock *Handler = createBasicBlock("handler." + CheckName);
+  Builder.CreateCondBr(Checked, Cont, Handler);
+  EmitBlock(Handler);
+
+  llvm::Constant *Info = llvm::ConstantStruct::getAnon(StaticArgs);
+  llvm::GlobalValue *InfoPtr =
+      new llvm::GlobalVariable(CGM.getModule(), Info->getType(), true,
+                               llvm::GlobalVariable::PrivateLinkage, Info);
+  InfoPtr->setUnnamedAddr(true);
+
+  llvm::SmallVector<llvm::Value *, 4> Args;
+  llvm::SmallVector<llvm::Type *, 4> ArgTypes;
+  Args.reserve(DynamicArgs.size() + 1);
+  ArgTypes.reserve(DynamicArgs.size() + 1);
+
+  // Handler functions take an i8* pointing to the (handler-specific) static
+  // information block, followed by a sequence of intptr_t arguments
+  // representing operand values.
+  Args.push_back(Builder.CreateBitCast(InfoPtr, Int8PtrTy));
+  ArgTypes.push_back(Int8PtrTy);
+  for (size_t i = 0, n = DynamicArgs.size(); i != n; ++i) {
+    Args.push_back(EmitCheckValue(DynamicArgs[i]));
+    ArgTypes.push_back(IntPtrTy);
+  }
+
+  llvm::FunctionType *FnType =
+    llvm::FunctionType::get(CGM.VoidTy, ArgTypes, false);
+  llvm::AttrBuilder B;
+  if (!Recoverable) {
+    B.addAttribute(llvm::Attributes::NoReturn)
+     .addAttribute(llvm::Attributes::NoUnwind);
+  }
+  B.addAttribute(llvm::Attributes::UWTable);
+  llvm::Value *Fn = CGM.CreateRuntimeFunction(FnType,
+                                          ("__ubsan_handle_" + CheckName).str(),
+                                         llvm::Attributes::get(getLLVMContext(),
+                                                               B));
+  llvm::CallInst *HandlerCall = Builder.CreateCall(Fn, Args);
+  if (Recoverable) {
+    Builder.CreateBr(Cont);
+  } else {
+    HandlerCall->setDoesNotReturn();
+    HandlerCall->setDoesNotThrow();
+    Builder.CreateUnreachable();
   }
-  TrapBB = createBasicBlock("trap");
-  EmitBlock(TrapBB);
 
-  llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::trap);
-  llvm::CallInst *TrapCall = Builder.CreateCall(F);
-  TrapCall->setDoesNotReturn();
-  TrapCall->setDoesNotThrow();
-  Builder.CreateUnreachable();
+  EmitBlock(Cont);
+}
 
-  if (Cont)
-    EmitBlock(Cont);
-  return TrapBB;
+void CodeGenFunction::EmitTrapvCheck(llvm::Value *Checked) {
+  llvm::BasicBlock *Cont = createBasicBlock("cont");
+
+  // If we're optimizing, collapse all calls to trap down to just one per
+  // function to save on code size.
+  if (!CGM.getCodeGenOpts().OptimizationLevel || !TrapBB) {
+    TrapBB = createBasicBlock("trap");
+    Builder.CreateCondBr(Checked, Cont, TrapBB);
+    EmitBlock(TrapBB);
+    llvm::Value *F = CGM.getIntrinsic(llvm::Intrinsic::trap);
+    llvm::CallInst *TrapCall = Builder.CreateCall(F);
+    TrapCall->setDoesNotReturn();
+    TrapCall->setDoesNotThrow();
+    Builder.CreateUnreachable();
+  } else {
+    Builder.CreateCondBr(Checked, Cont, TrapBB);
+  }
+
+  EmitBlock(Cont);
 }
 
 /// isSimpleArrayDecayOperand - If the specified expr is a simple decay from an
@@ -2007,14 +2181,14 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E) {
     // Propagate the alignment from the array itself to the result.
     ArrayAlignment = ArrayLV.getAlignment();
 
-    if (getContext().getLangOpts().isSignedOverflowDefined())
+    if (getLangOpts().isSignedOverflowDefined())
       Address = Builder.CreateGEP(ArrayPtr, Args, "arrayidx");
     else
       Address = Builder.CreateInBoundsGEP(ArrayPtr, Args, "arrayidx");
   } else {
     // The base must be a pointer, which is not an aggregate.  Emit it.
     llvm::Value *Base = EmitScalarExpr(E->getBase());
-    if (getContext().getLangOpts().isSignedOverflowDefined())
+    if (getLangOpts().isSignedOverflowDefined())
       Address = Builder.CreateGEP(Base, Idx, "arrayidx");
     else
       Address = Builder.CreateInBoundsGEP(Base, Idx, "arrayidx");
@@ -2037,8 +2211,8 @@ LValue CodeGenFunction::EmitArraySubscriptExpr(const ArraySubscriptExpr *E) {
 
   LV.getQuals().setAddressSpace(E->getBase()->getType().getAddressSpace());
 
-  if (getContext().getLangOpts().ObjC1 &&
-      getContext().getLangOpts().getGC() != LangOptions::NonGC) {
+  if (getLangOpts().ObjC1 &&
+      getLangOpts().getGC() != LangOptions::NonGC) {
     LV.setNonGC(!E->isOBJCGCCandidate(getContext()));
     setObjCGCLValueClass(getContext(), E, LV);
   }
@@ -2114,11 +2288,13 @@ LValue CodeGenFunction::EmitMemberExpr(const MemberExpr *E) {
 
   // If this is s.x, emit s as an lvalue.  If it is s->x, emit s as a scalar.
   LValue BaseLV;
-  if (E->isArrow())
-    BaseLV = MakeNaturalAlignAddrLValue(EmitScalarExpr(BaseExpr),
-                                        BaseExpr->getType()->getPointeeType());
-  else
-    BaseLV = EmitLValue(BaseExpr);
+  if (E->isArrow()) {
+    llvm::Value *Ptr = EmitScalarExpr(BaseExpr);
+    QualType PtrTy = BaseExpr->getType()->getPointeeType();
+    EmitTypeCheck(TCK_MemberAccess, E->getExprLoc(), Ptr, PtrTy);
+    BaseLV = MakeNaturalAlignAddrLValue(Ptr, PtrTy);
+  } else
+    BaseLV = EmitCheckedLValue(BaseExpr, TCK_MemberAccess);
 
   NamedDecl *ND = E->getMemberDecl();
   if (FieldDecl *Field = dyn_cast<FieldDecl>(ND)) {
@@ -2355,7 +2531,10 @@ LValue CodeGenFunction::EmitCastLValue(const CastExpr *E) {
 
   case CK_Dependent:
     llvm_unreachable("dependent cast kind in IR gen!");
- 
+
+  case CK_BuiltinFnToFnPtr:
+    llvm_unreachable("builtin functions are handled elsewhere");
+
   // These two casts are currently treated as no-ops, although they could
   // potentially be real operations depending on the target's ABI.
   case CK_NonAtomicToAtomic:
@@ -2546,7 +2725,7 @@ RValue CodeGenFunction::EmitCallExpr(const CallExpr *E,
   if (const CXXPseudoDestructorExpr *PseudoDtor 
           = dyn_cast<CXXPseudoDestructorExpr>(E->getCallee()->IgnoreParens())) {
     QualType DestroyedType = PseudoDtor->getDestroyedType();
-    if (getContext().getLangOpts().ObjCAutoRefCount &&
+    if (getLangOpts().ObjCAutoRefCount &&
         DestroyedType->isObjCLifetimeType() &&
         (DestroyedType.getObjCLifetime() == Qualifiers::OCL_Strong ||
          DestroyedType.getObjCLifetime() == Qualifiers::OCL_Weak)) {
@@ -2635,7 +2814,7 @@ LValue CodeGenFunction::EmitBinaryOperatorLValue(const BinaryOperator *E) {
     }
 
     RValue RV = EmitAnyExpr(E->getRHS());
-    LValue LV = EmitLValue(E->getLHS());
+    LValue LV = EmitCheckedLValue(E->getLHS(), TCK_Store);
     EmitStoreThroughLValue(RV, LV);
     return LV;
   }
@@ -2677,6 +2856,14 @@ CodeGenFunction::EmitCXXTypeidLValue(const CXXTypeidExpr *E) {
   return MakeAddrLValue(EmitCXXTypeidExpr(E), E->getType());
 }
 
+llvm::Value *CodeGenFunction::EmitCXXUuidofExpr(const CXXUuidofExpr *E) {
+  return CGM.GetAddrOfUuidDescriptor(E);
+}
+
+LValue CodeGenFunction::EmitCXXUuidofLValue(const CXXUuidofExpr *E) {
+  return MakeAddrLValue(EmitCXXUuidofExpr(E), E->getType());
+}
+
 LValue
 CodeGenFunction::EmitCXXBindTemporaryLValue(const CXXBindTemporaryExpr *E) {
   AggValueSlot Slot = CreateAggTemp(E->getType(), "temp.lvalue");
@@ -2977,11 +3164,10 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) {
   uint64_t Size = sizeChars.getQuantity();
   CharUnits alignChars = getContext().getTypeAlignInChars(AtomicTy);
   unsigned Align = alignChars.getQuantity();
-  unsigned MaxInlineWidth =
-      getContext().getTargetInfo().getMaxAtomicInlineWidth();
-  bool UseLibcall = (Size != Align || Size > MaxInlineWidth);
-
-
+  unsigned MaxInlineWidthInBits =
+    getContext().getTargetInfo().getMaxAtomicInlineWidth();
+  bool UseLibcall = (Size != Align ||
+                     getContext().toBits(sizeChars) > MaxInlineWidthInBits);
 
   llvm::Value *Ptr, *Order, *OrderFail = 0, *Val1 = 0, *Val2 = 0;
   Ptr = EmitScalarExpr(E->getPtr());
@@ -3177,6 +3363,13 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) {
     return ConvertTempToRValue(*this, E->getType(), Dest);
   }
 
+  bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
+                 E->getOp() == AtomicExpr::AO__atomic_store ||
+                 E->getOp() == AtomicExpr::AO__atomic_store_n;
+  bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
+                E->getOp() == AtomicExpr::AO__atomic_load ||
+                E->getOp() == AtomicExpr::AO__atomic_load_n;
+
   llvm::Type *IPtrTy =
       llvm::IntegerType::get(getLLVMContext(), Size * 8)->getPointerTo();
   llvm::Value *OrigDest = Dest;
@@ -3194,14 +3387,20 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) {
       break;
     case 1:  // memory_order_consume
     case 2:  // memory_order_acquire
+      if (IsStore)
+        break; // Avoid crashing on code with undefined behavior
       EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
                    llvm::Acquire);
       break;
     case 3:  // memory_order_release
+      if (IsLoad)
+        break; // Avoid crashing on code with undefined behavior
       EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
                    llvm::Release);
       break;
     case 4:  // memory_order_acq_rel
+      if (IsLoad || IsStore)
+        break; // Avoid crashing on code with undefined behavior
       EmitAtomicOp(*this, E, Dest, Ptr, Val1, Val2, Size, Align,
                    llvm::AcquireRelease);
       break;
@@ -3221,13 +3420,6 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E, llvm::Value *Dest) {
 
   // Long case, when Order isn't obviously constant.
 
-  bool IsStore = E->getOp() == AtomicExpr::AO__c11_atomic_store ||
-                 E->getOp() == AtomicExpr::AO__atomic_store ||
-                 E->getOp() == AtomicExpr::AO__atomic_store_n;
-  bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load ||
-                E->getOp() == AtomicExpr::AO__atomic_load ||
-                E->getOp() == AtomicExpr::AO__atomic_load_n;
-
   // Create all the relevant BB's
   llvm::BasicBlock *MonotonicBB = 0, *AcquireBB = 0, *ReleaseBB = 0,
                    *AcqRelBB = 0, *SeqCstBB = 0;
diff --git a/lib/CodeGen/CGExprAgg.cpp b/lib/CodeGen/CGExprAgg.cpp
index 61f7362..718e8f9 100644
--- a/lib/CodeGen/CGExprAgg.cpp
+++ b/lib/CodeGen/CGExprAgg.cpp
@@ -549,8 +549,10 @@ AggExprEmitter::VisitCompoundLiteralExpr(CompoundLiteralExpr *E) {
 void AggExprEmitter::VisitCastExpr(CastExpr *E) {
   switch (E->getCastKind()) {
   case CK_Dynamic: {
+    // FIXME: Can this actually happen? We have no test coverage for it.
     assert(isa<CXXDynamicCastExpr>(E) && "CK_Dynamic without a dynamic_cast?");
-    LValue LV = CGF.EmitCheckedLValue(E->getSubExpr());
+    LValue LV = CGF.EmitCheckedLValue(E->getSubExpr(),
+                                      CodeGenFunction::TCK_Load);
     // FIXME: Do we also need to handle property references here?
     if (LV.isSimple())
       CGF.EmitDynamicCast(LV.getAddress(), cast<CXXDynamicCastExpr>(E));
@@ -645,6 +647,7 @@ void AggExprEmitter::VisitCastExpr(CastExpr *E) {
   case CK_ARCReclaimReturnedObject:
   case CK_ARCExtendBlockObject:
   case CK_CopyAndAutoreleaseBlockObject:
+  case CK_BuiltinFnToFnPtr:
     llvm_unreachable("cast kind invalid for aggregate types");
   }
 }
@@ -771,7 +774,7 @@ void AggExprEmitter::VisitBinAssign(const BinaryOperator *E) {
     Visit(E->getRHS());
 
     // Now emit the LHS and copy into it.
-    LValue LHS = CGF.EmitLValue(E->getLHS());
+    LValue LHS = CGF.EmitCheckedLValue(E->getLHS(), CodeGenFunction::TCK_Store);
 
     EmitCopy(E->getLHS()->getType(),
              AggValueSlot::forLValue(LHS, AggValueSlot::IsDestructed,
@@ -1205,7 +1208,7 @@ static void CheckAggExprForMemSetUse(AggValueSlot &Slot, const Expr *E,
   if (Slot.isZeroed() || Slot.isVolatile() || Slot.getAddr() == 0) return;
 
   // C++ objects with a user-declared constructor don't need zero'ing.
-  if (CGF.getContext().getLangOpts().CPlusPlus)
+  if (CGF.getLangOpts().CPlusPlus)
     if (const RecordType *RT = CGF.getContext()
                        .getBaseElementType(E->getType())->getAs<RecordType>()) {
       const CXXRecordDecl *RD = cast<CXXRecordDecl>(RT->getDecl());
@@ -1271,10 +1274,11 @@ LValue CodeGenFunction::EmitAggExprToLValue(const Expr *E) {
 void CodeGenFunction::EmitAggregateCopy(llvm::Value *DestPtr,
                                         llvm::Value *SrcPtr, QualType Ty,
                                         bool isVolatile,
-                                        CharUnits alignment) {
+                                        CharUnits alignment,
+                                        bool isAssignment) {
   assert(!Ty->isAnyComplexType() && "Shouldn't happen for complex");
 
-  if (getContext().getLangOpts().CPlusPlus) {
+  if (getLangOpts().CPlusPlus) {
     if (const RecordType *RT = Ty->getAs<RecordType>()) {
       CXXRecordDecl *Record = cast<CXXRecordDecl>(RT->getDecl());
       assert((Record->hasTrivialCopyConstructor() || 
@@ -1300,9 +1304,13 @@ void CodeGenFunction::EmitAggregateCopy(llvm::Value *DestPtr,
   // implementation handles this case safely.  If there is a libc that does not
   // safely handle this, we can add a target hook.
 
-  // Get size and alignment info for this aggregate.
-  std::pair<CharUnits, CharUnits> TypeInfo = 
-    getContext().getTypeInfoInChars(Ty);
+  // Get data size and alignment info for this aggregate. If this is an
+  // assignment don't copy the tail padding. Otherwise copying it is fine.
+  std::pair<CharUnits, CharUnits> TypeInfo;
+  if (isAssignment)
+    TypeInfo = getContext().getTypeInfoDataSizeInChars(Ty);
+  else
+    TypeInfo = getContext().getTypeInfoInChars(Ty);
 
   if (alignment.isZero())
     alignment = TypeInfo.second;
@@ -1359,11 +1367,17 @@ void CodeGenFunction::EmitAggregateCopy(llvm::Value *DestPtr,
       }
     }
   }
+
+  // Determine the metadata to describe the position of any padding in this
+  // memcpy, as well as the TBAA tags for the members of the struct, in case
+  // the optimizer wishes to expand it in to scalar memory operations.
+  llvm::MDNode *TBAAStructTag = CGM.getTBAAStructInfo(Ty);
   
   Builder.CreateMemCpy(DestPtr, SrcPtr,
                        llvm::ConstantInt::get(IntPtrTy, 
                                               TypeInfo.first.getQuantity()),
-                       alignment.getQuantity(), isVolatile);
+                       alignment.getQuantity(), isVolatile,
+                       /*TBAATag=*/0, TBAAStructTag);
 }
 
 void CodeGenFunction::MaybeEmitStdInitializerListCleanup(llvm::Value *loc,
diff --git a/lib/CodeGen/CGExprCXX.cpp b/lib/CodeGen/CGExprCXX.cpp
index 31ea1b5..7f640f6 100644
--- a/lib/CodeGen/CGExprCXX.cpp
+++ b/lib/CodeGen/CGExprCXX.cpp
@@ -24,6 +24,7 @@ using namespace clang;
 using namespace CodeGen;
 
 RValue CodeGenFunction::EmitCXXMemberCall(const CXXMethodDecl *MD,
+                                          SourceLocation CallLoc,
                                           llvm::Value *Callee,
                                           ReturnValueSlot ReturnValue,
                                           llvm::Value *This,
@@ -33,6 +34,13 @@ RValue CodeGenFunction::EmitCXXMemberCall(const CXXMethodDecl *MD,
   assert(MD->isInstance() &&
          "Trying to emit a member call expr on a static method!");
 
+  // C++11 [class.mfct.non-static]p2:
+  //   If a non-static member function of a class X is called for an object that
+  //   is not of type X, or of a type derived from X, the behavior is undefined.
+  EmitTypeCheck(isa<CXXConstructorDecl>(MD) ? TCK_ConstructorCall
+                                            : TCK_MemberCall,
+                CallLoc, This, getContext().getRecordType(MD->getParent()));
+
   CallArgList Args;
 
   // Push the this ptr.
@@ -168,8 +176,9 @@ RValue CodeGenFunction::EmitCXXMemberCallExpr(const CXXMemberCallExpr *CE,
   const CXXMethodDecl *MD = cast<CXXMethodDecl>(ME->getMemberDecl());
 
   CGDebugInfo *DI = getDebugInfo();
-  if (DI && CGM.getCodeGenOpts().DebugInfo == CodeGenOptions::LimitedDebugInfo
-      && !isa<CallExpr>(ME->getBase())) {
+  if (DI &&
+      CGM.getCodeGenOpts().getDebugInfo() == CodeGenOptions::LimitedDebugInfo &&
+      !isa<CallExpr>(ME->getBase())) {
     QualType PQTy = ME->getBase()->IgnoreParenImpCasts()->getType();
     if (const PointerType * PTy = dyn_cast<PointerType>(PQTy)) {
       DI->getOrCreateRecordType(PTy->getPointeeType(), 
@@ -235,7 +244,7 @@ RValue CodeGenFunction::EmitCXXMemberCallExpr(const CXXMemberCallExpr *CE,
       // We don't like to generate the trivial copy/move assignment operator
       // when it isn't necessary; just produce the proper effect here.
       llvm::Value *RHS = EmitLValue(*CE->arg_begin()).getAddress();
-      EmitAggregateCopy(This, RHS, CE->getType());
+      EmitAggregateAssign(This, RHS, CE->getType());
       return RValue::get(This);
     }
     
@@ -251,16 +260,16 @@ RValue CodeGenFunction::EmitCXXMemberCallExpr(const CXXMemberCallExpr *CE,
   }
 
   // Compute the function type we're calling.
+  const CXXMethodDecl *CalleeDecl = DevirtualizedMethod ? DevirtualizedMethod : MD;
   const CGFunctionInfo *FInfo = 0;
-  if (isa<CXXDestructorDecl>(MD))
-    FInfo = &CGM.getTypes().arrangeCXXDestructor(cast<CXXDestructorDecl>(MD),
+  if (const CXXDestructorDecl *Dtor = dyn_cast<CXXDestructorDecl>(CalleeDecl))
+    FInfo = &CGM.getTypes().arrangeCXXDestructor(Dtor,
                                                  Dtor_Complete);
-  else if (isa<CXXConstructorDecl>(MD))
-    FInfo = &CGM.getTypes().arrangeCXXConstructorDeclaration(
-                                                 cast<CXXConstructorDecl>(MD),
-                                                 Ctor_Complete);
+  else if (const CXXConstructorDecl *Ctor = dyn_cast<CXXConstructorDecl>(CalleeDecl))
+    FInfo = &CGM.getTypes().arrangeCXXConstructorDeclaration(Ctor,
+                                                             Ctor_Complete);
   else
-    FInfo = &CGM.getTypes().arrangeCXXMethodDeclaration(MD);
+    FInfo = &CGM.getTypes().arrangeCXXMethodDeclaration(CalleeDecl);
 
   llvm::Type *Ty = CGM.getTypes().GetFunctionType(*FInfo);
 
@@ -277,7 +286,7 @@ RValue CodeGenFunction::EmitCXXMemberCallExpr(const CXXMemberCallExpr *CE,
     if (UseVirtualCall) {
       Callee = BuildVirtualCall(Dtor, Dtor_Complete, This, Ty);
     } else {
-      if (getContext().getLangOpts().AppleKext &&
+      if (getLangOpts().AppleKext &&
           MD->isVirtual() &&
           ME->hasQualifier())
         Callee = BuildAppleKextVirtualCall(MD, ME->getQualifier(), Ty);
@@ -295,7 +304,7 @@ RValue CodeGenFunction::EmitCXXMemberCallExpr(const CXXMemberCallExpr *CE,
   } else if (UseVirtualCall) {
       Callee = BuildVirtualCall(MD, This, Ty); 
   } else {
-    if (getContext().getLangOpts().AppleKext &&
+    if (getLangOpts().AppleKext &&
         MD->isVirtual() &&
         ME->hasQualifier())
       Callee = BuildAppleKextVirtualCall(MD, ME->getQualifier(), Ty);
@@ -306,8 +315,8 @@ RValue CodeGenFunction::EmitCXXMemberCallExpr(const CXXMemberCallExpr *CE,
     }
   }
 
-  return EmitCXXMemberCall(MD, Callee, ReturnValue, This, /*VTT=*/0,
-                           CE->arg_begin(), CE->arg_end());
+  return EmitCXXMemberCall(MD, CE->getExprLoc(), Callee, ReturnValue, This,
+                           /*VTT=*/0, CE->arg_begin(), CE->arg_end());
 }
 
 RValue
@@ -337,6 +346,9 @@ CodeGenFunction::EmitCXXMemberPointerCallExpr(const CXXMemberCallExpr *E,
   else 
     This = EmitLValue(BaseExpr).getAddress();
 
+  EmitTypeCheck(TCK_MemberCall, E->getExprLoc(), This,
+                QualType(MPT->getClass(), 0));
+
   // Ask the ABI to load the callee.  Note that This is modified.
   llvm::Value *Callee =
     CGM.getCXXABI().EmitLoadOfMemberFunctionPointer(*this, This, MemFnPtr, MPT);
@@ -370,13 +382,13 @@ CodeGenFunction::EmitCXXOperatorMemberCallExpr(const CXXOperatorCallExpr *E,
       MD->isTrivial()) {
     llvm::Value *Src = EmitLValue(E->getArg(1)).getAddress();
     QualType Ty = E->getType();
-    EmitAggregateCopy(This, Src, Ty);
+    EmitAggregateAssign(This, Src, Ty);
     return RValue::get(This);
   }
 
   llvm::Value *Callee = EmitCXXOperatorMemberCallee(E, MD, This);
-  return EmitCXXMemberCall(MD, Callee, ReturnValue, This, /*VTT=*/0,
-                           E->arg_begin() + 1, E->arg_end());
+  return EmitCXXMemberCall(MD, E->getExprLoc(), Callee, ReturnValue, This,
+                           /*VTT=*/0, E->arg_begin() + 1, E->arg_end());
 }
 
 RValue CodeGenFunction::EmitCUDAKernelCallExpr(const CUDAKernelCallExpr *E,
@@ -457,7 +469,7 @@ CodeGenFunction::EmitCXXConstructExpr(const CXXConstructExpr *E,
   // Elide the constructor if we're constructing from a temporary.
   // The temporary check is required because Sema sets this on NRVO
   // returns.
-  if (getContext().getLangOpts().ElideConstructors && E->isElidable()) {
+  if (getLangOpts().ElideConstructors && E->isElidable()) {
     assert(getContext().hasSameUnqualifiedType(E->getType(),
                                                E->getArg(0)->getType()));
     if (E->getArg(0)->isTemporaryObject(getContext(), CD->getParent())) {
@@ -878,7 +890,7 @@ CodeGenFunction::EmitNewArrayInitializer(const CXXNewExpr *E,
     if (constNum->getZExtValue() <= initializerElements) {
       // If there was a cleanup, deactivate it.
       if (cleanupDominator)
-        DeactivateCleanupBlock(cleanup, cleanupDominator);;
+        DeactivateCleanupBlock(cleanup, cleanupDominator);
       return;
     }
   } else {
@@ -949,7 +961,6 @@ static void EmitNewInitializer(CodeGenFunction &CGF, const CXXNewExpr *E,
   if (E->isArray()) {
     if (const CXXConstructExpr *CCE = dyn_cast_or_null<CXXConstructExpr>(Init)){
       CXXConstructorDecl *Ctor = CCE->getConstructor();
-      bool RequiresZeroInitialization = false;
       if (Ctor->isTrivial()) {
         // If new expression did not specify value-initialization, then there
         // is no initialization.
@@ -962,13 +973,11 @@ static void EmitNewInitializer(CodeGenFunction &CGF, const CXXNewExpr *E,
           EmitZeroMemSet(CGF, ElementType, NewPtr, AllocSizeWithoutCookie);
           return;
         }
-
-        RequiresZeroInitialization = true;
       }
 
       CGF.EmitCXXAggrConstructorCall(Ctor, NumElements, NewPtr,
                                      CCE->arg_begin(),  CCE->arg_end(),
-                                     RequiresZeroInitialization);
+                                     CCE->requiresZeroInitialization());
       return;
     } else if (Init && isa<ImplicitValueInitExpr>(Init) &&
                CGF.CGM.getTypes().isZeroInitializable(ElementType)) {
@@ -1230,8 +1239,7 @@ llvm::Value *CodeGenFunction::EmitCXXNewExpr(const CXXNewExpr *E) {
   llvm::BasicBlock *contBB = 0;
 
   llvm::Value *allocation = RV.getScalarVal();
-  unsigned AS =
-    cast<llvm::PointerType>(allocation->getType())->getAddressSpace();
+  unsigned AS = allocation->getType()->getPointerAddressSpace();
 
   // The null-check means that the initializer is conditionally
   // evaluated.
@@ -1377,8 +1385,14 @@ static void EmitObjectDelete(CodeGenFunction &CGF,
         if (UseGlobalDelete) {
           // If we're supposed to call the global delete, make sure we do so
           // even if the destructor throws.
+
+          // Derive the complete-object pointer, which is what we need
+          // to pass to the deallocation function.
+          llvm::Value *completePtr =
+            CGF.CGM.getCXXABI().adjustToCompleteObject(CGF, Ptr, ElementType);
+
           CGF.EHStack.pushCleanup<CallObjectDelete>(NormalAndEHCleanup,
-                                                    Ptr, OperatorDelete, 
+                                                    completePtr, OperatorDelete,
                                                     ElementType);
         }
         
@@ -1390,8 +1404,9 @@ static void EmitObjectDelete(CodeGenFunction &CGF,
           = CGF.BuildVirtualCall(Dtor, 
                                  UseGlobalDelete? Dtor_Complete : Dtor_Deleting,
                                  Ptr, Ty);
-        CGF.EmitCXXMemberCall(Dtor, Callee, ReturnValueSlot(), Ptr, /*VTT=*/0,
-                              0, 0);
+        // FIXME: Provide a source location here.
+        CGF.EmitCXXMemberCall(Dtor, SourceLocation(), Callee, ReturnValueSlot(),
+                              Ptr, /*VTT=*/0, 0, 0);
 
         if (UseGlobalDelete) {
           CGF.PopCleanupBlock();
diff --git a/lib/CodeGen/CGExprComplex.cpp b/lib/CodeGen/CGExprComplex.cpp
index 0233745..66b6f86 100644
--- a/lib/CodeGen/CGExprComplex.cpp
+++ b/lib/CodeGen/CGExprComplex.cpp
@@ -427,6 +427,7 @@ ComplexPairTy ComplexExprEmitter::EmitCast(CastExpr::CastKind CK, Expr *Op,
   case CK_ARCReclaimReturnedObject:
   case CK_ARCExtendBlockObject:
   case CK_CopyAndAutoreleaseBlockObject:
+  case CK_BuiltinFnToFnPtr:
     llvm_unreachable("invalid cast kind for complex value");
 
   case CK_FloatingRealToComplex:
@@ -640,7 +641,7 @@ EmitCompoundAssign(const CompoundAssignOperator *E,
   LValue LV = EmitCompoundAssignLValue(E, Func, Val);
 
   // The result of an assignment in C is the assigned r-value.
-  if (!CGF.getContext().getLangOpts().CPlusPlus)
+  if (!CGF.getLangOpts().CPlusPlus)
     return Val;
 
   // If the lvalue is non-volatile, return the computed value of the assignment.
@@ -675,7 +676,7 @@ ComplexPairTy ComplexExprEmitter::VisitBinAssign(const BinaryOperator *E) {
   LValue LV = EmitBinAssignLValue(E, Val);
 
   // The result of an assignment in C is the assigned r-value.
-  if (!CGF.getContext().getLangOpts().CPlusPlus)
+  if (!CGF.getLangOpts().CPlusPlus)
     return Val;
 
   // If the lvalue is non-volatile, return the computed value of the assignment.
diff --git a/lib/CodeGen/CGExprConstant.cpp b/lib/CodeGen/CGExprConstant.cpp
index a17a436..206f74a 100644
--- a/lib/CodeGen/CGExprConstant.cpp
+++ b/lib/CodeGen/CGExprConstant.cpp
@@ -24,7 +24,7 @@
 #include "llvm/Constants.h"
 #include "llvm/Function.h"
 #include "llvm/GlobalVariable.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 using namespace clang;
 using namespace CodeGen;
 
@@ -79,12 +79,12 @@ private:
   CharUnits getAlignment(const llvm::Constant *C) const {
     if (Packed)  return CharUnits::One();
     return CharUnits::fromQuantity(
-        CGM.getTargetData().getABITypeAlignment(C->getType()));
+        CGM.getDataLayout().getABITypeAlignment(C->getType()));
   }
 
   CharUnits getSizeInChars(const llvm::Constant *C) const {
     return CharUnits::fromQuantity(
-        CGM.getTargetData().getTypeAllocSize(C->getType()));
+        CGM.getDataLayout().getTypeAllocSize(C->getType()));
   }
 };
 
@@ -204,7 +204,7 @@ void ConstStructBuilder::AppendBitField(const FieldDecl *Field,
     if (!FitsCompletelyInPreviousByte) {
       unsigned NewFieldWidth = FieldSize - BitsInPreviousByte;
 
-      if (CGM.getTargetData().isBigEndian()) {
+      if (CGM.getDataLayout().isBigEndian()) {
         Tmp = Tmp.lshr(NewFieldWidth);
         Tmp = Tmp.trunc(BitsInPreviousByte);
 
@@ -220,7 +220,7 @@ void ConstStructBuilder::AppendBitField(const FieldDecl *Field,
     }
 
     Tmp = Tmp.zext(CharWidth);
-    if (CGM.getTargetData().isBigEndian()) {
+    if (CGM.getDataLayout().isBigEndian()) {
       if (FitsCompletelyInPreviousByte)
         Tmp = Tmp.shl(BitsInPreviousByte - FieldValue.getBitWidth());
     } else {
@@ -269,7 +269,7 @@ void ConstStructBuilder::AppendBitField(const FieldDecl *Field,
   while (FieldValue.getBitWidth() > CharWidth) {
     llvm::APInt Tmp;
 
-    if (CGM.getTargetData().isBigEndian()) {
+    if (CGM.getDataLayout().isBigEndian()) {
       // We want the high bits.
       Tmp = 
         FieldValue.lshr(FieldValue.getBitWidth() - CharWidth).trunc(CharWidth);
@@ -292,7 +292,7 @@ void ConstStructBuilder::AppendBitField(const FieldDecl *Field,
          "Should not have more than a byte left!");
 
   if (FieldValue.getBitWidth() < CharWidth) {
-    if (CGM.getTargetData().isBigEndian()) {
+    if (CGM.getDataLayout().isBigEndian()) {
       unsigned BitWidth = FieldValue.getBitWidth();
 
       FieldValue = FieldValue.zext(CharWidth) << (CharWidth - BitWidth);
@@ -337,7 +337,7 @@ void ConstStructBuilder::ConvertStructToPacked() {
     llvm::Constant *C = Elements[i];
 
     CharUnits ElementAlign = CharUnits::fromQuantity(
-      CGM.getTargetData().getABITypeAlignment(C->getType()));
+      CGM.getDataLayout().getABITypeAlignment(C->getType()));
     CharUnits AlignedElementOffsetInChars =
       ElementOffsetInChars.RoundUpToAlignment(ElementAlign);
 
@@ -379,7 +379,7 @@ bool ConstStructBuilder::Build(InitListExpr *ILE) {
   unsigned FieldNo = 0;
   unsigned ElementNo = 0;
   const FieldDecl *LastFD = 0;
-  bool IsMsStruct = RD->hasAttr<MsStructAttr>();
+  bool IsMsStruct = RD->isMsStruct(CGM.getContext());
   
   for (RecordDecl::field_iterator Field = RD->field_begin(),
        FieldEnd = RD->field_end(); Field != FieldEnd; ++Field, ++FieldNo) {
@@ -478,7 +478,7 @@ void ConstStructBuilder::Build(const APValue &Val, const RecordDecl *RD,
 
   unsigned FieldNo = 0;
   const FieldDecl *LastFD = 0;
-  bool IsMsStruct = RD->hasAttr<MsStructAttr>();
+  bool IsMsStruct = RD->isMsStruct(CGM.getContext());
   uint64_t OffsetBits = CGM.getContext().toBits(Offset);
 
   for (RecordDecl::field_iterator Field = RD->field_begin(),
@@ -665,8 +665,8 @@ public:
       SmallVector<llvm::Type*, 2> Types;
       Elts.push_back(C);
       Types.push_back(C->getType());
-      unsigned CurSize = CGM.getTargetData().getTypeAllocSize(C->getType());
-      unsigned TotalSize = CGM.getTargetData().getTypeAllocSize(destType);
+      unsigned CurSize = CGM.getDataLayout().getTypeAllocSize(C->getType());
+      unsigned TotalSize = CGM.getDataLayout().getTypeAllocSize(destType);
 
       assert(CurSize <= TotalSize && "Union size mismatch!");
       if (unsigned NumPadBytes = TotalSize - CurSize) {
@@ -691,6 +691,9 @@ public:
 
     case CK_Dependent: llvm_unreachable("saw dependent cast!");
 
+    case CK_BuiltinFnToFnPtr:
+      llvm_unreachable("builtin functions are handled elsewhere");
+
     case CK_ReinterpretMemberPointer:
     case CK_DerivedToBaseMemberPointer:
     case CK_BaseToDerivedMemberPointer:
@@ -811,11 +814,7 @@ public:
     return llvm::ConstantArray::get(AType, Elts);
   }
 
-  llvm::Constant *EmitStructInitialization(InitListExpr *ILE) {
-    return ConstStructBuilder::BuildStruct(CGM, CGF, ILE);
-  }
-
-  llvm::Constant *EmitUnionInitialization(InitListExpr *ILE) {
+  llvm::Constant *EmitRecordInitialization(InitListExpr *ILE) {
     return ConstStructBuilder::BuildStruct(CGM, CGF, ILE);
   }
 
@@ -828,10 +827,7 @@ public:
       return EmitArrayInitialization(ILE);
 
     if (ILE->getType()->isRecordType())
-      return EmitStructInitialization(ILE);
-
-    if (ILE->getType()->isUnionType())
-      return EmitUnionInitialization(ILE);
+      return EmitRecordInitialization(ILE);
 
     return 0;
   }
@@ -999,6 +995,9 @@ public:
         T = Typeid->getExprOperand()->getType();
       return CGM.GetAddrOfRTTIDescriptor(T);
     }
+    case Expr::CXXUuidofExprClass: {
+      return CGM.GetAddrOfUuidDescriptor(cast<CXXUuidofExpr>(E));
+    }
     }
 
     return 0;
diff --git a/lib/CodeGen/CGExprScalar.cpp b/lib/CodeGen/CGExprScalar.cpp
index 1cccafe..b429b1d 100644
--- a/lib/CodeGen/CGExprScalar.cpp
+++ b/lib/CodeGen/CGExprScalar.cpp
@@ -28,7 +28,7 @@
 #include "llvm/Intrinsics.h"
 #include "llvm/Module.h"
 #include "llvm/Support/CFG.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include <cstdarg>
 
 using namespace clang;
@@ -45,6 +45,7 @@ struct BinOpInfo {
   Value *RHS;
   QualType Ty;  // Computation Type.
   BinaryOperator::Opcode Opcode; // Opcode of BinOp to perform
+  bool FPContractable;
   const Expr *E;      // Entire expr, for error unsupported.  May not be binop.
 };
 
@@ -80,7 +81,11 @@ public:
 
   llvm::Type *ConvertType(QualType T) { return CGF.ConvertType(T); }
   LValue EmitLValue(const Expr *E) { return CGF.EmitLValue(E); }
-  LValue EmitCheckedLValue(const Expr *E) { return CGF.EmitCheckedLValue(E); }
+  LValue EmitCheckedLValue(const Expr *E, CodeGenFunction::TypeCheckKind TCK) {
+    return CGF.EmitCheckedLValue(E, TCK);
+  }
+
+  void EmitBinOpCheck(Value *Check, const BinOpInfo &Info);
 
   Value *EmitLoadOfLValue(LValue LV) {
     return CGF.EmitLoadOfLValue(LV).getScalarVal();
@@ -90,13 +95,19 @@ public:
   /// value l-value, this method emits the address of the l-value, then loads
   /// and returns the result.
   Value *EmitLoadOfLValue(const Expr *E) {
-    return EmitLoadOfLValue(EmitCheckedLValue(E));
+    return EmitLoadOfLValue(EmitCheckedLValue(E, CodeGenFunction::TCK_Load));
   }
 
   /// EmitConversionToBool - Convert the specified expression value to a
   /// boolean (i1) truth value.  This is equivalent to "Val != 0".
   Value *EmitConversionToBool(Value *Src, QualType DstTy);
 
+  /// \brief Emit a check that a conversion to or from a floating-point type
+  /// does not overflow.
+  void EmitFloatConversionCheck(Value *OrigSrc, QualType OrigSrcType,
+                                Value *Src, QualType SrcType,
+                                QualType DstType, llvm::Type *DstTy);
+
   /// EmitScalarConversion - Emit a conversion from the specified type to the
   /// specified destination type, both of which are LLVM scalar types.
   Value *EmitScalarConversion(Value *Src, QualType SrcTy, QualType DstTy);
@@ -391,34 +402,26 @@ public:
   // Binary Operators.
   Value *EmitMul(const BinOpInfo &Ops) {
     if (Ops.Ty->isSignedIntegerOrEnumerationType()) {
-      switch (CGF.getContext().getLangOpts().getSignedOverflowBehavior()) {
-      case LangOptions::SOB_Undefined:
-        return Builder.CreateNSWMul(Ops.LHS, Ops.RHS, "mul");
+      switch (CGF.getLangOpts().getSignedOverflowBehavior()) {
       case LangOptions::SOB_Defined:
         return Builder.CreateMul(Ops.LHS, Ops.RHS, "mul");
+      case LangOptions::SOB_Undefined:
+        if (!CGF.getLangOpts().SanitizeSignedIntegerOverflow)
+          return Builder.CreateNSWMul(Ops.LHS, Ops.RHS, "mul");
+        // Fall through.
       case LangOptions::SOB_Trapping:
         return EmitOverflowCheckedBinOp(Ops);
       }
     }
-    
+
     if (Ops.LHS->getType()->isFPOrFPVectorTy())
       return Builder.CreateFMul(Ops.LHS, Ops.RHS, "mul");
     return Builder.CreateMul(Ops.LHS, Ops.RHS, "mul");
   }
-  bool isTrapvOverflowBehavior() {
-    return CGF.getContext().getLangOpts().getSignedOverflowBehavior() 
-               == LangOptions::SOB_Trapping; 
-  }
   /// Create a binary op that checks for overflow.
   /// Currently only supports +, - and *.
   Value *EmitOverflowCheckedBinOp(const BinOpInfo &Ops);
-  // Emit the overflow BB when -ftrapv option is activated. 
-  void EmitOverflowBB(llvm::BasicBlock *overflowBB) {
-    Builder.SetInsertPoint(overflowBB);
-    llvm::Function *Trap = CGF.CGM.getIntrinsic(llvm::Intrinsic::trap);
-    Builder.CreateCall(Trap);
-    Builder.CreateUnreachable();
-  }
+
   // Check for undefined division and modulus behaviors.
   void EmitUndefinedBehaviorIntegerDivAndRemCheck(const BinOpInfo &Ops, 
                                                   llvm::Value *Zero,bool isDiv);
@@ -537,6 +540,110 @@ Value *ScalarExprEmitter::EmitConversionToBool(Value *Src, QualType SrcType) {
   return EmitPointerToBoolConversion(Src);
 }
 
+void ScalarExprEmitter::EmitFloatConversionCheck(Value *OrigSrc,
+                                                 QualType OrigSrcType,
+                                                 Value *Src, QualType SrcType,
+                                                 QualType DstType,
+                                                 llvm::Type *DstTy) {
+  using llvm::APFloat;
+  using llvm::APSInt;
+
+  llvm::Type *SrcTy = Src->getType();
+
+  llvm::Value *Check = 0;
+  if (llvm::IntegerType *IntTy = dyn_cast<llvm::IntegerType>(SrcTy)) {
+    // Integer to floating-point. This can fail for unsigned short -> __half
+    // or unsigned __int128 -> float.
+    assert(DstType->isFloatingType());
+    bool SrcIsUnsigned = OrigSrcType->isUnsignedIntegerOrEnumerationType();
+
+    APFloat LargestFloat =
+      APFloat::getLargest(CGF.getContext().getFloatTypeSemantics(DstType));
+    APSInt LargestInt(IntTy->getBitWidth(), SrcIsUnsigned);
+
+    bool IsExact;
+    if (LargestFloat.convertToInteger(LargestInt, APFloat::rmTowardZero,
+                                      &IsExact) != APFloat::opOK)
+      // The range of representable values of this floating point type includes
+      // all values of this integer type. Don't need an overflow check.
+      return;
+
+    llvm::Value *Max = llvm::ConstantInt::get(VMContext, LargestInt);
+    if (SrcIsUnsigned)
+      Check = Builder.CreateICmpULE(Src, Max);
+    else {
+      llvm::Value *Min = llvm::ConstantInt::get(VMContext, -LargestInt);
+      llvm::Value *GE = Builder.CreateICmpSGE(Src, Min);
+      llvm::Value *LE = Builder.CreateICmpSLE(Src, Max);
+      Check = Builder.CreateAnd(GE, LE);
+    }
+  } else {
+    // Floating-point to integer or floating-point to floating-point. This has
+    // undefined behavior if the source is +-Inf, NaN, or doesn't fit into the
+    // destination type.
+    const llvm::fltSemantics &SrcSema =
+      CGF.getContext().getFloatTypeSemantics(OrigSrcType);
+    APFloat MaxSrc(SrcSema, APFloat::uninitialized);
+    APFloat MinSrc(SrcSema, APFloat::uninitialized);
+
+    if (isa<llvm::IntegerType>(DstTy)) {
+      unsigned Width = CGF.getContext().getIntWidth(DstType);
+      bool Unsigned = DstType->isUnsignedIntegerOrEnumerationType();
+
+      APSInt Min = APSInt::getMinValue(Width, Unsigned);
+      if (MinSrc.convertFromAPInt(Min, !Unsigned, APFloat::rmTowardZero) &
+          APFloat::opOverflow)
+        // Don't need an overflow check for lower bound. Just check for
+        // -Inf/NaN.
+        MinSrc = APFloat::getLargest(SrcSema, true);
+
+      APSInt Max = APSInt::getMaxValue(Width, Unsigned);
+      if (MaxSrc.convertFromAPInt(Max, !Unsigned, APFloat::rmTowardZero) &
+          APFloat::opOverflow)
+        // Don't need an overflow check for upper bound. Just check for
+        // +Inf/NaN.
+        MaxSrc = APFloat::getLargest(SrcSema, false);
+    } else {
+      const llvm::fltSemantics &DstSema =
+        CGF.getContext().getFloatTypeSemantics(DstType);
+      bool IsInexact;
+
+      MinSrc = APFloat::getLargest(DstSema, true);
+      if (MinSrc.convert(SrcSema, APFloat::rmTowardZero, &IsInexact) &
+          APFloat::opOverflow)
+        MinSrc = APFloat::getLargest(SrcSema, true);
+
+      MaxSrc = APFloat::getLargest(DstSema, false);
+      if (MaxSrc.convert(SrcSema, APFloat::rmTowardZero, &IsInexact) &
+          APFloat::opOverflow)
+        MaxSrc = APFloat::getLargest(SrcSema, false);
+    }
+
+    // If we're converting from __half, convert the range to float to match
+    // the type of src.
+    if (OrigSrcType->isHalfType()) {
+      const llvm::fltSemantics &Sema =
+        CGF.getContext().getFloatTypeSemantics(SrcType);
+      bool IsInexact;
+      MinSrc.convert(Sema, APFloat::rmTowardZero, &IsInexact);
+      MaxSrc.convert(Sema, APFloat::rmTowardZero, &IsInexact);
+    }
+
+    llvm::Value *GE =
+      Builder.CreateFCmpOGE(Src, llvm::ConstantFP::get(VMContext, MinSrc));
+    llvm::Value *LE =
+      Builder.CreateFCmpOLE(Src, llvm::ConstantFP::get(VMContext, MaxSrc));
+    Check = Builder.CreateAnd(GE, LE);
+  }
+
+  // FIXME: Provide a SourceLocation.
+  llvm::Constant *StaticArgs[] = {
+    CGF.EmitCheckTypeDescriptor(OrigSrcType),
+    CGF.EmitCheckTypeDescriptor(DstType)
+  };
+  CGF.EmitCheck(Check, "float_cast_overflow", StaticArgs, OrigSrc);
+}
+
 /// EmitScalarConversion - Emit a conversion from the specified type to the
 /// specified destination type, both of which are LLVM scalar types.
 Value *ScalarExprEmitter::EmitScalarConversion(Value *Src, QualType SrcType,
@@ -547,6 +654,8 @@ Value *ScalarExprEmitter::EmitScalarConversion(Value *Src, QualType SrcType,
 
   if (DstType->isVoidType()) return 0;
 
+  llvm::Value *OrigSrc = Src;
+  QualType OrigSrcType = SrcType;
   llvm::Type *SrcTy = Src->getType();
 
   // Floating casts might be a bit special: if we're doing casts to / from half
@@ -620,6 +729,12 @@ Value *ScalarExprEmitter::EmitScalarConversion(Value *Src, QualType SrcType,
   Value *Res = NULL;
   llvm::Type *ResTy = DstTy;
 
+  // An overflowing conversion has undefined behavior if either the source type
+  // or the destination type is a floating-point type.
+  if (CGF.getLangOpts().SanitizeFloatCastOverflow &&
+      (OrigSrcType->isFloatingType() || DstType->isFloatingType()))
+    EmitFloatConversionCheck(OrigSrc, OrigSrcType, Src, SrcType, DstType, DstTy);
+
   // Cast to half via float
   if (DstType->isHalfType())
     DstTy = CGF.FloatTy;
@@ -686,6 +801,54 @@ Value *ScalarExprEmitter::EmitNullValue(QualType Ty) {
   return llvm::Constant::getNullValue(ConvertType(Ty));
 }
 
+/// \brief Emit a sanitization check for the given "binary" operation (which
+/// might actually be a unary increment which has been lowered to a binary
+/// operation). The check passes if \p Check, which is an \c i1, is \c true.
+void ScalarExprEmitter::EmitBinOpCheck(Value *Check, const BinOpInfo &Info) {
+  StringRef CheckName;
+  llvm::SmallVector<llvm::Constant *, 4> StaticData;
+  llvm::SmallVector<llvm::Value *, 2> DynamicData;
+
+  BinaryOperatorKind Opcode = Info.Opcode;
+  if (BinaryOperator::isCompoundAssignmentOp(Opcode))
+    Opcode = BinaryOperator::getOpForCompoundAssignment(Opcode);
+
+  StaticData.push_back(CGF.EmitCheckSourceLocation(Info.E->getExprLoc()));
+  const UnaryOperator *UO = dyn_cast<UnaryOperator>(Info.E);
+  if (UO && UO->getOpcode() == UO_Minus) {
+    CheckName = "negate_overflow";
+    StaticData.push_back(CGF.EmitCheckTypeDescriptor(UO->getType()));
+    DynamicData.push_back(Info.RHS);
+  } else {
+    if (BinaryOperator::isShiftOp(Opcode)) {
+      // Shift LHS negative or too large, or RHS out of bounds.
+      CheckName = "shift_out_of_bounds";
+      const BinaryOperator *BO = cast<BinaryOperator>(Info.E);
+      StaticData.push_back(
+        CGF.EmitCheckTypeDescriptor(BO->getLHS()->getType()));
+      StaticData.push_back(
+        CGF.EmitCheckTypeDescriptor(BO->getRHS()->getType()));
+    } else if (Opcode == BO_Div || Opcode == BO_Rem) {
+      // Divide or modulo by zero, or signed overflow (eg INT_MAX / -1).
+      CheckName = "divrem_overflow";
+      StaticData.push_back(CGF.EmitCheckTypeDescriptor(Info.E->getType()));
+    } else {
+      // Signed arithmetic overflow (+, -, *).
+      switch (Opcode) {
+      case BO_Add: CheckName = "add_overflow"; break;
+      case BO_Sub: CheckName = "sub_overflow"; break;
+      case BO_Mul: CheckName = "mul_overflow"; break;
+      default: llvm_unreachable("unexpected opcode for bin op check");
+      }
+      StaticData.push_back(CGF.EmitCheckTypeDescriptor(Info.E->getType()));
+    }
+    DynamicData.push_back(Info.LHS);
+    DynamicData.push_back(Info.RHS);
+  }
+
+  CGF.EmitCheck(Check, CheckName, StaticData, DynamicData);
+}
+
 //===----------------------------------------------------------------------===//
 //                            Visitor Methods
 //===----------------------------------------------------------------------===//
@@ -802,7 +965,8 @@ Value *ScalarExprEmitter::VisitMemberExpr(MemberExpr *E) {
   // debug info size.
   CGDebugInfo *DI = CGF.getDebugInfo();
   if (DI &&
-      CGF.CGM.getCodeGenOpts().DebugInfo == CodeGenOptions::LimitedDebugInfo) {
+      CGF.CGM.getCodeGenOpts().getDebugInfo()
+        == CodeGenOptions::LimitedDebugInfo) {
     QualType PQTy = E->getBase()->IgnoreParenImpCasts()->getType();
     if (const PointerType * PTy = dyn_cast<PointerType>(PQTy))
       if (FieldDecl *M = dyn_cast<FieldDecl>(E->getMemberDecl()))
@@ -1032,7 +1196,9 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
   // are in the same order as in the CastKind enum.
   switch (Kind) {
   case CK_Dependent: llvm_unreachable("dependent cast kind in IR gen!");
-      
+  case CK_BuiltinFnToFnPtr:
+    llvm_unreachable("builtin functions are handled elsewhere");
+
   case CK_LValueBitCast: 
   case CK_ObjCObjectLValueCast: {
     Value *V = EmitLValue(E).getAddress();
@@ -1055,19 +1221,18 @@ Value *ScalarExprEmitter::VisitCastExpr(CastExpr *CE) {
     return Visit(const_cast<Expr*>(E));
 
   case CK_BaseToDerived: {
-    const CXXRecordDecl *DerivedClassDecl = 
-      DestTy->getCXXRecordDeclForPointerType();
-    
-    return CGF.GetAddressOfDerivedClass(Visit(E), DerivedClassDecl, 
+    const CXXRecordDecl *DerivedClassDecl = DestTy->getPointeeCXXRecordDecl();
+    assert(DerivedClassDecl && "BaseToDerived arg isn't a C++ object pointer!");
+
+    return CGF.GetAddressOfDerivedClass(Visit(E), DerivedClassDecl,
                                         CE->path_begin(), CE->path_end(),
                                         ShouldNullCheckClassCastValue(CE));
   }
   case CK_UncheckedDerivedToBase:
   case CK_DerivedToBase: {
-    const RecordType *DerivedClassTy = 
-      E->getType()->getAs<PointerType>()->getPointeeType()->getAs<RecordType>();
-    CXXRecordDecl *DerivedClassDecl = 
-      cast<CXXRecordDecl>(DerivedClassTy->getDecl());
+    const CXXRecordDecl *DerivedClassDecl =
+      E->getType()->getPointeeCXXRecordDecl();
+    assert(DerivedClassDecl && "DerivedToBase arg isn't a C++ object pointer!");
 
     return CGF.GetAddressOfBaseClass(Visit(E), DerivedClassDecl, 
                                      CE->path_begin(), CE->path_end(),
@@ -1248,17 +1413,20 @@ llvm::Value *ScalarExprEmitter::
 EmitAddConsiderOverflowBehavior(const UnaryOperator *E,
                                 llvm::Value *InVal,
                                 llvm::Value *NextVal, bool IsInc) {
-  switch (CGF.getContext().getLangOpts().getSignedOverflowBehavior()) {
-  case LangOptions::SOB_Undefined:
-    return Builder.CreateNSWAdd(InVal, NextVal, IsInc ? "inc" : "dec");
+  switch (CGF.getLangOpts().getSignedOverflowBehavior()) {
   case LangOptions::SOB_Defined:
     return Builder.CreateAdd(InVal, NextVal, IsInc ? "inc" : "dec");
+  case LangOptions::SOB_Undefined:
+    if (!CGF.getLangOpts().SanitizeSignedIntegerOverflow)
+      return Builder.CreateNSWAdd(InVal, NextVal, IsInc ? "inc" : "dec");
+    // Fall through.
   case LangOptions::SOB_Trapping:
     BinOpInfo BinOp;
     BinOp.LHS = InVal;
     BinOp.RHS = NextVal;
     BinOp.Ty = E->getType();
     BinOp.Opcode = BO_Add;
+    BinOp.FPContractable = false;
     BinOp.E = E;
     return EmitOverflowCheckedBinOp(BinOp);
   }
@@ -1300,7 +1468,7 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
   // Most common case by far: integer increment.
   } else if (type->isIntegerType()) {
 
-    llvm::Value *amt = llvm::ConstantInt::get(value->getType(), amount);
+    llvm::Value *amt = llvm::ConstantInt::get(value->getType(), amount, true);
 
     // Note that signed integer inc/dec with width less than int can't
     // overflow because of promotion rules; we're just eliding a few steps here.
@@ -1320,7 +1488,7 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
           = CGF.getContext().getAsVariableArrayType(type)) {
       llvm::Value *numElts = CGF.getVLASize(vla).first;
       if (!isInc) numElts = Builder.CreateNSWNeg(numElts, "vla.negsize");
-      if (CGF.getContext().getLangOpts().isSignedOverflowDefined())
+      if (CGF.getLangOpts().isSignedOverflowDefined())
         value = Builder.CreateGEP(value, numElts, "vla.inc");
       else
         value = Builder.CreateInBoundsGEP(value, numElts, "vla.inc");
@@ -1330,7 +1498,7 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
       llvm::Value *amt = Builder.getInt32(amount);
 
       value = CGF.EmitCastToVoidPtr(value);
-      if (CGF.getContext().getLangOpts().isSignedOverflowDefined())
+      if (CGF.getLangOpts().isSignedOverflowDefined())
         value = Builder.CreateGEP(value, amt, "incdec.funcptr");
       else
         value = Builder.CreateInBoundsGEP(value, amt, "incdec.funcptr");
@@ -1339,7 +1507,7 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
     // For everything else, we can just do a simple increment.
     } else {
       llvm::Value *amt = Builder.getInt32(amount);
-      if (CGF.getContext().getLangOpts().isSignedOverflowDefined())
+      if (CGF.getLangOpts().isSignedOverflowDefined())
         value = Builder.CreateGEP(value, amt, "incdec.ptr");
       else
         value = Builder.CreateInBoundsGEP(value, amt, "incdec.ptr");
@@ -1400,7 +1568,7 @@ ScalarExprEmitter::EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
     llvm::Value *sizeValue =
       llvm::ConstantInt::get(CGF.SizeTy, size.getQuantity());
 
-    if (CGF.getContext().getLangOpts().isSignedOverflowDefined())
+    if (CGF.getLangOpts().isSignedOverflowDefined())
       value = Builder.CreateGEP(value, sizeValue, "incdec.objptr");
     else
       value = Builder.CreateInBoundsGEP(value, sizeValue, "incdec.objptr");
@@ -1444,6 +1612,7 @@ Value *ScalarExprEmitter::VisitUnaryMinus(const UnaryOperator *E) {
     BinOp.LHS = llvm::Constant::getNullValue(BinOp.RHS->getType());
   BinOp.Ty = E->getType();
   BinOp.Opcode = BO_Sub;
+  BinOp.FPContractable = false;
   BinOp.E = E;
   return EmitSub(BinOp);
 }
@@ -1652,6 +1821,7 @@ BinOpInfo ScalarExprEmitter::EmitBinOps(const BinaryOperator *E) {
   Result.RHS = Visit(E->getRHS());
   Result.Ty  = E->getType();
   Result.Opcode = E->getOpcode();
+  Result.FPContractable = E->isFPContractable();
   Result.E = E;
   return Result;
 }
@@ -1678,9 +1848,10 @@ LValue ScalarExprEmitter::EmitCompoundAssignLValue(
   OpInfo.RHS = Visit(E->getRHS());
   OpInfo.Ty = E->getComputationResultType();
   OpInfo.Opcode = E->getOpcode();
+  OpInfo.FPContractable = false;
   OpInfo.E = E;
   // Load/convert the LHS.
-  LValue LHSLV = EmitCheckedLValue(E->getLHS());
+  LValue LHSLV = EmitCheckedLValue(E->getLHS(), CodeGenFunction::TCK_Store);
   OpInfo.LHS = EmitLoadOfLValue(LHSLV);
 
   llvm::PHINode *atomicPHI = 0;
@@ -1740,7 +1911,7 @@ Value *ScalarExprEmitter::EmitCompoundAssign(const CompoundAssignOperator *E,
     return 0;
 
   // The result of an assignment in C is the assigned r-value.
-  if (!CGF.getContext().getLangOpts().CPlusPlus)
+  if (!CGF.getLangOpts().CPlusPlus)
     return RHS;
 
   // If the lvalue is non-volatile, return the computed value of the assignment.
@@ -1752,56 +1923,44 @@ Value *ScalarExprEmitter::EmitCompoundAssign(const CompoundAssignOperator *E,
 }
 
 void ScalarExprEmitter::EmitUndefinedBehaviorIntegerDivAndRemCheck(
-     					    const BinOpInfo &Ops, 
-				     	    llvm::Value *Zero, bool isDiv) {
-  llvm::Function::iterator insertPt = Builder.GetInsertBlock();
-  llvm::BasicBlock *contBB =
-    CGF.createBasicBlock(isDiv ? "div.cont" : "rem.cont", CGF.CurFn,
-                         llvm::next(insertPt));
-  llvm::BasicBlock *overflowBB = CGF.createBasicBlock("overflow", CGF.CurFn);
+    const BinOpInfo &Ops, llvm::Value *Zero, bool isDiv) {
+  llvm::Value *Cond = 0;
+
+  if (CGF.getLangOpts().SanitizeDivideByZero)
+    Cond = Builder.CreateICmpNE(Ops.RHS, Zero);
 
-  llvm::IntegerType *Ty = cast<llvm::IntegerType>(Zero->getType());
+  if (CGF.getLangOpts().SanitizeSignedIntegerOverflow &&
+      Ops.Ty->hasSignedIntegerRepresentation()) {
+    llvm::IntegerType *Ty = cast<llvm::IntegerType>(Zero->getType());
 
-  if (Ops.Ty->hasSignedIntegerRepresentation()) {
     llvm::Value *IntMin =
       Builder.getInt(llvm::APInt::getSignedMinValue(Ty->getBitWidth()));
     llvm::Value *NegOne = llvm::ConstantInt::get(Ty, -1ULL);
 
-    llvm::Value *Cond1 = Builder.CreateICmpEQ(Ops.RHS, Zero);
-    llvm::Value *LHSCmp = Builder.CreateICmpEQ(Ops.LHS, IntMin);
-    llvm::Value *RHSCmp = Builder.CreateICmpEQ(Ops.RHS, NegOne);
-    llvm::Value *Cond2 = Builder.CreateAnd(LHSCmp, RHSCmp, "and");
-    Builder.CreateCondBr(Builder.CreateOr(Cond1, Cond2, "or"), 
-                         overflowBB, contBB);
-  } else {
-    CGF.Builder.CreateCondBr(Builder.CreateICmpEQ(Ops.RHS, Zero), 
-                             overflowBB, contBB);
+    llvm::Value *LHSCmp = Builder.CreateICmpNE(Ops.LHS, IntMin);
+    llvm::Value *RHSCmp = Builder.CreateICmpNE(Ops.RHS, NegOne);
+    llvm::Value *Overflow = Builder.CreateOr(LHSCmp, RHSCmp, "or");
+    Cond = Cond ? Builder.CreateAnd(Cond, Overflow, "and") : Overflow;
   }
-  EmitOverflowBB(overflowBB);
-  Builder.SetInsertPoint(contBB);
+
+  if (Cond)
+    EmitBinOpCheck(Cond, Ops);
 }
 
 Value *ScalarExprEmitter::EmitDiv(const BinOpInfo &Ops) {
-  if (isTrapvOverflowBehavior()) { 
+  if (CGF.getLangOpts().SanitizeDivideByZero ||
+      CGF.getLangOpts().SanitizeSignedIntegerOverflow) {
     llvm::Value *Zero = llvm::Constant::getNullValue(ConvertType(Ops.Ty));
 
     if (Ops.Ty->isIntegerType())
       EmitUndefinedBehaviorIntegerDivAndRemCheck(Ops, Zero, true);
-    else if (Ops.Ty->isRealFloatingType()) {
-      llvm::Function::iterator insertPt = Builder.GetInsertBlock();
-      llvm::BasicBlock *DivCont = CGF.createBasicBlock("div.cont", CGF.CurFn,
-                                                       llvm::next(insertPt));
-      llvm::BasicBlock *overflowBB = CGF.createBasicBlock("overflow",
-                                                          CGF.CurFn);
-      CGF.Builder.CreateCondBr(Builder.CreateFCmpOEQ(Ops.RHS, Zero), 
-                               overflowBB, DivCont);
-      EmitOverflowBB(overflowBB);
-      Builder.SetInsertPoint(DivCont);
-    }
+    else if (CGF.getLangOpts().SanitizeDivideByZero &&
+             Ops.Ty->isRealFloatingType())
+      EmitBinOpCheck(Builder.CreateFCmpUNE(Ops.RHS, Zero), Ops);
   }
   if (Ops.LHS->getType()->isFPOrFPVectorTy()) {
     llvm::Value *Val = Builder.CreateFDiv(Ops.LHS, Ops.RHS, "div");
-    if (CGF.getContext().getLangOpts().OpenCL) {
+    if (CGF.getLangOpts().OpenCL) {
       // OpenCL 1.1 7.4: minimum accuracy of single precision / is 2.5ulp
       llvm::Type *ValTy = Val->getType();
       if (ValTy->isFloatTy() ||
@@ -1819,7 +1978,7 @@ Value *ScalarExprEmitter::EmitDiv(const BinOpInfo &Ops) {
 
 Value *ScalarExprEmitter::EmitRem(const BinOpInfo &Ops) {
   // Rem in C can't be a floating point type: C99 6.5.5p2.
-  if (isTrapvOverflowBehavior()) {
+  if (CGF.getLangOpts().SanitizeDivideByZero) {
     llvm::Value *Zero = llvm::Constant::getNullValue(ConvertType(Ops.Ty));
 
     if (Ops.Ty->isIntegerType()) 
@@ -1866,6 +2025,19 @@ Value *ScalarExprEmitter::EmitOverflowCheckedBinOp(const BinOpInfo &Ops) {
   Value *result = Builder.CreateExtractValue(resultAndOverflow, 0);
   Value *overflow = Builder.CreateExtractValue(resultAndOverflow, 1);
 
+  // Handle overflow with llvm.trap if no custom handler has been specified.
+  const std::string *handlerName =
+    &CGF.getLangOpts().OverflowHandler;
+  if (handlerName->empty()) {
+    // If the signed-integer-overflow sanitizer is enabled, emit a call to its
+    // runtime. Otherwise, this is a -ftrapv check, so just emit a trap.
+    if (CGF.getLangOpts().SanitizeSignedIntegerOverflow)
+      EmitBinOpCheck(Builder.CreateNot(overflow), Ops);
+    else
+      CGF.EmitTrapvCheck(Builder.CreateNot(overflow));
+    return result;
+  }
+
   // Branch in case of overflow.
   llvm::BasicBlock *initialBB = Builder.GetInsertBlock();
   llvm::Function::iterator insertPt = initialBB;
@@ -1875,15 +2047,6 @@ Value *ScalarExprEmitter::EmitOverflowCheckedBinOp(const BinOpInfo &Ops) {
 
   Builder.CreateCondBr(overflow, overflowBB, continueBB);
 
-  // Handle overflow with llvm.trap.
-  const std::string *handlerName = 
-    &CGF.getContext().getLangOpts().OverflowHandler;
-  if (handlerName->empty()) {
-    EmitOverflowBB(overflowBB);
-    Builder.SetInsertPoint(continueBB);
-    return result;
-  }
-
   // If an overflow handler is set, then we want to call it and then use its
   // result, if it returns.
   Builder.SetInsertPoint(overflowBB);
@@ -2001,24 +2164,106 @@ static Value *emitPointerArithmetic(CodeGenFunction &CGF,
   return CGF.Builder.CreateInBoundsGEP(pointer, index, "add.ptr");
 }
 
+// Construct an fmuladd intrinsic to represent a fused mul-add of MulOp and
+// Addend. Use negMul and negAdd to negate the first operand of the Mul or
+// the add operand respectively. This allows fmuladd to represent a*b-c, or
+// c-a*b. Patterns in LLVM should catch the negated forms and translate them to
+// efficient operations.
+static Value* buildFMulAdd(llvm::BinaryOperator *MulOp, Value *Addend,
+                           const CodeGenFunction &CGF, CGBuilderTy &Builder,
+                           bool negMul, bool negAdd) {
+  assert(!(negMul && negAdd) && "Only one of negMul and negAdd should be set.");
+ 
+  Value *MulOp0 = MulOp->getOperand(0);
+  Value *MulOp1 = MulOp->getOperand(1);
+  if (negMul) {
+    MulOp0 =
+      Builder.CreateFSub(
+        llvm::ConstantFP::getZeroValueForNegation(MulOp0->getType()), MulOp0,
+        "neg");
+  } else if (negAdd) {
+    Addend =
+      Builder.CreateFSub(
+        llvm::ConstantFP::getZeroValueForNegation(Addend->getType()), Addend,
+        "neg");
+  }
+
+  Value *FMulAdd =
+    Builder.CreateCall3(
+      CGF.CGM.getIntrinsic(llvm::Intrinsic::fmuladd, Addend->getType()),
+                           MulOp0, MulOp1, Addend);
+   MulOp->eraseFromParent();
+
+   return FMulAdd;
+}
+
+// Check whether it would be legal to emit an fmuladd intrinsic call to
+// represent op and if so, build the fmuladd.
+//
+// Checks that (a) the operation is fusable, and (b) -ffp-contract=on.
+// Does NOT check the type of the operation - it's assumed that this function
+// will be called from contexts where it's known that the type is contractable.
+static Value* tryEmitFMulAdd(const BinOpInfo &op, 
+                         const CodeGenFunction &CGF, CGBuilderTy &Builder,
+                         bool isSub=false) {
+
+  assert((op.Opcode == BO_Add || op.Opcode == BO_AddAssign ||
+          op.Opcode == BO_Sub || op.Opcode == BO_SubAssign) &&
+         "Only fadd/fsub can be the root of an fmuladd.");
+
+  // Check whether this op is marked as fusable.
+  if (!op.FPContractable)
+    return 0;
+
+  // Check whether -ffp-contract=on. (If -ffp-contract=off/fast, fusing is
+  // either disabled, or handled entirely by the LLVM backend).
+  if (CGF.getLangOpts().getFPContractMode() != LangOptions::FPC_On)
+    return 0;
+
+  // We have a potentially fusable op. Look for a mul on one of the operands.
+  if (llvm::BinaryOperator* LHSBinOp = dyn_cast<llvm::BinaryOperator>(op.LHS)) {
+    if (LHSBinOp->getOpcode() == llvm::Instruction::FMul) {
+      assert(LHSBinOp->getNumUses() == 0 &&
+             "Operations with multiple uses shouldn't be contracted.");
+      return buildFMulAdd(LHSBinOp, op.RHS, CGF, Builder, false, isSub);
+    }
+  } else if (llvm::BinaryOperator* RHSBinOp =
+               dyn_cast<llvm::BinaryOperator>(op.RHS)) {
+    if (RHSBinOp->getOpcode() == llvm::Instruction::FMul) {
+      assert(RHSBinOp->getNumUses() == 0 &&
+             "Operations with multiple uses shouldn't be contracted.");
+      return buildFMulAdd(RHSBinOp, op.LHS, CGF, Builder, isSub, false);
+    }
+  }
+
+  return 0;
+}
+
 Value *ScalarExprEmitter::EmitAdd(const BinOpInfo &op) {
   if (op.LHS->getType()->isPointerTy() ||
       op.RHS->getType()->isPointerTy())
     return emitPointerArithmetic(CGF, op, /*subtraction*/ false);
 
   if (op.Ty->isSignedIntegerOrEnumerationType()) {
-    switch (CGF.getContext().getLangOpts().getSignedOverflowBehavior()) {
-    case LangOptions::SOB_Undefined:
-      return Builder.CreateNSWAdd(op.LHS, op.RHS, "add");
+    switch (CGF.getLangOpts().getSignedOverflowBehavior()) {
     case LangOptions::SOB_Defined:
       return Builder.CreateAdd(op.LHS, op.RHS, "add");
+    case LangOptions::SOB_Undefined:
+      if (!CGF.getLangOpts().SanitizeSignedIntegerOverflow)
+        return Builder.CreateNSWAdd(op.LHS, op.RHS, "add");
+      // Fall through.
     case LangOptions::SOB_Trapping:
       return EmitOverflowCheckedBinOp(op);
     }
   }
     
-  if (op.LHS->getType()->isFPOrFPVectorTy())
+  if (op.LHS->getType()->isFPOrFPVectorTy()) {
+    // Try to form an fmuladd.
+    if (Value *FMulAdd = tryEmitFMulAdd(op, CGF, Builder))
+      return FMulAdd;
+
     return Builder.CreateFAdd(op.LHS, op.RHS, "add");
+  }
 
   return Builder.CreateAdd(op.LHS, op.RHS, "add");
 }
@@ -2027,18 +2272,24 @@ Value *ScalarExprEmitter::EmitSub(const BinOpInfo &op) {
   // The LHS is always a pointer if either side is.
   if (!op.LHS->getType()->isPointerTy()) {
     if (op.Ty->isSignedIntegerOrEnumerationType()) {
-      switch (CGF.getContext().getLangOpts().getSignedOverflowBehavior()) {
-      case LangOptions::SOB_Undefined:
-        return Builder.CreateNSWSub(op.LHS, op.RHS, "sub");
+      switch (CGF.getLangOpts().getSignedOverflowBehavior()) {
       case LangOptions::SOB_Defined:
         return Builder.CreateSub(op.LHS, op.RHS, "sub");
+      case LangOptions::SOB_Undefined:
+        if (!CGF.getLangOpts().SanitizeSignedIntegerOverflow)
+          return Builder.CreateNSWSub(op.LHS, op.RHS, "sub");
+        // Fall through.
       case LangOptions::SOB_Trapping:
         return EmitOverflowCheckedBinOp(op);
       }
     }
     
-    if (op.LHS->getType()->isFPOrFPVectorTy())
+    if (op.LHS->getType()->isFPOrFPVectorTy()) {
+      // Try to form an fmuladd.
+      if (Value *FMulAdd = tryEmitFMulAdd(op, CGF, Builder, true))
+        return FMulAdd;
       return Builder.CreateFSub(op.LHS, op.RHS, "sub");
+    }
 
     return Builder.CreateSub(op.LHS, op.RHS, "sub");
   }
@@ -2108,14 +2359,34 @@ Value *ScalarExprEmitter::EmitShl(const BinOpInfo &Ops) {
   if (Ops.LHS->getType() != RHS->getType())
     RHS = Builder.CreateIntCast(RHS, Ops.LHS->getType(), false, "sh_prom");
 
-  if (CGF.CatchUndefined 
-      && isa<llvm::IntegerType>(Ops.LHS->getType())) {
+  if (CGF.getLangOpts().SanitizeShift &&
+      isa<llvm::IntegerType>(Ops.LHS->getType())) {
     unsigned Width = cast<llvm::IntegerType>(Ops.LHS->getType())->getBitWidth();
-    llvm::BasicBlock *Cont = CGF.createBasicBlock("cont");
-    CGF.Builder.CreateCondBr(Builder.CreateICmpULT(RHS,
-                                 llvm::ConstantInt::get(RHS->getType(), Width)),
-                             Cont, CGF.getTrapBB());
-    CGF.EmitBlock(Cont);
+    llvm::Value *WidthMinusOne =
+      llvm::ConstantInt::get(RHS->getType(), Width - 1);
+    // FIXME: Emit the branching explicitly rather than emitting the check
+    // twice.
+    EmitBinOpCheck(Builder.CreateICmpULE(RHS, WidthMinusOne), Ops);
+
+    if (Ops.Ty->hasSignedIntegerRepresentation()) {
+      // Check whether we are shifting any non-zero bits off the top of the
+      // integer.
+      llvm::Value *BitsShiftedOff =
+        Builder.CreateLShr(Ops.LHS,
+                           Builder.CreateSub(WidthMinusOne, RHS, "shl.zeros",
+                                             /*NUW*/true, /*NSW*/true),
+                           "shl.check");
+      if (CGF.getLangOpts().CPlusPlus) {
+        // In C99, we are not permitted to shift a 1 bit into the sign bit.
+        // Under C++11's rules, shifting a 1 bit into the sign bit is
+        // OK, but shifting a 1 bit out of it is not. (C89 and C++03 don't
+        // define signed left shifts, so we use the C99 and C++11 rules there).
+        llvm::Value *One = llvm::ConstantInt::get(BitsShiftedOff->getType(), 1);
+        BitsShiftedOff = Builder.CreateLShr(BitsShiftedOff, One);
+      }
+      llvm::Value *Zero = llvm::ConstantInt::get(BitsShiftedOff->getType(), 0);
+      EmitBinOpCheck(Builder.CreateICmpEQ(BitsShiftedOff, Zero), Ops);
+    }
   }
 
   return Builder.CreateShl(Ops.LHS, RHS, "shl");
@@ -2128,14 +2399,11 @@ Value *ScalarExprEmitter::EmitShr(const BinOpInfo &Ops) {
   if (Ops.LHS->getType() != RHS->getType())
     RHS = Builder.CreateIntCast(RHS, Ops.LHS->getType(), false, "sh_prom");
 
-  if (CGF.CatchUndefined 
-      && isa<llvm::IntegerType>(Ops.LHS->getType())) {
+  if (CGF.getLangOpts().SanitizeShift &&
+      isa<llvm::IntegerType>(Ops.LHS->getType())) {
     unsigned Width = cast<llvm::IntegerType>(Ops.LHS->getType())->getBitWidth();
-    llvm::BasicBlock *Cont = CGF.createBasicBlock("cont");
-    CGF.Builder.CreateCondBr(Builder.CreateICmpULT(RHS,
-                                 llvm::ConstantInt::get(RHS->getType(), Width)),
-                             Cont, CGF.getTrapBB());
-    CGF.EmitBlock(Cont);
+    llvm::Value *WidthVal = llvm::ConstantInt::get(RHS->getType(), Width);
+    EmitBinOpCheck(Builder.CreateICmpULT(RHS, WidthVal), Ops);
   }
 
   if (Ops.Ty->hasUnsignedIntegerRepresentation())
@@ -2326,7 +2594,7 @@ Value *ScalarExprEmitter::VisitBinAssign(const BinaryOperator *E) {
 
   case Qualifiers::OCL_Weak:
     RHS = Visit(E->getRHS());
-    LHS = EmitCheckedLValue(E->getLHS());    
+    LHS = EmitCheckedLValue(E->getLHS(), CodeGenFunction::TCK_Store);
     RHS = CGF.EmitARCStoreWeak(LHS.getAddress(), RHS, Ignore);
     break;
 
@@ -2336,7 +2604,7 @@ Value *ScalarExprEmitter::VisitBinAssign(const BinaryOperator *E) {
     // __block variables need to have the rhs evaluated first, plus
     // this should improve codegen just a little.
     RHS = Visit(E->getRHS());
-    LHS = EmitCheckedLValue(E->getLHS());
+    LHS = EmitCheckedLValue(E->getLHS(), CodeGenFunction::TCK_Store);
 
     // Store the value into the LHS.  Bit-fields are handled specially
     // because the result is altered by the store, i.e., [C99 6.5.16p1]
@@ -2353,7 +2621,7 @@ Value *ScalarExprEmitter::VisitBinAssign(const BinaryOperator *E) {
     return 0;
 
   // The result of an assignment in C is the assigned r-value.
-  if (!CGF.getContext().getLangOpts().CPlusPlus)
+  if (!CGF.getLangOpts().CPlusPlus)
     return RHS;
 
   // If the lvalue is non-volatile, return the computed value of the assignment.
@@ -2567,7 +2835,7 @@ VisitAbstractConditionalOperator(const AbstractConditionalOperator *E) {
 
   // OpenCL: If the condition is a vector, we can treat this condition like
   // the select function.
-  if (CGF.getContext().getLangOpts().OpenCL 
+  if (CGF.getLangOpts().OpenCL 
       && condExpr->getType()->isVectorType()) {
     llvm::Value *CondV = CGF.EmitScalarExpr(condExpr);
     llvm::Value *LHS = Visit(lhsExpr);
diff --git a/lib/CodeGen/CGObjC.cpp b/lib/CodeGen/CGObjC.cpp
index 4ac172d..c90e4ec 100644
--- a/lib/CodeGen/CGObjC.cpp
+++ b/lib/CodeGen/CGObjC.cpp
@@ -21,7 +21,7 @@
 #include "clang/AST/StmtObjC.h"
 #include "clang/Basic/Diagnostic.h"
 #include "llvm/ADT/STLExtras.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/InlineAsm.h"
 using namespace clang;
 using namespace CodeGen;
@@ -440,8 +440,8 @@ void CodeGenFunction::StartObjCMethod(const ObjCMethodDecl *OMD,
                                       SourceLocation StartLoc) {
   FunctionArgList args;
   // Check if we should generate debug info for this method.
-  if (CGM.getModuleDebugInfo() && !OMD->hasAttr<NoDebugAttr>())
-    DebugInfo = CGM.getModuleDebugInfo();
+  if (!OMD->hasAttr<NoDebugAttr>())
+    maybeInitializeDebugInfo();
 
   llvm::Function *Fn = CGM.getObjCRuntime().GenerateMethod(OMD, CD);
 
@@ -613,7 +613,16 @@ PropertyImplStrategy::PropertyImplStrategy(CodeGenModule &CGM,
     // which translates to objc_storeStrong.  This isn't required, but
     // it's slightly nicer.
     } else if (CGM.getLangOpts().ObjCAutoRefCount && !IsAtomic) {
-      Kind = Expression;
+      // Using standard expression emission for the setter is only
+      // acceptable if the ivar is __strong, which won't be true if
+      // the property is annotated with __attribute__((NSObject)).
+      // TODO: falling all the way back to objc_setProperty here is
+      // just laziness, though;  we could still use objc_storeStrong
+      // if we hacked it right.
+      if (ivarType.getObjCLifetime() == Qualifiers::OCL_Strong)
+        Kind = Expression;
+      else
+        Kind = SetPropertyAndExpressionGet;
       return;
 
     // Otherwise, we need to at least use setProperty.  However, if
@@ -801,6 +810,10 @@ CodeGenFunction::generateObjCGetterBody(const ObjCImplementationDecl *classImpl,
   PropertyImplStrategy strategy(CGM, propImpl);
   switch (strategy.getKind()) {
   case PropertyImplStrategy::Native: {
+    // We don't need to do anything for a zero-size struct.
+    if (strategy.getIvarSize().isZero())
+      return;
+
     LValue LV = EmitLValueForIvar(TypeOfSelfObject(), LoadObjCSelf(), ivar, 0);
 
     // Currently, all atomic accesses have to be through integer
@@ -1032,12 +1045,7 @@ static bool hasTrivialSetExpr(const ObjCPropertyImplDecl *PID) {
 static bool UseOptimizedSetter(CodeGenModule &CGM) {
   if (CGM.getLangOpts().getGC() != LangOptions::NonGC)
     return false;
-  const TargetInfo &Target = CGM.getContext().getTargetInfo();
-
-  if (Target.getPlatformName() != "macosx")
-    return false;
-
-  return Target.getPlatformMinVersion() >= VersionTuple(10, 8);
+  return CGM.getLangOpts().ObjCRuntime.hasOptimizedSetter();
 }
 
 void
@@ -1064,6 +1072,10 @@ CodeGenFunction::generateObjCSetterBody(const ObjCImplementationDecl *classImpl,
   PropertyImplStrategy strategy(CGM, propImpl);
   switch (strategy.getKind()) {
   case PropertyImplStrategy::Native: {
+    // We don't need to do anything for a zero-size struct.
+    if (strategy.getIvarSize().isZero())
+      return;
+
     llvm::Value *argAddr = LocalDeclMap[*setterMethod->param_begin()];
 
     LValue ivarLValue =
@@ -1097,7 +1109,7 @@ CodeGenFunction::generateObjCSetterBody(const ObjCImplementationDecl *classImpl,
     llvm::Value *setOptimizedPropertyFn = 0;
     llvm::Value *setPropertyFn = 0;
     if (UseOptimizedSetter(CGM)) {
-      // 10.8 code and GC is off
+      // 10.8 and iOS 6.0 code and GC is off
       setOptimizedPropertyFn = 
         CGM.getObjCRuntime()
            .GetOptimizedPropertySetFunction(strategy.isAtomic(),
@@ -1209,7 +1221,7 @@ CodeGenFunction::generateObjCSetterBody(const ObjCImplementationDecl *classImpl,
 
   BinaryOperator assign(&ivarRef, finalArg, BO_Assign,
                         ivarRef.getType(), VK_RValue, OK_Ordinary,
-                        SourceLocation());
+                        SourceLocation(), false);
   EmitStmt(&assign);
 }
 
@@ -1697,11 +1709,11 @@ static llvm::Constant *createARCRuntimeFunction(CodeGenModule &CGM,
   // references to the runtime support library.  We don't really
   // permit this to fail, but we need a particular relocation style.
   if (llvm::Function *f = dyn_cast<llvm::Function>(fn)) {
-    if (!CGM.getLangOpts().ObjCRuntime.hasARC())
+    if (!CGM.getLangOpts().ObjCRuntime.hasNativeARC())
       f->setLinkage(llvm::Function::ExternalWeakLinkage);
     // set nonlazybind attribute for these APIs for performance.
     if (fnName == "objc_retain" || fnName  == "objc_release")
-      f->addFnAttr(llvm::Attribute::NonLazyBind);
+      f->addFnAttr(llvm::Attributes::NonLazyBind);
   }
 
   return fn;
@@ -1945,6 +1957,28 @@ void CodeGenFunction::EmitARCRelease(llvm::Value *value, bool precise) {
   }
 }
 
+/// Destroy a __strong variable.
+///
+/// At -O0, emit a call to store 'null' into the address;
+/// instrumenting tools prefer this because the address is exposed,
+/// but it's relatively cumbersome to optimize.
+///
+/// At -O1 and above, just load and call objc_release.
+///
+///   call void \@objc_storeStrong(i8** %addr, i8* null)
+void CodeGenFunction::EmitARCDestroyStrong(llvm::Value *addr, bool precise) {
+  if (CGM.getCodeGenOpts().OptimizationLevel == 0) {
+    llvm::PointerType *addrTy = cast<llvm::PointerType>(addr->getType());
+    llvm::Value *null = llvm::ConstantPointerNull::get(
+                          cast<llvm::PointerType>(addrTy->getElementType()));
+    EmitARCStoreStrongCall(addr, null, /*ignored*/ true);
+    return;
+  }
+
+  llvm::Value *value = Builder.CreateLoad(addr);
+  EmitARCRelease(value, precise);
+}
+
 /// Store into a strong object.  Always calls this:
 ///   call void \@objc_storeStrong(i8** %addr, i8* %value)
 llvm::Value *CodeGenFunction::EmitARCStoreStrongCall(llvm::Value *addr,
@@ -2218,15 +2252,13 @@ void CodeGenFunction::EmitObjCMRRAutoreleasePoolPop(llvm::Value *Arg) {
 void CodeGenFunction::destroyARCStrongPrecise(CodeGenFunction &CGF,
                                               llvm::Value *addr,
                                               QualType type) {
-  llvm::Value *ptr = CGF.Builder.CreateLoad(addr, "strongdestroy");
-  CGF.EmitARCRelease(ptr, /*precise*/ true);
+  CGF.EmitARCDestroyStrong(addr, /*precise*/ true);
 }
 
 void CodeGenFunction::destroyARCStrongImprecise(CodeGenFunction &CGF,
                                                 llvm::Value *addr,
                                                 QualType type) {
-  llvm::Value *ptr = CGF.Builder.CreateLoad(addr, "strongdestroy");
-  CGF.EmitARCRelease(ptr, /*precise*/ false);  
+  CGF.EmitARCDestroyStrong(addr, /*precise*/ false);
 }
 
 void CodeGenFunction::destroyARCWeak(CodeGenFunction &CGF,
@@ -2730,7 +2762,7 @@ void CodeGenFunction::EmitObjCAutoreleasePoolStmt(
 
   // Keep track of the current cleanup stack depth.
   RunCleanupsScope Scope(*this);
-  if (CGM.getLangOpts().ObjCRuntime.hasARC()) {
+  if (CGM.getLangOpts().ObjCRuntime.hasNativeARC()) {
     llvm::Value *token = EmitObjCAutoreleasePoolPush();
     EHStack.pushCleanup<CallObjCAutoreleasePoolObject>(NormalCleanup, token);
   } else {
@@ -2826,9 +2858,8 @@ CodeGenFunction::GenerateObjCAtomicSetterCopyHelperFunction(
                            "__assign_helper_atomic_property_",
                            &CGM.getModule());
   
-  if (CGM.getModuleDebugInfo())
-    DebugInfo = CGM.getModuleDebugInfo();
-  
+  // Initialize debug info if needed.
+  maybeInitializeDebugInfo();
   
   StartFunction(FD, C.VoidTy, Fn, FI, args, SourceLocation());
   
@@ -2845,8 +2876,8 @@ CodeGenFunction::GenerateObjCAtomicSetterCopyHelperFunction(
   Expr *Args[2] = { &DST, &SRC };
   CallExpr *CalleeExp = cast<CallExpr>(PID->getSetterCXXAssignment());
   CXXOperatorCallExpr TheCall(C, OO_Equal, CalleeExp->getCallee(),
-                              Args, 2, DestTy->getPointeeType(), 
-                              VK_LValue, SourceLocation());
+                              Args, DestTy->getPointeeType(),
+                              VK_LValue, SourceLocation(), false);
   
   EmitStmt(&TheCall);
 
@@ -2912,9 +2943,8 @@ CodeGenFunction::GenerateObjCAtomicGetterCopyHelperFunction(
   llvm::Function::Create(LTy, llvm::GlobalValue::InternalLinkage,
                          "__copy_helper_atomic_property_", &CGM.getModule());
   
-  if (CGM.getModuleDebugInfo())
-    DebugInfo = CGM.getModuleDebugInfo();
-  
+  // Initialize debug info if needed.
+  maybeInitializeDebugInfo();
   
   StartFunction(FD, C.VoidTy, Fn, FI, args, SourceLocation());
   
@@ -2940,7 +2970,7 @@ CodeGenFunction::GenerateObjCAtomicGetterCopyHelperFunction(
     CXXConstructExpr::Create(C, Ty, SourceLocation(),
                              CXXConstExpr->getConstructor(),
                              CXXConstExpr->isElidable(),
-                             &ConstructorArgs[0], ConstructorArgs.size(),
+                             ConstructorArgs,
                              CXXConstExpr->hadMultipleCandidates(),
                              CXXConstExpr->isListInitialization(),
                              CXXConstExpr->requiresZeroInitialization(),
diff --git a/lib/CodeGen/CGObjCGNU.cpp b/lib/CodeGen/CGObjCGNU.cpp
index 6d129d0..68d234d 100644
--- a/lib/CodeGen/CGObjCGNU.cpp
+++ b/lib/CodeGen/CGObjCGNU.cpp
@@ -33,7 +33,7 @@
 #include "llvm/ADT/StringMap.h"
 #include "llvm/Support/CallSite.h"
 #include "llvm/Support/Compiler.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 
 #include <cstdarg>
 
@@ -224,6 +224,25 @@ protected:
     llvm::ArrayType *ArrayTy = llvm::ArrayType::get(Ty, V.size());
     return MakeGlobal(ArrayTy, V, Name, linkage);
   }
+  /// Returns a property name and encoding string.
+  llvm::Constant *MakePropertyEncodingString(const ObjCPropertyDecl *PD,
+                                             const Decl *Container) {
+    ObjCRuntime R = CGM.getLangOpts().ObjCRuntime;
+    if ((R.getKind() == ObjCRuntime::GNUstep) &&
+        (R.getVersion() >= VersionTuple(1, 6))) {
+      std::string NameAndAttributes;
+      std::string TypeStr;
+      CGM.getContext().getObjCEncodingForPropertyDecl(PD, Container, TypeStr);
+      NameAndAttributes += '\0';
+      NameAndAttributes += TypeStr.length() + 3;
+      NameAndAttributes += TypeStr;
+      NameAndAttributes += '\0';
+      NameAndAttributes += PD->getNameAsString();
+      return llvm::ConstantExpr::getGetElementPtr(
+          CGM.GetAddrOfConstantString(NameAndAttributes), Zeros);
+    }
+    return MakeConstantString(PD->getNameAsString());
+  }
   /// Ensures that the value has the required type, by inserting a bitcast if
   /// required.  This function lets us avoid inserting bitcasts that are
   /// redundant.
@@ -514,7 +533,10 @@ public:
                                              const CGBlockInfo &blockInfo) {
     return NULLPtr;
   }
-  
+  virtual llvm::Constant *BuildRCBlockLayout(CodeGenModule &CGM,
+                                             const CGBlockInfo &blockInfo) {
+    return NULLPtr;
+  }
   virtual llvm::GlobalVariable *GetClassGlobal(const std::string &Name) {
     return 0;
   }
@@ -578,6 +600,8 @@ class CGObjCGNUstep : public CGObjCGNU {
     /// Type of an slot structure pointer.  This is returned by the various
     /// lookup functions.
     llvm::Type *SlotTy;
+  public:
+    virtual llvm::Constant *GetEHType(QualType T);
   protected:
     virtual llvm::Value *LookupIMP(CodeGenFunction &CGF,
                                    llvm::Value *&Receiver,
@@ -653,11 +677,40 @@ class CGObjCGNUstep : public CGObjCGNU {
     }
 };
 
-/// The ObjFW runtime, which closely follows the GCC runtime's
-/// compiler ABI.  Support here is due to Jonathan Schleifer, the
-/// ObjFW maintainer.
-class CGObjCObjFW : public CGObjCGCC {
-  /// Emit class references unconditionally as direct symbol references.
+/// Support for the ObjFW runtime. Support here is due to
+/// Jonathan Schleifer <js@webkeks.org>, the ObjFW maintainer.
+class CGObjCObjFW: public CGObjCGNU {
+protected:
+  /// The GCC ABI message lookup function.  Returns an IMP pointing to the
+  /// method implementation for this message.
+  LazyRuntimeFunction MsgLookupFn;
+  /// The GCC ABI superclass message lookup function.  Takes a pointer to a
+  /// structure describing the receiver and the class, and a selector as
+  /// arguments.  Returns the IMP for the corresponding method.
+  LazyRuntimeFunction MsgLookupSuperFn;
+
+  virtual llvm::Value *LookupIMP(CodeGenFunction &CGF,
+                                 llvm::Value *&Receiver,
+                                 llvm::Value *cmd,
+                                 llvm::MDNode *node) {
+    CGBuilderTy &Builder = CGF.Builder;
+    llvm::Value *args[] = {
+            EnforceType(Builder, Receiver, IdTy),
+            EnforceType(Builder, cmd, SelectorTy) };
+    llvm::CallSite imp = CGF.EmitCallOrInvoke(MsgLookupFn, args);
+    imp->setMetadata(msgSendMDKind, node);
+    return imp.getInstruction();
+  }
+
+  virtual llvm::Value *LookupIMPSuper(CodeGenFunction &CGF,
+                                      llvm::Value *ObjCSuper,
+                                      llvm::Value *cmd) {
+      CGBuilderTy &Builder = CGF.Builder;
+      llvm::Value *lookupArgs[] = {EnforceType(Builder, ObjCSuper,
+          PtrToObjCSuperTy), cmd};
+      return Builder.CreateCall(MsgLookupSuperFn, lookupArgs);
+    }
+
   virtual llvm::Value *GetClassNamed(CGBuilderTy &Builder,
                                      const std::string &Name, bool isWeak) {
     if (isWeak)
@@ -678,7 +731,13 @@ class CGObjCObjFW : public CGObjCGCC {
   }
 
 public:
-  CGObjCObjFW(CodeGenModule &Mod): CGObjCGCC(Mod) {}
+  CGObjCObjFW(CodeGenModule &Mod): CGObjCGNU(Mod, 9, 3) {
+    // IMP objc_msg_lookup(id, SEL);
+    MsgLookupFn.init(&CGM, "objc_msg_lookup", IMPTy, IdTy, SelectorTy, NULL);
+    // IMP objc_msg_lookup_super(struct objc_super*, SEL);
+    MsgLookupSuperFn.init(&CGM, "objc_msg_lookup_super", IMPTy,
+                          PtrToObjCSuperTy, SelectorTy, NULL);
+  }
 };
 } // end anonymous namespace
 
@@ -909,29 +968,30 @@ llvm::Value *CGObjCGNU::GetSelector(CGBuilderTy &Builder, const ObjCMethodDecl
 }
 
 llvm::Constant *CGObjCGNU::GetEHType(QualType T) {
-  if (!CGM.getLangOpts().CPlusPlus) {
-      if (T->isObjCIdType()
-          || T->isObjCQualifiedIdType()) {
-        // With the old ABI, there was only one kind of catchall, which broke
-        // foreign exceptions.  With the new ABI, we use __objc_id_typeinfo as
-        // a pointer indicating object catchalls, and NULL to indicate real
-        // catchalls
-        if (CGM.getLangOpts().ObjCRuntime.isNonFragile()) {
-          return MakeConstantString("@id");
-        } else {
-          return 0;
-        }
-      }
-
-      // All other types should be Objective-C interface pointer types.
-      const ObjCObjectPointerType *OPT =
-        T->getAs<ObjCObjectPointerType>();
-      assert(OPT && "Invalid @catch type.");
-      const ObjCInterfaceDecl *IDecl =
-        OPT->getObjectType()->getInterface();
-      assert(IDecl && "Invalid @catch type.");
-      return MakeConstantString(IDecl->getIdentifier()->getName());
+  if (T->isObjCIdType() || T->isObjCQualifiedIdType()) {
+    // With the old ABI, there was only one kind of catchall, which broke
+    // foreign exceptions.  With the new ABI, we use __objc_id_typeinfo as
+    // a pointer indicating object catchalls, and NULL to indicate real
+    // catchalls
+    if (CGM.getLangOpts().ObjCRuntime.isNonFragile()) {
+      return MakeConstantString("@id");
+    } else {
+      return 0;
+    }
   }
+
+  // All other types should be Objective-C interface pointer types.
+  const ObjCObjectPointerType *OPT = T->getAs<ObjCObjectPointerType>();
+  assert(OPT && "Invalid @catch type.");
+  const ObjCInterfaceDecl *IDecl = OPT->getObjectType()->getInterface();
+  assert(IDecl && "Invalid @catch type.");
+  return MakeConstantString(IDecl->getIdentifier()->getName());
+}
+
+llvm::Constant *CGObjCGNUstep::GetEHType(QualType T) {
+  if (!CGM.getLangOpts().CPlusPlus)
+    return CGObjCGNU::GetEHType(T);
+
   // For Objective-C++, we want to provide the ability to catch both C++ and
   // Objective-C objects in the same function.
 
@@ -1436,7 +1496,7 @@ llvm::Constant *CGObjCGNU::GenerateClassStructure(
   Elements.push_back(Zero);
   Elements.push_back(llvm::ConstantInt::get(LongTy, info));
   if (isMeta) {
-    llvm::TargetData td(&TheModule);
+    llvm::DataLayout td(&TheModule);
     Elements.push_back(
         llvm::ConstantInt::get(LongTy,
                                td.getTypeSizeInBits(ClassTy) /
@@ -1595,13 +1655,13 @@ void CGObjCGNU::GenerateProtocol(const ObjCProtocolDecl *PD) {
     std::string TypeStr;
     Context.getObjCEncodingForMethodDecl(*iter, TypeStr);
     if ((*iter)->getImplementationControl() == ObjCMethodDecl::Optional) {
-      InstanceMethodNames.push_back(
-          MakeConstantString((*iter)->getSelector().getAsString()));
-      InstanceMethodTypes.push_back(MakeConstantString(TypeStr));
-    } else {
       OptionalInstanceMethodNames.push_back(
           MakeConstantString((*iter)->getSelector().getAsString()));
       OptionalInstanceMethodTypes.push_back(MakeConstantString(TypeStr));
+    } else {
+      InstanceMethodNames.push_back(
+          MakeConstantString((*iter)->getSelector().getAsString()));
+      InstanceMethodTypes.push_back(MakeConstantString(TypeStr));
     }
   }
   // Collect information about class methods:
@@ -1615,13 +1675,13 @@ void CGObjCGNU::GenerateProtocol(const ObjCProtocolDecl *PD) {
     std::string TypeStr;
     Context.getObjCEncodingForMethodDecl((*iter),TypeStr);
     if ((*iter)->getImplementationControl() == ObjCMethodDecl::Optional) {
-      ClassMethodNames.push_back(
-          MakeConstantString((*iter)->getSelector().getAsString()));
-      ClassMethodTypes.push_back(MakeConstantString(TypeStr));
-    } else {
       OptionalClassMethodNames.push_back(
           MakeConstantString((*iter)->getSelector().getAsString()));
       OptionalClassMethodTypes.push_back(MakeConstantString(TypeStr));
+    } else {
+      ClassMethodNames.push_back(
+          MakeConstantString((*iter)->getSelector().getAsString()));
+      ClassMethodTypes.push_back(MakeConstantString(TypeStr));
     }
   }
 
@@ -1656,7 +1716,9 @@ void CGObjCGNU::GenerateProtocol(const ObjCProtocolDecl *PD) {
     std::vector<llvm::Constant*> Fields;
     ObjCPropertyDecl *property = *iter;
 
-    Fields.push_back(MakeConstantString(property->getNameAsString()));
+
+    Fields.push_back(MakePropertyEncodingString(property, PD));
+
     Fields.push_back(llvm::ConstantInt::get(Int8Ty,
                 property->getPropertyAttributes()));
     Fields.push_back(llvm::ConstantInt::get(Int8Ty, 0));
@@ -1909,7 +1971,7 @@ llvm::Constant *CGObjCGNU::GeneratePropertyList(const ObjCImplementationDecl *OI
     bool isSynthesized = (propertyImpl->getPropertyImplementation() == 
         ObjCPropertyImplDecl::Synthesize);
 
-    Fields.push_back(MakeConstantString(property->getNameAsString()));
+    Fields.push_back(MakePropertyEncodingString(property, OID));
     Fields.push_back(llvm::ConstantInt::get(Int8Ty,
                 property->getPropertyAttributes()));
     Fields.push_back(llvm::ConstantInt::get(Int8Ty, isSynthesized));
@@ -2011,7 +2073,7 @@ void CGObjCGNU::GenerateClass(const ObjCImplementationDecl *OID) {
     Context.getASTObjCInterfaceLayout(SuperClassDecl).getSize().getQuantity();
   // For non-fragile ivars, set the instance size to 0 - {the size of just this
   // class}.  The runtime will then set this to the correct value on load.
-  if (CGM.getContext().getLangOpts().ObjCRuntime.isNonFragile()) {
+  if (CGM.getLangOpts().ObjCRuntime.isNonFragile()) {
     instanceSize = 0 - (instanceSize - superInstanceSize);
   }
 
@@ -2026,7 +2088,7 @@ void CGObjCGNU::GenerateClass(const ObjCImplementationDecl *OID) {
       // Get the offset
       uint64_t BaseOffset = ComputeIvarBaseOffset(CGM, OID, IVD);
       uint64_t Offset = BaseOffset;
-      if (CGM.getContext().getLangOpts().ObjCRuntime.isNonFragile()) {
+      if (CGM.getLangOpts().ObjCRuntime.isNonFragile()) {
         Offset = BaseOffset - superInstanceSize;
       }
       llvm::Constant *OffsetValue = llvm::ConstantInt::get(IntTy, Offset);
@@ -2334,7 +2396,7 @@ llvm::Function *CGObjCGNU::ModuleInitFunction() {
   // Runtime version, used for ABI compatibility checking.
   Elements.push_back(llvm::ConstantInt::get(LongTy, RuntimeVersion));
   // sizeof(ModuleTy)
-  llvm::TargetData td(&TheModule);
+  llvm::DataLayout td(&TheModule);
   Elements.push_back(
     llvm::ConstantInt::get(LongTy,
                            td.getTypeSizeInBits(ModuleTy) /
@@ -2488,7 +2550,7 @@ void CGObjCGNU::EmitTryStmt(CodeGenFunction &CGF,
   // Unlike the Apple non-fragile runtimes, which also uses
   // unwind-based zero cost exceptions, the GNU Objective C runtime's
   // EH support isn't a veneer over C++ EH.  Instead, exception
-  // objects are created by __objc_exception_throw and destroyed by
+  // objects are created by objc_exception_throw and destroyed by
   // the personality function; this avoids the need for bracketing
   // catch handlers with calls to __blah_begin_catch/__blah_end_catch
   // (or even _Unwind_DeleteException), but probably doesn't
@@ -2513,7 +2575,9 @@ void CGObjCGNU::EmitThrowStmt(CodeGenFunction &CGF,
     ExceptionAsObject = CGF.ObjCEHValueStack.back();
   }
   ExceptionAsObject = CGF.Builder.CreateBitCast(ExceptionAsObject, IdTy);
-  CGF.EmitCallOrInvoke(ExceptionThrowFn, ExceptionAsObject);
+  llvm::CallSite Throw =
+      CGF.EmitCallOrInvoke(ExceptionThrowFn, ExceptionAsObject);
+  Throw.setDoesNotReturn();
   CGF.Builder.CreateUnreachable();
   CGF.Builder.ClearInsertionPoint();
 }
diff --git a/lib/CodeGen/CGObjCMac.cpp b/lib/CodeGen/CGObjCMac.cpp
index ef802a3..2203f01 100644
--- a/lib/CodeGen/CGObjCMac.cpp
+++ b/lib/CodeGen/CGObjCMac.cpp
@@ -36,7 +36,7 @@
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/Support/CallSite.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include <cstdio>
 
 using namespace clang;
@@ -66,7 +66,8 @@ private:
     return CGM.CreateRuntimeFunction(llvm::FunctionType::get(ObjectPtrTy,
                                                              params, true),
                                      "objc_msgSend",
-                                     llvm::Attribute::NonLazyBind);
+                                     llvm::Attributes::get(CGM.getLLVMContext(),
+                                                llvm::Attributes::NonLazyBind));
   }
 
   /// void objc_msgSend_stret (id, SEL, ...)
@@ -433,19 +434,19 @@ public:
   
   /// SyncEnterFn - LLVM object_sync_enter function.
   llvm::Constant *getSyncEnterFn() {
-    // void objc_sync_enter (id)
+    // int objc_sync_enter (id)
     llvm::Type *args[] = { ObjectPtrTy };
     llvm::FunctionType *FTy =
-      llvm::FunctionType::get(CGM.VoidTy, args, false);
+      llvm::FunctionType::get(CGM.IntTy, args, false);
     return CGM.CreateRuntimeFunction(FTy, "objc_sync_enter");
   }
 
   /// SyncExitFn - LLVM object_sync_exit function.
   llvm::Constant *getSyncExitFn() {
-    // void objc_sync_exit (id)
+    // int objc_sync_exit (id)
     llvm::Type *args[] = { ObjectPtrTy };
     llvm::FunctionType *FTy =
-      llvm::FunctionType::get(CGM.VoidTy, args, false);
+      llvm::FunctionType::get(CGM.IntTy, args, false);
     return CGM.CreateRuntimeFunction(FTy, "objc_sync_exit");
   }
 
@@ -583,7 +584,8 @@ public:
     return CGM.CreateRuntimeFunction(llvm::FunctionType::get(CGM.Int32Ty,
                                                              params, false),
                                      "_setjmp",
-                                     llvm::Attribute::ReturnsTwice);
+                                     llvm::Attributes::get(CGM.getLLVMContext(),
+                                                llvm::Attributes::NonLazyBind));
   }
 
 public:
@@ -753,6 +755,74 @@ public:
       : skip(_skip), scan(_scan) {}
   };
 
+  /// opcode for captured block variables layout 'instructions'.
+  /// In the following descriptions, 'I' is the value of the immediate field.
+  /// (field following the opcode).
+  ///
+  enum BLOCK_LAYOUT_OPCODE {
+    /// An operator which affects how the following layout should be
+    /// interpreted.
+    ///   I == 0: Halt interpretation and treat everything else as
+    ///           a non-pointer.  Note that this instruction is equal
+    ///           to '\0'.
+    ///   I != 0: Currently unused.
+    BLOCK_LAYOUT_OPERATOR            = 0,
+    
+    /// The next I+1 bytes do not contain a value of object pointer type.
+    /// Note that this can leave the stream unaligned, meaning that
+    /// subsequent word-size instructions do not begin at a multiple of
+    /// the pointer size.
+    BLOCK_LAYOUT_NON_OBJECT_BYTES    = 1,
+    
+    /// The next I+1 words do not contain a value of object pointer type.
+    /// This is simply an optimized version of BLOCK_LAYOUT_BYTES for
+    /// when the required skip quantity is a multiple of the pointer size.
+    BLOCK_LAYOUT_NON_OBJECT_WORDS    = 2,
+    
+    /// The next I+1 words are __strong pointers to Objective-C
+    /// objects or blocks.
+    BLOCK_LAYOUT_STRONG              = 3,
+    
+    /// The next I+1 words are pointers to __block variables.
+    BLOCK_LAYOUT_BYREF               = 4,
+    
+    /// The next I+1 words are __weak pointers to Objective-C
+    /// objects or blocks.
+    BLOCK_LAYOUT_WEAK                = 5,
+    
+    /// The next I+1 words are __unsafe_unretained pointers to
+    /// Objective-C objects or blocks.
+    BLOCK_LAYOUT_UNRETAINED          = 6
+    
+    /// The next I+1 words are block or object pointers with some
+    /// as-yet-unspecified ownership semantics.  If we add more
+    /// flavors of ownership semantics, values will be taken from
+    /// this range.
+    ///
+    /// This is included so that older tools can at least continue
+    /// processing the layout past such things.
+    //BLOCK_LAYOUT_OWNERSHIP_UNKNOWN = 7..10,
+    
+    /// All other opcodes are reserved.  Halt interpretation and
+    /// treat everything else as opaque.
+  };
+ 
+  class RUN_SKIP {
+  public:
+    enum BLOCK_LAYOUT_OPCODE opcode;
+    CharUnits block_var_bytepos;
+    CharUnits block_var_size;
+    RUN_SKIP(enum BLOCK_LAYOUT_OPCODE Opcode = BLOCK_LAYOUT_OPERATOR,
+             CharUnits BytePos = CharUnits::Zero(),
+             CharUnits Size = CharUnits::Zero())
+    : opcode(Opcode), block_var_bytepos(BytePos),  block_var_size(Size) {}
+    
+    // Allow sorting based on byte pos.
+    bool operator<(const RUN_SKIP &b) const {
+      return block_var_bytepos < b.block_var_bytepos;
+    }
+  };
+  
 protected:
   llvm::LLVMContext &VMContext;
   // FIXME! May not be needing this after all.
@@ -761,6 +831,9 @@ protected:
   // gc ivar layout bitmap calculation helper caches.
   SmallVector<GC_IVAR, 16> SkipIvars;
   SmallVector<GC_IVAR, 16> IvarsInfo;
+  
+  // arc/mrr layout of captured block literal variables.
+  SmallVector<RUN_SKIP, 16> RunSkipBlockVars;
 
   /// LazySymbols - Symbols to generate a lazy reference for. See
   /// DefinedSymbols and FinishModule().
@@ -869,6 +942,24 @@ protected:
                            ArrayRef<const FieldDecl*> RecFields,
                            unsigned int BytePos, bool ForStrongLayout,
                            bool &HasUnion);
+  
+  Qualifiers::ObjCLifetime getBlockCaptureLifetime(QualType QT);
+  
+  void UpdateRunSkipBlockVars(bool IsByref,
+                              Qualifiers::ObjCLifetime LifeTime,
+                              CharUnits FieldOffset,
+                              CharUnits FieldSize);
+  
+  void BuildRCBlockVarRecordLayout(const RecordType *RT,
+                                   CharUnits BytePos, bool &HasUnion);
+  
+  void BuildRCRecordLayout(const llvm::StructLayout *RecLayout,
+                           const RecordDecl *RD,
+                           ArrayRef<const FieldDecl*> RecFields,
+                           CharUnits BytePos, bool &HasUnion);
+  
+  uint64_t InlineLayoutInstruction(SmallVectorImpl<unsigned char> &Layout);
+  
 
   /// GetIvarLayoutName - Returns a unique constant for the given
   /// ivar layout bitmap.
@@ -959,6 +1050,8 @@ public:
   virtual llvm::Constant *GetOrEmitProtocolRef(const ObjCProtocolDecl *PD)=0;
   virtual llvm::Constant *BuildGCBlockLayout(CodeGen::CodeGenModule &CGM,
                                              const CGBlockInfo &blockInfo);
+  virtual llvm::Constant *BuildRCBlockLayout(CodeGen::CodeGenModule &CGM,
+                                             const CGBlockInfo &blockInfo);
   
 };
 
@@ -1787,8 +1880,8 @@ static Qualifiers::GC GetGCAttrTypeForType(ASTContext &Ctx, QualType FQT) {
 
 llvm::Constant *CGObjCCommonMac::BuildGCBlockLayout(CodeGenModule &CGM,
                                                 const CGBlockInfo &blockInfo) {
+  
   llvm::Constant *nullPtr = llvm::Constant::getNullValue(CGM.Int8PtrTy);
-
   if (CGM.getLangOpts().getGC() == LangOptions::NonGC &&
       !CGM.getLangOpts().ObjCAutoRefCount)
     return nullPtr;
@@ -1807,7 +1900,7 @@ llvm::Constant *CGObjCCommonMac::BuildGCBlockLayout(CodeGenModule &CGM,
 
   // Calculate the basic layout of the block structure.
   const llvm::StructLayout *layout =
-    CGM.getTargetData().getStructLayout(blockInfo.StructureType);
+    CGM.getDataLayout().getStructLayout(blockInfo.StructureType);
 
   // Ignore the optional 'this' capture: C++ objects are not assumed
   // to be GC'ed.
@@ -1860,7 +1953,7 @@ llvm::Constant *CGObjCCommonMac::BuildGCBlockLayout(CodeGenModule &CGM,
   llvm::Constant *C = BuildIvarLayoutBitmap(BitMap);
   if (CGM.getLangOpts().ObjCGCBitmapPrint) {
     printf("\n block variable layout for block: ");
-    const unsigned char *s = (unsigned char*)BitMap.c_str();
+    const unsigned char *s = (const unsigned char*)BitMap.c_str();
     for (unsigned i = 0, e = BitMap.size(); i < e; i++)
       if (!(s[i] & 0xf0))
         printf("0x0%x%s", s[i], s[i] != 0 ? ", " : "");
@@ -1872,6 +1965,476 @@ llvm::Constant *CGObjCCommonMac::BuildGCBlockLayout(CodeGenModule &CGM,
   return C;
 }
 
+/// getBlockCaptureLifetime - This routine returns life time of the captured
+/// block variable for the purpose of block layout meta-data generation. FQT is
+/// the type of the variable captured in the block.
+Qualifiers::ObjCLifetime CGObjCCommonMac::getBlockCaptureLifetime(QualType FQT) {
+  if (CGM.getLangOpts().ObjCAutoRefCount)
+    return FQT.getObjCLifetime();
+  
+  // MRR.
+  if (FQT->isObjCObjectPointerType() || FQT->isBlockPointerType())
+    return Qualifiers::OCL_ExplicitNone;
+  
+  return Qualifiers::OCL_None;
+}
+
+void CGObjCCommonMac::UpdateRunSkipBlockVars(bool IsByref,
+                                             Qualifiers::ObjCLifetime LifeTime,
+                                             CharUnits FieldOffset,
+                                             CharUnits FieldSize) {
+  // __block variables are passed by their descriptor address.
+  if (IsByref)
+    RunSkipBlockVars.push_back(RUN_SKIP(BLOCK_LAYOUT_BYREF, FieldOffset,
+                                        FieldSize));
+  else if (LifeTime == Qualifiers::OCL_Strong)
+    RunSkipBlockVars.push_back(RUN_SKIP(BLOCK_LAYOUT_STRONG, FieldOffset,
+                                        FieldSize));
+  else if (LifeTime == Qualifiers::OCL_Weak)
+    RunSkipBlockVars.push_back(RUN_SKIP(BLOCK_LAYOUT_WEAK, FieldOffset,
+                                        FieldSize));
+  else if (LifeTime == Qualifiers::OCL_ExplicitNone)
+    RunSkipBlockVars.push_back(RUN_SKIP(BLOCK_LAYOUT_UNRETAINED, FieldOffset,
+                                        FieldSize));
+  else
+    RunSkipBlockVars.push_back(RUN_SKIP(BLOCK_LAYOUT_NON_OBJECT_BYTES,
+                                        FieldOffset,
+                                        FieldSize));
+}
+
+void CGObjCCommonMac::BuildRCRecordLayout(const llvm::StructLayout *RecLayout,
+                                          const RecordDecl *RD,
+                                          ArrayRef<const FieldDecl*> RecFields,
+                                          CharUnits BytePos, bool &HasUnion) {
+  bool IsUnion = (RD && RD->isUnion());
+  CharUnits MaxUnionSize = CharUnits::Zero();
+  const FieldDecl *MaxField = 0;
+  const FieldDecl *LastFieldBitfieldOrUnnamed = 0;
+  CharUnits MaxFieldOffset = CharUnits::Zero();
+  CharUnits LastBitfieldOrUnnamedOffset = CharUnits::Zero();
+  
+  if (RecFields.empty())
+    return;
+  unsigned ByteSizeInBits = CGM.getContext().getTargetInfo().getCharWidth();
+  
+  for (unsigned i = 0, e = RecFields.size(); i != e; ++i) {
+    const FieldDecl *Field = RecFields[i];
+    // Note that 'i' here is actually the field index inside RD of Field,
+    // although this dependency is hidden.
+    const ASTRecordLayout &RL = CGM.getContext().getASTRecordLayout(RD);
+    CharUnits FieldOffset =
+      CGM.getContext().toCharUnitsFromBits(RL.getFieldOffset(i));
+    
+    // Skip over unnamed or bitfields
+    if (!Field->getIdentifier() || Field->isBitField()) {
+      LastFieldBitfieldOrUnnamed = Field;
+      LastBitfieldOrUnnamedOffset = FieldOffset;
+      continue;
+    }
+    
+    LastFieldBitfieldOrUnnamed = 0;
+    QualType FQT = Field->getType();
+    if (FQT->isRecordType() || FQT->isUnionType()) {
+      if (FQT->isUnionType())
+        HasUnion = true;
+      
+      BuildRCBlockVarRecordLayout(FQT->getAs<RecordType>(),
+                                  BytePos + FieldOffset, HasUnion);
+      continue;
+    }
+    
+    if (const ArrayType *Array = CGM.getContext().getAsArrayType(FQT)) {
+      const ConstantArrayType *CArray =
+        dyn_cast_or_null<ConstantArrayType>(Array);
+      uint64_t ElCount = CArray->getSize().getZExtValue();
+      assert(CArray && "only array with known element size is supported");
+      FQT = CArray->getElementType();
+      while (const ArrayType *Array = CGM.getContext().getAsArrayType(FQT)) {
+        const ConstantArrayType *CArray =
+          dyn_cast_or_null<ConstantArrayType>(Array);
+        ElCount *= CArray->getSize().getZExtValue();
+        FQT = CArray->getElementType();
+      }
+      
+      assert(!FQT->isUnionType() &&
+             "layout for array of unions not supported");
+      if (FQT->isRecordType() && ElCount) {
+        int OldIndex = RunSkipBlockVars.size() - 1;
+        const RecordType *RT = FQT->getAs<RecordType>();
+        BuildRCBlockVarRecordLayout(RT, BytePos + FieldOffset,
+                                    HasUnion);
+        
+        // Replicate layout information for each array element. Note that
+        // one element is already done.
+        uint64_t ElIx = 1;
+        for (int FirstIndex = RunSkipBlockVars.size() - 1 ;ElIx < ElCount; ElIx++) {
+          CharUnits Size = CGM.getContext().getTypeSizeInChars(RT);
+          for (int i = OldIndex+1; i <= FirstIndex; ++i)
+            RunSkipBlockVars.push_back(
+              RUN_SKIP(RunSkipBlockVars[i].opcode,
+              RunSkipBlockVars[i].block_var_bytepos + Size*ElIx,
+              RunSkipBlockVars[i].block_var_size));
+        }
+        continue;
+      }
+    }
+    CharUnits FieldSize = CGM.getContext().getTypeSizeInChars(Field->getType());
+    if (IsUnion) {
+      CharUnits UnionIvarSize = FieldSize;
+      if (UnionIvarSize > MaxUnionSize) {
+        MaxUnionSize = UnionIvarSize;
+        MaxField = Field;
+        MaxFieldOffset = FieldOffset;
+      }
+    } else {
+      UpdateRunSkipBlockVars(false,
+                             getBlockCaptureLifetime(FQT),
+                             BytePos + FieldOffset,
+                             FieldSize);
+    }
+  }
+  
+  if (LastFieldBitfieldOrUnnamed) {
+    if (LastFieldBitfieldOrUnnamed->isBitField()) {
+      // Last field was a bitfield. Must update the info.
+      uint64_t BitFieldSize
+        = LastFieldBitfieldOrUnnamed->getBitWidthValue(CGM.getContext());
+      unsigned UnsSize = (BitFieldSize / ByteSizeInBits) +
+                        ((BitFieldSize % ByteSizeInBits) != 0);
+      CharUnits Size = CharUnits::fromQuantity(UnsSize);
+      Size += LastBitfieldOrUnnamedOffset;
+      UpdateRunSkipBlockVars(false,
+                             getBlockCaptureLifetime(LastFieldBitfieldOrUnnamed->getType()),
+                             BytePos + LastBitfieldOrUnnamedOffset,
+                             Size);
+    } else {
+      assert(!LastFieldBitfieldOrUnnamed->getIdentifier() &&"Expected unnamed");
+      // Last field was unnamed. Must update skip info.
+      CharUnits FieldSize
+        = CGM.getContext().getTypeSizeInChars(LastFieldBitfieldOrUnnamed->getType());
+      UpdateRunSkipBlockVars(false,
+                             getBlockCaptureLifetime(LastFieldBitfieldOrUnnamed->getType()),
+                             BytePos + LastBitfieldOrUnnamedOffset,
+                             FieldSize);
+    }
+  }
+  
+  if (MaxField)
+    UpdateRunSkipBlockVars(false,
+                           getBlockCaptureLifetime(MaxField->getType()),
+                           BytePos + MaxFieldOffset,
+                           MaxUnionSize);
+}
+
+void CGObjCCommonMac::BuildRCBlockVarRecordLayout(const RecordType *RT,
+                                                  CharUnits BytePos,
+                                                  bool &HasUnion) {
+  const RecordDecl *RD = RT->getDecl();
+  SmallVector<const FieldDecl*, 16> Fields;
+  for (RecordDecl::field_iterator i = RD->field_begin(),
+       e = RD->field_end(); i != e; ++i)
+    Fields.push_back(*i);
+  llvm::Type *Ty = CGM.getTypes().ConvertType(QualType(RT, 0));
+  const llvm::StructLayout *RecLayout =
+    CGM.getDataLayout().getStructLayout(cast<llvm::StructType>(Ty));
+  
+  BuildRCRecordLayout(RecLayout, RD, Fields, BytePos, HasUnion);
+}
+
+/// InlineLayoutInstruction - This routine produce an inline instruction for the
+/// block variable layout if it can. If not, it returns 0. Rules are as follow:
+/// If ((uintptr_t) layout) < (1 << 12), the layout is inline. In the 64bit world,
+/// an inline layout of value 0x0000000000000xyz is interpreted as follows:
+/// x captured object pointers of BLOCK_LAYOUT_STRONG. Followed by
+/// y captured object of BLOCK_LAYOUT_BYREF. Followed by
+/// z captured object of BLOCK_LAYOUT_WEAK. If any of the above is missing, zero
+/// replaces it. For example, 0x00000x00 means x BLOCK_LAYOUT_STRONG and no
+/// BLOCK_LAYOUT_BYREF and no BLOCK_LAYOUT_WEAK objects are captured.
+uint64_t CGObjCCommonMac::InlineLayoutInstruction(
+                                    SmallVectorImpl<unsigned char> &Layout) {
+  uint64_t Result = 0;
+  if (Layout.size() <= 3) {
+    unsigned size = Layout.size();
+    unsigned strong_word_count = 0, byref_word_count=0, weak_word_count=0;
+    unsigned char inst;
+    enum BLOCK_LAYOUT_OPCODE opcode ;
+    switch (size) {
+      case 3:
+        inst = Layout[0];
+        opcode = (enum BLOCK_LAYOUT_OPCODE) (inst >> 4);
+        if (opcode == BLOCK_LAYOUT_STRONG)
+          strong_word_count = (inst & 0xF)+1;
+        else
+          return 0;
+        inst = Layout[1];
+        opcode = (enum BLOCK_LAYOUT_OPCODE) (inst >> 4);
+        if (opcode == BLOCK_LAYOUT_BYREF)
+          byref_word_count = (inst & 0xF)+1;
+        else
+          return 0;
+        inst = Layout[2];
+        opcode = (enum BLOCK_LAYOUT_OPCODE) (inst >> 4);
+        if (opcode == BLOCK_LAYOUT_WEAK)
+          weak_word_count = (inst & 0xF)+1;
+        else
+          return 0;
+        break;
+        
+      case 2:
+        inst = Layout[0];
+        opcode = (enum BLOCK_LAYOUT_OPCODE) (inst >> 4);
+        if (opcode == BLOCK_LAYOUT_STRONG) {
+          strong_word_count = (inst & 0xF)+1;
+          inst = Layout[1];
+          opcode = (enum BLOCK_LAYOUT_OPCODE) (inst >> 4);
+          if (opcode == BLOCK_LAYOUT_BYREF)
+            byref_word_count = (inst & 0xF)+1;
+          else if (opcode == BLOCK_LAYOUT_WEAK)
+            weak_word_count = (inst & 0xF)+1;
+          else
+            return 0;
+        }
+        else if (opcode == BLOCK_LAYOUT_BYREF) {
+          byref_word_count = (inst & 0xF)+1;
+          inst = Layout[1];
+          opcode = (enum BLOCK_LAYOUT_OPCODE) (inst >> 4);
+          if (opcode == BLOCK_LAYOUT_WEAK)
+            weak_word_count = (inst & 0xF)+1;
+          else
+            return 0;
+        }
+        else
+          return 0;
+        break;
+        
+      case 1:
+        inst = Layout[0];
+        opcode = (enum BLOCK_LAYOUT_OPCODE) (inst >> 4);
+        if (opcode == BLOCK_LAYOUT_STRONG)
+          strong_word_count = (inst & 0xF)+1;
+        else if (opcode == BLOCK_LAYOUT_BYREF)
+          byref_word_count = (inst & 0xF)+1;
+        else if (opcode == BLOCK_LAYOUT_WEAK)
+          weak_word_count = (inst & 0xF)+1;
+        else
+          return 0;
+        break;
+        
+      default:
+        return 0;
+    }
+    
+    // Cannot inline when any of the word counts is 15. Because this is one less
+    // than the actual work count (so 15 means 16 actual word counts),
+    // and we can only display 0 thru 15 word counts.
+    if (strong_word_count == 16 || byref_word_count == 16 || weak_word_count == 16)
+      return 0;
+    
+    unsigned count =
+      (strong_word_count != 0) + (byref_word_count != 0) + (weak_word_count != 0);
+    
+    if (size == count) {
+      if (strong_word_count)
+        Result = strong_word_count;
+      Result <<= 4;
+      if (byref_word_count)
+        Result += byref_word_count;
+      Result <<= 4;
+      if (weak_word_count)
+        Result += weak_word_count;
+    }
+  }
+  return Result;
+}
+
+llvm::Constant *CGObjCCommonMac::BuildRCBlockLayout(CodeGenModule &CGM,
+                                                    const CGBlockInfo &blockInfo) {
+  assert(CGM.getLangOpts().getGC() == LangOptions::NonGC);
+  
+  llvm::Constant *nullPtr = llvm::Constant::getNullValue(CGM.Int8PtrTy);
+  
+  RunSkipBlockVars.clear();
+  bool hasUnion = false;
+  
+  unsigned WordSizeInBits = CGM.getContext().getTargetInfo().getPointerWidth(0);
+  unsigned ByteSizeInBits = CGM.getContext().getTargetInfo().getCharWidth();
+  unsigned WordSizeInBytes = WordSizeInBits/ByteSizeInBits;
+  
+  const BlockDecl *blockDecl = blockInfo.getBlockDecl();
+  
+  // Calculate the basic layout of the block structure.
+  const llvm::StructLayout *layout =
+  CGM.getDataLayout().getStructLayout(blockInfo.StructureType);
+  
+  // Ignore the optional 'this' capture: C++ objects are not assumed
+  // to be GC'ed.
+  
+  // Walk the captured variables.
+  for (BlockDecl::capture_const_iterator ci = blockDecl->capture_begin(),
+       ce = blockDecl->capture_end(); ci != ce; ++ci) {
+    const VarDecl *variable = ci->getVariable();
+    QualType type = variable->getType();
+    
+    const CGBlockInfo::Capture &capture = blockInfo.getCapture(variable);
+    
+    // Ignore constant captures.
+    if (capture.isConstant()) continue;
+    
+    CharUnits fieldOffset =
+       CharUnits::fromQuantity(layout->getElementOffset(capture.getIndex()));
+    
+    assert(!type->isArrayType() && "array variable should not be caught");
+    if (const RecordType *record = type->getAs<RecordType>()) {
+      BuildRCBlockVarRecordLayout(record, fieldOffset, hasUnion);
+      continue;
+    }
+    CharUnits fieldSize;
+    if (ci->isByRef())
+      fieldSize = CharUnits::fromQuantity(WordSizeInBytes);
+    else
+      fieldSize = CGM.getContext().getTypeSizeInChars(type);
+    UpdateRunSkipBlockVars(ci->isByRef(), getBlockCaptureLifetime(type),
+                           fieldOffset, fieldSize);
+  }
+  
+  if (RunSkipBlockVars.empty())
+    return nullPtr;
+  
+  // Sort on byte position; captures might not be allocated in order,
+  // and unions can do funny things.
+  llvm::array_pod_sort(RunSkipBlockVars.begin(), RunSkipBlockVars.end());
+  SmallVector<unsigned char, 16> Layout;
+
+  unsigned size = RunSkipBlockVars.size();
+  for (unsigned i = 0; i < size; i++) {
+    enum BLOCK_LAYOUT_OPCODE opcode = RunSkipBlockVars[i].opcode;
+    CharUnits start_byte_pos = RunSkipBlockVars[i].block_var_bytepos;
+    CharUnits end_byte_pos = start_byte_pos;
+    unsigned j = i+1;
+    while (j < size) {
+      if (opcode == RunSkipBlockVars[j].opcode) {
+        end_byte_pos = RunSkipBlockVars[j++].block_var_bytepos;
+        i++;
+      }
+      else
+        break;
+    }
+    CharUnits size_in_bytes =
+      end_byte_pos - start_byte_pos + RunSkipBlockVars[j-1].block_var_size;
+    if (j < size) {
+      CharUnits gap =
+        RunSkipBlockVars[j].block_var_bytepos -
+        RunSkipBlockVars[j-1].block_var_bytepos - RunSkipBlockVars[j-1].block_var_size;
+      size_in_bytes += gap;
+    }
+    CharUnits residue_in_bytes = CharUnits::Zero();
+    if (opcode == BLOCK_LAYOUT_NON_OBJECT_BYTES) {
+      residue_in_bytes = size_in_bytes % WordSizeInBytes;
+      size_in_bytes -= residue_in_bytes;
+      opcode = BLOCK_LAYOUT_NON_OBJECT_WORDS;
+    }
+
+    unsigned size_in_words = size_in_bytes.getQuantity() / WordSizeInBytes;
+    while (size_in_words >= 16) {
+      // Note that value in imm. is one less that the actual
+      // value. So, 0xf means 16 words follow!
+      unsigned char inst = (opcode << 4) | 0xf;
+      Layout.push_back(inst);
+      size_in_words -= 16;
+    }
+    if (size_in_words > 0) {
+      // Note that value in imm. is one less that the actual
+      // value. So, we subtract 1 away!
+      unsigned char inst = (opcode << 4) | (size_in_words-1);
+      Layout.push_back(inst);
+    }
+    if (residue_in_bytes > CharUnits::Zero()) {
+      unsigned char inst =
+        (BLOCK_LAYOUT_NON_OBJECT_BYTES << 4) | (residue_in_bytes.getQuantity()-1);
+      Layout.push_back(inst);
+    }
+  }
+  
+  int e = Layout.size()-1;
+  while (e >= 0) {
+    unsigned char inst = Layout[e--];
+    enum BLOCK_LAYOUT_OPCODE opcode = (enum BLOCK_LAYOUT_OPCODE) (inst >> 4);
+    if (opcode == BLOCK_LAYOUT_NON_OBJECT_BYTES || opcode == BLOCK_LAYOUT_NON_OBJECT_WORDS)
+      Layout.pop_back();
+    else
+      break;
+  }
+  
+  uint64_t Result = InlineLayoutInstruction(Layout);
+  if (Result != 0) {
+    // Block variable layout instruction has been inlined.
+    if (CGM.getLangOpts().ObjCGCBitmapPrint) {
+      printf("\n Inline instruction for block variable layout: ");
+      printf("0x0%llx\n", (unsigned long long)Result);
+    }
+    if (WordSizeInBytes == 8) {
+      const llvm::APInt Instruction(64, Result);
+      return llvm::Constant::getIntegerValue(CGM.Int64Ty, Instruction);
+    }
+    else {
+      const llvm::APInt Instruction(32, Result);
+      return llvm::Constant::getIntegerValue(CGM.Int32Ty, Instruction);
+    }
+  }
+  
+  unsigned char inst = (BLOCK_LAYOUT_OPERATOR << 4) | 0;
+  Layout.push_back(inst);
+  std::string BitMap;
+  for (unsigned i = 0, e = Layout.size(); i != e; i++)
+    BitMap += Layout[i];
+  
+  if (CGM.getLangOpts().ObjCGCBitmapPrint) {
+    printf("\n block variable layout: ");
+    for (unsigned i = 0, e = BitMap.size(); i != e; i++) {
+      unsigned char inst = BitMap[i];
+      enum BLOCK_LAYOUT_OPCODE opcode = (enum BLOCK_LAYOUT_OPCODE) (inst >> 4);
+      unsigned delta = 1;
+      switch (opcode) {
+        case BLOCK_LAYOUT_OPERATOR:
+          printf("BL_OPERATOR:");
+          delta = 0;
+          break;
+        case BLOCK_LAYOUT_NON_OBJECT_BYTES:
+          printf("BL_NON_OBJECT_BYTES:");
+          break;
+        case BLOCK_LAYOUT_NON_OBJECT_WORDS:
+          printf("BL_NON_OBJECT_WORD:");
+          break;
+        case BLOCK_LAYOUT_STRONG:
+          printf("BL_STRONG:");
+          break;
+        case BLOCK_LAYOUT_BYREF:
+          printf("BL_BYREF:");
+          break;
+        case BLOCK_LAYOUT_WEAK:
+          printf("BL_WEAK:");
+          break;
+        case BLOCK_LAYOUT_UNRETAINED:
+          printf("BL_UNRETAINED:");
+          break;
+      } 
+      // Actual value of word count is one more that what is in the imm.
+      // field of the instruction
+      printf("%d", (inst & 0xf) + delta); 
+      if (i < e-1)
+        printf(", ");
+      else
+        printf("\n");
+    }
+  }
+  
+  llvm::GlobalVariable * Entry =
+    CreateMetadataVar("\01L_OBJC_CLASS_NAME_",
+                    llvm::ConstantDataArray::getString(VMContext, BitMap,false),
+                    "__TEXT,__objc_classname,cstring_literals", 1, true);
+  return getConstantGEP(VMContext, Entry, 0, 0);
+}
+
 llvm::Value *CGObjCMac::GenerateProtocolRef(CGBuilderTy &Builder,
                                             const ObjCProtocolDecl *PD) {
   // FIXME: I don't understand why gcc generates this, or where it is
@@ -2040,7 +2603,7 @@ CGObjCMac::EmitProtocolExtension(const ObjCProtocolDecl *PD,
                                  ArrayRef<llvm::Constant*> OptClassMethods,
                                  ArrayRef<llvm::Constant*> MethodTypesExt) {
   uint64_t Size =
-    CGM.getTargetData().getTypeAllocSize(ObjCTypes.ProtocolExtensionTy);
+    CGM.getDataLayout().getTypeAllocSize(ObjCTypes.ProtocolExtensionTy);
   llvm::Constant *Values[] = {
     llvm::ConstantInt::get(ObjCTypes.IntTy, Size),
     EmitMethodDescList("\01L_OBJC_PROTOCOL_INSTANCE_METHODS_OPT_"
@@ -2180,7 +2743,7 @@ llvm::Constant *CGObjCCommonMac::EmitPropertyList(Twine Name,
     return llvm::Constant::getNullValue(ObjCTypes.PropertyListPtrTy);
 
   unsigned PropertySize =
-    CGM.getTargetData().getTypeAllocSize(ObjCTypes.PropertyTy);
+    CGM.getDataLayout().getTypeAllocSize(ObjCTypes.PropertyTy);
   llvm::Constant *Values[3];
   Values[0] = llvm::ConstantInt::get(ObjCTypes.IntTy, PropertySize);
   Values[1] = llvm::ConstantInt::get(ObjCTypes.IntTy, Properties.size());
@@ -2269,7 +2832,7 @@ CGObjCMac::EmitMethodDescList(Twine Name, const char *Section,
   };
 */
 void CGObjCMac::GenerateCategory(const ObjCCategoryImplDecl *OCD) {
-  unsigned Size = CGM.getTargetData().getTypeAllocSize(ObjCTypes.CategoryTy);
+  unsigned Size = CGM.getDataLayout().getTypeAllocSize(ObjCTypes.CategoryTy);
 
   // FIXME: This is poor design, the OCD should have a pointer to the category
   // decl. Additionally, note that Category can be null for the @implementation
@@ -2338,15 +2901,37 @@ void CGObjCMac::GenerateCategory(const ObjCCategoryImplDecl *OCD) {
   MethodDefinitions.clear();
 }
 
-// FIXME: Get from somewhere?
-enum ClassFlags {
-  eClassFlags_Factory              = 0x00001,
-  eClassFlags_Meta                 = 0x00002,
-  // <rdr://5142207>
-  eClassFlags_HasCXXStructors      = 0x02000,
-  eClassFlags_Hidden               = 0x20000,
-  eClassFlags_ABI2_Hidden          = 0x00010,
-  eClassFlags_ABI2_HasCXXStructors = 0x00004   // <rdr://4923634>
+enum FragileClassFlags {
+  FragileABI_Class_Factory                 = 0x00001,
+  FragileABI_Class_Meta                    = 0x00002,
+  FragileABI_Class_HasCXXStructors         = 0x02000,
+  FragileABI_Class_Hidden                  = 0x20000
+};
+
+enum NonFragileClassFlags {
+  /// Is a meta-class.
+  NonFragileABI_Class_Meta                 = 0x00001,
+
+  /// Is a root class.
+  NonFragileABI_Class_Root                 = 0x00002,
+
+  /// Has a C++ constructor and destructor.
+  NonFragileABI_Class_HasCXXStructors      = 0x00004,
+
+  /// Has hidden visibility.
+  NonFragileABI_Class_Hidden               = 0x00010,
+
+  /// Has the exception attribute.
+  NonFragileABI_Class_Exception            = 0x00020,
+
+  /// (Obsolete) ARC-specific: this class has a .release_ivars method
+  NonFragileABI_Class_HasIvarReleaser      = 0x00040,
+
+  /// Class implementation was compiled under ARC.
+  NonFragileABI_Class_CompiledByARC        = 0x00080,
+
+  /// Class has non-trivial destructors, but zero-initialization is okay.
+  NonFragileABI_Class_HasCXXDestructorOnly = 0x00100
 };
 
 /*
@@ -2379,15 +2964,15 @@ void CGObjCMac::GenerateClass(const ObjCImplementationDecl *ID) {
     EmitProtocolList("\01L_OBJC_CLASS_PROTOCOLS_" + ID->getName(),
                      Interface->all_referenced_protocol_begin(),
                      Interface->all_referenced_protocol_end());
-  unsigned Flags = eClassFlags_Factory;
-  if (ID->hasCXXStructors())
-    Flags |= eClassFlags_HasCXXStructors;
+  unsigned Flags = FragileABI_Class_Factory;
+  if (ID->hasNonZeroConstructors() || ID->hasDestructors())
+    Flags |= FragileABI_Class_HasCXXStructors;
   unsigned Size =
     CGM.getContext().getASTObjCImplementationLayout(ID).getSize().getQuantity();
 
   // FIXME: Set CXX-structors flag.
   if (ID->getClassInterface()->getVisibility() == HiddenVisibility)
-    Flags |= eClassFlags_Hidden;
+    Flags |= FragileABI_Class_Hidden;
 
   llvm::SmallVector<llvm::Constant*, 16> InstanceMethods, ClassMethods;
   for (ObjCImplementationDecl::instmeth_iterator
@@ -2470,11 +3055,11 @@ void CGObjCMac::GenerateClass(const ObjCImplementationDecl *ID) {
 llvm::Constant *CGObjCMac::EmitMetaClass(const ObjCImplementationDecl *ID,
                                          llvm::Constant *Protocols,
                                          ArrayRef<llvm::Constant*> Methods) {
-  unsigned Flags = eClassFlags_Meta;
-  unsigned Size = CGM.getTargetData().getTypeAllocSize(ObjCTypes.ClassTy);
+  unsigned Flags = FragileABI_Class_Meta;
+  unsigned Size = CGM.getDataLayout().getTypeAllocSize(ObjCTypes.ClassTy);
 
   if (ID->getClassInterface()->getVisibility() == HiddenVisibility)
-    Flags |= eClassFlags_Hidden;
+    Flags |= FragileABI_Class_Hidden;
 
   llvm::Constant *Values[12];
   // The isa for the metaclass is the root of the hierarchy.
@@ -2588,7 +3173,7 @@ llvm::Value *CGObjCMac::EmitSuperClassRef(const ObjCInterfaceDecl *ID) {
 llvm::Constant *
 CGObjCMac::EmitClassExtension(const ObjCImplementationDecl *ID) {
   uint64_t Size =
-    CGM.getTargetData().getTypeAllocSize(ObjCTypes.ClassExtensionTy);
+    CGM.getDataLayout().getTypeAllocSize(ObjCTypes.ClassExtensionTy);
 
   llvm::Constant *Values[3];
   Values[0] = llvm::ConstantInt::get(ObjCTypes.IntTy, Size);
@@ -3481,7 +4066,7 @@ void CGObjCMac::EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF,
                                    llvm::Value *src, llvm::Value *dst) {
   llvm::Type * SrcTy = src->getType();
   if (!isa<llvm::PointerType>(SrcTy)) {
-    unsigned Size = CGM.getTargetData().getTypeAllocSize(SrcTy);
+    unsigned Size = CGM.getDataLayout().getTypeAllocSize(SrcTy);
     assert(Size <= 8 && "does not support size > 8");
     src = (Size == 4) ? CGF.Builder.CreateBitCast(src, ObjCTypes.IntTy)
       : CGF.Builder.CreateBitCast(src, ObjCTypes.LongLongTy);
@@ -3502,7 +4087,7 @@ void CGObjCMac::EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF,
                                      bool threadlocal) {
   llvm::Type * SrcTy = src->getType();
   if (!isa<llvm::PointerType>(SrcTy)) {
-    unsigned Size = CGM.getTargetData().getTypeAllocSize(SrcTy);
+    unsigned Size = CGM.getDataLayout().getTypeAllocSize(SrcTy);
     assert(Size <= 8 && "does not support size > 8");
     src = (Size == 4) ? CGF.Builder.CreateBitCast(src, ObjCTypes.IntTy)
       : CGF.Builder.CreateBitCast(src, ObjCTypes.LongLongTy);
@@ -3528,7 +4113,7 @@ void CGObjCMac::EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF,
   assert(ivarOffset && "EmitObjCIvarAssign - ivarOffset is NULL");
   llvm::Type * SrcTy = src->getType();
   if (!isa<llvm::PointerType>(SrcTy)) {
-    unsigned Size = CGM.getTargetData().getTypeAllocSize(SrcTy);
+    unsigned Size = CGM.getDataLayout().getTypeAllocSize(SrcTy);
     assert(Size <= 8 && "does not support size > 8");
     src = (Size == 4) ? CGF.Builder.CreateBitCast(src, ObjCTypes.IntTy)
       : CGF.Builder.CreateBitCast(src, ObjCTypes.LongLongTy);
@@ -3548,7 +4133,7 @@ void CGObjCMac::EmitObjCStrongCastAssign(CodeGen::CodeGenFunction &CGF,
                                          llvm::Value *src, llvm::Value *dst) {
   llvm::Type * SrcTy = src->getType();
   if (!isa<llvm::PointerType>(SrcTy)) {
-    unsigned Size = CGM.getTargetData().getTypeAllocSize(SrcTy);
+    unsigned Size = CGM.getDataLayout().getTypeAllocSize(SrcTy);
     assert(Size <= 8 && "does not support size > 8");
     src = (Size == 4) ? CGF.Builder.CreateBitCast(src, ObjCTypes.IntTy)
       : CGF.Builder.CreateBitCast(src, ObjCTypes.LongLongTy);
@@ -3679,7 +4264,7 @@ void CGObjCCommonMac::EmitImageInfo() {
 static const int ModuleVersion = 7;
 
 void CGObjCMac::EmitModuleInfo() {
-  uint64_t Size = CGM.getTargetData().getTypeAllocSize(ObjCTypes.ModuleTy);
+  uint64_t Size = CGM.getDataLayout().getTypeAllocSize(ObjCTypes.ModuleTy);
 
   llvm::Constant *Values[] = {
     llvm::ConstantInt::get(ObjCTypes.LongTy, ModuleVersion),
@@ -3824,7 +4409,7 @@ void CGObjCCommonMac::BuildAggrIvarRecordLayout(const RecordType *RT,
     Fields.push_back(*i);
   llvm::Type *Ty = CGM.getTypes().ConvertType(QualType(RT, 0));
   const llvm::StructLayout *RecLayout =
-    CGM.getTargetData().getStructLayout(cast<llvm::StructType>(Ty));
+    CGM.getDataLayout().getStructLayout(cast<llvm::StructType>(Ty));
 
   BuildAggrIvarLayout(0, RecLayout, RD, Fields, BytePos,
                       ForStrongLayout, HasUnion);
@@ -3951,7 +4536,7 @@ void CGObjCCommonMac::BuildAggrIvarLayout(const ObjCImplementationDecl *OI,
       if (IsUnion) {
         // FIXME: Why the asymmetry? We divide by word size in bits on other
         // side.
-        uint64_t UnionIvarSize = FieldSize;
+        uint64_t UnionIvarSize = FieldSize / ByteSizeInBits;
         if (UnionIvarSize > MaxSkippedUnionIvarSize) {
           MaxSkippedUnionIvarSize = UnionIvarSize;
           MaxSkippedField = Field;
@@ -4005,7 +4590,7 @@ llvm::Constant *CGObjCCommonMac::BuildIvarLayoutBitmap(std::string &BitMap) {
   // Build the string of skip/scan nibbles
   SmallVector<SKIP_SCAN, 32> SkipScanIvars;
   unsigned int WordSize =
-  CGM.getTypes().getTargetData().getTypeAllocSize(PtrTy);
+  CGM.getTypes().getDataLayout().getTypeAllocSize(PtrTy);
   if (IvarsInfo[0].ivar_bytepos == 0) {
     WordsToSkip = 0;
     WordsToScan = IvarsInfo[0].ivar_size;
@@ -4187,7 +4772,7 @@ llvm::Constant *CGObjCCommonMac::BuildIvarLayout(
     printf("\n%s ivar layout for class '%s': ",
            ForStrongLayout ? "strong" : "weak",
            OMD->getClassInterface()->getName().data());
-    const unsigned char *s = (unsigned char*)BitMap.c_str();
+    const unsigned char *s = (const unsigned char*)BitMap.c_str();
     for (unsigned i = 0, e = BitMap.size(); i < e; i++)
       if (!(s[i] & 0xf0))
         printf("0x0%x%s", s[i], s[i] != 0 ? ", " : "");
@@ -4835,7 +5420,7 @@ AddModuleClassList(ArrayRef<llvm::GlobalValue*> Container,
                              llvm::GlobalValue::InternalLinkage,
                              Init,
                              SymbolName);
-  GV->setAlignment(CGM.getTargetData().getABITypeAlignment(Init->getType()));
+  GV->setAlignment(CGM.getDataLayout().getABITypeAlignment(Init->getType()));
   GV->setSection(SectionName);
   CGM.AddUsedGlobal(GV);
 }
@@ -4941,19 +5526,6 @@ bool CGObjCNonFragileABIMac::isVTableDispatchedSelector(Selector Sel) {
   return VTableDispatchMethods.count(Sel);
 }
 
-// Metadata flags
-enum MetaDataDlags {
-  CLS = 0x0,
-  CLS_META = 0x1,
-  CLS_ROOT = 0x2,
-  OBJC2_CLS_HIDDEN = 0x10,
-  CLS_EXCEPTION = 0x20,
-
-  /// (Obsolete) ARC-specific: this class has a .release_ivars method
-  CLS_HAS_IVAR_RELEASER = 0x40,
-  /// class was compiled with -fobjc-arr
-  CLS_COMPILED_BY_ARC = 0x80  // (1<<7)
-};
 /// BuildClassRoTInitializer - generate meta-data for:
 /// struct _class_ro_t {
 ///   uint32_t const flags;
@@ -4978,19 +5550,20 @@ llvm::GlobalVariable * CGObjCNonFragileABIMac::BuildClassRoTInitializer(
   llvm::Constant *Values[10]; // 11 for 64bit targets!
 
   if (CGM.getLangOpts().ObjCAutoRefCount)
-    flags |= CLS_COMPILED_BY_ARC;
+    flags |= NonFragileABI_Class_CompiledByARC;
 
   Values[ 0] = llvm::ConstantInt::get(ObjCTypes.IntTy, flags);
   Values[ 1] = llvm::ConstantInt::get(ObjCTypes.IntTy, InstanceStart);
   Values[ 2] = llvm::ConstantInt::get(ObjCTypes.IntTy, InstanceSize);
   // FIXME. For 64bit targets add 0 here.
-  Values[ 3] = (flags & CLS_META) ? GetIvarLayoutName(0, ObjCTypes)
+  Values[ 3] = (flags & NonFragileABI_Class_Meta)
+    ? GetIvarLayoutName(0, ObjCTypes)
     : BuildIvarLayout(ID, true);
   Values[ 4] = GetClassName(ID->getIdentifier());
   // const struct _method_list_t * const baseMethods;
   std::vector<llvm::Constant*> Methods;
   std::string MethodListName("\01l_OBJC_$_");
-  if (flags & CLS_META) {
+  if (flags & NonFragileABI_Class_Meta) {
     MethodListName += "CLASS_METHODS_" + ID->getNameAsString();
     for (ObjCImplementationDecl::classmeth_iterator
            i = ID->classmeth_begin(), e = ID->classmeth_end(); i != e; ++i) {
@@ -5030,28 +5603,27 @@ llvm::GlobalVariable * CGObjCNonFragileABIMac::BuildClassRoTInitializer(
                                 OID->all_referenced_protocol_begin(),
                                 OID->all_referenced_protocol_end());
 
-  if (flags & CLS_META)
+  if (flags & NonFragileABI_Class_Meta) {
     Values[ 7] = llvm::Constant::getNullValue(ObjCTypes.IvarListnfABIPtrTy);
-  else
-    Values[ 7] = EmitIvarList(ID);
-  Values[ 8] = (flags & CLS_META) ? GetIvarLayoutName(0, ObjCTypes)
-    : BuildIvarLayout(ID, false);
-  if (flags & CLS_META)
+    Values[ 8] = GetIvarLayoutName(0, ObjCTypes);
     Values[ 9] = llvm::Constant::getNullValue(ObjCTypes.PropertyListPtrTy);
-  else
+  } else {
+    Values[ 7] = EmitIvarList(ID);
+    Values[ 8] = BuildIvarLayout(ID, false);
     Values[ 9] = EmitPropertyList("\01l_OBJC_$_PROP_LIST_" + ID->getName(),
                                   ID, ID->getClassInterface(), ObjCTypes);
+  }
   llvm::Constant *Init = llvm::ConstantStruct::get(ObjCTypes.ClassRonfABITy,
                                                    Values);
   llvm::GlobalVariable *CLASS_RO_GV =
     new llvm::GlobalVariable(CGM.getModule(), ObjCTypes.ClassRonfABITy, false,
                              llvm::GlobalValue::InternalLinkage,
                              Init,
-                             (flags & CLS_META) ?
+                             (flags & NonFragileABI_Class_Meta) ?
                              std::string("\01l_OBJC_METACLASS_RO_$_")+ClassName :
                              std::string("\01l_OBJC_CLASS_RO_$_")+ClassName);
   CLASS_RO_GV->setAlignment(
-    CGM.getTargetData().getABITypeAlignment(ObjCTypes.ClassRonfABITy));
+    CGM.getDataLayout().getABITypeAlignment(ObjCTypes.ClassRonfABITy));
   CLASS_RO_GV->setSection("__DATA, __objc_const");
   return CLASS_RO_GV;
 
@@ -5088,7 +5660,7 @@ llvm::GlobalVariable * CGObjCNonFragileABIMac::BuildClassMetaData(
   GV->setInitializer(Init);
   GV->setSection("__DATA, __objc_data");
   GV->setAlignment(
-    CGM.getTargetData().getABITypeAlignment(ObjCTypes.ClassnfABITy));
+    CGM.getDataLayout().getABITypeAlignment(ObjCTypes.ClassnfABITy));
   if (HiddenVisibility)
     GV->setVisibility(llvm::GlobalValue::HiddenVisibility);
   return GV;
@@ -5138,23 +5710,31 @@ void CGObjCNonFragileABIMac::GenerateClass(const ObjCImplementationDecl *ID) {
          "CGObjCNonFragileABIMac::GenerateClass - class is 0");
   // FIXME: Is this correct (that meta class size is never computed)?
   uint32_t InstanceStart =
-    CGM.getTargetData().getTypeAllocSize(ObjCTypes.ClassnfABITy);
+    CGM.getDataLayout().getTypeAllocSize(ObjCTypes.ClassnfABITy);
   uint32_t InstanceSize = InstanceStart;
-  uint32_t flags = CLS_META;
+  uint32_t flags = NonFragileABI_Class_Meta;
   std::string ObjCMetaClassName(getMetaclassSymbolPrefix());
   std::string ObjCClassName(getClassSymbolPrefix());
 
   llvm::GlobalVariable *SuperClassGV, *IsAGV;
 
+  // Build the flags for the metaclass.
   bool classIsHidden =
     ID->getClassInterface()->getVisibility() == HiddenVisibility;
   if (classIsHidden)
-    flags |= OBJC2_CLS_HIDDEN;
-  if (ID->hasCXXStructors())
-    flags |= eClassFlags_ABI2_HasCXXStructors;
+    flags |= NonFragileABI_Class_Hidden;
+
+  // FIXME: why is this flag set on the metaclass?
+  // ObjC metaclasses have no fields and don't really get constructed.
+  if (ID->hasNonZeroConstructors() || ID->hasDestructors()) {
+    flags |= NonFragileABI_Class_HasCXXStructors;
+    if (!ID->hasNonZeroConstructors())
+      flags |= NonFragileABI_Class_HasCXXDestructorOnly;  
+  }
+
   if (!ID->getClassInterface()->getSuperClass()) {
     // class is root
-    flags |= CLS_ROOT;
+    flags |= NonFragileABI_Class_Root;
     SuperClassGV = GetClassGlobal(ObjCClassName + ClassName);
     IsAGV = GetClassGlobal(ObjCMetaClassName + ClassName);
   } else {
@@ -5183,17 +5763,28 @@ void CGObjCNonFragileABIMac::GenerateClass(const ObjCImplementationDecl *ID) {
   DefinedMetaClasses.push_back(MetaTClass);
 
   // Metadata for the class
-  flags = CLS;
+  flags = 0;
   if (classIsHidden)
-    flags |= OBJC2_CLS_HIDDEN;
-  if (ID->hasCXXStructors())
-    flags |= eClassFlags_ABI2_HasCXXStructors;
+    flags |= NonFragileABI_Class_Hidden;
+
+  if (ID->hasNonZeroConstructors() || ID->hasDestructors()) {
+    flags |= NonFragileABI_Class_HasCXXStructors;
+
+    // Set a flag to enable a runtime optimization when a class has
+    // fields that require destruction but which don't require
+    // anything except zero-initialization during construction.  This
+    // is most notably true of __strong and __weak types, but you can
+    // also imagine there being C++ types with non-trivial default
+    // constructors that merely set all fields to null.
+    if (!ID->hasNonZeroConstructors())
+      flags |= NonFragileABI_Class_HasCXXDestructorOnly;
+  }
 
   if (hasObjCExceptionAttribute(CGM.getContext(), ID->getClassInterface()))
-    flags |= CLS_EXCEPTION;
+    flags |= NonFragileABI_Class_Exception;
 
   if (!ID->getClassInterface()->getSuperClass()) {
-    flags |= CLS_ROOT;
+    flags |= NonFragileABI_Class_Root;
     SuperClassGV = 0;
   } else {
     // Has a root. Current class is not a root.
@@ -5220,7 +5811,7 @@ void CGObjCNonFragileABIMac::GenerateClass(const ObjCImplementationDecl *ID) {
     DefinedNonLazyClasses.push_back(ClassMD);
 
   // Force the definition of the EHType if necessary.
-  if (flags & CLS_EXCEPTION)
+  if (flags & NonFragileABI_Class_Exception)
     GetInterfaceEHType(ID->getClassInterface(), true);
   // Make sure method definition entries are all clear for next implementation.
   MethodDefinitions.clear();
@@ -5344,7 +5935,7 @@ void CGObjCNonFragileABIMac::GenerateCategory(const ObjCCategoryImplDecl *OCD) {
                                Init,
                                ExtCatName);
   GCATV->setAlignment(
-    CGM.getTargetData().getABITypeAlignment(ObjCTypes.CategorynfABITy));
+    CGM.getDataLayout().getABITypeAlignment(ObjCTypes.CategorynfABITy));
   GCATV->setSection("__DATA, __objc_const");
   CGM.AddUsedGlobal(GCATV);
   DefinedCategories.push_back(GCATV);
@@ -5391,7 +5982,7 @@ CGObjCNonFragileABIMac::EmitMethodList(Twine Name,
 
   llvm::Constant *Values[3];
   // sizeof(struct _objc_method)
-  unsigned Size = CGM.getTargetData().getTypeAllocSize(ObjCTypes.MethodTy);
+  unsigned Size = CGM.getDataLayout().getTypeAllocSize(ObjCTypes.MethodTy);
   Values[0] = llvm::ConstantInt::get(ObjCTypes.IntTy, Size);
   // method_count
   Values[1] = llvm::ConstantInt::get(ObjCTypes.IntTy, Methods.size());
@@ -5403,7 +5994,7 @@ CGObjCNonFragileABIMac::EmitMethodList(Twine Name,
   llvm::GlobalVariable *GV =
     new llvm::GlobalVariable(CGM.getModule(), Init->getType(), false,
                              llvm::GlobalValue::InternalLinkage, Init, Name);
-  GV->setAlignment(CGM.getTargetData().getABITypeAlignment(Init->getType()));
+  GV->setAlignment(CGM.getDataLayout().getABITypeAlignment(Init->getType()));
   GV->setSection(Section);
   CGM.AddUsedGlobal(GV);
   return llvm::ConstantExpr::getBitCast(GV, ObjCTypes.MethodListnfABIPtrTy);
@@ -5437,7 +6028,7 @@ CGObjCNonFragileABIMac::EmitIvarOffsetVar(const ObjCInterfaceDecl *ID,
   IvarOffsetGV->setInitializer(llvm::ConstantInt::get(ObjCTypes.LongTy,
                                                       Offset));
   IvarOffsetGV->setAlignment(
-    CGM.getTargetData().getABITypeAlignment(ObjCTypes.LongTy));
+    CGM.getDataLayout().getABITypeAlignment(ObjCTypes.LongTy));
 
   // FIXME: This matches gcc, but shouldn't the visibility be set on the use as
   // well (i.e., in ObjCIvarOffsetVariable).
@@ -5490,7 +6081,7 @@ llvm::Constant *CGObjCNonFragileABIMac::EmitIvarList(
     Ivar[2] = GetMethodVarType(IVD);
     llvm::Type *FieldTy =
       CGM.getTypes().ConvertTypeForMem(IVD->getType());
-    unsigned Size = CGM.getTargetData().getTypeAllocSize(FieldTy);
+    unsigned Size = CGM.getDataLayout().getTypeAllocSize(FieldTy);
     unsigned Align = CGM.getContext().getPreferredTypeAlign(
       IVD->getType().getTypePtr()) >> 3;
     Align = llvm::Log2_32(Align);
@@ -5508,7 +6099,7 @@ llvm::Constant *CGObjCNonFragileABIMac::EmitIvarList(
     return llvm::Constant::getNullValue(ObjCTypes.IvarListnfABIPtrTy);
 
   llvm::Constant *Values[3];
-  unsigned Size = CGM.getTargetData().getTypeAllocSize(ObjCTypes.IvarnfABITy);
+  unsigned Size = CGM.getDataLayout().getTypeAllocSize(ObjCTypes.IvarnfABITy);
   Values[0] = llvm::ConstantInt::get(ObjCTypes.IntTy, Size);
   Values[1] = llvm::ConstantInt::get(ObjCTypes.IntTy, Ivars.size());
   llvm::ArrayType *AT = llvm::ArrayType::get(ObjCTypes.IvarnfABITy,
@@ -5522,7 +6113,7 @@ llvm::Constant *CGObjCNonFragileABIMac::EmitIvarList(
                              Init,
                              Prefix + OID->getName());
   GV->setAlignment(
-    CGM.getTargetData().getABITypeAlignment(Init->getType()));
+    CGM.getDataLayout().getABITypeAlignment(Init->getType()));
   GV->setSection("__DATA, __objc_const");
 
   CGM.AddUsedGlobal(GV);
@@ -5644,7 +6235,7 @@ llvm::Constant *CGObjCNonFragileABIMac::GetOrEmitProtocol(
   Values[7] = EmitPropertyList("\01l_OBJC_$_PROP_LIST_" + PD->getName(),
                                0, PD, ObjCTypes);
   uint32_t Size =
-    CGM.getTargetData().getTypeAllocSize(ObjCTypes.ProtocolnfABITy);
+    CGM.getDataLayout().getTypeAllocSize(ObjCTypes.ProtocolnfABITy);
   Values[8] = llvm::ConstantInt::get(ObjCTypes.IntTy, Size);
   Values[9] = llvm::Constant::getNullValue(ObjCTypes.IntTy);
   Values[10] = EmitProtocolMethodTypes("\01l_OBJC_$_PROTOCOL_METHOD_TYPES_"
@@ -5663,7 +6254,7 @@ llvm::Constant *CGObjCNonFragileABIMac::GetOrEmitProtocol(
                                false, llvm::GlobalValue::WeakAnyLinkage, Init,
                                "\01l_OBJC_PROTOCOL_$_" + PD->getName());
     Entry->setAlignment(
-      CGM.getTargetData().getABITypeAlignment(ObjCTypes.ProtocolnfABITy));
+      CGM.getDataLayout().getABITypeAlignment(ObjCTypes.ProtocolnfABITy));
     Entry->setSection("__DATA,__datacoal_nt,coalesced");
 
     Protocols[PD->getIdentifier()] = Entry;
@@ -5678,7 +6269,7 @@ llvm::Constant *CGObjCNonFragileABIMac::GetOrEmitProtocol(
                              false, llvm::GlobalValue::WeakAnyLinkage, Entry,
                              "\01l_OBJC_LABEL_PROTOCOL_$_" + PD->getName());
   PTGV->setAlignment(
-    CGM.getTargetData().getABITypeAlignment(ObjCTypes.ProtocolnfABIPtrTy));
+    CGM.getDataLayout().getABITypeAlignment(ObjCTypes.ProtocolnfABIPtrTy));
   PTGV->setSection("__DATA, __objc_protolist, coalesced, no_dead_strip");
   PTGV->setVisibility(llvm::GlobalValue::HiddenVisibility);
   CGM.AddUsedGlobal(PTGV);
@@ -5732,7 +6323,7 @@ CGObjCNonFragileABIMac::EmitProtocolList(Twine Name,
                                 Init, Name);
   GV->setSection("__DATA, __objc_const");
   GV->setAlignment(
-    CGM.getTargetData().getABITypeAlignment(Init->getType()));
+    CGM.getDataLayout().getABITypeAlignment(Init->getType()));
   CGM.AddUsedGlobal(GV);
   return llvm::ConstantExpr::getBitCast(GV,
                                         ObjCTypes.ProtocolListnfABIPtrTy);
@@ -5970,7 +6561,7 @@ llvm::Value *CGObjCNonFragileABIMac::EmitClassRefFromId(CGBuilderTy &Builder,
                              ClassGV,
                              "\01L_OBJC_CLASSLIST_REFERENCES_$_");
     Entry->setAlignment(
-                        CGM.getTargetData().getABITypeAlignment(
+                        CGM.getDataLayout().getABITypeAlignment(
                                                                 ObjCTypes.ClassnfABIPtrTy));
     Entry->setSection("__DATA, __objc_classrefs, regular, no_dead_strip");
     CGM.AddUsedGlobal(Entry);
@@ -6004,7 +6595,7 @@ CGObjCNonFragileABIMac::EmitSuperClassRef(CGBuilderTy &Builder,
                                ClassGV,
                                "\01L_OBJC_CLASSLIST_SUP_REFS_$_");
     Entry->setAlignment(
-      CGM.getTargetData().getABITypeAlignment(
+      CGM.getDataLayout().getABITypeAlignment(
         ObjCTypes.ClassnfABIPtrTy));
     Entry->setSection("__DATA, __objc_superrefs, regular, no_dead_strip");
     CGM.AddUsedGlobal(Entry);
@@ -6030,7 +6621,7 @@ llvm::Value *CGObjCNonFragileABIMac::EmitMetaClassRef(CGBuilderTy &Builder,
                              MetaClassGV,
                              "\01L_OBJC_CLASSLIST_SUP_REFS_$_");
   Entry->setAlignment(
-    CGM.getTargetData().getABITypeAlignment(
+    CGM.getDataLayout().getABITypeAlignment(
       ObjCTypes.ClassnfABIPtrTy));
 
   Entry->setSection("__DATA, __objc_superrefs, regular, no_dead_strip");
@@ -6079,16 +6670,9 @@ CGObjCNonFragileABIMac::GenerateMessageSendSuper(CodeGen::CodeGenFunction &CGF,
 
   // If this is a class message the metaclass is passed as the target.
   llvm::Value *Target;
-  if (IsClassMessage) {
-    if (isCategoryImpl) {
-      // Message sent to "super' in a class method defined in
-      // a category implementation.
-      Target = EmitClassRef(CGF.Builder, Class);
-      Target = CGF.Builder.CreateStructGEP(Target, 0);
-      Target = CGF.Builder.CreateLoad(Target);
-    } else
+  if (IsClassMessage)
       Target = EmitMetaClassRef(CGF.Builder, Class);
-  } else
+  else
     Target = EmitSuperClassRef(CGF.Builder, Class);
 
   // FIXME: We shouldn't need to do this cast, rectify the ASTContext and
@@ -6143,7 +6727,7 @@ void CGObjCNonFragileABIMac::EmitObjCIvarAssign(CodeGen::CodeGenFunction &CGF,
                                                 llvm::Value *ivarOffset) {
   llvm::Type * SrcTy = src->getType();
   if (!isa<llvm::PointerType>(SrcTy)) {
-    unsigned Size = CGM.getTargetData().getTypeAllocSize(SrcTy);
+    unsigned Size = CGM.getDataLayout().getTypeAllocSize(SrcTy);
     assert(Size <= 8 && "does not support size > 8");
     src = (Size == 4 ? CGF.Builder.CreateBitCast(src, ObjCTypes.IntTy)
            : CGF.Builder.CreateBitCast(src, ObjCTypes.LongTy));
@@ -6164,7 +6748,7 @@ void CGObjCNonFragileABIMac::EmitObjCStrongCastAssign(
   llvm::Value *src, llvm::Value *dst) {
   llvm::Type * SrcTy = src->getType();
   if (!isa<llvm::PointerType>(SrcTy)) {
-    unsigned Size = CGM.getTargetData().getTypeAllocSize(SrcTy);
+    unsigned Size = CGM.getDataLayout().getTypeAllocSize(SrcTy);
     assert(Size <= 8 && "does not support size > 8");
     src = (Size == 4 ? CGF.Builder.CreateBitCast(src, ObjCTypes.IntTy)
            : CGF.Builder.CreateBitCast(src, ObjCTypes.LongTy));
@@ -6211,7 +6795,7 @@ void CGObjCNonFragileABIMac::EmitObjCWeakAssign(CodeGen::CodeGenFunction &CGF,
                                                 llvm::Value *src, llvm::Value *dst) {
   llvm::Type * SrcTy = src->getType();
   if (!isa<llvm::PointerType>(SrcTy)) {
-    unsigned Size = CGM.getTargetData().getTypeAllocSize(SrcTy);
+    unsigned Size = CGM.getDataLayout().getTypeAllocSize(SrcTy);
     assert(Size <= 8 && "does not support size > 8");
     src = (Size == 4 ? CGF.Builder.CreateBitCast(src, ObjCTypes.IntTy)
            : CGF.Builder.CreateBitCast(src, ObjCTypes.LongTy));
@@ -6232,7 +6816,7 @@ void CGObjCNonFragileABIMac::EmitObjCGlobalAssign(CodeGen::CodeGenFunction &CGF,
                                           bool threadlocal) {
   llvm::Type * SrcTy = src->getType();
   if (!isa<llvm::PointerType>(SrcTy)) {
-    unsigned Size = CGM.getTargetData().getTypeAllocSize(SrcTy);
+    unsigned Size = CGM.getDataLayout().getTypeAllocSize(SrcTy);
     assert(Size <= 8 && "does not support size > 8");
     src = (Size == 4 ? CGF.Builder.CreateBitCast(src, ObjCTypes.IntTy)
            : CGF.Builder.CreateBitCast(src, ObjCTypes.LongTy));
@@ -6364,7 +6948,7 @@ CGObjCNonFragileABIMac::GetInterfaceEHType(const ObjCInterfaceDecl *ID,
 
   if (CGM.getLangOpts().getVisibilityMode() == HiddenVisibility)
     Entry->setVisibility(llvm::GlobalValue::HiddenVisibility);
-  Entry->setAlignment(CGM.getTargetData().getABITypeAlignment(
+  Entry->setAlignment(CGM.getDataLayout().getABITypeAlignment(
       ObjCTypes.EHTypeTy));
 
   if (ForDefinition) {
diff --git a/lib/CodeGen/CGObjCRuntime.cpp b/lib/CodeGen/CGObjCRuntime.cpp
index 9aa6837..6932dd7 100644
--- a/lib/CodeGen/CGObjCRuntime.cpp
+++ b/lib/CodeGen/CGObjCRuntime.cpp
@@ -78,6 +78,13 @@ uint64_t CGObjCRuntime::ComputeIvarBaseOffset(CodeGen::CodeGenModule &CGM,
     CGM.getContext().getCharWidth();
 }
 
+unsigned CGObjCRuntime::ComputeBitfieldBitOffset(
+    CodeGen::CodeGenModule &CGM,
+    const ObjCInterfaceDecl *ID,
+    const ObjCIvarDecl *Ivar) {
+  return LookupFieldBitOffset(CGM, ID, ID->getImplementation(), Ivar);
+}
+
 LValue CGObjCRuntime::EmitValueForIvarAtOffset(CodeGen::CodeGenFunction &CGF,
                                                const ObjCInterfaceDecl *OID,
                                                llvm::Value *BaseValue,
diff --git a/lib/CodeGen/CGObjCRuntime.h b/lib/CodeGen/CGObjCRuntime.h
index 219a3e4..3e77875 100644
--- a/lib/CodeGen/CGObjCRuntime.h
+++ b/lib/CodeGen/CGObjCRuntime.h
@@ -261,6 +261,8 @@ public:
                                         llvm::Value *Size) = 0;
   virtual llvm::Constant *BuildGCBlockLayout(CodeGen::CodeGenModule &CGM,
                                   const CodeGen::CGBlockInfo &blockInfo) = 0;
+  virtual llvm::Constant *BuildRCBlockLayout(CodeGen::CodeGenModule &CGM,
+                                  const CodeGen::CGBlockInfo &blockInfo) = 0;
   virtual llvm::GlobalVariable *GetClassGlobal(const std::string &Name) = 0;
 
   struct MessageSendInfo {
@@ -275,6 +277,12 @@ public:
   MessageSendInfo getMessageSendInfo(const ObjCMethodDecl *method,
                                      QualType resultType,
                                      CallArgList &callArgs);
+
+  // FIXME: This probably shouldn't be here, but the code to compute
+  // it is here.
+  unsigned ComputeBitfieldBitOffset(CodeGen::CodeGenModule &CGM,
+                                    const ObjCInterfaceDecl *ID,
+                                    const ObjCIvarDecl *Ivar);
 };
 
 /// Creates an instance of an Objective-C runtime class.
diff --git a/lib/CodeGen/CGRTTI.cpp b/lib/CodeGen/CGRTTI.cpp
index d1b370a..7c83d39 100644
--- a/lib/CodeGen/CGRTTI.cpp
+++ b/lib/CodeGen/CGRTTI.cpp
@@ -105,7 +105,6 @@ public:
   /// BuildTypeInfo - Build the RTTI type info struct for the given type.
   ///
   /// \param Force - true to force the creation of this RTTI value
-  /// \param ForEH - true if this is for exception handling
   llvm::Constant *BuildTypeInfo(QualType Ty, bool Force = false);
 };
 }
@@ -779,28 +778,24 @@ static unsigned ComputeVMIClassTypeInfoFlags(const CXXBaseSpecifier *Base,
     cast<CXXRecordDecl>(Base->getType()->getAs<RecordType>()->getDecl());
   
   if (Base->isVirtual()) {
-    if (Bases.VirtualBases.count(BaseDecl)) {
+    // Mark the virtual base as seen.
+    if (!Bases.VirtualBases.insert(BaseDecl)) {
       // If this virtual base has been seen before, then the class is diamond
       // shaped.
       Flags |= RTTIBuilder::VMI_DiamondShaped;
     } else {
       if (Bases.NonVirtualBases.count(BaseDecl))
         Flags |= RTTIBuilder::VMI_NonDiamondRepeat;
-
-      // Mark the virtual base as seen.
-      Bases.VirtualBases.insert(BaseDecl);
     }
   } else {
-    if (Bases.NonVirtualBases.count(BaseDecl)) {
+    // Mark the non-virtual base as seen.
+    if (!Bases.NonVirtualBases.insert(BaseDecl)) {
       // If this non-virtual base has been seen before, then the class has non-
       // diamond shaped repeated inheritance.
       Flags |= RTTIBuilder::VMI_NonDiamondRepeat;
     } else {
       if (Bases.VirtualBases.count(BaseDecl))
         Flags |= RTTIBuilder::VMI_NonDiamondRepeat;
-        
-      // Mark the non-virtual base as seen.
-      Bases.NonVirtualBases.insert(BaseDecl);
     }
   }
 
@@ -891,7 +886,7 @@ void RTTIBuilder::BuildVMIClassTypeInfo(const CXXRecordDecl *RD) {
       Offset = Layout.getBaseClassOffset(BaseDecl);
     };
     
-    OffsetFlags = Offset.getQuantity() << 8;
+    OffsetFlags = uint64_t(Offset.getQuantity()) << 8;
     
     // The low-order byte of __offset_flags contains flags, as given by the 
     // masks from the enumeration __offset_flags_masks.
@@ -982,7 +977,7 @@ llvm::Constant *CodeGenModule::GetAddrOfRTTIDescriptor(QualType Ty,
   // Return a bogus pointer if RTTI is disabled, unless it's for EH.
   // FIXME: should we even be calling this method if RTTI is disabled
   // and it's not for EH?
-  if (!ForEH && !getContext().getLangOpts().RTTI)
+  if (!ForEH && !getLangOpts().RTTI)
     return llvm::Constant::getNullValue(Int8PtrTy);
   
   if (ForEH && Ty->isObjCObjectPointerType() &&
diff --git a/lib/CodeGen/CGRecordLayout.h b/lib/CodeGen/CGRecordLayout.h
index 94c822f..3db5e04 100644
--- a/lib/CodeGen/CGRecordLayout.h
+++ b/lib/CodeGen/CGRecordLayout.h
@@ -166,8 +166,8 @@ public:
 class CGRecordLayout {
   friend class CodeGenTypes;
 
-  CGRecordLayout(const CGRecordLayout&); // DO NOT IMPLEMENT
-  void operator=(const CGRecordLayout&); // DO NOT IMPLEMENT
+  CGRecordLayout(const CGRecordLayout &) LLVM_DELETED_FUNCTION;
+  void operator=(const CGRecordLayout &) LLVM_DELETED_FUNCTION;
 
 private:
   /// The LLVM type corresponding to this record layout; used when
diff --git a/lib/CodeGen/CGRecordLayoutBuilder.cpp b/lib/CodeGen/CGRecordLayoutBuilder.cpp
index d642ef8..26ef3ef 100644
--- a/lib/CodeGen/CGRecordLayoutBuilder.cpp
+++ b/lib/CodeGen/CGRecordLayoutBuilder.cpp
@@ -25,7 +25,7 @@
 #include "llvm/Type.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/raw_ostream.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 using namespace clang;
 using namespace CodeGen;
 
@@ -206,7 +206,7 @@ void CGRecordLayoutBuilder::Layout(const RecordDecl *D) {
   Alignment = Types.getContext().getASTRecordLayout(D).getAlignment();
   Packed = D->hasAttr<PackedAttr>();
   
-  IsMsStruct = D->hasAttr<MsStructAttr>();
+  IsMsStruct = D->isMsStruct(Types.getContext());
 
   if (D->isUnion()) {
     LayoutUnion(D);
@@ -239,7 +239,7 @@ CGBitFieldInfo CGBitFieldInfo::MakeInfo(CodeGenTypes &Types,
 
   llvm::Type *Ty = Types.ConvertTypeForMem(FD->getType());
   CharUnits TypeSizeInBytes =
-    CharUnits::fromQuantity(Types.getTargetData().getTypeAllocSize(Ty));
+    CharUnits::fromQuantity(Types.getDataLayout().getTypeAllocSize(Ty));
   uint64_t TypeSizeInBits = Types.getContext().toBits(TypeSizeInBytes);
 
   bool IsSigned = FD->getType()->isSignedIntegerOrEnumerationType();
@@ -259,7 +259,7 @@ CGBitFieldInfo CGBitFieldInfo::MakeInfo(CodeGenTypes &Types,
 
   // in big-endian machines the first fields are in higher bit positions,
   // so revert the offset. The byte offsets are reversed(back) later.
-  if (Types.getTargetData().isBigEndian()) {
+  if (Types.getDataLayout().isBigEndian()) {
     FieldOffset = ((ContainingTypeSizeInBits)-FieldOffset-FieldSize);
   }
 
@@ -334,7 +334,7 @@ CGBitFieldInfo CGBitFieldInfo::MakeInfo(CodeGenTypes &Types,
     // on big-endian machines we reverted the bit offset because first fields are
     // in higher bits. But this also reverts the bytes, so fix this here by reverting
     // the byte offset on big-endian machines.
-    if (Types.getTargetData().isBigEndian()) {
+    if (Types.getDataLayout().isBigEndian()) {
       AI.FieldByteOffset = Types.getContext().toCharUnitsFromBits(
           ContainingTypeSizeInBits - AccessStart - AccessWidth);
     } else {
@@ -553,9 +553,9 @@ void CGRecordLayoutBuilder::LayoutUnion(const RecordDecl *D) {
     hasOnlyZeroSizedBitFields = false;
 
     CharUnits fieldAlign = CharUnits::fromQuantity(
-                          Types.getTargetData().getABITypeAlignment(fieldType));
+                          Types.getDataLayout().getABITypeAlignment(fieldType));
     CharUnits fieldSize = CharUnits::fromQuantity(
-                             Types.getTargetData().getTypeAllocSize(fieldType));
+                             Types.getDataLayout().getTypeAllocSize(fieldType));
 
     if (fieldAlign < unionAlign)
       continue;
@@ -884,7 +884,7 @@ void CGRecordLayoutBuilder::AppendTailPadding(CharUnits RecordSize) {
 void CGRecordLayoutBuilder::AppendField(CharUnits fieldOffset,
                                         llvm::Type *fieldType) {
   CharUnits fieldSize =
-    CharUnits::fromQuantity(Types.getTargetData().getTypeAllocSize(fieldType));
+    CharUnits::fromQuantity(Types.getDataLayout().getTypeAllocSize(fieldType));
 
   FieldTypes.push_back(fieldType);
 
@@ -957,7 +957,7 @@ CharUnits CGRecordLayoutBuilder::getTypeAlignment(llvm::Type *Ty) const {
   if (Packed)
     return CharUnits::One();
 
-  return CharUnits::fromQuantity(Types.getTargetData().getABITypeAlignment(Ty));
+  return CharUnits::fromQuantity(Types.getDataLayout().getABITypeAlignment(Ty));
 }
 
 CharUnits CGRecordLayoutBuilder::getAlignmentAsLLVMStruct() const {
@@ -1036,7 +1036,7 @@ CGRecordLayout *CodeGenTypes::ComputeRecordLayout(const RecordDecl *D,
   const ASTRecordLayout &Layout = getContext().getASTRecordLayout(D);
 
   uint64_t TypeSizeInBits = getContext().toBits(Layout.getSize());
-  assert(TypeSizeInBits == getTargetData().getTypeAllocSizeInBits(Ty) &&
+  assert(TypeSizeInBits == getDataLayout().getTypeAllocSizeInBits(Ty) &&
          "Type size mismatch!");
 
   if (BaseTy) {
@@ -1049,19 +1049,19 @@ CGRecordLayout *CodeGenTypes::ComputeRecordLayout(const RecordDecl *D,
       getContext().toBits(AlignedNonVirtualTypeSize);
 
     assert(AlignedNonVirtualTypeSizeInBits == 
-           getTargetData().getTypeAllocSizeInBits(BaseTy) &&
+           getDataLayout().getTypeAllocSizeInBits(BaseTy) &&
            "Type size mismatch!");
   }
                                      
   // Verify that the LLVM and AST field offsets agree.
   llvm::StructType *ST =
     dyn_cast<llvm::StructType>(RL->getLLVMType());
-  const llvm::StructLayout *SL = getTargetData().getStructLayout(ST);
+  const llvm::StructLayout *SL = getDataLayout().getStructLayout(ST);
 
   const ASTRecordLayout &AST_RL = getContext().getASTRecordLayout(D);
   RecordDecl::field_iterator it = D->field_begin();
   const FieldDecl *LastFD = 0;
-  bool IsMsStruct = D->hasAttr<MsStructAttr>();
+  bool IsMsStruct = D->isMsStruct(getContext());
   for (unsigned i = 0, e = AST_RL.getFieldCount(); i != e; ++i, ++it) {
     const FieldDecl *FD = *it;
 
diff --git a/lib/CodeGen/CGStmt.cpp b/lib/CodeGen/CGStmt.cpp
index d78908d..3548dba 100644
--- a/lib/CodeGen/CGStmt.cpp
+++ b/lib/CodeGen/CGStmt.cpp
@@ -21,7 +21,7 @@
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/InlineAsm.h"
 #include "llvm/Intrinsics.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 using namespace clang;
 using namespace CodeGen;
 
@@ -132,8 +132,8 @@ void CodeGenFunction::EmitStmt(const Stmt *S) {
   case Stmt::ReturnStmtClass:   EmitReturnStmt(cast<ReturnStmt>(*S));     break;
 
   case Stmt::SwitchStmtClass:   EmitSwitchStmt(cast<SwitchStmt>(*S));     break;
-  case Stmt::AsmStmtClass:      EmitAsmStmt(cast<AsmStmt>(*S));           break;
-  case Stmt::MSAsmStmtClass:    EmitMSAsmStmt(cast<MSAsmStmt>(*S));       break;
+  case Stmt::GCCAsmStmtClass:   // Intentional fall-through.
+  case Stmt::MSAsmStmtClass:    EmitAsmStmt(cast<AsmStmt>(*S));           break;
 
   case Stmt::ObjCAtTryStmtClass:
     EmitObjCAtTryStmt(cast<ObjCAtTryStmt>(*S));
@@ -237,6 +237,10 @@ void CodeGenFunction::SimplifyForwardingBlocks(llvm::BasicBlock *BB) {
   if (!BI || !BI->isUnconditional())
     return;
 
+  // Can only simplify empty blocks.
+  if (BI != BB->begin())
+    return;
+
   BB->replaceAllUsesWith(BI->getSuccessor(0));
   BI->eraseFromParent();
   BB->eraseFromParent();
@@ -743,6 +747,17 @@ void CodeGenFunction::EmitReturnStmt(const ReturnStmt &S) {
   // Emit the result value, even if unused, to evalute the side effects.
   const Expr *RV = S.getRetValue();
 
+  // Treat block literals in a return expression as if they appeared
+  // in their own scope.  This permits a small, easily-implemented
+  // exception to our over-conservative rules about not jumping to
+  // statements following block literals with non-trivial cleanups.
+  RunCleanupsScope cleanupScope(*this);
+  if (const ExprWithCleanups *cleanups =
+        dyn_cast_or_null<ExprWithCleanups>(RV)) {
+    enterFullExpression(cleanups);
+    RV = cleanups->getSubExpr();
+  }
+
   // FIXME: Clean this up by using an LValue for ReturnTemp,
   // EmitStoreThroughLValue, and EmitAnyExpr.
   if (S.getNRVOCandidate() && S.getNRVOCandidate()->isNRVOVariable() &&
@@ -779,6 +794,7 @@ void CodeGenFunction::EmitReturnStmt(const ReturnStmt &S) {
                                           AggValueSlot::IsNotAliased));
   }
 
+  cleanupScope.ForceCleanup();
   EmitBranchThroughCleanup(ReturnBlock);
 }
 
@@ -899,7 +915,8 @@ void CodeGenFunction::EmitCaseStmt(const CaseStmt &S) {
 
   // If the body of the case is just a 'break', and if there was no fallthrough,
   // try to not emit an empty block.
-  if ((CGM.getCodeGenOpts().OptimizationLevel > 0) && isa<BreakStmt>(S.getSubStmt())) {
+  if ((CGM.getCodeGenOpts().OptimizationLevel > 0) &&
+      isa<BreakStmt>(S.getSubStmt())) {
     JumpDest Block = BreakContinueStack.back().BreakBlock;
 
     // Only do this optimization if there are no cleanups that need emitting.
@@ -1263,6 +1280,10 @@ SimplifyConstraint(const char *Constraint, const TargetInfo &Target,
     case '=': // Will see this and the following in mult-alt constraints.
     case '+':
       break;
+    case '#': // Ignore the rest of the constraint alternative.
+      while (Constraint[1] && Constraint[1] != ',')
+	Constraint++;
+      break;
     case ',':
       Result += "|";
       break;
@@ -1323,8 +1344,7 @@ AddVariableConstraints(const std::string &Constraint, const Expr &AsmExpr,
 }
 
 llvm::Value*
-CodeGenFunction::EmitAsmInputLValue(const AsmStmt &S,
-                                    const TargetInfo::ConstraintInfo &Info,
+CodeGenFunction::EmitAsmInputLValue(const TargetInfo::ConstraintInfo &Info,
                                     LValue InputValue, QualType InputType,
                                     std::string &ConstraintStr) {
   llvm::Value *Arg;
@@ -1333,7 +1353,7 @@ CodeGenFunction::EmitAsmInputLValue(const AsmStmt &S,
       Arg = EmitLoadOfLValue(InputValue).getScalarVal();
     } else {
       llvm::Type *Ty = ConvertType(InputType);
-      uint64_t Size = CGM.getTargetData().getTypeSizeInBits(Ty);
+      uint64_t Size = CGM.getDataLayout().getTypeSizeInBits(Ty);
       if (Size <= 64 && llvm::isPowerOf2_64(Size)) {
         Ty = llvm::IntegerType::get(getLLVMContext(), Size);
         Ty = llvm::PointerType::getUnqual(Ty);
@@ -1353,7 +1373,7 @@ CodeGenFunction::EmitAsmInputLValue(const AsmStmt &S,
   return Arg;
 }
 
-llvm::Value* CodeGenFunction::EmitAsmInput(const AsmStmt &S,
+llvm::Value* CodeGenFunction::EmitAsmInput(
                                          const TargetInfo::ConstraintInfo &Info,
                                            const Expr *InputExpr,
                                            std::string &ConstraintStr) {
@@ -1363,7 +1383,7 @@ llvm::Value* CodeGenFunction::EmitAsmInput(const AsmStmt &S,
 
   InputExpr = InputExpr->IgnoreParenNoopCasts(getContext());
   LValue Dest = EmitLValue(InputExpr);
-  return EmitAsmInputLValue(S, Info, Dest, InputExpr->getType(), ConstraintStr);
+  return EmitAsmInputLValue(Info, Dest, InputExpr->getType(), ConstraintStr);
 }
 
 /// getAsmSrcLocInfo - Return the !srcloc metadata node to attach to an inline
@@ -1396,23 +1416,8 @@ static llvm::MDNode *getAsmSrcLocInfo(const StringLiteral *Str,
 }
 
 void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
-  // Analyze the asm string to decompose it into its pieces.  We know that Sema
-  // has already done this, so it is guaranteed to be successful.
-  SmallVector<AsmStmt::AsmStringPiece, 4> Pieces;
-  unsigned DiagOffs;
-  S.AnalyzeAsmString(Pieces, getContext(), DiagOffs);
-
-  // Assemble the pieces into the final asm string.
-  std::string AsmString;
-  for (unsigned i = 0, e = Pieces.size(); i != e; ++i) {
-    if (Pieces[i].isString())
-      AsmString += Pieces[i].getString();
-    else if (Pieces[i].getModifier() == '\0')
-      AsmString += '$' + llvm::utostr(Pieces[i].getOperandNo());
-    else
-      AsmString += "${" + llvm::utostr(Pieces[i].getOperandNo()) + ':' +
-                   Pieces[i].getModifier() + '}';
-  }
+  // Assemble the final asm string.
+  std::string AsmString = S.generateAsmString(getContext());
 
   // Get all the output and input constraints together.
   SmallVector<TargetInfo::ConstraintInfo, 4> OutputConstraintInfos;
@@ -1511,7 +1516,7 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
       InOutConstraints += ',';
 
       const Expr *InputExpr = S.getOutputExpr(i);
-      llvm::Value *Arg = EmitAsmInputLValue(S, Info, Dest, InputExpr->getType(),
+      llvm::Value *Arg = EmitAsmInputLValue(Info, Dest, InputExpr->getType(),
                                             InOutConstraints);
 
       if (llvm::Type* AdjTy =
@@ -1549,7 +1554,7 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
                             *InputExpr->IgnoreParenNoopCasts(getContext()),
                             Target, CGM, S);
 
-    llvm::Value *Arg = EmitAsmInput(S, Info, InputExpr, Constraints);
+    llvm::Value *Arg = EmitAsmInput(Info, InputExpr, Constraints);
 
     // If this input argument is tied to a larger output result, extend the
     // input to be the same size as the output.  The LLVM backend wants to see
@@ -1596,7 +1601,7 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
 
   // Clobbers
   for (unsigned i = 0, e = S.getNumClobbers(); i != e; i++) {
-    StringRef Clobber = S.getClobber(i)->getString();
+    StringRef Clobber = S.getClobber(i);
 
     if (Clobber != "memory" && Clobber != "cc")
     Clobber = Target.getNormalizedGCCRegisterName(Clobber);
@@ -1628,15 +1633,22 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
   llvm::FunctionType *FTy =
     llvm::FunctionType::get(ResultType, ArgTypes, false);
 
+  bool HasSideEffect = S.isVolatile() || S.getNumOutputs() == 0;
+  llvm::InlineAsm::AsmDialect AsmDialect = isa<MSAsmStmt>(&S) ?
+    llvm::InlineAsm::AD_Intel : llvm::InlineAsm::AD_ATT;
   llvm::InlineAsm *IA =
-    llvm::InlineAsm::get(FTy, AsmString, Constraints,
-                         S.isVolatile() || S.getNumOutputs() == 0);
+    llvm::InlineAsm::get(FTy, AsmString, Constraints, HasSideEffect,
+                         /* IsAlignStack */ false, AsmDialect);
   llvm::CallInst *Result = Builder.CreateCall(IA, Args);
-  Result->addAttribute(~0, llvm::Attribute::NoUnwind);
+  Result->addAttribute(llvm::AttrListPtr::FunctionIndex,
+                       llvm::Attributes::get(getLLVMContext(),
+                                             llvm::Attributes::NoUnwind));
 
   // Slap the source location of the inline asm into a !srcloc metadata on the
-  // call.
-  Result->setMetadata("srcloc", getAsmSrcLocInfo(S.getAsmString(), *this));
+  // call.  FIXME: Handle metadata for MS-style inline asms.
+  if (const GCCAsmStmt *gccAsmStmt = dyn_cast<GCCAsmStmt>(&S))
+    Result->setMetadata("srcloc", getAsmSrcLocInfo(gccAsmStmt->getAsmString(),
+                                                   *this));
 
   // Extract all of the register value results from the asm.
   std::vector<llvm::Value*> RegResults;
@@ -1662,12 +1674,12 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
       if (TruncTy->isFloatingPointTy())
         Tmp = Builder.CreateFPTrunc(Tmp, TruncTy);
       else if (TruncTy->isPointerTy() && Tmp->getType()->isIntegerTy()) {
-        uint64_t ResSize = CGM.getTargetData().getTypeSizeInBits(TruncTy);
+        uint64_t ResSize = CGM.getDataLayout().getTypeSizeInBits(TruncTy);
         Tmp = Builder.CreateTrunc(Tmp,
                    llvm::IntegerType::get(getLLVMContext(), (unsigned)ResSize));
         Tmp = Builder.CreateIntToPtr(Tmp, TruncTy);
       } else if (Tmp->getType()->isPointerTy() && TruncTy->isIntegerTy()) {
-        uint64_t TmpSize =CGM.getTargetData().getTypeSizeInBits(Tmp->getType());
+        uint64_t TmpSize =CGM.getDataLayout().getTypeSizeInBits(Tmp->getType());
         Tmp = Builder.CreatePtrToInt(Tmp,
                    llvm::IntegerType::get(getLLVMContext(), (unsigned)TmpSize));
         Tmp = Builder.CreateTrunc(Tmp, TruncTy);
@@ -1681,47 +1693,3 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
     EmitStoreThroughLValue(RValue::get(Tmp), ResultRegDests[i]);
   }
 }
-
-void CodeGenFunction::EmitMSAsmStmt(const MSAsmStmt &S) {
-  // MS-style inline assembly is not fully supported, so sema emits a warning.
-  if (!CGM.getCodeGenOpts().EmitMicrosoftInlineAsm)
-    return;
-
-  assert (S.isSimple() && "CodeGen can only handle simple MSAsmStmts.");
-
-  std::vector<llvm::Value*> Args;
-  std::vector<llvm::Type *> ArgTypes;
-  std::string Constraints;
-
-  // Clobbers
-  for (unsigned i = 0, e = S.getNumClobbers(); i != e; ++i) {
-    StringRef Clobber = S.getClobber(i);
-
-    if (Clobber != "memory" && Clobber != "cc")
-      Clobber = Target.getNormalizedGCCRegisterName(Clobber);
-
-    if (i != 0)
-      Constraints += ',';
-
-    Constraints += "~{";
-    Constraints += Clobber;
-    Constraints += '}';
-  }
-
-  // Add machine specific clobbers
-  std::string MachineClobbers = Target.getClobbers();
-  if (!MachineClobbers.empty()) {
-    if (!Constraints.empty())
-      Constraints += ',';
-    Constraints += MachineClobbers;
-  }
-
-  llvm::FunctionType *FTy =
-    llvm::FunctionType::get(VoidTy, ArgTypes, false);
-
-  llvm::InlineAsm *IA =
-    llvm::InlineAsm::get(FTy, *S.getAsmString(), Constraints, true);
-  llvm::CallInst *Result = Builder.CreateCall(IA, Args);
-  Result->addAttribute(~0, llvm::Attribute::NoUnwind);
-  Result->addAttribute(~0, llvm::Attribute::IANSDialect);
-}
diff --git a/lib/CodeGen/CGVTables.cpp b/lib/CodeGen/CGVTables.cpp
index cdaa26a..5b37fe4 100644
--- a/lib/CodeGen/CGVTables.cpp
+++ b/lib/CodeGen/CGVTables.cpp
@@ -79,15 +79,16 @@ llvm::Constant *CodeGenModule::GetAddrOfThunk(GlobalDecl GD,
 static llvm::Value *PerformTypeAdjustment(CodeGenFunction &CGF,
                                           llvm::Value *Ptr,
                                           int64_t NonVirtualAdjustment,
-                                          int64_t VirtualAdjustment) {
+                                          int64_t VirtualAdjustment,
+                                          bool IsReturnAdjustment) {
   if (!NonVirtualAdjustment && !VirtualAdjustment)
     return Ptr;
 
   llvm::Type *Int8PtrTy = CGF.Int8PtrTy;
   llvm::Value *V = CGF.Builder.CreateBitCast(Ptr, Int8PtrTy);
 
-  if (NonVirtualAdjustment) {
-    // Do the non-virtual adjustment.
+  if (NonVirtualAdjustment && !IsReturnAdjustment) {
+    // Perform the non-virtual adjustment for a base-to-derived cast.
     V = CGF.Builder.CreateConstInBoundsGEP1_64(V, NonVirtualAdjustment);
   }
 
@@ -95,7 +96,7 @@ static llvm::Value *PerformTypeAdjustment(CodeGenFunction &CGF,
     llvm::Type *PtrDiffTy = 
       CGF.ConvertType(CGF.getContext().getPointerDiffType());
 
-    // Do the virtual adjustment.
+    // Perform the virtual adjustment.
     llvm::Value *VTablePtrPtr = 
       CGF.Builder.CreateBitCast(V, Int8PtrTy->getPointerTo());
     
@@ -113,6 +114,11 @@ static llvm::Value *PerformTypeAdjustment(CodeGenFunction &CGF,
     V = CGF.Builder.CreateInBoundsGEP(V, Offset);
   }
 
+  if (NonVirtualAdjustment && IsReturnAdjustment) {
+    // Perform the non-virtual adjustment for a derived-to-base cast.
+    V = CGF.Builder.CreateConstInBoundsGEP1_64(V, NonVirtualAdjustment);
+  }
+
   // Cast back to the original type.
   return CGF.Builder.CreateBitCast(V, Ptr->getType());
 }
@@ -150,8 +156,7 @@ static void setThunkVisibility(CodeGenModule &CGM, const CXXMethodDecl *MD,
 
   case TSK_ExplicitSpecialization:
   case TSK_ImplicitInstantiation:
-    if (!CGM.getCodeGenOpts().HiddenWeakTemplateVTables)
-      return;
+    return;
     break;
   }
 
@@ -199,7 +204,8 @@ static RValue PerformReturnAdjustment(CodeGenFunction &CGF,
   
   ReturnValue = PerformTypeAdjustment(CGF, ReturnValue, 
                                       Thunk.Return.NonVirtual, 
-                                      Thunk.Return.VBaseOffsetOffset);
+                                      Thunk.Return.VBaseOffsetOffset,
+                                      /*IsReturnAdjustment*/true);
   
   if (NullCheckValue) {
     CGF.Builder.CreateBr(AdjustEnd);
@@ -248,7 +254,9 @@ void CodeGenFunction::GenerateVarArgsThunk(
   llvm::Function *BaseFn = cast<llvm::Function>(Callee);
 
   // Clone to thunk.
-  llvm::Function *NewFn = llvm::CloneFunction(BaseFn);
+  llvm::ValueToValueMapTy VMap;
+  llvm::Function *NewFn = llvm::CloneFunction(BaseFn, VMap,
+                                              /*ModuleLevelChanges=*/false);
   CGM.getModule().getFunctionList().push_back(NewFn);
   Fn->replaceAllUsesWith(NewFn);
   NewFn->takeName(Fn);
@@ -281,7 +289,8 @@ void CodeGenFunction::GenerateVarArgsThunk(
   llvm::Value *AdjustedThisPtr = 
     PerformTypeAdjustment(*this, ThisPtr, 
                           Thunk.This.NonVirtual, 
-                          Thunk.This.VCallOffsetOffset);
+                          Thunk.This.VCallOffsetOffset,
+                          /*IsReturnAdjustment*/false);
   ThisStore->setOperand(0, AdjustedThisPtr);
 
   if (!Thunk.Return.isEmpty()) {
@@ -324,7 +333,10 @@ void CodeGenFunction::GenerateThunk(llvm::Function *Fn,
     
     FunctionArgs.push_back(Param);
   }
-  
+
+  // Initialize debug info if needed.
+  maybeInitializeDebugInfo();
+
   StartFunction(GlobalDecl(), ResultType, Fn, FnInfo, FunctionArgs,
                 SourceLocation());
 
@@ -335,7 +347,8 @@ void CodeGenFunction::GenerateThunk(llvm::Function *Fn,
   llvm::Value *AdjustedThisPtr = 
     PerformTypeAdjustment(*this, LoadCXXThis(), 
                           Thunk.This.NonVirtual, 
-                          Thunk.This.VCallOffsetOffset);
+                          Thunk.This.VCallOffsetOffset,
+                          /*IsReturnAdjustment*/false);
   
   CallArgList CallArgs;
   
@@ -455,6 +468,8 @@ void CodeGenVTables::EmitThunk(GlobalDecl GD, const ThunkInfo &Thunk,
     return;
   }
 
+  CGM.SetLLVMFunctionAttributesForDefinition(GD.getDecl(), ThunkFn);
+
   if (ThunkFn->isVarArg()) {
     // Varargs thunks are special; we can't just generate a call because
     // we can't copy the varargs.  Our implementation is rather
@@ -524,7 +539,7 @@ CodeGenVTables::CreateVTableInitializer(const CXXRecordDecl *RD,
   
   unsigned NextVTableThunkIndex = 0;
   
-  llvm::Constant* PureVirtualFn = 0;
+  llvm::Constant *PureVirtualFn = 0, *DeletedVirtualFn = 0;
 
   for (unsigned I = 0; I != NumComponents; ++I) {
     VTableComponent Component = Components[I];
@@ -573,14 +588,25 @@ CodeGenVTables::CreateVTableInitializer(const CXXRecordDecl *RD,
       if (cast<CXXMethodDecl>(GD.getDecl())->isPure()) {
         // We have a pure virtual member function.
         if (!PureVirtualFn) {
-          llvm::FunctionType *Ty = 
-            llvm::FunctionType::get(CGM.VoidTy, /*isVarArg=*/false);
-          StringRef PureCallName = CGM.getCXXABI().GetPureVirtualCallName();
-          PureVirtualFn = CGM.CreateRuntimeFunction(Ty, PureCallName);
-          PureVirtualFn = llvm::ConstantExpr::getBitCast(PureVirtualFn,
+          llvm::FunctionType *Ty = 
+            llvm::FunctionType::get(CGM.VoidTy, /*isVarArg=*/false);
+          StringRef PureCallName = CGM.getCXXABI().GetPureVirtualCallName();
+          PureVirtualFn = CGM.CreateRuntimeFunction(Ty, PureCallName);
+          PureVirtualFn = llvm::ConstantExpr::getBitCast(PureVirtualFn,
                                                          CGM.Int8PtrTy);
         }
         Init = PureVirtualFn;
+      } else if (cast<CXXMethodDecl>(GD.getDecl())->isDeleted()) {
+        if (!DeletedVirtualFn) {
+          llvm::FunctionType *Ty =
+            llvm::FunctionType::get(CGM.VoidTy, /*isVarArg=*/false);
+          StringRef DeletedCallName =
+            CGM.getCXXABI().GetDeletedVirtualCallName();
+          DeletedVirtualFn = CGM.CreateRuntimeFunction(Ty, DeletedCallName);
+          DeletedVirtualFn = llvm::ConstantExpr::getBitCast(DeletedVirtualFn,
+                                                         CGM.Int8PtrTy);
+        }
+        Init = DeletedVirtualFn;
       } else {
         // Check if we should use a thunk.
         if (NextVTableThunkIndex < NumVTableThunks &&
diff --git a/lib/CodeGen/CodeGenAction.cpp b/lib/CodeGen/CodeGenAction.cpp
index dd32167..9d6d183 100644
--- a/lib/CodeGen/CodeGenAction.cpp
+++ b/lib/CodeGen/CodeGenAction.cpp
@@ -65,9 +65,11 @@ namespace clang {
       TargetOpts(targetopts),
       LangOpts(langopts),
       AsmOutStream(OS),
+      Context(), 
       LLVMIRGeneration("LLVM IR Generation Time"),
       Gen(CreateLLVMCodeGen(Diags, infile, compopts, C)),
-      LinkModule(LinkModule) {
+      LinkModule(LinkModule)
+    {
       llvm::TimePassesIsEnabled = TimePasses;
     }
 
@@ -379,7 +381,7 @@ void CodeGenAction::ExecuteAction() {
     // FIXME: This is stupid, IRReader shouldn't take ownership.
     llvm::MemoryBuffer *MainFileCopy =
       llvm::MemoryBuffer::getMemBufferCopy(MainFile->getBuffer(),
-                                           getCurrentFile().c_str());
+                                           getCurrentFile());
 
     llvm::SMDiagnostic Err;
     TheModule.reset(ParseIR(MainFileCopy, Err, *VMContext));
diff --git a/lib/CodeGen/CodeGenFunction.cpp b/lib/CodeGen/CodeGenFunction.cpp
index 1d02861..18f1623 100644
--- a/lib/CodeGen/CodeGenFunction.cpp
+++ b/lib/CodeGen/CodeGenFunction.cpp
@@ -24,7 +24,7 @@
 #include "clang/Frontend/CodeGenOptions.h"
 #include "llvm/Intrinsics.h"
 #include "llvm/MDBuilder.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 using namespace clang;
 using namespace CodeGen;
 
@@ -32,6 +32,10 @@ CodeGenFunction::CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext)
   : CodeGenTypeCache(cgm), CGM(cgm),
     Target(CGM.getContext().getTargetInfo()),
     Builder(cgm.getModule().getContext()),
+    SanitizePerformTypeCheck(CGM.getLangOpts().SanitizeNull |
+                             CGM.getLangOpts().SanitizeAlignment |
+                             CGM.getLangOpts().SanitizeObjectSize |
+                             CGM.getLangOpts().SanitizeVptr),
     AutoreleaseResult(false), BlockInfo(0), BlockPointer(0),
     LambdaThisCaptureField(0), NormalCleanupDest(0), NextCleanupDestIndex(1),
     FirstBlockInfo(0), EHResumeBlock(0), ExceptionSlot(0), EHSelectorSlot(0),
@@ -40,8 +44,6 @@ CodeGenFunction::CodeGenFunction(CodeGenModule &cgm, bool suppressNewContext)
     CXXABIThisDecl(0), CXXABIThisValue(0), CXXThisValue(0), CXXVTTDecl(0),
     CXXVTTValue(0), OutermostConditional(0), TerminateLandingPad(0),
     TerminateHandler(0), TrapBB(0) {
-
-  CatchUndefined = getContext().getLangOpts().CatchUndefined;
   if (!suppressNewContext)
     CGM.getCXXABI().getMangleContext().startNewFunction();
 }
@@ -348,11 +350,11 @@ void CodeGenFunction::StartFunction(GlobalDecl GD, QualType RetTy,
       for (FunctionDecl::redecl_iterator RI = FD->redecls_begin(),
              RE = FD->redecls_end(); RI != RE; ++RI)
         if (RI->isInlineSpecified()) {
-          Fn->addFnAttr(llvm::Attribute::InlineHint);
+          Fn->addFnAttr(llvm::Attributes::InlineHint);
           break;
         }
 
-  if (getContext().getLangOpts().OpenCL) {
+  if (getLangOpts().OpenCL) {
     // Add metadata for a kernel function.
     if (const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(D))
       EmitOpenCLKernelMetadata(FD, Fn);
@@ -485,7 +487,7 @@ static void TryMarkNoThrow(llvm::Function *F) {
       } else if (isa<llvm::ResumeInst>(&*BI)) {
         return;
       }
-  F->setDoesNotThrow(true);
+  F->setDoesNotThrow();
 }
 
 void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn,
@@ -493,8 +495,8 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn,
   const FunctionDecl *FD = cast<FunctionDecl>(GD.getDecl());
   
   // Check if we should generate debug info for this function.
-  if (CGM.getModuleDebugInfo() && !FD->hasAttr<NoDebugAttr>())
-    DebugInfo = CGM.getModuleDebugInfo();
+  if (!FD->hasAttr<NoDebugAttr>())
+    maybeInitializeDebugInfo();
 
   FunctionArgList Args;
   QualType ResTy = FD->getResultType();
@@ -517,7 +519,7 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn,
     EmitDestructorBody(Args);
   else if (isa<CXXConstructorDecl>(FD))
     EmitConstructorBody(Args);
-  else if (getContext().getLangOpts().CUDA &&
+  else if (getLangOpts().CUDA &&
            !CGM.getCodeGenOpts().CUDAIsDevice &&
            FD->hasAttr<CUDAGlobalAttr>())
     CGM.getCUDARuntime().EmitDeviceStubBody(*this, Args);
@@ -535,6 +537,24 @@ void CodeGenFunction::GenerateCode(GlobalDecl GD, llvm::Function *Fn,
   else
     EmitFunctionBody(Args);
 
+  // C++11 [stmt.return]p2:
+  //   Flowing off the end of a function [...] results in undefined behavior in
+  //   a value-returning function.
+  // C11 6.9.1p12:
+  //   If the '}' that terminates a function is reached, and the value of the
+  //   function call is used by the caller, the behavior is undefined.
+  if (getLangOpts().CPlusPlus && !FD->hasImplicitReturnZero() &&
+      !FD->getResultType()->isVoidType() && Builder.GetInsertBlock()) {
+    if (getLangOpts().SanitizeReturn)
+      EmitCheck(Builder.getFalse(), "missing_return",
+                EmitCheckSourceLocation(FD->getLocation()),
+                llvm::ArrayRef<llvm::Value*>());
+    else if (CGM.getCodeGenOpts().OptimizationLevel == 0)
+      Builder.CreateCall(CGM.getIntrinsic(llvm::Intrinsic::trap));
+    Builder.CreateUnreachable();
+    Builder.ClearInsertionPoint();
+  }
+
   // Emit the standard function epilogue.
   FinishFunction(BodyRange.getEnd());
 
@@ -806,7 +826,7 @@ static void emitNonZeroVLAInit(CodeGenFunction &CGF, QualType baseType,
 void
 CodeGenFunction::EmitNullInitialization(llvm::Value *DestPtr, QualType Ty) {
   // Ignore empty classes in C++.
-  if (getContext().getLangOpts().CPlusPlus) {
+  if (getLangOpts().CPlusPlus) {
     if (const RecordType *RT = Ty->getAs<RecordType>()) {
       if (cast<CXXRecordDecl>(RT->getDecl())->isEmpty())
         return;
@@ -983,8 +1003,7 @@ llvm::Value *CodeGenFunction::emitArrayLength(const ArrayType *origArrayType,
       arrayType = getContext().getAsArrayType(eltType);
     }
 
-    unsigned AddressSpace =
-        cast<llvm::PointerType>(addr->getType())->getAddressSpace();
+    unsigned AddressSpace = addr->getType()->getPointerAddressSpace();
     llvm::Type *BaseType = ConvertType(eltType)->getPointerTo(AddressSpace);
     addr = Builder.CreateBitCast(addr, BaseType, "array.begin");
   } else {
@@ -1027,6 +1046,7 @@ CodeGenFunction::getVLASize(const VariableArrayType *type) {
       numElements = vlaSize;
     } else {
       // It's undefined behavior if this wraps around, so mark it that way.
+      // FIXME: Teach -fcatch-undefined-behavior to trap this.
       numElements = Builder.CreateNUWMul(numElements, vlaSize);
     }
   } while ((type = getContext().getAsVariableArrayType(elementType)));
@@ -1104,10 +1124,26 @@ void CodeGenFunction::EmitVariablyModifiedType(QualType type) {
         // e.g. with a typedef and a pointer to it.
         llvm::Value *&entry = VLASizeMap[size];
         if (!entry) {
+          llvm::Value *Size = EmitScalarExpr(size);
+
+          // C11 6.7.6.2p5:
+          //   If the size is an expression that is not an integer constant
+          //   expression [...] each time it is evaluated it shall have a value
+          //   greater than zero.
+          if (getLangOpts().SanitizeVLABound &&
+              size->getType()->isSignedIntegerType()) {
+            llvm::Value *Zero = llvm::Constant::getNullValue(Size->getType());
+            llvm::Constant *StaticArgs[] = {
+              EmitCheckSourceLocation(size->getLocStart()),
+              EmitCheckTypeDescriptor(size->getType())
+            };
+            EmitCheck(Builder.CreateICmpSGT(Size, Zero),
+                      "vla_bound_not_positive", StaticArgs, Size);
+          }
+
           // Always zexting here would be wrong if it weren't
           // undefined behavior to have a negative bound.
-          entry = Builder.CreateIntCast(EmitScalarExpr(size), SizeTy,
-                                        /*signed*/ false);
+          entry = Builder.CreateIntCast(Size, SizeTy, /*signed*/ false);
         }
       }
       type = vat->getElementType();
@@ -1156,7 +1192,7 @@ void CodeGenFunction::EmitDeclRefExprDbgValue(const DeclRefExpr *E,
                                               llvm::Constant *Init) {
   assert (Init && "Invalid DeclRefExpr initializer!");
   if (CGDebugInfo *Dbg = getDebugInfo())
-    if (CGM.getCodeGenOpts().DebugInfo >= CodeGenOptions::LimitedDebugInfo)
+    if (CGM.getCodeGenOpts().getDebugInfo() >= CodeGenOptions::LimitedDebugInfo)
       Dbg->EmitGlobalVariable(E->getDecl(), Init);
 }
 
diff --git a/lib/CodeGen/CodeGenFunction.h b/lib/CodeGen/CodeGenFunction.h
index ed3e43b..f2ab226 100644
--- a/lib/CodeGen/CodeGenFunction.h
+++ b/lib/CodeGen/CodeGenFunction.h
@@ -222,8 +222,7 @@ public:
     /// immediately-enclosing context of the cleanup scope.  For
     /// EH cleanups, this is run in a terminate context.
     ///
-    // \param IsForEHCleanup true if this is for an EH cleanup, false
-    ///  if for a normal cleanup.
+    // \param flags cleanup kind.
     virtual void Emit(CodeGenFunction &CGF, Flags flags) = 0;
   };
 
@@ -533,8 +532,8 @@ public:
 /// CodeGenFunction - This class organizes the per-function state that is used
 /// while generating LLVM code.
 class CodeGenFunction : public CodeGenTypeCache {
-  CodeGenFunction(const CodeGenFunction&); // DO NOT IMPLEMENT
-  void operator=(const CodeGenFunction&);  // DO NOT IMPLEMENT
+  CodeGenFunction(const CodeGenFunction &) LLVM_DELETED_FUNCTION;
+  void operator=(const CodeGenFunction &) LLVM_DELETED_FUNCTION;
 
   friend class CGCXXABI;
 public:
@@ -595,8 +594,9 @@ public:
   /// potentially higher performance penalties.
   unsigned char BoundsChecking;
 
-  /// CatchUndefined - Emit run-time checks to catch undefined behaviors.
-  bool CatchUndefined;
+  /// \brief Whether any type-checking sanitizers are enabled. If \c false,
+  /// calls to EmitTypeCheck can be skipped.
+  bool SanitizePerformTypeCheck;
 
   /// In ARC, whether we should autorelease the return value.
   bool AutoreleaseResult;
@@ -795,8 +795,8 @@ public:
     bool OldDidCallStackSave;
     bool PerformCleanup;
 
-    RunCleanupsScope(const RunCleanupsScope &); // DO NOT IMPLEMENT
-    RunCleanupsScope &operator=(const RunCleanupsScope &); // DO NOT IMPLEMENT
+    RunCleanupsScope(const RunCleanupsScope &) LLVM_DELETED_FUNCTION;
+    void operator=(const RunCleanupsScope &) LLVM_DELETED_FUNCTION;
 
   protected:
     CodeGenFunction& CGF;
@@ -839,8 +839,8 @@ public:
     SourceRange Range;
     bool PopDebugStack;
 
-    LexicalScope(const LexicalScope &); // DO NOT IMPLEMENT THESE
-    LexicalScope &operator=(const LexicalScope &);
+    LexicalScope(const LexicalScope &) LLVM_DELETED_FUNCTION;
+    void operator=(const LexicalScope &) LLVM_DELETED_FUNCTION;
 
   public:
     /// \brief Enter a new cleanup scope.
@@ -908,7 +908,7 @@ public:
   /// themselves).
   void popCatchScope();
 
-  llvm::BasicBlock *getEHResumeBlock();
+  llvm::BasicBlock *getEHResumeBlock(bool isCleanup);
   llvm::BasicBlock *getEHDispatchBlock(EHScopeStack::stable_iterator scope);
 
   /// An object to manage conditionally-evaluated expressions.
@@ -1213,6 +1213,14 @@ public:
 
   CodeGenTypes &getTypes() const { return CGM.getTypes(); }
   ASTContext &getContext() const { return CGM.getContext(); }
+  /// Returns true if DebugInfo is actually initialized.
+  bool maybeInitializeDebugInfo() {
+    if (CGM.getModuleDebugInfo()) {
+      DebugInfo = CGM.getModuleDebugInfo();
+      return true;
+    }
+    return false;
+  }
   CGDebugInfo *getDebugInfo() { 
     if (DisableDebugInfo) 
       return NULL;
@@ -1504,7 +1512,7 @@ public:
   static bool hasAggregateLLVMType(QualType T);
 
   /// createBasicBlock - Create an LLVM basic block.
-  llvm::BasicBlock *createBasicBlock(StringRef name = "",
+  llvm::BasicBlock *createBasicBlock(const Twine &name = "",
                                      llvm::Function *parent = 0,
                                      llvm::BasicBlock *before = 0) {
 #ifdef NDEBUG
@@ -1631,7 +1639,7 @@ public:
   /// aggregate expression, the aggloc/agglocvolatile arguments indicate where
   /// the result should be returned.
   ///
-  /// \param IgnoreResult - True if the resulting value isn't used.
+  /// \param ignoreResult True if the resulting value isn't used.
   RValue EmitAnyExpr(const Expr *E,
                      AggValueSlot aggSlot = AggValueSlot::ignored(),
                      bool ignoreResult = false);
@@ -1654,13 +1662,26 @@ public:
   void EmitExprAsInit(const Expr *init, const ValueDecl *D,
                       LValue lvalue, bool capturedByInit);
 
+  /// EmitAggregateCopy - Emit an aggrate assignment.
+  ///
+  /// The difference to EmitAggregateCopy is that tail padding is not copied.
+  /// This is required for correctness when assigning non-POD structures in C++.
+  void EmitAggregateAssign(llvm::Value *DestPtr, llvm::Value *SrcPtr,
+                           QualType EltTy, bool isVolatile=false,
+                           CharUnits Alignment = CharUnits::Zero()) {
+    EmitAggregateCopy(DestPtr, SrcPtr, EltTy, isVolatile, Alignment, true);
+  }
+
   /// EmitAggregateCopy - Emit an aggrate copy.
   ///
   /// \param isVolatile - True iff either the source or the destination is
   /// volatile.
+  /// \param isAssignment - If false, allow padding to be copied.  This often
+  /// yields more efficient.
   void EmitAggregateCopy(llvm::Value *DestPtr, llvm::Value *SrcPtr,
                          QualType EltTy, bool isVolatile=false,
-                         CharUnits Alignment = CharUnits::Zero());
+                         CharUnits Alignment = CharUnits::Zero(),
+                         bool isAssignment = false);
 
   /// StartBlock - Start new block named N. If insert block is a dummy block
   /// then reuse it.
@@ -1829,12 +1850,37 @@ public:
 
   llvm::Value* EmitCXXTypeidExpr(const CXXTypeidExpr *E);
   llvm::Value *EmitDynamicCast(llvm::Value *V, const CXXDynamicCastExpr *DCE);
+  llvm::Value* EmitCXXUuidofExpr(const CXXUuidofExpr *E);
 
   void MaybeEmitStdInitializerListCleanup(llvm::Value *loc, const Expr *init);
   void EmitStdInitializerListCleanup(llvm::Value *loc,
                                      const InitListExpr *init);
 
-  void EmitCheck(llvm::Value *, unsigned Size);
+  /// \brief Situations in which we might emit a check for the suitability of a
+  ///        pointer or glvalue.
+  enum TypeCheckKind {
+    /// Checking the operand of a load. Must be suitably sized and aligned.
+    TCK_Load,
+    /// Checking the destination of a store. Must be suitably sized and aligned.
+    TCK_Store,
+    /// Checking the bound value in a reference binding. Must be suitably sized
+    /// and aligned, but is not required to refer to an object (until the
+    /// reference is used), per core issue 453.
+    TCK_ReferenceBinding,
+    /// Checking the object expression in a non-static data member access. Must
+    /// be an object within its lifetime.
+    TCK_MemberAccess,
+    /// Checking the 'this' pointer for a call to a non-static member function.
+    /// Must be an object within its lifetime.
+    TCK_MemberCall,
+    /// Checking the 'this' pointer for a constructor call.
+    TCK_ConstructorCall
+  };
+
+  /// \brief Emit a check that \p V is the address of storage of the
+  /// appropriate size and alignment for an object of type \p Type.
+  void EmitTypeCheck(TypeCheckKind TCK, SourceLocation Loc, llvm::Value *V,
+                     QualType Type, CharUnits Alignment = CharUnits::Zero());
 
   llvm::Value *EmitScalarPrePostIncDec(const UnaryOperator *E, LValue LV,
                                        bool isInc, bool isPre);
@@ -1981,7 +2027,6 @@ public:
   void EmitCaseStmt(const CaseStmt &S);
   void EmitCaseStmtRange(const CaseStmt &S);
   void EmitAsmStmt(const AsmStmt &S);
-  void EmitMSAsmStmt(const MSAsmStmt &S);
 
   void EmitObjCForCollectionStmt(const ObjCForCollectionStmt &S);
   void EmitObjCAtTryStmt(const ObjCAtTryStmt &S);
@@ -2033,11 +2078,10 @@ public:
   ///
   LValue EmitLValue(const Expr *E);
 
-  /// EmitCheckedLValue - Same as EmitLValue but additionally we generate
-  /// checking code to guard against undefined behavior.  This is only
-  /// suitable when we know that the address will be used to access the
-  /// object.
-  LValue EmitCheckedLValue(const Expr *E);
+  /// \brief Same as EmitLValue but additionally we generate checking code to
+  /// guard against undefined behavior.  This is only suitable when we know
+  /// that the address will be used to access the object.
+  LValue EmitCheckedLValue(const Expr *E, TypeCheckKind TCK);
 
   /// EmitToMemory - Change a scalar value from its value
   /// representation to its in-memory representation.
@@ -2178,6 +2222,7 @@ public:
   LValue EmitCXXBindTemporaryLValue(const CXXBindTemporaryExpr *E);
   LValue EmitLambdaLValue(const LambdaExpr *E);
   LValue EmitCXXTypeidLValue(const CXXTypeidExpr *E);
+  LValue EmitCXXUuidofLValue(const CXXUuidofExpr *E);
 
   LValue EmitObjCMessageExprLValue(const ObjCMessageExpr *E);
   LValue EmitObjCIvarRefLValue(const ObjCIvarRefExpr *E);
@@ -2230,6 +2275,7 @@ public:
                                                    const CXXRecordDecl *RD);
 
   RValue EmitCXXMemberCall(const CXXMethodDecl *MD,
+                           SourceLocation CallLoc,
                            llvm::Value *Callee,
                            ReturnValueSlot ReturnValue,
                            llvm::Value *This,
@@ -2310,6 +2356,7 @@ public:
   llvm::Value *EmitARCRetain(QualType type, llvm::Value *value);
   llvm::Value *EmitARCRetainNonBlock(llvm::Value *value);
   llvm::Value *EmitARCRetainBlock(llvm::Value *value, bool mandatory);
+  void EmitARCDestroyStrong(llvm::Value *addr, bool precise);
   void EmitARCRelease(llvm::Value *value, bool precise);
   llvm::Value *EmitARCAutorelease(llvm::Value *value);
   llvm::Value *EmitARCAutoreleaseReturnValue(llvm::Value *value);
@@ -2516,9 +2563,29 @@ public:
   void EmitBranchOnBoolExpr(const Expr *Cond, llvm::BasicBlock *TrueBlock,
                             llvm::BasicBlock *FalseBlock);
 
-  /// getTrapBB - Create a basic block that will call the trap intrinsic.  We'll
-  /// generate a branch around the created basic block as necessary.
-  llvm::BasicBlock *getTrapBB();
+  /// \brief Emit a description of a type in a format suitable for passing to
+  /// a runtime sanitizer handler.
+  llvm::Constant *EmitCheckTypeDescriptor(QualType T);
+
+  /// \brief Convert a value into a format suitable for passing to a runtime
+  /// sanitizer handler.
+  llvm::Value *EmitCheckValue(llvm::Value *V);
+
+  /// \brief Emit a description of a source location in a format suitable for
+  /// passing to a runtime sanitizer handler.
+  llvm::Constant *EmitCheckSourceLocation(SourceLocation Loc);
+
+  /// \brief Create a basic block that will call a handler function in a
+  /// sanitizer runtime with the provided arguments, and create a conditional
+  /// branch to it.
+  void EmitCheck(llvm::Value *Checked, StringRef CheckName,
+                 llvm::ArrayRef<llvm::Constant *> StaticArgs,
+                 llvm::ArrayRef<llvm::Value *> DynamicArgs,
+                 bool Recoverable = false);
+
+  /// \brief Create a basic block that will call the trap intrinsic, and emit a
+  /// conditional branch to it, for the -ftrapv checks.
+  void EmitTrapvCheck(llvm::Value *Checked);
 
   /// EmitCallArg - Emit a single call argument.
   void EmitCallArg(CallArgList &args, const Expr *E, QualType ArgType);
@@ -2553,12 +2620,10 @@ private:
                         SmallVector<llvm::Value*, 16> &Args,
                         llvm::FunctionType *IRFuncTy);
 
-  llvm::Value* EmitAsmInput(const AsmStmt &S,
-                            const TargetInfo::ConstraintInfo &Info,
+  llvm::Value* EmitAsmInput(const TargetInfo::ConstraintInfo &Info,
                             const Expr *InputExpr, std::string &ConstraintStr);
 
-  llvm::Value* EmitAsmInputLValue(const AsmStmt &S,
-                                  const TargetInfo::ConstraintInfo &Info,
+  llvm::Value* EmitAsmInputLValue(const TargetInfo::ConstraintInfo &Info,
                                   LValue InputValue, QualType InputType,
                                   std::string &ConstraintStr);
 
@@ -2624,15 +2689,9 @@ private:
 
   void AddObjCARCExceptionMetadata(llvm::Instruction *Inst);
 
-  /// GetPointeeAlignment - Given an expression with a pointer type, find the
-  /// alignment of the type referenced by the pointer.  Skip over implicit
-  /// casts.
-  unsigned GetPointeeAlignment(const Expr *Addr);
-
-  /// GetPointeeAlignmentValue - Given an expression with a pointer type, find
-  /// the alignment of the type referenced by the pointer.  Skip over implicit
-  /// casts.  Return the alignment as an llvm::Value.
-  llvm::Value *GetPointeeAlignmentValue(const Expr *Addr);
+  /// GetPointeeAlignment - Given an expression with a pointer type, emit the
+  /// value and compute our best estimate of the alignment of the pointee.
+  std::pair<llvm::Value*, unsigned> EmitPointerWithAlignment(const Expr *Addr);
 };
 
 /// Helper class with most of the code for saving a value for a
diff --git a/lib/CodeGen/CodeGenModule.cpp b/lib/CodeGen/CodeGenModule.cpp
index 3ae3c52..17972e2 100644
--- a/lib/CodeGen/CodeGenModule.cpp
+++ b/lib/CodeGen/CodeGenModule.cpp
@@ -42,7 +42,7 @@
 #include "llvm/ADT/APSInt.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/Target/Mangler.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/Support/CallSite.h"
 #include "llvm/Support/ErrorHandling.h"
 using namespace clang;
@@ -62,10 +62,10 @@ static CGCXXABI &createCXXABI(CodeGenModule &CGM) {
 
 
 CodeGenModule::CodeGenModule(ASTContext &C, const CodeGenOptions &CGO,
-                             llvm::Module &M, const llvm::TargetData &TD,
+                             llvm::Module &M, const llvm::DataLayout &TD,
                              DiagnosticsEngine &diags)
   : Context(C), LangOpts(C.getLangOpts()), CodeGenOpts(CGO), TheModule(M),
-    TheTargetData(TD), TheTargetCodeGenInfo(0), Diags(diags),
+    TheDataLayout(TD), TheTargetCodeGenInfo(0), Diags(diags),
     ABI(createCXXABI(*this)), 
     Types(*this),
     TBAA(0),
@@ -103,14 +103,14 @@ CodeGenModule::CodeGenModule(ASTContext &C, const CodeGenOptions &CGO,
     createCUDARuntime();
 
   // Enable TBAA unless it's suppressed. ThreadSanitizer needs TBAA even at O0.
-  if (LangOpts.ThreadSanitizer ||
+  if (LangOpts.SanitizeThread ||
       (!CodeGenOpts.RelaxedAliasing && CodeGenOpts.OptimizationLevel > 0))
     TBAA = new CodeGenTBAA(Context, VMContext, CodeGenOpts, getLangOpts(),
                            ABI.getMangleContext());
 
   // If debug info or coverage generation is enabled, create the CGDebugInfo
   // object.
-  if (CodeGenOpts.DebugInfo != CodeGenOptions::NoDebugInfo ||
+  if (CodeGenOpts.getDebugInfo() != CodeGenOptions::NoDebugInfo ||
       CodeGenOpts.EmitGcovArcs ||
       CodeGenOpts.EmitGcovNotes)
     DebugInfo = new CGDebugInfo(*this);
@@ -202,6 +202,12 @@ llvm::MDNode *CodeGenModule::getTBAAInfoForVTablePtr() {
   return TBAA->getTBAAInfoForVTablePtr();
 }
 
+llvm::MDNode *CodeGenModule::getTBAAStructInfo(QualType QTy) {
+  if (!TBAA)
+    return 0;
+  return TBAA->getTBAAStructInfo(QTy);
+}
+
 void CodeGenModule::DecorateInstruction(llvm::Instruction *Inst,
                                         llvm::MDNode *TBAAInfo) {
   Inst->setMetadata(llvm::LLVMContext::MD_tbaa, TBAAInfo);
@@ -287,7 +293,7 @@ void CodeGenModule::setTLSMode(llvm::GlobalVariable *GV,
   assert(D.isThreadSpecified() && "setting TLS mode on non-TLS var!");
 
   llvm::GlobalVariable::ThreadLocalMode TLM;
-  TLM = GetLLVMTLSModel(CodeGenOpts.DefaultTLSModel);
+  TLM = GetLLVMTLSModel(CodeGenOpts.getDefaultTLSModel());
 
   // Override the TLS model if it is explicitly specified.
   if (D.hasAttr<TLSModelAttr>()) {
@@ -347,9 +353,7 @@ void CodeGenModule::setTypeVisibility(llvm::GlobalValue *GV,
   // to deal with mixed-visibility symbols.
   case TSK_ExplicitSpecialization:
   case TSK_ImplicitInstantiation:
-    if (!CodeGenOpts.HiddenWeakTemplateVTables)
-      return;
-    break;
+    return;
   }
 
   // If there's a key function, there may be translation units
@@ -529,7 +533,7 @@ void CodeGenModule::SetLLVMFunctionAttributes(const Decl *D,
   unsigned CallingConv;
   AttributeListType AttributeList;
   ConstructAttributeList(Info, D, AttributeList, CallingConv);
-  F->setAttributes(llvm::AttrListPtr::get(AttributeList));
+  F->setAttributes(llvm::AttrListPtr::get(getLLVMContext(), AttributeList));
   F->setCallingConv(static_cast<llvm::CallingConv::ID>(CallingConv));
 }
 
@@ -559,39 +563,46 @@ void CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D,
     F->setHasUWTable();
 
   if (!hasUnwindExceptions(LangOpts))
-    F->addFnAttr(llvm::Attribute::NoUnwind);
+    F->addFnAttr(llvm::Attributes::NoUnwind);
 
   if (D->hasAttr<NakedAttr>()) {
     // Naked implies noinline: we should not be inlining such functions.
-    F->addFnAttr(llvm::Attribute::Naked);
-    F->addFnAttr(llvm::Attribute::NoInline);
+    F->addFnAttr(llvm::Attributes::Naked);
+    F->addFnAttr(llvm::Attributes::NoInline);
   }
 
   if (D->hasAttr<NoInlineAttr>())
-    F->addFnAttr(llvm::Attribute::NoInline);
+    F->addFnAttr(llvm::Attributes::NoInline);
 
   // (noinline wins over always_inline, and we can't specify both in IR)
   if ((D->hasAttr<AlwaysInlineAttr>() || D->hasAttr<ForceInlineAttr>()) &&
-      !F->hasFnAttr(llvm::Attribute::NoInline))
-    F->addFnAttr(llvm::Attribute::AlwaysInline);
+      !F->getFnAttributes().hasAttribute(llvm::Attributes::NoInline))
+    F->addFnAttr(llvm::Attributes::AlwaysInline);
 
   // FIXME: Communicate hot and cold attributes to LLVM more directly.
   if (D->hasAttr<ColdAttr>())
-    F->addFnAttr(llvm::Attribute::OptimizeForSize);
+    F->addFnAttr(llvm::Attributes::OptimizeForSize);
+
+  if (D->hasAttr<MinSizeAttr>())
+    F->addFnAttr(llvm::Attributes::MinSize);
 
   if (isa<CXXConstructorDecl>(D) || isa<CXXDestructorDecl>(D))
     F->setUnnamedAddr(true);
 
+  if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(D))
+    if (MD->isVirtual())
+      F->setUnnamedAddr(true);
+
   if (LangOpts.getStackProtector() == LangOptions::SSPOn)
-    F->addFnAttr(llvm::Attribute::StackProtect);
+    F->addFnAttr(llvm::Attributes::StackProtect);
   else if (LangOpts.getStackProtector() == LangOptions::SSPReq)
-    F->addFnAttr(llvm::Attribute::StackProtectReq);
+    F->addFnAttr(llvm::Attributes::StackProtectReq);
   
-  if (LangOpts.AddressSanitizer) {
+  if (LangOpts.SanitizeAddress) {
     // When AddressSanitizer is enabled, set AddressSafety attribute
     // unless __attribute__((no_address_safety_analysis)) is used.
     if (!D->hasAttr<NoAddressSafetyAnalysisAttr>())
-      F->addFnAttr(llvm::Attribute::AddressSafety);
+      F->addFnAttr(llvm::Attributes::AddressSafety);
   }
 
   unsigned alignment = D->getMaxAlignment() / Context.getCharWidth();
@@ -636,7 +647,8 @@ void CodeGenModule::SetFunctionAttributes(GlobalDecl GD,
   if (unsigned IID = F->getIntrinsicID()) {
     // If this is an intrinsic function, set the function's attributes
     // to the intrinsic's attributes.
-    F->setAttributes(llvm::Intrinsic::getAttributes((llvm::Intrinsic::ID)IID));
+    F->setAttributes(llvm::Intrinsic::getAttributes(getLLVMContext(),
+                                                    (llvm::Intrinsic::ID)IID));
     return;
   }
 
@@ -822,6 +834,49 @@ bool CodeGenModule::MayDeferGeneration(const ValueDecl *Global) {
   return !getContext().DeclMustBeEmitted(Global);
 }
 
+llvm::Constant *CodeGenModule::GetAddrOfUuidDescriptor(
+    const CXXUuidofExpr* E) {
+  // Sema has verified that IIDSource has a __declspec(uuid()), and that its
+  // well-formed.
+  StringRef Uuid;
+  if (E->isTypeOperand())
+    Uuid = CXXUuidofExpr::GetUuidAttrOfType(E->getTypeOperand())->getGuid();
+  else {
+    // Special case: __uuidof(0) means an all-zero GUID.
+    Expr *Op = E->getExprOperand();
+    if (!Op->isNullPointerConstant(Context, Expr::NPC_ValueDependentIsNull))
+      Uuid = CXXUuidofExpr::GetUuidAttrOfType(Op->getType())->getGuid();
+    else
+      Uuid = "00000000-0000-0000-0000-000000000000";
+  }
+  std::string Name = "__uuid_" + Uuid.str();
+
+  // Look for an existing global.
+  if (llvm::GlobalVariable *GV = getModule().getNamedGlobal(Name))
+    return GV;
+
+  llvm::Constant *Init = EmitUuidofInitializer(Uuid, E->getType());
+  assert(Init && "failed to initialize as constant");
+
+  // GUIDs are assumed to be 16 bytes, spread over 4-2-2-8 bytes. However, the
+  // first field is declared as "long", which for many targets is 8 bytes.
+  // Those architectures are not supported. (With the MS abi, long is always 4
+  // bytes.)
+  llvm::Type *GuidType = getTypes().ConvertType(E->getType());
+  if (Init->getType() != GuidType) {
+    DiagnosticsEngine &Diags = getDiags();
+    unsigned DiagID = Diags.getCustomDiagID(DiagnosticsEngine::Error,
+        "__uuidof codegen is not supported on this architecture");
+    Diags.Report(E->getExprLoc(), DiagID) << E->getSourceRange();
+    Init = llvm::UndefValue::get(GuidType);
+  }
+
+  llvm::GlobalVariable *GV = new llvm::GlobalVariable(getModule(), GuidType,
+      /*isConstant=*/true, llvm::GlobalValue::PrivateLinkage, Init, Name);
+  GV->setUnnamedAddr(true);
+  return GV;
+}
+
 llvm::Constant *CodeGenModule::GetWeakRefReference(const ValueDecl *VD) {
   const AliasAttr *AA = VD->getAttr<AliasAttr>();
   assert(AA && "No alias?");
@@ -830,19 +885,23 @@ llvm::Constant *CodeGenModule::GetWeakRefReference(const ValueDecl *VD) {
 
   // See if there is already something with the target's name in the module.
   llvm::GlobalValue *Entry = GetGlobalValue(AA->getAliasee());
+  if (Entry) {
+    unsigned AS = getContext().getTargetAddressSpace(VD->getType());
+    return llvm::ConstantExpr::getBitCast(Entry, DeclTy->getPointerTo(AS));
+  }
 
   llvm::Constant *Aliasee;
   if (isa<llvm::FunctionType>(DeclTy))
-    Aliasee = GetOrCreateLLVMFunction(AA->getAliasee(), DeclTy, GlobalDecl(),
+    Aliasee = GetOrCreateLLVMFunction(AA->getAliasee(), DeclTy,
+                                      GlobalDecl(cast<FunctionDecl>(VD)),
                                       /*ForVTable=*/false);
   else
     Aliasee = GetOrCreateLLVMGlobal(AA->getAliasee(),
                                     llvm::PointerType::getUnqual(DeclTy), 0);
-  if (!Entry) {
-    llvm::GlobalValue* F = cast<llvm::GlobalValue>(Aliasee);
-    F->setLinkage(llvm::Function::ExternalWeakLinkage);    
-    WeakRefReferences.insert(F);
-  }
+
+  llvm::GlobalValue* F = cast<llvm::GlobalValue>(Aliasee);
+  F->setLinkage(llvm::Function::ExternalWeakLinkage);
+  WeakRefReferences.insert(F);
 
   return Aliasee;
 }
@@ -1051,12 +1110,10 @@ CodeGenModule::GetOrCreateLLVMFunction(StringRef MangledName,
   // Lookup the entry, lazily creating it if necessary.
   llvm::GlobalValue *Entry = GetGlobalValue(MangledName);
   if (Entry) {
-    if (WeakRefReferences.count(Entry)) {
+    if (WeakRefReferences.erase(Entry)) {
       const FunctionDecl *FD = cast_or_null<FunctionDecl>(D.getDecl());
       if (FD && !FD->hasAttr<WeakAttr>())
         Entry->setLinkage(llvm::Function::ExternalLinkage);
-
-      WeakRefReferences.erase(Entry);
     }
 
     if (Entry->getType()->getElementType() == Ty)
@@ -1085,8 +1142,8 @@ CodeGenModule::GetOrCreateLLVMFunction(StringRef MangledName,
   assert(F->getName() == MangledName && "name was uniqued!");
   if (D.getDecl())
     SetFunctionAttributes(D, F, IsIncompleteFunction);
-  if (ExtraAttrs != llvm::Attribute::None)
-    F->addFnAttr(ExtraAttrs);
+  if (ExtraAttrs.hasAttributes())
+    F->addAttribute(llvm::AttrListPtr::FunctionIndex, ExtraAttrs);
 
   // This is the first use or definition of a mangled name.  If there is a
   // deferred decl with this name, remember that we need to emit it at the end
@@ -1197,11 +1254,9 @@ CodeGenModule::GetOrCreateLLVMGlobal(StringRef MangledName,
   // Lookup the entry, lazily creating it if necessary.
   llvm::GlobalValue *Entry = GetGlobalValue(MangledName);
   if (Entry) {
-    if (WeakRefReferences.count(Entry)) {
+    if (WeakRefReferences.erase(Entry)) {
       if (D && !D->hasAttr<WeakAttr>())
         Entry->setLinkage(llvm::Function::ExternalLinkage);
-
-      WeakRefReferences.erase(Entry);
     }
 
     if (UnnamedAddr)
@@ -1279,7 +1334,7 @@ CodeGenModule::CreateOrReplaceCXXRuntimeVariable(StringRef Name,
 
     // Because C++ name mangling, the only way we can end up with an already
     // existing global with the same name is if it has been declared extern "C".
-      assert(GV->isDeclaration() && "Declaration has wrong type!");
+    assert(GV->isDeclaration() && "Declaration has wrong type!");
     OldGV = GV;
   }
   
@@ -1424,7 +1479,7 @@ CodeGenModule::getVTableLinkage(const CXXRecordDecl *RD) {
 
 CharUnits CodeGenModule::GetTargetTypeStoreSize(llvm::Type *Ty) const {
     return Context.toCharUnitsFromBits(
-      TheTargetData.getTypeStoreSizeInBits(Ty));
+      TheDataLayout.getTypeStoreSizeInBits(Ty));
 }
 
 llvm::Constant *
@@ -1473,10 +1528,10 @@ CodeGenModule::MaybeEmitGlobalStdInitializerListInitializer(const VarDecl *D,
   // Now clone the InitListExpr to initialize the array instead.
   // Incredible hack: we want to use the existing InitListExpr here, so we need
   // to tell it that it no longer initializes a std::initializer_list.
-  Expr *arrayInit = new (ctx) InitListExpr(ctx, init->getLBraceLoc(),
-                                    const_cast<InitListExpr*>(init)->getInits(),
-                                                   init->getNumInits(),
-                                                   init->getRBraceLoc());
+  ArrayRef<Expr*> Inits(const_cast<InitListExpr*>(init)->getInits(),
+                        init->getNumInits());
+  Expr *arrayInit = new (ctx) InitListExpr(ctx, init->getLBraceLoc(), Inits,
+                                           init->getRBraceLoc());
   arrayInit->setType(arrayType);
 
   if (!cleanups.empty())
@@ -1682,9 +1737,21 @@ void CodeGenModule::EmitGlobalVarDefinition(const VarDecl *D) {
   if (NeedsGlobalCtor || NeedsGlobalDtor)
     EmitCXXGlobalVarDeclInitFunc(D, GV, NeedsGlobalCtor);
 
+  // If we are compiling with ASan, add metadata indicating dynamically
+  // initialized globals.
+  if (LangOpts.SanitizeAddress && NeedsGlobalCtor) {
+    llvm::Module &M = getModule();
+
+    llvm::NamedMDNode *DynamicInitializers =
+        M.getOrInsertNamedMetadata("llvm.asan.dynamically_initialized_globals");
+    llvm::Value *GlobalToAdd[] = { GV };
+    llvm::MDNode *ThisGlobal = llvm::MDNode::get(VMContext, GlobalToAdd);
+    DynamicInitializers->addOperand(ThisGlobal);
+  }
+
   // Emit global variable debug information.
   if (CGDebugInfo *DI = getModuleDebugInfo())
-    if (getCodeGenOpts().DebugInfo >= CodeGenOptions::LimitedDebugInfo)
+    if (getCodeGenOpts().getDebugInfo() >= CodeGenOptions::LimitedDebugInfo)
       DI->EmitGlobalVariable(GV, D);
 }
 
@@ -1758,8 +1825,10 @@ static void ReplaceUsesOfNonProtoTypeWithRealFunction(llvm::GlobalValue *Old,
     llvm::Attributes RAttrs = AttrList.getRetAttributes();
 
     // Add the return attributes.
-    if (RAttrs)
-      AttrVec.push_back(llvm::AttributeWithIndex::get(0, RAttrs));
+    if (RAttrs.hasAttributes())
+      AttrVec.push_back(llvm::
+                        AttributeWithIndex::get(llvm::AttrListPtr::ReturnIndex,
+                                                RAttrs));
 
     // If the function was passed too few arguments, don't transform.  If extra
     // arguments were passed, we silently drop them.  If any of the types
@@ -1775,14 +1844,18 @@ static void ReplaceUsesOfNonProtoTypeWithRealFunction(llvm::GlobalValue *Old,
       }
 
       // Add any parameter attributes.
-      if (llvm::Attributes PAttrs = AttrList.getParamAttributes(ArgNo + 1))
+      llvm::Attributes PAttrs = AttrList.getParamAttributes(ArgNo + 1);
+      if (PAttrs.hasAttributes())
         AttrVec.push_back(llvm::AttributeWithIndex::get(ArgNo + 1, PAttrs));
     }
     if (DontTransform)
       continue;
 
-    if (llvm::Attributes FnAttrs =  AttrList.getFnAttributes())
-      AttrVec.push_back(llvm::AttributeWithIndex::get(~0, FnAttrs));
+    llvm::Attributes FnAttrs =  AttrList.getFnAttributes();
+    if (FnAttrs.hasAttributes())
+      AttrVec.push_back(llvm::
+                       AttributeWithIndex::get(llvm::AttrListPtr::FunctionIndex,
+                                               FnAttrs));
 
     // Okay, we can transform this.  Create the new call instruction and copy
     // over the required information.
@@ -1791,7 +1864,7 @@ static void ReplaceUsesOfNonProtoTypeWithRealFunction(llvm::GlobalValue *Old,
     ArgList.clear();
     if (!NewCall->getType()->isVoidTy())
       NewCall->takeName(CI);
-    NewCall->setAttributes(llvm::AttrListPtr::get(AttrVec));
+    NewCall->setAttributes(llvm::AttrListPtr::get(OldFn->getContext(), AttrVec));
     NewCall->setCallingConv(CI->getCallingConv());
 
     // Finally, remove the old call, replacing any uses with the new one.
@@ -1911,7 +1984,7 @@ void CodeGenModule::EmitAliasDefinition(GlobalDecl GD) {
   // if a deferred decl.
   llvm::Constant *Aliasee;
   if (isa<llvm::FunctionType>(DeclTy))
-    Aliasee = GetOrCreateLLVMFunction(AA->getAliasee(), DeclTy, GlobalDecl(),
+    Aliasee = GetOrCreateLLVMFunction(AA->getAliasee(), DeclTy, GD,
                                       /*ForVTable=*/false);
   else
     Aliasee = GetOrCreateLLVMGlobal(AA->getAliasee(),
@@ -1987,7 +2060,7 @@ GetConstantCFStringEntry(llvm::StringMap<llvm::Constant*> &Map,
   IsUTF16 = true;
 
   SmallVector<UTF16, 128> ToBuf(NumBytes + 1); // +1 for ending nulls.
-  const UTF8 *FromPtr = (UTF8 *)String.data();
+  const UTF8 *FromPtr = (const UTF8 *)String.data();
   UTF16 *ToPtr = &ToBuf[0];
 
   (void)ConvertUTF8toUTF16(&FromPtr, FromPtr + NumBytes,
@@ -2019,7 +2092,7 @@ CodeGenModule::GetAddrOfConstantCFString(const StringLiteral *Literal) {
   bool isUTF16 = false;
   llvm::StringMapEntry<llvm::Constant*> &Entry =
     GetConstantCFStringEntry(CFConstantStringMap, Literal,
-                             getTargetData().isLittleEndian(),
+                             getDataLayout().isLittleEndian(),
                              isUTF16, StringLength);
 
   if (llvm::Constant *C = Entry.getValue())
@@ -2429,7 +2502,7 @@ void CodeGenModule::EmitObjCPropertyImplementations(const
       ObjCPropertyDecl *PD = PID->getPropertyDecl();
 
       // Determine which methods need to be implemented, some may have
-      // been overridden. Note that ::isSynthesized is not the method
+      // been overridden. Note that ::isPropertyAccessor is not the method
       // we want, that just indicates if the decl came from a
       // property. What we want to know is if the method is defined in
       // this implementation.
@@ -2465,11 +2538,11 @@ void CodeGenModule::EmitObjCIvarInitializations(ObjCImplementationDecl *D) {
       ObjCMethodDecl::Create(getContext(), D->getLocation(), D->getLocation(),
                              cxxSelector, getContext().VoidTy, 0, D,
                              /*isInstance=*/true, /*isVariadic=*/false,
-                          /*isSynthesized=*/true, /*isImplicitlyDeclared=*/true,
+                          /*isPropertyAccessor=*/true, /*isImplicitlyDeclared=*/true,
                              /*isDefined=*/false, ObjCMethodDecl::Required);
     D->addInstanceMethod(DTORMethod);
     CodeGenFunction(*this).GenerateObjCCtorDtorMethod(D, DTORMethod, false);
-    D->setHasCXXStructors(true);
+    D->setHasDestructors(true);
   }
 
   // If the implementation doesn't have any ivar initializers, we don't need
@@ -2487,13 +2560,13 @@ void CodeGenModule::EmitObjCIvarInitializations(ObjCImplementationDecl *D) {
                                                 getContext().getObjCIdType(), 0, 
                                                 D, /*isInstance=*/true,
                                                 /*isVariadic=*/false,
-                                                /*isSynthesized=*/true,
+                                                /*isPropertyAccessor=*/true,
                                                 /*isImplicitlyDeclared=*/true,
                                                 /*isDefined=*/false,
                                                 ObjCMethodDecl::Required);
   D->addInstanceMethod(CTORMethod);
   CodeGenFunction(*this).GenerateObjCCtorDtorMethod(D, CTORMethod, true);
-  D->setHasCXXStructors(true);
+  D->setHasNonZeroConstructors(true);
 }
 
 /// EmitNamespace - Emit all declarations in a namespace.
@@ -2512,8 +2585,17 @@ void CodeGenModule::EmitLinkageSpec(const LinkageSpecDecl *LSD) {
   }
 
   for (RecordDecl::decl_iterator I = LSD->decls_begin(), E = LSD->decls_end();
-       I != E; ++I)
+       I != E; ++I) {
+    // Meta-data for ObjC class includes references to implemented methods.
+    // Generate class's method definitions first.
+    if (ObjCImplDecl *OID = dyn_cast<ObjCImplDecl>(*I)) {
+      for (ObjCContainerDecl::method_iterator M = OID->meth_begin(),
+           MEnd = OID->meth_end();
+           M != MEnd; ++M)
+        EmitTopLevelDecl(*M);
+    }
     EmitTopLevelDecl(*I);
+  }
 }
 
 /// EmitTopLevelDecl - Emit code for a single top level declaration.
@@ -2737,3 +2819,32 @@ void CodeGenModule::EmitCoverageFile() {
     }
   }
 }
+
+llvm::Constant *CodeGenModule::EmitUuidofInitializer(StringRef Uuid,
+                                                     QualType GuidType) {
+  // Sema has checked that all uuid strings are of the form
+  // "12345678-1234-1234-1234-1234567890ab".
+  assert(Uuid.size() == 36);
+  const char *Uuidstr = Uuid.data();
+  for (int i = 0; i < 36; ++i) {
+    if (i == 8 || i == 13 || i == 18 || i == 23) assert(Uuidstr[i] == '-');
+    else                                         assert(isxdigit(Uuidstr[i]));
+  }
+  
+  llvm::APInt Field0(32, StringRef(Uuidstr     , 8), 16);
+  llvm::APInt Field1(16, StringRef(Uuidstr +  9, 4), 16);
+  llvm::APInt Field2(16, StringRef(Uuidstr + 14, 4), 16);
+  static const int Field3ValueOffsets[] = { 19, 21, 24, 26, 28, 30, 32, 34 };
+
+  APValue InitStruct(APValue::UninitStruct(), /*NumBases=*/0, /*NumFields=*/4);
+  InitStruct.getStructField(0) = APValue(llvm::APSInt(Field0));
+  InitStruct.getStructField(1) = APValue(llvm::APSInt(Field1));
+  InitStruct.getStructField(2) = APValue(llvm::APSInt(Field2));
+  APValue& Arr = InitStruct.getStructField(3);
+  Arr = APValue(APValue::UninitArray(), 8, 8);
+  for (int t = 0; t < 8; ++t)
+    Arr.getArrayInitializedElt(t) = APValue(llvm::APSInt(
+          llvm::APInt(8, StringRef(Uuidstr + Field3ValueOffsets[t], 2), 16)));
+
+  return EmitConstantValue(InitStruct, GuidType);
+}
diff --git a/lib/CodeGen/CodeGenModule.h b/lib/CodeGen/CodeGenModule.h
index d6ff50d..1167c87 100644
--- a/lib/CodeGen/CodeGenModule.h
+++ b/lib/CodeGen/CodeGenModule.h
@@ -35,7 +35,7 @@ namespace llvm {
   class ConstantInt;
   class Function;
   class GlobalValue;
-  class TargetData;
+  class DataLayout;
   class FunctionType;
   class LLVMContext;
 }
@@ -210,8 +210,8 @@ struct ARCEntrypoints {
 /// CodeGenModule - This class organizes the cross-function state that is used
 /// while generating LLVM code.
 class CodeGenModule : public CodeGenTypeCache {
-  CodeGenModule(const CodeGenModule&);  // DO NOT IMPLEMENT
-  void operator=(const CodeGenModule&); // DO NOT IMPLEMENT
+  CodeGenModule(const CodeGenModule &) LLVM_DELETED_FUNCTION;
+  void operator=(const CodeGenModule &) LLVM_DELETED_FUNCTION;
 
   typedef std::vector<std::pair<llvm::Constant*, int> > CtorList;
 
@@ -219,7 +219,7 @@ class CodeGenModule : public CodeGenTypeCache {
   const LangOptions &LangOpts;
   const CodeGenOptions &CodeGenOpts;
   llvm::Module &TheModule;
-  const llvm::TargetData &TheTargetData;
+  const llvm::DataLayout &TheDataLayout;
   mutable const TargetCodeGenInfo *TheTargetCodeGenInfo;
   DiagnosticsEngine &Diags;
   CGCXXABI &ABI;
@@ -296,11 +296,18 @@ class CodeGenModule : public CodeGenTypeCache {
   /// order.
   llvm::DenseMap<const Decl*, unsigned> DelayedCXXInitPosition;
   
+  typedef std::pair<OrderGlobalInits, llvm::Function*> GlobalInitData;
+
+  struct GlobalInitPriorityCmp {
+    bool operator()(const GlobalInitData &LHS,
+                    const GlobalInitData &RHS) const {
+      return LHS.first.priority < RHS.first.priority;
+    }
+  };
+
   /// - Global variables with initializers whose order of initialization
   /// is set by init_priority attribute.
-  
-  SmallVector<std::pair<OrderGlobalInits, llvm::Function*>, 8> 
-    PrioritizedCXXGlobalInits;
+  SmallVector<GlobalInitData, 8> PrioritizedCXXGlobalInits;
 
   /// CXXGlobalDtors - Global destructor functions and arguments that need to
   /// run on termination.
@@ -357,7 +364,7 @@ class CodeGenModule : public CodeGenTypeCache {
   /// @}
 public:
   CodeGenModule(ASTContext &C, const CodeGenOptions &CodeGenOpts,
-                llvm::Module &M, const llvm::TargetData &TD,
+                llvm::Module &M, const llvm::DataLayout &TD,
                 DiagnosticsEngine &Diags);
 
   ~CodeGenModule();
@@ -451,7 +458,7 @@ public:
   CodeGenVTables &getVTables() { return VTables; }
   VTableContext &getVTableContext() { return VTables.getVTableContext(); }
   DiagnosticsEngine &getDiags() const { return Diags; }
-  const llvm::TargetData &getTargetData() const { return TheTargetData; }
+  const llvm::DataLayout &getDataLayout() const { return TheDataLayout; }
   const TargetInfo &getTarget() const { return Context.getTargetInfo(); }
   llvm::LLVMContext &getLLVMContext() { return VMContext; }
   const TargetCodeGenInfo &getTargetCodeGenInfo();
@@ -461,6 +468,7 @@ public:
 
   llvm::MDNode *getTBAAInfo(QualType QTy);
   llvm::MDNode *getTBAAInfoForVTablePtr();
+  llvm::MDNode *getTBAAStructInfo(QualType QTy);
 
   bool isTypeConstant(QualType QTy, bool ExcludeCtorDtor);
 
@@ -548,6 +556,9 @@ public:
   /// for the given type.
   llvm::Constant *GetAddrOfRTTIDescriptor(QualType Ty, bool ForEH = false);
 
+  /// GetAddrOfUuidDescriptor - Get the address of a uuid descriptor .
+  llvm::Constant *GetAddrOfUuidDescriptor(const CXXUuidofExpr* E);
+
   /// GetAddrOfThunk - Get the address of the thunk for the given global decl.
   llvm::Constant *GetAddrOfThunk(GlobalDecl GD, const ThunkInfo &Thunk);
 
@@ -701,7 +712,7 @@ public:
   llvm::Constant *CreateRuntimeFunction(llvm::FunctionType *Ty,
                                         StringRef Name,
                                         llvm::Attributes ExtraAttrs =
-                                          llvm::Attribute::None);
+                                          llvm::Attributes());
   /// CreateRuntimeVariable - Create a new runtime global variable with the
   /// specified type and name.
   llvm::Constant *CreateRuntimeVariable(llvm::Type *Ty,
@@ -880,7 +891,7 @@ private:
                                           GlobalDecl D,
                                           bool ForVTable,
                                           llvm::Attributes ExtraAttrs =
-                                            llvm::Attribute::None);
+                                            llvm::Attributes());
   llvm::Constant *GetOrCreateLLVMGlobal(StringRef MangledName,
                                         llvm::PointerType *PTy,
                                         const VarDecl *D,
@@ -984,6 +995,9 @@ private:
   /// to emit the .gcno and .gcda files in a way that persists in .bc files.
   void EmitCoverageFile();
 
+  /// Emits the initializer for a uuidof string.
+  llvm::Constant *EmitUuidofInitializer(StringRef uuidstr, QualType IIDType);
+
   /// MayDeferGeneration - Determine if the given decl can be emitted
   /// lazily; this is only relevant for definitions. The given decl
   /// must be either a function or var decl.
diff --git a/lib/CodeGen/CodeGenTBAA.cpp b/lib/CodeGen/CodeGenTBAA.cpp
index bab60af..d9004a0 100644
--- a/lib/CodeGen/CodeGenTBAA.cpp
+++ b/lib/CodeGen/CodeGenTBAA.cpp
@@ -17,6 +17,7 @@
 
 #include "CodeGenTBAA.h"
 #include "clang/AST/ASTContext.h"
+#include "clang/AST/RecordLayout.h"
 #include "clang/AST/Mangle.h"
 #include "clang/Frontend/CodeGenOptions.h"
 #include "llvm/LLVMContext.h"
@@ -167,3 +168,59 @@ CodeGenTBAA::getTBAAInfo(QualType QTy) {
 llvm::MDNode *CodeGenTBAA::getTBAAInfoForVTablePtr() {
   return MDHelper.createTBAANode("vtable pointer", getRoot());
 }
+
+bool
+CodeGenTBAA::CollectFields(uint64_t BaseOffset,
+                           QualType QTy,
+                           SmallVectorImpl<llvm::MDBuilder::TBAAStructField> &
+                             Fields,
+                           bool MayAlias) {
+  /* Things not handled yet include: C++ base classes, bitfields, */
+
+  if (const RecordType *TTy = QTy->getAs<RecordType>()) {
+    const RecordDecl *RD = TTy->getDecl()->getDefinition();
+    if (RD->hasFlexibleArrayMember())
+      return false;
+
+    // TODO: Handle C++ base classes.
+    if (const CXXRecordDecl *Decl = dyn_cast<CXXRecordDecl>(RD))
+      if (Decl->bases_begin() != Decl->bases_end())
+        return false;
+
+    const ASTRecordLayout &Layout = Context.getASTRecordLayout(RD);
+
+    unsigned idx = 0;
+    for (RecordDecl::field_iterator i = RD->field_begin(),
+         e = RD->field_end(); i != e; ++i, ++idx) {
+      uint64_t Offset = BaseOffset +
+                        Layout.getFieldOffset(idx) / Context.getCharWidth();
+      QualType FieldQTy = i->getType();
+      if (!CollectFields(Offset, FieldQTy, Fields,
+                         MayAlias || TypeHasMayAlias(FieldQTy)))
+        return false;
+    }
+    return true;
+  }
+
+  /* Otherwise, treat whatever it is as a field. */
+  uint64_t Offset = BaseOffset;
+  uint64_t Size = Context.getTypeSizeInChars(QTy).getQuantity();
+  llvm::MDNode *TBAAInfo = MayAlias ? getChar() : getTBAAInfo(QTy);
+  Fields.push_back(llvm::MDBuilder::TBAAStructField(Offset, Size, TBAAInfo));
+  return true;
+}
+
+llvm::MDNode *
+CodeGenTBAA::getTBAAStructInfo(QualType QTy) {
+  const Type *Ty = Context.getCanonicalType(QTy).getTypePtr();
+
+  if (llvm::MDNode *N = StructMetadataCache[Ty])
+    return N;
+
+  SmallVector<llvm::MDBuilder::TBAAStructField, 4> Fields;
+  if (CollectFields(0, QTy, Fields, TypeHasMayAlias(QTy)))
+    return MDHelper.createTBAAStructNode(Fields);
+
+  // For now, handle any other kind of type conservatively.
+  return StructMetadataCache[Ty] = NULL;
+}
diff --git a/lib/CodeGen/CodeGenTBAA.h b/lib/CodeGen/CodeGenTBAA.h
index c17a5cf..eedb996 100644
--- a/lib/CodeGen/CodeGenTBAA.h
+++ b/lib/CodeGen/CodeGenTBAA.h
@@ -49,6 +49,10 @@ class CodeGenTBAA {
   /// MetadataCache - This maps clang::Types to llvm::MDNodes describing them.
   llvm::DenseMap<const Type *, llvm::MDNode *> MetadataCache;
 
+  /// StructMetadataCache - This maps clang::Types to llvm::MDNodes describing
+  /// them for struct assignments.
+  llvm::DenseMap<const Type *, llvm::MDNode *> StructMetadataCache;
+
   llvm::MDNode *Root;
   llvm::MDNode *Char;
 
@@ -60,6 +64,13 @@ class CodeGenTBAA {
   /// considered to be equivalent to it.
   llvm::MDNode *getChar();
 
+  /// CollectFields - Collect information about the fields of a type for
+  /// !tbaa.struct metadata formation. Return false for an unsupported type.
+  bool CollectFields(uint64_t BaseOffset,
+                     QualType Ty,
+                     SmallVectorImpl<llvm::MDBuilder::TBAAStructField> &Fields,
+                     bool MayAlias);
+
 public:
   CodeGenTBAA(ASTContext &Ctx, llvm::LLVMContext &VMContext,
               const CodeGenOptions &CGO,
@@ -74,6 +85,10 @@ public:
   /// getTBAAInfoForVTablePtr - Get the TBAA MDNode to be used for a
   /// dereference of a vtable pointer.
   llvm::MDNode *getTBAAInfoForVTablePtr();
+
+  /// getTBAAStructInfo - Get the TBAAStruct MDNode to be used for a memcpy of
+  /// the given type.
+  llvm::MDNode *getTBAAStructInfo(QualType QTy);
 };
 
 }  // end namespace CodeGen
diff --git a/lib/CodeGen/CodeGenTypes.cpp b/lib/CodeGen/CodeGenTypes.cpp
index 9a78dae..3c6c5c9 100644
--- a/lib/CodeGen/CodeGenTypes.cpp
+++ b/lib/CodeGen/CodeGenTypes.cpp
@@ -23,13 +23,13 @@
 #include "clang/AST/RecordLayout.h"
 #include "llvm/DerivedTypes.h"
 #include "llvm/Module.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 using namespace clang;
 using namespace CodeGen;
 
 CodeGenTypes::CodeGenTypes(CodeGenModule &CGM)
   : Context(CGM.getContext()), Target(Context.getTargetInfo()),
-    TheModule(CGM.getModule()), TheTargetData(CGM.getTargetData()),
+    TheModule(CGM.getModule()), TheDataLayout(CGM.getDataLayout()),
     TheABIInfo(CGM.getTargetCodeGenInfo().getABIInfo()),
     TheCXXABI(CGM.getCXXABI()),
     CodeGenOpts(CGM.getCodeGenOpts()), CGM(CGM) {
diff --git a/lib/CodeGen/CodeGenTypes.h b/lib/CodeGen/CodeGenTypes.h
index 3c29d2d..0519911 100644
--- a/lib/CodeGen/CodeGenTypes.h
+++ b/lib/CodeGen/CodeGenTypes.h
@@ -23,7 +23,7 @@
 namespace llvm {
   class FunctionType;
   class Module;
-  class TargetData;
+  class DataLayout;
   class Type;
   class LLVMContext;
   class StructType;
@@ -62,7 +62,7 @@ class CodeGenTypes {
   ASTContext &Context;
   const TargetInfo &Target;
   llvm::Module &TheModule;
-  const llvm::TargetData &TheTargetData;
+  const llvm::DataLayout &TheDataLayout;
   const ABIInfo &TheABIInfo;
   CGCXXABI &TheCXXABI;
   const CodeGenOptions &CodeGenOpts;
@@ -108,7 +108,7 @@ public:
   CodeGenTypes(CodeGenModule &CGM);
   ~CodeGenTypes();
 
-  const llvm::TargetData &getTargetData() const { return TheTargetData; }
+  const llvm::DataLayout &getDataLayout() const { return TheDataLayout; }
   const TargetInfo &getTarget() const { return Target; }
   ASTContext &getContext() const { return Context; }
   const ABIInfo &getABIInfo() const { return TheABIInfo; }
diff --git a/lib/CodeGen/ItaniumCXXABI.cpp b/lib/CodeGen/ItaniumCXXABI.cpp
index 0b7ce36..245150c 100644
--- a/lib/CodeGen/ItaniumCXXABI.cpp
+++ b/lib/CodeGen/ItaniumCXXABI.cpp
@@ -23,11 +23,11 @@
 #include "CGVTables.h"
 #include "CodeGenFunction.h"
 #include "CodeGenModule.h"
-#include <clang/AST/Mangle.h>
-#include <clang/AST/Type.h>
-#include <llvm/Intrinsics.h>
-#include <llvm/Target/TargetData.h>
-#include <llvm/Value.h>
+#include "clang/AST/Mangle.h"
+#include "clang/AST/Type.h"
+#include "llvm/Intrinsics.h"
+#include "llvm/DataLayout.h"
+#include "llvm/Value.h"
 
 using namespace clang;
 using namespace CodeGen;
@@ -92,6 +92,10 @@ public:
                                           llvm::Value *Addr,
                                           const MemberPointerType *MPT);
 
+  llvm::Value *adjustToCompleteObject(CodeGenFunction &CGF,
+                                      llvm::Value *ptr,
+                                      QualType type);
+
   void BuildConstructorSignature(const CXXConstructorDecl *Ctor,
                                  CXXCtorType T,
                                  CanQualType &ResTy,
@@ -109,6 +113,7 @@ public:
   void EmitInstanceFunctionProlog(CodeGenFunction &CGF);
 
   StringRef GetPureVirtualCallName() { return "__cxa_pure_virtual"; }
+  StringRef GetDeletedVirtualCallName() { return "__cxa_deleted_virtual"; }
 
   CharUnits getArrayCookieSizeImpl(QualType elementType);
   llvm::Value *InitializeArrayCookie(CodeGenFunction &CGF,
@@ -299,7 +304,7 @@ llvm::Value *ItaniumCXXABI::EmitMemberDataPointerAddress(CodeGenFunction &CGF,
 
   CGBuilderTy &Builder = CGF.Builder;
 
-  unsigned AS = cast<llvm::PointerType>(Base->getType())->getAddressSpace();
+  unsigned AS = Base->getType()->getPointerAddressSpace();
 
   // Cast to char*.
   Base = Builder.CreateBitCast(Base, Builder.getInt8Ty()->getPointerTo(AS));
@@ -677,6 +682,25 @@ bool ItaniumCXXABI::isZeroInitializable(const MemberPointerType *MPT) {
   return MPT->getPointeeType()->isFunctionType();
 }
 
+/// The Itanium ABI always places an offset to the complete object
+/// at entry -2 in the vtable.
+llvm::Value *ItaniumCXXABI::adjustToCompleteObject(CodeGenFunction &CGF,
+                                                   llvm::Value *ptr,
+                                                   QualType type) {
+  // Grab the vtable pointer as an intptr_t*.
+  llvm::Value *vtable = CGF.GetVTablePtr(ptr, CGF.IntPtrTy->getPointerTo());
+
+  // Track back to entry -2 and pull out the offset there.
+  llvm::Value *offsetPtr = 
+    CGF.Builder.CreateConstInBoundsGEP1_64(vtable, -2, "complete-offset.ptr");
+  llvm::LoadInst *offset = CGF.Builder.CreateLoad(offsetPtr);
+  offset->setAlignment(CGF.PointerAlignInBytes);
+
+  // Apply the offset.
+  ptr = CGF.Builder.CreateBitCast(ptr, CGF.Int8PtrTy);
+  return CGF.Builder.CreateInBoundsGEP(ptr, offset);
+}
+
 /// The generic ABI passes 'this', plus a VTT if it's initializing a
 /// base subobject.
 void ItaniumCXXABI::BuildConstructorSignature(const CXXConstructorDecl *Ctor,
@@ -810,7 +834,7 @@ llvm::Value *ItaniumCXXABI::InitializeArrayCookie(CodeGenFunction &CGF,
                                                   QualType ElementType) {
   assert(requiresArrayCookie(expr));
 
-  unsigned AS = cast<llvm::PointerType>(NewPtr->getType())->getAddressSpace();
+  unsigned AS = NewPtr->getType()->getPointerAddressSpace();
 
   ASTContext &Ctx = getContext();
   QualType SizeTy = Ctx.getSizeType();
@@ -852,7 +876,7 @@ llvm::Value *ItaniumCXXABI::readArrayCookieImpl(CodeGenFunction &CGF,
       CGF.Builder.CreateConstInBoundsGEP1_64(numElementsPtr,
                                              numElementsOffset.getQuantity());
 
-  unsigned AS = cast<llvm::PointerType>(allocPtr->getType())->getAddressSpace();
+  unsigned AS = allocPtr->getType()->getPointerAddressSpace();
   numElementsPtr = 
     CGF.Builder.CreateBitCast(numElementsPtr, CGF.SizeTy->getPointerTo(AS));
   return CGF.Builder.CreateLoad(numElementsPtr);
@@ -878,7 +902,7 @@ llvm::Value *ARMCXXABI::InitializeArrayCookie(CodeGenFunction &CGF,
 
   // NewPtr is a char*.
 
-  unsigned AS = cast<llvm::PointerType>(NewPtr->getType())->getAddressSpace();
+  unsigned AS = NewPtr->getType()->getPointerAddressSpace();
 
   ASTContext &Ctx = getContext();
   CharUnits SizeSize = Ctx.getTypeSizeInChars(Ctx.getSizeType());
@@ -913,7 +937,7 @@ llvm::Value *ARMCXXABI::readArrayCookieImpl(CodeGenFunction &CGF,
   llvm::Value *numElementsPtr
     = CGF.Builder.CreateConstInBoundsGEP1_64(allocPtr, CGF.SizeSizeInBytes);
 
-  unsigned AS = cast<llvm::PointerType>(allocPtr->getType())->getAddressSpace();
+  unsigned AS = allocPtr->getType()->getPointerAddressSpace();
   numElementsPtr = 
     CGF.Builder.CreateBitCast(numElementsPtr, CGF.SizeTy->getPointerTo(AS));
   return CGF.Builder.CreateLoad(numElementsPtr);
@@ -927,9 +951,9 @@ static llvm::Constant *getGuardAcquireFn(CodeGenModule &CGM,
   llvm::FunctionType *FTy =
     llvm::FunctionType::get(CGM.getTypes().ConvertType(CGM.getContext().IntTy),
                             GuardPtrTy, /*isVarArg=*/false);
-  
   return CGM.CreateRuntimeFunction(FTy, "__cxa_guard_acquire",
-                                   llvm::Attribute::NoUnwind);
+                                   llvm::Attributes::get(CGM.getLLVMContext(),
+                                                 llvm::Attributes::NoUnwind));
 }
 
 static llvm::Constant *getGuardReleaseFn(CodeGenModule &CGM,
@@ -937,9 +961,9 @@ static llvm::Constant *getGuardReleaseFn(CodeGenModule &CGM,
   // void __cxa_guard_release(__guard *guard_object);
   llvm::FunctionType *FTy =
     llvm::FunctionType::get(CGM.VoidTy, GuardPtrTy, /*isVarArg=*/false);
-  
   return CGM.CreateRuntimeFunction(FTy, "__cxa_guard_release",
-                                   llvm::Attribute::NoUnwind);
+                                   llvm::Attributes::get(CGM.getLLVMContext(),
+                                                 llvm::Attributes::NoUnwind));
 }
 
 static llvm::Constant *getGuardAbortFn(CodeGenModule &CGM,
@@ -947,9 +971,9 @@ static llvm::Constant *getGuardAbortFn(CodeGenModule &CGM,
   // void __cxa_guard_abort(__guard *guard_object);
   llvm::FunctionType *FTy =
     llvm::FunctionType::get(CGM.VoidTy, GuardPtrTy, /*isVarArg=*/false);
-  
   return CGM.CreateRuntimeFunction(FTy, "__cxa_guard_abort",
-                                   llvm::Attribute::NoUnwind);
+                                   llvm::Attributes::get(CGM.getLLVMContext(),
+                                                 llvm::Attributes::NoUnwind));
 }
 
 namespace {
@@ -1149,7 +1173,7 @@ void ItaniumCXXABI::registerGlobalDtor(CodeGenFunction &CGF,
 
   // In Apple kexts, we want to add a global destructor entry.
   // FIXME: shouldn't this be guarded by some variable?
-  if (CGM.getContext().getLangOpts().AppleKext) {
+  if (CGM.getLangOpts().AppleKext) {
     // Generate a global destructor entry.
     return CGM.AddCXXDtorEntry(dtor, addr);
   }
diff --git a/lib/CodeGen/MicrosoftCXXABI.cpp b/lib/CodeGen/MicrosoftCXXABI.cpp
index 6a2925b..8d205c3 100644
--- a/lib/CodeGen/MicrosoftCXXABI.cpp
+++ b/lib/CodeGen/MicrosoftCXXABI.cpp
@@ -29,14 +29,18 @@ public:
   MicrosoftCXXABI(CodeGenModule &CGM) : CGCXXABI(CGM) {}
 
   StringRef GetPureVirtualCallName() { return "_purecall"; }
+  // No known support for deleted functions in MSVC yet, so this choice is
+  // arbitrary.
+  StringRef GetDeletedVirtualCallName() { return "_purecall"; }
+
+  llvm::Value *adjustToCompleteObject(CodeGenFunction &CGF,
+                                      llvm::Value *ptr,
+                                      QualType type);
 
   void BuildConstructorSignature(const CXXConstructorDecl *Ctor,
                                  CXXCtorType Type,
                                  CanQualType &ResTy,
-                                 SmallVectorImpl<CanQualType> &ArgTys) {
-    // 'this' is already in place
-    // TODO: 'for base' flag
-  }  
+                                 SmallVectorImpl<CanQualType> &ArgTys);
 
   void BuildDestructorSignature(const CXXDestructorDecl *Ctor,
                                 CXXDtorType Type,
@@ -48,15 +52,9 @@ public:
 
   void BuildInstanceFunctionParams(CodeGenFunction &CGF,
                                    QualType &ResTy,
-                                   FunctionArgList &Params) {
-    BuildThisParam(CGF, Params);
-    // TODO: 'for base' flag
-  }
+                                   FunctionArgList &Params);
 
-  void EmitInstanceFunctionProlog(CodeGenFunction &CGF) {
-    EmitThisParam(CGF);
-    // TODO: 'for base' flag
-  }
+  void EmitInstanceFunctionProlog(CodeGenFunction &CGF);
 
   void EmitGuardedInit(CodeGenFunction &CGF, const VarDecl &D,
                        llvm::GlobalVariable *DeclPtr,
@@ -99,10 +97,49 @@ public:
   llvm::Value *readArrayCookieImpl(CodeGenFunction &CGF,
                                    llvm::Value *allocPtr,
                                    CharUnits cookieSize);
+  static bool needThisReturn(GlobalDecl GD);
 };
 
 }
 
+llvm::Value *MicrosoftCXXABI::adjustToCompleteObject(CodeGenFunction &CGF,
+                                                     llvm::Value *ptr,
+                                                     QualType type) {
+  // FIXME: implement
+  return ptr;
+}
+
+bool MicrosoftCXXABI::needThisReturn(GlobalDecl GD) {
+  const CXXMethodDecl* MD = cast<CXXMethodDecl>(GD.getDecl());
+  return isa<CXXConstructorDecl>(MD);
+}
+
+void MicrosoftCXXABI::BuildConstructorSignature(const CXXConstructorDecl *Ctor,
+                                 CXXCtorType Type,
+                                 CanQualType &ResTy,
+                                 SmallVectorImpl<CanQualType> &ArgTys) {
+  // 'this' is already in place
+  // TODO: 'for base' flag
+  // Ctor returns this ptr
+  ResTy = ArgTys[0];
+}
+
+void MicrosoftCXXABI::BuildInstanceFunctionParams(CodeGenFunction &CGF,
+                                                  QualType &ResTy,
+                                                  FunctionArgList &Params) {
+  BuildThisParam(CGF, Params);
+  if (needThisReturn(CGF.CurGD)) {
+    ResTy = Params[0]->getType();
+  }
+}
+
+void MicrosoftCXXABI::EmitInstanceFunctionProlog(CodeGenFunction &CGF) {
+  EmitThisParam(CGF);
+  if (needThisReturn(CGF.CurGD)) {
+    CGF.Builder.CreateStore(getThisValue(CGF), CGF.ReturnValue);
+  }
+}
+
 bool MicrosoftCXXABI::requiresArrayCookie(const CXXDeleteExpr *expr,
                                    QualType elementType) {
   // Microsoft seems to completely ignore the possibility of a
@@ -127,7 +164,7 @@ CharUnits MicrosoftCXXABI::getArrayCookieSizeImpl(QualType type) {
 llvm::Value *MicrosoftCXXABI::readArrayCookieImpl(CodeGenFunction &CGF,
                                                   llvm::Value *allocPtr,
                                                   CharUnits cookieSize) {
-  unsigned AS = cast<llvm::PointerType>(allocPtr->getType())->getAddressSpace();
+  unsigned AS = allocPtr->getType()->getPointerAddressSpace();
   llvm::Value *numElementsPtr =
     CGF.Builder.CreateBitCast(allocPtr, CGF.SizeTy->getPointerTo(AS));
   return CGF.Builder.CreateLoad(numElementsPtr);
@@ -147,7 +184,7 @@ llvm::Value* MicrosoftCXXABI::InitializeArrayCookie(CodeGenFunction &CGF,
   llvm::Value *cookiePtr = newPtr;
 
   // Write the number of elements into the appropriate slot.
-  unsigned AS = cast<llvm::PointerType>(newPtr->getType())->getAddressSpace();
+  unsigned AS = newPtr->getType()->getPointerAddressSpace();
   llvm::Value *numElementsPtr
     = CGF.Builder.CreateBitCast(cookiePtr, CGF.SizeTy->getPointerTo(AS));
   CGF.Builder.CreateStore(numElements, numElementsPtr);
diff --git a/lib/CodeGen/ModuleBuilder.cpp b/lib/CodeGen/ModuleBuilder.cpp
index ea2389e..0125559 100644
--- a/lib/CodeGen/ModuleBuilder.cpp
+++ b/lib/CodeGen/ModuleBuilder.cpp
@@ -21,14 +21,14 @@
 #include "clang/Basic/TargetInfo.h"
 #include "llvm/LLVMContext.h"
 #include "llvm/Module.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/ADT/OwningPtr.h"
 using namespace clang;
 
 namespace {
   class CodeGeneratorImpl : public CodeGenerator {
     DiagnosticsEngine &Diags;
-    OwningPtr<const llvm::TargetData> TD;
+    OwningPtr<const llvm::DataLayout> TD;
     ASTContext *Ctx;
     const CodeGenOptions CodeGenOpts;  // Intentionally copied in.
   protected:
@@ -54,7 +54,7 @@ namespace {
 
       M->setTargetTriple(Ctx->getTargetInfo().getTriple().getTriple());
       M->setDataLayout(Ctx->getTargetInfo().getTargetDescription());
-      TD.reset(new llvm::TargetData(Ctx->getTargetInfo().getTargetDescription()));
+      TD.reset(new llvm::DataLayout(Ctx->getTargetInfo().getTargetDescription()));
       Builder.reset(new CodeGen::CodeGenModule(Context, CodeGenOpts,
                                                *M, *TD, Diags));
     }
diff --git a/lib/CodeGen/TargetInfo.cpp b/lib/CodeGen/TargetInfo.cpp
index 9c23ed9..ffff0d0 100644
--- a/lib/CodeGen/TargetInfo.cpp
+++ b/lib/CodeGen/TargetInfo.cpp
@@ -18,7 +18,7 @@
 #include "clang/AST/RecordLayout.h"
 #include "clang/Frontend/CodeGenOptions.h"
 #include "llvm/Type.h"
-#include "llvm/Target/TargetData.h"
+#include "llvm/DataLayout.h"
 #include "llvm/ADT/Triple.h"
 #include "llvm/Support/raw_ostream.h"
 using namespace clang;
@@ -51,8 +51,8 @@ llvm::LLVMContext &ABIInfo::getVMContext() const {
   return CGT.getLLVMContext();
 }
 
-const llvm::TargetData &ABIInfo::getTargetData() const {
-  return CGT.getTargetData();
+const llvm::DataLayout &ABIInfo::getDataLayout() const {
+  return CGT.getDataLayout();
 }
 
 
@@ -389,6 +389,90 @@ ABIArgInfo DefaultABIInfo::classifyReturnType(QualType RetTy) const {
           ABIArgInfo::getExtend() : ABIArgInfo::getDirect());
 }
 
+//===----------------------------------------------------------------------===//
+// le32/PNaCl bitcode ABI Implementation
+//===----------------------------------------------------------------------===//
+
+class PNaClABIInfo : public ABIInfo {
+ public:
+  PNaClABIInfo(CodeGen::CodeGenTypes &CGT) : ABIInfo(CGT) {}
+
+  ABIArgInfo classifyReturnType(QualType RetTy) const;
+  ABIArgInfo classifyArgumentType(QualType RetTy, unsigned &FreeRegs) const;
+
+  virtual void computeInfo(CGFunctionInfo &FI) const;
+  virtual llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
+                                 CodeGenFunction &CGF) const;
+};
+
+class PNaClTargetCodeGenInfo : public TargetCodeGenInfo {
+ public:
+  PNaClTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT)
+    : TargetCodeGenInfo(new PNaClABIInfo(CGT)) {}
+};
+
+void PNaClABIInfo::computeInfo(CGFunctionInfo &FI) const {
+    FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
+
+    unsigned FreeRegs = FI.getHasRegParm() ? FI.getRegParm() : 0;
+
+    for (CGFunctionInfo::arg_iterator it = FI.arg_begin(), ie = FI.arg_end();
+         it != ie; ++it)
+      it->info = classifyArgumentType(it->type, FreeRegs);
+  }
+
+llvm::Value *PNaClABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
+                                       CodeGenFunction &CGF) const {
+  return 0;
+}
+
+ABIArgInfo PNaClABIInfo::classifyArgumentType(QualType Ty,
+                                              unsigned &FreeRegs) const {
+  if (isAggregateTypeForABI(Ty)) {
+    // Records with non trivial destructors/constructors should not be passed
+    // by value.
+    FreeRegs = 0;
+    if (isRecordWithNonTrivialDestructorOrCopyConstructor(Ty))
+      return ABIArgInfo::getIndirect(0, /*ByVal=*/false);
+
+    return ABIArgInfo::getIndirect(0);
+  }
+
+  // Treat an enum type as its underlying type.
+  if (const EnumType *EnumTy = Ty->getAs<EnumType>())
+    Ty = EnumTy->getDecl()->getIntegerType();
+
+  ABIArgInfo BaseInfo = (Ty->isPromotableIntegerType() ?
+          ABIArgInfo::getExtend() : ABIArgInfo::getDirect());
+
+  // Regparm regs hold 32 bits.
+  unsigned SizeInRegs = (getContext().getTypeSize(Ty) + 31) / 32;
+  if (SizeInRegs == 0) return BaseInfo;
+  if (SizeInRegs > FreeRegs) {
+    FreeRegs = 0;
+    return BaseInfo;
+  }
+  FreeRegs -= SizeInRegs;
+  return BaseInfo.isDirect() ?
+      ABIArgInfo::getDirectInReg(BaseInfo.getCoerceToType()) :
+      ABIArgInfo::getExtendInReg(BaseInfo.getCoerceToType());
+}
+
+ABIArgInfo PNaClABIInfo::classifyReturnType(QualType RetTy) const {
+  if (RetTy->isVoidType())
+    return ABIArgInfo::getIgnore();
+
+  if (isAggregateTypeForABI(RetTy))
+    return ABIArgInfo::getIndirect(0);
+
+  // Treat an enum type as its underlying type.
+  if (const EnumType *EnumTy = RetTy->getAs<EnumType>())
+    RetTy = EnumTy->getDecl()->getIntegerType();
+
+  return (RetTy->isPromotableIntegerType() ?
+          ABIArgInfo::getExtend() : ABIArgInfo::getDirect());
+}
+
 /// UseX86_MMXType - Return true if this is an MMX type that should use the
 /// special x86_mmx type.
 bool UseX86_MMXType(llvm::Type *IRType) {
@@ -435,7 +519,8 @@ class X86_32ABIInfo : public ABIInfo {
 
   /// getIndirectResult - Give a source type \arg Ty, return a suitable result
   /// such that the argument will be passed in memory.
-  ABIArgInfo getIndirectResult(QualType Ty, bool ByVal = true) const;
+  ABIArgInfo getIndirectResult(QualType Ty, bool ByVal,
+                               unsigned &FreeRegs) const;
 
   /// \brief Return the alignment to use for the given type on the stack.
   unsigned getTypeStackAlignInBytes(QualType Ty, unsigned Align) const;
@@ -443,9 +528,10 @@ class X86_32ABIInfo : public ABIInfo {
   Class classify(QualType Ty) const;
   ABIArgInfo classifyReturnType(QualType RetTy,
                                 unsigned callingConvention) const;
-  ABIArgInfo classifyArgumentTypeWithReg(QualType RetTy,
-                                         unsigned &FreeRegs) const;
-  ABIArgInfo classifyArgumentType(QualType RetTy) const;
+  ABIArgInfo classifyArgumentType(QualType RetTy, unsigned &FreeRegs,
+                                  bool IsFastCall) const;
+  bool shouldUseInReg(QualType Ty, unsigned &FreeRegs,
+                      bool IsFastCall, bool &NeedsPadding) const;
 
 public:
 
@@ -682,9 +768,15 @@ unsigned X86_32ABIInfo::getTypeStackAlignInBytes(QualType Ty,
   return MinABIStackAlignInBytes;
 }
 
-ABIArgInfo X86_32ABIInfo::getIndirectResult(QualType Ty, bool ByVal) const {
-  if (!ByVal)
+ABIArgInfo X86_32ABIInfo::getIndirectResult(QualType Ty, bool ByVal,
+                                            unsigned &FreeRegs) const {
+  if (!ByVal) {
+    if (FreeRegs) {
+      --FreeRegs; // Non byval indirects just use one pointer.
+      return ABIArgInfo::getIndirectInReg(0, false);
+    }
     return ABIArgInfo::getIndirect(0, false);
+  }
 
   // Compute the byval alignment.
   unsigned TypeAlign = getContext().getTypeAlign(Ty) / 8;
@@ -714,45 +806,51 @@ X86_32ABIInfo::Class X86_32ABIInfo::classify(QualType Ty) const {
   return Integer;
 }
 
-ABIArgInfo
-X86_32ABIInfo::classifyArgumentTypeWithReg(QualType Ty,
-                                           unsigned &FreeRegs) const {
-  // Common case first.
-  if (FreeRegs == 0)
-    return classifyArgumentType(Ty);
-
+bool X86_32ABIInfo::shouldUseInReg(QualType Ty, unsigned &FreeRegs,
+                                   bool IsFastCall, bool &NeedsPadding) const {
+  NeedsPadding = false;
   Class C = classify(Ty);
   if (C == Float)
-    return classifyArgumentType(Ty);
+    return false;
+
+  unsigned Size = getContext().getTypeSize(Ty);
+  unsigned SizeInRegs = (Size + 31) / 32;
 
-  unsigned SizeInRegs = (getContext().getTypeSize(Ty) + 31) / 32;
   if (SizeInRegs == 0)
-    return classifyArgumentType(Ty);
+    return false;
 
   if (SizeInRegs > FreeRegs) {
     FreeRegs = 0;
-    return classifyArgumentType(Ty);
+    return false;
   }
-  assert(SizeInRegs >= 1 && SizeInRegs <= 3);
+
   FreeRegs -= SizeInRegs;
 
-  // If it is a simple scalar, keep the type so that we produce a cleaner IR.
-  ABIArgInfo Foo = classifyArgumentType(Ty);
-  if (Foo.isDirect() && !Foo.getDirectOffset() && !Foo.getPaddingType())
-    return ABIArgInfo::getDirectInReg(Foo.getCoerceToType());
-  if (Foo.isExtend())
-    return ABIArgInfo::getExtendInReg(Foo.getCoerceToType());
+  if (IsFastCall) {
+    if (Size > 32)
+      return false;
+
+    if (Ty->isIntegralOrEnumerationType())
+      return true;
+
+    if (Ty->isPointerType())
+      return true;
+
+    if (Ty->isReferenceType())
+      return true;
+
+    if (FreeRegs)
+      NeedsPadding = true;
 
-  llvm::LLVMContext &LLVMContext = getVMContext();
-  llvm::Type *Int32 = llvm::Type::getInt32Ty(LLVMContext);
-  SmallVector<llvm::Type*, 3> Elements;
-  for (unsigned I = 0; I < SizeInRegs; ++I)
-    Elements.push_back(Int32);
-  llvm::Type *Result = llvm::StructType::get(LLVMContext, Elements);
-  return ABIArgInfo::getDirectInReg(Result);
+    return false;
+  }
+
+  return true;
 }
 
-ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty) const {
+ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty,
+                                               unsigned &FreeRegs,
+                                               bool IsFastCall) const {
   // FIXME: Set alignment on indirect arguments.
   if (isAggregateTypeForABI(Ty)) {
     // Structures with flexible arrays are always indirect.
@@ -760,25 +858,38 @@ ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty) const {
       // Structures with either a non-trivial destructor or a non-trivial
       // copy constructor are always indirect.
       if (hasNonTrivialDestructorOrCopyConstructor(RT))
-        return getIndirectResult(Ty, /*ByVal=*/false);
+        return getIndirectResult(Ty, false, FreeRegs);
 
       if (RT->getDecl()->hasFlexibleArrayMember())
-        return getIndirectResult(Ty);
+        return getIndirectResult(Ty, true, FreeRegs);
     }
 
     // Ignore empty structs/unions.
     if (isEmptyRecord(getContext(), Ty, true))
       return ABIArgInfo::getIgnore();
 
+    llvm::LLVMContext &LLVMContext = getVMContext();
+    llvm::IntegerType *Int32 = llvm::Type::getInt32Ty(LLVMContext);
+    bool NeedsPadding;
+    if (shouldUseInReg(Ty, FreeRegs, IsFastCall, NeedsPadding)) {
+      unsigned SizeInRegs = (getContext().getTypeSize(Ty) + 31) / 32;
+      SmallVector<llvm::Type*, 3> Elements;
+      for (unsigned I = 0; I < SizeInRegs; ++I)
+        Elements.push_back(Int32);
+      llvm::Type *Result = llvm::StructType::get(LLVMContext, Elements);
+      return ABIArgInfo::getDirectInReg(Result);
+    }
+    llvm::IntegerType *PaddingType = NeedsPadding ? Int32 : 0;
+
     // Expand small (<= 128-bit) record types when we know that the stack layout
     // of those arguments will match the struct. This is important because the
     // LLVM backend isn't smart enough to remove byval, which inhibits many
     // optimizations.
     if (getContext().getTypeSize(Ty) <= 4*32 &&
         canExpandIndirectArgument(Ty, getContext()))
-      return ABIArgInfo::getExpand();
+      return ABIArgInfo::getExpandWithPadding(IsFastCall, PaddingType);
 
-    return getIndirectResult(Ty);
+    return getIndirectResult(Ty, true, FreeRegs);
   }
 
   if (const VectorType *VT = Ty->getAs<VectorType>()) {
@@ -809,16 +920,32 @@ ABIArgInfo X86_32ABIInfo::classifyArgumentType(QualType Ty) const {
   if (const EnumType *EnumTy = Ty->getAs<EnumType>())
     Ty = EnumTy->getDecl()->getIntegerType();
 
-  return (Ty->isPromotableIntegerType() ?
-          ABIArgInfo::getExtend() : ABIArgInfo::getDirect());
+  bool NeedsPadding;
+  bool InReg = shouldUseInReg(Ty, FreeRegs, IsFastCall, NeedsPadding);
+
+  if (Ty->isPromotableIntegerType()) {
+    if (InReg)
+      return ABIArgInfo::getExtendInReg();
+    return ABIArgInfo::getExtend();
+  }
+  if (InReg)
+    return ABIArgInfo::getDirectInReg();
+  return ABIArgInfo::getDirect();
 }
 
 void X86_32ABIInfo::computeInfo(CGFunctionInfo &FI) const {
   FI.getReturnInfo() = classifyReturnType(FI.getReturnType(),
                                           FI.getCallingConvention());
 
-  unsigned FreeRegs = FI.getHasRegParm() ? FI.getRegParm() :
-    DefaultNumRegisterParameters;
+  unsigned CC = FI.getCallingConvention();
+  bool IsFastCall = CC == llvm::CallingConv::X86_FastCall;
+  unsigned FreeRegs;
+  if (IsFastCall)
+    FreeRegs = 2;
+  else if (FI.getHasRegParm())
+    FreeRegs = FI.getRegParm();
+  else
+    FreeRegs = DefaultNumRegisterParameters;
 
   // If the return value is indirect, then the hidden argument is consuming one
   // integer register.
@@ -832,7 +959,7 @@ void X86_32ABIInfo::computeInfo(CGFunctionInfo &FI) const {
 
   for (CGFunctionInfo::arg_iterator it = FI.arg_begin(), ie = FI.arg_end();
        it != ie; ++it)
-    it->info = classifyArgumentTypeWithReg(it->type, FreeRegs);
+    it->info = classifyArgumentType(it->type, FreeRegs, IsFastCall);
 }
 
 llvm::Value *X86_32ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
@@ -884,7 +1011,10 @@ void X86_32TargetCodeGenInfo::SetTargetAttributes(const Decl *D,
       llvm::Function *Fn = cast<llvm::Function>(GV);
 
       // Now add the 'alignstack' attribute with a value of 16.
-      Fn->addFnAttr(llvm::Attribute::constructStackAlignmentFromInt(16));
+      llvm::AttrBuilder B;
+      B.addStackAlignmentAttr(16);
+      Fn->addAttribute(llvm::AttrListPtr::FunctionIndex,
+                       llvm::Attributes::get(CGM.getLLVMContext(), B));
     }
   }
 }
@@ -1030,10 +1160,15 @@ class X86_64ABIInfo : public ABIInfo {
   }
 
   bool HasAVX;
+  // Some ABIs (e.g. X32 ABI and Native Client OS) use 32 bit pointers on
+  // 64-bit hardware.
+  bool Has64BitPointers;
 
 public:
   X86_64ABIInfo(CodeGen::CodeGenTypes &CGT, bool hasavx) :
-      ABIInfo(CGT), HasAVX(hasavx) {}
+      ABIInfo(CGT), HasAVX(hasavx),
+      Has64BitPointers(CGT.getDataLayout().getPointerSize(0) == 8) {
+  }
 
   bool isPassedUsingAVXType(QualType type) const {
     unsigned neededInt, neededSSE;
@@ -1070,7 +1205,7 @@ public:
 class X86_64TargetCodeGenInfo : public TargetCodeGenInfo {
 public:
   X86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool HasAVX)
-    : TargetCodeGenInfo(new X86_64ABIInfo(CGT, HasAVX)) {}
+      : TargetCodeGenInfo(new X86_64ABIInfo(CGT, HasAVX)) {}
 
   const X86_64ABIInfo &getABIInfo() const {
     return static_cast<const X86_64ABIInfo&>(TargetCodeGenInfo::getABIInfo());
@@ -1243,7 +1378,10 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase,
       Hi = Integer;
     } else if (k >= BuiltinType::Bool && k <= BuiltinType::LongLong) {
       Current = Integer;
-    } else if (k == BuiltinType::Float || k == BuiltinType::Double) {
+    } else if ((k == BuiltinType::Float || k == BuiltinType::Double) ||
+               (k == BuiltinType::LongDouble &&
+                getContext().getTargetInfo().getTriple().getOS() ==
+                llvm::Triple::NativeClient)) {
       Current = SSE;
     } else if (k == BuiltinType::LongDouble) {
       Lo = X87;
@@ -1266,7 +1404,7 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase,
   }
 
   if (Ty->isMemberPointerType()) {
-    if (Ty->isMemberFunctionPointerType())
+    if (Ty->isMemberFunctionPointerType() && Has64BitPointers)
       Lo = Hi = Integer;
     else
       Current = Integer;
@@ -1329,7 +1467,10 @@ void X86_64ABIInfo::classify(QualType Ty, uint64_t OffsetBase,
         Lo = Hi = Integer;
     } else if (ET == getContext().FloatTy)
       Current = SSE;
-    else if (ET == getContext().DoubleTy)
+    else if (ET == getContext().DoubleTy ||
+             (ET == getContext().LongDoubleTy &&
+              getContext().getTargetInfo().getTriple().getOS() ==
+              llvm::Triple::NativeClient))
       Lo = Hi = SSE;
     else if (ET == getContext().LongDoubleTy)
       Current = ComplexX87;
@@ -1708,7 +1849,7 @@ static bool BitsContainNoUserData(QualType Ty, unsigned StartBit,
 /// float at offset 4.  It is conservatively correct for this routine to return
 /// false.
 static bool ContainsFloatAtOffset(llvm::Type *IRType, unsigned IROffset,
-                                  const llvm::TargetData &TD) {
+                                  const llvm::DataLayout &TD) {
   // Base case if we find a float.
   if (IROffset == 0 && IRType->isFloatTy())
     return true;
@@ -1748,8 +1889,8 @@ GetSSETypeAtOffset(llvm::Type *IRType, unsigned IROffset,
   // We want to pass as <2 x float> if the LLVM IR type contains a float at
   // offset+0 and offset+4.  Walk the LLVM IR type to find out if this is the
   // case.
-  if (ContainsFloatAtOffset(IRType, IROffset, getTargetData()) &&
-      ContainsFloatAtOffset(IRType, IROffset+4, getTargetData()))
+  if (ContainsFloatAtOffset(IRType, IROffset, getDataLayout()) &&
+      ContainsFloatAtOffset(IRType, IROffset+4, getDataLayout()))
     return llvm::VectorType::get(llvm::Type::getFloatTy(getVMContext()), 2);
 
   return llvm::Type::getDoubleTy(getVMContext());
@@ -1777,7 +1918,8 @@ GetINTEGERTypeAtOffset(llvm::Type *IRType, unsigned IROffset,
   // returning an 8-byte unit starting with it.  See if we can safely use it.
   if (IROffset == 0) {
     // Pointers and int64's always fill the 8-byte unit.
-    if (isa<llvm::PointerType>(IRType) || IRType->isIntegerTy(64))
+    if ((isa<llvm::PointerType>(IRType) && Has64BitPointers) ||
+        IRType->isIntegerTy(64))
       return IRType;
 
     // If we have a 1/2/4-byte integer, we can use it only if the rest of the
@@ -1787,8 +1929,10 @@ GetINTEGERTypeAtOffset(llvm::Type *IRType, unsigned IROffset,
     // have to do this analysis on the source type because we can't depend on
     // unions being lowered a specific way etc.
     if (IRType->isIntegerTy(8) || IRType->isIntegerTy(16) ||
-        IRType->isIntegerTy(32)) {
-      unsigned BitWidth = cast<llvm::IntegerType>(IRType)->getBitWidth();
+        IRType->isIntegerTy(32) ||
+        (isa<llvm::PointerType>(IRType) && !Has64BitPointers)) {
+      unsigned BitWidth = isa<llvm::PointerType>(IRType) ? 32 :
+          cast<llvm::IntegerType>(IRType)->getBitWidth();
 
       if (BitsContainNoUserData(SourceTy, SourceOffset*8+BitWidth,
                                 SourceOffset*8+64, getContext()))
@@ -1798,7 +1942,7 @@ GetINTEGERTypeAtOffset(llvm::Type *IRType, unsigned IROffset,
 
   if (llvm::StructType *STy = dyn_cast<llvm::StructType>(IRType)) {
     // If this is a struct, recurse into the field at the specified offset.
-    const llvm::StructLayout *SL = getTargetData().getStructLayout(STy);
+    const llvm::StructLayout *SL = getDataLayout().getStructLayout(STy);
     if (IROffset < SL->getSizeInBytes()) {
       unsigned FieldIdx = SL->getElementContainingOffset(IROffset);
       IROffset -= SL->getElementOffset(FieldIdx);
@@ -1810,7 +1954,7 @@ GetINTEGERTypeAtOffset(llvm::Type *IRType, unsigned IROffset,
 
   if (llvm::ArrayType *ATy = dyn_cast<llvm::ArrayType>(IRType)) {
     llvm::Type *EltTy = ATy->getElementType();
-    unsigned EltSize = getTargetData().getTypeAllocSize(EltTy);
+    unsigned EltSize = getDataLayout().getTypeAllocSize(EltTy);
     unsigned EltOffset = IROffset/EltSize*EltSize;
     return GetINTEGERTypeAtOffset(EltTy, IROffset-EltOffset, SourceTy,
                                   SourceOffset);
@@ -1837,14 +1981,14 @@ GetINTEGERTypeAtOffset(llvm::Type *IRType, unsigned IROffset,
 /// return {i32*, float}.
 static llvm::Type *
 GetX86_64ByValArgumentPair(llvm::Type *Lo, llvm::Type *Hi,
-                           const llvm::TargetData &TD) {
+                           const llvm::DataLayout &TD) {
   // In order to correctly satisfy the ABI, we need to the high part to start
   // at offset 8.  If the high and low parts we inferred are both 4-byte types
   // (e.g. i32 and i32) then the resultant struct type ({i32,i32}) won't have
   // the second element at offset 8.  Check for this:
   unsigned LoSize = (unsigned)TD.getTypeAllocSize(Lo);
   unsigned HiAlign = TD.getABITypeAlignment(Hi);
-  unsigned HiStart = llvm::TargetData::RoundUpAlignment(LoSize, HiAlign);
+  unsigned HiStart = llvm::DataLayout::RoundUpAlignment(LoSize, HiAlign);
   assert(HiStart != 0 && HiStart <= 8 && "Invalid x86-64 argument pair!");
 
   // To handle this, we have to increase the size of the low part so that the
@@ -1996,7 +2140,7 @@ classifyReturnType(QualType RetTy) const {
   // known to pass in the high eightbyte of the result.  We do this by forming a
   // first class struct aggregate with the high and low part: {low, high}
   if (HighPart)
-    ResType = GetX86_64ByValArgumentPair(ResType, HighPart, getTargetData());
+    ResType = GetX86_64ByValArgumentPair(ResType, HighPart, getDataLayout());
 
   return ABIArgInfo::getDirect(ResType);
 }
@@ -2122,7 +2266,7 @@ ABIArgInfo X86_64ABIInfo::classifyArgumentType(
   // known to pass in the high eightbyte of the result.  We do this by forming a
   // first class struct aggregate with the high and low part: {low, high}
   if (HighPart)
-    ResType = GetX86_64ByValArgumentPair(ResType, HighPart, getTargetData());
+    ResType = GetX86_64ByValArgumentPair(ResType, HighPart, getDataLayout());
 
   return ABIArgInfo::getDirect(ResType);
 }
@@ -2435,6 +2579,43 @@ llvm::Value *WinX86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
   return AddrTyped;
 }
 
+namespace {
+
+class NaClX86_64ABIInfo : public ABIInfo {
+ public:
+  NaClX86_64ABIInfo(CodeGen::CodeGenTypes &CGT, bool HasAVX)
+      : ABIInfo(CGT), PInfo(CGT), NInfo(CGT, HasAVX) {}
+  virtual void computeInfo(CGFunctionInfo &FI) const;
+  virtual llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
+                                 CodeGenFunction &CGF) const;
+ private:
+  PNaClABIInfo PInfo;  // Used for generating calls with pnaclcall callingconv.
+  X86_64ABIInfo NInfo; // Used for everything else.
+};
+
+class NaClX86_64TargetCodeGenInfo : public TargetCodeGenInfo  {
+ public:
+  NaClX86_64TargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, bool HasAVX)
+      : TargetCodeGenInfo(new NaClX86_64ABIInfo(CGT, HasAVX)) {}
+};
+
+}
+
+void NaClX86_64ABIInfo::computeInfo(CGFunctionInfo &FI) const {
+  if (FI.getASTCallingConvention() == CC_PnaclCall)
+    PInfo.computeInfo(FI);
+  else
+    NInfo.computeInfo(FI);
+}
+
+llvm::Value *NaClX86_64ABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
+                                          CodeGenFunction &CGF) const {
+  // Always use the native convention; calling pnacl-style varargs functions
+  // is unuspported.
+  return NInfo.EmitVAArg(VAListAddr, Ty, CGF);
+}
+
+
 // PowerPC-32
 
 namespace {
@@ -2497,6 +2678,62 @@ PPC32TargetCodeGenInfo::initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
 // PowerPC-64
 
 namespace {
+/// PPC64_SVR4_ABIInfo - The 64-bit PowerPC ELF (SVR4) ABI information.
+class PPC64_SVR4_ABIInfo : public DefaultABIInfo {
+
+public:
+  PPC64_SVR4_ABIInfo(CodeGen::CodeGenTypes &CGT) : DefaultABIInfo(CGT) {}
+
+  bool isPromotableTypeForABI(QualType Ty) const;
+
+  ABIArgInfo classifyReturnType(QualType RetTy) const;
+  ABIArgInfo classifyArgumentType(QualType Ty) const;
+
+  // TODO: We can add more logic to computeInfo to improve performance.
+  // Example: For aggregate arguments that fit in a register, we could
+  // use getDirectInReg (as is done below for structs containing a single
+  // floating-point value) to avoid pushing them to memory on function
+  // entry.  This would require changing the logic in PPCISelLowering
+  // when lowering the parameters in the caller and args in the callee.
+  virtual void computeInfo(CGFunctionInfo &FI) const {
+    FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
+    for (CGFunctionInfo::arg_iterator it = FI.arg_begin(), ie = FI.arg_end();
+         it != ie; ++it) {
+      // We rely on the default argument classification for the most part.
+      // One exception:  An aggregate containing a single floating-point
+      // item must be passed in a register if one is available.
+      const Type *T = isSingleElementStruct(it->type, getContext());
+      if (T) {
+        const BuiltinType *BT = T->getAs<BuiltinType>();
+        if (BT && BT->isFloatingPoint()) {
+          QualType QT(T, 0);
+          it->info = ABIArgInfo::getDirectInReg(CGT.ConvertType(QT));
+          continue;
+        }
+      }
+      it->info = classifyArgumentType(it->type);
+    }
+  }
+
+  virtual llvm::Value *EmitVAArg(llvm::Value *VAListAddr, 
+                                 QualType Ty,
+                                 CodeGenFunction &CGF) const;
+};
+
+class PPC64_SVR4_TargetCodeGenInfo : public TargetCodeGenInfo {
+public:
+  PPC64_SVR4_TargetCodeGenInfo(CodeGenTypes &CGT)
+    : TargetCodeGenInfo(new PPC64_SVR4_ABIInfo(CGT)) {}
+
+  int getDwarfEHStackPointer(CodeGen::CodeGenModule &M) const {
+    // This is recovered from gcc output.
+    return 1; // r1 is the dedicated stack pointer
+  }
+
+  bool initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
+                               llvm::Value *Address) const;
+};
+
 class PPC64TargetCodeGenInfo : public DefaultTargetCodeGenInfo {
 public:
   PPC64TargetCodeGenInfo(CodeGenTypes &CGT) : DefaultTargetCodeGenInfo(CGT) {}
@@ -2512,9 +2749,94 @@ public:
 
 }
 
+// Return true if the ABI requires Ty to be passed sign- or zero-
+// extended to 64 bits.
 bool
-PPC64TargetCodeGenInfo::initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
-                                                llvm::Value *Address) const {
+PPC64_SVR4_ABIInfo::isPromotableTypeForABI(QualType Ty) const {
+  // Treat an enum type as its underlying type.
+  if (const EnumType *EnumTy = Ty->getAs<EnumType>())
+    Ty = EnumTy->getDecl()->getIntegerType();
+
+  // Promotable integer types are required to be promoted by the ABI.
+  if (Ty->isPromotableIntegerType())
+    return true;
+
+  // In addition to the usual promotable integer types, we also need to
+  // extend all 32-bit types, since the ABI requires promotion to 64 bits.
+  if (const BuiltinType *BT = Ty->getAs<BuiltinType>())
+    switch (BT->getKind()) {
+    case BuiltinType::Int:
+    case BuiltinType::UInt:
+      return true;
+    default:
+      break;
+    }
+
+  return false;
+}
+
+ABIArgInfo
+PPC64_SVR4_ABIInfo::classifyArgumentType(QualType Ty) const {
+  if (isAggregateTypeForABI(Ty)) {
+    // Records with non trivial destructors/constructors should not be passed
+    // by value.
+    if (isRecordWithNonTrivialDestructorOrCopyConstructor(Ty))
+      return ABIArgInfo::getIndirect(0, /*ByVal=*/false);
+
+    return ABIArgInfo::getIndirect(0);
+  }
+
+  return (isPromotableTypeForABI(Ty) ?
+          ABIArgInfo::getExtend() : ABIArgInfo::getDirect());
+}
+
+ABIArgInfo
+PPC64_SVR4_ABIInfo::classifyReturnType(QualType RetTy) const {
+  if (RetTy->isVoidType())
+    return ABIArgInfo::getIgnore();
+
+  if (isAggregateTypeForABI(RetTy))
+    return ABIArgInfo::getIndirect(0);
+
+  return (isPromotableTypeForABI(RetTy) ?
+          ABIArgInfo::getExtend() : ABIArgInfo::getDirect());
+}
+
+// Based on ARMABIInfo::EmitVAArg, adjusted for 64-bit machine.
+llvm::Value *PPC64_SVR4_ABIInfo::EmitVAArg(llvm::Value *VAListAddr,
+                                           QualType Ty,
+                                           CodeGenFunction &CGF) const {
+  llvm::Type *BP = CGF.Int8PtrTy;
+  llvm::Type *BPP = CGF.Int8PtrPtrTy;
+
+  CGBuilderTy &Builder = CGF.Builder;
+  llvm::Value *VAListAddrAsBPP = Builder.CreateBitCast(VAListAddr, BPP, "ap");
+  llvm::Value *Addr = Builder.CreateLoad(VAListAddrAsBPP, "ap.cur");
+
+  // Update the va_list pointer.
+  unsigned SizeInBytes = CGF.getContext().getTypeSize(Ty) / 8;
+  unsigned Offset = llvm::RoundUpToAlignment(SizeInBytes, 8);
+  llvm::Value *NextAddr =
+    Builder.CreateGEP(Addr, llvm::ConstantInt::get(CGF.Int64Ty, Offset),
+                      "ap.next");
+  Builder.CreateStore(NextAddr, VAListAddrAsBPP);
+
+  // If the argument is smaller than 8 bytes, it is right-adjusted in
+  // its doubleword slot.  Adjust the pointer to pick it up from the
+  // correct offset.
+  if (SizeInBytes < 8) {
+    llvm::Value *AddrAsInt = Builder.CreatePtrToInt(Addr, CGF.Int64Ty);
+    AddrAsInt = Builder.CreateAdd(AddrAsInt, Builder.getInt64(8 - SizeInBytes));
+    Addr = Builder.CreateIntToPtr(AddrAsInt, BP);
+  }
+
+  llvm::Type *PTy = llvm::PointerType::getUnqual(CGF.ConvertType(Ty));
+  return Builder.CreateBitCast(Addr, PTy);
+}
+
+static bool
+PPC64_initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
+                              llvm::Value *Address) {
   // This is calculated from the LLVM and GCC tables and verified
   // against gcc output.  AFAIK all ABIs use the same encoding.
 
@@ -2553,6 +2875,21 @@ PPC64TargetCodeGenInfo::initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
   return false;
 }
 
+bool
+PPC64_SVR4_TargetCodeGenInfo::initDwarfEHRegSizeTable(
+  CodeGen::CodeGenFunction &CGF,
+  llvm::Value *Address) const {
+
+  return PPC64_initDwarfEHRegSizeTable(CGF, Address);
+}
+
+bool
+PPC64TargetCodeGenInfo::initDwarfEHRegSizeTable(CodeGen::CodeGenFunction &CGF,
+                                                llvm::Value *Address) const {
+
+  return PPC64_initDwarfEHRegSizeTable(CGF, Address);
+}
+
 //===----------------------------------------------------------------------===//
 // ARM ABI Implementation
 //===----------------------------------------------------------------------===//
@@ -2576,14 +2913,18 @@ public:
   bool isEABI() const {
     StringRef Env =
       getContext().getTargetInfo().getTriple().getEnvironmentName();
-    return (Env == "gnueabi" || Env == "eabi" || Env == "androideabi");
+    return (Env == "gnueabi" || Env == "eabi" ||
+            Env == "android" || Env == "androideabi");
   }
 
 private:
   ABIKind getABIKind() const { return Kind; }
 
   ABIArgInfo classifyReturnType(QualType RetTy) const;
-  ABIArgInfo classifyArgumentType(QualType RetTy) const;
+  ABIArgInfo classifyArgumentType(QualType RetTy, int *VFPRegs,
+                                  unsigned &AllocatedVFP,
+                                  bool &IsHA) const;
+  bool isIllegalVectorType(QualType Ty) const;
 
   virtual void computeInfo(CGFunctionInfo &FI) const;
 
@@ -2626,10 +2967,33 @@ public:
 }
 
 void ARMABIInfo::computeInfo(CGFunctionInfo &FI) const {
+  // To correctly handle Homogeneous Aggregate, we need to keep track of the
+  // VFP registers allocated so far.
+  // C.1.vfp If the argument is a VFP CPRC and there are sufficient consecutive
+  // VFP registers of the appropriate type unallocated then the argument is
+  // allocated to the lowest-numbered sequence of such registers.
+  // C.2.vfp If the argument is a VFP CPRC then any VFP registers that are
+  // unallocated are marked as unavailable. 
+  unsigned AllocatedVFP = 0;
+  int VFPRegs[16] = { 0 };
   FI.getReturnInfo() = classifyReturnType(FI.getReturnType());
   for (CGFunctionInfo::arg_iterator it = FI.arg_begin(), ie = FI.arg_end();
-       it != ie; ++it)
-    it->info = classifyArgumentType(it->type);
+       it != ie; ++it) {
+    unsigned PreAllocation = AllocatedVFP;
+    bool IsHA = false;
+    // 6.1.2.3 There is one VFP co-processor register class using registers
+    // s0-s15 (d0-d7) for passing arguments.
+    const unsigned NumVFPs = 16;
+    it->info = classifyArgumentType(it->type, VFPRegs, AllocatedVFP, IsHA);
+    // If we do not have enough VFP registers for the HA, any VFP registers
+    // that are unallocated are marked as unavailable. To achieve this, we add
+    // padding of (NumVFPs - PreAllocation) floats.
+    if (IsHA && AllocatedVFP > NumVFPs && PreAllocation < NumVFPs) {
+      llvm::Type *PaddingTy = llvm::ArrayType::get(
+          llvm::Type::getFloatTy(getVMContext()), NumVFPs - PreAllocation);
+      it->info = ABIArgInfo::getExpandWithPadding(false, PaddingTy);
+    }
+  }
 
   // Always honor user-specified calling convention.
   if (FI.getCallingConvention() != llvm::CallingConv::C)
@@ -2637,7 +3001,9 @@ void ARMABIInfo::computeInfo(CGFunctionInfo &FI) const {
 
   // Calling convention as default by an ABI.
   llvm::CallingConv::ID DefaultCC;
-  if (isEABI())
+  if (getContext().getTargetInfo().getTriple().getEnvironmentName()=="gnueabihf")
+    DefaultCC = llvm::CallingConv::ARM_AAPCS_VFP;
+  else if (isEABI())
     DefaultCC = llvm::CallingConv::ARM_AAPCS;
   else
     DefaultCC = llvm::CallingConv::ARM_APCS;
@@ -2729,7 +3095,88 @@ static bool isHomogeneousAggregate(QualType Ty, const Type *&Base,
   return (Members > 0 && Members <= 4);
 }
 
-ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty) const {
+/// markAllocatedVFPs - update VFPRegs according to the alignment and
+/// number of VFP registers (unit is S register) requested.
+static void markAllocatedVFPs(int *VFPRegs, unsigned &AllocatedVFP,
+                              unsigned Alignment,
+                              unsigned NumRequired) {
+  // Early Exit.
+  if (AllocatedVFP >= 16)
+    return;
+  // C.1.vfp If the argument is a VFP CPRC and there are sufficient consecutive
+  // VFP registers of the appropriate type unallocated then the argument is
+  // allocated to the lowest-numbered sequence of such registers.
+  for (unsigned I = 0; I < 16; I += Alignment) {
+    bool FoundSlot = true;
+    for (unsigned J = I, JEnd = I + NumRequired; J < JEnd; J++)
+      if (J >= 16 || VFPRegs[J]) {
+         FoundSlot = false;
+         break;
+      }
+    if (FoundSlot) {
+      for (unsigned J = I, JEnd = I + NumRequired; J < JEnd; J++)
+        VFPRegs[J] = 1;
+      AllocatedVFP += NumRequired;
+      return;
+    }
+  }
+  // C.2.vfp If the argument is a VFP CPRC then any VFP registers that are
+  // unallocated are marked as unavailable.
+  for (unsigned I = 0; I < 16; I++)
+    VFPRegs[I] = 1;
+  AllocatedVFP = 17; // We do not have enough VFP registers.
+}
+
+ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty, int *VFPRegs,
+                                            unsigned &AllocatedVFP,
+                                            bool &IsHA) const {
+  // We update number of allocated VFPs according to
+  // 6.1.2.1 The following argument types are VFP CPRCs:
+  //   A single-precision floating-point type (including promoted
+  //   half-precision types); A double-precision floating-point type;
+  //   A 64-bit or 128-bit containerized vector type; Homogeneous Aggregate
+  //   with a Base Type of a single- or double-precision floating-point type,
+  //   64-bit containerized vectors or 128-bit containerized vectors with one
+  //   to four Elements.
+
+  // Handle illegal vector types here.
+  if (isIllegalVectorType(Ty)) {
+    uint64_t Size = getContext().getTypeSize(Ty);
+    if (Size <= 32) {
+      llvm::Type *ResType =
+          llvm::Type::getInt32Ty(getVMContext());
+      return ABIArgInfo::getDirect(ResType);
+    }
+    if (Size == 64) {
+      llvm::Type *ResType = llvm::VectorType::get(
+          llvm::Type::getInt32Ty(getVMContext()), 2);
+      markAllocatedVFPs(VFPRegs, AllocatedVFP, 2, 2);
+      return ABIArgInfo::getDirect(ResType);
+    }
+    if (Size == 128) {
+      llvm::Type *ResType = llvm::VectorType::get(
+          llvm::Type::getInt32Ty(getVMContext()), 4);
+      markAllocatedVFPs(VFPRegs, AllocatedVFP, 4, 4);
+      return ABIArgInfo::getDirect(ResType);
+    }
+    return ABIArgInfo::getIndirect(0, /*ByVal=*/false);
+  }
+  // Update VFPRegs for legal vector types.
+  if (const VectorType *VT = Ty->getAs<VectorType>()) {
+    uint64_t Size = getContext().getTypeSize(VT);
+    // Size of a legal vector should be power of 2 and above 64.
+    markAllocatedVFPs(VFPRegs, AllocatedVFP, Size >= 128 ? 4 : 2, Size / 32);
+  }
+  // Update VFPRegs for floating point types.
+  if (const BuiltinType *BT = Ty->getAs<BuiltinType>()) {
+    if (BT->getKind() == BuiltinType::Half ||
+        BT->getKind() == BuiltinType::Float)
+      markAllocatedVFPs(VFPRegs, AllocatedVFP, 1, 1);
+    if (BT->getKind() == BuiltinType::Double ||
+        BT->getKind() == BuiltinType::LongDouble)
+      markAllocatedVFPs(VFPRegs, AllocatedVFP, 2, 2);
+  }
+
   if (!isAggregateTypeForABI(Ty)) {
     // Treat an enum type as its underlying type.
     if (const EnumType *EnumTy = Ty->getAs<EnumType>())
@@ -2749,18 +3196,42 @@ ABIArgInfo ARMABIInfo::classifyArgumentType(QualType Ty) const {
     return ABIArgInfo::getIndirect(0, /*ByVal=*/false);
 
   if (getABIKind() == ARMABIInfo::AAPCS_VFP) {
-    // Homogeneous Aggregates need to be expanded.
+    // Homogeneous Aggregates need to be expanded when we can fit the aggregate
+    // into VFP registers.
     const Type *Base = 0;
-    if (isHomogeneousAggregate(Ty, Base, getContext())) {
+    uint64_t Members = 0;
+    if (isHomogeneousAggregate(Ty, Base, getContext(), &Members)) {
       assert(Base && "Base class should be set for homogeneous aggregate");
+      // Base can be a floating-point or a vector.
+      if (Base->isVectorType()) {
+        // ElementSize is in number of floats.
+        unsigned ElementSize = getContext().getTypeSize(Base) == 64 ? 2 : 4;
+        markAllocatedVFPs(VFPRegs, AllocatedVFP, ElementSize,
+                          Members * ElementSize);
+      } else if (Base->isSpecificBuiltinType(BuiltinType::Float))
+        markAllocatedVFPs(VFPRegs, AllocatedVFP, 1, Members);
+      else {
+        assert(Base->isSpecificBuiltinType(BuiltinType::Double) ||
+               Base->isSpecificBuiltinType(BuiltinType::LongDouble));
+        markAllocatedVFPs(VFPRegs, AllocatedVFP, 2, Members * 2);
+      }
+      IsHA = true;
       return ABIArgInfo::getExpand();
     }
   }
 
   // Support byval for ARM.
-  if (getContext().getTypeSizeInChars(Ty) > CharUnits::fromQuantity(64) ||
-      getContext().getTypeAlign(Ty) > 64) {
-    return ABIArgInfo::getIndirect(0, /*ByVal=*/true);
+  // The ABI alignment for APCS is 4-byte and for AAPCS at least 4-byte and at
+  // most 8-byte. We realign the indirect argument if type alignment is bigger
+  // than ABI alignment.
+  uint64_t ABIAlign = 4;
+  uint64_t TyAlign = getContext().getTypeAlign(Ty) / 8;
+  if (getABIKind() == ARMABIInfo::AAPCS_VFP ||
+      getABIKind() == ARMABIInfo::AAPCS)
+    ABIAlign = std::min(std::max(TyAlign, (uint64_t)4), (uint64_t)8);
+  if (getContext().getTypeSizeInChars(Ty) > CharUnits::fromQuantity(64)) {
+    return ABIArgInfo::getIndirect(0, /*ByVal=*/true,
+           /*Realign=*/TyAlign > ABIAlign);
   }
 
   // Otherwise, pass by coercing to a structure of the appropriate size.
@@ -2946,6 +3417,21 @@ ABIArgInfo ARMABIInfo::classifyReturnType(QualType RetTy) const {
   return ABIArgInfo::getIndirect(0);
 }
 
+/// isIllegalVector - check whether Ty is an illegal vector type.
+bool ARMABIInfo::isIllegalVectorType(QualType Ty) const {
+  if (const VectorType *VT = Ty->getAs<VectorType>()) {
+    // Check whether VT is legal.
+    unsigned NumElements = VT->getNumElements();
+    uint64_t Size = getContext().getTypeSize(VT);
+    // NumElements should be power of 2.
+    if ((NumElements & (NumElements - 1)) != 0)
+      return true;
+    // Size should be greater than 32 bits.
+    return Size <= 32;
+  }
+  return false;
+}
+
 llvm::Value *ARMABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
                                    CodeGenFunction &CGF) const {
   llvm::Type *BP = CGF.Int8PtrTy;
@@ -2954,30 +3440,104 @@ llvm::Value *ARMABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
   CGBuilderTy &Builder = CGF.Builder;
   llvm::Value *VAListAddrAsBPP = Builder.CreateBitCast(VAListAddr, BPP, "ap");
   llvm::Value *Addr = Builder.CreateLoad(VAListAddrAsBPP, "ap.cur");
-  // Handle address alignment for type alignment > 32 bits
+
+  uint64_t Size = CGF.getContext().getTypeSize(Ty) / 8;
   uint64_t TyAlign = CGF.getContext().getTypeAlign(Ty) / 8;
+  bool IsIndirect = false;
+
+  // The ABI alignment for 64-bit or 128-bit vectors is 8 for AAPCS and 4 for
+  // APCS. For AAPCS, the ABI alignment is at least 4-byte and at most 8-byte.
+  if (getABIKind() == ARMABIInfo::AAPCS_VFP ||
+      getABIKind() == ARMABIInfo::AAPCS)
+    TyAlign = std::min(std::max(TyAlign, (uint64_t)4), (uint64_t)8);
+  else
+    TyAlign = 4;
+  // Use indirect if size of the illegal vector is bigger than 16 bytes.
+  if (isIllegalVectorType(Ty) && Size > 16) {
+    IsIndirect = true;
+    Size = 4;
+    TyAlign = 4;
+  }
+
+  // Handle address alignment for ABI alignment > 4 bytes.
   if (TyAlign > 4) {
     assert((TyAlign & (TyAlign - 1)) == 0 &&
            "Alignment is not power of 2!");
     llvm::Value *AddrAsInt = Builder.CreatePtrToInt(Addr, CGF.Int32Ty);
     AddrAsInt = Builder.CreateAdd(AddrAsInt, Builder.getInt32(TyAlign - 1));
     AddrAsInt = Builder.CreateAnd(AddrAsInt, Builder.getInt32(~(TyAlign - 1)));
-    Addr = Builder.CreateIntToPtr(AddrAsInt, BP);
+    Addr = Builder.CreateIntToPtr(AddrAsInt, BP, "ap.align");
   }
-  llvm::Type *PTy =
-    llvm::PointerType::getUnqual(CGF.ConvertType(Ty));
-  llvm::Value *AddrTyped = Builder.CreateBitCast(Addr, PTy);
 
   uint64_t Offset =
-    llvm::RoundUpToAlignment(CGF.getContext().getTypeSize(Ty) / 8, 4);
+    llvm::RoundUpToAlignment(Size, 4);
   llvm::Value *NextAddr =
     Builder.CreateGEP(Addr, llvm::ConstantInt::get(CGF.Int32Ty, Offset),
                       "ap.next");
   Builder.CreateStore(NextAddr, VAListAddrAsBPP);
 
+  if (IsIndirect)
+    Addr = Builder.CreateLoad(Builder.CreateBitCast(Addr, BPP));
+  else if (TyAlign < CGF.getContext().getTypeAlign(Ty) / 8) {
+    // We can't directly cast ap.cur to pointer to a vector type, since ap.cur
+    // may not be correctly aligned for the vector type. We create an aligned
+    // temporary space and copy the content over from ap.cur to the temporary
+    // space. This is necessary if the natural alignment of the type is greater
+    // than the ABI alignment.
+    llvm::Type *I8PtrTy = Builder.getInt8PtrTy();
+    CharUnits CharSize = getContext().getTypeSizeInChars(Ty);
+    llvm::Value *AlignedTemp = CGF.CreateTempAlloca(CGF.ConvertType(Ty),
+                                                    "var.align");
+    llvm::Value *Dst = Builder.CreateBitCast(AlignedTemp, I8PtrTy);
+    llvm::Value *Src = Builder.CreateBitCast(Addr, I8PtrTy);
+    Builder.CreateMemCpy(Dst, Src,
+        llvm::ConstantInt::get(CGF.IntPtrTy, CharSize.getQuantity()),
+        TyAlign, false);
+    Addr = AlignedTemp; //The content is in aligned location.
+  }
+  llvm::Type *PTy =
+    llvm::PointerType::getUnqual(CGF.ConvertType(Ty));
+  llvm::Value *AddrTyped = Builder.CreateBitCast(Addr, PTy);
+
   return AddrTyped;
 }
 
+namespace {
+
+class NaClARMABIInfo : public ABIInfo {
+ public:
+  NaClARMABIInfo(CodeGen::CodeGenTypes &CGT, ARMABIInfo::ABIKind Kind)
+      : ABIInfo(CGT), PInfo(CGT), NInfo(CGT, Kind) {}
+  virtual void computeInfo(CGFunctionInfo &FI) const;
+  virtual llvm::Value *EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
+                                 CodeGenFunction &CGF) const;
+ private:
+  PNaClABIInfo PInfo; // Used for generating calls with pnaclcall callingconv.
+  ARMABIInfo NInfo; // Used for everything else.
+};
+
+class NaClARMTargetCodeGenInfo : public TargetCodeGenInfo  {
+ public:
+  NaClARMTargetCodeGenInfo(CodeGen::CodeGenTypes &CGT, ARMABIInfo::ABIKind Kind)
+      : TargetCodeGenInfo(new NaClARMABIInfo(CGT, Kind)) {}
+};
+
+}
+
+void NaClARMABIInfo::computeInfo(CGFunctionInfo &FI) const {
+  if (FI.getASTCallingConvention() == CC_PnaclCall)
+    PInfo.computeInfo(FI);
+  else
+    static_cast<const ABIInfo&>(NInfo).computeInfo(FI);
+}
+
+llvm::Value *NaClARMABIInfo::EmitVAArg(llvm::Value *VAListAddr, QualType Ty,
+                                       CodeGenFunction &CGF) const {
+  // Always use the native convention; calling pnacl-style varargs functions
+  // is unsupported.
+  return static_cast<const ABIInfo&>(NInfo).EmitVAArg(VAListAddr, Ty, CGF);
+}
+
 //===----------------------------------------------------------------------===//
 // NVPTX ABI Implementation
 //===----------------------------------------------------------------------===//
@@ -3072,7 +3632,7 @@ SetTargetAttributes(const Decl *D, llvm::GlobalValue *GV,
       // OpenCL __kernel functions get a kernel calling convention
       F->setCallingConv(llvm::CallingConv::PTX_Kernel);
       // And kernel functions are not subject to inlining
-      F->addFnAttr(llvm::Attribute::NoInline);
+      F->addFnAttr(llvm::Attributes::NoInline);
     }
   }
 
@@ -3188,7 +3748,7 @@ void MBlazeTargetCodeGenInfo::SetTargetAttributes(const Decl *D,
       F->setCallingConv(CC);
 
       // Step 2: Add attributes goodness.
-      F->addFnAttr(llvm::Attribute::NoInline);
+      F->addFnAttr(llvm::Attributes::NoInline);
   }
 
   // Step 3: Emit _interrupt_handler alias.
@@ -3226,7 +3786,7 @@ void MSP430TargetCodeGenInfo::SetTargetAttributes(const Decl *D,
       F->setCallingConv(llvm::CallingConv::MSP430_INTR);
 
       // Step 2: Add attributes goodness.
-      F->addFnAttr(llvm::Attribute::NoInline);
+      F->addFnAttr(llvm::Attributes::NoInline);
 
       // Step 3: Emit ISR vector alias.
       unsigned Num = attr->getNumber() + 0xffe0;
@@ -3583,7 +4143,7 @@ void TCETargetCodeGenInfo::SetTargetAttributes(const Decl *D,
   if (M.getLangOpts().OpenCL) {
     if (FD->hasAttr<OpenCLKernelAttr>()) {
       // OpenCL C Kernel functions are not subject to inlining
-      F->addFnAttr(llvm::Attribute::NoInline);
+      F->addFnAttr(llvm::Attributes::NoInline);
           
       if (FD->hasAttr<ReqdWorkGroupSizeAttr>()) {
 
@@ -3767,6 +4327,8 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() {
   default:
     return *(TheTargetCodeGenInfo = new DefaultTargetCodeGenInfo(Types));
 
+  case llvm::Triple::le32:
+    return *(TheTargetCodeGenInfo = new PNaClTargetCodeGenInfo(Types));
   case llvm::Triple::mips:
   case llvm::Triple::mipsel:
     return *(TheTargetCodeGenInfo = new MIPSTargetCodeGenInfo(Types, true));
@@ -3779,19 +4341,29 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() {
   case llvm::Triple::thumb:
     {
       ARMABIInfo::ABIKind Kind = ARMABIInfo::AAPCS;
-
       if (strcmp(getContext().getTargetInfo().getABI(), "apcs-gnu") == 0)
         Kind = ARMABIInfo::APCS;
-      else if (CodeGenOpts.FloatABI == "hard")
+      else if (CodeGenOpts.FloatABI == "hard" ||
+               (CodeGenOpts.FloatABI != "soft" && Triple.getEnvironment()==llvm::Triple::GNUEABIHF))
         Kind = ARMABIInfo::AAPCS_VFP;
 
-      return *(TheTargetCodeGenInfo = new ARMTargetCodeGenInfo(Types, Kind));
+      switch (Triple.getOS()) {
+        case llvm::Triple::NativeClient:
+          return *(TheTargetCodeGenInfo =
+                   new NaClARMTargetCodeGenInfo(Types, Kind));
+        default:
+          return *(TheTargetCodeGenInfo =
+                   new ARMTargetCodeGenInfo(Types, Kind));
+      }
     }
 
   case llvm::Triple::ppc:
     return *(TheTargetCodeGenInfo = new PPC32TargetCodeGenInfo(Types));
   case llvm::Triple::ppc64:
-    return *(TheTargetCodeGenInfo = new PPC64TargetCodeGenInfo(Types));
+    if (Triple.isOSBinFormatELF())
+      return *(TheTargetCodeGenInfo = new PPC64_SVR4_TargetCodeGenInfo(Types));
+    else
+      return *(TheTargetCodeGenInfo = new PPC64TargetCodeGenInfo(Types));
 
   case llvm::Triple::nvptx:
   case llvm::Triple::nvptx64:
@@ -3848,6 +4420,8 @@ const TargetCodeGenInfo &CodeGenModule::getTargetCodeGenInfo() {
     case llvm::Triple::MinGW32:
     case llvm::Triple::Cygwin:
       return *(TheTargetCodeGenInfo = new WinX86_64TargetCodeGenInfo(Types));
+    case llvm::Triple::NativeClient:
+      return *(TheTargetCodeGenInfo = new NaClX86_64TargetCodeGenInfo(Types, HasAVX));
     default:
       return *(TheTargetCodeGenInfo = new X86_64TargetCodeGenInfo(Types,
                                                                   HasAVX));
diff --git a/lib/Driver/Arg.cpp b/lib/Driver/Arg.cpp
index c0a2a50..93d70a9 100644
--- a/lib/Driver/Arg.cpp
+++ b/lib/Driver/Arg.cpp
@@ -8,6 +8,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Driver/Arg.h"
+#include "clang/Basic/LLVM.h"
 #include "clang/Driver/ArgList.h"
 #include "clang/Driver/Option.h"
 #include "llvm/ADT/SmallString.h"
@@ -15,22 +16,23 @@
 #include "llvm/Support/raw_ostream.h"
 
 using namespace clang::driver;
+using clang::StringRef;
 
-Arg::Arg(const Option *_Opt, unsigned _Index, const Arg *_BaseArg)
-  : Opt(_Opt), BaseArg(_BaseArg), Index(_Index),
+Arg::Arg(const Option _Opt, StringRef S, unsigned _Index, const Arg *_BaseArg)
+  : Opt(_Opt), BaseArg(_BaseArg), Spelling(S), Index(_Index),
     Claimed(false), OwnsValues(false) {
 }
 
-Arg::Arg(const Option *_Opt, unsigned _Index, 
+Arg::Arg(const Option _Opt, StringRef S, unsigned _Index,
          const char *Value0, const Arg *_BaseArg)
-  : Opt(_Opt), BaseArg(_BaseArg), Index(_Index),
+  : Opt(_Opt), BaseArg(_BaseArg), Spelling(S), Index(_Index),
     Claimed(false), OwnsValues(false) {
   Values.push_back(Value0);
 }
 
-Arg::Arg(const Option *_Opt, unsigned _Index, 
+Arg::Arg(const Option _Opt, StringRef S, unsigned _Index,
          const char *Value0, const char *Value1, const Arg *_BaseArg)
-  : Opt(_Opt), BaseArg(_BaseArg), Index(_Index),
+  : Opt(_Opt), BaseArg(_BaseArg), Spelling(S), Index(_Index),
     Claimed(false), OwnsValues(false) {
   Values.push_back(Value0);
   Values.push_back(Value1);
@@ -47,7 +49,7 @@ void Arg::dump() const {
   llvm::errs() << "<";
 
   llvm::errs() << " Opt:";
-  Opt->dump();
+  Opt.dump();
 
   llvm::errs() << " Index:" << Index;
 
@@ -83,39 +85,39 @@ void Arg::renderAsInput(const ArgList &Args, ArgStringList &Output) const {
   }
 
   for (unsigned i = 0, e = getNumValues(); i != e; ++i)
-    Output.push_back(getValue(Args, i));
+    Output.push_back(getValue(i));
 }
 
 void Arg::render(const ArgList &Args, ArgStringList &Output) const {
   switch (getOption().getRenderStyle()) {
   case Option::RenderValuesStyle:
     for (unsigned i = 0, e = getNumValues(); i != e; ++i)
-      Output.push_back(getValue(Args, i));
+      Output.push_back(getValue(i));
     break;
 
   case Option::RenderCommaJoinedStyle: {
     SmallString<256> Res;
     llvm::raw_svector_ostream OS(Res);
-    OS << getOption().getName();
+    OS << getSpelling();
     for (unsigned i = 0, e = getNumValues(); i != e; ++i) {
       if (i) OS << ',';
-      OS << getValue(Args, i);
+      OS << getValue(i);
     }
     Output.push_back(Args.MakeArgString(OS.str()));
     break;
   }
- 
+
  case Option::RenderJoinedStyle:
     Output.push_back(Args.GetOrMakeJoinedArgString(
-                       getIndex(), getOption().getName(), getValue(Args, 0)));
+                       getIndex(), getSpelling(), getValue(0)));
     for (unsigned i = 1, e = getNumValues(); i != e; ++i)
-      Output.push_back(getValue(Args, i));
+      Output.push_back(getValue(i));
     break;
 
   case Option::RenderSeparateStyle:
-    Output.push_back(getOption().getName().data());
+    Output.push_back(Args.MakeArgString(getSpelling()));
     for (unsigned i = 0, e = getNumValues(); i != e; ++i)
-      Output.push_back(getValue(Args, i));
+      Output.push_back(getValue(i));
     break;
   }
 }
diff --git a/lib/Driver/ArgList.cpp b/lib/Driver/ArgList.cpp
index 7fd439e..b3a43df 100644
--- a/lib/Driver/ArgList.cpp
+++ b/lib/Driver/ArgList.cpp
@@ -211,7 +211,7 @@ bool ArgList::hasFlag(OptSpecifier Pos, OptSpecifier Neg, bool Default) const {
 StringRef ArgList::getLastArgValue(OptSpecifier Id,
                                          StringRef Default) const {
   if (Arg *A = getLastArg(Id))
-    return A->getValue(*this);
+    return A->getValue();
   return Default;
 }
 
@@ -220,10 +220,10 @@ int ArgList::getLastArgIntValue(OptSpecifier Id, int Default,
   int Res = Default;
 
   if (Arg *A = getLastArg(Id)) {
-    if (StringRef(A->getValue(*this)).getAsInteger(10, Res)) {
+    if (StringRef(A->getValue()).getAsInteger(10, Res)) {
       if (Diags)
         Diags->Report(diag::err_drv_invalid_int_value)
-          << A->getAsString(*this) << A->getValue(*this);
+          << A->getAsString(*this) << A->getValue();
     }
   }
 
@@ -258,7 +258,7 @@ void ArgList::AddAllArgValues(ArgStringList &Output, OptSpecifier Id0,
          ie = filtered_end(); it != ie; ++it) {
     (*it)->claim();
     for (unsigned i = 0, e = (*it)->getNumValues(); i != e; ++i)
-      Output.push_back((*it)->getValue(*this, i));
+      Output.push_back((*it)->getValue(i));
   }
 }
 
@@ -271,10 +271,10 @@ void ArgList::AddAllArgsTranslated(ArgStringList &Output, OptSpecifier Id0,
 
     if (Joined) {
       Output.push_back(MakeArgString(StringRef(Translation) +
-                                     (*it)->getValue(*this, 0)));
+                                     (*it)->getValue(0)));
     } else {
       Output.push_back(Translation);
-      Output.push_back((*it)->getValue(*this, 0));
+      Output.push_back((*it)->getValue(0));
     }
   }
 }
@@ -362,33 +362,40 @@ const char *DerivedArgList::MakeArgString(StringRef Str) const {
   return BaseArgs.MakeArgString(Str);
 }
 
-Arg *DerivedArgList::MakeFlagArg(const Arg *BaseArg, const Option *Opt) const {
-  Arg *A = new Arg(Opt, BaseArgs.MakeIndex(Opt->getName()), BaseArg);
+Arg *DerivedArgList::MakeFlagArg(const Arg *BaseArg, const Option Opt) const {
+  Arg *A = new Arg(Opt, ArgList::MakeArgString(Twine(Opt.getPrefix()) +
+                                               Twine(Opt.getName())),
+                   BaseArgs.MakeIndex(Opt.getName()), BaseArg);
   SynthesizedArgs.push_back(A);
   return A;
 }
 
-Arg *DerivedArgList::MakePositionalArg(const Arg *BaseArg, const Option *Opt,
+Arg *DerivedArgList::MakePositionalArg(const Arg *BaseArg, const Option Opt,
                                        StringRef Value) const {
   unsigned Index = BaseArgs.MakeIndex(Value);
-  Arg *A = new Arg(Opt, Index, BaseArgs.getArgString(Index), BaseArg);
+  Arg *A = new Arg(Opt, ArgList::MakeArgString(Twine(Opt.getPrefix()) +
+                                               Twine(Opt.getName())),
+                   Index, BaseArgs.getArgString(Index), BaseArg);
   SynthesizedArgs.push_back(A);
   return A;
 }
 
-Arg *DerivedArgList::MakeSeparateArg(const Arg *BaseArg, const Option *Opt,
+Arg *DerivedArgList::MakeSeparateArg(const Arg *BaseArg, const Option Opt,
                                      StringRef Value) const {
-  unsigned Index = BaseArgs.MakeIndex(Opt->getName(), Value);
-  Arg *A = new Arg(Opt, Index, BaseArgs.getArgString(Index + 1), BaseArg);
+  unsigned Index = BaseArgs.MakeIndex(Opt.getName(), Value);
+  Arg *A = new Arg(Opt, ArgList::MakeArgString(Twine(Opt.getPrefix()) +
+                                               Twine(Opt.getName())),
+                   Index, BaseArgs.getArgString(Index + 1), BaseArg);
   SynthesizedArgs.push_back(A);
   return A;
 }
 
-Arg *DerivedArgList::MakeJoinedArg(const Arg *BaseArg, const Option *Opt,
+Arg *DerivedArgList::MakeJoinedArg(const Arg *BaseArg, const Option Opt,
                                    StringRef Value) const {
-  unsigned Index = BaseArgs.MakeIndex(Opt->getName().str() + Value.str());
-  Arg *A = new Arg(Opt, Index,
-                   BaseArgs.getArgString(Index) + Opt->getName().size(),
+  unsigned Index = BaseArgs.MakeIndex(Opt.getName().str() + Value.str());
+  Arg *A = new Arg(Opt, ArgList::MakeArgString(Twine(Opt.getPrefix()) +
+                                               Twine(Opt.getName())), Index,
+                   BaseArgs.getArgString(Index) + Opt.getName().size(),
                    BaseArg);
   SynthesizedArgs.push_back(A);
   return A;
diff --git a/lib/Driver/CC1AsOptions.cpp b/lib/Driver/CC1AsOptions.cpp
index ea80f5a..4f89b73 100644
--- a/lib/Driver/CC1AsOptions.cpp
+++ b/lib/Driver/CC1AsOptions.cpp
@@ -15,11 +15,19 @@ using namespace clang::driver;
 using namespace clang::driver::options;
 using namespace clang::driver::cc1asoptions;
 
+#define PREFIX(NAME, VALUE) const char *const NAME[] = VALUE;
+#define OPTION(PREFIX, NAME, ID, KIND, GROUP, ALIAS, FLAGS, PARAM, \
+               HELPTEXT, METAVAR)
+#include "clang/Driver/CC1AsOptions.inc"
+#undef OPTION
+#undef PREFIX
+
 static const OptTable::Info CC1AsInfoTable[] = {
-#define OPTION(NAME, ID, KIND, GROUP, ALIAS, FLAGS, PARAM, \
+#define PREFIX(NAME, VALUE)
+#define OPTION(PREFIX, NAME, ID, KIND, GROUP, ALIAS, FLAGS, PARAM, \
                HELPTEXT, METAVAR)   \
-  { NAME, HELPTEXT, METAVAR, Option::KIND##Class, PARAM, FLAGS, \
-    OPT_##GROUP, OPT_##ALIAS },
+  { PREFIX, NAME, HELPTEXT, METAVAR, OPT_##ID, Option::KIND##Class, PARAM, \
+    FLAGS, OPT_##GROUP, OPT_##ALIAS },
 #include "clang/Driver/CC1AsOptions.inc"
 };
 
diff --git a/lib/Driver/Compilation.cpp b/lib/Driver/Compilation.cpp
index c962fca..124e50c 100644
--- a/lib/Driver/Compilation.cpp
+++ b/lib/Driver/Compilation.cpp
@@ -17,6 +17,7 @@
 #include "clang/Driver/ToolChain.h"
 
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/StringSwitch.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/Program.h"
 #include <sys/stat.h>
@@ -101,6 +102,105 @@ void Compilation::PrintJob(raw_ostream &OS, const Job &J,
   }
 }
 
+static bool skipArg(const char *Flag, bool &SkipNextArg) {
+  StringRef FlagRef(Flag);
+
+  // Assume we're going to see -Flag <Arg>.
+  SkipNextArg = true;
+
+  // These flags are all of the form -Flag <Arg> and are treated as two
+  // arguments.  Therefore, we need to skip the flag and the next argument.
+  bool Res = llvm::StringSwitch<bool>(Flag)
+    .Cases("-I", "-MF", "-MT", "-MQ", true)
+    .Cases("-o", "-coverage-file", "-dependency-file", true)
+    .Cases("-fdebug-compilation-dir", "-fmodule-cache-path", "-idirafter", true)
+    .Cases("-include", "-include-pch", "-internal-isystem", true)
+    .Cases("-internal-externc-isystem", "-iprefix", "-iwithprefix", true)
+    .Cases("-iwithprefixbefore", "-isysroot", "-isystem", "-iquote", true)
+    .Cases("-resource-dir", "-serialize-diagnostic-file", true)
+    .Case("-dwarf-debug-flags", true)
+    .Default(false);
+
+  // Match found.
+  if (Res)
+    return Res;
+
+  // The remaining flags are treated as a single argument.
+  SkipNextArg = false;
+
+  // These flags are all of the form -Flag and have no second argument.
+  Res = llvm::StringSwitch<bool>(Flag)
+    .Cases("-M", "-MM", "-MG", "-MP", "-MD", true)
+    .Case("-MMD", true)
+    .Default(false);
+
+  // Match found.
+  if (Res)
+    return Res;
+
+  // These flags are treated as a single argument (e.g., -F<Dir>).
+  if (FlagRef.startswith("-F") || FlagRef.startswith("-I"))
+    return true;
+
+  return false;
+}
+
+static bool quoteNextArg(const char *flag) {
+  return llvm::StringSwitch<bool>(flag)
+    .Case("-D", true)
+    .Default(false);
+}
+
+void Compilation::PrintDiagnosticJob(raw_ostream &OS, const Job &J) const {
+  if (const Command *C = dyn_cast<Command>(&J)) {
+    OS << C->getExecutable();
+    unsigned QuoteNextArg = 0;
+    for (ArgStringList::const_iterator it = C->getArguments().begin(),
+           ie = C->getArguments().end(); it != ie; ++it) {
+
+      bool SkipNext;
+      if (skipArg(*it, SkipNext)) {
+        if (SkipNext) ++it;
+        continue;
+      }
+
+      if (!QuoteNextArg)
+        QuoteNextArg = quoteNextArg(*it) ? 2 : 0;
+
+      OS << ' ';
+
+      if (QuoteNextArg == 1)
+        OS << '"';
+
+      if (!std::strpbrk(*it, " \"\\$")) {
+        OS << *it;
+      } else {
+        // Quote the argument and escape shell special characters; this isn't
+        // really complete but is good enough.
+        OS << '"';
+        for (const char *s = *it; *s; ++s) {
+          if (*s == '"' || *s == '\\' || *s == '$')
+            OS << '\\';
+          OS << *s;
+        }
+        OS << '"';
+      }
+
+      if (QuoteNextArg) {
+        if (QuoteNextArg == 1)
+          OS << '"';
+        --QuoteNextArg;
+      }
+    }
+    OS << '\n';
+  } else {
+    const JobList *Jobs = cast<JobList>(&J);
+    for (JobList::const_iterator
+           it = Jobs->begin(), ie = Jobs->end(); it != ie; ++it)
+      PrintDiagnosticJob(OS, **it);
+  }
+}
+
 bool Compilation::CleanupFileList(const ArgStringList &Files,
                                   bool IssueErrors) const {
   bool Success = true;
diff --git a/lib/Driver/Driver.cpp b/lib/Driver/Driver.cpp
index 87d533d..68471ec 100644
--- a/lib/Driver/Driver.cpp
+++ b/lib/Driver/Driver.cpp
@@ -52,25 +52,13 @@ Driver::Driver(StringRef ClangExecutable,
     ClangExecutable(ClangExecutable), SysRoot(DEFAULT_SYSROOT),
     UseStdLib(true), DefaultTargetTriple(DefaultTargetTriple),
     DefaultImageName(DefaultImageName),
-    DriverTitle("clang \"gcc-compatible\" driver"),
+    DriverTitle("clang LLVM compiler"),
     CCPrintOptionsFilename(0), CCPrintHeadersFilename(0),
     CCLogDiagnosticsFilename(0), CCCIsCXX(false),
     CCCIsCPP(false),CCCEcho(false), CCCPrintBindings(false),
     CCPrintOptions(false), CCPrintHeaders(false), CCLogDiagnostics(false),
     CCGenDiagnostics(false), CCCGenericGCCName(""), CheckInputsExist(true),
-    CCCUseClang(true), CCCUseClangCXX(true), CCCUseClangCPP(true),
-    ForcedClangUse(false), CCCUsePCH(true), SuppressMissingInputWarning(false) {
-  if (IsProduction) {
-    // In a "production" build, only use clang on architectures we expect to
-    // work.
-    //
-    // During development its more convenient to always have the driver use
-    // clang, but we don't want users to be confused when things don't work, or
-    // to file bugs for things we don't support.
-    CCCClangArchs.insert(llvm::Triple::x86);
-    CCCClangArchs.insert(llvm::Triple::x86_64);
-    CCCClangArchs.insert(llvm::Triple::arm);
-  }
+    CCCUsePCH(true), SuppressMissingInputWarning(false) {
 
   Name = llvm::sys::path::stem(ClangExecutable);
   Dir  = llvm::sys::path::parent_path(ClangExecutable);
@@ -109,7 +97,7 @@ InputArgList *Driver::ParseArgStrings(ArrayRef<const char *> ArgList) {
   for (ArgList::const_iterator it = Args->begin(), ie = Args->end();
        it != ie; ++it) {
     Arg *A = *it;
-    if (A->getOption().isUnsupported()) {
+    if (A->getOption().hasFlag(options::Unsupported)) {
       Diag(clang::diag::err_drv_unsupported_opt) << A->getAsString(*Args);
       continue;
     }
@@ -186,9 +174,9 @@ DerivedArgList *Driver::TranslateInputArgs(const InputArgList &Args) const {
 
       // Add the remaining values as Xlinker arguments.
       for (unsigned i = 0, e = A->getNumValues(); i != e; ++i)
-        if (StringRef(A->getValue(Args, i)) != "--no-demangle")
+        if (StringRef(A->getValue(i)) != "--no-demangle")
           DAL->AddSeparateArg(A, Opts->getOption(options::OPT_Xlinker),
-                              A->getValue(Args, i));
+                              A->getValue(i));
 
       continue;
     }
@@ -197,22 +185,22 @@ DerivedArgList *Driver::TranslateInputArgs(const InputArgList &Args) const {
     // some build systems. We don't try to be complete here because we don't
     // care to encourage this usage model.
     if (A->getOption().matches(options::OPT_Wp_COMMA) &&
-        A->getNumValues() == 2 &&
-        (A->getValue(Args, 0) == StringRef("-MD") ||
-         A->getValue(Args, 0) == StringRef("-MMD"))) {
+        (A->getValue(0) == StringRef("-MD") ||
+         A->getValue(0) == StringRef("-MMD"))) {
       // Rewrite to -MD/-MMD along with -MF.
-      if (A->getValue(Args, 0) == StringRef("-MD"))
+      if (A->getValue(0) == StringRef("-MD"))
         DAL->AddFlagArg(A, Opts->getOption(options::OPT_MD));
       else
         DAL->AddFlagArg(A, Opts->getOption(options::OPT_MMD));
-      DAL->AddSeparateArg(A, Opts->getOption(options::OPT_MF),
-                          A->getValue(Args, 1));
+      if (A->getNumValues() == 2)
+        DAL->AddSeparateArg(A, Opts->getOption(options::OPT_MF),
+                            A->getValue(1));
       continue;
     }
 
     // Rewrite reserved library names.
     if (A->getOption().matches(options::OPT_l)) {
-      StringRef Value = A->getValue(Args);
+      StringRef Value = A->getValue();
 
       // Rewrite unless -nostdlib is present.
       if (!HasNostdlib && Value == "stdc++") {
@@ -285,48 +273,23 @@ Compilation *Driver::BuildCompilation(ArrayRef<const char *> ArgList) {
   CCCIsCXX = Args->hasArg(options::OPT_ccc_cxx) || CCCIsCXX;
   CCCEcho = Args->hasArg(options::OPT_ccc_echo);
   if (const Arg *A = Args->getLastArg(options::OPT_ccc_gcc_name))
-    CCCGenericGCCName = A->getValue(*Args);
-  CCCUseClangCXX = Args->hasFlag(options::OPT_ccc_clang_cxx,
-                                 options::OPT_ccc_no_clang_cxx,
-                                 CCCUseClangCXX);
+    CCCGenericGCCName = A->getValue();
   CCCUsePCH = Args->hasFlag(options::OPT_ccc_pch_is_pch,
                             options::OPT_ccc_pch_is_pth);
-  CCCUseClang = !Args->hasArg(options::OPT_ccc_no_clang);
-  CCCUseClangCPP = !Args->hasArg(options::OPT_ccc_no_clang_cpp);
-  if (const Arg *A = Args->getLastArg(options::OPT_ccc_clang_archs)) {
-    StringRef Cur = A->getValue(*Args);
-
-    CCCClangArchs.clear();
-    while (!Cur.empty()) {
-      std::pair<StringRef, StringRef> Split = Cur.split(',');
-
-      if (!Split.first.empty()) {
-        llvm::Triple::ArchType Arch =
-          llvm::Triple(Split.first, "", "").getArch();
-
-        if (Arch == llvm::Triple::UnknownArch)
-          Diag(clang::diag::err_drv_invalid_arch_name) << Split.first;
-
-        CCCClangArchs.insert(Arch);
-      }
-
-      Cur = Split.second;
-    }
-  }
   // FIXME: DefaultTargetTriple is used by the target-prefixed calls to as/ld
   // and getToolChain is const.
   if (const Arg *A = Args->getLastArg(options::OPT_target))
-    DefaultTargetTriple = A->getValue(*Args);
+    DefaultTargetTriple = A->getValue();
   if (const Arg *A = Args->getLastArg(options::OPT_ccc_install_dir))
-    Dir = InstalledDir = A->getValue(*Args);
+    Dir = InstalledDir = A->getValue();
   for (arg_iterator it = Args->filtered_begin(options::OPT_B),
          ie = Args->filtered_end(); it != ie; ++it) {
     const Arg *A = *it;
     A->claim();
-    PrefixDirs.push_back(A->getValue(*Args, 0));
+    PrefixDirs.push_back(A->getValue(0));
   }
   if (const Arg *A = Args->getLastArg(options::OPT__sysroot_EQ))
-    SysRoot = A->getValue(*Args);
+    SysRoot = A->getValue();
   if (Args->hasArg(options::OPT_nostdlib))
     UseStdLib = false;
 
@@ -399,11 +362,11 @@ void Driver::generateCompilationDiagnostics(Compilation &C,
   std::string Cmd;
   llvm::raw_string_ostream OS(Cmd);
   if (FailingCommand)
-    C.PrintJob(OS, *FailingCommand, "\n", false);
+    C.PrintDiagnosticJob(OS, *FailingCommand);
   else
     // Crash triggered by FORCE_CLANG_DIAGNOSTICS_CRASH, which doesn't have an
     // associated FailingCommand, so just pass all jobs.
-    C.PrintJob(OS, C.getJobs(), "\n", false);
+    C.PrintDiagnosticJob(OS, C.getJobs());
   OS.flush();
 
   // Clear stale state and suppress tool output.
@@ -418,7 +381,7 @@ void Driver::generateCompilationDiagnostics(Compilation &C,
     bool IgnoreInput = false;
 
     // Ignore input from stdin or any inputs that cannot be preprocessed.
-    if (!strcmp(it->second->getValue(C.getArgs()), "-")) {
+    if (!strcmp(it->second->getValue(), "-")) {
       Diag(clang::diag::note_drv_command_failed_diag_msg)
         << "Error generating preprocessed source(s) - ignoring input from stdin"
         ".";
@@ -442,7 +405,7 @@ void Driver::generateCompilationDiagnostics(Compilation &C,
        it != ie; ++it) {
     Arg *A = *it;
     if (A->getOption().matches(options::OPT_arch)) {
-      StringRef ArchName = A->getValue(C.getArgs());
+      StringRef ArchName = A->getValue();
       ArchNames.insert(ArchName);
     }
   }
@@ -501,57 +464,6 @@ void Driver::generateCompilationDiagnostics(Compilation &C,
         Diag(clang::diag::note_drv_command_failed_diag_msg)
           << "Error generating run script: " + Script + " " + Err;
       } else {
-        // Strip away options not necessary to reproduce the crash.
-        // FIXME: This doesn't work with quotes (e.g., -D "foo bar").
-        SmallVector<std::string, 16> Flag;
-        Flag.push_back("-D ");
-        Flag.push_back("-F");
-        Flag.push_back("-I ");
-        Flag.push_back("-M ");
-        Flag.push_back("-MD ");
-        Flag.push_back("-MF ");
-        Flag.push_back("-MG ");
-        Flag.push_back("-MM ");
-        Flag.push_back("-MMD ");
-        Flag.push_back("-MP ");
-        Flag.push_back("-MQ ");
-        Flag.push_back("-MT ");
-        Flag.push_back("-o ");
-        Flag.push_back("-coverage-file ");
-        Flag.push_back("-dependency-file ");
-        Flag.push_back("-fdebug-compilation-dir ");
-        Flag.push_back("-fmodule-cache-path ");
-        Flag.push_back("-idirafter ");
-        Flag.push_back("-include ");
-        Flag.push_back("-include-pch ");
-        Flag.push_back("-internal-isystem ");
-        Flag.push_back("-internal-externc-isystem ");
-        Flag.push_back("-iprefix ");
-        Flag.push_back("-iwithprefix ");
-        Flag.push_back("-iwithprefixbefore ");
-        Flag.push_back("-isysroot ");
-        Flag.push_back("-isystem ");
-        Flag.push_back("-iquote ");
-        Flag.push_back("-resource-dir ");
-        Flag.push_back("-serialize-diagnostic-file ");
-        for (unsigned i = 0, e = Flag.size(); i < e; ++i) {
-          size_t I = 0, E = 0;
-          do {
-            I = Cmd.find(Flag[i], I);
-            if (I == std::string::npos) break;
-
-            E = Cmd.find(" ", I + Flag[i].length());
-            if (E == std::string::npos) break;
-            // The -D option is not removed.  Instead, the argument is quoted.
-            if (Flag[i] != "-D ") {
-              Cmd.erase(I, E - I + 1);
-            } else {
-              Cmd.insert(I+3, "\"");
-              Cmd.insert(++E, "\"");
-              I = E;
-            }
-          } while(1);
-        }
         // Append the new filename with correct preprocessed suffix.
         size_t I, E;
         I = Cmd.find("-main-file-name ");
@@ -639,12 +551,12 @@ void Driver::PrintOptions(const ArgList &Args) const {
        it != ie; ++it, ++i) {
     Arg *A = *it;
     llvm::errs() << "Option " << i << " - "
-                 << "Name: \"" << A->getOption().getName() << "\", "
+                 << "Name: \"" << A->getOption().getPrefixedName() << "\", "
                  << "Values: {";
     for (unsigned j = 0; j < A->getNumValues(); ++j) {
       if (j)
         llvm::errs() << ", ";
-      llvm::errs() << '"' << A->getValue(Args, j) << '"';
+      llvm::errs() << '"' << A->getValue(j) << '"';
     }
     llvm::errs() << "}\n";
   }
@@ -652,7 +564,9 @@ void Driver::PrintOptions(const ArgList &Args) const {
 
 void Driver::PrintHelp(bool ShowHidden) const {
   getOpts().PrintHelp(llvm::outs(), Name.c_str(), DriverTitle.c_str(),
-                      ShowHidden);
+                      /*Include*/0,
+                      /*Exclude*/options::NoDriverOption |
+                      (ShowHidden ? 0 : options::HelpHidden));
 }
 
 void Driver::PrintVersion(const Compilation &C, raw_ostream &OS) const {
@@ -750,12 +664,12 @@ bool Driver::HandleImmediateArgs(const Compilation &C) {
   // FIXME: The following handlers should use a callback mechanism, we don't
   // know what the client would like to do.
   if (Arg *A = C.getArgs().getLastArg(options::OPT_print_file_name_EQ)) {
-    llvm::outs() << GetFilePath(A->getValue(C.getArgs()), TC) << "\n";
+    llvm::outs() << GetFilePath(A->getValue(), TC) << "\n";
     return false;
   }
 
   if (Arg *A = C.getArgs().getLastArg(options::OPT_print_prog_name_EQ)) {
-    llvm::outs() << GetProgramPath(A->getValue(C.getArgs()), TC) << "\n";
+    llvm::outs() << GetProgramPath(A->getValue(), TC) << "\n";
     return false;
   }
 
@@ -818,7 +732,7 @@ static unsigned PrintActions1(const Compilation &C, Action *A,
 
   os << Action::getClassName(A->getKind()) << ", ";
   if (InputAction *IA = dyn_cast<InputAction>(A)) {
-    os << "\"" << IA->getInputArg().getValue(C.getArgs()) << "\"";
+    os << "\"" << IA->getInputArg().getValue() << "\"";
   } else if (BindArchAction *BIA = dyn_cast<BindArchAction>(A)) {
     os << '"' << BIA->getArchName() << '"'
        << ", {" << PrintActions1(C, *BIA->begin(), Ids) << "}";
@@ -878,7 +792,7 @@ void Driver::BuildUniversalActions(const ToolChain &TC,
       // Validate the option here; we don't save the type here because its
       // particular spelling may participate in other driver choices.
       llvm::Triple::ArchType Arch =
-        llvm::Triple::getArchTypeForDarwinArchName(A->getValue(Args));
+        tools::darwin::getArchTypeForDarwinArchName(A->getValue());
       if (Arch == llvm::Triple::UnknownArch) {
         Diag(clang::diag::err_drv_invalid_arch_name)
           << A->getAsString(Args);
@@ -886,15 +800,15 @@ void Driver::BuildUniversalActions(const ToolChain &TC,
       }
 
       A->claim();
-      if (ArchNames.insert(A->getValue(Args)))
-        Archs.push_back(A->getValue(Args));
+      if (ArchNames.insert(A->getValue()))
+        Archs.push_back(A->getValue());
     }
   }
 
   // When there is no explicit arch for this platform, make sure we still bind
   // the architecture (to the default) so that -Xarch_ is handled correctly.
   if (!Archs.size())
-    Archs.push_back(Args.MakeArgString(TC.getArchName()));
+    Archs.push_back(Args.MakeArgString(TC.getDefaultUniversalArchName()));
 
   // FIXME: We killed off some others but these aren't yet detected in a
   // functional manner. If we added information to jobs about which "auxiliary"
@@ -981,8 +895,8 @@ void Driver::BuildInputs(const ToolChain &TC, const DerivedArgList &Args,
        it != ie; ++it) {
     Arg *A = *it;
 
-    if (isa<InputOption>(A->getOption())) {
-      const char *Value = A->getValue(Args);
+    if (A->getOption().getKind() == Option::InputClass) {
+      const char *Value = A->getValue();
       types::ID Ty = types::TY_INVALID;
 
       // Infer the input type if necessary.
@@ -1049,8 +963,8 @@ void Driver::BuildInputs(const ToolChain &TC, const DerivedArgList &Args,
       if (CheckInputsExist && memcmp(Value, "-", 2) != 0) {
         SmallString<64> Path(Value);
         if (Arg *WorkDir = Args.getLastArg(options::OPT_working_directory)) {
-          SmallString<64> Directory(WorkDir->getValue(Args));
-          if (llvm::sys::path::is_absolute(Directory.str())) {
+          if (!llvm::sys::path::is_absolute(Path.str())) {
+            SmallString<64> Directory(WorkDir->getValue());
             llvm::sys::path::append(Directory, Value);
             Path.assign(Directory);
           }
@@ -1064,21 +978,21 @@ void Driver::BuildInputs(const ToolChain &TC, const DerivedArgList &Args,
       } else
         Inputs.push_back(std::make_pair(Ty, A));
 
-    } else if (A->getOption().isLinkerInput()) {
+    } else if (A->getOption().hasFlag(options::LinkerInput)) {
       // Just treat as object type, we could make a special type for this if
       // necessary.
       Inputs.push_back(std::make_pair(types::TY_Object, A));
 
     } else if (A->getOption().matches(options::OPT_x)) {
       InputTypeArg = A;
-      InputType = types::lookupTypeForTypeSpecifier(A->getValue(Args));
+      InputType = types::lookupTypeForTypeSpecifier(A->getValue());
       A->claim();
 
       // Follow gcc behavior and treat as linker input for invalid -x
       // options. Its not clear why we shouldn't just revert to unknown; but
       // this isn't very important, we might as well be bug compatible.
       if (!InputType) {
-        Diag(clang::diag::err_drv_unknown_language) << A->getValue(Args);
+        Diag(clang::diag::err_drv_unknown_language) << A->getValue();
         InputType = types::TY_Object;
       }
     }
@@ -1301,7 +1215,7 @@ void Driver::BuildJobs(Compilation &C) const {
     const char *LinkingOutput = 0;
     if (isa<LipoJobAction>(A)) {
       if (FinalOutput)
-        LinkingOutput = FinalOutput->getValue(C.getArgs());
+        LinkingOutput = FinalOutput->getValue();
       else
         LinkingOutput = DefaultImageName.c_str();
     }
@@ -1331,13 +1245,13 @@ void Driver::BuildJobs(Compilation &C) const {
     // DiagnosticsEngine, so that extra values, position, and so on could be
     // printed.
     if (!A->isClaimed()) {
-      if (A->getOption().hasNoArgumentUnused())
+      if (A->getOption().hasFlag(options::NoArgumentUnused))
         continue;
 
       // Suppress the warning automatically if this is just a flag, and it is an
       // instance of an argument we already claimed.
       const Option &Opt = A->getOption();
-      if (isa<FlagOption>(Opt)) {
+      if (Opt.getKind() == Option::FlagClass) {
         bool DuplicateClaimed = false;
 
         for (arg_iterator it = C.getArgs().filtered_begin(&Opt),
@@ -1392,6 +1306,7 @@ static const Tool &SelectToolForJob(Compilation &C, const ToolChain *TC,
       !C.getArgs().hasArg(options::OPT_no_integrated_cpp) &&
       !C.getArgs().hasArg(options::OPT_traditional_cpp) &&
       !C.getArgs().hasArg(options::OPT_save_temps) &&
+      !C.getArgs().hasArg(options::OPT_rewrite_objc) &&
       ToolForJob->hasIntegratedCPP())
     Inputs = &(*Inputs)[0]->getInputs();
 
@@ -1413,7 +1328,7 @@ void Driver::BuildJobsForAction(Compilation &C,
     const Arg &Input = IA->getInputArg();
     Input.claim();
     if (Input.getOption().matches(options::OPT_INPUT)) {
-      const char *Name = Input.getValue(C.getArgs());
+      const char *Name = Input.getValue();
       Result = InputInfo(Name, A->getType(), Name);
     } else
       Result = InputInfo(&Input, A->getType(), "");
@@ -1502,7 +1417,7 @@ const char *Driver::GetNamedOutputPath(Compilation &C,
   if (AtTopLevel && !isa<DsymutilJobAction>(JA) &&
       !isa<VerifyJobAction>(JA)) {
     if (Arg *FinalOutput = C.getArgs().getLastArg(options::OPT_o))
-      return C.addResultFile(FinalOutput->getValue(C.getArgs()));
+      return C.addResultFile(FinalOutput->getValue());
   }
 
   // Default to writing to stdout?
@@ -1580,7 +1495,7 @@ const char *Driver::GetNamedOutputPath(Compilation &C,
 
 std::string Driver::GetFilePath(const char *Name, const ToolChain &TC) const {
   // Respect a limited subset of the '-Bprefix' functionality in GCC by
-  // attempting to use this prefix when lokup up program paths.
+  // attempting to use this prefix when looking for file paths.
   for (Driver::prefix_list::const_iterator it = PrefixDirs.begin(),
        ie = PrefixDirs.end(); it != ie; ++it) {
     std::string Dir(*it);
@@ -1619,26 +1534,26 @@ std::string Driver::GetFilePath(const char *Name, const ToolChain &TC) const {
   return Name;
 }
 
-static bool isPathExecutable(llvm::sys::Path &P, bool WantFile) {
-    bool Exists;
-    return (WantFile ? !llvm::sys::fs::exists(P.str(), Exists) && Exists
-                 : P.canExecute());
-}
-
-std::string Driver::GetProgramPath(const char *Name, const ToolChain &TC,
-                                   bool WantFile) const {
+std::string Driver::GetProgramPath(const char *Name,
+                                   const ToolChain &TC) const {
   // FIXME: Needs a better variable than DefaultTargetTriple
   std::string TargetSpecificExecutable(DefaultTargetTriple + "-" + Name);
   // Respect a limited subset of the '-Bprefix' functionality in GCC by
-  // attempting to use this prefix when lokup up program paths.
+  // attempting to use this prefix when looking for program paths.
   for (Driver::prefix_list::const_iterator it = PrefixDirs.begin(),
        ie = PrefixDirs.end(); it != ie; ++it) {
-    llvm::sys::Path P(*it);
-    P.appendComponent(TargetSpecificExecutable);
-    if (isPathExecutable(P, WantFile)) return P.str();
-    P.eraseComponent();
-    P.appendComponent(Name);
-    if (isPathExecutable(P, WantFile)) return P.str();
+    bool IsDirectory;
+    if (!llvm::sys::fs::is_directory(*it, IsDirectory) && IsDirectory) {
+      llvm::sys::Path P(*it);
+      P.appendComponent(TargetSpecificExecutable);
+      if (P.canExecute()) return P.str();
+      P.eraseComponent();
+      P.appendComponent(Name);
+      if (P.canExecute()) return P.str();
+    } else {
+      llvm::sys::Path P(*it + Name);
+      if (P.canExecute()) return P.str();
+    }
   }
 
   const ToolChain::path_list &List = TC.getProgramPaths();
@@ -1646,10 +1561,10 @@ std::string Driver::GetProgramPath(const char *Name, const ToolChain &TC,
          it = List.begin(), ie = List.end(); it != ie; ++it) {
     llvm::sys::Path P(*it);
     P.appendComponent(TargetSpecificExecutable);
-    if (isPathExecutable(P, WantFile)) return P.str();
+    if (P.canExecute()) return P.str();
     P.eraseComponent();
     P.appendComponent(Name);
-    if (isPathExecutable(P, WantFile)) return P.str();
+    if (P.canExecute()) return P.str();
   }
 
   // If all else failed, search the path.
@@ -1701,7 +1616,7 @@ static llvm::Triple computeTargetTriple(StringRef DefaultTargetTriple,
                                         StringRef DarwinArchName) {
   // FIXME: Already done in Compilation *Driver::BuildCompilation
   if (const Arg *A = Args.getLastArg(options::OPT_target))
-    DefaultTargetTriple = A->getValue(Args);
+    DefaultTargetTriple = A->getValue();
 
   llvm::Triple Target(llvm::Triple::normalize(DefaultTargetTriple));
 
@@ -1710,14 +1625,14 @@ static llvm::Triple computeTargetTriple(StringRef DefaultTargetTriple,
     // If an explict Darwin arch name is given, that trumps all.
     if (!DarwinArchName.empty()) {
       Target.setArch(
-        llvm::Triple::getArchTypeForDarwinArchName(DarwinArchName));
+        tools::darwin::getArchTypeForDarwinArchName(DarwinArchName));
       return Target;
     }
 
     // Handle the Darwin '-arch' flag.
     if (Arg *A = Args.getLastArg(options::OPT_arch)) {
       llvm::Triple::ArchType DarwinArch
-        = llvm::Triple::getArchTypeForDarwinArchName(A->getValue(Args));
+        = tools::darwin::getArchTypeForDarwinArchName(A->getValue());
       if (DarwinArch != llvm::Triple::UnknownArch)
         Target.setArch(DarwinArch);
     }
@@ -1820,37 +1735,14 @@ bool Driver::ShouldUseClangCompiler(const Compilation &C, const JobAction &JA,
                                     const llvm::Triple &Triple) const {
   // Check if user requested no clang, or clang doesn't understand this type (we
   // only handle single inputs for now).
-  if (!CCCUseClang || JA.size() != 1 ||
+  if (JA.size() != 1 ||
       !types::isAcceptedByClang((*JA.begin())->getType()))
     return false;
 
   // Otherwise make sure this is an action clang understands.
-  if (isa<PreprocessJobAction>(JA)) {
-    if (!CCCUseClangCPP) {
-      Diag(clang::diag::warn_drv_not_using_clang_cpp);
-      return false;
-    }
-  } else if (!isa<PrecompileJobAction>(JA) && !isa<CompileJobAction>(JA))
-    return false;
-
-  // Use clang for C++?
-  if (!CCCUseClangCXX && types::isCXX((*JA.begin())->getType())) {
-    Diag(clang::diag::warn_drv_not_using_clang_cxx);
+  if (!isa<PreprocessJobAction>(JA) && !isa<PrecompileJobAction>(JA) &&
+      !isa<CompileJobAction>(JA))
     return false;
-  }
-
-  // Always use clang for precompiling, AST generation, and rewriting,
-  // regardless of archs.
-  if (isa<PrecompileJobAction>(JA) ||
-      types::isOnlyAcceptedByClang(JA.getType()))
-    return true;
-
-  // Finally, don't use clang if this isn't one of the user specified archs to
-  // build.
-  if (!CCCClangArchs.empty() && !CCCClangArchs.count(Triple.getArch())) {
-    Diag(clang::diag::warn_drv_not_using_clang_arch) << Triple.getArchName();
-    return false;
-  }
 
   return true;
 }
diff --git a/lib/Driver/DriverOptions.cpp b/lib/Driver/DriverOptions.cpp
index 715819d..3925b8a 100644
--- a/lib/Driver/DriverOptions.cpp
+++ b/lib/Driver/DriverOptions.cpp
@@ -14,11 +14,19 @@
 using namespace clang::driver;
 using namespace clang::driver::options;
 
+#define PREFIX(NAME, VALUE) const char *const NAME[] = VALUE;
+#define OPTION(PREFIX, NAME, ID, KIND, GROUP, ALIAS, FLAGS, PARAM, \
+               HELPTEXT, METAVAR)
+#include "clang/Driver/Options.inc"
+#undef OPTION
+#undef PREFIX
+
 static const OptTable::Info InfoTable[] = {
-#define OPTION(NAME, ID, KIND, GROUP, ALIAS, FLAGS, PARAM, \
+#define PREFIX(NAME, VALUE)
+#define OPTION(PREFIX, NAME, ID, KIND, GROUP, ALIAS, FLAGS, PARAM, \
                HELPTEXT, METAVAR)   \
-  { NAME, HELPTEXT, METAVAR, Option::KIND##Class, PARAM, FLAGS, \
-    OPT_##GROUP, OPT_##ALIAS },
+  { PREFIX, NAME, HELPTEXT, METAVAR, OPT_##ID, Option::KIND##Class, PARAM, \
+    FLAGS, OPT_##GROUP, OPT_##ALIAS },
 #include "clang/Driver/Options.inc"
 };
 
diff --git a/lib/Driver/OptTable.cpp b/lib/Driver/OptTable.cpp
index a3e38b2..6e7b695 100644
--- a/lib/Driver/OptTable.cpp
+++ b/lib/Driver/OptTable.cpp
@@ -11,6 +11,7 @@
 #include "clang/Driver/Arg.h"
 #include "clang/Driver/ArgList.h"
 #include "clang/Driver/Option.h"
+#include "clang/Driver/Options.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/ErrorHandling.h"
 #include <algorithm>
@@ -54,6 +55,13 @@ static inline bool operator<(const OptTable::Info &A, const OptTable::Info &B) {
   if (int N = StrCmpOptionName(A.Name, B.Name))
     return N == -1;
 
+  for (const char * const *APre = A.Prefixes,
+                  * const *BPre = B.Prefixes;
+                          *APre != 0 && *BPre != 0; ++APre, ++BPre) {
+    if (int N = StrCmpOptionName(*APre, *BPre))
+      return N == -1;
+  }
+
   // Names are the same, check that classes are in order; exactly one
   // should be joined, and it should succeed the other.
   assert(((A.Kind == Option::JoinedClass) ^ (B.Kind == Option::JoinedClass)) &&
@@ -78,23 +86,24 @@ OptSpecifier::OptSpecifier(const Option *Opt) : ID(Opt->getID()) {}
 //
 
 OptTable::OptTable(const Info *_OptionInfos, unsigned _NumOptionInfos)
-  : OptionInfos(_OptionInfos), NumOptionInfos(_NumOptionInfos),
-    Options(new Option*[NumOptionInfos]),
-    TheInputOption(0), TheUnknownOption(0), FirstSearchableIndex(0)
+  : OptionInfos(_OptionInfos),
+    NumOptionInfos(_NumOptionInfos),
+    TheInputOptionID(0),
+    TheUnknownOptionID(0),
+    FirstSearchableIndex(0)
 {
   // Explicitly zero initialize the error to work around a bug in array
   // value-initialization on MinGW with gcc 4.3.5.
-  memset(Options, 0, sizeof(*Options) * NumOptionInfos);
 
   // Find start of normal options.
   for (unsigned i = 0, e = getNumOptions(); i != e; ++i) {
     unsigned Kind = getInfo(i + 1).Kind;
     if (Kind == Option::InputClass) {
-      assert(!TheInputOption && "Cannot have multiple input options!");
-      TheInputOption = getOption(i + 1);
+      assert(!TheInputOptionID && "Cannot have multiple input options!");
+      TheInputOptionID = getInfo(i + 1).ID;
     } else if (Kind == Option::UnknownClass) {
-      assert(!TheUnknownOption && "Cannot have multiple input options!");
-      TheUnknownOption = getOption(i + 1);
+      assert(!TheUnknownOptionID && "Cannot have multiple unknown options!");
+      TheUnknownOptionID = getInfo(i + 1).ID;
     } else if (Kind != Option::GroupClass) {
       FirstSearchableIndex = i;
       break;
@@ -115,91 +124,80 @@ OptTable::OptTable(const Info *_OptionInfos, unsigned _NumOptionInfos)
   // Check that options are in order.
   for (unsigned i = FirstSearchableIndex+1, e = getNumOptions(); i != e; ++i) {
     if (!(getInfo(i) < getInfo(i + 1))) {
-      getOption(i)->dump();
-      getOption(i + 1)->dump();
+      getOption(i).dump();
+      getOption(i + 1).dump();
       llvm_unreachable("Options are not in order!");
     }
   }
 #endif
+
+  // Build prefixes.
+  for (unsigned i = FirstSearchableIndex+1, e = getNumOptions(); i != e; ++i) {
+    if (const char *const *P = getInfo(i).Prefixes) {
+      for (; *P != 0; ++P) {
+        PrefixesUnion.insert(*P);
+      }
+    }
+  }
+
+  // Build prefix chars.
+  for (llvm::StringSet<>::const_iterator I = PrefixesUnion.begin(),
+                                         E = PrefixesUnion.end(); I != E; ++I) {
+    StringRef Prefix = I->getKey();
+    for (StringRef::const_iterator C = Prefix.begin(), CE = Prefix.end();
+                                   C != CE; ++C)
+      if (std::find(PrefixChars.begin(), PrefixChars.end(), *C)
+            == PrefixChars.end())
+        PrefixChars.push_back(*C);
+  }
 }
 
 OptTable::~OptTable() {
-  for (unsigned i = 0, e = getNumOptions(); i != e; ++i)
-    delete Options[i];
-  delete[] Options;
 }
 
-Option *OptTable::CreateOption(unsigned id) const {
-  const Info &info = getInfo(id);
-  const OptionGroup *Group =
-    cast_or_null<OptionGroup>(getOption(info.GroupID));
-  const Option *Alias = getOption(info.AliasID);
-
-  Option *Opt = 0;
-  switch (info.Kind) {
-  case Option::InputClass:
-    Opt = new InputOption(id); break;
-  case Option::UnknownClass:
-    Opt = new UnknownOption(id); break;
-  case Option::GroupClass:
-    Opt = new OptionGroup(id, info.Name, Group); break;
-  case Option::FlagClass:
-    Opt = new FlagOption(id, info.Name, Group, Alias); break;
-  case Option::JoinedClass:
-    Opt = new JoinedOption(id, info.Name, Group, Alias); break;
-  case Option::SeparateClass:
-    Opt = new SeparateOption(id, info.Name, Group, Alias); break;
-  case Option::CommaJoinedClass:
-    Opt = new CommaJoinedOption(id, info.Name, Group, Alias); break;
-  case Option::MultiArgClass:
-    Opt = new MultiArgOption(id, info.Name, Group, Alias, info.Param); break;
-  case Option::JoinedOrSeparateClass:
-    Opt = new JoinedOrSeparateOption(id, info.Name, Group, Alias); break;
-  case Option::JoinedAndSeparateClass:
-    Opt = new JoinedAndSeparateOption(id, info.Name, Group, Alias); break;
-  }
+const Option OptTable::getOption(OptSpecifier Opt) const {
+  unsigned id = Opt.getID();
+  if (id == 0)
+    return Option(0, 0);
+  assert((unsigned) (id - 1) < getNumOptions() && "Invalid ID.");
+  return Option(&getInfo(id), this);
+}
 
-  if (info.Flags & DriverOption)
-    Opt->setDriverOption(true);
-  if (info.Flags & LinkerInput)
-    Opt->setLinkerInput(true);
-  if (info.Flags & NoArgumentUnused)
-    Opt->setNoArgumentUnused(true);
-  if (info.Flags & NoForward)
-    Opt->setNoForward(true);
-  if (info.Flags & RenderAsInput)
-    Opt->setNoOptAsInput(true);
-  if (info.Flags & RenderJoined) {
-    assert((info.Kind == Option::JoinedOrSeparateClass ||
-            info.Kind == Option::SeparateClass) && "Invalid option.");
-    Opt->setRenderStyle(Option::RenderJoinedStyle);
-  }
-  if (info.Flags & RenderSeparate) {
-    assert((info.Kind == Option::JoinedOrSeparateClass ||
-            info.Kind == Option::JoinedClass) && "Invalid option.");
-    Opt->setRenderStyle(Option::RenderSeparateStyle);
-  }
-  if (info.Flags & Unsupported)
-    Opt->setUnsupported(true);
-  if (info.Flags & CC1Option)
-    Opt->setIsCC1Option(true);
+static bool isInput(const llvm::StringSet<> &Prefixes, StringRef Arg) {
+  if (Arg == "-")
+    return true;
+  for (llvm::StringSet<>::const_iterator I = Prefixes.begin(),
+                                         E = Prefixes.end(); I != E; ++I)
+    if (Arg.startswith(I->getKey()))
+      return false;
+  return true;
+}
 
-  return Opt;
+/// \returns Matched size. 0 means no match.
+static unsigned matchOption(const OptTable::Info *I, StringRef Str) {
+  for (const char * const *Pre = I->Prefixes; *Pre != 0; ++Pre) {
+    StringRef Prefix(*Pre);
+    if (Str.startswith(Prefix) && Str.substr(Prefix.size()).startswith(I->Name))
+      return Prefix.size() + StringRef(I->Name).size();
+  }
+  return 0;
 }
 
 Arg *OptTable::ParseOneArg(const ArgList &Args, unsigned &Index) const {
   unsigned Prev = Index;
   const char *Str = Args.getArgString(Index);
 
-  // Anything that doesn't start with '-' is an input, as is '-' itself.
-  if (Str[0] != '-' || Str[1] == '\0')
-    return new Arg(TheInputOption, Index++, Str);
+  // Anything that doesn't start with PrefixesUnion is an input, as is '-'
+  // itself.
+  if (isInput(PrefixesUnion, Str))
+    return new Arg(getOption(TheInputOptionID), Str, Index++, Str);
 
   const Info *Start = OptionInfos + FirstSearchableIndex;
   const Info *End = OptionInfos + getNumOptions();
+  StringRef Name = StringRef(Str).ltrim(PrefixChars);
 
   // Search for the first next option which could be a prefix.
-  Start = std::lower_bound(Start, End, Str);
+  Start = std::lower_bound(Start, End, Name.data());
 
   // Options are stored in sorted order, with '\0' at the end of the
   // alphabet. Since the only options which can accept a string must
@@ -210,15 +208,16 @@ Arg *OptTable::ParseOneArg(const ArgList &Args, unsigned &Index) const {
   // blanking on the simplest way to make it fast. We can solve this
   // problem when we move to TableGen.
   for (; Start != End; ++Start) {
+    unsigned ArgSize = 0;
     // Scan for first option which is a proper prefix.
     for (; Start != End; ++Start)
-      if (memcmp(Str, Start->Name, strlen(Start->Name)) == 0)
+      if ((ArgSize = matchOption(Start, Str)))
         break;
     if (Start == End)
       break;
 
     // See if this option matches.
-    if (Arg *A = getOption(Start - OptionInfos + 1)->accept(Args, Index))
+    if (Arg *A = Option(Start, this).accept(Args, Index, ArgSize))
       return A;
 
     // Otherwise, see if this argument was missing values.
@@ -226,7 +225,7 @@ Arg *OptTable::ParseOneArg(const ArgList &Args, unsigned &Index) const {
       return 0;
   }
 
-  return new Arg(TheUnknownOption, Index++, Str);
+  return new Arg(getOption(TheUnknownOptionID), Str, Index++, Str);
 }
 
 InputArgList *OptTable::ParseArgs(const char* const *ArgBegin,
@@ -266,10 +265,11 @@ InputArgList *OptTable::ParseArgs(const char* const *ArgBegin,
 }
 
 static std::string getOptionHelpName(const OptTable &Opts, OptSpecifier Id) {
-  std::string Name = Opts.getOptionName(Id);
+  const Option O = Opts.getOption(Id);
+  std::string Name = O.getPrefixedName();
 
   // Add metavar, if used.
-  switch (Opts.getOptionKind(Id)) {
+  switch (O.getKind()) {
   case Option::GroupClass: case Option::InputClass: case Option::UnknownClass:
     llvm_unreachable("Invalid option with help text.");
 
@@ -346,7 +346,8 @@ static const char *getOptionHelpGroup(const OptTable &Opts, OptSpecifier Id) {
 }
 
 void OptTable::PrintHelp(raw_ostream &OS, const char *Name,
-                         const char *Title, bool ShowHidden) const {
+                         const char *Title, unsigned short FlagsToInclude,
+                         unsigned short FlagsToExclude) const {
   OS << "OVERVIEW: " << Title << "\n";
   OS << '\n';
   OS << "USAGE: " << Name << " [options] <inputs>\n";
@@ -365,7 +366,8 @@ void OptTable::PrintHelp(raw_ostream &OS, const char *Name,
     if (getOptionKind(Id) == Option::GroupClass)
       continue;
 
-    if (!ShowHidden && isOptionHelpHidden(Id))
+    if ((FlagsToInclude && !(getInfo(Id).Flags & FlagsToInclude)) ||
+        getInfo(Id).Flags & FlagsToExclude)
       continue;
 
     if (const char *Text = getOptionHelpText(Id)) {
diff --git a/lib/Driver/Option.cpp b/lib/Driver/Option.cpp
index 03360ea..9a34df5 100644
--- a/lib/Driver/Option.cpp
+++ b/lib/Driver/Option.cpp
@@ -13,46 +13,20 @@
 #include "clang/Driver/ArgList.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/ADT/Twine.h"
 #include <cassert>
 #include <algorithm>
 using namespace clang::driver;
 
-Option::Option(OptionClass _Kind, OptSpecifier _ID, const char *_Name,
-               const OptionGroup *_Group, const Option *_Alias)
-  : Kind(_Kind), ID(_ID.getID()), Name(_Name), Group(_Group), Alias(_Alias),
-    Unsupported(false), LinkerInput(false), NoOptAsInput(false),
-    DriverOption(false), NoArgumentUnused(false), NoForward(false) {
+Option::Option(const OptTable::Info *info, const OptTable *owner)
+  : Info(info), Owner(owner) {
 
   // Multi-level aliases are not supported, and alias options cannot
   // have groups. This just simplifies option tracking, it is not an
   // inherent limitation.
-  assert((!Alias || (!Alias->Alias && !Group)) &&
+  assert((!Info || !getAlias().isValid() || (!getAlias().getAlias().isValid() &&
+         !getGroup().isValid())) &&
          "Multi-level aliases and aliases with groups are unsupported.");
-
-  // Initialize rendering options based on the class.
-  switch (Kind) {
-  case GroupClass:
-  case InputClass:
-  case UnknownClass:
-    RenderStyle = RenderValuesStyle;
-    break;
-
-  case JoinedClass:
-  case JoinedAndSeparateClass:
-    RenderStyle = RenderJoinedStyle;
-    break;
-
-  case CommaJoinedClass:
-    RenderStyle = RenderCommaJoinedStyle;
-    break;
-
-  case FlagClass:
-  case SeparateClass:
-  case MultiArgClass:
-  case JoinedOrSeparateClass:
-    RenderStyle = RenderSeparateStyle;
-    break;
-  }
 }
 
 Option::~Option() {
@@ -60,7 +34,7 @@ Option::~Option() {
 
 void Option::dump() const {
   llvm::errs() << "<";
-  switch (Kind) {
+  switch (getKind()) {
 #define P(N) case N: llvm::errs() << #N; break
     P(GroupClass);
     P(InputClass);
@@ -75,206 +49,153 @@ void Option::dump() const {
 #undef P
   }
 
-  llvm::errs() << " Name:\"" << Name << '"';
+  llvm::errs() << " Prefixes:[";
+  for (const char * const *Pre = Info->Prefixes; *Pre != 0; ++Pre) {
+    llvm::errs() << '"' << *Pre << (*(Pre + 1) == 0 ? "\"" : "\", ");
+  }
+  llvm::errs() << ']';
 
-  if (Group) {
+  llvm::errs() << " Name:\"" << getName() << '"';
+
+  const Option Group = getGroup();
+  if (Group.isValid()) {
     llvm::errs() << " Group:";
-    Group->dump();
+    Group.dump();
   }
 
-  if (Alias) {
+  const Option Alias = getAlias();
+  if (Alias.isValid()) {
     llvm::errs() << " Alias:";
-    Alias->dump();
+    Alias.dump();
   }
 
-  if (const MultiArgOption *MOA = dyn_cast<MultiArgOption>(this))
-    llvm::errs() << " NumArgs:" << MOA->getNumArgs();
+  if (getKind() == MultiArgClass)
+    llvm::errs() << " NumArgs:" << getNumArgs();
 
   llvm::errs() << ">\n";
 }
 
 bool Option::matches(OptSpecifier Opt) const {
   // Aliases are never considered in matching, look through them.
-  if (Alias)
-    return Alias->matches(Opt);
+  const Option Alias = getAlias();
+  if (Alias.isValid())
+    return Alias.matches(Opt);
 
   // Check exact match.
-  if (ID == Opt)
+  if (getID() == Opt.getID())
     return true;
 
-  if (Group)
-    return Group->matches(Opt);
+  const Option Group = getGroup();
+  if (Group.isValid())
+    return Group.matches(Opt);
   return false;
 }
 
-OptionGroup::OptionGroup(OptSpecifier ID, const char *Name,
-                         const OptionGroup *Group)
-  : Option(Option::GroupClass, ID, Name, Group, 0) {
-}
-
-Arg *OptionGroup::accept(const ArgList &Args, unsigned &Index) const {
-  llvm_unreachable("accept() should never be called on an OptionGroup");
-}
-
-InputOption::InputOption(OptSpecifier ID)
-  : Option(Option::InputClass, ID, "<input>", 0, 0) {
-}
-
-Arg *InputOption::accept(const ArgList &Args, unsigned &Index) const {
-  llvm_unreachable("accept() should never be called on an InputOption");
-}
-
-UnknownOption::UnknownOption(OptSpecifier ID)
-  : Option(Option::UnknownClass, ID, "<unknown>", 0, 0) {
-}
-
-Arg *UnknownOption::accept(const ArgList &Args, unsigned &Index) const {
-  llvm_unreachable("accept() should never be called on an UnknownOption");
-}
-
-FlagOption::FlagOption(OptSpecifier ID, const char *Name,
-                       const OptionGroup *Group, const Option *Alias)
-  : Option(Option::FlagClass, ID, Name, Group, Alias) {
-}
-
-Arg *FlagOption::accept(const ArgList &Args, unsigned &Index) const {
-  // Matches iff this is an exact match.
-  // FIXME: Avoid strlen.
-  if (getName().size() != strlen(Args.getArgString(Index)))
-    return 0;
-
-  return new Arg(getUnaliasedOption(), Index++);
-}
-
-JoinedOption::JoinedOption(OptSpecifier ID, const char *Name,
-                           const OptionGroup *Group, const Option *Alias)
-  : Option(Option::JoinedClass, ID, Name, Group, Alias) {
-}
-
-Arg *JoinedOption::accept(const ArgList &Args, unsigned &Index) const {
-  // Always matches.
-  const char *Value = Args.getArgString(Index) + getName().size();
-  return new Arg(getUnaliasedOption(), Index++, Value);
-}
-
-CommaJoinedOption::CommaJoinedOption(OptSpecifier ID, const char *Name,
-                                     const OptionGroup *Group,
-                                     const Option *Alias)
-  : Option(Option::CommaJoinedClass, ID, Name, Group, Alias) {
-}
-
-Arg *CommaJoinedOption::accept(const ArgList &Args,
-                               unsigned &Index) const {
-  // Always matches.
-  const char *Str = Args.getArgString(Index) + getName().size();
-  Arg *A = new Arg(getUnaliasedOption(), Index++);
+Arg *Option::accept(const ArgList &Args,
+                    unsigned &Index,
+                    unsigned ArgSize) const {
+  const Option &UnaliasedOption = getUnaliasedOption();
+  StringRef Spelling;
+  // If the option was an alias, get the spelling from the unaliased one.
+  if (getID() == UnaliasedOption.getID()) {
+    Spelling = StringRef(Args.getArgString(Index), ArgSize);
+  } else {
+    Spelling = Args.MakeArgString(Twine(UnaliasedOption.getPrefix()) +
+                                  Twine(UnaliasedOption.getName()));
+  }
 
-  // Parse out the comma separated values.
-  const char *Prev = Str;
-  for (;; ++Str) {
-    char c = *Str;
+  switch (getKind()) {
+  case FlagClass:
+    if (ArgSize != strlen(Args.getArgString(Index)))
+      return 0;
 
-    if (!c || c == ',') {
-      if (Prev != Str) {
-        char *Value = new char[Str - Prev + 1];
-        memcpy(Value, Prev, Str - Prev);
-        Value[Str - Prev] = '\0';
-        A->getValues().push_back(Value);
+    return new Arg(UnaliasedOption, Spelling, Index++);
+  case JoinedClass: {
+    const char *Value = Args.getArgString(Index) + ArgSize;
+    return new Arg(UnaliasedOption, Spelling, Index++, Value);
+  }
+  case CommaJoinedClass: {
+    // Always matches.
+    const char *Str = Args.getArgString(Index) + ArgSize;
+    Arg *A = new Arg(UnaliasedOption, Spelling, Index++);
+
+    // Parse out the comma separated values.
+    const char *Prev = Str;
+    for (;; ++Str) {
+      char c = *Str;
+
+      if (!c || c == ',') {
+        if (Prev != Str) {
+          char *Value = new char[Str - Prev + 1];
+          memcpy(Value, Prev, Str - Prev);
+          Value[Str - Prev] = '\0';
+          A->getValues().push_back(Value);
+        }
+
+        if (!c)
+          break;
+
+        Prev = Str + 1;
       }
-
-      if (!c)
-        break;
-
-      Prev = Str + 1;
     }
-  }
-  A->setOwnsValues(true);
-
-  return A;
-}
-
-SeparateOption::SeparateOption(OptSpecifier ID, const char *Name,
-                               const OptionGroup *Group, const Option *Alias)
-  : Option(Option::SeparateClass, ID, Name, Group, Alias) {
-}
-
-Arg *SeparateOption::accept(const ArgList &Args, unsigned &Index) const {
-  // Matches iff this is an exact match.
-  // FIXME: Avoid strlen.
-  if (getName().size() != strlen(Args.getArgString(Index)))
-    return 0;
-
-  Index += 2;
-  if (Index > Args.getNumInputArgStrings())
-    return 0;
-
-  return new Arg(getUnaliasedOption(), Index - 2, Args.getArgString(Index - 1));
-}
-
-MultiArgOption::MultiArgOption(OptSpecifier ID, const char *Name,
-                               const OptionGroup *Group, const Option *Alias,
-                               unsigned _NumArgs)
-  : Option(Option::MultiArgClass, ID, Name, Group, Alias), NumArgs(_NumArgs) {
-  assert(NumArgs > 1  && "Invalid MultiArgOption!");
-}
-
-Arg *MultiArgOption::accept(const ArgList &Args, unsigned &Index) const {
-  // Matches iff this is an exact match.
-  // FIXME: Avoid strlen.
-  if (getName().size() != strlen(Args.getArgString(Index)))
-    return 0;
-
-  Index += 1 + NumArgs;
-  if (Index > Args.getNumInputArgStrings())
-    return 0;
+    A->setOwnsValues(true);
 
-  Arg *A = new Arg(getUnaliasedOption(), Index - 1 - NumArgs,
-                   Args.getArgString(Index - NumArgs));
-  for (unsigned i = 1; i != NumArgs; ++i)
-    A->getValues().push_back(Args.getArgString(Index - NumArgs + i));
-  return A;
-}
-
-JoinedOrSeparateOption::JoinedOrSeparateOption(OptSpecifier ID,
-                                               const char *Name,
-                                               const OptionGroup *Group,
-                                               const Option *Alias)
-  : Option(Option::JoinedOrSeparateClass, ID, Name, Group, Alias) {
-}
-
-Arg *JoinedOrSeparateOption::accept(const ArgList &Args,
-                                    unsigned &Index) const {
-  // If this is not an exact match, it is a joined arg.
-  // FIXME: Avoid strlen.
-  if (getName().size() != strlen(Args.getArgString(Index))) {
-    const char *Value = Args.getArgString(Index) + getName().size();
-    return new Arg(this, Index++, Value);
+    return A;
   }
+  case SeparateClass:
+    // Matches iff this is an exact match.
+    // FIXME: Avoid strlen.
+    if (ArgSize != strlen(Args.getArgString(Index)))
+      return 0;
+
+    Index += 2;
+    if (Index > Args.getNumInputArgStrings())
+      return 0;
+
+    return new Arg(UnaliasedOption, Spelling,
+                   Index - 2, Args.getArgString(Index - 1));
+  case MultiArgClass: {
+    // Matches iff this is an exact match.
+    // FIXME: Avoid strlen.
+    if (ArgSize != strlen(Args.getArgString(Index)))
+      return 0;
+
+    Index += 1 + getNumArgs();
+    if (Index > Args.getNumInputArgStrings())
+      return 0;
+
+    Arg *A = new Arg(UnaliasedOption, Spelling, Index - 1 - getNumArgs(),
+                      Args.getArgString(Index - getNumArgs()));
+    for (unsigned i = 1; i != getNumArgs(); ++i)
+      A->getValues().push_back(Args.getArgString(Index - getNumArgs() + i));
+    return A;
+  }
+  case JoinedOrSeparateClass: {
+    // If this is not an exact match, it is a joined arg.
+    // FIXME: Avoid strlen.
+    if (ArgSize != strlen(Args.getArgString(Index))) {
+      const char *Value = Args.getArgString(Index) + ArgSize;
+      return new Arg(*this, Spelling, Index++, Value);
+    }
 
-  // Otherwise it must be separate.
-  Index += 2;
-  if (Index > Args.getNumInputArgStrings())
-    return 0;
-
-  return new Arg(getUnaliasedOption(), Index - 2, Args.getArgString(Index - 1));
-}
-
-JoinedAndSeparateOption::JoinedAndSeparateOption(OptSpecifier ID,
-                                                 const char *Name,
-                                                 const OptionGroup *Group,
-                                                 const Option *Alias)
-  : Option(Option::JoinedAndSeparateClass, ID, Name, Group, Alias) {
-}
-
-Arg *JoinedAndSeparateOption::accept(const ArgList &Args,
-                                     unsigned &Index) const {
-  // Always matches.
-
-  Index += 2;
-  if (Index > Args.getNumInputArgStrings())
-    return 0;
+    // Otherwise it must be separate.
+    Index += 2;
+    if (Index > Args.getNumInputArgStrings())
+      return 0;
 
-  return new Arg(getUnaliasedOption(), Index - 2,
-                 Args.getArgString(Index-2)+getName().size(),
-                 Args.getArgString(Index-1));
+    return new Arg(UnaliasedOption, Spelling,
+                   Index - 2, Args.getArgString(Index - 1));
+  }
+  case JoinedAndSeparateClass:
+    // Always matches.
+    Index += 2;
+    if (Index > Args.getNumInputArgStrings())
+      return 0;
+
+    return new Arg(UnaliasedOption, Spelling, Index - 2,
+                   Args.getArgString(Index - 2) + ArgSize,
+                   Args.getArgString(Index - 1));
+  default:
+    llvm_unreachable("Invalid option kind!");
+  }
 }
diff --git a/lib/Driver/SanitizerArgs.h b/lib/Driver/SanitizerArgs.h
new file mode 100644
index 0000000..ecb396e
--- /dev/null
+++ b/lib/Driver/SanitizerArgs.h
@@ -0,0 +1,106 @@
+//===--- SanitizerArgs.h - Arguments for sanitizer tools  -------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+#ifndef CLANG_LIB_DRIVER_SANITIZERARGS_H_
+#define CLANG_LIB_DRIVER_SANITIZERARGS_H_
+
+#include "clang/Driver/ArgList.h"
+
+namespace clang {
+namespace driver {
+
+class SanitizerArgs {
+  /// Assign ordinals to sanitizer flags. We'll use the ordinal values as
+  /// bit positions within \c Kind.
+  enum SanitizeOrdinal {
+#define SANITIZER(NAME, ID) SO_##ID,
+#include "clang/Basic/Sanitizers.def"
+    SO_Count
+  };
+
+  /// Bugs to catch at runtime.
+  enum SanitizeKind {
+#define SANITIZER(NAME, ID) ID = 1 << SO_##ID,
+#define SANITIZER_GROUP(NAME, ID, ALIAS) ID = ALIAS,
+#include "clang/Basic/Sanitizers.def"
+    NeedsAsanRt = Address,
+    NeedsTsanRt = Thread,
+    NeedsUbsanRt = Undefined
+  };
+  unsigned Kind;
+
+ public:
+  SanitizerArgs() : Kind(0) {}
+  /// Parses the sanitizer arguments from an argument list.
+  SanitizerArgs(const Driver &D, const ArgList &Args);
+
+  bool needsAsanRt() const { return Kind & NeedsAsanRt; }
+  bool needsTsanRt() const { return Kind & NeedsTsanRt; }
+  bool needsUbsanRt() const { return Kind & NeedsUbsanRt; }
+
+  bool sanitizesVptr() const { return Kind & Vptr; }
+  
+  void addArgs(const ArgList &Args, ArgStringList &CmdArgs) const {
+    if (!Kind)
+      return;
+    llvm::SmallString<256> SanitizeOpt("-fsanitize=");
+#define SANITIZER(NAME, ID) \
+    if (Kind & ID) \
+      SanitizeOpt += NAME ",";
+#include "clang/Basic/Sanitizers.def"
+    SanitizeOpt.pop_back();
+    CmdArgs.push_back(Args.MakeArgString(SanitizeOpt));
+  }
+
+ private:
+  /// Parse a single value from a -fsanitize= or -fno-sanitize= value list.
+  /// Returns a member of the \c SanitizeKind enumeration, or \c 0 if \p Value
+  /// is not known.
+  static unsigned parse(const char *Value) {
+    return llvm::StringSwitch<SanitizeKind>(Value)
+#define SANITIZER(NAME, ID) .Case(NAME, ID)
+#define SANITIZER_GROUP(NAME, ID, ALIAS) .Case(NAME, ID)
+#include "clang/Basic/Sanitizers.def"
+      .Default(SanitizeKind());
+  }
+
+  /// Parse a -fsanitize= or -fno-sanitize= argument's values, diagnosing any
+  /// invalid components.
+  static unsigned parse(const Driver &D, const Arg *A) {
+    unsigned Kind = 0;
+    for (unsigned I = 0, N = A->getNumValues(); I != N; ++I) {
+      if (unsigned K = parse(A->getValue(I)))
+        Kind |= K;
+      else
+        D.Diag(diag::err_drv_unsupported_option_argument)
+          << A->getOption().getName() << A->getValue(I);
+    }
+    return Kind;
+  }
+
+  /// Produce an argument string from argument \p A, which shows how it provides
+  /// a value in \p Mask. For instance, the argument
+  /// "-fsanitize=address,alignment" with mask \c NeedsUbsanRt would produce
+  /// "-fsanitize=alignment".
+  static std::string describeSanitizeArg(const ArgList &Args, const Arg *A,
+                                         unsigned Mask) {
+    if (!A->getOption().matches(options::OPT_fsanitize_EQ))
+      return A->getAsString(Args);
+
+    for (unsigned I = 0, N = A->getNumValues(); I != N; ++I)
+      if (parse(A->getValue(I)) & Mask)
+        return std::string("-fsanitize=") + A->getValue(I);
+
+    llvm_unreachable("arg didn't provide expected value");
+  }
+};
+
+}  // namespace driver
+}  // namespace clang
+
+#endif // CLANG_LIB_DRIVER_SANITIZERARGS_H_
diff --git a/lib/Driver/ToolChain.cpp b/lib/Driver/ToolChain.cpp
index 48ed044..de8ed1d 100644
--- a/lib/Driver/ToolChain.cpp
+++ b/lib/Driver/ToolChain.cpp
@@ -14,6 +14,7 @@
 #include "clang/Driver/ArgList.h"
 #include "clang/Driver/Driver.h"
 #include "clang/Driver/DriverDiagnostic.h"
+#include "clang/Driver/Option.h"
 #include "clang/Driver/Options.h"
 #include "llvm/ADT/StringSwitch.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -32,13 +33,32 @@ const Driver &ToolChain::getDriver() const {
  return D;
 }
 
+std::string ToolChain::getDefaultUniversalArchName() const {
+  // In universal driver terms, the arch name accepted by -arch isn't exactly
+  // the same as the ones that appear in the triple. Roughly speaking, this is
+  // an inverse of the darwin::getArchTypeForDarwinArchName() function, but the
+  // only interesting special case is powerpc.
+  switch (Triple.getArch()) {
+  case llvm::Triple::ppc:
+    return "ppc";
+  case llvm::Triple::ppc64:
+    return "ppc64";
+  default:
+    return Triple.getArchName();
+  }
+}
+
+bool ToolChain::IsUnwindTablesDefault() const {
+  return false;
+}
+
 std::string ToolChain::GetFilePath(const char *Name) const {
   return D.GetFilePath(Name, *this);
 
 }
 
-std::string ToolChain::GetProgramPath(const char *Name, bool WantFile) const {
-  return D.GetProgramPath(Name, *this, WantFile);
+std::string ToolChain::GetProgramPath(const char *Name) const {
+  return D.GetProgramPath(Name, *this);
 }
 
 types::ID ToolChain::LookupTypeForExtension(const char *Ext) const {
@@ -66,13 +86,13 @@ static const char *getARMTargetCPU(const ArgList &Args,
     // FIXME: Warn on inconsistent use of -mcpu and -march.
     // If we have -mcpu=, use that.
     if (Arg *A = Args.getLastArg(options::OPT_mcpu_EQ))
-      return A->getValue(Args);
+      return A->getValue();
   }
 
   StringRef MArch;
   if (Arg *A = Args.getLastArg(options::OPT_march_EQ)) {
     // Otherwise, if we have -march= choose the base CPU for that arch.
-    MArch = A->getValue(Args);
+    MArch = A->getValue();
   } else {
     // Otherwise, use the Arch from the triple.
     MArch = Triple.getArchName();
@@ -91,6 +111,8 @@ static const char *getARMTargetCPU(const ArgList &Args,
     .Cases("armv6z", "armv6zk", "arm1176jzf-s")
     .Case("armv6t2", "arm1156t2-s")
     .Cases("armv7", "armv7a", "armv7-a", "cortex-a8")
+    .Cases("armv7f", "armv7-f", "cortex-a9-mp")
+    .Cases("armv7s", "armv7-s", "swift")
     .Cases("armv7r", "armv7-r", "cortex-r4")
     .Cases("armv7m", "armv7-m", "cortex-m3")
     .Case("ep9312", "ep9312")
@@ -119,10 +141,12 @@ static const char *getLLVMArchSuffixForARM(StringRef CPU) {
     .Cases("arm1136j-s",  "arm1136jf-s",  "arm1176jz-s", "v6")
     .Cases("arm1176jzf-s",  "mpcorenovfp",  "mpcore", "v6")
     .Cases("arm1156t2-s",  "arm1156t2f-s", "v6t2")
-    .Cases("cortex-a8", "cortex-a9", "v7")
+    .Cases("cortex-a8", "cortex-a9", "cortex-a15", "v7")
     .Case("cortex-m3", "v7m")
     .Case("cortex-m4", "v7m")
     .Case("cortex-m0", "v6m")
+    .Case("cortex-a9-mp", "v7f")
+    .Case("swift", "v7s")
     .Default("");
 }
 
@@ -142,7 +166,7 @@ std::string ToolChain::ComputeLLVMTriple(const ArgList &Args,
     // FIXME: Thumb should just be another -target-feaure, not in the triple.
     StringRef Suffix =
       getLLVMArchSuffixForARM(getARMTargetCPU(Args, Triple));
-    bool ThumbDefault = (Suffix == "v7" && getTriple().isOSDarwin());
+    bool ThumbDefault = (Suffix.startswith("v7") && getTriple().isOSDarwin());
     std::string ArchName = "arm";
 
     // Assembly files should start in ARM mode.
@@ -180,7 +204,7 @@ ToolChain::RuntimeLibType ToolChain::GetRuntimeLibType(
   const ArgList &Args) const
 {
   if (Arg *A = Args.getLastArg(options::OPT_rtlib_EQ)) {
-    StringRef Value = A->getValue(Args);
+    StringRef Value = A->getValue();
     if (Value == "compiler-rt")
       return ToolChain::RLT_CompilerRT;
     if (Value == "libgcc")
@@ -194,7 +218,7 @@ ToolChain::RuntimeLibType ToolChain::GetRuntimeLibType(
 
 ToolChain::CXXStdlibType ToolChain::GetCXXStdlibType(const ArgList &Args) const{
   if (Arg *A = Args.getLastArg(options::OPT_stdlib_EQ)) {
-    StringRef Value = A->getValue(Args);
+    StringRef Value = A->getValue();
     if (Value == "libc++")
       return ToolChain::CST_Libcxx;
     if (Value == "libstdc++")
@@ -273,3 +297,24 @@ void ToolChain::AddCCKextLibArgs(const ArgList &Args,
                                  ArgStringList &CmdArgs) const {
   CmdArgs.push_back("-lcc_kext");
 }
+
+bool ToolChain::AddFastMathRuntimeIfAvailable(const ArgList &Args,
+                                              ArgStringList &CmdArgs) const {
+  // Check if -ffast-math or -funsafe-math is enabled.
+  Arg *A = Args.getLastArg(options::OPT_ffast_math,
+                           options::OPT_fno_fast_math,
+                           options::OPT_funsafe_math_optimizations,
+                           options::OPT_fno_unsafe_math_optimizations);
+
+  if (!A || A->getOption().getID() == options::OPT_fno_fast_math ||
+      A->getOption().getID() == options::OPT_fno_unsafe_math_optimizations)
+    return false;
+
+  // If crtfastmath.o exists add it to the arguments.
+  std::string Path = GetFilePath("crtfastmath.o");
+  if (Path == "crtfastmath.o") // Not found.
+    return false;
+
+  CmdArgs.push_back(Args.MakeArgString(Path));
+  return true;
+}
diff --git a/lib/Driver/ToolChains.cpp b/lib/Driver/ToolChains.cpp
index 01c6623..a2ccb35 100644
--- a/lib/Driver/ToolChains.cpp
+++ b/lib/Driver/ToolChains.cpp
@@ -31,6 +31,8 @@
 #include "llvm/Support/Path.h"
 #include "llvm/Support/system_error.h"
 
+#include "SanitizerArgs.h"
+
 #include <cstdlib> // ::getenv
 
 #include "clang/Config/config.h" // for GCC_INSTALL_PREFIX
@@ -80,17 +82,11 @@ bool Darwin::HasNativeLLVMSupport() const {
 
 /// Darwin provides an ARC runtime starting in MacOS X 10.7 and iOS 5.0.
 ObjCRuntime Darwin::getDefaultObjCRuntime(bool isNonFragile) const {
-  if (isTargetIPhoneOS()) {
+  if (isTargetIPhoneOS())
     return ObjCRuntime(ObjCRuntime::iOS, TargetVersion);
-  } else if (TargetSimulatorVersionFromDefines != VersionTuple()) {
-    return ObjCRuntime(ObjCRuntime::iOS, TargetSimulatorVersionFromDefines);
-  } else {
-    if (isNonFragile) {
-      return ObjCRuntime(ObjCRuntime::MacOSX, TargetVersion);
-    } else {
-      return ObjCRuntime(ObjCRuntime::FragileMacOSX, TargetVersion);
-    }
-  }
+  if (isNonFragile)
+    return ObjCRuntime(ObjCRuntime::MacOSX, TargetVersion);
+  return ObjCRuntime(ObjCRuntime::FragileMacOSX, TargetVersion);
 }
 
 /// Darwin provides a blocks runtime starting in MacOS X 10.6 and iOS 3.2.
@@ -111,6 +107,9 @@ static const char *GetArmArchForMArch(StringRef Value) {
     .Cases("armv7a", "armv7-a", "armv7")
     .Cases("armv7r", "armv7-r", "armv7")
     .Cases("armv7m", "armv7-m", "armv7")
+    .Cases("armv7f", "armv7-f", "armv7f")
+    .Cases("armv7k", "armv7-k", "armv7k")
+    .Cases("armv7s", "armv7-s", "armv7s")
     .Default(0);
 }
 
@@ -122,7 +121,10 @@ static const char *GetArmArchForMCpu(StringRef Value) {
     .Case("xscale", "xscale")
     .Cases("arm1136j-s", "arm1136jf-s", "arm1176jz-s",
            "arm1176jzf-s", "cortex-m0", "armv6")
-    .Cases("cortex-a8", "cortex-r4", "cortex-m3", "cortex-a9", "armv7")
+    .Cases("cortex-a8", "cortex-r4", "cortex-m3", "cortex-a9", "cortex-a15",
+           "armv7")
+    .Case("cortex-a9-mp", "armv7f")
+    .Case("swift", "armv7s")
     .Default(0);
 }
 
@@ -134,11 +136,11 @@ StringRef Darwin::getDarwinArchName(const ArgList &Args) const {
   case llvm::Triple::thumb:
   case llvm::Triple::arm: {
     if (const Arg *A = Args.getLastArg(options::OPT_march_EQ))
-      if (const char *Arch = GetArmArchForMArch(A->getValue(Args)))
+      if (const char *Arch = GetArmArchForMArch(A->getValue()))
         return Arch;
 
     if (const Arg *A = Args.getLastArg(options::OPT_mcpu_EQ))
-      if (const char *Arch = GetArmArchForMCpu(A->getValue(Args)))
+      if (const char *Arch = GetArmArchForMCpu(A->getValue()))
         return Arch;
 
     return "arm";
@@ -175,24 +177,11 @@ void Generic_ELF::anchor() {}
 Tool &Darwin::SelectTool(const Compilation &C, const JobAction &JA,
                          const ActionList &Inputs) const {
   Action::ActionClass Key = JA.getKind();
-  bool useClang = false;
 
   if (getDriver().ShouldUseClangCompiler(C, JA, getTriple())) {
-    useClang = true;
-    // Fallback to llvm-gcc for i386 kext compiles, we don't support that ABI.
-    if (!getDriver().shouldForceClangUse() &&
-        Inputs.size() == 1 &&
-        types::isCXX(Inputs[0]->getType()) &&
-        getTriple().isOSDarwin() &&
-        getTriple().getArch() == llvm::Triple::x86 &&
-        (C.getArgs().getLastArg(options::OPT_fapple_kext) ||
-         C.getArgs().getLastArg(options::OPT_mkernel)))
-      useClang = false;
-  }
-
-  // FIXME: This seems like a hacky way to choose clang frontend.
-  if (useClang)
+    // FIXME: This seems like a hacky way to choose clang frontend.
     Key = Action::AnalyzeJobClass;
+  }
 
   bool UseIntegratedAs = C.getArgs().hasFlag(options::OPT_integrated_as,
                                              options::OPT_no_integrated_as,
@@ -245,30 +234,6 @@ DarwinClang::DarwinClang(const Driver &D, const llvm::Triple& Triple)
   getProgramPaths().push_back(getDriver().getInstalledDir());
   if (getDriver().getInstalledDir() != getDriver().Dir)
     getProgramPaths().push_back(getDriver().Dir);
-
-  // For fallback, we need to know how to find the GCC cc1 executables, so we
-  // also add the GCC libexec paths. This is legacy code that can be removed
-  // once fallback is no longer useful.
-  AddGCCLibexecPath(DarwinVersion[0]);
-  AddGCCLibexecPath(DarwinVersion[0] - 2);
-  AddGCCLibexecPath(DarwinVersion[0] - 1);
-  AddGCCLibexecPath(DarwinVersion[0] + 1);
-  AddGCCLibexecPath(DarwinVersion[0] + 2);
-}
-
-void DarwinClang::AddGCCLibexecPath(unsigned darwinVersion) {
-  std::string ToolChainDir = "i686-apple-darwin";
-  ToolChainDir += llvm::utostr(darwinVersion);
-  ToolChainDir += "/4.2.1";
-
-  std::string Path = getDriver().Dir;
-  Path += "/../llvm-gcc-4.2/libexec/gcc/";
-  Path += ToolChainDir;
-  getProgramPaths().push_back(Path);
-
-  Path = "/usr/llvm-gcc-4.2/libexec/gcc/";
-  Path += ToolChainDir;
-  getProgramPaths().push_back(Path);
 }
 
 void DarwinClang::AddLinkARCArgs(const ArgList &Args,
@@ -287,9 +252,6 @@ void DarwinClang::AddLinkARCArgs(const ArgList &Args,
     s += "iphonesimulator";
   else if (isTargetIPhoneOS())
     s += "iphoneos";
-  // FIXME: Remove this once we depend fully on -mios-simulator-version-min.
-  else if (TargetSimulatorVersionFromDefines != VersionTuple())
-    s += "iphonesimulator";
   else
     s += "macosx";
   s += ".a";
@@ -320,13 +282,15 @@ void DarwinClang::AddLinkRuntimeLibArgs(const ArgList &Args,
     break;
   default:
     getDriver().Diag(diag::err_drv_unsupported_rtlib_for_platform)
-      << Args.getLastArg(options::OPT_rtlib_EQ)->getValue(Args) << "darwin";
+      << Args.getLastArg(options::OPT_rtlib_EQ)->getValue() << "darwin";
     return;
   }
 
   // Darwin doesn't support real static executables, don't link any runtime
   // libraries with -static.
-  if (Args.hasArg(options::OPT_static))
+  if (Args.hasArg(options::OPT_static) ||
+      Args.hasArg(options::OPT_fapple_kext) ||
+      Args.hasArg(options::OPT_mkernel))
     return;
 
   // Reject -static-libgcc for now, we can deal with this when and if someone
@@ -351,15 +315,16 @@ void DarwinClang::AddLinkRuntimeLibArgs(const ArgList &Args,
     }
   }
 
+  SanitizerArgs Sanitize(getDriver(), Args);
+
   // Add ASAN runtime library, if required. Dynamic libraries and bundles
   // should not be linked with the runtime library.
-  if (Args.hasFlag(options::OPT_faddress_sanitizer,
-                   options::OPT_fno_address_sanitizer, false)) {
+  if (Sanitize.needsAsanRt()) {
     if (Args.hasArg(options::OPT_dynamiclib) ||
         Args.hasArg(options::OPT_bundle)) return;
     if (isTargetIPhoneOS()) {
       getDriver().Diag(diag::err_drv_clang_unsupported_per_platform)
-        << "-faddress-sanitizer";
+        << "-fsanitize=address";
     } else {
       AddLinkRuntimeLib(Args, CmdArgs, "libclang_rt.asan_osx.a");
 
@@ -410,67 +375,28 @@ void DarwinClang::AddLinkRuntimeLibArgs(const ArgList &Args,
   }
 }
 
-static inline StringRef SimulatorVersionDefineName() {
-  return "__IPHONE_OS_VERSION_MIN_REQUIRED";
-}
-
-/// \brief Parse the simulator version define:
-/// __IPHONE_OS_VERSION_MIN_REQUIRED=([0-9])([0-9][0-9])([0-9][0-9])
-// and return the grouped values as integers, e.g:
-//   __IPHONE_OS_VERSION_MIN_REQUIRED=40201
-// will return Major=4, Minor=2, Micro=1.
-static bool GetVersionFromSimulatorDefine(StringRef define,
-                                          unsigned &Major, unsigned &Minor,
-                                          unsigned &Micro) {
-  assert(define.startswith(SimulatorVersionDefineName()));
-  StringRef name, version;
-  llvm::tie(name, version) = define.split('=');
-  if (version.empty())
-    return false;
-  std::string verstr = version.str();
-  char *end;
-  unsigned num = (unsigned) strtol(verstr.c_str(), &end, 10);
-  if (*end != '\0')
-    return false;
-  Major = num / 10000;
-  num = num % 10000;
-  Minor = num / 100;
-  Micro = num % 100;
-  return true;
-}
-
 void Darwin::AddDeploymentTarget(DerivedArgList &Args) const {
   const OptTable &Opts = getDriver().getOpts();
 
+  // Support allowing the SDKROOT environment variable used by xcrun and other
+  // Xcode tools to define the default sysroot, by making it the default for
+  // isysroot.
+  if (!Args.hasArg(options::OPT_isysroot)) {
+    if (char *env = ::getenv("SDKROOT")) {
+      // We only use this value as the default if it is an absolute path and
+      // exists.
+      if (llvm::sys::path::is_absolute(env) && llvm::sys::fs::exists(env)) {
+        Args.append(Args.MakeSeparateArg(
+                      0, Opts.getOption(options::OPT_isysroot), env));
+      }
+    }
+  }
+
   Arg *OSXVersion = Args.getLastArg(options::OPT_mmacosx_version_min_EQ);
   Arg *iOSVersion = Args.getLastArg(options::OPT_miphoneos_version_min_EQ);
   Arg *iOSSimVersion = Args.getLastArg(
     options::OPT_mios_simulator_version_min_EQ);
 
-  // FIXME: HACK! When compiling for the simulator we don't get a
-  // '-miphoneos-version-min' to help us know whether there is an ARC runtime
-  // or not; try to parse a __IPHONE_OS_VERSION_MIN_REQUIRED
-  // define passed in command-line.
-  if (!iOSVersion && !iOSSimVersion) {
-    for (arg_iterator it = Args.filtered_begin(options::OPT_D),
-           ie = Args.filtered_end(); it != ie; ++it) {
-      StringRef define = (*it)->getValue(Args);
-      if (define.startswith(SimulatorVersionDefineName())) {
-        unsigned Major = 0, Minor = 0, Micro = 0;
-        if (GetVersionFromSimulatorDefine(define, Major, Minor, Micro) &&
-            Major < 10 && Minor < 100 && Micro < 100) {
-          TargetSimulatorVersionFromDefines = VersionTuple(Major, Minor, Micro);
-        }
-        // When using the define to indicate the simulator, we force
-        // 10.6 macosx target.
-        const Option *O = Opts.getOption(options::OPT_mmacosx_version_min_EQ);
-        OSXVersion = Args.MakeJoinedArg(0, O, "10.6");
-        Args.append(OSXVersion);
-        break;
-      }
-    }
-  }
-
   if (OSXVersion && (iOSVersion || iOSSimVersion)) {
     getDriver().Diag(diag::err_drv_argument_not_allowed_with)
           << OSXVersion->getAsString(Args)
@@ -500,7 +426,7 @@ void Darwin::AddDeploymentTarget(DerivedArgList &Args) const {
     if (iOSTarget.empty()) {
       if (const Arg *A = Args.getLastArg(options::OPT_isysroot)) {
         StringRef first, second;
-        StringRef isysroot = A->getValue(Args);
+        StringRef isysroot = A->getValue();
         llvm::tie(first, second) = isysroot.split(StringRef("SDKs/iPhoneOS"));
         if (second != "")
           iOSTarget = second.substr(0,3);
@@ -510,7 +436,8 @@ void Darwin::AddDeploymentTarget(DerivedArgList &Args) const {
     // If no OSX or iOS target has been specified and we're compiling for armv7,
     // go ahead as assume we're targeting iOS.
     if (OSXTarget.empty() && iOSTarget.empty() &&
-        getDarwinArchName(Args) == "armv7")
+        (getDarwinArchName(Args) == "armv7" ||
+         getDarwinArchName(Args) == "armv7s"))
         iOSTarget = iOSVersionMin;
 
     // Handle conflicting deployment targets
@@ -536,21 +463,21 @@ void Darwin::AddDeploymentTarget(DerivedArgList &Args) const {
     }
 
     if (!OSXTarget.empty()) {
-      const Option *O = Opts.getOption(options::OPT_mmacosx_version_min_EQ);
+      const Option O = Opts.getOption(options::OPT_mmacosx_version_min_EQ);
       OSXVersion = Args.MakeJoinedArg(0, O, OSXTarget);
       Args.append(OSXVersion);
     } else if (!iOSTarget.empty()) {
-      const Option *O = Opts.getOption(options::OPT_miphoneos_version_min_EQ);
+      const Option O = Opts.getOption(options::OPT_miphoneos_version_min_EQ);
       iOSVersion = Args.MakeJoinedArg(0, O, iOSTarget);
       Args.append(iOSVersion);
     } else if (!iOSSimTarget.empty()) {
-      const Option *O = Opts.getOption(
+      const Option O = Opts.getOption(
         options::OPT_mios_simulator_version_min_EQ);
       iOSSimVersion = Args.MakeJoinedArg(0, O, iOSSimTarget);
       Args.append(iOSSimVersion);
     } else {
       // Otherwise, assume we are targeting OS X.
-      const Option *O = Opts.getOption(options::OPT_mmacosx_version_min_EQ);
+      const Option O = Opts.getOption(options::OPT_mmacosx_version_min_EQ);
       OSXVersion = Args.MakeJoinedArg(0, O, MacosxVersionMin);
       Args.append(OSXVersion);
     }
@@ -568,7 +495,7 @@ void Darwin::AddDeploymentTarget(DerivedArgList &Args) const {
   bool HadExtra;
   if (OSXVersion) {
     assert((!iOSVersion && !iOSSimVersion) && "Unknown target platform!");
-    if (!Driver::GetReleaseVersion(OSXVersion->getValue(Args), Major, Minor,
+    if (!Driver::GetReleaseVersion(OSXVersion->getValue(), Major, Minor,
                                    Micro, HadExtra) || HadExtra ||
         Major != 10 || Minor >= 100 || Micro >= 100)
       getDriver().Diag(diag::err_drv_invalid_version_number)
@@ -576,7 +503,7 @@ void Darwin::AddDeploymentTarget(DerivedArgList &Args) const {
   } else {
     const Arg *Version = iOSVersion ? iOSVersion : iOSSimVersion;
     assert(Version && "Unknown target platform!");
-    if (!Driver::GetReleaseVersion(Version->getValue(Args), Major, Minor,
+    if (!Driver::GetReleaseVersion(Version->getValue(), Major, Minor,
                                    Micro, HadExtra) || HadExtra ||
         Major >= 10 || Minor >= 100 || Micro >= 100)
       getDriver().Diag(diag::err_drv_invalid_version_number)
@@ -614,7 +541,7 @@ void DarwinClang::AddCXXStdlibLibArgs(const ArgList &Args,
     // Check in the sysroot first.
     bool Exists;
     if (const Arg *A = Args.getLastArg(options::OPT_isysroot)) {
-      llvm::sys::Path P(A->getValue(Args));
+      llvm::sys::Path P(A->getValue());
       P.appendComponent("usr");
       P.appendComponent("lib");
       P.appendComponent("libstdc++.dylib");
@@ -655,7 +582,14 @@ void DarwinClang::AddCCKextLibArgs(const ArgList &Args,
   llvm::sys::Path P(getDriver().ResourceDir);
   P.appendComponent("lib");
   P.appendComponent("darwin");
-  P.appendComponent("libclang_rt.cc_kext.a");
+
+  // Use the newer cc_kext for iOS ARM after 6.0.
+  if (!isTargetIPhoneOS() || isTargetIOSSimulator() ||
+      !isIPhoneOSVersionLT(6, 0)) {
+    P.appendComponent("libclang_rt.cc_kext.a");
+  } else {
+    P.appendComponent("libclang_rt.cc_kext_ios5.a");
+  }
 
   // For now, allow missing resource libraries to support developers who may
   // not have compiler-rt checked out or integrated into their build.
@@ -683,15 +617,15 @@ DerivedArgList *Darwin::TranslateArgs(const DerivedArgList &Args,
     if (A->getOption().matches(options::OPT_Xarch__)) {
       // Skip this argument unless the architecture matches either the toolchain
       // triple arch, or the arch being bound.
-      //
-      // FIXME: Canonicalize name.
-      StringRef XarchArch = A->getValue(Args, 0);
-      if (!(XarchArch == getArchName()  ||
-            (BoundArch && XarchArch == BoundArch)))
+      llvm::Triple::ArchType XarchArch =
+        tools::darwin::getArchTypeForDarwinArchName(A->getValue(0));
+      if (!(XarchArch == getArch()  ||
+            (BoundArch && XarchArch ==
+             tools::darwin::getArchTypeForDarwinArchName(BoundArch))))
         continue;
 
       Arg *OriginalArg = A;
-      unsigned Index = Args.getBaseArgs().MakeIndex(A->getValue(Args, 1));
+      unsigned Index = Args.getBaseArgs().MakeIndex(A->getValue(1));
       unsigned Prev = Index;
       Arg *XarchArg = Opts.ParseOneArg(Args, Index);
 
@@ -707,7 +641,7 @@ DerivedArgList *Darwin::TranslateArgs(const DerivedArgList &Args,
         getDriver().Diag(diag::err_drv_invalid_Xarch_argument_with_args)
           << A->getAsString(Args);
         continue;
-      } else if (XarchArg->getOption().isDriverOption()) {
+      } else if (XarchArg->getOption().hasFlag(options::DriverOption)) {
         getDriver().Diag(diag::err_drv_invalid_Xarch_argument_isdriver)
           << A->getAsString(Args);
         continue;
@@ -721,12 +655,12 @@ DerivedArgList *Darwin::TranslateArgs(const DerivedArgList &Args,
       // Linker input arguments require custom handling. The problem is that we
       // have already constructed the phase actions, so we can not treat them as
       // "input arguments".
-      if (A->getOption().isLinkerInput()) {
+      if (A->getOption().hasFlag(options::LinkerInput)) {
         // Convert the argument into individual Zlinker_input_args.
         for (unsigned i = 0, e = A->getNumValues(); i != e; ++i) {
           DAL->AddSeparateArg(OriginalArg,
                               Opts.getOption(options::OPT_Zlinker_input),
-                              A->getValue(Args, i));
+                              A->getValue(i));
 
         }
         continue;
@@ -749,7 +683,7 @@ DerivedArgList *Darwin::TranslateArgs(const DerivedArgList &Args,
 
     case options::OPT_dependency_file:
       DAL->AddSeparateArg(A, Opts.getOption(options::OPT_MF),
-                          A->getValue(Args));
+                          A->getValue());
       break;
 
     case options::OPT_gfull:
@@ -805,8 +739,8 @@ DerivedArgList *Darwin::TranslateArgs(const DerivedArgList &Args,
   // how the driver driver works.
   if (BoundArch) {
     StringRef Name = BoundArch;
-    const Option *MCpu = Opts.getOption(options::OPT_mcpu_EQ);
-    const Option *MArch = Opts.getOption(options::OPT_march_EQ);
+    const Option MCpu = Opts.getOption(options::OPT_mcpu_EQ);
+    const Option MArch = Opts.getOption(options::OPT_march_EQ);
 
     // This code must be kept in sync with LLVM's getArchTypeForDarwinArch,
     // which defines the list of which architectures we accept.
@@ -864,6 +798,12 @@ DerivedArgList *Darwin::TranslateArgs(const DerivedArgList &Args,
       DAL->AddJoinedArg(0, MArch, "armv6k");
     else if (Name == "armv7")
       DAL->AddJoinedArg(0, MArch, "armv7a");
+    else if (Name == "armv7f")
+      DAL->AddJoinedArg(0, MArch, "armv7f");
+    else if (Name == "armv7k")
+      DAL->AddJoinedArg(0, MArch, "armv7k");
+    else if (Name == "armv7s")
+      DAL->AddJoinedArg(0, MArch, "armv7s");
 
     else
       llvm_unreachable("invalid Darwin arch");
@@ -875,6 +815,25 @@ DerivedArgList *Darwin::TranslateArgs(const DerivedArgList &Args,
   if (BoundArch)
     AddDeploymentTarget(*DAL);
 
+  // For iOS 6, undo the translation to add -static for -mkernel/-fapple-kext.
+  // FIXME: It would be far better to avoid inserting those -static arguments,
+  // but we can't check the deployment target in the translation code until
+  // it is set here.
+  if (isTargetIPhoneOS() && !isIPhoneOSVersionLT(6, 0)) {
+    for (ArgList::iterator it = DAL->begin(), ie = DAL->end(); it != ie; ) {
+      Arg *A = *it;
+      ++it;
+      if (A->getOption().getID() != options::OPT_mkernel &&
+          A->getOption().getID() != options::OPT_fapple_kext)
+        continue;
+      assert(it != ie && "unexpected argument translation");
+      A = *it;
+      assert(A->getOption().getID() == options::OPT_static &&
+             "missing expected -static argument");
+      it = DAL->getArgs().erase(it);
+    }
+  }
+
   // Validate the C++ standard library choice.
   CXXStdlibType Type = GetCXXStdlibType(*DAL);
   if (Type == ToolChain::CST_Libcxx) {
@@ -882,13 +841,8 @@ DerivedArgList *Darwin::TranslateArgs(const DerivedArgList &Args,
     StringRef where;
 
     // Complain about targetting iOS < 5.0 in any way.
-    if (TargetSimulatorVersionFromDefines != VersionTuple()) {
-      if (TargetSimulatorVersionFromDefines < VersionTuple(5, 0))
-        where = "iOS 5.0";
-    } else if (isTargetIPhoneOS()) {
-      if (isIPhoneOSVersionLT(5, 0))
-        where = "iOS 5.0";
-    }
+    if (isTargetIPhoneOS() && isIPhoneOSVersionLT(5, 0))
+      where = "iOS 5.0";
 
     if (where != StringRef()) {
       getDriver().Diag(clang::diag::err_drv_invalid_libcxx_deployment)
@@ -900,9 +854,7 @@ DerivedArgList *Darwin::TranslateArgs(const DerivedArgList &Args,
 }
 
 bool Darwin::IsUnwindTablesDefault() const {
-  // FIXME: Gross; we should probably have some separate target
-  // definition, possibly even reusing the one in clang.
-  return getArchName() == "x86_64";
+  return getArch() == llvm::Triple::x86_64;
 }
 
 bool Darwin::UseDwarfDebugFlags() const {
@@ -917,19 +869,17 @@ bool Darwin::UseSjLjExceptions() const {
           getTriple().getArch() == llvm::Triple::thumb);
 }
 
-const char *Darwin::GetDefaultRelocationModel() const {
-  return "pic";
+bool Darwin::isPICDefault() const {
+  return true;
 }
 
-const char *Darwin::GetForcedPicModel() const {
-  if (getArchName() == "x86_64")
-    return "pic";
-  return 0;
+bool Darwin::isPICDefaultForced() const {
+  return getArch() == llvm::Triple::x86_64;
 }
 
 bool Darwin::SupportsProfiling() const {
   // Profiling instrumentation is only supported on x86.
-  return getArchName() == "i386" || getArchName() == "x86_64";
+  return getArch() == llvm::Triple::x86 || getArch() == llvm::Triple::x86_64;
 }
 
 bool Darwin::SupportsObjCGC() const {
@@ -937,8 +887,10 @@ bool Darwin::SupportsObjCGC() const {
   return !isTargetIPhoneOS();
 }
 
-bool Darwin::SupportsObjCARC() const {
-  return isTargetIPhoneOS() || !isMacosxVersionLT(10, 6);
+void Darwin::CheckObjCARC() const {
+  if (isTargetIPhoneOS() || !isMacosxVersionLT(10, 6))
+    return;
+  getDriver().Diag(diag::err_arc_unsupported_on_toolchain);
 }
 
 std::string
@@ -1013,7 +965,7 @@ bool Generic_GCC::GCCVersion::operator<(const GCCVersion &RHS) const {
 static StringRef getGCCToolchainDir(const ArgList &Args) {
   const Arg *A = Args.getLastArg(options::OPT_gcc_toolchain);
   if (A)
-    return A->getValue(Args);
+    return A->getValue();
   return GCC_INSTALL_PREFIX;
 }
 
@@ -1072,7 +1024,8 @@ Generic_GCC::GCCInstallationDetector::GCCInstallationDetector(
       if (!llvm::sys::fs::exists(LibDir))
         continue;
       for (unsigned k = 0, ke = CandidateTripleAliases.size(); k < ke; ++k)
-        ScanLibDirForGCCTriple(TargetArch, LibDir, CandidateTripleAliases[k]);
+        ScanLibDirForGCCTriple(TargetArch, Args, LibDir,
+                               CandidateTripleAliases[k]);
     }
     for (unsigned j = 0, je = CandidateMultiarchLibDirs.size(); j < je; ++j) {
       const std::string LibDir
@@ -1081,7 +1034,7 @@ Generic_GCC::GCCInstallationDetector::GCCInstallationDetector(
         continue;
       for (unsigned k = 0, ke = CandidateMultiarchTripleAliases.size(); k < ke;
            ++k)
-        ScanLibDirForGCCTriple(TargetArch, LibDir,
+        ScanLibDirForGCCTriple(TargetArch, Args, LibDir,
                                CandidateMultiarchTripleAliases[k],
                                /*NeedsMultiarchSuffix=*/true);
     }
@@ -1136,7 +1089,10 @@ Generic_GCC::GCCInstallationDetector::GCCInstallationDetector(
   static const char *const MIPSLibDirs[] = { "/lib" };
   static const char *const MIPSTriples[] = { "mips-linux-gnu" };
   static const char *const MIPSELLibDirs[] = { "/lib" };
-  static const char *const MIPSELTriples[] = { "mipsel-linux-gnu" };
+  static const char *const MIPSELTriples[] = {
+    "mipsel-linux-gnu",
+    "mipsel-linux-android"
+  };
 
   static const char *const MIPS64LibDirs[] = { "/lib64", "/lib" };
   static const char *const MIPS64Triples[] = { "mips64-linux-gnu" };
@@ -1264,8 +1220,32 @@ Generic_GCC::GCCInstallationDetector::GCCInstallationDetector(
     MultiarchTripleAliases.push_back(MultiarchTriple.str());
 }
 
+// FIXME: There is the same routine in the Tools.cpp.
+static bool hasMipsN32ABIArg(const ArgList &Args) {
+  Arg *A = Args.getLastArg(options::OPT_mabi_EQ);
+  return A && (A->getValue() == StringRef("n32"));
+}
+
+static StringRef getTargetMultiarchSuffix(llvm::Triple::ArchType TargetArch,
+                                          const ArgList &Args) {
+  if (TargetArch == llvm::Triple::x86_64 ||
+      TargetArch == llvm::Triple::ppc64)
+    return "/64";
+
+  if (TargetArch == llvm::Triple::mips64 ||
+      TargetArch == llvm::Triple::mips64el) {
+    if (hasMipsN32ABIArg(Args))
+      return "/n32";
+    else
+      return "/64";
+  }
+
+  return "/32";
+}
+
 void Generic_GCC::GCCInstallationDetector::ScanLibDirForGCCTriple(
-    llvm::Triple::ArchType TargetArch, const std::string &LibDir,
+    llvm::Triple::ArchType TargetArch, const ArgList &Args,
+    const std::string &LibDir,
     StringRef CandidateTriple, bool NeedsMultiarchSuffix) {
   // There are various different suffixes involving the triple we
   // check for. We also record what is necessary to walk from each back
@@ -1274,6 +1254,10 @@ void Generic_GCC::GCCInstallationDetector::ScanLibDirForGCCTriple(
     "/gcc/" + CandidateTriple.str(),
     "/" + CandidateTriple.str() + "/gcc/" + CandidateTriple.str(),
 
+    // The Freescale PPC SDK has the gcc libraries in
+    // <sysroot>/usr/lib/<triple>/x.y.z so have a look there as well.
+    "/" + CandidateTriple.str(),
+
     // Ubuntu has a strange mis-matched pair of triples that this happens to
     // match.
     // FIXME: It may be worthwhile to generalize this and look for a second
@@ -1283,6 +1267,7 @@ void Generic_GCC::GCCInstallationDetector::ScanLibDirForGCCTriple(
   const std::string InstallSuffixes[] = {
     "/../../..",
     "/../../../..",
+    "/../..",
     "/../../../.."
   };
   // Only look at the final, weird Ubuntu suffix for i386-linux-gnu.
@@ -1307,11 +1292,7 @@ void Generic_GCC::GCCInstallationDetector::ScanLibDirForGCCTriple(
       // *if* there is a subdirectory of the right name with crtbegin.o in it,
       // we use that. If not, and if not a multiarch triple, we look for
       // crtbegin.o without the subdirectory.
-      StringRef MultiarchSuffix
-        = (TargetArch == llvm::Triple::x86_64 ||
-           TargetArch == llvm::Triple::ppc64 ||
-           TargetArch == llvm::Triple::mips64 ||
-           TargetArch == llvm::Triple::mips64el) ? "/64" : "/32";
+      StringRef MultiarchSuffix = getTargetMultiarchSuffix(TargetArch, Args);
       if (llvm::sys::fs::exists(LI->path() + MultiarchSuffix + "/crtbegin.o")) {
         GCCMultiarchSuffix = MultiarchSuffix.str();
       } else {
@@ -1392,18 +1373,17 @@ Tool &Generic_GCC::SelectTool(const Compilation &C,
 }
 
 bool Generic_GCC::IsUnwindTablesDefault() const {
-  // FIXME: Gross; we should probably have some separate target
-  // definition, possibly even reusing the one in clang.
-  return getArchName() == "x86_64";
+  return getArch() == llvm::Triple::x86_64;
 }
 
-const char *Generic_GCC::GetDefaultRelocationModel() const {
-  return "static";
+bool Generic_GCC::isPICDefault() const {
+  return false;
 }
 
-const char *Generic_GCC::GetForcedPicModel() const {
-  return 0;
+bool Generic_GCC::isPICDefaultForced() const {
+  return false;
 }
+
 /// Hexagon Toolchain
 
 Hexagon_TC::Hexagon_TC(const Driver &D, const llvm::Triple& Triple)
@@ -1457,21 +1437,14 @@ Tool &Hexagon_TC::SelectTool(const Compilation &C,
   return *T;
 }
 
-bool Hexagon_TC::IsUnwindTablesDefault() const {
-  // FIXME: Gross; we should probably have some separate target
-  // definition, possibly even reusing the one in clang.
-  return getArchName() == "x86_64";
+bool Hexagon_TC::isPICDefault() const {
+  return false;
 }
 
-const char *Hexagon_TC::GetDefaultRelocationModel() const {
-  return "static";
+bool Hexagon_TC::isPICDefaultForced() const {
+  return false;
 }
 
-const char *Hexagon_TC::GetForcedPicModel() const {
-  return 0;
-} // End Hexagon
-
-
 /// TCEToolChain - A tool chain using the llvm bitcode tools to perform
 /// all subcommands. See http://tce.cs.tut.fi for our peculiar target.
 /// Currently does not support anything else but compilation.
@@ -1495,16 +1468,12 @@ bool TCEToolChain::IsMathErrnoDefault() const {
   return true;
 }
 
-bool TCEToolChain::IsUnwindTablesDefault() const {
+bool TCEToolChain::isPICDefault() const {
   return false;
 }
 
-const char *TCEToolChain::GetDefaultRelocationModel() const {
-  return "static";
-}
-
-const char *TCEToolChain::GetForcedPicModel() const {
-  return 0;
+bool TCEToolChain::isPICDefaultForced() const {
+  return false;
 }
 
 Tool &TCEToolChain::SelectTool(const Compilation &C,
@@ -1613,19 +1582,43 @@ void Bitrig::AddClangCXXStdlibIncludeArgs(const ArgList &DriverArgs,
       DriverArgs.hasArg(options::OPT_nostdincxx))
     return;
 
-  std::string Triple = getTriple().str();
-  if (Triple.substr(0, 5) == "amd64")
-    Triple.replace(0, 5, "x86_64");
-
-  addSystemInclude(DriverArgs, CC1Args, "/usr/include/c++/4.6.2");
-  addSystemInclude(DriverArgs, CC1Args, "/usr/include/c++/4.6.2/backward");
-  addSystemInclude(DriverArgs, CC1Args, "/usr/include/c++/4.6.2/" + Triple);
+  switch (GetCXXStdlibType(DriverArgs)) {
+  case ToolChain::CST_Libcxx:
+    addSystemInclude(DriverArgs, CC1Args,
+                     getDriver().SysRoot + "/usr/include/c++/");
+    break;
+  case ToolChain::CST_Libstdcxx:
+    addSystemInclude(DriverArgs, CC1Args,
+                     getDriver().SysRoot + "/usr/include/c++/stdc++");
+    addSystemInclude(DriverArgs, CC1Args,
+                     getDriver().SysRoot + "/usr/include/c++/stdc++/backward");
 
+    StringRef Triple = getTriple().str();
+    if (Triple.startswith("amd64"))
+      addSystemInclude(DriverArgs, CC1Args,
+                       getDriver().SysRoot + "/usr/include/c++/stdc++/x86_64" +
+                       Triple.substr(5));
+    else
+      addSystemInclude(DriverArgs, CC1Args,
+                       getDriver().SysRoot + "/usr/include/c++/stdc++/" +
+                       Triple);
+    break;
+  }
 }
 
 void Bitrig::AddCXXStdlibLibArgs(const ArgList &Args,
                                  ArgStringList &CmdArgs) const {
-  CmdArgs.push_back("-lstdc++");
+  switch (GetCXXStdlibType(Args)) {
+  case ToolChain::CST_Libcxx:
+    CmdArgs.push_back("-lc++");
+    CmdArgs.push_back("-lcxxrt");
+    // Include supc++ to provide Unwind until provided by libcxx.
+    CmdArgs.push_back("-lgcc");
+    break;
+  case ToolChain::CST_Libstdcxx:
+    CmdArgs.push_back("-lstdc++");
+    break;
+  }
 }
 
 /// FreeBSD - FreeBSD tool chain which can call as(1) and ld(1) directly.
@@ -2020,6 +2013,46 @@ static void addPathIfExists(Twine Path, ToolChain::path_list &Paths) {
   if (llvm::sys::fs::exists(Path)) Paths.push_back(Path.str());
 }
 
+static bool isMipsArch(llvm::Triple::ArchType Arch) {
+  return Arch == llvm::Triple::mips ||
+         Arch == llvm::Triple::mipsel ||
+         Arch == llvm::Triple::mips64 ||
+         Arch == llvm::Triple::mips64el;
+}
+
+static bool isMipsR2Arch(llvm::Triple::ArchType Arch,
+                         const ArgList &Args) {
+  if (Arch != llvm::Triple::mips &&
+      Arch != llvm::Triple::mipsel)
+    return false;
+
+  Arg *A = Args.getLastArg(options::OPT_march_EQ,
+                           options::OPT_mcpu_EQ,
+                           options::OPT_mips_CPUs_Group);
+
+  if (!A)
+    return false;
+
+  if (A->getOption().matches(options::OPT_mips_CPUs_Group))
+    return A->getOption().matches(options::OPT_mips32r2);
+
+  return A->getValue() == StringRef("mips32r2");
+}
+
+static StringRef getMultilibDir(const llvm::Triple &Triple,
+                                const ArgList &Args) {
+  if (!isMipsArch(Triple.getArch()))
+    return Triple.isArch32Bit() ? "lib32" : "lib64";
+
+  // lib32 directory has a special meaning on MIPS targets.
+  // It contains N32 ABI binaries. Use this folder if produce
+  // code for N32 ABI only.
+  if (hasMipsN32ABIArg(Args))
+    return "lib32";
+
+  return Triple.isArch32Bit() ? "lib" : "lib64";
+}
+
 Linux::Linux(const Driver &D, const llvm::Triple &Triple, const ArgList &Args)
   : Generic_ELF(D, Triple, Args) {
   llvm::Triple::ArchType Arch = Triple.getArch();
@@ -2043,19 +2076,14 @@ Linux::Linux(const Driver &D, const llvm::Triple &Triple, const ArgList &Args)
   if (Arch == llvm::Triple::arm || Arch == llvm::Triple::thumb)
     ExtraOpts.push_back("-X");
 
-  const bool IsMips = Arch == llvm::Triple::mips ||
-                      Arch == llvm::Triple::mipsel ||
-                      Arch == llvm::Triple::mips64 ||
-                      Arch == llvm::Triple::mips64el;
-
-  const bool IsAndroid = Triple.getEnvironment() == llvm::Triple::ANDROIDEABI;
+  const bool IsAndroid = Triple.getEnvironment() == llvm::Triple::Android;
 
   // Do not use 'gnu' hash style for Mips targets because .gnu.hash
   // and the MIPS ABI require .dynsym to be sorted in different ways.
   // .gnu.hash needs symbols to be grouped by hash code whereas the MIPS
   // ABI requires a mapping between the GOT and the symbol table.
   // Android loader does not support .gnu.hash.
-  if (!IsMips && !IsAndroid) {
+  if (!isMipsArch(Arch) && !IsAndroid) {
     if (IsRedhat(Distro) || IsOpenSuse(Distro) ||
         (IsUbuntu(Distro) && Distro >= UbuntuMaverick))
       ExtraOpts.push_back("--hash-style=gnu");
@@ -2084,16 +2112,23 @@ Linux::Linux(const Driver &D, const llvm::Triple &Triple, const ArgList &Args)
   // to the link paths.
   path_list &Paths = getFilePaths();
 
-  const std::string Multilib = Triple.isArch32Bit() ? "lib32" : "lib64";
+  const std::string Multilib = getMultilibDir(Triple, Args);
   const std::string MultiarchTriple = getMultiarchTriple(Triple, SysRoot);
 
   // Add the multilib suffixed paths where they are available.
   if (GCCInstallation.isValid()) {
     const llvm::Triple &GCCTriple = GCCInstallation.getTriple();
     const std::string &LibPath = GCCInstallation.getParentLibPath();
-    addPathIfExists((GCCInstallation.getInstallPath() +
-                     GCCInstallation.getMultiarchSuffix()),
-                    Paths);
+
+    if (IsAndroid && isMipsR2Arch(Triple.getArch(), Args))
+      addPathIfExists(GCCInstallation.getInstallPath() +
+                      GCCInstallation.getMultiarchSuffix() +
+                      "/mips-r2",
+                      Paths);
+    else
+      addPathIfExists((GCCInstallation.getInstallPath() +
+                       GCCInstallation.getMultiarchSuffix()),
+                      Paths);
 
     // If the GCC installation we found is inside of the sysroot, we want to
     // prefer libraries installed in the parent prefix of the GCC installation.
@@ -2108,6 +2143,11 @@ Linux::Linux(const Driver &D, const llvm::Triple &Triple, const ArgList &Args)
       addPathIfExists(LibPath + "/" + MultiarchTriple, Paths);
       addPathIfExists(LibPath + "/../" + Multilib, Paths);
     }
+    // On Android, libraries in the parent prefix of the GCC installation are
+    // preferred to the ones under sysroot.
+    if (IsAndroid) {
+      addPathIfExists(LibPath + "/../" + GCCTriple.str() + "/lib", Paths);
+    }
   }
   addPathIfExists(SysRoot + "/lib/" + MultiarchTriple, Paths);
   addPathIfExists(SysRoot + "/lib/../" + Multilib, Paths);
@@ -2326,16 +2366,25 @@ void Linux::AddClangCXXStdlibIncludeArgs(const ArgList &DriverArgs,
   StringRef LibDir = GCCInstallation.getParentLibPath();
   StringRef InstallDir = GCCInstallation.getInstallPath();
   StringRef Version = GCCInstallation.getVersion().Text;
-  if (!addLibStdCXXIncludePaths(LibDir + "/../include/c++/" + Version,
-                                (GCCInstallation.getTriple().str() +
-                                 GCCInstallation.getMultiarchSuffix()),
-                                DriverArgs, CC1Args)) {
+  StringRef TripleStr = GCCInstallation.getTriple().str();
+
+  const std::string IncludePathCandidates[] = {
+    LibDir.str() + "/../include/c++/" + Version.str(),
     // Gentoo is weird and places its headers inside the GCC install, so if the
     // first attempt to find the headers fails, try this pattern.
-    addLibStdCXXIncludePaths(InstallDir + "/include/g++-v4",
-                             (GCCInstallation.getTriple().str() +
-                              GCCInstallation.getMultiarchSuffix()),
-                             DriverArgs, CC1Args);
+    InstallDir.str() + "/include/g++-v4",
+    // Android standalone toolchain has C++ headers in yet another place.
+    LibDir.str() + "/../" + TripleStr.str() + "/include/c++/" + Version.str(),
+    // Freescale SDK C++ headers are directly in <sysroot>/usr/include/c++,
+    // without a subdirectory corresponding to the gcc version.
+    LibDir.str() + "/../include/c++",
+  };
+
+  for (unsigned i = 0; i < llvm::array_lengthof(IncludePathCandidates); ++i) {
+    if (addLibStdCXXIncludePaths(IncludePathCandidates[i], (TripleStr +
+                GCCInstallation.getMultiarchSuffix()),
+            DriverArgs, CC1Args))
+      break;
   }
 }
 
diff --git a/lib/Driver/ToolChains.h b/lib/Driver/ToolChains.h
index 95a11be..4c267e8 100644
--- a/lib/Driver/ToolChains.h
+++ b/lib/Driver/ToolChains.h
@@ -111,6 +111,7 @@ protected:
       SmallVectorImpl<StringRef> &MultiarchTripleAliases);
 
     void ScanLibDirForGCCTriple(llvm::Triple::ArchType TargetArch,
+                                const ArgList &Args,
                                 const std::string &LibDir,
                                 StringRef CandidateTriple,
                                 bool NeedsMultiarchSuffix = false);
@@ -128,8 +129,8 @@ public:
                            const ActionList &Inputs) const;
 
   virtual bool IsUnwindTablesDefault() const;
-  virtual const char *GetDefaultRelocationModel() const;
-  virtual const char *GetForcedPicModel() const;
+  virtual bool isPICDefault() const;
+  virtual bool isPICDefaultForced() const;
 
 protected:
   /// \name ToolChain Implementation Helper Functions
@@ -155,9 +156,8 @@ public:
   virtual Tool &SelectTool(const Compilation &C, const JobAction &JA,
                            const ActionList &Inputs) const;
 
-  virtual bool IsUnwindTablesDefault() const;
-  virtual const char *GetDefaultRelocationModel() const;
-  virtual const char *GetForcedPicModel() const;
+  virtual bool isPICDefault() const;
+  virtual bool isPICDefaultForced() const;
 };
 
   /// Darwin - The base Darwin tool chain.
@@ -185,11 +185,6 @@ private:
   /// The OS version we are targeting.
   mutable VersionTuple TargetVersion;
 
-protected:
-  // FIXME: Remove this once there is a proper way to detect an ARC runtime
-  // for the simulator.
-  mutable VersionTuple TargetSimulatorVersionFromDefines;
-
 private:
   /// The default macosx-version-min of this tool chain; empty until
   /// initialized.
@@ -243,9 +238,7 @@ public:
   }
 
   bool isTargetMacOS() const {
-    return !isTargetIOSSimulator() &&
-           !isTargetIPhoneOS() &&
-           TargetSimulatorVersionFromDefines == VersionTuple();
+    return !isTargetIOSSimulator() && !isTargetIPhoneOS();
   }
 
   bool isTargetInitialized() const { return TargetInitialized; }
@@ -325,6 +318,10 @@ public:
     return true;
   }
 
+  virtual bool IsEncodeExtendedBlockSignatureDefault() const {
+    return true;
+  }
+  
   virtual bool IsObjCNonFragileABIDefault() const {
     // Non-fragile ABI is default for everything but i386.
     return getTriple().getArch() != llvm::Triple::x86;
@@ -347,14 +344,14 @@ public:
   virtual RuntimeLibType GetDefaultRuntimeLibType() const {
     return ToolChain::RLT_CompilerRT;
   }
-  virtual const char *GetDefaultRelocationModel() const;
-  virtual const char *GetForcedPicModel() const;
+  virtual bool isPICDefault() const;
+  virtual bool isPICDefaultForced() const;
 
   virtual bool SupportsProfiling() const;
 
   virtual bool SupportsObjCGC() const;
 
-  virtual bool SupportsObjCARC() const;
+  virtual void CheckObjCARC() const;
 
   virtual bool UseDwarfDebugFlags() const;
 
@@ -365,9 +362,6 @@ public:
 
 /// DarwinClang - The Darwin toolchain used by Clang.
 class LLVM_LIBRARY_VISIBILITY DarwinClang : public Darwin {
-private:
-  void AddGCCLibexecPath(unsigned darwinVersion);
-
 public:
   DarwinClang(const Driver &D, const llvm::Triple& Triple);
 
@@ -399,7 +393,7 @@ public:
   std::string ComputeEffectiveClangTriple(const ArgList &Args,
                                           types::ID InputType) const;
 
-  virtual const char *GetDefaultRelocationModel() const { return "pic"; }
+  virtual bool isPICDefault() const { return false; };
 };
 
 class LLVM_LIBRARY_VISIBILITY Generic_ELF : public Generic_GCC {
@@ -540,9 +534,8 @@ public:
   virtual Tool &SelectTool(const Compilation &C, const JobAction &JA,
                            const ActionList &Inputs) const;
   bool IsMathErrnoDefault() const;
-  bool IsUnwindTablesDefault() const;
-  const char* GetDefaultRelocationModel() const;
-  const char* GetForcedPicModel() const;
+  bool isPICDefault() const;
+  bool isPICDefaultForced() const;
 
 private:
   mutable llvm::DenseMap<unsigned, Tool*> Tools;
@@ -564,8 +557,8 @@ public:
 
   virtual bool IsIntegratedAssemblerDefault() const;
   virtual bool IsUnwindTablesDefault() const;
-  virtual const char *GetDefaultRelocationModel() const;
-  virtual const char *GetForcedPicModel() const;
+  virtual bool isPICDefault() const;
+  virtual bool isPICDefaultForced() const;
 
   virtual void AddClangSystemIncludeArgs(const ArgList &DriverArgs,
                                          ArgStringList &CC1Args) const;
diff --git a/lib/Driver/Tools.cpp b/lib/Driver/Tools.cpp
index ed67f7b..e37959b 100644
--- a/lib/Driver/Tools.cpp
+++ b/lib/Driver/Tools.cpp
@@ -33,6 +33,7 @@
 #include "llvm/Support/ErrorHandling.h"
 
 #include "InputInfo.h"
+#include "SanitizerArgs.h"
 #include "ToolChains.h"
 
 using namespace clang::driver;
@@ -93,9 +94,15 @@ static void addDirectoryList(const ArgList &Args,
                              const char *ArgName,
                              const char *EnvVar) {
   const char *DirList = ::getenv(EnvVar);
+  bool CombinedArg = false;
+
   if (!DirList)
     return; // Nothing to do.
 
+  StringRef Name(ArgName);
+  if (Name.equals("-I") || Name.equals("-L"))
+    CombinedArg = true;
+
   StringRef Dirs(DirList);
   if (Dirs.empty()) // Empty string should not add '.'.
     return;
@@ -103,21 +110,37 @@ static void addDirectoryList(const ArgList &Args,
   StringRef::size_type Delim;
   while ((Delim = Dirs.find(llvm::sys::PathSeparator)) != StringRef::npos) {
     if (Delim == 0) { // Leading colon.
-      CmdArgs.push_back(ArgName);
-      CmdArgs.push_back(".");
+      if (CombinedArg) {
+        CmdArgs.push_back(Args.MakeArgString(std::string(ArgName) + "."));
+      } else {
+        CmdArgs.push_back(ArgName);
+        CmdArgs.push_back(".");
+      }
     } else {
-      CmdArgs.push_back(ArgName);
-      CmdArgs.push_back(Args.MakeArgString(Dirs.substr(0, Delim)));
+      if (CombinedArg) {
+        CmdArgs.push_back(Args.MakeArgString(std::string(ArgName) + Dirs.substr(0, Delim)));
+      } else {
+        CmdArgs.push_back(ArgName);
+        CmdArgs.push_back(Args.MakeArgString(Dirs.substr(0, Delim)));
+      }
     }
     Dirs = Dirs.substr(Delim + 1);
   }
 
   if (Dirs.empty()) { // Trailing colon.
-    CmdArgs.push_back(ArgName);
-    CmdArgs.push_back(".");
+    if (CombinedArg) {
+      CmdArgs.push_back(Args.MakeArgString(std::string(ArgName) + "."));
+    } else {
+      CmdArgs.push_back(ArgName);
+      CmdArgs.push_back(".");
+    }
   } else { // Add the last path.
-    CmdArgs.push_back(ArgName);
-    CmdArgs.push_back(Args.MakeArgString(Dirs));
+    if (CombinedArg) {
+      CmdArgs.push_back(Args.MakeArgString(std::string(ArgName) + Dirs));
+    } else {
+      CmdArgs.push_back(ArgName);
+      CmdArgs.push_back(Args.MakeArgString(Dirs));
+    }
   }
 }
 
@@ -200,6 +223,12 @@ static void addProfileRT(const ToolChain &TC, const ArgList &Args,
   CmdArgs.push_back(Args.MakeArgString(ProfileRT));
 }
 
+static bool forwardToGCC(const Option &O) {
+  return !O.hasFlag(options::NoForward) &&
+         !O.hasFlag(options::DriverOption) &&
+         !O.hasFlag(options::LinkerInput);
+}
+
 void Clang::AddPreprocessingOptions(Compilation &C,
                                     const Driver &D,
                                     const ArgList &Args,
@@ -219,11 +248,11 @@ void Clang::AddPreprocessingOptions(Compilation &C,
       (A = Args.getLastArg(options::OPT_MMD))) {
     // Determine the output location.
     const char *DepFile;
-    if (Output.getType() == types::TY_Dependencies) {
-      DepFile = Output.getFilename();
-    } else if (Arg *MF = Args.getLastArg(options::OPT_MF)) {
-      DepFile = MF->getValue(Args);
+    if (Arg *MF = Args.getLastArg(options::OPT_MF)) {
+      DepFile = MF->getValue();
       C.addFailureResultFile(DepFile);
+    } else if (Output.getType() == types::TY_Dependencies) {
+      DepFile = Output.getFilename();
     } else if (A->getOption().matches(options::OPT_M) ||
                A->getOption().matches(options::OPT_MM)) {
       DepFile = "-";
@@ -242,7 +271,7 @@ void Clang::AddPreprocessingOptions(Compilation &C,
       // when we are only generating a dependency file.
       Arg *OutputOpt = Args.getLastArg(options::OPT_o);
       if (OutputOpt && Output.getType() != types::TY_Dependencies) {
-        DepTarget = OutputOpt->getValue(Args);
+        DepTarget = OutputOpt->getValue();
       } else {
         // Otherwise derive from the base input.
         //
@@ -282,7 +311,7 @@ void Clang::AddPreprocessingOptions(Compilation &C,
     if (A->getOption().matches(options::OPT_MQ)) {
       CmdArgs.push_back("-MT");
       SmallString<128> Quoted;
-      QuoteTarget(A->getValue(Args), Quoted);
+      QuoteTarget(A->getValue(), Quoted);
       CmdArgs.push_back(Args.MakeArgString(Quoted));
 
     // -MT flag - no change
@@ -310,7 +339,7 @@ void Clang::AddPreprocessingOptions(Compilation &C,
 
       bool FoundPTH = false;
       bool FoundPCH = false;
-      llvm::sys::Path P(A->getValue(Args));
+      llvm::sys::Path P(A->getValue());
       bool Exists;
       if (UsePCH) {
         P.appendSuffix("pch");
@@ -442,10 +471,12 @@ static const char *getLLVMArchSuffixForARM(StringRef CPU) {
     .Cases("arm1136j-s",  "arm1136jf-s",  "arm1176jz-s", "v6")
     .Cases("arm1176jzf-s",  "mpcorenovfp",  "mpcore", "v6")
     .Cases("arm1156t2-s",  "arm1156t2f-s", "v6t2")
-    .Cases("cortex-a8", "cortex-a9", "v7")
+    .Cases("cortex-a8", "cortex-a9", "cortex-a15", "v7")
     .Case("cortex-m3", "v7m")
     .Case("cortex-m4", "v7m")
     .Case("cortex-m0", "v6m")
+    .Case("cortex-a9-mp", "v7f")
+    .Case("swift", "v7s")
     .Default("");
 }
 
@@ -458,7 +489,7 @@ static std::string getARMTargetCPU(const ArgList &Args,
 
   // If we have -mcpu=, use that.
   if (Arg *A = Args.getLastArg(options::OPT_mcpu_EQ)) {
-    StringRef MCPU = A->getValue(Args);
+    StringRef MCPU = A->getValue();
     // Handle -mcpu=native.
     if (MCPU == "native")
       return llvm::sys::getHostCPUName();
@@ -469,7 +500,7 @@ static std::string getARMTargetCPU(const ArgList &Args,
   StringRef MArch;
   if (Arg *A = Args.getLastArg(options::OPT_march_EQ)) {
     // Otherwise, if we have -march= choose the base CPU for that arch.
-    MArch = A->getValue(Args);
+    MArch = A->getValue();
   } else {
     // Otherwise, use the Arch from the triple.
     MArch = Triple.getArchName();
@@ -500,6 +531,8 @@ static std::string getARMTargetCPU(const ArgList &Args,
     .Cases("armv6z", "armv6zk", "arm1176jzf-s")
     .Case("armv6t2", "arm1156t2-s")
     .Cases("armv7", "armv7a", "armv7-a", "cortex-a8")
+    .Cases("armv7f", "armv7-f", "cortex-a9-mp")
+    .Cases("armv7s", "armv7-s", "swift")
     .Cases("armv7r", "armv7-r", "cortex-r4")
     .Cases("armv7m", "armv7-m", "cortex-m3")
     .Case("ep9312", "ep9312")
@@ -531,7 +564,7 @@ static bool isSignedCharDefault(const llvm::Triple &Triple) {
 // frontend target.
 static void addFPUArgs(const Driver &D, const Arg *A, const ArgList &Args,
                        ArgStringList &CmdArgs) {
-  StringRef FPU = A->getValue(Args);
+  StringRef FPU = A->getValue();
 
   // Set the target features based on the FPU.
   if (FPU == "fpa" || FPU == "fpe2" || FPU == "fpe3" || FPU == "maverick") {
@@ -569,14 +602,15 @@ static void addFPUArgs(const Driver &D, const Arg *A, const ArgList &Args,
 // Handle -mfpmath=.
 static void addFPMathArgs(const Driver &D, const Arg *A, const ArgList &Args,
                           ArgStringList &CmdArgs, StringRef CPU) {
-  StringRef FPMath = A->getValue(Args);
+  StringRef FPMath = A->getValue();
   
   // Set the target features based on the FPMath.
   if (FPMath == "neon") {
     CmdArgs.push_back("-target-feature");
     CmdArgs.push_back("+neonfp");
     
-    if (CPU != "cortex-a8" && CPU != "cortex-a9" && CPU != "cortex-a9-mp")    
+    if (CPU != "cortex-a8" && CPU != "cortex-a9" && CPU != "cortex-a9-mp" &&
+        CPU != "cortex-a15")
       D.Diag(diag::err_drv_invalid_feature) << "-mfpmath=neon" << CPU;
     
   } else if (FPMath == "vfp" || FPMath == "vfp2" || FPMath == "vfp3" ||
@@ -603,7 +637,7 @@ static StringRef getARMFloatABI(const Driver &D,
     else if (A->getOption().matches(options::OPT_mhard_float))
       FloatABI = "hard";
     else {
-      FloatABI = A->getValue(Args);
+      FloatABI = A->getValue();
       if (FloatABI != "soft" && FloatABI != "softfp" && FloatABI != "hard") {
         D.Diag(diag::err_drv_invalid_mfloat_abi)
           << A->getAsString(Args);
@@ -643,7 +677,7 @@ static StringRef getARMFloatABI(const Driver &D,
         // EABI is always AAPCS, and if it was not marked 'hard', it's softfp
         FloatABI = "softfp";
         break;
-      case llvm::Triple::ANDROIDEABI: {
+      case llvm::Triple::Android: {
         std::string ArchName =
           getLLVMArchSuffixForARM(getARMTargetCPU(Args, Triple));
         if (StringRef(ArchName).startswith("v7"))
@@ -669,18 +703,29 @@ void Clang::AddARMTargetArgs(const ArgList &Args,
                              ArgStringList &CmdArgs,
                              bool KernelOrKext) const {
   const Driver &D = getToolChain().getDriver();
-  llvm::Triple Triple = getToolChain().getTriple();
+  // Get the effective triple, which takes into account the deployment target.
+  std::string TripleStr = getToolChain().ComputeEffectiveClangTriple(Args);
+  llvm::Triple Triple(TripleStr);
+  std::string CPUName = getARMTargetCPU(Args, Triple);
 
   // Select the ABI to use.
   //
   // FIXME: Support -meabi.
   const char *ABIName = 0;
   if (Arg *A = Args.getLastArg(options::OPT_mabi_EQ)) {
-    ABIName = A->getValue(Args);
+    ABIName = A->getValue();
+  } else if (Triple.isOSDarwin()) {
+    // The backend is hardwired to assume AAPCS for M-class processors, ensure
+    // the frontend matches that.
+    if (StringRef(CPUName).startswith("cortex-m")) {
+      ABIName = "aapcs";
+    } else {
+      ABIName = "apcs-gnu";
+    }
   } else {
     // Select the default based on the platform.
     switch(Triple.getEnvironment()) {
-    case llvm::Triple::ANDROIDEABI:
+    case llvm::Triple::Android:
     case llvm::Triple::GNUEABI:
     case llvm::Triple::GNUEABIHF:
       ABIName = "aapcs-linux";
@@ -697,7 +742,7 @@ void Clang::AddARMTargetArgs(const ArgList &Args,
 
   // Set the CPU based on -march= and -mcpu=.
   CmdArgs.push_back("-target-cpu");
-  CmdArgs.push_back(Args.MakeArgString(getARMTargetCPU(Args, Triple)));
+  CmdArgs.push_back(Args.MakeArgString(CPUName));
 
   // Determine floating point ABI from the options & target defaults.
   StringRef FloatABI = getARMFloatABI(D, Args, Triple);
@@ -754,8 +799,10 @@ void Clang::AddARMTargetArgs(const ArgList &Args,
 
   // Kernel code has more strict alignment requirements.
   if (KernelOrKext) {
-    CmdArgs.push_back("-backend-option");
-    CmdArgs.push_back("-arm-long-calls");
+    if (Triple.getOS() != llvm::Triple::IOS || Triple.isOSVersionLT(6)) {
+      CmdArgs.push_back("-backend-option");
+      CmdArgs.push_back("-arm-long-calls");
+    }
 
     CmdArgs.push_back("-backend-option");
     CmdArgs.push_back("-arm-strict-align");
@@ -777,44 +824,18 @@ void Clang::AddARMTargetArgs(const ArgList &Args,
     CmdArgs.push_back("-no-implicit-float");
 }
 
-// Get default architecture.
-static const char* getMipsArchFromCPU(StringRef CPUName) {
-  if (CPUName == "mips32" || CPUName == "mips32r2")
-    return "mips";
-
-  assert((CPUName == "mips64" || CPUName == "mips64r2") &&
-         "Unexpected cpu name.");
-
-  return "mips64";
-}
-
-// Check that ArchName is a known Mips architecture name.
-static bool checkMipsArchName(StringRef ArchName) {
-  return ArchName == "mips" ||
-         ArchName == "mipsel" ||
-         ArchName == "mips64" ||
-         ArchName == "mips64el";
-}
-
-// Get default target cpu.
-static const char* getMipsCPUFromArch(StringRef ArchName) {
-  if (ArchName == "mips" || ArchName == "mipsel")
+// Translate MIPS CPU name alias option to CPU name.
+static StringRef getMipsCPUFromAlias(const Arg &A) {
+  if (A.getOption().matches(options::OPT_mips32))
     return "mips32";
-
-  assert((ArchName == "mips64" || ArchName == "mips64el") &&
-         "Unexpected arch name.");
-
-  return "mips64";
-}
-
-// Get default ABI.
-static const char* getMipsABIFromArch(StringRef ArchName) {
-    if (ArchName == "mips" || ArchName == "mipsel")
-      return "o32";
-    
-    assert((ArchName == "mips64" || ArchName == "mips64el") &&
-           "Unexpected arch name.");
-    return "n64";
+  if (A.getOption().matches(options::OPT_mips32r2))
+    return "mips32r2";
+  if (A.getOption().matches(options::OPT_mips64))
+    return "mips64";
+  if (A.getOption().matches(options::OPT_mips64r2))
+    return "mips64r2";
+  llvm_unreachable("Unexpected option");
+  return "";
 }
 
 // Get CPU and ABI names. They are not independent
@@ -823,26 +844,53 @@ static void getMipsCPUAndABI(const ArgList &Args,
                              const ToolChain &TC,
                              StringRef &CPUName,
                              StringRef &ABIName) {
-  StringRef ArchName;
-
-  // Select target cpu and architecture.
-  if (Arg *A = Args.getLastArg(options::OPT_mcpu_EQ)) {
-    CPUName = A->getValue(Args);
-    ArchName = getMipsArchFromCPU(CPUName);
-  }
-  else {
-    ArchName = Args.MakeArgString(TC.getArchName());
-    if (!checkMipsArchName(ArchName))
-      TC.getDriver().Diag(diag::err_drv_invalid_arch_name) << ArchName;
+  const char *DefMips32CPU = "mips32";
+  const char *DefMips64CPU = "mips64";
+
+  if (Arg *A = Args.getLastArg(options::OPT_march_EQ,
+                               options::OPT_mcpu_EQ,
+                               options::OPT_mips_CPUs_Group)) {
+    if (A->getOption().matches(options::OPT_mips_CPUs_Group))
+      CPUName = getMipsCPUFromAlias(*A);
     else
-      CPUName = getMipsCPUFromArch(ArchName);
+      CPUName = A->getValue();
   }
- 
-  // Select the ABI to use.
+
   if (Arg *A = Args.getLastArg(options::OPT_mabi_EQ))
-    ABIName = A->getValue(Args);
-  else 
-    ABIName = getMipsABIFromArch(ArchName);
+    ABIName = A->getValue();
+
+  // Setup default CPU and ABI names.
+  if (CPUName.empty() && ABIName.empty()) {
+    switch (TC.getTriple().getArch()) {
+    default:
+      llvm_unreachable("Unexpected triple arch name");
+    case llvm::Triple::mips:
+    case llvm::Triple::mipsel:
+      CPUName = DefMips32CPU;
+      break;
+    case llvm::Triple::mips64:
+    case llvm::Triple::mips64el:
+      CPUName = DefMips64CPU;
+      break;
+    }
+  }
+
+  if (!ABIName.empty()) {
+    // Deduce CPU name from ABI name.
+    CPUName = llvm::StringSwitch<const char *>(ABIName)
+      .Cases("o32", "eabi", DefMips32CPU)
+      .Cases("n32", "n64", DefMips64CPU)
+      .Default("");
+  }
+  else if (!CPUName.empty()) {
+    // Deduce ABI name from CPU name.
+    ABIName = llvm::StringSwitch<const char *>(CPUName)
+      .Cases("mips32", "mips32r2", "o32")
+      .Cases("mips64", "mips64r2", "n64")
+      .Default("");
+  }
+
+  // FIXME: Warn on inconsistent cpu and abi usage.
 }
 
 // Select the MIPS float ABI as determined by -msoft-float, -mhard-float,
@@ -859,7 +907,7 @@ static StringRef getMipsFloatABI(const Driver &D, const ArgList &Args) {
     else if (A->getOption().matches(options::OPT_mhard_float))
       FloatABI = "hard";
     else {
-      FloatABI = A->getValue(Args);
+      FloatABI = A->getValue();
       if (FloatABI != "soft" && FloatABI != "single" && FloatABI != "hard") {
         D.Diag(diag::err_drv_invalid_mfloat_abi) << A->getAsString(Args);
         FloatABI = "hard";
@@ -941,12 +989,19 @@ void Clang::AddMIPSTargetArgs(const ArgList &Args,
   AddTargetFeature(Args, CmdArgs,
                    options::OPT_mdspr2, options::OPT_mno_dspr2,
                    "dspr2");
+
+  if (Arg *A = Args.getLastArg(options::OPT_G)) {
+    StringRef v = A->getValue();
+    CmdArgs.push_back("-mllvm");
+    CmdArgs.push_back(Args.MakeArgString("-mips-ssection-threshold=" + v));
+    A->claim();
+  }
 }
 
 /// getPPCTargetCPU - Get the (LLVM) name of the PowerPC cpu we are targeting.
 static std::string getPPCTargetCPU(const ArgList &Args) {
   if (Arg *A = Args.getLastArg(options::OPT_mcpu_EQ)) {
-    StringRef CPUName = A->getValue(Args);
+    StringRef CPUName = A->getValue();
 
     if (CPUName == "native") {
       std::string CPU = llvm::sys::getHostCPUName();
@@ -978,6 +1033,8 @@ static std::string getPPCTargetCPU(const ArgList &Args) {
       .Case("970", "970")
       .Case("G5", "g5")
       .Case("a2", "a2")
+      .Case("e500mc", "e500mc")
+      .Case("e5500", "e5500")
       .Case("power6", "pwr6")
       .Case("power7", "pwr7")
       .Case("powerpc", "ppc")
@@ -1015,7 +1072,7 @@ void Clang::AddSparcTargetArgs(const ArgList &Args,
 
   if (const Arg *A = Args.getLastArg(options::OPT_march_EQ)) {
     CmdArgs.push_back("-target-cpu");
-    CmdArgs.push_back(A->getValue(Args));
+    CmdArgs.push_back(A->getValue());
   }
 
   // Select the float ABI as determined by -msoft-float, -mhard-float, and
@@ -1054,6 +1111,8 @@ void Clang::AddSparcTargetArgs(const ArgList &Args,
 
 void Clang::AddX86TargetArgs(const ArgList &Args,
                              ArgStringList &CmdArgs) const {
+  const bool isAndroid =
+    getToolChain().getTriple().getEnvironment() == llvm::Triple::Android;
   if (!Args.hasFlag(options::OPT_mred_zone,
                     options::OPT_mno_red_zone,
                     true) ||
@@ -1068,7 +1127,7 @@ void Clang::AddX86TargetArgs(const ArgList &Args,
 
   const char *CPUName = 0;
   if (const Arg *A = Args.getLastArg(options::OPT_march_EQ)) {
-    if (StringRef(A->getValue(Args)) == "native") {
+    if (StringRef(A->getValue()) == "native") {
       // FIXME: Reject attempts to use -march=native unless the target matches
       // the host.
       //
@@ -1078,7 +1137,7 @@ void Clang::AddX86TargetArgs(const ArgList &Args,
       if (!CPU.empty() && CPU != "generic")
         CPUName = Args.MakeArgString(CPU);
     } else
-      CPUName = A->getValue(Args);
+      CPUName = A->getValue();
   }
 
   // Select the default CPU if none was given (or detection failed).
@@ -1118,7 +1177,9 @@ void Clang::AddX86TargetArgs(const ArgList &Args,
       if (getToolChain().getArch() == llvm::Triple::x86_64)
         CPUName = "x86-64";
       else if (getToolChain().getArch() == llvm::Triple::x86)
-        CPUName = "pentium4";
+        // All x86 devices running Android have core2 as their common
+        // denominator. This makes a better choice than pentium4.
+        CPUName = isAndroid ? "core2" : "pentium4";
     }
   }
 
@@ -1141,8 +1202,8 @@ void Clang::AddX86TargetArgs(const ArgList &Args,
     (*it)->claim();
 
     // Skip over "-m".
-    assert(Name.startswith("-m") && "Invalid feature name.");
-    Name = Name.substr(2);
+    assert(Name.startswith("m") && "Invalid feature name.");
+    Name = Name.substr(1);
 
     bool IsNegative = Name.startswith("no-");
     if (IsNegative)
@@ -1174,7 +1235,7 @@ static Arg* getLastHexagonArchArg (const ArgList &Args)
       A->claim();
     }
     else if ((*it)->getOption().matches(options::OPT_m_Joined)){
-      StringRef Value = (*it)->getValue(Args,0);
+      StringRef Value = (*it)->getValue(0);
       if (Value.startswith("v")) {
         A = *it;
         A->claim();
@@ -1191,7 +1252,7 @@ static StringRef getHexagonTargetCPU(const ArgList &Args)
 
   // Select the default CPU (v4) if none was given or detection failed.
   if ((A = getLastHexagonArchArg (Args))) {
-    WhichHexagon = A->getValue(Args);
+    WhichHexagon = A->getValue();
     if (WhichHexagon == "")
       return "v4";
     else
@@ -1216,7 +1277,7 @@ void Clang::AddHexagonTargetArgs(const ArgList &Args,
   if (Arg *A = Args.getLastArg(options::OPT_G,
                                options::OPT_msmall_data_threshold_EQ)) {
     std::string SmallDataThreshold="-small-data-threshold=";
-    SmallDataThreshold += A->getValue(Args);
+    SmallDataThreshold += A->getValue();
     CmdArgs.push_back ("-mllvm");
     CmdArgs.push_back(Args.MakeArgString(SmallDataThreshold));
     A->claim();
@@ -1392,25 +1453,80 @@ static bool UseRelaxAll(Compilation &C, const ArgList &Args) {
     RelaxDefault);
 }
 
+SanitizerArgs::SanitizerArgs(const Driver &D, const ArgList &Args) {
+  Kind = 0;
+
+  const Arg *AsanArg, *TsanArg, *UbsanArg;
+  for (ArgList::const_iterator I = Args.begin(), E = Args.end(); I != E; ++I) {
+    unsigned Add = 0, Remove = 0;
+    const char *DeprecatedReplacement = 0;
+    if ((*I)->getOption().matches(options::OPT_faddress_sanitizer)) {
+      Add = Address;
+      DeprecatedReplacement = "-fsanitize=address";
+    } else if ((*I)->getOption().matches(options::OPT_fno_address_sanitizer)) {
+      Remove = Address;
+      DeprecatedReplacement = "-fno-sanitize=address";
+    } else if ((*I)->getOption().matches(options::OPT_fthread_sanitizer)) {
+      Add = Thread;
+      DeprecatedReplacement = "-fsanitize=thread";
+    } else if ((*I)->getOption().matches(options::OPT_fno_thread_sanitizer)) {
+      Remove = Thread;
+      DeprecatedReplacement = "-fno-sanitize=thread";
+    } else if ((*I)->getOption().matches(options::OPT_fcatch_undefined_behavior)) {
+      Add = Undefined;
+      DeprecatedReplacement = "-fsanitize=undefined";
+    } else if ((*I)->getOption().matches(options::OPT_fsanitize_EQ)) {
+      Add = parse(D, *I);
+    } else if ((*I)->getOption().matches(options::OPT_fno_sanitize_EQ)) {
+      Remove = parse(D, *I);
+    } else {
+      continue;
+    }
+
+    (*I)->claim();
+
+    Kind |= Add;
+    Kind &= ~Remove;
+
+    if (Add & NeedsAsanRt) AsanArg = *I;
+    if (Add & NeedsTsanRt) TsanArg = *I;
+    if (Add & NeedsUbsanRt) UbsanArg = *I;
+
+    // If this is a deprecated synonym, produce a warning directing users
+    // towards the new spelling.
+    if (DeprecatedReplacement)
+      D.Diag(diag::warn_drv_deprecated_arg)
+        << (*I)->getAsString(Args) << DeprecatedReplacement;
+  }
+
+  // Only one runtime library can be used at once.
+  // FIXME: Allow Ubsan to be combined with the other two.
+  bool NeedsAsan = needsAsanRt();
+  bool NeedsTsan = needsTsanRt();
+  bool NeedsUbsan = needsUbsanRt();
+  if (NeedsAsan + NeedsTsan + NeedsUbsan > 1)
+    D.Diag(diag::err_drv_argument_not_allowed_with)
+      << describeSanitizeArg(Args, NeedsAsan ? AsanArg : TsanArg,
+                             NeedsAsan ? NeedsAsanRt : NeedsTsanRt)
+      << describeSanitizeArg(Args, NeedsUbsan ? UbsanArg : TsanArg,
+                             NeedsUbsan ? NeedsUbsanRt : NeedsTsanRt);
+}
+
 /// If AddressSanitizer is enabled, add appropriate linker flags (Linux).
 /// This needs to be called before we add the C run-time (malloc, etc).
 static void addAsanRTLinux(const ToolChain &TC, const ArgList &Args,
                            ArgStringList &CmdArgs) {
-  if (!Args.hasFlag(options::OPT_faddress_sanitizer,
-                    options::OPT_fno_address_sanitizer, false))
-    return;
-  if(TC.getTriple().getEnvironment() == llvm::Triple::ANDROIDEABI) {
+  if(TC.getTriple().getEnvironment() == llvm::Triple::Android) {
     if (!Args.hasArg(options::OPT_shared)) {
       if (!Args.hasArg(options::OPT_pie))
         TC.getDriver().Diag(diag::err_drv_asan_android_requires_pie);
-      // For an executable, we add a .preinit_array stub.
-      CmdArgs.push_back("-u");
-      CmdArgs.push_back("__asan_preinit");
-      CmdArgs.push_back("-lasan");
     }
 
-    CmdArgs.push_back("-lasan_preload");
-    CmdArgs.push_back("-ldl");
+    SmallString<128> LibAsan(TC.getDriver().ResourceDir);
+    llvm::sys::path::append(LibAsan, "lib", "linux",
+        (Twine("libclang_rt.asan-") +
+            TC.getArchName() + "-android.so"));
+    CmdArgs.push_back(Args.MakeArgString(LibAsan));
   } else {
     if (!Args.hasArg(options::OPT_shared)) {
       // LibAsan is "libclang_rt.asan-<ArchName>.a" in the Linux library
@@ -1431,9 +1547,6 @@ static void addAsanRTLinux(const ToolChain &TC, const ArgList &Args,
 /// This needs to be called before we add the C run-time (malloc, etc).
 static void addTsanRTLinux(const ToolChain &TC, const ArgList &Args,
                            ArgStringList &CmdArgs) {
-  if (!Args.hasFlag(options::OPT_fthread_sanitizer,
-                    options::OPT_fno_thread_sanitizer, false))
-    return;
   if (!Args.hasArg(options::OPT_shared)) {
     // LibTsan is "libclang_rt.tsan-<ArchName>.a" in the Linux library
     // resource directory.
@@ -1448,6 +1561,22 @@ static void addTsanRTLinux(const ToolChain &TC, const ArgList &Args,
   }
 }
 
+/// If UndefinedBehaviorSanitizer is enabled, add appropriate linker flags
+/// (Linux).
+static void addUbsanRTLinux(const ToolChain &TC, const ArgList &Args,
+                            ArgStringList &CmdArgs) {
+  if (!Args.hasArg(options::OPT_shared)) {
+    // LibUbsan is "libclang_rt.ubsan-<ArchName>.a" in the Linux library
+    // resource directory.
+    SmallString<128> LibUbsan(TC.getDriver().ResourceDir);
+    llvm::sys::path::append(LibUbsan, "lib", "linux",
+                            (Twine("libclang_rt.ubsan-") +
+                             TC.getArchName() + ".a"));
+    CmdArgs.push_back(Args.MakeArgString(LibUbsan));
+    CmdArgs.push_back("-lpthread");
+  }
+}
+
 static bool shouldUseFramePointer(const ArgList &Args,
                                   const llvm::Triple &Triple) {
   if (Arg *A = Args.getLastArg(options::OPT_fno_omit_frame_pointer,
@@ -1516,7 +1645,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
       A->claim();
 
       for (unsigned i = 0, e = A->getNumValues(); i != e; ++i) {
-        StringRef Value = A->getValue(Args, i);
+        StringRef Value = A->getValue(i);
 
         if (Value == "-force_cpusubtype_ALL") {
           // Do nothing, this is the default and we don't support anything else.
@@ -1600,8 +1729,6 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
     CmdArgs.push_back("-analyzer-eagerly-assume");
 
-    CmdArgs.push_back("-analyzer-ipa=inlining");
-
     // Add default argument set.
     if (!Args.hasArg(options::OPT__analyzer_no_default_checks)) {
       CmdArgs.push_back("-analyzer-checker=core");
@@ -1627,7 +1754,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     // reasons.
     CmdArgs.push_back("-analyzer-output");
     if (Arg *A = Args.getLastArg(options::OPT__analyzer_output))
-      CmdArgs.push_back(A->getValue(Args));
+      CmdArgs.push_back(A->getValue());
     else
       CmdArgs.push_back("plist");
 
@@ -1642,67 +1769,90 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
   CheckCodeGenerationOptions(D, Args);
 
-  // Perform argument translation for LLVM backend. This
-  // takes some care in reconciling with llvm-gcc. The
-  // issue is that llvm-gcc translates these options based on
-  // the values in cc1, whereas we are processing based on
-  // the driver arguments.
-
-  // This comes from the default translation the driver + cc1
-  // would do to enable flag_pic.
-
-  Arg *LastPICArg = Args.getLastArg(options::OPT_fPIC, options::OPT_fno_PIC,
-                                    options::OPT_fpic, options::OPT_fno_pic,
-                                    options::OPT_fPIE, options::OPT_fno_PIE,
-                                    options::OPT_fpie, options::OPT_fno_pie);
-  bool PICDisabled = false;
-  bool PICEnabled = false;
-  bool PICForPIE = false;
-  if (LastPICArg) {
-    PICForPIE = (LastPICArg->getOption().matches(options::OPT_fPIE) ||
-                 LastPICArg->getOption().matches(options::OPT_fpie));
-    PICEnabled = (PICForPIE ||
-                  LastPICArg->getOption().matches(options::OPT_fPIC) ||
-                  LastPICArg->getOption().matches(options::OPT_fpic));
-    PICDisabled = !PICEnabled;
+  // For the PIC and PIE flag options, this logic is different from the legacy
+  // logic in very old versions of GCC, as that logic was just a bug no one had
+  // ever fixed. This logic is both more rational and consistent with GCC's new
+  // logic now that the bugs are fixed. The last argument relating to either
+  // PIC or PIE wins, and no other argument is used. If the last argument is
+  // any flavor of the '-fno-...' arguments, both PIC and PIE are disabled. Any
+  // PIE option implicitly enables PIC at the same level.
+  bool PIE = false;
+  bool PIC = getToolChain().isPICDefault();
+  bool IsPICLevelTwo = PIC;
+  if (Arg *A = Args.getLastArg(options::OPT_fPIC, options::OPT_fno_PIC,
+                               options::OPT_fpic, options::OPT_fno_pic,
+                               options::OPT_fPIE, options::OPT_fno_PIE,
+                               options::OPT_fpie, options::OPT_fno_pie)) {
+    Option O = A->getOption();
+    if (O.matches(options::OPT_fPIC) || O.matches(options::OPT_fpic) ||
+        O.matches(options::OPT_fPIE) || O.matches(options::OPT_fpie)) {
+      PIE = O.matches(options::OPT_fPIE) || O.matches(options::OPT_fpie);
+      PIC = PIE || O.matches(options::OPT_fPIC) || O.matches(options::OPT_fpic);
+      IsPICLevelTwo = O.matches(options::OPT_fPIE) ||
+                      O.matches(options::OPT_fPIC);
+    } else {
+      PIE = PIC = false;
+    }
   }
+  // Check whether the tool chain trumps the PIC-ness decision. If the PIC-ness
+  // is forced, then neither PIC nor PIE flags will have no effect.
+  if (getToolChain().isPICDefaultForced()) {
+    PIE = false;
+    PIC = getToolChain().isPICDefault();
+    IsPICLevelTwo = PIC;
+  }
+
+  // Inroduce a Darwin-specific hack. If the default is PIC but the flags
+  // specified while enabling PIC enabled level 1 PIC, just force it back to
+  // level 2 PIC instead. This matches the behavior of Darwin GCC (based on my
+  // informal testing).
+  if (PIC && getToolChain().getTriple().isOSDarwin())
+    IsPICLevelTwo |= getToolChain().isPICDefault();
+
   // Note that these flags are trump-cards. Regardless of the order w.r.t. the
   // PIC or PIE options above, if these show up, PIC is disabled.
-  if (Args.hasArg(options::OPT_mkernel))
-    PICDisabled = true;
+  llvm::Triple Triple(TripleStr);
+  if ((Args.hasArg(options::OPT_mkernel) ||
+       Args.hasArg(options::OPT_fapple_kext)) &&
+      (Triple.getOS() != llvm::Triple::IOS ||
+       Triple.isOSVersionLT(6)))
+    PIC = PIE = false;
   if (Args.hasArg(options::OPT_static))
-    PICDisabled = true;
-  bool DynamicNoPIC = Args.hasArg(options::OPT_mdynamic_no_pic);
-
-  // Select the relocation model.
-  const char *Model = getToolChain().GetForcedPicModel();
-  if (!Model) {
-    if (DynamicNoPIC)
-      Model = "dynamic-no-pic";
-    else if (PICDisabled)
-      Model = "static";
-    else if (PICEnabled)
-      Model = "pic";
-    else
-      Model = getToolChain().GetDefaultRelocationModel();
-  }
-  StringRef ModelStr = Model ? Model : "";
-  if (Model && ModelStr != "pic") {
-    CmdArgs.push_back("-mrelocation-model");
-    CmdArgs.push_back(Model);
-  }
+    PIC = PIE = false;
 
-  // Infer the __PIC__ and __PIE__ values.
-  if (ModelStr == "pic" && PICForPIE) {
-    CmdArgs.push_back("-pie-level");
-    CmdArgs.push_back((LastPICArg &&
-                       LastPICArg->getOption().matches(options::OPT_fPIE)) ?
-                      "2" : "1");
-  } else if (ModelStr == "pic" || ModelStr == "dynamic-no-pic") {
-    CmdArgs.push_back("-pic-level");
-    CmdArgs.push_back(((ModelStr != "dynamic-no-pic" && LastPICArg &&
-                        LastPICArg->getOption().matches(options::OPT_fPIC)) ||
-                       getToolChain().getTriple().isOSDarwin()) ? "2" : "1");
+  if (Arg *A = Args.getLastArg(options::OPT_mdynamic_no_pic)) {
+    // This is a very special mode. It trumps the other modes, almost no one
+    // uses it, and it isn't even valid on any OS but Darwin.
+    if (!getToolChain().getTriple().isOSDarwin())
+      D.Diag(diag::err_drv_unsupported_opt_for_target)
+        << A->getSpelling() << getToolChain().getTriple().str();
+
+    // FIXME: Warn when this flag trumps some other PIC or PIE flag.
+
+    CmdArgs.push_back("-mrelocation-model");
+    CmdArgs.push_back("dynamic-no-pic");
+
+    // Only a forced PIC mode can cause the actual compile to have PIC defines
+    // etc., no flags are sufficient. This behavior was selected to closely
+    // match that of llvm-gcc and Apple GCC before that.
+    if (getToolChain().isPICDefault() && getToolChain().isPICDefaultForced()) {
+      CmdArgs.push_back("-pic-level");
+      CmdArgs.push_back("2");
+    }
+  } else {
+    // Currently, LLVM only knows about PIC vs. static; the PIE differences are
+    // handled in Clang's IRGen by the -pie-level flag.
+    CmdArgs.push_back("-mrelocation-model");
+    CmdArgs.push_back(PIC ? "pic" : "static");
+
+    if (PIC) {
+      CmdArgs.push_back("-pic-level");
+      CmdArgs.push_back(IsPICLevelTwo ? "2" : "1");
+      if (PIE) {
+        CmdArgs.push_back("-pie-level");
+        CmdArgs.push_back(IsPICLevelTwo ? "2" : "1");
+      }
+    }
   }
 
   if (!Args.hasFlag(options::OPT_fmerge_all_constants,
@@ -1713,7 +1863,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
   if (Arg *A = Args.getLastArg(options::OPT_mregparm_EQ)) {
     CmdArgs.push_back("-mregparm");
-    CmdArgs.push_back(A->getValue(Args));
+    CmdArgs.push_back(A->getValue());
   }
 
   if (Args.hasFlag(options::OPT_mrtd, options::OPT_mno_rtd, false))
@@ -1741,25 +1891,30 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   // flag disables them after the flag enabling them, enable the codegen
   // optimization. This is complicated by several "umbrella" flags.
   if (Arg *A = Args.getLastArg(options::OPT_ffast_math,
+                               options::OPT_fno_fast_math,
                                options::OPT_ffinite_math_only,
                                options::OPT_fno_finite_math_only,
                                options::OPT_fhonor_infinities,
                                options::OPT_fno_honor_infinities))
-    if (A->getOption().getID() != options::OPT_fno_finite_math_only &&
+    if (A->getOption().getID() != options::OPT_fno_fast_math &&
+        A->getOption().getID() != options::OPT_fno_finite_math_only &&
         A->getOption().getID() != options::OPT_fhonor_infinities)
       CmdArgs.push_back("-menable-no-infs");
   if (Arg *A = Args.getLastArg(options::OPT_ffast_math,
+                               options::OPT_fno_fast_math,
                                options::OPT_ffinite_math_only,
                                options::OPT_fno_finite_math_only,
                                options::OPT_fhonor_nans,
                                options::OPT_fno_honor_nans))
-    if (A->getOption().getID() != options::OPT_fno_finite_math_only &&
+    if (A->getOption().getID() != options::OPT_fno_fast_math &&
+        A->getOption().getID() != options::OPT_fno_finite_math_only &&
         A->getOption().getID() != options::OPT_fhonor_nans)
       CmdArgs.push_back("-menable-no-nans");
 
   // -fmath-errno is the default on some platforms, e.g. BSD-derived OSes.
   bool MathErrno = getToolChain().IsMathErrnoDefault();
   if (Arg *A = Args.getLastArg(options::OPT_ffast_math,
+                               options::OPT_fno_fast_math,
                                options::OPT_fmath_errno,
                                options::OPT_fno_math_errno))
     MathErrno = A->getOption().getID() == options::OPT_fmath_errno;
@@ -1772,38 +1927,46 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   // madness.
   bool AssociativeMath = false;
   if (Arg *A = Args.getLastArg(options::OPT_ffast_math,
+                               options::OPT_fno_fast_math,
                                options::OPT_funsafe_math_optimizations,
                                options::OPT_fno_unsafe_math_optimizations,
                                options::OPT_fassociative_math,
                                options::OPT_fno_associative_math))
-    if (A->getOption().getID() != options::OPT_fno_unsafe_math_optimizations &&
+    if (A->getOption().getID() != options::OPT_fno_fast_math &&
+        A->getOption().getID() != options::OPT_fno_unsafe_math_optimizations &&
         A->getOption().getID() != options::OPT_fno_associative_math)
       AssociativeMath = true;
   bool ReciprocalMath = false;
   if (Arg *A = Args.getLastArg(options::OPT_ffast_math,
+                               options::OPT_fno_fast_math,
                                options::OPT_funsafe_math_optimizations,
                                options::OPT_fno_unsafe_math_optimizations,
                                options::OPT_freciprocal_math,
                                options::OPT_fno_reciprocal_math))
-    if (A->getOption().getID() != options::OPT_fno_unsafe_math_optimizations &&
+    if (A->getOption().getID() != options::OPT_fno_fast_math &&
+        A->getOption().getID() != options::OPT_fno_unsafe_math_optimizations &&
         A->getOption().getID() != options::OPT_fno_reciprocal_math)
       ReciprocalMath = true;
   bool SignedZeros = true;
   if (Arg *A = Args.getLastArg(options::OPT_ffast_math,
+                               options::OPT_fno_fast_math,
                                options::OPT_funsafe_math_optimizations,
                                options::OPT_fno_unsafe_math_optimizations,
                                options::OPT_fsigned_zeros,
                                options::OPT_fno_signed_zeros))
-    if (A->getOption().getID() != options::OPT_fno_unsafe_math_optimizations &&
+    if (A->getOption().getID() != options::OPT_fno_fast_math &&
+        A->getOption().getID() != options::OPT_fno_unsafe_math_optimizations &&
         A->getOption().getID() != options::OPT_fsigned_zeros)
       SignedZeros = false;
   bool TrappingMath = true;
   if (Arg *A = Args.getLastArg(options::OPT_ffast_math,
+                               options::OPT_fno_fast_math,
                                options::OPT_funsafe_math_optimizations,
                                options::OPT_fno_unsafe_math_optimizations,
                                options::OPT_ftrapping_math,
                                options::OPT_fno_trapping_math))
-    if (A->getOption().getID() != options::OPT_fno_unsafe_math_optimizations &&
+    if (A->getOption().getID() != options::OPT_fno_fast_math &&
+        A->getOption().getID() != options::OPT_fno_unsafe_math_optimizations &&
         A->getOption().getID() != options::OPT_ftrapping_math)
       TrappingMath = false;
   if (!MathErrno && AssociativeMath && ReciprocalMath && !SignedZeros &&
@@ -1813,16 +1976,17 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
   // Validate and pass through -fp-contract option. 
   if (Arg *A = Args.getLastArg(options::OPT_ffast_math,
+                               options::OPT_fno_fast_math,
                                options::OPT_ffp_contract)) {
     if (A->getOption().getID() == options::OPT_ffp_contract) {
-      StringRef Val = A->getValue(Args);
+      StringRef Val = A->getValue();
       if (Val == "fast" || Val == "on" || Val == "off") {
         CmdArgs.push_back(Args.MakeArgString("-ffp-contract=" + Val));
       } else {
         D.Diag(diag::err_drv_unsupported_option_argument)
           << A->getOption().getName() << Val;
       }
-    } else { // A is OPT_ffast_math
+    } else if (A->getOption().getID() == options::OPT_ffast_math) {
       // If fast-math is set then set the fp-contract mode to fast.
       CmdArgs.push_back(Args.MakeArgString("-ffp-contract=fast"));
     }
@@ -1833,10 +1997,12 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   // preprocessor macros. This is distinct from enabling any optimizations as
   // these options induce language changes which must survive serialization
   // and deserialization, etc.
-  if (Args.hasArg(options::OPT_ffast_math))
-    CmdArgs.push_back("-ffast-math");
-  if (Args.hasArg(options::OPT_ffinite_math_only))
-    CmdArgs.push_back("-ffinite-math-only");
+  if (Arg *A = Args.getLastArg(options::OPT_ffast_math, options::OPT_fno_fast_math))
+    if (A->getOption().matches(options::OPT_ffast_math))
+      CmdArgs.push_back("-ffast-math");
+  if (Arg *A = Args.getLastArg(options::OPT_ffinite_math_only, options::OPT_fno_fast_math))
+    if (A->getOption().matches(options::OPT_ffinite_math_only))
+      CmdArgs.push_back("-ffinite-math-only");
 
   // Decide whether to use verbose asm. Verbose assembly is the default on
   // toolchains which have the integrated assembler on by default.
@@ -1885,7 +2051,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
   if (Arg *A = Args.getLastArg(options::OPT_flimited_precision_EQ)) {
     CmdArgs.push_back("-mlimit-float-precision");
-    CmdArgs.push_back(A->getValue(Args));
+    CmdArgs.push_back(A->getValue());
   }
 
   // FIXME: Handle -mtune=.
@@ -1893,7 +2059,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
   if (Arg *A = Args.getLastArg(options::OPT_mcmodel_EQ)) {
     CmdArgs.push_back("-mcode-model");
-    CmdArgs.push_back(A->getValue(Args));
+    CmdArgs.push_back(A->getValue());
   }
 
   // Add target specific cpu and features flags.
@@ -1937,7 +2103,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   // Pass the linker version in use.
   if (Arg *A = Args.getLastArg(options::OPT_mlinker_version_EQ)) {
     CmdArgs.push_back("-target-linker-version");
-    CmdArgs.push_back(A->getValue(Args));
+    CmdArgs.push_back(A->getValue());
   }
 
   // -mno-omit-leaf-frame-pointer is the default on Darwin.
@@ -1991,6 +2157,8 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
   // We ignore flags -gstrict-dwarf and -grecord-gcc-switches for now.
   Args.ClaimAllArgs(options::OPT_g_flags_Group);
+  if (Args.hasArg(options::OPT_gcolumn_info))
+    CmdArgs.push_back("-dwarf-column-info");
 
   Args.AddAllArgs(CmdArgs, options::OPT_ffunction_sections);
   Args.AddAllArgs(CmdArgs, options::OPT_fdata_sections);
@@ -2008,7 +2176,9 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
       C.getArgs().hasArg(options::OPT_S)) {
     if (Output.isFilename()) {
       CmdArgs.push_back("-coverage-file");
-      CmdArgs.push_back(Args.MakeArgString(Output.getFilename()));
+      SmallString<128> absFilename(Output.getFilename());
+      llvm::sys::fs::make_absolute(absFilename);
+      CmdArgs.push_back(Args.MakeArgString(absFilename));
     }
   }
 
@@ -2047,7 +2217,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
       case options::OPT_ccc_arcmt_migrate:
         CmdArgs.push_back("-arcmt-migrate");
         CmdArgs.push_back("-mt-migrate-directory");
-        CmdArgs.push_back(A->getValue(Args));
+        CmdArgs.push_back(A->getValue());
 
         Args.AddLastArg(CmdArgs, options::OPT_arcmt_migrate_report_output);
         Args.AddLastArg(CmdArgs, options::OPT_arcmt_migrate_emit_arc_errors);
@@ -2062,7 +2232,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
         << A->getAsString(Args) << "-ccc-arcmt-migrate";
     }
     CmdArgs.push_back("-mt-migrate-directory");
-    CmdArgs.push_back(A->getValue(Args));
+    CmdArgs.push_back(A->getValue());
 
     if (!Args.hasArg(options::OPT_objcmt_migrate_literals,
                      options::OPT_objcmt_migrate_subscripting)) {
@@ -2094,7 +2264,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     if (A->getOption().matches(options::OPT_O4))
       CmdArgs.push_back("-O3");
     else if (A->getOption().matches(options::OPT_O) &&
-             A->getValue(Args)[0] == '\0')
+             A->getValue()[0] == '\0')
       CmdArgs.push_back("-O2");
     else
       A->render(Args, CmdArgs);
@@ -2132,8 +2302,11 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     // eventually we want to do all the standard defaulting here instead of
     // splitting it between the driver and clang -cc1.
     if (!types::isCXX(InputType))
-        Args.AddAllArgsTranslated(CmdArgs, options::OPT_std_default_EQ,
-                                  "-std=", /*Joined=*/true);
+      Args.AddAllArgsTranslated(CmdArgs, options::OPT_std_default_EQ,
+                                "-std=", /*Joined=*/true);
+    else if (getToolChain().getTriple().getOS() == llvm::Triple::Win32)
+      CmdArgs.push_back("-std=c++11");
+
     Args.AddLastArg(CmdArgs, options::OPT_trigraphs);
   }
 
@@ -2182,18 +2355,18 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   if (Arg *A = Args.getLastArg(options::OPT_ftemplate_depth_,
                                options::OPT_ftemplate_depth_EQ)) {
     CmdArgs.push_back("-ftemplate-depth");
-    CmdArgs.push_back(A->getValue(Args));
+    CmdArgs.push_back(A->getValue());
   }
 
   if (Arg *A = Args.getLastArg(options::OPT_fconstexpr_depth_EQ)) {
     CmdArgs.push_back("-fconstexpr-depth");
-    CmdArgs.push_back(A->getValue(Args));
+    CmdArgs.push_back(A->getValue());
   }
 
   if (Arg *A = Args.getLastArg(options::OPT_Wlarge_by_value_copy_EQ,
                                options::OPT_Wlarge_by_value_copy_def)) {
     if (A->getNumValues()) {
-      StringRef bytes = A->getValue(Args);
+      StringRef bytes = A->getValue();
       CmdArgs.push_back(Args.MakeArgString("-Wlarge-by-value-copy=" + bytes));
     } else
       CmdArgs.push_back("-Wlarge-by-value-copy=64"); // default value
@@ -2202,50 +2375,50 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   if (Arg *A = Args.getLastArg(options::OPT_fbounds_checking,
                                options::OPT_fbounds_checking_EQ)) {
     if (A->getNumValues()) {
-      StringRef val = A->getValue(Args);
+      StringRef val = A->getValue();
       CmdArgs.push_back(Args.MakeArgString("-fbounds-checking=" + val));
     } else
       CmdArgs.push_back("-fbounds-checking=1");
   }
 
-  if (Args.hasArg(options::OPT__relocatable_pch))
+  if (Args.hasArg(options::OPT_relocatable_pch))
     CmdArgs.push_back("-relocatable-pch");
 
   if (Arg *A = Args.getLastArg(options::OPT_fconstant_string_class_EQ)) {
     CmdArgs.push_back("-fconstant-string-class");
-    CmdArgs.push_back(A->getValue(Args));
+    CmdArgs.push_back(A->getValue());
   }
 
   if (Arg *A = Args.getLastArg(options::OPT_ftabstop_EQ)) {
     CmdArgs.push_back("-ftabstop");
-    CmdArgs.push_back(A->getValue(Args));
+    CmdArgs.push_back(A->getValue());
   }
 
   CmdArgs.push_back("-ferror-limit");
   if (Arg *A = Args.getLastArg(options::OPT_ferror_limit_EQ))
-    CmdArgs.push_back(A->getValue(Args));
+    CmdArgs.push_back(A->getValue());
   else
     CmdArgs.push_back("19");
 
   if (Arg *A = Args.getLastArg(options::OPT_fmacro_backtrace_limit_EQ)) {
     CmdArgs.push_back("-fmacro-backtrace-limit");
-    CmdArgs.push_back(A->getValue(Args));
+    CmdArgs.push_back(A->getValue());
   }
 
   if (Arg *A = Args.getLastArg(options::OPT_ftemplate_backtrace_limit_EQ)) {
     CmdArgs.push_back("-ftemplate-backtrace-limit");
-    CmdArgs.push_back(A->getValue(Args));
+    CmdArgs.push_back(A->getValue());
   }
 
   if (Arg *A = Args.getLastArg(options::OPT_fconstexpr_backtrace_limit_EQ)) {
     CmdArgs.push_back("-fconstexpr-backtrace-limit");
-    CmdArgs.push_back(A->getValue(Args));
+    CmdArgs.push_back(A->getValue());
   }
 
   // Pass -fmessage-length=.
   CmdArgs.push_back("-fmessage-length");
   if (Arg *A = Args.getLastArg(options::OPT_fmessage_length_EQ)) {
-    CmdArgs.push_back(A->getValue(Args));
+    CmdArgs.push_back(A->getValue());
   } else {
     // If -fmessage-length=N was not specified, determine whether this is a
     // terminal and, if so, implicitly define -fmessage-length appropriately.
@@ -2255,7 +2428,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
   if (const Arg *A = Args.getLastArg(options::OPT_fvisibility_EQ)) {
     CmdArgs.push_back("-fvisibility");
-    CmdArgs.push_back(A->getValue(Args));
+    CmdArgs.push_back(A->getValue());
   }
 
   Args.AddLastArg(CmdArgs, options::OPT_fvisibility_inlines_hidden);
@@ -2268,7 +2441,6 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back("-ffreestanding");
 
   // Forward -f (flag) options which we can pass directly.
-  Args.AddLastArg(CmdArgs, options::OPT_fcatch_undefined_behavior);
   Args.AddLastArg(CmdArgs, options::OPT_femit_all_decls);
   Args.AddLastArg(CmdArgs, options::OPT_fheinous_gnu_extensions);
   Args.AddLastArg(CmdArgs, options::OPT_flimit_debug_info);
@@ -2278,6 +2450,9 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   Args.AddLastArg(CmdArgs, options::OPT_fdiagnostics_show_template_tree);
   Args.AddLastArg(CmdArgs, options::OPT_fno_elide_type);
 
+  SanitizerArgs Sanitize(D, Args);
+  Sanitize.addArgs(Args, CmdArgs);
+
   // Report and error for -faltivec on anything other then PowerPC.
   if (const Arg *A = Args.getLastArg(options::OPT_faltivec))
     if (!(getToolChain().getTriple().getArch() == llvm::Triple::ppc ||
@@ -2288,14 +2463,6 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   if (getToolChain().SupportsProfiling())
     Args.AddLastArg(CmdArgs, options::OPT_pg);
 
-  if (Args.hasFlag(options::OPT_faddress_sanitizer,
-                   options::OPT_fno_address_sanitizer, false))
-    CmdArgs.push_back("-faddress-sanitizer");
-
-  if (Args.hasFlag(options::OPT_fthread_sanitizer,
-                   options::OPT_fno_thread_sanitizer, false))
-    CmdArgs.push_back("-fthread-sanitizer");
-
   // -flax-vector-conversions is default.
   if (!Args.hasFlag(options::OPT_flax_vector_conversions,
                     options::OPT_fno_lax_vector_conversions))
@@ -2316,7 +2483,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
   if (Arg *A = Args.getLastArg(options::OPT_ftrapv_handler_EQ)) {
     CmdArgs.push_back("-ftrapv-handler");
-    CmdArgs.push_back(A->getValue(Args));
+    CmdArgs.push_back(A->getValue());
   }
 
   Args.AddLastArg(CmdArgs, options::OPT_ftrap_function_EQ);
@@ -2337,6 +2504,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
   Args.AddLastArg(CmdArgs, options::OPT_pthread);
 
+
   // -stack-protector=0 is default.
   unsigned StackProtectorLevel = 0;
   if (Arg *A = Args.getLastArg(options::OPT_fno_stack_protector,
@@ -2355,6 +2523,20 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back(Args.MakeArgString(Twine(StackProtectorLevel)));
   }
 
+  // --param ssp-buffer-size=
+  for (arg_iterator it = Args.filtered_begin(options::OPT__param),
+       ie = Args.filtered_end(); it != ie; ++it) {
+    StringRef Str((*it)->getValue());
+    if (Str.startswith("ssp-buffer-size=")) {
+      if (StackProtectorLevel) {
+        CmdArgs.push_back("-stack-protector-buffer-size");
+        // FIXME: Verify the argument is a valid integer.
+        CmdArgs.push_back(Args.MakeArgString(Str.drop_front(16)));
+      }
+      (*it)->claim();
+    }
+  }
+
   // Translate -mstackrealign
   if (Args.hasFlag(options::OPT_mstackrealign, options::OPT_mno_stackrealign,
                    false)) {
@@ -2370,6 +2552,10 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     StringRef alignment = Args.getLastArgValue(options::OPT_mstack_alignment);
     CmdArgs.push_back(Args.MakeArgString("-mstack-alignment=" + alignment));
   }
+  if (Args.hasArg(options::OPT_mstrict_align)) {
+    CmdArgs.push_back("-backend-option");
+    CmdArgs.push_back("-arm-strict-align");
+  }
 
   // Forward -f options with positive and negative forms; we translate
   // these by hand.
@@ -2427,9 +2613,20 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
   // -frtti is default.
   if (!Args.hasFlag(options::OPT_frtti, options::OPT_fno_rtti) ||
-      KernelOrKext)
+      KernelOrKext) {
     CmdArgs.push_back("-fno-rtti");
 
+    // -fno-rtti cannot usefully be combined with -fsanitize=vptr.
+    if (Sanitize.sanitizesVptr()) {
+      std::string NoRttiArg =
+        Args.getLastArg(options::OPT_mkernel,
+                        options::OPT_fapple_kext,
+                        options::OPT_fno_rtti)->getAsString(Args);
+      D.Diag(diag::err_drv_argument_not_allowed_with)
+        << "-fsanitize=vptr" << NoRttiArg;
+    }
+  }
+
   // -fshort-enums=0 is default for all architectures except Hexagon.
   if (Args.hasFlag(options::OPT_fshort_enums,
                    options::OPT_fno_short_enums,
@@ -2537,12 +2734,16 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back("-fobjc-default-synthesize-properties");
   }
 
+  // -fencode-extended-block-signature=1 is default.
+  if (getToolChain().IsEncodeExtendedBlockSignatureDefault()) {
+    CmdArgs.push_back("-fencode-extended-block-signature");
+  }
+  
   // Allow -fno-objc-arr to trump -fobjc-arr/-fobjc-arc.
   // NOTE: This logic is duplicated in ToolChains.cpp.
   bool ARC = isObjCAutoRefCount(Args);
   if (ARC) {
-    if (!getToolChain().SupportsObjCARC())
-      D.Diag(diag::err_arc_unsupported);
+    getToolChain().CheckObjCARC();
 
     CmdArgs.push_back("-fobjc-arc");
 
@@ -2629,7 +2830,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   // -fno-pack-struct doesn't apply to -fpack-struct=.
   if (Arg *A = Args.getLastArg(options::OPT_fpack_struct_EQ)) {
     std::string PackStructStr = "-fpack-struct=";
-    PackStructStr += A->getValue(Args);
+    PackStructStr += A->getValue();
     CmdArgs.push_back(Args.MakeArgString(PackStructStr));
   } else if (Args.hasFlag(options::OPT_fpack_struct,
                           options::OPT_fno_pack_struct, false)) {
@@ -2678,13 +2879,13 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   if (const Arg *A =
         Args.getLastArg(options::OPT_fdiagnostics_show_category_EQ)) {
     CmdArgs.push_back("-fdiagnostics-show-category");
-    CmdArgs.push_back(A->getValue(Args));
+    CmdArgs.push_back(A->getValue());
   }
 
   if (const Arg *A =
         Args.getLastArg(options::OPT_fdiagnostics_format_EQ)) {
     CmdArgs.push_back("-fdiagnostics-format");
-    CmdArgs.push_back(A->getValue(Args));
+    CmdArgs.push_back(A->getValue());
   }
 
   if (Arg *A = Args.getLastArg(
@@ -2776,9 +2977,12 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
   // Handle serialized diagnostics.
   if (Arg *A = Args.getLastArg(options::OPT__serialize_diags)) {
     CmdArgs.push_back("-serialize-diagnostic-file");
-    CmdArgs.push_back(Args.MakeArgString(A->getValue(Args)));
+    CmdArgs.push_back(Args.MakeArgString(A->getValue()));
   }
 
+  if (Args.hasArg(options::OPT_fretain_comments_from_system_headers))
+    CmdArgs.push_back("-fretain-comments-from-system-headers");
+
   // Forward -Xclang arguments to -cc1, and -mllvm arguments to the LLVM option
   // parser.
   Args.AddAllArgValues(CmdArgs, options::OPT_Xclang);
@@ -2788,7 +2992,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
 
     // We translate this by hand to the -cc1 argument, since nightly test uses
     // it and developers have been trained to spell it with -mllvm.
-    if (StringRef((*it)->getValue(Args, 0)) == "-disable-llvm-optzns")
+    if (StringRef((*it)->getValue(0)) == "-disable-llvm-optzns")
       CmdArgs.push_back("-disable-llvm-optzns");
     else
       (*it)->render(Args, CmdArgs);
@@ -2807,7 +3011,10 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
          it = Inputs.begin(), ie = Inputs.end(); it != ie; ++it) {
     const InputInfo &II = *it;
     CmdArgs.push_back("-x");
-    CmdArgs.push_back(types::getTypeName(II.getType()));
+    if (Args.hasArg(options::OPT_rewrite_objc))
+      CmdArgs.push_back(types::getTypeName(types::TY_PP_ObjCXX));
+    else
+      CmdArgs.push_back(types::getTypeName(II.getType()));
     if (II.isFilename())
       CmdArgs.push_back(II.getFilename());
     else
@@ -2894,7 +3101,7 @@ ObjCRuntime Clang::AddObjCRuntimeArgs(const ArgList &args,
   if (runtimeArg &&
       runtimeArg->getOption().matches(options::OPT_fobjc_runtime_EQ)) {
     ObjCRuntime runtime;
-    StringRef value = runtimeArg->getValue(args);
+    StringRef value = runtimeArg->getValue();
     if (runtime.tryParse(value)) {
       getToolChain().getDriver().Diag(diag::err_drv_unknown_objc_runtime)
         << value;
@@ -2912,7 +3119,7 @@ ObjCRuntime Clang::AddObjCRuntimeArgs(const ArgList &args,
   unsigned objcABIVersion = 1;
   // If -fobjc-abi-version= is present, use that to set the version.
   if (Arg *abiArg = args.getLastArg(options::OPT_fobjc_abi_version_EQ)) {
-    StringRef value = abiArg->getValue(args);
+    StringRef value = abiArg->getValue();
     if (value == "1")
       objcABIVersion = 1;
     else if (value == "2")
@@ -2940,7 +3147,7 @@ ObjCRuntime Clang::AddObjCRuntimeArgs(const ArgList &args,
 
       if (Arg *abiArg = args.getLastArg(
             options::OPT_fobjc_nonfragile_abi_version_EQ)) {
-        StringRef value = abiArg->getValue(args);
+        StringRef value = abiArg->getValue();
         if (value == "1")
           nonFragileABIVersion = 1;
         else if (value == "2")
@@ -2993,7 +3200,7 @@ ObjCRuntime Clang::AddObjCRuntimeArgs(const ArgList &args,
     // Legacy behaviour is to target the gnustep runtime if we are i
     // non-fragile mode or the GCC runtime in fragile mode.
     if (isNonFragile)
-      runtime = ObjCRuntime(ObjCRuntime::GNUstep, VersionTuple());
+      runtime = ObjCRuntime(ObjCRuntime::GNUstep, VersionTuple(1,6));
     else
       runtime = ObjCRuntime(ObjCRuntime::GCC, VersionTuple());
   }
@@ -3116,7 +3323,7 @@ void gcc::Common::ConstructJob(Compilation &C, const JobAction &JA,
   for (ArgList::const_iterator
          it = Args.begin(), ie = Args.end(); it != ie; ++it) {
     Arg *A = *it;
-    if (A->getOption().hasForwardToGCC()) {
+    if (forwardToGCC(A->getOption())) {
       // Don't forward any -g arguments to assembly steps.
       if (isa<AssembleJobAction>(JA) &&
           A->getOption().matches(options::OPT_g_Group))
@@ -3134,17 +3341,17 @@ void gcc::Common::ConstructJob(Compilation &C, const JobAction &JA,
   RenderExtraToolArgs(JA, CmdArgs);
 
   // If using a driver driver, force the arch.
-  const std::string &Arch = getToolChain().getArchName();
+  llvm::Triple::ArchType Arch = getToolChain().getArch();
   if (getToolChain().getTriple().isOSDarwin()) {
     CmdArgs.push_back("-arch");
 
     // FIXME: Remove these special cases.
-    if (Arch == "powerpc")
+    if (Arch == llvm::Triple::ppc)
       CmdArgs.push_back("ppc");
-    else if (Arch == "powerpc64")
+    else if (Arch == llvm::Triple::ppc64)
       CmdArgs.push_back("ppc64");
     else
-      CmdArgs.push_back(Args.MakeArgString(Arch));
+      CmdArgs.push_back(Args.MakeArgString(getToolChain().getArchName()));
   }
 
   // Try to force gcc to match the tool chain we want, if we recognize
@@ -3152,9 +3359,9 @@ void gcc::Common::ConstructJob(Compilation &C, const JobAction &JA,
   //
   // FIXME: The triple class should directly provide the information we want
   // here.
-  if (Arch == "i386" || Arch == "powerpc")
+  if (Arch == llvm::Triple::x86 || Arch == llvm::Triple::ppc)
     CmdArgs.push_back("-m32");
-  else if (Arch == "x86_64" || Arch == "powerpc64")
+  else if (Arch == llvm::Triple::x86_64 || Arch == llvm::Triple::x86_64)
     CmdArgs.push_back("-m64");
 
   if (Output.isFilename()) {
@@ -3341,7 +3548,7 @@ void hexagon::Link::ConstructJob(Compilation &C, const JobAction &JA,
   for (ArgList::const_iterator
          it = Args.begin(), ie = Args.end(); it != ie; ++it) {
     Arg *A = *it;
-    if (A->getOption().hasForwardToGCC()) {
+    if (forwardToGCC(A->getOption())) {
       // Don't forward any -g arguments to assembly steps.
       if (isa<AssembleJobAction>(JA) &&
           A->getOption().matches(options::OPT_g_Group))
@@ -3409,6 +3616,37 @@ void hexagon::Link::ConstructJob(Compilation &C, const JobAction &JA,
 }
 // Hexagon tools end.
 
+llvm::Triple::ArchType darwin::getArchTypeForDarwinArchName(StringRef Str) {
+  // See arch(3) and llvm-gcc's driver-driver.c. We don't implement support for
+  // archs which Darwin doesn't use.
+
+  // The matching this routine does is fairly pointless, since it is neither the
+  // complete architecture list, nor a reasonable subset. The problem is that
+  // historically the driver driver accepts this and also ties its -march=
+  // handling to the architecture name, so we need to be careful before removing
+  // support for it.
+
+  // This code must be kept in sync with Clang's Darwin specific argument
+  // translation.
+
+  return llvm::StringSwitch<llvm::Triple::ArchType>(Str)
+    .Cases("ppc", "ppc601", "ppc603", "ppc604", "ppc604e", llvm::Triple::ppc)
+    .Cases("ppc750", "ppc7400", "ppc7450", "ppc970", llvm::Triple::ppc)
+    .Case("ppc64", llvm::Triple::ppc64)
+    .Cases("i386", "i486", "i486SX", "i586", "i686", llvm::Triple::x86)
+    .Cases("pentium", "pentpro", "pentIIm3", "pentIIm5", "pentium4",
+           llvm::Triple::x86)
+    .Case("x86_64", llvm::Triple::x86_64)
+    // This is derived from the driver driver.
+    .Cases("arm", "armv4t", "armv5", "armv6", llvm::Triple::arm)
+    .Cases("armv7", "armv7f", "armv7k", "armv7s", "xscale", llvm::Triple::arm)
+    .Case("r600", llvm::Triple::r600)
+    .Case("nvptx", llvm::Triple::nvptx)
+    .Case("nvptx64", llvm::Triple::nvptx64)
+    .Case("amdil", llvm::Triple::amdil)
+    .Case("spir", llvm::Triple::spir)
+    .Default(llvm::Triple::UnknownArch);
+}
 
 const char *darwin::CC1::getCC1Name(types::ID Type) const {
   switch (Type) {
@@ -3457,7 +3695,7 @@ darwin::CC1::getDependencyFileName(const ArgList &Args,
   std::string Res;
 
   if (Arg *OutputOpt = Args.getLastArg(options::OPT_o)) {
-    std::string Str(OutputOpt->getValue(Args));
+    std::string Str(OutputOpt->getValue());
     Res = Str.substr(0, Str.rfind('.'));
   } else {
     Res = darwin::CC1::getBaseInputStem(Args, Inputs);
@@ -3546,6 +3784,7 @@ void darwin::CC1::RemoveCC1UnsupportedArgs(ArgStringList &CmdArgs) const {
         .Case("duplicate-method-arg", true)
         .Case("dynamic-class-memaccess", true)
         .Case("enum-compare", true)
+        .Case("enum-conversion", true)
         .Case("exit-time-destructors", true)
         .Case("gnu", true)
         .Case("gnu-designator", true)
@@ -3555,6 +3794,7 @@ void darwin::CC1::RemoveCC1UnsupportedArgs(ArgStringList &CmdArgs) const {
         .Case("implicit-atomic-properties", true)
         .Case("incompatible-pointer-types", true)
         .Case("incomplete-implementation", true)
+        .Case("int-conversion", true)
         .Case("initializer-overrides", true)
         .Case("invalid-noreturn", true)
         .Case("invalid-token-paste", true)
@@ -3619,7 +3859,10 @@ void darwin::CC1::AddCC1Args(const ArgList &Args,
   CheckCodeGenerationOptions(D, Args);
 
   // Derived from cc1 spec.
-  if (!Args.hasArg(options::OPT_mkernel) && !Args.hasArg(options::OPT_static) &&
+  if ((!Args.hasArg(options::OPT_mkernel) ||
+       (getDarwinToolChain().isTargetIPhoneOS() &&
+        !getDarwinToolChain().isIPhoneOSVersionLT(6, 0))) &&
+      !Args.hasArg(options::OPT_static) &&
       !Args.hasArg(options::OPT_mdynamic_no_pic))
     CmdArgs.push_back("-fPIC");
 
@@ -3672,7 +3915,7 @@ void darwin::CC1::AddCC1OptionsArgs(const ArgList &Args, ArgStringList &CmdArgs,
       Args.hasArg(options::OPT_o)) {
     Arg *OutputOpt = Args.getLastArg(options::OPT_o);
     CmdArgs.push_back("-auxbase-strip");
-    CmdArgs.push_back(OutputOpt->getValue(Args));
+    CmdArgs.push_back(OutputOpt->getValue());
   } else {
     CmdArgs.push_back("-auxbase");
     CmdArgs.push_back(darwin::CC1::getBaseInputStem(Args, Inputs));
@@ -3811,7 +4054,7 @@ void darwin::CC1::AddCPPUniqueOptionsArgs(const ArgList &Args,
   Args.AddLastArg(CmdArgs, options::OPT_P);
 
   // FIXME: Handle %I properly.
-  if (getToolChain().getArchName() == "x86_64") {
+  if (getToolChain().getArch() == llvm::Triple::x86_64) {
     CmdArgs.push_back("-imultilib");
     CmdArgs.push_back("x86_64");
   }
@@ -3837,7 +4080,7 @@ void darwin::CC1::AddCPPUniqueOptionsArgs(const ArgList &Args,
       (Args.hasArg(options::OPT_MD) || Args.hasArg(options::OPT_MMD))) {
     if (Arg *OutputOpt = Args.getLastArg(options::OPT_o)) {
       CmdArgs.push_back("-MQ");
-      CmdArgs.push_back(OutputOpt->getValue(Args));
+      CmdArgs.push_back(OutputOpt->getValue());
     }
   }
 
@@ -4073,9 +4316,11 @@ void darwin::Assemble::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back("-force_cpusubtype_ALL");
 
   if (getToolChain().getTriple().getArch() != llvm::Triple::x86_64 &&
-      (Args.hasArg(options::OPT_mkernel) ||
-       Args.hasArg(options::OPT_static) ||
-       Args.hasArg(options::OPT_fapple_kext)))
+      (((Args.hasArg(options::OPT_mkernel) ||
+         Args.hasArg(options::OPT_fapple_kext)) &&
+        (!getDarwinToolChain().isTargetIPhoneOS() ||
+         getDarwinToolChain().isIPhoneOSVersionLT(6, 0))) ||
+       Args.hasArg(options::OPT_static)))
     CmdArgs.push_back("-static");
 
   Args.AddAllArgValues(CmdArgs, options::OPT_Wa_COMMA,
@@ -4110,16 +4355,29 @@ void darwin::DarwinTool::AddDarwinArch(const ArgList &Args,
     CmdArgs.push_back("-force_cpusubtype_ALL");
 }
 
+bool darwin::Link::NeedsTempPath(const InputInfoList &Inputs) const {
+  // We only need to generate a temp path for LTO if we aren't compiling object
+  // files. When compiling source files, we run 'dsymutil' after linking. We
+  // don't run 'dsymutil' when compiling object files.
+  for (InputInfoList::const_iterator
+         it = Inputs.begin(), ie = Inputs.end(); it != ie; ++it)
+    if (it->getType() != types::TY_Object)
+      return true;
+
+  return false;
+}
+
 void darwin::Link::AddLinkArgs(Compilation &C,
                                const ArgList &Args,
-                               ArgStringList &CmdArgs) const {
+                               ArgStringList &CmdArgs,
+                               const InputInfoList &Inputs) const {
   const Driver &D = getToolChain().getDriver();
   const toolchains::Darwin &DarwinTC = getDarwinToolChain();
 
   unsigned Version[3] = { 0, 0, 0 };
   if (Arg *A = Args.getLastArg(options::OPT_mlinker_version_EQ)) {
     bool HadExtra;
-    if (!Driver::GetReleaseVersion(A->getValue(Args), Version[0],
+    if (!Driver::GetReleaseVersion(A->getValue(), Version[0],
                                    Version[1], Version[2], HadExtra) ||
         HadExtra)
       D.Diag(diag::err_drv_invalid_version_number)
@@ -4140,7 +4398,7 @@ void darwin::Link::AddLinkArgs(Compilation &C,
              ie = Args.filtered_end(); it != ie; ++it) {
         const Arg *A = *it;
         for (unsigned i = 0, e = A->getNumValues(); i != e; ++i)
-          if (StringRef(A->getValue(Args, i)) == "-kext")
+          if (StringRef(A->getValue(i)) == "-kext")
             UsesLdClassic = true;
       }
     }
@@ -4151,7 +4409,7 @@ void darwin::Link::AddLinkArgs(Compilation &C,
   // If we are using LTO, then automatically create a temporary file path for
   // the linker to use, so that it's lifetime will extend past a possible
   // dsymutil step.
-  if (Version[0] >= 116 && D.IsUsingLTO(Args)) {
+  if (Version[0] >= 116 && D.IsUsingLTO(Args) && NeedsTempPath(Inputs)) {
     const char *TmpPath = C.getArgs().MakeArgString(
       D.GetTemporaryPath("cc", types::getTypeTempSuffix(types::TY_Object)));
     C.addTempFile(TmpPath);
@@ -4286,7 +4544,7 @@ void darwin::Link::AddLinkArgs(Compilation &C,
     CmdArgs.push_back(C.getArgs().MakeArgString(sysroot));
   } else if (const Arg *A = Args.getLastArg(options::OPT_isysroot)) {
     CmdArgs.push_back("-syslibroot");
-    CmdArgs.push_back(A->getValue(Args));
+    CmdArgs.push_back(A->getValue());
   }
 
   Args.AddLastArg(CmdArgs, options::OPT_twolevel__namespace);
@@ -4338,7 +4596,7 @@ void darwin::Link::ConstructJob(Compilation &C, const JobAction &JA,
 
   // I'm not sure why this particular decomposition exists in gcc, but
   // we follow suite for ease of comparison.
-  AddLinkArgs(C, Args, CmdArgs);
+  AddLinkArgs(C, Args, CmdArgs, Inputs);
 
   Args.AddAllArgs(CmdArgs, options::OPT_d_Flag);
   Args.AddAllArgs(CmdArgs, options::OPT_s);
@@ -4423,7 +4681,7 @@ void darwin::Link::ConstructJob(Compilation &C, const JobAction &JA,
             } else if (getDarwinToolChain().isTargetIPhoneOS()) {
               if (getDarwinToolChain().isIPhoneOSVersionLT(3, 1))
                 CmdArgs.push_back("-lcrt1.o");
-              else
+              else if (getDarwinToolChain().isIPhoneOSVersionLT(6, 0))
                 CmdArgs.push_back("-lcrt1.3.1.o");
             } else {
               if (getDarwinToolChain().isMacosxVersionLT(10, 5))
@@ -4451,11 +4709,12 @@ void darwin::Link::ConstructJob(Compilation &C, const JobAction &JA,
 
   Args.AddAllArgs(CmdArgs, options::OPT_L);
 
-  // If we're building a dynamic lib with -faddress-sanitizer, unresolved
-  // symbols may appear. Mark all of them as dynamic_lookup.
-  // Linking executables is handled in lib/Driver/ToolChains.cpp.
-  if (Args.hasFlag(options::OPT_faddress_sanitizer,
-                   options::OPT_fno_address_sanitizer, false)) {
+  SanitizerArgs Sanitize(getToolChain().getDriver(), Args);
+  // If we're building a dynamic lib with -fsanitize=address, or
+  // -fsanitize=undefined, unresolved symbols may appear. Mark all
+  // of them as dynamic_lookup. Linking executables is handled in
+  // lib/Driver/ToolChains.cpp.
+  if (Sanitize.needsAsanRt() || Sanitize.needsUbsanRt()) {
     if (Args.hasArg(options::OPT_dynamiclib) ||
         Args.hasArg(options::OPT_bundle)) {
       CmdArgs.push_back("-undefined");
@@ -4474,14 +4733,14 @@ void darwin::Link::ConstructJob(Compilation &C, const JobAction &JA,
       !Args.hasArg(options::OPT_nodefaultlibs)) {
     // Avoid linking compatibility stubs on i386 mac.
     if (!getDarwinToolChain().isTargetMacOS() ||
-        getDarwinToolChain().getArchName() != "i386") {
+        getDarwinToolChain().getArch() != llvm::Triple::x86) {
       // If we don't have ARC or subscripting runtime support, link in the
       // runtime stubs.  We have to do this *before* adding any of the normal
       // linker inputs so that its initializer gets run first.
       ObjCRuntime runtime =
         getDarwinToolChain().getDefaultObjCRuntime(/*nonfragile*/ true);
       // We use arclite library for both ARC and subscripting support.
-      if ((!runtime.hasARC() && isObjCAutoRefCount(Args)) ||
+      if ((!runtime.hasNativeARC() && isObjCAutoRefCount(Args)) ||
           !runtime.hasSubscripting())
         getDarwinToolChain().AddLinkARCArgs(Args, CmdArgs);
     }
@@ -4937,14 +5196,21 @@ void openbsd::Link::ConstructJob(Compilation &C, const JobAction &JA,
     // the default system libraries. Just mimic this for now.
     CmdArgs.push_back("-lgcc");
 
-    if (Args.hasArg(options::OPT_pthread))
-      CmdArgs.push_back("-lpthread");
+    if (Args.hasArg(options::OPT_pthread)) {
+      if (!Args.hasArg(options::OPT_shared) &&
+          Args.hasArg(options::OPT_pg))
+         CmdArgs.push_back("-lpthread_p");
+      else
+         CmdArgs.push_back("-lpthread");
+    }
+
     if (!Args.hasArg(options::OPT_shared)) {
-      if (Args.hasArg(options::OPT_pg)) 
+      if (Args.hasArg(options::OPT_pg))
          CmdArgs.push_back("-lc_p");
       else
          CmdArgs.push_back("-lc");
     }
+
     CmdArgs.push_back("-lgcc");
   }
 
@@ -5056,8 +5322,14 @@ void bitrig::Link::ConstructJob(Compilation &C, const JobAction &JA,
         CmdArgs.push_back("-lm");
     }
 
-    if (Args.hasArg(options::OPT_pthread))
-      CmdArgs.push_back("-lpthread");
+    if (Args.hasArg(options::OPT_pthread)) {
+      if (!Args.hasArg(options::OPT_shared) &&
+          Args.hasArg(options::OPT_pg))
+        CmdArgs.push_back("-lpthread_p");
+      else
+        CmdArgs.push_back("-lpthread");
+    }
+
     if (!Args.hasArg(options::OPT_shared)) {
       if (Args.hasArg(options::OPT_pg))
         CmdArgs.push_back("-lc_p");
@@ -5108,17 +5380,48 @@ void freebsd::Assemble::ConstructJob(Compilation &C, const JobAction &JA,
 
   // When building 32-bit code on FreeBSD/amd64, we have to explicitly
   // instruct as in the base system to assemble 32-bit code.
-  if (getToolChain().getArchName() == "i386")
+  if (getToolChain().getArch() == llvm::Triple::x86)
     CmdArgs.push_back("--32");
-
-  if (getToolChain().getArchName() == "powerpc")
+  else if (getToolChain().getArch() == llvm::Triple::ppc)
     CmdArgs.push_back("-a32");
+  else if (getToolChain().getArch() == llvm::Triple::mips ||
+           getToolChain().getArch() == llvm::Triple::mipsel ||
+           getToolChain().getArch() == llvm::Triple::mips64 ||
+           getToolChain().getArch() == llvm::Triple::mips64el) {
+    StringRef CPUName;
+    StringRef ABIName;
+    getMipsCPUAndABI(Args, getToolChain(), CPUName, ABIName);
 
-  // Set byte order explicitly
-  if (getToolChain().getArchName() == "mips")
-    CmdArgs.push_back("-EB");
-  else if (getToolChain().getArchName() == "mipsel")
-    CmdArgs.push_back("-EL");
+    CmdArgs.push_back("-march");
+    CmdArgs.push_back(CPUName.data());
+
+    // Convert ABI name to the GNU tools acceptable variant.
+    if (ABIName == "o32")
+      ABIName = "32";
+    else if (ABIName == "n64")
+      ABIName = "64";
+
+    CmdArgs.push_back("-mabi");
+    CmdArgs.push_back(ABIName.data());
+
+    if (getToolChain().getArch() == llvm::Triple::mips ||
+        getToolChain().getArch() == llvm::Triple::mips64)
+      CmdArgs.push_back("-EB");
+    else
+      CmdArgs.push_back("-EL");
+
+    Arg *LastPICArg = Args.getLastArg(options::OPT_fPIC, options::OPT_fno_PIC,
+                                      options::OPT_fpic, options::OPT_fno_pic,
+                                      options::OPT_fPIE, options::OPT_fno_PIE,
+                                      options::OPT_fpie, options::OPT_fno_pie);
+    if (LastPICArg &&
+        (LastPICArg->getOption().matches(options::OPT_fPIC) ||
+         LastPICArg->getOption().matches(options::OPT_fpic) ||
+         LastPICArg->getOption().matches(options::OPT_fPIE) ||
+         LastPICArg->getOption().matches(options::OPT_fpie))) {
+      CmdArgs.push_back("-KPIC");
+    }
+  }
 
   Args.AddAllArgValues(CmdArgs, options::OPT_Wa_COMMA,
                        options::OPT_Xassembler);
@@ -5142,7 +5445,9 @@ void freebsd::Link::ConstructJob(Compilation &C, const JobAction &JA,
                                  const InputInfoList &Inputs,
                                  const ArgList &Args,
                                  const char *LinkingOutput) const {
-  const Driver &D = getToolChain().getDriver();
+  const toolchains::FreeBSD& ToolChain = 
+    static_cast<const toolchains::FreeBSD&>(getToolChain());
+  const Driver &D = ToolChain.getDriver();
   ArgStringList CmdArgs;
 
   // Silence warning for "clang -g foo.o -o foo"
@@ -5156,6 +5461,9 @@ void freebsd::Link::ConstructJob(Compilation &C, const JobAction &JA,
   if (!D.SysRoot.empty())
     CmdArgs.push_back(Args.MakeArgString("--sysroot=" + D.SysRoot));
 
+  if (Args.hasArg(options::OPT_pie))
+    CmdArgs.push_back("-pie");
+
   if (Args.hasArg(options::OPT_static)) {
     CmdArgs.push_back("-Bstatic");
   } else {
@@ -5168,8 +5476,8 @@ void freebsd::Link::ConstructJob(Compilation &C, const JobAction &JA,
       CmdArgs.push_back("-dynamic-linker");
       CmdArgs.push_back("/libexec/ld-elf.so.1");
     }
-    if (getToolChain().getTriple().getOSMajorVersion() >= 9) {
-      llvm::Triple::ArchType Arch = getToolChain().getArch();
+    if (ToolChain.getTriple().getOSMajorVersion() >= 9) {
+      llvm::Triple::ArchType Arch = ToolChain.getArch();
       if (Arch == llvm::Triple::arm || Arch == llvm::Triple::sparc ||
           Arch == llvm::Triple::x86 || Arch == llvm::Triple::x86_64) {
         CmdArgs.push_back("--hash-style=both");
@@ -5180,12 +5488,12 @@ void freebsd::Link::ConstructJob(Compilation &C, const JobAction &JA,
 
   // When building 32-bit code on FreeBSD/amd64, we have to explicitly
   // instruct ld in the base system to link 32-bit code.
-  if (getToolChain().getArchName() == "i386") {
+  if (ToolChain.getArch() == llvm::Triple::x86) {
     CmdArgs.push_back("-m");
     CmdArgs.push_back("elf_i386_fbsd");
   }
 
-  if (getToolChain().getArchName() == "powerpc") {
+  if (ToolChain.getArch() == llvm::Triple::ppc) {
     CmdArgs.push_back("-m");
     CmdArgs.push_back("elf32ppc_fbsd");
   }
@@ -5199,29 +5507,33 @@ void freebsd::Link::ConstructJob(Compilation &C, const JobAction &JA,
 
   if (!Args.hasArg(options::OPT_nostdlib) &&
       !Args.hasArg(options::OPT_nostartfiles)) {
+    const char *crt1 = NULL;
     if (!Args.hasArg(options::OPT_shared)) {
       if (Args.hasArg(options::OPT_pg))
-        CmdArgs.push_back(Args.MakeArgString(
-                                getToolChain().GetFilePath("gcrt1.o")));
-      else {
-        const char *crt = Args.hasArg(options::OPT_pie) ? "Scrt1.o" : "crt1.o";
-        CmdArgs.push_back(Args.MakeArgString(
-                                getToolChain().GetFilePath(crt)));
-      }
-      CmdArgs.push_back(Args.MakeArgString(
-                              getToolChain().GetFilePath("crti.o")));
-      CmdArgs.push_back(Args.MakeArgString(
-                              getToolChain().GetFilePath("crtbegin.o")));
-    } else {
-      CmdArgs.push_back(Args.MakeArgString(
-                              getToolChain().GetFilePath("crti.o")));
-      CmdArgs.push_back(Args.MakeArgString(
-                              getToolChain().GetFilePath("crtbeginS.o")));
+        crt1 = "gcrt1.o";
+      else if (Args.hasArg(options::OPT_pie))
+        crt1 = "Scrt1.o";
+      else
+        crt1 = "crt1.o";
     }
+    if (crt1)
+      CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath(crt1)));
+
+    CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath("crti.o")));
+
+    const char *crtbegin = NULL;
+    if (Args.hasArg(options::OPT_static))
+      crtbegin = "crtbeginT.o";
+    else if (Args.hasArg(options::OPT_shared) || Args.hasArg(options::OPT_pie))
+      crtbegin = "crtbeginS.o";
+    else
+      crtbegin = "crtbegin.o";
+
+    CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath(crtbegin)));
   }
 
   Args.AddAllArgs(CmdArgs, options::OPT_L);
-  const ToolChain::path_list Paths = getToolChain().getFilePaths();
+  const ToolChain::path_list Paths = ToolChain.getFilePaths();
   for (ToolChain::path_list::const_iterator i = Paths.begin(), e = Paths.end();
        i != e; ++i)
     CmdArgs.push_back(Args.MakeArgString(StringRef("-L") + *i));
@@ -5232,12 +5544,12 @@ void freebsd::Link::ConstructJob(Compilation &C, const JobAction &JA,
   Args.AddAllArgs(CmdArgs, options::OPT_Z_Flag);
   Args.AddAllArgs(CmdArgs, options::OPT_r);
 
-  AddLinkerInputs(getToolChain(), Inputs, Args, CmdArgs);
+  AddLinkerInputs(ToolChain, Inputs, Args, CmdArgs);
 
   if (!Args.hasArg(options::OPT_nostdlib) &&
       !Args.hasArg(options::OPT_nodefaultlibs)) {
     if (D.CCCIsCXX) {
-      getToolChain().AddCXXStdlibLibArgs(Args, CmdArgs);
+      ToolChain.AddCXXStdlibLibArgs(Args, CmdArgs);
       if (Args.hasArg(options::OPT_pg))
         CmdArgs.push_back("-lm_p");
       else
@@ -5290,20 +5602,17 @@ void freebsd::Link::ConstructJob(Compilation &C, const JobAction &JA,
 
   if (!Args.hasArg(options::OPT_nostdlib) &&
       !Args.hasArg(options::OPT_nostartfiles)) {
-    if (!Args.hasArg(options::OPT_shared))
-      CmdArgs.push_back(Args.MakeArgString(getToolChain().GetFilePath(
-                                                                  "crtend.o")));
+    if (Args.hasArg(options::OPT_shared) || Args.hasArg(options::OPT_pie))
+      CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath("crtendS.o")));
     else
-      CmdArgs.push_back(Args.MakeArgString(getToolChain().GetFilePath(
-                                                                 "crtendS.o")));
-    CmdArgs.push_back(Args.MakeArgString(getToolChain().GetFilePath(
-                                                                    "crtn.o")));
+      CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath("crtend.o")));
+    CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath("crtn.o")));
   }
 
-  addProfileRT(getToolChain(), Args, CmdArgs, getToolChain().getTriple());
+  addProfileRT(ToolChain, Args, CmdArgs, ToolChain.getTriple());
 
   const char *Exec =
-    Args.MakeArgString(getToolChain().GetProgramPath("ld"));
+    Args.MakeArgString(ToolChain.GetProgramPath("ld"));
   C.addCommand(new Command(JA, *this, Exec, CmdArgs));
 }
 
@@ -5320,9 +5629,9 @@ void netbsd::Assemble::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back("--32");
 
   // Set byte order explicitly
-  if (getToolChain().getArchName() == "mips")
+  if (getToolChain().getArch() == llvm::Triple::mips)
     CmdArgs.push_back("-EB");
-  else if (getToolChain().getArchName() == "mipsel")
+  else if (getToolChain().getArch() == llvm::Triple::mipsel)
     CmdArgs.push_back("-EL");
 
   Args.AddAllArgValues(CmdArgs, options::OPT_Wa_COMMA,
@@ -5547,7 +5856,7 @@ void linuxtools::Assemble::ConstructJob(Compilation &C, const JobAction &JA,
 
 static void AddLibgcc(llvm::Triple Triple, const Driver &D,
                       ArgStringList &CmdArgs, const ArgList &Args) {
-  bool isAndroid = Triple.getEnvironment() == llvm::Triple::ANDROIDEABI;
+  bool isAndroid = Triple.getEnvironment() == llvm::Triple::Android;
   bool StaticLibgcc = isAndroid || Args.hasArg(options::OPT_static) ||
     Args.hasArg(options::OPT_static_libgcc);
   if (!D.CCCIsCXX)
@@ -5570,6 +5879,11 @@ static void AddLibgcc(llvm::Triple Triple, const Driver &D,
     CmdArgs.push_back("-lgcc");
 }
 
+static bool hasMipsN32ABIArg(const ArgList &Args) {
+  Arg *A = Args.getLastArg(options::OPT_mabi_EQ);
+  return A && (A->getValue() == StringRef("n32"));
+}
+
 void linuxtools::Link::ConstructJob(Compilation &C, const JobAction &JA,
                                     const InputInfo &Output,
                                     const InputInfoList &Inputs,
@@ -5578,8 +5892,8 @@ void linuxtools::Link::ConstructJob(Compilation &C, const JobAction &JA,
   const toolchains::Linux& ToolChain =
     static_cast<const toolchains::Linux&>(getToolChain());
   const Driver &D = ToolChain.getDriver();
-  const bool isAndroid = ToolChain.getTriple().getEnvironment() ==
-    llvm::Triple::ANDROIDEABI;
+  const bool isAndroid =
+    ToolChain.getTriple().getEnvironment() == llvm::Triple::Android;
 
   ArgStringList CmdArgs;
 
@@ -5626,10 +5940,18 @@ void linuxtools::Link::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back("elf32btsmip");
   else if (ToolChain.getArch() == llvm::Triple::mipsel)
     CmdArgs.push_back("elf32ltsmip");
-  else if (ToolChain.getArch() == llvm::Triple::mips64)
-    CmdArgs.push_back("elf64btsmip");
-  else if (ToolChain.getArch() == llvm::Triple::mips64el)
-    CmdArgs.push_back("elf64ltsmip");
+  else if (ToolChain.getArch() == llvm::Triple::mips64) {
+    if (hasMipsN32ABIArg(Args))
+      CmdArgs.push_back("elf32btsmipn32");
+    else
+      CmdArgs.push_back("elf64btsmip");
+  }
+  else if (ToolChain.getArch() == llvm::Triple::mips64el) {
+    if (hasMipsN32ABIArg(Args))
+      CmdArgs.push_back("elf32ltsmipn32");
+    else
+      CmdArgs.push_back("elf64ltsmip");
+  }
   else
     CmdArgs.push_back("elf_x86_64");
 
@@ -5641,8 +5963,7 @@ void linuxtools::Link::ConstructJob(Compilation &C, const JobAction &JA,
       CmdArgs.push_back("-static");
   } else if (Args.hasArg(options::OPT_shared)) {
     CmdArgs.push_back("-shared");
-    if ((ToolChain.getArch() == llvm::Triple::arm
-         || ToolChain.getArch() == llvm::Triple::thumb) && isAndroid) {
+    if (isAndroid) {
       CmdArgs.push_back("-Bsymbolic");
     }
   }
@@ -5667,8 +5988,12 @@ void linuxtools::Link::ConstructJob(Compilation &C, const JobAction &JA,
              ToolChain.getArch() == llvm::Triple::mipsel)
       CmdArgs.push_back("/lib/ld.so.1");
     else if (ToolChain.getArch() == llvm::Triple::mips64 ||
-             ToolChain.getArch() == llvm::Triple::mips64el)
-      CmdArgs.push_back("/lib64/ld.so.1");
+             ToolChain.getArch() == llvm::Triple::mips64el) {
+      if (hasMipsN32ABIArg(Args))
+        CmdArgs.push_back("/lib32/ld.so.1");
+      else
+        CmdArgs.push_back("/lib64/ld.so.1");
+    }
     else if (ToolChain.getArch() == llvm::Triple::ppc)
       CmdArgs.push_back("/lib/ld.so.1");
     else if (ToolChain.getArch() == llvm::Triple::ppc64)
@@ -5699,11 +6024,16 @@ void linuxtools::Link::ConstructJob(Compilation &C, const JobAction &JA,
     const char *crtbegin;
     if (Args.hasArg(options::OPT_static))
       crtbegin = isAndroid ? "crtbegin_static.o" : "crtbeginT.o";
-    else if (Args.hasArg(options::OPT_shared) || Args.hasArg(options::OPT_pie))
+    else if (Args.hasArg(options::OPT_shared))
       crtbegin = isAndroid ? "crtbegin_so.o" : "crtbeginS.o";
+    else if (Args.hasArg(options::OPT_pie))
+      crtbegin = isAndroid ? "crtbegin_dynamic.o" : "crtbeginS.o";
     else
       crtbegin = isAndroid ? "crtbegin_dynamic.o" : "crtbegin.o";
     CmdArgs.push_back(Args.MakeArgString(ToolChain.GetFilePath(crtbegin)));
+
+    // Add crtfastmath.o if available and fast math is enabled.
+    ToolChain.AddFastMathRuntimeIfAvailable(Args, CmdArgs);
   }
 
   Args.AddAllArgs(CmdArgs, options::OPT_L);
@@ -5728,6 +6058,12 @@ void linuxtools::Link::ConstructJob(Compilation &C, const JobAction &JA,
 
   AddLinkerInputs(ToolChain, Inputs, Args, CmdArgs);
 
+  SanitizerArgs Sanitize(D, Args);
+
+  // Call this before we add the C++ ABI library.
+  if (Sanitize.needsUbsanRt())
+    addUbsanRTLinux(getToolChain(), Args, CmdArgs);
+
   if (D.CCCIsCXX &&
       !Args.hasArg(options::OPT_nostdlib) &&
       !Args.hasArg(options::OPT_nodefaultlibs)) {
@@ -5742,8 +6078,10 @@ void linuxtools::Link::ConstructJob(Compilation &C, const JobAction &JA,
   }
 
   // Call this before we add the C run-time.
-  addAsanRTLinux(getToolChain(), Args, CmdArgs);
-  addTsanRTLinux(getToolChain(), Args, CmdArgs);
+  if (Sanitize.needsAsanRt())
+    addAsanRTLinux(getToolChain(), Args, CmdArgs);
+  if (Sanitize.needsTsanRt())
+    addTsanRTLinux(getToolChain(), Args, CmdArgs);
 
   if (!Args.hasArg(options::OPT_nostdlib)) {
     if (!Args.hasArg(options::OPT_nodefaultlibs)) {
@@ -5766,8 +6104,10 @@ void linuxtools::Link::ConstructJob(Compilation &C, const JobAction &JA,
 
     if (!Args.hasArg(options::OPT_nostartfiles)) {
       const char *crtend;
-      if (Args.hasArg(options::OPT_shared) || Args.hasArg(options::OPT_pie))
+      if (Args.hasArg(options::OPT_shared))
         crtend = isAndroid ? "crtend_so.o" : "crtendS.o";
+      else if (Args.hasArg(options::OPT_pie))
+        crtend = isAndroid ? "crtend_android.o" : "crtendS.o";
       else
         crtend = isAndroid ? "crtend_android.o" : "crtend.o";
 
@@ -5873,7 +6213,7 @@ void dragonfly::Assemble::ConstructJob(Compilation &C, const JobAction &JA,
 
   // When building 32-bit code on DragonFly/pc64, we have to explicitly
   // instruct as in the base system to assemble 32-bit code.
-  if (getToolChain().getArchName() == "i386")
+  if (getToolChain().getArch() == llvm::Triple::x86)
     CmdArgs.push_back("--32");
 
   Args.AddAllArgValues(CmdArgs, options::OPT_Wa_COMMA,
@@ -5917,7 +6257,7 @@ void dragonfly::Link::ConstructJob(Compilation &C, const JobAction &JA,
 
   // When building 32-bit code on DragonFly/pc64, we have to explicitly
   // instruct ld in the base system to link 32-bit code.
-  if (getToolChain().getArchName() == "i386") {
+  if (getToolChain().getArch() == llvm::Triple::x86) {
     CmdArgs.push_back("-m");
     CmdArgs.push_back("elf_i386");
   }
diff --git a/lib/Driver/Tools.h b/lib/Driver/Tools.h
index 999c57a..5898c66 100644
--- a/lib/Driver/Tools.h
+++ b/lib/Driver/Tools.h
@@ -202,6 +202,8 @@ namespace hexagon {
 
 
 namespace darwin {
+  llvm::Triple::ArchType getArchTypeForDarwinArchName(StringRef Str);
+
   class LLVM_LIBRARY_VISIBILITY DarwinTool : public Tool {
     virtual void anchor();
   protected:
@@ -288,8 +290,9 @@ namespace darwin {
   };
 
   class LLVM_LIBRARY_VISIBILITY Link : public DarwinTool  {
+    bool NeedsTempPath(const InputInfoList &Inputs) const;
     void AddLinkArgs(Compilation &C, const ArgList &Args,
-                     ArgStringList &CmdArgs) const;
+                     ArgStringList &CmdArgs, const InputInfoList &Inputs) const;
 
   public:
     Link(const ToolChain &TC) : DarwinTool("darwin::Link", "linker", TC) {}
diff --git a/lib/Driver/Types.cpp b/lib/Driver/Types.cpp
index 9d8fcfd..862025e 100644
--- a/lib/Driver/Types.cpp
+++ b/lib/Driver/Types.cpp
@@ -94,20 +94,6 @@ bool types::isAcceptedByClang(ID Id) {
   }
 }
 
-bool types::isOnlyAcceptedByClang(ID Id) {
-  switch (Id) {
-  default:
-    return false;
-
-  case TY_AST:
-  case TY_LLVM_IR:
-  case TY_LLVM_BC:
-  case TY_RewrittenObjC:
-  case TY_RewrittenLegacyObjC:
-    return true;
-  }
-}
-
 bool types::isObjC(ID Id) {
   switch (Id) {
   default:
diff --git a/lib/Driver/WindowsToolChain.cpp b/lib/Driver/WindowsToolChain.cpp
index 6827034..de2d535 100644
--- a/lib/Driver/WindowsToolChain.cpp
+++ b/lib/Driver/WindowsToolChain.cpp
@@ -81,19 +81,15 @@ bool Windows::IsIntegratedAssemblerDefault() const {
 }
 
 bool Windows::IsUnwindTablesDefault() const {
-  // FIXME: Gross; we should probably have some separate target
-  // definition, possibly even reusing the one in clang.
-  return getArchName() == "x86_64";
+  return getArch() == llvm::Triple::x86_64;
 }
 
-const char *Windows::GetDefaultRelocationModel() const {
-  return "static";
+bool Windows::isPICDefault() const {
+  return getArch() == llvm::Triple::x86_64;
 }
 
-const char *Windows::GetForcedPicModel() const {
-  if (getArchName() == "x86_64")
-    return "pic";
-  return 0;
+bool Windows::isPICDefaultForced() const {
+  return getArch() == llvm::Triple::x86_64;
 }
 
 // FIXME: This probably should goto to some platform utils place.
diff --git a/lib/Edit/RewriteObjCFoundationAPI.cpp b/lib/Edit/RewriteObjCFoundationAPI.cpp
index d15b7a7..de96fee 100644
--- a/lib/Edit/RewriteObjCFoundationAPI.cpp
+++ b/lib/Edit/RewriteObjCFoundationAPI.cpp
@@ -920,6 +920,7 @@ static bool rewriteToNumericBoxedExpression(const ObjCMessageExpr *Msg,
     case CK_ARCExtendBlockObject:
     case CK_NonAtomicToAtomic:
     case CK_CopyAndAutoreleaseBlockObject:
+    case CK_BuiltinFnToFnPtr:
       return false;
     }
   }
diff --git a/lib/Frontend/ASTConsumers.cpp b/lib/Frontend/ASTConsumers.cpp
index 0f0d835..882d400 100644
--- a/lib/Frontend/ASTConsumers.cpp
+++ b/lib/Frontend/ASTConsumers.cpp
@@ -58,7 +58,7 @@ namespace {
     bool shouldWalkTypesOfTypeLocs() const { return false; }
 
     bool TraverseDecl(Decl *D) {
-      if (filterMatches(D)) {
+      if (D != NULL && filterMatches(D)) {
         Out.changeColor(llvm::raw_ostream::BLUE) <<
             (Dump ? "Dumping " : "Printing ") << getName(D) << ":\n";
         Out.resetColor();
@@ -66,6 +66,7 @@ namespace {
           D->dump(Out);
         else
           D->print(Out, /*Indentation=*/0, /*PrintInstantiation=*/true);
+        Out << "\n";
         // Don't traverse child nodes to avoid output duplication.
         return true;
       }
@@ -89,8 +90,6 @@ namespace {
 
   class ASTDeclNodeLister : public ASTConsumer,
                      public RecursiveASTVisitor<ASTDeclNodeLister> {
-    typedef RecursiveASTVisitor<ASTDeclNodeLister> base;
-
   public:
     ASTDeclNodeLister(raw_ostream *Out = NULL)
         : Out(Out ? *Out : llvm::outs()) {}
diff --git a/lib/Frontend/ASTMerge.cpp b/lib/Frontend/ASTMerge.cpp
index 9feb3de..31b1df4 100644
--- a/lib/Frontend/ASTMerge.cpp
+++ b/lib/Frontend/ASTMerge.cpp
@@ -41,8 +41,9 @@ void ASTMergeAction::ExecuteAction() {
       DiagIDs(CI.getDiagnostics().getDiagnosticIDs());
   for (unsigned I = 0, N = ASTFiles.size(); I != N; ++I) {
     IntrusiveRefCntPtr<DiagnosticsEngine>
-        Diags(new DiagnosticsEngine(DiagIDs, CI.getDiagnostics().getClient(),
-                             /*ShouldOwnClient=*/false));
+        Diags(new DiagnosticsEngine(DiagIDs, &CI.getDiagnosticOpts(),
+                                    CI.getDiagnostics().getClient(),
+                                    /*ShouldOwnClient=*/false));
     ASTUnit *Unit = ASTUnit::LoadFromASTFile(ASTFiles[I], Diags,
                                              CI.getFileSystemOpts(), false);
     if (!Unit)
diff --git a/lib/Frontend/ASTUnit.cpp b/lib/Frontend/ASTUnit.cpp
index 42a6772..5576854 100644
--- a/lib/Frontend/ASTUnit.cpp
+++ b/lib/Frontend/ASTUnit.cpp
@@ -27,6 +27,7 @@
 #include "clang/Serialization/ASTWriter.h"
 #include "clang/Lex/HeaderSearch.h"
 #include "clang/Lex/Preprocessor.h"
+#include "clang/Lex/PreprocessorOptions.h"
 #include "clang/Basic/TargetOptions.h"
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Basic/Diagnostic.h"
@@ -180,6 +181,14 @@ void OnDiskData::Cleanup() {
   CleanPreambleFile();
 }
 
+struct ASTUnit::ASTWriterData {
+  SmallString<128> Buffer;
+  llvm::BitstreamWriter Stream;
+  ASTWriter Writer;
+
+  ASTWriterData() : Stream(Buffer), Writer(Stream) { }
+};
+
 void ASTUnit::clearFileLevelDecls() {
   for (FileDeclsTy::iterator
          I = FileDecls.begin(), E = FileDecls.end(); I != E; ++I)
@@ -495,8 +504,8 @@ class ASTInfoCollector : public ASTReaderListener {
   ASTContext &Context;
   LangOptions &LangOpt;
   HeaderSearch &HSI;
+  IntrusiveRefCntPtr<TargetOptions> &TargetOpts;
   IntrusiveRefCntPtr<TargetInfo> &Target;
-  std::string &Predefines;
   unsigned &Counter;
 
   unsigned NumHeaderInfos;
@@ -504,54 +513,38 @@ class ASTInfoCollector : public ASTReaderListener {
   bool InitializedLanguage;
 public:
   ASTInfoCollector(Preprocessor &PP, ASTContext &Context, LangOptions &LangOpt, 
-                   HeaderSearch &HSI,
+                   HeaderSearch &HSI, 
+                   IntrusiveRefCntPtr<TargetOptions> &TargetOpts,
                    IntrusiveRefCntPtr<TargetInfo> &Target,
-                   std::string &Predefines,
                    unsigned &Counter)
-    : PP(PP), Context(Context), LangOpt(LangOpt), HSI(HSI), Target(Target),
-      Predefines(Predefines), Counter(Counter), NumHeaderInfos(0),
+    : PP(PP), Context(Context), LangOpt(LangOpt), HSI(HSI), 
+      TargetOpts(TargetOpts), Target(Target),
+      Counter(Counter), NumHeaderInfos(0),
       InitializedLanguage(false) {}
 
-  virtual bool ReadLanguageOptions(const LangOptions &LangOpts) {
+  virtual bool ReadLanguageOptions(const LangOptions &LangOpts,
+                                   bool Complain) {
     if (InitializedLanguage)
       return false;
     
     LangOpt = LangOpts;
-    
-    // Initialize the preprocessor.
-    PP.Initialize(*Target);
-    
-    // Initialize the ASTContext
-    Context.InitBuiltinTypes(*Target);
-    
     InitializedLanguage = true;
+    
+    updated();
     return false;
   }
 
-  virtual bool ReadTargetTriple(StringRef Triple) {
+  virtual bool ReadTargetOptions(const TargetOptions &TargetOpts,
+                                 bool Complain) {
     // If we've already initialized the target, don't do it again.
     if (Target)
       return false;
     
-    // FIXME: This is broken, we should store the TargetOptions in the AST file.
-    TargetOptions TargetOpts;
-    TargetOpts.ABI = "";
-    TargetOpts.CXXABI = "";
-    TargetOpts.CPU = "";
-    TargetOpts.Features.clear();
-    TargetOpts.Triple = Triple;
-    Target = TargetInfo::CreateTargetInfo(PP.getDiagnostics(), TargetOpts);
-    return false;
-  }
+    this->TargetOpts = new TargetOptions(TargetOpts);
+    Target = TargetInfo::CreateTargetInfo(PP.getDiagnostics(), 
+                                          *this->TargetOpts);
 
-  virtual bool ReadPredefinesBuffer(const PCHPredefinesBlocks &Buffers,
-                                    StringRef OriginalFileName,
-                                    std::string &SuggestedPredefines,
-                                    FileManager &FileMgr) {
-    Predefines = Buffers[0].Data;
-    for (unsigned I = 1, N = Buffers.size(); I != N; ++I) {
-      Predefines += Buffers[I].Data;
-    }
+    updated();
     return false;
   }
 
@@ -559,9 +552,27 @@ public:
     HSI.setHeaderFileInfoForUID(HFI, NumHeaderInfos++);
   }
 
-  virtual void ReadCounter(unsigned Value) {
+  virtual void ReadCounter(const serialization::ModuleFile &M, unsigned Value) {
     Counter = Value;
   }
+
+private:
+  void updated() {
+    if (!Target || !InitializedLanguage)
+      return;
+
+    // Inform the target of the language options.
+    //
+    // FIXME: We shouldn't need to do this, the target should be immutable once
+    // created. This complexity should be lifted elsewhere.
+    Target->setForcedLangOptions(LangOpt);
+
+    // Initialize the preprocessor.
+    PP.Initialize(*Target);
+
+    // Initialize the ASTContext
+    Context.InitBuiltinTypes(*Target);
+  }
 };
 
 class StoredDiagnosticConsumer : public DiagnosticConsumer {
@@ -621,8 +632,10 @@ void StoredDiagnosticConsumer::HandleDiagnostic(DiagnosticsEngine::Level Level,
   StoredDiags.push_back(StoredDiagnostic(Level, Info));
 }
 
-const std::string &ASTUnit::getOriginalSourceFileName() {
-  return OriginalSourceFile;
+ASTDeserializationListener *ASTUnit::getDeserializationListener() {
+  if (WriterData)
+    return &WriterData->Writer;
+  return 0;
 }
 
 llvm::MemoryBuffer *ASTUnit::getBufferForFile(StringRef Filename,
@@ -638,11 +651,11 @@ void ASTUnit::ConfigureDiags(IntrusiveRefCntPtr<DiagnosticsEngine> &Diags,
   if (!Diags.getPtr()) {
     // No diagnostics engine was provided, so create our own diagnostics object
     // with the default options.
-    DiagnosticOptions DiagOpts;
     DiagnosticConsumer *Client = 0;
     if (CaptureDiagnostics)
       Client = new StoredDiagnosticConsumer(AST.StoredDiagnostics);
-    Diags = CompilerInstance::createDiagnostics(DiagOpts, ArgEnd-ArgBegin,
+    Diags = CompilerInstance::createDiagnostics(new DiagnosticOptions(),
+                                                ArgEnd-ArgBegin,
                                                 ArgBegin, Client,
                                                 /*ShouldOwnClient=*/true,
                                                 /*ShouldCloneClient=*/false);
@@ -679,7 +692,10 @@ ASTUnit *ASTUnit::LoadFromASTFile(const std::string &Filename,
   AST->SourceMgr = new SourceManager(AST->getDiagnostics(),
                                      AST->getFileManager(),
                                      UserFilesAreVolatile);
-  AST->HeaderInfo.reset(new HeaderSearch(AST->getFileManager(),
+  AST->HSOpts = new HeaderSearchOptions();
+  
+  AST->HeaderInfo.reset(new HeaderSearch(AST->HSOpts,
+                                         AST->getFileManager(),
                                          AST->getDiagnostics(),
                                          AST->ASTFileLangOpts,
                                          /*Target=*/0));
@@ -734,12 +750,12 @@ ASTUnit *ASTUnit::LoadFromASTFile(const std::string &Filename,
   // Gather Info for preprocessor construction later on.
 
   HeaderSearch &HeaderInfo = *AST->HeaderInfo.get();
-  std::string Predefines;
   unsigned Counter;
 
   OwningPtr<ASTReader> Reader;
 
-  AST->PP = new Preprocessor(AST->getDiagnostics(), AST->ASTFileLangOpts, 
+  AST->PP = new Preprocessor(new PreprocessorOptions(),
+                             AST->getDiagnostics(), AST->ASTFileLangOpts,
                              /*Target=*/0, AST->getSourceManager(), HeaderInfo, 
                              *AST, 
                              /*IILookup=*/0,
@@ -757,10 +773,12 @@ ASTUnit *ASTUnit::LoadFromASTFile(const std::string &Filename,
                             /*DelayInitialization=*/true);
   ASTContext &Context = *AST->Ctx;
 
+  bool disableValid = false;
+  if (::getenv("LIBCLANG_DISABLE_PCH_VALIDATION"))
+    disableValid = true;
   Reader.reset(new ASTReader(PP, Context,
                              /*isysroot=*/"",
-                             /*DisableValidation=*/false,
-                             /*DisableStatCache=*/false,
+                             /*DisableValidation=*/disableValid,
                              AllowPCHWithCompilerErrors));
   
   // Recover resources if we crash before exiting this method.
@@ -769,21 +787,25 @@ ASTUnit *ASTUnit::LoadFromASTFile(const std::string &Filename,
 
   Reader->setListener(new ASTInfoCollector(*AST->PP, Context,
                                            AST->ASTFileLangOpts, HeaderInfo, 
-                                           AST->Target, Predefines, Counter));
+                                           AST->TargetOpts, AST->Target, 
+                                           Counter));
 
-  switch (Reader->ReadAST(Filename, serialization::MK_MainFile)) {
+  switch (Reader->ReadAST(Filename, serialization::MK_MainFile,
+                          ASTReader::ARR_None)) {
   case ASTReader::Success:
     break;
 
   case ASTReader::Failure:
-  case ASTReader::IgnorePCH:
+  case ASTReader::OutOfDate:
+  case ASTReader::VersionMismatch:
+  case ASTReader::ConfigurationMismatch:
+  case ASTReader::HadErrors:
     AST->getDiagnostics().Report(diag::err_fe_unable_to_load_pch);
     return NULL;
   }
 
   AST->OriginalSourceFile = Reader->getOriginalSourceFile();
 
-  PP.setPredefines(Reader->getSuggestedPredefines());
   PP.setCounterValue(Counter);
 
   // Attach the AST reader to the AST context as an external AST
@@ -897,6 +919,10 @@ public:
     for (DeclGroupRef::iterator it = D.begin(), ie = D.end(); it != ie; ++it)
       handleTopLevelDecl(*it);
   }
+
+  virtual ASTDeserializationListener *GetASTDeserializationListener() {
+    return Unit.getDeserializationListener();
+  }
 };
 
 class TopLevelDeclTrackerAction : public ASTFrontendAction {
@@ -1047,14 +1073,13 @@ bool ASTUnit::Parse(llvm::MemoryBuffer *OverrideMainBuffer) {
     CCInvocation(new CompilerInvocation(*Invocation));
 
   Clang->setInvocation(CCInvocation.getPtr());
-  OriginalSourceFile = Clang->getFrontendOpts().Inputs[0].File;
+  OriginalSourceFile = Clang->getFrontendOpts().Inputs[0].getFile();
     
   // Set up diagnostics, capturing any diagnostics that would
   // otherwise be dropped.
   Clang->setDiagnostics(&getDiagnostics());
   
   // Create the target instance.
-  Clang->getTargetOpts().Features = TargetFeatures;
   Clang->setTarget(TargetInfo::CreateTargetInfo(Clang->getDiagnostics(),
                    Clang->getTargetOpts()));
   if (!Clang->hasTarget()) {
@@ -1070,9 +1095,9 @@ bool ASTUnit::Parse(llvm::MemoryBuffer *OverrideMainBuffer) {
   
   assert(Clang->getFrontendOpts().Inputs.size() == 1 &&
          "Invocation must have exactly one source file!");
-  assert(Clang->getFrontendOpts().Inputs[0].Kind != IK_AST &&
+  assert(Clang->getFrontendOpts().Inputs[0].getKind() != IK_AST &&
          "FIXME: AST inputs not yet supported here!");
-  assert(Clang->getFrontendOpts().Inputs[0].Kind != IK_LLVM_IR &&
+  assert(Clang->getFrontendOpts().Inputs[0].getKind() != IK_LLVM_IR &&
          "IR inputs not support here!");
 
   // Configure the various subsystems.
@@ -1217,7 +1242,7 @@ ASTUnit::ComputePreamble(CompilerInvocation &Invocation,
   // command line (to another file) or directly through the compiler invocation
   // (to a memory buffer).
   llvm::MemoryBuffer *Buffer = 0;
-  llvm::sys::PathWithStatus MainFilePath(FrontendOpts.Inputs[0].File);
+  llvm::sys::PathWithStatus MainFilePath(FrontendOpts.Inputs[0].getFile());
   if (const llvm::sys::FileStatus *MainFileStatus = MainFilePath.getFileStatus()) {
     // Check whether there is a file-file remapping of the main file
     for (PreprocessorOptions::remapped_file_iterator
@@ -1267,7 +1292,7 @@ ASTUnit::ComputePreamble(CompilerInvocation &Invocation,
   
   // If the main source file was not remapped, load it now.
   if (!Buffer) {
-    Buffer = getBufferForFile(FrontendOpts.Inputs[0].File);
+    Buffer = getBufferForFile(FrontendOpts.Inputs[0].getFile());
     if (!Buffer)
       return std::make_pair((llvm::MemoryBuffer*)0, std::make_pair(0, true));    
     
@@ -1429,7 +1454,7 @@ llvm::MemoryBuffer *ASTUnit::getMainBufferWithPrecompiledPreamble(
         // buffer size we reserved when creating the preamble.
         return CreatePaddedMainFileBuffer(NewPreamble.first, 
                                           PreambleReservedSize,
-                                          FrontendOpts.Inputs[0].File);
+                                          FrontendOpts.Inputs[0].getFile());
       }
     }
 
@@ -1482,7 +1507,7 @@ llvm::MemoryBuffer *ASTUnit::getMainBufferWithPrecompiledPreamble(
 
   // Save the preamble text for later; we'll need to compare against it for
   // subsequent reparses.
-  StringRef MainFilename = PreambleInvocation->getFrontendOpts().Inputs[0].File;
+  StringRef MainFilename = PreambleInvocation->getFrontendOpts().Inputs[0].getFile();
   Preamble.assign(FileMgr->getFile(MainFilename),
                   NewPreamble.first->getBufferStart(), 
                   NewPreamble.first->getBufferStart() 
@@ -1492,7 +1517,7 @@ llvm::MemoryBuffer *ASTUnit::getMainBufferWithPrecompiledPreamble(
   delete PreambleBuffer;
   PreambleBuffer
     = llvm::MemoryBuffer::getNewUninitMemBuffer(PreambleReservedSize,
-                                                FrontendOpts.Inputs[0].File);
+                                                FrontendOpts.Inputs[0].getFile());
   memcpy(const_cast<char*>(PreambleBuffer->getBufferStart()), 
          NewPreamble.first->getBufferStart(), Preamble.size());
   memset(const_cast<char*>(PreambleBuffer->getBufferStart()) + Preamble.size(), 
@@ -1500,7 +1525,7 @@ llvm::MemoryBuffer *ASTUnit::getMainBufferWithPrecompiledPreamble(
   const_cast<char*>(PreambleBuffer->getBufferEnd())[-1] = '\n';  
   
   // Remap the main source file to the preamble buffer.
-  llvm::sys::PathWithStatus MainFilePath(FrontendOpts.Inputs[0].File);
+  llvm::sys::PathWithStatus MainFilePath(FrontendOpts.Inputs[0].getFile());
   PreprocessorOpts.addRemappedFile(MainFilePath.str(), PreambleBuffer);
   
   // Tell the compiler invocation to generate a temporary precompiled header.
@@ -1518,15 +1543,14 @@ llvm::MemoryBuffer *ASTUnit::getMainBufferWithPrecompiledPreamble(
     CICleanup(Clang.get());
 
   Clang->setInvocation(&*PreambleInvocation);
-  OriginalSourceFile = Clang->getFrontendOpts().Inputs[0].File;
+  OriginalSourceFile = Clang->getFrontendOpts().Inputs[0].getFile();
   
   // Set up diagnostics, capturing all of the diagnostics produced.
   Clang->setDiagnostics(&getDiagnostics());
   
   // Create the target instance.
-  Clang->getTargetOpts().Features = TargetFeatures;
   Clang->setTarget(TargetInfo::CreateTargetInfo(Clang->getDiagnostics(),
-                                               Clang->getTargetOpts()));
+                                                Clang->getTargetOpts()));
   if (!Clang->hasTarget()) {
     llvm::sys::Path(FrontendOpts.OutputFile).eraseFromDisk();
     Preamble.clear();
@@ -1544,9 +1568,9 @@ llvm::MemoryBuffer *ASTUnit::getMainBufferWithPrecompiledPreamble(
   
   assert(Clang->getFrontendOpts().Inputs.size() == 1 &&
          "Invocation must have exactly one source file!");
-  assert(Clang->getFrontendOpts().Inputs[0].Kind != IK_AST &&
+  assert(Clang->getFrontendOpts().Inputs[0].getKind() != IK_AST &&
          "FIXME: AST inputs not yet supported here!");
-  assert(Clang->getFrontendOpts().Inputs[0].Kind != IK_LLVM_IR &&
+  assert(Clang->getFrontendOpts().Inputs[0].getKind() != IK_LLVM_IR &&
          "IR inputs not support here!");
   
   // Clear out old caches and data.
@@ -1633,7 +1657,7 @@ llvm::MemoryBuffer *ASTUnit::getMainBufferWithPrecompiledPreamble(
   
   return CreatePaddedMainFileBuffer(NewPreamble.first, 
                                     PreambleReservedSize,
-                                    FrontendOpts.Inputs[0].File);
+                                    FrontendOpts.Inputs[0].getFile());
 }
 
 void ASTUnit::RealizeTopLevelDeclsFromPreamble() {
@@ -1664,7 +1688,7 @@ void ASTUnit::transferASTDataFromCompilerInstance(CompilerInstance &CI) {
 }
 
 StringRef ASTUnit::getMainFileName() const {
-  return Invocation->getFrontendOpts().Inputs[0].File;
+  return Invocation->getFrontendOpts().Inputs[0].getFile();
 }
 
 ASTUnit *ASTUnit::create(CompilerInvocation *CI,
@@ -1733,9 +1757,6 @@ ASTUnit *ASTUnit::LoadFromCompilerInvocationAction(CompilerInvocation *CI,
   CI->getFrontendOpts().DisableFree = false;
   ProcessWarningOptions(AST->getDiagnostics(), CI->getDiagnosticOpts());
 
-  // Save the target features.
-  AST->TargetFeatures = CI->getTargetOpts().Features;
-
   // Create the compiler instance to use for building the AST.
   OwningPtr<CompilerInstance> Clang(new CompilerInstance());
 
@@ -1744,14 +1765,13 @@ ASTUnit *ASTUnit::LoadFromCompilerInvocationAction(CompilerInvocation *CI,
     CICleanup(Clang.get());
 
   Clang->setInvocation(CI);
-  AST->OriginalSourceFile = Clang->getFrontendOpts().Inputs[0].File;
+  AST->OriginalSourceFile = Clang->getFrontendOpts().Inputs[0].getFile();
     
   // Set up diagnostics, capturing any diagnostics that would
   // otherwise be dropped.
   Clang->setDiagnostics(&AST->getDiagnostics());
   
   // Create the target instance.
-  Clang->getTargetOpts().Features = AST->TargetFeatures;
   Clang->setTarget(TargetInfo::CreateTargetInfo(Clang->getDiagnostics(),
                    Clang->getTargetOpts()));
   if (!Clang->hasTarget())
@@ -1765,9 +1785,9 @@ ASTUnit *ASTUnit::LoadFromCompilerInvocationAction(CompilerInvocation *CI,
   
   assert(Clang->getFrontendOpts().Inputs.size() == 1 &&
          "Invocation must have exactly one source file!");
-  assert(Clang->getFrontendOpts().Inputs[0].Kind != IK_AST &&
+  assert(Clang->getFrontendOpts().Inputs[0].getKind() != IK_AST &&
          "FIXME: AST inputs not yet supported here!");
-  assert(Clang->getFrontendOpts().Inputs[0].Kind != IK_LLVM_IR &&
+  assert(Clang->getFrontendOpts().Inputs[0].getKind() != IK_LLVM_IR &&
          "IR inputs not supported here!");
 
   // Configure the various subsystems.
@@ -1840,9 +1860,6 @@ bool ASTUnit::LoadFromCompilerInvocation(bool PrecompilePreamble) {
   Invocation->getFrontendOpts().DisableFree = false;
   ProcessWarningOptions(getDiagnostics(), Invocation->getDiagnosticOpts());
 
-  // Save the target features.
-  TargetFeatures = Invocation->getTargetOpts().Features;
-  
   llvm::MemoryBuffer *OverrideMainBuffer = 0;
   if (PrecompilePreamble) {
     PreambleRebuildCounter = 2;
@@ -1909,12 +1926,13 @@ ASTUnit *ASTUnit::LoadFromCommandLine(const char **ArgBegin,
                                       bool AllowPCHWithCompilerErrors,
                                       bool SkipFunctionBodies,
                                       bool UserFilesAreVolatile,
+                                      bool ForSerialization,
                                       OwningPtr<ASTUnit> *ErrAST) {
   if (!Diags.getPtr()) {
     // No diagnostics engine was provided, so create our own diagnostics object
     // with the default options.
-    DiagnosticOptions DiagOpts;
-    Diags = CompilerInstance::createDiagnostics(DiagOpts, ArgEnd - ArgBegin, 
+    Diags = CompilerInstance::createDiagnostics(new DiagnosticOptions(),
+                                                ArgEnd - ArgBegin,
                                                 ArgBegin);
   }
 
@@ -1972,6 +1990,8 @@ ASTUnit *ASTUnit::LoadFromCommandLine(const char **ArgBegin,
   AST->NumStoredDiagnosticsFromDriver = StoredDiagnostics.size();
   AST->StoredDiagnostics.swap(StoredDiagnostics);
   AST->Invocation = CI;
+  if (ForSerialization)
+    AST->WriterData.reset(new ASTWriterData());
   CI = 0; // Zero out now to ease cleanup during crash recovery.
   
   // Recover resources if we crash before exiting this method.
@@ -2002,7 +2022,6 @@ bool ASTUnit::Reparse(RemappedFile *RemappedFiles, unsigned NumRemappedFiles) {
 
   // Remap files.
   PreprocessorOptions &PPOpts = Invocation->getPreprocessorOpts();
-  PPOpts.DisableStatCache = true;
   for (PreprocessorOptions::remapped_file_buffer_iterator 
          R = PPOpts.remapped_file_buffer_begin(),
          REnd = PPOpts.remapped_file_buffer_end();
@@ -2238,7 +2257,6 @@ void AugmentedCodeCompleteConsumer::ProcessCodeCompleteResults(Sema &S,
     
     // Adjust priority based on similar type classes.
     unsigned Priority = C->Priority;
-    CXCursorKind CursorKind = C->Kind;
     CodeCompletionString *Completion = C->Completion;
     if (!Context.getPreferredType().isNull()) {
       if (C->Kind == CXCursor_MacroDefinition) {
@@ -2272,12 +2290,11 @@ void AugmentedCodeCompleteConsumer::ProcessCodeCompleteResults(Sema &S,
       CodeCompletionBuilder Builder(getAllocator(), getCodeCompletionTUInfo(),
                                     CCP_CodePattern, C->Availability);
       Builder.AddTypedTextChunk(C->Completion->getTypedText());
-      CursorKind = CXCursor_NotImplemented;
       Priority = CCP_CodePattern;
       Completion = Builder.TakeString();
     }
     
-    AllResults.push_back(Result(Completion, Priority, CursorKind, 
+    AllResults.push_back(Result(Completion, Priority, C->Kind,
                                 C->Availability));
   }
   
@@ -2341,7 +2358,7 @@ void ASTUnit::CodeComplete(StringRef File, unsigned Line, unsigned Column,
     CICleanup(Clang.get());
 
   Clang->setInvocation(&*CCInvocation);
-  OriginalSourceFile = Clang->getFrontendOpts().Inputs[0].File;
+  OriginalSourceFile = Clang->getFrontendOpts().Inputs[0].getFile();
     
   // Set up diagnostics, capturing any diagnostics produced.
   Clang->setDiagnostics(&Diag);
@@ -2351,7 +2368,6 @@ void ASTUnit::CodeComplete(StringRef File, unsigned Line, unsigned Column,
                                     StoredDiagnostics);
   
   // Create the target instance.
-  Clang->getTargetOpts().Features = TargetFeatures;
   Clang->setTarget(TargetInfo::CreateTargetInfo(Clang->getDiagnostics(),
                                                Clang->getTargetOpts()));
   if (!Clang->hasTarget()) {
@@ -2367,9 +2383,9 @@ void ASTUnit::CodeComplete(StringRef File, unsigned Line, unsigned Column,
   
   assert(Clang->getFrontendOpts().Inputs.size() == 1 &&
          "Invocation must have exactly one source file!");
-  assert(Clang->getFrontendOpts().Inputs[0].Kind != IK_AST &&
+  assert(Clang->getFrontendOpts().Inputs[0].getKind() != IK_AST &&
          "FIXME: AST inputs not yet supported here!");
-  assert(Clang->getFrontendOpts().Inputs[0].Kind != IK_LLVM_IR &&
+  assert(Clang->getFrontendOpts().Inputs[0].getKind() != IK_LLVM_IR &&
          "IR inputs not support here!");
 
   
@@ -2398,8 +2414,6 @@ void ASTUnit::CodeComplete(StringRef File, unsigned Line, unsigned Column,
     = new AugmentedCodeCompleteConsumer(*this, Consumer, CodeCompleteOpts);
   Clang->setCodeCompletionConsumer(AugmentedConsumer);
 
-  Clang->getFrontendOpts().SkipFunctionBodies = true;
-
   // If we have a precompiled preamble, try to use it. We only allow
   // the use of the precompiled preamble if we're if the completion
   // point is within the main file, after the end of the precompiled
@@ -2420,7 +2434,6 @@ void ASTUnit::CodeComplete(StringRef File, unsigned Line, unsigned Column,
 
   // If the main file has been overridden due to the use of a preamble,
   // make that override happen and introduce the preamble.
-  PreprocessorOpts.DisableStatCache = true;
   StoredDiagnostics.insert(StoredDiagnostics.end(),
                            stored_diag_begin(),
                            stored_diag_afterDriver_begin());
@@ -2438,8 +2451,9 @@ void ASTUnit::CodeComplete(StringRef File, unsigned Line, unsigned Column,
     PreprocessorOpts.PrecompiledPreambleBytes.second = false;
   }
 
-  // Disable the preprocessing record
-  PreprocessorOpts.DetailedRecord = false;
+  // Disable the preprocessing record if modules are not enabled.
+  if (!Clang->getLangOpts().Modules)
+    PreprocessorOpts.DetailedRecord = false;
   
   OwningPtr<SyntaxOnlyAction> Act;
   Act.reset(new SyntaxOnlyAction);
@@ -2457,7 +2471,7 @@ void ASTUnit::CodeComplete(StringRef File, unsigned Line, unsigned Column,
   checkAndSanitizeDiags(StoredDiagnostics, getSourceManager());
 }
 
-CXSaveError ASTUnit::Save(StringRef File) {
+bool ASTUnit::Save(StringRef File) {
   // Write to a temporary file and later rename it to the actual file, to avoid
   // possible race conditions.
   SmallString<128> TempPath;
@@ -2466,7 +2480,7 @@ CXSaveError ASTUnit::Save(StringRef File) {
   int fd;
   if (llvm::sys::fs::unique_file(TempPath.str(), fd, TempPath,
                                  /*makeAbsolute=*/false))
-    return CXSaveError_Unknown;
+    return true;
 
   // FIXME: Can we somehow regenerate the stat cache here, or do we need to 
   // unconditionally create a stat cache when we parse the file?
@@ -2476,32 +2490,43 @@ CXSaveError ASTUnit::Save(StringRef File) {
   Out.close();
   if (Out.has_error()) {
     Out.clear_error();
-    return CXSaveError_Unknown;
+    return true;
   }
 
   if (llvm::sys::fs::rename(TempPath.str(), File)) {
     bool exists;
     llvm::sys::fs::remove(TempPath.str(), exists);
-    return CXSaveError_Unknown;
+    return true;
   }
 
-  return CXSaveError_None;
+  return false;
+}
+
+static bool serializeUnit(ASTWriter &Writer,
+                          SmallVectorImpl<char> &Buffer,
+                          Sema &S,
+                          bool hasErrors,
+                          raw_ostream &OS) {
+  Writer.WriteAST(S, std::string(), 0, "", hasErrors);
+
+  // Write the generated bitstream to "Out".
+  if (!Buffer.empty())
+    OS.write(Buffer.data(), Buffer.size());
+
+  return false;
 }
 
 bool ASTUnit::serialize(raw_ostream &OS) {
   bool hasErrors = getDiagnostics().hasErrorOccurred();
 
+  if (WriterData)
+    return serializeUnit(WriterData->Writer, WriterData->Buffer,
+                         getSema(), hasErrors, OS);
+
   SmallString<128> Buffer;
   llvm::BitstreamWriter Stream(Buffer);
   ASTWriter Writer(Stream);
-  // FIXME: Handle modules
-  Writer.WriteAST(getSema(), 0, std::string(), 0, "", hasErrors);
-  
-  // Write the generated bitstream to "Out".
-  if (!Buffer.empty())
-    OS.write((char *)&Buffer.front(), Buffer.size());
-
-  return false;
+  return serializeUnit(Writer, Buffer, getSema(), hasErrors, OS);
 }
 
 typedef ContinuousRangeMap<unsigned, int, 2> SLocRemap;
@@ -2761,6 +2786,85 @@ SourceLocation ASTUnit::getStartOfMainFileID() {
   return SourceMgr->getLocForStartOfFile(FID);
 }
 
+std::pair<PreprocessingRecord::iterator, PreprocessingRecord::iterator>
+ASTUnit::getLocalPreprocessingEntities() const {
+  if (isMainFileAST()) {
+    serialization::ModuleFile &
+      Mod = Reader->getModuleManager().getPrimaryModule();
+    return Reader->getModulePreprocessedEntities(Mod);
+  }
+
+  if (PreprocessingRecord *PPRec = PP->getPreprocessingRecord())
+    return std::make_pair(PPRec->local_begin(), PPRec->local_end());
+
+  return std::make_pair(PreprocessingRecord::iterator(),
+                        PreprocessingRecord::iterator());
+}
+
+bool ASTUnit::visitLocalTopLevelDecls(void *context, DeclVisitorFn Fn) {
+  if (isMainFileAST()) {
+    serialization::ModuleFile &
+      Mod = Reader->getModuleManager().getPrimaryModule();
+    ASTReader::ModuleDeclIterator MDI, MDE;
+    llvm::tie(MDI, MDE) = Reader->getModuleFileLevelDecls(Mod);
+    for (; MDI != MDE; ++MDI) {
+      if (!Fn(context, *MDI))
+        return false;
+    }
+
+    return true;
+  }
+
+  for (ASTUnit::top_level_iterator TL = top_level_begin(),
+                                TLEnd = top_level_end();
+         TL != TLEnd; ++TL) {
+    if (!Fn(context, *TL))
+      return false;
+  }
+
+  return true;
+}
+
+namespace {
+struct PCHLocatorInfo {
+  serialization::ModuleFile *Mod;
+  PCHLocatorInfo() : Mod(0) {}
+};
+}
+
+static bool PCHLocator(serialization::ModuleFile &M, void *UserData) {
+  PCHLocatorInfo &Info = *static_cast<PCHLocatorInfo*>(UserData);
+  switch (M.Kind) {
+  case serialization::MK_Module:
+    return true; // skip dependencies.
+  case serialization::MK_PCH:
+    Info.Mod = &M;
+    return true; // found it.
+  case serialization::MK_Preamble:
+    return false; // look in dependencies.
+  case serialization::MK_MainFile:
+    return false; // look in dependencies.
+  }
+
+  return true;
+}
+
+const FileEntry *ASTUnit::getPCHFile() {
+  if (!Reader)
+    return 0;
+
+  PCHLocatorInfo Info;
+  Reader->getModuleManager().visit(PCHLocator, &Info);
+  if (Info.Mod)
+    return Info.Mod->File;
+
+  return 0;
+}
+
+bool ASTUnit::isModuleFile() {
+  return isMainFileAST() && !ASTFileLangOpts.CurrentModule.empty();
+}
+
 void ASTUnit::PreambleData::countLines() const {
   NumLines = 0;
   if (empty())
diff --git a/lib/Frontend/ChainedDiagnosticConsumer.cpp b/lib/Frontend/ChainedDiagnosticConsumer.cpp
index c1d3db8..d77fd18 100644
--- a/lib/Frontend/ChainedDiagnosticConsumer.cpp
+++ b/lib/Frontend/ChainedDiagnosticConsumer.cpp
@@ -1,4 +1,4 @@
-//===- ChainedDiagnosticConsumer.cpp - Chain Diagnostic Clients -*- C++ -*-===//
+//===- ChainedDiagnosticConsumer.cpp - Chain Diagnostic Clients -----------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
diff --git a/lib/Frontend/ChainedIncludesSource.cpp b/lib/Frontend/ChainedIncludesSource.cpp
index dbb06bd..2d58640 100644
--- a/lib/Frontend/ChainedIncludesSource.cpp
+++ b/lib/Frontend/ChainedIncludesSource.cpp
@@ -39,14 +39,18 @@ static ASTReader *createASTReader(CompilerInstance &CI,
     Reader->addInMemoryBuffer(sr, memBufs[ti]);
   }
   Reader->setDeserializationListener(deserialListener);
-  switch (Reader->ReadAST(pchFile, serialization::MK_PCH)) {
+  switch (Reader->ReadAST(pchFile, serialization::MK_PCH,
+                          ASTReader::ARR_None)) {
   case ASTReader::Success:
     // Set the predefines buffer as suggested by the PCH reader.
     PP.setPredefines(Reader->getSuggestedPredefines());
     return Reader.take();
 
   case ASTReader::Failure:
-  case ASTReader::IgnorePCH:
+  case ASTReader::OutOfDate:
+  case ASTReader::VersionMismatch:
+  case ASTReader::ConfigurationMismatch:
+  case ASTReader::HadErrors:
     break;
   }
   return 0;
@@ -63,7 +67,7 @@ ChainedIncludesSource *ChainedIncludesSource::create(CompilerInstance &CI) {
   assert(!includes.empty() && "No '-chain-include' in options!");
 
   OwningPtr<ChainedIncludesSource> source(new ChainedIncludesSource());
-  InputKind IK = CI.getFrontendOpts().Inputs[0].Kind;
+  InputKind IK = CI.getFrontendOpts().Inputs[0].getKind();
 
   SmallVector<llvm::MemoryBuffer *, 4> serialBufs;
   SmallVector<std::string, 4> serialBufNames;
@@ -82,14 +86,14 @@ ChainedIncludesSource *ChainedIncludesSource::create(CompilerInstance &CI) {
     CInvok->getPreprocessorOpts().Macros.clear();
     
     CInvok->getFrontendOpts().Inputs.clear();
-    CInvok->getFrontendOpts().Inputs.push_back(FrontendInputFile(includes[i],
-                                                                 IK));
+    FrontendInputFile InputFile(includes[i], IK);
+    CInvok->getFrontendOpts().Inputs.push_back(InputFile);
 
     TextDiagnosticPrinter *DiagClient =
-      new TextDiagnosticPrinter(llvm::errs(), DiagnosticOptions());
+      new TextDiagnosticPrinter(llvm::errs(), new DiagnosticOptions());
     IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
     IntrusiveRefCntPtr<DiagnosticsEngine> Diags(
-        new DiagnosticsEngine(DiagID, DiagClient));
+        new DiagnosticsEngine(DiagID, &CI.getDiagnosticOpts(), DiagClient));
 
     OwningPtr<CompilerInstance> Clang(new CompilerInstance());
     Clang->setInvocation(CInvok.take());
@@ -108,6 +112,8 @@ ChainedIncludesSource *ChainedIncludesSource::create(CompilerInstance &CI) {
     OwningPtr<ASTConsumer> consumer;
     consumer.reset(new PCHGenerator(Clang->getPreprocessor(), "-", 0,
                                     /*isysroot=*/"", &OS));
+    Clang->getPreprocessor().setPPMutationListener(
+                                            consumer->GetPPMutationListener());
     Clang->getASTContext().setASTMutationListener(
                                             consumer->GetASTMutationListener());
     Clang->setASTConsumer(consumer.take());
@@ -141,7 +147,7 @@ ChainedIncludesSource *ChainedIncludesSource::create(CompilerInstance &CI) {
       Clang->getASTContext().setExternalSource(Reader);
     }
     
-    if (!Clang->InitializeSourceManager(includes[i]))
+    if (!Clang->InitializeSourceManager(InputFile))
       return 0;
 
     ParseAST(Clang->getSema());
diff --git a/lib/Frontend/CompilerInstance.cpp b/lib/Frontend/CompilerInstance.cpp
index 6de1531..22a74fc 100644
--- a/lib/Frontend/CompilerInstance.cpp
+++ b/lib/Frontend/CompilerInstance.cpp
@@ -52,6 +52,7 @@ CompilerInstance::CompilerInstance()
 }
 
 CompilerInstance::~CompilerInstance() {
+  assert(OutputFiles.empty() && "Still output files in flight?");
 }
 
 void CompilerInstance::setInvocation(CompilerInvocation *Value) {
@@ -88,19 +89,18 @@ void CompilerInstance::setASTConsumer(ASTConsumer *Value) {
 
 void CompilerInstance::setCodeCompletionConsumer(CodeCompleteConsumer *Value) {
   CompletionConsumer.reset(Value);
-  getFrontendOpts().SkipFunctionBodies = Value != 0;
 }
 
 // Diagnostics
-static void SetUpBuildDumpLog(const DiagnosticOptions &DiagOpts,
+static void SetUpBuildDumpLog(DiagnosticOptions *DiagOpts,
                               unsigned argc, const char* const *argv,
                               DiagnosticsEngine &Diags) {
   std::string ErrorInfo;
   OwningPtr<raw_ostream> OS(
-    new llvm::raw_fd_ostream(DiagOpts.DumpBuildInformation.c_str(), ErrorInfo));
+    new llvm::raw_fd_ostream(DiagOpts->DumpBuildInformation.c_str(),ErrorInfo));
   if (!ErrorInfo.empty()) {
     Diags.Report(diag::err_fe_unable_to_open_logfile)
-                 << DiagOpts.DumpBuildInformation << ErrorInfo;
+                 << DiagOpts->DumpBuildInformation << ErrorInfo;
     return;
   }
 
@@ -115,20 +115,20 @@ static void SetUpBuildDumpLog(const DiagnosticOptions &DiagOpts,
   Diags.setClient(new ChainedDiagnosticConsumer(Diags.takeClient(), Logger));
 }
 
-static void SetUpDiagnosticLog(const DiagnosticOptions &DiagOpts,
+static void SetUpDiagnosticLog(DiagnosticOptions *DiagOpts,
                                const CodeGenOptions *CodeGenOpts,
                                DiagnosticsEngine &Diags) {
   std::string ErrorInfo;
   bool OwnsStream = false;
   raw_ostream *OS = &llvm::errs();
-  if (DiagOpts.DiagnosticLogFile != "-") {
+  if (DiagOpts->DiagnosticLogFile != "-") {
     // Create the output stream.
     llvm::raw_fd_ostream *FileOS(
-      new llvm::raw_fd_ostream(DiagOpts.DiagnosticLogFile.c_str(),
+      new llvm::raw_fd_ostream(DiagOpts->DiagnosticLogFile.c_str(),
                                ErrorInfo, llvm::raw_fd_ostream::F_Append));
     if (!ErrorInfo.empty()) {
       Diags.Report(diag::warn_fe_cc_log_diagnostics_failure)
-        << DiagOpts.DumpBuildInformation << ErrorInfo;
+        << DiagOpts->DumpBuildInformation << ErrorInfo;
     } else {
       FileOS->SetUnbuffered();
       FileOS->SetUseAtomicWrites(true);
@@ -145,7 +145,7 @@ static void SetUpDiagnosticLog(const DiagnosticOptions &DiagOpts,
   Diags.setClient(new ChainedDiagnosticConsumer(Diags.takeClient(), Logger));
 }
 
-static void SetupSerializedDiagnostics(const DiagnosticOptions &DiagOpts,
+static void SetupSerializedDiagnostics(DiagnosticOptions *DiagOpts,
                                        DiagnosticsEngine &Diags,
                                        StringRef OutputFile) {
   std::string ErrorInfo;
@@ -171,13 +171,13 @@ void CompilerInstance::createDiagnostics(int Argc, const char* const *Argv,
                                          DiagnosticConsumer *Client,
                                          bool ShouldOwnClient,
                                          bool ShouldCloneClient) {
-  Diagnostics = createDiagnostics(getDiagnosticOpts(), Argc, Argv, Client,
+  Diagnostics = createDiagnostics(&getDiagnosticOpts(), Argc, Argv, Client,
                                   ShouldOwnClient, ShouldCloneClient,
                                   &getCodeGenOpts());
 }
 
 IntrusiveRefCntPtr<DiagnosticsEngine>
-CompilerInstance::createDiagnostics(const DiagnosticOptions &Opts,
+CompilerInstance::createDiagnostics(DiagnosticOptions *Opts,
                                     int Argc, const char* const *Argv,
                                     DiagnosticConsumer *Client,
                                     bool ShouldOwnClient,
@@ -185,7 +185,7 @@ CompilerInstance::createDiagnostics(const DiagnosticOptions &Opts,
                                     const CodeGenOptions *CodeGenOpts) {
   IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
   IntrusiveRefCntPtr<DiagnosticsEngine>
-      Diags(new DiagnosticsEngine(DiagID));
+      Diags(new DiagnosticsEngine(DiagID, Opts));
 
   // Create the diagnostic client for reporting errors or for
   // implementing -verify.
@@ -198,22 +198,22 @@ CompilerInstance::createDiagnostics(const DiagnosticOptions &Opts,
     Diags->setClient(new TextDiagnosticPrinter(llvm::errs(), Opts));
 
   // Chain in -verify checker, if requested.
-  if (Opts.VerifyDiagnostics)
+  if (Opts->VerifyDiagnostics)
     Diags->setClient(new VerifyDiagnosticConsumer(*Diags));
 
   // Chain in -diagnostic-log-file dumper, if requested.
-  if (!Opts.DiagnosticLogFile.empty())
+  if (!Opts->DiagnosticLogFile.empty())
     SetUpDiagnosticLog(Opts, CodeGenOpts, *Diags);
 
-  if (!Opts.DumpBuildInformation.empty())
+  if (!Opts->DumpBuildInformation.empty())
     SetUpBuildDumpLog(Opts, Argc, Argv, *Diags);
 
-  if (!Opts.DiagnosticSerializationFile.empty())
+  if (!Opts->DiagnosticSerializationFile.empty())
     SetupSerializedDiagnostics(Opts, *Diags,
-                               Opts.DiagnosticSerializationFile);
+                               Opts->DiagnosticSerializationFile);
   
   // Configure our handling of diagnostics.
-  ProcessWarningOptions(*Diags, Opts);
+  ProcessWarningOptions(*Diags, *Opts);
 
   return Diags;
 }
@@ -241,11 +241,13 @@ void CompilerInstance::createPreprocessor() {
     PTHMgr = PTHManager::Create(PPOpts.TokenCache, getDiagnostics());
 
   // Create the Preprocessor.
-  HeaderSearch *HeaderInfo = new HeaderSearch(getFileManager(), 
+  HeaderSearch *HeaderInfo = new HeaderSearch(&getHeaderSearchOpts(),
+                                              getFileManager(),
                                               getDiagnostics(),
                                               getLangOpts(),
                                               &getTarget());
-  PP = new Preprocessor(getDiagnostics(), getLangOpts(), &getTarget(),
+  PP = new Preprocessor(&getPreprocessorOpts(),
+                        getDiagnostics(), getLangOpts(), &getTarget(),
                         getSourceManager(), *HeaderInfo, *this, PTHMgr,
                         /*OwnsHeaderSearch=*/true);
 
@@ -306,14 +308,12 @@ void CompilerInstance::createASTContext() {
 
 void CompilerInstance::createPCHExternalASTSource(StringRef Path,
                                                   bool DisablePCHValidation,
-                                                  bool DisableStatCache,
                                                 bool AllowPCHWithCompilerErrors,
                                                  void *DeserializationListener){
   OwningPtr<ExternalASTSource> Source;
   bool Preamble = getPreprocessorOpts().PrecompiledPreambleBytes.first != 0;
   Source.reset(createPCHExternalASTSource(Path, getHeaderSearchOpts().Sysroot,
                                           DisablePCHValidation,
-                                          DisableStatCache,
                                           AllowPCHWithCompilerErrors,
                                           getPreprocessor(), getASTContext(),
                                           DeserializationListener,
@@ -326,7 +326,6 @@ ExternalASTSource *
 CompilerInstance::createPCHExternalASTSource(StringRef Path,
                                              const std::string &Sysroot,
                                              bool DisablePCHValidation,
-                                             bool DisableStatCache,
                                              bool AllowPCHWithCompilerErrors,
                                              Preprocessor &PP,
                                              ASTContext &Context,
@@ -335,14 +334,15 @@ CompilerInstance::createPCHExternalASTSource(StringRef Path,
   OwningPtr<ASTReader> Reader;
   Reader.reset(new ASTReader(PP, Context,
                              Sysroot.empty() ? "" : Sysroot.c_str(),
-                             DisablePCHValidation, DisableStatCache,
+                             DisablePCHValidation,
                              AllowPCHWithCompilerErrors));
 
   Reader->setDeserializationListener(
             static_cast<ASTDeserializationListener *>(DeserializationListener));
   switch (Reader->ReadAST(Path,
                           Preamble ? serialization::MK_Preamble
-                                   : serialization::MK_PCH)) {
+                                   : serialization::MK_PCH,
+                          ASTReader::ARR_None)) {
   case ASTReader::Success:
     // Set the predefines buffer as suggested by the PCH reader. Typically, the
     // predefines buffer will be empty.
@@ -353,7 +353,10 @@ CompilerInstance::createPCHExternalASTSource(StringRef Path,
     // Unrecoverable failure: don't even try to process the input file.
     break;
 
-  case ASTReader::IgnorePCH:
+  case ASTReader::OutOfDate:
+  case ASTReader::VersionMismatch:
+  case ASTReader::ConfigurationMismatch:
+  case ASTReader::HadErrors:
     // No suitable PCH file could be found. Return an error.
     break;
   }
@@ -586,19 +589,29 @@ CompilerInstance::createOutputFile(StringRef OutputPath,
 
 // Initialization Utilities
 
-bool CompilerInstance::InitializeSourceManager(StringRef InputFile,
-                                               SrcMgr::CharacteristicKind Kind){
-  return InitializeSourceManager(InputFile, Kind, getDiagnostics(), 
+bool CompilerInstance::InitializeSourceManager(const FrontendInputFile &Input){
+  return InitializeSourceManager(Input, getDiagnostics(),
                                  getFileManager(), getSourceManager(), 
                                  getFrontendOpts());
 }
 
-bool CompilerInstance::InitializeSourceManager(StringRef InputFile,
-                                               SrcMgr::CharacteristicKind Kind,
+bool CompilerInstance::InitializeSourceManager(const FrontendInputFile &Input,
                                                DiagnosticsEngine &Diags,
                                                FileManager &FileMgr,
                                                SourceManager &SourceMgr,
                                                const FrontendOptions &Opts) {
+  SrcMgr::CharacteristicKind
+    Kind = Input.isSystem() ? SrcMgr::C_System : SrcMgr::C_User;
+
+  if (Input.isBuffer()) {
+    SourceMgr.createMainFileIDForMemBuffer(Input.getBuffer(), Kind);
+    assert(!SourceMgr.getMainFileID().isInvalid() &&
+           "Couldn't establish MainFileID!");
+    return true;
+  }
+
+  StringRef InputFile = Input.getFile();
+
   // Figure out where to get and map in the main file.
   if (InputFile != "-") {
     const FileEntry *File = FileMgr.getFile(InputFile);
@@ -607,6 +620,19 @@ bool CompilerInstance::InitializeSourceManager(StringRef InputFile,
       return false;
     }
     SourceMgr.createMainFileID(File, Kind);
+
+    // The natural SourceManager infrastructure can't currently handle named
+    // pipes, but we would at least like to accept them for the main
+    // file. Detect them here, read them with the more generic MemoryBuffer
+    // function, and simply override their contents as we do for STDIN.
+    if (File->isNamedPipe()) {
+      OwningPtr<llvm::MemoryBuffer> MB;
+      if (llvm::error_code ec = llvm::MemoryBuffer::getFile(InputFile, MB)) {
+        Diags.Report(diag::err_cannot_open_file) << InputFile << ec.message();
+        return false;
+      }
+      SourceMgr.overrideFileContents(File, MB.take());
+    }
   } else {
     OwningPtr<llvm::MemoryBuffer> SB;
     if (llvm::MemoryBuffer::getSTDIN(SB)) {
@@ -746,7 +772,7 @@ static void compileModule(CompilerInstance &ImportingInstance,
     // Someone else is responsible for building the module. Wait for them to
     // finish.
     Locked.waitForUnlock();
-    break;
+    return;
   }
 
   ModuleMap &ModMap 
@@ -836,6 +862,7 @@ static void compileModule(CompilerInstance &ImportingInstance,
   // FIXME: Even though we're executing under crash protection, it would still
   // be nice to do this with RemoveFileOnSignal when we can. However, that
   // doesn't make sense for all clients, so clean this up manually.
+  Instance.clearOutputFiles(/*EraseFiles=*/true);
   if (!TempModuleMapFileName.empty())
     llvm::sys::Path(TempModuleMapFileName).eraseFromDisk();
 }
@@ -939,13 +966,14 @@ Module *CompilerInstance::loadModule(SourceLocation ImportLoc,
       const PreprocessorOptions &PPOpts = getPreprocessorOpts();
       ModuleManager = new ASTReader(getPreprocessor(), *Context,
                                     Sysroot.empty() ? "" : Sysroot.c_str(),
-                                    PPOpts.DisablePCHValidation,
-                                    PPOpts.DisableStatCache);
+                                    PPOpts.DisablePCHValidation);
       if (hasASTConsumer()) {
         ModuleManager->setDeserializationListener(
           getASTConsumer().GetASTDeserializationListener());
         getASTContext().setASTMutationListener(
           getASTConsumer().GetASTMutationListener());
+        getPreprocessor().setPPMutationListener(
+          getASTConsumer().GetPPMutationListener());
       }
       OwningPtr<ExternalASTSource> Source;
       Source.reset(ModuleManager);
@@ -957,12 +985,39 @@ Module *CompilerInstance::loadModule(SourceLocation ImportLoc,
     }
 
     // Try to load the module we found.
+    unsigned ARRFlags = ASTReader::ARR_None;
+    if (Module)
+      ARRFlags |= ASTReader::ARR_OutOfDate;
     switch (ModuleManager->ReadAST(ModuleFile->getName(),
-                                   serialization::MK_Module)) {
+                                   serialization::MK_Module,
+                                   ARRFlags)) {
     case ASTReader::Success:
       break;
 
-    case ASTReader::IgnorePCH:
+    case ASTReader::OutOfDate: {
+      // The module file is out-of-date. Rebuild it.
+      getFileManager().invalidateCache(ModuleFile);
+      bool Existed;
+      llvm::sys::fs::remove(ModuleFileName, Existed);
+      compileModule(*this, Module, ModuleFileName);
+
+      // Try loading the module again.
+      ModuleFile = FileMgr->getFile(ModuleFileName);
+      if (!ModuleFile ||
+          ModuleManager->ReadAST(ModuleFileName,
+                                 serialization::MK_Module,
+                                 ASTReader::ARR_None) != ASTReader::Success) {
+        KnownModules[Path[0].first] = 0;
+        return 0;
+      }
+
+      // Okay, we've rebuilt and now loaded the module.
+      break;
+    }
+
+    case ASTReader::VersionMismatch:
+    case ASTReader::ConfigurationMismatch:
+    case ASTReader::HadErrors:
       // FIXME: The ASTReader will already have complained, but can we showhorn
       // that diagnostic information into a more useful form?
       KnownModules[Path[0].first] = 0;
@@ -980,6 +1035,9 @@ Module *CompilerInstance::loadModule(SourceLocation ImportLoc,
       Module = PP->getHeaderSearchInfo().getModuleMap()
                  .findModule((Path[0].first->getName()));
     }
+
+    if (Module)
+      Module->setASTFile(ModuleFile);
     
     // Cache the result of this top-level module lookup for later.
     Known = KnownModules.insert(std::make_pair(Path[0].first, Module)).first;
@@ -1079,9 +1137,12 @@ Module *CompilerInstance::loadModule(SourceLocation ImportLoc,
   // implicit import declaration to capture it in the AST.
   if (IsInclusionDirective && hasASTContext()) {
     TranslationUnitDecl *TU = getASTContext().getTranslationUnitDecl();
-    TU->addDecl(ImportDecl::CreateImplicit(getASTContext(), TU,
-                                           ImportLoc, Module, 
-                                           Path.back().second));
+    ImportDecl *ImportD = ImportDecl::CreateImplicit(getASTContext(), TU,
+                                                     ImportLoc, Module,
+                                                     Path.back().second);
+    TU->addDecl(ImportD);
+    if (Consumer)
+      Consumer->HandleImplicitImportDecl(ImportD);
   }
   
   LastModuleImportLoc = ImportLoc;
diff --git a/lib/Frontend/CompilerInvocation.cpp b/lib/Frontend/CompilerInvocation.cpp
index d39679c..b9c198b 100644
--- a/lib/Frontend/CompilerInvocation.cpp
+++ b/lib/Frontend/CompilerInvocation.cpp
@@ -11,6 +11,7 @@
 #include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/Version.h"
 #include "clang/Basic/FileManager.h"
+#include "clang/Lex/HeaderSearchOptions.h"
 #include "clang/Driver/Arg.h"
 #include "clang/Driver/ArgList.h"
 #include "clang/Driver/Options.h"
@@ -20,6 +21,7 @@
 #include "clang/Frontend/CompilerInvocation.h"
 #include "clang/Frontend/LangStandard.h"
 #include "clang/Serialization/ASTReader.h"
+#include "llvm/ADT/Hashing.h"
 #include "llvm/ADT/OwningPtr.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
@@ -35,917 +37,21 @@ using namespace clang;
 //===----------------------------------------------------------------------===//
 
 CompilerInvocationBase::CompilerInvocationBase()
-  : LangOpts(new LangOptions()) {}
+  : LangOpts(new LangOptions()), TargetOpts(new TargetOptions()),
+    DiagnosticOpts(new DiagnosticOptions()),
+    HeaderSearchOpts(new HeaderSearchOptions()),
+    PreprocessorOpts(new PreprocessorOptions()) {}
 
 CompilerInvocationBase::CompilerInvocationBase(const CompilerInvocationBase &X)
   : RefCountedBase<CompilerInvocation>(),
-    LangOpts(new LangOptions(*X.getLangOpts())) {}
+    LangOpts(new LangOptions(*X.getLangOpts())), 
+    TargetOpts(new TargetOptions(X.getTargetOpts())),
+    DiagnosticOpts(new DiagnosticOptions(X.getDiagnosticOpts())),
+    HeaderSearchOpts(new HeaderSearchOptions(X.getHeaderSearchOpts())),
+    PreprocessorOpts(new PreprocessorOptions(X.getPreprocessorOpts())) {}
 
 //===----------------------------------------------------------------------===//
-// Utility functions.
-//===----------------------------------------------------------------------===//
-
-static const char *getAnalysisStoreName(AnalysisStores Kind) {
-  switch (Kind) {
-  default:
-    llvm_unreachable("Unknown analysis store!");
-#define ANALYSIS_STORE(NAME, CMDFLAG, DESC, CREATFN) \
-  case NAME##Model: return CMDFLAG;
-#include "clang/Frontend/Analyses.def"
-  }
-}
-
-static const char *getAnalysisConstraintName(AnalysisConstraints Kind) {
-  switch (Kind) {
-  default:
-    llvm_unreachable("Unknown analysis constraints!");
-#define ANALYSIS_CONSTRAINTS(NAME, CMDFLAG, DESC, CREATFN) \
-  case NAME##Model: return CMDFLAG;
-#include "clang/Frontend/Analyses.def"
-  }
-}
-
-static const char *getAnalysisDiagClientName(AnalysisDiagClients Kind) {
-  switch (Kind) {
-  default:
-    llvm_unreachable("Unknown analysis client!");
-#define ANALYSIS_DIAGNOSTICS(NAME, CMDFLAG, DESC, CREATFN, AUTOCREATE) \
-  case PD_##NAME: return CMDFLAG;
-#include "clang/Frontend/Analyses.def"
-  }
-}
-
-static const char *getAnalysisPurgeModeName(AnalysisPurgeMode Kind) {
-  switch (Kind) {
-  default:
-    llvm_unreachable("Unknown analysis purge mode!");
-#define ANALYSIS_PURGE(NAME, CMDFLAG, DESC) \
-  case NAME: return CMDFLAG;
-#include "clang/Frontend/Analyses.def"
-  }
-}
-
-static const char *getAnalysisIPAModeName(AnalysisIPAMode Kind) {
-  switch (Kind) {
-  default:
-    llvm_unreachable("Unknown analysis ipa mode!");
-#define ANALYSIS_IPA(NAME, CMDFLAG, DESC) \
-  case NAME: return CMDFLAG;
-#include "clang/Frontend/Analyses.def"
-  }
-}
-
-static const char *
-  getAnalysisInliningModeName(AnalysisInliningMode Kind) {
-  switch (Kind) {
-  default:
-    llvm_unreachable("Unknown analysis inlining mode!");
-#define ANALYSIS_INLINE_SELECTION(NAME, CMDFLAG, DESC) \
-  case NAME: return CMDFLAG;
-#include "clang/Frontend/Analyses.def"
-  }
-}
-
-//===----------------------------------------------------------------------===//
-// Serialization (to args)
-//===----------------------------------------------------------------------===//
-
-namespace {
-  /// ToArgsList - Helper class to create a list of std::strings.
-  class ToArgsList {
-    std::vector<std::string> &Res;
-  public:
-    explicit ToArgsList(std::vector<std::string> &Res) : Res(Res) {}
-
-    void push_back(StringRef Str) {
-      // Avoid creating a temporary string.
-      Res.push_back(std::string());
-      Res.back().assign(Str.data(), Str.size());
-    }
-
-    void push_back(StringRef Str1, StringRef Str2) {
-      push_back(Str1);
-      push_back(Str2);
-    }
-  };
-}
-
-static void AnalyzerOptsToArgs(const AnalyzerOptions &Opts, ToArgsList &Res) {
-  if (Opts.ShowCheckerHelp)
-    Res.push_back("-analyzer-checker-help");
-  if (Opts.AnalysisStoreOpt != RegionStoreModel)
-    Res.push_back("-analyzer-store",
-                  getAnalysisStoreName(Opts.AnalysisStoreOpt));
-  if (Opts.AnalysisConstraintsOpt != RangeConstraintsModel)
-    Res.push_back("-analyzer-constraints",
-                  getAnalysisConstraintName(Opts.AnalysisConstraintsOpt));
-  if (Opts.AnalysisDiagOpt != PD_HTML)
-    Res.push_back("-analyzer-output",
-                  getAnalysisDiagClientName(Opts.AnalysisDiagOpt));
-  if (Opts.AnalysisPurgeOpt != PurgeStmt)
-    Res.push_back("-analyzer-purge",
-                  getAnalysisPurgeModeName(Opts.AnalysisPurgeOpt));
-  if (!Opts.AnalyzeSpecificFunction.empty())
-    Res.push_back("-analyze-function", Opts.AnalyzeSpecificFunction);
-  if (Opts.IPAMode != Inlining)
-    Res.push_back("-analyzer-ipa", getAnalysisIPAModeName(Opts.IPAMode));
-  if (Opts.InliningMode != NoRedundancy)
-    Res.push_back("-analyzer-inlining-mode",
-                  getAnalysisInliningModeName(Opts.InliningMode));
-
-  if (Opts.AnalyzeAll)
-    Res.push_back("-analyzer-opt-analyze-headers");
-  if (Opts.AnalyzerDisplayProgress)
-    Res.push_back("-analyzer-display-progress");
-  if (Opts.AnalyzeNestedBlocks)
-    Res.push_back("-analyzer-opt-analyze-nested-blocks");
-  if (Opts.EagerlyAssume)
-    Res.push_back("-analyzer-eagerly-assume");
-  if (Opts.TrimGraph)
-    Res.push_back("-trim-egraph");
-  if (Opts.VisualizeEGDot)
-    Res.push_back("-analyzer-viz-egraph-graphviz");
-  if (Opts.VisualizeEGUbi)
-    Res.push_back("-analyzer-viz-egraph-ubigraph");
-  if (Opts.NoRetryExhausted)
-    Res.push_back("-analyzer-disable-retry-exhausted");
-
-  for (unsigned i = 0, e = Opts.CheckersControlList.size(); i != e; ++i) {
-    const std::pair<std::string, bool> &opt = Opts.CheckersControlList[i];
-    if (opt.second)
-      Res.push_back("-analyzer-disable-checker");
-    else
-      Res.push_back("-analyzer-checker");
-    Res.push_back(opt.first);
-  }
-}
-
-static void CodeGenOptsToArgs(const CodeGenOptions &Opts, ToArgsList &Res) {
-  switch (Opts.DebugInfo) {
-    case CodeGenOptions::NoDebugInfo:
-      break;
-    case CodeGenOptions::DebugLineTablesOnly:
-      Res.push_back("-gline-tables-only");
-      break;
-    case CodeGenOptions::LimitedDebugInfo:
-      Res.push_back("-g");
-      Res.push_back("-flimit-debug-info");
-      break;
-    case CodeGenOptions::FullDebugInfo:
-      Res.push_back("-g");
-      Res.push_back("-fno-limit-debug-info");
-      break;
-  }
-  if (Opts.DisableLLVMOpts)
-    Res.push_back("-disable-llvm-optzns");
-  if (Opts.DisableRedZone)
-    Res.push_back("-disable-red-zone");
-  if (Opts.DisableTailCalls)
-    Res.push_back("-mdisable-tail-calls");
-  if (!Opts.DebugCompilationDir.empty())
-    Res.push_back("-fdebug-compilation-dir", Opts.DebugCompilationDir);
-  if (!Opts.DwarfDebugFlags.empty())
-    Res.push_back("-dwarf-debug-flags", Opts.DwarfDebugFlags);
-  if (Opts.EmitGcovArcs)
-    Res.push_back("-femit-coverage-data");
-  if (Opts.EmitGcovNotes)
-    Res.push_back("-femit-coverage-notes");
-  if (Opts.EmitOpenCLArgMetadata)
-    Res.push_back("-cl-kernel-arg-info");
-  if (!Opts.MergeAllConstants)
-    Res.push_back("-fno-merge-all-constants");
-  if (Opts.NoCommon)
-    Res.push_back("-fno-common");
-  if (Opts.ForbidGuardVariables)
-    Res.push_back("-fforbid-guard-variables");
-  if (Opts.UseRegisterSizedBitfieldAccess)
-    Res.push_back("-fuse-register-sized-bitfield-access");
-  if (Opts.NoImplicitFloat)
-    Res.push_back("-no-implicit-float");
-  if (Opts.OmitLeafFramePointer)
-    Res.push_back("-momit-leaf-frame-pointer");
-  if (Opts.OptimizeSize) {
-    assert(Opts.OptimizationLevel == 2 && "Invalid options!");
-    Opts.OptimizeSize == 1 ? Res.push_back("-Os") : Res.push_back("-Oz");
-  } else if (Opts.OptimizationLevel != 0)
-    Res.push_back("-O" + llvm::utostr(Opts.OptimizationLevel));
-  if (!Opts.MainFileName.empty())
-    Res.push_back("-main-file-name", Opts.MainFileName);
-  if (Opts.NoInfsFPMath)
-    Res.push_back("-menable-no-infinities");
-  if (Opts.NoNaNsFPMath)
-    Res.push_back("-menable-no-nans");
-  // SimplifyLibCalls is only derived.
-  // TimePasses is only derived.
-  // UnitAtATime is unused.
-  // Inlining is only derived.
-
-  // UnrollLoops is derived, but also accepts an option, no
-  // harm in pushing it back here.
-  if (Opts.UnrollLoops)
-    Res.push_back("-funroll-loops");
-  if (Opts.DataSections)
-    Res.push_back("-fdata-sections");
-  if (Opts.FunctionSections)
-    Res.push_back("-ffunction-sections");
-  if (Opts.AsmVerbose)
-    Res.push_back("-masm-verbose");
-  if (!Opts.CodeModel.empty())
-    Res.push_back("-mcode-model", Opts.CodeModel);
-  if (Opts.CUDAIsDevice)
-    Res.push_back("-fcuda-is-device");
-  if (!Opts.CXAAtExit)
-    Res.push_back("-fno-use-cxa-atexit");
-  if (Opts.CXXCtorDtorAliases)
-    Res.push_back("-mconstructor-aliases");
-  if (Opts.ObjCAutoRefCountExceptions)
-    Res.push_back("-fobjc-arc-eh");
-  if (!Opts.DebugPass.empty()) {
-    Res.push_back("-mdebug-pass", Opts.DebugPass);
-  }
-  if (Opts.DisableFPElim)
-    Res.push_back("-mdisable-fp-elim");
-  if (!Opts.FloatABI.empty())
-    Res.push_back("-mfloat-abi", Opts.FloatABI);
-  if (!Opts.LimitFloatPrecision.empty())
-    Res.push_back("-mlimit-float-precision", Opts.LimitFloatPrecision);
-  if (Opts.NoZeroInitializedInBSS)
-    Res.push_back("-mno-zero-initialized-bss");
-  switch (Opts.getObjCDispatchMethod()) {
-  case CodeGenOptions::Legacy:
-    break;
-  case CodeGenOptions::Mixed:
-    Res.push_back("-fobjc-dispatch-method=mixed");
-    break;
-  case CodeGenOptions::NonLegacy:
-    Res.push_back("-fobjc-dispatch-method=non-legacy");
-    break;
-  }
-  if (Opts.BoundsChecking > 0)
-    Res.push_back("-fbounds-checking=" + llvm::utostr(Opts.BoundsChecking));
-  if (Opts.NumRegisterParameters)
-    Res.push_back("-mregparm", llvm::utostr(Opts.NumRegisterParameters));
-  if (Opts.NoGlobalMerge)
-    Res.push_back("-mno-global-merge");
-  if (Opts.NoExecStack)
-    Res.push_back("-mnoexecstack");
-  if (Opts.RelaxAll)
-    Res.push_back("-mrelax-all");
-  if (Opts.SaveTempLabels)
-    Res.push_back("-msave-temp-labels");
-  if (Opts.NoDwarf2CFIAsm)
-    Res.push_back("-fno-dwarf2-cfi-asm");
-  if (Opts.NoDwarfDirectoryAsm)
-    Res.push_back("-fno-dwarf-directory-asm");
-  if (Opts.SoftFloat)
-    Res.push_back("-msoft-float");
-  if (Opts.StrictEnums)
-    Res.push_back("-fstrict-enums");
-  if (Opts.UnwindTables)
-    Res.push_back("-munwind-tables");
-  if (Opts.RelocationModel != "pic")
-    Res.push_back("-mrelocation-model", Opts.RelocationModel);
-  if (!Opts.VerifyModule)
-    Res.push_back("-disable-llvm-verifier");
-  for (unsigned i = 0, e = Opts.BackendOptions.size(); i != e; ++i)
-    Res.push_back("-backend-option", Opts.BackendOptions[i]);
-
-  switch (Opts.DefaultTLSModel) {
-  case CodeGenOptions::GeneralDynamicTLSModel:
-    break;
-  case CodeGenOptions::LocalDynamicTLSModel:
-    Res.push_back("-ftls-model=local-dynamic");
-    break;
-  case CodeGenOptions::InitialExecTLSModel:
-    Res.push_back("-ftls-model=initial-exec");
-    break;
-  case CodeGenOptions::LocalExecTLSModel:
-    Res.push_back("-ftls-model=local-exec");
-    break;
-  }
-}
-
-static void DependencyOutputOptsToArgs(const DependencyOutputOptions &Opts,
-                                       ToArgsList &Res) {
-  if (Opts.IncludeSystemHeaders)
-    Res.push_back("-sys-header-deps");
-  if (Opts.ShowHeaderIncludes)
-    Res.push_back("-H");
-  if (!Opts.HeaderIncludeOutputFile.empty())
-    Res.push_back("-header-include-file", Opts.HeaderIncludeOutputFile);
-  if (Opts.UsePhonyTargets)
-    Res.push_back("-MP");
-  if (!Opts.OutputFile.empty())
-    Res.push_back("-dependency-file", Opts.OutputFile);
-  for (unsigned i = 0, e = Opts.Targets.size(); i != e; ++i)
-    Res.push_back("-MT", Opts.Targets[i]);
-}
-
-static void DiagnosticOptsToArgs(const DiagnosticOptions &Opts,
-                                 ToArgsList &Res) {
-  if (Opts.IgnoreWarnings)
-    Res.push_back("-w");
-  if (Opts.NoRewriteMacros)
-    Res.push_back("-Wno-rewrite-macros");
-  if (Opts.Pedantic)
-    Res.push_back("-pedantic");
-  if (Opts.PedanticErrors)
-    Res.push_back("-pedantic-errors");
-  if (!Opts.ShowColumn)
-    Res.push_back("-fno-show-column");
-  if (!Opts.ShowLocation)
-    Res.push_back("-fno-show-source-location");
-  if (!Opts.ShowCarets)
-    Res.push_back("-fno-caret-diagnostics");
-  if (!Opts.ShowFixits)
-    Res.push_back("-fno-diagnostics-fixit-info");
-  if (Opts.ShowSourceRanges)
-    Res.push_back("-fdiagnostics-print-source-range-info");
-  if (Opts.ShowParseableFixits)
-    Res.push_back("-fdiagnostics-parseable-fixits");
-  if (Opts.ShowColors)
-    Res.push_back("-fcolor-diagnostics");
-  if (Opts.VerifyDiagnostics)
-    Res.push_back("-verify");
-  if (Opts.ShowOptionNames)
-    Res.push_back("-fdiagnostics-show-option");
-  if (Opts.ShowCategories == 1)
-    Res.push_back("-fdiagnostics-show-category=id");
-  else if (Opts.ShowCategories == 2)
-    Res.push_back("-fdiagnostics-show-category=name");
-  switch (Opts.Format) {
-  case DiagnosticOptions::Clang: 
-    Res.push_back("-fdiagnostics-format=clang"); break;
-  case DiagnosticOptions::Msvc:  
-    Res.push_back("-fdiagnostics-format=msvc");  break;
-  case DiagnosticOptions::Vi:    
-    Res.push_back("-fdiagnostics-format=vi");    break;
-  }
-  if (Opts.ErrorLimit)
-    Res.push_back("-ferror-limit", llvm::utostr(Opts.ErrorLimit));
-  if (!Opts.DiagnosticLogFile.empty())
-    Res.push_back("-diagnostic-log-file", Opts.DiagnosticLogFile);
-  if (Opts.MacroBacktraceLimit
-                        != DiagnosticOptions::DefaultMacroBacktraceLimit)
-    Res.push_back("-fmacro-backtrace-limit",
-                  llvm::utostr(Opts.MacroBacktraceLimit));
-  if (Opts.TemplateBacktraceLimit
-                        != DiagnosticOptions::DefaultTemplateBacktraceLimit)
-    Res.push_back("-ftemplate-backtrace-limit",
-                  llvm::utostr(Opts.TemplateBacktraceLimit));
-  if (Opts.ConstexprBacktraceLimit
-                        != DiagnosticOptions::DefaultConstexprBacktraceLimit)
-    Res.push_back("-fconstexpr-backtrace-limit",
-                  llvm::utostr(Opts.ConstexprBacktraceLimit));
-
-  if (Opts.TabStop != DiagnosticOptions::DefaultTabStop)
-    Res.push_back("-ftabstop", llvm::utostr(Opts.TabStop));
-  if (Opts.MessageLength)
-    Res.push_back("-fmessage-length", llvm::utostr(Opts.MessageLength));
-  if (!Opts.DumpBuildInformation.empty())
-    Res.push_back("-dump-build-information", Opts.DumpBuildInformation);
-  for (unsigned i = 0, e = Opts.Warnings.size(); i != e; ++i)
-    Res.push_back("-W" + Opts.Warnings[i]);
-}
-
-static const char *getInputKindName(InputKind Kind) {
-  switch (Kind) {
-  case IK_None:              break;
-  case IK_AST:               return "ast";
-  case IK_Asm:               return "assembler-with-cpp";
-  case IK_C:                 return "c";
-  case IK_CXX:               return "c++";
-  case IK_LLVM_IR:           return "ir";
-  case IK_ObjC:              return "objective-c";
-  case IK_ObjCXX:            return "objective-c++";
-  case IK_OpenCL:            return "cl";
-  case IK_CUDA:              return "cuda";
-  case IK_PreprocessedC:     return "cpp-output";
-  case IK_PreprocessedCXX:   return "c++-cpp-output";
-  case IK_PreprocessedObjC:  return "objective-c-cpp-output";
-  case IK_PreprocessedObjCXX:return "objective-c++-cpp-output";
-  }
-
-  llvm_unreachable("Unexpected language kind!");
-}
-
-static const char *getActionName(frontend::ActionKind Kind) {
-  switch (Kind) {
-  case frontend::PluginAction:
-    llvm_unreachable("Invalid kind!");
-
-  case frontend::ASTDeclList:            return "-ast-list";
-  case frontend::ASTDump:                return "-ast-dump";
-  case frontend::ASTDumpXML:             return "-ast-dump-xml";
-  case frontend::ASTPrint:               return "-ast-print";
-  case frontend::ASTView:                return "-ast-view";
-  case frontend::DumpRawTokens:          return "-dump-raw-tokens";
-  case frontend::DumpTokens:             return "-dump-tokens";
-  case frontend::EmitAssembly:           return "-S";
-  case frontend::EmitBC:                 return "-emit-llvm-bc";
-  case frontend::EmitHTML:               return "-emit-html";
-  case frontend::EmitLLVM:               return "-emit-llvm";
-  case frontend::EmitLLVMOnly:           return "-emit-llvm-only";
-  case frontend::EmitCodeGenOnly:        return "-emit-codegen-only";
-  case frontend::EmitObj:                return "-emit-obj";
-  case frontend::FixIt:                  return "-fixit";
-  case frontend::GenerateModule:         return "-emit-module";
-  case frontend::GeneratePCH:            return "-emit-pch";
-  case frontend::GeneratePTH:            return "-emit-pth";
-  case frontend::InitOnly:               return "-init-only";
-  case frontend::ParseSyntaxOnly:        return "-fsyntax-only";
-  case frontend::PrintDeclContext:       return "-print-decl-contexts";
-  case frontend::PrintPreamble:          return "-print-preamble";
-  case frontend::PrintPreprocessedInput: return "-E";
-  case frontend::RewriteMacros:          return "-rewrite-macros";
-  case frontend::RewriteObjC:            return "-rewrite-objc";
-  case frontend::RewriteTest:            return "-rewrite-test";
-  case frontend::RunAnalysis:            return "-analyze";
-  case frontend::MigrateSource:          return "-migrate";
-  case frontend::RunPreprocessorOnly:    return "-Eonly";
-  }
-
-  llvm_unreachable("Unexpected language kind!");
-}
-
-static void FileSystemOptsToArgs(const FileSystemOptions &Opts, ToArgsList &Res){
-  if (!Opts.WorkingDir.empty())
-    Res.push_back("-working-directory", Opts.WorkingDir);
-}
-
-static void CodeCompleteOptionsToArgs(const CodeCompleteOptions &Opts,
-                                      ToArgsList &Res) {
-  if (Opts.IncludeMacros)
-    Res.push_back("-code-completion-macros");
-  if (Opts.IncludeCodePatterns)
-    Res.push_back("-code-completion-patterns");
-  if (!Opts.IncludeGlobals)
-    Res.push_back("-no-code-completion-globals");
-  if (Opts.IncludeBriefComments)
-    Res.push_back("-code-completion-brief-comments");
-}
-
-static void FrontendOptsToArgs(const FrontendOptions &Opts, ToArgsList &Res) {
-  if (Opts.DisableFree)
-    Res.push_back("-disable-free");
-  if (Opts.RelocatablePCH)
-    Res.push_back("-relocatable-pch");
-  if (Opts.ShowHelp)
-    Res.push_back("-help");
-  if (Opts.ShowStats)
-    Res.push_back("-print-stats");
-  if (Opts.ShowTimers)
-    Res.push_back("-ftime-report");
-  if (Opts.ShowVersion)
-    Res.push_back("-version");
-  if (Opts.FixWhatYouCan)
-    Res.push_back("-fix-what-you-can");
-  if (Opts.FixOnlyWarnings)
-    Res.push_back("-fix-only-warnings");
-  if (Opts.FixAndRecompile)
-    Res.push_back("-fixit-recompile");
-  if (Opts.FixToTemporaries)
-    Res.push_back("-fixit-to-temporary");
-  switch (Opts.ARCMTAction) {
-  case FrontendOptions::ARCMT_None:
-    break;
-  case FrontendOptions::ARCMT_Check:
-    Res.push_back("-arcmt-check");
-    break;
-  case FrontendOptions::ARCMT_Modify:
-    Res.push_back("-arcmt-modify");
-    break;
-  case FrontendOptions::ARCMT_Migrate:
-    Res.push_back("-arcmt-migrate");
-    break;
-  }
-  CodeCompleteOptionsToArgs(Opts.CodeCompleteOpts, Res);
-  if (!Opts.MTMigrateDir.empty())
-    Res.push_back("-mt-migrate-directory", Opts.MTMigrateDir);
-  if (!Opts.ARCMTMigrateReportOut.empty())
-    Res.push_back("-arcmt-migrate-report-output", Opts.ARCMTMigrateReportOut);
-  if (Opts.ARCMTMigrateEmitARCErrors)
-    Res.push_back("-arcmt-migrate-emit-errors");
-
-  if (Opts.ObjCMTAction & ~FrontendOptions::ObjCMT_Literals)
-    Res.push_back("-objcmt-migrate-literals");
-  if (Opts.ObjCMTAction & ~FrontendOptions::ObjCMT_Subscripting)
-    Res.push_back("-objcmt-migrate-subscripting");
-
-  bool NeedLang = false;
-  for (unsigned i = 0, e = Opts.Inputs.size(); i != e; ++i)
-    if (FrontendOptions::getInputKindForExtension(Opts.Inputs[i].File) !=
-        Opts.Inputs[i].Kind)
-      NeedLang = true;
-  if (NeedLang)
-    Res.push_back("-x", getInputKindName(Opts.Inputs[0].Kind));
-  for (unsigned i = 0, e = Opts.Inputs.size(); i != e; ++i) {
-    assert((!NeedLang || Opts.Inputs[i].Kind == Opts.Inputs[0].Kind) &&
-           "Unable to represent this input vector!");
-    Res.push_back(Opts.Inputs[i].File);
-  }
-
-  if (!Opts.OutputFile.empty())
-    Res.push_back("-o", Opts.OutputFile);
-  if (!Opts.CodeCompletionAt.FileName.empty())
-    Res.push_back("-code-completion-at",
-                  Opts.CodeCompletionAt.FileName + ":" +
-                  llvm::utostr(Opts.CodeCompletionAt.Line) + ":" +
-                  llvm::utostr(Opts.CodeCompletionAt.Column));
-  if (Opts.ProgramAction != frontend::PluginAction)
-    Res.push_back(getActionName(Opts.ProgramAction));
-  if (!Opts.ActionName.empty()) {
-    Res.push_back("-plugin", Opts.ActionName);
-    for(unsigned i = 0, e = Opts.PluginArgs.size(); i != e; ++i)
-      Res.push_back("-plugin-arg-" + Opts.ActionName, Opts.PluginArgs[i]);
-  }
-  if (!Opts.ASTDumpFilter.empty())
-    Res.push_back("-ast-dump-filter", Opts.ASTDumpFilter);
-  for (unsigned i = 0, e = Opts.Plugins.size(); i != e; ++i)
-    Res.push_back("-load", Opts.Plugins[i]);
-  for (unsigned i = 0, e = Opts.AddPluginActions.size(); i != e; ++i) {
-    Res.push_back("-add-plugin", Opts.AddPluginActions[i]);
-    for(unsigned ai = 0, ae = Opts.AddPluginArgs.size(); ai != ae; ++ai)
-      Res.push_back("-plugin-arg-" + Opts.AddPluginActions[i],
-                    Opts.AddPluginArgs[i][ai]);
-  }
-  for (unsigned i = 0, e = Opts.ASTMergeFiles.size(); i != e; ++i)
-    Res.push_back("-ast-merge", Opts.ASTMergeFiles[i]);
-  for (unsigned i = 0, e = Opts.LLVMArgs.size(); i != e; ++i)
-    Res.push_back("-mllvm", Opts.LLVMArgs[i]);
-  if (!Opts.OverrideRecordLayoutsFile.empty())
-    Res.push_back("-foverride-record-layout=" + Opts.OverrideRecordLayoutsFile);
-}
-
-static void HeaderSearchOptsToArgs(const HeaderSearchOptions &Opts,
-                                   ToArgsList &Res) {
-  if (Opts.Sysroot != "/") {
-    Res.push_back("-isysroot");
-    Res.push_back(Opts.Sysroot);
-  }
-
-  /// User specified include entries.
-  for (unsigned i = 0, e = Opts.UserEntries.size(); i != e; ++i) {
-    const HeaderSearchOptions::Entry &E = Opts.UserEntries[i];
-    if (E.IsFramework && (E.Group != frontend::Angled || !E.IsUserSupplied))
-      llvm::report_fatal_error("Invalid option set!");
-    if (E.IsUserSupplied) {
-      switch (E.Group) {
-      case frontend::After:
-        Res.push_back("-idirafter");
-        break;
-        
-      case frontend::Quoted:
-        Res.push_back("-iquote");
-        break;
-        
-      case frontend::System:
-        Res.push_back("-isystem");
-        break;
-        
-      case frontend::IndexHeaderMap:
-        Res.push_back("-index-header-map");
-        Res.push_back(E.IsFramework? "-F" : "-I");
-        break;
-        
-      case frontend::CSystem:
-        Res.push_back("-c-isystem");
-        break;
-
-      case frontend::CXXSystem:
-        Res.push_back("-cxx-isystem");
-        break;
-
-      case frontend::ObjCSystem:
-        Res.push_back("-objc-isystem");
-        break;
-
-      case frontend::ObjCXXSystem:
-        Res.push_back("-objcxx-isystem");
-        break;
-        
-      case frontend::Angled:
-        Res.push_back(E.IsFramework ? "-F" : "-I");
-        break;
-      }
-    } else {
-      if (E.IsInternal) {
-        assert(E.Group == frontend::System && "Unexpected header search group");
-        if (E.ImplicitExternC)
-          Res.push_back("-internal-externc-isystem");
-        else
-          Res.push_back("-internal-isystem");
-      } else {
-        if (E.Group != frontend::Angled && E.Group != frontend::System)
-          llvm::report_fatal_error("Invalid option set!");
-        Res.push_back(E.Group == frontend::Angled ? "-iwithprefixbefore" :
-                      "-iwithprefix");
-      }
-    }
-    Res.push_back(E.Path);
-  }
-
-  /// User-specified system header prefixes.
-  for (unsigned i = 0, e = Opts.SystemHeaderPrefixes.size(); i != e; ++i) {
-    if (Opts.SystemHeaderPrefixes[i].IsSystemHeader)
-      Res.push_back("-isystem-prefix");
-    else
-      Res.push_back("-ino-system-prefix");
-
-    Res.push_back(Opts.SystemHeaderPrefixes[i].Prefix);
-  }
-
-  if (!Opts.ResourceDir.empty())
-    Res.push_back("-resource-dir", Opts.ResourceDir);
-  if (!Opts.ModuleCachePath.empty())
-    Res.push_back("-fmodule-cache-path", Opts.ModuleCachePath);
-  if (!Opts.UseStandardSystemIncludes)
-    Res.push_back("-nostdsysteminc");
-  if (!Opts.UseStandardCXXIncludes)
-    Res.push_back("-nostdinc++");
-  if (Opts.UseLibcxx)
-    Res.push_back("-stdlib=libc++");
-  if (Opts.Verbose)
-    Res.push_back("-v");
-}
-
-static void LangOptsToArgs(const LangOptions &Opts, ToArgsList &Res) {
-  LangOptions DefaultLangOpts;
-
-  // FIXME: Need to set -std to get all the implicit options.
-
-  // FIXME: We want to only pass options relative to the defaults, which
-  // requires constructing a target. :(
-  //
-  // It would be better to push the all target specific choices into the driver,
-  // so that everything below that was more uniform.
-
-  if (Opts.Trigraphs)
-    Res.push_back("-trigraphs");
-  // Implicit based on the input kind:
-  //   AsmPreprocessor, CPlusPlus, ObjC1, ObjC2, OpenCL
-  // Implicit based on the input language standard:
-  //   BCPLComment, C99, CPlusPlus0x, Digraphs, GNUInline, ImplicitInt, GNUMode
-  if (Opts.DollarIdents)
-    Res.push_back("-fdollars-in-identifiers");
-  if (Opts.GNUMode && !Opts.GNUKeywords)
-    Res.push_back("-fno-gnu-keywords");
-  if (!Opts.GNUMode && Opts.GNUKeywords)
-    Res.push_back("-fgnu-keywords");
-  if (Opts.MicrosoftExt)
-    Res.push_back("-fms-extensions");
-  if (Opts.MicrosoftMode)
-    Res.push_back("-fms-compatibility");
-  if (Opts.MSCVersion != 0)
-    Res.push_back("-fmsc-version=" + llvm::utostr(Opts.MSCVersion));
-  if (Opts.Borland)
-    Res.push_back("-fborland-extensions");
-  if (Opts.ObjCDefaultSynthProperties)
-    Res.push_back("-fobjc-default-synthesize-properties");
-  // NoInline is implicit.
-  if (!Opts.CXXOperatorNames)
-    Res.push_back("-fno-operator-names");
-  if (Opts.PascalStrings)
-    Res.push_back("-fpascal-strings");
-  if (Opts.CatchUndefined)
-    Res.push_back("-fcatch-undefined-behavior");
-  if (Opts.AddressSanitizer)
-    Res.push_back("-faddress-sanitizer");
-  if (Opts.ThreadSanitizer)
-    Res.push_back("-fthread-sanitizer");
-  if (Opts.WritableStrings)
-    Res.push_back("-fwritable-strings");
-  if (Opts.ConstStrings)
-    Res.push_back("-fconst-strings");
-  if (!Opts.LaxVectorConversions)
-    Res.push_back("-fno-lax-vector-conversions");
-  if (Opts.AltiVec)
-    Res.push_back("-faltivec");
-  if (Opts.Exceptions)
-    Res.push_back("-fexceptions");
-  if (Opts.ObjCExceptions)
-    Res.push_back("-fobjc-exceptions");
-  if (Opts.CXXExceptions)
-    Res.push_back("-fcxx-exceptions");
-  if (Opts.SjLjExceptions)
-    Res.push_back("-fsjlj-exceptions");
-  if (Opts.TraditionalCPP)
-    Res.push_back("-traditional-cpp");
-  if (!Opts.RTTI)
-    Res.push_back("-fno-rtti");
-  if (Opts.MSBitfields)
-    Res.push_back("-mms-bitfields");
-  if (Opts.Freestanding)
-    Res.push_back("-ffreestanding");
-  if (Opts.NoBuiltin)
-    Res.push_back("-fno-builtin");
-  if (!Opts.AssumeSaneOperatorNew)
-    Res.push_back("-fno-assume-sane-operator-new");
-  if (!Opts.ThreadsafeStatics)
-    Res.push_back("-fno-threadsafe-statics");
-  if (Opts.POSIXThreads)
-    Res.push_back("-pthread");
-  if (Opts.Blocks)
-    Res.push_back("-fblocks");
-  if (Opts.BlocksRuntimeOptional)
-    Res.push_back("-fblocks-runtime-optional");
-  if (Opts.Modules)
-    Res.push_back("-fmodules");
-  if (Opts.EmitAllDecls)
-    Res.push_back("-femit-all-decls");
-  if (Opts.MathErrno)
-    Res.push_back("-fmath-errno");
-  switch (Opts.getSignedOverflowBehavior()) {
-  case LangOptions::SOB_Undefined: break;
-  case LangOptions::SOB_Defined:   Res.push_back("-fwrapv"); break;
-  case LangOptions::SOB_Trapping:
-    Res.push_back("-ftrapv");
-    if (!Opts.OverflowHandler.empty())
-      Res.push_back("-ftrapv-handler", Opts.OverflowHandler);
-    break;
-  }
-  switch (Opts.getFPContractMode()) {
-  case LangOptions::FPC_Off:  Res.push_back("-ffp-contract=off"); break;
-  case LangOptions::FPC_On:   Res.push_back("-ffp-contract=on"); break;
-  case LangOptions::FPC_Fast: Res.push_back("-ffp-contract=fast"); break;
-  }
-  if (Opts.HeinousExtensions)
-    Res.push_back("-fheinous-gnu-extensions");
-  // Optimize is implicit.
-  // OptimizeSize is implicit.
-  if (Opts.FastMath)
-    Res.push_back("-ffast-math");
-  if (Opts.Static)
-    Res.push_back("-static-define");
-  if (Opts.DumpRecordLayoutsSimple)
-    Res.push_back("-fdump-record-layouts-simple");
-  else if (Opts.DumpRecordLayouts)
-    Res.push_back("-fdump-record-layouts");
-  if (Opts.DumpVTableLayouts)
-    Res.push_back("-fdump-vtable-layouts");
-  if (Opts.NoBitFieldTypeAlign)
-    Res.push_back("-fno-bitfield-type-alignment");
-  if (Opts.PICLevel)
-    Res.push_back("-pic-level", llvm::utostr(Opts.PICLevel));
-  if (Opts.PIELevel)
-    Res.push_back("-pie-level", llvm::utostr(Opts.PIELevel));
-  if (Opts.ObjCGCBitmapPrint)
-    Res.push_back("-print-ivar-layout");
-  if (Opts.NoConstantCFStrings)
-    Res.push_back("-fno-constant-cfstrings");
-  if (!Opts.AccessControl)
-    Res.push_back("-fno-access-control");
-  if (!Opts.CharIsSigned)
-    Res.push_back("-fno-signed-char");
-  if (Opts.ShortWChar)
-    Res.push_back("-fshort-wchar");
-  if (!Opts.ElideConstructors)
-    Res.push_back("-fno-elide-constructors");
-  if (Opts.getGC() != LangOptions::NonGC) {
-    if (Opts.getGC() == LangOptions::HybridGC) {
-      Res.push_back("-fobjc-gc");
-    } else {
-      assert(Opts.getGC() == LangOptions::GCOnly && "Invalid GC mode!");
-      Res.push_back("-fobjc-gc-only");
-    }
-  }
-  Res.push_back("-fobjc-runtime=" + Opts.ObjCRuntime.getAsString());
-  if (Opts.ObjCAutoRefCount)
-    Res.push_back("-fobjc-arc");
-  if (Opts.ObjCRuntimeHasWeak)
-    Res.push_back("-fobjc-runtime-has-weak");
-  if (!Opts.ObjCInferRelatedResultType)
-    Res.push_back("-fno-objc-infer-related-result-type");
-  
-  if (Opts.AppleKext)
-    Res.push_back("-fapple-kext");
-
-  if (Opts.getVisibilityMode() != DefaultVisibility) {
-    Res.push_back("-fvisibility");
-    if (Opts.getVisibilityMode() == HiddenVisibility) {
-      Res.push_back("hidden");
-    } else {
-      assert(Opts.getVisibilityMode() == ProtectedVisibility &&
-             "Invalid visibility!");
-      Res.push_back("protected");
-    }
-  }
-  if (Opts.InlineVisibilityHidden)
-    Res.push_back("-fvisibility-inlines-hidden");
-
-  if (Opts.getStackProtector() != 0)
-    Res.push_back("-stack-protector", llvm::utostr(Opts.getStackProtector()));
-  if (Opts.InstantiationDepth != DefaultLangOpts.InstantiationDepth)
-    Res.push_back("-ftemplate-depth", llvm::utostr(Opts.InstantiationDepth));
-  if (Opts.ConstexprCallDepth != DefaultLangOpts.ConstexprCallDepth)
-    Res.push_back("-fconstexpr-depth", llvm::utostr(Opts.ConstexprCallDepth));
-  if (!Opts.ObjCConstantStringClass.empty())
-    Res.push_back("-fconstant-string-class", Opts.ObjCConstantStringClass);
-  if (Opts.FakeAddressSpaceMap)
-    Res.push_back("-ffake-address-space-map");
-  if (Opts.ParseUnknownAnytype)
-    Res.push_back("-funknown-anytype");
-  if (Opts.DebuggerSupport)
-    Res.push_back("-fdebugger-support");
-  if (Opts.DebuggerCastResultToId)
-    Res.push_back("-fdebugger-cast-result-to-id");
-  if (Opts.DebuggerObjCLiteral)
-    Res.push_back("-fdebugger-objc-literal");
-  if (Opts.DelayedTemplateParsing)
-    Res.push_back("-fdelayed-template-parsing");
-  if (Opts.Deprecated)
-    Res.push_back("-fdeprecated-macro");
-  if (Opts.ApplePragmaPack)
-    Res.push_back("-fapple-pragma-pack");
-  if (!Opts.CurrentModule.empty())
-    Res.push_back("-fmodule-name=" + Opts.CurrentModule);
-}
-
-static void PreprocessorOptsToArgs(const PreprocessorOptions &Opts,
-                                   ToArgsList &Res) {
-  for (unsigned i = 0, e = Opts.Macros.size(); i != e; ++i)
-    Res.push_back(std::string(Opts.Macros[i].second ? "-U" : "-D") +
-                  Opts.Macros[i].first);
-  for (unsigned i = 0, e = Opts.Includes.size(); i != e; ++i) {
-    // FIXME: We need to avoid reincluding the implicit PCH and PTH includes.
-    Res.push_back("-include", Opts.Includes[i]);
-  }
-  for (unsigned i = 0, e = Opts.MacroIncludes.size(); i != e; ++i)
-    Res.push_back("-imacros", Opts.MacroIncludes[i]);
-  if (!Opts.UsePredefines)
-    Res.push_back("-undef");
-  if (Opts.DetailedRecord)
-    Res.push_back("-detailed-preprocessing-record");
-  if (!Opts.ImplicitPCHInclude.empty())
-    Res.push_back("-include-pch", Opts.ImplicitPCHInclude);
-  if (!Opts.ImplicitPTHInclude.empty())
-    Res.push_back("-include-pth", Opts.ImplicitPTHInclude);
-  if (!Opts.TokenCache.empty()) {
-    if (Opts.ImplicitPTHInclude.empty())
-      Res.push_back("-token-cache", Opts.TokenCache);
-    else
-      assert(Opts.ImplicitPTHInclude == Opts.TokenCache &&
-             "Unsupported option combination!");
-  }
-  for (unsigned i = 0, e = Opts.ChainedIncludes.size(); i != e; ++i)
-    Res.push_back("-chain-include", Opts.ChainedIncludes[i]);
-  for (unsigned i = 0, e = Opts.RemappedFiles.size(); i != e; ++i) {
-    Res.push_back("-remap-file", Opts.RemappedFiles[i].first + ";" +
-                                 Opts.RemappedFiles[i].second);
-  }
-}
-
-static void PreprocessorOutputOptsToArgs(const PreprocessorOutputOptions &Opts,
-                                         ToArgsList &Res) {
-  if (!Opts.ShowCPP && !Opts.ShowMacros)
-    llvm::report_fatal_error("Invalid option combination!");
-
-  if (Opts.ShowCPP && Opts.ShowMacros)
-    Res.push_back("-dD");
-  else if (!Opts.ShowCPP && Opts.ShowMacros)
-    Res.push_back("-dM");
-
-  if (!Opts.ShowLineMarkers)
-    Res.push_back("-P");
-  if (Opts.ShowComments)
-    Res.push_back("-C");
-  if (Opts.ShowMacroComments)
-    Res.push_back("-CC");
-}
-
-static void TargetOptsToArgs(const TargetOptions &Opts,
-                             ToArgsList &Res) {
-  Res.push_back("-triple");
-  Res.push_back(Opts.Triple);
-  if (!Opts.CPU.empty())
-    Res.push_back("-target-cpu", Opts.CPU);
-  if (!Opts.ABI.empty())
-    Res.push_back("-target-abi", Opts.ABI);
-  if (!Opts.LinkerVersion.empty())
-    Res.push_back("-target-linker-version", Opts.LinkerVersion);
-  if (!Opts.CXXABI.empty())
-    Res.push_back("-cxx-abi", Opts.CXXABI);
-  for (unsigned i = 0, e = Opts.Features.size(); i != e; ++i)
-    Res.push_back("-target-feature", Opts.Features[i]);
-}
-
-void CompilerInvocation::toArgs(std::vector<std::string> &Res) const {
-  ToArgsList List(Res);
-  AnalyzerOptsToArgs(getAnalyzerOpts(), List);
-  CodeGenOptsToArgs(getCodeGenOpts(), List);
-  DependencyOutputOptsToArgs(getDependencyOutputOpts(), List);
-  DiagnosticOptsToArgs(getDiagnosticOpts(), List);
-  FileSystemOptsToArgs(getFileSystemOpts(), List);
-  FrontendOptsToArgs(getFrontendOpts(), List);
-  HeaderSearchOptsToArgs(getHeaderSearchOpts(), List);
-  LangOptsToArgs(*getLangOpts(), List);
-  PreprocessorOptsToArgs(getPreprocessorOpts(), List);
-  PreprocessorOutputOptsToArgs(getPreprocessorOutputOpts(), List);
-  TargetOptsToArgs(getTargetOpts(), List);
-}
-
-//===----------------------------------------------------------------------===//
-// Deserialization (to args)
+// Deserialization (from args)
 //===----------------------------------------------------------------------===//
 
 using namespace clang::driver;
@@ -965,7 +71,7 @@ static unsigned getOptimizationLevel(ArgList &Args, InputKind IK,
 
     assert (A->getOption().matches(options::OPT_O));
 
-    llvm::StringRef S(A->getValue(Args));
+    llvm::StringRef S(A->getValue());
     if (S == "s" || S == "z" || S.empty())
       return 2;
 
@@ -979,7 +85,7 @@ static unsigned getOptimizationLevelSize(ArgList &Args, InputKind IK,
                                          DiagnosticsEngine &Diags) {
   if (Arg *A = Args.getLastArg(options::OPT_O_Group)) {
     if (A->getOption().matches(options::OPT_O)) {
-      switch (A->getValue(Args)[0]) {
+      switch (A->getValue()[0]) {
       default:
         return 0;
       case 's':
@@ -996,14 +102,14 @@ static void addWarningArgs(ArgList &Args, std::vector<std::string> &Warnings) {
   for (arg_iterator I = Args.filtered_begin(OPT_W_Group),
          E = Args.filtered_end(); I != E; ++I) {
     Arg *A = *I;
-    // If the argument is a pure flag, add its name (minus the "-W" at the beginning)
+    // If the argument is a pure flag, add its name (minus the "W" at the beginning)
     // to the warning list. Else, add its value (for the OPT_W case).
     if (A->getOption().getKind() == Option::FlagClass) {
-      Warnings.push_back(A->getOption().getName().substr(2));
+      Warnings.push_back(A->getOption().getName().substr(1));
     } else {
       for (unsigned Idx = 0, End = A->getNumValues();
            Idx < End; ++Idx) {
-        StringRef V = A->getValue(Args, Idx);
+        StringRef V = A->getValue(Idx);
         // "-Wl," and such are not warning options.
         // FIXME: Should be handled by putting these in separate flags.
         if (V.startswith("l,") || V.startswith("a,") || V.startswith("p,"))
@@ -1020,11 +126,11 @@ static bool ParseAnalyzerArgs(AnalyzerOptions &Opts, ArgList &Args,
   using namespace options;
   bool Success = true;
   if (Arg *A = Args.getLastArg(OPT_analyzer_store)) {
-    StringRef Name = A->getValue(Args);
+    StringRef Name = A->getValue();
     AnalysisStores Value = llvm::StringSwitch<AnalysisStores>(Name)
 #define ANALYSIS_STORE(NAME, CMDFLAG, DESC, CREATFN) \
       .Case(CMDFLAG, NAME##Model)
-#include "clang/Frontend/Analyses.def"
+#include "clang/StaticAnalyzer/Core/Analyses.def"
       .Default(NumStores);
     if (Value == NumStores) {
       Diags.Report(diag::err_drv_invalid_value)
@@ -1036,11 +142,11 @@ static bool ParseAnalyzerArgs(AnalyzerOptions &Opts, ArgList &Args,
   }
 
   if (Arg *A = Args.getLastArg(OPT_analyzer_constraints)) {
-    StringRef Name = A->getValue(Args);
+    StringRef Name = A->getValue();
     AnalysisConstraints Value = llvm::StringSwitch<AnalysisConstraints>(Name)
 #define ANALYSIS_CONSTRAINTS(NAME, CMDFLAG, DESC, CREATFN) \
       .Case(CMDFLAG, NAME##Model)
-#include "clang/Frontend/Analyses.def"
+#include "clang/StaticAnalyzer/Core/Analyses.def"
       .Default(NumConstraints);
     if (Value == NumConstraints) {
       Diags.Report(diag::err_drv_invalid_value)
@@ -1052,11 +158,11 @@ static bool ParseAnalyzerArgs(AnalyzerOptions &Opts, ArgList &Args,
   }
 
   if (Arg *A = Args.getLastArg(OPT_analyzer_output)) {
-    StringRef Name = A->getValue(Args);
+    StringRef Name = A->getValue();
     AnalysisDiagClients Value = llvm::StringSwitch<AnalysisDiagClients>(Name)
 #define ANALYSIS_DIAGNOSTICS(NAME, CMDFLAG, DESC, CREATFN, AUTOCREAT) \
       .Case(CMDFLAG, PD_##NAME)
-#include "clang/Frontend/Analyses.def"
+#include "clang/StaticAnalyzer/Core/Analyses.def"
       .Default(NUM_ANALYSIS_DIAG_CLIENTS);
     if (Value == NUM_ANALYSIS_DIAG_CLIENTS) {
       Diags.Report(diag::err_drv_invalid_value)
@@ -1068,11 +174,11 @@ static bool ParseAnalyzerArgs(AnalyzerOptions &Opts, ArgList &Args,
   }
 
   if (Arg *A = Args.getLastArg(OPT_analyzer_purge)) {
-    StringRef Name = A->getValue(Args);
+    StringRef Name = A->getValue();
     AnalysisPurgeMode Value = llvm::StringSwitch<AnalysisPurgeMode>(Name)
 #define ANALYSIS_PURGE(NAME, CMDFLAG, DESC) \
       .Case(CMDFLAG, NAME)
-#include "clang/Frontend/Analyses.def"
+#include "clang/StaticAnalyzer/Core/Analyses.def"
       .Default(NumPurgeModes);
     if (Value == NumPurgeModes) {
       Diags.Report(diag::err_drv_invalid_value)
@@ -1084,11 +190,11 @@ static bool ParseAnalyzerArgs(AnalyzerOptions &Opts, ArgList &Args,
   }
 
   if (Arg *A = Args.getLastArg(OPT_analyzer_ipa)) {
-    StringRef Name = A->getValue(Args);
+    StringRef Name = A->getValue();
     AnalysisIPAMode Value = llvm::StringSwitch<AnalysisIPAMode>(Name)
 #define ANALYSIS_IPA(NAME, CMDFLAG, DESC) \
       .Case(CMDFLAG, NAME)
-#include "clang/Frontend/Analyses.def"
+#include "clang/StaticAnalyzer/Core/Analyses.def"
       .Default(NumIPAModes);
     if (Value == NumIPAModes) {
       Diags.Report(diag::err_drv_invalid_value)
@@ -1100,11 +206,11 @@ static bool ParseAnalyzerArgs(AnalyzerOptions &Opts, ArgList &Args,
   }
 
   if (Arg *A = Args.getLastArg(OPT_analyzer_inlining_mode)) {
-    StringRef Name = A->getValue(Args);
+    StringRef Name = A->getValue();
     AnalysisInliningMode Value = llvm::StringSwitch<AnalysisInliningMode>(Name)
 #define ANALYSIS_INLINING_MODE(NAME, CMDFLAG, DESC) \
       .Case(CMDFLAG, NAME)
-#include "clang/Frontend/Analyses.def"
+#include "clang/StaticAnalyzer/Core/Analyses.def"
       .Default(NumInliningModes);
     if (Value == NumInliningModes) {
       Diags.Report(diag::err_drv_invalid_value)
@@ -1116,21 +222,21 @@ static bool ParseAnalyzerArgs(AnalyzerOptions &Opts, ArgList &Args,
   }
 
   Opts.ShowCheckerHelp = Args.hasArg(OPT_analyzer_checker_help);
-  Opts.VisualizeEGDot = Args.hasArg(OPT_analyzer_viz_egraph_graphviz);
-  Opts.VisualizeEGUbi = Args.hasArg(OPT_analyzer_viz_egraph_ubigraph);
+  Opts.visualizeExplodedGraphWithGraphViz =
+    Args.hasArg(OPT_analyzer_viz_egraph_graphviz);
+  Opts.visualizeExplodedGraphWithUbiGraph =
+    Args.hasArg(OPT_analyzer_viz_egraph_ubigraph);
   Opts.NoRetryExhausted = Args.hasArg(OPT_analyzer_disable_retry_exhausted);
   Opts.AnalyzeAll = Args.hasArg(OPT_analyzer_opt_analyze_headers);
   Opts.AnalyzerDisplayProgress = Args.hasArg(OPT_analyzer_display_progress);
   Opts.AnalyzeNestedBlocks =
     Args.hasArg(OPT_analyzer_opt_analyze_nested_blocks);
-  Opts.EagerlyAssume = Args.hasArg(OPT_analyzer_eagerly_assume);
+  Opts.eagerlyAssumeBinOpBifurcation = Args.hasArg(OPT_analyzer_eagerly_assume);
   Opts.AnalyzeSpecificFunction = Args.getLastArgValue(OPT_analyze_function);
   Opts.UnoptimizedCFG = Args.hasArg(OPT_analysis_UnoptimizedCFG);
-  Opts.CFGAddImplicitDtors = Args.hasArg(OPT_analysis_CFGAddImplicitDtors);
   Opts.TrimGraph = Args.hasArg(OPT_trim_egraph);
   Opts.MaxNodes = Args.getLastArgIntValue(OPT_analyzer_max_nodes, 150000,Diags);
-  Opts.MaxLoop = Args.getLastArgIntValue(OPT_analyzer_max_loop, 4, Diags);
-  Opts.EagerlyTrimEGraph = !Args.hasArg(OPT_analyzer_no_eagerly_trim_egraph);
+  Opts.maxBlockVisitOnPath = Args.getLastArgIntValue(OPT_analyzer_max_loop, 4, Diags);
   Opts.PrintStats = Args.hasArg(OPT_analyzer_stats);
   Opts.InlineMaxStackDepth =
     Args.getLastArgIntValue(OPT_analyzer_inline_max_stack_depth,
@@ -1148,12 +254,42 @@ static bool ParseAnalyzerArgs(AnalyzerOptions &Opts, ArgList &Args,
     bool enable = (A->getOption().getID() == OPT_analyzer_checker);
     // We can have a list of comma separated checker names, e.g:
     // '-analyzer-checker=cocoa,unix'
-    StringRef checkerList = A->getValue(Args);
+    StringRef checkerList = A->getValue();
     SmallVector<StringRef, 4> checkers;
     checkerList.split(checkers, ",");
     for (unsigned i = 0, e = checkers.size(); i != e; ++i)
       Opts.CheckersControlList.push_back(std::make_pair(checkers[i], enable));
   }
+  
+  // Go through the analyzer configuration options.
+  for (arg_iterator it = Args.filtered_begin(OPT_analyzer_config),
+       ie = Args.filtered_end(); it != ie; ++it) {
+    const Arg *A = *it;
+    A->claim();
+    // We can have a list of comma separated config names, e.g:
+    // '-analyzer-config key1=val1,key2=val2'
+    StringRef configList = A->getValue();
+    SmallVector<StringRef, 4> configVals;
+    configList.split(configVals, ",");
+    for (unsigned i = 0, e = configVals.size(); i != e; ++i) {
+      StringRef key, val;
+      llvm::tie(key, val) = configVals[i].split("=");
+      if (val.empty()) {
+        Diags.Report(SourceLocation(),
+                     diag::err_analyzer_config_no_value) << configVals[i];
+        Success = false;
+        break;
+      }
+      if (val.find('=') != StringRef::npos) {
+        Diags.Report(SourceLocation(),
+                     diag::err_analyzer_config_multiple_values)
+          << configVals[i];
+        Success = false;
+        break;
+      }
+      Opts.Config[key] = val;
+    }
+  }
 
   return Success;
 }
@@ -1179,21 +315,23 @@ static bool ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args, InputKind IK,
   Opts.OptimizationLevel = OptLevel;
 
   // We must always run at least the always inlining pass.
-  Opts.Inlining = (Opts.OptimizationLevel > 1) ? CodeGenOptions::NormalInlining
-    : CodeGenOptions::OnlyAlwaysInlining;
+  Opts.setInlining(
+    (Opts.OptimizationLevel > 1) ? CodeGenOptions::NormalInlining
+                                 : CodeGenOptions::OnlyAlwaysInlining);
   // -fno-inline-functions overrides OptimizationLevel > 1.
   Opts.NoInline = Args.hasArg(OPT_fno_inline);
-  Opts.Inlining = Args.hasArg(OPT_fno_inline_functions) ?
-    CodeGenOptions::OnlyAlwaysInlining : Opts.Inlining;
+  Opts.setInlining(Args.hasArg(OPT_fno_inline_functions) ?
+                     CodeGenOptions::OnlyAlwaysInlining : Opts.getInlining());
 
   if (Args.hasArg(OPT_gline_tables_only)) {
-    Opts.DebugInfo = CodeGenOptions::DebugLineTablesOnly;
+    Opts.setDebugInfo(CodeGenOptions::DebugLineTablesOnly);
   } else if (Args.hasArg(OPT_g_Flag)) {
     if (Args.hasFlag(OPT_flimit_debug_info, OPT_fno_limit_debug_info, true))
-      Opts.DebugInfo = CodeGenOptions::LimitedDebugInfo;
+      Opts.setDebugInfo(CodeGenOptions::LimitedDebugInfo);
     else
-      Opts.DebugInfo = CodeGenOptions::FullDebugInfo;
+      Opts.setDebugInfo(CodeGenOptions::FullDebugInfo);
   }
+  Opts.DebugColumnInfo = Args.hasArg(OPT_dwarf_column_info);
 
   Opts.DisableLLVMOpts = Args.hasArg(OPT_disable_llvm_optzns);
   Opts.DisableRedZone = Args.hasArg(OPT_disable_red_zone);
@@ -1263,18 +401,21 @@ static bool ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args, InputKind IK,
   Opts.EmitGcovArcs = Args.hasArg(OPT_femit_coverage_data);
   Opts.EmitGcovNotes = Args.hasArg(OPT_femit_coverage_notes);
   Opts.EmitOpenCLArgMetadata = Args.hasArg(OPT_cl_kernel_arg_info);
-  Opts.EmitMicrosoftInlineAsm = Args.hasArg(OPT_fenable_experimental_ms_inline_asm);
   Opts.CoverageFile = Args.getLastArgValue(OPT_coverage_file);
   Opts.DebugCompilationDir = Args.getLastArgValue(OPT_fdebug_compilation_dir);
   Opts.LinkBitcodeFile = Args.getLastArgValue(OPT_mlink_bitcode_file);
+  Opts.SSPBufferSize =
+    Args.getLastArgIntValue(OPT_stack_protector_buffer_size, 8, Diags);
   Opts.StackRealignment = Args.hasArg(OPT_mstackrealign);
   if (Arg *A = Args.getLastArg(OPT_mstack_alignment)) {
-    StringRef Val = A->getValue(Args);
-    Val.getAsInteger(10, Opts.StackAlignment);
+    StringRef Val = A->getValue();
+    unsigned StackAlignment = Opts.StackAlignment;
+    Val.getAsInteger(10, StackAlignment);
+    Opts.StackAlignment = StackAlignment;
   }
 
   if (Arg *A = Args.getLastArg(OPT_fobjc_dispatch_method_EQ)) {
-    StringRef Name = A->getValue(Args);
+    StringRef Name = A->getValue();
     unsigned Method = llvm::StringSwitch<unsigned>(Name)
       .Case("legacy", CodeGenOptions::Legacy)
       .Case("non-legacy", CodeGenOptions::NonLegacy)
@@ -1284,12 +425,13 @@ static bool ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args, InputKind IK,
       Diags.Report(diag::err_drv_invalid_value) << A->getAsString(Args) << Name;
       Success = false;
     } else {
-      Opts.ObjCDispatchMethod = Method;
+      Opts.setObjCDispatchMethod(
+        static_cast<CodeGenOptions::ObjCDispatchMethodKind>(Method));
     }
   }
 
   if (Arg *A = Args.getLastArg(OPT_ftlsmodel_EQ)) {
-    StringRef Name = A->getValue(Args);
+    StringRef Name = A->getValue();
     unsigned Model = llvm::StringSwitch<unsigned>(Name)
         .Case("global-dynamic", CodeGenOptions::GeneralDynamicTLSModel)
         .Case("local-dynamic", CodeGenOptions::LocalDynamicTLSModel)
@@ -1300,7 +442,7 @@ static bool ParseCodeGenArgs(CodeGenOptions &Opts, ArgList &Args, InputKind IK,
       Diags.Report(diag::err_drv_invalid_value) << A->getAsString(Args) << Name;
       Success = false;
     } else {
-      Opts.DefaultTLSModel = static_cast<CodeGenOptions::TLSModel>(Model);
+      Opts.setDefaultTLSModel(static_cast<CodeGenOptions::TLSModel>(Model));
     }
   }
 
@@ -1351,9 +493,9 @@ bool clang::ParseDiagnosticArgs(DiagnosticOptions &Opts, ArgList &Args,
   StringRef ShowOverloads =
     Args.getLastArgValue(OPT_fshow_overloads_EQ, "all");
   if (ShowOverloads == "best")
-    Opts.ShowOverloads = DiagnosticsEngine::Ovl_Best;
+    Opts.setShowOverloads(Ovl_Best);
   else if (ShowOverloads == "all")
-    Opts.ShowOverloads = DiagnosticsEngine::Ovl_All;
+    Opts.setShowOverloads(Ovl_All);
   else {
     Success = false;
     if (Diags)
@@ -1381,11 +523,11 @@ bool clang::ParseDiagnosticArgs(DiagnosticOptions &Opts, ArgList &Args,
   StringRef Format =
     Args.getLastArgValue(OPT_fdiagnostics_format, "clang");
   if (Format == "clang")
-    Opts.Format = DiagnosticOptions::Clang;
+    Opts.setFormat(DiagnosticOptions::Clang);
   else if (Format == "msvc")
-    Opts.Format = DiagnosticOptions::Msvc;
+    Opts.setFormat(DiagnosticOptions::Msvc);
   else if (Format == "vi")
-    Opts.Format = DiagnosticOptions::Vi;
+    Opts.setFormat(DiagnosticOptions::Vi);
   else {
     Success = false;
     if (Diags)
@@ -1467,7 +609,7 @@ static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
     case OPT_emit_obj:
       Opts.ProgramAction = frontend::EmitObj; break;
     case OPT_fixit_EQ:
-      Opts.FixItSuffix = A->getValue(Args);
+      Opts.FixItSuffix = A->getValue();
       // fall-through!
     case OPT_fixit:
       Opts.ProgramAction = frontend::FixIt; break;
@@ -1503,14 +645,14 @@ static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
   }
 
   if (const Arg* A = Args.getLastArg(OPT_plugin)) {
-    Opts.Plugins.push_back(A->getValue(Args,0));
+    Opts.Plugins.push_back(A->getValue(0));
     Opts.ProgramAction = frontend::PluginAction;
-    Opts.ActionName = A->getValue(Args);
+    Opts.ActionName = A->getValue();
 
     for (arg_iterator it = Args.filtered_begin(OPT_plugin_arg),
            end = Args.filtered_end(); it != end; ++it) {
-      if ((*it)->getValue(Args, 0) == Opts.ActionName)
-        Opts.PluginArgs.push_back((*it)->getValue(Args, 1));
+      if ((*it)->getValue(0) == Opts.ActionName)
+        Opts.PluginArgs.push_back((*it)->getValue(1));
     }
   }
 
@@ -1519,17 +661,17 @@ static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
   for (int i = 0, e = Opts.AddPluginActions.size(); i != e; ++i) {
     for (arg_iterator it = Args.filtered_begin(OPT_plugin_arg),
            end = Args.filtered_end(); it != end; ++it) {
-      if ((*it)->getValue(Args, 0) == Opts.AddPluginActions[i])
-        Opts.AddPluginArgs[i].push_back((*it)->getValue(Args, 1));
+      if ((*it)->getValue(0) == Opts.AddPluginActions[i])
+        Opts.AddPluginArgs[i].push_back((*it)->getValue(1));
     }
   }
 
   if (const Arg *A = Args.getLastArg(OPT_code_completion_at)) {
     Opts.CodeCompletionAt =
-      ParsedSourceLocation::FromString(A->getValue(Args));
+      ParsedSourceLocation::FromString(A->getValue());
     if (Opts.CodeCompletionAt.FileName.empty())
       Diags.Report(diag::err_drv_invalid_value)
-        << A->getAsString(Args) << A->getValue(Args);
+        << A->getAsString(Args) << A->getValue();
   }
   Opts.DisableFree = Args.hasArg(OPT_disable_free);
 
@@ -1595,7 +737,7 @@ static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
 
   InputKind DashX = IK_None;
   if (const Arg *A = Args.getLastArg(OPT_x)) {
-    DashX = llvm::StringSwitch<InputKind>(A->getValue(Args))
+    DashX = llvm::StringSwitch<InputKind>(A->getValue())
       .Case("c", IK_C)
       .Case("cl", IK_OpenCL)
       .Case("cuda", IK_CUDA)
@@ -1619,7 +761,7 @@ static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
       .Default(IK_None);
     if (DashX == IK_None)
       Diags.Report(diag::err_drv_invalid_value)
-        << A->getAsString(Args) << A->getValue(Args);
+        << A->getAsString(Args) << A->getValue();
   }
 
   // '-' is the default input if none is given.
@@ -1667,7 +809,7 @@ static void ParseHeaderSearchArgs(HeaderSearchOptions &Opts, ArgList &Args) {
   Opts.UseStandardSystemIncludes = !Args.hasArg(OPT_nostdsysteminc);
   Opts.UseStandardCXXIncludes = !Args.hasArg(OPT_nostdincxx);
   if (const Arg *A = Args.getLastArg(OPT_stdlib_EQ))
-    Opts.UseLibcxx = (strcmp(A->getValue(Args), "libc++") == 0);
+    Opts.UseLibcxx = (strcmp(A->getValue(), "libc++") == 0);
   Opts.ResourceDir = Args.getLastArgValue(OPT_resource_dir);
   Opts.ModuleCachePath = Args.getLastArgValue(OPT_fmodule_cache_path);
   Opts.DisableModuleHash = Args.hasArg(OPT_fdisable_module_hash);
@@ -1686,7 +828,7 @@ static void ParseHeaderSearchArgs(HeaderSearchOptions &Opts, ArgList &Args) {
     frontend::IncludeDirGroup Group 
       = IsIndexHeaderMap? frontend::IndexHeaderMap : frontend::Angled;
     
-    Opts.AddPath((*it)->getValue(Args), Group, true,
+    Opts.AddPath((*it)->getValue(), Group, true,
                  /*IsFramework=*/ (*it)->getOption().matches(OPT_F), false);
     IsIndexHeaderMap = false;
   }
@@ -1698,43 +840,43 @@ static void ParseHeaderSearchArgs(HeaderSearchOptions &Opts, ArgList &Args) {
          ie = Args.filtered_end(); it != ie; ++it) {
     const Arg *A = *it;
     if (A->getOption().matches(OPT_iprefix))
-      Prefix = A->getValue(Args);
+      Prefix = A->getValue();
     else if (A->getOption().matches(OPT_iwithprefix))
-      Opts.AddPath(Prefix.str() + A->getValue(Args),
+      Opts.AddPath(Prefix.str() + A->getValue(),
                    frontend::System, false, false, false);
     else
-      Opts.AddPath(Prefix.str() + A->getValue(Args),
+      Opts.AddPath(Prefix.str() + A->getValue(),
                    frontend::Angled, false, false, false);
   }
 
   for (arg_iterator it = Args.filtered_begin(OPT_idirafter),
          ie = Args.filtered_end(); it != ie; ++it)
-    Opts.AddPath((*it)->getValue(Args), frontend::After, true, false, false);
+    Opts.AddPath((*it)->getValue(), frontend::After, true, false, false);
   for (arg_iterator it = Args.filtered_begin(OPT_iquote),
          ie = Args.filtered_end(); it != ie; ++it)
-    Opts.AddPath((*it)->getValue(Args), frontend::Quoted, true, false, false);
+    Opts.AddPath((*it)->getValue(), frontend::Quoted, true, false, false);
   for (arg_iterator it = Args.filtered_begin(OPT_isystem,
          OPT_iwithsysroot), ie = Args.filtered_end(); it != ie; ++it)
-    Opts.AddPath((*it)->getValue(Args), frontend::System, true, false,
+    Opts.AddPath((*it)->getValue(), frontend::System, true, false,
                  !(*it)->getOption().matches(OPT_iwithsysroot));
   for (arg_iterator it = Args.filtered_begin(OPT_iframework),
          ie = Args.filtered_end(); it != ie; ++it)
-    Opts.AddPath((*it)->getValue(Args), frontend::System, true, true,
+    Opts.AddPath((*it)->getValue(), frontend::System, true, true,
                  true);
 
   // Add the paths for the various language specific isystem flags.
   for (arg_iterator it = Args.filtered_begin(OPT_c_isystem),
        ie = Args.filtered_end(); it != ie; ++it)
-    Opts.AddPath((*it)->getValue(Args), frontend::CSystem, true, false, true);
+    Opts.AddPath((*it)->getValue(), frontend::CSystem, true, false, true);
   for (arg_iterator it = Args.filtered_begin(OPT_cxx_isystem),
        ie = Args.filtered_end(); it != ie; ++it)
-    Opts.AddPath((*it)->getValue(Args), frontend::CXXSystem, true, false, true);
+    Opts.AddPath((*it)->getValue(), frontend::CXXSystem, true, false, true);
   for (arg_iterator it = Args.filtered_begin(OPT_objc_isystem),
        ie = Args.filtered_end(); it != ie; ++it)
-    Opts.AddPath((*it)->getValue(Args), frontend::ObjCSystem, true, false,true);
+    Opts.AddPath((*it)->getValue(), frontend::ObjCSystem, true, false,true);
   for (arg_iterator it = Args.filtered_begin(OPT_objcxx_isystem),
        ie = Args.filtered_end(); it != ie; ++it)
-    Opts.AddPath((*it)->getValue(Args), frontend::ObjCXXSystem, true, false,
+    Opts.AddPath((*it)->getValue(), frontend::ObjCXXSystem, true, false,
                  true);
 
   // Add the internal paths from a driver that detects standard include paths.
@@ -1742,7 +884,7 @@ static void ParseHeaderSearchArgs(HeaderSearchOptions &Opts, ArgList &Args) {
                                             OPT_internal_externc_isystem),
                     E = Args.filtered_end();
        I != E; ++I)
-    Opts.AddPath((*I)->getValue(Args), frontend::System,
+    Opts.AddPath((*I)->getValue(), frontend::System,
                  false, false, /*IgnoreSysRoot=*/true, /*IsInternal=*/true,
                  (*I)->getOption().matches(OPT_internal_externc_isystem));
 
@@ -1751,7 +893,7 @@ static void ParseHeaderSearchArgs(HeaderSearchOptions &Opts, ArgList &Args) {
                                             OPT_ino_system_prefix),
                     E = Args.filtered_end();
        I != E; ++I)
-    Opts.AddSystemHeaderPrefix((*I)->getValue(Args),
+    Opts.AddSystemHeaderPrefix((*I)->getValue(),
                                (*I)->getOption().matches(OPT_isystem_prefix));
 }
 
@@ -1799,11 +941,12 @@ void CompilerInvocation::setLangDefaults(LangOptions &Opts, InputKind IK,
   }
 
   const LangStandard &Std = LangStandard::getLangStandardForKind(LangStd);
-  Opts.BCPLComment = Std.hasBCPLComments();
+  Opts.LineComment = Std.hasLineComments();
   Opts.C99 = Std.isC99();
   Opts.C11 = Std.isC11();
   Opts.CPlusPlus = Std.isCPlusPlus();
   Opts.CPlusPlus0x = Std.isCPlusPlus0x();
+  Opts.CPlusPlus1y = Std.isCPlusPlus1y();
   Opts.Digraphs = Std.hasDigraphs();
   Opts.GNUMode = Std.isGNUMode();
   Opts.GNUInline = !Std.isC99();
@@ -1838,6 +981,9 @@ void CompilerInvocation::setLangDefaults(LangOptions &Opts, InputKind IK,
   // OpenCL and C++ both have bool, true, false keywords.
   Opts.Bool = Opts.OpenCL || Opts.CPlusPlus;
 
+  // C++ has wchar_t keyword.
+  Opts.WChar = Opts.CPlusPlus;
+
   Opts.GNUKeywords = Opts.GNUMode;
   Opts.CXXOperatorNames = Opts.CPlusPlus;
 
@@ -1853,14 +999,14 @@ static void ParseLangArgs(LangOptions &Opts, ArgList &Args, InputKind IK,
   // FIXME: Cleanup per-file based stuff.
   LangStandard::Kind LangStd = LangStandard::lang_unspecified;
   if (const Arg *A = Args.getLastArg(OPT_std_EQ)) {
-    LangStd = llvm::StringSwitch<LangStandard::Kind>(A->getValue(Args))
+    LangStd = llvm::StringSwitch<LangStandard::Kind>(A->getValue())
 #define LANGSTANDARD(id, name, desc, features) \
       .Case(name, LangStandard::lang_##id)
 #include "clang/Frontend/LangStandards.def"
       .Default(LangStandard::lang_unspecified);
     if (LangStd == LangStandard::lang_unspecified)
       Diags.Report(diag::err_drv_invalid_value)
-        << A->getAsString(Args) << A->getValue(Args);
+        << A->getAsString(Args) << A->getValue();
     else {
       // Valid standard, check to make sure language and standard are compatable.    
       const LangStandard &Std = LangStandard::getLangStandardForKind(LangStd);
@@ -1901,7 +1047,7 @@ static void ParseLangArgs(LangOptions &Opts, ArgList &Args, InputKind IK,
   // Override the -std option in this case.
   if (const Arg *A = Args.getLastArg(OPT_cl_std_EQ)) {
     LangStandard::Kind OpenCLLangStd
-    = llvm::StringSwitch<LangStandard::Kind>(A->getValue(Args))
+    = llvm::StringSwitch<LangStandard::Kind>(A->getValue())
     .Case("CL", LangStandard::lang_opencl)
     .Case("CL1.1", LangStandard::lang_opencl11)
     .Case("CL1.2", LangStandard::lang_opencl12)
@@ -1909,7 +1055,7 @@ static void ParseLangArgs(LangOptions &Opts, ArgList &Args, InputKind IK,
     
     if (OpenCLLangStd == LangStandard::lang_unspecified) {
       Diags.Report(diag::err_drv_invalid_value)
-      << A->getAsString(Args) << A->getValue(Args);
+      << A->getAsString(Args) << A->getValue();
     }
     else
       LangStd = OpenCLLangStd;
@@ -1930,7 +1076,7 @@ static void ParseLangArgs(LangOptions &Opts, ArgList &Args, InputKind IK,
 
   if (Opts.ObjC1) {
     if (Arg *arg = Args.getLastArg(OPT_fobjc_runtime_EQ)) {
-      StringRef value = arg->getValue(Args);
+      StringRef value = arg->getValue();
       if (Opts.ObjCRuntime.tryParse(value))
         Diags.Report(diag::err_drv_unknown_objc_runtime) << value;
     }
@@ -1941,14 +1087,16 @@ static void ParseLangArgs(LangOptions &Opts, ArgList &Args, InputKind IK,
       Opts.setGC(LangOptions::HybridGC);
     else if (Args.hasArg(OPT_fobjc_arc)) {
       Opts.ObjCAutoRefCount = 1;
-      if (!Opts.ObjCRuntime.isNonFragile())
-        Diags.Report(diag::err_arc_nonfragile_abi);
+      if (!Opts.ObjCRuntime.allowsARC())
+        Diags.Report(diag::err_arc_unsupported_on_runtime);
+
+      // Only set ObjCARCWeak if ARC is enabled.
+      if (Args.hasArg(OPT_fobjc_runtime_has_weak))
+        Opts.ObjCARCWeak = 1;
+      else
+        Opts.ObjCARCWeak = Opts.ObjCRuntime.allowsWeak();
     }
 
-    Opts.ObjCRuntimeHasWeak = Opts.ObjCRuntime.hasWeak();
-    if (Args.hasArg(OPT_fobjc_runtime_has_weak))
-      Opts.ObjCRuntimeHasWeak = 1;
-
     if (Args.hasArg(OPT_fno_objc_infer_related_result_type))
       Opts.ObjCInferRelatedResultType = 0;
   }
@@ -1990,7 +1138,7 @@ static void ParseLangArgs(LangOptions &Opts, ArgList &Args, InputKind IK,
       << Args.getLastArg(OPT_fvisibility)->getAsString(Args) << Vis;
 
   if (Arg *A = Args.getLastArg(OPT_ffp_contract)) {
-    StringRef Val = A->getValue(Args);
+    StringRef Val = A->getValue();
     if (Val == "fast")
       Opts.setFPContractMode(LangOptions::FPC_Fast);
     else if (Val == "on")
@@ -2043,6 +1191,7 @@ static void ParseLangArgs(LangOptions &Opts, ArgList &Args, InputKind IK,
   Opts.BlocksRuntimeOptional = Args.hasArg(OPT_fblocks_runtime_optional);
   Opts.Modules = Args.hasArg(OPT_fmodules);
   Opts.CharIsSigned = !Args.hasArg(OPT_fno_signed_char);
+  Opts.WChar = Opts.CPlusPlus && !Args.hasArg(OPT_fno_wchar);
   Opts.ShortWChar = Args.hasArg(OPT_fshort_wchar);
   Opts.ShortEnums = Args.hasArg(OPT_fshort_enums);
   Opts.Freestanding = Args.hasArg(OPT_ffreestanding);
@@ -2064,7 +1213,8 @@ static void ParseLangArgs(LangOptions &Opts, ArgList &Args, InputKind IK,
     Args.getLastArgValue(OPT_fconstant_string_class);
   Opts.ObjCDefaultSynthProperties =
     Args.hasArg(OPT_fobjc_default_synthesize_properties);
-  Opts.CatchUndefined = Args.hasArg(OPT_fcatch_undefined_behavior);
+  Opts.EncodeExtendedBlockSig =
+    Args.hasArg(OPT_fencode_extended_block_signature);
   Opts.EmitAllDecls = Args.hasArg(OPT_femit_all_decls);
   Opts.PackStruct = Args.getLastArgIntValue(OPT_fpack_struct_EQ, 0, Diags);
   Opts.PICLevel = Args.getLastArgIntValue(OPT_pic_level, 0, Diags);
@@ -2085,8 +1235,6 @@ static void ParseLangArgs(LangOptions &Opts, ArgList &Args, InputKind IK,
   Opts.DebuggerSupport = Args.hasArg(OPT_fdebugger_support);
   Opts.DebuggerCastResultToId = Args.hasArg(OPT_fdebugger_cast_result_to_id);
   Opts.DebuggerObjCLiteral = Args.hasArg(OPT_fdebugger_objc_literal);
-  Opts.AddressSanitizer = Args.hasArg(OPT_faddress_sanitizer);
-  Opts.ThreadSanitizer = Args.hasArg(OPT_fthread_sanitizer);
   Opts.ApplePragmaPack = Args.hasArg(OPT_fapple_pragma_pack);
   Opts.CurrentModule = Args.getLastArgValue(OPT_fmodule_name);
 
@@ -2109,6 +1257,11 @@ static void ParseLangArgs(LangOptions &Opts, ArgList &Args, InputKind IK,
   Opts.FastMath = Args.hasArg(OPT_ffast_math);
   Opts.FiniteMathOnly = Args.hasArg(OPT_ffinite_math_only);
 
+  Opts.EmitMicrosoftInlineAsm = Args.hasArg(OPT_fenable_experimental_ms_inline_asm);
+
+  Opts.RetainCommentsFromSystemHeaders =
+      Args.hasArg(OPT_fretain_comments_from_system_headers);
+
   unsigned SSP = Args.getLastArgIntValue(OPT_stack_protector, 0, Diags);
   switch (SSP) {
   default:
@@ -2119,6 +1272,37 @@ static void ParseLangArgs(LangOptions &Opts, ArgList &Args, InputKind IK,
   case 1: Opts.setStackProtector(LangOptions::SSPOn);  break;
   case 2: Opts.setStackProtector(LangOptions::SSPReq); break;
   }
+
+  // Parse -fsanitize= arguments.
+  std::vector<std::string> Sanitizers = Args.getAllArgValues(OPT_fsanitize_EQ);
+  for (unsigned I = 0, N = Sanitizers.size(); I != N; ++I) {
+    // Since the Opts.Sanitize* values are bitfields, it's a little tricky to
+    // efficiently map string values to them. Perform the mapping indirectly:
+    // convert strings to enumerated values, then switch over the enum to set
+    // the right bitfield value.
+    enum Sanitizer {
+#define SANITIZER(NAME, ID) \
+      ID,
+#include "clang/Basic/Sanitizers.def"
+      Unknown
+    };
+    switch (llvm::StringSwitch<unsigned>(Sanitizers[I])
+#define SANITIZER(NAME, ID) \
+              .Case(NAME, ID)
+#include "clang/Basic/Sanitizers.def"
+              .Default(Unknown)) {
+#define SANITIZER(NAME, ID) \
+    case ID: \
+      Opts.Sanitize##ID = true; \
+      break;
+#include "clang/Basic/Sanitizers.def"
+
+    case Unknown:
+      Diags.Report(diag::err_drv_invalid_value)
+        << "-fsanitize=" << Sanitizers[I];
+      break;
+    }
+  }
 }
 
 static void ParsePreprocessorArgs(PreprocessorOptions &Opts, ArgList &Args,
@@ -2128,7 +1312,7 @@ static void ParsePreprocessorArgs(PreprocessorOptions &Opts, ArgList &Args,
   Opts.ImplicitPCHInclude = Args.getLastArgValue(OPT_include_pch);
   Opts.ImplicitPTHInclude = Args.getLastArgValue(OPT_include_pth);
   if (const Arg *A = Args.getLastArg(OPT_token_cache))
-      Opts.TokenCache = A->getValue(Args);
+      Opts.TokenCache = A->getValue();
   else
     Opts.TokenCache = Opts.ImplicitPTHInclude;
   Opts.UsePredefines = !Args.hasArg(OPT_undef);
@@ -2139,11 +1323,11 @@ static void ParsePreprocessorArgs(PreprocessorOptions &Opts, ArgList &Args,
   for (arg_iterator it = Args.filtered_begin(OPT_error_on_deserialized_pch_decl),
          ie = Args.filtered_end(); it != ie; ++it) {
     const Arg *A = *it;
-    Opts.DeserializedPCHDeclsToErrorOn.insert(A->getValue(Args));
+    Opts.DeserializedPCHDeclsToErrorOn.insert(A->getValue());
   }
 
   if (const Arg *A = Args.getLastArg(OPT_preamble_bytes_EQ)) {
-    StringRef Value(A->getValue(Args));
+    StringRef Value(A->getValue());
     size_t Comma = Value.find(',');
     unsigned Bytes = 0;
     unsigned EndOfLine = 0;
@@ -2162,9 +1346,9 @@ static void ParsePreprocessorArgs(PreprocessorOptions &Opts, ArgList &Args,
   for (arg_iterator it = Args.filtered_begin(OPT_D, OPT_U),
          ie = Args.filtered_end(); it != ie; ++it) {
     if ((*it)->getOption().matches(OPT_D))
-      Opts.addMacroDef((*it)->getValue(Args));
+      Opts.addMacroDef((*it)->getValue());
     else
-      Opts.addMacroUndef((*it)->getValue(Args));
+      Opts.addMacroUndef((*it)->getValue());
   }
 
   Opts.MacroIncludes = Args.getAllArgValues(OPT_imacros);
@@ -2174,22 +1358,13 @@ static void ParsePreprocessorArgs(PreprocessorOptions &Opts, ArgList &Args,
                                              OPT_include_pth),
          ie = Args.filtered_end(); it != ie; ++it) {
     const Arg *A = *it;
-    // PCH is handled specially, we need to extra the original include path.
-    if (A->getOption().matches(OPT_include_pch)) {
-      std::string OriginalFile =
-        ASTReader::getOriginalSourceFile(A->getValue(Args), FileMgr, Diags);
-      if (OriginalFile.empty())
-        continue;
-
-      Opts.Includes.push_back(OriginalFile);
-    } else
-      Opts.Includes.push_back(A->getValue(Args));
+    Opts.Includes.push_back(A->getValue());
   }
 
   for (arg_iterator it = Args.filtered_begin(OPT_chain_include),
          ie = Args.filtered_end(); it != ie; ++it) {
     const Arg *A = *it;
-    Opts.ChainedIncludes.push_back(A->getValue(Args));
+    Opts.ChainedIncludes.push_back(A->getValue());
   }
 
   // Include 'altivec.h' if -faltivec option present
@@ -2200,7 +1375,7 @@ static void ParsePreprocessorArgs(PreprocessorOptions &Opts, ArgList &Args,
          ie = Args.filtered_end(); it != ie; ++it) {
     const Arg *A = *it;
     std::pair<StringRef,StringRef> Split =
-      StringRef(A->getValue(Args)).split(';');
+      StringRef(A->getValue()).split(';');
 
     if (Split.second.empty()) {
       Diags.Report(diag::err_drv_invalid_remap_file) << A->getAsString(Args);
@@ -2211,7 +1386,7 @@ static void ParsePreprocessorArgs(PreprocessorOptions &Opts, ArgList &Args,
   }
   
   if (Arg *A = Args.getLastArg(OPT_fobjc_arc_cxxlib_EQ)) {
-    StringRef Name = A->getValue(Args);
+    StringRef Name = A->getValue();
     unsigned Library = llvm::StringSwitch<unsigned>(Name)
       .Case("libc++", ARCXX_libcxx)
       .Case("libstdc++", ARCXX_libstdcxx)
@@ -2240,7 +1415,7 @@ static void ParseTargetArgs(TargetOptions &Opts, ArgList &Args) {
   Opts.ABI = Args.getLastArgValue(OPT_target_abi);
   Opts.CXXABI = Args.getLastArgValue(OPT_cxx_abi);
   Opts.CPU = Args.getLastArgValue(OPT_target_cpu);
-  Opts.Features = Args.getAllArgValues(OPT_target_feature);
+  Opts.FeaturesAsWritten = Args.getAllArgValues(OPT_target_feature);
   Opts.LinkerVersion = Args.getLastArgValue(OPT_target_linker_version);
   Opts.Triple = llvm::Triple::normalize(Args.getLastArgValue(OPT_triple));
 
@@ -2280,13 +1455,13 @@ bool CompilerInvocation::CreateFromArgs(CompilerInvocation &Res,
   // Issue errors on arguments that are not valid for CC1.
   for (ArgList::iterator I = Args->begin(), E = Args->end();
        I != E; ++I) {
-    if (!(*I)->getOption().isCC1Option()) {
+    if (!(*I)->getOption().hasFlag(options::CC1Option)) {
       Diags.Report(diag::err_drv_unknown_argument) << (*I)->getAsString(*Args);
       Success = false;
     }
   }
 
-  Success = ParseAnalyzerArgs(Res.getAnalyzerOpts(), *Args, Diags) && Success;
+  Success = ParseAnalyzerArgs(*Res.getAnalyzerOpts(), *Args, Diags) && Success;
   Success = ParseMigratorArgs(Res.getMigratorOpts(), *Args) && Success;
   ParseDependencyOutputArgs(Res.getDependencyOutputOpts(), *Args);
   Success = ParseDiagnosticArgs(Res.getDiagnosticOpts(), *Args, &Diags)
@@ -2368,53 +1543,49 @@ llvm::APInt ModuleSignature::getAsInteger() const {
 }
 
 std::string CompilerInvocation::getModuleHash() const {
-  ModuleSignature Signature;
-  
+  using llvm::hash_code;
+  using llvm::hash_value;
+  using llvm::hash_combine;
+
   // Start the signature with the compiler version.
-  // FIXME: The full version string can be quite long.  Omit it from the
-  // module hash for now to avoid failures where the path name becomes too
-  // long.  An MD5 or similar checksum would work well here.
-  // Signature.add(getClangFullRepositoryVersion());
-  
+  // FIXME: We'd rather use something more cryptographically sound than
+  // CityHash, but this will do for now.
+  hash_code code = hash_value(getClangFullRepositoryVersion());
+
   // Extend the signature with the language options
 #define LANGOPT(Name, Bits, Default, Description) \
-  Signature.add(LangOpts->Name, Bits);
+   code = hash_combine(code, LangOpts->Name);
 #define ENUM_LANGOPT(Name, Type, Bits, Default, Description) \
-  Signature.add(static_cast<unsigned>(LangOpts->get##Name()), Bits);
+  code = hash_combine(code, static_cast<unsigned>(LangOpts->get##Name()));
 #define BENIGN_LANGOPT(Name, Bits, Default, Description)
 #define BENIGN_ENUM_LANGOPT(Name, Type, Bits, Default, Description)
 #include "clang/Basic/LangOptions.def"
   
-  // Extend the signature with the target triple
-  llvm::Triple T(TargetOpts.Triple);
-  Signature.add((unsigned)T.getArch(), 5);
-  Signature.add((unsigned)T.getVendor(), 4);
-  Signature.add((unsigned)T.getOS(), 5);
-  Signature.add((unsigned)T.getEnvironment(), 4);
+  // Extend the signature with the target options.
+  code = hash_combine(code, TargetOpts->Triple, TargetOpts->CPU,
+                      TargetOpts->ABI, TargetOpts->CXXABI,
+                      TargetOpts->LinkerVersion);
+  for (unsigned i = 0, n = TargetOpts->FeaturesAsWritten.size(); i != n; ++i)
+    code = hash_combine(code, TargetOpts->FeaturesAsWritten[i]);
 
   // Extend the signature with preprocessor options.
-  Signature.add(getPreprocessorOpts().UsePredefines, 1);
-  Signature.add(getPreprocessorOpts().DetailedRecord, 1);
-  
-  // Hash the preprocessor defines.
-  // FIXME: This is terrible. Use an MD5 sum of the preprocessor defines.
+  const PreprocessorOptions &ppOpts = getPreprocessorOpts();
+  code = hash_combine(code, ppOpts.UsePredefines, ppOpts.DetailedRecord);
+
   std::vector<StringRef> MacroDefs;
   for (std::vector<std::pair<std::string, bool/*isUndef*/> >::const_iterator 
             I = getPreprocessorOpts().Macros.begin(),
          IEnd = getPreprocessorOpts().Macros.end();
        I != IEnd; ++I) {
-    if (!I->second)
-      MacroDefs.push_back(I->first);
+    code = hash_combine(code, I->first, I->second);
   }
-  llvm::array_pod_sort(MacroDefs.begin(), MacroDefs.end());
-       
-  unsigned PPHashResult = 0;
-  for (unsigned I = 0, N = MacroDefs.size(); I != N; ++I)
-    PPHashResult = llvm::HashString(MacroDefs[I], PPHashResult);
-  Signature.add(PPHashResult, 32);
-  
-  // We've generated the signature. Treat it as one large APInt that we'll
-  // encode in base-36 and return.
-  Signature.flush();
-  return Signature.getAsInteger().toString(36, /*Signed=*/false);
+
+  // Extend the signature with the sysroot.
+  const HeaderSearchOptions &hsOpts = getHeaderSearchOpts();
+  code = hash_combine(code, hsOpts.Sysroot, hsOpts.UseBuiltinIncludes,
+                      hsOpts.UseStandardSystemIncludes,
+                      hsOpts.UseStandardCXXIncludes,
+                      hsOpts.UseLibcxx);
+
+  return llvm::APInt(64, code).toString(36, /*Signed=*/false);
 }
diff --git a/lib/Frontend/CreateInvocationFromCommandLine.cpp b/lib/Frontend/CreateInvocationFromCommandLine.cpp
index 0aca86e..d82cb6d 100644
--- a/lib/Frontend/CreateInvocationFromCommandLine.cpp
+++ b/lib/Frontend/CreateInvocationFromCommandLine.cpp
@@ -11,9 +11,9 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "clang/Basic/DiagnosticOptions.h"
 #include "clang/Frontend/Utils.h"
 #include "clang/Frontend/CompilerInstance.h"
-#include "clang/Frontend/DiagnosticOptions.h"
 #include "clang/Frontend/FrontendDiagnostic.h"
 #include "clang/Driver/Compilation.h"
 #include "clang/Driver/Driver.h"
@@ -34,8 +34,8 @@ clang::createInvocationFromCommandLine(ArrayRef<const char *> ArgList,
   if (!Diags.getPtr()) {
     // No diagnostics engine was provided, so create our own diagnostics object
     // with the default options.
-    DiagnosticOptions DiagOpts;
-    Diags = CompilerInstance::createDiagnostics(DiagOpts, ArgList.size(),
+    Diags = CompilerInstance::createDiagnostics(new DiagnosticOptions,
+                                                ArgList.size(),
                                                 ArgList.begin());
   }
 
@@ -49,11 +49,6 @@ clang::createInvocationFromCommandLine(ArrayRef<const char *> ArgList,
   // FIXME: We shouldn't have to pass in the path info.
   driver::Driver TheDriver("clang", llvm::sys::getDefaultTargetTriple(),
                            "a.out", false, *Diags);
-  // Force driver to use clang.
-  // FIXME: This seems like a hack. Maybe the "Clang" tool subclass should be
-  // available for using it to get the arguments, thus avoiding the overkill
-  // of using the driver.
-  TheDriver.setForcedClangUse();
 
   // Don't check that inputs exist, they may have been remapped.
   TheDriver.setCheckInputsExist(false);
diff --git a/lib/Frontend/DependencyFile.cpp b/lib/Frontend/DependencyFile.cpp
index 21f5daa..53ea8be 100644
--- a/lib/Frontend/DependencyFile.cpp
+++ b/lib/Frontend/DependencyFile.cpp
@@ -59,10 +59,11 @@ public:
                                   const Token &IncludeTok,
                                   StringRef FileName,
                                   bool IsAngled,
+                                  CharSourceRange FilenameRange,
                                   const FileEntry *File,
-                                  SourceLocation EndLoc,
                                   StringRef SearchPath,
-                                  StringRef RelativePath);
+                                  StringRef RelativePath,
+                                  const Module *Imported);
 
   virtual void EndOfMainFile() {
     OutputDependencyFile();
@@ -132,10 +133,11 @@ void DependencyFileCallback::InclusionDirective(SourceLocation HashLoc,
                                                 const Token &IncludeTok,
                                                 StringRef FileName,
                                                 bool IsAngled,
+                                                CharSourceRange FilenameRange,
                                                 const FileEntry *File,
-                                                SourceLocation EndLoc,
                                                 StringRef SearchPath,
-                                                StringRef RelativePath) {
+                                                StringRef RelativePath,
+                                                const Module *Imported) {
   if (!File) {
     if (AddMissingHeaderDeps)
       AddFilename(FileName);
diff --git a/lib/Frontend/DependencyGraph.cpp b/lib/Frontend/DependencyGraph.cpp
index eebaf0c..28d9c5d 100644
--- a/lib/Frontend/DependencyGraph.cpp
+++ b/lib/Frontend/DependencyGraph.cpp
@@ -51,10 +51,11 @@ public:
                                   const Token &IncludeTok,
                                   StringRef FileName,
                                   bool IsAngled,
+                                  CharSourceRange FilenameRange,
                                   const FileEntry *File,
-                                  SourceLocation EndLoc,
                                   StringRef SearchPath,
-                                  StringRef RelativePath);
+                                  StringRef RelativePath,
+                                  const Module *Imported);
 
   virtual void EndOfMainFile() {
     OutputGraphFile();
@@ -72,10 +73,11 @@ void DependencyGraphCallback::InclusionDirective(SourceLocation HashLoc,
                                                  const Token &IncludeTok,
                                                  StringRef FileName,
                                                  bool IsAngled,
+                                                 CharSourceRange FilenameRange,
                                                  const FileEntry *File,
-                                                 SourceLocation EndLoc,
                                                  StringRef SearchPath,
-                                                 StringRef RelativePath) {
+                                                 StringRef RelativePath,
+                                                 const Module *Imported) {
   if (!File)
     return;
   
diff --git a/lib/Frontend/DiagnosticRenderer.cpp b/lib/Frontend/DiagnosticRenderer.cpp
index f052f90..359b82b 100644
--- a/lib/Frontend/DiagnosticRenderer.cpp
+++ b/lib/Frontend/DiagnosticRenderer.cpp
@@ -8,9 +8,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Frontend/DiagnosticRenderer.h"
+#include "clang/Basic/DiagnosticOptions.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/SourceManager.h"
-#include "clang/Frontend/DiagnosticOptions.h"
 #include "clang/Lex/Lexer.h"
 #include "clang/Edit/EditedSource.h"
 #include "clang/Edit/Commit.h"
@@ -18,6 +18,7 @@
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/SmallString.h"
 #include <algorithm>
 using namespace clang;
@@ -60,8 +61,8 @@ static StringRef getImmediateMacroName(SourceLocation Loc,
 }
 
 DiagnosticRenderer::DiagnosticRenderer(const LangOptions &LangOpts,
-                                       const DiagnosticOptions &DiagOpts)
-: LangOpts(LangOpts), DiagOpts(DiagOpts), LastLevel() {}
+                                       DiagnosticOptions *DiagOpts)
+  : LangOpts(LangOpts), DiagOpts(DiagOpts), LastLevel() {}
 
 DiagnosticRenderer::~DiagnosticRenderer() {}
 
@@ -194,7 +195,7 @@ void DiagnosticRenderer::emitIncludeStack(SourceLocation Loc,
     return;
   LastIncludeLoc = Loc;
   
-  if (!DiagOpts.ShowNoteIncludeStack && Level == DiagnosticsEngine::Note)
+  if (!DiagOpts->ShowNoteIncludeStack && Level == DiagnosticsEngine::Note)
     return;
   
   emitIncludeStackRecursively(Loc, SM);
@@ -218,6 +219,53 @@ void DiagnosticRenderer::emitIncludeStackRecursively(SourceLocation Loc,
   emitIncludeLocation(Loc, PLoc, SM);
 }
 
+// Helper function to fix up source ranges.  It takes in an array of ranges,
+// and outputs an array of ranges where we want to draw the range highlighting
+// around the location specified by CaretLoc.
+//
+// To find locations which correspond to the caret, we crawl the macro caller
+// chain for the beginning and end of each range.  If the caret location
+// is in a macro expansion, we search each chain for a location
+// in the same expansion as the caret; otherwise, we crawl to the top of
+// each chain. Two locations are part of the same macro expansion
+// iff the FileID is the same.
+static void mapDiagnosticRanges(
+    SourceLocation CaretLoc,
+    const SmallVectorImpl<CharSourceRange>& Ranges,
+    SmallVectorImpl<CharSourceRange>& SpellingRanges,
+    const SourceManager *SM) {
+  FileID CaretLocFileID = SM->getFileID(CaretLoc);
+
+  for (SmallVectorImpl<CharSourceRange>::const_iterator I = Ranges.begin(),
+       E = Ranges.end();
+       I != E; ++I) {
+    SourceLocation Begin = I->getBegin(), End = I->getEnd();
+    bool IsTokenRange = I->isTokenRange();
+
+    // Search the macro caller chain for the beginning of the range.
+    while (Begin.isMacroID() && SM->getFileID(Begin) != CaretLocFileID)
+      Begin = SM->getImmediateMacroCallerLoc(Begin);
+
+    // Search the macro caller chain for the beginning of the range.
+    while (End.isMacroID() && SM->getFileID(End) != CaretLocFileID) {
+      // The computation of the next End is an inlined version of
+      // getImmediateMacroCallerLoc, except it chooses the end of an
+      // expansion range.
+      if (SM->isMacroArgExpansion(End)) {
+        End = SM->getImmediateSpellingLoc(End);
+      } else {
+        End = SM->getImmediateExpansionRange(End).second;
+      }
+    }
+
+    // Return the spelling location of the beginning and end of the range.
+    Begin = SM->getSpellingLoc(Begin);
+    End = SM->getSpellingLoc(End);
+    SpellingRanges.push_back(CharSourceRange(SourceRange(Begin, End),
+                                             IsTokenRange));
+  }
+}
+
 /// \brief Recursively emit notes for each macro expansion and caret
 /// diagnostics where appropriate.
 ///
@@ -245,9 +293,13 @@ void DiagnosticRenderer::emitMacroExpansionsAndCarets(
   // If this is a file source location, directly emit the source snippet and
   // caret line. Also record the macro depth reached.
   if (Loc.isFileID()) {
+    // Map the ranges.
+    SmallVector<CharSourceRange, 4> SpellingRanges;
+    mapDiagnosticRanges(Loc, Ranges, SpellingRanges, &SM);
+
     assert(MacroDepth == 0 && "We shouldn't hit a leaf node twice!");
     MacroDepth = OnMacroInst;
-    emitCodeContext(Loc, Level, Ranges, Hints, SM);
+    emitCodeContext(Loc, Level, SpellingRanges, Hints, SM);
     return;
   }
   // Otherwise recurse through each macro expansion layer.
@@ -257,8 +309,7 @@ void DiagnosticRenderer::emitMacroExpansionsAndCarets(
   Loc = SM.skipToMacroArgExpansion(Loc);
   
   SourceLocation OneLevelUp = SM.getImmediateMacroCallerLoc(Loc);
-  
-  // FIXME: Map ranges?
+
   emitMacroExpansionsAndCarets(OneLevelUp, Level, Ranges, Hints, SM, MacroDepth,
                                OnMacroInst + 1);
   
@@ -269,28 +320,17 @@ void DiagnosticRenderer::emitMacroExpansionsAndCarets(
   Loc = SM.getImmediateMacroCalleeLoc(Loc);
   
   unsigned MacroSkipStart = 0, MacroSkipEnd = 0;
-  if (MacroDepth > DiagOpts.MacroBacktraceLimit &&
-      DiagOpts.MacroBacktraceLimit != 0) {
-    MacroSkipStart = DiagOpts.MacroBacktraceLimit / 2 +
-    DiagOpts.MacroBacktraceLimit % 2;
-    MacroSkipEnd = MacroDepth - DiagOpts.MacroBacktraceLimit / 2;
+  if (MacroDepth > DiagOpts->MacroBacktraceLimit &&
+      DiagOpts->MacroBacktraceLimit != 0) {
+    MacroSkipStart = DiagOpts->MacroBacktraceLimit / 2 +
+    DiagOpts->MacroBacktraceLimit % 2;
+    MacroSkipEnd = MacroDepth - DiagOpts->MacroBacktraceLimit / 2;
   }
   
   // Whether to suppress printing this macro expansion.
   bool Suppressed = (OnMacroInst >= MacroSkipStart &&
                      OnMacroInst < MacroSkipEnd);
   
-  // Map the ranges.
-  for (SmallVectorImpl<CharSourceRange>::iterator I = Ranges.begin(),
-       E = Ranges.end();
-       I != E; ++I) {
-    SourceLocation Start = I->getBegin(), End = I->getEnd();
-    if (Start.isMacroID())
-      I->setBegin(SM.getImmediateMacroCalleeLoc(Start));
-    if (End.isMacroID())
-      I->setEnd(SM.getImmediateMacroCalleeLoc(End));
-  }
-  
   if (Suppressed) {
     // Tell the user that we've skipped contexts.
     if (OnMacroInst == MacroSkipStart) {
@@ -303,14 +343,18 @@ void DiagnosticRenderer::emitMacroExpansionsAndCarets(
     }
     return;
   }
-  
+
+  // Map the ranges.
+  SmallVector<CharSourceRange, 4> SpellingRanges;
+  mapDiagnosticRanges(MacroLoc, Ranges, SpellingRanges, &SM);
+
   SmallString<100> MessageStorage;
   llvm::raw_svector_ostream Message(MessageStorage);
   Message << "expanded from macro '"
           << getImmediateMacroName(MacroLoc, SM, LangOpts) << "'";
   emitDiagnostic(SM.getSpellingLoc(Loc), DiagnosticsEngine::Note,
                  Message.str(),
-                 Ranges, ArrayRef<FixItHint>(), &SM);
+                 SpellingRanges, ArrayRef<FixItHint>(), &SM);
 }
 
 DiagnosticNoteRenderer::~DiagnosticNoteRenderer() {}
diff --git a/lib/Frontend/FrontendAction.cpp b/lib/Frontend/FrontendAction.cpp
index a4321e7..2e9a791 100644
--- a/lib/Frontend/FrontendAction.cpp
+++ b/lib/Frontend/FrontendAction.cpp
@@ -23,10 +23,12 @@
 #include "clang/Parse/ParseAST.h"
 #include "clang/Serialization/ASTDeserializationListener.h"
 #include "clang/Serialization/ASTReader.h"
-#include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/Timer.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/system_error.h"
+#include "llvm/Support/Timer.h"
 using namespace clang;
 
 namespace {
@@ -155,20 +157,22 @@ ASTConsumer* FrontendAction::CreateWrappedASTConsumer(CompilerInstance &CI,
   return new MultiplexConsumer(Consumers);
 }
 
+
 bool FrontendAction::BeginSourceFile(CompilerInstance &CI,
                                      const FrontendInputFile &Input) {
   assert(!Instance && "Already processing a source file!");
-  assert(!Input.File.empty() && "Unexpected empty filename!");
+  assert(!Input.isEmpty() && "Unexpected empty filename!");
   setCurrentInput(Input);
   setCompilerInstance(&CI);
 
+  StringRef InputFile = Input.getFile();
   bool HasBegunSourceFile = false;
   if (!BeginInvocation(CI))
     goto failure;
 
   // AST files follow a very different path, since they share objects via the
   // AST unit.
-  if (Input.Kind == IK_AST) {
+  if (Input.getKind() == IK_AST) {
     assert(!usesPreprocessorOnly() &&
            "Attempt to pass AST file to preprocessor only action!");
     assert(hasASTFileSupport() &&
@@ -176,7 +180,7 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI,
 
     IntrusiveRefCntPtr<DiagnosticsEngine> Diags(&CI.getDiagnostics());
     std::string Error;
-    ASTUnit *AST = ASTUnit::LoadFromASTFile(Input.File, Diags,
+    ASTUnit *AST = ASTUnit::LoadFromASTFile(InputFile, Diags,
                                             CI.getFileSystemOpts());
     if (!AST)
       goto failure;
@@ -191,11 +195,11 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI,
     CI.setASTContext(&AST->getASTContext());
 
     // Initialize the action.
-    if (!BeginSourceFileAction(CI, Input.File))
+    if (!BeginSourceFileAction(CI, InputFile))
       goto failure;
 
     /// Create the AST consumer.
-    CI.setASTConsumer(CreateWrappedASTConsumer(CI, Input.File));
+    CI.setASTConsumer(CreateWrappedASTConsumer(CI, InputFile));
     if (!CI.hasASTConsumer())
       goto failure;
 
@@ -209,7 +213,7 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI,
     CI.createSourceManager(CI.getFileManager());
 
   // IR files bypass the rest of initialization.
-  if (Input.Kind == IK_LLVM_IR) {
+  if (Input.getKind() == IK_LLVM_IR) {
     assert(hasIRSupport() &&
            "This action does not have IR file support!");
 
@@ -218,12 +222,51 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI,
     HasBegunSourceFile = true;
 
     // Initialize the action.
-    if (!BeginSourceFileAction(CI, Input.File))
+    if (!BeginSourceFileAction(CI, InputFile))
       goto failure;
 
     return true;
   }
 
+  // If the implicit PCH include is actually a directory, rather than
+  // a single file, search for a suitable PCH file in that directory.
+  if (!CI.getPreprocessorOpts().ImplicitPCHInclude.empty()) {
+    FileManager &FileMgr = CI.getFileManager();
+    PreprocessorOptions &PPOpts = CI.getPreprocessorOpts();
+    StringRef PCHInclude = PPOpts.ImplicitPCHInclude;
+    if (const DirectoryEntry *PCHDir = FileMgr.getDirectory(PCHInclude)) {
+      llvm::error_code EC;
+      SmallString<128> DirNative;
+      llvm::sys::path::native(PCHDir->getName(), DirNative);
+      bool Found = false;
+      for (llvm::sys::fs::directory_iterator Dir(DirNative.str(), EC), DirEnd;
+           Dir != DirEnd && !EC; Dir.increment(EC)) {
+        // Check whether this is an acceptable AST file.
+        if (ASTReader::isAcceptableASTFile(Dir->path(), FileMgr,
+                                           CI.getLangOpts(),
+                                           CI.getTargetOpts(),
+                                           CI.getPreprocessorOpts())) {
+          for (unsigned I = 0, N = PPOpts.Includes.size(); I != N; ++I) {
+            if (PPOpts.Includes[I] == PPOpts.ImplicitPCHInclude) {
+              PPOpts.Includes[I] = Dir->path();
+              PPOpts.ImplicitPCHInclude = Dir->path();
+              Found = true;
+              break;
+            }
+          }
+
+          assert(Found && "Implicit PCH include not in includes list?");
+          break;
+        }
+      }
+
+      if (!Found) {
+        CI.getDiagnostics().Report(diag::err_fe_no_pch_in_dir) << PCHInclude;
+        return true;
+      }
+    }
+  }
+
   // Set up the preprocessor.
   CI.createPreprocessor();
 
@@ -233,7 +276,7 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI,
   HasBegunSourceFile = true;
 
   // Initialize the action.
-  if (!BeginSourceFileAction(CI, Input.File))
+  if (!BeginSourceFileAction(CI, InputFile))
     goto failure;
 
   /// Create the AST context and consumer unless this is a preprocessor only
@@ -242,12 +285,14 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI,
     CI.createASTContext();
 
     OwningPtr<ASTConsumer> Consumer(
-                                   CreateWrappedASTConsumer(CI, Input.File));
+                                   CreateWrappedASTConsumer(CI, InputFile));
     if (!Consumer)
       goto failure;
 
     CI.getASTContext().setASTMutationListener(Consumer->GetASTMutationListener());
-
+    CI.getPreprocessor().setPPMutationListener(
+      Consumer->GetPPMutationListener());
+    
     if (!CI.getPreprocessorOpts().ChainedIncludes.empty()) {
       // Convert headers to PCH and chain them.
       OwningPtr<ExternalASTSource> source;
@@ -270,7 +315,6 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI,
       CI.createPCHExternalASTSource(
                                 CI.getPreprocessorOpts().ImplicitPCHInclude,
                                 CI.getPreprocessorOpts().DisablePCHValidation,
-                                CI.getPreprocessorOpts().DisableStatCache,
                             CI.getPreprocessorOpts().AllowPCHWithCompilerErrors,
                                 DeserialListener);
       if (!CI.getASTContext().getExternalSource())
@@ -314,6 +358,7 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI,
 
   if (HasBegunSourceFile)
     CI.getDiagnosticClient().EndSourceFile();
+  CI.clearOutputFiles(/*EraseFiles=*/true);
   setCurrentInput(FrontendInputFile());
   setCompilerInstance(0);
   return false;
@@ -325,10 +370,7 @@ bool FrontendAction::Execute() {
   // Initialize the main file entry. This needs to be delayed until after PCH
   // has loaded.
   if (!isCurrentFileAST()) {
-    if (!CI.InitializeSourceManager(getCurrentFile(),
-                                    getCurrentInput().IsSystem
-                                      ? SrcMgr::C_System
-                                      : SrcMgr::C_User))
+    if (!CI.InitializeSourceManager(getCurrentInput()))
       return false;
   }
 
diff --git a/lib/Frontend/FrontendActions.cpp b/lib/Frontend/FrontendActions.cpp
index 24960cf..47063f7 100644
--- a/lib/Frontend/FrontendActions.cpp
+++ b/lib/Frontend/FrontendActions.cpp
@@ -131,8 +131,31 @@ ASTConsumer *GenerateModuleAction::CreateASTConsumer(CompilerInstance &CI,
                           Sysroot, OS);
 }
 
+static SmallVectorImpl<char> &
+operator+=(SmallVectorImpl<char> &Includes, StringRef RHS) {
+  Includes.append(RHS.begin(), RHS.end());
+  return Includes;
+}
+
+static void addHeaderInclude(StringRef HeaderName,
+                             SmallVectorImpl<char> &Includes,
+                             const LangOptions &LangOpts) {
+  if (LangOpts.ObjC1)
+    Includes += "#import \"";
+  else
+    Includes += "#include \"";
+  Includes += HeaderName;
+  Includes += "\"\n";
+}
+
+static void addHeaderInclude(const FileEntry *Header,
+                             SmallVectorImpl<char> &Includes,
+                             const LangOptions &LangOpts) {
+  addHeaderInclude(Header->getName(), Includes, LangOpts);
+}
+
 /// \brief Collect the set of header includes needed to construct the given 
-/// module.
+/// module and update the TopHeaders file set of the module.
 ///
 /// \param Module The module we're collecting includes from.
 ///
@@ -142,30 +165,23 @@ static void collectModuleHeaderIncludes(const LangOptions &LangOpts,
                                         FileManager &FileMgr,
                                         ModuleMap &ModMap,
                                         clang::Module *Module,
-                                        SmallString<256> &Includes) {
+                                        SmallVectorImpl<char> &Includes) {
   // Don't collect any headers for unavailable modules.
   if (!Module->isAvailable())
     return;
 
   // Add includes for each of these headers.
   for (unsigned I = 0, N = Module->Headers.size(); I != N; ++I) {
-    if (LangOpts.ObjC1)
-      Includes += "#import \"";
-    else
-      Includes += "#include \"";
-    Includes += Module->Headers[I]->getName();
-    Includes += "\"\n";
+    const FileEntry *Header = Module->Headers[I];
+    Module->TopHeaders.insert(Header);
+    addHeaderInclude(Header, Includes, LangOpts);
   }
 
   if (const FileEntry *UmbrellaHeader = Module->getUmbrellaHeader()) {
+    Module->TopHeaders.insert(UmbrellaHeader);
     if (Module->Parent) {
       // Include the umbrella header for submodules.
-      if (LangOpts.ObjC1)
-        Includes += "#import \"";
-      else
-        Includes += "#include \"";
-      Includes += UmbrellaHeader->getName();
-      Includes += "\"\n";
+      addHeaderInclude(UmbrellaHeader, Includes, LangOpts);
     }
   } else if (const DirectoryEntry *UmbrellaDir = Module->getUmbrellaDir()) {
     // Add all of the headers we find in this subdirectory.
@@ -184,17 +200,14 @@ static void collectModuleHeaderIncludes(const LangOptions &LangOpts,
       
       // If this header is marked 'unavailable' in this module, don't include 
       // it.
-      if (const FileEntry *Header = FileMgr.getFile(Dir->path()))
+      if (const FileEntry *Header = FileMgr.getFile(Dir->path())) {
         if (ModMap.isHeaderInUnavailableModule(Header))
           continue;
+        Module->TopHeaders.insert(Header);
+      }
       
       // Include this header umbrella header for submodules.
-      if (LangOpts.ObjC1)
-        Includes += "#import \"";
-      else
-        Includes += "#include \"";
-      Includes += Dir->path();
-      Includes += "\"\n";
+      addHeaderInclude(Dir->path(), Includes, LangOpts);
     }
   }
   
@@ -250,77 +263,29 @@ bool GenerateModuleAction::BeginSourceFileAction(CompilerInstance &CI,
     return false;
   }
 
-  // Do we have an umbrella header for this module?
-  const FileEntry *UmbrellaHeader = Module->getUmbrellaHeader();
-  
+  FileManager &FileMgr = CI.getFileManager();
+
   // Collect the set of #includes we need to build the module.
   SmallString<256> HeaderContents;
-  collectModuleHeaderIncludes(CI.getLangOpts(), CI.getFileManager(),
+  if (const FileEntry *UmbrellaHeader = Module->getUmbrellaHeader())
+    addHeaderInclude(UmbrellaHeader, HeaderContents, CI.getLangOpts());
+  collectModuleHeaderIncludes(CI.getLangOpts(), FileMgr,
     CI.getPreprocessor().getHeaderSearchInfo().getModuleMap(),
     Module, HeaderContents);
-  if (UmbrellaHeader && HeaderContents.empty()) {
-    // Simple case: we have an umbrella header and there are no additional
-    // includes, we can just parse the umbrella header directly.
-    setCurrentInput(FrontendInputFile(UmbrellaHeader->getName(),
-                                      getCurrentFileKind(),
-                                      Module->IsSystem));
-    return true;
-  }
-  
-  FileManager &FileMgr = CI.getFileManager();
-  SmallString<128> HeaderName;
-  time_t ModTime;
-  if (UmbrellaHeader) {
-    // Read in the umbrella header.
-    // FIXME: Go through the source manager; the umbrella header may have
-    // been overridden.
-    std::string ErrorStr;
-    llvm::MemoryBuffer *UmbrellaContents
-      = FileMgr.getBufferForFile(UmbrellaHeader, &ErrorStr);
-    if (!UmbrellaContents) {
-      CI.getDiagnostics().Report(diag::err_missing_umbrella_header)
-        << UmbrellaHeader->getName() << ErrorStr;
-      return false;
-    }
-    
-    // Combine the contents of the umbrella header with the automatically-
-    // generated includes.
-    SmallString<256> OldContents = HeaderContents;
-    HeaderContents = UmbrellaContents->getBuffer();
-    HeaderContents += "\n\n";
-    HeaderContents += "/* Module includes */\n";
-    HeaderContents += OldContents;
-
-    // Pretend that we're parsing the umbrella header.
-    HeaderName = UmbrellaHeader->getName();
-    ModTime = UmbrellaHeader->getModificationTime();
-    
-    delete UmbrellaContents;
-  } else {
-    // Pick an innocuous-sounding name for the umbrella header.
-    HeaderName = Module->Name + ".h";
-    if (FileMgr.getFile(HeaderName, /*OpenFile=*/false, 
-                        /*CacheFailure=*/false)) {
-      // Try again!
-      HeaderName = Module->Name + "-module.h";      
-      if (FileMgr.getFile(HeaderName, /*OpenFile=*/false, 
-                          /*CacheFailure=*/false)) {
-        // Pick something ridiculous and go with it.
-        HeaderName = Module->Name + "-module.hmod";
-      }
-    }
-    ModTime = time(0);
-  }
-  
-  // Remap the contents of the header name we're using to our synthesized
-  // buffer.
-  const FileEntry *HeaderFile = FileMgr.getVirtualFile(HeaderName, 
+
+  StringRef InputName = Module::getModuleInputBufferName();
+
+  // We consistently construct a buffer as input to build the module.
+  // This means the main file for modules will always be a virtual one.
+  // FIXME: Maybe allow using a memory buffer as input directly instead of
+  // messing with virtual files.
+  const FileEntry *HeaderFile = FileMgr.getVirtualFile(InputName, 
                                                        HeaderContents.size(), 
-                                                       ModTime);
+                                                       time(0));
   llvm::MemoryBuffer *HeaderContentsBuf
     = llvm::MemoryBuffer::getMemBufferCopy(HeaderContents);
   CI.getSourceManager().overrideFileContents(HeaderFile, HeaderContentsBuf);  
-  setCurrentInput(FrontendInputFile(HeaderName, getCurrentFileKind(),
+  setCurrentInput(FrontendInputFile(InputName, getCurrentFileKind(),
                                     Module->IsSystem));
   return true;
 }
diff --git a/lib/Frontend/InitHeaderSearch.cpp b/lib/Frontend/InitHeaderSearch.cpp
index 8178f7a..4fddd11 100644
--- a/lib/Frontend/InitHeaderSearch.cpp
+++ b/lib/Frontend/InitHeaderSearch.cpp
@@ -14,7 +14,7 @@
 #include "clang/Frontend/Utils.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/LangOptions.h"
-#include "clang/Frontend/HeaderSearchOptions.h"
+#include "clang/Lex/HeaderSearchOptions.h"
 #include "clang/Lex/HeaderSearch.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallPtrSet.h"
@@ -320,7 +320,9 @@ void InitHeaderSearch::AddDefaultCIncludePaths(const llvm::Triple &triple,
       P.appendComponent("../../../include"); // <sysroot>/include
       AddPath(P.str(), System, true, false, false);
       AddPath("/mingw/include", System, true, false, false);
+#if defined(_WIN32)
       AddPath("c:/mingw/include", System, true, false, false); 
+#endif
     }
     break;
       
@@ -396,12 +398,14 @@ AddDefaultCPlusPlusIncludePaths(const llvm::Triple &triple, const HeaderSearchOp
     AddMinGW64CXXPaths(HSOpts.ResourceDir, "4.7.0");
     // mingw.org C++ include paths
     AddMinGWCPlusPlusIncludePaths("/mingw/lib/gcc", "mingw32", "4.5.2"); //MSYS
+#if defined(_WIN32)
     AddMinGWCPlusPlusIncludePaths("c:/MinGW/lib/gcc", "mingw32", "4.6.2");
     AddMinGWCPlusPlusIncludePaths("c:/MinGW/lib/gcc", "mingw32", "4.6.1");
     AddMinGWCPlusPlusIncludePaths("c:/MinGW/lib/gcc", "mingw32", "4.5.2");
     AddMinGWCPlusPlusIncludePaths("c:/MinGW/lib/gcc", "mingw32", "4.5.0");
     AddMinGWCPlusPlusIncludePaths("c:/MinGW/lib/gcc", "mingw32", "4.4.0");
     AddMinGWCPlusPlusIncludePaths("c:/MinGW/lib/gcc", "mingw32", "4.3.0");
+#endif
     break;
   case llvm::Triple::DragonFly:
     AddPath("/usr/include/c++/4.1", CXXSystem, true, false, false);
diff --git a/lib/Frontend/InitPreprocessor.cpp b/lib/Frontend/InitPreprocessor.cpp
index 1440da6..4bbd033 100644
--- a/lib/Frontend/InitPreprocessor.cpp
+++ b/lib/Frontend/InitPreprocessor.cpp
@@ -17,11 +17,12 @@
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Frontend/FrontendDiagnostic.h"
 #include "clang/Frontend/FrontendOptions.h"
-#include "clang/Frontend/PreprocessorOptions.h"
 #include "clang/Lex/HeaderSearch.h"
 #include "clang/Lex/Preprocessor.h"
+#include "clang/Lex/PreprocessorOptions.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/SourceManager.h"
+#include "clang/Serialization/ASTReader.h"
 #include "llvm/ADT/APFloat.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/MemoryBuffer.h"
@@ -83,6 +84,19 @@ static void AddImplicitIncludePTH(MacroBuilder &Builder, Preprocessor &PP,
   AddImplicitInclude(Builder, OriginalFile, PP.getFileManager());
 }
 
+/// \brief Add an implicit \#include using the original file used to generate
+/// a PCH file.
+static void AddImplicitIncludePCH(MacroBuilder &Builder, Preprocessor &PP,
+                                  StringRef ImplicitIncludePCH) {
+  std::string OriginalFile =
+    ASTReader::getOriginalSourceFile(ImplicitIncludePCH, PP.getFileManager(),
+                                     PP.getDiagnostics());
+  if (OriginalFile.empty())
+    return;
+
+  AddImplicitInclude(Builder, OriginalFile, PP.getFileManager());
+}
+
 /// PickFP - This is used to pick a value based on the FP semantics of the
 /// specified FP model.
 template <typename T>
@@ -102,51 +116,51 @@ static T PickFP(const llvm::fltSemantics *Sem, T IEEESingleVal,
 }
 
 static void DefineFloatMacros(MacroBuilder &Builder, StringRef Prefix,
-                              const llvm::fltSemantics *Sem) {
+                              const llvm::fltSemantics *Sem, StringRef Ext) {
   const char *DenormMin, *Epsilon, *Max, *Min;
-  DenormMin = PickFP(Sem, "1.40129846e-45F", "4.9406564584124654e-324",
-                     "3.64519953188247460253e-4951L",
-                     "4.94065645841246544176568792868221e-324L",
-                     "6.47517511943802511092443895822764655e-4966L");
+  DenormMin = PickFP(Sem, "1.40129846e-45", "4.9406564584124654e-324",
+                     "3.64519953188247460253e-4951",
+                     "4.94065645841246544176568792868221e-324",
+                     "6.47517511943802511092443895822764655e-4966");
   int Digits = PickFP(Sem, 6, 15, 18, 31, 33);
-  Epsilon = PickFP(Sem, "1.19209290e-7F", "2.2204460492503131e-16",
-                   "1.08420217248550443401e-19L",
-                   "4.94065645841246544176568792868221e-324L",
-                   "1.92592994438723585305597794258492732e-34L");
+  Epsilon = PickFP(Sem, "1.19209290e-7", "2.2204460492503131e-16",
+                   "1.08420217248550443401e-19",
+                   "4.94065645841246544176568792868221e-324",
+                   "1.92592994438723585305597794258492732e-34");
   int MantissaDigits = PickFP(Sem, 24, 53, 64, 106, 113);
   int Min10Exp = PickFP(Sem, -37, -307, -4931, -291, -4931);
   int Max10Exp = PickFP(Sem, 38, 308, 4932, 308, 4932);
   int MinExp = PickFP(Sem, -125, -1021, -16381, -968, -16381);
   int MaxExp = PickFP(Sem, 128, 1024, 16384, 1024, 16384);
-  Min = PickFP(Sem, "1.17549435e-38F", "2.2250738585072014e-308",
-               "3.36210314311209350626e-4932L",
-               "2.00416836000897277799610805135016e-292L",
-               "3.36210314311209350626267781732175260e-4932L");
-  Max = PickFP(Sem, "3.40282347e+38F", "1.7976931348623157e+308",
-               "1.18973149535723176502e+4932L",
-               "1.79769313486231580793728971405301e+308L",
-               "1.18973149535723176508575932662800702e+4932L");
+  Min = PickFP(Sem, "1.17549435e-38", "2.2250738585072014e-308",
+               "3.36210314311209350626e-4932",
+               "2.00416836000897277799610805135016e-292",
+               "3.36210314311209350626267781732175260e-4932");
+  Max = PickFP(Sem, "3.40282347e+38", "1.7976931348623157e+308",
+               "1.18973149535723176502e+4932",
+               "1.79769313486231580793728971405301e+308",
+               "1.18973149535723176508575932662800702e+4932");
 
   SmallString<32> DefPrefix;
   DefPrefix = "__";
   DefPrefix += Prefix;
   DefPrefix += "_";
 
-  Builder.defineMacro(DefPrefix + "DENORM_MIN__", DenormMin);
+  Builder.defineMacro(DefPrefix + "DENORM_MIN__", Twine(DenormMin)+Ext);
   Builder.defineMacro(DefPrefix + "HAS_DENORM__");
   Builder.defineMacro(DefPrefix + "DIG__", Twine(Digits));
-  Builder.defineMacro(DefPrefix + "EPSILON__", Twine(Epsilon));
+  Builder.defineMacro(DefPrefix + "EPSILON__", Twine(Epsilon)+Ext);
   Builder.defineMacro(DefPrefix + "HAS_INFINITY__");
   Builder.defineMacro(DefPrefix + "HAS_QUIET_NAN__");
   Builder.defineMacro(DefPrefix + "MANT_DIG__", Twine(MantissaDigits));
 
   Builder.defineMacro(DefPrefix + "MAX_10_EXP__", Twine(Max10Exp));
   Builder.defineMacro(DefPrefix + "MAX_EXP__", Twine(MaxExp));
-  Builder.defineMacro(DefPrefix + "MAX__", Twine(Max));
+  Builder.defineMacro(DefPrefix + "MAX__", Twine(Max)+Ext);
 
   Builder.defineMacro(DefPrefix + "MIN_10_EXP__","("+Twine(Min10Exp)+")");
   Builder.defineMacro(DefPrefix + "MIN_EXP__", "("+Twine(MinExp)+")");
-  Builder.defineMacro(DefPrefix + "MIN__", Twine(Min));
+  Builder.defineMacro(DefPrefix + "MIN__", Twine(Min)+Ext);
 }
 
 
@@ -247,7 +261,7 @@ static void AddObjCXXARCLibstdcxxDefines(const LangOptions &LangOpts,
         << "};\n"
         << "\n";
       
-    if (LangOpts.ObjCRuntimeHasWeak) {
+    if (LangOpts.ObjCARCWeak) {
       Out << "template<typename _Tp>\n"
           << "struct __is_scalar<__attribute__((objc_ownership(weak))) _Tp> {\n"
           << "  enum { __value = 0 };\n"
@@ -288,6 +302,8 @@ static void InitializeStandardPredefinedMacros(const TargetInfo &TI,
     else if (!LangOpts.GNUMode && LangOpts.Digraphs)
       Builder.defineMacro("__STDC_VERSION__", "199409L");
   } else {
+    // FIXME: LangOpts.CPlusPlus1y
+
     // C++11 [cpp.predefined]p1:
     //   The name __cplusplus is defined to the value 201103L when compiling a
     //   C++ translation unit.
@@ -325,8 +341,8 @@ static void InitializePredefinedMacros(const TargetInfo &TI,
   Builder.defineMacro("__clang_patchlevel__", "0");
 #endif
   Builder.defineMacro("__clang_version__", 
-                      "\"" CLANG_VERSION_STRING " ("
-                      + getClangFullRepositoryVersion() + ")\"");
+                      "\"" CLANG_VERSION_STRING " "
+                      + getClangFullRepositoryVersion() + "\"");
 #undef TOSTR
 #undef TOSTR2
   if (!LangOpts.MicrosoftMode) {
@@ -420,19 +436,17 @@ static void InitializePredefinedMacros(const TargetInfo &TI,
     // Both __PRETTY_FUNCTION__ and __FUNCTION__ are GCC extensions, however
     // VC++ appears to only like __FUNCTION__.
     Builder.defineMacro("__PRETTY_FUNCTION__", "__FUNCTION__");
-    // Work around some issues with Visual C++ headerws.
-    if (LangOpts.CPlusPlus) {
-      // Since we define wchar_t in C++ mode.
+    // Work around some issues with Visual C++ headers.
+    if (LangOpts.WChar) {
+      // wchar_t supported as a keyword.
       Builder.defineMacro("_WCHAR_T_DEFINED");
       Builder.defineMacro("_NATIVE_WCHAR_T_DEFINED");
+    }
+    if (LangOpts.CPlusPlus) {
       // FIXME: Support Microsoft's __identifier extension in the lexer.
       Builder.append("#define __identifier(x) x");
       Builder.append("class type_info;");
     }
-
-    if (LangOpts.CPlusPlus0x) {
-      Builder.defineMacro("_HAS_CHAR16_T_LANGUAGE_SUPPORT", "1");
-    }
   }
 
   if (LangOpts.Optimize)
@@ -511,9 +525,9 @@ static void InitializePredefinedMacros(const TargetInfo &TI,
   DefineType("__CHAR16_TYPE__", TI.getChar16Type(), Builder);
   DefineType("__CHAR32_TYPE__", TI.getChar32Type(), Builder);
 
-  DefineFloatMacros(Builder, "FLT", &TI.getFloatFormat());
-  DefineFloatMacros(Builder, "DBL", &TI.getDoubleFormat());
-  DefineFloatMacros(Builder, "LDBL", &TI.getLongDoubleFormat());
+  DefineFloatMacros(Builder, "FLT", &TI.getFloatFormat(), "F");
+  DefineFloatMacros(Builder, "DBL", &TI.getDoubleFormat(), "");
+  DefineFloatMacros(Builder, "LDBL", &TI.getLongDoubleFormat(), "L");
 
   // Define a __POINTER_WIDTH__ macro for stdint.h.
   Builder.defineMacro("__POINTER_WIDTH__",
@@ -763,6 +777,8 @@ void clang::InitializePreprocessor(Preprocessor &PP,
     const std::string &Path = InitOpts.Includes[i];
     if (Path == InitOpts.ImplicitPTHInclude)
       AddImplicitIncludePTH(Builder, PP, Path);
+    else if (Path == InitOpts.ImplicitPCHInclude)
+      AddImplicitIncludePCH(Builder, PP, Path);
     else
       AddImplicitInclude(Builder, Path, PP.getFileManager());
   }
diff --git a/lib/Frontend/LogDiagnosticPrinter.cpp b/lib/Frontend/LogDiagnosticPrinter.cpp
index 3fee957..3a04f18 100644
--- a/lib/Frontend/LogDiagnosticPrinter.cpp
+++ b/lib/Frontend/LogDiagnosticPrinter.cpp
@@ -8,6 +8,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Frontend/LogDiagnosticPrinter.h"
+#include "clang/Basic/DiagnosticOptions.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/SourceManager.h"
 #include "llvm/ADT/SmallString.h"
@@ -16,9 +17,9 @@
 using namespace clang;
 
 LogDiagnosticPrinter::LogDiagnosticPrinter(raw_ostream &os,
-                                           const DiagnosticOptions &diags,
+                                           DiagnosticOptions *diags,
                                            bool _OwnsOutputStream)
-  : OS(os), LangOpts(0), DiagOpts(&diags),
+  : OS(os), LangOpts(0), DiagOpts(diags),
     OwnsOutputStream(_OwnsOutputStream) {
 }
 
@@ -172,6 +173,6 @@ void LogDiagnosticPrinter::HandleDiagnostic(DiagnosticsEngine::Level Level,
 
 DiagnosticConsumer *
 LogDiagnosticPrinter::clone(DiagnosticsEngine &Diags) const {
-  return new LogDiagnosticPrinter(OS, *DiagOpts, /*OwnsOutputStream=*/false);
+  return new LogDiagnosticPrinter(OS, &*DiagOpts, /*OwnsOutputStream=*/false);
 }
 
diff --git a/lib/Frontend/PrintPreprocessedOutput.cpp b/lib/Frontend/PrintPreprocessedOutput.cpp
index 5311ed5..30707dc 100644
--- a/lib/Frontend/PrintPreprocessedOutput.cpp
+++ b/lib/Frontend/PrintPreprocessedOutput.cpp
@@ -570,8 +570,12 @@ static void DoPrintMacros(Preprocessor &PP, raw_ostream *OS) {
   do PP.Lex(Tok);
   while (Tok.isNot(tok::eof));
 
-  SmallVector<id_macro_pair, 128>
-    MacrosByID(PP.macro_begin(), PP.macro_end());
+  SmallVector<id_macro_pair, 128> MacrosByID;
+  for (Preprocessor::macro_iterator I = PP.macro_begin(), E = PP.macro_end();
+       I != E; ++I) {
+    if (I->first->hasMacroDefinition())
+      MacrosByID.push_back(id_macro_pair(I->first, I->second));
+  }
   llvm::array_pod_sort(MacrosByID.begin(), MacrosByID.end(), MacroIDCompare);
 
   for (unsigned i = 0, e = MacrosByID.size(); i != e; ++i) {
diff --git a/lib/Frontend/SerializedDiagnosticPrinter.cpp b/lib/Frontend/SerializedDiagnosticPrinter.cpp
index a20f30d..5f8fc1e 100644
--- a/lib/Frontend/SerializedDiagnosticPrinter.cpp
+++ b/lib/Frontend/SerializedDiagnosticPrinter.cpp
@@ -12,6 +12,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/DenseSet.h"
+#include "clang/Basic/DiagnosticOptions.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/Diagnostic.h"
@@ -50,13 +51,10 @@ class SDiagsWriter;
   
 class SDiagsRenderer : public DiagnosticNoteRenderer {
   SDiagsWriter &Writer;
-  RecordData &Record;
 public:
-  SDiagsRenderer(SDiagsWriter &Writer, RecordData &Record,
-                 const LangOptions &LangOpts,
-                 const DiagnosticOptions &DiagOpts)
-    : DiagnosticNoteRenderer(LangOpts, DiagOpts),
-      Writer(Writer), Record(Record){}
+  SDiagsRenderer(SDiagsWriter &Writer, const LangOptions &LangOpts,
+                 DiagnosticOptions *DiagOpts)
+    : DiagnosticNoteRenderer(LangOpts, DiagOpts), Writer(Writer) {}
 
   virtual ~SDiagsRenderer() {}
   
@@ -73,15 +71,16 @@ protected:
                                  DiagnosticsEngine::Level Level,
                                  ArrayRef<CharSourceRange> Ranges,
                                  const SourceManager &SM) {}
-  
-  void emitNote(SourceLocation Loc, StringRef Message, const SourceManager *SM);
-  
+
+  virtual void emitNote(SourceLocation Loc, StringRef Message,
+                        const SourceManager *SM);
+
   virtual void emitCodeContext(SourceLocation Loc,
                                DiagnosticsEngine::Level Level,
                                SmallVectorImpl<CharSourceRange>& Ranges,
                                ArrayRef<FixItHint> Hints,
                                const SourceManager &SM);
-  
+
   virtual void beginDiagnostic(DiagOrStoredDiag D,
                                DiagnosticsEngine::Level Level);
   virtual void endDiagnostic(DiagOrStoredDiag D,
@@ -91,13 +90,12 @@ protected:
 class SDiagsWriter : public DiagnosticConsumer {
   friend class SDiagsRenderer;
 public:  
-  explicit SDiagsWriter(llvm::raw_ostream *os, const DiagnosticOptions &diags) 
-    : LangOpts(0), DiagOpts(diags),
-      Stream(Buffer), OS(os), inNonNoteDiagnostic(false)
-  { 
+  explicit SDiagsWriter(llvm::raw_ostream *os, DiagnosticOptions *diags)
+    : LangOpts(0), DiagOpts(diags), Stream(Buffer), OS(os),
+      EmittedAnyDiagBlocks(false) {
     EmitPreamble();
   }
-  
+
   ~SDiagsWriter() {}
   
   void HandleDiagnostic(DiagnosticsEngine::Level DiagLevel,
@@ -124,7 +122,26 @@ private:
 
   /// \brief Emit the META data block.
   void EmitMetaBlock();
-  
+
+  /// \brief Start a DIAG block.
+  void EnterDiagBlock();
+
+  /// \brief End a DIAG block.
+  void ExitDiagBlock();
+
+  /// \brief Emit a DIAG record.
+  void EmitDiagnosticMessage(SourceLocation Loc,
+                             PresumedLoc PLoc,
+                             DiagnosticsEngine::Level Level,
+                             StringRef Message,
+                             const SourceManager *SM,
+                             DiagOrStoredDiag D);
+
+  /// \brief Emit FIXIT and SOURCE_RANGE records for a diagnostic.
+  void EmitCodeContext(SmallVectorImpl<CharSourceRange> &Ranges,
+                       ArrayRef<FixItHint> Hints,
+                       const SourceManager &SM);
+
   /// \brief Emit a record for a CharSourceRange.
   void EmitCharSourceRange(CharSourceRange R, const SourceManager &SM);
   
@@ -159,7 +176,7 @@ private:
   enum { Version = 1 };
 
   const LangOptions *LangOpts;
-  const DiagnosticOptions &DiagOpts;
+  llvm::IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts;
   
   /// \brief The byte buffer for the serialized content.
   SmallString<1024> Buffer;
@@ -190,17 +207,18 @@ private:
 
   /// \brief Map for uniquing strings.
   DiagFlagsTy DiagFlags;
-  
-  /// \brief Flag indicating whether or not we are in the process of
-  /// emitting a non-note diagnostic.
-  bool inNonNoteDiagnostic;
+
+  /// \brief Whether we have already started emission of any DIAG blocks. Once
+  /// this becomes \c true, we never close a DIAG block until we know that we're
+  /// starting another one or we're done.
+  bool EmittedAnyDiagBlocks;
 };
 } // end anonymous namespace
 
 namespace clang {
 namespace serialized_diags {
 DiagnosticConsumer *create(llvm::raw_ostream *OS,
-                           const DiagnosticOptions &diags) {
+                           DiagnosticOptions *diags) {
   return new SDiagsWriter(OS, diags);
 }
 } // end namespace serialized_diags
@@ -474,31 +492,69 @@ unsigned SDiagsWriter::getEmitDiagnosticFlag(DiagnosticsEngine::Level DiagLevel,
 
 void SDiagsWriter::HandleDiagnostic(DiagnosticsEngine::Level DiagLevel,
                                     const Diagnostic &Info) {
+  // Enter the block for a non-note diagnostic immediately, rather than waiting
+  // for beginDiagnostic, in case associated notes are emitted before we get
+  // there.
   if (DiagLevel != DiagnosticsEngine::Note) {
-    if (inNonNoteDiagnostic) {
-      // We have encountered a non-note diagnostic.  Finish up the previous
-      // diagnostic block before starting a new one.
-      Stream.ExitBlock();
-    }
-    inNonNoteDiagnostic = true;
+    if (EmittedAnyDiagBlocks)
+      ExitDiagBlock();
+
+    EnterDiagBlock();
+    EmittedAnyDiagBlocks = true;
   }
 
   // Compute the diagnostic text.
-  diagBuf.clear();   
+  diagBuf.clear();
   Info.FormatDiagnostic(diagBuf);
 
-  const SourceManager *
-    SM = Info.hasSourceManager() ? &Info.getSourceManager() : 0;
-  SDiagsRenderer Renderer(*this, Record, *LangOpts, DiagOpts);
+  if (Info.getLocation().isInvalid()) {
+    // Special-case diagnostics with no location. We may not have entered a
+    // source file in this case, so we can't use the normal DiagnosticsRenderer
+    // machinery.
+    EmitDiagnosticMessage(SourceLocation(), PresumedLoc(), DiagLevel,
+                          diagBuf, 0, &Info);
+    return;
+  }
+
+  assert(Info.hasSourceManager() && LangOpts &&
+         "Unexpected diagnostic with valid location outside of a source file");
+  SDiagsRenderer Renderer(*this, *LangOpts, &*DiagOpts);
   Renderer.emitDiagnostic(Info.getLocation(), DiagLevel,
                           diagBuf.str(),
                           Info.getRanges(),
                           llvm::makeArrayRef(Info.getFixItHints(),
                                              Info.getNumFixItHints()),
-                          SM,
+                          &Info.getSourceManager(),
                           &Info);
 }
 
+void SDiagsWriter::EmitDiagnosticMessage(SourceLocation Loc,
+                                         PresumedLoc PLoc,
+                                         DiagnosticsEngine::Level Level,
+                                         StringRef Message,
+                                         const SourceManager *SM,
+                                         DiagOrStoredDiag D) {
+  // Emit the RECORD_DIAG record.
+  Record.clear();
+  Record.push_back(RECORD_DIAG);
+  Record.push_back(Level);
+  AddLocToRecord(Loc, SM, PLoc, Record);
+
+  if (const Diagnostic *Info = D.dyn_cast<const Diagnostic*>()) {
+    // Emit the category string lazily and get the category ID.
+    unsigned DiagID = DiagnosticIDs::getCategoryNumberForDiag(Info->getID());
+    Record.push_back(getEmitCategory(DiagID));
+    // Emit the diagnostic flag string lazily and get the mapped ID.
+    Record.push_back(getEmitDiagnosticFlag(Level, Info->getID()));
+  } else {
+    Record.push_back(getEmitCategory());
+    Record.push_back(getEmitDiagnosticFlag(Level));
+  }
+
+  Record.push_back(Message.size());
+  Stream.EmitRecordWithBlob(Abbrevs.get(RECORD_DIAG), Record, Message);
+}
+
 void
 SDiagsRenderer::emitDiagnosticMessage(SourceLocation Loc,
                                       PresumedLoc PLoc,
@@ -507,94 +563,80 @@ SDiagsRenderer::emitDiagnosticMessage(SourceLocation Loc,
                                       ArrayRef<clang::CharSourceRange> Ranges,
                                       const SourceManager *SM,
                                       DiagOrStoredDiag D) {
-  // Emit the RECORD_DIAG record.
-  Writer.Record.clear();
-  Writer.Record.push_back(RECORD_DIAG);
-  Writer.Record.push_back(Level);
-  Writer.AddLocToRecord(Loc, SM, PLoc, Record);
+  Writer.EmitDiagnosticMessage(Loc, PLoc, Level, Message, SM, D);
+}
 
-  if (const Diagnostic *Info = D.dyn_cast<const Diagnostic*>()) {
-    // Emit the category string lazily and get the category ID.
-    unsigned DiagID = DiagnosticIDs::getCategoryNumberForDiag(Info->getID());
-    Writer.Record.push_back(Writer.getEmitCategory(DiagID));
-    // Emit the diagnostic flag string lazily and get the mapped ID.
-    Writer.Record.push_back(Writer.getEmitDiagnosticFlag(Level, Info->getID()));
-  }
-  else {
-    Writer.Record.push_back(Writer.getEmitCategory());
-    Writer.Record.push_back(Writer.getEmitDiagnosticFlag(Level));
-  }
+void SDiagsWriter::EnterDiagBlock() {
+  Stream.EnterSubblock(BLOCK_DIAG, 4);
+}
 
-  Writer.Record.push_back(Message.size());
-  Writer.Stream.EmitRecordWithBlob(Writer.Abbrevs.get(RECORD_DIAG),
-                                   Writer.Record, Message);
+void SDiagsWriter::ExitDiagBlock() {
+  Stream.ExitBlock();
 }
 
 void SDiagsRenderer::beginDiagnostic(DiagOrStoredDiag D,
                                      DiagnosticsEngine::Level Level) {
-  Writer.Stream.EnterSubblock(BLOCK_DIAG, 4);  
+  if (Level == DiagnosticsEngine::Note)
+    Writer.EnterDiagBlock();
 }
 
 void SDiagsRenderer::endDiagnostic(DiagOrStoredDiag D,
                                    DiagnosticsEngine::Level Level) {
-  if (D && Level != DiagnosticsEngine::Note)
-    return;
-  Writer.Stream.ExitBlock();
+  // Only end note diagnostics here, because we can't be sure when we've seen
+  // the last note associated with a non-note diagnostic.
+  if (Level == DiagnosticsEngine::Note)
+    Writer.ExitDiagBlock();
 }
 
-void SDiagsRenderer::emitCodeContext(SourceLocation Loc,
-                                     DiagnosticsEngine::Level Level,
-                                     SmallVectorImpl<CharSourceRange> &Ranges,
-                                     ArrayRef<FixItHint> Hints,
-                                     const SourceManager &SM) {  
+void SDiagsWriter::EmitCodeContext(SmallVectorImpl<CharSourceRange> &Ranges,
+                                   ArrayRef<FixItHint> Hints,
+                                   const SourceManager &SM) {
   // Emit Source Ranges.
-  for (ArrayRef<CharSourceRange>::iterator it=Ranges.begin(), ei=Ranges.end();
-       it != ei; ++it) {
-    if (it->isValid())
-      Writer.EmitCharSourceRange(*it, SM);
-  }
-  
+  for (ArrayRef<CharSourceRange>::iterator I = Ranges.begin(), E = Ranges.end();
+       I != E; ++I)
+    if (I->isValid())
+      EmitCharSourceRange(*I, SM);
+
   // Emit FixIts.
-  for (ArrayRef<FixItHint>::iterator it = Hints.begin(), et = Hints.end();
-       it != et; ++it) {
-    const FixItHint &fix = *it;
-    if (fix.isNull())
+  for (ArrayRef<FixItHint>::iterator I = Hints.begin(), E = Hints.end();
+       I != E; ++I) {
+    const FixItHint &Fix = *I;
+    if (Fix.isNull())
       continue;
-    Writer.Record.clear();
-    Writer.Record.push_back(RECORD_FIXIT);
-    Writer.AddCharSourceRangeToRecord(fix.RemoveRange, Record, SM);
-    Writer.Record.push_back(fix.CodeToInsert.size());
-    Writer.Stream.EmitRecordWithBlob(Writer.Abbrevs.get(RECORD_FIXIT), Record,
-                                     fix.CodeToInsert);    
+    Record.clear();
+    Record.push_back(RECORD_FIXIT);
+    AddCharSourceRangeToRecord(Fix.RemoveRange, Record, SM);
+    Record.push_back(Fix.CodeToInsert.size());
+    Stream.EmitRecordWithBlob(Abbrevs.get(RECORD_FIXIT), Record,
+                              Fix.CodeToInsert);
   }
 }
 
+void SDiagsRenderer::emitCodeContext(SourceLocation Loc,
+                                     DiagnosticsEngine::Level Level,
+                                     SmallVectorImpl<CharSourceRange> &Ranges,
+                                     ArrayRef<FixItHint> Hints,
+                                     const SourceManager &SM) {
+  Writer.EmitCodeContext(Ranges, Hints, SM);
+}
+
 void SDiagsRenderer::emitNote(SourceLocation Loc, StringRef Message,
                               const SourceManager *SM) {
-  Writer.Stream.EnterSubblock(BLOCK_DIAG, 4);
-  RecordData Record;
-  Record.push_back(RECORD_DIAG);
-  Record.push_back(DiagnosticsEngine::Note);
-  Writer.AddLocToRecord(Loc, Record, SM);
-  Record.push_back(Writer.getEmitCategory());
-  Record.push_back(Writer.getEmitDiagnosticFlag(DiagnosticsEngine::Note));
-  Record.push_back(Message.size());
-  Writer.Stream.EmitRecordWithBlob(Writer.Abbrevs.get(RECORD_DIAG),
-                                   Record, Message);
-  Writer.Stream.ExitBlock();
+  Writer.EnterDiagBlock();
+  PresumedLoc PLoc = SM ? SM->getPresumedLoc(Loc) : PresumedLoc();
+  Writer.EmitDiagnosticMessage(Loc, PLoc, DiagnosticsEngine::Note,
+                               Message, SM, DiagOrStoredDiag());
+  Writer.ExitDiagBlock();
 }
 
 void SDiagsWriter::finish() {
-  if (inNonNoteDiagnostic) {
-    // Finish off any diagnostics we were in the process of emitting.
-    Stream.ExitBlock();
-    inNonNoteDiagnostic = false;
-  }
+  // Finish off any diagnostic we were in the process of emitting.
+  if (EmittedAnyDiagBlocks)
+    ExitDiagBlock();
 
   // Write the generated bitstream to "Out".
   OS->write((char *)&Buffer.front(), Buffer.size());
   OS->flush();
-  
+
   OS.reset(0);
 }
-
diff --git a/lib/Frontend/TextDiagnostic.cpp b/lib/Frontend/TextDiagnostic.cpp
index 9bb3e1d..35dabad 100644
--- a/lib/Frontend/TextDiagnostic.cpp
+++ b/lib/Frontend/TextDiagnostic.cpp
@@ -11,7 +11,7 @@
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/ConvertUTF.h"
-#include "clang/Frontend/DiagnosticOptions.h"
+#include "clang/Basic/DiagnosticOptions.h"
 #include "clang/Lex/Lexer.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/raw_ostream.h"
@@ -43,19 +43,22 @@ static const enum raw_ostream::Colors savedColor =
 /// \brief Add highlights to differences in template strings.
 static void applyTemplateHighlighting(raw_ostream &OS, StringRef Str,
                                       bool &Normal, bool Bold) {
-  for (unsigned i = 0, e = Str.size(); i < e; ++i)
-    if (Str[i] != ToggleHighlight) {
-      OS << Str[i];
-    } else {
-      if (Normal)
-        OS.changeColor(templateColor, true);
-      else {
-        OS.resetColor();
-        if (Bold)
-          OS.changeColor(savedColor, true);
-      }
-      Normal = !Normal;
+  while (1) {
+    size_t Pos = Str.find(ToggleHighlight);
+    OS << Str.slice(0, Pos);
+    if (Pos == StringRef::npos)
+      break;
+
+    Str = Str.substr(Pos + 1);
+    if (Normal)
+      OS.changeColor(templateColor, true);
+    else {
+      OS.resetColor();
+      if (Bold)
+        OS.changeColor(savedColor, true);
     }
+    Normal = !Normal;
+  }
 }
 
 /// \brief Number of spaces to indent when word-wrapping.
@@ -110,28 +113,15 @@ printableTextForNextCharacter(StringRef SourceLine, size_t *i,
     return std::make_pair(expandedTab, true);
   }
 
-  // FIXME: this data is copied from the private implementation of ConvertUTF.h
-  static const char trailingBytesForUTF8[256] = {
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-    1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
-    2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5
-  };
-
   unsigned char const *begin, *end;
   begin = reinterpret_cast<unsigned char const *>(&*(SourceLine.begin() + *i));
-  end = begin + SourceLine.size();
+  end = begin + (SourceLine.size() - *i);
   
   if (isLegalUTF8Sequence(begin, end)) {
     UTF32 c;
     UTF32 *cptr = &c;
     unsigned char const *original_begin = begin;
-    char trailingBytes = trailingBytesForUTF8[(unsigned char)SourceLine[*i]];
-    unsigned char const *cp_end = begin+trailingBytes+1;
+    unsigned char const *cp_end = begin+getNumBytesForUTF8(SourceLine[*i]);
 
     ConversionResult res = ConvertUTF8toUTF32(&begin, cp_end, &cptr, cptr+1,
                                               strictConversion);
@@ -274,14 +264,44 @@ struct SourceColumnMap {
   }
   int columns() const { return m_byteToColumn.back(); }
   int bytes() const { return m_columnToByte.back(); }
+
+  /// \brief Map a byte to the column which it is at the start of, or return -1
+  /// if it is not at the start of a column (for a UTF-8 trailing byte).
   int byteToColumn(int n) const {
     assert(0<=n && n<static_cast<int>(m_byteToColumn.size()));
     return m_byteToColumn[n];
   }
+
+  /// \brief Map a byte to the first column which contains it.
+  int byteToContainingColumn(int N) const {
+    assert(0 <= N && N < static_cast<int>(m_byteToColumn.size()));
+    while (m_byteToColumn[N] == -1)
+      --N;
+    return m_byteToColumn[N];
+  }
+
+  /// \brief Map a column to the byte which starts the column, or return -1 if
+  /// the column the second or subsequent column of an expanded tab or similar
+  /// multi-column entity.
   int columnToByte(int n) const {
     assert(0<=n && n<static_cast<int>(m_columnToByte.size()));
     return m_columnToByte[n];
   }
+
+  /// \brief Map from a byte index to the next byte which starts a column.
+  int startOfNextColumn(int N) const {
+    assert(0 <= N && N < static_cast<int>(m_columnToByte.size() - 1));
+    while (byteToColumn(++N) == -1) {}
+    return N;
+  }
+
+  /// \brief Map from a byte index to the previous byte which starts a column.
+  int startOfPreviousColumn(int N) const {
+    assert(0 < N && N < static_cast<int>(m_columnToByte.size()));
+    while (byteToColumn(--N) == -1) {}
+    return N;
+  }
+
   StringRef getSourceLine() const {
     return m_SourceLine;
   }
@@ -398,25 +418,24 @@ static void selectInterestingSourceRegion(std::string &SourceLine,
     bool ExpandedRegion = false;
 
     if (SourceStart>0) {
-      unsigned NewStart = SourceStart-1;
+      unsigned NewStart = map.startOfPreviousColumn(SourceStart);
 
       // Skip over any whitespace we see here; we're looking for
       // another bit of interesting text.
+      // FIXME: Detect non-ASCII whitespace characters too.
       while (NewStart &&
-             (map.byteToColumn(NewStart)==-1 ||
-             isspace(static_cast<unsigned char>(SourceLine[NewStart]))))
-        --NewStart;
+             isspace(static_cast<unsigned char>(SourceLine[NewStart])))
+        NewStart = map.startOfPreviousColumn(NewStart);
 
       // Skip over this bit of "interesting" text.
-      while (NewStart &&
-             (map.byteToColumn(NewStart)!=-1 &&
-             !isspace(static_cast<unsigned char>(SourceLine[NewStart]))))
-        --NewStart;
-
-      // Move up to the non-whitespace character we just saw.
-      if (NewStart)
-        ++NewStart;
+      while (NewStart) {
+        unsigned Prev = map.startOfPreviousColumn(NewStart);
+        if (isspace(static_cast<unsigned char>(SourceLine[Prev])))
+          break;
+        NewStart = Prev;
+      }
 
+      assert(map.byteToColumn(NewStart) != -1);
       unsigned NewColumns = map.byteToColumn(SourceEnd) -
                               map.byteToColumn(NewStart);
       if (NewColumns <= TargetColumns) {
@@ -426,21 +445,21 @@ static void selectInterestingSourceRegion(std::string &SourceLine,
     }
 
     if (SourceEnd<SourceLine.size()) {
-      unsigned NewEnd = SourceEnd+1;
+      unsigned NewEnd = map.startOfNextColumn(SourceEnd);
 
       // Skip over any whitespace we see here; we're looking for
       // another bit of interesting text.
-      while (NewEnd<SourceLine.size() &&
-             (map.byteToColumn(NewEnd)==-1 ||
-             isspace(static_cast<unsigned char>(SourceLine[NewEnd]))))
-        ++NewEnd;
+      // FIXME: Detect non-ASCII whitespace characters too.
+      while (NewEnd < SourceLine.size() &&
+             isspace(static_cast<unsigned char>(SourceLine[NewEnd])))
+        NewEnd = map.startOfNextColumn(NewEnd);
 
       // Skip over this bit of "interesting" text.
-      while (NewEnd<SourceLine.size() &&
-             (map.byteToColumn(NewEnd)!=-1 &&
-             !isspace(static_cast<unsigned char>(SourceLine[NewEnd]))))
-        ++NewEnd;
+      while (NewEnd < SourceLine.size() &&
+             !isspace(static_cast<unsigned char>(SourceLine[NewEnd])))
+        NewEnd = map.startOfNextColumn(NewEnd);
 
+      assert(map.byteToColumn(NewEnd) != -1);
       unsigned NewColumns = map.byteToColumn(NewEnd) -
                               map.byteToColumn(SourceStart);
       if (NewColumns <= TargetColumns) {
@@ -475,7 +494,7 @@ static void selectInterestingSourceRegion(std::string &SourceLine,
 
   // The line needs some trunctiona, and we'd prefer to keep the front
   //  if possible, so remove the back
-  if (BackColumnsRemoved)
+  if (BackColumnsRemoved > strlen(back_ellipse))
     SourceLine.replace(SourceEnd, std::string::npos, back_ellipse);
 
   // If that's enough then we're done
@@ -483,7 +502,7 @@ static void selectInterestingSourceRegion(std::string &SourceLine,
     return;
 
   // Otherwise remove the front as well
-  if (FrontColumnsRemoved) {
+  if (FrontColumnsRemoved > strlen(front_ellipse)) {
     SourceLine.replace(0, SourceStart, front_ellipse);
     CaretLine.replace(0, CaretStart, front_space);
     if (!FixItInsertionLine.empty())
@@ -651,7 +670,7 @@ static bool printWordWrapped(raw_ostream &OS, StringRef Str,
 
 TextDiagnostic::TextDiagnostic(raw_ostream &OS,
                                const LangOptions &LangOpts,
-                               const DiagnosticOptions &DiagOpts)
+                               DiagnosticOptions *DiagOpts)
   : DiagnosticRenderer(LangOpts, DiagOpts), OS(OS) {}
 
 TextDiagnostic::~TextDiagnostic() {}
@@ -670,13 +689,13 @@ TextDiagnostic::emitDiagnosticMessage(SourceLocation Loc,
   if (Loc.isValid())
     emitDiagnosticLoc(Loc, PLoc, Level, Ranges, *SM);
   
-  if (DiagOpts.ShowColors)
+  if (DiagOpts->ShowColors)
     OS.resetColor();
   
-  printDiagnosticLevel(OS, Level, DiagOpts.ShowColors);
+  printDiagnosticLevel(OS, Level, DiagOpts->ShowColors);
   printDiagnosticMessage(OS, Level, Message,
                          OS.tell() - StartOfLocationInfo,
-                         DiagOpts.MessageLength, DiagOpts.ShowColors);
+                         DiagOpts->MessageLength, DiagOpts->ShowColors);
 }
 
 /*static*/ void
@@ -770,36 +789,36 @@ void TextDiagnostic::emitDiagnosticLoc(SourceLocation Loc, PresumedLoc PLoc,
   }
   unsigned LineNo = PLoc.getLine();
 
-  if (!DiagOpts.ShowLocation)
+  if (!DiagOpts->ShowLocation)
     return;
 
-  if (DiagOpts.ShowColors)
+  if (DiagOpts->ShowColors)
     OS.changeColor(savedColor, true);
 
   OS << PLoc.getFilename();
-  switch (DiagOpts.Format) {
+  switch (DiagOpts->getFormat()) {
   case DiagnosticOptions::Clang: OS << ':'  << LineNo; break;
   case DiagnosticOptions::Msvc:  OS << '('  << LineNo; break;
   case DiagnosticOptions::Vi:    OS << " +" << LineNo; break;
   }
 
-  if (DiagOpts.ShowColumn)
+  if (DiagOpts->ShowColumn)
     // Compute the column number.
     if (unsigned ColNo = PLoc.getColumn()) {
-      if (DiagOpts.Format == DiagnosticOptions::Msvc) {
+      if (DiagOpts->getFormat() == DiagnosticOptions::Msvc) {
         OS << ',';
         ColNo--;
       } else
         OS << ':';
       OS << ColNo;
     }
-  switch (DiagOpts.Format) {
+  switch (DiagOpts->getFormat()) {
   case DiagnosticOptions::Clang:
   case DiagnosticOptions::Vi:    OS << ':';    break;
   case DiagnosticOptions::Msvc:  OS << ") : "; break;
   }
 
-  if (DiagOpts.ShowSourceRanges && !Ranges.empty()) {
+  if (DiagOpts->ShowSourceRanges && !Ranges.empty()) {
     FileID CaretFileID =
       SM.getFileID(SM.getExpansionLoc(Loc));
     bool PrintedRange = false;
@@ -858,7 +877,7 @@ void TextDiagnostic::emitBasicNote(StringRef Message) {
 void TextDiagnostic::emitIncludeLocation(SourceLocation Loc,
                                          PresumedLoc PLoc,
                                          const SourceManager &SM) {
-  if (DiagOpts.ShowLocation)
+  if (DiagOpts->ShowLocation)
     OS << "In file included from " << PLoc.getFilename() << ':'
        << PLoc.getLine() << ":\n";
   else
@@ -886,7 +905,7 @@ void TextDiagnostic::emitSnippetAndCaret(
   // was part of a different warning or error diagnostic, or if the
   // diagnostic has ranges.  We don't want to emit the same caret
   // multiple times if one loc has multiple diagnostics.
-  if (!DiagOpts.ShowCarets)
+  if (!DiagOpts->ShowCarets)
     return;
   if (Loc == LastLoc && Ranges.empty() && Hints.empty() &&
       (LastLevel != DiagnosticsEngine::Note || Level == LastLevel))
@@ -924,7 +943,7 @@ void TextDiagnostic::emitSnippetAndCaret(
   // length as the line of source code.
   std::string CaretLine(LineEnd-LineStart, ' ');
 
-  const SourceColumnMap sourceColMap(SourceLine, DiagOpts.TabStop);
+  const SourceColumnMap sourceColMap(SourceLine, DiagOpts->TabStop);
 
   // Highlight all of the characters covered by Ranges with ~ characters.
   for (SmallVectorImpl<CharSourceRange>::iterator I = Ranges.begin(),
@@ -933,7 +952,7 @@ void TextDiagnostic::emitSnippetAndCaret(
     highlightRange(*I, LineNo, FID, sourceColMap, CaretLine, SM);
 
   // Next, insert the caret itself.
-  ColNo = sourceColMap.byteToColumn(ColNo-1);
+  ColNo = sourceColMap.byteToContainingColumn(ColNo-1);
   if (CaretLine.size()<ColNo+1)
     CaretLine.resize(ColNo+1, ' ');
   CaretLine[ColNo] = '^';
@@ -944,7 +963,7 @@ void TextDiagnostic::emitSnippetAndCaret(
 
   // If the source line is too long for our terminal, select only the
   // "interesting" source region within that line.
-  unsigned Columns = DiagOpts.MessageLength;
+  unsigned Columns = DiagOpts->MessageLength;
   if (Columns)
     selectInterestingSourceRegion(SourceLine, CaretLine, FixItInsertionLine,
                                   Columns, sourceColMap);
@@ -953,7 +972,7 @@ void TextDiagnostic::emitSnippetAndCaret(
   // to produce easily machine parsable output.  Add a space before the
   // source line and the caret to make it trivial to tell the main diagnostic
   // line from what the user is intended to see.
-  if (DiagOpts.ShowSourceRanges) {
+  if (DiagOpts->ShowSourceRanges) {
     SourceLine = ' ' + SourceLine;
     CaretLine = ' ' + CaretLine;
   }
@@ -965,20 +984,20 @@ void TextDiagnostic::emitSnippetAndCaret(
   // Emit what we have computed.
   emitSnippet(SourceLine);
 
-  if (DiagOpts.ShowColors)
+  if (DiagOpts->ShowColors)
     OS.changeColor(caretColor, true);
   OS << CaretLine << '\n';
-  if (DiagOpts.ShowColors)
+  if (DiagOpts->ShowColors)
     OS.resetColor();
 
   if (!FixItInsertionLine.empty()) {
-    if (DiagOpts.ShowColors)
+    if (DiagOpts->ShowColors)
       // Print fixit line in color
       OS.changeColor(fixitColor, false);
-    if (DiagOpts.ShowSourceRanges)
+    if (DiagOpts->ShowSourceRanges)
       OS << ' ';
     OS << FixItInsertionLine << '\n';
-    if (DiagOpts.ShowColors)
+    if (DiagOpts->ShowColors)
       OS.resetColor();
   }
 
@@ -997,15 +1016,15 @@ void TextDiagnostic::emitSnippet(StringRef line) {
   
   while (i<line.size()) {
     std::pair<SmallString<16>,bool> res
-        = printableTextForNextCharacter(line, &i, DiagOpts.TabStop);
+        = printableTextForNextCharacter(line, &i, DiagOpts->TabStop);
     bool was_printable = res.second;
     
-    if (DiagOpts.ShowColors && was_printable == print_reversed) {
+    if (DiagOpts->ShowColors && was_printable == print_reversed) {
       if (print_reversed)
         OS.reverseColor();
       OS << to_print;
       to_print.clear();
-      if (DiagOpts.ShowColors)
+      if (DiagOpts->ShowColors)
         OS.resetColor();
     }
     
@@ -1013,10 +1032,10 @@ void TextDiagnostic::emitSnippet(StringRef line) {
     to_print += res.first.str();
   }
   
-  if (print_reversed && DiagOpts.ShowColors)
+  if (print_reversed && DiagOpts->ShowColors)
     OS.reverseColor();
   OS << to_print;
-  if (print_reversed && DiagOpts.ShowColors)
+  if (print_reversed && DiagOpts->ShowColors)
     OS.resetColor();
   
   OS << '\n';
@@ -1030,16 +1049,8 @@ void TextDiagnostic::highlightRange(const CharSourceRange &R,
                                     const SourceManager &SM) {
   if (!R.isValid()) return;
 
-  SourceLocation Begin = SM.getExpansionLoc(R.getBegin());
-  SourceLocation End = SM.getExpansionLoc(R.getEnd());
-
-  // If the End location and the start location are the same and are a macro
-  // location, then the range was something that came from a macro expansion
-  // or _Pragma.  If this is an object-like macro, the best we can do is to
-  // highlight the range.  If this is a function-like macro, we'd also like to
-  // highlight the arguments.
-  if (Begin == End && R.getEnd().isMacroID())
-    End = SM.getExpansionRange(R.getEnd()).second;
+  SourceLocation Begin = R.getBegin();
+  SourceLocation End = R.getEnd();
 
   unsigned StartLineNo = SM.getExpansionLineNumber(Begin);
   if (StartLineNo > LineNo || SM.getFileID(Begin) != FID)
@@ -1080,7 +1091,7 @@ void TextDiagnostic::highlightRange(const CharSourceRange &R,
     while (StartColNo < map.getSourceLine().size() &&
            (map.getSourceLine()[StartColNo] == ' ' ||
             map.getSourceLine()[StartColNo] == '\t'))
-      ++StartColNo;
+      StartColNo = map.startOfNextColumn(StartColNo);
 
     // Pick the last non-whitespace column.
     if (EndColNo > map.getSourceLine().size())
@@ -1088,7 +1099,7 @@ void TextDiagnostic::highlightRange(const CharSourceRange &R,
     while (EndColNo-1 &&
            (map.getSourceLine()[EndColNo-1] == ' ' ||
             map.getSourceLine()[EndColNo-1] == '\t'))
-      --EndColNo;
+      EndColNo = map.startOfPreviousColumn(EndColNo);
 
     // If the start/end passed each other, then we are trying to highlight a
     // range that just exists in whitespace, which must be some sort of other
@@ -1100,8 +1111,8 @@ void TextDiagnostic::highlightRange(const CharSourceRange &R,
   assert(EndColNo <= map.getSourceLine().size() && "Invalid range!");
 
   // Fill the range with ~'s.
-  StartColNo = map.byteToColumn(StartColNo);
-  EndColNo = map.byteToColumn(EndColNo);
+  StartColNo = map.byteToContainingColumn(StartColNo);
+  EndColNo = map.byteToContainingColumn(EndColNo);
 
   assert(StartColNo <= EndColNo && "Invalid range!");
   if (CaretLine.size() < EndColNo)
@@ -1116,7 +1127,7 @@ std::string TextDiagnostic::buildFixItInsertionLine(
   const SourceManager &SM) {
 
   std::string FixItInsertionLine;
-  if (Hints.empty() || !DiagOpts.ShowFixits)
+  if (Hints.empty() || !DiagOpts->ShowFixits)
     return FixItInsertionLine;
   unsigned PrevHintEndCol = 0;
 
@@ -1139,7 +1150,7 @@ std::string TextDiagnostic::buildFixItInsertionLine(
 
         // The hint must start inside the source or right at the end
         assert(HintByteOffset < static_cast<unsigned>(map.bytes())+1);
-        unsigned HintCol = map.byteToColumn(HintByteOffset);
+        unsigned HintCol = map.byteToContainingColumn(HintByteOffset);
 
         // If we inserted a long previous hint, push this one forwards, and add
         // an extra space to show that this is not part of the previous
@@ -1176,14 +1187,14 @@ std::string TextDiagnostic::buildFixItInsertionLine(
     }
   }
 
-  expandTabs(FixItInsertionLine, DiagOpts.TabStop);
+  expandTabs(FixItInsertionLine, DiagOpts->TabStop);
 
   return FixItInsertionLine;
 }
 
 void TextDiagnostic::emitParseableFixits(ArrayRef<FixItHint> Hints,
                                          const SourceManager &SM) {
-  if (!DiagOpts.ShowParseableFixits)
+  if (!DiagOpts->ShowParseableFixits)
     return;
 
   // We follow FixItRewriter's example in not (yet) handling
diff --git a/lib/Frontend/TextDiagnosticPrinter.cpp b/lib/Frontend/TextDiagnosticPrinter.cpp
index 382e156..aa7a61a 100644
--- a/lib/Frontend/TextDiagnosticPrinter.cpp
+++ b/lib/Frontend/TextDiagnosticPrinter.cpp
@@ -12,9 +12,9 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Frontend/TextDiagnosticPrinter.h"
+#include "clang/Basic/DiagnosticOptions.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/SourceManager.h"
-#include "clang/Frontend/DiagnosticOptions.h"
 #include "clang/Frontend/TextDiagnostic.h"
 #include "clang/Lex/Lexer.h"
 #include "llvm/Support/MemoryBuffer.h"
@@ -25,9 +25,9 @@
 using namespace clang;
 
 TextDiagnosticPrinter::TextDiagnosticPrinter(raw_ostream &os,
-                                             const DiagnosticOptions &diags,
+                                             DiagnosticOptions *diags,
                                              bool _OwnsOutputStream)
-  : OS(os), DiagOpts(&diags),
+  : OS(os), DiagOpts(diags),
     OwnsOutputStream(_OwnsOutputStream) {
 }
 
@@ -39,7 +39,7 @@ TextDiagnosticPrinter::~TextDiagnosticPrinter() {
 void TextDiagnosticPrinter::BeginSourceFile(const LangOptions &LO,
                                             const Preprocessor *PP) {
   // Build the TextDiagnostic utility.
-  TextDiag.reset(new TextDiagnostic(OS, LO, *DiagOpts));
+  TextDiag.reset(new TextDiagnostic(OS, LO, &*DiagOpts));
 }
 
 void TextDiagnosticPrinter::EndSourceFile() {
@@ -158,5 +158,5 @@ void TextDiagnosticPrinter::HandleDiagnostic(DiagnosticsEngine::Level Level,
 
 DiagnosticConsumer *
 TextDiagnosticPrinter::clone(DiagnosticsEngine &Diags) const {
-  return new TextDiagnosticPrinter(OS, *DiagOpts, /*OwnsOutputStream=*/false);
+  return new TextDiagnosticPrinter(OS, &*DiagOpts, /*OwnsOutputStream=*/false);
 }
diff --git a/lib/Frontend/VerifyDiagnosticConsumer.cpp b/lib/Frontend/VerifyDiagnosticConsumer.cpp
index a9378a1..1750946 100644
--- a/lib/Frontend/VerifyDiagnosticConsumer.cpp
+++ b/lib/Frontend/VerifyDiagnosticConsumer.cpp
@@ -31,14 +31,17 @@ VerifyDiagnosticConsumer::VerifyDiagnosticConsumer(DiagnosticsEngine &_Diags)
   : Diags(_Diags),
     PrimaryClient(Diags.getClient()), OwnsPrimaryClient(Diags.ownsClient()),
     Buffer(new TextDiagnosticBuffer()), CurrentPreprocessor(0),
-    ActiveSourceFiles(0)
+    LangOpts(0), SrcManager(0), ActiveSourceFiles(0), Status(HasNoDirectives)
 {
   Diags.takeClient();
+  if (Diags.hasSourceManager())
+    setSourceManager(Diags.getSourceManager());
 }
 
 VerifyDiagnosticConsumer::~VerifyDiagnosticConsumer() {
   assert(!ActiveSourceFiles && "Incomplete parsing of source files!");
   assert(!CurrentPreprocessor && "CurrentPreprocessor should be invalid!");
+  SrcManager = 0;
   CheckDiagnostics();  
   Diags.takeClient();
   if (OwnsPrimaryClient)
@@ -48,21 +51,20 @@ VerifyDiagnosticConsumer::~VerifyDiagnosticConsumer() {
 #ifndef NDEBUG
 namespace {
 class VerifyFileTracker : public PPCallbacks {
-  typedef VerifyDiagnosticConsumer::FilesParsedForDirectivesSet ListType;
-  ListType &FilesList;
+  VerifyDiagnosticConsumer &Verify;
   SourceManager &SM;
 
 public:
-  VerifyFileTracker(ListType &FilesList, SourceManager &SM)
-    : FilesList(FilesList), SM(SM) { }
+  VerifyFileTracker(VerifyDiagnosticConsumer &Verify, SourceManager &SM)
+    : Verify(Verify), SM(SM) { }
 
   /// \brief Hook into the preprocessor and update the list of parsed
   /// files when the preprocessor indicates a new file is entered.
   virtual void FileChanged(SourceLocation Loc, FileChangeReason Reason,
                            SrcMgr::CharacteristicKind FileType,
                            FileID PrevFID) {
-    if (const FileEntry *E = SM.getFileEntryForID(SM.getFileID(Loc)))
-      FilesList.insert(E);
+    Verify.UpdateParsedFileStatus(SM, SM.getFileID(Loc),
+                                  VerifyDiagnosticConsumer::IsParsed);
   }
 };
 } // End anonymous namespace.
@@ -76,10 +78,12 @@ void VerifyDiagnosticConsumer::BeginSourceFile(const LangOptions &LangOpts,
   if (++ActiveSourceFiles == 1) {
     if (PP) {
       CurrentPreprocessor = PP;
+      this->LangOpts = &LangOpts;
+      setSourceManager(PP->getSourceManager());
       const_cast<Preprocessor*>(PP)->addCommentHandler(this);
 #ifndef NDEBUG
-      VerifyFileTracker *V = new VerifyFileTracker(FilesParsedForDirectives,
-                                                   PP->getSourceManager());
+      // Debug build tracks parsed files.
+      VerifyFileTracker *V = new VerifyFileTracker(*this, *SrcManager);
       const_cast<Preprocessor*>(PP)->addPPCallbacks(V);
 #endif
     }
@@ -101,18 +105,40 @@ void VerifyDiagnosticConsumer::EndSourceFile() {
     // Check diagnostics once last file completed.
     CheckDiagnostics();
     CurrentPreprocessor = 0;
+    LangOpts = 0;
   }
 }
 
 void VerifyDiagnosticConsumer::HandleDiagnostic(
       DiagnosticsEngine::Level DiagLevel, const Diagnostic &Info) {
+  if (Info.hasSourceManager())
+    setSourceManager(Info.getSourceManager());
+
 #ifndef NDEBUG
-  if (Info.hasSourceManager()) {
-    FileID FID = Info.getSourceManager().getFileID(Info.getLocation());
-    if (!FID.isInvalid())
-      FilesWithDiagnostics.insert(FID);
+  // Debug build tracks unparsed files for possible
+  // unparsed expected-* directives.
+  if (SrcManager) {
+    SourceLocation Loc = Info.getLocation();
+    if (Loc.isValid()) {
+      ParsedStatus PS = IsUnparsed;
+
+      Loc = SrcManager->getExpansionLoc(Loc);
+      FileID FID = SrcManager->getFileID(Loc);
+
+      const FileEntry *FE = SrcManager->getFileEntryForID(FID);
+      if (FE && CurrentPreprocessor && SrcManager->isLoadedFileID(FID)) {
+        // If the file is a modules header file it shall not be parsed
+        // for expected-* directives.
+        HeaderSearch &HS = CurrentPreprocessor->getHeaderSearchInfo();
+        if (HS.findModuleForHeader(FE))
+          PS = IsUnparsedNoDirectives;
+      }
+
+      UpdateParsedFileStatus(*SrcManager, FID, PS);
+    }
   }
 #endif
+
   // Send the diagnostic to the buffer, we will check it once we reach the end
   // of the source file (or are destructed).
   Buffer->HandleDiagnostic(DiagLevel, Info);
@@ -200,10 +226,22 @@ public:
 
   // Return true if string literal is found.
   // When true, P marks begin-position of S in content.
-  bool Search(StringRef S) {
-    P = std::search(C, End, S.begin(), S.end());
-    PEnd = P + S.size();
-    return P != End;
+  bool Search(StringRef S, bool EnsureStartOfWord = false) {
+    do {
+      P = std::search(C, End, S.begin(), S.end());
+      PEnd = P + S.size();
+      if (P == End)
+        break;
+      if (!EnsureStartOfWord
+            // Check if string literal starts a new word.
+            || P == Begin || isspace(P[-1])
+            // Or it could be preceeded by the start of a comment.
+            || (P > (Begin + 1) && (P[-1] == '/' || P[-1] == '*')
+                                &&  P[-2] == '/'))
+        return true;
+      // Otherwise, skip and search again.
+    } while (Advance());
+    return false;
   }
 
   // Advance 1-past previous next/search.
@@ -240,12 +278,13 @@ private:
 ///
 /// Returns true if any valid directives were found.
 static bool ParseDirective(StringRef S, ExpectedData *ED, SourceManager &SM,
-                           SourceLocation Pos, DiagnosticsEngine &Diags) {
+                           SourceLocation Pos, DiagnosticsEngine &Diags,
+                           VerifyDiagnosticConsumer::DirectiveStatus &Status) {
   // A single comment may contain multiple directives.
   bool FoundDirective = false;
   for (ParseHelper PH(S); !PH.Done();) {
     // Search for token: expected
-    if (!PH.Search("expected"))
+    if (!PH.Search("expected", true))
       break;
     PH.Advance();
 
@@ -262,10 +301,24 @@ static bool ParseDirective(StringRef S, ExpectedData *ED, SourceManager &SM,
       DL = ED ? &ED->Warnings : NULL;
     else if (PH.Next("note"))
       DL = ED ? &ED->Notes : NULL;
-    else
+    else if (PH.Next("no-diagnostics")) {
+      if (Status == VerifyDiagnosticConsumer::HasOtherExpectedDirectives)
+        Diags.Report(Pos, diag::err_verify_invalid_no_diags)
+          << /*IsExpectedNoDiagnostics=*/true;
+      else
+        Status = VerifyDiagnosticConsumer::HasExpectedNoDiagnostics;
+      continue;
+    } else
       continue;
     PH.Advance();
 
+    if (Status == VerifyDiagnosticConsumer::HasExpectedNoDiagnostics) {
+      Diags.Report(Pos, diag::err_verify_invalid_no_diags)
+        << /*IsExpectedNoDiagnostics=*/false;
+      continue;
+    }
+    Status = VerifyDiagnosticConsumer::HasOtherExpectedDirectives;
+
     // If a directive has been found but we're not interested
     // in storing the directive information, return now.
     if (!DL)
@@ -412,7 +465,7 @@ bool VerifyDiagnosticConsumer::HandleComment(Preprocessor &PP,
   // Fold any "\<EOL>" sequences
   size_t loc = C.find('\\');
   if (loc == StringRef::npos) {
-    ParseDirective(C, &ED, SM, CommentBegin, PP.getDiagnostics());
+    ParseDirective(C, &ED, SM, CommentBegin, PP.getDiagnostics(), Status);
     return false;
   }
 
@@ -442,7 +495,7 @@ bool VerifyDiagnosticConsumer::HandleComment(Preprocessor &PP,
   }
 
   if (!C2.empty())
-    ParseDirective(C2, &ED, SM, CommentBegin, PP.getDiagnostics());
+    ParseDirective(C2, &ED, SM, CommentBegin, PP.getDiagnostics(), Status);
   return false;
 }
 
@@ -452,34 +505,36 @@ bool VerifyDiagnosticConsumer::HandleComment(Preprocessor &PP,
 /// Preprocessor, directives inside skipped #if blocks will still be found.
 ///
 /// \return true if any directives were found.
-static bool findDirectives(const Preprocessor &PP, FileID FID) {
+static bool findDirectives(SourceManager &SM, FileID FID,
+                           const LangOptions &LangOpts) {
   // Create a raw lexer to pull all the comments out of FID.
   if (FID.isInvalid())
     return false;
 
-  SourceManager& SM = PP.getSourceManager();
   // Create a lexer to lex all the tokens of the main file in raw mode.
   const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID);
-  Lexer RawLex(FID, FromFile, SM, PP.getLangOpts());
+  Lexer RawLex(FID, FromFile, SM, LangOpts);
 
   // Return comments as tokens, this is how we find expected diagnostics.
   RawLex.SetCommentRetentionState(true);
 
   Token Tok;
   Tok.setKind(tok::comment);
-  bool Found = false;
+  VerifyDiagnosticConsumer::DirectiveStatus Status =
+    VerifyDiagnosticConsumer::HasNoDirectives;
   while (Tok.isNot(tok::eof)) {
     RawLex.Lex(Tok);
     if (!Tok.is(tok::comment)) continue;
 
-    std::string Comment = PP.getSpelling(Tok);
+    std::string Comment = RawLex.getSpelling(Tok, SM, LangOpts);
     if (Comment.empty()) continue;
 
-    // Find all expected errors/warnings/notes.
-    Found |= ParseDirective(Comment, 0, SM, Tok.getLocation(),
-                            PP.getDiagnostics());
+    // Find first directive.
+    if (ParseDirective(Comment, 0, SM, Tok.getLocation(),
+                       SM.getDiagnostics(), Status))
+      return true;
   }
-  return Found;
+  return false;
 }
 #endif // !NDEBUG
 
@@ -601,41 +656,95 @@ static unsigned CheckResults(DiagnosticsEngine &Diags, SourceManager &SourceMgr,
   return NumProblems;
 }
 
+void VerifyDiagnosticConsumer::UpdateParsedFileStatus(SourceManager &SM,
+                                                      FileID FID,
+                                                      ParsedStatus PS) {
+  // Check SourceManager hasn't changed.
+  setSourceManager(SM);
+
+#ifndef NDEBUG
+  if (FID.isInvalid())
+    return;
+
+  const FileEntry *FE = SM.getFileEntryForID(FID);
+
+  if (PS == IsParsed) {
+    // Move the FileID from the unparsed set to the parsed set.
+    UnparsedFiles.erase(FID);
+    ParsedFiles.insert(std::make_pair(FID, FE));
+  } else if (!ParsedFiles.count(FID) && !UnparsedFiles.count(FID)) {
+    // Add the FileID to the unparsed set if we haven't seen it before.
+
+    // Check for directives.
+    bool FoundDirectives;
+    if (PS == IsUnparsedNoDirectives)
+      FoundDirectives = false;
+    else
+      FoundDirectives = !LangOpts || findDirectives(SM, FID, *LangOpts);
+
+    // Add the FileID to the unparsed set.
+    UnparsedFiles.insert(std::make_pair(FID,
+                                      UnparsedFileStatus(FE, FoundDirectives)));
+  }
+#endif
+}
+
 void VerifyDiagnosticConsumer::CheckDiagnostics() {
   // Ensure any diagnostics go to the primary client.
   bool OwnsCurClient = Diags.ownsClient();
   DiagnosticConsumer *CurClient = Diags.takeClient();
   Diags.setClient(PrimaryClient, false);
 
-  // If we have a preprocessor, scan the source for expected diagnostic
-  // markers. If not then any diagnostics are unexpected.
-  if (CurrentPreprocessor) {
-    SourceManager &SM = CurrentPreprocessor->getSourceManager();
-
 #ifndef NDEBUG
-    // In a debug build, scan through any files that may have been missed
-    // during parsing and issue a fatal error if directives are contained
-    // within these files.  If a fatal error occurs, this suggests that
-    // this file is being parsed separately from the main file.
-    HeaderSearch &HS = CurrentPreprocessor->getHeaderSearchInfo();
-    for (FilesWithDiagnosticsSet::iterator I = FilesWithDiagnostics.begin(),
-                                         End = FilesWithDiagnostics.end();
-            I != End; ++I) {
-      const FileEntry *E = SM.getFileEntryForID(*I);
-      // Don't check files already parsed or those handled as modules.
-      if (E && (FilesParsedForDirectives.count(E)
-                  || HS.findModuleForHeader(E)))
+  // In a debug build, scan through any files that may have been missed
+  // during parsing and issue a fatal error if directives are contained
+  // within these files.  If a fatal error occurs, this suggests that
+  // this file is being parsed separately from the main file, in which
+  // case consider moving the directives to the correct place, if this
+  // is applicable.
+  if (UnparsedFiles.size() > 0) {
+    // Generate a cache of parsed FileEntry pointers for alias lookups.
+    llvm::SmallPtrSet<const FileEntry *, 8> ParsedFileCache;
+    for (ParsedFilesMap::iterator I = ParsedFiles.begin(),
+                                End = ParsedFiles.end(); I != End; ++I) {
+      if (const FileEntry *FE = I->second)
+        ParsedFileCache.insert(FE);
+    }
+
+    // Iterate through list of unparsed files.
+    for (UnparsedFilesMap::iterator I = UnparsedFiles.begin(),
+                                  End = UnparsedFiles.end(); I != End; ++I) {
+      const UnparsedFileStatus &Status = I->second;
+      const FileEntry *FE = Status.getFile();
+
+      // Skip files that have been parsed via an alias.
+      if (FE && ParsedFileCache.count(FE))
         continue;
 
-      if (findDirectives(*CurrentPreprocessor, *I))
+      // Report a fatal error if this file contained directives.
+      if (Status.foundDirectives()) {
         llvm::report_fatal_error(Twine("-verify directives found after rather"
                                        " than during normal parsing of ",
-                                 StringRef(E ? E->getName() : "(unknown)")));
+                                 StringRef(FE ? FE->getName() : "(unknown)")));
+      }
+    }
+
+    // UnparsedFiles has been processed now, so clear it.
+    UnparsedFiles.clear();
+  }
+#endif // !NDEBUG
+
+  if (SrcManager) {
+    // Produce an error if no expected-* directives could be found in the
+    // source file(s) processed.
+    if (Status == HasNoDirectives) {
+      Diags.Report(diag::err_verify_no_directives).setForceEmit();
+      ++NumErrors;
+      Status = HasNoDirectivesReported;
     }
-#endif
 
     // Check that the expected diagnostics occurred.
-    NumErrors += CheckResults(Diags, SM, *Buffer, ED);
+    NumErrors += CheckResults(Diags, *SrcManager, *Buffer, ED);
   } else {
     NumErrors += (PrintUnexpected(Diags, 0, Buffer->err_begin(),
                                   Buffer->err_end(), "error") +
diff --git a/lib/Frontend/Warnings.cpp b/lib/Frontend/Warnings.cpp
index b7d4a3b..f789b7f 100644
--- a/lib/Frontend/Warnings.cpp
+++ b/lib/Frontend/Warnings.cpp
@@ -24,7 +24,7 @@
 #include "clang/Basic/Diagnostic.h"
 #include "clang/Sema/SemaDiagnostic.h"
 #include "clang/Lex/LexDiagnostic.h"
-#include "clang/Frontend/DiagnosticOptions.h"
+#include "clang/Basic/DiagnosticOptions.h"
 #include "clang/Frontend/FrontendDiagnostic.h"
 #include <cstring>
 #include <utility>
@@ -51,8 +51,7 @@ void clang::ProcessWarningOptions(DiagnosticsEngine &Diags,
                                   const DiagnosticOptions &Opts) {
   Diags.setSuppressSystemWarnings(true);  // Default to -Wno-system-headers
   Diags.setIgnoreAllWarnings(Opts.IgnoreWarnings);
-  Diags.setShowOverloads(
-    static_cast<DiagnosticsEngine::OverloadsShown>(Opts.ShowOverloads));
+  Diags.setShowOverloads(Opts.getShowOverloads());
 
   Diags.setElideType(Opts.ElideType);
   Diags.setPrintTemplateTree(Opts.ShowTemplateTree);
diff --git a/lib/FrontendTool/CMakeLists.txt b/lib/FrontendTool/CMakeLists.txt
index fe9d589..176511b 100644
--- a/lib/FrontendTool/CMakeLists.txt
+++ b/lib/FrontendTool/CMakeLists.txt
@@ -11,7 +11,8 @@ add_dependencies(clangFrontendTool
 target_link_libraries(clangFrontendTool
   clangDriver
   clangFrontend
-  clangRewrite
+  clangRewriteCore
+  clangRewriteFrontend
   clangCodeGen
   clangStaticAnalyzerFrontend
   clangStaticAnalyzerCheckers
diff --git a/lib/FrontendTool/ExecuteCompilerInvocation.cpp b/lib/FrontendTool/ExecuteCompilerInvocation.cpp
index bd50083..c7c55b0 100644
--- a/lib/FrontendTool/ExecuteCompilerInvocation.cpp
+++ b/lib/FrontendTool/ExecuteCompilerInvocation.cpp
@@ -16,6 +16,7 @@
 #include "clang/StaticAnalyzer/Frontend/FrontendActions.h"
 #include "clang/ARCMigrate/ARCMTActions.h"
 #include "clang/CodeGen/CodeGenAction.h"
+#include "clang/Driver/Option.h"
 #include "clang/Driver/Options.h"
 #include "clang/Driver/OptTable.h"
 #include "clang/Frontend/CompilerInvocation.h"
@@ -23,7 +24,7 @@
 #include "clang/Frontend/FrontendActions.h"
 #include "clang/Frontend/FrontendDiagnostic.h"
 #include "clang/Frontend/FrontendPluginRegistry.h"
-#include "clang/Rewrite/FrontendActions.h"
+#include "clang/Rewrite/Frontend/FrontendActions.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/DynamicLibrary.h"
 using namespace clang;
@@ -137,7 +138,9 @@ bool clang::ExecuteCompilerInvocation(CompilerInstance *Clang) {
   if (Clang->getFrontendOpts().ShowHelp) {
     OwningPtr<driver::OptTable> Opts(driver::createDriverOptTable());
     Opts->PrintHelp(llvm::outs(), "clang -cc1",
-                    "LLVM 'Clang' Compiler: http://clang.llvm.org");
+                    "LLVM 'Clang' Compiler: http://clang.llvm.org",
+                    /*Include=*/driver::options::CC1Option,
+                    /*Exclude=*/0);
     return 0;
   }
 
@@ -175,7 +178,7 @@ bool clang::ExecuteCompilerInvocation(CompilerInstance *Clang) {
 
   // Honor -analyzer-checker-help.
   // This should happen AFTER plugins have been loaded!
-  if (Clang->getAnalyzerOpts().ShowCheckerHelp) {
+  if (Clang->getAnalyzerOpts()->ShowCheckerHelp) {
     ento::printCheckerHelp(llvm::outs(), Clang->getFrontendOpts().Plugins);
     return 0;
   }
diff --git a/lib/Headers/CMakeLists.txt b/lib/Headers/CMakeLists.txt
index 6e9cc68..25e4d90 100644
--- a/lib/Headers/CMakeLists.txt
+++ b/lib/Headers/CMakeLists.txt
@@ -6,6 +6,7 @@ set(files
   bmiintrin.h
   bmi2intrin.h
   emmintrin.h
+  f16cintrin.h
   float.h
   fma4intrin.h
   fmaintrin.h
@@ -19,6 +20,7 @@ set(files
   nmmintrin.h
   pmmintrin.h
   popcntintrin.h
+  rtmintrin.h
   smmintrin.h
   stdalign.h
   stdarg.h
@@ -29,6 +31,8 @@ set(files
   tmmintrin.h
   varargs.h
   wmmintrin.h
+  __wmmintrin_aes.h
+  __wmmintrin_pclmul.h
   x86intrin.h
   xmmintrin.h
   xopintrin.h
diff --git a/lib/Headers/__wmmintrin_aes.h b/lib/Headers/__wmmintrin_aes.h
new file mode 100644
index 0000000..2bfa027
--- /dev/null
+++ b/lib/Headers/__wmmintrin_aes.h
@@ -0,0 +1,67 @@
+/*===---- __wmmintrin_aes.h - AES intrinsics -------------------------------===
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ *===-----------------------------------------------------------------------===
+ */
+#ifndef _WMMINTRIN_AES_H
+#define _WMMINTRIN_AES_H
+
+#include <emmintrin.h>
+
+#if !defined (__AES__)
+#  error "AES instructions not enabled"
+#else
+
+static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
+_mm_aesenc_si128(__m128i __V, __m128i __R)
+{
+  return (__m128i)__builtin_ia32_aesenc128(__V, __R);
+}
+
+static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
+_mm_aesenclast_si128(__m128i __V, __m128i __R)
+{
+  return (__m128i)__builtin_ia32_aesenclast128(__V, __R);
+}
+
+static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
+_mm_aesdec_si128(__m128i __V, __m128i __R)
+{
+  return (__m128i)__builtin_ia32_aesdec128(__V, __R);
+}
+
+static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
+_mm_aesdeclast_si128(__m128i __V, __m128i __R)
+{
+  return (__m128i)__builtin_ia32_aesdeclast128(__V, __R);
+}
+
+static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
+_mm_aesimc_si128(__m128i __V)
+{
+  return (__m128i)__builtin_ia32_aesimc128(__V);
+}
+
+#define _mm_aeskeygenassist_si128(C, R) \
+  __builtin_ia32_aeskeygenassist128((C), (R))
+
+#endif
+
+#endif  /* _WMMINTRIN_AES_H */
diff --git a/lib/Headers/__wmmintrin_pclmul.h b/lib/Headers/__wmmintrin_pclmul.h
new file mode 100644
index 0000000..8d1f1b7
--- /dev/null
+++ b/lib/Headers/__wmmintrin_pclmul.h
@@ -0,0 +1,34 @@
+/*===---- __wmmintrin_pclmul.h - AES intrinsics ----------------------------===
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ *===-----------------------------------------------------------------------===
+ */
+#ifndef _WMMINTRIN_PCLMUL_H
+#define _WMMINTRIN_PCLMUL_H
+
+#if !defined (__PCLMUL__)
+# error "PCLMUL instruction is not enabled"
+#else
+#define _mm_clmulepi64_si128(__X, __Y, __I) \
+  ((__m128i)__builtin_ia32_pclmulqdq128((__v2di)(__m128i)(__X), \
+                                        (__v2di)(__m128i)(__Y), (char)(__I)))
+#endif
+
+#endif /* _WMMINTRIN_PCLMUL_H */
diff --git a/lib/Headers/altivec.h b/lib/Headers/altivec.h
index a225378..2bf53fb 100644
--- a/lib/Headers/altivec.h
+++ b/lib/Headers/altivec.h
@@ -4363,14 +4363,14 @@ vec_perm(vector float a, vector float b, vector unsigned char c)
 
 /* vec_vperm */
 
-vector signed char __ATTRS_o_ai
+static vector signed char __ATTRS_o_ai
 vec_vperm(vector signed char a, vector signed char b, vector unsigned char c)
 {
   return (vector signed char)
            __builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
 }
 
-vector unsigned char __ATTRS_o_ai
+static vector unsigned char __ATTRS_o_ai
 vec_vperm(vector unsigned char a,
           vector unsigned char b,
           vector unsigned char c)
@@ -4379,21 +4379,21 @@ vec_vperm(vector unsigned char a,
            __builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
 }
 
-vector bool char __ATTRS_o_ai
+static vector bool char __ATTRS_o_ai
 vec_vperm(vector bool char a, vector bool char b, vector unsigned char c)
 {
   return (vector bool char)
            __builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
 }
 
-vector short __ATTRS_o_ai
+static vector short __ATTRS_o_ai
 vec_vperm(vector short a, vector short b, vector unsigned char c)
 {
   return (vector short)
            __builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
 }
 
-vector unsigned short __ATTRS_o_ai
+static vector unsigned short __ATTRS_o_ai
 vec_vperm(vector unsigned short a,
           vector unsigned short b,
           vector unsigned char c)
@@ -4402,41 +4402,41 @@ vec_vperm(vector unsigned short a,
            __builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
 }
 
-vector bool short __ATTRS_o_ai
+static vector bool short __ATTRS_o_ai
 vec_vperm(vector bool short a, vector bool short b, vector unsigned char c)
 {
   return (vector bool short)
            __builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
 }
 
-vector pixel __ATTRS_o_ai
+static vector pixel __ATTRS_o_ai
 vec_vperm(vector pixel a, vector pixel b, vector unsigned char c)
 {
   return (vector pixel)
            __builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
 }
 
-vector int __ATTRS_o_ai
+static vector int __ATTRS_o_ai
 vec_vperm(vector int a, vector int b, vector unsigned char c)
 {
   return (vector int)__builtin_altivec_vperm_4si(a, b, c);
 }
 
-vector unsigned int __ATTRS_o_ai
+static vector unsigned int __ATTRS_o_ai
 vec_vperm(vector unsigned int a, vector unsigned int b, vector unsigned char c)
 {
   return (vector unsigned int)
            __builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
 }
 
-vector bool int __ATTRS_o_ai
+static vector bool int __ATTRS_o_ai
 vec_vperm(vector bool int a, vector bool int b, vector unsigned char c)
 {
   return (vector bool int)
            __builtin_altivec_vperm_4si((vector int)a, (vector int)b, c);
 }
 
-vector float __ATTRS_o_ai
+static vector float __ATTRS_o_ai
 vec_vperm(vector float a, vector float b, vector unsigned char c)
 {
   return (vector float)
@@ -4445,7 +4445,7 @@ vec_vperm(vector float a, vector float b, vector unsigned char c)
 
 /* vec_re */
 
-vector float __attribute__((__always_inline__))
+static vector float __attribute__((__always_inline__))
 vec_re(vector float a)
 {
   return __builtin_altivec_vrefp(a);
@@ -4453,7 +4453,7 @@ vec_re(vector float a)
 
 /* vec_vrefp */
 
-vector float __attribute__((__always_inline__))
+static vector float __attribute__((__always_inline__))
 vec_vrefp(vector float a)
 {
   return __builtin_altivec_vrefp(a);
diff --git a/lib/Headers/bmi2intrin.h b/lib/Headers/bmi2intrin.h
index c60b0c4..a05cfad 100644
--- a/lib/Headers/bmi2intrin.h
+++ b/lib/Headers/bmi2intrin.h
@@ -70,6 +70,25 @@ _pext_u64(unsigned long long __X, unsigned long long __Y)
   return __builtin_ia32_pext_di(__X, __Y);
 }
 
+static __inline__ unsigned long long __attribute__((__always_inline__, __nodebug__))
+_mulx_u64 (unsigned long long __X, unsigned long long __Y,
+	   unsigned long long *__P)
+{
+  unsigned __int128 __res = (unsigned __int128) __X * __Y;
+  *__P = (unsigned long long) (__res >> 64);
+  return (unsigned long long) __res;
+}
+
+#else /* !__x86_64__ */
+
+static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__))
+_mulx_u32 (unsigned int __X, unsigned int __Y, unsigned int *__P)
+{
+  unsigned long long __res = (unsigned long long) __X * __Y;
+  *__P = (unsigned int) (__res >> 32);
+  return (unsigned int) __res;
+}
+
 #endif /* !__x86_64__  */
 
 #endif /* __BMI2INTRIN_H */
diff --git a/lib/Headers/cpuid.h b/lib/Headers/cpuid.h
index 05c293f..33df7c2 100644
--- a/lib/Headers/cpuid.h
+++ b/lib/Headers/cpuid.h
@@ -28,6 +28,6 @@
 static inline int __get_cpuid (unsigned int level, unsigned int *eax,
                                unsigned int *ebx, unsigned int *ecx,
                                unsigned int *edx) {
-    asm("cpuid" : "=a"(*eax), "=b" (*ebx), "=c"(*ecx), "=d"(*edx) : "0"(level));
+    __asm("cpuid" : "=a"(*eax), "=b" (*ebx), "=c"(*ecx), "=d"(*edx) : "0"(level));
     return 1;
 }
diff --git a/lib/Headers/f16cintrin.h b/lib/Headers/f16cintrin.h
new file mode 100644
index 0000000..2c96952
--- /dev/null
+++ b/lib/Headers/f16cintrin.h
@@ -0,0 +1,58 @@
+/*===---- f16cintrin.h - F16C intrinsics ---------------------------------===
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ *===-----------------------------------------------------------------------===
+ */
+
+#if !defined __X86INTRIN_H && !defined __IMMINTRIN_H
+#error "Never use <f16cintrin.h> directly; include <x86intrin.h> instead."
+#endif
+
+#ifndef __F16C__
+# error "F16C instruction is not enabled"
+#endif /* __F16C__ */
+
+#ifndef __F16CINTRIN_H
+#define __F16CINTRIN_H
+
+typedef float __v8sf __attribute__ ((__vector_size__ (32)));
+typedef float __m256 __attribute__ ((__vector_size__ (32)));
+
+#define _mm_cvtps_ph(a, imm) __extension__ ({ \
+  __m128 __a = (a); \
+ (__m128i)__builtin_ia32_vcvtps2ph((__v4sf)__a, (imm)); })
+
+#define _mm256_cvtps_ph(a, imm) __extension__ ({ \
+  __m256 __a = (a); \
+ (__m128i)__builtin_ia32_vcvtps2ph256((__v8sf)__a, (imm)); })
+
+static __inline __m128 __attribute__((__always_inline__, __nodebug__))
+_mm_cvtph_ps(__m128i a)
+{
+  return (__m128)__builtin_ia32_vcvtph2ps((__v8hi)a);
+}
+
+static __inline __m256 __attribute__((__always_inline__, __nodebug__))
+_mm256_cvtph_ps(__m128i a)
+{
+  return (__m256)__builtin_ia32_vcvtph2ps256((__v8hi)a);
+}
+
+#endif /* __F16CINTRIN_H */
diff --git a/lib/Headers/immintrin.h b/lib/Headers/immintrin.h
index 15b65f3..cd733bf 100644
--- a/lib/Headers/immintrin.h
+++ b/lib/Headers/immintrin.h
@@ -98,4 +98,8 @@ _rdrand64_step(unsigned long long *__p)
 #endif
 #endif /* __RDRND__ */
 
+#ifdef __RTM__
+#include <rtmintrin.h>
+#endif
+
 #endif /* __IMMINTRIN_H */
diff --git a/lib/Headers/module.map b/lib/Headers/module.map
index 418ba50..b24bccc 100644
--- a/lib/Headers/module.map
+++ b/lib/Headers/module.map
@@ -25,6 +25,11 @@ module _Builtin_intrinsics [system] {
       header "mmintrin.h"
     }
 
+    explicit module f16c {
+      requires f16c
+      header "f16cintrin.h"
+    }
+
     explicit module sse {
       requires sse
       export mmx
@@ -62,6 +67,12 @@ module _Builtin_intrinsics [system] {
       header "nmmintrin.h"
     }
 
+    explicit module sse4a {
+      requires sse4a
+      export sse3
+      header "ammintrin.h"
+    }
+
     explicit module avx {
       requires avx
       export sse4_2
@@ -84,6 +95,11 @@ module _Builtin_intrinsics [system] {
       header "bmi2intrin.h"
     }
 
+    explicit module fma {
+      requires fma
+      header "fmaintrin.h"
+    }
+
     explicit module fma4 {
       requires fma4
       export sse3
@@ -104,5 +120,26 @@ module _Builtin_intrinsics [system] {
       requires mm3dnow
       header "mm3dnow.h"
     }
+
+    explicit module xop {
+      requires xop
+      export fma4
+      header "xopintrin.h"
+    }
+
+    explicit module aes_pclmul {
+      requires aes, pclmul
+      header "wmmintrin.h"
+    }
+
+    explicit module aes {
+      requires aes
+      header "__wmmintrin_aes.h"
+    }
+
+    explicit module pclmul {
+      requires pclmul
+      header "__wmmintrin_pclmul.h"
+    }
   }
 }
diff --git a/lib/Headers/rtmintrin.h b/lib/Headers/rtmintrin.h
new file mode 100644
index 0000000..bdc2b99
--- /dev/null
+++ b/lib/Headers/rtmintrin.h
@@ -0,0 +1,49 @@
+/*===---- rtmintrin.h - RTM intrinsics -------------------------------------===
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ *===-----------------------------------------------------------------------===
+ */
+
+#ifndef __IMMINTRIN_H
+#error "Never use <rtmintrin.h> directly; include <immintrin.h> instead."
+#endif
+
+#define _XBEGIN_STARTED   (~0u)
+#define _XABORT_EXPLICIT  (1 << 0)
+#define _XABORT_RETRY     (1 << 1)
+#define _XABORT_CONFLICT  (1 << 2)
+#define _XABORT_CAPACITY  (1 << 3)
+#define _XABORT_DEBUG     (1 << 4)
+#define _XABORT_NESTED    (1 << 5)
+#define _XABORT_CODE(x)   (((x) >> 24) & 0xFF)
+
+static __inline__ unsigned int __attribute__((__always_inline__, __nodebug__))
+_xbegin(void)
+{
+  return __builtin_ia32_xbegin();
+}
+
+static __inline__ void __attribute__((__always_inline__, __nodebug__))
+_xend(void)
+{
+  __builtin_ia32_xend();
+}
+
+#define _xabort(imm) __builtin_ia32_xabort((imm))
diff --git a/lib/Headers/unwind.h b/lib/Headers/unwind.h
index a065920..6520b83 100644
--- a/lib/Headers/unwind.h
+++ b/lib/Headers/unwind.h
@@ -100,7 +100,7 @@ typedef enum {
   _UVRSR_FAILED = 2  
 } _Unwind_VRS_Result; 
 
-_Unwind_VRS_Result _Unwind_VRS_Get(_Unwind_Context *context,
+_Unwind_VRS_Result _Unwind_VRS_Get(struct _Unwind_Context *context,
   _Unwind_VRS_RegClass regclass,
   uint32_t regno,
   _Unwind_VRS_DataRepresentation representation,
diff --git a/lib/Headers/wmmintrin.h b/lib/Headers/wmmintrin.h
index dca896f..369e3c2 100644
--- a/lib/Headers/wmmintrin.h
+++ b/lib/Headers/wmmintrin.h
@@ -31,48 +31,11 @@
 #else
 
 #ifdef __AES__
-
-static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
-_mm_aesenc_si128(__m128i __V, __m128i __R)
-{
-  return (__m128i)__builtin_ia32_aesenc128(__V, __R);
-}
-
-static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
-_mm_aesenclast_si128(__m128i __V, __m128i __R)
-{
-  return (__m128i)__builtin_ia32_aesenclast128(__V, __R);
-}
-
-static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
-_mm_aesdec_si128(__m128i __V, __m128i __R)
-{
-  return (__m128i)__builtin_ia32_aesdec128(__V, __R);
-}
-
-static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
-_mm_aesdeclast_si128(__m128i __V, __m128i __R)
-{
-  return (__m128i)__builtin_ia32_aesdeclast128(__V, __R);
-}
-
-static __inline__ __m128i __attribute__((__always_inline__, __nodebug__))
-_mm_aesimc_si128(__m128i __V)
-{
-  return (__m128i)__builtin_ia32_aesimc128(__V);
-}
-
-#define _mm_aeskeygenassist_si128(C, R) \
-  __builtin_ia32_aeskeygenassist128((C), (R))
-
+#include <__wmmintrin_aes.h>
 #endif /* __AES__ */
 
 #ifdef __PCLMUL__
-
-#define _mm_clmulepi64_si128(__X, __Y, __I) \
-  ((__m128i)__builtin_ia32_pclmulqdq128((__v2di)(__m128i)(__X), \
-                                        (__v2di)(__m128i)(__Y), (char)(__I)))
-
+#include <__wmmintrin_pclmul.h>
 #endif /* __PCLMUL__ */
 
 #endif /* __AES__ || __PCLMUL__ */
diff --git a/lib/Headers/x86intrin.h b/lib/Headers/x86intrin.h
index 556cd01..68ce106 100644
--- a/lib/Headers/x86intrin.h
+++ b/lib/Headers/x86intrin.h
@@ -58,6 +58,10 @@
 #include <xopintrin.h>
 #endif
 
+#ifdef __F16C__
+#include <f16cintrin.h>
+#endif
+
 // FIXME: LWP
 
 #endif /* __X86INTRIN_H */
diff --git a/lib/Headers/xmmintrin.h b/lib/Headers/xmmintrin.h
index e616157..e2480ec 100644
--- a/lib/Headers/xmmintrin.h
+++ b/lib/Headers/xmmintrin.h
@@ -95,7 +95,8 @@ _mm_div_ps(__m128 a, __m128 b)
 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
 _mm_sqrt_ss(__m128 a)
 {
-  return __builtin_ia32_sqrtss(a);
+  __m128 c = __builtin_ia32_sqrtss(a);
+  return (__m128) { c[0], a[1], a[2], a[3] };
 }
 
 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
@@ -107,7 +108,8 @@ _mm_sqrt_ps(__m128 a)
 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
 _mm_rcp_ss(__m128 a)
 {
-  return __builtin_ia32_rcpss(a);
+  __m128 c = __builtin_ia32_rcpss(a);
+  return (__m128) { c[0], a[1], a[2], a[3] };
 }
 
 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
@@ -119,7 +121,8 @@ _mm_rcp_ps(__m128 a)
 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
 _mm_rsqrt_ss(__m128 a)
 {
-  return __builtin_ia32_rsqrtss(a);
+  __m128 c = __builtin_ia32_rsqrtss(a);
+  return (__m128) { c[0], a[1], a[2], a[3] };
 }
 
 static __inline__ __m128 __attribute__((__always_inline__, __nodebug__))
diff --git a/lib/Lex/HeaderMap.cpp b/lib/Lex/HeaderMap.cpp
index bbfc1df..7dc0491 100644
--- a/lib/Lex/HeaderMap.cpp
+++ b/lib/Lex/HeaderMap.cpp
@@ -144,7 +144,7 @@ HMapBucket HeaderMap::getBucket(unsigned BucketNo) const {
                                         sizeof(HMapHeader));
 
   const HMapBucket *BucketPtr = BucketArray+BucketNo;
-  if ((char*)(BucketPtr+1) > FileBuffer->getBufferEnd()) {
+  if ((const char*)(BucketPtr+1) > FileBuffer->getBufferEnd()) {
     Result.Prefix = 0;
     Result.Suffix = 0;
     return Result;  // Invalid buffer, corrupt hmap.
diff --git a/lib/Lex/HeaderSearch.cpp b/lib/Lex/HeaderSearch.cpp
index bb3a673..67000b68 100644
--- a/lib/Lex/HeaderSearch.cpp
+++ b/lib/Lex/HeaderSearch.cpp
@@ -12,6 +12,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Lex/HeaderSearch.h"
+#include "clang/Lex/HeaderSearchOptions.h"
 #include "clang/Lex/HeaderMap.h"
 #include "clang/Lex/Lexer.h"
 #include "clang/Basic/Diagnostic.h"
@@ -38,10 +39,11 @@ HeaderFileInfo::getControllingMacro(ExternalIdentifierLookup *External) {
 
 ExternalHeaderFileInfoSource::~ExternalHeaderFileInfoSource() {}
 
-HeaderSearch::HeaderSearch(FileManager &FM, DiagnosticsEngine &Diags,
+HeaderSearch::HeaderSearch(llvm::IntrusiveRefCntPtr<HeaderSearchOptions> HSOpts,
+                           FileManager &FM, DiagnosticsEngine &Diags,
                            const LangOptions &LangOpts, 
                            const TargetInfo *Target)
-  : FileMgr(FM), Diags(Diags), FrameworkMap(64), 
+  : HSOpts(HSOpts), FileMgr(FM), FrameworkMap(64),
     ModMap(FileMgr, *Diags.getClient(), LangOpts, Target)
 {
   AngledDirIdx = 0;
@@ -905,7 +907,20 @@ Module *HeaderSearch::loadFrameworkModule(StringRef Name,
   SubmodulePath.push_back(Name);
   
   // Walk the directory structure to find any enclosing frameworks.
+#ifdef LLVM_ON_UNIX
+  // Note: as an egregious but useful hack we use the real path here, because
+  // frameworks moving from top-level frameworks to embedded frameworks tend
+  // to be symlinked from the top-level location to the embedded location,
+  // and we need to resolve lookups as if we had found the embedded location.
+  char RealDirName[PATH_MAX];
+  StringRef DirName;
+  if (realpath(Dir->getName(), RealDirName))
+    DirName = RealDirName;
+  else
+    DirName = Dir->getName();
+#else
   StringRef DirName = Dir->getName();
+#endif
   do {
     // Get the parent directory name.
     DirName = llvm::sys::path::parent_path(DirName);
@@ -924,7 +939,33 @@ Module *HeaderSearch::loadFrameworkModule(StringRef Name,
       TopFrameworkDir = Dir;
     }
   } while (true);
-  
+
+  // Determine whether we're allowed to infer a module map.
+  bool canInfer = false;
+  if (llvm::sys::path::has_parent_path(TopFrameworkDir->getName())) {
+    // Figure out the parent path.
+    StringRef Parent = llvm::sys::path::parent_path(TopFrameworkDir->getName());
+    if (const DirectoryEntry *ParentDir = FileMgr.getDirectory(Parent)) {
+      // If there's a module map file in the parent directory, it can
+      // explicitly allow us to infer framework modules.
+      switch (loadModuleMapFile(ParentDir)) {
+        case LMM_AlreadyLoaded:
+        case LMM_NewlyLoaded: {
+          StringRef Name = llvm::sys::path::stem(TopFrameworkDir->getName());
+          canInfer = ModMap.canInferFrameworkModule(ParentDir, Name, IsSystem);
+          break;
+        }
+        case LMM_InvalidModuleMap:
+        case LMM_NoDirectory:
+          break;
+      }
+    }
+  }
+
+  // If we're not allowed to infer a module map, we're done.
+  if (!canInfer)
+    return 0;
+
   // Try to infer a module map from the top-level framework directory.
   Module *Result = ModMap.inferFrameworkModule(SubmodulePath.back(), 
                                                TopFrameworkDir,
diff --git a/lib/Lex/Lexer.cpp b/lib/Lex/Lexer.cpp
index 5212dd8..a5ba7db 100644
--- a/lib/Lex/Lexer.cpp
+++ b/lib/Lex/Lexer.cpp
@@ -513,10 +513,13 @@ Lexer::ComputePreamble(const llvm::MemoryBuffer *Buffer,
   // "fake" file source location at offset 1 so that the lexer will track our
   // position within the file.
   const unsigned StartOffset = 1;
-  SourceLocation StartLoc = SourceLocation::getFromRawEncoding(StartOffset);
-  Lexer TheLexer(StartLoc, LangOpts, Buffer->getBufferStart(),
+  SourceLocation FileLoc = SourceLocation::getFromRawEncoding(StartOffset);
+  Lexer TheLexer(FileLoc, LangOpts, Buffer->getBufferStart(),
                  Buffer->getBufferStart(), Buffer->getBufferEnd());
-  
+
+  // StartLoc will differ from FileLoc if there is a BOM that was skipped.
+  SourceLocation StartLoc = TheLexer.getSourceLocation();
+
   bool InPreprocessorDirective = false;
   Token TheTok;
   Token IfStartTok;
@@ -1534,7 +1537,7 @@ FinishIdentifier:
 
 /// isHexaLiteral - Return true if Start points to a hex constant.
 /// in microsoft mode (where this is supposed to be several different tokens).
-static bool isHexaLiteral(const char *Start, const LangOptions &LangOpts) {
+bool Lexer::isHexaLiteral(const char *Start, const LangOptions &LangOpts) {
   unsigned Size;
   char C1 = Lexer::getCharAndSizeNoWarn(Start, Size, LangOpts);
   if (C1 != '0')
@@ -1813,17 +1816,18 @@ void Lexer::LexCharConstant(Token &Result, const char *CurPtr,
 
   while (C != '\'') {
     // Skip escaped characters.
-    if (C == '\\') {
-      // Skip the escaped character.
-      // FIXME: UCN's
-      getAndAdvanceChar(CurPtr, Result);
-    } else if (C == '\n' || C == '\r' ||             // Newline.
-               (C == 0 && CurPtr-1 == BufferEnd)) {  // End of file.
+    if (C == '\\')
+      C = getAndAdvanceChar(CurPtr, Result);
+
+    if (C == '\n' || C == '\r' ||             // Newline.
+        (C == 0 && CurPtr-1 == BufferEnd)) {  // End of file.
       if (!isLexingRawMode() && !LangOpts.AsmPreprocessor)
         Diag(BufferPtr, diag::ext_unterminated_char);
       FormTokenWithChars(Result, CurPtr-1, tok::unknown);
       return;
-    } else if (C == 0) {
+    }
+
+    if (C == 0) {
       if (isCodeCompletionPoint(CurPtr-1)) {
         PP->CodeCompleteNaturalLanguage();
         FormTokenWithChars(Result, CurPtr-1, tok::unknown);
@@ -1895,21 +1899,21 @@ bool Lexer::SkipWhitespace(Token &Result, const char *CurPtr) {
   return false;
 }
 
-// SkipBCPLComment - We have just read the // characters from input.  Skip until
-// we find the newline character thats terminate the comment.  Then update
-/// BufferPtr and return.
+/// We have just read the // characters from input.  Skip until we find the
+/// newline character thats terminate the comment.  Then update BufferPtr and
+/// return.
 ///
 /// If we're in KeepCommentMode or any CommentHandler has inserted
 /// some tokens, this will store the first token and return true.
-bool Lexer::SkipBCPLComment(Token &Result, const char *CurPtr) {
-  // If BCPL comments aren't explicitly enabled for this language, emit an
+bool Lexer::SkipLineComment(Token &Result, const char *CurPtr) {
+  // If Line comments aren't explicitly enabled for this language, emit an
   // extension warning.
-  if (!LangOpts.BCPLComment && !isLexingRawMode()) {
-    Diag(BufferPtr, diag::ext_bcpl_comment);
+  if (!LangOpts.LineComment && !isLexingRawMode()) {
+    Diag(BufferPtr, diag::ext_line_comment);
 
     // Mark them enabled so we only emit one warning for this translation
     // unit.
-    LangOpts.BCPLComment = true;
+    LangOpts.LineComment = true;
   }
 
   // Scan over the body of the comment.  The common case, when scanning, is that
@@ -1973,7 +1977,7 @@ bool Lexer::SkipBCPLComment(Token &Result, const char *CurPtr) {
           }
 
           if (!isLexingRawMode())
-            Diag(OldPtr-1, diag::ext_multi_line_bcpl_comment);
+            Diag(OldPtr-1, diag::ext_multi_line_line_comment);
           break;
         }
     }
@@ -2002,7 +2006,7 @@ bool Lexer::SkipBCPLComment(Token &Result, const char *CurPtr) {
 
   // If we are returning comments as tokens, return this comment as a token.
   if (inKeepCommentMode())
-    return SaveBCPLComment(Result, CurPtr);
+    return SaveLineComment(Result, CurPtr);
 
   // If we are inside a preprocessor directive and we see the end of line,
   // return immediately, so that the lexer can return this as an EOD token.
@@ -2026,9 +2030,9 @@ bool Lexer::SkipBCPLComment(Token &Result, const char *CurPtr) {
   return false;
 }
 
-/// SaveBCPLComment - If in save-comment mode, package up this BCPL comment in
-/// an appropriate way and return it.
-bool Lexer::SaveBCPLComment(Token &Result, const char *CurPtr) {
+/// If in save-comment mode, package up this Line comment in an appropriate
+/// way and return it.
+bool Lexer::SaveLineComment(Token &Result, const char *CurPtr) {
   // If we're not in a preprocessor directive, just return the // comment
   // directly.
   FormTokenWithChars(Result, CurPtr, tok::comment);
@@ -2036,19 +2040,19 @@ bool Lexer::SaveBCPLComment(Token &Result, const char *CurPtr) {
   if (!ParsingPreprocessorDirective || LexingRawMode)
     return true;
 
-  // If this BCPL-style comment is in a macro definition, transmogrify it into
+  // If this Line-style comment is in a macro definition, transmogrify it into
   // a C-style block comment.
   bool Invalid = false;
   std::string Spelling = PP->getSpelling(Result, &Invalid);
   if (Invalid)
     return true;
   
-  assert(Spelling[0] == '/' && Spelling[1] == '/' && "Not bcpl comment?");
+  assert(Spelling[0] == '/' && Spelling[1] == '/' && "Not line comment?");
   Spelling[1] = '*';   // Change prefix to "/*".
   Spelling += "*/";    // add suffix.
 
   Result.setKind(tok::comment);
-  PP->CreateString(&Spelling[0], Spelling.size(), Result,
+  PP->CreateString(Spelling, Result,
                    Result.getLocation(), Result.getLocation());
   return true;
 }
@@ -2179,7 +2183,8 @@ bool Lexer::SkipBlockComment(Token &Result, const char *CurPtr) {
 #ifdef __SSE2__
       __m128i Slashes = _mm_set1_epi8('/');
       while (CurPtr+16 <= BufferEnd) {
-        int cmp = _mm_movemask_epi8(_mm_cmpeq_epi8(*(__m128i*)CurPtr, Slashes));
+        int cmp = _mm_movemask_epi8(_mm_cmpeq_epi8(*(const __m128i*)CurPtr,
+                                    Slashes));
         if (cmp != 0) {
           // Adjust the pointer to point directly after the first slash. It's
           // not necessary to set C here, it will be overwritten at the end of
@@ -2669,8 +2674,8 @@ LexNextToken:
     // If the next token is obviously a // or /* */ comment, skip it efficiently
     // too (without going through the big switch stmt).
     if (CurPtr[0] == '/' && CurPtr[1] == '/' && !inKeepCommentMode() &&
-        LangOpts.BCPLComment && !LangOpts.TraditionalCPP) {
-      if (SkipBCPLComment(Result, CurPtr+2))
+        LangOpts.LineComment && !LangOpts.TraditionalCPP) {
+      if (SkipLineComment(Result, CurPtr+2))
         return; // There is a token to return.
       goto SkipIgnoredUnits;
     } else if (CurPtr[0] == '/' && CurPtr[1] == '*' && !inKeepCommentMode()) {
@@ -2955,19 +2960,19 @@ LexNextToken:
   case '/':
     // 6.4.9: Comments
     Char = getCharAndSize(CurPtr, SizeTmp);
-    if (Char == '/') {         // BCPL comment.
-      // Even if BCPL comments are disabled (e.g. in C89 mode), we generally
+    if (Char == '/') {         // Line comment.
+      // Even if Line comments are disabled (e.g. in C89 mode), we generally
       // want to lex this as a comment.  There is one problem with this though,
       // that in one particular corner case, this can change the behavior of the
       // resultant program.  For example, In  "foo //**/ bar", C89 would lex
-      // this as "foo / bar" and langauges with BCPL comments would lex it as
+      // this as "foo / bar" and langauges with Line comments would lex it as
       // "foo".  Check to see if the character after the second slash is a '*'.
       // If so, we will lex that as a "/" instead of the start of a comment.
       // However, we never do this in -traditional-cpp mode.
-      if ((LangOpts.BCPLComment ||
+      if ((LangOpts.LineComment ||
            getCharAndSize(CurPtr+SizeTmp, SizeTmp2) != '*') &&
           !LangOpts.TraditionalCPP) {
-        if (SkipBCPLComment(Result, ConsumeChar(CurPtr, SizeTmp, Result)))
+        if (SkipLineComment(Result, ConsumeChar(CurPtr, SizeTmp, Result)))
           return; // There is a token to return.
 
         // It is common for the tokens immediately after a // comment to be
diff --git a/lib/Lex/LiteralSupport.cpp b/lib/Lex/LiteralSupport.cpp
index 9e3c778..e30612e 100644
--- a/lib/Lex/LiteralSupport.cpp
+++ b/lib/Lex/LiteralSupport.cpp
@@ -49,12 +49,46 @@ static unsigned getCharWidth(tok::TokenKind kind, const TargetInfo &Target) {
   }
 }
 
+static CharSourceRange MakeCharSourceRange(const LangOptions &Features,
+                                           FullSourceLoc TokLoc,
+                                           const char *TokBegin,
+                                           const char *TokRangeBegin,
+                                           const char *TokRangeEnd) {
+  SourceLocation Begin =
+    Lexer::AdvanceToTokenCharacter(TokLoc, TokRangeBegin - TokBegin,
+                                   TokLoc.getManager(), Features);
+  SourceLocation End =
+    Lexer::AdvanceToTokenCharacter(Begin, TokRangeEnd - TokRangeBegin,
+                                   TokLoc.getManager(), Features);
+  return CharSourceRange::getCharRange(Begin, End);
+}
+
+/// \brief Produce a diagnostic highlighting some portion of a literal.
+///
+/// Emits the diagnostic \p DiagID, highlighting the range of characters from
+/// \p TokRangeBegin (inclusive) to \p TokRangeEnd (exclusive), which must be
+/// a substring of a spelling buffer for the token beginning at \p TokBegin.
+static DiagnosticBuilder Diag(DiagnosticsEngine *Diags,
+                              const LangOptions &Features, FullSourceLoc TokLoc,
+                              const char *TokBegin, const char *TokRangeBegin,
+                              const char *TokRangeEnd, unsigned DiagID) {
+  SourceLocation Begin =
+    Lexer::AdvanceToTokenCharacter(TokLoc, TokRangeBegin - TokBegin,
+                                   TokLoc.getManager(), Features);
+  return Diags->Report(Begin, DiagID) <<
+    MakeCharSourceRange(Features, TokLoc, TokBegin, TokRangeBegin, TokRangeEnd);
+}
+
 /// ProcessCharEscape - Parse a standard C escape sequence, which can occur in
 /// either a character or a string literal.
-static unsigned ProcessCharEscape(const char *&ThisTokBuf,
+static unsigned ProcessCharEscape(const char *ThisTokBegin,
+                                  const char *&ThisTokBuf,
                                   const char *ThisTokEnd, bool &HadError,
                                   FullSourceLoc Loc, unsigned CharWidth,
-                                  DiagnosticsEngine *Diags) {
+                                  DiagnosticsEngine *Diags,
+                                  const LangOptions &Features) {
+  const char *EscapeBegin = ThisTokBuf;
+
   // Skip the '\' char.
   ++ThisTokBuf;
 
@@ -75,12 +109,14 @@ static unsigned ProcessCharEscape(const char *&ThisTokBuf,
     break;
   case 'e':
     if (Diags)
-      Diags->Report(Loc, diag::ext_nonstandard_escape) << "e";
+      Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
+           diag::ext_nonstandard_escape) << "e";
     ResultChar = 27;
     break;
   case 'E':
     if (Diags)
-      Diags->Report(Loc, diag::ext_nonstandard_escape) << "E";
+      Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
+           diag::ext_nonstandard_escape) << "E";
     ResultChar = 27;
     break;
   case 'f':
@@ -102,7 +138,8 @@ static unsigned ProcessCharEscape(const char *&ThisTokBuf,
     ResultChar = 0;
     if (ThisTokBuf == ThisTokEnd || !isxdigit(*ThisTokBuf)) {
       if (Diags)
-        Diags->Report(Loc, diag::err_hex_escape_no_digits);
+        Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
+             diag::err_hex_escape_no_digits);
       HadError = 1;
       break;
     }
@@ -126,7 +163,8 @@ static unsigned ProcessCharEscape(const char *&ThisTokBuf,
 
     // Check for overflow.
     if (Overflow && Diags)   // Too many digits to fit in
-      Diags->Report(Loc, diag::warn_hex_escape_too_large);
+      Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
+           diag::warn_hex_escape_too_large);
     break;
   }
   case '0': case '1': case '2': case '3':
@@ -148,7 +186,8 @@ static unsigned ProcessCharEscape(const char *&ThisTokBuf,
     // Check for overflow.  Reject '\777', but not L'\777'.
     if (CharWidth != 32 && (ResultChar >> CharWidth) != 0) {
       if (Diags)
-        Diags->Report(Loc, diag::warn_octal_escape_too_large);
+        Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
+             diag::warn_octal_escape_too_large);
       ResultChar &= ~0U >> (32-CharWidth);
     }
     break;
@@ -158,19 +197,22 @@ static unsigned ProcessCharEscape(const char *&ThisTokBuf,
   case '(': case '{': case '[': case '%':
     // GCC accepts these as extensions.  We warn about them as such though.
     if (Diags)
-      Diags->Report(Loc, diag::ext_nonstandard_escape)
-        << std::string()+(char)ResultChar;
+      Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
+           diag::ext_nonstandard_escape)
+        << std::string(1, ResultChar);
     break;
   default:
     if (Diags == 0)
       break;
-      
+
     if (isgraph(ResultChar))
-      Diags->Report(Loc, diag::ext_unknown_escape)
-        << std::string()+(char)ResultChar;
+      Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
+           diag::ext_unknown_escape)
+        << std::string(1, ResultChar);
     else
-      Diags->Report(Loc, diag::ext_unknown_escape)
-        << "x"+llvm::utohexstr(ResultChar);
+      Diag(Diags, Features, Loc, ThisTokBegin, EscapeBegin, ThisTokBuf,
+           diag::ext_unknown_escape)
+        << "x" + llvm::utohexstr(ResultChar);
     break;
   }
 
@@ -185,9 +227,6 @@ static bool ProcessUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf,
                              FullSourceLoc Loc, DiagnosticsEngine *Diags, 
                              const LangOptions &Features,
                              bool in_char_string_literal = false) {
-  if (!Features.CPlusPlus && !Features.C99 && Diags)
-    Diags->Report(Loc, diag::warn_ucn_not_valid_in_c89);
-
   const char *UcnBegin = ThisTokBuf;
 
   // Skip the '\u' char's.
@@ -195,7 +234,8 @@ static bool ProcessUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf,
 
   if (ThisTokBuf == ThisTokEnd || !isxdigit(*ThisTokBuf)) {
     if (Diags)
-      Diags->Report(Loc, diag::err_ucn_escape_no_digits);
+      Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
+           diag::err_ucn_escape_no_digits);
     return false;
   }
   UcnLen = (ThisTokBuf[-1] == 'u' ? 4 : 8);
@@ -208,12 +248,9 @@ static bool ProcessUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf,
   }
   // If we didn't consume the proper number of digits, there is a problem.
   if (UcnLenSave) {
-    if (Diags) {
-      SourceLocation L =
-        Lexer::AdvanceToTokenCharacter(Loc, UcnBegin - ThisTokBegin,
-                                       Loc.getManager(), Features);
-      Diags->Report(L, diag::err_ucn_escape_incomplete);
-    }
+    if (Diags)
+      Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
+           diag::err_ucn_escape_incomplete);
     return false;
   }
 
@@ -221,7 +258,8 @@ static bool ProcessUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf,
   if ((0xD800 <= UcnVal && UcnVal <= 0xDFFF) || // surrogate codepoints
       UcnVal > 0x10FFFF) {                      // maximum legal UTF32 value
     if (Diags)
-      Diags->Report(Loc, diag::err_ucn_escape_invalid);
+      Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
+           diag::err_ucn_escape_invalid);
     return false;
   }
 
@@ -231,22 +269,25 @@ static bool ProcessUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf,
       (UcnVal != 0x24 && UcnVal != 0x40 && UcnVal != 0x60)) {  // $, @, `
     bool IsError = (!Features.CPlusPlus0x || !in_char_string_literal);
     if (Diags) {
-      SourceLocation UcnBeginLoc =
-        Lexer::AdvanceToTokenCharacter(Loc, UcnBegin - ThisTokBegin,
-                                       Loc.getManager(), Features);
       char BasicSCSChar = UcnVal;
       if (UcnVal >= 0x20 && UcnVal < 0x7f)
-        Diags->Report(UcnBeginLoc, IsError ? diag::err_ucn_escape_basic_scs :
-                      diag::warn_cxx98_compat_literal_ucn_escape_basic_scs)
-          << StringRef(&BasicSCSChar, 1);
+        Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
+             IsError ? diag::err_ucn_escape_basic_scs :
+                       diag::warn_cxx98_compat_literal_ucn_escape_basic_scs)
+            << StringRef(&BasicSCSChar, 1);
       else
-        Diags->Report(UcnBeginLoc, IsError ? diag::err_ucn_control_character :
-                      diag::warn_cxx98_compat_literal_ucn_control_character);
+        Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
+             IsError ? diag::err_ucn_control_character :
+                       diag::warn_cxx98_compat_literal_ucn_control_character);
     }
     if (IsError)
       return false;
   }
 
+  if (!Features.CPlusPlus && !Features.C99 && Diags)
+    Diag(Diags, Features, Loc, ThisTokBegin, UcnBegin, ThisTokBuf,
+         diag::warn_ucn_not_valid_in_c89);
+
   return true;
 }
 
@@ -365,10 +406,10 @@ static void EncodeUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf,
   // Finally, we write the bytes into ResultBuf.
   ResultBuf += bytesToWrite;
   switch (bytesToWrite) { // note: everything falls through.
-    case 4: *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6;
-    case 3: *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6;
-    case 2: *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6;
-    case 1: *--ResultBuf = (UTF8) (UcnVal | firstByteMark[bytesToWrite]);
+  case 4: *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6;
+  case 3: *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6;
+  case 2: *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6;
+  case 1: *--ResultBuf = (UTF8) (UcnVal | firstByteMark[bytesToWrite]);
   }
   // Update the buffer.
   ResultBuf += bytesToWrite;
@@ -417,19 +458,19 @@ static void EncodeUCNEscape(const char *ThisTokBegin, const char *&ThisTokBuf,
 ///       floating-constant: [C99 6.4.4.2]
 ///         TODO: add rules...
 ///
-NumericLiteralParser::
-NumericLiteralParser(const char *begin, const char *end,
-                     SourceLocation TokLoc, Preprocessor &pp)
-  : PP(pp), ThisTokBegin(begin), ThisTokEnd(end) {
+NumericLiteralParser::NumericLiteralParser(StringRef TokSpelling,
+                                           SourceLocation TokLoc,
+                                           Preprocessor &PP)
+  : PP(PP), ThisTokBegin(TokSpelling.begin()), ThisTokEnd(TokSpelling.end()) {
 
   // This routine assumes that the range begin/end matches the regex for integer
   // and FP constants (specifically, the 'pp-number' regex), and assumes that
   // the byte at "*end" is both valid and not part of the regex.  Because of
   // this, it doesn't have to check for 'overscan' in various places.
-  assert(!isalnum(*end) && *end != '.' && *end != '_' &&
+  assert(!isalnum(*ThisTokEnd) && *ThisTokEnd != '.' && *ThisTokEnd != '_' &&
          "Lexer didn't maximally munch?");
 
-  s = DigitsBegin = begin;
+  s = DigitsBegin = ThisTokBegin;
   saw_exponent = false;
   saw_period = false;
   saw_ud_suffix = false;
@@ -451,7 +492,7 @@ NumericLiteralParser(const char *begin, const char *end,
     if (s == ThisTokEnd) {
       // Done.
     } else if (isxdigit(*s) && !(*s == 'e' || *s == 'E')) {
-      PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-begin),
+      PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s - ThisTokBegin),
               diag::err_invalid_decimal_digit) << StringRef(s, 1);
       hadError = true;
       return;
@@ -469,7 +510,7 @@ NumericLiteralParser(const char *begin, const char *end,
       if (first_non_digit != s) {
         s = first_non_digit;
       } else {
-        PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, Exponent-begin),
+        PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, Exponent - ThisTokBegin),
                 diag::err_exponent_has_no_digits);
         hadError = true;
         return;
@@ -565,7 +606,7 @@ NumericLiteralParser(const char *begin, const char *end,
     case 'j':
     case 'J':
       if (isImaginary) break;   // Cannot be repeated.
-      PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-begin),
+      PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s - ThisTokBegin),
               diag::ext_imaginary_constant);
       isImaginary = true;
       continue;  // Success.
@@ -583,7 +624,7 @@ NumericLiteralParser(const char *begin, const char *end,
     }
 
     // Report an error if there are any.
-    PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, SuffixBegin-begin),
+    PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, SuffixBegin - ThisTokBegin),
             isFPConstant ? diag::err_invalid_suffix_float_constant :
                            diag::err_invalid_suffix_integer_constant)
       << StringRef(SuffixBegin, ThisTokEnd-SuffixBegin);
@@ -619,7 +660,7 @@ void NumericLiteralParser::ParseNumberStartingWithZero(SourceLocation TokLoc) {
     }
 
     if (noSignificand) {
-      PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-ThisTokBegin), \
+      PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s - ThisTokBegin),
         diag::err_hexconstant_requires_digits);
       hadError = true;
       return;
@@ -722,6 +763,20 @@ void NumericLiteralParser::ParseNumberStartingWithZero(SourceLocation TokLoc) {
   }
 }
 
+static bool alwaysFitsInto64Bits(unsigned Radix, unsigned NumDigits) {
+  switch (Radix) {
+  case 2:
+    return NumDigits <= 64;
+  case 8:
+    return NumDigits <= 64 / 3; // Digits are groups of 3 bits.
+  case 10:
+    return NumDigits <= 19; // floor(log10(2^64))
+  case 16:
+    return NumDigits <= 64 / 4; // Digits are groups of 4 bits.
+  default:
+    llvm_unreachable("impossible Radix");
+  }
+}
 
 /// GetIntegerValue - Convert this numeric literal value to an APInt that
 /// matches Val's input width.  If there is an overflow, set Val to the low bits
@@ -733,13 +788,11 @@ bool NumericLiteralParser::GetIntegerValue(llvm::APInt &Val) {
   // integer. This avoids the expensive overflow checking below, and
   // handles the common cases that matter (small decimal integers and
   // hex/octal values which don't overflow).
-  unsigned MaxBitsPerDigit = 1;
-  while ((1U << MaxBitsPerDigit) < radix)
-    MaxBitsPerDigit += 1;
-  if ((SuffixBegin - DigitsBegin) * MaxBitsPerDigit <= 64) {
+  const unsigned NumDigits = SuffixBegin - DigitsBegin;
+  if (alwaysFitsInto64Bits(radix, NumDigits)) {
     uint64_t N = 0;
-    for (s = DigitsBegin; s != SuffixBegin; ++s)
-      N = N*radix + HexDigitValue(*s);
+    for (const char *Ptr = DigitsBegin; Ptr != SuffixBegin; ++Ptr)
+      N = N * radix + HexDigitValue(*Ptr);
 
     // This will truncate the value to Val's input width. Simply check
     // for overflow by comparing.
@@ -748,15 +801,15 @@ bool NumericLiteralParser::GetIntegerValue(llvm::APInt &Val) {
   }
 
   Val = 0;
-  s = DigitsBegin;
+  const char *Ptr = DigitsBegin;
 
   llvm::APInt RadixVal(Val.getBitWidth(), radix);
   llvm::APInt CharVal(Val.getBitWidth(), 0);
   llvm::APInt OldVal = Val;
 
   bool OverflowOccurred = false;
-  while (s < SuffixBegin) {
-    unsigned C = HexDigitValue(*s++);
+  while (Ptr < SuffixBegin) {
+    unsigned C = HexDigitValue(*Ptr++);
 
     // If this letter is out of bound for this radix, reject it.
     assert(C < radix && "NumericLiteralParser ctor should have rejected this");
@@ -943,7 +996,7 @@ CharLiteralParser::CharLiteralParser(const char *begin, const char *end,
         HadError = true;
       } else if (*buffer_begin > largest_character_for_kind) {
         HadError = true;
-        PP.Diag(Loc,diag::err_character_too_large);
+        PP.Diag(Loc, diag::err_character_too_large);
       }
 
       ++buffer_begin;
@@ -951,9 +1004,9 @@ CharLiteralParser::CharLiteralParser(const char *begin, const char *end,
     }
     unsigned CharWidth = getCharWidth(Kind, PP.getTargetInfo());
     uint64_t result =
-    ProcessCharEscape(begin, end, HadError,
-                      FullSourceLoc(Loc,PP.getSourceManager()),
-                      CharWidth, &PP.getDiagnostics());
+      ProcessCharEscape(TokBegin, begin, end, HadError,
+                        FullSourceLoc(Loc,PP.getSourceManager()),
+                        CharWidth, &PP.getDiagnostics(), PP.getLangOpts());
     *buffer_begin++ = result;
   }
 
@@ -1110,7 +1163,7 @@ void StringLiteralParser::init(const Token *StringToks, unsigned NumStringToks){
         Kind = StringToks[i].getKind();
       } else {
         if (Diags)
-          Diags->Report(FullSourceLoc(StringToks[i].getLocation(), SM),
+          Diags->Report(StringToks[i].getLocation(),
                         diag::err_unsupported_string_concat);
         hadError = true;
       }
@@ -1218,9 +1271,9 @@ void StringLiteralParser::init(const Token *StringToks, unsigned NumStringToks){
       assert(ThisTokEnd >= ThisTokBuf && "malformed raw string literal");
 
       // Copy the string over
-      if (CopyStringFragment(StringRef(ThisTokBuf, ThisTokEnd - ThisTokBuf)))
-        if (DiagnoseBadString(StringToks[i]))
-          hadError = true;
+      if (CopyStringFragment(StringToks[i], ThisTokBegin,
+                             StringRef(ThisTokBuf, ThisTokEnd - ThisTokBuf)))
+        hadError = true;
     } else {
       if (ThisTokBuf[0] != '"') {
         // The file may have come from PCH and then changed after loading the
@@ -1251,9 +1304,9 @@ void StringLiteralParser::init(const Token *StringToks, unsigned NumStringToks){
           } while (ThisTokBuf != ThisTokEnd && ThisTokBuf[0] != '\\');
 
           // Copy the character span over.
-          if (CopyStringFragment(StringRef(InStart, ThisTokBuf - InStart)))
-            if (DiagnoseBadString(StringToks[i]))
-              hadError = true;
+          if (CopyStringFragment(StringToks[i], ThisTokBegin,
+                                 StringRef(InStart, ThisTokBuf - InStart)))
+            hadError = true;
           continue;
         }
         // Is this a Universal Character Name escape?
@@ -1266,9 +1319,9 @@ void StringLiteralParser::init(const Token *StringToks, unsigned NumStringToks){
         }
         // Otherwise, this is a non-UCN escape character.  Process it.
         unsigned ResultChar =
-          ProcessCharEscape(ThisTokBuf, ThisTokEnd, hadError,
+          ProcessCharEscape(ThisTokBegin, ThisTokBuf, ThisTokEnd, hadError,
                             FullSourceLoc(StringToks[i].getLocation(), SM),
-                            CharByteWidth*8, Diags);
+                            CharByteWidth*8, Diags, Features);
 
         if (CharByteWidth == 4) {
           // FIXME: Make the type of the result buffer correct instead of
@@ -1308,8 +1361,8 @@ void StringLiteralParser::init(const Token *StringToks, unsigned NumStringToks){
 
     // Verify that pascal strings aren't too large.
     if (GetStringLength() > 256) {
-      if (Diags) 
-        Diags->Report(FullSourceLoc(StringToks[0].getLocation(), SM),
+      if (Diags)
+        Diags->Report(StringToks[0].getLocation(),
                       diag::err_pascal_string_too_long)
           << SourceRange(StringToks[0].getLocation(),
                          StringToks[NumStringToks-1].getLocation());
@@ -1319,9 +1372,9 @@ void StringLiteralParser::init(const Token *StringToks, unsigned NumStringToks){
   } else if (Diags) {
     // Complain if this string literal has too many characters.
     unsigned MaxChars = Features.CPlusPlus? 65536 : Features.C99 ? 4095 : 509;
-    
+
     if (GetNumStringChars() > MaxChars)
-      Diags->Report(FullSourceLoc(StringToks[0].getLocation(), SM),
+      Diags->Report(StringToks[0].getLocation(),
                     diag::ext_string_too_long)
         << GetNumStringChars() << MaxChars
         << (Features.CPlusPlus ? 2 : Features.C99 ? 1 : 0)
@@ -1330,21 +1383,61 @@ void StringLiteralParser::init(const Token *StringToks, unsigned NumStringToks){
   }
 }
 
-/// copyStringFragment - This function copies from Start to End into ResultPtr.
-/// Performs widening for multi-byte characters.
-bool StringLiteralParser::CopyStringFragment(StringRef Fragment) {
-  return !ConvertUTF8toWide(CharByteWidth, Fragment, ResultPtr);
+static const char *resyncUTF8(const char *Err, const char *End) {
+  if (Err == End)
+    return End;
+  End = Err + std::min<unsigned>(getNumBytesForUTF8(*Err), End-Err);
+  while (++Err != End && (*Err & 0xC0) == 0x80)
+    ;
+  return Err;
 }
 
-bool StringLiteralParser::DiagnoseBadString(const Token &Tok) {
+/// \brief This function copies from Fragment, which is a sequence of bytes
+/// within Tok's contents (which begin at TokBegin) into ResultPtr.
+/// Performs widening for multi-byte characters.
+bool StringLiteralParser::CopyStringFragment(const Token &Tok,
+                                             const char *TokBegin,
+                                             StringRef Fragment) {
+  const UTF8 *ErrorPtrTmp;
+  if (ConvertUTF8toWide(CharByteWidth, Fragment, ResultPtr, ErrorPtrTmp))
+    return false;
+
   // If we see bad encoding for unprefixed string literals, warn and
   // simply copy the byte values, for compatibility with gcc and older
   // versions of clang.
   bool NoErrorOnBadEncoding = isAscii();
-  unsigned Msg = NoErrorOnBadEncoding ? diag::warn_bad_string_encoding :
-                                        diag::err_bad_string_encoding;
-  if (Diags)
-    Diags->Report(FullSourceLoc(Tok.getLocation(), SM), Msg);
+  if (NoErrorOnBadEncoding) {
+    memcpy(ResultPtr, Fragment.data(), Fragment.size());
+    ResultPtr += Fragment.size();
+  }
+
+  if (Diags) {
+    const char *ErrorPtr = reinterpret_cast<const char *>(ErrorPtrTmp);
+
+    FullSourceLoc SourceLoc(Tok.getLocation(), SM);
+    const DiagnosticBuilder &Builder =
+      Diag(Diags, Features, SourceLoc, TokBegin,
+           ErrorPtr, resyncUTF8(ErrorPtr, Fragment.end()),
+           NoErrorOnBadEncoding ? diag::warn_bad_string_encoding
+                                : diag::err_bad_string_encoding);
+
+    const char *NextStart = resyncUTF8(ErrorPtr, Fragment.end());
+    StringRef NextFragment(NextStart, Fragment.end()-NextStart);
+
+    // Decode into a dummy buffer.
+    SmallString<512> Dummy;
+    Dummy.reserve(Fragment.size() * CharByteWidth);
+    char *Ptr = Dummy.data();
+
+    while (!Builder.hasMaxRanges() &&
+           !ConvertUTF8toWide(CharByteWidth, NextFragment, Ptr, ErrorPtrTmp)) {
+      const char *ErrorPtr = reinterpret_cast<const char *>(ErrorPtrTmp);
+      NextStart = resyncUTF8(ErrorPtr, Fragment.end());
+      Builder << MakeCharSourceRange(Features, SourceLoc, TokBegin,
+                                     ErrorPtr, NextStart);
+      NextFragment = StringRef(NextStart, Fragment.end()-NextStart);
+    }
+  }
   return !NoErrorOnBadEncoding;
 }
 
@@ -1422,9 +1515,9 @@ unsigned StringLiteralParser::getOffsetOfStringByte(const Token &Tok,
       }
       ByteNo -= Len;
     } else {
-      ProcessCharEscape(SpellingPtr, SpellingEnd, HadError,
+      ProcessCharEscape(SpellingStart, SpellingPtr, SpellingEnd, HadError,
                         FullSourceLoc(Tok.getLocation(), SM),
-                        CharByteWidth*8, Diags);
+                        CharByteWidth*8, Diags, Features);
       --ByteNo;
     }
     assert(!HadError && "This method isn't valid on erroneous strings");
diff --git a/lib/Lex/MacroArgs.cpp b/lib/Lex/MacroArgs.cpp
index e2b251a..ed8873d 100644
--- a/lib/Lex/MacroArgs.cpp
+++ b/lib/Lex/MacroArgs.cpp
@@ -291,7 +291,7 @@ Token MacroArgs::StringifyArgument(const Token *ArgToks,
     }
   }
 
-  PP.CreateString(&Result[0], Result.size(), Tok,
+  PP.CreateString(Result, Tok,
                   ExpansionLocStart, ExpansionLocEnd);
   return Tok;
 }
diff --git a/lib/Lex/MacroInfo.cpp b/lib/Lex/MacroInfo.cpp
index 3d0c9a1..904f04e 100644
--- a/lib/Lex/MacroInfo.cpp
+++ b/lib/Lex/MacroInfo.cpp
@@ -15,47 +15,65 @@
 #include "clang/Lex/Preprocessor.h"
 using namespace clang;
 
-MacroInfo::MacroInfo(SourceLocation DefLoc) : Location(DefLoc) {
-  IsFunctionLike = false;
-  IsC99Varargs = false;
-  IsGNUVarargs = false;
-  IsBuiltinMacro = false;
-  IsFromAST = false;
-  ChangedAfterLoad = false;
-  IsDisabled = false;
-  IsUsed = false;
-  IsAllowRedefinitionsWithoutWarning = false;
-  IsWarnIfUnused = false;
-  IsDefinitionLengthCached = false;
-  IsPublic = true;
-  
-  ArgumentList = 0;
-  NumArguments = 0;
+MacroInfo::MacroInfo(SourceLocation DefLoc)
+  : Location(DefLoc),
+    PreviousDefinition(0),
+    ArgumentList(0),
+    NumArguments(0),
+    IsDefinitionLengthCached(false),
+    IsFunctionLike(false),
+    IsC99Varargs(false),
+    IsGNUVarargs(false),
+    IsBuiltinMacro(false),
+    IsFromAST(false),
+    ChangedAfterLoad(false),
+    IsDisabled(false),
+    IsUsed(false),
+    IsAllowRedefinitionsWithoutWarning(false),
+    IsWarnIfUnused(false),
+    IsPublic(true),
+    IsHidden(false),
+    IsAmbiguous(false) {
 }
 
-MacroInfo::MacroInfo(const MacroInfo &MI, llvm::BumpPtrAllocator &PPAllocator) {
-  Location = MI.Location;
-  EndLocation = MI.EndLocation;
-  ReplacementTokens = MI.ReplacementTokens;
-  IsFunctionLike = MI.IsFunctionLike;
-  IsC99Varargs = MI.IsC99Varargs;
-  IsGNUVarargs = MI.IsGNUVarargs;
-  IsBuiltinMacro = MI.IsBuiltinMacro;
-  IsFromAST = MI.IsFromAST;
-  ChangedAfterLoad = MI.ChangedAfterLoad;
-  IsDisabled = MI.IsDisabled;
-  IsUsed = MI.IsUsed;
-  IsAllowRedefinitionsWithoutWarning = MI.IsAllowRedefinitionsWithoutWarning;
-  IsWarnIfUnused = MI.IsWarnIfUnused;
-  IsDefinitionLengthCached = MI.IsDefinitionLengthCached;
-  DefinitionLength = MI.DefinitionLength;
-  IsPublic = MI.IsPublic;
-  
-  ArgumentList = 0;
-  NumArguments = 0;
+MacroInfo::MacroInfo(const MacroInfo &MI, llvm::BumpPtrAllocator &PPAllocator)
+  : Location(MI.Location),
+    EndLocation(MI.EndLocation),
+    UndefLocation(MI.UndefLocation),
+    PreviousDefinition(0),
+    ArgumentList(0),
+    NumArguments(0),
+    ReplacementTokens(MI.ReplacementTokens),
+    DefinitionLength(MI.DefinitionLength),
+    IsDefinitionLengthCached(MI.IsDefinitionLengthCached),
+    IsFunctionLike(MI.IsFunctionLike),
+    IsC99Varargs(MI.IsC99Varargs),
+    IsGNUVarargs(MI.IsGNUVarargs),
+    IsBuiltinMacro(MI.IsBuiltinMacro),
+    IsFromAST(MI.IsFromAST),
+    ChangedAfterLoad(MI.ChangedAfterLoad),
+    IsDisabled(MI.IsDisabled),
+    IsUsed(MI.IsUsed),
+    IsAllowRedefinitionsWithoutWarning(MI.IsAllowRedefinitionsWithoutWarning),
+    IsWarnIfUnused(MI.IsWarnIfUnused),
+    IsPublic(MI.IsPublic),
+    IsHidden(MI.IsHidden),
+    IsAmbiguous(MI.IsAmbiguous) {
   setArgumentList(MI.ArgumentList, MI.NumArguments, PPAllocator);
 }
 
+const MacroInfo *MacroInfo::findDefinitionAtLoc(SourceLocation L,
+                                                SourceManager &SM) const {
+  assert(L.isValid() && "SourceLocation is invalid.");
+  for (const MacroInfo *MI = this; MI; MI = MI->PreviousDefinition) {
+    if (MI->Location.isInvalid() ||  // For macros defined on the command line.
+        SM.isBeforeInTranslationUnit(MI->Location, L))
+      return (MI->UndefLocation.isInvalid() ||
+              SM.isBeforeInTranslationUnit(L, MI->UndefLocation)) ? MI : NULL;
+  }
+  return NULL;
+}
+
 unsigned MacroInfo::getDefinitionLengthSlow(SourceManager &SM) const {
   assert(!IsDefinitionLengthCached);
   IsDefinitionLengthCached = true;
diff --git a/lib/Lex/ModuleMap.cpp b/lib/Lex/ModuleMap.cpp
index 5304311..8a936fa 100644
--- a/lib/Lex/ModuleMap.cpp
+++ b/lib/Lex/ModuleMap.cpp
@@ -16,6 +16,7 @@
 #include "clang/Lex/LiteralSupport.h"
 #include "clang/Lex/LexDiagnostic.h"
 #include "clang/Basic/Diagnostic.h"
+#include "clang/Basic/DiagnosticOptions.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/TargetInfo.h"
 #include "clang/Basic/TargetOptions.h"
@@ -26,6 +27,7 @@
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSwitch.h"
+#include <stdlib.h>
 using namespace clang;
 
 Module::ExportDecl 
@@ -75,7 +77,7 @@ ModuleMap::ModuleMap(FileManager &FileMgr, const DiagnosticConsumer &DC,
 {
   IntrusiveRefCntPtr<DiagnosticIDs> DiagIDs(new DiagnosticIDs);
   Diags = IntrusiveRefCntPtr<DiagnosticsEngine>(
-            new DiagnosticsEngine(DiagIDs));
+            new DiagnosticsEngine(DiagIDs, new DiagnosticOptions));
   Diags->setClient(DC.clone(*Diags), /*ShouldOwnClient=*/true);
   SourceMgr = new SourceManager(*Diags, FileMgr);
 }
@@ -96,16 +98,62 @@ void ModuleMap::setTarget(const TargetInfo &Target) {
   this->Target = &Target;
 }
 
+/// \brief "Sanitize" a filename so that it can be used as an identifier.
+static StringRef sanitizeFilenameAsIdentifier(StringRef Name,
+                                              SmallVectorImpl<char> &Buffer) {
+  if (Name.empty())
+    return Name;
+
+  // Check whether the filename is already an identifier; this is the common
+  // case.
+  bool isIdentifier = true;
+  for (unsigned I = 0, N = Name.size(); I != N; ++I) {
+    if (isalpha(Name[I]) || Name[I] == '_' || (isdigit(Name[I]) && I > 0))
+      continue;
+
+    isIdentifier = false;
+    break;
+  }
+
+  if (!isIdentifier) {
+    // If we don't already have something with the form of an identifier,
+    // create a buffer with the sanitized name.
+    Buffer.clear();
+    if (isdigit(Name[0]))
+      Buffer.push_back('_');
+    Buffer.reserve(Buffer.size() + Name.size());
+    for (unsigned I = 0, N = Name.size(); I != N; ++I) {
+      if (isalnum(Name[I]) || isspace(Name[I]))
+        Buffer.push_back(Name[I]);
+      else
+        Buffer.push_back('_');
+    }
+
+    Name = StringRef(Buffer.data(), Buffer.size());
+  }
+
+  while (llvm::StringSwitch<bool>(Name)
+#define KEYWORD(Keyword,Conditions) .Case(#Keyword, true)
+#define ALIAS(Keyword, AliasOf, Conditions) .Case(Keyword, true)
+#include "clang/Basic/TokenKinds.def"
+           .Default(false)) {
+    if (Name.data() != Buffer.data())
+      Buffer.append(Name.begin(), Name.end());
+    Buffer.push_back('_');
+    Name = StringRef(Buffer.data(), Buffer.size());
+  }
+
+  return Name;
+}
+
 Module *ModuleMap::findModuleForHeader(const FileEntry *File) {
-  llvm::DenseMap<const FileEntry *, Module *>::iterator Known
-    = Headers.find(File);
+  HeadersMap::iterator Known = Headers.find(File);
   if (Known != Headers.end()) {
-    // If a header corresponds to an unavailable module, don't report
-    // that it maps to anything.
-    if (!Known->second->isAvailable())
+    // If a header is not available, don't report that it maps to anything.
+    if (!Known->second.isAvailable())
       return 0;
 
-    return Known->second;
+    return Known->second.getModule();
   }
   
   const DirectoryEntry *Dir = File->getDir();
@@ -134,7 +182,10 @@ Module *ModuleMap::findModuleForHeader(const FileEntry *File) {
         
         for (unsigned I = SkippedDirs.size(); I != 0; --I) {
           // Find or create the module that corresponds to this directory name.
-          StringRef Name = llvm::sys::path::stem(SkippedDirs[I-1]->getName());
+          SmallString<32> NameBuf;
+          StringRef Name = sanitizeFilenameAsIdentifier(
+                             llvm::sys::path::stem(SkippedDirs[I-1]->getName()),
+                             NameBuf);
           Result = findOrCreateModule(Name, Result, /*IsFramework=*/false,
                                       Explicit).first;
           
@@ -148,9 +199,12 @@ Module *ModuleMap::findModuleForHeader(const FileEntry *File) {
         }
         
         // Infer a submodule with the same name as this header file.
-        StringRef Name = llvm::sys::path::stem(File->getName());
+        SmallString<32> NameBuf;
+        StringRef Name = sanitizeFilenameAsIdentifier(
+                           llvm::sys::path::stem(File->getName()), NameBuf);
         Result = findOrCreateModule(Name, Result, /*IsFramework=*/false,
                                     Explicit).first;
+        Result->TopHeaders.insert(File);
         
         // If inferred submodules export everything they import, add a 
         // wildcard to the set of exports.
@@ -163,7 +217,7 @@ Module *ModuleMap::findModuleForHeader(const FileEntry *File) {
           UmbrellaDirs[SkippedDirs[I]] = Result;
       }
       
-      Headers[File] = Result;
+      Headers[File] = KnownHeader(Result, /*Excluded=*/false);
 
       // If a header corresponds to an unavailable module, don't report
       // that it maps to anything.
@@ -188,10 +242,9 @@ Module *ModuleMap::findModuleForHeader(const FileEntry *File) {
 }
 
 bool ModuleMap::isHeaderInUnavailableModule(const FileEntry *Header) {
-  llvm::DenseMap<const FileEntry *, Module *>::iterator Known
-    = Headers.find(Header);
+  HeadersMap::iterator Known = Headers.find(Header);
   if (Known != Headers.end())
-    return !Known->second->isAvailable();
+    return !Known->second.isAvailable();
   
   const DirectoryEntry *Dir = Header->getDir();
   llvm::SmallVector<const DirectoryEntry *, 2> SkippedDirs;
@@ -216,7 +269,10 @@ bool ModuleMap::isHeaderInUnavailableModule(const FileEntry *Header) {
       if (UmbrellaModule->InferSubmodules) {
         for (unsigned I = SkippedDirs.size(); I != 0; --I) {
           // Find or create the module that corresponds to this directory name.
-          StringRef Name = llvm::sys::path::stem(SkippedDirs[I-1]->getName());
+          SmallString<32> NameBuf;
+          StringRef Name = sanitizeFilenameAsIdentifier(
+                             llvm::sys::path::stem(SkippedDirs[I-1]->getName()),
+                             NameBuf);
           Found = lookupModuleQualified(Name, Found);
           if (!Found)
             return false;
@@ -225,7 +281,10 @@ bool ModuleMap::isHeaderInUnavailableModule(const FileEntry *Header) {
         }
         
         // Infer a submodule with the same name as this header file.
-        StringRef Name = llvm::sys::path::stem(Header->getName());
+        SmallString<32> NameBuf;
+        StringRef Name = sanitizeFilenameAsIdentifier(
+                           llvm::sys::path::stem(Header->getName()),
+                           NameBuf);
         Found = lookupModuleQualified(Name, Found);
         if (!Found)
           return false;
@@ -287,8 +346,32 @@ ModuleMap::findOrCreateModule(StringRef Name, Module *Parent, bool IsFramework,
   return std::make_pair(Result, true);
 }
 
+bool ModuleMap::canInferFrameworkModule(const DirectoryEntry *ParentDir,
+                                        StringRef Name, bool &IsSystem) {
+  // Check whether we have already looked into the parent directory
+  // for a module map.
+  llvm::DenseMap<const DirectoryEntry *, InferredDirectory>::iterator
+    inferred = InferredDirectories.find(ParentDir);
+  if (inferred == InferredDirectories.end())
+    return false;
+
+  if (!inferred->second.InferModules)
+    return false;
+
+  // We're allowed to infer for this directory, but make sure it's okay
+  // to infer this particular module.
+  bool canInfer = std::find(inferred->second.ExcludedModules.begin(),
+                            inferred->second.ExcludedModules.end(),
+                            Name) == inferred->second.ExcludedModules.end();
+
+  if (canInfer && inferred->second.InferSystemModules)
+    IsSystem = true;
+
+  return canInfer;
+}
+
 Module *
-ModuleMap::inferFrameworkModule(StringRef ModuleName, 
+ModuleMap::inferFrameworkModule(StringRef ModuleName,
                                 const DirectoryEntry *FrameworkDir,
                                 bool IsSystem,
                                 Module *Parent) {
@@ -297,7 +380,54 @@ ModuleMap::inferFrameworkModule(StringRef ModuleName,
     return Mod;
   
   FileManager &FileMgr = SourceMgr->getFileManager();
-  
+
+  // If the framework has a parent path from which we're allowed to infer
+  // a framework module, do so.
+  if (!Parent) {
+    bool canInfer = false;
+    if (llvm::sys::path::has_parent_path(FrameworkDir->getName())) {
+      // Figure out the parent path.
+      StringRef Parent = llvm::sys::path::parent_path(FrameworkDir->getName());
+      if (const DirectoryEntry *ParentDir = FileMgr.getDirectory(Parent)) {
+        // Check whether we have already looked into the parent directory
+        // for a module map.
+        llvm::DenseMap<const DirectoryEntry *, InferredDirectory>::iterator
+          inferred = InferredDirectories.find(ParentDir);
+        if (inferred == InferredDirectories.end()) {
+          // We haven't looked here before. Load a module map, if there is
+          // one.
+          SmallString<128> ModMapPath = Parent;
+          llvm::sys::path::append(ModMapPath, "module.map");
+          if (const FileEntry *ModMapFile = FileMgr.getFile(ModMapPath)) {
+            parseModuleMapFile(ModMapFile);
+            inferred = InferredDirectories.find(ParentDir);
+          }
+
+          if (inferred == InferredDirectories.end())
+            inferred = InferredDirectories.insert(
+                         std::make_pair(ParentDir, InferredDirectory())).first;
+        }
+
+        if (inferred->second.InferModules) {
+          // We're allowed to infer for this directory, but make sure it's okay
+          // to infer this particular module.
+          StringRef Name = llvm::sys::path::filename(FrameworkDir->getName());
+          canInfer = std::find(inferred->second.ExcludedModules.begin(),
+                               inferred->second.ExcludedModules.end(),
+                               Name) == inferred->second.ExcludedModules.end();
+
+          if (inferred->second.InferSystemModules)
+            IsSystem = true;
+        }
+      }
+    }
+
+    // If we're not allowed to infer a framework module, don't.
+    if (!canInfer)
+      return 0;
+  }
+
+
   // Look for an umbrella header.
   SmallString<128> UmbrellaName = StringRef(FrameworkDir->getName());
   llvm::sys::path::append(UmbrellaName, "Headers");
@@ -320,7 +450,7 @@ ModuleMap::inferFrameworkModule(StringRef ModuleName,
   
   // umbrella header "umbrella-header-name"
   Result->Umbrella = UmbrellaHeader;
-  Headers[UmbrellaHeader] = Result;
+  Headers[UmbrellaHeader] = KnownHeader(Result, /*Excluded=*/false);
   UmbrellaDirs[UmbrellaHeader->getDir()] = Result;
   
   // export *
@@ -343,12 +473,42 @@ ModuleMap::inferFrameworkModule(StringRef ModuleName,
        Dir != DirEnd && !EC; Dir.increment(EC)) {
     if (!StringRef(Dir->path()).endswith(".framework"))
       continue;
-    
+
     if (const DirectoryEntry *SubframeworkDir
           = FileMgr.getDirectory(Dir->path())) {
+      // Note: as an egregious but useful hack, we use the real path here and
+      // check whether it is actually a subdirectory of the parent directory.
+      // This will not be the case if the 'subframework' is actually a symlink
+      // out to a top-level framework.
+#ifdef LLVM_ON_UNIX
+      char RealSubframeworkDirName[PATH_MAX];
+      if (realpath(Dir->path().c_str(), RealSubframeworkDirName)) {
+        StringRef SubframeworkDirName = RealSubframeworkDirName;
+
+        bool FoundParent = false;
+        do {
+          // Get the parent directory name.
+          SubframeworkDirName
+            = llvm::sys::path::parent_path(SubframeworkDirName);
+          if (SubframeworkDirName.empty())
+            break;
+
+          if (FileMgr.getDirectory(SubframeworkDirName) == FrameworkDir) {
+            FoundParent = true;
+            break;
+          }
+        } while (true);
+
+        if (!FoundParent)
+          continue;
+      }
+#endif
+
       // FIXME: Do we want to warn about subframeworks without umbrella headers?
-      inferFrameworkModule(llvm::sys::path::stem(Dir->path()), SubframeworkDir,
-                           IsSystem, Result);
+      SmallString<32> NameBuf;
+      inferFrameworkModule(sanitizeFilenameAsIdentifier(
+                             llvm::sys::path::stem(Dir->path()), NameBuf),
+                           SubframeworkDir, IsSystem, Result);
     }
   }
 
@@ -356,7 +516,7 @@ ModuleMap::inferFrameworkModule(StringRef ModuleName,
 }
 
 void ModuleMap::setUmbrellaHeader(Module *Mod, const FileEntry *UmbrellaHeader){
-  Headers[UmbrellaHeader] = Mod;
+  Headers[UmbrellaHeader] = KnownHeader(Mod, /*Excluded=*/false);
   Mod->Umbrella = UmbrellaHeader;
   UmbrellaDirs[UmbrellaHeader->getDir()] = Mod;
 }
@@ -366,9 +526,13 @@ void ModuleMap::setUmbrellaDir(Module *Mod, const DirectoryEntry *UmbrellaDir) {
   UmbrellaDirs[UmbrellaDir] = Mod;
 }
 
-void ModuleMap::addHeader(Module *Mod, const FileEntry *Header) {
-  Mod->Headers.push_back(Header);
-  Headers[Header] = Mod;
+void ModuleMap::addHeader(Module *Mod, const FileEntry *Header,
+                          bool Excluded) {
+  if (Excluded)
+    Mod->ExcludedHeaders.push_back(Header);
+  else
+    Mod->Headers.push_back(Header);
+  Headers[Header] = KnownHeader(Mod, Excluded);
 }
 
 const FileEntry *
@@ -388,12 +552,10 @@ void ModuleMap::dump() {
     M->getValue()->print(llvm::errs(), 2);
   
   llvm::errs() << "Headers:";
-  for (llvm::DenseMap<const FileEntry *, Module *>::iterator 
-            H = Headers.begin(),
-         HEnd = Headers.end();
+  for (HeadersMap::iterator H = Headers.begin(), HEnd = Headers.end();
        H != HEnd; ++H) {
     llvm::errs() << "  \"" << H->first->getName() << "\" -> " 
-                 << H->second->getFullModuleName() << "\n";
+                 << H->second.getModule()->getFullModuleName() << "\n";
   }
 }
 
@@ -454,6 +616,7 @@ namespace clang {
       EndOfFile,
       HeaderKeyword,
       Identifier,
+      ExcludeKeyword,
       ExplicitKeyword,
       ExportKeyword,
       FrameworkKeyword,
@@ -490,10 +653,24 @@ namespace clang {
       return StringRef(StringData, StringLength);
     }
   };
+
+  /// \brief The set of attributes that can be attached to a module.
+  struct Attributes {
+    Attributes() : IsSystem() { }
+
+    /// \brief Whether this is a system module.
+    unsigned IsSystem : 1;
+  };
   
+
   class ModuleMapParser {
     Lexer &L;
     SourceManager &SourceMgr;
+
+    /// \brief Default target information, used only for string literal
+    /// parsing.
+    const TargetInfo *Target;
+
     DiagnosticsEngine &Diags;
     ModuleMap &Map;
     
@@ -505,11 +682,7 @@ namespace clang {
 
     /// \brief Whether an error occurred.
     bool HadError;
-    
-    /// \brief Default target information, used only for string literal
-    /// parsing.
-    OwningPtr<TargetInfo> Target;
-    
+        
     /// \brief Stores string data for the various string literals referenced
     /// during parsing.
     llvm::BumpPtrAllocator StringData;
@@ -532,27 +705,25 @@ namespace clang {
     bool parseModuleId(ModuleId &Id);
     void parseModuleDecl();
     void parseRequiresDecl();
-    void parseHeaderDecl(SourceLocation UmbrellaLoc);
+    void parseHeaderDecl(SourceLocation UmbrellaLoc, SourceLocation ExcludeLoc);
     void parseUmbrellaDirDecl(SourceLocation UmbrellaLoc);
     void parseExportDecl();
-    void parseInferredSubmoduleDecl(bool Explicit);
-    
+    void parseInferredModuleDecl(bool Framework, bool Explicit);
+    bool parseOptionalAttributes(Attributes &Attrs);
+
     const DirectoryEntry *getOverriddenHeaderSearchDir();
     
   public:
     explicit ModuleMapParser(Lexer &L, SourceManager &SourceMgr, 
+                             const TargetInfo *Target,
                              DiagnosticsEngine &Diags,
                              ModuleMap &Map,
                              const DirectoryEntry *Directory,
                              const DirectoryEntry *BuiltinIncludeDir)
-      : L(L), SourceMgr(SourceMgr), Diags(Diags), Map(Map), 
+      : L(L), SourceMgr(SourceMgr), Target(Target), Diags(Diags), Map(Map), 
         Directory(Directory), BuiltinIncludeDir(BuiltinIncludeDir), 
         HadError(false), ActiveModule(0)
     {
-      TargetOptions TargetOpts;
-      TargetOpts.Triple = llvm::sys::getDefaultTargetTriple();
-      Target.reset(TargetInfo::CreateTargetInfo(Diags, TargetOpts));
-      
       Tok.clear();
       consumeToken();
     }
@@ -575,6 +746,7 @@ retry:
     Tok.StringLength = LToken.getLength();
     Tok.Kind = llvm::StringSwitch<MMToken::TokenKind>(Tok.getString())
                  .Case("header", MMToken::HeaderKeyword)
+                 .Case("exclude", MMToken::ExcludeKeyword)
                  .Case("explicit", MMToken::ExplicitKeyword)
                  .Case("export", MMToken::ExportKeyword)
                  .Case("framework", MMToken::FrameworkKeyword)
@@ -743,13 +915,6 @@ namespace {
 ///     'explicit'[opt] 'framework'[opt] 'module' module-id attributes[opt] 
 ///       { module-member* }
 ///
-///   attributes:
-///     attribute attributes
-///     attribute
-///
-///   attribute:
-///     [ identifier ]
-///
 ///   module-member:
 ///     requires-declaration
 ///     header-declaration
@@ -791,7 +956,7 @@ void ModuleMapParser::parseModuleDecl() {
   // If we have a wildcard for the module name, this is an inferred submodule.
   // Parse it. 
   if (Tok.is(MMToken::Star))
-    return parseInferredSubmoduleDecl(Explicit);
+    return parseInferredModuleDecl(Framework, Explicit);
   
   // Parse the module name.
   ModuleId Id;
@@ -799,7 +964,7 @@ void ModuleMapParser::parseModuleDecl() {
     HadError = true;
     return;
   }
-  
+
   if (ActiveModule) {
     if (Id.size() > 1) {
       Diags.Report(Id.front().second, diag::err_mmap_nested_submodule_id)
@@ -842,47 +1007,8 @@ void ModuleMapParser::parseModuleDecl() {
   SourceLocation ModuleNameLoc = Id.back().second;
   
   // Parse the optional attribute list.
-  bool IsSystem = false;
-  while (Tok.is(MMToken::LSquare)) {
-    // Consume the '['.
-    SourceLocation LSquareLoc = consumeToken();
-    
-    // Check whether we have an attribute name here.
-    if (!Tok.is(MMToken::Identifier)) {
-      Diags.Report(Tok.getLocation(), diag::err_mmap_expected_attribute);
-      skipUntil(MMToken::RSquare);
-      if (Tok.is(MMToken::RSquare))
-        consumeToken();
-      continue;
-    }
-    
-    // Decode the attribute name.
-    AttributeKind Attribute 
-      = llvm::StringSwitch<AttributeKind>(Tok.getString())
-        .Case("system", AT_system)
-        .Default(AT_unknown);
-    switch (Attribute) {
-    case AT_unknown:
-      Diags.Report(Tok.getLocation(), diag::warn_mmap_unknown_attribute)
-        << Tok.getString();
-      break;
-        
-    case AT_system:
-      IsSystem = true;
-      break;
-    }
-    consumeToken();
-    
-    // Consume the ']'.
-    if (!Tok.is(MMToken::RSquare)) {
-      Diags.Report(Tok.getLocation(), diag::err_mmap_expected_rsquare);
-      Diags.Report(LSquareLoc, diag::note_mmap_lsquare_match);
-      skipUntil(MMToken::RSquare);
-    }
-
-    if (Tok.is(MMToken::RSquare))
-      consumeToken();
-  }
+  Attributes Attrs;
+  parseOptionalAttributes(Attrs);
   
   // Parse the opening brace.
   if (!Tok.is(MMToken::LBrace)) {
@@ -925,7 +1051,7 @@ void ModuleMapParser::parseModuleDecl() {
   ActiveModule = Map.findOrCreateModule(ModuleName, ActiveModule, Framework,
                                         Explicit).first;
   ActiveModule->DefinitionLoc = ModuleNameLoc;
-  if (IsSystem)
+  if (Attrs.IsSystem)
     ActiveModule->IsSystem = true;
   
   bool Done = false;
@@ -953,14 +1079,25 @@ void ModuleMapParser::parseModuleDecl() {
     case MMToken::UmbrellaKeyword: {
       SourceLocation UmbrellaLoc = consumeToken();
       if (Tok.is(MMToken::HeaderKeyword))
-        parseHeaderDecl(UmbrellaLoc);
+        parseHeaderDecl(UmbrellaLoc, SourceLocation());
       else
         parseUmbrellaDirDecl(UmbrellaLoc);
       break;
     }
         
+    case MMToken::ExcludeKeyword: {
+      SourceLocation ExcludeLoc = consumeToken();
+      if (Tok.is(MMToken::HeaderKeyword)) {
+        parseHeaderDecl(SourceLocation(), ExcludeLoc);
+      } else {
+        Diags.Report(Tok.getLocation(), diag::err_mmap_expected_header)
+          << "exclude";
+      }
+      break;
+    }
+      
     case MMToken::HeaderKeyword:
-      parseHeaderDecl(SourceLocation());
+      parseHeaderDecl(SourceLocation(), SourceLocation());
       break;
         
     default:
@@ -1062,12 +1199,15 @@ static bool isBuiltinHeader(StringRef FileName) {
 ///
 ///   header-declaration:
 ///     'umbrella'[opt] 'header' string-literal
-void ModuleMapParser::parseHeaderDecl(SourceLocation UmbrellaLoc) {
+///     'exclude'[opt] 'header' string-literal
+void ModuleMapParser::parseHeaderDecl(SourceLocation UmbrellaLoc,
+                                      SourceLocation ExcludeLoc) {
   assert(Tok.is(MMToken::HeaderKeyword));
   consumeToken();
 
   bool Umbrella = UmbrellaLoc.isValid();
-  
+  bool Exclude = ExcludeLoc.isValid();
+  assert(!(Umbrella && Exclude) && "Cannot have both 'umbrella' and 'exclude'");
   // Parse the header name.
   if (!Tok.is(MMToken::StringLiteral)) {
     Diags.Report(Tok.getLocation(), diag::err_mmap_expected_header) 
@@ -1145,15 +1285,15 @@ void ModuleMapParser::parseHeaderDecl(SourceLocation UmbrellaLoc) {
   // FIXME: We shouldn't be eagerly stat'ing every file named in a module map.
   // Come up with a lazy way to do this.
   if (File) {
-    if (const Module *OwningModule = Map.Headers[File]) {
+    if (ModuleMap::KnownHeader OwningModule = Map.Headers[File]) {
       Diags.Report(FileNameLoc, diag::err_mmap_header_conflict)
-        << FileName << OwningModule->getFullModuleName();
+        << FileName << OwningModule.getModule()->getFullModuleName();
       HadError = true;
     } else if (Umbrella) {
       const DirectoryEntry *UmbrellaDir = File->getDir();
-      if ((OwningModule = Map.UmbrellaDirs[UmbrellaDir])) {
+      if (Module *UmbrellaModule = Map.UmbrellaDirs[UmbrellaDir]) {
         Diags.Report(UmbrellaLoc, diag::err_mmap_umbrella_clash)
-          << OwningModule->getFullModuleName();
+          << UmbrellaModule->getFullModuleName();
         HadError = true;
       } else {
         // Record this umbrella header.
@@ -1161,11 +1301,11 @@ void ModuleMapParser::parseHeaderDecl(SourceLocation UmbrellaLoc) {
       }
     } else {
       // Record this header.
-      Map.addHeader(ActiveModule, File);
+      Map.addHeader(ActiveModule, File, Exclude);
       
       // If there is a builtin counterpart to this file, add it now.
       if (BuiltinFile)
-        Map.addHeader(ActiveModule, BuiltinFile);
+        Map.addHeader(ActiveModule, BuiltinFile, Exclude);
     }
   } else {
     Diags.Report(FileNameLoc, diag::err_mmap_header_not_found)
@@ -1274,32 +1414,52 @@ void ModuleMapParser::parseExportDecl() {
   ActiveModule->UnresolvedExports.push_back(Unresolved);
 }
 
-void ModuleMapParser::parseInferredSubmoduleDecl(bool Explicit) {
+/// \brief Parse an inferried module declaration (wildcard modules).
+///
+///   module-declaration:
+///     'explicit'[opt] 'framework'[opt] 'module' * attributes[opt]
+///       { inferred-module-member* }
+///
+///   inferred-module-member:
+///     'export' '*'
+///     'exclude' identifier
+void ModuleMapParser::parseInferredModuleDecl(bool Framework, bool Explicit) {
   assert(Tok.is(MMToken::Star));
   SourceLocation StarLoc = consumeToken();
   bool Failed = false;
-  
+
   // Inferred modules must be submodules.
-  if (!ActiveModule) {
+  if (!ActiveModule && !Framework) {
     Diags.Report(StarLoc, diag::err_mmap_top_level_inferred_submodule);
     Failed = true;
   }
-  
-  // Inferred modules must have umbrella directories.
-  if (!Failed && !ActiveModule->getUmbrellaDir()) {
-    Diags.Report(StarLoc, diag::err_mmap_inferred_no_umbrella);
-    Failed = true;
-  }
-  
-  // Check for redefinition of an inferred module.
-  if (!Failed && ActiveModule->InferSubmodules) {
-    Diags.Report(StarLoc, diag::err_mmap_inferred_redef);
-    if (ActiveModule->InferredSubmoduleLoc.isValid())
-      Diags.Report(ActiveModule->InferredSubmoduleLoc,
-                   diag::note_mmap_prev_definition);
-    Failed = true;
+
+  if (ActiveModule) {
+    // Inferred modules must have umbrella directories.
+    if (!Failed && !ActiveModule->getUmbrellaDir()) {
+      Diags.Report(StarLoc, diag::err_mmap_inferred_no_umbrella);
+      Failed = true;
+    }
+    
+    // Check for redefinition of an inferred module.
+    if (!Failed && ActiveModule->InferSubmodules) {
+      Diags.Report(StarLoc, diag::err_mmap_inferred_redef);
+      if (ActiveModule->InferredSubmoduleLoc.isValid())
+        Diags.Report(ActiveModule->InferredSubmoduleLoc,
+                     diag::note_mmap_prev_definition);
+      Failed = true;
+    }
+
+    // Check for the 'framework' keyword, which is not permitted here.
+    if (Framework) {
+      Diags.Report(StarLoc, diag::err_mmap_inferred_framework_submodule);
+      Framework = false;
+    }
+  } else if (Explicit) {
+    Diags.Report(StarLoc, diag::err_mmap_explicit_inferred_framework);
+    Explicit = false;
   }
-  
+
   // If there were any problems with this inferred submodule, skip its body.
   if (Failed) {
     if (Tok.is(MMToken::LBrace)) {
@@ -1311,12 +1471,22 @@ void ModuleMapParser::parseInferredSubmoduleDecl(bool Explicit) {
     HadError = true;
     return;
   }
-  
-  // Note that we have an inferred submodule.
-  ActiveModule->InferSubmodules = true;
-  ActiveModule->InferredSubmoduleLoc = StarLoc;
-  ActiveModule->InferExplicitSubmodules = Explicit;
-  
+
+  // Parse optional attributes.
+  Attributes Attrs;
+  parseOptionalAttributes(Attrs);
+
+  if (ActiveModule) {
+    // Note that we have an inferred submodule.
+    ActiveModule->InferSubmodules = true;
+    ActiveModule->InferredSubmoduleLoc = StarLoc;
+    ActiveModule->InferExplicitSubmodules = Explicit;
+  } else {
+    // We'll be inferring framework modules for this directory.
+    Map.InferredDirectories[Directory].InferModules = true;
+    Map.InferredDirectories[Directory].InferSystemModules = Attrs.IsSystem;
+  }
+
   // Parse the opening brace.
   if (!Tok.is(MMToken::LBrace)) {
     Diags.Report(Tok.getLocation(), diag::err_mmap_expected_lbrace_wildcard);
@@ -1333,8 +1503,35 @@ void ModuleMapParser::parseInferredSubmoduleDecl(bool Explicit) {
     case MMToken::RBrace:
       Done = true;
       break;
-      
-    case MMToken::ExportKeyword: {
+
+    case MMToken::ExcludeKeyword: {
+      if (ActiveModule) {
+        Diags.Report(Tok.getLocation(), diag::err_mmap_expected_inferred_member)
+          << (ActiveModule != 0);
+        consumeToken();
+        break;
+      }
+
+      consumeToken();
+      if (!Tok.is(MMToken::Identifier)) {
+        Diags.Report(Tok.getLocation(), diag::err_mmap_missing_exclude_name);
+        break;
+      }
+
+      Map.InferredDirectories[Directory].ExcludedModules
+        .push_back(Tok.getString());
+      consumeToken();
+      break;
+    }
+
+    case MMToken::ExportKeyword:
+      if (!ActiveModule) {
+        Diags.Report(Tok.getLocation(), diag::err_mmap_expected_inferred_member)
+          << (ActiveModule != 0);
+        consumeToken();
+        break;
+      }
+
       consumeToken();
       if (Tok.is(MMToken::Star)) 
         ActiveModule->InferExportWildcard = true;
@@ -1343,14 +1540,14 @@ void ModuleMapParser::parseInferredSubmoduleDecl(bool Explicit) {
                      diag::err_mmap_expected_export_wildcard);
       consumeToken();
       break;
-    }
-      
+
     case MMToken::ExplicitKeyword:
     case MMToken::ModuleKeyword:
     case MMToken::HeaderKeyword:
     case MMToken::UmbrellaKeyword:
     default:
-      Diags.Report(Tok.getLocation(), diag::err_mmap_expected_wildcard_member);
+      Diags.Report(Tok.getLocation(), diag::err_mmap_expected_inferred_member)
+          << (ActiveModule != 0);
       consumeToken();
       break;        
     }
@@ -1365,6 +1562,66 @@ void ModuleMapParser::parseInferredSubmoduleDecl(bool Explicit) {
   }
 }
 
+/// \brief Parse optional attributes.
+///
+///   attributes:
+///     attribute attributes
+///     attribute
+///
+///   attribute:
+///     [ identifier ]
+///
+/// \param Attrs Will be filled in with the parsed attributes.
+///
+/// \returns true if an error occurred, false otherwise.
+bool ModuleMapParser::parseOptionalAttributes(Attributes &Attrs) {
+  bool HadError = false;
+  
+  while (Tok.is(MMToken::LSquare)) {
+    // Consume the '['.
+    SourceLocation LSquareLoc = consumeToken();
+
+    // Check whether we have an attribute name here.
+    if (!Tok.is(MMToken::Identifier)) {
+      Diags.Report(Tok.getLocation(), diag::err_mmap_expected_attribute);
+      skipUntil(MMToken::RSquare);
+      if (Tok.is(MMToken::RSquare))
+        consumeToken();
+      HadError = true;
+    }
+
+    // Decode the attribute name.
+    AttributeKind Attribute
+      = llvm::StringSwitch<AttributeKind>(Tok.getString())
+          .Case("system", AT_system)
+          .Default(AT_unknown);
+    switch (Attribute) {
+    case AT_unknown:
+      Diags.Report(Tok.getLocation(), diag::warn_mmap_unknown_attribute)
+        << Tok.getString();
+      break;
+
+    case AT_system:
+      Attrs.IsSystem = true;
+      break;
+    }
+    consumeToken();
+
+    // Consume the ']'.
+    if (!Tok.is(MMToken::RSquare)) {
+      Diags.Report(Tok.getLocation(), diag::err_mmap_expected_rsquare);
+      Diags.Report(LSquareLoc, diag::note_mmap_lsquare_match);
+      skipUntil(MMToken::RSquare);
+      HadError = true;
+    }
+
+    if (Tok.is(MMToken::RSquare))
+      consumeToken();
+  }
+
+  return HadError;
+}
+
 /// \brief If there is a specific header search directory due the presence
 /// of an umbrella directory, retrieve that directory. Otherwise, returns null.
 const DirectoryEntry *ModuleMapParser::getOverriddenHeaderSearchDir() {
@@ -1398,6 +1655,7 @@ bool ModuleMapParser::parseModuleMapFile() {
       break;
       
     case MMToken::Comma:
+    case MMToken::ExcludeKeyword:
     case MMToken::ExportKeyword:
     case MMToken::HeaderKeyword:
     case MMToken::Identifier:
@@ -1428,7 +1686,7 @@ bool ModuleMap::parseModuleMapFile(const FileEntry *File) {
   // Parse this module map file.
   Lexer L(ID, SourceMgr->getBuffer(ID), *SourceMgr, MMapLangOpts);
   Diags->getClient()->BeginSourceFile(MMapLangOpts);
-  ModuleMapParser Parser(L, *SourceMgr, *Diags, *this, File->getDir(),
+  ModuleMapParser Parser(L, *SourceMgr, Target, *Diags, *this, File->getDir(),
                          BuiltinIncludeDir);
   bool Result = Parser.parseModuleMapFile();
   Diags->getClient()->EndSourceFile();
diff --git a/lib/Lex/PPDirectives.cpp b/lib/Lex/PPDirectives.cpp
index 74b9cbc..b7c1846 100644
--- a/lib/Lex/PPDirectives.cpp
+++ b/lib/Lex/PPDirectives.cpp
@@ -1296,7 +1296,7 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc,
   case tok::string_literal:
     Filename = getSpelling(FilenameTok, FilenameBuffer);
     End = FilenameTok.getLocation();
-    CharEnd = End.getLocWithOffset(Filename.size());
+    CharEnd = End.getLocWithOffset(FilenameTok.getLength());
     break;
 
   case tok::less:
@@ -1306,7 +1306,7 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc,
     if (ConcatenateIncludeName(FilenameBuffer, End))
       return;   // Found <eod> but no ">"?  Diagnostic already emitted.
     Filename = FilenameBuffer.str();
-    CharEnd = getLocForEndOfToken(End);
+    CharEnd = End.getLocWithOffset(1);
     break;
   default:
     Diag(FilenameTok.getLocation(), diag::err_pp_expects_filename);
@@ -1314,6 +1314,8 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc,
     return;
   }
 
+  CharSourceRange FilenameRange
+    = CharSourceRange::getCharRange(FilenameTok.getLocation(), CharEnd);
   StringRef OriginalFilename = Filename;
   bool isAngled =
     GetIncludeFilenameSpelling(FilenameTok.getLocation(), Filename);
@@ -1384,9 +1386,13 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc,
       }
     }
     
-    // Notify the callback object that we've seen an inclusion directive.
-    Callbacks->InclusionDirective(HashLoc, IncludeTok, Filename, isAngled, File,
-                                  End, SearchPath, RelativePath);
+    if (!SuggestedModule) {
+      // Notify the callback object that we've seen an inclusion directive.
+      Callbacks->InclusionDirective(HashLoc, IncludeTok, Filename, isAngled,
+                                    FilenameRange, File,
+                                    SearchPath, RelativePath,
+                                    /*ImportedModule=*/0);
+    }
   }
   
   if (File == 0) {
@@ -1480,10 +1486,28 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc,
     Module *Imported
       = TheModuleLoader.loadModule(IncludeTok.getLocation(), Path, Visibility,
                                    /*IsIncludeDirective=*/true);
+    assert((Imported == 0 || Imported == SuggestedModule) &&
+           "the imported module is different than the suggested one");
     
     // If this header isn't part of the module we're building, we're done.
-    if (!BuildingImportedModule && Imported)
+    if (!BuildingImportedModule && Imported) {
+      if (Callbacks) {
+        Callbacks->InclusionDirective(HashLoc, IncludeTok, Filename, isAngled,
+                                      FilenameRange, File,
+                                      SearchPath, RelativePath, Imported);
+      }
       return;
+    }
+  }
+
+  if (Callbacks && SuggestedModule) {
+    // We didn't notify the callback object that we've seen an inclusion
+    // directive before. Now that we are parsing the include normally and not
+    // turning it to a module import, notify the callback object.
+    Callbacks->InclusionDirective(HashLoc, IncludeTok, Filename, isAngled,
+                                  FilenameRange, File,
+                                  SearchPath, RelativePath,
+                                  /*ImportedModule=*/0);
   }
   
   // The #included file will be considered to be a system header if either it is
@@ -1849,7 +1873,7 @@ void Preprocessor::HandleDefineDirective(Token &DefineTok) {
   MI->setDefinitionEndLoc(LastTok.getLocation());
 
   // Finally, if this identifier already had a macro defined for it, verify that
-  // the macro bodies are identical and free the old definition.
+  // the macro bodies are identical, and issue diagnostics if they are not.
   if (MacroInfo *OtherMI = getMacroInfo(MacroNameTok.getIdentifierInfo())) {
     // It is very common for system headers to have tons of macro redefinitions
     // and for warnings to be disabled in system headers.  If this is the case,
@@ -1870,7 +1894,6 @@ void Preprocessor::HandleDefineDirective(Token &DefineTok) {
     }
     if (OtherMI->isWarnIfUnused())
       WarnUnusedMacroLocs.erase(OtherMI->getDefinitionLoc());
-    ReleaseMacroInfo(OtherMI);
   }
 
   setMacroInfo(MacroNameTok.getIdentifierInfo(), MI);
@@ -1921,9 +1944,20 @@ void Preprocessor::HandleUndefDirective(Token &UndefTok) {
   if (MI->isWarnIfUnused())
     WarnUnusedMacroLocs.erase(MI->getDefinitionLoc());
 
-  // Free macro definition.
-  ReleaseMacroInfo(MI);
-  setMacroInfo(MacroNameTok.getIdentifierInfo(), 0);
+  UndefineMacro(MacroNameTok.getIdentifierInfo(), MI,
+                MacroNameTok.getLocation());
+}
+
+void Preprocessor::UndefineMacro(IdentifierInfo *II, MacroInfo *MI,
+                                 SourceLocation UndefLoc) {
+  MI->setUndefLoc(UndefLoc);
+  if (MI->isFromAST()) {
+    MI->setChangedAfterLoad();
+    if (Listener)
+      Listener->UndefinedMacro(MI);
+  }
+
+  clearMacroInfo(II);
 }
 
 
diff --git a/lib/Lex/PPExpressions.cpp b/lib/Lex/PPExpressions.cpp
index 7cac63e..d5a88db 100644
--- a/lib/Lex/PPExpressions.cpp
+++ b/lib/Lex/PPExpressions.cpp
@@ -178,7 +178,9 @@ static bool EvaluateValue(PPValue &Result, Token &PeekTok, DefinedTracker &DT,
     // preprocessor keywords and it wasn't macro expanded, it turns
     // into a simple 0, unless it is the C++ keyword "true", in which case it
     // turns into "1".
-    if (ValueLive)
+    if (ValueLive &&
+        II->getTokenID() != tok::kw_true &&
+        II->getTokenID() != tok::kw_false)
       PP.Diag(PeekTok, diag::warn_pp_undef_identifier) << II;
     Result.Val = II->getTokenID() == tok::kw_true;
     Result.Val.setIsUnsigned(false);  // "0" is signed intmax_t 0.
@@ -204,8 +206,7 @@ static bool EvaluateValue(PPValue &Result, Token &PeekTok, DefinedTracker &DT,
     if (NumberInvalid)
       return true; // a diagnostic was already reported
 
-    NumericLiteralParser Literal(Spelling.begin(), Spelling.end(),
-                                 PeekTok.getLocation(), PP);
+    NumericLiteralParser Literal(Spelling, PeekTok.getLocation(), PP);
     if (Literal.hadError)
       return true; // a diagnostic was already reported.
 
@@ -219,10 +220,15 @@ static bool EvaluateValue(PPValue &Result, Token &PeekTok, DefinedTracker &DT,
     if (Literal.hasUDSuffix())
       PP.Diag(PeekTok, diag::err_pp_invalid_udl) << /*integer*/1;
 
-    // long long is a C99 feature.
-    if (!PP.getLangOpts().C99 && Literal.isLongLong)
-      PP.Diag(PeekTok, PP.getLangOpts().CPlusPlus0x ?
-              diag::warn_cxx98_compat_longlong : diag::ext_longlong);
+    // 'long long' is a C99 or C++11 feature.
+    if (!PP.getLangOpts().C99 && Literal.isLongLong) {
+      if (PP.getLangOpts().CPlusPlus)
+        PP.Diag(PeekTok,
+             PP.getLangOpts().CPlusPlus0x ?
+             diag::warn_cxx98_compat_longlong : diag::ext_cxx11_longlong);
+      else
+        PP.Diag(PeekTok, diag::ext_c99_longlong);
+    }
 
     // Parse the integer literal into Result.
     if (Literal.GetIntegerValue(Result.Val)) {
diff --git a/lib/Lex/PPLexerChange.cpp b/lib/Lex/PPLexerChange.cpp
index e824320..d827f58 100644
--- a/lib/Lex/PPLexerChange.cpp
+++ b/lib/Lex/PPLexerChange.cpp
@@ -157,15 +157,15 @@ void Preprocessor::EnterSourceFileWithPTH(PTHLexer *PL,
 /// EnterMacro - Add a Macro to the top of the include stack and start lexing
 /// tokens from it instead of the current buffer.
 void Preprocessor::EnterMacro(Token &Tok, SourceLocation ILEnd,
-                              MacroArgs *Args) {
+                              MacroInfo *Macro, MacroArgs *Args) {
   PushIncludeMacroStack();
   CurDirLookup = 0;
 
   if (NumCachedTokenLexers == 0) {
-    CurTokenLexer.reset(new TokenLexer(Tok, ILEnd, Args, *this));
+    CurTokenLexer.reset(new TokenLexer(Tok, ILEnd, Macro, Args, *this));
   } else {
     CurTokenLexer.reset(TokenLexerCache[--NumCachedTokenLexers]);
-    CurTokenLexer->Init(Tok, ILEnd, Args);
+    CurTokenLexer->Init(Tok, ILEnd, Macro, Args);
   }
   if (CurLexerKind != CLK_LexAfterModuleImport)
     CurLexerKind = CLK_TokenLexer;
diff --git a/lib/Lex/PPMacroExpansion.cpp b/lib/Lex/PPMacroExpansion.cpp
index ebdb644..eee4342 100644
--- a/lib/Lex/PPMacroExpansion.cpp
+++ b/lib/Lex/PPMacroExpansion.cpp
@@ -27,39 +27,138 @@
 #include "llvm/Config/llvm-config.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/Format.h"
 #include <cstdio>
 #include <ctime>
 using namespace clang;
 
-MacroInfo *Preprocessor::getInfoForMacro(IdentifierInfo *II) const {
-  assert(II->hasMacroDefinition() && "Identifier is not a macro!");
-  
-  llvm::DenseMap<IdentifierInfo*, MacroInfo*>::const_iterator Pos
-    = Macros.find(II);
-  if (Pos == Macros.end()) {
-    // Load this macro from the external source.
-    getExternalSource()->LoadMacroDefinition(II);
-    Pos = Macros.find(II);
-  }
+MacroInfo *Preprocessor::getMacroInfoHistory(IdentifierInfo *II) const {
+  assert(II->hadMacroDefinition() && "Identifier has not been not a macro!");
+
+  macro_iterator Pos = Macros.find(II);
   assert(Pos != Macros.end() && "Identifier macro info is missing!");
   return Pos->second;
 }
 
 /// setMacroInfo - Specify a macro for this identifier.
 ///
-void Preprocessor::setMacroInfo(IdentifierInfo *II, MacroInfo *MI,
-                                bool LoadedFromAST) {
-  if (MI) {
-    Macros[II] = MI;
-    II->setHasMacroDefinition(true);
-    if (II->isFromAST() && !LoadedFromAST)
-      II->setChangedSinceDeserialization();
-  } else if (II->hasMacroDefinition()) {
-    Macros.erase(II);
-    II->setHasMacroDefinition(false);
-    if (II->isFromAST() && !LoadedFromAST)
-      II->setChangedSinceDeserialization();
+void Preprocessor::setMacroInfo(IdentifierInfo *II, MacroInfo *MI) {
+  assert(MI && "MacroInfo should be non-zero!");
+  assert(MI->getUndefLoc().isInvalid() &&
+         "Undefined macros cannot be registered");
+
+  MacroInfo *&StoredMI = Macros[II];
+  MI->setPreviousDefinition(StoredMI);
+  StoredMI = MI;
+  II->setHasMacroDefinition(MI->getUndefLoc().isInvalid());
+  if (II->isFromAST())
+    II->setChangedSinceDeserialization();
+}
+
+void Preprocessor::addLoadedMacroInfo(IdentifierInfo *II, MacroInfo *MI,
+                                      MacroInfo *Hint) {
+  assert(MI && "Missing macro?");
+  assert(MI->isFromAST() && "Macro is not from an AST?");
+  assert(!MI->getPreviousDefinition() && "Macro already in chain?");
+  
+  MacroInfo *&StoredMI = Macros[II];
+
+  // Easy case: this is the first macro definition for this macro.
+  if (!StoredMI) {
+    StoredMI = MI;
+
+    if (MI->isDefined())
+      II->setHasMacroDefinition(true);
+    return;
+  }
+
+  // If this macro is a definition and this identifier has been neither
+  // defined nor undef'd in the current translation unit, add this macro
+  // to the end of the chain of definitions.
+  if (MI->isDefined() && StoredMI->isFromAST()) {
+    // Simple case: if this is the first actual definition, just put it at
+    // th beginning.
+    if (!StoredMI->isDefined()) {
+      MI->setPreviousDefinition(StoredMI);
+      StoredMI = MI;
+
+      II->setHasMacroDefinition(true);
+      return;
+    }
+
+    // Find the end of the definition chain.
+    MacroInfo *Prev;
+    MacroInfo *PrevPrev = StoredMI;
+    bool Ambiguous = StoredMI->isAmbiguous();
+    bool MatchedOther = false;
+    do {
+      Prev = PrevPrev;
+
+      // If the macros are not identical, we have an ambiguity.
+      if (!Prev->isIdenticalTo(*MI, *this)) {
+        if (!Ambiguous) {
+          Ambiguous = true;
+          StoredMI->setAmbiguous(true);
+        }
+      } else {
+        MatchedOther = true;
+      }
+    } while ((PrevPrev = Prev->getPreviousDefinition()) &&
+             PrevPrev->isDefined());
+
+    // If there are ambiguous definitions, and we didn't match any other
+    // definition, then mark us as ambiguous.
+    if (Ambiguous && !MatchedOther)
+      MI->setAmbiguous(true);
+
+    // Wire this macro information into the chain.
+    MI->setPreviousDefinition(Prev->getPreviousDefinition());
+    Prev->setPreviousDefinition(MI);
+    return;
+  }
+
+  // The macro is not a definition; put it at the end of the list.
+  MacroInfo *Prev = Hint? Hint : StoredMI;
+  while (Prev->getPreviousDefinition())
+    Prev = Prev->getPreviousDefinition();
+  Prev->setPreviousDefinition(MI);
+}
+
+void Preprocessor::makeLoadedMacroInfoVisible(IdentifierInfo *II,
+                                              MacroInfo *MI) {
+  assert(MI->isFromAST() && "Macro must be from the AST");
+
+  MacroInfo *&StoredMI = Macros[II];
+  if (StoredMI == MI) {
+    // Easy case: this is the first macro anyway.
+    II->setHasMacroDefinition(MI->isDefined());
+    return;
   }
+
+  // Go find the macro and pull it out of the list.
+  // FIXME: Yes, this is O(N), and making a pile of macros visible or hidden
+  // would be quadratic, but it's extremely rare.
+  MacroInfo *Prev = StoredMI;
+  while (Prev->getPreviousDefinition() != MI)
+    Prev = Prev->getPreviousDefinition();
+  Prev->setPreviousDefinition(MI->getPreviousDefinition());
+  MI->setPreviousDefinition(0);
+
+  // Add the macro back to the list.
+  addLoadedMacroInfo(II, MI);
+
+  II->setHasMacroDefinition(StoredMI->isDefined());
+  if (II->isFromAST())
+    II->setChangedSinceDeserialization();
+}
+
+/// \brief Undefine a macro for this identifier.
+void Preprocessor::clearMacroInfo(IdentifierInfo *II) {
+  assert(II->hasMacroDefinition() && "Macro is not defined!");
+  assert(Macros[II]->getUndefLoc().isValid() && "Macro is still defined!");
+  II->setHasMacroDefinition(false);
+  if (II->isFromAST())
+    II->setChangedSinceDeserialization();
 }
 
 /// RegisterBuiltinMacro - Register the specified identifier in the identifier
@@ -100,6 +199,20 @@ void Preprocessor::RegisterBuiltinMacros() {
   Ident__has_include_next = RegisterBuiltinMacro(*this, "__has_include_next");
   Ident__has_warning      = RegisterBuiltinMacro(*this, "__has_warning");
 
+  // Modules.
+  if (LangOpts.Modules) {
+    Ident__building_module  = RegisterBuiltinMacro(*this, "__building_module");
+
+    // __MODULE__
+    if (!LangOpts.CurrentModule.empty())
+      Ident__MODULE__ = RegisterBuiltinMacro(*this, "__MODULE__");
+    else
+      Ident__MODULE__ = 0;
+  } else {
+    Ident__building_module = 0;
+    Ident__MODULE__ = 0;
+  }
+  
   // Microsoft Extensions.
   if (LangOpts.MicrosoftExt) 
     Ident__pragma = RegisterBuiltinMacro(*this, "__pragma");
@@ -263,7 +376,23 @@ bool Preprocessor::HandleMacroExpandedIdentifier(Token &Identifier,
       }
     }
   }
-  
+
+  // If the macro definition is ambiguous, complain.
+  if (MI->isAmbiguous()) {
+    Diag(Identifier, diag::warn_pp_ambiguous_macro)
+      << Identifier.getIdentifierInfo();
+    Diag(MI->getDefinitionLoc(), diag::note_pp_ambiguous_macro_chosen)
+      << Identifier.getIdentifierInfo();
+    for (MacroInfo *PrevMI = MI->getPreviousDefinition();
+         PrevMI && PrevMI->isDefined();
+         PrevMI = PrevMI->getPreviousDefinition()) {
+      if (PrevMI->isAmbiguous()) {
+        Diag(PrevMI->getDefinitionLoc(), diag::note_pp_ambiguous_macro_other)
+          << Identifier.getIdentifierInfo();
+      }
+    }
+  }
+
   // If we started lexing a macro, enter the macro expansion body.
 
   // If this macro expands to no tokens, don't bother to push it onto the
@@ -337,7 +466,7 @@ bool Preprocessor::HandleMacroExpandedIdentifier(Token &Identifier,
   }
 
   // Start expanding the macro.
-  EnterMacro(Identifier, ExpansionEnd, Args);
+  EnterMacro(Identifier, ExpansionEnd, MI, Args);
 
   // Now that the macro is at the top of the include stack, ask the
   // preprocessor to read the next token from it.
@@ -581,27 +710,27 @@ static void ComputeDATE_TIME(SourceLocation &DATELoc, SourceLocation &TIMELoc,
     "Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"
   };
 
-  char TmpBuffer[32];
-#ifdef LLVM_ON_WIN32
-  sprintf(TmpBuffer, "\"%s %2d %4d\"", Months[TM->tm_mon], TM->tm_mday,
-          TM->tm_year+1900);
-#else
-  snprintf(TmpBuffer, sizeof(TmpBuffer), "\"%s %2d %4d\"", Months[TM->tm_mon], TM->tm_mday,
-          TM->tm_year+1900);
-#endif
-
-  Token TmpTok;
-  TmpTok.startToken();
-  PP.CreateString(TmpBuffer, strlen(TmpBuffer), TmpTok);
-  DATELoc = TmpTok.getLocation();
-
-#ifdef LLVM_ON_WIN32
-  sprintf(TmpBuffer, "\"%02d:%02d:%02d\"", TM->tm_hour, TM->tm_min, TM->tm_sec);
-#else
-  snprintf(TmpBuffer, sizeof(TmpBuffer), "\"%02d:%02d:%02d\"", TM->tm_hour, TM->tm_min, TM->tm_sec);
-#endif
-  PP.CreateString(TmpBuffer, strlen(TmpBuffer), TmpTok);
-  TIMELoc = TmpTok.getLocation();
+  {
+    SmallString<32> TmpBuffer;
+    llvm::raw_svector_ostream TmpStream(TmpBuffer);
+    TmpStream << llvm::format("\"%s %2d %4d\"", Months[TM->tm_mon],
+                              TM->tm_mday, TM->tm_year + 1900);
+    Token TmpTok;
+    TmpTok.startToken();
+    PP.CreateString(TmpStream.str(), TmpTok);
+    DATELoc = TmpTok.getLocation();
+  }
+
+  {
+    SmallString<32> TmpBuffer;
+    llvm::raw_svector_ostream TmpStream(TmpBuffer);
+    TmpStream << llvm::format("\"%02d:%02d:%02d\"",
+                              TM->tm_hour, TM->tm_min, TM->tm_sec);
+    Token TmpTok;
+    TmpTok.startToken();
+    PP.CreateString(TmpStream.str(), TmpTok);
+    TIMELoc = TmpTok.getLocation();
+  }
 }
 
 
@@ -616,7 +745,7 @@ static bool HasFeature(const Preprocessor &PP, const IdentifierInfo *II) {
     Feature = Feature.substr(2, Feature.size() - 4);
 
   return llvm::StringSwitch<bool>(Feature)
-           .Case("address_sanitizer", LangOpts.AddressSanitizer)
+           .Case("address_sanitizer", LangOpts.SanitizeAddress)
            .Case("attribute_analyzer_noreturn", true)
            .Case("attribute_availability", true)
            .Case("attribute_availability_with_message", true)
@@ -641,8 +770,7 @@ static bool HasFeature(const Preprocessor &PP, const IdentifierInfo *II) {
            // Objective-C features
            .Case("objc_arr", LangOpts.ObjCAutoRefCount) // FIXME: REMOVE?
            .Case("objc_arc", LangOpts.ObjCAutoRefCount)
-           .Case("objc_arc_weak", LangOpts.ObjCAutoRefCount && 
-                 LangOpts.ObjCRuntimeHasWeak)
+           .Case("objc_arc_weak", LangOpts.ObjCARCWeak)
            .Case("objc_default_synthesize_properties", LangOpts.ObjC2)
            .Case("objc_fixed_enum", LangOpts.ObjC2)
            .Case("objc_instancetype", LangOpts.ObjC2)
@@ -716,22 +844,12 @@ static bool HasFeature(const Preprocessor &PP, const IdentifierInfo *II) {
             // "struct __is_empty" parsing hack hasn't been needed in this
             // translation unit. If it has, __is_empty reverts to a normal
             // identifier and __has_feature(is_empty) evaluates false.
-           .Case("is_empty", 
-                 LangOpts.CPlusPlus && 
-                 PP.getIdentifierInfo("__is_empty")->getTokenID()
-                                                            != tok::identifier)
+           .Case("is_empty", LangOpts.CPlusPlus)
            .Case("is_enum", LangOpts.CPlusPlus)
            .Case("is_final", LangOpts.CPlusPlus)
            .Case("is_literal", LangOpts.CPlusPlus)
            .Case("is_standard_layout", LangOpts.CPlusPlus)
-           // __is_pod is available only if the horrible
-           // "struct __is_pod" parsing hack hasn't been needed in this
-           // translation unit. If it has, __is_pod reverts to a normal
-           // identifier and __has_feature(is_pod) evaluates false.
-           .Case("is_pod", 
-                 LangOpts.CPlusPlus && 
-                 PP.getIdentifierInfo("__is_pod")->getTokenID()
-                                                            != tok::identifier)
+           .Case("is_pod", LangOpts.CPlusPlus)
            .Case("is_polymorphic", LangOpts.CPlusPlus)
            .Case("is_trivial", LangOpts.CPlusPlus)
            .Case("is_trivially_assignable", LangOpts.CPlusPlus)
@@ -807,22 +925,30 @@ static bool HasAttribute(const IdentifierInfo *II) {
 static bool EvaluateHasIncludeCommon(Token &Tok,
                                      IdentifierInfo *II, Preprocessor &PP,
                                      const DirectoryLookup *LookupFrom) {
-  SourceLocation LParenLoc;
+  // Save the location of the current token.  If a '(' is later found, use
+  // that location.  If no, use the end of this location instead.
+  SourceLocation LParenLoc = Tok.getLocation();
 
   // Get '('.
   PP.LexNonComment(Tok);
 
   // Ensure we have a '('.
   if (Tok.isNot(tok::l_paren)) {
-    PP.Diag(Tok.getLocation(), diag::err_pp_missing_lparen) << II->getName();
-    return false;
-  }
-
-  // Save '(' location for possible missing ')' message.
-  LParenLoc = Tok.getLocation();
+    // No '(', use end of last token.
+    LParenLoc = PP.getLocForEndOfToken(LParenLoc);
+    PP.Diag(LParenLoc, diag::err_pp_missing_lparen) << II->getName();
+    // If the next token looks like a filename or the start of one,
+    // assume it is and process it as such.
+    if (!Tok.is(tok::angle_string_literal) && !Tok.is(tok::string_literal) &&
+        !Tok.is(tok::less))
+      return false;
+  } else {
+    // Save '(' location for possible missing ')' message.
+    LParenLoc = Tok.getLocation();
 
-  // Get the file name.
-  PP.getCurrentLexer()->LexIncludeFilename(Tok);
+    // Get the file name.
+    PP.getCurrentLexer()->LexIncludeFilename(Tok);
+  }
 
   // Reserve a buffer to get the spelling.
   SmallString<128> FilenameBuffer;
@@ -847,8 +973,11 @@ static bool EvaluateHasIncludeCommon(Token &Tok,
     // This could be a <foo/bar.h> file coming from a macro expansion.  In this
     // case, glue the tokens together into FilenameBuffer and interpret those.
     FilenameBuffer.push_back('<');
-    if (PP.ConcatenateIncludeName(FilenameBuffer, EndLoc))
+    if (PP.ConcatenateIncludeName(FilenameBuffer, EndLoc)) {
+      // Let the caller know a <eod> was found by changing the Token kind.
+      Tok.setKind(tok::eod);
       return false;   // Found <eod> but no ">"?  Diagnostic already emitted.
+    }
     Filename = FilenameBuffer.str();
     break;
   default:
@@ -856,12 +985,15 @@ static bool EvaluateHasIncludeCommon(Token &Tok,
     return false;
   }
 
+  SourceLocation FilenameLoc = Tok.getLocation();
+
   // Get ')'.
   PP.LexNonComment(Tok);
 
   // Ensure we have a trailing ).
   if (Tok.isNot(tok::r_paren)) {
-    PP.Diag(Tok.getLocation(), diag::err_pp_missing_rparen) << II->getName();
+    PP.Diag(PP.getLocForEndOfToken(FilenameLoc), diag::err_pp_missing_rparen)
+        << II->getName();
     PP.Diag(LParenLoc, diag::note_matching) << "(";
     return false;
   }
@@ -909,6 +1041,47 @@ static bool EvaluateHasIncludeNext(Token &Tok,
   return EvaluateHasIncludeCommon(Tok, II, PP, Lookup);
 }
 
+/// \brief Process __building_module(identifier) expression.
+/// \returns true if we are building the named module, false otherwise.
+static bool EvaluateBuildingModule(Token &Tok,
+                                   IdentifierInfo *II, Preprocessor &PP) {
+  // Get '('.
+  PP.LexNonComment(Tok);
+
+  // Ensure we have a '('.
+  if (Tok.isNot(tok::l_paren)) {
+    PP.Diag(Tok.getLocation(), diag::err_pp_missing_lparen) << II->getName();
+    return false;
+  }
+
+  // Save '(' location for possible missing ')' message.
+  SourceLocation LParenLoc = Tok.getLocation();
+
+  // Get the module name.
+  PP.LexNonComment(Tok);
+
+  // Ensure that we have an identifier.
+  if (Tok.isNot(tok::identifier)) {
+    PP.Diag(Tok.getLocation(), diag::err_expected_id_building_module);
+    return false;
+  }
+
+  bool Result
+    = Tok.getIdentifierInfo()->getName() == PP.getLangOpts().CurrentModule;
+
+  // Get ')'.
+  PP.LexNonComment(Tok);
+
+  // Ensure we have a trailing ).
+  if (Tok.isNot(tok::r_paren)) {
+    PP.Diag(Tok.getLocation(), diag::err_pp_missing_rparen) << II->getName();
+    PP.Diag(LParenLoc, diag::note_matching) << "(";
+    return false;
+  }
+
+  return Result;
+}
+
 /// ExpandBuiltinMacro - If an identifier token is read that is to be expanded
 /// as a builtin macro, handle it and return the next token as 'Tok'.
 void Preprocessor::ExpandBuiltinMacro(Token &Tok) {
@@ -1093,7 +1266,8 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) {
     else
       Value = EvaluateHasIncludeNext(Tok, II, *this);
     OS << (int)Value;
-    Tok.setKind(tok::numeric_constant);
+    if (Tok.is(tok::r_paren))
+      Tok.setKind(tok::numeric_constant);
   } else if (II == Ident__has_warning) {
     // The argument should be a parenthesized string literal.
     // The argument to these builtins should be a parenthesized identifier.
@@ -1164,11 +1338,22 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) {
 
     OS << (int)Value;
     Tok.setKind(tok::numeric_constant);
+  } else if (II == Ident__building_module) {
+    // The argument to this builtin should be an identifier. The
+    // builtin evaluates to 1 when that identifier names the module we are
+    // currently building.
+    OS << (int)EvaluateBuildingModule(Tok, II, *this);
+    Tok.setKind(tok::numeric_constant);
+  } else if (II == Ident__MODULE__) {
+    // The current module as an identifier.
+    OS << getLangOpts().CurrentModule;
+    IdentifierInfo *ModuleII = getIdentifierInfo(getLangOpts().CurrentModule);
+    Tok.setIdentifierInfo(ModuleII);
+    Tok.setKind(ModuleII->getTokenID());
   } else {
     llvm_unreachable("Unknown identifier!");
   }
-  CreateString(OS.str().data(), OS.str().size(), Tok,
-               Tok.getLocation(), Tok.getLocation());
+  CreateString(OS.str(), Tok, Tok.getLocation(), Tok.getLocation());
 }
 
 void Preprocessor::markMacroAsUsed(MacroInfo *MI) {
diff --git a/lib/Lex/PTHLexer.cpp b/lib/Lex/PTHLexer.cpp
index 67738e9..b167172 100644
--- a/lib/Lex/PTHLexer.cpp
+++ b/lib/Lex/PTHLexer.cpp
@@ -198,12 +198,11 @@ bool PTHLexer::SkipBlock() {
   assert(LastHashTokPtr && "No known '#' token.");
 
   const unsigned char* HashEntryI = 0;
-  uint32_t Offset;
   uint32_t TableIdx;
 
   do {
     // Read the token offset from the side-table.
-    Offset = ReadLE32(CurPPCondPtr);
+    uint32_t Offset = ReadLE32(CurPPCondPtr);
 
     // Read the target table index from the side-table.
     TableIdx = ReadLE32(CurPPCondPtr);
@@ -223,13 +222,11 @@ bool PTHLexer::SkipBlock() {
         PPCond + TableIdx*(sizeof(uint32_t)*2);
       assert(NextPPCondPtr >= CurPPCondPtr);
       // Read where we should jump to.
-      uint32_t TmpOffset = ReadLE32(NextPPCondPtr);
-      const unsigned char* HashEntryJ = TokBuf + TmpOffset;
+      const unsigned char* HashEntryJ = TokBuf + ReadLE32(NextPPCondPtr);
 
       if (HashEntryJ <= LastHashTokPtr) {
         // Jump directly to the next entry in the side table.
         HashEntryI = HashEntryJ;
-        Offset = TmpOffset;
         TableIdx = ReadLE32(NextPPCondPtr);
         CurPPCondPtr = NextPPCondPtr;
       }
@@ -448,8 +445,8 @@ PTHManager *PTHManager::Create(const std::string &file,
 
   // Get the buffer ranges and check if there are at least three 32-bit
   // words at the end of the file.
-  const unsigned char *BufBeg = (unsigned char*)File->getBufferStart();
-  const unsigned char *BufEnd = (unsigned char*)File->getBufferEnd();
+  const unsigned char *BufBeg = (const unsigned char*)File->getBufferStart();
+  const unsigned char *BufEnd = (const unsigned char*)File->getBufferEnd();
 
   // Check the prologue of the file.
   if ((BufEnd - BufBeg) < (signed)(sizeof("cfe-pth") + 4 + 4) ||
diff --git a/lib/Lex/Pragma.cpp b/lib/Lex/Pragma.cpp
index c9cc4ad..e7e6c37 100644
--- a/lib/Lex/Pragma.cpp
+++ b/lib/Lex/Pragma.cpp
@@ -43,7 +43,6 @@ void EmptyPragmaHandler::HandlePragma(Preprocessor &PP,
 // PragmaNamespace Implementation.
 //===----------------------------------------------------------------------===//
 
-
 PragmaNamespace::~PragmaNamespace() {
   for (llvm::StringMap<PragmaHandler*>::iterator
          I = Handlers.begin(), E = Handlers.end(); I != E; ++I)
@@ -251,7 +250,7 @@ void Preprocessor::Handle_Pragma(Token &Tok) {
   // where we can lex it.
   Token TmpTok;
   TmpTok.startToken();
-  CreateString(&StrVal[0], StrVal.size(), TmpTok);
+  CreateString(StrVal, TmpTok);
   SourceLocation TokLoc = TmpTok.getLocation();
 
   // Make and enter a lexer object so that we lex and expand the tokens just
@@ -683,7 +682,7 @@ IdentifierInfo *Preprocessor::ParsePragmaPushOrPopMacro(Token &Tok) {
   Token MacroTok;
   MacroTok.startToken();
   MacroTok.setKind(tok::raw_identifier);
-  CreateString(&StrVal[1], StrVal.size() - 2, MacroTok);
+  CreateString(StringRef(&StrVal[1], StrVal.size() - 2), MacroTok);
 
   // Get the IdentifierInfo of MacroToPushTok.
   return LookUpIdentifierInfo(MacroTok);
@@ -733,19 +732,22 @@ void Preprocessor::HandlePragmaPopMacro(Token &PopMacroTok) {
   llvm::DenseMap<IdentifierInfo*, std::vector<MacroInfo*> >::iterator iter =
     PragmaPushMacroInfo.find(IdentInfo);
   if (iter != PragmaPushMacroInfo.end()) {
-    // Release the MacroInfo currently associated with IdentInfo.
-    MacroInfo *CurrentMI = getMacroInfo(IdentInfo);
-    if (CurrentMI) {
+    // Forget the MacroInfo currently associated with IdentInfo.
+    if (MacroInfo *CurrentMI = getMacroInfo(IdentInfo)) {
       if (CurrentMI->isWarnIfUnused())
         WarnUnusedMacroLocs.erase(CurrentMI->getDefinitionLoc());
-      ReleaseMacroInfo(CurrentMI);
+      UndefineMacro(IdentInfo, CurrentMI, MessageLoc);
     }
 
     // Get the MacroInfo we want to reinstall.
     MacroInfo *MacroToReInstall = iter->second.back();
 
-    // Reinstall the previously pushed macro.
-    setMacroInfo(IdentInfo, MacroToReInstall);
+    if (MacroToReInstall) {
+      // Reinstall the previously pushed macro.
+      setMacroInfo(IdentInfo, MacroToReInstall);
+    } else if (IdentInfo->hasMacroDefinition()) {
+      clearMacroInfo(IdentInfo);
+    }
 
     // Pop PragmaPushMacroInfo stack.
     iter->second.pop_back();
@@ -1009,7 +1011,7 @@ struct PragmaDebugHandler : public PragmaHandler {
     if (II->isStr("assert")) {
       llvm_unreachable("This is an assertion!");
     } else if (II->isStr("crash")) {
-      *(volatile int*) 0x11 = 0;
+      LLVM_BUILTIN_TRAP;
     } else if (II->isStr("parser_crash")) {
       Token Crasher;
       Crasher.setKind(tok::annot_pragma_parser_crash);
diff --git a/lib/Lex/PreprocessingRecord.cpp b/lib/Lex/PreprocessingRecord.cpp
index dfdeba3..01f3665e 100644
--- a/lib/Lex/PreprocessingRecord.cpp
+++ b/lib/Lex/PreprocessingRecord.cpp
@@ -25,10 +25,11 @@ ExternalPreprocessingRecordSource::~ExternalPreprocessingRecordSource() { }
 InclusionDirective::InclusionDirective(PreprocessingRecord &PPRec,
                                        InclusionKind Kind, 
                                        StringRef FileName, 
-                                       bool InQuotes, const FileEntry *File, 
+                                       bool InQuotes, bool ImportedModule,
+                                       const FileEntry *File,
                                        SourceRange Range)
   : PreprocessingDirective(InclusionDirectiveKind, Range), 
-    InQuotes(InQuotes), Kind(Kind), File(File) 
+    InQuotes(InQuotes), Kind(Kind), ImportedModule(ImportedModule), File(File)
 { 
   char *Memory 
     = (char*)PPRec.Allocate(FileName.size() + 1, llvm::alignOf<char>());
@@ -59,8 +60,7 @@ PreprocessingRecord::getPreprocessedEntitiesInRange(SourceRange Range) {
                           iterator(this, CachedRangeQuery.Result.second));
   }
 
-  std::pair<PPEntityID, PPEntityID>
-    Res = getPreprocessedEntitiesInRangeSlow(Range);
+  std::pair<int, int> Res = getPreprocessedEntitiesInRangeSlow(Range);
   
   CachedRangeQuery.Range = Range;
   CachedRangeQuery.Result = Res;
@@ -95,12 +95,12 @@ bool PreprocessingRecord::isEntityInFileID(iterator PPEI, FileID FID) {
   if (FID.isInvalid())
     return false;
 
-  PPEntityID PPID = PPEI.Position;
-  if (PPID < 0) {
-    assert(unsigned(-PPID-1) < LoadedPreprocessedEntities.size() &&
+  int Pos = PPEI.Position;
+  if (Pos < 0) {
+    assert(unsigned(-Pos-1) < LoadedPreprocessedEntities.size() &&
            "Out-of bounds loaded preprocessed entity");
     assert(ExternalSource && "No external source to load from");
-    unsigned LoadedIndex = LoadedPreprocessedEntities.size()+PPID;
+    unsigned LoadedIndex = LoadedPreprocessedEntities.size()+Pos;
     if (PreprocessedEntity *PPE = LoadedPreprocessedEntities[LoadedIndex])
       return isPreprocessedEntityIfInFileID(PPE, FID, SourceMgr);
 
@@ -118,15 +118,15 @@ bool PreprocessingRecord::isEntityInFileID(iterator PPEI, FileID FID) {
                                           FID, SourceMgr);
   }
 
-  assert(unsigned(PPID) < PreprocessedEntities.size() &&
+  assert(unsigned(Pos) < PreprocessedEntities.size() &&
          "Out-of bounds local preprocessed entity");
-  return isPreprocessedEntityIfInFileID(PreprocessedEntities[PPID],
+  return isPreprocessedEntityIfInFileID(PreprocessedEntities[Pos],
                                         FID, SourceMgr);
 }
 
 /// \brief Returns a pair of [Begin, End) iterators of preprocessed entities
 /// that source range \arg R encompasses.
-std::pair<PreprocessingRecord::PPEntityID, PreprocessingRecord::PPEntityID>
+std::pair<int, int>
 PreprocessingRecord::getPreprocessedEntitiesInRangeSlow(SourceRange Range) {
   assert(Range.isValid());
   assert(!SourceMgr.isBeforeInTranslationUnit(Range.getEnd(),Range.getBegin()));
@@ -319,14 +319,19 @@ void PreprocessingRecord::RegisterMacroDefinition(MacroInfo *Macro,
 
 /// \brief Retrieve the preprocessed entity at the given ID.
 PreprocessedEntity *PreprocessingRecord::getPreprocessedEntity(PPEntityID PPID){
-  if (PPID < 0) {
-    assert(unsigned(-PPID-1) < LoadedPreprocessedEntities.size() &&
+  if (PPID.ID < 0) {
+    unsigned Index = -PPID.ID - 1;
+    assert(Index < LoadedPreprocessedEntities.size() &&
            "Out-of bounds loaded preprocessed entity");
-    return getLoadedPreprocessedEntity(LoadedPreprocessedEntities.size()+PPID);
+    return getLoadedPreprocessedEntity(Index);
   }
-  assert(unsigned(PPID) < PreprocessedEntities.size() &&
+
+  if (PPID.ID == 0)
+    return 0;
+  unsigned Index = PPID.ID - 1;
+  assert(Index < PreprocessedEntities.size() &&
          "Out-of bounds local preprocessed entity");
-  return PreprocessedEntities[PPID];
+  return PreprocessedEntities[Index];
 }
 
 /// \brief Retrieve the loaded preprocessed entity at the given index.
@@ -389,10 +394,11 @@ void PreprocessingRecord::InclusionDirective(
     const clang::Token &IncludeTok,
     StringRef FileName,
     bool IsAngled,
+    CharSourceRange FilenameRange,
     const FileEntry *File,
-    clang::SourceLocation EndLoc,
     StringRef SearchPath,
-    StringRef RelativePath) {
+    StringRef RelativePath,
+    const Module *Imported) {
   InclusionDirective::InclusionKind Kind = InclusionDirective::Include;
   
   switch (IncludeTok.getIdentifierInfo()->getPPKeywordID()) {
@@ -415,9 +421,19 @@ void PreprocessingRecord::InclusionDirective(
   default:
     llvm_unreachable("Unknown include directive kind");
   }
-  
+
+  SourceLocation EndLoc;
+  if (!IsAngled) {
+    EndLoc = FilenameRange.getBegin();
+  } else {
+    EndLoc = FilenameRange.getEnd();
+    if (FilenameRange.isCharRange())
+      EndLoc = EndLoc.getLocWithOffset(-1); // the InclusionDirective expects
+                                            // a token range.
+  }
   clang::InclusionDirective *ID
-    = new (*this) clang::InclusionDirective(*this, Kind, FileName, !IsAngled, 
+    = new (*this) clang::InclusionDirective(*this, Kind, FileName, !IsAngled,
+                                            (bool)Imported,
                                             File, SourceRange(HashLoc, EndLoc));
   addPreprocessedEntity(ID);
 }
diff --git a/lib/Lex/Preprocessor.cpp b/lib/Lex/Preprocessor.cpp
index 614530c..3b070ce 100644
--- a/lib/Lex/Preprocessor.cpp
+++ b/lib/Lex/Preprocessor.cpp
@@ -26,6 +26,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Lex/Preprocessor.h"
+#include "clang/Lex/PreprocessorOptions.h"
 #include "MacroArgs.h"
 #include "clang/Lex/ExternalPreprocessorSource.h"
 #include "clang/Lex/HeaderSearch.h"
@@ -49,21 +50,25 @@ using namespace clang;
 //===----------------------------------------------------------------------===//
 ExternalPreprocessorSource::~ExternalPreprocessorSource() { }
 
-Preprocessor::Preprocessor(DiagnosticsEngine &diags, LangOptions &opts,
+PPMutationListener::~PPMutationListener() { }
+
+Preprocessor::Preprocessor(llvm::IntrusiveRefCntPtr<PreprocessorOptions> PPOpts,
+                           DiagnosticsEngine &diags, LangOptions &opts,
                            const TargetInfo *target, SourceManager &SM,
                            HeaderSearch &Headers, ModuleLoader &TheModuleLoader,
                            IdentifierInfoLookup* IILookup,
                            bool OwnsHeaders,
                            bool DelayInitialization,
                            bool IncrProcessing)
-  : Diags(&diags), LangOpts(opts), Target(target),FileMgr(Headers.getFileMgr()),
+  : PPOpts(PPOpts), Diags(&diags), LangOpts(opts), Target(target),
+    FileMgr(Headers.getFileMgr()),
     SourceMgr(SM), HeaderInfo(Headers), TheModuleLoader(TheModuleLoader),
     ExternalSource(0), Identifiers(opts, IILookup), 
     IncrementalProcessing(IncrProcessing), CodeComplete(0), 
     CodeCompletionFile(0), CodeCompletionOffset(0), CodeCompletionReached(0),
     SkipMainFilePreamble(0, true), CurPPLexer(0), 
-    CurDirLookup(0), CurLexerKind(CLK_Lexer), Callbacks(0), MacroArgCache(0), 
-    Record(0), MIChainHead(0), MICache(0) 
+    CurDirLookup(0), CurLexerKind(CLK_Lexer), Callbacks(0), Listener(0),
+    MacroArgCache(0), Record(0), MIChainHead(0), MICache(0) 
 {
   OwnsHeaderSearch = OwnsHeaders;
   
@@ -285,6 +290,39 @@ Preprocessor::macro_end(bool IncludeExternalMacros) const {
   return Macros.end();
 }
 
+/// \brief Compares macro tokens with a specified token value sequence.
+static bool MacroDefinitionEquals(const MacroInfo *MI,
+                                  llvm::ArrayRef<TokenValue> Tokens) {
+  return Tokens.size() == MI->getNumTokens() &&
+      std::equal(Tokens.begin(), Tokens.end(), MI->tokens_begin());
+}
+
+StringRef Preprocessor::getLastMacroWithSpelling(
+                                    SourceLocation Loc,
+                                    ArrayRef<TokenValue> Tokens) const {
+  SourceLocation BestLocation;
+  StringRef BestSpelling;
+  for (Preprocessor::macro_iterator I = macro_begin(), E = macro_end();
+       I != E; ++I) {
+    if (!I->second->isObjectLike())
+      continue;
+    const MacroInfo *MI = I->second->findDefinitionAtLoc(Loc, SourceMgr);
+    if (!MI)
+      continue;
+    if (!MacroDefinitionEquals(MI, Tokens))
+      continue;
+    SourceLocation Location = I->second->getDefinitionLoc();
+    // Choose the macro defined latest.
+    if (BestLocation.isInvalid() ||
+        (Location.isValid() &&
+         SourceMgr.isBeforeInTranslationUnit(BestLocation, Location))) {
+      BestLocation = Location;
+      BestSpelling = I->first->getName();
+    }
+  }
+  return BestSpelling;
+}
+
 void Preprocessor::recomputeCurLexerKind() {
   if (CurLexer)
     CurLexerKind = CLK_Lexer;
@@ -378,17 +416,17 @@ StringRef Preprocessor::getSpelling(const Token &Tok,
 /// CreateString - Plop the specified string into a scratch buffer and return a
 /// location for it.  If specified, the source location provides a source
 /// location for the token.
-void Preprocessor::CreateString(const char *Buf, unsigned Len, Token &Tok,
+void Preprocessor::CreateString(StringRef Str, Token &Tok,
                                 SourceLocation ExpansionLocStart,
                                 SourceLocation ExpansionLocEnd) {
-  Tok.setLength(Len);
+  Tok.setLength(Str.size());
 
   const char *DestPtr;
-  SourceLocation Loc = ScratchBuf->getToken(Buf, Len, DestPtr);
+  SourceLocation Loc = ScratchBuf->getToken(Str.data(), Str.size(), DestPtr);
 
   if (ExpansionLocStart.isValid())
     Loc = SourceMgr.createExpansionLoc(Loc, ExpansionLocStart,
-                                       ExpansionLocEnd, Len);
+                                       ExpansionLocEnd, Str.size());
   Tok.setLocation(Loc);
 
   // If this is a raw identifier or a literal token, set the pointer data.
@@ -641,10 +679,14 @@ void Preprocessor::LexAfterModuleImport(Token &Result) {
   }
 
   // If we have a non-empty module path, load the named module.
-  if (!ModuleImportPath.empty())
-    (void)TheModuleLoader.loadModule(ModuleImportLoc, ModuleImportPath,
-                                     Module::MacrosVisible,
-                                     /*IsIncludeDirective=*/false);
+  if (!ModuleImportPath.empty()) {
+    Module *Imported = TheModuleLoader.loadModule(ModuleImportLoc,
+                                                  ModuleImportPath,
+                                                  Module::MacrosVisible,
+                                                  /*IsIncludeDirective=*/false);
+    if (Callbacks)
+      Callbacks->moduleImport(ModuleImportLoc, ModuleImportPath, Imported);
+  }
 }
 
 void Preprocessor::addCommentHandler(CommentHandler *Handler) {
diff --git a/lib/Lex/TokenLexer.cpp b/lib/Lex/TokenLexer.cpp
index ade40da..59b7478 100644
--- a/lib/Lex/TokenLexer.cpp
+++ b/lib/Lex/TokenLexer.cpp
@@ -23,12 +23,13 @@ using namespace clang;
 
 /// Create a TokenLexer for the specified macro with the specified actual
 /// arguments.  Note that this ctor takes ownership of the ActualArgs pointer.
-void TokenLexer::Init(Token &Tok, SourceLocation ELEnd, MacroArgs *Actuals) {
+void TokenLexer::Init(Token &Tok, SourceLocation ELEnd, MacroInfo *MI,
+                      MacroArgs *Actuals) {
   // If the client is reusing a TokenLexer, make sure to free any memory
   // associated with it.
   destroy();
 
-  Macro = PP.getMacroInfo(Tok.getIdentifierInfo());
+  Macro = MI;
   ActualArgs = Actuals;
   CurToken = 0;
 
@@ -118,6 +119,55 @@ void TokenLexer::destroy() {
   if (ActualArgs) ActualArgs->destroy(PP);
 }
 
+/// Remove comma ahead of __VA_ARGS__, if present, according to compiler dialect
+/// settings.  Returns true if the comma is removed.
+static bool MaybeRemoveCommaBeforeVaArgs(SmallVector<Token, 128> &ResultToks,
+                                         bool &NextTokGetsSpace,
+                                         bool HasPasteOperator,
+                                         MacroInfo *Macro, unsigned MacroArgNo,
+                                         Preprocessor &PP) {
+  // Is the macro argument __VA_ARGS__?
+  if (!Macro->isVariadic() || MacroArgNo != Macro->getNumArgs()-1)
+    return false;
+
+  // In Microsoft-compatibility mode, a comma is removed in the expansion
+  // of " ... , __VA_ARGS__ " if __VA_ARGS__ is empty.  This extension is
+  // not supported by gcc.
+  if (!HasPasteOperator && !PP.getLangOpts().MicrosoftMode)
+    return false;
+
+  // GCC removes the comma in the expansion of " ... , ## __VA_ARGS__ " if
+  // __VA_ARGS__ is empty, but not in strict C99 mode where there are no
+  // named arguments, where it remains.  In all other modes, including C99
+  // with GNU extensions, it is removed regardless of named arguments.
+  // Microsoft also appears to support this extension, unofficially.
+  if (PP.getLangOpts().C99 && !PP.getLangOpts().GNUMode
+        && Macro->getNumArgs() < 2)
+    return false;
+
+  // Is a comma available to be removed?
+  if (ResultToks.empty() || !ResultToks.back().is(tok::comma))
+    return false;
+
+  // Issue an extension diagnostic for the paste operator.
+  if (HasPasteOperator)
+    PP.Diag(ResultToks.back().getLocation(), diag::ext_paste_comma);
+
+  // Remove the comma.
+  ResultToks.pop_back();
+
+  // If the comma was right after another paste (e.g. "X##,##__VA_ARGS__"),
+  // then removal of the comma should produce a placemarker token (in C99
+  // terms) which we model by popping off the previous ##, giving us a plain
+  // "X" when __VA_ARGS__ is empty.
+  if (!ResultToks.empty() && ResultToks.back().is(tok::hashhash))
+    ResultToks.pop_back();
+
+  // Never add a space, even if the comma, ##, or arg had a space.
+  NextTokGetsSpace = false;
+  return true;
+}
+
 /// Expand the arguments of a function-like macro so that we can quickly
 /// return preexpanded tokens from Tokens.
 void TokenLexer::ExpandFunctionArguments() {
@@ -198,6 +248,14 @@ void TokenLexer::ExpandFunctionArguments() {
       !ResultToks.empty() && ResultToks.back().is(tok::hashhash);
     bool PasteAfter = i+1 != e && Tokens[i+1].is(tok::hashhash);
 
+    // In Microsoft mode, remove the comma before __VA_ARGS__ to ensure there
+    // are no trailing commas if __VA_ARGS__ is empty.
+    if (!PasteBefore && ActualArgs->isVarargsElidedUse() &&
+        MaybeRemoveCommaBeforeVaArgs(ResultToks, NextTokGetsSpace,
+                                     /*HasPasteOperator=*/false,
+                                     Macro, ArgNo, PP))
+      continue;
+
     // If it is not the LHS/RHS of a ## operator, we must pre-expand the
     // argument and substitute the expanded tokens into the result.  This is
     // C99 6.10.3.1p1.
@@ -320,23 +378,13 @@ void TokenLexer::ExpandFunctionArguments() {
 
     // If this is the __VA_ARGS__ token, and if the argument wasn't provided,
     // and if the macro had at least one real argument, and if the token before
-    // the ## was a comma, remove the comma.
-    if ((unsigned)ArgNo == Macro->getNumArgs()-1 && // is __VA_ARGS__
-        ActualArgs->isVarargsElidedUse() &&       // Argument elided.
-        !ResultToks.empty() && ResultToks.back().is(tok::comma)) {
-      // Never add a space, even if the comma, ##, or arg had a space.
-      NextTokGetsSpace = false;
-      // Remove the paste operator, report use of the extension.
-      PP.Diag(ResultToks.back().getLocation(), diag::ext_paste_comma);
-      ResultToks.pop_back();
-      
-      // If the comma was right after another paste (e.g. "X##,##__VA_ARGS__"),
-      // then removal of the comma should produce a placemarker token (in C99
-      // terms) which we model by popping off the previous ##, giving us a plain
-      // "X" when __VA_ARGS__ is empty.
-      if (!ResultToks.empty() && ResultToks.back().is(tok::hashhash))
-        ResultToks.pop_back();
-    }
+    // the ## was a comma, remove the comma.  This is a GCC extension which is
+    // disabled when using -std=c99.
+    if (ActualArgs->isVarargsElidedUse())
+      MaybeRemoveCommaBeforeVaArgs(ResultToks, NextTokGetsSpace,
+                                   /*HasPasteOperator=*/true,
+                                   Macro, ArgNo, PP);
+
     continue;
   }
 
@@ -494,7 +542,7 @@ bool TokenLexer::PasteTokens(Token &Tok) {
     // Claim that the tmp token is a string_literal so that we can get the
     // character pointer back from CreateString in getLiteralData().
     ResultTokTmp.setKind(tok::string_literal);
-    PP.CreateString(&Buffer[0], Buffer.size(), ResultTokTmp);
+    PP.CreateString(Buffer, ResultTokTmp);
     SourceLocation ResultTokLoc = ResultTokTmp.getLocation();
     ResultTokStrPtr = ResultTokTmp.getLiteralData();
 
diff --git a/lib/Parse/ParseAST.cpp b/lib/Parse/ParseAST.cpp
index bd4f859..7d68e1f 100644
--- a/lib/Parse/ParseAST.cpp
+++ b/lib/Parse/ParseAST.cpp
@@ -78,7 +78,6 @@ void clang::ParseAST(Sema &S, bool PrintStats, bool SkipFunctionBodies) {
 
   S.getPreprocessor().EnterMainSourceFile();
   P.Initialize();
-  S.Initialize();
 
   // C11 6.9p1 says translation units must have at least one top-level
   // declaration. C++ doesn't have this restriction. We also don't want to
diff --git a/lib/Parse/ParseCXXInlineMethods.cpp b/lib/Parse/ParseCXXInlineMethods.cpp
index abce27c..9c5c0597 100644
--- a/lib/Parse/ParseCXXInlineMethods.cpp
+++ b/lib/Parse/ParseCXXInlineMethods.cpp
@@ -34,7 +34,7 @@ Decl *Parser::ParseCXXInlineMethodDef(AccessSpecifier AS,
           Tok.is(tok::equal)) &&
          "Current token not a '{', ':', '=', or 'try'!");
 
-  MultiTemplateParamsArg TemplateParams(Actions,
+  MultiTemplateParamsArg TemplateParams(
           TemplateInfo.TemplateParams ? TemplateInfo.TemplateParams->data() : 0,
           TemplateInfo.TemplateParams ? TemplateInfo.TemplateParams->size() : 0);
 
@@ -42,10 +42,10 @@ Decl *Parser::ParseCXXInlineMethodDef(AccessSpecifier AS,
   D.setFunctionDefinitionKind(DefinitionKind);
   if (D.getDeclSpec().isFriendSpecified())
     FnD = Actions.ActOnFriendFunctionDecl(getCurScope(), D,
-                                          move(TemplateParams));
+                                          TemplateParams);
   else {
     FnD = Actions.ActOnCXXMemberDeclarator(getCurScope(), AS, D,
-                                           move(TemplateParams), 0, 
+                                           TemplateParams, 0,
                                            VS, ICIS_NoInit);
     if (FnD) {
       Actions.ProcessDeclAttributeList(getCurScope(), FnD, AccessAttrs,
diff --git a/lib/Parse/ParseDecl.cpp b/lib/Parse/ParseDecl.cpp
index cb865cc..f73907a 100644
--- a/lib/Parse/ParseDecl.cpp
+++ b/lib/Parse/ParseDecl.cpp
@@ -143,7 +143,7 @@ void Parser::ParseGNUAttributes(ParsedAttributes &attrs,
 
           // Attributes in a class are parsed at the end of the class, along
           // with other late-parsed declarations.
-          if (!ClassStack.empty())
+          if (!ClassStack.empty() && !LateAttrs->parseSoon())
             getCurrentClass().LateParsedDeclarations.push_back(LA);
 
           // consume everything up to and including the matching right parens
@@ -154,7 +154,8 @@ void Parser::ParseGNUAttributes(ParsedAttributes &attrs,
           Eof.setLocation(Tok.getLocation());
           LA->Toks.push_back(Eof);
         } else {
-          ParseGNUAttributeArgs(AttrName, AttrNameLoc, attrs, endLoc);
+          ParseGNUAttributeArgs(AttrName, AttrNameLoc, attrs, endLoc,
+                                0, SourceLocation(), AttributeList::AS_GNU);
         }
       } else {
         attrs.addNew(AttrName, AttrNameLoc, 0, AttrNameLoc,
@@ -173,11 +174,15 @@ void Parser::ParseGNUAttributes(ParsedAttributes &attrs,
 }
 
 
-/// Parse the arguments to a parameterized GNU attribute
+/// Parse the arguments to a parameterized GNU attribute or
+/// a C++11 attribute in "gnu" namespace.
 void Parser::ParseGNUAttributeArgs(IdentifierInfo *AttrName,
                                    SourceLocation AttrNameLoc,
                                    ParsedAttributes &Attrs,
-                                   SourceLocation *EndLoc) {
+                                   SourceLocation *EndLoc,
+                                   IdentifierInfo *ScopeName,
+                                   SourceLocation ScopeLoc,
+                                   AttributeList::Syntax Syntax) {
 
   assert(Tok.is(tok::l_paren) && "Attribute arg list not starting with '('");
 
@@ -236,7 +241,7 @@ void Parser::ParseGNUAttributeArgs(IdentifierInfo *AttrName,
     break;
   }
 
-  ExprVector ArgExprs(Actions);
+  ExprVector ArgExprs;
 
   if (!BuiltinType &&
       (ParmLoc.isValid() ? Tok.is(tok::comma) : Tok.isNot(tok::r_paren))) {
@@ -277,10 +282,11 @@ void Parser::ParseGNUAttributeArgs(IdentifierInfo *AttrName,
 
   SourceLocation RParen = Tok.getLocation();
   if (!ExpectAndConsume(tok::r_paren, diag::err_expected_rparen)) {
+    SourceLocation AttrLoc = ScopeLoc.isValid() ? ScopeLoc : AttrNameLoc;
     AttributeList *attr =
-      Attrs.addNew(AttrName, SourceRange(AttrNameLoc, RParen), 0, AttrNameLoc,
-                   ParmName, ParmLoc, ArgExprs.take(), ArgExprs.size(),
-                   AttributeList::AS_GNU);
+      Attrs.addNew(AttrName, SourceRange(AttrLoc, RParen),
+                   ScopeName, ScopeLoc, ParmName, ParmLoc,
+                   ArgExprs.data(), ArgExprs.size(), Syntax);
     if (BuiltinType && attr->getKind() == AttributeList::AT_IBOutletCollection)
       Diag(Tok, diag::err_iboutletcollection_builtintype);
   }
@@ -851,10 +857,6 @@ void Parser::ParseLexedAttributes(ParsingClass &Class) {
     Actions.ActOnStartDelayedMemberDeclarations(getCurScope(),
                                                 Class.TagOrTemplate);
   if (!Class.LateParsedDeclarations.empty()) {
-    // Allow 'this' within late-parsed attributes.
-    Sema::CXXThisScopeRAII ThisScope(Actions, Class.TagOrTemplate,
-                                     /*TypeQuals=*/0);
-
     for (unsigned i = 0, ni = Class.LateParsedDeclarations.size(); i < ni; ++i){
       Class.LateParsedDeclarations[i]->ParseLexedAttributes();
     }
@@ -869,6 +871,8 @@ void Parser::ParseLexedAttributes(ParsingClass &Class) {
 /// \brief Parse all attributes in LAs, and attach them to Decl D.
 void Parser::ParseLexedAttributeList(LateParsedAttrList &LAs, Decl *D,
                                      bool EnterScope, bool OnDefinition) {
+  assert(LAs.parseSoon() &&
+         "Attribute list should be marked for immediate parsing.");
   for (unsigned i = 0, ni = LAs.size(); i < ni; ++i) {
     if (D)
       LAs[i]->addDecl(D);
@@ -904,34 +908,45 @@ void Parser::ParseLexedAttribute(LateParsedAttribute &LA,
   ParsedAttributes Attrs(AttrFactory);
   SourceLocation endLoc;
 
-  if (LA.Decls.size() == 1) {
+  if (LA.Decls.size() > 0) {
     Decl *D = LA.Decls[0];
+    NamedDecl *ND  = dyn_cast<NamedDecl>(D);
+    RecordDecl *RD = dyn_cast_or_null<RecordDecl>(D->getDeclContext());
 
-    // If the Decl is templatized, add template parameters to scope.
-    bool HasTemplateScope = EnterScope && D->isTemplateDecl();
-    ParseScope TempScope(this, Scope::TemplateParamScope, HasTemplateScope);
-    if (HasTemplateScope)
-      Actions.ActOnReenterTemplateScope(Actions.CurScope, D);
-
-    // If the Decl is on a function, add function parameters to the scope.
-    bool HasFunctionScope = EnterScope && D->isFunctionOrFunctionTemplate();
-    ParseScope FnScope(this, Scope::FnScope|Scope::DeclScope, HasFunctionScope);
-    if (HasFunctionScope)
-      Actions.ActOnReenterFunctionContext(Actions.CurScope, D);
-
-    ParseGNUAttributeArgs(&LA.AttrName, LA.AttrNameLoc, Attrs, &endLoc);
-
-    if (HasFunctionScope) {
-      Actions.ActOnExitFunctionContext();
-      FnScope.Exit();  // Pop scope, and remove Decls from IdResolver
-    }
-    if (HasTemplateScope) {
-      TempScope.Exit();
+    // Allow 'this' within late-parsed attributes.
+    Sema::CXXThisScopeRAII ThisScope(Actions, RD,
+                                     /*TypeQuals=*/0,
+                                     ND && RD && ND->isCXXInstanceMember());
+
+    if (LA.Decls.size() == 1) {
+      // If the Decl is templatized, add template parameters to scope.
+      bool HasTemplateScope = EnterScope && D->isTemplateDecl();
+      ParseScope TempScope(this, Scope::TemplateParamScope, HasTemplateScope);
+      if (HasTemplateScope)
+        Actions.ActOnReenterTemplateScope(Actions.CurScope, D);
+
+      // If the Decl is on a function, add function parameters to the scope.
+      bool HasFunScope = EnterScope && D->isFunctionOrFunctionTemplate();
+      ParseScope FnScope(this, Scope::FnScope|Scope::DeclScope, HasFunScope);
+      if (HasFunScope)
+        Actions.ActOnReenterFunctionContext(Actions.CurScope, D);
+
+      ParseGNUAttributeArgs(&LA.AttrName, LA.AttrNameLoc, Attrs, &endLoc,
+                            0, SourceLocation(), AttributeList::AS_GNU);
+
+      if (HasFunScope) {
+        Actions.ActOnExitFunctionContext();
+        FnScope.Exit();  // Pop scope, and remove Decls from IdResolver
+      }
+      if (HasTemplateScope) {
+        TempScope.Exit();
+      }
+    } else {
+      // If there are multiple decls, then the decl cannot be within the
+      // function scope.
+      ParseGNUAttributeArgs(&LA.AttrName, LA.AttrNameLoc, Attrs, &endLoc,
+                            0, SourceLocation(), AttributeList::AS_GNU);
     }
-  } else if (LA.Decls.size() > 0) {
-    // If there are multiple decls, then the decl cannot be within the
-    // function scope.
-    ParseGNUAttributeArgs(&LA.AttrName, LA.AttrNameLoc, Attrs, &endLoc);
   } else {
     Diag(Tok, diag::warn_attribute_no_decl) << LA.AttrName.getName();
   }
@@ -998,7 +1013,7 @@ void Parser::ParseThreadSafetyAttribute(IdentifierInfo &AttrName,
   BalancedDelimiterTracker T(*this, tok::l_paren);
   T.consumeOpen();
 
-  ExprVector ArgExprs(Actions);
+  ExprVector ArgExprs;
   bool ArgExprsOk = true;
 
   // now parse the list of expressions
@@ -1018,7 +1033,7 @@ void Parser::ParseThreadSafetyAttribute(IdentifierInfo &AttrName,
   // Match the ')'.
   if (ArgExprsOk && !T.consumeClose()) {
     Attrs.addNew(&AttrName, AttrNameLoc, 0, AttrNameLoc, 0, SourceLocation(),
-                 ArgExprs.take(), ArgExprs.size(), AttributeList::AS_GNU);
+                 ArgExprs.data(), ArgExprs.size(), AttributeList::AS_GNU);
   }
   if (EndLoc)
     *EndLoc = T.getCloseLocation();
@@ -1127,6 +1142,18 @@ void Parser::DiagnoseProhibitedAttributes(ParsedAttributesWithRange &attrs) {
     << attrs.Range;
 }
 
+void Parser::ProhibitCXX11Attributes(ParsedAttributesWithRange &attrs) {
+  AttributeList *AttrList = attrs.getList();
+  while (AttrList) {
+    if (AttrList->isCXX0XAttribute()) {
+      Diag(AttrList->getLoc(), diag::warn_attribute_no_decl) 
+        << AttrList->getName();
+      AttrList->setInvalid();
+    }
+    AttrList = AttrList->getNext();
+  }
+}
+
 /// ParseDeclaration - Parse a full 'declaration', which consists of
 /// declaration-specifiers, some number of declarators, and a semicolon.
 /// 'Context' should be a Declarator::TheContext value.  This returns the
@@ -1400,7 +1427,8 @@ Parser::DeclGroupPtrTy Parser::ParseDeclGroup(ParsingDeclSpec &DS,
 
   // Save late-parsed attributes for now; they need to be parsed in the
   // appropriate function scope after the function Decl has been constructed.
-  LateParsedAttrList LateParsedAttrs;
+  // These will be parsed in ParseFunctionDefinition or ParseLexedAttrList.
+  LateParsedAttrList LateParsedAttrs(true);
   if (D.isFunctionDeclarator())
     MaybeParseGNUAttributes(D, &LateParsedAttrs);
 
@@ -1587,9 +1615,7 @@ Decl *Parser::ParseDeclarationAfterDeclaratorAndAttributes(Declarator &D,
   case ParsedTemplateInfo::Template:
   case ParsedTemplateInfo::ExplicitSpecialization:
     ThisDecl = Actions.ActOnTemplateDeclarator(getCurScope(),
-                             MultiTemplateParamsArg(Actions,
-                                          TemplateInfo.TemplateParams->data(),
-                                          TemplateInfo.TemplateParams->size()),
+                                               *TemplateInfo.TemplateParams,
                                                D);
     break;
 
@@ -1660,7 +1686,7 @@ Decl *Parser::ParseDeclarationAfterDeclaratorAndAttributes(Declarator &D,
     BalancedDelimiterTracker T(*this, tok::l_paren);
     T.consumeOpen();
 
-    ExprVector Exprs(Actions);
+    ExprVector Exprs;
     CommaLocsTy CommaLocs;
 
     if (getLangOpts().CPlusPlus && D.getCXXScopeSpec().isSet()) {
@@ -1669,6 +1695,7 @@ Decl *Parser::ParseDeclarationAfterDeclaratorAndAttributes(Declarator &D,
     }
 
     if (ParseExpressionList(Exprs, CommaLocs)) {
+      Actions.ActOnInitializerError(ThisDecl);
       SkipUntil(tok::r_paren);
 
       if (getLangOpts().CPlusPlus && D.getCXXScopeSpec().isSet()) {
@@ -1689,7 +1716,7 @@ Decl *Parser::ParseDeclarationAfterDeclaratorAndAttributes(Declarator &D,
 
       ExprResult Initializer = Actions.ActOnParenListExpr(T.getOpenLocation(),
                                                           T.getCloseLocation(),
-                                                          move_arg(Exprs));
+                                                          Exprs);
       Actions.AddInitializerToDecl(ThisDecl, Initializer.take(),
                                    /*DirectInit=*/true, TypeContainsAuto);
     }
@@ -1872,6 +1899,9 @@ bool Parser::ParseImplicitInt(DeclSpec &DS, CXXScopeSpec *SS,
         TagName="union" ; FixitTagName = "union " ;TagKind=tok::kw_union ;break;
       case DeclSpec::TST_struct:
         TagName="struct"; FixitTagName = "struct ";TagKind=tok::kw_struct;break;
+      case DeclSpec::TST_interface:
+        TagName="__interface"; FixitTagName = "__interface ";
+        TagKind=tok::kw___interface;break;
       case DeclSpec::TST_class:
         TagName="class" ; FixitTagName = "class " ;TagKind=tok::kw_class ;break;
     }
@@ -2069,13 +2099,13 @@ void Parser::ParseAlignmentSpecifier(ParsedAttributes &Attrs,
     return;
   }
 
-  ExprVector ArgExprs(Actions);
+  ExprVector ArgExprs;
   ArgExprs.push_back(ArgExpr.release());
   // FIXME: This should not be GNU, but we since the attribute used is
   //        based on the spelling, and there is no true spelling for
   //        C++11 attributes, this isn't accepted.
   Attrs.addNew(PP.getIdentifierInfo("aligned"), KWLoc, 0, KWLoc,
-               0, T.getOpenLocation(), ArgExprs.take(), 1,
+               0, T.getOpenLocation(), ArgExprs.data(), 1,
                AttributeList::AS_GNU);
 }
 
@@ -2130,8 +2160,14 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
     DoneWithDeclSpec:
       if (!AttrsLastTime)
         ProhibitAttributes(attrs);
-      else
+      else {
+        // Reject C++11 attributes that appertain to decl specifiers as
+        // we don't support any C++11 attributes that appertain to decl
+        // specifiers. This also conforms to what g++ 4.8 is doing.
+        ProhibitCXX11Attributes(attrs);
+
         DS.takeAttributesFrom(attrs);
+      }
 
       // If this is not a declaration specifier token, we're done reading decl
       // specifiers.  First verify that DeclSpec's are consistent.
@@ -2271,6 +2307,8 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
           isInvalid = DS.SetTypeSpecType(DeclSpec::TST_typename,
                                          Tok.getAnnotationEndLoc(),
                                          PrevSpec, DiagID, T);
+          if (isInvalid)
+            break;
         }
         else
           DS.SetTypeSpecError();
@@ -2476,12 +2514,12 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
     case tok::kw___forceinline: {
       isInvalid = DS.SetFunctionSpecInline(Loc, PrevSpec, DiagID);
       IdentifierInfo *AttrName = Tok.getIdentifierInfo();
-      SourceLocation AttrNameLoc = ConsumeToken();
+      SourceLocation AttrNameLoc = Tok.getLocation();
       // FIXME: This does not work correctly if it is set to be a declspec
       //        attribute, and a GNU attribute is simply incorrect.
       DS.getAttributes().addNew(AttrName, AttrNameLoc, 0, AttrNameLoc, 0,
                                 SourceLocation(), 0, 0, AttributeList::AS_GNU);
-      continue;
+      break;
     }
 
     case tok::kw___ptr64:
@@ -2706,6 +2744,7 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
     // class-specifier:
     case tok::kw_class:
     case tok::kw_struct:
+    case tok::kw___interface:
     case tok::kw_union: {
       tok::TokenKind Kind = Tok.getKind();
       ConsumeToken();
@@ -2723,15 +2762,15 @@ void Parser::ParseDeclarationSpecifiers(DeclSpec &DS,
     // cv-qualifier:
     case tok::kw_const:
       isInvalid = DS.SetTypeQual(DeclSpec::TQ_const, Loc, PrevSpec, DiagID,
-                                 getLangOpts(), /*IsTypeSpec*/true);
+                                 getLangOpts());
       break;
     case tok::kw_volatile:
       isInvalid = DS.SetTypeQual(DeclSpec::TQ_volatile, Loc, PrevSpec, DiagID,
-                                 getLangOpts(), /*IsTypeSpec*/true);
+                                 getLangOpts());
       break;
     case tok::kw_restrict:
       isInvalid = DS.SetTypeQual(DeclSpec::TQ_restrict, Loc, PrevSpec, DiagID,
-                                 getLangOpts(), /*IsTypeSpec*/true);
+                                 getLangOpts());
       break;
 
     // C++ typename-specifier:
@@ -3165,6 +3204,8 @@ void Parser::ParseEnumSpecifier(SourceLocation StartLoc, DeclSpec &DS,
       // anything that's a simple-type-specifier followed by '(' as an
       // expression. This suffices because function types are not valid
       // underlying types anyway.
+      EnterExpressionEvaluationContext Unevaluated(Actions,
+                                                   Sema::ConstantEvaluated);
       TPResult TPR = isExpressionOrTypeSpecifierSimple(NextToken().getKind());
       // If the next token starts an expression, we know we're parsing a
       // bit-field. This is the common case.
@@ -3213,11 +3254,14 @@ void Parser::ParseEnumSpecifier(SourceLocation StartLoc, DeclSpec &DS,
       SourceRange Range;
       BaseType = ParseTypeName(&Range);
 
-      if (!getLangOpts().CPlusPlus0x && !getLangOpts().ObjC2)
-        Diag(StartLoc, diag::ext_ms_enum_fixed_underlying_type)
-          << Range;
-      if (getLangOpts().CPlusPlus0x)
+      if (getLangOpts().CPlusPlus0x) {
         Diag(StartLoc, diag::warn_cxx98_compat_enum_fixed_underlying_type);
+      } else if (!getLangOpts().ObjC2) {
+        if (getLangOpts().CPlusPlus)
+          Diag(StartLoc, diag::ext_cxx11_enum_fixed_underlying_type) << Range;
+        else
+          Diag(StartLoc, diag::ext_c_enum_fixed_underlying_type) << Range;
+      }
     }
   }
 
@@ -3526,6 +3570,7 @@ bool Parser::isKnownToBeTypeSpecifier(const Token &Tok) const {
     // struct-or-union-specifier (C99) or class-specifier (C++)
   case tok::kw_class:
   case tok::kw_struct:
+  case tok::kw___interface:
   case tok::kw_union:
     // enum-specifier
   case tok::kw_enum:
@@ -3597,6 +3642,7 @@ bool Parser::isTypeSpecifierQualifier() {
     // struct-or-union-specifier (C99) or class-specifier (C++)
   case tok::kw_class:
   case tok::kw_struct:
+  case tok::kw___interface:
   case tok::kw_union:
     // enum-specifier
   case tok::kw_enum:
@@ -3735,6 +3781,7 @@ bool Parser::isDeclarationSpecifier(bool DisambiguatingWithExpression) {
   case tok::kw_class:
   case tok::kw_struct:
   case tok::kw_union:
+  case tok::kw___interface:
     // enum-specifier
   case tok::kw_enum:
 
@@ -3748,6 +3795,9 @@ bool Parser::isDeclarationSpecifier(bool DisambiguatingWithExpression) {
   case tok::kw_virtual:
   case tok::kw_explicit:
 
+    // friend keyword.
+  case tok::kw_friend:
+
     // static_assert-declaration
   case tok::kw__Static_assert:
 
@@ -3756,11 +3806,10 @@ bool Parser::isDeclarationSpecifier(bool DisambiguatingWithExpression) {
 
     // GNU attributes.
   case tok::kw___attribute:
-    return true;
 
-    // C++0x decltype.
+    // C++11 decltype and constexpr.
   case tok::annot_decltype:
-    return true;
+  case tok::kw_constexpr:
 
     // C11 _Atomic()
   case tok::kw__Atomic:
@@ -3925,15 +3974,15 @@ void Parser::ParseTypeQualifierListOpt(DeclSpec &DS,
 
     case tok::kw_const:
       isInvalid = DS.SetTypeQual(DeclSpec::TQ_const   , Loc, PrevSpec, DiagID,
-                                 getLangOpts(), /*IsTypeSpec*/false);
+                                 getLangOpts());
       break;
     case tok::kw_volatile:
       isInvalid = DS.SetTypeQual(DeclSpec::TQ_volatile, Loc, PrevSpec, DiagID,
-                                 getLangOpts(), /*IsTypeSpec*/false);
+                                 getLangOpts());
       break;
     case tok::kw_restrict:
       isInvalid = DS.SetTypeQual(DeclSpec::TQ_restrict, Loc, PrevSpec, DiagID,
-                                 getLangOpts(), /*IsTypeSpec*/false);
+                                 getLangOpts());
       break;
 
     // OpenCL qualifiers:
@@ -4344,7 +4393,7 @@ void Parser::ParseDirectDeclarator(Declarator &D) {
     D.SetIdentifier(0, Tok.getLocation());
   } else {
     if (Tok.getKind() == tok::annot_pragma_parser_crash)
-      *(volatile int*) 0x11 = 0;
+      LLVM_BUILTIN_TRAP;
     if (D.getContext() == Declarator::MemberContext)
       Diag(Tok, diag::err_expected_member_name_or_semi)
         << D.getDeclSpec().getSourceRange();
@@ -4374,9 +4423,15 @@ void Parser::ParseDirectDeclarator(Declarator &D) {
       // In such a case, check if we actually have a function declarator; if it
       // is not, the declarator has been fully parsed.
       bool IsAmbiguous = false;
-      if (getLangOpts().CPlusPlus && D.mayBeFollowedByCXXDirectInit() &&
-          !isCXXFunctionDeclarator(&IsAmbiguous))
-        break;
+      if (getLangOpts().CPlusPlus && D.mayBeFollowedByCXXDirectInit()) {
+        // The name of the declarator, if any, is tentatively declared within
+        // a possible direct initializer.
+        TentativelyDeclaredIdentifiers.push_back(D.getIdentifier());
+        bool IsFunctionDecl = isCXXFunctionDeclarator(&IsAmbiguous);
+        TentativelyDeclaredIdentifiers.pop_back();
+        if (!IsFunctionDecl)
+          break;
+      }
       ParsedAttributes attrs(AttrFactory);
       BalancedDelimiterTracker T(*this, tok::l_paren);
       T.consumeOpen();
@@ -4553,7 +4608,14 @@ void Parser::ParseFunctionDeclarator(Declarator &D,
 
   Actions.ActOnStartFunctionDeclarator();
 
-  SourceLocation EndLoc;
+  /* LocalEndLoc is the end location for the local FunctionTypeLoc.
+     EndLoc is the end location for the function declarator.
+     They differ for trailing return types. */
+  SourceLocation StartLoc, LocalEndLoc, EndLoc;
+  SourceLocation LParenLoc, RParenLoc;
+  LParenLoc = Tracker.getOpenLocation();
+  StartLoc = LParenLoc;
+
   if (isFunctionDeclaratorIdentifierList()) {
     if (RequiresArg)
       Diag(Tok, diag::err_argument_required_after_attribute);
@@ -4561,7 +4623,9 @@ void Parser::ParseFunctionDeclarator(Declarator &D,
     ParseFunctionDeclaratorIdentifierList(D, ParamInfo);
 
     Tracker.consumeClose();
-    EndLoc = Tracker.getCloseLocation();
+    RParenLoc = Tracker.getCloseLocation();
+    LocalEndLoc = RParenLoc;
+    EndLoc = RParenLoc;
   } else {
     if (Tok.isNot(tok::r_paren))
       ParseParameterDeclarationClause(D, FirstArgAttrs, ParamInfo, EllipsisLoc);
@@ -4572,7 +4636,9 @@ void Parser::ParseFunctionDeclarator(Declarator &D,
 
     // If we have the closing ')', eat it.
     Tracker.consumeClose();
-    EndLoc = Tracker.getCloseLocation();
+    RParenLoc = Tracker.getCloseLocation();
+    LocalEndLoc = RParenLoc;
+    EndLoc = RParenLoc;
 
     if (getLangOpts().CPlusPlus) {
       // FIXME: Accept these components in any order, and produce fixits to
@@ -4628,21 +4694,25 @@ void Parser::ParseFunctionDeclarator(Declarator &D,
       MaybeParseCXX0XAttributes(FnAttrs);
 
       // Parse trailing-return-type[opt].
+      LocalEndLoc = EndLoc;
       if (getLangOpts().CPlusPlus0x && Tok.is(tok::arrow)) {
         Diag(Tok, diag::warn_cxx98_compat_trailing_return_type);
+        if (D.getDeclSpec().getTypeSpecType() == TST_auto)
+          StartLoc = D.getDeclSpec().getTypeSpecTypeLoc();
+        LocalEndLoc = Tok.getLocation();
         SourceRange Range;
         TrailingReturnType = ParseTrailingReturnType(Range);
-        if (Range.getEnd().isValid())
-          EndLoc = Range.getEnd();
+        EndLoc = Range.getEnd();
       }
     }
   }
 
   // Remember that we parsed a function type, and remember the attributes.
   D.AddTypeInfo(DeclaratorChunk::getFunction(HasProto,
-                                             /*isVariadic=*/EllipsisLoc.isValid(),
-                                             IsAmbiguous, EllipsisLoc,
+                                             IsAmbiguous,
+                                             LParenLoc,
                                              ParamInfo.data(), ParamInfo.size(),
+                                             EllipsisLoc, RParenLoc,
                                              DS.getTypeQualifiers(),
                                              RefQualifierIsLValueRef,
                                              RefQualifierLoc, ConstQualifierLoc,
@@ -4654,8 +4724,7 @@ void Parser::ParseFunctionDeclarator(Declarator &D,
                                              DynamicExceptions.size(),
                                              NoexceptExpr.isUsable() ?
                                                NoexceptExpr.get() : 0,
-                                             Tracker.getOpenLocation(),
-                                             EndLoc, D,
+                                             StartLoc, LocalEndLoc, D,
                                              TrailingReturnType),
                 FnAttrs, EndLoc);
 
@@ -5058,7 +5127,8 @@ void Parser::ParseTypeofSpecifier(DeclSpec &DS) {
 
   const bool hasParens = Tok.is(tok::l_paren);
 
-  EnterExpressionEvaluationContext Unevaluated(Actions, Sema::Unevaluated);
+  EnterExpressionEvaluationContext Unevaluated(Actions, Sema::Unevaluated,
+                                               Sema::ReuseLambdaContextDecl);
 
   bool isCastExpr;
   ParsedType CastTy;
diff --git a/lib/Parse/ParseDeclCXX.cpp b/lib/Parse/ParseDeclCXX.cpp
index 3dc96cf..4cb14e2 100644
--- a/lib/Parse/ParseDeclCXX.cpp
+++ b/lib/Parse/ParseDeclCXX.cpp
@@ -18,6 +18,7 @@
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/ParsedTemplate.h"
 #include "clang/Sema/PrettyDeclStackTrace.h"
+#include "clang/Sema/SemaDiagnostic.h"
 #include "llvm/ADT/SmallString.h"
 #include "RAIIObjectsForParser.h"
 using namespace clang;
@@ -87,6 +88,12 @@ Decl *Parser::ParseNamespace(unsigned Context,
   }
 
   if (Tok.is(tok::equal)) {
+    if (Ident == 0) {
+      Diag(Tok, diag::err_expected_ident);
+      // Skip to end of the definition and eat the ';'.
+      SkipUntil(tok::semi);
+      return 0;
+    }
     if (!attrs.empty())
       Diag(attrTok, diag::err_unexpected_namespace_attributes_alias);
     if (InlineLoc.isValid())
@@ -581,7 +588,7 @@ Decl *Parser::ParseUsingDeclaration(unsigned Context,
 
   if (IsAliasDecl) {
     TemplateParameterLists *TemplateParams = TemplateInfo.TemplateParams;
-    MultiTemplateParamsArg TemplateParamsArg(Actions,
+    MultiTemplateParamsArg TemplateParamsArg(
       TemplateParams ? TemplateParams->data() : 0,
       TemplateParams ? TemplateParams->size() : 0);
     // FIXME: Propagate attributes.
@@ -616,12 +623,13 @@ Decl *Parser::ParseStaticAssertDeclaration(SourceLocation &DeclEnd){
   BalancedDelimiterTracker T(*this, tok::l_paren);
   if (T.consumeOpen()) {
     Diag(Tok, diag::err_expected_lparen);
+    SkipMalformedDecl();
     return 0;
   }
 
   ExprResult AssertExpr(ParseConstantExpression());
   if (AssertExpr.isInvalid()) {
-    SkipUntil(tok::semi);
+    SkipMalformedDecl();
     return 0;
   }
 
@@ -630,13 +638,13 @@ Decl *Parser::ParseStaticAssertDeclaration(SourceLocation &DeclEnd){
 
   if (!isTokenStringLiteral()) {
     Diag(Tok, diag::err_expected_string_literal);
-    SkipUntil(tok::semi);
+    SkipMalformedDecl();
     return 0;
   }
 
   ExprResult AssertMessage(ParseStringLiteralExpression());
   if (AssertMessage.isInvalid()) {
-    SkipUntil(tok::semi);
+    SkipMalformedDecl();
     return 0;
   }
 
@@ -694,9 +702,22 @@ SourceLocation Parser::ParseDecltypeSpecifier(DeclSpec &DS) {
                                                  0, /*IsDecltype=*/true);
     Result = ParseExpression();
     if (Result.isInvalid()) {
-      SkipUntil(tok::r_paren);
       DS.SetTypeSpecError();
-      return StartLoc;
+      if (SkipUntil(tok::r_paren, /*StopAtSemi=*/true, /*DontConsume=*/true)) {
+        EndLoc = ConsumeParen();
+      } else {
+        assert(Tok.is(tok::semi));
+        if (PP.isBacktrackEnabled()) {
+          // Backtrack to get the location of the last token before the semi.
+          PP.RevertCachedTokens(2);
+          ConsumeToken(); // the semi.
+          EndLoc = ConsumeAnyToken();
+          assert(Tok.is(tok::semi));
+        } else {
+          EndLoc = Tok.getLocation();
+        }
+      }
+      return EndLoc;
     }
 
     // Match the ')'
@@ -1034,6 +1055,8 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
   DeclSpec::TST TagType;
   if (TagTokKind == tok::kw_struct)
     TagType = DeclSpec::TST_struct;
+  else if (TagTokKind == tok::kw___interface)
+    TagType = DeclSpec::TST_interface;
   else if (TagTokKind == tok::kw_class)
     TagType = DeclSpec::TST_class;
   else {
@@ -1151,7 +1174,8 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
         << (TemplateInfo.Kind == ParsedTemplateInfo::ExplicitInstantiation)
         << (TagType == DeclSpec::TST_class? 0
             : TagType == DeclSpec::TST_struct? 1
-            : 2)
+            : TagType == DeclSpec::TST_interface? 2
+            : 3)
         << Name
         << SourceRange(LAngleLoc, RAngleLoc);
 
@@ -1243,8 +1267,7 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
     if (Tok.isNot(tok::semi)) {
       // A semicolon was missing after this declaration. Diagnose and recover.
       ExpectAndConsume(tok::semi, diag::err_expected_semi_after_tagdecl,
-                       TagType == DeclSpec::TST_class ? "class" :
-                       TagType == DeclSpec::TST_struct ? "struct" : "union");
+        DeclSpec::getSpecifierName(TagType));
       PP.EnterToken(Tok);
       Tok.setKind(tok::semi);
     }
@@ -1279,8 +1302,7 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
   if (TemplateId) {
     // Explicit specialization, class template partial specialization,
     // or explicit instantiation.
-    ASTTemplateArgsPtr TemplateArgsPtr(Actions,
-                                       TemplateId->getTemplateArgs(),
+    ASTTemplateArgsPtr TemplateArgsPtr(TemplateId->getTemplateArgs(),
                                        TemplateId->NumArgs);
     if (TemplateInfo.Kind == ParsedTemplateInfo::ExplicitInstantiation &&
         TUK == Sema::TUK_Declaration) {
@@ -1362,7 +1384,7 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
                        TemplateArgsPtr,
                        TemplateId->RAngleLoc,
                        attrs.getList(),
-                       MultiTemplateParamsArg(Actions,
+                       MultiTemplateParamsArg(
                                     TemplateParams? &(*TemplateParams)[0] : 0,
                                  TemplateParams? TemplateParams->size() : 0));
     }
@@ -1389,7 +1411,7 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
       Actions.ActOnTemplatedFriendTag(getCurScope(), DS.getFriendSpecLoc(),
                                       TagType, StartLoc, SS,
                                       Name, NameLoc, attrs.getList(),
-                                      MultiTemplateParamsArg(Actions,
+                                      MultiTemplateParamsArg(
                                     TemplateParams? &(*TemplateParams)[0] : 0,
                                  TemplateParams? TemplateParams->size() : 0));
   } else {
@@ -1470,8 +1492,7 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
   if (TUK == Sema::TUK_Definition &&
       (TemplateInfo.Kind || !isValidAfterTypeSpecifier(false))) {
     ExpectAndConsume(tok::semi, diag::err_expected_semi_after_tagdecl,
-                     TagType == DeclSpec::TST_class ? "class" :
-                     TagType == DeclSpec::TST_struct ? "struct" : "union");
+      DeclSpec::getSpecifierName(TagType));
     // Push this token back into the preprocessor and change our current token
     // to ';' so that the rest of the code recovers as though there were an
     // ';' after the definition.
@@ -1667,7 +1688,8 @@ VirtSpecifiers::Specifier Parser::isCXX0XVirtSpecifier(const Token &Tok) const {
 ///       virt-specifier-seq:
 ///         virt-specifier
 ///         virt-specifier-seq virt-specifier
-void Parser::ParseOptionalCXX0XVirtSpecifierSeq(VirtSpecifiers &VS) {
+void Parser::ParseOptionalCXX0XVirtSpecifierSeq(VirtSpecifiers &VS,
+                                                bool IsInterface) {
   while (true) {
     VirtSpecifiers::Specifier Specifier = isCXX0XVirtSpecifier();
     if (Specifier == VirtSpecifiers::VS_None)
@@ -1681,10 +1703,15 @@ void Parser::ParseOptionalCXX0XVirtSpecifierSeq(VirtSpecifiers &VS) {
         << PrevSpec
         << FixItHint::CreateRemoval(Tok.getLocation());
 
-    Diag(Tok.getLocation(), getLangOpts().CPlusPlus0x ?
-         diag::warn_cxx98_compat_override_control_keyword :
-         diag::ext_override_control_keyword)
-      << VirtSpecifiers::getSpecifierName(Specifier);
+    if (IsInterface && Specifier == VirtSpecifiers::VS_Final) {
+      Diag(Tok.getLocation(), diag::err_override_control_interface)
+        << VirtSpecifiers::getSpecifierName(Specifier);
+    } else {
+      Diag(Tok.getLocation(), getLangOpts().CPlusPlus0x ?
+           diag::warn_cxx98_compat_override_control_keyword :
+           diag::ext_override_control_keyword)
+        << VirtSpecifiers::getSpecifierName(Specifier);
+    }
     ConsumeToken();
   }
 }
@@ -1869,7 +1896,7 @@ void Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS,
   ParseDeclarationSpecifiers(DS, TemplateInfo, AS, DSC_class,
                              &CommonLateParsedAttrs);
 
-  MultiTemplateParamsArg TemplateParams(Actions,
+  MultiTemplateParamsArg TemplateParams(
       TemplateInfo.TemplateParams? TemplateInfo.TemplateParams->data() : 0,
       TemplateInfo.TemplateParams? TemplateInfo.TemplateParams->size() : 0);
 
@@ -1905,7 +1932,7 @@ void Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS,
       return;
     }
 
-    ParseOptionalCXX0XVirtSpecifierSeq(VS);
+    ParseOptionalCXX0XVirtSpecifierSeq(VS, getCurrentClass().IsInterface);
 
     // If attributes exist after the declarator, but before an '{', parse them.
     MaybeParseGNUAttributes(DeclaratorInfo, &LateParsedAttrs);
@@ -2026,7 +2053,7 @@ void Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS,
 
     // FIXME: When g++ adds support for this, we'll need to check whether it
     // goes before or after the GNU attributes and __asm__.
-    ParseOptionalCXX0XVirtSpecifierSeq(VS);
+    ParseOptionalCXX0XVirtSpecifierSeq(VS, getCurrentClass().IsInterface);
 
     InClassInitStyle HasInClassInit = ICIS_NoInit;
     if ((Tok.is(tok::equal) || Tok.is(tok::l_brace)) && !HasInitializer) {
@@ -2052,11 +2079,11 @@ void Parser::ParseCXXClassMemberDeclaration(AccessSpecifier AS,
     if (DS.isFriendSpecified()) {
       // TODO: handle initializers, bitfields, 'delete'
       ThisDecl = Actions.ActOnFriendFunctionDecl(getCurScope(), DeclaratorInfo,
-                                                 move(TemplateParams));
+                                                 TemplateParams);
     } else {
       ThisDecl = Actions.ActOnCXXMemberDeclarator(getCurScope(), AS,
                                                   DeclaratorInfo,
-                                                  move(TemplateParams),
+                                                  TemplateParams,
                                                   BitfieldSize.release(),
                                                   VS, HasInClassInit);
       if (AccessAttrs)
@@ -2240,6 +2267,7 @@ ExprResult Parser::ParseCXXMemberInitializer(Decl *D, bool IsFunction,
 void Parser::ParseCXXMemberSpecification(SourceLocation RecordLoc,
                                          unsigned TagType, Decl *TagDecl) {
   assert((TagType == DeclSpec::TST_struct ||
+         TagType == DeclSpec::TST_interface ||
          TagType == DeclSpec::TST_union  ||
          TagType == DeclSpec::TST_class) && "Invalid TagType!");
 
@@ -2254,6 +2282,15 @@ void Parser::ParseCXXMemberSpecification(SourceLocation RecordLoc,
       if (S->isClassScope()) {
         // We're inside a class scope, so this is a nested class.
         NonNestedClass = false;
+
+        // The Microsoft extension __interface does not permit nested classes.
+        if (getCurrentClass().IsInterface) {
+          Diag(RecordLoc, diag::err_invalid_member_in_interface)
+            << /*ErrorType=*/6
+            << (isa<NamedDecl>(TagDecl)
+                  ? cast<NamedDecl>(TagDecl)->getQualifiedNameAsString()
+                  : "<anonymous>");
+        }
         break;
       }
 
@@ -2274,7 +2311,8 @@ void Parser::ParseCXXMemberSpecification(SourceLocation RecordLoc,
   ParseScope ClassScope(this, Scope::ClassScope|Scope::DeclScope);
 
   // Note that we are parsing a new (potentially-nested) class definition.
-  ParsingClassDefinition ParsingDef(*this, TagDecl, NonNestedClass);
+  ParsingClassDefinition ParsingDef(*this, TagDecl, NonNestedClass,
+                                    TagType == DeclSpec::TST_interface);
 
   if (TagDecl)
     Actions.ActOnTagStartDefinition(getCurScope(), TagDecl);
@@ -2286,9 +2324,14 @@ void Parser::ParseCXXMemberSpecification(SourceLocation RecordLoc,
     assert(isCXX0XFinalKeyword() && "not a class definition");
     FinalLoc = ConsumeToken();
 
-    Diag(FinalLoc, getLangOpts().CPlusPlus0x ?
-         diag::warn_cxx98_compat_override_control_keyword :
-         diag::ext_override_control_keyword) << "final";
+    if (TagType == DeclSpec::TST_interface) {
+      Diag(FinalLoc, diag::err_override_control_interface)
+        << "final";
+    } else {
+      Diag(FinalLoc, getLangOpts().CPlusPlus0x ?
+           diag::warn_cxx98_compat_override_control_keyword :
+           diag::ext_override_control_keyword) << "final";
+    }
   }
 
   if (Tok.is(tok::colon)) {
@@ -2348,6 +2391,11 @@ void Parser::ParseCXXMemberSpecification(SourceLocation RecordLoc,
         continue;
       }
 
+      if (Tok.is(tok::annot_pragma_align)) {
+        HandlePragmaAlign();
+        continue;
+      }
+
       AccessSpecifier AS = getAccessSpecifierIfPresent();
       if (AS != AS_none) {
         // Current token is a C++ access specifier.
@@ -2373,6 +2421,13 @@ void Parser::ParseCXXMemberSpecification(SourceLocation RecordLoc,
             << FixItHint::CreateInsertion(EndLoc, ":");
         }
 
+        // The Microsoft extension __interface does not permit non-public
+        // access specifiers.
+        if (TagType == DeclSpec::TST_interface && CurAS != AS_public) {
+          Diag(ASLoc, diag::err_access_specifier_interface)
+            << (CurAS == AS_protected);
+        }
+
         if (Actions.ActOnAccessSpecifier(AS, ASLoc, EndLoc,
                                          AccessAttrs.getList())) {
           // found another attribute than only annotations
@@ -2571,7 +2626,7 @@ Parser::MemInitResult Parser::ParseMemInitializer(Decl *ConstructorDecl) {
     T.consumeOpen();
 
     // Parse the optional expression-list.
-    ExprVector ArgExprs(Actions);
+    ExprVector ArgExprs;
     CommaLocsTy CommaLocs;
     if (Tok.isNot(tok::r_paren) && ParseExpressionList(ArgExprs, CommaLocs)) {
       SkipUntil(tok::r_paren);
@@ -2586,7 +2641,7 @@ Parser::MemInitResult Parser::ParseMemInitializer(Decl *ConstructorDecl) {
 
     return Actions.ActOnMemInitializer(ConstructorDecl, getCurScope(), SS, II,
                                        TemplateTypeTy, DS, IdLoc,
-                                       T.getOpenLocation(), ArgExprs.take(),
+                                       T.getOpenLocation(), ArgExprs.data(),
                                        ArgExprs.size(), T.getCloseLocation(),
                                        EllipsisLoc);
   }
@@ -2752,10 +2807,11 @@ TypeResult Parser::ParseTrailingReturnType(SourceRange &Range) {
 /// so push that class onto our stack of classes that is currently
 /// being parsed.
 Sema::ParsingClassState
-Parser::PushParsingClass(Decl *ClassDecl, bool NonNestedClass) {
+Parser::PushParsingClass(Decl *ClassDecl, bool NonNestedClass,
+                         bool IsInterface) {
   assert((NonNestedClass || !ClassStack.empty()) &&
          "Nested class without outer class");
-  ClassStack.push(new ParsingClass(ClassDecl, NonNestedClass));
+  ClassStack.push(new ParsingClass(ClassDecl, NonNestedClass, IsInterface));
   return Actions.PushParsingClass();
 }
 
@@ -2773,9 +2829,6 @@ void Parser::DeallocateParsedClasses(Parser::ParsingClass *Class) {
 /// This routine should be called when we have finished parsing the
 /// definition of a class, but have not yet popped the Scope
 /// associated with the class's definition.
-///
-/// \returns true if the class we've popped is a top-level class,
-/// false otherwise.
 void Parser::PopParsingClass(Sema::ParsingClassState state) {
   assert(!ClassStack.empty() && "Mismatched push/pop for class parsing");
 
@@ -2850,6 +2903,21 @@ IdentifierInfo *Parser::TryParseCXX11AttributeIdentifier(SourceLocation &Loc) {
   }
 }
 
+static bool IsBuiltInOrStandardCXX11Attribute(IdentifierInfo *AttrName,
+                                               IdentifierInfo *ScopeName) {
+  switch (AttributeList::getKind(AttrName, ScopeName,
+                                 AttributeList::AS_CXX11)) {
+  case AttributeList::AT_CarriesDependency:
+  case AttributeList::AT_FallThrough:
+  case AttributeList::AT_NoReturn: {
+    return true;
+  }
+
+  default:
+    return false;
+  }
+}
+
 /// ParseCXX11AttributeSpecifier - Parse a C++11 attribute-specifier. Currently
 /// only parses standard attributes.
 ///
@@ -2934,46 +3002,38 @@ void Parser::ParseCXX11AttributeSpecifier(ParsedAttributes &attrs,
       }
     }
 
+    bool StandardAttr = IsBuiltInOrStandardCXX11Attribute(AttrName,ScopeName);
     bool AttrParsed = false;
-    switch (AttributeList::getKind(AttrName, ScopeName,
-                                   AttributeList::AS_CXX11)) {
-    // No arguments
-    case AttributeList::AT_CarriesDependency:
-    // FIXME: implement generic support of attributes with C++11 syntax
-    // see Parse/ParseDecl.cpp: ParseGNUAttributes
-    case AttributeList::AT_FallThrough:
-    case AttributeList::AT_NoReturn: {
-      if (Tok.is(tok::l_paren)) {
-        Diag(Tok.getLocation(), diag::err_cxx11_attribute_forbids_arguments)
-          << AttrName->getName();
-        break;
+
+    // Parse attribute arguments
+    if (Tok.is(tok::l_paren)) {
+      if (ScopeName && ScopeName->getName() == "gnu") {
+        ParseGNUAttributeArgs(AttrName, AttrLoc, attrs, endLoc,
+                              ScopeName, ScopeLoc, AttributeList::AS_CXX11);
+        AttrParsed = true;
+      } else {
+        if (StandardAttr)
+          Diag(Tok.getLocation(), diag::err_cxx11_attribute_forbids_arguments)
+            << AttrName->getName();
+
+        // FIXME: handle other formats of c++11 attribute arguments
+        ConsumeParen();
+        SkipUntil(tok::r_paren, false);
       }
+    }
 
+    if (!AttrParsed)
       attrs.addNew(AttrName,
                    SourceRange(ScopeLoc.isValid() ? ScopeLoc : AttrLoc,
                                AttrLoc),
                    ScopeName, ScopeLoc, 0,
                    SourceLocation(), 0, 0, AttributeList::AS_CXX11);
-      AttrParsed = true;
-      break;
-    }
-
-    // Silence warnings
-    default: break;
-    }
-
-    // Skip the entire parameter clause, if any
-    if (!AttrParsed && Tok.is(tok::l_paren)) {
-      ConsumeParen();
-      // SkipUntil maintains the balancedness of tokens.
-      SkipUntil(tok::r_paren, false);
-    }
 
     if (Tok.is(tok::ellipsis)) {
-      if (AttrParsed)
-        Diag(Tok, diag::err_cxx11_attribute_forbids_ellipsis)
-          << AttrName->getName();
       ConsumeToken();
+
+      Diag(Tok, diag::err_cxx11_attribute_forbids_ellipsis)
+        << AttrName->getName();
     }
   }
 
diff --git a/lib/Parse/ParseExpr.cpp b/lib/Parse/ParseExpr.cpp
index 8d4668b..c7be0d3 100644
--- a/lib/Parse/ParseExpr.cpp
+++ b/lib/Parse/ParseExpr.cpp
@@ -179,7 +179,7 @@ static prec::Level getBinOpPrecedence(tok::TokenKind Kind,
 /// \endverbatim
 ExprResult Parser::ParseExpression(TypeCastState isTypeCast) {
   ExprResult LHS(ParseAssignmentExpression(isTypeCast));
-  return ParseRHSOfBinaryExpression(move(LHS), prec::Comma);
+  return ParseRHSOfBinaryExpression(LHS, prec::Comma);
 }
 
 /// This routine is called when the '@' is seen and consumed.
@@ -190,7 +190,7 @@ ExprResult Parser::ParseExpression(TypeCastState isTypeCast) {
 ExprResult
 Parser::ParseExpressionWithLeadingAt(SourceLocation AtLoc) {
   ExprResult LHS(ParseObjCAtExpression(AtLoc));
-  return ParseRHSOfBinaryExpression(move(LHS), prec::Comma);
+  return ParseRHSOfBinaryExpression(LHS, prec::Comma);
 }
 
 /// This routine is called when a leading '__extension__' is seen and
@@ -210,7 +210,7 @@ Parser::ParseExpressionWithLeadingExtension(SourceLocation ExtLoc) {
     LHS = Actions.ActOnUnaryOp(getCurScope(), ExtLoc, tok::kw___extension__,
                                LHS.take());
 
-  return ParseRHSOfBinaryExpression(move(LHS), prec::Comma);
+  return ParseRHSOfBinaryExpression(LHS, prec::Comma);
 }
 
 /// \brief Parse an expr that doesn't include (top-level) commas.
@@ -227,7 +227,7 @@ ExprResult Parser::ParseAssignmentExpression(TypeCastState isTypeCast) {
   ExprResult LHS = ParseCastExpression(/*isUnaryExpression=*/false,
                                        /*isAddressOfOperand=*/false,
                                        isTypeCast);
-  return ParseRHSOfBinaryExpression(move(LHS), prec::Assignment);
+  return ParseRHSOfBinaryExpression(LHS, prec::Assignment);
 }
 
 /// \brief Parse an assignment expression where part of an Objective-C message
@@ -265,6 +265,17 @@ ExprResult Parser::ParseConstantExpression(TypeCastState isTypeCast) {
   return Actions.ActOnConstantExpression(Res);
 }
 
+bool Parser::isNotExpressionStart() {
+  tok::TokenKind K = Tok.getKind();
+  if (K == tok::l_brace || K == tok::r_brace  ||
+      K == tok::kw_for  || K == tok::kw_while ||
+      K == tok::kw_if   || K == tok::kw_else  ||
+      K == tok::kw_goto || K == tok::kw_try)
+    return true;
+  // If this is a decl-specifier, we can't be at the start of an expression.
+  return isKnownToBeDeclarationSpecifier();
+}
+
 /// \brief Parse a binary expression that starts with \p LHS and has a
 /// precedence of at least \p MinPrec.
 ExprResult
@@ -279,12 +290,23 @@ Parser::ParseRHSOfBinaryExpression(ExprResult LHS, prec::Level MinPrec) {
     // because we are called recursively, or because the token is not a binop),
     // then we are done!
     if (NextTokPrec < MinPrec)
-      return move(LHS);
+      return LHS;
 
     // Consume the operator, saving the operator token for error reporting.
     Token OpToken = Tok;
     ConsumeToken();
 
+    // Bail out when encountering a comma followed by a token which can't
+    // possibly be the start of an expression. For instance:
+    //   int f() { return 1, }
+    // We can't do this before consuming the comma, because
+    // isNotExpressionStart() looks at the token stream.
+    if (OpToken.is(tok::comma) && isNotExpressionStart()) {
+      PP.EnterToken(Tok);
+      Tok = OpToken;
+      return LHS;
+    }
+
     // Special case handling for the ternary operator.
     ExprResult TernaryMiddle(true);
     if (NextTokPrec == prec::Conditional) {
@@ -458,7 +480,7 @@ ExprResult Parser::ParseCastExpression(bool isUnaryExpression,
                                        isTypeCast);
   if (NotCastExpr)
     Diag(Tok, diag::err_expected_expression);
-  return move(Res);
+  return Res;
 }
 
 namespace {
@@ -698,7 +720,7 @@ ExprResult Parser::ParseCastExpression(bool isUnaryExpression,
     case CastExpr:
       // We have parsed the cast-expression and no postfix-expr pieces are
       // following.
-      return move(Res);
+      return Res;
     }
 
     break;
@@ -741,6 +763,53 @@ ExprResult Parser::ParseCastExpression(bool isUnaryExpression,
       // Avoid the unnecessary parse-time lookup in the common case
       // where the syntax forbids a type.
       const Token &Next = NextToken();
+
+      // If this identifier was reverted from a token ID, and the next token
+      // is a parenthesis, this is likely to be a use of a type trait. Check
+      // those tokens.
+      if (Next.is(tok::l_paren) &&
+          Tok.is(tok::identifier) &&
+          Tok.getIdentifierInfo()->hasRevertedTokenIDToIdentifier()) {
+        IdentifierInfo *II = Tok.getIdentifierInfo();
+        // Build up the mapping of revertable type traits, for future use.
+        if (RevertableTypeTraits.empty()) {
+#define RTT_JOIN(X,Y) X##Y
+#define REVERTABLE_TYPE_TRAIT(Name)                         \
+          RevertableTypeTraits[PP.getIdentifierInfo(#Name)] \
+            = RTT_JOIN(tok::kw_,Name)
+
+          REVERTABLE_TYPE_TRAIT(__is_arithmetic);
+          REVERTABLE_TYPE_TRAIT(__is_convertible);
+          REVERTABLE_TYPE_TRAIT(__is_empty);
+          REVERTABLE_TYPE_TRAIT(__is_floating_point);
+          REVERTABLE_TYPE_TRAIT(__is_function);
+          REVERTABLE_TYPE_TRAIT(__is_fundamental);
+          REVERTABLE_TYPE_TRAIT(__is_integral);
+          REVERTABLE_TYPE_TRAIT(__is_member_function_pointer);
+          REVERTABLE_TYPE_TRAIT(__is_member_pointer);
+          REVERTABLE_TYPE_TRAIT(__is_pod);
+          REVERTABLE_TYPE_TRAIT(__is_pointer);
+          REVERTABLE_TYPE_TRAIT(__is_same);
+          REVERTABLE_TYPE_TRAIT(__is_scalar);
+          REVERTABLE_TYPE_TRAIT(__is_signed);
+          REVERTABLE_TYPE_TRAIT(__is_unsigned);
+          REVERTABLE_TYPE_TRAIT(__is_void);
+#undef REVERTABLE_TYPE_TRAIT
+#undef RTT_JOIN
+          }
+
+          // If we find that this is in fact the name of a type trait,
+          // update the token kind in place and parse again to treat it as
+          // the appropriate kind of type trait.
+          llvm::SmallDenseMap<IdentifierInfo *, tok::TokenKind>::iterator Known
+            = RevertableTypeTraits.find(II);
+          if (Known != RevertableTypeTraits.end()) {
+            Tok.setKind(Known->second);
+            return ParseCastExpression(isUnaryExpression, isAddressOfOperand,
+                                       NotCastExpr, isTypeCast);
+          }
+        }
+
       if (Next.is(tok::coloncolon) ||
           (!ColonIsSacred && Next.is(tok::colon)) ||
           Next.is(tok::less) ||
@@ -758,7 +827,7 @@ ExprResult Parser::ParseCastExpression(bool isUnaryExpression,
     // '.'.
     IdentifierInfo &II = *Tok.getIdentifierInfo();
     SourceLocation ILoc = ConsumeToken();
-    
+
     // Support 'Class.property' and 'super.property' notation.
     if (getLangOpts().ObjC1 && Tok.is(tok::period) &&
         (Actions.getTypeName(II, ILoc, getCurScope()) ||
@@ -888,7 +957,7 @@ ExprResult Parser::ParseCastExpression(bool isUnaryExpression,
     Res = ParseCastExpression(!getLangOpts().CPlusPlus);
     if (!Res.isInvalid())
       Res = Actions.ActOnUnaryOp(getCurScope(), SavedLoc, SavedKind, Res.get());
-    return move(Res);
+    return Res;
   }
   case tok::amp: {         // unary-expression: '&' cast-expression
     // Special treatment because of member pointers
@@ -896,7 +965,7 @@ ExprResult Parser::ParseCastExpression(bool isUnaryExpression,
     Res = ParseCastExpression(false, true);
     if (!Res.isInvalid())
       Res = Actions.ActOnUnaryOp(getCurScope(), SavedLoc, SavedKind, Res.get());
-    return move(Res);
+    return Res;
   }
 
   case tok::star:          // unary-expression: '*' cast-expression
@@ -910,7 +979,7 @@ ExprResult Parser::ParseCastExpression(bool isUnaryExpression,
     Res = ParseCastExpression(false);
     if (!Res.isInvalid())
       Res = Actions.ActOnUnaryOp(getCurScope(), SavedLoc, SavedKind, Res.get());
-    return move(Res);
+    return Res;
   }
 
   case tok::kw___extension__:{//unary-expression:'__extension__' cast-expr [GNU]
@@ -920,7 +989,7 @@ ExprResult Parser::ParseCastExpression(bool isUnaryExpression,
     Res = ParseCastExpression(false);
     if (!Res.isInvalid())
       Res = Actions.ActOnUnaryOp(getCurScope(), SavedLoc, SavedKind, Res.get());
-    return move(Res);
+    return Res;
   }
   case tok::kw__Alignof:   // unary-expression: '_Alignof' '(' type-name ')'
     if (!getLangOpts().C11)
@@ -946,7 +1015,7 @@ ExprResult Parser::ParseCastExpression(bool isUnaryExpression,
                                                 Tok.getLocation());
     Res = Actions.ActOnAddrLabel(AmpAmpLoc, Tok.getLocation(), LD);
     ConsumeToken();
-    return move(Res);
+    return Res;
   }
   case tok::kw_const_cast:
   case tok::kw_dynamic_cast:
@@ -1132,13 +1201,14 @@ ExprResult Parser::ParseCastExpression(bool isUnaryExpression,
     if (!Result.isInvalid())
       Result = Actions.ActOnNoexceptExpr(KeyLoc, T.getOpenLocation(), 
                                          Result.take(), T.getCloseLocation());
-    return move(Result);
+    return Result;
   }
 
   case tok::kw___is_abstract: // [GNU] unary-type-trait
   case tok::kw___is_class:
   case tok::kw___is_empty:
   case tok::kw___is_enum:
+  case tok::kw___is_interface_class:
   case tok::kw___is_literal:
   case tok::kw___is_arithmetic:
   case tok::kw___is_integral:
@@ -1270,7 +1340,7 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
     switch (Tok.getKind()) {
     case tok::code_completion:
       if (InMessageExpression)
-        return move(LHS);
+        return LHS;
         
       Actions.CodeCompletePostfixExpression(getCurScope(), LHS);
       cutOffParsing();
@@ -1290,7 +1360,7 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
       // Fall through; this isn't a message send.
                 
     default:  // Not a postfix-expression suffix.
-      return move(LHS);
+      return LHS;
     case tok::l_square: {  // postfix-expression: p-e '[' expression ']'
       // If we have a array postfix expression that starts on a new line and
       // Objective-C is enabled, it is highly likely that the user forgot a
@@ -1300,7 +1370,7 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
       // expression and recover by pretending there is no suffix.
       if (getLangOpts().ObjC1 && Tok.isAtStartOfLine() &&
           isSimpleObjCMessageExpression())
-        return move(LHS);
+        return LHS;
 
       // Reject array indices starting with a lambda-expression. '[[' is
       // reserved for attributes.
@@ -1341,7 +1411,7 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
       BalancedDelimiterTracker PT(*this, tok::l_paren);
 
       if (OpKind == tok::lesslessless) {
-        ExprVector ExecConfigExprs(Actions);
+        ExprVector ExecConfigExprs;
         CommaLocsTy ExecConfigCommaLocs;
         SourceLocation OpenLoc = ConsumeToken();
 
@@ -1372,7 +1442,7 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
         if (!LHS.isInvalid()) {
           ExprResult ECResult = Actions.ActOnCUDAExecConfigExpr(getCurScope(),
                                     OpenLoc, 
-                                    move_arg(ExecConfigExprs), 
+                                    ExecConfigExprs, 
                                     CloseLoc);
           if (ECResult.isInvalid())
             LHS = ExprError();
@@ -1384,7 +1454,7 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
         Loc = PT.getOpenLocation();
       }
 
-      ExprVector ArgExprs(Actions);
+      ExprVector ArgExprs;
       CommaLocsTy CommaLocs;
       
       if (Tok.is(tok::code_completion)) {
@@ -1414,7 +1484,7 @@ Parser::ParsePostfixExpressionSuffix(ExprResult LHS) {
                 ArgExprs.size()-1 == CommaLocs.size())&&
                "Unexpected number of commas!");
         LHS = Actions.ActOnCallExpr(getCurScope(), LHS.take(), Loc,
-                                    move_arg(ArgExprs), Tok.getLocation(),
+                                    ArgExprs, Tok.getLocation(),
                                     ExecConfig);
         PT.consumeClose();
       }
@@ -1583,7 +1653,7 @@ Parser::ParseExprAfterUnaryExprOrTypeTrait(const Token &OpTok,
 
   // If we get here, the operand to the typeof/sizeof/alignof was an expresion.
   isCastExpr = false;
-  return move(Operand);
+  return Operand;
 }
 
 
@@ -1653,7 +1723,8 @@ ExprResult Parser::ParseUnaryExprOrTypeTraitExpression() {
   if (OpTok.is(tok::kw_alignof) || OpTok.is(tok::kw__Alignof))
     Diag(OpTok, diag::warn_cxx98_compat_alignof);
 
-  EnterExpressionEvaluationContext Unevaluated(Actions, Sema::Unevaluated);
+  EnterExpressionEvaluationContext Unevaluated(Actions, Sema::Unevaluated,
+                                               Sema::ReuseLambdaContextDecl);
 
   bool isCastExpr;
   ParsedType CastTy;
@@ -1684,7 +1755,7 @@ ExprResult Parser::ParseUnaryExprOrTypeTraitExpression() {
                                                     /*isType=*/false,
                                                     Operand.release(),
                                                     CastRange);
-  return move(Operand);
+  return Operand;
 }
 
 /// ParseBuiltinPrimaryExpression
@@ -1796,7 +1867,7 @@ ExprResult Parser::ParseBuiltinPrimaryExpression() {
         Res = ParseExpression();
         if (Res.isInvalid()) {
           SkipUntil(tok::r_paren);
-          return move(Res);
+          return Res;
         }
         Comps.back().U.E = Res.release();
 
@@ -1823,7 +1894,7 @@ ExprResult Parser::ParseBuiltinPrimaryExpression() {
     ExprResult Cond(ParseAssignmentExpression());
     if (Cond.isInvalid()) {
       SkipUntil(tok::r_paren);
-      return move(Cond);
+      return Cond;
     }
     if (ExpectAndConsume(tok::comma, diag::err_expected_comma, "",tok::r_paren))
       return ExprError();
@@ -1831,7 +1902,7 @@ ExprResult Parser::ParseBuiltinPrimaryExpression() {
     ExprResult Expr1(ParseAssignmentExpression());
     if (Expr1.isInvalid()) {
       SkipUntil(tok::r_paren);
-      return move(Expr1);
+      return Expr1;
     }
     if (ExpectAndConsume(tok::comma, diag::err_expected_comma, "",tok::r_paren))
       return ExprError();
@@ -1839,7 +1910,7 @@ ExprResult Parser::ParseBuiltinPrimaryExpression() {
     ExprResult Expr2(ParseAssignmentExpression());
     if (Expr2.isInvalid()) {
       SkipUntil(tok::r_paren);
-      return move(Expr2);
+      return Expr2;
     }
     if (Tok.isNot(tok::r_paren)) {
       Diag(Tok, diag::err_expected_rparen);
@@ -2083,7 +2154,7 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
                                          DeclaratorInfo, CastTy, 
                                          RParenLoc, Result.take());
         }
-        return move(Result);
+        return Result;
       }
 
       Diag(Tok, diag::err_expected_lbrace_in_compound_literal);
@@ -2093,13 +2164,13 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
     // Parse the expression-list.
     InMessageExpressionRAIIObject InMessage(*this, false);
     
-    ExprVector ArgExprs(Actions);
+    ExprVector ArgExprs;
     CommaLocsTy CommaLocs;
 
     if (!ParseExpressionList(ArgExprs, CommaLocs)) {
       ExprType = SimpleExpr;
       Result = Actions.ActOnParenListExpr(OpenLoc, Tok.getLocation(),
-                                          move_arg(ArgExprs));
+                                          ArgExprs);
     }
   } else {
     InMessageExpressionRAIIObject InMessage(*this, false);
@@ -2120,7 +2191,7 @@ Parser::ParseParenExpression(ParenParseOption &ExprType, bool stopIfCastExpr,
 
   T.consumeClose();
   RParenLoc = T.getCloseLocation();
-  return move(Result);
+  return Result;
 }
 
 /// ParseCompoundLiteralExpression - We have parsed the parenthesized type-name
@@ -2141,7 +2212,7 @@ Parser::ParseCompoundLiteralExpression(ParsedType Ty,
   ExprResult Result = ParseInitializer();
   if (!Result.isInvalid() && Ty)
     return Actions.ActOnCompoundLiteral(LParenLoc, Ty, RParenLoc, Result.take());
-  return move(Result);
+  return Result;
 }
 
 /// ParseStringLiteralExpression - This handles the various token types that
@@ -2211,8 +2282,8 @@ ExprResult Parser::ParseGenericSelectionExpression() {
   }
 
   SourceLocation DefaultLoc;
-  TypeVector Types(Actions);
-  ExprVector Exprs(Actions);
+  TypeVector Types;
+  ExprVector Exprs;
   while (1) {
     ParsedType Ty;
     if (Tok.is(tok::kw_default)) {
@@ -2263,7 +2334,7 @@ ExprResult Parser::ParseGenericSelectionExpression() {
   return Actions.ActOnGenericSelectionExpr(KeyLoc, DefaultLoc, 
                                            T.getCloseLocation(),
                                            ControllingExpr.release(),
-                                           move_arg(Types), move_arg(Exprs));
+                                           Types, Exprs);
 }
 
 /// ParseExpressionList - Used for C/C++ (argument-)expression-list.
@@ -2415,18 +2486,28 @@ ExprResult Parser::ParseBlockLiteralExpression() {
   } else {
     // Otherwise, pretend we saw (void).
     ParsedAttributes attrs(AttrFactory);
-    ParamInfo.AddTypeInfo(DeclaratorChunk::getFunction(true, false, false,
-                                                       SourceLocation(),
-                                                       0, 0, 0,
-                                                       true, SourceLocation(),
-                                                       SourceLocation(),
-                                                       SourceLocation(),
-                                                       SourceLocation(),
-                                                       EST_None,
-                                                       SourceLocation(),
-                                                       0, 0, 0, 0,
-                                                       CaretLoc, CaretLoc,
-                                                       ParamInfo),
+    SourceLocation NoLoc;
+    ParamInfo.AddTypeInfo(DeclaratorChunk::getFunction(/*HasProto=*/true,
+                                             /*IsAmbiguous=*/false,
+                                             /*RParenLoc=*/NoLoc,
+                                             /*ArgInfo=*/0,
+                                             /*NumArgs=*/0,
+                                             /*EllipsisLoc=*/NoLoc,
+                                             /*RParenLoc=*/NoLoc,
+                                             /*TypeQuals=*/0,
+                                             /*RefQualifierIsLvalueRef=*/true,
+                                             /*RefQualifierLoc=*/NoLoc,
+                                             /*ConstQualifierLoc=*/NoLoc,
+                                             /*VolatileQualifierLoc=*/NoLoc,
+                                             /*MutableLoc=*/NoLoc,
+                                             EST_None,
+                                             /*ESpecLoc=*/NoLoc,
+                                             /*Exceptions=*/0,
+                                             /*ExceptionRanges=*/0,
+                                             /*NumExceptions=*/0,
+                                             /*NoexceptExpr=*/0,
+                                             CaretLoc, CaretLoc,
+                                             ParamInfo),
                           attrs, CaretLoc);
 
     MaybeParseGNUAttributes(ParamInfo);
@@ -2450,7 +2531,7 @@ ExprResult Parser::ParseBlockLiteralExpression() {
     Result = Actions.ActOnBlockStmtExpr(CaretLoc, Stmt.take(), getCurScope());
   else
     Actions.ActOnBlockError(CaretLoc, getCurScope());
-  return move(Result);
+  return Result;
 }
 
 /// ParseObjCBoolLiteral - This handles the objective-c Boolean literals.
diff --git a/lib/Parse/ParseExprCXX.cpp b/lib/Parse/ParseExprCXX.cpp
index afac257..2f615e1 100644
--- a/lib/Parse/ParseExprCXX.cpp
+++ b/lib/Parse/ParseExprCXX.cpp
@@ -96,6 +96,45 @@ void Parser::CheckForTemplateAndDigraph(Token &Next, ParsedType ObjectType,
              /*AtDigraph*/false);
 }
 
+/// \brief Emits an error for a left parentheses after a double colon.
+///
+/// When a '(' is found after a '::', emit an error.  Attempt to fix the token
+/// stream by removing the '(', and the matching ')' if it found.
+void Parser::CheckForLParenAfterColonColon() {
+  if (!Tok.is(tok::l_paren))
+    return;
+
+  SourceLocation l_parenLoc = ConsumeParen(), r_parenLoc;
+  Token Tok1 = getCurToken();
+  if (!Tok1.is(tok::identifier) && !Tok1.is(tok::star))
+    return;
+
+  if (Tok1.is(tok::identifier)) {
+    Token Tok2 = GetLookAheadToken(1);
+    if (Tok2.is(tok::r_paren)) {
+      ConsumeToken();
+      PP.EnterToken(Tok1);
+      r_parenLoc = ConsumeParen();
+    }
+  } else if (Tok1.is(tok::star)) {
+    Token Tok2 = GetLookAheadToken(1);
+    if (Tok2.is(tok::identifier)) {
+      Token Tok3 = GetLookAheadToken(2);
+      if (Tok3.is(tok::r_paren)) {
+        ConsumeToken();
+        ConsumeToken();
+        PP.EnterToken(Tok2);
+        PP.EnterToken(Tok1);
+        r_parenLoc = ConsumeParen();
+      }
+    }
+  }
+
+  Diag(l_parenLoc, diag::err_paren_after_colon_colon)
+      << FixItHint::CreateRemoval(l_parenLoc)
+      << FixItHint::CreateRemoval(r_parenLoc);
+}
+
 /// \brief Parse global scope or nested-name-specifier if present.
 ///
 /// Parses a C++ global scope specifier ('::') or nested-name-specifier (which
@@ -160,7 +199,9 @@ bool Parser::ParseOptionalCXXScopeSpecifier(CXXScopeSpec &SS,
     // '::' - Global scope qualifier.
     if (Actions.ActOnCXXGlobalScopeSpecifier(getCurScope(), ConsumeToken(), SS))
       return true;
-    
+
+    CheckForLParenAfterColonColon();
+
     HasScopeSpecifier = true;
   }
 
@@ -301,8 +342,7 @@ bool Parser::ParseOptionalCXXScopeSpecifier(CXXScopeSpec &SS,
 
       HasScopeSpecifier = true;
       
-      ASTTemplateArgsPtr TemplateArgsPtr(Actions,
-                                         TemplateId->getTemplateArgs(),
+      ASTTemplateArgsPtr TemplateArgsPtr(TemplateId->getTemplateArgs(),
                                          TemplateId->NumArgs);
       
       if (Actions.ActOnCXXNestedNameSpecifier(getCurScope(),
@@ -372,6 +412,8 @@ bool Parser::ParseOptionalCXXScopeSpecifier(CXXScopeSpec &SS,
              "NextToken() not working properly!");
       SourceLocation CCLoc = ConsumeToken();
 
+      CheckForLParenAfterColonColon();
+
       HasScopeSpecifier = true;
       if (Actions.ActOnCXXNestedNameSpecifier(getCurScope(), II, IdLoc, CCLoc,
                                               ObjectType, EnteringContext, SS))
@@ -757,10 +799,10 @@ ExprResult Parser::ParseLambdaExpressionAfterIntroducer(
                               Scope::FunctionPrototypeScope |
                               Scope::DeclScope);
 
-    SourceLocation DeclLoc, DeclEndLoc;
+    SourceLocation DeclEndLoc;
     BalancedDelimiterTracker T(*this, tok::l_paren);
     T.consumeOpen();
-    DeclLoc = T.getOpenLocation();
+    SourceLocation LParenLoc = T.getOpenLocation();
 
     // Parse parameter-declaration-clause.
     ParsedAttributes Attr(AttrFactory);
@@ -771,7 +813,8 @@ ExprResult Parser::ParseLambdaExpressionAfterIntroducer(
       ParseParameterDeclarationClause(D, Attr, ParamInfo, EllipsisLoc);
 
     T.consumeClose();
-    DeclEndLoc = T.getCloseLocation();
+    SourceLocation RParenLoc = T.getCloseLocation();
+    DeclEndLoc = RParenLoc;
 
     // Parse 'mutable'[opt].
     SourceLocation MutableLoc;
@@ -797,9 +840,12 @@ ExprResult Parser::ParseLambdaExpressionAfterIntroducer(
     // Parse attribute-specifier[opt].
     MaybeParseCXX0XAttributes(Attr, &DeclEndLoc);
 
+    SourceLocation FunLocalRangeEnd = DeclEndLoc;
+
     // Parse trailing-return-type[opt].
     TypeResult TrailingReturnType;
     if (Tok.is(tok::arrow)) {
+      FunLocalRangeEnd = Tok.getLocation();
       SourceRange Range;
       TrailingReturnType = ParseTrailingReturnType(Range);
       if (Range.getEnd().isValid())
@@ -808,15 +854,17 @@ ExprResult Parser::ParseLambdaExpressionAfterIntroducer(
 
     PrototypeScope.Exit();
 
+    SourceLocation NoLoc;
     D.AddTypeInfo(DeclaratorChunk::getFunction(/*hasProto=*/true,
-                                           /*isVariadic=*/EllipsisLoc.isValid(),
-                                           /*isAmbiguous=*/false, EllipsisLoc,
+                                           /*isAmbiguous=*/false,
+                                           LParenLoc,
                                            ParamInfo.data(), ParamInfo.size(),
+                                           EllipsisLoc, RParenLoc,
                                            DS.getTypeQualifiers(),
                                            /*RefQualifierIsLValueRef=*/true,
-                                           /*RefQualifierLoc=*/SourceLocation(),
-                                         /*ConstQualifierLoc=*/SourceLocation(),
-                                      /*VolatileQualifierLoc=*/SourceLocation(),
+                                           /*RefQualifierLoc=*/NoLoc,
+                                           /*ConstQualifierLoc=*/NoLoc,
+                                           /*VolatileQualifierLoc=*/NoLoc,
                                            MutableLoc,
                                            ESpecType, ESpecRange.getBegin(),
                                            DynamicExceptions.data(),
@@ -824,7 +872,7 @@ ExprResult Parser::ParseLambdaExpressionAfterIntroducer(
                                            DynamicExceptions.size(),
                                            NoexceptExpr.isUsable() ?
                                              NoexceptExpr.get() : 0,
-                                           DeclLoc, DeclEndLoc, D,
+                                           LParenLoc, FunLocalRangeEnd, D,
                                            TrailingReturnType),
                   Attr, DeclEndLoc);
   } else if (Tok.is(tok::kw_mutable) || Tok.is(tok::arrow)) {
@@ -853,25 +901,28 @@ ExprResult Parser::ParseLambdaExpressionAfterIntroducer(
     }
 
     ParsedAttributes Attr(AttrFactory);
+    SourceLocation NoLoc;
     D.AddTypeInfo(DeclaratorChunk::getFunction(/*hasProto=*/true,
-                     /*isVariadic=*/false,
-                     /*isAmbiguous=*/false,
-                     /*EllipsisLoc=*/SourceLocation(),
-                     /*Params=*/0, /*NumParams=*/0,
-                     /*TypeQuals=*/0,
-                     /*RefQualifierIsLValueRef=*/true,
-                     /*RefQualifierLoc=*/SourceLocation(),
-                     /*ConstQualifierLoc=*/SourceLocation(),
-                     /*VolatileQualifierLoc=*/SourceLocation(),
-                     MutableLoc,
-                     EST_None, 
-                     /*ESpecLoc=*/SourceLocation(),
-                     /*Exceptions=*/0,
-                     /*ExceptionRanges=*/0,
-                     /*NumExceptions=*/0,
-                     /*NoexceptExpr=*/0,
-                     DeclLoc, DeclEndLoc, D,
-                     TrailingReturnType),
+                                               /*isAmbiguous=*/false,
+                                               /*LParenLoc=*/NoLoc,
+                                               /*Params=*/0,
+                                               /*NumParams=*/0,
+                                               /*EllipsisLoc=*/NoLoc,
+                                               /*RParenLoc=*/NoLoc,
+                                               /*TypeQuals=*/0,
+                                               /*RefQualifierIsLValueRef=*/true,
+                                               /*RefQualifierLoc=*/NoLoc,
+                                               /*ConstQualifierLoc=*/NoLoc,
+                                               /*VolatileQualifierLoc=*/NoLoc,
+                                               MutableLoc,
+                                               EST_None,
+                                               /*ESpecLoc=*/NoLoc,
+                                               /*Exceptions=*/0,
+                                               /*ExceptionRanges=*/0,
+                                               /*NumExceptions=*/0,
+                                               /*NoexceptExpr=*/0,
+                                               DeclLoc, DeclEndLoc, D,
+                                               TrailingReturnType),
                   Attr, DeclEndLoc);
   }
   
@@ -926,10 +977,11 @@ ExprResult Parser::ParseCXXCasts() {
 
   // Check for "<::" which is parsed as "[:".  If found, fix token stream,
   // diagnose error, suggest fix, and recover parsing.
-  Token Next = NextToken();
-  if (Tok.is(tok::l_square) && Tok.getLength() == 2 && Next.is(tok::colon) &&
-      areTokensAdjacent(Tok, Next))
-    FixDigraph(*this, PP, Tok, Next, Kind, /*AtDigraph*/true);
+  if (Tok.is(tok::l_square) && Tok.getLength() == 2) {
+    Token Next = NextToken();
+    if (Next.is(tok::colon) && areTokensAdjacent(Tok, Next))
+      FixDigraph(*this, PP, Tok, Next, Kind, /*AtDigraph*/true);
+  }
 
   if (ExpectAndConsume(tok::less, diag::err_expected_less_after, CastName))
     return ExprError();
@@ -965,7 +1017,7 @@ ExprResult Parser::ParseCXXCasts() {
                                        T.getOpenLocation(), Result.take(), 
                                        T.getCloseLocation());
 
-  return move(Result);
+  return Result;
 }
 
 /// ParseCXXTypeid - This handles the C++ typeid expression.
@@ -988,6 +1040,22 @@ ExprResult Parser::ParseCXXTypeid() {
 
   ExprResult Result;
 
+  // C++0x [expr.typeid]p3:
+  //   When typeid is applied to an expression other than an lvalue of a
+  //   polymorphic class type [...] The expression is an unevaluated
+  //   operand (Clause 5).
+  //
+  // Note that we can't tell whether the expression is an lvalue of a
+  // polymorphic class type until after we've parsed the expression; we
+  // speculatively assume the subexpression is unevaluated, and fix it up
+  // later.
+  //
+  // We enter the unevaluated context before trying to determine whether we
+  // have a type-id, because the tentative parse logic will try to resolve
+  // names, and must treat them as unevaluated.
+  EnterExpressionEvaluationContext Unevaluated(Actions, Sema::Unevaluated,
+                                               Sema::ReuseLambdaContextDecl);
+
   if (isTypeIdInParens()) {
     TypeResult Ty = ParseTypeName();
 
@@ -1000,16 +1068,6 @@ ExprResult Parser::ParseCXXTypeid() {
     Result = Actions.ActOnCXXTypeid(OpLoc, LParenLoc, /*isType=*/true,
                                     Ty.get().getAsOpaquePtr(), RParenLoc);
   } else {
-    // C++0x [expr.typeid]p3:
-    //   When typeid is applied to an expression other than an lvalue of a
-    //   polymorphic class type [...] The expression is an unevaluated
-    //   operand (Clause 5).
-    //
-    // Note that we can't tell whether the expression is an lvalue of a
-    // polymorphic class type until after we've parsed the expression; we
-    // speculatively assume the subexpression is unevaluated, and fix it up
-    // later.
-    EnterExpressionEvaluationContext Unevaluated(Actions, Sema::Unevaluated);
     Result = ParseExpression();
 
     // Match the ')'.
@@ -1026,7 +1084,7 @@ ExprResult Parser::ParseCXXTypeid() {
     }
   }
 
-  return move(Result);
+  return Result;
 }
 
 /// ParseCXXUuidof - This handles the Microsoft C++ __uuidof expression.
@@ -1074,7 +1132,7 @@ ExprResult Parser::ParseCXXUuidof() {
     }
   }
 
-  return move(Result);
+  return Result;
 }
 
 /// \brief Parse a C++ pseudo-destructor expression after the base,
@@ -1196,7 +1254,7 @@ ExprResult Parser::ParseThrowExpression() {
 
   default:
     ExprResult Expr(ParseAssignmentExpression());
-    if (Expr.isInvalid()) return move(Expr);
+    if (Expr.isInvalid()) return Expr;
     return Actions.ActOnCXXThrow(getCurScope(), ThrowLoc, Expr.take());
   }
 }
@@ -1245,7 +1303,7 @@ Parser::ParseCXXTypeConstructExpression(const DeclSpec &DS) {
     BalancedDelimiterTracker T(*this, tok::l_paren);
     T.consumeOpen();
 
-    ExprVector Exprs(Actions);
+    ExprVector Exprs;
     CommaLocsTy CommaLocs;
 
     if (Tok.isNot(tok::r_paren)) {
@@ -1265,7 +1323,7 @@ Parser::ParseCXXTypeConstructExpression(const DeclSpec &DS) {
     assert((Exprs.size() == 0 || Exprs.size()-1 == CommaLocs.size())&&
            "Unexpected number of commas!");
     return Actions.ActOnCXXTypeConstructExpr(TypeRep, T.getOpenLocation(), 
-                                             move_arg(Exprs),
+                                             Exprs,
                                              T.getCloseLocation());
   }
 }
@@ -1280,11 +1338,11 @@ Parser::ParseCXXTypeConstructExpression(const DeclSpec &DS) {
 /// [GNU]   type-specifier-seq declarator simple-asm-expr[opt] attributes[opt]
 ///             '=' assignment-expression
 ///
-/// \param ExprResult if the condition was parsed as an expression, the
-/// parsed expression.
+/// \param ExprOut if the condition was parsed as an expression, the parsed
+/// expression.
 ///
-/// \param DeclResult if the condition was parsed as a declaration, the
-/// parsed declaration.
+/// \param DeclOut if the condition was parsed as a declaration, the parsed
+/// declaration.
 ///
 /// \param Loc The location of the start of the statement that requires this
 /// condition, e.g., the "for" in a for loop.
@@ -1714,8 +1772,7 @@ bool Parser::ParseUnqualifiedIdTemplateId(CXXScopeSpec &SS,
   }
 
   // Bundle the template arguments together.
-  ASTTemplateArgsPtr TemplateArgsPtr(Actions, TemplateArgs.data(),
-                                     TemplateArgs.size());
+  ASTTemplateArgsPtr TemplateArgsPtr(TemplateArgs);
 
   // Constructor and destructor names.
   TypeResult Type
@@ -1762,7 +1819,7 @@ bool Parser::ParseUnqualifiedIdTemplateId(CXXScopeSpec &SS,
 ///         ptr-operator conversion-declarator[opt]
 /// \endcode
 ///
-/// \param The nested-name-specifier that preceded this unqualified-id. If
+/// \param SS The nested-name-specifier that preceded this unqualified-id. If
 /// non-empty, then we are parsing the unqualified-id of a qualified-id.
 ///
 /// \param EnteringContext whether we are entering the scope of the 
@@ -1867,8 +1924,9 @@ bool Parser::ParseUnqualifiedIdOperator(CXXScopeSpec &SS, bool EnteringContext,
 
   // Parse a literal-operator-id.
   //
-  //   literal-operator-id: [C++0x 13.5.8]
-  //     operator "" identifier
+  //   literal-operator-id: C++11 [over.literal]
+  //     operator string-literal identifier
+  //     operator user-defined-string-literal
 
   if (getLangOpts().CPlusPlus0x && isTokenStringLiteral()) {
     Diag(Tok.getLocation(), diag::warn_cxx98_compat_literal_operator);
@@ -1882,6 +1940,9 @@ bool Parser::ParseUnqualifiedIdOperator(CXXScopeSpec &SS, bool EnteringContext,
     llvm::SmallVector<SourceLocation, 4> TokLocs;
     while (isTokenStringLiteral()) {
       if (!Tok.is(tok::string_literal) && !DiagId) {
+        // C++11 [over.literal]p1:
+        //   The string-literal or user-defined-string-literal in a
+        //   literal-operator-id shall have no encoding-prefix [...].
         DiagLoc = Tok.getLocation();
         DiagId = diag::err_literal_operator_string_prefix;
       }
@@ -1903,9 +1964,6 @@ bool Parser::ParseUnqualifiedIdOperator(CXXScopeSpec &SS, bool EnteringContext,
         Lexer::AdvanceToTokenCharacter(TokLocs[Literal.getUDSuffixToken()],
                                        Literal.getUDSuffixOffset(),
                                        PP.getSourceManager(), getLangOpts());
-      // This form is not permitted by the standard (yet).
-      DiagLoc = SuffixLoc;
-      DiagId = diag::err_literal_operator_missing_space;
     } else if (Tok.is(tok::identifier)) {
       II = Tok.getIdentifierInfo();
       SuffixLoc = ConsumeToken();
@@ -1917,6 +1975,10 @@ bool Parser::ParseUnqualifiedIdOperator(CXXScopeSpec &SS, bool EnteringContext,
 
     // The string literal must be empty.
     if (!Literal.GetString().empty() || Literal.Pascal) {
+      // C++11 [over.literal]p1:
+      //   The string-literal or user-defined-string-literal in a
+      //   literal-operator-id shall [...] contain no characters
+      //   other than the implicit terminating '\0'.
       DiagLoc = TokLocs.front();
       DiagId = diag::err_literal_operator_string_not_empty;
     }
@@ -1981,7 +2043,7 @@ bool Parser::ParseUnqualifiedIdOperator(CXXScopeSpec &SS, bool EnteringContext,
 ///
 /// \endcode
 ///
-/// \param The nested-name-specifier that preceded this unqualified-id. If
+/// \param SS The nested-name-specifier that preceded this unqualified-id. If
 /// non-empty, then we are parsing the unqualified-id of a qualified-id.
 ///
 /// \param EnteringContext whether we are entering the scope of the 
@@ -2209,7 +2271,7 @@ Parser::ParseCXXNewExpression(bool UseGlobal, SourceLocation Start) {
   // A '(' now can be a new-placement or the '(' wrapping the type-id in the
   // second form of new-expression. It can't be a new-type-id.
 
-  ExprVector PlacementArgs(Actions);
+  ExprVector PlacementArgs;
   SourceLocation PlacementLParen, PlacementRParen;
 
   SourceRange TypeIdParens;
@@ -2279,7 +2341,7 @@ Parser::ParseCXXNewExpression(bool UseGlobal, SourceLocation Start) {
 
   if (Tok.is(tok::l_paren)) {
     SourceLocation ConstructorLParen, ConstructorRParen;
-    ExprVector ConstructorArgs(Actions);
+    ExprVector ConstructorArgs;
     BalancedDelimiterTracker T(*this, tok::l_paren);
     T.consumeOpen();
     ConstructorLParen = T.getOpenLocation();
@@ -2298,7 +2360,7 @@ Parser::ParseCXXNewExpression(bool UseGlobal, SourceLocation Start) {
     }
     Initializer = Actions.ActOnParenListExpr(ConstructorLParen,
                                              ConstructorRParen,
-                                             move_arg(ConstructorArgs));
+                                             ConstructorArgs);
   } else if (Tok.is(tok::l_brace) && getLangOpts().CPlusPlus0x) {
     Diag(Tok.getLocation(),
          diag::warn_cxx98_compat_generalized_initializer_lists);
@@ -2308,7 +2370,7 @@ Parser::ParseCXXNewExpression(bool UseGlobal, SourceLocation Start) {
     return Initializer;
 
   return Actions.ActOnCXXNew(Start, UseGlobal, PlacementLParen,
-                             move_arg(PlacementArgs), PlacementRParen,
+                             PlacementArgs, PlacementRParen,
                              TypeIdParens, DeclaratorInfo, Initializer.take());
 }
 
@@ -2422,7 +2484,7 @@ Parser::ParseCXXDeleteExpression(bool UseGlobal, SourceLocation Start) {
 
   ExprResult Operand(ParseCastExpression(false));
   if (Operand.isInvalid())
-    return move(Operand);
+    return Operand;
 
   return Actions.ActOnCXXDelete(Start, UseGlobal, ArrayDelete, Operand.take());
 }
@@ -2453,6 +2515,7 @@ static UnaryTypeTrait UnaryTypeTraitFromTokKind(tok::TokenKind kind) {
   case tok::kw___is_function:                return UTT_IsFunction;
   case tok::kw___is_fundamental:             return UTT_IsFundamental;
   case tok::kw___is_integral:                return UTT_IsIntegral;
+  case tok::kw___is_interface_class:         return UTT_IsInterfaceClass;
   case tok::kw___is_lvalue_reference:        return UTT_IsLvalueReference;
   case tok::kw___is_member_function_pointer: return UTT_IsMemberFunctionPointer;
   case tok::kw___is_member_object_pointer:   return UTT_IsMemberObjectPointer;
@@ -2804,7 +2867,7 @@ Parser::ParseCXXAmbiguousParenExpression(ParenParseOption &ExprType,
       Result = Actions.ActOnCastExpr(getCurScope(), Tracker.getOpenLocation(),
                                     DeclaratorInfo, CastTy,
                                     Tracker.getCloseLocation(), Result.take());
-    return move(Result);
+    return Result;
   }
 
   // Not a compound literal, and not followed by a cast-expression.
@@ -2823,5 +2886,5 @@ Parser::ParseCXXAmbiguousParenExpression(ParenParseOption &ExprType,
   }
 
   Tracker.consumeClose();
-  return move(Result);
+  return Result;
 }
diff --git a/lib/Parse/ParseInit.cpp b/lib/Parse/ParseInit.cpp
index 1c349fd..e47fd9b 100644
--- a/lib/Parse/ParseInit.cpp
+++ b/lib/Parse/ParseInit.cpp
@@ -313,7 +313,7 @@ ExprResult Parser::ParseInitializerWithPotentialDesignator() {
       Idx = ParseAssignmentExpression();
       if (Idx.isInvalid()) {
         SkipUntil(tok::r_square);
-        return move(Idx);
+        return Idx;
       }
     }
 
@@ -341,7 +341,7 @@ ExprResult Parser::ParseInitializerWithPotentialDesignator() {
       ExprResult RHS(ParseConstantExpression());
       if (RHS.isInvalid()) {
         SkipUntil(tok::r_square);
-        return move(RHS);
+        return RHS;
       }
       Desig.AddDesignator(Designator::getArrayRange(Idx.release(),
                                                     RHS.release(),
@@ -405,15 +405,14 @@ ExprResult Parser::ParseBraceInitializer() {
 
   /// InitExprs - This is the actual list of expressions contained in the
   /// initializer.
-  ExprVector InitExprs(Actions);
+  ExprVector InitExprs;
 
   if (Tok.is(tok::r_brace)) {
     // Empty initializers are a C++ feature and a GNU extension to C.
     if (!getLangOpts().CPlusPlus)
       Diag(LBraceLoc, diag::ext_gnu_empty_initializer);
     // Match the '}'.
-    return Actions.ActOnInitList(LBraceLoc, MultiExprArg(Actions),
-                                 ConsumeBrace());
+    return Actions.ActOnInitList(LBraceLoc, MultiExprArg(), ConsumeBrace());
   }
 
   bool InitExprsOk = true;
@@ -476,7 +475,7 @@ ExprResult Parser::ParseBraceInitializer() {
   bool closed = !T.consumeClose();
 
   if (InitExprsOk && closed)
-    return Actions.ActOnInitList(LBraceLoc, move_arg(InitExprs),
+    return Actions.ActOnInitList(LBraceLoc, InitExprs,
                                  T.getCloseLocation());
 
   return ExprError(); // an error occurred.
diff --git a/lib/Parse/ParseObjc.cpp b/lib/Parse/ParseObjc.cpp
index db35a38..d321baf 100644
--- a/lib/Parse/ParseObjc.cpp
+++ b/lib/Parse/ParseObjc.cpp
@@ -18,6 +18,7 @@
 #include "clang/Sema/PrettyDeclStackTrace.h"
 #include "clang/Sema/Scope.h"
 #include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringExtras.h"
 using namespace clang;
 
 
@@ -1031,7 +1032,6 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc,
                             Scope::FunctionPrototypeScope|Scope::DeclScope);
 
   AttributePool allParamAttrs(AttrFactory);
-  
   while (1) {
     ParsedAttributes paramAttrs(AttrFactory);
     Sema::ObjCArgInfo ArgInfo;
@@ -1102,6 +1102,14 @@ Decl *Parser::ParseObjCMethodDecl(SourceLocation mLoc,
     SelIdent = ParseObjCSelectorPiece(selLoc);
     if (!SelIdent && Tok.isNot(tok::colon))
       break;
+    if (!SelIdent) {
+      SourceLocation ColonLoc = Tok.getLocation();
+      if (PP.getLocForEndOfToken(ArgInfo.NameLoc) == ColonLoc) {
+        Diag(ArgInfo.NameLoc, diag::warn_missing_selector_name) << ArgInfo.Name;
+        Diag(ArgInfo.NameLoc, diag::note_missing_selector_name) << ArgInfo.Name;
+        Diag(ColonLoc, diag::note_force_empty_selector_name) << ArgInfo.Name;
+      }
+    }
     // We have a selector or a colon, continue parsing.
   }
 
@@ -1806,7 +1814,7 @@ StmtResult Parser::ParseObjCTryStmt(SourceLocation atLoc) {
     Diag(Tok, diag::err_expected_lbrace);
     return StmtError();
   }
-  StmtVector CatchStmts(Actions);
+  StmtVector CatchStmts;
   StmtResult FinallyStmt;
   ParseScope TryScope(this, Scope::DeclScope);
   StmtResult TryBody(ParseCompoundStatementBody());
@@ -1894,7 +1902,7 @@ StmtResult Parser::ParseObjCTryStmt(SourceLocation atLoc) {
   }
   
   return Actions.ActOnObjCAtTryStmt(atLoc, TryBody.take(), 
-                                    move_arg(CatchStmts),
+                                    CatchStmts,
                                     FinallyStmt.take());
 }
 
@@ -2061,13 +2069,13 @@ ExprResult Parser::ParseObjCAtExpression(SourceLocation AtLoc) {
 
     ExprResult Lit(Actions.ActOnNumericConstant(Tok));
     if (Lit.isInvalid()) {
-      return move(Lit);
+      return Lit;
     }
     ConsumeToken(); // Consume the literal token.
 
     Lit = Actions.ActOnUnaryOp(getCurScope(), OpLoc, Kind, Lit.take());
     if (Lit.isInvalid())
-      return move(Lit);
+      return Lit;
 
     return ParsePostfixExpressionSuffix(
              Actions.BuildObjCNumericLiteral(AtLoc, Lit.take()));
@@ -2134,7 +2142,7 @@ ExprResult Parser::ParseObjCAtExpression(SourceLocation AtLoc) {
   }
 }
 
-/// \brirg Parse the receiver of an Objective-C++ message send.
+/// \brief Parse the receiver of an Objective-C++ message send.
 ///
 /// This routine parses the receiver of a message send in
 /// Objective-C++ either as a type or as an expression. Note that this
@@ -2346,7 +2354,7 @@ ExprResult Parser::ParseObjCMessageExpression() {
   ExprResult Res(ParseExpression());
   if (Res.isInvalid()) {
     SkipUntil(tok::r_square);
-    return move(Res);
+    return Res;
   }
 
   return ParseObjCMessageExpressionBody(LBracLoc, SourceLocation(),
@@ -2418,7 +2426,7 @@ Parser::ParseObjCMessageExpressionBody(SourceLocation LBracLoc,
 
   SmallVector<IdentifierInfo *, 12> KeyIdents;
   SmallVector<SourceLocation, 12> KeyLocs;
-  ExprVector KeyExprs(Actions);
+  ExprVector KeyExprs;
 
   if (Tok.is(tok::colon)) {
     while (1) {
@@ -2465,7 +2473,7 @@ Parser::ParseObjCMessageExpressionBody(SourceLocation LBracLoc,
         // stop at the ']' when it skips to the ';'.  We want it to skip beyond
         // the enclosing expression.
         SkipUntil(tok::r_square);
-        return move(Res);
+        return Res;
       }
 
       // We have a valid expression.
@@ -2512,7 +2520,7 @@ Parser::ParseObjCMessageExpressionBody(SourceLocation LBracLoc,
         // stop at the ']' when it skips to the ';'.  We want it to skip beyond
         // the enclosing expression.
         SkipUntil(tok::r_square);
-        return move(Res);
+        return Res;
       }
 
       // We have a valid expression.
@@ -2551,32 +2559,23 @@ Parser::ParseObjCMessageExpressionBody(SourceLocation LBracLoc,
 
   if (SuperLoc.isValid())
     return Actions.ActOnSuperMessage(getCurScope(), SuperLoc, Sel,
-                                     LBracLoc, KeyLocs, RBracLoc,
-                                     MultiExprArg(Actions, 
-                                                  KeyExprs.take(),
-                                                  KeyExprs.size()));
+                                     LBracLoc, KeyLocs, RBracLoc, KeyExprs);
   else if (ReceiverType)
     return Actions.ActOnClassMessage(getCurScope(), ReceiverType, Sel,
-                                     LBracLoc, KeyLocs, RBracLoc,
-                                     MultiExprArg(Actions, 
-                                                  KeyExprs.take(), 
-                                                  KeyExprs.size()));
+                                     LBracLoc, KeyLocs, RBracLoc, KeyExprs);
   return Actions.ActOnInstanceMessage(getCurScope(), ReceiverExpr, Sel,
-                                      LBracLoc, KeyLocs, RBracLoc,
-                                      MultiExprArg(Actions, 
-                                                   KeyExprs.take(), 
-                                                   KeyExprs.size()));
+                                      LBracLoc, KeyLocs, RBracLoc, KeyExprs);
 }
 
 ExprResult Parser::ParseObjCStringLiteral(SourceLocation AtLoc) {
   ExprResult Res(ParseStringLiteralExpression());
-  if (Res.isInvalid()) return move(Res);
+  if (Res.isInvalid()) return Res;
 
   // @"foo" @"bar" is a valid concatenated string.  Eat any subsequent string
   // expressions.  At this point, we know that the only valid thing that starts
   // with '@' is an @"".
   SmallVector<SourceLocation, 4> AtLocs;
-  ExprVector AtStrings(Actions);
+  ExprVector AtStrings;
   AtLocs.push_back(AtLoc);
   AtStrings.push_back(Res.release());
 
@@ -2589,12 +2588,12 @@ ExprResult Parser::ParseObjCStringLiteral(SourceLocation AtLoc) {
 
     ExprResult Lit(ParseStringLiteralExpression());
     if (Lit.isInvalid())
-      return move(Lit);
+      return Lit;
 
     AtStrings.push_back(Lit.release());
   }
 
-  return Owned(Actions.ParseObjCStringLiteral(&AtLocs[0], AtStrings.take(),
+  return Owned(Actions.ParseObjCStringLiteral(&AtLocs[0], AtStrings.data(),
                                               AtStrings.size()));
 }
 
@@ -2615,7 +2614,7 @@ ExprResult Parser::ParseObjCBooleanLiteral(SourceLocation AtLoc,
 ExprResult Parser::ParseObjCCharacterLiteral(SourceLocation AtLoc) {
   ExprResult Lit(Actions.ActOnCharacterConstant(Tok));
   if (Lit.isInvalid()) {
-    return move(Lit);
+    return Lit;
   }
   ConsumeToken(); // Consume the literal token.
   return Owned(Actions.BuildObjCNumericLiteral(AtLoc, Lit.take()));
@@ -2629,7 +2628,7 @@ ExprResult Parser::ParseObjCCharacterLiteral(SourceLocation AtLoc) {
 ExprResult Parser::ParseObjCNumericLiteral(SourceLocation AtLoc) {
   ExprResult Lit(Actions.ActOnNumericConstant(Tok));
   if (Lit.isInvalid()) {
-    return move(Lit);
+    return Lit;
   }
   ConsumeToken(); // Consume the literal token.
   return Owned(Actions.BuildObjCNumericLiteral(AtLoc, Lit.take()));
@@ -2661,7 +2660,7 @@ Parser::ParseObjCBoxedExpr(SourceLocation AtLoc) {
 }
 
 ExprResult Parser::ParseObjCArrayLiteral(SourceLocation AtLoc) {
-  ExprVector ElementExprs(Actions);                   // array elements.
+  ExprVector ElementExprs;                   // array elements.
   ConsumeBracket(); // consume the l_square.
 
   while (Tok.isNot(tok::r_square)) {
@@ -2672,7 +2671,7 @@ ExprResult Parser::ParseObjCArrayLiteral(SourceLocation AtLoc) {
       // stop at the ']' when it skips to the ';'.  We want it to skip beyond
       // the enclosing expression.
       SkipUntil(tok::r_square);
-      return move(Res);
+      return Res;
     }    
     
     // Parse the ellipsis that indicates a pack expansion.
@@ -2689,7 +2688,7 @@ ExprResult Parser::ParseObjCArrayLiteral(SourceLocation AtLoc) {
      return ExprError(Diag(Tok, diag::err_expected_rsquare_or_comma));
   }
   SourceLocation EndLoc = ConsumeBracket(); // location of ']'
-  MultiExprArg Args(Actions, ElementExprs.take(), ElementExprs.size());
+  MultiExprArg Args(ElementExprs);
   return Owned(Actions.BuildObjCArrayLiteral(SourceRange(AtLoc, EndLoc), Args));
 }
 
@@ -2707,7 +2706,7 @@ ExprResult Parser::ParseObjCDictionaryLiteral(SourceLocation AtLoc) {
         // stop at the '}' when it skips to the ';'.  We want it to skip beyond
         // the enclosing expression.
         SkipUntil(tok::r_brace);
-        return move(KeyExpr);
+        return KeyExpr;
       }
     }
 
@@ -2723,7 +2722,7 @@ ExprResult Parser::ParseObjCDictionaryLiteral(SourceLocation AtLoc) {
       // stop at the '}' when it skips to the ';'.  We want it to skip beyond
       // the enclosing expression.
       SkipUntil(tok::r_brace);
-      return move(ValueExpr);
+      return ValueExpr;
     }
     
     // Parse the ellipsis that designates this as a pack expansion.
@@ -2752,7 +2751,7 @@ ExprResult Parser::ParseObjCDictionaryLiteral(SourceLocation AtLoc) {
 }
 
 ///    objc-encode-expression:
-///      @encode ( type-name )
+///      \@encode ( type-name )
 ExprResult
 Parser::ParseObjCEncodeExpression(SourceLocation AtLoc) {
   assert(Tok.isObjCAtKeyword(tok::objc_encode) && "Not an @encode expression!");
diff --git a/lib/Parse/ParsePragma.cpp b/lib/Parse/ParsePragma.cpp
index eb13e0d..a7605f0 100644
--- a/lib/Parse/ParsePragma.cpp
+++ b/lib/Parse/ParsePragma.cpp
@@ -40,7 +40,7 @@ void Parser::HandlePragmaVisibility() {
 struct PragmaPackInfo {
   Sema::PragmaPackKind Kind;
   IdentifierInfo *Name;
-  Expr *Alignment;
+  Token Alignment;
   SourceLocation LParenLoc;
   SourceLocation RParenLoc;
 };
@@ -50,10 +50,107 @@ void Parser::HandlePragmaPack() {
   PragmaPackInfo *Info =
     static_cast<PragmaPackInfo *>(Tok.getAnnotationValue());
   SourceLocation PragmaLoc = ConsumeToken();
-  Actions.ActOnPragmaPack(Info->Kind, Info->Name, Info->Alignment, PragmaLoc,
+  ExprResult Alignment;
+  if (Info->Alignment.is(tok::numeric_constant)) {
+    Alignment = Actions.ActOnNumericConstant(Info->Alignment);
+    if (Alignment.isInvalid())
+      return;
+  }
+  Actions.ActOnPragmaPack(Info->Kind, Info->Name, Alignment.get(), PragmaLoc,
                           Info->LParenLoc, Info->RParenLoc);
 }
 
+void Parser::HandlePragmaMSStruct() {
+  assert(Tok.is(tok::annot_pragma_msstruct));
+  Sema::PragmaMSStructKind Kind =
+    static_cast<Sema::PragmaMSStructKind>(
+    reinterpret_cast<uintptr_t>(Tok.getAnnotationValue()));
+  Actions.ActOnPragmaMSStruct(Kind);
+  ConsumeToken(); // The annotation token.
+}
+
+void Parser::HandlePragmaAlign() {
+  assert(Tok.is(tok::annot_pragma_align));
+  Sema::PragmaOptionsAlignKind Kind =
+    static_cast<Sema::PragmaOptionsAlignKind>(
+    reinterpret_cast<uintptr_t>(Tok.getAnnotationValue()));
+  SourceLocation PragmaLoc = ConsumeToken();
+  Actions.ActOnPragmaOptionsAlign(Kind, PragmaLoc);
+}
+
+void Parser::HandlePragmaWeak() {
+  assert(Tok.is(tok::annot_pragma_weak));
+  SourceLocation PragmaLoc = ConsumeToken();
+  Actions.ActOnPragmaWeakID(Tok.getIdentifierInfo(), PragmaLoc,
+                            Tok.getLocation());
+  ConsumeToken(); // The weak name.
+}
+
+void Parser::HandlePragmaWeakAlias() {
+  assert(Tok.is(tok::annot_pragma_weakalias));
+  SourceLocation PragmaLoc = ConsumeToken();
+  IdentifierInfo *WeakName = Tok.getIdentifierInfo();
+  SourceLocation WeakNameLoc = Tok.getLocation();
+  ConsumeToken();
+  IdentifierInfo *AliasName = Tok.getIdentifierInfo();
+  SourceLocation AliasNameLoc = Tok.getLocation();
+  ConsumeToken();
+  Actions.ActOnPragmaWeakAlias(WeakName, AliasName, PragmaLoc,
+                               WeakNameLoc, AliasNameLoc);
+
+}
+
+void Parser::HandlePragmaRedefineExtname() {
+  assert(Tok.is(tok::annot_pragma_redefine_extname));
+  SourceLocation RedefLoc = ConsumeToken();
+  IdentifierInfo *RedefName = Tok.getIdentifierInfo();
+  SourceLocation RedefNameLoc = Tok.getLocation();
+  ConsumeToken();
+  IdentifierInfo *AliasName = Tok.getIdentifierInfo();
+  SourceLocation AliasNameLoc = Tok.getLocation();
+  ConsumeToken();
+  Actions.ActOnPragmaRedefineExtname(RedefName, AliasName, RedefLoc,
+                                     RedefNameLoc, AliasNameLoc);
+}
+
+void Parser::HandlePragmaFPContract() {
+  assert(Tok.is(tok::annot_pragma_fp_contract));
+  tok::OnOffSwitch OOS =
+    static_cast<tok::OnOffSwitch>(
+    reinterpret_cast<uintptr_t>(Tok.getAnnotationValue()));
+  Actions.ActOnPragmaFPContract(OOS);
+  ConsumeToken(); // The annotation token.
+}
+
+namespace {
+  typedef llvm::PointerIntPair<IdentifierInfo *, 1, bool> OpenCLExtData;
+}
+
+void Parser::HandlePragmaOpenCLExtension() {
+  assert(Tok.is(tok::annot_pragma_opencl_extension));
+  OpenCLExtData data =
+      OpenCLExtData::getFromOpaqueValue(Tok.getAnnotationValue());
+  unsigned state = data.getInt();
+  IdentifierInfo *ename = data.getPointer();
+  SourceLocation NameLoc = Tok.getLocation();
+  ConsumeToken(); // The annotation token.
+
+  OpenCLOptions &f = Actions.getOpenCLOptions();
+  // OpenCL 1.1 9.1: "The all variant sets the behavior for all extensions,
+  // overriding all previously issued extension directives, but only if the
+  // behavior is set to disable."
+  if (state == 0 && ename->isStr("all")) {
+#define OPENCLEXT(nm)   f.nm = 0;
+#include "clang/Basic/OpenCLExtensions.def"
+  }
+#define OPENCLEXT(nm) else if (ename->isStr(#nm)) { f.nm = state; }
+#include "clang/Basic/OpenCLExtensions.def"
+  else {
+    PP.Diag(NameLoc, diag::warn_pragma_unknown_extension) << ename;
+    return;
+  }
+}
+
 // #pragma GCC visibility comes in two variants:
 //   'push' '(' [visibility] ')'
 //   'pop'
@@ -130,13 +227,12 @@ void PragmaPackHandler::HandlePragma(Preprocessor &PP,
 
   Sema::PragmaPackKind Kind = Sema::PPK_Default;
   IdentifierInfo *Name = 0;
-  ExprResult Alignment;
+  Token Alignment;
+  Alignment.startToken();
   SourceLocation LParenLoc = Tok.getLocation();
   PP.Lex(Tok);
   if (Tok.is(tok::numeric_constant)) {
-    Alignment = Actions.ActOnNumericConstant(Tok);
-    if (Alignment.isInvalid())
-      return;
+    Alignment = Tok;
 
     PP.Lex(Tok);
 
@@ -165,9 +261,7 @@ void PragmaPackHandler::HandlePragma(Preprocessor &PP,
         PP.Lex(Tok);
 
         if (Tok.is(tok::numeric_constant)) {
-          Alignment = Actions.ActOnNumericConstant(Tok);
-          if (Alignment.isInvalid())
-            return;
+          Alignment = Tok;
 
           PP.Lex(Tok);
         } else if (Tok.is(tok::identifier)) {
@@ -182,9 +276,7 @@ void PragmaPackHandler::HandlePragma(Preprocessor &PP,
               return;
             }
 
-            Alignment = Actions.ActOnNumericConstant(Tok);
-            if (Alignment.isInvalid())
-              return;
+            Alignment = Tok;
 
             PP.Lex(Tok);
           }
@@ -219,7 +311,7 @@ void PragmaPackHandler::HandlePragma(Preprocessor &PP,
   new (Info) PragmaPackInfo();
   Info->Kind = Kind;
   Info->Name = Name;
-  Info->Alignment = Alignment.release();
+  Info->Alignment = Alignment;
   Info->LParenLoc = LParenLoc;
   Info->RParenLoc = RParenLoc;
 
@@ -265,12 +357,23 @@ void PragmaMSStructHandler::HandlePragma(Preprocessor &PP,
       << "ms_struct";
     return;
   }
-  Actions.ActOnPragmaMSStruct(Kind);
+
+  Token *Toks =
+    (Token*) PP.getPreprocessorAllocator().Allocate(
+      sizeof(Token) * 1, llvm::alignOf<Token>());
+  new (Toks) Token();
+  Toks[0].startToken();
+  Toks[0].setKind(tok::annot_pragma_msstruct);
+  Toks[0].setLocation(MSStructTok.getLocation());
+  Toks[0].setAnnotationValue(reinterpret_cast<void*>(
+                             static_cast<uintptr_t>(Kind)));
+  PP.EnterTokenStream(Toks, 1, /*DisableMacroExpansion=*/true,
+                      /*OwnsTokens=*/false);
 }
 
 // #pragma 'align' '=' {'native','natural','mac68k','power','reset'}
 // #pragma 'options 'align' '=' {'native','natural','mac68k','power','reset'}
-static void ParseAlignPragma(Sema &Actions, Preprocessor &PP, Token &FirstTok,
+static void ParseAlignPragma(Preprocessor &PP, Token &FirstTok,
                              bool IsOptions) {
   Token Tok;
 
@@ -317,7 +420,6 @@ static void ParseAlignPragma(Sema &Actions, Preprocessor &PP, Token &FirstTok,
     return;
   }
 
-  SourceLocation KindLoc = Tok.getLocation();
   PP.Lex(Tok);
   if (Tok.isNot(tok::eod)) {
     PP.Diag(Tok.getLocation(), diag::warn_pragma_extra_tokens_at_eol)
@@ -325,19 +427,29 @@ static void ParseAlignPragma(Sema &Actions, Preprocessor &PP, Token &FirstTok,
     return;
   }
 
-  Actions.ActOnPragmaOptionsAlign(Kind, FirstTok.getLocation(), KindLoc);
+  Token *Toks =
+    (Token*) PP.getPreprocessorAllocator().Allocate(
+      sizeof(Token) * 1, llvm::alignOf<Token>());
+  new (Toks) Token();
+  Toks[0].startToken();
+  Toks[0].setKind(tok::annot_pragma_align);
+  Toks[0].setLocation(FirstTok.getLocation());
+  Toks[0].setAnnotationValue(reinterpret_cast<void*>(
+                             static_cast<uintptr_t>(Kind)));
+  PP.EnterTokenStream(Toks, 1, /*DisableMacroExpansion=*/true,
+                      /*OwnsTokens=*/false);
 }
 
 void PragmaAlignHandler::HandlePragma(Preprocessor &PP, 
                                       PragmaIntroducerKind Introducer,
                                       Token &AlignTok) {
-  ParseAlignPragma(Actions, PP, AlignTok, /*IsOptions=*/false);
+  ParseAlignPragma(PP, AlignTok, /*IsOptions=*/false);
 }
 
 void PragmaOptionsHandler::HandlePragma(Preprocessor &PP, 
                                         PragmaIntroducerKind Introducer,
                                         Token &OptionsTok) {
-  ParseAlignPragma(Actions, PP, OptionsTok, /*IsOptions=*/true);
+  ParseAlignPragma(PP, OptionsTok, /*IsOptions=*/true);
 }
 
 // #pragma unused(identifier)
@@ -426,7 +538,6 @@ void PragmaUnusedHandler::HandlePragma(Preprocessor &PP,
 void PragmaWeakHandler::HandlePragma(Preprocessor &PP, 
                                      PragmaIntroducerKind Introducer,
                                      Token &WeakTok) {
-  // FIXME: Should we be expanding macros here? My guess is no.
   SourceLocation WeakLoc = WeakTok.getLocation();
 
   Token Tok;
@@ -436,19 +547,20 @@ void PragmaWeakHandler::HandlePragma(Preprocessor &PP,
     return;
   }
 
-  IdentifierInfo *WeakName = Tok.getIdentifierInfo(), *AliasName = 0;
-  SourceLocation WeakNameLoc = Tok.getLocation(), AliasNameLoc;
+  Token WeakName = Tok;
+  bool HasAlias = false;
+  Token AliasName;
 
   PP.Lex(Tok);
   if (Tok.is(tok::equal)) {
+    HasAlias = true;
     PP.Lex(Tok);
     if (Tok.isNot(tok::identifier)) {
       PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_identifier)
           << "weak";
       return;
     }
-    AliasName = Tok.getIdentifierInfo();
-    AliasNameLoc = Tok.getLocation();
+    AliasName = Tok;
     PP.Lex(Tok);
   }
 
@@ -457,11 +569,29 @@ void PragmaWeakHandler::HandlePragma(Preprocessor &PP,
     return;
   }
 
-  if (AliasName) {
-    Actions.ActOnPragmaWeakAlias(WeakName, AliasName, WeakLoc, WeakNameLoc,
-                                 AliasNameLoc);
+  if (HasAlias) {
+    Token *Toks = 
+      (Token*) PP.getPreprocessorAllocator().Allocate(
+        sizeof(Token) * 3, llvm::alignOf<Token>());
+    Token &pragmaUnusedTok = Toks[0];
+    pragmaUnusedTok.startToken();
+    pragmaUnusedTok.setKind(tok::annot_pragma_weakalias);
+    pragmaUnusedTok.setLocation(WeakLoc);
+    Toks[1] = WeakName;
+    Toks[2] = AliasName;
+    PP.EnterTokenStream(Toks, 3,
+                        /*DisableMacroExpansion=*/true, /*OwnsTokens=*/false);
   } else {
-    Actions.ActOnPragmaWeakID(WeakName, WeakLoc, WeakNameLoc);
+    Token *Toks = 
+      (Token*) PP.getPreprocessorAllocator().Allocate(
+        sizeof(Token) * 2, llvm::alignOf<Token>());
+    Token &pragmaUnusedTok = Toks[0];
+    pragmaUnusedTok.startToken();
+    pragmaUnusedTok.setKind(tok::annot_pragma_weak);
+    pragmaUnusedTok.setLocation(WeakLoc);
+    Toks[1] = WeakName;
+    PP.EnterTokenStream(Toks, 2,
+                        /*DisableMacroExpansion=*/true, /*OwnsTokens=*/false);
   }
 }
 
@@ -479,17 +609,16 @@ void PragmaRedefineExtnameHandler::HandlePragma(Preprocessor &PP,
     return;
   }
 
-  IdentifierInfo *RedefName = Tok.getIdentifierInfo(), *AliasName = 0;
-  SourceLocation RedefNameLoc = Tok.getLocation(), AliasNameLoc;
-
+  Token RedefName = Tok;
   PP.Lex(Tok);
+
   if (Tok.isNot(tok::identifier)) {
     PP.Diag(Tok.getLocation(), diag::warn_pragma_expected_identifier)
         << "redefine_extname";
     return;
   }
-  AliasName = Tok.getIdentifierInfo();
-  AliasNameLoc = Tok.getLocation();
+
+  Token AliasName = Tok;
   PP.Lex(Tok);
 
   if (Tok.isNot(tok::eod)) {
@@ -498,8 +627,17 @@ void PragmaRedefineExtnameHandler::HandlePragma(Preprocessor &PP,
     return;
   }
 
-  Actions.ActOnPragmaRedefineExtname(RedefName, AliasName, RedefLoc,
-      RedefNameLoc, AliasNameLoc);
+  Token *Toks = 
+    (Token*) PP.getPreprocessorAllocator().Allocate(
+      sizeof(Token) * 3, llvm::alignOf<Token>());
+  Token &pragmaRedefTok = Toks[0];
+  pragmaRedefTok.startToken();
+  pragmaRedefTok.setKind(tok::annot_pragma_redefine_extname);
+  pragmaRedefTok.setLocation(RedefLoc);
+  Toks[1] = RedefName;
+  Toks[2] = AliasName;
+  PP.EnterTokenStream(Toks, 3,
+                      /*DisableMacroExpansion=*/true, /*OwnsTokens=*/false);
 }
 
 
@@ -511,7 +649,17 @@ PragmaFPContractHandler::HandlePragma(Preprocessor &PP,
   if (PP.LexOnOffSwitch(OOS))
     return;
 
-  Actions.ActOnPragmaFPContract(OOS);
+  Token *Toks =
+    (Token*) PP.getPreprocessorAllocator().Allocate(
+      sizeof(Token) * 1, llvm::alignOf<Token>());
+  new (Toks) Token();
+  Toks[0].startToken();
+  Toks[0].setKind(tok::annot_pragma_fp_contract);
+  Toks[0].setLocation(Tok.getLocation());
+  Toks[0].setAnnotationValue(reinterpret_cast<void*>(
+                             static_cast<uintptr_t>(OOS)));
+  PP.EnterTokenStream(Toks, 1, /*DisableMacroExpansion=*/true,
+                      /*OwnsTokens=*/false);
 }
 
 void 
@@ -550,19 +698,23 @@ PragmaOpenCLExtensionHandler::HandlePragma(Preprocessor &PP,
     return;
   }
 
-  OpenCLOptions &f = Actions.getOpenCLOptions();
-  // OpenCL 1.1 9.1: "The all variant sets the behavior for all extensions,
-  // overriding all previously issued extension directives, but only if the
-  // behavior is set to disable."
-  if (state == 0 && ename->isStr("all")) {
-#define OPENCLEXT(nm)   f.nm = 0;
-#include "clang/Basic/OpenCLExtensions.def"
-  }
-#define OPENCLEXT(nm) else if (ename->isStr(#nm)) { f.nm = state; }
-#include "clang/Basic/OpenCLExtensions.def"
-  else {
-    PP.Diag(NameLoc, diag::warn_pragma_unknown_extension) << ename;
+  PP.Lex(Tok);
+  if (Tok.isNot(tok::eod)) {
+    PP.Diag(Tok.getLocation(), diag::warn_pragma_extra_tokens_at_eol) <<
+      "OPENCL EXTENSION";
     return;
   }
+
+  OpenCLExtData data(ename, state);
+  Token *Toks =
+    (Token*) PP.getPreprocessorAllocator().Allocate(
+      sizeof(Token) * 1, llvm::alignOf<Token>());
+  new (Toks) Token();
+  Toks[0].startToken();
+  Toks[0].setKind(tok::annot_pragma_opencl_extension);
+  Toks[0].setLocation(NameLoc);
+  Toks[0].setAnnotationValue(data.getOpaqueValue());
+  PP.EnterTokenStream(Toks, 1, /*DisableMacroExpansion=*/true,
+                      /*OwnsTokens=*/false);
 }
 
diff --git a/lib/Parse/ParsePragma.h b/lib/Parse/ParsePragma.h
index fef6960..b9a2a25 100644
--- a/lib/Parse/ParsePragma.h
+++ b/lib/Parse/ParsePragma.h
@@ -21,9 +21,8 @@ namespace clang {
   class Parser;
 
 class PragmaAlignHandler : public PragmaHandler {
-  Sema &Actions;
 public:
-  explicit PragmaAlignHandler(Sema &A) : PragmaHandler("align"), Actions(A) {}
+  explicit PragmaAlignHandler() : PragmaHandler("align") {}
 
   virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
                             Token &FirstToken);
@@ -31,38 +30,31 @@ public:
 
 class PragmaGCCVisibilityHandler : public PragmaHandler {
 public:
-  explicit PragmaGCCVisibilityHandler(Sema &/*A*/)
-    : PragmaHandler("visibility") {}
+  explicit PragmaGCCVisibilityHandler() : PragmaHandler("visibility") {}
 
   virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
                             Token &FirstToken);
 };
 
 class PragmaOptionsHandler : public PragmaHandler {
-  Sema &Actions;
 public:
-  explicit PragmaOptionsHandler(Sema &A) : PragmaHandler("options"),
-                                           Actions(A) {}
+  explicit PragmaOptionsHandler() : PragmaHandler("options") {}
 
   virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
                             Token &FirstToken);
 };
 
 class PragmaPackHandler : public PragmaHandler {
-  Sema &Actions;
 public:
-  explicit PragmaPackHandler(Sema &A) : PragmaHandler("pack"),
-                                        Actions(A) {}
+  explicit PragmaPackHandler() : PragmaHandler("pack") {}
 
   virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
                             Token &FirstToken);
 };
   
 class PragmaMSStructHandler : public PragmaHandler {
-  Sema &Actions;
 public:
-  explicit PragmaMSStructHandler(Sema &A) : PragmaHandler("ms_struct"),
-  Actions(A) {}
+  explicit PragmaMSStructHandler() : PragmaHandler("ms_struct") {}
     
   virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
                             Token &FirstToken);
@@ -70,48 +62,39 @@ public:
 
 class PragmaUnusedHandler : public PragmaHandler {
 public:
-  PragmaUnusedHandler(Sema &/*A*/)
-    : PragmaHandler("unused") {}
+  PragmaUnusedHandler() : PragmaHandler("unused") {}
 
   virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
                             Token &FirstToken);
 };
 
 class PragmaWeakHandler : public PragmaHandler {
-  Sema &Actions;
 public:
-  explicit PragmaWeakHandler(Sema &A)
-    : PragmaHandler("weak"), Actions(A) {}
+  explicit PragmaWeakHandler() : PragmaHandler("weak") {}
 
   virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
                             Token &FirstToken);
 };
 
 class PragmaRedefineExtnameHandler : public PragmaHandler {
-  Sema &Actions;
 public:
-  explicit PragmaRedefineExtnameHandler(Sema &A)
-    : PragmaHandler("redefine_extname"), Actions(A) {}
+  explicit PragmaRedefineExtnameHandler() : PragmaHandler("redefine_extname") {}
 
   virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
                             Token &FirstToken);
 };
 
 class PragmaOpenCLExtensionHandler : public PragmaHandler {
-  Sema &Actions;
 public:
-  PragmaOpenCLExtensionHandler(Sema &A) :
-    PragmaHandler("EXTENSION"), Actions(A) {}
+  PragmaOpenCLExtensionHandler() : PragmaHandler("EXTENSION") {}
   virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
                             Token &FirstToken);
 };
   
 
 class PragmaFPContractHandler : public PragmaHandler {
-  Sema &Actions;
 public:
-  PragmaFPContractHandler(Sema &A) :
-    PragmaHandler("FP_CONTRACT"), Actions(A) {}
+  PragmaFPContractHandler() : PragmaHandler("FP_CONTRACT") {}
   virtual void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer,
                             Token &FirstToken);
 };
diff --git a/lib/Parse/ParseStmt.cpp b/lib/Parse/ParseStmt.cpp
index df9b996..f604e03 100644
--- a/lib/Parse/ParseStmt.cpp
+++ b/lib/Parse/ParseStmt.cpp
@@ -17,6 +17,7 @@
 #include "clang/Sema/DeclSpec.h"
 #include "clang/Sema/PrettyDeclStackTrace.h"
 #include "clang/Sema/Scope.h"
+#include "clang/Sema/TypoCorrection.h"
 #include "clang/Basic/Diagnostic.h"
 #include "clang/Basic/PrettyStackTrace.h"
 #include "clang/Basic/SourceManager.h"
@@ -130,96 +131,38 @@ Retry:
       return ParseLabeledStatement(Attrs);
     }
 
+    // Look up the identifier, and typo-correct it to a keyword if it's not
+    // found.
     if (Next.isNot(tok::coloncolon)) {
-      CXXScopeSpec SS;
-      IdentifierInfo *Name = Tok.getIdentifierInfo();
-      SourceLocation NameLoc = Tok.getLocation();
-
-      if (getLangOpts().CPlusPlus)
-        CheckForTemplateAndDigraph(Next, ParsedType(),
-                                   /*EnteringContext=*/false, *Name, SS);
-
-      Sema::NameClassification Classification
-        = Actions.ClassifyName(getCurScope(), SS, Name, NameLoc, Next);
-      switch (Classification.getKind()) {
-      case Sema::NC_Keyword:
-        // The identifier was corrected to a keyword. Update the token
-        // to this keyword, and try again.
-        if (Name->getTokenID() != tok::identifier) {
-          Tok.setIdentifierInfo(Name);
-          Tok.setKind(Name->getTokenID());
-          goto Retry;
-        }
-
-        // Fall through via the normal error path.
-        // FIXME: This seems like it could only happen for context-sensitive
-        // keywords.
-
-      case Sema::NC_Error:
+      // Try to limit which sets of keywords should be included in typo
+      // correction based on what the next token is.
+      // FIXME: Pass the next token into the CorrectionCandidateCallback and
+      //        do this filtering in a more fine-grained manner.
+      CorrectionCandidateCallback DefaultValidator;
+      DefaultValidator.WantTypeSpecifiers =
+          Next.is(tok::l_paren) || Next.is(tok::less) ||
+          Next.is(tok::identifier) || Next.is(tok::star) ||
+          Next.is(tok::amp) || Next.is(tok::l_square);
+      DefaultValidator.WantExpressionKeywords =
+          Next.is(tok::l_paren) || Next.is(tok::identifier) ||
+          Next.is(tok::arrow) || Next.is(tok::period);
+      DefaultValidator.WantRemainingKeywords =
+          Next.is(tok::l_paren) || Next.is(tok::semi) ||
+          Next.is(tok::identifier) || Next.is(tok::l_brace);
+      DefaultValidator.WantCXXNamedCasts = false;
+      if (TryAnnotateName(/*IsAddressOfOperand*/false, &DefaultValidator)
+            == ANK_Error) {
         // Handle errors here by skipping up to the next semicolon or '}', and
         // eat the semicolon if that's what stopped us.
         SkipUntil(tok::r_brace, /*StopAtSemi=*/true, /*DontConsume=*/true);
         if (Tok.is(tok::semi))
           ConsumeToken();
         return StmtError();
-
-      case Sema::NC_Unknown:
-        // Either we don't know anything about this identifier, or we know that
-        // we're in a syntactic context we haven't handled yet.
-        break;
-
-      case Sema::NC_Type:
-        Tok.setKind(tok::annot_typename);
-        setTypeAnnotation(Tok, Classification.getType());
-        Tok.setAnnotationEndLoc(NameLoc);
-        PP.AnnotateCachedTokens(Tok);
-        break;
-
-      case Sema::NC_Expression:
-        Tok.setKind(tok::annot_primary_expr);
-        setExprAnnotation(Tok, Classification.getExpression());
-        Tok.setAnnotationEndLoc(NameLoc);
-        PP.AnnotateCachedTokens(Tok);
-        break;
-
-      case Sema::NC_TypeTemplate:
-      case Sema::NC_FunctionTemplate: {
-        ConsumeToken(); // the identifier
-        UnqualifiedId Id;
-        Id.setIdentifier(Name, NameLoc);
-        if (AnnotateTemplateIdToken(
-                            TemplateTy::make(Classification.getTemplateName()),
-                                    Classification.getTemplateNameKind(),
-                                    SS, SourceLocation(), Id,
-                                    /*AllowTypeAnnotation=*/false)) {
-          // Handle errors here by skipping up to the next semicolon or '}', and
-          // eat the semicolon if that's what stopped us.
-          SkipUntil(tok::r_brace, /*StopAtSemi=*/true, /*DontConsume=*/true);
-          if (Tok.is(tok::semi))
-            ConsumeToken();
-          return StmtError();
-        }
-
-        // If the next token is '::', jump right into parsing a
-        // nested-name-specifier. We don't want to leave the template-id
-        // hanging.
-        if (NextToken().is(tok::coloncolon) && TryAnnotateCXXScopeToken(false)){
-          // Handle errors here by skipping up to the next semicolon or '}', and
-          // eat the semicolon if that's what stopped us.
-          SkipUntil(tok::r_brace, /*StopAtSemi=*/true, /*DontConsume=*/true);
-          if (Tok.is(tok::semi))
-            ConsumeToken();
-          return StmtError();
-        }
-
-        // We've annotated a template-id, so try again now.
-        goto Retry;
       }
 
-      case Sema::NC_NestedNameSpecifier:
-        // FIXME: Implement this!
-        break;
-      }
+      // If the identifier was typo-corrected, try again.
+      if (Tok.isNot(tok::identifier))
+        goto Retry;
     }
 
     // Fall through
@@ -289,7 +232,7 @@ Retry:
     bool msAsm = false;
     Res = ParseAsmStatement(msAsm);
     Res = Actions.ActOnFinishFullStmt(Res.get());
-    if (msAsm) return move(Res);
+    if (msAsm) return Res;
     SemiError = "asm";
     break;
   }
@@ -310,6 +253,41 @@ Retry:
     ProhibitAttributes(Attrs);
     HandlePragmaPack();
     return StmtEmpty();
+
+  case tok::annot_pragma_msstruct:
+    ProhibitAttributes(Attrs);
+    HandlePragmaMSStruct();
+    return StmtEmpty();
+
+  case tok::annot_pragma_align:
+    ProhibitAttributes(Attrs);
+    HandlePragmaAlign();
+    return StmtEmpty();
+
+  case tok::annot_pragma_weak:
+    ProhibitAttributes(Attrs);
+    HandlePragmaWeak();
+    return StmtEmpty();
+
+  case tok::annot_pragma_weakalias:
+    ProhibitAttributes(Attrs);
+    HandlePragmaWeakAlias();
+    return StmtEmpty();
+
+  case tok::annot_pragma_redefine_extname:
+    ProhibitAttributes(Attrs);
+    HandlePragmaRedefineExtname();
+    return StmtEmpty();
+
+  case tok::annot_pragma_fp_contract:
+    Diag(Tok, diag::err_pragma_fp_contract_scope);
+    ConsumeToken();
+    return StmtError();
+
+  case tok::annot_pragma_opencl_extension:
+    ProhibitAttributes(Attrs);
+    HandlePragmaOpenCLExtension();
+    return StmtEmpty();
   }
 
   // If we reached this code, the statement must end in a semicolon.
@@ -324,7 +302,7 @@ Retry:
     SkipUntil(tok::r_brace, true, true);
   }
 
-  return move(Res);
+  return Res;
 }
 
 /// \brief Parse an expression statement.
@@ -381,7 +359,7 @@ StmtResult Parser::ParseSEHTryBlockCommon(SourceLocation TryLoc) {
 
   StmtResult TryBlock(ParseCompoundStatement());
   if(TryBlock.isInvalid())
-    return move(TryBlock);
+    return TryBlock;
 
   StmtResult Handler;
   if (Tok.is(tok::identifier) &&
@@ -396,7 +374,7 @@ StmtResult Parser::ParseSEHTryBlockCommon(SourceLocation TryLoc) {
   }
 
   if(Handler.isInvalid())
-    return move(Handler);
+    return Handler;
 
   return Actions.ActOnSEHTryBlock(false /* IsCXXTry */,
                                   TryLoc,
@@ -441,7 +419,7 @@ StmtResult Parser::ParseSEHExceptBlock(SourceLocation ExceptLoc) {
   StmtResult Block(ParseCompoundStatement());
 
   if(Block.isInvalid())
-    return move(Block);
+    return Block;
 
   return Actions.ActOnSEHExceptBlock(ExceptLoc, FilterExpr.take(), Block.take());
 }
@@ -458,7 +436,7 @@ StmtResult Parser::ParseSEHFinallyBlock(SourceLocation FinallyBlock) {
 
   StmtResult Block(ParseCompoundStatement());
   if(Block.isInvalid())
-    return move(Block);
+    return Block;
 
   return Actions.ActOnSEHFinallyBlock(FinallyBlock,Block.take());
 }
@@ -603,7 +581,7 @@ StmtResult Parser::ParseCaseStatement(bool MissingCase, ExprResult Expr) {
       // Otherwise we link it into the current chain.
       Stmt *NextDeepest = Case.get();
       if (TopLevelCase.isInvalid())
-        TopLevelCase = move(Case);
+        TopLevelCase = Case;
       else
         Actions.ActOnCaseStmtBody(DeepestParsedCaseStmt, Case.get());
       DeepestParsedCaseStmt = NextDeepest;
@@ -636,7 +614,7 @@ StmtResult Parser::ParseCaseStatement(bool MissingCase, ExprResult Expr) {
   Actions.ActOnCaseStmtBody(DeepestParsedCaseStmt, SubStmt.get());
 
   // Return the top level parsed statement tree.
-  return move(TopLevelCase);
+  return TopLevelCase;
 }
 
 /// ParseDefaultStatement
@@ -728,6 +706,48 @@ StmtResult Parser::ParseCompoundStatement(bool isStmtExpr,
   return ParseCompoundStatementBody(isStmtExpr);
 }
 
+/// Parse any pragmas at the start of the compound expression. We handle these
+/// separately since some pragmas (FP_CONTRACT) must appear before any C
+/// statement in the compound, but may be intermingled with other pragmas.
+void Parser::ParseCompoundStatementLeadingPragmas() {
+  bool checkForPragmas = true;
+  while (checkForPragmas) {
+    switch (Tok.getKind()) {
+    case tok::annot_pragma_vis:
+      HandlePragmaVisibility();
+      break;
+    case tok::annot_pragma_pack:
+      HandlePragmaPack();
+      break;
+    case tok::annot_pragma_msstruct:
+      HandlePragmaMSStruct();
+      break;
+    case tok::annot_pragma_align:
+      HandlePragmaAlign();
+      break;
+    case tok::annot_pragma_weak:
+      HandlePragmaWeak();
+      break;
+    case tok::annot_pragma_weakalias:
+      HandlePragmaWeakAlias();
+      break;
+    case tok::annot_pragma_redefine_extname:
+      HandlePragmaRedefineExtname();
+      break;
+    case tok::annot_pragma_opencl_extension:
+      HandlePragmaOpenCLExtension();
+      break;
+    case tok::annot_pragma_fp_contract:
+      HandlePragmaFPContract();
+      break;
+    default:
+      checkForPragmas = false;
+      break;
+    }
+  }
+
+}
+
 /// ParseCompoundStatementBody - Parse a sequence of statements and invoke the
 /// ActOnCompoundStmt action.  This expects the '{' to be the current token, and
 /// consume the '}' at the end of the block.  It does not manipulate the scope
@@ -736,6 +756,11 @@ StmtResult Parser::ParseCompoundStatementBody(bool isStmtExpr) {
   PrettyStackTraceLoc CrashInfo(PP.getSourceManager(),
                                 Tok.getLocation(),
                                 "in compound statement ('{}')");
+
+  // Record the state of the FP_CONTRACT pragma, restore on leaving the
+  // compound statement.
+  Sema::FPContractStateRAII SaveFPContractState(Actions);
+
   InMessageExpressionRAIIObject InMessage(*this, false);
   BalancedDelimiterTracker T(*this, tok::l_brace);
   if (T.consumeOpen())
@@ -743,7 +768,10 @@ StmtResult Parser::ParseCompoundStatementBody(bool isStmtExpr) {
 
   Sema::CompoundScopeRAII CompoundScope(Actions);
 
-  StmtVector Stmts(Actions);
+  // Parse any pragmas at the beginning of the compound statement.
+  ParseCompoundStatementLeadingPragmas();
+
+  StmtVector Stmts;
 
   // "__label__ X, Y, Z;" is the GNU "Local Label" extension.  These are
   // only allowed at the start of a compound stmt regardless of the language.
@@ -850,7 +878,7 @@ StmtResult Parser::ParseCompoundStatementBody(bool isStmtExpr) {
   }
 
   return Actions.ActOnCompoundStmt(T.getOpenLocation(), CloseLoc,
-                                   move_arg(Stmts), isStmtExpr);
+                                   Stmts, isStmtExpr);
 }
 
 /// ParseParenExprOrCondition:
@@ -1096,7 +1124,7 @@ StmtResult Parser::ParseSwitchStatement(SourceLocation *TrailingElseLoc) {
       SkipUntil(tok::r_brace, false, false);
     } else
       SkipUntil(tok::semi);
-    return move(Switch);
+    return Switch;
   }
 
   // C99 6.8.4p3 - In C99, the body of the switch statement is a scope, even if
@@ -1375,7 +1403,7 @@ StmtResult Parser::ParseForStatement(SourceLocation *TrailingElseLoc) {
     ColonProtectionRAIIObject ColonProtection(*this, MightBeForRangeStmt);
 
     SourceLocation DeclStart = Tok.getLocation(), DeclEnd;
-    StmtVector Stmts(Actions);
+    StmtVector Stmts;
     DeclGroupPtrTy DG = ParseSimpleDeclaration(Stmts, Declarator::ForContext,
                                                DeclEnd, attrs, false,
                                                MightBeForRangeStmt ?
@@ -1498,7 +1526,8 @@ StmtResult Parser::ParseForStatement(SourceLocation *TrailingElseLoc) {
     ForRangeStmt = Actions.ActOnCXXForRangeStmt(ForLoc, FirstPart.take(),
                                                 ForRangeInit.ColonLoc,
                                                 ForRangeInit.RangeExpr.get(),
-                                                T.getCloseLocation());
+                                                T.getCloseLocation(),
+                                                Sema::BFRK_Build);
 
 
   // Similarly, we need to do the semantic analysis for a for-range
@@ -1580,7 +1609,7 @@ StmtResult Parser::ParseGotoStatement() {
     return StmtError();
   }
 
-  return move(Res);
+  return Res;
 }
 
 /// ParseContinueStatement
@@ -1653,6 +1682,9 @@ StmtResult Parser::ParseReturnStatement() {
 ///         ms-asm-line '\n' ms-asm-instruction-block
 ///
 StmtResult Parser::ParseMicrosoftAsmStatement(SourceLocation AsmLoc) {
+  // MS-style inline assembly is not fully supported, so emit a warning.
+  Diag(AsmLoc, diag::warn_unsupported_msasm);
+
   SourceManager &SrcMgr = PP.getSourceManager();
   SourceLocation EndLoc = AsmLoc;
   SmallVector<Token, 4> AsmToks;
@@ -1745,6 +1777,21 @@ StmtResult Parser::ParseMicrosoftAsmStatement(SourceLocation AsmLoc) {
     return StmtError();
   }
 
+  // If MS-style inline assembly is disabled, then build an empty asm.
+  if (!getLangOpts().EmitMicrosoftInlineAsm) {
+    Token t;
+    t.setKind(tok::string_literal);
+    t.setLiteralData("\"/*FIXME: not done*/\"");
+    t.clearFlag(Token::NeedsCleaning);
+    t.setLength(21);
+    ExprResult AsmString(Actions.ActOnStringLiteral(&t, 1));
+    ExprVector Constraints;
+    ExprVector Exprs;
+    ExprVector Clobbers;
+    return Actions.ActOnGCCAsmStmt(AsmLoc, true, true, 0, 0, 0, Constraints,
+                                   Exprs, AsmString.take(), Clobbers, EndLoc);
+  }
+
   // FIXME: We should be passing source locations for better diagnostics.
   return Actions.ActOnMSAsmStmt(AsmLoc, LBraceLoc,
                                 llvm::makeArrayRef(AsmToks), EndLoc);
@@ -1806,18 +1853,17 @@ StmtResult Parser::ParseAsmStatement(bool &msAsm) {
   }
 
   SmallVector<IdentifierInfo *, 4> Names;
-  ExprVector Constraints(Actions);
-  ExprVector Exprs(Actions);
-  ExprVector Clobbers(Actions);
+  ExprVector Constraints;
+  ExprVector Exprs;
+  ExprVector Clobbers;
 
   if (Tok.is(tok::r_paren)) {
     // We have a simple asm expression like 'asm("foo")'.
     T.consumeClose();
-    return Actions.ActOnAsmStmt(AsmLoc, /*isSimple*/ true, isVolatile,
-                                /*NumOutputs*/ 0, /*NumInputs*/ 0, 0,
-                                move_arg(Constraints), move_arg(Exprs),
-                                AsmString.take(), move_arg(Clobbers),
-                                T.getCloseLocation());
+    return Actions.ActOnGCCAsmStmt(AsmLoc, /*isSimple*/ true, isVolatile,
+                                   /*NumOutputs*/ 0, /*NumInputs*/ 0, 0,
+                                   Constraints, Exprs, AsmString.take(),
+                                   Clobbers, T.getCloseLocation());
   }
 
   // Parse Outputs, if present.
@@ -1878,11 +1924,10 @@ StmtResult Parser::ParseAsmStatement(bool &msAsm) {
   }
 
   T.consumeClose();
-  return Actions.ActOnAsmStmt(AsmLoc, false, isVolatile,
-                              NumOutputs, NumInputs, Names.data(),
-                              move_arg(Constraints), move_arg(Exprs),
-                              AsmString.take(), move_arg(Clobbers),
-                              T.getCloseLocation());
+  return Actions.ActOnGCCAsmStmt(AsmLoc, false, isVolatile, NumOutputs,
+                                 NumInputs, Names.data(), Constraints, Exprs,
+                                 AsmString.take(), Clobbers,
+                                 T.getCloseLocation());
 }
 
 /// ParseAsmOperands - Parse the asm-operands production as used by
@@ -1975,7 +2020,7 @@ Decl *Parser::ParseFunctionStatementBody(Decl *Decl, ParseScope &BodyScope) {
   if (FnBody.isInvalid()) {
     Sema::CompoundScopeRAII CompoundScope(Actions);
     FnBody = Actions.ActOnCompoundStmt(LBraceLoc, LBraceLoc,
-                                       MultiStmtArg(Actions), false);
+                                       MultiStmtArg(), false);
   }
 
   BodyScope.Exit();
@@ -2006,13 +2051,13 @@ Decl *Parser::ParseFunctionTryBlock(Decl *Decl, ParseScope &BodyScope) {
   }
 
   SourceLocation LBraceLoc = Tok.getLocation();
-  StmtResult FnBody(ParseCXXTryBlockCommon(TryLoc));
+  StmtResult FnBody(ParseCXXTryBlockCommon(TryLoc, /*FnTry*/true));
   // If we failed to parse the try-catch, we just give the function an empty
   // compound statement as the body.
   if (FnBody.isInvalid()) {
     Sema::CompoundScopeRAII CompoundScope(Actions);
     FnBody = Actions.ActOnCompoundStmt(LBraceLoc, LBraceLoc,
-                                       MultiStmtArg(Actions), false);
+                                       MultiStmtArg(), false);
   }
 
   BodyScope.Exit();
@@ -2024,12 +2069,18 @@ bool Parser::trySkippingFunctionBody() {
   assert(SkipFunctionBodies &&
          "Should only be called when SkipFunctionBodies is enabled");
 
+  if (!PP.isCodeCompletionEnabled()) {
+    ConsumeBrace();
+    SkipUntil(tok::r_brace, /*StopAtSemi=*/false, /*DontConsume=*/false);
+    return true;
+  }
+
   // We're in code-completion mode. Skip parsing for all function bodies unless
   // the body contains the code-completion point.
   TentativeParsingAction PA(*this);
   ConsumeBrace();
   if (SkipUntil(tok::r_brace, /*StopAtSemi=*/false, /*DontConsume=*/false,
-                /*StopAtCodeCompletion=*/PP.isCodeCompletionEnabled())) {
+                /*StopAtCodeCompletion=*/true)) {
     PA.Commit();
     return true;
   }
@@ -2066,15 +2117,16 @@ StmtResult Parser::ParseCXXTryBlock() {
 ///         'try' compound-statement seh-except-block
 ///         'try' compound-statment  seh-finally-block
 ///
-StmtResult Parser::ParseCXXTryBlockCommon(SourceLocation TryLoc) {
+StmtResult Parser::ParseCXXTryBlockCommon(SourceLocation TryLoc, bool FnTry) {
   if (Tok.isNot(tok::l_brace))
     return StmtError(Diag(Tok, diag::err_expected_lbrace));
   // FIXME: Possible draft standard bug: attribute-specifier should be allowed?
 
   StmtResult TryBlock(ParseCompoundStatement(/*isStmtExpr=*/false,
-                                             Scope::DeclScope|Scope::TryScope));
+                      Scope::DeclScope |
+                        (FnTry ? Scope::FnTryScope : Scope::TryScope)));
   if (TryBlock.isInvalid())
-    return move(TryBlock);
+    return TryBlock;
 
   // Borland allows SEH-handlers with 'try'
 
@@ -2092,7 +2144,7 @@ StmtResult Parser::ParseCXXTryBlockCommon(SourceLocation TryLoc) {
       Handler = ParseSEHFinallyBlock(Loc);
     }
     if(Handler.isInvalid())
-      return move(Handler);
+      return Handler;
 
     return Actions.ActOnSEHTryBlock(true /* IsCXXTry */,
                                     TryLoc,
@@ -2100,7 +2152,7 @@ StmtResult Parser::ParseCXXTryBlockCommon(SourceLocation TryLoc) {
                                     Handler.take());
   }
   else {
-    StmtVector Handlers(Actions);
+    StmtVector Handlers;
     ParsedAttributesWithRange attrs(AttrFactory);
     MaybeParseCXX0XAttributes(attrs);
     ProhibitAttributes(attrs);
@@ -2108,7 +2160,7 @@ StmtResult Parser::ParseCXXTryBlockCommon(SourceLocation TryLoc) {
     if (Tok.isNot(tok::kw_catch))
       return StmtError(Diag(Tok, diag::err_expected_catch));
     while (Tok.is(tok::kw_catch)) {
-      StmtResult Handler(ParseCXXCatchBlock());
+      StmtResult Handler(ParseCXXCatchBlock(FnTry));
       if (!Handler.isInvalid())
         Handlers.push_back(Handler.release());
     }
@@ -2117,7 +2169,7 @@ StmtResult Parser::ParseCXXTryBlockCommon(SourceLocation TryLoc) {
     if (Handlers.empty())
       return StmtError();
 
-    return Actions.ActOnCXXTryBlock(TryLoc, TryBlock.take(),move_arg(Handlers));
+    return Actions.ActOnCXXTryBlock(TryLoc, TryBlock.take(),Handlers);
   }
 }
 
@@ -2132,7 +2184,7 @@ StmtResult Parser::ParseCXXTryBlockCommon(SourceLocation TryLoc) {
 ///         type-specifier-seq
 ///         '...'
 ///
-StmtResult Parser::ParseCXXCatchBlock() {
+StmtResult Parser::ParseCXXCatchBlock(bool FnCatch) {
   assert(Tok.is(tok::kw_catch) && "Expected 'catch'");
 
   SourceLocation CatchLoc = ConsumeToken();
@@ -2144,7 +2196,8 @@ StmtResult Parser::ParseCXXCatchBlock() {
   // C++ 3.3.2p3:
   // The name in a catch exception-declaration is local to the handler and
   // shall not be redeclared in the outermost block of the handler.
-  ParseScope CatchScope(this, Scope::DeclScope | Scope::ControlScope);
+  ParseScope CatchScope(this, Scope::DeclScope | Scope::ControlScope |
+                          (FnCatch ? Scope::FnCatchScope : 0));
 
   // exception-declaration is equivalent to '...' or a parameter-declaration
   // without default arguments.
@@ -2169,7 +2222,7 @@ StmtResult Parser::ParseCXXCatchBlock() {
   // FIXME: Possible draft standard bug: attribute-specifier should be allowed?
   StmtResult Block(ParseCompoundStatement());
   if (Block.isInvalid())
-    return move(Block);
+    return Block;
 
   return Actions.ActOnCXXCatchBlock(CatchLoc, ExceptionDecl, Block.take());
 }
diff --git a/lib/Parse/ParseTemplate.cpp b/lib/Parse/ParseTemplate.cpp
index ade918f..2e0411e 100644
--- a/lib/Parse/ParseTemplate.cpp
+++ b/lib/Parse/ParseTemplate.cpp
@@ -246,7 +246,7 @@ Parser::ParseSingleDeclarationAfterTemplate(
     return 0;
   }
 
-  LateParsedAttrList LateParsedAttrs;
+  LateParsedAttrList LateParsedAttrs(true);
   if (DeclaratorInfo.isFunctionDeclarator())
     MaybeParseGNUAttributes(DeclaratorInfo, &LateParsedAttrs);
 
@@ -889,8 +889,7 @@ bool Parser::AnnotateTemplateIdToken(TemplateTy Template, TemplateNameKind TNK,
     return true;
   }
 
-  ASTTemplateArgsPtr TemplateArgsPtr(Actions, TemplateArgs.data(),
-                                     TemplateArgs.size());
+  ASTTemplateArgsPtr TemplateArgsPtr(TemplateArgs);
 
   // Build the annotation token.
   if (TNK == TNK_Type_template && AllowTypeAnnotation) {
@@ -942,8 +941,6 @@ bool Parser::AnnotateTemplateIdToken(TemplateTy Template, TemplateNameKind TNK,
       Tok.setLocation(TemplateKWLoc);
     else
       Tok.setLocation(TemplateNameLoc);
-
-    TemplateArgsPtr.release();
   }
 
   // Common fields for the annotation token
@@ -969,8 +966,7 @@ void Parser::AnnotateTemplateIdTokenAsType() {
           TemplateId->Kind == TNK_Dependent_template_name) &&
          "Only works for type and dependent templates");
 
-  ASTTemplateArgsPtr TemplateArgsPtr(Actions,
-                                     TemplateId->getTemplateArgs(),
+  ASTTemplateArgsPtr TemplateArgsPtr(TemplateId->getTemplateArgs(),
                                      TemplateId->NumArgs);
 
   TypeResult Type
diff --git a/lib/Parse/ParseTentative.cpp b/lib/Parse/ParseTentative.cpp
index 1a4df47..40c4eee 100644
--- a/lib/Parse/ParseTentative.cpp
+++ b/lib/Parse/ParseTentative.cpp
@@ -104,16 +104,27 @@ bool Parser::isCXXSimpleDeclaration(bool AllowForRangeDecl) {
   // isCXXDeclarationSpecifier will return TPResult::Ambiguous() only in such
   // a case.
 
-  TPResult TPR = isCXXDeclarationSpecifier();
+  bool InvalidAsDeclaration = false;
+  TPResult TPR = isCXXDeclarationSpecifier(TPResult::False(),
+                                           &InvalidAsDeclaration);
   if (TPR != TPResult::Ambiguous())
     return TPR != TPResult::False(); // Returns true for TPResult::True() or
                                      // TPResult::Error().
 
+  // FIXME: TryParseSimpleDeclaration doesn't look past the first initializer,
+  // and so gets some cases wrong. We can't carry on if we've already seen
+  // something which makes this statement invalid as a declaration in this case,
+  // since it can cause us to misparse valid code. Revisit this once
+  // TryParseInitDeclaratorList is fixed.
+  if (InvalidAsDeclaration)
+    return false;
+
   // FIXME: Add statistics about the number of ambiguous statements encountered
   // and how they were resolved (number of declarations+number of expressions).
 
-  // Ok, we have a simple-type-specifier/typename-specifier followed by a '('.
-  // We need tentative parsing...
+  // Ok, we have a simple-type-specifier/typename-specifier followed by a '(',
+  // or an identifier which doesn't resolve as anything. We need tentative
+  // parsing...
 
   TentativeParsingAction PA(*this);
   TPR = TryParseSimpleDeclaration(AllowForRangeDecl);
@@ -140,20 +151,28 @@ bool Parser::isCXXSimpleDeclaration(bool AllowForRangeDecl) {
 ///    attribute-specifier-seqopt type-specifier-seq declarator
 ///
 Parser::TPResult Parser::TryParseSimpleDeclaration(bool AllowForRangeDecl) {
-  // We know that we have a simple-type-specifier/typename-specifier followed
-  // by a '('.
-  assert(isCXXDeclarationSpecifier() == TPResult::Ambiguous());
-
   if (Tok.is(tok::kw_typeof))
     TryParseTypeofSpecifier();
   else {
+    if (Tok.is(tok::annot_cxxscope))
+      ConsumeToken();
     ConsumeToken();
-    
+
     if (getLangOpts().ObjC1 && Tok.is(tok::less))
       TryParseProtocolQualifiers();
   }
-  
-  assert(Tok.is(tok::l_paren) && "Expected '('");
+
+  // Two decl-specifiers in a row conclusively disambiguate this as being a
+  // simple-declaration. Don't bother calling isCXXDeclarationSpecifier in the
+  // overwhelmingly common case that the next token is a '('.
+  if (Tok.isNot(tok::l_paren)) {
+    TPResult TPR = isCXXDeclarationSpecifier();
+    if (TPR == TPResult::Ambiguous())
+      return TPResult::True();
+    if (TPR == TPResult::True() || TPR == TPResult::Error())
+      return TPR;
+    assert(TPR == TPResult::False());
+  }
 
   TPResult TPR = TryParseInitDeclaratorList();
   if (TPR != TPResult::Ambiguous())
@@ -623,6 +642,8 @@ Parser::TPResult Parser::TryParseDeclarator(bool mayBeAbstract,
     // declarator-id
     if (Tok.is(tok::annot_cxxscope))
       ConsumeToken();
+    else
+      TentativelyDeclaredIdentifiers.push_back(Tok.getIdentifierInfo());
     ConsumeToken();
   } else if (Tok.is(tok::l_paren)) {
     ConsumeParen();
@@ -761,6 +782,7 @@ Parser::isExpressionOrTypeSpecifierSimple(tok::TokenKind Kind) {
   case tok::kw___is_convertible_to:
   case tok::kw___is_empty:
   case tok::kw___is_enum:
+  case tok::kw___is_interface_class:
   case tok::kw___is_final:
   case tok::kw___is_literal:
   case tok::kw___is_literal_type:
@@ -824,6 +846,12 @@ Parser::isExpressionOrTypeSpecifierSimple(tok::TokenKind Kind) {
   return TPResult::Ambiguous();
 }
 
+bool Parser::isTentativelyDeclared(IdentifierInfo *II) {
+  return std::find(TentativelyDeclaredIdentifiers.begin(),
+                   TentativelyDeclaredIdentifiers.end(), II)
+      != TentativelyDeclaredIdentifiers.end();
+}
+
 /// isCXXDeclarationSpecifier - Returns TPResult::True() if it is a declaration
 /// specifier, TPResult::False() if it is not, TPResult::Ambiguous() if it could
 /// be either a decl-specifier or a function-style cast, and TPResult::Error()
@@ -831,7 +859,10 @@ Parser::isExpressionOrTypeSpecifierSimple(tok::TokenKind Kind) {
 ///
 /// If HasMissingTypename is provided, a name with a dependent scope specifier
 /// will be treated as ambiguous if the 'typename' keyword is missing. If this
-/// happens, *HasMissingTypename will be set to 'true'.
+/// happens, *HasMissingTypename will be set to 'true'. This will also be used
+/// as an indicator that undeclared identifiers (which will trigger a later
+/// parse error) should be treated as types. Returns TPResult::Ambiguous() in
+/// such cases.
 ///
 ///         decl-specifier:
 ///           storage-class-specifier
@@ -927,22 +958,64 @@ Parser::TPResult
 Parser::isCXXDeclarationSpecifier(Parser::TPResult BracedCastResult,
                                   bool *HasMissingTypename) {
   switch (Tok.getKind()) {
-  case tok::identifier:   // foo::bar
+  case tok::identifier: {
     // Check for need to substitute AltiVec __vector keyword
     // for "vector" identifier.
     if (TryAltiVecVectorToken())
       return TPResult::True();
-    // Fall through.
+
+    const Token &Next = NextToken();
+    // In 'foo bar', 'foo' is always a type name outside of Objective-C.
+    if (!getLangOpts().ObjC1 && Next.is(tok::identifier))
+      return TPResult::True();
+
+    if (Next.isNot(tok::coloncolon) && Next.isNot(tok::less)) {
+      // Determine whether this is a valid expression. If not, we will hit
+      // a parse error one way or another. In that case, tell the caller that
+      // this is ambiguous. Typo-correct to type and expression keywords and
+      // to types and identifiers, in order to try to recover from errors.
+      CorrectionCandidateCallback TypoCorrection;
+      TypoCorrection.WantRemainingKeywords = false;
+      switch (TryAnnotateName(false /* no nested name specifier */,
+                              &TypoCorrection)) {
+      case ANK_Error:
+        return TPResult::Error();
+      case ANK_TentativeDecl:
+        return TPResult::False();
+      case ANK_TemplateName:
+        // A bare type template-name which can't be a template template
+        // argument is an error, and was probably intended to be a type.
+        return GreaterThanIsOperator ? TPResult::True() : TPResult::False();
+      case ANK_Unresolved:
+        return HasMissingTypename ? TPResult::Ambiguous() : TPResult::False();
+      case ANK_Success:
+        break;
+      }
+      assert(Tok.isNot(tok::identifier) &&
+             "TryAnnotateName succeeded without producing an annotation");
+    } else {
+      // This might possibly be a type with a dependent scope specifier and
+      // a missing 'typename' keyword. Don't use TryAnnotateName in this case,
+      // since it will annotate as a primary expression, and we want to use the
+      // "missing 'typename'" logic.
+      if (TryAnnotateTypeOrScopeToken())
+        return TPResult::Error();
+      // If annotation failed, assume it's a non-type.
+      // FIXME: If this happens due to an undeclared identifier, treat it as
+      // ambiguous.
+      if (Tok.is(tok::identifier))
+        return TPResult::False();
+    }
+
+    // We annotated this token as something. Recurse to handle whatever we got.
+    return isCXXDeclarationSpecifier(BracedCastResult, HasMissingTypename);
+  }
+
   case tok::kw_typename:  // typename T::type
     // Annotate typenames and C++ scope specifiers.  If we get one, just
     // recurse to handle whatever we get.
     if (TryAnnotateTypeOrScopeToken())
       return TPResult::Error();
-    if (Tok.is(tok::identifier)) {
-      const Token &Next = NextToken();
-      return (!getLangOpts().ObjC1 && Next.is(tok::identifier)) ?
-          TPResult::True() : TPResult::False();
-    }
     return isCXXDeclarationSpecifier(BracedCastResult, HasMissingTypename);
 
   case tok::coloncolon: {    // ::foo::bar
@@ -1073,6 +1146,28 @@ Parser::isCXXDeclarationSpecifier(Parser::TPResult BracedCastResult,
             *HasMissingTypename = true;
             return TPResult::Ambiguous();
           }
+        } else {
+          // Try to resolve the name. If it doesn't exist, assume it was
+          // intended to name a type and keep disambiguating.
+          switch (TryAnnotateName(false /* SS is not dependent */)) {
+          case ANK_Error:
+            return TPResult::Error();
+          case ANK_TentativeDecl:
+            return TPResult::False();
+          case ANK_TemplateName:
+            // A bare type template-name which can't be a template template
+            // argument is an error, and was probably intended to be a type.
+            return GreaterThanIsOperator ? TPResult::True() : TPResult::False();
+          case ANK_Unresolved:
+            return HasMissingTypename ? TPResult::Ambiguous()
+                                      : TPResult::False();
+          case ANK_Success:
+            // Annotated it, check again.
+            assert(Tok.isNot(tok::annot_cxxscope) ||
+                   NextToken().isNot(tok::identifier));
+            return isCXXDeclarationSpecifier(BracedCastResult,
+                                             HasMissingTypename);
+          }
         }
       }
       return TPResult::False();
diff --git a/lib/Parse/Parser.cpp b/lib/Parse/Parser.cpp
index 3725e2b..f4cdd61 100644
--- a/lib/Parse/Parser.cpp
+++ b/lib/Parse/Parser.cpp
@@ -23,6 +23,7 @@
 #include "clang/AST/ASTConsumer.h"
 using namespace clang;
 
+
 namespace {
 /// \brief A comment handler that passes comments found by the preprocessor
 /// to the parser action.
@@ -47,11 +48,13 @@ IdentifierInfo *Parser::getSEHExceptKeyword() {
   return Ident__except;
 }
 
-Parser::Parser(Preprocessor &pp, Sema &actions, bool SkipFunctionBodies)
+Parser::Parser(Preprocessor &pp, Sema &actions, bool skipFunctionBodies)
   : PP(pp), Actions(actions), Diags(PP.getDiagnostics()),
     GreaterThanIsOperator(true), ColonIsSacred(false), 
     InMessageExpression(false), TemplateParameterDepth(0),
-    ParsingInObjCContainer(false), SkipFunctionBodies(SkipFunctionBodies) {
+    ParsingInObjCContainer(false) {
+  SkipFunctionBodies = pp.isCodeCompletionEnabled() || skipFunctionBodies;
+  Tok.startToken();
   Tok.setKind(tok::eof);
   Actions.CurScope = 0;
   NumCachedScopes = 0;
@@ -60,35 +63,35 @@ Parser::Parser(Preprocessor &pp, Sema &actions, bool SkipFunctionBodies)
 
   // Add #pragma handlers. These are removed and destroyed in the
   // destructor.
-  AlignHandler.reset(new PragmaAlignHandler(actions));
+  AlignHandler.reset(new PragmaAlignHandler());
   PP.AddPragmaHandler(AlignHandler.get());
 
-  GCCVisibilityHandler.reset(new PragmaGCCVisibilityHandler(actions));
+  GCCVisibilityHandler.reset(new PragmaGCCVisibilityHandler());
   PP.AddPragmaHandler("GCC", GCCVisibilityHandler.get());
 
-  OptionsHandler.reset(new PragmaOptionsHandler(actions));
+  OptionsHandler.reset(new PragmaOptionsHandler());
   PP.AddPragmaHandler(OptionsHandler.get());
 
-  PackHandler.reset(new PragmaPackHandler(actions));
+  PackHandler.reset(new PragmaPackHandler());
   PP.AddPragmaHandler(PackHandler.get());
     
-  MSStructHandler.reset(new PragmaMSStructHandler(actions));
+  MSStructHandler.reset(new PragmaMSStructHandler());
   PP.AddPragmaHandler(MSStructHandler.get());
 
-  UnusedHandler.reset(new PragmaUnusedHandler(actions));
+  UnusedHandler.reset(new PragmaUnusedHandler());
   PP.AddPragmaHandler(UnusedHandler.get());
 
-  WeakHandler.reset(new PragmaWeakHandler(actions));
+  WeakHandler.reset(new PragmaWeakHandler());
   PP.AddPragmaHandler(WeakHandler.get());
 
-  RedefineExtnameHandler.reset(new PragmaRedefineExtnameHandler(actions));
+  RedefineExtnameHandler.reset(new PragmaRedefineExtnameHandler());
   PP.AddPragmaHandler(RedefineExtnameHandler.get());
 
-  FPContractHandler.reset(new PragmaFPContractHandler(actions));
+  FPContractHandler.reset(new PragmaFPContractHandler());
   PP.AddPragmaHandler("STDC", FPContractHandler.get());
 
   if (getLangOpts().OpenCL) {
-    OpenCLExtensionHandler.reset(new PragmaOpenCLExtensionHandler(actions));
+    OpenCLExtensionHandler.reset(new PragmaOpenCLExtensionHandler());
     PP.AddPragmaHandler("OPENCL", OpenCLExtensionHandler.get());
 
     PP.AddPragmaHandler("OPENCL", FPContractHandler.get());
@@ -135,7 +138,7 @@ DiagnosticBuilder Parser::Diag(const Token &Tok, unsigned DiagID) {
 /// given range.
 ///
 /// \param Loc The location where we'll emit the diagnostic.
-/// \param Loc The kind of diagnostic to emit.
+/// \param DK The kind of diagnostic to emit.
 /// \param ParenRange Source range enclosing code that should be parenthesized.
 void Parser::SuggestParentheses(SourceLocation Loc, unsigned DK,
                                 SourceRange ParenRange) {
@@ -154,7 +157,8 @@ void Parser::SuggestParentheses(SourceLocation Loc, unsigned DK,
 
 static bool IsCommonTypo(tok::TokenKind ExpectedTok, const Token &Tok) {
   switch (ExpectedTok) {
-  case tok::semi: return Tok.is(tok::colon); // : for ;
+  case tok::semi:
+    return Tok.is(tok::colon) || Tok.is(tok::comma); // : or , for ;
   default: return false;
   }
 }
@@ -466,9 +470,6 @@ void Parser::Initialize() {
   EnterScope(Scope::DeclScope);
   Actions.ActOnTranslationUnitScope(getCurScope());
 
-  // Prime the lexer look-ahead.
-  ConsumeToken();
-
   // Initialization for Objective-C context sensitive keywords recognition.
   // Referenced in Parser::ParseObjCTypeQualifierList.
   if (getLangOpts().ObjC1) {
@@ -523,6 +524,11 @@ void Parser::Initialize() {
     PP.SetPoisonReason(Ident___abnormal_termination,diag::err_seh___finally_block);
     PP.SetPoisonReason(Ident_AbnormalTermination,diag::err_seh___finally_block);
   }
+
+  Actions.Initialize();
+
+  // Prime the lexer look-ahead.
+  ConsumeToken();
 }
 
 namespace {
@@ -634,6 +640,27 @@ Parser::ParseExternalDeclaration(ParsedAttributesWithRange &attrs,
   case tok::annot_pragma_pack:
     HandlePragmaPack();
     return DeclGroupPtrTy();
+  case tok::annot_pragma_msstruct:
+    HandlePragmaMSStruct();
+    return DeclGroupPtrTy();
+  case tok::annot_pragma_align:
+    HandlePragmaAlign();
+    return DeclGroupPtrTy();
+  case tok::annot_pragma_weak:
+    HandlePragmaWeak();
+    return DeclGroupPtrTy();
+  case tok::annot_pragma_weakalias:
+    HandlePragmaWeakAlias();
+    return DeclGroupPtrTy();
+  case tok::annot_pragma_redefine_extname:
+    HandlePragmaRedefineExtname();
+    return DeclGroupPtrTy();
+  case tok::annot_pragma_fp_contract:
+    HandlePragmaFPContract();
+    return DeclGroupPtrTy();
+  case tok::annot_pragma_opencl_extension:
+    HandlePragmaOpenCLExtension();
+    return DeclGroupPtrTy();
   case tok::semi:
     ConsumeExtraSemi(OutsideFunction);
     // TODO: Invoke action for top-level semicolon.
@@ -693,7 +720,7 @@ Parser::ParseExternalDeclaration(ParsedAttributesWithRange &attrs,
     // A function definition cannot start with any of these keywords.
     {
       SourceLocation DeclEnd;
-      StmtVector Stmts(Actions);
+      StmtVector Stmts;
       return ParseDeclaration(Stmts, Declarator::FileContext, DeclEnd, attrs);
     }
 
@@ -704,7 +731,7 @@ Parser::ParseExternalDeclaration(ParsedAttributesWithRange &attrs,
       Diag(ConsumeToken(), diag::warn_static_inline_explicit_inst_ignored)
         << 0;
       SourceLocation DeclEnd;
-      StmtVector Stmts(Actions);
+      StmtVector Stmts;
       return ParseDeclaration(Stmts, Declarator::FileContext, DeclEnd, attrs);  
     }
     goto dont_know;
@@ -716,7 +743,7 @@ Parser::ParseExternalDeclaration(ParsedAttributesWithRange &attrs,
       // Inline namespaces. Allowed as an extension even in C++03.
       if (NextKind == tok::kw_namespace) {
         SourceLocation DeclEnd;
-        StmtVector Stmts(Actions);
+        StmtVector Stmts;
         return ParseDeclaration(Stmts, Declarator::FileContext, DeclEnd, attrs);
       }
       
@@ -726,7 +753,7 @@ Parser::ParseExternalDeclaration(ParsedAttributesWithRange &attrs,
         Diag(ConsumeToken(), diag::warn_static_inline_explicit_inst_ignored)
           << 1;
         SourceLocation DeclEnd;
-        StmtVector Stmts(Actions);
+        StmtVector Stmts;
         return ParseDeclaration(Stmts, Declarator::FileContext, DeclEnd, attrs);  
       }
     }
@@ -972,16 +999,14 @@ Decl *Parser::ParseFunctionDefinition(ParsingDeclarator &D,
   if (getLangOpts().DelayedTemplateParsing &&
       Tok.isNot(tok::equal) &&
       TemplateInfo.Kind == ParsedTemplateInfo::Template) {
-    MultiTemplateParamsArg TemplateParameterLists(Actions,
-                                         TemplateInfo.TemplateParams->data(),
-                                         TemplateInfo.TemplateParams->size());
+    MultiTemplateParamsArg TemplateParameterLists(*TemplateInfo.TemplateParams);
     
     ParseScope BodyScope(this, Scope::FnScope|Scope::DeclScope);
     Scope *ParentScope = getCurScope()->getParent();
 
     D.setFunctionDefinitionKind(FDK_Definition);
     Decl *DP = Actions.HandleDeclarator(ParentScope, D,
-                                        move(TemplateParameterLists));
+                                        TemplateParameterLists);
     D.complete(DP);
     D.getMutableDeclSpec().abort();
 
@@ -1009,13 +1034,12 @@ Decl *Parser::ParseFunctionDefinition(ParsingDeclarator &D,
            (Tok.is(tok::l_brace) || Tok.is(tok::kw_try) ||
             Tok.is(tok::colon)) && 
       Actions.CurContext->isTranslationUnit()) {
-    MultiTemplateParamsArg TemplateParameterLists(Actions, 0, 0);
     ParseScope BodyScope(this, Scope::FnScope|Scope::DeclScope);
     Scope *ParentScope = getCurScope()->getParent();
     
     D.setFunctionDefinitionKind(FDK_Definition);
     Decl *FuncDecl = Actions.HandleDeclarator(ParentScope, D,
-                                        move(TemplateParameterLists));
+                                              MultiTemplateParamsArg());
     D.complete(FuncDecl);
     D.getMutableDeclSpec().abort();
     if (FuncDecl) {
@@ -1033,10 +1057,7 @@ Decl *Parser::ParseFunctionDefinition(ParsingDeclarator &D,
   // specified Declarator for the function.
   Decl *Res = TemplateInfo.TemplateParams?
       Actions.ActOnStartOfFunctionTemplateDef(getCurScope(),
-                              MultiTemplateParamsArg(Actions,
-                                          TemplateInfo.TemplateParams->data(),
-                                         TemplateInfo.TemplateParams->size()),
-                                              D)
+                                              *TemplateInfo.TemplateParams, D)
     : Actions.ActOnStartOfFunctionDef(getCurScope(), D);
 
   // Break out of the ParsingDeclarator context before we parse the body.
@@ -1288,7 +1309,7 @@ Parser::ExprResult Parser::ParseSimpleAsm(SourceLocation *EndLoc) {
       *EndLoc = T.getCloseLocation();
   }
 
-  return move(Result);
+  return Result;
 }
 
 /// \brief Get the TemplateIdAnnotation from the token and put it in the
@@ -1301,6 +1322,143 @@ TemplateIdAnnotation *Parser::takeTemplateIdAnnotation(const Token &tok) {
   return Id;
 }
 
+void Parser::AnnotateScopeToken(CXXScopeSpec &SS, bool IsNewAnnotation) {
+  // Push the current token back into the token stream (or revert it if it is
+  // cached) and use an annotation scope token for current token.
+  if (PP.isBacktrackEnabled())
+    PP.RevertCachedTokens(1);
+  else
+    PP.EnterToken(Tok);
+  Tok.setKind(tok::annot_cxxscope);
+  Tok.setAnnotationValue(Actions.SaveNestedNameSpecifierAnnotation(SS));
+  Tok.setAnnotationRange(SS.getRange());
+
+  // In case the tokens were cached, have Preprocessor replace them
+  // with the annotation token.  We don't need to do this if we've
+  // just reverted back to a prior state.
+  if (IsNewAnnotation)
+    PP.AnnotateCachedTokens(Tok);
+}
+
+/// \brief Attempt to classify the name at the current token position. This may
+/// form a type, scope or primary expression annotation, or replace the token
+/// with a typo-corrected keyword. This is only appropriate when the current
+/// name must refer to an entity which has already been declared.
+///
+/// \param IsAddressOfOperand Must be \c true if the name is preceded by an '&'
+///        and might possibly have a dependent nested name specifier.
+/// \param CCC Indicates how to perform typo-correction for this name. If NULL,
+///        no typo correction will be performed.
+Parser::AnnotatedNameKind
+Parser::TryAnnotateName(bool IsAddressOfOperand,
+                        CorrectionCandidateCallback *CCC) {
+  assert(Tok.is(tok::identifier) || Tok.is(tok::annot_cxxscope));
+
+  const bool EnteringContext = false;
+  const bool WasScopeAnnotation = Tok.is(tok::annot_cxxscope);
+
+  CXXScopeSpec SS;
+  if (getLangOpts().CPlusPlus &&
+      ParseOptionalCXXScopeSpecifier(SS, ParsedType(), EnteringContext))
+    return ANK_Error;
+
+  if (Tok.isNot(tok::identifier) || SS.isInvalid()) {
+    if (TryAnnotateTypeOrScopeTokenAfterScopeSpec(EnteringContext, false, SS,
+                                                  !WasScopeAnnotation))
+      return ANK_Error;
+    return ANK_Unresolved;
+  }
+
+  IdentifierInfo *Name = Tok.getIdentifierInfo();
+  SourceLocation NameLoc = Tok.getLocation();
+
+  // FIXME: Move the tentative declaration logic into ClassifyName so we can
+  // typo-correct to tentatively-declared identifiers.
+  if (isTentativelyDeclared(Name)) {
+    // Identifier has been tentatively declared, and thus cannot be resolved as
+    // an expression. Fall back to annotating it as a type.
+    if (TryAnnotateTypeOrScopeTokenAfterScopeSpec(EnteringContext, false, SS,
+                                                  !WasScopeAnnotation))
+      return ANK_Error;
+    return Tok.is(tok::annot_typename) ? ANK_Success : ANK_TentativeDecl;
+  }
+
+  Token Next = NextToken();
+
+  // Look up and classify the identifier. We don't perform any typo-correction
+  // after a scope specifier, because in general we can't recover from typos
+  // there (eg, after correcting 'A::tempalte B<X>::C', we would need to jump
+  // back into scope specifier parsing).
+  Sema::NameClassification Classification
+    = Actions.ClassifyName(getCurScope(), SS, Name, NameLoc, Next,
+                           IsAddressOfOperand, SS.isEmpty() ? CCC : 0);
+
+  switch (Classification.getKind()) {
+  case Sema::NC_Error:
+    return ANK_Error;
+
+  case Sema::NC_Keyword:
+    // The identifier was typo-corrected to a keyword.
+    Tok.setIdentifierInfo(Name);
+    Tok.setKind(Name->getTokenID());
+    PP.TypoCorrectToken(Tok);
+    if (SS.isNotEmpty())
+      AnnotateScopeToken(SS, !WasScopeAnnotation);
+    // We've "annotated" this as a keyword.
+    return ANK_Success;
+
+  case Sema::NC_Unknown:
+    // It's not something we know about. Leave it unannotated.
+    break;
+
+  case Sema::NC_Type:
+    Tok.setKind(tok::annot_typename);
+    setTypeAnnotation(Tok, Classification.getType());
+    Tok.setAnnotationEndLoc(NameLoc);
+    if (SS.isNotEmpty())
+      Tok.setLocation(SS.getBeginLoc());
+    PP.AnnotateCachedTokens(Tok);
+    return ANK_Success;
+
+  case Sema::NC_Expression:
+    Tok.setKind(tok::annot_primary_expr);
+    setExprAnnotation(Tok, Classification.getExpression());
+    Tok.setAnnotationEndLoc(NameLoc);
+    if (SS.isNotEmpty())
+      Tok.setLocation(SS.getBeginLoc());
+    PP.AnnotateCachedTokens(Tok);
+    return ANK_Success;
+
+  case Sema::NC_TypeTemplate:
+    if (Next.isNot(tok::less)) {
+      // This may be a type template being used as a template template argument.
+      if (SS.isNotEmpty())
+        AnnotateScopeToken(SS, !WasScopeAnnotation);
+      return ANK_TemplateName;
+    }
+    // Fall through.
+  case Sema::NC_FunctionTemplate: {
+    // We have a type or function template followed by '<'.
+    ConsumeToken();
+    UnqualifiedId Id;
+    Id.setIdentifier(Name, NameLoc);
+    if (AnnotateTemplateIdToken(
+            TemplateTy::make(Classification.getTemplateName()),
+            Classification.getTemplateNameKind(), SS, SourceLocation(), Id))
+      return ANK_Error;
+    return ANK_Success;
+  }
+
+  case Sema::NC_NestedNameSpecifier:
+    llvm_unreachable("already parsed nested name specifier");
+  }
+
+  // Unable to classify the name, but maybe we can annotate a scope specifier.
+  if (SS.isNotEmpty())
+    AnnotateScopeToken(SS, !WasScopeAnnotation);
+  return ANK_Unresolved;
+}
+
 /// TryAnnotateTypeOrScopeToken - If the current token position is on a
 /// typename (possibly qualified in C++) or a C++ scope specifier not followed
 /// by a typename, TryAnnotateTypeOrScopeToken will replace one or more tokens
@@ -1377,8 +1535,7 @@ bool Parser::TryAnnotateTypeOrScopeToken(bool EnteringContext, bool NeedType) {
         return true;
       }
 
-      ASTTemplateArgsPtr TemplateArgsPtr(Actions,
-                                         TemplateId->getTemplateArgs(),
+      ASTTemplateArgsPtr TemplateArgsPtr(TemplateId->getTemplateArgs(),
                                          TemplateId->NumArgs);
 
       Ty = Actions.ActOnTypenameType(getCurScope(), TypenameLoc, SS,
@@ -1404,13 +1561,24 @@ bool Parser::TryAnnotateTypeOrScopeToken(bool EnteringContext, bool NeedType) {
   }
 
   // Remembers whether the token was originally a scope annotation.
-  bool wasScopeAnnotation = Tok.is(tok::annot_cxxscope);
+  bool WasScopeAnnotation = Tok.is(tok::annot_cxxscope);
 
   CXXScopeSpec SS;
   if (getLangOpts().CPlusPlus)
     if (ParseOptionalCXXScopeSpecifier(SS, ParsedType(), EnteringContext))
       return true;
 
+  return TryAnnotateTypeOrScopeTokenAfterScopeSpec(EnteringContext, NeedType,
+                                                   SS, !WasScopeAnnotation);
+}
+
+/// \brief Try to annotate a type or scope token, having already parsed an
+/// optional scope specifier. \p IsNewScope should be \c true unless the scope
+/// specifier was extracted from an existing tok::annot_cxxscope annotation.
+bool Parser::TryAnnotateTypeOrScopeTokenAfterScopeSpec(bool EnteringContext,
+                                                       bool NeedType,
+                                                       CXXScopeSpec &SS,
+                                                       bool IsNewScope) {
   if (Tok.is(tok::identifier)) {
     IdentifierInfo *CorrectedII = 0;
     // Determine whether the identifier is a type name.
@@ -1492,21 +1660,7 @@ bool Parser::TryAnnotateTypeOrScopeToken(bool EnteringContext, bool NeedType) {
     return false;
 
   // A C++ scope specifier that isn't followed by a typename.
-  // Push the current token back into the token stream (or revert it if it is
-  // cached) and use an annotation scope token for current token.
-  if (PP.isBacktrackEnabled())
-    PP.RevertCachedTokens(1);
-  else
-    PP.EnterToken(Tok);
-  Tok.setKind(tok::annot_cxxscope);
-  Tok.setAnnotationValue(Actions.SaveNestedNameSpecifierAnnotation(SS));
-  Tok.setAnnotationRange(SS.getRange());
-
-  // In case the tokens were cached, have Preprocessor replace them
-  // with the annotation token.  We don't need to do this if we've
-  // just reverted back to the state we were in before being called.
-  if (!wasScopeAnnotation)
-    PP.AnnotateCachedTokens(Tok);
+  AnnotateScopeToken(SS, IsNewScope);
   return false;
 }
 
@@ -1529,19 +1683,7 @@ bool Parser::TryAnnotateCXXScopeToken(bool EnteringContext) {
   if (SS.isEmpty())
     return false;
 
-  // Push the current token back into the token stream (or revert it if it is
-  // cached) and use an annotation scope token for current token.
-  if (PP.isBacktrackEnabled())
-    PP.RevertCachedTokens(1);
-  else
-    PP.EnterToken(Tok);
-  Tok.setKind(tok::annot_cxxscope);
-  Tok.setAnnotationValue(Actions.SaveNestedNameSpecifierAnnotation(SS));
-  Tok.setAnnotationRange(SS.getRange());
-
-  // In case the tokens were cached, have Preprocessor replace them with the
-  // annotation token.
-  PP.AnnotateCachedTokens(Tok);
+  AnnotateScopeToken(SS, true);
   return false;
 }
 
@@ -1798,8 +1940,8 @@ bool BalancedDelimiterTracker::diagnoseMissingClose() {
   }
   P.Diag(P.Tok, DID);
   P.Diag(LOpen, diag::note_matching) << LHSName;
-  if (P.SkipUntil(Close))
-    LClose = P.Tok.getLocation();
+  if (P.SkipUntil(Close, /*StopAtSemi*/ true, /*DontConsume*/ true))
+    LClose = P.ConsumeAnyToken();
   return true;
 }
 
diff --git a/lib/Parse/RAIIObjectsForParser.h b/lib/Parse/RAIIObjectsForParser.h
index 455c4af..060fd20 100644
--- a/lib/Parse/RAIIObjectsForParser.h
+++ b/lib/Parse/RAIIObjectsForParser.h
@@ -87,9 +87,8 @@ namespace clang {
     Sema::ParsingDeclState State;
     bool Popped;
 
-    // Do not implement.
-    ParsingDeclRAIIObject(const ParsingDeclRAIIObject &other);
-    ParsingDeclRAIIObject &operator=(const ParsingDeclRAIIObject &other);
+    ParsingDeclRAIIObject(const ParsingDeclRAIIObject &) LLVM_DELETED_FUNCTION;
+    void operator=(const ParsingDeclRAIIObject &) LLVM_DELETED_FUNCTION;
 
   public:
     enum NoParent_t { NoParent };
@@ -245,8 +244,9 @@ namespace clang {
   /// the way they used to be.  This is used to handle __extension__ in the
   /// parser.
   class ExtensionRAIIObject {
-    void operator=(const ExtensionRAIIObject &);     // DO NOT IMPLEMENT
-    ExtensionRAIIObject(const ExtensionRAIIObject&); // DO NOT IMPLEMENT
+    ExtensionRAIIObject(const ExtensionRAIIObject &) LLVM_DELETED_FUNCTION;
+    void operator=(const ExtensionRAIIObject &) LLVM_DELETED_FUNCTION;
+
     DiagnosticsEngine &Diags;
   public:
     ExtensionRAIIObject(DiagnosticsEngine &diags) : Diags(diags) {
diff --git a/lib/Rewrite/CMakeLists.txt b/lib/Rewrite/CMakeLists.txt
index af8f6d4..d3d7543 100644
--- a/lib/Rewrite/CMakeLists.txt
+++ b/lib/Rewrite/CMakeLists.txt
@@ -1,33 +1,2 @@
-add_clang_library(clangRewrite
-  DeltaTree.cpp
-  FixItRewriter.cpp
-  FrontendActions.cpp
-  HTMLPrint.cpp
-  HTMLRewrite.cpp
-  InclusionRewriter.cpp
-  RewriteMacros.cpp
-  RewriteModernObjC.cpp
-  RewriteObjC.cpp
-  RewriteRope.cpp
-  RewriteTest.cpp
-  Rewriter.cpp
-  TokenRewriter.cpp
-  )
-
-add_dependencies(clangRewrite
-  ClangAttrClasses
-  ClangAttrList
-  ClangAttrParsedAttrList
-  ClangCommentNodes
-  ClangDeclNodes
-  ClangDiagnosticCommon
-  ClangDiagnosticFrontend
-  ClangStmtNodes
-  )
-
-target_link_libraries(clangRewrite
-  clangBasic
-  clangAST
-  clangParse
-  clangFrontend
-  )
+add_subdirectory(Core)
+add_subdirectory(Frontend)
diff --git a/lib/Rewrite/Core/CMakeLists.txt b/lib/Rewrite/Core/CMakeLists.txt
new file mode 100644
index 0000000..0797818
--- /dev/null
+++ b/lib/Rewrite/Core/CMakeLists.txt
@@ -0,0 +1,24 @@
+add_clang_library(clangRewriteCore
+  DeltaTree.cpp
+  HTMLRewrite.cpp
+  RewriteRope.cpp
+  Rewriter.cpp
+  TokenRewriter.cpp
+  )
+
+add_dependencies(clangRewriteCore
+  ClangAttrClasses
+  ClangAttrList
+  ClangAttrParsedAttrList
+  ClangCommentNodes
+  ClangDeclNodes
+  ClangDiagnosticCommon
+  ClangDiagnosticFrontend
+  ClangStmtNodes
+  )
+
+target_link_libraries(clangRewriteCore
+  clangBasic
+  clangAST
+  clangParse
+  )
diff --git a/lib/Rewrite/Core/DeltaTree.cpp b/lib/Rewrite/Core/DeltaTree.cpp
new file mode 100644
index 0000000..4692277
--- /dev/null
+++ b/lib/Rewrite/Core/DeltaTree.cpp
@@ -0,0 +1,464 @@
+//===--- DeltaTree.cpp - B-Tree for Rewrite Delta tracking ----------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the DeltaTree and related classes.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Rewrite/Core/DeltaTree.h"
+#include "clang/Basic/LLVM.h"
+#include <cstring>
+#include <cstdio>
+using namespace clang;
+
+/// The DeltaTree class is a multiway search tree (BTree) structure with some
+/// fancy features.  B-Trees are generally more memory and cache efficient
+/// than binary trees, because they store multiple keys/values in each node.
+///
+/// DeltaTree implements a key/value mapping from FileIndex to Delta, allowing
+/// fast lookup by FileIndex.  However, an added (important) bonus is that it
+/// can also efficiently tell us the full accumulated delta for a specific
+/// file offset as well, without traversing the whole tree.
+///
+/// The nodes of the tree are made up of instances of two classes:
+/// DeltaTreeNode and DeltaTreeInteriorNode.  The later subclasses the
+/// former and adds children pointers.  Each node knows the full delta of all
+/// entries (recursively) contained inside of it, which allows us to get the
+/// full delta implied by a whole subtree in constant time.
+
+namespace {
+  /// SourceDelta - As code in the original input buffer is added and deleted,
+  /// SourceDelta records are used to keep track of how the input SourceLocation
+  /// object is mapped into the output buffer.
+  struct SourceDelta {
+    unsigned FileLoc;
+    int Delta;
+
+    static SourceDelta get(unsigned Loc, int D) {
+      SourceDelta Delta;
+      Delta.FileLoc = Loc;
+      Delta.Delta = D;
+      return Delta;
+    }
+  };
+  
+  /// DeltaTreeNode - The common part of all nodes.
+  ///
+  class DeltaTreeNode {
+  public:
+    struct InsertResult {
+      DeltaTreeNode *LHS, *RHS;
+      SourceDelta Split;
+    };
+    
+  private:
+    friend class DeltaTreeInteriorNode;
+
+    /// WidthFactor - This controls the number of K/V slots held in the BTree:
+    /// how wide it is.  Each level of the BTree is guaranteed to have at least
+    /// WidthFactor-1 K/V pairs (except the root) and may have at most
+    /// 2*WidthFactor-1 K/V pairs.
+    enum { WidthFactor = 8 };
+
+    /// Values - This tracks the SourceDelta's currently in this node.
+    ///
+    SourceDelta Values[2*WidthFactor-1];
+
+    /// NumValuesUsed - This tracks the number of values this node currently
+    /// holds.
+    unsigned char NumValuesUsed;
+
+    /// IsLeaf - This is true if this is a leaf of the btree.  If false, this is
+    /// an interior node, and is actually an instance of DeltaTreeInteriorNode.
+    bool IsLeaf;
+
+    /// FullDelta - This is the full delta of all the values in this node and
+    /// all children nodes.
+    int FullDelta;
+  public:
+    DeltaTreeNode(bool isLeaf = true)
+      : NumValuesUsed(0), IsLeaf(isLeaf), FullDelta(0) {}
+
+    bool isLeaf() const { return IsLeaf; }
+    int getFullDelta() const { return FullDelta; }
+    bool isFull() const { return NumValuesUsed == 2*WidthFactor-1; }
+
+    unsigned getNumValuesUsed() const { return NumValuesUsed; }
+    const SourceDelta &getValue(unsigned i) const {
+      assert(i < NumValuesUsed && "Invalid value #");
+      return Values[i];
+    }
+    SourceDelta &getValue(unsigned i) {
+      assert(i < NumValuesUsed && "Invalid value #");
+      return Values[i];
+    }
+
+    /// DoInsertion - Do an insertion of the specified FileIndex/Delta pair into
+    /// this node.  If insertion is easy, do it and return false.  Otherwise,
+    /// split the node, populate InsertRes with info about the split, and return
+    /// true.
+    bool DoInsertion(unsigned FileIndex, int Delta, InsertResult *InsertRes);
+
+    void DoSplit(InsertResult &InsertRes);
+
+
+    /// RecomputeFullDeltaLocally - Recompute the FullDelta field by doing a
+    /// local walk over our contained deltas.
+    void RecomputeFullDeltaLocally();
+
+    void Destroy();
+  };
+} // end anonymous namespace
+
+namespace {
+  /// DeltaTreeInteriorNode - When isLeaf = false, a node has child pointers.
+  /// This class tracks them.
+  class DeltaTreeInteriorNode : public DeltaTreeNode {
+    DeltaTreeNode *Children[2*WidthFactor];
+    ~DeltaTreeInteriorNode() {
+      for (unsigned i = 0, e = NumValuesUsed+1; i != e; ++i)
+        Children[i]->Destroy();
+    }
+    friend class DeltaTreeNode;
+  public:
+    DeltaTreeInteriorNode() : DeltaTreeNode(false /*nonleaf*/) {}
+
+    DeltaTreeInteriorNode(const InsertResult &IR)
+      : DeltaTreeNode(false /*nonleaf*/) {
+      Children[0] = IR.LHS;
+      Children[1] = IR.RHS;
+      Values[0] = IR.Split;
+      FullDelta = IR.LHS->getFullDelta()+IR.RHS->getFullDelta()+IR.Split.Delta;
+      NumValuesUsed = 1;
+    }
+
+    const DeltaTreeNode *getChild(unsigned i) const {
+      assert(i < getNumValuesUsed()+1 && "Invalid child");
+      return Children[i];
+    }
+    DeltaTreeNode *getChild(unsigned i) {
+      assert(i < getNumValuesUsed()+1 && "Invalid child");
+      return Children[i];
+    }
+
+    static inline bool classof(const DeltaTreeNode *N) { return !N->isLeaf(); }
+  };
+}
+
+
+/// Destroy - A 'virtual' destructor.
+void DeltaTreeNode::Destroy() {
+  if (isLeaf())
+    delete this;
+  else
+    delete cast<DeltaTreeInteriorNode>(this);
+}
+
+/// RecomputeFullDeltaLocally - Recompute the FullDelta field by doing a
+/// local walk over our contained deltas.
+void DeltaTreeNode::RecomputeFullDeltaLocally() {
+  int NewFullDelta = 0;
+  for (unsigned i = 0, e = getNumValuesUsed(); i != e; ++i)
+    NewFullDelta += Values[i].Delta;
+  if (DeltaTreeInteriorNode *IN = dyn_cast<DeltaTreeInteriorNode>(this))
+    for (unsigned i = 0, e = getNumValuesUsed()+1; i != e; ++i)
+      NewFullDelta += IN->getChild(i)->getFullDelta();
+  FullDelta = NewFullDelta;
+}
+
+/// DoInsertion - Do an insertion of the specified FileIndex/Delta pair into
+/// this node.  If insertion is easy, do it and return false.  Otherwise,
+/// split the node, populate InsertRes with info about the split, and return
+/// true.
+bool DeltaTreeNode::DoInsertion(unsigned FileIndex, int Delta,
+                                InsertResult *InsertRes) {
+  // Maintain full delta for this node.
+  FullDelta += Delta;
+
+  // Find the insertion point, the first delta whose index is >= FileIndex.
+  unsigned i = 0, e = getNumValuesUsed();
+  while (i != e && FileIndex > getValue(i).FileLoc)
+    ++i;
+
+  // If we found an a record for exactly this file index, just merge this
+  // value into the pre-existing record and finish early.
+  if (i != e && getValue(i).FileLoc == FileIndex) {
+    // NOTE: Delta could drop to zero here.  This means that the delta entry is
+    // useless and could be removed.  Supporting erases is more complex than
+    // leaving an entry with Delta=0, so we just leave an entry with Delta=0 in
+    // the tree.
+    Values[i].Delta += Delta;
+    return false;
+  }
+
+  // Otherwise, we found an insertion point, and we know that the value at the
+  // specified index is > FileIndex.  Handle the leaf case first.
+  if (isLeaf()) {
+    if (!isFull()) {
+      // For an insertion into a non-full leaf node, just insert the value in
+      // its sorted position.  This requires moving later values over.
+      if (i != e)
+        memmove(&Values[i+1], &Values[i], sizeof(Values[0])*(e-i));
+      Values[i] = SourceDelta::get(FileIndex, Delta);
+      ++NumValuesUsed;
+      return false;
+    }
+
+    // Otherwise, if this is leaf is full, split the node at its median, insert
+    // the value into one of the children, and return the result.
+    assert(InsertRes && "No result location specified");
+    DoSplit(*InsertRes);
+
+    if (InsertRes->Split.FileLoc > FileIndex)
+      InsertRes->LHS->DoInsertion(FileIndex, Delta, 0 /*can't fail*/);
+    else
+      InsertRes->RHS->DoInsertion(FileIndex, Delta, 0 /*can't fail*/);
+    return true;
+  }
+
+  // Otherwise, this is an interior node.  Send the request down the tree.
+  DeltaTreeInteriorNode *IN = cast<DeltaTreeInteriorNode>(this);
+  if (!IN->Children[i]->DoInsertion(FileIndex, Delta, InsertRes))
+    return false; // If there was space in the child, just return.
+
+  // Okay, this split the subtree, producing a new value and two children to
+  // insert here.  If this node is non-full, we can just insert it directly.
+  if (!isFull()) {
+    // Now that we have two nodes and a new element, insert the perclated value
+    // into ourself by moving all the later values/children down, then inserting
+    // the new one.
+    if (i != e)
+      memmove(&IN->Children[i+2], &IN->Children[i+1],
+              (e-i)*sizeof(IN->Children[0]));
+    IN->Children[i] = InsertRes->LHS;
+    IN->Children[i+1] = InsertRes->RHS;
+
+    if (e != i)
+      memmove(&Values[i+1], &Values[i], (e-i)*sizeof(Values[0]));
+    Values[i] = InsertRes->Split;
+    ++NumValuesUsed;
+    return false;
+  }
+
+  // Finally, if this interior node was full and a node is percolated up, split
+  // ourself and return that up the chain.  Start by saving all our info to
+  // avoid having the split clobber it.
+  IN->Children[i] = InsertRes->LHS;
+  DeltaTreeNode *SubRHS = InsertRes->RHS;
+  SourceDelta SubSplit = InsertRes->Split;
+
+  // Do the split.
+  DoSplit(*InsertRes);
+
+  // Figure out where to insert SubRHS/NewSplit.
+  DeltaTreeInteriorNode *InsertSide;
+  if (SubSplit.FileLoc < InsertRes->Split.FileLoc)
+    InsertSide = cast<DeltaTreeInteriorNode>(InsertRes->LHS);
+  else
+    InsertSide = cast<DeltaTreeInteriorNode>(InsertRes->RHS);
+
+  // We now have a non-empty interior node 'InsertSide' to insert
+  // SubRHS/SubSplit into.  Find out where to insert SubSplit.
+
+  // Find the insertion point, the first delta whose index is >SubSplit.FileLoc.
+  i = 0; e = InsertSide->getNumValuesUsed();
+  while (i != e && SubSplit.FileLoc > InsertSide->getValue(i).FileLoc)
+    ++i;
+
+  // Now we know that i is the place to insert the split value into.  Insert it
+  // and the child right after it.
+  if (i != e)
+    memmove(&InsertSide->Children[i+2], &InsertSide->Children[i+1],
+            (e-i)*sizeof(IN->Children[0]));
+  InsertSide->Children[i+1] = SubRHS;
+
+  if (e != i)
+    memmove(&InsertSide->Values[i+1], &InsertSide->Values[i],
+            (e-i)*sizeof(Values[0]));
+  InsertSide->Values[i] = SubSplit;
+  ++InsertSide->NumValuesUsed;
+  InsertSide->FullDelta += SubSplit.Delta + SubRHS->getFullDelta();
+  return true;
+}
+
+/// DoSplit - Split the currently full node (which has 2*WidthFactor-1 values)
+/// into two subtrees each with "WidthFactor-1" values and a pivot value.
+/// Return the pieces in InsertRes.
+void DeltaTreeNode::DoSplit(InsertResult &InsertRes) {
+  assert(isFull() && "Why split a non-full node?");
+
+  // Since this node is full, it contains 2*WidthFactor-1 values.  We move
+  // the first 'WidthFactor-1' values to the LHS child (which we leave in this
+  // node), propagate one value up, and move the last 'WidthFactor-1' values
+  // into the RHS child.
+
+  // Create the new child node.
+  DeltaTreeNode *NewNode;
+  if (DeltaTreeInteriorNode *IN = dyn_cast<DeltaTreeInteriorNode>(this)) {
+    // If this is an interior node, also move over 'WidthFactor' children
+    // into the new node.
+    DeltaTreeInteriorNode *New = new DeltaTreeInteriorNode();
+    memcpy(&New->Children[0], &IN->Children[WidthFactor],
+           WidthFactor*sizeof(IN->Children[0]));
+    NewNode = New;
+  } else {
+    // Just create the new leaf node.
+    NewNode = new DeltaTreeNode();
+  }
+
+  // Move over the last 'WidthFactor-1' values from here to NewNode.
+  memcpy(&NewNode->Values[0], &Values[WidthFactor],
+         (WidthFactor-1)*sizeof(Values[0]));
+
+  // Decrease the number of values in the two nodes.
+  NewNode->NumValuesUsed = NumValuesUsed = WidthFactor-1;
+
+  // Recompute the two nodes' full delta.
+  NewNode->RecomputeFullDeltaLocally();
+  RecomputeFullDeltaLocally();
+
+  InsertRes.LHS = this;
+  InsertRes.RHS = NewNode;
+  InsertRes.Split = Values[WidthFactor-1];
+}
+
+
+
+//===----------------------------------------------------------------------===//
+//                        DeltaTree Implementation
+//===----------------------------------------------------------------------===//
+
+//#define VERIFY_TREE
+
+#ifdef VERIFY_TREE
+/// VerifyTree - Walk the btree performing assertions on various properties to
+/// verify consistency.  This is useful for debugging new changes to the tree.
+static void VerifyTree(const DeltaTreeNode *N) {
+  const DeltaTreeInteriorNode *IN = dyn_cast<DeltaTreeInteriorNode>(N);
+  if (IN == 0) {
+    // Verify leaves, just ensure that FullDelta matches up and the elements
+    // are in proper order.
+    int FullDelta = 0;
+    for (unsigned i = 0, e = N->getNumValuesUsed(); i != e; ++i) {
+      if (i)
+        assert(N->getValue(i-1).FileLoc < N->getValue(i).FileLoc);
+      FullDelta += N->getValue(i).Delta;
+    }
+    assert(FullDelta == N->getFullDelta());
+    return;
+  }
+
+  // Verify interior nodes: Ensure that FullDelta matches up and the
+  // elements are in proper order and the children are in proper order.
+  int FullDelta = 0;
+  for (unsigned i = 0, e = IN->getNumValuesUsed(); i != e; ++i) {
+    const SourceDelta &IVal = N->getValue(i);
+    const DeltaTreeNode *IChild = IN->getChild(i);
+    if (i)
+      assert(IN->getValue(i-1).FileLoc < IVal.FileLoc);
+    FullDelta += IVal.Delta;
+    FullDelta += IChild->getFullDelta();
+
+    // The largest value in child #i should be smaller than FileLoc.
+    assert(IChild->getValue(IChild->getNumValuesUsed()-1).FileLoc <
+           IVal.FileLoc);
+
+    // The smallest value in child #i+1 should be larger than FileLoc.
+    assert(IN->getChild(i+1)->getValue(0).FileLoc > IVal.FileLoc);
+    VerifyTree(IChild);
+  }
+
+  FullDelta += IN->getChild(IN->getNumValuesUsed())->getFullDelta();
+
+  assert(FullDelta == N->getFullDelta());
+}
+#endif  // VERIFY_TREE
+
+static DeltaTreeNode *getRoot(void *Root) {
+  return (DeltaTreeNode*)Root;
+}
+
+DeltaTree::DeltaTree() {
+  Root = new DeltaTreeNode();
+}
+DeltaTree::DeltaTree(const DeltaTree &RHS) {
+  // Currently we only support copying when the RHS is empty.
+  assert(getRoot(RHS.Root)->getNumValuesUsed() == 0 &&
+         "Can only copy empty tree");
+  Root = new DeltaTreeNode();
+}
+
+DeltaTree::~DeltaTree() {
+  getRoot(Root)->Destroy();
+}
+
+/// getDeltaAt - Return the accumulated delta at the specified file offset.
+/// This includes all insertions or delections that occurred *before* the
+/// specified file index.
+int DeltaTree::getDeltaAt(unsigned FileIndex) const {
+  const DeltaTreeNode *Node = getRoot(Root);
+
+  int Result = 0;
+
+  // Walk down the tree.
+  while (1) {
+    // For all nodes, include any local deltas before the specified file
+    // index by summing them up directly.  Keep track of how many were
+    // included.
+    unsigned NumValsGreater = 0;
+    for (unsigned e = Node->getNumValuesUsed(); NumValsGreater != e;
+         ++NumValsGreater) {
+      const SourceDelta &Val = Node->getValue(NumValsGreater);
+
+      if (Val.FileLoc >= FileIndex)
+        break;
+      Result += Val.Delta;
+    }
+
+    // If we have an interior node, include information about children and
+    // recurse.  Otherwise, if we have a leaf, we're done.
+    const DeltaTreeInteriorNode *IN = dyn_cast<DeltaTreeInteriorNode>(Node);
+    if (!IN) return Result;
+
+    // Include any children to the left of the values we skipped, all of
+    // their deltas should be included as well.
+    for (unsigned i = 0; i != NumValsGreater; ++i)
+      Result += IN->getChild(i)->getFullDelta();
+
+    // If we found exactly the value we were looking for, break off the
+    // search early.  There is no need to search the RHS of the value for
+    // partial results.
+    if (NumValsGreater != Node->getNumValuesUsed() &&
+        Node->getValue(NumValsGreater).FileLoc == FileIndex)
+      return Result+IN->getChild(NumValsGreater)->getFullDelta();
+
+    // Otherwise, traverse down the tree.  The selected subtree may be
+    // partially included in the range.
+    Node = IN->getChild(NumValsGreater);
+  }
+  // NOT REACHED.
+}
+
+/// AddDelta - When a change is made that shifts around the text buffer,
+/// this method is used to record that info.  It inserts a delta of 'Delta'
+/// into the current DeltaTree at offset FileIndex.
+void DeltaTree::AddDelta(unsigned FileIndex, int Delta) {
+  assert(Delta && "Adding a noop?");
+  DeltaTreeNode *MyRoot = getRoot(Root);
+
+  DeltaTreeNode::InsertResult InsertRes;
+  if (MyRoot->DoInsertion(FileIndex, Delta, &InsertRes)) {
+    Root = MyRoot = new DeltaTreeInteriorNode(InsertRes);
+  }
+
+#ifdef VERIFY_TREE
+  VerifyTree(MyRoot);
+#endif
+}
+
diff --git a/lib/Rewrite/Core/HTMLRewrite.cpp b/lib/Rewrite/Core/HTMLRewrite.cpp
new file mode 100644
index 0000000..0e8e4fe
--- /dev/null
+++ b/lib/Rewrite/Core/HTMLRewrite.cpp
@@ -0,0 +1,584 @@
+//== HTMLRewrite.cpp - Translate source code into prettified HTML --*- C++ -*-//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines the HTMLRewriter clas, which is used to translate the
+//  text of a source file into prettified HTML.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Rewrite/Core/Rewriter.h"
+#include "clang/Rewrite/Core/HTMLRewrite.h"
+#include "clang/Lex/TokenConcatenation.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Basic/SourceManager.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/OwningPtr.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/raw_ostream.h"
+using namespace clang;
+
+
+/// HighlightRange - Highlight a range in the source code with the specified
+/// start/end tags.  B/E must be in the same file.  This ensures that
+/// start/end tags are placed at the start/end of each line if the range is
+/// multiline.
+void html::HighlightRange(Rewriter &R, SourceLocation B, SourceLocation E,
+                          const char *StartTag, const char *EndTag) {
+  SourceManager &SM = R.getSourceMgr();
+  B = SM.getExpansionLoc(B);
+  E = SM.getExpansionLoc(E);
+  FileID FID = SM.getFileID(B);
+  assert(SM.getFileID(E) == FID && "B/E not in the same file!");
+
+  unsigned BOffset = SM.getFileOffset(B);
+  unsigned EOffset = SM.getFileOffset(E);
+
+  // Include the whole end token in the range.
+  EOffset += Lexer::MeasureTokenLength(E, R.getSourceMgr(), R.getLangOpts());
+
+  bool Invalid = false;
+  const char *BufferStart = SM.getBufferData(FID, &Invalid).data();
+  if (Invalid)
+    return;
+  
+  HighlightRange(R.getEditBuffer(FID), BOffset, EOffset,
+                 BufferStart, StartTag, EndTag);
+}
+
+/// HighlightRange - This is the same as the above method, but takes
+/// decomposed file locations.
+void html::HighlightRange(RewriteBuffer &RB, unsigned B, unsigned E,
+                          const char *BufferStart,
+                          const char *StartTag, const char *EndTag) {
+  // Insert the tag at the absolute start/end of the range.
+  RB.InsertTextAfter(B, StartTag);
+  RB.InsertTextBefore(E, EndTag);
+
+  // Scan the range to see if there is a \r or \n.  If so, and if the line is
+  // not blank, insert tags on that line as well.
+  bool HadOpenTag = true;
+
+  unsigned LastNonWhiteSpace = B;
+  for (unsigned i = B; i != E; ++i) {
+    switch (BufferStart[i]) {
+    case '\r':
+    case '\n':
+      // Okay, we found a newline in the range.  If we have an open tag, we need
+      // to insert a close tag at the first non-whitespace before the newline.
+      if (HadOpenTag)
+        RB.InsertTextBefore(LastNonWhiteSpace+1, EndTag);
+
+      // Instead of inserting an open tag immediately after the newline, we
+      // wait until we see a non-whitespace character.  This prevents us from
+      // inserting tags around blank lines, and also allows the open tag to
+      // be put *after* whitespace on a non-blank line.
+      HadOpenTag = false;
+      break;
+    case '\0':
+    case ' ':
+    case '\t':
+    case '\f':
+    case '\v':
+      // Ignore whitespace.
+      break;
+
+    default:
+      // If there is no tag open, do it now.
+      if (!HadOpenTag) {
+        RB.InsertTextAfter(i, StartTag);
+        HadOpenTag = true;
+      }
+
+      // Remember this character.
+      LastNonWhiteSpace = i;
+      break;
+    }
+  }
+}
+
+void html::EscapeText(Rewriter &R, FileID FID,
+                      bool EscapeSpaces, bool ReplaceTabs) {
+
+  const llvm::MemoryBuffer *Buf = R.getSourceMgr().getBuffer(FID);
+  const char* C = Buf->getBufferStart();
+  const char* FileEnd = Buf->getBufferEnd();
+
+  assert (C <= FileEnd);
+
+  RewriteBuffer &RB = R.getEditBuffer(FID);
+
+  unsigned ColNo = 0;
+  for (unsigned FilePos = 0; C != FileEnd ; ++C, ++FilePos) {
+    switch (*C) {
+    default: ++ColNo; break;
+    case '\n':
+    case '\r':
+      ColNo = 0;
+      break;
+
+    case ' ':
+      if (EscapeSpaces)
+        RB.ReplaceText(FilePos, 1, "&nbsp;");
+      ++ColNo;
+      break;
+    case '\f':
+      RB.ReplaceText(FilePos, 1, "<hr>");
+      ColNo = 0;
+      break;
+
+    case '\t': {
+      if (!ReplaceTabs)
+        break;
+      unsigned NumSpaces = 8-(ColNo&7);
+      if (EscapeSpaces)
+        RB.ReplaceText(FilePos, 1,
+                       StringRef("&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;"
+                                       "&nbsp;&nbsp;&nbsp;", 6*NumSpaces));
+      else
+        RB.ReplaceText(FilePos, 1, StringRef("        ", NumSpaces));
+      ColNo += NumSpaces;
+      break;
+    }
+    case '<':
+      RB.ReplaceText(FilePos, 1, "&lt;");
+      ++ColNo;
+      break;
+
+    case '>':
+      RB.ReplaceText(FilePos, 1, "&gt;");
+      ++ColNo;
+      break;
+
+    case '&':
+      RB.ReplaceText(FilePos, 1, "&amp;");
+      ++ColNo;
+      break;
+    }
+  }
+}
+
+std::string html::EscapeText(const std::string& s, bool EscapeSpaces,
+                             bool ReplaceTabs) {
+
+  unsigned len = s.size();
+  std::string Str;
+  llvm::raw_string_ostream os(Str);
+
+  for (unsigned i = 0 ; i < len; ++i) {
+
+    char c = s[i];
+    switch (c) {
+    default:
+      os << c; break;
+
+    case ' ':
+      if (EscapeSpaces) os << "&nbsp;";
+      else os << ' ';
+      break;
+
+    case '\t':
+      if (ReplaceTabs) {
+        if (EscapeSpaces)
+          for (unsigned i = 0; i < 4; ++i)
+            os << "&nbsp;";
+        else
+          for (unsigned i = 0; i < 4; ++i)
+            os << " ";
+      }
+      else
+        os << c;
+
+      break;
+
+    case '<': os << "&lt;"; break;
+    case '>': os << "&gt;"; break;
+    case '&': os << "&amp;"; break;
+    }
+  }
+
+  return os.str();
+}
+
+static void AddLineNumber(RewriteBuffer &RB, unsigned LineNo,
+                          unsigned B, unsigned E) {
+  SmallString<256> Str;
+  llvm::raw_svector_ostream OS(Str);
+
+  OS << "<tr><td class=\"num\" id=\"LN"
+     << LineNo << "\">"
+     << LineNo << "</td><td class=\"line\">";
+
+  if (B == E) { // Handle empty lines.
+    OS << " </td></tr>";
+    RB.InsertTextBefore(B, OS.str());
+  } else {
+    RB.InsertTextBefore(B, OS.str());
+    RB.InsertTextBefore(E, "</td></tr>");
+  }
+}
+
+void html::AddLineNumbers(Rewriter& R, FileID FID) {
+
+  const llvm::MemoryBuffer *Buf = R.getSourceMgr().getBuffer(FID);
+  const char* FileBeg = Buf->getBufferStart();
+  const char* FileEnd = Buf->getBufferEnd();
+  const char* C = FileBeg;
+  RewriteBuffer &RB = R.getEditBuffer(FID);
+
+  assert (C <= FileEnd);
+
+  unsigned LineNo = 0;
+  unsigned FilePos = 0;
+
+  while (C != FileEnd) {
+
+    ++LineNo;
+    unsigned LineStartPos = FilePos;
+    unsigned LineEndPos = FileEnd - FileBeg;
+
+    assert (FilePos <= LineEndPos);
+    assert (C < FileEnd);
+
+    // Scan until the newline (or end-of-file).
+
+    while (C != FileEnd) {
+      char c = *C;
+      ++C;
+
+      if (c == '\n') {
+        LineEndPos = FilePos++;
+        break;
+      }
+
+      ++FilePos;
+    }
+
+    AddLineNumber(RB, LineNo, LineStartPos, LineEndPos);
+  }
+
+  // Add one big table tag that surrounds all of the code.
+  RB.InsertTextBefore(0, "<table class=\"code\">\n");
+  RB.InsertTextAfter(FileEnd - FileBeg, "</table>");
+}
+
+void html::AddHeaderFooterInternalBuiltinCSS(Rewriter& R, FileID FID,
+                                             const char *title) {
+
+  const llvm::MemoryBuffer *Buf = R.getSourceMgr().getBuffer(FID);
+  const char* FileStart = Buf->getBufferStart();
+  const char* FileEnd = Buf->getBufferEnd();
+
+  SourceLocation StartLoc = R.getSourceMgr().getLocForStartOfFile(FID);
+  SourceLocation EndLoc = StartLoc.getLocWithOffset(FileEnd-FileStart);
+
+  std::string s;
+  llvm::raw_string_ostream os(s);
+  os << "<!doctype html>\n" // Use HTML 5 doctype
+        "<html>\n<head>\n";
+
+  if (title)
+    os << "<title>" << html::EscapeText(title) << "</title>\n";
+
+  os << "<style type=\"text/css\">\n"
+      " body { color:#000000; background-color:#ffffff }\n"
+      " body { font-family:Helvetica, sans-serif; font-size:10pt }\n"
+      " h1 { font-size:14pt }\n"
+      " .code { border-collapse:collapse; width:100%; }\n"
+      " .code { font-family: \"Monospace\", monospace; font-size:10pt }\n"
+      " .code { line-height: 1.2em }\n"
+      " .comment { color: green; font-style: oblique }\n"
+      " .keyword { color: blue }\n"
+      " .string_literal { color: red }\n"
+      " .directive { color: darkmagenta }\n"
+      // Macro expansions.
+      " .expansion { display: none; }\n"
+      " .macro:hover .expansion { display: block; border: 2px solid #FF0000; "
+          "padding: 2px; background-color:#FFF0F0; font-weight: normal; "
+          "  -webkit-border-radius:5px;  -webkit-box-shadow:1px 1px 7px #000; "
+          "position: absolute; top: -1em; left:10em; z-index: 1 } \n"
+      " .macro { color: darkmagenta; background-color:LemonChiffon;"
+             // Macros are position: relative to provide base for expansions.
+             " position: relative }\n"
+      " .num { width:2.5em; padding-right:2ex; background-color:#eeeeee }\n"
+      " .num { text-align:right; font-size:8pt }\n"
+      " .num { color:#444444 }\n"
+      " .line { padding-left: 1ex; border-left: 3px solid #ccc }\n"
+      " .line { white-space: pre }\n"
+      " .msg { -webkit-box-shadow:1px 1px 7px #000 }\n"
+      " .msg { -webkit-border-radius:5px }\n"
+      " .msg { font-family:Helvetica, sans-serif; font-size:8pt }\n"
+      " .msg { float:left }\n"
+      " .msg { padding:0.25em 1ex 0.25em 1ex }\n"
+      " .msg { margin-top:10px; margin-bottom:10px }\n"
+      " .msg { font-weight:bold }\n"
+      " .msg { max-width:60em; word-wrap: break-word; white-space: pre-wrap }\n"
+      " .msgT { padding:0x; spacing:0x }\n"
+      " .msgEvent { background-color:#fff8b4; color:#000000 }\n"
+      " .msgControl { background-color:#bbbbbb; color:#000000 }\n"
+      " .mrange { background-color:#dfddf3 }\n"
+      " .mrange { border-bottom:1px solid #6F9DBE }\n"
+      " .PathIndex { font-weight: bold; padding:0px 5px; "
+        "margin-right:5px; }\n"
+      " .PathIndex { -webkit-border-radius:8px }\n"
+      " .PathIndexEvent { background-color:#bfba87 }\n"
+      " .PathIndexControl { background-color:#8c8c8c }\n"
+      " .PathNav a { text-decoration:none; font-size: larger }\n"
+      " .CodeInsertionHint { font-weight: bold; background-color: #10dd10 }\n"
+      " .CodeRemovalHint { background-color:#de1010 }\n"
+      " .CodeRemovalHint { border-bottom:1px solid #6F9DBE }\n"
+      " table.simpletable {\n"
+      "   padding: 5px;\n"
+      "   font-size:12pt;\n"
+      "   margin:20px;\n"
+      "   border-collapse: collapse; border-spacing: 0px;\n"
+      " }\n"
+      " td.rowname {\n"
+      "   text-align:right; font-weight:bold; color:#444444;\n"
+      "   padding-right:2ex; }\n"
+      "</style>\n</head>\n<body>";
+
+  // Generate header
+  R.InsertTextBefore(StartLoc, os.str());
+  // Generate footer
+
+  R.InsertTextAfter(EndLoc, "</body></html>\n");
+}
+
+/// SyntaxHighlight - Relex the specified FileID and annotate the HTML with
+/// information about keywords, macro expansions etc.  This uses the macro
+/// table state from the end of the file, so it won't be perfectly perfect,
+/// but it will be reasonably close.
+void html::SyntaxHighlight(Rewriter &R, FileID FID, const Preprocessor &PP) {
+  RewriteBuffer &RB = R.getEditBuffer(FID);
+
+  const SourceManager &SM = PP.getSourceManager();
+  const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID);
+  Lexer L(FID, FromFile, SM, PP.getLangOpts());
+  const char *BufferStart = L.getBufferStart();
+
+  // Inform the preprocessor that we want to retain comments as tokens, so we
+  // can highlight them.
+  L.SetCommentRetentionState(true);
+
+  // Lex all the tokens in raw mode, to avoid entering #includes or expanding
+  // macros.
+  Token Tok;
+  L.LexFromRawLexer(Tok);
+
+  while (Tok.isNot(tok::eof)) {
+    // Since we are lexing unexpanded tokens, all tokens are from the main
+    // FileID.
+    unsigned TokOffs = SM.getFileOffset(Tok.getLocation());
+    unsigned TokLen = Tok.getLength();
+    switch (Tok.getKind()) {
+    default: break;
+    case tok::identifier:
+      llvm_unreachable("tok::identifier in raw lexing mode!");
+    case tok::raw_identifier: {
+      // Fill in Result.IdentifierInfo and update the token kind,
+      // looking up the identifier in the identifier table.
+      PP.LookUpIdentifierInfo(Tok);
+
+      // If this is a pp-identifier, for a keyword, highlight it as such.
+      if (Tok.isNot(tok::identifier))
+        HighlightRange(RB, TokOffs, TokOffs+TokLen, BufferStart,
+                       "<span class='keyword'>", "</span>");
+      break;
+    }
+    case tok::comment:
+      HighlightRange(RB, TokOffs, TokOffs+TokLen, BufferStart,
+                     "<span class='comment'>", "</span>");
+      break;
+    case tok::utf8_string_literal:
+      // Chop off the u part of u8 prefix
+      ++TokOffs;
+      --TokLen;
+      // FALL THROUGH to chop the 8
+    case tok::wide_string_literal:
+    case tok::utf16_string_literal:
+    case tok::utf32_string_literal:
+      // Chop off the L, u, U or 8 prefix
+      ++TokOffs;
+      --TokLen;
+      // FALL THROUGH.
+    case tok::string_literal:
+      // FIXME: Exclude the optional ud-suffix from the highlighted range.
+      HighlightRange(RB, TokOffs, TokOffs+TokLen, BufferStart,
+                     "<span class='string_literal'>", "</span>");
+      break;
+    case tok::hash: {
+      // If this is a preprocessor directive, all tokens to end of line are too.
+      if (!Tok.isAtStartOfLine())
+        break;
+
+      // Eat all of the tokens until we get to the next one at the start of
+      // line.
+      unsigned TokEnd = TokOffs+TokLen;
+      L.LexFromRawLexer(Tok);
+      while (!Tok.isAtStartOfLine() && Tok.isNot(tok::eof)) {
+        TokEnd = SM.getFileOffset(Tok.getLocation())+Tok.getLength();
+        L.LexFromRawLexer(Tok);
+      }
+
+      // Find end of line.  This is a hack.
+      HighlightRange(RB, TokOffs, TokEnd, BufferStart,
+                     "<span class='directive'>", "</span>");
+
+      // Don't skip the next token.
+      continue;
+    }
+    }
+
+    L.LexFromRawLexer(Tok);
+  }
+}
+
+/// HighlightMacros - This uses the macro table state from the end of the
+/// file, to re-expand macros and insert (into the HTML) information about the
+/// macro expansions.  This won't be perfectly perfect, but it will be
+/// reasonably close.
+void html::HighlightMacros(Rewriter &R, FileID FID, const Preprocessor& PP) {
+  // Re-lex the raw token stream into a token buffer.
+  const SourceManager &SM = PP.getSourceManager();
+  std::vector<Token> TokenStream;
+
+  const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID);
+  Lexer L(FID, FromFile, SM, PP.getLangOpts());
+
+  // Lex all the tokens in raw mode, to avoid entering #includes or expanding
+  // macros.
+  while (1) {
+    Token Tok;
+    L.LexFromRawLexer(Tok);
+
+    // If this is a # at the start of a line, discard it from the token stream.
+    // We don't want the re-preprocess step to see #defines, #includes or other
+    // preprocessor directives.
+    if (Tok.is(tok::hash) && Tok.isAtStartOfLine())
+      continue;
+
+    // If this is a ## token, change its kind to unknown so that repreprocessing
+    // it will not produce an error.
+    if (Tok.is(tok::hashhash))
+      Tok.setKind(tok::unknown);
+
+    // If this raw token is an identifier, the raw lexer won't have looked up
+    // the corresponding identifier info for it.  Do this now so that it will be
+    // macro expanded when we re-preprocess it.
+    if (Tok.is(tok::raw_identifier))
+      PP.LookUpIdentifierInfo(Tok);
+
+    TokenStream.push_back(Tok);
+
+    if (Tok.is(tok::eof)) break;
+  }
+
+  // Temporarily change the diagnostics object so that we ignore any generated
+  // diagnostics from this pass.
+  DiagnosticsEngine TmpDiags(PP.getDiagnostics().getDiagnosticIDs(),
+                             &PP.getDiagnostics().getDiagnosticOptions(),
+                      new IgnoringDiagConsumer);
+
+  // FIXME: This is a huge hack; we reuse the input preprocessor because we want
+  // its state, but we aren't actually changing it (we hope). This should really
+  // construct a copy of the preprocessor.
+  Preprocessor &TmpPP = const_cast<Preprocessor&>(PP);
+  DiagnosticsEngine *OldDiags = &TmpPP.getDiagnostics();
+  TmpPP.setDiagnostics(TmpDiags);
+
+  // Inform the preprocessor that we don't want comments.
+  TmpPP.SetCommentRetentionState(false, false);
+
+  // We don't want pragmas either. Although we filtered out #pragma, removing
+  // _Pragma and __pragma is much harder.
+  bool PragmasPreviouslyEnabled = TmpPP.getPragmasEnabled();
+  TmpPP.setPragmasEnabled(false);
+
+  // Enter the tokens we just lexed.  This will cause them to be macro expanded
+  // but won't enter sub-files (because we removed #'s).
+  TmpPP.EnterTokenStream(&TokenStream[0], TokenStream.size(), false, false);
+
+  TokenConcatenation ConcatInfo(TmpPP);
+
+  // Lex all the tokens.
+  Token Tok;
+  TmpPP.Lex(Tok);
+  while (Tok.isNot(tok::eof)) {
+    // Ignore non-macro tokens.
+    if (!Tok.getLocation().isMacroID()) {
+      TmpPP.Lex(Tok);
+      continue;
+    }
+
+    // Okay, we have the first token of a macro expansion: highlight the
+    // expansion by inserting a start tag before the macro expansion and
+    // end tag after it.
+    std::pair<SourceLocation, SourceLocation> LLoc =
+      SM.getExpansionRange(Tok.getLocation());
+
+    // Ignore tokens whose instantiation location was not the main file.
+    if (SM.getFileID(LLoc.first) != FID) {
+      TmpPP.Lex(Tok);
+      continue;
+    }
+
+    assert(SM.getFileID(LLoc.second) == FID &&
+           "Start and end of expansion must be in the same ultimate file!");
+
+    std::string Expansion = EscapeText(TmpPP.getSpelling(Tok));
+    unsigned LineLen = Expansion.size();
+
+    Token PrevPrevTok;
+    Token PrevTok = Tok;
+    // Okay, eat this token, getting the next one.
+    TmpPP.Lex(Tok);
+
+    // Skip all the rest of the tokens that are part of this macro
+    // instantiation.  It would be really nice to pop up a window with all the
+    // spelling of the tokens or something.
+    while (!Tok.is(tok::eof) &&
+           SM.getExpansionLoc(Tok.getLocation()) == LLoc.first) {
+      // Insert a newline if the macro expansion is getting large.
+      if (LineLen > 60) {
+        Expansion += "<br>";
+        LineLen = 0;
+      }
+
+      LineLen -= Expansion.size();
+
+      // If the tokens were already space separated, or if they must be to avoid
+      // them being implicitly pasted, add a space between them.
+      if (Tok.hasLeadingSpace() ||
+          ConcatInfo.AvoidConcat(PrevPrevTok, PrevTok, Tok))
+        Expansion += ' ';
+
+      // Escape any special characters in the token text.
+      Expansion += EscapeText(TmpPP.getSpelling(Tok));
+      LineLen += Expansion.size();
+
+      PrevPrevTok = PrevTok;
+      PrevTok = Tok;
+      TmpPP.Lex(Tok);
+    }
+
+
+    // Insert the expansion as the end tag, so that multi-line macros all get
+    // highlighted.
+    Expansion = "<span class='expansion'>" + Expansion + "</span></span>";
+
+    HighlightRange(R, LLoc.first, LLoc.second,
+                   "<span class='macro'>", Expansion.c_str());
+  }
+
+  // Restore the preprocessor's old state.
+  TmpPP.setDiagnostics(*OldDiags);
+  TmpPP.setPragmasEnabled(PragmasPreviouslyEnabled);
+}
diff --git a/lib/Rewrite/Core/Makefile b/lib/Rewrite/Core/Makefile
new file mode 100644
index 0000000..8c8d2e4
--- /dev/null
+++ b/lib/Rewrite/Core/Makefile
@@ -0,0 +1,18 @@
+##===- clang/lib/Rewrite/Makefile --------------------------*- Makefile -*-===##
+# 
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+# 
+##===----------------------------------------------------------------------===##
+#
+# This implements code transformation / rewriting facilities.
+#
+##===----------------------------------------------------------------------===##
+
+CLANG_LEVEL := ../../..
+LIBRARYNAME := clangRewriteCore
+
+include $(CLANG_LEVEL)/Makefile
+
diff --git a/lib/Rewrite/Core/RewriteRope.cpp b/lib/Rewrite/Core/RewriteRope.cpp
new file mode 100644
index 0000000..fe7aa2d
--- /dev/null
+++ b/lib/Rewrite/Core/RewriteRope.cpp
@@ -0,0 +1,807 @@
+//===--- RewriteRope.cpp - Rope specialized for rewriter --------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file implements the RewriteRope class, which is a powerful string.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Rewrite/Core/RewriteRope.h"
+#include "clang/Basic/LLVM.h"
+#include <algorithm>
+using namespace clang;
+
+/// RewriteRope is a "strong" string class, designed to make insertions and
+/// deletions in the middle of the string nearly constant time (really, they are
+/// O(log N), but with a very low constant factor).
+///
+/// The implementation of this datastructure is a conceptual linear sequence of
+/// RopePiece elements.  Each RopePiece represents a view on a separately
+/// allocated and reference counted string.  This means that splitting a very
+/// long string can be done in constant time by splitting a RopePiece that
+/// references the whole string into two rope pieces that reference each half.
+/// Once split, another string can be inserted in between the two halves by
+/// inserting a RopePiece in between the two others.  All of this is very
+/// inexpensive: it takes time proportional to the number of RopePieces, not the
+/// length of the strings they represent.
+///
+/// While a linear sequences of RopePieces is the conceptual model, the actual
+/// implementation captures them in an adapted B+ Tree.  Using a B+ tree (which
+/// is a tree that keeps the values in the leaves and has where each node
+/// contains a reasonable number of pointers to children/values) allows us to
+/// maintain efficient operation when the RewriteRope contains a *huge* number
+/// of RopePieces.  The basic idea of the B+ Tree is that it allows us to find
+/// the RopePiece corresponding to some offset very efficiently, and it
+/// automatically balances itself on insertions of RopePieces (which can happen
+/// for both insertions and erases of string ranges).
+///
+/// The one wrinkle on the theory is that we don't attempt to keep the tree
+/// properly balanced when erases happen.  Erases of string data can both insert
+/// new RopePieces (e.g. when the middle of some other rope piece is deleted,
+/// which results in two rope pieces, which is just like an insert) or it can
+/// reduce the number of RopePieces maintained by the B+Tree.  In the case when
+/// the number of RopePieces is reduced, we don't attempt to maintain the
+/// standard 'invariant' that each node in the tree contains at least
+/// 'WidthFactor' children/values.  For our use cases, this doesn't seem to
+/// matter.
+///
+/// The implementation below is primarily implemented in terms of three classes:
+///   RopePieceBTreeNode - Common base class for:
+///
+///     RopePieceBTreeLeaf - Directly manages up to '2*WidthFactor' RopePiece
+///          nodes.  This directly represents a chunk of the string with those
+///          RopePieces contatenated.
+///     RopePieceBTreeInterior - An interior node in the B+ Tree, which manages
+///          up to '2*WidthFactor' other nodes in the tree.
+
+
+//===----------------------------------------------------------------------===//
+// RopePieceBTreeNode Class
+//===----------------------------------------------------------------------===//
+
+namespace {
+  /// RopePieceBTreeNode - Common base class of RopePieceBTreeLeaf and
+  /// RopePieceBTreeInterior.  This provides some 'virtual' dispatching methods
+  /// and a flag that determines which subclass the instance is.  Also
+  /// important, this node knows the full extend of the node, including any
+  /// children that it has.  This allows efficient skipping over entire subtrees
+  /// when looking for an offset in the BTree.
+  class RopePieceBTreeNode {
+  protected:
+    /// WidthFactor - This controls the number of K/V slots held in the BTree:
+    /// how wide it is.  Each level of the BTree is guaranteed to have at least
+    /// 'WidthFactor' elements in it (either ropepieces or children), (except
+    /// the root, which may have less) and may have at most 2*WidthFactor
+    /// elements.
+    enum { WidthFactor = 8 };
+
+    /// Size - This is the number of bytes of file this node (including any
+    /// potential children) covers.
+    unsigned Size;
+
+    /// IsLeaf - True if this is an instance of RopePieceBTreeLeaf, false if it
+    /// is an instance of RopePieceBTreeInterior.
+    bool IsLeaf;
+
+    RopePieceBTreeNode(bool isLeaf) : Size(0), IsLeaf(isLeaf) {}
+    ~RopePieceBTreeNode() {}
+  public:
+
+    bool isLeaf() const { return IsLeaf; }
+    unsigned size() const { return Size; }
+
+    void Destroy();
+
+    /// split - Split the range containing the specified offset so that we are
+    /// guaranteed that there is a place to do an insertion at the specified
+    /// offset.  The offset is relative, so "0" is the start of the node.
+    ///
+    /// If there is no space in this subtree for the extra piece, the extra tree
+    /// node is returned and must be inserted into a parent.
+    RopePieceBTreeNode *split(unsigned Offset);
+
+    /// insert - Insert the specified ropepiece into this tree node at the
+    /// specified offset.  The offset is relative, so "0" is the start of the
+    /// node.
+    ///
+    /// If there is no space in this subtree for the extra piece, the extra tree
+    /// node is returned and must be inserted into a parent.
+    RopePieceBTreeNode *insert(unsigned Offset, const RopePiece &R);
+
+    /// erase - Remove NumBytes from this node at the specified offset.  We are
+    /// guaranteed that there is a split at Offset.
+    void erase(unsigned Offset, unsigned NumBytes);
+
+  };
+} // end anonymous namespace
+
+//===----------------------------------------------------------------------===//
+// RopePieceBTreeLeaf Class
+//===----------------------------------------------------------------------===//
+
+namespace {
+  /// RopePieceBTreeLeaf - Directly manages up to '2*WidthFactor' RopePiece
+  /// nodes.  This directly represents a chunk of the string with those
+  /// RopePieces contatenated.  Since this is a B+Tree, all values (in this case
+  /// instances of RopePiece) are stored in leaves like this.  To make iteration
+  /// over the leaves efficient, they maintain a singly linked list through the
+  /// NextLeaf field.  This allows the B+Tree forward iterator to be constant
+  /// time for all increments.
+  class RopePieceBTreeLeaf : public RopePieceBTreeNode {
+    /// NumPieces - This holds the number of rope pieces currently active in the
+    /// Pieces array.
+    unsigned char NumPieces;
+
+    /// Pieces - This tracks the file chunks currently in this leaf.
+    ///
+    RopePiece Pieces[2*WidthFactor];
+
+    /// NextLeaf - This is a pointer to the next leaf in the tree, allowing
+    /// efficient in-order forward iteration of the tree without traversal.
+    RopePieceBTreeLeaf **PrevLeaf, *NextLeaf;
+  public:
+    RopePieceBTreeLeaf() : RopePieceBTreeNode(true), NumPieces(0),
+                           PrevLeaf(0), NextLeaf(0) {}
+    ~RopePieceBTreeLeaf() {
+      if (PrevLeaf || NextLeaf)
+        removeFromLeafInOrder();
+      clear();
+    }
+
+    bool isFull() const { return NumPieces == 2*WidthFactor; }
+
+    /// clear - Remove all rope pieces from this leaf.
+    void clear() {
+      while (NumPieces)
+        Pieces[--NumPieces] = RopePiece();
+      Size = 0;
+    }
+
+    unsigned getNumPieces() const { return NumPieces; }
+
+    const RopePiece &getPiece(unsigned i) const {
+      assert(i < getNumPieces() && "Invalid piece ID");
+      return Pieces[i];
+    }
+
+    const RopePieceBTreeLeaf *getNextLeafInOrder() const { return NextLeaf; }
+    void insertAfterLeafInOrder(RopePieceBTreeLeaf *Node) {
+      assert(PrevLeaf == 0 && NextLeaf == 0 && "Already in ordering");
+
+      NextLeaf = Node->NextLeaf;
+      if (NextLeaf)
+        NextLeaf->PrevLeaf = &NextLeaf;
+      PrevLeaf = &Node->NextLeaf;
+      Node->NextLeaf = this;
+    }
+
+    void removeFromLeafInOrder() {
+      if (PrevLeaf) {
+        *PrevLeaf = NextLeaf;
+        if (NextLeaf)
+          NextLeaf->PrevLeaf = PrevLeaf;
+      } else if (NextLeaf) {
+        NextLeaf->PrevLeaf = 0;
+      }
+    }
+
+    /// FullRecomputeSizeLocally - This method recomputes the 'Size' field by
+    /// summing the size of all RopePieces.
+    void FullRecomputeSizeLocally() {
+      Size = 0;
+      for (unsigned i = 0, e = getNumPieces(); i != e; ++i)
+        Size += getPiece(i).size();
+    }
+
+    /// split - Split the range containing the specified offset so that we are
+    /// guaranteed that there is a place to do an insertion at the specified
+    /// offset.  The offset is relative, so "0" is the start of the node.
+    ///
+    /// If there is no space in this subtree for the extra piece, the extra tree
+    /// node is returned and must be inserted into a parent.
+    RopePieceBTreeNode *split(unsigned Offset);
+
+    /// insert - Insert the specified ropepiece into this tree node at the
+    /// specified offset.  The offset is relative, so "0" is the start of the
+    /// node.
+    ///
+    /// If there is no space in this subtree for the extra piece, the extra tree
+    /// node is returned and must be inserted into a parent.
+    RopePieceBTreeNode *insert(unsigned Offset, const RopePiece &R);
+
+
+    /// erase - Remove NumBytes from this node at the specified offset.  We are
+    /// guaranteed that there is a split at Offset.
+    void erase(unsigned Offset, unsigned NumBytes);
+
+    static inline bool classof(const RopePieceBTreeNode *N) {
+      return N->isLeaf();
+    }
+  };
+} // end anonymous namespace
+
+/// split - Split the range containing the specified offset so that we are
+/// guaranteed that there is a place to do an insertion at the specified
+/// offset.  The offset is relative, so "0" is the start of the node.
+///
+/// If there is no space in this subtree for the extra piece, the extra tree
+/// node is returned and must be inserted into a parent.
+RopePieceBTreeNode *RopePieceBTreeLeaf::split(unsigned Offset) {
+  // Find the insertion point.  We are guaranteed that there is a split at the
+  // specified offset so find it.
+  if (Offset == 0 || Offset == size()) {
+    // Fastpath for a common case.  There is already a splitpoint at the end.
+    return 0;
+  }
+
+  // Find the piece that this offset lands in.
+  unsigned PieceOffs = 0;
+  unsigned i = 0;
+  while (Offset >= PieceOffs+Pieces[i].size()) {
+    PieceOffs += Pieces[i].size();
+    ++i;
+  }
+
+  // If there is already a split point at the specified offset, just return
+  // success.
+  if (PieceOffs == Offset)
+    return 0;
+
+  // Otherwise, we need to split piece 'i' at Offset-PieceOffs.  Convert Offset
+  // to being Piece relative.
+  unsigned IntraPieceOffset = Offset-PieceOffs;
+
+  // We do this by shrinking the RopePiece and then doing an insert of the tail.
+  RopePiece Tail(Pieces[i].StrData, Pieces[i].StartOffs+IntraPieceOffset,
+                 Pieces[i].EndOffs);
+  Size -= Pieces[i].size();
+  Pieces[i].EndOffs = Pieces[i].StartOffs+IntraPieceOffset;
+  Size += Pieces[i].size();
+
+  return insert(Offset, Tail);
+}
+
+
+/// insert - Insert the specified RopePiece into this tree node at the
+/// specified offset.  The offset is relative, so "0" is the start of the node.
+///
+/// If there is no space in this subtree for the extra piece, the extra tree
+/// node is returned and must be inserted into a parent.
+RopePieceBTreeNode *RopePieceBTreeLeaf::insert(unsigned Offset,
+                                               const RopePiece &R) {
+  // If this node is not full, insert the piece.
+  if (!isFull()) {
+    // Find the insertion point.  We are guaranteed that there is a split at the
+    // specified offset so find it.
+    unsigned i = 0, e = getNumPieces();
+    if (Offset == size()) {
+      // Fastpath for a common case.
+      i = e;
+    } else {
+      unsigned SlotOffs = 0;
+      for (; Offset > SlotOffs; ++i)
+        SlotOffs += getPiece(i).size();
+      assert(SlotOffs == Offset && "Split didn't occur before insertion!");
+    }
+
+    // For an insertion into a non-full leaf node, just insert the value in
+    // its sorted position.  This requires moving later values over.
+    for (; i != e; --e)
+      Pieces[e] = Pieces[e-1];
+    Pieces[i] = R;
+    ++NumPieces;
+    Size += R.size();
+    return 0;
+  }
+
+  // Otherwise, if this is leaf is full, split it in two halves.  Since this
+  // node is full, it contains 2*WidthFactor values.  We move the first
+  // 'WidthFactor' values to the LHS child (which we leave in this node) and
+  // move the last 'WidthFactor' values into the RHS child.
+
+  // Create the new node.
+  RopePieceBTreeLeaf *NewNode = new RopePieceBTreeLeaf();
+
+  // Move over the last 'WidthFactor' values from here to NewNode.
+  std::copy(&Pieces[WidthFactor], &Pieces[2*WidthFactor],
+            &NewNode->Pieces[0]);
+  // Replace old pieces with null RopePieces to drop refcounts.
+  std::fill(&Pieces[WidthFactor], &Pieces[2*WidthFactor], RopePiece());
+
+  // Decrease the number of values in the two nodes.
+  NewNode->NumPieces = NumPieces = WidthFactor;
+
+  // Recompute the two nodes' size.
+  NewNode->FullRecomputeSizeLocally();
+  FullRecomputeSizeLocally();
+
+  // Update the list of leaves.
+  NewNode->insertAfterLeafInOrder(this);
+
+  // These insertions can't fail.
+  if (this->size() >= Offset)
+    this->insert(Offset, R);
+  else
+    NewNode->insert(Offset - this->size(), R);
+  return NewNode;
+}
+
+/// erase - Remove NumBytes from this node at the specified offset.  We are
+/// guaranteed that there is a split at Offset.
+void RopePieceBTreeLeaf::erase(unsigned Offset, unsigned NumBytes) {
+  // Since we are guaranteed that there is a split at Offset, we start by
+  // finding the Piece that starts there.
+  unsigned PieceOffs = 0;
+  unsigned i = 0;
+  for (; Offset > PieceOffs; ++i)
+    PieceOffs += getPiece(i).size();
+  assert(PieceOffs == Offset && "Split didn't occur before erase!");
+
+  unsigned StartPiece = i;
+
+  // Figure out how many pieces completely cover 'NumBytes'.  We want to remove
+  // all of them.
+  for (; Offset+NumBytes > PieceOffs+getPiece(i).size(); ++i)
+    PieceOffs += getPiece(i).size();
+
+  // If we exactly include the last one, include it in the region to delete.
+  if (Offset+NumBytes == PieceOffs+getPiece(i).size())
+    PieceOffs += getPiece(i).size(), ++i;
+
+  // If we completely cover some RopePieces, erase them now.
+  if (i != StartPiece) {
+    unsigned NumDeleted = i-StartPiece;
+    for (; i != getNumPieces(); ++i)
+      Pieces[i-NumDeleted] = Pieces[i];
+
+    // Drop references to dead rope pieces.
+    std::fill(&Pieces[getNumPieces()-NumDeleted], &Pieces[getNumPieces()],
+              RopePiece());
+    NumPieces -= NumDeleted;
+
+    unsigned CoverBytes = PieceOffs-Offset;
+    NumBytes -= CoverBytes;
+    Size -= CoverBytes;
+  }
+
+  // If we completely removed some stuff, we could be done.
+  if (NumBytes == 0) return;
+
+  // Okay, now might be erasing part of some Piece.  If this is the case, then
+  // move the start point of the piece.
+  assert(getPiece(StartPiece).size() > NumBytes);
+  Pieces[StartPiece].StartOffs += NumBytes;
+
+  // The size of this node just shrunk by NumBytes.
+  Size -= NumBytes;
+}
+
+//===----------------------------------------------------------------------===//
+// RopePieceBTreeInterior Class
+//===----------------------------------------------------------------------===//
+
+namespace {
+  /// RopePieceBTreeInterior - This represents an interior node in the B+Tree,
+  /// which holds up to 2*WidthFactor pointers to child nodes.
+  class RopePieceBTreeInterior : public RopePieceBTreeNode {
+    /// NumChildren - This holds the number of children currently active in the
+    /// Children array.
+    unsigned char NumChildren;
+    RopePieceBTreeNode *Children[2*WidthFactor];
+  public:
+    RopePieceBTreeInterior() : RopePieceBTreeNode(false), NumChildren(0) {}
+
+    RopePieceBTreeInterior(RopePieceBTreeNode *LHS, RopePieceBTreeNode *RHS)
+    : RopePieceBTreeNode(false) {
+      Children[0] = LHS;
+      Children[1] = RHS;
+      NumChildren = 2;
+      Size = LHS->size() + RHS->size();
+    }
+
+    ~RopePieceBTreeInterior() {
+      for (unsigned i = 0, e = getNumChildren(); i != e; ++i)
+        Children[i]->Destroy();
+    }
+
+    bool isFull() const { return NumChildren == 2*WidthFactor; }
+
+    unsigned getNumChildren() const { return NumChildren; }
+    const RopePieceBTreeNode *getChild(unsigned i) const {
+      assert(i < NumChildren && "invalid child #");
+      return Children[i];
+    }
+    RopePieceBTreeNode *getChild(unsigned i) {
+      assert(i < NumChildren && "invalid child #");
+      return Children[i];
+    }
+
+    /// FullRecomputeSizeLocally - Recompute the Size field of this node by
+    /// summing up the sizes of the child nodes.
+    void FullRecomputeSizeLocally() {
+      Size = 0;
+      for (unsigned i = 0, e = getNumChildren(); i != e; ++i)
+        Size += getChild(i)->size();
+    }
+
+
+    /// split - Split the range containing the specified offset so that we are
+    /// guaranteed that there is a place to do an insertion at the specified
+    /// offset.  The offset is relative, so "0" is the start of the node.
+    ///
+    /// If there is no space in this subtree for the extra piece, the extra tree
+    /// node is returned and must be inserted into a parent.
+    RopePieceBTreeNode *split(unsigned Offset);
+
+
+    /// insert - Insert the specified ropepiece into this tree node at the
+    /// specified offset.  The offset is relative, so "0" is the start of the
+    /// node.
+    ///
+    /// If there is no space in this subtree for the extra piece, the extra tree
+    /// node is returned and must be inserted into a parent.
+    RopePieceBTreeNode *insert(unsigned Offset, const RopePiece &R);
+
+    /// HandleChildPiece - A child propagated an insertion result up to us.
+    /// Insert the new child, and/or propagate the result further up the tree.
+    RopePieceBTreeNode *HandleChildPiece(unsigned i, RopePieceBTreeNode *RHS);
+
+    /// erase - Remove NumBytes from this node at the specified offset.  We are
+    /// guaranteed that there is a split at Offset.
+    void erase(unsigned Offset, unsigned NumBytes);
+
+    static inline bool classof(const RopePieceBTreeNode *N) {
+      return !N->isLeaf();
+    }
+  };
+} // end anonymous namespace
+
+/// split - Split the range containing the specified offset so that we are
+/// guaranteed that there is a place to do an insertion at the specified
+/// offset.  The offset is relative, so "0" is the start of the node.
+///
+/// If there is no space in this subtree for the extra piece, the extra tree
+/// node is returned and must be inserted into a parent.
+RopePieceBTreeNode *RopePieceBTreeInterior::split(unsigned Offset) {
+  // Figure out which child to split.
+  if (Offset == 0 || Offset == size())
+    return 0;  // If we have an exact offset, we're already split.
+
+  unsigned ChildOffset = 0;
+  unsigned i = 0;
+  for (; Offset >= ChildOffset+getChild(i)->size(); ++i)
+    ChildOffset += getChild(i)->size();
+
+  // If already split there, we're done.
+  if (ChildOffset == Offset)
+    return 0;
+
+  // Otherwise, recursively split the child.
+  if (RopePieceBTreeNode *RHS = getChild(i)->split(Offset-ChildOffset))
+    return HandleChildPiece(i, RHS);
+  return 0;  // Done!
+}
+
+/// insert - Insert the specified ropepiece into this tree node at the
+/// specified offset.  The offset is relative, so "0" is the start of the
+/// node.
+///
+/// If there is no space in this subtree for the extra piece, the extra tree
+/// node is returned and must be inserted into a parent.
+RopePieceBTreeNode *RopePieceBTreeInterior::insert(unsigned Offset,
+                                                   const RopePiece &R) {
+  // Find the insertion point.  We are guaranteed that there is a split at the
+  // specified offset so find it.
+  unsigned i = 0, e = getNumChildren();
+
+  unsigned ChildOffs = 0;
+  if (Offset == size()) {
+    // Fastpath for a common case.  Insert at end of last child.
+    i = e-1;
+    ChildOffs = size()-getChild(i)->size();
+  } else {
+    for (; Offset > ChildOffs+getChild(i)->size(); ++i)
+      ChildOffs += getChild(i)->size();
+  }
+
+  Size += R.size();
+
+  // Insert at the end of this child.
+  if (RopePieceBTreeNode *RHS = getChild(i)->insert(Offset-ChildOffs, R))
+    return HandleChildPiece(i, RHS);
+
+  return 0;
+}
+
+/// HandleChildPiece - A child propagated an insertion result up to us.
+/// Insert the new child, and/or propagate the result further up the tree.
+RopePieceBTreeNode *
+RopePieceBTreeInterior::HandleChildPiece(unsigned i, RopePieceBTreeNode *RHS) {
+  // Otherwise the child propagated a subtree up to us as a new child.  See if
+  // we have space for it here.
+  if (!isFull()) {
+    // Insert RHS after child 'i'.
+    if (i + 1 != getNumChildren())
+      memmove(&Children[i+2], &Children[i+1],
+              (getNumChildren()-i-1)*sizeof(Children[0]));
+    Children[i+1] = RHS;
+    ++NumChildren;
+    return 0;
+  }
+
+  // Okay, this node is full.  Split it in half, moving WidthFactor children to
+  // a newly allocated interior node.
+
+  // Create the new node.
+  RopePieceBTreeInterior *NewNode = new RopePieceBTreeInterior();
+
+  // Move over the last 'WidthFactor' values from here to NewNode.
+  memcpy(&NewNode->Children[0], &Children[WidthFactor],
+         WidthFactor*sizeof(Children[0]));
+
+  // Decrease the number of values in the two nodes.
+  NewNode->NumChildren = NumChildren = WidthFactor;
+
+  // Finally, insert the two new children in the side the can (now) hold them.
+  // These insertions can't fail.
+  if (i < WidthFactor)
+    this->HandleChildPiece(i, RHS);
+  else
+    NewNode->HandleChildPiece(i-WidthFactor, RHS);
+
+  // Recompute the two nodes' size.
+  NewNode->FullRecomputeSizeLocally();
+  FullRecomputeSizeLocally();
+  return NewNode;
+}
+
+/// erase - Remove NumBytes from this node at the specified offset.  We are
+/// guaranteed that there is a split at Offset.
+void RopePieceBTreeInterior::erase(unsigned Offset, unsigned NumBytes) {
+  // This will shrink this node by NumBytes.
+  Size -= NumBytes;
+
+  // Find the first child that overlaps with Offset.
+  unsigned i = 0;
+  for (; Offset >= getChild(i)->size(); ++i)
+    Offset -= getChild(i)->size();
+
+  // Propagate the delete request into overlapping children, or completely
+  // delete the children as appropriate.
+  while (NumBytes) {
+    RopePieceBTreeNode *CurChild = getChild(i);
+
+    // If we are deleting something contained entirely in the child, pass on the
+    // request.
+    if (Offset+NumBytes < CurChild->size()) {
+      CurChild->erase(Offset, NumBytes);
+      return;
+    }
+
+    // If this deletion request starts somewhere in the middle of the child, it
+    // must be deleting to the end of the child.
+    if (Offset) {
+      unsigned BytesFromChild = CurChild->size()-Offset;
+      CurChild->erase(Offset, BytesFromChild);
+      NumBytes -= BytesFromChild;
+      // Start at the beginning of the next child.
+      Offset = 0;
+      ++i;
+      continue;
+    }
+
+    // If the deletion request completely covers the child, delete it and move
+    // the rest down.
+    NumBytes -= CurChild->size();
+    CurChild->Destroy();
+    --NumChildren;
+    if (i != getNumChildren())
+      memmove(&Children[i], &Children[i+1],
+              (getNumChildren()-i)*sizeof(Children[0]));
+  }
+}
+
+//===----------------------------------------------------------------------===//
+// RopePieceBTreeNode Implementation
+//===----------------------------------------------------------------------===//
+
+void RopePieceBTreeNode::Destroy() {
+  if (RopePieceBTreeLeaf *Leaf = dyn_cast<RopePieceBTreeLeaf>(this))
+    delete Leaf;
+  else
+    delete cast<RopePieceBTreeInterior>(this);
+}
+
+/// split - Split the range containing the specified offset so that we are
+/// guaranteed that there is a place to do an insertion at the specified
+/// offset.  The offset is relative, so "0" is the start of the node.
+///
+/// If there is no space in this subtree for the extra piece, the extra tree
+/// node is returned and must be inserted into a parent.
+RopePieceBTreeNode *RopePieceBTreeNode::split(unsigned Offset) {
+  assert(Offset <= size() && "Invalid offset to split!");
+  if (RopePieceBTreeLeaf *Leaf = dyn_cast<RopePieceBTreeLeaf>(this))
+    return Leaf->split(Offset);
+  return cast<RopePieceBTreeInterior>(this)->split(Offset);
+}
+
+/// insert - Insert the specified ropepiece into this tree node at the
+/// specified offset.  The offset is relative, so "0" is the start of the
+/// node.
+///
+/// If there is no space in this subtree for the extra piece, the extra tree
+/// node is returned and must be inserted into a parent.
+RopePieceBTreeNode *RopePieceBTreeNode::insert(unsigned Offset,
+                                               const RopePiece &R) {
+  assert(Offset <= size() && "Invalid offset to insert!");
+  if (RopePieceBTreeLeaf *Leaf = dyn_cast<RopePieceBTreeLeaf>(this))
+    return Leaf->insert(Offset, R);
+  return cast<RopePieceBTreeInterior>(this)->insert(Offset, R);
+}
+
+/// erase - Remove NumBytes from this node at the specified offset.  We are
+/// guaranteed that there is a split at Offset.
+void RopePieceBTreeNode::erase(unsigned Offset, unsigned NumBytes) {
+  assert(Offset+NumBytes <= size() && "Invalid offset to erase!");
+  if (RopePieceBTreeLeaf *Leaf = dyn_cast<RopePieceBTreeLeaf>(this))
+    return Leaf->erase(Offset, NumBytes);
+  return cast<RopePieceBTreeInterior>(this)->erase(Offset, NumBytes);
+}
+
+
+//===----------------------------------------------------------------------===//
+// RopePieceBTreeIterator Implementation
+//===----------------------------------------------------------------------===//
+
+static const RopePieceBTreeLeaf *getCN(const void *P) {
+  return static_cast<const RopePieceBTreeLeaf*>(P);
+}
+
+// begin iterator.
+RopePieceBTreeIterator::RopePieceBTreeIterator(const void *n) {
+  const RopePieceBTreeNode *N = static_cast<const RopePieceBTreeNode*>(n);
+
+  // Walk down the left side of the tree until we get to a leaf.
+  while (const RopePieceBTreeInterior *IN = dyn_cast<RopePieceBTreeInterior>(N))
+    N = IN->getChild(0);
+
+  // We must have at least one leaf.
+  CurNode = cast<RopePieceBTreeLeaf>(N);
+
+  // If we found a leaf that happens to be empty, skip over it until we get
+  // to something full.
+  while (CurNode && getCN(CurNode)->getNumPieces() == 0)
+    CurNode = getCN(CurNode)->getNextLeafInOrder();
+
+  if (CurNode != 0)
+    CurPiece = &getCN(CurNode)->getPiece(0);
+  else  // Empty tree, this is an end() iterator.
+    CurPiece = 0;
+  CurChar = 0;
+}
+
+void RopePieceBTreeIterator::MoveToNextPiece() {
+  if (CurPiece != &getCN(CurNode)->getPiece(getCN(CurNode)->getNumPieces()-1)) {
+    CurChar = 0;
+    ++CurPiece;
+    return;
+  }
+
+  // Find the next non-empty leaf node.
+  do
+    CurNode = getCN(CurNode)->getNextLeafInOrder();
+  while (CurNode && getCN(CurNode)->getNumPieces() == 0);
+
+  if (CurNode != 0)
+    CurPiece = &getCN(CurNode)->getPiece(0);
+  else // Hit end().
+    CurPiece = 0;
+  CurChar = 0;
+}
+
+//===----------------------------------------------------------------------===//
+// RopePieceBTree Implementation
+//===----------------------------------------------------------------------===//
+
+static RopePieceBTreeNode *getRoot(void *P) {
+  return static_cast<RopePieceBTreeNode*>(P);
+}
+
+RopePieceBTree::RopePieceBTree() {
+  Root = new RopePieceBTreeLeaf();
+}
+RopePieceBTree::RopePieceBTree(const RopePieceBTree &RHS) {
+  assert(RHS.empty() && "Can't copy non-empty tree yet");
+  Root = new RopePieceBTreeLeaf();
+}
+RopePieceBTree::~RopePieceBTree() {
+  getRoot(Root)->Destroy();
+}
+
+unsigned RopePieceBTree::size() const {
+  return getRoot(Root)->size();
+}
+
+void RopePieceBTree::clear() {
+  if (RopePieceBTreeLeaf *Leaf = dyn_cast<RopePieceBTreeLeaf>(getRoot(Root)))
+    Leaf->clear();
+  else {
+    getRoot(Root)->Destroy();
+    Root = new RopePieceBTreeLeaf();
+  }
+}
+
+void RopePieceBTree::insert(unsigned Offset, const RopePiece &R) {
+  // #1. Split at Offset.
+  if (RopePieceBTreeNode *RHS = getRoot(Root)->split(Offset))
+    Root = new RopePieceBTreeInterior(getRoot(Root), RHS);
+
+  // #2. Do the insertion.
+  if (RopePieceBTreeNode *RHS = getRoot(Root)->insert(Offset, R))
+    Root = new RopePieceBTreeInterior(getRoot(Root), RHS);
+}
+
+void RopePieceBTree::erase(unsigned Offset, unsigned NumBytes) {
+  // #1. Split at Offset.
+  if (RopePieceBTreeNode *RHS = getRoot(Root)->split(Offset))
+    Root = new RopePieceBTreeInterior(getRoot(Root), RHS);
+
+  // #2. Do the erasing.
+  getRoot(Root)->erase(Offset, NumBytes);
+}
+
+//===----------------------------------------------------------------------===//
+// RewriteRope Implementation
+//===----------------------------------------------------------------------===//
+
+/// MakeRopeString - This copies the specified byte range into some instance of
+/// RopeRefCountString, and return a RopePiece that represents it.  This uses
+/// the AllocBuffer object to aggregate requests for small strings into one
+/// allocation instead of doing tons of tiny allocations.
+RopePiece RewriteRope::MakeRopeString(const char *Start, const char *End) {
+  unsigned Len = End-Start;
+  assert(Len && "Zero length RopePiece is invalid!");
+
+  // If we have space for this string in the current alloc buffer, use it.
+  if (AllocOffs+Len <= AllocChunkSize) {
+    memcpy(AllocBuffer->Data+AllocOffs, Start, Len);
+    AllocOffs += Len;
+    return RopePiece(AllocBuffer, AllocOffs-Len, AllocOffs);
+  }
+
+  // If we don't have enough room because this specific allocation is huge,
+  // just allocate a new rope piece for it alone.
+  if (Len > AllocChunkSize) {
+    unsigned Size = End-Start+sizeof(RopeRefCountString)-1;
+    RopeRefCountString *Res =
+      reinterpret_cast<RopeRefCountString *>(new char[Size]);
+    Res->RefCount = 0;
+    memcpy(Res->Data, Start, End-Start);
+    return RopePiece(Res, 0, End-Start);
+  }
+
+  // Otherwise, this was a small request but we just don't have space for it
+  // Make a new chunk and share it with later allocations.
+
+  // If we had an old allocation, drop our reference to it.
+  if (AllocBuffer && --AllocBuffer->RefCount == 0)
+    delete [] (char*)AllocBuffer;
+
+  unsigned AllocSize = offsetof(RopeRefCountString, Data) + AllocChunkSize;
+  AllocBuffer = reinterpret_cast<RopeRefCountString *>(new char[AllocSize]);
+  AllocBuffer->RefCount = 0;
+  memcpy(AllocBuffer->Data, Start, Len);
+  AllocOffs = Len;
+
+  // Start out the new allocation with a refcount of 1, since we have an
+  // internal reference to it.
+  AllocBuffer->addRef();
+  return RopePiece(AllocBuffer, 0, Len);
+}
+
+
diff --git a/lib/Rewrite/Core/Rewriter.cpp b/lib/Rewrite/Core/Rewriter.cpp
new file mode 100644
index 0000000..4df967f
--- /dev/null
+++ b/lib/Rewrite/Core/Rewriter.cpp
@@ -0,0 +1,486 @@
+//===--- Rewriter.cpp - Code rewriting interface --------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines the Rewriter class, which is used for code
+//  transformations.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Rewrite/Core/Rewriter.h"
+#include "clang/AST/Stmt.h"
+#include "clang/AST/Decl.h"
+#include "clang/Basic/DiagnosticIDs.h"
+#include "clang/Basic/FileManager.h"
+#include "clang/Basic/SourceManager.h"
+#include "clang/Lex/Lexer.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/Support/FileSystem.h"
+using namespace clang;
+
+raw_ostream &RewriteBuffer::write(raw_ostream &os) const {
+  // FIXME: eliminate the copy by writing out each chunk at a time
+  os << std::string(begin(), end());
+  return os;
+}
+
+/// \brief Return true if this character is non-new-line whitespace:
+/// ' ', '\\t', '\\f', '\\v', '\\r'.
+static inline bool isWhitespace(unsigned char c) {
+  switch (c) {
+  case ' ':
+  case '\t':
+  case '\f':
+  case '\v':
+  case '\r':
+    return true;
+  default:
+    return false;
+  }
+}
+
+void RewriteBuffer::RemoveText(unsigned OrigOffset, unsigned Size,
+                               bool removeLineIfEmpty) {
+  // Nothing to remove, exit early.
+  if (Size == 0) return;
+
+  unsigned RealOffset = getMappedOffset(OrigOffset, true);
+  assert(RealOffset+Size < Buffer.size() && "Invalid location");
+
+  // Remove the dead characters.
+  Buffer.erase(RealOffset, Size);
+
+  // Add a delta so that future changes are offset correctly.
+  AddReplaceDelta(OrigOffset, -Size);
+
+  if (removeLineIfEmpty) {
+    // Find the line that the remove occurred and if it is completely empty
+    // remove the line as well.
+
+    iterator curLineStart = begin();
+    unsigned curLineStartOffs = 0;
+    iterator posI = begin();
+    for (unsigned i = 0; i != RealOffset; ++i) {
+      if (*posI == '\n') {
+        curLineStart = posI;
+        ++curLineStart;
+        curLineStartOffs = i + 1;
+      }
+      ++posI;
+    }
+  
+    unsigned lineSize = 0;
+    posI = curLineStart;
+    while (posI != end() && isWhitespace(*posI)) {
+      ++posI;
+      ++lineSize;
+    }
+    if (posI != end() && *posI == '\n') {
+      Buffer.erase(curLineStartOffs, lineSize + 1/* + '\n'*/);
+      AddReplaceDelta(curLineStartOffs, -(lineSize + 1/* + '\n'*/));
+    }
+  }
+}
+
+void RewriteBuffer::InsertText(unsigned OrigOffset, StringRef Str,
+                               bool InsertAfter) {
+
+  // Nothing to insert, exit early.
+  if (Str.empty()) return;
+
+  unsigned RealOffset = getMappedOffset(OrigOffset, InsertAfter);
+  Buffer.insert(RealOffset, Str.begin(), Str.end());
+
+  // Add a delta so that future changes are offset correctly.
+  AddInsertDelta(OrigOffset, Str.size());
+}
+
+/// ReplaceText - This method replaces a range of characters in the input
+/// buffer with a new string.  This is effectively a combined "remove+insert"
+/// operation.
+void RewriteBuffer::ReplaceText(unsigned OrigOffset, unsigned OrigLength,
+                                StringRef NewStr) {
+  unsigned RealOffset = getMappedOffset(OrigOffset, true);
+  Buffer.erase(RealOffset, OrigLength);
+  Buffer.insert(RealOffset, NewStr.begin(), NewStr.end());
+  if (OrigLength != NewStr.size())
+    AddReplaceDelta(OrigOffset, NewStr.size() - OrigLength);
+}
+
+
+//===----------------------------------------------------------------------===//
+// Rewriter class
+//===----------------------------------------------------------------------===//
+
+/// getRangeSize - Return the size in bytes of the specified range if they
+/// are in the same file.  If not, this returns -1.
+int Rewriter::getRangeSize(const CharSourceRange &Range,
+                           RewriteOptions opts) const {
+  if (!isRewritable(Range.getBegin()) ||
+      !isRewritable(Range.getEnd())) return -1;
+
+  FileID StartFileID, EndFileID;
+  unsigned StartOff, EndOff;
+
+  StartOff = getLocationOffsetAndFileID(Range.getBegin(), StartFileID);
+  EndOff   = getLocationOffsetAndFileID(Range.getEnd(), EndFileID);
+
+  if (StartFileID != EndFileID)
+    return -1;
+
+  // If edits have been made to this buffer, the delta between the range may
+  // have changed.
+  std::map<FileID, RewriteBuffer>::const_iterator I =
+    RewriteBuffers.find(StartFileID);
+  if (I != RewriteBuffers.end()) {
+    const RewriteBuffer &RB = I->second;
+    EndOff = RB.getMappedOffset(EndOff, opts.IncludeInsertsAtEndOfRange);
+    StartOff = RB.getMappedOffset(StartOff, !opts.IncludeInsertsAtBeginOfRange);
+  }
+
+
+  // Adjust the end offset to the end of the last token, instead of being the
+  // start of the last token if this is a token range.
+  if (Range.isTokenRange())
+    EndOff += Lexer::MeasureTokenLength(Range.getEnd(), *SourceMgr, *LangOpts);
+
+  return EndOff-StartOff;
+}
+
+int Rewriter::getRangeSize(SourceRange Range, RewriteOptions opts) const {
+  return getRangeSize(CharSourceRange::getTokenRange(Range), opts);
+}
+
+
+/// getRewrittenText - Return the rewritten form of the text in the specified
+/// range.  If the start or end of the range was unrewritable or if they are
+/// in different buffers, this returns an empty string.
+///
+/// Note that this method is not particularly efficient.
+///
+std::string Rewriter::getRewrittenText(SourceRange Range) const {
+  if (!isRewritable(Range.getBegin()) ||
+      !isRewritable(Range.getEnd()))
+    return "";
+
+  FileID StartFileID, EndFileID;
+  unsigned StartOff, EndOff;
+  StartOff = getLocationOffsetAndFileID(Range.getBegin(), StartFileID);
+  EndOff   = getLocationOffsetAndFileID(Range.getEnd(), EndFileID);
+
+  if (StartFileID != EndFileID)
+    return ""; // Start and end in different buffers.
+
+  // If edits have been made to this buffer, the delta between the range may
+  // have changed.
+  std::map<FileID, RewriteBuffer>::const_iterator I =
+    RewriteBuffers.find(StartFileID);
+  if (I == RewriteBuffers.end()) {
+    // If the buffer hasn't been rewritten, just return the text from the input.
+    const char *Ptr = SourceMgr->getCharacterData(Range.getBegin());
+
+    // Adjust the end offset to the end of the last token, instead of being the
+    // start of the last token.
+    EndOff += Lexer::MeasureTokenLength(Range.getEnd(), *SourceMgr, *LangOpts);
+    return std::string(Ptr, Ptr+EndOff-StartOff);
+  }
+
+  const RewriteBuffer &RB = I->second;
+  EndOff = RB.getMappedOffset(EndOff, true);
+  StartOff = RB.getMappedOffset(StartOff);
+
+  // Adjust the end offset to the end of the last token, instead of being the
+  // start of the last token.
+  EndOff += Lexer::MeasureTokenLength(Range.getEnd(), *SourceMgr, *LangOpts);
+
+  // Advance the iterators to the right spot, yay for linear time algorithms.
+  RewriteBuffer::iterator Start = RB.begin();
+  std::advance(Start, StartOff);
+  RewriteBuffer::iterator End = Start;
+  std::advance(End, EndOff-StartOff);
+
+  return std::string(Start, End);
+}
+
+unsigned Rewriter::getLocationOffsetAndFileID(SourceLocation Loc,
+                                              FileID &FID) const {
+  assert(Loc.isValid() && "Invalid location");
+  std::pair<FileID,unsigned> V = SourceMgr->getDecomposedLoc(Loc);
+  FID = V.first;
+  return V.second;
+}
+
+
+/// getEditBuffer - Get or create a RewriteBuffer for the specified FileID.
+///
+RewriteBuffer &Rewriter::getEditBuffer(FileID FID) {
+  std::map<FileID, RewriteBuffer>::iterator I =
+    RewriteBuffers.lower_bound(FID);
+  if (I != RewriteBuffers.end() && I->first == FID)
+    return I->second;
+  I = RewriteBuffers.insert(I, std::make_pair(FID, RewriteBuffer()));
+
+  StringRef MB = SourceMgr->getBufferData(FID);
+  I->second.Initialize(MB.begin(), MB.end());
+
+  return I->second;
+}
+
+/// InsertText - Insert the specified string at the specified location in the
+/// original buffer.
+bool Rewriter::InsertText(SourceLocation Loc, StringRef Str,
+                          bool InsertAfter, bool indentNewLines) {
+  if (!isRewritable(Loc)) return true;
+  FileID FID;
+  unsigned StartOffs = getLocationOffsetAndFileID(Loc, FID);
+
+  SmallString<128> indentedStr;
+  if (indentNewLines && Str.find('\n') != StringRef::npos) {
+    StringRef MB = SourceMgr->getBufferData(FID);
+
+    unsigned lineNo = SourceMgr->getLineNumber(FID, StartOffs) - 1;
+    const SrcMgr::ContentCache *
+        Content = SourceMgr->getSLocEntry(FID).getFile().getContentCache();
+    unsigned lineOffs = Content->SourceLineCache[lineNo];
+
+    // Find the whitespace at the start of the line.
+    StringRef indentSpace;
+    {
+      unsigned i = lineOffs;
+      while (isWhitespace(MB[i]))
+        ++i;
+      indentSpace = MB.substr(lineOffs, i-lineOffs);
+    }
+
+    SmallVector<StringRef, 4> lines;
+    Str.split(lines, "\n");
+
+    for (unsigned i = 0, e = lines.size(); i != e; ++i) {
+      indentedStr += lines[i];
+      if (i < e-1) {
+        indentedStr += '\n';
+        indentedStr += indentSpace;
+      }
+    }
+    Str = indentedStr.str();
+  }
+
+  getEditBuffer(FID).InsertText(StartOffs, Str, InsertAfter);
+  return false;
+}
+
+bool Rewriter::InsertTextAfterToken(SourceLocation Loc, StringRef Str) {
+  if (!isRewritable(Loc)) return true;
+  FileID FID;
+  unsigned StartOffs = getLocationOffsetAndFileID(Loc, FID);
+  RewriteOptions rangeOpts;
+  rangeOpts.IncludeInsertsAtBeginOfRange = false;
+  StartOffs += getRangeSize(SourceRange(Loc, Loc), rangeOpts);
+  getEditBuffer(FID).InsertText(StartOffs, Str, /*InsertAfter*/true);
+  return false;
+}
+
+/// RemoveText - Remove the specified text region.
+bool Rewriter::RemoveText(SourceLocation Start, unsigned Length,
+                          RewriteOptions opts) {
+  if (!isRewritable(Start)) return true;
+  FileID FID;
+  unsigned StartOffs = getLocationOffsetAndFileID(Start, FID);
+  getEditBuffer(FID).RemoveText(StartOffs, Length, opts.RemoveLineIfEmpty);
+  return false;
+}
+
+/// ReplaceText - This method replaces a range of characters in the input
+/// buffer with a new string.  This is effectively a combined "remove/insert"
+/// operation.
+bool Rewriter::ReplaceText(SourceLocation Start, unsigned OrigLength,
+                           StringRef NewStr) {
+  if (!isRewritable(Start)) return true;
+  FileID StartFileID;
+  unsigned StartOffs = getLocationOffsetAndFileID(Start, StartFileID);
+
+  getEditBuffer(StartFileID).ReplaceText(StartOffs, OrigLength, NewStr);
+  return false;
+}
+
+bool Rewriter::ReplaceText(SourceRange range, SourceRange replacementRange) {
+  if (!isRewritable(range.getBegin())) return true;
+  if (!isRewritable(range.getEnd())) return true;
+  if (replacementRange.isInvalid()) return true;
+  SourceLocation start = range.getBegin();
+  unsigned origLength = getRangeSize(range);
+  unsigned newLength = getRangeSize(replacementRange);
+  FileID FID;
+  unsigned newOffs = getLocationOffsetAndFileID(replacementRange.getBegin(),
+                                                FID);
+  StringRef MB = SourceMgr->getBufferData(FID);
+  return ReplaceText(start, origLength, MB.substr(newOffs, newLength));
+}
+
+/// ReplaceStmt - This replaces a Stmt/Expr with another, using the pretty
+/// printer to generate the replacement code.  This returns true if the input
+/// could not be rewritten, or false if successful.
+bool Rewriter::ReplaceStmt(Stmt *From, Stmt *To) {
+  // Measaure the old text.
+  int Size = getRangeSize(From->getSourceRange());
+  if (Size == -1)
+    return true;
+
+  // Get the new text.
+  std::string SStr;
+  llvm::raw_string_ostream S(SStr);
+  To->printPretty(S, 0, PrintingPolicy(*LangOpts));
+  const std::string &Str = S.str();
+
+  ReplaceText(From->getLocStart(), Size, Str);
+  return false;
+}
+
+std::string Rewriter::ConvertToString(Stmt *From) {
+  std::string SStr;
+  llvm::raw_string_ostream S(SStr);
+  From->printPretty(S, 0, PrintingPolicy(*LangOpts));
+  return S.str();
+}
+
+bool Rewriter::IncreaseIndentation(CharSourceRange range,
+                                   SourceLocation parentIndent) {
+  if (range.isInvalid()) return true;
+  if (!isRewritable(range.getBegin())) return true;
+  if (!isRewritable(range.getEnd())) return true;
+  if (!isRewritable(parentIndent)) return true;
+
+  FileID StartFileID, EndFileID, parentFileID;
+  unsigned StartOff, EndOff, parentOff;
+
+  StartOff = getLocationOffsetAndFileID(range.getBegin(), StartFileID);
+  EndOff   = getLocationOffsetAndFileID(range.getEnd(), EndFileID);
+  parentOff = getLocationOffsetAndFileID(parentIndent, parentFileID);
+
+  if (StartFileID != EndFileID || StartFileID != parentFileID)
+    return true;
+  if (StartOff > EndOff)
+    return true;
+
+  FileID FID = StartFileID;
+  StringRef MB = SourceMgr->getBufferData(FID);
+
+  unsigned parentLineNo = SourceMgr->getLineNumber(FID, parentOff) - 1;
+  unsigned startLineNo = SourceMgr->getLineNumber(FID, StartOff) - 1;
+  unsigned endLineNo = SourceMgr->getLineNumber(FID, EndOff) - 1;
+  
+  const SrcMgr::ContentCache *
+      Content = SourceMgr->getSLocEntry(FID).getFile().getContentCache();
+  
+  // Find where the lines start.
+  unsigned parentLineOffs = Content->SourceLineCache[parentLineNo];
+  unsigned startLineOffs = Content->SourceLineCache[startLineNo];
+
+  // Find the whitespace at the start of each line.
+  StringRef parentSpace, startSpace;
+  {
+    unsigned i = parentLineOffs;
+    while (isWhitespace(MB[i]))
+      ++i;
+    parentSpace = MB.substr(parentLineOffs, i-parentLineOffs);
+
+    i = startLineOffs;
+    while (isWhitespace(MB[i]))
+      ++i;
+    startSpace = MB.substr(startLineOffs, i-startLineOffs);
+  }
+  if (parentSpace.size() >= startSpace.size())
+    return true;
+  if (!startSpace.startswith(parentSpace))
+    return true;
+
+  StringRef indent = startSpace.substr(parentSpace.size());
+
+  // Indent the lines between start/end offsets.
+  RewriteBuffer &RB = getEditBuffer(FID);
+  for (unsigned lineNo = startLineNo; lineNo <= endLineNo; ++lineNo) {
+    unsigned offs = Content->SourceLineCache[lineNo];
+    unsigned i = offs;
+    while (isWhitespace(MB[i]))
+      ++i;
+    StringRef origIndent = MB.substr(offs, i-offs);
+    if (origIndent.startswith(startSpace))
+      RB.InsertText(offs, indent, /*InsertAfter=*/false);
+  }
+
+  return false;
+}
+
+// A wrapper for a file stream that atomically overwrites the target.
+//
+// Creates a file output stream for a temporary file in the constructor,
+// which is later accessible via getStream() if ok() return true.
+// Flushes the stream and moves the temporary file to the target location
+// in the destructor.
+class AtomicallyMovedFile {
+public:
+  AtomicallyMovedFile(DiagnosticsEngine &Diagnostics, StringRef Filename,
+                      bool &AllWritten)
+    : Diagnostics(Diagnostics), Filename(Filename), AllWritten(AllWritten) {
+    TempFilename = Filename;
+    TempFilename += "-%%%%%%%%";
+    int FD;
+    if (llvm::sys::fs::unique_file(TempFilename.str(), FD, TempFilename,
+                                    /*makeAbsolute=*/true, 0664)) {
+      AllWritten = false;
+      Diagnostics.Report(clang::diag::err_unable_to_make_temp)
+        << TempFilename;
+    } else {
+      FileStream.reset(new llvm::raw_fd_ostream(FD, /*shouldClose=*/true));
+    }
+  }
+
+  ~AtomicallyMovedFile() {
+    if (!ok()) return;
+
+    FileStream->flush();
+#ifdef _WIN32
+    // Win32 does not allow rename/removing opened files.
+    FileStream.reset();
+#endif
+    if (llvm::error_code ec =
+          llvm::sys::fs::rename(TempFilename.str(), Filename)) {
+      AllWritten = false;
+      Diagnostics.Report(clang::diag::err_unable_to_rename_temp)
+        << TempFilename << Filename << ec.message();
+      bool existed;
+      // If the remove fails, there's not a lot we can do - this is already an
+      // error.
+      llvm::sys::fs::remove(TempFilename.str(), existed);
+    }
+  }
+
+  bool ok() { return FileStream; }
+  llvm::raw_ostream &getStream() { return *FileStream; }
+
+private:
+  DiagnosticsEngine &Diagnostics;
+  StringRef Filename;
+  SmallString<128> TempFilename;
+  OwningPtr<llvm::raw_fd_ostream> FileStream;
+  bool &AllWritten;
+};
+
+bool Rewriter::overwriteChangedFiles() {
+  bool AllWritten = true;
+  for (buffer_iterator I = buffer_begin(), E = buffer_end(); I != E; ++I) {
+    const FileEntry *Entry =
+        getSourceMgr().getFileEntryForID(I->first);
+    AtomicallyMovedFile File(getSourceMgr().getDiagnostics(), Entry->getName(),
+                             AllWritten);
+    if (File.ok()) {
+      I->second.write(File.getStream());
+    }
+  }
+  return !AllWritten;
+}
diff --git a/lib/Rewrite/Core/TokenRewriter.cpp b/lib/Rewrite/Core/TokenRewriter.cpp
new file mode 100644
index 0000000..940ece2
--- /dev/null
+++ b/lib/Rewrite/Core/TokenRewriter.cpp
@@ -0,0 +1,99 @@
+//===--- TokenRewriter.cpp - Token-based code rewriting interface ---------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file implements the TokenRewriter class, which is used for code
+//  transformations.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Rewrite/Core/TokenRewriter.h"
+#include "clang/Lex/Lexer.h"
+#include "clang/Lex/ScratchBuffer.h"
+#include "clang/Basic/SourceManager.h"
+using namespace clang;
+
+TokenRewriter::TokenRewriter(FileID FID, SourceManager &SM,
+                             const LangOptions &LangOpts) {
+  ScratchBuf.reset(new ScratchBuffer(SM));
+
+  // Create a lexer to lex all the tokens of the main file in raw mode.
+  const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID);
+  Lexer RawLex(FID, FromFile, SM, LangOpts);
+
+  // Return all comments and whitespace as tokens.
+  RawLex.SetKeepWhitespaceMode(true);
+
+  // Lex the file, populating our datastructures.
+  Token RawTok;
+  RawLex.LexFromRawLexer(RawTok);
+  while (RawTok.isNot(tok::eof)) {
+#if 0
+    if (Tok.is(tok::raw_identifier)) {
+      // Look up the identifier info for the token.  This should use
+      // IdentifierTable directly instead of PP.
+      PP.LookUpIdentifierInfo(Tok);
+    }
+#endif
+
+    AddToken(RawTok, TokenList.end());
+    RawLex.LexFromRawLexer(RawTok);
+  }
+}
+
+TokenRewriter::~TokenRewriter() {
+}
+
+
+/// RemapIterator - Convert from token_iterator (a const iterator) to
+/// TokenRefTy (a non-const iterator).
+TokenRewriter::TokenRefTy TokenRewriter::RemapIterator(token_iterator I) {
+  if (I == token_end()) return TokenList.end();
+
+  // FIXME: This is horrible, we should use our own list or something to avoid
+  // this.
+  std::map<SourceLocation, TokenRefTy>::iterator MapIt =
+    TokenAtLoc.find(I->getLocation());
+  assert(MapIt != TokenAtLoc.end() && "iterator not in rewriter?");
+  return MapIt->second;
+}
+
+
+/// AddToken - Add the specified token into the Rewriter before the other
+/// position.
+TokenRewriter::TokenRefTy
+TokenRewriter::AddToken(const Token &T, TokenRefTy Where) {
+  Where = TokenList.insert(Where, T);
+
+  bool InsertSuccess = TokenAtLoc.insert(std::make_pair(T.getLocation(),
+                                                        Where)).second;
+  assert(InsertSuccess && "Token location already in rewriter!");
+  (void)InsertSuccess;
+  return Where;
+}
+
+
+TokenRewriter::token_iterator
+TokenRewriter::AddTokenBefore(token_iterator I, const char *Val) {
+  unsigned Len = strlen(Val);
+
+  // Plop the string into the scratch buffer, then create a token for this
+  // string.
+  Token Tok;
+  Tok.startToken();
+  const char *Spelling;
+  Tok.setLocation(ScratchBuf->getToken(Val, Len, Spelling));
+  Tok.setLength(Len);
+
+  // TODO: Form a whole lexer around this and relex the token!  For now, just
+  // set kind to tok::unknown.
+  Tok.setKind(tok::unknown);
+
+  return AddToken(Tok, RemapIterator(I));
+}
+
diff --git a/lib/Rewrite/DeltaTree.cpp b/lib/Rewrite/DeltaTree.cpp
deleted file mode 100644
index 4297dc8..0000000
--- a/lib/Rewrite/DeltaTree.cpp
+++ /dev/null
@@ -1,467 +0,0 @@
-//===--- DeltaTree.cpp - B-Tree for Rewrite Delta tracking ----------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file implements the DeltaTree and related classes.
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/Rewrite/DeltaTree.h"
-#include "clang/Basic/LLVM.h"
-#include <cstring>
-#include <cstdio>
-using namespace clang;
-
-/// The DeltaTree class is a multiway search tree (BTree) structure with some
-/// fancy features.  B-Trees are generally more memory and cache efficient
-/// than binary trees, because they store multiple keys/values in each node.
-///
-/// DeltaTree implements a key/value mapping from FileIndex to Delta, allowing
-/// fast lookup by FileIndex.  However, an added (important) bonus is that it
-/// can also efficiently tell us the full accumulated delta for a specific
-/// file offset as well, without traversing the whole tree.
-///
-/// The nodes of the tree are made up of instances of two classes:
-/// DeltaTreeNode and DeltaTreeInteriorNode.  The later subclasses the
-/// former and adds children pointers.  Each node knows the full delta of all
-/// entries (recursively) contained inside of it, which allows us to get the
-/// full delta implied by a whole subtree in constant time.
-
-namespace {
-  /// SourceDelta - As code in the original input buffer is added and deleted,
-  /// SourceDelta records are used to keep track of how the input SourceLocation
-  /// object is mapped into the output buffer.
-  struct SourceDelta {
-    unsigned FileLoc;
-    int Delta;
-
-    static SourceDelta get(unsigned Loc, int D) {
-      SourceDelta Delta;
-      Delta.FileLoc = Loc;
-      Delta.Delta = D;
-      return Delta;
-    }
-  };
-  
-  /// DeltaTreeNode - The common part of all nodes.
-  ///
-  class DeltaTreeNode {
-  public:
-    struct InsertResult {
-      DeltaTreeNode *LHS, *RHS;
-      SourceDelta Split;
-    };
-    
-  private:
-    friend class DeltaTreeInteriorNode;
-
-    /// WidthFactor - This controls the number of K/V slots held in the BTree:
-    /// how wide it is.  Each level of the BTree is guaranteed to have at least
-    /// WidthFactor-1 K/V pairs (except the root) and may have at most
-    /// 2*WidthFactor-1 K/V pairs.
-    enum { WidthFactor = 8 };
-
-    /// Values - This tracks the SourceDelta's currently in this node.
-    ///
-    SourceDelta Values[2*WidthFactor-1];
-
-    /// NumValuesUsed - This tracks the number of values this node currently
-    /// holds.
-    unsigned char NumValuesUsed;
-
-    /// IsLeaf - This is true if this is a leaf of the btree.  If false, this is
-    /// an interior node, and is actually an instance of DeltaTreeInteriorNode.
-    bool IsLeaf;
-
-    /// FullDelta - This is the full delta of all the values in this node and
-    /// all children nodes.
-    int FullDelta;
-  public:
-    DeltaTreeNode(bool isLeaf = true)
-      : NumValuesUsed(0), IsLeaf(isLeaf), FullDelta(0) {}
-
-    bool isLeaf() const { return IsLeaf; }
-    int getFullDelta() const { return FullDelta; }
-    bool isFull() const { return NumValuesUsed == 2*WidthFactor-1; }
-
-    unsigned getNumValuesUsed() const { return NumValuesUsed; }
-    const SourceDelta &getValue(unsigned i) const {
-      assert(i < NumValuesUsed && "Invalid value #");
-      return Values[i];
-    }
-    SourceDelta &getValue(unsigned i) {
-      assert(i < NumValuesUsed && "Invalid value #");
-      return Values[i];
-    }
-
-    /// DoInsertion - Do an insertion of the specified FileIndex/Delta pair into
-    /// this node.  If insertion is easy, do it and return false.  Otherwise,
-    /// split the node, populate InsertRes with info about the split, and return
-    /// true.
-    bool DoInsertion(unsigned FileIndex, int Delta, InsertResult *InsertRes);
-
-    void DoSplit(InsertResult &InsertRes);
-
-
-    /// RecomputeFullDeltaLocally - Recompute the FullDelta field by doing a
-    /// local walk over our contained deltas.
-    void RecomputeFullDeltaLocally();
-
-    void Destroy();
-
-    //static inline bool classof(const DeltaTreeNode *) { return true; }
-  };
-} // end anonymous namespace
-
-namespace {
-  /// DeltaTreeInteriorNode - When isLeaf = false, a node has child pointers.
-  /// This class tracks them.
-  class DeltaTreeInteriorNode : public DeltaTreeNode {
-    DeltaTreeNode *Children[2*WidthFactor];
-    ~DeltaTreeInteriorNode() {
-      for (unsigned i = 0, e = NumValuesUsed+1; i != e; ++i)
-        Children[i]->Destroy();
-    }
-    friend class DeltaTreeNode;
-  public:
-    DeltaTreeInteriorNode() : DeltaTreeNode(false /*nonleaf*/) {}
-
-    DeltaTreeInteriorNode(const InsertResult &IR)
-      : DeltaTreeNode(false /*nonleaf*/) {
-      Children[0] = IR.LHS;
-      Children[1] = IR.RHS;
-      Values[0] = IR.Split;
-      FullDelta = IR.LHS->getFullDelta()+IR.RHS->getFullDelta()+IR.Split.Delta;
-      NumValuesUsed = 1;
-    }
-
-    const DeltaTreeNode *getChild(unsigned i) const {
-      assert(i < getNumValuesUsed()+1 && "Invalid child");
-      return Children[i];
-    }
-    DeltaTreeNode *getChild(unsigned i) {
-      assert(i < getNumValuesUsed()+1 && "Invalid child");
-      return Children[i];
-    }
-
-  //static inline bool classof(const DeltaTreeInteriorNode *) { return true; }
-    static inline bool classof(const DeltaTreeNode *N) { return !N->isLeaf(); }
-  };
-}
-
-
-/// Destroy - A 'virtual' destructor.
-void DeltaTreeNode::Destroy() {
-  if (isLeaf())
-    delete this;
-  else
-    delete cast<DeltaTreeInteriorNode>(this);
-}
-
-/// RecomputeFullDeltaLocally - Recompute the FullDelta field by doing a
-/// local walk over our contained deltas.
-void DeltaTreeNode::RecomputeFullDeltaLocally() {
-  int NewFullDelta = 0;
-  for (unsigned i = 0, e = getNumValuesUsed(); i != e; ++i)
-    NewFullDelta += Values[i].Delta;
-  if (DeltaTreeInteriorNode *IN = dyn_cast<DeltaTreeInteriorNode>(this))
-    for (unsigned i = 0, e = getNumValuesUsed()+1; i != e; ++i)
-      NewFullDelta += IN->getChild(i)->getFullDelta();
-  FullDelta = NewFullDelta;
-}
-
-/// DoInsertion - Do an insertion of the specified FileIndex/Delta pair into
-/// this node.  If insertion is easy, do it and return false.  Otherwise,
-/// split the node, populate InsertRes with info about the split, and return
-/// true.
-bool DeltaTreeNode::DoInsertion(unsigned FileIndex, int Delta,
-                                InsertResult *InsertRes) {
-  // Maintain full delta for this node.
-  FullDelta += Delta;
-
-  // Find the insertion point, the first delta whose index is >= FileIndex.
-  unsigned i = 0, e = getNumValuesUsed();
-  while (i != e && FileIndex > getValue(i).FileLoc)
-    ++i;
-
-  // If we found an a record for exactly this file index, just merge this
-  // value into the pre-existing record and finish early.
-  if (i != e && getValue(i).FileLoc == FileIndex) {
-    // NOTE: Delta could drop to zero here.  This means that the delta entry is
-    // useless and could be removed.  Supporting erases is more complex than
-    // leaving an entry with Delta=0, so we just leave an entry with Delta=0 in
-    // the tree.
-    Values[i].Delta += Delta;
-    return false;
-  }
-
-  // Otherwise, we found an insertion point, and we know that the value at the
-  // specified index is > FileIndex.  Handle the leaf case first.
-  if (isLeaf()) {
-    if (!isFull()) {
-      // For an insertion into a non-full leaf node, just insert the value in
-      // its sorted position.  This requires moving later values over.
-      if (i != e)
-        memmove(&Values[i+1], &Values[i], sizeof(Values[0])*(e-i));
-      Values[i] = SourceDelta::get(FileIndex, Delta);
-      ++NumValuesUsed;
-      return false;
-    }
-
-    // Otherwise, if this is leaf is full, split the node at its median, insert
-    // the value into one of the children, and return the result.
-    assert(InsertRes && "No result location specified");
-    DoSplit(*InsertRes);
-
-    if (InsertRes->Split.FileLoc > FileIndex)
-      InsertRes->LHS->DoInsertion(FileIndex, Delta, 0 /*can't fail*/);
-    else
-      InsertRes->RHS->DoInsertion(FileIndex, Delta, 0 /*can't fail*/);
-    return true;
-  }
-
-  // Otherwise, this is an interior node.  Send the request down the tree.
-  DeltaTreeInteriorNode *IN = cast<DeltaTreeInteriorNode>(this);
-  if (!IN->Children[i]->DoInsertion(FileIndex, Delta, InsertRes))
-    return false; // If there was space in the child, just return.
-
-  // Okay, this split the subtree, producing a new value and two children to
-  // insert here.  If this node is non-full, we can just insert it directly.
-  if (!isFull()) {
-    // Now that we have two nodes and a new element, insert the perclated value
-    // into ourself by moving all the later values/children down, then inserting
-    // the new one.
-    if (i != e)
-      memmove(&IN->Children[i+2], &IN->Children[i+1],
-              (e-i)*sizeof(IN->Children[0]));
-    IN->Children[i] = InsertRes->LHS;
-    IN->Children[i+1] = InsertRes->RHS;
-
-    if (e != i)
-      memmove(&Values[i+1], &Values[i], (e-i)*sizeof(Values[0]));
-    Values[i] = InsertRes->Split;
-    ++NumValuesUsed;
-    return false;
-  }
-
-  // Finally, if this interior node was full and a node is percolated up, split
-  // ourself and return that up the chain.  Start by saving all our info to
-  // avoid having the split clobber it.
-  IN->Children[i] = InsertRes->LHS;
-  DeltaTreeNode *SubRHS = InsertRes->RHS;
-  SourceDelta SubSplit = InsertRes->Split;
-
-  // Do the split.
-  DoSplit(*InsertRes);
-
-  // Figure out where to insert SubRHS/NewSplit.
-  DeltaTreeInteriorNode *InsertSide;
-  if (SubSplit.FileLoc < InsertRes->Split.FileLoc)
-    InsertSide = cast<DeltaTreeInteriorNode>(InsertRes->LHS);
-  else
-    InsertSide = cast<DeltaTreeInteriorNode>(InsertRes->RHS);
-
-  // We now have a non-empty interior node 'InsertSide' to insert
-  // SubRHS/SubSplit into.  Find out where to insert SubSplit.
-
-  // Find the insertion point, the first delta whose index is >SubSplit.FileLoc.
-  i = 0; e = InsertSide->getNumValuesUsed();
-  while (i != e && SubSplit.FileLoc > InsertSide->getValue(i).FileLoc)
-    ++i;
-
-  // Now we know that i is the place to insert the split value into.  Insert it
-  // and the child right after it.
-  if (i != e)
-    memmove(&InsertSide->Children[i+2], &InsertSide->Children[i+1],
-            (e-i)*sizeof(IN->Children[0]));
-  InsertSide->Children[i+1] = SubRHS;
-
-  if (e != i)
-    memmove(&InsertSide->Values[i+1], &InsertSide->Values[i],
-            (e-i)*sizeof(Values[0]));
-  InsertSide->Values[i] = SubSplit;
-  ++InsertSide->NumValuesUsed;
-  InsertSide->FullDelta += SubSplit.Delta + SubRHS->getFullDelta();
-  return true;
-}
-
-/// DoSplit - Split the currently full node (which has 2*WidthFactor-1 values)
-/// into two subtrees each with "WidthFactor-1" values and a pivot value.
-/// Return the pieces in InsertRes.
-void DeltaTreeNode::DoSplit(InsertResult &InsertRes) {
-  assert(isFull() && "Why split a non-full node?");
-
-  // Since this node is full, it contains 2*WidthFactor-1 values.  We move
-  // the first 'WidthFactor-1' values to the LHS child (which we leave in this
-  // node), propagate one value up, and move the last 'WidthFactor-1' values
-  // into the RHS child.
-
-  // Create the new child node.
-  DeltaTreeNode *NewNode;
-  if (DeltaTreeInteriorNode *IN = dyn_cast<DeltaTreeInteriorNode>(this)) {
-    // If this is an interior node, also move over 'WidthFactor' children
-    // into the new node.
-    DeltaTreeInteriorNode *New = new DeltaTreeInteriorNode();
-    memcpy(&New->Children[0], &IN->Children[WidthFactor],
-           WidthFactor*sizeof(IN->Children[0]));
-    NewNode = New;
-  } else {
-    // Just create the new leaf node.
-    NewNode = new DeltaTreeNode();
-  }
-
-  // Move over the last 'WidthFactor-1' values from here to NewNode.
-  memcpy(&NewNode->Values[0], &Values[WidthFactor],
-         (WidthFactor-1)*sizeof(Values[0]));
-
-  // Decrease the number of values in the two nodes.
-  NewNode->NumValuesUsed = NumValuesUsed = WidthFactor-1;
-
-  // Recompute the two nodes' full delta.
-  NewNode->RecomputeFullDeltaLocally();
-  RecomputeFullDeltaLocally();
-
-  InsertRes.LHS = this;
-  InsertRes.RHS = NewNode;
-  InsertRes.Split = Values[WidthFactor-1];
-}
-
-
-
-//===----------------------------------------------------------------------===//
-//                        DeltaTree Implementation
-//===----------------------------------------------------------------------===//
-
-//#define VERIFY_TREE
-
-#ifdef VERIFY_TREE
-/// VerifyTree - Walk the btree performing assertions on various properties to
-/// verify consistency.  This is useful for debugging new changes to the tree.
-static void VerifyTree(const DeltaTreeNode *N) {
-  const DeltaTreeInteriorNode *IN = dyn_cast<DeltaTreeInteriorNode>(N);
-  if (IN == 0) {
-    // Verify leaves, just ensure that FullDelta matches up and the elements
-    // are in proper order.
-    int FullDelta = 0;
-    for (unsigned i = 0, e = N->getNumValuesUsed(); i != e; ++i) {
-      if (i)
-        assert(N->getValue(i-1).FileLoc < N->getValue(i).FileLoc);
-      FullDelta += N->getValue(i).Delta;
-    }
-    assert(FullDelta == N->getFullDelta());
-    return;
-  }
-
-  // Verify interior nodes: Ensure that FullDelta matches up and the
-  // elements are in proper order and the children are in proper order.
-  int FullDelta = 0;
-  for (unsigned i = 0, e = IN->getNumValuesUsed(); i != e; ++i) {
-    const SourceDelta &IVal = N->getValue(i);
-    const DeltaTreeNode *IChild = IN->getChild(i);
-    if (i)
-      assert(IN->getValue(i-1).FileLoc < IVal.FileLoc);
-    FullDelta += IVal.Delta;
-    FullDelta += IChild->getFullDelta();
-
-    // The largest value in child #i should be smaller than FileLoc.
-    assert(IChild->getValue(IChild->getNumValuesUsed()-1).FileLoc <
-           IVal.FileLoc);
-
-    // The smallest value in child #i+1 should be larger than FileLoc.
-    assert(IN->getChild(i+1)->getValue(0).FileLoc > IVal.FileLoc);
-    VerifyTree(IChild);
-  }
-
-  FullDelta += IN->getChild(IN->getNumValuesUsed())->getFullDelta();
-
-  assert(FullDelta == N->getFullDelta());
-}
-#endif  // VERIFY_TREE
-
-static DeltaTreeNode *getRoot(void *Root) {
-  return (DeltaTreeNode*)Root;
-}
-
-DeltaTree::DeltaTree() {
-  Root = new DeltaTreeNode();
-}
-DeltaTree::DeltaTree(const DeltaTree &RHS) {
-  // Currently we only support copying when the RHS is empty.
-  assert(getRoot(RHS.Root)->getNumValuesUsed() == 0 &&
-         "Can only copy empty tree");
-  Root = new DeltaTreeNode();
-}
-
-DeltaTree::~DeltaTree() {
-  getRoot(Root)->Destroy();
-}
-
-/// getDeltaAt - Return the accumulated delta at the specified file offset.
-/// This includes all insertions or delections that occurred *before* the
-/// specified file index.
-int DeltaTree::getDeltaAt(unsigned FileIndex) const {
-  const DeltaTreeNode *Node = getRoot(Root);
-
-  int Result = 0;
-
-  // Walk down the tree.
-  while (1) {
-    // For all nodes, include any local deltas before the specified file
-    // index by summing them up directly.  Keep track of how many were
-    // included.
-    unsigned NumValsGreater = 0;
-    for (unsigned e = Node->getNumValuesUsed(); NumValsGreater != e;
-         ++NumValsGreater) {
-      const SourceDelta &Val = Node->getValue(NumValsGreater);
-
-      if (Val.FileLoc >= FileIndex)
-        break;
-      Result += Val.Delta;
-    }
-
-    // If we have an interior node, include information about children and
-    // recurse.  Otherwise, if we have a leaf, we're done.
-    const DeltaTreeInteriorNode *IN = dyn_cast<DeltaTreeInteriorNode>(Node);
-    if (!IN) return Result;
-
-    // Include any children to the left of the values we skipped, all of
-    // their deltas should be included as well.
-    for (unsigned i = 0; i != NumValsGreater; ++i)
-      Result += IN->getChild(i)->getFullDelta();
-
-    // If we found exactly the value we were looking for, break off the
-    // search early.  There is no need to search the RHS of the value for
-    // partial results.
-    if (NumValsGreater != Node->getNumValuesUsed() &&
-        Node->getValue(NumValsGreater).FileLoc == FileIndex)
-      return Result+IN->getChild(NumValsGreater)->getFullDelta();
-
-    // Otherwise, traverse down the tree.  The selected subtree may be
-    // partially included in the range.
-    Node = IN->getChild(NumValsGreater);
-  }
-  // NOT REACHED.
-}
-
-/// AddDelta - When a change is made that shifts around the text buffer,
-/// this method is used to record that info.  It inserts a delta of 'Delta'
-/// into the current DeltaTree at offset FileIndex.
-void DeltaTree::AddDelta(unsigned FileIndex, int Delta) {
-  assert(Delta && "Adding a noop?");
-  DeltaTreeNode *MyRoot = getRoot(Root);
-
-  DeltaTreeNode::InsertResult InsertRes;
-  if (MyRoot->DoInsertion(FileIndex, Delta, &InsertRes)) {
-    Root = MyRoot = new DeltaTreeInteriorNode(InsertRes);
-  }
-
-#ifdef VERIFY_TREE
-  VerifyTree(MyRoot);
-#endif
-}
-
diff --git a/lib/Rewrite/FixItRewriter.cpp b/lib/Rewrite/FixItRewriter.cpp
deleted file mode 100644
index 3863adb..0000000
--- a/lib/Rewrite/FixItRewriter.cpp
+++ /dev/null
@@ -1,205 +0,0 @@
-//===--- FixItRewriter.cpp - Fix-It Rewriter Diagnostic Client --*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This is a diagnostic client adaptor that performs rewrites as
-// suggested by code modification hints attached to diagnostics. It
-// then forwards any diagnostics to the adapted diagnostic client.
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/Rewrite/FixItRewriter.h"
-#include "clang/Edit/Commit.h"
-#include "clang/Edit/EditsReceiver.h"
-#include "clang/Basic/FileManager.h"
-#include "clang/Basic/SourceLocation.h"
-#include "clang/Basic/SourceManager.h"
-#include "clang/Frontend/FrontendDiagnostic.h"
-#include "llvm/Support/raw_ostream.h"
-#include "llvm/Support/Path.h"
-#include "llvm/ADT/OwningPtr.h"
-#include <cstdio>
-
-using namespace clang;
-
-FixItRewriter::FixItRewriter(DiagnosticsEngine &Diags, SourceManager &SourceMgr,
-                             const LangOptions &LangOpts,
-                             FixItOptions *FixItOpts)
-  : Diags(Diags),
-    Editor(SourceMgr, LangOpts),
-    Rewrite(SourceMgr, LangOpts),
-    FixItOpts(FixItOpts),
-    NumFailures(0),
-    PrevDiagSilenced(false) {
-  OwnsClient = Diags.ownsClient();
-  Client = Diags.takeClient();
-  Diags.setClient(this);
-}
-
-FixItRewriter::~FixItRewriter() {
-  Diags.takeClient();
-  Diags.setClient(Client, OwnsClient);
-}
-
-bool FixItRewriter::WriteFixedFile(FileID ID, raw_ostream &OS) {
-  const RewriteBuffer *RewriteBuf = Rewrite.getRewriteBufferFor(ID);
-  if (!RewriteBuf) return true;
-  RewriteBuf->write(OS);
-  OS.flush();
-  return false;
-}
-
-namespace {
-
-class RewritesReceiver : public edit::EditsReceiver {
-  Rewriter &Rewrite;
-
-public:
-  RewritesReceiver(Rewriter &Rewrite) : Rewrite(Rewrite) { }
-
-  virtual void insert(SourceLocation loc, StringRef text) {
-    Rewrite.InsertText(loc, text);
-  }
-  virtual void replace(CharSourceRange range, StringRef text) {
-    Rewrite.ReplaceText(range.getBegin(), Rewrite.getRangeSize(range), text);
-  }
-};
-
-}
-
-bool FixItRewriter::WriteFixedFiles(
-            std::vector<std::pair<std::string, std::string> > *RewrittenFiles) {
-  if (NumFailures > 0 && !FixItOpts->FixWhatYouCan) {
-    Diag(FullSourceLoc(), diag::warn_fixit_no_changes);
-    return true;
-  }
-
-  RewritesReceiver Rec(Rewrite);
-  Editor.applyRewrites(Rec);
-
-  for (iterator I = buffer_begin(), E = buffer_end(); I != E; ++I) {
-    const FileEntry *Entry = Rewrite.getSourceMgr().getFileEntryForID(I->first);
-    int fd;
-    std::string Filename = FixItOpts->RewriteFilename(Entry->getName(), fd);
-    std::string Err;
-    OwningPtr<llvm::raw_fd_ostream> OS;
-    if (fd != -1) {
-      OS.reset(new llvm::raw_fd_ostream(fd, /*shouldClose=*/true));
-    } else {
-      OS.reset(new llvm::raw_fd_ostream(Filename.c_str(), Err,
-                                        llvm::raw_fd_ostream::F_Binary));
-    }
-    if (!Err.empty()) {
-      Diags.Report(clang::diag::err_fe_unable_to_open_output)
-          << Filename << Err;
-      continue;
-    }
-    RewriteBuffer &RewriteBuf = I->second;
-    RewriteBuf.write(*OS);
-    OS->flush();
-
-    if (RewrittenFiles)
-      RewrittenFiles->push_back(std::make_pair(Entry->getName(), Filename));
-  }
-
-  return false;
-}
-
-bool FixItRewriter::IncludeInDiagnosticCounts() const {
-  return Client ? Client->IncludeInDiagnosticCounts() : true;
-}
-
-void FixItRewriter::HandleDiagnostic(DiagnosticsEngine::Level DiagLevel,
-                                     const Diagnostic &Info) {
-  // Default implementation (Warnings/errors count).
-  DiagnosticConsumer::HandleDiagnostic(DiagLevel, Info);
-
-  if (!FixItOpts->Silent ||
-      DiagLevel >= DiagnosticsEngine::Error ||
-      (DiagLevel == DiagnosticsEngine::Note && !PrevDiagSilenced) ||
-      (DiagLevel > DiagnosticsEngine::Note && Info.getNumFixItHints())) {
-    Client->HandleDiagnostic(DiagLevel, Info);
-    PrevDiagSilenced = false;
-  } else {
-    PrevDiagSilenced = true;
-  }
-
-  // Skip over any diagnostics that are ignored or notes.
-  if (DiagLevel <= DiagnosticsEngine::Note)
-    return;
-  // Skip over errors if we are only fixing warnings.
-  if (DiagLevel >= DiagnosticsEngine::Error && FixItOpts->FixOnlyWarnings) {
-    ++NumFailures;
-    return;
-  }
-
-  // Make sure that we can perform all of the modifications we
-  // in this diagnostic.
-  edit::Commit commit(Editor);
-  for (unsigned Idx = 0, Last = Info.getNumFixItHints();
-       Idx < Last; ++Idx) {
-    const FixItHint &Hint = Info.getFixItHint(Idx);
-
-    if (Hint.CodeToInsert.empty()) {
-      if (Hint.InsertFromRange.isValid())
-        commit.insertFromRange(Hint.RemoveRange.getBegin(),
-                           Hint.InsertFromRange, /*afterToken=*/false,
-                           Hint.BeforePreviousInsertions);
-      else
-        commit.remove(Hint.RemoveRange);
-    } else {
-      if (Hint.RemoveRange.isTokenRange() ||
-          Hint.RemoveRange.getBegin() != Hint.RemoveRange.getEnd())
-        commit.replace(Hint.RemoveRange, Hint.CodeToInsert);
-      else
-        commit.insert(Hint.RemoveRange.getBegin(), Hint.CodeToInsert,
-                    /*afterToken=*/false, Hint.BeforePreviousInsertions);
-    }
-  }
-  bool CanRewrite = Info.getNumFixItHints() > 0 && commit.isCommitable();
-
-  if (!CanRewrite) {
-    if (Info.getNumFixItHints() > 0)
-      Diag(Info.getLocation(), diag::note_fixit_in_macro);
-
-    // If this was an error, refuse to perform any rewriting.
-    if (DiagLevel >= DiagnosticsEngine::Error) {
-      if (++NumFailures == 1)
-        Diag(Info.getLocation(), diag::note_fixit_unfixed_error);
-    }
-    return;
-  }
-  
-  if (!Editor.commit(commit)) {
-    ++NumFailures;
-    Diag(Info.getLocation(), diag::note_fixit_failed);
-    return;
-  }
-
-  Diag(Info.getLocation(), diag::note_fixit_applied);
-}
-
-/// \brief Emit a diagnostic via the adapted diagnostic client.
-void FixItRewriter::Diag(SourceLocation Loc, unsigned DiagID) {
-  // When producing this diagnostic, we temporarily bypass ourselves,
-  // clear out any current diagnostic, and let the downstream client
-  // format the diagnostic.
-  Diags.takeClient();
-  Diags.setClient(Client);
-  Diags.Clear();
-  Diags.Report(Loc, DiagID);
-  Diags.takeClient();
-  Diags.setClient(this);
-}
-
-DiagnosticConsumer *FixItRewriter::clone(DiagnosticsEngine &Diags) const {
-  return new FixItRewriter(Diags, Diags.getSourceManager(), 
-                           Rewrite.getLangOpts(), FixItOpts);
-}
-
-FixItOptions::~FixItOptions() {}
diff --git a/lib/Rewrite/Frontend/CMakeLists.txt b/lib/Rewrite/Frontend/CMakeLists.txt
new file mode 100644
index 0000000..9017e47
--- /dev/null
+++ b/lib/Rewrite/Frontend/CMakeLists.txt
@@ -0,0 +1,28 @@
+add_clang_library(clangRewriteFrontend
+  FixItRewriter.cpp
+  FrontendActions.cpp
+  HTMLPrint.cpp
+  InclusionRewriter.cpp
+  RewriteMacros.cpp
+  RewriteModernObjC.cpp
+  RewriteObjC.cpp
+  RewriteTest.cpp
+  )
+
+add_dependencies(clangRewriteFrontend
+  ClangAttrClasses
+  ClangAttrList
+  ClangAttrParsedAttrList
+  ClangCommentNodes
+  ClangDeclNodes
+  ClangDiagnosticCommon
+  ClangDiagnosticFrontend
+  ClangStmtNodes
+  )
+
+target_link_libraries(clangRewriteFrontend
+  clangBasic
+  clangAST
+  clangParse
+  clangFrontend
+  )
diff --git a/lib/Rewrite/Frontend/FixItRewriter.cpp b/lib/Rewrite/Frontend/FixItRewriter.cpp
new file mode 100644
index 0000000..43a1ab1
--- /dev/null
+++ b/lib/Rewrite/Frontend/FixItRewriter.cpp
@@ -0,0 +1,205 @@
+//===--- FixItRewriter.cpp - Fix-It Rewriter Diagnostic Client --*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This is a diagnostic client adaptor that performs rewrites as
+// suggested by code modification hints attached to diagnostics. It
+// then forwards any diagnostics to the adapted diagnostic client.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Rewrite/Frontend/FixItRewriter.h"
+#include "clang/Edit/Commit.h"
+#include "clang/Edit/EditsReceiver.h"
+#include "clang/Basic/FileManager.h"
+#include "clang/Basic/SourceLocation.h"
+#include "clang/Basic/SourceManager.h"
+#include "clang/Frontend/FrontendDiagnostic.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/Path.h"
+#include "llvm/ADT/OwningPtr.h"
+#include <cstdio>
+
+using namespace clang;
+
+FixItRewriter::FixItRewriter(DiagnosticsEngine &Diags, SourceManager &SourceMgr,
+                             const LangOptions &LangOpts,
+                             FixItOptions *FixItOpts)
+  : Diags(Diags),
+    Editor(SourceMgr, LangOpts),
+    Rewrite(SourceMgr, LangOpts),
+    FixItOpts(FixItOpts),
+    NumFailures(0),
+    PrevDiagSilenced(false) {
+  OwnsClient = Diags.ownsClient();
+  Client = Diags.takeClient();
+  Diags.setClient(this);
+}
+
+FixItRewriter::~FixItRewriter() {
+  Diags.takeClient();
+  Diags.setClient(Client, OwnsClient);
+}
+
+bool FixItRewriter::WriteFixedFile(FileID ID, raw_ostream &OS) {
+  const RewriteBuffer *RewriteBuf = Rewrite.getRewriteBufferFor(ID);
+  if (!RewriteBuf) return true;
+  RewriteBuf->write(OS);
+  OS.flush();
+  return false;
+}
+
+namespace {
+
+class RewritesReceiver : public edit::EditsReceiver {
+  Rewriter &Rewrite;
+
+public:
+  RewritesReceiver(Rewriter &Rewrite) : Rewrite(Rewrite) { }
+
+  virtual void insert(SourceLocation loc, StringRef text) {
+    Rewrite.InsertText(loc, text);
+  }
+  virtual void replace(CharSourceRange range, StringRef text) {
+    Rewrite.ReplaceText(range.getBegin(), Rewrite.getRangeSize(range), text);
+  }
+};
+
+}
+
+bool FixItRewriter::WriteFixedFiles(
+            std::vector<std::pair<std::string, std::string> > *RewrittenFiles) {
+  if (NumFailures > 0 && !FixItOpts->FixWhatYouCan) {
+    Diag(FullSourceLoc(), diag::warn_fixit_no_changes);
+    return true;
+  }
+
+  RewritesReceiver Rec(Rewrite);
+  Editor.applyRewrites(Rec);
+
+  for (iterator I = buffer_begin(), E = buffer_end(); I != E; ++I) {
+    const FileEntry *Entry = Rewrite.getSourceMgr().getFileEntryForID(I->first);
+    int fd;
+    std::string Filename = FixItOpts->RewriteFilename(Entry->getName(), fd);
+    std::string Err;
+    OwningPtr<llvm::raw_fd_ostream> OS;
+    if (fd != -1) {
+      OS.reset(new llvm::raw_fd_ostream(fd, /*shouldClose=*/true));
+    } else {
+      OS.reset(new llvm::raw_fd_ostream(Filename.c_str(), Err,
+                                        llvm::raw_fd_ostream::F_Binary));
+    }
+    if (!Err.empty()) {
+      Diags.Report(clang::diag::err_fe_unable_to_open_output)
+          << Filename << Err;
+      continue;
+    }
+    RewriteBuffer &RewriteBuf = I->second;
+    RewriteBuf.write(*OS);
+    OS->flush();
+
+    if (RewrittenFiles)
+      RewrittenFiles->push_back(std::make_pair(Entry->getName(), Filename));
+  }
+
+  return false;
+}
+
+bool FixItRewriter::IncludeInDiagnosticCounts() const {
+  return Client ? Client->IncludeInDiagnosticCounts() : true;
+}
+
+void FixItRewriter::HandleDiagnostic(DiagnosticsEngine::Level DiagLevel,
+                                     const Diagnostic &Info) {
+  // Default implementation (Warnings/errors count).
+  DiagnosticConsumer::HandleDiagnostic(DiagLevel, Info);
+
+  if (!FixItOpts->Silent ||
+      DiagLevel >= DiagnosticsEngine::Error ||
+      (DiagLevel == DiagnosticsEngine::Note && !PrevDiagSilenced) ||
+      (DiagLevel > DiagnosticsEngine::Note && Info.getNumFixItHints())) {
+    Client->HandleDiagnostic(DiagLevel, Info);
+    PrevDiagSilenced = false;
+  } else {
+    PrevDiagSilenced = true;
+  }
+
+  // Skip over any diagnostics that are ignored or notes.
+  if (DiagLevel <= DiagnosticsEngine::Note)
+    return;
+  // Skip over errors if we are only fixing warnings.
+  if (DiagLevel >= DiagnosticsEngine::Error && FixItOpts->FixOnlyWarnings) {
+    ++NumFailures;
+    return;
+  }
+
+  // Make sure that we can perform all of the modifications we
+  // in this diagnostic.
+  edit::Commit commit(Editor);
+  for (unsigned Idx = 0, Last = Info.getNumFixItHints();
+       Idx < Last; ++Idx) {
+    const FixItHint &Hint = Info.getFixItHint(Idx);
+
+    if (Hint.CodeToInsert.empty()) {
+      if (Hint.InsertFromRange.isValid())
+        commit.insertFromRange(Hint.RemoveRange.getBegin(),
+                           Hint.InsertFromRange, /*afterToken=*/false,
+                           Hint.BeforePreviousInsertions);
+      else
+        commit.remove(Hint.RemoveRange);
+    } else {
+      if (Hint.RemoveRange.isTokenRange() ||
+          Hint.RemoveRange.getBegin() != Hint.RemoveRange.getEnd())
+        commit.replace(Hint.RemoveRange, Hint.CodeToInsert);
+      else
+        commit.insert(Hint.RemoveRange.getBegin(), Hint.CodeToInsert,
+                    /*afterToken=*/false, Hint.BeforePreviousInsertions);
+    }
+  }
+  bool CanRewrite = Info.getNumFixItHints() > 0 && commit.isCommitable();
+
+  if (!CanRewrite) {
+    if (Info.getNumFixItHints() > 0)
+      Diag(Info.getLocation(), diag::note_fixit_in_macro);
+
+    // If this was an error, refuse to perform any rewriting.
+    if (DiagLevel >= DiagnosticsEngine::Error) {
+      if (++NumFailures == 1)
+        Diag(Info.getLocation(), diag::note_fixit_unfixed_error);
+    }
+    return;
+  }
+  
+  if (!Editor.commit(commit)) {
+    ++NumFailures;
+    Diag(Info.getLocation(), diag::note_fixit_failed);
+    return;
+  }
+
+  Diag(Info.getLocation(), diag::note_fixit_applied);
+}
+
+/// \brief Emit a diagnostic via the adapted diagnostic client.
+void FixItRewriter::Diag(SourceLocation Loc, unsigned DiagID) {
+  // When producing this diagnostic, we temporarily bypass ourselves,
+  // clear out any current diagnostic, and let the downstream client
+  // format the diagnostic.
+  Diags.takeClient();
+  Diags.setClient(Client);
+  Diags.Clear();
+  Diags.Report(Loc, DiagID);
+  Diags.takeClient();
+  Diags.setClient(this);
+}
+
+DiagnosticConsumer *FixItRewriter::clone(DiagnosticsEngine &Diags) const {
+  return new FixItRewriter(Diags, Diags.getSourceManager(), 
+                           Rewrite.getLangOpts(), FixItOpts);
+}
+
+FixItOptions::~FixItOptions() {}
diff --git a/lib/Rewrite/Frontend/FrontendActions.cpp b/lib/Rewrite/Frontend/FrontendActions.cpp
new file mode 100644
index 0000000..7d29b6d
--- /dev/null
+++ b/lib/Rewrite/Frontend/FrontendActions.cpp
@@ -0,0 +1,192 @@
+//===--- FrontendActions.cpp ----------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Rewrite/Frontend/FrontendActions.h"
+#include "clang/AST/ASTConsumer.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Parse/Parser.h"
+#include "clang/Basic/FileManager.h"
+#include "clang/Frontend/FrontendActions.h"
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Frontend/FrontendDiagnostic.h"
+#include "clang/Frontend/Utils.h"
+#include "clang/Rewrite/Frontend/ASTConsumers.h"
+#include "clang/Rewrite/Frontend/FixItRewriter.h"
+#include "clang/Rewrite/Frontend/Rewriters.h"
+#include "llvm/ADT/OwningPtr.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/FileSystem.h"
+
+using namespace clang;
+
+//===----------------------------------------------------------------------===//
+// AST Consumer Actions
+//===----------------------------------------------------------------------===//
+
+ASTConsumer *HTMLPrintAction::CreateASTConsumer(CompilerInstance &CI,
+                                                StringRef InFile) {
+  if (raw_ostream *OS = CI.createDefaultOutputFile(false, InFile))
+    return CreateHTMLPrinter(OS, CI.getPreprocessor());
+  return 0;
+}
+
+FixItAction::FixItAction() {}
+FixItAction::~FixItAction() {}
+
+ASTConsumer *FixItAction::CreateASTConsumer(CompilerInstance &CI,
+                                            StringRef InFile) {
+  return new ASTConsumer();
+}
+
+namespace {
+class FixItRewriteInPlace : public FixItOptions {
+public:
+  std::string RewriteFilename(const std::string &Filename, int &fd) {
+    fd = -1;
+    return Filename;
+  }
+};
+
+class FixItActionSuffixInserter : public FixItOptions {
+  std::string NewSuffix;
+
+public:
+  FixItActionSuffixInserter(std::string NewSuffix, bool FixWhatYouCan)
+    : NewSuffix(NewSuffix) {
+      this->FixWhatYouCan = FixWhatYouCan;
+  }
+
+  std::string RewriteFilename(const std::string &Filename, int &fd) {
+    fd = -1;
+    SmallString<128> Path(Filename);
+    llvm::sys::path::replace_extension(Path,
+      NewSuffix + llvm::sys::path::extension(Path));
+    return Path.str();
+  }
+};
+
+class FixItRewriteToTemp : public FixItOptions {
+public:
+  std::string RewriteFilename(const std::string &Filename, int &fd) {
+    SmallString<128> Path;
+    Path = llvm::sys::path::filename(Filename);
+    Path += "-%%%%%%%%";
+    Path += llvm::sys::path::extension(Filename);
+    SmallString<128> NewPath;
+    llvm::sys::fs::unique_file(Path.str(), fd, NewPath);
+    return NewPath.str();
+  }
+};
+} // end anonymous namespace
+
+bool FixItAction::BeginSourceFileAction(CompilerInstance &CI,
+                                        StringRef Filename) {
+  const FrontendOptions &FEOpts = getCompilerInstance().getFrontendOpts();
+  if (!FEOpts.FixItSuffix.empty()) {
+    FixItOpts.reset(new FixItActionSuffixInserter(FEOpts.FixItSuffix,
+                                                  FEOpts.FixWhatYouCan));
+  } else {
+    FixItOpts.reset(new FixItRewriteInPlace);
+    FixItOpts->FixWhatYouCan = FEOpts.FixWhatYouCan;
+  }
+  Rewriter.reset(new FixItRewriter(CI.getDiagnostics(), CI.getSourceManager(),
+                                   CI.getLangOpts(), FixItOpts.get()));
+  return true;
+}
+
+void FixItAction::EndSourceFileAction() {
+  // Otherwise rewrite all files.
+  Rewriter->WriteFixedFiles();
+}
+
+bool FixItRecompile::BeginInvocation(CompilerInstance &CI) {
+
+  std::vector<std::pair<std::string, std::string> > RewrittenFiles;
+  bool err = false;
+  {
+    const FrontendOptions &FEOpts = CI.getFrontendOpts();
+    OwningPtr<FrontendAction> FixAction(new SyntaxOnlyAction());
+    if (FixAction->BeginSourceFile(CI, FEOpts.Inputs[0])) {
+      OwningPtr<FixItOptions> FixItOpts;
+      if (FEOpts.FixToTemporaries)
+        FixItOpts.reset(new FixItRewriteToTemp());
+      else
+        FixItOpts.reset(new FixItRewriteInPlace());
+      FixItOpts->Silent = true;
+      FixItOpts->FixWhatYouCan = FEOpts.FixWhatYouCan;
+      FixItOpts->FixOnlyWarnings = FEOpts.FixOnlyWarnings;
+      FixItRewriter Rewriter(CI.getDiagnostics(), CI.getSourceManager(),
+                             CI.getLangOpts(), FixItOpts.get());
+      FixAction->Execute();
+  
+      err = Rewriter.WriteFixedFiles(&RewrittenFiles);
+    
+      FixAction->EndSourceFile();
+      CI.setSourceManager(0);
+      CI.setFileManager(0);
+    } else {
+      err = true;
+    }
+  }
+  if (err)
+    return false;
+  CI.getDiagnosticClient().clear();
+  CI.getDiagnostics().Reset();
+
+  PreprocessorOptions &PPOpts = CI.getPreprocessorOpts();
+  PPOpts.RemappedFiles.insert(PPOpts.RemappedFiles.end(),
+                              RewrittenFiles.begin(), RewrittenFiles.end());
+  PPOpts.RemappedFilesKeepOriginalName = false;
+
+  return true;
+}
+
+//===----------------------------------------------------------------------===//
+// Preprocessor Actions
+//===----------------------------------------------------------------------===//
+
+ASTConsumer *RewriteObjCAction::CreateASTConsumer(CompilerInstance &CI,
+                                                  StringRef InFile) {
+  if (raw_ostream *OS = CI.createDefaultOutputFile(false, InFile, "cpp")) {
+    if (CI.getLangOpts().ObjCRuntime.isNonFragile())
+      return CreateModernObjCRewriter(InFile, OS,
+                                CI.getDiagnostics(), CI.getLangOpts(),
+                                CI.getDiagnosticOpts().NoRewriteMacros);
+    return CreateObjCRewriter(InFile, OS,
+                              CI.getDiagnostics(), CI.getLangOpts(),
+                              CI.getDiagnosticOpts().NoRewriteMacros);
+  }
+  return 0;
+}
+
+void RewriteMacrosAction::ExecuteAction() {
+  CompilerInstance &CI = getCompilerInstance();
+  raw_ostream *OS = CI.createDefaultOutputFile(true, getCurrentFile());
+  if (!OS) return;
+
+  RewriteMacrosInInput(CI.getPreprocessor(), OS);
+}
+
+void RewriteTestAction::ExecuteAction() {
+  CompilerInstance &CI = getCompilerInstance();
+  raw_ostream *OS = CI.createDefaultOutputFile(false, getCurrentFile());
+  if (!OS) return;
+
+  DoRewriteTest(CI.getPreprocessor(), OS);
+}
+
+void RewriteIncludesAction::ExecuteAction() {
+  CompilerInstance &CI = getCompilerInstance();
+  raw_ostream *OS = CI.createDefaultOutputFile(true, getCurrentFile());
+  if (!OS) return;
+
+  RewriteIncludesInInput(CI.getPreprocessor(), OS,
+                         CI.getPreprocessorOutputOpts());
+}
diff --git a/lib/Rewrite/Frontend/HTMLPrint.cpp b/lib/Rewrite/Frontend/HTMLPrint.cpp
new file mode 100644
index 0000000..79e4447
--- /dev/null
+++ b/lib/Rewrite/Frontend/HTMLPrint.cpp
@@ -0,0 +1,94 @@
+//===--- HTMLPrint.cpp - Source code -> HTML pretty-printing --------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Pretty-printing of source code to HTML.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Rewrite/Frontend/ASTConsumers.h"
+#include "clang/AST/ASTConsumer.h"
+#include "clang/AST/ASTContext.h"
+#include "clang/AST/Decl.h"
+#include "clang/Basic/Diagnostic.h"
+#include "clang/Basic/FileManager.h"
+#include "clang/Basic/SourceManager.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Rewrite/Core/HTMLRewrite.h"
+#include "clang/Rewrite/Core/Rewriter.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/raw_ostream.h"
+using namespace clang;
+
+//===----------------------------------------------------------------------===//
+// Functional HTML pretty-printing.
+//===----------------------------------------------------------------------===//
+
+namespace {
+  class HTMLPrinter : public ASTConsumer {
+    Rewriter R;
+    raw_ostream *Out;
+    Preprocessor &PP;
+    bool SyntaxHighlight, HighlightMacros;
+
+  public:
+    HTMLPrinter(raw_ostream *OS, Preprocessor &pp,
+                bool _SyntaxHighlight, bool _HighlightMacros)
+      : Out(OS), PP(pp), SyntaxHighlight(_SyntaxHighlight),
+        HighlightMacros(_HighlightMacros) {}
+
+    void Initialize(ASTContext &context);
+    void HandleTranslationUnit(ASTContext &Ctx);
+  };
+}
+
+ASTConsumer* clang::CreateHTMLPrinter(raw_ostream *OS,
+                                      Preprocessor &PP,
+                                      bool SyntaxHighlight,
+                                      bool HighlightMacros) {
+  return new HTMLPrinter(OS, PP, SyntaxHighlight, HighlightMacros);
+}
+
+void HTMLPrinter::Initialize(ASTContext &context) {
+  R.setSourceMgr(context.getSourceManager(), context.getLangOpts());
+}
+
+void HTMLPrinter::HandleTranslationUnit(ASTContext &Ctx) {
+  if (PP.getDiagnostics().hasErrorOccurred())
+    return;
+
+  // Format the file.
+  FileID FID = R.getSourceMgr().getMainFileID();
+  const FileEntry* Entry = R.getSourceMgr().getFileEntryForID(FID);
+  const char* Name;
+  // In some cases, in particular the case where the input is from stdin,
+  // there is no entry.  Fall back to the memory buffer for a name in those
+  // cases.
+  if (Entry)
+    Name = Entry->getName();
+  else
+    Name = R.getSourceMgr().getBuffer(FID)->getBufferIdentifier();
+
+  html::AddLineNumbers(R, FID);
+  html::AddHeaderFooterInternalBuiltinCSS(R, FID, Name);
+
+  // If we have a preprocessor, relex the file and syntax highlight.
+  // We might not have a preprocessor if we come from a deserialized AST file,
+  // for example.
+
+  if (SyntaxHighlight) html::SyntaxHighlight(R, FID, PP);
+  if (HighlightMacros) html::HighlightMacros(R, FID, PP);
+  html::EscapeText(R, FID, false, true);
+
+  // Emit the HTML.
+  const RewriteBuffer &RewriteBuf = R.getEditBuffer(FID);
+  char *Buffer = (char*)malloc(RewriteBuf.size());
+  std::copy(RewriteBuf.begin(), RewriteBuf.end(), Buffer);
+  Out->write(Buffer, RewriteBuf.size());
+  free(Buffer);
+}
diff --git a/lib/Rewrite/Frontend/InclusionRewriter.cpp b/lib/Rewrite/Frontend/InclusionRewriter.cpp
new file mode 100644
index 0000000..9d1bec9
--- /dev/null
+++ b/lib/Rewrite/Frontend/InclusionRewriter.cpp
@@ -0,0 +1,363 @@
+//===--- InclusionRewriter.cpp - Rewrite includes into their expansions ---===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This code rewrites include invocations into their expansions.  This gives you
+// a file with all included files merged into it.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Rewrite/Frontend/Rewriters.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Basic/SourceManager.h"
+#include "clang/Frontend/PreprocessorOutputOptions.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace clang;
+using namespace llvm;
+
+namespace {
+
+class InclusionRewriter : public PPCallbacks {
+  /// Information about which #includes were actually performed,
+  /// created by preprocessor callbacks.
+  struct FileChange {
+    SourceLocation From;
+    FileID Id;
+    SrcMgr::CharacteristicKind FileType;
+    FileChange(SourceLocation From) : From(From) {
+    }
+  };
+  Preprocessor &PP; ///< Used to find inclusion directives.
+  SourceManager &SM; ///< Used to read and manage source files.
+  raw_ostream &OS; ///< The destination stream for rewritten contents.
+  bool ShowLineMarkers; ///< Show #line markers.
+  bool UseLineDirective; ///< Use of line directives or line markers.
+  typedef std::map<unsigned, FileChange> FileChangeMap;
+  FileChangeMap FileChanges; /// Tracks which files were included where.
+  /// Used transitively for building up the FileChanges mapping over the
+  /// various \c PPCallbacks callbacks.
+  FileChangeMap::iterator LastInsertedFileChange;
+public:
+  InclusionRewriter(Preprocessor &PP, raw_ostream &OS, bool ShowLineMarkers);
+  bool Process(FileID FileId, SrcMgr::CharacteristicKind FileType);
+private:
+  virtual void FileChanged(SourceLocation Loc, FileChangeReason Reason,
+                           SrcMgr::CharacteristicKind FileType,
+                           FileID PrevFID);
+  virtual void FileSkipped(const FileEntry &ParentFile,
+                           const Token &FilenameTok,
+                           SrcMgr::CharacteristicKind FileType);
+  virtual void InclusionDirective(SourceLocation HashLoc,
+                                  const Token &IncludeTok,
+                                  StringRef FileName,
+                                  bool IsAngled,
+                                  CharSourceRange FilenameRange,
+                                  const FileEntry *File,
+                                  StringRef SearchPath,
+                                  StringRef RelativePath,
+                                  const Module *Imported);
+  void WriteLineInfo(const char *Filename, int Line,
+                     SrcMgr::CharacteristicKind FileType,
+                     StringRef EOL, StringRef Extra = StringRef());
+  void OutputContentUpTo(const MemoryBuffer &FromFile,
+                         unsigned &WriteFrom, unsigned WriteTo,
+                         StringRef EOL, int &lines,
+                         bool EnsureNewline = false);
+  void CommentOutDirective(Lexer &DirectivesLex, const Token &StartToken,
+                           const MemoryBuffer &FromFile, StringRef EOL,
+                           unsigned &NextToWrite, int &Lines);
+  const FileChange *FindFileChangeLocation(SourceLocation Loc) const;
+  StringRef NextIdentifierName(Lexer &RawLex, Token &RawToken);
+};
+
+}  // end anonymous namespace
+
+/// Initializes an InclusionRewriter with a \p PP source and \p OS destination.
+InclusionRewriter::InclusionRewriter(Preprocessor &PP, raw_ostream &OS,
+                                     bool ShowLineMarkers)
+    : PP(PP), SM(PP.getSourceManager()), OS(OS),
+    ShowLineMarkers(ShowLineMarkers),
+    LastInsertedFileChange(FileChanges.end()) {
+  // If we're in microsoft mode, use normal #line instead of line markers.
+  UseLineDirective = PP.getLangOpts().MicrosoftExt;
+}
+
+/// Write appropriate line information as either #line directives or GNU line
+/// markers depending on what mode we're in, including the \p Filename and
+/// \p Line we are located at, using the specified \p EOL line separator, and
+/// any \p Extra context specifiers in GNU line directives.
+void InclusionRewriter::WriteLineInfo(const char *Filename, int Line,
+                                      SrcMgr::CharacteristicKind FileType,
+                                      StringRef EOL, StringRef Extra) {
+  if (!ShowLineMarkers)
+    return;
+  if (UseLineDirective) {
+    OS << "#line" << ' ' << Line << ' ' << '"' << Filename << '"';
+  } else {
+    // Use GNU linemarkers as described here:
+    // http://gcc.gnu.org/onlinedocs/cpp/Preprocessor-Output.html
+    OS << '#' << ' ' << Line << ' ' << '"' << Filename << '"';
+    if (!Extra.empty())
+      OS << Extra;
+    if (FileType == SrcMgr::C_System)
+      // "`3' This indicates that the following text comes from a system header
+      // file, so certain warnings should be suppressed."
+      OS << " 3";
+    else if (FileType == SrcMgr::C_ExternCSystem)
+      // as above for `3', plus "`4' This indicates that the following text
+      // should be treated as being wrapped in an implicit extern "C" block."
+      OS << " 3 4";
+  }
+  OS << EOL;
+}
+
+/// FileChanged - Whenever the preprocessor enters or exits a #include file
+/// it invokes this handler.
+void InclusionRewriter::FileChanged(SourceLocation Loc,
+                                    FileChangeReason Reason,
+                                    SrcMgr::CharacteristicKind NewFileType,
+                                    FileID) {
+  if (Reason != EnterFile)
+    return;
+  if (LastInsertedFileChange == FileChanges.end())
+    // we didn't reach this file (eg: the main file) via an inclusion directive
+    return;
+  LastInsertedFileChange->second.Id = FullSourceLoc(Loc, SM).getFileID();
+  LastInsertedFileChange->second.FileType = NewFileType;
+  LastInsertedFileChange = FileChanges.end();
+}
+
+/// Called whenever an inclusion is skipped due to canonical header protection
+/// macros.
+void InclusionRewriter::FileSkipped(const FileEntry &/*ParentFile*/,
+                                    const Token &/*FilenameTok*/,
+                                    SrcMgr::CharacteristicKind /*FileType*/) {
+  assert(LastInsertedFileChange != FileChanges.end() && "A file, that wasn't "
+    "found via an inclusion directive, was skipped");
+  FileChanges.erase(LastInsertedFileChange);
+  LastInsertedFileChange = FileChanges.end();
+}
+
+/// This should be called whenever the preprocessor encounters include
+/// directives. It does not say whether the file has been included, but it
+/// provides more information about the directive (hash location instead
+/// of location inside the included file). It is assumed that the matching
+/// FileChanged() or FileSkipped() is called after this.
+void InclusionRewriter::InclusionDirective(SourceLocation HashLoc,
+                                           const Token &/*IncludeTok*/,
+                                           StringRef /*FileName*/,
+                                           bool /*IsAngled*/,
+                                           CharSourceRange /*FilenameRange*/,
+                                           const FileEntry * /*File*/,
+                                           StringRef /*SearchPath*/,
+                                           StringRef /*RelativePath*/,
+                                           const Module * /*Imported*/) {
+  assert(LastInsertedFileChange == FileChanges.end() && "Another inclusion "
+    "directive was found before the previous one was processed");
+  std::pair<FileChangeMap::iterator, bool> p = FileChanges.insert(
+    std::make_pair(HashLoc.getRawEncoding(), FileChange(HashLoc)));
+  assert(p.second && "Unexpected revisitation of the same include directive");
+  LastInsertedFileChange = p.first;
+}
+
+/// Simple lookup for a SourceLocation (specifically one denoting the hash in
+/// an inclusion directive) in the map of inclusion information, FileChanges.
+const InclusionRewriter::FileChange *
+InclusionRewriter::FindFileChangeLocation(SourceLocation Loc) const {
+  FileChangeMap::const_iterator I = FileChanges.find(Loc.getRawEncoding());
+  if (I != FileChanges.end())
+    return &I->second;
+  return NULL;
+}
+
+/// Detect the likely line ending style of \p FromFile by examining the first
+/// newline found within it.
+static StringRef DetectEOL(const MemoryBuffer &FromFile) {
+  // detect what line endings the file uses, so that added content does not mix
+  // the style
+  const char *Pos = strchr(FromFile.getBufferStart(), '\n');
+  if (Pos == NULL)
+    return "\n";
+  if (Pos + 1 < FromFile.getBufferEnd() && Pos[1] == '\r')
+    return "\n\r";
+  if (Pos - 1 >= FromFile.getBufferStart() && Pos[-1] == '\r')
+    return "\r\n";
+  return "\n";
+}
+
+/// Writes out bytes from \p FromFile, starting at \p NextToWrite and ending at
+/// \p WriteTo - 1.
+void InclusionRewriter::OutputContentUpTo(const MemoryBuffer &FromFile,
+                                          unsigned &WriteFrom, unsigned WriteTo,
+                                          StringRef EOL, int &Line,
+                                          bool EnsureNewline) {
+  if (WriteTo <= WriteFrom)
+    return;
+  OS.write(FromFile.getBufferStart() + WriteFrom, WriteTo - WriteFrom);
+  // count lines manually, it's faster than getPresumedLoc()
+  Line += std::count(FromFile.getBufferStart() + WriteFrom,
+                     FromFile.getBufferStart() + WriteTo, '\n');
+  if (EnsureNewline) {
+    char LastChar = FromFile.getBufferStart()[WriteTo - 1];
+    if (LastChar != '\n' && LastChar != '\r')
+      OS << EOL;
+  }
+  WriteFrom = WriteTo;
+}
+
+/// Print characters from \p FromFile starting at \p NextToWrite up until the
+/// inclusion directive at \p StartToken, then print out the inclusion
+/// inclusion directive disabled by a #if directive, updating \p NextToWrite
+/// and \p Line to track the number of source lines visited and the progress
+/// through the \p FromFile buffer.
+void InclusionRewriter::CommentOutDirective(Lexer &DirectiveLex,
+                                            const Token &StartToken,
+                                            const MemoryBuffer &FromFile,
+                                            StringRef EOL,
+                                            unsigned &NextToWrite, int &Line) {
+  OutputContentUpTo(FromFile, NextToWrite,
+    SM.getFileOffset(StartToken.getLocation()), EOL, Line);
+  Token DirectiveToken;
+  do {
+    DirectiveLex.LexFromRawLexer(DirectiveToken);
+  } while (!DirectiveToken.is(tok::eod) && DirectiveToken.isNot(tok::eof));
+  OS << "#if 0 /* expanded by -frewrite-includes */" << EOL;
+  OutputContentUpTo(FromFile, NextToWrite,
+    SM.getFileOffset(DirectiveToken.getLocation()) + DirectiveToken.getLength(),
+    EOL, Line);
+  OS << "#endif /* expanded by -frewrite-includes */" << EOL;
+}
+
+/// Find the next identifier in the pragma directive specified by \p RawToken.
+StringRef InclusionRewriter::NextIdentifierName(Lexer &RawLex,
+                                                Token &RawToken) {
+  RawLex.LexFromRawLexer(RawToken);
+  if (RawToken.is(tok::raw_identifier))
+    PP.LookUpIdentifierInfo(RawToken);
+  if (RawToken.is(tok::identifier))
+    return RawToken.getIdentifierInfo()->getName();
+  return StringRef();
+}
+
+/// Use a raw lexer to analyze \p FileId, inccrementally copying parts of it
+/// and including content of included files recursively.
+bool InclusionRewriter::Process(FileID FileId,
+                                SrcMgr::CharacteristicKind FileType)
+{
+  bool Invalid;
+  const MemoryBuffer &FromFile = *SM.getBuffer(FileId, &Invalid);
+  if (Invalid) // invalid inclusion
+    return true;
+  const char *FileName = FromFile.getBufferIdentifier();
+  Lexer RawLex(FileId, &FromFile, PP.getSourceManager(), PP.getLangOpts());
+  RawLex.SetCommentRetentionState(false);
+
+  StringRef EOL = DetectEOL(FromFile);
+
+  // Per the GNU docs: "1" indicates the start of a new file.
+  WriteLineInfo(FileName, 1, FileType, EOL, " 1");
+
+  if (SM.getFileIDSize(FileId) == 0)
+    return true;
+
+  // The next byte to be copied from the source file
+  unsigned NextToWrite = 0;
+  int Line = 1; // The current input file line number.
+
+  Token RawToken;
+  RawLex.LexFromRawLexer(RawToken);
+
+  // TODO: Consider adding a switch that strips possibly unimportant content,
+  // such as comments, to reduce the size of repro files.
+  while (RawToken.isNot(tok::eof)) {
+    if (RawToken.is(tok::hash) && RawToken.isAtStartOfLine()) {
+      RawLex.setParsingPreprocessorDirective(true);
+      Token HashToken = RawToken;
+      RawLex.LexFromRawLexer(RawToken);
+      if (RawToken.is(tok::raw_identifier))
+        PP.LookUpIdentifierInfo(RawToken);
+      if (RawToken.is(tok::identifier)) {
+        switch (RawToken.getIdentifierInfo()->getPPKeywordID()) {
+          case tok::pp_include:
+          case tok::pp_include_next:
+          case tok::pp_import: {
+            CommentOutDirective(RawLex, HashToken, FromFile, EOL, NextToWrite,
+              Line);
+            if (const FileChange *Change = FindFileChangeLocation(
+                HashToken.getLocation())) {
+              // now include and recursively process the file
+              if (Process(Change->Id, Change->FileType))
+                // and set lineinfo back to this file, if the nested one was
+                // actually included
+                // `2' indicates returning to a file (after having included
+                // another file.
+                WriteLineInfo(FileName, Line, FileType, EOL, " 2");
+            } else
+              // fix up lineinfo (since commented out directive changed line
+              // numbers) for inclusions that were skipped due to header guards
+              WriteLineInfo(FileName, Line, FileType, EOL);
+            break;
+          }
+          case tok::pp_pragma: {
+            StringRef Identifier = NextIdentifierName(RawLex, RawToken);
+            if (Identifier == "clang" || Identifier == "GCC") {
+              if (NextIdentifierName(RawLex, RawToken) == "system_header") {
+                // keep the directive in, commented out
+                CommentOutDirective(RawLex, HashToken, FromFile, EOL,
+                  NextToWrite, Line);
+                // update our own type
+                FileType = SM.getFileCharacteristic(RawToken.getLocation());
+                WriteLineInfo(FileName, Line, FileType, EOL);
+              }
+            } else if (Identifier == "once") {
+              // keep the directive in, commented out
+              CommentOutDirective(RawLex, HashToken, FromFile, EOL,
+                NextToWrite, Line);
+              WriteLineInfo(FileName, Line, FileType, EOL);
+            }
+            break;
+          }
+          default:
+            break;
+        }
+      }
+      RawLex.setParsingPreprocessorDirective(false);
+    }
+    RawLex.LexFromRawLexer(RawToken);
+  }
+  OutputContentUpTo(FromFile, NextToWrite,
+    SM.getFileOffset(SM.getLocForEndOfFile(FileId)) + 1, EOL, Line,
+    /*EnsureNewline*/true);
+  return true;
+}
+
+/// InclusionRewriterInInput - Implement -frewrite-includes mode.
+void clang::RewriteIncludesInInput(Preprocessor &PP, raw_ostream *OS,
+                                   const PreprocessorOutputOptions &Opts) {
+  SourceManager &SM = PP.getSourceManager();
+  InclusionRewriter *Rewrite = new InclusionRewriter(PP, *OS,
+                                                     Opts.ShowLineMarkers);
+  PP.addPPCallbacks(Rewrite);
+
+  // First let the preprocessor process the entire file and call callbacks.
+  // Callbacks will record which #include's were actually performed.
+  PP.EnterMainSourceFile();
+  Token Tok;
+  // Only preprocessor directives matter here, so disable macro expansion
+  // everywhere else as an optimization.
+  // TODO: It would be even faster if the preprocessor could be switched
+  // to a mode where it would parse only preprocessor directives and comments,
+  // nothing else matters for parsing or processing.
+  PP.SetMacroExpansionOnlyInDirectives();
+  do {
+    PP.Lex(Tok);
+  } while (Tok.isNot(tok::eof));
+  Rewrite->Process(SM.getMainFileID(), SrcMgr::C_User);
+  OS->flush();
+}
diff --git a/lib/Rewrite/Frontend/Makefile b/lib/Rewrite/Frontend/Makefile
new file mode 100644
index 0000000..ac97d40
--- /dev/null
+++ b/lib/Rewrite/Frontend/Makefile
@@ -0,0 +1,18 @@
+##===- clang/lib/Rewrite/Makefile --------------------------*- Makefile -*-===##
+# 
+#                     The LLVM Compiler Infrastructure
+#
+# This file is distributed under the University of Illinois Open Source
+# License. See LICENSE.TXT for details.
+# 
+##===----------------------------------------------------------------------===##
+#
+# This implements code transformation / rewriting facilities.
+#
+##===----------------------------------------------------------------------===##
+
+CLANG_LEVEL := ../../..
+LIBRARYNAME := clangRewriteFrontend
+
+include $(CLANG_LEVEL)/Makefile
+
diff --git a/lib/Rewrite/Frontend/RewriteMacros.cpp b/lib/Rewrite/Frontend/RewriteMacros.cpp
new file mode 100644
index 0000000..f399dd5
--- /dev/null
+++ b/lib/Rewrite/Frontend/RewriteMacros.cpp
@@ -0,0 +1,217 @@
+//===--- RewriteMacros.cpp - Rewrite macros into their expansions ---------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This code rewrites macro invocations into their expansions.  This gives you
+// a macro expanded file that retains comments and #includes.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Rewrite/Frontend/Rewriters.h"
+#include "clang/Rewrite/Core/Rewriter.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Basic/SourceManager.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/Path.h"
+#include "llvm/ADT/OwningPtr.h"
+#include <cstdio>
+
+using namespace clang;
+
+/// isSameToken - Return true if the two specified tokens start have the same
+/// content.
+static bool isSameToken(Token &RawTok, Token &PPTok) {
+  // If two tokens have the same kind and the same identifier info, they are
+  // obviously the same.
+  if (PPTok.getKind() == RawTok.getKind() &&
+      PPTok.getIdentifierInfo() == RawTok.getIdentifierInfo())
+    return true;
+
+  // Otherwise, if they are different but have the same identifier info, they
+  // are also considered to be the same.  This allows keywords and raw lexed
+  // identifiers with the same name to be treated the same.
+  if (PPTok.getIdentifierInfo() &&
+      PPTok.getIdentifierInfo() == RawTok.getIdentifierInfo())
+    return true;
+
+  return false;
+}
+
+
+/// GetNextRawTok - Return the next raw token in the stream, skipping over
+/// comments if ReturnComment is false.
+static const Token &GetNextRawTok(const std::vector<Token> &RawTokens,
+                                  unsigned &CurTok, bool ReturnComment) {
+  assert(CurTok < RawTokens.size() && "Overran eof!");
+
+  // If the client doesn't want comments and we have one, skip it.
+  if (!ReturnComment && RawTokens[CurTok].is(tok::comment))
+    ++CurTok;
+
+  return RawTokens[CurTok++];
+}
+
+
+/// LexRawTokensFromMainFile - Lets all the raw tokens from the main file into
+/// the specified vector.
+static void LexRawTokensFromMainFile(Preprocessor &PP,
+                                     std::vector<Token> &RawTokens) {
+  SourceManager &SM = PP.getSourceManager();
+
+  // Create a lexer to lex all the tokens of the main file in raw mode.  Even
+  // though it is in raw mode, it will not return comments.
+  const llvm::MemoryBuffer *FromFile = SM.getBuffer(SM.getMainFileID());
+  Lexer RawLex(SM.getMainFileID(), FromFile, SM, PP.getLangOpts());
+
+  // Switch on comment lexing because we really do want them.
+  RawLex.SetCommentRetentionState(true);
+
+  Token RawTok;
+  do {
+    RawLex.LexFromRawLexer(RawTok);
+
+    // If we have an identifier with no identifier info for our raw token, look
+    // up the indentifier info.  This is important for equality comparison of
+    // identifier tokens.
+    if (RawTok.is(tok::raw_identifier))
+      PP.LookUpIdentifierInfo(RawTok);
+
+    RawTokens.push_back(RawTok);
+  } while (RawTok.isNot(tok::eof));
+}
+
+
+/// RewriteMacrosInInput - Implement -rewrite-macros mode.
+void clang::RewriteMacrosInInput(Preprocessor &PP, raw_ostream *OS) {
+  SourceManager &SM = PP.getSourceManager();
+
+  Rewriter Rewrite;
+  Rewrite.setSourceMgr(SM, PP.getLangOpts());
+  RewriteBuffer &RB = Rewrite.getEditBuffer(SM.getMainFileID());
+
+  std::vector<Token> RawTokens;
+  LexRawTokensFromMainFile(PP, RawTokens);
+  unsigned CurRawTok = 0;
+  Token RawTok = GetNextRawTok(RawTokens, CurRawTok, false);
+
+
+  // Get the first preprocessing token.
+  PP.EnterMainSourceFile();
+  Token PPTok;
+  PP.Lex(PPTok);
+
+  // Preprocess the input file in parallel with raw lexing the main file. Ignore
+  // all tokens that are preprocessed from a file other than the main file (e.g.
+  // a header).  If we see tokens that are in the preprocessed file but not the
+  // lexed file, we have a macro expansion.  If we see tokens in the lexed file
+  // that aren't in the preprocessed view, we have macros that expand to no
+  // tokens, or macro arguments etc.
+  while (RawTok.isNot(tok::eof) || PPTok.isNot(tok::eof)) {
+    SourceLocation PPLoc = SM.getExpansionLoc(PPTok.getLocation());
+
+    // If PPTok is from a different source file, ignore it.
+    if (!SM.isFromMainFile(PPLoc)) {
+      PP.Lex(PPTok);
+      continue;
+    }
+
+    // If the raw file hits a preprocessor directive, they will be extra tokens
+    // in the raw file that don't exist in the preprocsesed file.  However, we
+    // choose to preserve them in the output file and otherwise handle them
+    // specially.
+    if (RawTok.is(tok::hash) && RawTok.isAtStartOfLine()) {
+      // If this is a #warning directive or #pragma mark (GNU extensions),
+      // comment the line out.
+      if (RawTokens[CurRawTok].is(tok::identifier)) {
+        const IdentifierInfo *II = RawTokens[CurRawTok].getIdentifierInfo();
+        if (II->getName() == "warning") {
+          // Comment out #warning.
+          RB.InsertTextAfter(SM.getFileOffset(RawTok.getLocation()), "//");
+        } else if (II->getName() == "pragma" &&
+                   RawTokens[CurRawTok+1].is(tok::identifier) &&
+                   (RawTokens[CurRawTok+1].getIdentifierInfo()->getName() ==
+                    "mark")) {
+          // Comment out #pragma mark.
+          RB.InsertTextAfter(SM.getFileOffset(RawTok.getLocation()), "//");
+        }
+      }
+
+      // Otherwise, if this is a #include or some other directive, just leave it
+      // in the file by skipping over the line.
+      RawTok = GetNextRawTok(RawTokens, CurRawTok, false);
+      while (!RawTok.isAtStartOfLine() && RawTok.isNot(tok::eof))
+        RawTok = GetNextRawTok(RawTokens, CurRawTok, false);
+      continue;
+    }
+
+    // Okay, both tokens are from the same file.  Get their offsets from the
+    // start of the file.
+    unsigned PPOffs = SM.getFileOffset(PPLoc);
+    unsigned RawOffs = SM.getFileOffset(RawTok.getLocation());
+
+    // If the offsets are the same and the token kind is the same, ignore them.
+    if (PPOffs == RawOffs && isSameToken(RawTok, PPTok)) {
+      RawTok = GetNextRawTok(RawTokens, CurRawTok, false);
+      PP.Lex(PPTok);
+      continue;
+    }
+
+    // If the PP token is farther along than the raw token, something was
+    // deleted.  Comment out the raw token.
+    if (RawOffs <= PPOffs) {
+      // Comment out a whole run of tokens instead of bracketing each one with
+      // comments.  Add a leading space if RawTok didn't have one.
+      bool HasSpace = RawTok.hasLeadingSpace();
+      RB.InsertTextAfter(RawOffs, &" /*"[HasSpace]);
+      unsigned EndPos;
+
+      do {
+        EndPos = RawOffs+RawTok.getLength();
+
+        RawTok = GetNextRawTok(RawTokens, CurRawTok, true);
+        RawOffs = SM.getFileOffset(RawTok.getLocation());
+
+        if (RawTok.is(tok::comment)) {
+          // Skip past the comment.
+          RawTok = GetNextRawTok(RawTokens, CurRawTok, false);
+          break;
+        }
+
+      } while (RawOffs <= PPOffs && !RawTok.isAtStartOfLine() &&
+               (PPOffs != RawOffs || !isSameToken(RawTok, PPTok)));
+
+      RB.InsertTextBefore(EndPos, "*/");
+      continue;
+    }
+
+    // Otherwise, there was a replacement an expansion.  Insert the new token
+    // in the output buffer.  Insert the whole run of new tokens at once to get
+    // them in the right order.
+    unsigned InsertPos = PPOffs;
+    std::string Expansion;
+    while (PPOffs < RawOffs) {
+      Expansion += ' ' + PP.getSpelling(PPTok);
+      PP.Lex(PPTok);
+      PPLoc = SM.getExpansionLoc(PPTok.getLocation());
+      PPOffs = SM.getFileOffset(PPLoc);
+    }
+    Expansion += ' ';
+    RB.InsertTextBefore(InsertPos, Expansion);
+  }
+
+  // Get the buffer corresponding to MainFileID.  If we haven't changed it, then
+  // we are done.
+  if (const RewriteBuffer *RewriteBuf =
+      Rewrite.getRewriteBufferFor(SM.getMainFileID())) {
+    //printf("Changed:\n");
+    *OS << std::string(RewriteBuf->begin(), RewriteBuf->end());
+  } else {
+    fprintf(stderr, "No changes\n");
+  }
+  OS->flush();
+}
diff --git a/lib/Rewrite/Frontend/RewriteModernObjC.cpp b/lib/Rewrite/Frontend/RewriteModernObjC.cpp
new file mode 100644
index 0000000..4b56b37
--- /dev/null
+++ b/lib/Rewrite/Frontend/RewriteModernObjC.cpp
@@ -0,0 +1,7607 @@
+//===--- RewriteObjC.cpp - Playground for the code rewriter ---------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Hacks and fun related to the code rewriter.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Rewrite/Frontend/ASTConsumers.h"
+#include "clang/Rewrite/Core/Rewriter.h"
+#include "clang/AST/AST.h"
+#include "clang/AST/ASTConsumer.h"
+#include "clang/AST/ParentMap.h"
+#include "clang/Basic/SourceManager.h"
+#include "clang/Basic/IdentifierTable.h"
+#include "clang/Basic/Diagnostic.h"
+#include "clang/Lex/Lexer.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/OwningPtr.h"
+#include "llvm/ADT/DenseSet.h"
+
+using namespace clang;
+using llvm::utostr;
+
+namespace {
+  class RewriteModernObjC : public ASTConsumer {
+  protected:
+    
+    enum {
+      BLOCK_FIELD_IS_OBJECT   =  3,  /* id, NSObject, __attribute__((NSObject)),
+                                        block, ... */
+      BLOCK_FIELD_IS_BLOCK    =  7,  /* a block variable */
+      BLOCK_FIELD_IS_BYREF    =  8,  /* the on stack structure holding the 
+                                        __block variable */
+      BLOCK_FIELD_IS_WEAK     = 16,  /* declared __weak, only used in byref copy
+                                        helpers */
+      BLOCK_BYREF_CALLER      = 128, /* called from __block (byref) copy/dispose
+                                        support routines */
+      BLOCK_BYREF_CURRENT_MAX = 256
+    };
+    
+    enum {
+      BLOCK_NEEDS_FREE =        (1 << 24),
+      BLOCK_HAS_COPY_DISPOSE =  (1 << 25),
+      BLOCK_HAS_CXX_OBJ =       (1 << 26),
+      BLOCK_IS_GC =             (1 << 27),
+      BLOCK_IS_GLOBAL =         (1 << 28),
+      BLOCK_HAS_DESCRIPTOR =    (1 << 29)
+    };
+    static const int OBJC_ABI_VERSION = 7;
+    
+    Rewriter Rewrite;
+    DiagnosticsEngine &Diags;
+    const LangOptions &LangOpts;
+    ASTContext *Context;
+    SourceManager *SM;
+    TranslationUnitDecl *TUDecl;
+    FileID MainFileID;
+    const char *MainFileStart, *MainFileEnd;
+    Stmt *CurrentBody;
+    ParentMap *PropParentMap; // created lazily.
+    std::string InFileName;
+    raw_ostream* OutFile;
+    std::string Preamble;
+    
+    TypeDecl *ProtocolTypeDecl;
+    VarDecl *GlobalVarDecl;
+    Expr *GlobalConstructionExp;
+    unsigned RewriteFailedDiag;
+    unsigned GlobalBlockRewriteFailedDiag;
+    // ObjC string constant support.
+    unsigned NumObjCStringLiterals;
+    VarDecl *ConstantStringClassReference;
+    RecordDecl *NSStringRecord;
+
+    // ObjC foreach break/continue generation support.
+    int BcLabelCount;
+    
+    unsigned TryFinallyContainsReturnDiag;
+    // Needed for super.
+    ObjCMethodDecl *CurMethodDef;
+    RecordDecl *SuperStructDecl;
+    RecordDecl *ConstantStringDecl;
+    
+    FunctionDecl *MsgSendFunctionDecl;
+    FunctionDecl *MsgSendSuperFunctionDecl;
+    FunctionDecl *MsgSendStretFunctionDecl;
+    FunctionDecl *MsgSendSuperStretFunctionDecl;
+    FunctionDecl *MsgSendFpretFunctionDecl;
+    FunctionDecl *GetClassFunctionDecl;
+    FunctionDecl *GetMetaClassFunctionDecl;
+    FunctionDecl *GetSuperClassFunctionDecl;
+    FunctionDecl *SelGetUidFunctionDecl;
+    FunctionDecl *CFStringFunctionDecl;
+    FunctionDecl *SuperContructorFunctionDecl;
+    FunctionDecl *CurFunctionDef;
+
+    /* Misc. containers needed for meta-data rewrite. */
+    SmallVector<ObjCImplementationDecl *, 8> ClassImplementation;
+    SmallVector<ObjCCategoryImplDecl *, 8> CategoryImplementation;
+    llvm::SmallPtrSet<ObjCInterfaceDecl*, 8> ObjCSynthesizedStructs;
+    llvm::SmallPtrSet<ObjCProtocolDecl*, 8> ObjCSynthesizedProtocols;
+    llvm::SmallPtrSet<ObjCInterfaceDecl*, 8> ObjCWrittenInterfaces;
+    llvm::SmallPtrSet<TagDecl*, 32> GlobalDefinedTags;
+    SmallVector<ObjCInterfaceDecl*, 32> ObjCInterfacesSeen;
+    /// DefinedNonLazyClasses - List of defined "non-lazy" classes.
+    SmallVector<ObjCInterfaceDecl*, 8> DefinedNonLazyClasses;
+    
+    /// DefinedNonLazyCategories - List of defined "non-lazy" categories.
+    llvm::SmallVector<ObjCCategoryDecl*, 8> DefinedNonLazyCategories;
+    
+    SmallVector<Stmt *, 32> Stmts;
+    SmallVector<int, 8> ObjCBcLabelNo;
+    // Remember all the @protocol(<expr>) expressions.
+    llvm::SmallPtrSet<ObjCProtocolDecl *, 32> ProtocolExprDecls;
+    
+    llvm::DenseSet<uint64_t> CopyDestroyCache;
+
+    // Block expressions.
+    SmallVector<BlockExpr *, 32> Blocks;
+    SmallVector<int, 32> InnerDeclRefsCount;
+    SmallVector<DeclRefExpr *, 32> InnerDeclRefs;
+    
+    SmallVector<DeclRefExpr *, 32> BlockDeclRefs;
+
+    // Block related declarations.
+    SmallVector<ValueDecl *, 8> BlockByCopyDecls;
+    llvm::SmallPtrSet<ValueDecl *, 8> BlockByCopyDeclsPtrSet;
+    SmallVector<ValueDecl *, 8> BlockByRefDecls;
+    llvm::SmallPtrSet<ValueDecl *, 8> BlockByRefDeclsPtrSet;
+    llvm::DenseMap<ValueDecl *, unsigned> BlockByRefDeclNo;
+    llvm::SmallPtrSet<ValueDecl *, 8> ImportedBlockDecls;
+    llvm::SmallPtrSet<VarDecl *, 8> ImportedLocalExternalDecls;
+    
+    llvm::DenseMap<BlockExpr *, std::string> RewrittenBlockExprs;
+    llvm::DenseMap<ObjCInterfaceDecl *, 
+                    llvm::SmallPtrSet<ObjCIvarDecl *, 8> > ReferencedIvars;
+    
+    // This maps an original source AST to it's rewritten form. This allows
+    // us to avoid rewriting the same node twice (which is very uncommon).
+    // This is needed to support some of the exotic property rewriting.
+    llvm::DenseMap<Stmt *, Stmt *> ReplacedNodes;
+
+    // Needed for header files being rewritten
+    bool IsHeader;
+    bool SilenceRewriteMacroWarning;
+    bool objc_impl_method;
+    
+    bool DisableReplaceStmt;
+    class DisableReplaceStmtScope {
+      RewriteModernObjC &R;
+      bool SavedValue;
+    
+    public:
+      DisableReplaceStmtScope(RewriteModernObjC &R)
+        : R(R), SavedValue(R.DisableReplaceStmt) {
+        R.DisableReplaceStmt = true;
+      }
+      ~DisableReplaceStmtScope() {
+        R.DisableReplaceStmt = SavedValue;
+      }
+    };
+    void InitializeCommon(ASTContext &context);
+
+  public:
+    llvm::DenseMap<ObjCMethodDecl*, std::string> MethodInternalNames;
+    // Top Level Driver code.
+    virtual bool HandleTopLevelDecl(DeclGroupRef D) {
+      for (DeclGroupRef::iterator I = D.begin(), E = D.end(); I != E; ++I) {
+        if (ObjCInterfaceDecl *Class = dyn_cast<ObjCInterfaceDecl>(*I)) {
+          if (!Class->isThisDeclarationADefinition()) {
+            RewriteForwardClassDecl(D);
+            break;
+          } else {
+            // Keep track of all interface declarations seen.
+            ObjCInterfacesSeen.push_back(Class);
+            break;
+          }
+        }
+
+        if (ObjCProtocolDecl *Proto = dyn_cast<ObjCProtocolDecl>(*I)) {
+          if (!Proto->isThisDeclarationADefinition()) {
+            RewriteForwardProtocolDecl(D);
+            break;
+          }
+        }
+
+        HandleTopLevelSingleDecl(*I);
+      }
+      return true;
+    }
+    void HandleTopLevelSingleDecl(Decl *D);
+    void HandleDeclInMainFile(Decl *D);
+    RewriteModernObjC(std::string inFile, raw_ostream *OS,
+                DiagnosticsEngine &D, const LangOptions &LOpts,
+                bool silenceMacroWarn);
+    
+    ~RewriteModernObjC() {}
+    
+    virtual void HandleTranslationUnit(ASTContext &C);
+
+    void ReplaceStmt(Stmt *Old, Stmt *New) {
+      Stmt *ReplacingStmt = ReplacedNodes[Old];
+
+      if (ReplacingStmt)
+        return; // We can't rewrite the same node twice.
+
+      if (DisableReplaceStmt)
+        return;
+
+      // If replacement succeeded or warning disabled return with no warning.
+      if (!Rewrite.ReplaceStmt(Old, New)) {
+        ReplacedNodes[Old] = New;
+        return;
+      }
+      if (SilenceRewriteMacroWarning)
+        return;
+      Diags.Report(Context->getFullLoc(Old->getLocStart()), RewriteFailedDiag)
+                   << Old->getSourceRange();
+    }
+
+    void ReplaceStmtWithRange(Stmt *Old, Stmt *New, SourceRange SrcRange) {
+      if (DisableReplaceStmt)
+        return;
+
+      // Measure the old text.
+      int Size = Rewrite.getRangeSize(SrcRange);
+      if (Size == -1) {
+        Diags.Report(Context->getFullLoc(Old->getLocStart()), RewriteFailedDiag)
+                     << Old->getSourceRange();
+        return;
+      }
+      // Get the new text.
+      std::string SStr;
+      llvm::raw_string_ostream S(SStr);
+      New->printPretty(S, 0, PrintingPolicy(LangOpts));
+      const std::string &Str = S.str();
+
+      // If replacement succeeded or warning disabled return with no warning.
+      if (!Rewrite.ReplaceText(SrcRange.getBegin(), Size, Str)) {
+        ReplacedNodes[Old] = New;
+        return;
+      }
+      if (SilenceRewriteMacroWarning)
+        return;
+      Diags.Report(Context->getFullLoc(Old->getLocStart()), RewriteFailedDiag)
+                   << Old->getSourceRange();
+    }
+
+    void InsertText(SourceLocation Loc, StringRef Str,
+                    bool InsertAfter = true) {
+      // If insertion succeeded or warning disabled return with no warning.
+      if (!Rewrite.InsertText(Loc, Str, InsertAfter) ||
+          SilenceRewriteMacroWarning)
+        return;
+
+      Diags.Report(Context->getFullLoc(Loc), RewriteFailedDiag);
+    }
+
+    void ReplaceText(SourceLocation Start, unsigned OrigLength,
+                     StringRef Str) {
+      // If removal succeeded or warning disabled return with no warning.
+      if (!Rewrite.ReplaceText(Start, OrigLength, Str) ||
+          SilenceRewriteMacroWarning)
+        return;
+
+      Diags.Report(Context->getFullLoc(Start), RewriteFailedDiag);
+    }
+
+    // Syntactic Rewriting.
+    void RewriteRecordBody(RecordDecl *RD);
+    void RewriteInclude();
+    void RewriteLineDirective(const Decl *D);
+    void ConvertSourceLocationToLineDirective(SourceLocation Loc,
+                                              std::string &LineString);
+    void RewriteForwardClassDecl(DeclGroupRef D);
+    void RewriteForwardClassDecl(const llvm::SmallVector<Decl*, 8> &DG);
+    void RewriteForwardClassEpilogue(ObjCInterfaceDecl *ClassDecl, 
+                                     const std::string &typedefString);
+    void RewriteImplementations();
+    void RewritePropertyImplDecl(ObjCPropertyImplDecl *PID,
+                                 ObjCImplementationDecl *IMD,
+                                 ObjCCategoryImplDecl *CID);
+    void RewriteInterfaceDecl(ObjCInterfaceDecl *Dcl);
+    void RewriteImplementationDecl(Decl *Dcl);
+    void RewriteObjCMethodDecl(const ObjCInterfaceDecl *IDecl,
+                               ObjCMethodDecl *MDecl, std::string &ResultStr);
+    void RewriteTypeIntoString(QualType T, std::string &ResultStr,
+                               const FunctionType *&FPRetType);
+    void RewriteByRefString(std::string &ResultStr, const std::string &Name,
+                            ValueDecl *VD, bool def=false);
+    void RewriteCategoryDecl(ObjCCategoryDecl *Dcl);
+    void RewriteProtocolDecl(ObjCProtocolDecl *Dcl);
+    void RewriteForwardProtocolDecl(DeclGroupRef D);
+    void RewriteForwardProtocolDecl(const llvm::SmallVector<Decl*, 8> &DG);
+    void RewriteMethodDeclaration(ObjCMethodDecl *Method);
+    void RewriteProperty(ObjCPropertyDecl *prop);
+    void RewriteFunctionDecl(FunctionDecl *FD);
+    void RewriteBlockPointerType(std::string& Str, QualType Type);
+    void RewriteBlockPointerTypeVariable(std::string& Str, ValueDecl *VD);
+    void RewriteBlockLiteralFunctionDecl(FunctionDecl *FD);
+    void RewriteObjCQualifiedInterfaceTypes(Decl *Dcl);
+    void RewriteTypeOfDecl(VarDecl *VD);
+    void RewriteObjCQualifiedInterfaceTypes(Expr *E);
+    
+    std::string getIvarAccessString(ObjCIvarDecl *D);
+  
+    // Expression Rewriting.
+    Stmt *RewriteFunctionBodyOrGlobalInitializer(Stmt *S);
+    Stmt *RewriteAtEncode(ObjCEncodeExpr *Exp);
+    Stmt *RewritePropertyOrImplicitGetter(PseudoObjectExpr *Pseudo);
+    Stmt *RewritePropertyOrImplicitSetter(PseudoObjectExpr *Pseudo);
+    Stmt *RewriteAtSelector(ObjCSelectorExpr *Exp);
+    Stmt *RewriteMessageExpr(ObjCMessageExpr *Exp);
+    Stmt *RewriteObjCStringLiteral(ObjCStringLiteral *Exp);
+    Stmt *RewriteObjCBoolLiteralExpr(ObjCBoolLiteralExpr *Exp);
+    Stmt *RewriteObjCBoxedExpr(ObjCBoxedExpr *Exp);
+    Stmt *RewriteObjCArrayLiteralExpr(ObjCArrayLiteral *Exp);
+    Stmt *RewriteObjCDictionaryLiteralExpr(ObjCDictionaryLiteral *Exp);
+    Stmt *RewriteObjCProtocolExpr(ObjCProtocolExpr *Exp);
+    Stmt *RewriteObjCTryStmt(ObjCAtTryStmt *S);
+    Stmt *RewriteObjCAutoreleasePoolStmt(ObjCAutoreleasePoolStmt  *S);
+    Stmt *RewriteObjCSynchronizedStmt(ObjCAtSynchronizedStmt *S);
+    Stmt *RewriteObjCThrowStmt(ObjCAtThrowStmt *S);
+    Stmt *RewriteObjCForCollectionStmt(ObjCForCollectionStmt *S,
+                                       SourceLocation OrigEnd);
+    Stmt *RewriteBreakStmt(BreakStmt *S);
+    Stmt *RewriteContinueStmt(ContinueStmt *S);
+    void RewriteCastExpr(CStyleCastExpr *CE);
+    void RewriteImplicitCastObjCExpr(CastExpr *IE);
+    void RewriteLinkageSpec(LinkageSpecDecl *LSD);
+    
+    // Block rewriting.
+    void RewriteBlocksInFunctionProtoType(QualType funcType, NamedDecl *D);
+    
+    // Block specific rewrite rules.
+    void RewriteBlockPointerDecl(NamedDecl *VD);
+    void RewriteByRefVar(VarDecl *VD, bool firstDecl, bool lastDecl);
+    Stmt *RewriteBlockDeclRefExpr(DeclRefExpr *VD);
+    Stmt *RewriteLocalVariableExternalStorage(DeclRefExpr *DRE);
+    void RewriteBlockPointerFunctionArgs(FunctionDecl *FD);
+    
+    void RewriteObjCInternalStruct(ObjCInterfaceDecl *CDecl,
+                                      std::string &Result);
+    
+    void RewriteObjCFieldDecl(FieldDecl *fieldDecl, std::string &Result);
+    bool IsTagDefinedInsideClass(ObjCContainerDecl *IDecl, TagDecl *Tag,
+                                 bool &IsNamedDefinition);
+    void RewriteLocallyDefinedNamedAggregates(FieldDecl *fieldDecl, 
+                                              std::string &Result);
+    
+    bool RewriteObjCFieldDeclType(QualType &Type, std::string &Result);
+    
+    void RewriteIvarOffsetSymbols(ObjCInterfaceDecl *CDecl,
+                                  std::string &Result);
+    
+    virtual void Initialize(ASTContext &context);
+    
+    // Misc. AST transformation routines. Sometimes they end up calling
+    // rewriting routines on the new ASTs.
+    CallExpr *SynthesizeCallToFunctionDecl(FunctionDecl *FD,
+                                           Expr **args, unsigned nargs,
+                                           SourceLocation StartLoc=SourceLocation(),
+                                           SourceLocation EndLoc=SourceLocation());
+    
+    Expr *SynthMsgSendStretCallExpr(FunctionDecl *MsgSendStretFlavor,
+                                        QualType msgSendType, 
+                                        QualType returnType, 
+                                        SmallVectorImpl<QualType> &ArgTypes,
+                                        SmallVectorImpl<Expr*> &MsgExprs,
+                                        ObjCMethodDecl *Method);
+
+    Stmt *SynthMessageExpr(ObjCMessageExpr *Exp,
+                           SourceLocation StartLoc=SourceLocation(),
+                           SourceLocation EndLoc=SourceLocation());
+    
+    void SynthCountByEnumWithState(std::string &buf);
+    void SynthMsgSendFunctionDecl();
+    void SynthMsgSendSuperFunctionDecl();
+    void SynthMsgSendStretFunctionDecl();
+    void SynthMsgSendFpretFunctionDecl();
+    void SynthMsgSendSuperStretFunctionDecl();
+    void SynthGetClassFunctionDecl();
+    void SynthGetMetaClassFunctionDecl();
+    void SynthGetSuperClassFunctionDecl();
+    void SynthSelGetUidFunctionDecl();
+    void SynthSuperContructorFunctionDecl();
+    
+    // Rewriting metadata
+    template<typename MethodIterator>
+    void RewriteObjCMethodsMetaData(MethodIterator MethodBegin,
+                                    MethodIterator MethodEnd,
+                                    bool IsInstanceMethod,
+                                    StringRef prefix,
+                                    StringRef ClassName,
+                                    std::string &Result);
+    void RewriteObjCProtocolMetaData(ObjCProtocolDecl *Protocol,
+                                     std::string &Result);
+    void RewriteObjCProtocolListMetaData(
+                   const ObjCList<ObjCProtocolDecl> &Prots,
+                   StringRef prefix, StringRef ClassName, std::string &Result);
+    void RewriteObjCClassMetaData(ObjCImplementationDecl *IDecl,
+                                          std::string &Result);
+    void RewriteClassSetupInitHook(std::string &Result);
+    
+    void RewriteMetaDataIntoBuffer(std::string &Result);
+    void WriteImageInfo(std::string &Result);
+    void RewriteObjCCategoryImplDecl(ObjCCategoryImplDecl *CDecl,
+                                             std::string &Result);
+    void RewriteCategorySetupInitHook(std::string &Result);
+    
+    // Rewriting ivar
+    void RewriteIvarOffsetComputation(ObjCIvarDecl *ivar,
+                                              std::string &Result);
+    Stmt *RewriteObjCIvarRefExpr(ObjCIvarRefExpr *IV);
+
+    
+    std::string SynthesizeByrefCopyDestroyHelper(VarDecl *VD, int flag);
+    std::string SynthesizeBlockHelperFuncs(BlockExpr *CE, int i,
+                                      StringRef funcName, std::string Tag);
+    std::string SynthesizeBlockFunc(BlockExpr *CE, int i,
+                                      StringRef funcName, std::string Tag);
+    std::string SynthesizeBlockImpl(BlockExpr *CE, 
+                                    std::string Tag, std::string Desc);
+    std::string SynthesizeBlockDescriptor(std::string DescTag, 
+                                          std::string ImplTag,
+                                          int i, StringRef funcName,
+                                          unsigned hasCopy);
+    Stmt *SynthesizeBlockCall(CallExpr *Exp, const Expr* BlockExp);
+    void SynthesizeBlockLiterals(SourceLocation FunLocStart,
+                                 StringRef FunName);
+    FunctionDecl *SynthBlockInitFunctionDecl(StringRef name);
+    Stmt *SynthBlockInitExpr(BlockExpr *Exp,
+            const SmallVector<DeclRefExpr *, 8> &InnerBlockDeclRefs);
+
+    // Misc. helper routines.
+    QualType getProtocolType();
+    void WarnAboutReturnGotoStmts(Stmt *S);
+    void CheckFunctionPointerDecl(QualType dType, NamedDecl *ND);
+    void InsertBlockLiteralsWithinFunction(FunctionDecl *FD);
+    void InsertBlockLiteralsWithinMethod(ObjCMethodDecl *MD);
+
+    bool IsDeclStmtInForeachHeader(DeclStmt *DS);
+    void CollectBlockDeclRefInfo(BlockExpr *Exp);
+    void GetBlockDeclRefExprs(Stmt *S);
+    void GetInnerBlockDeclRefExprs(Stmt *S, 
+                SmallVector<DeclRefExpr *, 8> &InnerBlockDeclRefs,
+                llvm::SmallPtrSet<const DeclContext *, 8> &InnerContexts);
+
+    // We avoid calling Type::isBlockPointerType(), since it operates on the
+    // canonical type. We only care if the top-level type is a closure pointer.
+    bool isTopLevelBlockPointerType(QualType T) {
+      return isa<BlockPointerType>(T);
+    }
+
+    /// convertBlockPointerToFunctionPointer - Converts a block-pointer type
+    /// to a function pointer type and upon success, returns true; false
+    /// otherwise.
+    bool convertBlockPointerToFunctionPointer(QualType &T) {
+      if (isTopLevelBlockPointerType(T)) {
+        const BlockPointerType *BPT = T->getAs<BlockPointerType>();
+        T = Context->getPointerType(BPT->getPointeeType());
+        return true;
+      }
+      return false;
+    }
+    
+    bool convertObjCTypeToCStyleType(QualType &T);
+    
+    bool needToScanForQualifiers(QualType T);
+    QualType getSuperStructType();
+    QualType getConstantStringStructType();
+    QualType convertFunctionTypeOfBlocks(const FunctionType *FT);
+    bool BufferContainsPPDirectives(const char *startBuf, const char *endBuf);
+    
+    void convertToUnqualifiedObjCType(QualType &T) {
+      if (T->isObjCQualifiedIdType()) {
+        bool isConst = T.isConstQualified();
+        T = isConst ? Context->getObjCIdType().withConst() 
+                    : Context->getObjCIdType();
+      }
+      else if (T->isObjCQualifiedClassType())
+        T = Context->getObjCClassType();
+      else if (T->isObjCObjectPointerType() &&
+               T->getPointeeType()->isObjCQualifiedInterfaceType()) {
+        if (const ObjCObjectPointerType * OBJPT =
+              T->getAsObjCInterfacePointerType()) {
+          const ObjCInterfaceType *IFaceT = OBJPT->getInterfaceType();
+          T = QualType(IFaceT, 0);
+          T = Context->getPointerType(T);
+        }
+     }
+    }
+    
+    // FIXME: This predicate seems like it would be useful to add to ASTContext.
+    bool isObjCType(QualType T) {
+      if (!LangOpts.ObjC1 && !LangOpts.ObjC2)
+        return false;
+
+      QualType OCT = Context->getCanonicalType(T).getUnqualifiedType();
+
+      if (OCT == Context->getCanonicalType(Context->getObjCIdType()) ||
+          OCT == Context->getCanonicalType(Context->getObjCClassType()))
+        return true;
+
+      if (const PointerType *PT = OCT->getAs<PointerType>()) {
+        if (isa<ObjCInterfaceType>(PT->getPointeeType()) ||
+            PT->getPointeeType()->isObjCQualifiedIdType())
+          return true;
+      }
+      return false;
+    }
+    bool PointerTypeTakesAnyBlockArguments(QualType QT);
+    bool PointerTypeTakesAnyObjCQualifiedType(QualType QT);
+    void GetExtentOfArgList(const char *Name, const char *&LParen,
+                            const char *&RParen);
+    
+    void QuoteDoublequotes(std::string &From, std::string &To) {
+      for (unsigned i = 0; i < From.length(); i++) {
+        if (From[i] == '"')
+          To += "\\\"";
+        else
+          To += From[i];
+      }
+    }
+
+    QualType getSimpleFunctionType(QualType result,
+                                   const QualType *args,
+                                   unsigned numArgs,
+                                   bool variadic = false) {
+      if (result == Context->getObjCInstanceType())
+        result =  Context->getObjCIdType();
+      FunctionProtoType::ExtProtoInfo fpi;
+      fpi.Variadic = variadic;
+      return Context->getFunctionType(result, args, numArgs, fpi);
+    }
+
+    // Helper function: create a CStyleCastExpr with trivial type source info.
+    CStyleCastExpr* NoTypeInfoCStyleCastExpr(ASTContext *Ctx, QualType Ty,
+                                             CastKind Kind, Expr *E) {
+      TypeSourceInfo *TInfo = Ctx->getTrivialTypeSourceInfo(Ty, SourceLocation());
+      return CStyleCastExpr::Create(*Ctx, Ty, VK_RValue, Kind, E, 0, TInfo,
+                                    SourceLocation(), SourceLocation());
+    }
+    
+    bool ImplementationIsNonLazy(const ObjCImplDecl *OD) const {
+      IdentifierInfo* II = &Context->Idents.get("load");
+      Selector LoadSel = Context->Selectors.getSelector(0, &II);
+      return OD->getClassMethod(LoadSel) != 0;
+    }
+  };
+  
+}
+
+void RewriteModernObjC::RewriteBlocksInFunctionProtoType(QualType funcType,
+                                                   NamedDecl *D) {
+  if (const FunctionProtoType *fproto
+      = dyn_cast<FunctionProtoType>(funcType.IgnoreParens())) {
+    for (FunctionProtoType::arg_type_iterator I = fproto->arg_type_begin(),
+         E = fproto->arg_type_end(); I && (I != E); ++I)
+      if (isTopLevelBlockPointerType(*I)) {
+        // All the args are checked/rewritten. Don't call twice!
+        RewriteBlockPointerDecl(D);
+        break;
+      }
+  }
+}
+
+void RewriteModernObjC::CheckFunctionPointerDecl(QualType funcType, NamedDecl *ND) {
+  const PointerType *PT = funcType->getAs<PointerType>();
+  if (PT && PointerTypeTakesAnyBlockArguments(funcType))
+    RewriteBlocksInFunctionProtoType(PT->getPointeeType(), ND);
+}
+
+static bool IsHeaderFile(const std::string &Filename) {
+  std::string::size_type DotPos = Filename.rfind('.');
+
+  if (DotPos == std::string::npos) {
+    // no file extension
+    return false;
+  }
+
+  std::string Ext = std::string(Filename.begin()+DotPos+1, Filename.end());
+  // C header: .h
+  // C++ header: .hh or .H;
+  return Ext == "h" || Ext == "hh" || Ext == "H";
+}
+
+RewriteModernObjC::RewriteModernObjC(std::string inFile, raw_ostream* OS,
+                         DiagnosticsEngine &D, const LangOptions &LOpts,
+                         bool silenceMacroWarn)
+      : Diags(D), LangOpts(LOpts), InFileName(inFile), OutFile(OS),
+        SilenceRewriteMacroWarning(silenceMacroWarn) {
+  IsHeader = IsHeaderFile(inFile);
+  RewriteFailedDiag = Diags.getCustomDiagID(DiagnosticsEngine::Warning,
+               "rewriting sub-expression within a macro (may not be correct)");
+  // FIXME. This should be an error. But if block is not called, it is OK. And it
+  // may break including some headers.
+  GlobalBlockRewriteFailedDiag = Diags.getCustomDiagID(DiagnosticsEngine::Warning,
+    "rewriting block literal declared in global scope is not implemented");
+          
+  TryFinallyContainsReturnDiag = Diags.getCustomDiagID(
+               DiagnosticsEngine::Warning,
+               "rewriter doesn't support user-specified control flow semantics "
+               "for @try/@finally (code may not execute properly)");
+}
+
+ASTConsumer *clang::CreateModernObjCRewriter(const std::string& InFile,
+                                       raw_ostream* OS,
+                                       DiagnosticsEngine &Diags,
+                                       const LangOptions &LOpts,
+                                       bool SilenceRewriteMacroWarning) {
+    return new RewriteModernObjC(InFile, OS, Diags, LOpts, SilenceRewriteMacroWarning);
+}
+
+void RewriteModernObjC::InitializeCommon(ASTContext &context) {
+  Context = &context;
+  SM = &Context->getSourceManager();
+  TUDecl = Context->getTranslationUnitDecl();
+  MsgSendFunctionDecl = 0;
+  MsgSendSuperFunctionDecl = 0;
+  MsgSendStretFunctionDecl = 0;
+  MsgSendSuperStretFunctionDecl = 0;
+  MsgSendFpretFunctionDecl = 0;
+  GetClassFunctionDecl = 0;
+  GetMetaClassFunctionDecl = 0;
+  GetSuperClassFunctionDecl = 0;
+  SelGetUidFunctionDecl = 0;
+  CFStringFunctionDecl = 0;
+  ConstantStringClassReference = 0;
+  NSStringRecord = 0;
+  CurMethodDef = 0;
+  CurFunctionDef = 0;
+  GlobalVarDecl = 0;
+  GlobalConstructionExp = 0;
+  SuperStructDecl = 0;
+  ProtocolTypeDecl = 0;
+  ConstantStringDecl = 0;
+  BcLabelCount = 0;
+  SuperContructorFunctionDecl = 0;
+  NumObjCStringLiterals = 0;
+  PropParentMap = 0;
+  CurrentBody = 0;
+  DisableReplaceStmt = false;
+  objc_impl_method = false;
+
+  // Get the ID and start/end of the main file.
+  MainFileID = SM->getMainFileID();
+  const llvm::MemoryBuffer *MainBuf = SM->getBuffer(MainFileID);
+  MainFileStart = MainBuf->getBufferStart();
+  MainFileEnd = MainBuf->getBufferEnd();
+
+  Rewrite.setSourceMgr(Context->getSourceManager(), Context->getLangOpts());
+}
+
+//===----------------------------------------------------------------------===//
+// Top Level Driver Code
+//===----------------------------------------------------------------------===//
+
+void RewriteModernObjC::HandleTopLevelSingleDecl(Decl *D) {
+  if (Diags.hasErrorOccurred())
+    return;
+
+  // Two cases: either the decl could be in the main file, or it could be in a
+  // #included file.  If the former, rewrite it now.  If the later, check to see
+  // if we rewrote the #include/#import.
+  SourceLocation Loc = D->getLocation();
+  Loc = SM->getExpansionLoc(Loc);
+
+  // If this is for a builtin, ignore it.
+  if (Loc.isInvalid()) return;
+
+  // Look for built-in declarations that we need to refer during the rewrite.
+  if (FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
+    RewriteFunctionDecl(FD);
+  } else if (VarDecl *FVD = dyn_cast<VarDecl>(D)) {
+    // declared in <Foundation/NSString.h>
+    if (FVD->getName() == "_NSConstantStringClassReference") {
+      ConstantStringClassReference = FVD;
+      return;
+    }
+  } else if (ObjCCategoryDecl *CD = dyn_cast<ObjCCategoryDecl>(D)) {
+    RewriteCategoryDecl(CD);
+  } else if (ObjCProtocolDecl *PD = dyn_cast<ObjCProtocolDecl>(D)) {
+    if (PD->isThisDeclarationADefinition())
+      RewriteProtocolDecl(PD);
+  } else if (LinkageSpecDecl *LSD = dyn_cast<LinkageSpecDecl>(D)) {
+    // FIXME. This will not work in all situations and leaving it out
+    // is harmless.
+    // RewriteLinkageSpec(LSD);
+    
+    // Recurse into linkage specifications
+    for (DeclContext::decl_iterator DI = LSD->decls_begin(),
+                                 DIEnd = LSD->decls_end();
+         DI != DIEnd; ) {
+      if (ObjCInterfaceDecl *IFace = dyn_cast<ObjCInterfaceDecl>((*DI))) {
+        if (!IFace->isThisDeclarationADefinition()) {
+          SmallVector<Decl *, 8> DG;
+          SourceLocation StartLoc = IFace->getLocStart();
+          do {
+            if (isa<ObjCInterfaceDecl>(*DI) &&
+                !cast<ObjCInterfaceDecl>(*DI)->isThisDeclarationADefinition() &&
+                StartLoc == (*DI)->getLocStart())
+              DG.push_back(*DI);
+            else
+              break;
+            
+            ++DI;
+          } while (DI != DIEnd);
+          RewriteForwardClassDecl(DG);
+          continue;
+        }
+        else {
+          // Keep track of all interface declarations seen.
+          ObjCInterfacesSeen.push_back(IFace);
+          ++DI;
+          continue;
+        }
+      }
+
+      if (ObjCProtocolDecl *Proto = dyn_cast<ObjCProtocolDecl>((*DI))) {
+        if (!Proto->isThisDeclarationADefinition()) {
+          SmallVector<Decl *, 8> DG;
+          SourceLocation StartLoc = Proto->getLocStart();
+          do {
+            if (isa<ObjCProtocolDecl>(*DI) &&
+                !cast<ObjCProtocolDecl>(*DI)->isThisDeclarationADefinition() &&
+                StartLoc == (*DI)->getLocStart())
+              DG.push_back(*DI);
+            else
+              break;
+            
+            ++DI;
+          } while (DI != DIEnd);
+          RewriteForwardProtocolDecl(DG);
+          continue;
+        }
+      }
+      
+      HandleTopLevelSingleDecl(*DI);
+      ++DI;
+    }
+  }
+  // If we have a decl in the main file, see if we should rewrite it.
+  if (SM->isFromMainFile(Loc))
+    return HandleDeclInMainFile(D);
+}
+
+//===----------------------------------------------------------------------===//
+// Syntactic (non-AST) Rewriting Code
+//===----------------------------------------------------------------------===//
+
+void RewriteModernObjC::RewriteInclude() {
+  SourceLocation LocStart = SM->getLocForStartOfFile(MainFileID);
+  StringRef MainBuf = SM->getBufferData(MainFileID);
+  const char *MainBufStart = MainBuf.begin();
+  const char *MainBufEnd = MainBuf.end();
+  size_t ImportLen = strlen("import");
+
+  // Loop over the whole file, looking for includes.
+  for (const char *BufPtr = MainBufStart; BufPtr < MainBufEnd; ++BufPtr) {
+    if (*BufPtr == '#') {
+      if (++BufPtr == MainBufEnd)
+        return;
+      while (*BufPtr == ' ' || *BufPtr == '\t')
+        if (++BufPtr == MainBufEnd)
+          return;
+      if (!strncmp(BufPtr, "import", ImportLen)) {
+        // replace import with include
+        SourceLocation ImportLoc =
+          LocStart.getLocWithOffset(BufPtr-MainBufStart);
+        ReplaceText(ImportLoc, ImportLen, "include");
+        BufPtr += ImportLen;
+      }
+    }
+  }
+}
+
+static void WriteInternalIvarName(const ObjCInterfaceDecl *IDecl,
+                                  ObjCIvarDecl *IvarDecl, std::string &Result) {
+  Result += "OBJC_IVAR_$_";
+  Result += IDecl->getName();
+  Result += "$";
+  Result += IvarDecl->getName();
+}
+
+std::string 
+RewriteModernObjC::getIvarAccessString(ObjCIvarDecl *D) {
+  const ObjCInterfaceDecl *ClassDecl = D->getContainingInterface();
+  
+  // Build name of symbol holding ivar offset.
+  std::string IvarOffsetName;
+  WriteInternalIvarName(ClassDecl, D, IvarOffsetName);
+  
+  
+  std::string S = "(*(";
+  QualType IvarT = D->getType();
+  
+  if (!isa<TypedefType>(IvarT) && IvarT->isRecordType()) {
+    RecordDecl *RD = IvarT->getAs<RecordType>()->getDecl();
+    RD = RD->getDefinition();
+    if (RD && !RD->getDeclName().getAsIdentifierInfo()) {
+      // decltype(((Foo_IMPL*)0)->bar) *
+      ObjCContainerDecl *CDecl = 
+      dyn_cast<ObjCContainerDecl>(D->getDeclContext());
+      // ivar in class extensions requires special treatment.
+      if (ObjCCategoryDecl *CatDecl = dyn_cast<ObjCCategoryDecl>(CDecl))
+        CDecl = CatDecl->getClassInterface();
+      std::string RecName = CDecl->getName();
+      RecName += "_IMPL";
+      RecordDecl *RD = RecordDecl::Create(*Context, TTK_Struct, TUDecl,
+                                          SourceLocation(), SourceLocation(),
+                                          &Context->Idents.get(RecName.c_str()));
+      QualType PtrStructIMPL = Context->getPointerType(Context->getTagDeclType(RD));
+      unsigned UnsignedIntSize = 
+      static_cast<unsigned>(Context->getTypeSize(Context->UnsignedIntTy));
+      Expr *Zero = IntegerLiteral::Create(*Context,
+                                          llvm::APInt(UnsignedIntSize, 0),
+                                          Context->UnsignedIntTy, SourceLocation());
+      Zero = NoTypeInfoCStyleCastExpr(Context, PtrStructIMPL, CK_BitCast, Zero);
+      ParenExpr *PE = new (Context) ParenExpr(SourceLocation(), SourceLocation(),
+                                              Zero);
+      FieldDecl *FD = FieldDecl::Create(*Context, 0, SourceLocation(),
+                                        SourceLocation(),
+                                        &Context->Idents.get(D->getNameAsString()),
+                                        IvarT, 0,
+                                        /*BitWidth=*/0, /*Mutable=*/true,
+                                        ICIS_NoInit);
+      MemberExpr *ME = new (Context) MemberExpr(PE, true, FD, SourceLocation(),
+                                                FD->getType(), VK_LValue,
+                                                OK_Ordinary);
+      IvarT = Context->getDecltypeType(ME, ME->getType());
+    }
+  }
+  convertObjCTypeToCStyleType(IvarT);
+  QualType castT = Context->getPointerType(IvarT);
+  std::string TypeString(castT.getAsString(Context->getPrintingPolicy()));
+  S += TypeString;
+  S += ")";
+  
+  // ((char *)self + IVAR_OFFSET_SYMBOL_NAME)
+  S += "((char *)self + ";
+  S += IvarOffsetName;
+  S += "))";
+  ReferencedIvars[const_cast<ObjCInterfaceDecl *>(ClassDecl)].insert(D);
+  return S;
+}
+
+/// mustSynthesizeSetterGetterMethod - returns true if setter or getter has not
+/// been found in the class implementation. In this case, it must be synthesized.
+static bool mustSynthesizeSetterGetterMethod(ObjCImplementationDecl *IMP,
+                                             ObjCPropertyDecl *PD,
+                                             bool getter) {
+  return getter ? !IMP->getInstanceMethod(PD->getGetterName())
+                : !IMP->getInstanceMethod(PD->getSetterName());
+  
+}
+
+void RewriteModernObjC::RewritePropertyImplDecl(ObjCPropertyImplDecl *PID,
+                                          ObjCImplementationDecl *IMD,
+                                          ObjCCategoryImplDecl *CID) {
+  static bool objcGetPropertyDefined = false;
+  static bool objcSetPropertyDefined = false;
+  SourceLocation startGetterSetterLoc;
+  
+  if (PID->getLocStart().isValid()) {
+    SourceLocation startLoc = PID->getLocStart();
+    InsertText(startLoc, "// ");
+    const char *startBuf = SM->getCharacterData(startLoc);
+    assert((*startBuf == '@') && "bogus @synthesize location");
+    const char *semiBuf = strchr(startBuf, ';');
+    assert((*semiBuf == ';') && "@synthesize: can't find ';'");
+    startGetterSetterLoc = startLoc.getLocWithOffset(semiBuf-startBuf+1);
+  }
+  else
+    startGetterSetterLoc = IMD ? IMD->getLocEnd() : CID->getLocEnd();
+
+  if (PID->getPropertyImplementation() == ObjCPropertyImplDecl::Dynamic)
+    return; // FIXME: is this correct?
+
+  // Generate the 'getter' function.
+  ObjCPropertyDecl *PD = PID->getPropertyDecl();
+  ObjCIvarDecl *OID = PID->getPropertyIvarDecl();
+
+  if (!OID)
+    return;
+  unsigned Attributes = PD->getPropertyAttributes();
+  if (mustSynthesizeSetterGetterMethod(IMD, PD, true /*getter*/)) {
+    bool GenGetProperty = !(Attributes & ObjCPropertyDecl::OBJC_PR_nonatomic) &&
+                          (Attributes & (ObjCPropertyDecl::OBJC_PR_retain | 
+                                         ObjCPropertyDecl::OBJC_PR_copy));
+    std::string Getr;
+    if (GenGetProperty && !objcGetPropertyDefined) {
+      objcGetPropertyDefined = true;
+      // FIXME. Is this attribute correct in all cases?
+      Getr = "\nextern \"C\" __declspec(dllimport) "
+            "id objc_getProperty(id, SEL, long, bool);\n";
+    }
+    RewriteObjCMethodDecl(OID->getContainingInterface(),  
+                          PD->getGetterMethodDecl(), Getr);
+    Getr += "{ ";
+    // Synthesize an explicit cast to gain access to the ivar.
+    // See objc-act.c:objc_synthesize_new_getter() for details.
+    if (GenGetProperty) {
+      // return objc_getProperty(self, _cmd, offsetof(ClassDecl, OID), 1)
+      Getr += "typedef ";
+      const FunctionType *FPRetType = 0;
+      RewriteTypeIntoString(PD->getGetterMethodDecl()->getResultType(), Getr, 
+                            FPRetType);
+      Getr += " _TYPE";
+      if (FPRetType) {
+        Getr += ")"; // close the precedence "scope" for "*".
+      
+        // Now, emit the argument types (if any).
+        if (const FunctionProtoType *FT = dyn_cast<FunctionProtoType>(FPRetType)){
+          Getr += "(";
+          for (unsigned i = 0, e = FT->getNumArgs(); i != e; ++i) {
+            if (i) Getr += ", ";
+            std::string ParamStr = FT->getArgType(i).getAsString(
+                                                          Context->getPrintingPolicy());
+            Getr += ParamStr;
+          }
+          if (FT->isVariadic()) {
+            if (FT->getNumArgs()) Getr += ", ";
+            Getr += "...";
+          }
+          Getr += ")";
+        } else
+          Getr += "()";
+      }
+      Getr += ";\n";
+      Getr += "return (_TYPE)";
+      Getr += "objc_getProperty(self, _cmd, ";
+      RewriteIvarOffsetComputation(OID, Getr);
+      Getr += ", 1)";
+    }
+    else
+      Getr += "return " + getIvarAccessString(OID);
+    Getr += "; }";
+    InsertText(startGetterSetterLoc, Getr);
+  }
+  
+  if (PD->isReadOnly() || 
+      !mustSynthesizeSetterGetterMethod(IMD, PD, false /*setter*/))
+    return;
+
+  // Generate the 'setter' function.
+  std::string Setr;
+  bool GenSetProperty = Attributes & (ObjCPropertyDecl::OBJC_PR_retain | 
+                                      ObjCPropertyDecl::OBJC_PR_copy);
+  if (GenSetProperty && !objcSetPropertyDefined) {
+    objcSetPropertyDefined = true;
+    // FIXME. Is this attribute correct in all cases?
+    Setr = "\nextern \"C\" __declspec(dllimport) "
+    "void objc_setProperty (id, SEL, long, id, bool, bool);\n";
+  }
+  
+  RewriteObjCMethodDecl(OID->getContainingInterface(), 
+                        PD->getSetterMethodDecl(), Setr);
+  Setr += "{ ";
+  // Synthesize an explicit cast to initialize the ivar.
+  // See objc-act.c:objc_synthesize_new_setter() for details.
+  if (GenSetProperty) {
+    Setr += "objc_setProperty (self, _cmd, ";
+    RewriteIvarOffsetComputation(OID, Setr);
+    Setr += ", (id)";
+    Setr += PD->getName();
+    Setr += ", ";
+    if (Attributes & ObjCPropertyDecl::OBJC_PR_nonatomic)
+      Setr += "0, ";
+    else
+      Setr += "1, ";
+    if (Attributes & ObjCPropertyDecl::OBJC_PR_copy)
+      Setr += "1)";
+    else
+      Setr += "0)";
+  }
+  else {
+    Setr += getIvarAccessString(OID) + " = ";
+    Setr += PD->getName();
+  }
+  Setr += "; }\n";
+  InsertText(startGetterSetterLoc, Setr);
+}
+
+static void RewriteOneForwardClassDecl(ObjCInterfaceDecl *ForwardDecl,
+                                       std::string &typedefString) {
+  typedefString += "#ifndef _REWRITER_typedef_";
+  typedefString += ForwardDecl->getNameAsString();
+  typedefString += "\n";
+  typedefString += "#define _REWRITER_typedef_";
+  typedefString += ForwardDecl->getNameAsString();
+  typedefString += "\n";
+  typedefString += "typedef struct objc_object ";
+  typedefString += ForwardDecl->getNameAsString();
+  // typedef struct { } _objc_exc_Classname;
+  typedefString += ";\ntypedef struct {} _objc_exc_";
+  typedefString += ForwardDecl->getNameAsString();
+  typedefString += ";\n#endif\n";
+}
+
+void RewriteModernObjC::RewriteForwardClassEpilogue(ObjCInterfaceDecl *ClassDecl,
+                                              const std::string &typedefString) {
+    SourceLocation startLoc = ClassDecl->getLocStart();
+    const char *startBuf = SM->getCharacterData(startLoc);
+    const char *semiPtr = strchr(startBuf, ';'); 
+    // Replace the @class with typedefs corresponding to the classes.
+    ReplaceText(startLoc, semiPtr-startBuf+1, typedefString);  
+}
+
+void RewriteModernObjC::RewriteForwardClassDecl(DeclGroupRef D) {
+  std::string typedefString;
+  for (DeclGroupRef::iterator I = D.begin(), E = D.end(); I != E; ++I) {
+    ObjCInterfaceDecl *ForwardDecl = cast<ObjCInterfaceDecl>(*I);
+    if (I == D.begin()) {
+      // Translate to typedef's that forward reference structs with the same name
+      // as the class. As a convenience, we include the original declaration
+      // as a comment.
+      typedefString += "// @class ";
+      typedefString += ForwardDecl->getNameAsString();
+      typedefString += ";\n";
+    }
+    RewriteOneForwardClassDecl(ForwardDecl, typedefString);
+  }
+  DeclGroupRef::iterator I = D.begin();
+  RewriteForwardClassEpilogue(cast<ObjCInterfaceDecl>(*I), typedefString);
+}
+
+void RewriteModernObjC::RewriteForwardClassDecl(
+                                const llvm::SmallVector<Decl*, 8> &D) {
+  std::string typedefString;
+  for (unsigned i = 0; i < D.size(); i++) {
+    ObjCInterfaceDecl *ForwardDecl = cast<ObjCInterfaceDecl>(D[i]);
+    if (i == 0) {
+      typedefString += "// @class ";
+      typedefString += ForwardDecl->getNameAsString();
+      typedefString += ";\n";
+    }
+    RewriteOneForwardClassDecl(ForwardDecl, typedefString);
+  }
+  RewriteForwardClassEpilogue(cast<ObjCInterfaceDecl>(D[0]), typedefString);
+}
+
+void RewriteModernObjC::RewriteMethodDeclaration(ObjCMethodDecl *Method) {
+  // When method is a synthesized one, such as a getter/setter there is
+  // nothing to rewrite.
+  if (Method->isImplicit())
+    return;
+  SourceLocation LocStart = Method->getLocStart();
+  SourceLocation LocEnd = Method->getLocEnd();
+
+  if (SM->getExpansionLineNumber(LocEnd) >
+      SM->getExpansionLineNumber(LocStart)) {
+    InsertText(LocStart, "#if 0\n");
+    ReplaceText(LocEnd, 1, ";\n#endif\n");
+  } else {
+    InsertText(LocStart, "// ");
+  }
+}
+
+void RewriteModernObjC::RewriteProperty(ObjCPropertyDecl *prop) {
+  SourceLocation Loc = prop->getAtLoc();
+
+  ReplaceText(Loc, 0, "// ");
+  // FIXME: handle properties that are declared across multiple lines.
+}
+
+void RewriteModernObjC::RewriteCategoryDecl(ObjCCategoryDecl *CatDecl) {
+  SourceLocation LocStart = CatDecl->getLocStart();
+
+  // FIXME: handle category headers that are declared across multiple lines.
+  if (CatDecl->getIvarRBraceLoc().isValid()) {
+    ReplaceText(LocStart, 1, "/** ");
+    ReplaceText(CatDecl->getIvarRBraceLoc(), 1, "**/ ");
+  }
+  else {
+    ReplaceText(LocStart, 0, "// ");
+  }
+  
+  for (ObjCCategoryDecl::prop_iterator I = CatDecl->prop_begin(),
+       E = CatDecl->prop_end(); I != E; ++I)
+    RewriteProperty(*I);
+  
+  for (ObjCCategoryDecl::instmeth_iterator
+         I = CatDecl->instmeth_begin(), E = CatDecl->instmeth_end();
+       I != E; ++I)
+    RewriteMethodDeclaration(*I);
+  for (ObjCCategoryDecl::classmeth_iterator
+         I = CatDecl->classmeth_begin(), E = CatDecl->classmeth_end();
+       I != E; ++I)
+    RewriteMethodDeclaration(*I);
+
+  // Lastly, comment out the @end.
+  ReplaceText(CatDecl->getAtEndRange().getBegin(), 
+              strlen("@end"), "/* @end */");
+}
+
+void RewriteModernObjC::RewriteProtocolDecl(ObjCProtocolDecl *PDecl) {
+  SourceLocation LocStart = PDecl->getLocStart();
+  assert(PDecl->isThisDeclarationADefinition());
+  
+  // FIXME: handle protocol headers that are declared across multiple lines.
+  ReplaceText(LocStart, 0, "// ");
+
+  for (ObjCProtocolDecl::instmeth_iterator
+         I = PDecl->instmeth_begin(), E = PDecl->instmeth_end();
+       I != E; ++I)
+    RewriteMethodDeclaration(*I);
+  for (ObjCProtocolDecl::classmeth_iterator
+         I = PDecl->classmeth_begin(), E = PDecl->classmeth_end();
+       I != E; ++I)
+    RewriteMethodDeclaration(*I);
+
+  for (ObjCInterfaceDecl::prop_iterator I = PDecl->prop_begin(),
+       E = PDecl->prop_end(); I != E; ++I)
+    RewriteProperty(*I);
+  
+  // Lastly, comment out the @end.
+  SourceLocation LocEnd = PDecl->getAtEndRange().getBegin();
+  ReplaceText(LocEnd, strlen("@end"), "/* @end */");
+
+  // Must comment out @optional/@required
+  const char *startBuf = SM->getCharacterData(LocStart);
+  const char *endBuf = SM->getCharacterData(LocEnd);
+  for (const char *p = startBuf; p < endBuf; p++) {
+    if (*p == '@' && !strncmp(p+1, "optional", strlen("optional"))) {
+      SourceLocation OptionalLoc = LocStart.getLocWithOffset(p-startBuf);
+      ReplaceText(OptionalLoc, strlen("@optional"), "/* @optional */");
+
+    }
+    else if (*p == '@' && !strncmp(p+1, "required", strlen("required"))) {
+      SourceLocation OptionalLoc = LocStart.getLocWithOffset(p-startBuf);
+      ReplaceText(OptionalLoc, strlen("@required"), "/* @required */");
+
+    }
+  }
+}
+
+void RewriteModernObjC::RewriteForwardProtocolDecl(DeclGroupRef D) {
+  SourceLocation LocStart = (*D.begin())->getLocStart();
+  if (LocStart.isInvalid())
+    llvm_unreachable("Invalid SourceLocation");
+  // FIXME: handle forward protocol that are declared across multiple lines.
+  ReplaceText(LocStart, 0, "// ");
+}
+
+void 
+RewriteModernObjC::RewriteForwardProtocolDecl(const llvm::SmallVector<Decl*, 8> &DG) {
+  SourceLocation LocStart = DG[0]->getLocStart();
+  if (LocStart.isInvalid())
+    llvm_unreachable("Invalid SourceLocation");
+  // FIXME: handle forward protocol that are declared across multiple lines.
+  ReplaceText(LocStart, 0, "// ");
+}
+
+void 
+RewriteModernObjC::RewriteLinkageSpec(LinkageSpecDecl *LSD) {
+  SourceLocation LocStart = LSD->getExternLoc();
+  if (LocStart.isInvalid())
+    llvm_unreachable("Invalid extern SourceLocation");
+  
+  ReplaceText(LocStart, 0, "// ");
+  if (!LSD->hasBraces())
+    return;
+  // FIXME. We don't rewrite well if '{' is not on same line as 'extern'.
+  SourceLocation LocRBrace = LSD->getRBraceLoc();
+  if (LocRBrace.isInvalid())
+    llvm_unreachable("Invalid rbrace SourceLocation");
+  ReplaceText(LocRBrace, 0, "// ");
+}
+
+void RewriteModernObjC::RewriteTypeIntoString(QualType T, std::string &ResultStr,
+                                        const FunctionType *&FPRetType) {
+  if (T->isObjCQualifiedIdType())
+    ResultStr += "id";
+  else if (T->isFunctionPointerType() ||
+           T->isBlockPointerType()) {
+    // needs special handling, since pointer-to-functions have special
+    // syntax (where a decaration models use).
+    QualType retType = T;
+    QualType PointeeTy;
+    if (const PointerType* PT = retType->getAs<PointerType>())
+      PointeeTy = PT->getPointeeType();
+    else if (const BlockPointerType *BPT = retType->getAs<BlockPointerType>())
+      PointeeTy = BPT->getPointeeType();
+    if ((FPRetType = PointeeTy->getAs<FunctionType>())) {
+      ResultStr += FPRetType->getResultType().getAsString(
+        Context->getPrintingPolicy());
+      ResultStr += "(*";
+    }
+  } else
+    ResultStr += T.getAsString(Context->getPrintingPolicy());
+}
+
+void RewriteModernObjC::RewriteObjCMethodDecl(const ObjCInterfaceDecl *IDecl,
+                                        ObjCMethodDecl *OMD,
+                                        std::string &ResultStr) {
+  //fprintf(stderr,"In RewriteObjCMethodDecl\n");
+  const FunctionType *FPRetType = 0;
+  ResultStr += "\nstatic ";
+  RewriteTypeIntoString(OMD->getResultType(), ResultStr, FPRetType);
+  ResultStr += " ";
+
+  // Unique method name
+  std::string NameStr;
+
+  if (OMD->isInstanceMethod())
+    NameStr += "_I_";
+  else
+    NameStr += "_C_";
+
+  NameStr += IDecl->getNameAsString();
+  NameStr += "_";
+
+  if (ObjCCategoryImplDecl *CID =
+      dyn_cast<ObjCCategoryImplDecl>(OMD->getDeclContext())) {
+    NameStr += CID->getNameAsString();
+    NameStr += "_";
+  }
+  // Append selector names, replacing ':' with '_'
+  {
+    std::string selString = OMD->getSelector().getAsString();
+    int len = selString.size();
+    for (int i = 0; i < len; i++)
+      if (selString[i] == ':')
+        selString[i] = '_';
+    NameStr += selString;
+  }
+  // Remember this name for metadata emission
+  MethodInternalNames[OMD] = NameStr;
+  ResultStr += NameStr;
+
+  // Rewrite arguments
+  ResultStr += "(";
+
+  // invisible arguments
+  if (OMD->isInstanceMethod()) {
+    QualType selfTy = Context->getObjCInterfaceType(IDecl);
+    selfTy = Context->getPointerType(selfTy);
+    if (!LangOpts.MicrosoftExt) {
+      if (ObjCSynthesizedStructs.count(const_cast<ObjCInterfaceDecl*>(IDecl)))
+        ResultStr += "struct ";
+    }
+    // When rewriting for Microsoft, explicitly omit the structure name.
+    ResultStr += IDecl->getNameAsString();
+    ResultStr += " *";
+  }
+  else
+    ResultStr += Context->getObjCClassType().getAsString(
+      Context->getPrintingPolicy());
+
+  ResultStr += " self, ";
+  ResultStr += Context->getObjCSelType().getAsString(Context->getPrintingPolicy());
+  ResultStr += " _cmd";
+
+  // Method arguments.
+  for (ObjCMethodDecl::param_iterator PI = OMD->param_begin(),
+       E = OMD->param_end(); PI != E; ++PI) {
+    ParmVarDecl *PDecl = *PI;
+    ResultStr += ", ";
+    if (PDecl->getType()->isObjCQualifiedIdType()) {
+      ResultStr += "id ";
+      ResultStr += PDecl->getNameAsString();
+    } else {
+      std::string Name = PDecl->getNameAsString();
+      QualType QT = PDecl->getType();
+      // Make sure we convert "t (^)(...)" to "t (*)(...)".
+      (void)convertBlockPointerToFunctionPointer(QT);
+      QT.getAsStringInternal(Name, Context->getPrintingPolicy());
+      ResultStr += Name;
+    }
+  }
+  if (OMD->isVariadic())
+    ResultStr += ", ...";
+  ResultStr += ") ";
+
+  if (FPRetType) {
+    ResultStr += ")"; // close the precedence "scope" for "*".
+
+    // Now, emit the argument types (if any).
+    if (const FunctionProtoType *FT = dyn_cast<FunctionProtoType>(FPRetType)) {
+      ResultStr += "(";
+      for (unsigned i = 0, e = FT->getNumArgs(); i != e; ++i) {
+        if (i) ResultStr += ", ";
+        std::string ParamStr = FT->getArgType(i).getAsString(
+          Context->getPrintingPolicy());
+        ResultStr += ParamStr;
+      }
+      if (FT->isVariadic()) {
+        if (FT->getNumArgs()) ResultStr += ", ";
+        ResultStr += "...";
+      }
+      ResultStr += ")";
+    } else {
+      ResultStr += "()";
+    }
+  }
+}
+void RewriteModernObjC::RewriteImplementationDecl(Decl *OID) {
+  ObjCImplementationDecl *IMD = dyn_cast<ObjCImplementationDecl>(OID);
+  ObjCCategoryImplDecl *CID = dyn_cast<ObjCCategoryImplDecl>(OID);
+
+  if (IMD) {
+    if (IMD->getIvarRBraceLoc().isValid()) {
+      ReplaceText(IMD->getLocStart(), 1, "/** ");
+      ReplaceText(IMD->getIvarRBraceLoc(), 1, "**/ ");
+    }
+    else {
+      InsertText(IMD->getLocStart(), "// ");
+    }
+  }
+  else
+    InsertText(CID->getLocStart(), "// ");
+
+  for (ObjCCategoryImplDecl::instmeth_iterator
+       I = IMD ? IMD->instmeth_begin() : CID->instmeth_begin(),
+       E = IMD ? IMD->instmeth_end() : CID->instmeth_end();
+       I != E; ++I) {
+    std::string ResultStr;
+    ObjCMethodDecl *OMD = *I;
+    RewriteObjCMethodDecl(OMD->getClassInterface(), OMD, ResultStr);
+    SourceLocation LocStart = OMD->getLocStart();
+    SourceLocation LocEnd = OMD->getCompoundBody()->getLocStart();
+
+    const char *startBuf = SM->getCharacterData(LocStart);
+    const char *endBuf = SM->getCharacterData(LocEnd);
+    ReplaceText(LocStart, endBuf-startBuf, ResultStr);
+  }
+
+  for (ObjCCategoryImplDecl::classmeth_iterator
+       I = IMD ? IMD->classmeth_begin() : CID->classmeth_begin(),
+       E = IMD ? IMD->classmeth_end() : CID->classmeth_end();
+       I != E; ++I) {
+    std::string ResultStr;
+    ObjCMethodDecl *OMD = *I;
+    RewriteObjCMethodDecl(OMD->getClassInterface(), OMD, ResultStr);
+    SourceLocation LocStart = OMD->getLocStart();
+    SourceLocation LocEnd = OMD->getCompoundBody()->getLocStart();
+
+    const char *startBuf = SM->getCharacterData(LocStart);
+    const char *endBuf = SM->getCharacterData(LocEnd);
+    ReplaceText(LocStart, endBuf-startBuf, ResultStr);
+  }
+  for (ObjCCategoryImplDecl::propimpl_iterator
+       I = IMD ? IMD->propimpl_begin() : CID->propimpl_begin(),
+       E = IMD ? IMD->propimpl_end() : CID->propimpl_end();
+       I != E; ++I) {
+    RewritePropertyImplDecl(*I, IMD, CID);
+  }
+
+  InsertText(IMD ? IMD->getLocEnd() : CID->getLocEnd(), "// ");
+}
+
+void RewriteModernObjC::RewriteInterfaceDecl(ObjCInterfaceDecl *ClassDecl) {
+  // Do not synthesize more than once.
+  if (ObjCSynthesizedStructs.count(ClassDecl))
+    return;
+  // Make sure super class's are written before current class is written.
+  ObjCInterfaceDecl *SuperClass = ClassDecl->getSuperClass();
+  while (SuperClass) {
+    RewriteInterfaceDecl(SuperClass);
+    SuperClass = SuperClass->getSuperClass();
+  }
+  std::string ResultStr;
+  if (!ObjCWrittenInterfaces.count(ClassDecl->getCanonicalDecl())) {
+    // we haven't seen a forward decl - generate a typedef.
+    RewriteOneForwardClassDecl(ClassDecl, ResultStr);
+    RewriteIvarOffsetSymbols(ClassDecl, ResultStr);
+    
+    RewriteObjCInternalStruct(ClassDecl, ResultStr);
+    // Mark this typedef as having been written into its c++ equivalent.
+    ObjCWrittenInterfaces.insert(ClassDecl->getCanonicalDecl());
+  
+    for (ObjCInterfaceDecl::prop_iterator I = ClassDecl->prop_begin(),
+         E = ClassDecl->prop_end(); I != E; ++I)
+      RewriteProperty(*I);
+    for (ObjCInterfaceDecl::instmeth_iterator
+         I = ClassDecl->instmeth_begin(), E = ClassDecl->instmeth_end();
+         I != E; ++I)
+      RewriteMethodDeclaration(*I);
+    for (ObjCInterfaceDecl::classmeth_iterator
+         I = ClassDecl->classmeth_begin(), E = ClassDecl->classmeth_end();
+         I != E; ++I)
+      RewriteMethodDeclaration(*I);
+
+    // Lastly, comment out the @end.
+    ReplaceText(ClassDecl->getAtEndRange().getBegin(), strlen("@end"), 
+                "/* @end */");
+  }
+}
+
+Stmt *RewriteModernObjC::RewritePropertyOrImplicitSetter(PseudoObjectExpr *PseudoOp) {
+  SourceRange OldRange = PseudoOp->getSourceRange();
+
+  // We just magically know some things about the structure of this
+  // expression.
+  ObjCMessageExpr *OldMsg =
+    cast<ObjCMessageExpr>(PseudoOp->getSemanticExpr(
+                            PseudoOp->getNumSemanticExprs() - 1));
+
+  // Because the rewriter doesn't allow us to rewrite rewritten code,
+  // we need to suppress rewriting the sub-statements.
+  Expr *Base;
+  SmallVector<Expr*, 2> Args;
+  {
+    DisableReplaceStmtScope S(*this);
+
+    // Rebuild the base expression if we have one.
+    Base = 0;
+    if (OldMsg->getReceiverKind() == ObjCMessageExpr::Instance) {
+      Base = OldMsg->getInstanceReceiver();
+      Base = cast<OpaqueValueExpr>(Base)->getSourceExpr();
+      Base = cast<Expr>(RewriteFunctionBodyOrGlobalInitializer(Base));
+    }
+  
+    unsigned numArgs = OldMsg->getNumArgs();
+    for (unsigned i = 0; i < numArgs; i++) {
+      Expr *Arg = OldMsg->getArg(i);
+      if (isa<OpaqueValueExpr>(Arg))
+        Arg = cast<OpaqueValueExpr>(Arg)->getSourceExpr();
+      Arg = cast<Expr>(RewriteFunctionBodyOrGlobalInitializer(Arg));
+      Args.push_back(Arg);
+    }
+  }
+
+  // TODO: avoid this copy.
+  SmallVector<SourceLocation, 1> SelLocs;
+  OldMsg->getSelectorLocs(SelLocs);
+
+  ObjCMessageExpr *NewMsg = 0;
+  switch (OldMsg->getReceiverKind()) {
+  case ObjCMessageExpr::Class:
+    NewMsg = ObjCMessageExpr::Create(*Context, OldMsg->getType(),
+                                     OldMsg->getValueKind(),
+                                     OldMsg->getLeftLoc(),
+                                     OldMsg->getClassReceiverTypeInfo(),
+                                     OldMsg->getSelector(),
+                                     SelLocs,
+                                     OldMsg->getMethodDecl(),
+                                     Args,
+                                     OldMsg->getRightLoc(),
+                                     OldMsg->isImplicit());
+    break;
+
+  case ObjCMessageExpr::Instance:
+    NewMsg = ObjCMessageExpr::Create(*Context, OldMsg->getType(),
+                                     OldMsg->getValueKind(),
+                                     OldMsg->getLeftLoc(),
+                                     Base,
+                                     OldMsg->getSelector(),
+                                     SelLocs,
+                                     OldMsg->getMethodDecl(),
+                                     Args,
+                                     OldMsg->getRightLoc(),
+                                     OldMsg->isImplicit());
+    break;
+
+  case ObjCMessageExpr::SuperClass:
+  case ObjCMessageExpr::SuperInstance:
+    NewMsg = ObjCMessageExpr::Create(*Context, OldMsg->getType(),
+                                     OldMsg->getValueKind(),
+                                     OldMsg->getLeftLoc(),
+                                     OldMsg->getSuperLoc(),
+                 OldMsg->getReceiverKind() == ObjCMessageExpr::SuperInstance,
+                                     OldMsg->getSuperType(),
+                                     OldMsg->getSelector(),
+                                     SelLocs,
+                                     OldMsg->getMethodDecl(),
+                                     Args,
+                                     OldMsg->getRightLoc(),
+                                     OldMsg->isImplicit());
+    break;
+  }
+
+  Stmt *Replacement = SynthMessageExpr(NewMsg);
+  ReplaceStmtWithRange(PseudoOp, Replacement, OldRange);
+  return Replacement;
+}
+
+Stmt *RewriteModernObjC::RewritePropertyOrImplicitGetter(PseudoObjectExpr *PseudoOp) {
+  SourceRange OldRange = PseudoOp->getSourceRange();
+
+  // We just magically know some things about the structure of this
+  // expression.
+  ObjCMessageExpr *OldMsg =
+    cast<ObjCMessageExpr>(PseudoOp->getResultExpr()->IgnoreImplicit());
+
+  // Because the rewriter doesn't allow us to rewrite rewritten code,
+  // we need to suppress rewriting the sub-statements.
+  Expr *Base = 0;
+  SmallVector<Expr*, 1> Args;
+  {
+    DisableReplaceStmtScope S(*this);
+    // Rebuild the base expression if we have one.
+    if (OldMsg->getReceiverKind() == ObjCMessageExpr::Instance) {
+      Base = OldMsg->getInstanceReceiver();
+      Base = cast<OpaqueValueExpr>(Base)->getSourceExpr();
+      Base = cast<Expr>(RewriteFunctionBodyOrGlobalInitializer(Base));
+    }
+    unsigned numArgs = OldMsg->getNumArgs();
+    for (unsigned i = 0; i < numArgs; i++) {
+      Expr *Arg = OldMsg->getArg(i);
+      if (isa<OpaqueValueExpr>(Arg))
+        Arg = cast<OpaqueValueExpr>(Arg)->getSourceExpr();
+      Arg = cast<Expr>(RewriteFunctionBodyOrGlobalInitializer(Arg));
+      Args.push_back(Arg);
+    }
+  }
+
+  // Intentionally empty.
+  SmallVector<SourceLocation, 1> SelLocs;
+
+  ObjCMessageExpr *NewMsg = 0;
+  switch (OldMsg->getReceiverKind()) {
+  case ObjCMessageExpr::Class:
+    NewMsg = ObjCMessageExpr::Create(*Context, OldMsg->getType(),
+                                     OldMsg->getValueKind(),
+                                     OldMsg->getLeftLoc(),
+                                     OldMsg->getClassReceiverTypeInfo(),
+                                     OldMsg->getSelector(),
+                                     SelLocs,
+                                     OldMsg->getMethodDecl(),
+                                     Args,
+                                     OldMsg->getRightLoc(),
+                                     OldMsg->isImplicit());
+    break;
+
+  case ObjCMessageExpr::Instance:
+    NewMsg = ObjCMessageExpr::Create(*Context, OldMsg->getType(),
+                                     OldMsg->getValueKind(),
+                                     OldMsg->getLeftLoc(),
+                                     Base,
+                                     OldMsg->getSelector(),
+                                     SelLocs,
+                                     OldMsg->getMethodDecl(),
+                                     Args,
+                                     OldMsg->getRightLoc(),
+                                     OldMsg->isImplicit());
+    break;
+
+  case ObjCMessageExpr::SuperClass:
+  case ObjCMessageExpr::SuperInstance:
+    NewMsg = ObjCMessageExpr::Create(*Context, OldMsg->getType(),
+                                     OldMsg->getValueKind(),
+                                     OldMsg->getLeftLoc(),
+                                     OldMsg->getSuperLoc(),
+                 OldMsg->getReceiverKind() == ObjCMessageExpr::SuperInstance,
+                                     OldMsg->getSuperType(),
+                                     OldMsg->getSelector(),
+                                     SelLocs,
+                                     OldMsg->getMethodDecl(),
+                                     Args,
+                                     OldMsg->getRightLoc(),
+                                     OldMsg->isImplicit());
+    break;
+  }
+
+  Stmt *Replacement = SynthMessageExpr(NewMsg);
+  ReplaceStmtWithRange(PseudoOp, Replacement, OldRange);
+  return Replacement;
+}
+
+/// SynthCountByEnumWithState - To print:
+/// ((unsigned int (*)
+///  (id, SEL, struct __objcFastEnumerationState *, id *, unsigned int))
+///  (void *)objc_msgSend)((id)l_collection,
+///                        sel_registerName(
+///                          "countByEnumeratingWithState:objects:count:"),
+///                        &enumState,
+///                        (id *)__rw_items, (unsigned int)16)
+///
+void RewriteModernObjC::SynthCountByEnumWithState(std::string &buf) {
+  buf += "((unsigned int (*) (id, SEL, struct __objcFastEnumerationState *, "
+  "id *, unsigned int))(void *)objc_msgSend)";
+  buf += "\n\t\t";
+  buf += "((id)l_collection,\n\t\t";
+  buf += "sel_registerName(\"countByEnumeratingWithState:objects:count:\"),";
+  buf += "\n\t\t";
+  buf += "&enumState, "
+         "(id *)__rw_items, (unsigned int)16)";
+}
+
+/// RewriteBreakStmt - Rewrite for a break-stmt inside an ObjC2's foreach
+/// statement to exit to its outer synthesized loop.
+///
+Stmt *RewriteModernObjC::RewriteBreakStmt(BreakStmt *S) {
+  if (Stmts.empty() || !isa<ObjCForCollectionStmt>(Stmts.back()))
+    return S;
+  // replace break with goto __break_label
+  std::string buf;
+
+  SourceLocation startLoc = S->getLocStart();
+  buf = "goto __break_label_";
+  buf += utostr(ObjCBcLabelNo.back());
+  ReplaceText(startLoc, strlen("break"), buf);
+
+  return 0;
+}
+
+void RewriteModernObjC::ConvertSourceLocationToLineDirective(
+                                          SourceLocation Loc,
+                                          std::string &LineString) {
+  if (Loc.isFileID()) {
+    LineString += "\n#line ";
+    PresumedLoc PLoc = SM->getPresumedLoc(Loc);
+    LineString += utostr(PLoc.getLine());
+    LineString += " \"";
+    LineString += Lexer::Stringify(PLoc.getFilename());
+    LineString += "\"\n";
+  }
+}
+
+/// RewriteContinueStmt - Rewrite for a continue-stmt inside an ObjC2's foreach
+/// statement to continue with its inner synthesized loop.
+///
+Stmt *RewriteModernObjC::RewriteContinueStmt(ContinueStmt *S) {
+  if (Stmts.empty() || !isa<ObjCForCollectionStmt>(Stmts.back()))
+    return S;
+  // replace continue with goto __continue_label
+  std::string buf;
+
+  SourceLocation startLoc = S->getLocStart();
+  buf = "goto __continue_label_";
+  buf += utostr(ObjCBcLabelNo.back());
+  ReplaceText(startLoc, strlen("continue"), buf);
+
+  return 0;
+}
+
+/// RewriteObjCForCollectionStmt - Rewriter for ObjC2's foreach statement.
+///  It rewrites:
+/// for ( type elem in collection) { stmts; }
+
+/// Into:
+/// {
+///   type elem;
+///   struct __objcFastEnumerationState enumState = { 0 };
+///   id __rw_items[16];
+///   id l_collection = (id)collection;
+///   unsigned long limit = [l_collection countByEnumeratingWithState:&enumState
+///                                       objects:__rw_items count:16];
+/// if (limit) {
+///   unsigned long startMutations = *enumState.mutationsPtr;
+///   do {
+///        unsigned long counter = 0;
+///        do {
+///             if (startMutations != *enumState.mutationsPtr)
+///               objc_enumerationMutation(l_collection);
+///             elem = (type)enumState.itemsPtr[counter++];
+///             stmts;
+///             __continue_label: ;
+///        } while (counter < limit);
+///   } while (limit = [l_collection countByEnumeratingWithState:&enumState
+///                                  objects:__rw_items count:16]);
+///   elem = nil;
+///   __break_label: ;
+///  }
+///  else
+///       elem = nil;
+///  }
+///
+Stmt *RewriteModernObjC::RewriteObjCForCollectionStmt(ObjCForCollectionStmt *S,
+                                                SourceLocation OrigEnd) {
+  assert(!Stmts.empty() && "ObjCForCollectionStmt - Statement stack empty");
+  assert(isa<ObjCForCollectionStmt>(Stmts.back()) &&
+         "ObjCForCollectionStmt Statement stack mismatch");
+  assert(!ObjCBcLabelNo.empty() &&
+         "ObjCForCollectionStmt - Label No stack empty");
+
+  SourceLocation startLoc = S->getLocStart();
+  const char *startBuf = SM->getCharacterData(startLoc);
+  StringRef elementName;
+  std::string elementTypeAsString;
+  std::string buf;
+  // line directive first.
+  SourceLocation ForEachLoc = S->getForLoc();
+  ConvertSourceLocationToLineDirective(ForEachLoc, buf);
+  buf += "{\n\t";
+  if (DeclStmt *DS = dyn_cast<DeclStmt>(S->getElement())) {
+    // type elem;
+    NamedDecl* D = cast<NamedDecl>(DS->getSingleDecl());
+    QualType ElementType = cast<ValueDecl>(D)->getType();
+    if (ElementType->isObjCQualifiedIdType() ||
+        ElementType->isObjCQualifiedInterfaceType())
+      // Simply use 'id' for all qualified types.
+      elementTypeAsString = "id";
+    else
+      elementTypeAsString = ElementType.getAsString(Context->getPrintingPolicy());
+    buf += elementTypeAsString;
+    buf += " ";
+    elementName = D->getName();
+    buf += elementName;
+    buf += ";\n\t";
+  }
+  else {
+    DeclRefExpr *DR = cast<DeclRefExpr>(S->getElement());
+    elementName = DR->getDecl()->getName();
+    ValueDecl *VD = cast<ValueDecl>(DR->getDecl());
+    if (VD->getType()->isObjCQualifiedIdType() ||
+        VD->getType()->isObjCQualifiedInterfaceType())
+      // Simply use 'id' for all qualified types.
+      elementTypeAsString = "id";
+    else
+      elementTypeAsString = VD->getType().getAsString(Context->getPrintingPolicy());
+  }
+
+  // struct __objcFastEnumerationState enumState = { 0 };
+  buf += "struct __objcFastEnumerationState enumState = { 0 };\n\t";
+  // id __rw_items[16];
+  buf += "id __rw_items[16];\n\t";
+  // id l_collection = (id)
+  buf += "id l_collection = (id)";
+  // Find start location of 'collection' the hard way!
+  const char *startCollectionBuf = startBuf;
+  startCollectionBuf += 3;  // skip 'for'
+  startCollectionBuf = strchr(startCollectionBuf, '(');
+  startCollectionBuf++; // skip '('
+  // find 'in' and skip it.
+  while (*startCollectionBuf != ' ' ||
+         *(startCollectionBuf+1) != 'i' || *(startCollectionBuf+2) != 'n' ||
+         (*(startCollectionBuf+3) != ' ' &&
+          *(startCollectionBuf+3) != '[' && *(startCollectionBuf+3) != '('))
+    startCollectionBuf++;
+  startCollectionBuf += 3;
+
+  // Replace: "for (type element in" with string constructed thus far.
+  ReplaceText(startLoc, startCollectionBuf - startBuf, buf);
+  // Replace ')' in for '(' type elem in collection ')' with ';'
+  SourceLocation rightParenLoc = S->getRParenLoc();
+  const char *rparenBuf = SM->getCharacterData(rightParenLoc);
+  SourceLocation lparenLoc = startLoc.getLocWithOffset(rparenBuf-startBuf);
+  buf = ";\n\t";
+
+  // unsigned long limit = [l_collection countByEnumeratingWithState:&enumState
+  //                                   objects:__rw_items count:16];
+  // which is synthesized into:
+  // unsigned int limit =
+  // ((unsigned int (*)
+  //  (id, SEL, struct __objcFastEnumerationState *, id *, unsigned int))
+  //  (void *)objc_msgSend)((id)l_collection,
+  //                        sel_registerName(
+  //                          "countByEnumeratingWithState:objects:count:"),
+  //                        (struct __objcFastEnumerationState *)&state,
+  //                        (id *)__rw_items, (unsigned int)16);
+  buf += "unsigned long limit =\n\t\t";
+  SynthCountByEnumWithState(buf);
+  buf += ";\n\t";
+  /// if (limit) {
+  ///   unsigned long startMutations = *enumState.mutationsPtr;
+  ///   do {
+  ///        unsigned long counter = 0;
+  ///        do {
+  ///             if (startMutations != *enumState.mutationsPtr)
+  ///               objc_enumerationMutation(l_collection);
+  ///             elem = (type)enumState.itemsPtr[counter++];
+  buf += "if (limit) {\n\t";
+  buf += "unsigned long startMutations = *enumState.mutationsPtr;\n\t";
+  buf += "do {\n\t\t";
+  buf += "unsigned long counter = 0;\n\t\t";
+  buf += "do {\n\t\t\t";
+  buf += "if (startMutations != *enumState.mutationsPtr)\n\t\t\t\t";
+  buf += "objc_enumerationMutation(l_collection);\n\t\t\t";
+  buf += elementName;
+  buf += " = (";
+  buf += elementTypeAsString;
+  buf += ")enumState.itemsPtr[counter++];";
+  // Replace ')' in for '(' type elem in collection ')' with all of these.
+  ReplaceText(lparenLoc, 1, buf);
+
+  ///            __continue_label: ;
+  ///        } while (counter < limit);
+  ///   } while (limit = [l_collection countByEnumeratingWithState:&enumState
+  ///                                  objects:__rw_items count:16]);
+  ///   elem = nil;
+  ///   __break_label: ;
+  ///  }
+  ///  else
+  ///       elem = nil;
+  ///  }
+  ///
+  buf = ";\n\t";
+  buf += "__continue_label_";
+  buf += utostr(ObjCBcLabelNo.back());
+  buf += ": ;";
+  buf += "\n\t\t";
+  buf += "} while (counter < limit);\n\t";
+  buf += "} while (limit = ";
+  SynthCountByEnumWithState(buf);
+  buf += ");\n\t";
+  buf += elementName;
+  buf += " = ((";
+  buf += elementTypeAsString;
+  buf += ")0);\n\t";
+  buf += "__break_label_";
+  buf += utostr(ObjCBcLabelNo.back());
+  buf += ": ;\n\t";
+  buf += "}\n\t";
+  buf += "else\n\t\t";
+  buf += elementName;
+  buf += " = ((";
+  buf += elementTypeAsString;
+  buf += ")0);\n\t";
+  buf += "}\n";
+
+  // Insert all these *after* the statement body.
+  // FIXME: If this should support Obj-C++, support CXXTryStmt
+  if (isa<CompoundStmt>(S->getBody())) {
+    SourceLocation endBodyLoc = OrigEnd.getLocWithOffset(1);
+    InsertText(endBodyLoc, buf);
+  } else {
+    /* Need to treat single statements specially. For example:
+     *
+     *     for (A *a in b) if (stuff()) break;
+     *     for (A *a in b) xxxyy;
+     *
+     * The following code simply scans ahead to the semi to find the actual end.
+     */
+    const char *stmtBuf = SM->getCharacterData(OrigEnd);
+    const char *semiBuf = strchr(stmtBuf, ';');
+    assert(semiBuf && "Can't find ';'");
+    SourceLocation endBodyLoc = OrigEnd.getLocWithOffset(semiBuf-stmtBuf+1);
+    InsertText(endBodyLoc, buf);
+  }
+  Stmts.pop_back();
+  ObjCBcLabelNo.pop_back();
+  return 0;
+}
+
+static void Write_RethrowObject(std::string &buf) {
+  buf += "{ struct _FIN { _FIN(id reth) : rethrow(reth) {}\n";
+  buf += "\t~_FIN() { if (rethrow) objc_exception_throw(rethrow); }\n";
+  buf += "\tid rethrow;\n";
+  buf += "\t} _fin_force_rethow(_rethrow);";
+}
+
+/// RewriteObjCSynchronizedStmt -
+/// This routine rewrites @synchronized(expr) stmt;
+/// into:
+/// objc_sync_enter(expr);
+/// @try stmt @finally { objc_sync_exit(expr); }
+///
+Stmt *RewriteModernObjC::RewriteObjCSynchronizedStmt(ObjCAtSynchronizedStmt *S) {
+  // Get the start location and compute the semi location.
+  SourceLocation startLoc = S->getLocStart();
+  const char *startBuf = SM->getCharacterData(startLoc);
+
+  assert((*startBuf == '@') && "bogus @synchronized location");
+
+  std::string buf;
+  SourceLocation SynchLoc = S->getAtSynchronizedLoc();
+  ConvertSourceLocationToLineDirective(SynchLoc, buf);
+  buf += "{ id _rethrow = 0; id _sync_obj = ";
+  
+  const char *lparenBuf = startBuf;
+  while (*lparenBuf != '(') lparenBuf++;
+  ReplaceText(startLoc, lparenBuf-startBuf+1, buf);
+  
+  buf = "; objc_sync_enter(_sync_obj);\n";
+  buf += "try {\n\tstruct _SYNC_EXIT { _SYNC_EXIT(id arg) : sync_exit(arg) {}";
+  buf += "\n\t~_SYNC_EXIT() {objc_sync_exit(sync_exit);}";
+  buf += "\n\tid sync_exit;";
+  buf += "\n\t} _sync_exit(_sync_obj);\n";
+
+  // We can't use S->getSynchExpr()->getLocEnd() to find the end location, since
+  // the sync expression is typically a message expression that's already
+  // been rewritten! (which implies the SourceLocation's are invalid).
+  SourceLocation RParenExprLoc = S->getSynchBody()->getLocStart();
+  const char *RParenExprLocBuf = SM->getCharacterData(RParenExprLoc);
+  while (*RParenExprLocBuf != ')') RParenExprLocBuf--;
+  RParenExprLoc = startLoc.getLocWithOffset(RParenExprLocBuf-startBuf);
+  
+  SourceLocation LBranceLoc = S->getSynchBody()->getLocStart();
+  const char *LBraceLocBuf = SM->getCharacterData(LBranceLoc);
+  assert (*LBraceLocBuf == '{');
+  ReplaceText(RParenExprLoc, (LBraceLocBuf - SM->getCharacterData(RParenExprLoc) + 1), buf);
+  
+  SourceLocation startRBraceLoc = S->getSynchBody()->getLocEnd();
+  assert((*SM->getCharacterData(startRBraceLoc) == '}') &&
+         "bogus @synchronized block");
+  
+  buf = "} catch (id e) {_rethrow = e;}\n";
+  Write_RethrowObject(buf);
+  buf += "}\n";
+  buf += "}\n";
+
+  ReplaceText(startRBraceLoc, 1, buf);
+
+  return 0;
+}
+
+void RewriteModernObjC::WarnAboutReturnGotoStmts(Stmt *S)
+{
+  // Perform a bottom up traversal of all children.
+  for (Stmt::child_range CI = S->children(); CI; ++CI)
+    if (*CI)
+      WarnAboutReturnGotoStmts(*CI);
+
+  if (isa<ReturnStmt>(S) || isa<GotoStmt>(S)) {
+    Diags.Report(Context->getFullLoc(S->getLocStart()),
+                 TryFinallyContainsReturnDiag);
+  }
+  return;
+}
+
+Stmt *RewriteModernObjC::RewriteObjCAutoreleasePoolStmt(ObjCAutoreleasePoolStmt  *S) {
+  SourceLocation startLoc = S->getAtLoc();
+  ReplaceText(startLoc, strlen("@autoreleasepool"), "/* @autoreleasepool */");
+  ReplaceText(S->getSubStmt()->getLocStart(), 1, 
+              "{ __AtAutoreleasePool __autoreleasepool; ");
+  
+  return 0;
+}
+
+Stmt *RewriteModernObjC::RewriteObjCTryStmt(ObjCAtTryStmt *S) {
+  ObjCAtFinallyStmt *finalStmt = S->getFinallyStmt();
+  bool noCatch = S->getNumCatchStmts() == 0;
+  std::string buf;
+  SourceLocation TryLocation = S->getAtTryLoc();
+  ConvertSourceLocationToLineDirective(TryLocation, buf);
+  
+  if (finalStmt) {
+    if (noCatch)
+      buf += "{ id volatile _rethrow = 0;\n";
+    else {
+      buf += "{ id volatile _rethrow = 0;\ntry {\n";
+    }
+  }
+  // Get the start location and compute the semi location.
+  SourceLocation startLoc = S->getLocStart();
+  const char *startBuf = SM->getCharacterData(startLoc);
+
+  assert((*startBuf == '@') && "bogus @try location");
+  if (finalStmt)
+    ReplaceText(startLoc, 1, buf);
+  else
+    // @try -> try
+    ReplaceText(startLoc, 1, "");
+  
+  for (unsigned I = 0, N = S->getNumCatchStmts(); I != N; ++I) {
+    ObjCAtCatchStmt *Catch = S->getCatchStmt(I);
+    VarDecl *catchDecl = Catch->getCatchParamDecl();
+    
+    startLoc = Catch->getLocStart();
+    bool AtRemoved = false;
+    if (catchDecl) {
+      QualType t = catchDecl->getType();
+      if (const ObjCObjectPointerType *Ptr = t->getAs<ObjCObjectPointerType>()) {
+        // Should be a pointer to a class.
+        ObjCInterfaceDecl *IDecl = Ptr->getObjectType()->getInterface();
+        if (IDecl) {
+          std::string Result;
+          ConvertSourceLocationToLineDirective(Catch->getLocStart(), Result);
+          
+          startBuf = SM->getCharacterData(startLoc);
+          assert((*startBuf == '@') && "bogus @catch location");
+          SourceLocation rParenLoc = Catch->getRParenLoc();
+          const char *rParenBuf = SM->getCharacterData(rParenLoc);
+          
+          // _objc_exc_Foo *_e as argument to catch.
+          Result += "catch (_objc_exc_"; Result += IDecl->getNameAsString();
+          Result += " *_"; Result += catchDecl->getNameAsString();
+          Result += ")";
+          ReplaceText(startLoc, rParenBuf-startBuf+1, Result);
+          // Foo *e = (Foo *)_e;
+          Result.clear();
+          Result = "{ ";
+          Result += IDecl->getNameAsString();
+          Result += " *"; Result += catchDecl->getNameAsString();
+          Result += " = ("; Result += IDecl->getNameAsString(); Result += "*)";
+          Result += "_"; Result += catchDecl->getNameAsString();
+          
+          Result += "; ";
+          SourceLocation lBraceLoc = Catch->getCatchBody()->getLocStart();
+          ReplaceText(lBraceLoc, 1, Result);
+          AtRemoved = true;
+        }
+      }
+    }
+    if (!AtRemoved)
+      // @catch -> catch
+      ReplaceText(startLoc, 1, "");
+      
+  }
+  if (finalStmt) {
+    buf.clear();
+    SourceLocation FinallyLoc = finalStmt->getLocStart();
+    
+    if (noCatch) {
+      ConvertSourceLocationToLineDirective(FinallyLoc, buf);
+      buf += "catch (id e) {_rethrow = e;}\n";
+    }
+    else {
+      buf += "}\n";
+      ConvertSourceLocationToLineDirective(FinallyLoc, buf);
+      buf += "catch (id e) {_rethrow = e;}\n";
+    }
+    
+    SourceLocation startFinalLoc = finalStmt->getLocStart();
+    ReplaceText(startFinalLoc, 8, buf);
+    Stmt *body = finalStmt->getFinallyBody();
+    SourceLocation startFinalBodyLoc = body->getLocStart();
+    buf.clear();
+    Write_RethrowObject(buf);
+    ReplaceText(startFinalBodyLoc, 1, buf);
+    
+    SourceLocation endFinalBodyLoc = body->getLocEnd();
+    ReplaceText(endFinalBodyLoc, 1, "}\n}");
+    // Now check for any return/continue/go statements within the @try.
+    WarnAboutReturnGotoStmts(S->getTryBody());
+  }
+
+  return 0;
+}
+
+// This can't be done with ReplaceStmt(S, ThrowExpr), since
+// the throw expression is typically a message expression that's already
+// been rewritten! (which implies the SourceLocation's are invalid).
+Stmt *RewriteModernObjC::RewriteObjCThrowStmt(ObjCAtThrowStmt *S) {
+  // Get the start location and compute the semi location.
+  SourceLocation startLoc = S->getLocStart();
+  const char *startBuf = SM->getCharacterData(startLoc);
+
+  assert((*startBuf == '@') && "bogus @throw location");
+
+  std::string buf;
+  /* void objc_exception_throw(id) __attribute__((noreturn)); */
+  if (S->getThrowExpr())
+    buf = "objc_exception_throw(";
+  else
+    buf = "throw";
+
+  // handle "@  throw" correctly.
+  const char *wBuf = strchr(startBuf, 'w');
+  assert((*wBuf == 'w') && "@throw: can't find 'w'");
+  ReplaceText(startLoc, wBuf-startBuf+1, buf);
+
+  const char *semiBuf = strchr(startBuf, ';');
+  assert((*semiBuf == ';') && "@throw: can't find ';'");
+  SourceLocation semiLoc = startLoc.getLocWithOffset(semiBuf-startBuf);
+  if (S->getThrowExpr())
+    ReplaceText(semiLoc, 1, ");");
+  return 0;
+}
+
+Stmt *RewriteModernObjC::RewriteAtEncode(ObjCEncodeExpr *Exp) {
+  // Create a new string expression.
+  QualType StrType = Context->getPointerType(Context->CharTy);
+  std::string StrEncoding;
+  Context->getObjCEncodingForType(Exp->getEncodedType(), StrEncoding);
+  Expr *Replacement = StringLiteral::Create(*Context, StrEncoding,
+                                            StringLiteral::Ascii, false,
+                                            StrType, SourceLocation());
+  ReplaceStmt(Exp, Replacement);
+
+  // Replace this subexpr in the parent.
+  // delete Exp; leak for now, see RewritePropertyOrImplicitSetter() usage for more info.
+  return Replacement;
+}
+
+Stmt *RewriteModernObjC::RewriteAtSelector(ObjCSelectorExpr *Exp) {
+  if (!SelGetUidFunctionDecl)
+    SynthSelGetUidFunctionDecl();
+  assert(SelGetUidFunctionDecl && "Can't find sel_registerName() decl");
+  // Create a call to sel_registerName("selName").
+  SmallVector<Expr*, 8> SelExprs;
+  QualType argType = Context->getPointerType(Context->CharTy);
+  SelExprs.push_back(StringLiteral::Create(*Context,
+                                           Exp->getSelector().getAsString(),
+                                           StringLiteral::Ascii, false,
+                                           argType, SourceLocation()));
+  CallExpr *SelExp = SynthesizeCallToFunctionDecl(SelGetUidFunctionDecl,
+                                                 &SelExprs[0], SelExprs.size());
+  ReplaceStmt(Exp, SelExp);
+  // delete Exp; leak for now, see RewritePropertyOrImplicitSetter() usage for more info.
+  return SelExp;
+}
+
+CallExpr *RewriteModernObjC::SynthesizeCallToFunctionDecl(
+  FunctionDecl *FD, Expr **args, unsigned nargs, SourceLocation StartLoc,
+                                                    SourceLocation EndLoc) {
+  // Get the type, we will need to reference it in a couple spots.
+  QualType msgSendType = FD->getType();
+
+  // Create a reference to the objc_msgSend() declaration.
+  DeclRefExpr *DRE =
+    new (Context) DeclRefExpr(FD, false, msgSendType, VK_LValue, SourceLocation());
+
+  // Now, we cast the reference to a pointer to the objc_msgSend type.
+  QualType pToFunc = Context->getPointerType(msgSendType);
+  ImplicitCastExpr *ICE = 
+    ImplicitCastExpr::Create(*Context, pToFunc, CK_FunctionToPointerDecay,
+                             DRE, 0, VK_RValue);
+
+  const FunctionType *FT = msgSendType->getAs<FunctionType>();
+
+  CallExpr *Exp =  
+    new (Context) CallExpr(*Context, ICE, llvm::makeArrayRef(args, nargs),
+                           FT->getCallResultType(*Context),
+                           VK_RValue, EndLoc);
+  return Exp;
+}
+
+static bool scanForProtocolRefs(const char *startBuf, const char *endBuf,
+                                const char *&startRef, const char *&endRef) {
+  while (startBuf < endBuf) {
+    if (*startBuf == '<')
+      startRef = startBuf; // mark the start.
+    if (*startBuf == '>') {
+      if (startRef && *startRef == '<') {
+        endRef = startBuf; // mark the end.
+        return true;
+      }
+      return false;
+    }
+    startBuf++;
+  }
+  return false;
+}
+
+static void scanToNextArgument(const char *&argRef) {
+  int angle = 0;
+  while (*argRef != ')' && (*argRef != ',' || angle > 0)) {
+    if (*argRef == '<')
+      angle++;
+    else if (*argRef == '>')
+      angle--;
+    argRef++;
+  }
+  assert(angle == 0 && "scanToNextArgument - bad protocol type syntax");
+}
+
+bool RewriteModernObjC::needToScanForQualifiers(QualType T) {
+  if (T->isObjCQualifiedIdType())
+    return true;
+  if (const PointerType *PT = T->getAs<PointerType>()) {
+    if (PT->getPointeeType()->isObjCQualifiedIdType())
+      return true;
+  }
+  if (T->isObjCObjectPointerType()) {
+    T = T->getPointeeType();
+    return T->isObjCQualifiedInterfaceType();
+  }
+  if (T->isArrayType()) {
+    QualType ElemTy = Context->getBaseElementType(T);
+    return needToScanForQualifiers(ElemTy);
+  }
+  return false;
+}
+
+void RewriteModernObjC::RewriteObjCQualifiedInterfaceTypes(Expr *E) {
+  QualType Type = E->getType();
+  if (needToScanForQualifiers(Type)) {
+    SourceLocation Loc, EndLoc;
+
+    if (const CStyleCastExpr *ECE = dyn_cast<CStyleCastExpr>(E)) {
+      Loc = ECE->getLParenLoc();
+      EndLoc = ECE->getRParenLoc();
+    } else {
+      Loc = E->getLocStart();
+      EndLoc = E->getLocEnd();
+    }
+    // This will defend against trying to rewrite synthesized expressions.
+    if (Loc.isInvalid() || EndLoc.isInvalid())
+      return;
+
+    const char *startBuf = SM->getCharacterData(Loc);
+    const char *endBuf = SM->getCharacterData(EndLoc);
+    const char *startRef = 0, *endRef = 0;
+    if (scanForProtocolRefs(startBuf, endBuf, startRef, endRef)) {
+      // Get the locations of the startRef, endRef.
+      SourceLocation LessLoc = Loc.getLocWithOffset(startRef-startBuf);
+      SourceLocation GreaterLoc = Loc.getLocWithOffset(endRef-startBuf+1);
+      // Comment out the protocol references.
+      InsertText(LessLoc, "/*");
+      InsertText(GreaterLoc, "*/");
+    }
+  }
+}
+
+void RewriteModernObjC::RewriteObjCQualifiedInterfaceTypes(Decl *Dcl) {
+  SourceLocation Loc;
+  QualType Type;
+  const FunctionProtoType *proto = 0;
+  if (VarDecl *VD = dyn_cast<VarDecl>(Dcl)) {
+    Loc = VD->getLocation();
+    Type = VD->getType();
+  }
+  else if (FunctionDecl *FD = dyn_cast<FunctionDecl>(Dcl)) {
+    Loc = FD->getLocation();
+    // Check for ObjC 'id' and class types that have been adorned with protocol
+    // information (id<p>, C<p>*). The protocol references need to be rewritten!
+    const FunctionType *funcType = FD->getType()->getAs<FunctionType>();
+    assert(funcType && "missing function type");
+    proto = dyn_cast<FunctionProtoType>(funcType);
+    if (!proto)
+      return;
+    Type = proto->getResultType();
+  }
+  else if (FieldDecl *FD = dyn_cast<FieldDecl>(Dcl)) {
+    Loc = FD->getLocation();
+    Type = FD->getType();
+  }
+  else
+    return;
+
+  if (needToScanForQualifiers(Type)) {
+    // Since types are unique, we need to scan the buffer.
+
+    const char *endBuf = SM->getCharacterData(Loc);
+    const char *startBuf = endBuf;
+    while (*startBuf != ';' && *startBuf != '<' && startBuf != MainFileStart)
+      startBuf--; // scan backward (from the decl location) for return type.
+    const char *startRef = 0, *endRef = 0;
+    if (scanForProtocolRefs(startBuf, endBuf, startRef, endRef)) {
+      // Get the locations of the startRef, endRef.
+      SourceLocation LessLoc = Loc.getLocWithOffset(startRef-endBuf);
+      SourceLocation GreaterLoc = Loc.getLocWithOffset(endRef-endBuf+1);
+      // Comment out the protocol references.
+      InsertText(LessLoc, "/*");
+      InsertText(GreaterLoc, "*/");
+    }
+  }
+  if (!proto)
+      return; // most likely, was a variable
+  // Now check arguments.
+  const char *startBuf = SM->getCharacterData(Loc);
+  const char *startFuncBuf = startBuf;
+  for (unsigned i = 0; i < proto->getNumArgs(); i++) {
+    if (needToScanForQualifiers(proto->getArgType(i))) {
+      // Since types are unique, we need to scan the buffer.
+
+      const char *endBuf = startBuf;
+      // scan forward (from the decl location) for argument types.
+      scanToNextArgument(endBuf);
+      const char *startRef = 0, *endRef = 0;
+      if (scanForProtocolRefs(startBuf, endBuf, startRef, endRef)) {
+        // Get the locations of the startRef, endRef.
+        SourceLocation LessLoc =
+          Loc.getLocWithOffset(startRef-startFuncBuf);
+        SourceLocation GreaterLoc =
+          Loc.getLocWithOffset(endRef-startFuncBuf+1);
+        // Comment out the protocol references.
+        InsertText(LessLoc, "/*");
+        InsertText(GreaterLoc, "*/");
+      }
+      startBuf = ++endBuf;
+    }
+    else {
+      // If the function name is derived from a macro expansion, then the
+      // argument buffer will not follow the name. Need to speak with Chris.
+      while (*startBuf && *startBuf != ')' && *startBuf != ',')
+        startBuf++; // scan forward (from the decl location) for argument types.
+      startBuf++;
+    }
+  }
+}
+
+void RewriteModernObjC::RewriteTypeOfDecl(VarDecl *ND) {
+  QualType QT = ND->getType();
+  const Type* TypePtr = QT->getAs<Type>();
+  if (!isa<TypeOfExprType>(TypePtr))
+    return;
+  while (isa<TypeOfExprType>(TypePtr)) {
+    const TypeOfExprType *TypeOfExprTypePtr = cast<TypeOfExprType>(TypePtr);
+    QT = TypeOfExprTypePtr->getUnderlyingExpr()->getType();
+    TypePtr = QT->getAs<Type>();
+  }
+  // FIXME. This will not work for multiple declarators; as in:
+  // __typeof__(a) b,c,d;
+  std::string TypeAsString(QT.getAsString(Context->getPrintingPolicy()));
+  SourceLocation DeclLoc = ND->getTypeSpecStartLoc();
+  const char *startBuf = SM->getCharacterData(DeclLoc);
+  if (ND->getInit()) {
+    std::string Name(ND->getNameAsString());
+    TypeAsString += " " + Name + " = ";
+    Expr *E = ND->getInit();
+    SourceLocation startLoc;
+    if (const CStyleCastExpr *ECE = dyn_cast<CStyleCastExpr>(E))
+      startLoc = ECE->getLParenLoc();
+    else
+      startLoc = E->getLocStart();
+    startLoc = SM->getExpansionLoc(startLoc);
+    const char *endBuf = SM->getCharacterData(startLoc);
+    ReplaceText(DeclLoc, endBuf-startBuf-1, TypeAsString);
+  }
+  else {
+    SourceLocation X = ND->getLocEnd();
+    X = SM->getExpansionLoc(X);
+    const char *endBuf = SM->getCharacterData(X);
+    ReplaceText(DeclLoc, endBuf-startBuf-1, TypeAsString);
+  }
+}
+
+// SynthSelGetUidFunctionDecl - SEL sel_registerName(const char *str);
+void RewriteModernObjC::SynthSelGetUidFunctionDecl() {
+  IdentifierInfo *SelGetUidIdent = &Context->Idents.get("sel_registerName");
+  SmallVector<QualType, 16> ArgTys;
+  ArgTys.push_back(Context->getPointerType(Context->CharTy.withConst()));
+  QualType getFuncType =
+    getSimpleFunctionType(Context->getObjCSelType(), &ArgTys[0], ArgTys.size());
+  SelGetUidFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
+                                           SourceLocation(),
+                                           SourceLocation(),
+                                           SelGetUidIdent, getFuncType, 0,
+                                           SC_Extern,
+                                           SC_None, false);
+}
+
+void RewriteModernObjC::RewriteFunctionDecl(FunctionDecl *FD) {
+  // declared in <objc/objc.h>
+  if (FD->getIdentifier() &&
+      FD->getName() == "sel_registerName") {
+    SelGetUidFunctionDecl = FD;
+    return;
+  }
+  RewriteObjCQualifiedInterfaceTypes(FD);
+}
+
+void RewriteModernObjC::RewriteBlockPointerType(std::string& Str, QualType Type) {
+  std::string TypeString(Type.getAsString(Context->getPrintingPolicy()));
+  const char *argPtr = TypeString.c_str();
+  if (!strchr(argPtr, '^')) {
+    Str += TypeString;
+    return;
+  }
+  while (*argPtr) {
+    Str += (*argPtr == '^' ? '*' : *argPtr);
+    argPtr++;
+  }
+}
+
+// FIXME. Consolidate this routine with RewriteBlockPointerType.
+void RewriteModernObjC::RewriteBlockPointerTypeVariable(std::string& Str,
+                                                  ValueDecl *VD) {
+  QualType Type = VD->getType();
+  std::string TypeString(Type.getAsString(Context->getPrintingPolicy()));
+  const char *argPtr = TypeString.c_str();
+  int paren = 0;
+  while (*argPtr) {
+    switch (*argPtr) {
+      case '(':
+        Str += *argPtr;
+        paren++;
+        break;
+      case ')':
+        Str += *argPtr;
+        paren--;
+        break;
+      case '^':
+        Str += '*';
+        if (paren == 1)
+          Str += VD->getNameAsString();
+        break;
+      default:
+        Str += *argPtr;
+        break;
+    }
+    argPtr++;
+  }
+}
+
+void RewriteModernObjC::RewriteBlockLiteralFunctionDecl(FunctionDecl *FD) {
+  SourceLocation FunLocStart = FD->getTypeSpecStartLoc();
+  const FunctionType *funcType = FD->getType()->getAs<FunctionType>();
+  const FunctionProtoType *proto = dyn_cast<FunctionProtoType>(funcType);
+  if (!proto)
+    return;
+  QualType Type = proto->getResultType();
+  std::string FdStr = Type.getAsString(Context->getPrintingPolicy());
+  FdStr += " ";
+  FdStr += FD->getName();
+  FdStr +=  "(";
+  unsigned numArgs = proto->getNumArgs();
+  for (unsigned i = 0; i < numArgs; i++) {
+  QualType ArgType = proto->getArgType(i);
+  RewriteBlockPointerType(FdStr, ArgType);
+  if (i+1 < numArgs)
+    FdStr += ", ";
+  }
+  if (FD->isVariadic()) {
+    FdStr +=  (numArgs > 0) ? ", ...);\n" : "...);\n";
+  }
+  else
+    FdStr +=  ");\n";
+  InsertText(FunLocStart, FdStr);
+}
+
+// SynthSuperContructorFunctionDecl - id __rw_objc_super(id obj, id super);
+void RewriteModernObjC::SynthSuperContructorFunctionDecl() {
+  if (SuperContructorFunctionDecl)
+    return;
+  IdentifierInfo *msgSendIdent = &Context->Idents.get("__rw_objc_super");
+  SmallVector<QualType, 16> ArgTys;
+  QualType argT = Context->getObjCIdType();
+  assert(!argT.isNull() && "Can't find 'id' type");
+  ArgTys.push_back(argT);
+  ArgTys.push_back(argT);
+  QualType msgSendType = getSimpleFunctionType(Context->getObjCIdType(),
+                                               &ArgTys[0], ArgTys.size());
+  SuperContructorFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
+                                         SourceLocation(),
+                                         SourceLocation(),
+                                         msgSendIdent, msgSendType, 0,
+                                         SC_Extern,
+                                         SC_None, false);
+}
+
+// SynthMsgSendFunctionDecl - id objc_msgSend(id self, SEL op, ...);
+void RewriteModernObjC::SynthMsgSendFunctionDecl() {
+  IdentifierInfo *msgSendIdent = &Context->Idents.get("objc_msgSend");
+  SmallVector<QualType, 16> ArgTys;
+  QualType argT = Context->getObjCIdType();
+  assert(!argT.isNull() && "Can't find 'id' type");
+  ArgTys.push_back(argT);
+  argT = Context->getObjCSelType();
+  assert(!argT.isNull() && "Can't find 'SEL' type");
+  ArgTys.push_back(argT);
+  QualType msgSendType = getSimpleFunctionType(Context->getObjCIdType(),
+                                               &ArgTys[0], ArgTys.size(),
+                                               true /*isVariadic*/);
+  MsgSendFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
+                                         SourceLocation(),
+                                         SourceLocation(),
+                                         msgSendIdent, msgSendType, 0,
+                                         SC_Extern,
+                                         SC_None, false);
+}
+
+// SynthMsgSendSuperFunctionDecl - id objc_msgSendSuper(void);
+void RewriteModernObjC::SynthMsgSendSuperFunctionDecl() {
+  IdentifierInfo *msgSendIdent = &Context->Idents.get("objc_msgSendSuper");
+  SmallVector<QualType, 2> ArgTys;
+  ArgTys.push_back(Context->VoidTy);
+  QualType msgSendType = getSimpleFunctionType(Context->getObjCIdType(),
+                                               &ArgTys[0], 1,
+                                               true /*isVariadic*/);
+  MsgSendSuperFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
+                                              SourceLocation(),
+                                              SourceLocation(),
+                                              msgSendIdent, msgSendType, 0,
+                                              SC_Extern,
+                                              SC_None, false);
+}
+
+// SynthMsgSendStretFunctionDecl - id objc_msgSend_stret(id self, SEL op, ...);
+void RewriteModernObjC::SynthMsgSendStretFunctionDecl() {
+  IdentifierInfo *msgSendIdent = &Context->Idents.get("objc_msgSend_stret");
+  SmallVector<QualType, 16> ArgTys;
+  QualType argT = Context->getObjCIdType();
+  assert(!argT.isNull() && "Can't find 'id' type");
+  ArgTys.push_back(argT);
+  argT = Context->getObjCSelType();
+  assert(!argT.isNull() && "Can't find 'SEL' type");
+  ArgTys.push_back(argT);
+  QualType msgSendType = getSimpleFunctionType(Context->getObjCIdType(),
+                                               &ArgTys[0], ArgTys.size(),
+                                               true /*isVariadic*/);
+  MsgSendStretFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
+                                         SourceLocation(),
+                                         SourceLocation(),
+                                         msgSendIdent, msgSendType, 0,
+                                         SC_Extern,
+                                         SC_None, false);
+}
+
+// SynthMsgSendSuperStretFunctionDecl -
+// id objc_msgSendSuper_stret(void);
+void RewriteModernObjC::SynthMsgSendSuperStretFunctionDecl() {
+  IdentifierInfo *msgSendIdent =
+    &Context->Idents.get("objc_msgSendSuper_stret");
+  SmallVector<QualType, 2> ArgTys;
+  ArgTys.push_back(Context->VoidTy);
+  QualType msgSendType = getSimpleFunctionType(Context->getObjCIdType(),
+                                               &ArgTys[0], 1,
+                                               true /*isVariadic*/);
+  MsgSendSuperStretFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
+                                                       SourceLocation(),
+                                                       SourceLocation(),
+                                              msgSendIdent, msgSendType, 0,
+                                              SC_Extern,
+                                              SC_None, false);
+}
+
+// SynthMsgSendFpretFunctionDecl - double objc_msgSend_fpret(id self, SEL op, ...);
+void RewriteModernObjC::SynthMsgSendFpretFunctionDecl() {
+  IdentifierInfo *msgSendIdent = &Context->Idents.get("objc_msgSend_fpret");
+  SmallVector<QualType, 16> ArgTys;
+  QualType argT = Context->getObjCIdType();
+  assert(!argT.isNull() && "Can't find 'id' type");
+  ArgTys.push_back(argT);
+  argT = Context->getObjCSelType();
+  assert(!argT.isNull() && "Can't find 'SEL' type");
+  ArgTys.push_back(argT);
+  QualType msgSendType = getSimpleFunctionType(Context->DoubleTy,
+                                               &ArgTys[0], ArgTys.size(),
+                                               true /*isVariadic*/);
+  MsgSendFpretFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
+                                              SourceLocation(),
+                                              SourceLocation(),
+                                              msgSendIdent, msgSendType, 0,
+                                              SC_Extern,
+                                              SC_None, false);
+}
+
+// SynthGetClassFunctionDecl - Class objc_getClass(const char *name);
+void RewriteModernObjC::SynthGetClassFunctionDecl() {
+  IdentifierInfo *getClassIdent = &Context->Idents.get("objc_getClass");
+  SmallVector<QualType, 16> ArgTys;
+  ArgTys.push_back(Context->getPointerType(Context->CharTy.withConst()));
+  QualType getClassType = getSimpleFunctionType(Context->getObjCClassType(),
+                                                &ArgTys[0], ArgTys.size());
+  GetClassFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
+                                          SourceLocation(),
+                                          SourceLocation(),
+                                          getClassIdent, getClassType, 0,
+                                          SC_Extern,
+                                          SC_None, false);
+}
+
+// SynthGetSuperClassFunctionDecl - Class class_getSuperclass(Class cls);
+void RewriteModernObjC::SynthGetSuperClassFunctionDecl() {
+  IdentifierInfo *getSuperClassIdent = 
+    &Context->Idents.get("class_getSuperclass");
+  SmallVector<QualType, 16> ArgTys;
+  ArgTys.push_back(Context->getObjCClassType());
+  QualType getClassType = getSimpleFunctionType(Context->getObjCClassType(),
+                                                &ArgTys[0], ArgTys.size());
+  GetSuperClassFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
+                                                   SourceLocation(),
+                                                   SourceLocation(),
+                                                   getSuperClassIdent,
+                                                   getClassType, 0,
+                                                   SC_Extern,
+                                                   SC_None,
+                                                   false);
+}
+
+// SynthGetMetaClassFunctionDecl - Class objc_getMetaClass(const char *name);
+void RewriteModernObjC::SynthGetMetaClassFunctionDecl() {
+  IdentifierInfo *getClassIdent = &Context->Idents.get("objc_getMetaClass");
+  SmallVector<QualType, 16> ArgTys;
+  ArgTys.push_back(Context->getPointerType(Context->CharTy.withConst()));
+  QualType getClassType = getSimpleFunctionType(Context->getObjCClassType(),
+                                                &ArgTys[0], ArgTys.size());
+  GetMetaClassFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
+                                              SourceLocation(),
+                                              SourceLocation(),
+                                              getClassIdent, getClassType, 0,
+                                              SC_Extern,
+                                              SC_None, false);
+}
+
+Stmt *RewriteModernObjC::RewriteObjCStringLiteral(ObjCStringLiteral *Exp) {
+  QualType strType = getConstantStringStructType();
+
+  std::string S = "__NSConstantStringImpl_";
+
+  std::string tmpName = InFileName;
+  unsigned i;
+  for (i=0; i < tmpName.length(); i++) {
+    char c = tmpName.at(i);
+    // replace any non alphanumeric characters with '_'.
+    if (!isalpha(c) && (c < '0' || c > '9'))
+      tmpName[i] = '_';
+  }
+  S += tmpName;
+  S += "_";
+  S += utostr(NumObjCStringLiterals++);
+
+  Preamble += "static __NSConstantStringImpl " + S;
+  Preamble += " __attribute__ ((section (\"__DATA, __cfstring\"))) = {__CFConstantStringClassReference,";
+  Preamble += "0x000007c8,"; // utf8_str
+  // The pretty printer for StringLiteral handles escape characters properly.
+  std::string prettyBufS;
+  llvm::raw_string_ostream prettyBuf(prettyBufS);
+  Exp->getString()->printPretty(prettyBuf, 0, PrintingPolicy(LangOpts));
+  Preamble += prettyBuf.str();
+  Preamble += ",";
+  Preamble += utostr(Exp->getString()->getByteLength()) + "};\n";
+
+  VarDecl *NewVD = VarDecl::Create(*Context, TUDecl, SourceLocation(),
+                                   SourceLocation(), &Context->Idents.get(S),
+                                   strType, 0, SC_Static, SC_None);
+  DeclRefExpr *DRE = new (Context) DeclRefExpr(NewVD, false, strType, VK_LValue,
+                                               SourceLocation());
+  Expr *Unop = new (Context) UnaryOperator(DRE, UO_AddrOf,
+                                 Context->getPointerType(DRE->getType()),
+                                           VK_RValue, OK_Ordinary,
+                                           SourceLocation());
+  // cast to NSConstantString *
+  CastExpr *cast = NoTypeInfoCStyleCastExpr(Context, Exp->getType(),
+                                            CK_CPointerToObjCPointerCast, Unop);
+  ReplaceStmt(Exp, cast);
+  // delete Exp; leak for now, see RewritePropertyOrImplicitSetter() usage for more info.
+  return cast;
+}
+
+Stmt *RewriteModernObjC::RewriteObjCBoolLiteralExpr(ObjCBoolLiteralExpr *Exp) {
+  unsigned IntSize =
+    static_cast<unsigned>(Context->getTypeSize(Context->IntTy));
+  
+  Expr *FlagExp = IntegerLiteral::Create(*Context, 
+                                         llvm::APInt(IntSize, Exp->getValue()), 
+                                         Context->IntTy, Exp->getLocation());
+  CastExpr *cast = NoTypeInfoCStyleCastExpr(Context, Context->ObjCBuiltinBoolTy,
+                                            CK_BitCast, FlagExp);
+  ParenExpr *PE = new (Context) ParenExpr(Exp->getLocation(), Exp->getExprLoc(), 
+                                          cast);
+  ReplaceStmt(Exp, PE);
+  return PE;
+}
+
+Stmt *RewriteModernObjC::RewriteObjCBoxedExpr(ObjCBoxedExpr *Exp) {
+  // synthesize declaration of helper functions needed in this routine.
+  if (!SelGetUidFunctionDecl)
+    SynthSelGetUidFunctionDecl();
+  // use objc_msgSend() for all.
+  if (!MsgSendFunctionDecl)
+    SynthMsgSendFunctionDecl();
+  if (!GetClassFunctionDecl)
+    SynthGetClassFunctionDecl();
+  
+  FunctionDecl *MsgSendFlavor = MsgSendFunctionDecl;
+  SourceLocation StartLoc = Exp->getLocStart();
+  SourceLocation EndLoc = Exp->getLocEnd();
+  
+  // Synthesize a call to objc_msgSend().
+  SmallVector<Expr*, 4> MsgExprs;
+  SmallVector<Expr*, 4> ClsExprs;
+  QualType argType = Context->getPointerType(Context->CharTy);
+  
+  // Create a call to objc_getClass("<BoxingClass>"). It will be the 1st argument.
+  ObjCMethodDecl *BoxingMethod = Exp->getBoxingMethod();
+  ObjCInterfaceDecl *BoxingClass = BoxingMethod->getClassInterface();
+  
+  IdentifierInfo *clsName = BoxingClass->getIdentifier();
+  ClsExprs.push_back(StringLiteral::Create(*Context,
+                                           clsName->getName(),
+                                           StringLiteral::Ascii, false,
+                                           argType, SourceLocation()));
+  CallExpr *Cls = SynthesizeCallToFunctionDecl(GetClassFunctionDecl,
+                                               &ClsExprs[0],
+                                               ClsExprs.size(), 
+                                               StartLoc, EndLoc);
+  MsgExprs.push_back(Cls);
+  
+  // Create a call to sel_registerName("<BoxingMethod>:"), etc.
+  // it will be the 2nd argument.
+  SmallVector<Expr*, 4> SelExprs;
+  SelExprs.push_back(StringLiteral::Create(*Context,
+                                           BoxingMethod->getSelector().getAsString(),
+                                           StringLiteral::Ascii, false,
+                                           argType, SourceLocation()));
+  CallExpr *SelExp = SynthesizeCallToFunctionDecl(SelGetUidFunctionDecl,
+                                                  &SelExprs[0], SelExprs.size(),
+                                                  StartLoc, EndLoc);
+  MsgExprs.push_back(SelExp);
+  
+  // User provided sub-expression is the 3rd, and last, argument.
+  Expr *subExpr  = Exp->getSubExpr();
+  if (ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(subExpr)) {
+    QualType type = ICE->getType();
+    const Expr *SubExpr = ICE->IgnoreParenImpCasts();
+    CastKind CK = CK_BitCast;
+    if (SubExpr->getType()->isIntegralType(*Context) && type->isBooleanType())
+      CK = CK_IntegralToBoolean;
+    subExpr = NoTypeInfoCStyleCastExpr(Context, type, CK, subExpr);
+  }
+  MsgExprs.push_back(subExpr);
+  
+  SmallVector<QualType, 4> ArgTypes;
+  ArgTypes.push_back(Context->getObjCIdType());
+  ArgTypes.push_back(Context->getObjCSelType());
+  for (ObjCMethodDecl::param_iterator PI = BoxingMethod->param_begin(),
+       E = BoxingMethod->param_end(); PI != E; ++PI)
+    ArgTypes.push_back((*PI)->getType());
+  
+  QualType returnType = Exp->getType();
+  // Get the type, we will need to reference it in a couple spots.
+  QualType msgSendType = MsgSendFlavor->getType();
+  
+  // Create a reference to the objc_msgSend() declaration.
+  DeclRefExpr *DRE = new (Context) DeclRefExpr(MsgSendFlavor, false, msgSendType,
+                                               VK_LValue, SourceLocation());
+  
+  CastExpr *cast = NoTypeInfoCStyleCastExpr(Context,
+                                            Context->getPointerType(Context->VoidTy),
+                                            CK_BitCast, DRE);
+  
+  // Now do the "normal" pointer to function cast.
+  QualType castType =
+  getSimpleFunctionType(returnType, &ArgTypes[0], ArgTypes.size(),
+                        BoxingMethod->isVariadic());
+  castType = Context->getPointerType(castType);
+  cast = NoTypeInfoCStyleCastExpr(Context, castType, CK_BitCast,
+                                  cast);
+  
+  // Don't forget the parens to enforce the proper binding.
+  ParenExpr *PE = new (Context) ParenExpr(StartLoc, EndLoc, cast);
+  
+  const FunctionType *FT = msgSendType->getAs<FunctionType>();
+  CallExpr *CE = new (Context) CallExpr(*Context, PE, MsgExprs,
+                                        FT->getResultType(), VK_RValue,
+                                        EndLoc);
+  ReplaceStmt(Exp, CE);
+  return CE;
+}
+
+Stmt *RewriteModernObjC::RewriteObjCArrayLiteralExpr(ObjCArrayLiteral *Exp) {
+  // synthesize declaration of helper functions needed in this routine.
+  if (!SelGetUidFunctionDecl)
+    SynthSelGetUidFunctionDecl();
+  // use objc_msgSend() for all.
+  if (!MsgSendFunctionDecl)
+    SynthMsgSendFunctionDecl();
+  if (!GetClassFunctionDecl)
+    SynthGetClassFunctionDecl();
+  
+  FunctionDecl *MsgSendFlavor = MsgSendFunctionDecl;
+  SourceLocation StartLoc = Exp->getLocStart();
+  SourceLocation EndLoc = Exp->getLocEnd();
+  
+  // Build the expression: __NSContainer_literal(int, ...).arr
+  QualType IntQT = Context->IntTy;
+  QualType NSArrayFType =
+    getSimpleFunctionType(Context->VoidTy, &IntQT, 1, true);
+  std::string NSArrayFName("__NSContainer_literal");
+  FunctionDecl *NSArrayFD = SynthBlockInitFunctionDecl(NSArrayFName);
+  DeclRefExpr *NSArrayDRE = 
+    new (Context) DeclRefExpr(NSArrayFD, false, NSArrayFType, VK_RValue,
+                              SourceLocation());
+
+  SmallVector<Expr*, 16> InitExprs;
+  unsigned NumElements = Exp->getNumElements();
+  unsigned UnsignedIntSize = 
+    static_cast<unsigned>(Context->getTypeSize(Context->UnsignedIntTy));
+  Expr *count = IntegerLiteral::Create(*Context,
+                                       llvm::APInt(UnsignedIntSize, NumElements),
+                                       Context->UnsignedIntTy, SourceLocation());
+  InitExprs.push_back(count);
+  for (unsigned i = 0; i < NumElements; i++)
+    InitExprs.push_back(Exp->getElement(i));
+  Expr *NSArrayCallExpr = 
+    new (Context) CallExpr(*Context, NSArrayDRE, InitExprs,
+                           NSArrayFType, VK_LValue, SourceLocation());
+  
+  FieldDecl *ARRFD = FieldDecl::Create(*Context, 0, SourceLocation(),
+                                    SourceLocation(),
+                                    &Context->Idents.get("arr"),
+                                    Context->getPointerType(Context->VoidPtrTy), 0,
+                                    /*BitWidth=*/0, /*Mutable=*/true,
+                                    ICIS_NoInit);
+  MemberExpr *ArrayLiteralME = 
+    new (Context) MemberExpr(NSArrayCallExpr, false, ARRFD, 
+                             SourceLocation(),
+                             ARRFD->getType(), VK_LValue,
+                             OK_Ordinary);
+  QualType ConstIdT = Context->getObjCIdType().withConst();
+  CStyleCastExpr * ArrayLiteralObjects = 
+    NoTypeInfoCStyleCastExpr(Context, 
+                             Context->getPointerType(ConstIdT),
+                             CK_BitCast,
+                             ArrayLiteralME);
+  
+  // Synthesize a call to objc_msgSend().
+  SmallVector<Expr*, 32> MsgExprs;
+  SmallVector<Expr*, 4> ClsExprs;
+  QualType argType = Context->getPointerType(Context->CharTy);
+  QualType expType = Exp->getType();
+  
+  // Create a call to objc_getClass("NSArray"). It will be th 1st argument.
+  ObjCInterfaceDecl *Class = 
+    expType->getPointeeType()->getAs<ObjCObjectType>()->getInterface();
+  
+  IdentifierInfo *clsName = Class->getIdentifier();
+  ClsExprs.push_back(StringLiteral::Create(*Context,
+                                           clsName->getName(),
+                                           StringLiteral::Ascii, false,
+                                           argType, SourceLocation()));
+  CallExpr *Cls = SynthesizeCallToFunctionDecl(GetClassFunctionDecl,
+                                               &ClsExprs[0],
+                                               ClsExprs.size(), 
+                                               StartLoc, EndLoc);
+  MsgExprs.push_back(Cls);
+  
+  // Create a call to sel_registerName("arrayWithObjects:count:").
+  // it will be the 2nd argument.
+  SmallVector<Expr*, 4> SelExprs;
+  ObjCMethodDecl *ArrayMethod = Exp->getArrayWithObjectsMethod();
+  SelExprs.push_back(StringLiteral::Create(*Context,
+                                           ArrayMethod->getSelector().getAsString(),
+                                           StringLiteral::Ascii, false,
+                                           argType, SourceLocation()));
+  CallExpr *SelExp = SynthesizeCallToFunctionDecl(SelGetUidFunctionDecl,
+                                                  &SelExprs[0], SelExprs.size(),
+                                                  StartLoc, EndLoc);
+  MsgExprs.push_back(SelExp);
+  
+  // (const id [])objects
+  MsgExprs.push_back(ArrayLiteralObjects);
+  
+  // (NSUInteger)cnt
+  Expr *cnt = IntegerLiteral::Create(*Context,
+                                     llvm::APInt(UnsignedIntSize, NumElements),
+                                     Context->UnsignedIntTy, SourceLocation());
+  MsgExprs.push_back(cnt);
+  
+  
+  SmallVector<QualType, 4> ArgTypes;
+  ArgTypes.push_back(Context->getObjCIdType());
+  ArgTypes.push_back(Context->getObjCSelType());
+  for (ObjCMethodDecl::param_iterator PI = ArrayMethod->param_begin(),
+       E = ArrayMethod->param_end(); PI != E; ++PI)
+    ArgTypes.push_back((*PI)->getType());
+  
+  QualType returnType = Exp->getType();
+  // Get the type, we will need to reference it in a couple spots.
+  QualType msgSendType = MsgSendFlavor->getType();
+  
+  // Create a reference to the objc_msgSend() declaration.
+  DeclRefExpr *DRE = new (Context) DeclRefExpr(MsgSendFlavor, false, msgSendType,
+                                               VK_LValue, SourceLocation());
+  
+  CastExpr *cast = NoTypeInfoCStyleCastExpr(Context,
+                                            Context->getPointerType(Context->VoidTy),
+                                            CK_BitCast, DRE);
+  
+  // Now do the "normal" pointer to function cast.
+  QualType castType =
+  getSimpleFunctionType(returnType, &ArgTypes[0], ArgTypes.size(),
+                        ArrayMethod->isVariadic());
+  castType = Context->getPointerType(castType);
+  cast = NoTypeInfoCStyleCastExpr(Context, castType, CK_BitCast,
+                                  cast);
+  
+  // Don't forget the parens to enforce the proper binding.
+  ParenExpr *PE = new (Context) ParenExpr(StartLoc, EndLoc, cast);
+  
+  const FunctionType *FT = msgSendType->getAs<FunctionType>();
+  CallExpr *CE = new (Context) CallExpr(*Context, PE, MsgExprs,
+                                        FT->getResultType(), VK_RValue,
+                                        EndLoc);
+  ReplaceStmt(Exp, CE);
+  return CE;
+}
+
+Stmt *RewriteModernObjC::RewriteObjCDictionaryLiteralExpr(ObjCDictionaryLiteral *Exp) {
+  // synthesize declaration of helper functions needed in this routine.
+  if (!SelGetUidFunctionDecl)
+    SynthSelGetUidFunctionDecl();
+  // use objc_msgSend() for all.
+  if (!MsgSendFunctionDecl)
+    SynthMsgSendFunctionDecl();
+  if (!GetClassFunctionDecl)
+    SynthGetClassFunctionDecl();
+  
+  FunctionDecl *MsgSendFlavor = MsgSendFunctionDecl;
+  SourceLocation StartLoc = Exp->getLocStart();
+  SourceLocation EndLoc = Exp->getLocEnd();
+  
+  // Build the expression: __NSContainer_literal(int, ...).arr
+  QualType IntQT = Context->IntTy;
+  QualType NSDictFType =
+    getSimpleFunctionType(Context->VoidTy, &IntQT, 1, true);
+  std::string NSDictFName("__NSContainer_literal");
+  FunctionDecl *NSDictFD = SynthBlockInitFunctionDecl(NSDictFName);
+  DeclRefExpr *NSDictDRE = 
+    new (Context) DeclRefExpr(NSDictFD, false, NSDictFType, VK_RValue,
+                              SourceLocation());
+  
+  SmallVector<Expr*, 16> KeyExprs;
+  SmallVector<Expr*, 16> ValueExprs;
+  
+  unsigned NumElements = Exp->getNumElements();
+  unsigned UnsignedIntSize = 
+    static_cast<unsigned>(Context->getTypeSize(Context->UnsignedIntTy));
+  Expr *count = IntegerLiteral::Create(*Context,
+                                       llvm::APInt(UnsignedIntSize, NumElements),
+                                       Context->UnsignedIntTy, SourceLocation());
+  KeyExprs.push_back(count);
+  ValueExprs.push_back(count);
+  for (unsigned i = 0; i < NumElements; i++) {
+    ObjCDictionaryElement Element = Exp->getKeyValueElement(i);
+    KeyExprs.push_back(Element.Key);
+    ValueExprs.push_back(Element.Value);
+  }
+  
+  // (const id [])objects
+  Expr *NSValueCallExpr = 
+    new (Context) CallExpr(*Context, NSDictDRE, ValueExprs,
+                           NSDictFType, VK_LValue, SourceLocation());
+  
+  FieldDecl *ARRFD = FieldDecl::Create(*Context, 0, SourceLocation(),
+                                       SourceLocation(),
+                                       &Context->Idents.get("arr"),
+                                       Context->getPointerType(Context->VoidPtrTy), 0,
+                                       /*BitWidth=*/0, /*Mutable=*/true,
+                                       ICIS_NoInit);
+  MemberExpr *DictLiteralValueME = 
+    new (Context) MemberExpr(NSValueCallExpr, false, ARRFD, 
+                             SourceLocation(),
+                             ARRFD->getType(), VK_LValue,
+                             OK_Ordinary);
+  QualType ConstIdT = Context->getObjCIdType().withConst();
+  CStyleCastExpr * DictValueObjects = 
+    NoTypeInfoCStyleCastExpr(Context, 
+                             Context->getPointerType(ConstIdT),
+                             CK_BitCast,
+                             DictLiteralValueME);
+  // (const id <NSCopying> [])keys
+  Expr *NSKeyCallExpr = 
+    new (Context) CallExpr(*Context, NSDictDRE, KeyExprs,
+                           NSDictFType, VK_LValue, SourceLocation());
+  
+  MemberExpr *DictLiteralKeyME = 
+    new (Context) MemberExpr(NSKeyCallExpr, false, ARRFD, 
+                             SourceLocation(),
+                             ARRFD->getType(), VK_LValue,
+                             OK_Ordinary);
+  
+  CStyleCastExpr * DictKeyObjects = 
+    NoTypeInfoCStyleCastExpr(Context, 
+                             Context->getPointerType(ConstIdT),
+                             CK_BitCast,
+                             DictLiteralKeyME);
+  
+  
+  
+  // Synthesize a call to objc_msgSend().
+  SmallVector<Expr*, 32> MsgExprs;
+  SmallVector<Expr*, 4> ClsExprs;
+  QualType argType = Context->getPointerType(Context->CharTy);
+  QualType expType = Exp->getType();
+  
+  // Create a call to objc_getClass("NSArray"). It will be th 1st argument.
+  ObjCInterfaceDecl *Class = 
+  expType->getPointeeType()->getAs<ObjCObjectType>()->getInterface();
+  
+  IdentifierInfo *clsName = Class->getIdentifier();
+  ClsExprs.push_back(StringLiteral::Create(*Context,
+                                           clsName->getName(),
+                                           StringLiteral::Ascii, false,
+                                           argType, SourceLocation()));
+  CallExpr *Cls = SynthesizeCallToFunctionDecl(GetClassFunctionDecl,
+                                               &ClsExprs[0],
+                                               ClsExprs.size(), 
+                                               StartLoc, EndLoc);
+  MsgExprs.push_back(Cls);
+  
+  // Create a call to sel_registerName("arrayWithObjects:count:").
+  // it will be the 2nd argument.
+  SmallVector<Expr*, 4> SelExprs;
+  ObjCMethodDecl *DictMethod = Exp->getDictWithObjectsMethod();
+  SelExprs.push_back(StringLiteral::Create(*Context,
+                                           DictMethod->getSelector().getAsString(),
+                                           StringLiteral::Ascii, false,
+                                           argType, SourceLocation()));
+  CallExpr *SelExp = SynthesizeCallToFunctionDecl(SelGetUidFunctionDecl,
+                                                  &SelExprs[0], SelExprs.size(),
+                                                  StartLoc, EndLoc);
+  MsgExprs.push_back(SelExp);
+  
+  // (const id [])objects
+  MsgExprs.push_back(DictValueObjects);
+  
+  // (const id <NSCopying> [])keys
+  MsgExprs.push_back(DictKeyObjects);
+  
+  // (NSUInteger)cnt
+  Expr *cnt = IntegerLiteral::Create(*Context,
+                                     llvm::APInt(UnsignedIntSize, NumElements),
+                                     Context->UnsignedIntTy, SourceLocation());
+  MsgExprs.push_back(cnt);
+  
+  
+  SmallVector<QualType, 8> ArgTypes;
+  ArgTypes.push_back(Context->getObjCIdType());
+  ArgTypes.push_back(Context->getObjCSelType());
+  for (ObjCMethodDecl::param_iterator PI = DictMethod->param_begin(),
+       E = DictMethod->param_end(); PI != E; ++PI) {
+    QualType T = (*PI)->getType();
+    if (const PointerType* PT = T->getAs<PointerType>()) {
+      QualType PointeeTy = PT->getPointeeType();
+      convertToUnqualifiedObjCType(PointeeTy);
+      T = Context->getPointerType(PointeeTy);
+    }
+    ArgTypes.push_back(T);
+  }
+  
+  QualType returnType = Exp->getType();
+  // Get the type, we will need to reference it in a couple spots.
+  QualType msgSendType = MsgSendFlavor->getType();
+  
+  // Create a reference to the objc_msgSend() declaration.
+  DeclRefExpr *DRE = new (Context) DeclRefExpr(MsgSendFlavor, false, msgSendType,
+                                               VK_LValue, SourceLocation());
+  
+  CastExpr *cast = NoTypeInfoCStyleCastExpr(Context,
+                                            Context->getPointerType(Context->VoidTy),
+                                            CK_BitCast, DRE);
+  
+  // Now do the "normal" pointer to function cast.
+  QualType castType =
+  getSimpleFunctionType(returnType, &ArgTypes[0], ArgTypes.size(),
+                        DictMethod->isVariadic());
+  castType = Context->getPointerType(castType);
+  cast = NoTypeInfoCStyleCastExpr(Context, castType, CK_BitCast,
+                                  cast);
+  
+  // Don't forget the parens to enforce the proper binding.
+  ParenExpr *PE = new (Context) ParenExpr(StartLoc, EndLoc, cast);
+  
+  const FunctionType *FT = msgSendType->getAs<FunctionType>();
+  CallExpr *CE = new (Context) CallExpr(*Context, PE, MsgExprs,
+                                        FT->getResultType(), VK_RValue,
+                                        EndLoc);
+  ReplaceStmt(Exp, CE);
+  return CE;
+}
+
+// struct __rw_objc_super { 
+//   struct objc_object *object; struct objc_object *superClass; 
+// };
+QualType RewriteModernObjC::getSuperStructType() {
+  if (!SuperStructDecl) {
+    SuperStructDecl = RecordDecl::Create(*Context, TTK_Struct, TUDecl,
+                                         SourceLocation(), SourceLocation(),
+                                         &Context->Idents.get("__rw_objc_super"));
+    QualType FieldTypes[2];
+
+    // struct objc_object *object;
+    FieldTypes[0] = Context->getObjCIdType();
+    // struct objc_object *superClass;
+    FieldTypes[1] = Context->getObjCIdType();
+
+    // Create fields
+    for (unsigned i = 0; i < 2; ++i) {
+      SuperStructDecl->addDecl(FieldDecl::Create(*Context, SuperStructDecl,
+                                                 SourceLocation(),
+                                                 SourceLocation(), 0,
+                                                 FieldTypes[i], 0,
+                                                 /*BitWidth=*/0,
+                                                 /*Mutable=*/false,
+                                                 ICIS_NoInit));
+    }
+
+    SuperStructDecl->completeDefinition();
+  }
+  return Context->getTagDeclType(SuperStructDecl);
+}
+
+QualType RewriteModernObjC::getConstantStringStructType() {
+  if (!ConstantStringDecl) {
+    ConstantStringDecl = RecordDecl::Create(*Context, TTK_Struct, TUDecl,
+                                            SourceLocation(), SourceLocation(),
+                         &Context->Idents.get("__NSConstantStringImpl"));
+    QualType FieldTypes[4];
+
+    // struct objc_object *receiver;
+    FieldTypes[0] = Context->getObjCIdType();
+    // int flags;
+    FieldTypes[1] = Context->IntTy;
+    // char *str;
+    FieldTypes[2] = Context->getPointerType(Context->CharTy);
+    // long length;
+    FieldTypes[3] = Context->LongTy;
+
+    // Create fields
+    for (unsigned i = 0; i < 4; ++i) {
+      ConstantStringDecl->addDecl(FieldDecl::Create(*Context,
+                                                    ConstantStringDecl,
+                                                    SourceLocation(),
+                                                    SourceLocation(), 0,
+                                                    FieldTypes[i], 0,
+                                                    /*BitWidth=*/0,
+                                                    /*Mutable=*/true,
+                                                    ICIS_NoInit));
+    }
+
+    ConstantStringDecl->completeDefinition();
+  }
+  return Context->getTagDeclType(ConstantStringDecl);
+}
+
+/// getFunctionSourceLocation - returns start location of a function
+/// definition. Complication arises when function has declared as
+/// extern "C" or extern "C" {...}
+static SourceLocation getFunctionSourceLocation (RewriteModernObjC &R,
+                                                 FunctionDecl *FD) {
+  if (FD->isExternC()  && !FD->isMain()) {
+    const DeclContext *DC = FD->getDeclContext();
+    if (const LinkageSpecDecl *LSD = dyn_cast<LinkageSpecDecl>(DC))
+      // if it is extern "C" {...}, return function decl's own location.
+      if (!LSD->getRBraceLoc().isValid())
+        return LSD->getExternLoc();
+  }
+  if (FD->getStorageClassAsWritten() != SC_None)
+    R.RewriteBlockLiteralFunctionDecl(FD);
+  return FD->getTypeSpecStartLoc();
+}
+
+void RewriteModernObjC::RewriteLineDirective(const Decl *D) {
+  
+  SourceLocation Location = D->getLocation();
+  
+  if (Location.isFileID()) {
+    std::string LineString("\n#line ");
+    PresumedLoc PLoc = SM->getPresumedLoc(Location);
+    LineString += utostr(PLoc.getLine());
+    LineString += " \"";
+    LineString += Lexer::Stringify(PLoc.getFilename());
+    if (isa<ObjCMethodDecl>(D))
+      LineString += "\"";
+    else LineString += "\"\n";
+    
+    Location = D->getLocStart();
+    if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
+      if (FD->isExternC()  && !FD->isMain()) {
+        const DeclContext *DC = FD->getDeclContext();
+        if (const LinkageSpecDecl *LSD = dyn_cast<LinkageSpecDecl>(DC))
+          // if it is extern "C" {...}, return function decl's own location.
+          if (!LSD->getRBraceLoc().isValid())
+            Location = LSD->getExternLoc();
+      }
+    }
+    InsertText(Location, LineString);
+  }
+}
+
+/// SynthMsgSendStretCallExpr - This routine translates message expression
+/// into a call to objc_msgSend_stret() entry point. Tricky part is that
+/// nil check on receiver must be performed before calling objc_msgSend_stret.
+/// MsgSendStretFlavor - function declaration objc_msgSend_stret(...)
+/// msgSendType - function type of objc_msgSend_stret(...)
+/// returnType - Result type of the method being synthesized.
+/// ArgTypes - type of the arguments passed to objc_msgSend_stret, starting with receiver type.
+/// MsgExprs - list of argument expressions being passed to objc_msgSend_stret, 
+/// starting with receiver.
+/// Method - Method being rewritten.
+Expr *RewriteModernObjC::SynthMsgSendStretCallExpr(FunctionDecl *MsgSendStretFlavor,
+                                                 QualType msgSendType, 
+                                                 QualType returnType, 
+                                                 SmallVectorImpl<QualType> &ArgTypes,
+                                                 SmallVectorImpl<Expr*> &MsgExprs,
+                                                 ObjCMethodDecl *Method) {
+  // Now do the "normal" pointer to function cast.
+  QualType castType = getSimpleFunctionType(returnType, &ArgTypes[0], ArgTypes.size(),
+                                            Method ? Method->isVariadic() : false);
+  castType = Context->getPointerType(castType);
+  
+  // build type for containing the objc_msgSend_stret object.
+  static unsigned stretCount=0;
+  std::string name = "__Stret"; name += utostr(stretCount);
+  std::string str = 
+    "extern \"C\" void * __cdecl memset(void *_Dst, int _Val, size_t _Size);\n";
+  str += "struct "; str += name;
+  str += " {\n\t";
+  str += name;
+  str += "(id receiver, SEL sel";
+  for (unsigned i = 2; i < ArgTypes.size(); i++) {
+    std::string ArgName = "arg"; ArgName += utostr(i);
+    ArgTypes[i].getAsStringInternal(ArgName, Context->getPrintingPolicy());
+    str += ", "; str += ArgName;
+  }
+  // could be vararg.
+  for (unsigned i = ArgTypes.size(); i < MsgExprs.size(); i++) {
+    std::string ArgName = "arg"; ArgName += utostr(i);
+    MsgExprs[i]->getType().getAsStringInternal(ArgName,
+                                               Context->getPrintingPolicy());
+    str += ", "; str += ArgName;
+  }
+  
+  str += ") {\n";
+  str += "\t  if (receiver == 0)\n";
+  str += "\t    memset((void*)&s, 0, sizeof(s));\n";
+  str += "\t  else\n";
+  str += "\t    s = (("; str += castType.getAsString(Context->getPrintingPolicy());
+  str += ")(void *)objc_msgSend_stret)(receiver, sel";
+  for (unsigned i = 2; i < ArgTypes.size(); i++) {
+    str += ", arg"; str += utostr(i);
+  }
+  // could be vararg.
+  for (unsigned i = ArgTypes.size(); i < MsgExprs.size(); i++) {
+    str += ", arg"; str += utostr(i);
+  }
+  
+  str += ");\n";
+  str += "\t}\n";
+  str += "\t"; str += returnType.getAsString(Context->getPrintingPolicy());
+  str += " s;\n";
+  str += "};\n\n";
+  SourceLocation FunLocStart;
+  if (CurFunctionDef)
+    FunLocStart = getFunctionSourceLocation(*this, CurFunctionDef);
+  else {
+    assert(CurMethodDef && "SynthMsgSendStretCallExpr - CurMethodDef is null");
+    FunLocStart = CurMethodDef->getLocStart();
+  }
+
+  InsertText(FunLocStart, str);
+  ++stretCount;
+  
+  // AST for __Stretn(receiver, args).s;
+  IdentifierInfo *ID = &Context->Idents.get(name);
+  FunctionDecl *FD = FunctionDecl::Create(*Context, TUDecl, SourceLocation(),
+                                          SourceLocation(), ID, castType, 0, SC_Extern,
+                                          SC_None, false, false);
+  DeclRefExpr *DRE = new (Context) DeclRefExpr(FD, false, castType, VK_RValue,
+                                               SourceLocation());
+  CallExpr *STCE = new (Context) CallExpr(*Context, DRE, MsgExprs,
+                                          castType, VK_LValue, SourceLocation());
+  
+  FieldDecl *FieldD = FieldDecl::Create(*Context, 0, SourceLocation(),
+                                    SourceLocation(),
+                                    &Context->Idents.get("s"),
+                                    returnType, 0,
+                                    /*BitWidth=*/0, /*Mutable=*/true,
+                                    ICIS_NoInit);
+  MemberExpr *ME = new (Context) MemberExpr(STCE, false, FieldD, SourceLocation(),
+                                            FieldD->getType(), VK_LValue,
+                                            OK_Ordinary);
+
+  return ME;
+}
+
+Stmt *RewriteModernObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
+                                    SourceLocation StartLoc,
+                                    SourceLocation EndLoc) {
+  if (!SelGetUidFunctionDecl)
+    SynthSelGetUidFunctionDecl();
+  if (!MsgSendFunctionDecl)
+    SynthMsgSendFunctionDecl();
+  if (!MsgSendSuperFunctionDecl)
+    SynthMsgSendSuperFunctionDecl();
+  if (!MsgSendStretFunctionDecl)
+    SynthMsgSendStretFunctionDecl();
+  if (!MsgSendSuperStretFunctionDecl)
+    SynthMsgSendSuperStretFunctionDecl();
+  if (!MsgSendFpretFunctionDecl)
+    SynthMsgSendFpretFunctionDecl();
+  if (!GetClassFunctionDecl)
+    SynthGetClassFunctionDecl();
+  if (!GetSuperClassFunctionDecl)
+    SynthGetSuperClassFunctionDecl();
+  if (!GetMetaClassFunctionDecl)
+    SynthGetMetaClassFunctionDecl();
+
+  // default to objc_msgSend().
+  FunctionDecl *MsgSendFlavor = MsgSendFunctionDecl;
+  // May need to use objc_msgSend_stret() as well.
+  FunctionDecl *MsgSendStretFlavor = 0;
+  if (ObjCMethodDecl *mDecl = Exp->getMethodDecl()) {
+    QualType resultType = mDecl->getResultType();
+    if (resultType->isRecordType())
+      MsgSendStretFlavor = MsgSendStretFunctionDecl;
+    else if (resultType->isRealFloatingType())
+      MsgSendFlavor = MsgSendFpretFunctionDecl;
+  }
+
+  // Synthesize a call to objc_msgSend().
+  SmallVector<Expr*, 8> MsgExprs;
+  switch (Exp->getReceiverKind()) {
+  case ObjCMessageExpr::SuperClass: {
+    MsgSendFlavor = MsgSendSuperFunctionDecl;
+    if (MsgSendStretFlavor)
+      MsgSendStretFlavor = MsgSendSuperStretFunctionDecl;
+    assert(MsgSendFlavor && "MsgSendFlavor is NULL!");
+
+    ObjCInterfaceDecl *ClassDecl = CurMethodDef->getClassInterface();
+
+    SmallVector<Expr*, 4> InitExprs;
+
+    // set the receiver to self, the first argument to all methods.
+    InitExprs.push_back(
+      NoTypeInfoCStyleCastExpr(Context, Context->getObjCIdType(),
+                               CK_BitCast,
+                   new (Context) DeclRefExpr(CurMethodDef->getSelfDecl(),
+                                             false,
+                                             Context->getObjCIdType(),
+                                             VK_RValue,
+                                             SourceLocation()))
+                        ); // set the 'receiver'.
+
+    // (id)class_getSuperclass((Class)objc_getClass("CurrentClass"))
+    SmallVector<Expr*, 8> ClsExprs;
+    QualType argType = Context->getPointerType(Context->CharTy);
+    ClsExprs.push_back(StringLiteral::Create(*Context,
+                                   ClassDecl->getIdentifier()->getName(),
+                                   StringLiteral::Ascii, false,
+                                   argType, SourceLocation()));
+    // (Class)objc_getClass("CurrentClass")
+    CallExpr *Cls = SynthesizeCallToFunctionDecl(GetMetaClassFunctionDecl,
+                                                 &ClsExprs[0],
+                                                 ClsExprs.size(),
+                                                 StartLoc,
+                                                 EndLoc);
+    ClsExprs.clear();
+    ClsExprs.push_back(Cls);
+    Cls = SynthesizeCallToFunctionDecl(GetSuperClassFunctionDecl,
+                                       &ClsExprs[0], ClsExprs.size(),
+                                       StartLoc, EndLoc);
+    
+    // (id)class_getSuperclass((Class)objc_getClass("CurrentClass"))
+    // To turn off a warning, type-cast to 'id'
+    InitExprs.push_back( // set 'super class', using class_getSuperclass().
+                        NoTypeInfoCStyleCastExpr(Context,
+                                                 Context->getObjCIdType(),
+                                                 CK_BitCast, Cls));
+    // struct __rw_objc_super
+    QualType superType = getSuperStructType();
+    Expr *SuperRep;
+
+    if (LangOpts.MicrosoftExt) {
+      SynthSuperContructorFunctionDecl();
+      // Simulate a contructor call...
+      DeclRefExpr *DRE = new (Context) DeclRefExpr(SuperContructorFunctionDecl,
+                                                   false, superType, VK_LValue,
+                                                   SourceLocation());
+      SuperRep = new (Context) CallExpr(*Context, DRE, InitExprs,
+                                        superType, VK_LValue,
+                                        SourceLocation());
+      // The code for super is a little tricky to prevent collision with
+      // the structure definition in the header. The rewriter has it's own
+      // internal definition (__rw_objc_super) that is uses. This is why
+      // we need the cast below. For example:
+      // (struct __rw_objc_super *)&__rw_objc_super((id)self, (id)objc_getClass("SUPER"))
+      //
+      SuperRep = new (Context) UnaryOperator(SuperRep, UO_AddrOf,
+                               Context->getPointerType(SuperRep->getType()),
+                                             VK_RValue, OK_Ordinary,
+                                             SourceLocation());
+      SuperRep = NoTypeInfoCStyleCastExpr(Context,
+                                          Context->getPointerType(superType),
+                                          CK_BitCast, SuperRep);
+    } else {
+      // (struct __rw_objc_super) { <exprs from above> }
+      InitListExpr *ILE =
+        new (Context) InitListExpr(*Context, SourceLocation(), InitExprs,
+                                   SourceLocation());
+      TypeSourceInfo *superTInfo
+        = Context->getTrivialTypeSourceInfo(superType);
+      SuperRep = new (Context) CompoundLiteralExpr(SourceLocation(), superTInfo,
+                                                   superType, VK_LValue,
+                                                   ILE, false);
+      // struct __rw_objc_super *
+      SuperRep = new (Context) UnaryOperator(SuperRep, UO_AddrOf,
+                               Context->getPointerType(SuperRep->getType()),
+                                             VK_RValue, OK_Ordinary,
+                                             SourceLocation());
+    }
+    MsgExprs.push_back(SuperRep);
+    break;
+  }
+
+  case ObjCMessageExpr::Class: {
+    SmallVector<Expr*, 8> ClsExprs;
+    QualType argType = Context->getPointerType(Context->CharTy);
+    ObjCInterfaceDecl *Class
+      = Exp->getClassReceiver()->getAs<ObjCObjectType>()->getInterface();
+    IdentifierInfo *clsName = Class->getIdentifier();
+    ClsExprs.push_back(StringLiteral::Create(*Context,
+                                             clsName->getName(),
+                                             StringLiteral::Ascii, false,
+                                             argType, SourceLocation()));
+    CallExpr *Cls = SynthesizeCallToFunctionDecl(GetClassFunctionDecl,
+                                                 &ClsExprs[0],
+                                                 ClsExprs.size(), 
+                                                 StartLoc, EndLoc);
+    CastExpr *ArgExpr = NoTypeInfoCStyleCastExpr(Context,
+                                                 Context->getObjCIdType(),
+                                                 CK_BitCast, Cls);
+    MsgExprs.push_back(ArgExpr);
+    break;
+  }
+
+  case ObjCMessageExpr::SuperInstance:{
+    MsgSendFlavor = MsgSendSuperFunctionDecl;
+    if (MsgSendStretFlavor)
+      MsgSendStretFlavor = MsgSendSuperStretFunctionDecl;
+    assert(MsgSendFlavor && "MsgSendFlavor is NULL!");
+    ObjCInterfaceDecl *ClassDecl = CurMethodDef->getClassInterface();
+    SmallVector<Expr*, 4> InitExprs;
+
+    InitExprs.push_back(
+      NoTypeInfoCStyleCastExpr(Context, Context->getObjCIdType(),
+                               CK_BitCast,
+                   new (Context) DeclRefExpr(CurMethodDef->getSelfDecl(),
+                                             false,
+                                             Context->getObjCIdType(),
+                                             VK_RValue, SourceLocation()))
+                        ); // set the 'receiver'.
+    
+    // (id)class_getSuperclass((Class)objc_getClass("CurrentClass"))
+    SmallVector<Expr*, 8> ClsExprs;
+    QualType argType = Context->getPointerType(Context->CharTy);
+    ClsExprs.push_back(StringLiteral::Create(*Context,
+                                   ClassDecl->getIdentifier()->getName(),
+                                   StringLiteral::Ascii, false, argType,
+                                   SourceLocation()));
+    // (Class)objc_getClass("CurrentClass")
+    CallExpr *Cls = SynthesizeCallToFunctionDecl(GetClassFunctionDecl,
+                                                 &ClsExprs[0],
+                                                 ClsExprs.size(), 
+                                                 StartLoc, EndLoc);
+    ClsExprs.clear();
+    ClsExprs.push_back(Cls);
+    Cls = SynthesizeCallToFunctionDecl(GetSuperClassFunctionDecl,
+                                       &ClsExprs[0], ClsExprs.size(),
+                                       StartLoc, EndLoc);
+    
+    // (id)class_getSuperclass((Class)objc_getClass("CurrentClass"))
+    // To turn off a warning, type-cast to 'id'
+    InitExprs.push_back(
+      // set 'super class', using class_getSuperclass().
+      NoTypeInfoCStyleCastExpr(Context, Context->getObjCIdType(),
+                               CK_BitCast, Cls));
+    // struct __rw_objc_super
+    QualType superType = getSuperStructType();
+    Expr *SuperRep;
+
+    if (LangOpts.MicrosoftExt) {
+      SynthSuperContructorFunctionDecl();
+      // Simulate a contructor call...
+      DeclRefExpr *DRE = new (Context) DeclRefExpr(SuperContructorFunctionDecl,
+                                                   false, superType, VK_LValue,
+                                                   SourceLocation());
+      SuperRep = new (Context) CallExpr(*Context, DRE, InitExprs,
+                                        superType, VK_LValue, SourceLocation());
+      // The code for super is a little tricky to prevent collision with
+      // the structure definition in the header. The rewriter has it's own
+      // internal definition (__rw_objc_super) that is uses. This is why
+      // we need the cast below. For example:
+      // (struct __rw_objc_super *)&__rw_objc_super((id)self, (id)objc_getClass("SUPER"))
+      //
+      SuperRep = new (Context) UnaryOperator(SuperRep, UO_AddrOf,
+                               Context->getPointerType(SuperRep->getType()),
+                               VK_RValue, OK_Ordinary,
+                               SourceLocation());
+      SuperRep = NoTypeInfoCStyleCastExpr(Context,
+                               Context->getPointerType(superType),
+                               CK_BitCast, SuperRep);
+    } else {
+      // (struct __rw_objc_super) { <exprs from above> }
+      InitListExpr *ILE =
+        new (Context) InitListExpr(*Context, SourceLocation(), InitExprs,
+                                   SourceLocation());
+      TypeSourceInfo *superTInfo
+        = Context->getTrivialTypeSourceInfo(superType);
+      SuperRep = new (Context) CompoundLiteralExpr(SourceLocation(), superTInfo,
+                                                   superType, VK_RValue, ILE,
+                                                   false);
+    }
+    MsgExprs.push_back(SuperRep);
+    break;
+  }
+
+  case ObjCMessageExpr::Instance: {
+    // Remove all type-casts because it may contain objc-style types; e.g.
+    // Foo<Proto> *.
+    Expr *recExpr = Exp->getInstanceReceiver();
+    while (CStyleCastExpr *CE = dyn_cast<CStyleCastExpr>(recExpr))
+      recExpr = CE->getSubExpr();
+    CastKind CK = recExpr->getType()->isObjCObjectPointerType()
+                    ? CK_BitCast : recExpr->getType()->isBlockPointerType()
+                                     ? CK_BlockPointerToObjCPointerCast
+                                     : CK_CPointerToObjCPointerCast;
+
+    recExpr = NoTypeInfoCStyleCastExpr(Context, Context->getObjCIdType(),
+                                       CK, recExpr);
+    MsgExprs.push_back(recExpr);
+    break;
+  }
+  }
+
+  // Create a call to sel_registerName("selName"), it will be the 2nd argument.
+  SmallVector<Expr*, 8> SelExprs;
+  QualType argType = Context->getPointerType(Context->CharTy);
+  SelExprs.push_back(StringLiteral::Create(*Context,
+                                       Exp->getSelector().getAsString(),
+                                       StringLiteral::Ascii, false,
+                                       argType, SourceLocation()));
+  CallExpr *SelExp = SynthesizeCallToFunctionDecl(SelGetUidFunctionDecl,
+                                                 &SelExprs[0], SelExprs.size(),
+                                                  StartLoc,
+                                                  EndLoc);
+  MsgExprs.push_back(SelExp);
+
+  // Now push any user supplied arguments.
+  for (unsigned i = 0; i < Exp->getNumArgs(); i++) {
+    Expr *userExpr = Exp->getArg(i);
+    // Make all implicit casts explicit...ICE comes in handy:-)
+    if (ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(userExpr)) {
+      // Reuse the ICE type, it is exactly what the doctor ordered.
+      QualType type = ICE->getType();
+      if (needToScanForQualifiers(type))
+        type = Context->getObjCIdType();
+      // Make sure we convert "type (^)(...)" to "type (*)(...)".
+      (void)convertBlockPointerToFunctionPointer(type);
+      const Expr *SubExpr = ICE->IgnoreParenImpCasts();
+      CastKind CK;
+      if (SubExpr->getType()->isIntegralType(*Context) && 
+          type->isBooleanType()) {
+        CK = CK_IntegralToBoolean;
+      } else if (type->isObjCObjectPointerType()) {
+        if (SubExpr->getType()->isBlockPointerType()) {
+          CK = CK_BlockPointerToObjCPointerCast;
+        } else if (SubExpr->getType()->isPointerType()) {
+          CK = CK_CPointerToObjCPointerCast;
+        } else {
+          CK = CK_BitCast;
+        }
+      } else {
+        CK = CK_BitCast;
+      }
+
+      userExpr = NoTypeInfoCStyleCastExpr(Context, type, CK, userExpr);
+    }
+    // Make id<P...> cast into an 'id' cast.
+    else if (CStyleCastExpr *CE = dyn_cast<CStyleCastExpr>(userExpr)) {
+      if (CE->getType()->isObjCQualifiedIdType()) {
+        while ((CE = dyn_cast<CStyleCastExpr>(userExpr)))
+          userExpr = CE->getSubExpr();
+        CastKind CK;
+        if (userExpr->getType()->isIntegralType(*Context)) {
+          CK = CK_IntegralToPointer;
+        } else if (userExpr->getType()->isBlockPointerType()) {
+          CK = CK_BlockPointerToObjCPointerCast;
+        } else if (userExpr->getType()->isPointerType()) {
+          CK = CK_CPointerToObjCPointerCast;
+        } else {
+          CK = CK_BitCast;
+        }
+        userExpr = NoTypeInfoCStyleCastExpr(Context, Context->getObjCIdType(),
+                                            CK, userExpr);
+      }
+    }
+    MsgExprs.push_back(userExpr);
+    // We've transferred the ownership to MsgExprs. For now, we *don't* null
+    // out the argument in the original expression (since we aren't deleting
+    // the ObjCMessageExpr). See RewritePropertyOrImplicitSetter() usage for more info.
+    //Exp->setArg(i, 0);
+  }
+  // Generate the funky cast.
+  CastExpr *cast;
+  SmallVector<QualType, 8> ArgTypes;
+  QualType returnType;
+
+  // Push 'id' and 'SEL', the 2 implicit arguments.
+  if (MsgSendFlavor == MsgSendSuperFunctionDecl)
+    ArgTypes.push_back(Context->getPointerType(getSuperStructType()));
+  else
+    ArgTypes.push_back(Context->getObjCIdType());
+  ArgTypes.push_back(Context->getObjCSelType());
+  if (ObjCMethodDecl *OMD = Exp->getMethodDecl()) {
+    // Push any user argument types.
+    for (ObjCMethodDecl::param_iterator PI = OMD->param_begin(),
+         E = OMD->param_end(); PI != E; ++PI) {
+      QualType t = (*PI)->getType()->isObjCQualifiedIdType()
+                     ? Context->getObjCIdType()
+                     : (*PI)->getType();
+      // Make sure we convert "t (^)(...)" to "t (*)(...)".
+      (void)convertBlockPointerToFunctionPointer(t);
+      ArgTypes.push_back(t);
+    }
+    returnType = Exp->getType();
+    convertToUnqualifiedObjCType(returnType);
+    (void)convertBlockPointerToFunctionPointer(returnType);
+  } else {
+    returnType = Context->getObjCIdType();
+  }
+  // Get the type, we will need to reference it in a couple spots.
+  QualType msgSendType = MsgSendFlavor->getType();
+
+  // Create a reference to the objc_msgSend() declaration.
+  DeclRefExpr *DRE = new (Context) DeclRefExpr(MsgSendFlavor, false, msgSendType,
+                                               VK_LValue, SourceLocation());
+
+  // Need to cast objc_msgSend to "void *" (to workaround a GCC bandaid).
+  // If we don't do this cast, we get the following bizarre warning/note:
+  // xx.m:13: warning: function called through a non-compatible type
+  // xx.m:13: note: if this code is reached, the program will abort
+  cast = NoTypeInfoCStyleCastExpr(Context,
+                                  Context->getPointerType(Context->VoidTy),
+                                  CK_BitCast, DRE);
+
+  // Now do the "normal" pointer to function cast.
+  QualType castType =
+    getSimpleFunctionType(returnType, &ArgTypes[0], ArgTypes.size(),
+      // If we don't have a method decl, force a variadic cast.
+      Exp->getMethodDecl() ? Exp->getMethodDecl()->isVariadic() : true);
+  castType = Context->getPointerType(castType);
+  cast = NoTypeInfoCStyleCastExpr(Context, castType, CK_BitCast,
+                                  cast);
+
+  // Don't forget the parens to enforce the proper binding.
+  ParenExpr *PE = new (Context) ParenExpr(StartLoc, EndLoc, cast);
+
+  const FunctionType *FT = msgSendType->getAs<FunctionType>();
+  CallExpr *CE = new (Context) CallExpr(*Context, PE, MsgExprs,
+                                        FT->getResultType(), VK_RValue, EndLoc);
+  Stmt *ReplacingStmt = CE;
+  if (MsgSendStretFlavor) {
+    // We have the method which returns a struct/union. Must also generate
+    // call to objc_msgSend_stret and hang both varieties on a conditional
+    // expression which dictate which one to envoke depending on size of
+    // method's return type.
+
+    Expr *STCE = SynthMsgSendStretCallExpr(MsgSendStretFlavor, 
+                                           msgSendType, returnType, 
+                                           ArgTypes, MsgExprs,
+                                           Exp->getMethodDecl());
+
+    // Build sizeof(returnType)
+    UnaryExprOrTypeTraitExpr *sizeofExpr =
+       new (Context) UnaryExprOrTypeTraitExpr(UETT_SizeOf,
+                                 Context->getTrivialTypeSourceInfo(returnType),
+                                 Context->getSizeType(), SourceLocation(),
+                                 SourceLocation());
+    // (sizeof(returnType) <= 8 ? objc_msgSend(...) : objc_msgSend_stret(...))
+    // FIXME: Value of 8 is base on ppc32/x86 ABI for the most common cases.
+    // For X86 it is more complicated and some kind of target specific routine
+    // is needed to decide what to do.
+    unsigned IntSize =
+      static_cast<unsigned>(Context->getTypeSize(Context->IntTy));
+    IntegerLiteral *limit = IntegerLiteral::Create(*Context,
+                                                   llvm::APInt(IntSize, 8),
+                                                   Context->IntTy,
+                                                   SourceLocation());
+    BinaryOperator *lessThanExpr = 
+      new (Context) BinaryOperator(sizeofExpr, limit, BO_LE, Context->IntTy,
+                                   VK_RValue, OK_Ordinary, SourceLocation(),
+                                   false);
+    // (sizeof(returnType) <= 8 ? objc_msgSend(...) : objc_msgSend_stret(...))
+    ConditionalOperator *CondExpr =
+      new (Context) ConditionalOperator(lessThanExpr,
+                                        SourceLocation(), CE,
+                                        SourceLocation(), STCE,
+                                        returnType, VK_RValue, OK_Ordinary);
+    ReplacingStmt = new (Context) ParenExpr(SourceLocation(), SourceLocation(), 
+                                            CondExpr);
+  }
+  // delete Exp; leak for now, see RewritePropertyOrImplicitSetter() usage for more info.
+  return ReplacingStmt;
+}
+
+Stmt *RewriteModernObjC::RewriteMessageExpr(ObjCMessageExpr *Exp) {
+  Stmt *ReplacingStmt = SynthMessageExpr(Exp, Exp->getLocStart(),
+                                         Exp->getLocEnd());
+
+  // Now do the actual rewrite.
+  ReplaceStmt(Exp, ReplacingStmt);
+
+  // delete Exp; leak for now, see RewritePropertyOrImplicitSetter() usage for more info.
+  return ReplacingStmt;
+}
+
+// typedef struct objc_object Protocol;
+QualType RewriteModernObjC::getProtocolType() {
+  if (!ProtocolTypeDecl) {
+    TypeSourceInfo *TInfo
+      = Context->getTrivialTypeSourceInfo(Context->getObjCIdType());
+    ProtocolTypeDecl = TypedefDecl::Create(*Context, TUDecl,
+                                           SourceLocation(), SourceLocation(),
+                                           &Context->Idents.get("Protocol"),
+                                           TInfo);
+  }
+  return Context->getTypeDeclType(ProtocolTypeDecl);
+}
+
+/// RewriteObjCProtocolExpr - Rewrite a protocol expression into
+/// a synthesized/forward data reference (to the protocol's metadata).
+/// The forward references (and metadata) are generated in
+/// RewriteModernObjC::HandleTranslationUnit().
+Stmt *RewriteModernObjC::RewriteObjCProtocolExpr(ObjCProtocolExpr *Exp) {
+  std::string Name = "_OBJC_PROTOCOL_REFERENCE_$_" + 
+                      Exp->getProtocol()->getNameAsString();
+  IdentifierInfo *ID = &Context->Idents.get(Name);
+  VarDecl *VD = VarDecl::Create(*Context, TUDecl, SourceLocation(),
+                                SourceLocation(), ID, getProtocolType(), 0,
+                                SC_Extern, SC_None);
+  DeclRefExpr *DRE = new (Context) DeclRefExpr(VD, false, getProtocolType(),
+                                               VK_LValue, SourceLocation());
+  Expr *DerefExpr = new (Context) UnaryOperator(DRE, UO_AddrOf,
+                             Context->getPointerType(DRE->getType()),
+                             VK_RValue, OK_Ordinary, SourceLocation());
+  CastExpr *castExpr = NoTypeInfoCStyleCastExpr(Context, DerefExpr->getType(),
+                                                CK_BitCast,
+                                                DerefExpr);
+  ReplaceStmt(Exp, castExpr);
+  ProtocolExprDecls.insert(Exp->getProtocol()->getCanonicalDecl());
+  // delete Exp; leak for now, see RewritePropertyOrImplicitSetter() usage for more info.
+  return castExpr;
+
+}
+
+bool RewriteModernObjC::BufferContainsPPDirectives(const char *startBuf,
+                                             const char *endBuf) {
+  while (startBuf < endBuf) {
+    if (*startBuf == '#') {
+      // Skip whitespace.
+      for (++startBuf; startBuf[0] == ' ' || startBuf[0] == '\t'; ++startBuf)
+        ;
+      if (!strncmp(startBuf, "if", strlen("if")) ||
+          !strncmp(startBuf, "ifdef", strlen("ifdef")) ||
+          !strncmp(startBuf, "ifndef", strlen("ifndef")) ||
+          !strncmp(startBuf, "define", strlen("define")) ||
+          !strncmp(startBuf, "undef", strlen("undef")) ||
+          !strncmp(startBuf, "else", strlen("else")) ||
+          !strncmp(startBuf, "elif", strlen("elif")) ||
+          !strncmp(startBuf, "endif", strlen("endif")) ||
+          !strncmp(startBuf, "pragma", strlen("pragma")) ||
+          !strncmp(startBuf, "include", strlen("include")) ||
+          !strncmp(startBuf, "import", strlen("import")) ||
+          !strncmp(startBuf, "include_next", strlen("include_next")))
+        return true;
+    }
+    startBuf++;
+  }
+  return false;
+}
+
+/// IsTagDefinedInsideClass - This routine checks that a named tagged type 
+/// is defined inside an objective-c class. If so, it returns true. 
+bool RewriteModernObjC::IsTagDefinedInsideClass(ObjCContainerDecl *IDecl, 
+                                                TagDecl *Tag,
+                                                bool &IsNamedDefinition) {
+  if (!IDecl)
+    return false;
+  SourceLocation TagLocation;
+  if (RecordDecl *RD = dyn_cast<RecordDecl>(Tag)) {
+    RD = RD->getDefinition();
+    if (!RD || !RD->getDeclName().getAsIdentifierInfo())
+      return false;
+    IsNamedDefinition = true;
+    TagLocation = RD->getLocation();
+    return Context->getSourceManager().isBeforeInTranslationUnit(
+                                          IDecl->getLocation(), TagLocation);
+  }
+  if (EnumDecl *ED = dyn_cast<EnumDecl>(Tag)) {
+    if (!ED || !ED->getDeclName().getAsIdentifierInfo())
+      return false;
+    IsNamedDefinition = true;
+    TagLocation = ED->getLocation();
+    return Context->getSourceManager().isBeforeInTranslationUnit(
+                                          IDecl->getLocation(), TagLocation);
+
+  }
+  return false;
+}
+
+/// RewriteObjCFieldDeclType - This routine rewrites a type into the buffer.
+/// It handles elaborated types, as well as enum types in the process.
+bool RewriteModernObjC::RewriteObjCFieldDeclType(QualType &Type, 
+                                                 std::string &Result) {
+  if (isa<TypedefType>(Type)) {
+    Result += "\t";
+    return false;
+  }
+    
+  if (Type->isArrayType()) {
+    QualType ElemTy = Context->getBaseElementType(Type);
+    return RewriteObjCFieldDeclType(ElemTy, Result);
+  }
+  else if (Type->isRecordType()) {
+    RecordDecl *RD = Type->getAs<RecordType>()->getDecl();
+    if (RD->isCompleteDefinition()) {
+      if (RD->isStruct())
+        Result += "\n\tstruct ";
+      else if (RD->isUnion())
+        Result += "\n\tunion ";
+      else
+        assert(false && "class not allowed as an ivar type");
+      
+      Result += RD->getName();
+      if (GlobalDefinedTags.count(RD)) {
+        // struct/union is defined globally, use it.
+        Result += " ";
+        return true;
+      }
+      Result += " {\n";
+      for (RecordDecl::field_iterator i = RD->field_begin(), 
+           e = RD->field_end(); i != e; ++i) {
+        FieldDecl *FD = *i;
+        RewriteObjCFieldDecl(FD, Result);
+      }
+      Result += "\t} "; 
+      return true;
+    }
+  }
+  else if (Type->isEnumeralType()) {
+    EnumDecl *ED = Type->getAs<EnumType>()->getDecl();
+    if (ED->isCompleteDefinition()) {
+      Result += "\n\tenum ";
+      Result += ED->getName();
+      if (GlobalDefinedTags.count(ED)) {
+        // Enum is globall defined, use it.
+        Result += " ";
+        return true;
+      }
+      
+      Result += " {\n";
+      for (EnumDecl::enumerator_iterator EC = ED->enumerator_begin(),
+           ECEnd = ED->enumerator_end(); EC != ECEnd; ++EC) {
+        Result += "\t"; Result += EC->getName(); Result += " = ";
+        llvm::APSInt Val = EC->getInitVal();
+        Result += Val.toString(10);
+        Result += ",\n";
+      }
+      Result += "\t} "; 
+      return true;
+    }
+  }
+  
+  Result += "\t";
+  convertObjCTypeToCStyleType(Type);
+  return false;
+}
+
+
+/// RewriteObjCFieldDecl - This routine rewrites a field into the buffer.
+/// It handles elaborated types, as well as enum types in the process.
+void RewriteModernObjC::RewriteObjCFieldDecl(FieldDecl *fieldDecl, 
+                                             std::string &Result) {
+  QualType Type = fieldDecl->getType();
+  std::string Name = fieldDecl->getNameAsString();
+  
+  bool EleboratedType = RewriteObjCFieldDeclType(Type, Result); 
+  if (!EleboratedType)
+    Type.getAsStringInternal(Name, Context->getPrintingPolicy());
+  Result += Name;
+  if (fieldDecl->isBitField()) {
+    Result += " : "; Result += utostr(fieldDecl->getBitWidthValue(*Context));
+  }
+  else if (EleboratedType && Type->isArrayType()) {
+    CanQualType CType = Context->getCanonicalType(Type);
+    while (isa<ArrayType>(CType)) {
+      if (const ConstantArrayType *CAT = Context->getAsConstantArrayType(CType)) {
+        Result += "[";
+        llvm::APInt Dim = CAT->getSize();
+        Result += utostr(Dim.getZExtValue());
+        Result += "]";
+      }
+      CType = CType->getAs<ArrayType>()->getElementType();
+    }
+  }
+  
+  Result += ";\n";
+}
+
+/// RewriteLocallyDefinedNamedAggregates - This routine rewrites locally defined
+/// named aggregate types into the input buffer.
+void RewriteModernObjC::RewriteLocallyDefinedNamedAggregates(FieldDecl *fieldDecl, 
+                                             std::string &Result) {
+  QualType Type = fieldDecl->getType();
+  if (isa<TypedefType>(Type))
+    return;
+  if (Type->isArrayType())
+    Type = Context->getBaseElementType(Type);
+  ObjCContainerDecl *IDecl = 
+    dyn_cast<ObjCContainerDecl>(fieldDecl->getDeclContext());
+  
+  TagDecl *TD = 0;
+  if (Type->isRecordType()) {
+    TD = Type->getAs<RecordType>()->getDecl();
+  }
+  else if (Type->isEnumeralType()) {
+    TD = Type->getAs<EnumType>()->getDecl();
+  }
+  
+  if (TD) {
+    if (GlobalDefinedTags.count(TD))
+      return;
+    
+    bool IsNamedDefinition = false;
+    if (IsTagDefinedInsideClass(IDecl, TD, IsNamedDefinition)) {
+      RewriteObjCFieldDeclType(Type, Result);
+      Result += ";";
+    }
+    if (IsNamedDefinition)
+      GlobalDefinedTags.insert(TD);
+  }
+    
+}
+
+/// RewriteObjCInternalStruct - Rewrite one internal struct corresponding to
+/// an objective-c class with ivars.
+void RewriteModernObjC::RewriteObjCInternalStruct(ObjCInterfaceDecl *CDecl,
+                                               std::string &Result) {
+  assert(CDecl && "Class missing in SynthesizeObjCInternalStruct");
+  assert(CDecl->getName() != "" &&
+         "Name missing in SynthesizeObjCInternalStruct");
+  ObjCInterfaceDecl *RCDecl = CDecl->getSuperClass();
+  SmallVector<ObjCIvarDecl *, 8> IVars;
+  for (ObjCIvarDecl *IVD = CDecl->all_declared_ivar_begin();
+       IVD; IVD = IVD->getNextIvar())
+    IVars.push_back(IVD);
+  
+  SourceLocation LocStart = CDecl->getLocStart();
+  SourceLocation LocEnd = CDecl->getEndOfDefinitionLoc();
+  
+  const char *startBuf = SM->getCharacterData(LocStart);
+  const char *endBuf = SM->getCharacterData(LocEnd);
+  
+  // If no ivars and no root or if its root, directly or indirectly,
+  // have no ivars (thus not synthesized) then no need to synthesize this class.
+  if ((!CDecl->isThisDeclarationADefinition() || IVars.size() == 0) &&
+      (!RCDecl || !ObjCSynthesizedStructs.count(RCDecl))) {
+    endBuf += Lexer::MeasureTokenLength(LocEnd, *SM, LangOpts);
+    ReplaceText(LocStart, endBuf-startBuf, Result);
+    return;
+  }
+  
+  // Insert named struct/union definitions inside class to
+  // outer scope. This follows semantics of locally defined
+  // struct/unions in objective-c classes.
+  for (unsigned i = 0, e = IVars.size(); i < e; i++)
+    RewriteLocallyDefinedNamedAggregates(IVars[i], Result);
+
+  Result += "\nstruct ";
+  Result += CDecl->getNameAsString();
+  Result += "_IMPL {\n";
+  
+  if (RCDecl && ObjCSynthesizedStructs.count(RCDecl)) {
+    Result += "\tstruct "; Result += RCDecl->getNameAsString();
+    Result += "_IMPL "; Result += RCDecl->getNameAsString();
+    Result += "_IVARS;\n";
+  }
+  
+  for (unsigned i = 0, e = IVars.size(); i < e; i++)
+    RewriteObjCFieldDecl(IVars[i], Result);
+
+  Result += "};\n";
+  endBuf += Lexer::MeasureTokenLength(LocEnd, *SM, LangOpts);
+  ReplaceText(LocStart, endBuf-startBuf, Result);
+  // Mark this struct as having been generated.
+  if (!ObjCSynthesizedStructs.insert(CDecl))
+    llvm_unreachable("struct already synthesize- RewriteObjCInternalStruct");
+}
+
+/// RewriteIvarOffsetSymbols - Rewrite ivar offset symbols of those ivars which
+/// have been referenced in an ivar access expression.
+void RewriteModernObjC::RewriteIvarOffsetSymbols(ObjCInterfaceDecl *CDecl,
+                                                  std::string &Result) {
+  // write out ivar offset symbols which have been referenced in an ivar
+  // access expression.
+  llvm::SmallPtrSet<ObjCIvarDecl *, 8> Ivars = ReferencedIvars[CDecl];
+  if (Ivars.empty())
+    return;
+  for (llvm::SmallPtrSet<ObjCIvarDecl *, 8>::iterator i = Ivars.begin(),
+       e = Ivars.end(); i != e; i++) {
+    ObjCIvarDecl *IvarDecl = (*i);
+    Result += "\n";
+    if (LangOpts.MicrosoftExt)
+      Result += "__declspec(allocate(\".objc_ivar$B\")) ";
+    Result += "extern \"C\" ";
+    if (LangOpts.MicrosoftExt && 
+        IvarDecl->getAccessControl() != ObjCIvarDecl::Private &&
+        IvarDecl->getAccessControl() != ObjCIvarDecl::Package)
+        Result += "__declspec(dllimport) ";
+
+    Result += "unsigned long ";
+    WriteInternalIvarName(CDecl, IvarDecl, Result);
+    Result += ";";
+  }
+}
+
+//===----------------------------------------------------------------------===//
+// Meta Data Emission
+//===----------------------------------------------------------------------===//
+
+
+/// RewriteImplementations - This routine rewrites all method implementations
+/// and emits meta-data.
+
+void RewriteModernObjC::RewriteImplementations() {
+  int ClsDefCount = ClassImplementation.size();
+  int CatDefCount = CategoryImplementation.size();
+
+  // Rewrite implemented methods
+  for (int i = 0; i < ClsDefCount; i++) {
+    ObjCImplementationDecl *OIMP = ClassImplementation[i];
+    ObjCInterfaceDecl *CDecl = OIMP->getClassInterface();
+    if (CDecl->isImplicitInterfaceDecl())
+      assert(false &&
+             "Legacy implicit interface rewriting not supported in moder abi");
+    RewriteImplementationDecl(OIMP);
+  }
+
+  for (int i = 0; i < CatDefCount; i++) {
+    ObjCCategoryImplDecl *CIMP = CategoryImplementation[i];
+    ObjCInterfaceDecl *CDecl = CIMP->getClassInterface();
+    if (CDecl->isImplicitInterfaceDecl())
+      assert(false &&
+             "Legacy implicit interface rewriting not supported in moder abi");
+    RewriteImplementationDecl(CIMP);
+  }
+}
+
+void RewriteModernObjC::RewriteByRefString(std::string &ResultStr, 
+                                     const std::string &Name,
+                                     ValueDecl *VD, bool def) {
+  assert(BlockByRefDeclNo.count(VD) && 
+         "RewriteByRefString: ByRef decl missing");
+  if (def)
+    ResultStr += "struct ";
+  ResultStr += "__Block_byref_" + Name + 
+    "_" + utostr(BlockByRefDeclNo[VD]) ;
+}
+
+static bool HasLocalVariableExternalStorage(ValueDecl *VD) {
+  if (VarDecl *Var = dyn_cast<VarDecl>(VD))
+    return (Var->isFunctionOrMethodVarDecl() && !Var->hasLocalStorage());
+  return false;
+}
+
+std::string RewriteModernObjC::SynthesizeBlockFunc(BlockExpr *CE, int i,
+                                                   StringRef funcName,
+                                                   std::string Tag) {
+  const FunctionType *AFT = CE->getFunctionType();
+  QualType RT = AFT->getResultType();
+  std::string StructRef = "struct " + Tag;
+  SourceLocation BlockLoc = CE->getExprLoc();
+  std::string S;
+  ConvertSourceLocationToLineDirective(BlockLoc, S);
+  
+  S += "static " + RT.getAsString(Context->getPrintingPolicy()) + " __" +
+         funcName.str() + "_block_func_" + utostr(i);
+
+  BlockDecl *BD = CE->getBlockDecl();
+
+  if (isa<FunctionNoProtoType>(AFT)) {
+    // No user-supplied arguments. Still need to pass in a pointer to the
+    // block (to reference imported block decl refs).
+    S += "(" + StructRef + " *__cself)";
+  } else if (BD->param_empty()) {
+    S += "(" + StructRef + " *__cself)";
+  } else {
+    const FunctionProtoType *FT = cast<FunctionProtoType>(AFT);
+    assert(FT && "SynthesizeBlockFunc: No function proto");
+    S += '(';
+    // first add the implicit argument.
+    S += StructRef + " *__cself, ";
+    std::string ParamStr;
+    for (BlockDecl::param_iterator AI = BD->param_begin(),
+         E = BD->param_end(); AI != E; ++AI) {
+      if (AI != BD->param_begin()) S += ", ";
+      ParamStr = (*AI)->getNameAsString();
+      QualType QT = (*AI)->getType();
+      (void)convertBlockPointerToFunctionPointer(QT);
+      QT.getAsStringInternal(ParamStr, Context->getPrintingPolicy());
+      S += ParamStr;
+    }
+    if (FT->isVariadic()) {
+      if (!BD->param_empty()) S += ", ";
+      S += "...";
+    }
+    S += ')';
+  }
+  S += " {\n";
+
+  // Create local declarations to avoid rewriting all closure decl ref exprs.
+  // First, emit a declaration for all "by ref" decls.
+  for (SmallVector<ValueDecl*,8>::iterator I = BlockByRefDecls.begin(),
+       E = BlockByRefDecls.end(); I != E; ++I) {
+    S += "  ";
+    std::string Name = (*I)->getNameAsString();
+    std::string TypeString;
+    RewriteByRefString(TypeString, Name, (*I));
+    TypeString += " *";
+    Name = TypeString + Name;
+    S += Name + " = __cself->" + (*I)->getNameAsString() + "; // bound by ref\n";
+  }
+  // Next, emit a declaration for all "by copy" declarations.
+  for (SmallVector<ValueDecl*,8>::iterator I = BlockByCopyDecls.begin(),
+       E = BlockByCopyDecls.end(); I != E; ++I) {
+    S += "  ";
+    // Handle nested closure invocation. For example:
+    //
+    //   void (^myImportedClosure)(void);
+    //   myImportedClosure  = ^(void) { setGlobalInt(x + y); };
+    //
+    //   void (^anotherClosure)(void);
+    //   anotherClosure = ^(void) {
+    //     myImportedClosure(); // import and invoke the closure
+    //   };
+    //
+    if (isTopLevelBlockPointerType((*I)->getType())) {
+      RewriteBlockPointerTypeVariable(S, (*I));
+      S += " = (";
+      RewriteBlockPointerType(S, (*I)->getType());
+      S += ")";
+      S += "__cself->" + (*I)->getNameAsString() + "; // bound by copy\n";
+    }
+    else {
+      std::string Name = (*I)->getNameAsString();
+      QualType QT = (*I)->getType();
+      if (HasLocalVariableExternalStorage(*I))
+        QT = Context->getPointerType(QT);
+      QT.getAsStringInternal(Name, Context->getPrintingPolicy());
+      S += Name + " = __cself->" + 
+                              (*I)->getNameAsString() + "; // bound by copy\n";
+    }
+  }
+  std::string RewrittenStr = RewrittenBlockExprs[CE];
+  const char *cstr = RewrittenStr.c_str();
+  while (*cstr++ != '{') ;
+  S += cstr;
+  S += "\n";
+  return S;
+}
+
+std::string RewriteModernObjC::SynthesizeBlockHelperFuncs(BlockExpr *CE, int i,
+                                                   StringRef funcName,
+                                                   std::string Tag) {
+  std::string StructRef = "struct " + Tag;
+  std::string S = "static void __";
+
+  S += funcName;
+  S += "_block_copy_" + utostr(i);
+  S += "(" + StructRef;
+  S += "*dst, " + StructRef;
+  S += "*src) {";
+  for (llvm::SmallPtrSet<ValueDecl*,8>::iterator I = ImportedBlockDecls.begin(),
+      E = ImportedBlockDecls.end(); I != E; ++I) {
+    ValueDecl *VD = (*I);
+    S += "_Block_object_assign((void*)&dst->";
+    S += (*I)->getNameAsString();
+    S += ", (void*)src->";
+    S += (*I)->getNameAsString();
+    if (BlockByRefDeclsPtrSet.count((*I)))
+      S += ", " + utostr(BLOCK_FIELD_IS_BYREF) + "/*BLOCK_FIELD_IS_BYREF*/);";
+    else if (VD->getType()->isBlockPointerType())
+      S += ", " + utostr(BLOCK_FIELD_IS_BLOCK) + "/*BLOCK_FIELD_IS_BLOCK*/);";
+    else
+      S += ", " + utostr(BLOCK_FIELD_IS_OBJECT) + "/*BLOCK_FIELD_IS_OBJECT*/);";
+  }
+  S += "}\n";
+  
+  S += "\nstatic void __";
+  S += funcName;
+  S += "_block_dispose_" + utostr(i);
+  S += "(" + StructRef;
+  S += "*src) {";
+  for (llvm::SmallPtrSet<ValueDecl*,8>::iterator I = ImportedBlockDecls.begin(),
+      E = ImportedBlockDecls.end(); I != E; ++I) {
+    ValueDecl *VD = (*I);
+    S += "_Block_object_dispose((void*)src->";
+    S += (*I)->getNameAsString();
+    if (BlockByRefDeclsPtrSet.count((*I)))
+      S += ", " + utostr(BLOCK_FIELD_IS_BYREF) + "/*BLOCK_FIELD_IS_BYREF*/);";
+    else if (VD->getType()->isBlockPointerType())
+      S += ", " + utostr(BLOCK_FIELD_IS_BLOCK) + "/*BLOCK_FIELD_IS_BLOCK*/);";
+    else
+      S += ", " + utostr(BLOCK_FIELD_IS_OBJECT) + "/*BLOCK_FIELD_IS_OBJECT*/);";
+  }
+  S += "}\n";
+  return S;
+}
+
+std::string RewriteModernObjC::SynthesizeBlockImpl(BlockExpr *CE, std::string Tag, 
+                                             std::string Desc) {
+  std::string S = "\nstruct " + Tag;
+  std::string Constructor = "  " + Tag;
+
+  S += " {\n  struct __block_impl impl;\n";
+  S += "  struct " + Desc;
+  S += "* Desc;\n";
+
+  Constructor += "(void *fp, "; // Invoke function pointer.
+  Constructor += "struct " + Desc; // Descriptor pointer.
+  Constructor += " *desc";
+
+  if (BlockDeclRefs.size()) {
+    // Output all "by copy" declarations.
+    for (SmallVector<ValueDecl*,8>::iterator I = BlockByCopyDecls.begin(),
+         E = BlockByCopyDecls.end(); I != E; ++I) {
+      S += "  ";
+      std::string FieldName = (*I)->getNameAsString();
+      std::string ArgName = "_" + FieldName;
+      // Handle nested closure invocation. For example:
+      //
+      //   void (^myImportedBlock)(void);
+      //   myImportedBlock  = ^(void) { setGlobalInt(x + y); };
+      //
+      //   void (^anotherBlock)(void);
+      //   anotherBlock = ^(void) {
+      //     myImportedBlock(); // import and invoke the closure
+      //   };
+      //
+      if (isTopLevelBlockPointerType((*I)->getType())) {
+        S += "struct __block_impl *";
+        Constructor += ", void *" + ArgName;
+      } else {
+        QualType QT = (*I)->getType();
+        if (HasLocalVariableExternalStorage(*I))
+          QT = Context->getPointerType(QT);
+        QT.getAsStringInternal(FieldName, Context->getPrintingPolicy());
+        QT.getAsStringInternal(ArgName, Context->getPrintingPolicy());
+        Constructor += ", " + ArgName;
+      }
+      S += FieldName + ";\n";
+    }
+    // Output all "by ref" declarations.
+    for (SmallVector<ValueDecl*,8>::iterator I = BlockByRefDecls.begin(),
+         E = BlockByRefDecls.end(); I != E; ++I) {
+      S += "  ";
+      std::string FieldName = (*I)->getNameAsString();
+      std::string ArgName = "_" + FieldName;
+      {
+        std::string TypeString;
+        RewriteByRefString(TypeString, FieldName, (*I));
+        TypeString += " *";
+        FieldName = TypeString + FieldName;
+        ArgName = TypeString + ArgName;
+        Constructor += ", " + ArgName;
+      }
+      S += FieldName + "; // by ref\n";
+    }
+    // Finish writing the constructor.
+    Constructor += ", int flags=0)";
+    // Initialize all "by copy" arguments.
+    bool firsTime = true;
+    for (SmallVector<ValueDecl*,8>::iterator I = BlockByCopyDecls.begin(),
+         E = BlockByCopyDecls.end(); I != E; ++I) {
+      std::string Name = (*I)->getNameAsString();
+        if (firsTime) {
+          Constructor += " : ";
+          firsTime = false;
+        }
+        else
+          Constructor += ", ";
+        if (isTopLevelBlockPointerType((*I)->getType()))
+          Constructor += Name + "((struct __block_impl *)_" + Name + ")";
+        else
+          Constructor += Name + "(_" + Name + ")";
+    }
+    // Initialize all "by ref" arguments.
+    for (SmallVector<ValueDecl*,8>::iterator I = BlockByRefDecls.begin(),
+         E = BlockByRefDecls.end(); I != E; ++I) {
+      std::string Name = (*I)->getNameAsString();
+      if (firsTime) {
+        Constructor += " : ";
+        firsTime = false;
+      }
+      else
+        Constructor += ", ";
+      Constructor += Name + "(_" + Name + "->__forwarding)";
+    }
+    
+    Constructor += " {\n";
+    if (GlobalVarDecl)
+      Constructor += "    impl.isa = &_NSConcreteGlobalBlock;\n";
+    else
+      Constructor += "    impl.isa = &_NSConcreteStackBlock;\n";
+    Constructor += "    impl.Flags = flags;\n    impl.FuncPtr = fp;\n";
+
+    Constructor += "    Desc = desc;\n";
+  } else {
+    // Finish writing the constructor.
+    Constructor += ", int flags=0) {\n";
+    if (GlobalVarDecl)
+      Constructor += "    impl.isa = &_NSConcreteGlobalBlock;\n";
+    else
+      Constructor += "    impl.isa = &_NSConcreteStackBlock;\n";
+    Constructor += "    impl.Flags = flags;\n    impl.FuncPtr = fp;\n";
+    Constructor += "    Desc = desc;\n";
+  }
+  Constructor += "  ";
+  Constructor += "}\n";
+  S += Constructor;
+  S += "};\n";
+  return S;
+}
+
+std::string RewriteModernObjC::SynthesizeBlockDescriptor(std::string DescTag, 
+                                                   std::string ImplTag, int i,
+                                                   StringRef FunName,
+                                                   unsigned hasCopy) {
+  std::string S = "\nstatic struct " + DescTag;
+  
+  S += " {\n  size_t reserved;\n";
+  S += "  size_t Block_size;\n";
+  if (hasCopy) {
+    S += "  void (*copy)(struct ";
+    S += ImplTag; S += "*, struct ";
+    S += ImplTag; S += "*);\n";
+    
+    S += "  void (*dispose)(struct ";
+    S += ImplTag; S += "*);\n";
+  }
+  S += "} ";
+
+  S += DescTag + "_DATA = { 0, sizeof(struct ";
+  S += ImplTag + ")";
+  if (hasCopy) {
+    S += ", __" + FunName.str() + "_block_copy_" + utostr(i);
+    S += ", __" + FunName.str() + "_block_dispose_" + utostr(i);
+  }
+  S += "};\n";
+  return S;
+}
+
+void RewriteModernObjC::SynthesizeBlockLiterals(SourceLocation FunLocStart,
+                                          StringRef FunName) {
+  bool RewriteSC = (GlobalVarDecl &&
+                    !Blocks.empty() &&
+                    GlobalVarDecl->getStorageClass() == SC_Static &&
+                    GlobalVarDecl->getType().getCVRQualifiers());
+  if (RewriteSC) {
+    std::string SC(" void __");
+    SC += GlobalVarDecl->getNameAsString();
+    SC += "() {}";
+    InsertText(FunLocStart, SC);
+  }
+  
+  // Insert closures that were part of the function.
+  for (unsigned i = 0, count=0; i < Blocks.size(); i++) {
+    CollectBlockDeclRefInfo(Blocks[i]);
+    // Need to copy-in the inner copied-in variables not actually used in this
+    // block.
+    for (int j = 0; j < InnerDeclRefsCount[i]; j++) {
+      DeclRefExpr *Exp = InnerDeclRefs[count++];
+      ValueDecl *VD = Exp->getDecl();
+      BlockDeclRefs.push_back(Exp);
+      if (!VD->hasAttr<BlocksAttr>()) {
+        if (!BlockByCopyDeclsPtrSet.count(VD)) {
+          BlockByCopyDeclsPtrSet.insert(VD);
+          BlockByCopyDecls.push_back(VD);
+        }
+        continue;
+      }
+
+      if (!BlockByRefDeclsPtrSet.count(VD)) {
+        BlockByRefDeclsPtrSet.insert(VD);
+        BlockByRefDecls.push_back(VD);
+      }
+
+      // imported objects in the inner blocks not used in the outer
+      // blocks must be copied/disposed in the outer block as well.
+      if (VD->getType()->isObjCObjectPointerType() || 
+          VD->getType()->isBlockPointerType())
+        ImportedBlockDecls.insert(VD);
+    }
+
+    std::string ImplTag = "__" + FunName.str() + "_block_impl_" + utostr(i);
+    std::string DescTag = "__" + FunName.str() + "_block_desc_" + utostr(i);
+
+    std::string CI = SynthesizeBlockImpl(Blocks[i], ImplTag, DescTag);
+
+    InsertText(FunLocStart, CI);
+
+    std::string CF = SynthesizeBlockFunc(Blocks[i], i, FunName, ImplTag);
+
+    InsertText(FunLocStart, CF);
+
+    if (ImportedBlockDecls.size()) {
+      std::string HF = SynthesizeBlockHelperFuncs(Blocks[i], i, FunName, ImplTag);
+      InsertText(FunLocStart, HF);
+    }
+    std::string BD = SynthesizeBlockDescriptor(DescTag, ImplTag, i, FunName,
+                                               ImportedBlockDecls.size() > 0);
+    InsertText(FunLocStart, BD);
+
+    BlockDeclRefs.clear();
+    BlockByRefDecls.clear();
+    BlockByRefDeclsPtrSet.clear();
+    BlockByCopyDecls.clear();
+    BlockByCopyDeclsPtrSet.clear();
+    ImportedBlockDecls.clear();
+  }
+  if (RewriteSC) {
+    // Must insert any 'const/volatile/static here. Since it has been
+    // removed as result of rewriting of block literals.
+    std::string SC;
+    if (GlobalVarDecl->getStorageClass() == SC_Static)
+      SC = "static ";
+    if (GlobalVarDecl->getType().isConstQualified())
+      SC += "const ";
+    if (GlobalVarDecl->getType().isVolatileQualified())
+      SC += "volatile ";
+    if (GlobalVarDecl->getType().isRestrictQualified())
+      SC += "restrict ";
+    InsertText(FunLocStart, SC);
+  }
+  if (GlobalConstructionExp) {
+    // extra fancy dance for global literal expression.
+    
+    // Always the latest block expression on the block stack.
+    std::string Tag = "__";
+    Tag += FunName;
+    Tag += "_block_impl_";
+    Tag += utostr(Blocks.size()-1);
+    std::string globalBuf = "static ";
+    globalBuf += Tag; globalBuf += " ";
+    std::string SStr;
+  
+    llvm::raw_string_ostream constructorExprBuf(SStr);
+    GlobalConstructionExp->printPretty(constructorExprBuf, 0,
+                                         PrintingPolicy(LangOpts));
+    globalBuf += constructorExprBuf.str();
+    globalBuf += ";\n";
+    InsertText(FunLocStart, globalBuf);
+    GlobalConstructionExp = 0;
+  }
+
+  Blocks.clear();
+  InnerDeclRefsCount.clear();
+  InnerDeclRefs.clear();
+  RewrittenBlockExprs.clear();
+}
+
+void RewriteModernObjC::InsertBlockLiteralsWithinFunction(FunctionDecl *FD) {
+  SourceLocation FunLocStart = 
+    (!Blocks.empty()) ? getFunctionSourceLocation(*this, FD)
+                      : FD->getTypeSpecStartLoc();
+  StringRef FuncName = FD->getName();
+
+  SynthesizeBlockLiterals(FunLocStart, FuncName);
+}
+
+static void BuildUniqueMethodName(std::string &Name,
+                                  ObjCMethodDecl *MD) {
+  ObjCInterfaceDecl *IFace = MD->getClassInterface();
+  Name = IFace->getName();
+  Name += "__" + MD->getSelector().getAsString();
+  // Convert colons to underscores.
+  std::string::size_type loc = 0;
+  while ((loc = Name.find(":", loc)) != std::string::npos)
+    Name.replace(loc, 1, "_");
+}
+
+void RewriteModernObjC::InsertBlockLiteralsWithinMethod(ObjCMethodDecl *MD) {
+  //fprintf(stderr,"In InsertBlockLiteralsWitinMethod\n");
+  //SourceLocation FunLocStart = MD->getLocStart();
+  SourceLocation FunLocStart = MD->getLocStart();
+  std::string FuncName;
+  BuildUniqueMethodName(FuncName, MD);
+  SynthesizeBlockLiterals(FunLocStart, FuncName);
+}
+
+void RewriteModernObjC::GetBlockDeclRefExprs(Stmt *S) {
+  for (Stmt::child_range CI = S->children(); CI; ++CI)
+    if (*CI) {
+      if (BlockExpr *CBE = dyn_cast<BlockExpr>(*CI))
+        GetBlockDeclRefExprs(CBE->getBody());
+      else
+        GetBlockDeclRefExprs(*CI);
+    }
+  // Handle specific things.
+  if (DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(S)) {
+    if (DRE->refersToEnclosingLocal()) {
+      // FIXME: Handle enums.
+      if (!isa<FunctionDecl>(DRE->getDecl()))
+        BlockDeclRefs.push_back(DRE);
+      if (HasLocalVariableExternalStorage(DRE->getDecl()))
+        BlockDeclRefs.push_back(DRE);
+    }
+  }
+  
+  return;
+}
+
+void RewriteModernObjC::GetInnerBlockDeclRefExprs(Stmt *S, 
+                SmallVector<DeclRefExpr *, 8> &InnerBlockDeclRefs,
+                llvm::SmallPtrSet<const DeclContext *, 8> &InnerContexts) {
+  for (Stmt::child_range CI = S->children(); CI; ++CI)
+    if (*CI) {
+      if (BlockExpr *CBE = dyn_cast<BlockExpr>(*CI)) {
+        InnerContexts.insert(cast<DeclContext>(CBE->getBlockDecl()));
+        GetInnerBlockDeclRefExprs(CBE->getBody(),
+                                  InnerBlockDeclRefs,
+                                  InnerContexts);
+      }
+      else
+        GetInnerBlockDeclRefExprs(*CI,
+                                  InnerBlockDeclRefs,
+                                  InnerContexts);
+
+    }
+  // Handle specific things.
+  if (DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(S)) {
+    if (DRE->refersToEnclosingLocal()) {
+      if (!isa<FunctionDecl>(DRE->getDecl()) &&
+          !InnerContexts.count(DRE->getDecl()->getDeclContext()))
+        InnerBlockDeclRefs.push_back(DRE);
+      if (VarDecl *Var = dyn_cast<VarDecl>(DRE->getDecl()))
+        if (Var->isFunctionOrMethodVarDecl())
+          ImportedLocalExternalDecls.insert(Var);
+    }
+  }
+  
+  return;
+}
+
+/// convertObjCTypeToCStyleType - This routine converts such objc types
+/// as qualified objects, and blocks to their closest c/c++ types that
+/// it can. It returns true if input type was modified.
+bool RewriteModernObjC::convertObjCTypeToCStyleType(QualType &T) {
+  QualType oldT = T;
+  convertBlockPointerToFunctionPointer(T);
+  if (T->isFunctionPointerType()) {
+    QualType PointeeTy;
+    if (const PointerType* PT = T->getAs<PointerType>()) {
+      PointeeTy = PT->getPointeeType();
+      if (const FunctionType *FT = PointeeTy->getAs<FunctionType>()) {
+        T = convertFunctionTypeOfBlocks(FT);
+        T = Context->getPointerType(T);
+      }
+    }
+  }
+  
+  convertToUnqualifiedObjCType(T);
+  return T != oldT;
+}
+
+/// convertFunctionTypeOfBlocks - This routine converts a function type
+/// whose result type may be a block pointer or whose argument type(s)
+/// might be block pointers to an equivalent function type replacing
+/// all block pointers to function pointers.
+QualType RewriteModernObjC::convertFunctionTypeOfBlocks(const FunctionType *FT) {
+  const FunctionProtoType *FTP = dyn_cast<FunctionProtoType>(FT);
+  // FTP will be null for closures that don't take arguments.
+  // Generate a funky cast.
+  SmallVector<QualType, 8> ArgTypes;
+  QualType Res = FT->getResultType();
+  bool modified = convertObjCTypeToCStyleType(Res);
+  
+  if (FTP) {
+    for (FunctionProtoType::arg_type_iterator I = FTP->arg_type_begin(),
+         E = FTP->arg_type_end(); I && (I != E); ++I) {
+      QualType t = *I;
+      // Make sure we convert "t (^)(...)" to "t (*)(...)".
+      if (convertObjCTypeToCStyleType(t))
+        modified = true;
+      ArgTypes.push_back(t);
+    }
+  }
+  QualType FuncType;
+  if (modified)
+    FuncType = getSimpleFunctionType(Res, &ArgTypes[0], ArgTypes.size());
+  else FuncType = QualType(FT, 0);
+  return FuncType;
+}
+
+Stmt *RewriteModernObjC::SynthesizeBlockCall(CallExpr *Exp, const Expr *BlockExp) {
+  // Navigate to relevant type information.
+  const BlockPointerType *CPT = 0;
+
+  if (const DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(BlockExp)) {
+    CPT = DRE->getType()->getAs<BlockPointerType>();
+  } else if (const MemberExpr *MExpr = dyn_cast<MemberExpr>(BlockExp)) {
+    CPT = MExpr->getType()->getAs<BlockPointerType>();
+  } 
+  else if (const ParenExpr *PRE = dyn_cast<ParenExpr>(BlockExp)) {
+    return SynthesizeBlockCall(Exp, PRE->getSubExpr());
+  }
+  else if (const ImplicitCastExpr *IEXPR = dyn_cast<ImplicitCastExpr>(BlockExp)) 
+    CPT = IEXPR->getType()->getAs<BlockPointerType>();
+  else if (const ConditionalOperator *CEXPR = 
+            dyn_cast<ConditionalOperator>(BlockExp)) {
+    Expr *LHSExp = CEXPR->getLHS();
+    Stmt *LHSStmt = SynthesizeBlockCall(Exp, LHSExp);
+    Expr *RHSExp = CEXPR->getRHS();
+    Stmt *RHSStmt = SynthesizeBlockCall(Exp, RHSExp);
+    Expr *CONDExp = CEXPR->getCond();
+    ConditionalOperator *CondExpr =
+      new (Context) ConditionalOperator(CONDExp,
+                                      SourceLocation(), cast<Expr>(LHSStmt),
+                                      SourceLocation(), cast<Expr>(RHSStmt),
+                                      Exp->getType(), VK_RValue, OK_Ordinary);
+    return CondExpr;
+  } else if (const ObjCIvarRefExpr *IRE = dyn_cast<ObjCIvarRefExpr>(BlockExp)) {
+    CPT = IRE->getType()->getAs<BlockPointerType>();
+  } else if (const PseudoObjectExpr *POE
+               = dyn_cast<PseudoObjectExpr>(BlockExp)) {
+    CPT = POE->getType()->castAs<BlockPointerType>();
+  } else {
+    assert(1 && "RewriteBlockClass: Bad type");
+  }
+  assert(CPT && "RewriteBlockClass: Bad type");
+  const FunctionType *FT = CPT->getPointeeType()->getAs<FunctionType>();
+  assert(FT && "RewriteBlockClass: Bad type");
+  const FunctionProtoType *FTP = dyn_cast<FunctionProtoType>(FT);
+  // FTP will be null for closures that don't take arguments.
+
+  RecordDecl *RD = RecordDecl::Create(*Context, TTK_Struct, TUDecl,
+                                      SourceLocation(), SourceLocation(),
+                                      &Context->Idents.get("__block_impl"));
+  QualType PtrBlock = Context->getPointerType(Context->getTagDeclType(RD));
+
+  // Generate a funky cast.
+  SmallVector<QualType, 8> ArgTypes;
+
+  // Push the block argument type.
+  ArgTypes.push_back(PtrBlock);
+  if (FTP) {
+    for (FunctionProtoType::arg_type_iterator I = FTP->arg_type_begin(),
+         E = FTP->arg_type_end(); I && (I != E); ++I) {
+      QualType t = *I;
+      // Make sure we convert "t (^)(...)" to "t (*)(...)".
+      if (!convertBlockPointerToFunctionPointer(t))
+        convertToUnqualifiedObjCType(t);
+      ArgTypes.push_back(t);
+    }
+  }
+  // Now do the pointer to function cast.
+  QualType PtrToFuncCastType
+    = getSimpleFunctionType(Exp->getType(), &ArgTypes[0], ArgTypes.size());
+
+  PtrToFuncCastType = Context->getPointerType(PtrToFuncCastType);
+
+  CastExpr *BlkCast = NoTypeInfoCStyleCastExpr(Context, PtrBlock,
+                                               CK_BitCast,
+                                               const_cast<Expr*>(BlockExp));
+  // Don't forget the parens to enforce the proper binding.
+  ParenExpr *PE = new (Context) ParenExpr(SourceLocation(), SourceLocation(),
+                                          BlkCast);
+  //PE->dump();
+
+  FieldDecl *FD = FieldDecl::Create(*Context, 0, SourceLocation(),
+                                    SourceLocation(),
+                                    &Context->Idents.get("FuncPtr"),
+                                    Context->VoidPtrTy, 0,
+                                    /*BitWidth=*/0, /*Mutable=*/true,
+                                    ICIS_NoInit);
+  MemberExpr *ME = new (Context) MemberExpr(PE, true, FD, SourceLocation(),
+                                            FD->getType(), VK_LValue,
+                                            OK_Ordinary);
+
+  
+  CastExpr *FunkCast = NoTypeInfoCStyleCastExpr(Context, PtrToFuncCastType,
+                                                CK_BitCast, ME);
+  PE = new (Context) ParenExpr(SourceLocation(), SourceLocation(), FunkCast);
+
+  SmallVector<Expr*, 8> BlkExprs;
+  // Add the implicit argument.
+  BlkExprs.push_back(BlkCast);
+  // Add the user arguments.
+  for (CallExpr::arg_iterator I = Exp->arg_begin(),
+       E = Exp->arg_end(); I != E; ++I) {
+    BlkExprs.push_back(*I);
+  }
+  CallExpr *CE = new (Context) CallExpr(*Context, PE, BlkExprs,
+                                        Exp->getType(), VK_RValue,
+                                        SourceLocation());
+  return CE;
+}
+
+// We need to return the rewritten expression to handle cases where the
+// DeclRefExpr is embedded in another expression being rewritten.
+// For example:
+//
+// int main() {
+//    __block Foo *f;
+//    __block int i;
+//
+//    void (^myblock)() = ^() {
+//        [f test]; // f is a DeclRefExpr embedded in a message (which is being rewritten).
+//        i = 77;
+//    };
+//}
+Stmt *RewriteModernObjC::RewriteBlockDeclRefExpr(DeclRefExpr *DeclRefExp) {
+  // Rewrite the byref variable into BYREFVAR->__forwarding->BYREFVAR 
+  // for each DeclRefExp where BYREFVAR is name of the variable.
+  ValueDecl *VD = DeclRefExp->getDecl();
+  bool isArrow = DeclRefExp->refersToEnclosingLocal();
+  
+  FieldDecl *FD = FieldDecl::Create(*Context, 0, SourceLocation(),
+                                    SourceLocation(),
+                                    &Context->Idents.get("__forwarding"), 
+                                    Context->VoidPtrTy, 0,
+                                    /*BitWidth=*/0, /*Mutable=*/true,
+                                    ICIS_NoInit);
+  MemberExpr *ME = new (Context) MemberExpr(DeclRefExp, isArrow,
+                                            FD, SourceLocation(),
+                                            FD->getType(), VK_LValue,
+                                            OK_Ordinary);
+
+  StringRef Name = VD->getName();
+  FD = FieldDecl::Create(*Context, 0, SourceLocation(), SourceLocation(),
+                         &Context->Idents.get(Name), 
+                         Context->VoidPtrTy, 0,
+                         /*BitWidth=*/0, /*Mutable=*/true,
+                         ICIS_NoInit);
+  ME = new (Context) MemberExpr(ME, true, FD, SourceLocation(),
+                                DeclRefExp->getType(), VK_LValue, OK_Ordinary);
+  
+  
+  
+  // Need parens to enforce precedence.
+  ParenExpr *PE = new (Context) ParenExpr(DeclRefExp->getExprLoc(), 
+                                          DeclRefExp->getExprLoc(), 
+                                          ME);
+  ReplaceStmt(DeclRefExp, PE);
+  return PE;
+}
+
+// Rewrites the imported local variable V with external storage 
+// (static, extern, etc.) as *V
+//
+Stmt *RewriteModernObjC::RewriteLocalVariableExternalStorage(DeclRefExpr *DRE) {
+  ValueDecl *VD = DRE->getDecl();
+  if (VarDecl *Var = dyn_cast<VarDecl>(VD))
+    if (!ImportedLocalExternalDecls.count(Var))
+      return DRE;
+  Expr *Exp = new (Context) UnaryOperator(DRE, UO_Deref, DRE->getType(),
+                                          VK_LValue, OK_Ordinary,
+                                          DRE->getLocation());
+  // Need parens to enforce precedence.
+  ParenExpr *PE = new (Context) ParenExpr(SourceLocation(), SourceLocation(), 
+                                          Exp);
+  ReplaceStmt(DRE, PE);
+  return PE;
+}
+
+void RewriteModernObjC::RewriteCastExpr(CStyleCastExpr *CE) {
+  SourceLocation LocStart = CE->getLParenLoc();
+  SourceLocation LocEnd = CE->getRParenLoc();
+
+  // Need to avoid trying to rewrite synthesized casts.
+  if (LocStart.isInvalid())
+    return;
+  // Need to avoid trying to rewrite casts contained in macros.
+  if (!Rewriter::isRewritable(LocStart) || !Rewriter::isRewritable(LocEnd))
+    return;
+
+  const char *startBuf = SM->getCharacterData(LocStart);
+  const char *endBuf = SM->getCharacterData(LocEnd);
+  QualType QT = CE->getType();
+  const Type* TypePtr = QT->getAs<Type>();
+  if (isa<TypeOfExprType>(TypePtr)) {
+    const TypeOfExprType *TypeOfExprTypePtr = cast<TypeOfExprType>(TypePtr);
+    QT = TypeOfExprTypePtr->getUnderlyingExpr()->getType();
+    std::string TypeAsString = "(";
+    RewriteBlockPointerType(TypeAsString, QT);
+    TypeAsString += ")";
+    ReplaceText(LocStart, endBuf-startBuf+1, TypeAsString);
+    return;
+  }
+  // advance the location to startArgList.
+  const char *argPtr = startBuf;
+
+  while (*argPtr++ && (argPtr < endBuf)) {
+    switch (*argPtr) {
+    case '^':
+      // Replace the '^' with '*'.
+      LocStart = LocStart.getLocWithOffset(argPtr-startBuf);
+      ReplaceText(LocStart, 1, "*");
+      break;
+    }
+  }
+  return;
+}
+
+void RewriteModernObjC::RewriteImplicitCastObjCExpr(CastExpr *IC) {
+  CastKind CastKind = IC->getCastKind();
+  if (CastKind != CK_BlockPointerToObjCPointerCast &&
+      CastKind != CK_AnyPointerToBlockPointerCast)
+    return;
+  
+  QualType QT = IC->getType();
+  (void)convertBlockPointerToFunctionPointer(QT);
+  std::string TypeString(QT.getAsString(Context->getPrintingPolicy()));
+  std::string Str = "(";
+  Str += TypeString;
+  Str += ")";
+  InsertText(IC->getSubExpr()->getLocStart(), &Str[0], Str.size());
+
+  return;
+}
+
+void RewriteModernObjC::RewriteBlockPointerFunctionArgs(FunctionDecl *FD) {
+  SourceLocation DeclLoc = FD->getLocation();
+  unsigned parenCount = 0;
+
+  // We have 1 or more arguments that have closure pointers.
+  const char *startBuf = SM->getCharacterData(DeclLoc);
+  const char *startArgList = strchr(startBuf, '(');
+
+  assert((*startArgList == '(') && "Rewriter fuzzy parser confused");
+
+  parenCount++;
+  // advance the location to startArgList.
+  DeclLoc = DeclLoc.getLocWithOffset(startArgList-startBuf);
+  assert((DeclLoc.isValid()) && "Invalid DeclLoc");
+
+  const char *argPtr = startArgList;
+
+  while (*argPtr++ && parenCount) {
+    switch (*argPtr) {
+    case '^':
+      // Replace the '^' with '*'.
+      DeclLoc = DeclLoc.getLocWithOffset(argPtr-startArgList);
+      ReplaceText(DeclLoc, 1, "*");
+      break;
+    case '(':
+      parenCount++;
+      break;
+    case ')':
+      parenCount--;
+      break;
+    }
+  }
+  return;
+}
+
+bool RewriteModernObjC::PointerTypeTakesAnyBlockArguments(QualType QT) {
+  const FunctionProtoType *FTP;
+  const PointerType *PT = QT->getAs<PointerType>();
+  if (PT) {
+    FTP = PT->getPointeeType()->getAs<FunctionProtoType>();
+  } else {
+    const BlockPointerType *BPT = QT->getAs<BlockPointerType>();
+    assert(BPT && "BlockPointerTypeTakeAnyBlockArguments(): not a block pointer type");
+    FTP = BPT->getPointeeType()->getAs<FunctionProtoType>();
+  }
+  if (FTP) {
+    for (FunctionProtoType::arg_type_iterator I = FTP->arg_type_begin(),
+         E = FTP->arg_type_end(); I != E; ++I)
+      if (isTopLevelBlockPointerType(*I))
+        return true;
+  }
+  return false;
+}
+
+bool RewriteModernObjC::PointerTypeTakesAnyObjCQualifiedType(QualType QT) {
+  const FunctionProtoType *FTP;
+  const PointerType *PT = QT->getAs<PointerType>();
+  if (PT) {
+    FTP = PT->getPointeeType()->getAs<FunctionProtoType>();
+  } else {
+    const BlockPointerType *BPT = QT->getAs<BlockPointerType>();
+    assert(BPT && "BlockPointerTypeTakeAnyBlockArguments(): not a block pointer type");
+    FTP = BPT->getPointeeType()->getAs<FunctionProtoType>();
+  }
+  if (FTP) {
+    for (FunctionProtoType::arg_type_iterator I = FTP->arg_type_begin(),
+         E = FTP->arg_type_end(); I != E; ++I) {
+      if ((*I)->isObjCQualifiedIdType())
+        return true;
+      if ((*I)->isObjCObjectPointerType() &&
+          (*I)->getPointeeType()->isObjCQualifiedInterfaceType())
+        return true;
+    }
+        
+  }
+  return false;
+}
+
+void RewriteModernObjC::GetExtentOfArgList(const char *Name, const char *&LParen,
+                                     const char *&RParen) {
+  const char *argPtr = strchr(Name, '(');
+  assert((*argPtr == '(') && "Rewriter fuzzy parser confused");
+
+  LParen = argPtr; // output the start.
+  argPtr++; // skip past the left paren.
+  unsigned parenCount = 1;
+
+  while (*argPtr && parenCount) {
+    switch (*argPtr) {
+    case '(': parenCount++; break;
+    case ')': parenCount--; break;
+    default: break;
+    }
+    if (parenCount) argPtr++;
+  }
+  assert((*argPtr == ')') && "Rewriter fuzzy parser confused");
+  RParen = argPtr; // output the end
+}
+
+void RewriteModernObjC::RewriteBlockPointerDecl(NamedDecl *ND) {
+  if (FunctionDecl *FD = dyn_cast<FunctionDecl>(ND)) {
+    RewriteBlockPointerFunctionArgs(FD);
+    return;
+  }
+  // Handle Variables and Typedefs.
+  SourceLocation DeclLoc = ND->getLocation();
+  QualType DeclT;
+  if (VarDecl *VD = dyn_cast<VarDecl>(ND))
+    DeclT = VD->getType();
+  else if (TypedefNameDecl *TDD = dyn_cast<TypedefNameDecl>(ND))
+    DeclT = TDD->getUnderlyingType();
+  else if (FieldDecl *FD = dyn_cast<FieldDecl>(ND))
+    DeclT = FD->getType();
+  else
+    llvm_unreachable("RewriteBlockPointerDecl(): Decl type not yet handled");
+
+  const char *startBuf = SM->getCharacterData(DeclLoc);
+  const char *endBuf = startBuf;
+  // scan backward (from the decl location) for the end of the previous decl.
+  while (*startBuf != '^' && *startBuf != ';' && startBuf != MainFileStart)
+    startBuf--;
+  SourceLocation Start = DeclLoc.getLocWithOffset(startBuf-endBuf);
+  std::string buf;
+  unsigned OrigLength=0;
+  // *startBuf != '^' if we are dealing with a pointer to function that
+  // may take block argument types (which will be handled below).
+  if (*startBuf == '^') {
+    // Replace the '^' with '*', computing a negative offset.
+    buf = '*';
+    startBuf++;
+    OrigLength++;
+  }
+  while (*startBuf != ')') {
+    buf += *startBuf;
+    startBuf++;
+    OrigLength++;
+  }
+  buf += ')';
+  OrigLength++;
+  
+  if (PointerTypeTakesAnyBlockArguments(DeclT) ||
+      PointerTypeTakesAnyObjCQualifiedType(DeclT)) {
+    // Replace the '^' with '*' for arguments.
+    // Replace id<P> with id/*<>*/
+    DeclLoc = ND->getLocation();
+    startBuf = SM->getCharacterData(DeclLoc);
+    const char *argListBegin, *argListEnd;
+    GetExtentOfArgList(startBuf, argListBegin, argListEnd);
+    while (argListBegin < argListEnd) {
+      if (*argListBegin == '^')
+        buf += '*';
+      else if (*argListBegin ==  '<') {
+        buf += "/*"; 
+        buf += *argListBegin++;
+        OrigLength++;
+        while (*argListBegin != '>') {
+          buf += *argListBegin++;
+          OrigLength++;
+        }
+        buf += *argListBegin;
+        buf += "*/";
+      }
+      else
+        buf += *argListBegin;
+      argListBegin++;
+      OrigLength++;
+    }
+    buf += ')';
+    OrigLength++;
+  }
+  ReplaceText(Start, OrigLength, buf);
+  
+  return;
+}
+
+
+/// SynthesizeByrefCopyDestroyHelper - This routine synthesizes:
+/// void __Block_byref_id_object_copy(struct Block_byref_id_object *dst,
+///                    struct Block_byref_id_object *src) {
+///  _Block_object_assign (&_dest->object, _src->object, 
+///                        BLOCK_BYREF_CALLER | BLOCK_FIELD_IS_OBJECT
+///                        [|BLOCK_FIELD_IS_WEAK]) // object
+///  _Block_object_assign(&_dest->object, _src->object, 
+///                       BLOCK_BYREF_CALLER | BLOCK_FIELD_IS_BLOCK
+///                       [|BLOCK_FIELD_IS_WEAK]) // block
+/// }
+/// And:
+/// void __Block_byref_id_object_dispose(struct Block_byref_id_object *_src) {
+///  _Block_object_dispose(_src->object, 
+///                        BLOCK_BYREF_CALLER | BLOCK_FIELD_IS_OBJECT
+///                        [|BLOCK_FIELD_IS_WEAK]) // object
+///  _Block_object_dispose(_src->object, 
+///                         BLOCK_BYREF_CALLER | BLOCK_FIELD_IS_BLOCK
+///                         [|BLOCK_FIELD_IS_WEAK]) // block
+/// }
+
+std::string RewriteModernObjC::SynthesizeByrefCopyDestroyHelper(VarDecl *VD,
+                                                          int flag) {
+  std::string S;
+  if (CopyDestroyCache.count(flag))
+    return S;
+  CopyDestroyCache.insert(flag);
+  S = "static void __Block_byref_id_object_copy_";
+  S += utostr(flag);
+  S += "(void *dst, void *src) {\n";
+  
+  // offset into the object pointer is computed as:
+  // void * + void* + int + int + void* + void *
+  unsigned IntSize = 
+  static_cast<unsigned>(Context->getTypeSize(Context->IntTy));
+  unsigned VoidPtrSize = 
+  static_cast<unsigned>(Context->getTypeSize(Context->VoidPtrTy));
+  
+  unsigned offset = (VoidPtrSize*4 + IntSize + IntSize)/Context->getCharWidth();
+  S += " _Block_object_assign((char*)dst + ";
+  S += utostr(offset);
+  S += ", *(void * *) ((char*)src + ";
+  S += utostr(offset);
+  S += "), ";
+  S += utostr(flag);
+  S += ");\n}\n";
+  
+  S += "static void __Block_byref_id_object_dispose_";
+  S += utostr(flag);
+  S += "(void *src) {\n";
+  S += " _Block_object_dispose(*(void * *) ((char*)src + ";
+  S += utostr(offset);
+  S += "), ";
+  S += utostr(flag);
+  S += ");\n}\n";
+  return S;
+}
+
+/// RewriteByRefVar - For each __block typex ND variable this routine transforms
+/// the declaration into:
+/// struct __Block_byref_ND {
+/// void *__isa;                  // NULL for everything except __weak pointers
+/// struct __Block_byref_ND *__forwarding;
+/// int32_t __flags;
+/// int32_t __size;
+/// void *__Block_byref_id_object_copy; // If variable is __block ObjC object
+/// void *__Block_byref_id_object_dispose; // If variable is __block ObjC object
+/// typex ND;
+/// };
+///
+/// It then replaces declaration of ND variable with:
+/// struct __Block_byref_ND ND = {__isa=0B, __forwarding=&ND, __flags=some_flag, 
+///                               __size=sizeof(struct __Block_byref_ND), 
+///                               ND=initializer-if-any};
+///
+///
+void RewriteModernObjC::RewriteByRefVar(VarDecl *ND, bool firstDecl,
+                                        bool lastDecl) {
+  int flag = 0;
+  int isa = 0;
+  SourceLocation DeclLoc = ND->getTypeSpecStartLoc();
+  if (DeclLoc.isInvalid())
+    // If type location is missing, it is because of missing type (a warning).
+    // Use variable's location which is good for this case.
+    DeclLoc = ND->getLocation();
+  const char *startBuf = SM->getCharacterData(DeclLoc);
+  SourceLocation X = ND->getLocEnd();
+  X = SM->getExpansionLoc(X);
+  const char *endBuf = SM->getCharacterData(X);
+  std::string Name(ND->getNameAsString());
+  std::string ByrefType;
+  RewriteByRefString(ByrefType, Name, ND, true);
+  ByrefType += " {\n";
+  ByrefType += "  void *__isa;\n";
+  RewriteByRefString(ByrefType, Name, ND);
+  ByrefType += " *__forwarding;\n";
+  ByrefType += " int __flags;\n";
+  ByrefType += " int __size;\n";
+  // Add void *__Block_byref_id_object_copy; 
+  // void *__Block_byref_id_object_dispose; if needed.
+  QualType Ty = ND->getType();
+  bool HasCopyAndDispose = Context->BlockRequiresCopying(Ty);
+  if (HasCopyAndDispose) {
+    ByrefType += " void (*__Block_byref_id_object_copy)(void*, void*);\n";
+    ByrefType += " void (*__Block_byref_id_object_dispose)(void*);\n";
+  }
+
+  QualType T = Ty;
+  (void)convertBlockPointerToFunctionPointer(T);
+  T.getAsStringInternal(Name, Context->getPrintingPolicy());
+    
+  ByrefType += " " + Name + ";\n";
+  ByrefType += "};\n";
+  // Insert this type in global scope. It is needed by helper function.
+  SourceLocation FunLocStart;
+  if (CurFunctionDef)
+     FunLocStart = getFunctionSourceLocation(*this, CurFunctionDef);
+  else {
+    assert(CurMethodDef && "RewriteByRefVar - CurMethodDef is null");
+    FunLocStart = CurMethodDef->getLocStart();
+  }
+  InsertText(FunLocStart, ByrefType);
+  
+  if (Ty.isObjCGCWeak()) {
+    flag |= BLOCK_FIELD_IS_WEAK;
+    isa = 1;
+  }
+  if (HasCopyAndDispose) {
+    flag = BLOCK_BYREF_CALLER;
+    QualType Ty = ND->getType();
+    // FIXME. Handle __weak variable (BLOCK_FIELD_IS_WEAK) as well.
+    if (Ty->isBlockPointerType())
+      flag |= BLOCK_FIELD_IS_BLOCK;
+    else
+      flag |= BLOCK_FIELD_IS_OBJECT;
+    std::string HF = SynthesizeByrefCopyDestroyHelper(ND, flag);
+    if (!HF.empty())
+      InsertText(FunLocStart, HF);
+  }
+  
+  // struct __Block_byref_ND ND = 
+  // {0, &ND, some_flag, __size=sizeof(struct __Block_byref_ND), 
+  //  initializer-if-any};
+  bool hasInit = (ND->getInit() != 0);
+  // FIXME. rewriter does not support __block c++ objects which
+  // require construction.
+  if (hasInit)
+    if (CXXConstructExpr *CExp = dyn_cast<CXXConstructExpr>(ND->getInit())) {
+      CXXConstructorDecl *CXXDecl = CExp->getConstructor();
+      if (CXXDecl && CXXDecl->isDefaultConstructor())
+        hasInit = false;
+    }
+  
+  unsigned flags = 0;
+  if (HasCopyAndDispose)
+    flags |= BLOCK_HAS_COPY_DISPOSE;
+  Name = ND->getNameAsString();
+  ByrefType.clear();
+  RewriteByRefString(ByrefType, Name, ND);
+  std::string ForwardingCastType("(");
+  ForwardingCastType += ByrefType + " *)";
+  ByrefType += " " + Name + " = {(void*)";
+  ByrefType += utostr(isa);
+  ByrefType += "," +  ForwardingCastType + "&" + Name + ", ";
+  ByrefType += utostr(flags);
+  ByrefType += ", ";
+  ByrefType += "sizeof(";
+  RewriteByRefString(ByrefType, Name, ND);
+  ByrefType += ")";
+  if (HasCopyAndDispose) {
+    ByrefType += ", __Block_byref_id_object_copy_";
+    ByrefType += utostr(flag);
+    ByrefType += ", __Block_byref_id_object_dispose_";
+    ByrefType += utostr(flag);
+  }
+  
+  if (!firstDecl) {
+    // In multiple __block declarations, and for all but 1st declaration,
+    // find location of the separating comma. This would be start location
+    // where new text is to be inserted.
+    DeclLoc = ND->getLocation();
+    const char *startDeclBuf = SM->getCharacterData(DeclLoc);
+    const char *commaBuf = startDeclBuf;
+    while (*commaBuf != ',')
+      commaBuf--;
+    assert((*commaBuf == ',') && "RewriteByRefVar: can't find ','");
+    DeclLoc = DeclLoc.getLocWithOffset(commaBuf - startDeclBuf);
+    startBuf = commaBuf;
+  }
+  
+  if (!hasInit) {
+    ByrefType += "};\n";
+    unsigned nameSize = Name.size();
+    // for block or function pointer declaration. Name is aleady
+    // part of the declaration.
+    if (Ty->isBlockPointerType() || Ty->isFunctionPointerType())
+      nameSize = 1;
+    ReplaceText(DeclLoc, endBuf-startBuf+nameSize, ByrefType);
+  }
+  else {
+    ByrefType += ", ";
+    SourceLocation startLoc;
+    Expr *E = ND->getInit();
+    if (const CStyleCastExpr *ECE = dyn_cast<CStyleCastExpr>(E))
+      startLoc = ECE->getLParenLoc();
+    else
+      startLoc = E->getLocStart();
+    startLoc = SM->getExpansionLoc(startLoc);
+    endBuf = SM->getCharacterData(startLoc);
+    ReplaceText(DeclLoc, endBuf-startBuf, ByrefType);
+
+    const char separator = lastDecl ? ';' : ',';
+    const char *startInitializerBuf = SM->getCharacterData(startLoc);
+    const char *separatorBuf = strchr(startInitializerBuf, separator);
+    assert((*separatorBuf == separator) && 
+           "RewriteByRefVar: can't find ';' or ','");
+    SourceLocation separatorLoc =
+      startLoc.getLocWithOffset(separatorBuf-startInitializerBuf);
+    
+    InsertText(separatorLoc, lastDecl ? "}" : "};\n");
+  }
+  return;
+}
+
+void RewriteModernObjC::CollectBlockDeclRefInfo(BlockExpr *Exp) {
+  // Add initializers for any closure decl refs.
+  GetBlockDeclRefExprs(Exp->getBody());
+  if (BlockDeclRefs.size()) {
+    // Unique all "by copy" declarations.
+    for (unsigned i = 0; i < BlockDeclRefs.size(); i++)
+      if (!BlockDeclRefs[i]->getDecl()->hasAttr<BlocksAttr>()) {
+        if (!BlockByCopyDeclsPtrSet.count(BlockDeclRefs[i]->getDecl())) {
+          BlockByCopyDeclsPtrSet.insert(BlockDeclRefs[i]->getDecl());
+          BlockByCopyDecls.push_back(BlockDeclRefs[i]->getDecl());
+        }
+      }
+    // Unique all "by ref" declarations.
+    for (unsigned i = 0; i < BlockDeclRefs.size(); i++)
+      if (BlockDeclRefs[i]->getDecl()->hasAttr<BlocksAttr>()) {
+        if (!BlockByRefDeclsPtrSet.count(BlockDeclRefs[i]->getDecl())) {
+          BlockByRefDeclsPtrSet.insert(BlockDeclRefs[i]->getDecl());
+          BlockByRefDecls.push_back(BlockDeclRefs[i]->getDecl());
+        }
+      }
+    // Find any imported blocks...they will need special attention.
+    for (unsigned i = 0; i < BlockDeclRefs.size(); i++)
+      if (BlockDeclRefs[i]->getDecl()->hasAttr<BlocksAttr>() ||
+          BlockDeclRefs[i]->getType()->isObjCObjectPointerType() || 
+          BlockDeclRefs[i]->getType()->isBlockPointerType())
+        ImportedBlockDecls.insert(BlockDeclRefs[i]->getDecl());
+  }
+}
+
+FunctionDecl *RewriteModernObjC::SynthBlockInitFunctionDecl(StringRef name) {
+  IdentifierInfo *ID = &Context->Idents.get(name);
+  QualType FType = Context->getFunctionNoProtoType(Context->VoidPtrTy);
+  return FunctionDecl::Create(*Context, TUDecl, SourceLocation(),
+                              SourceLocation(), ID, FType, 0, SC_Extern,
+                              SC_None, false, false);
+}
+
+Stmt *RewriteModernObjC::SynthBlockInitExpr(BlockExpr *Exp,
+          const SmallVector<DeclRefExpr *, 8> &InnerBlockDeclRefs) {
+  
+  const BlockDecl *block = Exp->getBlockDecl();
+  
+  Blocks.push_back(Exp);
+
+  CollectBlockDeclRefInfo(Exp);
+  
+  // Add inner imported variables now used in current block.
+ int countOfInnerDecls = 0;
+  if (!InnerBlockDeclRefs.empty()) {
+    for (unsigned i = 0; i < InnerBlockDeclRefs.size(); i++) {
+      DeclRefExpr *Exp = InnerBlockDeclRefs[i];
+      ValueDecl *VD = Exp->getDecl();
+      if (!VD->hasAttr<BlocksAttr>() && !BlockByCopyDeclsPtrSet.count(VD)) {
+      // We need to save the copied-in variables in nested
+      // blocks because it is needed at the end for some of the API generations.
+      // See SynthesizeBlockLiterals routine.
+        InnerDeclRefs.push_back(Exp); countOfInnerDecls++;
+        BlockDeclRefs.push_back(Exp);
+        BlockByCopyDeclsPtrSet.insert(VD);
+        BlockByCopyDecls.push_back(VD);
+      }
+      if (VD->hasAttr<BlocksAttr>() && !BlockByRefDeclsPtrSet.count(VD)) {
+        InnerDeclRefs.push_back(Exp); countOfInnerDecls++;
+        BlockDeclRefs.push_back(Exp);
+        BlockByRefDeclsPtrSet.insert(VD);
+        BlockByRefDecls.push_back(VD);
+      }
+    }
+    // Find any imported blocks...they will need special attention.
+    for (unsigned i = 0; i < InnerBlockDeclRefs.size(); i++)
+      if (InnerBlockDeclRefs[i]->getDecl()->hasAttr<BlocksAttr>() ||
+          InnerBlockDeclRefs[i]->getType()->isObjCObjectPointerType() || 
+          InnerBlockDeclRefs[i]->getType()->isBlockPointerType())
+        ImportedBlockDecls.insert(InnerBlockDeclRefs[i]->getDecl());
+  }
+  InnerDeclRefsCount.push_back(countOfInnerDecls);
+  
+  std::string FuncName;
+
+  if (CurFunctionDef)
+    FuncName = CurFunctionDef->getNameAsString();
+  else if (CurMethodDef)
+    BuildUniqueMethodName(FuncName, CurMethodDef);
+  else if (GlobalVarDecl)
+    FuncName = std::string(GlobalVarDecl->getNameAsString());
+
+  bool GlobalBlockExpr = 
+    block->getDeclContext()->getRedeclContext()->isFileContext();
+  
+  if (GlobalBlockExpr && !GlobalVarDecl) {
+    Diags.Report(block->getLocation(), GlobalBlockRewriteFailedDiag);
+    GlobalBlockExpr = false;
+  }
+  
+  std::string BlockNumber = utostr(Blocks.size()-1);
+
+  std::string Func = "__" + FuncName + "_block_func_" + BlockNumber;
+
+  // Get a pointer to the function type so we can cast appropriately.
+  QualType BFT = convertFunctionTypeOfBlocks(Exp->getFunctionType());
+  QualType FType = Context->getPointerType(BFT);
+
+  FunctionDecl *FD;
+  Expr *NewRep;
+
+  // Simulate a contructor call...
+  std::string Tag;
+  
+  if (GlobalBlockExpr)
+    Tag = "__global_";
+  else
+    Tag = "__";
+  Tag += FuncName + "_block_impl_" + BlockNumber;
+  
+  FD = SynthBlockInitFunctionDecl(Tag);
+  DeclRefExpr *DRE = new (Context) DeclRefExpr(FD, false, FType, VK_RValue,
+                                               SourceLocation());
+
+  SmallVector<Expr*, 4> InitExprs;
+
+  // Initialize the block function.
+  FD = SynthBlockInitFunctionDecl(Func);
+  DeclRefExpr *Arg = new (Context) DeclRefExpr(FD, false, FD->getType(),
+                                               VK_LValue, SourceLocation());
+  CastExpr *castExpr = NoTypeInfoCStyleCastExpr(Context, Context->VoidPtrTy,
+                                                CK_BitCast, Arg);
+  InitExprs.push_back(castExpr);
+
+  // Initialize the block descriptor.
+  std::string DescData = "__" + FuncName + "_block_desc_" + BlockNumber + "_DATA";
+
+  VarDecl *NewVD = VarDecl::Create(*Context, TUDecl,
+                                   SourceLocation(), SourceLocation(),
+                                   &Context->Idents.get(DescData.c_str()),
+                                   Context->VoidPtrTy, 0,
+                                   SC_Static, SC_None);
+  UnaryOperator *DescRefExpr =
+    new (Context) UnaryOperator(new (Context) DeclRefExpr(NewVD, false,
+                                                          Context->VoidPtrTy,
+                                                          VK_LValue,
+                                                          SourceLocation()), 
+                                UO_AddrOf,
+                                Context->getPointerType(Context->VoidPtrTy), 
+                                VK_RValue, OK_Ordinary,
+                                SourceLocation());
+  InitExprs.push_back(DescRefExpr); 
+  
+  // Add initializers for any closure decl refs.
+  if (BlockDeclRefs.size()) {
+    Expr *Exp;
+    // Output all "by copy" declarations.
+    for (SmallVector<ValueDecl*,8>::iterator I = BlockByCopyDecls.begin(),
+         E = BlockByCopyDecls.end(); I != E; ++I) {
+      if (isObjCType((*I)->getType())) {
+        // FIXME: Conform to ABI ([[obj retain] autorelease]).
+        FD = SynthBlockInitFunctionDecl((*I)->getName());
+        Exp = new (Context) DeclRefExpr(FD, false, FD->getType(),
+                                        VK_LValue, SourceLocation());
+        if (HasLocalVariableExternalStorage(*I)) {
+          QualType QT = (*I)->getType();
+          QT = Context->getPointerType(QT);
+          Exp = new (Context) UnaryOperator(Exp, UO_AddrOf, QT, VK_RValue,
+                                            OK_Ordinary, SourceLocation());
+        }
+      } else if (isTopLevelBlockPointerType((*I)->getType())) {
+        FD = SynthBlockInitFunctionDecl((*I)->getName());
+        Arg = new (Context) DeclRefExpr(FD, false, FD->getType(),
+                                        VK_LValue, SourceLocation());
+        Exp = NoTypeInfoCStyleCastExpr(Context, Context->VoidPtrTy,
+                                       CK_BitCast, Arg);
+      } else {
+        FD = SynthBlockInitFunctionDecl((*I)->getName());
+        Exp = new (Context) DeclRefExpr(FD, false, FD->getType(),
+                                        VK_LValue, SourceLocation());
+        if (HasLocalVariableExternalStorage(*I)) {
+          QualType QT = (*I)->getType();
+          QT = Context->getPointerType(QT);
+          Exp = new (Context) UnaryOperator(Exp, UO_AddrOf, QT, VK_RValue,
+                                            OK_Ordinary, SourceLocation());
+        }
+        
+      }
+      InitExprs.push_back(Exp);
+    }
+    // Output all "by ref" declarations.
+    for (SmallVector<ValueDecl*,8>::iterator I = BlockByRefDecls.begin(),
+         E = BlockByRefDecls.end(); I != E; ++I) {
+      ValueDecl *ND = (*I);
+      std::string Name(ND->getNameAsString());
+      std::string RecName;
+      RewriteByRefString(RecName, Name, ND, true);
+      IdentifierInfo *II = &Context->Idents.get(RecName.c_str() 
+                                                + sizeof("struct"));
+      RecordDecl *RD = RecordDecl::Create(*Context, TTK_Struct, TUDecl,
+                                          SourceLocation(), SourceLocation(),
+                                          II);
+      assert(RD && "SynthBlockInitExpr(): Can't find RecordDecl");
+      QualType castT = Context->getPointerType(Context->getTagDeclType(RD));
+      
+      FD = SynthBlockInitFunctionDecl((*I)->getName());
+      Exp = new (Context) DeclRefExpr(FD, false, FD->getType(), VK_LValue,
+                                      SourceLocation());
+      bool isNestedCapturedVar = false;
+      if (block)
+        for (BlockDecl::capture_const_iterator ci = block->capture_begin(),
+             ce = block->capture_end(); ci != ce; ++ci) {
+          const VarDecl *variable = ci->getVariable();
+          if (variable == ND && ci->isNested()) {
+            assert (ci->isByRef() && 
+                    "SynthBlockInitExpr - captured block variable is not byref");
+            isNestedCapturedVar = true;
+            break;
+          }
+        }
+      // captured nested byref variable has its address passed. Do not take
+      // its address again.
+      if (!isNestedCapturedVar)
+          Exp = new (Context) UnaryOperator(Exp, UO_AddrOf,
+                                     Context->getPointerType(Exp->getType()),
+                                     VK_RValue, OK_Ordinary, SourceLocation());
+      Exp = NoTypeInfoCStyleCastExpr(Context, castT, CK_BitCast, Exp);
+      InitExprs.push_back(Exp);
+    }
+  }
+  if (ImportedBlockDecls.size()) {
+    // generate BLOCK_HAS_COPY_DISPOSE(have helper funcs) | BLOCK_HAS_DESCRIPTOR
+    int flag = (BLOCK_HAS_COPY_DISPOSE | BLOCK_HAS_DESCRIPTOR);
+    unsigned IntSize = 
+      static_cast<unsigned>(Context->getTypeSize(Context->IntTy));
+    Expr *FlagExp = IntegerLiteral::Create(*Context, llvm::APInt(IntSize, flag), 
+                                           Context->IntTy, SourceLocation());
+    InitExprs.push_back(FlagExp);
+  }
+  NewRep = new (Context) CallExpr(*Context, DRE, InitExprs,
+                                  FType, VK_LValue, SourceLocation());
+  
+  if (GlobalBlockExpr) {
+    assert (GlobalConstructionExp == 0 && 
+            "SynthBlockInitExpr - GlobalConstructionExp must be null");
+    GlobalConstructionExp = NewRep;
+    NewRep = DRE;
+  }
+  
+  NewRep = new (Context) UnaryOperator(NewRep, UO_AddrOf,
+                             Context->getPointerType(NewRep->getType()),
+                             VK_RValue, OK_Ordinary, SourceLocation());
+  NewRep = NoTypeInfoCStyleCastExpr(Context, FType, CK_BitCast,
+                                    NewRep);
+  BlockDeclRefs.clear();
+  BlockByRefDecls.clear();
+  BlockByRefDeclsPtrSet.clear();
+  BlockByCopyDecls.clear();
+  BlockByCopyDeclsPtrSet.clear();
+  ImportedBlockDecls.clear();
+  return NewRep;
+}
+
+bool RewriteModernObjC::IsDeclStmtInForeachHeader(DeclStmt *DS) {
+  if (const ObjCForCollectionStmt * CS = 
+      dyn_cast<ObjCForCollectionStmt>(Stmts.back()))
+        return CS->getElement() == DS;
+  return false;
+}
+
+//===----------------------------------------------------------------------===//
+// Function Body / Expression rewriting
+//===----------------------------------------------------------------------===//
+
+Stmt *RewriteModernObjC::RewriteFunctionBodyOrGlobalInitializer(Stmt *S) {
+  if (isa<SwitchStmt>(S) || isa<WhileStmt>(S) ||
+      isa<DoStmt>(S) || isa<ForStmt>(S))
+    Stmts.push_back(S);
+  else if (isa<ObjCForCollectionStmt>(S)) {
+    Stmts.push_back(S);
+    ObjCBcLabelNo.push_back(++BcLabelCount);
+  }
+
+  // Pseudo-object operations and ivar references need special
+  // treatment because we're going to recursively rewrite them.
+  if (PseudoObjectExpr *PseudoOp = dyn_cast<PseudoObjectExpr>(S)) {
+    if (isa<BinaryOperator>(PseudoOp->getSyntacticForm())) {
+      return RewritePropertyOrImplicitSetter(PseudoOp);
+    } else {
+      return RewritePropertyOrImplicitGetter(PseudoOp);
+    }
+  } else if (ObjCIvarRefExpr *IvarRefExpr = dyn_cast<ObjCIvarRefExpr>(S)) {
+    return RewriteObjCIvarRefExpr(IvarRefExpr);
+  }
+
+  SourceRange OrigStmtRange = S->getSourceRange();
+
+  // Perform a bottom up rewrite of all children.
+  for (Stmt::child_range CI = S->children(); CI; ++CI)
+    if (*CI) {
+      Stmt *childStmt = (*CI);
+      Stmt *newStmt = RewriteFunctionBodyOrGlobalInitializer(childStmt);
+      if (newStmt) {
+        *CI = newStmt;
+      }
+    }
+
+  if (BlockExpr *BE = dyn_cast<BlockExpr>(S)) {
+    SmallVector<DeclRefExpr *, 8> InnerBlockDeclRefs;
+    llvm::SmallPtrSet<const DeclContext *, 8> InnerContexts;
+    InnerContexts.insert(BE->getBlockDecl());
+    ImportedLocalExternalDecls.clear();
+    GetInnerBlockDeclRefExprs(BE->getBody(),
+                              InnerBlockDeclRefs, InnerContexts);
+    // Rewrite the block body in place.
+    Stmt *SaveCurrentBody = CurrentBody;
+    CurrentBody = BE->getBody();
+    PropParentMap = 0;
+    // block literal on rhs of a property-dot-sytax assignment
+    // must be replaced by its synthesize ast so getRewrittenText
+    // works as expected. In this case, what actually ends up on RHS
+    // is the blockTranscribed which is the helper function for the
+    // block literal; as in: self.c = ^() {[ace ARR];};
+    bool saveDisableReplaceStmt = DisableReplaceStmt;
+    DisableReplaceStmt = false;
+    RewriteFunctionBodyOrGlobalInitializer(BE->getBody());
+    DisableReplaceStmt = saveDisableReplaceStmt;
+    CurrentBody = SaveCurrentBody;
+    PropParentMap = 0;
+    ImportedLocalExternalDecls.clear();
+    // Now we snarf the rewritten text and stash it away for later use.
+    std::string Str = Rewrite.getRewrittenText(BE->getSourceRange());
+    RewrittenBlockExprs[BE] = Str;
+
+    Stmt *blockTranscribed = SynthBlockInitExpr(BE, InnerBlockDeclRefs);
+                            
+    //blockTranscribed->dump();
+    ReplaceStmt(S, blockTranscribed);
+    return blockTranscribed;
+  }
+  // Handle specific things.
+  if (ObjCEncodeExpr *AtEncode = dyn_cast<ObjCEncodeExpr>(S))
+    return RewriteAtEncode(AtEncode);
+
+  if (ObjCSelectorExpr *AtSelector = dyn_cast<ObjCSelectorExpr>(S))
+    return RewriteAtSelector(AtSelector);
+
+  if (ObjCStringLiteral *AtString = dyn_cast<ObjCStringLiteral>(S))
+    return RewriteObjCStringLiteral(AtString);
+  
+  if (ObjCBoolLiteralExpr *BoolLitExpr = dyn_cast<ObjCBoolLiteralExpr>(S))
+    return RewriteObjCBoolLiteralExpr(BoolLitExpr);
+  
+  if (ObjCBoxedExpr *BoxedExpr = dyn_cast<ObjCBoxedExpr>(S))
+    return RewriteObjCBoxedExpr(BoxedExpr);
+  
+  if (ObjCArrayLiteral *ArrayLitExpr = dyn_cast<ObjCArrayLiteral>(S))
+    return RewriteObjCArrayLiteralExpr(ArrayLitExpr);
+  
+  if (ObjCDictionaryLiteral *DictionaryLitExpr = 
+        dyn_cast<ObjCDictionaryLiteral>(S))
+    return RewriteObjCDictionaryLiteralExpr(DictionaryLitExpr);
+
+  if (ObjCMessageExpr *MessExpr = dyn_cast<ObjCMessageExpr>(S)) {
+#if 0
+    // Before we rewrite it, put the original message expression in a comment.
+    SourceLocation startLoc = MessExpr->getLocStart();
+    SourceLocation endLoc = MessExpr->getLocEnd();
+
+    const char *startBuf = SM->getCharacterData(startLoc);
+    const char *endBuf = SM->getCharacterData(endLoc);
+
+    std::string messString;
+    messString += "// ";
+    messString.append(startBuf, endBuf-startBuf+1);
+    messString += "\n";
+
+    // FIXME: Missing definition of
+    // InsertText(clang::SourceLocation, char const*, unsigned int).
+    // InsertText(startLoc, messString.c_str(), messString.size());
+    // Tried this, but it didn't work either...
+    // ReplaceText(startLoc, 0, messString.c_str(), messString.size());
+#endif
+    return RewriteMessageExpr(MessExpr);
+  }
+
+  if (ObjCAutoreleasePoolStmt *StmtAutoRelease = 
+        dyn_cast<ObjCAutoreleasePoolStmt>(S)) {
+    return RewriteObjCAutoreleasePoolStmt(StmtAutoRelease);
+  }
+  
+  if (ObjCAtTryStmt *StmtTry = dyn_cast<ObjCAtTryStmt>(S))
+    return RewriteObjCTryStmt(StmtTry);
+
+  if (ObjCAtSynchronizedStmt *StmtTry = dyn_cast<ObjCAtSynchronizedStmt>(S))
+    return RewriteObjCSynchronizedStmt(StmtTry);
+
+  if (ObjCAtThrowStmt *StmtThrow = dyn_cast<ObjCAtThrowStmt>(S))
+    return RewriteObjCThrowStmt(StmtThrow);
+
+  if (ObjCProtocolExpr *ProtocolExp = dyn_cast<ObjCProtocolExpr>(S))
+    return RewriteObjCProtocolExpr(ProtocolExp);
+
+  if (ObjCForCollectionStmt *StmtForCollection =
+        dyn_cast<ObjCForCollectionStmt>(S))
+    return RewriteObjCForCollectionStmt(StmtForCollection,
+                                        OrigStmtRange.getEnd());
+  if (BreakStmt *StmtBreakStmt =
+      dyn_cast<BreakStmt>(S))
+    return RewriteBreakStmt(StmtBreakStmt);
+  if (ContinueStmt *StmtContinueStmt =
+      dyn_cast<ContinueStmt>(S))
+    return RewriteContinueStmt(StmtContinueStmt);
+
+  // Need to check for protocol refs (id <P>, Foo <P> *) in variable decls
+  // and cast exprs.
+  if (DeclStmt *DS = dyn_cast<DeclStmt>(S)) {
+    // FIXME: What we're doing here is modifying the type-specifier that
+    // precedes the first Decl.  In the future the DeclGroup should have
+    // a separate type-specifier that we can rewrite.
+    // NOTE: We need to avoid rewriting the DeclStmt if it is within
+    // the context of an ObjCForCollectionStmt. For example:
+    //   NSArray *someArray;
+    //   for (id <FooProtocol> index in someArray) ;
+    // This is because RewriteObjCForCollectionStmt() does textual rewriting 
+    // and it depends on the original text locations/positions.
+    if (Stmts.empty() || !IsDeclStmtInForeachHeader(DS))
+      RewriteObjCQualifiedInterfaceTypes(*DS->decl_begin());
+
+    // Blocks rewrite rules.
+    for (DeclStmt::decl_iterator DI = DS->decl_begin(), DE = DS->decl_end();
+         DI != DE; ++DI) {
+      Decl *SD = *DI;
+      if (ValueDecl *ND = dyn_cast<ValueDecl>(SD)) {
+        if (isTopLevelBlockPointerType(ND->getType()))
+          RewriteBlockPointerDecl(ND);
+        else if (ND->getType()->isFunctionPointerType())
+          CheckFunctionPointerDecl(ND->getType(), ND);
+        if (VarDecl *VD = dyn_cast<VarDecl>(SD)) {
+          if (VD->hasAttr<BlocksAttr>()) {
+            static unsigned uniqueByrefDeclCount = 0;
+            assert(!BlockByRefDeclNo.count(ND) &&
+              "RewriteFunctionBodyOrGlobalInitializer: Duplicate byref decl");
+            BlockByRefDeclNo[ND] = uniqueByrefDeclCount++;
+            RewriteByRefVar(VD, (DI == DS->decl_begin()), ((DI+1) == DE));
+          }
+          else           
+            RewriteTypeOfDecl(VD);
+        }
+      }
+      if (TypedefNameDecl *TD = dyn_cast<TypedefNameDecl>(SD)) {
+        if (isTopLevelBlockPointerType(TD->getUnderlyingType()))
+          RewriteBlockPointerDecl(TD);
+        else if (TD->getUnderlyingType()->isFunctionPointerType())
+          CheckFunctionPointerDecl(TD->getUnderlyingType(), TD);
+      }
+    }
+  }
+
+  if (CStyleCastExpr *CE = dyn_cast<CStyleCastExpr>(S))
+    RewriteObjCQualifiedInterfaceTypes(CE);
+
+  if (isa<SwitchStmt>(S) || isa<WhileStmt>(S) ||
+      isa<DoStmt>(S) || isa<ForStmt>(S)) {
+    assert(!Stmts.empty() && "Statement stack is empty");
+    assert ((isa<SwitchStmt>(Stmts.back()) || isa<WhileStmt>(Stmts.back()) ||
+             isa<DoStmt>(Stmts.back()) || isa<ForStmt>(Stmts.back()))
+            && "Statement stack mismatch");
+    Stmts.pop_back();
+  }
+  // Handle blocks rewriting.
+  if (DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(S)) {
+    ValueDecl *VD = DRE->getDecl(); 
+    if (VD->hasAttr<BlocksAttr>())
+      return RewriteBlockDeclRefExpr(DRE);
+    if (HasLocalVariableExternalStorage(VD))
+      return RewriteLocalVariableExternalStorage(DRE);
+  }
+  
+  if (CallExpr *CE = dyn_cast<CallExpr>(S)) {
+    if (CE->getCallee()->getType()->isBlockPointerType()) {
+      Stmt *BlockCall = SynthesizeBlockCall(CE, CE->getCallee());
+      ReplaceStmt(S, BlockCall);
+      return BlockCall;
+    }
+  }
+  if (CStyleCastExpr *CE = dyn_cast<CStyleCastExpr>(S)) {
+    RewriteCastExpr(CE);
+  }
+  if (ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(S)) {
+    RewriteImplicitCastObjCExpr(ICE);
+  }
+#if 0
+
+  if (ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(S)) {
+    CastExpr *Replacement = new (Context) CastExpr(ICE->getType(),
+                                                   ICE->getSubExpr(),
+                                                   SourceLocation());
+    // Get the new text.
+    std::string SStr;
+    llvm::raw_string_ostream Buf(SStr);
+    Replacement->printPretty(Buf);
+    const std::string &Str = Buf.str();
+
+    printf("CAST = %s\n", &Str[0]);
+    InsertText(ICE->getSubExpr()->getLocStart(), &Str[0], Str.size());
+    delete S;
+    return Replacement;
+  }
+#endif
+  // Return this stmt unmodified.
+  return S;
+}
+
+void RewriteModernObjC::RewriteRecordBody(RecordDecl *RD) {
+  for (RecordDecl::field_iterator i = RD->field_begin(), 
+                                  e = RD->field_end(); i != e; ++i) {
+    FieldDecl *FD = *i;
+    if (isTopLevelBlockPointerType(FD->getType()))
+      RewriteBlockPointerDecl(FD);
+    if (FD->getType()->isObjCQualifiedIdType() ||
+        FD->getType()->isObjCQualifiedInterfaceType())
+      RewriteObjCQualifiedInterfaceTypes(FD);
+  }
+}
+
+/// HandleDeclInMainFile - This is called for each top-level decl defined in the
+/// main file of the input.
+void RewriteModernObjC::HandleDeclInMainFile(Decl *D) {
+  switch (D->getKind()) {
+    case Decl::Function: {
+      FunctionDecl *FD = cast<FunctionDecl>(D);
+      if (FD->isOverloadedOperator())
+        return;
+
+      // Since function prototypes don't have ParmDecl's, we check the function
+      // prototype. This enables us to rewrite function declarations and
+      // definitions using the same code.
+      RewriteBlocksInFunctionProtoType(FD->getType(), FD);
+
+      if (!FD->isThisDeclarationADefinition())
+        break;
+
+      // FIXME: If this should support Obj-C++, support CXXTryStmt
+      if (CompoundStmt *Body = dyn_cast_or_null<CompoundStmt>(FD->getBody())) {
+        CurFunctionDef = FD;
+        CurrentBody = Body;
+        Body =
+        cast_or_null<CompoundStmt>(RewriteFunctionBodyOrGlobalInitializer(Body));
+        FD->setBody(Body);
+        CurrentBody = 0;
+        if (PropParentMap) {
+          delete PropParentMap;
+          PropParentMap = 0;
+        }
+        // This synthesizes and inserts the block "impl" struct, invoke function,
+        // and any copy/dispose helper functions.
+        InsertBlockLiteralsWithinFunction(FD);
+        RewriteLineDirective(D);
+        CurFunctionDef = 0;
+      }
+      break;
+    }
+    case Decl::ObjCMethod: {
+      ObjCMethodDecl *MD = cast<ObjCMethodDecl>(D);
+      if (CompoundStmt *Body = MD->getCompoundBody()) {
+        CurMethodDef = MD;
+        CurrentBody = Body;
+        Body =
+          cast_or_null<CompoundStmt>(RewriteFunctionBodyOrGlobalInitializer(Body));
+        MD->setBody(Body);
+        CurrentBody = 0;
+        if (PropParentMap) {
+          delete PropParentMap;
+          PropParentMap = 0;
+        }
+        InsertBlockLiteralsWithinMethod(MD);
+        RewriteLineDirective(D);
+        CurMethodDef = 0;
+      }
+      break;
+    }
+    case Decl::ObjCImplementation: {
+      ObjCImplementationDecl *CI = cast<ObjCImplementationDecl>(D);
+      ClassImplementation.push_back(CI);
+      break;
+    }
+    case Decl::ObjCCategoryImpl: {
+      ObjCCategoryImplDecl *CI = cast<ObjCCategoryImplDecl>(D);
+      CategoryImplementation.push_back(CI);
+      break;
+    }
+    case Decl::Var: {
+      VarDecl *VD = cast<VarDecl>(D);
+      RewriteObjCQualifiedInterfaceTypes(VD);
+      if (isTopLevelBlockPointerType(VD->getType()))
+        RewriteBlockPointerDecl(VD);
+      else if (VD->getType()->isFunctionPointerType()) {
+        CheckFunctionPointerDecl(VD->getType(), VD);
+        if (VD->getInit()) {
+          if (CStyleCastExpr *CE = dyn_cast<CStyleCastExpr>(VD->getInit())) {
+            RewriteCastExpr(CE);
+          }
+        }
+      } else if (VD->getType()->isRecordType()) {
+        RecordDecl *RD = VD->getType()->getAs<RecordType>()->getDecl();
+        if (RD->isCompleteDefinition())
+          RewriteRecordBody(RD);
+      }
+      if (VD->getInit()) {
+        GlobalVarDecl = VD;
+        CurrentBody = VD->getInit();
+        RewriteFunctionBodyOrGlobalInitializer(VD->getInit());
+        CurrentBody = 0;
+        if (PropParentMap) {
+          delete PropParentMap;
+          PropParentMap = 0;
+        }
+        SynthesizeBlockLiterals(VD->getTypeSpecStartLoc(), VD->getName());
+        GlobalVarDecl = 0;
+          
+        // This is needed for blocks.
+        if (CStyleCastExpr *CE = dyn_cast<CStyleCastExpr>(VD->getInit())) {
+            RewriteCastExpr(CE);
+        }
+      }
+      break;
+    }
+    case Decl::TypeAlias:
+    case Decl::Typedef: {
+      if (TypedefNameDecl *TD = dyn_cast<TypedefNameDecl>(D)) {
+        if (isTopLevelBlockPointerType(TD->getUnderlyingType()))
+          RewriteBlockPointerDecl(TD);
+        else if (TD->getUnderlyingType()->isFunctionPointerType())
+          CheckFunctionPointerDecl(TD->getUnderlyingType(), TD);
+      }
+      break;
+    }
+    case Decl::CXXRecord:
+    case Decl::Record: {
+      RecordDecl *RD = cast<RecordDecl>(D);
+      if (RD->isCompleteDefinition()) 
+        RewriteRecordBody(RD);
+      break;
+    }
+    default:
+      break;
+  }
+  // Nothing yet.
+}
+
+/// Write_ProtocolExprReferencedMetadata - This routine writer out the
+/// protocol reference symbols in the for of:
+/// struct _protocol_t *PROTOCOL_REF = &PROTOCOL_METADATA.
+static void Write_ProtocolExprReferencedMetadata(ASTContext *Context, 
+                                                 ObjCProtocolDecl *PDecl,
+                                                 std::string &Result) {
+  // Also output .objc_protorefs$B section and its meta-data.
+  if (Context->getLangOpts().MicrosoftExt)
+    Result += "static ";
+  Result += "struct _protocol_t *";
+  Result += "_OBJC_PROTOCOL_REFERENCE_$_";
+  Result += PDecl->getNameAsString();
+  Result += " = &";
+  Result += "_OBJC_PROTOCOL_"; Result += PDecl->getNameAsString();
+  Result += ";\n";
+}
+
+void RewriteModernObjC::HandleTranslationUnit(ASTContext &C) {
+  if (Diags.hasErrorOccurred())
+    return;
+
+  RewriteInclude();
+
+  // Here's a great place to add any extra declarations that may be needed.
+  // Write out meta data for each @protocol(<expr>).
+  for (llvm::SmallPtrSet<ObjCProtocolDecl *,8>::iterator I = ProtocolExprDecls.begin(),
+       E = ProtocolExprDecls.end(); I != E; ++I) {
+    RewriteObjCProtocolMetaData(*I, Preamble);
+    Write_ProtocolExprReferencedMetadata(Context, (*I), Preamble);
+  }
+
+  InsertText(SM->getLocForStartOfFile(MainFileID), Preamble, false);
+  
+  if (ClassImplementation.size() || CategoryImplementation.size())
+    RewriteImplementations();
+  
+  for (unsigned i = 0, e = ObjCInterfacesSeen.size(); i < e; i++) {
+    ObjCInterfaceDecl *CDecl = ObjCInterfacesSeen[i];
+    // Write struct declaration for the class matching its ivar declarations.
+    // Note that for modern abi, this is postponed until the end of TU
+    // because class extensions and the implementation might declare their own
+    // private ivars.
+    RewriteInterfaceDecl(CDecl);
+  }
+
+  // Get the buffer corresponding to MainFileID.  If we haven't changed it, then
+  // we are done.
+  if (const RewriteBuffer *RewriteBuf =
+      Rewrite.getRewriteBufferFor(MainFileID)) {
+    //printf("Changed:\n");
+    *OutFile << std::string(RewriteBuf->begin(), RewriteBuf->end());
+  } else {
+    llvm::errs() << "No changes\n";
+  }
+
+  if (ClassImplementation.size() || CategoryImplementation.size() ||
+      ProtocolExprDecls.size()) {
+    // Rewrite Objective-c meta data*
+    std::string ResultStr;
+    RewriteMetaDataIntoBuffer(ResultStr);
+    // Emit metadata.
+    *OutFile << ResultStr;
+  }
+  // Emit ImageInfo;
+  {
+    std::string ResultStr;
+    WriteImageInfo(ResultStr);
+    *OutFile << ResultStr;
+  }
+  OutFile->flush();
+}
+
+void RewriteModernObjC::Initialize(ASTContext &context) {
+  InitializeCommon(context);
+  
+  Preamble += "#ifndef __OBJC2__\n";
+  Preamble += "#define __OBJC2__\n";
+  Preamble += "#endif\n";
+
+  // declaring objc_selector outside the parameter list removes a silly
+  // scope related warning...
+  if (IsHeader)
+    Preamble = "#pragma once\n";
+  Preamble += "struct objc_selector; struct objc_class;\n";
+  Preamble += "struct __rw_objc_super { \n\tstruct objc_object *object; ";
+  Preamble += "\n\tstruct objc_object *superClass; ";
+  // Add a constructor for creating temporary objects.
+  Preamble += "\n\t__rw_objc_super(struct objc_object *o, struct objc_object *s) ";
+  Preamble += ": object(o), superClass(s) {} ";
+  Preamble += "\n};\n";
+  
+  if (LangOpts.MicrosoftExt) {
+    // Define all sections using syntax that makes sense.
+    // These are currently generated.
+    Preamble += "\n#pragma section(\".objc_classlist$B\", long, read, write)\n";
+    Preamble += "#pragma section(\".objc_catlist$B\", long, read, write)\n";
+    Preamble += "#pragma section(\".objc_imageinfo$B\", long, read, write)\n";
+    Preamble += "#pragma section(\".objc_nlclslist$B\", long, read, write)\n";
+    Preamble += "#pragma section(\".objc_nlcatlist$B\", long, read, write)\n";
+    // These are generated but not necessary for functionality.
+    Preamble += "#pragma section(\".cat_cls_meth$B\", long, read, write)\n";
+    Preamble += "#pragma section(\".inst_meth$B\", long, read, write)\n";
+    Preamble += "#pragma section(\".cls_meth$B\", long, read, write)\n";
+    Preamble += "#pragma section(\".objc_ivar$B\", long, read, write)\n";
+    
+    // These need be generated for performance. Currently they are not,
+    // using API calls instead.
+    Preamble += "#pragma section(\".objc_selrefs$B\", long, read, write)\n";
+    Preamble += "#pragma section(\".objc_classrefs$B\", long, read, write)\n";
+    Preamble += "#pragma section(\".objc_superrefs$B\", long, read, write)\n";
+    
+  }
+  Preamble += "#ifndef _REWRITER_typedef_Protocol\n";
+  Preamble += "typedef struct objc_object Protocol;\n";
+  Preamble += "#define _REWRITER_typedef_Protocol\n";
+  Preamble += "#endif\n";
+  if (LangOpts.MicrosoftExt) {
+    Preamble += "#define __OBJC_RW_DLLIMPORT extern \"C\" __declspec(dllimport)\n";
+    Preamble += "#define __OBJC_RW_STATICIMPORT extern \"C\"\n";
+  } 
+  else
+    Preamble += "#define __OBJC_RW_DLLIMPORT extern\n";
+  
+  Preamble += "__OBJC_RW_DLLIMPORT void objc_msgSend(void);\n";
+  Preamble += "__OBJC_RW_DLLIMPORT void objc_msgSendSuper(void);\n";
+  Preamble += "__OBJC_RW_DLLIMPORT void objc_msgSend_stret(void);\n";
+  Preamble += "__OBJC_RW_DLLIMPORT void objc_msgSendSuper_stret(void);\n";
+  Preamble += "__OBJC_RW_DLLIMPORT void objc_msgSend_fpret(void);\n";
+
+  Preamble += "__OBJC_RW_DLLIMPORT struct objc_class *objc_getClass";
+  Preamble += "(const char *);\n";
+  Preamble += "__OBJC_RW_DLLIMPORT struct objc_class *class_getSuperclass";
+  Preamble += "(struct objc_class *);\n";
+  Preamble += "__OBJC_RW_DLLIMPORT struct objc_class *objc_getMetaClass";
+  Preamble += "(const char *);\n";
+  Preamble += "__OBJC_RW_DLLIMPORT void objc_exception_throw( struct objc_object *);\n";
+  // @synchronized hooks.
+  Preamble += "__OBJC_RW_DLLIMPORT int objc_sync_enter( struct objc_object *);\n";
+  Preamble += "__OBJC_RW_DLLIMPORT int objc_sync_exit( struct objc_object *);\n";
+  Preamble += "__OBJC_RW_DLLIMPORT Protocol *objc_getProtocol(const char *);\n";
+  Preamble += "#ifndef __FASTENUMERATIONSTATE\n";
+  Preamble += "struct __objcFastEnumerationState {\n\t";
+  Preamble += "unsigned long state;\n\t";
+  Preamble += "void **itemsPtr;\n\t";
+  Preamble += "unsigned long *mutationsPtr;\n\t";
+  Preamble += "unsigned long extra[5];\n};\n";
+  Preamble += "__OBJC_RW_DLLIMPORT void objc_enumerationMutation(struct objc_object *);\n";
+  Preamble += "#define __FASTENUMERATIONSTATE\n";
+  Preamble += "#endif\n";
+  Preamble += "#ifndef __NSCONSTANTSTRINGIMPL\n";
+  Preamble += "struct __NSConstantStringImpl {\n";
+  Preamble += "  int *isa;\n";
+  Preamble += "  int flags;\n";
+  Preamble += "  char *str;\n";
+  Preamble += "  long length;\n";
+  Preamble += "};\n";
+  Preamble += "#ifdef CF_EXPORT_CONSTANT_STRING\n";
+  Preamble += "extern \"C\" __declspec(dllexport) int __CFConstantStringClassReference[];\n";
+  Preamble += "#else\n";
+  Preamble += "__OBJC_RW_DLLIMPORT int __CFConstantStringClassReference[];\n";
+  Preamble += "#endif\n";
+  Preamble += "#define __NSCONSTANTSTRINGIMPL\n";
+  Preamble += "#endif\n";
+  // Blocks preamble.
+  Preamble += "#ifndef BLOCK_IMPL\n";
+  Preamble += "#define BLOCK_IMPL\n";
+  Preamble += "struct __block_impl {\n";
+  Preamble += "  void *isa;\n";
+  Preamble += "  int Flags;\n";
+  Preamble += "  int Reserved;\n";
+  Preamble += "  void *FuncPtr;\n";
+  Preamble += "};\n";
+  Preamble += "// Runtime copy/destroy helper functions (from Block_private.h)\n";
+  Preamble += "#ifdef __OBJC_EXPORT_BLOCKS\n";
+  Preamble += "extern \"C\" __declspec(dllexport) "
+  "void _Block_object_assign(void *, const void *, const int);\n";
+  Preamble += "extern \"C\" __declspec(dllexport) void _Block_object_dispose(const void *, const int);\n";
+  Preamble += "extern \"C\" __declspec(dllexport) void *_NSConcreteGlobalBlock[32];\n";
+  Preamble += "extern \"C\" __declspec(dllexport) void *_NSConcreteStackBlock[32];\n";
+  Preamble += "#else\n";
+  Preamble += "__OBJC_RW_DLLIMPORT void _Block_object_assign(void *, const void *, const int);\n";
+  Preamble += "__OBJC_RW_DLLIMPORT void _Block_object_dispose(const void *, const int);\n";
+  Preamble += "__OBJC_RW_DLLIMPORT void *_NSConcreteGlobalBlock[32];\n";
+  Preamble += "__OBJC_RW_DLLIMPORT void *_NSConcreteStackBlock[32];\n";
+  Preamble += "#endif\n";
+  Preamble += "#endif\n";
+  if (LangOpts.MicrosoftExt) {
+    Preamble += "#undef __OBJC_RW_DLLIMPORT\n";
+    Preamble += "#undef __OBJC_RW_STATICIMPORT\n";
+    Preamble += "#ifndef KEEP_ATTRIBUTES\n";  // We use this for clang tests.
+    Preamble += "#define __attribute__(X)\n";
+    Preamble += "#endif\n";
+    Preamble += "#ifndef __weak\n";
+    Preamble += "#define __weak\n";
+    Preamble += "#endif\n";
+    Preamble += "#ifndef __block\n";
+    Preamble += "#define __block\n";
+    Preamble += "#endif\n";
+  }
+  else {
+    Preamble += "#define __block\n";
+    Preamble += "#define __weak\n";
+  }
+  
+  // Declarations required for modern objective-c array and dictionary literals.
+  Preamble += "\n#include <stdarg.h>\n";
+  Preamble += "struct __NSContainer_literal {\n";
+  Preamble += "  void * *arr;\n";
+  Preamble += "  __NSContainer_literal (unsigned int count, ...) {\n";
+  Preamble += "\tva_list marker;\n";
+  Preamble += "\tva_start(marker, count);\n";
+  Preamble += "\tarr = new void *[count];\n";
+  Preamble += "\tfor (unsigned i = 0; i < count; i++)\n";
+  Preamble += "\t  arr[i] = va_arg(marker, void *);\n";
+  Preamble += "\tva_end( marker );\n";
+  Preamble += "  };\n";
+  Preamble += "  ~__NSContainer_literal() {\n";
+  Preamble += "\tdelete[] arr;\n";
+  Preamble += "  }\n";
+  Preamble += "};\n";
+  
+  // Declaration required for implementation of @autoreleasepool statement.
+  Preamble += "extern \"C\" __declspec(dllimport) void * objc_autoreleasePoolPush(void);\n";
+  Preamble += "extern \"C\" __declspec(dllimport) void objc_autoreleasePoolPop(void *);\n\n";
+  Preamble += "struct __AtAutoreleasePool {\n";
+  Preamble += "  __AtAutoreleasePool() {atautoreleasepoolobj = objc_autoreleasePoolPush();}\n";
+  Preamble += "  ~__AtAutoreleasePool() {objc_autoreleasePoolPop(atautoreleasepoolobj);}\n";
+  Preamble += "  void * atautoreleasepoolobj;\n";
+  Preamble += "};\n";
+  
+  // NOTE! Windows uses LLP64 for 64bit mode. So, cast pointer to long long
+  // as this avoids warning in any 64bit/32bit compilation model.
+  Preamble += "\n#define __OFFSETOFIVAR__(TYPE, MEMBER) ((long long) &((TYPE *)0)->MEMBER)\n";
+}
+
+/// RewriteIvarOffsetComputation - This rutine synthesizes computation of
+/// ivar offset.
+void RewriteModernObjC::RewriteIvarOffsetComputation(ObjCIvarDecl *ivar,
+                                                         std::string &Result) {
+  if (ivar->isBitField()) {
+    // FIXME: The hack below doesn't work for bitfields. For now, we simply
+    // place all bitfields at offset 0.
+    Result += "0";
+  } else {
+    Result += "__OFFSETOFIVAR__(struct ";
+    Result += ivar->getContainingInterface()->getNameAsString();
+    if (LangOpts.MicrosoftExt)
+      Result += "_IMPL";
+    Result += ", ";
+    Result += ivar->getNameAsString();
+    Result += ")";
+  }
+}
+
+/// WriteModernMetadataDeclarations - Writes out metadata declarations for modern ABI.
+/// struct _prop_t {
+///   const char *name;
+///   char *attributes;
+/// }
+
+/// struct _prop_list_t {
+///   uint32_t entsize;      // sizeof(struct _prop_t)
+///   uint32_t count_of_properties;
+///   struct _prop_t prop_list[count_of_properties];
+/// }
+
+/// struct _protocol_t;
+
+/// struct _protocol_list_t {
+///   long protocol_count;   // Note, this is 32/64 bit
+///   struct _protocol_t * protocol_list[protocol_count];
+/// }
+
+/// struct _objc_method {
+///   SEL _cmd;
+///   const char *method_type;
+///   char *_imp;
+/// }
+
+/// struct _method_list_t {
+///   uint32_t entsize;  // sizeof(struct _objc_method)
+///   uint32_t method_count;
+///   struct _objc_method method_list[method_count];
+/// }
+
+/// struct _protocol_t {
+///   id isa;  // NULL
+///   const char *protocol_name;
+///   const struct _protocol_list_t * protocol_list; // super protocols
+///   const struct method_list_t *instance_methods;
+///   const struct method_list_t *class_methods;
+///   const struct method_list_t *optionalInstanceMethods;
+///   const struct method_list_t *optionalClassMethods;
+///   const struct _prop_list_t * properties;
+///   const uint32_t size;  // sizeof(struct _protocol_t)
+///   const uint32_t flags;  // = 0
+///   const char ** extendedMethodTypes;
+/// }
+
+/// struct _ivar_t {
+///   unsigned long int *offset;  // pointer to ivar offset location
+///   const char *name;
+///   const char *type;
+///   uint32_t alignment;
+///   uint32_t size;
+/// }
+
+/// struct _ivar_list_t {
+///   uint32 entsize;  // sizeof(struct _ivar_t)
+///   uint32 count;
+///   struct _ivar_t list[count];
+/// }
+
+/// struct _class_ro_t {
+///   uint32_t flags;
+///   uint32_t instanceStart;
+///   uint32_t instanceSize;
+///   uint32_t reserved;  // only when building for 64bit targets
+///   const uint8_t *ivarLayout;
+///   const char *name;
+///   const struct _method_list_t *baseMethods;
+///   const struct _protocol_list_t *baseProtocols;
+///   const struct _ivar_list_t *ivars;
+///   const uint8_t *weakIvarLayout;
+///   const struct _prop_list_t *properties;
+/// }
+
+/// struct _class_t {
+///   struct _class_t *isa;
+///   struct _class_t *superclass;
+///   void *cache;
+///   IMP *vtable;
+///   struct _class_ro_t *ro;
+/// }
+
+/// struct _category_t {
+///   const char *name;
+///   struct _class_t *cls;
+///   const struct _method_list_t *instance_methods;
+///   const struct _method_list_t *class_methods;
+///   const struct _protocol_list_t *protocols;
+///   const struct _prop_list_t *properties;
+/// }
+
+/// MessageRefTy - LLVM for:
+/// struct _message_ref_t {
+///   IMP messenger;
+///   SEL name;
+/// };
+
+/// SuperMessageRefTy - LLVM for:
+/// struct _super_message_ref_t {
+///   SUPER_IMP messenger;
+///   SEL name;
+/// };
+
+static void WriteModernMetadataDeclarations(ASTContext *Context, std::string &Result) {
+  static bool meta_data_declared = false;
+  if (meta_data_declared)
+    return;
+  
+  Result += "\nstruct _prop_t {\n";
+  Result += "\tconst char *name;\n";
+  Result += "\tconst char *attributes;\n";
+  Result += "};\n";
+  
+  Result += "\nstruct _protocol_t;\n";
+  
+  Result += "\nstruct _objc_method {\n";
+  Result += "\tstruct objc_selector * _cmd;\n";
+  Result += "\tconst char *method_type;\n";
+  Result += "\tvoid  *_imp;\n";
+  Result += "};\n";
+  
+  Result += "\nstruct _protocol_t {\n";
+  Result += "\tvoid * isa;  // NULL\n";
+  Result += "\tconst char *protocol_name;\n";
+  Result += "\tconst struct _protocol_list_t * protocol_list; // super protocols\n";
+  Result += "\tconst struct method_list_t *instance_methods;\n";
+  Result += "\tconst struct method_list_t *class_methods;\n";
+  Result += "\tconst struct method_list_t *optionalInstanceMethods;\n";
+  Result += "\tconst struct method_list_t *optionalClassMethods;\n";
+  Result += "\tconst struct _prop_list_t * properties;\n";
+  Result += "\tconst unsigned int size;  // sizeof(struct _protocol_t)\n";
+  Result += "\tconst unsigned int flags;  // = 0\n";
+  Result += "\tconst char ** extendedMethodTypes;\n";
+  Result += "};\n";
+  
+  Result += "\nstruct _ivar_t {\n";
+  Result += "\tunsigned long int *offset;  // pointer to ivar offset location\n";
+  Result += "\tconst char *name;\n";
+  Result += "\tconst char *type;\n";
+  Result += "\tunsigned int alignment;\n";
+  Result += "\tunsigned int  size;\n";
+  Result += "};\n";
+  
+  Result += "\nstruct _class_ro_t {\n";
+  Result += "\tunsigned int flags;\n";
+  Result += "\tunsigned int instanceStart;\n";
+  Result += "\tunsigned int instanceSize;\n";
+  const llvm::Triple &Triple(Context->getTargetInfo().getTriple());
+  if (Triple.getArch() == llvm::Triple::x86_64)
+    Result += "\tunsigned int reserved;\n";
+  Result += "\tconst unsigned char *ivarLayout;\n";
+  Result += "\tconst char *name;\n";
+  Result += "\tconst struct _method_list_t *baseMethods;\n";
+  Result += "\tconst struct _objc_protocol_list *baseProtocols;\n";
+  Result += "\tconst struct _ivar_list_t *ivars;\n";
+  Result += "\tconst unsigned char *weakIvarLayout;\n";
+  Result += "\tconst struct _prop_list_t *properties;\n";
+  Result += "};\n";
+  
+  Result += "\nstruct _class_t {\n";
+  Result += "\tstruct _class_t *isa;\n";
+  Result += "\tstruct _class_t *superclass;\n";
+  Result += "\tvoid *cache;\n";
+  Result += "\tvoid *vtable;\n";
+  Result += "\tstruct _class_ro_t *ro;\n";
+  Result += "};\n";
+  
+  Result += "\nstruct _category_t {\n";
+  Result += "\tconst char *name;\n";
+  Result += "\tstruct _class_t *cls;\n";
+  Result += "\tconst struct _method_list_t *instance_methods;\n";
+  Result += "\tconst struct _method_list_t *class_methods;\n";
+  Result += "\tconst struct _protocol_list_t *protocols;\n";
+  Result += "\tconst struct _prop_list_t *properties;\n";
+  Result += "};\n";
+  
+  Result += "extern \"C\" __declspec(dllimport) struct objc_cache _objc_empty_cache;\n";
+  Result += "#pragma warning(disable:4273)\n";
+  meta_data_declared = true;
+}
+
+static void Write_protocol_list_t_TypeDecl(std::string &Result,
+                                           long super_protocol_count) {
+  Result += "struct /*_protocol_list_t*/"; Result += " {\n";
+  Result += "\tlong protocol_count;  // Note, this is 32/64 bit\n";
+  Result += "\tstruct _protocol_t *super_protocols[";
+  Result += utostr(super_protocol_count); Result += "];\n";
+  Result += "}";
+}
+
+static void Write_method_list_t_TypeDecl(std::string &Result,
+                                         unsigned int method_count) {
+  Result += "struct /*_method_list_t*/"; Result += " {\n";
+  Result += "\tunsigned int entsize;  // sizeof(struct _objc_method)\n";
+  Result += "\tunsigned int method_count;\n";
+  Result += "\tstruct _objc_method method_list[";
+  Result += utostr(method_count); Result += "];\n";
+  Result += "}";
+}
+
+static void Write__prop_list_t_TypeDecl(std::string &Result,
+                                        unsigned int property_count) {
+  Result += "struct /*_prop_list_t*/"; Result += " {\n";
+  Result += "\tunsigned int entsize;  // sizeof(struct _prop_t)\n";
+  Result += "\tunsigned int count_of_properties;\n";
+  Result += "\tstruct _prop_t prop_list[";
+  Result += utostr(property_count); Result += "];\n";
+  Result += "}";
+}
+
+static void Write__ivar_list_t_TypeDecl(std::string &Result,
+                                        unsigned int ivar_count) {
+  Result += "struct /*_ivar_list_t*/"; Result += " {\n";
+  Result += "\tunsigned int entsize;  // sizeof(struct _prop_t)\n";
+  Result += "\tunsigned int count;\n";
+  Result += "\tstruct _ivar_t ivar_list[";
+  Result += utostr(ivar_count); Result += "];\n";
+  Result += "}";
+}
+
+static void Write_protocol_list_initializer(ASTContext *Context, std::string &Result,
+                                            ArrayRef<ObjCProtocolDecl *> SuperProtocols,
+                                            StringRef VarName,
+                                            StringRef ProtocolName) {
+  if (SuperProtocols.size() > 0) {
+    Result += "\nstatic ";
+    Write_protocol_list_t_TypeDecl(Result, SuperProtocols.size());
+    Result += " "; Result += VarName;
+    Result += ProtocolName; 
+    Result += " __attribute__ ((used, section (\"__DATA,__objc_const\"))) = {\n";
+    Result += "\t"; Result += utostr(SuperProtocols.size()); Result += ",\n";
+    for (unsigned i = 0, e = SuperProtocols.size(); i < e; i++) {
+      ObjCProtocolDecl *SuperPD = SuperProtocols[i];
+      Result += "\t&"; Result += "_OBJC_PROTOCOL_"; 
+      Result += SuperPD->getNameAsString();
+      if (i == e-1)
+        Result += "\n};\n";
+      else
+        Result += ",\n";
+    }
+  }
+}
+
+static void Write_method_list_t_initializer(RewriteModernObjC &RewriteObj,
+                                            ASTContext *Context, std::string &Result,
+                                            ArrayRef<ObjCMethodDecl *> Methods,
+                                            StringRef VarName,
+                                            StringRef TopLevelDeclName,
+                                            bool MethodImpl) {
+  if (Methods.size() > 0) {
+    Result += "\nstatic ";
+    Write_method_list_t_TypeDecl(Result, Methods.size());
+    Result += " "; Result += VarName;
+    Result += TopLevelDeclName; 
+    Result += " __attribute__ ((used, section (\"__DATA,__objc_const\"))) = {\n";
+    Result += "\t"; Result += "sizeof(_objc_method)"; Result += ",\n";
+    Result += "\t"; Result += utostr(Methods.size()); Result += ",\n";
+    for (unsigned i = 0, e = Methods.size(); i < e; i++) {
+      ObjCMethodDecl *MD = Methods[i];
+      if (i == 0)
+        Result += "\t{{(struct objc_selector *)\"";
+      else
+        Result += "\t{(struct objc_selector *)\"";
+      Result += (MD)->getSelector().getAsString(); Result += "\"";
+      Result += ", ";
+      std::string MethodTypeString;
+      Context->getObjCEncodingForMethodDecl(MD, MethodTypeString);
+      Result += "\""; Result += MethodTypeString; Result += "\"";
+      Result += ", ";
+      if (!MethodImpl)
+        Result += "0";
+      else {
+        Result += "(void *)";
+        Result += RewriteObj.MethodInternalNames[MD];
+      }
+      if (i  == e-1)
+        Result += "}}\n";
+      else
+        Result += "},\n";
+    }
+    Result += "};\n";
+  }
+}
+
+static void Write_prop_list_t_initializer(RewriteModernObjC &RewriteObj,
+                                           ASTContext *Context, std::string &Result,
+                                           ArrayRef<ObjCPropertyDecl *> Properties,
+                                           const Decl *Container,
+                                           StringRef VarName,
+                                           StringRef ProtocolName) {
+  if (Properties.size() > 0) {
+    Result += "\nstatic ";
+    Write__prop_list_t_TypeDecl(Result, Properties.size());
+    Result += " "; Result += VarName;
+    Result += ProtocolName; 
+    Result += " __attribute__ ((used, section (\"__DATA,__objc_const\"))) = {\n";
+    Result += "\t"; Result += "sizeof(_prop_t)"; Result += ",\n";
+    Result += "\t"; Result += utostr(Properties.size()); Result += ",\n";
+    for (unsigned i = 0, e = Properties.size(); i < e; i++) {
+      ObjCPropertyDecl *PropDecl = Properties[i];
+      if (i == 0)
+        Result += "\t{{\"";
+      else
+        Result += "\t{\"";
+      Result += PropDecl->getName(); Result += "\",";
+      std::string PropertyTypeString, QuotePropertyTypeString;
+      Context->getObjCEncodingForPropertyDecl(PropDecl, Container, PropertyTypeString);
+      RewriteObj.QuoteDoublequotes(PropertyTypeString, QuotePropertyTypeString);
+      Result += "\""; Result += QuotePropertyTypeString; Result += "\"";
+      if (i  == e-1)
+        Result += "}}\n";
+      else
+        Result += "},\n";
+    }
+    Result += "};\n";
+  }
+}
+
+// Metadata flags
+enum MetaDataDlags {
+  CLS = 0x0,
+  CLS_META = 0x1,
+  CLS_ROOT = 0x2,
+  OBJC2_CLS_HIDDEN = 0x10,
+  CLS_EXCEPTION = 0x20,
+  
+  /// (Obsolete) ARC-specific: this class has a .release_ivars method
+  CLS_HAS_IVAR_RELEASER = 0x40,
+  /// class was compiled with -fobjc-arr
+  CLS_COMPILED_BY_ARC = 0x80  // (1<<7)
+};
+
+static void Write__class_ro_t_initializer(ASTContext *Context, std::string &Result, 
+                                          unsigned int flags, 
+                                          const std::string &InstanceStart, 
+                                          const std::string &InstanceSize,
+                                          ArrayRef<ObjCMethodDecl *>baseMethods,
+                                          ArrayRef<ObjCProtocolDecl *>baseProtocols,
+                                          ArrayRef<ObjCIvarDecl *>ivars,
+                                          ArrayRef<ObjCPropertyDecl *>Properties,
+                                          StringRef VarName,
+                                          StringRef ClassName) {
+  Result += "\nstatic struct _class_ro_t ";
+  Result += VarName; Result += ClassName;
+  Result += " __attribute__ ((used, section (\"__DATA,__objc_const\"))) = {\n";
+  Result += "\t"; 
+  Result += llvm::utostr(flags); Result += ", "; 
+  Result += InstanceStart; Result += ", ";
+  Result += InstanceSize; Result += ", \n";
+  Result += "\t";
+  const llvm::Triple &Triple(Context->getTargetInfo().getTriple());
+  if (Triple.getArch() == llvm::Triple::x86_64)
+    // uint32_t const reserved; // only when building for 64bit targets
+    Result += "(unsigned int)0, \n\t";
+  // const uint8_t * const ivarLayout;
+  Result += "0, \n\t";
+  Result += "\""; Result += ClassName; Result += "\",\n\t";
+  bool metaclass = ((flags & CLS_META) != 0);
+  if (baseMethods.size() > 0) {
+    Result += "(const struct _method_list_t *)&";
+    if (metaclass)
+      Result += "_OBJC_$_CLASS_METHODS_";
+    else
+      Result += "_OBJC_$_INSTANCE_METHODS_";
+    Result += ClassName;
+    Result += ",\n\t";
+  }
+  else
+    Result += "0, \n\t";
+
+  if (!metaclass && baseProtocols.size() > 0) {
+    Result += "(const struct _objc_protocol_list *)&";
+    Result += "_OBJC_CLASS_PROTOCOLS_$_"; Result += ClassName;
+    Result += ",\n\t";
+  }
+  else
+    Result += "0, \n\t";
+
+  if (!metaclass && ivars.size() > 0) {
+    Result += "(const struct _ivar_list_t *)&";
+    Result += "_OBJC_$_INSTANCE_VARIABLES_"; Result += ClassName;
+    Result += ",\n\t";
+  }
+  else
+    Result += "0, \n\t";
+
+  // weakIvarLayout
+  Result += "0, \n\t";
+  if (!metaclass && Properties.size() > 0) {
+    Result += "(const struct _prop_list_t *)&";
+    Result += "_OBJC_$_PROP_LIST_"; Result += ClassName;
+    Result += ",\n";
+  }
+  else
+    Result += "0, \n";
+
+  Result += "};\n";
+}
+
+static void Write_class_t(ASTContext *Context, std::string &Result,
+                          StringRef VarName,
+                          const ObjCInterfaceDecl *CDecl, bool metaclass) {
+  bool rootClass = (!CDecl->getSuperClass());
+  const ObjCInterfaceDecl *RootClass = CDecl;
+  
+  if (!rootClass) {
+    // Find the Root class
+    RootClass = CDecl->getSuperClass();
+    while (RootClass->getSuperClass()) {
+      RootClass = RootClass->getSuperClass();
+    }
+  }
+
+  if (metaclass && rootClass) {
+    // Need to handle a case of use of forward declaration.
+    Result += "\n";
+    Result += "extern \"C\" ";
+    if (CDecl->getImplementation())
+      Result += "__declspec(dllexport) ";
+    else
+      Result += "__declspec(dllimport) ";
+    
+    Result += "struct _class_t OBJC_CLASS_$_";
+    Result += CDecl->getNameAsString();
+    Result += ";\n";
+  }
+  // Also, for possibility of 'super' metadata class not having been defined yet.
+  if (!rootClass) {
+    ObjCInterfaceDecl *SuperClass = CDecl->getSuperClass();
+    Result += "\n";
+    Result += "extern \"C\" ";
+    if (SuperClass->getImplementation())
+      Result += "__declspec(dllexport) ";
+    else
+      Result += "__declspec(dllimport) ";
+
+    Result += "struct _class_t "; 
+    Result += VarName;
+    Result += SuperClass->getNameAsString();
+    Result += ";\n";
+    
+    if (metaclass && RootClass != SuperClass) {
+      Result += "extern \"C\" ";
+      if (RootClass->getImplementation())
+        Result += "__declspec(dllexport) ";
+      else
+        Result += "__declspec(dllimport) ";
+
+      Result += "struct _class_t "; 
+      Result += VarName;
+      Result += RootClass->getNameAsString();
+      Result += ";\n";
+    }
+  }
+  
+  Result += "\nextern \"C\" __declspec(dllexport) struct _class_t "; 
+  Result += VarName; Result += CDecl->getNameAsString();
+  Result += " __attribute__ ((used, section (\"__DATA,__objc_data\"))) = {\n";
+  Result += "\t";
+  if (metaclass) {
+    if (!rootClass) {
+      Result += "0, // &"; Result += VarName;
+      Result += RootClass->getNameAsString();
+      Result += ",\n\t";
+      Result += "0, // &"; Result += VarName;
+      Result += CDecl->getSuperClass()->getNameAsString();
+      Result += ",\n\t";
+    }
+    else {
+      Result += "0, // &"; Result += VarName; 
+      Result += CDecl->getNameAsString();
+      Result += ",\n\t";
+      Result += "0, // &OBJC_CLASS_$_"; Result += CDecl->getNameAsString();
+      Result += ",\n\t";
+    }
+  }
+  else {
+    Result += "0, // &OBJC_METACLASS_$_"; 
+    Result += CDecl->getNameAsString();
+    Result += ",\n\t";
+    if (!rootClass) {
+      Result += "0, // &"; Result += VarName;
+      Result += CDecl->getSuperClass()->getNameAsString();
+      Result += ",\n\t";
+    }
+    else 
+      Result += "0,\n\t";
+  }
+  Result += "0, // (void *)&_objc_empty_cache,\n\t";
+  Result += "0, // unused, was (void *)&_objc_empty_vtable,\n\t";
+  if (metaclass)
+    Result += "&_OBJC_METACLASS_RO_$_";
+  else
+    Result += "&_OBJC_CLASS_RO_$_";
+  Result += CDecl->getNameAsString();
+  Result += ",\n};\n";
+  
+  // Add static function to initialize some of the meta-data fields.
+  // avoid doing it twice.
+  if (metaclass)
+    return;
+  
+  const ObjCInterfaceDecl *SuperClass = 
+    rootClass ? CDecl : CDecl->getSuperClass();
+  
+  Result += "static void OBJC_CLASS_SETUP_$_";
+  Result += CDecl->getNameAsString();
+  Result += "(void ) {\n";
+  Result += "\tOBJC_METACLASS_$_"; Result += CDecl->getNameAsString();
+  Result += ".isa = "; Result += "&OBJC_METACLASS_$_";
+  Result += RootClass->getNameAsString(); Result += ";\n";
+  
+  Result += "\tOBJC_METACLASS_$_"; Result += CDecl->getNameAsString();
+  Result += ".superclass = ";
+  if (rootClass)
+    Result += "&OBJC_CLASS_$_";
+  else
+     Result += "&OBJC_METACLASS_$_";
+
+  Result += SuperClass->getNameAsString(); Result += ";\n";
+  
+  Result += "\tOBJC_METACLASS_$_"; Result += CDecl->getNameAsString();
+  Result += ".cache = "; Result += "&_objc_empty_cache"; Result += ";\n";
+  
+  Result += "\tOBJC_CLASS_$_"; Result += CDecl->getNameAsString();
+  Result += ".isa = "; Result += "&OBJC_METACLASS_$_";
+  Result += CDecl->getNameAsString(); Result += ";\n";
+  
+  if (!rootClass) {
+    Result += "\tOBJC_CLASS_$_"; Result += CDecl->getNameAsString();
+    Result += ".superclass = "; Result += "&OBJC_CLASS_$_";
+    Result += SuperClass->getNameAsString(); Result += ";\n";
+  }
+  
+  Result += "\tOBJC_CLASS_$_"; Result += CDecl->getNameAsString();
+  Result += ".cache = "; Result += "&_objc_empty_cache"; Result += ";\n";
+  Result += "}\n";
+}
+
+static void Write_category_t(RewriteModernObjC &RewriteObj, ASTContext *Context, 
+                             std::string &Result,
+                             ObjCCategoryDecl *CatDecl,
+                             ObjCInterfaceDecl *ClassDecl,
+                             ArrayRef<ObjCMethodDecl *> InstanceMethods,
+                             ArrayRef<ObjCMethodDecl *> ClassMethods,
+                             ArrayRef<ObjCProtocolDecl *> RefedProtocols,
+                             ArrayRef<ObjCPropertyDecl *> ClassProperties) {
+  StringRef CatName = CatDecl->getName();
+  StringRef ClassName = ClassDecl->getName();
+  // must declare an extern class object in case this class is not implemented 
+  // in this TU.
+  Result += "\n";
+  Result += "extern \"C\" ";
+  if (ClassDecl->getImplementation())
+    Result += "__declspec(dllexport) ";
+  else
+    Result += "__declspec(dllimport) ";
+  
+  Result += "struct _class_t ";
+  Result += "OBJC_CLASS_$_"; Result += ClassName;
+  Result += ";\n";
+  
+  Result += "\nstatic struct _category_t ";
+  Result += "_OBJC_$_CATEGORY_";
+  Result += ClassName; Result += "_$_"; Result += CatName;
+  Result += " __attribute__ ((used, section (\"__DATA,__objc_const\"))) = \n";
+  Result += "{\n";
+  Result += "\t\""; Result += ClassName; Result += "\",\n";
+  Result += "\t0, // &"; Result += "OBJC_CLASS_$_"; Result += ClassName;
+  Result += ",\n";
+  if (InstanceMethods.size() > 0) {
+    Result += "\t(const struct _method_list_t *)&";  
+    Result += "_OBJC_$_CATEGORY_INSTANCE_METHODS_";
+    Result += ClassName; Result += "_$_"; Result += CatName;
+    Result += ",\n";
+  }
+  else
+    Result += "\t0,\n";
+  
+  if (ClassMethods.size() > 0) {
+    Result += "\t(const struct _method_list_t *)&";  
+    Result += "_OBJC_$_CATEGORY_CLASS_METHODS_";
+    Result += ClassName; Result += "_$_"; Result += CatName;
+    Result += ",\n";
+  }
+  else
+    Result += "\t0,\n";
+  
+  if (RefedProtocols.size() > 0) {
+    Result += "\t(const struct _protocol_list_t *)&";  
+    Result += "_OBJC_CATEGORY_PROTOCOLS_$_";
+    Result += ClassName; Result += "_$_"; Result += CatName;
+    Result += ",\n";
+  }
+  else
+    Result += "\t0,\n";
+  
+  if (ClassProperties.size() > 0) {
+    Result += "\t(const struct _prop_list_t *)&";  Result += "_OBJC_$_PROP_LIST_";
+    Result += ClassName; Result += "_$_"; Result += CatName;
+    Result += ",\n";
+  }
+  else
+    Result += "\t0,\n";
+  
+  Result += "};\n";
+  
+  // Add static function to initialize the class pointer in the category structure.
+  Result += "static void OBJC_CATEGORY_SETUP_$_";
+  Result += ClassDecl->getNameAsString();
+  Result += "_$_";
+  Result += CatName;
+  Result += "(void ) {\n";
+  Result += "\t_OBJC_$_CATEGORY_"; 
+  Result += ClassDecl->getNameAsString();
+  Result += "_$_";
+  Result += CatName;
+  Result += ".cls = "; Result += "&OBJC_CLASS_$_"; Result += ClassName;
+  Result += ";\n}\n";
+}
+
+static void Write__extendedMethodTypes_initializer(RewriteModernObjC &RewriteObj,
+                                           ASTContext *Context, std::string &Result,
+                                           ArrayRef<ObjCMethodDecl *> Methods,
+                                           StringRef VarName,
+                                           StringRef ProtocolName) {
+  if (Methods.size() == 0)
+    return;
+  
+  Result += "\nstatic const char *";
+  Result += VarName; Result += ProtocolName;
+  Result += " [] __attribute__ ((used, section (\"__DATA,__objc_const\"))) = \n";
+  Result += "{\n";
+  for (unsigned i = 0, e = Methods.size(); i < e; i++) {
+    ObjCMethodDecl *MD = Methods[i];
+    std::string MethodTypeString, QuoteMethodTypeString;
+    Context->getObjCEncodingForMethodDecl(MD, MethodTypeString, true);
+    RewriteObj.QuoteDoublequotes(MethodTypeString, QuoteMethodTypeString);
+    Result += "\t\""; Result += QuoteMethodTypeString; Result += "\"";
+    if (i == e-1)
+      Result += "\n};\n";
+    else {
+      Result += ",\n";
+    }
+  }
+}
+
+static void Write_IvarOffsetVar(RewriteModernObjC &RewriteObj,
+                                ASTContext *Context,
+                                std::string &Result, 
+                                ArrayRef<ObjCIvarDecl *> Ivars, 
+                                ObjCInterfaceDecl *CDecl) {
+  // FIXME. visibilty of offset symbols may have to be set; for Darwin
+  // this is what happens:
+  /**
+   if (Ivar->getAccessControl() == ObjCIvarDecl::Private ||
+       Ivar->getAccessControl() == ObjCIvarDecl::Package ||
+       Class->getVisibility() == HiddenVisibility)
+     Visibility shoud be: HiddenVisibility;
+   else
+     Visibility shoud be: DefaultVisibility;
+  */
+  
+  Result += "\n";
+  for (unsigned i =0, e = Ivars.size(); i < e; i++) {
+    ObjCIvarDecl *IvarDecl = Ivars[i];
+    if (Context->getLangOpts().MicrosoftExt)
+      Result += "__declspec(allocate(\".objc_ivar$B\")) ";
+    
+    if (!Context->getLangOpts().MicrosoftExt ||
+        IvarDecl->getAccessControl() == ObjCIvarDecl::Private ||
+        IvarDecl->getAccessControl() == ObjCIvarDecl::Package)
+      Result += "extern \"C\" unsigned long int "; 
+    else
+      Result += "extern \"C\" __declspec(dllexport) unsigned long int ";
+    WriteInternalIvarName(CDecl, IvarDecl, Result);
+    Result += " __attribute__ ((used, section (\"__DATA,__objc_ivar\")))";
+    Result += " = ";
+    RewriteObj.RewriteIvarOffsetComputation(IvarDecl, Result);
+    Result += ";\n";
+  }
+}
+
+static void Write__ivar_list_t_initializer(RewriteModernObjC &RewriteObj,
+                                           ASTContext *Context, std::string &Result,
+                                           ArrayRef<ObjCIvarDecl *> Ivars,
+                                           StringRef VarName,
+                                           ObjCInterfaceDecl *CDecl) {
+  if (Ivars.size() > 0) {
+    Write_IvarOffsetVar(RewriteObj, Context, Result, Ivars, CDecl);
+    
+    Result += "\nstatic ";
+    Write__ivar_list_t_TypeDecl(Result, Ivars.size());
+    Result += " "; Result += VarName;
+    Result += CDecl->getNameAsString();
+    Result += " __attribute__ ((used, section (\"__DATA,__objc_const\"))) = {\n";
+    Result += "\t"; Result += "sizeof(_ivar_t)"; Result += ",\n";
+    Result += "\t"; Result += utostr(Ivars.size()); Result += ",\n";
+    for (unsigned i =0, e = Ivars.size(); i < e; i++) {
+      ObjCIvarDecl *IvarDecl = Ivars[i];
+      if (i == 0)
+        Result += "\t{{";
+      else
+        Result += "\t {";
+      Result += "(unsigned long int *)&";
+      WriteInternalIvarName(CDecl, IvarDecl, Result);
+      Result += ", ";
+      
+      Result += "\""; Result += IvarDecl->getName(); Result += "\", ";
+      std::string IvarTypeString, QuoteIvarTypeString;
+      Context->getObjCEncodingForType(IvarDecl->getType(), IvarTypeString,
+                                      IvarDecl);
+      RewriteObj.QuoteDoublequotes(IvarTypeString, QuoteIvarTypeString);
+      Result += "\""; Result += QuoteIvarTypeString; Result += "\", ";
+      
+      // FIXME. this alignment represents the host alignment and need be changed to
+      // represent the target alignment.
+      unsigned Align = Context->getTypeAlign(IvarDecl->getType())/8;
+      Align = llvm::Log2_32(Align);
+      Result += llvm::utostr(Align); Result += ", ";
+      CharUnits Size = Context->getTypeSizeInChars(IvarDecl->getType());
+      Result += llvm::utostr(Size.getQuantity());
+      if (i  == e-1)
+        Result += "}}\n";
+      else
+        Result += "},\n";
+    }
+    Result += "};\n";
+  }
+}
+
+/// RewriteObjCProtocolMetaData - Rewrite protocols meta-data.
+void RewriteModernObjC::RewriteObjCProtocolMetaData(ObjCProtocolDecl *PDecl, 
+                                                    std::string &Result) {
+  
+  // Do not synthesize the protocol more than once.
+  if (ObjCSynthesizedProtocols.count(PDecl->getCanonicalDecl()))
+    return;
+  WriteModernMetadataDeclarations(Context, Result);
+  
+  if (ObjCProtocolDecl *Def = PDecl->getDefinition())
+    PDecl = Def;
+  // Must write out all protocol definitions in current qualifier list,
+  // and in their nested qualifiers before writing out current definition.
+  for (ObjCProtocolDecl::protocol_iterator I = PDecl->protocol_begin(),
+       E = PDecl->protocol_end(); I != E; ++I)
+    RewriteObjCProtocolMetaData(*I, Result);
+  
+  // Construct method lists.
+  std::vector<ObjCMethodDecl *> InstanceMethods, ClassMethods;
+  std::vector<ObjCMethodDecl *> OptInstanceMethods, OptClassMethods;
+  for (ObjCProtocolDecl::instmeth_iterator
+       I = PDecl->instmeth_begin(), E = PDecl->instmeth_end();
+       I != E; ++I) {
+    ObjCMethodDecl *MD = *I;
+    if (MD->getImplementationControl() == ObjCMethodDecl::Optional) {
+      OptInstanceMethods.push_back(MD);
+    } else {
+      InstanceMethods.push_back(MD);
+    }
+  }
+  
+  for (ObjCProtocolDecl::classmeth_iterator
+       I = PDecl->classmeth_begin(), E = PDecl->classmeth_end();
+       I != E; ++I) {
+    ObjCMethodDecl *MD = *I;
+    if (MD->getImplementationControl() == ObjCMethodDecl::Optional) {
+      OptClassMethods.push_back(MD);
+    } else {
+      ClassMethods.push_back(MD);
+    }
+  }
+  std::vector<ObjCMethodDecl *> AllMethods;
+  for (unsigned i = 0, e = InstanceMethods.size(); i < e; i++)
+    AllMethods.push_back(InstanceMethods[i]);
+  for (unsigned i = 0, e = ClassMethods.size(); i < e; i++)
+    AllMethods.push_back(ClassMethods[i]);
+  for (unsigned i = 0, e = OptInstanceMethods.size(); i < e; i++)
+    AllMethods.push_back(OptInstanceMethods[i]);
+  for (unsigned i = 0, e = OptClassMethods.size(); i < e; i++)
+    AllMethods.push_back(OptClassMethods[i]);
+
+  Write__extendedMethodTypes_initializer(*this, Context, Result,
+                                         AllMethods,
+                                         "_OBJC_PROTOCOL_METHOD_TYPES_",
+                                         PDecl->getNameAsString());
+  // Protocol's super protocol list
+  std::vector<ObjCProtocolDecl *> SuperProtocols;
+  for (ObjCProtocolDecl::protocol_iterator I = PDecl->protocol_begin(),
+       E = PDecl->protocol_end(); I != E; ++I)
+    SuperProtocols.push_back(*I);
+  
+  Write_protocol_list_initializer(Context, Result, SuperProtocols,
+                                  "_OBJC_PROTOCOL_REFS_",
+                                  PDecl->getNameAsString());
+  
+  Write_method_list_t_initializer(*this, Context, Result, InstanceMethods, 
+                                  "_OBJC_PROTOCOL_INSTANCE_METHODS_",
+                                  PDecl->getNameAsString(), false);
+  
+  Write_method_list_t_initializer(*this, Context, Result, ClassMethods, 
+                                  "_OBJC_PROTOCOL_CLASS_METHODS_",
+                                  PDecl->getNameAsString(), false);
+
+  Write_method_list_t_initializer(*this, Context, Result, OptInstanceMethods, 
+                                  "_OBJC_PROTOCOL_OPT_INSTANCE_METHODS_",
+                                  PDecl->getNameAsString(), false);
+  
+  Write_method_list_t_initializer(*this, Context, Result, OptClassMethods, 
+                                  "_OBJC_PROTOCOL_OPT_CLASS_METHODS_",
+                                  PDecl->getNameAsString(), false);
+  
+  // Protocol's property metadata.
+  std::vector<ObjCPropertyDecl *> ProtocolProperties;
+  for (ObjCContainerDecl::prop_iterator I = PDecl->prop_begin(),
+       E = PDecl->prop_end(); I != E; ++I)
+    ProtocolProperties.push_back(*I);
+  
+  Write_prop_list_t_initializer(*this, Context, Result, ProtocolProperties,
+                                 /* Container */0,
+                                 "_OBJC_PROTOCOL_PROPERTIES_",
+                                 PDecl->getNameAsString());
+  
+  // Writer out root metadata for current protocol: struct _protocol_t
+  Result += "\n";
+  if (LangOpts.MicrosoftExt)
+    Result += "static ";
+  Result += "struct _protocol_t _OBJC_PROTOCOL_";
+  Result += PDecl->getNameAsString();
+  Result += " __attribute__ ((used, section (\"__DATA,__datacoal_nt,coalesced\"))) = {\n";
+  Result += "\t0,\n"; // id is; is null
+  Result += "\t\""; Result += PDecl->getNameAsString(); Result += "\",\n";
+  if (SuperProtocols.size() > 0) {
+    Result += "\t(const struct _protocol_list_t *)&"; Result += "_OBJC_PROTOCOL_REFS_";
+    Result += PDecl->getNameAsString(); Result += ",\n";
+  }
+  else
+    Result += "\t0,\n";
+  if (InstanceMethods.size() > 0) {
+    Result += "\t(const struct method_list_t *)&_OBJC_PROTOCOL_INSTANCE_METHODS_"; 
+    Result += PDecl->getNameAsString(); Result += ",\n";
+  }
+  else
+    Result += "\t0,\n";
+
+  if (ClassMethods.size() > 0) {
+    Result += "\t(const struct method_list_t *)&_OBJC_PROTOCOL_CLASS_METHODS_"; 
+    Result += PDecl->getNameAsString(); Result += ",\n";
+  }
+  else
+    Result += "\t0,\n";
+  
+  if (OptInstanceMethods.size() > 0) {
+    Result += "\t(const struct method_list_t *)&_OBJC_PROTOCOL_OPT_INSTANCE_METHODS_"; 
+    Result += PDecl->getNameAsString(); Result += ",\n";
+  }
+  else
+    Result += "\t0,\n";
+  
+  if (OptClassMethods.size() > 0) {
+    Result += "\t(const struct method_list_t *)&_OBJC_PROTOCOL_OPT_CLASS_METHODS_"; 
+    Result += PDecl->getNameAsString(); Result += ",\n";
+  }
+  else
+    Result += "\t0,\n";
+  
+  if (ProtocolProperties.size() > 0) {
+    Result += "\t(const struct _prop_list_t *)&_OBJC_PROTOCOL_PROPERTIES_"; 
+    Result += PDecl->getNameAsString(); Result += ",\n";
+  }
+  else
+    Result += "\t0,\n";
+  
+  Result += "\t"; Result += "sizeof(_protocol_t)"; Result += ",\n";
+  Result += "\t0,\n";
+  
+  if (AllMethods.size() > 0) {
+    Result += "\t(const char **)&"; Result += "_OBJC_PROTOCOL_METHOD_TYPES_";
+    Result += PDecl->getNameAsString();
+    Result += "\n};\n";
+  }
+  else
+    Result += "\t0\n};\n";
+  
+  if (LangOpts.MicrosoftExt)
+    Result += "static ";
+  Result += "struct _protocol_t *";
+  Result += "_OBJC_LABEL_PROTOCOL_$_"; Result += PDecl->getNameAsString();
+  Result += " = &_OBJC_PROTOCOL_"; Result += PDecl->getNameAsString();
+  Result += ";\n";
+    
+  // Mark this protocol as having been generated.
+  if (!ObjCSynthesizedProtocols.insert(PDecl->getCanonicalDecl()))
+    llvm_unreachable("protocol already synthesized");
+  
+}
+
+void RewriteModernObjC::RewriteObjCProtocolListMetaData(
+                                const ObjCList<ObjCProtocolDecl> &Protocols,
+                                StringRef prefix, StringRef ClassName,
+                                std::string &Result) {
+  if (Protocols.empty()) return;
+  
+  for (unsigned i = 0; i != Protocols.size(); i++)
+    RewriteObjCProtocolMetaData(Protocols[i], Result);
+  
+  // Output the top lovel protocol meta-data for the class.
+  /* struct _objc_protocol_list {
+   struct _objc_protocol_list *next;
+   int    protocol_count;
+   struct _objc_protocol *class_protocols[];
+   }
+   */
+  Result += "\n";
+  if (LangOpts.MicrosoftExt)
+    Result += "__declspec(allocate(\".cat_cls_meth$B\")) ";
+  Result += "static struct {\n";
+  Result += "\tstruct _objc_protocol_list *next;\n";
+  Result += "\tint    protocol_count;\n";
+  Result += "\tstruct _objc_protocol *class_protocols[";
+  Result += utostr(Protocols.size());
+  Result += "];\n} _OBJC_";
+  Result += prefix;
+  Result += "_PROTOCOLS_";
+  Result += ClassName;
+  Result += " __attribute__ ((used, section (\"__OBJC, __cat_cls_meth\")))= "
+  "{\n\t0, ";
+  Result += utostr(Protocols.size());
+  Result += "\n";
+  
+  Result += "\t,{&_OBJC_PROTOCOL_";
+  Result += Protocols[0]->getNameAsString();
+  Result += " \n";
+  
+  for (unsigned i = 1; i != Protocols.size(); i++) {
+    Result += "\t ,&_OBJC_PROTOCOL_";
+    Result += Protocols[i]->getNameAsString();
+    Result += "\n";
+  }
+  Result += "\t }\n};\n";
+}
+
+/// hasObjCExceptionAttribute - Return true if this class or any super
+/// class has the __objc_exception__ attribute.
+/// FIXME. Move this to ASTContext.cpp as it is also used for IRGen.
+static bool hasObjCExceptionAttribute(ASTContext &Context,
+                                      const ObjCInterfaceDecl *OID) {
+  if (OID->hasAttr<ObjCExceptionAttr>())
+    return true;
+  if (const ObjCInterfaceDecl *Super = OID->getSuperClass())
+    return hasObjCExceptionAttribute(Context, Super);
+  return false;
+}
+
+void RewriteModernObjC::RewriteObjCClassMetaData(ObjCImplementationDecl *IDecl,
+                                           std::string &Result) {
+  ObjCInterfaceDecl *CDecl = IDecl->getClassInterface();
+  
+  // Explicitly declared @interface's are already synthesized.
+  if (CDecl->isImplicitInterfaceDecl())
+    assert(false && 
+           "Legacy implicit interface rewriting not supported in moder abi");
+  
+  WriteModernMetadataDeclarations(Context, Result);
+  SmallVector<ObjCIvarDecl *, 8> IVars;
+  
+  for (ObjCIvarDecl *IVD = CDecl->all_declared_ivar_begin();
+      IVD; IVD = IVD->getNextIvar()) {
+    // Ignore unnamed bit-fields.
+    if (!IVD->getDeclName())
+      continue;
+    IVars.push_back(IVD);
+  }
+  
+  Write__ivar_list_t_initializer(*this, Context, Result, IVars, 
+                                 "_OBJC_$_INSTANCE_VARIABLES_",
+                                 CDecl);
+  
+  // Build _objc_method_list for class's instance methods if needed
+  SmallVector<ObjCMethodDecl *, 32>
+    InstanceMethods(IDecl->instmeth_begin(), IDecl->instmeth_end());
+  
+  // If any of our property implementations have associated getters or
+  // setters, produce metadata for them as well.
+  for (ObjCImplDecl::propimpl_iterator Prop = IDecl->propimpl_begin(),
+       PropEnd = IDecl->propimpl_end();
+       Prop != PropEnd; ++Prop) {
+    if (Prop->getPropertyImplementation() == ObjCPropertyImplDecl::Dynamic)
+      continue;
+    if (!Prop->getPropertyIvarDecl())
+      continue;
+    ObjCPropertyDecl *PD = Prop->getPropertyDecl();
+    if (!PD)
+      continue;
+    if (ObjCMethodDecl *Getter = PD->getGetterMethodDecl())
+      if (mustSynthesizeSetterGetterMethod(IDecl, PD, true /*getter*/))
+        InstanceMethods.push_back(Getter);
+    if (PD->isReadOnly())
+      continue;
+    if (ObjCMethodDecl *Setter = PD->getSetterMethodDecl())
+      if (mustSynthesizeSetterGetterMethod(IDecl, PD, false /*setter*/))
+        InstanceMethods.push_back(Setter);
+  }
+  
+  Write_method_list_t_initializer(*this, Context, Result, InstanceMethods,
+                                  "_OBJC_$_INSTANCE_METHODS_",
+                                  IDecl->getNameAsString(), true);
+  
+  SmallVector<ObjCMethodDecl *, 32>
+    ClassMethods(IDecl->classmeth_begin(), IDecl->classmeth_end());
+  
+  Write_method_list_t_initializer(*this, Context, Result, ClassMethods,
+                                  "_OBJC_$_CLASS_METHODS_",
+                                  IDecl->getNameAsString(), true);
+  
+  // Protocols referenced in class declaration?
+  // Protocol's super protocol list
+  std::vector<ObjCProtocolDecl *> RefedProtocols;
+  const ObjCList<ObjCProtocolDecl> &Protocols = CDecl->getReferencedProtocols();
+  for (ObjCList<ObjCProtocolDecl>::iterator I = Protocols.begin(),
+       E = Protocols.end();
+       I != E; ++I) {
+    RefedProtocols.push_back(*I);
+    // Must write out all protocol definitions in current qualifier list,
+    // and in their nested qualifiers before writing out current definition.
+    RewriteObjCProtocolMetaData(*I, Result);
+  }
+  
+  Write_protocol_list_initializer(Context, Result, 
+                                  RefedProtocols,
+                                  "_OBJC_CLASS_PROTOCOLS_$_",
+                                  IDecl->getNameAsString());
+  
+  // Protocol's property metadata.
+  std::vector<ObjCPropertyDecl *> ClassProperties;
+  for (ObjCContainerDecl::prop_iterator I = CDecl->prop_begin(),
+       E = CDecl->prop_end(); I != E; ++I)
+    ClassProperties.push_back(*I);
+  
+  Write_prop_list_t_initializer(*this, Context, Result, ClassProperties,
+                                 /* Container */IDecl,
+                                 "_OBJC_$_PROP_LIST_",
+                                 CDecl->getNameAsString());
+
+  
+  // Data for initializing _class_ro_t  metaclass meta-data
+  uint32_t flags = CLS_META;
+  std::string InstanceSize;
+  std::string InstanceStart;
+  
+  
+  bool classIsHidden = CDecl->getVisibility() == HiddenVisibility;
+  if (classIsHidden)
+    flags |= OBJC2_CLS_HIDDEN;
+  
+  if (!CDecl->getSuperClass())
+    // class is root
+    flags |= CLS_ROOT;
+  InstanceSize = "sizeof(struct _class_t)";
+  InstanceStart = InstanceSize;
+  Write__class_ro_t_initializer(Context, Result, flags, 
+                                InstanceStart, InstanceSize,
+                                ClassMethods,
+                                0,
+                                0,
+                                0,
+                                "_OBJC_METACLASS_RO_$_",
+                                CDecl->getNameAsString());
+
+  
+  // Data for initializing _class_ro_t meta-data
+  flags = CLS;
+  if (classIsHidden)
+    flags |= OBJC2_CLS_HIDDEN;
+  
+  if (hasObjCExceptionAttribute(*Context, CDecl))
+    flags |= CLS_EXCEPTION;
+
+  if (!CDecl->getSuperClass())
+    // class is root
+    flags |= CLS_ROOT;
+  
+  InstanceSize.clear();
+  InstanceStart.clear();
+  if (!ObjCSynthesizedStructs.count(CDecl)) {
+    InstanceSize = "0";
+    InstanceStart = "0";
+  }
+  else {
+    InstanceSize = "sizeof(struct ";
+    InstanceSize += CDecl->getNameAsString();
+    InstanceSize += "_IMPL)";
+    
+    ObjCIvarDecl *IVD = CDecl->all_declared_ivar_begin();
+    if (IVD) {
+      RewriteIvarOffsetComputation(IVD, InstanceStart);
+    }
+    else 
+      InstanceStart = InstanceSize;
+  }
+  Write__class_ro_t_initializer(Context, Result, flags, 
+                                InstanceStart, InstanceSize,
+                                InstanceMethods,
+                                RefedProtocols,
+                                IVars,
+                                ClassProperties,
+                                "_OBJC_CLASS_RO_$_",
+                                CDecl->getNameAsString());
+  
+  Write_class_t(Context, Result,
+                "OBJC_METACLASS_$_",
+                CDecl, /*metaclass*/true);
+  
+  Write_class_t(Context, Result,
+                "OBJC_CLASS_$_",
+                CDecl, /*metaclass*/false);
+  
+  if (ImplementationIsNonLazy(IDecl))
+    DefinedNonLazyClasses.push_back(CDecl);
+                
+}
+
+void RewriteModernObjC::RewriteClassSetupInitHook(std::string &Result) {
+  int ClsDefCount = ClassImplementation.size();
+  if (!ClsDefCount)
+    return;
+  Result += "#pragma section(\".objc_inithooks$B\", long, read, write)\n";
+  Result += "__declspec(allocate(\".objc_inithooks$B\")) ";
+  Result += "static void *OBJC_CLASS_SETUP[] = {\n";
+  for (int i = 0; i < ClsDefCount; i++) {
+    ObjCImplementationDecl *IDecl = ClassImplementation[i];
+    ObjCInterfaceDecl *CDecl = IDecl->getClassInterface();
+    Result += "\t(void *)&OBJC_CLASS_SETUP_$_";
+    Result  += CDecl->getName(); Result += ",\n";
+  }
+  Result += "};\n";
+}
+
+void RewriteModernObjC::RewriteMetaDataIntoBuffer(std::string &Result) {
+  int ClsDefCount = ClassImplementation.size();
+  int CatDefCount = CategoryImplementation.size();
+  
+  // For each implemented class, write out all its meta data.
+  for (int i = 0; i < ClsDefCount; i++)
+    RewriteObjCClassMetaData(ClassImplementation[i], Result);
+  
+  RewriteClassSetupInitHook(Result);
+  
+  // For each implemented category, write out all its meta data.
+  for (int i = 0; i < CatDefCount; i++)
+    RewriteObjCCategoryImplDecl(CategoryImplementation[i], Result);
+  
+  RewriteCategorySetupInitHook(Result);
+  
+  if (ClsDefCount > 0) {
+    if (LangOpts.MicrosoftExt)
+      Result += "__declspec(allocate(\".objc_classlist$B\")) ";
+    Result += "static struct _class_t *L_OBJC_LABEL_CLASS_$ [";
+    Result += llvm::utostr(ClsDefCount); Result += "]";
+    Result += 
+      " __attribute__((used, section (\"__DATA, __objc_classlist,"
+      "regular,no_dead_strip\")))= {\n";
+    for (int i = 0; i < ClsDefCount; i++) {
+      Result += "\t&OBJC_CLASS_$_";
+      Result += ClassImplementation[i]->getNameAsString();
+      Result += ",\n";
+    }
+    Result += "};\n";
+    
+    if (!DefinedNonLazyClasses.empty()) {
+      if (LangOpts.MicrosoftExt)
+        Result += "__declspec(allocate(\".objc_nlclslist$B\")) \n";
+      Result += "static struct _class_t *_OBJC_LABEL_NONLAZY_CLASS_$[] = {\n\t";
+      for (unsigned i = 0, e = DefinedNonLazyClasses.size(); i < e; i++) {
+        Result += "\t&OBJC_CLASS_$_"; Result += DefinedNonLazyClasses[i]->getNameAsString();
+        Result += ",\n";
+      }
+      Result += "};\n";
+    }
+  }
+  
+  if (CatDefCount > 0) {
+    if (LangOpts.MicrosoftExt)
+      Result += "__declspec(allocate(\".objc_catlist$B\")) ";
+    Result += "static struct _category_t *L_OBJC_LABEL_CATEGORY_$ [";
+    Result += llvm::utostr(CatDefCount); Result += "]";
+    Result += 
+    " __attribute__((used, section (\"__DATA, __objc_catlist,"
+    "regular,no_dead_strip\")))= {\n";
+    for (int i = 0; i < CatDefCount; i++) {
+      Result += "\t&_OBJC_$_CATEGORY_";
+      Result += 
+        CategoryImplementation[i]->getClassInterface()->getNameAsString(); 
+      Result += "_$_";
+      Result += CategoryImplementation[i]->getNameAsString();
+      Result += ",\n";
+    }
+    Result += "};\n";
+  }
+  
+  if (!DefinedNonLazyCategories.empty()) {
+    if (LangOpts.MicrosoftExt)
+      Result += "__declspec(allocate(\".objc_nlcatlist$B\")) \n";
+    Result += "static struct _category_t *_OBJC_LABEL_NONLAZY_CATEGORY_$[] = {\n\t";
+    for (unsigned i = 0, e = DefinedNonLazyCategories.size(); i < e; i++) {
+      Result += "\t&_OBJC_$_CATEGORY_";
+      Result += 
+        DefinedNonLazyCategories[i]->getClassInterface()->getNameAsString(); 
+      Result += "_$_";
+      Result += DefinedNonLazyCategories[i]->getNameAsString();
+      Result += ",\n";
+    }
+    Result += "};\n";
+  }
+}
+
+void RewriteModernObjC::WriteImageInfo(std::string &Result) {
+  if (LangOpts.MicrosoftExt)
+    Result += "__declspec(allocate(\".objc_imageinfo$B\")) \n";
+  
+  Result += "static struct IMAGE_INFO { unsigned version; unsigned flag; } ";
+  // version 0, ObjCABI is 2
+  Result += "_OBJC_IMAGE_INFO = { 0, 2 };\n";
+}
+
+/// RewriteObjCCategoryImplDecl - Rewrite metadata for each category
+/// implementation.
+void RewriteModernObjC::RewriteObjCCategoryImplDecl(ObjCCategoryImplDecl *IDecl,
+                                              std::string &Result) {
+  WriteModernMetadataDeclarations(Context, Result);
+  ObjCInterfaceDecl *ClassDecl = IDecl->getClassInterface();
+  // Find category declaration for this implementation.
+  ObjCCategoryDecl *CDecl=0;
+  for (CDecl = ClassDecl->getCategoryList(); CDecl;
+       CDecl = CDecl->getNextClassCategory())
+    if (CDecl->getIdentifier() == IDecl->getIdentifier())
+      break;
+  
+  std::string FullCategoryName = ClassDecl->getNameAsString();
+  FullCategoryName += "_$_";
+  FullCategoryName += CDecl->getNameAsString();
+  
+  // Build _objc_method_list for class's instance methods if needed
+  SmallVector<ObjCMethodDecl *, 32>
+  InstanceMethods(IDecl->instmeth_begin(), IDecl->instmeth_end());
+  
+  // If any of our property implementations have associated getters or
+  // setters, produce metadata for them as well.
+  for (ObjCImplDecl::propimpl_iterator Prop = IDecl->propimpl_begin(),
+       PropEnd = IDecl->propimpl_end();
+       Prop != PropEnd; ++Prop) {
+    if (Prop->getPropertyImplementation() == ObjCPropertyImplDecl::Dynamic)
+      continue;
+    if (!Prop->getPropertyIvarDecl())
+      continue;
+    ObjCPropertyDecl *PD = Prop->getPropertyDecl();
+    if (!PD)
+      continue;
+    if (ObjCMethodDecl *Getter = PD->getGetterMethodDecl())
+      InstanceMethods.push_back(Getter);
+    if (PD->isReadOnly())
+      continue;
+    if (ObjCMethodDecl *Setter = PD->getSetterMethodDecl())
+      InstanceMethods.push_back(Setter);
+  }
+  
+  Write_method_list_t_initializer(*this, Context, Result, InstanceMethods,
+                                  "_OBJC_$_CATEGORY_INSTANCE_METHODS_",
+                                  FullCategoryName, true);
+  
+  SmallVector<ObjCMethodDecl *, 32>
+    ClassMethods(IDecl->classmeth_begin(), IDecl->classmeth_end());
+  
+  Write_method_list_t_initializer(*this, Context, Result, ClassMethods,
+                                  "_OBJC_$_CATEGORY_CLASS_METHODS_",
+                                  FullCategoryName, true);
+  
+  // Protocols referenced in class declaration?
+  // Protocol's super protocol list
+  std::vector<ObjCProtocolDecl *> RefedProtocols;
+  for (ObjCInterfaceDecl::protocol_iterator I = CDecl->protocol_begin(),
+                                            E = CDecl->protocol_end();
+
+         I != E; ++I) {
+    RefedProtocols.push_back(*I);
+    // Must write out all protocol definitions in current qualifier list,
+    // and in their nested qualifiers before writing out current definition.
+    RewriteObjCProtocolMetaData(*I, Result);
+  }
+  
+  Write_protocol_list_initializer(Context, Result, 
+                                  RefedProtocols,
+                                  "_OBJC_CATEGORY_PROTOCOLS_$_",
+                                  FullCategoryName);
+  
+  // Protocol's property metadata.
+  std::vector<ObjCPropertyDecl *> ClassProperties;
+  for (ObjCContainerDecl::prop_iterator I = CDecl->prop_begin(),
+       E = CDecl->prop_end(); I != E; ++I)
+    ClassProperties.push_back(*I);
+  
+  Write_prop_list_t_initializer(*this, Context, Result, ClassProperties,
+                                /* Container */IDecl,
+                                "_OBJC_$_PROP_LIST_",
+                                FullCategoryName);
+  
+  Write_category_t(*this, Context, Result,
+                   CDecl,
+                   ClassDecl,
+                   InstanceMethods,
+                   ClassMethods,
+                   RefedProtocols,
+                   ClassProperties);
+  
+  // Determine if this category is also "non-lazy".
+  if (ImplementationIsNonLazy(IDecl))
+    DefinedNonLazyCategories.push_back(CDecl);
+    
+}
+
+void RewriteModernObjC::RewriteCategorySetupInitHook(std::string &Result) {
+  int CatDefCount = CategoryImplementation.size();
+  if (!CatDefCount)
+    return;
+  Result += "#pragma section(\".objc_inithooks$B\", long, read, write)\n";
+  Result += "__declspec(allocate(\".objc_inithooks$B\")) ";
+  Result += "static void *OBJC_CATEGORY_SETUP[] = {\n";
+  for (int i = 0; i < CatDefCount; i++) {
+    ObjCCategoryImplDecl *IDecl = CategoryImplementation[i];
+    ObjCCategoryDecl *CatDecl= IDecl->getCategoryDecl();
+    ObjCInterfaceDecl *ClassDecl = IDecl->getClassInterface();
+    Result += "\t(void *)&OBJC_CATEGORY_SETUP_$_";
+    Result += ClassDecl->getName();
+    Result += "_$_";
+    Result += CatDecl->getName();
+    Result += ",\n";
+  }
+  Result += "};\n";
+}
+
+// RewriteObjCMethodsMetaData - Rewrite methods metadata for instance or
+/// class methods.
+template<typename MethodIterator>
+void RewriteModernObjC::RewriteObjCMethodsMetaData(MethodIterator MethodBegin,
+                                             MethodIterator MethodEnd,
+                                             bool IsInstanceMethod,
+                                             StringRef prefix,
+                                             StringRef ClassName,
+                                             std::string &Result) {
+  if (MethodBegin == MethodEnd) return;
+  
+  if (!objc_impl_method) {
+    /* struct _objc_method {
+     SEL _cmd;
+     char *method_types;
+     void *_imp;
+     }
+     */
+    Result += "\nstruct _objc_method {\n";
+    Result += "\tSEL _cmd;\n";
+    Result += "\tchar *method_types;\n";
+    Result += "\tvoid *_imp;\n";
+    Result += "};\n";
+    
+    objc_impl_method = true;
+  }
+  
+  // Build _objc_method_list for class's methods if needed
+  
+  /* struct  {
+   struct _objc_method_list *next_method;
+   int method_count;
+   struct _objc_method method_list[];
+   }
+   */
+  unsigned NumMethods = std::distance(MethodBegin, MethodEnd);
+  Result += "\n";
+  if (LangOpts.MicrosoftExt) {
+    if (IsInstanceMethod)
+      Result += "__declspec(allocate(\".inst_meth$B\")) ";
+    else
+      Result += "__declspec(allocate(\".cls_meth$B\")) ";
+  }
+  Result += "static struct {\n";
+  Result += "\tstruct _objc_method_list *next_method;\n";
+  Result += "\tint method_count;\n";
+  Result += "\tstruct _objc_method method_list[";
+  Result += utostr(NumMethods);
+  Result += "];\n} _OBJC_";
+  Result += prefix;
+  Result += IsInstanceMethod ? "INSTANCE" : "CLASS";
+  Result += "_METHODS_";
+  Result += ClassName;
+  Result += " __attribute__ ((used, section (\"__OBJC, __";
+  Result += IsInstanceMethod ? "inst" : "cls";
+  Result += "_meth\")))= ";
+  Result += "{\n\t0, " + utostr(NumMethods) + "\n";
+  
+  Result += "\t,{{(SEL)\"";
+  Result += (*MethodBegin)->getSelector().getAsString().c_str();
+  std::string MethodTypeString;
+  Context->getObjCEncodingForMethodDecl(*MethodBegin, MethodTypeString);
+  Result += "\", \"";
+  Result += MethodTypeString;
+  Result += "\", (void *)";
+  Result += MethodInternalNames[*MethodBegin];
+  Result += "}\n";
+  for (++MethodBegin; MethodBegin != MethodEnd; ++MethodBegin) {
+    Result += "\t  ,{(SEL)\"";
+    Result += (*MethodBegin)->getSelector().getAsString().c_str();
+    std::string MethodTypeString;
+    Context->getObjCEncodingForMethodDecl(*MethodBegin, MethodTypeString);
+    Result += "\", \"";
+    Result += MethodTypeString;
+    Result += "\", (void *)";
+    Result += MethodInternalNames[*MethodBegin];
+    Result += "}\n";
+  }
+  Result += "\t }\n};\n";
+}
+
+Stmt *RewriteModernObjC::RewriteObjCIvarRefExpr(ObjCIvarRefExpr *IV) {
+  SourceRange OldRange = IV->getSourceRange();
+  Expr *BaseExpr = IV->getBase();
+  
+  // Rewrite the base, but without actually doing replaces.
+  {
+    DisableReplaceStmtScope S(*this);
+    BaseExpr = cast<Expr>(RewriteFunctionBodyOrGlobalInitializer(BaseExpr));
+    IV->setBase(BaseExpr);
+  }
+  
+  ObjCIvarDecl *D = IV->getDecl();
+  
+  Expr *Replacement = IV;
+  
+    if (BaseExpr->getType()->isObjCObjectPointerType()) {
+      const ObjCInterfaceType *iFaceDecl =
+        dyn_cast<ObjCInterfaceType>(BaseExpr->getType()->getPointeeType());
+      assert(iFaceDecl && "RewriteObjCIvarRefExpr - iFaceDecl is null");
+      // lookup which class implements the instance variable.
+      ObjCInterfaceDecl *clsDeclared = 0;
+      iFaceDecl->getDecl()->lookupInstanceVariable(D->getIdentifier(),
+                                                   clsDeclared);
+      assert(clsDeclared && "RewriteObjCIvarRefExpr(): Can't find class");
+      
+      // Build name of symbol holding ivar offset.
+      std::string IvarOffsetName;
+      WriteInternalIvarName(clsDeclared, D, IvarOffsetName);
+      
+      ReferencedIvars[clsDeclared].insert(D);
+      
+      // cast offset to "char *".
+      CastExpr *castExpr = NoTypeInfoCStyleCastExpr(Context, 
+                                                    Context->getPointerType(Context->CharTy),
+                                                    CK_BitCast,
+                                                    BaseExpr);
+      VarDecl *NewVD = VarDecl::Create(*Context, TUDecl, SourceLocation(),
+                                       SourceLocation(), &Context->Idents.get(IvarOffsetName),
+                                       Context->UnsignedLongTy, 0, SC_Extern, SC_None);
+      DeclRefExpr *DRE = new (Context) DeclRefExpr(NewVD, false,
+                                                   Context->UnsignedLongTy, VK_LValue,
+                                                   SourceLocation());
+      BinaryOperator *addExpr = 
+        new (Context) BinaryOperator(castExpr, DRE, BO_Add, 
+                                     Context->getPointerType(Context->CharTy),
+                                     VK_RValue, OK_Ordinary, SourceLocation(), false);
+      // Don't forget the parens to enforce the proper binding.
+      ParenExpr *PE = new (Context) ParenExpr(SourceLocation(),
+                                              SourceLocation(),
+                                              addExpr);
+      QualType IvarT = D->getType();
+
+      if (!isa<TypedefType>(IvarT) && IvarT->isRecordType()) {
+        RecordDecl *RD = IvarT->getAs<RecordType>()->getDecl();
+        RD = RD->getDefinition();
+        if (RD && !RD->getDeclName().getAsIdentifierInfo()) {
+          // decltype(((Foo_IMPL*)0)->bar) *
+          ObjCContainerDecl *CDecl = 
+            dyn_cast<ObjCContainerDecl>(D->getDeclContext());
+          // ivar in class extensions requires special treatment.
+          if (ObjCCategoryDecl *CatDecl = dyn_cast<ObjCCategoryDecl>(CDecl))
+            CDecl = CatDecl->getClassInterface();
+          std::string RecName = CDecl->getName();
+          RecName += "_IMPL";
+          RecordDecl *RD = RecordDecl::Create(*Context, TTK_Struct, TUDecl,
+                                              SourceLocation(), SourceLocation(),
+                                              &Context->Idents.get(RecName.c_str()));
+          QualType PtrStructIMPL = Context->getPointerType(Context->getTagDeclType(RD));
+          unsigned UnsignedIntSize = 
+            static_cast<unsigned>(Context->getTypeSize(Context->UnsignedIntTy));
+          Expr *Zero = IntegerLiteral::Create(*Context,
+                                              llvm::APInt(UnsignedIntSize, 0),
+                                              Context->UnsignedIntTy, SourceLocation());
+          Zero = NoTypeInfoCStyleCastExpr(Context, PtrStructIMPL, CK_BitCast, Zero);
+          ParenExpr *PE = new (Context) ParenExpr(SourceLocation(), SourceLocation(),
+                                                  Zero);
+          FieldDecl *FD = FieldDecl::Create(*Context, 0, SourceLocation(),
+                                            SourceLocation(),
+                                            &Context->Idents.get(D->getNameAsString()),
+                                            IvarT, 0,
+                                            /*BitWidth=*/0, /*Mutable=*/true,
+                                            ICIS_NoInit);
+          MemberExpr *ME = new (Context) MemberExpr(PE, true, FD, SourceLocation(),
+                                                    FD->getType(), VK_LValue,
+                                                    OK_Ordinary);
+          IvarT = Context->getDecltypeType(ME, ME->getType());
+        }
+      }
+      convertObjCTypeToCStyleType(IvarT);
+      QualType castT = Context->getPointerType(IvarT);
+          
+      castExpr = NoTypeInfoCStyleCastExpr(Context, 
+                                          castT,
+                                          CK_BitCast,
+                                          PE);
+      
+      
+      Expr *Exp = new (Context) UnaryOperator(castExpr, UO_Deref, IvarT,
+                                              VK_LValue, OK_Ordinary,
+                                              SourceLocation());
+      PE = new (Context) ParenExpr(OldRange.getBegin(),
+                                   OldRange.getEnd(),
+                                   Exp);
+
+      Replacement = PE;
+    }
+  
+    ReplaceStmtWithRange(IV, Replacement, OldRange);
+    return Replacement;  
+}
diff --git a/lib/Rewrite/Frontend/RewriteObjC.cpp b/lib/Rewrite/Frontend/RewriteObjC.cpp
new file mode 100644
index 0000000..a6dcc6b
--- /dev/null
+++ b/lib/Rewrite/Frontend/RewriteObjC.cpp
@@ -0,0 +1,6029 @@
+//===--- RewriteObjC.cpp - Playground for the code rewriter ---------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Hacks and fun related to the code rewriter.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Rewrite/Frontend/ASTConsumers.h"
+#include "clang/Rewrite/Core/Rewriter.h"
+#include "clang/AST/AST.h"
+#include "clang/AST/ASTConsumer.h"
+#include "clang/AST/ParentMap.h"
+#include "clang/Basic/SourceManager.h"
+#include "clang/Basic/IdentifierTable.h"
+#include "clang/Basic/Diagnostic.h"
+#include "clang/Lex/Lexer.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/OwningPtr.h"
+#include "llvm/ADT/DenseSet.h"
+
+using namespace clang;
+using llvm::utostr;
+
+namespace {
+  class RewriteObjC : public ASTConsumer {
+  protected:
+    
+    enum {
+      BLOCK_FIELD_IS_OBJECT   =  3,  /* id, NSObject, __attribute__((NSObject)),
+                                        block, ... */
+      BLOCK_FIELD_IS_BLOCK    =  7,  /* a block variable */
+      BLOCK_FIELD_IS_BYREF    =  8,  /* the on stack structure holding the 
+                                        __block variable */
+      BLOCK_FIELD_IS_WEAK     = 16,  /* declared __weak, only used in byref copy
+                                        helpers */
+      BLOCK_BYREF_CALLER      = 128, /* called from __block (byref) copy/dispose
+                                        support routines */
+      BLOCK_BYREF_CURRENT_MAX = 256
+    };
+    
+    enum {
+      BLOCK_NEEDS_FREE =        (1 << 24),
+      BLOCK_HAS_COPY_DISPOSE =  (1 << 25),
+      BLOCK_HAS_CXX_OBJ =       (1 << 26),
+      BLOCK_IS_GC =             (1 << 27),
+      BLOCK_IS_GLOBAL =         (1 << 28),
+      BLOCK_HAS_DESCRIPTOR =    (1 << 29)
+    };
+    static const int OBJC_ABI_VERSION = 7;
+    
+    Rewriter Rewrite;
+    DiagnosticsEngine &Diags;
+    const LangOptions &LangOpts;
+    ASTContext *Context;
+    SourceManager *SM;
+    TranslationUnitDecl *TUDecl;
+    FileID MainFileID;
+    const char *MainFileStart, *MainFileEnd;
+    Stmt *CurrentBody;
+    ParentMap *PropParentMap; // created lazily.
+    std::string InFileName;
+    raw_ostream* OutFile;
+    std::string Preamble;
+    
+    TypeDecl *ProtocolTypeDecl;
+    VarDecl *GlobalVarDecl;
+    unsigned RewriteFailedDiag;
+    // ObjC string constant support.
+    unsigned NumObjCStringLiterals;
+    VarDecl *ConstantStringClassReference;
+    RecordDecl *NSStringRecord;
+
+    // ObjC foreach break/continue generation support.
+    int BcLabelCount;
+    
+    unsigned TryFinallyContainsReturnDiag;
+    // Needed for super.
+    ObjCMethodDecl *CurMethodDef;
+    RecordDecl *SuperStructDecl;
+    RecordDecl *ConstantStringDecl;
+    
+    FunctionDecl *MsgSendFunctionDecl;
+    FunctionDecl *MsgSendSuperFunctionDecl;
+    FunctionDecl *MsgSendStretFunctionDecl;
+    FunctionDecl *MsgSendSuperStretFunctionDecl;
+    FunctionDecl *MsgSendFpretFunctionDecl;
+    FunctionDecl *GetClassFunctionDecl;
+    FunctionDecl *GetMetaClassFunctionDecl;
+    FunctionDecl *GetSuperClassFunctionDecl;
+    FunctionDecl *SelGetUidFunctionDecl;
+    FunctionDecl *CFStringFunctionDecl;
+    FunctionDecl *SuperContructorFunctionDecl;
+    FunctionDecl *CurFunctionDef;
+    FunctionDecl *CurFunctionDeclToDeclareForBlock;
+
+    /* Misc. containers needed for meta-data rewrite. */
+    SmallVector<ObjCImplementationDecl *, 8> ClassImplementation;
+    SmallVector<ObjCCategoryImplDecl *, 8> CategoryImplementation;
+    llvm::SmallPtrSet<ObjCInterfaceDecl*, 8> ObjCSynthesizedStructs;
+    llvm::SmallPtrSet<ObjCProtocolDecl*, 8> ObjCSynthesizedProtocols;
+    llvm::SmallPtrSet<ObjCInterfaceDecl*, 8> ObjCForwardDecls;
+    llvm::DenseMap<ObjCMethodDecl*, std::string> MethodInternalNames;
+    SmallVector<Stmt *, 32> Stmts;
+    SmallVector<int, 8> ObjCBcLabelNo;
+    // Remember all the @protocol(<expr>) expressions.
+    llvm::SmallPtrSet<ObjCProtocolDecl *, 32> ProtocolExprDecls;
+    
+    llvm::DenseSet<uint64_t> CopyDestroyCache;
+
+    // Block expressions.
+    SmallVector<BlockExpr *, 32> Blocks;
+    SmallVector<int, 32> InnerDeclRefsCount;
+    SmallVector<DeclRefExpr *, 32> InnerDeclRefs;
+    
+    SmallVector<DeclRefExpr *, 32> BlockDeclRefs;
+
+    // Block related declarations.
+    SmallVector<ValueDecl *, 8> BlockByCopyDecls;
+    llvm::SmallPtrSet<ValueDecl *, 8> BlockByCopyDeclsPtrSet;
+    SmallVector<ValueDecl *, 8> BlockByRefDecls;
+    llvm::SmallPtrSet<ValueDecl *, 8> BlockByRefDeclsPtrSet;
+    llvm::DenseMap<ValueDecl *, unsigned> BlockByRefDeclNo;
+    llvm::SmallPtrSet<ValueDecl *, 8> ImportedBlockDecls;
+    llvm::SmallPtrSet<VarDecl *, 8> ImportedLocalExternalDecls;
+    
+    llvm::DenseMap<BlockExpr *, std::string> RewrittenBlockExprs;
+
+    // This maps an original source AST to it's rewritten form. This allows
+    // us to avoid rewriting the same node twice (which is very uncommon).
+    // This is needed to support some of the exotic property rewriting.
+    llvm::DenseMap<Stmt *, Stmt *> ReplacedNodes;
+
+    // Needed for header files being rewritten
+    bool IsHeader;
+    bool SilenceRewriteMacroWarning;
+    bool objc_impl_method;
+    
+    bool DisableReplaceStmt;
+    class DisableReplaceStmtScope {
+      RewriteObjC &R;
+      bool SavedValue;
+    
+    public:
+      DisableReplaceStmtScope(RewriteObjC &R)
+        : R(R), SavedValue(R.DisableReplaceStmt) {
+        R.DisableReplaceStmt = true;
+      }
+      ~DisableReplaceStmtScope() {
+        R.DisableReplaceStmt = SavedValue;
+      }
+    };
+    void InitializeCommon(ASTContext &context);
+
+  public:
+
+    // Top Level Driver code.
+    virtual bool HandleTopLevelDecl(DeclGroupRef D) {
+      for (DeclGroupRef::iterator I = D.begin(), E = D.end(); I != E; ++I) {
+        if (ObjCInterfaceDecl *Class = dyn_cast<ObjCInterfaceDecl>(*I)) {
+          if (!Class->isThisDeclarationADefinition()) {
+            RewriteForwardClassDecl(D);
+            break;
+          }
+        }
+
+        if (ObjCProtocolDecl *Proto = dyn_cast<ObjCProtocolDecl>(*I)) {
+          if (!Proto->isThisDeclarationADefinition()) {
+            RewriteForwardProtocolDecl(D);
+            break;
+          }
+        }
+
+        HandleTopLevelSingleDecl(*I);
+      }
+      return true;
+    }
+    void HandleTopLevelSingleDecl(Decl *D);
+    void HandleDeclInMainFile(Decl *D);
+    RewriteObjC(std::string inFile, raw_ostream *OS,
+                DiagnosticsEngine &D, const LangOptions &LOpts,
+                bool silenceMacroWarn);
+
+    ~RewriteObjC() {}
+
+    virtual void HandleTranslationUnit(ASTContext &C);
+
+    void ReplaceStmt(Stmt *Old, Stmt *New) {
+      Stmt *ReplacingStmt = ReplacedNodes[Old];
+
+      if (ReplacingStmt)
+        return; // We can't rewrite the same node twice.
+
+      if (DisableReplaceStmt)
+        return;
+
+      // If replacement succeeded or warning disabled return with no warning.
+      if (!Rewrite.ReplaceStmt(Old, New)) {
+        ReplacedNodes[Old] = New;
+        return;
+      }
+      if (SilenceRewriteMacroWarning)
+        return;
+      Diags.Report(Context->getFullLoc(Old->getLocStart()), RewriteFailedDiag)
+                   << Old->getSourceRange();
+    }
+
+    void ReplaceStmtWithRange(Stmt *Old, Stmt *New, SourceRange SrcRange) {
+      if (DisableReplaceStmt)
+        return;
+
+      // Measure the old text.
+      int Size = Rewrite.getRangeSize(SrcRange);
+      if (Size == -1) {
+        Diags.Report(Context->getFullLoc(Old->getLocStart()), RewriteFailedDiag)
+                     << Old->getSourceRange();
+        return;
+      }
+      // Get the new text.
+      std::string SStr;
+      llvm::raw_string_ostream S(SStr);
+      New->printPretty(S, 0, PrintingPolicy(LangOpts));
+      const std::string &Str = S.str();
+
+      // If replacement succeeded or warning disabled return with no warning.
+      if (!Rewrite.ReplaceText(SrcRange.getBegin(), Size, Str)) {
+        ReplacedNodes[Old] = New;
+        return;
+      }
+      if (SilenceRewriteMacroWarning)
+        return;
+      Diags.Report(Context->getFullLoc(Old->getLocStart()), RewriteFailedDiag)
+                   << Old->getSourceRange();
+    }
+
+    void InsertText(SourceLocation Loc, StringRef Str,
+                    bool InsertAfter = true) {
+      // If insertion succeeded or warning disabled return with no warning.
+      if (!Rewrite.InsertText(Loc, Str, InsertAfter) ||
+          SilenceRewriteMacroWarning)
+        return;
+
+      Diags.Report(Context->getFullLoc(Loc), RewriteFailedDiag);
+    }
+
+    void ReplaceText(SourceLocation Start, unsigned OrigLength,
+                     StringRef Str) {
+      // If removal succeeded or warning disabled return with no warning.
+      if (!Rewrite.ReplaceText(Start, OrigLength, Str) ||
+          SilenceRewriteMacroWarning)
+        return;
+
+      Diags.Report(Context->getFullLoc(Start), RewriteFailedDiag);
+    }
+
+    // Syntactic Rewriting.
+    void RewriteRecordBody(RecordDecl *RD);
+    void RewriteInclude();
+    void RewriteForwardClassDecl(DeclGroupRef D);
+    void RewriteForwardClassDecl(const llvm::SmallVector<Decl*, 8> &DG);
+    void RewriteForwardClassEpilogue(ObjCInterfaceDecl *ClassDecl, 
+                                     const std::string &typedefString);
+    void RewriteImplementations();
+    void RewritePropertyImplDecl(ObjCPropertyImplDecl *PID,
+                                 ObjCImplementationDecl *IMD,
+                                 ObjCCategoryImplDecl *CID);
+    void RewriteInterfaceDecl(ObjCInterfaceDecl *Dcl);
+    void RewriteImplementationDecl(Decl *Dcl);
+    void RewriteObjCMethodDecl(const ObjCInterfaceDecl *IDecl,
+                               ObjCMethodDecl *MDecl, std::string &ResultStr);
+    void RewriteTypeIntoString(QualType T, std::string &ResultStr,
+                               const FunctionType *&FPRetType);
+    void RewriteByRefString(std::string &ResultStr, const std::string &Name,
+                            ValueDecl *VD, bool def=false);
+    void RewriteCategoryDecl(ObjCCategoryDecl *Dcl);
+    void RewriteProtocolDecl(ObjCProtocolDecl *Dcl);
+    void RewriteForwardProtocolDecl(DeclGroupRef D);
+    void RewriteForwardProtocolDecl(const llvm::SmallVector<Decl*, 8> &DG);
+    void RewriteMethodDeclaration(ObjCMethodDecl *Method);
+    void RewriteProperty(ObjCPropertyDecl *prop);
+    void RewriteFunctionDecl(FunctionDecl *FD);
+    void RewriteBlockPointerType(std::string& Str, QualType Type);
+    void RewriteBlockPointerTypeVariable(std::string& Str, ValueDecl *VD);
+    void RewriteBlockLiteralFunctionDecl(FunctionDecl *FD);
+    void RewriteObjCQualifiedInterfaceTypes(Decl *Dcl);
+    void RewriteTypeOfDecl(VarDecl *VD);
+    void RewriteObjCQualifiedInterfaceTypes(Expr *E);
+  
+    // Expression Rewriting.
+    Stmt *RewriteFunctionBodyOrGlobalInitializer(Stmt *S);
+    Stmt *RewriteAtEncode(ObjCEncodeExpr *Exp);
+    Stmt *RewritePropertyOrImplicitGetter(PseudoObjectExpr *Pseudo);
+    Stmt *RewritePropertyOrImplicitSetter(PseudoObjectExpr *Pseudo);
+    Stmt *RewriteAtSelector(ObjCSelectorExpr *Exp);
+    Stmt *RewriteMessageExpr(ObjCMessageExpr *Exp);
+    Stmt *RewriteObjCStringLiteral(ObjCStringLiteral *Exp);
+    Stmt *RewriteObjCProtocolExpr(ObjCProtocolExpr *Exp);
+    void RewriteTryReturnStmts(Stmt *S);
+    void RewriteSyncReturnStmts(Stmt *S, std::string buf);
+    Stmt *RewriteObjCTryStmt(ObjCAtTryStmt *S);
+    Stmt *RewriteObjCSynchronizedStmt(ObjCAtSynchronizedStmt *S);
+    Stmt *RewriteObjCThrowStmt(ObjCAtThrowStmt *S);
+    Stmt *RewriteObjCForCollectionStmt(ObjCForCollectionStmt *S,
+                                       SourceLocation OrigEnd);
+    Stmt *RewriteBreakStmt(BreakStmt *S);
+    Stmt *RewriteContinueStmt(ContinueStmt *S);
+    void RewriteCastExpr(CStyleCastExpr *CE);
+    
+    // Block rewriting.
+    void RewriteBlocksInFunctionProtoType(QualType funcType, NamedDecl *D);
+    
+    // Block specific rewrite rules.
+    void RewriteBlockPointerDecl(NamedDecl *VD);
+    void RewriteByRefVar(VarDecl *VD);
+    Stmt *RewriteBlockDeclRefExpr(DeclRefExpr *VD);
+    Stmt *RewriteLocalVariableExternalStorage(DeclRefExpr *DRE);
+    void RewriteBlockPointerFunctionArgs(FunctionDecl *FD);
+    
+    void RewriteObjCInternalStruct(ObjCInterfaceDecl *CDecl,
+                                      std::string &Result);
+    
+    virtual void Initialize(ASTContext &context) = 0;
+    
+    // Metadata Rewriting.
+    virtual void RewriteMetaDataIntoBuffer(std::string &Result) = 0;
+    virtual void RewriteObjCProtocolListMetaData(const ObjCList<ObjCProtocolDecl> &Prots,
+                                                 StringRef prefix,
+                                                 StringRef ClassName,
+                                                 std::string &Result) = 0;
+    virtual void RewriteObjCCategoryImplDecl(ObjCCategoryImplDecl *CDecl,
+                                             std::string &Result) = 0;
+    virtual void RewriteObjCProtocolMetaData(ObjCProtocolDecl *Protocol,
+                                     StringRef prefix,
+                                     StringRef ClassName,
+                                     std::string &Result) = 0;
+    virtual void RewriteObjCClassMetaData(ObjCImplementationDecl *IDecl,
+                                          std::string &Result) = 0;
+    
+    // Rewriting ivar access
+    virtual Stmt *RewriteObjCIvarRefExpr(ObjCIvarRefExpr *IV) = 0;
+    virtual void RewriteIvarOffsetComputation(ObjCIvarDecl *ivar,
+                                         std::string &Result) = 0;
+    
+    // Misc. AST transformation routines. Sometimes they end up calling
+    // rewriting routines on the new ASTs.
+    CallExpr *SynthesizeCallToFunctionDecl(FunctionDecl *FD,
+                                           Expr **args, unsigned nargs,
+                                           SourceLocation StartLoc=SourceLocation(),
+                                           SourceLocation EndLoc=SourceLocation());
+    CallExpr *SynthMsgSendStretCallExpr(FunctionDecl *MsgSendStretFlavor,
+                                        QualType msgSendType, 
+                                        QualType returnType, 
+                                        SmallVectorImpl<QualType> &ArgTypes,
+                                        SmallVectorImpl<Expr*> &MsgExprs,
+                                        ObjCMethodDecl *Method);
+    Stmt *SynthMessageExpr(ObjCMessageExpr *Exp,
+                           SourceLocation StartLoc=SourceLocation(),
+                           SourceLocation EndLoc=SourceLocation());
+    
+    void SynthCountByEnumWithState(std::string &buf);
+    void SynthMsgSendFunctionDecl();
+    void SynthMsgSendSuperFunctionDecl();
+    void SynthMsgSendStretFunctionDecl();
+    void SynthMsgSendFpretFunctionDecl();
+    void SynthMsgSendSuperStretFunctionDecl();
+    void SynthGetClassFunctionDecl();
+    void SynthGetMetaClassFunctionDecl();
+    void SynthGetSuperClassFunctionDecl();
+    void SynthSelGetUidFunctionDecl();
+    void SynthSuperContructorFunctionDecl();
+    
+    std::string SynthesizeByrefCopyDestroyHelper(VarDecl *VD, int flag);
+    std::string SynthesizeBlockHelperFuncs(BlockExpr *CE, int i,
+                                      StringRef funcName, std::string Tag);
+    std::string SynthesizeBlockFunc(BlockExpr *CE, int i,
+                                      StringRef funcName, std::string Tag);
+    std::string SynthesizeBlockImpl(BlockExpr *CE, 
+                                    std::string Tag, std::string Desc);
+    std::string SynthesizeBlockDescriptor(std::string DescTag, 
+                                          std::string ImplTag,
+                                          int i, StringRef funcName,
+                                          unsigned hasCopy);
+    Stmt *SynthesizeBlockCall(CallExpr *Exp, const Expr* BlockExp);
+    void SynthesizeBlockLiterals(SourceLocation FunLocStart,
+                                 StringRef FunName);
+    FunctionDecl *SynthBlockInitFunctionDecl(StringRef name);
+    Stmt *SynthBlockInitExpr(BlockExpr *Exp,
+            const SmallVector<DeclRefExpr *, 8> &InnerBlockDeclRefs);
+
+    // Misc. helper routines.
+    QualType getProtocolType();
+    void WarnAboutReturnGotoStmts(Stmt *S);
+    void HasReturnStmts(Stmt *S, bool &hasReturns);
+    void CheckFunctionPointerDecl(QualType dType, NamedDecl *ND);
+    void InsertBlockLiteralsWithinFunction(FunctionDecl *FD);
+    void InsertBlockLiteralsWithinMethod(ObjCMethodDecl *MD);
+
+    bool IsDeclStmtInForeachHeader(DeclStmt *DS);
+    void CollectBlockDeclRefInfo(BlockExpr *Exp);
+    void GetBlockDeclRefExprs(Stmt *S);
+    void GetInnerBlockDeclRefExprs(Stmt *S, 
+                SmallVector<DeclRefExpr *, 8> &InnerBlockDeclRefs,
+                llvm::SmallPtrSet<const DeclContext *, 8> &InnerContexts);
+
+    // We avoid calling Type::isBlockPointerType(), since it operates on the
+    // canonical type. We only care if the top-level type is a closure pointer.
+    bool isTopLevelBlockPointerType(QualType T) {
+      return isa<BlockPointerType>(T);
+    }
+
+    /// convertBlockPointerToFunctionPointer - Converts a block-pointer type
+    /// to a function pointer type and upon success, returns true; false
+    /// otherwise.
+    bool convertBlockPointerToFunctionPointer(QualType &T) {
+      if (isTopLevelBlockPointerType(T)) {
+        const BlockPointerType *BPT = T->getAs<BlockPointerType>();
+        T = Context->getPointerType(BPT->getPointeeType());
+        return true;
+      }
+      return false;
+    }
+    
+    bool needToScanForQualifiers(QualType T);
+    QualType getSuperStructType();
+    QualType getConstantStringStructType();
+    QualType convertFunctionTypeOfBlocks(const FunctionType *FT);
+    bool BufferContainsPPDirectives(const char *startBuf, const char *endBuf);
+    
+    void convertToUnqualifiedObjCType(QualType &T) {
+      if (T->isObjCQualifiedIdType())
+        T = Context->getObjCIdType();
+      else if (T->isObjCQualifiedClassType())
+        T = Context->getObjCClassType();
+      else if (T->isObjCObjectPointerType() &&
+               T->getPointeeType()->isObjCQualifiedInterfaceType()) {
+        if (const ObjCObjectPointerType * OBJPT =
+              T->getAsObjCInterfacePointerType()) {
+          const ObjCInterfaceType *IFaceT = OBJPT->getInterfaceType();
+          T = QualType(IFaceT, 0);
+          T = Context->getPointerType(T);
+        }
+     }
+    }
+    
+    // FIXME: This predicate seems like it would be useful to add to ASTContext.
+    bool isObjCType(QualType T) {
+      if (!LangOpts.ObjC1 && !LangOpts.ObjC2)
+        return false;
+
+      QualType OCT = Context->getCanonicalType(T).getUnqualifiedType();
+
+      if (OCT == Context->getCanonicalType(Context->getObjCIdType()) ||
+          OCT == Context->getCanonicalType(Context->getObjCClassType()))
+        return true;
+
+      if (const PointerType *PT = OCT->getAs<PointerType>()) {
+        if (isa<ObjCInterfaceType>(PT->getPointeeType()) ||
+            PT->getPointeeType()->isObjCQualifiedIdType())
+          return true;
+      }
+      return false;
+    }
+    bool PointerTypeTakesAnyBlockArguments(QualType QT);
+    bool PointerTypeTakesAnyObjCQualifiedType(QualType QT);
+    void GetExtentOfArgList(const char *Name, const char *&LParen,
+                            const char *&RParen);
+    
+    void QuoteDoublequotes(std::string &From, std::string &To) {
+      for (unsigned i = 0; i < From.length(); i++) {
+        if (From[i] == '"')
+          To += "\\\"";
+        else
+          To += From[i];
+      }
+    }
+
+    QualType getSimpleFunctionType(QualType result,
+                                   const QualType *args,
+                                   unsigned numArgs,
+                                   bool variadic = false) {
+      if (result == Context->getObjCInstanceType())
+        result =  Context->getObjCIdType();
+      FunctionProtoType::ExtProtoInfo fpi;
+      fpi.Variadic = variadic;
+      return Context->getFunctionType(result, args, numArgs, fpi);
+    }
+
+    // Helper function: create a CStyleCastExpr with trivial type source info.
+    CStyleCastExpr* NoTypeInfoCStyleCastExpr(ASTContext *Ctx, QualType Ty,
+                                             CastKind Kind, Expr *E) {
+      TypeSourceInfo *TInfo = Ctx->getTrivialTypeSourceInfo(Ty, SourceLocation());
+      return CStyleCastExpr::Create(*Ctx, Ty, VK_RValue, Kind, E, 0, TInfo,
+                                    SourceLocation(), SourceLocation());
+    }
+  };
+  
+  class RewriteObjCFragileABI : public RewriteObjC {
+  public:
+    
+    RewriteObjCFragileABI(std::string inFile, raw_ostream *OS,
+                DiagnosticsEngine &D, const LangOptions &LOpts,
+                bool silenceMacroWarn) : RewriteObjC(inFile, OS,
+                                                     D, LOpts,
+                                                     silenceMacroWarn) {}
+    
+    ~RewriteObjCFragileABI() {}
+    virtual void Initialize(ASTContext &context);
+    
+    // Rewriting metadata
+    template<typename MethodIterator>
+    void RewriteObjCMethodsMetaData(MethodIterator MethodBegin,
+                                    MethodIterator MethodEnd,
+                                    bool IsInstanceMethod,
+                                    StringRef prefix,
+                                    StringRef ClassName,
+                                    std::string &Result);
+    virtual void RewriteObjCProtocolMetaData(ObjCProtocolDecl *Protocol,
+                                             StringRef prefix,
+                                             StringRef ClassName,
+                                             std::string &Result);
+    virtual void RewriteObjCProtocolListMetaData(
+                  const ObjCList<ObjCProtocolDecl> &Prots,
+                  StringRef prefix, StringRef ClassName, std::string &Result);
+    virtual void RewriteObjCClassMetaData(ObjCImplementationDecl *IDecl,
+                                          std::string &Result);
+    virtual void RewriteMetaDataIntoBuffer(std::string &Result);
+    virtual void RewriteObjCCategoryImplDecl(ObjCCategoryImplDecl *CDecl,
+                                             std::string &Result);
+    
+    // Rewriting ivar
+    virtual void RewriteIvarOffsetComputation(ObjCIvarDecl *ivar,
+                                              std::string &Result);
+    virtual Stmt *RewriteObjCIvarRefExpr(ObjCIvarRefExpr *IV);
+  };
+}
+
+void RewriteObjC::RewriteBlocksInFunctionProtoType(QualType funcType,
+                                                   NamedDecl *D) {
+  if (const FunctionProtoType *fproto
+      = dyn_cast<FunctionProtoType>(funcType.IgnoreParens())) {
+    for (FunctionProtoType::arg_type_iterator I = fproto->arg_type_begin(),
+         E = fproto->arg_type_end(); I && (I != E); ++I)
+      if (isTopLevelBlockPointerType(*I)) {
+        // All the args are checked/rewritten. Don't call twice!
+        RewriteBlockPointerDecl(D);
+        break;
+      }
+  }
+}
+
+void RewriteObjC::CheckFunctionPointerDecl(QualType funcType, NamedDecl *ND) {
+  const PointerType *PT = funcType->getAs<PointerType>();
+  if (PT && PointerTypeTakesAnyBlockArguments(funcType))
+    RewriteBlocksInFunctionProtoType(PT->getPointeeType(), ND);
+}
+
+static bool IsHeaderFile(const std::string &Filename) {
+  std::string::size_type DotPos = Filename.rfind('.');
+
+  if (DotPos == std::string::npos) {
+    // no file extension
+    return false;
+  }
+
+  std::string Ext = std::string(Filename.begin()+DotPos+1, Filename.end());
+  // C header: .h
+  // C++ header: .hh or .H;
+  return Ext == "h" || Ext == "hh" || Ext == "H";
+}
+
+RewriteObjC::RewriteObjC(std::string inFile, raw_ostream* OS,
+                         DiagnosticsEngine &D, const LangOptions &LOpts,
+                         bool silenceMacroWarn)
+      : Diags(D), LangOpts(LOpts), InFileName(inFile), OutFile(OS),
+        SilenceRewriteMacroWarning(silenceMacroWarn) {
+  IsHeader = IsHeaderFile(inFile);
+  RewriteFailedDiag = Diags.getCustomDiagID(DiagnosticsEngine::Warning,
+               "rewriting sub-expression within a macro (may not be correct)");
+  TryFinallyContainsReturnDiag = Diags.getCustomDiagID(
+               DiagnosticsEngine::Warning,
+               "rewriter doesn't support user-specified control flow semantics "
+               "for @try/@finally (code may not execute properly)");
+}
+
+ASTConsumer *clang::CreateObjCRewriter(const std::string& InFile,
+                                       raw_ostream* OS,
+                                       DiagnosticsEngine &Diags,
+                                       const LangOptions &LOpts,
+                                       bool SilenceRewriteMacroWarning) {
+  return new RewriteObjCFragileABI(InFile, OS, Diags, LOpts, SilenceRewriteMacroWarning);
+}
+
+void RewriteObjC::InitializeCommon(ASTContext &context) {
+  Context = &context;
+  SM = &Context->getSourceManager();
+  TUDecl = Context->getTranslationUnitDecl();
+  MsgSendFunctionDecl = 0;
+  MsgSendSuperFunctionDecl = 0;
+  MsgSendStretFunctionDecl = 0;
+  MsgSendSuperStretFunctionDecl = 0;
+  MsgSendFpretFunctionDecl = 0;
+  GetClassFunctionDecl = 0;
+  GetMetaClassFunctionDecl = 0;
+  GetSuperClassFunctionDecl = 0;
+  SelGetUidFunctionDecl = 0;
+  CFStringFunctionDecl = 0;
+  ConstantStringClassReference = 0;
+  NSStringRecord = 0;
+  CurMethodDef = 0;
+  CurFunctionDef = 0;
+  CurFunctionDeclToDeclareForBlock = 0;
+  GlobalVarDecl = 0;
+  SuperStructDecl = 0;
+  ProtocolTypeDecl = 0;
+  ConstantStringDecl = 0;
+  BcLabelCount = 0;
+  SuperContructorFunctionDecl = 0;
+  NumObjCStringLiterals = 0;
+  PropParentMap = 0;
+  CurrentBody = 0;
+  DisableReplaceStmt = false;
+  objc_impl_method = false;
+
+  // Get the ID and start/end of the main file.
+  MainFileID = SM->getMainFileID();
+  const llvm::MemoryBuffer *MainBuf = SM->getBuffer(MainFileID);
+  MainFileStart = MainBuf->getBufferStart();
+  MainFileEnd = MainBuf->getBufferEnd();
+
+  Rewrite.setSourceMgr(Context->getSourceManager(), Context->getLangOpts());
+}
+
+//===----------------------------------------------------------------------===//
+// Top Level Driver Code
+//===----------------------------------------------------------------------===//
+
+void RewriteObjC::HandleTopLevelSingleDecl(Decl *D) {
+  if (Diags.hasErrorOccurred())
+    return;
+
+  // Two cases: either the decl could be in the main file, or it could be in a
+  // #included file.  If the former, rewrite it now.  If the later, check to see
+  // if we rewrote the #include/#import.
+  SourceLocation Loc = D->getLocation();
+  Loc = SM->getExpansionLoc(Loc);
+
+  // If this is for a builtin, ignore it.
+  if (Loc.isInvalid()) return;
+
+  // Look for built-in declarations that we need to refer during the rewrite.
+  if (FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
+    RewriteFunctionDecl(FD);
+  } else if (VarDecl *FVD = dyn_cast<VarDecl>(D)) {
+    // declared in <Foundation/NSString.h>
+    if (FVD->getName() == "_NSConstantStringClassReference") {
+      ConstantStringClassReference = FVD;
+      return;
+    }
+  } else if (ObjCInterfaceDecl *ID = dyn_cast<ObjCInterfaceDecl>(D)) {
+    if (ID->isThisDeclarationADefinition())
+      RewriteInterfaceDecl(ID);
+  } else if (ObjCCategoryDecl *CD = dyn_cast<ObjCCategoryDecl>(D)) {
+    RewriteCategoryDecl(CD);
+  } else if (ObjCProtocolDecl *PD = dyn_cast<ObjCProtocolDecl>(D)) {
+    if (PD->isThisDeclarationADefinition())
+      RewriteProtocolDecl(PD);
+  } else if (LinkageSpecDecl *LSD = dyn_cast<LinkageSpecDecl>(D)) {
+    // Recurse into linkage specifications
+    for (DeclContext::decl_iterator DI = LSD->decls_begin(),
+                                 DIEnd = LSD->decls_end();
+         DI != DIEnd; ) {
+      if (ObjCInterfaceDecl *IFace = dyn_cast<ObjCInterfaceDecl>((*DI))) {
+        if (!IFace->isThisDeclarationADefinition()) {
+          SmallVector<Decl *, 8> DG;
+          SourceLocation StartLoc = IFace->getLocStart();
+          do {
+            if (isa<ObjCInterfaceDecl>(*DI) &&
+                !cast<ObjCInterfaceDecl>(*DI)->isThisDeclarationADefinition() &&
+                StartLoc == (*DI)->getLocStart())
+              DG.push_back(*DI);
+            else
+              break;
+            
+            ++DI;
+          } while (DI != DIEnd);
+          RewriteForwardClassDecl(DG);
+          continue;
+        }
+      }
+
+      if (ObjCProtocolDecl *Proto = dyn_cast<ObjCProtocolDecl>((*DI))) {
+        if (!Proto->isThisDeclarationADefinition()) {
+          SmallVector<Decl *, 8> DG;
+          SourceLocation StartLoc = Proto->getLocStart();
+          do {
+            if (isa<ObjCProtocolDecl>(*DI) &&
+                !cast<ObjCProtocolDecl>(*DI)->isThisDeclarationADefinition() &&
+                StartLoc == (*DI)->getLocStart())
+              DG.push_back(*DI);
+            else
+              break;
+            
+            ++DI;
+          } while (DI != DIEnd);
+          RewriteForwardProtocolDecl(DG);
+          continue;
+        }
+      }
+      
+      HandleTopLevelSingleDecl(*DI);
+      ++DI;
+    }
+  }
+  // If we have a decl in the main file, see if we should rewrite it.
+  if (SM->isFromMainFile(Loc))
+    return HandleDeclInMainFile(D);
+}
+
+//===----------------------------------------------------------------------===//
+// Syntactic (non-AST) Rewriting Code
+//===----------------------------------------------------------------------===//
+
+void RewriteObjC::RewriteInclude() {
+  SourceLocation LocStart = SM->getLocForStartOfFile(MainFileID);
+  StringRef MainBuf = SM->getBufferData(MainFileID);
+  const char *MainBufStart = MainBuf.begin();
+  const char *MainBufEnd = MainBuf.end();
+  size_t ImportLen = strlen("import");
+
+  // Loop over the whole file, looking for includes.
+  for (const char *BufPtr = MainBufStart; BufPtr < MainBufEnd; ++BufPtr) {
+    if (*BufPtr == '#') {
+      if (++BufPtr == MainBufEnd)
+        return;
+      while (*BufPtr == ' ' || *BufPtr == '\t')
+        if (++BufPtr == MainBufEnd)
+          return;
+      if (!strncmp(BufPtr, "import", ImportLen)) {
+        // replace import with include
+        SourceLocation ImportLoc =
+          LocStart.getLocWithOffset(BufPtr-MainBufStart);
+        ReplaceText(ImportLoc, ImportLen, "include");
+        BufPtr += ImportLen;
+      }
+    }
+  }
+}
+
+static std::string getIvarAccessString(ObjCIvarDecl *OID) {
+  const ObjCInterfaceDecl *ClassDecl = OID->getContainingInterface();
+  std::string S;
+  S = "((struct ";
+  S += ClassDecl->getIdentifier()->getName();
+  S += "_IMPL *)self)->";
+  S += OID->getName();
+  return S;
+}
+
+void RewriteObjC::RewritePropertyImplDecl(ObjCPropertyImplDecl *PID,
+                                          ObjCImplementationDecl *IMD,
+                                          ObjCCategoryImplDecl *CID) {
+  static bool objcGetPropertyDefined = false;
+  static bool objcSetPropertyDefined = false;
+  SourceLocation startLoc = PID->getLocStart();
+  InsertText(startLoc, "// ");
+  const char *startBuf = SM->getCharacterData(startLoc);
+  assert((*startBuf == '@') && "bogus @synthesize location");
+  const char *semiBuf = strchr(startBuf, ';');
+  assert((*semiBuf == ';') && "@synthesize: can't find ';'");
+  SourceLocation onePastSemiLoc =
+    startLoc.getLocWithOffset(semiBuf-startBuf+1);
+
+  if (PID->getPropertyImplementation() == ObjCPropertyImplDecl::Dynamic)
+    return; // FIXME: is this correct?
+
+  // Generate the 'getter' function.
+  ObjCPropertyDecl *PD = PID->getPropertyDecl();
+  ObjCIvarDecl *OID = PID->getPropertyIvarDecl();
+
+  if (!OID)
+    return;
+  unsigned Attributes = PD->getPropertyAttributes();
+  if (!PD->getGetterMethodDecl()->isDefined()) {
+    bool GenGetProperty = !(Attributes & ObjCPropertyDecl::OBJC_PR_nonatomic) &&
+                          (Attributes & (ObjCPropertyDecl::OBJC_PR_retain | 
+                                         ObjCPropertyDecl::OBJC_PR_copy));
+    std::string Getr;
+    if (GenGetProperty && !objcGetPropertyDefined) {
+      objcGetPropertyDefined = true;
+      // FIXME. Is this attribute correct in all cases?
+      Getr = "\nextern \"C\" __declspec(dllimport) "
+            "id objc_getProperty(id, SEL, long, bool);\n";
+    }
+    RewriteObjCMethodDecl(OID->getContainingInterface(),  
+                          PD->getGetterMethodDecl(), Getr);
+    Getr += "{ ";
+    // Synthesize an explicit cast to gain access to the ivar.
+    // See objc-act.c:objc_synthesize_new_getter() for details.
+    if (GenGetProperty) {
+      // return objc_getProperty(self, _cmd, offsetof(ClassDecl, OID), 1)
+      Getr += "typedef ";
+      const FunctionType *FPRetType = 0;
+      RewriteTypeIntoString(PD->getGetterMethodDecl()->getResultType(), Getr, 
+                            FPRetType);
+      Getr += " _TYPE";
+      if (FPRetType) {
+        Getr += ")"; // close the precedence "scope" for "*".
+      
+        // Now, emit the argument types (if any).
+        if (const FunctionProtoType *FT = dyn_cast<FunctionProtoType>(FPRetType)){
+          Getr += "(";
+          for (unsigned i = 0, e = FT->getNumArgs(); i != e; ++i) {
+            if (i) Getr += ", ";
+            std::string ParamStr = FT->getArgType(i).getAsString(
+                                                          Context->getPrintingPolicy());
+            Getr += ParamStr;
+          }
+          if (FT->isVariadic()) {
+            if (FT->getNumArgs()) Getr += ", ";
+            Getr += "...";
+          }
+          Getr += ")";
+        } else
+          Getr += "()";
+      }
+      Getr += ";\n";
+      Getr += "return (_TYPE)";
+      Getr += "objc_getProperty(self, _cmd, ";
+      RewriteIvarOffsetComputation(OID, Getr);
+      Getr += ", 1)";
+    }
+    else
+      Getr += "return " + getIvarAccessString(OID);
+    Getr += "; }";
+    InsertText(onePastSemiLoc, Getr);
+  }
+  
+  if (PD->isReadOnly() || PD->getSetterMethodDecl()->isDefined())
+    return;
+
+  // Generate the 'setter' function.
+  std::string Setr;
+  bool GenSetProperty = Attributes & (ObjCPropertyDecl::OBJC_PR_retain | 
+                                      ObjCPropertyDecl::OBJC_PR_copy);
+  if (GenSetProperty && !objcSetPropertyDefined) {
+    objcSetPropertyDefined = true;
+    // FIXME. Is this attribute correct in all cases?
+    Setr = "\nextern \"C\" __declspec(dllimport) "
+    "void objc_setProperty (id, SEL, long, id, bool, bool);\n";
+  }
+  
+  RewriteObjCMethodDecl(OID->getContainingInterface(), 
+                        PD->getSetterMethodDecl(), Setr);
+  Setr += "{ ";
+  // Synthesize an explicit cast to initialize the ivar.
+  // See objc-act.c:objc_synthesize_new_setter() for details.
+  if (GenSetProperty) {
+    Setr += "objc_setProperty (self, _cmd, ";
+    RewriteIvarOffsetComputation(OID, Setr);
+    Setr += ", (id)";
+    Setr += PD->getName();
+    Setr += ", ";
+    if (Attributes & ObjCPropertyDecl::OBJC_PR_nonatomic)
+      Setr += "0, ";
+    else
+      Setr += "1, ";
+    if (Attributes & ObjCPropertyDecl::OBJC_PR_copy)
+      Setr += "1)";
+    else
+      Setr += "0)";
+  }
+  else {
+    Setr += getIvarAccessString(OID) + " = ";
+    Setr += PD->getName();
+  }
+  Setr += "; }";
+  InsertText(onePastSemiLoc, Setr);
+}
+
+static void RewriteOneForwardClassDecl(ObjCInterfaceDecl *ForwardDecl,
+                                       std::string &typedefString) {
+  typedefString += "#ifndef _REWRITER_typedef_";
+  typedefString += ForwardDecl->getNameAsString();
+  typedefString += "\n";
+  typedefString += "#define _REWRITER_typedef_";
+  typedefString += ForwardDecl->getNameAsString();
+  typedefString += "\n";
+  typedefString += "typedef struct objc_object ";
+  typedefString += ForwardDecl->getNameAsString();
+  typedefString += ";\n#endif\n";
+}
+
+void RewriteObjC::RewriteForwardClassEpilogue(ObjCInterfaceDecl *ClassDecl,
+                                              const std::string &typedefString) {
+    SourceLocation startLoc = ClassDecl->getLocStart();
+    const char *startBuf = SM->getCharacterData(startLoc);
+    const char *semiPtr = strchr(startBuf, ';'); 
+    // Replace the @class with typedefs corresponding to the classes.
+    ReplaceText(startLoc, semiPtr-startBuf+1, typedefString);  
+}
+
+void RewriteObjC::RewriteForwardClassDecl(DeclGroupRef D) {
+  std::string typedefString;
+  for (DeclGroupRef::iterator I = D.begin(), E = D.end(); I != E; ++I) {
+    ObjCInterfaceDecl *ForwardDecl = cast<ObjCInterfaceDecl>(*I);
+    if (I == D.begin()) {
+      // Translate to typedef's that forward reference structs with the same name
+      // as the class. As a convenience, we include the original declaration
+      // as a comment.
+      typedefString += "// @class ";
+      typedefString += ForwardDecl->getNameAsString();
+      typedefString += ";\n";
+    }
+    RewriteOneForwardClassDecl(ForwardDecl, typedefString);
+  }
+  DeclGroupRef::iterator I = D.begin();
+  RewriteForwardClassEpilogue(cast<ObjCInterfaceDecl>(*I), typedefString);
+}
+
+void RewriteObjC::RewriteForwardClassDecl(
+                                const llvm::SmallVector<Decl*, 8> &D) {
+  std::string typedefString;
+  for (unsigned i = 0; i < D.size(); i++) {
+    ObjCInterfaceDecl *ForwardDecl = cast<ObjCInterfaceDecl>(D[i]);
+    if (i == 0) {
+      typedefString += "// @class ";
+      typedefString += ForwardDecl->getNameAsString();
+      typedefString += ";\n";
+    }
+    RewriteOneForwardClassDecl(ForwardDecl, typedefString);
+  }
+  RewriteForwardClassEpilogue(cast<ObjCInterfaceDecl>(D[0]), typedefString);
+}
+
+void RewriteObjC::RewriteMethodDeclaration(ObjCMethodDecl *Method) {
+  // When method is a synthesized one, such as a getter/setter there is
+  // nothing to rewrite.
+  if (Method->isImplicit())
+    return;
+  SourceLocation LocStart = Method->getLocStart();
+  SourceLocation LocEnd = Method->getLocEnd();
+
+  if (SM->getExpansionLineNumber(LocEnd) >
+      SM->getExpansionLineNumber(LocStart)) {
+    InsertText(LocStart, "#if 0\n");
+    ReplaceText(LocEnd, 1, ";\n#endif\n");
+  } else {
+    InsertText(LocStart, "// ");
+  }
+}
+
+void RewriteObjC::RewriteProperty(ObjCPropertyDecl *prop) {
+  SourceLocation Loc = prop->getAtLoc();
+
+  ReplaceText(Loc, 0, "// ");
+  // FIXME: handle properties that are declared across multiple lines.
+}
+
+void RewriteObjC::RewriteCategoryDecl(ObjCCategoryDecl *CatDecl) {
+  SourceLocation LocStart = CatDecl->getLocStart();
+
+  // FIXME: handle category headers that are declared across multiple lines.
+  ReplaceText(LocStart, 0, "// ");
+
+  for (ObjCCategoryDecl::prop_iterator I = CatDecl->prop_begin(),
+       E = CatDecl->prop_end(); I != E; ++I)
+    RewriteProperty(*I);
+  
+  for (ObjCCategoryDecl::instmeth_iterator
+         I = CatDecl->instmeth_begin(), E = CatDecl->instmeth_end();
+       I != E; ++I)
+    RewriteMethodDeclaration(*I);
+  for (ObjCCategoryDecl::classmeth_iterator
+         I = CatDecl->classmeth_begin(), E = CatDecl->classmeth_end();
+       I != E; ++I)
+    RewriteMethodDeclaration(*I);
+
+  // Lastly, comment out the @end.
+  ReplaceText(CatDecl->getAtEndRange().getBegin(), 
+              strlen("@end"), "/* @end */");
+}
+
+void RewriteObjC::RewriteProtocolDecl(ObjCProtocolDecl *PDecl) {
+  SourceLocation LocStart = PDecl->getLocStart();
+  assert(PDecl->isThisDeclarationADefinition());
+  
+  // FIXME: handle protocol headers that are declared across multiple lines.
+  ReplaceText(LocStart, 0, "// ");
+
+  for (ObjCProtocolDecl::instmeth_iterator
+         I = PDecl->instmeth_begin(), E = PDecl->instmeth_end();
+       I != E; ++I)
+    RewriteMethodDeclaration(*I);
+  for (ObjCProtocolDecl::classmeth_iterator
+         I = PDecl->classmeth_begin(), E = PDecl->classmeth_end();
+       I != E; ++I)
+    RewriteMethodDeclaration(*I);
+
+  for (ObjCInterfaceDecl::prop_iterator I = PDecl->prop_begin(),
+       E = PDecl->prop_end(); I != E; ++I)
+    RewriteProperty(*I);
+  
+  // Lastly, comment out the @end.
+  SourceLocation LocEnd = PDecl->getAtEndRange().getBegin();
+  ReplaceText(LocEnd, strlen("@end"), "/* @end */");
+
+  // Must comment out @optional/@required
+  const char *startBuf = SM->getCharacterData(LocStart);
+  const char *endBuf = SM->getCharacterData(LocEnd);
+  for (const char *p = startBuf; p < endBuf; p++) {
+    if (*p == '@' && !strncmp(p+1, "optional", strlen("optional"))) {
+      SourceLocation OptionalLoc = LocStart.getLocWithOffset(p-startBuf);
+      ReplaceText(OptionalLoc, strlen("@optional"), "/* @optional */");
+
+    }
+    else if (*p == '@' && !strncmp(p+1, "required", strlen("required"))) {
+      SourceLocation OptionalLoc = LocStart.getLocWithOffset(p-startBuf);
+      ReplaceText(OptionalLoc, strlen("@required"), "/* @required */");
+
+    }
+  }
+}
+
+void RewriteObjC::RewriteForwardProtocolDecl(DeclGroupRef D) {
+  SourceLocation LocStart = (*D.begin())->getLocStart();
+  if (LocStart.isInvalid())
+    llvm_unreachable("Invalid SourceLocation");
+  // FIXME: handle forward protocol that are declared across multiple lines.
+  ReplaceText(LocStart, 0, "// ");
+}
+
+void 
+RewriteObjC::RewriteForwardProtocolDecl(const llvm::SmallVector<Decl*, 8> &DG) {
+  SourceLocation LocStart = DG[0]->getLocStart();
+  if (LocStart.isInvalid())
+    llvm_unreachable("Invalid SourceLocation");
+  // FIXME: handle forward protocol that are declared across multiple lines.
+  ReplaceText(LocStart, 0, "// ");
+}
+
+void RewriteObjC::RewriteTypeIntoString(QualType T, std::string &ResultStr,
+                                        const FunctionType *&FPRetType) {
+  if (T->isObjCQualifiedIdType())
+    ResultStr += "id";
+  else if (T->isFunctionPointerType() ||
+           T->isBlockPointerType()) {
+    // needs special handling, since pointer-to-functions have special
+    // syntax (where a decaration models use).
+    QualType retType = T;
+    QualType PointeeTy;
+    if (const PointerType* PT = retType->getAs<PointerType>())
+      PointeeTy = PT->getPointeeType();
+    else if (const BlockPointerType *BPT = retType->getAs<BlockPointerType>())
+      PointeeTy = BPT->getPointeeType();
+    if ((FPRetType = PointeeTy->getAs<FunctionType>())) {
+      ResultStr += FPRetType->getResultType().getAsString(
+        Context->getPrintingPolicy());
+      ResultStr += "(*";
+    }
+  } else
+    ResultStr += T.getAsString(Context->getPrintingPolicy());
+}
+
+void RewriteObjC::RewriteObjCMethodDecl(const ObjCInterfaceDecl *IDecl,
+                                        ObjCMethodDecl *OMD,
+                                        std::string &ResultStr) {
+  //fprintf(stderr,"In RewriteObjCMethodDecl\n");
+  const FunctionType *FPRetType = 0;
+  ResultStr += "\nstatic ";
+  RewriteTypeIntoString(OMD->getResultType(), ResultStr, FPRetType);
+  ResultStr += " ";
+
+  // Unique method name
+  std::string NameStr;
+
+  if (OMD->isInstanceMethod())
+    NameStr += "_I_";
+  else
+    NameStr += "_C_";
+
+  NameStr += IDecl->getNameAsString();
+  NameStr += "_";
+
+  if (ObjCCategoryImplDecl *CID =
+      dyn_cast<ObjCCategoryImplDecl>(OMD->getDeclContext())) {
+    NameStr += CID->getNameAsString();
+    NameStr += "_";
+  }
+  // Append selector names, replacing ':' with '_'
+  {
+    std::string selString = OMD->getSelector().getAsString();
+    int len = selString.size();
+    for (int i = 0; i < len; i++)
+      if (selString[i] == ':')
+        selString[i] = '_';
+    NameStr += selString;
+  }
+  // Remember this name for metadata emission
+  MethodInternalNames[OMD] = NameStr;
+  ResultStr += NameStr;
+
+  // Rewrite arguments
+  ResultStr += "(";
+
+  // invisible arguments
+  if (OMD->isInstanceMethod()) {
+    QualType selfTy = Context->getObjCInterfaceType(IDecl);
+    selfTy = Context->getPointerType(selfTy);
+    if (!LangOpts.MicrosoftExt) {
+      if (ObjCSynthesizedStructs.count(const_cast<ObjCInterfaceDecl*>(IDecl)))
+        ResultStr += "struct ";
+    }
+    // When rewriting for Microsoft, explicitly omit the structure name.
+    ResultStr += IDecl->getNameAsString();
+    ResultStr += " *";
+  }
+  else
+    ResultStr += Context->getObjCClassType().getAsString(
+      Context->getPrintingPolicy());
+
+  ResultStr += " self, ";
+  ResultStr += Context->getObjCSelType().getAsString(Context->getPrintingPolicy());
+  ResultStr += " _cmd";
+
+  // Method arguments.
+  for (ObjCMethodDecl::param_iterator PI = OMD->param_begin(),
+       E = OMD->param_end(); PI != E; ++PI) {
+    ParmVarDecl *PDecl = *PI;
+    ResultStr += ", ";
+    if (PDecl->getType()->isObjCQualifiedIdType()) {
+      ResultStr += "id ";
+      ResultStr += PDecl->getNameAsString();
+    } else {
+      std::string Name = PDecl->getNameAsString();
+      QualType QT = PDecl->getType();
+      // Make sure we convert "t (^)(...)" to "t (*)(...)".
+      (void)convertBlockPointerToFunctionPointer(QT);
+      QT.getAsStringInternal(Name, Context->getPrintingPolicy());
+      ResultStr += Name;
+    }
+  }
+  if (OMD->isVariadic())
+    ResultStr += ", ...";
+  ResultStr += ") ";
+
+  if (FPRetType) {
+    ResultStr += ")"; // close the precedence "scope" for "*".
+
+    // Now, emit the argument types (if any).
+    if (const FunctionProtoType *FT = dyn_cast<FunctionProtoType>(FPRetType)) {
+      ResultStr += "(";
+      for (unsigned i = 0, e = FT->getNumArgs(); i != e; ++i) {
+        if (i) ResultStr += ", ";
+        std::string ParamStr = FT->getArgType(i).getAsString(
+          Context->getPrintingPolicy());
+        ResultStr += ParamStr;
+      }
+      if (FT->isVariadic()) {
+        if (FT->getNumArgs()) ResultStr += ", ";
+        ResultStr += "...";
+      }
+      ResultStr += ")";
+    } else {
+      ResultStr += "()";
+    }
+  }
+}
+void RewriteObjC::RewriteImplementationDecl(Decl *OID) {
+  ObjCImplementationDecl *IMD = dyn_cast<ObjCImplementationDecl>(OID);
+  ObjCCategoryImplDecl *CID = dyn_cast<ObjCCategoryImplDecl>(OID);
+
+  InsertText(IMD ? IMD->getLocStart() : CID->getLocStart(), "// ");
+
+  for (ObjCCategoryImplDecl::instmeth_iterator
+       I = IMD ? IMD->instmeth_begin() : CID->instmeth_begin(),
+       E = IMD ? IMD->instmeth_end() : CID->instmeth_end();
+       I != E; ++I) {
+    std::string ResultStr;
+    ObjCMethodDecl *OMD = *I;
+    RewriteObjCMethodDecl(OMD->getClassInterface(), OMD, ResultStr);
+    SourceLocation LocStart = OMD->getLocStart();
+    SourceLocation LocEnd = OMD->getCompoundBody()->getLocStart();
+
+    const char *startBuf = SM->getCharacterData(LocStart);
+    const char *endBuf = SM->getCharacterData(LocEnd);
+    ReplaceText(LocStart, endBuf-startBuf, ResultStr);
+  }
+
+  for (ObjCCategoryImplDecl::classmeth_iterator
+       I = IMD ? IMD->classmeth_begin() : CID->classmeth_begin(),
+       E = IMD ? IMD->classmeth_end() : CID->classmeth_end();
+       I != E; ++I) {
+    std::string ResultStr;
+    ObjCMethodDecl *OMD = *I;
+    RewriteObjCMethodDecl(OMD->getClassInterface(), OMD, ResultStr);
+    SourceLocation LocStart = OMD->getLocStart();
+    SourceLocation LocEnd = OMD->getCompoundBody()->getLocStart();
+
+    const char *startBuf = SM->getCharacterData(LocStart);
+    const char *endBuf = SM->getCharacterData(LocEnd);
+    ReplaceText(LocStart, endBuf-startBuf, ResultStr);
+  }
+  for (ObjCCategoryImplDecl::propimpl_iterator
+       I = IMD ? IMD->propimpl_begin() : CID->propimpl_begin(),
+       E = IMD ? IMD->propimpl_end() : CID->propimpl_end();
+       I != E; ++I) {
+    RewritePropertyImplDecl(*I, IMD, CID);
+  }
+
+  InsertText(IMD ? IMD->getLocEnd() : CID->getLocEnd(), "// ");
+}
+
+void RewriteObjC::RewriteInterfaceDecl(ObjCInterfaceDecl *ClassDecl) {
+  std::string ResultStr;
+  if (!ObjCForwardDecls.count(ClassDecl->getCanonicalDecl())) {
+    // we haven't seen a forward decl - generate a typedef.
+    ResultStr = "#ifndef _REWRITER_typedef_";
+    ResultStr += ClassDecl->getNameAsString();
+    ResultStr += "\n";
+    ResultStr += "#define _REWRITER_typedef_";
+    ResultStr += ClassDecl->getNameAsString();
+    ResultStr += "\n";
+    ResultStr += "typedef struct objc_object ";
+    ResultStr += ClassDecl->getNameAsString();
+    ResultStr += ";\n#endif\n";
+    // Mark this typedef as having been generated.
+    ObjCForwardDecls.insert(ClassDecl->getCanonicalDecl());
+  }
+  RewriteObjCInternalStruct(ClassDecl, ResultStr);
+
+  for (ObjCInterfaceDecl::prop_iterator I = ClassDecl->prop_begin(),
+         E = ClassDecl->prop_end(); I != E; ++I)
+    RewriteProperty(*I);
+  for (ObjCInterfaceDecl::instmeth_iterator
+         I = ClassDecl->instmeth_begin(), E = ClassDecl->instmeth_end();
+       I != E; ++I)
+    RewriteMethodDeclaration(*I);
+  for (ObjCInterfaceDecl::classmeth_iterator
+         I = ClassDecl->classmeth_begin(), E = ClassDecl->classmeth_end();
+       I != E; ++I)
+    RewriteMethodDeclaration(*I);
+
+  // Lastly, comment out the @end.
+  ReplaceText(ClassDecl->getAtEndRange().getBegin(), strlen("@end"), 
+              "/* @end */");
+}
+
+Stmt *RewriteObjC::RewritePropertyOrImplicitSetter(PseudoObjectExpr *PseudoOp) {
+  SourceRange OldRange = PseudoOp->getSourceRange();
+
+  // We just magically know some things about the structure of this
+  // expression.
+  ObjCMessageExpr *OldMsg =
+    cast<ObjCMessageExpr>(PseudoOp->getSemanticExpr(
+                            PseudoOp->getNumSemanticExprs() - 1));
+
+  // Because the rewriter doesn't allow us to rewrite rewritten code,
+  // we need to suppress rewriting the sub-statements.
+  Expr *Base, *RHS;
+  {
+    DisableReplaceStmtScope S(*this);
+
+    // Rebuild the base expression if we have one.
+    Base = 0;
+    if (OldMsg->getReceiverKind() == ObjCMessageExpr::Instance) {
+      Base = OldMsg->getInstanceReceiver();
+      Base = cast<OpaqueValueExpr>(Base)->getSourceExpr();
+      Base = cast<Expr>(RewriteFunctionBodyOrGlobalInitializer(Base));
+    }
+
+    // Rebuild the RHS.
+    RHS = cast<BinaryOperator>(PseudoOp->getSyntacticForm())->getRHS();
+    RHS = cast<OpaqueValueExpr>(RHS)->getSourceExpr();
+    RHS = cast<Expr>(RewriteFunctionBodyOrGlobalInitializer(RHS));
+  }
+
+  // TODO: avoid this copy.
+  SmallVector<SourceLocation, 1> SelLocs;
+  OldMsg->getSelectorLocs(SelLocs);
+
+  ObjCMessageExpr *NewMsg = 0;
+  switch (OldMsg->getReceiverKind()) {
+  case ObjCMessageExpr::Class:
+    NewMsg = ObjCMessageExpr::Create(*Context, OldMsg->getType(),
+                                     OldMsg->getValueKind(),
+                                     OldMsg->getLeftLoc(),
+                                     OldMsg->getClassReceiverTypeInfo(),
+                                     OldMsg->getSelector(),
+                                     SelLocs,
+                                     OldMsg->getMethodDecl(),
+                                     RHS,
+                                     OldMsg->getRightLoc(),
+                                     OldMsg->isImplicit());
+    break;
+
+  case ObjCMessageExpr::Instance:
+    NewMsg = ObjCMessageExpr::Create(*Context, OldMsg->getType(),
+                                     OldMsg->getValueKind(),
+                                     OldMsg->getLeftLoc(),
+                                     Base,
+                                     OldMsg->getSelector(),
+                                     SelLocs,
+                                     OldMsg->getMethodDecl(),
+                                     RHS,
+                                     OldMsg->getRightLoc(),
+                                     OldMsg->isImplicit());
+    break;
+
+  case ObjCMessageExpr::SuperClass:
+  case ObjCMessageExpr::SuperInstance:
+    NewMsg = ObjCMessageExpr::Create(*Context, OldMsg->getType(),
+                                     OldMsg->getValueKind(),
+                                     OldMsg->getLeftLoc(),
+                                     OldMsg->getSuperLoc(),
+                 OldMsg->getReceiverKind() == ObjCMessageExpr::SuperInstance,
+                                     OldMsg->getSuperType(),
+                                     OldMsg->getSelector(),
+                                     SelLocs,
+                                     OldMsg->getMethodDecl(),
+                                     RHS,
+                                     OldMsg->getRightLoc(),
+                                     OldMsg->isImplicit());
+    break;
+  }
+
+  Stmt *Replacement = SynthMessageExpr(NewMsg);
+  ReplaceStmtWithRange(PseudoOp, Replacement, OldRange);
+  return Replacement;
+}
+
+Stmt *RewriteObjC::RewritePropertyOrImplicitGetter(PseudoObjectExpr *PseudoOp) {
+  SourceRange OldRange = PseudoOp->getSourceRange();
+
+  // We just magically know some things about the structure of this
+  // expression.
+  ObjCMessageExpr *OldMsg =
+    cast<ObjCMessageExpr>(PseudoOp->getResultExpr()->IgnoreImplicit());
+
+  // Because the rewriter doesn't allow us to rewrite rewritten code,
+  // we need to suppress rewriting the sub-statements.
+  Expr *Base = 0;
+  {
+    DisableReplaceStmtScope S(*this);
+
+    // Rebuild the base expression if we have one.
+    if (OldMsg->getReceiverKind() == ObjCMessageExpr::Instance) {
+      Base = OldMsg->getInstanceReceiver();
+      Base = cast<OpaqueValueExpr>(Base)->getSourceExpr();
+      Base = cast<Expr>(RewriteFunctionBodyOrGlobalInitializer(Base));
+    }
+  }
+
+  // Intentionally empty.
+  SmallVector<SourceLocation, 1> SelLocs;
+  SmallVector<Expr*, 1> Args;
+
+  ObjCMessageExpr *NewMsg = 0;
+  switch (OldMsg->getReceiverKind()) {
+  case ObjCMessageExpr::Class:
+    NewMsg = ObjCMessageExpr::Create(*Context, OldMsg->getType(),
+                                     OldMsg->getValueKind(),
+                                     OldMsg->getLeftLoc(),
+                                     OldMsg->getClassReceiverTypeInfo(),
+                                     OldMsg->getSelector(),
+                                     SelLocs,
+                                     OldMsg->getMethodDecl(),
+                                     Args,
+                                     OldMsg->getRightLoc(),
+                                     OldMsg->isImplicit());
+    break;
+
+  case ObjCMessageExpr::Instance:
+    NewMsg = ObjCMessageExpr::Create(*Context, OldMsg->getType(),
+                                     OldMsg->getValueKind(),
+                                     OldMsg->getLeftLoc(),
+                                     Base,
+                                     OldMsg->getSelector(),
+                                     SelLocs,
+                                     OldMsg->getMethodDecl(),
+                                     Args,
+                                     OldMsg->getRightLoc(),
+                                     OldMsg->isImplicit());
+    break;
+
+  case ObjCMessageExpr::SuperClass:
+  case ObjCMessageExpr::SuperInstance:
+    NewMsg = ObjCMessageExpr::Create(*Context, OldMsg->getType(),
+                                     OldMsg->getValueKind(),
+                                     OldMsg->getLeftLoc(),
+                                     OldMsg->getSuperLoc(),
+                 OldMsg->getReceiverKind() == ObjCMessageExpr::SuperInstance,
+                                     OldMsg->getSuperType(),
+                                     OldMsg->getSelector(),
+                                     SelLocs,
+                                     OldMsg->getMethodDecl(),
+                                     Args,
+                                     OldMsg->getRightLoc(),
+                                     OldMsg->isImplicit());
+    break;
+  }
+
+  Stmt *Replacement = SynthMessageExpr(NewMsg);
+  ReplaceStmtWithRange(PseudoOp, Replacement, OldRange);
+  return Replacement;
+}
+
+/// SynthCountByEnumWithState - To print:
+/// ((unsigned int (*)
+///  (id, SEL, struct __objcFastEnumerationState *, id *, unsigned int))
+///  (void *)objc_msgSend)((id)l_collection,
+///                        sel_registerName(
+///                          "countByEnumeratingWithState:objects:count:"),
+///                        &enumState,
+///                        (id *)__rw_items, (unsigned int)16)
+///
+void RewriteObjC::SynthCountByEnumWithState(std::string &buf) {
+  buf += "((unsigned int (*) (id, SEL, struct __objcFastEnumerationState *, "
+  "id *, unsigned int))(void *)objc_msgSend)";
+  buf += "\n\t\t";
+  buf += "((id)l_collection,\n\t\t";
+  buf += "sel_registerName(\"countByEnumeratingWithState:objects:count:\"),";
+  buf += "\n\t\t";
+  buf += "&enumState, "
+         "(id *)__rw_items, (unsigned int)16)";
+}
+
+/// RewriteBreakStmt - Rewrite for a break-stmt inside an ObjC2's foreach
+/// statement to exit to its outer synthesized loop.
+///
+Stmt *RewriteObjC::RewriteBreakStmt(BreakStmt *S) {
+  if (Stmts.empty() || !isa<ObjCForCollectionStmt>(Stmts.back()))
+    return S;
+  // replace break with goto __break_label
+  std::string buf;
+
+  SourceLocation startLoc = S->getLocStart();
+  buf = "goto __break_label_";
+  buf += utostr(ObjCBcLabelNo.back());
+  ReplaceText(startLoc, strlen("break"), buf);
+
+  return 0;
+}
+
+/// RewriteContinueStmt - Rewrite for a continue-stmt inside an ObjC2's foreach
+/// statement to continue with its inner synthesized loop.
+///
+Stmt *RewriteObjC::RewriteContinueStmt(ContinueStmt *S) {
+  if (Stmts.empty() || !isa<ObjCForCollectionStmt>(Stmts.back()))
+    return S;
+  // replace continue with goto __continue_label
+  std::string buf;
+
+  SourceLocation startLoc = S->getLocStart();
+  buf = "goto __continue_label_";
+  buf += utostr(ObjCBcLabelNo.back());
+  ReplaceText(startLoc, strlen("continue"), buf);
+
+  return 0;
+}
+
+/// RewriteObjCForCollectionStmt - Rewriter for ObjC2's foreach statement.
+///  It rewrites:
+/// for ( type elem in collection) { stmts; }
+
+/// Into:
+/// {
+///   type elem;
+///   struct __objcFastEnumerationState enumState = { 0 };
+///   id __rw_items[16];
+///   id l_collection = (id)collection;
+///   unsigned long limit = [l_collection countByEnumeratingWithState:&enumState
+///                                       objects:__rw_items count:16];
+/// if (limit) {
+///   unsigned long startMutations = *enumState.mutationsPtr;
+///   do {
+///        unsigned long counter = 0;
+///        do {
+///             if (startMutations != *enumState.mutationsPtr)
+///               objc_enumerationMutation(l_collection);
+///             elem = (type)enumState.itemsPtr[counter++];
+///             stmts;
+///             __continue_label: ;
+///        } while (counter < limit);
+///   } while (limit = [l_collection countByEnumeratingWithState:&enumState
+///                                  objects:__rw_items count:16]);
+///   elem = nil;
+///   __break_label: ;
+///  }
+///  else
+///       elem = nil;
+///  }
+///
+Stmt *RewriteObjC::RewriteObjCForCollectionStmt(ObjCForCollectionStmt *S,
+                                                SourceLocation OrigEnd) {
+  assert(!Stmts.empty() && "ObjCForCollectionStmt - Statement stack empty");
+  assert(isa<ObjCForCollectionStmt>(Stmts.back()) &&
+         "ObjCForCollectionStmt Statement stack mismatch");
+  assert(!ObjCBcLabelNo.empty() &&
+         "ObjCForCollectionStmt - Label No stack empty");
+
+  SourceLocation startLoc = S->getLocStart();
+  const char *startBuf = SM->getCharacterData(startLoc);
+  StringRef elementName;
+  std::string elementTypeAsString;
+  std::string buf;
+  buf = "\n{\n\t";
+  if (DeclStmt *DS = dyn_cast<DeclStmt>(S->getElement())) {
+    // type elem;
+    NamedDecl* D = cast<NamedDecl>(DS->getSingleDecl());
+    QualType ElementType = cast<ValueDecl>(D)->getType();
+    if (ElementType->isObjCQualifiedIdType() ||
+        ElementType->isObjCQualifiedInterfaceType())
+      // Simply use 'id' for all qualified types.
+      elementTypeAsString = "id";
+    else
+      elementTypeAsString = ElementType.getAsString(Context->getPrintingPolicy());
+    buf += elementTypeAsString;
+    buf += " ";
+    elementName = D->getName();
+    buf += elementName;
+    buf += ";\n\t";
+  }
+  else {
+    DeclRefExpr *DR = cast<DeclRefExpr>(S->getElement());
+    elementName = DR->getDecl()->getName();
+    ValueDecl *VD = cast<ValueDecl>(DR->getDecl());
+    if (VD->getType()->isObjCQualifiedIdType() ||
+        VD->getType()->isObjCQualifiedInterfaceType())
+      // Simply use 'id' for all qualified types.
+      elementTypeAsString = "id";
+    else
+      elementTypeAsString = VD->getType().getAsString(Context->getPrintingPolicy());
+  }
+
+  // struct __objcFastEnumerationState enumState = { 0 };
+  buf += "struct __objcFastEnumerationState enumState = { 0 };\n\t";
+  // id __rw_items[16];
+  buf += "id __rw_items[16];\n\t";
+  // id l_collection = (id)
+  buf += "id l_collection = (id)";
+  // Find start location of 'collection' the hard way!
+  const char *startCollectionBuf = startBuf;
+  startCollectionBuf += 3;  // skip 'for'
+  startCollectionBuf = strchr(startCollectionBuf, '(');
+  startCollectionBuf++; // skip '('
+  // find 'in' and skip it.
+  while (*startCollectionBuf != ' ' ||
+         *(startCollectionBuf+1) != 'i' || *(startCollectionBuf+2) != 'n' ||
+         (*(startCollectionBuf+3) != ' ' &&
+          *(startCollectionBuf+3) != '[' && *(startCollectionBuf+3) != '('))
+    startCollectionBuf++;
+  startCollectionBuf += 3;
+
+  // Replace: "for (type element in" with string constructed thus far.
+  ReplaceText(startLoc, startCollectionBuf - startBuf, buf);
+  // Replace ')' in for '(' type elem in collection ')' with ';'
+  SourceLocation rightParenLoc = S->getRParenLoc();
+  const char *rparenBuf = SM->getCharacterData(rightParenLoc);
+  SourceLocation lparenLoc = startLoc.getLocWithOffset(rparenBuf-startBuf);
+  buf = ";\n\t";
+
+  // unsigned long limit = [l_collection countByEnumeratingWithState:&enumState
+  //                                   objects:__rw_items count:16];
+  // which is synthesized into:
+  // unsigned int limit =
+  // ((unsigned int (*)
+  //  (id, SEL, struct __objcFastEnumerationState *, id *, unsigned int))
+  //  (void *)objc_msgSend)((id)l_collection,
+  //                        sel_registerName(
+  //                          "countByEnumeratingWithState:objects:count:"),
+  //                        (struct __objcFastEnumerationState *)&state,
+  //                        (id *)__rw_items, (unsigned int)16);
+  buf += "unsigned long limit =\n\t\t";
+  SynthCountByEnumWithState(buf);
+  buf += ";\n\t";
+  /// if (limit) {
+  ///   unsigned long startMutations = *enumState.mutationsPtr;
+  ///   do {
+  ///        unsigned long counter = 0;
+  ///        do {
+  ///             if (startMutations != *enumState.mutationsPtr)
+  ///               objc_enumerationMutation(l_collection);
+  ///             elem = (type)enumState.itemsPtr[counter++];
+  buf += "if (limit) {\n\t";
+  buf += "unsigned long startMutations = *enumState.mutationsPtr;\n\t";
+  buf += "do {\n\t\t";
+  buf += "unsigned long counter = 0;\n\t\t";
+  buf += "do {\n\t\t\t";
+  buf += "if (startMutations != *enumState.mutationsPtr)\n\t\t\t\t";
+  buf += "objc_enumerationMutation(l_collection);\n\t\t\t";
+  buf += elementName;
+  buf += " = (";
+  buf += elementTypeAsString;
+  buf += ")enumState.itemsPtr[counter++];";
+  // Replace ')' in for '(' type elem in collection ')' with all of these.
+  ReplaceText(lparenLoc, 1, buf);
+
+  ///            __continue_label: ;
+  ///        } while (counter < limit);
+  ///   } while (limit = [l_collection countByEnumeratingWithState:&enumState
+  ///                                  objects:__rw_items count:16]);
+  ///   elem = nil;
+  ///   __break_label: ;
+  ///  }
+  ///  else
+  ///       elem = nil;
+  ///  }
+  ///
+  buf = ";\n\t";
+  buf += "__continue_label_";
+  buf += utostr(ObjCBcLabelNo.back());
+  buf += ": ;";
+  buf += "\n\t\t";
+  buf += "} while (counter < limit);\n\t";
+  buf += "} while (limit = ";
+  SynthCountByEnumWithState(buf);
+  buf += ");\n\t";
+  buf += elementName;
+  buf += " = ((";
+  buf += elementTypeAsString;
+  buf += ")0);\n\t";
+  buf += "__break_label_";
+  buf += utostr(ObjCBcLabelNo.back());
+  buf += ": ;\n\t";
+  buf += "}\n\t";
+  buf += "else\n\t\t";
+  buf += elementName;
+  buf += " = ((";
+  buf += elementTypeAsString;
+  buf += ")0);\n\t";
+  buf += "}\n";
+
+  // Insert all these *after* the statement body.
+  // FIXME: If this should support Obj-C++, support CXXTryStmt
+  if (isa<CompoundStmt>(S->getBody())) {
+    SourceLocation endBodyLoc = OrigEnd.getLocWithOffset(1);
+    InsertText(endBodyLoc, buf);
+  } else {
+    /* Need to treat single statements specially. For example:
+     *
+     *     for (A *a in b) if (stuff()) break;
+     *     for (A *a in b) xxxyy;
+     *
+     * The following code simply scans ahead to the semi to find the actual end.
+     */
+    const char *stmtBuf = SM->getCharacterData(OrigEnd);
+    const char *semiBuf = strchr(stmtBuf, ';');
+    assert(semiBuf && "Can't find ';'");
+    SourceLocation endBodyLoc = OrigEnd.getLocWithOffset(semiBuf-stmtBuf+1);
+    InsertText(endBodyLoc, buf);
+  }
+  Stmts.pop_back();
+  ObjCBcLabelNo.pop_back();
+  return 0;
+}
+
+/// RewriteObjCSynchronizedStmt -
+/// This routine rewrites @synchronized(expr) stmt;
+/// into:
+/// objc_sync_enter(expr);
+/// @try stmt @finally { objc_sync_exit(expr); }
+///
+Stmt *RewriteObjC::RewriteObjCSynchronizedStmt(ObjCAtSynchronizedStmt *S) {
+  // Get the start location and compute the semi location.
+  SourceLocation startLoc = S->getLocStart();
+  const char *startBuf = SM->getCharacterData(startLoc);
+
+  assert((*startBuf == '@') && "bogus @synchronized location");
+
+  std::string buf;
+  buf = "objc_sync_enter((id)";
+  const char *lparenBuf = startBuf;
+  while (*lparenBuf != '(') lparenBuf++;
+  ReplaceText(startLoc, lparenBuf-startBuf+1, buf);
+  // We can't use S->getSynchExpr()->getLocEnd() to find the end location, since
+  // the sync expression is typically a message expression that's already
+  // been rewritten! (which implies the SourceLocation's are invalid).
+  SourceLocation endLoc = S->getSynchBody()->getLocStart();
+  const char *endBuf = SM->getCharacterData(endLoc);
+  while (*endBuf != ')') endBuf--;
+  SourceLocation rparenLoc = startLoc.getLocWithOffset(endBuf-startBuf);
+  buf = ");\n";
+  // declare a new scope with two variables, _stack and _rethrow.
+  buf += "/* @try scope begin */ \n{ struct _objc_exception_data {\n";
+  buf += "int buf[18/*32-bit i386*/];\n";
+  buf += "char *pointers[4];} _stack;\n";
+  buf += "id volatile _rethrow = 0;\n";
+  buf += "objc_exception_try_enter(&_stack);\n";
+  buf += "if (!_setjmp(_stack.buf)) /* @try block continue */\n";
+  ReplaceText(rparenLoc, 1, buf);
+  startLoc = S->getSynchBody()->getLocEnd();
+  startBuf = SM->getCharacterData(startLoc);
+
+  assert((*startBuf == '}') && "bogus @synchronized block");
+  SourceLocation lastCurlyLoc = startLoc;
+  buf = "}\nelse {\n";
+  buf += "  _rethrow = objc_exception_extract(&_stack);\n";
+  buf += "}\n";
+  buf += "{ /* implicit finally clause */\n";
+  buf += "  if (!_rethrow) objc_exception_try_exit(&_stack);\n";
+  
+  std::string syncBuf;
+  syncBuf += " objc_sync_exit(";
+
+  Expr *syncExpr = S->getSynchExpr();
+  CastKind CK = syncExpr->getType()->isObjCObjectPointerType()
+                  ? CK_BitCast :
+                syncExpr->getType()->isBlockPointerType()
+                  ? CK_BlockPointerToObjCPointerCast
+                  : CK_CPointerToObjCPointerCast;
+  syncExpr = NoTypeInfoCStyleCastExpr(Context, Context->getObjCIdType(),
+                                      CK, syncExpr);
+  std::string syncExprBufS;
+  llvm::raw_string_ostream syncExprBuf(syncExprBufS);
+  syncExpr->printPretty(syncExprBuf, 0, PrintingPolicy(LangOpts));
+  syncBuf += syncExprBuf.str();
+  syncBuf += ");";
+  
+  buf += syncBuf;
+  buf += "\n  if (_rethrow) objc_exception_throw(_rethrow);\n";
+  buf += "}\n";
+  buf += "}";
+
+  ReplaceText(lastCurlyLoc, 1, buf);
+
+  bool hasReturns = false;
+  HasReturnStmts(S->getSynchBody(), hasReturns);
+  if (hasReturns)
+    RewriteSyncReturnStmts(S->getSynchBody(), syncBuf);
+
+  return 0;
+}
+
+void RewriteObjC::WarnAboutReturnGotoStmts(Stmt *S)
+{
+  // Perform a bottom up traversal of all children.
+  for (Stmt::child_range CI = S->children(); CI; ++CI)
+    if (*CI)
+      WarnAboutReturnGotoStmts(*CI);
+
+  if (isa<ReturnStmt>(S) || isa<GotoStmt>(S)) {
+    Diags.Report(Context->getFullLoc(S->getLocStart()),
+                 TryFinallyContainsReturnDiag);
+  }
+  return;
+}
+
+void RewriteObjC::HasReturnStmts(Stmt *S, bool &hasReturns) 
+{  
+  // Perform a bottom up traversal of all children.
+  for (Stmt::child_range CI = S->children(); CI; ++CI)
+   if (*CI)
+     HasReturnStmts(*CI, hasReturns);
+
+ if (isa<ReturnStmt>(S))
+   hasReturns = true;
+ return;
+}
+
+void RewriteObjC::RewriteTryReturnStmts(Stmt *S) {
+ // Perform a bottom up traversal of all children.
+ for (Stmt::child_range CI = S->children(); CI; ++CI)
+   if (*CI) {
+     RewriteTryReturnStmts(*CI);
+   }
+ if (isa<ReturnStmt>(S)) {
+   SourceLocation startLoc = S->getLocStart();
+   const char *startBuf = SM->getCharacterData(startLoc);
+
+   const char *semiBuf = strchr(startBuf, ';');
+   assert((*semiBuf == ';') && "RewriteTryReturnStmts: can't find ';'");
+   SourceLocation onePastSemiLoc = startLoc.getLocWithOffset(semiBuf-startBuf+1);
+
+   std::string buf;
+   buf = "{ objc_exception_try_exit(&_stack); return";
+   
+   ReplaceText(startLoc, 6, buf);
+   InsertText(onePastSemiLoc, "}");
+ }
+ return;
+}
+
+void RewriteObjC::RewriteSyncReturnStmts(Stmt *S, std::string syncExitBuf) {
+  // Perform a bottom up traversal of all children.
+  for (Stmt::child_range CI = S->children(); CI; ++CI)
+    if (*CI) {
+      RewriteSyncReturnStmts(*CI, syncExitBuf);
+    }
+  if (isa<ReturnStmt>(S)) {
+    SourceLocation startLoc = S->getLocStart();
+    const char *startBuf = SM->getCharacterData(startLoc);
+
+    const char *semiBuf = strchr(startBuf, ';');
+    assert((*semiBuf == ';') && "RewriteSyncReturnStmts: can't find ';'");
+    SourceLocation onePastSemiLoc = startLoc.getLocWithOffset(semiBuf-startBuf+1);
+
+    std::string buf;
+    buf = "{ objc_exception_try_exit(&_stack);";
+    buf += syncExitBuf;
+    buf += " return";
+    
+    ReplaceText(startLoc, 6, buf);
+    InsertText(onePastSemiLoc, "}");
+  }
+  return;
+}
+
+Stmt *RewriteObjC::RewriteObjCTryStmt(ObjCAtTryStmt *S) {
+  // Get the start location and compute the semi location.
+  SourceLocation startLoc = S->getLocStart();
+  const char *startBuf = SM->getCharacterData(startLoc);
+
+  assert((*startBuf == '@') && "bogus @try location");
+
+  std::string buf;
+  // declare a new scope with two variables, _stack and _rethrow.
+  buf = "/* @try scope begin */ { struct _objc_exception_data {\n";
+  buf += "int buf[18/*32-bit i386*/];\n";
+  buf += "char *pointers[4];} _stack;\n";
+  buf += "id volatile _rethrow = 0;\n";
+  buf += "objc_exception_try_enter(&_stack);\n";
+  buf += "if (!_setjmp(_stack.buf)) /* @try block continue */\n";
+
+  ReplaceText(startLoc, 4, buf);
+
+  startLoc = S->getTryBody()->getLocEnd();
+  startBuf = SM->getCharacterData(startLoc);
+
+  assert((*startBuf == '}') && "bogus @try block");
+
+  SourceLocation lastCurlyLoc = startLoc;
+  if (S->getNumCatchStmts()) {
+    startLoc = startLoc.getLocWithOffset(1);
+    buf = " /* @catch begin */ else {\n";
+    buf += " id _caught = objc_exception_extract(&_stack);\n";
+    buf += " objc_exception_try_enter (&_stack);\n";
+    buf += " if (_setjmp(_stack.buf))\n";
+    buf += "   _rethrow = objc_exception_extract(&_stack);\n";
+    buf += " else { /* @catch continue */";
+
+    InsertText(startLoc, buf);
+  } else { /* no catch list */
+    buf = "}\nelse {\n";
+    buf += "  _rethrow = objc_exception_extract(&_stack);\n";
+    buf += "}";
+    ReplaceText(lastCurlyLoc, 1, buf);
+  }
+  Stmt *lastCatchBody = 0;
+  for (unsigned I = 0, N = S->getNumCatchStmts(); I != N; ++I) {
+    ObjCAtCatchStmt *Catch = S->getCatchStmt(I);
+    VarDecl *catchDecl = Catch->getCatchParamDecl();
+
+    if (I == 0)
+      buf = "if ("; // we are generating code for the first catch clause
+    else
+      buf = "else if (";
+    startLoc = Catch->getLocStart();
+    startBuf = SM->getCharacterData(startLoc);
+
+    assert((*startBuf == '@') && "bogus @catch location");
+
+    const char *lParenLoc = strchr(startBuf, '(');
+
+    if (Catch->hasEllipsis()) {
+      // Now rewrite the body...
+      lastCatchBody = Catch->getCatchBody();
+      SourceLocation bodyLoc = lastCatchBody->getLocStart();
+      const char *bodyBuf = SM->getCharacterData(bodyLoc);
+      assert(*SM->getCharacterData(Catch->getRParenLoc()) == ')' &&
+             "bogus @catch paren location");
+      assert((*bodyBuf == '{') && "bogus @catch body location");
+
+      buf += "1) { id _tmp = _caught;";
+      Rewrite.ReplaceText(startLoc, bodyBuf-startBuf+1, buf);
+    } else if (catchDecl) {
+      QualType t = catchDecl->getType();
+      if (t == Context->getObjCIdType()) {
+        buf += "1) { ";
+        ReplaceText(startLoc, lParenLoc-startBuf+1, buf);
+      } else if (const ObjCObjectPointerType *Ptr =
+                   t->getAs<ObjCObjectPointerType>()) {
+        // Should be a pointer to a class.
+        ObjCInterfaceDecl *IDecl = Ptr->getObjectType()->getInterface();
+        if (IDecl) {
+          buf += "objc_exception_match((struct objc_class *)objc_getClass(\"";
+          buf += IDecl->getNameAsString();
+          buf += "\"), (struct objc_object *)_caught)) { ";
+          ReplaceText(startLoc, lParenLoc-startBuf+1, buf);
+        }
+      }
+      // Now rewrite the body...
+      lastCatchBody = Catch->getCatchBody();
+      SourceLocation rParenLoc = Catch->getRParenLoc();
+      SourceLocation bodyLoc = lastCatchBody->getLocStart();
+      const char *bodyBuf = SM->getCharacterData(bodyLoc);
+      const char *rParenBuf = SM->getCharacterData(rParenLoc);
+      assert((*rParenBuf == ')') && "bogus @catch paren location");
+      assert((*bodyBuf == '{') && "bogus @catch body location");
+
+      // Here we replace ") {" with "= _caught;" (which initializes and
+      // declares the @catch parameter).
+      ReplaceText(rParenLoc, bodyBuf-rParenBuf+1, " = _caught;");
+    } else {
+      llvm_unreachable("@catch rewrite bug");
+    }
+  }
+  // Complete the catch list...
+  if (lastCatchBody) {
+    SourceLocation bodyLoc = lastCatchBody->getLocEnd();
+    assert(*SM->getCharacterData(bodyLoc) == '}' &&
+           "bogus @catch body location");
+
+    // Insert the last (implicit) else clause *before* the right curly brace.
+    bodyLoc = bodyLoc.getLocWithOffset(-1);
+    buf = "} /* last catch end */\n";
+    buf += "else {\n";
+    buf += " _rethrow = _caught;\n";
+    buf += " objc_exception_try_exit(&_stack);\n";
+    buf += "} } /* @catch end */\n";
+    if (!S->getFinallyStmt())
+      buf += "}\n";
+    InsertText(bodyLoc, buf);
+
+    // Set lastCurlyLoc
+    lastCurlyLoc = lastCatchBody->getLocEnd();
+  }
+  if (ObjCAtFinallyStmt *finalStmt = S->getFinallyStmt()) {
+    startLoc = finalStmt->getLocStart();
+    startBuf = SM->getCharacterData(startLoc);
+    assert((*startBuf == '@') && "bogus @finally start");
+
+    ReplaceText(startLoc, 8, "/* @finally */");
+
+    Stmt *body = finalStmt->getFinallyBody();
+    SourceLocation startLoc = body->getLocStart();
+    SourceLocation endLoc = body->getLocEnd();
+    assert(*SM->getCharacterData(startLoc) == '{' &&
+           "bogus @finally body location");
+    assert(*SM->getCharacterData(endLoc) == '}' &&
+           "bogus @finally body location");
+
+    startLoc = startLoc.getLocWithOffset(1);
+    InsertText(startLoc, " if (!_rethrow) objc_exception_try_exit(&_stack);\n");
+    endLoc = endLoc.getLocWithOffset(-1);
+    InsertText(endLoc, " if (_rethrow) objc_exception_throw(_rethrow);\n");
+
+    // Set lastCurlyLoc
+    lastCurlyLoc = body->getLocEnd();
+
+    // Now check for any return/continue/go statements within the @try.
+    WarnAboutReturnGotoStmts(S->getTryBody());
+  } else { /* no finally clause - make sure we synthesize an implicit one */
+    buf = "{ /* implicit finally clause */\n";
+    buf += " if (!_rethrow) objc_exception_try_exit(&_stack);\n";
+    buf += " if (_rethrow) objc_exception_throw(_rethrow);\n";
+    buf += "}";
+    ReplaceText(lastCurlyLoc, 1, buf);
+    
+    // Now check for any return/continue/go statements within the @try.
+    // The implicit finally clause won't called if the @try contains any
+    // jump statements.
+    bool hasReturns = false;
+    HasReturnStmts(S->getTryBody(), hasReturns);
+    if (hasReturns)
+      RewriteTryReturnStmts(S->getTryBody());
+  }
+  // Now emit the final closing curly brace...
+  lastCurlyLoc = lastCurlyLoc.getLocWithOffset(1);
+  InsertText(lastCurlyLoc, " } /* @try scope end */\n");
+  return 0;
+}
+
+// This can't be done with ReplaceStmt(S, ThrowExpr), since
+// the throw expression is typically a message expression that's already
+// been rewritten! (which implies the SourceLocation's are invalid).
+Stmt *RewriteObjC::RewriteObjCThrowStmt(ObjCAtThrowStmt *S) {
+  // Get the start location and compute the semi location.
+  SourceLocation startLoc = S->getLocStart();
+  const char *startBuf = SM->getCharacterData(startLoc);
+
+  assert((*startBuf == '@') && "bogus @throw location");
+
+  std::string buf;
+  /* void objc_exception_throw(id) __attribute__((noreturn)); */
+  if (S->getThrowExpr())
+    buf = "objc_exception_throw(";
+  else // add an implicit argument
+    buf = "objc_exception_throw(_caught";
+
+  // handle "@  throw" correctly.
+  const char *wBuf = strchr(startBuf, 'w');
+  assert((*wBuf == 'w') && "@throw: can't find 'w'");
+  ReplaceText(startLoc, wBuf-startBuf+1, buf);
+
+  const char *semiBuf = strchr(startBuf, ';');
+  assert((*semiBuf == ';') && "@throw: can't find ';'");
+  SourceLocation semiLoc = startLoc.getLocWithOffset(semiBuf-startBuf);
+  ReplaceText(semiLoc, 1, ");");
+  return 0;
+}
+
+Stmt *RewriteObjC::RewriteAtEncode(ObjCEncodeExpr *Exp) {
+  // Create a new string expression.
+  QualType StrType = Context->getPointerType(Context->CharTy);
+  std::string StrEncoding;
+  Context->getObjCEncodingForType(Exp->getEncodedType(), StrEncoding);
+  Expr *Replacement = StringLiteral::Create(*Context, StrEncoding,
+                                            StringLiteral::Ascii, false,
+                                            StrType, SourceLocation());
+  ReplaceStmt(Exp, Replacement);
+
+  // Replace this subexpr in the parent.
+  // delete Exp; leak for now, see RewritePropertyOrImplicitSetter() usage for more info.
+  return Replacement;
+}
+
+Stmt *RewriteObjC::RewriteAtSelector(ObjCSelectorExpr *Exp) {
+  if (!SelGetUidFunctionDecl)
+    SynthSelGetUidFunctionDecl();
+  assert(SelGetUidFunctionDecl && "Can't find sel_registerName() decl");
+  // Create a call to sel_registerName("selName").
+  SmallVector<Expr*, 8> SelExprs;
+  QualType argType = Context->getPointerType(Context->CharTy);
+  SelExprs.push_back(StringLiteral::Create(*Context,
+                                           Exp->getSelector().getAsString(),
+                                           StringLiteral::Ascii, false,
+                                           argType, SourceLocation()));
+  CallExpr *SelExp = SynthesizeCallToFunctionDecl(SelGetUidFunctionDecl,
+                                                 &SelExprs[0], SelExprs.size());
+  ReplaceStmt(Exp, SelExp);
+  // delete Exp; leak for now, see RewritePropertyOrImplicitSetter() usage for more info.
+  return SelExp;
+}
+
+CallExpr *RewriteObjC::SynthesizeCallToFunctionDecl(
+  FunctionDecl *FD, Expr **args, unsigned nargs, SourceLocation StartLoc,
+                                                    SourceLocation EndLoc) {
+  // Get the type, we will need to reference it in a couple spots.
+  QualType msgSendType = FD->getType();
+
+  // Create a reference to the objc_msgSend() declaration.
+  DeclRefExpr *DRE = new (Context) DeclRefExpr(FD, false, msgSendType,
+                                               VK_LValue, SourceLocation());
+
+  // Now, we cast the reference to a pointer to the objc_msgSend type.
+  QualType pToFunc = Context->getPointerType(msgSendType);
+  ImplicitCastExpr *ICE = 
+    ImplicitCastExpr::Create(*Context, pToFunc, CK_FunctionToPointerDecay,
+                             DRE, 0, VK_RValue);
+
+  const FunctionType *FT = msgSendType->getAs<FunctionType>();
+
+  CallExpr *Exp =  
+    new (Context) CallExpr(*Context, ICE, llvm::makeArrayRef(args, nargs),
+                           FT->getCallResultType(*Context),
+                           VK_RValue, EndLoc);
+  return Exp;
+}
+
+static bool scanForProtocolRefs(const char *startBuf, const char *endBuf,
+                                const char *&startRef, const char *&endRef) {
+  while (startBuf < endBuf) {
+    if (*startBuf == '<')
+      startRef = startBuf; // mark the start.
+    if (*startBuf == '>') {
+      if (startRef && *startRef == '<') {
+        endRef = startBuf; // mark the end.
+        return true;
+      }
+      return false;
+    }
+    startBuf++;
+  }
+  return false;
+}
+
+static void scanToNextArgument(const char *&argRef) {
+  int angle = 0;
+  while (*argRef != ')' && (*argRef != ',' || angle > 0)) {
+    if (*argRef == '<')
+      angle++;
+    else if (*argRef == '>')
+      angle--;
+    argRef++;
+  }
+  assert(angle == 0 && "scanToNextArgument - bad protocol type syntax");
+}
+
+bool RewriteObjC::needToScanForQualifiers(QualType T) {
+  if (T->isObjCQualifiedIdType())
+    return true;
+  if (const PointerType *PT = T->getAs<PointerType>()) {
+    if (PT->getPointeeType()->isObjCQualifiedIdType())
+      return true;
+  }
+  if (T->isObjCObjectPointerType()) {
+    T = T->getPointeeType();
+    return T->isObjCQualifiedInterfaceType();
+  }
+  if (T->isArrayType()) {
+    QualType ElemTy = Context->getBaseElementType(T);
+    return needToScanForQualifiers(ElemTy);
+  }
+  return false;
+}
+
+void RewriteObjC::RewriteObjCQualifiedInterfaceTypes(Expr *E) {
+  QualType Type = E->getType();
+  if (needToScanForQualifiers(Type)) {
+    SourceLocation Loc, EndLoc;
+
+    if (const CStyleCastExpr *ECE = dyn_cast<CStyleCastExpr>(E)) {
+      Loc = ECE->getLParenLoc();
+      EndLoc = ECE->getRParenLoc();
+    } else {
+      Loc = E->getLocStart();
+      EndLoc = E->getLocEnd();
+    }
+    // This will defend against trying to rewrite synthesized expressions.
+    if (Loc.isInvalid() || EndLoc.isInvalid())
+      return;
+
+    const char *startBuf = SM->getCharacterData(Loc);
+    const char *endBuf = SM->getCharacterData(EndLoc);
+    const char *startRef = 0, *endRef = 0;
+    if (scanForProtocolRefs(startBuf, endBuf, startRef, endRef)) {
+      // Get the locations of the startRef, endRef.
+      SourceLocation LessLoc = Loc.getLocWithOffset(startRef-startBuf);
+      SourceLocation GreaterLoc = Loc.getLocWithOffset(endRef-startBuf+1);
+      // Comment out the protocol references.
+      InsertText(LessLoc, "/*");
+      InsertText(GreaterLoc, "*/");
+    }
+  }
+}
+
+void RewriteObjC::RewriteObjCQualifiedInterfaceTypes(Decl *Dcl) {
+  SourceLocation Loc;
+  QualType Type;
+  const FunctionProtoType *proto = 0;
+  if (VarDecl *VD = dyn_cast<VarDecl>(Dcl)) {
+    Loc = VD->getLocation();
+    Type = VD->getType();
+  }
+  else if (FunctionDecl *FD = dyn_cast<FunctionDecl>(Dcl)) {
+    Loc = FD->getLocation();
+    // Check for ObjC 'id' and class types that have been adorned with protocol
+    // information (id<p>, C<p>*). The protocol references need to be rewritten!
+    const FunctionType *funcType = FD->getType()->getAs<FunctionType>();
+    assert(funcType && "missing function type");
+    proto = dyn_cast<FunctionProtoType>(funcType);
+    if (!proto)
+      return;
+    Type = proto->getResultType();
+  }
+  else if (FieldDecl *FD = dyn_cast<FieldDecl>(Dcl)) {
+    Loc = FD->getLocation();
+    Type = FD->getType();
+  }
+  else
+    return;
+
+  if (needToScanForQualifiers(Type)) {
+    // Since types are unique, we need to scan the buffer.
+
+    const char *endBuf = SM->getCharacterData(Loc);
+    const char *startBuf = endBuf;
+    while (*startBuf != ';' && *startBuf != '<' && startBuf != MainFileStart)
+      startBuf--; // scan backward (from the decl location) for return type.
+    const char *startRef = 0, *endRef = 0;
+    if (scanForProtocolRefs(startBuf, endBuf, startRef, endRef)) {
+      // Get the locations of the startRef, endRef.
+      SourceLocation LessLoc = Loc.getLocWithOffset(startRef-endBuf);
+      SourceLocation GreaterLoc = Loc.getLocWithOffset(endRef-endBuf+1);
+      // Comment out the protocol references.
+      InsertText(LessLoc, "/*");
+      InsertText(GreaterLoc, "*/");
+    }
+  }
+  if (!proto)
+      return; // most likely, was a variable
+  // Now check arguments.
+  const char *startBuf = SM->getCharacterData(Loc);
+  const char *startFuncBuf = startBuf;
+  for (unsigned i = 0; i < proto->getNumArgs(); i++) {
+    if (needToScanForQualifiers(proto->getArgType(i))) {
+      // Since types are unique, we need to scan the buffer.
+
+      const char *endBuf = startBuf;
+      // scan forward (from the decl location) for argument types.
+      scanToNextArgument(endBuf);
+      const char *startRef = 0, *endRef = 0;
+      if (scanForProtocolRefs(startBuf, endBuf, startRef, endRef)) {
+        // Get the locations of the startRef, endRef.
+        SourceLocation LessLoc =
+          Loc.getLocWithOffset(startRef-startFuncBuf);
+        SourceLocation GreaterLoc =
+          Loc.getLocWithOffset(endRef-startFuncBuf+1);
+        // Comment out the protocol references.
+        InsertText(LessLoc, "/*");
+        InsertText(GreaterLoc, "*/");
+      }
+      startBuf = ++endBuf;
+    }
+    else {
+      // If the function name is derived from a macro expansion, then the
+      // argument buffer will not follow the name. Need to speak with Chris.
+      while (*startBuf && *startBuf != ')' && *startBuf != ',')
+        startBuf++; // scan forward (from the decl location) for argument types.
+      startBuf++;
+    }
+  }
+}
+
+void RewriteObjC::RewriteTypeOfDecl(VarDecl *ND) {
+  QualType QT = ND->getType();
+  const Type* TypePtr = QT->getAs<Type>();
+  if (!isa<TypeOfExprType>(TypePtr))
+    return;
+  while (isa<TypeOfExprType>(TypePtr)) {
+    const TypeOfExprType *TypeOfExprTypePtr = cast<TypeOfExprType>(TypePtr);
+    QT = TypeOfExprTypePtr->getUnderlyingExpr()->getType();
+    TypePtr = QT->getAs<Type>();
+  }
+  // FIXME. This will not work for multiple declarators; as in:
+  // __typeof__(a) b,c,d;
+  std::string TypeAsString(QT.getAsString(Context->getPrintingPolicy()));
+  SourceLocation DeclLoc = ND->getTypeSpecStartLoc();
+  const char *startBuf = SM->getCharacterData(DeclLoc);
+  if (ND->getInit()) {
+    std::string Name(ND->getNameAsString());
+    TypeAsString += " " + Name + " = ";
+    Expr *E = ND->getInit();
+    SourceLocation startLoc;
+    if (const CStyleCastExpr *ECE = dyn_cast<CStyleCastExpr>(E))
+      startLoc = ECE->getLParenLoc();
+    else
+      startLoc = E->getLocStart();
+    startLoc = SM->getExpansionLoc(startLoc);
+    const char *endBuf = SM->getCharacterData(startLoc);
+    ReplaceText(DeclLoc, endBuf-startBuf-1, TypeAsString);
+  }
+  else {
+    SourceLocation X = ND->getLocEnd();
+    X = SM->getExpansionLoc(X);
+    const char *endBuf = SM->getCharacterData(X);
+    ReplaceText(DeclLoc, endBuf-startBuf-1, TypeAsString);
+  }
+}
+
+// SynthSelGetUidFunctionDecl - SEL sel_registerName(const char *str);
+void RewriteObjC::SynthSelGetUidFunctionDecl() {
+  IdentifierInfo *SelGetUidIdent = &Context->Idents.get("sel_registerName");
+  SmallVector<QualType, 16> ArgTys;
+  ArgTys.push_back(Context->getPointerType(Context->CharTy.withConst()));
+  QualType getFuncType =
+    getSimpleFunctionType(Context->getObjCSelType(), &ArgTys[0], ArgTys.size());
+  SelGetUidFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
+                                           SourceLocation(),
+                                           SourceLocation(),
+                                           SelGetUidIdent, getFuncType, 0,
+                                           SC_Extern,
+                                           SC_None, false);
+}
+
+void RewriteObjC::RewriteFunctionDecl(FunctionDecl *FD) {
+  // declared in <objc/objc.h>
+  if (FD->getIdentifier() &&
+      FD->getName() == "sel_registerName") {
+    SelGetUidFunctionDecl = FD;
+    return;
+  }
+  RewriteObjCQualifiedInterfaceTypes(FD);
+}
+
+void RewriteObjC::RewriteBlockPointerType(std::string& Str, QualType Type) {
+  std::string TypeString(Type.getAsString(Context->getPrintingPolicy()));
+  const char *argPtr = TypeString.c_str();
+  if (!strchr(argPtr, '^')) {
+    Str += TypeString;
+    return;
+  }
+  while (*argPtr) {
+    Str += (*argPtr == '^' ? '*' : *argPtr);
+    argPtr++;
+  }
+}
+
+// FIXME. Consolidate this routine with RewriteBlockPointerType.
+void RewriteObjC::RewriteBlockPointerTypeVariable(std::string& Str,
+                                                  ValueDecl *VD) {
+  QualType Type = VD->getType();
+  std::string TypeString(Type.getAsString(Context->getPrintingPolicy()));
+  const char *argPtr = TypeString.c_str();
+  int paren = 0;
+  while (*argPtr) {
+    switch (*argPtr) {
+      case '(':
+        Str += *argPtr;
+        paren++;
+        break;
+      case ')':
+        Str += *argPtr;
+        paren--;
+        break;
+      case '^':
+        Str += '*';
+        if (paren == 1)
+          Str += VD->getNameAsString();
+        break;
+      default:
+        Str += *argPtr;
+        break;
+    }
+    argPtr++;
+  }
+}
+
+
+void RewriteObjC::RewriteBlockLiteralFunctionDecl(FunctionDecl *FD) {
+  SourceLocation FunLocStart = FD->getTypeSpecStartLoc();
+  const FunctionType *funcType = FD->getType()->getAs<FunctionType>();
+  const FunctionProtoType *proto = dyn_cast<FunctionProtoType>(funcType);
+  if (!proto)
+    return;
+  QualType Type = proto->getResultType();
+  std::string FdStr = Type.getAsString(Context->getPrintingPolicy());
+  FdStr += " ";
+  FdStr += FD->getName();
+  FdStr +=  "(";
+  unsigned numArgs = proto->getNumArgs();
+  for (unsigned i = 0; i < numArgs; i++) {
+    QualType ArgType = proto->getArgType(i);
+    RewriteBlockPointerType(FdStr, ArgType);
+    if (i+1 < numArgs)
+      FdStr += ", ";
+  }
+  FdStr +=  ");\n";
+  InsertText(FunLocStart, FdStr);
+  CurFunctionDeclToDeclareForBlock = 0;
+}
+
+// SynthSuperContructorFunctionDecl - id objc_super(id obj, id super);
+void RewriteObjC::SynthSuperContructorFunctionDecl() {
+  if (SuperContructorFunctionDecl)
+    return;
+  IdentifierInfo *msgSendIdent = &Context->Idents.get("__rw_objc_super");
+  SmallVector<QualType, 16> ArgTys;
+  QualType argT = Context->getObjCIdType();
+  assert(!argT.isNull() && "Can't find 'id' type");
+  ArgTys.push_back(argT);
+  ArgTys.push_back(argT);
+  QualType msgSendType = getSimpleFunctionType(Context->getObjCIdType(),
+                                               &ArgTys[0], ArgTys.size());
+  SuperContructorFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
+                                         SourceLocation(),
+                                         SourceLocation(),
+                                         msgSendIdent, msgSendType, 0,
+                                         SC_Extern,
+                                         SC_None, false);
+}
+
+// SynthMsgSendFunctionDecl - id objc_msgSend(id self, SEL op, ...);
+void RewriteObjC::SynthMsgSendFunctionDecl() {
+  IdentifierInfo *msgSendIdent = &Context->Idents.get("objc_msgSend");
+  SmallVector<QualType, 16> ArgTys;
+  QualType argT = Context->getObjCIdType();
+  assert(!argT.isNull() && "Can't find 'id' type");
+  ArgTys.push_back(argT);
+  argT = Context->getObjCSelType();
+  assert(!argT.isNull() && "Can't find 'SEL' type");
+  ArgTys.push_back(argT);
+  QualType msgSendType = getSimpleFunctionType(Context->getObjCIdType(),
+                                               &ArgTys[0], ArgTys.size(),
+                                               true /*isVariadic*/);
+  MsgSendFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
+                                         SourceLocation(),
+                                         SourceLocation(),
+                                         msgSendIdent, msgSendType, 0,
+                                         SC_Extern,
+                                         SC_None, false);
+}
+
+// SynthMsgSendSuperFunctionDecl - id objc_msgSendSuper(struct objc_super *, SEL op, ...);
+void RewriteObjC::SynthMsgSendSuperFunctionDecl() {
+  IdentifierInfo *msgSendIdent = &Context->Idents.get("objc_msgSendSuper");
+  SmallVector<QualType, 16> ArgTys;
+  RecordDecl *RD = RecordDecl::Create(*Context, TTK_Struct, TUDecl,
+                                      SourceLocation(), SourceLocation(),
+                                      &Context->Idents.get("objc_super"));
+  QualType argT = Context->getPointerType(Context->getTagDeclType(RD));
+  assert(!argT.isNull() && "Can't build 'struct objc_super *' type");
+  ArgTys.push_back(argT);
+  argT = Context->getObjCSelType();
+  assert(!argT.isNull() && "Can't find 'SEL' type");
+  ArgTys.push_back(argT);
+  QualType msgSendType = getSimpleFunctionType(Context->getObjCIdType(),
+                                               &ArgTys[0], ArgTys.size(),
+                                               true /*isVariadic*/);
+  MsgSendSuperFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
+                                              SourceLocation(),
+                                              SourceLocation(),
+                                              msgSendIdent, msgSendType, 0,
+                                              SC_Extern,
+                                              SC_None, false);
+}
+
+// SynthMsgSendStretFunctionDecl - id objc_msgSend_stret(id self, SEL op, ...);
+void RewriteObjC::SynthMsgSendStretFunctionDecl() {
+  IdentifierInfo *msgSendIdent = &Context->Idents.get("objc_msgSend_stret");
+  SmallVector<QualType, 16> ArgTys;
+  QualType argT = Context->getObjCIdType();
+  assert(!argT.isNull() && "Can't find 'id' type");
+  ArgTys.push_back(argT);
+  argT = Context->getObjCSelType();
+  assert(!argT.isNull() && "Can't find 'SEL' type");
+  ArgTys.push_back(argT);
+  QualType msgSendType = getSimpleFunctionType(Context->getObjCIdType(),
+                                               &ArgTys[0], ArgTys.size(),
+                                               true /*isVariadic*/);
+  MsgSendStretFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
+                                         SourceLocation(),
+                                         SourceLocation(),
+                                         msgSendIdent, msgSendType, 0,
+                                         SC_Extern,
+                                         SC_None, false);
+}
+
+// SynthMsgSendSuperStretFunctionDecl -
+// id objc_msgSendSuper_stret(struct objc_super *, SEL op, ...);
+void RewriteObjC::SynthMsgSendSuperStretFunctionDecl() {
+  IdentifierInfo *msgSendIdent =
+    &Context->Idents.get("objc_msgSendSuper_stret");
+  SmallVector<QualType, 16> ArgTys;
+  RecordDecl *RD = RecordDecl::Create(*Context, TTK_Struct, TUDecl,
+                                      SourceLocation(), SourceLocation(),
+                                      &Context->Idents.get("objc_super"));
+  QualType argT = Context->getPointerType(Context->getTagDeclType(RD));
+  assert(!argT.isNull() && "Can't build 'struct objc_super *' type");
+  ArgTys.push_back(argT);
+  argT = Context->getObjCSelType();
+  assert(!argT.isNull() && "Can't find 'SEL' type");
+  ArgTys.push_back(argT);
+  QualType msgSendType = getSimpleFunctionType(Context->getObjCIdType(),
+                                               &ArgTys[0], ArgTys.size(),
+                                               true /*isVariadic*/);
+  MsgSendSuperStretFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
+                                                       SourceLocation(),
+                                                       SourceLocation(),
+                                              msgSendIdent, msgSendType, 0,
+                                              SC_Extern,
+                                              SC_None, false);
+}
+
+// SynthMsgSendFpretFunctionDecl - double objc_msgSend_fpret(id self, SEL op, ...);
+void RewriteObjC::SynthMsgSendFpretFunctionDecl() {
+  IdentifierInfo *msgSendIdent = &Context->Idents.get("objc_msgSend_fpret");
+  SmallVector<QualType, 16> ArgTys;
+  QualType argT = Context->getObjCIdType();
+  assert(!argT.isNull() && "Can't find 'id' type");
+  ArgTys.push_back(argT);
+  argT = Context->getObjCSelType();
+  assert(!argT.isNull() && "Can't find 'SEL' type");
+  ArgTys.push_back(argT);
+  QualType msgSendType = getSimpleFunctionType(Context->DoubleTy,
+                                               &ArgTys[0], ArgTys.size(),
+                                               true /*isVariadic*/);
+  MsgSendFpretFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
+                                              SourceLocation(),
+                                              SourceLocation(),
+                                              msgSendIdent, msgSendType, 0,
+                                              SC_Extern,
+                                              SC_None, false);
+}
+
+// SynthGetClassFunctionDecl - id objc_getClass(const char *name);
+void RewriteObjC::SynthGetClassFunctionDecl() {
+  IdentifierInfo *getClassIdent = &Context->Idents.get("objc_getClass");
+  SmallVector<QualType, 16> ArgTys;
+  ArgTys.push_back(Context->getPointerType(Context->CharTy.withConst()));
+  QualType getClassType = getSimpleFunctionType(Context->getObjCIdType(),
+                                                &ArgTys[0], ArgTys.size());
+  GetClassFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
+                                          SourceLocation(),
+                                          SourceLocation(),
+                                          getClassIdent, getClassType, 0,
+                                          SC_Extern,
+                                          SC_None, false);
+}
+
+// SynthGetSuperClassFunctionDecl - Class class_getSuperclass(Class cls);
+void RewriteObjC::SynthGetSuperClassFunctionDecl() {
+  IdentifierInfo *getSuperClassIdent = 
+    &Context->Idents.get("class_getSuperclass");
+  SmallVector<QualType, 16> ArgTys;
+  ArgTys.push_back(Context->getObjCClassType());
+  QualType getClassType = getSimpleFunctionType(Context->getObjCClassType(),
+                                                &ArgTys[0], ArgTys.size());
+  GetSuperClassFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
+                                                   SourceLocation(),
+                                                   SourceLocation(),
+                                                   getSuperClassIdent,
+                                                   getClassType, 0,
+                                                   SC_Extern,
+                                                   SC_None,
+                                                   false);
+}
+
+// SynthGetMetaClassFunctionDecl - id objc_getMetaClass(const char *name);
+void RewriteObjC::SynthGetMetaClassFunctionDecl() {
+  IdentifierInfo *getClassIdent = &Context->Idents.get("objc_getMetaClass");
+  SmallVector<QualType, 16> ArgTys;
+  ArgTys.push_back(Context->getPointerType(Context->CharTy.withConst()));
+  QualType getClassType = getSimpleFunctionType(Context->getObjCIdType(),
+                                                &ArgTys[0], ArgTys.size());
+  GetMetaClassFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
+                                              SourceLocation(),
+                                              SourceLocation(),
+                                              getClassIdent, getClassType, 0,
+                                              SC_Extern,
+                                              SC_None, false);
+}
+
+Stmt *RewriteObjC::RewriteObjCStringLiteral(ObjCStringLiteral *Exp) {
+  QualType strType = getConstantStringStructType();
+
+  std::string S = "__NSConstantStringImpl_";
+
+  std::string tmpName = InFileName;
+  unsigned i;
+  for (i=0; i < tmpName.length(); i++) {
+    char c = tmpName.at(i);
+    // replace any non alphanumeric characters with '_'.
+    if (!isalpha(c) && (c < '0' || c > '9'))
+      tmpName[i] = '_';
+  }
+  S += tmpName;
+  S += "_";
+  S += utostr(NumObjCStringLiterals++);
+
+  Preamble += "static __NSConstantStringImpl " + S;
+  Preamble += " __attribute__ ((section (\"__DATA, __cfstring\"))) = {__CFConstantStringClassReference,";
+  Preamble += "0x000007c8,"; // utf8_str
+  // The pretty printer for StringLiteral handles escape characters properly.
+  std::string prettyBufS;
+  llvm::raw_string_ostream prettyBuf(prettyBufS);
+  Exp->getString()->printPretty(prettyBuf, 0, PrintingPolicy(LangOpts));
+  Preamble += prettyBuf.str();
+  Preamble += ",";
+  Preamble += utostr(Exp->getString()->getByteLength()) + "};\n";
+
+  VarDecl *NewVD = VarDecl::Create(*Context, TUDecl, SourceLocation(),
+                                   SourceLocation(), &Context->Idents.get(S),
+                                   strType, 0, SC_Static, SC_None);
+  DeclRefExpr *DRE = new (Context) DeclRefExpr(NewVD, false, strType, VK_LValue,
+                                               SourceLocation());
+  Expr *Unop = new (Context) UnaryOperator(DRE, UO_AddrOf,
+                                 Context->getPointerType(DRE->getType()),
+                                           VK_RValue, OK_Ordinary,
+                                           SourceLocation());
+  // cast to NSConstantString *
+  CastExpr *cast = NoTypeInfoCStyleCastExpr(Context, Exp->getType(),
+                                            CK_CPointerToObjCPointerCast, Unop);
+  ReplaceStmt(Exp, cast);
+  // delete Exp; leak for now, see RewritePropertyOrImplicitSetter() usage for more info.
+  return cast;
+}
+
+// struct objc_super { struct objc_object *receiver; struct objc_class *super; };
+QualType RewriteObjC::getSuperStructType() {
+  if (!SuperStructDecl) {
+    SuperStructDecl = RecordDecl::Create(*Context, TTK_Struct, TUDecl,
+                                         SourceLocation(), SourceLocation(),
+                                         &Context->Idents.get("objc_super"));
+    QualType FieldTypes[2];
+
+    // struct objc_object *receiver;
+    FieldTypes[0] = Context->getObjCIdType();
+    // struct objc_class *super;
+    FieldTypes[1] = Context->getObjCClassType();
+
+    // Create fields
+    for (unsigned i = 0; i < 2; ++i) {
+      SuperStructDecl->addDecl(FieldDecl::Create(*Context, SuperStructDecl,
+                                                 SourceLocation(),
+                                                 SourceLocation(), 0,
+                                                 FieldTypes[i], 0,
+                                                 /*BitWidth=*/0,
+                                                 /*Mutable=*/false,
+                                                 ICIS_NoInit));
+    }
+
+    SuperStructDecl->completeDefinition();
+  }
+  return Context->getTagDeclType(SuperStructDecl);
+}
+
+QualType RewriteObjC::getConstantStringStructType() {
+  if (!ConstantStringDecl) {
+    ConstantStringDecl = RecordDecl::Create(*Context, TTK_Struct, TUDecl,
+                                            SourceLocation(), SourceLocation(),
+                         &Context->Idents.get("__NSConstantStringImpl"));
+    QualType FieldTypes[4];
+
+    // struct objc_object *receiver;
+    FieldTypes[0] = Context->getObjCIdType();
+    // int flags;
+    FieldTypes[1] = Context->IntTy;
+    // char *str;
+    FieldTypes[2] = Context->getPointerType(Context->CharTy);
+    // long length;
+    FieldTypes[3] = Context->LongTy;
+
+    // Create fields
+    for (unsigned i = 0; i < 4; ++i) {
+      ConstantStringDecl->addDecl(FieldDecl::Create(*Context,
+                                                    ConstantStringDecl,
+                                                    SourceLocation(),
+                                                    SourceLocation(), 0,
+                                                    FieldTypes[i], 0,
+                                                    /*BitWidth=*/0,
+                                                    /*Mutable=*/true,
+                                                    ICIS_NoInit));
+    }
+
+    ConstantStringDecl->completeDefinition();
+  }
+  return Context->getTagDeclType(ConstantStringDecl);
+}
+
+CallExpr *RewriteObjC::SynthMsgSendStretCallExpr(FunctionDecl *MsgSendStretFlavor,
+                                                QualType msgSendType, 
+                                                QualType returnType, 
+                                                SmallVectorImpl<QualType> &ArgTypes,
+                                                SmallVectorImpl<Expr*> &MsgExprs,
+                                                ObjCMethodDecl *Method) {
+  // Create a reference to the objc_msgSend_stret() declaration.
+  DeclRefExpr *STDRE = new (Context) DeclRefExpr(MsgSendStretFlavor,
+                                                 false, msgSendType,
+                                                 VK_LValue, SourceLocation());
+  // Need to cast objc_msgSend_stret to "void *" (see above comment).
+  CastExpr *cast = NoTypeInfoCStyleCastExpr(Context,
+                                  Context->getPointerType(Context->VoidTy),
+                                  CK_BitCast, STDRE);
+  // Now do the "normal" pointer to function cast.
+  QualType castType = getSimpleFunctionType(returnType, &ArgTypes[0], ArgTypes.size(),
+                                            Method ? Method->isVariadic() : false);
+  castType = Context->getPointerType(castType);
+  cast = NoTypeInfoCStyleCastExpr(Context, castType, CK_BitCast,
+                                            cast);
+  
+  // Don't forget the parens to enforce the proper binding.
+  ParenExpr *PE = new (Context) ParenExpr(SourceLocation(), SourceLocation(), cast);
+  
+  const FunctionType *FT = msgSendType->getAs<FunctionType>();
+  CallExpr *STCE = new (Context) CallExpr(*Context, PE, MsgExprs,
+                                          FT->getResultType(), VK_RValue,
+                                          SourceLocation());
+  return STCE;
+  
+}
+
+
+Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
+                                    SourceLocation StartLoc,
+                                    SourceLocation EndLoc) {
+  if (!SelGetUidFunctionDecl)
+    SynthSelGetUidFunctionDecl();
+  if (!MsgSendFunctionDecl)
+    SynthMsgSendFunctionDecl();
+  if (!MsgSendSuperFunctionDecl)
+    SynthMsgSendSuperFunctionDecl();
+  if (!MsgSendStretFunctionDecl)
+    SynthMsgSendStretFunctionDecl();
+  if (!MsgSendSuperStretFunctionDecl)
+    SynthMsgSendSuperStretFunctionDecl();
+  if (!MsgSendFpretFunctionDecl)
+    SynthMsgSendFpretFunctionDecl();
+  if (!GetClassFunctionDecl)
+    SynthGetClassFunctionDecl();
+  if (!GetSuperClassFunctionDecl)
+    SynthGetSuperClassFunctionDecl();
+  if (!GetMetaClassFunctionDecl)
+    SynthGetMetaClassFunctionDecl();
+
+  // default to objc_msgSend().
+  FunctionDecl *MsgSendFlavor = MsgSendFunctionDecl;
+  // May need to use objc_msgSend_stret() as well.
+  FunctionDecl *MsgSendStretFlavor = 0;
+  if (ObjCMethodDecl *mDecl = Exp->getMethodDecl()) {
+    QualType resultType = mDecl->getResultType();
+    if (resultType->isRecordType())
+      MsgSendStretFlavor = MsgSendStretFunctionDecl;
+    else if (resultType->isRealFloatingType())
+      MsgSendFlavor = MsgSendFpretFunctionDecl;
+  }
+
+  // Synthesize a call to objc_msgSend().
+  SmallVector<Expr*, 8> MsgExprs;
+  switch (Exp->getReceiverKind()) {
+  case ObjCMessageExpr::SuperClass: {
+    MsgSendFlavor = MsgSendSuperFunctionDecl;
+    if (MsgSendStretFlavor)
+      MsgSendStretFlavor = MsgSendSuperStretFunctionDecl;
+    assert(MsgSendFlavor && "MsgSendFlavor is NULL!");
+
+    ObjCInterfaceDecl *ClassDecl = CurMethodDef->getClassInterface();
+
+    SmallVector<Expr*, 4> InitExprs;
+
+    // set the receiver to self, the first argument to all methods.
+    InitExprs.push_back(
+      NoTypeInfoCStyleCastExpr(Context, Context->getObjCIdType(),
+                               CK_BitCast,
+                   new (Context) DeclRefExpr(CurMethodDef->getSelfDecl(),
+                                             false,
+                                             Context->getObjCIdType(),
+                                             VK_RValue,
+                                             SourceLocation()))
+                        ); // set the 'receiver'.
+
+    // (id)class_getSuperclass((Class)objc_getClass("CurrentClass"))
+    SmallVector<Expr*, 8> ClsExprs;
+    QualType argType = Context->getPointerType(Context->CharTy);
+    ClsExprs.push_back(StringLiteral::Create(*Context,
+                                   ClassDecl->getIdentifier()->getName(),
+                                   StringLiteral::Ascii, false,
+                                   argType, SourceLocation()));
+    CallExpr *Cls = SynthesizeCallToFunctionDecl(GetMetaClassFunctionDecl,
+                                                 &ClsExprs[0],
+                                                 ClsExprs.size(),
+                                                 StartLoc,
+                                                 EndLoc);
+    // (Class)objc_getClass("CurrentClass")
+    CastExpr *ArgExpr = NoTypeInfoCStyleCastExpr(Context,
+                                             Context->getObjCClassType(),
+                                             CK_BitCast, Cls);
+    ClsExprs.clear();
+    ClsExprs.push_back(ArgExpr);
+    Cls = SynthesizeCallToFunctionDecl(GetSuperClassFunctionDecl,
+                                       &ClsExprs[0], ClsExprs.size(),
+                                       StartLoc, EndLoc);
+    
+    // (id)class_getSuperclass((Class)objc_getClass("CurrentClass"))
+    // To turn off a warning, type-cast to 'id'
+    InitExprs.push_back( // set 'super class', using class_getSuperclass().
+                        NoTypeInfoCStyleCastExpr(Context,
+                                                 Context->getObjCIdType(),
+                                                 CK_BitCast, Cls));
+    // struct objc_super
+    QualType superType = getSuperStructType();
+    Expr *SuperRep;
+
+    if (LangOpts.MicrosoftExt) {
+      SynthSuperContructorFunctionDecl();
+      // Simulate a contructor call...
+      DeclRefExpr *DRE = new (Context) DeclRefExpr(SuperContructorFunctionDecl,
+                                                   false, superType, VK_LValue,
+                                                   SourceLocation());
+      SuperRep = new (Context) CallExpr(*Context, DRE, InitExprs,
+                                        superType, VK_LValue,
+                                        SourceLocation());
+      // The code for super is a little tricky to prevent collision with
+      // the structure definition in the header. The rewriter has it's own
+      // internal definition (__rw_objc_super) that is uses. This is why
+      // we need the cast below. For example:
+      // (struct objc_super *)&__rw_objc_super((id)self, (id)objc_getClass("SUPER"))
+      //
+      SuperRep = new (Context) UnaryOperator(SuperRep, UO_AddrOf,
+                               Context->getPointerType(SuperRep->getType()),
+                                             VK_RValue, OK_Ordinary,
+                                             SourceLocation());
+      SuperRep = NoTypeInfoCStyleCastExpr(Context,
+                                          Context->getPointerType(superType),
+                                          CK_BitCast, SuperRep);
+    } else {
+      // (struct objc_super) { <exprs from above> }
+      InitListExpr *ILE =
+        new (Context) InitListExpr(*Context, SourceLocation(), InitExprs,
+                                   SourceLocation());
+      TypeSourceInfo *superTInfo
+        = Context->getTrivialTypeSourceInfo(superType);
+      SuperRep = new (Context) CompoundLiteralExpr(SourceLocation(), superTInfo,
+                                                   superType, VK_LValue,
+                                                   ILE, false);
+      // struct objc_super *
+      SuperRep = new (Context) UnaryOperator(SuperRep, UO_AddrOf,
+                               Context->getPointerType(SuperRep->getType()),
+                                             VK_RValue, OK_Ordinary,
+                                             SourceLocation());
+    }
+    MsgExprs.push_back(SuperRep);
+    break;
+  }
+
+  case ObjCMessageExpr::Class: {
+    SmallVector<Expr*, 8> ClsExprs;
+    QualType argType = Context->getPointerType(Context->CharTy);
+    ObjCInterfaceDecl *Class
+      = Exp->getClassReceiver()->getAs<ObjCObjectType>()->getInterface();
+    IdentifierInfo *clsName = Class->getIdentifier();
+    ClsExprs.push_back(StringLiteral::Create(*Context,
+                                             clsName->getName(),
+                                             StringLiteral::Ascii, false,
+                                             argType, SourceLocation()));
+    CallExpr *Cls = SynthesizeCallToFunctionDecl(GetClassFunctionDecl,
+                                                 &ClsExprs[0],
+                                                 ClsExprs.size(), 
+                                                 StartLoc, EndLoc);
+    MsgExprs.push_back(Cls);
+    break;
+  }
+
+  case ObjCMessageExpr::SuperInstance:{
+    MsgSendFlavor = MsgSendSuperFunctionDecl;
+    if (MsgSendStretFlavor)
+      MsgSendStretFlavor = MsgSendSuperStretFunctionDecl;
+    assert(MsgSendFlavor && "MsgSendFlavor is NULL!");
+    ObjCInterfaceDecl *ClassDecl = CurMethodDef->getClassInterface();
+    SmallVector<Expr*, 4> InitExprs;
+
+    InitExprs.push_back(
+      NoTypeInfoCStyleCastExpr(Context, Context->getObjCIdType(),
+                               CK_BitCast,
+                   new (Context) DeclRefExpr(CurMethodDef->getSelfDecl(),
+                                             false,
+                                             Context->getObjCIdType(),
+                                             VK_RValue, SourceLocation()))
+                        ); // set the 'receiver'.
+    
+    // (id)class_getSuperclass((Class)objc_getClass("CurrentClass"))
+    SmallVector<Expr*, 8> ClsExprs;
+    QualType argType = Context->getPointerType(Context->CharTy);
+    ClsExprs.push_back(StringLiteral::Create(*Context,
+                                   ClassDecl->getIdentifier()->getName(),
+                                   StringLiteral::Ascii, false, argType,
+                                   SourceLocation()));
+    CallExpr *Cls = SynthesizeCallToFunctionDecl(GetClassFunctionDecl,
+                                                 &ClsExprs[0],
+                                                 ClsExprs.size(), 
+                                                 StartLoc, EndLoc);
+    // (Class)objc_getClass("CurrentClass")
+    CastExpr *ArgExpr = NoTypeInfoCStyleCastExpr(Context,
+                                                 Context->getObjCClassType(),
+                                                 CK_BitCast, Cls);
+    ClsExprs.clear();
+    ClsExprs.push_back(ArgExpr);
+    Cls = SynthesizeCallToFunctionDecl(GetSuperClassFunctionDecl,
+                                       &ClsExprs[0], ClsExprs.size(),
+                                       StartLoc, EndLoc);
+    
+    // (id)class_getSuperclass((Class)objc_getClass("CurrentClass"))
+    // To turn off a warning, type-cast to 'id'
+    InitExprs.push_back(
+      // set 'super class', using class_getSuperclass().
+      NoTypeInfoCStyleCastExpr(Context, Context->getObjCIdType(),
+                               CK_BitCast, Cls));
+    // struct objc_super
+    QualType superType = getSuperStructType();
+    Expr *SuperRep;
+
+    if (LangOpts.MicrosoftExt) {
+      SynthSuperContructorFunctionDecl();
+      // Simulate a contructor call...
+      DeclRefExpr *DRE = new (Context) DeclRefExpr(SuperContructorFunctionDecl,
+                                                   false, superType, VK_LValue,
+                                                   SourceLocation());
+      SuperRep = new (Context) CallExpr(*Context, DRE, InitExprs,
+                                        superType, VK_LValue, SourceLocation());
+      // The code for super is a little tricky to prevent collision with
+      // the structure definition in the header. The rewriter has it's own
+      // internal definition (__rw_objc_super) that is uses. This is why
+      // we need the cast below. For example:
+      // (struct objc_super *)&__rw_objc_super((id)self, (id)objc_getClass("SUPER"))
+      //
+      SuperRep = new (Context) UnaryOperator(SuperRep, UO_AddrOf,
+                               Context->getPointerType(SuperRep->getType()),
+                               VK_RValue, OK_Ordinary,
+                               SourceLocation());
+      SuperRep = NoTypeInfoCStyleCastExpr(Context,
+                               Context->getPointerType(superType),
+                               CK_BitCast, SuperRep);
+    } else {
+      // (struct objc_super) { <exprs from above> }
+      InitListExpr *ILE =
+        new (Context) InitListExpr(*Context, SourceLocation(), InitExprs,
+                                   SourceLocation());
+      TypeSourceInfo *superTInfo
+        = Context->getTrivialTypeSourceInfo(superType);
+      SuperRep = new (Context) CompoundLiteralExpr(SourceLocation(), superTInfo,
+                                                   superType, VK_RValue, ILE,
+                                                   false);
+    }
+    MsgExprs.push_back(SuperRep);
+    break;
+  }
+
+  case ObjCMessageExpr::Instance: {
+    // Remove all type-casts because it may contain objc-style types; e.g.
+    // Foo<Proto> *.
+    Expr *recExpr = Exp->getInstanceReceiver();
+    while (CStyleCastExpr *CE = dyn_cast<CStyleCastExpr>(recExpr))
+      recExpr = CE->getSubExpr();
+    CastKind CK = recExpr->getType()->isObjCObjectPointerType()
+                    ? CK_BitCast : recExpr->getType()->isBlockPointerType()
+                                     ? CK_BlockPointerToObjCPointerCast
+                                     : CK_CPointerToObjCPointerCast;
+
+    recExpr = NoTypeInfoCStyleCastExpr(Context, Context->getObjCIdType(),
+                                       CK, recExpr);
+    MsgExprs.push_back(recExpr);
+    break;
+  }
+  }
+
+  // Create a call to sel_registerName("selName"), it will be the 2nd argument.
+  SmallVector<Expr*, 8> SelExprs;
+  QualType argType = Context->getPointerType(Context->CharTy);
+  SelExprs.push_back(StringLiteral::Create(*Context,
+                                       Exp->getSelector().getAsString(),
+                                       StringLiteral::Ascii, false,
+                                       argType, SourceLocation()));
+  CallExpr *SelExp = SynthesizeCallToFunctionDecl(SelGetUidFunctionDecl,
+                                                 &SelExprs[0], SelExprs.size(),
+                                                  StartLoc,
+                                                  EndLoc);
+  MsgExprs.push_back(SelExp);
+
+  // Now push any user supplied arguments.
+  for (unsigned i = 0; i < Exp->getNumArgs(); i++) {
+    Expr *userExpr = Exp->getArg(i);
+    // Make all implicit casts explicit...ICE comes in handy:-)
+    if (ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(userExpr)) {
+      // Reuse the ICE type, it is exactly what the doctor ordered.
+      QualType type = ICE->getType();
+      if (needToScanForQualifiers(type))
+        type = Context->getObjCIdType();
+      // Make sure we convert "type (^)(...)" to "type (*)(...)".
+      (void)convertBlockPointerToFunctionPointer(type);
+      const Expr *SubExpr = ICE->IgnoreParenImpCasts();
+      CastKind CK;
+      if (SubExpr->getType()->isIntegralType(*Context) && 
+          type->isBooleanType()) {
+        CK = CK_IntegralToBoolean;
+      } else if (type->isObjCObjectPointerType()) {
+        if (SubExpr->getType()->isBlockPointerType()) {
+          CK = CK_BlockPointerToObjCPointerCast;
+        } else if (SubExpr->getType()->isPointerType()) {
+          CK = CK_CPointerToObjCPointerCast;
+        } else {
+          CK = CK_BitCast;
+        }
+      } else {
+        CK = CK_BitCast;
+      }
+
+      userExpr = NoTypeInfoCStyleCastExpr(Context, type, CK, userExpr);
+    }
+    // Make id<P...> cast into an 'id' cast.
+    else if (CStyleCastExpr *CE = dyn_cast<CStyleCastExpr>(userExpr)) {
+      if (CE->getType()->isObjCQualifiedIdType()) {
+        while ((CE = dyn_cast<CStyleCastExpr>(userExpr)))
+          userExpr = CE->getSubExpr();
+        CastKind CK;
+        if (userExpr->getType()->isIntegralType(*Context)) {
+          CK = CK_IntegralToPointer;
+        } else if (userExpr->getType()->isBlockPointerType()) {
+          CK = CK_BlockPointerToObjCPointerCast;
+        } else if (userExpr->getType()->isPointerType()) {
+          CK = CK_CPointerToObjCPointerCast;
+        } else {
+          CK = CK_BitCast;
+        }
+        userExpr = NoTypeInfoCStyleCastExpr(Context, Context->getObjCIdType(),
+                                            CK, userExpr);
+      }
+    }
+    MsgExprs.push_back(userExpr);
+    // We've transferred the ownership to MsgExprs. For now, we *don't* null
+    // out the argument in the original expression (since we aren't deleting
+    // the ObjCMessageExpr). See RewritePropertyOrImplicitSetter() usage for more info.
+    //Exp->setArg(i, 0);
+  }
+  // Generate the funky cast.
+  CastExpr *cast;
+  SmallVector<QualType, 8> ArgTypes;
+  QualType returnType;
+
+  // Push 'id' and 'SEL', the 2 implicit arguments.
+  if (MsgSendFlavor == MsgSendSuperFunctionDecl)
+    ArgTypes.push_back(Context->getPointerType(getSuperStructType()));
+  else
+    ArgTypes.push_back(Context->getObjCIdType());
+  ArgTypes.push_back(Context->getObjCSelType());
+  if (ObjCMethodDecl *OMD = Exp->getMethodDecl()) {
+    // Push any user argument types.
+    for (ObjCMethodDecl::param_iterator PI = OMD->param_begin(),
+         E = OMD->param_end(); PI != E; ++PI) {
+      QualType t = (*PI)->getType()->isObjCQualifiedIdType()
+                     ? Context->getObjCIdType()
+                     : (*PI)->getType();
+      // Make sure we convert "t (^)(...)" to "t (*)(...)".
+      (void)convertBlockPointerToFunctionPointer(t);
+      ArgTypes.push_back(t);
+    }
+    returnType = Exp->getType();
+    convertToUnqualifiedObjCType(returnType);
+    (void)convertBlockPointerToFunctionPointer(returnType);
+  } else {
+    returnType = Context->getObjCIdType();
+  }
+  // Get the type, we will need to reference it in a couple spots.
+  QualType msgSendType = MsgSendFlavor->getType();
+
+  // Create a reference to the objc_msgSend() declaration.
+  DeclRefExpr *DRE = new (Context) DeclRefExpr(MsgSendFlavor, false, msgSendType,
+                                               VK_LValue, SourceLocation());
+
+  // Need to cast objc_msgSend to "void *" (to workaround a GCC bandaid).
+  // If we don't do this cast, we get the following bizarre warning/note:
+  // xx.m:13: warning: function called through a non-compatible type
+  // xx.m:13: note: if this code is reached, the program will abort
+  cast = NoTypeInfoCStyleCastExpr(Context,
+                                  Context->getPointerType(Context->VoidTy),
+                                  CK_BitCast, DRE);
+
+  // Now do the "normal" pointer to function cast.
+  QualType castType =
+    getSimpleFunctionType(returnType, &ArgTypes[0], ArgTypes.size(),
+      // If we don't have a method decl, force a variadic cast.
+      Exp->getMethodDecl() ? Exp->getMethodDecl()->isVariadic() : true);
+  castType = Context->getPointerType(castType);
+  cast = NoTypeInfoCStyleCastExpr(Context, castType, CK_BitCast,
+                                  cast);
+
+  // Don't forget the parens to enforce the proper binding.
+  ParenExpr *PE = new (Context) ParenExpr(StartLoc, EndLoc, cast);
+
+  const FunctionType *FT = msgSendType->getAs<FunctionType>();
+  CallExpr *CE = new (Context) CallExpr(*Context, PE, MsgExprs,
+                                        FT->getResultType(), VK_RValue,
+                                        EndLoc);
+  Stmt *ReplacingStmt = CE;
+  if (MsgSendStretFlavor) {
+    // We have the method which returns a struct/union. Must also generate
+    // call to objc_msgSend_stret and hang both varieties on a conditional
+    // expression which dictate which one to envoke depending on size of
+    // method's return type.
+    
+    CallExpr *STCE = SynthMsgSendStretCallExpr(MsgSendStretFlavor, 
+                                               msgSendType, returnType, 
+                                               ArgTypes, MsgExprs,
+                                               Exp->getMethodDecl());
+
+    // Build sizeof(returnType)
+    UnaryExprOrTypeTraitExpr *sizeofExpr =
+       new (Context) UnaryExprOrTypeTraitExpr(UETT_SizeOf,
+                                 Context->getTrivialTypeSourceInfo(returnType),
+                                 Context->getSizeType(), SourceLocation(),
+                                 SourceLocation());
+    // (sizeof(returnType) <= 8 ? objc_msgSend(...) : objc_msgSend_stret(...))
+    // FIXME: Value of 8 is base on ppc32/x86 ABI for the most common cases.
+    // For X86 it is more complicated and some kind of target specific routine
+    // is needed to decide what to do.
+    unsigned IntSize =
+      static_cast<unsigned>(Context->getTypeSize(Context->IntTy));
+    IntegerLiteral *limit = IntegerLiteral::Create(*Context,
+                                                   llvm::APInt(IntSize, 8),
+                                                   Context->IntTy,
+                                                   SourceLocation());
+    BinaryOperator *lessThanExpr = 
+      new (Context) BinaryOperator(sizeofExpr, limit, BO_LE, Context->IntTy,
+                                   VK_RValue, OK_Ordinary, SourceLocation(),
+                                   false);
+    // (sizeof(returnType) <= 8 ? objc_msgSend(...) : objc_msgSend_stret(...))
+    ConditionalOperator *CondExpr =
+      new (Context) ConditionalOperator(lessThanExpr,
+                                        SourceLocation(), CE,
+                                        SourceLocation(), STCE,
+                                        returnType, VK_RValue, OK_Ordinary);
+    ReplacingStmt = new (Context) ParenExpr(SourceLocation(), SourceLocation(), 
+                                            CondExpr);
+  }
+  // delete Exp; leak for now, see RewritePropertyOrImplicitSetter() usage for more info.
+  return ReplacingStmt;
+}
+
+Stmt *RewriteObjC::RewriteMessageExpr(ObjCMessageExpr *Exp) {
+  Stmt *ReplacingStmt = SynthMessageExpr(Exp, Exp->getLocStart(),
+                                         Exp->getLocEnd());
+
+  // Now do the actual rewrite.
+  ReplaceStmt(Exp, ReplacingStmt);
+
+  // delete Exp; leak for now, see RewritePropertyOrImplicitSetter() usage for more info.
+  return ReplacingStmt;
+}
+
+// typedef struct objc_object Protocol;
+QualType RewriteObjC::getProtocolType() {
+  if (!ProtocolTypeDecl) {
+    TypeSourceInfo *TInfo
+      = Context->getTrivialTypeSourceInfo(Context->getObjCIdType());
+    ProtocolTypeDecl = TypedefDecl::Create(*Context, TUDecl,
+                                           SourceLocation(), SourceLocation(),
+                                           &Context->Idents.get("Protocol"),
+                                           TInfo);
+  }
+  return Context->getTypeDeclType(ProtocolTypeDecl);
+}
+
+/// RewriteObjCProtocolExpr - Rewrite a protocol expression into
+/// a synthesized/forward data reference (to the protocol's metadata).
+/// The forward references (and metadata) are generated in
+/// RewriteObjC::HandleTranslationUnit().
+Stmt *RewriteObjC::RewriteObjCProtocolExpr(ObjCProtocolExpr *Exp) {
+  std::string Name = "_OBJC_PROTOCOL_" + Exp->getProtocol()->getNameAsString();
+  IdentifierInfo *ID = &Context->Idents.get(Name);
+  VarDecl *VD = VarDecl::Create(*Context, TUDecl, SourceLocation(),
+                                SourceLocation(), ID, getProtocolType(), 0,
+                                SC_Extern, SC_None);
+  DeclRefExpr *DRE = new (Context) DeclRefExpr(VD, false, getProtocolType(),
+                                               VK_LValue, SourceLocation());
+  Expr *DerefExpr = new (Context) UnaryOperator(DRE, UO_AddrOf,
+                             Context->getPointerType(DRE->getType()),
+                             VK_RValue, OK_Ordinary, SourceLocation());
+  CastExpr *castExpr = NoTypeInfoCStyleCastExpr(Context, DerefExpr->getType(),
+                                                CK_BitCast,
+                                                DerefExpr);
+  ReplaceStmt(Exp, castExpr);
+  ProtocolExprDecls.insert(Exp->getProtocol()->getCanonicalDecl());
+  // delete Exp; leak for now, see RewritePropertyOrImplicitSetter() usage for more info.
+  return castExpr;
+
+}
+
+bool RewriteObjC::BufferContainsPPDirectives(const char *startBuf,
+                                             const char *endBuf) {
+  while (startBuf < endBuf) {
+    if (*startBuf == '#') {
+      // Skip whitespace.
+      for (++startBuf; startBuf[0] == ' ' || startBuf[0] == '\t'; ++startBuf)
+        ;
+      if (!strncmp(startBuf, "if", strlen("if")) ||
+          !strncmp(startBuf, "ifdef", strlen("ifdef")) ||
+          !strncmp(startBuf, "ifndef", strlen("ifndef")) ||
+          !strncmp(startBuf, "define", strlen("define")) ||
+          !strncmp(startBuf, "undef", strlen("undef")) ||
+          !strncmp(startBuf, "else", strlen("else")) ||
+          !strncmp(startBuf, "elif", strlen("elif")) ||
+          !strncmp(startBuf, "endif", strlen("endif")) ||
+          !strncmp(startBuf, "pragma", strlen("pragma")) ||
+          !strncmp(startBuf, "include", strlen("include")) ||
+          !strncmp(startBuf, "import", strlen("import")) ||
+          !strncmp(startBuf, "include_next", strlen("include_next")))
+        return true;
+    }
+    startBuf++;
+  }
+  return false;
+}
+
+/// RewriteObjCInternalStruct - Rewrite one internal struct corresponding to
+/// an objective-c class with ivars.
+void RewriteObjC::RewriteObjCInternalStruct(ObjCInterfaceDecl *CDecl,
+                                               std::string &Result) {
+  assert(CDecl && "Class missing in SynthesizeObjCInternalStruct");
+  assert(CDecl->getName() != "" &&
+         "Name missing in SynthesizeObjCInternalStruct");
+  // Do not synthesize more than once.
+  if (ObjCSynthesizedStructs.count(CDecl))
+    return;
+  ObjCInterfaceDecl *RCDecl = CDecl->getSuperClass();
+  int NumIvars = CDecl->ivar_size();
+  SourceLocation LocStart = CDecl->getLocStart();
+  SourceLocation LocEnd = CDecl->getEndOfDefinitionLoc();
+
+  const char *startBuf = SM->getCharacterData(LocStart);
+  const char *endBuf = SM->getCharacterData(LocEnd);
+
+  // If no ivars and no root or if its root, directly or indirectly,
+  // have no ivars (thus not synthesized) then no need to synthesize this class.
+  if ((!CDecl->isThisDeclarationADefinition() || NumIvars == 0) &&
+      (!RCDecl || !ObjCSynthesizedStructs.count(RCDecl))) {
+    endBuf += Lexer::MeasureTokenLength(LocEnd, *SM, LangOpts);
+    ReplaceText(LocStart, endBuf-startBuf, Result);
+    return;
+  }
+
+  // FIXME: This has potential of causing problem. If
+  // SynthesizeObjCInternalStruct is ever called recursively.
+  Result += "\nstruct ";
+  Result += CDecl->getNameAsString();
+  if (LangOpts.MicrosoftExt)
+    Result += "_IMPL";
+
+  if (NumIvars > 0) {
+    const char *cursor = strchr(startBuf, '{');
+    assert((cursor && endBuf)
+           && "SynthesizeObjCInternalStruct - malformed @interface");
+    // If the buffer contains preprocessor directives, we do more fine-grained
+    // rewrites. This is intended to fix code that looks like (which occurs in
+    // NSURL.h, for example):
+    //
+    // #ifdef XYZ
+    // @interface Foo : NSObject
+    // #else
+    // @interface FooBar : NSObject
+    // #endif
+    // {
+    //    int i;
+    // }
+    // @end
+    //
+    // This clause is segregated to avoid breaking the common case.
+    if (BufferContainsPPDirectives(startBuf, cursor)) {
+      SourceLocation L = RCDecl ? CDecl->getSuperClassLoc() :
+                                  CDecl->getAtStartLoc();
+      const char *endHeader = SM->getCharacterData(L);
+      endHeader += Lexer::MeasureTokenLength(L, *SM, LangOpts);
+
+      if (CDecl->protocol_begin() != CDecl->protocol_end()) {
+        // advance to the end of the referenced protocols.
+        while (endHeader < cursor && *endHeader != '>') endHeader++;
+        endHeader++;
+      }
+      // rewrite the original header
+      ReplaceText(LocStart, endHeader-startBuf, Result);
+    } else {
+      // rewrite the original header *without* disturbing the '{'
+      ReplaceText(LocStart, cursor-startBuf, Result);
+    }
+    if (RCDecl && ObjCSynthesizedStructs.count(RCDecl)) {
+      Result = "\n    struct ";
+      Result += RCDecl->getNameAsString();
+      Result += "_IMPL ";
+      Result += RCDecl->getNameAsString();
+      Result += "_IVARS;\n";
+
+      // insert the super class structure definition.
+      SourceLocation OnePastCurly =
+        LocStart.getLocWithOffset(cursor-startBuf+1);
+      InsertText(OnePastCurly, Result);
+    }
+    cursor++; // past '{'
+
+    // Now comment out any visibility specifiers.
+    while (cursor < endBuf) {
+      if (*cursor == '@') {
+        SourceLocation atLoc = LocStart.getLocWithOffset(cursor-startBuf);
+        // Skip whitespace.
+        for (++cursor; cursor[0] == ' ' || cursor[0] == '\t'; ++cursor)
+          /*scan*/;
+
+        // FIXME: presence of @public, etc. inside comment results in
+        // this transformation as well, which is still correct c-code.
+        if (!strncmp(cursor, "public", strlen("public")) ||
+            !strncmp(cursor, "private", strlen("private")) ||
+            !strncmp(cursor, "package", strlen("package")) ||
+            !strncmp(cursor, "protected", strlen("protected")))
+          InsertText(atLoc, "// ");
+      }
+      // FIXME: If there are cases where '<' is used in ivar declaration part
+      // of user code, then scan the ivar list and use needToScanForQualifiers
+      // for type checking.
+      else if (*cursor == '<') {
+        SourceLocation atLoc = LocStart.getLocWithOffset(cursor-startBuf);
+        InsertText(atLoc, "/* ");
+        cursor = strchr(cursor, '>');
+        cursor++;
+        atLoc = LocStart.getLocWithOffset(cursor-startBuf);
+        InsertText(atLoc, " */");
+      } else if (*cursor == '^') { // rewrite block specifier.
+        SourceLocation caretLoc = LocStart.getLocWithOffset(cursor-startBuf);
+        ReplaceText(caretLoc, 1, "*");
+      }
+      cursor++;
+    }
+    // Don't forget to add a ';'!!
+    InsertText(LocEnd.getLocWithOffset(1), ";");
+  } else { // we don't have any instance variables - insert super struct.
+    endBuf += Lexer::MeasureTokenLength(LocEnd, *SM, LangOpts);
+    Result += " {\n    struct ";
+    Result += RCDecl->getNameAsString();
+    Result += "_IMPL ";
+    Result += RCDecl->getNameAsString();
+    Result += "_IVARS;\n};\n";
+    ReplaceText(LocStart, endBuf-startBuf, Result);
+  }
+  // Mark this struct as having been generated.
+  if (!ObjCSynthesizedStructs.insert(CDecl))
+    llvm_unreachable("struct already synthesize- SynthesizeObjCInternalStruct");
+}
+
+//===----------------------------------------------------------------------===//
+// Meta Data Emission
+//===----------------------------------------------------------------------===//
+
+
+/// RewriteImplementations - This routine rewrites all method implementations
+/// and emits meta-data.
+
+void RewriteObjC::RewriteImplementations() {
+  int ClsDefCount = ClassImplementation.size();
+  int CatDefCount = CategoryImplementation.size();
+
+  // Rewrite implemented methods
+  for (int i = 0; i < ClsDefCount; i++)
+    RewriteImplementationDecl(ClassImplementation[i]);
+
+  for (int i = 0; i < CatDefCount; i++)
+    RewriteImplementationDecl(CategoryImplementation[i]);
+}
+
+void RewriteObjC::RewriteByRefString(std::string &ResultStr, 
+                                     const std::string &Name,
+                                     ValueDecl *VD, bool def) {
+  assert(BlockByRefDeclNo.count(VD) && 
+         "RewriteByRefString: ByRef decl missing");
+  if (def)
+    ResultStr += "struct ";
+  ResultStr += "__Block_byref_" + Name + 
+    "_" + utostr(BlockByRefDeclNo[VD]) ;
+}
+
+static bool HasLocalVariableExternalStorage(ValueDecl *VD) {
+  if (VarDecl *Var = dyn_cast<VarDecl>(VD))
+    return (Var->isFunctionOrMethodVarDecl() && !Var->hasLocalStorage());
+  return false;
+}
+
+std::string RewriteObjC::SynthesizeBlockFunc(BlockExpr *CE, int i,
+                                                   StringRef funcName,
+                                                   std::string Tag) {
+  const FunctionType *AFT = CE->getFunctionType();
+  QualType RT = AFT->getResultType();
+  std::string StructRef = "struct " + Tag;
+  std::string S = "static " + RT.getAsString(Context->getPrintingPolicy()) + " __" +
+                  funcName.str() + "_" + "block_func_" + utostr(i);
+
+  BlockDecl *BD = CE->getBlockDecl();
+
+  if (isa<FunctionNoProtoType>(AFT)) {
+    // No user-supplied arguments. Still need to pass in a pointer to the
+    // block (to reference imported block decl refs).
+    S += "(" + StructRef + " *__cself)";
+  } else if (BD->param_empty()) {
+    S += "(" + StructRef + " *__cself)";
+  } else {
+    const FunctionProtoType *FT = cast<FunctionProtoType>(AFT);
+    assert(FT && "SynthesizeBlockFunc: No function proto");
+    S += '(';
+    // first add the implicit argument.
+    S += StructRef + " *__cself, ";
+    std::string ParamStr;
+    for (BlockDecl::param_iterator AI = BD->param_begin(),
+         E = BD->param_end(); AI != E; ++AI) {
+      if (AI != BD->param_begin()) S += ", ";
+      ParamStr = (*AI)->getNameAsString();
+      QualType QT = (*AI)->getType();
+      (void)convertBlockPointerToFunctionPointer(QT);
+      QT.getAsStringInternal(ParamStr, Context->getPrintingPolicy());
+      S += ParamStr;
+    }
+    if (FT->isVariadic()) {
+      if (!BD->param_empty()) S += ", ";
+      S += "...";
+    }
+    S += ')';
+  }
+  S += " {\n";
+
+  // Create local declarations to avoid rewriting all closure decl ref exprs.
+  // First, emit a declaration for all "by ref" decls.
+  for (SmallVector<ValueDecl*,8>::iterator I = BlockByRefDecls.begin(),
+       E = BlockByRefDecls.end(); I != E; ++I) {
+    S += "  ";
+    std::string Name = (*I)->getNameAsString();
+    std::string TypeString;
+    RewriteByRefString(TypeString, Name, (*I));
+    TypeString += " *";
+    Name = TypeString + Name;
+    S += Name + " = __cself->" + (*I)->getNameAsString() + "; // bound by ref\n";
+  }
+  // Next, emit a declaration for all "by copy" declarations.
+  for (SmallVector<ValueDecl*,8>::iterator I = BlockByCopyDecls.begin(),
+       E = BlockByCopyDecls.end(); I != E; ++I) {
+    S += "  ";
+    // Handle nested closure invocation. For example:
+    //
+    //   void (^myImportedClosure)(void);
+    //   myImportedClosure  = ^(void) { setGlobalInt(x + y); };
+    //
+    //   void (^anotherClosure)(void);
+    //   anotherClosure = ^(void) {
+    //     myImportedClosure(); // import and invoke the closure
+    //   };
+    //
+    if (isTopLevelBlockPointerType((*I)->getType())) {
+      RewriteBlockPointerTypeVariable(S, (*I));
+      S += " = (";
+      RewriteBlockPointerType(S, (*I)->getType());
+      S += ")";
+      S += "__cself->" + (*I)->getNameAsString() + "; // bound by copy\n";
+    }
+    else {
+      std::string Name = (*I)->getNameAsString();
+      QualType QT = (*I)->getType();
+      if (HasLocalVariableExternalStorage(*I))
+        QT = Context->getPointerType(QT);
+      QT.getAsStringInternal(Name, Context->getPrintingPolicy());
+      S += Name + " = __cself->" + 
+                              (*I)->getNameAsString() + "; // bound by copy\n";
+    }
+  }
+  std::string RewrittenStr = RewrittenBlockExprs[CE];
+  const char *cstr = RewrittenStr.c_str();
+  while (*cstr++ != '{') ;
+  S += cstr;
+  S += "\n";
+  return S;
+}
+
+std::string RewriteObjC::SynthesizeBlockHelperFuncs(BlockExpr *CE, int i,
+                                                   StringRef funcName,
+                                                   std::string Tag) {
+  std::string StructRef = "struct " + Tag;
+  std::string S = "static void __";
+
+  S += funcName;
+  S += "_block_copy_" + utostr(i);
+  S += "(" + StructRef;
+  S += "*dst, " + StructRef;
+  S += "*src) {";
+  for (llvm::SmallPtrSet<ValueDecl*,8>::iterator I = ImportedBlockDecls.begin(),
+      E = ImportedBlockDecls.end(); I != E; ++I) {
+    ValueDecl *VD = (*I);
+    S += "_Block_object_assign((void*)&dst->";
+    S += (*I)->getNameAsString();
+    S += ", (void*)src->";
+    S += (*I)->getNameAsString();
+    if (BlockByRefDeclsPtrSet.count((*I)))
+      S += ", " + utostr(BLOCK_FIELD_IS_BYREF) + "/*BLOCK_FIELD_IS_BYREF*/);";
+    else if (VD->getType()->isBlockPointerType())
+      S += ", " + utostr(BLOCK_FIELD_IS_BLOCK) + "/*BLOCK_FIELD_IS_BLOCK*/);";
+    else
+      S += ", " + utostr(BLOCK_FIELD_IS_OBJECT) + "/*BLOCK_FIELD_IS_OBJECT*/);";
+  }
+  S += "}\n";
+  
+  S += "\nstatic void __";
+  S += funcName;
+  S += "_block_dispose_" + utostr(i);
+  S += "(" + StructRef;
+  S += "*src) {";
+  for (llvm::SmallPtrSet<ValueDecl*,8>::iterator I = ImportedBlockDecls.begin(),
+      E = ImportedBlockDecls.end(); I != E; ++I) {
+    ValueDecl *VD = (*I);
+    S += "_Block_object_dispose((void*)src->";
+    S += (*I)->getNameAsString();
+    if (BlockByRefDeclsPtrSet.count((*I)))
+      S += ", " + utostr(BLOCK_FIELD_IS_BYREF) + "/*BLOCK_FIELD_IS_BYREF*/);";
+    else if (VD->getType()->isBlockPointerType())
+      S += ", " + utostr(BLOCK_FIELD_IS_BLOCK) + "/*BLOCK_FIELD_IS_BLOCK*/);";
+    else
+      S += ", " + utostr(BLOCK_FIELD_IS_OBJECT) + "/*BLOCK_FIELD_IS_OBJECT*/);";
+  }
+  S += "}\n";
+  return S;
+}
+
+std::string RewriteObjC::SynthesizeBlockImpl(BlockExpr *CE, std::string Tag, 
+                                             std::string Desc) {
+  std::string S = "\nstruct " + Tag;
+  std::string Constructor = "  " + Tag;
+
+  S += " {\n  struct __block_impl impl;\n";
+  S += "  struct " + Desc;
+  S += "* Desc;\n";
+
+  Constructor += "(void *fp, "; // Invoke function pointer.
+  Constructor += "struct " + Desc; // Descriptor pointer.
+  Constructor += " *desc";
+
+  if (BlockDeclRefs.size()) {
+    // Output all "by copy" declarations.
+    for (SmallVector<ValueDecl*,8>::iterator I = BlockByCopyDecls.begin(),
+         E = BlockByCopyDecls.end(); I != E; ++I) {
+      S += "  ";
+      std::string FieldName = (*I)->getNameAsString();
+      std::string ArgName = "_" + FieldName;
+      // Handle nested closure invocation. For example:
+      //
+      //   void (^myImportedBlock)(void);
+      //   myImportedBlock  = ^(void) { setGlobalInt(x + y); };
+      //
+      //   void (^anotherBlock)(void);
+      //   anotherBlock = ^(void) {
+      //     myImportedBlock(); // import and invoke the closure
+      //   };
+      //
+      if (isTopLevelBlockPointerType((*I)->getType())) {
+        S += "struct __block_impl *";
+        Constructor += ", void *" + ArgName;
+      } else {
+        QualType QT = (*I)->getType();
+        if (HasLocalVariableExternalStorage(*I))
+          QT = Context->getPointerType(QT);
+        QT.getAsStringInternal(FieldName, Context->getPrintingPolicy());
+        QT.getAsStringInternal(ArgName, Context->getPrintingPolicy());
+        Constructor += ", " + ArgName;
+      }
+      S += FieldName + ";\n";
+    }
+    // Output all "by ref" declarations.
+    for (SmallVector<ValueDecl*,8>::iterator I = BlockByRefDecls.begin(),
+         E = BlockByRefDecls.end(); I != E; ++I) {
+      S += "  ";
+      std::string FieldName = (*I)->getNameAsString();
+      std::string ArgName = "_" + FieldName;
+      {
+        std::string TypeString;
+        RewriteByRefString(TypeString, FieldName, (*I));
+        TypeString += " *";
+        FieldName = TypeString + FieldName;
+        ArgName = TypeString + ArgName;
+        Constructor += ", " + ArgName;
+      }
+      S += FieldName + "; // by ref\n";
+    }
+    // Finish writing the constructor.
+    Constructor += ", int flags=0)";
+    // Initialize all "by copy" arguments.
+    bool firsTime = true;
+    for (SmallVector<ValueDecl*,8>::iterator I = BlockByCopyDecls.begin(),
+         E = BlockByCopyDecls.end(); I != E; ++I) {
+      std::string Name = (*I)->getNameAsString();
+        if (firsTime) {
+          Constructor += " : ";
+          firsTime = false;
+        }
+        else
+          Constructor += ", ";
+        if (isTopLevelBlockPointerType((*I)->getType()))
+          Constructor += Name + "((struct __block_impl *)_" + Name + ")";
+        else
+          Constructor += Name + "(_" + Name + ")";
+    }
+    // Initialize all "by ref" arguments.
+    for (SmallVector<ValueDecl*,8>::iterator I = BlockByRefDecls.begin(),
+         E = BlockByRefDecls.end(); I != E; ++I) {
+      std::string Name = (*I)->getNameAsString();
+      if (firsTime) {
+        Constructor += " : ";
+        firsTime = false;
+      }
+      else
+        Constructor += ", ";
+      Constructor += Name + "(_" + Name + "->__forwarding)";
+    }
+    
+    Constructor += " {\n";
+    if (GlobalVarDecl)
+      Constructor += "    impl.isa = &_NSConcreteGlobalBlock;\n";
+    else
+      Constructor += "    impl.isa = &_NSConcreteStackBlock;\n";
+    Constructor += "    impl.Flags = flags;\n    impl.FuncPtr = fp;\n";
+
+    Constructor += "    Desc = desc;\n";
+  } else {
+    // Finish writing the constructor.
+    Constructor += ", int flags=0) {\n";
+    if (GlobalVarDecl)
+      Constructor += "    impl.isa = &_NSConcreteGlobalBlock;\n";
+    else
+      Constructor += "    impl.isa = &_NSConcreteStackBlock;\n";
+    Constructor += "    impl.Flags = flags;\n    impl.FuncPtr = fp;\n";
+    Constructor += "    Desc = desc;\n";
+  }
+  Constructor += "  ";
+  Constructor += "}\n";
+  S += Constructor;
+  S += "};\n";
+  return S;
+}
+
+std::string RewriteObjC::SynthesizeBlockDescriptor(std::string DescTag, 
+                                                   std::string ImplTag, int i,
+                                                   StringRef FunName,
+                                                   unsigned hasCopy) {
+  std::string S = "\nstatic struct " + DescTag;
+  
+  S += " {\n  unsigned long reserved;\n";
+  S += "  unsigned long Block_size;\n";
+  if (hasCopy) {
+    S += "  void (*copy)(struct ";
+    S += ImplTag; S += "*, struct ";
+    S += ImplTag; S += "*);\n";
+    
+    S += "  void (*dispose)(struct ";
+    S += ImplTag; S += "*);\n";
+  }
+  S += "} ";
+
+  S += DescTag + "_DATA = { 0, sizeof(struct ";
+  S += ImplTag + ")";
+  if (hasCopy) {
+    S += ", __" + FunName.str() + "_block_copy_" + utostr(i);
+    S += ", __" + FunName.str() + "_block_dispose_" + utostr(i);
+  }
+  S += "};\n";
+  return S;
+}
+
+void RewriteObjC::SynthesizeBlockLiterals(SourceLocation FunLocStart,
+                                          StringRef FunName) {
+  // Insert declaration for the function in which block literal is used.
+  if (CurFunctionDeclToDeclareForBlock && !Blocks.empty())
+    RewriteBlockLiteralFunctionDecl(CurFunctionDeclToDeclareForBlock);
+  bool RewriteSC = (GlobalVarDecl &&
+                    !Blocks.empty() &&
+                    GlobalVarDecl->getStorageClass() == SC_Static &&
+                    GlobalVarDecl->getType().getCVRQualifiers());
+  if (RewriteSC) {
+    std::string SC(" void __");
+    SC += GlobalVarDecl->getNameAsString();
+    SC += "() {}";
+    InsertText(FunLocStart, SC);
+  }
+  
+  // Insert closures that were part of the function.
+  for (unsigned i = 0, count=0; i < Blocks.size(); i++) {
+    CollectBlockDeclRefInfo(Blocks[i]);
+    // Need to copy-in the inner copied-in variables not actually used in this
+    // block.
+    for (int j = 0; j < InnerDeclRefsCount[i]; j++) {
+      DeclRefExpr *Exp = InnerDeclRefs[count++];
+      ValueDecl *VD = Exp->getDecl();
+      BlockDeclRefs.push_back(Exp);
+      if (!VD->hasAttr<BlocksAttr>() && !BlockByCopyDeclsPtrSet.count(VD)) {
+        BlockByCopyDeclsPtrSet.insert(VD);
+        BlockByCopyDecls.push_back(VD);
+      }
+      if (VD->hasAttr<BlocksAttr>() && !BlockByRefDeclsPtrSet.count(VD)) {
+        BlockByRefDeclsPtrSet.insert(VD);
+        BlockByRefDecls.push_back(VD);
+      }
+      // imported objects in the inner blocks not used in the outer
+      // blocks must be copied/disposed in the outer block as well.
+      if (VD->hasAttr<BlocksAttr>() ||
+          VD->getType()->isObjCObjectPointerType() || 
+          VD->getType()->isBlockPointerType())
+        ImportedBlockDecls.insert(VD);
+    }
+
+    std::string ImplTag = "__" + FunName.str() + "_block_impl_" + utostr(i);
+    std::string DescTag = "__" + FunName.str() + "_block_desc_" + utostr(i);
+
+    std::string CI = SynthesizeBlockImpl(Blocks[i], ImplTag, DescTag);
+
+    InsertText(FunLocStart, CI);
+
+    std::string CF = SynthesizeBlockFunc(Blocks[i], i, FunName, ImplTag);
+
+    InsertText(FunLocStart, CF);
+
+    if (ImportedBlockDecls.size()) {
+      std::string HF = SynthesizeBlockHelperFuncs(Blocks[i], i, FunName, ImplTag);
+      InsertText(FunLocStart, HF);
+    }
+    std::string BD = SynthesizeBlockDescriptor(DescTag, ImplTag, i, FunName,
+                                               ImportedBlockDecls.size() > 0);
+    InsertText(FunLocStart, BD);
+
+    BlockDeclRefs.clear();
+    BlockByRefDecls.clear();
+    BlockByRefDeclsPtrSet.clear();
+    BlockByCopyDecls.clear();
+    BlockByCopyDeclsPtrSet.clear();
+    ImportedBlockDecls.clear();
+  }
+  if (RewriteSC) {
+    // Must insert any 'const/volatile/static here. Since it has been
+    // removed as result of rewriting of block literals.
+    std::string SC;
+    if (GlobalVarDecl->getStorageClass() == SC_Static)
+      SC = "static ";
+    if (GlobalVarDecl->getType().isConstQualified())
+      SC += "const ";
+    if (GlobalVarDecl->getType().isVolatileQualified())
+      SC += "volatile ";
+    if (GlobalVarDecl->getType().isRestrictQualified())
+      SC += "restrict ";
+    InsertText(FunLocStart, SC);
+  }
+  
+  Blocks.clear();
+  InnerDeclRefsCount.clear();
+  InnerDeclRefs.clear();
+  RewrittenBlockExprs.clear();
+}
+
+void RewriteObjC::InsertBlockLiteralsWithinFunction(FunctionDecl *FD) {
+  SourceLocation FunLocStart = FD->getTypeSpecStartLoc();
+  StringRef FuncName = FD->getName();
+
+  SynthesizeBlockLiterals(FunLocStart, FuncName);
+}
+
+static void BuildUniqueMethodName(std::string &Name,
+                                  ObjCMethodDecl *MD) {
+  ObjCInterfaceDecl *IFace = MD->getClassInterface();
+  Name = IFace->getName();
+  Name += "__" + MD->getSelector().getAsString();
+  // Convert colons to underscores.
+  std::string::size_type loc = 0;
+  while ((loc = Name.find(":", loc)) != std::string::npos)
+    Name.replace(loc, 1, "_");
+}
+
+void RewriteObjC::InsertBlockLiteralsWithinMethod(ObjCMethodDecl *MD) {
+  //fprintf(stderr,"In InsertBlockLiteralsWitinMethod\n");
+  //SourceLocation FunLocStart = MD->getLocStart();
+  SourceLocation FunLocStart = MD->getLocStart();
+  std::string FuncName;
+  BuildUniqueMethodName(FuncName, MD);
+  SynthesizeBlockLiterals(FunLocStart, FuncName);
+}
+
+void RewriteObjC::GetBlockDeclRefExprs(Stmt *S) {
+  for (Stmt::child_range CI = S->children(); CI; ++CI)
+    if (*CI) {
+      if (BlockExpr *CBE = dyn_cast<BlockExpr>(*CI))
+        GetBlockDeclRefExprs(CBE->getBody());
+      else
+        GetBlockDeclRefExprs(*CI);
+    }
+  // Handle specific things.
+  if (DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(S)) {
+    if (DRE->refersToEnclosingLocal()) {
+      // FIXME: Handle enums.
+      if (!isa<FunctionDecl>(DRE->getDecl()))
+        BlockDeclRefs.push_back(DRE);
+      if (HasLocalVariableExternalStorage(DRE->getDecl()))
+        BlockDeclRefs.push_back(DRE);
+    }
+  }
+  
+  return;
+}
+
+void RewriteObjC::GetInnerBlockDeclRefExprs(Stmt *S, 
+                SmallVector<DeclRefExpr *, 8> &InnerBlockDeclRefs,
+                llvm::SmallPtrSet<const DeclContext *, 8> &InnerContexts) {
+  for (Stmt::child_range CI = S->children(); CI; ++CI)
+    if (*CI) {
+      if (BlockExpr *CBE = dyn_cast<BlockExpr>(*CI)) {
+        InnerContexts.insert(cast<DeclContext>(CBE->getBlockDecl()));
+        GetInnerBlockDeclRefExprs(CBE->getBody(),
+                                  InnerBlockDeclRefs,
+                                  InnerContexts);
+      }
+      else
+        GetInnerBlockDeclRefExprs(*CI,
+                                  InnerBlockDeclRefs,
+                                  InnerContexts);
+
+    }
+  // Handle specific things.
+  if (DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(S)) {
+    if (DRE->refersToEnclosingLocal()) {
+      if (!isa<FunctionDecl>(DRE->getDecl()) &&
+          !InnerContexts.count(DRE->getDecl()->getDeclContext()))
+        InnerBlockDeclRefs.push_back(DRE);
+      if (VarDecl *Var = dyn_cast<VarDecl>(DRE->getDecl()))
+        if (Var->isFunctionOrMethodVarDecl())
+          ImportedLocalExternalDecls.insert(Var);
+    }
+  }
+  
+  return;
+}
+
+/// convertFunctionTypeOfBlocks - This routine converts a function type
+/// whose result type may be a block pointer or whose argument type(s)
+/// might be block pointers to an equivalent function type replacing
+/// all block pointers to function pointers.
+QualType RewriteObjC::convertFunctionTypeOfBlocks(const FunctionType *FT) {
+  const FunctionProtoType *FTP = dyn_cast<FunctionProtoType>(FT);
+  // FTP will be null for closures that don't take arguments.
+  // Generate a funky cast.
+  SmallVector<QualType, 8> ArgTypes;
+  QualType Res = FT->getResultType();
+  bool HasBlockType = convertBlockPointerToFunctionPointer(Res);
+  
+  if (FTP) {
+    for (FunctionProtoType::arg_type_iterator I = FTP->arg_type_begin(),
+         E = FTP->arg_type_end(); I && (I != E); ++I) {
+      QualType t = *I;
+      // Make sure we convert "t (^)(...)" to "t (*)(...)".
+      if (convertBlockPointerToFunctionPointer(t))
+        HasBlockType = true;
+      ArgTypes.push_back(t);
+    }
+  }
+  QualType FuncType;
+  // FIXME. Does this work if block takes no argument but has a return type
+  // which is of block type?
+  if (HasBlockType)
+    FuncType = getSimpleFunctionType(Res, &ArgTypes[0], ArgTypes.size());
+  else FuncType = QualType(FT, 0);
+  return FuncType;
+}
+
+Stmt *RewriteObjC::SynthesizeBlockCall(CallExpr *Exp, const Expr *BlockExp) {
+  // Navigate to relevant type information.
+  const BlockPointerType *CPT = 0;
+
+  if (const DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(BlockExp)) {
+    CPT = DRE->getType()->getAs<BlockPointerType>();
+  } else if (const MemberExpr *MExpr = dyn_cast<MemberExpr>(BlockExp)) {
+    CPT = MExpr->getType()->getAs<BlockPointerType>();
+  } 
+  else if (const ParenExpr *PRE = dyn_cast<ParenExpr>(BlockExp)) {
+    return SynthesizeBlockCall(Exp, PRE->getSubExpr());
+  }
+  else if (const ImplicitCastExpr *IEXPR = dyn_cast<ImplicitCastExpr>(BlockExp)) 
+    CPT = IEXPR->getType()->getAs<BlockPointerType>();
+  else if (const ConditionalOperator *CEXPR = 
+            dyn_cast<ConditionalOperator>(BlockExp)) {
+    Expr *LHSExp = CEXPR->getLHS();
+    Stmt *LHSStmt = SynthesizeBlockCall(Exp, LHSExp);
+    Expr *RHSExp = CEXPR->getRHS();
+    Stmt *RHSStmt = SynthesizeBlockCall(Exp, RHSExp);
+    Expr *CONDExp = CEXPR->getCond();
+    ConditionalOperator *CondExpr =
+      new (Context) ConditionalOperator(CONDExp,
+                                      SourceLocation(), cast<Expr>(LHSStmt),
+                                      SourceLocation(), cast<Expr>(RHSStmt),
+                                      Exp->getType(), VK_RValue, OK_Ordinary);
+    return CondExpr;
+  } else if (const ObjCIvarRefExpr *IRE = dyn_cast<ObjCIvarRefExpr>(BlockExp)) {
+    CPT = IRE->getType()->getAs<BlockPointerType>();
+  } else if (const PseudoObjectExpr *POE
+               = dyn_cast<PseudoObjectExpr>(BlockExp)) {
+    CPT = POE->getType()->castAs<BlockPointerType>();
+  } else {
+    assert(1 && "RewriteBlockClass: Bad type");
+  }
+  assert(CPT && "RewriteBlockClass: Bad type");
+  const FunctionType *FT = CPT->getPointeeType()->getAs<FunctionType>();
+  assert(FT && "RewriteBlockClass: Bad type");
+  const FunctionProtoType *FTP = dyn_cast<FunctionProtoType>(FT);
+  // FTP will be null for closures that don't take arguments.
+
+  RecordDecl *RD = RecordDecl::Create(*Context, TTK_Struct, TUDecl,
+                                      SourceLocation(), SourceLocation(),
+                                      &Context->Idents.get("__block_impl"));
+  QualType PtrBlock = Context->getPointerType(Context->getTagDeclType(RD));
+
+  // Generate a funky cast.
+  SmallVector<QualType, 8> ArgTypes;
+
+  // Push the block argument type.
+  ArgTypes.push_back(PtrBlock);
+  if (FTP) {
+    for (FunctionProtoType::arg_type_iterator I = FTP->arg_type_begin(),
+         E = FTP->arg_type_end(); I && (I != E); ++I) {
+      QualType t = *I;
+      // Make sure we convert "t (^)(...)" to "t (*)(...)".
+      if (!convertBlockPointerToFunctionPointer(t))
+        convertToUnqualifiedObjCType(t);
+      ArgTypes.push_back(t);
+    }
+  }
+  // Now do the pointer to function cast.
+  QualType PtrToFuncCastType
+    = getSimpleFunctionType(Exp->getType(), &ArgTypes[0], ArgTypes.size());
+
+  PtrToFuncCastType = Context->getPointerType(PtrToFuncCastType);
+
+  CastExpr *BlkCast = NoTypeInfoCStyleCastExpr(Context, PtrBlock,
+                                               CK_BitCast,
+                                               const_cast<Expr*>(BlockExp));
+  // Don't forget the parens to enforce the proper binding.
+  ParenExpr *PE = new (Context) ParenExpr(SourceLocation(), SourceLocation(),
+                                          BlkCast);
+  //PE->dump();
+
+  FieldDecl *FD = FieldDecl::Create(*Context, 0, SourceLocation(),
+                                    SourceLocation(),
+                                    &Context->Idents.get("FuncPtr"),
+                                    Context->VoidPtrTy, 0,
+                                    /*BitWidth=*/0, /*Mutable=*/true,
+                                    ICIS_NoInit);
+  MemberExpr *ME = new (Context) MemberExpr(PE, true, FD, SourceLocation(),
+                                            FD->getType(), VK_LValue,
+                                            OK_Ordinary);
+
+  
+  CastExpr *FunkCast = NoTypeInfoCStyleCastExpr(Context, PtrToFuncCastType,
+                                                CK_BitCast, ME);
+  PE = new (Context) ParenExpr(SourceLocation(), SourceLocation(), FunkCast);
+
+  SmallVector<Expr*, 8> BlkExprs;
+  // Add the implicit argument.
+  BlkExprs.push_back(BlkCast);
+  // Add the user arguments.
+  for (CallExpr::arg_iterator I = Exp->arg_begin(),
+       E = Exp->arg_end(); I != E; ++I) {
+    BlkExprs.push_back(*I);
+  }
+  CallExpr *CE = new (Context) CallExpr(*Context, PE, BlkExprs,
+                                        Exp->getType(), VK_RValue,
+                                        SourceLocation());
+  return CE;
+}
+
+// We need to return the rewritten expression to handle cases where the
+// BlockDeclRefExpr is embedded in another expression being rewritten.
+// For example:
+//
+// int main() {
+//    __block Foo *f;
+//    __block int i;
+//
+//    void (^myblock)() = ^() {
+//        [f test]; // f is a BlockDeclRefExpr embedded in a message (which is being rewritten).
+//        i = 77;
+//    };
+//}
+Stmt *RewriteObjC::RewriteBlockDeclRefExpr(DeclRefExpr *DeclRefExp) {
+  // Rewrite the byref variable into BYREFVAR->__forwarding->BYREFVAR 
+  // for each DeclRefExp where BYREFVAR is name of the variable.
+  ValueDecl *VD = DeclRefExp->getDecl();
+  bool isArrow = DeclRefExp->refersToEnclosingLocal();
+  
+  FieldDecl *FD = FieldDecl::Create(*Context, 0, SourceLocation(),
+                                    SourceLocation(),
+                                    &Context->Idents.get("__forwarding"), 
+                                    Context->VoidPtrTy, 0,
+                                    /*BitWidth=*/0, /*Mutable=*/true,
+                                    ICIS_NoInit);
+  MemberExpr *ME = new (Context) MemberExpr(DeclRefExp, isArrow,
+                                            FD, SourceLocation(),
+                                            FD->getType(), VK_LValue,
+                                            OK_Ordinary);
+
+  StringRef Name = VD->getName();
+  FD = FieldDecl::Create(*Context, 0, SourceLocation(), SourceLocation(),
+                         &Context->Idents.get(Name), 
+                         Context->VoidPtrTy, 0,
+                         /*BitWidth=*/0, /*Mutable=*/true,
+                         ICIS_NoInit);
+  ME = new (Context) MemberExpr(ME, true, FD, SourceLocation(),
+                                DeclRefExp->getType(), VK_LValue, OK_Ordinary);
+  
+  
+  
+  // Need parens to enforce precedence.
+  ParenExpr *PE = new (Context) ParenExpr(DeclRefExp->getExprLoc(), 
+                                          DeclRefExp->getExprLoc(), 
+                                          ME);
+  ReplaceStmt(DeclRefExp, PE);
+  return PE;
+}
+
+// Rewrites the imported local variable V with external storage 
+// (static, extern, etc.) as *V
+//
+Stmt *RewriteObjC::RewriteLocalVariableExternalStorage(DeclRefExpr *DRE) {
+  ValueDecl *VD = DRE->getDecl();
+  if (VarDecl *Var = dyn_cast<VarDecl>(VD))
+    if (!ImportedLocalExternalDecls.count(Var))
+      return DRE;
+  Expr *Exp = new (Context) UnaryOperator(DRE, UO_Deref, DRE->getType(),
+                                          VK_LValue, OK_Ordinary,
+                                          DRE->getLocation());
+  // Need parens to enforce precedence.
+  ParenExpr *PE = new (Context) ParenExpr(SourceLocation(), SourceLocation(), 
+                                          Exp);
+  ReplaceStmt(DRE, PE);
+  return PE;
+}
+
+void RewriteObjC::RewriteCastExpr(CStyleCastExpr *CE) {
+  SourceLocation LocStart = CE->getLParenLoc();
+  SourceLocation LocEnd = CE->getRParenLoc();
+
+  // Need to avoid trying to rewrite synthesized casts.
+  if (LocStart.isInvalid())
+    return;
+  // Need to avoid trying to rewrite casts contained in macros.
+  if (!Rewriter::isRewritable(LocStart) || !Rewriter::isRewritable(LocEnd))
+    return;
+
+  const char *startBuf = SM->getCharacterData(LocStart);
+  const char *endBuf = SM->getCharacterData(LocEnd);
+  QualType QT = CE->getType();
+  const Type* TypePtr = QT->getAs<Type>();
+  if (isa<TypeOfExprType>(TypePtr)) {
+    const TypeOfExprType *TypeOfExprTypePtr = cast<TypeOfExprType>(TypePtr);
+    QT = TypeOfExprTypePtr->getUnderlyingExpr()->getType();
+    std::string TypeAsString = "(";
+    RewriteBlockPointerType(TypeAsString, QT);
+    TypeAsString += ")";
+    ReplaceText(LocStart, endBuf-startBuf+1, TypeAsString);
+    return;
+  }
+  // advance the location to startArgList.
+  const char *argPtr = startBuf;
+
+  while (*argPtr++ && (argPtr < endBuf)) {
+    switch (*argPtr) {
+    case '^':
+      // Replace the '^' with '*'.
+      LocStart = LocStart.getLocWithOffset(argPtr-startBuf);
+      ReplaceText(LocStart, 1, "*");
+      break;
+    }
+  }
+  return;
+}
+
+void RewriteObjC::RewriteBlockPointerFunctionArgs(FunctionDecl *FD) {
+  SourceLocation DeclLoc = FD->getLocation();
+  unsigned parenCount = 0;
+
+  // We have 1 or more arguments that have closure pointers.
+  const char *startBuf = SM->getCharacterData(DeclLoc);
+  const char *startArgList = strchr(startBuf, '(');
+
+  assert((*startArgList == '(') && "Rewriter fuzzy parser confused");
+
+  parenCount++;
+  // advance the location to startArgList.
+  DeclLoc = DeclLoc.getLocWithOffset(startArgList-startBuf);
+  assert((DeclLoc.isValid()) && "Invalid DeclLoc");
+
+  const char *argPtr = startArgList;
+
+  while (*argPtr++ && parenCount) {
+    switch (*argPtr) {
+    case '^':
+      // Replace the '^' with '*'.
+      DeclLoc = DeclLoc.getLocWithOffset(argPtr-startArgList);
+      ReplaceText(DeclLoc, 1, "*");
+      break;
+    case '(':
+      parenCount++;
+      break;
+    case ')':
+      parenCount--;
+      break;
+    }
+  }
+  return;
+}
+
+bool RewriteObjC::PointerTypeTakesAnyBlockArguments(QualType QT) {
+  const FunctionProtoType *FTP;
+  const PointerType *PT = QT->getAs<PointerType>();
+  if (PT) {
+    FTP = PT->getPointeeType()->getAs<FunctionProtoType>();
+  } else {
+    const BlockPointerType *BPT = QT->getAs<BlockPointerType>();
+    assert(BPT && "BlockPointerTypeTakeAnyBlockArguments(): not a block pointer type");
+    FTP = BPT->getPointeeType()->getAs<FunctionProtoType>();
+  }
+  if (FTP) {
+    for (FunctionProtoType::arg_type_iterator I = FTP->arg_type_begin(),
+         E = FTP->arg_type_end(); I != E; ++I)
+      if (isTopLevelBlockPointerType(*I))
+        return true;
+  }
+  return false;
+}
+
+bool RewriteObjC::PointerTypeTakesAnyObjCQualifiedType(QualType QT) {
+  const FunctionProtoType *FTP;
+  const PointerType *PT = QT->getAs<PointerType>();
+  if (PT) {
+    FTP = PT->getPointeeType()->getAs<FunctionProtoType>();
+  } else {
+    const BlockPointerType *BPT = QT->getAs<BlockPointerType>();
+    assert(BPT && "BlockPointerTypeTakeAnyBlockArguments(): not a block pointer type");
+    FTP = BPT->getPointeeType()->getAs<FunctionProtoType>();
+  }
+  if (FTP) {
+    for (FunctionProtoType::arg_type_iterator I = FTP->arg_type_begin(),
+         E = FTP->arg_type_end(); I != E; ++I) {
+      if ((*I)->isObjCQualifiedIdType())
+        return true;
+      if ((*I)->isObjCObjectPointerType() &&
+          (*I)->getPointeeType()->isObjCQualifiedInterfaceType())
+        return true;
+    }
+        
+  }
+  return false;
+}
+
+void RewriteObjC::GetExtentOfArgList(const char *Name, const char *&LParen,
+                                     const char *&RParen) {
+  const char *argPtr = strchr(Name, '(');
+  assert((*argPtr == '(') && "Rewriter fuzzy parser confused");
+
+  LParen = argPtr; // output the start.
+  argPtr++; // skip past the left paren.
+  unsigned parenCount = 1;
+
+  while (*argPtr && parenCount) {
+    switch (*argPtr) {
+    case '(': parenCount++; break;
+    case ')': parenCount--; break;
+    default: break;
+    }
+    if (parenCount) argPtr++;
+  }
+  assert((*argPtr == ')') && "Rewriter fuzzy parser confused");
+  RParen = argPtr; // output the end
+}
+
+void RewriteObjC::RewriteBlockPointerDecl(NamedDecl *ND) {
+  if (FunctionDecl *FD = dyn_cast<FunctionDecl>(ND)) {
+    RewriteBlockPointerFunctionArgs(FD);
+    return;
+  }
+  // Handle Variables and Typedefs.
+  SourceLocation DeclLoc = ND->getLocation();
+  QualType DeclT;
+  if (VarDecl *VD = dyn_cast<VarDecl>(ND))
+    DeclT = VD->getType();
+  else if (TypedefNameDecl *TDD = dyn_cast<TypedefNameDecl>(ND))
+    DeclT = TDD->getUnderlyingType();
+  else if (FieldDecl *FD = dyn_cast<FieldDecl>(ND))
+    DeclT = FD->getType();
+  else
+    llvm_unreachable("RewriteBlockPointerDecl(): Decl type not yet handled");
+
+  const char *startBuf = SM->getCharacterData(DeclLoc);
+  const char *endBuf = startBuf;
+  // scan backward (from the decl location) for the end of the previous decl.
+  while (*startBuf != '^' && *startBuf != ';' && startBuf != MainFileStart)
+    startBuf--;
+  SourceLocation Start = DeclLoc.getLocWithOffset(startBuf-endBuf);
+  std::string buf;
+  unsigned OrigLength=0;
+  // *startBuf != '^' if we are dealing with a pointer to function that
+  // may take block argument types (which will be handled below).
+  if (*startBuf == '^') {
+    // Replace the '^' with '*', computing a negative offset.
+    buf = '*';
+    startBuf++;
+    OrigLength++;
+  }
+  while (*startBuf != ')') {
+    buf += *startBuf;
+    startBuf++;
+    OrigLength++;
+  }
+  buf += ')';
+  OrigLength++;
+  
+  if (PointerTypeTakesAnyBlockArguments(DeclT) ||
+      PointerTypeTakesAnyObjCQualifiedType(DeclT)) {
+    // Replace the '^' with '*' for arguments.
+    // Replace id<P> with id/*<>*/
+    DeclLoc = ND->getLocation();
+    startBuf = SM->getCharacterData(DeclLoc);
+    const char *argListBegin, *argListEnd;
+    GetExtentOfArgList(startBuf, argListBegin, argListEnd);
+    while (argListBegin < argListEnd) {
+      if (*argListBegin == '^')
+        buf += '*';
+      else if (*argListBegin ==  '<') {
+        buf += "/*"; 
+        buf += *argListBegin++;
+        OrigLength++;
+        while (*argListBegin != '>') {
+          buf += *argListBegin++;
+          OrigLength++;
+        }
+        buf += *argListBegin;
+        buf += "*/";
+      }
+      else
+        buf += *argListBegin;
+      argListBegin++;
+      OrigLength++;
+    }
+    buf += ')';
+    OrigLength++;
+  }
+  ReplaceText(Start, OrigLength, buf);
+  
+  return;
+}
+
+
+/// SynthesizeByrefCopyDestroyHelper - This routine synthesizes:
+/// void __Block_byref_id_object_copy(struct Block_byref_id_object *dst,
+///                    struct Block_byref_id_object *src) {
+///  _Block_object_assign (&_dest->object, _src->object, 
+///                        BLOCK_BYREF_CALLER | BLOCK_FIELD_IS_OBJECT
+///                        [|BLOCK_FIELD_IS_WEAK]) // object
+///  _Block_object_assign(&_dest->object, _src->object, 
+///                       BLOCK_BYREF_CALLER | BLOCK_FIELD_IS_BLOCK
+///                       [|BLOCK_FIELD_IS_WEAK]) // block
+/// }
+/// And:
+/// void __Block_byref_id_object_dispose(struct Block_byref_id_object *_src) {
+///  _Block_object_dispose(_src->object, 
+///                        BLOCK_BYREF_CALLER | BLOCK_FIELD_IS_OBJECT
+///                        [|BLOCK_FIELD_IS_WEAK]) // object
+///  _Block_object_dispose(_src->object, 
+///                         BLOCK_BYREF_CALLER | BLOCK_FIELD_IS_BLOCK
+///                         [|BLOCK_FIELD_IS_WEAK]) // block
+/// }
+
+std::string RewriteObjC::SynthesizeByrefCopyDestroyHelper(VarDecl *VD,
+                                                          int flag) {
+  std::string S;
+  if (CopyDestroyCache.count(flag))
+    return S;
+  CopyDestroyCache.insert(flag);
+  S = "static void __Block_byref_id_object_copy_";
+  S += utostr(flag);
+  S += "(void *dst, void *src) {\n";
+  
+  // offset into the object pointer is computed as:
+  // void * + void* + int + int + void* + void *
+  unsigned IntSize = 
+  static_cast<unsigned>(Context->getTypeSize(Context->IntTy));
+  unsigned VoidPtrSize = 
+  static_cast<unsigned>(Context->getTypeSize(Context->VoidPtrTy));
+  
+  unsigned offset = (VoidPtrSize*4 + IntSize + IntSize)/Context->getCharWidth();
+  S += " _Block_object_assign((char*)dst + ";
+  S += utostr(offset);
+  S += ", *(void * *) ((char*)src + ";
+  S += utostr(offset);
+  S += "), ";
+  S += utostr(flag);
+  S += ");\n}\n";
+  
+  S += "static void __Block_byref_id_object_dispose_";
+  S += utostr(flag);
+  S += "(void *src) {\n";
+  S += " _Block_object_dispose(*(void * *) ((char*)src + ";
+  S += utostr(offset);
+  S += "), ";
+  S += utostr(flag);
+  S += ");\n}\n";
+  return S;
+}
+
+/// RewriteByRefVar - For each __block typex ND variable this routine transforms
+/// the declaration into:
+/// struct __Block_byref_ND {
+/// void *__isa;                  // NULL for everything except __weak pointers
+/// struct __Block_byref_ND *__forwarding;
+/// int32_t __flags;
+/// int32_t __size;
+/// void *__Block_byref_id_object_copy; // If variable is __block ObjC object
+/// void *__Block_byref_id_object_dispose; // If variable is __block ObjC object
+/// typex ND;
+/// };
+///
+/// It then replaces declaration of ND variable with:
+/// struct __Block_byref_ND ND = {__isa=0B, __forwarding=&ND, __flags=some_flag, 
+///                               __size=sizeof(struct __Block_byref_ND), 
+///                               ND=initializer-if-any};
+///
+///
+void RewriteObjC::RewriteByRefVar(VarDecl *ND) {
+  // Insert declaration for the function in which block literal is
+  // used.
+  if (CurFunctionDeclToDeclareForBlock)
+    RewriteBlockLiteralFunctionDecl(CurFunctionDeclToDeclareForBlock);
+  int flag = 0;
+  int isa = 0;
+  SourceLocation DeclLoc = ND->getTypeSpecStartLoc();
+  if (DeclLoc.isInvalid())
+    // If type location is missing, it is because of missing type (a warning).
+    // Use variable's location which is good for this case.
+    DeclLoc = ND->getLocation();
+  const char *startBuf = SM->getCharacterData(DeclLoc);
+  SourceLocation X = ND->getLocEnd();
+  X = SM->getExpansionLoc(X);
+  const char *endBuf = SM->getCharacterData(X);
+  std::string Name(ND->getNameAsString());
+  std::string ByrefType;
+  RewriteByRefString(ByrefType, Name, ND, true);
+  ByrefType += " {\n";
+  ByrefType += "  void *__isa;\n";
+  RewriteByRefString(ByrefType, Name, ND);
+  ByrefType += " *__forwarding;\n";
+  ByrefType += " int __flags;\n";
+  ByrefType += " int __size;\n";
+  // Add void *__Block_byref_id_object_copy; 
+  // void *__Block_byref_id_object_dispose; if needed.
+  QualType Ty = ND->getType();
+  bool HasCopyAndDispose = Context->BlockRequiresCopying(Ty);
+  if (HasCopyAndDispose) {
+    ByrefType += " void (*__Block_byref_id_object_copy)(void*, void*);\n";
+    ByrefType += " void (*__Block_byref_id_object_dispose)(void*);\n";
+  }
+
+  QualType T = Ty;
+  (void)convertBlockPointerToFunctionPointer(T);
+  T.getAsStringInternal(Name, Context->getPrintingPolicy());
+    
+  ByrefType += " " + Name + ";\n";
+  ByrefType += "};\n";
+  // Insert this type in global scope. It is needed by helper function.
+  SourceLocation FunLocStart;
+  if (CurFunctionDef)
+     FunLocStart = CurFunctionDef->getTypeSpecStartLoc();
+  else {
+    assert(CurMethodDef && "RewriteByRefVar - CurMethodDef is null");
+    FunLocStart = CurMethodDef->getLocStart();
+  }
+  InsertText(FunLocStart, ByrefType);
+  if (Ty.isObjCGCWeak()) {
+    flag |= BLOCK_FIELD_IS_WEAK;
+    isa = 1;
+  }
+  
+  if (HasCopyAndDispose) {
+    flag = BLOCK_BYREF_CALLER;
+    QualType Ty = ND->getType();
+    // FIXME. Handle __weak variable (BLOCK_FIELD_IS_WEAK) as well.
+    if (Ty->isBlockPointerType())
+      flag |= BLOCK_FIELD_IS_BLOCK;
+    else
+      flag |= BLOCK_FIELD_IS_OBJECT;
+    std::string HF = SynthesizeByrefCopyDestroyHelper(ND, flag);
+    if (!HF.empty())
+      InsertText(FunLocStart, HF);
+  }
+  
+  // struct __Block_byref_ND ND = 
+  // {0, &ND, some_flag, __size=sizeof(struct __Block_byref_ND), 
+  //  initializer-if-any};
+  bool hasInit = (ND->getInit() != 0);
+  unsigned flags = 0;
+  if (HasCopyAndDispose)
+    flags |= BLOCK_HAS_COPY_DISPOSE;
+  Name = ND->getNameAsString();
+  ByrefType.clear();
+  RewriteByRefString(ByrefType, Name, ND);
+  std::string ForwardingCastType("(");
+  ForwardingCastType += ByrefType + " *)";
+  if (!hasInit) {
+    ByrefType += " " + Name + " = {(void*)";
+    ByrefType += utostr(isa);
+    ByrefType += "," +  ForwardingCastType + "&" + Name + ", ";
+    ByrefType += utostr(flags);
+    ByrefType += ", ";
+    ByrefType += "sizeof(";
+    RewriteByRefString(ByrefType, Name, ND);
+    ByrefType += ")";
+    if (HasCopyAndDispose) {
+      ByrefType += ", __Block_byref_id_object_copy_";
+      ByrefType += utostr(flag);
+      ByrefType += ", __Block_byref_id_object_dispose_";
+      ByrefType += utostr(flag);
+    }
+    ByrefType += "};\n";
+    unsigned nameSize = Name.size();
+    // for block or function pointer declaration. Name is aleady
+    // part of the declaration.
+    if (Ty->isBlockPointerType() || Ty->isFunctionPointerType())
+      nameSize = 1;
+    ReplaceText(DeclLoc, endBuf-startBuf+nameSize, ByrefType);
+  }
+  else {
+    SourceLocation startLoc;
+    Expr *E = ND->getInit();
+    if (const CStyleCastExpr *ECE = dyn_cast<CStyleCastExpr>(E))
+      startLoc = ECE->getLParenLoc();
+    else
+      startLoc = E->getLocStart();
+    startLoc = SM->getExpansionLoc(startLoc);
+    endBuf = SM->getCharacterData(startLoc);
+    ByrefType += " " + Name;
+    ByrefType += " = {(void*)";
+    ByrefType += utostr(isa);
+    ByrefType += "," +  ForwardingCastType + "&" + Name + ", ";
+    ByrefType += utostr(flags);
+    ByrefType += ", ";
+    ByrefType += "sizeof(";
+    RewriteByRefString(ByrefType, Name, ND);
+    ByrefType += "), ";
+    if (HasCopyAndDispose) {
+      ByrefType += "__Block_byref_id_object_copy_";
+      ByrefType += utostr(flag);
+      ByrefType += ", __Block_byref_id_object_dispose_";
+      ByrefType += utostr(flag);
+      ByrefType += ", ";
+    }
+    ReplaceText(DeclLoc, endBuf-startBuf, ByrefType);
+    
+    // Complete the newly synthesized compound expression by inserting a right
+    // curly brace before the end of the declaration.
+    // FIXME: This approach avoids rewriting the initializer expression. It
+    // also assumes there is only one declarator. For example, the following
+    // isn't currently supported by this routine (in general):
+    // 
+    // double __block BYREFVAR = 1.34, BYREFVAR2 = 1.37;
+    //
+    const char *startInitializerBuf = SM->getCharacterData(startLoc);
+    const char *semiBuf = strchr(startInitializerBuf, ';');
+    assert((*semiBuf == ';') && "RewriteByRefVar: can't find ';'");
+    SourceLocation semiLoc =
+      startLoc.getLocWithOffset(semiBuf-startInitializerBuf);
+
+    InsertText(semiLoc, "}");
+  }
+  return;
+}
+
+void RewriteObjC::CollectBlockDeclRefInfo(BlockExpr *Exp) {
+  // Add initializers for any closure decl refs.
+  GetBlockDeclRefExprs(Exp->getBody());
+  if (BlockDeclRefs.size()) {
+    // Unique all "by copy" declarations.
+    for (unsigned i = 0; i < BlockDeclRefs.size(); i++)
+      if (!BlockDeclRefs[i]->getDecl()->hasAttr<BlocksAttr>()) {
+        if (!BlockByCopyDeclsPtrSet.count(BlockDeclRefs[i]->getDecl())) {
+          BlockByCopyDeclsPtrSet.insert(BlockDeclRefs[i]->getDecl());
+          BlockByCopyDecls.push_back(BlockDeclRefs[i]->getDecl());
+        }
+      }
+    // Unique all "by ref" declarations.
+    for (unsigned i = 0; i < BlockDeclRefs.size(); i++)
+      if (BlockDeclRefs[i]->getDecl()->hasAttr<BlocksAttr>()) {
+        if (!BlockByRefDeclsPtrSet.count(BlockDeclRefs[i]->getDecl())) {
+          BlockByRefDeclsPtrSet.insert(BlockDeclRefs[i]->getDecl());
+          BlockByRefDecls.push_back(BlockDeclRefs[i]->getDecl());
+        }
+      }
+    // Find any imported blocks...they will need special attention.
+    for (unsigned i = 0; i < BlockDeclRefs.size(); i++)
+      if (BlockDeclRefs[i]->getDecl()->hasAttr<BlocksAttr>() ||
+          BlockDeclRefs[i]->getType()->isObjCObjectPointerType() || 
+          BlockDeclRefs[i]->getType()->isBlockPointerType())
+        ImportedBlockDecls.insert(BlockDeclRefs[i]->getDecl());
+  }
+}
+
+FunctionDecl *RewriteObjC::SynthBlockInitFunctionDecl(StringRef name) {
+  IdentifierInfo *ID = &Context->Idents.get(name);
+  QualType FType = Context->getFunctionNoProtoType(Context->VoidPtrTy);
+  return FunctionDecl::Create(*Context, TUDecl, SourceLocation(),
+                              SourceLocation(), ID, FType, 0, SC_Extern,
+                              SC_None, false, false);
+}
+
+Stmt *RewriteObjC::SynthBlockInitExpr(BlockExpr *Exp,
+          const SmallVector<DeclRefExpr *, 8> &InnerBlockDeclRefs) {
+  const BlockDecl *block = Exp->getBlockDecl();
+  Blocks.push_back(Exp);
+
+  CollectBlockDeclRefInfo(Exp);
+  
+  // Add inner imported variables now used in current block.
+ int countOfInnerDecls = 0;
+  if (!InnerBlockDeclRefs.empty()) {
+    for (unsigned i = 0; i < InnerBlockDeclRefs.size(); i++) {
+      DeclRefExpr *Exp = InnerBlockDeclRefs[i];
+      ValueDecl *VD = Exp->getDecl();
+      if (!VD->hasAttr<BlocksAttr>() && !BlockByCopyDeclsPtrSet.count(VD)) {
+      // We need to save the copied-in variables in nested
+      // blocks because it is needed at the end for some of the API generations.
+      // See SynthesizeBlockLiterals routine.
+        InnerDeclRefs.push_back(Exp); countOfInnerDecls++;
+        BlockDeclRefs.push_back(Exp);
+        BlockByCopyDeclsPtrSet.insert(VD);
+        BlockByCopyDecls.push_back(VD);
+      }
+      if (VD->hasAttr<BlocksAttr>() && !BlockByRefDeclsPtrSet.count(VD)) {
+        InnerDeclRefs.push_back(Exp); countOfInnerDecls++;
+        BlockDeclRefs.push_back(Exp);
+        BlockByRefDeclsPtrSet.insert(VD);
+        BlockByRefDecls.push_back(VD);
+      }
+    }
+    // Find any imported blocks...they will need special attention.
+    for (unsigned i = 0; i < InnerBlockDeclRefs.size(); i++)
+      if (InnerBlockDeclRefs[i]->getDecl()->hasAttr<BlocksAttr>() ||
+          InnerBlockDeclRefs[i]->getType()->isObjCObjectPointerType() || 
+          InnerBlockDeclRefs[i]->getType()->isBlockPointerType())
+        ImportedBlockDecls.insert(InnerBlockDeclRefs[i]->getDecl());
+  }
+  InnerDeclRefsCount.push_back(countOfInnerDecls);
+  
+  std::string FuncName;
+
+  if (CurFunctionDef)
+    FuncName = CurFunctionDef->getNameAsString();
+  else if (CurMethodDef)
+    BuildUniqueMethodName(FuncName, CurMethodDef);
+  else if (GlobalVarDecl)
+    FuncName = std::string(GlobalVarDecl->getNameAsString());
+
+  std::string BlockNumber = utostr(Blocks.size()-1);
+
+  std::string Tag = "__" + FuncName + "_block_impl_" + BlockNumber;
+  std::string Func = "__" + FuncName + "_block_func_" + BlockNumber;
+
+  // Get a pointer to the function type so we can cast appropriately.
+  QualType BFT = convertFunctionTypeOfBlocks(Exp->getFunctionType());
+  QualType FType = Context->getPointerType(BFT);
+
+  FunctionDecl *FD;
+  Expr *NewRep;
+
+  // Simulate a contructor call...
+  FD = SynthBlockInitFunctionDecl(Tag);
+  DeclRefExpr *DRE = new (Context) DeclRefExpr(FD, false, FType, VK_RValue,
+                                               SourceLocation());
+
+  SmallVector<Expr*, 4> InitExprs;
+
+  // Initialize the block function.
+  FD = SynthBlockInitFunctionDecl(Func);
+  DeclRefExpr *Arg = new (Context) DeclRefExpr(FD, false, FD->getType(),
+                                               VK_LValue, SourceLocation());
+  CastExpr *castExpr = NoTypeInfoCStyleCastExpr(Context, Context->VoidPtrTy,
+                                                CK_BitCast, Arg);
+  InitExprs.push_back(castExpr);
+
+  // Initialize the block descriptor.
+  std::string DescData = "__" + FuncName + "_block_desc_" + BlockNumber + "_DATA";
+
+  VarDecl *NewVD = VarDecl::Create(*Context, TUDecl,
+                                   SourceLocation(), SourceLocation(),
+                                   &Context->Idents.get(DescData.c_str()),
+                                   Context->VoidPtrTy, 0,
+                                   SC_Static, SC_None);
+  UnaryOperator *DescRefExpr =
+    new (Context) UnaryOperator(new (Context) DeclRefExpr(NewVD, false,
+                                                          Context->VoidPtrTy,
+                                                          VK_LValue,
+                                                          SourceLocation()), 
+                                UO_AddrOf,
+                                Context->getPointerType(Context->VoidPtrTy), 
+                                VK_RValue, OK_Ordinary,
+                                SourceLocation());
+  InitExprs.push_back(DescRefExpr); 
+  
+  // Add initializers for any closure decl refs.
+  if (BlockDeclRefs.size()) {
+    Expr *Exp;
+    // Output all "by copy" declarations.
+    for (SmallVector<ValueDecl*,8>::iterator I = BlockByCopyDecls.begin(),
+         E = BlockByCopyDecls.end(); I != E; ++I) {
+      if (isObjCType((*I)->getType())) {
+        // FIXME: Conform to ABI ([[obj retain] autorelease]).
+        FD = SynthBlockInitFunctionDecl((*I)->getName());
+        Exp = new (Context) DeclRefExpr(FD, false, FD->getType(), VK_LValue,
+                                        SourceLocation());
+        if (HasLocalVariableExternalStorage(*I)) {
+          QualType QT = (*I)->getType();
+          QT = Context->getPointerType(QT);
+          Exp = new (Context) UnaryOperator(Exp, UO_AddrOf, QT, VK_RValue,
+                                            OK_Ordinary, SourceLocation());
+        }
+      } else if (isTopLevelBlockPointerType((*I)->getType())) {
+        FD = SynthBlockInitFunctionDecl((*I)->getName());
+        Arg = new (Context) DeclRefExpr(FD, false, FD->getType(), VK_LValue,
+                                        SourceLocation());
+        Exp = NoTypeInfoCStyleCastExpr(Context, Context->VoidPtrTy,
+                                       CK_BitCast, Arg);
+      } else {
+        FD = SynthBlockInitFunctionDecl((*I)->getName());
+        Exp = new (Context) DeclRefExpr(FD, false, FD->getType(), VK_LValue,
+                                        SourceLocation());
+        if (HasLocalVariableExternalStorage(*I)) {
+          QualType QT = (*I)->getType();
+          QT = Context->getPointerType(QT);
+          Exp = new (Context) UnaryOperator(Exp, UO_AddrOf, QT, VK_RValue,
+                                            OK_Ordinary, SourceLocation());
+        }
+        
+      }
+      InitExprs.push_back(Exp);
+    }
+    // Output all "by ref" declarations.
+    for (SmallVector<ValueDecl*,8>::iterator I = BlockByRefDecls.begin(),
+         E = BlockByRefDecls.end(); I != E; ++I) {
+      ValueDecl *ND = (*I);
+      std::string Name(ND->getNameAsString());
+      std::string RecName;
+      RewriteByRefString(RecName, Name, ND, true);
+      IdentifierInfo *II = &Context->Idents.get(RecName.c_str() 
+                                                + sizeof("struct"));
+      RecordDecl *RD = RecordDecl::Create(*Context, TTK_Struct, TUDecl,
+                                          SourceLocation(), SourceLocation(),
+                                          II);
+      assert(RD && "SynthBlockInitExpr(): Can't find RecordDecl");
+      QualType castT = Context->getPointerType(Context->getTagDeclType(RD));
+      
+      FD = SynthBlockInitFunctionDecl((*I)->getName());
+      Exp = new (Context) DeclRefExpr(FD, false, FD->getType(), VK_LValue,
+                                      SourceLocation());
+      bool isNestedCapturedVar = false;
+      if (block)
+        for (BlockDecl::capture_const_iterator ci = block->capture_begin(),
+             ce = block->capture_end(); ci != ce; ++ci) {
+          const VarDecl *variable = ci->getVariable();
+          if (variable == ND && ci->isNested()) {
+            assert (ci->isByRef() && 
+                    "SynthBlockInitExpr - captured block variable is not byref");
+            isNestedCapturedVar = true;
+            break;
+          }
+        }
+      // captured nested byref variable has its address passed. Do not take
+      // its address again.
+      if (!isNestedCapturedVar)
+          Exp = new (Context) UnaryOperator(Exp, UO_AddrOf,
+                                     Context->getPointerType(Exp->getType()),
+                                     VK_RValue, OK_Ordinary, SourceLocation());
+      Exp = NoTypeInfoCStyleCastExpr(Context, castT, CK_BitCast, Exp);
+      InitExprs.push_back(Exp);
+    }
+  }
+  if (ImportedBlockDecls.size()) {
+    // generate BLOCK_HAS_COPY_DISPOSE(have helper funcs) | BLOCK_HAS_DESCRIPTOR
+    int flag = (BLOCK_HAS_COPY_DISPOSE | BLOCK_HAS_DESCRIPTOR);
+    unsigned IntSize = 
+      static_cast<unsigned>(Context->getTypeSize(Context->IntTy));
+    Expr *FlagExp = IntegerLiteral::Create(*Context, llvm::APInt(IntSize, flag), 
+                                           Context->IntTy, SourceLocation());
+    InitExprs.push_back(FlagExp);
+  }
+  NewRep = new (Context) CallExpr(*Context, DRE, InitExprs,
+                                  FType, VK_LValue, SourceLocation());
+  NewRep = new (Context) UnaryOperator(NewRep, UO_AddrOf,
+                             Context->getPointerType(NewRep->getType()),
+                             VK_RValue, OK_Ordinary, SourceLocation());
+  NewRep = NoTypeInfoCStyleCastExpr(Context, FType, CK_BitCast,
+                                    NewRep);
+  BlockDeclRefs.clear();
+  BlockByRefDecls.clear();
+  BlockByRefDeclsPtrSet.clear();
+  BlockByCopyDecls.clear();
+  BlockByCopyDeclsPtrSet.clear();
+  ImportedBlockDecls.clear();
+  return NewRep;
+}
+
+bool RewriteObjC::IsDeclStmtInForeachHeader(DeclStmt *DS) {
+  if (const ObjCForCollectionStmt * CS = 
+      dyn_cast<ObjCForCollectionStmt>(Stmts.back()))
+        return CS->getElement() == DS;
+  return false;
+}
+
+//===----------------------------------------------------------------------===//
+// Function Body / Expression rewriting
+//===----------------------------------------------------------------------===//
+
+Stmt *RewriteObjC::RewriteFunctionBodyOrGlobalInitializer(Stmt *S) {
+  if (isa<SwitchStmt>(S) || isa<WhileStmt>(S) ||
+      isa<DoStmt>(S) || isa<ForStmt>(S))
+    Stmts.push_back(S);
+  else if (isa<ObjCForCollectionStmt>(S)) {
+    Stmts.push_back(S);
+    ObjCBcLabelNo.push_back(++BcLabelCount);
+  }
+
+  // Pseudo-object operations and ivar references need special
+  // treatment because we're going to recursively rewrite them.
+  if (PseudoObjectExpr *PseudoOp = dyn_cast<PseudoObjectExpr>(S)) {
+    if (isa<BinaryOperator>(PseudoOp->getSyntacticForm())) {
+      return RewritePropertyOrImplicitSetter(PseudoOp);
+    } else {
+      return RewritePropertyOrImplicitGetter(PseudoOp);
+    }
+  } else if (ObjCIvarRefExpr *IvarRefExpr = dyn_cast<ObjCIvarRefExpr>(S)) {
+    return RewriteObjCIvarRefExpr(IvarRefExpr);
+  }
+
+  SourceRange OrigStmtRange = S->getSourceRange();
+
+  // Perform a bottom up rewrite of all children.
+  for (Stmt::child_range CI = S->children(); CI; ++CI)
+    if (*CI) {
+      Stmt *childStmt = (*CI);
+      Stmt *newStmt = RewriteFunctionBodyOrGlobalInitializer(childStmt);
+      if (newStmt) {
+        *CI = newStmt;
+      }
+    }
+
+  if (BlockExpr *BE = dyn_cast<BlockExpr>(S)) {
+    SmallVector<DeclRefExpr *, 8> InnerBlockDeclRefs;
+    llvm::SmallPtrSet<const DeclContext *, 8> InnerContexts;
+    InnerContexts.insert(BE->getBlockDecl());
+    ImportedLocalExternalDecls.clear();
+    GetInnerBlockDeclRefExprs(BE->getBody(),
+                              InnerBlockDeclRefs, InnerContexts);
+    // Rewrite the block body in place.
+    Stmt *SaveCurrentBody = CurrentBody;
+    CurrentBody = BE->getBody();
+    PropParentMap = 0;
+    // block literal on rhs of a property-dot-sytax assignment
+    // must be replaced by its synthesize ast so getRewrittenText
+    // works as expected. In this case, what actually ends up on RHS
+    // is the blockTranscribed which is the helper function for the
+    // block literal; as in: self.c = ^() {[ace ARR];};
+    bool saveDisableReplaceStmt = DisableReplaceStmt;
+    DisableReplaceStmt = false;
+    RewriteFunctionBodyOrGlobalInitializer(BE->getBody());
+    DisableReplaceStmt = saveDisableReplaceStmt;
+    CurrentBody = SaveCurrentBody;
+    PropParentMap = 0;
+    ImportedLocalExternalDecls.clear();
+    // Now we snarf the rewritten text and stash it away for later use.
+    std::string Str = Rewrite.getRewrittenText(BE->getSourceRange());
+    RewrittenBlockExprs[BE] = Str;
+
+    Stmt *blockTranscribed = SynthBlockInitExpr(BE, InnerBlockDeclRefs);
+                            
+    //blockTranscribed->dump();
+    ReplaceStmt(S, blockTranscribed);
+    return blockTranscribed;
+  }
+  // Handle specific things.
+  if (ObjCEncodeExpr *AtEncode = dyn_cast<ObjCEncodeExpr>(S))
+    return RewriteAtEncode(AtEncode);
+
+  if (ObjCSelectorExpr *AtSelector = dyn_cast<ObjCSelectorExpr>(S))
+    return RewriteAtSelector(AtSelector);
+
+  if (ObjCStringLiteral *AtString = dyn_cast<ObjCStringLiteral>(S))
+    return RewriteObjCStringLiteral(AtString);
+
+  if (ObjCMessageExpr *MessExpr = dyn_cast<ObjCMessageExpr>(S)) {
+#if 0
+    // Before we rewrite it, put the original message expression in a comment.
+    SourceLocation startLoc = MessExpr->getLocStart();
+    SourceLocation endLoc = MessExpr->getLocEnd();
+
+    const char *startBuf = SM->getCharacterData(startLoc);
+    const char *endBuf = SM->getCharacterData(endLoc);
+
+    std::string messString;
+    messString += "// ";
+    messString.append(startBuf, endBuf-startBuf+1);
+    messString += "\n";
+
+    // FIXME: Missing definition of
+    // InsertText(clang::SourceLocation, char const*, unsigned int).
+    // InsertText(startLoc, messString.c_str(), messString.size());
+    // Tried this, but it didn't work either...
+    // ReplaceText(startLoc, 0, messString.c_str(), messString.size());
+#endif
+    return RewriteMessageExpr(MessExpr);
+  }
+
+  if (ObjCAtTryStmt *StmtTry = dyn_cast<ObjCAtTryStmt>(S))
+    return RewriteObjCTryStmt(StmtTry);
+
+  if (ObjCAtSynchronizedStmt *StmtTry = dyn_cast<ObjCAtSynchronizedStmt>(S))
+    return RewriteObjCSynchronizedStmt(StmtTry);
+
+  if (ObjCAtThrowStmt *StmtThrow = dyn_cast<ObjCAtThrowStmt>(S))
+    return RewriteObjCThrowStmt(StmtThrow);
+
+  if (ObjCProtocolExpr *ProtocolExp = dyn_cast<ObjCProtocolExpr>(S))
+    return RewriteObjCProtocolExpr(ProtocolExp);
+
+  if (ObjCForCollectionStmt *StmtForCollection =
+        dyn_cast<ObjCForCollectionStmt>(S))
+    return RewriteObjCForCollectionStmt(StmtForCollection,
+                                        OrigStmtRange.getEnd());
+  if (BreakStmt *StmtBreakStmt =
+      dyn_cast<BreakStmt>(S))
+    return RewriteBreakStmt(StmtBreakStmt);
+  if (ContinueStmt *StmtContinueStmt =
+      dyn_cast<ContinueStmt>(S))
+    return RewriteContinueStmt(StmtContinueStmt);
+
+  // Need to check for protocol refs (id <P>, Foo <P> *) in variable decls
+  // and cast exprs.
+  if (DeclStmt *DS = dyn_cast<DeclStmt>(S)) {
+    // FIXME: What we're doing here is modifying the type-specifier that
+    // precedes the first Decl.  In the future the DeclGroup should have
+    // a separate type-specifier that we can rewrite.
+    // NOTE: We need to avoid rewriting the DeclStmt if it is within
+    // the context of an ObjCForCollectionStmt. For example:
+    //   NSArray *someArray;
+    //   for (id <FooProtocol> index in someArray) ;
+    // This is because RewriteObjCForCollectionStmt() does textual rewriting 
+    // and it depends on the original text locations/positions.
+    if (Stmts.empty() || !IsDeclStmtInForeachHeader(DS))
+      RewriteObjCQualifiedInterfaceTypes(*DS->decl_begin());
+
+    // Blocks rewrite rules.
+    for (DeclStmt::decl_iterator DI = DS->decl_begin(), DE = DS->decl_end();
+         DI != DE; ++DI) {
+      Decl *SD = *DI;
+      if (ValueDecl *ND = dyn_cast<ValueDecl>(SD)) {
+        if (isTopLevelBlockPointerType(ND->getType()))
+          RewriteBlockPointerDecl(ND);
+        else if (ND->getType()->isFunctionPointerType())
+          CheckFunctionPointerDecl(ND->getType(), ND);
+        if (VarDecl *VD = dyn_cast<VarDecl>(SD)) {
+          if (VD->hasAttr<BlocksAttr>()) {
+            static unsigned uniqueByrefDeclCount = 0;
+            assert(!BlockByRefDeclNo.count(ND) &&
+              "RewriteFunctionBodyOrGlobalInitializer: Duplicate byref decl");
+            BlockByRefDeclNo[ND] = uniqueByrefDeclCount++;
+            RewriteByRefVar(VD);
+          }
+          else           
+            RewriteTypeOfDecl(VD);
+        }
+      }
+      if (TypedefNameDecl *TD = dyn_cast<TypedefNameDecl>(SD)) {
+        if (isTopLevelBlockPointerType(TD->getUnderlyingType()))
+          RewriteBlockPointerDecl(TD);
+        else if (TD->getUnderlyingType()->isFunctionPointerType())
+          CheckFunctionPointerDecl(TD->getUnderlyingType(), TD);
+      }
+    }
+  }
+
+  if (CStyleCastExpr *CE = dyn_cast<CStyleCastExpr>(S))
+    RewriteObjCQualifiedInterfaceTypes(CE);
+
+  if (isa<SwitchStmt>(S) || isa<WhileStmt>(S) ||
+      isa<DoStmt>(S) || isa<ForStmt>(S)) {
+    assert(!Stmts.empty() && "Statement stack is empty");
+    assert ((isa<SwitchStmt>(Stmts.back()) || isa<WhileStmt>(Stmts.back()) ||
+             isa<DoStmt>(Stmts.back()) || isa<ForStmt>(Stmts.back()))
+            && "Statement stack mismatch");
+    Stmts.pop_back();
+  }
+  // Handle blocks rewriting.
+  if (DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(S)) {
+    ValueDecl *VD = DRE->getDecl(); 
+    if (VD->hasAttr<BlocksAttr>())
+      return RewriteBlockDeclRefExpr(DRE);
+    if (HasLocalVariableExternalStorage(VD))
+      return RewriteLocalVariableExternalStorage(DRE);
+  }
+  
+  if (CallExpr *CE = dyn_cast<CallExpr>(S)) {
+    if (CE->getCallee()->getType()->isBlockPointerType()) {
+      Stmt *BlockCall = SynthesizeBlockCall(CE, CE->getCallee());
+      ReplaceStmt(S, BlockCall);
+      return BlockCall;
+    }
+  }
+  if (CStyleCastExpr *CE = dyn_cast<CStyleCastExpr>(S)) {
+    RewriteCastExpr(CE);
+  }
+#if 0
+  if (ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(S)) {
+    CastExpr *Replacement = new (Context) CastExpr(ICE->getType(),
+                                                   ICE->getSubExpr(),
+                                                   SourceLocation());
+    // Get the new text.
+    std::string SStr;
+    llvm::raw_string_ostream Buf(SStr);
+    Replacement->printPretty(Buf);
+    const std::string &Str = Buf.str();
+
+    printf("CAST = %s\n", &Str[0]);
+    InsertText(ICE->getSubExpr()->getLocStart(), &Str[0], Str.size());
+    delete S;
+    return Replacement;
+  }
+#endif
+  // Return this stmt unmodified.
+  return S;
+}
+
+void RewriteObjC::RewriteRecordBody(RecordDecl *RD) {
+  for (RecordDecl::field_iterator i = RD->field_begin(), 
+                                  e = RD->field_end(); i != e; ++i) {
+    FieldDecl *FD = *i;
+    if (isTopLevelBlockPointerType(FD->getType()))
+      RewriteBlockPointerDecl(FD);
+    if (FD->getType()->isObjCQualifiedIdType() ||
+        FD->getType()->isObjCQualifiedInterfaceType())
+      RewriteObjCQualifiedInterfaceTypes(FD);
+  }
+}
+
+/// HandleDeclInMainFile - This is called for each top-level decl defined in the
+/// main file of the input.
+void RewriteObjC::HandleDeclInMainFile(Decl *D) {
+  switch (D->getKind()) {
+    case Decl::Function: {
+      FunctionDecl *FD = cast<FunctionDecl>(D);
+      if (FD->isOverloadedOperator())
+        return;
+
+      // Since function prototypes don't have ParmDecl's, we check the function
+      // prototype. This enables us to rewrite function declarations and
+      // definitions using the same code.
+      RewriteBlocksInFunctionProtoType(FD->getType(), FD);
+
+      if (!FD->isThisDeclarationADefinition())
+        break;
+
+      // FIXME: If this should support Obj-C++, support CXXTryStmt
+      if (CompoundStmt *Body = dyn_cast_or_null<CompoundStmt>(FD->getBody())) {
+        CurFunctionDef = FD;
+        CurFunctionDeclToDeclareForBlock = FD;
+        CurrentBody = Body;
+        Body =
+        cast_or_null<CompoundStmt>(RewriteFunctionBodyOrGlobalInitializer(Body));
+        FD->setBody(Body);
+        CurrentBody = 0;
+        if (PropParentMap) {
+          delete PropParentMap;
+          PropParentMap = 0;
+        }
+        // This synthesizes and inserts the block "impl" struct, invoke function,
+        // and any copy/dispose helper functions.
+        InsertBlockLiteralsWithinFunction(FD);
+        CurFunctionDef = 0;
+        CurFunctionDeclToDeclareForBlock = 0;
+      }
+      break;
+    }
+    case Decl::ObjCMethod: {
+      ObjCMethodDecl *MD = cast<ObjCMethodDecl>(D);
+      if (CompoundStmt *Body = MD->getCompoundBody()) {
+        CurMethodDef = MD;
+        CurrentBody = Body;
+        Body =
+          cast_or_null<CompoundStmt>(RewriteFunctionBodyOrGlobalInitializer(Body));
+        MD->setBody(Body);
+        CurrentBody = 0;
+        if (PropParentMap) {
+          delete PropParentMap;
+          PropParentMap = 0;
+        }
+        InsertBlockLiteralsWithinMethod(MD);
+        CurMethodDef = 0;
+      }
+      break;
+    }
+    case Decl::ObjCImplementation: {
+      ObjCImplementationDecl *CI = cast<ObjCImplementationDecl>(D);
+      ClassImplementation.push_back(CI);
+      break;
+    }
+    case Decl::ObjCCategoryImpl: {
+      ObjCCategoryImplDecl *CI = cast<ObjCCategoryImplDecl>(D);
+      CategoryImplementation.push_back(CI);
+      break;
+    }
+    case Decl::Var: {
+      VarDecl *VD = cast<VarDecl>(D);
+      RewriteObjCQualifiedInterfaceTypes(VD);
+      if (isTopLevelBlockPointerType(VD->getType()))
+        RewriteBlockPointerDecl(VD);
+      else if (VD->getType()->isFunctionPointerType()) {
+        CheckFunctionPointerDecl(VD->getType(), VD);
+        if (VD->getInit()) {
+          if (CStyleCastExpr *CE = dyn_cast<CStyleCastExpr>(VD->getInit())) {
+            RewriteCastExpr(CE);
+          }
+        }
+      } else if (VD->getType()->isRecordType()) {
+        RecordDecl *RD = VD->getType()->getAs<RecordType>()->getDecl();
+        if (RD->isCompleteDefinition())
+          RewriteRecordBody(RD);
+      }
+      if (VD->getInit()) {
+        GlobalVarDecl = VD;
+        CurrentBody = VD->getInit();
+        RewriteFunctionBodyOrGlobalInitializer(VD->getInit());
+        CurrentBody = 0;
+        if (PropParentMap) {
+          delete PropParentMap;
+          PropParentMap = 0;
+        }
+        SynthesizeBlockLiterals(VD->getTypeSpecStartLoc(), VD->getName());
+        GlobalVarDecl = 0;
+          
+        // This is needed for blocks.
+        if (CStyleCastExpr *CE = dyn_cast<CStyleCastExpr>(VD->getInit())) {
+            RewriteCastExpr(CE);
+        }
+      }
+      break;
+    }
+    case Decl::TypeAlias:
+    case Decl::Typedef: {
+      if (TypedefNameDecl *TD = dyn_cast<TypedefNameDecl>(D)) {
+        if (isTopLevelBlockPointerType(TD->getUnderlyingType()))
+          RewriteBlockPointerDecl(TD);
+        else if (TD->getUnderlyingType()->isFunctionPointerType())
+          CheckFunctionPointerDecl(TD->getUnderlyingType(), TD);
+      }
+      break;
+    }
+    case Decl::CXXRecord:
+    case Decl::Record: {
+      RecordDecl *RD = cast<RecordDecl>(D);
+      if (RD->isCompleteDefinition()) 
+        RewriteRecordBody(RD);
+      break;
+    }
+    default:
+      break;
+  }
+  // Nothing yet.
+}
+
+void RewriteObjC::HandleTranslationUnit(ASTContext &C) {
+  if (Diags.hasErrorOccurred())
+    return;
+
+  RewriteInclude();
+
+  // Here's a great place to add any extra declarations that may be needed.
+  // Write out meta data for each @protocol(<expr>).
+  for (llvm::SmallPtrSet<ObjCProtocolDecl *,8>::iterator I = ProtocolExprDecls.begin(),
+       E = ProtocolExprDecls.end(); I != E; ++I)
+    RewriteObjCProtocolMetaData(*I, "", "", Preamble);
+
+  InsertText(SM->getLocForStartOfFile(MainFileID), Preamble, false);
+  if (ClassImplementation.size() || CategoryImplementation.size())
+    RewriteImplementations();
+
+  // Get the buffer corresponding to MainFileID.  If we haven't changed it, then
+  // we are done.
+  if (const RewriteBuffer *RewriteBuf =
+      Rewrite.getRewriteBufferFor(MainFileID)) {
+    //printf("Changed:\n");
+    *OutFile << std::string(RewriteBuf->begin(), RewriteBuf->end());
+  } else {
+    llvm::errs() << "No changes\n";
+  }
+
+  if (ClassImplementation.size() || CategoryImplementation.size() ||
+      ProtocolExprDecls.size()) {
+    // Rewrite Objective-c meta data*
+    std::string ResultStr;
+    RewriteMetaDataIntoBuffer(ResultStr);
+    // Emit metadata.
+    *OutFile << ResultStr;
+  }
+  OutFile->flush();
+}
+
+void RewriteObjCFragileABI::Initialize(ASTContext &context) {
+  InitializeCommon(context);
+  
+  // declaring objc_selector outside the parameter list removes a silly
+  // scope related warning...
+  if (IsHeader)
+    Preamble = "#pragma once\n";
+  Preamble += "struct objc_selector; struct objc_class;\n";
+  Preamble += "struct __rw_objc_super { struct objc_object *object; ";
+  Preamble += "struct objc_object *superClass; ";
+  if (LangOpts.MicrosoftExt) {
+    // Add a constructor for creating temporary objects.
+    Preamble += "__rw_objc_super(struct objc_object *o, struct objc_object *s) "
+    ": ";
+    Preamble += "object(o), superClass(s) {} ";
+  }
+  Preamble += "};\n";
+  Preamble += "#ifndef _REWRITER_typedef_Protocol\n";
+  Preamble += "typedef struct objc_object Protocol;\n";
+  Preamble += "#define _REWRITER_typedef_Protocol\n";
+  Preamble += "#endif\n";
+  if (LangOpts.MicrosoftExt) {
+    Preamble += "#define __OBJC_RW_DLLIMPORT extern \"C\" __declspec(dllimport)\n";
+    Preamble += "#define __OBJC_RW_STATICIMPORT extern \"C\"\n";
+  } else
+    Preamble += "#define __OBJC_RW_DLLIMPORT extern\n";
+  Preamble += "__OBJC_RW_DLLIMPORT struct objc_object *objc_msgSend";
+  Preamble += "(struct objc_object *, struct objc_selector *, ...);\n";
+  Preamble += "__OBJC_RW_DLLIMPORT struct objc_object *objc_msgSendSuper";
+  Preamble += "(struct objc_super *, struct objc_selector *, ...);\n";
+  Preamble += "__OBJC_RW_DLLIMPORT struct objc_object* objc_msgSend_stret";
+  Preamble += "(struct objc_object *, struct objc_selector *, ...);\n";
+  Preamble += "__OBJC_RW_DLLIMPORT struct objc_object* objc_msgSendSuper_stret";
+  Preamble += "(struct objc_super *, struct objc_selector *, ...);\n";
+  Preamble += "__OBJC_RW_DLLIMPORT double objc_msgSend_fpret";
+  Preamble += "(struct objc_object *, struct objc_selector *, ...);\n";
+  Preamble += "__OBJC_RW_DLLIMPORT struct objc_object *objc_getClass";
+  Preamble += "(const char *);\n";
+  Preamble += "__OBJC_RW_DLLIMPORT struct objc_class *class_getSuperclass";
+  Preamble += "(struct objc_class *);\n";
+  Preamble += "__OBJC_RW_DLLIMPORT struct objc_object *objc_getMetaClass";
+  Preamble += "(const char *);\n";
+  Preamble += "__OBJC_RW_DLLIMPORT void objc_exception_throw(struct objc_object *);\n";
+  Preamble += "__OBJC_RW_DLLIMPORT void objc_exception_try_enter(void *);\n";
+  Preamble += "__OBJC_RW_DLLIMPORT void objc_exception_try_exit(void *);\n";
+  Preamble += "__OBJC_RW_DLLIMPORT struct objc_object *objc_exception_extract(void *);\n";
+  Preamble += "__OBJC_RW_DLLIMPORT int objc_exception_match";
+  Preamble += "(struct objc_class *, struct objc_object *);\n";
+  // @synchronized hooks.
+  Preamble += "__OBJC_RW_DLLIMPORT int objc_sync_enter(struct objc_object *);\n";
+  Preamble += "__OBJC_RW_DLLIMPORT int objc_sync_exit(struct objc_object *);\n";
+  Preamble += "__OBJC_RW_DLLIMPORT Protocol *objc_getProtocol(const char *);\n";
+  Preamble += "#ifndef __FASTENUMERATIONSTATE\n";
+  Preamble += "struct __objcFastEnumerationState {\n\t";
+  Preamble += "unsigned long state;\n\t";
+  Preamble += "void **itemsPtr;\n\t";
+  Preamble += "unsigned long *mutationsPtr;\n\t";
+  Preamble += "unsigned long extra[5];\n};\n";
+  Preamble += "__OBJC_RW_DLLIMPORT void objc_enumerationMutation(struct objc_object *);\n";
+  Preamble += "#define __FASTENUMERATIONSTATE\n";
+  Preamble += "#endif\n";
+  Preamble += "#ifndef __NSCONSTANTSTRINGIMPL\n";
+  Preamble += "struct __NSConstantStringImpl {\n";
+  Preamble += "  int *isa;\n";
+  Preamble += "  int flags;\n";
+  Preamble += "  char *str;\n";
+  Preamble += "  long length;\n";
+  Preamble += "};\n";
+  Preamble += "#ifdef CF_EXPORT_CONSTANT_STRING\n";
+  Preamble += "extern \"C\" __declspec(dllexport) int __CFConstantStringClassReference[];\n";
+  Preamble += "#else\n";
+  Preamble += "__OBJC_RW_DLLIMPORT int __CFConstantStringClassReference[];\n";
+  Preamble += "#endif\n";
+  Preamble += "#define __NSCONSTANTSTRINGIMPL\n";
+  Preamble += "#endif\n";
+  // Blocks preamble.
+  Preamble += "#ifndef BLOCK_IMPL\n";
+  Preamble += "#define BLOCK_IMPL\n";
+  Preamble += "struct __block_impl {\n";
+  Preamble += "  void *isa;\n";
+  Preamble += "  int Flags;\n";
+  Preamble += "  int Reserved;\n";
+  Preamble += "  void *FuncPtr;\n";
+  Preamble += "};\n";
+  Preamble += "// Runtime copy/destroy helper functions (from Block_private.h)\n";
+  Preamble += "#ifdef __OBJC_EXPORT_BLOCKS\n";
+  Preamble += "extern \"C\" __declspec(dllexport) "
+  "void _Block_object_assign(void *, const void *, const int);\n";
+  Preamble += "extern \"C\" __declspec(dllexport) void _Block_object_dispose(const void *, const int);\n";
+  Preamble += "extern \"C\" __declspec(dllexport) void *_NSConcreteGlobalBlock[32];\n";
+  Preamble += "extern \"C\" __declspec(dllexport) void *_NSConcreteStackBlock[32];\n";
+  Preamble += "#else\n";
+  Preamble += "__OBJC_RW_DLLIMPORT void _Block_object_assign(void *, const void *, const int);\n";
+  Preamble += "__OBJC_RW_DLLIMPORT void _Block_object_dispose(const void *, const int);\n";
+  Preamble += "__OBJC_RW_DLLIMPORT void *_NSConcreteGlobalBlock[32];\n";
+  Preamble += "__OBJC_RW_DLLIMPORT void *_NSConcreteStackBlock[32];\n";
+  Preamble += "#endif\n";
+  Preamble += "#endif\n";
+  if (LangOpts.MicrosoftExt) {
+    Preamble += "#undef __OBJC_RW_DLLIMPORT\n";
+    Preamble += "#undef __OBJC_RW_STATICIMPORT\n";
+    Preamble += "#ifndef KEEP_ATTRIBUTES\n";  // We use this for clang tests.
+    Preamble += "#define __attribute__(X)\n";
+    Preamble += "#endif\n";
+    Preamble += "#define __weak\n";
+  }
+  else {
+    Preamble += "#define __block\n";
+    Preamble += "#define __weak\n";
+  }
+  // NOTE! Windows uses LLP64 for 64bit mode. So, cast pointer to long long
+  // as this avoids warning in any 64bit/32bit compilation model.
+  Preamble += "\n#define __OFFSETOFIVAR__(TYPE, MEMBER) ((long long) &((TYPE *)0)->MEMBER)\n";
+}
+
+/// RewriteIvarOffsetComputation - This rutine synthesizes computation of
+/// ivar offset.
+void RewriteObjCFragileABI::RewriteIvarOffsetComputation(ObjCIvarDecl *ivar,
+                                                         std::string &Result) {
+  if (ivar->isBitField()) {
+    // FIXME: The hack below doesn't work for bitfields. For now, we simply
+    // place all bitfields at offset 0.
+    Result += "0";
+  } else {
+    Result += "__OFFSETOFIVAR__(struct ";
+    Result += ivar->getContainingInterface()->getNameAsString();
+    if (LangOpts.MicrosoftExt)
+      Result += "_IMPL";
+    Result += ", ";
+    Result += ivar->getNameAsString();
+    Result += ")";
+  }
+}
+
+/// RewriteObjCProtocolMetaData - Rewrite protocols meta-data.
+void RewriteObjCFragileABI::RewriteObjCProtocolMetaData(
+                            ObjCProtocolDecl *PDecl, StringRef prefix,
+                            StringRef ClassName, std::string &Result) {
+  static bool objc_protocol_methods = false;
+  
+  // Output struct protocol_methods holder of method selector and type.
+  if (!objc_protocol_methods && PDecl->hasDefinition()) {
+    /* struct protocol_methods {
+     SEL _cmd;
+     char *method_types;
+     }
+     */
+    Result += "\nstruct _protocol_methods {\n";
+    Result += "\tstruct objc_selector *_cmd;\n";
+    Result += "\tchar *method_types;\n";
+    Result += "};\n";
+    
+    objc_protocol_methods = true;
+  }
+  // Do not synthesize the protocol more than once.
+  if (ObjCSynthesizedProtocols.count(PDecl->getCanonicalDecl()))
+    return;
+  
+  if (ObjCProtocolDecl *Def = PDecl->getDefinition())
+    PDecl = Def;
+  
+  if (PDecl->instmeth_begin() != PDecl->instmeth_end()) {
+    unsigned NumMethods = std::distance(PDecl->instmeth_begin(),
+                                        PDecl->instmeth_end());
+    /* struct _objc_protocol_method_list {
+     int protocol_method_count;
+     struct protocol_methods protocols[];
+     }
+     */
+    Result += "\nstatic struct {\n";
+    Result += "\tint protocol_method_count;\n";
+    Result += "\tstruct _protocol_methods protocol_methods[";
+    Result += utostr(NumMethods);
+    Result += "];\n} _OBJC_PROTOCOL_INSTANCE_METHODS_";
+    Result += PDecl->getNameAsString();
+    Result += " __attribute__ ((used, section (\"__OBJC, __cat_inst_meth\")))= "
+    "{\n\t" + utostr(NumMethods) + "\n";
+    
+    // Output instance methods declared in this protocol.
+    for (ObjCProtocolDecl::instmeth_iterator
+         I = PDecl->instmeth_begin(), E = PDecl->instmeth_end();
+         I != E; ++I) {
+      if (I == PDecl->instmeth_begin())
+        Result += "\t  ,{{(struct objc_selector *)\"";
+      else
+        Result += "\t  ,{(struct objc_selector *)\"";
+      Result += (*I)->getSelector().getAsString();
+      std::string MethodTypeString;
+      Context->getObjCEncodingForMethodDecl((*I), MethodTypeString);
+      Result += "\", \"";
+      Result += MethodTypeString;
+      Result += "\"}\n";
+    }
+    Result += "\t }\n};\n";
+  }
+  
+  // Output class methods declared in this protocol.
+  unsigned NumMethods = std::distance(PDecl->classmeth_begin(),
+                                      PDecl->classmeth_end());
+  if (NumMethods > 0) {
+    /* struct _objc_protocol_method_list {
+     int protocol_method_count;
+     struct protocol_methods protocols[];
+     }
+     */
+    Result += "\nstatic struct {\n";
+    Result += "\tint protocol_method_count;\n";
+    Result += "\tstruct _protocol_methods protocol_methods[";
+    Result += utostr(NumMethods);
+    Result += "];\n} _OBJC_PROTOCOL_CLASS_METHODS_";
+    Result += PDecl->getNameAsString();
+    Result += " __attribute__ ((used, section (\"__OBJC, __cat_cls_meth\")))= "
+    "{\n\t";
+    Result += utostr(NumMethods);
+    Result += "\n";
+    
+    // Output instance methods declared in this protocol.
+    for (ObjCProtocolDecl::classmeth_iterator
+         I = PDecl->classmeth_begin(), E = PDecl->classmeth_end();
+         I != E; ++I) {
+      if (I == PDecl->classmeth_begin())
+        Result += "\t  ,{{(struct objc_selector *)\"";
+      else
+        Result += "\t  ,{(struct objc_selector *)\"";
+      Result += (*I)->getSelector().getAsString();
+      std::string MethodTypeString;
+      Context->getObjCEncodingForMethodDecl((*I), MethodTypeString);
+      Result += "\", \"";
+      Result += MethodTypeString;
+      Result += "\"}\n";
+    }
+    Result += "\t }\n};\n";
+  }
+  
+  // Output:
+  /* struct _objc_protocol {
+   // Objective-C 1.0 extensions
+   struct _objc_protocol_extension *isa;
+   char *protocol_name;
+   struct _objc_protocol **protocol_list;
+   struct _objc_protocol_method_list *instance_methods;
+   struct _objc_protocol_method_list *class_methods;
+   };
+   */
+  static bool objc_protocol = false;
+  if (!objc_protocol) {
+    Result += "\nstruct _objc_protocol {\n";
+    Result += "\tstruct _objc_protocol_extension *isa;\n";
+    Result += "\tchar *protocol_name;\n";
+    Result += "\tstruct _objc_protocol **protocol_list;\n";
+    Result += "\tstruct _objc_protocol_method_list *instance_methods;\n";
+    Result += "\tstruct _objc_protocol_method_list *class_methods;\n";
+    Result += "};\n";
+    
+    objc_protocol = true;
+  }
+  
+  Result += "\nstatic struct _objc_protocol _OBJC_PROTOCOL_";
+  Result += PDecl->getNameAsString();
+  Result += " __attribute__ ((used, section (\"__OBJC, __protocol\")))= "
+  "{\n\t0, \"";
+  Result += PDecl->getNameAsString();
+  Result += "\", 0, ";
+  if (PDecl->instmeth_begin() != PDecl->instmeth_end()) {
+    Result += "(struct _objc_protocol_method_list *)&_OBJC_PROTOCOL_INSTANCE_METHODS_";
+    Result += PDecl->getNameAsString();
+    Result += ", ";
+  }
+  else
+    Result += "0, ";
+  if (PDecl->classmeth_begin() != PDecl->classmeth_end()) {
+    Result += "(struct _objc_protocol_method_list *)&_OBJC_PROTOCOL_CLASS_METHODS_";
+    Result += PDecl->getNameAsString();
+    Result += "\n";
+  }
+  else
+    Result += "0\n";
+  Result += "};\n";
+  
+  // Mark this protocol as having been generated.
+  if (!ObjCSynthesizedProtocols.insert(PDecl->getCanonicalDecl()))
+    llvm_unreachable("protocol already synthesized");
+  
+}
+
+void RewriteObjCFragileABI::RewriteObjCProtocolListMetaData(
+                                const ObjCList<ObjCProtocolDecl> &Protocols,
+                                StringRef prefix, StringRef ClassName,
+                                std::string &Result) {
+  if (Protocols.empty()) return;
+  
+  for (unsigned i = 0; i != Protocols.size(); i++)
+    RewriteObjCProtocolMetaData(Protocols[i], prefix, ClassName, Result);
+  
+  // Output the top lovel protocol meta-data for the class.
+  /* struct _objc_protocol_list {
+   struct _objc_protocol_list *next;
+   int    protocol_count;
+   struct _objc_protocol *class_protocols[];
+   }
+   */
+  Result += "\nstatic struct {\n";
+  Result += "\tstruct _objc_protocol_list *next;\n";
+  Result += "\tint    protocol_count;\n";
+  Result += "\tstruct _objc_protocol *class_protocols[";
+  Result += utostr(Protocols.size());
+  Result += "];\n} _OBJC_";
+  Result += prefix;
+  Result += "_PROTOCOLS_";
+  Result += ClassName;
+  Result += " __attribute__ ((used, section (\"__OBJC, __cat_cls_meth\")))= "
+  "{\n\t0, ";
+  Result += utostr(Protocols.size());
+  Result += "\n";
+  
+  Result += "\t,{&_OBJC_PROTOCOL_";
+  Result += Protocols[0]->getNameAsString();
+  Result += " \n";
+  
+  for (unsigned i = 1; i != Protocols.size(); i++) {
+    Result += "\t ,&_OBJC_PROTOCOL_";
+    Result += Protocols[i]->getNameAsString();
+    Result += "\n";
+  }
+  Result += "\t }\n};\n";
+}
+
+void RewriteObjCFragileABI::RewriteObjCClassMetaData(ObjCImplementationDecl *IDecl,
+                                           std::string &Result) {
+  ObjCInterfaceDecl *CDecl = IDecl->getClassInterface();
+  
+  // Explicitly declared @interface's are already synthesized.
+  if (CDecl->isImplicitInterfaceDecl()) {
+    // FIXME: Implementation of a class with no @interface (legacy) does not
+    // produce correct synthesis as yet.
+    RewriteObjCInternalStruct(CDecl, Result);
+  }
+  
+  // Build _objc_ivar_list metadata for classes ivars if needed
+  unsigned NumIvars = !IDecl->ivar_empty()
+  ? IDecl->ivar_size()
+  : (CDecl ? CDecl->ivar_size() : 0);
+  if (NumIvars > 0) {
+    static bool objc_ivar = false;
+    if (!objc_ivar) {
+      /* struct _objc_ivar {
+       char *ivar_name;
+       char *ivar_type;
+       int ivar_offset;
+       };
+       */
+      Result += "\nstruct _objc_ivar {\n";
+      Result += "\tchar *ivar_name;\n";
+      Result += "\tchar *ivar_type;\n";
+      Result += "\tint ivar_offset;\n";
+      Result += "};\n";
+      
+      objc_ivar = true;
+    }
+    
+    /* struct {
+     int ivar_count;
+     struct _objc_ivar ivar_list[nIvars];
+     };
+     */
+    Result += "\nstatic struct {\n";
+    Result += "\tint ivar_count;\n";
+    Result += "\tstruct _objc_ivar ivar_list[";
+    Result += utostr(NumIvars);
+    Result += "];\n} _OBJC_INSTANCE_VARIABLES_";
+    Result += IDecl->getNameAsString();
+    Result += " __attribute__ ((used, section (\"__OBJC, __instance_vars\")))= "
+    "{\n\t";
+    Result += utostr(NumIvars);
+    Result += "\n";
+    
+    ObjCInterfaceDecl::ivar_iterator IVI, IVE;
+    SmallVector<ObjCIvarDecl *, 8> IVars;
+    if (!IDecl->ivar_empty()) {
+      for (ObjCInterfaceDecl::ivar_iterator
+           IV = IDecl->ivar_begin(), IVEnd = IDecl->ivar_end();
+           IV != IVEnd; ++IV)
+        IVars.push_back(*IV);
+      IVI = IDecl->ivar_begin();
+      IVE = IDecl->ivar_end();
+    } else {
+      IVI = CDecl->ivar_begin();
+      IVE = CDecl->ivar_end();
+    }
+    Result += "\t,{{\"";
+    Result += IVI->getNameAsString();
+    Result += "\", \"";
+    std::string TmpString, StrEncoding;
+    Context->getObjCEncodingForType(IVI->getType(), TmpString, *IVI);
+    QuoteDoublequotes(TmpString, StrEncoding);
+    Result += StrEncoding;
+    Result += "\", ";
+    RewriteIvarOffsetComputation(*IVI, Result);
+    Result += "}\n";
+    for (++IVI; IVI != IVE; ++IVI) {
+      Result += "\t  ,{\"";
+      Result += IVI->getNameAsString();
+      Result += "\", \"";
+      std::string TmpString, StrEncoding;
+      Context->getObjCEncodingForType(IVI->getType(), TmpString, *IVI);
+      QuoteDoublequotes(TmpString, StrEncoding);
+      Result += StrEncoding;
+      Result += "\", ";
+      RewriteIvarOffsetComputation(*IVI, Result);
+      Result += "}\n";
+    }
+    
+    Result += "\t }\n};\n";
+  }
+  
+  // Build _objc_method_list for class's instance methods if needed
+  SmallVector<ObjCMethodDecl *, 32>
+  InstanceMethods(IDecl->instmeth_begin(), IDecl->instmeth_end());
+  
+  // If any of our property implementations have associated getters or
+  // setters, produce metadata for them as well.
+  for (ObjCImplDecl::propimpl_iterator Prop = IDecl->propimpl_begin(),
+       PropEnd = IDecl->propimpl_end();
+       Prop != PropEnd; ++Prop) {
+    if (Prop->getPropertyImplementation() == ObjCPropertyImplDecl::Dynamic)
+      continue;
+    if (!Prop->getPropertyIvarDecl())
+      continue;
+    ObjCPropertyDecl *PD = Prop->getPropertyDecl();
+    if (!PD)
+      continue;
+    if (ObjCMethodDecl *Getter = PD->getGetterMethodDecl())
+      if (!Getter->isDefined())
+        InstanceMethods.push_back(Getter);
+    if (PD->isReadOnly())
+      continue;
+    if (ObjCMethodDecl *Setter = PD->getSetterMethodDecl())
+      if (!Setter->isDefined())
+        InstanceMethods.push_back(Setter);
+  }
+  RewriteObjCMethodsMetaData(InstanceMethods.begin(), InstanceMethods.end(),
+                             true, "", IDecl->getName(), Result);
+  
+  // Build _objc_method_list for class's class methods if needed
+  RewriteObjCMethodsMetaData(IDecl->classmeth_begin(), IDecl->classmeth_end(),
+                             false, "", IDecl->getName(), Result);
+  
+  // Protocols referenced in class declaration?
+  RewriteObjCProtocolListMetaData(CDecl->getReferencedProtocols(),
+                                  "CLASS", CDecl->getName(), Result);
+  
+  // Declaration of class/meta-class metadata
+  /* struct _objc_class {
+   struct _objc_class *isa; // or const char *root_class_name when metadata
+   const char *super_class_name;
+   char *name;
+   long version;
+   long info;
+   long instance_size;
+   struct _objc_ivar_list *ivars;
+   struct _objc_method_list *methods;
+   struct objc_cache *cache;
+   struct objc_protocol_list *protocols;
+   const char *ivar_layout;
+   struct _objc_class_ext  *ext;
+   };
+   */
+  static bool objc_class = false;
+  if (!objc_class) {
+    Result += "\nstruct _objc_class {\n";
+    Result += "\tstruct _objc_class *isa;\n";
+    Result += "\tconst char *super_class_name;\n";
+    Result += "\tchar *name;\n";
+    Result += "\tlong version;\n";
+    Result += "\tlong info;\n";
+    Result += "\tlong instance_size;\n";
+    Result += "\tstruct _objc_ivar_list *ivars;\n";
+    Result += "\tstruct _objc_method_list *methods;\n";
+    Result += "\tstruct objc_cache *cache;\n";
+    Result += "\tstruct _objc_protocol_list *protocols;\n";
+    Result += "\tconst char *ivar_layout;\n";
+    Result += "\tstruct _objc_class_ext  *ext;\n";
+    Result += "};\n";
+    objc_class = true;
+  }
+  
+  // Meta-class metadata generation.
+  ObjCInterfaceDecl *RootClass = 0;
+  ObjCInterfaceDecl *SuperClass = CDecl->getSuperClass();
+  while (SuperClass) {
+    RootClass = SuperClass;
+    SuperClass = SuperClass->getSuperClass();
+  }
+  SuperClass = CDecl->getSuperClass();
+  
+  Result += "\nstatic struct _objc_class _OBJC_METACLASS_";
+  Result += CDecl->getNameAsString();
+  Result += " __attribute__ ((used, section (\"__OBJC, __meta_class\")))= "
+  "{\n\t(struct _objc_class *)\"";
+  Result += (RootClass ? RootClass->getNameAsString() : CDecl->getNameAsString());
+  Result += "\"";
+  
+  if (SuperClass) {
+    Result += ", \"";
+    Result += SuperClass->getNameAsString();
+    Result += "\", \"";
+    Result += CDecl->getNameAsString();
+    Result += "\"";
+  }
+  else {
+    Result += ", 0, \"";
+    Result += CDecl->getNameAsString();
+    Result += "\"";
+  }
+  // Set 'ivars' field for root class to 0. ObjC1 runtime does not use it.
+  // 'info' field is initialized to CLS_META(2) for metaclass
+  Result += ", 0,2, sizeof(struct _objc_class), 0";
+  if (IDecl->classmeth_begin() != IDecl->classmeth_end()) {
+    Result += "\n\t, (struct _objc_method_list *)&_OBJC_CLASS_METHODS_";
+    Result += IDecl->getNameAsString();
+    Result += "\n";
+  }
+  else
+    Result += ", 0\n";
+  if (CDecl->protocol_begin() != CDecl->protocol_end()) {
+    Result += "\t,0, (struct _objc_protocol_list *)&_OBJC_CLASS_PROTOCOLS_";
+    Result += CDecl->getNameAsString();
+    Result += ",0,0\n";
+  }
+  else
+    Result += "\t,0,0,0,0\n";
+  Result += "};\n";
+  
+  // class metadata generation.
+  Result += "\nstatic struct _objc_class _OBJC_CLASS_";
+  Result += CDecl->getNameAsString();
+  Result += " __attribute__ ((used, section (\"__OBJC, __class\")))= "
+  "{\n\t&_OBJC_METACLASS_";
+  Result += CDecl->getNameAsString();
+  if (SuperClass) {
+    Result += ", \"";
+    Result += SuperClass->getNameAsString();
+    Result += "\", \"";
+    Result += CDecl->getNameAsString();
+    Result += "\"";
+  }
+  else {
+    Result += ", 0, \"";
+    Result += CDecl->getNameAsString();
+    Result += "\"";
+  }
+  // 'info' field is initialized to CLS_CLASS(1) for class
+  Result += ", 0,1";
+  if (!ObjCSynthesizedStructs.count(CDecl))
+    Result += ",0";
+  else {
+    // class has size. Must synthesize its size.
+    Result += ",sizeof(struct ";
+    Result += CDecl->getNameAsString();
+    if (LangOpts.MicrosoftExt)
+      Result += "_IMPL";
+    Result += ")";
+  }
+  if (NumIvars > 0) {
+    Result += ", (struct _objc_ivar_list *)&_OBJC_INSTANCE_VARIABLES_";
+    Result += CDecl->getNameAsString();
+    Result += "\n\t";
+  }
+  else
+    Result += ",0";
+  if (IDecl->instmeth_begin() != IDecl->instmeth_end()) {
+    Result += ", (struct _objc_method_list *)&_OBJC_INSTANCE_METHODS_";
+    Result += CDecl->getNameAsString();
+    Result += ", 0\n\t";
+  }
+  else
+    Result += ",0,0";
+  if (CDecl->protocol_begin() != CDecl->protocol_end()) {
+    Result += ", (struct _objc_protocol_list*)&_OBJC_CLASS_PROTOCOLS_";
+    Result += CDecl->getNameAsString();
+    Result += ", 0,0\n";
+  }
+  else
+    Result += ",0,0,0\n";
+  Result += "};\n";
+}
+
+void RewriteObjCFragileABI::RewriteMetaDataIntoBuffer(std::string &Result) {
+  int ClsDefCount = ClassImplementation.size();
+  int CatDefCount = CategoryImplementation.size();
+  
+  // For each implemented class, write out all its meta data.
+  for (int i = 0; i < ClsDefCount; i++)
+    RewriteObjCClassMetaData(ClassImplementation[i], Result);
+  
+  // For each implemented category, write out all its meta data.
+  for (int i = 0; i < CatDefCount; i++)
+    RewriteObjCCategoryImplDecl(CategoryImplementation[i], Result);
+  
+  // Write objc_symtab metadata
+  /*
+   struct _objc_symtab
+   {
+   long sel_ref_cnt;
+   SEL *refs;
+   short cls_def_cnt;
+   short cat_def_cnt;
+   void *defs[cls_def_cnt + cat_def_cnt];
+   };
+   */
+  
+  Result += "\nstruct _objc_symtab {\n";
+  Result += "\tlong sel_ref_cnt;\n";
+  Result += "\tSEL *refs;\n";
+  Result += "\tshort cls_def_cnt;\n";
+  Result += "\tshort cat_def_cnt;\n";
+  Result += "\tvoid *defs[" + utostr(ClsDefCount + CatDefCount)+ "];\n";
+  Result += "};\n\n";
+  
+  Result += "static struct _objc_symtab "
+  "_OBJC_SYMBOLS __attribute__((used, section (\"__OBJC, __symbols\")))= {\n";
+  Result += "\t0, 0, " + utostr(ClsDefCount)
+  + ", " + utostr(CatDefCount) + "\n";
+  for (int i = 0; i < ClsDefCount; i++) {
+    Result += "\t,&_OBJC_CLASS_";
+    Result += ClassImplementation[i]->getNameAsString();
+    Result += "\n";
+  }
+  
+  for (int i = 0; i < CatDefCount; i++) {
+    Result += "\t,&_OBJC_CATEGORY_";
+    Result += CategoryImplementation[i]->getClassInterface()->getNameAsString();
+    Result += "_";
+    Result += CategoryImplementation[i]->getNameAsString();
+    Result += "\n";
+  }
+  
+  Result += "};\n\n";
+  
+  // Write objc_module metadata
+  
+  /*
+   struct _objc_module {
+   long version;
+   long size;
+   const char *name;
+   struct _objc_symtab *symtab;
+   }
+   */
+  
+  Result += "\nstruct _objc_module {\n";
+  Result += "\tlong version;\n";
+  Result += "\tlong size;\n";
+  Result += "\tconst char *name;\n";
+  Result += "\tstruct _objc_symtab *symtab;\n";
+  Result += "};\n\n";
+  Result += "static struct _objc_module "
+  "_OBJC_MODULES __attribute__ ((used, section (\"__OBJC, __module_info\")))= {\n";
+  Result += "\t" + utostr(OBJC_ABI_VERSION) +
+  ", sizeof(struct _objc_module), \"\", &_OBJC_SYMBOLS\n";
+  Result += "};\n\n";
+  
+  if (LangOpts.MicrosoftExt) {
+    if (ProtocolExprDecls.size()) {
+      Result += "#pragma section(\".objc_protocol$B\",long,read,write)\n";
+      Result += "#pragma data_seg(push, \".objc_protocol$B\")\n";
+      for (llvm::SmallPtrSet<ObjCProtocolDecl *,8>::iterator I = ProtocolExprDecls.begin(),
+           E = ProtocolExprDecls.end(); I != E; ++I) {
+        Result += "static struct _objc_protocol *_POINTER_OBJC_PROTOCOL_";
+        Result += (*I)->getNameAsString();
+        Result += " = &_OBJC_PROTOCOL_";
+        Result += (*I)->getNameAsString();
+        Result += ";\n";
+      }
+      Result += "#pragma data_seg(pop)\n\n";
+    }
+    Result += "#pragma section(\".objc_module_info$B\",long,read,write)\n";
+    Result += "#pragma data_seg(push, \".objc_module_info$B\")\n";
+    Result += "static struct _objc_module *_POINTER_OBJC_MODULES = ";
+    Result += "&_OBJC_MODULES;\n";
+    Result += "#pragma data_seg(pop)\n\n";
+  }
+}
+
+/// RewriteObjCCategoryImplDecl - Rewrite metadata for each category
+/// implementation.
+void RewriteObjCFragileABI::RewriteObjCCategoryImplDecl(ObjCCategoryImplDecl *IDecl,
+                                              std::string &Result) {
+  ObjCInterfaceDecl *ClassDecl = IDecl->getClassInterface();
+  // Find category declaration for this implementation.
+  ObjCCategoryDecl *CDecl;
+  for (CDecl = ClassDecl->getCategoryList(); CDecl;
+       CDecl = CDecl->getNextClassCategory())
+    if (CDecl->getIdentifier() == IDecl->getIdentifier())
+      break;
+  
+  std::string FullCategoryName = ClassDecl->getNameAsString();
+  FullCategoryName += '_';
+  FullCategoryName += IDecl->getNameAsString();
+  
+  // Build _objc_method_list for class's instance methods if needed
+  SmallVector<ObjCMethodDecl *, 32>
+  InstanceMethods(IDecl->instmeth_begin(), IDecl->instmeth_end());
+  
+  // If any of our property implementations have associated getters or
+  // setters, produce metadata for them as well.
+  for (ObjCImplDecl::propimpl_iterator Prop = IDecl->propimpl_begin(),
+       PropEnd = IDecl->propimpl_end();
+       Prop != PropEnd; ++Prop) {
+    if (Prop->getPropertyImplementation() == ObjCPropertyImplDecl::Dynamic)
+      continue;
+    if (!Prop->getPropertyIvarDecl())
+      continue;
+    ObjCPropertyDecl *PD = Prop->getPropertyDecl();
+    if (!PD)
+      continue;
+    if (ObjCMethodDecl *Getter = PD->getGetterMethodDecl())
+      InstanceMethods.push_back(Getter);
+    if (PD->isReadOnly())
+      continue;
+    if (ObjCMethodDecl *Setter = PD->getSetterMethodDecl())
+      InstanceMethods.push_back(Setter);
+  }
+  RewriteObjCMethodsMetaData(InstanceMethods.begin(), InstanceMethods.end(),
+                             true, "CATEGORY_", FullCategoryName.c_str(),
+                             Result);
+  
+  // Build _objc_method_list for class's class methods if needed
+  RewriteObjCMethodsMetaData(IDecl->classmeth_begin(), IDecl->classmeth_end(),
+                             false, "CATEGORY_", FullCategoryName.c_str(),
+                             Result);
+  
+  // Protocols referenced in class declaration?
+  // Null CDecl is case of a category implementation with no category interface
+  if (CDecl)
+    RewriteObjCProtocolListMetaData(CDecl->getReferencedProtocols(), "CATEGORY",
+                                    FullCategoryName, Result);
+  /* struct _objc_category {
+   char *category_name;
+   char *class_name;
+   struct _objc_method_list *instance_methods;
+   struct _objc_method_list *class_methods;
+   struct _objc_protocol_list *protocols;
+   // Objective-C 1.0 extensions
+   uint32_t size;     // sizeof (struct _objc_category)
+   struct _objc_property_list *instance_properties;  // category's own
+   // @property decl.
+   };
+   */
+  
+  static bool objc_category = false;
+  if (!objc_category) {
+    Result += "\nstruct _objc_category {\n";
+    Result += "\tchar *category_name;\n";
+    Result += "\tchar *class_name;\n";
+    Result += "\tstruct _objc_method_list *instance_methods;\n";
+    Result += "\tstruct _objc_method_list *class_methods;\n";
+    Result += "\tstruct _objc_protocol_list *protocols;\n";
+    Result += "\tunsigned int size;\n";
+    Result += "\tstruct _objc_property_list *instance_properties;\n";
+    Result += "};\n";
+    objc_category = true;
+  }
+  Result += "\nstatic struct _objc_category _OBJC_CATEGORY_";
+  Result += FullCategoryName;
+  Result += " __attribute__ ((used, section (\"__OBJC, __category\")))= {\n\t\"";
+  Result += IDecl->getNameAsString();
+  Result += "\"\n\t, \"";
+  Result += ClassDecl->getNameAsString();
+  Result += "\"\n";
+  
+  if (IDecl->instmeth_begin() != IDecl->instmeth_end()) {
+    Result += "\t, (struct _objc_method_list *)"
+    "&_OBJC_CATEGORY_INSTANCE_METHODS_";
+    Result += FullCategoryName;
+    Result += "\n";
+  }
+  else
+    Result += "\t, 0\n";
+  if (IDecl->classmeth_begin() != IDecl->classmeth_end()) {
+    Result += "\t, (struct _objc_method_list *)"
+    "&_OBJC_CATEGORY_CLASS_METHODS_";
+    Result += FullCategoryName;
+    Result += "\n";
+  }
+  else
+    Result += "\t, 0\n";
+  
+  if (CDecl && CDecl->protocol_begin() != CDecl->protocol_end()) {
+    Result += "\t, (struct _objc_protocol_list *)&_OBJC_CATEGORY_PROTOCOLS_";
+    Result += FullCategoryName;
+    Result += "\n";
+  }
+  else
+    Result += "\t, 0\n";
+  Result += "\t, sizeof(struct _objc_category), 0\n};\n";
+}
+
+// RewriteObjCMethodsMetaData - Rewrite methods metadata for instance or
+/// class methods.
+template<typename MethodIterator>
+void RewriteObjCFragileABI::RewriteObjCMethodsMetaData(MethodIterator MethodBegin,
+                                             MethodIterator MethodEnd,
+                                             bool IsInstanceMethod,
+                                             StringRef prefix,
+                                             StringRef ClassName,
+                                             std::string &Result) {
+  if (MethodBegin == MethodEnd) return;
+  
+  if (!objc_impl_method) {
+    /* struct _objc_method {
+     SEL _cmd;
+     char *method_types;
+     void *_imp;
+     }
+     */
+    Result += "\nstruct _objc_method {\n";
+    Result += "\tSEL _cmd;\n";
+    Result += "\tchar *method_types;\n";
+    Result += "\tvoid *_imp;\n";
+    Result += "};\n";
+    
+    objc_impl_method = true;
+  }
+  
+  // Build _objc_method_list for class's methods if needed
+  
+  /* struct  {
+   struct _objc_method_list *next_method;
+   int method_count;
+   struct _objc_method method_list[];
+   }
+   */
+  unsigned NumMethods = std::distance(MethodBegin, MethodEnd);
+  Result += "\nstatic struct {\n";
+  Result += "\tstruct _objc_method_list *next_method;\n";
+  Result += "\tint method_count;\n";
+  Result += "\tstruct _objc_method method_list[";
+  Result += utostr(NumMethods);
+  Result += "];\n} _OBJC_";
+  Result += prefix;
+  Result += IsInstanceMethod ? "INSTANCE" : "CLASS";
+  Result += "_METHODS_";
+  Result += ClassName;
+  Result += " __attribute__ ((used, section (\"__OBJC, __";
+  Result += IsInstanceMethod ? "inst" : "cls";
+  Result += "_meth\")))= ";
+  Result += "{\n\t0, " + utostr(NumMethods) + "\n";
+  
+  Result += "\t,{{(SEL)\"";
+  Result += (*MethodBegin)->getSelector().getAsString().c_str();
+  std::string MethodTypeString;
+  Context->getObjCEncodingForMethodDecl(*MethodBegin, MethodTypeString);
+  Result += "\", \"";
+  Result += MethodTypeString;
+  Result += "\", (void *)";
+  Result += MethodInternalNames[*MethodBegin];
+  Result += "}\n";
+  for (++MethodBegin; MethodBegin != MethodEnd; ++MethodBegin) {
+    Result += "\t  ,{(SEL)\"";
+    Result += (*MethodBegin)->getSelector().getAsString().c_str();
+    std::string MethodTypeString;
+    Context->getObjCEncodingForMethodDecl(*MethodBegin, MethodTypeString);
+    Result += "\", \"";
+    Result += MethodTypeString;
+    Result += "\", (void *)";
+    Result += MethodInternalNames[*MethodBegin];
+    Result += "}\n";
+  }
+  Result += "\t }\n};\n";
+}
+
+Stmt *RewriteObjCFragileABI::RewriteObjCIvarRefExpr(ObjCIvarRefExpr *IV) {
+  SourceRange OldRange = IV->getSourceRange();
+  Expr *BaseExpr = IV->getBase();
+  
+  // Rewrite the base, but without actually doing replaces.
+  {
+    DisableReplaceStmtScope S(*this);
+    BaseExpr = cast<Expr>(RewriteFunctionBodyOrGlobalInitializer(BaseExpr));
+    IV->setBase(BaseExpr);
+  }
+  
+  ObjCIvarDecl *D = IV->getDecl();
+  
+  Expr *Replacement = IV;
+  if (CurMethodDef) {
+    if (BaseExpr->getType()->isObjCObjectPointerType()) {
+      const ObjCInterfaceType *iFaceDecl =
+      dyn_cast<ObjCInterfaceType>(BaseExpr->getType()->getPointeeType());
+      assert(iFaceDecl && "RewriteObjCIvarRefExpr - iFaceDecl is null");
+      // lookup which class implements the instance variable.
+      ObjCInterfaceDecl *clsDeclared = 0;
+      iFaceDecl->getDecl()->lookupInstanceVariable(D->getIdentifier(),
+                                                   clsDeclared);
+      assert(clsDeclared && "RewriteObjCIvarRefExpr(): Can't find class");
+      
+      // Synthesize an explicit cast to gain access to the ivar.
+      std::string RecName = clsDeclared->getIdentifier()->getName();
+      RecName += "_IMPL";
+      IdentifierInfo *II = &Context->Idents.get(RecName);
+      RecordDecl *RD = RecordDecl::Create(*Context, TTK_Struct, TUDecl,
+                                          SourceLocation(), SourceLocation(),
+                                          II);
+      assert(RD && "RewriteObjCIvarRefExpr(): Can't find RecordDecl");
+      QualType castT = Context->getPointerType(Context->getTagDeclType(RD));
+      CastExpr *castExpr = NoTypeInfoCStyleCastExpr(Context, castT,
+                                                    CK_BitCast,
+                                                    IV->getBase());
+      // Don't forget the parens to enforce the proper binding.
+      ParenExpr *PE = new (Context) ParenExpr(OldRange.getBegin(),
+                                              OldRange.getEnd(),
+                                              castExpr);
+      if (IV->isFreeIvar() &&
+          declaresSameEntity(CurMethodDef->getClassInterface(), iFaceDecl->getDecl())) {
+        MemberExpr *ME = new (Context) MemberExpr(PE, true, D,
+                                                  IV->getLocation(),
+                                                  D->getType(),
+                                                  VK_LValue, OK_Ordinary);
+        Replacement = ME;
+      } else {
+        IV->setBase(PE);
+      }
+    }
+  } else { // we are outside a method.
+    assert(!IV->isFreeIvar() && "Cannot have a free standing ivar outside a method");
+    
+    // Explicit ivar refs need to have a cast inserted.
+    // FIXME: consider sharing some of this code with the code above.
+    if (BaseExpr->getType()->isObjCObjectPointerType()) {
+      const ObjCInterfaceType *iFaceDecl =
+      dyn_cast<ObjCInterfaceType>(BaseExpr->getType()->getPointeeType());
+      // lookup which class implements the instance variable.
+      ObjCInterfaceDecl *clsDeclared = 0;
+      iFaceDecl->getDecl()->lookupInstanceVariable(D->getIdentifier(),
+                                                   clsDeclared);
+      assert(clsDeclared && "RewriteObjCIvarRefExpr(): Can't find class");
+      
+      // Synthesize an explicit cast to gain access to the ivar.
+      std::string RecName = clsDeclared->getIdentifier()->getName();
+      RecName += "_IMPL";
+      IdentifierInfo *II = &Context->Idents.get(RecName);
+      RecordDecl *RD = RecordDecl::Create(*Context, TTK_Struct, TUDecl,
+                                          SourceLocation(), SourceLocation(),
+                                          II);
+      assert(RD && "RewriteObjCIvarRefExpr(): Can't find RecordDecl");
+      QualType castT = Context->getPointerType(Context->getTagDeclType(RD));
+      CastExpr *castExpr = NoTypeInfoCStyleCastExpr(Context, castT,
+                                                    CK_BitCast,
+                                                    IV->getBase());
+      // Don't forget the parens to enforce the proper binding.
+      ParenExpr *PE = new (Context) ParenExpr(IV->getBase()->getLocStart(),
+                                              IV->getBase()->getLocEnd(), castExpr);
+      // Cannot delete IV->getBase(), since PE points to it.
+      // Replace the old base with the cast. This is important when doing
+      // embedded rewrites. For example, [newInv->_container addObject:0].
+      IV->setBase(PE);
+    }
+  }
+  
+  ReplaceStmtWithRange(IV, Replacement, OldRange);
+  return Replacement;  
+}
diff --git a/lib/Rewrite/Frontend/RewriteTest.cpp b/lib/Rewrite/Frontend/RewriteTest.cpp
new file mode 100644
index 0000000..722c5e8
--- /dev/null
+++ b/lib/Rewrite/Frontend/RewriteTest.cpp
@@ -0,0 +1,39 @@
+//===--- RewriteTest.cpp - Rewriter playground ----------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This is a testbed.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Rewrite/Frontend/Rewriters.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Rewrite/Core/TokenRewriter.h"
+#include "llvm/Support/raw_ostream.h"
+
+void clang::DoRewriteTest(Preprocessor &PP, raw_ostream* OS) {
+  SourceManager &SM = PP.getSourceManager();
+  const LangOptions &LangOpts = PP.getLangOpts();
+
+  TokenRewriter Rewriter(SM.getMainFileID(), SM, LangOpts);
+
+  // Throw <i> </i> tags around comments.
+  for (TokenRewriter::token_iterator I = Rewriter.token_begin(),
+       E = Rewriter.token_end(); I != E; ++I) {
+    if (I->isNot(tok::comment)) continue;
+
+    Rewriter.AddTokenBefore(I, "<i>");
+    Rewriter.AddTokenAfter(I, "</i>");
+  }
+
+
+  // Print out the output.
+  for (TokenRewriter::token_iterator I = Rewriter.token_begin(),
+       E = Rewriter.token_end(); I != E; ++I)
+    *OS << PP.getSpelling(*I);
+}
diff --git a/lib/Rewrite/FrontendActions.cpp b/lib/Rewrite/FrontendActions.cpp
deleted file mode 100644
index 9bc218e..0000000
--- a/lib/Rewrite/FrontendActions.cpp
+++ /dev/null
@@ -1,192 +0,0 @@
-//===--- FrontendActions.cpp ----------------------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/Rewrite/FrontendActions.h"
-#include "clang/AST/ASTConsumer.h"
-#include "clang/Lex/Preprocessor.h"
-#include "clang/Parse/Parser.h"
-#include "clang/Basic/FileManager.h"
-#include "clang/Frontend/FrontendActions.h"
-#include "clang/Frontend/CompilerInstance.h"
-#include "clang/Frontend/FrontendDiagnostic.h"
-#include "clang/Frontend/Utils.h"
-#include "clang/Rewrite/ASTConsumers.h"
-#include "clang/Rewrite/FixItRewriter.h"
-#include "clang/Rewrite/Rewriters.h"
-#include "llvm/ADT/OwningPtr.h"
-#include "llvm/Support/raw_ostream.h"
-#include "llvm/Support/Path.h"
-#include "llvm/Support/FileSystem.h"
-
-using namespace clang;
-
-//===----------------------------------------------------------------------===//
-// AST Consumer Actions
-//===----------------------------------------------------------------------===//
-
-ASTConsumer *HTMLPrintAction::CreateASTConsumer(CompilerInstance &CI,
-                                                StringRef InFile) {
-  if (raw_ostream *OS = CI.createDefaultOutputFile(false, InFile))
-    return CreateHTMLPrinter(OS, CI.getPreprocessor());
-  return 0;
-}
-
-FixItAction::FixItAction() {}
-FixItAction::~FixItAction() {}
-
-ASTConsumer *FixItAction::CreateASTConsumer(CompilerInstance &CI,
-                                            StringRef InFile) {
-  return new ASTConsumer();
-}
-
-namespace {
-class FixItRewriteInPlace : public FixItOptions {
-public:
-  std::string RewriteFilename(const std::string &Filename, int &fd) {
-    fd = -1;
-    return Filename;
-  }
-};
-
-class FixItActionSuffixInserter : public FixItOptions {
-  std::string NewSuffix;
-
-public:
-  FixItActionSuffixInserter(std::string NewSuffix, bool FixWhatYouCan)
-    : NewSuffix(NewSuffix) {
-      this->FixWhatYouCan = FixWhatYouCan;
-  }
-
-  std::string RewriteFilename(const std::string &Filename, int &fd) {
-    fd = -1;
-    SmallString<128> Path(Filename);
-    llvm::sys::path::replace_extension(Path,
-      NewSuffix + llvm::sys::path::extension(Path));
-    return Path.str();
-  }
-};
-
-class FixItRewriteToTemp : public FixItOptions {
-public:
-  std::string RewriteFilename(const std::string &Filename, int &fd) {
-    SmallString<128> Path;
-    Path = llvm::sys::path::filename(Filename);
-    Path += "-%%%%%%%%";
-    Path += llvm::sys::path::extension(Filename);
-    SmallString<128> NewPath;
-    llvm::sys::fs::unique_file(Path.str(), fd, NewPath);
-    return NewPath.str();
-  }
-};
-} // end anonymous namespace
-
-bool FixItAction::BeginSourceFileAction(CompilerInstance &CI,
-                                        StringRef Filename) {
-  const FrontendOptions &FEOpts = getCompilerInstance().getFrontendOpts();
-  if (!FEOpts.FixItSuffix.empty()) {
-    FixItOpts.reset(new FixItActionSuffixInserter(FEOpts.FixItSuffix,
-                                                  FEOpts.FixWhatYouCan));
-  } else {
-    FixItOpts.reset(new FixItRewriteInPlace);
-    FixItOpts->FixWhatYouCan = FEOpts.FixWhatYouCan;
-  }
-  Rewriter.reset(new FixItRewriter(CI.getDiagnostics(), CI.getSourceManager(),
-                                   CI.getLangOpts(), FixItOpts.get()));
-  return true;
-}
-
-void FixItAction::EndSourceFileAction() {
-  // Otherwise rewrite all files.
-  Rewriter->WriteFixedFiles();
-}
-
-bool FixItRecompile::BeginInvocation(CompilerInstance &CI) {
-
-  std::vector<std::pair<std::string, std::string> > RewrittenFiles;
-  bool err = false;
-  {
-    const FrontendOptions &FEOpts = CI.getFrontendOpts();
-    OwningPtr<FrontendAction> FixAction(new SyntaxOnlyAction());
-    if (FixAction->BeginSourceFile(CI, FEOpts.Inputs[0])) {
-      OwningPtr<FixItOptions> FixItOpts;
-      if (FEOpts.FixToTemporaries)
-        FixItOpts.reset(new FixItRewriteToTemp());
-      else
-        FixItOpts.reset(new FixItRewriteInPlace());
-      FixItOpts->Silent = true;
-      FixItOpts->FixWhatYouCan = FEOpts.FixWhatYouCan;
-      FixItOpts->FixOnlyWarnings = FEOpts.FixOnlyWarnings;
-      FixItRewriter Rewriter(CI.getDiagnostics(), CI.getSourceManager(),
-                             CI.getLangOpts(), FixItOpts.get());
-      FixAction->Execute();
-  
-      err = Rewriter.WriteFixedFiles(&RewrittenFiles);
-    
-      FixAction->EndSourceFile();
-      CI.setSourceManager(0);
-      CI.setFileManager(0);
-    } else {
-      err = true;
-    }
-  }
-  if (err)
-    return false;
-  CI.getDiagnosticClient().clear();
-  CI.getDiagnostics().Reset();
-
-  PreprocessorOptions &PPOpts = CI.getPreprocessorOpts();
-  PPOpts.RemappedFiles.insert(PPOpts.RemappedFiles.end(),
-                              RewrittenFiles.begin(), RewrittenFiles.end());
-  PPOpts.RemappedFilesKeepOriginalName = false;
-
-  return true;
-}
-
-//===----------------------------------------------------------------------===//
-// Preprocessor Actions
-//===----------------------------------------------------------------------===//
-
-ASTConsumer *RewriteObjCAction::CreateASTConsumer(CompilerInstance &CI,
-                                                  StringRef InFile) {
-  if (raw_ostream *OS = CI.createDefaultOutputFile(false, InFile, "cpp")) {
-    if (CI.getLangOpts().ObjCRuntime.isNonFragile())
-      return CreateModernObjCRewriter(InFile, OS,
-                                CI.getDiagnostics(), CI.getLangOpts(),
-                                CI.getDiagnosticOpts().NoRewriteMacros);
-    return CreateObjCRewriter(InFile, OS,
-                              CI.getDiagnostics(), CI.getLangOpts(),
-                              CI.getDiagnosticOpts().NoRewriteMacros);
-  }
-  return 0;
-}
-
-void RewriteMacrosAction::ExecuteAction() {
-  CompilerInstance &CI = getCompilerInstance();
-  raw_ostream *OS = CI.createDefaultOutputFile(true, getCurrentFile());
-  if (!OS) return;
-
-  RewriteMacrosInInput(CI.getPreprocessor(), OS);
-}
-
-void RewriteTestAction::ExecuteAction() {
-  CompilerInstance &CI = getCompilerInstance();
-  raw_ostream *OS = CI.createDefaultOutputFile(false, getCurrentFile());
-  if (!OS) return;
-
-  DoRewriteTest(CI.getPreprocessor(), OS);
-}
-
-void RewriteIncludesAction::ExecuteAction() {
-  CompilerInstance &CI = getCompilerInstance();
-  raw_ostream *OS = CI.createDefaultOutputFile(true, getCurrentFile());
-  if (!OS) return;
-
-  RewriteIncludesInInput(CI.getPreprocessor(), OS,
-                         CI.getPreprocessorOutputOpts());
-}
diff --git a/lib/Rewrite/HTMLPrint.cpp b/lib/Rewrite/HTMLPrint.cpp
deleted file mode 100644
index 3d190ab..0000000
--- a/lib/Rewrite/HTMLPrint.cpp
+++ /dev/null
@@ -1,94 +0,0 @@
-//===--- HTMLPrint.cpp - Source code -> HTML pretty-printing --------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Pretty-printing of source code to HTML.
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/Rewrite/ASTConsumers.h"
-#include "clang/AST/ASTConsumer.h"
-#include "clang/AST/ASTContext.h"
-#include "clang/AST/Decl.h"
-#include "clang/Basic/Diagnostic.h"
-#include "clang/Basic/FileManager.h"
-#include "clang/Basic/SourceManager.h"
-#include "clang/Lex/Preprocessor.h"
-#include "clang/Rewrite/HTMLRewrite.h"
-#include "clang/Rewrite/Rewriter.h"
-#include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/raw_ostream.h"
-using namespace clang;
-
-//===----------------------------------------------------------------------===//
-// Functional HTML pretty-printing.
-//===----------------------------------------------------------------------===//
-
-namespace {
-  class HTMLPrinter : public ASTConsumer {
-    Rewriter R;
-    raw_ostream *Out;
-    Preprocessor &PP;
-    bool SyntaxHighlight, HighlightMacros;
-
-  public:
-    HTMLPrinter(raw_ostream *OS, Preprocessor &pp,
-                bool _SyntaxHighlight, bool _HighlightMacros)
-      : Out(OS), PP(pp), SyntaxHighlight(_SyntaxHighlight),
-        HighlightMacros(_HighlightMacros) {}
-
-    void Initialize(ASTContext &context);
-    void HandleTranslationUnit(ASTContext &Ctx);
-  };
-}
-
-ASTConsumer* clang::CreateHTMLPrinter(raw_ostream *OS,
-                                      Preprocessor &PP,
-                                      bool SyntaxHighlight,
-                                      bool HighlightMacros) {
-  return new HTMLPrinter(OS, PP, SyntaxHighlight, HighlightMacros);
-}
-
-void HTMLPrinter::Initialize(ASTContext &context) {
-  R.setSourceMgr(context.getSourceManager(), context.getLangOpts());
-}
-
-void HTMLPrinter::HandleTranslationUnit(ASTContext &Ctx) {
-  if (PP.getDiagnostics().hasErrorOccurred())
-    return;
-
-  // Format the file.
-  FileID FID = R.getSourceMgr().getMainFileID();
-  const FileEntry* Entry = R.getSourceMgr().getFileEntryForID(FID);
-  const char* Name;
-  // In some cases, in particular the case where the input is from stdin,
-  // there is no entry.  Fall back to the memory buffer for a name in those
-  // cases.
-  if (Entry)
-    Name = Entry->getName();
-  else
-    Name = R.getSourceMgr().getBuffer(FID)->getBufferIdentifier();
-
-  html::AddLineNumbers(R, FID);
-  html::AddHeaderFooterInternalBuiltinCSS(R, FID, Name);
-
-  // If we have a preprocessor, relex the file and syntax highlight.
-  // We might not have a preprocessor if we come from a deserialized AST file,
-  // for example.
-
-  if (SyntaxHighlight) html::SyntaxHighlight(R, FID, PP);
-  if (HighlightMacros) html::HighlightMacros(R, FID, PP);
-  html::EscapeText(R, FID, false, true);
-
-  // Emit the HTML.
-  const RewriteBuffer &RewriteBuf = R.getEditBuffer(FID);
-  char *Buffer = (char*)malloc(RewriteBuf.size());
-  std::copy(RewriteBuf.begin(), RewriteBuf.end(), Buffer);
-  Out->write(Buffer, RewriteBuf.size());
-  free(Buffer);
-}
diff --git a/lib/Rewrite/HTMLRewrite.cpp b/lib/Rewrite/HTMLRewrite.cpp
deleted file mode 100644
index 236b98f..0000000
--- a/lib/Rewrite/HTMLRewrite.cpp
+++ /dev/null
@@ -1,583 +0,0 @@
-//== HTMLRewrite.cpp - Translate source code into prettified HTML --*- C++ -*-//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//  This file defines the HTMLRewriter clas, which is used to translate the
-//  text of a source file into prettified HTML.
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/Lex/Preprocessor.h"
-#include "clang/Rewrite/Rewriter.h"
-#include "clang/Rewrite/HTMLRewrite.h"
-#include "clang/Lex/TokenConcatenation.h"
-#include "clang/Lex/Preprocessor.h"
-#include "clang/Basic/SourceManager.h"
-#include "llvm/ADT/SmallString.h"
-#include "llvm/ADT/OwningPtr.h"
-#include "llvm/Support/ErrorHandling.h"
-#include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/raw_ostream.h"
-using namespace clang;
-
-
-/// HighlightRange - Highlight a range in the source code with the specified
-/// start/end tags.  B/E must be in the same file.  This ensures that
-/// start/end tags are placed at the start/end of each line if the range is
-/// multiline.
-void html::HighlightRange(Rewriter &R, SourceLocation B, SourceLocation E,
-                          const char *StartTag, const char *EndTag) {
-  SourceManager &SM = R.getSourceMgr();
-  B = SM.getExpansionLoc(B);
-  E = SM.getExpansionLoc(E);
-  FileID FID = SM.getFileID(B);
-  assert(SM.getFileID(E) == FID && "B/E not in the same file!");
-
-  unsigned BOffset = SM.getFileOffset(B);
-  unsigned EOffset = SM.getFileOffset(E);
-
-  // Include the whole end token in the range.
-  EOffset += Lexer::MeasureTokenLength(E, R.getSourceMgr(), R.getLangOpts());
-
-  bool Invalid = false;
-  const char *BufferStart = SM.getBufferData(FID, &Invalid).data();
-  if (Invalid)
-    return;
-  
-  HighlightRange(R.getEditBuffer(FID), BOffset, EOffset,
-                 BufferStart, StartTag, EndTag);
-}
-
-/// HighlightRange - This is the same as the above method, but takes
-/// decomposed file locations.
-void html::HighlightRange(RewriteBuffer &RB, unsigned B, unsigned E,
-                          const char *BufferStart,
-                          const char *StartTag, const char *EndTag) {
-  // Insert the tag at the absolute start/end of the range.
-  RB.InsertTextAfter(B, StartTag);
-  RB.InsertTextBefore(E, EndTag);
-
-  // Scan the range to see if there is a \r or \n.  If so, and if the line is
-  // not blank, insert tags on that line as well.
-  bool HadOpenTag = true;
-
-  unsigned LastNonWhiteSpace = B;
-  for (unsigned i = B; i != E; ++i) {
-    switch (BufferStart[i]) {
-    case '\r':
-    case '\n':
-      // Okay, we found a newline in the range.  If we have an open tag, we need
-      // to insert a close tag at the first non-whitespace before the newline.
-      if (HadOpenTag)
-        RB.InsertTextBefore(LastNonWhiteSpace+1, EndTag);
-
-      // Instead of inserting an open tag immediately after the newline, we
-      // wait until we see a non-whitespace character.  This prevents us from
-      // inserting tags around blank lines, and also allows the open tag to
-      // be put *after* whitespace on a non-blank line.
-      HadOpenTag = false;
-      break;
-    case '\0':
-    case ' ':
-    case '\t':
-    case '\f':
-    case '\v':
-      // Ignore whitespace.
-      break;
-
-    default:
-      // If there is no tag open, do it now.
-      if (!HadOpenTag) {
-        RB.InsertTextAfter(i, StartTag);
-        HadOpenTag = true;
-      }
-
-      // Remember this character.
-      LastNonWhiteSpace = i;
-      break;
-    }
-  }
-}
-
-void html::EscapeText(Rewriter &R, FileID FID,
-                      bool EscapeSpaces, bool ReplaceTabs) {
-
-  const llvm::MemoryBuffer *Buf = R.getSourceMgr().getBuffer(FID);
-  const char* C = Buf->getBufferStart();
-  const char* FileEnd = Buf->getBufferEnd();
-
-  assert (C <= FileEnd);
-
-  RewriteBuffer &RB = R.getEditBuffer(FID);
-
-  unsigned ColNo = 0;
-  for (unsigned FilePos = 0; C != FileEnd ; ++C, ++FilePos) {
-    switch (*C) {
-    default: ++ColNo; break;
-    case '\n':
-    case '\r':
-      ColNo = 0;
-      break;
-
-    case ' ':
-      if (EscapeSpaces)
-        RB.ReplaceText(FilePos, 1, "&nbsp;");
-      ++ColNo;
-      break;
-    case '\f':
-      RB.ReplaceText(FilePos, 1, "<hr>");
-      ColNo = 0;
-      break;
-
-    case '\t': {
-      if (!ReplaceTabs)
-        break;
-      unsigned NumSpaces = 8-(ColNo&7);
-      if (EscapeSpaces)
-        RB.ReplaceText(FilePos, 1,
-                       StringRef("&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;"
-                                       "&nbsp;&nbsp;&nbsp;", 6*NumSpaces));
-      else
-        RB.ReplaceText(FilePos, 1, StringRef("        ", NumSpaces));
-      ColNo += NumSpaces;
-      break;
-    }
-    case '<':
-      RB.ReplaceText(FilePos, 1, "&lt;");
-      ++ColNo;
-      break;
-
-    case '>':
-      RB.ReplaceText(FilePos, 1, "&gt;");
-      ++ColNo;
-      break;
-
-    case '&':
-      RB.ReplaceText(FilePos, 1, "&amp;");
-      ++ColNo;
-      break;
-    }
-  }
-}
-
-std::string html::EscapeText(const std::string& s, bool EscapeSpaces,
-                             bool ReplaceTabs) {
-
-  unsigned len = s.size();
-  std::string Str;
-  llvm::raw_string_ostream os(Str);
-
-  for (unsigned i = 0 ; i < len; ++i) {
-
-    char c = s[i];
-    switch (c) {
-    default:
-      os << c; break;
-
-    case ' ':
-      if (EscapeSpaces) os << "&nbsp;";
-      else os << ' ';
-      break;
-
-    case '\t':
-      if (ReplaceTabs) {
-        if (EscapeSpaces)
-          for (unsigned i = 0; i < 4; ++i)
-            os << "&nbsp;";
-        else
-          for (unsigned i = 0; i < 4; ++i)
-            os << " ";
-      }
-      else
-        os << c;
-
-      break;
-
-    case '<': os << "&lt;"; break;
-    case '>': os << "&gt;"; break;
-    case '&': os << "&amp;"; break;
-    }
-  }
-
-  return os.str();
-}
-
-static void AddLineNumber(RewriteBuffer &RB, unsigned LineNo,
-                          unsigned B, unsigned E) {
-  SmallString<256> Str;
-  llvm::raw_svector_ostream OS(Str);
-
-  OS << "<tr><td class=\"num\" id=\"LN"
-     << LineNo << "\">"
-     << LineNo << "</td><td class=\"line\">";
-
-  if (B == E) { // Handle empty lines.
-    OS << " </td></tr>";
-    RB.InsertTextBefore(B, OS.str());
-  } else {
-    RB.InsertTextBefore(B, OS.str());
-    RB.InsertTextBefore(E, "</td></tr>");
-  }
-}
-
-void html::AddLineNumbers(Rewriter& R, FileID FID) {
-
-  const llvm::MemoryBuffer *Buf = R.getSourceMgr().getBuffer(FID);
-  const char* FileBeg = Buf->getBufferStart();
-  const char* FileEnd = Buf->getBufferEnd();
-  const char* C = FileBeg;
-  RewriteBuffer &RB = R.getEditBuffer(FID);
-
-  assert (C <= FileEnd);
-
-  unsigned LineNo = 0;
-  unsigned FilePos = 0;
-
-  while (C != FileEnd) {
-
-    ++LineNo;
-    unsigned LineStartPos = FilePos;
-    unsigned LineEndPos = FileEnd - FileBeg;
-
-    assert (FilePos <= LineEndPos);
-    assert (C < FileEnd);
-
-    // Scan until the newline (or end-of-file).
-
-    while (C != FileEnd) {
-      char c = *C;
-      ++C;
-
-      if (c == '\n') {
-        LineEndPos = FilePos++;
-        break;
-      }
-
-      ++FilePos;
-    }
-
-    AddLineNumber(RB, LineNo, LineStartPos, LineEndPos);
-  }
-
-  // Add one big table tag that surrounds all of the code.
-  RB.InsertTextBefore(0, "<table class=\"code\">\n");
-  RB.InsertTextAfter(FileEnd - FileBeg, "</table>");
-}
-
-void html::AddHeaderFooterInternalBuiltinCSS(Rewriter& R, FileID FID,
-                                             const char *title) {
-
-  const llvm::MemoryBuffer *Buf = R.getSourceMgr().getBuffer(FID);
-  const char* FileStart = Buf->getBufferStart();
-  const char* FileEnd = Buf->getBufferEnd();
-
-  SourceLocation StartLoc = R.getSourceMgr().getLocForStartOfFile(FID);
-  SourceLocation EndLoc = StartLoc.getLocWithOffset(FileEnd-FileStart);
-
-  std::string s;
-  llvm::raw_string_ostream os(s);
-  os << "<!doctype html>\n" // Use HTML 5 doctype
-        "<html>\n<head>\n";
-
-  if (title)
-    os << "<title>" << html::EscapeText(title) << "</title>\n";
-
-  os << "<style type=\"text/css\">\n"
-      " body { color:#000000; background-color:#ffffff }\n"
-      " body { font-family:Helvetica, sans-serif; font-size:10pt }\n"
-      " h1 { font-size:14pt }\n"
-      " .code { border-collapse:collapse; width:100%; }\n"
-      " .code { font-family: \"Monospace\", monospace; font-size:10pt }\n"
-      " .code { line-height: 1.2em }\n"
-      " .comment { color: green; font-style: oblique }\n"
-      " .keyword { color: blue }\n"
-      " .string_literal { color: red }\n"
-      " .directive { color: darkmagenta }\n"
-      // Macro expansions.
-      " .expansion { display: none; }\n"
-      " .macro:hover .expansion { display: block; border: 2px solid #FF0000; "
-          "padding: 2px; background-color:#FFF0F0; font-weight: normal; "
-          "  -webkit-border-radius:5px;  -webkit-box-shadow:1px 1px 7px #000; "
-          "position: absolute; top: -1em; left:10em; z-index: 1 } \n"
-      " .macro { color: darkmagenta; background-color:LemonChiffon;"
-             // Macros are position: relative to provide base for expansions.
-             " position: relative }\n"
-      " .num { width:2.5em; padding-right:2ex; background-color:#eeeeee }\n"
-      " .num { text-align:right; font-size:8pt }\n"
-      " .num { color:#444444 }\n"
-      " .line { padding-left: 1ex; border-left: 3px solid #ccc }\n"
-      " .line { white-space: pre }\n"
-      " .msg { -webkit-box-shadow:1px 1px 7px #000 }\n"
-      " .msg { -webkit-border-radius:5px }\n"
-      " .msg { font-family:Helvetica, sans-serif; font-size:8pt }\n"
-      " .msg { float:left }\n"
-      " .msg { padding:0.25em 1ex 0.25em 1ex }\n"
-      " .msg { margin-top:10px; margin-bottom:10px }\n"
-      " .msg { font-weight:bold }\n"
-      " .msg { max-width:60em; word-wrap: break-word; white-space: pre-wrap }\n"
-      " .msgT { padding:0x; spacing:0x }\n"
-      " .msgEvent { background-color:#fff8b4; color:#000000 }\n"
-      " .msgControl { background-color:#bbbbbb; color:#000000 }\n"
-      " .mrange { background-color:#dfddf3 }\n"
-      " .mrange { border-bottom:1px solid #6F9DBE }\n"
-      " .PathIndex { font-weight: bold; padding:0px 5px; "
-        "margin-right:5px; }\n"
-      " .PathIndex { -webkit-border-radius:8px }\n"
-      " .PathIndexEvent { background-color:#bfba87 }\n"
-      " .PathIndexControl { background-color:#8c8c8c }\n"
-      " .PathNav a { text-decoration:none; font-size: larger }\n"
-      " .CodeInsertionHint { font-weight: bold; background-color: #10dd10 }\n"
-      " .CodeRemovalHint { background-color:#de1010 }\n"
-      " .CodeRemovalHint { border-bottom:1px solid #6F9DBE }\n"
-      " table.simpletable {\n"
-      "   padding: 5px;\n"
-      "   font-size:12pt;\n"
-      "   margin:20px;\n"
-      "   border-collapse: collapse; border-spacing: 0px;\n"
-      " }\n"
-      " td.rowname {\n"
-      "   text-align:right; font-weight:bold; color:#444444;\n"
-      "   padding-right:2ex; }\n"
-      "</style>\n</head>\n<body>";
-
-  // Generate header
-  R.InsertTextBefore(StartLoc, os.str());
-  // Generate footer
-
-  R.InsertTextAfter(EndLoc, "</body></html>\n");
-}
-
-/// SyntaxHighlight - Relex the specified FileID and annotate the HTML with
-/// information about keywords, macro expansions etc.  This uses the macro
-/// table state from the end of the file, so it won't be perfectly perfect,
-/// but it will be reasonably close.
-void html::SyntaxHighlight(Rewriter &R, FileID FID, const Preprocessor &PP) {
-  RewriteBuffer &RB = R.getEditBuffer(FID);
-
-  const SourceManager &SM = PP.getSourceManager();
-  const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID);
-  Lexer L(FID, FromFile, SM, PP.getLangOpts());
-  const char *BufferStart = L.getBufferStart();
-
-  // Inform the preprocessor that we want to retain comments as tokens, so we
-  // can highlight them.
-  L.SetCommentRetentionState(true);
-
-  // Lex all the tokens in raw mode, to avoid entering #includes or expanding
-  // macros.
-  Token Tok;
-  L.LexFromRawLexer(Tok);
-
-  while (Tok.isNot(tok::eof)) {
-    // Since we are lexing unexpanded tokens, all tokens are from the main
-    // FileID.
-    unsigned TokOffs = SM.getFileOffset(Tok.getLocation());
-    unsigned TokLen = Tok.getLength();
-    switch (Tok.getKind()) {
-    default: break;
-    case tok::identifier:
-      llvm_unreachable("tok::identifier in raw lexing mode!");
-    case tok::raw_identifier: {
-      // Fill in Result.IdentifierInfo and update the token kind,
-      // looking up the identifier in the identifier table.
-      PP.LookUpIdentifierInfo(Tok);
-
-      // If this is a pp-identifier, for a keyword, highlight it as such.
-      if (Tok.isNot(tok::identifier))
-        HighlightRange(RB, TokOffs, TokOffs+TokLen, BufferStart,
-                       "<span class='keyword'>", "</span>");
-      break;
-    }
-    case tok::comment:
-      HighlightRange(RB, TokOffs, TokOffs+TokLen, BufferStart,
-                     "<span class='comment'>", "</span>");
-      break;
-    case tok::utf8_string_literal:
-      // Chop off the u part of u8 prefix
-      ++TokOffs;
-      --TokLen;
-      // FALL THROUGH to chop the 8
-    case tok::wide_string_literal:
-    case tok::utf16_string_literal:
-    case tok::utf32_string_literal:
-      // Chop off the L, u, U or 8 prefix
-      ++TokOffs;
-      --TokLen;
-      // FALL THROUGH.
-    case tok::string_literal:
-      // FIXME: Exclude the optional ud-suffix from the highlighted range.
-      HighlightRange(RB, TokOffs, TokOffs+TokLen, BufferStart,
-                     "<span class='string_literal'>", "</span>");
-      break;
-    case tok::hash: {
-      // If this is a preprocessor directive, all tokens to end of line are too.
-      if (!Tok.isAtStartOfLine())
-        break;
-
-      // Eat all of the tokens until we get to the next one at the start of
-      // line.
-      unsigned TokEnd = TokOffs+TokLen;
-      L.LexFromRawLexer(Tok);
-      while (!Tok.isAtStartOfLine() && Tok.isNot(tok::eof)) {
-        TokEnd = SM.getFileOffset(Tok.getLocation())+Tok.getLength();
-        L.LexFromRawLexer(Tok);
-      }
-
-      // Find end of line.  This is a hack.
-      HighlightRange(RB, TokOffs, TokEnd, BufferStart,
-                     "<span class='directive'>", "</span>");
-
-      // Don't skip the next token.
-      continue;
-    }
-    }
-
-    L.LexFromRawLexer(Tok);
-  }
-}
-
-/// HighlightMacros - This uses the macro table state from the end of the
-/// file, to re-expand macros and insert (into the HTML) information about the
-/// macro expansions.  This won't be perfectly perfect, but it will be
-/// reasonably close.
-void html::HighlightMacros(Rewriter &R, FileID FID, const Preprocessor& PP) {
-  // Re-lex the raw token stream into a token buffer.
-  const SourceManager &SM = PP.getSourceManager();
-  std::vector<Token> TokenStream;
-
-  const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID);
-  Lexer L(FID, FromFile, SM, PP.getLangOpts());
-
-  // Lex all the tokens in raw mode, to avoid entering #includes or expanding
-  // macros.
-  while (1) {
-    Token Tok;
-    L.LexFromRawLexer(Tok);
-
-    // If this is a # at the start of a line, discard it from the token stream.
-    // We don't want the re-preprocess step to see #defines, #includes or other
-    // preprocessor directives.
-    if (Tok.is(tok::hash) && Tok.isAtStartOfLine())
-      continue;
-
-    // If this is a ## token, change its kind to unknown so that repreprocessing
-    // it will not produce an error.
-    if (Tok.is(tok::hashhash))
-      Tok.setKind(tok::unknown);
-
-    // If this raw token is an identifier, the raw lexer won't have looked up
-    // the corresponding identifier info for it.  Do this now so that it will be
-    // macro expanded when we re-preprocess it.
-    if (Tok.is(tok::raw_identifier))
-      PP.LookUpIdentifierInfo(Tok);
-
-    TokenStream.push_back(Tok);
-
-    if (Tok.is(tok::eof)) break;
-  }
-
-  // Temporarily change the diagnostics object so that we ignore any generated
-  // diagnostics from this pass.
-  DiagnosticsEngine TmpDiags(PP.getDiagnostics().getDiagnosticIDs(),
-                      new IgnoringDiagConsumer);
-
-  // FIXME: This is a huge hack; we reuse the input preprocessor because we want
-  // its state, but we aren't actually changing it (we hope). This should really
-  // construct a copy of the preprocessor.
-  Preprocessor &TmpPP = const_cast<Preprocessor&>(PP);
-  DiagnosticsEngine *OldDiags = &TmpPP.getDiagnostics();
-  TmpPP.setDiagnostics(TmpDiags);
-
-  // Inform the preprocessor that we don't want comments.
-  TmpPP.SetCommentRetentionState(false, false);
-
-  // We don't want pragmas either. Although we filtered out #pragma, removing
-  // _Pragma and __pragma is much harder.
-  bool PragmasPreviouslyEnabled = TmpPP.getPragmasEnabled();
-  TmpPP.setPragmasEnabled(false);
-
-  // Enter the tokens we just lexed.  This will cause them to be macro expanded
-  // but won't enter sub-files (because we removed #'s).
-  TmpPP.EnterTokenStream(&TokenStream[0], TokenStream.size(), false, false);
-
-  TokenConcatenation ConcatInfo(TmpPP);
-
-  // Lex all the tokens.
-  Token Tok;
-  TmpPP.Lex(Tok);
-  while (Tok.isNot(tok::eof)) {
-    // Ignore non-macro tokens.
-    if (!Tok.getLocation().isMacroID()) {
-      TmpPP.Lex(Tok);
-      continue;
-    }
-
-    // Okay, we have the first token of a macro expansion: highlight the
-    // expansion by inserting a start tag before the macro expansion and
-    // end tag after it.
-    std::pair<SourceLocation, SourceLocation> LLoc =
-      SM.getExpansionRange(Tok.getLocation());
-
-    // Ignore tokens whose instantiation location was not the main file.
-    if (SM.getFileID(LLoc.first) != FID) {
-      TmpPP.Lex(Tok);
-      continue;
-    }
-
-    assert(SM.getFileID(LLoc.second) == FID &&
-           "Start and end of expansion must be in the same ultimate file!");
-
-    std::string Expansion = EscapeText(TmpPP.getSpelling(Tok));
-    unsigned LineLen = Expansion.size();
-
-    Token PrevPrevTok;
-    Token PrevTok = Tok;
-    // Okay, eat this token, getting the next one.
-    TmpPP.Lex(Tok);
-
-    // Skip all the rest of the tokens that are part of this macro
-    // instantiation.  It would be really nice to pop up a window with all the
-    // spelling of the tokens or something.
-    while (!Tok.is(tok::eof) &&
-           SM.getExpansionLoc(Tok.getLocation()) == LLoc.first) {
-      // Insert a newline if the macro expansion is getting large.
-      if (LineLen > 60) {
-        Expansion += "<br>";
-        LineLen = 0;
-      }
-
-      LineLen -= Expansion.size();
-
-      // If the tokens were already space separated, or if they must be to avoid
-      // them being implicitly pasted, add a space between them.
-      if (Tok.hasLeadingSpace() ||
-          ConcatInfo.AvoidConcat(PrevPrevTok, PrevTok, Tok))
-        Expansion += ' ';
-
-      // Escape any special characters in the token text.
-      Expansion += EscapeText(TmpPP.getSpelling(Tok));
-      LineLen += Expansion.size();
-
-      PrevPrevTok = PrevTok;
-      PrevTok = Tok;
-      TmpPP.Lex(Tok);
-    }
-
-
-    // Insert the expansion as the end tag, so that multi-line macros all get
-    // highlighted.
-    Expansion = "<span class='expansion'>" + Expansion + "</span></span>";
-
-    HighlightRange(R, LLoc.first, LLoc.second,
-                   "<span class='macro'>", Expansion.c_str());
-  }
-
-  // Restore the preprocessor's old state.
-  TmpPP.setDiagnostics(*OldDiags);
-  TmpPP.setPragmasEnabled(PragmasPreviouslyEnabled);
-}
diff --git a/lib/Rewrite/InclusionRewriter.cpp b/lib/Rewrite/InclusionRewriter.cpp
deleted file mode 100644
index 3dfc3b0..0000000
--- a/lib/Rewrite/InclusionRewriter.cpp
+++ /dev/null
@@ -1,361 +0,0 @@
-//===--- InclusionRewriter.cpp - Rewrite includes into their expansions ---===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This code rewrites include invocations into their expansions.  This gives you
-// a file with all included files merged into it.
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/Rewrite/Rewriters.h"
-#include "clang/Lex/Preprocessor.h"
-#include "clang/Basic/SourceManager.h"
-#include "clang/Frontend/PreprocessorOutputOptions.h"
-#include "llvm/Support/raw_ostream.h"
-
-using namespace clang;
-using namespace llvm;
-
-namespace {
-
-class InclusionRewriter : public PPCallbacks {
-  /// Information about which #includes were actually performed,
-  /// created by preprocessor callbacks.
-  struct FileChange {
-    SourceLocation From;
-    FileID Id;
-    SrcMgr::CharacteristicKind FileType;
-    FileChange(SourceLocation From) : From(From) {
-    }
-  };
-  Preprocessor &PP; ///< Used to find inclusion directives.
-  SourceManager &SM; ///< Used to read and manage source files.
-  raw_ostream &OS; ///< The destination stream for rewritten contents.
-  bool ShowLineMarkers; ///< Show #line markers.
-  bool UseLineDirective; ///< Use of line directives or line markers.
-  typedef std::map<unsigned, FileChange> FileChangeMap;
-  FileChangeMap FileChanges; /// Tracks which files were included where.
-  /// Used transitively for building up the FileChanges mapping over the
-  /// various \c PPCallbacks callbacks.
-  FileChangeMap::iterator LastInsertedFileChange;
-public:
-  InclusionRewriter(Preprocessor &PP, raw_ostream &OS, bool ShowLineMarkers);
-  bool Process(FileID FileId, SrcMgr::CharacteristicKind FileType);
-private:
-  virtual void FileChanged(SourceLocation Loc, FileChangeReason Reason,
-                           SrcMgr::CharacteristicKind FileType,
-                           FileID PrevFID);
-  virtual void FileSkipped(const FileEntry &ParentFile,
-                           const Token &FilenameTok,
-                           SrcMgr::CharacteristicKind FileType);
-  virtual void InclusionDirective(SourceLocation HashLoc,
-                                  const Token &IncludeTok,
-                                  StringRef FileName,
-                                  bool IsAngled,
-                                  const FileEntry *File,
-                                  SourceLocation EndLoc,
-                                  StringRef SearchPath,
-                                  StringRef RelativePath);
-  void WriteLineInfo(const char *Filename, int Line,
-                     SrcMgr::CharacteristicKind FileType,
-                     StringRef EOL, StringRef Extra = StringRef());
-  void OutputContentUpTo(const MemoryBuffer &FromFile,
-                         unsigned &WriteFrom, unsigned WriteTo,
-                         StringRef EOL, int &lines,
-                         bool EnsureNewline = false);
-  void CommentOutDirective(Lexer &DirectivesLex, const Token &StartToken,
-                           const MemoryBuffer &FromFile, StringRef EOL,
-                           unsigned &NextToWrite, int &Lines);
-  const FileChange *FindFileChangeLocation(SourceLocation Loc) const;
-  StringRef NextIdentifierName(Lexer &RawLex, Token &RawToken);
-};
-
-}  // end anonymous namespace
-
-/// Initializes an InclusionRewriter with a \p PP source and \p OS destination.
-InclusionRewriter::InclusionRewriter(Preprocessor &PP, raw_ostream &OS,
-                                     bool ShowLineMarkers)
-    : PP(PP), SM(PP.getSourceManager()), OS(OS),
-    ShowLineMarkers(ShowLineMarkers),
-    LastInsertedFileChange(FileChanges.end()) {
-  // If we're in microsoft mode, use normal #line instead of line markers.
-  UseLineDirective = PP.getLangOpts().MicrosoftExt;
-}
-
-/// Write appropriate line information as either #line directives or GNU line
-/// markers depending on what mode we're in, including the \p Filename and
-/// \p Line we are located at, using the specified \p EOL line separator, and
-/// any \p Extra context specifiers in GNU line directives.
-void InclusionRewriter::WriteLineInfo(const char *Filename, int Line,
-                                      SrcMgr::CharacteristicKind FileType,
-                                      StringRef EOL, StringRef Extra) {
-  if (!ShowLineMarkers)
-    return;
-  if (UseLineDirective) {
-    OS << "#line" << ' ' << Line << ' ' << '"' << Filename << '"';
-  } else {
-    // Use GNU linemarkers as described here:
-    // http://gcc.gnu.org/onlinedocs/cpp/Preprocessor-Output.html
-    OS << '#' << ' ' << Line << ' ' << '"' << Filename << '"';
-    if (!Extra.empty())
-      OS << Extra;
-    if (FileType == SrcMgr::C_System)
-      // "`3' This indicates that the following text comes from a system header
-      // file, so certain warnings should be suppressed."
-      OS << " 3";
-    else if (FileType == SrcMgr::C_ExternCSystem)
-      // as above for `3', plus "`4' This indicates that the following text
-      // should be treated as being wrapped in an implicit extern "C" block."
-      OS << " 3 4";
-  }
-  OS << EOL;
-}
-
-/// FileChanged - Whenever the preprocessor enters or exits a #include file
-/// it invokes this handler.
-void InclusionRewriter::FileChanged(SourceLocation Loc,
-                                    FileChangeReason Reason,
-                                    SrcMgr::CharacteristicKind NewFileType,
-                                    FileID) {
-  if (Reason != EnterFile)
-    return;
-  if (LastInsertedFileChange == FileChanges.end())
-    // we didn't reach this file (eg: the main file) via an inclusion directive
-    return;
-  LastInsertedFileChange->second.Id = FullSourceLoc(Loc, SM).getFileID();
-  LastInsertedFileChange->second.FileType = NewFileType;
-  LastInsertedFileChange = FileChanges.end();
-}
-
-/// Called whenever an inclusion is skipped due to canonical header protection
-/// macros.
-void InclusionRewriter::FileSkipped(const FileEntry &/*ParentFile*/,
-                                    const Token &/*FilenameTok*/,
-                                    SrcMgr::CharacteristicKind /*FileType*/) {
-  assert(LastInsertedFileChange != FileChanges.end() && "A file, that wasn't "
-    "found via an inclusion directive, was skipped");
-  FileChanges.erase(LastInsertedFileChange);
-  LastInsertedFileChange = FileChanges.end();
-}
-
-/// This should be called whenever the preprocessor encounters include
-/// directives. It does not say whether the file has been included, but it
-/// provides more information about the directive (hash location instead
-/// of location inside the included file). It is assumed that the matching
-/// FileChanged() or FileSkipped() is called after this.
-void InclusionRewriter::InclusionDirective(SourceLocation HashLoc,
-                                           const Token &/*IncludeTok*/,
-                                           StringRef /*FileName*/,
-                                           bool /*IsAngled*/,
-                                           const FileEntry * /*File*/,
-                                           SourceLocation /*EndLoc*/,
-                                           StringRef /*SearchPath*/,
-                                           StringRef /*RelativePath*/) {
-  assert(LastInsertedFileChange == FileChanges.end() && "Another inclusion "
-    "directive was found before the previous one was processed");
-  std::pair<FileChangeMap::iterator, bool> p = FileChanges.insert(
-    std::make_pair(HashLoc.getRawEncoding(), FileChange(HashLoc)));
-  assert(p.second && "Unexpected revisitation of the same include directive");
-  LastInsertedFileChange = p.first;
-}
-
-/// Simple lookup for a SourceLocation (specifically one denoting the hash in
-/// an inclusion directive) in the map of inclusion information, FileChanges.
-const InclusionRewriter::FileChange *
-InclusionRewriter::FindFileChangeLocation(SourceLocation Loc) const {
-  FileChangeMap::const_iterator I = FileChanges.find(Loc.getRawEncoding());
-  if (I != FileChanges.end())
-    return &I->second;
-  return NULL;
-}
-
-/// Detect the likely line ending style of \p FromFile by examining the first
-/// newline found within it.
-static StringRef DetectEOL(const MemoryBuffer &FromFile) {
-  // detect what line endings the file uses, so that added content does not mix
-  // the style
-  const char *Pos = strchr(FromFile.getBufferStart(), '\n');
-  if (Pos == NULL)
-    return "\n";
-  if (Pos + 1 < FromFile.getBufferEnd() && Pos[1] == '\r')
-    return "\n\r";
-  if (Pos - 1 >= FromFile.getBufferStart() && Pos[-1] == '\r')
-    return "\r\n";
-  return "\n";
-}
-
-/// Writes out bytes from \p FromFile, starting at \p NextToWrite and ending at
-/// \p WriteTo - 1.
-void InclusionRewriter::OutputContentUpTo(const MemoryBuffer &FromFile,
-                                          unsigned &WriteFrom, unsigned WriteTo,
-                                          StringRef EOL, int &Line,
-                                          bool EnsureNewline) {
-  if (WriteTo <= WriteFrom)
-    return;
-  OS.write(FromFile.getBufferStart() + WriteFrom, WriteTo - WriteFrom);
-  // count lines manually, it's faster than getPresumedLoc()
-  Line += std::count(FromFile.getBufferStart() + WriteFrom,
-                     FromFile.getBufferStart() + WriteTo, '\n');
-  if (EnsureNewline) {
-    char LastChar = FromFile.getBufferStart()[WriteTo - 1];
-    if (LastChar != '\n' && LastChar != '\r')
-      OS << EOL;
-  }
-  WriteFrom = WriteTo;
-}
-
-/// Print characters from \p FromFile starting at \p NextToWrite up until the
-/// inclusion directive at \p StartToken, then print out the inclusion
-/// inclusion directive disabled by a #if directive, updating \p NextToWrite
-/// and \p Line to track the number of source lines visited and the progress
-/// through the \p FromFile buffer.
-void InclusionRewriter::CommentOutDirective(Lexer &DirectiveLex,
-                                            const Token &StartToken,
-                                            const MemoryBuffer &FromFile,
-                                            StringRef EOL,
-                                            unsigned &NextToWrite, int &Line) {
-  OutputContentUpTo(FromFile, NextToWrite,
-    SM.getFileOffset(StartToken.getLocation()), EOL, Line);
-  Token DirectiveToken;
-  do {
-    DirectiveLex.LexFromRawLexer(DirectiveToken);
-  } while (!DirectiveToken.is(tok::eod) && DirectiveToken.isNot(tok::eof));
-  OS << "#if 0 /* expanded by -frewrite-includes */" << EOL;
-  OutputContentUpTo(FromFile, NextToWrite,
-    SM.getFileOffset(DirectiveToken.getLocation()) + DirectiveToken.getLength(),
-    EOL, Line);
-  OS << "#endif /* expanded by -frewrite-includes */" << EOL;
-}
-
-/// Find the next identifier in the pragma directive specified by \p RawToken.
-StringRef InclusionRewriter::NextIdentifierName(Lexer &RawLex,
-                                                Token &RawToken) {
-  RawLex.LexFromRawLexer(RawToken);
-  if (RawToken.is(tok::raw_identifier))
-    PP.LookUpIdentifierInfo(RawToken);
-  if (RawToken.is(tok::identifier))
-    return RawToken.getIdentifierInfo()->getName();
-  return StringRef();
-}
-
-/// Use a raw lexer to analyze \p FileId, inccrementally copying parts of it
-/// and including content of included files recursively.
-bool InclusionRewriter::Process(FileID FileId,
-                                SrcMgr::CharacteristicKind FileType)
-{
-  bool Invalid;
-  const MemoryBuffer &FromFile = *SM.getBuffer(FileId, &Invalid);
-  if (Invalid) // invalid inclusion
-    return true;
-  const char *FileName = FromFile.getBufferIdentifier();
-  Lexer RawLex(FileId, &FromFile, PP.getSourceManager(), PP.getLangOpts());
-  RawLex.SetCommentRetentionState(false);
-
-  StringRef EOL = DetectEOL(FromFile);
-
-  // Per the GNU docs: "1" indicates the start of a new file.
-  WriteLineInfo(FileName, 1, FileType, EOL, " 1");
-
-  if (SM.getFileIDSize(FileId) == 0)
-    return true;
-
-  // The next byte to be copied from the source file
-  unsigned NextToWrite = 0;
-  int Line = 1; // The current input file line number.
-
-  Token RawToken;
-  RawLex.LexFromRawLexer(RawToken);
-
-  // TODO: Consider adding a switch that strips possibly unimportant content,
-  // such as comments, to reduce the size of repro files.
-  while (RawToken.isNot(tok::eof)) {
-    if (RawToken.is(tok::hash) && RawToken.isAtStartOfLine()) {
-      RawLex.setParsingPreprocessorDirective(true);
-      Token HashToken = RawToken;
-      RawLex.LexFromRawLexer(RawToken);
-      if (RawToken.is(tok::raw_identifier))
-        PP.LookUpIdentifierInfo(RawToken);
-      if (RawToken.is(tok::identifier)) {
-        switch (RawToken.getIdentifierInfo()->getPPKeywordID()) {
-          case tok::pp_include:
-          case tok::pp_include_next:
-          case tok::pp_import: {
-            CommentOutDirective(RawLex, HashToken, FromFile, EOL, NextToWrite,
-              Line);
-            if (const FileChange *Change = FindFileChangeLocation(
-                HashToken.getLocation())) {
-              // now include and recursively process the file
-              if (Process(Change->Id, Change->FileType))
-                // and set lineinfo back to this file, if the nested one was
-                // actually included
-                // `2' indicates returning to a file (after having included
-                // another file.
-                WriteLineInfo(FileName, Line, FileType, EOL, " 2");
-            } else
-              // fix up lineinfo (since commented out directive changed line
-              // numbers) for inclusions that were skipped due to header guards
-              WriteLineInfo(FileName, Line, FileType, EOL);
-            break;
-          }
-          case tok::pp_pragma: {
-            StringRef Identifier = NextIdentifierName(RawLex, RawToken);
-            if (Identifier == "clang" || Identifier == "GCC") {
-              if (NextIdentifierName(RawLex, RawToken) == "system_header") {
-                // keep the directive in, commented out
-                CommentOutDirective(RawLex, HashToken, FromFile, EOL,
-                  NextToWrite, Line);
-                // update our own type
-                FileType = SM.getFileCharacteristic(RawToken.getLocation());
-                WriteLineInfo(FileName, Line, FileType, EOL);
-              }
-            } else if (Identifier == "once") {
-              // keep the directive in, commented out
-              CommentOutDirective(RawLex, HashToken, FromFile, EOL,
-                NextToWrite, Line);
-              WriteLineInfo(FileName, Line, FileType, EOL);
-            }
-            break;
-          }
-          default:
-            break;
-        }
-      }
-      RawLex.setParsingPreprocessorDirective(false);
-    }
-    RawLex.LexFromRawLexer(RawToken);
-  }
-  OutputContentUpTo(FromFile, NextToWrite,
-    SM.getFileOffset(SM.getLocForEndOfFile(FileId)) + 1, EOL, Line,
-    /*EnsureNewline*/true);
-  return true;
-}
-
-/// InclusionRewriterInInput - Implement -frewrite-includes mode.
-void clang::RewriteIncludesInInput(Preprocessor &PP, raw_ostream *OS,
-                                   const PreprocessorOutputOptions &Opts) {
-  SourceManager &SM = PP.getSourceManager();
-  InclusionRewriter *Rewrite = new InclusionRewriter(PP, *OS,
-                                                     Opts.ShowLineMarkers);
-  PP.addPPCallbacks(Rewrite);
-
-  // First let the preprocessor process the entire file and call callbacks.
-  // Callbacks will record which #include's were actually performed.
-  PP.EnterMainSourceFile();
-  Token Tok;
-  // Only preprocessor directives matter here, so disable macro expansion
-  // everywhere else as an optimization.
-  // TODO: It would be even faster if the preprocessor could be switched
-  // to a mode where it would parse only preprocessor directives and comments,
-  // nothing else matters for parsing or processing.
-  PP.SetMacroExpansionOnlyInDirectives();
-  do {
-    PP.Lex(Tok);
-  } while (Tok.isNot(tok::eof));
-  Rewrite->Process(SM.getMainFileID(), SrcMgr::C_User);
-  OS->flush();
-}
diff --git a/lib/Rewrite/Makefile b/lib/Rewrite/Makefile
index 5fef9b2..0be84d4 100644
--- a/lib/Rewrite/Makefile
+++ b/lib/Rewrite/Makefile
@@ -1,4 +1,4 @@
-##===- clang/lib/Rewrite/Makefile --------------------------*- Makefile -*-===##
+##===- clang/lib/StaticAnalyzer/Makefile -------------------*- Makefile -*-===##
 # 
 #                     The LLVM Compiler Infrastructure
 #
@@ -6,13 +6,9 @@
 # License. See LICENSE.TXT for details.
 # 
 ##===----------------------------------------------------------------------===##
-#
-# This implements code transformation / rewriting facilities.
-#
-##===----------------------------------------------------------------------===##
 
 CLANG_LEVEL := ../..
-LIBRARYNAME := clangRewrite
+DIRS := Frontend
+PARALLEL_DIRS := Core
 
 include $(CLANG_LEVEL)/Makefile
-
diff --git a/lib/Rewrite/RewriteMacros.cpp b/lib/Rewrite/RewriteMacros.cpp
deleted file mode 100644
index 3fa0bdb..0000000
--- a/lib/Rewrite/RewriteMacros.cpp
+++ /dev/null
@@ -1,217 +0,0 @@
-//===--- RewriteMacros.cpp - Rewrite macros into their expansions ---------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This code rewrites macro invocations into their expansions.  This gives you
-// a macro expanded file that retains comments and #includes.
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/Rewrite/Rewriters.h"
-#include "clang/Rewrite/Rewriter.h"
-#include "clang/Lex/Preprocessor.h"
-#include "clang/Basic/SourceManager.h"
-#include "llvm/Support/raw_ostream.h"
-#include "llvm/Support/Path.h"
-#include "llvm/ADT/OwningPtr.h"
-#include <cstdio>
-
-using namespace clang;
-
-/// isSameToken - Return true if the two specified tokens start have the same
-/// content.
-static bool isSameToken(Token &RawTok, Token &PPTok) {
-  // If two tokens have the same kind and the same identifier info, they are
-  // obviously the same.
-  if (PPTok.getKind() == RawTok.getKind() &&
-      PPTok.getIdentifierInfo() == RawTok.getIdentifierInfo())
-    return true;
-
-  // Otherwise, if they are different but have the same identifier info, they
-  // are also considered to be the same.  This allows keywords and raw lexed
-  // identifiers with the same name to be treated the same.
-  if (PPTok.getIdentifierInfo() &&
-      PPTok.getIdentifierInfo() == RawTok.getIdentifierInfo())
-    return true;
-
-  return false;
-}
-
-
-/// GetNextRawTok - Return the next raw token in the stream, skipping over
-/// comments if ReturnComment is false.
-static const Token &GetNextRawTok(const std::vector<Token> &RawTokens,
-                                  unsigned &CurTok, bool ReturnComment) {
-  assert(CurTok < RawTokens.size() && "Overran eof!");
-
-  // If the client doesn't want comments and we have one, skip it.
-  if (!ReturnComment && RawTokens[CurTok].is(tok::comment))
-    ++CurTok;
-
-  return RawTokens[CurTok++];
-}
-
-
-/// LexRawTokensFromMainFile - Lets all the raw tokens from the main file into
-/// the specified vector.
-static void LexRawTokensFromMainFile(Preprocessor &PP,
-                                     std::vector<Token> &RawTokens) {
-  SourceManager &SM = PP.getSourceManager();
-
-  // Create a lexer to lex all the tokens of the main file in raw mode.  Even
-  // though it is in raw mode, it will not return comments.
-  const llvm::MemoryBuffer *FromFile = SM.getBuffer(SM.getMainFileID());
-  Lexer RawLex(SM.getMainFileID(), FromFile, SM, PP.getLangOpts());
-
-  // Switch on comment lexing because we really do want them.
-  RawLex.SetCommentRetentionState(true);
-
-  Token RawTok;
-  do {
-    RawLex.LexFromRawLexer(RawTok);
-
-    // If we have an identifier with no identifier info for our raw token, look
-    // up the indentifier info.  This is important for equality comparison of
-    // identifier tokens.
-    if (RawTok.is(tok::raw_identifier))
-      PP.LookUpIdentifierInfo(RawTok);
-
-    RawTokens.push_back(RawTok);
-  } while (RawTok.isNot(tok::eof));
-}
-
-
-/// RewriteMacrosInInput - Implement -rewrite-macros mode.
-void clang::RewriteMacrosInInput(Preprocessor &PP, raw_ostream *OS) {
-  SourceManager &SM = PP.getSourceManager();
-
-  Rewriter Rewrite;
-  Rewrite.setSourceMgr(SM, PP.getLangOpts());
-  RewriteBuffer &RB = Rewrite.getEditBuffer(SM.getMainFileID());
-
-  std::vector<Token> RawTokens;
-  LexRawTokensFromMainFile(PP, RawTokens);
-  unsigned CurRawTok = 0;
-  Token RawTok = GetNextRawTok(RawTokens, CurRawTok, false);
-
-
-  // Get the first preprocessing token.
-  PP.EnterMainSourceFile();
-  Token PPTok;
-  PP.Lex(PPTok);
-
-  // Preprocess the input file in parallel with raw lexing the main file. Ignore
-  // all tokens that are preprocessed from a file other than the main file (e.g.
-  // a header).  If we see tokens that are in the preprocessed file but not the
-  // lexed file, we have a macro expansion.  If we see tokens in the lexed file
-  // that aren't in the preprocessed view, we have macros that expand to no
-  // tokens, or macro arguments etc.
-  while (RawTok.isNot(tok::eof) || PPTok.isNot(tok::eof)) {
-    SourceLocation PPLoc = SM.getExpansionLoc(PPTok.getLocation());
-
-    // If PPTok is from a different source file, ignore it.
-    if (!SM.isFromMainFile(PPLoc)) {
-      PP.Lex(PPTok);
-      continue;
-    }
-
-    // If the raw file hits a preprocessor directive, they will be extra tokens
-    // in the raw file that don't exist in the preprocsesed file.  However, we
-    // choose to preserve them in the output file and otherwise handle them
-    // specially.
-    if (RawTok.is(tok::hash) && RawTok.isAtStartOfLine()) {
-      // If this is a #warning directive or #pragma mark (GNU extensions),
-      // comment the line out.
-      if (RawTokens[CurRawTok].is(tok::identifier)) {
-        const IdentifierInfo *II = RawTokens[CurRawTok].getIdentifierInfo();
-        if (II->getName() == "warning") {
-          // Comment out #warning.
-          RB.InsertTextAfter(SM.getFileOffset(RawTok.getLocation()), "//");
-        } else if (II->getName() == "pragma" &&
-                   RawTokens[CurRawTok+1].is(tok::identifier) &&
-                   (RawTokens[CurRawTok+1].getIdentifierInfo()->getName() ==
-                    "mark")) {
-          // Comment out #pragma mark.
-          RB.InsertTextAfter(SM.getFileOffset(RawTok.getLocation()), "//");
-        }
-      }
-
-      // Otherwise, if this is a #include or some other directive, just leave it
-      // in the file by skipping over the line.
-      RawTok = GetNextRawTok(RawTokens, CurRawTok, false);
-      while (!RawTok.isAtStartOfLine() && RawTok.isNot(tok::eof))
-        RawTok = GetNextRawTok(RawTokens, CurRawTok, false);
-      continue;
-    }
-
-    // Okay, both tokens are from the same file.  Get their offsets from the
-    // start of the file.
-    unsigned PPOffs = SM.getFileOffset(PPLoc);
-    unsigned RawOffs = SM.getFileOffset(RawTok.getLocation());
-
-    // If the offsets are the same and the token kind is the same, ignore them.
-    if (PPOffs == RawOffs && isSameToken(RawTok, PPTok)) {
-      RawTok = GetNextRawTok(RawTokens, CurRawTok, false);
-      PP.Lex(PPTok);
-      continue;
-    }
-
-    // If the PP token is farther along than the raw token, something was
-    // deleted.  Comment out the raw token.
-    if (RawOffs <= PPOffs) {
-      // Comment out a whole run of tokens instead of bracketing each one with
-      // comments.  Add a leading space if RawTok didn't have one.
-      bool HasSpace = RawTok.hasLeadingSpace();
-      RB.InsertTextAfter(RawOffs, &" /*"[HasSpace]);
-      unsigned EndPos;
-
-      do {
-        EndPos = RawOffs+RawTok.getLength();
-
-        RawTok = GetNextRawTok(RawTokens, CurRawTok, true);
-        RawOffs = SM.getFileOffset(RawTok.getLocation());
-
-        if (RawTok.is(tok::comment)) {
-          // Skip past the comment.
-          RawTok = GetNextRawTok(RawTokens, CurRawTok, false);
-          break;
-        }
-
-      } while (RawOffs <= PPOffs && !RawTok.isAtStartOfLine() &&
-               (PPOffs != RawOffs || !isSameToken(RawTok, PPTok)));
-
-      RB.InsertTextBefore(EndPos, "*/");
-      continue;
-    }
-
-    // Otherwise, there was a replacement an expansion.  Insert the new token
-    // in the output buffer.  Insert the whole run of new tokens at once to get
-    // them in the right order.
-    unsigned InsertPos = PPOffs;
-    std::string Expansion;
-    while (PPOffs < RawOffs) {
-      Expansion += ' ' + PP.getSpelling(PPTok);
-      PP.Lex(PPTok);
-      PPLoc = SM.getExpansionLoc(PPTok.getLocation());
-      PPOffs = SM.getFileOffset(PPLoc);
-    }
-    Expansion += ' ';
-    RB.InsertTextBefore(InsertPos, Expansion);
-  }
-
-  // Get the buffer corresponding to MainFileID.  If we haven't changed it, then
-  // we are done.
-  if (const RewriteBuffer *RewriteBuf =
-      Rewrite.getRewriteBufferFor(SM.getMainFileID())) {
-    //printf("Changed:\n");
-    *OS << std::string(RewriteBuf->begin(), RewriteBuf->end());
-  } else {
-    fprintf(stderr, "No changes\n");
-  }
-  OS->flush();
-}
diff --git a/lib/Rewrite/RewriteModernObjC.cpp b/lib/Rewrite/RewriteModernObjC.cpp
deleted file mode 100644
index dcd003f..0000000
--- a/lib/Rewrite/RewriteModernObjC.cpp
+++ /dev/null
@@ -1,7543 +0,0 @@
-//===--- RewriteObjC.cpp - Playground for the code rewriter ---------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Hacks and fun related to the code rewriter.
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/Rewrite/ASTConsumers.h"
-#include "clang/Rewrite/Rewriter.h"
-#include "clang/AST/AST.h"
-#include "clang/AST/ASTConsumer.h"
-#include "clang/AST/ParentMap.h"
-#include "clang/Basic/SourceManager.h"
-#include "clang/Basic/IdentifierTable.h"
-#include "clang/Basic/Diagnostic.h"
-#include "clang/Lex/Lexer.h"
-#include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/raw_ostream.h"
-#include "llvm/ADT/StringExtras.h"
-#include "llvm/ADT/SmallPtrSet.h"
-#include "llvm/ADT/OwningPtr.h"
-#include "llvm/ADT/DenseSet.h"
-
-using namespace clang;
-using llvm::utostr;
-
-namespace {
-  class RewriteModernObjC : public ASTConsumer {
-  protected:
-    
-    enum {
-      BLOCK_FIELD_IS_OBJECT   =  3,  /* id, NSObject, __attribute__((NSObject)),
-                                        block, ... */
-      BLOCK_FIELD_IS_BLOCK    =  7,  /* a block variable */
-      BLOCK_FIELD_IS_BYREF    =  8,  /* the on stack structure holding the 
-                                        __block variable */
-      BLOCK_FIELD_IS_WEAK     = 16,  /* declared __weak, only used in byref copy
-                                        helpers */
-      BLOCK_BYREF_CALLER      = 128, /* called from __block (byref) copy/dispose
-                                        support routines */
-      BLOCK_BYREF_CURRENT_MAX = 256
-    };
-    
-    enum {
-      BLOCK_NEEDS_FREE =        (1 << 24),
-      BLOCK_HAS_COPY_DISPOSE =  (1 << 25),
-      BLOCK_HAS_CXX_OBJ =       (1 << 26),
-      BLOCK_IS_GC =             (1 << 27),
-      BLOCK_IS_GLOBAL =         (1 << 28),
-      BLOCK_HAS_DESCRIPTOR =    (1 << 29)
-    };
-    static const int OBJC_ABI_VERSION = 7;
-    
-    Rewriter Rewrite;
-    DiagnosticsEngine &Diags;
-    const LangOptions &LangOpts;
-    ASTContext *Context;
-    SourceManager *SM;
-    TranslationUnitDecl *TUDecl;
-    FileID MainFileID;
-    const char *MainFileStart, *MainFileEnd;
-    Stmt *CurrentBody;
-    ParentMap *PropParentMap; // created lazily.
-    std::string InFileName;
-    raw_ostream* OutFile;
-    std::string Preamble;
-    
-    TypeDecl *ProtocolTypeDecl;
-    VarDecl *GlobalVarDecl;
-    Expr *GlobalConstructionExp;
-    unsigned RewriteFailedDiag;
-    unsigned GlobalBlockRewriteFailedDiag;
-    // ObjC string constant support.
-    unsigned NumObjCStringLiterals;
-    VarDecl *ConstantStringClassReference;
-    RecordDecl *NSStringRecord;
-
-    // ObjC foreach break/continue generation support.
-    int BcLabelCount;
-    
-    unsigned TryFinallyContainsReturnDiag;
-    // Needed for super.
-    ObjCMethodDecl *CurMethodDef;
-    RecordDecl *SuperStructDecl;
-    RecordDecl *ConstantStringDecl;
-    
-    FunctionDecl *MsgSendFunctionDecl;
-    FunctionDecl *MsgSendSuperFunctionDecl;
-    FunctionDecl *MsgSendStretFunctionDecl;
-    FunctionDecl *MsgSendSuperStretFunctionDecl;
-    FunctionDecl *MsgSendFpretFunctionDecl;
-    FunctionDecl *GetClassFunctionDecl;
-    FunctionDecl *GetMetaClassFunctionDecl;
-    FunctionDecl *GetSuperClassFunctionDecl;
-    FunctionDecl *SelGetUidFunctionDecl;
-    FunctionDecl *CFStringFunctionDecl;
-    FunctionDecl *SuperContructorFunctionDecl;
-    FunctionDecl *CurFunctionDef;
-
-    /* Misc. containers needed for meta-data rewrite. */
-    SmallVector<ObjCImplementationDecl *, 8> ClassImplementation;
-    SmallVector<ObjCCategoryImplDecl *, 8> CategoryImplementation;
-    llvm::SmallPtrSet<ObjCInterfaceDecl*, 8> ObjCSynthesizedStructs;
-    llvm::SmallPtrSet<ObjCProtocolDecl*, 8> ObjCSynthesizedProtocols;
-    llvm::SmallPtrSet<ObjCInterfaceDecl*, 8> ObjCWrittenInterfaces;
-    llvm::SmallPtrSet<TagDecl*, 32> GlobalDefinedTags;
-    SmallVector<ObjCInterfaceDecl*, 32> ObjCInterfacesSeen;
-    /// DefinedNonLazyClasses - List of defined "non-lazy" classes.
-    SmallVector<ObjCInterfaceDecl*, 8> DefinedNonLazyClasses;
-    
-    /// DefinedNonLazyCategories - List of defined "non-lazy" categories.
-    llvm::SmallVector<ObjCCategoryDecl*, 8> DefinedNonLazyCategories;
-    
-    SmallVector<Stmt *, 32> Stmts;
-    SmallVector<int, 8> ObjCBcLabelNo;
-    // Remember all the @protocol(<expr>) expressions.
-    llvm::SmallPtrSet<ObjCProtocolDecl *, 32> ProtocolExprDecls;
-    
-    llvm::DenseSet<uint64_t> CopyDestroyCache;
-
-    // Block expressions.
-    SmallVector<BlockExpr *, 32> Blocks;
-    SmallVector<int, 32> InnerDeclRefsCount;
-    SmallVector<DeclRefExpr *, 32> InnerDeclRefs;
-    
-    SmallVector<DeclRefExpr *, 32> BlockDeclRefs;
-
-    // Block related declarations.
-    SmallVector<ValueDecl *, 8> BlockByCopyDecls;
-    llvm::SmallPtrSet<ValueDecl *, 8> BlockByCopyDeclsPtrSet;
-    SmallVector<ValueDecl *, 8> BlockByRefDecls;
-    llvm::SmallPtrSet<ValueDecl *, 8> BlockByRefDeclsPtrSet;
-    llvm::DenseMap<ValueDecl *, unsigned> BlockByRefDeclNo;
-    llvm::SmallPtrSet<ValueDecl *, 8> ImportedBlockDecls;
-    llvm::SmallPtrSet<VarDecl *, 8> ImportedLocalExternalDecls;
-    
-    llvm::DenseMap<BlockExpr *, std::string> RewrittenBlockExprs;
-    llvm::DenseMap<ObjCInterfaceDecl *, 
-                    llvm::SmallPtrSet<ObjCIvarDecl *, 8> > ReferencedIvars;
-    
-    // This maps an original source AST to it's rewritten form. This allows
-    // us to avoid rewriting the same node twice (which is very uncommon).
-    // This is needed to support some of the exotic property rewriting.
-    llvm::DenseMap<Stmt *, Stmt *> ReplacedNodes;
-
-    // Needed for header files being rewritten
-    bool IsHeader;
-    bool SilenceRewriteMacroWarning;
-    bool objc_impl_method;
-    
-    bool DisableReplaceStmt;
-    class DisableReplaceStmtScope {
-      RewriteModernObjC &R;
-      bool SavedValue;
-    
-    public:
-      DisableReplaceStmtScope(RewriteModernObjC &R)
-        : R(R), SavedValue(R.DisableReplaceStmt) {
-        R.DisableReplaceStmt = true;
-      }
-      ~DisableReplaceStmtScope() {
-        R.DisableReplaceStmt = SavedValue;
-      }
-    };
-    void InitializeCommon(ASTContext &context);
-
-  public:
-    llvm::DenseMap<ObjCMethodDecl*, std::string> MethodInternalNames;
-    // Top Level Driver code.
-    virtual bool HandleTopLevelDecl(DeclGroupRef D) {
-      for (DeclGroupRef::iterator I = D.begin(), E = D.end(); I != E; ++I) {
-        if (ObjCInterfaceDecl *Class = dyn_cast<ObjCInterfaceDecl>(*I)) {
-          if (!Class->isThisDeclarationADefinition()) {
-            RewriteForwardClassDecl(D);
-            break;
-          } else {
-            // Keep track of all interface declarations seen.
-            ObjCInterfacesSeen.push_back(Class);
-            break;
-          }
-        }
-
-        if (ObjCProtocolDecl *Proto = dyn_cast<ObjCProtocolDecl>(*I)) {
-          if (!Proto->isThisDeclarationADefinition()) {
-            RewriteForwardProtocolDecl(D);
-            break;
-          }
-        }
-
-        HandleTopLevelSingleDecl(*I);
-      }
-      return true;
-    }
-    void HandleTopLevelSingleDecl(Decl *D);
-    void HandleDeclInMainFile(Decl *D);
-    RewriteModernObjC(std::string inFile, raw_ostream *OS,
-                DiagnosticsEngine &D, const LangOptions &LOpts,
-                bool silenceMacroWarn);
-    
-    ~RewriteModernObjC() {}
-    
-    virtual void HandleTranslationUnit(ASTContext &C);
-
-    void ReplaceStmt(Stmt *Old, Stmt *New) {
-      Stmt *ReplacingStmt = ReplacedNodes[Old];
-
-      if (ReplacingStmt)
-        return; // We can't rewrite the same node twice.
-
-      if (DisableReplaceStmt)
-        return;
-
-      // If replacement succeeded or warning disabled return with no warning.
-      if (!Rewrite.ReplaceStmt(Old, New)) {
-        ReplacedNodes[Old] = New;
-        return;
-      }
-      if (SilenceRewriteMacroWarning)
-        return;
-      Diags.Report(Context->getFullLoc(Old->getLocStart()), RewriteFailedDiag)
-                   << Old->getSourceRange();
-    }
-
-    void ReplaceStmtWithRange(Stmt *Old, Stmt *New, SourceRange SrcRange) {
-      if (DisableReplaceStmt)
-        return;
-
-      // Measure the old text.
-      int Size = Rewrite.getRangeSize(SrcRange);
-      if (Size == -1) {
-        Diags.Report(Context->getFullLoc(Old->getLocStart()), RewriteFailedDiag)
-                     << Old->getSourceRange();
-        return;
-      }
-      // Get the new text.
-      std::string SStr;
-      llvm::raw_string_ostream S(SStr);
-      New->printPretty(S, 0, PrintingPolicy(LangOpts));
-      const std::string &Str = S.str();
-
-      // If replacement succeeded or warning disabled return with no warning.
-      if (!Rewrite.ReplaceText(SrcRange.getBegin(), Size, Str)) {
-        ReplacedNodes[Old] = New;
-        return;
-      }
-      if (SilenceRewriteMacroWarning)
-        return;
-      Diags.Report(Context->getFullLoc(Old->getLocStart()), RewriteFailedDiag)
-                   << Old->getSourceRange();
-    }
-
-    void InsertText(SourceLocation Loc, StringRef Str,
-                    bool InsertAfter = true) {
-      // If insertion succeeded or warning disabled return with no warning.
-      if (!Rewrite.InsertText(Loc, Str, InsertAfter) ||
-          SilenceRewriteMacroWarning)
-        return;
-
-      Diags.Report(Context->getFullLoc(Loc), RewriteFailedDiag);
-    }
-
-    void ReplaceText(SourceLocation Start, unsigned OrigLength,
-                     StringRef Str) {
-      // If removal succeeded or warning disabled return with no warning.
-      if (!Rewrite.ReplaceText(Start, OrigLength, Str) ||
-          SilenceRewriteMacroWarning)
-        return;
-
-      Diags.Report(Context->getFullLoc(Start), RewriteFailedDiag);
-    }
-
-    // Syntactic Rewriting.
-    void RewriteRecordBody(RecordDecl *RD);
-    void RewriteInclude();
-    void RewriteForwardClassDecl(DeclGroupRef D);
-    void RewriteForwardClassDecl(const llvm::SmallVector<Decl*, 8> &DG);
-    void RewriteForwardClassEpilogue(ObjCInterfaceDecl *ClassDecl, 
-                                     const std::string &typedefString);
-    void RewriteImplementations();
-    void RewritePropertyImplDecl(ObjCPropertyImplDecl *PID,
-                                 ObjCImplementationDecl *IMD,
-                                 ObjCCategoryImplDecl *CID);
-    void RewriteInterfaceDecl(ObjCInterfaceDecl *Dcl);
-    void RewriteImplementationDecl(Decl *Dcl);
-    void RewriteObjCMethodDecl(const ObjCInterfaceDecl *IDecl,
-                               ObjCMethodDecl *MDecl, std::string &ResultStr);
-    void RewriteTypeIntoString(QualType T, std::string &ResultStr,
-                               const FunctionType *&FPRetType);
-    void RewriteByRefString(std::string &ResultStr, const std::string &Name,
-                            ValueDecl *VD, bool def=false);
-    void RewriteCategoryDecl(ObjCCategoryDecl *Dcl);
-    void RewriteProtocolDecl(ObjCProtocolDecl *Dcl);
-    void RewriteForwardProtocolDecl(DeclGroupRef D);
-    void RewriteForwardProtocolDecl(const llvm::SmallVector<Decl*, 8> &DG);
-    void RewriteMethodDeclaration(ObjCMethodDecl *Method);
-    void RewriteProperty(ObjCPropertyDecl *prop);
-    void RewriteFunctionDecl(FunctionDecl *FD);
-    void RewriteBlockPointerType(std::string& Str, QualType Type);
-    void RewriteBlockPointerTypeVariable(std::string& Str, ValueDecl *VD);
-    void RewriteBlockLiteralFunctionDecl(FunctionDecl *FD);
-    void RewriteObjCQualifiedInterfaceTypes(Decl *Dcl);
-    void RewriteTypeOfDecl(VarDecl *VD);
-    void RewriteObjCQualifiedInterfaceTypes(Expr *E);
-    
-    std::string getIvarAccessString(ObjCIvarDecl *D);
-  
-    // Expression Rewriting.
-    Stmt *RewriteFunctionBodyOrGlobalInitializer(Stmt *S);
-    Stmt *RewriteAtEncode(ObjCEncodeExpr *Exp);
-    Stmt *RewritePropertyOrImplicitGetter(PseudoObjectExpr *Pseudo);
-    Stmt *RewritePropertyOrImplicitSetter(PseudoObjectExpr *Pseudo);
-    Stmt *RewriteAtSelector(ObjCSelectorExpr *Exp);
-    Stmt *RewriteMessageExpr(ObjCMessageExpr *Exp);
-    Stmt *RewriteObjCStringLiteral(ObjCStringLiteral *Exp);
-    Stmt *RewriteObjCBoolLiteralExpr(ObjCBoolLiteralExpr *Exp);
-    Stmt *RewriteObjCBoxedExpr(ObjCBoxedExpr *Exp);
-    Stmt *RewriteObjCArrayLiteralExpr(ObjCArrayLiteral *Exp);
-    Stmt *RewriteObjCDictionaryLiteralExpr(ObjCDictionaryLiteral *Exp);
-    Stmt *RewriteObjCProtocolExpr(ObjCProtocolExpr *Exp);
-    Stmt *RewriteObjCTryStmt(ObjCAtTryStmt *S);
-    Stmt *RewriteObjCAutoreleasePoolStmt(ObjCAutoreleasePoolStmt  *S);
-    Stmt *RewriteObjCSynchronizedStmt(ObjCAtSynchronizedStmt *S);
-    Stmt *RewriteObjCThrowStmt(ObjCAtThrowStmt *S);
-    Stmt *RewriteObjCForCollectionStmt(ObjCForCollectionStmt *S,
-                                       SourceLocation OrigEnd);
-    Stmt *RewriteBreakStmt(BreakStmt *S);
-    Stmt *RewriteContinueStmt(ContinueStmt *S);
-    void RewriteCastExpr(CStyleCastExpr *CE);
-    void RewriteImplicitCastObjCExpr(CastExpr *IE);
-    void RewriteLinkageSpec(LinkageSpecDecl *LSD);
-    
-    // Block rewriting.
-    void RewriteBlocksInFunctionProtoType(QualType funcType, NamedDecl *D);
-    
-    // Block specific rewrite rules.
-    void RewriteBlockPointerDecl(NamedDecl *VD);
-    void RewriteByRefVar(VarDecl *VD, bool firstDecl, bool lastDecl);
-    Stmt *RewriteBlockDeclRefExpr(DeclRefExpr *VD);
-    Stmt *RewriteLocalVariableExternalStorage(DeclRefExpr *DRE);
-    void RewriteBlockPointerFunctionArgs(FunctionDecl *FD);
-    
-    void RewriteObjCInternalStruct(ObjCInterfaceDecl *CDecl,
-                                      std::string &Result);
-    
-    void RewriteObjCFieldDecl(FieldDecl *fieldDecl, std::string &Result);
-    bool IsTagDefinedInsideClass(ObjCContainerDecl *IDecl, TagDecl *Tag,
-                                 bool &IsNamedDefinition);
-    void RewriteLocallyDefinedNamedAggregates(FieldDecl *fieldDecl, 
-                                              std::string &Result);
-    
-    bool RewriteObjCFieldDeclType(QualType &Type, std::string &Result);
-    
-    void RewriteIvarOffsetSymbols(ObjCInterfaceDecl *CDecl,
-                                  std::string &Result);
-    
-    virtual void Initialize(ASTContext &context);
-    
-    // Misc. AST transformation routines. Sometimes they end up calling
-    // rewriting routines on the new ASTs.
-    CallExpr *SynthesizeCallToFunctionDecl(FunctionDecl *FD,
-                                           Expr **args, unsigned nargs,
-                                           SourceLocation StartLoc=SourceLocation(),
-                                           SourceLocation EndLoc=SourceLocation());
-    
-    Expr *SynthMsgSendStretCallExpr(FunctionDecl *MsgSendStretFlavor,
-                                        QualType msgSendType, 
-                                        QualType returnType, 
-                                        SmallVectorImpl<QualType> &ArgTypes,
-                                        SmallVectorImpl<Expr*> &MsgExprs,
-                                        ObjCMethodDecl *Method);
-
-    Stmt *SynthMessageExpr(ObjCMessageExpr *Exp,
-                           SourceLocation StartLoc=SourceLocation(),
-                           SourceLocation EndLoc=SourceLocation());
-    
-    void SynthCountByEnumWithState(std::string &buf);
-    void SynthMsgSendFunctionDecl();
-    void SynthMsgSendSuperFunctionDecl();
-    void SynthMsgSendStretFunctionDecl();
-    void SynthMsgSendFpretFunctionDecl();
-    void SynthMsgSendSuperStretFunctionDecl();
-    void SynthGetClassFunctionDecl();
-    void SynthGetMetaClassFunctionDecl();
-    void SynthGetSuperClassFunctionDecl();
-    void SynthSelGetUidFunctionDecl();
-    void SynthSuperContructorFunctionDecl();
-    
-    // Rewriting metadata
-    template<typename MethodIterator>
-    void RewriteObjCMethodsMetaData(MethodIterator MethodBegin,
-                                    MethodIterator MethodEnd,
-                                    bool IsInstanceMethod,
-                                    StringRef prefix,
-                                    StringRef ClassName,
-                                    std::string &Result);
-    void RewriteObjCProtocolMetaData(ObjCProtocolDecl *Protocol,
-                                     std::string &Result);
-    void RewriteObjCProtocolListMetaData(
-                   const ObjCList<ObjCProtocolDecl> &Prots,
-                   StringRef prefix, StringRef ClassName, std::string &Result);
-    void RewriteObjCClassMetaData(ObjCImplementationDecl *IDecl,
-                                          std::string &Result);
-    void RewriteClassSetupInitHook(std::string &Result);
-    
-    void RewriteMetaDataIntoBuffer(std::string &Result);
-    void WriteImageInfo(std::string &Result);
-    void RewriteObjCCategoryImplDecl(ObjCCategoryImplDecl *CDecl,
-                                             std::string &Result);
-    void RewriteCategorySetupInitHook(std::string &Result);
-    
-    // Rewriting ivar
-    void RewriteIvarOffsetComputation(ObjCIvarDecl *ivar,
-                                              std::string &Result);
-    Stmt *RewriteObjCIvarRefExpr(ObjCIvarRefExpr *IV);
-
-    
-    std::string SynthesizeByrefCopyDestroyHelper(VarDecl *VD, int flag);
-    std::string SynthesizeBlockHelperFuncs(BlockExpr *CE, int i,
-                                      StringRef funcName, std::string Tag);
-    std::string SynthesizeBlockFunc(BlockExpr *CE, int i,
-                                      StringRef funcName, std::string Tag);
-    std::string SynthesizeBlockImpl(BlockExpr *CE, 
-                                    std::string Tag, std::string Desc);
-    std::string SynthesizeBlockDescriptor(std::string DescTag, 
-                                          std::string ImplTag,
-                                          int i, StringRef funcName,
-                                          unsigned hasCopy);
-    Stmt *SynthesizeBlockCall(CallExpr *Exp, const Expr* BlockExp);
-    void SynthesizeBlockLiterals(SourceLocation FunLocStart,
-                                 StringRef FunName);
-    FunctionDecl *SynthBlockInitFunctionDecl(StringRef name);
-    Stmt *SynthBlockInitExpr(BlockExpr *Exp,
-            const SmallVector<DeclRefExpr *, 8> &InnerBlockDeclRefs);
-
-    // Misc. helper routines.
-    QualType getProtocolType();
-    void WarnAboutReturnGotoStmts(Stmt *S);
-    void CheckFunctionPointerDecl(QualType dType, NamedDecl *ND);
-    void InsertBlockLiteralsWithinFunction(FunctionDecl *FD);
-    void InsertBlockLiteralsWithinMethod(ObjCMethodDecl *MD);
-
-    bool IsDeclStmtInForeachHeader(DeclStmt *DS);
-    void CollectBlockDeclRefInfo(BlockExpr *Exp);
-    void GetBlockDeclRefExprs(Stmt *S);
-    void GetInnerBlockDeclRefExprs(Stmt *S, 
-                SmallVector<DeclRefExpr *, 8> &InnerBlockDeclRefs,
-                llvm::SmallPtrSet<const DeclContext *, 8> &InnerContexts);
-
-    // We avoid calling Type::isBlockPointerType(), since it operates on the
-    // canonical type. We only care if the top-level type is a closure pointer.
-    bool isTopLevelBlockPointerType(QualType T) {
-      return isa<BlockPointerType>(T);
-    }
-
-    /// convertBlockPointerToFunctionPointer - Converts a block-pointer type
-    /// to a function pointer type and upon success, returns true; false
-    /// otherwise.
-    bool convertBlockPointerToFunctionPointer(QualType &T) {
-      if (isTopLevelBlockPointerType(T)) {
-        const BlockPointerType *BPT = T->getAs<BlockPointerType>();
-        T = Context->getPointerType(BPT->getPointeeType());
-        return true;
-      }
-      return false;
-    }
-    
-    bool convertObjCTypeToCStyleType(QualType &T);
-    
-    bool needToScanForQualifiers(QualType T);
-    QualType getSuperStructType();
-    QualType getConstantStringStructType();
-    QualType convertFunctionTypeOfBlocks(const FunctionType *FT);
-    bool BufferContainsPPDirectives(const char *startBuf, const char *endBuf);
-    
-    void convertToUnqualifiedObjCType(QualType &T) {
-      if (T->isObjCQualifiedIdType()) {
-        bool isConst = T.isConstQualified();
-        T = isConst ? Context->getObjCIdType().withConst() 
-                    : Context->getObjCIdType();
-      }
-      else if (T->isObjCQualifiedClassType())
-        T = Context->getObjCClassType();
-      else if (T->isObjCObjectPointerType() &&
-               T->getPointeeType()->isObjCQualifiedInterfaceType()) {
-        if (const ObjCObjectPointerType * OBJPT =
-              T->getAsObjCInterfacePointerType()) {
-          const ObjCInterfaceType *IFaceT = OBJPT->getInterfaceType();
-          T = QualType(IFaceT, 0);
-          T = Context->getPointerType(T);
-        }
-     }
-    }
-    
-    // FIXME: This predicate seems like it would be useful to add to ASTContext.
-    bool isObjCType(QualType T) {
-      if (!LangOpts.ObjC1 && !LangOpts.ObjC2)
-        return false;
-
-      QualType OCT = Context->getCanonicalType(T).getUnqualifiedType();
-
-      if (OCT == Context->getCanonicalType(Context->getObjCIdType()) ||
-          OCT == Context->getCanonicalType(Context->getObjCClassType()))
-        return true;
-
-      if (const PointerType *PT = OCT->getAs<PointerType>()) {
-        if (isa<ObjCInterfaceType>(PT->getPointeeType()) ||
-            PT->getPointeeType()->isObjCQualifiedIdType())
-          return true;
-      }
-      return false;
-    }
-    bool PointerTypeTakesAnyBlockArguments(QualType QT);
-    bool PointerTypeTakesAnyObjCQualifiedType(QualType QT);
-    void GetExtentOfArgList(const char *Name, const char *&LParen,
-                            const char *&RParen);
-    
-    void QuoteDoublequotes(std::string &From, std::string &To) {
-      for (unsigned i = 0; i < From.length(); i++) {
-        if (From[i] == '"')
-          To += "\\\"";
-        else
-          To += From[i];
-      }
-    }
-
-    QualType getSimpleFunctionType(QualType result,
-                                   const QualType *args,
-                                   unsigned numArgs,
-                                   bool variadic = false) {
-      if (result == Context->getObjCInstanceType())
-        result =  Context->getObjCIdType();
-      FunctionProtoType::ExtProtoInfo fpi;
-      fpi.Variadic = variadic;
-      return Context->getFunctionType(result, args, numArgs, fpi);
-    }
-
-    // Helper function: create a CStyleCastExpr with trivial type source info.
-    CStyleCastExpr* NoTypeInfoCStyleCastExpr(ASTContext *Ctx, QualType Ty,
-                                             CastKind Kind, Expr *E) {
-      TypeSourceInfo *TInfo = Ctx->getTrivialTypeSourceInfo(Ty, SourceLocation());
-      return CStyleCastExpr::Create(*Ctx, Ty, VK_RValue, Kind, E, 0, TInfo,
-                                    SourceLocation(), SourceLocation());
-    }
-    
-    bool ImplementationIsNonLazy(const ObjCImplDecl *OD) const {
-      IdentifierInfo* II = &Context->Idents.get("load");
-      Selector LoadSel = Context->Selectors.getSelector(0, &II);
-      return OD->getClassMethod(LoadSel) != 0;
-    }
-  };
-  
-}
-
-void RewriteModernObjC::RewriteBlocksInFunctionProtoType(QualType funcType,
-                                                   NamedDecl *D) {
-  if (const FunctionProtoType *fproto
-      = dyn_cast<FunctionProtoType>(funcType.IgnoreParens())) {
-    for (FunctionProtoType::arg_type_iterator I = fproto->arg_type_begin(),
-         E = fproto->arg_type_end(); I && (I != E); ++I)
-      if (isTopLevelBlockPointerType(*I)) {
-        // All the args are checked/rewritten. Don't call twice!
-        RewriteBlockPointerDecl(D);
-        break;
-      }
-  }
-}
-
-void RewriteModernObjC::CheckFunctionPointerDecl(QualType funcType, NamedDecl *ND) {
-  const PointerType *PT = funcType->getAs<PointerType>();
-  if (PT && PointerTypeTakesAnyBlockArguments(funcType))
-    RewriteBlocksInFunctionProtoType(PT->getPointeeType(), ND);
-}
-
-static bool IsHeaderFile(const std::string &Filename) {
-  std::string::size_type DotPos = Filename.rfind('.');
-
-  if (DotPos == std::string::npos) {
-    // no file extension
-    return false;
-  }
-
-  std::string Ext = std::string(Filename.begin()+DotPos+1, Filename.end());
-  // C header: .h
-  // C++ header: .hh or .H;
-  return Ext == "h" || Ext == "hh" || Ext == "H";
-}
-
-RewriteModernObjC::RewriteModernObjC(std::string inFile, raw_ostream* OS,
-                         DiagnosticsEngine &D, const LangOptions &LOpts,
-                         bool silenceMacroWarn)
-      : Diags(D), LangOpts(LOpts), InFileName(inFile), OutFile(OS),
-        SilenceRewriteMacroWarning(silenceMacroWarn) {
-  IsHeader = IsHeaderFile(inFile);
-  RewriteFailedDiag = Diags.getCustomDiagID(DiagnosticsEngine::Warning,
-               "rewriting sub-expression within a macro (may not be correct)");
-  // FIXME. This should be an error. But if block is not called, it is OK. And it
-  // may break including some headers.
-  GlobalBlockRewriteFailedDiag = Diags.getCustomDiagID(DiagnosticsEngine::Warning,
-    "rewriting block literal declared in global scope is not implemented");
-          
-  TryFinallyContainsReturnDiag = Diags.getCustomDiagID(
-               DiagnosticsEngine::Warning,
-               "rewriter doesn't support user-specified control flow semantics "
-               "for @try/@finally (code may not execute properly)");
-}
-
-ASTConsumer *clang::CreateModernObjCRewriter(const std::string& InFile,
-                                       raw_ostream* OS,
-                                       DiagnosticsEngine &Diags,
-                                       const LangOptions &LOpts,
-                                       bool SilenceRewriteMacroWarning) {
-    return new RewriteModernObjC(InFile, OS, Diags, LOpts, SilenceRewriteMacroWarning);
-}
-
-void RewriteModernObjC::InitializeCommon(ASTContext &context) {
-  Context = &context;
-  SM = &Context->getSourceManager();
-  TUDecl = Context->getTranslationUnitDecl();
-  MsgSendFunctionDecl = 0;
-  MsgSendSuperFunctionDecl = 0;
-  MsgSendStretFunctionDecl = 0;
-  MsgSendSuperStretFunctionDecl = 0;
-  MsgSendFpretFunctionDecl = 0;
-  GetClassFunctionDecl = 0;
-  GetMetaClassFunctionDecl = 0;
-  GetSuperClassFunctionDecl = 0;
-  SelGetUidFunctionDecl = 0;
-  CFStringFunctionDecl = 0;
-  ConstantStringClassReference = 0;
-  NSStringRecord = 0;
-  CurMethodDef = 0;
-  CurFunctionDef = 0;
-  GlobalVarDecl = 0;
-  GlobalConstructionExp = 0;
-  SuperStructDecl = 0;
-  ProtocolTypeDecl = 0;
-  ConstantStringDecl = 0;
-  BcLabelCount = 0;
-  SuperContructorFunctionDecl = 0;
-  NumObjCStringLiterals = 0;
-  PropParentMap = 0;
-  CurrentBody = 0;
-  DisableReplaceStmt = false;
-  objc_impl_method = false;
-
-  // Get the ID and start/end of the main file.
-  MainFileID = SM->getMainFileID();
-  const llvm::MemoryBuffer *MainBuf = SM->getBuffer(MainFileID);
-  MainFileStart = MainBuf->getBufferStart();
-  MainFileEnd = MainBuf->getBufferEnd();
-
-  Rewrite.setSourceMgr(Context->getSourceManager(), Context->getLangOpts());
-}
-
-//===----------------------------------------------------------------------===//
-// Top Level Driver Code
-//===----------------------------------------------------------------------===//
-
-void RewriteModernObjC::HandleTopLevelSingleDecl(Decl *D) {
-  if (Diags.hasErrorOccurred())
-    return;
-
-  // Two cases: either the decl could be in the main file, or it could be in a
-  // #included file.  If the former, rewrite it now.  If the later, check to see
-  // if we rewrote the #include/#import.
-  SourceLocation Loc = D->getLocation();
-  Loc = SM->getExpansionLoc(Loc);
-
-  // If this is for a builtin, ignore it.
-  if (Loc.isInvalid()) return;
-
-  // Look for built-in declarations that we need to refer during the rewrite.
-  if (FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
-    RewriteFunctionDecl(FD);
-  } else if (VarDecl *FVD = dyn_cast<VarDecl>(D)) {
-    // declared in <Foundation/NSString.h>
-    if (FVD->getName() == "_NSConstantStringClassReference") {
-      ConstantStringClassReference = FVD;
-      return;
-    }
-  } else if (ObjCCategoryDecl *CD = dyn_cast<ObjCCategoryDecl>(D)) {
-    RewriteCategoryDecl(CD);
-  } else if (ObjCProtocolDecl *PD = dyn_cast<ObjCProtocolDecl>(D)) {
-    if (PD->isThisDeclarationADefinition())
-      RewriteProtocolDecl(PD);
-  } else if (LinkageSpecDecl *LSD = dyn_cast<LinkageSpecDecl>(D)) {
-    // FIXME. This will not work in all situations and leaving it out
-    // is harmless.
-    // RewriteLinkageSpec(LSD);
-    
-    // Recurse into linkage specifications
-    for (DeclContext::decl_iterator DI = LSD->decls_begin(),
-                                 DIEnd = LSD->decls_end();
-         DI != DIEnd; ) {
-      if (ObjCInterfaceDecl *IFace = dyn_cast<ObjCInterfaceDecl>((*DI))) {
-        if (!IFace->isThisDeclarationADefinition()) {
-          SmallVector<Decl *, 8> DG;
-          SourceLocation StartLoc = IFace->getLocStart();
-          do {
-            if (isa<ObjCInterfaceDecl>(*DI) &&
-                !cast<ObjCInterfaceDecl>(*DI)->isThisDeclarationADefinition() &&
-                StartLoc == (*DI)->getLocStart())
-              DG.push_back(*DI);
-            else
-              break;
-            
-            ++DI;
-          } while (DI != DIEnd);
-          RewriteForwardClassDecl(DG);
-          continue;
-        }
-        else {
-          // Keep track of all interface declarations seen.
-          ObjCInterfacesSeen.push_back(IFace);
-          ++DI;
-          continue;
-        }
-      }
-
-      if (ObjCProtocolDecl *Proto = dyn_cast<ObjCProtocolDecl>((*DI))) {
-        if (!Proto->isThisDeclarationADefinition()) {
-          SmallVector<Decl *, 8> DG;
-          SourceLocation StartLoc = Proto->getLocStart();
-          do {
-            if (isa<ObjCProtocolDecl>(*DI) &&
-                !cast<ObjCProtocolDecl>(*DI)->isThisDeclarationADefinition() &&
-                StartLoc == (*DI)->getLocStart())
-              DG.push_back(*DI);
-            else
-              break;
-            
-            ++DI;
-          } while (DI != DIEnd);
-          RewriteForwardProtocolDecl(DG);
-          continue;
-        }
-      }
-      
-      HandleTopLevelSingleDecl(*DI);
-      ++DI;
-    }
-  }
-  // If we have a decl in the main file, see if we should rewrite it.
-  if (SM->isFromMainFile(Loc))
-    return HandleDeclInMainFile(D);
-}
-
-//===----------------------------------------------------------------------===//
-// Syntactic (non-AST) Rewriting Code
-//===----------------------------------------------------------------------===//
-
-void RewriteModernObjC::RewriteInclude() {
-  SourceLocation LocStart = SM->getLocForStartOfFile(MainFileID);
-  StringRef MainBuf = SM->getBufferData(MainFileID);
-  const char *MainBufStart = MainBuf.begin();
-  const char *MainBufEnd = MainBuf.end();
-  size_t ImportLen = strlen("import");
-
-  // Loop over the whole file, looking for includes.
-  for (const char *BufPtr = MainBufStart; BufPtr < MainBufEnd; ++BufPtr) {
-    if (*BufPtr == '#') {
-      if (++BufPtr == MainBufEnd)
-        return;
-      while (*BufPtr == ' ' || *BufPtr == '\t')
-        if (++BufPtr == MainBufEnd)
-          return;
-      if (!strncmp(BufPtr, "import", ImportLen)) {
-        // replace import with include
-        SourceLocation ImportLoc =
-          LocStart.getLocWithOffset(BufPtr-MainBufStart);
-        ReplaceText(ImportLoc, ImportLen, "include");
-        BufPtr += ImportLen;
-      }
-    }
-  }
-}
-
-static void WriteInternalIvarName(const ObjCInterfaceDecl *IDecl,
-                                  ObjCIvarDecl *IvarDecl, std::string &Result) {
-  Result += "OBJC_IVAR_$_";
-  Result += IDecl->getName();
-  Result += "$";
-  Result += IvarDecl->getName();
-}
-
-std::string 
-RewriteModernObjC::getIvarAccessString(ObjCIvarDecl *D) {
-  const ObjCInterfaceDecl *ClassDecl = D->getContainingInterface();
-  
-  // Build name of symbol holding ivar offset.
-  std::string IvarOffsetName;
-  WriteInternalIvarName(ClassDecl, D, IvarOffsetName);
-  
-  
-  std::string S = "(*(";
-  QualType IvarT = D->getType();
-  
-  if (!isa<TypedefType>(IvarT) && IvarT->isRecordType()) {
-    RecordDecl *RD = IvarT->getAs<RecordType>()->getDecl();
-    RD = RD->getDefinition();
-    if (RD && !RD->getDeclName().getAsIdentifierInfo()) {
-      // decltype(((Foo_IMPL*)0)->bar) *
-      ObjCContainerDecl *CDecl = 
-      dyn_cast<ObjCContainerDecl>(D->getDeclContext());
-      // ivar in class extensions requires special treatment.
-      if (ObjCCategoryDecl *CatDecl = dyn_cast<ObjCCategoryDecl>(CDecl))
-        CDecl = CatDecl->getClassInterface();
-      std::string RecName = CDecl->getName();
-      RecName += "_IMPL";
-      RecordDecl *RD = RecordDecl::Create(*Context, TTK_Struct, TUDecl,
-                                          SourceLocation(), SourceLocation(),
-                                          &Context->Idents.get(RecName.c_str()));
-      QualType PtrStructIMPL = Context->getPointerType(Context->getTagDeclType(RD));
-      unsigned UnsignedIntSize = 
-      static_cast<unsigned>(Context->getTypeSize(Context->UnsignedIntTy));
-      Expr *Zero = IntegerLiteral::Create(*Context,
-                                          llvm::APInt(UnsignedIntSize, 0),
-                                          Context->UnsignedIntTy, SourceLocation());
-      Zero = NoTypeInfoCStyleCastExpr(Context, PtrStructIMPL, CK_BitCast, Zero);
-      ParenExpr *PE = new (Context) ParenExpr(SourceLocation(), SourceLocation(),
-                                              Zero);
-      FieldDecl *FD = FieldDecl::Create(*Context, 0, SourceLocation(),
-                                        SourceLocation(),
-                                        &Context->Idents.get(D->getNameAsString()),
-                                        IvarT, 0,
-                                        /*BitWidth=*/0, /*Mutable=*/true,
-                                        ICIS_NoInit);
-      MemberExpr *ME = new (Context) MemberExpr(PE, true, FD, SourceLocation(),
-                                                FD->getType(), VK_LValue,
-                                                OK_Ordinary);
-      IvarT = Context->getDecltypeType(ME, ME->getType());
-    }
-  }
-  convertObjCTypeToCStyleType(IvarT);
-  QualType castT = Context->getPointerType(IvarT);
-  std::string TypeString(castT.getAsString(Context->getPrintingPolicy()));
-  S += TypeString;
-  S += ")";
-  
-  // ((char *)self + IVAR_OFFSET_SYMBOL_NAME)
-  S += "((char *)self + ";
-  S += IvarOffsetName;
-  S += "))";
-  ReferencedIvars[const_cast<ObjCInterfaceDecl *>(ClassDecl)].insert(D);
-  return S;
-}
-
-/// mustSynthesizeSetterGetterMethod - returns true if setter or getter has not
-/// been found in the class implementation. In this case, it must be synthesized.
-static bool mustSynthesizeSetterGetterMethod(ObjCImplementationDecl *IMP,
-                                             ObjCPropertyDecl *PD,
-                                             bool getter) {
-  return getter ? !IMP->getInstanceMethod(PD->getGetterName())
-                : !IMP->getInstanceMethod(PD->getSetterName());
-  
-}
-
-void RewriteModernObjC::RewritePropertyImplDecl(ObjCPropertyImplDecl *PID,
-                                          ObjCImplementationDecl *IMD,
-                                          ObjCCategoryImplDecl *CID) {
-  static bool objcGetPropertyDefined = false;
-  static bool objcSetPropertyDefined = false;
-  SourceLocation startGetterSetterLoc;
-  
-  if (PID->getLocStart().isValid()) {
-    SourceLocation startLoc = PID->getLocStart();
-    InsertText(startLoc, "// ");
-    const char *startBuf = SM->getCharacterData(startLoc);
-    assert((*startBuf == '@') && "bogus @synthesize location");
-    const char *semiBuf = strchr(startBuf, ';');
-    assert((*semiBuf == ';') && "@synthesize: can't find ';'");
-    startGetterSetterLoc = startLoc.getLocWithOffset(semiBuf-startBuf+1);
-  }
-  else
-    startGetterSetterLoc = IMD ? IMD->getLocEnd() : CID->getLocEnd();
-
-  if (PID->getPropertyImplementation() == ObjCPropertyImplDecl::Dynamic)
-    return; // FIXME: is this correct?
-
-  // Generate the 'getter' function.
-  ObjCPropertyDecl *PD = PID->getPropertyDecl();
-  ObjCIvarDecl *OID = PID->getPropertyIvarDecl();
-
-  if (!OID)
-    return;
-  unsigned Attributes = PD->getPropertyAttributes();
-  if (mustSynthesizeSetterGetterMethod(IMD, PD, true /*getter*/)) {
-    bool GenGetProperty = !(Attributes & ObjCPropertyDecl::OBJC_PR_nonatomic) &&
-                          (Attributes & (ObjCPropertyDecl::OBJC_PR_retain | 
-                                         ObjCPropertyDecl::OBJC_PR_copy));
-    std::string Getr;
-    if (GenGetProperty && !objcGetPropertyDefined) {
-      objcGetPropertyDefined = true;
-      // FIXME. Is this attribute correct in all cases?
-      Getr = "\nextern \"C\" __declspec(dllimport) "
-            "id objc_getProperty(id, SEL, long, bool);\n";
-    }
-    RewriteObjCMethodDecl(OID->getContainingInterface(),  
-                          PD->getGetterMethodDecl(), Getr);
-    Getr += "{ ";
-    // Synthesize an explicit cast to gain access to the ivar.
-    // See objc-act.c:objc_synthesize_new_getter() for details.
-    if (GenGetProperty) {
-      // return objc_getProperty(self, _cmd, offsetof(ClassDecl, OID), 1)
-      Getr += "typedef ";
-      const FunctionType *FPRetType = 0;
-      RewriteTypeIntoString(PD->getGetterMethodDecl()->getResultType(), Getr, 
-                            FPRetType);
-      Getr += " _TYPE";
-      if (FPRetType) {
-        Getr += ")"; // close the precedence "scope" for "*".
-      
-        // Now, emit the argument types (if any).
-        if (const FunctionProtoType *FT = dyn_cast<FunctionProtoType>(FPRetType)){
-          Getr += "(";
-          for (unsigned i = 0, e = FT->getNumArgs(); i != e; ++i) {
-            if (i) Getr += ", ";
-            std::string ParamStr = FT->getArgType(i).getAsString(
-                                                          Context->getPrintingPolicy());
-            Getr += ParamStr;
-          }
-          if (FT->isVariadic()) {
-            if (FT->getNumArgs()) Getr += ", ";
-            Getr += "...";
-          }
-          Getr += ")";
-        } else
-          Getr += "()";
-      }
-      Getr += ";\n";
-      Getr += "return (_TYPE)";
-      Getr += "objc_getProperty(self, _cmd, ";
-      RewriteIvarOffsetComputation(OID, Getr);
-      Getr += ", 1)";
-    }
-    else
-      Getr += "return " + getIvarAccessString(OID);
-    Getr += "; }";
-    InsertText(startGetterSetterLoc, Getr);
-  }
-  
-  if (PD->isReadOnly() || 
-      !mustSynthesizeSetterGetterMethod(IMD, PD, false /*setter*/))
-    return;
-
-  // Generate the 'setter' function.
-  std::string Setr;
-  bool GenSetProperty = Attributes & (ObjCPropertyDecl::OBJC_PR_retain | 
-                                      ObjCPropertyDecl::OBJC_PR_copy);
-  if (GenSetProperty && !objcSetPropertyDefined) {
-    objcSetPropertyDefined = true;
-    // FIXME. Is this attribute correct in all cases?
-    Setr = "\nextern \"C\" __declspec(dllimport) "
-    "void objc_setProperty (id, SEL, long, id, bool, bool);\n";
-  }
-  
-  RewriteObjCMethodDecl(OID->getContainingInterface(), 
-                        PD->getSetterMethodDecl(), Setr);
-  Setr += "{ ";
-  // Synthesize an explicit cast to initialize the ivar.
-  // See objc-act.c:objc_synthesize_new_setter() for details.
-  if (GenSetProperty) {
-    Setr += "objc_setProperty (self, _cmd, ";
-    RewriteIvarOffsetComputation(OID, Setr);
-    Setr += ", (id)";
-    Setr += PD->getName();
-    Setr += ", ";
-    if (Attributes & ObjCPropertyDecl::OBJC_PR_nonatomic)
-      Setr += "0, ";
-    else
-      Setr += "1, ";
-    if (Attributes & ObjCPropertyDecl::OBJC_PR_copy)
-      Setr += "1)";
-    else
-      Setr += "0)";
-  }
-  else {
-    Setr += getIvarAccessString(OID) + " = ";
-    Setr += PD->getName();
-  }
-  Setr += "; }\n";
-  InsertText(startGetterSetterLoc, Setr);
-}
-
-static void RewriteOneForwardClassDecl(ObjCInterfaceDecl *ForwardDecl,
-                                       std::string &typedefString) {
-  typedefString += "#ifndef _REWRITER_typedef_";
-  typedefString += ForwardDecl->getNameAsString();
-  typedefString += "\n";
-  typedefString += "#define _REWRITER_typedef_";
-  typedefString += ForwardDecl->getNameAsString();
-  typedefString += "\n";
-  typedefString += "typedef struct objc_object ";
-  typedefString += ForwardDecl->getNameAsString();
-  // typedef struct { } _objc_exc_Classname;
-  typedefString += ";\ntypedef struct {} _objc_exc_";
-  typedefString += ForwardDecl->getNameAsString();
-  typedefString += ";\n#endif\n";
-}
-
-void RewriteModernObjC::RewriteForwardClassEpilogue(ObjCInterfaceDecl *ClassDecl,
-                                              const std::string &typedefString) {
-    SourceLocation startLoc = ClassDecl->getLocStart();
-    const char *startBuf = SM->getCharacterData(startLoc);
-    const char *semiPtr = strchr(startBuf, ';'); 
-    // Replace the @class with typedefs corresponding to the classes.
-    ReplaceText(startLoc, semiPtr-startBuf+1, typedefString);  
-}
-
-void RewriteModernObjC::RewriteForwardClassDecl(DeclGroupRef D) {
-  std::string typedefString;
-  for (DeclGroupRef::iterator I = D.begin(), E = D.end(); I != E; ++I) {
-    ObjCInterfaceDecl *ForwardDecl = cast<ObjCInterfaceDecl>(*I);
-    if (I == D.begin()) {
-      // Translate to typedef's that forward reference structs with the same name
-      // as the class. As a convenience, we include the original declaration
-      // as a comment.
-      typedefString += "// @class ";
-      typedefString += ForwardDecl->getNameAsString();
-      typedefString += ";\n";
-    }
-    RewriteOneForwardClassDecl(ForwardDecl, typedefString);
-  }
-  DeclGroupRef::iterator I = D.begin();
-  RewriteForwardClassEpilogue(cast<ObjCInterfaceDecl>(*I), typedefString);
-}
-
-void RewriteModernObjC::RewriteForwardClassDecl(
-                                const llvm::SmallVector<Decl*, 8> &D) {
-  std::string typedefString;
-  for (unsigned i = 0; i < D.size(); i++) {
-    ObjCInterfaceDecl *ForwardDecl = cast<ObjCInterfaceDecl>(D[i]);
-    if (i == 0) {
-      typedefString += "// @class ";
-      typedefString += ForwardDecl->getNameAsString();
-      typedefString += ";\n";
-    }
-    RewriteOneForwardClassDecl(ForwardDecl, typedefString);
-  }
-  RewriteForwardClassEpilogue(cast<ObjCInterfaceDecl>(D[0]), typedefString);
-}
-
-void RewriteModernObjC::RewriteMethodDeclaration(ObjCMethodDecl *Method) {
-  // When method is a synthesized one, such as a getter/setter there is
-  // nothing to rewrite.
-  if (Method->isImplicit())
-    return;
-  SourceLocation LocStart = Method->getLocStart();
-  SourceLocation LocEnd = Method->getLocEnd();
-
-  if (SM->getExpansionLineNumber(LocEnd) >
-      SM->getExpansionLineNumber(LocStart)) {
-    InsertText(LocStart, "#if 0\n");
-    ReplaceText(LocEnd, 1, ";\n#endif\n");
-  } else {
-    InsertText(LocStart, "// ");
-  }
-}
-
-void RewriteModernObjC::RewriteProperty(ObjCPropertyDecl *prop) {
-  SourceLocation Loc = prop->getAtLoc();
-
-  ReplaceText(Loc, 0, "// ");
-  // FIXME: handle properties that are declared across multiple lines.
-}
-
-void RewriteModernObjC::RewriteCategoryDecl(ObjCCategoryDecl *CatDecl) {
-  SourceLocation LocStart = CatDecl->getLocStart();
-
-  // FIXME: handle category headers that are declared across multiple lines.
-  if (CatDecl->getIvarRBraceLoc().isValid()) {
-    ReplaceText(LocStart, 1, "/** ");
-    ReplaceText(CatDecl->getIvarRBraceLoc(), 1, "**/ ");
-  }
-  else {
-    ReplaceText(LocStart, 0, "// ");
-  }
-  
-  for (ObjCCategoryDecl::prop_iterator I = CatDecl->prop_begin(),
-       E = CatDecl->prop_end(); I != E; ++I)
-    RewriteProperty(*I);
-  
-  for (ObjCCategoryDecl::instmeth_iterator
-         I = CatDecl->instmeth_begin(), E = CatDecl->instmeth_end();
-       I != E; ++I)
-    RewriteMethodDeclaration(*I);
-  for (ObjCCategoryDecl::classmeth_iterator
-         I = CatDecl->classmeth_begin(), E = CatDecl->classmeth_end();
-       I != E; ++I)
-    RewriteMethodDeclaration(*I);
-
-  // Lastly, comment out the @end.
-  ReplaceText(CatDecl->getAtEndRange().getBegin(), 
-              strlen("@end"), "/* @end */");
-}
-
-void RewriteModernObjC::RewriteProtocolDecl(ObjCProtocolDecl *PDecl) {
-  SourceLocation LocStart = PDecl->getLocStart();
-  assert(PDecl->isThisDeclarationADefinition());
-  
-  // FIXME: handle protocol headers that are declared across multiple lines.
-  ReplaceText(LocStart, 0, "// ");
-
-  for (ObjCProtocolDecl::instmeth_iterator
-         I = PDecl->instmeth_begin(), E = PDecl->instmeth_end();
-       I != E; ++I)
-    RewriteMethodDeclaration(*I);
-  for (ObjCProtocolDecl::classmeth_iterator
-         I = PDecl->classmeth_begin(), E = PDecl->classmeth_end();
-       I != E; ++I)
-    RewriteMethodDeclaration(*I);
-
-  for (ObjCInterfaceDecl::prop_iterator I = PDecl->prop_begin(),
-       E = PDecl->prop_end(); I != E; ++I)
-    RewriteProperty(*I);
-  
-  // Lastly, comment out the @end.
-  SourceLocation LocEnd = PDecl->getAtEndRange().getBegin();
-  ReplaceText(LocEnd, strlen("@end"), "/* @end */");
-
-  // Must comment out @optional/@required
-  const char *startBuf = SM->getCharacterData(LocStart);
-  const char *endBuf = SM->getCharacterData(LocEnd);
-  for (const char *p = startBuf; p < endBuf; p++) {
-    if (*p == '@' && !strncmp(p+1, "optional", strlen("optional"))) {
-      SourceLocation OptionalLoc = LocStart.getLocWithOffset(p-startBuf);
-      ReplaceText(OptionalLoc, strlen("@optional"), "/* @optional */");
-
-    }
-    else if (*p == '@' && !strncmp(p+1, "required", strlen("required"))) {
-      SourceLocation OptionalLoc = LocStart.getLocWithOffset(p-startBuf);
-      ReplaceText(OptionalLoc, strlen("@required"), "/* @required */");
-
-    }
-  }
-}
-
-void RewriteModernObjC::RewriteForwardProtocolDecl(DeclGroupRef D) {
-  SourceLocation LocStart = (*D.begin())->getLocStart();
-  if (LocStart.isInvalid())
-    llvm_unreachable("Invalid SourceLocation");
-  // FIXME: handle forward protocol that are declared across multiple lines.
-  ReplaceText(LocStart, 0, "// ");
-}
-
-void 
-RewriteModernObjC::RewriteForwardProtocolDecl(const llvm::SmallVector<Decl*, 8> &DG) {
-  SourceLocation LocStart = DG[0]->getLocStart();
-  if (LocStart.isInvalid())
-    llvm_unreachable("Invalid SourceLocation");
-  // FIXME: handle forward protocol that are declared across multiple lines.
-  ReplaceText(LocStart, 0, "// ");
-}
-
-void 
-RewriteModernObjC::RewriteLinkageSpec(LinkageSpecDecl *LSD) {
-  SourceLocation LocStart = LSD->getExternLoc();
-  if (LocStart.isInvalid())
-    llvm_unreachable("Invalid extern SourceLocation");
-  
-  ReplaceText(LocStart, 0, "// ");
-  if (!LSD->hasBraces())
-    return;
-  // FIXME. We don't rewrite well if '{' is not on same line as 'extern'.
-  SourceLocation LocRBrace = LSD->getRBraceLoc();
-  if (LocRBrace.isInvalid())
-    llvm_unreachable("Invalid rbrace SourceLocation");
-  ReplaceText(LocRBrace, 0, "// ");
-}
-
-void RewriteModernObjC::RewriteTypeIntoString(QualType T, std::string &ResultStr,
-                                        const FunctionType *&FPRetType) {
-  if (T->isObjCQualifiedIdType())
-    ResultStr += "id";
-  else if (T->isFunctionPointerType() ||
-           T->isBlockPointerType()) {
-    // needs special handling, since pointer-to-functions have special
-    // syntax (where a decaration models use).
-    QualType retType = T;
-    QualType PointeeTy;
-    if (const PointerType* PT = retType->getAs<PointerType>())
-      PointeeTy = PT->getPointeeType();
-    else if (const BlockPointerType *BPT = retType->getAs<BlockPointerType>())
-      PointeeTy = BPT->getPointeeType();
-    if ((FPRetType = PointeeTy->getAs<FunctionType>())) {
-      ResultStr += FPRetType->getResultType().getAsString(
-        Context->getPrintingPolicy());
-      ResultStr += "(*";
-    }
-  } else
-    ResultStr += T.getAsString(Context->getPrintingPolicy());
-}
-
-void RewriteModernObjC::RewriteObjCMethodDecl(const ObjCInterfaceDecl *IDecl,
-                                        ObjCMethodDecl *OMD,
-                                        std::string &ResultStr) {
-  //fprintf(stderr,"In RewriteObjCMethodDecl\n");
-  const FunctionType *FPRetType = 0;
-  ResultStr += "\nstatic ";
-  RewriteTypeIntoString(OMD->getResultType(), ResultStr, FPRetType);
-  ResultStr += " ";
-
-  // Unique method name
-  std::string NameStr;
-
-  if (OMD->isInstanceMethod())
-    NameStr += "_I_";
-  else
-    NameStr += "_C_";
-
-  NameStr += IDecl->getNameAsString();
-  NameStr += "_";
-
-  if (ObjCCategoryImplDecl *CID =
-      dyn_cast<ObjCCategoryImplDecl>(OMD->getDeclContext())) {
-    NameStr += CID->getNameAsString();
-    NameStr += "_";
-  }
-  // Append selector names, replacing ':' with '_'
-  {
-    std::string selString = OMD->getSelector().getAsString();
-    int len = selString.size();
-    for (int i = 0; i < len; i++)
-      if (selString[i] == ':')
-        selString[i] = '_';
-    NameStr += selString;
-  }
-  // Remember this name for metadata emission
-  MethodInternalNames[OMD] = NameStr;
-  ResultStr += NameStr;
-
-  // Rewrite arguments
-  ResultStr += "(";
-
-  // invisible arguments
-  if (OMD->isInstanceMethod()) {
-    QualType selfTy = Context->getObjCInterfaceType(IDecl);
-    selfTy = Context->getPointerType(selfTy);
-    if (!LangOpts.MicrosoftExt) {
-      if (ObjCSynthesizedStructs.count(const_cast<ObjCInterfaceDecl*>(IDecl)))
-        ResultStr += "struct ";
-    }
-    // When rewriting for Microsoft, explicitly omit the structure name.
-    ResultStr += IDecl->getNameAsString();
-    ResultStr += " *";
-  }
-  else
-    ResultStr += Context->getObjCClassType().getAsString(
-      Context->getPrintingPolicy());
-
-  ResultStr += " self, ";
-  ResultStr += Context->getObjCSelType().getAsString(Context->getPrintingPolicy());
-  ResultStr += " _cmd";
-
-  // Method arguments.
-  for (ObjCMethodDecl::param_iterator PI = OMD->param_begin(),
-       E = OMD->param_end(); PI != E; ++PI) {
-    ParmVarDecl *PDecl = *PI;
-    ResultStr += ", ";
-    if (PDecl->getType()->isObjCQualifiedIdType()) {
-      ResultStr += "id ";
-      ResultStr += PDecl->getNameAsString();
-    } else {
-      std::string Name = PDecl->getNameAsString();
-      QualType QT = PDecl->getType();
-      // Make sure we convert "t (^)(...)" to "t (*)(...)".
-      (void)convertBlockPointerToFunctionPointer(QT);
-      QT.getAsStringInternal(Name, Context->getPrintingPolicy());
-      ResultStr += Name;
-    }
-  }
-  if (OMD->isVariadic())
-    ResultStr += ", ...";
-  ResultStr += ") ";
-
-  if (FPRetType) {
-    ResultStr += ")"; // close the precedence "scope" for "*".
-
-    // Now, emit the argument types (if any).
-    if (const FunctionProtoType *FT = dyn_cast<FunctionProtoType>(FPRetType)) {
-      ResultStr += "(";
-      for (unsigned i = 0, e = FT->getNumArgs(); i != e; ++i) {
-        if (i) ResultStr += ", ";
-        std::string ParamStr = FT->getArgType(i).getAsString(
-          Context->getPrintingPolicy());
-        ResultStr += ParamStr;
-      }
-      if (FT->isVariadic()) {
-        if (FT->getNumArgs()) ResultStr += ", ";
-        ResultStr += "...";
-      }
-      ResultStr += ")";
-    } else {
-      ResultStr += "()";
-    }
-  }
-}
-void RewriteModernObjC::RewriteImplementationDecl(Decl *OID) {
-  ObjCImplementationDecl *IMD = dyn_cast<ObjCImplementationDecl>(OID);
-  ObjCCategoryImplDecl *CID = dyn_cast<ObjCCategoryImplDecl>(OID);
-
-  if (IMD) {
-    if (IMD->getIvarRBraceLoc().isValid()) {
-      ReplaceText(IMD->getLocStart(), 1, "/** ");
-      ReplaceText(IMD->getIvarRBraceLoc(), 1, "**/ ");
-    }
-    else {
-      InsertText(IMD->getLocStart(), "// ");
-    }
-  }
-  else
-    InsertText(CID->getLocStart(), "// ");
-
-  for (ObjCCategoryImplDecl::instmeth_iterator
-       I = IMD ? IMD->instmeth_begin() : CID->instmeth_begin(),
-       E = IMD ? IMD->instmeth_end() : CID->instmeth_end();
-       I != E; ++I) {
-    std::string ResultStr;
-    ObjCMethodDecl *OMD = *I;
-    RewriteObjCMethodDecl(OMD->getClassInterface(), OMD, ResultStr);
-    SourceLocation LocStart = OMD->getLocStart();
-    SourceLocation LocEnd = OMD->getCompoundBody()->getLocStart();
-
-    const char *startBuf = SM->getCharacterData(LocStart);
-    const char *endBuf = SM->getCharacterData(LocEnd);
-    ReplaceText(LocStart, endBuf-startBuf, ResultStr);
-  }
-
-  for (ObjCCategoryImplDecl::classmeth_iterator
-       I = IMD ? IMD->classmeth_begin() : CID->classmeth_begin(),
-       E = IMD ? IMD->classmeth_end() : CID->classmeth_end();
-       I != E; ++I) {
-    std::string ResultStr;
-    ObjCMethodDecl *OMD = *I;
-    RewriteObjCMethodDecl(OMD->getClassInterface(), OMD, ResultStr);
-    SourceLocation LocStart = OMD->getLocStart();
-    SourceLocation LocEnd = OMD->getCompoundBody()->getLocStart();
-
-    const char *startBuf = SM->getCharacterData(LocStart);
-    const char *endBuf = SM->getCharacterData(LocEnd);
-    ReplaceText(LocStart, endBuf-startBuf, ResultStr);
-  }
-  for (ObjCCategoryImplDecl::propimpl_iterator
-       I = IMD ? IMD->propimpl_begin() : CID->propimpl_begin(),
-       E = IMD ? IMD->propimpl_end() : CID->propimpl_end();
-       I != E; ++I) {
-    RewritePropertyImplDecl(*I, IMD, CID);
-  }
-
-  InsertText(IMD ? IMD->getLocEnd() : CID->getLocEnd(), "// ");
-}
-
-void RewriteModernObjC::RewriteInterfaceDecl(ObjCInterfaceDecl *ClassDecl) {
-  // Do not synthesize more than once.
-  if (ObjCSynthesizedStructs.count(ClassDecl))
-    return;
-  // Make sure super class's are written before current class is written.
-  ObjCInterfaceDecl *SuperClass = ClassDecl->getSuperClass();
-  while (SuperClass) {
-    RewriteInterfaceDecl(SuperClass);
-    SuperClass = SuperClass->getSuperClass();
-  }
-  std::string ResultStr;
-  if (!ObjCWrittenInterfaces.count(ClassDecl->getCanonicalDecl())) {
-    // we haven't seen a forward decl - generate a typedef.
-    RewriteOneForwardClassDecl(ClassDecl, ResultStr);
-    RewriteIvarOffsetSymbols(ClassDecl, ResultStr);
-    
-    RewriteObjCInternalStruct(ClassDecl, ResultStr);
-    // Mark this typedef as having been written into its c++ equivalent.
-    ObjCWrittenInterfaces.insert(ClassDecl->getCanonicalDecl());
-  
-    for (ObjCInterfaceDecl::prop_iterator I = ClassDecl->prop_begin(),
-         E = ClassDecl->prop_end(); I != E; ++I)
-      RewriteProperty(*I);
-    for (ObjCInterfaceDecl::instmeth_iterator
-         I = ClassDecl->instmeth_begin(), E = ClassDecl->instmeth_end();
-         I != E; ++I)
-      RewriteMethodDeclaration(*I);
-    for (ObjCInterfaceDecl::classmeth_iterator
-         I = ClassDecl->classmeth_begin(), E = ClassDecl->classmeth_end();
-         I != E; ++I)
-      RewriteMethodDeclaration(*I);
-
-    // Lastly, comment out the @end.
-    ReplaceText(ClassDecl->getAtEndRange().getBegin(), strlen("@end"), 
-                "/* @end */");
-  }
-}
-
-Stmt *RewriteModernObjC::RewritePropertyOrImplicitSetter(PseudoObjectExpr *PseudoOp) {
-  SourceRange OldRange = PseudoOp->getSourceRange();
-
-  // We just magically know some things about the structure of this
-  // expression.
-  ObjCMessageExpr *OldMsg =
-    cast<ObjCMessageExpr>(PseudoOp->getSemanticExpr(
-                            PseudoOp->getNumSemanticExprs() - 1));
-
-  // Because the rewriter doesn't allow us to rewrite rewritten code,
-  // we need to suppress rewriting the sub-statements.
-  Expr *Base;
-  SmallVector<Expr*, 2> Args;
-  {
-    DisableReplaceStmtScope S(*this);
-
-    // Rebuild the base expression if we have one.
-    Base = 0;
-    if (OldMsg->getReceiverKind() == ObjCMessageExpr::Instance) {
-      Base = OldMsg->getInstanceReceiver();
-      Base = cast<OpaqueValueExpr>(Base)->getSourceExpr();
-      Base = cast<Expr>(RewriteFunctionBodyOrGlobalInitializer(Base));
-    }
-  
-    unsigned numArgs = OldMsg->getNumArgs();
-    for (unsigned i = 0; i < numArgs; i++) {
-      Expr *Arg = OldMsg->getArg(i);
-      if (isa<OpaqueValueExpr>(Arg))
-        Arg = cast<OpaqueValueExpr>(Arg)->getSourceExpr();
-      Arg = cast<Expr>(RewriteFunctionBodyOrGlobalInitializer(Arg));
-      Args.push_back(Arg);
-    }
-  }
-
-  // TODO: avoid this copy.
-  SmallVector<SourceLocation, 1> SelLocs;
-  OldMsg->getSelectorLocs(SelLocs);
-
-  ObjCMessageExpr *NewMsg = 0;
-  switch (OldMsg->getReceiverKind()) {
-  case ObjCMessageExpr::Class:
-    NewMsg = ObjCMessageExpr::Create(*Context, OldMsg->getType(),
-                                     OldMsg->getValueKind(),
-                                     OldMsg->getLeftLoc(),
-                                     OldMsg->getClassReceiverTypeInfo(),
-                                     OldMsg->getSelector(),
-                                     SelLocs,
-                                     OldMsg->getMethodDecl(),
-                                     Args,
-                                     OldMsg->getRightLoc(),
-                                     OldMsg->isImplicit());
-    break;
-
-  case ObjCMessageExpr::Instance:
-    NewMsg = ObjCMessageExpr::Create(*Context, OldMsg->getType(),
-                                     OldMsg->getValueKind(),
-                                     OldMsg->getLeftLoc(),
-                                     Base,
-                                     OldMsg->getSelector(),
-                                     SelLocs,
-                                     OldMsg->getMethodDecl(),
-                                     Args,
-                                     OldMsg->getRightLoc(),
-                                     OldMsg->isImplicit());
-    break;
-
-  case ObjCMessageExpr::SuperClass:
-  case ObjCMessageExpr::SuperInstance:
-    NewMsg = ObjCMessageExpr::Create(*Context, OldMsg->getType(),
-                                     OldMsg->getValueKind(),
-                                     OldMsg->getLeftLoc(),
-                                     OldMsg->getSuperLoc(),
-                 OldMsg->getReceiverKind() == ObjCMessageExpr::SuperInstance,
-                                     OldMsg->getSuperType(),
-                                     OldMsg->getSelector(),
-                                     SelLocs,
-                                     OldMsg->getMethodDecl(),
-                                     Args,
-                                     OldMsg->getRightLoc(),
-                                     OldMsg->isImplicit());
-    break;
-  }
-
-  Stmt *Replacement = SynthMessageExpr(NewMsg);
-  ReplaceStmtWithRange(PseudoOp, Replacement, OldRange);
-  return Replacement;
-}
-
-Stmt *RewriteModernObjC::RewritePropertyOrImplicitGetter(PseudoObjectExpr *PseudoOp) {
-  SourceRange OldRange = PseudoOp->getSourceRange();
-
-  // We just magically know some things about the structure of this
-  // expression.
-  ObjCMessageExpr *OldMsg =
-    cast<ObjCMessageExpr>(PseudoOp->getResultExpr()->IgnoreImplicit());
-
-  // Because the rewriter doesn't allow us to rewrite rewritten code,
-  // we need to suppress rewriting the sub-statements.
-  Expr *Base = 0;
-  SmallVector<Expr*, 1> Args;
-  {
-    DisableReplaceStmtScope S(*this);
-    // Rebuild the base expression if we have one.
-    if (OldMsg->getReceiverKind() == ObjCMessageExpr::Instance) {
-      Base = OldMsg->getInstanceReceiver();
-      Base = cast<OpaqueValueExpr>(Base)->getSourceExpr();
-      Base = cast<Expr>(RewriteFunctionBodyOrGlobalInitializer(Base));
-    }
-    unsigned numArgs = OldMsg->getNumArgs();
-    for (unsigned i = 0; i < numArgs; i++) {
-      Expr *Arg = OldMsg->getArg(i);
-      if (isa<OpaqueValueExpr>(Arg))
-        Arg = cast<OpaqueValueExpr>(Arg)->getSourceExpr();
-      Arg = cast<Expr>(RewriteFunctionBodyOrGlobalInitializer(Arg));
-      Args.push_back(Arg);
-    }
-  }
-
-  // Intentionally empty.
-  SmallVector<SourceLocation, 1> SelLocs;
-
-  ObjCMessageExpr *NewMsg = 0;
-  switch (OldMsg->getReceiverKind()) {
-  case ObjCMessageExpr::Class:
-    NewMsg = ObjCMessageExpr::Create(*Context, OldMsg->getType(),
-                                     OldMsg->getValueKind(),
-                                     OldMsg->getLeftLoc(),
-                                     OldMsg->getClassReceiverTypeInfo(),
-                                     OldMsg->getSelector(),
-                                     SelLocs,
-                                     OldMsg->getMethodDecl(),
-                                     Args,
-                                     OldMsg->getRightLoc(),
-                                     OldMsg->isImplicit());
-    break;
-
-  case ObjCMessageExpr::Instance:
-    NewMsg = ObjCMessageExpr::Create(*Context, OldMsg->getType(),
-                                     OldMsg->getValueKind(),
-                                     OldMsg->getLeftLoc(),
-                                     Base,
-                                     OldMsg->getSelector(),
-                                     SelLocs,
-                                     OldMsg->getMethodDecl(),
-                                     Args,
-                                     OldMsg->getRightLoc(),
-                                     OldMsg->isImplicit());
-    break;
-
-  case ObjCMessageExpr::SuperClass:
-  case ObjCMessageExpr::SuperInstance:
-    NewMsg = ObjCMessageExpr::Create(*Context, OldMsg->getType(),
-                                     OldMsg->getValueKind(),
-                                     OldMsg->getLeftLoc(),
-                                     OldMsg->getSuperLoc(),
-                 OldMsg->getReceiverKind() == ObjCMessageExpr::SuperInstance,
-                                     OldMsg->getSuperType(),
-                                     OldMsg->getSelector(),
-                                     SelLocs,
-                                     OldMsg->getMethodDecl(),
-                                     Args,
-                                     OldMsg->getRightLoc(),
-                                     OldMsg->isImplicit());
-    break;
-  }
-
-  Stmt *Replacement = SynthMessageExpr(NewMsg);
-  ReplaceStmtWithRange(PseudoOp, Replacement, OldRange);
-  return Replacement;
-}
-
-/// SynthCountByEnumWithState - To print:
-/// ((unsigned int (*)
-///  (id, SEL, struct __objcFastEnumerationState *, id *, unsigned int))
-///  (void *)objc_msgSend)((id)l_collection,
-///                        sel_registerName(
-///                          "countByEnumeratingWithState:objects:count:"),
-///                        &enumState,
-///                        (id *)__rw_items, (unsigned int)16)
-///
-void RewriteModernObjC::SynthCountByEnumWithState(std::string &buf) {
-  buf += "((unsigned int (*) (id, SEL, struct __objcFastEnumerationState *, "
-  "id *, unsigned int))(void *)objc_msgSend)";
-  buf += "\n\t\t";
-  buf += "((id)l_collection,\n\t\t";
-  buf += "sel_registerName(\"countByEnumeratingWithState:objects:count:\"),";
-  buf += "\n\t\t";
-  buf += "&enumState, "
-         "(id *)__rw_items, (unsigned int)16)";
-}
-
-/// RewriteBreakStmt - Rewrite for a break-stmt inside an ObjC2's foreach
-/// statement to exit to its outer synthesized loop.
-///
-Stmt *RewriteModernObjC::RewriteBreakStmt(BreakStmt *S) {
-  if (Stmts.empty() || !isa<ObjCForCollectionStmt>(Stmts.back()))
-    return S;
-  // replace break with goto __break_label
-  std::string buf;
-
-  SourceLocation startLoc = S->getLocStart();
-  buf = "goto __break_label_";
-  buf += utostr(ObjCBcLabelNo.back());
-  ReplaceText(startLoc, strlen("break"), buf);
-
-  return 0;
-}
-
-/// RewriteContinueStmt - Rewrite for a continue-stmt inside an ObjC2's foreach
-/// statement to continue with its inner synthesized loop.
-///
-Stmt *RewriteModernObjC::RewriteContinueStmt(ContinueStmt *S) {
-  if (Stmts.empty() || !isa<ObjCForCollectionStmt>(Stmts.back()))
-    return S;
-  // replace continue with goto __continue_label
-  std::string buf;
-
-  SourceLocation startLoc = S->getLocStart();
-  buf = "goto __continue_label_";
-  buf += utostr(ObjCBcLabelNo.back());
-  ReplaceText(startLoc, strlen("continue"), buf);
-
-  return 0;
-}
-
-/// RewriteObjCForCollectionStmt - Rewriter for ObjC2's foreach statement.
-///  It rewrites:
-/// for ( type elem in collection) { stmts; }
-
-/// Into:
-/// {
-///   type elem;
-///   struct __objcFastEnumerationState enumState = { 0 };
-///   id __rw_items[16];
-///   id l_collection = (id)collection;
-///   unsigned long limit = [l_collection countByEnumeratingWithState:&enumState
-///                                       objects:__rw_items count:16];
-/// if (limit) {
-///   unsigned long startMutations = *enumState.mutationsPtr;
-///   do {
-///        unsigned long counter = 0;
-///        do {
-///             if (startMutations != *enumState.mutationsPtr)
-///               objc_enumerationMutation(l_collection);
-///             elem = (type)enumState.itemsPtr[counter++];
-///             stmts;
-///             __continue_label: ;
-///        } while (counter < limit);
-///   } while (limit = [l_collection countByEnumeratingWithState:&enumState
-///                                  objects:__rw_items count:16]);
-///   elem = nil;
-///   __break_label: ;
-///  }
-///  else
-///       elem = nil;
-///  }
-///
-Stmt *RewriteModernObjC::RewriteObjCForCollectionStmt(ObjCForCollectionStmt *S,
-                                                SourceLocation OrigEnd) {
-  assert(!Stmts.empty() && "ObjCForCollectionStmt - Statement stack empty");
-  assert(isa<ObjCForCollectionStmt>(Stmts.back()) &&
-         "ObjCForCollectionStmt Statement stack mismatch");
-  assert(!ObjCBcLabelNo.empty() &&
-         "ObjCForCollectionStmt - Label No stack empty");
-
-  SourceLocation startLoc = S->getLocStart();
-  const char *startBuf = SM->getCharacterData(startLoc);
-  StringRef elementName;
-  std::string elementTypeAsString;
-  std::string buf;
-  buf = "\n{\n\t";
-  if (DeclStmt *DS = dyn_cast<DeclStmt>(S->getElement())) {
-    // type elem;
-    NamedDecl* D = cast<NamedDecl>(DS->getSingleDecl());
-    QualType ElementType = cast<ValueDecl>(D)->getType();
-    if (ElementType->isObjCQualifiedIdType() ||
-        ElementType->isObjCQualifiedInterfaceType())
-      // Simply use 'id' for all qualified types.
-      elementTypeAsString = "id";
-    else
-      elementTypeAsString = ElementType.getAsString(Context->getPrintingPolicy());
-    buf += elementTypeAsString;
-    buf += " ";
-    elementName = D->getName();
-    buf += elementName;
-    buf += ";\n\t";
-  }
-  else {
-    DeclRefExpr *DR = cast<DeclRefExpr>(S->getElement());
-    elementName = DR->getDecl()->getName();
-    ValueDecl *VD = cast<ValueDecl>(DR->getDecl());
-    if (VD->getType()->isObjCQualifiedIdType() ||
-        VD->getType()->isObjCQualifiedInterfaceType())
-      // Simply use 'id' for all qualified types.
-      elementTypeAsString = "id";
-    else
-      elementTypeAsString = VD->getType().getAsString(Context->getPrintingPolicy());
-  }
-
-  // struct __objcFastEnumerationState enumState = { 0 };
-  buf += "struct __objcFastEnumerationState enumState = { 0 };\n\t";
-  // id __rw_items[16];
-  buf += "id __rw_items[16];\n\t";
-  // id l_collection = (id)
-  buf += "id l_collection = (id)";
-  // Find start location of 'collection' the hard way!
-  const char *startCollectionBuf = startBuf;
-  startCollectionBuf += 3;  // skip 'for'
-  startCollectionBuf = strchr(startCollectionBuf, '(');
-  startCollectionBuf++; // skip '('
-  // find 'in' and skip it.
-  while (*startCollectionBuf != ' ' ||
-         *(startCollectionBuf+1) != 'i' || *(startCollectionBuf+2) != 'n' ||
-         (*(startCollectionBuf+3) != ' ' &&
-          *(startCollectionBuf+3) != '[' && *(startCollectionBuf+3) != '('))
-    startCollectionBuf++;
-  startCollectionBuf += 3;
-
-  // Replace: "for (type element in" with string constructed thus far.
-  ReplaceText(startLoc, startCollectionBuf - startBuf, buf);
-  // Replace ')' in for '(' type elem in collection ')' with ';'
-  SourceLocation rightParenLoc = S->getRParenLoc();
-  const char *rparenBuf = SM->getCharacterData(rightParenLoc);
-  SourceLocation lparenLoc = startLoc.getLocWithOffset(rparenBuf-startBuf);
-  buf = ";\n\t";
-
-  // unsigned long limit = [l_collection countByEnumeratingWithState:&enumState
-  //                                   objects:__rw_items count:16];
-  // which is synthesized into:
-  // unsigned int limit =
-  // ((unsigned int (*)
-  //  (id, SEL, struct __objcFastEnumerationState *, id *, unsigned int))
-  //  (void *)objc_msgSend)((id)l_collection,
-  //                        sel_registerName(
-  //                          "countByEnumeratingWithState:objects:count:"),
-  //                        (struct __objcFastEnumerationState *)&state,
-  //                        (id *)__rw_items, (unsigned int)16);
-  buf += "unsigned long limit =\n\t\t";
-  SynthCountByEnumWithState(buf);
-  buf += ";\n\t";
-  /// if (limit) {
-  ///   unsigned long startMutations = *enumState.mutationsPtr;
-  ///   do {
-  ///        unsigned long counter = 0;
-  ///        do {
-  ///             if (startMutations != *enumState.mutationsPtr)
-  ///               objc_enumerationMutation(l_collection);
-  ///             elem = (type)enumState.itemsPtr[counter++];
-  buf += "if (limit) {\n\t";
-  buf += "unsigned long startMutations = *enumState.mutationsPtr;\n\t";
-  buf += "do {\n\t\t";
-  buf += "unsigned long counter = 0;\n\t\t";
-  buf += "do {\n\t\t\t";
-  buf += "if (startMutations != *enumState.mutationsPtr)\n\t\t\t\t";
-  buf += "objc_enumerationMutation(l_collection);\n\t\t\t";
-  buf += elementName;
-  buf += " = (";
-  buf += elementTypeAsString;
-  buf += ")enumState.itemsPtr[counter++];";
-  // Replace ')' in for '(' type elem in collection ')' with all of these.
-  ReplaceText(lparenLoc, 1, buf);
-
-  ///            __continue_label: ;
-  ///        } while (counter < limit);
-  ///   } while (limit = [l_collection countByEnumeratingWithState:&enumState
-  ///                                  objects:__rw_items count:16]);
-  ///   elem = nil;
-  ///   __break_label: ;
-  ///  }
-  ///  else
-  ///       elem = nil;
-  ///  }
-  ///
-  buf = ";\n\t";
-  buf += "__continue_label_";
-  buf += utostr(ObjCBcLabelNo.back());
-  buf += ": ;";
-  buf += "\n\t\t";
-  buf += "} while (counter < limit);\n\t";
-  buf += "} while (limit = ";
-  SynthCountByEnumWithState(buf);
-  buf += ");\n\t";
-  buf += elementName;
-  buf += " = ((";
-  buf += elementTypeAsString;
-  buf += ")0);\n\t";
-  buf += "__break_label_";
-  buf += utostr(ObjCBcLabelNo.back());
-  buf += ": ;\n\t";
-  buf += "}\n\t";
-  buf += "else\n\t\t";
-  buf += elementName;
-  buf += " = ((";
-  buf += elementTypeAsString;
-  buf += ")0);\n\t";
-  buf += "}\n";
-
-  // Insert all these *after* the statement body.
-  // FIXME: If this should support Obj-C++, support CXXTryStmt
-  if (isa<CompoundStmt>(S->getBody())) {
-    SourceLocation endBodyLoc = OrigEnd.getLocWithOffset(1);
-    InsertText(endBodyLoc, buf);
-  } else {
-    /* Need to treat single statements specially. For example:
-     *
-     *     for (A *a in b) if (stuff()) break;
-     *     for (A *a in b) xxxyy;
-     *
-     * The following code simply scans ahead to the semi to find the actual end.
-     */
-    const char *stmtBuf = SM->getCharacterData(OrigEnd);
-    const char *semiBuf = strchr(stmtBuf, ';');
-    assert(semiBuf && "Can't find ';'");
-    SourceLocation endBodyLoc = OrigEnd.getLocWithOffset(semiBuf-stmtBuf+1);
-    InsertText(endBodyLoc, buf);
-  }
-  Stmts.pop_back();
-  ObjCBcLabelNo.pop_back();
-  return 0;
-}
-
-static void Write_RethrowObject(std::string &buf) {
-  buf += "{ struct _FIN { _FIN(id reth) : rethrow(reth) {}\n";
-  buf += "\t~_FIN() { if (rethrow) objc_exception_throw(rethrow); }\n";
-  buf += "\tid rethrow;\n";
-  buf += "\t} _fin_force_rethow(_rethrow);";
-}
-
-/// RewriteObjCSynchronizedStmt -
-/// This routine rewrites @synchronized(expr) stmt;
-/// into:
-/// objc_sync_enter(expr);
-/// @try stmt @finally { objc_sync_exit(expr); }
-///
-Stmt *RewriteModernObjC::RewriteObjCSynchronizedStmt(ObjCAtSynchronizedStmt *S) {
-  // Get the start location and compute the semi location.
-  SourceLocation startLoc = S->getLocStart();
-  const char *startBuf = SM->getCharacterData(startLoc);
-
-  assert((*startBuf == '@') && "bogus @synchronized location");
-
-  std::string buf;
-  buf = "{ id _rethrow = 0; id _sync_obj = ";
-  
-  const char *lparenBuf = startBuf;
-  while (*lparenBuf != '(') lparenBuf++;
-  ReplaceText(startLoc, lparenBuf-startBuf+1, buf);
-  
-  buf = "; objc_sync_enter(_sync_obj);\n";
-  buf += "try {\n\tstruct _SYNC_EXIT { _SYNC_EXIT(id arg) : sync_exit(arg) {}";
-  buf += "\n\t~_SYNC_EXIT() {objc_sync_exit(sync_exit);}";
-  buf += "\n\tid sync_exit;";
-  buf += "\n\t} _sync_exit(_sync_obj);\n";
-
-  // We can't use S->getSynchExpr()->getLocEnd() to find the end location, since
-  // the sync expression is typically a message expression that's already
-  // been rewritten! (which implies the SourceLocation's are invalid).
-  SourceLocation RParenExprLoc = S->getSynchBody()->getLocStart();
-  const char *RParenExprLocBuf = SM->getCharacterData(RParenExprLoc);
-  while (*RParenExprLocBuf != ')') RParenExprLocBuf--;
-  RParenExprLoc = startLoc.getLocWithOffset(RParenExprLocBuf-startBuf);
-  
-  SourceLocation LBranceLoc = S->getSynchBody()->getLocStart();
-  const char *LBraceLocBuf = SM->getCharacterData(LBranceLoc);
-  assert (*LBraceLocBuf == '{');
-  ReplaceText(RParenExprLoc, (LBraceLocBuf - SM->getCharacterData(RParenExprLoc) + 1), buf);
-  
-  SourceLocation startRBraceLoc = S->getSynchBody()->getLocEnd();
-  assert((*SM->getCharacterData(startRBraceLoc) == '}') &&
-         "bogus @synchronized block");
-  
-  buf = "} catch (id e) {_rethrow = e;}\n";
-  Write_RethrowObject(buf);
-  buf += "}\n";
-  buf += "}\n";
-
-  ReplaceText(startRBraceLoc, 1, buf);
-
-  return 0;
-}
-
-void RewriteModernObjC::WarnAboutReturnGotoStmts(Stmt *S)
-{
-  // Perform a bottom up traversal of all children.
-  for (Stmt::child_range CI = S->children(); CI; ++CI)
-    if (*CI)
-      WarnAboutReturnGotoStmts(*CI);
-
-  if (isa<ReturnStmt>(S) || isa<GotoStmt>(S)) {
-    Diags.Report(Context->getFullLoc(S->getLocStart()),
-                 TryFinallyContainsReturnDiag);
-  }
-  return;
-}
-
-Stmt *RewriteModernObjC::RewriteObjCAutoreleasePoolStmt(ObjCAutoreleasePoolStmt  *S) {
-  SourceLocation startLoc = S->getAtLoc();
-  ReplaceText(startLoc, strlen("@autoreleasepool"), "/* @autoreleasepool */");
-  ReplaceText(S->getSubStmt()->getLocStart(), 1, 
-              "{ __AtAutoreleasePool __autoreleasepool; ");
-  
-  return 0;
-}
-
-Stmt *RewriteModernObjC::RewriteObjCTryStmt(ObjCAtTryStmt *S) {
-  ObjCAtFinallyStmt *finalStmt = S->getFinallyStmt();
-  bool noCatch = S->getNumCatchStmts() == 0;
-  std::string buf;
-  
-  if (finalStmt) {
-    if (noCatch)
-      buf = "{ id volatile _rethrow = 0;\n";
-    else {
-      buf = "{ id volatile _rethrow = 0;\ntry {\n";
-    }
-  }
-  // Get the start location and compute the semi location.
-  SourceLocation startLoc = S->getLocStart();
-  const char *startBuf = SM->getCharacterData(startLoc);
-
-  assert((*startBuf == '@') && "bogus @try location");
-  if (finalStmt)
-    ReplaceText(startLoc, 1, buf);
-  else
-    // @try -> try
-    ReplaceText(startLoc, 1, "");
-  
-  for (unsigned I = 0, N = S->getNumCatchStmts(); I != N; ++I) {
-    ObjCAtCatchStmt *Catch = S->getCatchStmt(I);
-    VarDecl *catchDecl = Catch->getCatchParamDecl();
-    
-    startLoc = Catch->getLocStart();
-    bool AtRemoved = false;
-    if (catchDecl) {
-      QualType t = catchDecl->getType();
-      if (const ObjCObjectPointerType *Ptr = t->getAs<ObjCObjectPointerType>()) {
-        // Should be a pointer to a class.
-        ObjCInterfaceDecl *IDecl = Ptr->getObjectType()->getInterface();
-        if (IDecl) {
-          std::string Result;
-          startBuf = SM->getCharacterData(startLoc);
-          assert((*startBuf == '@') && "bogus @catch location");
-          SourceLocation rParenLoc = Catch->getRParenLoc();
-          const char *rParenBuf = SM->getCharacterData(rParenLoc);
-          
-          // _objc_exc_Foo *_e as argument to catch.
-          Result = "catch (_objc_exc_"; Result += IDecl->getNameAsString();
-          Result += " *_"; Result += catchDecl->getNameAsString();
-          Result += ")";
-          ReplaceText(startLoc, rParenBuf-startBuf+1, Result);
-          // Foo *e = (Foo *)_e;
-          Result.clear();
-          Result = "{ ";
-          Result += IDecl->getNameAsString();
-          Result += " *"; Result += catchDecl->getNameAsString();
-          Result += " = ("; Result += IDecl->getNameAsString(); Result += "*)";
-          Result += "_"; Result += catchDecl->getNameAsString();
-          
-          Result += "; ";
-          SourceLocation lBraceLoc = Catch->getCatchBody()->getLocStart();
-          ReplaceText(lBraceLoc, 1, Result);
-          AtRemoved = true;
-        }
-      }
-    }
-    if (!AtRemoved)
-      // @catch -> catch
-      ReplaceText(startLoc, 1, "");
-      
-  }
-  if (finalStmt) {
-    buf.clear();
-    if (noCatch)
-      buf = "catch (id e) {_rethrow = e;}\n";
-    else 
-      buf = "}\ncatch (id e) {_rethrow = e;}\n";
-
-    SourceLocation startFinalLoc = finalStmt->getLocStart();
-    ReplaceText(startFinalLoc, 8, buf);
-    Stmt *body = finalStmt->getFinallyBody();
-    SourceLocation startFinalBodyLoc = body->getLocStart();
-    buf.clear();
-    Write_RethrowObject(buf);
-    ReplaceText(startFinalBodyLoc, 1, buf);
-    
-    SourceLocation endFinalBodyLoc = body->getLocEnd();
-    ReplaceText(endFinalBodyLoc, 1, "}\n}");
-    // Now check for any return/continue/go statements within the @try.
-    WarnAboutReturnGotoStmts(S->getTryBody());
-  }
-
-  return 0;
-}
-
-// This can't be done with ReplaceStmt(S, ThrowExpr), since
-// the throw expression is typically a message expression that's already
-// been rewritten! (which implies the SourceLocation's are invalid).
-Stmt *RewriteModernObjC::RewriteObjCThrowStmt(ObjCAtThrowStmt *S) {
-  // Get the start location and compute the semi location.
-  SourceLocation startLoc = S->getLocStart();
-  const char *startBuf = SM->getCharacterData(startLoc);
-
-  assert((*startBuf == '@') && "bogus @throw location");
-
-  std::string buf;
-  /* void objc_exception_throw(id) __attribute__((noreturn)); */
-  if (S->getThrowExpr())
-    buf = "objc_exception_throw(";
-  else
-    buf = "throw";
-
-  // handle "@  throw" correctly.
-  const char *wBuf = strchr(startBuf, 'w');
-  assert((*wBuf == 'w') && "@throw: can't find 'w'");
-  ReplaceText(startLoc, wBuf-startBuf+1, buf);
-
-  const char *semiBuf = strchr(startBuf, ';');
-  assert((*semiBuf == ';') && "@throw: can't find ';'");
-  SourceLocation semiLoc = startLoc.getLocWithOffset(semiBuf-startBuf);
-  if (S->getThrowExpr())
-    ReplaceText(semiLoc, 1, ");");
-  return 0;
-}
-
-Stmt *RewriteModernObjC::RewriteAtEncode(ObjCEncodeExpr *Exp) {
-  // Create a new string expression.
-  QualType StrType = Context->getPointerType(Context->CharTy);
-  std::string StrEncoding;
-  Context->getObjCEncodingForType(Exp->getEncodedType(), StrEncoding);
-  Expr *Replacement = StringLiteral::Create(*Context, StrEncoding,
-                                            StringLiteral::Ascii, false,
-                                            StrType, SourceLocation());
-  ReplaceStmt(Exp, Replacement);
-
-  // Replace this subexpr in the parent.
-  // delete Exp; leak for now, see RewritePropertyOrImplicitSetter() usage for more info.
-  return Replacement;
-}
-
-Stmt *RewriteModernObjC::RewriteAtSelector(ObjCSelectorExpr *Exp) {
-  if (!SelGetUidFunctionDecl)
-    SynthSelGetUidFunctionDecl();
-  assert(SelGetUidFunctionDecl && "Can't find sel_registerName() decl");
-  // Create a call to sel_registerName("selName").
-  SmallVector<Expr*, 8> SelExprs;
-  QualType argType = Context->getPointerType(Context->CharTy);
-  SelExprs.push_back(StringLiteral::Create(*Context,
-                                           Exp->getSelector().getAsString(),
-                                           StringLiteral::Ascii, false,
-                                           argType, SourceLocation()));
-  CallExpr *SelExp = SynthesizeCallToFunctionDecl(SelGetUidFunctionDecl,
-                                                 &SelExprs[0], SelExprs.size());
-  ReplaceStmt(Exp, SelExp);
-  // delete Exp; leak for now, see RewritePropertyOrImplicitSetter() usage for more info.
-  return SelExp;
-}
-
-CallExpr *RewriteModernObjC::SynthesizeCallToFunctionDecl(
-  FunctionDecl *FD, Expr **args, unsigned nargs, SourceLocation StartLoc,
-                                                    SourceLocation EndLoc) {
-  // Get the type, we will need to reference it in a couple spots.
-  QualType msgSendType = FD->getType();
-
-  // Create a reference to the objc_msgSend() declaration.
-  DeclRefExpr *DRE =
-    new (Context) DeclRefExpr(FD, false, msgSendType, VK_LValue, SourceLocation());
-
-  // Now, we cast the reference to a pointer to the objc_msgSend type.
-  QualType pToFunc = Context->getPointerType(msgSendType);
-  ImplicitCastExpr *ICE = 
-    ImplicitCastExpr::Create(*Context, pToFunc, CK_FunctionToPointerDecay,
-                             DRE, 0, VK_RValue);
-
-  const FunctionType *FT = msgSendType->getAs<FunctionType>();
-
-  CallExpr *Exp =  
-    new (Context) CallExpr(*Context, ICE, args, nargs, 
-                           FT->getCallResultType(*Context),
-                           VK_RValue, EndLoc);
-  return Exp;
-}
-
-static bool scanForProtocolRefs(const char *startBuf, const char *endBuf,
-                                const char *&startRef, const char *&endRef) {
-  while (startBuf < endBuf) {
-    if (*startBuf == '<')
-      startRef = startBuf; // mark the start.
-    if (*startBuf == '>') {
-      if (startRef && *startRef == '<') {
-        endRef = startBuf; // mark the end.
-        return true;
-      }
-      return false;
-    }
-    startBuf++;
-  }
-  return false;
-}
-
-static void scanToNextArgument(const char *&argRef) {
-  int angle = 0;
-  while (*argRef != ')' && (*argRef != ',' || angle > 0)) {
-    if (*argRef == '<')
-      angle++;
-    else if (*argRef == '>')
-      angle--;
-    argRef++;
-  }
-  assert(angle == 0 && "scanToNextArgument - bad protocol type syntax");
-}
-
-bool RewriteModernObjC::needToScanForQualifiers(QualType T) {
-  if (T->isObjCQualifiedIdType())
-    return true;
-  if (const PointerType *PT = T->getAs<PointerType>()) {
-    if (PT->getPointeeType()->isObjCQualifiedIdType())
-      return true;
-  }
-  if (T->isObjCObjectPointerType()) {
-    T = T->getPointeeType();
-    return T->isObjCQualifiedInterfaceType();
-  }
-  if (T->isArrayType()) {
-    QualType ElemTy = Context->getBaseElementType(T);
-    return needToScanForQualifiers(ElemTy);
-  }
-  return false;
-}
-
-void RewriteModernObjC::RewriteObjCQualifiedInterfaceTypes(Expr *E) {
-  QualType Type = E->getType();
-  if (needToScanForQualifiers(Type)) {
-    SourceLocation Loc, EndLoc;
-
-    if (const CStyleCastExpr *ECE = dyn_cast<CStyleCastExpr>(E)) {
-      Loc = ECE->getLParenLoc();
-      EndLoc = ECE->getRParenLoc();
-    } else {
-      Loc = E->getLocStart();
-      EndLoc = E->getLocEnd();
-    }
-    // This will defend against trying to rewrite synthesized expressions.
-    if (Loc.isInvalid() || EndLoc.isInvalid())
-      return;
-
-    const char *startBuf = SM->getCharacterData(Loc);
-    const char *endBuf = SM->getCharacterData(EndLoc);
-    const char *startRef = 0, *endRef = 0;
-    if (scanForProtocolRefs(startBuf, endBuf, startRef, endRef)) {
-      // Get the locations of the startRef, endRef.
-      SourceLocation LessLoc = Loc.getLocWithOffset(startRef-startBuf);
-      SourceLocation GreaterLoc = Loc.getLocWithOffset(endRef-startBuf+1);
-      // Comment out the protocol references.
-      InsertText(LessLoc, "/*");
-      InsertText(GreaterLoc, "*/");
-    }
-  }
-}
-
-void RewriteModernObjC::RewriteObjCQualifiedInterfaceTypes(Decl *Dcl) {
-  SourceLocation Loc;
-  QualType Type;
-  const FunctionProtoType *proto = 0;
-  if (VarDecl *VD = dyn_cast<VarDecl>(Dcl)) {
-    Loc = VD->getLocation();
-    Type = VD->getType();
-  }
-  else if (FunctionDecl *FD = dyn_cast<FunctionDecl>(Dcl)) {
-    Loc = FD->getLocation();
-    // Check for ObjC 'id' and class types that have been adorned with protocol
-    // information (id<p>, C<p>*). The protocol references need to be rewritten!
-    const FunctionType *funcType = FD->getType()->getAs<FunctionType>();
-    assert(funcType && "missing function type");
-    proto = dyn_cast<FunctionProtoType>(funcType);
-    if (!proto)
-      return;
-    Type = proto->getResultType();
-  }
-  else if (FieldDecl *FD = dyn_cast<FieldDecl>(Dcl)) {
-    Loc = FD->getLocation();
-    Type = FD->getType();
-  }
-  else
-    return;
-
-  if (needToScanForQualifiers(Type)) {
-    // Since types are unique, we need to scan the buffer.
-
-    const char *endBuf = SM->getCharacterData(Loc);
-    const char *startBuf = endBuf;
-    while (*startBuf != ';' && *startBuf != '<' && startBuf != MainFileStart)
-      startBuf--; // scan backward (from the decl location) for return type.
-    const char *startRef = 0, *endRef = 0;
-    if (scanForProtocolRefs(startBuf, endBuf, startRef, endRef)) {
-      // Get the locations of the startRef, endRef.
-      SourceLocation LessLoc = Loc.getLocWithOffset(startRef-endBuf);
-      SourceLocation GreaterLoc = Loc.getLocWithOffset(endRef-endBuf+1);
-      // Comment out the protocol references.
-      InsertText(LessLoc, "/*");
-      InsertText(GreaterLoc, "*/");
-    }
-  }
-  if (!proto)
-      return; // most likely, was a variable
-  // Now check arguments.
-  const char *startBuf = SM->getCharacterData(Loc);
-  const char *startFuncBuf = startBuf;
-  for (unsigned i = 0; i < proto->getNumArgs(); i++) {
-    if (needToScanForQualifiers(proto->getArgType(i))) {
-      // Since types are unique, we need to scan the buffer.
-
-      const char *endBuf = startBuf;
-      // scan forward (from the decl location) for argument types.
-      scanToNextArgument(endBuf);
-      const char *startRef = 0, *endRef = 0;
-      if (scanForProtocolRefs(startBuf, endBuf, startRef, endRef)) {
-        // Get the locations of the startRef, endRef.
-        SourceLocation LessLoc =
-          Loc.getLocWithOffset(startRef-startFuncBuf);
-        SourceLocation GreaterLoc =
-          Loc.getLocWithOffset(endRef-startFuncBuf+1);
-        // Comment out the protocol references.
-        InsertText(LessLoc, "/*");
-        InsertText(GreaterLoc, "*/");
-      }
-      startBuf = ++endBuf;
-    }
-    else {
-      // If the function name is derived from a macro expansion, then the
-      // argument buffer will not follow the name. Need to speak with Chris.
-      while (*startBuf && *startBuf != ')' && *startBuf != ',')
-        startBuf++; // scan forward (from the decl location) for argument types.
-      startBuf++;
-    }
-  }
-}
-
-void RewriteModernObjC::RewriteTypeOfDecl(VarDecl *ND) {
-  QualType QT = ND->getType();
-  const Type* TypePtr = QT->getAs<Type>();
-  if (!isa<TypeOfExprType>(TypePtr))
-    return;
-  while (isa<TypeOfExprType>(TypePtr)) {
-    const TypeOfExprType *TypeOfExprTypePtr = cast<TypeOfExprType>(TypePtr);
-    QT = TypeOfExprTypePtr->getUnderlyingExpr()->getType();
-    TypePtr = QT->getAs<Type>();
-  }
-  // FIXME. This will not work for multiple declarators; as in:
-  // __typeof__(a) b,c,d;
-  std::string TypeAsString(QT.getAsString(Context->getPrintingPolicy()));
-  SourceLocation DeclLoc = ND->getTypeSpecStartLoc();
-  const char *startBuf = SM->getCharacterData(DeclLoc);
-  if (ND->getInit()) {
-    std::string Name(ND->getNameAsString());
-    TypeAsString += " " + Name + " = ";
-    Expr *E = ND->getInit();
-    SourceLocation startLoc;
-    if (const CStyleCastExpr *ECE = dyn_cast<CStyleCastExpr>(E))
-      startLoc = ECE->getLParenLoc();
-    else
-      startLoc = E->getLocStart();
-    startLoc = SM->getExpansionLoc(startLoc);
-    const char *endBuf = SM->getCharacterData(startLoc);
-    ReplaceText(DeclLoc, endBuf-startBuf-1, TypeAsString);
-  }
-  else {
-    SourceLocation X = ND->getLocEnd();
-    X = SM->getExpansionLoc(X);
-    const char *endBuf = SM->getCharacterData(X);
-    ReplaceText(DeclLoc, endBuf-startBuf-1, TypeAsString);
-  }
-}
-
-// SynthSelGetUidFunctionDecl - SEL sel_registerName(const char *str);
-void RewriteModernObjC::SynthSelGetUidFunctionDecl() {
-  IdentifierInfo *SelGetUidIdent = &Context->Idents.get("sel_registerName");
-  SmallVector<QualType, 16> ArgTys;
-  ArgTys.push_back(Context->getPointerType(Context->CharTy.withConst()));
-  QualType getFuncType =
-    getSimpleFunctionType(Context->getObjCSelType(), &ArgTys[0], ArgTys.size());
-  SelGetUidFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
-                                           SourceLocation(),
-                                           SourceLocation(),
-                                           SelGetUidIdent, getFuncType, 0,
-                                           SC_Extern,
-                                           SC_None, false);
-}
-
-void RewriteModernObjC::RewriteFunctionDecl(FunctionDecl *FD) {
-  // declared in <objc/objc.h>
-  if (FD->getIdentifier() &&
-      FD->getName() == "sel_registerName") {
-    SelGetUidFunctionDecl = FD;
-    return;
-  }
-  RewriteObjCQualifiedInterfaceTypes(FD);
-}
-
-void RewriteModernObjC::RewriteBlockPointerType(std::string& Str, QualType Type) {
-  std::string TypeString(Type.getAsString(Context->getPrintingPolicy()));
-  const char *argPtr = TypeString.c_str();
-  if (!strchr(argPtr, '^')) {
-    Str += TypeString;
-    return;
-  }
-  while (*argPtr) {
-    Str += (*argPtr == '^' ? '*' : *argPtr);
-    argPtr++;
-  }
-}
-
-// FIXME. Consolidate this routine with RewriteBlockPointerType.
-void RewriteModernObjC::RewriteBlockPointerTypeVariable(std::string& Str,
-                                                  ValueDecl *VD) {
-  QualType Type = VD->getType();
-  std::string TypeString(Type.getAsString(Context->getPrintingPolicy()));
-  const char *argPtr = TypeString.c_str();
-  int paren = 0;
-  while (*argPtr) {
-    switch (*argPtr) {
-      case '(':
-        Str += *argPtr;
-        paren++;
-        break;
-      case ')':
-        Str += *argPtr;
-        paren--;
-        break;
-      case '^':
-        Str += '*';
-        if (paren == 1)
-          Str += VD->getNameAsString();
-        break;
-      default:
-        Str += *argPtr;
-        break;
-    }
-    argPtr++;
-  }
-}
-
-void RewriteModernObjC::RewriteBlockLiteralFunctionDecl(FunctionDecl *FD) {
-  SourceLocation FunLocStart = FD->getTypeSpecStartLoc();
-  const FunctionType *funcType = FD->getType()->getAs<FunctionType>();
-  const FunctionProtoType *proto = dyn_cast<FunctionProtoType>(funcType);
-  if (!proto)
-    return;
-  QualType Type = proto->getResultType();
-  std::string FdStr = Type.getAsString(Context->getPrintingPolicy());
-  FdStr += " ";
-  FdStr += FD->getName();
-  FdStr +=  "(";
-  unsigned numArgs = proto->getNumArgs();
-  for (unsigned i = 0; i < numArgs; i++) {
-  QualType ArgType = proto->getArgType(i);
-  RewriteBlockPointerType(FdStr, ArgType);
-  if (i+1 < numArgs)
-    FdStr += ", ";
-  }
-  if (FD->isVariadic()) {
-    FdStr +=  (numArgs > 0) ? ", ...);\n" : "...);\n";
-  }
-  else
-    FdStr +=  ");\n";
-  InsertText(FunLocStart, FdStr);
-}
-
-// SynthSuperContructorFunctionDecl - id __rw_objc_super(id obj, id super);
-void RewriteModernObjC::SynthSuperContructorFunctionDecl() {
-  if (SuperContructorFunctionDecl)
-    return;
-  IdentifierInfo *msgSendIdent = &Context->Idents.get("__rw_objc_super");
-  SmallVector<QualType, 16> ArgTys;
-  QualType argT = Context->getObjCIdType();
-  assert(!argT.isNull() && "Can't find 'id' type");
-  ArgTys.push_back(argT);
-  ArgTys.push_back(argT);
-  QualType msgSendType = getSimpleFunctionType(Context->getObjCIdType(),
-                                               &ArgTys[0], ArgTys.size());
-  SuperContructorFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
-                                         SourceLocation(),
-                                         SourceLocation(),
-                                         msgSendIdent, msgSendType, 0,
-                                         SC_Extern,
-                                         SC_None, false);
-}
-
-// SynthMsgSendFunctionDecl - id objc_msgSend(id self, SEL op, ...);
-void RewriteModernObjC::SynthMsgSendFunctionDecl() {
-  IdentifierInfo *msgSendIdent = &Context->Idents.get("objc_msgSend");
-  SmallVector<QualType, 16> ArgTys;
-  QualType argT = Context->getObjCIdType();
-  assert(!argT.isNull() && "Can't find 'id' type");
-  ArgTys.push_back(argT);
-  argT = Context->getObjCSelType();
-  assert(!argT.isNull() && "Can't find 'SEL' type");
-  ArgTys.push_back(argT);
-  QualType msgSendType = getSimpleFunctionType(Context->getObjCIdType(),
-                                               &ArgTys[0], ArgTys.size(),
-                                               true /*isVariadic*/);
-  MsgSendFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
-                                         SourceLocation(),
-                                         SourceLocation(),
-                                         msgSendIdent, msgSendType, 0,
-                                         SC_Extern,
-                                         SC_None, false);
-}
-
-// SynthMsgSendSuperFunctionDecl - id objc_msgSendSuper(void);
-void RewriteModernObjC::SynthMsgSendSuperFunctionDecl() {
-  IdentifierInfo *msgSendIdent = &Context->Idents.get("objc_msgSendSuper");
-  SmallVector<QualType, 2> ArgTys;
-  ArgTys.push_back(Context->VoidTy);
-  QualType msgSendType = getSimpleFunctionType(Context->getObjCIdType(),
-                                               &ArgTys[0], 1,
-                                               true /*isVariadic*/);
-  MsgSendSuperFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
-                                              SourceLocation(),
-                                              SourceLocation(),
-                                              msgSendIdent, msgSendType, 0,
-                                              SC_Extern,
-                                              SC_None, false);
-}
-
-// SynthMsgSendStretFunctionDecl - id objc_msgSend_stret(id self, SEL op, ...);
-void RewriteModernObjC::SynthMsgSendStretFunctionDecl() {
-  IdentifierInfo *msgSendIdent = &Context->Idents.get("objc_msgSend_stret");
-  SmallVector<QualType, 16> ArgTys;
-  QualType argT = Context->getObjCIdType();
-  assert(!argT.isNull() && "Can't find 'id' type");
-  ArgTys.push_back(argT);
-  argT = Context->getObjCSelType();
-  assert(!argT.isNull() && "Can't find 'SEL' type");
-  ArgTys.push_back(argT);
-  QualType msgSendType = getSimpleFunctionType(Context->getObjCIdType(),
-                                               &ArgTys[0], ArgTys.size(),
-                                               true /*isVariadic*/);
-  MsgSendStretFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
-                                         SourceLocation(),
-                                         SourceLocation(),
-                                         msgSendIdent, msgSendType, 0,
-                                         SC_Extern,
-                                         SC_None, false);
-}
-
-// SynthMsgSendSuperStretFunctionDecl -
-// id objc_msgSendSuper_stret(void);
-void RewriteModernObjC::SynthMsgSendSuperStretFunctionDecl() {
-  IdentifierInfo *msgSendIdent =
-    &Context->Idents.get("objc_msgSendSuper_stret");
-  SmallVector<QualType, 2> ArgTys;
-  ArgTys.push_back(Context->VoidTy);
-  QualType msgSendType = getSimpleFunctionType(Context->getObjCIdType(),
-                                               &ArgTys[0], 1,
-                                               true /*isVariadic*/);
-  MsgSendSuperStretFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
-                                                       SourceLocation(),
-                                                       SourceLocation(),
-                                              msgSendIdent, msgSendType, 0,
-                                              SC_Extern,
-                                              SC_None, false);
-}
-
-// SynthMsgSendFpretFunctionDecl - double objc_msgSend_fpret(id self, SEL op, ...);
-void RewriteModernObjC::SynthMsgSendFpretFunctionDecl() {
-  IdentifierInfo *msgSendIdent = &Context->Idents.get("objc_msgSend_fpret");
-  SmallVector<QualType, 16> ArgTys;
-  QualType argT = Context->getObjCIdType();
-  assert(!argT.isNull() && "Can't find 'id' type");
-  ArgTys.push_back(argT);
-  argT = Context->getObjCSelType();
-  assert(!argT.isNull() && "Can't find 'SEL' type");
-  ArgTys.push_back(argT);
-  QualType msgSendType = getSimpleFunctionType(Context->DoubleTy,
-                                               &ArgTys[0], ArgTys.size(),
-                                               true /*isVariadic*/);
-  MsgSendFpretFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
-                                              SourceLocation(),
-                                              SourceLocation(),
-                                              msgSendIdent, msgSendType, 0,
-                                              SC_Extern,
-                                              SC_None, false);
-}
-
-// SynthGetClassFunctionDecl - Class objc_getClass(const char *name);
-void RewriteModernObjC::SynthGetClassFunctionDecl() {
-  IdentifierInfo *getClassIdent = &Context->Idents.get("objc_getClass");
-  SmallVector<QualType, 16> ArgTys;
-  ArgTys.push_back(Context->getPointerType(Context->CharTy.withConst()));
-  QualType getClassType = getSimpleFunctionType(Context->getObjCClassType(),
-                                                &ArgTys[0], ArgTys.size());
-  GetClassFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
-                                          SourceLocation(),
-                                          SourceLocation(),
-                                          getClassIdent, getClassType, 0,
-                                          SC_Extern,
-                                          SC_None, false);
-}
-
-// SynthGetSuperClassFunctionDecl - Class class_getSuperclass(Class cls);
-void RewriteModernObjC::SynthGetSuperClassFunctionDecl() {
-  IdentifierInfo *getSuperClassIdent = 
-    &Context->Idents.get("class_getSuperclass");
-  SmallVector<QualType, 16> ArgTys;
-  ArgTys.push_back(Context->getObjCClassType());
-  QualType getClassType = getSimpleFunctionType(Context->getObjCClassType(),
-                                                &ArgTys[0], ArgTys.size());
-  GetSuperClassFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
-                                                   SourceLocation(),
-                                                   SourceLocation(),
-                                                   getSuperClassIdent,
-                                                   getClassType, 0,
-                                                   SC_Extern,
-                                                   SC_None,
-                                                   false);
-}
-
-// SynthGetMetaClassFunctionDecl - Class objc_getMetaClass(const char *name);
-void RewriteModernObjC::SynthGetMetaClassFunctionDecl() {
-  IdentifierInfo *getClassIdent = &Context->Idents.get("objc_getMetaClass");
-  SmallVector<QualType, 16> ArgTys;
-  ArgTys.push_back(Context->getPointerType(Context->CharTy.withConst()));
-  QualType getClassType = getSimpleFunctionType(Context->getObjCClassType(),
-                                                &ArgTys[0], ArgTys.size());
-  GetMetaClassFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
-                                              SourceLocation(),
-                                              SourceLocation(),
-                                              getClassIdent, getClassType, 0,
-                                              SC_Extern,
-                                              SC_None, false);
-}
-
-Stmt *RewriteModernObjC::RewriteObjCStringLiteral(ObjCStringLiteral *Exp) {
-  QualType strType = getConstantStringStructType();
-
-  std::string S = "__NSConstantStringImpl_";
-
-  std::string tmpName = InFileName;
-  unsigned i;
-  for (i=0; i < tmpName.length(); i++) {
-    char c = tmpName.at(i);
-    // replace any non alphanumeric characters with '_'.
-    if (!isalpha(c) && (c < '0' || c > '9'))
-      tmpName[i] = '_';
-  }
-  S += tmpName;
-  S += "_";
-  S += utostr(NumObjCStringLiterals++);
-
-  Preamble += "static __NSConstantStringImpl " + S;
-  Preamble += " __attribute__ ((section (\"__DATA, __cfstring\"))) = {__CFConstantStringClassReference,";
-  Preamble += "0x000007c8,"; // utf8_str
-  // The pretty printer for StringLiteral handles escape characters properly.
-  std::string prettyBufS;
-  llvm::raw_string_ostream prettyBuf(prettyBufS);
-  Exp->getString()->printPretty(prettyBuf, 0, PrintingPolicy(LangOpts));
-  Preamble += prettyBuf.str();
-  Preamble += ",";
-  Preamble += utostr(Exp->getString()->getByteLength()) + "};\n";
-
-  VarDecl *NewVD = VarDecl::Create(*Context, TUDecl, SourceLocation(),
-                                   SourceLocation(), &Context->Idents.get(S),
-                                   strType, 0, SC_Static, SC_None);
-  DeclRefExpr *DRE = new (Context) DeclRefExpr(NewVD, false, strType, VK_LValue,
-                                               SourceLocation());
-  Expr *Unop = new (Context) UnaryOperator(DRE, UO_AddrOf,
-                                 Context->getPointerType(DRE->getType()),
-                                           VK_RValue, OK_Ordinary,
-                                           SourceLocation());
-  // cast to NSConstantString *
-  CastExpr *cast = NoTypeInfoCStyleCastExpr(Context, Exp->getType(),
-                                            CK_CPointerToObjCPointerCast, Unop);
-  ReplaceStmt(Exp, cast);
-  // delete Exp; leak for now, see RewritePropertyOrImplicitSetter() usage for more info.
-  return cast;
-}
-
-Stmt *RewriteModernObjC::RewriteObjCBoolLiteralExpr(ObjCBoolLiteralExpr *Exp) {
-  unsigned IntSize =
-    static_cast<unsigned>(Context->getTypeSize(Context->IntTy));
-  
-  Expr *FlagExp = IntegerLiteral::Create(*Context, 
-                                         llvm::APInt(IntSize, Exp->getValue()), 
-                                         Context->IntTy, Exp->getLocation());
-  CastExpr *cast = NoTypeInfoCStyleCastExpr(Context, Context->ObjCBuiltinBoolTy,
-                                            CK_BitCast, FlagExp);
-  ParenExpr *PE = new (Context) ParenExpr(Exp->getLocation(), Exp->getExprLoc(), 
-                                          cast);
-  ReplaceStmt(Exp, PE);
-  return PE;
-}
-
-Stmt *RewriteModernObjC::RewriteObjCBoxedExpr(ObjCBoxedExpr *Exp) {
-  // synthesize declaration of helper functions needed in this routine.
-  if (!SelGetUidFunctionDecl)
-    SynthSelGetUidFunctionDecl();
-  // use objc_msgSend() for all.
-  if (!MsgSendFunctionDecl)
-    SynthMsgSendFunctionDecl();
-  if (!GetClassFunctionDecl)
-    SynthGetClassFunctionDecl();
-  
-  FunctionDecl *MsgSendFlavor = MsgSendFunctionDecl;
-  SourceLocation StartLoc = Exp->getLocStart();
-  SourceLocation EndLoc = Exp->getLocEnd();
-  
-  // Synthesize a call to objc_msgSend().
-  SmallVector<Expr*, 4> MsgExprs;
-  SmallVector<Expr*, 4> ClsExprs;
-  QualType argType = Context->getPointerType(Context->CharTy);
-  
-  // Create a call to objc_getClass("<BoxingClass>"). It will be the 1st argument.
-  ObjCMethodDecl *BoxingMethod = Exp->getBoxingMethod();
-  ObjCInterfaceDecl *BoxingClass = BoxingMethod->getClassInterface();
-  
-  IdentifierInfo *clsName = BoxingClass->getIdentifier();
-  ClsExprs.push_back(StringLiteral::Create(*Context,
-                                           clsName->getName(),
-                                           StringLiteral::Ascii, false,
-                                           argType, SourceLocation()));
-  CallExpr *Cls = SynthesizeCallToFunctionDecl(GetClassFunctionDecl,
-                                               &ClsExprs[0],
-                                               ClsExprs.size(), 
-                                               StartLoc, EndLoc);
-  MsgExprs.push_back(Cls);
-  
-  // Create a call to sel_registerName("<BoxingMethod>:"), etc.
-  // it will be the 2nd argument.
-  SmallVector<Expr*, 4> SelExprs;
-  SelExprs.push_back(StringLiteral::Create(*Context,
-                                           BoxingMethod->getSelector().getAsString(),
-                                           StringLiteral::Ascii, false,
-                                           argType, SourceLocation()));
-  CallExpr *SelExp = SynthesizeCallToFunctionDecl(SelGetUidFunctionDecl,
-                                                  &SelExprs[0], SelExprs.size(),
-                                                  StartLoc, EndLoc);
-  MsgExprs.push_back(SelExp);
-  
-  // User provided sub-expression is the 3rd, and last, argument.
-  Expr *subExpr  = Exp->getSubExpr();
-  if (ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(subExpr)) {
-    QualType type = ICE->getType();
-    const Expr *SubExpr = ICE->IgnoreParenImpCasts();
-    CastKind CK = CK_BitCast;
-    if (SubExpr->getType()->isIntegralType(*Context) && type->isBooleanType())
-      CK = CK_IntegralToBoolean;
-    subExpr = NoTypeInfoCStyleCastExpr(Context, type, CK, subExpr);
-  }
-  MsgExprs.push_back(subExpr);
-  
-  SmallVector<QualType, 4> ArgTypes;
-  ArgTypes.push_back(Context->getObjCIdType());
-  ArgTypes.push_back(Context->getObjCSelType());
-  for (ObjCMethodDecl::param_iterator PI = BoxingMethod->param_begin(),
-       E = BoxingMethod->param_end(); PI != E; ++PI)
-    ArgTypes.push_back((*PI)->getType());
-  
-  QualType returnType = Exp->getType();
-  // Get the type, we will need to reference it in a couple spots.
-  QualType msgSendType = MsgSendFlavor->getType();
-  
-  // Create a reference to the objc_msgSend() declaration.
-  DeclRefExpr *DRE = new (Context) DeclRefExpr(MsgSendFlavor, false, msgSendType,
-                                               VK_LValue, SourceLocation());
-  
-  CastExpr *cast = NoTypeInfoCStyleCastExpr(Context,
-                                            Context->getPointerType(Context->VoidTy),
-                                            CK_BitCast, DRE);
-  
-  // Now do the "normal" pointer to function cast.
-  QualType castType =
-  getSimpleFunctionType(returnType, &ArgTypes[0], ArgTypes.size(),
-                        BoxingMethod->isVariadic());
-  castType = Context->getPointerType(castType);
-  cast = NoTypeInfoCStyleCastExpr(Context, castType, CK_BitCast,
-                                  cast);
-  
-  // Don't forget the parens to enforce the proper binding.
-  ParenExpr *PE = new (Context) ParenExpr(StartLoc, EndLoc, cast);
-  
-  const FunctionType *FT = msgSendType->getAs<FunctionType>();
-  CallExpr *CE = new (Context) CallExpr(*Context, PE, &MsgExprs[0],
-                                        MsgExprs.size(),
-                                        FT->getResultType(), VK_RValue,
-                                        EndLoc);
-  ReplaceStmt(Exp, CE);
-  return CE;
-}
-
-Stmt *RewriteModernObjC::RewriteObjCArrayLiteralExpr(ObjCArrayLiteral *Exp) {
-  // synthesize declaration of helper functions needed in this routine.
-  if (!SelGetUidFunctionDecl)
-    SynthSelGetUidFunctionDecl();
-  // use objc_msgSend() for all.
-  if (!MsgSendFunctionDecl)
-    SynthMsgSendFunctionDecl();
-  if (!GetClassFunctionDecl)
-    SynthGetClassFunctionDecl();
-  
-  FunctionDecl *MsgSendFlavor = MsgSendFunctionDecl;
-  SourceLocation StartLoc = Exp->getLocStart();
-  SourceLocation EndLoc = Exp->getLocEnd();
-  
-  // Build the expression: __NSContainer_literal(int, ...).arr
-  QualType IntQT = Context->IntTy;
-  QualType NSArrayFType =
-    getSimpleFunctionType(Context->VoidTy, &IntQT, 1, true);
-  std::string NSArrayFName("__NSContainer_literal");
-  FunctionDecl *NSArrayFD = SynthBlockInitFunctionDecl(NSArrayFName);
-  DeclRefExpr *NSArrayDRE = 
-    new (Context) DeclRefExpr(NSArrayFD, false, NSArrayFType, VK_RValue,
-                              SourceLocation());
-
-  SmallVector<Expr*, 16> InitExprs;
-  unsigned NumElements = Exp->getNumElements();
-  unsigned UnsignedIntSize = 
-    static_cast<unsigned>(Context->getTypeSize(Context->UnsignedIntTy));
-  Expr *count = IntegerLiteral::Create(*Context,
-                                       llvm::APInt(UnsignedIntSize, NumElements),
-                                       Context->UnsignedIntTy, SourceLocation());
-  InitExprs.push_back(count);
-  for (unsigned i = 0; i < NumElements; i++)
-    InitExprs.push_back(Exp->getElement(i));
-  Expr *NSArrayCallExpr = 
-    new (Context) CallExpr(*Context, NSArrayDRE, &InitExprs[0], InitExprs.size(),
-                           NSArrayFType, VK_LValue, SourceLocation());
-  
-  FieldDecl *ARRFD = FieldDecl::Create(*Context, 0, SourceLocation(),
-                                    SourceLocation(),
-                                    &Context->Idents.get("arr"),
-                                    Context->getPointerType(Context->VoidPtrTy), 0,
-                                    /*BitWidth=*/0, /*Mutable=*/true,
-                                    ICIS_NoInit);
-  MemberExpr *ArrayLiteralME = 
-    new (Context) MemberExpr(NSArrayCallExpr, false, ARRFD, 
-                             SourceLocation(),
-                             ARRFD->getType(), VK_LValue,
-                             OK_Ordinary);
-  QualType ConstIdT = Context->getObjCIdType().withConst();
-  CStyleCastExpr * ArrayLiteralObjects = 
-    NoTypeInfoCStyleCastExpr(Context, 
-                             Context->getPointerType(ConstIdT),
-                             CK_BitCast,
-                             ArrayLiteralME);
-  
-  // Synthesize a call to objc_msgSend().
-  SmallVector<Expr*, 32> MsgExprs;
-  SmallVector<Expr*, 4> ClsExprs;
-  QualType argType = Context->getPointerType(Context->CharTy);
-  QualType expType = Exp->getType();
-  
-  // Create a call to objc_getClass("NSArray"). It will be th 1st argument.
-  ObjCInterfaceDecl *Class = 
-    expType->getPointeeType()->getAs<ObjCObjectType>()->getInterface();
-  
-  IdentifierInfo *clsName = Class->getIdentifier();
-  ClsExprs.push_back(StringLiteral::Create(*Context,
-                                           clsName->getName(),
-                                           StringLiteral::Ascii, false,
-                                           argType, SourceLocation()));
-  CallExpr *Cls = SynthesizeCallToFunctionDecl(GetClassFunctionDecl,
-                                               &ClsExprs[0],
-                                               ClsExprs.size(), 
-                                               StartLoc, EndLoc);
-  MsgExprs.push_back(Cls);
-  
-  // Create a call to sel_registerName("arrayWithObjects:count:").
-  // it will be the 2nd argument.
-  SmallVector<Expr*, 4> SelExprs;
-  ObjCMethodDecl *ArrayMethod = Exp->getArrayWithObjectsMethod();
-  SelExprs.push_back(StringLiteral::Create(*Context,
-                                           ArrayMethod->getSelector().getAsString(),
-                                           StringLiteral::Ascii, false,
-                                           argType, SourceLocation()));
-  CallExpr *SelExp = SynthesizeCallToFunctionDecl(SelGetUidFunctionDecl,
-                                                  &SelExprs[0], SelExprs.size(),
-                                                  StartLoc, EndLoc);
-  MsgExprs.push_back(SelExp);
-  
-  // (const id [])objects
-  MsgExprs.push_back(ArrayLiteralObjects);
-  
-  // (NSUInteger)cnt
-  Expr *cnt = IntegerLiteral::Create(*Context,
-                                     llvm::APInt(UnsignedIntSize, NumElements),
-                                     Context->UnsignedIntTy, SourceLocation());
-  MsgExprs.push_back(cnt);
-  
-  
-  SmallVector<QualType, 4> ArgTypes;
-  ArgTypes.push_back(Context->getObjCIdType());
-  ArgTypes.push_back(Context->getObjCSelType());
-  for (ObjCMethodDecl::param_iterator PI = ArrayMethod->param_begin(),
-       E = ArrayMethod->param_end(); PI != E; ++PI)
-    ArgTypes.push_back((*PI)->getType());
-  
-  QualType returnType = Exp->getType();
-  // Get the type, we will need to reference it in a couple spots.
-  QualType msgSendType = MsgSendFlavor->getType();
-  
-  // Create a reference to the objc_msgSend() declaration.
-  DeclRefExpr *DRE = new (Context) DeclRefExpr(MsgSendFlavor, false, msgSendType,
-                                               VK_LValue, SourceLocation());
-  
-  CastExpr *cast = NoTypeInfoCStyleCastExpr(Context,
-                                            Context->getPointerType(Context->VoidTy),
-                                            CK_BitCast, DRE);
-  
-  // Now do the "normal" pointer to function cast.
-  QualType castType =
-  getSimpleFunctionType(returnType, &ArgTypes[0], ArgTypes.size(),
-                        ArrayMethod->isVariadic());
-  castType = Context->getPointerType(castType);
-  cast = NoTypeInfoCStyleCastExpr(Context, castType, CK_BitCast,
-                                  cast);
-  
-  // Don't forget the parens to enforce the proper binding.
-  ParenExpr *PE = new (Context) ParenExpr(StartLoc, EndLoc, cast);
-  
-  const FunctionType *FT = msgSendType->getAs<FunctionType>();
-  CallExpr *CE = new (Context) CallExpr(*Context, PE, &MsgExprs[0],
-                                        MsgExprs.size(),
-                                        FT->getResultType(), VK_RValue,
-                                        EndLoc);
-  ReplaceStmt(Exp, CE);
-  return CE;
-}
-
-Stmt *RewriteModernObjC::RewriteObjCDictionaryLiteralExpr(ObjCDictionaryLiteral *Exp) {
-  // synthesize declaration of helper functions needed in this routine.
-  if (!SelGetUidFunctionDecl)
-    SynthSelGetUidFunctionDecl();
-  // use objc_msgSend() for all.
-  if (!MsgSendFunctionDecl)
-    SynthMsgSendFunctionDecl();
-  if (!GetClassFunctionDecl)
-    SynthGetClassFunctionDecl();
-  
-  FunctionDecl *MsgSendFlavor = MsgSendFunctionDecl;
-  SourceLocation StartLoc = Exp->getLocStart();
-  SourceLocation EndLoc = Exp->getLocEnd();
-  
-  // Build the expression: __NSContainer_literal(int, ...).arr
-  QualType IntQT = Context->IntTy;
-  QualType NSDictFType =
-    getSimpleFunctionType(Context->VoidTy, &IntQT, 1, true);
-  std::string NSDictFName("__NSContainer_literal");
-  FunctionDecl *NSDictFD = SynthBlockInitFunctionDecl(NSDictFName);
-  DeclRefExpr *NSDictDRE = 
-    new (Context) DeclRefExpr(NSDictFD, false, NSDictFType, VK_RValue,
-                              SourceLocation());
-  
-  SmallVector<Expr*, 16> KeyExprs;
-  SmallVector<Expr*, 16> ValueExprs;
-  
-  unsigned NumElements = Exp->getNumElements();
-  unsigned UnsignedIntSize = 
-    static_cast<unsigned>(Context->getTypeSize(Context->UnsignedIntTy));
-  Expr *count = IntegerLiteral::Create(*Context,
-                                       llvm::APInt(UnsignedIntSize, NumElements),
-                                       Context->UnsignedIntTy, SourceLocation());
-  KeyExprs.push_back(count);
-  ValueExprs.push_back(count);
-  for (unsigned i = 0; i < NumElements; i++) {
-    ObjCDictionaryElement Element = Exp->getKeyValueElement(i);
-    KeyExprs.push_back(Element.Key);
-    ValueExprs.push_back(Element.Value);
-  }
-  
-  // (const id [])objects
-  Expr *NSValueCallExpr = 
-    new (Context) CallExpr(*Context, NSDictDRE, &ValueExprs[0], ValueExprs.size(),
-                           NSDictFType, VK_LValue, SourceLocation());
-  
-  FieldDecl *ARRFD = FieldDecl::Create(*Context, 0, SourceLocation(),
-                                       SourceLocation(),
-                                       &Context->Idents.get("arr"),
-                                       Context->getPointerType(Context->VoidPtrTy), 0,
-                                       /*BitWidth=*/0, /*Mutable=*/true,
-                                       ICIS_NoInit);
-  MemberExpr *DictLiteralValueME = 
-    new (Context) MemberExpr(NSValueCallExpr, false, ARRFD, 
-                             SourceLocation(),
-                             ARRFD->getType(), VK_LValue,
-                             OK_Ordinary);
-  QualType ConstIdT = Context->getObjCIdType().withConst();
-  CStyleCastExpr * DictValueObjects = 
-    NoTypeInfoCStyleCastExpr(Context, 
-                             Context->getPointerType(ConstIdT),
-                             CK_BitCast,
-                             DictLiteralValueME);
-  // (const id <NSCopying> [])keys
-  Expr *NSKeyCallExpr = 
-    new (Context) CallExpr(*Context, NSDictDRE, &KeyExprs[0], KeyExprs.size(),
-                           NSDictFType, VK_LValue, SourceLocation());
-  
-  MemberExpr *DictLiteralKeyME = 
-    new (Context) MemberExpr(NSKeyCallExpr, false, ARRFD, 
-                             SourceLocation(),
-                             ARRFD->getType(), VK_LValue,
-                             OK_Ordinary);
-  
-  CStyleCastExpr * DictKeyObjects = 
-    NoTypeInfoCStyleCastExpr(Context, 
-                             Context->getPointerType(ConstIdT),
-                             CK_BitCast,
-                             DictLiteralKeyME);
-  
-  
-  
-  // Synthesize a call to objc_msgSend().
-  SmallVector<Expr*, 32> MsgExprs;
-  SmallVector<Expr*, 4> ClsExprs;
-  QualType argType = Context->getPointerType(Context->CharTy);
-  QualType expType = Exp->getType();
-  
-  // Create a call to objc_getClass("NSArray"). It will be th 1st argument.
-  ObjCInterfaceDecl *Class = 
-  expType->getPointeeType()->getAs<ObjCObjectType>()->getInterface();
-  
-  IdentifierInfo *clsName = Class->getIdentifier();
-  ClsExprs.push_back(StringLiteral::Create(*Context,
-                                           clsName->getName(),
-                                           StringLiteral::Ascii, false,
-                                           argType, SourceLocation()));
-  CallExpr *Cls = SynthesizeCallToFunctionDecl(GetClassFunctionDecl,
-                                               &ClsExprs[0],
-                                               ClsExprs.size(), 
-                                               StartLoc, EndLoc);
-  MsgExprs.push_back(Cls);
-  
-  // Create a call to sel_registerName("arrayWithObjects:count:").
-  // it will be the 2nd argument.
-  SmallVector<Expr*, 4> SelExprs;
-  ObjCMethodDecl *DictMethod = Exp->getDictWithObjectsMethod();
-  SelExprs.push_back(StringLiteral::Create(*Context,
-                                           DictMethod->getSelector().getAsString(),
-                                           StringLiteral::Ascii, false,
-                                           argType, SourceLocation()));
-  CallExpr *SelExp = SynthesizeCallToFunctionDecl(SelGetUidFunctionDecl,
-                                                  &SelExprs[0], SelExprs.size(),
-                                                  StartLoc, EndLoc);
-  MsgExprs.push_back(SelExp);
-  
-  // (const id [])objects
-  MsgExprs.push_back(DictValueObjects);
-  
-  // (const id <NSCopying> [])keys
-  MsgExprs.push_back(DictKeyObjects);
-  
-  // (NSUInteger)cnt
-  Expr *cnt = IntegerLiteral::Create(*Context,
-                                     llvm::APInt(UnsignedIntSize, NumElements),
-                                     Context->UnsignedIntTy, SourceLocation());
-  MsgExprs.push_back(cnt);
-  
-  
-  SmallVector<QualType, 8> ArgTypes;
-  ArgTypes.push_back(Context->getObjCIdType());
-  ArgTypes.push_back(Context->getObjCSelType());
-  for (ObjCMethodDecl::param_iterator PI = DictMethod->param_begin(),
-       E = DictMethod->param_end(); PI != E; ++PI) {
-    QualType T = (*PI)->getType();
-    if (const PointerType* PT = T->getAs<PointerType>()) {
-      QualType PointeeTy = PT->getPointeeType();
-      convertToUnqualifiedObjCType(PointeeTy);
-      T = Context->getPointerType(PointeeTy);
-    }
-    ArgTypes.push_back(T);
-  }
-  
-  QualType returnType = Exp->getType();
-  // Get the type, we will need to reference it in a couple spots.
-  QualType msgSendType = MsgSendFlavor->getType();
-  
-  // Create a reference to the objc_msgSend() declaration.
-  DeclRefExpr *DRE = new (Context) DeclRefExpr(MsgSendFlavor, false, msgSendType,
-                                               VK_LValue, SourceLocation());
-  
-  CastExpr *cast = NoTypeInfoCStyleCastExpr(Context,
-                                            Context->getPointerType(Context->VoidTy),
-                                            CK_BitCast, DRE);
-  
-  // Now do the "normal" pointer to function cast.
-  QualType castType =
-  getSimpleFunctionType(returnType, &ArgTypes[0], ArgTypes.size(),
-                        DictMethod->isVariadic());
-  castType = Context->getPointerType(castType);
-  cast = NoTypeInfoCStyleCastExpr(Context, castType, CK_BitCast,
-                                  cast);
-  
-  // Don't forget the parens to enforce the proper binding.
-  ParenExpr *PE = new (Context) ParenExpr(StartLoc, EndLoc, cast);
-  
-  const FunctionType *FT = msgSendType->getAs<FunctionType>();
-  CallExpr *CE = new (Context) CallExpr(*Context, PE, &MsgExprs[0],
-                                        MsgExprs.size(),
-                                        FT->getResultType(), VK_RValue,
-                                        EndLoc);
-  ReplaceStmt(Exp, CE);
-  return CE;
-}
-
-// struct __rw_objc_super { 
-//   struct objc_object *object; struct objc_object *superClass; 
-// };
-QualType RewriteModernObjC::getSuperStructType() {
-  if (!SuperStructDecl) {
-    SuperStructDecl = RecordDecl::Create(*Context, TTK_Struct, TUDecl,
-                                         SourceLocation(), SourceLocation(),
-                                         &Context->Idents.get("__rw_objc_super"));
-    QualType FieldTypes[2];
-
-    // struct objc_object *object;
-    FieldTypes[0] = Context->getObjCIdType();
-    // struct objc_object *superClass;
-    FieldTypes[1] = Context->getObjCIdType();
-
-    // Create fields
-    for (unsigned i = 0; i < 2; ++i) {
-      SuperStructDecl->addDecl(FieldDecl::Create(*Context, SuperStructDecl,
-                                                 SourceLocation(),
-                                                 SourceLocation(), 0,
-                                                 FieldTypes[i], 0,
-                                                 /*BitWidth=*/0,
-                                                 /*Mutable=*/false,
-                                                 ICIS_NoInit));
-    }
-
-    SuperStructDecl->completeDefinition();
-  }
-  return Context->getTagDeclType(SuperStructDecl);
-}
-
-QualType RewriteModernObjC::getConstantStringStructType() {
-  if (!ConstantStringDecl) {
-    ConstantStringDecl = RecordDecl::Create(*Context, TTK_Struct, TUDecl,
-                                            SourceLocation(), SourceLocation(),
-                         &Context->Idents.get("__NSConstantStringImpl"));
-    QualType FieldTypes[4];
-
-    // struct objc_object *receiver;
-    FieldTypes[0] = Context->getObjCIdType();
-    // int flags;
-    FieldTypes[1] = Context->IntTy;
-    // char *str;
-    FieldTypes[2] = Context->getPointerType(Context->CharTy);
-    // long length;
-    FieldTypes[3] = Context->LongTy;
-
-    // Create fields
-    for (unsigned i = 0; i < 4; ++i) {
-      ConstantStringDecl->addDecl(FieldDecl::Create(*Context,
-                                                    ConstantStringDecl,
-                                                    SourceLocation(),
-                                                    SourceLocation(), 0,
-                                                    FieldTypes[i], 0,
-                                                    /*BitWidth=*/0,
-                                                    /*Mutable=*/true,
-                                                    ICIS_NoInit));
-    }
-
-    ConstantStringDecl->completeDefinition();
-  }
-  return Context->getTagDeclType(ConstantStringDecl);
-}
-
-/// getFunctionSourceLocation - returns start location of a function
-/// definition. Complication arises when function has declared as
-/// extern "C" or extern "C" {...}
-static SourceLocation getFunctionSourceLocation (RewriteModernObjC &R,
-                                                 FunctionDecl *FD) {
-  if (FD->isExternC()  && !FD->isMain()) {
-    const DeclContext *DC = FD->getDeclContext();
-    if (const LinkageSpecDecl *LSD = dyn_cast<LinkageSpecDecl>(DC))
-      // if it is extern "C" {...}, return function decl's own location.
-      if (!LSD->getRBraceLoc().isValid())
-        return LSD->getExternLoc();
-  }
-  if (FD->getStorageClassAsWritten() != SC_None)
-    R.RewriteBlockLiteralFunctionDecl(FD);
-  return FD->getTypeSpecStartLoc();
-}
-
-/// SynthMsgSendStretCallExpr - This routine translates message expression
-/// into a call to objc_msgSend_stret() entry point. Tricky part is that
-/// nil check on receiver must be performed before calling objc_msgSend_stret.
-/// MsgSendStretFlavor - function declaration objc_msgSend_stret(...)
-/// msgSendType - function type of objc_msgSend_stret(...)
-/// returnType - Result type of the method being synthesized.
-/// ArgTypes - type of the arguments passed to objc_msgSend_stret, starting with receiver type.
-/// MsgExprs - list of argument expressions being passed to objc_msgSend_stret, 
-/// starting with receiver.
-/// Method - Method being rewritten.
-Expr *RewriteModernObjC::SynthMsgSendStretCallExpr(FunctionDecl *MsgSendStretFlavor,
-                                                 QualType msgSendType, 
-                                                 QualType returnType, 
-                                                 SmallVectorImpl<QualType> &ArgTypes,
-                                                 SmallVectorImpl<Expr*> &MsgExprs,
-                                                 ObjCMethodDecl *Method) {
-  // Now do the "normal" pointer to function cast.
-  QualType castType = getSimpleFunctionType(returnType, &ArgTypes[0], ArgTypes.size(),
-                                            Method ? Method->isVariadic() : false);
-  castType = Context->getPointerType(castType);
-  
-  // build type for containing the objc_msgSend_stret object.
-  static unsigned stretCount=0;
-  std::string name = "__Stret"; name += utostr(stretCount);
-  std::string str = 
-    "extern \"C\" void * __cdecl memset(void *_Dst, int _Val, size_t _Size);\n";
-  str += "struct "; str += name;
-  str += " {\n\t";
-  str += name;
-  str += "(id receiver, SEL sel";
-  for (unsigned i = 2; i < ArgTypes.size(); i++) {
-    std::string ArgName = "arg"; ArgName += utostr(i);
-    ArgTypes[i].getAsStringInternal(ArgName, Context->getPrintingPolicy());
-    str += ", "; str += ArgName;
-  }
-  // could be vararg.
-  for (unsigned i = ArgTypes.size(); i < MsgExprs.size(); i++) {
-    std::string ArgName = "arg"; ArgName += utostr(i);
-    MsgExprs[i]->getType().getAsStringInternal(ArgName,
-                                               Context->getPrintingPolicy());
-    str += ", "; str += ArgName;
-  }
-  
-  str += ") {\n";
-  str += "\t  if (receiver == 0)\n";
-  str += "\t    memset((void*)&s, 0, sizeof(s));\n";
-  str += "\t  else\n";
-  str += "\t    s = (("; str += castType.getAsString(Context->getPrintingPolicy());
-  str += ")(void *)objc_msgSend_stret)(receiver, sel";
-  for (unsigned i = 2; i < ArgTypes.size(); i++) {
-    str += ", arg"; str += utostr(i);
-  }
-  // could be vararg.
-  for (unsigned i = ArgTypes.size(); i < MsgExprs.size(); i++) {
-    str += ", arg"; str += utostr(i);
-  }
-  
-  str += ");\n";
-  str += "\t}\n";
-  str += "\t"; str += returnType.getAsString(Context->getPrintingPolicy());
-  str += " s;\n";
-  str += "};\n\n";
-  SourceLocation FunLocStart = getFunctionSourceLocation(*this, CurFunctionDef);
-  InsertText(FunLocStart, str);
-  ++stretCount;
-  
-  // AST for __Stretn(receiver, args).s;
-  IdentifierInfo *ID = &Context->Idents.get(name);
-  FunctionDecl *FD = FunctionDecl::Create(*Context, TUDecl, SourceLocation(),
-                                          SourceLocation(), ID, castType, 0, SC_Extern,
-                                          SC_None, false, false);
-  DeclRefExpr *DRE = new (Context) DeclRefExpr(FD, false, castType, VK_RValue,
-                                               SourceLocation());
-  CallExpr *STCE = new (Context) CallExpr(*Context, DRE, &MsgExprs[0], MsgExprs.size(),
-                                          castType, VK_LValue, SourceLocation());
-  
-  FieldDecl *FieldD = FieldDecl::Create(*Context, 0, SourceLocation(),
-                                    SourceLocation(),
-                                    &Context->Idents.get("s"),
-                                    returnType, 0,
-                                    /*BitWidth=*/0, /*Mutable=*/true,
-                                    ICIS_NoInit);
-  MemberExpr *ME = new (Context) MemberExpr(STCE, false, FieldD, SourceLocation(),
-                                            FieldD->getType(), VK_LValue,
-                                            OK_Ordinary);
-
-  return ME;
-}
-
-Stmt *RewriteModernObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
-                                    SourceLocation StartLoc,
-                                    SourceLocation EndLoc) {
-  if (!SelGetUidFunctionDecl)
-    SynthSelGetUidFunctionDecl();
-  if (!MsgSendFunctionDecl)
-    SynthMsgSendFunctionDecl();
-  if (!MsgSendSuperFunctionDecl)
-    SynthMsgSendSuperFunctionDecl();
-  if (!MsgSendStretFunctionDecl)
-    SynthMsgSendStretFunctionDecl();
-  if (!MsgSendSuperStretFunctionDecl)
-    SynthMsgSendSuperStretFunctionDecl();
-  if (!MsgSendFpretFunctionDecl)
-    SynthMsgSendFpretFunctionDecl();
-  if (!GetClassFunctionDecl)
-    SynthGetClassFunctionDecl();
-  if (!GetSuperClassFunctionDecl)
-    SynthGetSuperClassFunctionDecl();
-  if (!GetMetaClassFunctionDecl)
-    SynthGetMetaClassFunctionDecl();
-
-  // default to objc_msgSend().
-  FunctionDecl *MsgSendFlavor = MsgSendFunctionDecl;
-  // May need to use objc_msgSend_stret() as well.
-  FunctionDecl *MsgSendStretFlavor = 0;
-  if (ObjCMethodDecl *mDecl = Exp->getMethodDecl()) {
-    QualType resultType = mDecl->getResultType();
-    if (resultType->isRecordType())
-      MsgSendStretFlavor = MsgSendStretFunctionDecl;
-    else if (resultType->isRealFloatingType())
-      MsgSendFlavor = MsgSendFpretFunctionDecl;
-  }
-
-  // Synthesize a call to objc_msgSend().
-  SmallVector<Expr*, 8> MsgExprs;
-  switch (Exp->getReceiverKind()) {
-  case ObjCMessageExpr::SuperClass: {
-    MsgSendFlavor = MsgSendSuperFunctionDecl;
-    if (MsgSendStretFlavor)
-      MsgSendStretFlavor = MsgSendSuperStretFunctionDecl;
-    assert(MsgSendFlavor && "MsgSendFlavor is NULL!");
-
-    ObjCInterfaceDecl *ClassDecl = CurMethodDef->getClassInterface();
-
-    SmallVector<Expr*, 4> InitExprs;
-
-    // set the receiver to self, the first argument to all methods.
-    InitExprs.push_back(
-      NoTypeInfoCStyleCastExpr(Context, Context->getObjCIdType(),
-                               CK_BitCast,
-                   new (Context) DeclRefExpr(CurMethodDef->getSelfDecl(),
-                                             false,
-                                             Context->getObjCIdType(),
-                                             VK_RValue,
-                                             SourceLocation()))
-                        ); // set the 'receiver'.
-
-    // (id)class_getSuperclass((Class)objc_getClass("CurrentClass"))
-    SmallVector<Expr*, 8> ClsExprs;
-    QualType argType = Context->getPointerType(Context->CharTy);
-    ClsExprs.push_back(StringLiteral::Create(*Context,
-                                   ClassDecl->getIdentifier()->getName(),
-                                   StringLiteral::Ascii, false,
-                                   argType, SourceLocation()));
-    // (Class)objc_getClass("CurrentClass")
-    CallExpr *Cls = SynthesizeCallToFunctionDecl(GetMetaClassFunctionDecl,
-                                                 &ClsExprs[0],
-                                                 ClsExprs.size(),
-                                                 StartLoc,
-                                                 EndLoc);
-    ClsExprs.clear();
-    ClsExprs.push_back(Cls);
-    Cls = SynthesizeCallToFunctionDecl(GetSuperClassFunctionDecl,
-                                       &ClsExprs[0], ClsExprs.size(),
-                                       StartLoc, EndLoc);
-    
-    // (id)class_getSuperclass((Class)objc_getClass("CurrentClass"))
-    // To turn off a warning, type-cast to 'id'
-    InitExprs.push_back( // set 'super class', using class_getSuperclass().
-                        NoTypeInfoCStyleCastExpr(Context,
-                                                 Context->getObjCIdType(),
-                                                 CK_BitCast, Cls));
-    // struct __rw_objc_super
-    QualType superType = getSuperStructType();
-    Expr *SuperRep;
-
-    if (LangOpts.MicrosoftExt) {
-      SynthSuperContructorFunctionDecl();
-      // Simulate a contructor call...
-      DeclRefExpr *DRE = new (Context) DeclRefExpr(SuperContructorFunctionDecl,
-                                                   false, superType, VK_LValue,
-                                                   SourceLocation());
-      SuperRep = new (Context) CallExpr(*Context, DRE, &InitExprs[0],
-                                        InitExprs.size(),
-                                        superType, VK_LValue,
-                                        SourceLocation());
-      // The code for super is a little tricky to prevent collision with
-      // the structure definition in the header. The rewriter has it's own
-      // internal definition (__rw_objc_super) that is uses. This is why
-      // we need the cast below. For example:
-      // (struct __rw_objc_super *)&__rw_objc_super((id)self, (id)objc_getClass("SUPER"))
-      //
-      SuperRep = new (Context) UnaryOperator(SuperRep, UO_AddrOf,
-                               Context->getPointerType(SuperRep->getType()),
-                                             VK_RValue, OK_Ordinary,
-                                             SourceLocation());
-      SuperRep = NoTypeInfoCStyleCastExpr(Context,
-                                          Context->getPointerType(superType),
-                                          CK_BitCast, SuperRep);
-    } else {
-      // (struct __rw_objc_super) { <exprs from above> }
-      InitListExpr *ILE =
-        new (Context) InitListExpr(*Context, SourceLocation(),
-                                   &InitExprs[0], InitExprs.size(),
-                                   SourceLocation());
-      TypeSourceInfo *superTInfo
-        = Context->getTrivialTypeSourceInfo(superType);
-      SuperRep = new (Context) CompoundLiteralExpr(SourceLocation(), superTInfo,
-                                                   superType, VK_LValue,
-                                                   ILE, false);
-      // struct __rw_objc_super *
-      SuperRep = new (Context) UnaryOperator(SuperRep, UO_AddrOf,
-                               Context->getPointerType(SuperRep->getType()),
-                                             VK_RValue, OK_Ordinary,
-                                             SourceLocation());
-    }
-    MsgExprs.push_back(SuperRep);
-    break;
-  }
-
-  case ObjCMessageExpr::Class: {
-    SmallVector<Expr*, 8> ClsExprs;
-    QualType argType = Context->getPointerType(Context->CharTy);
-    ObjCInterfaceDecl *Class
-      = Exp->getClassReceiver()->getAs<ObjCObjectType>()->getInterface();
-    IdentifierInfo *clsName = Class->getIdentifier();
-    ClsExprs.push_back(StringLiteral::Create(*Context,
-                                             clsName->getName(),
-                                             StringLiteral::Ascii, false,
-                                             argType, SourceLocation()));
-    CallExpr *Cls = SynthesizeCallToFunctionDecl(GetClassFunctionDecl,
-                                                 &ClsExprs[0],
-                                                 ClsExprs.size(), 
-                                                 StartLoc, EndLoc);
-    CastExpr *ArgExpr = NoTypeInfoCStyleCastExpr(Context,
-                                                 Context->getObjCIdType(),
-                                                 CK_BitCast, Cls);
-    MsgExprs.push_back(ArgExpr);
-    break;
-  }
-
-  case ObjCMessageExpr::SuperInstance:{
-    MsgSendFlavor = MsgSendSuperFunctionDecl;
-    if (MsgSendStretFlavor)
-      MsgSendStretFlavor = MsgSendSuperStretFunctionDecl;
-    assert(MsgSendFlavor && "MsgSendFlavor is NULL!");
-    ObjCInterfaceDecl *ClassDecl = CurMethodDef->getClassInterface();
-    SmallVector<Expr*, 4> InitExprs;
-
-    InitExprs.push_back(
-      NoTypeInfoCStyleCastExpr(Context, Context->getObjCIdType(),
-                               CK_BitCast,
-                   new (Context) DeclRefExpr(CurMethodDef->getSelfDecl(),
-                                             false,
-                                             Context->getObjCIdType(),
-                                             VK_RValue, SourceLocation()))
-                        ); // set the 'receiver'.
-    
-    // (id)class_getSuperclass((Class)objc_getClass("CurrentClass"))
-    SmallVector<Expr*, 8> ClsExprs;
-    QualType argType = Context->getPointerType(Context->CharTy);
-    ClsExprs.push_back(StringLiteral::Create(*Context,
-                                   ClassDecl->getIdentifier()->getName(),
-                                   StringLiteral::Ascii, false, argType,
-                                   SourceLocation()));
-    // (Class)objc_getClass("CurrentClass")
-    CallExpr *Cls = SynthesizeCallToFunctionDecl(GetClassFunctionDecl,
-                                                 &ClsExprs[0],
-                                                 ClsExprs.size(), 
-                                                 StartLoc, EndLoc);
-    ClsExprs.clear();
-    ClsExprs.push_back(Cls);
-    Cls = SynthesizeCallToFunctionDecl(GetSuperClassFunctionDecl,
-                                       &ClsExprs[0], ClsExprs.size(),
-                                       StartLoc, EndLoc);
-    
-    // (id)class_getSuperclass((Class)objc_getClass("CurrentClass"))
-    // To turn off a warning, type-cast to 'id'
-    InitExprs.push_back(
-      // set 'super class', using class_getSuperclass().
-      NoTypeInfoCStyleCastExpr(Context, Context->getObjCIdType(),
-                               CK_BitCast, Cls));
-    // struct __rw_objc_super
-    QualType superType = getSuperStructType();
-    Expr *SuperRep;
-
-    if (LangOpts.MicrosoftExt) {
-      SynthSuperContructorFunctionDecl();
-      // Simulate a contructor call...
-      DeclRefExpr *DRE = new (Context) DeclRefExpr(SuperContructorFunctionDecl,
-                                                   false, superType, VK_LValue,
-                                                   SourceLocation());
-      SuperRep = new (Context) CallExpr(*Context, DRE, &InitExprs[0],
-                                        InitExprs.size(),
-                                        superType, VK_LValue, SourceLocation());
-      // The code for super is a little tricky to prevent collision with
-      // the structure definition in the header. The rewriter has it's own
-      // internal definition (__rw_objc_super) that is uses. This is why
-      // we need the cast below. For example:
-      // (struct __rw_objc_super *)&__rw_objc_super((id)self, (id)objc_getClass("SUPER"))
-      //
-      SuperRep = new (Context) UnaryOperator(SuperRep, UO_AddrOf,
-                               Context->getPointerType(SuperRep->getType()),
-                               VK_RValue, OK_Ordinary,
-                               SourceLocation());
-      SuperRep = NoTypeInfoCStyleCastExpr(Context,
-                               Context->getPointerType(superType),
-                               CK_BitCast, SuperRep);
-    } else {
-      // (struct __rw_objc_super) { <exprs from above> }
-      InitListExpr *ILE =
-        new (Context) InitListExpr(*Context, SourceLocation(),
-                                   &InitExprs[0], InitExprs.size(),
-                                   SourceLocation());
-      TypeSourceInfo *superTInfo
-        = Context->getTrivialTypeSourceInfo(superType);
-      SuperRep = new (Context) CompoundLiteralExpr(SourceLocation(), superTInfo,
-                                                   superType, VK_RValue, ILE,
-                                                   false);
-    }
-    MsgExprs.push_back(SuperRep);
-    break;
-  }
-
-  case ObjCMessageExpr::Instance: {
-    // Remove all type-casts because it may contain objc-style types; e.g.
-    // Foo<Proto> *.
-    Expr *recExpr = Exp->getInstanceReceiver();
-    while (CStyleCastExpr *CE = dyn_cast<CStyleCastExpr>(recExpr))
-      recExpr = CE->getSubExpr();
-    CastKind CK = recExpr->getType()->isObjCObjectPointerType()
-                    ? CK_BitCast : recExpr->getType()->isBlockPointerType()
-                                     ? CK_BlockPointerToObjCPointerCast
-                                     : CK_CPointerToObjCPointerCast;
-
-    recExpr = NoTypeInfoCStyleCastExpr(Context, Context->getObjCIdType(),
-                                       CK, recExpr);
-    MsgExprs.push_back(recExpr);
-    break;
-  }
-  }
-
-  // Create a call to sel_registerName("selName"), it will be the 2nd argument.
-  SmallVector<Expr*, 8> SelExprs;
-  QualType argType = Context->getPointerType(Context->CharTy);
-  SelExprs.push_back(StringLiteral::Create(*Context,
-                                       Exp->getSelector().getAsString(),
-                                       StringLiteral::Ascii, false,
-                                       argType, SourceLocation()));
-  CallExpr *SelExp = SynthesizeCallToFunctionDecl(SelGetUidFunctionDecl,
-                                                 &SelExprs[0], SelExprs.size(),
-                                                  StartLoc,
-                                                  EndLoc);
-  MsgExprs.push_back(SelExp);
-
-  // Now push any user supplied arguments.
-  for (unsigned i = 0; i < Exp->getNumArgs(); i++) {
-    Expr *userExpr = Exp->getArg(i);
-    // Make all implicit casts explicit...ICE comes in handy:-)
-    if (ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(userExpr)) {
-      // Reuse the ICE type, it is exactly what the doctor ordered.
-      QualType type = ICE->getType();
-      if (needToScanForQualifiers(type))
-        type = Context->getObjCIdType();
-      // Make sure we convert "type (^)(...)" to "type (*)(...)".
-      (void)convertBlockPointerToFunctionPointer(type);
-      const Expr *SubExpr = ICE->IgnoreParenImpCasts();
-      CastKind CK;
-      if (SubExpr->getType()->isIntegralType(*Context) && 
-          type->isBooleanType()) {
-        CK = CK_IntegralToBoolean;
-      } else if (type->isObjCObjectPointerType()) {
-        if (SubExpr->getType()->isBlockPointerType()) {
-          CK = CK_BlockPointerToObjCPointerCast;
-        } else if (SubExpr->getType()->isPointerType()) {
-          CK = CK_CPointerToObjCPointerCast;
-        } else {
-          CK = CK_BitCast;
-        }
-      } else {
-        CK = CK_BitCast;
-      }
-
-      userExpr = NoTypeInfoCStyleCastExpr(Context, type, CK, userExpr);
-    }
-    // Make id<P...> cast into an 'id' cast.
-    else if (CStyleCastExpr *CE = dyn_cast<CStyleCastExpr>(userExpr)) {
-      if (CE->getType()->isObjCQualifiedIdType()) {
-        while ((CE = dyn_cast<CStyleCastExpr>(userExpr)))
-          userExpr = CE->getSubExpr();
-        CastKind CK;
-        if (userExpr->getType()->isIntegralType(*Context)) {
-          CK = CK_IntegralToPointer;
-        } else if (userExpr->getType()->isBlockPointerType()) {
-          CK = CK_BlockPointerToObjCPointerCast;
-        } else if (userExpr->getType()->isPointerType()) {
-          CK = CK_CPointerToObjCPointerCast;
-        } else {
-          CK = CK_BitCast;
-        }
-        userExpr = NoTypeInfoCStyleCastExpr(Context, Context->getObjCIdType(),
-                                            CK, userExpr);
-      }
-    }
-    MsgExprs.push_back(userExpr);
-    // We've transferred the ownership to MsgExprs. For now, we *don't* null
-    // out the argument in the original expression (since we aren't deleting
-    // the ObjCMessageExpr). See RewritePropertyOrImplicitSetter() usage for more info.
-    //Exp->setArg(i, 0);
-  }
-  // Generate the funky cast.
-  CastExpr *cast;
-  SmallVector<QualType, 8> ArgTypes;
-  QualType returnType;
-
-  // Push 'id' and 'SEL', the 2 implicit arguments.
-  if (MsgSendFlavor == MsgSendSuperFunctionDecl)
-    ArgTypes.push_back(Context->getPointerType(getSuperStructType()));
-  else
-    ArgTypes.push_back(Context->getObjCIdType());
-  ArgTypes.push_back(Context->getObjCSelType());
-  if (ObjCMethodDecl *OMD = Exp->getMethodDecl()) {
-    // Push any user argument types.
-    for (ObjCMethodDecl::param_iterator PI = OMD->param_begin(),
-         E = OMD->param_end(); PI != E; ++PI) {
-      QualType t = (*PI)->getType()->isObjCQualifiedIdType()
-                     ? Context->getObjCIdType()
-                     : (*PI)->getType();
-      // Make sure we convert "t (^)(...)" to "t (*)(...)".
-      (void)convertBlockPointerToFunctionPointer(t);
-      ArgTypes.push_back(t);
-    }
-    returnType = Exp->getType();
-    convertToUnqualifiedObjCType(returnType);
-    (void)convertBlockPointerToFunctionPointer(returnType);
-  } else {
-    returnType = Context->getObjCIdType();
-  }
-  // Get the type, we will need to reference it in a couple spots.
-  QualType msgSendType = MsgSendFlavor->getType();
-
-  // Create a reference to the objc_msgSend() declaration.
-  DeclRefExpr *DRE = new (Context) DeclRefExpr(MsgSendFlavor, false, msgSendType,
-                                               VK_LValue, SourceLocation());
-
-  // Need to cast objc_msgSend to "void *" (to workaround a GCC bandaid).
-  // If we don't do this cast, we get the following bizarre warning/note:
-  // xx.m:13: warning: function called through a non-compatible type
-  // xx.m:13: note: if this code is reached, the program will abort
-  cast = NoTypeInfoCStyleCastExpr(Context,
-                                  Context->getPointerType(Context->VoidTy),
-                                  CK_BitCast, DRE);
-
-  // Now do the "normal" pointer to function cast.
-  QualType castType =
-    getSimpleFunctionType(returnType, &ArgTypes[0], ArgTypes.size(),
-      // If we don't have a method decl, force a variadic cast.
-      Exp->getMethodDecl() ? Exp->getMethodDecl()->isVariadic() : true);
-  castType = Context->getPointerType(castType);
-  cast = NoTypeInfoCStyleCastExpr(Context, castType, CK_BitCast,
-                                  cast);
-
-  // Don't forget the parens to enforce the proper binding.
-  ParenExpr *PE = new (Context) ParenExpr(StartLoc, EndLoc, cast);
-
-  const FunctionType *FT = msgSendType->getAs<FunctionType>();
-  CallExpr *CE = new (Context) CallExpr(*Context, PE, &MsgExprs[0],
-                                        MsgExprs.size(),
-                                        FT->getResultType(), VK_RValue,
-                                        EndLoc);
-  Stmt *ReplacingStmt = CE;
-  if (MsgSendStretFlavor) {
-    // We have the method which returns a struct/union. Must also generate
-    // call to objc_msgSend_stret and hang both varieties on a conditional
-    // expression which dictate which one to envoke depending on size of
-    // method's return type.
-
-    Expr *STCE = SynthMsgSendStretCallExpr(MsgSendStretFlavor, 
-                                           msgSendType, returnType, 
-                                           ArgTypes, MsgExprs,
-                                           Exp->getMethodDecl());
-
-    // Build sizeof(returnType)
-    UnaryExprOrTypeTraitExpr *sizeofExpr =
-       new (Context) UnaryExprOrTypeTraitExpr(UETT_SizeOf,
-                                 Context->getTrivialTypeSourceInfo(returnType),
-                                 Context->getSizeType(), SourceLocation(),
-                                 SourceLocation());
-    // (sizeof(returnType) <= 8 ? objc_msgSend(...) : objc_msgSend_stret(...))
-    // FIXME: Value of 8 is base on ppc32/x86 ABI for the most common cases.
-    // For X86 it is more complicated and some kind of target specific routine
-    // is needed to decide what to do.
-    unsigned IntSize =
-      static_cast<unsigned>(Context->getTypeSize(Context->IntTy));
-    IntegerLiteral *limit = IntegerLiteral::Create(*Context,
-                                                   llvm::APInt(IntSize, 8),
-                                                   Context->IntTy,
-                                                   SourceLocation());
-    BinaryOperator *lessThanExpr = 
-      new (Context) BinaryOperator(sizeofExpr, limit, BO_LE, Context->IntTy,
-                                   VK_RValue, OK_Ordinary, SourceLocation());
-    // (sizeof(returnType) <= 8 ? objc_msgSend(...) : objc_msgSend_stret(...))
-    ConditionalOperator *CondExpr =
-      new (Context) ConditionalOperator(lessThanExpr,
-                                        SourceLocation(), CE,
-                                        SourceLocation(), STCE,
-                                        returnType, VK_RValue, OK_Ordinary);
-    ReplacingStmt = new (Context) ParenExpr(SourceLocation(), SourceLocation(), 
-                                            CondExpr);
-  }
-  // delete Exp; leak for now, see RewritePropertyOrImplicitSetter() usage for more info.
-  return ReplacingStmt;
-}
-
-Stmt *RewriteModernObjC::RewriteMessageExpr(ObjCMessageExpr *Exp) {
-  Stmt *ReplacingStmt = SynthMessageExpr(Exp, Exp->getLocStart(),
-                                         Exp->getLocEnd());
-
-  // Now do the actual rewrite.
-  ReplaceStmt(Exp, ReplacingStmt);
-
-  // delete Exp; leak for now, see RewritePropertyOrImplicitSetter() usage for more info.
-  return ReplacingStmt;
-}
-
-// typedef struct objc_object Protocol;
-QualType RewriteModernObjC::getProtocolType() {
-  if (!ProtocolTypeDecl) {
-    TypeSourceInfo *TInfo
-      = Context->getTrivialTypeSourceInfo(Context->getObjCIdType());
-    ProtocolTypeDecl = TypedefDecl::Create(*Context, TUDecl,
-                                           SourceLocation(), SourceLocation(),
-                                           &Context->Idents.get("Protocol"),
-                                           TInfo);
-  }
-  return Context->getTypeDeclType(ProtocolTypeDecl);
-}
-
-/// RewriteObjCProtocolExpr - Rewrite a protocol expression into
-/// a synthesized/forward data reference (to the protocol's metadata).
-/// The forward references (and metadata) are generated in
-/// RewriteModernObjC::HandleTranslationUnit().
-Stmt *RewriteModernObjC::RewriteObjCProtocolExpr(ObjCProtocolExpr *Exp) {
-  std::string Name = "_OBJC_PROTOCOL_REFERENCE_$_" + 
-                      Exp->getProtocol()->getNameAsString();
-  IdentifierInfo *ID = &Context->Idents.get(Name);
-  VarDecl *VD = VarDecl::Create(*Context, TUDecl, SourceLocation(),
-                                SourceLocation(), ID, getProtocolType(), 0,
-                                SC_Extern, SC_None);
-  DeclRefExpr *DRE = new (Context) DeclRefExpr(VD, false, getProtocolType(),
-                                               VK_LValue, SourceLocation());
-  Expr *DerefExpr = new (Context) UnaryOperator(DRE, UO_AddrOf,
-                             Context->getPointerType(DRE->getType()),
-                             VK_RValue, OK_Ordinary, SourceLocation());
-  CastExpr *castExpr = NoTypeInfoCStyleCastExpr(Context, DerefExpr->getType(),
-                                                CK_BitCast,
-                                                DerefExpr);
-  ReplaceStmt(Exp, castExpr);
-  ProtocolExprDecls.insert(Exp->getProtocol()->getCanonicalDecl());
-  // delete Exp; leak for now, see RewritePropertyOrImplicitSetter() usage for more info.
-  return castExpr;
-
-}
-
-bool RewriteModernObjC::BufferContainsPPDirectives(const char *startBuf,
-                                             const char *endBuf) {
-  while (startBuf < endBuf) {
-    if (*startBuf == '#') {
-      // Skip whitespace.
-      for (++startBuf; startBuf[0] == ' ' || startBuf[0] == '\t'; ++startBuf)
-        ;
-      if (!strncmp(startBuf, "if", strlen("if")) ||
-          !strncmp(startBuf, "ifdef", strlen("ifdef")) ||
-          !strncmp(startBuf, "ifndef", strlen("ifndef")) ||
-          !strncmp(startBuf, "define", strlen("define")) ||
-          !strncmp(startBuf, "undef", strlen("undef")) ||
-          !strncmp(startBuf, "else", strlen("else")) ||
-          !strncmp(startBuf, "elif", strlen("elif")) ||
-          !strncmp(startBuf, "endif", strlen("endif")) ||
-          !strncmp(startBuf, "pragma", strlen("pragma")) ||
-          !strncmp(startBuf, "include", strlen("include")) ||
-          !strncmp(startBuf, "import", strlen("import")) ||
-          !strncmp(startBuf, "include_next", strlen("include_next")))
-        return true;
-    }
-    startBuf++;
-  }
-  return false;
-}
-
-/// IsTagDefinedInsideClass - This routine checks that a named tagged type 
-/// is defined inside an objective-c class. If so, it returns true. 
-bool RewriteModernObjC::IsTagDefinedInsideClass(ObjCContainerDecl *IDecl, 
-                                                TagDecl *Tag,
-                                                bool &IsNamedDefinition) {
-  if (!IDecl)
-    return false;
-  SourceLocation TagLocation;
-  if (RecordDecl *RD = dyn_cast<RecordDecl>(Tag)) {
-    RD = RD->getDefinition();
-    if (!RD || !RD->getDeclName().getAsIdentifierInfo())
-      return false;
-    IsNamedDefinition = true;
-    TagLocation = RD->getLocation();
-    return Context->getSourceManager().isBeforeInTranslationUnit(
-                                          IDecl->getLocation(), TagLocation);
-  }
-  if (EnumDecl *ED = dyn_cast<EnumDecl>(Tag)) {
-    if (!ED || !ED->getDeclName().getAsIdentifierInfo())
-      return false;
-    IsNamedDefinition = true;
-    TagLocation = ED->getLocation();
-    return Context->getSourceManager().isBeforeInTranslationUnit(
-                                          IDecl->getLocation(), TagLocation);
-
-  }
-  return false;
-}
-
-/// RewriteObjCFieldDeclType - This routine rewrites a type into the buffer.
-/// It handles elaborated types, as well as enum types in the process.
-bool RewriteModernObjC::RewriteObjCFieldDeclType(QualType &Type, 
-                                                 std::string &Result) {
-  if (isa<TypedefType>(Type)) {
-    Result += "\t";
-    return false;
-  }
-    
-  if (Type->isArrayType()) {
-    QualType ElemTy = Context->getBaseElementType(Type);
-    return RewriteObjCFieldDeclType(ElemTy, Result);
-  }
-  else if (Type->isRecordType()) {
-    RecordDecl *RD = Type->getAs<RecordType>()->getDecl();
-    if (RD->isCompleteDefinition()) {
-      if (RD->isStruct())
-        Result += "\n\tstruct ";
-      else if (RD->isUnion())
-        Result += "\n\tunion ";
-      else
-        assert(false && "class not allowed as an ivar type");
-      
-      Result += RD->getName();
-      if (GlobalDefinedTags.count(RD)) {
-        // struct/union is defined globally, use it.
-        Result += " ";
-        return true;
-      }
-      Result += " {\n";
-      for (RecordDecl::field_iterator i = RD->field_begin(), 
-           e = RD->field_end(); i != e; ++i) {
-        FieldDecl *FD = *i;
-        RewriteObjCFieldDecl(FD, Result);
-      }
-      Result += "\t} "; 
-      return true;
-    }
-  }
-  else if (Type->isEnumeralType()) {
-    EnumDecl *ED = Type->getAs<EnumType>()->getDecl();
-    if (ED->isCompleteDefinition()) {
-      Result += "\n\tenum ";
-      Result += ED->getName();
-      if (GlobalDefinedTags.count(ED)) {
-        // Enum is globall defined, use it.
-        Result += " ";
-        return true;
-      }
-      
-      Result += " {\n";
-      for (EnumDecl::enumerator_iterator EC = ED->enumerator_begin(),
-           ECEnd = ED->enumerator_end(); EC != ECEnd; ++EC) {
-        Result += "\t"; Result += EC->getName(); Result += " = ";
-        llvm::APSInt Val = EC->getInitVal();
-        Result += Val.toString(10);
-        Result += ",\n";
-      }
-      Result += "\t} "; 
-      return true;
-    }
-  }
-  
-  Result += "\t";
-  convertObjCTypeToCStyleType(Type);
-  return false;
-}
-
-
-/// RewriteObjCFieldDecl - This routine rewrites a field into the buffer.
-/// It handles elaborated types, as well as enum types in the process.
-void RewriteModernObjC::RewriteObjCFieldDecl(FieldDecl *fieldDecl, 
-                                             std::string &Result) {
-  QualType Type = fieldDecl->getType();
-  std::string Name = fieldDecl->getNameAsString();
-  
-  bool EleboratedType = RewriteObjCFieldDeclType(Type, Result); 
-  if (!EleboratedType)
-    Type.getAsStringInternal(Name, Context->getPrintingPolicy());
-  Result += Name;
-  if (fieldDecl->isBitField()) {
-    Result += " : "; Result += utostr(fieldDecl->getBitWidthValue(*Context));
-  }
-  else if (EleboratedType && Type->isArrayType()) {
-    CanQualType CType = Context->getCanonicalType(Type);
-    while (isa<ArrayType>(CType)) {
-      if (const ConstantArrayType *CAT = Context->getAsConstantArrayType(CType)) {
-        Result += "[";
-        llvm::APInt Dim = CAT->getSize();
-        Result += utostr(Dim.getZExtValue());
-        Result += "]";
-      }
-      CType = CType->getAs<ArrayType>()->getElementType();
-    }
-  }
-  
-  Result += ";\n";
-}
-
-/// RewriteLocallyDefinedNamedAggregates - This routine rewrites locally defined
-/// named aggregate types into the input buffer.
-void RewriteModernObjC::RewriteLocallyDefinedNamedAggregates(FieldDecl *fieldDecl, 
-                                             std::string &Result) {
-  QualType Type = fieldDecl->getType();
-  if (isa<TypedefType>(Type))
-    return;
-  if (Type->isArrayType())
-    Type = Context->getBaseElementType(Type);
-  ObjCContainerDecl *IDecl = 
-    dyn_cast<ObjCContainerDecl>(fieldDecl->getDeclContext());
-  
-  TagDecl *TD = 0;
-  if (Type->isRecordType()) {
-    TD = Type->getAs<RecordType>()->getDecl();
-  }
-  else if (Type->isEnumeralType()) {
-    TD = Type->getAs<EnumType>()->getDecl();
-  }
-  
-  if (TD) {
-    if (GlobalDefinedTags.count(TD))
-      return;
-    
-    bool IsNamedDefinition = false;
-    if (IsTagDefinedInsideClass(IDecl, TD, IsNamedDefinition)) {
-      RewriteObjCFieldDeclType(Type, Result);
-      Result += ";";
-    }
-    if (IsNamedDefinition)
-      GlobalDefinedTags.insert(TD);
-  }
-    
-}
-
-/// RewriteObjCInternalStruct - Rewrite one internal struct corresponding to
-/// an objective-c class with ivars.
-void RewriteModernObjC::RewriteObjCInternalStruct(ObjCInterfaceDecl *CDecl,
-                                               std::string &Result) {
-  assert(CDecl && "Class missing in SynthesizeObjCInternalStruct");
-  assert(CDecl->getName() != "" &&
-         "Name missing in SynthesizeObjCInternalStruct");
-  ObjCInterfaceDecl *RCDecl = CDecl->getSuperClass();
-  SmallVector<ObjCIvarDecl *, 8> IVars;
-  for (ObjCIvarDecl *IVD = CDecl->all_declared_ivar_begin();
-       IVD; IVD = IVD->getNextIvar())
-    IVars.push_back(IVD);
-  
-  SourceLocation LocStart = CDecl->getLocStart();
-  SourceLocation LocEnd = CDecl->getEndOfDefinitionLoc();
-  
-  const char *startBuf = SM->getCharacterData(LocStart);
-  const char *endBuf = SM->getCharacterData(LocEnd);
-  
-  // If no ivars and no root or if its root, directly or indirectly,
-  // have no ivars (thus not synthesized) then no need to synthesize this class.
-  if ((!CDecl->isThisDeclarationADefinition() || IVars.size() == 0) &&
-      (!RCDecl || !ObjCSynthesizedStructs.count(RCDecl))) {
-    endBuf += Lexer::MeasureTokenLength(LocEnd, *SM, LangOpts);
-    ReplaceText(LocStart, endBuf-startBuf, Result);
-    return;
-  }
-  
-  // Insert named struct/union definitions inside class to
-  // outer scope. This follows semantics of locally defined
-  // struct/unions in objective-c classes.
-  for (unsigned i = 0, e = IVars.size(); i < e; i++)
-    RewriteLocallyDefinedNamedAggregates(IVars[i], Result);
-
-  Result += "\nstruct ";
-  Result += CDecl->getNameAsString();
-  Result += "_IMPL {\n";
-  
-  if (RCDecl && ObjCSynthesizedStructs.count(RCDecl)) {
-    Result += "\tstruct "; Result += RCDecl->getNameAsString();
-    Result += "_IMPL "; Result += RCDecl->getNameAsString();
-    Result += "_IVARS;\n";
-  }
-  
-  for (unsigned i = 0, e = IVars.size(); i < e; i++)
-    RewriteObjCFieldDecl(IVars[i], Result);
-
-  Result += "};\n";
-  endBuf += Lexer::MeasureTokenLength(LocEnd, *SM, LangOpts);
-  ReplaceText(LocStart, endBuf-startBuf, Result);
-  // Mark this struct as having been generated.
-  if (!ObjCSynthesizedStructs.insert(CDecl))
-    llvm_unreachable("struct already synthesize- RewriteObjCInternalStruct");
-}
-
-/// RewriteIvarOffsetSymbols - Rewrite ivar offset symbols of those ivars which
-/// have been referenced in an ivar access expression.
-void RewriteModernObjC::RewriteIvarOffsetSymbols(ObjCInterfaceDecl *CDecl,
-                                                  std::string &Result) {
-  // write out ivar offset symbols which have been referenced in an ivar
-  // access expression.
-  llvm::SmallPtrSet<ObjCIvarDecl *, 8> Ivars = ReferencedIvars[CDecl];
-  if (Ivars.empty())
-    return;
-  for (llvm::SmallPtrSet<ObjCIvarDecl *, 8>::iterator i = Ivars.begin(),
-       e = Ivars.end(); i != e; i++) {
-    ObjCIvarDecl *IvarDecl = (*i);
-    Result += "\n";
-    if (LangOpts.MicrosoftExt)
-      Result += "__declspec(allocate(\".objc_ivar$B\")) ";
-    Result += "extern \"C\" ";
-    if (LangOpts.MicrosoftExt && 
-        IvarDecl->getAccessControl() != ObjCIvarDecl::Private &&
-        IvarDecl->getAccessControl() != ObjCIvarDecl::Package)
-        Result += "__declspec(dllimport) ";
-
-    Result += "unsigned long ";
-    WriteInternalIvarName(CDecl, IvarDecl, Result);
-    Result += ";";
-  }
-}
-
-//===----------------------------------------------------------------------===//
-// Meta Data Emission
-//===----------------------------------------------------------------------===//
-
-
-/// RewriteImplementations - This routine rewrites all method implementations
-/// and emits meta-data.
-
-void RewriteModernObjC::RewriteImplementations() {
-  int ClsDefCount = ClassImplementation.size();
-  int CatDefCount = CategoryImplementation.size();
-
-  // Rewrite implemented methods
-  for (int i = 0; i < ClsDefCount; i++) {
-    ObjCImplementationDecl *OIMP = ClassImplementation[i];
-    ObjCInterfaceDecl *CDecl = OIMP->getClassInterface();
-    if (CDecl->isImplicitInterfaceDecl())
-      assert(false &&
-             "Legacy implicit interface rewriting not supported in moder abi");
-    RewriteImplementationDecl(OIMP);
-  }
-
-  for (int i = 0; i < CatDefCount; i++) {
-    ObjCCategoryImplDecl *CIMP = CategoryImplementation[i];
-    ObjCInterfaceDecl *CDecl = CIMP->getClassInterface();
-    if (CDecl->isImplicitInterfaceDecl())
-      assert(false &&
-             "Legacy implicit interface rewriting not supported in moder abi");
-    RewriteImplementationDecl(CIMP);
-  }
-}
-
-void RewriteModernObjC::RewriteByRefString(std::string &ResultStr, 
-                                     const std::string &Name,
-                                     ValueDecl *VD, bool def) {
-  assert(BlockByRefDeclNo.count(VD) && 
-         "RewriteByRefString: ByRef decl missing");
-  if (def)
-    ResultStr += "struct ";
-  ResultStr += "__Block_byref_" + Name + 
-    "_" + utostr(BlockByRefDeclNo[VD]) ;
-}
-
-static bool HasLocalVariableExternalStorage(ValueDecl *VD) {
-  if (VarDecl *Var = dyn_cast<VarDecl>(VD))
-    return (Var->isFunctionOrMethodVarDecl() && !Var->hasLocalStorage());
-  return false;
-}
-
-std::string RewriteModernObjC::SynthesizeBlockFunc(BlockExpr *CE, int i,
-                                                   StringRef funcName,
-                                                   std::string Tag) {
-  const FunctionType *AFT = CE->getFunctionType();
-  QualType RT = AFT->getResultType();
-  std::string StructRef = "struct " + Tag;
-  std::string S = "static " + RT.getAsString(Context->getPrintingPolicy()) + " __" +
-                  funcName.str() + "_block_func_" + utostr(i);
-
-  BlockDecl *BD = CE->getBlockDecl();
-
-  if (isa<FunctionNoProtoType>(AFT)) {
-    // No user-supplied arguments. Still need to pass in a pointer to the
-    // block (to reference imported block decl refs).
-    S += "(" + StructRef + " *__cself)";
-  } else if (BD->param_empty()) {
-    S += "(" + StructRef + " *__cself)";
-  } else {
-    const FunctionProtoType *FT = cast<FunctionProtoType>(AFT);
-    assert(FT && "SynthesizeBlockFunc: No function proto");
-    S += '(';
-    // first add the implicit argument.
-    S += StructRef + " *__cself, ";
-    std::string ParamStr;
-    for (BlockDecl::param_iterator AI = BD->param_begin(),
-         E = BD->param_end(); AI != E; ++AI) {
-      if (AI != BD->param_begin()) S += ", ";
-      ParamStr = (*AI)->getNameAsString();
-      QualType QT = (*AI)->getType();
-      (void)convertBlockPointerToFunctionPointer(QT);
-      QT.getAsStringInternal(ParamStr, Context->getPrintingPolicy());
-      S += ParamStr;
-    }
-    if (FT->isVariadic()) {
-      if (!BD->param_empty()) S += ", ";
-      S += "...";
-    }
-    S += ')';
-  }
-  S += " {\n";
-
-  // Create local declarations to avoid rewriting all closure decl ref exprs.
-  // First, emit a declaration for all "by ref" decls.
-  for (SmallVector<ValueDecl*,8>::iterator I = BlockByRefDecls.begin(),
-       E = BlockByRefDecls.end(); I != E; ++I) {
-    S += "  ";
-    std::string Name = (*I)->getNameAsString();
-    std::string TypeString;
-    RewriteByRefString(TypeString, Name, (*I));
-    TypeString += " *";
-    Name = TypeString + Name;
-    S += Name + " = __cself->" + (*I)->getNameAsString() + "; // bound by ref\n";
-  }
-  // Next, emit a declaration for all "by copy" declarations.
-  for (SmallVector<ValueDecl*,8>::iterator I = BlockByCopyDecls.begin(),
-       E = BlockByCopyDecls.end(); I != E; ++I) {
-    S += "  ";
-    // Handle nested closure invocation. For example:
-    //
-    //   void (^myImportedClosure)(void);
-    //   myImportedClosure  = ^(void) { setGlobalInt(x + y); };
-    //
-    //   void (^anotherClosure)(void);
-    //   anotherClosure = ^(void) {
-    //     myImportedClosure(); // import and invoke the closure
-    //   };
-    //
-    if (isTopLevelBlockPointerType((*I)->getType())) {
-      RewriteBlockPointerTypeVariable(S, (*I));
-      S += " = (";
-      RewriteBlockPointerType(S, (*I)->getType());
-      S += ")";
-      S += "__cself->" + (*I)->getNameAsString() + "; // bound by copy\n";
-    }
-    else {
-      std::string Name = (*I)->getNameAsString();
-      QualType QT = (*I)->getType();
-      if (HasLocalVariableExternalStorage(*I))
-        QT = Context->getPointerType(QT);
-      QT.getAsStringInternal(Name, Context->getPrintingPolicy());
-      S += Name + " = __cself->" + 
-                              (*I)->getNameAsString() + "; // bound by copy\n";
-    }
-  }
-  std::string RewrittenStr = RewrittenBlockExprs[CE];
-  const char *cstr = RewrittenStr.c_str();
-  while (*cstr++ != '{') ;
-  S += cstr;
-  S += "\n";
-  return S;
-}
-
-std::string RewriteModernObjC::SynthesizeBlockHelperFuncs(BlockExpr *CE, int i,
-                                                   StringRef funcName,
-                                                   std::string Tag) {
-  std::string StructRef = "struct " + Tag;
-  std::string S = "static void __";
-
-  S += funcName;
-  S += "_block_copy_" + utostr(i);
-  S += "(" + StructRef;
-  S += "*dst, " + StructRef;
-  S += "*src) {";
-  for (llvm::SmallPtrSet<ValueDecl*,8>::iterator I = ImportedBlockDecls.begin(),
-      E = ImportedBlockDecls.end(); I != E; ++I) {
-    ValueDecl *VD = (*I);
-    S += "_Block_object_assign((void*)&dst->";
-    S += (*I)->getNameAsString();
-    S += ", (void*)src->";
-    S += (*I)->getNameAsString();
-    if (BlockByRefDeclsPtrSet.count((*I)))
-      S += ", " + utostr(BLOCK_FIELD_IS_BYREF) + "/*BLOCK_FIELD_IS_BYREF*/);";
-    else if (VD->getType()->isBlockPointerType())
-      S += ", " + utostr(BLOCK_FIELD_IS_BLOCK) + "/*BLOCK_FIELD_IS_BLOCK*/);";
-    else
-      S += ", " + utostr(BLOCK_FIELD_IS_OBJECT) + "/*BLOCK_FIELD_IS_OBJECT*/);";
-  }
-  S += "}\n";
-  
-  S += "\nstatic void __";
-  S += funcName;
-  S += "_block_dispose_" + utostr(i);
-  S += "(" + StructRef;
-  S += "*src) {";
-  for (llvm::SmallPtrSet<ValueDecl*,8>::iterator I = ImportedBlockDecls.begin(),
-      E = ImportedBlockDecls.end(); I != E; ++I) {
-    ValueDecl *VD = (*I);
-    S += "_Block_object_dispose((void*)src->";
-    S += (*I)->getNameAsString();
-    if (BlockByRefDeclsPtrSet.count((*I)))
-      S += ", " + utostr(BLOCK_FIELD_IS_BYREF) + "/*BLOCK_FIELD_IS_BYREF*/);";
-    else if (VD->getType()->isBlockPointerType())
-      S += ", " + utostr(BLOCK_FIELD_IS_BLOCK) + "/*BLOCK_FIELD_IS_BLOCK*/);";
-    else
-      S += ", " + utostr(BLOCK_FIELD_IS_OBJECT) + "/*BLOCK_FIELD_IS_OBJECT*/);";
-  }
-  S += "}\n";
-  return S;
-}
-
-std::string RewriteModernObjC::SynthesizeBlockImpl(BlockExpr *CE, std::string Tag, 
-                                             std::string Desc) {
-  std::string S = "\nstruct " + Tag;
-  std::string Constructor = "  " + Tag;
-
-  S += " {\n  struct __block_impl impl;\n";
-  S += "  struct " + Desc;
-  S += "* Desc;\n";
-
-  Constructor += "(void *fp, "; // Invoke function pointer.
-  Constructor += "struct " + Desc; // Descriptor pointer.
-  Constructor += " *desc";
-
-  if (BlockDeclRefs.size()) {
-    // Output all "by copy" declarations.
-    for (SmallVector<ValueDecl*,8>::iterator I = BlockByCopyDecls.begin(),
-         E = BlockByCopyDecls.end(); I != E; ++I) {
-      S += "  ";
-      std::string FieldName = (*I)->getNameAsString();
-      std::string ArgName = "_" + FieldName;
-      // Handle nested closure invocation. For example:
-      //
-      //   void (^myImportedBlock)(void);
-      //   myImportedBlock  = ^(void) { setGlobalInt(x + y); };
-      //
-      //   void (^anotherBlock)(void);
-      //   anotherBlock = ^(void) {
-      //     myImportedBlock(); // import and invoke the closure
-      //   };
-      //
-      if (isTopLevelBlockPointerType((*I)->getType())) {
-        S += "struct __block_impl *";
-        Constructor += ", void *" + ArgName;
-      } else {
-        QualType QT = (*I)->getType();
-        if (HasLocalVariableExternalStorage(*I))
-          QT = Context->getPointerType(QT);
-        QT.getAsStringInternal(FieldName, Context->getPrintingPolicy());
-        QT.getAsStringInternal(ArgName, Context->getPrintingPolicy());
-        Constructor += ", " + ArgName;
-      }
-      S += FieldName + ";\n";
-    }
-    // Output all "by ref" declarations.
-    for (SmallVector<ValueDecl*,8>::iterator I = BlockByRefDecls.begin(),
-         E = BlockByRefDecls.end(); I != E; ++I) {
-      S += "  ";
-      std::string FieldName = (*I)->getNameAsString();
-      std::string ArgName = "_" + FieldName;
-      {
-        std::string TypeString;
-        RewriteByRefString(TypeString, FieldName, (*I));
-        TypeString += " *";
-        FieldName = TypeString + FieldName;
-        ArgName = TypeString + ArgName;
-        Constructor += ", " + ArgName;
-      }
-      S += FieldName + "; // by ref\n";
-    }
-    // Finish writing the constructor.
-    Constructor += ", int flags=0)";
-    // Initialize all "by copy" arguments.
-    bool firsTime = true;
-    for (SmallVector<ValueDecl*,8>::iterator I = BlockByCopyDecls.begin(),
-         E = BlockByCopyDecls.end(); I != E; ++I) {
-      std::string Name = (*I)->getNameAsString();
-        if (firsTime) {
-          Constructor += " : ";
-          firsTime = false;
-        }
-        else
-          Constructor += ", ";
-        if (isTopLevelBlockPointerType((*I)->getType()))
-          Constructor += Name + "((struct __block_impl *)_" + Name + ")";
-        else
-          Constructor += Name + "(_" + Name + ")";
-    }
-    // Initialize all "by ref" arguments.
-    for (SmallVector<ValueDecl*,8>::iterator I = BlockByRefDecls.begin(),
-         E = BlockByRefDecls.end(); I != E; ++I) {
-      std::string Name = (*I)->getNameAsString();
-      if (firsTime) {
-        Constructor += " : ";
-        firsTime = false;
-      }
-      else
-        Constructor += ", ";
-      Constructor += Name + "(_" + Name + "->__forwarding)";
-    }
-    
-    Constructor += " {\n";
-    if (GlobalVarDecl)
-      Constructor += "    impl.isa = &_NSConcreteGlobalBlock;\n";
-    else
-      Constructor += "    impl.isa = &_NSConcreteStackBlock;\n";
-    Constructor += "    impl.Flags = flags;\n    impl.FuncPtr = fp;\n";
-
-    Constructor += "    Desc = desc;\n";
-  } else {
-    // Finish writing the constructor.
-    Constructor += ", int flags=0) {\n";
-    if (GlobalVarDecl)
-      Constructor += "    impl.isa = &_NSConcreteGlobalBlock;\n";
-    else
-      Constructor += "    impl.isa = &_NSConcreteStackBlock;\n";
-    Constructor += "    impl.Flags = flags;\n    impl.FuncPtr = fp;\n";
-    Constructor += "    Desc = desc;\n";
-  }
-  Constructor += "  ";
-  Constructor += "}\n";
-  S += Constructor;
-  S += "};\n";
-  return S;
-}
-
-std::string RewriteModernObjC::SynthesizeBlockDescriptor(std::string DescTag, 
-                                                   std::string ImplTag, int i,
-                                                   StringRef FunName,
-                                                   unsigned hasCopy) {
-  std::string S = "\nstatic struct " + DescTag;
-  
-  S += " {\n  size_t reserved;\n";
-  S += "  size_t Block_size;\n";
-  if (hasCopy) {
-    S += "  void (*copy)(struct ";
-    S += ImplTag; S += "*, struct ";
-    S += ImplTag; S += "*);\n";
-    
-    S += "  void (*dispose)(struct ";
-    S += ImplTag; S += "*);\n";
-  }
-  S += "} ";
-
-  S += DescTag + "_DATA = { 0, sizeof(struct ";
-  S += ImplTag + ")";
-  if (hasCopy) {
-    S += ", __" + FunName.str() + "_block_copy_" + utostr(i);
-    S += ", __" + FunName.str() + "_block_dispose_" + utostr(i);
-  }
-  S += "};\n";
-  return S;
-}
-
-void RewriteModernObjC::SynthesizeBlockLiterals(SourceLocation FunLocStart,
-                                          StringRef FunName) {
-  bool RewriteSC = (GlobalVarDecl &&
-                    !Blocks.empty() &&
-                    GlobalVarDecl->getStorageClass() == SC_Static &&
-                    GlobalVarDecl->getType().getCVRQualifiers());
-  if (RewriteSC) {
-    std::string SC(" void __");
-    SC += GlobalVarDecl->getNameAsString();
-    SC += "() {}";
-    InsertText(FunLocStart, SC);
-  }
-  
-  // Insert closures that were part of the function.
-  for (unsigned i = 0, count=0; i < Blocks.size(); i++) {
-    CollectBlockDeclRefInfo(Blocks[i]);
-    // Need to copy-in the inner copied-in variables not actually used in this
-    // block.
-    for (int j = 0; j < InnerDeclRefsCount[i]; j++) {
-      DeclRefExpr *Exp = InnerDeclRefs[count++];
-      ValueDecl *VD = Exp->getDecl();
-      BlockDeclRefs.push_back(Exp);
-      if (!VD->hasAttr<BlocksAttr>()) {
-        if (!BlockByCopyDeclsPtrSet.count(VD)) {
-          BlockByCopyDeclsPtrSet.insert(VD);
-          BlockByCopyDecls.push_back(VD);
-        }
-        continue;
-      }
-
-      if (!BlockByRefDeclsPtrSet.count(VD)) {
-        BlockByRefDeclsPtrSet.insert(VD);
-        BlockByRefDecls.push_back(VD);
-      }
-
-      // imported objects in the inner blocks not used in the outer
-      // blocks must be copied/disposed in the outer block as well.
-      if (VD->getType()->isObjCObjectPointerType() || 
-          VD->getType()->isBlockPointerType())
-        ImportedBlockDecls.insert(VD);
-    }
-
-    std::string ImplTag = "__" + FunName.str() + "_block_impl_" + utostr(i);
-    std::string DescTag = "__" + FunName.str() + "_block_desc_" + utostr(i);
-
-    std::string CI = SynthesizeBlockImpl(Blocks[i], ImplTag, DescTag);
-
-    InsertText(FunLocStart, CI);
-
-    std::string CF = SynthesizeBlockFunc(Blocks[i], i, FunName, ImplTag);
-
-    InsertText(FunLocStart, CF);
-
-    if (ImportedBlockDecls.size()) {
-      std::string HF = SynthesizeBlockHelperFuncs(Blocks[i], i, FunName, ImplTag);
-      InsertText(FunLocStart, HF);
-    }
-    std::string BD = SynthesizeBlockDescriptor(DescTag, ImplTag, i, FunName,
-                                               ImportedBlockDecls.size() > 0);
-    InsertText(FunLocStart, BD);
-
-    BlockDeclRefs.clear();
-    BlockByRefDecls.clear();
-    BlockByRefDeclsPtrSet.clear();
-    BlockByCopyDecls.clear();
-    BlockByCopyDeclsPtrSet.clear();
-    ImportedBlockDecls.clear();
-  }
-  if (RewriteSC) {
-    // Must insert any 'const/volatile/static here. Since it has been
-    // removed as result of rewriting of block literals.
-    std::string SC;
-    if (GlobalVarDecl->getStorageClass() == SC_Static)
-      SC = "static ";
-    if (GlobalVarDecl->getType().isConstQualified())
-      SC += "const ";
-    if (GlobalVarDecl->getType().isVolatileQualified())
-      SC += "volatile ";
-    if (GlobalVarDecl->getType().isRestrictQualified())
-      SC += "restrict ";
-    InsertText(FunLocStart, SC);
-  }
-  if (GlobalConstructionExp) {
-    // extra fancy dance for global literal expression.
-    
-    // Always the latest block expression on the block stack.
-    std::string Tag = "__";
-    Tag += FunName;
-    Tag += "_block_impl_";
-    Tag += utostr(Blocks.size()-1);
-    std::string globalBuf = "static ";
-    globalBuf += Tag; globalBuf += " ";
-    std::string SStr;
-  
-    llvm::raw_string_ostream constructorExprBuf(SStr);
-    GlobalConstructionExp->printPretty(constructorExprBuf, 0,
-                                         PrintingPolicy(LangOpts));
-    globalBuf += constructorExprBuf.str();
-    globalBuf += ";\n";
-    InsertText(FunLocStart, globalBuf);
-    GlobalConstructionExp = 0;
-  }
-
-  Blocks.clear();
-  InnerDeclRefsCount.clear();
-  InnerDeclRefs.clear();
-  RewrittenBlockExprs.clear();
-}
-
-void RewriteModernObjC::InsertBlockLiteralsWithinFunction(FunctionDecl *FD) {
-  SourceLocation FunLocStart = 
-    (!Blocks.empty()) ? getFunctionSourceLocation(*this, FD)
-                      : FD->getTypeSpecStartLoc();
-  StringRef FuncName = FD->getName();
-
-  SynthesizeBlockLiterals(FunLocStart, FuncName);
-}
-
-static void BuildUniqueMethodName(std::string &Name,
-                                  ObjCMethodDecl *MD) {
-  ObjCInterfaceDecl *IFace = MD->getClassInterface();
-  Name = IFace->getName();
-  Name += "__" + MD->getSelector().getAsString();
-  // Convert colons to underscores.
-  std::string::size_type loc = 0;
-  while ((loc = Name.find(":", loc)) != std::string::npos)
-    Name.replace(loc, 1, "_");
-}
-
-void RewriteModernObjC::InsertBlockLiteralsWithinMethod(ObjCMethodDecl *MD) {
-  //fprintf(stderr,"In InsertBlockLiteralsWitinMethod\n");
-  //SourceLocation FunLocStart = MD->getLocStart();
-  SourceLocation FunLocStart = MD->getLocStart();
-  std::string FuncName;
-  BuildUniqueMethodName(FuncName, MD);
-  SynthesizeBlockLiterals(FunLocStart, FuncName);
-}
-
-void RewriteModernObjC::GetBlockDeclRefExprs(Stmt *S) {
-  for (Stmt::child_range CI = S->children(); CI; ++CI)
-    if (*CI) {
-      if (BlockExpr *CBE = dyn_cast<BlockExpr>(*CI))
-        GetBlockDeclRefExprs(CBE->getBody());
-      else
-        GetBlockDeclRefExprs(*CI);
-    }
-  // Handle specific things.
-  if (DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(S)) {
-    if (DRE->refersToEnclosingLocal()) {
-      // FIXME: Handle enums.
-      if (!isa<FunctionDecl>(DRE->getDecl()))
-        BlockDeclRefs.push_back(DRE);
-      if (HasLocalVariableExternalStorage(DRE->getDecl()))
-        BlockDeclRefs.push_back(DRE);
-    }
-  }
-  
-  return;
-}
-
-void RewriteModernObjC::GetInnerBlockDeclRefExprs(Stmt *S, 
-                SmallVector<DeclRefExpr *, 8> &InnerBlockDeclRefs,
-                llvm::SmallPtrSet<const DeclContext *, 8> &InnerContexts) {
-  for (Stmt::child_range CI = S->children(); CI; ++CI)
-    if (*CI) {
-      if (BlockExpr *CBE = dyn_cast<BlockExpr>(*CI)) {
-        InnerContexts.insert(cast<DeclContext>(CBE->getBlockDecl()));
-        GetInnerBlockDeclRefExprs(CBE->getBody(),
-                                  InnerBlockDeclRefs,
-                                  InnerContexts);
-      }
-      else
-        GetInnerBlockDeclRefExprs(*CI,
-                                  InnerBlockDeclRefs,
-                                  InnerContexts);
-
-    }
-  // Handle specific things.
-  if (DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(S)) {
-    if (DRE->refersToEnclosingLocal()) {
-      if (!isa<FunctionDecl>(DRE->getDecl()) &&
-          !InnerContexts.count(DRE->getDecl()->getDeclContext()))
-        InnerBlockDeclRefs.push_back(DRE);
-      if (VarDecl *Var = dyn_cast<VarDecl>(DRE->getDecl()))
-        if (Var->isFunctionOrMethodVarDecl())
-          ImportedLocalExternalDecls.insert(Var);
-    }
-  }
-  
-  return;
-}
-
-/// convertObjCTypeToCStyleType - This routine converts such objc types
-/// as qualified objects, and blocks to their closest c/c++ types that
-/// it can. It returns true if input type was modified.
-bool RewriteModernObjC::convertObjCTypeToCStyleType(QualType &T) {
-  QualType oldT = T;
-  convertBlockPointerToFunctionPointer(T);
-  if (T->isFunctionPointerType()) {
-    QualType PointeeTy;
-    if (const PointerType* PT = T->getAs<PointerType>()) {
-      PointeeTy = PT->getPointeeType();
-      if (const FunctionType *FT = PointeeTy->getAs<FunctionType>()) {
-        T = convertFunctionTypeOfBlocks(FT);
-        T = Context->getPointerType(T);
-      }
-    }
-  }
-  
-  convertToUnqualifiedObjCType(T);
-  return T != oldT;
-}
-
-/// convertFunctionTypeOfBlocks - This routine converts a function type
-/// whose result type may be a block pointer or whose argument type(s)
-/// might be block pointers to an equivalent function type replacing
-/// all block pointers to function pointers.
-QualType RewriteModernObjC::convertFunctionTypeOfBlocks(const FunctionType *FT) {
-  const FunctionProtoType *FTP = dyn_cast<FunctionProtoType>(FT);
-  // FTP will be null for closures that don't take arguments.
-  // Generate a funky cast.
-  SmallVector<QualType, 8> ArgTypes;
-  QualType Res = FT->getResultType();
-  bool modified = convertObjCTypeToCStyleType(Res);
-  
-  if (FTP) {
-    for (FunctionProtoType::arg_type_iterator I = FTP->arg_type_begin(),
-         E = FTP->arg_type_end(); I && (I != E); ++I) {
-      QualType t = *I;
-      // Make sure we convert "t (^)(...)" to "t (*)(...)".
-      if (convertObjCTypeToCStyleType(t))
-        modified = true;
-      ArgTypes.push_back(t);
-    }
-  }
-  QualType FuncType;
-  if (modified)
-    FuncType = getSimpleFunctionType(Res, &ArgTypes[0], ArgTypes.size());
-  else FuncType = QualType(FT, 0);
-  return FuncType;
-}
-
-Stmt *RewriteModernObjC::SynthesizeBlockCall(CallExpr *Exp, const Expr *BlockExp) {
-  // Navigate to relevant type information.
-  const BlockPointerType *CPT = 0;
-
-  if (const DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(BlockExp)) {
-    CPT = DRE->getType()->getAs<BlockPointerType>();
-  } else if (const MemberExpr *MExpr = dyn_cast<MemberExpr>(BlockExp)) {
-    CPT = MExpr->getType()->getAs<BlockPointerType>();
-  } 
-  else if (const ParenExpr *PRE = dyn_cast<ParenExpr>(BlockExp)) {
-    return SynthesizeBlockCall(Exp, PRE->getSubExpr());
-  }
-  else if (const ImplicitCastExpr *IEXPR = dyn_cast<ImplicitCastExpr>(BlockExp)) 
-    CPT = IEXPR->getType()->getAs<BlockPointerType>();
-  else if (const ConditionalOperator *CEXPR = 
-            dyn_cast<ConditionalOperator>(BlockExp)) {
-    Expr *LHSExp = CEXPR->getLHS();
-    Stmt *LHSStmt = SynthesizeBlockCall(Exp, LHSExp);
-    Expr *RHSExp = CEXPR->getRHS();
-    Stmt *RHSStmt = SynthesizeBlockCall(Exp, RHSExp);
-    Expr *CONDExp = CEXPR->getCond();
-    ConditionalOperator *CondExpr =
-      new (Context) ConditionalOperator(CONDExp,
-                                      SourceLocation(), cast<Expr>(LHSStmt),
-                                      SourceLocation(), cast<Expr>(RHSStmt),
-                                      Exp->getType(), VK_RValue, OK_Ordinary);
-    return CondExpr;
-  } else if (const ObjCIvarRefExpr *IRE = dyn_cast<ObjCIvarRefExpr>(BlockExp)) {
-    CPT = IRE->getType()->getAs<BlockPointerType>();
-  } else if (const PseudoObjectExpr *POE
-               = dyn_cast<PseudoObjectExpr>(BlockExp)) {
-    CPT = POE->getType()->castAs<BlockPointerType>();
-  } else {
-    assert(1 && "RewriteBlockClass: Bad type");
-  }
-  assert(CPT && "RewriteBlockClass: Bad type");
-  const FunctionType *FT = CPT->getPointeeType()->getAs<FunctionType>();
-  assert(FT && "RewriteBlockClass: Bad type");
-  const FunctionProtoType *FTP = dyn_cast<FunctionProtoType>(FT);
-  // FTP will be null for closures that don't take arguments.
-
-  RecordDecl *RD = RecordDecl::Create(*Context, TTK_Struct, TUDecl,
-                                      SourceLocation(), SourceLocation(),
-                                      &Context->Idents.get("__block_impl"));
-  QualType PtrBlock = Context->getPointerType(Context->getTagDeclType(RD));
-
-  // Generate a funky cast.
-  SmallVector<QualType, 8> ArgTypes;
-
-  // Push the block argument type.
-  ArgTypes.push_back(PtrBlock);
-  if (FTP) {
-    for (FunctionProtoType::arg_type_iterator I = FTP->arg_type_begin(),
-         E = FTP->arg_type_end(); I && (I != E); ++I) {
-      QualType t = *I;
-      // Make sure we convert "t (^)(...)" to "t (*)(...)".
-      if (!convertBlockPointerToFunctionPointer(t))
-        convertToUnqualifiedObjCType(t);
-      ArgTypes.push_back(t);
-    }
-  }
-  // Now do the pointer to function cast.
-  QualType PtrToFuncCastType
-    = getSimpleFunctionType(Exp->getType(), &ArgTypes[0], ArgTypes.size());
-
-  PtrToFuncCastType = Context->getPointerType(PtrToFuncCastType);
-
-  CastExpr *BlkCast = NoTypeInfoCStyleCastExpr(Context, PtrBlock,
-                                               CK_BitCast,
-                                               const_cast<Expr*>(BlockExp));
-  // Don't forget the parens to enforce the proper binding.
-  ParenExpr *PE = new (Context) ParenExpr(SourceLocation(), SourceLocation(),
-                                          BlkCast);
-  //PE->dump();
-
-  FieldDecl *FD = FieldDecl::Create(*Context, 0, SourceLocation(),
-                                    SourceLocation(),
-                                    &Context->Idents.get("FuncPtr"),
-                                    Context->VoidPtrTy, 0,
-                                    /*BitWidth=*/0, /*Mutable=*/true,
-                                    ICIS_NoInit);
-  MemberExpr *ME = new (Context) MemberExpr(PE, true, FD, SourceLocation(),
-                                            FD->getType(), VK_LValue,
-                                            OK_Ordinary);
-
-  
-  CastExpr *FunkCast = NoTypeInfoCStyleCastExpr(Context, PtrToFuncCastType,
-                                                CK_BitCast, ME);
-  PE = new (Context) ParenExpr(SourceLocation(), SourceLocation(), FunkCast);
-
-  SmallVector<Expr*, 8> BlkExprs;
-  // Add the implicit argument.
-  BlkExprs.push_back(BlkCast);
-  // Add the user arguments.
-  for (CallExpr::arg_iterator I = Exp->arg_begin(),
-       E = Exp->arg_end(); I != E; ++I) {
-    BlkExprs.push_back(*I);
-  }
-  CallExpr *CE = new (Context) CallExpr(*Context, PE, &BlkExprs[0],
-                                        BlkExprs.size(),
-                                        Exp->getType(), VK_RValue,
-                                        SourceLocation());
-  return CE;
-}
-
-// We need to return the rewritten expression to handle cases where the
-// DeclRefExpr is embedded in another expression being rewritten.
-// For example:
-//
-// int main() {
-//    __block Foo *f;
-//    __block int i;
-//
-//    void (^myblock)() = ^() {
-//        [f test]; // f is a DeclRefExpr embedded in a message (which is being rewritten).
-//        i = 77;
-//    };
-//}
-Stmt *RewriteModernObjC::RewriteBlockDeclRefExpr(DeclRefExpr *DeclRefExp) {
-  // Rewrite the byref variable into BYREFVAR->__forwarding->BYREFVAR 
-  // for each DeclRefExp where BYREFVAR is name of the variable.
-  ValueDecl *VD = DeclRefExp->getDecl();
-  bool isArrow = DeclRefExp->refersToEnclosingLocal();
-  
-  FieldDecl *FD = FieldDecl::Create(*Context, 0, SourceLocation(),
-                                    SourceLocation(),
-                                    &Context->Idents.get("__forwarding"), 
-                                    Context->VoidPtrTy, 0,
-                                    /*BitWidth=*/0, /*Mutable=*/true,
-                                    ICIS_NoInit);
-  MemberExpr *ME = new (Context) MemberExpr(DeclRefExp, isArrow,
-                                            FD, SourceLocation(),
-                                            FD->getType(), VK_LValue,
-                                            OK_Ordinary);
-
-  StringRef Name = VD->getName();
-  FD = FieldDecl::Create(*Context, 0, SourceLocation(), SourceLocation(),
-                         &Context->Idents.get(Name), 
-                         Context->VoidPtrTy, 0,
-                         /*BitWidth=*/0, /*Mutable=*/true,
-                         ICIS_NoInit);
-  ME = new (Context) MemberExpr(ME, true, FD, SourceLocation(),
-                                DeclRefExp->getType(), VK_LValue, OK_Ordinary);
-  
-  
-  
-  // Need parens to enforce precedence.
-  ParenExpr *PE = new (Context) ParenExpr(DeclRefExp->getExprLoc(), 
-                                          DeclRefExp->getExprLoc(), 
-                                          ME);
-  ReplaceStmt(DeclRefExp, PE);
-  return PE;
-}
-
-// Rewrites the imported local variable V with external storage 
-// (static, extern, etc.) as *V
-//
-Stmt *RewriteModernObjC::RewriteLocalVariableExternalStorage(DeclRefExpr *DRE) {
-  ValueDecl *VD = DRE->getDecl();
-  if (VarDecl *Var = dyn_cast<VarDecl>(VD))
-    if (!ImportedLocalExternalDecls.count(Var))
-      return DRE;
-  Expr *Exp = new (Context) UnaryOperator(DRE, UO_Deref, DRE->getType(),
-                                          VK_LValue, OK_Ordinary,
-                                          DRE->getLocation());
-  // Need parens to enforce precedence.
-  ParenExpr *PE = new (Context) ParenExpr(SourceLocation(), SourceLocation(), 
-                                          Exp);
-  ReplaceStmt(DRE, PE);
-  return PE;
-}
-
-void RewriteModernObjC::RewriteCastExpr(CStyleCastExpr *CE) {
-  SourceLocation LocStart = CE->getLParenLoc();
-  SourceLocation LocEnd = CE->getRParenLoc();
-
-  // Need to avoid trying to rewrite synthesized casts.
-  if (LocStart.isInvalid())
-    return;
-  // Need to avoid trying to rewrite casts contained in macros.
-  if (!Rewriter::isRewritable(LocStart) || !Rewriter::isRewritable(LocEnd))
-    return;
-
-  const char *startBuf = SM->getCharacterData(LocStart);
-  const char *endBuf = SM->getCharacterData(LocEnd);
-  QualType QT = CE->getType();
-  const Type* TypePtr = QT->getAs<Type>();
-  if (isa<TypeOfExprType>(TypePtr)) {
-    const TypeOfExprType *TypeOfExprTypePtr = cast<TypeOfExprType>(TypePtr);
-    QT = TypeOfExprTypePtr->getUnderlyingExpr()->getType();
-    std::string TypeAsString = "(";
-    RewriteBlockPointerType(TypeAsString, QT);
-    TypeAsString += ")";
-    ReplaceText(LocStart, endBuf-startBuf+1, TypeAsString);
-    return;
-  }
-  // advance the location to startArgList.
-  const char *argPtr = startBuf;
-
-  while (*argPtr++ && (argPtr < endBuf)) {
-    switch (*argPtr) {
-    case '^':
-      // Replace the '^' with '*'.
-      LocStart = LocStart.getLocWithOffset(argPtr-startBuf);
-      ReplaceText(LocStart, 1, "*");
-      break;
-    }
-  }
-  return;
-}
-
-void RewriteModernObjC::RewriteImplicitCastObjCExpr(CastExpr *IC) {
-  CastKind CastKind = IC->getCastKind();
-  if (CastKind != CK_BlockPointerToObjCPointerCast &&
-      CastKind != CK_AnyPointerToBlockPointerCast)
-    return;
-  
-  QualType QT = IC->getType();
-  (void)convertBlockPointerToFunctionPointer(QT);
-  std::string TypeString(QT.getAsString(Context->getPrintingPolicy()));
-  std::string Str = "(";
-  Str += TypeString;
-  Str += ")";
-  InsertText(IC->getSubExpr()->getLocStart(), &Str[0], Str.size());
-
-  return;
-}
-
-void RewriteModernObjC::RewriteBlockPointerFunctionArgs(FunctionDecl *FD) {
-  SourceLocation DeclLoc = FD->getLocation();
-  unsigned parenCount = 0;
-
-  // We have 1 or more arguments that have closure pointers.
-  const char *startBuf = SM->getCharacterData(DeclLoc);
-  const char *startArgList = strchr(startBuf, '(');
-
-  assert((*startArgList == '(') && "Rewriter fuzzy parser confused");
-
-  parenCount++;
-  // advance the location to startArgList.
-  DeclLoc = DeclLoc.getLocWithOffset(startArgList-startBuf);
-  assert((DeclLoc.isValid()) && "Invalid DeclLoc");
-
-  const char *argPtr = startArgList;
-
-  while (*argPtr++ && parenCount) {
-    switch (*argPtr) {
-    case '^':
-      // Replace the '^' with '*'.
-      DeclLoc = DeclLoc.getLocWithOffset(argPtr-startArgList);
-      ReplaceText(DeclLoc, 1, "*");
-      break;
-    case '(':
-      parenCount++;
-      break;
-    case ')':
-      parenCount--;
-      break;
-    }
-  }
-  return;
-}
-
-bool RewriteModernObjC::PointerTypeTakesAnyBlockArguments(QualType QT) {
-  const FunctionProtoType *FTP;
-  const PointerType *PT = QT->getAs<PointerType>();
-  if (PT) {
-    FTP = PT->getPointeeType()->getAs<FunctionProtoType>();
-  } else {
-    const BlockPointerType *BPT = QT->getAs<BlockPointerType>();
-    assert(BPT && "BlockPointerTypeTakeAnyBlockArguments(): not a block pointer type");
-    FTP = BPT->getPointeeType()->getAs<FunctionProtoType>();
-  }
-  if (FTP) {
-    for (FunctionProtoType::arg_type_iterator I = FTP->arg_type_begin(),
-         E = FTP->arg_type_end(); I != E; ++I)
-      if (isTopLevelBlockPointerType(*I))
-        return true;
-  }
-  return false;
-}
-
-bool RewriteModernObjC::PointerTypeTakesAnyObjCQualifiedType(QualType QT) {
-  const FunctionProtoType *FTP;
-  const PointerType *PT = QT->getAs<PointerType>();
-  if (PT) {
-    FTP = PT->getPointeeType()->getAs<FunctionProtoType>();
-  } else {
-    const BlockPointerType *BPT = QT->getAs<BlockPointerType>();
-    assert(BPT && "BlockPointerTypeTakeAnyBlockArguments(): not a block pointer type");
-    FTP = BPT->getPointeeType()->getAs<FunctionProtoType>();
-  }
-  if (FTP) {
-    for (FunctionProtoType::arg_type_iterator I = FTP->arg_type_begin(),
-         E = FTP->arg_type_end(); I != E; ++I) {
-      if ((*I)->isObjCQualifiedIdType())
-        return true;
-      if ((*I)->isObjCObjectPointerType() &&
-          (*I)->getPointeeType()->isObjCQualifiedInterfaceType())
-        return true;
-    }
-        
-  }
-  return false;
-}
-
-void RewriteModernObjC::GetExtentOfArgList(const char *Name, const char *&LParen,
-                                     const char *&RParen) {
-  const char *argPtr = strchr(Name, '(');
-  assert((*argPtr == '(') && "Rewriter fuzzy parser confused");
-
-  LParen = argPtr; // output the start.
-  argPtr++; // skip past the left paren.
-  unsigned parenCount = 1;
-
-  while (*argPtr && parenCount) {
-    switch (*argPtr) {
-    case '(': parenCount++; break;
-    case ')': parenCount--; break;
-    default: break;
-    }
-    if (parenCount) argPtr++;
-  }
-  assert((*argPtr == ')') && "Rewriter fuzzy parser confused");
-  RParen = argPtr; // output the end
-}
-
-void RewriteModernObjC::RewriteBlockPointerDecl(NamedDecl *ND) {
-  if (FunctionDecl *FD = dyn_cast<FunctionDecl>(ND)) {
-    RewriteBlockPointerFunctionArgs(FD);
-    return;
-  }
-  // Handle Variables and Typedefs.
-  SourceLocation DeclLoc = ND->getLocation();
-  QualType DeclT;
-  if (VarDecl *VD = dyn_cast<VarDecl>(ND))
-    DeclT = VD->getType();
-  else if (TypedefNameDecl *TDD = dyn_cast<TypedefNameDecl>(ND))
-    DeclT = TDD->getUnderlyingType();
-  else if (FieldDecl *FD = dyn_cast<FieldDecl>(ND))
-    DeclT = FD->getType();
-  else
-    llvm_unreachable("RewriteBlockPointerDecl(): Decl type not yet handled");
-
-  const char *startBuf = SM->getCharacterData(DeclLoc);
-  const char *endBuf = startBuf;
-  // scan backward (from the decl location) for the end of the previous decl.
-  while (*startBuf != '^' && *startBuf != ';' && startBuf != MainFileStart)
-    startBuf--;
-  SourceLocation Start = DeclLoc.getLocWithOffset(startBuf-endBuf);
-  std::string buf;
-  unsigned OrigLength=0;
-  // *startBuf != '^' if we are dealing with a pointer to function that
-  // may take block argument types (which will be handled below).
-  if (*startBuf == '^') {
-    // Replace the '^' with '*', computing a negative offset.
-    buf = '*';
-    startBuf++;
-    OrigLength++;
-  }
-  while (*startBuf != ')') {
-    buf += *startBuf;
-    startBuf++;
-    OrigLength++;
-  }
-  buf += ')';
-  OrigLength++;
-  
-  if (PointerTypeTakesAnyBlockArguments(DeclT) ||
-      PointerTypeTakesAnyObjCQualifiedType(DeclT)) {
-    // Replace the '^' with '*' for arguments.
-    // Replace id<P> with id/*<>*/
-    DeclLoc = ND->getLocation();
-    startBuf = SM->getCharacterData(DeclLoc);
-    const char *argListBegin, *argListEnd;
-    GetExtentOfArgList(startBuf, argListBegin, argListEnd);
-    while (argListBegin < argListEnd) {
-      if (*argListBegin == '^')
-        buf += '*';
-      else if (*argListBegin ==  '<') {
-        buf += "/*"; 
-        buf += *argListBegin++;
-        OrigLength++;;
-        while (*argListBegin != '>') {
-          buf += *argListBegin++;
-          OrigLength++;
-        }
-        buf += *argListBegin;
-        buf += "*/";
-      }
-      else
-        buf += *argListBegin;
-      argListBegin++;
-      OrigLength++;
-    }
-    buf += ')';
-    OrigLength++;
-  }
-  ReplaceText(Start, OrigLength, buf);
-  
-  return;
-}
-
-
-/// SynthesizeByrefCopyDestroyHelper - This routine synthesizes:
-/// void __Block_byref_id_object_copy(struct Block_byref_id_object *dst,
-///                    struct Block_byref_id_object *src) {
-///  _Block_object_assign (&_dest->object, _src->object, 
-///                        BLOCK_BYREF_CALLER | BLOCK_FIELD_IS_OBJECT
-///                        [|BLOCK_FIELD_IS_WEAK]) // object
-///  _Block_object_assign(&_dest->object, _src->object, 
-///                       BLOCK_BYREF_CALLER | BLOCK_FIELD_IS_BLOCK
-///                       [|BLOCK_FIELD_IS_WEAK]) // block
-/// }
-/// And:
-/// void __Block_byref_id_object_dispose(struct Block_byref_id_object *_src) {
-///  _Block_object_dispose(_src->object, 
-///                        BLOCK_BYREF_CALLER | BLOCK_FIELD_IS_OBJECT
-///                        [|BLOCK_FIELD_IS_WEAK]) // object
-///  _Block_object_dispose(_src->object, 
-///                         BLOCK_BYREF_CALLER | BLOCK_FIELD_IS_BLOCK
-///                         [|BLOCK_FIELD_IS_WEAK]) // block
-/// }
-
-std::string RewriteModernObjC::SynthesizeByrefCopyDestroyHelper(VarDecl *VD,
-                                                          int flag) {
-  std::string S;
-  if (CopyDestroyCache.count(flag))
-    return S;
-  CopyDestroyCache.insert(flag);
-  S = "static void __Block_byref_id_object_copy_";
-  S += utostr(flag);
-  S += "(void *dst, void *src) {\n";
-  
-  // offset into the object pointer is computed as:
-  // void * + void* + int + int + void* + void *
-  unsigned IntSize = 
-  static_cast<unsigned>(Context->getTypeSize(Context->IntTy));
-  unsigned VoidPtrSize = 
-  static_cast<unsigned>(Context->getTypeSize(Context->VoidPtrTy));
-  
-  unsigned offset = (VoidPtrSize*4 + IntSize + IntSize)/Context->getCharWidth();
-  S += " _Block_object_assign((char*)dst + ";
-  S += utostr(offset);
-  S += ", *(void * *) ((char*)src + ";
-  S += utostr(offset);
-  S += "), ";
-  S += utostr(flag);
-  S += ");\n}\n";
-  
-  S += "static void __Block_byref_id_object_dispose_";
-  S += utostr(flag);
-  S += "(void *src) {\n";
-  S += " _Block_object_dispose(*(void * *) ((char*)src + ";
-  S += utostr(offset);
-  S += "), ";
-  S += utostr(flag);
-  S += ");\n}\n";
-  return S;
-}
-
-/// RewriteByRefVar - For each __block typex ND variable this routine transforms
-/// the declaration into:
-/// struct __Block_byref_ND {
-/// void *__isa;                  // NULL for everything except __weak pointers
-/// struct __Block_byref_ND *__forwarding;
-/// int32_t __flags;
-/// int32_t __size;
-/// void *__Block_byref_id_object_copy; // If variable is __block ObjC object
-/// void *__Block_byref_id_object_dispose; // If variable is __block ObjC object
-/// typex ND;
-/// };
-///
-/// It then replaces declaration of ND variable with:
-/// struct __Block_byref_ND ND = {__isa=0B, __forwarding=&ND, __flags=some_flag, 
-///                               __size=sizeof(struct __Block_byref_ND), 
-///                               ND=initializer-if-any};
-///
-///
-void RewriteModernObjC::RewriteByRefVar(VarDecl *ND, bool firstDecl,
-                                        bool lastDecl) {
-  int flag = 0;
-  int isa = 0;
-  SourceLocation DeclLoc = ND->getTypeSpecStartLoc();
-  if (DeclLoc.isInvalid())
-    // If type location is missing, it is because of missing type (a warning).
-    // Use variable's location which is good for this case.
-    DeclLoc = ND->getLocation();
-  const char *startBuf = SM->getCharacterData(DeclLoc);
-  SourceLocation X = ND->getLocEnd();
-  X = SM->getExpansionLoc(X);
-  const char *endBuf = SM->getCharacterData(X);
-  std::string Name(ND->getNameAsString());
-  std::string ByrefType;
-  RewriteByRefString(ByrefType, Name, ND, true);
-  ByrefType += " {\n";
-  ByrefType += "  void *__isa;\n";
-  RewriteByRefString(ByrefType, Name, ND);
-  ByrefType += " *__forwarding;\n";
-  ByrefType += " int __flags;\n";
-  ByrefType += " int __size;\n";
-  // Add void *__Block_byref_id_object_copy; 
-  // void *__Block_byref_id_object_dispose; if needed.
-  QualType Ty = ND->getType();
-  bool HasCopyAndDispose = Context->BlockRequiresCopying(Ty);
-  if (HasCopyAndDispose) {
-    ByrefType += " void (*__Block_byref_id_object_copy)(void*, void*);\n";
-    ByrefType += " void (*__Block_byref_id_object_dispose)(void*);\n";
-  }
-
-  QualType T = Ty;
-  (void)convertBlockPointerToFunctionPointer(T);
-  T.getAsStringInternal(Name, Context->getPrintingPolicy());
-    
-  ByrefType += " " + Name + ";\n";
-  ByrefType += "};\n";
-  // Insert this type in global scope. It is needed by helper function.
-  SourceLocation FunLocStart;
-  if (CurFunctionDef)
-     FunLocStart = getFunctionSourceLocation(*this, CurFunctionDef);
-  else {
-    assert(CurMethodDef && "RewriteByRefVar - CurMethodDef is null");
-    FunLocStart = CurMethodDef->getLocStart();
-  }
-  InsertText(FunLocStart, ByrefType);
-  
-  if (Ty.isObjCGCWeak()) {
-    flag |= BLOCK_FIELD_IS_WEAK;
-    isa = 1;
-  }
-  if (HasCopyAndDispose) {
-    flag = BLOCK_BYREF_CALLER;
-    QualType Ty = ND->getType();
-    // FIXME. Handle __weak variable (BLOCK_FIELD_IS_WEAK) as well.
-    if (Ty->isBlockPointerType())
-      flag |= BLOCK_FIELD_IS_BLOCK;
-    else
-      flag |= BLOCK_FIELD_IS_OBJECT;
-    std::string HF = SynthesizeByrefCopyDestroyHelper(ND, flag);
-    if (!HF.empty())
-      InsertText(FunLocStart, HF);
-  }
-  
-  // struct __Block_byref_ND ND = 
-  // {0, &ND, some_flag, __size=sizeof(struct __Block_byref_ND), 
-  //  initializer-if-any};
-  bool hasInit = (ND->getInit() != 0);
-  // FIXME. rewriter does not support __block c++ objects which
-  // require construction.
-  if (hasInit)
-    if (CXXConstructExpr *CExp = dyn_cast<CXXConstructExpr>(ND->getInit())) {
-      CXXConstructorDecl *CXXDecl = CExp->getConstructor();
-      if (CXXDecl && CXXDecl->isDefaultConstructor())
-        hasInit = false;
-    }
-  
-  unsigned flags = 0;
-  if (HasCopyAndDispose)
-    flags |= BLOCK_HAS_COPY_DISPOSE;
-  Name = ND->getNameAsString();
-  ByrefType.clear();
-  RewriteByRefString(ByrefType, Name, ND);
-  std::string ForwardingCastType("(");
-  ForwardingCastType += ByrefType + " *)";
-  ByrefType += " " + Name + " = {(void*)";
-  ByrefType += utostr(isa);
-  ByrefType += "," +  ForwardingCastType + "&" + Name + ", ";
-  ByrefType += utostr(flags);
-  ByrefType += ", ";
-  ByrefType += "sizeof(";
-  RewriteByRefString(ByrefType, Name, ND);
-  ByrefType += ")";
-  if (HasCopyAndDispose) {
-    ByrefType += ", __Block_byref_id_object_copy_";
-    ByrefType += utostr(flag);
-    ByrefType += ", __Block_byref_id_object_dispose_";
-    ByrefType += utostr(flag);
-  }
-  
-  if (!firstDecl) {
-    // In multiple __block declarations, and for all but 1st declaration,
-    // find location of the separating comma. This would be start location
-    // where new text is to be inserted.
-    DeclLoc = ND->getLocation();
-    const char *startDeclBuf = SM->getCharacterData(DeclLoc);
-    const char *commaBuf = startDeclBuf;
-    while (*commaBuf != ',')
-      commaBuf--;
-    assert((*commaBuf == ',') && "RewriteByRefVar: can't find ','");
-    DeclLoc = DeclLoc.getLocWithOffset(commaBuf - startDeclBuf);
-    startBuf = commaBuf;
-  }
-  
-  if (!hasInit) {
-    ByrefType += "};\n";
-    unsigned nameSize = Name.size();
-    // for block or function pointer declaration. Name is aleady
-    // part of the declaration.
-    if (Ty->isBlockPointerType() || Ty->isFunctionPointerType())
-      nameSize = 1;
-    ReplaceText(DeclLoc, endBuf-startBuf+nameSize, ByrefType);
-  }
-  else {
-    ByrefType += ", ";
-    SourceLocation startLoc;
-    Expr *E = ND->getInit();
-    if (const CStyleCastExpr *ECE = dyn_cast<CStyleCastExpr>(E))
-      startLoc = ECE->getLParenLoc();
-    else
-      startLoc = E->getLocStart();
-    startLoc = SM->getExpansionLoc(startLoc);
-    endBuf = SM->getCharacterData(startLoc);
-    ReplaceText(DeclLoc, endBuf-startBuf, ByrefType);
-
-    const char separator = lastDecl ? ';' : ',';
-    const char *startInitializerBuf = SM->getCharacterData(startLoc);
-    const char *separatorBuf = strchr(startInitializerBuf, separator);
-    assert((*separatorBuf == separator) && 
-           "RewriteByRefVar: can't find ';' or ','");
-    SourceLocation separatorLoc =
-      startLoc.getLocWithOffset(separatorBuf-startInitializerBuf);
-    
-    InsertText(separatorLoc, lastDecl ? "}" : "};\n");
-  }
-  return;
-}
-
-void RewriteModernObjC::CollectBlockDeclRefInfo(BlockExpr *Exp) {
-  // Add initializers for any closure decl refs.
-  GetBlockDeclRefExprs(Exp->getBody());
-  if (BlockDeclRefs.size()) {
-    // Unique all "by copy" declarations.
-    for (unsigned i = 0; i < BlockDeclRefs.size(); i++)
-      if (!BlockDeclRefs[i]->getDecl()->hasAttr<BlocksAttr>()) {
-        if (!BlockByCopyDeclsPtrSet.count(BlockDeclRefs[i]->getDecl())) {
-          BlockByCopyDeclsPtrSet.insert(BlockDeclRefs[i]->getDecl());
-          BlockByCopyDecls.push_back(BlockDeclRefs[i]->getDecl());
-        }
-      }
-    // Unique all "by ref" declarations.
-    for (unsigned i = 0; i < BlockDeclRefs.size(); i++)
-      if (BlockDeclRefs[i]->getDecl()->hasAttr<BlocksAttr>()) {
-        if (!BlockByRefDeclsPtrSet.count(BlockDeclRefs[i]->getDecl())) {
-          BlockByRefDeclsPtrSet.insert(BlockDeclRefs[i]->getDecl());
-          BlockByRefDecls.push_back(BlockDeclRefs[i]->getDecl());
-        }
-      }
-    // Find any imported blocks...they will need special attention.
-    for (unsigned i = 0; i < BlockDeclRefs.size(); i++)
-      if (BlockDeclRefs[i]->getDecl()->hasAttr<BlocksAttr>() ||
-          BlockDeclRefs[i]->getType()->isObjCObjectPointerType() || 
-          BlockDeclRefs[i]->getType()->isBlockPointerType())
-        ImportedBlockDecls.insert(BlockDeclRefs[i]->getDecl());
-  }
-}
-
-FunctionDecl *RewriteModernObjC::SynthBlockInitFunctionDecl(StringRef name) {
-  IdentifierInfo *ID = &Context->Idents.get(name);
-  QualType FType = Context->getFunctionNoProtoType(Context->VoidPtrTy);
-  return FunctionDecl::Create(*Context, TUDecl, SourceLocation(),
-                              SourceLocation(), ID, FType, 0, SC_Extern,
-                              SC_None, false, false);
-}
-
-Stmt *RewriteModernObjC::SynthBlockInitExpr(BlockExpr *Exp,
-          const SmallVector<DeclRefExpr *, 8> &InnerBlockDeclRefs) {
-  
-  const BlockDecl *block = Exp->getBlockDecl();
-  
-  Blocks.push_back(Exp);
-
-  CollectBlockDeclRefInfo(Exp);
-  
-  // Add inner imported variables now used in current block.
- int countOfInnerDecls = 0;
-  if (!InnerBlockDeclRefs.empty()) {
-    for (unsigned i = 0; i < InnerBlockDeclRefs.size(); i++) {
-      DeclRefExpr *Exp = InnerBlockDeclRefs[i];
-      ValueDecl *VD = Exp->getDecl();
-      if (!VD->hasAttr<BlocksAttr>() && !BlockByCopyDeclsPtrSet.count(VD)) {
-      // We need to save the copied-in variables in nested
-      // blocks because it is needed at the end for some of the API generations.
-      // See SynthesizeBlockLiterals routine.
-        InnerDeclRefs.push_back(Exp); countOfInnerDecls++;
-        BlockDeclRefs.push_back(Exp);
-        BlockByCopyDeclsPtrSet.insert(VD);
-        BlockByCopyDecls.push_back(VD);
-      }
-      if (VD->hasAttr<BlocksAttr>() && !BlockByRefDeclsPtrSet.count(VD)) {
-        InnerDeclRefs.push_back(Exp); countOfInnerDecls++;
-        BlockDeclRefs.push_back(Exp);
-        BlockByRefDeclsPtrSet.insert(VD);
-        BlockByRefDecls.push_back(VD);
-      }
-    }
-    // Find any imported blocks...they will need special attention.
-    for (unsigned i = 0; i < InnerBlockDeclRefs.size(); i++)
-      if (InnerBlockDeclRefs[i]->getDecl()->hasAttr<BlocksAttr>() ||
-          InnerBlockDeclRefs[i]->getType()->isObjCObjectPointerType() || 
-          InnerBlockDeclRefs[i]->getType()->isBlockPointerType())
-        ImportedBlockDecls.insert(InnerBlockDeclRefs[i]->getDecl());
-  }
-  InnerDeclRefsCount.push_back(countOfInnerDecls);
-  
-  std::string FuncName;
-
-  if (CurFunctionDef)
-    FuncName = CurFunctionDef->getNameAsString();
-  else if (CurMethodDef)
-    BuildUniqueMethodName(FuncName, CurMethodDef);
-  else if (GlobalVarDecl)
-    FuncName = std::string(GlobalVarDecl->getNameAsString());
-
-  bool GlobalBlockExpr = 
-    block->getDeclContext()->getRedeclContext()->isFileContext();
-  
-  if (GlobalBlockExpr && !GlobalVarDecl) {
-    Diags.Report(block->getLocation(), GlobalBlockRewriteFailedDiag);
-    GlobalBlockExpr = false;
-  }
-  
-  std::string BlockNumber = utostr(Blocks.size()-1);
-
-  std::string Func = "__" + FuncName + "_block_func_" + BlockNumber;
-
-  // Get a pointer to the function type so we can cast appropriately.
-  QualType BFT = convertFunctionTypeOfBlocks(Exp->getFunctionType());
-  QualType FType = Context->getPointerType(BFT);
-
-  FunctionDecl *FD;
-  Expr *NewRep;
-
-  // Simulate a contructor call...
-  std::string Tag;
-  
-  if (GlobalBlockExpr)
-    Tag = "__global_";
-  else
-    Tag = "__";
-  Tag += FuncName + "_block_impl_" + BlockNumber;
-  
-  FD = SynthBlockInitFunctionDecl(Tag);
-  DeclRefExpr *DRE = new (Context) DeclRefExpr(FD, false, FType, VK_RValue,
-                                               SourceLocation());
-
-  SmallVector<Expr*, 4> InitExprs;
-
-  // Initialize the block function.
-  FD = SynthBlockInitFunctionDecl(Func);
-  DeclRefExpr *Arg = new (Context) DeclRefExpr(FD, false, FD->getType(),
-                                               VK_LValue, SourceLocation());
-  CastExpr *castExpr = NoTypeInfoCStyleCastExpr(Context, Context->VoidPtrTy,
-                                                CK_BitCast, Arg);
-  InitExprs.push_back(castExpr);
-
-  // Initialize the block descriptor.
-  std::string DescData = "__" + FuncName + "_block_desc_" + BlockNumber + "_DATA";
-
-  VarDecl *NewVD = VarDecl::Create(*Context, TUDecl,
-                                   SourceLocation(), SourceLocation(),
-                                   &Context->Idents.get(DescData.c_str()),
-                                   Context->VoidPtrTy, 0,
-                                   SC_Static, SC_None);
-  UnaryOperator *DescRefExpr =
-    new (Context) UnaryOperator(new (Context) DeclRefExpr(NewVD, false,
-                                                          Context->VoidPtrTy,
-                                                          VK_LValue,
-                                                          SourceLocation()), 
-                                UO_AddrOf,
-                                Context->getPointerType(Context->VoidPtrTy), 
-                                VK_RValue, OK_Ordinary,
-                                SourceLocation());
-  InitExprs.push_back(DescRefExpr); 
-  
-  // Add initializers for any closure decl refs.
-  if (BlockDeclRefs.size()) {
-    Expr *Exp;
-    // Output all "by copy" declarations.
-    for (SmallVector<ValueDecl*,8>::iterator I = BlockByCopyDecls.begin(),
-         E = BlockByCopyDecls.end(); I != E; ++I) {
-      if (isObjCType((*I)->getType())) {
-        // FIXME: Conform to ABI ([[obj retain] autorelease]).
-        FD = SynthBlockInitFunctionDecl((*I)->getName());
-        Exp = new (Context) DeclRefExpr(FD, false, FD->getType(),
-                                        VK_LValue, SourceLocation());
-        if (HasLocalVariableExternalStorage(*I)) {
-          QualType QT = (*I)->getType();
-          QT = Context->getPointerType(QT);
-          Exp = new (Context) UnaryOperator(Exp, UO_AddrOf, QT, VK_RValue,
-                                            OK_Ordinary, SourceLocation());
-        }
-      } else if (isTopLevelBlockPointerType((*I)->getType())) {
-        FD = SynthBlockInitFunctionDecl((*I)->getName());
-        Arg = new (Context) DeclRefExpr(FD, false, FD->getType(),
-                                        VK_LValue, SourceLocation());
-        Exp = NoTypeInfoCStyleCastExpr(Context, Context->VoidPtrTy,
-                                       CK_BitCast, Arg);
-      } else {
-        FD = SynthBlockInitFunctionDecl((*I)->getName());
-        Exp = new (Context) DeclRefExpr(FD, false, FD->getType(),
-                                        VK_LValue, SourceLocation());
-        if (HasLocalVariableExternalStorage(*I)) {
-          QualType QT = (*I)->getType();
-          QT = Context->getPointerType(QT);
-          Exp = new (Context) UnaryOperator(Exp, UO_AddrOf, QT, VK_RValue,
-                                            OK_Ordinary, SourceLocation());
-        }
-        
-      }
-      InitExprs.push_back(Exp);
-    }
-    // Output all "by ref" declarations.
-    for (SmallVector<ValueDecl*,8>::iterator I = BlockByRefDecls.begin(),
-         E = BlockByRefDecls.end(); I != E; ++I) {
-      ValueDecl *ND = (*I);
-      std::string Name(ND->getNameAsString());
-      std::string RecName;
-      RewriteByRefString(RecName, Name, ND, true);
-      IdentifierInfo *II = &Context->Idents.get(RecName.c_str() 
-                                                + sizeof("struct"));
-      RecordDecl *RD = RecordDecl::Create(*Context, TTK_Struct, TUDecl,
-                                          SourceLocation(), SourceLocation(),
-                                          II);
-      assert(RD && "SynthBlockInitExpr(): Can't find RecordDecl");
-      QualType castT = Context->getPointerType(Context->getTagDeclType(RD));
-      
-      FD = SynthBlockInitFunctionDecl((*I)->getName());
-      Exp = new (Context) DeclRefExpr(FD, false, FD->getType(), VK_LValue,
-                                      SourceLocation());
-      bool isNestedCapturedVar = false;
-      if (block)
-        for (BlockDecl::capture_const_iterator ci = block->capture_begin(),
-             ce = block->capture_end(); ci != ce; ++ci) {
-          const VarDecl *variable = ci->getVariable();
-          if (variable == ND && ci->isNested()) {
-            assert (ci->isByRef() && 
-                    "SynthBlockInitExpr - captured block variable is not byref");
-            isNestedCapturedVar = true;
-            break;
-          }
-        }
-      // captured nested byref variable has its address passed. Do not take
-      // its address again.
-      if (!isNestedCapturedVar)
-          Exp = new (Context) UnaryOperator(Exp, UO_AddrOf,
-                                     Context->getPointerType(Exp->getType()),
-                                     VK_RValue, OK_Ordinary, SourceLocation());
-      Exp = NoTypeInfoCStyleCastExpr(Context, castT, CK_BitCast, Exp);
-      InitExprs.push_back(Exp);
-    }
-  }
-  if (ImportedBlockDecls.size()) {
-    // generate BLOCK_HAS_COPY_DISPOSE(have helper funcs) | BLOCK_HAS_DESCRIPTOR
-    int flag = (BLOCK_HAS_COPY_DISPOSE | BLOCK_HAS_DESCRIPTOR);
-    unsigned IntSize = 
-      static_cast<unsigned>(Context->getTypeSize(Context->IntTy));
-    Expr *FlagExp = IntegerLiteral::Create(*Context, llvm::APInt(IntSize, flag), 
-                                           Context->IntTy, SourceLocation());
-    InitExprs.push_back(FlagExp);
-  }
-  NewRep = new (Context) CallExpr(*Context, DRE, &InitExprs[0], InitExprs.size(),
-                                  FType, VK_LValue, SourceLocation());
-  
-  if (GlobalBlockExpr) {
-    assert (GlobalConstructionExp == 0 && 
-            "SynthBlockInitExpr - GlobalConstructionExp must be null");
-    GlobalConstructionExp = NewRep;
-    NewRep = DRE;
-  }
-  
-  NewRep = new (Context) UnaryOperator(NewRep, UO_AddrOf,
-                             Context->getPointerType(NewRep->getType()),
-                             VK_RValue, OK_Ordinary, SourceLocation());
-  NewRep = NoTypeInfoCStyleCastExpr(Context, FType, CK_BitCast,
-                                    NewRep);
-  BlockDeclRefs.clear();
-  BlockByRefDecls.clear();
-  BlockByRefDeclsPtrSet.clear();
-  BlockByCopyDecls.clear();
-  BlockByCopyDeclsPtrSet.clear();
-  ImportedBlockDecls.clear();
-  return NewRep;
-}
-
-bool RewriteModernObjC::IsDeclStmtInForeachHeader(DeclStmt *DS) {
-  if (const ObjCForCollectionStmt * CS = 
-      dyn_cast<ObjCForCollectionStmt>(Stmts.back()))
-        return CS->getElement() == DS;
-  return false;
-}
-
-//===----------------------------------------------------------------------===//
-// Function Body / Expression rewriting
-//===----------------------------------------------------------------------===//
-
-Stmt *RewriteModernObjC::RewriteFunctionBodyOrGlobalInitializer(Stmt *S) {
-  if (isa<SwitchStmt>(S) || isa<WhileStmt>(S) ||
-      isa<DoStmt>(S) || isa<ForStmt>(S))
-    Stmts.push_back(S);
-  else if (isa<ObjCForCollectionStmt>(S)) {
-    Stmts.push_back(S);
-    ObjCBcLabelNo.push_back(++BcLabelCount);
-  }
-
-  // Pseudo-object operations and ivar references need special
-  // treatment because we're going to recursively rewrite them.
-  if (PseudoObjectExpr *PseudoOp = dyn_cast<PseudoObjectExpr>(S)) {
-    if (isa<BinaryOperator>(PseudoOp->getSyntacticForm())) {
-      return RewritePropertyOrImplicitSetter(PseudoOp);
-    } else {
-      return RewritePropertyOrImplicitGetter(PseudoOp);
-    }
-  } else if (ObjCIvarRefExpr *IvarRefExpr = dyn_cast<ObjCIvarRefExpr>(S)) {
-    return RewriteObjCIvarRefExpr(IvarRefExpr);
-  }
-
-  SourceRange OrigStmtRange = S->getSourceRange();
-
-  // Perform a bottom up rewrite of all children.
-  for (Stmt::child_range CI = S->children(); CI; ++CI)
-    if (*CI) {
-      Stmt *childStmt = (*CI);
-      Stmt *newStmt = RewriteFunctionBodyOrGlobalInitializer(childStmt);
-      if (newStmt) {
-        *CI = newStmt;
-      }
-    }
-
-  if (BlockExpr *BE = dyn_cast<BlockExpr>(S)) {
-    SmallVector<DeclRefExpr *, 8> InnerBlockDeclRefs;
-    llvm::SmallPtrSet<const DeclContext *, 8> InnerContexts;
-    InnerContexts.insert(BE->getBlockDecl());
-    ImportedLocalExternalDecls.clear();
-    GetInnerBlockDeclRefExprs(BE->getBody(),
-                              InnerBlockDeclRefs, InnerContexts);
-    // Rewrite the block body in place.
-    Stmt *SaveCurrentBody = CurrentBody;
-    CurrentBody = BE->getBody();
-    PropParentMap = 0;
-    // block literal on rhs of a property-dot-sytax assignment
-    // must be replaced by its synthesize ast so getRewrittenText
-    // works as expected. In this case, what actually ends up on RHS
-    // is the blockTranscribed which is the helper function for the
-    // block literal; as in: self.c = ^() {[ace ARR];};
-    bool saveDisableReplaceStmt = DisableReplaceStmt;
-    DisableReplaceStmt = false;
-    RewriteFunctionBodyOrGlobalInitializer(BE->getBody());
-    DisableReplaceStmt = saveDisableReplaceStmt;
-    CurrentBody = SaveCurrentBody;
-    PropParentMap = 0;
-    ImportedLocalExternalDecls.clear();
-    // Now we snarf the rewritten text and stash it away for later use.
-    std::string Str = Rewrite.getRewrittenText(BE->getSourceRange());
-    RewrittenBlockExprs[BE] = Str;
-
-    Stmt *blockTranscribed = SynthBlockInitExpr(BE, InnerBlockDeclRefs);
-                            
-    //blockTranscribed->dump();
-    ReplaceStmt(S, blockTranscribed);
-    return blockTranscribed;
-  }
-  // Handle specific things.
-  if (ObjCEncodeExpr *AtEncode = dyn_cast<ObjCEncodeExpr>(S))
-    return RewriteAtEncode(AtEncode);
-
-  if (ObjCSelectorExpr *AtSelector = dyn_cast<ObjCSelectorExpr>(S))
-    return RewriteAtSelector(AtSelector);
-
-  if (ObjCStringLiteral *AtString = dyn_cast<ObjCStringLiteral>(S))
-    return RewriteObjCStringLiteral(AtString);
-  
-  if (ObjCBoolLiteralExpr *BoolLitExpr = dyn_cast<ObjCBoolLiteralExpr>(S))
-    return RewriteObjCBoolLiteralExpr(BoolLitExpr);
-  
-  if (ObjCBoxedExpr *BoxedExpr = dyn_cast<ObjCBoxedExpr>(S))
-    return RewriteObjCBoxedExpr(BoxedExpr);
-  
-  if (ObjCArrayLiteral *ArrayLitExpr = dyn_cast<ObjCArrayLiteral>(S))
-    return RewriteObjCArrayLiteralExpr(ArrayLitExpr);
-  
-  if (ObjCDictionaryLiteral *DictionaryLitExpr = 
-        dyn_cast<ObjCDictionaryLiteral>(S))
-    return RewriteObjCDictionaryLiteralExpr(DictionaryLitExpr);
-
-  if (ObjCMessageExpr *MessExpr = dyn_cast<ObjCMessageExpr>(S)) {
-#if 0
-    // Before we rewrite it, put the original message expression in a comment.
-    SourceLocation startLoc = MessExpr->getLocStart();
-    SourceLocation endLoc = MessExpr->getLocEnd();
-
-    const char *startBuf = SM->getCharacterData(startLoc);
-    const char *endBuf = SM->getCharacterData(endLoc);
-
-    std::string messString;
-    messString += "// ";
-    messString.append(startBuf, endBuf-startBuf+1);
-    messString += "\n";
-
-    // FIXME: Missing definition of
-    // InsertText(clang::SourceLocation, char const*, unsigned int).
-    // InsertText(startLoc, messString.c_str(), messString.size());
-    // Tried this, but it didn't work either...
-    // ReplaceText(startLoc, 0, messString.c_str(), messString.size());
-#endif
-    return RewriteMessageExpr(MessExpr);
-  }
-
-  if (ObjCAutoreleasePoolStmt *StmtAutoRelease = 
-        dyn_cast<ObjCAutoreleasePoolStmt>(S)) {
-    return RewriteObjCAutoreleasePoolStmt(StmtAutoRelease);
-  }
-  
-  if (ObjCAtTryStmt *StmtTry = dyn_cast<ObjCAtTryStmt>(S))
-    return RewriteObjCTryStmt(StmtTry);
-
-  if (ObjCAtSynchronizedStmt *StmtTry = dyn_cast<ObjCAtSynchronizedStmt>(S))
-    return RewriteObjCSynchronizedStmt(StmtTry);
-
-  if (ObjCAtThrowStmt *StmtThrow = dyn_cast<ObjCAtThrowStmt>(S))
-    return RewriteObjCThrowStmt(StmtThrow);
-
-  if (ObjCProtocolExpr *ProtocolExp = dyn_cast<ObjCProtocolExpr>(S))
-    return RewriteObjCProtocolExpr(ProtocolExp);
-
-  if (ObjCForCollectionStmt *StmtForCollection =
-        dyn_cast<ObjCForCollectionStmt>(S))
-    return RewriteObjCForCollectionStmt(StmtForCollection,
-                                        OrigStmtRange.getEnd());
-  if (BreakStmt *StmtBreakStmt =
-      dyn_cast<BreakStmt>(S))
-    return RewriteBreakStmt(StmtBreakStmt);
-  if (ContinueStmt *StmtContinueStmt =
-      dyn_cast<ContinueStmt>(S))
-    return RewriteContinueStmt(StmtContinueStmt);
-
-  // Need to check for protocol refs (id <P>, Foo <P> *) in variable decls
-  // and cast exprs.
-  if (DeclStmt *DS = dyn_cast<DeclStmt>(S)) {
-    // FIXME: What we're doing here is modifying the type-specifier that
-    // precedes the first Decl.  In the future the DeclGroup should have
-    // a separate type-specifier that we can rewrite.
-    // NOTE: We need to avoid rewriting the DeclStmt if it is within
-    // the context of an ObjCForCollectionStmt. For example:
-    //   NSArray *someArray;
-    //   for (id <FooProtocol> index in someArray) ;
-    // This is because RewriteObjCForCollectionStmt() does textual rewriting 
-    // and it depends on the original text locations/positions.
-    if (Stmts.empty() || !IsDeclStmtInForeachHeader(DS))
-      RewriteObjCQualifiedInterfaceTypes(*DS->decl_begin());
-
-    // Blocks rewrite rules.
-    for (DeclStmt::decl_iterator DI = DS->decl_begin(), DE = DS->decl_end();
-         DI != DE; ++DI) {
-      Decl *SD = *DI;
-      if (ValueDecl *ND = dyn_cast<ValueDecl>(SD)) {
-        if (isTopLevelBlockPointerType(ND->getType()))
-          RewriteBlockPointerDecl(ND);
-        else if (ND->getType()->isFunctionPointerType())
-          CheckFunctionPointerDecl(ND->getType(), ND);
-        if (VarDecl *VD = dyn_cast<VarDecl>(SD)) {
-          if (VD->hasAttr<BlocksAttr>()) {
-            static unsigned uniqueByrefDeclCount = 0;
-            assert(!BlockByRefDeclNo.count(ND) &&
-              "RewriteFunctionBodyOrGlobalInitializer: Duplicate byref decl");
-            BlockByRefDeclNo[ND] = uniqueByrefDeclCount++;
-            RewriteByRefVar(VD, (DI == DS->decl_begin()), ((DI+1) == DE));
-          }
-          else           
-            RewriteTypeOfDecl(VD);
-        }
-      }
-      if (TypedefNameDecl *TD = dyn_cast<TypedefNameDecl>(SD)) {
-        if (isTopLevelBlockPointerType(TD->getUnderlyingType()))
-          RewriteBlockPointerDecl(TD);
-        else if (TD->getUnderlyingType()->isFunctionPointerType())
-          CheckFunctionPointerDecl(TD->getUnderlyingType(), TD);
-      }
-    }
-  }
-
-  if (CStyleCastExpr *CE = dyn_cast<CStyleCastExpr>(S))
-    RewriteObjCQualifiedInterfaceTypes(CE);
-
-  if (isa<SwitchStmt>(S) || isa<WhileStmt>(S) ||
-      isa<DoStmt>(S) || isa<ForStmt>(S)) {
-    assert(!Stmts.empty() && "Statement stack is empty");
-    assert ((isa<SwitchStmt>(Stmts.back()) || isa<WhileStmt>(Stmts.back()) ||
-             isa<DoStmt>(Stmts.back()) || isa<ForStmt>(Stmts.back()))
-            && "Statement stack mismatch");
-    Stmts.pop_back();
-  }
-  // Handle blocks rewriting.
-  if (DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(S)) {
-    ValueDecl *VD = DRE->getDecl(); 
-    if (VD->hasAttr<BlocksAttr>())
-      return RewriteBlockDeclRefExpr(DRE);
-    if (HasLocalVariableExternalStorage(VD))
-      return RewriteLocalVariableExternalStorage(DRE);
-  }
-  
-  if (CallExpr *CE = dyn_cast<CallExpr>(S)) {
-    if (CE->getCallee()->getType()->isBlockPointerType()) {
-      Stmt *BlockCall = SynthesizeBlockCall(CE, CE->getCallee());
-      ReplaceStmt(S, BlockCall);
-      return BlockCall;
-    }
-  }
-  if (CStyleCastExpr *CE = dyn_cast<CStyleCastExpr>(S)) {
-    RewriteCastExpr(CE);
-  }
-  if (ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(S)) {
-    RewriteImplicitCastObjCExpr(ICE);
-  }
-#if 0
-
-  if (ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(S)) {
-    CastExpr *Replacement = new (Context) CastExpr(ICE->getType(),
-                                                   ICE->getSubExpr(),
-                                                   SourceLocation());
-    // Get the new text.
-    std::string SStr;
-    llvm::raw_string_ostream Buf(SStr);
-    Replacement->printPretty(Buf);
-    const std::string &Str = Buf.str();
-
-    printf("CAST = %s\n", &Str[0]);
-    InsertText(ICE->getSubExpr()->getLocStart(), &Str[0], Str.size());
-    delete S;
-    return Replacement;
-  }
-#endif
-  // Return this stmt unmodified.
-  return S;
-}
-
-void RewriteModernObjC::RewriteRecordBody(RecordDecl *RD) {
-  for (RecordDecl::field_iterator i = RD->field_begin(), 
-                                  e = RD->field_end(); i != e; ++i) {
-    FieldDecl *FD = *i;
-    if (isTopLevelBlockPointerType(FD->getType()))
-      RewriteBlockPointerDecl(FD);
-    if (FD->getType()->isObjCQualifiedIdType() ||
-        FD->getType()->isObjCQualifiedInterfaceType())
-      RewriteObjCQualifiedInterfaceTypes(FD);
-  }
-}
-
-/// HandleDeclInMainFile - This is called for each top-level decl defined in the
-/// main file of the input.
-void RewriteModernObjC::HandleDeclInMainFile(Decl *D) {
-  switch (D->getKind()) {
-    case Decl::Function: {
-      FunctionDecl *FD = cast<FunctionDecl>(D);
-      if (FD->isOverloadedOperator())
-        return;
-
-      // Since function prototypes don't have ParmDecl's, we check the function
-      // prototype. This enables us to rewrite function declarations and
-      // definitions using the same code.
-      RewriteBlocksInFunctionProtoType(FD->getType(), FD);
-
-      if (!FD->isThisDeclarationADefinition())
-        break;
-
-      // FIXME: If this should support Obj-C++, support CXXTryStmt
-      if (CompoundStmt *Body = dyn_cast_or_null<CompoundStmt>(FD->getBody())) {
-        CurFunctionDef = FD;
-        CurrentBody = Body;
-        Body =
-        cast_or_null<CompoundStmt>(RewriteFunctionBodyOrGlobalInitializer(Body));
-        FD->setBody(Body);
-        CurrentBody = 0;
-        if (PropParentMap) {
-          delete PropParentMap;
-          PropParentMap = 0;
-        }
-        // This synthesizes and inserts the block "impl" struct, invoke function,
-        // and any copy/dispose helper functions.
-        InsertBlockLiteralsWithinFunction(FD);
-        CurFunctionDef = 0;
-      }
-      break;
-    }
-    case Decl::ObjCMethod: {
-      ObjCMethodDecl *MD = cast<ObjCMethodDecl>(D);
-      if (CompoundStmt *Body = MD->getCompoundBody()) {
-        CurMethodDef = MD;
-        CurrentBody = Body;
-        Body =
-          cast_or_null<CompoundStmt>(RewriteFunctionBodyOrGlobalInitializer(Body));
-        MD->setBody(Body);
-        CurrentBody = 0;
-        if (PropParentMap) {
-          delete PropParentMap;
-          PropParentMap = 0;
-        }
-        InsertBlockLiteralsWithinMethod(MD);
-        CurMethodDef = 0;
-      }
-      break;
-    }
-    case Decl::ObjCImplementation: {
-      ObjCImplementationDecl *CI = cast<ObjCImplementationDecl>(D);
-      ClassImplementation.push_back(CI);
-      break;
-    }
-    case Decl::ObjCCategoryImpl: {
-      ObjCCategoryImplDecl *CI = cast<ObjCCategoryImplDecl>(D);
-      CategoryImplementation.push_back(CI);
-      break;
-    }
-    case Decl::Var: {
-      VarDecl *VD = cast<VarDecl>(D);
-      RewriteObjCQualifiedInterfaceTypes(VD);
-      if (isTopLevelBlockPointerType(VD->getType()))
-        RewriteBlockPointerDecl(VD);
-      else if (VD->getType()->isFunctionPointerType()) {
-        CheckFunctionPointerDecl(VD->getType(), VD);
-        if (VD->getInit()) {
-          if (CStyleCastExpr *CE = dyn_cast<CStyleCastExpr>(VD->getInit())) {
-            RewriteCastExpr(CE);
-          }
-        }
-      } else if (VD->getType()->isRecordType()) {
-        RecordDecl *RD = VD->getType()->getAs<RecordType>()->getDecl();
-        if (RD->isCompleteDefinition())
-          RewriteRecordBody(RD);
-      }
-      if (VD->getInit()) {
-        GlobalVarDecl = VD;
-        CurrentBody = VD->getInit();
-        RewriteFunctionBodyOrGlobalInitializer(VD->getInit());
-        CurrentBody = 0;
-        if (PropParentMap) {
-          delete PropParentMap;
-          PropParentMap = 0;
-        }
-        SynthesizeBlockLiterals(VD->getTypeSpecStartLoc(), VD->getName());
-        GlobalVarDecl = 0;
-          
-        // This is needed for blocks.
-        if (CStyleCastExpr *CE = dyn_cast<CStyleCastExpr>(VD->getInit())) {
-            RewriteCastExpr(CE);
-        }
-      }
-      break;
-    }
-    case Decl::TypeAlias:
-    case Decl::Typedef: {
-      if (TypedefNameDecl *TD = dyn_cast<TypedefNameDecl>(D)) {
-        if (isTopLevelBlockPointerType(TD->getUnderlyingType()))
-          RewriteBlockPointerDecl(TD);
-        else if (TD->getUnderlyingType()->isFunctionPointerType())
-          CheckFunctionPointerDecl(TD->getUnderlyingType(), TD);
-      }
-      break;
-    }
-    case Decl::CXXRecord:
-    case Decl::Record: {
-      RecordDecl *RD = cast<RecordDecl>(D);
-      if (RD->isCompleteDefinition()) 
-        RewriteRecordBody(RD);
-      break;
-    }
-    default:
-      break;
-  }
-  // Nothing yet.
-}
-
-/// Write_ProtocolExprReferencedMetadata - This routine writer out the
-/// protocol reference symbols in the for of:
-/// struct _protocol_t *PROTOCOL_REF = &PROTOCOL_METADATA.
-static void Write_ProtocolExprReferencedMetadata(ASTContext *Context, 
-                                                 ObjCProtocolDecl *PDecl,
-                                                 std::string &Result) {
-  // Also output .objc_protorefs$B section and its meta-data.
-  if (Context->getLangOpts().MicrosoftExt)
-    Result += "static ";
-  Result += "struct _protocol_t *";
-  Result += "_OBJC_PROTOCOL_REFERENCE_$_";
-  Result += PDecl->getNameAsString();
-  Result += " = &";
-  Result += "_OBJC_PROTOCOL_"; Result += PDecl->getNameAsString();
-  Result += ";\n";
-}
-
-void RewriteModernObjC::HandleTranslationUnit(ASTContext &C) {
-  if (Diags.hasErrorOccurred())
-    return;
-
-  RewriteInclude();
-
-  // Here's a great place to add any extra declarations that may be needed.
-  // Write out meta data for each @protocol(<expr>).
-  for (llvm::SmallPtrSet<ObjCProtocolDecl *,8>::iterator I = ProtocolExprDecls.begin(),
-       E = ProtocolExprDecls.end(); I != E; ++I) {
-    RewriteObjCProtocolMetaData(*I, Preamble);
-    Write_ProtocolExprReferencedMetadata(Context, (*I), Preamble);
-  }
-
-  InsertText(SM->getLocForStartOfFile(MainFileID), Preamble, false);
-  
-  if (ClassImplementation.size() || CategoryImplementation.size())
-    RewriteImplementations();
-  
-  for (unsigned i = 0, e = ObjCInterfacesSeen.size(); i < e; i++) {
-    ObjCInterfaceDecl *CDecl = ObjCInterfacesSeen[i];
-    // Write struct declaration for the class matching its ivar declarations.
-    // Note that for modern abi, this is postponed until the end of TU
-    // because class extensions and the implementation might declare their own
-    // private ivars.
-    RewriteInterfaceDecl(CDecl);
-  }
-
-  // Get the buffer corresponding to MainFileID.  If we haven't changed it, then
-  // we are done.
-  if (const RewriteBuffer *RewriteBuf =
-      Rewrite.getRewriteBufferFor(MainFileID)) {
-    //printf("Changed:\n");
-    *OutFile << std::string(RewriteBuf->begin(), RewriteBuf->end());
-  } else {
-    llvm::errs() << "No changes\n";
-  }
-
-  if (ClassImplementation.size() || CategoryImplementation.size() ||
-      ProtocolExprDecls.size()) {
-    // Rewrite Objective-c meta data*
-    std::string ResultStr;
-    RewriteMetaDataIntoBuffer(ResultStr);
-    // Emit metadata.
-    *OutFile << ResultStr;
-  }
-  // Emit ImageInfo;
-  {
-    std::string ResultStr;
-    WriteImageInfo(ResultStr);
-    *OutFile << ResultStr;
-  }
-  OutFile->flush();
-}
-
-void RewriteModernObjC::Initialize(ASTContext &context) {
-  InitializeCommon(context);
-  
-  Preamble += "#ifndef __OBJC2__\n";
-  Preamble += "#define __OBJC2__\n";
-  Preamble += "#endif\n";
-
-  // declaring objc_selector outside the parameter list removes a silly
-  // scope related warning...
-  if (IsHeader)
-    Preamble = "#pragma once\n";
-  Preamble += "struct objc_selector; struct objc_class;\n";
-  Preamble += "struct __rw_objc_super { \n\tstruct objc_object *object; ";
-  Preamble += "\n\tstruct objc_object *superClass; ";
-  // Add a constructor for creating temporary objects.
-  Preamble += "\n\t__rw_objc_super(struct objc_object *o, struct objc_object *s) ";
-  Preamble += ": object(o), superClass(s) {} ";
-  Preamble += "\n};\n";
-  
-  if (LangOpts.MicrosoftExt) {
-    // Define all sections using syntax that makes sense.
-    // These are currently generated.
-    Preamble += "\n#pragma section(\".objc_classlist$B\", long, read, write)\n";
-    Preamble += "#pragma section(\".objc_catlist$B\", long, read, write)\n";
-    Preamble += "#pragma section(\".objc_imageinfo$B\", long, read, write)\n";
-    Preamble += "#pragma section(\".objc_nlclslist$B\", long, read, write)\n";
-    Preamble += "#pragma section(\".objc_nlcatlist$B\", long, read, write)\n";
-    // These are generated but not necessary for functionality.
-    Preamble += "#pragma section(\".cat_cls_meth$B\", long, read, write)\n";
-    Preamble += "#pragma section(\".inst_meth$B\", long, read, write)\n";
-    Preamble += "#pragma section(\".cls_meth$B\", long, read, write)\n";
-    Preamble += "#pragma section(\".objc_ivar$B\", long, read, write)\n";
-    
-    // These need be generated for performance. Currently they are not,
-    // using API calls instead.
-    Preamble += "#pragma section(\".objc_selrefs$B\", long, read, write)\n";
-    Preamble += "#pragma section(\".objc_classrefs$B\", long, read, write)\n";
-    Preamble += "#pragma section(\".objc_superrefs$B\", long, read, write)\n";
-    
-  }
-  Preamble += "#ifndef _REWRITER_typedef_Protocol\n";
-  Preamble += "typedef struct objc_object Protocol;\n";
-  Preamble += "#define _REWRITER_typedef_Protocol\n";
-  Preamble += "#endif\n";
-  if (LangOpts.MicrosoftExt) {
-    Preamble += "#define __OBJC_RW_DLLIMPORT extern \"C\" __declspec(dllimport)\n";
-    Preamble += "#define __OBJC_RW_STATICIMPORT extern \"C\"\n";
-  } 
-  else
-    Preamble += "#define __OBJC_RW_DLLIMPORT extern\n";
-  
-  Preamble += "__OBJC_RW_DLLIMPORT void objc_msgSend(void);\n";
-  Preamble += "__OBJC_RW_DLLIMPORT void objc_msgSendSuper(void);\n";
-  Preamble += "__OBJC_RW_DLLIMPORT void objc_msgSend_stret(void);\n";
-  Preamble += "__OBJC_RW_DLLIMPORT void objc_msgSendSuper_stret(void);\n";
-  Preamble += "__OBJC_RW_DLLIMPORT void objc_msgSend_fpret(void);\n";
-
-  Preamble += "__OBJC_RW_DLLIMPORT struct objc_class *objc_getClass";
-  Preamble += "(const char *);\n";
-  Preamble += "__OBJC_RW_DLLIMPORT struct objc_class *class_getSuperclass";
-  Preamble += "(struct objc_class *);\n";
-  Preamble += "__OBJC_RW_DLLIMPORT struct objc_class *objc_getMetaClass";
-  Preamble += "(const char *);\n";
-  Preamble += "__OBJC_RW_DLLIMPORT void objc_exception_throw( struct objc_object *);\n";
-  // @synchronized hooks.
-  Preamble += "__OBJC_RW_DLLIMPORT void objc_sync_enter( struct objc_object *);\n";
-  Preamble += "__OBJC_RW_DLLIMPORT void objc_sync_exit( struct objc_object *);\n";
-  Preamble += "__OBJC_RW_DLLIMPORT Protocol *objc_getProtocol(const char *);\n";
-  Preamble += "#ifndef __FASTENUMERATIONSTATE\n";
-  Preamble += "struct __objcFastEnumerationState {\n\t";
-  Preamble += "unsigned long state;\n\t";
-  Preamble += "void **itemsPtr;\n\t";
-  Preamble += "unsigned long *mutationsPtr;\n\t";
-  Preamble += "unsigned long extra[5];\n};\n";
-  Preamble += "__OBJC_RW_DLLIMPORT void objc_enumerationMutation(struct objc_object *);\n";
-  Preamble += "#define __FASTENUMERATIONSTATE\n";
-  Preamble += "#endif\n";
-  Preamble += "#ifndef __NSCONSTANTSTRINGIMPL\n";
-  Preamble += "struct __NSConstantStringImpl {\n";
-  Preamble += "  int *isa;\n";
-  Preamble += "  int flags;\n";
-  Preamble += "  char *str;\n";
-  Preamble += "  long length;\n";
-  Preamble += "};\n";
-  Preamble += "#ifdef CF_EXPORT_CONSTANT_STRING\n";
-  Preamble += "extern \"C\" __declspec(dllexport) int __CFConstantStringClassReference[];\n";
-  Preamble += "#else\n";
-  Preamble += "__OBJC_RW_DLLIMPORT int __CFConstantStringClassReference[];\n";
-  Preamble += "#endif\n";
-  Preamble += "#define __NSCONSTANTSTRINGIMPL\n";
-  Preamble += "#endif\n";
-  // Blocks preamble.
-  Preamble += "#ifndef BLOCK_IMPL\n";
-  Preamble += "#define BLOCK_IMPL\n";
-  Preamble += "struct __block_impl {\n";
-  Preamble += "  void *isa;\n";
-  Preamble += "  int Flags;\n";
-  Preamble += "  int Reserved;\n";
-  Preamble += "  void *FuncPtr;\n";
-  Preamble += "};\n";
-  Preamble += "// Runtime copy/destroy helper functions (from Block_private.h)\n";
-  Preamble += "#ifdef __OBJC_EXPORT_BLOCKS\n";
-  Preamble += "extern \"C\" __declspec(dllexport) "
-  "void _Block_object_assign(void *, const void *, const int);\n";
-  Preamble += "extern \"C\" __declspec(dllexport) void _Block_object_dispose(const void *, const int);\n";
-  Preamble += "extern \"C\" __declspec(dllexport) void *_NSConcreteGlobalBlock[32];\n";
-  Preamble += "extern \"C\" __declspec(dllexport) void *_NSConcreteStackBlock[32];\n";
-  Preamble += "#else\n";
-  Preamble += "__OBJC_RW_DLLIMPORT void _Block_object_assign(void *, const void *, const int);\n";
-  Preamble += "__OBJC_RW_DLLIMPORT void _Block_object_dispose(const void *, const int);\n";
-  Preamble += "__OBJC_RW_DLLIMPORT void *_NSConcreteGlobalBlock[32];\n";
-  Preamble += "__OBJC_RW_DLLIMPORT void *_NSConcreteStackBlock[32];\n";
-  Preamble += "#endif\n";
-  Preamble += "#endif\n";
-  if (LangOpts.MicrosoftExt) {
-    Preamble += "#undef __OBJC_RW_DLLIMPORT\n";
-    Preamble += "#undef __OBJC_RW_STATICIMPORT\n";
-    Preamble += "#ifndef KEEP_ATTRIBUTES\n";  // We use this for clang tests.
-    Preamble += "#define __attribute__(X)\n";
-    Preamble += "#endif\n";
-    Preamble += "#ifndef __weak\n";
-    Preamble += "#define __weak\n";
-    Preamble += "#endif\n";
-    Preamble += "#ifndef __block\n";
-    Preamble += "#define __block\n";
-    Preamble += "#endif\n";
-  }
-  else {
-    Preamble += "#define __block\n";
-    Preamble += "#define __weak\n";
-  }
-  
-  // Declarations required for modern objective-c array and dictionary literals.
-  Preamble += "\n#include <stdarg.h>\n";
-  Preamble += "struct __NSContainer_literal {\n";
-  Preamble += "  void * *arr;\n";
-  Preamble += "  __NSContainer_literal (unsigned int count, ...) {\n";
-  Preamble += "\tva_list marker;\n";
-  Preamble += "\tva_start(marker, count);\n";
-  Preamble += "\tarr = new void *[count];\n";
-  Preamble += "\tfor (unsigned i = 0; i < count; i++)\n";
-  Preamble += "\t  arr[i] = va_arg(marker, void *);\n";
-  Preamble += "\tva_end( marker );\n";
-  Preamble += "  };\n";
-  Preamble += "  ~__NSContainer_literal() {\n";
-  Preamble += "\tdelete[] arr;\n";
-  Preamble += "  }\n";
-  Preamble += "};\n";
-  
-  // Declaration required for implementation of @autoreleasepool statement.
-  Preamble += "extern \"C\" __declspec(dllimport) void * objc_autoreleasePoolPush(void);\n";
-  Preamble += "extern \"C\" __declspec(dllimport) void objc_autoreleasePoolPop(void *);\n\n";
-  Preamble += "struct __AtAutoreleasePool {\n";
-  Preamble += "  __AtAutoreleasePool() {atautoreleasepoolobj = objc_autoreleasePoolPush();}\n";
-  Preamble += "  ~__AtAutoreleasePool() {objc_autoreleasePoolPop(atautoreleasepoolobj);}\n";
-  Preamble += "  void * atautoreleasepoolobj;\n";
-  Preamble += "};\n";
-  
-  // NOTE! Windows uses LLP64 for 64bit mode. So, cast pointer to long long
-  // as this avoids warning in any 64bit/32bit compilation model.
-  Preamble += "\n#define __OFFSETOFIVAR__(TYPE, MEMBER) ((long long) &((TYPE *)0)->MEMBER)\n";
-}
-
-/// RewriteIvarOffsetComputation - This rutine synthesizes computation of
-/// ivar offset.
-void RewriteModernObjC::RewriteIvarOffsetComputation(ObjCIvarDecl *ivar,
-                                                         std::string &Result) {
-  if (ivar->isBitField()) {
-    // FIXME: The hack below doesn't work for bitfields. For now, we simply
-    // place all bitfields at offset 0.
-    Result += "0";
-  } else {
-    Result += "__OFFSETOFIVAR__(struct ";
-    Result += ivar->getContainingInterface()->getNameAsString();
-    if (LangOpts.MicrosoftExt)
-      Result += "_IMPL";
-    Result += ", ";
-    Result += ivar->getNameAsString();
-    Result += ")";
-  }
-}
-
-/// WriteModernMetadataDeclarations - Writes out metadata declarations for modern ABI.
-/// struct _prop_t {
-///   const char *name;
-///   char *attributes;
-/// }
-
-/// struct _prop_list_t {
-///   uint32_t entsize;      // sizeof(struct _prop_t)
-///   uint32_t count_of_properties;
-///   struct _prop_t prop_list[count_of_properties];
-/// }
-
-/// struct _protocol_t;
-
-/// struct _protocol_list_t {
-///   long protocol_count;   // Note, this is 32/64 bit
-///   struct _protocol_t * protocol_list[protocol_count];
-/// }
-
-/// struct _objc_method {
-///   SEL _cmd;
-///   const char *method_type;
-///   char *_imp;
-/// }
-
-/// struct _method_list_t {
-///   uint32_t entsize;  // sizeof(struct _objc_method)
-///   uint32_t method_count;
-///   struct _objc_method method_list[method_count];
-/// }
-
-/// struct _protocol_t {
-///   id isa;  // NULL
-///   const char *protocol_name;
-///   const struct _protocol_list_t * protocol_list; // super protocols
-///   const struct method_list_t *instance_methods;
-///   const struct method_list_t *class_methods;
-///   const struct method_list_t *optionalInstanceMethods;
-///   const struct method_list_t *optionalClassMethods;
-///   const struct _prop_list_t * properties;
-///   const uint32_t size;  // sizeof(struct _protocol_t)
-///   const uint32_t flags;  // = 0
-///   const char ** extendedMethodTypes;
-/// }
-
-/// struct _ivar_t {
-///   unsigned long int *offset;  // pointer to ivar offset location
-///   const char *name;
-///   const char *type;
-///   uint32_t alignment;
-///   uint32_t size;
-/// }
-
-/// struct _ivar_list_t {
-///   uint32 entsize;  // sizeof(struct _ivar_t)
-///   uint32 count;
-///   struct _ivar_t list[count];
-/// }
-
-/// struct _class_ro_t {
-///   uint32_t flags;
-///   uint32_t instanceStart;
-///   uint32_t instanceSize;
-///   uint32_t reserved;  // only when building for 64bit targets
-///   const uint8_t *ivarLayout;
-///   const char *name;
-///   const struct _method_list_t *baseMethods;
-///   const struct _protocol_list_t *baseProtocols;
-///   const struct _ivar_list_t *ivars;
-///   const uint8_t *weakIvarLayout;
-///   const struct _prop_list_t *properties;
-/// }
-
-/// struct _class_t {
-///   struct _class_t *isa;
-///   struct _class_t *superclass;
-///   void *cache;
-///   IMP *vtable;
-///   struct _class_ro_t *ro;
-/// }
-
-/// struct _category_t {
-///   const char *name;
-///   struct _class_t *cls;
-///   const struct _method_list_t *instance_methods;
-///   const struct _method_list_t *class_methods;
-///   const struct _protocol_list_t *protocols;
-///   const struct _prop_list_t *properties;
-/// }
-
-/// MessageRefTy - LLVM for:
-/// struct _message_ref_t {
-///   IMP messenger;
-///   SEL name;
-/// };
-
-/// SuperMessageRefTy - LLVM for:
-/// struct _super_message_ref_t {
-///   SUPER_IMP messenger;
-///   SEL name;
-/// };
-
-static void WriteModernMetadataDeclarations(ASTContext *Context, std::string &Result) {
-  static bool meta_data_declared = false;
-  if (meta_data_declared)
-    return;
-  
-  Result += "\nstruct _prop_t {\n";
-  Result += "\tconst char *name;\n";
-  Result += "\tconst char *attributes;\n";
-  Result += "};\n";
-  
-  Result += "\nstruct _protocol_t;\n";
-  
-  Result += "\nstruct _objc_method {\n";
-  Result += "\tstruct objc_selector * _cmd;\n";
-  Result += "\tconst char *method_type;\n";
-  Result += "\tvoid  *_imp;\n";
-  Result += "};\n";
-  
-  Result += "\nstruct _protocol_t {\n";
-  Result += "\tvoid * isa;  // NULL\n";
-  Result += "\tconst char *protocol_name;\n";
-  Result += "\tconst struct _protocol_list_t * protocol_list; // super protocols\n";
-  Result += "\tconst struct method_list_t *instance_methods;\n";
-  Result += "\tconst struct method_list_t *class_methods;\n";
-  Result += "\tconst struct method_list_t *optionalInstanceMethods;\n";
-  Result += "\tconst struct method_list_t *optionalClassMethods;\n";
-  Result += "\tconst struct _prop_list_t * properties;\n";
-  Result += "\tconst unsigned int size;  // sizeof(struct _protocol_t)\n";
-  Result += "\tconst unsigned int flags;  // = 0\n";
-  Result += "\tconst char ** extendedMethodTypes;\n";
-  Result += "};\n";
-  
-  Result += "\nstruct _ivar_t {\n";
-  Result += "\tunsigned long int *offset;  // pointer to ivar offset location\n";
-  Result += "\tconst char *name;\n";
-  Result += "\tconst char *type;\n";
-  Result += "\tunsigned int alignment;\n";
-  Result += "\tunsigned int  size;\n";
-  Result += "};\n";
-  
-  Result += "\nstruct _class_ro_t {\n";
-  Result += "\tunsigned int flags;\n";
-  Result += "\tunsigned int instanceStart;\n";
-  Result += "\tunsigned int instanceSize;\n";
-  const llvm::Triple &Triple(Context->getTargetInfo().getTriple());
-  if (Triple.getArch() == llvm::Triple::x86_64)
-    Result += "\tunsigned int reserved;\n";
-  Result += "\tconst unsigned char *ivarLayout;\n";
-  Result += "\tconst char *name;\n";
-  Result += "\tconst struct _method_list_t *baseMethods;\n";
-  Result += "\tconst struct _objc_protocol_list *baseProtocols;\n";
-  Result += "\tconst struct _ivar_list_t *ivars;\n";
-  Result += "\tconst unsigned char *weakIvarLayout;\n";
-  Result += "\tconst struct _prop_list_t *properties;\n";
-  Result += "};\n";
-  
-  Result += "\nstruct _class_t {\n";
-  Result += "\tstruct _class_t *isa;\n";
-  Result += "\tstruct _class_t *superclass;\n";
-  Result += "\tvoid *cache;\n";
-  Result += "\tvoid *vtable;\n";
-  Result += "\tstruct _class_ro_t *ro;\n";
-  Result += "};\n";
-  
-  Result += "\nstruct _category_t {\n";
-  Result += "\tconst char *name;\n";
-  Result += "\tstruct _class_t *cls;\n";
-  Result += "\tconst struct _method_list_t *instance_methods;\n";
-  Result += "\tconst struct _method_list_t *class_methods;\n";
-  Result += "\tconst struct _protocol_list_t *protocols;\n";
-  Result += "\tconst struct _prop_list_t *properties;\n";
-  Result += "};\n";
-  
-  Result += "extern \"C\" __declspec(dllimport) struct objc_cache _objc_empty_cache;\n";
-  Result += "#pragma warning(disable:4273)\n";
-  meta_data_declared = true;
-}
-
-static void Write_protocol_list_t_TypeDecl(std::string &Result,
-                                           long super_protocol_count) {
-  Result += "struct /*_protocol_list_t*/"; Result += " {\n";
-  Result += "\tlong protocol_count;  // Note, this is 32/64 bit\n";
-  Result += "\tstruct _protocol_t *super_protocols[";
-  Result += utostr(super_protocol_count); Result += "];\n";
-  Result += "}";
-}
-
-static void Write_method_list_t_TypeDecl(std::string &Result,
-                                         unsigned int method_count) {
-  Result += "struct /*_method_list_t*/"; Result += " {\n";
-  Result += "\tunsigned int entsize;  // sizeof(struct _objc_method)\n";
-  Result += "\tunsigned int method_count;\n";
-  Result += "\tstruct _objc_method method_list[";
-  Result += utostr(method_count); Result += "];\n";
-  Result += "}";
-}
-
-static void Write__prop_list_t_TypeDecl(std::string &Result,
-                                        unsigned int property_count) {
-  Result += "struct /*_prop_list_t*/"; Result += " {\n";
-  Result += "\tunsigned int entsize;  // sizeof(struct _prop_t)\n";
-  Result += "\tunsigned int count_of_properties;\n";
-  Result += "\tstruct _prop_t prop_list[";
-  Result += utostr(property_count); Result += "];\n";
-  Result += "}";
-}
-
-static void Write__ivar_list_t_TypeDecl(std::string &Result,
-                                        unsigned int ivar_count) {
-  Result += "struct /*_ivar_list_t*/"; Result += " {\n";
-  Result += "\tunsigned int entsize;  // sizeof(struct _prop_t)\n";
-  Result += "\tunsigned int count;\n";
-  Result += "\tstruct _ivar_t ivar_list[";
-  Result += utostr(ivar_count); Result += "];\n";
-  Result += "}";
-}
-
-static void Write_protocol_list_initializer(ASTContext *Context, std::string &Result,
-                                            ArrayRef<ObjCProtocolDecl *> SuperProtocols,
-                                            StringRef VarName,
-                                            StringRef ProtocolName) {
-  if (SuperProtocols.size() > 0) {
-    Result += "\nstatic ";
-    Write_protocol_list_t_TypeDecl(Result, SuperProtocols.size());
-    Result += " "; Result += VarName;
-    Result += ProtocolName; 
-    Result += " __attribute__ ((used, section (\"__DATA,__objc_const\"))) = {\n";
-    Result += "\t"; Result += utostr(SuperProtocols.size()); Result += ",\n";
-    for (unsigned i = 0, e = SuperProtocols.size(); i < e; i++) {
-      ObjCProtocolDecl *SuperPD = SuperProtocols[i];
-      Result += "\t&"; Result += "_OBJC_PROTOCOL_"; 
-      Result += SuperPD->getNameAsString();
-      if (i == e-1)
-        Result += "\n};\n";
-      else
-        Result += ",\n";
-    }
-  }
-}
-
-static void Write_method_list_t_initializer(RewriteModernObjC &RewriteObj,
-                                            ASTContext *Context, std::string &Result,
-                                            ArrayRef<ObjCMethodDecl *> Methods,
-                                            StringRef VarName,
-                                            StringRef TopLevelDeclName,
-                                            bool MethodImpl) {
-  if (Methods.size() > 0) {
-    Result += "\nstatic ";
-    Write_method_list_t_TypeDecl(Result, Methods.size());
-    Result += " "; Result += VarName;
-    Result += TopLevelDeclName; 
-    Result += " __attribute__ ((used, section (\"__DATA,__objc_const\"))) = {\n";
-    Result += "\t"; Result += "sizeof(_objc_method)"; Result += ",\n";
-    Result += "\t"; Result += utostr(Methods.size()); Result += ",\n";
-    for (unsigned i = 0, e = Methods.size(); i < e; i++) {
-      ObjCMethodDecl *MD = Methods[i];
-      if (i == 0)
-        Result += "\t{{(struct objc_selector *)\"";
-      else
-        Result += "\t{(struct objc_selector *)\"";
-      Result += (MD)->getSelector().getAsString(); Result += "\"";
-      Result += ", ";
-      std::string MethodTypeString;
-      Context->getObjCEncodingForMethodDecl(MD, MethodTypeString);
-      Result += "\""; Result += MethodTypeString; Result += "\"";
-      Result += ", ";
-      if (!MethodImpl)
-        Result += "0";
-      else {
-        Result += "(void *)";
-        Result += RewriteObj.MethodInternalNames[MD];
-      }
-      if (i  == e-1)
-        Result += "}}\n";
-      else
-        Result += "},\n";
-    }
-    Result += "};\n";
-  }
-}
-
-static void Write_prop_list_t_initializer(RewriteModernObjC &RewriteObj,
-                                           ASTContext *Context, std::string &Result,
-                                           ArrayRef<ObjCPropertyDecl *> Properties,
-                                           const Decl *Container,
-                                           StringRef VarName,
-                                           StringRef ProtocolName) {
-  if (Properties.size() > 0) {
-    Result += "\nstatic ";
-    Write__prop_list_t_TypeDecl(Result, Properties.size());
-    Result += " "; Result += VarName;
-    Result += ProtocolName; 
-    Result += " __attribute__ ((used, section (\"__DATA,__objc_const\"))) = {\n";
-    Result += "\t"; Result += "sizeof(_prop_t)"; Result += ",\n";
-    Result += "\t"; Result += utostr(Properties.size()); Result += ",\n";
-    for (unsigned i = 0, e = Properties.size(); i < e; i++) {
-      ObjCPropertyDecl *PropDecl = Properties[i];
-      if (i == 0)
-        Result += "\t{{\"";
-      else
-        Result += "\t{\"";
-      Result += PropDecl->getName(); Result += "\",";
-      std::string PropertyTypeString, QuotePropertyTypeString;
-      Context->getObjCEncodingForPropertyDecl(PropDecl, Container, PropertyTypeString);
-      RewriteObj.QuoteDoublequotes(PropertyTypeString, QuotePropertyTypeString);
-      Result += "\""; Result += QuotePropertyTypeString; Result += "\"";
-      if (i  == e-1)
-        Result += "}}\n";
-      else
-        Result += "},\n";
-    }
-    Result += "};\n";
-  }
-}
-
-// Metadata flags
-enum MetaDataDlags {
-  CLS = 0x0,
-  CLS_META = 0x1,
-  CLS_ROOT = 0x2,
-  OBJC2_CLS_HIDDEN = 0x10,
-  CLS_EXCEPTION = 0x20,
-  
-  /// (Obsolete) ARC-specific: this class has a .release_ivars method
-  CLS_HAS_IVAR_RELEASER = 0x40,
-  /// class was compiled with -fobjc-arr
-  CLS_COMPILED_BY_ARC = 0x80  // (1<<7)
-};
-
-static void Write__class_ro_t_initializer(ASTContext *Context, std::string &Result, 
-                                          unsigned int flags, 
-                                          const std::string &InstanceStart, 
-                                          const std::string &InstanceSize,
-                                          ArrayRef<ObjCMethodDecl *>baseMethods,
-                                          ArrayRef<ObjCProtocolDecl *>baseProtocols,
-                                          ArrayRef<ObjCIvarDecl *>ivars,
-                                          ArrayRef<ObjCPropertyDecl *>Properties,
-                                          StringRef VarName,
-                                          StringRef ClassName) {
-  Result += "\nstatic struct _class_ro_t ";
-  Result += VarName; Result += ClassName;
-  Result += " __attribute__ ((used, section (\"__DATA,__objc_const\"))) = {\n";
-  Result += "\t"; 
-  Result += llvm::utostr(flags); Result += ", "; 
-  Result += InstanceStart; Result += ", ";
-  Result += InstanceSize; Result += ", \n";
-  Result += "\t";
-  const llvm::Triple &Triple(Context->getTargetInfo().getTriple());
-  if (Triple.getArch() == llvm::Triple::x86_64)
-    // uint32_t const reserved; // only when building for 64bit targets
-    Result += "(unsigned int)0, \n\t";
-  // const uint8_t * const ivarLayout;
-  Result += "0, \n\t";
-  Result += "\""; Result += ClassName; Result += "\",\n\t";
-  bool metaclass = ((flags & CLS_META) != 0);
-  if (baseMethods.size() > 0) {
-    Result += "(const struct _method_list_t *)&";
-    if (metaclass)
-      Result += "_OBJC_$_CLASS_METHODS_";
-    else
-      Result += "_OBJC_$_INSTANCE_METHODS_";
-    Result += ClassName;
-    Result += ",\n\t";
-  }
-  else
-    Result += "0, \n\t";
-
-  if (!metaclass && baseProtocols.size() > 0) {
-    Result += "(const struct _objc_protocol_list *)&";
-    Result += "_OBJC_CLASS_PROTOCOLS_$_"; Result += ClassName;
-    Result += ",\n\t";
-  }
-  else
-    Result += "0, \n\t";
-
-  if (!metaclass && ivars.size() > 0) {
-    Result += "(const struct _ivar_list_t *)&";
-    Result += "_OBJC_$_INSTANCE_VARIABLES_"; Result += ClassName;
-    Result += ",\n\t";
-  }
-  else
-    Result += "0, \n\t";
-
-  // weakIvarLayout
-  Result += "0, \n\t";
-  if (!metaclass && Properties.size() > 0) {
-    Result += "(const struct _prop_list_t *)&";
-    Result += "_OBJC_$_PROP_LIST_"; Result += ClassName;
-    Result += ",\n";
-  }
-  else
-    Result += "0, \n";
-
-  Result += "};\n";
-}
-
-static void Write_class_t(ASTContext *Context, std::string &Result,
-                          StringRef VarName,
-                          const ObjCInterfaceDecl *CDecl, bool metaclass) {
-  bool rootClass = (!CDecl->getSuperClass());
-  const ObjCInterfaceDecl *RootClass = CDecl;
-  
-  if (!rootClass) {
-    // Find the Root class
-    RootClass = CDecl->getSuperClass();
-    while (RootClass->getSuperClass()) {
-      RootClass = RootClass->getSuperClass();
-    }
-  }
-
-  if (metaclass && rootClass) {
-    // Need to handle a case of use of forward declaration.
-    Result += "\n";
-    Result += "extern \"C\" ";
-    if (CDecl->getImplementation())
-      Result += "__declspec(dllexport) ";
-    else
-      Result += "__declspec(dllimport) ";
-    
-    Result += "struct _class_t OBJC_CLASS_$_";
-    Result += CDecl->getNameAsString();
-    Result += ";\n";
-  }
-  // Also, for possibility of 'super' metadata class not having been defined yet.
-  if (!rootClass) {
-    ObjCInterfaceDecl *SuperClass = CDecl->getSuperClass();
-    Result += "\n";
-    Result += "extern \"C\" ";
-    if (SuperClass->getImplementation())
-      Result += "__declspec(dllexport) ";
-    else
-      Result += "__declspec(dllimport) ";
-
-    Result += "struct _class_t "; 
-    Result += VarName;
-    Result += SuperClass->getNameAsString();
-    Result += ";\n";
-    
-    if (metaclass && RootClass != SuperClass) {
-      Result += "extern \"C\" ";
-      if (RootClass->getImplementation())
-        Result += "__declspec(dllexport) ";
-      else
-        Result += "__declspec(dllimport) ";
-
-      Result += "struct _class_t "; 
-      Result += VarName;
-      Result += RootClass->getNameAsString();
-      Result += ";\n";
-    }
-  }
-  
-  Result += "\nextern \"C\" __declspec(dllexport) struct _class_t "; 
-  Result += VarName; Result += CDecl->getNameAsString();
-  Result += " __attribute__ ((used, section (\"__DATA,__objc_data\"))) = {\n";
-  Result += "\t";
-  if (metaclass) {
-    if (!rootClass) {
-      Result += "0, // &"; Result += VarName;
-      Result += RootClass->getNameAsString();
-      Result += ",\n\t";
-      Result += "0, // &"; Result += VarName;
-      Result += CDecl->getSuperClass()->getNameAsString();
-      Result += ",\n\t";
-    }
-    else {
-      Result += "0, // &"; Result += VarName; 
-      Result += CDecl->getNameAsString();
-      Result += ",\n\t";
-      Result += "0, // &OBJC_CLASS_$_"; Result += CDecl->getNameAsString();
-      Result += ",\n\t";
-    }
-  }
-  else {
-    Result += "0, // &OBJC_METACLASS_$_"; 
-    Result += CDecl->getNameAsString();
-    Result += ",\n\t";
-    if (!rootClass) {
-      Result += "0, // &"; Result += VarName;
-      Result += CDecl->getSuperClass()->getNameAsString();
-      Result += ",\n\t";
-    }
-    else 
-      Result += "0,\n\t";
-  }
-  Result += "0, // (void *)&_objc_empty_cache,\n\t";
-  Result += "0, // unused, was (void *)&_objc_empty_vtable,\n\t";
-  if (metaclass)
-    Result += "&_OBJC_METACLASS_RO_$_";
-  else
-    Result += "&_OBJC_CLASS_RO_$_";
-  Result += CDecl->getNameAsString();
-  Result += ",\n};\n";
-  
-  // Add static function to initialize some of the meta-data fields.
-  // avoid doing it twice.
-  if (metaclass)
-    return;
-  
-  const ObjCInterfaceDecl *SuperClass = 
-    rootClass ? CDecl : CDecl->getSuperClass();
-  
-  Result += "static void OBJC_CLASS_SETUP_$_";
-  Result += CDecl->getNameAsString();
-  Result += "(void ) {\n";
-  Result += "\tOBJC_METACLASS_$_"; Result += CDecl->getNameAsString();
-  Result += ".isa = "; Result += "&OBJC_METACLASS_$_";
-  Result += RootClass->getNameAsString(); Result += ";\n";
-  
-  Result += "\tOBJC_METACLASS_$_"; Result += CDecl->getNameAsString();
-  Result += ".superclass = ";
-  if (rootClass)
-    Result += "&OBJC_CLASS_$_";
-  else
-     Result += "&OBJC_METACLASS_$_";
-
-  Result += SuperClass->getNameAsString(); Result += ";\n";
-  
-  Result += "\tOBJC_METACLASS_$_"; Result += CDecl->getNameAsString();
-  Result += ".cache = "; Result += "&_objc_empty_cache"; Result += ";\n";
-  
-  Result += "\tOBJC_CLASS_$_"; Result += CDecl->getNameAsString();
-  Result += ".isa = "; Result += "&OBJC_METACLASS_$_";
-  Result += CDecl->getNameAsString(); Result += ";\n";
-  
-  if (!rootClass) {
-    Result += "\tOBJC_CLASS_$_"; Result += CDecl->getNameAsString();
-    Result += ".superclass = "; Result += "&OBJC_CLASS_$_";
-    Result += SuperClass->getNameAsString(); Result += ";\n";
-  }
-  
-  Result += "\tOBJC_CLASS_$_"; Result += CDecl->getNameAsString();
-  Result += ".cache = "; Result += "&_objc_empty_cache"; Result += ";\n";
-  Result += "}\n";
-}
-
-static void Write_category_t(RewriteModernObjC &RewriteObj, ASTContext *Context, 
-                             std::string &Result,
-                             ObjCCategoryDecl *CatDecl,
-                             ObjCInterfaceDecl *ClassDecl,
-                             ArrayRef<ObjCMethodDecl *> InstanceMethods,
-                             ArrayRef<ObjCMethodDecl *> ClassMethods,
-                             ArrayRef<ObjCProtocolDecl *> RefedProtocols,
-                             ArrayRef<ObjCPropertyDecl *> ClassProperties) {
-  StringRef CatName = CatDecl->getName();
-  StringRef ClassName = ClassDecl->getName();
-  // must declare an extern class object in case this class is not implemented 
-  // in this TU.
-  Result += "\n";
-  Result += "extern \"C\" ";
-  if (ClassDecl->getImplementation())
-    Result += "__declspec(dllexport) ";
-  else
-    Result += "__declspec(dllimport) ";
-  
-  Result += "struct _class_t ";
-  Result += "OBJC_CLASS_$_"; Result += ClassName;
-  Result += ";\n";
-  
-  Result += "\nstatic struct _category_t ";
-  Result += "_OBJC_$_CATEGORY_";
-  Result += ClassName; Result += "_$_"; Result += CatName;
-  Result += " __attribute__ ((used, section (\"__DATA,__objc_const\"))) = \n";
-  Result += "{\n";
-  Result += "\t\""; Result += ClassName; Result += "\",\n";
-  Result += "\t0, // &"; Result += "OBJC_CLASS_$_"; Result += ClassName;
-  Result += ",\n";
-  if (InstanceMethods.size() > 0) {
-    Result += "\t(const struct _method_list_t *)&";  
-    Result += "_OBJC_$_CATEGORY_INSTANCE_METHODS_";
-    Result += ClassName; Result += "_$_"; Result += CatName;
-    Result += ",\n";
-  }
-  else
-    Result += "\t0,\n";
-  
-  if (ClassMethods.size() > 0) {
-    Result += "\t(const struct _method_list_t *)&";  
-    Result += "_OBJC_$_CATEGORY_CLASS_METHODS_";
-    Result += ClassName; Result += "_$_"; Result += CatName;
-    Result += ",\n";
-  }
-  else
-    Result += "\t0,\n";
-  
-  if (RefedProtocols.size() > 0) {
-    Result += "\t(const struct _protocol_list_t *)&";  
-    Result += "_OBJC_CATEGORY_PROTOCOLS_$_";
-    Result += ClassName; Result += "_$_"; Result += CatName;
-    Result += ",\n";
-  }
-  else
-    Result += "\t0,\n";
-  
-  if (ClassProperties.size() > 0) {
-    Result += "\t(const struct _prop_list_t *)&";  Result += "_OBJC_$_PROP_LIST_";
-    Result += ClassName; Result += "_$_"; Result += CatName;
-    Result += ",\n";
-  }
-  else
-    Result += "\t0,\n";
-  
-  Result += "};\n";
-  
-  // Add static function to initialize the class pointer in the category structure.
-  Result += "static void OBJC_CATEGORY_SETUP_$_";
-  Result += ClassDecl->getNameAsString();
-  Result += "_$_";
-  Result += CatName;
-  Result += "(void ) {\n";
-  Result += "\t_OBJC_$_CATEGORY_"; 
-  Result += ClassDecl->getNameAsString();
-  Result += "_$_";
-  Result += CatName;
-  Result += ".cls = "; Result += "&OBJC_CLASS_$_"; Result += ClassName;
-  Result += ";\n}\n";
-}
-
-static void Write__extendedMethodTypes_initializer(RewriteModernObjC &RewriteObj,
-                                           ASTContext *Context, std::string &Result,
-                                           ArrayRef<ObjCMethodDecl *> Methods,
-                                           StringRef VarName,
-                                           StringRef ProtocolName) {
-  if (Methods.size() == 0)
-    return;
-  
-  Result += "\nstatic const char *";
-  Result += VarName; Result += ProtocolName;
-  Result += " [] __attribute__ ((used, section (\"__DATA,__objc_const\"))) = \n";
-  Result += "{\n";
-  for (unsigned i = 0, e = Methods.size(); i < e; i++) {
-    ObjCMethodDecl *MD = Methods[i];
-    std::string MethodTypeString, QuoteMethodTypeString;
-    Context->getObjCEncodingForMethodDecl(MD, MethodTypeString, true);
-    RewriteObj.QuoteDoublequotes(MethodTypeString, QuoteMethodTypeString);
-    Result += "\t\""; Result += QuoteMethodTypeString; Result += "\"";
-    if (i == e-1)
-      Result += "\n};\n";
-    else {
-      Result += ",\n";
-    }
-  }
-}
-
-static void Write_IvarOffsetVar(RewriteModernObjC &RewriteObj,
-                                ASTContext *Context,
-                                std::string &Result, 
-                                ArrayRef<ObjCIvarDecl *> Ivars, 
-                                ObjCInterfaceDecl *CDecl) {
-  // FIXME. visibilty of offset symbols may have to be set; for Darwin
-  // this is what happens:
-  /**
-   if (Ivar->getAccessControl() == ObjCIvarDecl::Private ||
-       Ivar->getAccessControl() == ObjCIvarDecl::Package ||
-       Class->getVisibility() == HiddenVisibility)
-     Visibility shoud be: HiddenVisibility;
-   else
-     Visibility shoud be: DefaultVisibility;
-  */
-  
-  Result += "\n";
-  for (unsigned i =0, e = Ivars.size(); i < e; i++) {
-    ObjCIvarDecl *IvarDecl = Ivars[i];
-    if (Context->getLangOpts().MicrosoftExt)
-      Result += "__declspec(allocate(\".objc_ivar$B\")) ";
-    
-    if (!Context->getLangOpts().MicrosoftExt ||
-        IvarDecl->getAccessControl() == ObjCIvarDecl::Private ||
-        IvarDecl->getAccessControl() == ObjCIvarDecl::Package)
-      Result += "extern \"C\" unsigned long int "; 
-    else
-      Result += "extern \"C\" __declspec(dllexport) unsigned long int ";
-    WriteInternalIvarName(CDecl, IvarDecl, Result);
-    Result += " __attribute__ ((used, section (\"__DATA,__objc_ivar\")))";
-    Result += " = ";
-    RewriteObj.RewriteIvarOffsetComputation(IvarDecl, Result);
-    Result += ";\n";
-  }
-}
-
-static void Write__ivar_list_t_initializer(RewriteModernObjC &RewriteObj,
-                                           ASTContext *Context, std::string &Result,
-                                           ArrayRef<ObjCIvarDecl *> Ivars,
-                                           StringRef VarName,
-                                           ObjCInterfaceDecl *CDecl) {
-  if (Ivars.size() > 0) {
-    Write_IvarOffsetVar(RewriteObj, Context, Result, Ivars, CDecl);
-    
-    Result += "\nstatic ";
-    Write__ivar_list_t_TypeDecl(Result, Ivars.size());
-    Result += " "; Result += VarName;
-    Result += CDecl->getNameAsString();
-    Result += " __attribute__ ((used, section (\"__DATA,__objc_const\"))) = {\n";
-    Result += "\t"; Result += "sizeof(_ivar_t)"; Result += ",\n";
-    Result += "\t"; Result += utostr(Ivars.size()); Result += ",\n";
-    for (unsigned i =0, e = Ivars.size(); i < e; i++) {
-      ObjCIvarDecl *IvarDecl = Ivars[i];
-      if (i == 0)
-        Result += "\t{{";
-      else
-        Result += "\t {";
-      Result += "(unsigned long int *)&";
-      WriteInternalIvarName(CDecl, IvarDecl, Result);
-      Result += ", ";
-      
-      Result += "\""; Result += IvarDecl->getName(); Result += "\", ";
-      std::string IvarTypeString, QuoteIvarTypeString;
-      Context->getObjCEncodingForType(IvarDecl->getType(), IvarTypeString,
-                                      IvarDecl);
-      RewriteObj.QuoteDoublequotes(IvarTypeString, QuoteIvarTypeString);
-      Result += "\""; Result += QuoteIvarTypeString; Result += "\", ";
-      
-      // FIXME. this alignment represents the host alignment and need be changed to
-      // represent the target alignment.
-      unsigned Align = Context->getTypeAlign(IvarDecl->getType())/8;
-      Align = llvm::Log2_32(Align);
-      Result += llvm::utostr(Align); Result += ", ";
-      CharUnits Size = Context->getTypeSizeInChars(IvarDecl->getType());
-      Result += llvm::utostr(Size.getQuantity());
-      if (i  == e-1)
-        Result += "}}\n";
-      else
-        Result += "},\n";
-    }
-    Result += "};\n";
-  }
-}
-
-/// RewriteObjCProtocolMetaData - Rewrite protocols meta-data.
-void RewriteModernObjC::RewriteObjCProtocolMetaData(ObjCProtocolDecl *PDecl, 
-                                                    std::string &Result) {
-  
-  // Do not synthesize the protocol more than once.
-  if (ObjCSynthesizedProtocols.count(PDecl->getCanonicalDecl()))
-    return;
-  WriteModernMetadataDeclarations(Context, Result);
-  
-  if (ObjCProtocolDecl *Def = PDecl->getDefinition())
-    PDecl = Def;
-  // Must write out all protocol definitions in current qualifier list,
-  // and in their nested qualifiers before writing out current definition.
-  for (ObjCProtocolDecl::protocol_iterator I = PDecl->protocol_begin(),
-       E = PDecl->protocol_end(); I != E; ++I)
-    RewriteObjCProtocolMetaData(*I, Result);
-  
-  // Construct method lists.
-  std::vector<ObjCMethodDecl *> InstanceMethods, ClassMethods;
-  std::vector<ObjCMethodDecl *> OptInstanceMethods, OptClassMethods;
-  for (ObjCProtocolDecl::instmeth_iterator
-       I = PDecl->instmeth_begin(), E = PDecl->instmeth_end();
-       I != E; ++I) {
-    ObjCMethodDecl *MD = *I;
-    if (MD->getImplementationControl() == ObjCMethodDecl::Optional) {
-      OptInstanceMethods.push_back(MD);
-    } else {
-      InstanceMethods.push_back(MD);
-    }
-  }
-  
-  for (ObjCProtocolDecl::classmeth_iterator
-       I = PDecl->classmeth_begin(), E = PDecl->classmeth_end();
-       I != E; ++I) {
-    ObjCMethodDecl *MD = *I;
-    if (MD->getImplementationControl() == ObjCMethodDecl::Optional) {
-      OptClassMethods.push_back(MD);
-    } else {
-      ClassMethods.push_back(MD);
-    }
-  }
-  std::vector<ObjCMethodDecl *> AllMethods;
-  for (unsigned i = 0, e = InstanceMethods.size(); i < e; i++)
-    AllMethods.push_back(InstanceMethods[i]);
-  for (unsigned i = 0, e = ClassMethods.size(); i < e; i++)
-    AllMethods.push_back(ClassMethods[i]);
-  for (unsigned i = 0, e = OptInstanceMethods.size(); i < e; i++)
-    AllMethods.push_back(OptInstanceMethods[i]);
-  for (unsigned i = 0, e = OptClassMethods.size(); i < e; i++)
-    AllMethods.push_back(OptClassMethods[i]);
-
-  Write__extendedMethodTypes_initializer(*this, Context, Result,
-                                         AllMethods,
-                                         "_OBJC_PROTOCOL_METHOD_TYPES_",
-                                         PDecl->getNameAsString());
-  // Protocol's super protocol list
-  std::vector<ObjCProtocolDecl *> SuperProtocols;
-  for (ObjCProtocolDecl::protocol_iterator I = PDecl->protocol_begin(),
-       E = PDecl->protocol_end(); I != E; ++I)
-    SuperProtocols.push_back(*I);
-  
-  Write_protocol_list_initializer(Context, Result, SuperProtocols,
-                                  "_OBJC_PROTOCOL_REFS_",
-                                  PDecl->getNameAsString());
-  
-  Write_method_list_t_initializer(*this, Context, Result, InstanceMethods, 
-                                  "_OBJC_PROTOCOL_INSTANCE_METHODS_",
-                                  PDecl->getNameAsString(), false);
-  
-  Write_method_list_t_initializer(*this, Context, Result, ClassMethods, 
-                                  "_OBJC_PROTOCOL_CLASS_METHODS_",
-                                  PDecl->getNameAsString(), false);
-
-  Write_method_list_t_initializer(*this, Context, Result, OptInstanceMethods, 
-                                  "_OBJC_PROTOCOL_OPT_INSTANCE_METHODS_",
-                                  PDecl->getNameAsString(), false);
-  
-  Write_method_list_t_initializer(*this, Context, Result, OptClassMethods, 
-                                  "_OBJC_PROTOCOL_OPT_CLASS_METHODS_",
-                                  PDecl->getNameAsString(), false);
-  
-  // Protocol's property metadata.
-  std::vector<ObjCPropertyDecl *> ProtocolProperties;
-  for (ObjCContainerDecl::prop_iterator I = PDecl->prop_begin(),
-       E = PDecl->prop_end(); I != E; ++I)
-    ProtocolProperties.push_back(*I);
-  
-  Write_prop_list_t_initializer(*this, Context, Result, ProtocolProperties,
-                                 /* Container */0,
-                                 "_OBJC_PROTOCOL_PROPERTIES_",
-                                 PDecl->getNameAsString());
-  
-  // Writer out root metadata for current protocol: struct _protocol_t
-  Result += "\n";
-  if (LangOpts.MicrosoftExt)
-    Result += "static ";
-  Result += "struct _protocol_t _OBJC_PROTOCOL_";
-  Result += PDecl->getNameAsString();
-  Result += " __attribute__ ((used, section (\"__DATA,__datacoal_nt,coalesced\"))) = {\n";
-  Result += "\t0,\n"; // id is; is null
-  Result += "\t\""; Result += PDecl->getNameAsString(); Result += "\",\n";
-  if (SuperProtocols.size() > 0) {
-    Result += "\t(const struct _protocol_list_t *)&"; Result += "_OBJC_PROTOCOL_REFS_";
-    Result += PDecl->getNameAsString(); Result += ",\n";
-  }
-  else
-    Result += "\t0,\n";
-  if (InstanceMethods.size() > 0) {
-    Result += "\t(const struct method_list_t *)&_OBJC_PROTOCOL_INSTANCE_METHODS_"; 
-    Result += PDecl->getNameAsString(); Result += ",\n";
-  }
-  else
-    Result += "\t0,\n";
-
-  if (ClassMethods.size() > 0) {
-    Result += "\t(const struct method_list_t *)&_OBJC_PROTOCOL_CLASS_METHODS_"; 
-    Result += PDecl->getNameAsString(); Result += ",\n";
-  }
-  else
-    Result += "\t0,\n";
-  
-  if (OptInstanceMethods.size() > 0) {
-    Result += "\t(const struct method_list_t *)&_OBJC_PROTOCOL_OPT_INSTANCE_METHODS_"; 
-    Result += PDecl->getNameAsString(); Result += ",\n";
-  }
-  else
-    Result += "\t0,\n";
-  
-  if (OptClassMethods.size() > 0) {
-    Result += "\t(const struct method_list_t *)&_OBJC_PROTOCOL_OPT_CLASS_METHODS_"; 
-    Result += PDecl->getNameAsString(); Result += ",\n";
-  }
-  else
-    Result += "\t0,\n";
-  
-  if (ProtocolProperties.size() > 0) {
-    Result += "\t(const struct _prop_list_t *)&_OBJC_PROTOCOL_PROPERTIES_"; 
-    Result += PDecl->getNameAsString(); Result += ",\n";
-  }
-  else
-    Result += "\t0,\n";
-  
-  Result += "\t"; Result += "sizeof(_protocol_t)"; Result += ",\n";
-  Result += "\t0,\n";
-  
-  if (AllMethods.size() > 0) {
-    Result += "\t(const char **)&"; Result += "_OBJC_PROTOCOL_METHOD_TYPES_";
-    Result += PDecl->getNameAsString();
-    Result += "\n};\n";
-  }
-  else
-    Result += "\t0\n};\n";
-  
-  if (LangOpts.MicrosoftExt)
-    Result += "static ";
-  Result += "struct _protocol_t *";
-  Result += "_OBJC_LABEL_PROTOCOL_$_"; Result += PDecl->getNameAsString();
-  Result += " = &_OBJC_PROTOCOL_"; Result += PDecl->getNameAsString();
-  Result += ";\n";
-    
-  // Mark this protocol as having been generated.
-  if (!ObjCSynthesizedProtocols.insert(PDecl->getCanonicalDecl()))
-    llvm_unreachable("protocol already synthesized");
-  
-}
-
-void RewriteModernObjC::RewriteObjCProtocolListMetaData(
-                                const ObjCList<ObjCProtocolDecl> &Protocols,
-                                StringRef prefix, StringRef ClassName,
-                                std::string &Result) {
-  if (Protocols.empty()) return;
-  
-  for (unsigned i = 0; i != Protocols.size(); i++)
-    RewriteObjCProtocolMetaData(Protocols[i], Result);
-  
-  // Output the top lovel protocol meta-data for the class.
-  /* struct _objc_protocol_list {
-   struct _objc_protocol_list *next;
-   int    protocol_count;
-   struct _objc_protocol *class_protocols[];
-   }
-   */
-  Result += "\n";
-  if (LangOpts.MicrosoftExt)
-    Result += "__declspec(allocate(\".cat_cls_meth$B\")) ";
-  Result += "static struct {\n";
-  Result += "\tstruct _objc_protocol_list *next;\n";
-  Result += "\tint    protocol_count;\n";
-  Result += "\tstruct _objc_protocol *class_protocols[";
-  Result += utostr(Protocols.size());
-  Result += "];\n} _OBJC_";
-  Result += prefix;
-  Result += "_PROTOCOLS_";
-  Result += ClassName;
-  Result += " __attribute__ ((used, section (\"__OBJC, __cat_cls_meth\")))= "
-  "{\n\t0, ";
-  Result += utostr(Protocols.size());
-  Result += "\n";
-  
-  Result += "\t,{&_OBJC_PROTOCOL_";
-  Result += Protocols[0]->getNameAsString();
-  Result += " \n";
-  
-  for (unsigned i = 1; i != Protocols.size(); i++) {
-    Result += "\t ,&_OBJC_PROTOCOL_";
-    Result += Protocols[i]->getNameAsString();
-    Result += "\n";
-  }
-  Result += "\t }\n};\n";
-}
-
-/// hasObjCExceptionAttribute - Return true if this class or any super
-/// class has the __objc_exception__ attribute.
-/// FIXME. Move this to ASTContext.cpp as it is also used for IRGen.
-static bool hasObjCExceptionAttribute(ASTContext &Context,
-                                      const ObjCInterfaceDecl *OID) {
-  if (OID->hasAttr<ObjCExceptionAttr>())
-    return true;
-  if (const ObjCInterfaceDecl *Super = OID->getSuperClass())
-    return hasObjCExceptionAttribute(Context, Super);
-  return false;
-}
-
-void RewriteModernObjC::RewriteObjCClassMetaData(ObjCImplementationDecl *IDecl,
-                                           std::string &Result) {
-  ObjCInterfaceDecl *CDecl = IDecl->getClassInterface();
-  
-  // Explicitly declared @interface's are already synthesized.
-  if (CDecl->isImplicitInterfaceDecl())
-    assert(false && 
-           "Legacy implicit interface rewriting not supported in moder abi");
-  
-  WriteModernMetadataDeclarations(Context, Result);
-  SmallVector<ObjCIvarDecl *, 8> IVars;
-  
-  for (ObjCIvarDecl *IVD = CDecl->all_declared_ivar_begin();
-      IVD; IVD = IVD->getNextIvar()) {
-    // Ignore unnamed bit-fields.
-    if (!IVD->getDeclName())
-      continue;
-    IVars.push_back(IVD);
-  }
-  
-  Write__ivar_list_t_initializer(*this, Context, Result, IVars, 
-                                 "_OBJC_$_INSTANCE_VARIABLES_",
-                                 CDecl);
-  
-  // Build _objc_method_list for class's instance methods if needed
-  SmallVector<ObjCMethodDecl *, 32>
-    InstanceMethods(IDecl->instmeth_begin(), IDecl->instmeth_end());
-  
-  // If any of our property implementations have associated getters or
-  // setters, produce metadata for them as well.
-  for (ObjCImplDecl::propimpl_iterator Prop = IDecl->propimpl_begin(),
-       PropEnd = IDecl->propimpl_end();
-       Prop != PropEnd; ++Prop) {
-    if (Prop->getPropertyImplementation() == ObjCPropertyImplDecl::Dynamic)
-      continue;
-    if (!Prop->getPropertyIvarDecl())
-      continue;
-    ObjCPropertyDecl *PD = Prop->getPropertyDecl();
-    if (!PD)
-      continue;
-    if (ObjCMethodDecl *Getter = PD->getGetterMethodDecl())
-      if (mustSynthesizeSetterGetterMethod(IDecl, PD, true /*getter*/))
-        InstanceMethods.push_back(Getter);
-    if (PD->isReadOnly())
-      continue;
-    if (ObjCMethodDecl *Setter = PD->getSetterMethodDecl())
-      if (mustSynthesizeSetterGetterMethod(IDecl, PD, false /*setter*/))
-        InstanceMethods.push_back(Setter);
-  }
-  
-  Write_method_list_t_initializer(*this, Context, Result, InstanceMethods,
-                                  "_OBJC_$_INSTANCE_METHODS_",
-                                  IDecl->getNameAsString(), true);
-  
-  SmallVector<ObjCMethodDecl *, 32>
-    ClassMethods(IDecl->classmeth_begin(), IDecl->classmeth_end());
-  
-  Write_method_list_t_initializer(*this, Context, Result, ClassMethods,
-                                  "_OBJC_$_CLASS_METHODS_",
-                                  IDecl->getNameAsString(), true);
-  
-  // Protocols referenced in class declaration?
-  // Protocol's super protocol list
-  std::vector<ObjCProtocolDecl *> RefedProtocols;
-  const ObjCList<ObjCProtocolDecl> &Protocols = CDecl->getReferencedProtocols();
-  for (ObjCList<ObjCProtocolDecl>::iterator I = Protocols.begin(),
-       E = Protocols.end();
-       I != E; ++I) {
-    RefedProtocols.push_back(*I);
-    // Must write out all protocol definitions in current qualifier list,
-    // and in their nested qualifiers before writing out current definition.
-    RewriteObjCProtocolMetaData(*I, Result);
-  }
-  
-  Write_protocol_list_initializer(Context, Result, 
-                                  RefedProtocols,
-                                  "_OBJC_CLASS_PROTOCOLS_$_",
-                                  IDecl->getNameAsString());
-  
-  // Protocol's property metadata.
-  std::vector<ObjCPropertyDecl *> ClassProperties;
-  for (ObjCContainerDecl::prop_iterator I = CDecl->prop_begin(),
-       E = CDecl->prop_end(); I != E; ++I)
-    ClassProperties.push_back(*I);
-  
-  Write_prop_list_t_initializer(*this, Context, Result, ClassProperties,
-                                 /* Container */IDecl,
-                                 "_OBJC_$_PROP_LIST_",
-                                 CDecl->getNameAsString());
-
-  
-  // Data for initializing _class_ro_t  metaclass meta-data
-  uint32_t flags = CLS_META;
-  std::string InstanceSize;
-  std::string InstanceStart;
-  
-  
-  bool classIsHidden = CDecl->getVisibility() == HiddenVisibility;
-  if (classIsHidden)
-    flags |= OBJC2_CLS_HIDDEN;
-  
-  if (!CDecl->getSuperClass())
-    // class is root
-    flags |= CLS_ROOT;
-  InstanceSize = "sizeof(struct _class_t)";
-  InstanceStart = InstanceSize;
-  Write__class_ro_t_initializer(Context, Result, flags, 
-                                InstanceStart, InstanceSize,
-                                ClassMethods,
-                                0,
-                                0,
-                                0,
-                                "_OBJC_METACLASS_RO_$_",
-                                CDecl->getNameAsString());
-
-  
-  // Data for initializing _class_ro_t meta-data
-  flags = CLS;
-  if (classIsHidden)
-    flags |= OBJC2_CLS_HIDDEN;
-  
-  if (hasObjCExceptionAttribute(*Context, CDecl))
-    flags |= CLS_EXCEPTION;
-
-  if (!CDecl->getSuperClass())
-    // class is root
-    flags |= CLS_ROOT;
-  
-  InstanceSize.clear();
-  InstanceStart.clear();
-  if (!ObjCSynthesizedStructs.count(CDecl)) {
-    InstanceSize = "0";
-    InstanceStart = "0";
-  }
-  else {
-    InstanceSize = "sizeof(struct ";
-    InstanceSize += CDecl->getNameAsString();
-    InstanceSize += "_IMPL)";
-    
-    ObjCIvarDecl *IVD = CDecl->all_declared_ivar_begin();
-    if (IVD) {
-      RewriteIvarOffsetComputation(IVD, InstanceStart);
-    }
-    else 
-      InstanceStart = InstanceSize;
-  }
-  Write__class_ro_t_initializer(Context, Result, flags, 
-                                InstanceStart, InstanceSize,
-                                InstanceMethods,
-                                RefedProtocols,
-                                IVars,
-                                ClassProperties,
-                                "_OBJC_CLASS_RO_$_",
-                                CDecl->getNameAsString());
-  
-  Write_class_t(Context, Result,
-                "OBJC_METACLASS_$_",
-                CDecl, /*metaclass*/true);
-  
-  Write_class_t(Context, Result,
-                "OBJC_CLASS_$_",
-                CDecl, /*metaclass*/false);
-  
-  if (ImplementationIsNonLazy(IDecl))
-    DefinedNonLazyClasses.push_back(CDecl);
-                
-}
-
-void RewriteModernObjC::RewriteClassSetupInitHook(std::string &Result) {
-  int ClsDefCount = ClassImplementation.size();
-  if (!ClsDefCount)
-    return;
-  Result += "#pragma section(\".objc_inithooks$B\", long, read, write)\n";
-  Result += "__declspec(allocate(\".objc_inithooks$B\")) ";
-  Result += "static void *OBJC_CLASS_SETUP[] = {\n";
-  for (int i = 0; i < ClsDefCount; i++) {
-    ObjCImplementationDecl *IDecl = ClassImplementation[i];
-    ObjCInterfaceDecl *CDecl = IDecl->getClassInterface();
-    Result += "\t(void *)&OBJC_CLASS_SETUP_$_";
-    Result  += CDecl->getName(); Result += ",\n";
-  }
-  Result += "};\n";
-}
-
-void RewriteModernObjC::RewriteMetaDataIntoBuffer(std::string &Result) {
-  int ClsDefCount = ClassImplementation.size();
-  int CatDefCount = CategoryImplementation.size();
-  
-  // For each implemented class, write out all its meta data.
-  for (int i = 0; i < ClsDefCount; i++)
-    RewriteObjCClassMetaData(ClassImplementation[i], Result);
-  
-  RewriteClassSetupInitHook(Result);
-  
-  // For each implemented category, write out all its meta data.
-  for (int i = 0; i < CatDefCount; i++)
-    RewriteObjCCategoryImplDecl(CategoryImplementation[i], Result);
-  
-  RewriteCategorySetupInitHook(Result);
-  
-  if (ClsDefCount > 0) {
-    if (LangOpts.MicrosoftExt)
-      Result += "__declspec(allocate(\".objc_classlist$B\")) ";
-    Result += "static struct _class_t *L_OBJC_LABEL_CLASS_$ [";
-    Result += llvm::utostr(ClsDefCount); Result += "]";
-    Result += 
-      " __attribute__((used, section (\"__DATA, __objc_classlist,"
-      "regular,no_dead_strip\")))= {\n";
-    for (int i = 0; i < ClsDefCount; i++) {
-      Result += "\t&OBJC_CLASS_$_";
-      Result += ClassImplementation[i]->getNameAsString();
-      Result += ",\n";
-    }
-    Result += "};\n";
-    
-    if (!DefinedNonLazyClasses.empty()) {
-      if (LangOpts.MicrosoftExt)
-        Result += "__declspec(allocate(\".objc_nlclslist$B\")) \n";
-      Result += "static struct _class_t *_OBJC_LABEL_NONLAZY_CLASS_$[] = {\n\t";
-      for (unsigned i = 0, e = DefinedNonLazyClasses.size(); i < e; i++) {
-        Result += "\t&OBJC_CLASS_$_"; Result += DefinedNonLazyClasses[i]->getNameAsString();
-        Result += ",\n";
-      }
-      Result += "};\n";
-    }
-  }
-  
-  if (CatDefCount > 0) {
-    if (LangOpts.MicrosoftExt)
-      Result += "__declspec(allocate(\".objc_catlist$B\")) ";
-    Result += "static struct _category_t *L_OBJC_LABEL_CATEGORY_$ [";
-    Result += llvm::utostr(CatDefCount); Result += "]";
-    Result += 
-    " __attribute__((used, section (\"__DATA, __objc_catlist,"
-    "regular,no_dead_strip\")))= {\n";
-    for (int i = 0; i < CatDefCount; i++) {
-      Result += "\t&_OBJC_$_CATEGORY_";
-      Result += 
-        CategoryImplementation[i]->getClassInterface()->getNameAsString(); 
-      Result += "_$_";
-      Result += CategoryImplementation[i]->getNameAsString();
-      Result += ",\n";
-    }
-    Result += "};\n";
-  }
-  
-  if (!DefinedNonLazyCategories.empty()) {
-    if (LangOpts.MicrosoftExt)
-      Result += "__declspec(allocate(\".objc_nlcatlist$B\")) \n";
-    Result += "static struct _category_t *_OBJC_LABEL_NONLAZY_CATEGORY_$[] = {\n\t";
-    for (unsigned i = 0, e = DefinedNonLazyCategories.size(); i < e; i++) {
-      Result += "\t&_OBJC_$_CATEGORY_";
-      Result += 
-        DefinedNonLazyCategories[i]->getClassInterface()->getNameAsString(); 
-      Result += "_$_";
-      Result += DefinedNonLazyCategories[i]->getNameAsString();
-      Result += ",\n";
-    }
-    Result += "};\n";
-  }
-}
-
-void RewriteModernObjC::WriteImageInfo(std::string &Result) {
-  if (LangOpts.MicrosoftExt)
-    Result += "__declspec(allocate(\".objc_imageinfo$B\")) \n";
-  
-  Result += "static struct IMAGE_INFO { unsigned version; unsigned flag; } ";
-  // version 0, ObjCABI is 2
-  Result += "_OBJC_IMAGE_INFO = { 0, 2 };\n";
-}
-
-/// RewriteObjCCategoryImplDecl - Rewrite metadata for each category
-/// implementation.
-void RewriteModernObjC::RewriteObjCCategoryImplDecl(ObjCCategoryImplDecl *IDecl,
-                                              std::string &Result) {
-  WriteModernMetadataDeclarations(Context, Result);
-  ObjCInterfaceDecl *ClassDecl = IDecl->getClassInterface();
-  // Find category declaration for this implementation.
-  ObjCCategoryDecl *CDecl=0;
-  for (CDecl = ClassDecl->getCategoryList(); CDecl;
-       CDecl = CDecl->getNextClassCategory())
-    if (CDecl->getIdentifier() == IDecl->getIdentifier())
-      break;
-  
-  std::string FullCategoryName = ClassDecl->getNameAsString();
-  FullCategoryName += "_$_";
-  FullCategoryName += CDecl->getNameAsString();
-  
-  // Build _objc_method_list for class's instance methods if needed
-  SmallVector<ObjCMethodDecl *, 32>
-  InstanceMethods(IDecl->instmeth_begin(), IDecl->instmeth_end());
-  
-  // If any of our property implementations have associated getters or
-  // setters, produce metadata for them as well.
-  for (ObjCImplDecl::propimpl_iterator Prop = IDecl->propimpl_begin(),
-       PropEnd = IDecl->propimpl_end();
-       Prop != PropEnd; ++Prop) {
-    if (Prop->getPropertyImplementation() == ObjCPropertyImplDecl::Dynamic)
-      continue;
-    if (!Prop->getPropertyIvarDecl())
-      continue;
-    ObjCPropertyDecl *PD = Prop->getPropertyDecl();
-    if (!PD)
-      continue;
-    if (ObjCMethodDecl *Getter = PD->getGetterMethodDecl())
-      InstanceMethods.push_back(Getter);
-    if (PD->isReadOnly())
-      continue;
-    if (ObjCMethodDecl *Setter = PD->getSetterMethodDecl())
-      InstanceMethods.push_back(Setter);
-  }
-  
-  Write_method_list_t_initializer(*this, Context, Result, InstanceMethods,
-                                  "_OBJC_$_CATEGORY_INSTANCE_METHODS_",
-                                  FullCategoryName, true);
-  
-  SmallVector<ObjCMethodDecl *, 32>
-    ClassMethods(IDecl->classmeth_begin(), IDecl->classmeth_end());
-  
-  Write_method_list_t_initializer(*this, Context, Result, ClassMethods,
-                                  "_OBJC_$_CATEGORY_CLASS_METHODS_",
-                                  FullCategoryName, true);
-  
-  // Protocols referenced in class declaration?
-  // Protocol's super protocol list
-  std::vector<ObjCProtocolDecl *> RefedProtocols;
-  for (ObjCInterfaceDecl::protocol_iterator I = CDecl->protocol_begin(),
-                                            E = CDecl->protocol_end();
-
-         I != E; ++I) {
-    RefedProtocols.push_back(*I);
-    // Must write out all protocol definitions in current qualifier list,
-    // and in their nested qualifiers before writing out current definition.
-    RewriteObjCProtocolMetaData(*I, Result);
-  }
-  
-  Write_protocol_list_initializer(Context, Result, 
-                                  RefedProtocols,
-                                  "_OBJC_CATEGORY_PROTOCOLS_$_",
-                                  FullCategoryName);
-  
-  // Protocol's property metadata.
-  std::vector<ObjCPropertyDecl *> ClassProperties;
-  for (ObjCContainerDecl::prop_iterator I = CDecl->prop_begin(),
-       E = CDecl->prop_end(); I != E; ++I)
-    ClassProperties.push_back(*I);
-  
-  Write_prop_list_t_initializer(*this, Context, Result, ClassProperties,
-                                /* Container */IDecl,
-                                "_OBJC_$_PROP_LIST_",
-                                FullCategoryName);
-  
-  Write_category_t(*this, Context, Result,
-                   CDecl,
-                   ClassDecl,
-                   InstanceMethods,
-                   ClassMethods,
-                   RefedProtocols,
-                   ClassProperties);
-  
-  // Determine if this category is also "non-lazy".
-  if (ImplementationIsNonLazy(IDecl))
-    DefinedNonLazyCategories.push_back(CDecl);
-    
-}
-
-void RewriteModernObjC::RewriteCategorySetupInitHook(std::string &Result) {
-  int CatDefCount = CategoryImplementation.size();
-  if (!CatDefCount)
-    return;
-  Result += "#pragma section(\".objc_inithooks$B\", long, read, write)\n";
-  Result += "__declspec(allocate(\".objc_inithooks$B\")) ";
-  Result += "static void *OBJC_CATEGORY_SETUP[] = {\n";
-  for (int i = 0; i < CatDefCount; i++) {
-    ObjCCategoryImplDecl *IDecl = CategoryImplementation[i];
-    ObjCCategoryDecl *CatDecl= IDecl->getCategoryDecl();
-    ObjCInterfaceDecl *ClassDecl = IDecl->getClassInterface();
-    Result += "\t(void *)&OBJC_CATEGORY_SETUP_$_";
-    Result += ClassDecl->getName();
-    Result += "_$_";
-    Result += CatDecl->getName();
-    Result += ",\n";
-  }
-  Result += "};\n";
-}
-
-// RewriteObjCMethodsMetaData - Rewrite methods metadata for instance or
-/// class methods.
-template<typename MethodIterator>
-void RewriteModernObjC::RewriteObjCMethodsMetaData(MethodIterator MethodBegin,
-                                             MethodIterator MethodEnd,
-                                             bool IsInstanceMethod,
-                                             StringRef prefix,
-                                             StringRef ClassName,
-                                             std::string &Result) {
-  if (MethodBegin == MethodEnd) return;
-  
-  if (!objc_impl_method) {
-    /* struct _objc_method {
-     SEL _cmd;
-     char *method_types;
-     void *_imp;
-     }
-     */
-    Result += "\nstruct _objc_method {\n";
-    Result += "\tSEL _cmd;\n";
-    Result += "\tchar *method_types;\n";
-    Result += "\tvoid *_imp;\n";
-    Result += "};\n";
-    
-    objc_impl_method = true;
-  }
-  
-  // Build _objc_method_list for class's methods if needed
-  
-  /* struct  {
-   struct _objc_method_list *next_method;
-   int method_count;
-   struct _objc_method method_list[];
-   }
-   */
-  unsigned NumMethods = std::distance(MethodBegin, MethodEnd);
-  Result += "\n";
-  if (LangOpts.MicrosoftExt) {
-    if (IsInstanceMethod)
-      Result += "__declspec(allocate(\".inst_meth$B\")) ";
-    else
-      Result += "__declspec(allocate(\".cls_meth$B\")) ";
-  }
-  Result += "static struct {\n";
-  Result += "\tstruct _objc_method_list *next_method;\n";
-  Result += "\tint method_count;\n";
-  Result += "\tstruct _objc_method method_list[";
-  Result += utostr(NumMethods);
-  Result += "];\n} _OBJC_";
-  Result += prefix;
-  Result += IsInstanceMethod ? "INSTANCE" : "CLASS";
-  Result += "_METHODS_";
-  Result += ClassName;
-  Result += " __attribute__ ((used, section (\"__OBJC, __";
-  Result += IsInstanceMethod ? "inst" : "cls";
-  Result += "_meth\")))= ";
-  Result += "{\n\t0, " + utostr(NumMethods) + "\n";
-  
-  Result += "\t,{{(SEL)\"";
-  Result += (*MethodBegin)->getSelector().getAsString().c_str();
-  std::string MethodTypeString;
-  Context->getObjCEncodingForMethodDecl(*MethodBegin, MethodTypeString);
-  Result += "\", \"";
-  Result += MethodTypeString;
-  Result += "\", (void *)";
-  Result += MethodInternalNames[*MethodBegin];
-  Result += "}\n";
-  for (++MethodBegin; MethodBegin != MethodEnd; ++MethodBegin) {
-    Result += "\t  ,{(SEL)\"";
-    Result += (*MethodBegin)->getSelector().getAsString().c_str();
-    std::string MethodTypeString;
-    Context->getObjCEncodingForMethodDecl(*MethodBegin, MethodTypeString);
-    Result += "\", \"";
-    Result += MethodTypeString;
-    Result += "\", (void *)";
-    Result += MethodInternalNames[*MethodBegin];
-    Result += "}\n";
-  }
-  Result += "\t }\n};\n";
-}
-
-Stmt *RewriteModernObjC::RewriteObjCIvarRefExpr(ObjCIvarRefExpr *IV) {
-  SourceRange OldRange = IV->getSourceRange();
-  Expr *BaseExpr = IV->getBase();
-  
-  // Rewrite the base, but without actually doing replaces.
-  {
-    DisableReplaceStmtScope S(*this);
-    BaseExpr = cast<Expr>(RewriteFunctionBodyOrGlobalInitializer(BaseExpr));
-    IV->setBase(BaseExpr);
-  }
-  
-  ObjCIvarDecl *D = IV->getDecl();
-  
-  Expr *Replacement = IV;
-  
-    if (BaseExpr->getType()->isObjCObjectPointerType()) {
-      const ObjCInterfaceType *iFaceDecl =
-        dyn_cast<ObjCInterfaceType>(BaseExpr->getType()->getPointeeType());
-      assert(iFaceDecl && "RewriteObjCIvarRefExpr - iFaceDecl is null");
-      // lookup which class implements the instance variable.
-      ObjCInterfaceDecl *clsDeclared = 0;
-      iFaceDecl->getDecl()->lookupInstanceVariable(D->getIdentifier(),
-                                                   clsDeclared);
-      assert(clsDeclared && "RewriteObjCIvarRefExpr(): Can't find class");
-      
-      // Build name of symbol holding ivar offset.
-      std::string IvarOffsetName;
-      WriteInternalIvarName(clsDeclared, D, IvarOffsetName);
-      
-      ReferencedIvars[clsDeclared].insert(D);
-      
-      // cast offset to "char *".
-      CastExpr *castExpr = NoTypeInfoCStyleCastExpr(Context, 
-                                                    Context->getPointerType(Context->CharTy),
-                                                    CK_BitCast,
-                                                    BaseExpr);
-      VarDecl *NewVD = VarDecl::Create(*Context, TUDecl, SourceLocation(),
-                                       SourceLocation(), &Context->Idents.get(IvarOffsetName),
-                                       Context->UnsignedLongTy, 0, SC_Extern, SC_None);
-      DeclRefExpr *DRE = new (Context) DeclRefExpr(NewVD, false,
-                                                   Context->UnsignedLongTy, VK_LValue,
-                                                   SourceLocation());
-      BinaryOperator *addExpr = 
-        new (Context) BinaryOperator(castExpr, DRE, BO_Add, 
-                                     Context->getPointerType(Context->CharTy),
-                                     VK_RValue, OK_Ordinary, SourceLocation());
-      // Don't forget the parens to enforce the proper binding.
-      ParenExpr *PE = new (Context) ParenExpr(SourceLocation(),
-                                              SourceLocation(),
-                                              addExpr);
-      QualType IvarT = D->getType();
-
-      if (!isa<TypedefType>(IvarT) && IvarT->isRecordType()) {
-        RecordDecl *RD = IvarT->getAs<RecordType>()->getDecl();
-        RD = RD->getDefinition();
-        if (RD && !RD->getDeclName().getAsIdentifierInfo()) {
-          // decltype(((Foo_IMPL*)0)->bar) *
-          ObjCContainerDecl *CDecl = 
-            dyn_cast<ObjCContainerDecl>(D->getDeclContext());
-          // ivar in class extensions requires special treatment.
-          if (ObjCCategoryDecl *CatDecl = dyn_cast<ObjCCategoryDecl>(CDecl))
-            CDecl = CatDecl->getClassInterface();
-          std::string RecName = CDecl->getName();
-          RecName += "_IMPL";
-          RecordDecl *RD = RecordDecl::Create(*Context, TTK_Struct, TUDecl,
-                                              SourceLocation(), SourceLocation(),
-                                              &Context->Idents.get(RecName.c_str()));
-          QualType PtrStructIMPL = Context->getPointerType(Context->getTagDeclType(RD));
-          unsigned UnsignedIntSize = 
-            static_cast<unsigned>(Context->getTypeSize(Context->UnsignedIntTy));
-          Expr *Zero = IntegerLiteral::Create(*Context,
-                                              llvm::APInt(UnsignedIntSize, 0),
-                                              Context->UnsignedIntTy, SourceLocation());
-          Zero = NoTypeInfoCStyleCastExpr(Context, PtrStructIMPL, CK_BitCast, Zero);
-          ParenExpr *PE = new (Context) ParenExpr(SourceLocation(), SourceLocation(),
-                                                  Zero);
-          FieldDecl *FD = FieldDecl::Create(*Context, 0, SourceLocation(),
-                                            SourceLocation(),
-                                            &Context->Idents.get(D->getNameAsString()),
-                                            IvarT, 0,
-                                            /*BitWidth=*/0, /*Mutable=*/true,
-                                            ICIS_NoInit);
-          MemberExpr *ME = new (Context) MemberExpr(PE, true, FD, SourceLocation(),
-                                                    FD->getType(), VK_LValue,
-                                                    OK_Ordinary);
-          IvarT = Context->getDecltypeType(ME, ME->getType());
-        }
-      }
-      convertObjCTypeToCStyleType(IvarT);
-      QualType castT = Context->getPointerType(IvarT);
-          
-      castExpr = NoTypeInfoCStyleCastExpr(Context, 
-                                          castT,
-                                          CK_BitCast,
-                                          PE);
-      
-      
-      Expr *Exp = new (Context) UnaryOperator(castExpr, UO_Deref, IvarT,
-                                              VK_LValue, OK_Ordinary,
-                                              SourceLocation());
-      PE = new (Context) ParenExpr(OldRange.getBegin(),
-                                   OldRange.getEnd(),
-                                   Exp);
-
-      Replacement = PE;
-    }
-  
-    ReplaceStmtWithRange(IV, Replacement, OldRange);
-    return Replacement;  
-}
diff --git a/lib/Rewrite/RewriteObjC.cpp b/lib/Rewrite/RewriteObjC.cpp
deleted file mode 100644
index 37c17e6..0000000
--- a/lib/Rewrite/RewriteObjC.cpp
+++ /dev/null
@@ -1,6035 +0,0 @@
-//===--- RewriteObjC.cpp - Playground for the code rewriter ---------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// Hacks and fun related to the code rewriter.
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/Rewrite/ASTConsumers.h"
-#include "clang/Rewrite/Rewriter.h"
-#include "clang/AST/AST.h"
-#include "clang/AST/ASTConsumer.h"
-#include "clang/AST/ParentMap.h"
-#include "clang/Basic/SourceManager.h"
-#include "clang/Basic/IdentifierTable.h"
-#include "clang/Basic/Diagnostic.h"
-#include "clang/Lex/Lexer.h"
-#include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/raw_ostream.h"
-#include "llvm/ADT/StringExtras.h"
-#include "llvm/ADT/SmallPtrSet.h"
-#include "llvm/ADT/OwningPtr.h"
-#include "llvm/ADT/DenseSet.h"
-
-using namespace clang;
-using llvm::utostr;
-
-namespace {
-  class RewriteObjC : public ASTConsumer {
-  protected:
-    
-    enum {
-      BLOCK_FIELD_IS_OBJECT   =  3,  /* id, NSObject, __attribute__((NSObject)),
-                                        block, ... */
-      BLOCK_FIELD_IS_BLOCK    =  7,  /* a block variable */
-      BLOCK_FIELD_IS_BYREF    =  8,  /* the on stack structure holding the 
-                                        __block variable */
-      BLOCK_FIELD_IS_WEAK     = 16,  /* declared __weak, only used in byref copy
-                                        helpers */
-      BLOCK_BYREF_CALLER      = 128, /* called from __block (byref) copy/dispose
-                                        support routines */
-      BLOCK_BYREF_CURRENT_MAX = 256
-    };
-    
-    enum {
-      BLOCK_NEEDS_FREE =        (1 << 24),
-      BLOCK_HAS_COPY_DISPOSE =  (1 << 25),
-      BLOCK_HAS_CXX_OBJ =       (1 << 26),
-      BLOCK_IS_GC =             (1 << 27),
-      BLOCK_IS_GLOBAL =         (1 << 28),
-      BLOCK_HAS_DESCRIPTOR =    (1 << 29)
-    };
-    static const int OBJC_ABI_VERSION = 7;
-    
-    Rewriter Rewrite;
-    DiagnosticsEngine &Diags;
-    const LangOptions &LangOpts;
-    ASTContext *Context;
-    SourceManager *SM;
-    TranslationUnitDecl *TUDecl;
-    FileID MainFileID;
-    const char *MainFileStart, *MainFileEnd;
-    Stmt *CurrentBody;
-    ParentMap *PropParentMap; // created lazily.
-    std::string InFileName;
-    raw_ostream* OutFile;
-    std::string Preamble;
-    
-    TypeDecl *ProtocolTypeDecl;
-    VarDecl *GlobalVarDecl;
-    unsigned RewriteFailedDiag;
-    // ObjC string constant support.
-    unsigned NumObjCStringLiterals;
-    VarDecl *ConstantStringClassReference;
-    RecordDecl *NSStringRecord;
-
-    // ObjC foreach break/continue generation support.
-    int BcLabelCount;
-    
-    unsigned TryFinallyContainsReturnDiag;
-    // Needed for super.
-    ObjCMethodDecl *CurMethodDef;
-    RecordDecl *SuperStructDecl;
-    RecordDecl *ConstantStringDecl;
-    
-    FunctionDecl *MsgSendFunctionDecl;
-    FunctionDecl *MsgSendSuperFunctionDecl;
-    FunctionDecl *MsgSendStretFunctionDecl;
-    FunctionDecl *MsgSendSuperStretFunctionDecl;
-    FunctionDecl *MsgSendFpretFunctionDecl;
-    FunctionDecl *GetClassFunctionDecl;
-    FunctionDecl *GetMetaClassFunctionDecl;
-    FunctionDecl *GetSuperClassFunctionDecl;
-    FunctionDecl *SelGetUidFunctionDecl;
-    FunctionDecl *CFStringFunctionDecl;
-    FunctionDecl *SuperContructorFunctionDecl;
-    FunctionDecl *CurFunctionDef;
-    FunctionDecl *CurFunctionDeclToDeclareForBlock;
-
-    /* Misc. containers needed for meta-data rewrite. */
-    SmallVector<ObjCImplementationDecl *, 8> ClassImplementation;
-    SmallVector<ObjCCategoryImplDecl *, 8> CategoryImplementation;
-    llvm::SmallPtrSet<ObjCInterfaceDecl*, 8> ObjCSynthesizedStructs;
-    llvm::SmallPtrSet<ObjCProtocolDecl*, 8> ObjCSynthesizedProtocols;
-    llvm::SmallPtrSet<ObjCInterfaceDecl*, 8> ObjCForwardDecls;
-    llvm::DenseMap<ObjCMethodDecl*, std::string> MethodInternalNames;
-    SmallVector<Stmt *, 32> Stmts;
-    SmallVector<int, 8> ObjCBcLabelNo;
-    // Remember all the @protocol(<expr>) expressions.
-    llvm::SmallPtrSet<ObjCProtocolDecl *, 32> ProtocolExprDecls;
-    
-    llvm::DenseSet<uint64_t> CopyDestroyCache;
-
-    // Block expressions.
-    SmallVector<BlockExpr *, 32> Blocks;
-    SmallVector<int, 32> InnerDeclRefsCount;
-    SmallVector<DeclRefExpr *, 32> InnerDeclRefs;
-    
-    SmallVector<DeclRefExpr *, 32> BlockDeclRefs;
-
-    // Block related declarations.
-    SmallVector<ValueDecl *, 8> BlockByCopyDecls;
-    llvm::SmallPtrSet<ValueDecl *, 8> BlockByCopyDeclsPtrSet;
-    SmallVector<ValueDecl *, 8> BlockByRefDecls;
-    llvm::SmallPtrSet<ValueDecl *, 8> BlockByRefDeclsPtrSet;
-    llvm::DenseMap<ValueDecl *, unsigned> BlockByRefDeclNo;
-    llvm::SmallPtrSet<ValueDecl *, 8> ImportedBlockDecls;
-    llvm::SmallPtrSet<VarDecl *, 8> ImportedLocalExternalDecls;
-    
-    llvm::DenseMap<BlockExpr *, std::string> RewrittenBlockExprs;
-
-    // This maps an original source AST to it's rewritten form. This allows
-    // us to avoid rewriting the same node twice (which is very uncommon).
-    // This is needed to support some of the exotic property rewriting.
-    llvm::DenseMap<Stmt *, Stmt *> ReplacedNodes;
-
-    // Needed for header files being rewritten
-    bool IsHeader;
-    bool SilenceRewriteMacroWarning;
-    bool objc_impl_method;
-    
-    bool DisableReplaceStmt;
-    class DisableReplaceStmtScope {
-      RewriteObjC &R;
-      bool SavedValue;
-    
-    public:
-      DisableReplaceStmtScope(RewriteObjC &R)
-        : R(R), SavedValue(R.DisableReplaceStmt) {
-        R.DisableReplaceStmt = true;
-      }
-      ~DisableReplaceStmtScope() {
-        R.DisableReplaceStmt = SavedValue;
-      }
-    };
-    void InitializeCommon(ASTContext &context);
-
-  public:
-
-    // Top Level Driver code.
-    virtual bool HandleTopLevelDecl(DeclGroupRef D) {
-      for (DeclGroupRef::iterator I = D.begin(), E = D.end(); I != E; ++I) {
-        if (ObjCInterfaceDecl *Class = dyn_cast<ObjCInterfaceDecl>(*I)) {
-          if (!Class->isThisDeclarationADefinition()) {
-            RewriteForwardClassDecl(D);
-            break;
-          }
-        }
-
-        if (ObjCProtocolDecl *Proto = dyn_cast<ObjCProtocolDecl>(*I)) {
-          if (!Proto->isThisDeclarationADefinition()) {
-            RewriteForwardProtocolDecl(D);
-            break;
-          }
-        }
-
-        HandleTopLevelSingleDecl(*I);
-      }
-      return true;
-    }
-    void HandleTopLevelSingleDecl(Decl *D);
-    void HandleDeclInMainFile(Decl *D);
-    RewriteObjC(std::string inFile, raw_ostream *OS,
-                DiagnosticsEngine &D, const LangOptions &LOpts,
-                bool silenceMacroWarn);
-
-    ~RewriteObjC() {}
-
-    virtual void HandleTranslationUnit(ASTContext &C);
-
-    void ReplaceStmt(Stmt *Old, Stmt *New) {
-      Stmt *ReplacingStmt = ReplacedNodes[Old];
-
-      if (ReplacingStmt)
-        return; // We can't rewrite the same node twice.
-
-      if (DisableReplaceStmt)
-        return;
-
-      // If replacement succeeded or warning disabled return with no warning.
-      if (!Rewrite.ReplaceStmt(Old, New)) {
-        ReplacedNodes[Old] = New;
-        return;
-      }
-      if (SilenceRewriteMacroWarning)
-        return;
-      Diags.Report(Context->getFullLoc(Old->getLocStart()), RewriteFailedDiag)
-                   << Old->getSourceRange();
-    }
-
-    void ReplaceStmtWithRange(Stmt *Old, Stmt *New, SourceRange SrcRange) {
-      if (DisableReplaceStmt)
-        return;
-
-      // Measure the old text.
-      int Size = Rewrite.getRangeSize(SrcRange);
-      if (Size == -1) {
-        Diags.Report(Context->getFullLoc(Old->getLocStart()), RewriteFailedDiag)
-                     << Old->getSourceRange();
-        return;
-      }
-      // Get the new text.
-      std::string SStr;
-      llvm::raw_string_ostream S(SStr);
-      New->printPretty(S, 0, PrintingPolicy(LangOpts));
-      const std::string &Str = S.str();
-
-      // If replacement succeeded or warning disabled return with no warning.
-      if (!Rewrite.ReplaceText(SrcRange.getBegin(), Size, Str)) {
-        ReplacedNodes[Old] = New;
-        return;
-      }
-      if (SilenceRewriteMacroWarning)
-        return;
-      Diags.Report(Context->getFullLoc(Old->getLocStart()), RewriteFailedDiag)
-                   << Old->getSourceRange();
-    }
-
-    void InsertText(SourceLocation Loc, StringRef Str,
-                    bool InsertAfter = true) {
-      // If insertion succeeded or warning disabled return with no warning.
-      if (!Rewrite.InsertText(Loc, Str, InsertAfter) ||
-          SilenceRewriteMacroWarning)
-        return;
-
-      Diags.Report(Context->getFullLoc(Loc), RewriteFailedDiag);
-    }
-
-    void ReplaceText(SourceLocation Start, unsigned OrigLength,
-                     StringRef Str) {
-      // If removal succeeded or warning disabled return with no warning.
-      if (!Rewrite.ReplaceText(Start, OrigLength, Str) ||
-          SilenceRewriteMacroWarning)
-        return;
-
-      Diags.Report(Context->getFullLoc(Start), RewriteFailedDiag);
-    }
-
-    // Syntactic Rewriting.
-    void RewriteRecordBody(RecordDecl *RD);
-    void RewriteInclude();
-    void RewriteForwardClassDecl(DeclGroupRef D);
-    void RewriteForwardClassDecl(const llvm::SmallVector<Decl*, 8> &DG);
-    void RewriteForwardClassEpilogue(ObjCInterfaceDecl *ClassDecl, 
-                                     const std::string &typedefString);
-    void RewriteImplementations();
-    void RewritePropertyImplDecl(ObjCPropertyImplDecl *PID,
-                                 ObjCImplementationDecl *IMD,
-                                 ObjCCategoryImplDecl *CID);
-    void RewriteInterfaceDecl(ObjCInterfaceDecl *Dcl);
-    void RewriteImplementationDecl(Decl *Dcl);
-    void RewriteObjCMethodDecl(const ObjCInterfaceDecl *IDecl,
-                               ObjCMethodDecl *MDecl, std::string &ResultStr);
-    void RewriteTypeIntoString(QualType T, std::string &ResultStr,
-                               const FunctionType *&FPRetType);
-    void RewriteByRefString(std::string &ResultStr, const std::string &Name,
-                            ValueDecl *VD, bool def=false);
-    void RewriteCategoryDecl(ObjCCategoryDecl *Dcl);
-    void RewriteProtocolDecl(ObjCProtocolDecl *Dcl);
-    void RewriteForwardProtocolDecl(DeclGroupRef D);
-    void RewriteForwardProtocolDecl(const llvm::SmallVector<Decl*, 8> &DG);
-    void RewriteMethodDeclaration(ObjCMethodDecl *Method);
-    void RewriteProperty(ObjCPropertyDecl *prop);
-    void RewriteFunctionDecl(FunctionDecl *FD);
-    void RewriteBlockPointerType(std::string& Str, QualType Type);
-    void RewriteBlockPointerTypeVariable(std::string& Str, ValueDecl *VD);
-    void RewriteBlockLiteralFunctionDecl(FunctionDecl *FD);
-    void RewriteObjCQualifiedInterfaceTypes(Decl *Dcl);
-    void RewriteTypeOfDecl(VarDecl *VD);
-    void RewriteObjCQualifiedInterfaceTypes(Expr *E);
-  
-    // Expression Rewriting.
-    Stmt *RewriteFunctionBodyOrGlobalInitializer(Stmt *S);
-    Stmt *RewriteAtEncode(ObjCEncodeExpr *Exp);
-    Stmt *RewritePropertyOrImplicitGetter(PseudoObjectExpr *Pseudo);
-    Stmt *RewritePropertyOrImplicitSetter(PseudoObjectExpr *Pseudo);
-    Stmt *RewriteAtSelector(ObjCSelectorExpr *Exp);
-    Stmt *RewriteMessageExpr(ObjCMessageExpr *Exp);
-    Stmt *RewriteObjCStringLiteral(ObjCStringLiteral *Exp);
-    Stmt *RewriteObjCProtocolExpr(ObjCProtocolExpr *Exp);
-    void RewriteTryReturnStmts(Stmt *S);
-    void RewriteSyncReturnStmts(Stmt *S, std::string buf);
-    Stmt *RewriteObjCTryStmt(ObjCAtTryStmt *S);
-    Stmt *RewriteObjCSynchronizedStmt(ObjCAtSynchronizedStmt *S);
-    Stmt *RewriteObjCThrowStmt(ObjCAtThrowStmt *S);
-    Stmt *RewriteObjCForCollectionStmt(ObjCForCollectionStmt *S,
-                                       SourceLocation OrigEnd);
-    Stmt *RewriteBreakStmt(BreakStmt *S);
-    Stmt *RewriteContinueStmt(ContinueStmt *S);
-    void RewriteCastExpr(CStyleCastExpr *CE);
-    
-    // Block rewriting.
-    void RewriteBlocksInFunctionProtoType(QualType funcType, NamedDecl *D);
-    
-    // Block specific rewrite rules.
-    void RewriteBlockPointerDecl(NamedDecl *VD);
-    void RewriteByRefVar(VarDecl *VD);
-    Stmt *RewriteBlockDeclRefExpr(DeclRefExpr *VD);
-    Stmt *RewriteLocalVariableExternalStorage(DeclRefExpr *DRE);
-    void RewriteBlockPointerFunctionArgs(FunctionDecl *FD);
-    
-    void RewriteObjCInternalStruct(ObjCInterfaceDecl *CDecl,
-                                      std::string &Result);
-    
-    virtual void Initialize(ASTContext &context) = 0;
-    
-    // Metadata Rewriting.
-    virtual void RewriteMetaDataIntoBuffer(std::string &Result) = 0;
-    virtual void RewriteObjCProtocolListMetaData(const ObjCList<ObjCProtocolDecl> &Prots,
-                                                 StringRef prefix,
-                                                 StringRef ClassName,
-                                                 std::string &Result) = 0;
-    virtual void RewriteObjCCategoryImplDecl(ObjCCategoryImplDecl *CDecl,
-                                             std::string &Result) = 0;
-    virtual void RewriteObjCProtocolMetaData(ObjCProtocolDecl *Protocol,
-                                     StringRef prefix,
-                                     StringRef ClassName,
-                                     std::string &Result) = 0;
-    virtual void RewriteObjCClassMetaData(ObjCImplementationDecl *IDecl,
-                                          std::string &Result) = 0;
-    
-    // Rewriting ivar access
-    virtual Stmt *RewriteObjCIvarRefExpr(ObjCIvarRefExpr *IV) = 0;
-    virtual void RewriteIvarOffsetComputation(ObjCIvarDecl *ivar,
-                                         std::string &Result) = 0;
-    
-    // Misc. AST transformation routines. Sometimes they end up calling
-    // rewriting routines on the new ASTs.
-    CallExpr *SynthesizeCallToFunctionDecl(FunctionDecl *FD,
-                                           Expr **args, unsigned nargs,
-                                           SourceLocation StartLoc=SourceLocation(),
-                                           SourceLocation EndLoc=SourceLocation());
-    CallExpr *SynthMsgSendStretCallExpr(FunctionDecl *MsgSendStretFlavor,
-                                        QualType msgSendType, 
-                                        QualType returnType, 
-                                        SmallVectorImpl<QualType> &ArgTypes,
-                                        SmallVectorImpl<Expr*> &MsgExprs,
-                                        ObjCMethodDecl *Method);
-    Stmt *SynthMessageExpr(ObjCMessageExpr *Exp,
-                           SourceLocation StartLoc=SourceLocation(),
-                           SourceLocation EndLoc=SourceLocation());
-    
-    void SynthCountByEnumWithState(std::string &buf);
-    void SynthMsgSendFunctionDecl();
-    void SynthMsgSendSuperFunctionDecl();
-    void SynthMsgSendStretFunctionDecl();
-    void SynthMsgSendFpretFunctionDecl();
-    void SynthMsgSendSuperStretFunctionDecl();
-    void SynthGetClassFunctionDecl();
-    void SynthGetMetaClassFunctionDecl();
-    void SynthGetSuperClassFunctionDecl();
-    void SynthSelGetUidFunctionDecl();
-    void SynthSuperContructorFunctionDecl();
-    
-    std::string SynthesizeByrefCopyDestroyHelper(VarDecl *VD, int flag);
-    std::string SynthesizeBlockHelperFuncs(BlockExpr *CE, int i,
-                                      StringRef funcName, std::string Tag);
-    std::string SynthesizeBlockFunc(BlockExpr *CE, int i,
-                                      StringRef funcName, std::string Tag);
-    std::string SynthesizeBlockImpl(BlockExpr *CE, 
-                                    std::string Tag, std::string Desc);
-    std::string SynthesizeBlockDescriptor(std::string DescTag, 
-                                          std::string ImplTag,
-                                          int i, StringRef funcName,
-                                          unsigned hasCopy);
-    Stmt *SynthesizeBlockCall(CallExpr *Exp, const Expr* BlockExp);
-    void SynthesizeBlockLiterals(SourceLocation FunLocStart,
-                                 StringRef FunName);
-    FunctionDecl *SynthBlockInitFunctionDecl(StringRef name);
-    Stmt *SynthBlockInitExpr(BlockExpr *Exp,
-            const SmallVector<DeclRefExpr *, 8> &InnerBlockDeclRefs);
-
-    // Misc. helper routines.
-    QualType getProtocolType();
-    void WarnAboutReturnGotoStmts(Stmt *S);
-    void HasReturnStmts(Stmt *S, bool &hasReturns);
-    void CheckFunctionPointerDecl(QualType dType, NamedDecl *ND);
-    void InsertBlockLiteralsWithinFunction(FunctionDecl *FD);
-    void InsertBlockLiteralsWithinMethod(ObjCMethodDecl *MD);
-
-    bool IsDeclStmtInForeachHeader(DeclStmt *DS);
-    void CollectBlockDeclRefInfo(BlockExpr *Exp);
-    void GetBlockDeclRefExprs(Stmt *S);
-    void GetInnerBlockDeclRefExprs(Stmt *S, 
-                SmallVector<DeclRefExpr *, 8> &InnerBlockDeclRefs,
-                llvm::SmallPtrSet<const DeclContext *, 8> &InnerContexts);
-
-    // We avoid calling Type::isBlockPointerType(), since it operates on the
-    // canonical type. We only care if the top-level type is a closure pointer.
-    bool isTopLevelBlockPointerType(QualType T) {
-      return isa<BlockPointerType>(T);
-    }
-
-    /// convertBlockPointerToFunctionPointer - Converts a block-pointer type
-    /// to a function pointer type and upon success, returns true; false
-    /// otherwise.
-    bool convertBlockPointerToFunctionPointer(QualType &T) {
-      if (isTopLevelBlockPointerType(T)) {
-        const BlockPointerType *BPT = T->getAs<BlockPointerType>();
-        T = Context->getPointerType(BPT->getPointeeType());
-        return true;
-      }
-      return false;
-    }
-    
-    bool needToScanForQualifiers(QualType T);
-    QualType getSuperStructType();
-    QualType getConstantStringStructType();
-    QualType convertFunctionTypeOfBlocks(const FunctionType *FT);
-    bool BufferContainsPPDirectives(const char *startBuf, const char *endBuf);
-    
-    void convertToUnqualifiedObjCType(QualType &T) {
-      if (T->isObjCQualifiedIdType())
-        T = Context->getObjCIdType();
-      else if (T->isObjCQualifiedClassType())
-        T = Context->getObjCClassType();
-      else if (T->isObjCObjectPointerType() &&
-               T->getPointeeType()->isObjCQualifiedInterfaceType()) {
-        if (const ObjCObjectPointerType * OBJPT =
-              T->getAsObjCInterfacePointerType()) {
-          const ObjCInterfaceType *IFaceT = OBJPT->getInterfaceType();
-          T = QualType(IFaceT, 0);
-          T = Context->getPointerType(T);
-        }
-     }
-    }
-    
-    // FIXME: This predicate seems like it would be useful to add to ASTContext.
-    bool isObjCType(QualType T) {
-      if (!LangOpts.ObjC1 && !LangOpts.ObjC2)
-        return false;
-
-      QualType OCT = Context->getCanonicalType(T).getUnqualifiedType();
-
-      if (OCT == Context->getCanonicalType(Context->getObjCIdType()) ||
-          OCT == Context->getCanonicalType(Context->getObjCClassType()))
-        return true;
-
-      if (const PointerType *PT = OCT->getAs<PointerType>()) {
-        if (isa<ObjCInterfaceType>(PT->getPointeeType()) ||
-            PT->getPointeeType()->isObjCQualifiedIdType())
-          return true;
-      }
-      return false;
-    }
-    bool PointerTypeTakesAnyBlockArguments(QualType QT);
-    bool PointerTypeTakesAnyObjCQualifiedType(QualType QT);
-    void GetExtentOfArgList(const char *Name, const char *&LParen,
-                            const char *&RParen);
-    
-    void QuoteDoublequotes(std::string &From, std::string &To) {
-      for (unsigned i = 0; i < From.length(); i++) {
-        if (From[i] == '"')
-          To += "\\\"";
-        else
-          To += From[i];
-      }
-    }
-
-    QualType getSimpleFunctionType(QualType result,
-                                   const QualType *args,
-                                   unsigned numArgs,
-                                   bool variadic = false) {
-      if (result == Context->getObjCInstanceType())
-        result =  Context->getObjCIdType();
-      FunctionProtoType::ExtProtoInfo fpi;
-      fpi.Variadic = variadic;
-      return Context->getFunctionType(result, args, numArgs, fpi);
-    }
-
-    // Helper function: create a CStyleCastExpr with trivial type source info.
-    CStyleCastExpr* NoTypeInfoCStyleCastExpr(ASTContext *Ctx, QualType Ty,
-                                             CastKind Kind, Expr *E) {
-      TypeSourceInfo *TInfo = Ctx->getTrivialTypeSourceInfo(Ty, SourceLocation());
-      return CStyleCastExpr::Create(*Ctx, Ty, VK_RValue, Kind, E, 0, TInfo,
-                                    SourceLocation(), SourceLocation());
-    }
-  };
-  
-  class RewriteObjCFragileABI : public RewriteObjC {
-  public:
-    
-    RewriteObjCFragileABI(std::string inFile, raw_ostream *OS,
-                DiagnosticsEngine &D, const LangOptions &LOpts,
-                bool silenceMacroWarn) : RewriteObjC(inFile, OS,
-                                                     D, LOpts,
-                                                     silenceMacroWarn) {}
-    
-    ~RewriteObjCFragileABI() {}
-    virtual void Initialize(ASTContext &context);
-    
-    // Rewriting metadata
-    template<typename MethodIterator>
-    void RewriteObjCMethodsMetaData(MethodIterator MethodBegin,
-                                    MethodIterator MethodEnd,
-                                    bool IsInstanceMethod,
-                                    StringRef prefix,
-                                    StringRef ClassName,
-                                    std::string &Result);
-    virtual void RewriteObjCProtocolMetaData(ObjCProtocolDecl *Protocol,
-                                             StringRef prefix,
-                                             StringRef ClassName,
-                                             std::string &Result);
-    virtual void RewriteObjCProtocolListMetaData(
-                  const ObjCList<ObjCProtocolDecl> &Prots,
-                  StringRef prefix, StringRef ClassName, std::string &Result);
-    virtual void RewriteObjCClassMetaData(ObjCImplementationDecl *IDecl,
-                                          std::string &Result);
-    virtual void RewriteMetaDataIntoBuffer(std::string &Result);
-    virtual void RewriteObjCCategoryImplDecl(ObjCCategoryImplDecl *CDecl,
-                                             std::string &Result);
-    
-    // Rewriting ivar
-    virtual void RewriteIvarOffsetComputation(ObjCIvarDecl *ivar,
-                                              std::string &Result);
-    virtual Stmt *RewriteObjCIvarRefExpr(ObjCIvarRefExpr *IV);
-  };
-}
-
-void RewriteObjC::RewriteBlocksInFunctionProtoType(QualType funcType,
-                                                   NamedDecl *D) {
-  if (const FunctionProtoType *fproto
-      = dyn_cast<FunctionProtoType>(funcType.IgnoreParens())) {
-    for (FunctionProtoType::arg_type_iterator I = fproto->arg_type_begin(),
-         E = fproto->arg_type_end(); I && (I != E); ++I)
-      if (isTopLevelBlockPointerType(*I)) {
-        // All the args are checked/rewritten. Don't call twice!
-        RewriteBlockPointerDecl(D);
-        break;
-      }
-  }
-}
-
-void RewriteObjC::CheckFunctionPointerDecl(QualType funcType, NamedDecl *ND) {
-  const PointerType *PT = funcType->getAs<PointerType>();
-  if (PT && PointerTypeTakesAnyBlockArguments(funcType))
-    RewriteBlocksInFunctionProtoType(PT->getPointeeType(), ND);
-}
-
-static bool IsHeaderFile(const std::string &Filename) {
-  std::string::size_type DotPos = Filename.rfind('.');
-
-  if (DotPos == std::string::npos) {
-    // no file extension
-    return false;
-  }
-
-  std::string Ext = std::string(Filename.begin()+DotPos+1, Filename.end());
-  // C header: .h
-  // C++ header: .hh or .H;
-  return Ext == "h" || Ext == "hh" || Ext == "H";
-}
-
-RewriteObjC::RewriteObjC(std::string inFile, raw_ostream* OS,
-                         DiagnosticsEngine &D, const LangOptions &LOpts,
-                         bool silenceMacroWarn)
-      : Diags(D), LangOpts(LOpts), InFileName(inFile), OutFile(OS),
-        SilenceRewriteMacroWarning(silenceMacroWarn) {
-  IsHeader = IsHeaderFile(inFile);
-  RewriteFailedDiag = Diags.getCustomDiagID(DiagnosticsEngine::Warning,
-               "rewriting sub-expression within a macro (may not be correct)");
-  TryFinallyContainsReturnDiag = Diags.getCustomDiagID(
-               DiagnosticsEngine::Warning,
-               "rewriter doesn't support user-specified control flow semantics "
-               "for @try/@finally (code may not execute properly)");
-}
-
-ASTConsumer *clang::CreateObjCRewriter(const std::string& InFile,
-                                       raw_ostream* OS,
-                                       DiagnosticsEngine &Diags,
-                                       const LangOptions &LOpts,
-                                       bool SilenceRewriteMacroWarning) {
-  return new RewriteObjCFragileABI(InFile, OS, Diags, LOpts, SilenceRewriteMacroWarning);
-}
-
-void RewriteObjC::InitializeCommon(ASTContext &context) {
-  Context = &context;
-  SM = &Context->getSourceManager();
-  TUDecl = Context->getTranslationUnitDecl();
-  MsgSendFunctionDecl = 0;
-  MsgSendSuperFunctionDecl = 0;
-  MsgSendStretFunctionDecl = 0;
-  MsgSendSuperStretFunctionDecl = 0;
-  MsgSendFpretFunctionDecl = 0;
-  GetClassFunctionDecl = 0;
-  GetMetaClassFunctionDecl = 0;
-  GetSuperClassFunctionDecl = 0;
-  SelGetUidFunctionDecl = 0;
-  CFStringFunctionDecl = 0;
-  ConstantStringClassReference = 0;
-  NSStringRecord = 0;
-  CurMethodDef = 0;
-  CurFunctionDef = 0;
-  CurFunctionDeclToDeclareForBlock = 0;
-  GlobalVarDecl = 0;
-  SuperStructDecl = 0;
-  ProtocolTypeDecl = 0;
-  ConstantStringDecl = 0;
-  BcLabelCount = 0;
-  SuperContructorFunctionDecl = 0;
-  NumObjCStringLiterals = 0;
-  PropParentMap = 0;
-  CurrentBody = 0;
-  DisableReplaceStmt = false;
-  objc_impl_method = false;
-
-  // Get the ID and start/end of the main file.
-  MainFileID = SM->getMainFileID();
-  const llvm::MemoryBuffer *MainBuf = SM->getBuffer(MainFileID);
-  MainFileStart = MainBuf->getBufferStart();
-  MainFileEnd = MainBuf->getBufferEnd();
-
-  Rewrite.setSourceMgr(Context->getSourceManager(), Context->getLangOpts());
-}
-
-//===----------------------------------------------------------------------===//
-// Top Level Driver Code
-//===----------------------------------------------------------------------===//
-
-void RewriteObjC::HandleTopLevelSingleDecl(Decl *D) {
-  if (Diags.hasErrorOccurred())
-    return;
-
-  // Two cases: either the decl could be in the main file, or it could be in a
-  // #included file.  If the former, rewrite it now.  If the later, check to see
-  // if we rewrote the #include/#import.
-  SourceLocation Loc = D->getLocation();
-  Loc = SM->getExpansionLoc(Loc);
-
-  // If this is for a builtin, ignore it.
-  if (Loc.isInvalid()) return;
-
-  // Look for built-in declarations that we need to refer during the rewrite.
-  if (FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
-    RewriteFunctionDecl(FD);
-  } else if (VarDecl *FVD = dyn_cast<VarDecl>(D)) {
-    // declared in <Foundation/NSString.h>
-    if (FVD->getName() == "_NSConstantStringClassReference") {
-      ConstantStringClassReference = FVD;
-      return;
-    }
-  } else if (ObjCInterfaceDecl *ID = dyn_cast<ObjCInterfaceDecl>(D)) {
-    if (ID->isThisDeclarationADefinition())
-      RewriteInterfaceDecl(ID);
-  } else if (ObjCCategoryDecl *CD = dyn_cast<ObjCCategoryDecl>(D)) {
-    RewriteCategoryDecl(CD);
-  } else if (ObjCProtocolDecl *PD = dyn_cast<ObjCProtocolDecl>(D)) {
-    if (PD->isThisDeclarationADefinition())
-      RewriteProtocolDecl(PD);
-  } else if (LinkageSpecDecl *LSD = dyn_cast<LinkageSpecDecl>(D)) {
-    // Recurse into linkage specifications
-    for (DeclContext::decl_iterator DI = LSD->decls_begin(),
-                                 DIEnd = LSD->decls_end();
-         DI != DIEnd; ) {
-      if (ObjCInterfaceDecl *IFace = dyn_cast<ObjCInterfaceDecl>((*DI))) {
-        if (!IFace->isThisDeclarationADefinition()) {
-          SmallVector<Decl *, 8> DG;
-          SourceLocation StartLoc = IFace->getLocStart();
-          do {
-            if (isa<ObjCInterfaceDecl>(*DI) &&
-                !cast<ObjCInterfaceDecl>(*DI)->isThisDeclarationADefinition() &&
-                StartLoc == (*DI)->getLocStart())
-              DG.push_back(*DI);
-            else
-              break;
-            
-            ++DI;
-          } while (DI != DIEnd);
-          RewriteForwardClassDecl(DG);
-          continue;
-        }
-      }
-
-      if (ObjCProtocolDecl *Proto = dyn_cast<ObjCProtocolDecl>((*DI))) {
-        if (!Proto->isThisDeclarationADefinition()) {
-          SmallVector<Decl *, 8> DG;
-          SourceLocation StartLoc = Proto->getLocStart();
-          do {
-            if (isa<ObjCProtocolDecl>(*DI) &&
-                !cast<ObjCProtocolDecl>(*DI)->isThisDeclarationADefinition() &&
-                StartLoc == (*DI)->getLocStart())
-              DG.push_back(*DI);
-            else
-              break;
-            
-            ++DI;
-          } while (DI != DIEnd);
-          RewriteForwardProtocolDecl(DG);
-          continue;
-        }
-      }
-      
-      HandleTopLevelSingleDecl(*DI);
-      ++DI;
-    }
-  }
-  // If we have a decl in the main file, see if we should rewrite it.
-  if (SM->isFromMainFile(Loc))
-    return HandleDeclInMainFile(D);
-}
-
-//===----------------------------------------------------------------------===//
-// Syntactic (non-AST) Rewriting Code
-//===----------------------------------------------------------------------===//
-
-void RewriteObjC::RewriteInclude() {
-  SourceLocation LocStart = SM->getLocForStartOfFile(MainFileID);
-  StringRef MainBuf = SM->getBufferData(MainFileID);
-  const char *MainBufStart = MainBuf.begin();
-  const char *MainBufEnd = MainBuf.end();
-  size_t ImportLen = strlen("import");
-
-  // Loop over the whole file, looking for includes.
-  for (const char *BufPtr = MainBufStart; BufPtr < MainBufEnd; ++BufPtr) {
-    if (*BufPtr == '#') {
-      if (++BufPtr == MainBufEnd)
-        return;
-      while (*BufPtr == ' ' || *BufPtr == '\t')
-        if (++BufPtr == MainBufEnd)
-          return;
-      if (!strncmp(BufPtr, "import", ImportLen)) {
-        // replace import with include
-        SourceLocation ImportLoc =
-          LocStart.getLocWithOffset(BufPtr-MainBufStart);
-        ReplaceText(ImportLoc, ImportLen, "include");
-        BufPtr += ImportLen;
-      }
-    }
-  }
-}
-
-static std::string getIvarAccessString(ObjCIvarDecl *OID) {
-  const ObjCInterfaceDecl *ClassDecl = OID->getContainingInterface();
-  std::string S;
-  S = "((struct ";
-  S += ClassDecl->getIdentifier()->getName();
-  S += "_IMPL *)self)->";
-  S += OID->getName();
-  return S;
-}
-
-void RewriteObjC::RewritePropertyImplDecl(ObjCPropertyImplDecl *PID,
-                                          ObjCImplementationDecl *IMD,
-                                          ObjCCategoryImplDecl *CID) {
-  static bool objcGetPropertyDefined = false;
-  static bool objcSetPropertyDefined = false;
-  SourceLocation startLoc = PID->getLocStart();
-  InsertText(startLoc, "// ");
-  const char *startBuf = SM->getCharacterData(startLoc);
-  assert((*startBuf == '@') && "bogus @synthesize location");
-  const char *semiBuf = strchr(startBuf, ';');
-  assert((*semiBuf == ';') && "@synthesize: can't find ';'");
-  SourceLocation onePastSemiLoc =
-    startLoc.getLocWithOffset(semiBuf-startBuf+1);
-
-  if (PID->getPropertyImplementation() == ObjCPropertyImplDecl::Dynamic)
-    return; // FIXME: is this correct?
-
-  // Generate the 'getter' function.
-  ObjCPropertyDecl *PD = PID->getPropertyDecl();
-  ObjCIvarDecl *OID = PID->getPropertyIvarDecl();
-
-  if (!OID)
-    return;
-  unsigned Attributes = PD->getPropertyAttributes();
-  if (!PD->getGetterMethodDecl()->isDefined()) {
-    bool GenGetProperty = !(Attributes & ObjCPropertyDecl::OBJC_PR_nonatomic) &&
-                          (Attributes & (ObjCPropertyDecl::OBJC_PR_retain | 
-                                         ObjCPropertyDecl::OBJC_PR_copy));
-    std::string Getr;
-    if (GenGetProperty && !objcGetPropertyDefined) {
-      objcGetPropertyDefined = true;
-      // FIXME. Is this attribute correct in all cases?
-      Getr = "\nextern \"C\" __declspec(dllimport) "
-            "id objc_getProperty(id, SEL, long, bool);\n";
-    }
-    RewriteObjCMethodDecl(OID->getContainingInterface(),  
-                          PD->getGetterMethodDecl(), Getr);
-    Getr += "{ ";
-    // Synthesize an explicit cast to gain access to the ivar.
-    // See objc-act.c:objc_synthesize_new_getter() for details.
-    if (GenGetProperty) {
-      // return objc_getProperty(self, _cmd, offsetof(ClassDecl, OID), 1)
-      Getr += "typedef ";
-      const FunctionType *FPRetType = 0;
-      RewriteTypeIntoString(PD->getGetterMethodDecl()->getResultType(), Getr, 
-                            FPRetType);
-      Getr += " _TYPE";
-      if (FPRetType) {
-        Getr += ")"; // close the precedence "scope" for "*".
-      
-        // Now, emit the argument types (if any).
-        if (const FunctionProtoType *FT = dyn_cast<FunctionProtoType>(FPRetType)){
-          Getr += "(";
-          for (unsigned i = 0, e = FT->getNumArgs(); i != e; ++i) {
-            if (i) Getr += ", ";
-            std::string ParamStr = FT->getArgType(i).getAsString(
-                                                          Context->getPrintingPolicy());
-            Getr += ParamStr;
-          }
-          if (FT->isVariadic()) {
-            if (FT->getNumArgs()) Getr += ", ";
-            Getr += "...";
-          }
-          Getr += ")";
-        } else
-          Getr += "()";
-      }
-      Getr += ";\n";
-      Getr += "return (_TYPE)";
-      Getr += "objc_getProperty(self, _cmd, ";
-      RewriteIvarOffsetComputation(OID, Getr);
-      Getr += ", 1)";
-    }
-    else
-      Getr += "return " + getIvarAccessString(OID);
-    Getr += "; }";
-    InsertText(onePastSemiLoc, Getr);
-  }
-  
-  if (PD->isReadOnly() || PD->getSetterMethodDecl()->isDefined())
-    return;
-
-  // Generate the 'setter' function.
-  std::string Setr;
-  bool GenSetProperty = Attributes & (ObjCPropertyDecl::OBJC_PR_retain | 
-                                      ObjCPropertyDecl::OBJC_PR_copy);
-  if (GenSetProperty && !objcSetPropertyDefined) {
-    objcSetPropertyDefined = true;
-    // FIXME. Is this attribute correct in all cases?
-    Setr = "\nextern \"C\" __declspec(dllimport) "
-    "void objc_setProperty (id, SEL, long, id, bool, bool);\n";
-  }
-  
-  RewriteObjCMethodDecl(OID->getContainingInterface(), 
-                        PD->getSetterMethodDecl(), Setr);
-  Setr += "{ ";
-  // Synthesize an explicit cast to initialize the ivar.
-  // See objc-act.c:objc_synthesize_new_setter() for details.
-  if (GenSetProperty) {
-    Setr += "objc_setProperty (self, _cmd, ";
-    RewriteIvarOffsetComputation(OID, Setr);
-    Setr += ", (id)";
-    Setr += PD->getName();
-    Setr += ", ";
-    if (Attributes & ObjCPropertyDecl::OBJC_PR_nonatomic)
-      Setr += "0, ";
-    else
-      Setr += "1, ";
-    if (Attributes & ObjCPropertyDecl::OBJC_PR_copy)
-      Setr += "1)";
-    else
-      Setr += "0)";
-  }
-  else {
-    Setr += getIvarAccessString(OID) + " = ";
-    Setr += PD->getName();
-  }
-  Setr += "; }";
-  InsertText(onePastSemiLoc, Setr);
-}
-
-static void RewriteOneForwardClassDecl(ObjCInterfaceDecl *ForwardDecl,
-                                       std::string &typedefString) {
-  typedefString += "#ifndef _REWRITER_typedef_";
-  typedefString += ForwardDecl->getNameAsString();
-  typedefString += "\n";
-  typedefString += "#define _REWRITER_typedef_";
-  typedefString += ForwardDecl->getNameAsString();
-  typedefString += "\n";
-  typedefString += "typedef struct objc_object ";
-  typedefString += ForwardDecl->getNameAsString();
-  typedefString += ";\n#endif\n";
-}
-
-void RewriteObjC::RewriteForwardClassEpilogue(ObjCInterfaceDecl *ClassDecl,
-                                              const std::string &typedefString) {
-    SourceLocation startLoc = ClassDecl->getLocStart();
-    const char *startBuf = SM->getCharacterData(startLoc);
-    const char *semiPtr = strchr(startBuf, ';'); 
-    // Replace the @class with typedefs corresponding to the classes.
-    ReplaceText(startLoc, semiPtr-startBuf+1, typedefString);  
-}
-
-void RewriteObjC::RewriteForwardClassDecl(DeclGroupRef D) {
-  std::string typedefString;
-  for (DeclGroupRef::iterator I = D.begin(), E = D.end(); I != E; ++I) {
-    ObjCInterfaceDecl *ForwardDecl = cast<ObjCInterfaceDecl>(*I);
-    if (I == D.begin()) {
-      // Translate to typedef's that forward reference structs with the same name
-      // as the class. As a convenience, we include the original declaration
-      // as a comment.
-      typedefString += "// @class ";
-      typedefString += ForwardDecl->getNameAsString();
-      typedefString += ";\n";
-    }
-    RewriteOneForwardClassDecl(ForwardDecl, typedefString);
-  }
-  DeclGroupRef::iterator I = D.begin();
-  RewriteForwardClassEpilogue(cast<ObjCInterfaceDecl>(*I), typedefString);
-}
-
-void RewriteObjC::RewriteForwardClassDecl(
-                                const llvm::SmallVector<Decl*, 8> &D) {
-  std::string typedefString;
-  for (unsigned i = 0; i < D.size(); i++) {
-    ObjCInterfaceDecl *ForwardDecl = cast<ObjCInterfaceDecl>(D[i]);
-    if (i == 0) {
-      typedefString += "// @class ";
-      typedefString += ForwardDecl->getNameAsString();
-      typedefString += ";\n";
-    }
-    RewriteOneForwardClassDecl(ForwardDecl, typedefString);
-  }
-  RewriteForwardClassEpilogue(cast<ObjCInterfaceDecl>(D[0]), typedefString);
-}
-
-void RewriteObjC::RewriteMethodDeclaration(ObjCMethodDecl *Method) {
-  // When method is a synthesized one, such as a getter/setter there is
-  // nothing to rewrite.
-  if (Method->isImplicit())
-    return;
-  SourceLocation LocStart = Method->getLocStart();
-  SourceLocation LocEnd = Method->getLocEnd();
-
-  if (SM->getExpansionLineNumber(LocEnd) >
-      SM->getExpansionLineNumber(LocStart)) {
-    InsertText(LocStart, "#if 0\n");
-    ReplaceText(LocEnd, 1, ";\n#endif\n");
-  } else {
-    InsertText(LocStart, "// ");
-  }
-}
-
-void RewriteObjC::RewriteProperty(ObjCPropertyDecl *prop) {
-  SourceLocation Loc = prop->getAtLoc();
-
-  ReplaceText(Loc, 0, "// ");
-  // FIXME: handle properties that are declared across multiple lines.
-}
-
-void RewriteObjC::RewriteCategoryDecl(ObjCCategoryDecl *CatDecl) {
-  SourceLocation LocStart = CatDecl->getLocStart();
-
-  // FIXME: handle category headers that are declared across multiple lines.
-  ReplaceText(LocStart, 0, "// ");
-
-  for (ObjCCategoryDecl::prop_iterator I = CatDecl->prop_begin(),
-       E = CatDecl->prop_end(); I != E; ++I)
-    RewriteProperty(*I);
-  
-  for (ObjCCategoryDecl::instmeth_iterator
-         I = CatDecl->instmeth_begin(), E = CatDecl->instmeth_end();
-       I != E; ++I)
-    RewriteMethodDeclaration(*I);
-  for (ObjCCategoryDecl::classmeth_iterator
-         I = CatDecl->classmeth_begin(), E = CatDecl->classmeth_end();
-       I != E; ++I)
-    RewriteMethodDeclaration(*I);
-
-  // Lastly, comment out the @end.
-  ReplaceText(CatDecl->getAtEndRange().getBegin(), 
-              strlen("@end"), "/* @end */");
-}
-
-void RewriteObjC::RewriteProtocolDecl(ObjCProtocolDecl *PDecl) {
-  SourceLocation LocStart = PDecl->getLocStart();
-  assert(PDecl->isThisDeclarationADefinition());
-  
-  // FIXME: handle protocol headers that are declared across multiple lines.
-  ReplaceText(LocStart, 0, "// ");
-
-  for (ObjCProtocolDecl::instmeth_iterator
-         I = PDecl->instmeth_begin(), E = PDecl->instmeth_end();
-       I != E; ++I)
-    RewriteMethodDeclaration(*I);
-  for (ObjCProtocolDecl::classmeth_iterator
-         I = PDecl->classmeth_begin(), E = PDecl->classmeth_end();
-       I != E; ++I)
-    RewriteMethodDeclaration(*I);
-
-  for (ObjCInterfaceDecl::prop_iterator I = PDecl->prop_begin(),
-       E = PDecl->prop_end(); I != E; ++I)
-    RewriteProperty(*I);
-  
-  // Lastly, comment out the @end.
-  SourceLocation LocEnd = PDecl->getAtEndRange().getBegin();
-  ReplaceText(LocEnd, strlen("@end"), "/* @end */");
-
-  // Must comment out @optional/@required
-  const char *startBuf = SM->getCharacterData(LocStart);
-  const char *endBuf = SM->getCharacterData(LocEnd);
-  for (const char *p = startBuf; p < endBuf; p++) {
-    if (*p == '@' && !strncmp(p+1, "optional", strlen("optional"))) {
-      SourceLocation OptionalLoc = LocStart.getLocWithOffset(p-startBuf);
-      ReplaceText(OptionalLoc, strlen("@optional"), "/* @optional */");
-
-    }
-    else if (*p == '@' && !strncmp(p+1, "required", strlen("required"))) {
-      SourceLocation OptionalLoc = LocStart.getLocWithOffset(p-startBuf);
-      ReplaceText(OptionalLoc, strlen("@required"), "/* @required */");
-
-    }
-  }
-}
-
-void RewriteObjC::RewriteForwardProtocolDecl(DeclGroupRef D) {
-  SourceLocation LocStart = (*D.begin())->getLocStart();
-  if (LocStart.isInvalid())
-    llvm_unreachable("Invalid SourceLocation");
-  // FIXME: handle forward protocol that are declared across multiple lines.
-  ReplaceText(LocStart, 0, "// ");
-}
-
-void 
-RewriteObjC::RewriteForwardProtocolDecl(const llvm::SmallVector<Decl*, 8> &DG) {
-  SourceLocation LocStart = DG[0]->getLocStart();
-  if (LocStart.isInvalid())
-    llvm_unreachable("Invalid SourceLocation");
-  // FIXME: handle forward protocol that are declared across multiple lines.
-  ReplaceText(LocStart, 0, "// ");
-}
-
-void RewriteObjC::RewriteTypeIntoString(QualType T, std::string &ResultStr,
-                                        const FunctionType *&FPRetType) {
-  if (T->isObjCQualifiedIdType())
-    ResultStr += "id";
-  else if (T->isFunctionPointerType() ||
-           T->isBlockPointerType()) {
-    // needs special handling, since pointer-to-functions have special
-    // syntax (where a decaration models use).
-    QualType retType = T;
-    QualType PointeeTy;
-    if (const PointerType* PT = retType->getAs<PointerType>())
-      PointeeTy = PT->getPointeeType();
-    else if (const BlockPointerType *BPT = retType->getAs<BlockPointerType>())
-      PointeeTy = BPT->getPointeeType();
-    if ((FPRetType = PointeeTy->getAs<FunctionType>())) {
-      ResultStr += FPRetType->getResultType().getAsString(
-        Context->getPrintingPolicy());
-      ResultStr += "(*";
-    }
-  } else
-    ResultStr += T.getAsString(Context->getPrintingPolicy());
-}
-
-void RewriteObjC::RewriteObjCMethodDecl(const ObjCInterfaceDecl *IDecl,
-                                        ObjCMethodDecl *OMD,
-                                        std::string &ResultStr) {
-  //fprintf(stderr,"In RewriteObjCMethodDecl\n");
-  const FunctionType *FPRetType = 0;
-  ResultStr += "\nstatic ";
-  RewriteTypeIntoString(OMD->getResultType(), ResultStr, FPRetType);
-  ResultStr += " ";
-
-  // Unique method name
-  std::string NameStr;
-
-  if (OMD->isInstanceMethod())
-    NameStr += "_I_";
-  else
-    NameStr += "_C_";
-
-  NameStr += IDecl->getNameAsString();
-  NameStr += "_";
-
-  if (ObjCCategoryImplDecl *CID =
-      dyn_cast<ObjCCategoryImplDecl>(OMD->getDeclContext())) {
-    NameStr += CID->getNameAsString();
-    NameStr += "_";
-  }
-  // Append selector names, replacing ':' with '_'
-  {
-    std::string selString = OMD->getSelector().getAsString();
-    int len = selString.size();
-    for (int i = 0; i < len; i++)
-      if (selString[i] == ':')
-        selString[i] = '_';
-    NameStr += selString;
-  }
-  // Remember this name for metadata emission
-  MethodInternalNames[OMD] = NameStr;
-  ResultStr += NameStr;
-
-  // Rewrite arguments
-  ResultStr += "(";
-
-  // invisible arguments
-  if (OMD->isInstanceMethod()) {
-    QualType selfTy = Context->getObjCInterfaceType(IDecl);
-    selfTy = Context->getPointerType(selfTy);
-    if (!LangOpts.MicrosoftExt) {
-      if (ObjCSynthesizedStructs.count(const_cast<ObjCInterfaceDecl*>(IDecl)))
-        ResultStr += "struct ";
-    }
-    // When rewriting for Microsoft, explicitly omit the structure name.
-    ResultStr += IDecl->getNameAsString();
-    ResultStr += " *";
-  }
-  else
-    ResultStr += Context->getObjCClassType().getAsString(
-      Context->getPrintingPolicy());
-
-  ResultStr += " self, ";
-  ResultStr += Context->getObjCSelType().getAsString(Context->getPrintingPolicy());
-  ResultStr += " _cmd";
-
-  // Method arguments.
-  for (ObjCMethodDecl::param_iterator PI = OMD->param_begin(),
-       E = OMD->param_end(); PI != E; ++PI) {
-    ParmVarDecl *PDecl = *PI;
-    ResultStr += ", ";
-    if (PDecl->getType()->isObjCQualifiedIdType()) {
-      ResultStr += "id ";
-      ResultStr += PDecl->getNameAsString();
-    } else {
-      std::string Name = PDecl->getNameAsString();
-      QualType QT = PDecl->getType();
-      // Make sure we convert "t (^)(...)" to "t (*)(...)".
-      (void)convertBlockPointerToFunctionPointer(QT);
-      QT.getAsStringInternal(Name, Context->getPrintingPolicy());
-      ResultStr += Name;
-    }
-  }
-  if (OMD->isVariadic())
-    ResultStr += ", ...";
-  ResultStr += ") ";
-
-  if (FPRetType) {
-    ResultStr += ")"; // close the precedence "scope" for "*".
-
-    // Now, emit the argument types (if any).
-    if (const FunctionProtoType *FT = dyn_cast<FunctionProtoType>(FPRetType)) {
-      ResultStr += "(";
-      for (unsigned i = 0, e = FT->getNumArgs(); i != e; ++i) {
-        if (i) ResultStr += ", ";
-        std::string ParamStr = FT->getArgType(i).getAsString(
-          Context->getPrintingPolicy());
-        ResultStr += ParamStr;
-      }
-      if (FT->isVariadic()) {
-        if (FT->getNumArgs()) ResultStr += ", ";
-        ResultStr += "...";
-      }
-      ResultStr += ")";
-    } else {
-      ResultStr += "()";
-    }
-  }
-}
-void RewriteObjC::RewriteImplementationDecl(Decl *OID) {
-  ObjCImplementationDecl *IMD = dyn_cast<ObjCImplementationDecl>(OID);
-  ObjCCategoryImplDecl *CID = dyn_cast<ObjCCategoryImplDecl>(OID);
-
-  InsertText(IMD ? IMD->getLocStart() : CID->getLocStart(), "// ");
-
-  for (ObjCCategoryImplDecl::instmeth_iterator
-       I = IMD ? IMD->instmeth_begin() : CID->instmeth_begin(),
-       E = IMD ? IMD->instmeth_end() : CID->instmeth_end();
-       I != E; ++I) {
-    std::string ResultStr;
-    ObjCMethodDecl *OMD = *I;
-    RewriteObjCMethodDecl(OMD->getClassInterface(), OMD, ResultStr);
-    SourceLocation LocStart = OMD->getLocStart();
-    SourceLocation LocEnd = OMD->getCompoundBody()->getLocStart();
-
-    const char *startBuf = SM->getCharacterData(LocStart);
-    const char *endBuf = SM->getCharacterData(LocEnd);
-    ReplaceText(LocStart, endBuf-startBuf, ResultStr);
-  }
-
-  for (ObjCCategoryImplDecl::classmeth_iterator
-       I = IMD ? IMD->classmeth_begin() : CID->classmeth_begin(),
-       E = IMD ? IMD->classmeth_end() : CID->classmeth_end();
-       I != E; ++I) {
-    std::string ResultStr;
-    ObjCMethodDecl *OMD = *I;
-    RewriteObjCMethodDecl(OMD->getClassInterface(), OMD, ResultStr);
-    SourceLocation LocStart = OMD->getLocStart();
-    SourceLocation LocEnd = OMD->getCompoundBody()->getLocStart();
-
-    const char *startBuf = SM->getCharacterData(LocStart);
-    const char *endBuf = SM->getCharacterData(LocEnd);
-    ReplaceText(LocStart, endBuf-startBuf, ResultStr);
-  }
-  for (ObjCCategoryImplDecl::propimpl_iterator
-       I = IMD ? IMD->propimpl_begin() : CID->propimpl_begin(),
-       E = IMD ? IMD->propimpl_end() : CID->propimpl_end();
-       I != E; ++I) {
-    RewritePropertyImplDecl(*I, IMD, CID);
-  }
-
-  InsertText(IMD ? IMD->getLocEnd() : CID->getLocEnd(), "// ");
-}
-
-void RewriteObjC::RewriteInterfaceDecl(ObjCInterfaceDecl *ClassDecl) {
-  std::string ResultStr;
-  if (!ObjCForwardDecls.count(ClassDecl->getCanonicalDecl())) {
-    // we haven't seen a forward decl - generate a typedef.
-    ResultStr = "#ifndef _REWRITER_typedef_";
-    ResultStr += ClassDecl->getNameAsString();
-    ResultStr += "\n";
-    ResultStr += "#define _REWRITER_typedef_";
-    ResultStr += ClassDecl->getNameAsString();
-    ResultStr += "\n";
-    ResultStr += "typedef struct objc_object ";
-    ResultStr += ClassDecl->getNameAsString();
-    ResultStr += ";\n#endif\n";
-    // Mark this typedef as having been generated.
-    ObjCForwardDecls.insert(ClassDecl->getCanonicalDecl());
-  }
-  RewriteObjCInternalStruct(ClassDecl, ResultStr);
-
-  for (ObjCInterfaceDecl::prop_iterator I = ClassDecl->prop_begin(),
-         E = ClassDecl->prop_end(); I != E; ++I)
-    RewriteProperty(*I);
-  for (ObjCInterfaceDecl::instmeth_iterator
-         I = ClassDecl->instmeth_begin(), E = ClassDecl->instmeth_end();
-       I != E; ++I)
-    RewriteMethodDeclaration(*I);
-  for (ObjCInterfaceDecl::classmeth_iterator
-         I = ClassDecl->classmeth_begin(), E = ClassDecl->classmeth_end();
-       I != E; ++I)
-    RewriteMethodDeclaration(*I);
-
-  // Lastly, comment out the @end.
-  ReplaceText(ClassDecl->getAtEndRange().getBegin(), strlen("@end"), 
-              "/* @end */");
-}
-
-Stmt *RewriteObjC::RewritePropertyOrImplicitSetter(PseudoObjectExpr *PseudoOp) {
-  SourceRange OldRange = PseudoOp->getSourceRange();
-
-  // We just magically know some things about the structure of this
-  // expression.
-  ObjCMessageExpr *OldMsg =
-    cast<ObjCMessageExpr>(PseudoOp->getSemanticExpr(
-                            PseudoOp->getNumSemanticExprs() - 1));
-
-  // Because the rewriter doesn't allow us to rewrite rewritten code,
-  // we need to suppress rewriting the sub-statements.
-  Expr *Base, *RHS;
-  {
-    DisableReplaceStmtScope S(*this);
-
-    // Rebuild the base expression if we have one.
-    Base = 0;
-    if (OldMsg->getReceiverKind() == ObjCMessageExpr::Instance) {
-      Base = OldMsg->getInstanceReceiver();
-      Base = cast<OpaqueValueExpr>(Base)->getSourceExpr();
-      Base = cast<Expr>(RewriteFunctionBodyOrGlobalInitializer(Base));
-    }
-
-    // Rebuild the RHS.
-    RHS = cast<BinaryOperator>(PseudoOp->getSyntacticForm())->getRHS();
-    RHS = cast<OpaqueValueExpr>(RHS)->getSourceExpr();
-    RHS = cast<Expr>(RewriteFunctionBodyOrGlobalInitializer(RHS));
-  }
-
-  // TODO: avoid this copy.
-  SmallVector<SourceLocation, 1> SelLocs;
-  OldMsg->getSelectorLocs(SelLocs);
-
-  ObjCMessageExpr *NewMsg = 0;
-  switch (OldMsg->getReceiverKind()) {
-  case ObjCMessageExpr::Class:
-    NewMsg = ObjCMessageExpr::Create(*Context, OldMsg->getType(),
-                                     OldMsg->getValueKind(),
-                                     OldMsg->getLeftLoc(),
-                                     OldMsg->getClassReceiverTypeInfo(),
-                                     OldMsg->getSelector(),
-                                     SelLocs,
-                                     OldMsg->getMethodDecl(),
-                                     RHS,
-                                     OldMsg->getRightLoc(),
-                                     OldMsg->isImplicit());
-    break;
-
-  case ObjCMessageExpr::Instance:
-    NewMsg = ObjCMessageExpr::Create(*Context, OldMsg->getType(),
-                                     OldMsg->getValueKind(),
-                                     OldMsg->getLeftLoc(),
-                                     Base,
-                                     OldMsg->getSelector(),
-                                     SelLocs,
-                                     OldMsg->getMethodDecl(),
-                                     RHS,
-                                     OldMsg->getRightLoc(),
-                                     OldMsg->isImplicit());
-    break;
-
-  case ObjCMessageExpr::SuperClass:
-  case ObjCMessageExpr::SuperInstance:
-    NewMsg = ObjCMessageExpr::Create(*Context, OldMsg->getType(),
-                                     OldMsg->getValueKind(),
-                                     OldMsg->getLeftLoc(),
-                                     OldMsg->getSuperLoc(),
-                 OldMsg->getReceiverKind() == ObjCMessageExpr::SuperInstance,
-                                     OldMsg->getSuperType(),
-                                     OldMsg->getSelector(),
-                                     SelLocs,
-                                     OldMsg->getMethodDecl(),
-                                     RHS,
-                                     OldMsg->getRightLoc(),
-                                     OldMsg->isImplicit());
-    break;
-  }
-
-  Stmt *Replacement = SynthMessageExpr(NewMsg);
-  ReplaceStmtWithRange(PseudoOp, Replacement, OldRange);
-  return Replacement;
-}
-
-Stmt *RewriteObjC::RewritePropertyOrImplicitGetter(PseudoObjectExpr *PseudoOp) {
-  SourceRange OldRange = PseudoOp->getSourceRange();
-
-  // We just magically know some things about the structure of this
-  // expression.
-  ObjCMessageExpr *OldMsg =
-    cast<ObjCMessageExpr>(PseudoOp->getResultExpr()->IgnoreImplicit());
-
-  // Because the rewriter doesn't allow us to rewrite rewritten code,
-  // we need to suppress rewriting the sub-statements.
-  Expr *Base = 0;
-  {
-    DisableReplaceStmtScope S(*this);
-
-    // Rebuild the base expression if we have one.
-    if (OldMsg->getReceiverKind() == ObjCMessageExpr::Instance) {
-      Base = OldMsg->getInstanceReceiver();
-      Base = cast<OpaqueValueExpr>(Base)->getSourceExpr();
-      Base = cast<Expr>(RewriteFunctionBodyOrGlobalInitializer(Base));
-    }
-  }
-
-  // Intentionally empty.
-  SmallVector<SourceLocation, 1> SelLocs;
-  SmallVector<Expr*, 1> Args;
-
-  ObjCMessageExpr *NewMsg = 0;
-  switch (OldMsg->getReceiverKind()) {
-  case ObjCMessageExpr::Class:
-    NewMsg = ObjCMessageExpr::Create(*Context, OldMsg->getType(),
-                                     OldMsg->getValueKind(),
-                                     OldMsg->getLeftLoc(),
-                                     OldMsg->getClassReceiverTypeInfo(),
-                                     OldMsg->getSelector(),
-                                     SelLocs,
-                                     OldMsg->getMethodDecl(),
-                                     Args,
-                                     OldMsg->getRightLoc(),
-                                     OldMsg->isImplicit());
-    break;
-
-  case ObjCMessageExpr::Instance:
-    NewMsg = ObjCMessageExpr::Create(*Context, OldMsg->getType(),
-                                     OldMsg->getValueKind(),
-                                     OldMsg->getLeftLoc(),
-                                     Base,
-                                     OldMsg->getSelector(),
-                                     SelLocs,
-                                     OldMsg->getMethodDecl(),
-                                     Args,
-                                     OldMsg->getRightLoc(),
-                                     OldMsg->isImplicit());
-    break;
-
-  case ObjCMessageExpr::SuperClass:
-  case ObjCMessageExpr::SuperInstance:
-    NewMsg = ObjCMessageExpr::Create(*Context, OldMsg->getType(),
-                                     OldMsg->getValueKind(),
-                                     OldMsg->getLeftLoc(),
-                                     OldMsg->getSuperLoc(),
-                 OldMsg->getReceiverKind() == ObjCMessageExpr::SuperInstance,
-                                     OldMsg->getSuperType(),
-                                     OldMsg->getSelector(),
-                                     SelLocs,
-                                     OldMsg->getMethodDecl(),
-                                     Args,
-                                     OldMsg->getRightLoc(),
-                                     OldMsg->isImplicit());
-    break;
-  }
-
-  Stmt *Replacement = SynthMessageExpr(NewMsg);
-  ReplaceStmtWithRange(PseudoOp, Replacement, OldRange);
-  return Replacement;
-}
-
-/// SynthCountByEnumWithState - To print:
-/// ((unsigned int (*)
-///  (id, SEL, struct __objcFastEnumerationState *, id *, unsigned int))
-///  (void *)objc_msgSend)((id)l_collection,
-///                        sel_registerName(
-///                          "countByEnumeratingWithState:objects:count:"),
-///                        &enumState,
-///                        (id *)__rw_items, (unsigned int)16)
-///
-void RewriteObjC::SynthCountByEnumWithState(std::string &buf) {
-  buf += "((unsigned int (*) (id, SEL, struct __objcFastEnumerationState *, "
-  "id *, unsigned int))(void *)objc_msgSend)";
-  buf += "\n\t\t";
-  buf += "((id)l_collection,\n\t\t";
-  buf += "sel_registerName(\"countByEnumeratingWithState:objects:count:\"),";
-  buf += "\n\t\t";
-  buf += "&enumState, "
-         "(id *)__rw_items, (unsigned int)16)";
-}
-
-/// RewriteBreakStmt - Rewrite for a break-stmt inside an ObjC2's foreach
-/// statement to exit to its outer synthesized loop.
-///
-Stmt *RewriteObjC::RewriteBreakStmt(BreakStmt *S) {
-  if (Stmts.empty() || !isa<ObjCForCollectionStmt>(Stmts.back()))
-    return S;
-  // replace break with goto __break_label
-  std::string buf;
-
-  SourceLocation startLoc = S->getLocStart();
-  buf = "goto __break_label_";
-  buf += utostr(ObjCBcLabelNo.back());
-  ReplaceText(startLoc, strlen("break"), buf);
-
-  return 0;
-}
-
-/// RewriteContinueStmt - Rewrite for a continue-stmt inside an ObjC2's foreach
-/// statement to continue with its inner synthesized loop.
-///
-Stmt *RewriteObjC::RewriteContinueStmt(ContinueStmt *S) {
-  if (Stmts.empty() || !isa<ObjCForCollectionStmt>(Stmts.back()))
-    return S;
-  // replace continue with goto __continue_label
-  std::string buf;
-
-  SourceLocation startLoc = S->getLocStart();
-  buf = "goto __continue_label_";
-  buf += utostr(ObjCBcLabelNo.back());
-  ReplaceText(startLoc, strlen("continue"), buf);
-
-  return 0;
-}
-
-/// RewriteObjCForCollectionStmt - Rewriter for ObjC2's foreach statement.
-///  It rewrites:
-/// for ( type elem in collection) { stmts; }
-
-/// Into:
-/// {
-///   type elem;
-///   struct __objcFastEnumerationState enumState = { 0 };
-///   id __rw_items[16];
-///   id l_collection = (id)collection;
-///   unsigned long limit = [l_collection countByEnumeratingWithState:&enumState
-///                                       objects:__rw_items count:16];
-/// if (limit) {
-///   unsigned long startMutations = *enumState.mutationsPtr;
-///   do {
-///        unsigned long counter = 0;
-///        do {
-///             if (startMutations != *enumState.mutationsPtr)
-///               objc_enumerationMutation(l_collection);
-///             elem = (type)enumState.itemsPtr[counter++];
-///             stmts;
-///             __continue_label: ;
-///        } while (counter < limit);
-///   } while (limit = [l_collection countByEnumeratingWithState:&enumState
-///                                  objects:__rw_items count:16]);
-///   elem = nil;
-///   __break_label: ;
-///  }
-///  else
-///       elem = nil;
-///  }
-///
-Stmt *RewriteObjC::RewriteObjCForCollectionStmt(ObjCForCollectionStmt *S,
-                                                SourceLocation OrigEnd) {
-  assert(!Stmts.empty() && "ObjCForCollectionStmt - Statement stack empty");
-  assert(isa<ObjCForCollectionStmt>(Stmts.back()) &&
-         "ObjCForCollectionStmt Statement stack mismatch");
-  assert(!ObjCBcLabelNo.empty() &&
-         "ObjCForCollectionStmt - Label No stack empty");
-
-  SourceLocation startLoc = S->getLocStart();
-  const char *startBuf = SM->getCharacterData(startLoc);
-  StringRef elementName;
-  std::string elementTypeAsString;
-  std::string buf;
-  buf = "\n{\n\t";
-  if (DeclStmt *DS = dyn_cast<DeclStmt>(S->getElement())) {
-    // type elem;
-    NamedDecl* D = cast<NamedDecl>(DS->getSingleDecl());
-    QualType ElementType = cast<ValueDecl>(D)->getType();
-    if (ElementType->isObjCQualifiedIdType() ||
-        ElementType->isObjCQualifiedInterfaceType())
-      // Simply use 'id' for all qualified types.
-      elementTypeAsString = "id";
-    else
-      elementTypeAsString = ElementType.getAsString(Context->getPrintingPolicy());
-    buf += elementTypeAsString;
-    buf += " ";
-    elementName = D->getName();
-    buf += elementName;
-    buf += ";\n\t";
-  }
-  else {
-    DeclRefExpr *DR = cast<DeclRefExpr>(S->getElement());
-    elementName = DR->getDecl()->getName();
-    ValueDecl *VD = cast<ValueDecl>(DR->getDecl());
-    if (VD->getType()->isObjCQualifiedIdType() ||
-        VD->getType()->isObjCQualifiedInterfaceType())
-      // Simply use 'id' for all qualified types.
-      elementTypeAsString = "id";
-    else
-      elementTypeAsString = VD->getType().getAsString(Context->getPrintingPolicy());
-  }
-
-  // struct __objcFastEnumerationState enumState = { 0 };
-  buf += "struct __objcFastEnumerationState enumState = { 0 };\n\t";
-  // id __rw_items[16];
-  buf += "id __rw_items[16];\n\t";
-  // id l_collection = (id)
-  buf += "id l_collection = (id)";
-  // Find start location of 'collection' the hard way!
-  const char *startCollectionBuf = startBuf;
-  startCollectionBuf += 3;  // skip 'for'
-  startCollectionBuf = strchr(startCollectionBuf, '(');
-  startCollectionBuf++; // skip '('
-  // find 'in' and skip it.
-  while (*startCollectionBuf != ' ' ||
-         *(startCollectionBuf+1) != 'i' || *(startCollectionBuf+2) != 'n' ||
-         (*(startCollectionBuf+3) != ' ' &&
-          *(startCollectionBuf+3) != '[' && *(startCollectionBuf+3) != '('))
-    startCollectionBuf++;
-  startCollectionBuf += 3;
-
-  // Replace: "for (type element in" with string constructed thus far.
-  ReplaceText(startLoc, startCollectionBuf - startBuf, buf);
-  // Replace ')' in for '(' type elem in collection ')' with ';'
-  SourceLocation rightParenLoc = S->getRParenLoc();
-  const char *rparenBuf = SM->getCharacterData(rightParenLoc);
-  SourceLocation lparenLoc = startLoc.getLocWithOffset(rparenBuf-startBuf);
-  buf = ";\n\t";
-
-  // unsigned long limit = [l_collection countByEnumeratingWithState:&enumState
-  //                                   objects:__rw_items count:16];
-  // which is synthesized into:
-  // unsigned int limit =
-  // ((unsigned int (*)
-  //  (id, SEL, struct __objcFastEnumerationState *, id *, unsigned int))
-  //  (void *)objc_msgSend)((id)l_collection,
-  //                        sel_registerName(
-  //                          "countByEnumeratingWithState:objects:count:"),
-  //                        (struct __objcFastEnumerationState *)&state,
-  //                        (id *)__rw_items, (unsigned int)16);
-  buf += "unsigned long limit =\n\t\t";
-  SynthCountByEnumWithState(buf);
-  buf += ";\n\t";
-  /// if (limit) {
-  ///   unsigned long startMutations = *enumState.mutationsPtr;
-  ///   do {
-  ///        unsigned long counter = 0;
-  ///        do {
-  ///             if (startMutations != *enumState.mutationsPtr)
-  ///               objc_enumerationMutation(l_collection);
-  ///             elem = (type)enumState.itemsPtr[counter++];
-  buf += "if (limit) {\n\t";
-  buf += "unsigned long startMutations = *enumState.mutationsPtr;\n\t";
-  buf += "do {\n\t\t";
-  buf += "unsigned long counter = 0;\n\t\t";
-  buf += "do {\n\t\t\t";
-  buf += "if (startMutations != *enumState.mutationsPtr)\n\t\t\t\t";
-  buf += "objc_enumerationMutation(l_collection);\n\t\t\t";
-  buf += elementName;
-  buf += " = (";
-  buf += elementTypeAsString;
-  buf += ")enumState.itemsPtr[counter++];";
-  // Replace ')' in for '(' type elem in collection ')' with all of these.
-  ReplaceText(lparenLoc, 1, buf);
-
-  ///            __continue_label: ;
-  ///        } while (counter < limit);
-  ///   } while (limit = [l_collection countByEnumeratingWithState:&enumState
-  ///                                  objects:__rw_items count:16]);
-  ///   elem = nil;
-  ///   __break_label: ;
-  ///  }
-  ///  else
-  ///       elem = nil;
-  ///  }
-  ///
-  buf = ";\n\t";
-  buf += "__continue_label_";
-  buf += utostr(ObjCBcLabelNo.back());
-  buf += ": ;";
-  buf += "\n\t\t";
-  buf += "} while (counter < limit);\n\t";
-  buf += "} while (limit = ";
-  SynthCountByEnumWithState(buf);
-  buf += ");\n\t";
-  buf += elementName;
-  buf += " = ((";
-  buf += elementTypeAsString;
-  buf += ")0);\n\t";
-  buf += "__break_label_";
-  buf += utostr(ObjCBcLabelNo.back());
-  buf += ": ;\n\t";
-  buf += "}\n\t";
-  buf += "else\n\t\t";
-  buf += elementName;
-  buf += " = ((";
-  buf += elementTypeAsString;
-  buf += ")0);\n\t";
-  buf += "}\n";
-
-  // Insert all these *after* the statement body.
-  // FIXME: If this should support Obj-C++, support CXXTryStmt
-  if (isa<CompoundStmt>(S->getBody())) {
-    SourceLocation endBodyLoc = OrigEnd.getLocWithOffset(1);
-    InsertText(endBodyLoc, buf);
-  } else {
-    /* Need to treat single statements specially. For example:
-     *
-     *     for (A *a in b) if (stuff()) break;
-     *     for (A *a in b) xxxyy;
-     *
-     * The following code simply scans ahead to the semi to find the actual end.
-     */
-    const char *stmtBuf = SM->getCharacterData(OrigEnd);
-    const char *semiBuf = strchr(stmtBuf, ';');
-    assert(semiBuf && "Can't find ';'");
-    SourceLocation endBodyLoc = OrigEnd.getLocWithOffset(semiBuf-stmtBuf+1);
-    InsertText(endBodyLoc, buf);
-  }
-  Stmts.pop_back();
-  ObjCBcLabelNo.pop_back();
-  return 0;
-}
-
-/// RewriteObjCSynchronizedStmt -
-/// This routine rewrites @synchronized(expr) stmt;
-/// into:
-/// objc_sync_enter(expr);
-/// @try stmt @finally { objc_sync_exit(expr); }
-///
-Stmt *RewriteObjC::RewriteObjCSynchronizedStmt(ObjCAtSynchronizedStmt *S) {
-  // Get the start location and compute the semi location.
-  SourceLocation startLoc = S->getLocStart();
-  const char *startBuf = SM->getCharacterData(startLoc);
-
-  assert((*startBuf == '@') && "bogus @synchronized location");
-
-  std::string buf;
-  buf = "objc_sync_enter((id)";
-  const char *lparenBuf = startBuf;
-  while (*lparenBuf != '(') lparenBuf++;
-  ReplaceText(startLoc, lparenBuf-startBuf+1, buf);
-  // We can't use S->getSynchExpr()->getLocEnd() to find the end location, since
-  // the sync expression is typically a message expression that's already
-  // been rewritten! (which implies the SourceLocation's are invalid).
-  SourceLocation endLoc = S->getSynchBody()->getLocStart();
-  const char *endBuf = SM->getCharacterData(endLoc);
-  while (*endBuf != ')') endBuf--;
-  SourceLocation rparenLoc = startLoc.getLocWithOffset(endBuf-startBuf);
-  buf = ");\n";
-  // declare a new scope with two variables, _stack and _rethrow.
-  buf += "/* @try scope begin */ \n{ struct _objc_exception_data {\n";
-  buf += "int buf[18/*32-bit i386*/];\n";
-  buf += "char *pointers[4];} _stack;\n";
-  buf += "id volatile _rethrow = 0;\n";
-  buf += "objc_exception_try_enter(&_stack);\n";
-  buf += "if (!_setjmp(_stack.buf)) /* @try block continue */\n";
-  ReplaceText(rparenLoc, 1, buf);
-  startLoc = S->getSynchBody()->getLocEnd();
-  startBuf = SM->getCharacterData(startLoc);
-
-  assert((*startBuf == '}') && "bogus @synchronized block");
-  SourceLocation lastCurlyLoc = startLoc;
-  buf = "}\nelse {\n";
-  buf += "  _rethrow = objc_exception_extract(&_stack);\n";
-  buf += "}\n";
-  buf += "{ /* implicit finally clause */\n";
-  buf += "  if (!_rethrow) objc_exception_try_exit(&_stack);\n";
-  
-  std::string syncBuf;
-  syncBuf += " objc_sync_exit(";
-
-  Expr *syncExpr = S->getSynchExpr();
-  CastKind CK = syncExpr->getType()->isObjCObjectPointerType()
-                  ? CK_BitCast :
-                syncExpr->getType()->isBlockPointerType()
-                  ? CK_BlockPointerToObjCPointerCast
-                  : CK_CPointerToObjCPointerCast;
-  syncExpr = NoTypeInfoCStyleCastExpr(Context, Context->getObjCIdType(),
-                                      CK, syncExpr);
-  std::string syncExprBufS;
-  llvm::raw_string_ostream syncExprBuf(syncExprBufS);
-  syncExpr->printPretty(syncExprBuf, 0, PrintingPolicy(LangOpts));
-  syncBuf += syncExprBuf.str();
-  syncBuf += ");";
-  
-  buf += syncBuf;
-  buf += "\n  if (_rethrow) objc_exception_throw(_rethrow);\n";
-  buf += "}\n";
-  buf += "}";
-
-  ReplaceText(lastCurlyLoc, 1, buf);
-
-  bool hasReturns = false;
-  HasReturnStmts(S->getSynchBody(), hasReturns);
-  if (hasReturns)
-    RewriteSyncReturnStmts(S->getSynchBody(), syncBuf);
-
-  return 0;
-}
-
-void RewriteObjC::WarnAboutReturnGotoStmts(Stmt *S)
-{
-  // Perform a bottom up traversal of all children.
-  for (Stmt::child_range CI = S->children(); CI; ++CI)
-    if (*CI)
-      WarnAboutReturnGotoStmts(*CI);
-
-  if (isa<ReturnStmt>(S) || isa<GotoStmt>(S)) {
-    Diags.Report(Context->getFullLoc(S->getLocStart()),
-                 TryFinallyContainsReturnDiag);
-  }
-  return;
-}
-
-void RewriteObjC::HasReturnStmts(Stmt *S, bool &hasReturns) 
-{  
-  // Perform a bottom up traversal of all children.
-  for (Stmt::child_range CI = S->children(); CI; ++CI)
-   if (*CI)
-     HasReturnStmts(*CI, hasReturns);
-
- if (isa<ReturnStmt>(S))
-   hasReturns = true;
- return;
-}
-
-void RewriteObjC::RewriteTryReturnStmts(Stmt *S) {
- // Perform a bottom up traversal of all children.
- for (Stmt::child_range CI = S->children(); CI; ++CI)
-   if (*CI) {
-     RewriteTryReturnStmts(*CI);
-   }
- if (isa<ReturnStmt>(S)) {
-   SourceLocation startLoc = S->getLocStart();
-   const char *startBuf = SM->getCharacterData(startLoc);
-
-   const char *semiBuf = strchr(startBuf, ';');
-   assert((*semiBuf == ';') && "RewriteTryReturnStmts: can't find ';'");
-   SourceLocation onePastSemiLoc = startLoc.getLocWithOffset(semiBuf-startBuf+1);
-
-   std::string buf;
-   buf = "{ objc_exception_try_exit(&_stack); return";
-   
-   ReplaceText(startLoc, 6, buf);
-   InsertText(onePastSemiLoc, "}");
- }
- return;
-}
-
-void RewriteObjC::RewriteSyncReturnStmts(Stmt *S, std::string syncExitBuf) {
-  // Perform a bottom up traversal of all children.
-  for (Stmt::child_range CI = S->children(); CI; ++CI)
-    if (*CI) {
-      RewriteSyncReturnStmts(*CI, syncExitBuf);
-    }
-  if (isa<ReturnStmt>(S)) {
-    SourceLocation startLoc = S->getLocStart();
-    const char *startBuf = SM->getCharacterData(startLoc);
-
-    const char *semiBuf = strchr(startBuf, ';');
-    assert((*semiBuf == ';') && "RewriteSyncReturnStmts: can't find ';'");
-    SourceLocation onePastSemiLoc = startLoc.getLocWithOffset(semiBuf-startBuf+1);
-
-    std::string buf;
-    buf = "{ objc_exception_try_exit(&_stack);";
-    buf += syncExitBuf;
-    buf += " return";
-    
-    ReplaceText(startLoc, 6, buf);
-    InsertText(onePastSemiLoc, "}");
-  }
-  return;
-}
-
-Stmt *RewriteObjC::RewriteObjCTryStmt(ObjCAtTryStmt *S) {
-  // Get the start location and compute the semi location.
-  SourceLocation startLoc = S->getLocStart();
-  const char *startBuf = SM->getCharacterData(startLoc);
-
-  assert((*startBuf == '@') && "bogus @try location");
-
-  std::string buf;
-  // declare a new scope with two variables, _stack and _rethrow.
-  buf = "/* @try scope begin */ { struct _objc_exception_data {\n";
-  buf += "int buf[18/*32-bit i386*/];\n";
-  buf += "char *pointers[4];} _stack;\n";
-  buf += "id volatile _rethrow = 0;\n";
-  buf += "objc_exception_try_enter(&_stack);\n";
-  buf += "if (!_setjmp(_stack.buf)) /* @try block continue */\n";
-
-  ReplaceText(startLoc, 4, buf);
-
-  startLoc = S->getTryBody()->getLocEnd();
-  startBuf = SM->getCharacterData(startLoc);
-
-  assert((*startBuf == '}') && "bogus @try block");
-
-  SourceLocation lastCurlyLoc = startLoc;
-  if (S->getNumCatchStmts()) {
-    startLoc = startLoc.getLocWithOffset(1);
-    buf = " /* @catch begin */ else {\n";
-    buf += " id _caught = objc_exception_extract(&_stack);\n";
-    buf += " objc_exception_try_enter (&_stack);\n";
-    buf += " if (_setjmp(_stack.buf))\n";
-    buf += "   _rethrow = objc_exception_extract(&_stack);\n";
-    buf += " else { /* @catch continue */";
-
-    InsertText(startLoc, buf);
-  } else { /* no catch list */
-    buf = "}\nelse {\n";
-    buf += "  _rethrow = objc_exception_extract(&_stack);\n";
-    buf += "}";
-    ReplaceText(lastCurlyLoc, 1, buf);
-  }
-  Stmt *lastCatchBody = 0;
-  for (unsigned I = 0, N = S->getNumCatchStmts(); I != N; ++I) {
-    ObjCAtCatchStmt *Catch = S->getCatchStmt(I);
-    VarDecl *catchDecl = Catch->getCatchParamDecl();
-
-    if (I == 0)
-      buf = "if ("; // we are generating code for the first catch clause
-    else
-      buf = "else if (";
-    startLoc = Catch->getLocStart();
-    startBuf = SM->getCharacterData(startLoc);
-
-    assert((*startBuf == '@') && "bogus @catch location");
-
-    const char *lParenLoc = strchr(startBuf, '(');
-
-    if (Catch->hasEllipsis()) {
-      // Now rewrite the body...
-      lastCatchBody = Catch->getCatchBody();
-      SourceLocation bodyLoc = lastCatchBody->getLocStart();
-      const char *bodyBuf = SM->getCharacterData(bodyLoc);
-      assert(*SM->getCharacterData(Catch->getRParenLoc()) == ')' &&
-             "bogus @catch paren location");
-      assert((*bodyBuf == '{') && "bogus @catch body location");
-
-      buf += "1) { id _tmp = _caught;";
-      Rewrite.ReplaceText(startLoc, bodyBuf-startBuf+1, buf);
-    } else if (catchDecl) {
-      QualType t = catchDecl->getType();
-      if (t == Context->getObjCIdType()) {
-        buf += "1) { ";
-        ReplaceText(startLoc, lParenLoc-startBuf+1, buf);
-      } else if (const ObjCObjectPointerType *Ptr =
-                   t->getAs<ObjCObjectPointerType>()) {
-        // Should be a pointer to a class.
-        ObjCInterfaceDecl *IDecl = Ptr->getObjectType()->getInterface();
-        if (IDecl) {
-          buf += "objc_exception_match((struct objc_class *)objc_getClass(\"";
-          buf += IDecl->getNameAsString();
-          buf += "\"), (struct objc_object *)_caught)) { ";
-          ReplaceText(startLoc, lParenLoc-startBuf+1, buf);
-        }
-      }
-      // Now rewrite the body...
-      lastCatchBody = Catch->getCatchBody();
-      SourceLocation rParenLoc = Catch->getRParenLoc();
-      SourceLocation bodyLoc = lastCatchBody->getLocStart();
-      const char *bodyBuf = SM->getCharacterData(bodyLoc);
-      const char *rParenBuf = SM->getCharacterData(rParenLoc);
-      assert((*rParenBuf == ')') && "bogus @catch paren location");
-      assert((*bodyBuf == '{') && "bogus @catch body location");
-
-      // Here we replace ") {" with "= _caught;" (which initializes and
-      // declares the @catch parameter).
-      ReplaceText(rParenLoc, bodyBuf-rParenBuf+1, " = _caught;");
-    } else {
-      llvm_unreachable("@catch rewrite bug");
-    }
-  }
-  // Complete the catch list...
-  if (lastCatchBody) {
-    SourceLocation bodyLoc = lastCatchBody->getLocEnd();
-    assert(*SM->getCharacterData(bodyLoc) == '}' &&
-           "bogus @catch body location");
-
-    // Insert the last (implicit) else clause *before* the right curly brace.
-    bodyLoc = bodyLoc.getLocWithOffset(-1);
-    buf = "} /* last catch end */\n";
-    buf += "else {\n";
-    buf += " _rethrow = _caught;\n";
-    buf += " objc_exception_try_exit(&_stack);\n";
-    buf += "} } /* @catch end */\n";
-    if (!S->getFinallyStmt())
-      buf += "}\n";
-    InsertText(bodyLoc, buf);
-
-    // Set lastCurlyLoc
-    lastCurlyLoc = lastCatchBody->getLocEnd();
-  }
-  if (ObjCAtFinallyStmt *finalStmt = S->getFinallyStmt()) {
-    startLoc = finalStmt->getLocStart();
-    startBuf = SM->getCharacterData(startLoc);
-    assert((*startBuf == '@') && "bogus @finally start");
-
-    ReplaceText(startLoc, 8, "/* @finally */");
-
-    Stmt *body = finalStmt->getFinallyBody();
-    SourceLocation startLoc = body->getLocStart();
-    SourceLocation endLoc = body->getLocEnd();
-    assert(*SM->getCharacterData(startLoc) == '{' &&
-           "bogus @finally body location");
-    assert(*SM->getCharacterData(endLoc) == '}' &&
-           "bogus @finally body location");
-
-    startLoc = startLoc.getLocWithOffset(1);
-    InsertText(startLoc, " if (!_rethrow) objc_exception_try_exit(&_stack);\n");
-    endLoc = endLoc.getLocWithOffset(-1);
-    InsertText(endLoc, " if (_rethrow) objc_exception_throw(_rethrow);\n");
-
-    // Set lastCurlyLoc
-    lastCurlyLoc = body->getLocEnd();
-
-    // Now check for any return/continue/go statements within the @try.
-    WarnAboutReturnGotoStmts(S->getTryBody());
-  } else { /* no finally clause - make sure we synthesize an implicit one */
-    buf = "{ /* implicit finally clause */\n";
-    buf += " if (!_rethrow) objc_exception_try_exit(&_stack);\n";
-    buf += " if (_rethrow) objc_exception_throw(_rethrow);\n";
-    buf += "}";
-    ReplaceText(lastCurlyLoc, 1, buf);
-    
-    // Now check for any return/continue/go statements within the @try.
-    // The implicit finally clause won't called if the @try contains any
-    // jump statements.
-    bool hasReturns = false;
-    HasReturnStmts(S->getTryBody(), hasReturns);
-    if (hasReturns)
-      RewriteTryReturnStmts(S->getTryBody());
-  }
-  // Now emit the final closing curly brace...
-  lastCurlyLoc = lastCurlyLoc.getLocWithOffset(1);
-  InsertText(lastCurlyLoc, " } /* @try scope end */\n");
-  return 0;
-}
-
-// This can't be done with ReplaceStmt(S, ThrowExpr), since
-// the throw expression is typically a message expression that's already
-// been rewritten! (which implies the SourceLocation's are invalid).
-Stmt *RewriteObjC::RewriteObjCThrowStmt(ObjCAtThrowStmt *S) {
-  // Get the start location and compute the semi location.
-  SourceLocation startLoc = S->getLocStart();
-  const char *startBuf = SM->getCharacterData(startLoc);
-
-  assert((*startBuf == '@') && "bogus @throw location");
-
-  std::string buf;
-  /* void objc_exception_throw(id) __attribute__((noreturn)); */
-  if (S->getThrowExpr())
-    buf = "objc_exception_throw(";
-  else // add an implicit argument
-    buf = "objc_exception_throw(_caught";
-
-  // handle "@  throw" correctly.
-  const char *wBuf = strchr(startBuf, 'w');
-  assert((*wBuf == 'w') && "@throw: can't find 'w'");
-  ReplaceText(startLoc, wBuf-startBuf+1, buf);
-
-  const char *semiBuf = strchr(startBuf, ';');
-  assert((*semiBuf == ';') && "@throw: can't find ';'");
-  SourceLocation semiLoc = startLoc.getLocWithOffset(semiBuf-startBuf);
-  ReplaceText(semiLoc, 1, ");");
-  return 0;
-}
-
-Stmt *RewriteObjC::RewriteAtEncode(ObjCEncodeExpr *Exp) {
-  // Create a new string expression.
-  QualType StrType = Context->getPointerType(Context->CharTy);
-  std::string StrEncoding;
-  Context->getObjCEncodingForType(Exp->getEncodedType(), StrEncoding);
-  Expr *Replacement = StringLiteral::Create(*Context, StrEncoding,
-                                            StringLiteral::Ascii, false,
-                                            StrType, SourceLocation());
-  ReplaceStmt(Exp, Replacement);
-
-  // Replace this subexpr in the parent.
-  // delete Exp; leak for now, see RewritePropertyOrImplicitSetter() usage for more info.
-  return Replacement;
-}
-
-Stmt *RewriteObjC::RewriteAtSelector(ObjCSelectorExpr *Exp) {
-  if (!SelGetUidFunctionDecl)
-    SynthSelGetUidFunctionDecl();
-  assert(SelGetUidFunctionDecl && "Can't find sel_registerName() decl");
-  // Create a call to sel_registerName("selName").
-  SmallVector<Expr*, 8> SelExprs;
-  QualType argType = Context->getPointerType(Context->CharTy);
-  SelExprs.push_back(StringLiteral::Create(*Context,
-                                           Exp->getSelector().getAsString(),
-                                           StringLiteral::Ascii, false,
-                                           argType, SourceLocation()));
-  CallExpr *SelExp = SynthesizeCallToFunctionDecl(SelGetUidFunctionDecl,
-                                                 &SelExprs[0], SelExprs.size());
-  ReplaceStmt(Exp, SelExp);
-  // delete Exp; leak for now, see RewritePropertyOrImplicitSetter() usage for more info.
-  return SelExp;
-}
-
-CallExpr *RewriteObjC::SynthesizeCallToFunctionDecl(
-  FunctionDecl *FD, Expr **args, unsigned nargs, SourceLocation StartLoc,
-                                                    SourceLocation EndLoc) {
-  // Get the type, we will need to reference it in a couple spots.
-  QualType msgSendType = FD->getType();
-
-  // Create a reference to the objc_msgSend() declaration.
-  DeclRefExpr *DRE = new (Context) DeclRefExpr(FD, false, msgSendType,
-                                               VK_LValue, SourceLocation());
-
-  // Now, we cast the reference to a pointer to the objc_msgSend type.
-  QualType pToFunc = Context->getPointerType(msgSendType);
-  ImplicitCastExpr *ICE = 
-    ImplicitCastExpr::Create(*Context, pToFunc, CK_FunctionToPointerDecay,
-                             DRE, 0, VK_RValue);
-
-  const FunctionType *FT = msgSendType->getAs<FunctionType>();
-
-  CallExpr *Exp =  
-    new (Context) CallExpr(*Context, ICE, args, nargs, 
-                           FT->getCallResultType(*Context),
-                           VK_RValue, EndLoc);
-  return Exp;
-}
-
-static bool scanForProtocolRefs(const char *startBuf, const char *endBuf,
-                                const char *&startRef, const char *&endRef) {
-  while (startBuf < endBuf) {
-    if (*startBuf == '<')
-      startRef = startBuf; // mark the start.
-    if (*startBuf == '>') {
-      if (startRef && *startRef == '<') {
-        endRef = startBuf; // mark the end.
-        return true;
-      }
-      return false;
-    }
-    startBuf++;
-  }
-  return false;
-}
-
-static void scanToNextArgument(const char *&argRef) {
-  int angle = 0;
-  while (*argRef != ')' && (*argRef != ',' || angle > 0)) {
-    if (*argRef == '<')
-      angle++;
-    else if (*argRef == '>')
-      angle--;
-    argRef++;
-  }
-  assert(angle == 0 && "scanToNextArgument - bad protocol type syntax");
-}
-
-bool RewriteObjC::needToScanForQualifiers(QualType T) {
-  if (T->isObjCQualifiedIdType())
-    return true;
-  if (const PointerType *PT = T->getAs<PointerType>()) {
-    if (PT->getPointeeType()->isObjCQualifiedIdType())
-      return true;
-  }
-  if (T->isObjCObjectPointerType()) {
-    T = T->getPointeeType();
-    return T->isObjCQualifiedInterfaceType();
-  }
-  if (T->isArrayType()) {
-    QualType ElemTy = Context->getBaseElementType(T);
-    return needToScanForQualifiers(ElemTy);
-  }
-  return false;
-}
-
-void RewriteObjC::RewriteObjCQualifiedInterfaceTypes(Expr *E) {
-  QualType Type = E->getType();
-  if (needToScanForQualifiers(Type)) {
-    SourceLocation Loc, EndLoc;
-
-    if (const CStyleCastExpr *ECE = dyn_cast<CStyleCastExpr>(E)) {
-      Loc = ECE->getLParenLoc();
-      EndLoc = ECE->getRParenLoc();
-    } else {
-      Loc = E->getLocStart();
-      EndLoc = E->getLocEnd();
-    }
-    // This will defend against trying to rewrite synthesized expressions.
-    if (Loc.isInvalid() || EndLoc.isInvalid())
-      return;
-
-    const char *startBuf = SM->getCharacterData(Loc);
-    const char *endBuf = SM->getCharacterData(EndLoc);
-    const char *startRef = 0, *endRef = 0;
-    if (scanForProtocolRefs(startBuf, endBuf, startRef, endRef)) {
-      // Get the locations of the startRef, endRef.
-      SourceLocation LessLoc = Loc.getLocWithOffset(startRef-startBuf);
-      SourceLocation GreaterLoc = Loc.getLocWithOffset(endRef-startBuf+1);
-      // Comment out the protocol references.
-      InsertText(LessLoc, "/*");
-      InsertText(GreaterLoc, "*/");
-    }
-  }
-}
-
-void RewriteObjC::RewriteObjCQualifiedInterfaceTypes(Decl *Dcl) {
-  SourceLocation Loc;
-  QualType Type;
-  const FunctionProtoType *proto = 0;
-  if (VarDecl *VD = dyn_cast<VarDecl>(Dcl)) {
-    Loc = VD->getLocation();
-    Type = VD->getType();
-  }
-  else if (FunctionDecl *FD = dyn_cast<FunctionDecl>(Dcl)) {
-    Loc = FD->getLocation();
-    // Check for ObjC 'id' and class types that have been adorned with protocol
-    // information (id<p>, C<p>*). The protocol references need to be rewritten!
-    const FunctionType *funcType = FD->getType()->getAs<FunctionType>();
-    assert(funcType && "missing function type");
-    proto = dyn_cast<FunctionProtoType>(funcType);
-    if (!proto)
-      return;
-    Type = proto->getResultType();
-  }
-  else if (FieldDecl *FD = dyn_cast<FieldDecl>(Dcl)) {
-    Loc = FD->getLocation();
-    Type = FD->getType();
-  }
-  else
-    return;
-
-  if (needToScanForQualifiers(Type)) {
-    // Since types are unique, we need to scan the buffer.
-
-    const char *endBuf = SM->getCharacterData(Loc);
-    const char *startBuf = endBuf;
-    while (*startBuf != ';' && *startBuf != '<' && startBuf != MainFileStart)
-      startBuf--; // scan backward (from the decl location) for return type.
-    const char *startRef = 0, *endRef = 0;
-    if (scanForProtocolRefs(startBuf, endBuf, startRef, endRef)) {
-      // Get the locations of the startRef, endRef.
-      SourceLocation LessLoc = Loc.getLocWithOffset(startRef-endBuf);
-      SourceLocation GreaterLoc = Loc.getLocWithOffset(endRef-endBuf+1);
-      // Comment out the protocol references.
-      InsertText(LessLoc, "/*");
-      InsertText(GreaterLoc, "*/");
-    }
-  }
-  if (!proto)
-      return; // most likely, was a variable
-  // Now check arguments.
-  const char *startBuf = SM->getCharacterData(Loc);
-  const char *startFuncBuf = startBuf;
-  for (unsigned i = 0; i < proto->getNumArgs(); i++) {
-    if (needToScanForQualifiers(proto->getArgType(i))) {
-      // Since types are unique, we need to scan the buffer.
-
-      const char *endBuf = startBuf;
-      // scan forward (from the decl location) for argument types.
-      scanToNextArgument(endBuf);
-      const char *startRef = 0, *endRef = 0;
-      if (scanForProtocolRefs(startBuf, endBuf, startRef, endRef)) {
-        // Get the locations of the startRef, endRef.
-        SourceLocation LessLoc =
-          Loc.getLocWithOffset(startRef-startFuncBuf);
-        SourceLocation GreaterLoc =
-          Loc.getLocWithOffset(endRef-startFuncBuf+1);
-        // Comment out the protocol references.
-        InsertText(LessLoc, "/*");
-        InsertText(GreaterLoc, "*/");
-      }
-      startBuf = ++endBuf;
-    }
-    else {
-      // If the function name is derived from a macro expansion, then the
-      // argument buffer will not follow the name. Need to speak with Chris.
-      while (*startBuf && *startBuf != ')' && *startBuf != ',')
-        startBuf++; // scan forward (from the decl location) for argument types.
-      startBuf++;
-    }
-  }
-}
-
-void RewriteObjC::RewriteTypeOfDecl(VarDecl *ND) {
-  QualType QT = ND->getType();
-  const Type* TypePtr = QT->getAs<Type>();
-  if (!isa<TypeOfExprType>(TypePtr))
-    return;
-  while (isa<TypeOfExprType>(TypePtr)) {
-    const TypeOfExprType *TypeOfExprTypePtr = cast<TypeOfExprType>(TypePtr);
-    QT = TypeOfExprTypePtr->getUnderlyingExpr()->getType();
-    TypePtr = QT->getAs<Type>();
-  }
-  // FIXME. This will not work for multiple declarators; as in:
-  // __typeof__(a) b,c,d;
-  std::string TypeAsString(QT.getAsString(Context->getPrintingPolicy()));
-  SourceLocation DeclLoc = ND->getTypeSpecStartLoc();
-  const char *startBuf = SM->getCharacterData(DeclLoc);
-  if (ND->getInit()) {
-    std::string Name(ND->getNameAsString());
-    TypeAsString += " " + Name + " = ";
-    Expr *E = ND->getInit();
-    SourceLocation startLoc;
-    if (const CStyleCastExpr *ECE = dyn_cast<CStyleCastExpr>(E))
-      startLoc = ECE->getLParenLoc();
-    else
-      startLoc = E->getLocStart();
-    startLoc = SM->getExpansionLoc(startLoc);
-    const char *endBuf = SM->getCharacterData(startLoc);
-    ReplaceText(DeclLoc, endBuf-startBuf-1, TypeAsString);
-  }
-  else {
-    SourceLocation X = ND->getLocEnd();
-    X = SM->getExpansionLoc(X);
-    const char *endBuf = SM->getCharacterData(X);
-    ReplaceText(DeclLoc, endBuf-startBuf-1, TypeAsString);
-  }
-}
-
-// SynthSelGetUidFunctionDecl - SEL sel_registerName(const char *str);
-void RewriteObjC::SynthSelGetUidFunctionDecl() {
-  IdentifierInfo *SelGetUidIdent = &Context->Idents.get("sel_registerName");
-  SmallVector<QualType, 16> ArgTys;
-  ArgTys.push_back(Context->getPointerType(Context->CharTy.withConst()));
-  QualType getFuncType =
-    getSimpleFunctionType(Context->getObjCSelType(), &ArgTys[0], ArgTys.size());
-  SelGetUidFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
-                                           SourceLocation(),
-                                           SourceLocation(),
-                                           SelGetUidIdent, getFuncType, 0,
-                                           SC_Extern,
-                                           SC_None, false);
-}
-
-void RewriteObjC::RewriteFunctionDecl(FunctionDecl *FD) {
-  // declared in <objc/objc.h>
-  if (FD->getIdentifier() &&
-      FD->getName() == "sel_registerName") {
-    SelGetUidFunctionDecl = FD;
-    return;
-  }
-  RewriteObjCQualifiedInterfaceTypes(FD);
-}
-
-void RewriteObjC::RewriteBlockPointerType(std::string& Str, QualType Type) {
-  std::string TypeString(Type.getAsString(Context->getPrintingPolicy()));
-  const char *argPtr = TypeString.c_str();
-  if (!strchr(argPtr, '^')) {
-    Str += TypeString;
-    return;
-  }
-  while (*argPtr) {
-    Str += (*argPtr == '^' ? '*' : *argPtr);
-    argPtr++;
-  }
-}
-
-// FIXME. Consolidate this routine with RewriteBlockPointerType.
-void RewriteObjC::RewriteBlockPointerTypeVariable(std::string& Str,
-                                                  ValueDecl *VD) {
-  QualType Type = VD->getType();
-  std::string TypeString(Type.getAsString(Context->getPrintingPolicy()));
-  const char *argPtr = TypeString.c_str();
-  int paren = 0;
-  while (*argPtr) {
-    switch (*argPtr) {
-      case '(':
-        Str += *argPtr;
-        paren++;
-        break;
-      case ')':
-        Str += *argPtr;
-        paren--;
-        break;
-      case '^':
-        Str += '*';
-        if (paren == 1)
-          Str += VD->getNameAsString();
-        break;
-      default:
-        Str += *argPtr;
-        break;
-    }
-    argPtr++;
-  }
-}
-
-
-void RewriteObjC::RewriteBlockLiteralFunctionDecl(FunctionDecl *FD) {
-  SourceLocation FunLocStart = FD->getTypeSpecStartLoc();
-  const FunctionType *funcType = FD->getType()->getAs<FunctionType>();
-  const FunctionProtoType *proto = dyn_cast<FunctionProtoType>(funcType);
-  if (!proto)
-    return;
-  QualType Type = proto->getResultType();
-  std::string FdStr = Type.getAsString(Context->getPrintingPolicy());
-  FdStr += " ";
-  FdStr += FD->getName();
-  FdStr +=  "(";
-  unsigned numArgs = proto->getNumArgs();
-  for (unsigned i = 0; i < numArgs; i++) {
-    QualType ArgType = proto->getArgType(i);
-    RewriteBlockPointerType(FdStr, ArgType);
-    if (i+1 < numArgs)
-      FdStr += ", ";
-  }
-  FdStr +=  ");\n";
-  InsertText(FunLocStart, FdStr);
-  CurFunctionDeclToDeclareForBlock = 0;
-}
-
-// SynthSuperContructorFunctionDecl - id objc_super(id obj, id super);
-void RewriteObjC::SynthSuperContructorFunctionDecl() {
-  if (SuperContructorFunctionDecl)
-    return;
-  IdentifierInfo *msgSendIdent = &Context->Idents.get("__rw_objc_super");
-  SmallVector<QualType, 16> ArgTys;
-  QualType argT = Context->getObjCIdType();
-  assert(!argT.isNull() && "Can't find 'id' type");
-  ArgTys.push_back(argT);
-  ArgTys.push_back(argT);
-  QualType msgSendType = getSimpleFunctionType(Context->getObjCIdType(),
-                                               &ArgTys[0], ArgTys.size());
-  SuperContructorFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
-                                         SourceLocation(),
-                                         SourceLocation(),
-                                         msgSendIdent, msgSendType, 0,
-                                         SC_Extern,
-                                         SC_None, false);
-}
-
-// SynthMsgSendFunctionDecl - id objc_msgSend(id self, SEL op, ...);
-void RewriteObjC::SynthMsgSendFunctionDecl() {
-  IdentifierInfo *msgSendIdent = &Context->Idents.get("objc_msgSend");
-  SmallVector<QualType, 16> ArgTys;
-  QualType argT = Context->getObjCIdType();
-  assert(!argT.isNull() && "Can't find 'id' type");
-  ArgTys.push_back(argT);
-  argT = Context->getObjCSelType();
-  assert(!argT.isNull() && "Can't find 'SEL' type");
-  ArgTys.push_back(argT);
-  QualType msgSendType = getSimpleFunctionType(Context->getObjCIdType(),
-                                               &ArgTys[0], ArgTys.size(),
-                                               true /*isVariadic*/);
-  MsgSendFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
-                                         SourceLocation(),
-                                         SourceLocation(),
-                                         msgSendIdent, msgSendType, 0,
-                                         SC_Extern,
-                                         SC_None, false);
-}
-
-// SynthMsgSendSuperFunctionDecl - id objc_msgSendSuper(struct objc_super *, SEL op, ...);
-void RewriteObjC::SynthMsgSendSuperFunctionDecl() {
-  IdentifierInfo *msgSendIdent = &Context->Idents.get("objc_msgSendSuper");
-  SmallVector<QualType, 16> ArgTys;
-  RecordDecl *RD = RecordDecl::Create(*Context, TTK_Struct, TUDecl,
-                                      SourceLocation(), SourceLocation(),
-                                      &Context->Idents.get("objc_super"));
-  QualType argT = Context->getPointerType(Context->getTagDeclType(RD));
-  assert(!argT.isNull() && "Can't build 'struct objc_super *' type");
-  ArgTys.push_back(argT);
-  argT = Context->getObjCSelType();
-  assert(!argT.isNull() && "Can't find 'SEL' type");
-  ArgTys.push_back(argT);
-  QualType msgSendType = getSimpleFunctionType(Context->getObjCIdType(),
-                                               &ArgTys[0], ArgTys.size(),
-                                               true /*isVariadic*/);
-  MsgSendSuperFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
-                                              SourceLocation(),
-                                              SourceLocation(),
-                                              msgSendIdent, msgSendType, 0,
-                                              SC_Extern,
-                                              SC_None, false);
-}
-
-// SynthMsgSendStretFunctionDecl - id objc_msgSend_stret(id self, SEL op, ...);
-void RewriteObjC::SynthMsgSendStretFunctionDecl() {
-  IdentifierInfo *msgSendIdent = &Context->Idents.get("objc_msgSend_stret");
-  SmallVector<QualType, 16> ArgTys;
-  QualType argT = Context->getObjCIdType();
-  assert(!argT.isNull() && "Can't find 'id' type");
-  ArgTys.push_back(argT);
-  argT = Context->getObjCSelType();
-  assert(!argT.isNull() && "Can't find 'SEL' type");
-  ArgTys.push_back(argT);
-  QualType msgSendType = getSimpleFunctionType(Context->getObjCIdType(),
-                                               &ArgTys[0], ArgTys.size(),
-                                               true /*isVariadic*/);
-  MsgSendStretFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
-                                         SourceLocation(),
-                                         SourceLocation(),
-                                         msgSendIdent, msgSendType, 0,
-                                         SC_Extern,
-                                         SC_None, false);
-}
-
-// SynthMsgSendSuperStretFunctionDecl -
-// id objc_msgSendSuper_stret(struct objc_super *, SEL op, ...);
-void RewriteObjC::SynthMsgSendSuperStretFunctionDecl() {
-  IdentifierInfo *msgSendIdent =
-    &Context->Idents.get("objc_msgSendSuper_stret");
-  SmallVector<QualType, 16> ArgTys;
-  RecordDecl *RD = RecordDecl::Create(*Context, TTK_Struct, TUDecl,
-                                      SourceLocation(), SourceLocation(),
-                                      &Context->Idents.get("objc_super"));
-  QualType argT = Context->getPointerType(Context->getTagDeclType(RD));
-  assert(!argT.isNull() && "Can't build 'struct objc_super *' type");
-  ArgTys.push_back(argT);
-  argT = Context->getObjCSelType();
-  assert(!argT.isNull() && "Can't find 'SEL' type");
-  ArgTys.push_back(argT);
-  QualType msgSendType = getSimpleFunctionType(Context->getObjCIdType(),
-                                               &ArgTys[0], ArgTys.size(),
-                                               true /*isVariadic*/);
-  MsgSendSuperStretFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
-                                                       SourceLocation(),
-                                                       SourceLocation(),
-                                              msgSendIdent, msgSendType, 0,
-                                              SC_Extern,
-                                              SC_None, false);
-}
-
-// SynthMsgSendFpretFunctionDecl - double objc_msgSend_fpret(id self, SEL op, ...);
-void RewriteObjC::SynthMsgSendFpretFunctionDecl() {
-  IdentifierInfo *msgSendIdent = &Context->Idents.get("objc_msgSend_fpret");
-  SmallVector<QualType, 16> ArgTys;
-  QualType argT = Context->getObjCIdType();
-  assert(!argT.isNull() && "Can't find 'id' type");
-  ArgTys.push_back(argT);
-  argT = Context->getObjCSelType();
-  assert(!argT.isNull() && "Can't find 'SEL' type");
-  ArgTys.push_back(argT);
-  QualType msgSendType = getSimpleFunctionType(Context->DoubleTy,
-                                               &ArgTys[0], ArgTys.size(),
-                                               true /*isVariadic*/);
-  MsgSendFpretFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
-                                              SourceLocation(),
-                                              SourceLocation(),
-                                              msgSendIdent, msgSendType, 0,
-                                              SC_Extern,
-                                              SC_None, false);
-}
-
-// SynthGetClassFunctionDecl - id objc_getClass(const char *name);
-void RewriteObjC::SynthGetClassFunctionDecl() {
-  IdentifierInfo *getClassIdent = &Context->Idents.get("objc_getClass");
-  SmallVector<QualType, 16> ArgTys;
-  ArgTys.push_back(Context->getPointerType(Context->CharTy.withConst()));
-  QualType getClassType = getSimpleFunctionType(Context->getObjCIdType(),
-                                                &ArgTys[0], ArgTys.size());
-  GetClassFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
-                                          SourceLocation(),
-                                          SourceLocation(),
-                                          getClassIdent, getClassType, 0,
-                                          SC_Extern,
-                                          SC_None, false);
-}
-
-// SynthGetSuperClassFunctionDecl - Class class_getSuperclass(Class cls);
-void RewriteObjC::SynthGetSuperClassFunctionDecl() {
-  IdentifierInfo *getSuperClassIdent = 
-    &Context->Idents.get("class_getSuperclass");
-  SmallVector<QualType, 16> ArgTys;
-  ArgTys.push_back(Context->getObjCClassType());
-  QualType getClassType = getSimpleFunctionType(Context->getObjCClassType(),
-                                                &ArgTys[0], ArgTys.size());
-  GetSuperClassFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
-                                                   SourceLocation(),
-                                                   SourceLocation(),
-                                                   getSuperClassIdent,
-                                                   getClassType, 0,
-                                                   SC_Extern,
-                                                   SC_None,
-                                                   false);
-}
-
-// SynthGetMetaClassFunctionDecl - id objc_getMetaClass(const char *name);
-void RewriteObjC::SynthGetMetaClassFunctionDecl() {
-  IdentifierInfo *getClassIdent = &Context->Idents.get("objc_getMetaClass");
-  SmallVector<QualType, 16> ArgTys;
-  ArgTys.push_back(Context->getPointerType(Context->CharTy.withConst()));
-  QualType getClassType = getSimpleFunctionType(Context->getObjCIdType(),
-                                                &ArgTys[0], ArgTys.size());
-  GetMetaClassFunctionDecl = FunctionDecl::Create(*Context, TUDecl,
-                                              SourceLocation(),
-                                              SourceLocation(),
-                                              getClassIdent, getClassType, 0,
-                                              SC_Extern,
-                                              SC_None, false);
-}
-
-Stmt *RewriteObjC::RewriteObjCStringLiteral(ObjCStringLiteral *Exp) {
-  QualType strType = getConstantStringStructType();
-
-  std::string S = "__NSConstantStringImpl_";
-
-  std::string tmpName = InFileName;
-  unsigned i;
-  for (i=0; i < tmpName.length(); i++) {
-    char c = tmpName.at(i);
-    // replace any non alphanumeric characters with '_'.
-    if (!isalpha(c) && (c < '0' || c > '9'))
-      tmpName[i] = '_';
-  }
-  S += tmpName;
-  S += "_";
-  S += utostr(NumObjCStringLiterals++);
-
-  Preamble += "static __NSConstantStringImpl " + S;
-  Preamble += " __attribute__ ((section (\"__DATA, __cfstring\"))) = {__CFConstantStringClassReference,";
-  Preamble += "0x000007c8,"; // utf8_str
-  // The pretty printer for StringLiteral handles escape characters properly.
-  std::string prettyBufS;
-  llvm::raw_string_ostream prettyBuf(prettyBufS);
-  Exp->getString()->printPretty(prettyBuf, 0, PrintingPolicy(LangOpts));
-  Preamble += prettyBuf.str();
-  Preamble += ",";
-  Preamble += utostr(Exp->getString()->getByteLength()) + "};\n";
-
-  VarDecl *NewVD = VarDecl::Create(*Context, TUDecl, SourceLocation(),
-                                   SourceLocation(), &Context->Idents.get(S),
-                                   strType, 0, SC_Static, SC_None);
-  DeclRefExpr *DRE = new (Context) DeclRefExpr(NewVD, false, strType, VK_LValue,
-                                               SourceLocation());
-  Expr *Unop = new (Context) UnaryOperator(DRE, UO_AddrOf,
-                                 Context->getPointerType(DRE->getType()),
-                                           VK_RValue, OK_Ordinary,
-                                           SourceLocation());
-  // cast to NSConstantString *
-  CastExpr *cast = NoTypeInfoCStyleCastExpr(Context, Exp->getType(),
-                                            CK_CPointerToObjCPointerCast, Unop);
-  ReplaceStmt(Exp, cast);
-  // delete Exp; leak for now, see RewritePropertyOrImplicitSetter() usage for more info.
-  return cast;
-}
-
-// struct objc_super { struct objc_object *receiver; struct objc_class *super; };
-QualType RewriteObjC::getSuperStructType() {
-  if (!SuperStructDecl) {
-    SuperStructDecl = RecordDecl::Create(*Context, TTK_Struct, TUDecl,
-                                         SourceLocation(), SourceLocation(),
-                                         &Context->Idents.get("objc_super"));
-    QualType FieldTypes[2];
-
-    // struct objc_object *receiver;
-    FieldTypes[0] = Context->getObjCIdType();
-    // struct objc_class *super;
-    FieldTypes[1] = Context->getObjCClassType();
-
-    // Create fields
-    for (unsigned i = 0; i < 2; ++i) {
-      SuperStructDecl->addDecl(FieldDecl::Create(*Context, SuperStructDecl,
-                                                 SourceLocation(),
-                                                 SourceLocation(), 0,
-                                                 FieldTypes[i], 0,
-                                                 /*BitWidth=*/0,
-                                                 /*Mutable=*/false,
-                                                 ICIS_NoInit));
-    }
-
-    SuperStructDecl->completeDefinition();
-  }
-  return Context->getTagDeclType(SuperStructDecl);
-}
-
-QualType RewriteObjC::getConstantStringStructType() {
-  if (!ConstantStringDecl) {
-    ConstantStringDecl = RecordDecl::Create(*Context, TTK_Struct, TUDecl,
-                                            SourceLocation(), SourceLocation(),
-                         &Context->Idents.get("__NSConstantStringImpl"));
-    QualType FieldTypes[4];
-
-    // struct objc_object *receiver;
-    FieldTypes[0] = Context->getObjCIdType();
-    // int flags;
-    FieldTypes[1] = Context->IntTy;
-    // char *str;
-    FieldTypes[2] = Context->getPointerType(Context->CharTy);
-    // long length;
-    FieldTypes[3] = Context->LongTy;
-
-    // Create fields
-    for (unsigned i = 0; i < 4; ++i) {
-      ConstantStringDecl->addDecl(FieldDecl::Create(*Context,
-                                                    ConstantStringDecl,
-                                                    SourceLocation(),
-                                                    SourceLocation(), 0,
-                                                    FieldTypes[i], 0,
-                                                    /*BitWidth=*/0,
-                                                    /*Mutable=*/true,
-                                                    ICIS_NoInit));
-    }
-
-    ConstantStringDecl->completeDefinition();
-  }
-  return Context->getTagDeclType(ConstantStringDecl);
-}
-
-CallExpr *RewriteObjC::SynthMsgSendStretCallExpr(FunctionDecl *MsgSendStretFlavor,
-                                                QualType msgSendType, 
-                                                QualType returnType, 
-                                                SmallVectorImpl<QualType> &ArgTypes,
-                                                SmallVectorImpl<Expr*> &MsgExprs,
-                                                ObjCMethodDecl *Method) {
-  // Create a reference to the objc_msgSend_stret() declaration.
-  DeclRefExpr *STDRE = new (Context) DeclRefExpr(MsgSendStretFlavor,
-                                                 false, msgSendType,
-                                                 VK_LValue, SourceLocation());
-  // Need to cast objc_msgSend_stret to "void *" (see above comment).
-  CastExpr *cast = NoTypeInfoCStyleCastExpr(Context,
-                                  Context->getPointerType(Context->VoidTy),
-                                  CK_BitCast, STDRE);
-  // Now do the "normal" pointer to function cast.
-  QualType castType = getSimpleFunctionType(returnType, &ArgTypes[0], ArgTypes.size(),
-                                            Method ? Method->isVariadic() : false);
-  castType = Context->getPointerType(castType);
-  cast = NoTypeInfoCStyleCastExpr(Context, castType, CK_BitCast,
-                                            cast);
-  
-  // Don't forget the parens to enforce the proper binding.
-  ParenExpr *PE = new (Context) ParenExpr(SourceLocation(), SourceLocation(), cast);
-  
-  const FunctionType *FT = msgSendType->getAs<FunctionType>();
-  CallExpr *STCE = new (Context) CallExpr(*Context, PE, &MsgExprs[0],
-                                          MsgExprs.size(),
-                                          FT->getResultType(), VK_RValue,
-                                          SourceLocation());
-  return STCE;
-  
-}
-
-
-Stmt *RewriteObjC::SynthMessageExpr(ObjCMessageExpr *Exp,
-                                    SourceLocation StartLoc,
-                                    SourceLocation EndLoc) {
-  if (!SelGetUidFunctionDecl)
-    SynthSelGetUidFunctionDecl();
-  if (!MsgSendFunctionDecl)
-    SynthMsgSendFunctionDecl();
-  if (!MsgSendSuperFunctionDecl)
-    SynthMsgSendSuperFunctionDecl();
-  if (!MsgSendStretFunctionDecl)
-    SynthMsgSendStretFunctionDecl();
-  if (!MsgSendSuperStretFunctionDecl)
-    SynthMsgSendSuperStretFunctionDecl();
-  if (!MsgSendFpretFunctionDecl)
-    SynthMsgSendFpretFunctionDecl();
-  if (!GetClassFunctionDecl)
-    SynthGetClassFunctionDecl();
-  if (!GetSuperClassFunctionDecl)
-    SynthGetSuperClassFunctionDecl();
-  if (!GetMetaClassFunctionDecl)
-    SynthGetMetaClassFunctionDecl();
-
-  // default to objc_msgSend().
-  FunctionDecl *MsgSendFlavor = MsgSendFunctionDecl;
-  // May need to use objc_msgSend_stret() as well.
-  FunctionDecl *MsgSendStretFlavor = 0;
-  if (ObjCMethodDecl *mDecl = Exp->getMethodDecl()) {
-    QualType resultType = mDecl->getResultType();
-    if (resultType->isRecordType())
-      MsgSendStretFlavor = MsgSendStretFunctionDecl;
-    else if (resultType->isRealFloatingType())
-      MsgSendFlavor = MsgSendFpretFunctionDecl;
-  }
-
-  // Synthesize a call to objc_msgSend().
-  SmallVector<Expr*, 8> MsgExprs;
-  switch (Exp->getReceiverKind()) {
-  case ObjCMessageExpr::SuperClass: {
-    MsgSendFlavor = MsgSendSuperFunctionDecl;
-    if (MsgSendStretFlavor)
-      MsgSendStretFlavor = MsgSendSuperStretFunctionDecl;
-    assert(MsgSendFlavor && "MsgSendFlavor is NULL!");
-
-    ObjCInterfaceDecl *ClassDecl = CurMethodDef->getClassInterface();
-
-    SmallVector<Expr*, 4> InitExprs;
-
-    // set the receiver to self, the first argument to all methods.
-    InitExprs.push_back(
-      NoTypeInfoCStyleCastExpr(Context, Context->getObjCIdType(),
-                               CK_BitCast,
-                   new (Context) DeclRefExpr(CurMethodDef->getSelfDecl(),
-                                             false,
-                                             Context->getObjCIdType(),
-                                             VK_RValue,
-                                             SourceLocation()))
-                        ); // set the 'receiver'.
-
-    // (id)class_getSuperclass((Class)objc_getClass("CurrentClass"))
-    SmallVector<Expr*, 8> ClsExprs;
-    QualType argType = Context->getPointerType(Context->CharTy);
-    ClsExprs.push_back(StringLiteral::Create(*Context,
-                                   ClassDecl->getIdentifier()->getName(),
-                                   StringLiteral::Ascii, false,
-                                   argType, SourceLocation()));
-    CallExpr *Cls = SynthesizeCallToFunctionDecl(GetMetaClassFunctionDecl,
-                                                 &ClsExprs[0],
-                                                 ClsExprs.size(),
-                                                 StartLoc,
-                                                 EndLoc);
-    // (Class)objc_getClass("CurrentClass")
-    CastExpr *ArgExpr = NoTypeInfoCStyleCastExpr(Context,
-                                             Context->getObjCClassType(),
-                                             CK_BitCast, Cls);
-    ClsExprs.clear();
-    ClsExprs.push_back(ArgExpr);
-    Cls = SynthesizeCallToFunctionDecl(GetSuperClassFunctionDecl,
-                                       &ClsExprs[0], ClsExprs.size(),
-                                       StartLoc, EndLoc);
-    
-    // (id)class_getSuperclass((Class)objc_getClass("CurrentClass"))
-    // To turn off a warning, type-cast to 'id'
-    InitExprs.push_back( // set 'super class', using class_getSuperclass().
-                        NoTypeInfoCStyleCastExpr(Context,
-                                                 Context->getObjCIdType(),
-                                                 CK_BitCast, Cls));
-    // struct objc_super
-    QualType superType = getSuperStructType();
-    Expr *SuperRep;
-
-    if (LangOpts.MicrosoftExt) {
-      SynthSuperContructorFunctionDecl();
-      // Simulate a contructor call...
-      DeclRefExpr *DRE = new (Context) DeclRefExpr(SuperContructorFunctionDecl,
-                                                   false, superType, VK_LValue,
-                                                   SourceLocation());
-      SuperRep = new (Context) CallExpr(*Context, DRE, &InitExprs[0],
-                                        InitExprs.size(),
-                                        superType, VK_LValue,
-                                        SourceLocation());
-      // The code for super is a little tricky to prevent collision with
-      // the structure definition in the header. The rewriter has it's own
-      // internal definition (__rw_objc_super) that is uses. This is why
-      // we need the cast below. For example:
-      // (struct objc_super *)&__rw_objc_super((id)self, (id)objc_getClass("SUPER"))
-      //
-      SuperRep = new (Context) UnaryOperator(SuperRep, UO_AddrOf,
-                               Context->getPointerType(SuperRep->getType()),
-                                             VK_RValue, OK_Ordinary,
-                                             SourceLocation());
-      SuperRep = NoTypeInfoCStyleCastExpr(Context,
-                                          Context->getPointerType(superType),
-                                          CK_BitCast, SuperRep);
-    } else {
-      // (struct objc_super) { <exprs from above> }
-      InitListExpr *ILE =
-        new (Context) InitListExpr(*Context, SourceLocation(),
-                                   &InitExprs[0], InitExprs.size(),
-                                   SourceLocation());
-      TypeSourceInfo *superTInfo
-        = Context->getTrivialTypeSourceInfo(superType);
-      SuperRep = new (Context) CompoundLiteralExpr(SourceLocation(), superTInfo,
-                                                   superType, VK_LValue,
-                                                   ILE, false);
-      // struct objc_super *
-      SuperRep = new (Context) UnaryOperator(SuperRep, UO_AddrOf,
-                               Context->getPointerType(SuperRep->getType()),
-                                             VK_RValue, OK_Ordinary,
-                                             SourceLocation());
-    }
-    MsgExprs.push_back(SuperRep);
-    break;
-  }
-
-  case ObjCMessageExpr::Class: {
-    SmallVector<Expr*, 8> ClsExprs;
-    QualType argType = Context->getPointerType(Context->CharTy);
-    ObjCInterfaceDecl *Class
-      = Exp->getClassReceiver()->getAs<ObjCObjectType>()->getInterface();
-    IdentifierInfo *clsName = Class->getIdentifier();
-    ClsExprs.push_back(StringLiteral::Create(*Context,
-                                             clsName->getName(),
-                                             StringLiteral::Ascii, false,
-                                             argType, SourceLocation()));
-    CallExpr *Cls = SynthesizeCallToFunctionDecl(GetClassFunctionDecl,
-                                                 &ClsExprs[0],
-                                                 ClsExprs.size(), 
-                                                 StartLoc, EndLoc);
-    MsgExprs.push_back(Cls);
-    break;
-  }
-
-  case ObjCMessageExpr::SuperInstance:{
-    MsgSendFlavor = MsgSendSuperFunctionDecl;
-    if (MsgSendStretFlavor)
-      MsgSendStretFlavor = MsgSendSuperStretFunctionDecl;
-    assert(MsgSendFlavor && "MsgSendFlavor is NULL!");
-    ObjCInterfaceDecl *ClassDecl = CurMethodDef->getClassInterface();
-    SmallVector<Expr*, 4> InitExprs;
-
-    InitExprs.push_back(
-      NoTypeInfoCStyleCastExpr(Context, Context->getObjCIdType(),
-                               CK_BitCast,
-                   new (Context) DeclRefExpr(CurMethodDef->getSelfDecl(),
-                                             false,
-                                             Context->getObjCIdType(),
-                                             VK_RValue, SourceLocation()))
-                        ); // set the 'receiver'.
-    
-    // (id)class_getSuperclass((Class)objc_getClass("CurrentClass"))
-    SmallVector<Expr*, 8> ClsExprs;
-    QualType argType = Context->getPointerType(Context->CharTy);
-    ClsExprs.push_back(StringLiteral::Create(*Context,
-                                   ClassDecl->getIdentifier()->getName(),
-                                   StringLiteral::Ascii, false, argType,
-                                   SourceLocation()));
-    CallExpr *Cls = SynthesizeCallToFunctionDecl(GetClassFunctionDecl,
-                                                 &ClsExprs[0],
-                                                 ClsExprs.size(), 
-                                                 StartLoc, EndLoc);
-    // (Class)objc_getClass("CurrentClass")
-    CastExpr *ArgExpr = NoTypeInfoCStyleCastExpr(Context,
-                                                 Context->getObjCClassType(),
-                                                 CK_BitCast, Cls);
-    ClsExprs.clear();
-    ClsExprs.push_back(ArgExpr);
-    Cls = SynthesizeCallToFunctionDecl(GetSuperClassFunctionDecl,
-                                       &ClsExprs[0], ClsExprs.size(),
-                                       StartLoc, EndLoc);
-    
-    // (id)class_getSuperclass((Class)objc_getClass("CurrentClass"))
-    // To turn off a warning, type-cast to 'id'
-    InitExprs.push_back(
-      // set 'super class', using class_getSuperclass().
-      NoTypeInfoCStyleCastExpr(Context, Context->getObjCIdType(),
-                               CK_BitCast, Cls));
-    // struct objc_super
-    QualType superType = getSuperStructType();
-    Expr *SuperRep;
-
-    if (LangOpts.MicrosoftExt) {
-      SynthSuperContructorFunctionDecl();
-      // Simulate a contructor call...
-      DeclRefExpr *DRE = new (Context) DeclRefExpr(SuperContructorFunctionDecl,
-                                                   false, superType, VK_LValue,
-                                                   SourceLocation());
-      SuperRep = new (Context) CallExpr(*Context, DRE, &InitExprs[0],
-                                        InitExprs.size(),
-                                        superType, VK_LValue, SourceLocation());
-      // The code for super is a little tricky to prevent collision with
-      // the structure definition in the header. The rewriter has it's own
-      // internal definition (__rw_objc_super) that is uses. This is why
-      // we need the cast below. For example:
-      // (struct objc_super *)&__rw_objc_super((id)self, (id)objc_getClass("SUPER"))
-      //
-      SuperRep = new (Context) UnaryOperator(SuperRep, UO_AddrOf,
-                               Context->getPointerType(SuperRep->getType()),
-                               VK_RValue, OK_Ordinary,
-                               SourceLocation());
-      SuperRep = NoTypeInfoCStyleCastExpr(Context,
-                               Context->getPointerType(superType),
-                               CK_BitCast, SuperRep);
-    } else {
-      // (struct objc_super) { <exprs from above> }
-      InitListExpr *ILE =
-        new (Context) InitListExpr(*Context, SourceLocation(),
-                                   &InitExprs[0], InitExprs.size(),
-                                   SourceLocation());
-      TypeSourceInfo *superTInfo
-        = Context->getTrivialTypeSourceInfo(superType);
-      SuperRep = new (Context) CompoundLiteralExpr(SourceLocation(), superTInfo,
-                                                   superType, VK_RValue, ILE,
-                                                   false);
-    }
-    MsgExprs.push_back(SuperRep);
-    break;
-  }
-
-  case ObjCMessageExpr::Instance: {
-    // Remove all type-casts because it may contain objc-style types; e.g.
-    // Foo<Proto> *.
-    Expr *recExpr = Exp->getInstanceReceiver();
-    while (CStyleCastExpr *CE = dyn_cast<CStyleCastExpr>(recExpr))
-      recExpr = CE->getSubExpr();
-    CastKind CK = recExpr->getType()->isObjCObjectPointerType()
-                    ? CK_BitCast : recExpr->getType()->isBlockPointerType()
-                                     ? CK_BlockPointerToObjCPointerCast
-                                     : CK_CPointerToObjCPointerCast;
-
-    recExpr = NoTypeInfoCStyleCastExpr(Context, Context->getObjCIdType(),
-                                       CK, recExpr);
-    MsgExprs.push_back(recExpr);
-    break;
-  }
-  }
-
-  // Create a call to sel_registerName("selName"), it will be the 2nd argument.
-  SmallVector<Expr*, 8> SelExprs;
-  QualType argType = Context->getPointerType(Context->CharTy);
-  SelExprs.push_back(StringLiteral::Create(*Context,
-                                       Exp->getSelector().getAsString(),
-                                       StringLiteral::Ascii, false,
-                                       argType, SourceLocation()));
-  CallExpr *SelExp = SynthesizeCallToFunctionDecl(SelGetUidFunctionDecl,
-                                                 &SelExprs[0], SelExprs.size(),
-                                                  StartLoc,
-                                                  EndLoc);
-  MsgExprs.push_back(SelExp);
-
-  // Now push any user supplied arguments.
-  for (unsigned i = 0; i < Exp->getNumArgs(); i++) {
-    Expr *userExpr = Exp->getArg(i);
-    // Make all implicit casts explicit...ICE comes in handy:-)
-    if (ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(userExpr)) {
-      // Reuse the ICE type, it is exactly what the doctor ordered.
-      QualType type = ICE->getType();
-      if (needToScanForQualifiers(type))
-        type = Context->getObjCIdType();
-      // Make sure we convert "type (^)(...)" to "type (*)(...)".
-      (void)convertBlockPointerToFunctionPointer(type);
-      const Expr *SubExpr = ICE->IgnoreParenImpCasts();
-      CastKind CK;
-      if (SubExpr->getType()->isIntegralType(*Context) && 
-          type->isBooleanType()) {
-        CK = CK_IntegralToBoolean;
-      } else if (type->isObjCObjectPointerType()) {
-        if (SubExpr->getType()->isBlockPointerType()) {
-          CK = CK_BlockPointerToObjCPointerCast;
-        } else if (SubExpr->getType()->isPointerType()) {
-          CK = CK_CPointerToObjCPointerCast;
-        } else {
-          CK = CK_BitCast;
-        }
-      } else {
-        CK = CK_BitCast;
-      }
-
-      userExpr = NoTypeInfoCStyleCastExpr(Context, type, CK, userExpr);
-    }
-    // Make id<P...> cast into an 'id' cast.
-    else if (CStyleCastExpr *CE = dyn_cast<CStyleCastExpr>(userExpr)) {
-      if (CE->getType()->isObjCQualifiedIdType()) {
-        while ((CE = dyn_cast<CStyleCastExpr>(userExpr)))
-          userExpr = CE->getSubExpr();
-        CastKind CK;
-        if (userExpr->getType()->isIntegralType(*Context)) {
-          CK = CK_IntegralToPointer;
-        } else if (userExpr->getType()->isBlockPointerType()) {
-          CK = CK_BlockPointerToObjCPointerCast;
-        } else if (userExpr->getType()->isPointerType()) {
-          CK = CK_CPointerToObjCPointerCast;
-        } else {
-          CK = CK_BitCast;
-        }
-        userExpr = NoTypeInfoCStyleCastExpr(Context, Context->getObjCIdType(),
-                                            CK, userExpr);
-      }
-    }
-    MsgExprs.push_back(userExpr);
-    // We've transferred the ownership to MsgExprs. For now, we *don't* null
-    // out the argument in the original expression (since we aren't deleting
-    // the ObjCMessageExpr). See RewritePropertyOrImplicitSetter() usage for more info.
-    //Exp->setArg(i, 0);
-  }
-  // Generate the funky cast.
-  CastExpr *cast;
-  SmallVector<QualType, 8> ArgTypes;
-  QualType returnType;
-
-  // Push 'id' and 'SEL', the 2 implicit arguments.
-  if (MsgSendFlavor == MsgSendSuperFunctionDecl)
-    ArgTypes.push_back(Context->getPointerType(getSuperStructType()));
-  else
-    ArgTypes.push_back(Context->getObjCIdType());
-  ArgTypes.push_back(Context->getObjCSelType());
-  if (ObjCMethodDecl *OMD = Exp->getMethodDecl()) {
-    // Push any user argument types.
-    for (ObjCMethodDecl::param_iterator PI = OMD->param_begin(),
-         E = OMD->param_end(); PI != E; ++PI) {
-      QualType t = (*PI)->getType()->isObjCQualifiedIdType()
-                     ? Context->getObjCIdType()
-                     : (*PI)->getType();
-      // Make sure we convert "t (^)(...)" to "t (*)(...)".
-      (void)convertBlockPointerToFunctionPointer(t);
-      ArgTypes.push_back(t);
-    }
-    returnType = Exp->getType();
-    convertToUnqualifiedObjCType(returnType);
-    (void)convertBlockPointerToFunctionPointer(returnType);
-  } else {
-    returnType = Context->getObjCIdType();
-  }
-  // Get the type, we will need to reference it in a couple spots.
-  QualType msgSendType = MsgSendFlavor->getType();
-
-  // Create a reference to the objc_msgSend() declaration.
-  DeclRefExpr *DRE = new (Context) DeclRefExpr(MsgSendFlavor, false, msgSendType,
-                                               VK_LValue, SourceLocation());
-
-  // Need to cast objc_msgSend to "void *" (to workaround a GCC bandaid).
-  // If we don't do this cast, we get the following bizarre warning/note:
-  // xx.m:13: warning: function called through a non-compatible type
-  // xx.m:13: note: if this code is reached, the program will abort
-  cast = NoTypeInfoCStyleCastExpr(Context,
-                                  Context->getPointerType(Context->VoidTy),
-                                  CK_BitCast, DRE);
-
-  // Now do the "normal" pointer to function cast.
-  QualType castType =
-    getSimpleFunctionType(returnType, &ArgTypes[0], ArgTypes.size(),
-      // If we don't have a method decl, force a variadic cast.
-      Exp->getMethodDecl() ? Exp->getMethodDecl()->isVariadic() : true);
-  castType = Context->getPointerType(castType);
-  cast = NoTypeInfoCStyleCastExpr(Context, castType, CK_BitCast,
-                                  cast);
-
-  // Don't forget the parens to enforce the proper binding.
-  ParenExpr *PE = new (Context) ParenExpr(StartLoc, EndLoc, cast);
-
-  const FunctionType *FT = msgSendType->getAs<FunctionType>();
-  CallExpr *CE = new (Context) CallExpr(*Context, PE, &MsgExprs[0],
-                                        MsgExprs.size(),
-                                        FT->getResultType(), VK_RValue,
-                                        EndLoc);
-  Stmt *ReplacingStmt = CE;
-  if (MsgSendStretFlavor) {
-    // We have the method which returns a struct/union. Must also generate
-    // call to objc_msgSend_stret and hang both varieties on a conditional
-    // expression which dictate which one to envoke depending on size of
-    // method's return type.
-    
-    CallExpr *STCE = SynthMsgSendStretCallExpr(MsgSendStretFlavor, 
-                                               msgSendType, returnType, 
-                                               ArgTypes, MsgExprs,
-                                               Exp->getMethodDecl());
-
-    // Build sizeof(returnType)
-    UnaryExprOrTypeTraitExpr *sizeofExpr =
-       new (Context) UnaryExprOrTypeTraitExpr(UETT_SizeOf,
-                                 Context->getTrivialTypeSourceInfo(returnType),
-                                 Context->getSizeType(), SourceLocation(),
-                                 SourceLocation());
-    // (sizeof(returnType) <= 8 ? objc_msgSend(...) : objc_msgSend_stret(...))
-    // FIXME: Value of 8 is base on ppc32/x86 ABI for the most common cases.
-    // For X86 it is more complicated and some kind of target specific routine
-    // is needed to decide what to do.
-    unsigned IntSize =
-      static_cast<unsigned>(Context->getTypeSize(Context->IntTy));
-    IntegerLiteral *limit = IntegerLiteral::Create(*Context,
-                                                   llvm::APInt(IntSize, 8),
-                                                   Context->IntTy,
-                                                   SourceLocation());
-    BinaryOperator *lessThanExpr = 
-      new (Context) BinaryOperator(sizeofExpr, limit, BO_LE, Context->IntTy,
-                                   VK_RValue, OK_Ordinary, SourceLocation());
-    // (sizeof(returnType) <= 8 ? objc_msgSend(...) : objc_msgSend_stret(...))
-    ConditionalOperator *CondExpr =
-      new (Context) ConditionalOperator(lessThanExpr,
-                                        SourceLocation(), CE,
-                                        SourceLocation(), STCE,
-                                        returnType, VK_RValue, OK_Ordinary);
-    ReplacingStmt = new (Context) ParenExpr(SourceLocation(), SourceLocation(), 
-                                            CondExpr);
-  }
-  // delete Exp; leak for now, see RewritePropertyOrImplicitSetter() usage for more info.
-  return ReplacingStmt;
-}
-
-Stmt *RewriteObjC::RewriteMessageExpr(ObjCMessageExpr *Exp) {
-  Stmt *ReplacingStmt = SynthMessageExpr(Exp, Exp->getLocStart(),
-                                         Exp->getLocEnd());
-
-  // Now do the actual rewrite.
-  ReplaceStmt(Exp, ReplacingStmt);
-
-  // delete Exp; leak for now, see RewritePropertyOrImplicitSetter() usage for more info.
-  return ReplacingStmt;
-}
-
-// typedef struct objc_object Protocol;
-QualType RewriteObjC::getProtocolType() {
-  if (!ProtocolTypeDecl) {
-    TypeSourceInfo *TInfo
-      = Context->getTrivialTypeSourceInfo(Context->getObjCIdType());
-    ProtocolTypeDecl = TypedefDecl::Create(*Context, TUDecl,
-                                           SourceLocation(), SourceLocation(),
-                                           &Context->Idents.get("Protocol"),
-                                           TInfo);
-  }
-  return Context->getTypeDeclType(ProtocolTypeDecl);
-}
-
-/// RewriteObjCProtocolExpr - Rewrite a protocol expression into
-/// a synthesized/forward data reference (to the protocol's metadata).
-/// The forward references (and metadata) are generated in
-/// RewriteObjC::HandleTranslationUnit().
-Stmt *RewriteObjC::RewriteObjCProtocolExpr(ObjCProtocolExpr *Exp) {
-  std::string Name = "_OBJC_PROTOCOL_" + Exp->getProtocol()->getNameAsString();
-  IdentifierInfo *ID = &Context->Idents.get(Name);
-  VarDecl *VD = VarDecl::Create(*Context, TUDecl, SourceLocation(),
-                                SourceLocation(), ID, getProtocolType(), 0,
-                                SC_Extern, SC_None);
-  DeclRefExpr *DRE = new (Context) DeclRefExpr(VD, false, getProtocolType(),
-                                               VK_LValue, SourceLocation());
-  Expr *DerefExpr = new (Context) UnaryOperator(DRE, UO_AddrOf,
-                             Context->getPointerType(DRE->getType()),
-                             VK_RValue, OK_Ordinary, SourceLocation());
-  CastExpr *castExpr = NoTypeInfoCStyleCastExpr(Context, DerefExpr->getType(),
-                                                CK_BitCast,
-                                                DerefExpr);
-  ReplaceStmt(Exp, castExpr);
-  ProtocolExprDecls.insert(Exp->getProtocol()->getCanonicalDecl());
-  // delete Exp; leak for now, see RewritePropertyOrImplicitSetter() usage for more info.
-  return castExpr;
-
-}
-
-bool RewriteObjC::BufferContainsPPDirectives(const char *startBuf,
-                                             const char *endBuf) {
-  while (startBuf < endBuf) {
-    if (*startBuf == '#') {
-      // Skip whitespace.
-      for (++startBuf; startBuf[0] == ' ' || startBuf[0] == '\t'; ++startBuf)
-        ;
-      if (!strncmp(startBuf, "if", strlen("if")) ||
-          !strncmp(startBuf, "ifdef", strlen("ifdef")) ||
-          !strncmp(startBuf, "ifndef", strlen("ifndef")) ||
-          !strncmp(startBuf, "define", strlen("define")) ||
-          !strncmp(startBuf, "undef", strlen("undef")) ||
-          !strncmp(startBuf, "else", strlen("else")) ||
-          !strncmp(startBuf, "elif", strlen("elif")) ||
-          !strncmp(startBuf, "endif", strlen("endif")) ||
-          !strncmp(startBuf, "pragma", strlen("pragma")) ||
-          !strncmp(startBuf, "include", strlen("include")) ||
-          !strncmp(startBuf, "import", strlen("import")) ||
-          !strncmp(startBuf, "include_next", strlen("include_next")))
-        return true;
-    }
-    startBuf++;
-  }
-  return false;
-}
-
-/// RewriteObjCInternalStruct - Rewrite one internal struct corresponding to
-/// an objective-c class with ivars.
-void RewriteObjC::RewriteObjCInternalStruct(ObjCInterfaceDecl *CDecl,
-                                               std::string &Result) {
-  assert(CDecl && "Class missing in SynthesizeObjCInternalStruct");
-  assert(CDecl->getName() != "" &&
-         "Name missing in SynthesizeObjCInternalStruct");
-  // Do not synthesize more than once.
-  if (ObjCSynthesizedStructs.count(CDecl))
-    return;
-  ObjCInterfaceDecl *RCDecl = CDecl->getSuperClass();
-  int NumIvars = CDecl->ivar_size();
-  SourceLocation LocStart = CDecl->getLocStart();
-  SourceLocation LocEnd = CDecl->getEndOfDefinitionLoc();
-
-  const char *startBuf = SM->getCharacterData(LocStart);
-  const char *endBuf = SM->getCharacterData(LocEnd);
-
-  // If no ivars and no root or if its root, directly or indirectly,
-  // have no ivars (thus not synthesized) then no need to synthesize this class.
-  if ((!CDecl->isThisDeclarationADefinition() || NumIvars == 0) &&
-      (!RCDecl || !ObjCSynthesizedStructs.count(RCDecl))) {
-    endBuf += Lexer::MeasureTokenLength(LocEnd, *SM, LangOpts);
-    ReplaceText(LocStart, endBuf-startBuf, Result);
-    return;
-  }
-
-  // FIXME: This has potential of causing problem. If
-  // SynthesizeObjCInternalStruct is ever called recursively.
-  Result += "\nstruct ";
-  Result += CDecl->getNameAsString();
-  if (LangOpts.MicrosoftExt)
-    Result += "_IMPL";
-
-  if (NumIvars > 0) {
-    const char *cursor = strchr(startBuf, '{');
-    assert((cursor && endBuf)
-           && "SynthesizeObjCInternalStruct - malformed @interface");
-    // If the buffer contains preprocessor directives, we do more fine-grained
-    // rewrites. This is intended to fix code that looks like (which occurs in
-    // NSURL.h, for example):
-    //
-    // #ifdef XYZ
-    // @interface Foo : NSObject
-    // #else
-    // @interface FooBar : NSObject
-    // #endif
-    // {
-    //    int i;
-    // }
-    // @end
-    //
-    // This clause is segregated to avoid breaking the common case.
-    if (BufferContainsPPDirectives(startBuf, cursor)) {
-      SourceLocation L = RCDecl ? CDecl->getSuperClassLoc() :
-                                  CDecl->getAtStartLoc();
-      const char *endHeader = SM->getCharacterData(L);
-      endHeader += Lexer::MeasureTokenLength(L, *SM, LangOpts);
-
-      if (CDecl->protocol_begin() != CDecl->protocol_end()) {
-        // advance to the end of the referenced protocols.
-        while (endHeader < cursor && *endHeader != '>') endHeader++;
-        endHeader++;
-      }
-      // rewrite the original header
-      ReplaceText(LocStart, endHeader-startBuf, Result);
-    } else {
-      // rewrite the original header *without* disturbing the '{'
-      ReplaceText(LocStart, cursor-startBuf, Result);
-    }
-    if (RCDecl && ObjCSynthesizedStructs.count(RCDecl)) {
-      Result = "\n    struct ";
-      Result += RCDecl->getNameAsString();
-      Result += "_IMPL ";
-      Result += RCDecl->getNameAsString();
-      Result += "_IVARS;\n";
-
-      // insert the super class structure definition.
-      SourceLocation OnePastCurly =
-        LocStart.getLocWithOffset(cursor-startBuf+1);
-      InsertText(OnePastCurly, Result);
-    }
-    cursor++; // past '{'
-
-    // Now comment out any visibility specifiers.
-    while (cursor < endBuf) {
-      if (*cursor == '@') {
-        SourceLocation atLoc = LocStart.getLocWithOffset(cursor-startBuf);
-        // Skip whitespace.
-        for (++cursor; cursor[0] == ' ' || cursor[0] == '\t'; ++cursor)
-          /*scan*/;
-
-        // FIXME: presence of @public, etc. inside comment results in
-        // this transformation as well, which is still correct c-code.
-        if (!strncmp(cursor, "public", strlen("public")) ||
-            !strncmp(cursor, "private", strlen("private")) ||
-            !strncmp(cursor, "package", strlen("package")) ||
-            !strncmp(cursor, "protected", strlen("protected")))
-          InsertText(atLoc, "// ");
-      }
-      // FIXME: If there are cases where '<' is used in ivar declaration part
-      // of user code, then scan the ivar list and use needToScanForQualifiers
-      // for type checking.
-      else if (*cursor == '<') {
-        SourceLocation atLoc = LocStart.getLocWithOffset(cursor-startBuf);
-        InsertText(atLoc, "/* ");
-        cursor = strchr(cursor, '>');
-        cursor++;
-        atLoc = LocStart.getLocWithOffset(cursor-startBuf);
-        InsertText(atLoc, " */");
-      } else if (*cursor == '^') { // rewrite block specifier.
-        SourceLocation caretLoc = LocStart.getLocWithOffset(cursor-startBuf);
-        ReplaceText(caretLoc, 1, "*");
-      }
-      cursor++;
-    }
-    // Don't forget to add a ';'!!
-    InsertText(LocEnd.getLocWithOffset(1), ";");
-  } else { // we don't have any instance variables - insert super struct.
-    endBuf += Lexer::MeasureTokenLength(LocEnd, *SM, LangOpts);
-    Result += " {\n    struct ";
-    Result += RCDecl->getNameAsString();
-    Result += "_IMPL ";
-    Result += RCDecl->getNameAsString();
-    Result += "_IVARS;\n};\n";
-    ReplaceText(LocStart, endBuf-startBuf, Result);
-  }
-  // Mark this struct as having been generated.
-  if (!ObjCSynthesizedStructs.insert(CDecl))
-    llvm_unreachable("struct already synthesize- SynthesizeObjCInternalStruct");
-}
-
-//===----------------------------------------------------------------------===//
-// Meta Data Emission
-//===----------------------------------------------------------------------===//
-
-
-/// RewriteImplementations - This routine rewrites all method implementations
-/// and emits meta-data.
-
-void RewriteObjC::RewriteImplementations() {
-  int ClsDefCount = ClassImplementation.size();
-  int CatDefCount = CategoryImplementation.size();
-
-  // Rewrite implemented methods
-  for (int i = 0; i < ClsDefCount; i++)
-    RewriteImplementationDecl(ClassImplementation[i]);
-
-  for (int i = 0; i < CatDefCount; i++)
-    RewriteImplementationDecl(CategoryImplementation[i]);
-}
-
-void RewriteObjC::RewriteByRefString(std::string &ResultStr, 
-                                     const std::string &Name,
-                                     ValueDecl *VD, bool def) {
-  assert(BlockByRefDeclNo.count(VD) && 
-         "RewriteByRefString: ByRef decl missing");
-  if (def)
-    ResultStr += "struct ";
-  ResultStr += "__Block_byref_" + Name + 
-    "_" + utostr(BlockByRefDeclNo[VD]) ;
-}
-
-static bool HasLocalVariableExternalStorage(ValueDecl *VD) {
-  if (VarDecl *Var = dyn_cast<VarDecl>(VD))
-    return (Var->isFunctionOrMethodVarDecl() && !Var->hasLocalStorage());
-  return false;
-}
-
-std::string RewriteObjC::SynthesizeBlockFunc(BlockExpr *CE, int i,
-                                                   StringRef funcName,
-                                                   std::string Tag) {
-  const FunctionType *AFT = CE->getFunctionType();
-  QualType RT = AFT->getResultType();
-  std::string StructRef = "struct " + Tag;
-  std::string S = "static " + RT.getAsString(Context->getPrintingPolicy()) + " __" +
-                  funcName.str() + "_" + "block_func_" + utostr(i);
-
-  BlockDecl *BD = CE->getBlockDecl();
-
-  if (isa<FunctionNoProtoType>(AFT)) {
-    // No user-supplied arguments. Still need to pass in a pointer to the
-    // block (to reference imported block decl refs).
-    S += "(" + StructRef + " *__cself)";
-  } else if (BD->param_empty()) {
-    S += "(" + StructRef + " *__cself)";
-  } else {
-    const FunctionProtoType *FT = cast<FunctionProtoType>(AFT);
-    assert(FT && "SynthesizeBlockFunc: No function proto");
-    S += '(';
-    // first add the implicit argument.
-    S += StructRef + " *__cself, ";
-    std::string ParamStr;
-    for (BlockDecl::param_iterator AI = BD->param_begin(),
-         E = BD->param_end(); AI != E; ++AI) {
-      if (AI != BD->param_begin()) S += ", ";
-      ParamStr = (*AI)->getNameAsString();
-      QualType QT = (*AI)->getType();
-      (void)convertBlockPointerToFunctionPointer(QT);
-      QT.getAsStringInternal(ParamStr, Context->getPrintingPolicy());
-      S += ParamStr;
-    }
-    if (FT->isVariadic()) {
-      if (!BD->param_empty()) S += ", ";
-      S += "...";
-    }
-    S += ')';
-  }
-  S += " {\n";
-
-  // Create local declarations to avoid rewriting all closure decl ref exprs.
-  // First, emit a declaration for all "by ref" decls.
-  for (SmallVector<ValueDecl*,8>::iterator I = BlockByRefDecls.begin(),
-       E = BlockByRefDecls.end(); I != E; ++I) {
-    S += "  ";
-    std::string Name = (*I)->getNameAsString();
-    std::string TypeString;
-    RewriteByRefString(TypeString, Name, (*I));
-    TypeString += " *";
-    Name = TypeString + Name;
-    S += Name + " = __cself->" + (*I)->getNameAsString() + "; // bound by ref\n";
-  }
-  // Next, emit a declaration for all "by copy" declarations.
-  for (SmallVector<ValueDecl*,8>::iterator I = BlockByCopyDecls.begin(),
-       E = BlockByCopyDecls.end(); I != E; ++I) {
-    S += "  ";
-    // Handle nested closure invocation. For example:
-    //
-    //   void (^myImportedClosure)(void);
-    //   myImportedClosure  = ^(void) { setGlobalInt(x + y); };
-    //
-    //   void (^anotherClosure)(void);
-    //   anotherClosure = ^(void) {
-    //     myImportedClosure(); // import and invoke the closure
-    //   };
-    //
-    if (isTopLevelBlockPointerType((*I)->getType())) {
-      RewriteBlockPointerTypeVariable(S, (*I));
-      S += " = (";
-      RewriteBlockPointerType(S, (*I)->getType());
-      S += ")";
-      S += "__cself->" + (*I)->getNameAsString() + "; // bound by copy\n";
-    }
-    else {
-      std::string Name = (*I)->getNameAsString();
-      QualType QT = (*I)->getType();
-      if (HasLocalVariableExternalStorage(*I))
-        QT = Context->getPointerType(QT);
-      QT.getAsStringInternal(Name, Context->getPrintingPolicy());
-      S += Name + " = __cself->" + 
-                              (*I)->getNameAsString() + "; // bound by copy\n";
-    }
-  }
-  std::string RewrittenStr = RewrittenBlockExprs[CE];
-  const char *cstr = RewrittenStr.c_str();
-  while (*cstr++ != '{') ;
-  S += cstr;
-  S += "\n";
-  return S;
-}
-
-std::string RewriteObjC::SynthesizeBlockHelperFuncs(BlockExpr *CE, int i,
-                                                   StringRef funcName,
-                                                   std::string Tag) {
-  std::string StructRef = "struct " + Tag;
-  std::string S = "static void __";
-
-  S += funcName;
-  S += "_block_copy_" + utostr(i);
-  S += "(" + StructRef;
-  S += "*dst, " + StructRef;
-  S += "*src) {";
-  for (llvm::SmallPtrSet<ValueDecl*,8>::iterator I = ImportedBlockDecls.begin(),
-      E = ImportedBlockDecls.end(); I != E; ++I) {
-    ValueDecl *VD = (*I);
-    S += "_Block_object_assign((void*)&dst->";
-    S += (*I)->getNameAsString();
-    S += ", (void*)src->";
-    S += (*I)->getNameAsString();
-    if (BlockByRefDeclsPtrSet.count((*I)))
-      S += ", " + utostr(BLOCK_FIELD_IS_BYREF) + "/*BLOCK_FIELD_IS_BYREF*/);";
-    else if (VD->getType()->isBlockPointerType())
-      S += ", " + utostr(BLOCK_FIELD_IS_BLOCK) + "/*BLOCK_FIELD_IS_BLOCK*/);";
-    else
-      S += ", " + utostr(BLOCK_FIELD_IS_OBJECT) + "/*BLOCK_FIELD_IS_OBJECT*/);";
-  }
-  S += "}\n";
-  
-  S += "\nstatic void __";
-  S += funcName;
-  S += "_block_dispose_" + utostr(i);
-  S += "(" + StructRef;
-  S += "*src) {";
-  for (llvm::SmallPtrSet<ValueDecl*,8>::iterator I = ImportedBlockDecls.begin(),
-      E = ImportedBlockDecls.end(); I != E; ++I) {
-    ValueDecl *VD = (*I);
-    S += "_Block_object_dispose((void*)src->";
-    S += (*I)->getNameAsString();
-    if (BlockByRefDeclsPtrSet.count((*I)))
-      S += ", " + utostr(BLOCK_FIELD_IS_BYREF) + "/*BLOCK_FIELD_IS_BYREF*/);";
-    else if (VD->getType()->isBlockPointerType())
-      S += ", " + utostr(BLOCK_FIELD_IS_BLOCK) + "/*BLOCK_FIELD_IS_BLOCK*/);";
-    else
-      S += ", " + utostr(BLOCK_FIELD_IS_OBJECT) + "/*BLOCK_FIELD_IS_OBJECT*/);";
-  }
-  S += "}\n";
-  return S;
-}
-
-std::string RewriteObjC::SynthesizeBlockImpl(BlockExpr *CE, std::string Tag, 
-                                             std::string Desc) {
-  std::string S = "\nstruct " + Tag;
-  std::string Constructor = "  " + Tag;
-
-  S += " {\n  struct __block_impl impl;\n";
-  S += "  struct " + Desc;
-  S += "* Desc;\n";
-
-  Constructor += "(void *fp, "; // Invoke function pointer.
-  Constructor += "struct " + Desc; // Descriptor pointer.
-  Constructor += " *desc";
-
-  if (BlockDeclRefs.size()) {
-    // Output all "by copy" declarations.
-    for (SmallVector<ValueDecl*,8>::iterator I = BlockByCopyDecls.begin(),
-         E = BlockByCopyDecls.end(); I != E; ++I) {
-      S += "  ";
-      std::string FieldName = (*I)->getNameAsString();
-      std::string ArgName = "_" + FieldName;
-      // Handle nested closure invocation. For example:
-      //
-      //   void (^myImportedBlock)(void);
-      //   myImportedBlock  = ^(void) { setGlobalInt(x + y); };
-      //
-      //   void (^anotherBlock)(void);
-      //   anotherBlock = ^(void) {
-      //     myImportedBlock(); // import and invoke the closure
-      //   };
-      //
-      if (isTopLevelBlockPointerType((*I)->getType())) {
-        S += "struct __block_impl *";
-        Constructor += ", void *" + ArgName;
-      } else {
-        QualType QT = (*I)->getType();
-        if (HasLocalVariableExternalStorage(*I))
-          QT = Context->getPointerType(QT);
-        QT.getAsStringInternal(FieldName, Context->getPrintingPolicy());
-        QT.getAsStringInternal(ArgName, Context->getPrintingPolicy());
-        Constructor += ", " + ArgName;
-      }
-      S += FieldName + ";\n";
-    }
-    // Output all "by ref" declarations.
-    for (SmallVector<ValueDecl*,8>::iterator I = BlockByRefDecls.begin(),
-         E = BlockByRefDecls.end(); I != E; ++I) {
-      S += "  ";
-      std::string FieldName = (*I)->getNameAsString();
-      std::string ArgName = "_" + FieldName;
-      {
-        std::string TypeString;
-        RewriteByRefString(TypeString, FieldName, (*I));
-        TypeString += " *";
-        FieldName = TypeString + FieldName;
-        ArgName = TypeString + ArgName;
-        Constructor += ", " + ArgName;
-      }
-      S += FieldName + "; // by ref\n";
-    }
-    // Finish writing the constructor.
-    Constructor += ", int flags=0)";
-    // Initialize all "by copy" arguments.
-    bool firsTime = true;
-    for (SmallVector<ValueDecl*,8>::iterator I = BlockByCopyDecls.begin(),
-         E = BlockByCopyDecls.end(); I != E; ++I) {
-      std::string Name = (*I)->getNameAsString();
-        if (firsTime) {
-          Constructor += " : ";
-          firsTime = false;
-        }
-        else
-          Constructor += ", ";
-        if (isTopLevelBlockPointerType((*I)->getType()))
-          Constructor += Name + "((struct __block_impl *)_" + Name + ")";
-        else
-          Constructor += Name + "(_" + Name + ")";
-    }
-    // Initialize all "by ref" arguments.
-    for (SmallVector<ValueDecl*,8>::iterator I = BlockByRefDecls.begin(),
-         E = BlockByRefDecls.end(); I != E; ++I) {
-      std::string Name = (*I)->getNameAsString();
-      if (firsTime) {
-        Constructor += " : ";
-        firsTime = false;
-      }
-      else
-        Constructor += ", ";
-      Constructor += Name + "(_" + Name + "->__forwarding)";
-    }
-    
-    Constructor += " {\n";
-    if (GlobalVarDecl)
-      Constructor += "    impl.isa = &_NSConcreteGlobalBlock;\n";
-    else
-      Constructor += "    impl.isa = &_NSConcreteStackBlock;\n";
-    Constructor += "    impl.Flags = flags;\n    impl.FuncPtr = fp;\n";
-
-    Constructor += "    Desc = desc;\n";
-  } else {
-    // Finish writing the constructor.
-    Constructor += ", int flags=0) {\n";
-    if (GlobalVarDecl)
-      Constructor += "    impl.isa = &_NSConcreteGlobalBlock;\n";
-    else
-      Constructor += "    impl.isa = &_NSConcreteStackBlock;\n";
-    Constructor += "    impl.Flags = flags;\n    impl.FuncPtr = fp;\n";
-    Constructor += "    Desc = desc;\n";
-  }
-  Constructor += "  ";
-  Constructor += "}\n";
-  S += Constructor;
-  S += "};\n";
-  return S;
-}
-
-std::string RewriteObjC::SynthesizeBlockDescriptor(std::string DescTag, 
-                                                   std::string ImplTag, int i,
-                                                   StringRef FunName,
-                                                   unsigned hasCopy) {
-  std::string S = "\nstatic struct " + DescTag;
-  
-  S += " {\n  unsigned long reserved;\n";
-  S += "  unsigned long Block_size;\n";
-  if (hasCopy) {
-    S += "  void (*copy)(struct ";
-    S += ImplTag; S += "*, struct ";
-    S += ImplTag; S += "*);\n";
-    
-    S += "  void (*dispose)(struct ";
-    S += ImplTag; S += "*);\n";
-  }
-  S += "} ";
-
-  S += DescTag + "_DATA = { 0, sizeof(struct ";
-  S += ImplTag + ")";
-  if (hasCopy) {
-    S += ", __" + FunName.str() + "_block_copy_" + utostr(i);
-    S += ", __" + FunName.str() + "_block_dispose_" + utostr(i);
-  }
-  S += "};\n";
-  return S;
-}
-
-void RewriteObjC::SynthesizeBlockLiterals(SourceLocation FunLocStart,
-                                          StringRef FunName) {
-  // Insert declaration for the function in which block literal is used.
-  if (CurFunctionDeclToDeclareForBlock && !Blocks.empty())
-    RewriteBlockLiteralFunctionDecl(CurFunctionDeclToDeclareForBlock);
-  bool RewriteSC = (GlobalVarDecl &&
-                    !Blocks.empty() &&
-                    GlobalVarDecl->getStorageClass() == SC_Static &&
-                    GlobalVarDecl->getType().getCVRQualifiers());
-  if (RewriteSC) {
-    std::string SC(" void __");
-    SC += GlobalVarDecl->getNameAsString();
-    SC += "() {}";
-    InsertText(FunLocStart, SC);
-  }
-  
-  // Insert closures that were part of the function.
-  for (unsigned i = 0, count=0; i < Blocks.size(); i++) {
-    CollectBlockDeclRefInfo(Blocks[i]);
-    // Need to copy-in the inner copied-in variables not actually used in this
-    // block.
-    for (int j = 0; j < InnerDeclRefsCount[i]; j++) {
-      DeclRefExpr *Exp = InnerDeclRefs[count++];
-      ValueDecl *VD = Exp->getDecl();
-      BlockDeclRefs.push_back(Exp);
-      if (!VD->hasAttr<BlocksAttr>() && !BlockByCopyDeclsPtrSet.count(VD)) {
-        BlockByCopyDeclsPtrSet.insert(VD);
-        BlockByCopyDecls.push_back(VD);
-      }
-      if (VD->hasAttr<BlocksAttr>() && !BlockByRefDeclsPtrSet.count(VD)) {
-        BlockByRefDeclsPtrSet.insert(VD);
-        BlockByRefDecls.push_back(VD);
-      }
-      // imported objects in the inner blocks not used in the outer
-      // blocks must be copied/disposed in the outer block as well.
-      if (VD->hasAttr<BlocksAttr>() ||
-          VD->getType()->isObjCObjectPointerType() || 
-          VD->getType()->isBlockPointerType())
-        ImportedBlockDecls.insert(VD);
-    }
-
-    std::string ImplTag = "__" + FunName.str() + "_block_impl_" + utostr(i);
-    std::string DescTag = "__" + FunName.str() + "_block_desc_" + utostr(i);
-
-    std::string CI = SynthesizeBlockImpl(Blocks[i], ImplTag, DescTag);
-
-    InsertText(FunLocStart, CI);
-
-    std::string CF = SynthesizeBlockFunc(Blocks[i], i, FunName, ImplTag);
-
-    InsertText(FunLocStart, CF);
-
-    if (ImportedBlockDecls.size()) {
-      std::string HF = SynthesizeBlockHelperFuncs(Blocks[i], i, FunName, ImplTag);
-      InsertText(FunLocStart, HF);
-    }
-    std::string BD = SynthesizeBlockDescriptor(DescTag, ImplTag, i, FunName,
-                                               ImportedBlockDecls.size() > 0);
-    InsertText(FunLocStart, BD);
-
-    BlockDeclRefs.clear();
-    BlockByRefDecls.clear();
-    BlockByRefDeclsPtrSet.clear();
-    BlockByCopyDecls.clear();
-    BlockByCopyDeclsPtrSet.clear();
-    ImportedBlockDecls.clear();
-  }
-  if (RewriteSC) {
-    // Must insert any 'const/volatile/static here. Since it has been
-    // removed as result of rewriting of block literals.
-    std::string SC;
-    if (GlobalVarDecl->getStorageClass() == SC_Static)
-      SC = "static ";
-    if (GlobalVarDecl->getType().isConstQualified())
-      SC += "const ";
-    if (GlobalVarDecl->getType().isVolatileQualified())
-      SC += "volatile ";
-    if (GlobalVarDecl->getType().isRestrictQualified())
-      SC += "restrict ";
-    InsertText(FunLocStart, SC);
-  }
-  
-  Blocks.clear();
-  InnerDeclRefsCount.clear();
-  InnerDeclRefs.clear();
-  RewrittenBlockExprs.clear();
-}
-
-void RewriteObjC::InsertBlockLiteralsWithinFunction(FunctionDecl *FD) {
-  SourceLocation FunLocStart = FD->getTypeSpecStartLoc();
-  StringRef FuncName = FD->getName();
-
-  SynthesizeBlockLiterals(FunLocStart, FuncName);
-}
-
-static void BuildUniqueMethodName(std::string &Name,
-                                  ObjCMethodDecl *MD) {
-  ObjCInterfaceDecl *IFace = MD->getClassInterface();
-  Name = IFace->getName();
-  Name += "__" + MD->getSelector().getAsString();
-  // Convert colons to underscores.
-  std::string::size_type loc = 0;
-  while ((loc = Name.find(":", loc)) != std::string::npos)
-    Name.replace(loc, 1, "_");
-}
-
-void RewriteObjC::InsertBlockLiteralsWithinMethod(ObjCMethodDecl *MD) {
-  //fprintf(stderr,"In InsertBlockLiteralsWitinMethod\n");
-  //SourceLocation FunLocStart = MD->getLocStart();
-  SourceLocation FunLocStart = MD->getLocStart();
-  std::string FuncName;
-  BuildUniqueMethodName(FuncName, MD);
-  SynthesizeBlockLiterals(FunLocStart, FuncName);
-}
-
-void RewriteObjC::GetBlockDeclRefExprs(Stmt *S) {
-  for (Stmt::child_range CI = S->children(); CI; ++CI)
-    if (*CI) {
-      if (BlockExpr *CBE = dyn_cast<BlockExpr>(*CI))
-        GetBlockDeclRefExprs(CBE->getBody());
-      else
-        GetBlockDeclRefExprs(*CI);
-    }
-  // Handle specific things.
-  if (DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(S)) {
-    if (DRE->refersToEnclosingLocal()) {
-      // FIXME: Handle enums.
-      if (!isa<FunctionDecl>(DRE->getDecl()))
-        BlockDeclRefs.push_back(DRE);
-      if (HasLocalVariableExternalStorage(DRE->getDecl()))
-        BlockDeclRefs.push_back(DRE);
-    }
-  }
-  
-  return;
-}
-
-void RewriteObjC::GetInnerBlockDeclRefExprs(Stmt *S, 
-                SmallVector<DeclRefExpr *, 8> &InnerBlockDeclRefs,
-                llvm::SmallPtrSet<const DeclContext *, 8> &InnerContexts) {
-  for (Stmt::child_range CI = S->children(); CI; ++CI)
-    if (*CI) {
-      if (BlockExpr *CBE = dyn_cast<BlockExpr>(*CI)) {
-        InnerContexts.insert(cast<DeclContext>(CBE->getBlockDecl()));
-        GetInnerBlockDeclRefExprs(CBE->getBody(),
-                                  InnerBlockDeclRefs,
-                                  InnerContexts);
-      }
-      else
-        GetInnerBlockDeclRefExprs(*CI,
-                                  InnerBlockDeclRefs,
-                                  InnerContexts);
-
-    }
-  // Handle specific things.
-  if (DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(S)) {
-    if (DRE->refersToEnclosingLocal()) {
-      if (!isa<FunctionDecl>(DRE->getDecl()) &&
-          !InnerContexts.count(DRE->getDecl()->getDeclContext()))
-        InnerBlockDeclRefs.push_back(DRE);
-      if (VarDecl *Var = dyn_cast<VarDecl>(DRE->getDecl()))
-        if (Var->isFunctionOrMethodVarDecl())
-          ImportedLocalExternalDecls.insert(Var);
-    }
-  }
-  
-  return;
-}
-
-/// convertFunctionTypeOfBlocks - This routine converts a function type
-/// whose result type may be a block pointer or whose argument type(s)
-/// might be block pointers to an equivalent function type replacing
-/// all block pointers to function pointers.
-QualType RewriteObjC::convertFunctionTypeOfBlocks(const FunctionType *FT) {
-  const FunctionProtoType *FTP = dyn_cast<FunctionProtoType>(FT);
-  // FTP will be null for closures that don't take arguments.
-  // Generate a funky cast.
-  SmallVector<QualType, 8> ArgTypes;
-  QualType Res = FT->getResultType();
-  bool HasBlockType = convertBlockPointerToFunctionPointer(Res);
-  
-  if (FTP) {
-    for (FunctionProtoType::arg_type_iterator I = FTP->arg_type_begin(),
-         E = FTP->arg_type_end(); I && (I != E); ++I) {
-      QualType t = *I;
-      // Make sure we convert "t (^)(...)" to "t (*)(...)".
-      if (convertBlockPointerToFunctionPointer(t))
-        HasBlockType = true;
-      ArgTypes.push_back(t);
-    }
-  }
-  QualType FuncType;
-  // FIXME. Does this work if block takes no argument but has a return type
-  // which is of block type?
-  if (HasBlockType)
-    FuncType = getSimpleFunctionType(Res, &ArgTypes[0], ArgTypes.size());
-  else FuncType = QualType(FT, 0);
-  return FuncType;
-}
-
-Stmt *RewriteObjC::SynthesizeBlockCall(CallExpr *Exp, const Expr *BlockExp) {
-  // Navigate to relevant type information.
-  const BlockPointerType *CPT = 0;
-
-  if (const DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(BlockExp)) {
-    CPT = DRE->getType()->getAs<BlockPointerType>();
-  } else if (const MemberExpr *MExpr = dyn_cast<MemberExpr>(BlockExp)) {
-    CPT = MExpr->getType()->getAs<BlockPointerType>();
-  } 
-  else if (const ParenExpr *PRE = dyn_cast<ParenExpr>(BlockExp)) {
-    return SynthesizeBlockCall(Exp, PRE->getSubExpr());
-  }
-  else if (const ImplicitCastExpr *IEXPR = dyn_cast<ImplicitCastExpr>(BlockExp)) 
-    CPT = IEXPR->getType()->getAs<BlockPointerType>();
-  else if (const ConditionalOperator *CEXPR = 
-            dyn_cast<ConditionalOperator>(BlockExp)) {
-    Expr *LHSExp = CEXPR->getLHS();
-    Stmt *LHSStmt = SynthesizeBlockCall(Exp, LHSExp);
-    Expr *RHSExp = CEXPR->getRHS();
-    Stmt *RHSStmt = SynthesizeBlockCall(Exp, RHSExp);
-    Expr *CONDExp = CEXPR->getCond();
-    ConditionalOperator *CondExpr =
-      new (Context) ConditionalOperator(CONDExp,
-                                      SourceLocation(), cast<Expr>(LHSStmt),
-                                      SourceLocation(), cast<Expr>(RHSStmt),
-                                      Exp->getType(), VK_RValue, OK_Ordinary);
-    return CondExpr;
-  } else if (const ObjCIvarRefExpr *IRE = dyn_cast<ObjCIvarRefExpr>(BlockExp)) {
-    CPT = IRE->getType()->getAs<BlockPointerType>();
-  } else if (const PseudoObjectExpr *POE
-               = dyn_cast<PseudoObjectExpr>(BlockExp)) {
-    CPT = POE->getType()->castAs<BlockPointerType>();
-  } else {
-    assert(1 && "RewriteBlockClass: Bad type");
-  }
-  assert(CPT && "RewriteBlockClass: Bad type");
-  const FunctionType *FT = CPT->getPointeeType()->getAs<FunctionType>();
-  assert(FT && "RewriteBlockClass: Bad type");
-  const FunctionProtoType *FTP = dyn_cast<FunctionProtoType>(FT);
-  // FTP will be null for closures that don't take arguments.
-
-  RecordDecl *RD = RecordDecl::Create(*Context, TTK_Struct, TUDecl,
-                                      SourceLocation(), SourceLocation(),
-                                      &Context->Idents.get("__block_impl"));
-  QualType PtrBlock = Context->getPointerType(Context->getTagDeclType(RD));
-
-  // Generate a funky cast.
-  SmallVector<QualType, 8> ArgTypes;
-
-  // Push the block argument type.
-  ArgTypes.push_back(PtrBlock);
-  if (FTP) {
-    for (FunctionProtoType::arg_type_iterator I = FTP->arg_type_begin(),
-         E = FTP->arg_type_end(); I && (I != E); ++I) {
-      QualType t = *I;
-      // Make sure we convert "t (^)(...)" to "t (*)(...)".
-      if (!convertBlockPointerToFunctionPointer(t))
-        convertToUnqualifiedObjCType(t);
-      ArgTypes.push_back(t);
-    }
-  }
-  // Now do the pointer to function cast.
-  QualType PtrToFuncCastType
-    = getSimpleFunctionType(Exp->getType(), &ArgTypes[0], ArgTypes.size());
-
-  PtrToFuncCastType = Context->getPointerType(PtrToFuncCastType);
-
-  CastExpr *BlkCast = NoTypeInfoCStyleCastExpr(Context, PtrBlock,
-                                               CK_BitCast,
-                                               const_cast<Expr*>(BlockExp));
-  // Don't forget the parens to enforce the proper binding.
-  ParenExpr *PE = new (Context) ParenExpr(SourceLocation(), SourceLocation(),
-                                          BlkCast);
-  //PE->dump();
-
-  FieldDecl *FD = FieldDecl::Create(*Context, 0, SourceLocation(),
-                                    SourceLocation(),
-                                    &Context->Idents.get("FuncPtr"),
-                                    Context->VoidPtrTy, 0,
-                                    /*BitWidth=*/0, /*Mutable=*/true,
-                                    ICIS_NoInit);
-  MemberExpr *ME = new (Context) MemberExpr(PE, true, FD, SourceLocation(),
-                                            FD->getType(), VK_LValue,
-                                            OK_Ordinary);
-
-  
-  CastExpr *FunkCast = NoTypeInfoCStyleCastExpr(Context, PtrToFuncCastType,
-                                                CK_BitCast, ME);
-  PE = new (Context) ParenExpr(SourceLocation(), SourceLocation(), FunkCast);
-
-  SmallVector<Expr*, 8> BlkExprs;
-  // Add the implicit argument.
-  BlkExprs.push_back(BlkCast);
-  // Add the user arguments.
-  for (CallExpr::arg_iterator I = Exp->arg_begin(),
-       E = Exp->arg_end(); I != E; ++I) {
-    BlkExprs.push_back(*I);
-  }
-  CallExpr *CE = new (Context) CallExpr(*Context, PE, &BlkExprs[0],
-                                        BlkExprs.size(),
-                                        Exp->getType(), VK_RValue,
-                                        SourceLocation());
-  return CE;
-}
-
-// We need to return the rewritten expression to handle cases where the
-// BlockDeclRefExpr is embedded in another expression being rewritten.
-// For example:
-//
-// int main() {
-//    __block Foo *f;
-//    __block int i;
-//
-//    void (^myblock)() = ^() {
-//        [f test]; // f is a BlockDeclRefExpr embedded in a message (which is being rewritten).
-//        i = 77;
-//    };
-//}
-Stmt *RewriteObjC::RewriteBlockDeclRefExpr(DeclRefExpr *DeclRefExp) {
-  // Rewrite the byref variable into BYREFVAR->__forwarding->BYREFVAR 
-  // for each DeclRefExp where BYREFVAR is name of the variable.
-  ValueDecl *VD = DeclRefExp->getDecl();
-  bool isArrow = DeclRefExp->refersToEnclosingLocal();
-  
-  FieldDecl *FD = FieldDecl::Create(*Context, 0, SourceLocation(),
-                                    SourceLocation(),
-                                    &Context->Idents.get("__forwarding"), 
-                                    Context->VoidPtrTy, 0,
-                                    /*BitWidth=*/0, /*Mutable=*/true,
-                                    ICIS_NoInit);
-  MemberExpr *ME = new (Context) MemberExpr(DeclRefExp, isArrow,
-                                            FD, SourceLocation(),
-                                            FD->getType(), VK_LValue,
-                                            OK_Ordinary);
-
-  StringRef Name = VD->getName();
-  FD = FieldDecl::Create(*Context, 0, SourceLocation(), SourceLocation(),
-                         &Context->Idents.get(Name), 
-                         Context->VoidPtrTy, 0,
-                         /*BitWidth=*/0, /*Mutable=*/true,
-                         ICIS_NoInit);
-  ME = new (Context) MemberExpr(ME, true, FD, SourceLocation(),
-                                DeclRefExp->getType(), VK_LValue, OK_Ordinary);
-  
-  
-  
-  // Need parens to enforce precedence.
-  ParenExpr *PE = new (Context) ParenExpr(DeclRefExp->getExprLoc(), 
-                                          DeclRefExp->getExprLoc(), 
-                                          ME);
-  ReplaceStmt(DeclRefExp, PE);
-  return PE;
-}
-
-// Rewrites the imported local variable V with external storage 
-// (static, extern, etc.) as *V
-//
-Stmt *RewriteObjC::RewriteLocalVariableExternalStorage(DeclRefExpr *DRE) {
-  ValueDecl *VD = DRE->getDecl();
-  if (VarDecl *Var = dyn_cast<VarDecl>(VD))
-    if (!ImportedLocalExternalDecls.count(Var))
-      return DRE;
-  Expr *Exp = new (Context) UnaryOperator(DRE, UO_Deref, DRE->getType(),
-                                          VK_LValue, OK_Ordinary,
-                                          DRE->getLocation());
-  // Need parens to enforce precedence.
-  ParenExpr *PE = new (Context) ParenExpr(SourceLocation(), SourceLocation(), 
-                                          Exp);
-  ReplaceStmt(DRE, PE);
-  return PE;
-}
-
-void RewriteObjC::RewriteCastExpr(CStyleCastExpr *CE) {
-  SourceLocation LocStart = CE->getLParenLoc();
-  SourceLocation LocEnd = CE->getRParenLoc();
-
-  // Need to avoid trying to rewrite synthesized casts.
-  if (LocStart.isInvalid())
-    return;
-  // Need to avoid trying to rewrite casts contained in macros.
-  if (!Rewriter::isRewritable(LocStart) || !Rewriter::isRewritable(LocEnd))
-    return;
-
-  const char *startBuf = SM->getCharacterData(LocStart);
-  const char *endBuf = SM->getCharacterData(LocEnd);
-  QualType QT = CE->getType();
-  const Type* TypePtr = QT->getAs<Type>();
-  if (isa<TypeOfExprType>(TypePtr)) {
-    const TypeOfExprType *TypeOfExprTypePtr = cast<TypeOfExprType>(TypePtr);
-    QT = TypeOfExprTypePtr->getUnderlyingExpr()->getType();
-    std::string TypeAsString = "(";
-    RewriteBlockPointerType(TypeAsString, QT);
-    TypeAsString += ")";
-    ReplaceText(LocStart, endBuf-startBuf+1, TypeAsString);
-    return;
-  }
-  // advance the location to startArgList.
-  const char *argPtr = startBuf;
-
-  while (*argPtr++ && (argPtr < endBuf)) {
-    switch (*argPtr) {
-    case '^':
-      // Replace the '^' with '*'.
-      LocStart = LocStart.getLocWithOffset(argPtr-startBuf);
-      ReplaceText(LocStart, 1, "*");
-      break;
-    }
-  }
-  return;
-}
-
-void RewriteObjC::RewriteBlockPointerFunctionArgs(FunctionDecl *FD) {
-  SourceLocation DeclLoc = FD->getLocation();
-  unsigned parenCount = 0;
-
-  // We have 1 or more arguments that have closure pointers.
-  const char *startBuf = SM->getCharacterData(DeclLoc);
-  const char *startArgList = strchr(startBuf, '(');
-
-  assert((*startArgList == '(') && "Rewriter fuzzy parser confused");
-
-  parenCount++;
-  // advance the location to startArgList.
-  DeclLoc = DeclLoc.getLocWithOffset(startArgList-startBuf);
-  assert((DeclLoc.isValid()) && "Invalid DeclLoc");
-
-  const char *argPtr = startArgList;
-
-  while (*argPtr++ && parenCount) {
-    switch (*argPtr) {
-    case '^':
-      // Replace the '^' with '*'.
-      DeclLoc = DeclLoc.getLocWithOffset(argPtr-startArgList);
-      ReplaceText(DeclLoc, 1, "*");
-      break;
-    case '(':
-      parenCount++;
-      break;
-    case ')':
-      parenCount--;
-      break;
-    }
-  }
-  return;
-}
-
-bool RewriteObjC::PointerTypeTakesAnyBlockArguments(QualType QT) {
-  const FunctionProtoType *FTP;
-  const PointerType *PT = QT->getAs<PointerType>();
-  if (PT) {
-    FTP = PT->getPointeeType()->getAs<FunctionProtoType>();
-  } else {
-    const BlockPointerType *BPT = QT->getAs<BlockPointerType>();
-    assert(BPT && "BlockPointerTypeTakeAnyBlockArguments(): not a block pointer type");
-    FTP = BPT->getPointeeType()->getAs<FunctionProtoType>();
-  }
-  if (FTP) {
-    for (FunctionProtoType::arg_type_iterator I = FTP->arg_type_begin(),
-         E = FTP->arg_type_end(); I != E; ++I)
-      if (isTopLevelBlockPointerType(*I))
-        return true;
-  }
-  return false;
-}
-
-bool RewriteObjC::PointerTypeTakesAnyObjCQualifiedType(QualType QT) {
-  const FunctionProtoType *FTP;
-  const PointerType *PT = QT->getAs<PointerType>();
-  if (PT) {
-    FTP = PT->getPointeeType()->getAs<FunctionProtoType>();
-  } else {
-    const BlockPointerType *BPT = QT->getAs<BlockPointerType>();
-    assert(BPT && "BlockPointerTypeTakeAnyBlockArguments(): not a block pointer type");
-    FTP = BPT->getPointeeType()->getAs<FunctionProtoType>();
-  }
-  if (FTP) {
-    for (FunctionProtoType::arg_type_iterator I = FTP->arg_type_begin(),
-         E = FTP->arg_type_end(); I != E; ++I) {
-      if ((*I)->isObjCQualifiedIdType())
-        return true;
-      if ((*I)->isObjCObjectPointerType() &&
-          (*I)->getPointeeType()->isObjCQualifiedInterfaceType())
-        return true;
-    }
-        
-  }
-  return false;
-}
-
-void RewriteObjC::GetExtentOfArgList(const char *Name, const char *&LParen,
-                                     const char *&RParen) {
-  const char *argPtr = strchr(Name, '(');
-  assert((*argPtr == '(') && "Rewriter fuzzy parser confused");
-
-  LParen = argPtr; // output the start.
-  argPtr++; // skip past the left paren.
-  unsigned parenCount = 1;
-
-  while (*argPtr && parenCount) {
-    switch (*argPtr) {
-    case '(': parenCount++; break;
-    case ')': parenCount--; break;
-    default: break;
-    }
-    if (parenCount) argPtr++;
-  }
-  assert((*argPtr == ')') && "Rewriter fuzzy parser confused");
-  RParen = argPtr; // output the end
-}
-
-void RewriteObjC::RewriteBlockPointerDecl(NamedDecl *ND) {
-  if (FunctionDecl *FD = dyn_cast<FunctionDecl>(ND)) {
-    RewriteBlockPointerFunctionArgs(FD);
-    return;
-  }
-  // Handle Variables and Typedefs.
-  SourceLocation DeclLoc = ND->getLocation();
-  QualType DeclT;
-  if (VarDecl *VD = dyn_cast<VarDecl>(ND))
-    DeclT = VD->getType();
-  else if (TypedefNameDecl *TDD = dyn_cast<TypedefNameDecl>(ND))
-    DeclT = TDD->getUnderlyingType();
-  else if (FieldDecl *FD = dyn_cast<FieldDecl>(ND))
-    DeclT = FD->getType();
-  else
-    llvm_unreachable("RewriteBlockPointerDecl(): Decl type not yet handled");
-
-  const char *startBuf = SM->getCharacterData(DeclLoc);
-  const char *endBuf = startBuf;
-  // scan backward (from the decl location) for the end of the previous decl.
-  while (*startBuf != '^' && *startBuf != ';' && startBuf != MainFileStart)
-    startBuf--;
-  SourceLocation Start = DeclLoc.getLocWithOffset(startBuf-endBuf);
-  std::string buf;
-  unsigned OrigLength=0;
-  // *startBuf != '^' if we are dealing with a pointer to function that
-  // may take block argument types (which will be handled below).
-  if (*startBuf == '^') {
-    // Replace the '^' with '*', computing a negative offset.
-    buf = '*';
-    startBuf++;
-    OrigLength++;
-  }
-  while (*startBuf != ')') {
-    buf += *startBuf;
-    startBuf++;
-    OrigLength++;
-  }
-  buf += ')';
-  OrigLength++;
-  
-  if (PointerTypeTakesAnyBlockArguments(DeclT) ||
-      PointerTypeTakesAnyObjCQualifiedType(DeclT)) {
-    // Replace the '^' with '*' for arguments.
-    // Replace id<P> with id/*<>*/
-    DeclLoc = ND->getLocation();
-    startBuf = SM->getCharacterData(DeclLoc);
-    const char *argListBegin, *argListEnd;
-    GetExtentOfArgList(startBuf, argListBegin, argListEnd);
-    while (argListBegin < argListEnd) {
-      if (*argListBegin == '^')
-        buf += '*';
-      else if (*argListBegin ==  '<') {
-        buf += "/*"; 
-        buf += *argListBegin++;
-        OrigLength++;;
-        while (*argListBegin != '>') {
-          buf += *argListBegin++;
-          OrigLength++;
-        }
-        buf += *argListBegin;
-        buf += "*/";
-      }
-      else
-        buf += *argListBegin;
-      argListBegin++;
-      OrigLength++;
-    }
-    buf += ')';
-    OrigLength++;
-  }
-  ReplaceText(Start, OrigLength, buf);
-  
-  return;
-}
-
-
-/// SynthesizeByrefCopyDestroyHelper - This routine synthesizes:
-/// void __Block_byref_id_object_copy(struct Block_byref_id_object *dst,
-///                    struct Block_byref_id_object *src) {
-///  _Block_object_assign (&_dest->object, _src->object, 
-///                        BLOCK_BYREF_CALLER | BLOCK_FIELD_IS_OBJECT
-///                        [|BLOCK_FIELD_IS_WEAK]) // object
-///  _Block_object_assign(&_dest->object, _src->object, 
-///                       BLOCK_BYREF_CALLER | BLOCK_FIELD_IS_BLOCK
-///                       [|BLOCK_FIELD_IS_WEAK]) // block
-/// }
-/// And:
-/// void __Block_byref_id_object_dispose(struct Block_byref_id_object *_src) {
-///  _Block_object_dispose(_src->object, 
-///                        BLOCK_BYREF_CALLER | BLOCK_FIELD_IS_OBJECT
-///                        [|BLOCK_FIELD_IS_WEAK]) // object
-///  _Block_object_dispose(_src->object, 
-///                         BLOCK_BYREF_CALLER | BLOCK_FIELD_IS_BLOCK
-///                         [|BLOCK_FIELD_IS_WEAK]) // block
-/// }
-
-std::string RewriteObjC::SynthesizeByrefCopyDestroyHelper(VarDecl *VD,
-                                                          int flag) {
-  std::string S;
-  if (CopyDestroyCache.count(flag))
-    return S;
-  CopyDestroyCache.insert(flag);
-  S = "static void __Block_byref_id_object_copy_";
-  S += utostr(flag);
-  S += "(void *dst, void *src) {\n";
-  
-  // offset into the object pointer is computed as:
-  // void * + void* + int + int + void* + void *
-  unsigned IntSize = 
-  static_cast<unsigned>(Context->getTypeSize(Context->IntTy));
-  unsigned VoidPtrSize = 
-  static_cast<unsigned>(Context->getTypeSize(Context->VoidPtrTy));
-  
-  unsigned offset = (VoidPtrSize*4 + IntSize + IntSize)/Context->getCharWidth();
-  S += " _Block_object_assign((char*)dst + ";
-  S += utostr(offset);
-  S += ", *(void * *) ((char*)src + ";
-  S += utostr(offset);
-  S += "), ";
-  S += utostr(flag);
-  S += ");\n}\n";
-  
-  S += "static void __Block_byref_id_object_dispose_";
-  S += utostr(flag);
-  S += "(void *src) {\n";
-  S += " _Block_object_dispose(*(void * *) ((char*)src + ";
-  S += utostr(offset);
-  S += "), ";
-  S += utostr(flag);
-  S += ");\n}\n";
-  return S;
-}
-
-/// RewriteByRefVar - For each __block typex ND variable this routine transforms
-/// the declaration into:
-/// struct __Block_byref_ND {
-/// void *__isa;                  // NULL for everything except __weak pointers
-/// struct __Block_byref_ND *__forwarding;
-/// int32_t __flags;
-/// int32_t __size;
-/// void *__Block_byref_id_object_copy; // If variable is __block ObjC object
-/// void *__Block_byref_id_object_dispose; // If variable is __block ObjC object
-/// typex ND;
-/// };
-///
-/// It then replaces declaration of ND variable with:
-/// struct __Block_byref_ND ND = {__isa=0B, __forwarding=&ND, __flags=some_flag, 
-///                               __size=sizeof(struct __Block_byref_ND), 
-///                               ND=initializer-if-any};
-///
-///
-void RewriteObjC::RewriteByRefVar(VarDecl *ND) {
-  // Insert declaration for the function in which block literal is
-  // used.
-  if (CurFunctionDeclToDeclareForBlock)
-    RewriteBlockLiteralFunctionDecl(CurFunctionDeclToDeclareForBlock);
-  int flag = 0;
-  int isa = 0;
-  SourceLocation DeclLoc = ND->getTypeSpecStartLoc();
-  if (DeclLoc.isInvalid())
-    // If type location is missing, it is because of missing type (a warning).
-    // Use variable's location which is good for this case.
-    DeclLoc = ND->getLocation();
-  const char *startBuf = SM->getCharacterData(DeclLoc);
-  SourceLocation X = ND->getLocEnd();
-  X = SM->getExpansionLoc(X);
-  const char *endBuf = SM->getCharacterData(X);
-  std::string Name(ND->getNameAsString());
-  std::string ByrefType;
-  RewriteByRefString(ByrefType, Name, ND, true);
-  ByrefType += " {\n";
-  ByrefType += "  void *__isa;\n";
-  RewriteByRefString(ByrefType, Name, ND);
-  ByrefType += " *__forwarding;\n";
-  ByrefType += " int __flags;\n";
-  ByrefType += " int __size;\n";
-  // Add void *__Block_byref_id_object_copy; 
-  // void *__Block_byref_id_object_dispose; if needed.
-  QualType Ty = ND->getType();
-  bool HasCopyAndDispose = Context->BlockRequiresCopying(Ty);
-  if (HasCopyAndDispose) {
-    ByrefType += " void (*__Block_byref_id_object_copy)(void*, void*);\n";
-    ByrefType += " void (*__Block_byref_id_object_dispose)(void*);\n";
-  }
-
-  QualType T = Ty;
-  (void)convertBlockPointerToFunctionPointer(T);
-  T.getAsStringInternal(Name, Context->getPrintingPolicy());
-    
-  ByrefType += " " + Name + ";\n";
-  ByrefType += "};\n";
-  // Insert this type in global scope. It is needed by helper function.
-  SourceLocation FunLocStart;
-  if (CurFunctionDef)
-     FunLocStart = CurFunctionDef->getTypeSpecStartLoc();
-  else {
-    assert(CurMethodDef && "RewriteByRefVar - CurMethodDef is null");
-    FunLocStart = CurMethodDef->getLocStart();
-  }
-  InsertText(FunLocStart, ByrefType);
-  if (Ty.isObjCGCWeak()) {
-    flag |= BLOCK_FIELD_IS_WEAK;
-    isa = 1;
-  }
-  
-  if (HasCopyAndDispose) {
-    flag = BLOCK_BYREF_CALLER;
-    QualType Ty = ND->getType();
-    // FIXME. Handle __weak variable (BLOCK_FIELD_IS_WEAK) as well.
-    if (Ty->isBlockPointerType())
-      flag |= BLOCK_FIELD_IS_BLOCK;
-    else
-      flag |= BLOCK_FIELD_IS_OBJECT;
-    std::string HF = SynthesizeByrefCopyDestroyHelper(ND, flag);
-    if (!HF.empty())
-      InsertText(FunLocStart, HF);
-  }
-  
-  // struct __Block_byref_ND ND = 
-  // {0, &ND, some_flag, __size=sizeof(struct __Block_byref_ND), 
-  //  initializer-if-any};
-  bool hasInit = (ND->getInit() != 0);
-  unsigned flags = 0;
-  if (HasCopyAndDispose)
-    flags |= BLOCK_HAS_COPY_DISPOSE;
-  Name = ND->getNameAsString();
-  ByrefType.clear();
-  RewriteByRefString(ByrefType, Name, ND);
-  std::string ForwardingCastType("(");
-  ForwardingCastType += ByrefType + " *)";
-  if (!hasInit) {
-    ByrefType += " " + Name + " = {(void*)";
-    ByrefType += utostr(isa);
-    ByrefType += "," +  ForwardingCastType + "&" + Name + ", ";
-    ByrefType += utostr(flags);
-    ByrefType += ", ";
-    ByrefType += "sizeof(";
-    RewriteByRefString(ByrefType, Name, ND);
-    ByrefType += ")";
-    if (HasCopyAndDispose) {
-      ByrefType += ", __Block_byref_id_object_copy_";
-      ByrefType += utostr(flag);
-      ByrefType += ", __Block_byref_id_object_dispose_";
-      ByrefType += utostr(flag);
-    }
-    ByrefType += "};\n";
-    unsigned nameSize = Name.size();
-    // for block or function pointer declaration. Name is aleady
-    // part of the declaration.
-    if (Ty->isBlockPointerType() || Ty->isFunctionPointerType())
-      nameSize = 1;
-    ReplaceText(DeclLoc, endBuf-startBuf+nameSize, ByrefType);
-  }
-  else {
-    SourceLocation startLoc;
-    Expr *E = ND->getInit();
-    if (const CStyleCastExpr *ECE = dyn_cast<CStyleCastExpr>(E))
-      startLoc = ECE->getLParenLoc();
-    else
-      startLoc = E->getLocStart();
-    startLoc = SM->getExpansionLoc(startLoc);
-    endBuf = SM->getCharacterData(startLoc);
-    ByrefType += " " + Name;
-    ByrefType += " = {(void*)";
-    ByrefType += utostr(isa);
-    ByrefType += "," +  ForwardingCastType + "&" + Name + ", ";
-    ByrefType += utostr(flags);
-    ByrefType += ", ";
-    ByrefType += "sizeof(";
-    RewriteByRefString(ByrefType, Name, ND);
-    ByrefType += "), ";
-    if (HasCopyAndDispose) {
-      ByrefType += "__Block_byref_id_object_copy_";
-      ByrefType += utostr(flag);
-      ByrefType += ", __Block_byref_id_object_dispose_";
-      ByrefType += utostr(flag);
-      ByrefType += ", ";
-    }
-    ReplaceText(DeclLoc, endBuf-startBuf, ByrefType);
-    
-    // Complete the newly synthesized compound expression by inserting a right
-    // curly brace before the end of the declaration.
-    // FIXME: This approach avoids rewriting the initializer expression. It
-    // also assumes there is only one declarator. For example, the following
-    // isn't currently supported by this routine (in general):
-    // 
-    // double __block BYREFVAR = 1.34, BYREFVAR2 = 1.37;
-    //
-    const char *startInitializerBuf = SM->getCharacterData(startLoc);
-    const char *semiBuf = strchr(startInitializerBuf, ';');
-    assert((*semiBuf == ';') && "RewriteByRefVar: can't find ';'");
-    SourceLocation semiLoc =
-      startLoc.getLocWithOffset(semiBuf-startInitializerBuf);
-
-    InsertText(semiLoc, "}");
-  }
-  return;
-}
-
-void RewriteObjC::CollectBlockDeclRefInfo(BlockExpr *Exp) {
-  // Add initializers for any closure decl refs.
-  GetBlockDeclRefExprs(Exp->getBody());
-  if (BlockDeclRefs.size()) {
-    // Unique all "by copy" declarations.
-    for (unsigned i = 0; i < BlockDeclRefs.size(); i++)
-      if (!BlockDeclRefs[i]->getDecl()->hasAttr<BlocksAttr>()) {
-        if (!BlockByCopyDeclsPtrSet.count(BlockDeclRefs[i]->getDecl())) {
-          BlockByCopyDeclsPtrSet.insert(BlockDeclRefs[i]->getDecl());
-          BlockByCopyDecls.push_back(BlockDeclRefs[i]->getDecl());
-        }
-      }
-    // Unique all "by ref" declarations.
-    for (unsigned i = 0; i < BlockDeclRefs.size(); i++)
-      if (BlockDeclRefs[i]->getDecl()->hasAttr<BlocksAttr>()) {
-        if (!BlockByRefDeclsPtrSet.count(BlockDeclRefs[i]->getDecl())) {
-          BlockByRefDeclsPtrSet.insert(BlockDeclRefs[i]->getDecl());
-          BlockByRefDecls.push_back(BlockDeclRefs[i]->getDecl());
-        }
-      }
-    // Find any imported blocks...they will need special attention.
-    for (unsigned i = 0; i < BlockDeclRefs.size(); i++)
-      if (BlockDeclRefs[i]->getDecl()->hasAttr<BlocksAttr>() ||
-          BlockDeclRefs[i]->getType()->isObjCObjectPointerType() || 
-          BlockDeclRefs[i]->getType()->isBlockPointerType())
-        ImportedBlockDecls.insert(BlockDeclRefs[i]->getDecl());
-  }
-}
-
-FunctionDecl *RewriteObjC::SynthBlockInitFunctionDecl(StringRef name) {
-  IdentifierInfo *ID = &Context->Idents.get(name);
-  QualType FType = Context->getFunctionNoProtoType(Context->VoidPtrTy);
-  return FunctionDecl::Create(*Context, TUDecl, SourceLocation(),
-                              SourceLocation(), ID, FType, 0, SC_Extern,
-                              SC_None, false, false);
-}
-
-Stmt *RewriteObjC::SynthBlockInitExpr(BlockExpr *Exp,
-          const SmallVector<DeclRefExpr *, 8> &InnerBlockDeclRefs) {
-  const BlockDecl *block = Exp->getBlockDecl();
-  Blocks.push_back(Exp);
-
-  CollectBlockDeclRefInfo(Exp);
-  
-  // Add inner imported variables now used in current block.
- int countOfInnerDecls = 0;
-  if (!InnerBlockDeclRefs.empty()) {
-    for (unsigned i = 0; i < InnerBlockDeclRefs.size(); i++) {
-      DeclRefExpr *Exp = InnerBlockDeclRefs[i];
-      ValueDecl *VD = Exp->getDecl();
-      if (!VD->hasAttr<BlocksAttr>() && !BlockByCopyDeclsPtrSet.count(VD)) {
-      // We need to save the copied-in variables in nested
-      // blocks because it is needed at the end for some of the API generations.
-      // See SynthesizeBlockLiterals routine.
-        InnerDeclRefs.push_back(Exp); countOfInnerDecls++;
-        BlockDeclRefs.push_back(Exp);
-        BlockByCopyDeclsPtrSet.insert(VD);
-        BlockByCopyDecls.push_back(VD);
-      }
-      if (VD->hasAttr<BlocksAttr>() && !BlockByRefDeclsPtrSet.count(VD)) {
-        InnerDeclRefs.push_back(Exp); countOfInnerDecls++;
-        BlockDeclRefs.push_back(Exp);
-        BlockByRefDeclsPtrSet.insert(VD);
-        BlockByRefDecls.push_back(VD);
-      }
-    }
-    // Find any imported blocks...they will need special attention.
-    for (unsigned i = 0; i < InnerBlockDeclRefs.size(); i++)
-      if (InnerBlockDeclRefs[i]->getDecl()->hasAttr<BlocksAttr>() ||
-          InnerBlockDeclRefs[i]->getType()->isObjCObjectPointerType() || 
-          InnerBlockDeclRefs[i]->getType()->isBlockPointerType())
-        ImportedBlockDecls.insert(InnerBlockDeclRefs[i]->getDecl());
-  }
-  InnerDeclRefsCount.push_back(countOfInnerDecls);
-  
-  std::string FuncName;
-
-  if (CurFunctionDef)
-    FuncName = CurFunctionDef->getNameAsString();
-  else if (CurMethodDef)
-    BuildUniqueMethodName(FuncName, CurMethodDef);
-  else if (GlobalVarDecl)
-    FuncName = std::string(GlobalVarDecl->getNameAsString());
-
-  std::string BlockNumber = utostr(Blocks.size()-1);
-
-  std::string Tag = "__" + FuncName + "_block_impl_" + BlockNumber;
-  std::string Func = "__" + FuncName + "_block_func_" + BlockNumber;
-
-  // Get a pointer to the function type so we can cast appropriately.
-  QualType BFT = convertFunctionTypeOfBlocks(Exp->getFunctionType());
-  QualType FType = Context->getPointerType(BFT);
-
-  FunctionDecl *FD;
-  Expr *NewRep;
-
-  // Simulate a contructor call...
-  FD = SynthBlockInitFunctionDecl(Tag);
-  DeclRefExpr *DRE = new (Context) DeclRefExpr(FD, false, FType, VK_RValue,
-                                               SourceLocation());
-
-  SmallVector<Expr*, 4> InitExprs;
-
-  // Initialize the block function.
-  FD = SynthBlockInitFunctionDecl(Func);
-  DeclRefExpr *Arg = new (Context) DeclRefExpr(FD, false, FD->getType(),
-                                               VK_LValue, SourceLocation());
-  CastExpr *castExpr = NoTypeInfoCStyleCastExpr(Context, Context->VoidPtrTy,
-                                                CK_BitCast, Arg);
-  InitExprs.push_back(castExpr);
-
-  // Initialize the block descriptor.
-  std::string DescData = "__" + FuncName + "_block_desc_" + BlockNumber + "_DATA";
-
-  VarDecl *NewVD = VarDecl::Create(*Context, TUDecl,
-                                   SourceLocation(), SourceLocation(),
-                                   &Context->Idents.get(DescData.c_str()),
-                                   Context->VoidPtrTy, 0,
-                                   SC_Static, SC_None);
-  UnaryOperator *DescRefExpr =
-    new (Context) UnaryOperator(new (Context) DeclRefExpr(NewVD, false,
-                                                          Context->VoidPtrTy,
-                                                          VK_LValue,
-                                                          SourceLocation()), 
-                                UO_AddrOf,
-                                Context->getPointerType(Context->VoidPtrTy), 
-                                VK_RValue, OK_Ordinary,
-                                SourceLocation());
-  InitExprs.push_back(DescRefExpr); 
-  
-  // Add initializers for any closure decl refs.
-  if (BlockDeclRefs.size()) {
-    Expr *Exp;
-    // Output all "by copy" declarations.
-    for (SmallVector<ValueDecl*,8>::iterator I = BlockByCopyDecls.begin(),
-         E = BlockByCopyDecls.end(); I != E; ++I) {
-      if (isObjCType((*I)->getType())) {
-        // FIXME: Conform to ABI ([[obj retain] autorelease]).
-        FD = SynthBlockInitFunctionDecl((*I)->getName());
-        Exp = new (Context) DeclRefExpr(FD, false, FD->getType(), VK_LValue,
-                                        SourceLocation());
-        if (HasLocalVariableExternalStorage(*I)) {
-          QualType QT = (*I)->getType();
-          QT = Context->getPointerType(QT);
-          Exp = new (Context) UnaryOperator(Exp, UO_AddrOf, QT, VK_RValue,
-                                            OK_Ordinary, SourceLocation());
-        }
-      } else if (isTopLevelBlockPointerType((*I)->getType())) {
-        FD = SynthBlockInitFunctionDecl((*I)->getName());
-        Arg = new (Context) DeclRefExpr(FD, false, FD->getType(), VK_LValue,
-                                        SourceLocation());
-        Exp = NoTypeInfoCStyleCastExpr(Context, Context->VoidPtrTy,
-                                       CK_BitCast, Arg);
-      } else {
-        FD = SynthBlockInitFunctionDecl((*I)->getName());
-        Exp = new (Context) DeclRefExpr(FD, false, FD->getType(), VK_LValue,
-                                        SourceLocation());
-        if (HasLocalVariableExternalStorage(*I)) {
-          QualType QT = (*I)->getType();
-          QT = Context->getPointerType(QT);
-          Exp = new (Context) UnaryOperator(Exp, UO_AddrOf, QT, VK_RValue,
-                                            OK_Ordinary, SourceLocation());
-        }
-        
-      }
-      InitExprs.push_back(Exp);
-    }
-    // Output all "by ref" declarations.
-    for (SmallVector<ValueDecl*,8>::iterator I = BlockByRefDecls.begin(),
-         E = BlockByRefDecls.end(); I != E; ++I) {
-      ValueDecl *ND = (*I);
-      std::string Name(ND->getNameAsString());
-      std::string RecName;
-      RewriteByRefString(RecName, Name, ND, true);
-      IdentifierInfo *II = &Context->Idents.get(RecName.c_str() 
-                                                + sizeof("struct"));
-      RecordDecl *RD = RecordDecl::Create(*Context, TTK_Struct, TUDecl,
-                                          SourceLocation(), SourceLocation(),
-                                          II);
-      assert(RD && "SynthBlockInitExpr(): Can't find RecordDecl");
-      QualType castT = Context->getPointerType(Context->getTagDeclType(RD));
-      
-      FD = SynthBlockInitFunctionDecl((*I)->getName());
-      Exp = new (Context) DeclRefExpr(FD, false, FD->getType(), VK_LValue,
-                                      SourceLocation());
-      bool isNestedCapturedVar = false;
-      if (block)
-        for (BlockDecl::capture_const_iterator ci = block->capture_begin(),
-             ce = block->capture_end(); ci != ce; ++ci) {
-          const VarDecl *variable = ci->getVariable();
-          if (variable == ND && ci->isNested()) {
-            assert (ci->isByRef() && 
-                    "SynthBlockInitExpr - captured block variable is not byref");
-            isNestedCapturedVar = true;
-            break;
-          }
-        }
-      // captured nested byref variable has its address passed. Do not take
-      // its address again.
-      if (!isNestedCapturedVar)
-          Exp = new (Context) UnaryOperator(Exp, UO_AddrOf,
-                                     Context->getPointerType(Exp->getType()),
-                                     VK_RValue, OK_Ordinary, SourceLocation());
-      Exp = NoTypeInfoCStyleCastExpr(Context, castT, CK_BitCast, Exp);
-      InitExprs.push_back(Exp);
-    }
-  }
-  if (ImportedBlockDecls.size()) {
-    // generate BLOCK_HAS_COPY_DISPOSE(have helper funcs) | BLOCK_HAS_DESCRIPTOR
-    int flag = (BLOCK_HAS_COPY_DISPOSE | BLOCK_HAS_DESCRIPTOR);
-    unsigned IntSize = 
-      static_cast<unsigned>(Context->getTypeSize(Context->IntTy));
-    Expr *FlagExp = IntegerLiteral::Create(*Context, llvm::APInt(IntSize, flag), 
-                                           Context->IntTy, SourceLocation());
-    InitExprs.push_back(FlagExp);
-  }
-  NewRep = new (Context) CallExpr(*Context, DRE, &InitExprs[0], InitExprs.size(),
-                                  FType, VK_LValue, SourceLocation());
-  NewRep = new (Context) UnaryOperator(NewRep, UO_AddrOf,
-                             Context->getPointerType(NewRep->getType()),
-                             VK_RValue, OK_Ordinary, SourceLocation());
-  NewRep = NoTypeInfoCStyleCastExpr(Context, FType, CK_BitCast,
-                                    NewRep);
-  BlockDeclRefs.clear();
-  BlockByRefDecls.clear();
-  BlockByRefDeclsPtrSet.clear();
-  BlockByCopyDecls.clear();
-  BlockByCopyDeclsPtrSet.clear();
-  ImportedBlockDecls.clear();
-  return NewRep;
-}
-
-bool RewriteObjC::IsDeclStmtInForeachHeader(DeclStmt *DS) {
-  if (const ObjCForCollectionStmt * CS = 
-      dyn_cast<ObjCForCollectionStmt>(Stmts.back()))
-        return CS->getElement() == DS;
-  return false;
-}
-
-//===----------------------------------------------------------------------===//
-// Function Body / Expression rewriting
-//===----------------------------------------------------------------------===//
-
-Stmt *RewriteObjC::RewriteFunctionBodyOrGlobalInitializer(Stmt *S) {
-  if (isa<SwitchStmt>(S) || isa<WhileStmt>(S) ||
-      isa<DoStmt>(S) || isa<ForStmt>(S))
-    Stmts.push_back(S);
-  else if (isa<ObjCForCollectionStmt>(S)) {
-    Stmts.push_back(S);
-    ObjCBcLabelNo.push_back(++BcLabelCount);
-  }
-
-  // Pseudo-object operations and ivar references need special
-  // treatment because we're going to recursively rewrite them.
-  if (PseudoObjectExpr *PseudoOp = dyn_cast<PseudoObjectExpr>(S)) {
-    if (isa<BinaryOperator>(PseudoOp->getSyntacticForm())) {
-      return RewritePropertyOrImplicitSetter(PseudoOp);
-    } else {
-      return RewritePropertyOrImplicitGetter(PseudoOp);
-    }
-  } else if (ObjCIvarRefExpr *IvarRefExpr = dyn_cast<ObjCIvarRefExpr>(S)) {
-    return RewriteObjCIvarRefExpr(IvarRefExpr);
-  }
-
-  SourceRange OrigStmtRange = S->getSourceRange();
-
-  // Perform a bottom up rewrite of all children.
-  for (Stmt::child_range CI = S->children(); CI; ++CI)
-    if (*CI) {
-      Stmt *childStmt = (*CI);
-      Stmt *newStmt = RewriteFunctionBodyOrGlobalInitializer(childStmt);
-      if (newStmt) {
-        *CI = newStmt;
-      }
-    }
-
-  if (BlockExpr *BE = dyn_cast<BlockExpr>(S)) {
-    SmallVector<DeclRefExpr *, 8> InnerBlockDeclRefs;
-    llvm::SmallPtrSet<const DeclContext *, 8> InnerContexts;
-    InnerContexts.insert(BE->getBlockDecl());
-    ImportedLocalExternalDecls.clear();
-    GetInnerBlockDeclRefExprs(BE->getBody(),
-                              InnerBlockDeclRefs, InnerContexts);
-    // Rewrite the block body in place.
-    Stmt *SaveCurrentBody = CurrentBody;
-    CurrentBody = BE->getBody();
-    PropParentMap = 0;
-    // block literal on rhs of a property-dot-sytax assignment
-    // must be replaced by its synthesize ast so getRewrittenText
-    // works as expected. In this case, what actually ends up on RHS
-    // is the blockTranscribed which is the helper function for the
-    // block literal; as in: self.c = ^() {[ace ARR];};
-    bool saveDisableReplaceStmt = DisableReplaceStmt;
-    DisableReplaceStmt = false;
-    RewriteFunctionBodyOrGlobalInitializer(BE->getBody());
-    DisableReplaceStmt = saveDisableReplaceStmt;
-    CurrentBody = SaveCurrentBody;
-    PropParentMap = 0;
-    ImportedLocalExternalDecls.clear();
-    // Now we snarf the rewritten text and stash it away for later use.
-    std::string Str = Rewrite.getRewrittenText(BE->getSourceRange());
-    RewrittenBlockExprs[BE] = Str;
-
-    Stmt *blockTranscribed = SynthBlockInitExpr(BE, InnerBlockDeclRefs);
-                            
-    //blockTranscribed->dump();
-    ReplaceStmt(S, blockTranscribed);
-    return blockTranscribed;
-  }
-  // Handle specific things.
-  if (ObjCEncodeExpr *AtEncode = dyn_cast<ObjCEncodeExpr>(S))
-    return RewriteAtEncode(AtEncode);
-
-  if (ObjCSelectorExpr *AtSelector = dyn_cast<ObjCSelectorExpr>(S))
-    return RewriteAtSelector(AtSelector);
-
-  if (ObjCStringLiteral *AtString = dyn_cast<ObjCStringLiteral>(S))
-    return RewriteObjCStringLiteral(AtString);
-
-  if (ObjCMessageExpr *MessExpr = dyn_cast<ObjCMessageExpr>(S)) {
-#if 0
-    // Before we rewrite it, put the original message expression in a comment.
-    SourceLocation startLoc = MessExpr->getLocStart();
-    SourceLocation endLoc = MessExpr->getLocEnd();
-
-    const char *startBuf = SM->getCharacterData(startLoc);
-    const char *endBuf = SM->getCharacterData(endLoc);
-
-    std::string messString;
-    messString += "// ";
-    messString.append(startBuf, endBuf-startBuf+1);
-    messString += "\n";
-
-    // FIXME: Missing definition of
-    // InsertText(clang::SourceLocation, char const*, unsigned int).
-    // InsertText(startLoc, messString.c_str(), messString.size());
-    // Tried this, but it didn't work either...
-    // ReplaceText(startLoc, 0, messString.c_str(), messString.size());
-#endif
-    return RewriteMessageExpr(MessExpr);
-  }
-
-  if (ObjCAtTryStmt *StmtTry = dyn_cast<ObjCAtTryStmt>(S))
-    return RewriteObjCTryStmt(StmtTry);
-
-  if (ObjCAtSynchronizedStmt *StmtTry = dyn_cast<ObjCAtSynchronizedStmt>(S))
-    return RewriteObjCSynchronizedStmt(StmtTry);
-
-  if (ObjCAtThrowStmt *StmtThrow = dyn_cast<ObjCAtThrowStmt>(S))
-    return RewriteObjCThrowStmt(StmtThrow);
-
-  if (ObjCProtocolExpr *ProtocolExp = dyn_cast<ObjCProtocolExpr>(S))
-    return RewriteObjCProtocolExpr(ProtocolExp);
-
-  if (ObjCForCollectionStmt *StmtForCollection =
-        dyn_cast<ObjCForCollectionStmt>(S))
-    return RewriteObjCForCollectionStmt(StmtForCollection,
-                                        OrigStmtRange.getEnd());
-  if (BreakStmt *StmtBreakStmt =
-      dyn_cast<BreakStmt>(S))
-    return RewriteBreakStmt(StmtBreakStmt);
-  if (ContinueStmt *StmtContinueStmt =
-      dyn_cast<ContinueStmt>(S))
-    return RewriteContinueStmt(StmtContinueStmt);
-
-  // Need to check for protocol refs (id <P>, Foo <P> *) in variable decls
-  // and cast exprs.
-  if (DeclStmt *DS = dyn_cast<DeclStmt>(S)) {
-    // FIXME: What we're doing here is modifying the type-specifier that
-    // precedes the first Decl.  In the future the DeclGroup should have
-    // a separate type-specifier that we can rewrite.
-    // NOTE: We need to avoid rewriting the DeclStmt if it is within
-    // the context of an ObjCForCollectionStmt. For example:
-    //   NSArray *someArray;
-    //   for (id <FooProtocol> index in someArray) ;
-    // This is because RewriteObjCForCollectionStmt() does textual rewriting 
-    // and it depends on the original text locations/positions.
-    if (Stmts.empty() || !IsDeclStmtInForeachHeader(DS))
-      RewriteObjCQualifiedInterfaceTypes(*DS->decl_begin());
-
-    // Blocks rewrite rules.
-    for (DeclStmt::decl_iterator DI = DS->decl_begin(), DE = DS->decl_end();
-         DI != DE; ++DI) {
-      Decl *SD = *DI;
-      if (ValueDecl *ND = dyn_cast<ValueDecl>(SD)) {
-        if (isTopLevelBlockPointerType(ND->getType()))
-          RewriteBlockPointerDecl(ND);
-        else if (ND->getType()->isFunctionPointerType())
-          CheckFunctionPointerDecl(ND->getType(), ND);
-        if (VarDecl *VD = dyn_cast<VarDecl>(SD)) {
-          if (VD->hasAttr<BlocksAttr>()) {
-            static unsigned uniqueByrefDeclCount = 0;
-            assert(!BlockByRefDeclNo.count(ND) &&
-              "RewriteFunctionBodyOrGlobalInitializer: Duplicate byref decl");
-            BlockByRefDeclNo[ND] = uniqueByrefDeclCount++;
-            RewriteByRefVar(VD);
-          }
-          else           
-            RewriteTypeOfDecl(VD);
-        }
-      }
-      if (TypedefNameDecl *TD = dyn_cast<TypedefNameDecl>(SD)) {
-        if (isTopLevelBlockPointerType(TD->getUnderlyingType()))
-          RewriteBlockPointerDecl(TD);
-        else if (TD->getUnderlyingType()->isFunctionPointerType())
-          CheckFunctionPointerDecl(TD->getUnderlyingType(), TD);
-      }
-    }
-  }
-
-  if (CStyleCastExpr *CE = dyn_cast<CStyleCastExpr>(S))
-    RewriteObjCQualifiedInterfaceTypes(CE);
-
-  if (isa<SwitchStmt>(S) || isa<WhileStmt>(S) ||
-      isa<DoStmt>(S) || isa<ForStmt>(S)) {
-    assert(!Stmts.empty() && "Statement stack is empty");
-    assert ((isa<SwitchStmt>(Stmts.back()) || isa<WhileStmt>(Stmts.back()) ||
-             isa<DoStmt>(Stmts.back()) || isa<ForStmt>(Stmts.back()))
-            && "Statement stack mismatch");
-    Stmts.pop_back();
-  }
-  // Handle blocks rewriting.
-  if (DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(S)) {
-    ValueDecl *VD = DRE->getDecl(); 
-    if (VD->hasAttr<BlocksAttr>())
-      return RewriteBlockDeclRefExpr(DRE);
-    if (HasLocalVariableExternalStorage(VD))
-      return RewriteLocalVariableExternalStorage(DRE);
-  }
-  
-  if (CallExpr *CE = dyn_cast<CallExpr>(S)) {
-    if (CE->getCallee()->getType()->isBlockPointerType()) {
-      Stmt *BlockCall = SynthesizeBlockCall(CE, CE->getCallee());
-      ReplaceStmt(S, BlockCall);
-      return BlockCall;
-    }
-  }
-  if (CStyleCastExpr *CE = dyn_cast<CStyleCastExpr>(S)) {
-    RewriteCastExpr(CE);
-  }
-#if 0
-  if (ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(S)) {
-    CastExpr *Replacement = new (Context) CastExpr(ICE->getType(),
-                                                   ICE->getSubExpr(),
-                                                   SourceLocation());
-    // Get the new text.
-    std::string SStr;
-    llvm::raw_string_ostream Buf(SStr);
-    Replacement->printPretty(Buf);
-    const std::string &Str = Buf.str();
-
-    printf("CAST = %s\n", &Str[0]);
-    InsertText(ICE->getSubExpr()->getLocStart(), &Str[0], Str.size());
-    delete S;
-    return Replacement;
-  }
-#endif
-  // Return this stmt unmodified.
-  return S;
-}
-
-void RewriteObjC::RewriteRecordBody(RecordDecl *RD) {
-  for (RecordDecl::field_iterator i = RD->field_begin(), 
-                                  e = RD->field_end(); i != e; ++i) {
-    FieldDecl *FD = *i;
-    if (isTopLevelBlockPointerType(FD->getType()))
-      RewriteBlockPointerDecl(FD);
-    if (FD->getType()->isObjCQualifiedIdType() ||
-        FD->getType()->isObjCQualifiedInterfaceType())
-      RewriteObjCQualifiedInterfaceTypes(FD);
-  }
-}
-
-/// HandleDeclInMainFile - This is called for each top-level decl defined in the
-/// main file of the input.
-void RewriteObjC::HandleDeclInMainFile(Decl *D) {
-  switch (D->getKind()) {
-    case Decl::Function: {
-      FunctionDecl *FD = cast<FunctionDecl>(D);
-      if (FD->isOverloadedOperator())
-        return;
-
-      // Since function prototypes don't have ParmDecl's, we check the function
-      // prototype. This enables us to rewrite function declarations and
-      // definitions using the same code.
-      RewriteBlocksInFunctionProtoType(FD->getType(), FD);
-
-      if (!FD->isThisDeclarationADefinition())
-        break;
-
-      // FIXME: If this should support Obj-C++, support CXXTryStmt
-      if (CompoundStmt *Body = dyn_cast_or_null<CompoundStmt>(FD->getBody())) {
-        CurFunctionDef = FD;
-        CurFunctionDeclToDeclareForBlock = FD;
-        CurrentBody = Body;
-        Body =
-        cast_or_null<CompoundStmt>(RewriteFunctionBodyOrGlobalInitializer(Body));
-        FD->setBody(Body);
-        CurrentBody = 0;
-        if (PropParentMap) {
-          delete PropParentMap;
-          PropParentMap = 0;
-        }
-        // This synthesizes and inserts the block "impl" struct, invoke function,
-        // and any copy/dispose helper functions.
-        InsertBlockLiteralsWithinFunction(FD);
-        CurFunctionDef = 0;
-        CurFunctionDeclToDeclareForBlock = 0;
-      }
-      break;
-    }
-    case Decl::ObjCMethod: {
-      ObjCMethodDecl *MD = cast<ObjCMethodDecl>(D);
-      if (CompoundStmt *Body = MD->getCompoundBody()) {
-        CurMethodDef = MD;
-        CurrentBody = Body;
-        Body =
-          cast_or_null<CompoundStmt>(RewriteFunctionBodyOrGlobalInitializer(Body));
-        MD->setBody(Body);
-        CurrentBody = 0;
-        if (PropParentMap) {
-          delete PropParentMap;
-          PropParentMap = 0;
-        }
-        InsertBlockLiteralsWithinMethod(MD);
-        CurMethodDef = 0;
-      }
-      break;
-    }
-    case Decl::ObjCImplementation: {
-      ObjCImplementationDecl *CI = cast<ObjCImplementationDecl>(D);
-      ClassImplementation.push_back(CI);
-      break;
-    }
-    case Decl::ObjCCategoryImpl: {
-      ObjCCategoryImplDecl *CI = cast<ObjCCategoryImplDecl>(D);
-      CategoryImplementation.push_back(CI);
-      break;
-    }
-    case Decl::Var: {
-      VarDecl *VD = cast<VarDecl>(D);
-      RewriteObjCQualifiedInterfaceTypes(VD);
-      if (isTopLevelBlockPointerType(VD->getType()))
-        RewriteBlockPointerDecl(VD);
-      else if (VD->getType()->isFunctionPointerType()) {
-        CheckFunctionPointerDecl(VD->getType(), VD);
-        if (VD->getInit()) {
-          if (CStyleCastExpr *CE = dyn_cast<CStyleCastExpr>(VD->getInit())) {
-            RewriteCastExpr(CE);
-          }
-        }
-      } else if (VD->getType()->isRecordType()) {
-        RecordDecl *RD = VD->getType()->getAs<RecordType>()->getDecl();
-        if (RD->isCompleteDefinition())
-          RewriteRecordBody(RD);
-      }
-      if (VD->getInit()) {
-        GlobalVarDecl = VD;
-        CurrentBody = VD->getInit();
-        RewriteFunctionBodyOrGlobalInitializer(VD->getInit());
-        CurrentBody = 0;
-        if (PropParentMap) {
-          delete PropParentMap;
-          PropParentMap = 0;
-        }
-        SynthesizeBlockLiterals(VD->getTypeSpecStartLoc(), VD->getName());
-        GlobalVarDecl = 0;
-          
-        // This is needed for blocks.
-        if (CStyleCastExpr *CE = dyn_cast<CStyleCastExpr>(VD->getInit())) {
-            RewriteCastExpr(CE);
-        }
-      }
-      break;
-    }
-    case Decl::TypeAlias:
-    case Decl::Typedef: {
-      if (TypedefNameDecl *TD = dyn_cast<TypedefNameDecl>(D)) {
-        if (isTopLevelBlockPointerType(TD->getUnderlyingType()))
-          RewriteBlockPointerDecl(TD);
-        else if (TD->getUnderlyingType()->isFunctionPointerType())
-          CheckFunctionPointerDecl(TD->getUnderlyingType(), TD);
-      }
-      break;
-    }
-    case Decl::CXXRecord:
-    case Decl::Record: {
-      RecordDecl *RD = cast<RecordDecl>(D);
-      if (RD->isCompleteDefinition()) 
-        RewriteRecordBody(RD);
-      break;
-    }
-    default:
-      break;
-  }
-  // Nothing yet.
-}
-
-void RewriteObjC::HandleTranslationUnit(ASTContext &C) {
-  if (Diags.hasErrorOccurred())
-    return;
-
-  RewriteInclude();
-
-  // Here's a great place to add any extra declarations that may be needed.
-  // Write out meta data for each @protocol(<expr>).
-  for (llvm::SmallPtrSet<ObjCProtocolDecl *,8>::iterator I = ProtocolExprDecls.begin(),
-       E = ProtocolExprDecls.end(); I != E; ++I)
-    RewriteObjCProtocolMetaData(*I, "", "", Preamble);
-
-  InsertText(SM->getLocForStartOfFile(MainFileID), Preamble, false);
-  if (ClassImplementation.size() || CategoryImplementation.size())
-    RewriteImplementations();
-
-  // Get the buffer corresponding to MainFileID.  If we haven't changed it, then
-  // we are done.
-  if (const RewriteBuffer *RewriteBuf =
-      Rewrite.getRewriteBufferFor(MainFileID)) {
-    //printf("Changed:\n");
-    *OutFile << std::string(RewriteBuf->begin(), RewriteBuf->end());
-  } else {
-    llvm::errs() << "No changes\n";
-  }
-
-  if (ClassImplementation.size() || CategoryImplementation.size() ||
-      ProtocolExprDecls.size()) {
-    // Rewrite Objective-c meta data*
-    std::string ResultStr;
-    RewriteMetaDataIntoBuffer(ResultStr);
-    // Emit metadata.
-    *OutFile << ResultStr;
-  }
-  OutFile->flush();
-}
-
-void RewriteObjCFragileABI::Initialize(ASTContext &context) {
-  InitializeCommon(context);
-  
-  // declaring objc_selector outside the parameter list removes a silly
-  // scope related warning...
-  if (IsHeader)
-    Preamble = "#pragma once\n";
-  Preamble += "struct objc_selector; struct objc_class;\n";
-  Preamble += "struct __rw_objc_super { struct objc_object *object; ";
-  Preamble += "struct objc_object *superClass; ";
-  if (LangOpts.MicrosoftExt) {
-    // Add a constructor for creating temporary objects.
-    Preamble += "__rw_objc_super(struct objc_object *o, struct objc_object *s) "
-    ": ";
-    Preamble += "object(o), superClass(s) {} ";
-  }
-  Preamble += "};\n";
-  Preamble += "#ifndef _REWRITER_typedef_Protocol\n";
-  Preamble += "typedef struct objc_object Protocol;\n";
-  Preamble += "#define _REWRITER_typedef_Protocol\n";
-  Preamble += "#endif\n";
-  if (LangOpts.MicrosoftExt) {
-    Preamble += "#define __OBJC_RW_DLLIMPORT extern \"C\" __declspec(dllimport)\n";
-    Preamble += "#define __OBJC_RW_STATICIMPORT extern \"C\"\n";
-  } else
-    Preamble += "#define __OBJC_RW_DLLIMPORT extern\n";
-  Preamble += "__OBJC_RW_DLLIMPORT struct objc_object *objc_msgSend";
-  Preamble += "(struct objc_object *, struct objc_selector *, ...);\n";
-  Preamble += "__OBJC_RW_DLLIMPORT struct objc_object *objc_msgSendSuper";
-  Preamble += "(struct objc_super *, struct objc_selector *, ...);\n";
-  Preamble += "__OBJC_RW_DLLIMPORT struct objc_object* objc_msgSend_stret";
-  Preamble += "(struct objc_object *, struct objc_selector *, ...);\n";
-  Preamble += "__OBJC_RW_DLLIMPORT struct objc_object* objc_msgSendSuper_stret";
-  Preamble += "(struct objc_super *, struct objc_selector *, ...);\n";
-  Preamble += "__OBJC_RW_DLLIMPORT double objc_msgSend_fpret";
-  Preamble += "(struct objc_object *, struct objc_selector *, ...);\n";
-  Preamble += "__OBJC_RW_DLLIMPORT struct objc_object *objc_getClass";
-  Preamble += "(const char *);\n";
-  Preamble += "__OBJC_RW_DLLIMPORT struct objc_class *class_getSuperclass";
-  Preamble += "(struct objc_class *);\n";
-  Preamble += "__OBJC_RW_DLLIMPORT struct objc_object *objc_getMetaClass";
-  Preamble += "(const char *);\n";
-  Preamble += "__OBJC_RW_DLLIMPORT void objc_exception_throw(struct objc_object *);\n";
-  Preamble += "__OBJC_RW_DLLIMPORT void objc_exception_try_enter(void *);\n";
-  Preamble += "__OBJC_RW_DLLIMPORT void objc_exception_try_exit(void *);\n";
-  Preamble += "__OBJC_RW_DLLIMPORT struct objc_object *objc_exception_extract(void *);\n";
-  Preamble += "__OBJC_RW_DLLIMPORT int objc_exception_match";
-  Preamble += "(struct objc_class *, struct objc_object *);\n";
-  // @synchronized hooks.
-  Preamble += "__OBJC_RW_DLLIMPORT void objc_sync_enter(struct objc_object *);\n";
-  Preamble += "__OBJC_RW_DLLIMPORT void objc_sync_exit(struct objc_object *);\n";
-  Preamble += "__OBJC_RW_DLLIMPORT Protocol *objc_getProtocol(const char *);\n";
-  Preamble += "#ifndef __FASTENUMERATIONSTATE\n";
-  Preamble += "struct __objcFastEnumerationState {\n\t";
-  Preamble += "unsigned long state;\n\t";
-  Preamble += "void **itemsPtr;\n\t";
-  Preamble += "unsigned long *mutationsPtr;\n\t";
-  Preamble += "unsigned long extra[5];\n};\n";
-  Preamble += "__OBJC_RW_DLLIMPORT void objc_enumerationMutation(struct objc_object *);\n";
-  Preamble += "#define __FASTENUMERATIONSTATE\n";
-  Preamble += "#endif\n";
-  Preamble += "#ifndef __NSCONSTANTSTRINGIMPL\n";
-  Preamble += "struct __NSConstantStringImpl {\n";
-  Preamble += "  int *isa;\n";
-  Preamble += "  int flags;\n";
-  Preamble += "  char *str;\n";
-  Preamble += "  long length;\n";
-  Preamble += "};\n";
-  Preamble += "#ifdef CF_EXPORT_CONSTANT_STRING\n";
-  Preamble += "extern \"C\" __declspec(dllexport) int __CFConstantStringClassReference[];\n";
-  Preamble += "#else\n";
-  Preamble += "__OBJC_RW_DLLIMPORT int __CFConstantStringClassReference[];\n";
-  Preamble += "#endif\n";
-  Preamble += "#define __NSCONSTANTSTRINGIMPL\n";
-  Preamble += "#endif\n";
-  // Blocks preamble.
-  Preamble += "#ifndef BLOCK_IMPL\n";
-  Preamble += "#define BLOCK_IMPL\n";
-  Preamble += "struct __block_impl {\n";
-  Preamble += "  void *isa;\n";
-  Preamble += "  int Flags;\n";
-  Preamble += "  int Reserved;\n";
-  Preamble += "  void *FuncPtr;\n";
-  Preamble += "};\n";
-  Preamble += "// Runtime copy/destroy helper functions (from Block_private.h)\n";
-  Preamble += "#ifdef __OBJC_EXPORT_BLOCKS\n";
-  Preamble += "extern \"C\" __declspec(dllexport) "
-  "void _Block_object_assign(void *, const void *, const int);\n";
-  Preamble += "extern \"C\" __declspec(dllexport) void _Block_object_dispose(const void *, const int);\n";
-  Preamble += "extern \"C\" __declspec(dllexport) void *_NSConcreteGlobalBlock[32];\n";
-  Preamble += "extern \"C\" __declspec(dllexport) void *_NSConcreteStackBlock[32];\n";
-  Preamble += "#else\n";
-  Preamble += "__OBJC_RW_DLLIMPORT void _Block_object_assign(void *, const void *, const int);\n";
-  Preamble += "__OBJC_RW_DLLIMPORT void _Block_object_dispose(const void *, const int);\n";
-  Preamble += "__OBJC_RW_DLLIMPORT void *_NSConcreteGlobalBlock[32];\n";
-  Preamble += "__OBJC_RW_DLLIMPORT void *_NSConcreteStackBlock[32];\n";
-  Preamble += "#endif\n";
-  Preamble += "#endif\n";
-  if (LangOpts.MicrosoftExt) {
-    Preamble += "#undef __OBJC_RW_DLLIMPORT\n";
-    Preamble += "#undef __OBJC_RW_STATICIMPORT\n";
-    Preamble += "#ifndef KEEP_ATTRIBUTES\n";  // We use this for clang tests.
-    Preamble += "#define __attribute__(X)\n";
-    Preamble += "#endif\n";
-    Preamble += "#define __weak\n";
-  }
-  else {
-    Preamble += "#define __block\n";
-    Preamble += "#define __weak\n";
-  }
-  // NOTE! Windows uses LLP64 for 64bit mode. So, cast pointer to long long
-  // as this avoids warning in any 64bit/32bit compilation model.
-  Preamble += "\n#define __OFFSETOFIVAR__(TYPE, MEMBER) ((long long) &((TYPE *)0)->MEMBER)\n";
-}
-
-/// RewriteIvarOffsetComputation - This rutine synthesizes computation of
-/// ivar offset.
-void RewriteObjCFragileABI::RewriteIvarOffsetComputation(ObjCIvarDecl *ivar,
-                                                         std::string &Result) {
-  if (ivar->isBitField()) {
-    // FIXME: The hack below doesn't work for bitfields. For now, we simply
-    // place all bitfields at offset 0.
-    Result += "0";
-  } else {
-    Result += "__OFFSETOFIVAR__(struct ";
-    Result += ivar->getContainingInterface()->getNameAsString();
-    if (LangOpts.MicrosoftExt)
-      Result += "_IMPL";
-    Result += ", ";
-    Result += ivar->getNameAsString();
-    Result += ")";
-  }
-}
-
-/// RewriteObjCProtocolMetaData - Rewrite protocols meta-data.
-void RewriteObjCFragileABI::RewriteObjCProtocolMetaData(
-                            ObjCProtocolDecl *PDecl, StringRef prefix,
-                            StringRef ClassName, std::string &Result) {
-  static bool objc_protocol_methods = false;
-  
-  // Output struct protocol_methods holder of method selector and type.
-  if (!objc_protocol_methods && PDecl->hasDefinition()) {
-    /* struct protocol_methods {
-     SEL _cmd;
-     char *method_types;
-     }
-     */
-    Result += "\nstruct _protocol_methods {\n";
-    Result += "\tstruct objc_selector *_cmd;\n";
-    Result += "\tchar *method_types;\n";
-    Result += "};\n";
-    
-    objc_protocol_methods = true;
-  }
-  // Do not synthesize the protocol more than once.
-  if (ObjCSynthesizedProtocols.count(PDecl->getCanonicalDecl()))
-    return;
-  
-  if (ObjCProtocolDecl *Def = PDecl->getDefinition())
-    PDecl = Def;
-  
-  if (PDecl->instmeth_begin() != PDecl->instmeth_end()) {
-    unsigned NumMethods = std::distance(PDecl->instmeth_begin(),
-                                        PDecl->instmeth_end());
-    /* struct _objc_protocol_method_list {
-     int protocol_method_count;
-     struct protocol_methods protocols[];
-     }
-     */
-    Result += "\nstatic struct {\n";
-    Result += "\tint protocol_method_count;\n";
-    Result += "\tstruct _protocol_methods protocol_methods[";
-    Result += utostr(NumMethods);
-    Result += "];\n} _OBJC_PROTOCOL_INSTANCE_METHODS_";
-    Result += PDecl->getNameAsString();
-    Result += " __attribute__ ((used, section (\"__OBJC, __cat_inst_meth\")))= "
-    "{\n\t" + utostr(NumMethods) + "\n";
-    
-    // Output instance methods declared in this protocol.
-    for (ObjCProtocolDecl::instmeth_iterator
-         I = PDecl->instmeth_begin(), E = PDecl->instmeth_end();
-         I != E; ++I) {
-      if (I == PDecl->instmeth_begin())
-        Result += "\t  ,{{(struct objc_selector *)\"";
-      else
-        Result += "\t  ,{(struct objc_selector *)\"";
-      Result += (*I)->getSelector().getAsString();
-      std::string MethodTypeString;
-      Context->getObjCEncodingForMethodDecl((*I), MethodTypeString);
-      Result += "\", \"";
-      Result += MethodTypeString;
-      Result += "\"}\n";
-    }
-    Result += "\t }\n};\n";
-  }
-  
-  // Output class methods declared in this protocol.
-  unsigned NumMethods = std::distance(PDecl->classmeth_begin(),
-                                      PDecl->classmeth_end());
-  if (NumMethods > 0) {
-    /* struct _objc_protocol_method_list {
-     int protocol_method_count;
-     struct protocol_methods protocols[];
-     }
-     */
-    Result += "\nstatic struct {\n";
-    Result += "\tint protocol_method_count;\n";
-    Result += "\tstruct _protocol_methods protocol_methods[";
-    Result += utostr(NumMethods);
-    Result += "];\n} _OBJC_PROTOCOL_CLASS_METHODS_";
-    Result += PDecl->getNameAsString();
-    Result += " __attribute__ ((used, section (\"__OBJC, __cat_cls_meth\")))= "
-    "{\n\t";
-    Result += utostr(NumMethods);
-    Result += "\n";
-    
-    // Output instance methods declared in this protocol.
-    for (ObjCProtocolDecl::classmeth_iterator
-         I = PDecl->classmeth_begin(), E = PDecl->classmeth_end();
-         I != E; ++I) {
-      if (I == PDecl->classmeth_begin())
-        Result += "\t  ,{{(struct objc_selector *)\"";
-      else
-        Result += "\t  ,{(struct objc_selector *)\"";
-      Result += (*I)->getSelector().getAsString();
-      std::string MethodTypeString;
-      Context->getObjCEncodingForMethodDecl((*I), MethodTypeString);
-      Result += "\", \"";
-      Result += MethodTypeString;
-      Result += "\"}\n";
-    }
-    Result += "\t }\n};\n";
-  }
-  
-  // Output:
-  /* struct _objc_protocol {
-   // Objective-C 1.0 extensions
-   struct _objc_protocol_extension *isa;
-   char *protocol_name;
-   struct _objc_protocol **protocol_list;
-   struct _objc_protocol_method_list *instance_methods;
-   struct _objc_protocol_method_list *class_methods;
-   };
-   */
-  static bool objc_protocol = false;
-  if (!objc_protocol) {
-    Result += "\nstruct _objc_protocol {\n";
-    Result += "\tstruct _objc_protocol_extension *isa;\n";
-    Result += "\tchar *protocol_name;\n";
-    Result += "\tstruct _objc_protocol **protocol_list;\n";
-    Result += "\tstruct _objc_protocol_method_list *instance_methods;\n";
-    Result += "\tstruct _objc_protocol_method_list *class_methods;\n";
-    Result += "};\n";
-    
-    objc_protocol = true;
-  }
-  
-  Result += "\nstatic struct _objc_protocol _OBJC_PROTOCOL_";
-  Result += PDecl->getNameAsString();
-  Result += " __attribute__ ((used, section (\"__OBJC, __protocol\")))= "
-  "{\n\t0, \"";
-  Result += PDecl->getNameAsString();
-  Result += "\", 0, ";
-  if (PDecl->instmeth_begin() != PDecl->instmeth_end()) {
-    Result += "(struct _objc_protocol_method_list *)&_OBJC_PROTOCOL_INSTANCE_METHODS_";
-    Result += PDecl->getNameAsString();
-    Result += ", ";
-  }
-  else
-    Result += "0, ";
-  if (PDecl->classmeth_begin() != PDecl->classmeth_end()) {
-    Result += "(struct _objc_protocol_method_list *)&_OBJC_PROTOCOL_CLASS_METHODS_";
-    Result += PDecl->getNameAsString();
-    Result += "\n";
-  }
-  else
-    Result += "0\n";
-  Result += "};\n";
-  
-  // Mark this protocol as having been generated.
-  if (!ObjCSynthesizedProtocols.insert(PDecl->getCanonicalDecl()))
-    llvm_unreachable("protocol already synthesized");
-  
-}
-
-void RewriteObjCFragileABI::RewriteObjCProtocolListMetaData(
-                                const ObjCList<ObjCProtocolDecl> &Protocols,
-                                StringRef prefix, StringRef ClassName,
-                                std::string &Result) {
-  if (Protocols.empty()) return;
-  
-  for (unsigned i = 0; i != Protocols.size(); i++)
-    RewriteObjCProtocolMetaData(Protocols[i], prefix, ClassName, Result);
-  
-  // Output the top lovel protocol meta-data for the class.
-  /* struct _objc_protocol_list {
-   struct _objc_protocol_list *next;
-   int    protocol_count;
-   struct _objc_protocol *class_protocols[];
-   }
-   */
-  Result += "\nstatic struct {\n";
-  Result += "\tstruct _objc_protocol_list *next;\n";
-  Result += "\tint    protocol_count;\n";
-  Result += "\tstruct _objc_protocol *class_protocols[";
-  Result += utostr(Protocols.size());
-  Result += "];\n} _OBJC_";
-  Result += prefix;
-  Result += "_PROTOCOLS_";
-  Result += ClassName;
-  Result += " __attribute__ ((used, section (\"__OBJC, __cat_cls_meth\")))= "
-  "{\n\t0, ";
-  Result += utostr(Protocols.size());
-  Result += "\n";
-  
-  Result += "\t,{&_OBJC_PROTOCOL_";
-  Result += Protocols[0]->getNameAsString();
-  Result += " \n";
-  
-  for (unsigned i = 1; i != Protocols.size(); i++) {
-    Result += "\t ,&_OBJC_PROTOCOL_";
-    Result += Protocols[i]->getNameAsString();
-    Result += "\n";
-  }
-  Result += "\t }\n};\n";
-}
-
-void RewriteObjCFragileABI::RewriteObjCClassMetaData(ObjCImplementationDecl *IDecl,
-                                           std::string &Result) {
-  ObjCInterfaceDecl *CDecl = IDecl->getClassInterface();
-  
-  // Explicitly declared @interface's are already synthesized.
-  if (CDecl->isImplicitInterfaceDecl()) {
-    // FIXME: Implementation of a class with no @interface (legacy) does not
-    // produce correct synthesis as yet.
-    RewriteObjCInternalStruct(CDecl, Result);
-  }
-  
-  // Build _objc_ivar_list metadata for classes ivars if needed
-  unsigned NumIvars = !IDecl->ivar_empty()
-  ? IDecl->ivar_size()
-  : (CDecl ? CDecl->ivar_size() : 0);
-  if (NumIvars > 0) {
-    static bool objc_ivar = false;
-    if (!objc_ivar) {
-      /* struct _objc_ivar {
-       char *ivar_name;
-       char *ivar_type;
-       int ivar_offset;
-       };
-       */
-      Result += "\nstruct _objc_ivar {\n";
-      Result += "\tchar *ivar_name;\n";
-      Result += "\tchar *ivar_type;\n";
-      Result += "\tint ivar_offset;\n";
-      Result += "};\n";
-      
-      objc_ivar = true;
-    }
-    
-    /* struct {
-     int ivar_count;
-     struct _objc_ivar ivar_list[nIvars];
-     };
-     */
-    Result += "\nstatic struct {\n";
-    Result += "\tint ivar_count;\n";
-    Result += "\tstruct _objc_ivar ivar_list[";
-    Result += utostr(NumIvars);
-    Result += "];\n} _OBJC_INSTANCE_VARIABLES_";
-    Result += IDecl->getNameAsString();
-    Result += " __attribute__ ((used, section (\"__OBJC, __instance_vars\")))= "
-    "{\n\t";
-    Result += utostr(NumIvars);
-    Result += "\n";
-    
-    ObjCInterfaceDecl::ivar_iterator IVI, IVE;
-    SmallVector<ObjCIvarDecl *, 8> IVars;
-    if (!IDecl->ivar_empty()) {
-      for (ObjCInterfaceDecl::ivar_iterator
-           IV = IDecl->ivar_begin(), IVEnd = IDecl->ivar_end();
-           IV != IVEnd; ++IV)
-        IVars.push_back(*IV);
-      IVI = IDecl->ivar_begin();
-      IVE = IDecl->ivar_end();
-    } else {
-      IVI = CDecl->ivar_begin();
-      IVE = CDecl->ivar_end();
-    }
-    Result += "\t,{{\"";
-    Result += IVI->getNameAsString();
-    Result += "\", \"";
-    std::string TmpString, StrEncoding;
-    Context->getObjCEncodingForType(IVI->getType(), TmpString, *IVI);
-    QuoteDoublequotes(TmpString, StrEncoding);
-    Result += StrEncoding;
-    Result += "\", ";
-    RewriteIvarOffsetComputation(*IVI, Result);
-    Result += "}\n";
-    for (++IVI; IVI != IVE; ++IVI) {
-      Result += "\t  ,{\"";
-      Result += IVI->getNameAsString();
-      Result += "\", \"";
-      std::string TmpString, StrEncoding;
-      Context->getObjCEncodingForType(IVI->getType(), TmpString, *IVI);
-      QuoteDoublequotes(TmpString, StrEncoding);
-      Result += StrEncoding;
-      Result += "\", ";
-      RewriteIvarOffsetComputation(*IVI, Result);
-      Result += "}\n";
-    }
-    
-    Result += "\t }\n};\n";
-  }
-  
-  // Build _objc_method_list for class's instance methods if needed
-  SmallVector<ObjCMethodDecl *, 32>
-  InstanceMethods(IDecl->instmeth_begin(), IDecl->instmeth_end());
-  
-  // If any of our property implementations have associated getters or
-  // setters, produce metadata for them as well.
-  for (ObjCImplDecl::propimpl_iterator Prop = IDecl->propimpl_begin(),
-       PropEnd = IDecl->propimpl_end();
-       Prop != PropEnd; ++Prop) {
-    if (Prop->getPropertyImplementation() == ObjCPropertyImplDecl::Dynamic)
-      continue;
-    if (!Prop->getPropertyIvarDecl())
-      continue;
-    ObjCPropertyDecl *PD = Prop->getPropertyDecl();
-    if (!PD)
-      continue;
-    if (ObjCMethodDecl *Getter = PD->getGetterMethodDecl())
-      if (!Getter->isDefined())
-        InstanceMethods.push_back(Getter);
-    if (PD->isReadOnly())
-      continue;
-    if (ObjCMethodDecl *Setter = PD->getSetterMethodDecl())
-      if (!Setter->isDefined())
-        InstanceMethods.push_back(Setter);
-  }
-  RewriteObjCMethodsMetaData(InstanceMethods.begin(), InstanceMethods.end(),
-                             true, "", IDecl->getName(), Result);
-  
-  // Build _objc_method_list for class's class methods if needed
-  RewriteObjCMethodsMetaData(IDecl->classmeth_begin(), IDecl->classmeth_end(),
-                             false, "", IDecl->getName(), Result);
-  
-  // Protocols referenced in class declaration?
-  RewriteObjCProtocolListMetaData(CDecl->getReferencedProtocols(),
-                                  "CLASS", CDecl->getName(), Result);
-  
-  // Declaration of class/meta-class metadata
-  /* struct _objc_class {
-   struct _objc_class *isa; // or const char *root_class_name when metadata
-   const char *super_class_name;
-   char *name;
-   long version;
-   long info;
-   long instance_size;
-   struct _objc_ivar_list *ivars;
-   struct _objc_method_list *methods;
-   struct objc_cache *cache;
-   struct objc_protocol_list *protocols;
-   const char *ivar_layout;
-   struct _objc_class_ext  *ext;
-   };
-   */
-  static bool objc_class = false;
-  if (!objc_class) {
-    Result += "\nstruct _objc_class {\n";
-    Result += "\tstruct _objc_class *isa;\n";
-    Result += "\tconst char *super_class_name;\n";
-    Result += "\tchar *name;\n";
-    Result += "\tlong version;\n";
-    Result += "\tlong info;\n";
-    Result += "\tlong instance_size;\n";
-    Result += "\tstruct _objc_ivar_list *ivars;\n";
-    Result += "\tstruct _objc_method_list *methods;\n";
-    Result += "\tstruct objc_cache *cache;\n";
-    Result += "\tstruct _objc_protocol_list *protocols;\n";
-    Result += "\tconst char *ivar_layout;\n";
-    Result += "\tstruct _objc_class_ext  *ext;\n";
-    Result += "};\n";
-    objc_class = true;
-  }
-  
-  // Meta-class metadata generation.
-  ObjCInterfaceDecl *RootClass = 0;
-  ObjCInterfaceDecl *SuperClass = CDecl->getSuperClass();
-  while (SuperClass) {
-    RootClass = SuperClass;
-    SuperClass = SuperClass->getSuperClass();
-  }
-  SuperClass = CDecl->getSuperClass();
-  
-  Result += "\nstatic struct _objc_class _OBJC_METACLASS_";
-  Result += CDecl->getNameAsString();
-  Result += " __attribute__ ((used, section (\"__OBJC, __meta_class\")))= "
-  "{\n\t(struct _objc_class *)\"";
-  Result += (RootClass ? RootClass->getNameAsString() : CDecl->getNameAsString());
-  Result += "\"";
-  
-  if (SuperClass) {
-    Result += ", \"";
-    Result += SuperClass->getNameAsString();
-    Result += "\", \"";
-    Result += CDecl->getNameAsString();
-    Result += "\"";
-  }
-  else {
-    Result += ", 0, \"";
-    Result += CDecl->getNameAsString();
-    Result += "\"";
-  }
-  // Set 'ivars' field for root class to 0. ObjC1 runtime does not use it.
-  // 'info' field is initialized to CLS_META(2) for metaclass
-  Result += ", 0,2, sizeof(struct _objc_class), 0";
-  if (IDecl->classmeth_begin() != IDecl->classmeth_end()) {
-    Result += "\n\t, (struct _objc_method_list *)&_OBJC_CLASS_METHODS_";
-    Result += IDecl->getNameAsString();
-    Result += "\n";
-  }
-  else
-    Result += ", 0\n";
-  if (CDecl->protocol_begin() != CDecl->protocol_end()) {
-    Result += "\t,0, (struct _objc_protocol_list *)&_OBJC_CLASS_PROTOCOLS_";
-    Result += CDecl->getNameAsString();
-    Result += ",0,0\n";
-  }
-  else
-    Result += "\t,0,0,0,0\n";
-  Result += "};\n";
-  
-  // class metadata generation.
-  Result += "\nstatic struct _objc_class _OBJC_CLASS_";
-  Result += CDecl->getNameAsString();
-  Result += " __attribute__ ((used, section (\"__OBJC, __class\")))= "
-  "{\n\t&_OBJC_METACLASS_";
-  Result += CDecl->getNameAsString();
-  if (SuperClass) {
-    Result += ", \"";
-    Result += SuperClass->getNameAsString();
-    Result += "\", \"";
-    Result += CDecl->getNameAsString();
-    Result += "\"";
-  }
-  else {
-    Result += ", 0, \"";
-    Result += CDecl->getNameAsString();
-    Result += "\"";
-  }
-  // 'info' field is initialized to CLS_CLASS(1) for class
-  Result += ", 0,1";
-  if (!ObjCSynthesizedStructs.count(CDecl))
-    Result += ",0";
-  else {
-    // class has size. Must synthesize its size.
-    Result += ",sizeof(struct ";
-    Result += CDecl->getNameAsString();
-    if (LangOpts.MicrosoftExt)
-      Result += "_IMPL";
-    Result += ")";
-  }
-  if (NumIvars > 0) {
-    Result += ", (struct _objc_ivar_list *)&_OBJC_INSTANCE_VARIABLES_";
-    Result += CDecl->getNameAsString();
-    Result += "\n\t";
-  }
-  else
-    Result += ",0";
-  if (IDecl->instmeth_begin() != IDecl->instmeth_end()) {
-    Result += ", (struct _objc_method_list *)&_OBJC_INSTANCE_METHODS_";
-    Result += CDecl->getNameAsString();
-    Result += ", 0\n\t";
-  }
-  else
-    Result += ",0,0";
-  if (CDecl->protocol_begin() != CDecl->protocol_end()) {
-    Result += ", (struct _objc_protocol_list*)&_OBJC_CLASS_PROTOCOLS_";
-    Result += CDecl->getNameAsString();
-    Result += ", 0,0\n";
-  }
-  else
-    Result += ",0,0,0\n";
-  Result += "};\n";
-}
-
-void RewriteObjCFragileABI::RewriteMetaDataIntoBuffer(std::string &Result) {
-  int ClsDefCount = ClassImplementation.size();
-  int CatDefCount = CategoryImplementation.size();
-  
-  // For each implemented class, write out all its meta data.
-  for (int i = 0; i < ClsDefCount; i++)
-    RewriteObjCClassMetaData(ClassImplementation[i], Result);
-  
-  // For each implemented category, write out all its meta data.
-  for (int i = 0; i < CatDefCount; i++)
-    RewriteObjCCategoryImplDecl(CategoryImplementation[i], Result);
-  
-  // Write objc_symtab metadata
-  /*
-   struct _objc_symtab
-   {
-   long sel_ref_cnt;
-   SEL *refs;
-   short cls_def_cnt;
-   short cat_def_cnt;
-   void *defs[cls_def_cnt + cat_def_cnt];
-   };
-   */
-  
-  Result += "\nstruct _objc_symtab {\n";
-  Result += "\tlong sel_ref_cnt;\n";
-  Result += "\tSEL *refs;\n";
-  Result += "\tshort cls_def_cnt;\n";
-  Result += "\tshort cat_def_cnt;\n";
-  Result += "\tvoid *defs[" + utostr(ClsDefCount + CatDefCount)+ "];\n";
-  Result += "};\n\n";
-  
-  Result += "static struct _objc_symtab "
-  "_OBJC_SYMBOLS __attribute__((used, section (\"__OBJC, __symbols\")))= {\n";
-  Result += "\t0, 0, " + utostr(ClsDefCount)
-  + ", " + utostr(CatDefCount) + "\n";
-  for (int i = 0; i < ClsDefCount; i++) {
-    Result += "\t,&_OBJC_CLASS_";
-    Result += ClassImplementation[i]->getNameAsString();
-    Result += "\n";
-  }
-  
-  for (int i = 0; i < CatDefCount; i++) {
-    Result += "\t,&_OBJC_CATEGORY_";
-    Result += CategoryImplementation[i]->getClassInterface()->getNameAsString();
-    Result += "_";
-    Result += CategoryImplementation[i]->getNameAsString();
-    Result += "\n";
-  }
-  
-  Result += "};\n\n";
-  
-  // Write objc_module metadata
-  
-  /*
-   struct _objc_module {
-   long version;
-   long size;
-   const char *name;
-   struct _objc_symtab *symtab;
-   }
-   */
-  
-  Result += "\nstruct _objc_module {\n";
-  Result += "\tlong version;\n";
-  Result += "\tlong size;\n";
-  Result += "\tconst char *name;\n";
-  Result += "\tstruct _objc_symtab *symtab;\n";
-  Result += "};\n\n";
-  Result += "static struct _objc_module "
-  "_OBJC_MODULES __attribute__ ((used, section (\"__OBJC, __module_info\")))= {\n";
-  Result += "\t" + utostr(OBJC_ABI_VERSION) +
-  ", sizeof(struct _objc_module), \"\", &_OBJC_SYMBOLS\n";
-  Result += "};\n\n";
-  
-  if (LangOpts.MicrosoftExt) {
-    if (ProtocolExprDecls.size()) {
-      Result += "#pragma section(\".objc_protocol$B\",long,read,write)\n";
-      Result += "#pragma data_seg(push, \".objc_protocol$B\")\n";
-      for (llvm::SmallPtrSet<ObjCProtocolDecl *,8>::iterator I = ProtocolExprDecls.begin(),
-           E = ProtocolExprDecls.end(); I != E; ++I) {
-        Result += "static struct _objc_protocol *_POINTER_OBJC_PROTOCOL_";
-        Result += (*I)->getNameAsString();
-        Result += " = &_OBJC_PROTOCOL_";
-        Result += (*I)->getNameAsString();
-        Result += ";\n";
-      }
-      Result += "#pragma data_seg(pop)\n\n";
-    }
-    Result += "#pragma section(\".objc_module_info$B\",long,read,write)\n";
-    Result += "#pragma data_seg(push, \".objc_module_info$B\")\n";
-    Result += "static struct _objc_module *_POINTER_OBJC_MODULES = ";
-    Result += "&_OBJC_MODULES;\n";
-    Result += "#pragma data_seg(pop)\n\n";
-  }
-}
-
-/// RewriteObjCCategoryImplDecl - Rewrite metadata for each category
-/// implementation.
-void RewriteObjCFragileABI::RewriteObjCCategoryImplDecl(ObjCCategoryImplDecl *IDecl,
-                                              std::string &Result) {
-  ObjCInterfaceDecl *ClassDecl = IDecl->getClassInterface();
-  // Find category declaration for this implementation.
-  ObjCCategoryDecl *CDecl;
-  for (CDecl = ClassDecl->getCategoryList(); CDecl;
-       CDecl = CDecl->getNextClassCategory())
-    if (CDecl->getIdentifier() == IDecl->getIdentifier())
-      break;
-  
-  std::string FullCategoryName = ClassDecl->getNameAsString();
-  FullCategoryName += '_';
-  FullCategoryName += IDecl->getNameAsString();
-  
-  // Build _objc_method_list for class's instance methods if needed
-  SmallVector<ObjCMethodDecl *, 32>
-  InstanceMethods(IDecl->instmeth_begin(), IDecl->instmeth_end());
-  
-  // If any of our property implementations have associated getters or
-  // setters, produce metadata for them as well.
-  for (ObjCImplDecl::propimpl_iterator Prop = IDecl->propimpl_begin(),
-       PropEnd = IDecl->propimpl_end();
-       Prop != PropEnd; ++Prop) {
-    if (Prop->getPropertyImplementation() == ObjCPropertyImplDecl::Dynamic)
-      continue;
-    if (!Prop->getPropertyIvarDecl())
-      continue;
-    ObjCPropertyDecl *PD = Prop->getPropertyDecl();
-    if (!PD)
-      continue;
-    if (ObjCMethodDecl *Getter = PD->getGetterMethodDecl())
-      InstanceMethods.push_back(Getter);
-    if (PD->isReadOnly())
-      continue;
-    if (ObjCMethodDecl *Setter = PD->getSetterMethodDecl())
-      InstanceMethods.push_back(Setter);
-  }
-  RewriteObjCMethodsMetaData(InstanceMethods.begin(), InstanceMethods.end(),
-                             true, "CATEGORY_", FullCategoryName.c_str(),
-                             Result);
-  
-  // Build _objc_method_list for class's class methods if needed
-  RewriteObjCMethodsMetaData(IDecl->classmeth_begin(), IDecl->classmeth_end(),
-                             false, "CATEGORY_", FullCategoryName.c_str(),
-                             Result);
-  
-  // Protocols referenced in class declaration?
-  // Null CDecl is case of a category implementation with no category interface
-  if (CDecl)
-    RewriteObjCProtocolListMetaData(CDecl->getReferencedProtocols(), "CATEGORY",
-                                    FullCategoryName, Result);
-  /* struct _objc_category {
-   char *category_name;
-   char *class_name;
-   struct _objc_method_list *instance_methods;
-   struct _objc_method_list *class_methods;
-   struct _objc_protocol_list *protocols;
-   // Objective-C 1.0 extensions
-   uint32_t size;     // sizeof (struct _objc_category)
-   struct _objc_property_list *instance_properties;  // category's own
-   // @property decl.
-   };
-   */
-  
-  static bool objc_category = false;
-  if (!objc_category) {
-    Result += "\nstruct _objc_category {\n";
-    Result += "\tchar *category_name;\n";
-    Result += "\tchar *class_name;\n";
-    Result += "\tstruct _objc_method_list *instance_methods;\n";
-    Result += "\tstruct _objc_method_list *class_methods;\n";
-    Result += "\tstruct _objc_protocol_list *protocols;\n";
-    Result += "\tunsigned int size;\n";
-    Result += "\tstruct _objc_property_list *instance_properties;\n";
-    Result += "};\n";
-    objc_category = true;
-  }
-  Result += "\nstatic struct _objc_category _OBJC_CATEGORY_";
-  Result += FullCategoryName;
-  Result += " __attribute__ ((used, section (\"__OBJC, __category\")))= {\n\t\"";
-  Result += IDecl->getNameAsString();
-  Result += "\"\n\t, \"";
-  Result += ClassDecl->getNameAsString();
-  Result += "\"\n";
-  
-  if (IDecl->instmeth_begin() != IDecl->instmeth_end()) {
-    Result += "\t, (struct _objc_method_list *)"
-    "&_OBJC_CATEGORY_INSTANCE_METHODS_";
-    Result += FullCategoryName;
-    Result += "\n";
-  }
-  else
-    Result += "\t, 0\n";
-  if (IDecl->classmeth_begin() != IDecl->classmeth_end()) {
-    Result += "\t, (struct _objc_method_list *)"
-    "&_OBJC_CATEGORY_CLASS_METHODS_";
-    Result += FullCategoryName;
-    Result += "\n";
-  }
-  else
-    Result += "\t, 0\n";
-  
-  if (CDecl && CDecl->protocol_begin() != CDecl->protocol_end()) {
-    Result += "\t, (struct _objc_protocol_list *)&_OBJC_CATEGORY_PROTOCOLS_";
-    Result += FullCategoryName;
-    Result += "\n";
-  }
-  else
-    Result += "\t, 0\n";
-  Result += "\t, sizeof(struct _objc_category), 0\n};\n";
-}
-
-// RewriteObjCMethodsMetaData - Rewrite methods metadata for instance or
-/// class methods.
-template<typename MethodIterator>
-void RewriteObjCFragileABI::RewriteObjCMethodsMetaData(MethodIterator MethodBegin,
-                                             MethodIterator MethodEnd,
-                                             bool IsInstanceMethod,
-                                             StringRef prefix,
-                                             StringRef ClassName,
-                                             std::string &Result) {
-  if (MethodBegin == MethodEnd) return;
-  
-  if (!objc_impl_method) {
-    /* struct _objc_method {
-     SEL _cmd;
-     char *method_types;
-     void *_imp;
-     }
-     */
-    Result += "\nstruct _objc_method {\n";
-    Result += "\tSEL _cmd;\n";
-    Result += "\tchar *method_types;\n";
-    Result += "\tvoid *_imp;\n";
-    Result += "};\n";
-    
-    objc_impl_method = true;
-  }
-  
-  // Build _objc_method_list for class's methods if needed
-  
-  /* struct  {
-   struct _objc_method_list *next_method;
-   int method_count;
-   struct _objc_method method_list[];
-   }
-   */
-  unsigned NumMethods = std::distance(MethodBegin, MethodEnd);
-  Result += "\nstatic struct {\n";
-  Result += "\tstruct _objc_method_list *next_method;\n";
-  Result += "\tint method_count;\n";
-  Result += "\tstruct _objc_method method_list[";
-  Result += utostr(NumMethods);
-  Result += "];\n} _OBJC_";
-  Result += prefix;
-  Result += IsInstanceMethod ? "INSTANCE" : "CLASS";
-  Result += "_METHODS_";
-  Result += ClassName;
-  Result += " __attribute__ ((used, section (\"__OBJC, __";
-  Result += IsInstanceMethod ? "inst" : "cls";
-  Result += "_meth\")))= ";
-  Result += "{\n\t0, " + utostr(NumMethods) + "\n";
-  
-  Result += "\t,{{(SEL)\"";
-  Result += (*MethodBegin)->getSelector().getAsString().c_str();
-  std::string MethodTypeString;
-  Context->getObjCEncodingForMethodDecl(*MethodBegin, MethodTypeString);
-  Result += "\", \"";
-  Result += MethodTypeString;
-  Result += "\", (void *)";
-  Result += MethodInternalNames[*MethodBegin];
-  Result += "}\n";
-  for (++MethodBegin; MethodBegin != MethodEnd; ++MethodBegin) {
-    Result += "\t  ,{(SEL)\"";
-    Result += (*MethodBegin)->getSelector().getAsString().c_str();
-    std::string MethodTypeString;
-    Context->getObjCEncodingForMethodDecl(*MethodBegin, MethodTypeString);
-    Result += "\", \"";
-    Result += MethodTypeString;
-    Result += "\", (void *)";
-    Result += MethodInternalNames[*MethodBegin];
-    Result += "}\n";
-  }
-  Result += "\t }\n};\n";
-}
-
-Stmt *RewriteObjCFragileABI::RewriteObjCIvarRefExpr(ObjCIvarRefExpr *IV) {
-  SourceRange OldRange = IV->getSourceRange();
-  Expr *BaseExpr = IV->getBase();
-  
-  // Rewrite the base, but without actually doing replaces.
-  {
-    DisableReplaceStmtScope S(*this);
-    BaseExpr = cast<Expr>(RewriteFunctionBodyOrGlobalInitializer(BaseExpr));
-    IV->setBase(BaseExpr);
-  }
-  
-  ObjCIvarDecl *D = IV->getDecl();
-  
-  Expr *Replacement = IV;
-  if (CurMethodDef) {
-    if (BaseExpr->getType()->isObjCObjectPointerType()) {
-      const ObjCInterfaceType *iFaceDecl =
-      dyn_cast<ObjCInterfaceType>(BaseExpr->getType()->getPointeeType());
-      assert(iFaceDecl && "RewriteObjCIvarRefExpr - iFaceDecl is null");
-      // lookup which class implements the instance variable.
-      ObjCInterfaceDecl *clsDeclared = 0;
-      iFaceDecl->getDecl()->lookupInstanceVariable(D->getIdentifier(),
-                                                   clsDeclared);
-      assert(clsDeclared && "RewriteObjCIvarRefExpr(): Can't find class");
-      
-      // Synthesize an explicit cast to gain access to the ivar.
-      std::string RecName = clsDeclared->getIdentifier()->getName();
-      RecName += "_IMPL";
-      IdentifierInfo *II = &Context->Idents.get(RecName);
-      RecordDecl *RD = RecordDecl::Create(*Context, TTK_Struct, TUDecl,
-                                          SourceLocation(), SourceLocation(),
-                                          II);
-      assert(RD && "RewriteObjCIvarRefExpr(): Can't find RecordDecl");
-      QualType castT = Context->getPointerType(Context->getTagDeclType(RD));
-      CastExpr *castExpr = NoTypeInfoCStyleCastExpr(Context, castT,
-                                                    CK_BitCast,
-                                                    IV->getBase());
-      // Don't forget the parens to enforce the proper binding.
-      ParenExpr *PE = new (Context) ParenExpr(OldRange.getBegin(),
-                                              OldRange.getEnd(),
-                                              castExpr);
-      if (IV->isFreeIvar() &&
-          declaresSameEntity(CurMethodDef->getClassInterface(), iFaceDecl->getDecl())) {
-        MemberExpr *ME = new (Context) MemberExpr(PE, true, D,
-                                                  IV->getLocation(),
-                                                  D->getType(),
-                                                  VK_LValue, OK_Ordinary);
-        Replacement = ME;
-      } else {
-        IV->setBase(PE);
-      }
-    }
-  } else { // we are outside a method.
-    assert(!IV->isFreeIvar() && "Cannot have a free standing ivar outside a method");
-    
-    // Explicit ivar refs need to have a cast inserted.
-    // FIXME: consider sharing some of this code with the code above.
-    if (BaseExpr->getType()->isObjCObjectPointerType()) {
-      const ObjCInterfaceType *iFaceDecl =
-      dyn_cast<ObjCInterfaceType>(BaseExpr->getType()->getPointeeType());
-      // lookup which class implements the instance variable.
-      ObjCInterfaceDecl *clsDeclared = 0;
-      iFaceDecl->getDecl()->lookupInstanceVariable(D->getIdentifier(),
-                                                   clsDeclared);
-      assert(clsDeclared && "RewriteObjCIvarRefExpr(): Can't find class");
-      
-      // Synthesize an explicit cast to gain access to the ivar.
-      std::string RecName = clsDeclared->getIdentifier()->getName();
-      RecName += "_IMPL";
-      IdentifierInfo *II = &Context->Idents.get(RecName);
-      RecordDecl *RD = RecordDecl::Create(*Context, TTK_Struct, TUDecl,
-                                          SourceLocation(), SourceLocation(),
-                                          II);
-      assert(RD && "RewriteObjCIvarRefExpr(): Can't find RecordDecl");
-      QualType castT = Context->getPointerType(Context->getTagDeclType(RD));
-      CastExpr *castExpr = NoTypeInfoCStyleCastExpr(Context, castT,
-                                                    CK_BitCast,
-                                                    IV->getBase());
-      // Don't forget the parens to enforce the proper binding.
-      ParenExpr *PE = new (Context) ParenExpr(IV->getBase()->getLocStart(),
-                                              IV->getBase()->getLocEnd(), castExpr);
-      // Cannot delete IV->getBase(), since PE points to it.
-      // Replace the old base with the cast. This is important when doing
-      // embedded rewrites. For example, [newInv->_container addObject:0].
-      IV->setBase(PE);
-    }
-  }
-  
-  ReplaceStmtWithRange(IV, Replacement, OldRange);
-  return Replacement;  
-}
diff --git a/lib/Rewrite/RewriteRope.cpp b/lib/Rewrite/RewriteRope.cpp
deleted file mode 100644
index cc8de1b..0000000
--- a/lib/Rewrite/RewriteRope.cpp
+++ /dev/null
@@ -1,811 +0,0 @@
-//===--- RewriteRope.cpp - Rope specialized for rewriter --------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//  This file implements the RewriteRope class, which is a powerful string.
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/Rewrite/RewriteRope.h"
-#include "clang/Basic/LLVM.h"
-#include <algorithm>
-using namespace clang;
-
-/// RewriteRope is a "strong" string class, designed to make insertions and
-/// deletions in the middle of the string nearly constant time (really, they are
-/// O(log N), but with a very low constant factor).
-///
-/// The implementation of this datastructure is a conceptual linear sequence of
-/// RopePiece elements.  Each RopePiece represents a view on a separately
-/// allocated and reference counted string.  This means that splitting a very
-/// long string can be done in constant time by splitting a RopePiece that
-/// references the whole string into two rope pieces that reference each half.
-/// Once split, another string can be inserted in between the two halves by
-/// inserting a RopePiece in between the two others.  All of this is very
-/// inexpensive: it takes time proportional to the number of RopePieces, not the
-/// length of the strings they represent.
-///
-/// While a linear sequences of RopePieces is the conceptual model, the actual
-/// implementation captures them in an adapted B+ Tree.  Using a B+ tree (which
-/// is a tree that keeps the values in the leaves and has where each node
-/// contains a reasonable number of pointers to children/values) allows us to
-/// maintain efficient operation when the RewriteRope contains a *huge* number
-/// of RopePieces.  The basic idea of the B+ Tree is that it allows us to find
-/// the RopePiece corresponding to some offset very efficiently, and it
-/// automatically balances itself on insertions of RopePieces (which can happen
-/// for both insertions and erases of string ranges).
-///
-/// The one wrinkle on the theory is that we don't attempt to keep the tree
-/// properly balanced when erases happen.  Erases of string data can both insert
-/// new RopePieces (e.g. when the middle of some other rope piece is deleted,
-/// which results in two rope pieces, which is just like an insert) or it can
-/// reduce the number of RopePieces maintained by the B+Tree.  In the case when
-/// the number of RopePieces is reduced, we don't attempt to maintain the
-/// standard 'invariant' that each node in the tree contains at least
-/// 'WidthFactor' children/values.  For our use cases, this doesn't seem to
-/// matter.
-///
-/// The implementation below is primarily implemented in terms of three classes:
-///   RopePieceBTreeNode - Common base class for:
-///
-///     RopePieceBTreeLeaf - Directly manages up to '2*WidthFactor' RopePiece
-///          nodes.  This directly represents a chunk of the string with those
-///          RopePieces contatenated.
-///     RopePieceBTreeInterior - An interior node in the B+ Tree, which manages
-///          up to '2*WidthFactor' other nodes in the tree.
-
-
-//===----------------------------------------------------------------------===//
-// RopePieceBTreeNode Class
-//===----------------------------------------------------------------------===//
-
-namespace {
-  /// RopePieceBTreeNode - Common base class of RopePieceBTreeLeaf and
-  /// RopePieceBTreeInterior.  This provides some 'virtual' dispatching methods
-  /// and a flag that determines which subclass the instance is.  Also
-  /// important, this node knows the full extend of the node, including any
-  /// children that it has.  This allows efficient skipping over entire subtrees
-  /// when looking for an offset in the BTree.
-  class RopePieceBTreeNode {
-  protected:
-    /// WidthFactor - This controls the number of K/V slots held in the BTree:
-    /// how wide it is.  Each level of the BTree is guaranteed to have at least
-    /// 'WidthFactor' elements in it (either ropepieces or children), (except
-    /// the root, which may have less) and may have at most 2*WidthFactor
-    /// elements.
-    enum { WidthFactor = 8 };
-
-    /// Size - This is the number of bytes of file this node (including any
-    /// potential children) covers.
-    unsigned Size;
-
-    /// IsLeaf - True if this is an instance of RopePieceBTreeLeaf, false if it
-    /// is an instance of RopePieceBTreeInterior.
-    bool IsLeaf;
-
-    RopePieceBTreeNode(bool isLeaf) : Size(0), IsLeaf(isLeaf) {}
-    ~RopePieceBTreeNode() {}
-  public:
-
-    bool isLeaf() const { return IsLeaf; }
-    unsigned size() const { return Size; }
-
-    void Destroy();
-
-    /// split - Split the range containing the specified offset so that we are
-    /// guaranteed that there is a place to do an insertion at the specified
-    /// offset.  The offset is relative, so "0" is the start of the node.
-    ///
-    /// If there is no space in this subtree for the extra piece, the extra tree
-    /// node is returned and must be inserted into a parent.
-    RopePieceBTreeNode *split(unsigned Offset);
-
-    /// insert - Insert the specified ropepiece into this tree node at the
-    /// specified offset.  The offset is relative, so "0" is the start of the
-    /// node.
-    ///
-    /// If there is no space in this subtree for the extra piece, the extra tree
-    /// node is returned and must be inserted into a parent.
-    RopePieceBTreeNode *insert(unsigned Offset, const RopePiece &R);
-
-    /// erase - Remove NumBytes from this node at the specified offset.  We are
-    /// guaranteed that there is a split at Offset.
-    void erase(unsigned Offset, unsigned NumBytes);
-
-    //static inline bool classof(const RopePieceBTreeNode *) { return true; }
-
-  };
-} // end anonymous namespace
-
-//===----------------------------------------------------------------------===//
-// RopePieceBTreeLeaf Class
-//===----------------------------------------------------------------------===//
-
-namespace {
-  /// RopePieceBTreeLeaf - Directly manages up to '2*WidthFactor' RopePiece
-  /// nodes.  This directly represents a chunk of the string with those
-  /// RopePieces contatenated.  Since this is a B+Tree, all values (in this case
-  /// instances of RopePiece) are stored in leaves like this.  To make iteration
-  /// over the leaves efficient, they maintain a singly linked list through the
-  /// NextLeaf field.  This allows the B+Tree forward iterator to be constant
-  /// time for all increments.
-  class RopePieceBTreeLeaf : public RopePieceBTreeNode {
-    /// NumPieces - This holds the number of rope pieces currently active in the
-    /// Pieces array.
-    unsigned char NumPieces;
-
-    /// Pieces - This tracks the file chunks currently in this leaf.
-    ///
-    RopePiece Pieces[2*WidthFactor];
-
-    /// NextLeaf - This is a pointer to the next leaf in the tree, allowing
-    /// efficient in-order forward iteration of the tree without traversal.
-    RopePieceBTreeLeaf **PrevLeaf, *NextLeaf;
-  public:
-    RopePieceBTreeLeaf() : RopePieceBTreeNode(true), NumPieces(0),
-                           PrevLeaf(0), NextLeaf(0) {}
-    ~RopePieceBTreeLeaf() {
-      if (PrevLeaf || NextLeaf)
-        removeFromLeafInOrder();
-      clear();
-    }
-
-    bool isFull() const { return NumPieces == 2*WidthFactor; }
-
-    /// clear - Remove all rope pieces from this leaf.
-    void clear() {
-      while (NumPieces)
-        Pieces[--NumPieces] = RopePiece();
-      Size = 0;
-    }
-
-    unsigned getNumPieces() const { return NumPieces; }
-
-    const RopePiece &getPiece(unsigned i) const {
-      assert(i < getNumPieces() && "Invalid piece ID");
-      return Pieces[i];
-    }
-
-    const RopePieceBTreeLeaf *getNextLeafInOrder() const { return NextLeaf; }
-    void insertAfterLeafInOrder(RopePieceBTreeLeaf *Node) {
-      assert(PrevLeaf == 0 && NextLeaf == 0 && "Already in ordering");
-
-      NextLeaf = Node->NextLeaf;
-      if (NextLeaf)
-        NextLeaf->PrevLeaf = &NextLeaf;
-      PrevLeaf = &Node->NextLeaf;
-      Node->NextLeaf = this;
-    }
-
-    void removeFromLeafInOrder() {
-      if (PrevLeaf) {
-        *PrevLeaf = NextLeaf;
-        if (NextLeaf)
-          NextLeaf->PrevLeaf = PrevLeaf;
-      } else if (NextLeaf) {
-        NextLeaf->PrevLeaf = 0;
-      }
-    }
-
-    /// FullRecomputeSizeLocally - This method recomputes the 'Size' field by
-    /// summing the size of all RopePieces.
-    void FullRecomputeSizeLocally() {
-      Size = 0;
-      for (unsigned i = 0, e = getNumPieces(); i != e; ++i)
-        Size += getPiece(i).size();
-    }
-
-    /// split - Split the range containing the specified offset so that we are
-    /// guaranteed that there is a place to do an insertion at the specified
-    /// offset.  The offset is relative, so "0" is the start of the node.
-    ///
-    /// If there is no space in this subtree for the extra piece, the extra tree
-    /// node is returned and must be inserted into a parent.
-    RopePieceBTreeNode *split(unsigned Offset);
-
-    /// insert - Insert the specified ropepiece into this tree node at the
-    /// specified offset.  The offset is relative, so "0" is the start of the
-    /// node.
-    ///
-    /// If there is no space in this subtree for the extra piece, the extra tree
-    /// node is returned and must be inserted into a parent.
-    RopePieceBTreeNode *insert(unsigned Offset, const RopePiece &R);
-
-
-    /// erase - Remove NumBytes from this node at the specified offset.  We are
-    /// guaranteed that there is a split at Offset.
-    void erase(unsigned Offset, unsigned NumBytes);
-
-    //static inline bool classof(const RopePieceBTreeLeaf *) { return true; }
-    static inline bool classof(const RopePieceBTreeNode *N) {
-      return N->isLeaf();
-    }
-  };
-} // end anonymous namespace
-
-/// split - Split the range containing the specified offset so that we are
-/// guaranteed that there is a place to do an insertion at the specified
-/// offset.  The offset is relative, so "0" is the start of the node.
-///
-/// If there is no space in this subtree for the extra piece, the extra tree
-/// node is returned and must be inserted into a parent.
-RopePieceBTreeNode *RopePieceBTreeLeaf::split(unsigned Offset) {
-  // Find the insertion point.  We are guaranteed that there is a split at the
-  // specified offset so find it.
-  if (Offset == 0 || Offset == size()) {
-    // Fastpath for a common case.  There is already a splitpoint at the end.
-    return 0;
-  }
-
-  // Find the piece that this offset lands in.
-  unsigned PieceOffs = 0;
-  unsigned i = 0;
-  while (Offset >= PieceOffs+Pieces[i].size()) {
-    PieceOffs += Pieces[i].size();
-    ++i;
-  }
-
-  // If there is already a split point at the specified offset, just return
-  // success.
-  if (PieceOffs == Offset)
-    return 0;
-
-  // Otherwise, we need to split piece 'i' at Offset-PieceOffs.  Convert Offset
-  // to being Piece relative.
-  unsigned IntraPieceOffset = Offset-PieceOffs;
-
-  // We do this by shrinking the RopePiece and then doing an insert of the tail.
-  RopePiece Tail(Pieces[i].StrData, Pieces[i].StartOffs+IntraPieceOffset,
-                 Pieces[i].EndOffs);
-  Size -= Pieces[i].size();
-  Pieces[i].EndOffs = Pieces[i].StartOffs+IntraPieceOffset;
-  Size += Pieces[i].size();
-
-  return insert(Offset, Tail);
-}
-
-
-/// insert - Insert the specified RopePiece into this tree node at the
-/// specified offset.  The offset is relative, so "0" is the start of the node.
-///
-/// If there is no space in this subtree for the extra piece, the extra tree
-/// node is returned and must be inserted into a parent.
-RopePieceBTreeNode *RopePieceBTreeLeaf::insert(unsigned Offset,
-                                               const RopePiece &R) {
-  // If this node is not full, insert the piece.
-  if (!isFull()) {
-    // Find the insertion point.  We are guaranteed that there is a split at the
-    // specified offset so find it.
-    unsigned i = 0, e = getNumPieces();
-    if (Offset == size()) {
-      // Fastpath for a common case.
-      i = e;
-    } else {
-      unsigned SlotOffs = 0;
-      for (; Offset > SlotOffs; ++i)
-        SlotOffs += getPiece(i).size();
-      assert(SlotOffs == Offset && "Split didn't occur before insertion!");
-    }
-
-    // For an insertion into a non-full leaf node, just insert the value in
-    // its sorted position.  This requires moving later values over.
-    for (; i != e; --e)
-      Pieces[e] = Pieces[e-1];
-    Pieces[i] = R;
-    ++NumPieces;
-    Size += R.size();
-    return 0;
-  }
-
-  // Otherwise, if this is leaf is full, split it in two halves.  Since this
-  // node is full, it contains 2*WidthFactor values.  We move the first
-  // 'WidthFactor' values to the LHS child (which we leave in this node) and
-  // move the last 'WidthFactor' values into the RHS child.
-
-  // Create the new node.
-  RopePieceBTreeLeaf *NewNode = new RopePieceBTreeLeaf();
-
-  // Move over the last 'WidthFactor' values from here to NewNode.
-  std::copy(&Pieces[WidthFactor], &Pieces[2*WidthFactor],
-            &NewNode->Pieces[0]);
-  // Replace old pieces with null RopePieces to drop refcounts.
-  std::fill(&Pieces[WidthFactor], &Pieces[2*WidthFactor], RopePiece());
-
-  // Decrease the number of values in the two nodes.
-  NewNode->NumPieces = NumPieces = WidthFactor;
-
-  // Recompute the two nodes' size.
-  NewNode->FullRecomputeSizeLocally();
-  FullRecomputeSizeLocally();
-
-  // Update the list of leaves.
-  NewNode->insertAfterLeafInOrder(this);
-
-  // These insertions can't fail.
-  if (this->size() >= Offset)
-    this->insert(Offset, R);
-  else
-    NewNode->insert(Offset - this->size(), R);
-  return NewNode;
-}
-
-/// erase - Remove NumBytes from this node at the specified offset.  We are
-/// guaranteed that there is a split at Offset.
-void RopePieceBTreeLeaf::erase(unsigned Offset, unsigned NumBytes) {
-  // Since we are guaranteed that there is a split at Offset, we start by
-  // finding the Piece that starts there.
-  unsigned PieceOffs = 0;
-  unsigned i = 0;
-  for (; Offset > PieceOffs; ++i)
-    PieceOffs += getPiece(i).size();
-  assert(PieceOffs == Offset && "Split didn't occur before erase!");
-
-  unsigned StartPiece = i;
-
-  // Figure out how many pieces completely cover 'NumBytes'.  We want to remove
-  // all of them.
-  for (; Offset+NumBytes > PieceOffs+getPiece(i).size(); ++i)
-    PieceOffs += getPiece(i).size();
-
-  // If we exactly include the last one, include it in the region to delete.
-  if (Offset+NumBytes == PieceOffs+getPiece(i).size())
-    PieceOffs += getPiece(i).size(), ++i;
-
-  // If we completely cover some RopePieces, erase them now.
-  if (i != StartPiece) {
-    unsigned NumDeleted = i-StartPiece;
-    for (; i != getNumPieces(); ++i)
-      Pieces[i-NumDeleted] = Pieces[i];
-
-    // Drop references to dead rope pieces.
-    std::fill(&Pieces[getNumPieces()-NumDeleted], &Pieces[getNumPieces()],
-              RopePiece());
-    NumPieces -= NumDeleted;
-
-    unsigned CoverBytes = PieceOffs-Offset;
-    NumBytes -= CoverBytes;
-    Size -= CoverBytes;
-  }
-
-  // If we completely removed some stuff, we could be done.
-  if (NumBytes == 0) return;
-
-  // Okay, now might be erasing part of some Piece.  If this is the case, then
-  // move the start point of the piece.
-  assert(getPiece(StartPiece).size() > NumBytes);
-  Pieces[StartPiece].StartOffs += NumBytes;
-
-  // The size of this node just shrunk by NumBytes.
-  Size -= NumBytes;
-}
-
-//===----------------------------------------------------------------------===//
-// RopePieceBTreeInterior Class
-//===----------------------------------------------------------------------===//
-
-namespace {
-  /// RopePieceBTreeInterior - This represents an interior node in the B+Tree,
-  /// which holds up to 2*WidthFactor pointers to child nodes.
-  class RopePieceBTreeInterior : public RopePieceBTreeNode {
-    /// NumChildren - This holds the number of children currently active in the
-    /// Children array.
-    unsigned char NumChildren;
-    RopePieceBTreeNode *Children[2*WidthFactor];
-  public:
-    RopePieceBTreeInterior() : RopePieceBTreeNode(false), NumChildren(0) {}
-
-    RopePieceBTreeInterior(RopePieceBTreeNode *LHS, RopePieceBTreeNode *RHS)
-    : RopePieceBTreeNode(false) {
-      Children[0] = LHS;
-      Children[1] = RHS;
-      NumChildren = 2;
-      Size = LHS->size() + RHS->size();
-    }
-
-    ~RopePieceBTreeInterior() {
-      for (unsigned i = 0, e = getNumChildren(); i != e; ++i)
-        Children[i]->Destroy();
-    }
-
-    bool isFull() const { return NumChildren == 2*WidthFactor; }
-
-    unsigned getNumChildren() const { return NumChildren; }
-    const RopePieceBTreeNode *getChild(unsigned i) const {
-      assert(i < NumChildren && "invalid child #");
-      return Children[i];
-    }
-    RopePieceBTreeNode *getChild(unsigned i) {
-      assert(i < NumChildren && "invalid child #");
-      return Children[i];
-    }
-
-    /// FullRecomputeSizeLocally - Recompute the Size field of this node by
-    /// summing up the sizes of the child nodes.
-    void FullRecomputeSizeLocally() {
-      Size = 0;
-      for (unsigned i = 0, e = getNumChildren(); i != e; ++i)
-        Size += getChild(i)->size();
-    }
-
-
-    /// split - Split the range containing the specified offset so that we are
-    /// guaranteed that there is a place to do an insertion at the specified
-    /// offset.  The offset is relative, so "0" is the start of the node.
-    ///
-    /// If there is no space in this subtree for the extra piece, the extra tree
-    /// node is returned and must be inserted into a parent.
-    RopePieceBTreeNode *split(unsigned Offset);
-
-
-    /// insert - Insert the specified ropepiece into this tree node at the
-    /// specified offset.  The offset is relative, so "0" is the start of the
-    /// node.
-    ///
-    /// If there is no space in this subtree for the extra piece, the extra tree
-    /// node is returned and must be inserted into a parent.
-    RopePieceBTreeNode *insert(unsigned Offset, const RopePiece &R);
-
-    /// HandleChildPiece - A child propagated an insertion result up to us.
-    /// Insert the new child, and/or propagate the result further up the tree.
-    RopePieceBTreeNode *HandleChildPiece(unsigned i, RopePieceBTreeNode *RHS);
-
-    /// erase - Remove NumBytes from this node at the specified offset.  We are
-    /// guaranteed that there is a split at Offset.
-    void erase(unsigned Offset, unsigned NumBytes);
-
-    //static inline bool classof(const RopePieceBTreeInterior *) { return true; }
-    static inline bool classof(const RopePieceBTreeNode *N) {
-      return !N->isLeaf();
-    }
-  };
-} // end anonymous namespace
-
-/// split - Split the range containing the specified offset so that we are
-/// guaranteed that there is a place to do an insertion at the specified
-/// offset.  The offset is relative, so "0" is the start of the node.
-///
-/// If there is no space in this subtree for the extra piece, the extra tree
-/// node is returned and must be inserted into a parent.
-RopePieceBTreeNode *RopePieceBTreeInterior::split(unsigned Offset) {
-  // Figure out which child to split.
-  if (Offset == 0 || Offset == size())
-    return 0;  // If we have an exact offset, we're already split.
-
-  unsigned ChildOffset = 0;
-  unsigned i = 0;
-  for (; Offset >= ChildOffset+getChild(i)->size(); ++i)
-    ChildOffset += getChild(i)->size();
-
-  // If already split there, we're done.
-  if (ChildOffset == Offset)
-    return 0;
-
-  // Otherwise, recursively split the child.
-  if (RopePieceBTreeNode *RHS = getChild(i)->split(Offset-ChildOffset))
-    return HandleChildPiece(i, RHS);
-  return 0;  // Done!
-}
-
-/// insert - Insert the specified ropepiece into this tree node at the
-/// specified offset.  The offset is relative, so "0" is the start of the
-/// node.
-///
-/// If there is no space in this subtree for the extra piece, the extra tree
-/// node is returned and must be inserted into a parent.
-RopePieceBTreeNode *RopePieceBTreeInterior::insert(unsigned Offset,
-                                                   const RopePiece &R) {
-  // Find the insertion point.  We are guaranteed that there is a split at the
-  // specified offset so find it.
-  unsigned i = 0, e = getNumChildren();
-
-  unsigned ChildOffs = 0;
-  if (Offset == size()) {
-    // Fastpath for a common case.  Insert at end of last child.
-    i = e-1;
-    ChildOffs = size()-getChild(i)->size();
-  } else {
-    for (; Offset > ChildOffs+getChild(i)->size(); ++i)
-      ChildOffs += getChild(i)->size();
-  }
-
-  Size += R.size();
-
-  // Insert at the end of this child.
-  if (RopePieceBTreeNode *RHS = getChild(i)->insert(Offset-ChildOffs, R))
-    return HandleChildPiece(i, RHS);
-
-  return 0;
-}
-
-/// HandleChildPiece - A child propagated an insertion result up to us.
-/// Insert the new child, and/or propagate the result further up the tree.
-RopePieceBTreeNode *
-RopePieceBTreeInterior::HandleChildPiece(unsigned i, RopePieceBTreeNode *RHS) {
-  // Otherwise the child propagated a subtree up to us as a new child.  See if
-  // we have space for it here.
-  if (!isFull()) {
-    // Insert RHS after child 'i'.
-    if (i + 1 != getNumChildren())
-      memmove(&Children[i+2], &Children[i+1],
-              (getNumChildren()-i-1)*sizeof(Children[0]));
-    Children[i+1] = RHS;
-    ++NumChildren;
-    return 0;
-  }
-
-  // Okay, this node is full.  Split it in half, moving WidthFactor children to
-  // a newly allocated interior node.
-
-  // Create the new node.
-  RopePieceBTreeInterior *NewNode = new RopePieceBTreeInterior();
-
-  // Move over the last 'WidthFactor' values from here to NewNode.
-  memcpy(&NewNode->Children[0], &Children[WidthFactor],
-         WidthFactor*sizeof(Children[0]));
-
-  // Decrease the number of values in the two nodes.
-  NewNode->NumChildren = NumChildren = WidthFactor;
-
-  // Finally, insert the two new children in the side the can (now) hold them.
-  // These insertions can't fail.
-  if (i < WidthFactor)
-    this->HandleChildPiece(i, RHS);
-  else
-    NewNode->HandleChildPiece(i-WidthFactor, RHS);
-
-  // Recompute the two nodes' size.
-  NewNode->FullRecomputeSizeLocally();
-  FullRecomputeSizeLocally();
-  return NewNode;
-}
-
-/// erase - Remove NumBytes from this node at the specified offset.  We are
-/// guaranteed that there is a split at Offset.
-void RopePieceBTreeInterior::erase(unsigned Offset, unsigned NumBytes) {
-  // This will shrink this node by NumBytes.
-  Size -= NumBytes;
-
-  // Find the first child that overlaps with Offset.
-  unsigned i = 0;
-  for (; Offset >= getChild(i)->size(); ++i)
-    Offset -= getChild(i)->size();
-
-  // Propagate the delete request into overlapping children, or completely
-  // delete the children as appropriate.
-  while (NumBytes) {
-    RopePieceBTreeNode *CurChild = getChild(i);
-
-    // If we are deleting something contained entirely in the child, pass on the
-    // request.
-    if (Offset+NumBytes < CurChild->size()) {
-      CurChild->erase(Offset, NumBytes);
-      return;
-    }
-
-    // If this deletion request starts somewhere in the middle of the child, it
-    // must be deleting to the end of the child.
-    if (Offset) {
-      unsigned BytesFromChild = CurChild->size()-Offset;
-      CurChild->erase(Offset, BytesFromChild);
-      NumBytes -= BytesFromChild;
-      // Start at the beginning of the next child.
-      Offset = 0;
-      ++i;
-      continue;
-    }
-
-    // If the deletion request completely covers the child, delete it and move
-    // the rest down.
-    NumBytes -= CurChild->size();
-    CurChild->Destroy();
-    --NumChildren;
-    if (i != getNumChildren())
-      memmove(&Children[i], &Children[i+1],
-              (getNumChildren()-i)*sizeof(Children[0]));
-  }
-}
-
-//===----------------------------------------------------------------------===//
-// RopePieceBTreeNode Implementation
-//===----------------------------------------------------------------------===//
-
-void RopePieceBTreeNode::Destroy() {
-  if (RopePieceBTreeLeaf *Leaf = dyn_cast<RopePieceBTreeLeaf>(this))
-    delete Leaf;
-  else
-    delete cast<RopePieceBTreeInterior>(this);
-}
-
-/// split - Split the range containing the specified offset so that we are
-/// guaranteed that there is a place to do an insertion at the specified
-/// offset.  The offset is relative, so "0" is the start of the node.
-///
-/// If there is no space in this subtree for the extra piece, the extra tree
-/// node is returned and must be inserted into a parent.
-RopePieceBTreeNode *RopePieceBTreeNode::split(unsigned Offset) {
-  assert(Offset <= size() && "Invalid offset to split!");
-  if (RopePieceBTreeLeaf *Leaf = dyn_cast<RopePieceBTreeLeaf>(this))
-    return Leaf->split(Offset);
-  return cast<RopePieceBTreeInterior>(this)->split(Offset);
-}
-
-/// insert - Insert the specified ropepiece into this tree node at the
-/// specified offset.  The offset is relative, so "0" is the start of the
-/// node.
-///
-/// If there is no space in this subtree for the extra piece, the extra tree
-/// node is returned and must be inserted into a parent.
-RopePieceBTreeNode *RopePieceBTreeNode::insert(unsigned Offset,
-                                               const RopePiece &R) {
-  assert(Offset <= size() && "Invalid offset to insert!");
-  if (RopePieceBTreeLeaf *Leaf = dyn_cast<RopePieceBTreeLeaf>(this))
-    return Leaf->insert(Offset, R);
-  return cast<RopePieceBTreeInterior>(this)->insert(Offset, R);
-}
-
-/// erase - Remove NumBytes from this node at the specified offset.  We are
-/// guaranteed that there is a split at Offset.
-void RopePieceBTreeNode::erase(unsigned Offset, unsigned NumBytes) {
-  assert(Offset+NumBytes <= size() && "Invalid offset to erase!");
-  if (RopePieceBTreeLeaf *Leaf = dyn_cast<RopePieceBTreeLeaf>(this))
-    return Leaf->erase(Offset, NumBytes);
-  return cast<RopePieceBTreeInterior>(this)->erase(Offset, NumBytes);
-}
-
-
-//===----------------------------------------------------------------------===//
-// RopePieceBTreeIterator Implementation
-//===----------------------------------------------------------------------===//
-
-static const RopePieceBTreeLeaf *getCN(const void *P) {
-  return static_cast<const RopePieceBTreeLeaf*>(P);
-}
-
-// begin iterator.
-RopePieceBTreeIterator::RopePieceBTreeIterator(const void *n) {
-  const RopePieceBTreeNode *N = static_cast<const RopePieceBTreeNode*>(n);
-
-  // Walk down the left side of the tree until we get to a leaf.
-  while (const RopePieceBTreeInterior *IN = dyn_cast<RopePieceBTreeInterior>(N))
-    N = IN->getChild(0);
-
-  // We must have at least one leaf.
-  CurNode = cast<RopePieceBTreeLeaf>(N);
-
-  // If we found a leaf that happens to be empty, skip over it until we get
-  // to something full.
-  while (CurNode && getCN(CurNode)->getNumPieces() == 0)
-    CurNode = getCN(CurNode)->getNextLeafInOrder();
-
-  if (CurNode != 0)
-    CurPiece = &getCN(CurNode)->getPiece(0);
-  else  // Empty tree, this is an end() iterator.
-    CurPiece = 0;
-  CurChar = 0;
-}
-
-void RopePieceBTreeIterator::MoveToNextPiece() {
-  if (CurPiece != &getCN(CurNode)->getPiece(getCN(CurNode)->getNumPieces()-1)) {
-    CurChar = 0;
-    ++CurPiece;
-    return;
-  }
-
-  // Find the next non-empty leaf node.
-  do
-    CurNode = getCN(CurNode)->getNextLeafInOrder();
-  while (CurNode && getCN(CurNode)->getNumPieces() == 0);
-
-  if (CurNode != 0)
-    CurPiece = &getCN(CurNode)->getPiece(0);
-  else // Hit end().
-    CurPiece = 0;
-  CurChar = 0;
-}
-
-//===----------------------------------------------------------------------===//
-// RopePieceBTree Implementation
-//===----------------------------------------------------------------------===//
-
-static RopePieceBTreeNode *getRoot(void *P) {
-  return static_cast<RopePieceBTreeNode*>(P);
-}
-
-RopePieceBTree::RopePieceBTree() {
-  Root = new RopePieceBTreeLeaf();
-}
-RopePieceBTree::RopePieceBTree(const RopePieceBTree &RHS) {
-  assert(RHS.empty() && "Can't copy non-empty tree yet");
-  Root = new RopePieceBTreeLeaf();
-}
-RopePieceBTree::~RopePieceBTree() {
-  getRoot(Root)->Destroy();
-}
-
-unsigned RopePieceBTree::size() const {
-  return getRoot(Root)->size();
-}
-
-void RopePieceBTree::clear() {
-  if (RopePieceBTreeLeaf *Leaf = dyn_cast<RopePieceBTreeLeaf>(getRoot(Root)))
-    Leaf->clear();
-  else {
-    getRoot(Root)->Destroy();
-    Root = new RopePieceBTreeLeaf();
-  }
-}
-
-void RopePieceBTree::insert(unsigned Offset, const RopePiece &R) {
-  // #1. Split at Offset.
-  if (RopePieceBTreeNode *RHS = getRoot(Root)->split(Offset))
-    Root = new RopePieceBTreeInterior(getRoot(Root), RHS);
-
-  // #2. Do the insertion.
-  if (RopePieceBTreeNode *RHS = getRoot(Root)->insert(Offset, R))
-    Root = new RopePieceBTreeInterior(getRoot(Root), RHS);
-}
-
-void RopePieceBTree::erase(unsigned Offset, unsigned NumBytes) {
-  // #1. Split at Offset.
-  if (RopePieceBTreeNode *RHS = getRoot(Root)->split(Offset))
-    Root = new RopePieceBTreeInterior(getRoot(Root), RHS);
-
-  // #2. Do the erasing.
-  getRoot(Root)->erase(Offset, NumBytes);
-}
-
-//===----------------------------------------------------------------------===//
-// RewriteRope Implementation
-//===----------------------------------------------------------------------===//
-
-/// MakeRopeString - This copies the specified byte range into some instance of
-/// RopeRefCountString, and return a RopePiece that represents it.  This uses
-/// the AllocBuffer object to aggregate requests for small strings into one
-/// allocation instead of doing tons of tiny allocations.
-RopePiece RewriteRope::MakeRopeString(const char *Start, const char *End) {
-  unsigned Len = End-Start;
-  assert(Len && "Zero length RopePiece is invalid!");
-
-  // If we have space for this string in the current alloc buffer, use it.
-  if (AllocOffs+Len <= AllocChunkSize) {
-    memcpy(AllocBuffer->Data+AllocOffs, Start, Len);
-    AllocOffs += Len;
-    return RopePiece(AllocBuffer, AllocOffs-Len, AllocOffs);
-  }
-
-  // If we don't have enough room because this specific allocation is huge,
-  // just allocate a new rope piece for it alone.
-  if (Len > AllocChunkSize) {
-    unsigned Size = End-Start+sizeof(RopeRefCountString)-1;
-    RopeRefCountString *Res =
-      reinterpret_cast<RopeRefCountString *>(new char[Size]);
-    Res->RefCount = 0;
-    memcpy(Res->Data, Start, End-Start);
-    return RopePiece(Res, 0, End-Start);
-  }
-
-  // Otherwise, this was a small request but we just don't have space for it
-  // Make a new chunk and share it with later allocations.
-
-  // If we had an old allocation, drop our reference to it.
-  if (AllocBuffer && --AllocBuffer->RefCount == 0)
-    delete [] (char*)AllocBuffer;
-
-  unsigned AllocSize = offsetof(RopeRefCountString, Data) + AllocChunkSize;
-  AllocBuffer = reinterpret_cast<RopeRefCountString *>(new char[AllocSize]);
-  AllocBuffer->RefCount = 0;
-  memcpy(AllocBuffer->Data, Start, Len);
-  AllocOffs = Len;
-
-  // Start out the new allocation with a refcount of 1, since we have an
-  // internal reference to it.
-  AllocBuffer->addRef();
-  return RopePiece(AllocBuffer, 0, Len);
-}
-
-
diff --git a/lib/Rewrite/RewriteTest.cpp b/lib/Rewrite/RewriteTest.cpp
deleted file mode 100644
index 019e5e7..0000000
--- a/lib/Rewrite/RewriteTest.cpp
+++ /dev/null
@@ -1,39 +0,0 @@
-//===--- RewriteTest.cpp - Rewriter playground ----------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This is a testbed.
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/Rewrite/Rewriters.h"
-#include "clang/Lex/Preprocessor.h"
-#include "clang/Rewrite/TokenRewriter.h"
-#include "llvm/Support/raw_ostream.h"
-
-void clang::DoRewriteTest(Preprocessor &PP, raw_ostream* OS) {
-  SourceManager &SM = PP.getSourceManager();
-  const LangOptions &LangOpts = PP.getLangOpts();
-
-  TokenRewriter Rewriter(SM.getMainFileID(), SM, LangOpts);
-
-  // Throw <i> </i> tags around comments.
-  for (TokenRewriter::token_iterator I = Rewriter.token_begin(),
-       E = Rewriter.token_end(); I != E; ++I) {
-    if (I->isNot(tok::comment)) continue;
-
-    Rewriter.AddTokenBefore(I, "<i>");
-    Rewriter.AddTokenAfter(I, "</i>");
-  }
-
-
-  // Print out the output.
-  for (TokenRewriter::token_iterator I = Rewriter.token_begin(),
-       E = Rewriter.token_end(); I != E; ++I)
-    *OS << PP.getSpelling(*I);
-}
diff --git a/lib/Rewrite/Rewriter.cpp b/lib/Rewrite/Rewriter.cpp
deleted file mode 100644
index 7c27114..0000000
--- a/lib/Rewrite/Rewriter.cpp
+++ /dev/null
@@ -1,486 +0,0 @@
-//===--- Rewriter.cpp - Code rewriting interface --------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//  This file defines the Rewriter class, which is used for code
-//  transformations.
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/Rewrite/Rewriter.h"
-#include "clang/AST/Stmt.h"
-#include "clang/AST/Decl.h"
-#include "clang/Basic/DiagnosticIDs.h"
-#include "clang/Basic/FileManager.h"
-#include "clang/Basic/SourceManager.h"
-#include "clang/Lex/Lexer.h"
-#include "llvm/ADT/SmallString.h"
-#include "llvm/Support/FileSystem.h"
-using namespace clang;
-
-raw_ostream &RewriteBuffer::write(raw_ostream &os) const {
-  // FIXME: eliminate the copy by writing out each chunk at a time
-  os << std::string(begin(), end());
-  return os;
-}
-
-/// \brief Return true if this character is non-new-line whitespace:
-/// ' ', '\\t', '\\f', '\\v', '\\r'.
-static inline bool isWhitespace(unsigned char c) {
-  switch (c) {
-  case ' ':
-  case '\t':
-  case '\f':
-  case '\v':
-  case '\r':
-    return true;
-  default:
-    return false;
-  }
-}
-
-void RewriteBuffer::RemoveText(unsigned OrigOffset, unsigned Size,
-                               bool removeLineIfEmpty) {
-  // Nothing to remove, exit early.
-  if (Size == 0) return;
-
-  unsigned RealOffset = getMappedOffset(OrigOffset, true);
-  assert(RealOffset+Size < Buffer.size() && "Invalid location");
-
-  // Remove the dead characters.
-  Buffer.erase(RealOffset, Size);
-
-  // Add a delta so that future changes are offset correctly.
-  AddReplaceDelta(OrigOffset, -Size);
-
-  if (removeLineIfEmpty) {
-    // Find the line that the remove occurred and if it is completely empty
-    // remove the line as well.
-
-    iterator curLineStart = begin();
-    unsigned curLineStartOffs = 0;
-    iterator posI = begin();
-    for (unsigned i = 0; i != RealOffset; ++i) {
-      if (*posI == '\n') {
-        curLineStart = posI;
-        ++curLineStart;
-        curLineStartOffs = i + 1;
-      }
-      ++posI;
-    }
-  
-    unsigned lineSize = 0;
-    posI = curLineStart;
-    while (posI != end() && isWhitespace(*posI)) {
-      ++posI;
-      ++lineSize;
-    }
-    if (posI != end() && *posI == '\n') {
-      Buffer.erase(curLineStartOffs, lineSize + 1/* + '\n'*/);
-      AddReplaceDelta(curLineStartOffs, -(lineSize + 1/* + '\n'*/));
-    }
-  }
-}
-
-void RewriteBuffer::InsertText(unsigned OrigOffset, StringRef Str,
-                               bool InsertAfter) {
-
-  // Nothing to insert, exit early.
-  if (Str.empty()) return;
-
-  unsigned RealOffset = getMappedOffset(OrigOffset, InsertAfter);
-  Buffer.insert(RealOffset, Str.begin(), Str.end());
-
-  // Add a delta so that future changes are offset correctly.
-  AddInsertDelta(OrigOffset, Str.size());
-}
-
-/// ReplaceText - This method replaces a range of characters in the input
-/// buffer with a new string.  This is effectively a combined "remove+insert"
-/// operation.
-void RewriteBuffer::ReplaceText(unsigned OrigOffset, unsigned OrigLength,
-                                StringRef NewStr) {
-  unsigned RealOffset = getMappedOffset(OrigOffset, true);
-  Buffer.erase(RealOffset, OrigLength);
-  Buffer.insert(RealOffset, NewStr.begin(), NewStr.end());
-  if (OrigLength != NewStr.size())
-    AddReplaceDelta(OrigOffset, NewStr.size() - OrigLength);
-}
-
-
-//===----------------------------------------------------------------------===//
-// Rewriter class
-//===----------------------------------------------------------------------===//
-
-/// getRangeSize - Return the size in bytes of the specified range if they
-/// are in the same file.  If not, this returns -1.
-int Rewriter::getRangeSize(const CharSourceRange &Range,
-                           RewriteOptions opts) const {
-  if (!isRewritable(Range.getBegin()) ||
-      !isRewritable(Range.getEnd())) return -1;
-
-  FileID StartFileID, EndFileID;
-  unsigned StartOff, EndOff;
-
-  StartOff = getLocationOffsetAndFileID(Range.getBegin(), StartFileID);
-  EndOff   = getLocationOffsetAndFileID(Range.getEnd(), EndFileID);
-
-  if (StartFileID != EndFileID)
-    return -1;
-
-  // If edits have been made to this buffer, the delta between the range may
-  // have changed.
-  std::map<FileID, RewriteBuffer>::const_iterator I =
-    RewriteBuffers.find(StartFileID);
-  if (I != RewriteBuffers.end()) {
-    const RewriteBuffer &RB = I->second;
-    EndOff = RB.getMappedOffset(EndOff, opts.IncludeInsertsAtEndOfRange);
-    StartOff = RB.getMappedOffset(StartOff, !opts.IncludeInsertsAtBeginOfRange);
-  }
-
-
-  // Adjust the end offset to the end of the last token, instead of being the
-  // start of the last token if this is a token range.
-  if (Range.isTokenRange())
-    EndOff += Lexer::MeasureTokenLength(Range.getEnd(), *SourceMgr, *LangOpts);
-
-  return EndOff-StartOff;
-}
-
-int Rewriter::getRangeSize(SourceRange Range, RewriteOptions opts) const {
-  return getRangeSize(CharSourceRange::getTokenRange(Range), opts);
-}
-
-
-/// getRewrittenText - Return the rewritten form of the text in the specified
-/// range.  If the start or end of the range was unrewritable or if they are
-/// in different buffers, this returns an empty string.
-///
-/// Note that this method is not particularly efficient.
-///
-std::string Rewriter::getRewrittenText(SourceRange Range) const {
-  if (!isRewritable(Range.getBegin()) ||
-      !isRewritable(Range.getEnd()))
-    return "";
-
-  FileID StartFileID, EndFileID;
-  unsigned StartOff, EndOff;
-  StartOff = getLocationOffsetAndFileID(Range.getBegin(), StartFileID);
-  EndOff   = getLocationOffsetAndFileID(Range.getEnd(), EndFileID);
-
-  if (StartFileID != EndFileID)
-    return ""; // Start and end in different buffers.
-
-  // If edits have been made to this buffer, the delta between the range may
-  // have changed.
-  std::map<FileID, RewriteBuffer>::const_iterator I =
-    RewriteBuffers.find(StartFileID);
-  if (I == RewriteBuffers.end()) {
-    // If the buffer hasn't been rewritten, just return the text from the input.
-    const char *Ptr = SourceMgr->getCharacterData(Range.getBegin());
-
-    // Adjust the end offset to the end of the last token, instead of being the
-    // start of the last token.
-    EndOff += Lexer::MeasureTokenLength(Range.getEnd(), *SourceMgr, *LangOpts);
-    return std::string(Ptr, Ptr+EndOff-StartOff);
-  }
-
-  const RewriteBuffer &RB = I->second;
-  EndOff = RB.getMappedOffset(EndOff, true);
-  StartOff = RB.getMappedOffset(StartOff);
-
-  // Adjust the end offset to the end of the last token, instead of being the
-  // start of the last token.
-  EndOff += Lexer::MeasureTokenLength(Range.getEnd(), *SourceMgr, *LangOpts);
-
-  // Advance the iterators to the right spot, yay for linear time algorithms.
-  RewriteBuffer::iterator Start = RB.begin();
-  std::advance(Start, StartOff);
-  RewriteBuffer::iterator End = Start;
-  std::advance(End, EndOff-StartOff);
-
-  return std::string(Start, End);
-}
-
-unsigned Rewriter::getLocationOffsetAndFileID(SourceLocation Loc,
-                                              FileID &FID) const {
-  assert(Loc.isValid() && "Invalid location");
-  std::pair<FileID,unsigned> V = SourceMgr->getDecomposedLoc(Loc);
-  FID = V.first;
-  return V.second;
-}
-
-
-/// getEditBuffer - Get or create a RewriteBuffer for the specified FileID.
-///
-RewriteBuffer &Rewriter::getEditBuffer(FileID FID) {
-  std::map<FileID, RewriteBuffer>::iterator I =
-    RewriteBuffers.lower_bound(FID);
-  if (I != RewriteBuffers.end() && I->first == FID)
-    return I->second;
-  I = RewriteBuffers.insert(I, std::make_pair(FID, RewriteBuffer()));
-
-  StringRef MB = SourceMgr->getBufferData(FID);
-  I->second.Initialize(MB.begin(), MB.end());
-
-  return I->second;
-}
-
-/// InsertText - Insert the specified string at the specified location in the
-/// original buffer.
-bool Rewriter::InsertText(SourceLocation Loc, StringRef Str,
-                          bool InsertAfter, bool indentNewLines) {
-  if (!isRewritable(Loc)) return true;
-  FileID FID;
-  unsigned StartOffs = getLocationOffsetAndFileID(Loc, FID);
-
-  SmallString<128> indentedStr;
-  if (indentNewLines && Str.find('\n') != StringRef::npos) {
-    StringRef MB = SourceMgr->getBufferData(FID);
-
-    unsigned lineNo = SourceMgr->getLineNumber(FID, StartOffs) - 1;
-    const SrcMgr::ContentCache *
-        Content = SourceMgr->getSLocEntry(FID).getFile().getContentCache();
-    unsigned lineOffs = Content->SourceLineCache[lineNo];
-
-    // Find the whitespace at the start of the line.
-    StringRef indentSpace;
-    {
-      unsigned i = lineOffs;
-      while (isWhitespace(MB[i]))
-        ++i;
-      indentSpace = MB.substr(lineOffs, i-lineOffs);
-    }
-
-    SmallVector<StringRef, 4> lines;
-    Str.split(lines, "\n");
-
-    for (unsigned i = 0, e = lines.size(); i != e; ++i) {
-      indentedStr += lines[i];
-      if (i < e-1) {
-        indentedStr += '\n';
-        indentedStr += indentSpace;
-      }
-    }
-    Str = indentedStr.str();
-  }
-
-  getEditBuffer(FID).InsertText(StartOffs, Str, InsertAfter);
-  return false;
-}
-
-bool Rewriter::InsertTextAfterToken(SourceLocation Loc, StringRef Str) {
-  if (!isRewritable(Loc)) return true;
-  FileID FID;
-  unsigned StartOffs = getLocationOffsetAndFileID(Loc, FID);
-  RewriteOptions rangeOpts;
-  rangeOpts.IncludeInsertsAtBeginOfRange = false;
-  StartOffs += getRangeSize(SourceRange(Loc, Loc), rangeOpts);
-  getEditBuffer(FID).InsertText(StartOffs, Str, /*InsertAfter*/true);
-  return false;
-}
-
-/// RemoveText - Remove the specified text region.
-bool Rewriter::RemoveText(SourceLocation Start, unsigned Length,
-                          RewriteOptions opts) {
-  if (!isRewritable(Start)) return true;
-  FileID FID;
-  unsigned StartOffs = getLocationOffsetAndFileID(Start, FID);
-  getEditBuffer(FID).RemoveText(StartOffs, Length, opts.RemoveLineIfEmpty);
-  return false;
-}
-
-/// ReplaceText - This method replaces a range of characters in the input
-/// buffer with a new string.  This is effectively a combined "remove/insert"
-/// operation.
-bool Rewriter::ReplaceText(SourceLocation Start, unsigned OrigLength,
-                           StringRef NewStr) {
-  if (!isRewritable(Start)) return true;
-  FileID StartFileID;
-  unsigned StartOffs = getLocationOffsetAndFileID(Start, StartFileID);
-
-  getEditBuffer(StartFileID).ReplaceText(StartOffs, OrigLength, NewStr);
-  return false;
-}
-
-bool Rewriter::ReplaceText(SourceRange range, SourceRange replacementRange) {
-  if (!isRewritable(range.getBegin())) return true;
-  if (!isRewritable(range.getEnd())) return true;
-  if (replacementRange.isInvalid()) return true;
-  SourceLocation start = range.getBegin();
-  unsigned origLength = getRangeSize(range);
-  unsigned newLength = getRangeSize(replacementRange);
-  FileID FID;
-  unsigned newOffs = getLocationOffsetAndFileID(replacementRange.getBegin(),
-                                                FID);
-  StringRef MB = SourceMgr->getBufferData(FID);
-  return ReplaceText(start, origLength, MB.substr(newOffs, newLength));
-}
-
-/// ReplaceStmt - This replaces a Stmt/Expr with another, using the pretty
-/// printer to generate the replacement code.  This returns true if the input
-/// could not be rewritten, or false if successful.
-bool Rewriter::ReplaceStmt(Stmt *From, Stmt *To) {
-  // Measaure the old text.
-  int Size = getRangeSize(From->getSourceRange());
-  if (Size == -1)
-    return true;
-
-  // Get the new text.
-  std::string SStr;
-  llvm::raw_string_ostream S(SStr);
-  To->printPretty(S, 0, PrintingPolicy(*LangOpts));
-  const std::string &Str = S.str();
-
-  ReplaceText(From->getLocStart(), Size, Str);
-  return false;
-}
-
-std::string Rewriter::ConvertToString(Stmt *From) {
-  std::string SStr;
-  llvm::raw_string_ostream S(SStr);
-  From->printPretty(S, 0, PrintingPolicy(*LangOpts));
-  return S.str();
-}
-
-bool Rewriter::IncreaseIndentation(CharSourceRange range,
-                                   SourceLocation parentIndent) {
-  if (range.isInvalid()) return true;
-  if (!isRewritable(range.getBegin())) return true;
-  if (!isRewritable(range.getEnd())) return true;
-  if (!isRewritable(parentIndent)) return true;
-
-  FileID StartFileID, EndFileID, parentFileID;
-  unsigned StartOff, EndOff, parentOff;
-
-  StartOff = getLocationOffsetAndFileID(range.getBegin(), StartFileID);
-  EndOff   = getLocationOffsetAndFileID(range.getEnd(), EndFileID);
-  parentOff = getLocationOffsetAndFileID(parentIndent, parentFileID);
-
-  if (StartFileID != EndFileID || StartFileID != parentFileID)
-    return true;
-  if (StartOff > EndOff)
-    return true;
-
-  FileID FID = StartFileID;
-  StringRef MB = SourceMgr->getBufferData(FID);
-
-  unsigned parentLineNo = SourceMgr->getLineNumber(FID, parentOff) - 1;
-  unsigned startLineNo = SourceMgr->getLineNumber(FID, StartOff) - 1;
-  unsigned endLineNo = SourceMgr->getLineNumber(FID, EndOff) - 1;
-  
-  const SrcMgr::ContentCache *
-      Content = SourceMgr->getSLocEntry(FID).getFile().getContentCache();
-  
-  // Find where the lines start.
-  unsigned parentLineOffs = Content->SourceLineCache[parentLineNo];
-  unsigned startLineOffs = Content->SourceLineCache[startLineNo];
-
-  // Find the whitespace at the start of each line.
-  StringRef parentSpace, startSpace;
-  {
-    unsigned i = parentLineOffs;
-    while (isWhitespace(MB[i]))
-      ++i;
-    parentSpace = MB.substr(parentLineOffs, i-parentLineOffs);
-
-    i = startLineOffs;
-    while (isWhitespace(MB[i]))
-      ++i;
-    startSpace = MB.substr(startLineOffs, i-startLineOffs);
-  }
-  if (parentSpace.size() >= startSpace.size())
-    return true;
-  if (!startSpace.startswith(parentSpace))
-    return true;
-
-  StringRef indent = startSpace.substr(parentSpace.size());
-
-  // Indent the lines between start/end offsets.
-  RewriteBuffer &RB = getEditBuffer(FID);
-  for (unsigned lineNo = startLineNo; lineNo <= endLineNo; ++lineNo) {
-    unsigned offs = Content->SourceLineCache[lineNo];
-    unsigned i = offs;
-    while (isWhitespace(MB[i]))
-      ++i;
-    StringRef origIndent = MB.substr(offs, i-offs);
-    if (origIndent.startswith(startSpace))
-      RB.InsertText(offs, indent, /*InsertAfter=*/false);
-  }
-
-  return false;
-}
-
-// A wrapper for a file stream that atomically overwrites the target.
-//
-// Creates a file output stream for a temporary file in the constructor,
-// which is later accessible via getStream() if ok() return true.
-// Flushes the stream and moves the temporary file to the target location
-// in the destructor.
-class AtomicallyMovedFile {
-public:
-  AtomicallyMovedFile(DiagnosticsEngine &Diagnostics, StringRef Filename,
-                      bool &AllWritten)
-    : Diagnostics(Diagnostics), Filename(Filename), AllWritten(AllWritten) {
-    TempFilename = Filename;
-    TempFilename += "-%%%%%%%%";
-    int FD;
-    if (llvm::sys::fs::unique_file(TempFilename.str(), FD, TempFilename,
-                                    /*makeAbsolute=*/true, 0664)) {
-      AllWritten = false;
-      Diagnostics.Report(clang::diag::err_unable_to_make_temp)
-        << TempFilename;
-    } else {
-      FileStream.reset(new llvm::raw_fd_ostream(FD, /*shouldClose=*/true));
-    }
-  }
-
-  ~AtomicallyMovedFile() {
-    if (!ok()) return;
-
-    FileStream->flush();
-#ifdef _WIN32
-    // Win32 does not allow rename/removing opened files.
-    FileStream.reset();
-#endif
-    if (llvm::error_code ec =
-          llvm::sys::fs::rename(TempFilename.str(), Filename)) {
-      AllWritten = false;
-      Diagnostics.Report(clang::diag::err_unable_to_rename_temp)
-        << TempFilename << Filename << ec.message();
-      bool existed;
-      // If the remove fails, there's not a lot we can do - this is already an
-      // error.
-      llvm::sys::fs::remove(TempFilename.str(), existed);
-    }
-  }
-
-  bool ok() { return FileStream; }
-  llvm::raw_ostream &getStream() { return *FileStream; }
-
-private:
-  DiagnosticsEngine &Diagnostics;
-  StringRef Filename;
-  SmallString<128> TempFilename;
-  OwningPtr<llvm::raw_fd_ostream> FileStream;
-  bool &AllWritten;
-};
-
-bool Rewriter::overwriteChangedFiles() {
-  bool AllWritten = true;
-  for (buffer_iterator I = buffer_begin(), E = buffer_end(); I != E; ++I) {
-    const FileEntry *Entry =
-        getSourceMgr().getFileEntryForID(I->first);
-    AtomicallyMovedFile File(getSourceMgr().getDiagnostics(), Entry->getName(),
-                             AllWritten);
-    if (File.ok()) {
-      I->second.write(File.getStream());
-    }
-  }
-  return !AllWritten;
-}
diff --git a/lib/Rewrite/TokenRewriter.cpp b/lib/Rewrite/TokenRewriter.cpp
deleted file mode 100644
index 03ce63e..0000000
--- a/lib/Rewrite/TokenRewriter.cpp
+++ /dev/null
@@ -1,99 +0,0 @@
-//===--- TokenRewriter.cpp - Token-based code rewriting interface ---------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//  This file implements the TokenRewriter class, which is used for code
-//  transformations.
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/Rewrite/TokenRewriter.h"
-#include "clang/Lex/Lexer.h"
-#include "clang/Lex/ScratchBuffer.h"
-#include "clang/Basic/SourceManager.h"
-using namespace clang;
-
-TokenRewriter::TokenRewriter(FileID FID, SourceManager &SM,
-                             const LangOptions &LangOpts) {
-  ScratchBuf.reset(new ScratchBuffer(SM));
-
-  // Create a lexer to lex all the tokens of the main file in raw mode.
-  const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID);
-  Lexer RawLex(FID, FromFile, SM, LangOpts);
-
-  // Return all comments and whitespace as tokens.
-  RawLex.SetKeepWhitespaceMode(true);
-
-  // Lex the file, populating our datastructures.
-  Token RawTok;
-  RawLex.LexFromRawLexer(RawTok);
-  while (RawTok.isNot(tok::eof)) {
-#if 0
-    if (Tok.is(tok::raw_identifier)) {
-      // Look up the identifier info for the token.  This should use
-      // IdentifierTable directly instead of PP.
-      PP.LookUpIdentifierInfo(Tok);
-    }
-#endif
-
-    AddToken(RawTok, TokenList.end());
-    RawLex.LexFromRawLexer(RawTok);
-  }
-}
-
-TokenRewriter::~TokenRewriter() {
-}
-
-
-/// RemapIterator - Convert from token_iterator (a const iterator) to
-/// TokenRefTy (a non-const iterator).
-TokenRewriter::TokenRefTy TokenRewriter::RemapIterator(token_iterator I) {
-  if (I == token_end()) return TokenList.end();
-
-  // FIXME: This is horrible, we should use our own list or something to avoid
-  // this.
-  std::map<SourceLocation, TokenRefTy>::iterator MapIt =
-    TokenAtLoc.find(I->getLocation());
-  assert(MapIt != TokenAtLoc.end() && "iterator not in rewriter?");
-  return MapIt->second;
-}
-
-
-/// AddToken - Add the specified token into the Rewriter before the other
-/// position.
-TokenRewriter::TokenRefTy
-TokenRewriter::AddToken(const Token &T, TokenRefTy Where) {
-  Where = TokenList.insert(Where, T);
-
-  bool InsertSuccess = TokenAtLoc.insert(std::make_pair(T.getLocation(),
-                                                        Where)).second;
-  assert(InsertSuccess && "Token location already in rewriter!");
-  (void)InsertSuccess;
-  return Where;
-}
-
-
-TokenRewriter::token_iterator
-TokenRewriter::AddTokenBefore(token_iterator I, const char *Val) {
-  unsigned Len = strlen(Val);
-
-  // Plop the string into the scratch buffer, then create a token for this
-  // string.
-  Token Tok;
-  Tok.startToken();
-  const char *Spelling;
-  Tok.setLocation(ScratchBuf->getToken(Val, Len, Spelling));
-  Tok.setLength(Len);
-
-  // TODO: Form a whole lexer around this and relex the token!  For now, just
-  // set kind to tok::unknown.
-  Tok.setKind(tok::unknown);
-
-  return AddToken(Tok, RemapIterator(I));
-}
-
diff --git a/lib/Sema/AnalysisBasedWarnings.cpp b/lib/Sema/AnalysisBasedWarnings.cpp
index 19a7d6f..801a1b1 100644
--- a/lib/Sema/AnalysisBasedWarnings.cpp
+++ b/lib/Sema/AnalysisBasedWarnings.cpp
@@ -27,6 +27,7 @@
 #include "clang/AST/StmtObjC.h"
 #include "clang/AST/StmtCXX.h"
 #include "clang/AST/EvaluatedExprVisitor.h"
+#include "clang/AST/ParentMap.h"
 #include "clang/AST/StmtVisitor.h"
 #include "clang/AST/RecursiveASTVisitor.h"
 #include "clang/Analysis/AnalysisContext.h"
@@ -36,10 +37,12 @@
 #include "clang/Analysis/Analyses/ThreadSafety.h"
 #include "clang/Analysis/CFGStmtMap.h"
 #include "clang/Analysis/Analyses/UninitializedValues.h"
+#include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/FoldingSet.h"
 #include "llvm/ADT/ImmutableMap.h"
 #include "llvm/ADT/PostOrderIterator.h"
+#include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/Casting.h"
@@ -182,13 +185,6 @@ static ControlFlowKind CheckFallThrough(AnalysisDeclContext &AC) {
       HasFakeEdge = true;
       continue;
     }
-    if (const AsmStmt *AS = dyn_cast<AsmStmt>(S)) {
-      if (AS->isMSAsm()) {
-        HasFakeEdge = true;
-        HasLiveReturn = true;
-        continue;
-      }
-    }
     if (isa<MSAsmStmt>(S)) {
       // TODO: Verify this is correct.
       HasFakeEdge = true;
@@ -506,7 +502,7 @@ static void DiagUninitUse(Sema &S, const VarDecl *VD, const UninitUse &Use,
 
     // Information used when building the diagnostic.
     unsigned DiagKind;
-    const char *Str;
+    StringRef Str;
     SourceRange Range;
 
     // FixIts to suppress the diagnosic by removing the dead condition.
@@ -822,6 +818,18 @@ namespace {
 
 static void DiagnoseSwitchLabelsFallthrough(Sema &S, AnalysisDeclContext &AC,
                                             bool PerFunction) {
+  // Only perform this analysis when using C++11.  There is no good workflow
+  // for this warning when not using C++11.  There is no good way to silence
+  // the warning (no attribute is available) unless we are using C++11's support
+  // for generalized attributes.  Once could use pragmas to silence the warning,
+  // but as a general solution that is gross and not in the spirit of this
+  // warning.
+  //
+  // NOTE: This an intermediate solution.  There are on-going discussions on
+  // how to properly support this warning outside of C++11 with an annotation.
+  if (!AC.getASTContext().getLangOpts().CPlusPlus0x)
+    return;
+
   FallthroughMapper FM(S);
   FM.TraverseStmt(AC.getBody());
 
@@ -859,8 +867,21 @@ static void DiagnoseSwitchLabelsFallthrough(Sema &S, AnalysisDeclContext &AC,
       if (S.getLangOpts().CPlusPlus0x) {
         const Stmt *Term = B.getTerminator();
         if (!(B.empty() && Term && isa<BreakStmt>(Term))) {
+          Preprocessor &PP = S.getPreprocessor();
+          TokenValue Tokens[] = {
+            tok::l_square, tok::l_square, PP.getIdentifierInfo("clang"),
+            tok::coloncolon, PP.getIdentifierInfo("fallthrough"),
+            tok::r_square, tok::r_square
+          };
+          StringRef AnnotationSpelling = "[[clang::fallthrough]]";
+          StringRef MacroName = PP.getLastMacroWithSpelling(L, Tokens);
+          if (!MacroName.empty())
+            AnnotationSpelling = MacroName;
+          SmallString<64> TextToInsert(AnnotationSpelling);
+          TextToInsert += "; ";
           S.Diag(L, diag::note_insert_fallthrough_fixit) <<
-            FixItHint::CreateInsertion(L, "[[clang::fallthrough]]; ");
+              AnnotationSpelling <<
+              FixItHint::CreateInsertion(L, TextToInsert);
         }
       }
       S.Diag(L, diag::note_insert_break_fixit) <<
@@ -878,6 +899,199 @@ static void DiagnoseSwitchLabelsFallthrough(Sema &S, AnalysisDeclContext &AC,
 }
 
 namespace {
+typedef std::pair<const Stmt *,
+                  sema::FunctionScopeInfo::WeakObjectUseMap::const_iterator>
+        StmtUsesPair;
+
+class StmtUseSorter {
+  const SourceManager &SM;
+
+public:
+  explicit StmtUseSorter(const SourceManager &SM) : SM(SM) { }
+
+  bool operator()(const StmtUsesPair &LHS, const StmtUsesPair &RHS) {
+    return SM.isBeforeInTranslationUnit(LHS.first->getLocStart(),
+                                        RHS.first->getLocStart());
+  }
+};
+}
+
+static bool isInLoop(const ASTContext &Ctx, const ParentMap &PM,
+                     const Stmt *S) {
+  assert(S);
+
+  do {
+    switch (S->getStmtClass()) {
+    case Stmt::ForStmtClass:
+    case Stmt::WhileStmtClass:
+    case Stmt::CXXForRangeStmtClass:
+    case Stmt::ObjCForCollectionStmtClass:
+      return true;
+    case Stmt::DoStmtClass: {
+      const Expr *Cond = cast<DoStmt>(S)->getCond();
+      llvm::APSInt Val;
+      if (!Cond->EvaluateAsInt(Val, Ctx))
+        return true;
+      return Val.getBoolValue();
+    }
+    default:
+      break;
+    }
+  } while ((S = PM.getParent(S)));
+
+  return false;
+}
+
+
+static void diagnoseRepeatedUseOfWeak(Sema &S,
+                                      const sema::FunctionScopeInfo *CurFn,
+                                      const Decl *D,
+                                      const ParentMap &PM) {
+  typedef sema::FunctionScopeInfo::WeakObjectProfileTy WeakObjectProfileTy;
+  typedef sema::FunctionScopeInfo::WeakObjectUseMap WeakObjectUseMap;
+  typedef sema::FunctionScopeInfo::WeakUseVector WeakUseVector;
+
+  ASTContext &Ctx = S.getASTContext();
+
+  const WeakObjectUseMap &WeakMap = CurFn->getWeakObjectUses();
+
+  // Extract all weak objects that are referenced more than once.
+  SmallVector<StmtUsesPair, 8> UsesByStmt;
+  for (WeakObjectUseMap::const_iterator I = WeakMap.begin(), E = WeakMap.end();
+       I != E; ++I) {
+    const WeakUseVector &Uses = I->second;
+
+    // Find the first read of the weak object.
+    WeakUseVector::const_iterator UI = Uses.begin(), UE = Uses.end();
+    for ( ; UI != UE; ++UI) {
+      if (UI->isUnsafe())
+        break;
+    }
+
+    // If there were only writes to this object, don't warn.
+    if (UI == UE)
+      continue;
+
+    // If there was only one read, followed by any number of writes, and the
+    // read is not within a loop, don't warn. Additionally, don't warn in a
+    // loop if the base object is a local variable -- local variables are often
+    // changed in loops.
+    if (UI == Uses.begin()) {
+      WeakUseVector::const_iterator UI2 = UI;
+      for (++UI2; UI2 != UE; ++UI2)
+        if (UI2->isUnsafe())
+          break;
+
+      if (UI2 == UE) {
+        if (!isInLoop(Ctx, PM, UI->getUseExpr()))
+          continue;
+
+        const WeakObjectProfileTy &Profile = I->first;
+        if (!Profile.isExactProfile())
+          continue;
+
+        const NamedDecl *Base = Profile.getBase();
+        if (!Base)
+          Base = Profile.getProperty();
+        assert(Base && "A profile always has a base or property.");
+
+        if (const VarDecl *BaseVar = dyn_cast<VarDecl>(Base))
+          if (BaseVar->hasLocalStorage() && !isa<ParmVarDecl>(Base))
+            continue;
+      }
+    }
+
+    UsesByStmt.push_back(StmtUsesPair(UI->getUseExpr(), I));
+  }
+
+  if (UsesByStmt.empty())
+    return;
+
+  // Sort by first use so that we emit the warnings in a deterministic order.
+  std::sort(UsesByStmt.begin(), UsesByStmt.end(),
+            StmtUseSorter(S.getSourceManager()));
+
+  // Classify the current code body for better warning text.
+  // This enum should stay in sync with the cases in
+  // warn_arc_repeated_use_of_weak and warn_arc_possible_repeated_use_of_weak.
+  // FIXME: Should we use a common classification enum and the same set of
+  // possibilities all throughout Sema?
+  enum {
+    Function,
+    Method,
+    Block,
+    Lambda
+  } FunctionKind;
+
+  if (isa<sema::BlockScopeInfo>(CurFn))
+    FunctionKind = Block;
+  else if (isa<sema::LambdaScopeInfo>(CurFn))
+    FunctionKind = Lambda;
+  else if (isa<ObjCMethodDecl>(D))
+    FunctionKind = Method;
+  else
+    FunctionKind = Function;
+
+  // Iterate through the sorted problems and emit warnings for each.
+  for (SmallVectorImpl<StmtUsesPair>::const_iterator I = UsesByStmt.begin(),
+                                                     E = UsesByStmt.end();
+       I != E; ++I) {
+    const Stmt *FirstRead = I->first;
+    const WeakObjectProfileTy &Key = I->second->first;
+    const WeakUseVector &Uses = I->second->second;
+
+    // For complicated expressions like 'a.b.c' and 'x.b.c', WeakObjectProfileTy
+    // may not contain enough information to determine that these are different
+    // properties. We can only be 100% sure of a repeated use in certain cases,
+    // and we adjust the diagnostic kind accordingly so that the less certain
+    // case can be turned off if it is too noisy.
+    unsigned DiagKind;
+    if (Key.isExactProfile())
+      DiagKind = diag::warn_arc_repeated_use_of_weak;
+    else
+      DiagKind = diag::warn_arc_possible_repeated_use_of_weak;
+
+    // Classify the weak object being accessed for better warning text.
+    // This enum should stay in sync with the cases in
+    // warn_arc_repeated_use_of_weak and warn_arc_possible_repeated_use_of_weak.
+    enum {
+      Variable,
+      Property,
+      ImplicitProperty,
+      Ivar
+    } ObjectKind;
+
+    const NamedDecl *D = Key.getProperty();
+    if (isa<VarDecl>(D))
+      ObjectKind = Variable;
+    else if (isa<ObjCPropertyDecl>(D))
+      ObjectKind = Property;
+    else if (isa<ObjCMethodDecl>(D))
+      ObjectKind = ImplicitProperty;
+    else if (isa<ObjCIvarDecl>(D))
+      ObjectKind = Ivar;
+    else
+      llvm_unreachable("Unexpected weak object kind!");
+
+    // Show the first time the object was read.
+    S.Diag(FirstRead->getLocStart(), DiagKind)
+      << ObjectKind << D << FunctionKind
+      << FirstRead->getSourceRange();
+
+    // Print all the other accesses as notes.
+    for (WeakUseVector::const_iterator UI = Uses.begin(), UE = Uses.end();
+         UI != UE; ++UI) {
+      if (UI->getUseExpr() == FirstRead)
+        continue;
+      S.Diag(UI->getUseExpr()->getLocStart(),
+             diag::note_arc_weak_also_accessed_here)
+        << UI->getUseExpr()->getSourceRange();
+    }
+  }
+}
+
+
+namespace {
 struct SLocSort {
   bool operator()(const UninitUse &a, const UninitUse &b) {
     // Prefer a more confident report over a less confident one.
@@ -1091,27 +1305,47 @@ class ThreadSafetyReporter : public clang::thread_safety::ThreadSafetyHandler {
                         diag::warn_variable_requires_any_lock:
                         diag::warn_var_deref_requires_any_lock;
     PartialDiagnosticAt Warning(Loc, S.PDiag(DiagID)
-      << D->getName() << getLockKindFromAccessKind(AK));
+      << D->getNameAsString() << getLockKindFromAccessKind(AK));
     Warnings.push_back(DelayedDiag(Warning, OptionalNotes()));
   }
 
   void handleMutexNotHeld(const NamedDecl *D, ProtectedOperationKind POK,
-                          Name LockName, LockKind LK, SourceLocation Loc) {
+                          Name LockName, LockKind LK, SourceLocation Loc,
+                          Name *PossibleMatch) {
     unsigned DiagID = 0;
-    switch (POK) {
-      case POK_VarAccess:
-        DiagID = diag::warn_variable_requires_lock;
-        break;
-      case POK_VarDereference:
-        DiagID = diag::warn_var_deref_requires_lock;
-        break;
-      case POK_FunctionCall:
-        DiagID = diag::warn_fun_requires_lock;
-        break;
+    if (PossibleMatch) {
+      switch (POK) {
+        case POK_VarAccess:
+          DiagID = diag::warn_variable_requires_lock_precise;
+          break;
+        case POK_VarDereference:
+          DiagID = diag::warn_var_deref_requires_lock_precise;
+          break;
+        case POK_FunctionCall:
+          DiagID = diag::warn_fun_requires_lock_precise;
+          break;
+      }
+      PartialDiagnosticAt Warning(Loc, S.PDiag(DiagID)
+        << D->getNameAsString() << LockName << LK);
+      PartialDiagnosticAt Note(Loc, S.PDiag(diag::note_found_mutex_near_match)
+                               << *PossibleMatch);
+      Warnings.push_back(DelayedDiag(Warning, OptionalNotes(1, Note)));
+    } else {
+      switch (POK) {
+        case POK_VarAccess:
+          DiagID = diag::warn_variable_requires_lock;
+          break;
+        case POK_VarDereference:
+          DiagID = diag::warn_var_deref_requires_lock;
+          break;
+        case POK_FunctionCall:
+          DiagID = diag::warn_fun_requires_lock;
+          break;
+      }
+      PartialDiagnosticAt Warning(Loc, S.PDiag(DiagID)
+        << D->getNameAsString() << LockName << LK);
+      Warnings.push_back(DelayedDiag(Warning, OptionalNotes()));
     }
-    PartialDiagnosticAt Warning(Loc, S.PDiag(DiagID)
-      << D->getName() << LockName << LK);
-    Warnings.push_back(DelayedDiag(Warning, OptionalNotes()));
   }
 
   void handleFunExcludesLock(Name FunName, Name LockName, SourceLocation Loc) {
@@ -1206,7 +1440,8 @@ AnalysisBasedWarnings::IssueWarnings(sema::AnalysisBasedWarnings::Policy P,
   AC.getCFGBuildOptions().AddEHEdges = false;
   AC.getCFGBuildOptions().AddInitializers = true;
   AC.getCFGBuildOptions().AddImplicitDtors = true;
-  
+  AC.getCFGBuildOptions().AddTemporaryDtors = true;
+
   // Force that certain expressions appear as CFGElements in the CFG.  This
   // is used to speed up various analyses.
   // FIXME: This isn't the right factoring.  This is here for initial
@@ -1350,6 +1585,11 @@ AnalysisBasedWarnings::IssueWarnings(sema::AnalysisBasedWarnings::Policy P,
     DiagnoseSwitchLabelsFallthrough(S, AC, !FallThroughDiagFull);
   }
 
+  if (S.getLangOpts().ObjCARCWeak &&
+      Diags.getDiagnosticLevel(diag::warn_arc_repeated_use_of_weak,
+                               D->getLocStart()) != DiagnosticsEngine::Ignored)
+    diagnoseRepeatedUseOfWeak(S, fscope, D, AC.getParentMap());
+
   // Collect statistics about the CFG if it was built.
   if (S.CollectStats && AC.isCFGBuilt()) {
     ++NumFunctionsAnalyzed;
diff --git a/lib/Sema/CMakeLists.txt b/lib/Sema/CMakeLists.txt
index 46dfa05..7cfe3ae 100644
--- a/lib/Sema/CMakeLists.txt
+++ b/lib/Sema/CMakeLists.txt
@@ -13,7 +13,9 @@ add_clang_library(clangSema
   DelayedDiagnostic.cpp
   IdentifierResolver.cpp
   JumpDiagnostics.cpp
+  MultiplexExternalSemaSource.cpp
   Scope.cpp
+  ScopeInfo.cpp
   Sema.cpp
   SemaAccess.cpp
   SemaAttr.cpp
@@ -39,6 +41,7 @@ add_clang_library(clangSema
   SemaOverload.cpp
   SemaPseudoObject.cpp
   SemaStmt.cpp
+  SemaStmtAsm.cpp
   SemaStmtAttr.cpp
   SemaTemplate.cpp
   SemaTemplateDeduction.cpp
diff --git a/lib/Sema/CodeCompleteConsumer.cpp b/lib/Sema/CodeCompleteConsumer.cpp
index a835725..0a23601 100644
--- a/lib/Sema/CodeCompleteConsumer.cpp
+++ b/lib/Sema/CodeCompleteConsumer.cpp
@@ -193,11 +193,10 @@ CodeCompletionString::CodeCompletionString(const Chunk *Chunks,
                                            CXAvailabilityKind Availability,
                                            const char **Annotations,
                                            unsigned NumAnnotations,
-                                           CXCursorKind ParentKind,
                                            StringRef ParentName,
                                            const char *BriefComment)
   : NumChunks(NumChunks), NumAnnotations(NumAnnotations),
-    Priority(Priority), Availability(Availability), ParentKind(ParentKind),
+    Priority(Priority), Availability(Availability),
     ParentName(ParentName), BriefComment(BriefComment)
 { 
   assert(NumChunks <= 0xffff);
@@ -339,7 +338,7 @@ CodeCompletionString *CodeCompletionBuilder::TakeString() {
     = new (Mem) CodeCompletionString(Chunks.data(), Chunks.size(),
                                      Priority, Availability,
                                      Annotations.data(), Annotations.size(),
-                                     ParentKind, ParentName, BriefComment);
+                                     ParentName, BriefComment);
   Chunks.clear();
   return Result;
 }
@@ -380,7 +379,6 @@ void CodeCompletionBuilder::AddChunk(CodeCompletionString::ChunkKind CK,
 
 void CodeCompletionBuilder::addParentContext(DeclContext *DC) {
   if (DC->isTranslationUnit()) {
-    ParentKind = CXCursor_TranslationUnit;
     return;
   }
   
@@ -391,7 +389,6 @@ void CodeCompletionBuilder::addParentContext(DeclContext *DC) {
   if (!ND)
     return;
   
-  ParentKind = getCursorKindForDecl(ND);
   ParentName = getCodeCompletionTUInfo().getParentName(DC);
 }
 
diff --git a/lib/Sema/DeclSpec.cpp b/lib/Sema/DeclSpec.cpp
index d12ca78..b3066eb 100644
--- a/lib/Sema/DeclSpec.cpp
+++ b/lib/Sema/DeclSpec.cpp
@@ -144,11 +144,13 @@ CXXScopeSpec::getWithLocInContext(ASTContext &Context) const {
 
 /// DeclaratorChunk::getFunction - Return a DeclaratorChunk for a function.
 /// "TheDeclarator" is the declarator that this will be added to.
-DeclaratorChunk DeclaratorChunk::getFunction(bool hasProto, bool isVariadic,
+DeclaratorChunk DeclaratorChunk::getFunction(bool hasProto,
                                              bool isAmbiguous,
-                                             SourceLocation EllipsisLoc,
+                                             SourceLocation LParenLoc,
                                              ParamInfo *ArgInfo,
                                              unsigned NumArgs,
+                                             SourceLocation EllipsisLoc,
+                                             SourceLocation RParenLoc,
                                              unsigned TypeQuals,
                                              bool RefQualifierIsLvalueRef,
                                              SourceLocation RefQualifierLoc,
@@ -173,9 +175,11 @@ DeclaratorChunk DeclaratorChunk::getFunction(bool hasProto, bool isVariadic,
   I.EndLoc                      = LocalRangeEnd;
   I.Fun.AttrList                = 0;
   I.Fun.hasPrototype            = hasProto;
-  I.Fun.isVariadic              = isVariadic;
+  I.Fun.isVariadic              = EllipsisLoc.isValid();
   I.Fun.isAmbiguous             = isAmbiguous;
+  I.Fun.LParenLoc               = LParenLoc.getRawEncoding();
   I.Fun.EllipsisLoc             = EllipsisLoc.getRawEncoding();
+  I.Fun.RParenLoc               = RParenLoc.getRawEncoding();
   I.Fun.DeleteArgInfo           = false;
   I.Fun.TypeQuals               = TypeQuals;
   I.Fun.NumArgs                 = NumArgs;
@@ -270,6 +274,7 @@ bool Declarator::isDeclarationOfFunction() const {
     case TST_int:
     case TST_int128:
     case TST_struct:
+    case TST_interface:
     case TST_union:
     case TST_unknown_anytype:
     case TST_unspecified:
@@ -325,10 +330,14 @@ unsigned DeclSpec::getParsedSpecifiers() const {
 
 template <class T> static bool BadSpecifier(T TNew, T TPrev,
                                             const char *&PrevSpec,
-                                            unsigned &DiagID) {
+                                            unsigned &DiagID,
+                                            bool IsExtension = true) {
   PrevSpec = DeclSpec::getSpecifierName(TPrev);
-  DiagID = (TNew == TPrev ? diag::ext_duplicate_declspec
-            : diag::err_invalid_decl_spec_combination);
+  if (TNew != TPrev)
+    DiagID = diag::err_invalid_decl_spec_combination;
+  else
+    DiagID = IsExtension ? diag::ext_duplicate_declspec : 
+                           diag::warn_duplicate_declspec;    
   return true;
 }
 
@@ -396,6 +405,7 @@ const char *DeclSpec::getSpecifierName(DeclSpec::TST T) {
   case DeclSpec::TST_class:       return "class";
   case DeclSpec::TST_union:       return "union";
   case DeclSpec::TST_struct:      return "struct";
+  case DeclSpec::TST_interface:   return "__interface";
   case DeclSpec::TST_typename:    return "type-name";
   case DeclSpec::TST_typeofType:
   case DeclSpec::TST_typeofExpr:  return "typeof";
@@ -670,12 +680,16 @@ bool DeclSpec::SetTypeSpecError() {
 }
 
 bool DeclSpec::SetTypeQual(TQ T, SourceLocation Loc, const char *&PrevSpec,
-                           unsigned &DiagID, const LangOptions &Lang,
-                           bool IsTypeSpec) {
-  // Duplicates are permitted in C99, and are permitted in C++11 unless the
-  // cv-qualifier appears as a type-specifier.
-  if ((TypeQualifiers & T) && !Lang.C99 && (!Lang.CPlusPlus0x || IsTypeSpec))
-    return BadSpecifier(T, T, PrevSpec, DiagID);
+                           unsigned &DiagID, const LangOptions &Lang) {
+  // Duplicates are permitted in C99, but are not permitted in C++. However,
+  // since this is likely not what the user intended, we will always warn.  We
+  // do not need to set the qualifier's location since we already have it.
+  if (TypeQualifiers & T) {
+    bool IsExtension = true;
+    if (Lang.C99)
+      IsExtension = false;
+    return BadSpecifier(T, T, PrevSpec, DiagID, IsExtension);
+  }
   TypeQualifiers |= T;
 
   switch (T) {
diff --git a/lib/Sema/DelayedDiagnostic.cpp b/lib/Sema/DelayedDiagnostic.cpp
index 876f9d7..3100432 100644
--- a/lib/Sema/DelayedDiagnostic.cpp
+++ b/lib/Sema/DelayedDiagnostic.cpp
@@ -22,6 +22,7 @@ using namespace sema;
 DelayedDiagnostic DelayedDiagnostic::makeDeprecation(SourceLocation Loc,
                                     const NamedDecl *D,
                                     const ObjCInterfaceDecl *UnknownObjCClass,
+                                    const ObjCPropertyDecl  *ObjCProperty,
                                     StringRef Msg) {
   DelayedDiagnostic DD;
   DD.Kind = Deprecation;
@@ -29,6 +30,7 @@ DelayedDiagnostic DelayedDiagnostic::makeDeprecation(SourceLocation Loc,
   DD.Loc = Loc;
   DD.DeprecationData.Decl = D;
   DD.DeprecationData.UnknownObjCClass = UnknownObjCClass;
+  DD.DeprecationData.ObjCProperty = ObjCProperty;
   char *MessageData = 0;
   if (Msg.size()) {
     MessageData = new char [Msg.size()];
diff --git a/lib/Sema/IdentifierResolver.cpp b/lib/Sema/IdentifierResolver.cpp
index 4d62cab..0093915 100644
--- a/lib/Sema/IdentifierResolver.cpp
+++ b/lib/Sema/IdentifierResolver.cpp
@@ -135,8 +135,16 @@ bool IdentifierResolver::isDeclInScope(Decl *D, DeclContext *Ctx,
       // of the controlled statement.
       //
       assert(S->getParent() && "No TUScope?");
-      if (S->getParent()->getFlags() & Scope::ControlScope)
+      if (S->getFlags() & Scope::FnTryScope)
         return S->getParent()->isDeclScope(D);
+      if (S->getParent()->getFlags() & Scope::ControlScope) {
+        if (S->getParent()->getFlags() & Scope::FnCatchScope) {
+          S = S->getParent();
+          if (S->isDeclScope(D))
+            return true;
+        }
+        return S->getParent()->isDeclScope(D);
+      }
     }
     return false;
   }
diff --git a/lib/Sema/JumpDiagnostics.cpp b/lib/Sema/JumpDiagnostics.cpp
index ab786c6..e2ec1cc 100644
--- a/lib/Sema/JumpDiagnostics.cpp
+++ b/lib/Sema/JumpDiagnostics.cpp
@@ -123,7 +123,7 @@ typedef std::pair<unsigned,unsigned> ScopePair;
 /// diagnostic that should be emitted if control goes over it. If not, return 0.
 static ScopePair GetDiagForGotoScopeDecl(ASTContext &Context, const Decl *D) {
   if (const VarDecl *VD = dyn_cast<VarDecl>(D)) {
-    unsigned InDiag = 0, OutDiag = 0;
+    unsigned InDiag = 0;
     if (VD->getType()->isVariablyModifiedType())
       InDiag = diag::note_protected_by_vla;
 
@@ -164,43 +164,53 @@ static ScopePair GetDiagForGotoScopeDecl(ASTContext &Context, const Decl *D) {
       //   where it is in scope is ill-formed unless the variable has
       //   POD type and is declared without an initializer.
 
-      if (const Expr *init = VD->getInit()) {
-        // We actually give variables of record type (or array thereof)
-        // an initializer even if that initializer only calls a trivial
-        // ctor.  Detect that case.
-        // FIXME: With generalized initializer lists, this may
-        // classify "X x{};" as having no initializer.
-        unsigned inDiagToUse = diag::note_protected_by_variable_init;
-
-        const CXXRecordDecl *record = 0;
-
-        if (const CXXConstructExpr *cce = dyn_cast<CXXConstructExpr>(init)) {
-          const CXXConstructorDecl *ctor = cce->getConstructor();
-          record = ctor->getParent();
-
-          if (ctor->isTrivial() && ctor->isDefaultConstructor()) {
-            if (!record->hasTrivialDestructor())
-              inDiagToUse = diag::note_protected_by_variable_nontriv_destructor;
-            else if (!record->isPOD())
-              inDiagToUse = diag::note_protected_by_variable_non_pod;
-            else
-              inDiagToUse = 0;
-          }
-        } else if (VD->getType()->isArrayType()) {
-          record = VD->getType()->getBaseElementTypeUnsafe()
-                                ->getAsCXXRecordDecl();
+      const Expr *Init = VD->getInit();
+      if (!Init)
+        return ScopePair(InDiag, 0);
+
+      const ExprWithCleanups *EWC = dyn_cast<ExprWithCleanups>(Init);
+      if (EWC)
+        Init = EWC->getSubExpr();
+
+      const MaterializeTemporaryExpr *M = NULL;
+      Init = Init->findMaterializedTemporary(M);
+
+      SmallVector<SubobjectAdjustment, 2> Adjustments;
+      Init = Init->skipRValueSubobjectAdjustments(Adjustments);
+
+      QualType QT = Init->getType();
+      if (QT.isNull())
+        return ScopePair(diag::note_protected_by_variable_init, 0);
+
+      const Type *T = QT.getTypePtr();
+      if (T->isArrayType())
+        T = T->getBaseElementTypeUnsafe();
+
+      const CXXRecordDecl *Record = T->getAsCXXRecordDecl();
+      if (!Record)
+        return ScopePair(diag::note_protected_by_variable_init, 0);
+
+      // If we need to call a non trivial destructor for this variable,
+      // record an out diagnostic.
+      unsigned OutDiag = 0;
+      if (!Init->isGLValue() && !Record->hasTrivialDestructor())
+        OutDiag = diag::note_exits_dtor;
+
+      if (const CXXConstructExpr *cce = dyn_cast<CXXConstructExpr>(Init)) {
+        const CXXConstructorDecl *ctor = cce->getConstructor();
+        if (ctor->isTrivial() && ctor->isDefaultConstructor()) {
+          if (OutDiag)
+            InDiag = diag::note_protected_by_variable_nontriv_destructor;
+          else if (!Record->isPOD())
+            InDiag = diag::note_protected_by_variable_non_pod;
+          return ScopePair(InDiag, OutDiag);
         }
-
-        if (inDiagToUse)
-          InDiag = inDiagToUse;
-
-        // Also object to indirect jumps which leave scopes with dtors.
-        if (record && !record->hasTrivialDestructor())
-          OutDiag = diag::note_exits_dtor;
       }
+
+      return ScopePair(diag::note_protected_by_variable_init, OutDiag);
     }
-    
-    return ScopePair(InDiag, OutDiag);    
+
+    return ScopePair(InDiag, 0);
   }
 
   if (const TypedefDecl *TD = dyn_cast<TypedefDecl>(D)) {
@@ -322,6 +332,29 @@ void JumpScopeChecker::BuildScopeInformation(Stmt *S, unsigned &origParentScope)
     Jumps.push_back(S);
     break;
 
+  case Stmt::CXXTryStmtClass: {
+    CXXTryStmt *TS = cast<CXXTryStmt>(S);
+    unsigned newParentScope;
+    Scopes.push_back(GotoScope(ParentScope,
+                               diag::note_protected_by_cxx_try,
+                               diag::note_exits_cxx_try,
+                               TS->getSourceRange().getBegin()));
+    if (Stmt *TryBlock = TS->getTryBlock())
+      BuildScopeInformation(TryBlock, (newParentScope = Scopes.size()-1));
+
+    // Jump from the catch into the try is not allowed either.
+    for (unsigned I = 0, E = TS->getNumHandlers(); I != E; ++I) {
+      CXXCatchStmt *CS = TS->getHandler(I);
+      Scopes.push_back(GotoScope(ParentScope,
+                                 diag::note_protected_by_cxx_catch,
+                                 diag::note_exits_cxx_catch,
+                                 CS->getSourceRange().getBegin()));
+      BuildScopeInformation(CS->getHandlerBlock(), 
+                            (newParentScope = Scopes.size()-1));
+    }
+    return;
+  }
+
   default:
     break;
   }
@@ -418,30 +451,6 @@ void JumpScopeChecker::BuildScopeInformation(Stmt *S, unsigned &origParentScope)
       continue;
     }
 
-    // Disallow jumps into any part of a C++ try statement. This is pretty
-    // much the same as for Obj-C.
-    if (CXXTryStmt *TS = dyn_cast<CXXTryStmt>(SubStmt)) {
-      Scopes.push_back(GotoScope(ParentScope,
-                                 diag::note_protected_by_cxx_try,
-                                 diag::note_exits_cxx_try,
-                                 TS->getSourceRange().getBegin()));
-      if (Stmt *TryBlock = TS->getTryBlock())
-        BuildScopeInformation(TryBlock, (newParentScope = Scopes.size()-1));
-
-      // Jump from the catch into the try is not allowed either.
-      for (unsigned I = 0, E = TS->getNumHandlers(); I != E; ++I) {
-        CXXCatchStmt *CS = TS->getHandler(I);
-        Scopes.push_back(GotoScope(ParentScope,
-                                   diag::note_protected_by_cxx_catch,
-                                   diag::note_exits_cxx_catch,
-                                   CS->getSourceRange().getBegin()));
-        BuildScopeInformation(CS->getHandlerBlock(), 
-                              (newParentScope = Scopes.size()-1));
-      }
-
-      continue;
-    }
-
     // Disallow jumps into the protected statement of an @autoreleasepool.
     if (ObjCAutoreleasePoolStmt *AS = dyn_cast<ObjCAutoreleasePoolStmt>(SubStmt)){
       // Recursively walk the AST for the @autoreleasepool part, protected by a new
@@ -453,14 +462,19 @@ void JumpScopeChecker::BuildScopeInformation(Stmt *S, unsigned &origParentScope)
       BuildScopeInformation(AS->getSubStmt(), (newParentScope = Scopes.size()-1));
       continue;
     }
-    
-    if (const BlockExpr *BE = dyn_cast<BlockExpr>(SubStmt)) {
-        const BlockDecl *BDecl = BE->getBlockDecl();
+
+    // Disallow jumps past full-expressions that use blocks with
+    // non-trivial cleanups of their captures.  This is theoretically
+    // implementable but a lot of work which we haven't felt up to doing.
+    if (ExprWithCleanups *EWC = dyn_cast<ExprWithCleanups>(SubStmt)) {
+      for (unsigned i = 0, e = EWC->getNumObjects(); i != e; ++i) {
+        const BlockDecl *BDecl = EWC->getObject(i);
         for (BlockDecl::capture_const_iterator ci = BDecl->capture_begin(),
              ce = BDecl->capture_end(); ci != ce; ++ci) {
           VarDecl *variable = ci->getVariable();
           BuildScopeInformation(variable, BDecl, ParentScope);
         }
+      }
     }
     
     // Recursively walk the AST.
diff --git a/lib/Sema/MultiplexExternalSemaSource.cpp b/lib/Sema/MultiplexExternalSemaSource.cpp
new file mode 100644
index 0000000..f930fb3
--- /dev/null
+++ b/lib/Sema/MultiplexExternalSemaSource.cpp
@@ -0,0 +1,271 @@
+//===--- MultiplexExternalSemaSource.cpp  ---------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the event dispatching to the subscribed clients.
+//
+//===----------------------------------------------------------------------===//
+#include "clang/Sema/MultiplexExternalSemaSource.h"
+
+#include "clang/AST/DeclContextInternals.h"
+#include "clang/Sema/Lookup.h"
+
+using namespace clang;
+
+///\brief Constructs a new multiplexing external sema source and appends the
+/// given element to it.
+///
+///\param[in] source - An ExternalSemaSource.
+///
+MultiplexExternalSemaSource::MultiplexExternalSemaSource(ExternalSemaSource &s1,
+                                                        ExternalSemaSource &s2){
+  Sources.push_back(&s1);
+  Sources.push_back(&s2);
+}
+
+// pin the vtable here.
+MultiplexExternalSemaSource::~MultiplexExternalSemaSource() {}
+
+///\brief Appends new source to the source list.
+///
+///\param[in] source - An ExternalSemaSource.
+///
+void MultiplexExternalSemaSource::addSource(ExternalSemaSource &source) {
+  Sources.push_back(&source);
+}
+
+//===----------------------------------------------------------------------===//
+// ExternalASTSource.
+//===----------------------------------------------------------------------===//
+
+Decl *MultiplexExternalSemaSource::GetExternalDecl(uint32_t ID) {
+  for(size_t i = 0; i < Sources.size(); ++i)
+    if (Decl *Result = Sources[i]->GetExternalDecl(ID))
+      return Result;
+  return 0;
+}
+
+Selector MultiplexExternalSemaSource::GetExternalSelector(uint32_t ID) {
+  Selector Sel;
+  for(size_t i = 0; i < Sources.size(); ++i) {
+    Sel = Sources[i]->GetExternalSelector(ID);
+    if (!Sel.isNull())
+      return Sel;
+  }
+  return Sel;
+}
+
+uint32_t MultiplexExternalSemaSource::GetNumExternalSelectors() {
+  uint32_t total = 0;
+  for(size_t i = 0; i < Sources.size(); ++i)
+    total += Sources[i]->GetNumExternalSelectors();
+  return total;
+}
+
+Stmt *MultiplexExternalSemaSource::GetExternalDeclStmt(uint64_t Offset) {
+  for(size_t i = 0; i < Sources.size(); ++i)
+    if (Stmt *Result = Sources[i]->GetExternalDeclStmt(Offset))
+      return Result;
+  return 0;
+}
+
+CXXBaseSpecifier *MultiplexExternalSemaSource::GetExternalCXXBaseSpecifiers(
+                                                               uint64_t Offset){
+  for(size_t i = 0; i < Sources.size(); ++i)
+    if (CXXBaseSpecifier *R = Sources[i]->GetExternalCXXBaseSpecifiers(Offset))
+      return R;
+  return 0; 
+}
+
+DeclContextLookupResult MultiplexExternalSemaSource::
+FindExternalVisibleDeclsByName(const DeclContext *DC, DeclarationName Name) {
+  StoredDeclsList DeclsFound;
+  DeclContextLookupResult lookup;
+  for(size_t i = 0; i < Sources.size(); ++i) {
+    lookup = Sources[i]->FindExternalVisibleDeclsByName(DC, Name);
+    while(lookup.first != lookup.second) {
+      if (!DeclsFound.HandleRedeclaration(*lookup.first))
+        DeclsFound.AddSubsequentDecl(*lookup.first);
+      lookup.first++;
+    }
+  }
+  return DeclsFound.getLookupResult(); 
+}
+
+void MultiplexExternalSemaSource::completeVisibleDeclsMap(const DeclContext *DC){
+  for(size_t i = 0; i < Sources.size(); ++i)
+    Sources[i]->completeVisibleDeclsMap(DC);
+}
+
+ExternalLoadResult MultiplexExternalSemaSource::
+FindExternalLexicalDecls(const DeclContext *DC,
+                         bool (*isKindWeWant)(Decl::Kind),
+                         SmallVectorImpl<Decl*> &Result) {
+  for(size_t i = 0; i < Sources.size(); ++i)
+    // FIXME: The semantics of the return result is unclear to me...
+    Sources[i]->FindExternalLexicalDecls(DC, isKindWeWant, Result);
+
+  return ELR_Success;
+}
+
+void MultiplexExternalSemaSource::FindFileRegionDecls(FileID File, 
+                                                      unsigned Offset,
+                                                      unsigned Length,
+                                                SmallVectorImpl<Decl *> &Decls){
+  for(size_t i = 0; i < Sources.size(); ++i)
+    Sources[i]->FindFileRegionDecls(File, Offset, Length, Decls);
+}
+
+void MultiplexExternalSemaSource::CompleteType(TagDecl *Tag) {
+  for(size_t i = 0; i < Sources.size(); ++i)
+    Sources[i]->CompleteType(Tag);
+}
+
+void MultiplexExternalSemaSource::CompleteType(ObjCInterfaceDecl *Class) {
+  for(size_t i = 0; i < Sources.size(); ++i)
+    Sources[i]->CompleteType(Class);
+}
+
+void MultiplexExternalSemaSource::ReadComments() {
+  for(size_t i = 0; i < Sources.size(); ++i)
+    Sources[i]->ReadComments();
+}
+
+void MultiplexExternalSemaSource::StartedDeserializing() {
+  for(size_t i = 0; i < Sources.size(); ++i)
+    Sources[i]->StartedDeserializing();
+}
+
+void MultiplexExternalSemaSource::FinishedDeserializing() {
+  for(size_t i = 0; i < Sources.size(); ++i)
+    Sources[i]->FinishedDeserializing();
+}
+
+void MultiplexExternalSemaSource::StartTranslationUnit(ASTConsumer *Consumer) {
+  for(size_t i = 0; i < Sources.size(); ++i)
+    Sources[i]->StartTranslationUnit(Consumer);
+}
+
+void MultiplexExternalSemaSource::PrintStats() {
+  for(size_t i = 0; i < Sources.size(); ++i)
+    Sources[i]->PrintStats();
+}
+
+bool MultiplexExternalSemaSource::layoutRecordType(const RecordDecl *Record,
+                                                   uint64_t &Size, 
+                                                   uint64_t &Alignment,
+                      llvm::DenseMap<const FieldDecl *, uint64_t> &FieldOffsets,
+                  llvm::DenseMap<const CXXRecordDecl *, CharUnits> &BaseOffsets,
+          llvm::DenseMap<const CXXRecordDecl *, CharUnits> &VirtualBaseOffsets){
+  for(size_t i = 0; i < Sources.size(); ++i)
+    if (Sources[i]->layoutRecordType(Record, Size, Alignment, FieldOffsets, 
+                                     BaseOffsets, VirtualBaseOffsets))
+      return true;
+  return false;
+}
+
+void MultiplexExternalSemaSource::
+getMemoryBufferSizes(MemoryBufferSizes &sizes) const {
+  for(size_t i = 0; i < Sources.size(); ++i)
+    Sources[i]->getMemoryBufferSizes(sizes);
+
+}
+
+//===----------------------------------------------------------------------===//
+// ExternalSemaSource.
+//===----------------------------------------------------------------------===//
+
+
+void MultiplexExternalSemaSource::InitializeSema(Sema &S) {
+  for(size_t i = 0; i < Sources.size(); ++i)
+    Sources[i]->InitializeSema(S);
+}
+
+void MultiplexExternalSemaSource::ForgetSema() {
+  for(size_t i = 0; i < Sources.size(); ++i)
+    Sources[i]->ForgetSema();
+}
+
+void MultiplexExternalSemaSource::ReadMethodPool(Selector Sel) {
+  for(size_t i = 0; i < Sources.size(); ++i)
+    Sources[i]->ReadMethodPool(Sel);
+}
+
+void MultiplexExternalSemaSource::ReadKnownNamespaces(
+                                   SmallVectorImpl<NamespaceDecl*> &Namespaces){
+  for(size_t i = 0; i < Sources.size(); ++i)
+    Sources[i]->ReadKnownNamespaces(Namespaces);
+}
+  
+bool MultiplexExternalSemaSource::LookupUnqualified(LookupResult &R, Scope *S){ 
+  for(size_t i = 0; i < Sources.size(); ++i)
+    Sources[i]->LookupUnqualified(R, S);
+  
+  return !R.empty();
+}
+
+void MultiplexExternalSemaSource::ReadTentativeDefinitions(
+                                     SmallVectorImpl<VarDecl*> &TentativeDefs) {
+  for(size_t i = 0; i < Sources.size(); ++i)
+    Sources[i]->ReadTentativeDefinitions(TentativeDefs);
+}
+  
+void MultiplexExternalSemaSource::ReadUnusedFileScopedDecls(
+                                SmallVectorImpl<const DeclaratorDecl*> &Decls) {
+  for(size_t i = 0; i < Sources.size(); ++i)
+    Sources[i]->ReadUnusedFileScopedDecls(Decls);
+}
+  
+void MultiplexExternalSemaSource::ReadDelegatingConstructors(
+                                  SmallVectorImpl<CXXConstructorDecl*> &Decls) {
+  for(size_t i = 0; i < Sources.size(); ++i)
+    Sources[i]->ReadDelegatingConstructors(Decls);
+}
+
+void MultiplexExternalSemaSource::ReadExtVectorDecls(
+                                     SmallVectorImpl<TypedefNameDecl*> &Decls) {
+  for(size_t i = 0; i < Sources.size(); ++i)
+    Sources[i]->ReadExtVectorDecls(Decls);
+}
+
+void MultiplexExternalSemaSource::ReadDynamicClasses(
+                                       SmallVectorImpl<CXXRecordDecl*> &Decls) {
+  for(size_t i = 0; i < Sources.size(); ++i)
+    Sources[i]->ReadDynamicClasses(Decls);
+}
+
+void MultiplexExternalSemaSource::ReadLocallyScopedExternalDecls(
+                                           SmallVectorImpl<NamedDecl*> &Decls) {
+  for(size_t i = 0; i < Sources.size(); ++i)
+    Sources[i]->ReadLocallyScopedExternalDecls(Decls);
+}
+
+void MultiplexExternalSemaSource::ReadReferencedSelectors(
+                  SmallVectorImpl<std::pair<Selector, SourceLocation> > &Sels) {
+  for(size_t i = 0; i < Sources.size(); ++i)
+    Sources[i]->ReadReferencedSelectors(Sels);
+}
+
+void MultiplexExternalSemaSource::ReadWeakUndeclaredIdentifiers(
+                   SmallVectorImpl<std::pair<IdentifierInfo*, WeakInfo> > &WI) {
+  for(size_t i = 0; i < Sources.size(); ++i)
+    Sources[i]->ReadWeakUndeclaredIdentifiers(WI);
+}
+
+void MultiplexExternalSemaSource::ReadUsedVTables(
+                                  SmallVectorImpl<ExternalVTableUse> &VTables) {
+  for(size_t i = 0; i < Sources.size(); ++i)
+    Sources[i]->ReadUsedVTables(VTables);
+}
+
+void MultiplexExternalSemaSource::ReadPendingInstantiations(
+                                           SmallVectorImpl<std::pair<ValueDecl*,
+                                                   SourceLocation> > &Pending) {
+  for(size_t i = 0; i < Sources.size(); ++i)
+    Sources[i]->ReadPendingInstantiations(Pending);
+}
diff --git a/lib/Sema/ScopeInfo.cpp b/lib/Sema/ScopeInfo.cpp
new file mode 100644
index 0000000..4d29a34
--- /dev/null
+++ b/lib/Sema/ScopeInfo.cpp
@@ -0,0 +1,189 @@
+//===--- ScopeInfo.cpp - Information about a semantic context -------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements FunctionScopeInfo and its subclasses, which contain
+// information about a single function, block, lambda, or method body.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Sema/ScopeInfo.h"
+#include "clang/AST/Decl.h"
+#include "clang/AST/DeclObjC.h"
+#include "clang/AST/Expr.h"
+#include "clang/AST/ExprCXX.h"
+#include "clang/AST/ExprObjC.h"
+
+using namespace clang;
+using namespace sema;
+
+void FunctionScopeInfo::Clear() {
+  HasBranchProtectedScope = false;
+  HasBranchIntoScope = false;
+  HasIndirectGoto = false;
+
+  SwitchStack.clear();
+  Returns.clear();
+  ErrorTrap.reset();
+  PossiblyUnreachableDiags.clear();
+  WeakObjectUses.clear();
+}
+
+static const NamedDecl *getBestPropertyDecl(const ObjCPropertyRefExpr *PropE) {
+  if (PropE->isExplicitProperty())
+    return PropE->getExplicitProperty();
+
+  return PropE->getImplicitPropertyGetter();
+}
+
+FunctionScopeInfo::WeakObjectProfileTy::BaseInfoTy
+FunctionScopeInfo::WeakObjectProfileTy::getBaseInfo(const Expr *E) {
+  E = E->IgnoreParenCasts();
+
+  const NamedDecl *D = 0;
+  bool IsExact = false;
+
+  switch (E->getStmtClass()) {
+  case Stmt::DeclRefExprClass:
+    D = cast<DeclRefExpr>(E)->getDecl();
+    IsExact = isa<VarDecl>(D);
+    break;
+  case Stmt::MemberExprClass: {
+    const MemberExpr *ME = cast<MemberExpr>(E);
+    D = ME->getMemberDecl();
+    IsExact = isa<CXXThisExpr>(ME->getBase()->IgnoreParenImpCasts());
+    break;
+  }
+  case Stmt::ObjCIvarRefExprClass: {
+    const ObjCIvarRefExpr *IE = cast<ObjCIvarRefExpr>(E);
+    D = IE->getDecl();
+    IsExact = IE->getBase()->isObjCSelfExpr();
+    break;
+  }
+  case Stmt::PseudoObjectExprClass: {
+    const PseudoObjectExpr *POE = cast<PseudoObjectExpr>(E);
+    const ObjCPropertyRefExpr *BaseProp =
+      dyn_cast<ObjCPropertyRefExpr>(POE->getSyntacticForm());
+    if (BaseProp) {
+      D = getBestPropertyDecl(BaseProp);
+
+      const Expr *DoubleBase = BaseProp->getBase();
+      if (const OpaqueValueExpr *OVE = dyn_cast<OpaqueValueExpr>(DoubleBase))
+        DoubleBase = OVE->getSourceExpr();
+
+      IsExact = DoubleBase->isObjCSelfExpr();
+    }
+    break;
+  }
+  default:
+    break;
+  }
+
+  return BaseInfoTy(D, IsExact);
+}
+
+
+FunctionScopeInfo::WeakObjectProfileTy::WeakObjectProfileTy(
+                                          const ObjCPropertyRefExpr *PropE)
+    : Base(0, true), Property(getBestPropertyDecl(PropE)) {
+
+  if (PropE->isObjectReceiver()) {
+    const OpaqueValueExpr *OVE = cast<OpaqueValueExpr>(PropE->getBase());
+    const Expr *E = OVE->getSourceExpr();
+    Base = getBaseInfo(E);
+  } else if (PropE->isClassReceiver()) {
+    Base.setPointer(PropE->getClassReceiver());
+  } else {
+    assert(PropE->isSuperReceiver());
+  }
+}
+
+FunctionScopeInfo::WeakObjectProfileTy::WeakObjectProfileTy(const Expr *BaseE,
+                                                const ObjCPropertyDecl *Prop)
+    : Base(0, true), Property(Prop) {
+  if (BaseE)
+    Base = getBaseInfo(BaseE);
+  // else, this is a message accessing a property on super.
+}
+
+FunctionScopeInfo::WeakObjectProfileTy::WeakObjectProfileTy(
+                                                      const DeclRefExpr *DRE)
+  : Base(0, true), Property(DRE->getDecl()) {
+  assert(isa<VarDecl>(Property));
+}
+
+FunctionScopeInfo::WeakObjectProfileTy::WeakObjectProfileTy(
+                                                  const ObjCIvarRefExpr *IvarE)
+  : Base(getBaseInfo(IvarE->getBase())), Property(IvarE->getDecl()) {
+}
+
+void FunctionScopeInfo::recordUseOfWeak(const ObjCMessageExpr *Msg,
+                                        const ObjCPropertyDecl *Prop) {
+  assert(Msg && Prop);
+  WeakUseVector &Uses =
+    WeakObjectUses[WeakObjectProfileTy(Msg->getInstanceReceiver(), Prop)];
+  Uses.push_back(WeakUseTy(Msg, Msg->getNumArgs() == 0));
+}
+
+void FunctionScopeInfo::markSafeWeakUse(const Expr *E) {
+  E = E->IgnoreParenCasts();
+
+  if (const PseudoObjectExpr *POE = dyn_cast<PseudoObjectExpr>(E)) {
+    markSafeWeakUse(POE->getSyntacticForm());
+    return;
+  }
+
+  if (const ConditionalOperator *Cond = dyn_cast<ConditionalOperator>(E)) {
+    markSafeWeakUse(Cond->getTrueExpr());
+    markSafeWeakUse(Cond->getFalseExpr());
+    return;
+  }
+
+  if (const BinaryConditionalOperator *Cond =
+        dyn_cast<BinaryConditionalOperator>(E)) {
+    markSafeWeakUse(Cond->getCommon());
+    markSafeWeakUse(Cond->getFalseExpr());
+    return;
+  }
+
+  // Has this weak object been seen before?
+  FunctionScopeInfo::WeakObjectUseMap::iterator Uses;
+  if (const ObjCPropertyRefExpr *RefExpr = dyn_cast<ObjCPropertyRefExpr>(E))
+    Uses = WeakObjectUses.find(WeakObjectProfileTy(RefExpr));
+  else if (const ObjCIvarRefExpr *IvarE = dyn_cast<ObjCIvarRefExpr>(E))
+    Uses = WeakObjectUses.find(WeakObjectProfileTy(IvarE));
+  else if (const DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(E))
+    Uses = WeakObjectUses.find(WeakObjectProfileTy(DRE));
+  else if (const ObjCMessageExpr *MsgE = dyn_cast<ObjCMessageExpr>(E)) {
+    Uses = WeakObjectUses.end();
+    if (const ObjCMethodDecl *MD = MsgE->getMethodDecl()) {
+      if (const ObjCPropertyDecl *Prop = MD->findPropertyDecl()) {
+        Uses =
+          WeakObjectUses.find(WeakObjectProfileTy(MsgE->getInstanceReceiver(),
+                                                  Prop));
+      }
+    }
+  }
+  else
+    return;
+
+  if (Uses == WeakObjectUses.end())
+    return;
+
+  // Has there been a read from the object using this Expr?
+  FunctionScopeInfo::WeakUseVector::reverse_iterator ThisUse =
+    std::find(Uses->second.rbegin(), Uses->second.rend(), WeakUseTy(E, true));
+  if (ThisUse == Uses->second.rend())
+    return;
+
+  ThisUse->markSafe();
+}
+
+FunctionScopeInfo::~FunctionScopeInfo() { }
+BlockScopeInfo::~BlockScopeInfo() { }
+LambdaScopeInfo::~LambdaScopeInfo() { }
diff --git a/lib/Sema/Sema.cpp b/lib/Sema/Sema.cpp
index 7f79f0c..13a33b7 100644
--- a/lib/Sema/Sema.cpp
+++ b/lib/Sema/Sema.cpp
@@ -22,6 +22,7 @@
 #include "clang/Sema/CXXFieldCollector.h"
 #include "clang/Sema/TemplateDeduction.h"
 #include "clang/Sema/ExternalSemaSource.h"
+#include "clang/Sema/MultiplexExternalSemaSource.h"
 #include "clang/Sema/ObjCMethodList.h"
 #include "clang/Sema/PrettyDeclStackTrace.h"
 #include "clang/Sema/Scope.h"
@@ -43,22 +44,6 @@
 using namespace clang;
 using namespace sema;
 
-FunctionScopeInfo::~FunctionScopeInfo() { }
-
-void FunctionScopeInfo::Clear() {
-  HasBranchProtectedScope = false;
-  HasBranchIntoScope = false;
-  HasIndirectGoto = false;
-  
-  SwitchStack.clear();
-  Returns.clear();
-  ErrorTrap.reset();
-  PossiblyUnreachableDiags.clear();
-}
-
-BlockScopeInfo::~BlockScopeInfo() { }
-LambdaScopeInfo::~LambdaScopeInfo() { }
-
 PrintingPolicy Sema::getPrintingPolicy(const ASTContext &Context,
                                        const Preprocessor &PP) {
   PrintingPolicy Policy = Context.getPrintingPolicy();
@@ -84,12 +69,14 @@ void Sema::ActOnTranslationUnitScope(Scope *S) {
 Sema::Sema(Preprocessor &pp, ASTContext &ctxt, ASTConsumer &consumer,
            TranslationUnitKind TUKind,
            CodeCompleteConsumer *CodeCompleter)
-  : TheTargetAttributesSema(0), FPFeatures(pp.getLangOpts()),
+  : TheTargetAttributesSema(0), ExternalSource(0), 
+    isMultiplexExternalSource(false), FPFeatures(pp.getLangOpts()),
     LangOpts(pp.getLangOpts()), PP(pp), Context(ctxt), Consumer(consumer),
     Diags(PP.getDiagnostics()), SourceMgr(PP.getSourceManager()),
-    CollectStats(false), ExternalSource(0), CodeCompleter(CodeCompleter),
+    CollectStats(false), CodeCompleter(CodeCompleter),
     CurContext(0), OriginalLexicalContext(0),
     PackContext(0), MSStructPragmaOn(false), VisContext(0),
+    IsBuildingRecoveryCallExpr(false),
     ExprNeedsCleanups(false), LateTemplateParser(0), OpaqueParser(0),
     IdResolver(pp), StdInitializerList(0), CXXTypeInfoDecl(0), MSVCGuidDecl(0),
     NSNumberDecl(0),
@@ -203,6 +190,10 @@ Sema::~Sema() {
   if (ExternalSemaSource *ExternalSema
         = dyn_cast_or_null<ExternalSemaSource>(Context.getExternalSource()))
     ExternalSema->ForgetSema();
+
+  // If Sema's ExternalSource is the multiplexer - we own it.
+  if (isMultiplexExternalSource)
+    delete ExternalSource;
 }
 
 /// makeUnavailableInSystemHeader - There is an error in the current
@@ -234,6 +225,27 @@ ASTMutationListener *Sema::getASTMutationListener() const {
   return getASTConsumer().GetASTMutationListener();
 }
 
+///\brief Registers an external source. If an external source already exists,
+/// creates a multiplex external source and appends to it.
+///
+///\param[in] E - A non-null external sema source.
+///
+void Sema::addExternalSource(ExternalSemaSource *E) {
+  assert(E && "Cannot use with NULL ptr");
+
+  if (!ExternalSource) {
+    ExternalSource = E;
+    return;
+  }
+
+  if (isMultiplexExternalSource)
+    static_cast<MultiplexExternalSemaSource*>(ExternalSource)->addSource(*E);
+  else {
+    ExternalSource = new MultiplexExternalSemaSource(*ExternalSource, *E);
+    isMultiplexExternalSource = true;
+  }
+}
+
 /// \brief Print out statistics about the semantic analysis.
 void Sema::PrintStats() const {
   llvm::errs() << "\n*** Semantic Analysis Stats:\n";
@@ -507,6 +519,11 @@ void Sema::ActOnEndOfTranslationUnit() {
   assert(DelayedDiagnostics.getCurrentPool() == NULL
          && "reached end of translation unit with a pool attached?");
 
+  // If code completion is enabled, don't perform any end-of-translation-unit
+  // work.
+  if (PP.isCodeCompletionEnabled())
+    return;
+
   // Only complete translation units define vtables and perform implicit
   // instantiations.
   if (TUKind == TU_Complete) {
@@ -648,6 +665,8 @@ void Sema::ActOnEndOfTranslationUnit() {
                                    diag::err_tentative_def_incomplete_type))
       VD->setInvalidDecl();
 
+    CheckCompleteVariableDeclaration(VD);
+
     // Notify the consumer that we've completed a tentative definition.
     if (!VD->isInvalidDecl())
       Consumer.CompleteTentativeDefinition(VD);
@@ -1021,6 +1040,9 @@ LambdaScopeInfo *Sema::getCurLambda() {
 }
 
 void Sema::ActOnComment(SourceRange Comment) {
+  if (!LangOpts.RetainCommentsFromSystemHeaders &&
+      SourceMgr.isInSystemHeader(Comment.getBegin()))
+    return;
   RawComment RC(SourceMgr, Comment);
   if (RC.isAlmostTrailingComment()) {
     SourceRange MagicMarkerRange(Comment.getBegin(),
@@ -1165,8 +1187,7 @@ static void noteOverloads(Sema &S, const UnresolvedSetImpl &Overloads,
        DeclsEnd = Overloads.end(); It != DeclsEnd; ++It) {
     // FIXME: Magic number for max shown overloads stolen from
     // OverloadCandidateSet::NoteCandidates.
-    if (ShownOverloads >= 4 &&
-        S.Diags.getShowOverloads() == DiagnosticsEngine::Ovl_Best) {
+    if (ShownOverloads >= 4 && S.Diags.getShowOverloads() == Ovl_Best) {
       ++SuppressedOverloads;
       continue;
     }
@@ -1237,8 +1258,7 @@ bool Sema::tryToRecoverWithCall(ExprResult &E, const PartialDiagnostic &PD,
     // FIXME: Try this before emitting the fixit, and suppress diagnostics
     // while doing so.
     E = ActOnCallExpr(0, E.take(), ParenInsertionLoc,
-                      MultiExprArg(*this, 0, 0),
-                      ParenInsertionLoc.getLocWithOffset(1));
+                      MultiExprArg(), ParenInsertionLoc.getLocWithOffset(1));
     return true;
   }
 
diff --git a/lib/Sema/SemaAccess.cpp b/lib/Sema/SemaAccess.cpp
index 3481171..58b1a51 100644
--- a/lib/Sema/SemaAccess.cpp
+++ b/lib/Sema/SemaAccess.cpp
@@ -97,14 +97,19 @@ struct EffectiveContext {
     // functions (which can gain privileges through friendship), but we
     // take that as an oversight.
     while (true) {
+      // We want to add canonical declarations to the EC lists for
+      // simplicity of checking, but we need to walk up through the
+      // actual current DC chain.  Otherwise, something like a local
+      // extern or friend which happens to be the canonical
+      // declaration will really mess us up.
+
       if (isa<CXXRecordDecl>(DC)) {
-        CXXRecordDecl *Record = cast<CXXRecordDecl>(DC)->getCanonicalDecl();
-        Records.push_back(Record);
+        CXXRecordDecl *Record = cast<CXXRecordDecl>(DC);
+        Records.push_back(Record->getCanonicalDecl());
         DC = Record->getDeclContext();
       } else if (isa<FunctionDecl>(DC)) {
-        FunctionDecl *Function = cast<FunctionDecl>(DC)->getCanonicalDecl();
-        Functions.push_back(Function);
-        
+        FunctionDecl *Function = cast<FunctionDecl>(DC);
+        Functions.push_back(Function->getCanonicalDecl());
         if (Function->getFriendObjectKind())
           DC = Function->getLexicalDeclContext();
         else
@@ -1791,7 +1796,7 @@ void Sema::CheckLookupAccess(const LookupResult &R) {
 /// specifiers into account, but no member access expressions and such.
 ///
 /// \param Decl the declaration to check if it can be accessed
-/// \param Class the class/context from which to start the search
+/// \param Ctx the class/context from which to start the search
 /// \return true if the Decl is accessible from the Class, false otherwise.
 bool Sema::IsSimplyAccessible(NamedDecl *Decl, DeclContext *Ctx) {
   if (CXXRecordDecl *Class = dyn_cast<CXXRecordDecl>(Ctx)) {
diff --git a/lib/Sema/SemaAttr.cpp b/lib/Sema/SemaAttr.cpp
index e935fc7..f1154c1 100644
--- a/lib/Sema/SemaAttr.cpp
+++ b/lib/Sema/SemaAttr.cpp
@@ -136,23 +136,12 @@ void Sema::AddMsStructLayoutForRecord(RecordDecl *RD) {
 }
 
 void Sema::ActOnPragmaOptionsAlign(PragmaOptionsAlignKind Kind,
-                                   SourceLocation PragmaLoc,
-                                   SourceLocation KindLoc) {
+                                   SourceLocation PragmaLoc) {
   if (PackContext == 0)
     PackContext = new PragmaPackStack();
 
   PragmaPackStack *Context = static_cast<PragmaPackStack*>(PackContext);
 
-  // Reset just pops the top of the stack, or resets the current alignment to
-  // default.
-  if (Kind == Sema::POAK_Reset) {
-    if (!Context->pop(0, /*IsReset=*/true)) {
-      Diag(PragmaLoc, diag::warn_pragma_options_align_reset_failed)
-        << "stack empty";
-    }
-    return;
-  }
-
   switch (Kind) {
     // For all targets we support native and natural are the same.
     //
@@ -181,9 +170,13 @@ void Sema::ActOnPragmaOptionsAlign(PragmaOptionsAlignKind Kind,
     Context->setAlignment(PackStackEntry::kMac68kAlignmentSentinel);
     break;
 
-  default:
-    Diag(PragmaLoc, diag::warn_pragma_options_align_unsupported_option)
-      << KindLoc;
+  case POAK_Reset:
+    // Reset just pops the top of the stack, or resets the current alignment to
+    // default.
+    if (!Context->pop(0, /*IsReset=*/true)) {
+      Diag(PragmaLoc, diag::warn_pragma_options_align_reset_failed)
+        << "stack empty";
+    }
     break;
   }
 }
diff --git a/lib/Sema/SemaCXXScopeSpec.cpp b/lib/Sema/SemaCXXScopeSpec.cpp
index 0de9dd5..15bfd1c 100644
--- a/lib/Sema/SemaCXXScopeSpec.cpp
+++ b/lib/Sema/SemaCXXScopeSpec.cpp
@@ -520,7 +520,8 @@ bool Sema::BuildCXXNestedNameSpecifier(Scope *S,
       if (LookupCtx)
         Diag(Found.getNameLoc(), diag::err_no_member_suggest)
           << Name << LookupCtx << CorrectedQuotedStr << SS.getRange()
-          << FixItHint::CreateReplacement(Found.getNameLoc(), CorrectedStr);
+          << FixItHint::CreateReplacement(Corrected.getCorrectionRange(),
+                                          CorrectedStr);
       else
         Diag(Found.getNameLoc(), diag::err_undeclared_var_use_suggest)
           << Name << CorrectedQuotedStr
diff --git a/lib/Sema/SemaCast.cpp b/lib/Sema/SemaCast.cpp
index d8d51e7..bf25c61 100644
--- a/lib/Sema/SemaCast.cpp
+++ b/lib/Sema/SemaCast.cpp
@@ -227,7 +227,7 @@ Sema::ActOnCXXNamedCast(SourceLocation OpLoc, tok::TokenKind Kind,
     CheckExtraCXXDefaultArguments(D);
   }
 
-  return BuildCXXNamedCast(OpLoc, Kind, TInfo, move(E),
+  return BuildCXXNamedCast(OpLoc, Kind, TInfo, E,
                            SourceRange(LAngleBracketLoc, RAngleBracketLoc),
                            SourceRange(LParenLoc, RParenLoc));
 }
@@ -1331,8 +1331,7 @@ TryStaticImplicitCast(Sema &Self, ExprResult &SrcExpr, QualType DestType,
   if (InitSeq.Failed() && (CStyle || !DestType->isReferenceType()))
     return TC_NotApplicable;
     
-  ExprResult Result
-    = InitSeq.Perform(Self, Entity, InitKind, MultiExprArg(Self, &SrcExprRaw, 1));
+  ExprResult Result = InitSeq.Perform(Self, Entity, InitKind, SrcExprRaw);
   if (Result.isInvalid()) {
     msg = 0;
     return TC_Failed;
@@ -1343,7 +1342,7 @@ TryStaticImplicitCast(Sema &Self, ExprResult &SrcExpr, QualType DestType,
   else
     Kind = CK_NoOp;
   
-  SrcExpr = move(Result);
+  SrcExpr = Result;
   return TC_Success;
 }
 
@@ -1492,6 +1491,22 @@ static void DiagnoseCastOfObjCSEL(Sema &Self, const ExprResult &SrcExpr,
     }
 }
 
+static void checkIntToPointerCast(bool CStyle, SourceLocation Loc,
+                                  const Expr *SrcExpr, QualType DestType,
+                                  Sema &Self) {
+  QualType SrcType = SrcExpr->getType();
+
+  // Not warning on reinterpret_cast, boolean, constant expressions, etc
+  // are not explicit design choices, but consistent with GCC's behavior.
+  // Feel free to modify them if you've reason/evidence for an alternative.
+  if (CStyle && SrcType->isIntegralType(Self.Context)
+      && !SrcType->isBooleanType()
+      && !SrcType->isEnumeralType()
+      && !SrcExpr->isIntegerConstantExpr(Self.Context)
+      && Self.Context.getTypeSize(DestType) > Self.Context.getTypeSize(SrcType))
+    Self.Diag(Loc, diag::warn_int_to_pointer_cast) << SrcType << DestType;
+}
+
 static TryCastResult TryReinterpretCast(Sema &Self, ExprResult &SrcExpr,
                                         QualType DestType, bool CStyle,
                                         const SourceRange &OpRange,
@@ -1513,7 +1528,7 @@ static TryCastResult TryReinterpretCast(Sema &Self, ExprResult &SrcExpr,
           SingleFunctionExpr,
           Expr::getValueKindForType(DestType) == VK_RValue // Convert Fun to Ptr 
         ) && SingleFunctionExpr.isUsable()) {
-      SrcExpr = move(SingleFunctionExpr);
+      SrcExpr = SingleFunctionExpr;
       SrcType = SrcExpr.get()->getType();
     } else {
       return TC_NotApplicable;
@@ -1690,6 +1705,8 @@ static TryCastResult TryReinterpretCast(Sema &Self, ExprResult &SrcExpr,
 
   if (SrcType->isIntegralOrEnumerationType()) {
     assert(destIsPtr && "One type must be a pointer");
+    checkIntToPointerCast(CStyle, OpRange.getBegin(), SrcExpr.get(), DestType,
+                          Self);
     // C++ 5.2.10p5: A value of integral or enumeration type can be explicitly
     //   converted to a pointer.
     // C++ 5.2.10p9: [Note: ...a null pointer constant of integral type is not
@@ -1903,6 +1920,43 @@ void CastOperation::CheckCXXCStyleCast(bool FunctionalStyle,
     SrcExpr = ExprError();
 }
 
+/// DiagnoseBadFunctionCast - Warn whenever a function call is cast to a 
+///  non-matching type. Such as enum function call to int, int call to
+/// pointer; etc. Cast to 'void' is an exception.
+static void DiagnoseBadFunctionCast(Sema &Self, const ExprResult &SrcExpr,
+                                  QualType DestType) {
+  if (Self.Diags.getDiagnosticLevel(diag::warn_bad_function_cast,
+                                    SrcExpr.get()->getExprLoc()) 
+        == DiagnosticsEngine::Ignored)
+    return;
+  
+  if (!isa<CallExpr>(SrcExpr.get()))
+    return;
+  
+  QualType SrcType = SrcExpr.get()->getType();
+  if (DestType.getUnqualifiedType()->isVoidType())
+    return;
+  if ((SrcType->isAnyPointerType() || SrcType->isBlockPointerType())
+      && (DestType->isAnyPointerType() || DestType->isBlockPointerType()))
+    return;
+  if (SrcType->isIntegerType() && DestType->isIntegerType() &&
+      (SrcType->isBooleanType() == DestType->isBooleanType()) &&
+      (SrcType->isEnumeralType() == DestType->isEnumeralType()))
+    return;
+  if (SrcType->isRealFloatingType() && DestType->isRealFloatingType())
+    return;
+  if (SrcType->isEnumeralType() && DestType->isEnumeralType())
+    return;
+  if (SrcType->isComplexType() && DestType->isComplexType())
+    return;
+  if (SrcType->isComplexIntegerType() && DestType->isComplexIntegerType())
+    return;
+  
+  Self.Diag(SrcExpr.get()->getExprLoc(),
+            diag::warn_bad_function_cast)
+            << SrcType << DestType << SrcExpr.get()->getSourceRange();
+}
+
 /// Check the semantics of a C-style cast operation, in C.
 void CastOperation::CheckCStyleCast() {
   assert(!Self.getLangOpts().CPlusPlus);
@@ -2035,6 +2089,8 @@ void CastOperation::CheckCStyleCast() {
       SrcExpr = ExprError();
       return;
     }
+    checkIntToPointerCast(/* CStyle */ true, OpRange.getBegin(), SrcExpr.get(),
+                          DestType, Self);
   } else if (!SrcType->isArithmeticType()) {
     if (!DestType->isIntegralType(Self.Context) &&
         DestType->isArithmeticType()) {
@@ -2076,7 +2132,7 @@ void CastOperation::CheckCStyleCast() {
     }
   }
   DiagnoseCastOfObjCSEL(Self, SrcExpr, DestType);
-  
+  DiagnoseBadFunctionCast(Self, SrcExpr, DestType);
   Kind = Self.PrepareScalarCast(SrcExpr, DestType);
   if (SrcExpr.isInvalid())
     return;
diff --git a/lib/Sema/SemaChecking.cpp b/lib/Sema/SemaChecking.cpp
index 2559f00..692a210 100644
--- a/lib/Sema/SemaChecking.cpp
+++ b/lib/Sema/SemaChecking.cpp
@@ -266,11 +266,11 @@ Sema::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
   case Builtin::BI__sync_swap_4:
   case Builtin::BI__sync_swap_8:
   case Builtin::BI__sync_swap_16:
-    return SemaBuiltinAtomicOverloaded(move(TheCallResult));
+    return SemaBuiltinAtomicOverloaded(TheCallResult);
 #define BUILTIN(ID, TYPE, ATTRS)
 #define ATOMIC_BUILTIN(ID, TYPE, ATTRS) \
   case Builtin::BI##ID: \
-    return SemaAtomicOpsOverloaded(move(TheCallResult), AtomicExpr::AO##ID);
+    return SemaAtomicOpsOverloaded(TheCallResult, AtomicExpr::AO##ID);
 #include "clang/Basic/Builtins.def"
   case Builtin::BI__builtin_annotation:
     if (SemaBuiltinAnnotation(*this, TheCall))
@@ -299,7 +299,7 @@ Sema::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
     }
   }
 
-  return move(TheCallResult);
+  return TheCallResult;
 }
 
 // Get the valid immediate range for the specified NEON type code.
@@ -437,6 +437,11 @@ bool Sema::CheckMipsBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
   default: return false;
   case Mips::BI__builtin_mips_wrdsp: i = 1; l = 0; u = 63; break;
   case Mips::BI__builtin_mips_rddsp: i = 0; l = 0; u = 63; break;
+  case Mips::BI__builtin_mips_append: i = 2; l = 0; u = 31; break;
+  case Mips::BI__builtin_mips_balign: i = 2; l = 0; u = 3; break;
+  case Mips::BI__builtin_mips_precr_sra_ph_w: i = 2; l = 0; u = 31; break;
+  case Mips::BI__builtin_mips_precr_sra_r_ph_w: i = 2; l = 0; u = 31; break;
+  case Mips::BI__builtin_mips_prepend: i = 2; l = 0; u = 31; break;
   };
 
   // We can't check the value of a dependent argument.
@@ -490,9 +495,8 @@ void Sema::checkCall(NamedDecl *FDecl, Expr **Args,
                      SourceLocation Loc,
                      SourceRange Range,
                      VariadicCallType CallType) {
-  // FIXME: This mechanism should be abstracted to be less fragile and
-  // more efficient. For example, just map function ids to custom
-  // handlers.
+  if (CurContext->isDependentContext())
+    return;
 
   // Printf and scanf checking.
   bool HandledFormatString = false;
@@ -506,8 +510,11 @@ void Sema::checkCall(NamedDecl *FDecl, Expr **Args,
   // Refuse POD arguments that weren't caught by the format string
   // checks above.
   if (!HandledFormatString && CallType != VariadicDoesNotApply)
-    for (unsigned ArgIdx = NumProtoArgs; ArgIdx < NumArgs; ++ArgIdx)
-      variadicArgumentPODCheck(Args[ArgIdx], CallType);
+    for (unsigned ArgIdx = NumProtoArgs; ArgIdx < NumArgs; ++ArgIdx) {
+      // Args[ArgIdx] can be null in malformed code.
+      if (Expr *Arg = Args[ArgIdx])
+        variadicArgumentPODCheck(Arg, CallType);
+    }
 
   for (specific_attr_iterator<NonNullAttr>
          I = FDecl->specific_attr_begin<NonNullAttr>(),
@@ -538,11 +545,23 @@ void Sema::CheckConstructorCall(FunctionDecl *FDecl, Expr **Args,
 /// and safety properties not strictly enforced by the C type system.
 bool Sema::CheckFunctionCall(FunctionDecl *FDecl, CallExpr *TheCall,
                              const FunctionProtoType *Proto) {
-  bool IsMemberFunction = isa<CXXMemberCallExpr>(TheCall);
+  bool IsMemberOperatorCall = isa<CXXOperatorCallExpr>(TheCall) &&
+                              isa<CXXMethodDecl>(FDecl);
+  bool IsMemberFunction = isa<CXXMemberCallExpr>(TheCall) ||
+                          IsMemberOperatorCall;
   VariadicCallType CallType = getVariadicCallType(FDecl, Proto,
                                                   TheCall->getCallee());
   unsigned NumProtoArgs = Proto ? Proto->getNumArgs() : 0;
-  checkCall(FDecl, TheCall->getArgs(), TheCall->getNumArgs(), NumProtoArgs,
+  Expr** Args = TheCall->getArgs();
+  unsigned NumArgs = TheCall->getNumArgs();
+  if (IsMemberOperatorCall) {
+    // If this is a call to a member operator, hide the first argument
+    // from checkCall.
+    // FIXME: Our choice of AST representation here is less than ideal.
+    ++Args;
+    --NumArgs;
+  }
+  checkCall(FDecl, Args, NumArgs, NumProtoArgs,
             IsMemberFunction, TheCall->getRParenLoc(),
             TheCall->getCallee()->getSourceRange(), CallType);
 
@@ -737,6 +756,11 @@ ExprResult Sema::SemaAtomicOpsOverloaded(ExprResult TheCallResult,
         << Ptr->getType() << Ptr->getSourceRange();
       return ExprError();
     }
+    if (AtomTy.isConstQualified()) {
+      Diag(DRE->getLocStart(), diag::err_atomic_op_needs_non_const_atomic)
+        << Ptr->getType() << Ptr->getSourceRange();
+      return ExprError();
+    }
     ValType = AtomTy->getAs<AtomicType>()->getValueType();
   }
 
@@ -885,8 +909,7 @@ ExprResult Sema::SemaAtomicOpsOverloaded(ExprResult TheCallResult,
   }
 
   return Owned(new (Context) AtomicExpr(TheCall->getCallee()->getLocStart(),
-                                        SubExprs.data(), SubExprs.size(),
-                                        ResultType, Op,
+                                        SubExprs, ResultType, Op,
                                         TheCall->getRParenLoc()));
 }
 
@@ -1189,10 +1212,19 @@ Sema::SemaBuiltinAtomicOverloaded(ExprResult TheCallResult) {
   // concrete integer type we should convert to is.
   unsigned NewBuiltinID = BuiltinIndices[BuiltinIndex][SizeIndex];
   const char *NewBuiltinName = Context.BuiltinInfo.GetName(NewBuiltinID);
-  IdentifierInfo *NewBuiltinII = PP.getIdentifierInfo(NewBuiltinName);
-  FunctionDecl *NewBuiltinDecl =
-    cast<FunctionDecl>(LazilyCreateBuiltin(NewBuiltinII, NewBuiltinID,
-                                           TUScope, false, DRE->getLocStart()));
+  FunctionDecl *NewBuiltinDecl;
+  if (NewBuiltinID == BuiltinID)
+    NewBuiltinDecl = FDecl;
+  else {
+    // Perform builtin lookup to avoid redeclaring it.
+    DeclarationName DN(&Context.Idents.get(NewBuiltinName));
+    LookupResult Res(*this, DN, DRE->getLocStart(), LookupOrdinaryName);
+    LookupName(Res, TUScope, /*AllowBuiltinCreation=*/true);
+    assert(Res.getFoundDecl());
+    NewBuiltinDecl = dyn_cast<FunctionDecl>(Res.getFoundDecl());
+    if (NewBuiltinDecl == 0)
+      return ExprError();
+  }
 
   // The first argument --- the pointer --- has a fixed type; we
   // deduce the types of the rest of the arguments accordingly.  Walk
@@ -1228,14 +1260,14 @@ Sema::SemaBuiltinAtomicOverloaded(ExprResult TheCallResult) {
       NewBuiltinDecl,
       /*enclosing*/ false,
       DRE->getLocation(),
-      NewBuiltinDecl->getType(),
+      Context.BuiltinFnTy,
       DRE->getValueKind());
 
   // Set the callee in the CallExpr.
-  // FIXME: This leaks the original parens and implicit casts.
-  ExprResult PromotedCall = UsualUnaryConversions(NewDRE);
-  if (PromotedCall.isInvalid())
-    return ExprError();
+  // FIXME: This loses syntactic information.
+  QualType CalleePtrTy = Context.getPointerType(NewBuiltinDecl->getType());
+  ExprResult PromotedCall = ImpCastExprToType(NewDRE, CalleePtrTy,
+                                              CK_BuiltinFnToFnPtr);
   TheCall->setCallee(PromotedCall.take());
 
   // Change the result type of the call to match the original value type. This
@@ -1243,7 +1275,7 @@ Sema::SemaBuiltinAtomicOverloaded(ExprResult TheCallResult) {
   // gracefully.
   TheCall->setType(ResultType);
 
-  return move(TheCallResult);
+  return TheCallResult;
 }
 
 /// CheckObjCString - Checks that the argument to the builtin
@@ -1264,7 +1296,7 @@ bool Sema::CheckObjCString(Expr *Arg) {
     StringRef String = Literal->getString();
     unsigned NumBytes = String.size();
     SmallVector<UTF16, 128> ToBuf(NumBytes);
-    const UTF8 *FromPtr = (UTF8 *)String.data();
+    const UTF8 *FromPtr = (const UTF8 *)String.data();
     UTF16 *ToPtr = &ToBuf[0];
     
     ConversionResult Result = ConvertUTF8toUTF16(&FromPtr, FromPtr + NumBytes,
@@ -1503,8 +1535,7 @@ ExprResult Sema::SemaBuiltinShuffleVector(CallExpr *TheCall) {
     TheCall->setArg(i, 0);
   }
 
-  return Owned(new (Context) ShuffleVectorExpr(Context, exprs.begin(),
-                                            exprs.size(), resType,
+  return Owned(new (Context) ShuffleVectorExpr(Context, exprs, resType,
                                             TheCall->getCallee()->getLocStart(),
                                             TheCall->getRParenLoc()));
 }
@@ -1935,19 +1966,19 @@ public:
   void HandleIncompleteSpecifier(const char *startSpecifier,
                                  unsigned specifierLen);
 
+  void HandleInvalidLengthModifier(
+      const analyze_format_string::FormatSpecifier &FS,
+      const analyze_format_string::ConversionSpecifier &CS,
+      const char *startSpecifier, unsigned specifierLen, unsigned DiagID);
+
   void HandleNonStandardLengthModifier(
-      const analyze_format_string::LengthModifier &LM,
+      const analyze_format_string::FormatSpecifier &FS,
       const char *startSpecifier, unsigned specifierLen);
 
   void HandleNonStandardConversionSpecifier(
       const analyze_format_string::ConversionSpecifier &CS,
       const char *startSpecifier, unsigned specifierLen);
 
-  void HandleNonStandardConversionSpecification(
-      const analyze_format_string::LengthModifier &LM,
-      const analyze_format_string::ConversionSpecifier &CS,
-      const char *startSpecifier, unsigned specifierLen);
-
   virtual void HandlePosition(const char *startPos, unsigned posLen);
 
   virtual void HandleInvalidPosition(const char *startSpecifier,
@@ -1964,7 +1995,7 @@ public:
                                    PartialDiagnostic PDiag,
                                    SourceLocation StringLoc,
                                    bool IsStringLocation, Range StringRange,
-                                   FixItHint Fixit = FixItHint());
+                            ArrayRef<FixItHint> Fixit = ArrayRef<FixItHint>());
 
 protected:
   bool HandleInvalidConversionSpecifier(unsigned argIndex, SourceLocation Loc,
@@ -1991,7 +2022,7 @@ protected:
   template <typename Range>
   void EmitFormatDiagnostic(PartialDiagnostic PDiag, SourceLocation StringLoc,
                             bool IsStringLocation, Range StringRange,
-                            FixItHint Fixit = FixItHint());
+                            ArrayRef<FixItHint> Fixit = ArrayRef<FixItHint>());
 
   void CheckPositionalAndNonpositionalArgs(
       const analyze_format_string::FormatSpecifier *FS);
@@ -2025,35 +2056,95 @@ void CheckFormatHandler::HandleIncompleteSpecifier(const char *startSpecifier,
                        getSpecifierRange(startSpecifier, specifierLen));
 }
 
+void CheckFormatHandler::HandleInvalidLengthModifier(
+    const analyze_format_string::FormatSpecifier &FS,
+    const analyze_format_string::ConversionSpecifier &CS,
+    const char *startSpecifier, unsigned specifierLen, unsigned DiagID) {
+  using namespace analyze_format_string;
+
+  const LengthModifier &LM = FS.getLengthModifier();
+  CharSourceRange LMRange = getSpecifierRange(LM.getStart(), LM.getLength());
+
+  // See if we know how to fix this length modifier.
+  llvm::Optional<LengthModifier> FixedLM = FS.getCorrectedLengthModifier();
+  if (FixedLM) {
+    EmitFormatDiagnostic(S.PDiag(DiagID) << LM.toString() << CS.toString(),
+                         getLocationOfByte(LM.getStart()),
+                         /*IsStringLocation*/true,
+                         getSpecifierRange(startSpecifier, specifierLen));
+
+    S.Diag(getLocationOfByte(LM.getStart()), diag::note_format_fix_specifier)
+      << FixedLM->toString()
+      << FixItHint::CreateReplacement(LMRange, FixedLM->toString());
+
+  } else {
+    FixItHint Hint;
+    if (DiagID == diag::warn_format_nonsensical_length)
+      Hint = FixItHint::CreateRemoval(LMRange);
+
+    EmitFormatDiagnostic(S.PDiag(DiagID) << LM.toString() << CS.toString(),
+                         getLocationOfByte(LM.getStart()),
+                         /*IsStringLocation*/true,
+                         getSpecifierRange(startSpecifier, specifierLen),
+                         Hint);
+  }
+}
+
 void CheckFormatHandler::HandleNonStandardLengthModifier(
-    const analyze_format_string::LengthModifier &LM,
+    const analyze_format_string::FormatSpecifier &FS,
     const char *startSpecifier, unsigned specifierLen) {
-  EmitFormatDiagnostic(S.PDiag(diag::warn_format_non_standard) << LM.toString()
-                       << 0,
-                       getLocationOfByte(LM.getStart()),
-                       /*IsStringLocation*/true,
-                       getSpecifierRange(startSpecifier, specifierLen));
+  using namespace analyze_format_string;
+
+  const LengthModifier &LM = FS.getLengthModifier();
+  CharSourceRange LMRange = getSpecifierRange(LM.getStart(), LM.getLength());
+
+  // See if we know how to fix this length modifier.
+  llvm::Optional<LengthModifier> FixedLM = FS.getCorrectedLengthModifier();
+  if (FixedLM) {
+    EmitFormatDiagnostic(S.PDiag(diag::warn_format_non_standard)
+                           << LM.toString() << 0,
+                         getLocationOfByte(LM.getStart()),
+                         /*IsStringLocation*/true,
+                         getSpecifierRange(startSpecifier, specifierLen));
+
+    S.Diag(getLocationOfByte(LM.getStart()), diag::note_format_fix_specifier)
+      << FixedLM->toString()
+      << FixItHint::CreateReplacement(LMRange, FixedLM->toString());
+
+  } else {
+    EmitFormatDiagnostic(S.PDiag(diag::warn_format_non_standard)
+                           << LM.toString() << 0,
+                         getLocationOfByte(LM.getStart()),
+                         /*IsStringLocation*/true,
+                         getSpecifierRange(startSpecifier, specifierLen));
+  }
 }
 
 void CheckFormatHandler::HandleNonStandardConversionSpecifier(
     const analyze_format_string::ConversionSpecifier &CS,
     const char *startSpecifier, unsigned specifierLen) {
-  EmitFormatDiagnostic(S.PDiag(diag::warn_format_non_standard) << CS.toString()
-                       << 1,
-                       getLocationOfByte(CS.getStart()),
-                       /*IsStringLocation*/true,
-                       getSpecifierRange(startSpecifier, specifierLen));
-}
+  using namespace analyze_format_string;
 
-void CheckFormatHandler::HandleNonStandardConversionSpecification(
-    const analyze_format_string::LengthModifier &LM,
-    const analyze_format_string::ConversionSpecifier &CS,
-    const char *startSpecifier, unsigned specifierLen) {
-  EmitFormatDiagnostic(S.PDiag(diag::warn_format_non_standard_conversion_spec)
-                       << LM.toString() << CS.toString(),
-                       getLocationOfByte(LM.getStart()),
-                       /*IsStringLocation*/true,
-                       getSpecifierRange(startSpecifier, specifierLen));
+  // See if we know how to fix this conversion specifier.
+  llvm::Optional<ConversionSpecifier> FixedCS = CS.getStandardSpecifier();
+  if (FixedCS) {
+    EmitFormatDiagnostic(S.PDiag(diag::warn_format_non_standard)
+                          << CS.toString() << /*conversion specifier*/1,
+                         getLocationOfByte(CS.getStart()),
+                         /*IsStringLocation*/true,
+                         getSpecifierRange(startSpecifier, specifierLen));
+
+    CharSourceRange CSRange = getSpecifierRange(CS.getStart(), CS.getLength());
+    S.Diag(getLocationOfByte(CS.getStart()), diag::note_format_fix_specifier)
+      << FixedCS->toString()
+      << FixItHint::CreateReplacement(CSRange, FixedCS->toString());
+  } else {
+    EmitFormatDiagnostic(S.PDiag(diag::warn_format_non_standard)
+                          << CS.toString() << /*conversion specifier*/1,
+                         getLocationOfByte(CS.getStart()),
+                         /*IsStringLocation*/true,
+                         getSpecifierRange(startSpecifier, specifierLen));
+  }
 }
 
 void CheckFormatHandler::HandlePosition(const char *startPos,
@@ -2182,7 +2273,7 @@ void CheckFormatHandler::EmitFormatDiagnostic(PartialDiagnostic PDiag,
                                               SourceLocation Loc,
                                               bool IsStringLocation,
                                               Range StringRange,
-                                              FixItHint FixIt) {
+                                              ArrayRef<FixItHint> FixIt) {
   EmitFormatDiagnostic(S, inFunctionCall, Args[FormatIdx], PDiag,
                        Loc, IsStringLocation, StringRange, FixIt);
 }
@@ -2190,7 +2281,7 @@ void CheckFormatHandler::EmitFormatDiagnostic(PartialDiagnostic PDiag,
 /// \brief If the format string is not within the funcion call, emit a note
 /// so that the function call and string are in diagnostic messages.
 ///
-/// \param inFunctionCall if true, the format string is within the function
+/// \param InFunctionCall if true, the format string is within the function
 /// call and only one diagnostic message will be produced.  Otherwise, an
 /// extra note will be emitted pointing to location of the format string.
 ///
@@ -2213,7 +2304,7 @@ void CheckFormatHandler::EmitFormatDiagnostic(PartialDiagnostic PDiag,
 /// \param StringRange some or all of the string to highlight.  This is
 /// templated so it can accept either a CharSourceRange or a SourceRange.
 ///
-/// \param Fixit optional fix it hint for the format string.
+/// \param FixIt optional fix it hint for the format string.
 template<typename Range>
 void CheckFormatHandler::EmitFormatDiagnostic(Sema &S, bool InFunctionCall,
                                               const Expr *ArgumentExpr,
@@ -2221,15 +2312,27 @@ void CheckFormatHandler::EmitFormatDiagnostic(Sema &S, bool InFunctionCall,
                                               SourceLocation Loc,
                                               bool IsStringLocation,
                                               Range StringRange,
-                                              FixItHint FixIt) {
-  if (InFunctionCall)
-    S.Diag(Loc, PDiag) << StringRange << FixIt;
-  else {
+                                              ArrayRef<FixItHint> FixIt) {
+  if (InFunctionCall) {
+    const Sema::SemaDiagnosticBuilder &D = S.Diag(Loc, PDiag);
+    D << StringRange;
+    for (ArrayRef<FixItHint>::iterator I = FixIt.begin(), E = FixIt.end();
+         I != E; ++I) {
+      D << *I;
+    }
+  } else {
     S.Diag(IsStringLocation ? ArgumentExpr->getExprLoc() : Loc, PDiag)
       << ArgumentExpr->getSourceRange();
-    S.Diag(IsStringLocation ? Loc : StringRange.getBegin(),
-           diag::note_format_string_defined)
-      << StringRange << FixIt;
+
+    const Sema::SemaDiagnosticBuilder &Note =
+      S.Diag(IsStringLocation ? Loc : StringRange.getBegin(),
+             diag::note_format_string_defined);
+
+    Note << StringRange;
+    for (ArrayRef<FixItHint>::iterator I = FixIt.begin(), E = FixIt.end();
+         I != E; ++I) {
+      Note << *I;
+    }
   }
 }
 
@@ -2550,23 +2653,17 @@ CheckPrintfHandler::HandlePrintfSpecifier(const analyze_printf::PrintfSpecifier
             startSpecifier, specifierLen);
 
   // Check the length modifier is valid with the given conversion specifier.
-  const LengthModifier &LM = FS.getLengthModifier();
-  if (!FS.hasValidLengthModifier())
-    EmitFormatDiagnostic(S.PDiag(diag::warn_format_nonsensical_length)
-                           << LM.toString() << CS.toString(),
-                         getLocationOfByte(LM.getStart()),
-                         /*IsStringLocation*/true,
-                         getSpecifierRange(startSpecifier, specifierLen),
-                         FixItHint::CreateRemoval(
-                           getSpecifierRange(LM.getStart(),
-                                             LM.getLength())));
-  if (!FS.hasStandardLengthModifier())
-    HandleNonStandardLengthModifier(LM, startSpecifier, specifierLen);
+  if (!FS.hasValidLengthModifier(S.getASTContext().getTargetInfo()))
+    HandleInvalidLengthModifier(FS, CS, startSpecifier, specifierLen,
+                                diag::warn_format_nonsensical_length);
+  else if (!FS.hasStandardLengthModifier())
+    HandleNonStandardLengthModifier(FS, startSpecifier, specifierLen);
+  else if (!FS.hasStandardLengthConversionCombination())
+    HandleInvalidLengthModifier(FS, CS, startSpecifier, specifierLen,
+                                diag::warn_format_non_standard_conversion_spec);
+
   if (!FS.hasStandardConversionSpecifier(S.getLangOpts()))
     HandleNonStandardConversionSpecifier(CS, startSpecifier, specifierLen);
-  if (!FS.hasStandardLengthConversionCombination())
-    HandleNonStandardConversionSpecification(LM, CS, startSpecifier,
-                                             specifierLen);
 
   // The remaining checks depend on the data arguments.
   if (HasVAListArg)
@@ -2582,6 +2679,30 @@ CheckPrintfHandler::HandlePrintfSpecifier(const analyze_printf::PrintfSpecifier
   return checkFormatExpr(FS, startSpecifier, specifierLen, Arg);
 }
 
+static bool requiresParensToAddCast(const Expr *E) {
+  // FIXME: We should have a general way to reason about operator
+  // precedence and whether parens are actually needed here.
+  // Take care of a few common cases where they aren't.
+  const Expr *Inside = E->IgnoreImpCasts();
+  if (const PseudoObjectExpr *POE = dyn_cast<PseudoObjectExpr>(Inside))
+    Inside = POE->getSyntacticForm()->IgnoreImpCasts();
+
+  switch (Inside->getStmtClass()) {
+  case Stmt::ArraySubscriptExprClass:
+  case Stmt::CallExprClass:
+  case Stmt::DeclRefExprClass:
+  case Stmt::MemberExprClass:
+  case Stmt::ObjCIvarRefExprClass:
+  case Stmt::ObjCMessageExprClass:
+  case Stmt::ObjCPropertyRefExprClass:
+  case Stmt::ParenExprClass:
+  case Stmt::UnaryOperatorClass:
+    return false;
+  default:
+    return true;
+  }
+}
+
 bool
 CheckPrintfHandler::checkFormatExpr(const analyze_printf::PrintfSpecifier &FS,
                                     const char *StartSpecifier,
@@ -2593,81 +2714,151 @@ CheckPrintfHandler::checkFormatExpr(const analyze_printf::PrintfSpecifier &FS,
   // format specifier.
   const analyze_printf::ArgType &AT = FS.getArgType(S.Context,
                                                     ObjCContext);
-  if (AT.isValid() && !AT.matchesType(S.Context, E->getType())) {
-    // Look through argument promotions for our error message's reported type.
-    // This includes the integral and floating promotions, but excludes array
-    // and function pointer decay; seeing that an argument intended to be a
-    // string has type 'char [6]' is probably more confusing than 'char *'.
-    if (const ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(E)) {
-      if (ICE->getCastKind() == CK_IntegralCast ||
-          ICE->getCastKind() == CK_FloatingCast) {
-        E = ICE->getSubExpr();
-
-        // Check if we didn't match because of an implicit cast from a 'char'
-        // or 'short' to an 'int'.  This is done because printf is a varargs
-        // function.
-        if (ICE->getType() == S.Context.IntTy ||
-            ICE->getType() == S.Context.UnsignedIntTy) {
-          // All further checking is done on the subexpression.
-          if (AT.matchesType(S.Context, E->getType()))
-            return true;
-        }
+  if (!AT.isValid())
+    return true;
+
+  QualType IntendedTy = E->getType();
+  if (AT.matchesType(S.Context, IntendedTy))
+    return true;
+
+  // Look through argument promotions for our error message's reported type.
+  // This includes the integral and floating promotions, but excludes array
+  // and function pointer decay; seeing that an argument intended to be a
+  // string has type 'char [6]' is probably more confusing than 'char *'.
+  if (const ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(E)) {
+    if (ICE->getCastKind() == CK_IntegralCast ||
+        ICE->getCastKind() == CK_FloatingCast) {
+      E = ICE->getSubExpr();
+      IntendedTy = E->getType();
+
+      // Check if we didn't match because of an implicit cast from a 'char'
+      // or 'short' to an 'int'.  This is done because printf is a varargs
+      // function.
+      if (ICE->getType() == S.Context.IntTy ||
+          ICE->getType() == S.Context.UnsignedIntTy) {
+        // All further checking is done on the subexpression.
+        if (AT.matchesType(S.Context, IntendedTy))
+          return true;
       }
     }
+  }
 
-    // We may be able to offer a FixItHint if it is a supported type.
-    PrintfSpecifier fixedFS = FS;
-    bool success = fixedFS.fixType(E->getType(), S.getLangOpts(),
-                                   S.Context, ObjCContext);
+  if (S.Context.getTargetInfo().getTriple().isOSDarwin()) {
+    // Special-case some of Darwin's platform-independence types.
+    if (const TypedefType *UserTy = IntendedTy->getAs<TypedefType>()) {
+      StringRef Name = UserTy->getDecl()->getName();
+      IntendedTy = llvm::StringSwitch<QualType>(Name)
+        .Case("NSInteger", S.Context.LongTy)
+        .Case("NSUInteger", S.Context.UnsignedLongTy)
+        .Case("SInt32", S.Context.IntTy)
+        .Case("UInt32", S.Context.UnsignedIntTy)
+        .Default(IntendedTy);
+    }
+  }
 
-    if (success) {
-      // Get the fix string from the fixed format specifier
-      SmallString<16> buf;
-      llvm::raw_svector_ostream os(buf);
-      fixedFS.toString(os);
+  // We may be able to offer a FixItHint if it is a supported type.
+  PrintfSpecifier fixedFS = FS;
+  bool success = fixedFS.fixType(IntendedTy, S.getLangOpts(),
+                                 S.Context, ObjCContext);
+
+  if (success) {
+    // Get the fix string from the fixed format specifier
+    SmallString<16> buf;
+    llvm::raw_svector_ostream os(buf);
+    fixedFS.toString(os);
+
+    CharSourceRange SpecRange = getSpecifierRange(StartSpecifier, SpecifierLen);
+
+    if (IntendedTy != E->getType()) {
+      // The canonical type for formatting this value is different from the
+      // actual type of the expression. (This occurs, for example, with Darwin's
+      // NSInteger on 32-bit platforms, where it is typedef'd as 'int', but
+      // should be printed as 'long' for 64-bit compatibility.)
+      // Rather than emitting a normal format/argument mismatch, we want to
+      // add a cast to the recommended type (and correct the format string
+      // if necessary).
+      SmallString<16> CastBuf;
+      llvm::raw_svector_ostream CastFix(CastBuf);
+      CastFix << "(";
+      IntendedTy.print(CastFix, S.Context.getPrintingPolicy());
+      CastFix << ")";
+
+      SmallVector<FixItHint,4> Hints;
+      if (!AT.matchesType(S.Context, IntendedTy))
+        Hints.push_back(FixItHint::CreateReplacement(SpecRange, os.str()));
+
+      if (const CStyleCastExpr *CCast = dyn_cast<CStyleCastExpr>(E)) {
+        // If there's already a cast present, just replace it.
+        SourceRange CastRange(CCast->getLParenLoc(), CCast->getRParenLoc());
+        Hints.push_back(FixItHint::CreateReplacement(CastRange, CastFix.str()));
+
+      } else if (!requiresParensToAddCast(E)) {
+        // If the expression has high enough precedence,
+        // just write the C-style cast.
+        Hints.push_back(FixItHint::CreateInsertion(E->getLocStart(),
+                                                   CastFix.str()));
+      } else {
+        // Otherwise, add parens around the expression as well as the cast.
+        CastFix << "(";
+        Hints.push_back(FixItHint::CreateInsertion(E->getLocStart(),
+                                                   CastFix.str()));
+
+        SourceLocation After = S.PP.getLocForEndOfToken(E->getLocEnd());
+        Hints.push_back(FixItHint::CreateInsertion(After, ")"));
+      }
 
+      // We extract the name from the typedef because we don't want to show
+      // the underlying type in the diagnostic.
+      const TypedefType *UserTy = cast<TypedefType>(E->getType());
+      StringRef Name = UserTy->getDecl()->getName();
+
+      // Finally, emit the diagnostic.
+      EmitFormatDiagnostic(S.PDiag(diag::warn_format_argument_needs_cast)
+                             << Name << IntendedTy
+                             << E->getSourceRange(),
+                           E->getLocStart(), /*IsStringLocation=*/false,
+                           SpecRange, Hints);
+    } else {
       EmitFormatDiagnostic(
         S.PDiag(diag::warn_printf_conversion_argument_type_mismatch)
-          << AT.getRepresentativeTypeName(S.Context) << E->getType()
+          << AT.getRepresentativeTypeName(S.Context) << IntendedTy
           << E->getSourceRange(),
         E->getLocStart(),
         /*IsStringLocation*/false,
-        getSpecifierRange(StartSpecifier, SpecifierLen),
-        FixItHint::CreateReplacement(
-          getSpecifierRange(StartSpecifier, SpecifierLen),
-          os.str()));
-    } else {
-      const CharSourceRange &CSR = getSpecifierRange(StartSpecifier,
-                                                     SpecifierLen);
-      // Since the warning for passing non-POD types to variadic functions
-      // was deferred until now, we emit a warning for non-POD
-      // arguments here.
-      if (S.isValidVarArgType(E->getType()) == Sema::VAK_Invalid) {
-        unsigned DiagKind;
-        if (E->getType()->isObjCObjectType())
-          DiagKind = diag::err_cannot_pass_objc_interface_to_vararg_format;
-        else
-          DiagKind = diag::warn_non_pod_vararg_with_format_string;
-
-        EmitFormatDiagnostic(
-          S.PDiag(DiagKind)
-            << S.getLangOpts().CPlusPlus0x
-            << E->getType()
-            << CallType
-            << AT.getRepresentativeTypeName(S.Context)
-            << CSR
-            << E->getSourceRange(),
-          E->getLocStart(), /*IsStringLocation*/false, CSR);
-
-        checkForCStrMembers(AT, E, CSR);
-      } else
-        EmitFormatDiagnostic(
-          S.PDiag(diag::warn_printf_conversion_argument_type_mismatch)
-            << AT.getRepresentativeTypeName(S.Context) << E->getType()
-            << CSR
-            << E->getSourceRange(),
-          E->getLocStart(), /*IsStringLocation*/false, CSR);
+        SpecRange,
+        FixItHint::CreateReplacement(SpecRange, os.str()));
     }
+  } else {
+    const CharSourceRange &CSR = getSpecifierRange(StartSpecifier,
+                                                   SpecifierLen);
+    // Since the warning for passing non-POD types to variadic functions
+    // was deferred until now, we emit a warning for non-POD
+    // arguments here.
+    if (S.isValidVarArgType(E->getType()) == Sema::VAK_Invalid) {
+      unsigned DiagKind;
+      if (E->getType()->isObjCObjectType())
+        DiagKind = diag::err_cannot_pass_objc_interface_to_vararg_format;
+      else
+        DiagKind = diag::warn_non_pod_vararg_with_format_string;
+
+      EmitFormatDiagnostic(
+        S.PDiag(DiagKind)
+          << S.getLangOpts().CPlusPlus0x
+          << E->getType()
+          << CallType
+          << AT.getRepresentativeTypeName(S.Context)
+          << CSR
+          << E->getSourceRange(),
+        E->getLocStart(), /*IsStringLocation*/false, CSR);
+
+      checkForCStrMembers(AT, E, CSR);
+    } else
+      EmitFormatDiagnostic(
+        S.PDiag(diag::warn_printf_conversion_argument_type_mismatch)
+          << AT.getRepresentativeTypeName(S.Context) << E->getType()
+          << CSR
+          << E->getSourceRange(),
+        E->getLocStart(), /*IsStringLocation*/false, CSR);
   }
 
   return true;
@@ -2776,24 +2967,17 @@ bool CheckScanfHandler::HandleScanfSpecifier(
   }
   
   // Check the length modifier is valid with the given conversion specifier.
-  const LengthModifier &LM = FS.getLengthModifier();
-  if (!FS.hasValidLengthModifier()) {
-    const CharSourceRange &R = getSpecifierRange(LM.getStart(), LM.getLength());
-    EmitFormatDiagnostic(S.PDiag(diag::warn_format_nonsensical_length)
-                         << LM.toString() << CS.toString()
-                         << getSpecifierRange(startSpecifier, specifierLen),
-                         getLocationOfByte(LM.getStart()),
-                         /*IsStringLocation*/true, R,
-                         FixItHint::CreateRemoval(R));
-  }
+  if (!FS.hasValidLengthModifier(S.getASTContext().getTargetInfo()))
+    HandleInvalidLengthModifier(FS, CS, startSpecifier, specifierLen,
+                                diag::warn_format_nonsensical_length);
+  else if (!FS.hasStandardLengthModifier())
+    HandleNonStandardLengthModifier(FS, startSpecifier, specifierLen);
+  else if (!FS.hasStandardLengthConversionCombination())
+    HandleInvalidLengthModifier(FS, CS, startSpecifier, specifierLen,
+                                diag::warn_format_non_standard_conversion_spec);
 
-  if (!FS.hasStandardLengthModifier())
-    HandleNonStandardLengthModifier(LM, startSpecifier, specifierLen);
   if (!FS.hasStandardConversionSpecifier(S.getLangOpts()))
     HandleNonStandardConversionSpecifier(CS, startSpecifier, specifierLen);
-  if (!FS.hasStandardLengthConversionCombination())
-    HandleNonStandardConversionSpecification(LM, CS, startSpecifier,
-                                             specifierLen);
 
   // The remaining checks depend on the data arguments.
   if (HasVAListArg)
@@ -2881,7 +3065,8 @@ void Sema::CheckFormatString(const StringLiteral *FExpr,
                          inFunctionCall, CallType);
   
     if (!analyze_format_string::ParsePrintfString(H, Str, Str + StrLen,
-                                                  getLangOpts()))
+                                                  getLangOpts(),
+                                                  Context.getTargetInfo()))
       H.DoneProcessing();
   } else if (Type == FST_Scanf) {
     CheckScanfHandler H(*this, FExpr, OrigFormatExpr, firstDataArg, numDataArgs,
@@ -2889,7 +3074,8 @@ void Sema::CheckFormatString(const StringLiteral *FExpr,
                         inFunctionCall, CallType);
     
     if (!analyze_format_string::ParseScanfString(H, Str, Str + StrLen,
-                                                 getLangOpts()))
+                                                 getLangOpts(),
+                                                 Context.getTargetInfo()))
       H.DoneProcessing();
   } // TODO: handle other formats
 }
@@ -4138,6 +4324,44 @@ static void CheckTrivialUnsignedComparison(Sema &S, BinaryOperator *E) {
   }
 }
 
+static void DiagnoseOutOfRangeComparison(Sema &S, BinaryOperator *E,
+                                         Expr *Constant, Expr *Other,
+                                         llvm::APSInt Value,
+                                         bool RhsConstant) {
+  BinaryOperatorKind op = E->getOpcode();
+  QualType OtherT = Other->getType();
+  QualType ConstantT = Constant->getType();
+  if (S.Context.hasSameUnqualifiedType(OtherT, ConstantT))
+    return;
+  assert((OtherT->isIntegerType() && ConstantT->isIntegerType())
+         && "comparison with non-integer type");
+  // FIXME. handle cases for signedness to catch (signed char)N == 200
+  IntRange OtherRange = IntRange::forValueOfType(S.Context, OtherT);
+  IntRange LitRange = GetValueRange(S.Context, Value, Value.getBitWidth());
+  if (OtherRange.Width >= LitRange.Width)
+    return;
+  bool IsTrue = true;
+  if (op == BO_EQ)
+    IsTrue = false;
+  else if (op == BO_NE)
+    IsTrue = true;
+  else if (RhsConstant) {
+    if (op == BO_GT || op == BO_GE)
+      IsTrue = !LitRange.NonNegative;
+    else // op == BO_LT || op == BO_LE
+      IsTrue = LitRange.NonNegative;
+  } else {
+    if (op == BO_LT || op == BO_LE)
+      IsTrue = !LitRange.NonNegative;
+    else // op == BO_GT || op == BO_GE
+      IsTrue = LitRange.NonNegative;
+  }
+  SmallString<16> PrettySourceValue(Value.toString(10));
+  S.Diag(E->getOperatorLoc(), diag::warn_out_of_range_compare)
+  << PrettySourceValue << OtherT << IsTrue
+  << E->getLHS()->getSourceRange() << E->getRHS()->getSourceRange();
+}
+
 /// Analyze the operands of the given comparison.  Implements the
 /// fallback case from AnalyzeComparison.
 static void AnalyzeImpConvsInComparison(Sema &S, BinaryOperator *E) {
@@ -4153,20 +4377,42 @@ static void AnalyzeComparison(Sema &S, BinaryOperator *E) {
   QualType T = E->getLHS()->getType();
   assert(S.Context.hasSameUnqualifiedType(T, E->getRHS()->getType())
          && "comparison with mismatched types");
+  if (E->isValueDependent())
+    return AnalyzeImpConvsInComparison(S, E);
 
+  Expr *LHS = E->getLHS()->IgnoreParenImpCasts();
+  Expr *RHS = E->getRHS()->IgnoreParenImpCasts();
+  
+  bool IsComparisonConstant = false;
+  
+  // Check whether an integer constant comparison results in a value
+  // of 'true' or 'false'.
+  if (T->isIntegralType(S.Context)) {
+    llvm::APSInt RHSValue;
+    bool IsRHSIntegralLiteral = 
+      RHS->isIntegerConstantExpr(RHSValue, S.Context);
+    llvm::APSInt LHSValue;
+    bool IsLHSIntegralLiteral = 
+      LHS->isIntegerConstantExpr(LHSValue, S.Context);
+    if (IsRHSIntegralLiteral && !IsLHSIntegralLiteral)
+        DiagnoseOutOfRangeComparison(S, E, RHS, LHS, RHSValue, true);
+    else if (!IsRHSIntegralLiteral && IsLHSIntegralLiteral)
+      DiagnoseOutOfRangeComparison(S, E, LHS, RHS, LHSValue, false);
+    else
+      IsComparisonConstant = 
+        (IsRHSIntegralLiteral && IsLHSIntegralLiteral);
+  } else if (!T->hasUnsignedIntegerRepresentation())
+      IsComparisonConstant = E->isIntegerConstantExpr(S.Context);
+  
   // We don't do anything special if this isn't an unsigned integral
   // comparison:  we're only interested in integral comparisons, and
   // signed comparisons only happen in cases we don't care to warn about.
   //
   // We also don't care about value-dependent expressions or expressions
   // whose result is a constant.
-  if (!T->hasUnsignedIntegerRepresentation()
-      || E->isValueDependent() || E->isIntegerConstantExpr(S.Context))
+  if (!T->hasUnsignedIntegerRepresentation() || IsComparisonConstant)
     return AnalyzeImpConvsInComparison(S, E);
-
-  Expr *LHS = E->getLHS()->IgnoreParenImpCasts();
-  Expr *RHS = E->getRHS()->IgnoreParenImpCasts();
-
+  
   // Check to see if one of the (unmodified) operands is of different
   // signedness.
   Expr *signedOperand, *unsignedOperand;
@@ -4353,6 +4599,46 @@ std::string PrettyPrintInRange(const llvm::APSInt &Value, IntRange Range) {
   return ValueInRange.toString(10);
 }
 
+static bool IsImplicitBoolFloatConversion(Sema &S, Expr *Ex, bool ToBool) {
+  if (!isa<ImplicitCastExpr>(Ex))
+    return false;
+
+  Expr *InnerE = Ex->IgnoreParenImpCasts();
+  const Type *Target = S.Context.getCanonicalType(Ex->getType()).getTypePtr();
+  const Type *Source =
+    S.Context.getCanonicalType(InnerE->getType()).getTypePtr();
+  if (Target->isDependentType())
+    return false;
+
+  const BuiltinType *FloatCandidateBT =
+    dyn_cast<BuiltinType>(ToBool ? Source : Target);
+  const Type *BoolCandidateType = ToBool ? Target : Source;
+
+  return (BoolCandidateType->isSpecificBuiltinType(BuiltinType::Bool) &&
+          FloatCandidateBT && (FloatCandidateBT->isFloatingPoint()));
+}
+
+void CheckImplicitArgumentConversions(Sema &S, CallExpr *TheCall,
+                                      SourceLocation CC) {
+  unsigned NumArgs = TheCall->getNumArgs();
+  for (unsigned i = 0; i < NumArgs; ++i) {
+    Expr *CurrA = TheCall->getArg(i);
+    if (!IsImplicitBoolFloatConversion(S, CurrA, true))
+      continue;
+
+    bool IsSwapped = ((i > 0) &&
+        IsImplicitBoolFloatConversion(S, TheCall->getArg(i - 1), false));
+    IsSwapped |= ((i < (NumArgs - 1)) &&
+        IsImplicitBoolFloatConversion(S, TheCall->getArg(i + 1), false));
+    if (IsSwapped) {
+      // Warn on this floating-point to bool conversion.
+      DiagnoseImpCast(S, CurrA->IgnoreParenImpCasts(),
+                      CurrA->getType(), CC,
+                      diag::warn_impcast_floating_point_to_bool);
+    }
+  }
+}
+
 void CheckImplicitConversion(Sema &S, Expr *E, QualType T,
                              SourceLocation CC, bool *ICContext = 0) {
   if (E->isTypeDependent() || E->isValueDependent()) return;
@@ -4488,12 +4774,33 @@ void CheckImplicitConversion(Sema &S, Expr *E, QualType T,
       }
     }
 
+    // If the target is bool, warn if expr is a function or method call.
+    if (Target->isSpecificBuiltinType(BuiltinType::Bool) &&
+        isa<CallExpr>(E)) {
+      // Check last argument of function call to see if it is an
+      // implicit cast from a type matching the type the result
+      // is being cast to.
+      CallExpr *CEx = cast<CallExpr>(E);
+      unsigned NumArgs = CEx->getNumArgs();
+      if (NumArgs > 0) {
+        Expr *LastA = CEx->getArg(NumArgs - 1);
+        Expr *InnerE = LastA->IgnoreParenImpCasts();
+        const Type *InnerType =
+          S.Context.getCanonicalType(InnerE->getType()).getTypePtr();
+        if (isa<ImplicitCastExpr>(LastA) && (InnerType == Target)) {
+          // Warn on this floating-point to bool conversion
+          DiagnoseImpCast(S, E, T, CC,
+                          diag::warn_impcast_floating_point_to_bool);
+        }
+      }
+    }
     return;
   }
 
   if ((E->isNullPointerConstant(S.Context, Expr::NPC_ValueDependentIsNotNull)
            == Expr::NPCK_GNUNull) && !Target->isAnyPointerType()
-      && !Target->isBlockPointerType() && !Target->isMemberPointerType()) {
+      && !Target->isBlockPointerType() && !Target->isMemberPointerType()
+      && Target->isScalarType()) {
     SourceLocation Loc = E->getSourceRange().getBegin();
     if (Loc.isMacroID())
       Loc = S.SourceMgr.getImmediateExpansionRange(Loc).first;
@@ -4658,6 +4965,10 @@ void AnalyzeImplicitConversions(Sema &S, Expr *OrigE, SourceLocation CC) {
     return;
   }
 
+  // Check implicit argument conversions for function calls.
+  if (CallExpr *Call = dyn_cast<CallExpr>(E))
+    CheckImplicitArgumentConversions(S, Call, CC);
+
   // Go ahead and check any implicit conversions we might have skipped.
   // The non-canonical typecheck is just an optimization;
   // CheckImplicitConversion will filter out dead implicit conversions.
@@ -5077,7 +5388,8 @@ static bool considerVariable(VarDecl *var, Expr *ref, RetainCycleOwner &owner) {
     return false;
 
   owner.Variable = var;
-  owner.setLocsFrom(ref);
+  if (ref)
+    owner.setLocsFrom(ref);
   return true;
 }
 
@@ -5186,6 +5498,12 @@ namespace {
       if (block->getBlockDecl()->capturesVariable(Variable))
         Visit(block->getBlockDecl()->getBody());
     }
+    
+    void VisitOpaqueValueExpr(OpaqueValueExpr *OVE) {
+      if (Capturer) return;
+      if (OVE->getSourceExpr())
+        Visit(OVE->getSourceExpr());
+    }
   };
 }
 
@@ -5195,6 +5513,28 @@ static Expr *findCapturingExpr(Sema &S, Expr *e, RetainCycleOwner &owner) {
   assert(owner.Variable && owner.Loc.isValid());
 
   e = e->IgnoreParenCasts();
+
+  // Look through [^{...} copy] and Block_copy(^{...}).
+  if (ObjCMessageExpr *ME = dyn_cast<ObjCMessageExpr>(e)) {
+    Selector Cmd = ME->getSelector();
+    if (Cmd.isUnarySelector() && Cmd.getNameForSlot(0) == "copy") {
+      e = ME->getInstanceReceiver();
+      if (!e)
+        return 0;
+      e = e->IgnoreParenCasts();
+    }
+  } else if (CallExpr *CE = dyn_cast<CallExpr>(e)) {
+    if (CE->getNumArgs() == 1) {
+      FunctionDecl *Fn = dyn_cast_or_null<FunctionDecl>(CE->getCalleeDecl());
+      if (Fn) {
+        const IdentifierInfo *FnI = Fn->getIdentifier();
+        if (FnI && FnI->isStr("_Block_copy")) {
+          e = CE->getArg(0)->IgnoreParenCasts();
+        }
+      }
+    }
+  }
+  
   BlockExpr *block = dyn_cast<BlockExpr>(e);
   if (!block || !block->getBlockDecl()->capturesVariable(owner.Variable))
     return 0;
@@ -5271,6 +5611,20 @@ void Sema::checkRetainCycles(Expr *receiver, Expr *argument) {
     diagnoseRetainCycle(*this, capturer, owner);
 }
 
+void Sema::checkRetainCycles(VarDecl *Var, Expr *Init) {
+  RetainCycleOwner Owner;
+  if (!considerVariable(Var, /*DeclRefExpr=*/0, Owner))
+    return;
+  
+  // Because we don't have an expression for the variable, we have to set the
+  // location explicitly here.
+  Owner.Loc = Var->getLocation();
+  Owner.Range = Var->getSourceRange();
+  
+  if (Expr *Capturer = findCapturingExpr(*this, Init, Owner))
+    diagnoseRetainCycle(*this, Capturer, Owner);
+}
+
 bool Sema::checkUnsafeAssigns(SourceLocation Loc,
                               QualType LHS, Expr *RHS) {
   Qualifiers::ObjCLifetime LT = LHS.getObjCLifetime();
@@ -5304,9 +5658,19 @@ void Sema::checkUnsafeExprAssigns(SourceLocation Loc,
   
   if (LHSType.isNull())
     LHSType = LHS->getType();
+
+  Qualifiers::ObjCLifetime LT = LHSType.getObjCLifetime();
+
+  if (LT == Qualifiers::OCL_Weak) {
+    DiagnosticsEngine::Level Level =
+      Diags.getDiagnosticLevel(diag::warn_arc_repeated_use_of_weak, Loc);
+    if (Level != DiagnosticsEngine::Ignored)
+      getCurFunction()->markSafeWeakUse(LHS);
+  }
+
   if (checkUnsafeAssigns(Loc, LHSType, RHS))
     return;
-  Qualifiers::ObjCLifetime LT = LHSType.getObjCLifetime();
+
   // FIXME. Check for other life times.
   if (LT != Qualifiers::OCL_None)
     return;
@@ -5826,7 +6190,8 @@ void Sema::CheckArgumentWithTypeTag(const ArgumentWithTypeTagAttr *Attr,
   if (IsPointerAttr) {
     // Skip implicit cast of pointer to `void *' (as a function argument).
     if (const ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(ArgumentExpr))
-      if (ICE->getType()->isVoidPointerType())
+      if (ICE->getType()->isVoidPointerType() &&
+          ICE->getCastKind() == CK_BitCast)
         ArgumentExpr = ICE->getSubExpr();
   }
   QualType ArgumentType = ArgumentExpr->getType();
@@ -5881,4 +6246,3 @@ void Sema::CheckArgumentWithTypeTag(const ArgumentWithTypeTagAttr *Attr,
         << ArgumentExpr->getSourceRange()
         << TypeTagExpr->getSourceRange();
 }
-
diff --git a/lib/Sema/SemaCodeComplete.cpp b/lib/Sema/SemaCodeComplete.cpp
index adf1327..b1aead8 100644
--- a/lib/Sema/SemaCodeComplete.cpp
+++ b/lib/Sema/SemaCodeComplete.cpp
@@ -1059,10 +1059,12 @@ bool ResultBuilder::IsClassOrStruct(NamedDecl *ND) const {
   // Allow us to find class templates, too.
   if (ClassTemplateDecl *ClassTemplate = dyn_cast<ClassTemplateDecl>(ND))
     ND = ClassTemplate->getTemplatedDecl();
-  
+
+  // For purposes of this check, interfaces match too.
   if (RecordDecl *RD = dyn_cast<RecordDecl>(ND))
     return RD->getTagKind() == TTK_Class ||
-    RD->getTagKind() == TTK_Struct;
+    RD->getTagKind() == TTK_Struct ||
+    RD->getTagKind() == TTK_Interface;
   
   return false;
 }
@@ -1422,7 +1424,8 @@ static const char *GetCompletionTypeString(QualType T,
         if (!Tag->getIdentifier() && !Tag->getTypedefNameForAnonDecl()) {
           switch (Tag->getTagKind()) {
           case TTK_Struct: return "struct <anonymous>";
-          case TTK_Class:  return "class <anonymous>";            
+          case TTK_Interface: return "__interface <anonymous>";
+          case TTK_Class:  return "class <anonymous>";
           case TTK_Union:  return "union <anonymous>";
           case TTK_Enum:   return "enum <anonymous>";
           }
@@ -1449,7 +1452,7 @@ static void addThisCompletion(Sema &S, ResultBuilder &Results) {
                                                      Policy,
                                                      Allocator));
   Builder.AddTypedTextChunk("this");
-  Results.AddResult(CodeCompletionResult(Builder.TakeString()));            
+  Results.AddResult(CodeCompletionResult(Builder.TakeString()));
 }
 
 /// \brief Add language constructs that show up for "ordinary" names.
@@ -2480,7 +2483,6 @@ CodeCompletionResult::CreateCodeCompletionString(ASTContext &Ctx,
     
     if (Declaration) {
       Result.addParentContext(Declaration->getDeclContext());
-      Pattern->ParentKind = Result.getParentKind();
       Pattern->ParentName = Result.getParentName();
     }
     
@@ -2493,7 +2495,7 @@ CodeCompletionResult::CreateCodeCompletionString(ASTContext &Ctx,
   }
   
   if (Kind == RK_Macro) {
-    MacroInfo *MI = PP.getMacroInfo(Macro);
+    MacroInfo *MI = PP.getMacroInfoHistory(Macro);
     assert(MI && "Not a macro?");
 
     Result.AddTypedTextChunk(
@@ -2880,10 +2882,14 @@ CXCursorKind clang::getCursorKindForDecl(Decl *D) {
       case ObjCPropertyImplDecl::Synthesize:
         return CXCursor_ObjCSynthesizeDecl;
       }
+
+      case Decl::Import:
+        return CXCursor_ModuleImportDecl;
       
     default:
       if (TagDecl *TD = dyn_cast<TagDecl>(D)) {
         switch (TD->getTagKind()) {
+          case TTK_Interface:  // fall through
           case TTK_Struct: return CXCursor_StructDecl;
           case TTK_Class:  return CXCursor_ClassDecl;
           case TTK_Union:  return CXCursor_UnionDecl;
@@ -2896,6 +2902,7 @@ CXCursorKind clang::getCursorKindForDecl(Decl *D) {
 }
 
 static void AddMacroResults(Preprocessor &PP, ResultBuilder &Results,
+                            bool IncludeUndefined,
                             bool TargetTypeIsPointer = false) {
   typedef CodeCompletionResult Result;
   
@@ -2904,7 +2911,8 @@ static void AddMacroResults(Preprocessor &PP, ResultBuilder &Results,
   for (Preprocessor::macro_iterator M = PP.macro_begin(), 
                                  MEnd = PP.macro_end();
        M != MEnd; ++M) {
-    Results.AddResult(Result(M->first, 
+    if (IncludeUndefined || M->first->hasMacroDefinition())
+      Results.AddResult(Result(M->first,
                              getMacroUsagePriority(M->first->getName(),
                                                    PP.getLangOpts(),
                                                    TargetTypeIsPointer)));
@@ -3125,7 +3133,6 @@ void Sema::CodeCompleteModuleImport(SourceLocation ImportLoc,
 
 void Sema::CodeCompleteOrdinaryName(Scope *S, 
                                     ParserCompletionContext CompletionContext) {
-  typedef CodeCompletionResult Result;
   ResultBuilder Results(*this, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         mapCodeCompletionContext(*this, CompletionContext));
@@ -3204,7 +3211,7 @@ void Sema::CodeCompleteOrdinaryName(Scope *S,
   }
   
   if (CodeCompleter->includeMacros())
-    AddMacroResults(PP, Results);
+    AddMacroResults(PP, Results, false);
   
   HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
                             Results.data(),Results.size());
@@ -3296,7 +3303,6 @@ struct Sema::CodeCompleteExpressionData {
 /// type we're looking for.
 void Sema::CodeCompleteExpression(Scope *S, 
                                   const CodeCompleteExpressionData &Data) {
-  typedef CodeCompletionResult Result;
   ResultBuilder Results(*this, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_Expression);
@@ -3336,7 +3342,7 @@ void Sema::CodeCompleteExpression(Scope *S,
     AddPrettyFunctionResults(PP.getLangOpts(), Results);        
 
   if (CodeCompleter->includeMacros())
-    AddMacroResults(PP, Results, PreferredTypeIsPointer);
+    AddMacroResults(PP, Results, false, PreferredTypeIsPointer);
   HandleCodeCompleteResults(this, CodeCompleter, 
                 CodeCompletionContext(CodeCompletionContext::CCC_Expression, 
                                       Data.PreferredType),
@@ -3580,7 +3586,6 @@ void Sema::CodeCompleteTag(Scope *S, unsigned TagSpec) {
   if (!CodeCompleter)
     return;
   
-  typedef CodeCompletionResult Result;
   ResultBuilder::LookupFilter Filter = 0;
   enum CodeCompletionContext::Kind ContextKind
     = CodeCompletionContext::CCC_Other;
@@ -3597,6 +3602,7 @@ void Sema::CodeCompleteTag(Scope *S, unsigned TagSpec) {
     
   case DeclSpec::TST_struct:
   case DeclSpec::TST_class:
+  case DeclSpec::TST_interface:
     Filter = &ResultBuilder::IsClassOrStruct;
     ContextKind = CodeCompletionContext::CCC_ClassOrStructTag;
     break;
@@ -3728,7 +3734,7 @@ void Sema::CodeCompleteCase(Scope *S) {
   //so only say we include macros if the code completer says we do
   enum CodeCompletionContext::Kind kind = CodeCompletionContext::CCC_Other;
   if (CodeCompleter->includeMacros()) {
-    AddMacroResults(PP, Results);
+    AddMacroResults(PP, Results, false);
     kind = CodeCompletionContext::CCC_OtherWithMacros;
   }
   
@@ -3898,7 +3904,6 @@ void Sema::CodeCompleteReturn(Scope *S) {
 }
 
 void Sema::CodeCompleteAfterIf(Scope *S) {
-  typedef CodeCompletionResult Result;
   ResultBuilder Results(*this, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         mapCodeCompletionContext(*this, PCC_Statement));
@@ -3952,7 +3957,7 @@ void Sema::CodeCompleteAfterIf(Scope *S) {
     AddPrettyFunctionResults(PP.getLangOpts(), Results);        
   
   if (CodeCompleter->includeMacros())
-    AddMacroResults(PP, Results);
+    AddMacroResults(PP, Results, false);
   
   HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
                             Results.data(),Results.size());
@@ -4408,7 +4413,6 @@ static void AddObjCTopLevelResults(ResultBuilder &Results, bool NeedAt) {
 }
 
 void Sema::CodeCompleteObjCAtDirective(Scope *S) {
-  typedef CodeCompletionResult Result;
   ResultBuilder Results(*this, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_Other);
@@ -4595,26 +4599,23 @@ static bool ObjCPropertyFlagConflicts(unsigned Attributes, unsigned NewFlag) {
   
   // Check for collisions with "readonly".
   if ((Attributes & ObjCDeclSpec::DQ_PR_readonly) &&
-      (Attributes & (ObjCDeclSpec::DQ_PR_readwrite |
-                     ObjCDeclSpec::DQ_PR_assign |
-                     ObjCDeclSpec::DQ_PR_unsafe_unretained |
-                     ObjCDeclSpec::DQ_PR_copy |
-                     ObjCDeclSpec::DQ_PR_retain |
-                     ObjCDeclSpec::DQ_PR_strong)))
+      (Attributes & ObjCDeclSpec::DQ_PR_readwrite))
     return true;
   
-  // Check for more than one of { assign, copy, retain, strong }.
+  // Check for more than one of { assign, copy, retain, strong, weak }.
   unsigned AssignCopyRetMask = Attributes & (ObjCDeclSpec::DQ_PR_assign |
                                          ObjCDeclSpec::DQ_PR_unsafe_unretained |
                                              ObjCDeclSpec::DQ_PR_copy |
-                                             ObjCDeclSpec::DQ_PR_retain|
-                                             ObjCDeclSpec::DQ_PR_strong);
+                                             ObjCDeclSpec::DQ_PR_retain |
+                                             ObjCDeclSpec::DQ_PR_strong |
+                                             ObjCDeclSpec::DQ_PR_weak);
   if (AssignCopyRetMask &&
       AssignCopyRetMask != ObjCDeclSpec::DQ_PR_assign &&
       AssignCopyRetMask != ObjCDeclSpec::DQ_PR_unsafe_unretained &&
       AssignCopyRetMask != ObjCDeclSpec::DQ_PR_copy &&
       AssignCopyRetMask != ObjCDeclSpec::DQ_PR_retain &&
-      AssignCopyRetMask != ObjCDeclSpec::DQ_PR_strong)
+      AssignCopyRetMask != ObjCDeclSpec::DQ_PR_strong &&
+      AssignCopyRetMask != ObjCDeclSpec::DQ_PR_weak)
     return true;
   
   return false;
@@ -4626,7 +4627,6 @@ void Sema::CodeCompleteObjCPropertyFlags(Scope *S, ObjCDeclSpec &ODS) {
   
   unsigned Attributes = ODS.getPropertyAttributes();
   
-  typedef CodeCompletionResult Result;
   ResultBuilder Results(*this, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_Other);
@@ -4650,6 +4650,12 @@ void Sema::CodeCompleteObjCPropertyFlags(Scope *S, ObjCDeclSpec &ODS) {
     Results.AddResult(CodeCompletionResult("nonatomic"));
   if (!ObjCPropertyFlagConflicts(Attributes, ObjCDeclSpec::DQ_PR_atomic))
     Results.AddResult(CodeCompletionResult("atomic"));
+
+  // Only suggest "weak" if we're compiling for ARC-with-weak-references or GC.
+  if (getLangOpts().ObjCARCWeak || getLangOpts().getGC() != LangOptions::NonGC)
+    if (!ObjCPropertyFlagConflicts(Attributes, ObjCDeclSpec::DQ_PR_weak))
+      Results.AddResult(CodeCompletionResult("weak"));
+
   if (!ObjCPropertyFlagConflicts(Attributes, ObjCDeclSpec::DQ_PR_setter)) {
     CodeCompletionBuilder Setter(Results.getAllocator(),
                                  Results.getCodeCompletionTUInfo());
@@ -4837,8 +4843,6 @@ static void AddObjCMethods(ObjCContainerDecl *Container,
 
 
 void Sema::CodeCompleteObjCPropertyGetter(Scope *S) {
-  typedef CodeCompletionResult Result;
-
   // Try to find the interface where getters might live.
   ObjCInterfaceDecl *Class = dyn_cast_or_null<ObjCInterfaceDecl>(CurContext);
   if (!Class) {
@@ -4866,8 +4870,6 @@ void Sema::CodeCompleteObjCPropertyGetter(Scope *S) {
 }
 
 void Sema::CodeCompleteObjCPropertySetter(Scope *S) {
-  typedef CodeCompletionResult Result;
-
   // Try to find the interface where setters might live.
   ObjCInterfaceDecl *Class
     = dyn_cast_or_null<ObjCInterfaceDecl>(CurContext);
@@ -4898,7 +4900,6 @@ void Sema::CodeCompleteObjCPropertySetter(Scope *S) {
 
 void Sema::CodeCompleteObjCPassingType(Scope *S, ObjCDeclSpec &DS,
                                        bool IsParameter) {
-  typedef CodeCompletionResult Result;
   ResultBuilder Results(*this, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_Type);
@@ -4957,7 +4958,7 @@ void Sema::CodeCompleteObjCPassingType(Scope *S, ObjCDeclSpec &DS,
                      CodeCompleter->includeGlobals());
   
   if (CodeCompleter->includeMacros())
-    AddMacroResults(PP, Results);
+    AddMacroResults(PP, Results, false);
 
   HandleCodeCompleteResults(this, CodeCompleter,
                             CodeCompletionContext::CCC_Type,
@@ -5041,7 +5042,7 @@ static ObjCInterfaceDecl *GetAssumedMessageSendExprType(Expr *E) {
 ///
 /// \param S The semantic analysis object.
 ///
-/// \param S NeedSuperKeyword Whether we need to prefix this completion with
+/// \param NeedSuperKeyword Whether we need to prefix this completion with
 /// the "super" keyword. Otherwise, we just need to provide the arguments.
 ///
 /// \param SelIdents The identifiers in the selector that have already been
@@ -5187,7 +5188,7 @@ void Sema::CodeCompleteObjCMessageReceiver(Scope *S) {
   Results.ExitScope();
   
   if (CodeCompleter->includeMacros())
-    AddMacroResults(PP, Results);
+    AddMacroResults(PP, Results, false);
   HandleCodeCompleteResults(this, CodeCompleter, Results.getCompletionContext(),
                             Results.data(), Results.size());
   
@@ -5339,11 +5340,11 @@ static void AddClassMessageCompletions(Sema &SemaRef, Scope *S,
     
     // If we have an external source, load the entire class method
     // pool from the AST file.
-    if (SemaRef.ExternalSource) {
+    if (SemaRef.getExternalSource()) {
       for (uint32_t I = 0, 
-                    N = SemaRef.ExternalSource->GetNumExternalSelectors();
+                    N = SemaRef.getExternalSource()->GetNumExternalSelectors();
            I != N; ++I) {
-        Selector Sel = SemaRef.ExternalSource->GetExternalSelector(I);
+        Selector Sel = SemaRef.getExternalSource()->GetExternalSelector(I);
         if (Sel.isNull() || SemaRef.MethodPool.count(Sel))
           continue;
         
@@ -5868,7 +5869,6 @@ void Sema::CodeCompleteObjCImplementationCategory(Scope *S,
 }
 
 void Sema::CodeCompleteObjCPropertyDefinition(Scope *S) {
-  typedef CodeCompletionResult Result;
   ResultBuilder Results(*this, CodeCompleter->getAllocator(),
                         CodeCompleter->getCodeCompletionTUInfo(),
                         CodeCompletionContext::CCC_Other);
@@ -7169,7 +7169,9 @@ void Sema::CodeCompletePreprocessorMacroName(bool IsDefinition) {
          M != MEnd; ++M) {
       Builder.AddTypedTextChunk(Builder.getAllocator().CopyString(
                                            M->first->getName()));
-      Results.AddResult(Builder.TakeString());
+      Results.AddResult(CodeCompletionResult(Builder.TakeString(),
+                                             CCP_CodePattern,
+                                             CXCursor_MacroDefinition));
     }
     Results.ExitScope();
   } else if (IsDefinition) {
@@ -7186,7 +7188,7 @@ void Sema::CodeCompletePreprocessorExpression() {
                         CodeCompletionContext::CCC_PreprocessorExpression);
   
   if (!CodeCompleter || CodeCompleter->includeMacros())
-    AddMacroResults(PP, Results);
+    AddMacroResults(PP, Results, true);
   
     // defined (<macro>)
   Results.EnterNewScope();
@@ -7235,7 +7237,7 @@ void Sema::GatherGlobalCodeCompletions(CodeCompletionAllocator &Allocator,
   }
   
   if (!CodeCompleter || CodeCompleter->includeMacros())
-    AddMacroResults(PP, Builder);
+    AddMacroResults(PP, Builder, true);
   
   Results.clear();
   Results.insert(Results.end(), 
diff --git a/lib/Sema/SemaDecl.cpp b/lib/Sema/SemaDecl.cpp
index ea181de..0092d5d 100644
--- a/lib/Sema/SemaDecl.cpp
+++ b/lib/Sema/SemaDecl.cpp
@@ -350,8 +350,8 @@ ParsedType Sema::getTypeName(IdentifierInfo &II, SourceLocation NameLoc,
 /// isTagName() - This method is called *for error recovery purposes only*
 /// to determine if the specified name is a valid tag name ("struct foo").  If
 /// so, this returns the TST for the tag corresponding to it (TST_enum,
-/// TST_union, TST_struct, TST_class).  This is used to diagnose cases in C
-/// where the user forgot to specify the tag.
+/// TST_union, TST_struct, TST_interface, TST_class).  This is used to diagnose
+/// cases in C where the user forgot to specify the tag.
 DeclSpec::TST Sema::isTagName(IdentifierInfo &II, Scope *S) {
   // Do a tag name lookup in this scope.
   LookupResult R(*this, &II, SourceLocation(), LookupTagName);
@@ -361,6 +361,7 @@ DeclSpec::TST Sema::isTagName(IdentifierInfo &II, Scope *S) {
     if (const TagDecl *TD = R.getAsSingle<TagDecl>()) {
       switch (TD->getTagKind()) {
       case TTK_Struct: return DeclSpec::TST_struct;
+      case TTK_Interface: return DeclSpec::TST_interface;
       case TTK_Union:  return DeclSpec::TST_union;
       case TTK_Class:  return DeclSpec::TST_class;
       case TTK_Enum:   return DeclSpec::TST_enum;
@@ -434,7 +435,8 @@ bool Sema::DiagnoseUnknownTypeName(IdentifierInfo *&II,
       else if (DeclContext *DC = computeDeclContext(*SS, false))
         Diag(IILoc, diag::err_unknown_nested_typename_suggest)
           << II << DC << CorrectedQuotedStr << SS->getRange()
-          << FixItHint::CreateReplacement(SourceRange(IILoc), CorrectedStr);
+          << FixItHint::CreateReplacement(Corrected.getCorrectionRange(),
+                                          CorrectedStr);
       else
         llvm_unreachable("could not have corrected a typo here");
 
@@ -517,9 +519,9 @@ static bool isTagTypeWithMissingTag(Sema &SemaRef, LookupResult &Result,
                                     Scope *S, CXXScopeSpec &SS,
                                     IdentifierInfo *&Name,
                                     SourceLocation NameLoc) {
-  Result.clear(Sema::LookupTagName);
-  SemaRef.LookupParsedName(Result, S, &SS);
-  if (TagDecl *Tag = Result.getAsSingle<TagDecl>()) {
+  LookupResult R(SemaRef, Name, NameLoc, Sema::LookupTagName);
+  SemaRef.LookupParsedName(R, S, &SS);
+  if (TagDecl *Tag = R.getAsSingle<TagDecl>()) {
     const char *TagName = 0;
     const char *FixItTagName = 0;
     switch (Tag->getTagKind()) {
@@ -538,6 +540,11 @@ static bool isTagTypeWithMissingTag(Sema &SemaRef, LookupResult &Result,
         FixItTagName = "struct ";
         break;
 
+      case TTK_Interface:
+        TagName = "__interface";
+        FixItTagName = "__interface ";
+        break;
+
       case TTK_Union:
         TagName = "union";
         FixItTagName = "union ";
@@ -548,25 +555,42 @@ static bool isTagTypeWithMissingTag(Sema &SemaRef, LookupResult &Result,
       << Name << TagName << SemaRef.getLangOpts().CPlusPlus
       << FixItHint::CreateInsertion(NameLoc, FixItTagName);
 
-    LookupResult R(SemaRef, Name, NameLoc, Sema::LookupOrdinaryName);
-    if (SemaRef.LookupParsedName(R, S, &SS)) {
-      for (LookupResult::iterator I = R.begin(), IEnd = R.end();
-           I != IEnd; ++I)
-        SemaRef.Diag((*I)->getLocation(), diag::note_decl_hiding_tag_type)
-          << Name << TagName;
-    }
+    for (LookupResult::iterator I = Result.begin(), IEnd = Result.end();
+         I != IEnd; ++I)
+      SemaRef.Diag((*I)->getLocation(), diag::note_decl_hiding_tag_type)
+        << Name << TagName;
+
+    // Replace lookup results with just the tag decl.
+    Result.clear(Sema::LookupTagName);
+    SemaRef.LookupParsedName(Result, S, &SS);
     return true;
   }
 
-  Result.clear(Sema::LookupOrdinaryName);
   return false;
 }
 
+/// Build a ParsedType for a simple-type-specifier with a nested-name-specifier.
+static ParsedType buildNestedType(Sema &S, CXXScopeSpec &SS,
+                                  QualType T, SourceLocation NameLoc) {
+  ASTContext &Context = S.Context;
+
+  TypeLocBuilder Builder;
+  Builder.pushTypeSpec(T).setNameLoc(NameLoc);
+
+  T = S.getElaboratedType(ETK_None, SS, T);
+  ElaboratedTypeLoc ElabTL = Builder.push<ElaboratedTypeLoc>(T);
+  ElabTL.setElaboratedKeywordLoc(SourceLocation());
+  ElabTL.setQualifierLoc(SS.getWithLocInContext(Context));
+  return S.CreateParsedType(T, Builder.getTypeSourceInfo(Context, T));
+}
+
 Sema::NameClassification Sema::ClassifyName(Scope *S,
                                             CXXScopeSpec &SS,
                                             IdentifierInfo *&Name,
                                             SourceLocation NameLoc,
-                                            const Token &NextToken) {
+                                            const Token &NextToken,
+                                            bool IsAddressOfOperand,
+                                            CorrectionCandidateCallback *CCC) {
   DeclarationNameInfo NameInfo(Name, NameLoc);
   ObjCMethodDecl *CurMethod = getCurMethodDecl();
   
@@ -632,25 +656,11 @@ Corrected:
 
     // Perform typo correction to determine if there is another name that is
     // close to this name.
-    if (!SecondTry) {
+    if (!SecondTry && CCC) {
       SecondTry = true;
-      CorrectionCandidateCallback DefaultValidator;
-      // Try to limit which sets of keywords should be included in typo
-      // correction based on what the next token is.
-      DefaultValidator.WantTypeSpecifiers =
-          NextToken.is(tok::l_paren) || NextToken.is(tok::less) ||
-          NextToken.is(tok::identifier) || NextToken.is(tok::star) ||
-          NextToken.is(tok::amp) || NextToken.is(tok::l_square);
-      DefaultValidator.WantExpressionKeywords =
-          NextToken.is(tok::l_paren) || NextToken.is(tok::identifier) ||
-          NextToken.is(tok::arrow) || NextToken.is(tok::period);
-      DefaultValidator.WantRemainingKeywords =
-          NextToken.is(tok::l_paren) || NextToken.is(tok::semi) ||
-          NextToken.is(tok::identifier) || NextToken.is(tok::l_brace);
-      DefaultValidator.WantCXXNamedCasts = false;
       if (TypoCorrection Corrected = CorrectTypo(Result.getLookupNameInfo(),
                                                  Result.getLookupKind(), S, 
-                                                 &SS, DefaultValidator)) {
+                                                 &SS, *CCC)) {
         unsigned UnqualifiedDiag = diag::err_undeclared_var_use_suggest;
         unsigned QualifiedDiag = diag::err_no_member_suggest;
         std::string CorrectedStr(Corrected.getAsString(getLangOpts()));
@@ -675,11 +685,12 @@ Corrected:
           Diag(NameLoc, UnqualifiedDiag)
             << Name << CorrectedQuotedStr
             << FixItHint::CreateReplacement(NameLoc, CorrectedStr);
-        else
+        else // FIXME: is this even reachable? Test it.
           Diag(NameLoc, QualifiedDiag)
             << Name << computeDeclContext(SS, false) << CorrectedQuotedStr
             << SS.getRange()
-            << FixItHint::CreateReplacement(NameLoc, CorrectedStr);
+            << FixItHint::CreateReplacement(Corrected.getCorrectionRange(),
+                                            CorrectedStr);
 
         // Update the name, so that the caller has the new name.
         Name = Corrected.getCorrectionAsIdentifierInfo();
@@ -705,7 +716,7 @@ Corrected:
         if (ObjCIvarDecl *Ivar = Result.getAsSingle<ObjCIvarDecl>()) {
           Result.clear();
           ExprResult E(LookupInObjCMethod(Result, S, Ivar->getIdentifier()));
-          return move(E);
+          return E;
         }
         
         goto Corrected;
@@ -731,8 +742,9 @@ Corrected:
     // perform some heroics to see if we actually have a 
     // template-argument-list, which would indicate a missing 'template'
     // keyword here.
-    return BuildDependentDeclRefExpr(SS, /*TemplateKWLoc=*/SourceLocation(),
-                                     NameInfo, /*TemplateArgs=*/0);
+    return ActOnDependentIdExpression(SS, /*TemplateKWLoc=*/SourceLocation(),
+                                      NameInfo, IsAddressOfOperand,
+                                      /*TemplateArgs=*/0);
   }
 
   case LookupResult::Found:
@@ -808,14 +820,16 @@ Corrected:
       return NameClassification::TypeTemplate(Template);
     }
   }
-  
+
   NamedDecl *FirstDecl = (*Result.begin())->getUnderlyingDecl();
   if (TypeDecl *Type = dyn_cast<TypeDecl>(FirstDecl)) {
     DiagnoseUseOfDecl(Type, NameLoc);
     QualType T = Context.getTypeDeclType(Type);
+    if (SS.isNotEmpty())
+      return buildNestedType(*this, SS, T, NameLoc);
     return ParsedType::make(T);
   }
-  
+
   ObjCInterfaceDecl *Class = dyn_cast<ObjCInterfaceDecl>(FirstDecl);
   if (!Class) {
     // FIXME: It's unfortunate that we don't have a Type node for handling this.
@@ -838,24 +852,28 @@ Corrected:
     return ParsedType::make(T);
   }
 
+  // We can have a type template here if we're classifying a template argument.
+  if (isa<TemplateDecl>(FirstDecl) && !isa<FunctionTemplateDecl>(FirstDecl))
+    return NameClassification::TypeTemplate(
+        TemplateName(cast<TemplateDecl>(FirstDecl)));
+
   // Check for a tag type hidden by a non-type decl in a few cases where it
   // seems likely a type is wanted instead of the non-type that was found.
-  if (!getLangOpts().ObjC1 && FirstDecl && !isa<ClassTemplateDecl>(FirstDecl) &&
-      !isa<TypeAliasTemplateDecl>(FirstDecl)) {
+  if (!getLangOpts().ObjC1) {
     bool NextIsOp = NextToken.is(tok::amp) || NextToken.is(tok::star);
     if ((NextToken.is(tok::identifier) ||
          (NextIsOp && FirstDecl->isFunctionOrFunctionTemplate())) &&
         isTagTypeWithMissingTag(*this, Result, S, SS, Name, NameLoc)) {
-      FirstDecl = (*Result.begin())->getUnderlyingDecl();
-      if (TypeDecl *Type = dyn_cast<TypeDecl>(FirstDecl)) {
-        DiagnoseUseOfDecl(Type, NameLoc);
-        QualType T = Context.getTypeDeclType(Type);
-        return ParsedType::make(T);
-      }
+      TypeDecl *Type = Result.getAsSingle<TypeDecl>();
+      DiagnoseUseOfDecl(Type, NameLoc);
+      QualType T = Context.getTypeDeclType(Type);
+      if (SS.isNotEmpty())
+        return buildNestedType(*this, SS, T, NameLoc);
+      return ParsedType::make(T);
     }
   }
   
-  if (!Result.empty() && (*Result.begin())->isCXXClassMember())
+  if (FirstDecl->isCXXClassMember())
     return BuildPossibleImplicitMemberExpr(SS, SourceLocation(), Result, 0);
 
   bool ADL = UseArgumentDependentLookup(SS, Result, NextToken.is(tok::l_paren));
@@ -1186,8 +1204,14 @@ bool Sema::ShouldWarnIfUnusedFileScopedDecl(const DeclaratorDecl *D) const {
         Context.DeclMustBeEmitted(FD))
       return false;
   } else if (const VarDecl *VD = dyn_cast<VarDecl>(D)) {
+    // Don't warn on variables of const-qualified or reference type, since their
+    // values can be used even if though they're not odr-used, and because const
+    // qualified variables can appear in headers in contexts where they're not
+    // intended to be used.
+    // FIXME: Use more principled rules for these exemptions.
     if (!VD->isFileVarDecl() ||
-        VD->getType().isConstant(Context) ||
+        VD->getType().isConstQualified() ||
+        VD->getType()->isReferenceType() ||
         Context.DeclMustBeEmitted(VD))
       return false;
 
@@ -1248,7 +1272,7 @@ static bool ShouldDiagnoseUnusedDecl(const NamedDecl *D) {
     QualType Ty = VD->getType();
 
     // Only look at the outermost level of typedef.
-    if (const TypedefType *TT = dyn_cast<TypedefType>(Ty)) {
+    if (const TypedefType *TT = Ty->getAs<TypedefType>()) {
       if (TT->getDecl()->hasAttr<UnusedAttr>())
         return false;
     }
@@ -1268,6 +1292,8 @@ static bool ShouldDiagnoseUnusedDecl(const NamedDecl *D) {
           return false;
 
         if (const Expr *Init = VD->getInit()) {
+          if (const ExprWithCleanups *Cleanups = dyn_cast<ExprWithCleanups>(Init))
+            Init = Cleanups->getSubExpr();
           const CXXConstructExpr *Construct =
             dyn_cast<CXXConstructExpr>(Init);
           if (Construct && !Construct->isElidable()) {
@@ -1706,6 +1732,25 @@ DeclHasAttr(const Decl *D, const Attr *A) {
   if (AA)
     return false;
 
+  // The following thread safety attributes can also be duplicated.
+  switch (A->getKind()) {
+    case attr::ExclusiveLocksRequired:
+    case attr::SharedLocksRequired:
+    case attr::LocksExcluded:
+    case attr::ExclusiveLockFunction:
+    case attr::SharedLockFunction:
+    case attr::UnlockFunction:
+    case attr::ExclusiveTrylockFunction:
+    case attr::SharedTrylockFunction:
+    case attr::GuardedBy:
+    case attr::PtGuardedBy:
+    case attr::AcquiredBefore:
+    case attr::AcquiredAfter:
+      return false;
+    default:
+      ;
+  }
+
   const OwnershipAttr *OA = dyn_cast<OwnershipAttr>(A);
   const AnnotateAttr *Ann = dyn_cast<AnnotateAttr>(A);
   for (Decl::attr_iterator i = D->attr_begin(), e = D->attr_end(); i != e; ++i)
@@ -1908,6 +1953,19 @@ static bool canRedefineFunction(const FunctionDecl *FD,
           FD->getStorageClass() == SC_Extern);
 }
 
+/// Is the given calling convention the ABI default for the given
+/// declaration?
+static bool isABIDefaultCC(Sema &S, CallingConv CC, FunctionDecl *D) {
+  CallingConv ABIDefaultCC;
+  if (isa<CXXMethodDecl>(D) && cast<CXXMethodDecl>(D)->isInstance()) {
+    ABIDefaultCC = S.Context.getDefaultCXXMethodCallConv(D->isVariadic());
+  } else {
+    // Free C function or a static method.
+    ABIDefaultCC = (S.Context.getLangOpts().MRTD ? CC_X86StdCall : CC_C);
+  }
+  return ABIDefaultCC == CC;
+}
+
 /// MergeFunctionDecl - We just parsed a function 'New' from
 /// declarator D which has the same name and scope as a previous
 /// declaration 'Old'.  Figure out how to resolve this situation,
@@ -1976,6 +2034,9 @@ bool Sema::MergeFunctionDecl(FunctionDecl *New, Decl *OldD, Scope *S) {
   // later declared or defined without one, the second decl assumes the
   // calling convention of the first.
   //
+  // It's OK if a function is first declared without a calling convention,
+  // but is later declared or defined with the default calling convention.
+  //
   // For the new decl, we have to look at the NON-canonical type to tell the
   // difference between a function that really doesn't have a calling
   // convention and one that is declared cdecl. That's because in
@@ -1989,10 +2050,22 @@ bool Sema::MergeFunctionDecl(FunctionDecl *New, Decl *OldD, Scope *S) {
   FunctionType::ExtInfo OldTypeInfo = OldType->getExtInfo();
   FunctionType::ExtInfo NewTypeInfo = NewType->getExtInfo();
   bool RequiresAdjustment = false;
-  if (OldTypeInfo.getCC() != CC_Default &&
-      NewTypeInfo.getCC() == CC_Default) {
+  if (OldTypeInfo.getCC() == NewTypeInfo.getCC()) {
+    // Fast path: nothing to do.
+
+  // Inherit the CC from the previous declaration if it was specified
+  // there but not here.
+  } else if (NewTypeInfo.getCC() == CC_Default) {
     NewTypeInfo = NewTypeInfo.withCallingConv(OldTypeInfo.getCC());
     RequiresAdjustment = true;
+
+  // Don't complain about mismatches when the default CC is
+  // effectively the same as the explict one.
+  } else if (OldTypeInfo.getCC() == CC_Default &&
+             isABIDefaultCC(*this, NewTypeInfo.getCC(), New)) {
+    NewTypeInfo = NewTypeInfo.withCallingConv(OldTypeInfo.getCC());
+    RequiresAdjustment = true;
+
   } else if (!Context.isSameCallConv(OldTypeInfo.getCC(),
                                      NewTypeInfo.getCC())) {
     // Calling conventions really aren't compatible, so complain.
@@ -2398,7 +2471,7 @@ void Sema::MergeVarDeclTypes(VarDecl *New, VarDecl *Old) {
   }
   if (MergedT.isNull()) {
     Diag(New->getLocation(), diag::err_redefinition_different_type)
-      << New->getDeclName();
+      << New->getDeclName() << New->getType() << Old->getType();
     Diag(Old->getLocation(), diag::note_previous_definition);
     return New->setInvalidDecl();
   }
@@ -2551,8 +2624,7 @@ void Sema::MergeVarDecl(VarDecl *New, LookupResult &Previous) {
 /// no declarator (e.g. "struct foo;") is parsed.
 Decl *Sema::ParsedFreeStandingDeclSpec(Scope *S, AccessSpecifier AS,
                                        DeclSpec &DS) {
-  return ParsedFreeStandingDeclSpec(S, AS, DS,
-                                    MultiTemplateParamsArg(*this, 0, 0));
+  return ParsedFreeStandingDeclSpec(S, AS, DS, MultiTemplateParamsArg());
 }
 
 /// ParsedFreeStandingDeclSpec - This method is invoked when a declspec with
@@ -2565,6 +2637,7 @@ Decl *Sema::ParsedFreeStandingDeclSpec(Scope *S, AccessSpecifier AS,
   TagDecl *Tag = 0;
   if (DS.getTypeSpecType() == DeclSpec::TST_class ||
       DS.getTypeSpecType() == DeclSpec::TST_struct ||
+      DS.getTypeSpecType() == DeclSpec::TST_interface ||
       DS.getTypeSpecType() == DeclSpec::TST_union ||
       DS.getTypeSpecType() == DeclSpec::TST_enum) {
     TagD = DS.getRepAsDecl();
@@ -2603,7 +2676,8 @@ Decl *Sema::ParsedFreeStandingDeclSpec(Scope *S, AccessSpecifier AS,
       Diag(DS.getConstexprSpecLoc(), diag::err_constexpr_tag)
         << (DS.getTypeSpecType() == DeclSpec::TST_class ? 0 :
             DS.getTypeSpecType() == DeclSpec::TST_struct ? 1 :
-            DS.getTypeSpecType() == DeclSpec::TST_union ? 2 : 3);
+            DS.getTypeSpecType() == DeclSpec::TST_interface ? 2 :
+            DS.getTypeSpecType() == DeclSpec::TST_union ? 3 : 4);
     else
       Diag(DS.getConstexprSpecLoc(), diag::err_constexpr_no_declarators);
     // Don't emit warnings after this error.
@@ -2724,16 +2798,17 @@ Decl *Sema::ParsedFreeStandingDeclSpec(Scope *S, AccessSpecifier AS,
     DeclSpec::TST TypeSpecType = DS.getTypeSpecType();
     if (TypeSpecType == DeclSpec::TST_class ||
         TypeSpecType == DeclSpec::TST_struct ||
+        TypeSpecType == DeclSpec::TST_interface ||
         TypeSpecType == DeclSpec::TST_union ||
         TypeSpecType == DeclSpec::TST_enum) {
       AttributeList* attrs = DS.getAttributes().getList();
       while (attrs) {
-        Diag(attrs->getScopeLoc(),
-             diag::warn_declspec_attribute_ignored)
+        Diag(attrs->getLoc(), diag::warn_declspec_attribute_ignored)
         << attrs->getName()
         << (TypeSpecType == DeclSpec::TST_class ? 0 :
             TypeSpecType == DeclSpec::TST_struct ? 1 :
-            TypeSpecType == DeclSpec::TST_union ? 2 : 3);
+            TypeSpecType == DeclSpec::TST_union ? 2 :
+            TypeSpecType == DeclSpec::TST_interface ? 3 : 4);
         attrs = attrs->getNext();
       }
     }
@@ -3353,7 +3428,6 @@ static bool RebuildDeclaratorInCurrentInstantiation(Sema &S, Declarator &D,
   switch (DS.getTypeSpecType()) {
   case DeclSpec::TST_typename:
   case DeclSpec::TST_typeofType:
-  case DeclSpec::TST_decltype:
   case DeclSpec::TST_underlyingType:
   case DeclSpec::TST_atomic: {
     // Grab the type from the parser.
@@ -3377,6 +3451,7 @@ static bool RebuildDeclaratorInCurrentInstantiation(Sema &S, Declarator &D,
     break;
   }
 
+  case DeclSpec::TST_decltype:
   case DeclSpec::TST_typeofExpr: {
     Expr *E = DS.getRepAsExpr();
     ExprResult Result = S.RebuildExprInCurrentInstantiation(E);
@@ -3411,7 +3486,7 @@ static bool RebuildDeclaratorInCurrentInstantiation(Sema &S, Declarator &D,
 
 Decl *Sema::ActOnDeclarator(Scope *S, Declarator &D) {
   D.setFunctionDefinitionKind(FDK_Declaration);
-  Decl *Dcl = HandleDeclarator(S, D, MultiTemplateParamsArg(*this));
+  Decl *Dcl = HandleDeclarator(S, D, MultiTemplateParamsArg());
 
   if (OriginalLexicalContext && OriginalLexicalContext->isObjCContainer() &&
       Dcl && Dcl->getDeclContext()->isFileContext())
@@ -3476,7 +3551,8 @@ bool Sema::diagnoseQualifiedDeclaration(CXXScopeSpec &SS, DeclContext *DC,
   //   void X::f();
   // };
   if (Cur->Equals(DC)) {
-    Diag(Loc, diag::warn_member_extra_qualification)
+    Diag(Loc, LangOpts.MicrosoftExt? diag::warn_member_extra_qualification
+                                   : diag::err_member_extra_qualification)
       << Name << FixItHint::CreateRemoval(SS.getRange());
     SS.clear();
     return false;
@@ -3710,11 +3786,11 @@ Decl *Sema::HandleDeclarator(Scope *S, Declarator &D,
     New = ActOnTypedefDeclarator(S, D, DC, TInfo, Previous);
   } else if (R->isFunctionType()) {
     New = ActOnFunctionDeclarator(S, D, DC, TInfo, Previous,
-                                  move(TemplateParamLists),
+                                  TemplateParamLists,
                                   AddToScope);
   } else {
     New = ActOnVariableDeclarator(S, D, DC, TInfo, Previous,
-                                  move(TemplateParamLists));
+                                  TemplateParamLists);
   }
 
   if (New == 0)
@@ -3729,9 +3805,9 @@ Decl *Sema::HandleDeclarator(Scope *S, Declarator &D,
   return New;
 }
 
-/// TryToFixInvalidVariablyModifiedType - Helper method to turn variable array
-/// types into constant array types in certain situations which would otherwise
-/// be errors (for GCC compatibility).
+/// Helper method to turn variable array types into constant array
+/// types in certain situations which would otherwise be errors (for
+/// GCC compatibility).
 static QualType TryToFixInvalidVariablyModifiedType(QualType T,
                                                     ASTContext &Context,
                                                     bool &SizeIsNegative,
@@ -3799,6 +3875,52 @@ static QualType TryToFixInvalidVariablyModifiedType(QualType T,
                                       Res, ArrayType::Normal, 0);
 }
 
+static void
+FixInvalidVariablyModifiedTypeLoc(TypeLoc SrcTL, TypeLoc DstTL) {
+  if (PointerTypeLoc* SrcPTL = dyn_cast<PointerTypeLoc>(&SrcTL)) {
+    PointerTypeLoc* DstPTL = cast<PointerTypeLoc>(&DstTL);
+    FixInvalidVariablyModifiedTypeLoc(SrcPTL->getPointeeLoc(),
+                                      DstPTL->getPointeeLoc());
+    DstPTL->setStarLoc(SrcPTL->getStarLoc());
+    return;
+  }
+  if (ParenTypeLoc* SrcPTL = dyn_cast<ParenTypeLoc>(&SrcTL)) {
+    ParenTypeLoc* DstPTL = cast<ParenTypeLoc>(&DstTL);
+    FixInvalidVariablyModifiedTypeLoc(SrcPTL->getInnerLoc(),
+                                      DstPTL->getInnerLoc());
+    DstPTL->setLParenLoc(SrcPTL->getLParenLoc());
+    DstPTL->setRParenLoc(SrcPTL->getRParenLoc());
+    return;
+  }
+  ArrayTypeLoc* SrcATL = cast<ArrayTypeLoc>(&SrcTL);
+  ArrayTypeLoc* DstATL = cast<ArrayTypeLoc>(&DstTL);
+  TypeLoc SrcElemTL = SrcATL->getElementLoc();
+  TypeLoc DstElemTL = DstATL->getElementLoc();
+  DstElemTL.initializeFullCopy(SrcElemTL);
+  DstATL->setLBracketLoc(SrcATL->getLBracketLoc());
+  DstATL->setSizeExpr(SrcATL->getSizeExpr());
+  DstATL->setRBracketLoc(SrcATL->getRBracketLoc());
+}
+
+/// Helper method to turn variable array types into constant array
+/// types in certain situations which would otherwise be errors (for
+/// GCC compatibility).
+static TypeSourceInfo*
+TryToFixInvalidVariablyModifiedTypeSourceInfo(TypeSourceInfo *TInfo,
+                                              ASTContext &Context,
+                                              bool &SizeIsNegative,
+                                              llvm::APSInt &Oversized) {
+  QualType FixedTy
+    = TryToFixInvalidVariablyModifiedType(TInfo->getType(), Context,
+                                          SizeIsNegative, Oversized);
+  if (FixedTy.isNull())
+    return 0;
+  TypeSourceInfo *FixedTInfo = Context.getTrivialTypeSourceInfo(FixedTy);
+  FixInvalidVariablyModifiedTypeLoc(TInfo->getTypeLoc(),
+                                    FixedTInfo->getTypeLoc());
+  return FixedTInfo;
+}
+
 /// \brief Register the given locally-scoped external C declaration so
 /// that it can be found later for redeclarations
 void
@@ -3926,19 +4048,21 @@ Sema::CheckTypedefForVariablyModifiedType(Scope *S, TypedefNameDecl *NewTD) {
   // then it shall have block scope.
   // Note that variably modified types must be fixed before merging the decl so
   // that redeclarations will match.
-  QualType T = NewTD->getUnderlyingType();
+  TypeSourceInfo *TInfo = NewTD->getTypeSourceInfo();
+  QualType T = TInfo->getType();
   if (T->isVariablyModifiedType()) {
     getCurFunction()->setHasBranchProtectedScope();
 
     if (S->getFnParent() == 0) {
       bool SizeIsNegative;
       llvm::APSInt Oversized;
-      QualType FixedTy =
-          TryToFixInvalidVariablyModifiedType(T, Context, SizeIsNegative,
-                                              Oversized);
-      if (!FixedTy.isNull()) {
+      TypeSourceInfo *FixedTInfo =
+        TryToFixInvalidVariablyModifiedTypeSourceInfo(TInfo, Context,
+                                                      SizeIsNegative,
+                                                      Oversized);
+      if (FixedTInfo) {
         Diag(NewTD->getLocation(), diag::warn_illegal_constant_array_size);
-        NewTD->setTypeSourceInfo(Context.getTrivialTypeSourceInfo(FixedTy));
+        NewTD->setTypeSourceInfo(FixedTInfo);
       } else {
         if (SizeIsNegative)
           Diag(NewTD->getLocation(), diag::err_typecheck_negative_array_size);
@@ -4203,7 +4327,7 @@ Sema::ActOnVariableDeclarator(Scope *S, Declarator &D, DeclContext *DC,
                                   D.getDeclSpec().getLocStart(),
                                                   D.getIdentifierLoc(),
                                                   D.getCXXScopeSpec(),
-                                                  TemplateParamLists.get(),
+                                                  TemplateParamLists.data(),
                                                   TemplateParamLists.size(),
                                                   /*never a friend*/ false,
                                                   isExplicitSpecialization,
@@ -4244,7 +4368,7 @@ Sema::ActOnVariableDeclarator(Scope *S, Declarator &D, DeclContext *DC,
     if (TemplateParamLists.size() > 0 && D.getCXXScopeSpec().isSet()) {
       NewVD->setTemplateParameterListsInfo(Context,
                                            TemplateParamLists.size(),
-                                           TemplateParamLists.release());
+                                           TemplateParamLists.data());
     }
 
     if (D.getDeclSpec().isConstexprSpecified())
@@ -4281,6 +4405,14 @@ Sema::ActOnVariableDeclarator(Scope *S, Declarator &D, DeclContext *DC,
   // Handle attributes prior to checking for duplicates in MergeVarDecl
   ProcessDeclAttributes(S, NewVD, D);
 
+  if (getLangOpts().CUDA) {
+    // CUDA B.2.5: "__shared__ and __constant__ variables have implied static
+    // storage [duration]."
+    if (SC == SC_None && S->getFnParent() != 0 &&
+       (NewVD->hasAttr<CUDASharedAttr>() || NewVD->hasAttr<CUDAConstantAttr>()))
+      NewVD->setStorageClass(SC_Static);
+  }
+
   // In auto-retain/release, infer strong retension for variables of
   // retainable type.
   if (getLangOpts().ObjCAutoRefCount && inferObjCARCLifetime(NewVD))
@@ -4490,7 +4622,8 @@ bool Sema::CheckVariableDeclaration(VarDecl *NewVD,
   if (NewVD->isInvalidDecl())
     return false;
 
-  QualType T = NewVD->getType();
+  TypeSourceInfo *TInfo = NewVD->getTypeSourceInfo();
+  QualType T = TInfo->getType();
 
   if (T->isObjCObjectType()) {
     Diag(NewVD->getLocation(), diag::err_statically_allocated_object)
@@ -4522,8 +4655,10 @@ bool Sema::CheckVariableDeclaration(VarDecl *NewVD,
       && !NewVD->hasAttr<BlocksAttr>()) {
     if (getLangOpts().getGC() != LangOptions::NonGC)
       Diag(NewVD->getLocation(), diag::warn_gc_attribute_weak_on_local);
-    else
+    else {
+      assert(!getLangOpts().ObjCAutoRefCount);
       Diag(NewVD->getLocation(), diag::warn_attribute_weak_on_local);
+    }
   }
   
   bool isVM = T->isVariablyModifiedType();
@@ -4535,11 +4670,10 @@ bool Sema::CheckVariableDeclaration(VarDecl *NewVD,
       (T->isVariableArrayType() && NewVD->hasGlobalStorage())) {
     bool SizeIsNegative;
     llvm::APSInt Oversized;
-    QualType FixedTy =
-        TryToFixInvalidVariablyModifiedType(T, Context, SizeIsNegative,
-                                            Oversized);
-
-    if (FixedTy.isNull() && T->isVariableArrayType()) {
+    TypeSourceInfo *FixedTInfo =
+      TryToFixInvalidVariablyModifiedTypeSourceInfo(TInfo, Context,
+                                                    SizeIsNegative, Oversized);
+    if (FixedTInfo == 0 && T->isVariableArrayType()) {
       const VariableArrayType *VAT = Context.getAsVariableArrayType(T);
       // FIXME: This won't give the correct result for
       // int a[10][n];
@@ -4558,7 +4692,7 @@ bool Sema::CheckVariableDeclaration(VarDecl *NewVD,
       return false;
     }
 
-    if (FixedTy.isNull()) {
+    if (FixedTInfo == 0) {
       if (NewVD->isFileVarDecl())
         Diag(NewVD->getLocation(), diag::err_vm_decl_in_file_scope);
       else
@@ -4568,7 +4702,8 @@ bool Sema::CheckVariableDeclaration(VarDecl *NewVD,
     }
 
     Diag(NewVD->getLocation(), diag::warn_illegal_constant_array_size);
-    NewVD->setType(FixedTy);
+    NewVD->setType(FixedTInfo->getType());
+    NewVD->setTypeSourceInfo(FixedTInfo);
   }
 
   if (Previous.empty() && NewVD->isExternC()) {
@@ -4655,6 +4790,31 @@ static bool FindOverriddenMethod(const CXXBaseSpecifier *Specifier,
   return false;
 }
 
+namespace {
+  enum OverrideErrorKind { OEK_All, OEK_NonDeleted, OEK_Deleted };
+}
+/// \brief Report an error regarding overriding, along with any relevant
+/// overriden methods.
+///
+/// \param DiagID the primary error to report.
+/// \param MD the overriding method.
+/// \param OEK which overrides to include as notes.
+static void ReportOverrides(Sema& S, unsigned DiagID, const CXXMethodDecl *MD,
+                            OverrideErrorKind OEK = OEK_All) {
+  S.Diag(MD->getLocation(), DiagID) << MD->getDeclName();
+  for (CXXMethodDecl::method_iterator I = MD->begin_overridden_methods(),
+                                      E = MD->end_overridden_methods();
+       I != E; ++I) {
+    // This check (& the OEK parameter) could be replaced by a predicate, but
+    // without lambdas that would be overkill. This is still nicer than writing
+    // out the diag loop 3 times.
+    if ((OEK == OEK_All) ||
+        (OEK == OEK_NonDeleted && !(*I)->isDeleted()) ||
+        (OEK == OEK_Deleted && (*I)->isDeleted()))
+      S.Diag((*I)->getLocation(), diag::note_overridden_virtual_function);
+  }
+}
+
 /// AddOverriddenMethods - See if a method overrides any in the base classes,
 /// and if so, check that it's a valid override and remember it.
 bool Sema::AddOverriddenMethods(CXXRecordDecl *DC, CXXMethodDecl *MD) {
@@ -4663,6 +4823,8 @@ bool Sema::AddOverriddenMethods(CXXRecordDecl *DC, CXXMethodDecl *MD) {
   FindOverriddenMethodData Data;
   Data.Method = MD;
   Data.S = this;
+  bool hasDeletedOverridenMethods = false;
+  bool hasNonDeletedOverridenMethods = false;
   bool AddedAny = false;
   if (DC->lookupInBases(&FindOverriddenMethod, &Data, Paths)) {
     for (CXXBasePaths::decl_iterator I = Paths.found_decls_begin(),
@@ -4672,12 +4834,21 @@ bool Sema::AddOverriddenMethods(CXXRecordDecl *DC, CXXMethodDecl *MD) {
         if (!CheckOverridingFunctionReturnType(MD, OldMD) &&
             !CheckOverridingFunctionExceptionSpec(MD, OldMD) &&
             !CheckIfOverriddenFunctionIsMarkedFinal(MD, OldMD)) {
+          hasDeletedOverridenMethods |= OldMD->isDeleted();
+          hasNonDeletedOverridenMethods |= !OldMD->isDeleted();
           AddedAny = true;
         }
       }
     }
   }
-  
+
+  if (hasDeletedOverridenMethods && !MD->isDeleted()) {
+    ReportOverrides(*this, diag::err_non_deleted_override, MD, OEK_Deleted);
+  }
+  if (hasNonDeletedOverridenMethods && MD->isDeleted()) {
+    ReportOverrides(*this, diag::err_deleted_override, MD, OEK_NonDeleted);
+  }
+
   return AddedAny;
 }
 
@@ -4837,6 +5008,10 @@ static NamedDecl* DiagnoseInvalidRedeclaration(
   }
 
   if (Correction) {
+    // FIXME: use Correction.getCorrectionRange() instead of computing the range
+    // here. This requires passing in the CXXScopeSpec to CorrectTypo which in
+    // turn causes the correction to fully qualify the name. If we fix
+    // CorrectTypo to minimally qualify then this change should be good.
     SourceRange FixItLoc(NewFD->getLocation());
     CXXScopeSpec &SS = ExtraArgs.D.getCXXScopeSpec();
     if (Correction.getCorrectionSpecifier() && SS.isValid())
@@ -5072,6 +5247,22 @@ static FunctionDecl* CreateNewFunctionDecl(Sema &SemaRef, Declarator &D,
   }
 }
 
+void Sema::checkVoidParamDecl(ParmVarDecl *Param) {
+  // In C++, the empty parameter-type-list must be spelled "void"; a
+  // typedef of void is not permitted.
+  if (getLangOpts().CPlusPlus &&
+      Param->getType().getUnqualifiedType() != Context.VoidTy) {
+    bool IsTypeAlias = false;
+    if (const TypedefType *TT = Param->getType()->getAs<TypedefType>())
+      IsTypeAlias = isa<TypeAliasDecl>(TT->getDecl());
+    else if (const TemplateSpecializationType *TST =
+               Param->getType()->getAs<TemplateSpecializationType>())
+      IsTypeAlias = TST->isTypeAlias();
+    Diag(Param->getLocation(), diag::err_param_typedef_of_void)
+      << IsTypeAlias;
+  }
+}
+
 NamedDecl*
 Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
                               TypeSourceInfo *TInfo, LookupResult &Previous,
@@ -5138,6 +5329,15 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
       NewFD->setImplicitlyInline();
     }
 
+    // If this is a method defined in an __interface, and is not a constructor
+    // or an overloaded operator, then set the pure flag (isVirtual will already
+    // return true).
+    if (const CXXRecordDecl *Parent =
+          dyn_cast<CXXRecordDecl>(NewFD->getDeclContext())) {
+      if (Parent->isInterface() && cast<CXXMethodDecl>(NewFD)->isUserProvided())
+        NewFD->setPure(true);
+    }
+
     SetNestedNameSpecifier(NewFD, D);
     isExplicitSpecialization = false;
     isFunctionTemplateSpecialization = false;
@@ -5157,7 +5357,7 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
                                   D.getDeclSpec().getLocStart(),
                                   D.getIdentifierLoc(),
                                   D.getCXXScopeSpec(),
-                                  TemplateParamLists.get(),
+                                  TemplateParamLists.data(),
                                   TemplateParamLists.size(),
                                   isFriend,
                                   isExplicitSpecialization,
@@ -5196,7 +5396,7 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
         if (TemplateParamLists.size() > 1) {
           NewFD->setTemplateParameterListsInfo(Context,
                                                TemplateParamLists.size() - 1,
-                                               TemplateParamLists.release());
+                                               TemplateParamLists.data());
         }
       } else {
         // This is a function template specialization.
@@ -5204,7 +5404,7 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
         // For source fidelity, store all the template param lists.
         NewFD->setTemplateParameterListsInfo(Context,
                                              TemplateParamLists.size(),
-                                             TemplateParamLists.release());
+                                             TemplateParamLists.data());
 
         // C++0x [temp.expl.spec]p20 forbids "template<> friend void foo(int);".
         if (isFriend) {
@@ -5236,7 +5436,7 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
         // For source fidelity, store all the template param lists.
         NewFD->setTemplateParameterListsInfo(Context,
                                              TemplateParamLists.size(),
-                                             TemplateParamLists.release());
+                                             TemplateParamLists.data());
     }
 
     if (Invalid) {
@@ -5376,6 +5576,20 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
            diag::err_static_out_of_line)
         << FixItHint::CreateRemoval(D.getDeclSpec().getStorageClassSpecLoc());
     }
+
+    // C++11 [except.spec]p15:
+    //   A deallocation function with no exception-specification is treated
+    //   as if it were specified with noexcept(true).
+    const FunctionProtoType *FPT = R->getAs<FunctionProtoType>();
+    if ((Name.getCXXOverloadedOperator() == OO_Delete ||
+         Name.getCXXOverloadedOperator() == OO_Array_Delete) &&
+        getLangOpts().CPlusPlus0x && FPT && !FPT->hasExceptionSpec()) {
+      FunctionProtoType::ExtProtoInfo EPI = FPT->getExtProtoInfo();
+      EPI.ExceptionSpecType = EST_BasicNoexcept;
+      NewFD->setType(Context.getFunctionType(FPT->getResultType(),
+                                             FPT->arg_type_begin(),
+                                             FPT->getNumArgs(), EPI));
+    }
   }
 
   // Filter out previous declarations that don't match the scope.
@@ -5413,21 +5627,7 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
         FTI.ArgInfo[0].Param &&
         cast<ParmVarDecl>(FTI.ArgInfo[0].Param)->getType()->isVoidType()) {
       // Empty arg list, don't push any params.
-      ParmVarDecl *Param = cast<ParmVarDecl>(FTI.ArgInfo[0].Param);
-
-      // In C++, the empty parameter-type-list must be spelled "void"; a
-      // typedef of void is not permitted.
-      if (getLangOpts().CPlusPlus &&
-          Param->getType().getUnqualifiedType() != Context.VoidTy) {
-        bool IsTypeAlias = false;
-        if (const TypedefType *TT = Param->getType()->getAs<TypedefType>())
-          IsTypeAlias = isa<TypeAliasDecl>(TT->getDecl());
-        else if (const TemplateSpecializationType *TST =
-                   Param->getType()->getAs<TemplateSpecializationType>())
-          IsTypeAlias = TST->isTypeAlias();
-        Diag(Param->getLocation(), diag::err_param_typedef_of_void)
-          << IsTypeAlias;
-      }
+      checkVoidParamDecl(cast<ParmVarDecl>(FTI.ArgInfo[0].Param));
     } else if (FTI.NumArgs > 0 && FTI.ArgInfo[0].Param != 0) {
       for (unsigned i = 0, e = FTI.NumArgs; i != e; ++i) {
         ParmVarDecl *Param = cast<ParmVarDecl>(FTI.ArgInfo[i].Param);
@@ -5500,6 +5700,9 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
       D.setRedeclaration(CheckFunctionDeclaration(S, NewFD, Previous,
                                                   isExplicitSpecialization));
     }
+    // Make graceful recovery from an invalid redeclaration.
+    else if (!Previous.empty())
+           D.setRedeclaration(true);
     assert((NewFD->isInvalidDecl() || !D.isRedeclaration() ||
             Previous.getResultKind() != LookupResult::FoundOverloaded) &&
            "previous declaration set still overloaded");
@@ -5510,12 +5713,10 @@ Sema::ActOnFunctionDeclarator(Scope *S, Declarator &D, DeclContext *DC,
       TemplateIdAnnotation *TemplateId = D.getName().TemplateId;
       TemplateArgs.setLAngleLoc(TemplateId->LAngleLoc);
       TemplateArgs.setRAngleLoc(TemplateId->RAngleLoc);
-      ASTTemplateArgsPtr TemplateArgsPtr(*this,
-                                         TemplateId->getTemplateArgs(),
+      ASTTemplateArgsPtr TemplateArgsPtr(TemplateId->getTemplateArgs(),
                                          TemplateId->NumArgs);
       translateTemplateArguments(TemplateArgsPtr,
                                  TemplateArgs);
-      TemplateArgsPtr.release();
     
       HasExplicitTemplateArgs = true;
     
@@ -5969,20 +6170,12 @@ bool Sema::CheckFunctionDeclaration(Scope *S, FunctionDecl *NewFD,
     // Find any virtual functions that this function overrides.
     if (CXXMethodDecl *Method = dyn_cast<CXXMethodDecl>(NewFD)) {
       if (!Method->isFunctionTemplateSpecialization() && 
-          !Method->getDescribedFunctionTemplate()) {
+          !Method->getDescribedFunctionTemplate() &&
+          Method->isCanonicalDecl()) {
         if (AddOverriddenMethods(Method->getParent(), Method)) {
           // If the function was marked as "static", we have a problem.
           if (NewFD->getStorageClass() == SC_Static) {
-            Diag(NewFD->getLocation(), diag::err_static_overrides_virtual)
-              << NewFD->getDeclName();
-            for (CXXMethodDecl::method_iterator 
-                      Overridden = Method->begin_overridden_methods(),
-                   OverriddenEnd = Method->end_overridden_methods();
-                 Overridden != OverriddenEnd;
-                 ++Overridden) {
-              Diag((*Overridden)->getLocation(), 
-                   diag::note_overridden_virtual_function);
-            }
+            ReportOverrides(*this, diag::err_static_overrides_virtual, Method);
           }
         }
       }
@@ -6191,28 +6384,12 @@ namespace {
       }
     }
 
-    // Sometimes, the expression passed in lacks the casts that are used
-    // to determine which DeclRefExpr's to check.  Assume that the casts
-    // are present and continue visiting the expression.
-    void HandleExpr(Expr *E) {
-      // Skip checking T a = a where T is not a record or reference type.
-      // Doing so is a way to silence uninitialized warnings.
-      if (isRecordType || isReferenceType)
-        if (DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(E))
-          HandleDeclRefExpr(DRE);
-
-      if (ConditionalOperator *CO = dyn_cast<ConditionalOperator>(E)) {
-        HandleValue(CO->getTrueExpr());
-        HandleValue(CO->getFalseExpr());
-      }
-
-      Visit(E);
-    }
-
     // For most expressions, the cast is directly above the DeclRefExpr.
     // For conditional operators, the cast can be outside the conditional
     // operator if both expressions are DeclRefExpr's.
     void HandleValue(Expr *E) {
+      if (isReferenceType)
+        return;
       E = E->IgnoreParenImpCasts();
       if (DeclRefExpr* DRE = dyn_cast<DeclRefExpr>(E)) {
         HandleDeclRefExpr(DRE);
@@ -6222,11 +6399,32 @@ namespace {
       if (ConditionalOperator *CO = dyn_cast<ConditionalOperator>(E)) {
         HandleValue(CO->getTrueExpr());
         HandleValue(CO->getFalseExpr());
+        return;
+      }
+
+      if (isa<MemberExpr>(E)) {
+        Expr *Base = E->IgnoreParenImpCasts();
+        while (MemberExpr *ME = dyn_cast<MemberExpr>(Base)) {
+          // Check for static member variables and don't warn on them.
+          if (!isa<FieldDecl>(ME->getMemberDecl()))
+            return;
+          Base = ME->getBase()->IgnoreParenImpCasts();
+        }
+        if (DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(Base))
+          HandleDeclRefExpr(DRE);
+        return;
       }
     }
 
+    // Reference types are handled here since all uses of references are
+    // bad, not just r-value uses.
+    void VisitDeclRefExpr(DeclRefExpr *E) {
+      if (isReferenceType)
+        HandleDeclRefExpr(E);
+    }
+
     void VisitImplicitCastExpr(ImplicitCastExpr *E) {
-      if ((!isRecordType && E->getCastKind() == CK_LValueToRValue) ||
+      if (E->getCastKind() == CK_LValueToRValue ||
           (isRecordType && E->getCastKind() == CK_NoOp))
         HandleValue(E->getSubExpr());
 
@@ -6237,22 +6435,36 @@ namespace {
       // Don't warn on arrays since they can be treated as pointers.
       if (E->getType()->canDecayToPointerType()) return;
 
-      ValueDecl *VD = E->getMemberDecl();
-      CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(VD);
-      if (isa<FieldDecl>(VD) || (MD && !MD->isStatic()))
-        if (DeclRefExpr *DRE
-              = dyn_cast<DeclRefExpr>(E->getBase()->IgnoreParenImpCasts())) {
+      // Warn when a non-static method call is followed by non-static member
+      // field accesses, which is followed by a DeclRefExpr.
+      CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(E->getMemberDecl());
+      bool Warn = (MD && !MD->isStatic());
+      Expr *Base = E->getBase()->IgnoreParenImpCasts();
+      while (MemberExpr *ME = dyn_cast<MemberExpr>(Base)) {
+        if (!isa<FieldDecl>(ME->getMemberDecl()))
+          Warn = false;
+        Base = ME->getBase()->IgnoreParenImpCasts();
+      }
+
+      if (DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(Base)) {
+        if (Warn)
           HandleDeclRefExpr(DRE);
-          return;
-        }
+        return;
+      }
 
-      Inherited::VisitMemberExpr(E);
+      // The base of a MemberExpr is not a MemberExpr or a DeclRefExpr.
+      // Visit that expression.
+      Visit(Base);
     }
 
     void VisitUnaryOperator(UnaryOperator *E) {
       // For POD record types, addresses of its own members are well-defined.
-      if (E->getOpcode() == UO_AddrOf && isRecordType && isPODType &&
-          isa<MemberExpr>(E->getSubExpr()->IgnoreParens())) return;
+      if (E->getOpcode() == UO_AddrOf && isRecordType &&
+          isa<MemberExpr>(E->getSubExpr()->IgnoreParens())) {
+        if (!isPODType)
+          HandleValue(E->getSubExpr());
+        return;
+      }
       Inherited::VisitUnaryOperator(E);
     } 
 
@@ -6261,20 +6473,38 @@ namespace {
     void HandleDeclRefExpr(DeclRefExpr *DRE) {
       Decl* ReferenceDecl = DRE->getDecl(); 
       if (OrigDecl != ReferenceDecl) return;
-      LookupResult Result(S, DRE->getNameInfo(), Sema::LookupOrdinaryName,
-                          Sema::NotForRedeclaration);
+      unsigned diag = isReferenceType
+          ? diag::warn_uninit_self_reference_in_reference_init
+          : diag::warn_uninit_self_reference_in_init;
       S.DiagRuntimeBehavior(DRE->getLocStart(), DRE,
-                            S.PDiag(diag::warn_uninit_self_reference_in_init)
-                              << Result.getLookupName()
+                            S.PDiag(diag)
+                              << DRE->getNameInfo().getName()
                               << OrigDecl->getLocation()
                               << DRE->getSourceRange());
     }
   };
-}
 
-/// CheckSelfReference - Warns if OrigDecl is used in expression E.
-void Sema::CheckSelfReference(Decl* OrigDecl, Expr *E) {
-  SelfReferenceChecker(*this, OrigDecl).HandleExpr(E);
+  /// CheckSelfReference - Warns if OrigDecl is used in expression E.
+  static void CheckSelfReference(Sema &S, Decl* OrigDecl, Expr *E,
+                                 bool DirectInit) {
+    // Parameters arguments are occassionially constructed with itself,
+    // for instance, in recursive functions.  Skip them.
+    if (isa<ParmVarDecl>(OrigDecl))
+      return;
+
+    E = E->IgnoreParens();
+
+    // Skip checking T a = a where T is not a record or reference type.
+    // Doing so is a way to silence uninitialized warnings.
+    if (!DirectInit && !cast<VarDecl>(OrigDecl)->getType()->isRecordType())
+      if (ImplicitCastExpr *ICE = dyn_cast<ImplicitCastExpr>(E))
+        if (ICE->getCastKind() == CK_LValueToRValue)
+          if (DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(ICE->getSubExpr()))
+            if (DRE->getDecl() == OrigDecl)
+              return;
+
+    SelfReferenceChecker(S, OrigDecl).Visit(E);
+  }
 }
 
 /// AddInitializerToDecl - Adds the initializer Init to the
@@ -6311,15 +6541,6 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init,
     return;
   }
 
-  // Check for self-references within variable initializers.
-  // Variables declared within a function/method body (except for references)
-  // are handled by a dataflow analysis.
-  // Record types initialized by initializer list are handled here.
-  // Initialization by constructors are handled in TryConstructorInitialization.
-  if ((!VDecl->hasLocalStorage() || VDecl->getType()->isReferenceType()) &&
-      (isa<InitListExpr>(Init) || !VDecl->getType()->isRecordType()))
-    CheckSelfReference(RealDecl, Init);
-
   ParenListExpr *CXXDirectInit = dyn_cast<ParenListExpr>(Init);
 
   // C++11 [decl.spec.auto]p6. Deduce the type which 'auto' stands in for.
@@ -6495,8 +6716,7 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init,
     }
     InitializationSequence InitSeq(*this, Entity, Kind, Args, NumArgs);
     ExprResult Result = InitSeq.Perform(*this, Entity, Kind,
-                                              MultiExprArg(*this, Args,NumArgs),
-                                              &DclT);
+                                        MultiExprArg(Args, NumArgs), &DclT);
     if (Result.isInvalid()) {
       VDecl->setInvalidDecl();
       return;
@@ -6505,6 +6725,14 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init,
     Init = Result.takeAs<Expr>();
   }
 
+  // Check for self-references within variable initializers.
+  // Variables declared within a function/method body (except for references)
+  // are handled by a dataflow analysis.
+  if (!VDecl->hasLocalStorage() || VDecl->getType()->isRecordType() ||
+      VDecl->getType()->isReferenceType()) {
+    CheckSelfReference(*this, RealDecl, Init, DirectInit);
+  }
+
   // If the type changed, it means we had an incomplete type that was
   // completed by the initializer. For example:
   //   int ary[] = { 1, 3, 5 };
@@ -6515,9 +6743,28 @@ void Sema::AddInitializerToDecl(Decl *RealDecl, Expr *Init,
   // Check any implicit conversions within the expression.
   CheckImplicitConversions(Init, VDecl->getLocation());
 
-  if (!VDecl->isInvalidDecl())
+  if (!VDecl->isInvalidDecl()) {
     checkUnsafeAssigns(VDecl->getLocation(), VDecl->getType(), Init);
 
+    if (VDecl->hasAttr<BlocksAttr>())
+      checkRetainCycles(VDecl, Init);
+
+    // It is safe to assign a weak reference into a strong variable.
+    // Although this code can still have problems:
+    //   id x = self.weakProp;
+    //   id y = self.weakProp;
+    // we do not warn to warn spuriously when 'x' and 'y' are on separate
+    // paths through the function. This should be revisited if
+    // -Wrepeated-use-of-weak is made flow-sensitive.
+    if (VDecl->getType().getObjCLifetime() == Qualifiers::OCL_Strong) {
+      DiagnosticsEngine::Level Level =
+        Diags.getDiagnosticLevel(diag::warn_arc_repeated_use_of_weak,
+                                 Init->getLocStart());
+      if (Level != DiagnosticsEngine::Ignored)
+        getCurFunction()->markSafeWeakUse(Init);
+    }
+  }
+
   Init = MaybeCreateExprWithCleanups(Init);
   // Attach the initializer to the decl.
   VDecl->setInit(Init);
@@ -6758,8 +7005,10 @@ void Sema::ActOnUninitializedDecl(Decl *RealDecl,
                                  AbstractVariableType))
         Var->setInvalidDecl();
       if (!Type->isDependentType() && !Var->isInvalidDecl() &&
-          Var->getStorageClass() == SC_PrivateExtern)
+          Var->getStorageClass() == SC_PrivateExtern) {
         Diag(Var->getLocation(), diag::warn_private_extern);
+        Diag(Var->getLocation(), diag::note_private_extern);
+      }
         
       return;
 
@@ -6881,8 +7130,7 @@ void Sema::ActOnUninitializedDecl(Decl *RealDecl,
       = InitializationKind::CreateDefault(Var->getLocation());
     
     InitializationSequence InitSeq(*this, Entity, Kind, 0, 0);
-    ExprResult Init = InitSeq.Perform(*this, Entity, Kind,
-                                      MultiExprArg(*this, 0, 0));
+    ExprResult Init = InitSeq.Perform(*this, Entity, Kind, MultiExprArg());
     if (Init.isInvalid())
       Var->setInvalidDecl();
     else if (Init.get()) {
@@ -6957,11 +7205,22 @@ void Sema::CheckCompleteVariableDeclaration(VarDecl *var) {
     }
   }
 
+  if (var->isThisDeclarationADefinition() &&
+      var->getLinkage() == ExternalLinkage) {
+    // Find a previous declaration that's not a definition.
+    VarDecl *prev = var->getPreviousDecl();
+    while (prev && prev->isThisDeclarationADefinition())
+      prev = prev->getPreviousDecl();
+
+    if (!prev)
+      Diag(var->getLocation(), diag::warn_missing_variable_declarations) << var;
+  }
+
   // All the following checks are C++ only.
   if (!getLangOpts().CPlusPlus) return;
 
-  QualType baseType = Context.getBaseElementType(var->getType());
-  if (baseType->isDependentType()) return;
+  QualType type = var->getType();
+  if (type->isDependentType()) return;
 
   // __block variables might require us to capture a copy-initializer.
   if (var->hasAttr<BlocksAttr>()) {
@@ -6970,8 +7229,6 @@ void Sema::CheckCompleteVariableDeclaration(VarDecl *var) {
 
     // Regardless, we don't want to ignore array nesting when
     // constructing this copy.
-    QualType type = var->getType();
-
     if (type->isStructureOrClassType()) {
       SourceLocation poi = var->getLocation();
       Expr *varRef =new (Context) DeclRefExpr(var, false, type, VK_LValue, poi);
@@ -6989,8 +7246,10 @@ void Sema::CheckCompleteVariableDeclaration(VarDecl *var) {
 
   Expr *Init = var->getInit();
   bool IsGlobal = var->hasGlobalStorage() && !var->isStaticLocal();
+  QualType baseType = Context.getBaseElementType(type);
 
-  if (!var->getDeclContext()->isDependentContext() && Init) {
+  if (!var->getDeclContext()->isDependentContext() &&
+      Init && !Init->isValueDependent()) {
     if (IsGlobal && !var->isConstexpr() &&
         getDiagnostics().getDiagnosticLevel(diag::warn_global_constructor,
                                             var->getLocation())
@@ -7178,7 +7437,7 @@ void Sema::ActOnDocumentableDecls(Decl **Group, unsigned NumDecls) {
     // the lookahead in the lexer: we've consumed the semicolon and looked
     // ahead through comments.
     for (unsigned i = 0; i != NumDecls; ++i)
-      Context.getCommentForDecl(Group[i]);
+      Context.getCommentForDecl(Group[i], &PP);
   }
 }
 
@@ -7450,6 +7709,9 @@ void Sema::ActOnFinishKNRParamDeclarations(Scope *S, Declarator &D,
         unsigned DiagID; // unused
         DS.SetTypeSpecType(DeclSpec::TST_int, FTI.ArgInfo[i].IdentLoc,
                            PrevSpec, DiagID);
+        // Use the identifier location for the type source range.
+        DS.SetRangeStart(FTI.ArgInfo[i].IdentLoc);
+        DS.SetRangeEnd(FTI.ArgInfo[i].IdentLoc);
         Declarator ParamD(DS, Declarator::KNRTypeListContext);
         ParamD.SetIdentifier(FTI.ArgInfo[i].Ident, FTI.ArgInfo[i].IdentLoc);
         FTI.ArgInfo[i].Param = ActOnParamDeclarator(S, ParamD);
@@ -7464,8 +7726,7 @@ Decl *Sema::ActOnStartOfFunctionDef(Scope *FnBodyScope, Declarator &D) {
   Scope *ParentScope = FnBodyScope->getParent();
 
   D.setFunctionDefinitionKind(FDK_Definition);
-  Decl *DP = HandleDeclarator(ParentScope, D,
-                              MultiTemplateParamsArg(*this));
+  Decl *DP = HandleDeclarator(ParentScope, D, MultiTemplateParamsArg());
   return ActOnStartOfFunctionDef(FnBodyScope, DP);
 }
 
@@ -7707,7 +7968,7 @@ void Sema::computeNRVO(Stmt *Body, FunctionScopeInfo *Scope) {
 }
 
 Decl *Sema::ActOnFinishFunctionBody(Decl *D, Stmt *BodyArg) {
-  return ActOnFinishFunctionBody(D, move(BodyArg), false);
+  return ActOnFinishFunctionBody(D, BodyArg, false);
 }
 
 Decl *Sema::ActOnFinishFunctionBody(Decl *dcl, Stmt *Body,
@@ -7765,22 +8026,16 @@ Decl *Sema::ActOnFinishFunctionBody(Decl *dcl, Stmt *Body,
       if (Body)
         computeNRVO(Body, getCurFunction());
     }
-    if (getCurFunction()->ObjCShouldCallSuperDealloc) {
-      Diag(MD->getLocEnd(), diag::warn_objc_missing_super_dealloc);
-      getCurFunction()->ObjCShouldCallSuperDealloc = false;
-    }
-    if (getCurFunction()->ObjCShouldCallSuperFinalize) {
-      Diag(MD->getLocEnd(), diag::warn_objc_missing_super_finalize);
-      getCurFunction()->ObjCShouldCallSuperFinalize = false;
+    if (getCurFunction()->ObjCShouldCallSuper) {
+      Diag(MD->getLocEnd(), diag::warn_objc_missing_super_call)
+        << MD->getSelector().getAsString();
+      getCurFunction()->ObjCShouldCallSuper = false;
     }
   } else {
     return 0;
   }
 
-  assert(!getCurFunction()->ObjCShouldCallSuperDealloc &&
-         "This should only be set for ObjC methods, which should have been "
-         "handled in the block above.");
-  assert(!getCurFunction()->ObjCShouldCallSuperFinalize &&
+  assert(!getCurFunction()->ObjCShouldCallSuper &&
          "This should only be set for ObjC methods, which should have been "
          "handled in the block above.");
 
@@ -7916,13 +8171,28 @@ NamedDecl *Sema::ImplicitlyDefineFunction(SourceLocation Loc,
   bool Error = DS.SetTypeSpecType(DeclSpec::TST_int, Loc, Dummy, DiagID);
   (void)Error; // Silence warning.
   assert(!Error && "Error setting up implicit decl!");
+  SourceLocation NoLoc;
   Declarator D(DS, Declarator::BlockContext);
-  D.AddTypeInfo(DeclaratorChunk::getFunction(false, false, false,
-                                             SourceLocation(), 0, 0, 0, true, 
-                                             SourceLocation(), SourceLocation(),
-                                             SourceLocation(), SourceLocation(),
-                                             EST_None, SourceLocation(),
-                                             0, 0, 0, 0, Loc, Loc, D),
+  D.AddTypeInfo(DeclaratorChunk::getFunction(/*HasProto=*/false,
+                                             /*IsAmbiguous=*/false,
+                                             /*RParenLoc=*/NoLoc,
+                                             /*ArgInfo=*/0,
+                                             /*NumArgs=*/0,
+                                             /*EllipsisLoc=*/NoLoc,
+                                             /*RParenLoc=*/NoLoc,
+                                             /*TypeQuals=*/0,
+                                             /*RefQualifierIsLvalueRef=*/true,
+                                             /*RefQualifierLoc=*/NoLoc,
+                                             /*ConstQualifierLoc=*/NoLoc,
+                                             /*VolatileQualifierLoc=*/NoLoc,
+                                             /*MutableLoc=*/NoLoc,
+                                             EST_None,
+                                             /*ESpecLoc=*/NoLoc,
+                                             /*Exceptions=*/0,
+                                             /*ExceptionRanges=*/0,
+                                             /*NumExceptions=*/0,
+                                             /*NoexceptExpr=*/0,
+                                             Loc, Loc, D),
                 DS.getAttributes(),
                 SourceLocation());
   D.SetIdentifier(&II, Loc);
@@ -8071,6 +8341,7 @@ TypedefDecl *Sema::ParseTypedefDecl(Scope *S, Declarator &D, QualType T,
   switch (D.getDeclSpec().getTypeSpecType()) {
   case TST_enum:
   case TST_struct:
+  case TST_interface:
   case TST_union:
   case TST_class: {
     TagDecl *tagFromDeclSpec = cast<TagDecl>(D.getDeclSpec().getRepAsDecl());
@@ -8146,6 +8417,29 @@ bool Sema::CheckEnumRedeclaration(SourceLocation EnumLoc, bool IsScoped,
   return false;
 }
 
+/// \brief Get diagnostic %select index for tag kind for
+/// redeclaration diagnostic message.
+/// WARNING: Indexes apply to particular diagnostics only!
+///
+/// \returns diagnostic %select index.
+static unsigned getRedeclDiagFromTagKind(TagTypeKind Tag) {
+  switch (Tag) {
+  case TTK_Struct: return 0;
+  case TTK_Interface: return 1;
+  case TTK_Class:  return 2;
+  default: llvm_unreachable("Invalid tag kind for redecl diagnostic!");
+  }
+}
+
+/// \brief Determine if tag kind is a class-key compatible with
+/// class for redeclaration (class, struct, or __interface).
+///
+/// \returns true iff the tag kind is compatible.
+static bool isClassCompatTagKind(TagTypeKind Tag)
+{
+  return Tag == TTK_Struct || Tag == TTK_Class || Tag == TTK_Interface;
+}
+
 /// \brief Determine whether a tag with a given kind is acceptable
 /// as a redeclaration of the given tag declaration.
 ///
@@ -8168,12 +8462,11 @@ bool Sema::isAcceptableTagRedeclaration(const TagDecl *Previous,
   //   struct class-key shall be used to refer to a class (clause 9)
   //   declared using the class or struct class-key.
   TagTypeKind OldTag = Previous->getTagKind();
-  if (!isDefinition || (NewTag != TTK_Class && NewTag != TTK_Struct))
+  if (!isDefinition || !isClassCompatTagKind(NewTag))
     if (OldTag == NewTag)
       return true;
 
-  if ((OldTag == TTK_Struct || OldTag == TTK_Class) &&
-      (NewTag == TTK_Struct || NewTag == TTK_Class)) {
+  if (isClassCompatTagKind(OldTag) && isClassCompatTagKind(NewTag)) {
     // Warn about the struct/class tag mismatch.
     bool isTemplate = false;
     if (const CXXRecordDecl *Record = dyn_cast<CXXRecordDecl>(Previous))
@@ -8183,7 +8476,8 @@ bool Sema::isAcceptableTagRedeclaration(const TagDecl *Previous,
       // In a template instantiation, do not offer fix-its for tag mismatches
       // since they usually mess up the template instead of fixing the problem.
       Diag(NewTagLoc, diag::warn_struct_class_tag_mismatch)
-        << (NewTag == TTK_Class) << isTemplate << &Name;
+        << getRedeclDiagFromTagKind(NewTag) << isTemplate << &Name
+        << getRedeclDiagFromTagKind(OldTag);
       return true;
     }
 
@@ -8202,13 +8496,13 @@ bool Sema::isAcceptableTagRedeclaration(const TagDecl *Previous,
           if (!previousMismatch) {
             previousMismatch = true;
             Diag(NewTagLoc, diag::warn_struct_class_previous_tag_mismatch)
-              << (NewTag == TTK_Class) << isTemplate << &Name;
+              << getRedeclDiagFromTagKind(NewTag) << isTemplate << &Name
+              << getRedeclDiagFromTagKind(I->getTagKind());
           }
           Diag(I->getInnerLocStart(), diag::note_struct_class_suggestion)
-            << (NewTag == TTK_Class)
+            << getRedeclDiagFromTagKind(NewTag)
             << FixItHint::CreateReplacement(I->getInnerLocStart(),
-                                            NewTag == TTK_Class?
-                                            "class" : "struct");
+                 TypeWithKeyword::getTagTypeKindName(NewTag));
         }
       }
       return true;
@@ -8224,16 +8518,16 @@ bool Sema::isAcceptableTagRedeclaration(const TagDecl *Previous,
     }
 
     Diag(NewTagLoc, diag::warn_struct_class_tag_mismatch)
-      << (NewTag == TTK_Class)
-      << isTemplate << &Name;
+      << getRedeclDiagFromTagKind(NewTag) << isTemplate << &Name
+      << getRedeclDiagFromTagKind(OldTag);
     Diag(Redecl->getLocation(), diag::note_previous_use);
 
     // If there is a previous defintion, suggest a fix-it.
     if (Previous->getDefinition()) {
         Diag(NewTagLoc, diag::note_struct_class_suggestion)
-          << (Redecl->getTagKind() == TTK_Class)
+          << getRedeclDiagFromTagKind(Redecl->getTagKind())
           << FixItHint::CreateReplacement(SourceRange(NewTagLoc),
-                        Redecl->getTagKind() == TTK_Class? "class" : "struct");
+               TypeWithKeyword::getTagTypeKindName(Redecl->getTagKind()));
     }
 
     return true;
@@ -8276,7 +8570,7 @@ Decl *Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK,
       (SS.isNotEmpty() && TUK != TUK_Reference)) {
     if (TemplateParameterList *TemplateParams
           = MatchTemplateParametersToScopeSpecifier(KWLoc, NameLoc, SS,
-                                                TemplateParameterLists.get(),
+                                                TemplateParameterLists.data(),
                                                 TemplateParameterLists.size(),
                                                     TUK == TUK_Friend,
                                                     isExplicitSpecialization,
@@ -8293,8 +8587,8 @@ Decl *Sema::ActOnTag(Scope *S, unsigned TagSpec, TagUseKind TUK,
                                                SS, Name, NameLoc, Attr,
                                                TemplateParams, AS,
                                                ModulePrivateLoc,
-                                           TemplateParameterLists.size() - 1,
-                 (TemplateParameterList**) TemplateParameterLists.release());
+                                               TemplateParameterLists.size()-1,
+                                               TemplateParameterLists.data());
         return Result.get();
       } else {
         // The "template<>" header is extraneous.
@@ -8843,7 +9137,7 @@ CreateNewDecl:
       if (TemplateParameterLists.size() > 0) {
         New->setTemplateParameterListsInfo(Context,
                                            TemplateParameterLists.size(),
-                    (TemplateParameterList**) TemplateParameterLists.release());
+                                           TemplateParameterLists.data());
       }
     }
     else
@@ -9298,12 +9592,15 @@ FieldDecl *Sema::CheckFieldDecl(DeclarationName Name, QualType T,
   if (!InvalidDecl && T->isVariablyModifiedType()) {
     bool SizeIsNegative;
     llvm::APSInt Oversized;
-    QualType FixedTy = TryToFixInvalidVariablyModifiedType(T, Context,
-                                                           SizeIsNegative,
-                                                           Oversized);
-    if (!FixedTy.isNull()) {
+
+    TypeSourceInfo *FixedTInfo =
+      TryToFixInvalidVariablyModifiedTypeSourceInfo(TInfo, Context,
+                                                    SizeIsNegative,
+                                                    Oversized);
+    if (FixedTInfo) {
       Diag(Loc, diag::warn_illegal_constant_array_size);
-      T = FixedTy;
+      TInfo = FixedTInfo;
+      T = FixedTInfo->getType();
     } else {
       if (SizeIsNegative)
         Diag(Loc, diag::err_typecheck_negative_array_size);
@@ -9460,12 +9757,12 @@ bool Sema::CheckNontrivialField(FieldDecl *FD) {
   return false;
 }
 
-/// If the given constructor is user-provided, produce a diagnostic explaining
+/// If the given constructor is user-declared, produce a diagnostic explaining
 /// that it makes the class non-trivial.
-static bool DiagnoseNontrivialUserProvidedCtor(Sema &S, QualType QT,
+static bool diagnoseNonTrivialUserDeclaredCtor(Sema &S, QualType QT,
                                                CXXConstructorDecl *CD,
                                                Sema::CXXSpecialMember CSM) {
-  if (!CD->isUserProvided())
+  if (CD->isImplicit())
     return false;
 
   SourceLocation CtorLoc = CD->getLocation();
@@ -9488,17 +9785,17 @@ void Sema::DiagnoseNontrivial(const RecordType* T, CXXSpecialMember member) {
     if (RD->hasUserDeclaredConstructor()) {
       typedef CXXRecordDecl::ctor_iterator ctor_iter;
       for (ctor_iter CI = RD->ctor_begin(), CE = RD->ctor_end(); CI != CE; ++CI)
-        if (DiagnoseNontrivialUserProvidedCtor(*this, QT, *CI, member))
+        if (diagnoseNonTrivialUserDeclaredCtor(*this, QT, *CI, member))
           return;
 
-      // No user-provided constructors; look for constructor templates.
+      // No user-delcared constructors; look for constructor templates.
       typedef CXXRecordDecl::specific_decl_iterator<FunctionTemplateDecl>
           tmpl_iter;
       for (tmpl_iter TI(RD->decls_begin()), TE(RD->decls_end());
            TI != TE; ++TI) {
         CXXConstructorDecl *CD =
             dyn_cast<CXXConstructorDecl>(TI->getTemplatedDecl());
-        if (CD && DiagnoseNontrivialUserProvidedCtor(*this, QT, CD, member))
+        if (CD && diagnoseNonTrivialUserDeclaredCtor(*this, QT, CD, member))
           return;
       }
     }
@@ -10025,42 +10322,6 @@ void Sema::ActOnFields(Scope* S,
           Convs->setAccess(I, (*I)->getAccess());
         
         if (!CXXRecord->isDependentType()) {
-          // Objective-C Automatic Reference Counting:
-          //   If a class has a non-static data member of Objective-C pointer
-          //   type (or array thereof), it is a non-POD type and its
-          //   default constructor (if any), copy constructor, copy assignment
-          //   operator, and destructor are non-trivial.
-          //
-          // This rule is also handled by CXXRecordDecl::completeDefinition(). 
-          // However, here we check whether this particular class is only 
-          // non-POD because of the presence of an Objective-C pointer member. 
-          // If so, objects of this type cannot be shared between code compiled 
-          // with ARC and code compiled with manual retain/release.
-          if (getLangOpts().ObjCAutoRefCount &&
-              CXXRecord->hasObjectMember() && 
-              CXXRecord->getLinkage() == ExternalLinkage) {
-            if (CXXRecord->isPOD()) {
-              Diag(CXXRecord->getLocation(), 
-                   diag::warn_arc_non_pod_class_with_object_member)
-               << CXXRecord;
-            } else {
-              // FIXME: Fix-Its would be nice here, but finding a good location
-              // for them is going to be tricky.
-              if (CXXRecord->hasTrivialCopyConstructor())
-                Diag(CXXRecord->getLocation(), 
-                     diag::warn_arc_trivial_member_function_with_object_member)
-                  << CXXRecord << 0;
-              if (CXXRecord->hasTrivialCopyAssignment())
-                Diag(CXXRecord->getLocation(), 
-                     diag::warn_arc_trivial_member_function_with_object_member)
-                << CXXRecord << 1;
-              if (CXXRecord->hasTrivialDestructor())
-                Diag(CXXRecord->getLocation(), 
-                     diag::warn_arc_trivial_member_function_with_object_member)
-                << CXXRecord << 2;
-            }
-          }
-          
           // Adjust user-defined destructor exception spec.
           if (getLangOpts().CPlusPlus0x &&
               CXXRecord->hasUserDeclaredDestructor())
@@ -10092,7 +10353,7 @@ void Sema::ActOnFields(Scope* S,
                 //   class subobject has more than one final overrider the
                 //   program is ill-formed.
                 Diag(Record->getLocation(), diag::err_multiple_final_overriders)
-                  << (NamedDecl *)M->first << Record;
+                  << (const NamedDecl *)M->first << Record;
                 Diag(M->first->getLocation(), 
                      diag::note_overridden_virtual_function);
                 for (OverridingMethods::overriding_iterator 
@@ -10100,7 +10361,7 @@ void Sema::ActOnFields(Scope* S,
                        OMEnd = SO->second.end();
                      OM != OMEnd; ++OM)
                   Diag(OM->Method->getLocation(), diag::note_final_overrider)
-                    << (NamedDecl *)M->first << OM->Method->getParent();
+                    << (const NamedDecl *)M->first << OM->Method->getParent();
                 
                 Record->setInvalidDecl();
               }
@@ -10461,57 +10722,6 @@ Decl *Sema::ActOnEnumConstant(Scope *S, Decl *theEnumDecl, Decl *lastEnumConst,
   return New;
 }
 
-// Emits a warning if every element in the enum is the same value and if
-// every element is initialized with a integer or boolean literal.
-static void CheckForUniqueEnumValues(Sema &S, Decl **Elements,
-                                     unsigned NumElements, EnumDecl *Enum,
-                                     QualType EnumType) {
-  if (S.Diags.getDiagnosticLevel(diag::warn_identical_enum_values,
-                                 Enum->getLocation()) ==
-      DiagnosticsEngine::Ignored)
-    return;
-
-  if (NumElements < 2)
-    return;
-
-  if (!Enum->getIdentifier())
-    return;
-
-  llvm::APSInt FirstVal;
-
-  for (unsigned i = 0; i != NumElements; ++i) {
-    EnumConstantDecl *ECD = cast_or_null<EnumConstantDecl>(Elements[i]);
-    if (!ECD)
-      return;
-
-    Expr *InitExpr = ECD->getInitExpr();
-    if (!InitExpr)
-      return;
-    InitExpr = InitExpr->IgnoreImpCasts();
-    if (!isa<IntegerLiteral>(InitExpr) && !isa<CXXBoolLiteralExpr>(InitExpr))
-      return;
-
-    if (i == 0) {
-      FirstVal = ECD->getInitVal();
-      continue;
-    }
-
-    if (!llvm::APSInt::isSameValue(FirstVal, ECD->getInitVal()))
-      return;
-  }
-
-  S.Diag(Enum->getLocation(), diag::warn_identical_enum_values)
-      << EnumType << FirstVal.toString(10)
-      << Enum->getSourceRange();
-
-  EnumConstantDecl *Last = cast<EnumConstantDecl>(Elements[NumElements - 1]),
-                   *Next = cast<EnumConstantDecl>(Elements[NumElements - 2]);
-
-  S.Diag(Last->getLocation(), diag::note_identical_enum_values)
-    << FixItHint::CreateReplacement(Last->getInitExpr()->getSourceRange(),
-                                    Next->getName());
-}
-
 void Sema::ActOnEnumBody(SourceLocation EnumLoc, SourceLocation LBraceLoc,
                          SourceLocation RBraceLoc, Decl *EnumDeclX,
                          Decl **Elements, unsigned NumElements,
@@ -10734,8 +10944,6 @@ void Sema::ActOnEnumBody(SourceLocation EnumLoc, SourceLocation LBraceLoc,
   // it needs to go into the function scope.
   if (InFunctionDeclarator)
     DeclsInPrototypeScope.push_back(Enum);
-
-  CheckForUniqueEnumValues(*this, Elements, NumElements, Enum, EnumType);
 }
 
 Decl *Sema::ActOnFileScopeAsmDecl(Expr *expr,
@@ -10834,10 +11042,6 @@ Decl *Sema::getObjCDeclContext() const {
 }
 
 AvailabilityResult Sema::getCurContextAvailability() const {
-  const Decl *D = cast<Decl>(getCurLexicalContext());
-  // A category implicitly has the availability of the interface.
-  if (const ObjCCategoryDecl *CatD = dyn_cast<ObjCCategoryDecl>(D))
-    D = CatD->getClassInterface();
-  
+  const Decl *D = cast<Decl>(getCurObjCLexicalContext());
   return D->getAvailability();
 }
diff --git a/lib/Sema/SemaDeclAttr.cpp b/lib/Sema/SemaDeclAttr.cpp
index caa7b2f..e326a20 100644
--- a/lib/Sema/SemaDeclAttr.cpp
+++ b/lib/Sema/SemaDeclAttr.cpp
@@ -415,14 +415,19 @@ static void checkAttrArgsAreLockableObjs(Sema &S, Decl *D,
     }
 
     if (StringLiteral *StrLit = dyn_cast<StringLiteral>(ArgExp)) {
-      // Ignore empty strings without warnings
-      if (StrLit->getLength() == 0)
+      if (StrLit->getLength() == 0 ||
+          StrLit->getString() == StringRef("*")) {
+        // Pass empty strings to the analyzer without warnings.
+        // Treat "*" as the universal lock.
+        Args.push_back(ArgExp);
         continue;
+      }
 
       // We allow constant strings to be used as a placeholder for expressions
       // that are not valid C++ syntax, but warn that they are ignored.
       S.Diag(Attr.getLoc(), diag::warn_thread_attribute_ignored) <<
         Attr.getName();
+      Args.push_back(ArgExp);
       continue;
     }
 
@@ -859,7 +864,6 @@ static void handleLockReturnedAttr(Sema &S, Decl *D,
 
   if (!checkAttributeNumArgs(S, Attr, 1))
     return;
-  Expr *Arg = Attr.getArg(0);
 
   if (!isa<FunctionDecl>(D) && !isa<FunctionTemplateDecl>(D)) {
     S.Diag(Attr.getLoc(), diag::warn_thread_attribute_wrong_decl_type)
@@ -867,9 +871,6 @@ static void handleLockReturnedAttr(Sema &S, Decl *D,
     return;
   }
 
-  if (Arg->isTypeDependent())
-    return;
-
   // check that the argument is lockable object
   SmallVector<Expr*, 1> Args;
   checkAttrArgsAreLockableObjs(S, D, Attr, Args);
@@ -962,7 +963,8 @@ static void handlePackedAttr(Sema &S, Decl *D, const AttributeList &Attr) {
   else if (FieldDecl *FD = dyn_cast<FieldDecl>(D)) {
     // If the alignment is less than or equal to 8 bits, the packed attribute
     // has no effect.
-    if (!FD->getType()->isIncompleteType() &&
+    if (!FD->getType()->isDependentType() &&
+        !FD->getType()->isIncompleteType() &&
         S.Context.getTypeAlign(FD->getType()) <= 8)
       S.Diag(Attr.getLoc(), diag::warn_attribute_ignored_for_field_of_type)
         << Attr.getName() << FD->getType();
@@ -973,8 +975,8 @@ static void handlePackedAttr(Sema &S, Decl *D, const AttributeList &Attr) {
 }
 
 static void handleMsStructAttr(Sema &S, Decl *D, const AttributeList &Attr) {
-  if (TagDecl *TD = dyn_cast<TagDecl>(D))
-    TD->addAttr(::new (S.Context) MsStructAttr(Attr.getRange(), S.Context));
+  if (RecordDecl *RD = dyn_cast<RecordDecl>(D))
+    RD->addAttr(::new (S.Context) MsStructAttr(Attr.getRange(), S.Context));
   else
     S.Diag(Attr.getLoc(), diag::warn_attribute_ignored) << Attr.getName();
 }
@@ -1522,6 +1524,20 @@ static void handleAliasAttr(Sema &S, Decl *D, const AttributeList &Attr) {
                                          Str->getString()));
 }
 
+static void handleMinSizeAttr(Sema &S, Decl *D, const AttributeList &Attr) {
+  // Check the attribute arguments.
+  if (!checkAttributeNumArgs(S, Attr, 0))
+    return;
+
+  if (!isa<FunctionDecl>(D) && !isa<ObjCMethodDecl>(D)) {
+    S.Diag(Attr.getLoc(), diag::err_attribute_wrong_decl_type)
+      << Attr.getName() << ExpectedFunctionOrMethod;
+    return;
+  }
+
+  D->addAttr(::new (S.Context) MinSizeAttr(Attr.getRange(), S.Context));
+}
+
 static void handleColdAttr(Sema &S, Decl *D, const AttributeList &Attr) {
   // Check the attribute arguments.
   if (!checkAttributeNumArgs(S, Attr, 0))
@@ -2268,16 +2284,14 @@ static void handleObjCNSObject(Sema &S, Decl *D, const AttributeList &Attr) {
   }
   if (TypedefNameDecl *TD = dyn_cast<TypedefNameDecl>(D)) {
     QualType T = TD->getUnderlyingType();
-    if (!T->isPointerType() ||
-        !T->getAs<PointerType>()->getPointeeType()->isRecordType()) {
+    if (!T->isCARCBridgableType()) {
       S.Diag(TD->getLocation(), diag::err_nsobject_attribute);
       return;
     }
   }
   else if (ObjCPropertyDecl *PD = dyn_cast<ObjCPropertyDecl>(D)) {
     QualType T = PD->getType();
-    if (!T->isPointerType() ||
-        !T->getAs<PointerType>()->getPointeeType()->isRecordType()) {
+    if (!T->isCARCBridgableType()) {
       S.Diag(PD->getLocation(), diag::err_nsobject_attribute);
       return;
     }
@@ -3583,7 +3597,12 @@ static void handleCallConvAttr(Sema &S, Decl *D, const AttributeList &Attr) {
     }
 
     D->addAttr(::new (S.Context) PcsAttr(Attr.getRange(), S.Context, PCS));
+    return;
   }
+  case AttributeList::AT_PnaclCall:
+    D->addAttr(::new (S.Context) PnaclCallAttr(Attr.getRange(), S.Context));
+    return;
+
   default:
     llvm_unreachable("unexpected attribute kind");
   }
@@ -3636,9 +3655,17 @@ bool Sema::CheckCallingConvAttr(const AttributeList &attr, CallingConv &CC) {
     Diag(attr.getLoc(), diag::err_invalid_pcs);
     return true;
   }
+  case AttributeList::AT_PnaclCall: CC = CC_PnaclCall; break;
   default: llvm_unreachable("unexpected attribute kind");
   }
 
+  const TargetInfo &TI = Context.getTargetInfo();
+  TargetInfo::CallingConvCheckResult A = TI.checkCallingConvention(CC);
+  if (A == TargetInfo::CCCR_Warning) {
+    Diag(attr.getLoc(), diag::warn_cconv_ignored) << attr.getName();
+    CC = TI.getDefaultCallingConv();
+  }
+
   return false;
 }
 
@@ -3878,11 +3905,11 @@ static void handleNSReturnsRetainedAttr(Sema &S, Decl *D,
 
   if (ObjCMethodDecl *MD = dyn_cast<ObjCMethodDecl>(D))
     returnType = MD->getResultType();
-  else if (ObjCPropertyDecl *PD = dyn_cast<ObjCPropertyDecl>(D))
-    returnType = PD->getType();
   else if (S.getLangOpts().ObjCAutoRefCount && hasDeclarator(D) &&
            (Attr.getKind() == AttributeList::AT_NSReturnsRetained))
     return; // ignore: was handled as a type attribute
+  else if (ObjCPropertyDecl *PD = dyn_cast<ObjCPropertyDecl>(D))
+    returnType = PD->getType();
   else if (FunctionDecl *FD = dyn_cast<FunctionDecl>(D))
     returnType = FD->getResultType();
   else {
@@ -3971,6 +3998,33 @@ static void handleObjCReturnsInnerPointerAttr(Sema &S, Decl *D,
     ::new (S.Context) ObjCReturnsInnerPointerAttr(attr.getRange(), S.Context));
 }
 
+static void handleObjCRequiresSuperAttr(Sema &S, Decl *D,
+                                        const AttributeList &attr) {
+  SourceLocation loc = attr.getLoc();
+  ObjCMethodDecl *method = dyn_cast<ObjCMethodDecl>(D);
+  
+  if (!method) {
+   S.Diag(D->getLocStart(), diag::err_attribute_wrong_decl_type)
+   << SourceRange(loc, loc) << attr.getName() << ExpectedMethod;
+    return;
+  }
+  DeclContext *DC = method->getDeclContext();
+  if (const ObjCProtocolDecl *PDecl = dyn_cast_or_null<ObjCProtocolDecl>(DC)) {
+    S.Diag(D->getLocStart(), diag::warn_objc_requires_super_protocol)
+    << attr.getName() << 0;
+    S.Diag(PDecl->getLocation(), diag::note_protocol_decl);
+    return;
+  }
+  if (method->getMethodFamily() == OMF_dealloc) {
+    S.Diag(D->getLocStart(), diag::warn_objc_requires_super_protocol)
+    << attr.getName() << 1;
+    return;
+  }
+  
+  method->addAttr(
+    ::new (S.Context) ObjCRequiresSuperAttr(attr.getRange(), S.Context));
+}
+
 /// Handle cf_audited_transfer and cf_unknown_transfer.
 static void handleCFTransferAttr(Sema &S, Decl *D, const AttributeList &A) {
   if (!isa<FunctionDecl>(D)) {
@@ -4149,19 +4203,21 @@ static void handleUuidAttr(Sema &S, Decl *D, const AttributeList &Attr) {
 }
 
 static void handleInheritanceAttr(Sema &S, Decl *D, const AttributeList &Attr) {
-  if (S.LangOpts.MicrosoftExt) {
-    AttributeList::Kind Kind = Attr.getKind();
-    if (Kind == AttributeList::AT_SingleInheritance)
-      D->addAttr(
-          ::new (S.Context) SingleInheritanceAttr(Attr.getRange(), S.Context));
-    else if (Kind == AttributeList::AT_MultipleInheritance)
-      D->addAttr(
-          ::new (S.Context) MultipleInheritanceAttr(Attr.getRange(), S.Context));
-    else if (Kind == AttributeList::AT_VirtualInheritance)
-      D->addAttr(
-          ::new (S.Context) VirtualInheritanceAttr(Attr.getRange(), S.Context));
-  } else
+  if (!S.LangOpts.MicrosoftExt) {
     S.Diag(Attr.getLoc(), diag::warn_attribute_ignored) << Attr.getName();
+    return;
+  }
+
+  AttributeList::Kind Kind = Attr.getKind();
+  if (Kind == AttributeList::AT_SingleInheritance)
+    D->addAttr(
+        ::new (S.Context) SingleInheritanceAttr(Attr.getRange(), S.Context));
+  else if (Kind == AttributeList::AT_MultipleInheritance)
+    D->addAttr(
+        ::new (S.Context) MultipleInheritanceAttr(Attr.getRange(), S.Context));
+  else if (Kind == AttributeList::AT_VirtualInheritance)
+    D->addAttr(
+        ::new (S.Context) VirtualInheritanceAttr(Attr.getRange(), S.Context));
 }
 
 static void handlePortabilityAttr(Sema &S, Decl *D, const AttributeList &Attr) {
@@ -4246,6 +4302,9 @@ static void ProcessInheritableDeclAttr(Sema &S, Scope *scope, Decl *D,
   case AttributeList::AT_ExtVectorType:
     handleExtVectorTypeAttr(S, scope, D, Attr);
     break;
+  case AttributeList::AT_MinSize:
+    handleMinSizeAttr(S, D, Attr);
+    break;
   case AttributeList::AT_Format:      handleFormatAttr      (S, D, Attr); break;
   case AttributeList::AT_FormatArg:   handleFormatArgAttr   (S, D, Attr); break;
   case AttributeList::AT_CUDAGlobal:  handleGlobalAttr      (S, D, Attr); break;
@@ -4278,6 +4337,9 @@ static void ProcessInheritableDeclAttr(Sema &S, Scope *scope, Decl *D,
   case AttributeList::AT_ObjCReturnsInnerPointer:
     handleObjCReturnsInnerPointerAttr(S, D, Attr); break;
 
+  case AttributeList::AT_ObjCRequiresSuper:
+      handleObjCRequiresSuperAttr(S, D, Attr); break;
+      
   case AttributeList::AT_NSBridged:
     handleNSBridgedAttr(S, scope, D, Attr); break;
 
@@ -4360,6 +4422,7 @@ static void ProcessInheritableDeclAttr(Sema &S, Scope *scope, Decl *D,
   case AttributeList::AT_ThisCall:
   case AttributeList::AT_Pascal:
   case AttributeList::AT_Pcs:
+  case AttributeList::AT_PnaclCall:
     handleCallConvAttr(S, D, Attr);
     break;
   case AttributeList::AT_OpenCLKernel:
@@ -4774,18 +4837,25 @@ static bool isDeclDeprecated(Decl *D) {
 static void
 DoEmitDeprecationWarning(Sema &S, const NamedDecl *D, StringRef Message,
                          SourceLocation Loc,
-                         const ObjCInterfaceDecl *UnknownObjCClass) {
+                         const ObjCInterfaceDecl *UnknownObjCClass,
+                         const ObjCPropertyDecl *ObjCPropery) {
   DeclarationName Name = D->getDeclName();
   if (!Message.empty()) {
     S.Diag(Loc, diag::warn_deprecated_message) << Name << Message;
     S.Diag(D->getLocation(),
            isa<ObjCMethodDecl>(D) ? diag::note_method_declared_at
                                   : diag::note_previous_decl) << Name;
+    if (ObjCPropery)
+      S.Diag(ObjCPropery->getLocation(), diag::note_property_attribute)
+        << ObjCPropery->getDeclName() << 0;
   } else if (!UnknownObjCClass) {
     S.Diag(Loc, diag::warn_deprecated) << D->getDeclName();
     S.Diag(D->getLocation(),
            isa<ObjCMethodDecl>(D) ? diag::note_method_declared_at
                                   : diag::note_previous_decl) << Name;
+    if (ObjCPropery)
+      S.Diag(ObjCPropery->getLocation(), diag::note_property_attribute)
+        << ObjCPropery->getDeclName() << 0;
   } else {
     S.Diag(Loc, diag::warn_deprecated_fwdclass_message) << Name;
     S.Diag(UnknownObjCClass->getLocation(), diag::note_forward_class);
@@ -4800,16 +4870,19 @@ void Sema::HandleDelayedDeprecationCheck(DelayedDiagnostic &DD,
   DD.Triggered = true;
   DoEmitDeprecationWarning(*this, DD.getDeprecationDecl(),
                            DD.getDeprecationMessage(), DD.Loc,
-                           DD.getUnknownObjCClass());
+                           DD.getUnknownObjCClass(),
+                           DD.getObjCProperty());
 }
 
 void Sema::EmitDeprecationWarning(NamedDecl *D, StringRef Message,
                                   SourceLocation Loc,
-                                  const ObjCInterfaceDecl *UnknownObjCClass) {
+                                  const ObjCInterfaceDecl *UnknownObjCClass,
+                                  const ObjCPropertyDecl  *ObjCProperty) {
   // Delay if we're currently parsing a declaration.
   if (DelayedDiagnostics.shouldDelayDiagnostics()) {
     DelayedDiagnostics.add(DelayedDiagnostic::makeDeprecation(Loc, D, 
                                                               UnknownObjCClass,
+                                                              ObjCProperty,
                                                               Message));
     return;
   }
@@ -4817,5 +4890,5 @@ void Sema::EmitDeprecationWarning(NamedDecl *D, StringRef Message,
   // Otherwise, don't warn if our current context is deprecated.
   if (isDeclDeprecated(cast<Decl>(getCurLexicalContext())))
     return;
-  DoEmitDeprecationWarning(*this, D, Message, Loc, UnknownObjCClass);
+  DoEmitDeprecationWarning(*this, D, Message, Loc, UnknownObjCClass, ObjCProperty);
 }
diff --git a/lib/Sema/SemaDeclCXX.cpp b/lib/Sema/SemaDeclCXX.cpp
index eeac9b8..16eddf8 100644
--- a/lib/Sema/SemaDeclCXX.cpp
+++ b/lib/Sema/SemaDeclCXX.cpp
@@ -246,8 +246,7 @@ Sema::SetParamDefaultArgument(ParmVarDecl *Param, Expr *Arg,
   InitializationKind Kind = InitializationKind::CreateCopy(Param->getLocation(),
                                                            EqualLoc);
   InitializationSequence InitSeq(*this, Entity, Kind, &Arg, 1);
-  ExprResult Result = InitSeq.Perform(*this, Entity, Kind,
-                                      MultiExprArg(*this, &Arg, 1));
+  ExprResult Result = InitSeq.Perform(*this, Entity, Kind, Arg);
   if (Result.isInvalid())
     return true;
   Arg = Result.takeAs<Expr>();
@@ -374,10 +373,10 @@ void Sema::CheckExtraCXXDefaultArguments(Declarator &D) {
   }
 }
 
-// MergeCXXFunctionDecl - Merge two declarations of the same C++
-// function, once we already know that they have the same
-// type. Subroutine of MergeFunctionDecl. Returns true if there was an
-// error, false otherwise.
+/// MergeCXXFunctionDecl - Merge two declarations of the same C++
+/// function, once we already know that they have the same
+/// type. Subroutine of MergeFunctionDecl. Returns true if there was an
+/// error, false otherwise.
 bool Sema::MergeCXXFunctionDecl(FunctionDecl *New, FunctionDecl *Old,
                                 Scope *S) {
   bool Invalid = false;
@@ -676,6 +675,20 @@ static bool CheckConstexprParameterTypes(Sema &SemaRef,
   return true;
 }
 
+/// \brief Get diagnostic %select index for tag kind for
+/// record diagnostic message.
+/// WARNING: Indexes apply to particular diagnostics only!
+///
+/// \returns diagnostic %select index.
+static unsigned getRecordDiagFromTagKind(TagTypeKind Tag) {
+  switch (Tag) {
+  case TTK_Struct: return 0;
+  case TTK_Interface: return 1;
+  case TTK_Class:  return 2;
+  default: llvm_unreachable("Invalid tag kind for record diagnostic!");
+  }
+}
+
 // CheckConstexprFunctionDecl - Check whether a function declaration satisfies
 // the requirements of a constexpr function definition or a constexpr
 // constructor definition. If so, return true. If not, produce appropriate
@@ -692,8 +705,8 @@ bool Sema::CheckConstexprFunctionDecl(const FunctionDecl *NewFD) {
     const CXXRecordDecl *RD = MD->getParent();
     if (RD->getNumVBases()) {
       Diag(NewFD->getLocation(), diag::err_constexpr_virtual_base)
-        << isa<CXXConstructorDecl>(NewFD) << RD->isStruct()
-        << RD->getNumVBases();
+        << isa<CXXConstructorDecl>(NewFD)
+        << getRecordDiagFromTagKind(RD->getTagKind()) << RD->getNumVBases();
       for (CXXRecordDecl::base_class_const_iterator I = RD->vbases_begin(),
              E = RD->vbases_end(); I != E; ++I)
         Diag(I->getLocStart(),
@@ -1005,6 +1018,41 @@ bool Sema::isCurrentClassName(const IdentifierInfo &II, Scope *,
     return false;
 }
 
+/// \brief Determine whether the given class is a base class of the given
+/// class, including looking at dependent bases.
+static bool findCircularInheritance(const CXXRecordDecl *Class,
+                                    const CXXRecordDecl *Current) {
+  SmallVector<const CXXRecordDecl*, 8> Queue;
+
+  Class = Class->getCanonicalDecl();
+  while (true) {
+    for (CXXRecordDecl::base_class_const_iterator I = Current->bases_begin(),
+                                                  E = Current->bases_end();
+         I != E; ++I) {
+      CXXRecordDecl *Base = I->getType()->getAsCXXRecordDecl();
+      if (!Base)
+        continue;
+
+      Base = Base->getDefinition();
+      if (!Base)
+        continue;
+
+      if (Base->getCanonicalDecl() == Class)
+        return true;
+
+      Queue.push_back(Base);
+    }
+
+    if (Queue.empty())
+      return false;
+
+    Current = Queue.back();
+    Queue.pop_back();
+  }
+
+  return false;
+}
+
 /// \brief Check the validity of a C++ base class specifier.
 ///
 /// \returns a new CXXBaseSpecifier if well-formed, emits diagnostics
@@ -1031,13 +1079,32 @@ Sema::CheckBaseSpecifier(CXXRecordDecl *Class,
       << TInfo->getTypeLoc().getSourceRange();
     EllipsisLoc = SourceLocation();
   }
-  
-  if (BaseType->isDependentType())
+
+  SourceLocation BaseLoc = TInfo->getTypeLoc().getBeginLoc();
+
+  if (BaseType->isDependentType()) {
+    // Make sure that we don't have circular inheritance among our dependent
+    // bases. For non-dependent bases, the check for completeness below handles
+    // this.
+    if (CXXRecordDecl *BaseDecl = BaseType->getAsCXXRecordDecl()) {
+      if (BaseDecl->getCanonicalDecl() == Class->getCanonicalDecl() ||
+          ((BaseDecl = BaseDecl->getDefinition()) &&
+           findCircularInheritance(Class, BaseDecl))) {
+        Diag(BaseLoc, diag::err_circular_inheritance)
+          << BaseType << Context.getTypeDeclType(Class);
+
+        if (BaseDecl->getCanonicalDecl() != Class->getCanonicalDecl())
+          Diag(BaseDecl->getLocation(), diag::note_previous_decl)
+            << BaseType;
+            
+        return 0;
+      }
+    }
+
     return new (Context) CXXBaseSpecifier(SpecifierRange, Virtual,
                                           Class->getTagKind() == TTK_Class,
                                           Access, TInfo, EllipsisLoc);
-
-  SourceLocation BaseLoc = TInfo->getTypeLoc().getBeginLoc();
+  }
 
   // Base specifiers must be record types.
   if (!BaseType->isRecordType()) {
@@ -1165,10 +1232,21 @@ bool Sema::AttachBaseSpecifiers(CXXRecordDecl *Class, CXXBaseSpecifier **Bases,
       // Okay, add this new base class.
       KnownBase = Bases[idx];
       Bases[NumGoodBases++] = Bases[idx];
-      if (const RecordType *Record = NewBaseType->getAs<RecordType>())
-        if (const CXXRecordDecl *RD = cast<CXXRecordDecl>(Record->getDecl()))
-          if (RD->hasAttr<WeakAttr>())
-            Class->addAttr(::new (Context) WeakAttr(SourceRange(), Context));
+      if (const RecordType *Record = NewBaseType->getAs<RecordType>()) {
+        const CXXRecordDecl *RD = cast<CXXRecordDecl>(Record->getDecl());
+        if (Class->isInterface() &&
+              (!RD->isInterface() ||
+               KnownBase->getAccessSpecifier() != AS_public)) {
+          // The Microsoft extension __interface does not permit bases that
+          // are not themselves public interfaces.
+          Diag(KnownBase->getLocStart(), diag::err_invalid_base_in_interface)
+            << getRecordDiagFromTagKind(RD->getTagKind()) << RD->getName()
+            << RD->getSourceRange();
+          Invalid = true;
+        }
+        if (RD->hasAttr<WeakAttr>())
+          Class->addAttr(::new (Context) WeakAttr(SourceRange(), Context));
+      }
     }
   }
 
@@ -1407,6 +1485,9 @@ bool Sema::ActOnAccessSpecifier(AccessSpecifier Access,
 
 /// CheckOverrideControl - Check C++11 override control semantics.
 void Sema::CheckOverrideControl(Decl *D) {
+  if (D->isInvalidDecl())
+    return;
+
   const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(D);
 
   // Do we know which functions this declaration might be overriding?
@@ -1496,6 +1577,50 @@ Sema::ActOnCXXMemberDeclarator(Scope *S, AccessSpecifier AS, Declarator &D,
 
   bool isFunc = D.isDeclarationOfFunction();
 
+  if (cast<CXXRecordDecl>(CurContext)->isInterface()) {
+    // The Microsoft extension __interface only permits public member functions
+    // and prohibits constructors, destructors, operators, non-public member
+    // functions, static methods and data members.
+    unsigned InvalidDecl;
+    bool ShowDeclName = true;
+    if (!isFunc)
+      InvalidDecl = (DS.getStorageClassSpec() == DeclSpec::SCS_typedef) ? 0 : 1;
+    else if (AS != AS_public)
+      InvalidDecl = 2;
+    else if (DS.getStorageClassSpec() == DeclSpec::SCS_static)
+      InvalidDecl = 3;
+    else switch (Name.getNameKind()) {
+      case DeclarationName::CXXConstructorName:
+        InvalidDecl = 4;
+        ShowDeclName = false;
+        break;
+
+      case DeclarationName::CXXDestructorName:
+        InvalidDecl = 5;
+        ShowDeclName = false;
+        break;
+
+      case DeclarationName::CXXOperatorName:
+      case DeclarationName::CXXConversionFunctionName:
+        InvalidDecl = 6;
+        break;
+
+      default:
+        InvalidDecl = 0;
+        break;
+    }
+
+    if (InvalidDecl) {
+      if (ShowDeclName)
+        Diag(Loc, diag::err_invalid_member_in_interface)
+          << (InvalidDecl-1) << Name;
+      else
+        Diag(Loc, diag::err_invalid_member_in_interface)
+          << (InvalidDecl-1) << "";
+      return 0;
+    }
+  }
+
   // C++ 9.2p6: A member shall not be declared to have automatic storage
   // duration (auto, register) or with the extern storage-class-specifier.
   // C++ 7.1.1p8: The mutable specifier can be applied only to names of class
@@ -1548,7 +1673,7 @@ Sema::ActOnCXXMemberDeclarator(Scope *S, AccessSpecifier AS, Declarator &D,
     // Member field could not be with "template" keyword.
     // So TemplateParameterLists should be empty in this case.
     if (TemplateParameterLists.size()) {
-      TemplateParameterList* TemplateParams = TemplateParameterLists.get()[0];
+      TemplateParameterList* TemplateParams = TemplateParameterLists[0];
       if (TemplateParams->size()) {
         // There is no such thing as a member field template.
         Diag(D.getIdentifierLoc(), diag::err_template_member)
@@ -1588,7 +1713,7 @@ Sema::ActOnCXXMemberDeclarator(Scope *S, AccessSpecifier AS, Declarator &D,
   } else {
     assert(InitStyle == ICIS_NoInit);
 
-    Member = HandleDeclarator(S, D, move(TemplateParameterLists));
+    Member = HandleDeclarator(S, D, TemplateParameterLists);
     if (!Member) {
       return 0;
     }
@@ -1662,6 +1787,99 @@ Sema::ActOnCXXMemberDeclarator(Scope *S, AccessSpecifier AS, Declarator &D,
   return Member;
 }
 
+namespace {
+  class UninitializedFieldVisitor
+      : public EvaluatedExprVisitor<UninitializedFieldVisitor> {
+    Sema &S;
+    ValueDecl *VD;
+  public:
+    typedef EvaluatedExprVisitor<UninitializedFieldVisitor> Inherited;
+    UninitializedFieldVisitor(Sema &S, ValueDecl *VD) : Inherited(S.Context),
+                                                        S(S), VD(VD) {
+    }
+
+    void HandleExpr(Expr *E) {
+      if (!E) return;
+
+      // Expressions like x(x) sometimes lack the surrounding expressions
+      // but need to be checked anyways.
+      HandleValue(E);
+      Visit(E);
+    }
+
+    void HandleValue(Expr *E) {
+      E = E->IgnoreParens();
+
+      if (MemberExpr *ME = dyn_cast<MemberExpr>(E)) {
+        if (isa<EnumConstantDecl>(ME->getMemberDecl()))
+            return;
+        Expr *Base = E;
+        while (isa<MemberExpr>(Base)) {
+          ME = dyn_cast<MemberExpr>(Base);
+          if (VarDecl *VarD = dyn_cast<VarDecl>(ME->getMemberDecl()))
+            if (VarD->hasGlobalStorage())
+              return;
+          Base = ME->getBase();
+        }
+
+        if (VD == ME->getMemberDecl() && isa<CXXThisExpr>(Base)) {
+          unsigned diag = VD->getType()->isReferenceType()
+              ? diag::warn_reference_field_is_uninit
+              : diag::warn_field_is_uninit;
+          S.Diag(ME->getExprLoc(), diag) << ME->getMemberNameInfo().getName();
+          return;
+        }
+      }
+
+      if (ConditionalOperator *CO = dyn_cast<ConditionalOperator>(E)) {
+        HandleValue(CO->getTrueExpr());
+        HandleValue(CO->getFalseExpr());
+        return;
+      }
+
+      if (BinaryConditionalOperator *BCO =
+              dyn_cast<BinaryConditionalOperator>(E)) {
+        HandleValue(BCO->getCommon());
+        HandleValue(BCO->getFalseExpr());
+        return;
+      }
+
+      if (BinaryOperator *BO = dyn_cast<BinaryOperator>(E)) {
+        switch (BO->getOpcode()) {
+        default:
+          return;
+        case(BO_PtrMemD):
+        case(BO_PtrMemI):
+          HandleValue(BO->getLHS());
+          return;
+        case(BO_Comma):
+          HandleValue(BO->getRHS());
+          return;
+        }
+      }
+    }
+
+    void VisitImplicitCastExpr(ImplicitCastExpr *E) {
+      if (E->getCastKind() == CK_LValueToRValue)
+        HandleValue(E->getSubExpr());
+
+      Inherited::VisitImplicitCastExpr(E);
+    }
+
+    void VisitCXXMemberCallExpr(CXXMemberCallExpr *E) {
+      Expr *Callee = E->getCallee();
+      if (isa<MemberExpr>(Callee))
+        HandleValue(Callee);
+
+      Inherited::VisitCXXMemberCallExpr(E);
+    }
+  };
+  static void CheckInitExprContainsUninitializedFields(Sema &S, Expr *E,
+                                                       ValueDecl *VD) {
+    UninitializedFieldVisitor(S, VD).HandleExpr(E);
+  }
+} // namespace
+
 /// ActOnCXXInClassMemberInitializer - This is invoked after parsing an
 /// in-class initializer for a non-static C++ class member, and after
 /// instantiating an in-class initializer in a class template. Such actions
@@ -1685,8 +1903,17 @@ Sema::ActOnCXXInClassMemberInitializer(Decl *D, SourceLocation InitLoc,
     return;
   }
 
+  if (getDiagnostics().getDiagnosticLevel(diag::warn_field_is_uninit, InitLoc)
+      != DiagnosticsEngine::Ignored) {
+    CheckInitExprContainsUninitializedFields(*this, InitExpr, FD);
+  }
+
   ExprResult Init = InitExpr;
-  if (!FD->getType()->isDependentType() && !InitExpr->isTypeDependent()) {
+  if (!FD->getType()->isDependentType() && !InitExpr->isTypeDependent() &&
+      !FD->getDeclContext()->isDependentContext()) {
+    // Note: We don't type-check when we're in a dependent context, because
+    // the initialization-substitution code does not properly handle direct
+    // list initialization. We have the same hackaround for ctor-initializers.
     if (isa<InitListExpr>(InitExpr) && isStdInitializerList(FD->getType(), 0)) {
       Diag(FD->getLocation(), diag::warn_dangling_std_initializer_list)
         << /*at end of ctor*/1 << InitExpr->getSourceRange();
@@ -1795,7 +2022,8 @@ Sema::ActOnMemInitializer(Decl *ConstructorD,
                           Expr **Args, unsigned NumArgs,
                           SourceLocation RParenLoc,
                           SourceLocation EllipsisLoc) {
-  Expr *List = new (Context) ParenListExpr(Context, LParenLoc, Args, NumArgs,
+  Expr *List = new (Context) ParenListExpr(Context, LParenLoc,
+                                           llvm::makeArrayRef(Args, NumArgs),
                                            RParenLoc);
   return BuildMemInitializer(ConstructorD, S, SS, MemberOrBase, TemplateTypeTy,
                              DS, IdLoc, List, EllipsisLoc);
@@ -2044,96 +2272,6 @@ static void CheckForDanglingReferenceOrPointer(Sema &S, ValueDecl *Member,
     << (unsigned)IsPointer;
 }
 
-namespace {
-  class UninitializedFieldVisitor
-      : public EvaluatedExprVisitor<UninitializedFieldVisitor> {
-    Sema &S;
-    ValueDecl *VD;
-  public:
-    typedef EvaluatedExprVisitor<UninitializedFieldVisitor> Inherited;
-    UninitializedFieldVisitor(Sema &S, ValueDecl *VD) : Inherited(S.Context),
-                                                        S(S), VD(VD) {
-    }
-
-    void HandleExpr(Expr *E) {
-      if (!E) return;
-
-      // Expressions like x(x) sometimes lack the surrounding expressions
-      // but need to be checked anyways.
-      HandleValue(E);
-      Visit(E);
-    }
-
-    void HandleValue(Expr *E) {
-      E = E->IgnoreParens();
-
-      if (MemberExpr *ME = dyn_cast<MemberExpr>(E)) {
-        if (isa<EnumConstantDecl>(ME->getMemberDecl()))
-            return;
-        Expr *Base = E;
-        while (isa<MemberExpr>(Base)) {
-          ME = dyn_cast<MemberExpr>(Base);
-          if (VarDecl *VarD = dyn_cast<VarDecl>(ME->getMemberDecl()))
-            if (VarD->hasGlobalStorage())
-              return;
-          Base = ME->getBase();
-        }
-
-        if (VD == ME->getMemberDecl() && isa<CXXThisExpr>(Base)) {
-          S.Diag(ME->getExprLoc(), diag::warn_field_is_uninit);
-          return;
-        }
-      }
-
-      if (ConditionalOperator *CO = dyn_cast<ConditionalOperator>(E)) {
-        HandleValue(CO->getTrueExpr());
-        HandleValue(CO->getFalseExpr());
-        return;
-      }
-
-      if (BinaryConditionalOperator *BCO =
-              dyn_cast<BinaryConditionalOperator>(E)) {
-        HandleValue(BCO->getCommon());
-        HandleValue(BCO->getFalseExpr());
-        return;
-      }
-
-      if (BinaryOperator *BO = dyn_cast<BinaryOperator>(E)) {
-        switch (BO->getOpcode()) {
-        default:
-          return;
-        case(BO_PtrMemD):
-        case(BO_PtrMemI):
-          HandleValue(BO->getLHS());
-          return;
-        case(BO_Comma):
-          HandleValue(BO->getRHS());
-          return;
-        }
-      }
-    }
-
-    void VisitImplicitCastExpr(ImplicitCastExpr *E) {
-      if (E->getCastKind() == CK_LValueToRValue)
-        HandleValue(E->getSubExpr());
-
-      Inherited::VisitImplicitCastExpr(E);
-    }
-
-    void VisitCXXMemberCallExpr(CXXMemberCallExpr *E) {
-      Expr *Callee = E->getCallee();
-      if (isa<MemberExpr>(Callee))
-        HandleValue(Callee);
-
-      Inherited::VisitCXXMemberCallExpr(E);
-    }
-  };
-  static void CheckInitExprContainsUninitializedFields(Sema &S, Expr *E,
-                                                       ValueDecl *VD) {
-    UninitializedFieldVisitor(S, VD).HandleExpr(E);
-  }
-} // namespace
-
 MemInitResult
 Sema::BuildMemberInitializer(ValueDecl *Member, Expr *Init,
                              SourceLocation IdLoc) {
@@ -2167,11 +2305,13 @@ Sema::BuildMemberInitializer(ValueDecl *Member, Expr *Init,
         != DiagnosticsEngine::Ignored)
     for (unsigned i = 0; i < NumArgs; ++i)
       // FIXME: Warn about the case when other fields are used before being
-      // uninitialized. For example, let this field be the i'th field. When
+      // initialized. For example, let this field be the i'th field. When
       // initializing the i'th field, throw a warning if any of the >= i'th
       // fields are used, as they are not yet initialized.
       // Right now we are only handling the case where the i'th field uses
       // itself in its initializer.
+      // Also need to take into account that some fields may be initialized by
+      // in-class initializers, see C++11 [class.base.init]p9.
       CheckInitExprContainsUninitializedFields(*this, Args[i], Member);
 
   SourceRange InitRange = Init->getSourceRange();
@@ -2204,7 +2344,7 @@ Sema::BuildMemberInitializer(ValueDecl *Member, Expr *Init,
 
     InitializationSequence InitSeq(*this, MemberEntity, Kind, Args, NumArgs);
     ExprResult MemberInit = InitSeq.Perform(*this, MemberEntity, Kind,
-                                            MultiExprArg(*this, Args, NumArgs),
+                                            MultiExprArg(Args, NumArgs),
                                             0);
     if (MemberInit.isInvalid())
       return true;
@@ -2273,7 +2413,7 @@ Sema::BuildDelegatingInitializer(TypeSourceInfo *TInfo, Expr *Init,
                                                 InitRange.getEnd());
   InitializationSequence InitSeq(*this, DelegationEntity, Kind, Args, NumArgs);
   ExprResult DelegationInit = InitSeq.Perform(*this, DelegationEntity, Kind,
-                                              MultiExprArg(*this, Args,NumArgs),
+                                              MultiExprArg(Args, NumArgs),
                                               0);
   if (DelegationInit.isInvalid())
     return true;
@@ -2411,8 +2551,7 @@ Sema::BuildBaseInitializer(QualType BaseType, TypeSourceInfo *BaseTInfo,
                                                 InitRange.getEnd());
   InitializationSequence InitSeq(*this, BaseEntity, Kind, Args, NumArgs);
   ExprResult BaseInit = InitSeq.Perform(*this, BaseEntity, Kind,
-                                          MultiExprArg(*this, Args, NumArgs),
-                                          0);
+                                        MultiExprArg(Args, NumArgs), 0);
   if (BaseInit.isInvalid())
     return true;
 
@@ -2480,8 +2619,7 @@ BuildImplicitBaseInitializer(Sema &SemaRef, CXXConstructorDecl *Constructor,
     InitializationKind InitKind
       = InitializationKind::CreateDefault(Constructor->getLocation());
     InitializationSequence InitSeq(SemaRef, InitEntity, InitKind, 0, 0);
-    BaseInit = InitSeq.Perform(SemaRef, InitEntity, InitKind,
-                               MultiExprArg(SemaRef, 0, 0));
+    BaseInit = InitSeq.Perform(SemaRef, InitEntity, InitKind, MultiExprArg());
     break;
   }
 
@@ -2936,7 +3074,11 @@ bool Sema::SetCtorInitializers(CXXConstructorDecl *Constructor,
              NumInitializers * sizeof(CXXCtorInitializer*));
       Constructor->setCtorInitializers(baseOrMemberInitializers);
     }
-    
+
+    // Let template instantiation know whether we had errors.
+    if (AnyErrors)
+      Constructor->setInvalidDecl();
+
     return false;
   }
 
@@ -3324,11 +3466,10 @@ void Sema::ActOnMemInitializers(Decl *ConstructorDecl,
     } else {
       assert(Init->isDelegatingInitializer());
       // This must be the only initializer
-      if (i != 0 || NumMemInits > 1) {
-        Diag(MemInits[0]->getSourceLocation(),
+      if (NumMemInits != 1) {
+        Diag(Init->getSourceLocation(),
              diag::err_delegating_initializer_alone)
-          << MemInits[0]->getSourceRange();
-        HadError = true;
+          << Init->getSourceRange() << MemInits[i ? 0 : 1]->getSourceRange();
         // We will treat this as being the only initializer.
       }
       SetDelegatingInitializer(Constructor, MemInits[i]);
@@ -3812,6 +3953,11 @@ void Sema::CheckCompletedCXXClass(CXXRecordDecl *Record) {
            diag::warn_non_virtual_dtor) << Context.getRecordType(Record);
   }
 
+  if (Record->isAbstract() && Record->hasAttr<FinalAttr>()) {
+    Diag(Record->getLocation(), diag::warn_abstract_final_class);
+    DiagnoseAbstractType(Record);
+  }
+
   // See if a method overloads virtual methods in a base
   /// class without overriding any.
   if (!Record->isDependentType()) {
@@ -4065,7 +4211,7 @@ void Sema::CheckExplicitlyDefaultedSpecialMember(CXXMethodDecl *MD) {
 
   // Compute argument constness, constexpr, and triviality.
   bool CanHaveConstParam = false;
-  bool Trivial;
+  bool Trivial = false;
   switch (CSM) {
   case CXXDefaultConstructor:
     Trivial = RD->hasTrivialDefaultConstructor();
@@ -4304,7 +4450,7 @@ bool SpecialMemberDeletionInfo::isAccessible(Subobject Subobj,
   /// If we're operating on a base class, the object type is the
   /// type of this special member.
   QualType objectTy;
-  AccessSpecifier access = target->getAccess();;
+  AccessSpecifier access = target->getAccess();
   if (CXXBaseSpecifier *base = Subobj.dyn_cast<CXXBaseSpecifier*>()) {
     objectTy = S.Context.getTypeDeclType(MD->getParent());
     access = CXXRecordDecl::MergeAccess(base->getAccessSpecifier(), access);
@@ -4647,6 +4793,19 @@ namespace {
   };
 }
 
+/// \brief Check whether any most overriden method from MD in Methods
+static bool CheckMostOverridenMethods(const CXXMethodDecl *MD,
+                   const llvm::SmallPtrSet<const CXXMethodDecl *, 8>& Methods) {
+  if (MD->size_overridden_methods() == 0)
+    return Methods.count(MD->getCanonicalDecl());
+  for (CXXMethodDecl::method_iterator I = MD->begin_overridden_methods(),
+                                      E = MD->end_overridden_methods();
+       I != E; ++I)
+    if (CheckMostOverridenMethods(*I, Methods))
+      return true;
+  return false;
+}
+
 /// \brief Member lookup function that determines whether a given C++
 /// method overloads virtual methods in a base class without overriding any,
 /// to be used with CXXRecordDecl::lookupInBases().
@@ -4678,7 +4837,7 @@ static bool FindHiddenVirtualMethod(const CXXBaseSpecifier *Specifier,
       if (!Data.S->IsOverload(Data.Method, MD, false))
         return true;
       // Collect the overload only if its hidden.
-      if (!Data.OverridenAndUsingBaseMethods.count(MD))
+      if (!CheckMostOverridenMethods(MD, Data.OverridenAndUsingBaseMethods))
         overloadedMethods.push_back(MD);
     }
   }
@@ -4689,6 +4848,17 @@ static bool FindHiddenVirtualMethod(const CXXBaseSpecifier *Specifier,
   return foundSameNameMethod;
 }
 
+/// \brief Add the most overriden methods from MD to Methods
+static void AddMostOverridenMethods(const CXXMethodDecl *MD,
+                         llvm::SmallPtrSet<const CXXMethodDecl *, 8>& Methods) {
+  if (MD->size_overridden_methods() == 0)
+    Methods.insert(MD->getCanonicalDecl());
+  for (CXXMethodDecl::method_iterator I = MD->begin_overridden_methods(),
+                                      E = MD->end_overridden_methods();
+       I != E; ++I)
+    AddMostOverridenMethods(*I, Methods);
+}
+
 /// \brief See if a method overloads virtual methods in a base class without
 /// overriding any.
 void Sema::DiagnoseHiddenVirtualMethods(CXXRecordDecl *DC, CXXMethodDecl *MD) {
@@ -4709,14 +4879,11 @@ void Sema::DiagnoseHiddenVirtualMethods(CXXRecordDecl *DC, CXXMethodDecl *MD) {
   // by 'using' in a set. A base method not in this set is hidden.
   for (DeclContext::lookup_result res = DC->lookup(MD->getDeclName());
        res.first != res.second; ++res.first) {
-    if (CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(*res.first))
-      for (CXXMethodDecl::method_iterator I = MD->begin_overridden_methods(),
-                                          E = MD->end_overridden_methods();
-           I != E; ++I)
-        Data.OverridenAndUsingBaseMethods.insert((*I)->getCanonicalDecl());
+    NamedDecl *ND = *res.first;
     if (UsingShadowDecl *shad = dyn_cast<UsingShadowDecl>(*res.first))
-      if (CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(shad->getTargetDecl()))
-        Data.OverridenAndUsingBaseMethods.insert(MD->getCanonicalDecl());
+      ND = shad->getTargetDecl();
+    if (CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(ND))
+      AddMostOverridenMethods(MD, Data.OverridenAndUsingBaseMethods);
   }
 
   if (DC->lookupInBases(&FindHiddenVirtualMethod, &Data, Paths) &&
@@ -5307,7 +5474,47 @@ Decl *Sema::ActOnConversionDeclarator(CXXConversionDecl *Conversion) {
 // Namespace Handling
 //===----------------------------------------------------------------------===//
 
+/// \brief Diagnose a mismatch in 'inline' qualifiers when a namespace is
+/// reopened.
+static void DiagnoseNamespaceInlineMismatch(Sema &S, SourceLocation KeywordLoc,
+                                            SourceLocation Loc,
+                                            IdentifierInfo *II, bool *IsInline,
+                                            NamespaceDecl *PrevNS) {
+  assert(*IsInline != PrevNS->isInline());
+
+  // HACK: Work around a bug in libstdc++4.6's <atomic>, where
+  // std::__atomic[0,1,2] are defined as non-inline namespaces, then reopened as
+  // inline namespaces, with the intention of bringing names into namespace std.
+  //
+  // We support this just well enough to get that case working; this is not
+  // sufficient to support reopening namespaces as inline in general.
+  if (*IsInline && II && II->getName().startswith("__atomic") &&
+      S.getSourceManager().isInSystemHeader(Loc)) {
+    // Mark all prior declarations of the namespace as inline.
+    for (NamespaceDecl *NS = PrevNS->getMostRecentDecl(); NS;
+         NS = NS->getPreviousDecl())
+      NS->setInline(*IsInline);
+    // Patch up the lookup table for the containing namespace. This isn't really
+    // correct, but it's good enough for this particular case.
+    for (DeclContext::decl_iterator I = PrevNS->decls_begin(),
+                                    E = PrevNS->decls_end(); I != E; ++I)
+      if (NamedDecl *ND = dyn_cast<NamedDecl>(*I))
+        PrevNS->getParent()->makeDeclVisibleInContext(ND);
+    return;
+  }
+
+  if (PrevNS->isInline())
+    // The user probably just forgot the 'inline', so suggest that it
+    // be added back.
+    S.Diag(Loc, diag::warn_inline_namespace_reopened_noninline)
+      << FixItHint::CreateInsertion(KeywordLoc, "inline ");
+  else
+    S.Diag(Loc, diag::err_inline_namespace_mismatch)
+      << IsInline;
 
+  S.Diag(PrevNS->getLocation(), diag::note_previous_definition);
+  *IsInline = PrevNS->isInline();
+}
 
 /// ActOnStartNamespaceDef - This is called at the start of a namespace
 /// definition.
@@ -5357,21 +5564,9 @@ Decl *Sema::ActOnStartNamespaceDef(Scope *NamespcScope,
     
     if (PrevNS) {
       // This is an extended namespace definition.
-      if (IsInline != PrevNS->isInline()) {
-        // inline-ness must match
-        if (PrevNS->isInline()) {
-          // The user probably just forgot the 'inline', so suggest that it
-          // be added back.
-          Diag(Loc, diag::warn_inline_namespace_reopened_noninline)
-            << FixItHint::CreateInsertion(NamespaceLoc, "inline ");
-        } else {
-          Diag(Loc, diag::err_inline_namespace_mismatch)
-            << IsInline;
-        }
-        Diag(PrevNS->getLocation(), diag::note_previous_definition);
-        
-        IsInline = PrevNS->isInline();
-      }      
+      if (IsInline != PrevNS->isInline())
+        DiagnoseNamespaceInlineMismatch(*this, NamespaceLoc, Loc, II,
+                                        &IsInline, PrevNS);
     } else if (PrevDecl) {
       // This is an invalid name redefinition.
       Diag(Loc, diag::err_redefinition_different_kind)
@@ -5402,15 +5597,9 @@ Decl *Sema::ActOnStartNamespaceDef(Scope *NamespcScope,
       PrevNS = ND->getAnonymousNamespace();
     }
 
-    if (PrevNS && IsInline != PrevNS->isInline()) {
-      // inline-ness must match
-      Diag(Loc, diag::err_inline_namespace_mismatch)
-        << IsInline;
-      Diag(PrevNS->getLocation(), diag::note_previous_definition);
-      
-      // Recover by ignoring the new namespace's inline status.
-      IsInline = PrevNS->isInline();
-    }
+    if (PrevNS && IsInline != PrevNS->isInline())
+      DiagnoseNamespaceInlineMismatch(*this, NamespaceLoc, NamespaceLoc, II,
+                                      &IsInline, PrevNS);
   }
   
   NamespaceDecl *Namespc = NamespaceDecl::Create(Context, CurContext, IsInline,
@@ -5460,15 +5649,15 @@ Decl *Sema::ActOnStartNamespaceDef(Scope *NamespcScope,
 
     if (!PrevNS) {
       UsingDirectiveDecl* UD
-        = UsingDirectiveDecl::Create(Context, CurContext,
+        = UsingDirectiveDecl::Create(Context, Parent,
                                      /* 'using' */ LBrace,
                                      /* 'namespace' */ SourceLocation(),
                                      /* qualifier */ NestedNameSpecifierLoc(),
                                      /* identifier */ SourceLocation(),
                                      Namespc,
-                                     /* Ancestor */ CurContext);
+                                     /* Ancestor */ Parent);
       UD->setImplicit();
-      CurContext->addDecl(UD);
+      Parent->addDecl(UD);
     }
   }
 
@@ -5697,7 +5886,8 @@ static bool TryNamespaceTypoCorrection(Sema &S, LookupResult &R, Scope *Sc,
     if (DeclContext *DC = S.computeDeclContext(SS, false))
       S.Diag(IdentLoc, diag::err_using_directive_member_suggest)
         << Ident << DC << CorrectedQuotedStr << SS.getRange()
-        << FixItHint::CreateReplacement(IdentLoc, CorrectedStr);
+        << FixItHint::CreateReplacement(Corrected.getCorrectionRange(),
+                                        CorrectedStr);
     else
       S.Diag(IdentLoc, diag::err_using_directive_suggest)
         << Ident << CorrectedQuotedStr
@@ -6562,10 +6752,10 @@ Decl *Sema::ActOnAliasDeclaration(Scope *S,
 
     if (TemplateParamLists.size() != 1) {
       Diag(UsingLoc, diag::err_alias_template_extra_headers)
-        << SourceRange(TemplateParamLists.get()[1]->getTemplateLoc(),
-         TemplateParamLists.get()[TemplateParamLists.size()-1]->getRAngleLoc());
+        << SourceRange(TemplateParamLists[1]->getTemplateLoc(),
+         TemplateParamLists[TemplateParamLists.size()-1]->getRAngleLoc());
     }
-    TemplateParameterList *TemplateParams = TemplateParamLists.get()[0];
+    TemplateParameterList *TemplateParams = TemplateParamLists[0];
 
     // Only consider previous declarations in the same scope.
     FilterLookupForScope(Previous, CurContext, S, /*ConsiderLinkage*/false,
@@ -6696,28 +6886,6 @@ Decl *Sema::ActOnNamespaceAliasDef(Scope *S,
   return AliasDecl;
 }
 
-namespace {
-  /// \brief Scoped object used to handle the state changes required in Sema
-  /// to implicitly define the body of a C++ member function;
-  class ImplicitlyDefinedFunctionScope {
-    Sema &S;
-    Sema::ContextRAII SavedContext;
-    
-  public:
-    ImplicitlyDefinedFunctionScope(Sema &S, CXXMethodDecl *Method)
-      : S(S), SavedContext(S, Method) 
-    {
-      S.PushFunctionScope();
-      S.PushExpressionEvaluationContext(Sema::PotentiallyEvaluated);
-    }
-    
-    ~ImplicitlyDefinedFunctionScope() {
-      S.PopExpressionEvaluationContext();
-      S.PopFunctionScopeInfo();
-    }
-  };
-}
-
 Sema::ImplicitExceptionSpecification
 Sema::ComputeDefaultedDefaultCtorExceptionSpec(SourceLocation Loc,
                                                CXXMethodDecl *MD) {
@@ -6861,7 +7029,7 @@ void Sema::DefineImplicitDefaultConstructor(SourceLocation CurrentLocation,
   CXXRecordDecl *ClassDecl = Constructor->getParent();
   assert(ClassDecl && "DefineImplicitDefaultConstructor - invalid constructor");
 
-  ImplicitlyDefinedFunctionScope Scope(*this, Constructor);
+  SynthesizedFunctionScope Scope(*this, Constructor);
   DiagnosticErrorTrap Trap(Diags);
   if (SetCtorInitializers(Constructor, 0, 0, /*AnyErrors=*/false) ||
       Trap.hasErrorOccurred()) {
@@ -7173,7 +7341,7 @@ void Sema::DefineImplicitDestructor(SourceLocation CurrentLocation,
   if (Destructor->isInvalidDecl())
     return;
 
-  ImplicitlyDefinedFunctionScope Scope(*this, Destructor);
+  SynthesizedFunctionScope Scope(*this, Destructor);
 
   DiagnosticErrorTrap Trap(Diags);
   MarkBaseAndMemberDestructorsReferenced(Destructor->getLocation(),
@@ -7412,7 +7580,7 @@ BuildSingleCopyAssign(Sema &S, SourceLocation Loc, QualType T,
     = new (S.Context) BinaryOperator(IterationVarRefRVal,
                      IntegerLiteral::Create(S.Context, Upper, SizeType, Loc),
                                      BO_NE, S.Context.BoolTy,
-                                     VK_RValue, OK_Ordinary, Loc);
+                                     VK_RValue, OK_Ordinary, Loc, false);
   
   // Create the pre-increment of the iteration variable.
   Expr *Increment
@@ -7654,7 +7822,7 @@ void Sema::DefineImplicitCopyAssignment(SourceLocation CurrentLocation,
   
   CopyAssignOperator->setUsed();
 
-  ImplicitlyDefinedFunctionScope Scope(*this, CopyAssignOperator);
+  SynthesizedFunctionScope Scope(*this, CopyAssignOperator);
   DiagnosticErrorTrap Trap(Diags);
 
   // C++0x [class.copy]p30:
@@ -7666,7 +7834,7 @@ void Sema::DefineImplicitCopyAssignment(SourceLocation CurrentLocation,
   //   which they were declared in the class definition.
   
   // The statements that form the synthesized function body.
-  ASTOwningVector<Stmt*> Statements(*this);
+  SmallVector<Stmt*, 8> Statements;
   
   // The parameter for the "other" object, which we are copying from.
   ParmVarDecl *Other = CopyAssignOperator->getParamDecl(0);
@@ -7846,8 +8014,8 @@ void Sema::DefineImplicitCopyAssignment(SourceLocation CurrentLocation,
           }
         
           CollectableMemCpyRef = BuildDeclRefExpr(CollectableMemCpy, 
-                                                  CollectableMemCpy->getType(),
-                                                  VK_LValue, Loc, 0).take();
+                                                  Context.BuiltinFnTy,
+                                                  VK_RValue, Loc, 0).take();
           assert(CollectableMemCpyRef && "Builtin reference cannot fail");
         }
       }
@@ -7866,12 +8034,12 @@ void Sema::DefineImplicitCopyAssignment(SourceLocation CurrentLocation,
         }
 
         BuiltinMemCpyRef = BuildDeclRefExpr(BuiltinMemCpy, 
-                                            BuiltinMemCpy->getType(),
-                                            VK_LValue, Loc, 0).take();
+                                            Context.BuiltinFnTy,
+                                            VK_RValue, Loc, 0).take();
         assert(BuiltinMemCpyRef && "Builtin reference cannot fail");
       }
           
-      ASTOwningVector<Expr*> CallArgs(*this);
+      SmallVector<Expr*, 8> CallArgs;
       CallArgs.push_back(To.takeAs<Expr>());
       CallArgs.push_back(From.takeAs<Expr>());
       CallArgs.push_back(IntegerLiteral::Create(Context, Size, SizeType, Loc));
@@ -7879,12 +8047,12 @@ void Sema::DefineImplicitCopyAssignment(SourceLocation CurrentLocation,
       if (NeedsCollectableMemCpy)
         Call = ActOnCallExpr(/*Scope=*/0,
                              CollectableMemCpyRef,
-                             Loc, move_arg(CallArgs), 
+                             Loc, CallArgs,
                              Loc);
       else
         Call = ActOnCallExpr(/*Scope=*/0,
                              BuiltinMemCpyRef,
-                             Loc, move_arg(CallArgs), 
+                             Loc, CallArgs,
                              Loc);
           
       assert(!Call.isInvalid() && "Call to __builtin_memcpy cannot fail!");
@@ -7934,7 +8102,7 @@ void Sema::DefineImplicitCopyAssignment(SourceLocation CurrentLocation,
   StmtResult Body;
   {
     CompoundScopeRAII CompoundScope(*this);
-    Body = ActOnCompoundStmt(Loc, Loc, move_arg(Statements),
+    Body = ActOnCompoundStmt(Loc, Loc, Statements,
                              /*isStmtExpr=*/false);
     assert(!Body.isInvalid() && "Compound statement creation cannot fail");
   }
@@ -8040,7 +8208,7 @@ hasMoveOrIsTriviallyCopyable(Sema &S, QualType Type, bool IsConstructor) {
   // reference types, are supposed to return false here, but that appears
   // to be a standard defect.
   CXXRecordDecl *ClassDecl = Type->getAsCXXRecordDecl();
-  if (!ClassDecl || !ClassDecl->getDefinition())
+  if (!ClassDecl || !ClassDecl->getDefinition() || ClassDecl->isInvalidDecl())
     return true;
 
   if (Type.isTriviallyCopyableType(S.Context))
@@ -8195,7 +8363,7 @@ void Sema::DefineImplicitMoveAssignment(SourceLocation CurrentLocation,
   
   MoveAssignOperator->setUsed();
 
-  ImplicitlyDefinedFunctionScope Scope(*this, MoveAssignOperator);
+  SynthesizedFunctionScope Scope(*this, MoveAssignOperator);
   DiagnosticErrorTrap Trap(Diags);
 
   // C++0x [class.copy]p28:
@@ -8207,7 +8375,7 @@ void Sema::DefineImplicitMoveAssignment(SourceLocation CurrentLocation,
   //   definition.
 
   // The statements that form the synthesized function body.
-  ASTOwningVector<Stmt*> Statements(*this);
+  SmallVector<Stmt*, 8> Statements;
 
   // The parameter for the "other" object, which we are move from.
   ParmVarDecl *Other = MoveAssignOperator->getParamDecl(0);
@@ -8395,8 +8563,8 @@ void Sema::DefineImplicitMoveAssignment(SourceLocation CurrentLocation,
           }
         
           CollectableMemCpyRef = BuildDeclRefExpr(CollectableMemCpy, 
-                                                  CollectableMemCpy->getType(),
-                                                  VK_LValue, Loc, 0).take();
+                                                  Context.BuiltinFnTy,
+                                                  VK_RValue, Loc, 0).take();
           assert(CollectableMemCpyRef && "Builtin reference cannot fail");
         }
       }
@@ -8415,12 +8583,12 @@ void Sema::DefineImplicitMoveAssignment(SourceLocation CurrentLocation,
         }
 
         BuiltinMemCpyRef = BuildDeclRefExpr(BuiltinMemCpy, 
-                                            BuiltinMemCpy->getType(),
-                                            VK_LValue, Loc, 0).take();
+                                            Context.BuiltinFnTy,
+                                            VK_RValue, Loc, 0).take();
         assert(BuiltinMemCpyRef && "Builtin reference cannot fail");
       }
           
-      ASTOwningVector<Expr*> CallArgs(*this);
+      SmallVector<Expr*, 8> CallArgs;
       CallArgs.push_back(To.takeAs<Expr>());
       CallArgs.push_back(From.takeAs<Expr>());
       CallArgs.push_back(IntegerLiteral::Create(Context, Size, SizeType, Loc));
@@ -8428,12 +8596,12 @@ void Sema::DefineImplicitMoveAssignment(SourceLocation CurrentLocation,
       if (NeedsCollectableMemCpy)
         Call = ActOnCallExpr(/*Scope=*/0,
                              CollectableMemCpyRef,
-                             Loc, move_arg(CallArgs), 
+                             Loc, CallArgs,
                              Loc);
       else
         Call = ActOnCallExpr(/*Scope=*/0,
                              BuiltinMemCpyRef,
-                             Loc, move_arg(CallArgs), 
+                             Loc, CallArgs,
                              Loc);
           
       assert(!Call.isInvalid() && "Call to __builtin_memcpy cannot fail!");
@@ -8483,7 +8651,7 @@ void Sema::DefineImplicitMoveAssignment(SourceLocation CurrentLocation,
   StmtResult Body;
   {
     CompoundScopeRAII CompoundScope(*this);
-    Body = ActOnCompoundStmt(Loc, Loc, move_arg(Statements),
+    Body = ActOnCompoundStmt(Loc, Loc, Statements,
                              /*isStmtExpr=*/false);
     assert(!Body.isInvalid() && "Compound statement creation cannot fail");
   }
@@ -8691,7 +8859,7 @@ void Sema::DefineImplicitCopyConstructor(SourceLocation CurrentLocation,
   CXXRecordDecl *ClassDecl = CopyConstructor->getParent();
   assert(ClassDecl && "DefineImplicitCopyConstructor - invalid constructor");
 
-  ImplicitlyDefinedFunctionScope Scope(*this, CopyConstructor);
+  SynthesizedFunctionScope Scope(*this, CopyConstructor);
   DiagnosticErrorTrap Trap(Diags);
 
   if (SetCtorInitializers(CopyConstructor, 0, 0, /*AnyErrors=*/false) ||
@@ -8703,7 +8871,7 @@ void Sema::DefineImplicitCopyConstructor(SourceLocation CurrentLocation,
     Sema::CompoundScopeRAII CompoundScope(*this);
     CopyConstructor->setBody(ActOnCompoundStmt(CopyConstructor->getLocation(),
                                                CopyConstructor->getLocation(),
-                                               MultiStmtArg(*this, 0, 0),
+                                               MultiStmtArg(),
                                                /*isStmtExpr=*/false)
                                                               .takeAs<Stmt>());
     CopyConstructor->setImplicitlyDefined(true);
@@ -8874,7 +9042,7 @@ void Sema::DefineImplicitMoveConstructor(SourceLocation CurrentLocation,
   CXXRecordDecl *ClassDecl = MoveConstructor->getParent();
   assert(ClassDecl && "DefineImplicitMoveConstructor - invalid constructor");
 
-  ImplicitlyDefinedFunctionScope Scope(*this, MoveConstructor);
+  SynthesizedFunctionScope Scope(*this, MoveConstructor);
   DiagnosticErrorTrap Trap(Diags);
 
   if (SetCtorInitializers(MoveConstructor, 0, 0, /*AnyErrors=*/false) ||
@@ -8886,7 +9054,7 @@ void Sema::DefineImplicitMoveConstructor(SourceLocation CurrentLocation,
     Sema::CompoundScopeRAII CompoundScope(*this);
     MoveConstructor->setBody(ActOnCompoundStmt(MoveConstructor->getLocation(),
                                                MoveConstructor->getLocation(),
-                                               MultiStmtArg(*this, 0, 0),
+                                               MultiStmtArg(),
                                                /*isStmtExpr=*/false)
                                                               .takeAs<Stmt>());
     MoveConstructor->setImplicitlyDefined(true);
@@ -8926,7 +9094,7 @@ void Sema::DefineImplicitLambdaToFunctionPointerConversion(
   
   Conv->setUsed();
   
-  ImplicitlyDefinedFunctionScope Scope(*this, Conv);
+  SynthesizedFunctionScope Scope(*this, Conv);
   DiagnosticErrorTrap Trap(Diags);
   
   // Return the address of the __invoke function.
@@ -8959,7 +9127,7 @@ void Sema::DefineImplicitLambdaToBlockPointerConversion(
 {
   Conv->setUsed();
   
-  ImplicitlyDefinedFunctionScope Scope(*this, Conv);
+  SynthesizedFunctionScope Scope(*this, Conv);
   DiagnosticErrorTrap Trap(Diags);
   
   // Copy-initialize the lambda object as needed to capture it.
@@ -9014,12 +9182,12 @@ static bool hasOneRealArgument(MultiExprArg Args) {
     return false;
     
   default:
-    if (!Args.get()[1]->isDefaultArgument())
+    if (!Args[1]->isDefaultArgument())
       return false;
     
     // fall through
   case 1:
-    return !Args.get()[0]->isDefaultArgument();
+    return !Args[0]->isDefaultArgument();
   }
   
   return false;
@@ -9047,12 +9215,12 @@ Sema::BuildCXXConstructExpr(SourceLocation ConstructLoc, QualType DeclInitType,
   //       directly into the target of the omitted copy/move
   if (ConstructKind == CXXConstructExpr::CK_Complete &&
       Constructor->isCopyOrMoveConstructor() && hasOneRealArgument(ExprArgs)) {
-    Expr *SubExpr = ((Expr **)ExprArgs.get())[0];
+    Expr *SubExpr = ExprArgs[0];
     Elidable = SubExpr->isTemporaryObject(Context, Constructor->getParent());
   }
 
   return BuildCXXConstructExpr(ConstructLoc, DeclInitType, Constructor,
-                               Elidable, move(ExprArgs), HadMultipleCandidates,
+                               Elidable, ExprArgs, HadMultipleCandidates,
                                RequiresZeroInit, ConstructKind, ParenRange);
 }
 
@@ -9066,12 +9234,9 @@ Sema::BuildCXXConstructExpr(SourceLocation ConstructLoc, QualType DeclInitType,
                             bool RequiresZeroInit,
                             unsigned ConstructKind,
                             SourceRange ParenRange) {
-  unsigned NumExprs = ExprArgs.size();
-  Expr **Exprs = (Expr **)ExprArgs.release();
-
   MarkFunctionReferenced(ConstructLoc, Constructor);
   return Owned(CXXConstructExpr::Create(Context, DeclInitType, ConstructLoc,
-                                        Constructor, Elidable, Exprs, NumExprs,
+                                        Constructor, Elidable, ExprArgs,
                                         HadMultipleCandidates, /*FIXME*/false,
                                         RequiresZeroInit,
               static_cast<CXXConstructExpr::ConstructionKind>(ConstructKind),
@@ -9085,7 +9250,7 @@ bool Sema::InitializeVarWithConstructor(VarDecl *VD,
   // FIXME: Provide the correct paren SourceRange when available.
   ExprResult TempResult =
     BuildCXXConstructExpr(VD->getLocation(), VD->getType(), Constructor,
-                          move(Exprs), HadMultipleCandidates, false,
+                          Exprs, HadMultipleCandidates, false,
                           CXXConstructExpr::CK_Complete, SourceRange());
   if (TempResult.isInvalid())
     return true;
@@ -9135,11 +9300,11 @@ bool
 Sema::CompleteConstructorCall(CXXConstructorDecl *Constructor,
                               MultiExprArg ArgsPtr,
                               SourceLocation Loc,
-                              ASTOwningVector<Expr*> &ConvertedArgs,
+                              SmallVectorImpl<Expr*> &ConvertedArgs,
                               bool AllowExplicit) {
   // FIXME: This duplicates a lot of code from Sema::ConvertArgumentsForCall.
   unsigned NumArgs = ArgsPtr.size();
-  Expr **Args = (Expr **)ArgsPtr.get();
+  Expr **Args = ArgsPtr.data();
 
   const FunctionProtoType *Proto 
     = Constructor->getType()->getAs<FunctionProtoType>();
@@ -9268,7 +9433,7 @@ CheckOperatorNewDeclaration(Sema &SemaRef, const FunctionDecl *FnDecl) {
 }
 
 static bool
-CheckOperatorDeleteDeclaration(Sema &SemaRef, const FunctionDecl *FnDecl) {
+CheckOperatorDeleteDeclaration(Sema &SemaRef, FunctionDecl *FnDecl) {
   // C++ [basic.stc.dynamic.deallocation]p1:
   //   A program is ill-formed if deallocation functions are declared in a
   //   namespace scope other than global scope or declared static in global 
@@ -9825,7 +9990,7 @@ Decl *Sema::BuildStaticAssertDeclaration(SourceLocation StaticAssertLoc,
 /// \brief Perform semantic analysis of the given friend type declaration.
 ///
 /// \returns A friend declaration that.
-FriendDecl *Sema::CheckFriendTypeDecl(SourceLocation Loc,
+FriendDecl *Sema::CheckFriendTypeDecl(SourceLocation LocStart,
                                       SourceLocation FriendLoc,
                                       TypeSourceInfo *TSInfo) {
   assert(TSInfo && "NULL TypeSourceInfo for friend type declaration");
@@ -9864,7 +10029,7 @@ FriendDecl *Sema::CheckFriendTypeDecl(SourceLocation Loc,
              diag::warn_cxx98_compat_nonclass_type_friend :
              diag::ext_nonclass_type_friend)
         << T
-        << SourceRange(FriendLoc, TypeRange.getEnd());
+        << TypeRange;
     }
   } else if (T->getAs<EnumType>()) {
     Diag(FriendLoc,
@@ -9872,18 +10037,22 @@ FriendDecl *Sema::CheckFriendTypeDecl(SourceLocation Loc,
            diag::warn_cxx98_compat_enum_friend :
            diag::ext_enum_friend)
       << T
-      << SourceRange(FriendLoc, TypeRange.getEnd());
+      << TypeRange;
   }
   
-  // C++0x [class.friend]p3:
+  // C++11 [class.friend]p3:
+  //   A friend declaration that does not declare a function shall have one
+  //   of the following forms:
+  //     friend elaborated-type-specifier ;
+  //     friend simple-type-specifier ;
+  //     friend typename-specifier ;
+  if (getLangOpts().CPlusPlus0x && LocStart != FriendLoc)
+    Diag(FriendLoc, diag::err_friend_not_first_in_declaration) << T;
+
   //   If the type specifier in a friend declaration designates a (possibly
-  //   cv-qualified) class type, that class is declared as a friend; otherwise, 
+  //   cv-qualified) class type, that class is declared as a friend; otherwise,
   //   the friend declaration is ignored.
-  
-  // FIXME: C++0x has some syntactic restrictions on friend type declarations
-  // in [class.friend]p3 that we do not implement.
-  
-  return FriendDecl::Create(Context, CurContext, Loc, TSInfo, FriendLoc);
+  return FriendDecl::Create(Context, CurContext, LocStart, TSInfo, FriendLoc);
 }
 
 /// Handle a friend tag declaration where the scope specifier was
@@ -9901,7 +10070,7 @@ Decl *Sema::ActOnTemplatedFriendTag(Scope *S, SourceLocation FriendLoc,
 
   if (TemplateParameterList *TemplateParams
         = MatchTemplateParametersToScopeSpecifier(TagLoc, NameLoc, SS,
-                                                  TempParamLists.get(),
+                                                  TempParamLists.data(),
                                                   TempParamLists.size(),
                                                   /*friend*/ true,
                                                   isExplicitSpecialization,
@@ -9916,7 +10085,7 @@ Decl *Sema::ActOnTemplatedFriendTag(Scope *S, SourceLocation FriendLoc,
                                 TemplateParams, AS_public,
                                 /*ModulePrivateLoc=*/SourceLocation(),
                                 TempParamLists.size() - 1,
-                   (TemplateParameterList**) TempParamLists.release()).take();
+                                TempParamLists.data()).take();
     } else {
       // The "template<>" header is extraneous.
       Diag(TemplateParams->getTemplateLoc(), diag::err_template_tag_noparams)
@@ -9929,7 +10098,7 @@ Decl *Sema::ActOnTemplatedFriendTag(Scope *S, SourceLocation FriendLoc,
 
   bool isAllExplicitSpecializations = true;
   for (unsigned I = TempParamLists.size(); I-- > 0; ) {
-    if (TempParamLists.get()[I]->size()) {
+    if (TempParamLists[I]->size()) {
       isAllExplicitSpecializations = false;
       break;
     }
@@ -10076,7 +10245,7 @@ Decl *Sema::ActOnFriendTypeDecl(Scope *S, const DeclSpec &DS,
   if (unsigned NumTempParamLists = TempParams.size())
     D = FriendTemplateDecl::Create(Context, CurContext, Loc,
                                    NumTempParamLists,
-                                   TempParams.release(),
+                                   TempParams.data(),
                                    TSI,
                                    DS.getFriendSpecLoc());
   else
@@ -10318,7 +10487,7 @@ Decl *Sema::ActOnFriendFunctionDecl(Scope *S, Declarator &D,
   
   bool AddToScope = true;
   NamedDecl *ND = ActOnFunctionDeclarator(DCScope, D, DC, TInfo, Previous,
-                                          move(TemplateParams), AddToScope);
+                                          TemplateParams, AddToScope);
   if (!ND) return 0;
 
   assert(ND->getDeclContext() == DC);
@@ -10435,10 +10604,10 @@ void Sema::SetDeclDefaulted(Decl *Dcl, SourceLocation DefaultLoc) {
     // If this definition appears within the record, do the checking when
     // the record is complete.
     const FunctionDecl *Primary = MD;
-    if (MD->getTemplatedKind() != FunctionDecl::TK_NonTemplate)
+    if (const FunctionDecl *Pattern = MD->getTemplateInstantiationPattern())
       // Find the uninstantiated declaration that actually had the '= default'
       // on it.
-      MD->getTemplateInstantiationPattern()->isDefined(Primary);
+      Pattern->isDefined(Primary);
 
     if (Primary == Primary->getCanonicalDecl())
       return;
@@ -10963,14 +11132,16 @@ void DelegatingCycleHelper(CXXConstructorDecl* Ctor,
   if (Ctor->isInvalidDecl())
     return;
 
-  const FunctionDecl *FNTarget = 0;
-  CXXConstructorDecl *Target;
-  
-  // We ignore the result here since if we don't have a body, Target will be
-  // null below.
-  (void)Ctor->getTargetConstructor()->hasBody(FNTarget);
-  Target
-= const_cast<CXXConstructorDecl*>(cast_or_null<CXXConstructorDecl>(FNTarget));
+  CXXConstructorDecl *Target = Ctor->getTargetConstructor();
+
+  // Target may not be determinable yet, for instance if this is a dependent
+  // call in an uninstantiated template.
+  if (Target) {
+    const FunctionDecl *FNTarget = 0;
+    (void)Target->hasBody(FNTarget);
+    Target = const_cast<CXXConstructorDecl*>(
+      cast_or_null<CXXConstructorDecl>(FNTarget));
+  }
 
   CXXConstructorDecl *Canonical = Ctor->getCanonicalDecl(),
                      // Avoid dereferencing a null pointer here.
@@ -10994,17 +11165,18 @@ void DelegatingCycleHelper(CXXConstructorDecl* Ctor,
              diag::warn_delegating_ctor_cycle)
         << Ctor;
 
-      // Don't add a note for a function delegating directo to itself.
+      // Don't add a note for a function delegating directly to itself.
       if (TCanonical != Canonical)
         S.Diag(Target->getLocation(), diag::note_it_delegates_to);
 
       CXXConstructorDecl *C = Target;
       while (C->getCanonicalDecl() != Canonical) {
+        const FunctionDecl *FNTarget = 0;
         (void)C->getTargetConstructor()->hasBody(FNTarget);
         assert(FNTarget && "Ctor cycle through bodiless function");
 
-        C
-       = const_cast<CXXConstructorDecl*>(cast<CXXConstructorDecl>(FNTarget));
+        C = const_cast<CXXConstructorDecl*>(
+          cast<CXXConstructorDecl>(FNTarget));
         S.Diag(C->getLocation(), diag::note_which_delegates_to);
       }
     }
@@ -11027,9 +11199,8 @@ void Sema::CheckDelegatingCtorCycles() {
   for (DelegatingCtorDeclsType::iterator
          I = DelegatingCtorDecls.begin(ExternalSource),
          E = DelegatingCtorDecls.end();
-       I != E; ++I) {
-   DelegatingCycleHelper(*I, Valid, Invalid, Current, *this);
-  }
+       I != E; ++I)
+    DelegatingCycleHelper(*I, Valid, Invalid, Current, *this);
 
   for (CI = Invalid.begin(), CE = Invalid.end(); CI != CE; ++CI)
     (*CI)->setInvalidDecl();
diff --git a/lib/Sema/SemaDeclObjC.cpp b/lib/Sema/SemaDeclObjC.cpp
index 9da4d69..c4e91e8 100644
--- a/lib/Sema/SemaDeclObjC.cpp
+++ b/lib/Sema/SemaDeclObjC.cpp
@@ -282,6 +282,25 @@ void Sema::AddAnyMethodToGlobalPool(Decl *D) {
     AddFactoryMethodToGlobalPool(MDecl, true);
 }
 
+/// HasExplicitOwnershipAttr - returns true when pointer to ObjC pointer
+/// has explicit ownership attribute; false otherwise.
+static bool
+HasExplicitOwnershipAttr(Sema &S, ParmVarDecl *Param) {
+  QualType T = Param->getType();
+  
+  if (const PointerType *PT = T->getAs<PointerType>()) {
+    T = PT->getPointeeType();
+  } else if (const ReferenceType *RT = T->getAs<ReferenceType>()) {
+    T = RT->getPointeeType();
+  } else {
+    return true;
+  }
+  
+  // If we have a lifetime qualifier, but it's local, we must have 
+  // inferred it. So, it is implicit.
+  return !T.getLocalQualifiers().hasObjCLifetime();
+}
+
 /// ActOnStartOfObjCMethodDef - This routine sets up parameters; invisible
 /// and user declared, in the method definition's AST.
 void Sema::ActOnStartOfObjCMethodDef(Scope *FnBodyScope, Decl *D) {
@@ -313,6 +332,12 @@ void Sema::ActOnStartOfObjCMethodDef(Scope *FnBodyScope, Decl *D) {
         RequireCompleteType(Param->getLocation(), Param->getType(),
                             diag::err_typecheck_decl_incomplete_type))
           Param->setInvalidDecl();
+    if (!Param->isInvalidDecl() &&
+        getLangOpts().ObjCAutoRefCount &&
+        !HasExplicitOwnershipAttr(*this, Param))
+      Diag(Param->getLocation(), diag::warn_arc_strong_pointer_objc_pointer) <<
+            Param->getType();
+    
     if ((*PI)->getIdentifier())
       PushOnScopeChains(*PI, FnBodyScope);
   }
@@ -345,8 +370,10 @@ void Sema::ActOnStartOfObjCMethodDef(Scope *FnBodyScope, Decl *D) {
   // Warn on deprecated methods under -Wdeprecated-implementations,
   // and prepare for warning on missing super calls.
   if (ObjCInterfaceDecl *IC = MDecl->getClassInterface()) {
-    if (ObjCMethodDecl *IMD = 
-          IC->lookupMethod(MDecl->getSelector(), MDecl->isInstanceMethod()))
+    ObjCMethodDecl *IMD = 
+      IC->lookupMethod(MDecl->getSelector(), MDecl->isInstanceMethod());
+    
+    if (IMD)
       DiagnoseObjCImplementedDeprecations(*this, 
                                           dyn_cast<NamedDecl>(IMD), 
                                           MDecl->getLocation(), 0);
@@ -356,13 +383,23 @@ void Sema::ActOnStartOfObjCMethodDef(Scope *FnBodyScope, Decl *D) {
     // Finally, in ActOnFinishFunctionBody() (SemaDecl), warn if flag is set.
     // Only do this if the current class actually has a superclass.
     if (IC->getSuperClass()) {
-      getCurFunction()->ObjCShouldCallSuperDealloc = 
-        !(Context.getLangOpts().ObjCAutoRefCount ||
-          Context.getLangOpts().getGC() == LangOptions::GCOnly) &&
-        MDecl->getMethodFamily() == OMF_dealloc;
-      getCurFunction()->ObjCShouldCallSuperFinalize =
-        Context.getLangOpts().getGC() != LangOptions::NonGC &&
-        MDecl->getMethodFamily() == OMF_finalize;
+      ObjCMethodFamily Family = MDecl->getMethodFamily();
+      if (Family == OMF_dealloc) {
+        if (!(getLangOpts().ObjCAutoRefCount ||
+              getLangOpts().getGC() == LangOptions::GCOnly))
+          getCurFunction()->ObjCShouldCallSuper = true;
+
+      } else if (Family == OMF_finalize) {
+        if (Context.getLangOpts().getGC() != LangOptions::NonGC)
+          getCurFunction()->ObjCShouldCallSuper = true;
+        
+      } else {
+        const ObjCMethodDecl *SuperMethod =
+          IC->getSuperClass()->lookupMethod(MDecl->getSelector(),
+                                            MDecl->isInstanceMethod());
+        getCurFunction()->ObjCShouldCallSuper = 
+          (SuperMethod && SuperMethod->hasAttr<ObjCRequiresSuperAttr>());
+      }
     }
   }
 }
@@ -510,7 +547,7 @@ ActOnStartClassInterface(SourceLocation AtInterfaceLoc,
 
   // Check then save referenced protocols.
   if (NumProtoRefs) {
-    IDecl->setProtocolList((ObjCProtocolDecl**)ProtoRefs, NumProtoRefs,
+    IDecl->setProtocolList((ObjCProtocolDecl*const*)ProtoRefs, NumProtoRefs,
                            ProtoLocs, Context);
     IDecl->setEndOfDefinitionLoc(EndProtoLoc);
   }
@@ -652,7 +689,7 @@ Sema::ActOnStartProtocolInterface(SourceLocation AtProtoInterfaceLoc,
 
   if (!err && NumProtoRefs ) {
     /// Check then save referenced protocols.
-    PDecl->setProtocolList((ObjCProtocolDecl**)ProtoRefs, NumProtoRefs,
+    PDecl->setProtocolList((ObjCProtocolDecl*const*)ProtoRefs, NumProtoRefs,
                            ProtoLocs, Context);
   }
 
@@ -819,11 +856,11 @@ ActOnStartCategoryInterface(SourceLocation AtInterfaceLoc,
   CurContext->addDecl(CDecl);
 
   if (NumProtoRefs) {
-    CDecl->setProtocolList((ObjCProtocolDecl**)ProtoRefs, NumProtoRefs, 
+    CDecl->setProtocolList((ObjCProtocolDecl*const*)ProtoRefs, NumProtoRefs, 
                            ProtoLocs, Context);
     // Protocols in the class extension belong to the class.
     if (CDecl->IsClassExtension())
-     IDecl->mergeClassExtensionProtocolList((ObjCProtocolDecl**)ProtoRefs, 
+     IDecl->mergeClassExtensionProtocolList((ObjCProtocolDecl*const*)ProtoRefs, 
                                             NumProtoRefs, Context); 
   }
 
@@ -1545,9 +1582,9 @@ void Sema::CheckProtocolMethodDefs(SourceLocation ImpLoc,
          E = PDecl->instmeth_end(); I != E; ++I) {
       ObjCMethodDecl *method = *I;
       if (method->getImplementationControl() != ObjCMethodDecl::Optional &&
-          !method->isSynthesized() && !InsMap.count(method->getSelector()) &&
-          (!Super ||
-           !Super->lookupInstanceMethod(method->getSelector()))) {
+          !method->isPropertyAccessor() &&
+          !InsMap.count(method->getSelector()) &&
+          (!Super || !Super->lookupInstanceMethod(method->getSelector()))) {
             // If a method is not implemented in the category implementation but
             // has been declared in its primary class, superclass,
             // or in one of their protocols, no need to issue the warning. 
@@ -1560,7 +1597,7 @@ void Sema::CheckProtocolMethodDefs(SourceLocation ImpLoc,
             if (ObjCMethodDecl *MethodInClass =
                   IDecl->lookupInstanceMethod(method->getSelector(), 
                                               true /*shallowCategoryLookup*/))
-              if (C || MethodInClass->isSynthesized())
+              if (C || MethodInClass->isPropertyAccessor())
                 continue;
             unsigned DIAG = diag::warn_unimplemented_protocol_method;
             if (Diags.getDiagnosticLevel(DIAG, ImpLoc)
@@ -1621,7 +1658,7 @@ void Sema::MatchAllMethodDeclarations(const SelectorSet &InsMap,
     if (InsMapSeen.count((*I)->getSelector()))
         continue;
     InsMapSeen.insert((*I)->getSelector());
-    if (!(*I)->isSynthesized() &&
+    if (!(*I)->isPropertyAccessor() &&
         !InsMap.count((*I)->getSelector())) {
       if (ImmediateClass)
         WarnUndefinedMethod(IMPDecl->getLocation(), *I, IncompleteImpl,
@@ -1638,7 +1675,7 @@ void Sema::MatchAllMethodDeclarations(const SelectorSet &InsMap,
         if (!WarnCategoryMethodImpl)
           WarnConflictingTypedMethods(ImpMethodDecl, MethodDecl,
                                       isa<ObjCProtocolDecl>(CDecl));
-        else if (!MethodDecl->isSynthesized())
+        else if (!MethodDecl->isPropertyAccessor())
           WarnExactTypedMethods(ImpMethodDecl, MethodDecl,
                                 isa<ObjCProtocolDecl>(CDecl));
       }
@@ -1672,14 +1709,26 @@ void Sema::MatchAllMethodDeclarations(const SelectorSet &InsMap,
   }
   
   if (ObjCInterfaceDecl *I = dyn_cast<ObjCInterfaceDecl> (CDecl)) {
-    // Also methods in class extensions need be looked at next.
-    for (const ObjCCategoryDecl *ClsExtDecl = I->getFirstClassExtension(); 
-         ClsExtDecl; ClsExtDecl = ClsExtDecl->getNextClassExtension())
-      MatchAllMethodDeclarations(InsMap, ClsMap, InsMapSeen, ClsMapSeen,
-                                 IMPDecl,
-                                 const_cast<ObjCCategoryDecl *>(ClsExtDecl), 
-                                 IncompleteImpl, false, 
-                                 WarnCategoryMethodImpl);
+    // when checking that methods in implementation match their declaration,
+    // i.e. when WarnCategoryMethodImpl is false, check declarations in class
+    // extension; as well as those in categories.
+    if (!WarnCategoryMethodImpl)
+      for (const ObjCCategoryDecl *CDeclChain = I->getCategoryList();
+           CDeclChain; CDeclChain = CDeclChain->getNextClassCategory())
+        MatchAllMethodDeclarations(InsMap, ClsMap, InsMapSeen, ClsMapSeen,
+                                   IMPDecl,
+                                   const_cast<ObjCCategoryDecl *>(CDeclChain),
+                                   IncompleteImpl, false,
+                                   WarnCategoryMethodImpl);
+    else 
+      // Also methods in class extensions need be looked at next.
+      for (const ObjCCategoryDecl *ClsExtDecl = I->getFirstClassExtension(); 
+           ClsExtDecl; ClsExtDecl = ClsExtDecl->getNextClassExtension())
+        MatchAllMethodDeclarations(InsMap, ClsMap, InsMapSeen, ClsMapSeen,
+                                   IMPDecl,
+                                   const_cast<ObjCCategoryDecl *>(ClsExtDecl), 
+                                   IncompleteImpl, false, 
+                                   WarnCategoryMethodImpl);
     
     // Check for any implementation of a methods declared in protocol.
     for (ObjCInterfaceDecl::all_protocol_iterator
@@ -2339,11 +2388,11 @@ Decl *Sema::ActOnAtEnd(Scope *S, SourceRange AtEnd,
                CExtDecl; CExtDecl = CExtDecl->getNextClassExtension()) {
             if (ObjCMethodDecl *GetterMethod =
                 CExtDecl->getInstanceMethod(Property->getGetterName()))
-              GetterMethod->setSynthesized(true);
+              GetterMethod->setPropertyAccessor(true);
             if (!Property->isReadOnly())
               if (ObjCMethodDecl *SetterMethod =
                   CExtDecl->getInstanceMethod(Property->getSetterName()))
-                SetterMethod->setSynthesized(true);
+                SetterMethod->setPropertyAccessor(true);
           }        
         }
       }
@@ -2435,26 +2484,49 @@ CvtQTToAstBitMask(ObjCDeclSpec::ObjCDeclQualifier PQTVal) {
 }
 
 static inline
+unsigned countAlignAttr(const AttrVec &A) {
+  unsigned count=0;
+  for (AttrVec::const_iterator i = A.begin(), e = A.end(); i != e; ++i)
+    if ((*i)->getKind() == attr::Aligned)
+      ++count;
+  return count;
+}
+
+static inline
 bool containsInvalidMethodImplAttribute(ObjCMethodDecl *IMD,
                                         const AttrVec &A) {
   // If method is only declared in implementation (private method),
   // No need to issue any diagnostics on method definition with attributes.
   if (!IMD)
     return false;
-
+  
   // method declared in interface has no attribute. 
-  // But implementation has attributes. This is invalid
+  // But implementation has attributes. This is invalid.
+  // Except when implementation has 'Align' attribute which is
+  // immaterial to method declared in interface.
   if (!IMD->hasAttrs())
-    return true;
+    return (A.size() > countAlignAttr(A));
 
   const AttrVec &D = IMD->getAttrs();
-  if (D.size() != A.size())
-    return true;
 
+  unsigned countAlignOnImpl = countAlignAttr(A);
+  if (!countAlignOnImpl && (A.size() != D.size()))
+    return true;
+  else if (countAlignOnImpl) {
+    unsigned countAlignOnDecl = countAlignAttr(D);
+    if (countAlignOnDecl && (A.size() != D.size()))
+      return true;
+    else if (!countAlignOnDecl && 
+             ((A.size()-countAlignOnImpl) != D.size()))
+      return true;
+  }
+  
   // attributes on method declaration and definition must match exactly.
   // Note that we have at most a couple of attributes on methods, so this
   // n*n search is good enough.
   for (AttrVec::const_iterator i = A.begin(), e = A.end(); i != e; ++i) {
+    if ((*i)->getKind() == attr::Aligned)
+      continue;
     bool match = false;
     for (AttrVec::const_iterator i1 = D.begin(), e1 = D.end(); i1 != e1; ++i1) {
       if ((*i)->getKind() == (*i1)->getKind()) {
@@ -2465,6 +2537,7 @@ bool containsInvalidMethodImplAttribute(ObjCMethodDecl *IMD,
     if (!match)
       return true;
   }
+  
   return false;
 }
 
@@ -2525,7 +2598,7 @@ public:
     // with this selector before.
     Sema::GlobalMethodPool::iterator it = S.MethodPool.find(selector);
     if (it == S.MethodPool.end()) {
-      if (!S.ExternalSource) return;
+      if (!S.getExternalSource()) return;
       S.ReadMethodPool(selector);
       
       it = S.MethodPool.find(selector);
@@ -2767,7 +2840,7 @@ Decl *Sema::ActOnMethodDeclaration(
                            ResultTInfo,
                            CurContext,
                            MethodType == tok::minus, isVariadic,
-                           /*isSynthesized=*/false,
+                           /*isPropertyAccessor=*/false,
                            /*isImplicitlyDeclared=*/false, /*isDefined=*/false,
                            MethodDeclKind == tok::objc_optional 
                              ? ObjCMethodDecl::Optional
diff --git a/lib/Sema/SemaExceptionSpec.cpp b/lib/Sema/SemaExceptionSpec.cpp
index e6266fb..e1f4888 100644
--- a/lib/Sema/SemaExceptionSpec.cpp
+++ b/lib/Sema/SemaExceptionSpec.cpp
@@ -120,6 +120,24 @@ Sema::ResolveExceptionSpec(SourceLocation Loc, const FunctionProtoType *FPT) {
   return SourceDecl->getType()->castAs<FunctionProtoType>();
 }
 
+/// Determine whether a function has an implicitly-generated exception
+/// specification.
+static bool hasImplicitExceptionSpec(FunctionDecl *Decl) {
+  if (!isa<CXXDestructorDecl>(Decl) &&
+      Decl->getDeclName().getCXXOverloadedOperator() != OO_Delete &&
+      Decl->getDeclName().getCXXOverloadedOperator() != OO_Array_Delete)
+    return false;
+
+  // If the user didn't declare the function, its exception specification must
+  // be implicit.
+  if (!Decl->getTypeSourceInfo())
+    return true;
+
+  const FunctionProtoType *Ty =
+    Decl->getTypeSourceInfo()->getType()->getAs<FunctionProtoType>();
+  return !Ty->hasExceptionSpec();
+}
+
 bool Sema::CheckEquivalentExceptionSpec(FunctionDecl *Old, FunctionDecl *New) {
   OverloadedOperatorKind OO = New->getDeclName().getCXXOverloadedOperator();
   bool IsOperatorNew = OO == OO_New || OO == OO_Array_New;
@@ -129,25 +147,35 @@ bool Sema::CheckEquivalentExceptionSpec(FunctionDecl *Old, FunctionDecl *New) {
   if (getLangOpts().MicrosoftExt)
     DiagID = diag::warn_mismatched_exception_spec; 
 
-  if (!CheckEquivalentExceptionSpec(PDiag(DiagID),
-                                    PDiag(diag::note_previous_declaration),
-                                    Old->getType()->getAs<FunctionProtoType>(),
-                                    Old->getLocation(),
-                                    New->getType()->getAs<FunctionProtoType>(),
-                                    New->getLocation(),
-                                    &MissingExceptionSpecification,
-                                    &MissingEmptyExceptionSpecification,
-                                    /*AllowNoexceptAllMatchWithNoSpec=*/true,
-                                    IsOperatorNew))
+  // Check the types as written: they must match before any exception
+  // specification adjustment is applied.
+  if (!CheckEquivalentExceptionSpec(
+        PDiag(DiagID), PDiag(diag::note_previous_declaration),
+        Old->getType()->getAs<FunctionProtoType>(), Old->getLocation(),
+        New->getType()->getAs<FunctionProtoType>(), New->getLocation(),
+        &MissingExceptionSpecification, &MissingEmptyExceptionSpecification,
+        /*AllowNoexceptAllMatchWithNoSpec=*/true, IsOperatorNew)) {
+    // C++11 [except.spec]p4 [DR1492]:
+    //   If a declaration of a function has an implicit
+    //   exception-specification, other declarations of the function shall
+    //   not specify an exception-specification.
+    if (getLangOpts().CPlusPlus0x &&
+        hasImplicitExceptionSpec(Old) != hasImplicitExceptionSpec(New)) {
+      Diag(New->getLocation(), diag::ext_implicit_exception_spec_mismatch)
+        << hasImplicitExceptionSpec(Old);
+      if (!Old->getLocation().isInvalid())
+        Diag(Old->getLocation(), diag::note_previous_declaration);
+    }
     return false;
+  }
 
   // The failure was something other than an empty exception
   // specification; return an error.
   if (!MissingExceptionSpecification && !MissingEmptyExceptionSpecification)
     return true;
 
-  const FunctionProtoType *NewProto 
-    = New->getType()->getAs<FunctionProtoType>();
+  const FunctionProtoType *NewProto =
+    New->getType()->getAs<FunctionProtoType>();
 
   // The new function declaration is only missing an empty exception
   // specification "throw()". If the throw() specification came from a
@@ -172,8 +200,8 @@ bool Sema::CheckEquivalentExceptionSpec(FunctionDecl *Old, FunctionDecl *New) {
   }
 
   if (MissingExceptionSpecification && NewProto) {
-    const FunctionProtoType *OldProto
-      = Old->getType()->getAs<FunctionProtoType>();
+    const FunctionProtoType *OldProto =
+      Old->getType()->getAs<FunctionProtoType>();
 
     FunctionProtoType::ExtProtoInfo EPI = NewProto->getExtProtoInfo();
     EPI.ExceptionSpecType = OldProto->getExceptionSpecType();
@@ -290,14 +318,17 @@ bool Sema::CheckEquivalentExceptionSpec(
   unsigned DiagID = diag::err_mismatched_exception_spec;
   if (getLangOpts().MicrosoftExt)
     DiagID = diag::warn_mismatched_exception_spec; 
-  return CheckEquivalentExceptionSpec(
-                                      PDiag(DiagID),
+  return CheckEquivalentExceptionSpec(PDiag(DiagID),
                                       PDiag(diag::note_previous_declaration),
                                       Old, OldLoc, New, NewLoc);
 }
 
 /// CheckEquivalentExceptionSpec - Check if the two types have compatible
 /// exception specifications. See C++ [except.spec]p3.
+///
+/// \return \c false if the exception specifications match, \c true if there is
+/// a problem. If \c true is returned, either a diagnostic has already been
+/// produced or \c *MissingExceptionSpecification is set to \c true.
 bool Sema::CheckEquivalentExceptionSpec(const PartialDiagnostic &DiagID,
                                         const PartialDiagnostic & NoteID,
                                         const FunctionProtoType *Old,
@@ -1029,6 +1060,7 @@ CanThrowResult Sema::canThrow(const Expr *E) {
   case Expr::PseudoObjectExprClass:
   case Expr::SubstNonTypeTemplateParmExprClass:
   case Expr::SubstNonTypeTemplateParmPackExprClass:
+  case Expr::FunctionParmPackExprClass:
   case Expr::UnaryExprOrTypeTraitExprClass:
   case Expr::UnresolvedLookupExprClass:
   case Expr::UnresolvedMemberExprClass:
diff --git a/lib/Sema/SemaExpr.cpp b/lib/Sema/SemaExpr.cpp
index 3875ba1..bf4abfc 100644
--- a/lib/Sema/SemaExpr.cpp
+++ b/lib/Sema/SemaExpr.cpp
@@ -66,6 +66,15 @@ bool Sema::CanUseDecl(NamedDecl *D) {
   return true;
 }
 
+static void DiagnoseUnusedOfDecl(Sema &S, NamedDecl *D, SourceLocation Loc) {
+  // Warn if this is used but marked unused.
+  if (D->hasAttr<UnusedAttr>()) {
+    const Decl *DC = cast<Decl>(S.getCurObjCLexicalContext());
+    if (!DC->hasAttr<UnusedAttr>())
+      S.Diag(Loc, diag::warn_used_but_marked_unused) << D->getDeclName();
+  }
+}
+
 static AvailabilityResult DiagnoseAvailabilityOfDecl(Sema &S,
                               NamedDecl *D, SourceLocation Loc,
                               const ObjCInterfaceDecl *UnknownObjCClass) {
@@ -78,6 +87,17 @@ static AvailabilityResult DiagnoseAvailabilityOfDecl(Sema &S,
       if (const EnumDecl *TheEnumDecl = dyn_cast<EnumDecl>(DC))
         Result = TheEnumDecl->getAvailability(&Message);
     }
+
+  const ObjCPropertyDecl *ObjCPDecl = 0;
+  if (Result == AR_Deprecated || Result == AR_Unavailable) {
+    if (const ObjCMethodDecl *MD = dyn_cast<ObjCMethodDecl>(D)) {
+      if (const ObjCPropertyDecl *PD = MD->findPropertyDecl()) {
+        AvailabilityResult PDeclResult = PD->getAvailability(0);
+        if (PDeclResult == Result)
+          ObjCPDecl = PD;
+      }
+    }
+  }
   
   switch (Result) {
     case AR_Available:
@@ -85,23 +105,30 @@ static AvailabilityResult DiagnoseAvailabilityOfDecl(Sema &S,
       break;
             
     case AR_Deprecated:
-      S.EmitDeprecationWarning(D, Message, Loc, UnknownObjCClass);
+      S.EmitDeprecationWarning(D, Message, Loc, UnknownObjCClass, ObjCPDecl);
       break;
             
     case AR_Unavailable:
       if (S.getCurContextAvailability() != AR_Unavailable) {
         if (Message.empty()) {
-          if (!UnknownObjCClass)
+          if (!UnknownObjCClass) {
             S.Diag(Loc, diag::err_unavailable) << D->getDeclName();
+            if (ObjCPDecl)
+              S.Diag(ObjCPDecl->getLocation(), diag::note_property_attribute)
+                << ObjCPDecl->getDeclName() << 1;
+          }
           else
             S.Diag(Loc, diag::warn_unavailable_fwdclass_message) 
               << D->getDeclName();
         }
-        else 
+        else
           S.Diag(Loc, diag::err_unavailable_message) 
             << D->getDeclName() << Message;
-          S.Diag(D->getLocation(), diag::note_unavailable_here) 
-          << isa<FunctionDecl>(D) << false;
+        S.Diag(D->getLocation(), diag::note_unavailable_here)
+                  << isa<FunctionDecl>(D) << false;
+        if (ObjCPDecl)
+          S.Diag(ObjCPDecl->getLocation(), diag::note_property_attribute)
+          << ObjCPDecl->getDeclName() << 1;
       }
       break;
     }
@@ -250,9 +277,7 @@ bool Sema::DiagnoseUseOfDecl(NamedDecl *D, SourceLocation Loc,
   }
   DiagnoseAvailabilityOfDecl(*this, D, Loc, UnknownObjCClass);
 
-  // Warn if this is used but marked unused.
-  if (D->hasAttr<UnusedAttr>())
-    Diag(Loc, diag::warn_used_but_marked_unused) << D->getDeclName();
+  DiagnoseUnusedOfDecl(*this, D, Loc);
 
   diagnoseUseOfInternalDeclInInlineFunction(*this, D, Loc);
 
@@ -502,7 +527,7 @@ ExprResult Sema::DefaultFunctionArrayLvalueConversion(Expr *E) {
   Res = DefaultLvalueConversion(Res.take());
   if (Res.isInvalid())
     return ExprError();
-  return move(Res);
+  return Res;
 }
 
 
@@ -1098,8 +1123,8 @@ Sema::ActOnGenericSelectionExpr(SourceLocation KeyLoc,
   unsigned NumAssocs = ArgTypes.size();
   assert(NumAssocs == ArgExprs.size());
 
-  ParsedType *ParsedTypes = ArgTypes.release();
-  Expr **Exprs = ArgExprs.release();
+  ParsedType *ParsedTypes = ArgTypes.data();
+  Expr **Exprs = ArgExprs.data();
 
   TypeSourceInfo **Types = new TypeSourceInfo*[NumAssocs];
   for (unsigned i = 0; i < NumAssocs; ++i) {
@@ -1185,8 +1210,9 @@ Sema::CreateGenericSelectionExpr(SourceLocation KeyLoc,
   if (IsResultDependent)
     return Owned(new (Context) GenericSelectionExpr(
                    Context, KeyLoc, ControllingExpr,
-                   Types, Exprs, NumAssocs, DefaultLoc,
-                   RParenLoc, ContainsUnexpandedParameterPack));
+                   llvm::makeArrayRef(Types, NumAssocs),
+                   llvm::makeArrayRef(Exprs, NumAssocs),
+                   DefaultLoc, RParenLoc, ContainsUnexpandedParameterPack));
 
   SmallVector<unsigned, 1> CompatIndices;
   unsigned DefaultIndex = -1U;
@@ -1240,8 +1266,9 @@ Sema::CreateGenericSelectionExpr(SourceLocation KeyLoc,
 
   return Owned(new (Context) GenericSelectionExpr(
                  Context, KeyLoc, ControllingExpr,
-                 Types, Exprs, NumAssocs, DefaultLoc,
-                 RParenLoc, ContainsUnexpandedParameterPack,
+                 llvm::makeArrayRef(Types, NumAssocs),
+                 llvm::makeArrayRef(Exprs, NumAssocs),
+                 DefaultLoc, RParenLoc, ContainsUnexpandedParameterPack,
                  ResultIndex));
 }
 
@@ -1402,6 +1429,15 @@ Sema::BuildDeclRefExpr(ValueDecl *D, QualType Ty, ExprValueKind VK,
 
   MarkDeclRefReferenced(E);
 
+  if (getLangOpts().ObjCARCWeak && isa<VarDecl>(D) &&
+      Ty.getObjCLifetime() == Qualifiers::OCL_Weak) {
+    DiagnosticsEngine::Level Level =
+      Diags.getDiagnosticLevel(diag::warn_arc_repeated_use_of_weak,
+                               E->getLocStart());
+    if (Level != DiagnosticsEngine::Ignored)
+      getCurFunction()->recordUseOfWeak(E);
+  }
+
   // Just in case we're building an illegal pointer-to-member.
   FieldDecl *FD = dyn_cast<FieldDecl>(D);
   if (FD && FD->isBitField())
@@ -1428,11 +1464,9 @@ Sema::DecomposeUnqualifiedId(const UnqualifiedId &Id,
     Buffer.setLAngleLoc(Id.TemplateId->LAngleLoc);
     Buffer.setRAngleLoc(Id.TemplateId->RAngleLoc);
 
-    ASTTemplateArgsPtr TemplateArgsPtr(*this,
-                                       Id.TemplateId->getTemplateArgs(),
+    ASTTemplateArgsPtr TemplateArgsPtr(Id.TemplateId->getTemplateArgs(),
                                        Id.TemplateId->NumArgs);
     translateTemplateArguments(TemplateArgsPtr, Buffer);
-    TemplateArgsPtr.release();
 
     TemplateName TName = Id.TemplateId->Template.get();
     SourceLocation TNameLoc = Id.TemplateId->TemplateNameLoc;
@@ -1606,7 +1640,8 @@ bool Sema::DiagnoseEmptyLookup(Scope *S, CXXScopeSpec &SS, LookupResult &R,
           Diag(R.getNameLoc(), diag::err_no_member_suggest)
             << Name << computeDeclContext(SS, false) << CorrectedQuotedStr
             << SS.getRange()
-            << FixItHint::CreateReplacement(R.getNameLoc(), CorrectedStr);
+            << FixItHint::CreateReplacement(Corrected.getCorrectionRange(),
+                                            CorrectedStr);
         if (ND)
           Diag(ND->getLocation(), diag::note_previous_decl)
             << CorrectedQuotedStr;
@@ -1797,7 +1832,7 @@ ExprResult Sema::ActOnIdExpression(Scope *S,
         // lookup fails and no expression will be built to reference it.
         if (!E.isInvalid() && !E.get())
           return ExprError();
-        return move(E);
+        return E;
       }
     }
   }
@@ -1860,9 +1895,10 @@ ExprResult Sema::ActOnIdExpression(Scope *S,
 /// this path.
 ExprResult
 Sema::BuildQualifiedDeclarationNameExpr(CXXScopeSpec &SS,
-                                        const DeclarationNameInfo &NameInfo) {
-  DeclContext *DC;
-  if (!(DC = computeDeclContext(SS, false)) || DC->isDependentContext())
+                                        const DeclarationNameInfo &NameInfo,
+                                        bool IsAddressOfOperand) {
+  DeclContext *DC = computeDeclContext(SS, false);
+  if (!DC)
     return BuildDependentDeclRefExpr(SS, /*TemplateKWLoc=*/SourceLocation(),
                                      NameInfo, /*TemplateArgs=*/0);
 
@@ -1875,13 +1911,26 @@ Sema::BuildQualifiedDeclarationNameExpr(CXXScopeSpec &SS,
   if (R.isAmbiguous())
     return ExprError();
 
+  if (R.getResultKind() == LookupResult::NotFoundInCurrentInstantiation)
+    return BuildDependentDeclRefExpr(SS, /*TemplateKWLoc=*/SourceLocation(),
+                                     NameInfo, /*TemplateArgs=*/0);
+
   if (R.empty()) {
     Diag(NameInfo.getLoc(), diag::err_no_member)
       << NameInfo.getName() << DC << SS.getRange();
     return ExprError();
   }
 
-  return BuildDeclarationNameExpr(SS, R, /*ADL*/ false);
+  // Defend against this resolving to an implicit member access. We usually
+  // won't get here if this might be a legitimate a class member (we end up in
+  // BuildMemberReferenceExpr instead), but this can be valid if we're forming
+  // a pointer-to-member or in an unevaluated context in C++11.
+  if (!R.empty() && (*R.begin())->isCXXClassMember() && !IsAddressOfOperand)
+    return BuildPossibleImplicitMemberExpr(SS,
+                                           /*TemplateKWLoc=*/SourceLocation(),
+                                           R, /*TemplateArgs=*/0);
+
+  return BuildDeclarationNameExpr(SS, R, /* ADL */ false);
 }
 
 /// LookupInObjCMethod - The parser has read a name in, and Sema has
@@ -1965,9 +2014,25 @@ Sema::LookupInObjCMethod(LookupResult &Lookup, Scope *S,
       ObjCMethodFamily MF = CurMethod->getMethodFamily();
       if (MF != OMF_init && MF != OMF_dealloc && MF != OMF_finalize)
         Diag(Loc, diag::warn_direct_ivar_access) << IV->getDeclName();
-      return Owned(new (Context)
-                   ObjCIvarRefExpr(IV, IV->getType(), Loc,
-                                   SelfExpr.take(), true, true));
+
+      ObjCIvarRefExpr *Result = new (Context) ObjCIvarRefExpr(IV, IV->getType(),
+                                                              Loc,
+                                                              SelfExpr.take(),
+                                                              true, true);
+
+      if (getLangOpts().ObjCAutoRefCount) {
+        if (IV->getType().getObjCLifetime() == Qualifiers::OCL_Weak) {
+          DiagnosticsEngine::Level Level =
+            Diags.getDiagnosticLevel(diag::warn_arc_repeated_use_of_weak, Loc);
+          if (Level != DiagnosticsEngine::Ignored)
+            getCurFunction()->recordUseOfWeak(Result);
+        }
+        if (CurContext->isClosure())
+          Diag(Loc, diag::warn_implicitly_retains_self)
+            << FixItHint::CreateInsertion(Loc, "self->");
+      }
+      
+      return Owned(Result);
     }
   } else if (CurMethod->isInstanceMethod()) {
     // We should warn if a local variable hides an ivar.
@@ -2416,6 +2481,14 @@ Sema::BuildDeclarationNameExpr(const CXXScopeSpec &SS,
     }
         
     case Decl::Function: {
+      if (unsigned BID = cast<FunctionDecl>(VD)->getBuiltinID()) {
+        if (!Context.BuiltinInfo.isPredefinedLibFunction(BID)) {
+          type = Context.BuiltinFnTy;
+          valueKind = VK_RValue;
+          break;
+        }
+      }
+
       const FunctionType *fty = type->castAs<FunctionType>();
 
       // If we're referring to a function with an __unknown_anytype
@@ -2615,19 +2688,20 @@ ExprResult Sema::ActOnNumericConstant(const Token &Tok, Scope *UDLScope) {
     return ActOnIntegerConstant(Tok.getLocation(), Val-'0');
   }
 
-  SmallString<512> IntegerBuffer;
-  // Add padding so that NumericLiteralParser can overread by one character.
-  IntegerBuffer.resize(Tok.getLength()+1);
-  const char *ThisTokBegin = &IntegerBuffer[0];
+  SmallString<128> SpellingBuffer;
+  // NumericLiteralParser wants to overread by one character.  Add padding to
+  // the buffer in case the token is copied to the buffer.  If getSpelling()
+  // returns a StringRef to the memory buffer, it should have a null char at
+  // the EOF, so it is also safe.
+  SpellingBuffer.resize(Tok.getLength() + 1);
 
   // Get the spelling of the token, which eliminates trigraphs, etc.
   bool Invalid = false;
-  unsigned ActualLength = PP.getSpelling(Tok, ThisTokBegin, &Invalid);
+  StringRef TokSpelling = PP.getSpelling(Tok, SpellingBuffer, &Invalid);
   if (Invalid)
     return ExprError();
 
-  NumericLiteralParser Literal(ThisTokBegin, ThisTokBegin+ActualLength,
-                               Tok.getLocation(), PP);
+  NumericLiteralParser Literal(TokSpelling, Tok.getLocation(), PP);
   if (Literal.hadError)
     return ExprError();
 
@@ -2693,7 +2767,7 @@ ExprResult Sema::ActOnNumericConstant(const Token &Tok, Scope *UDLScope) {
           Context.CharTy, llvm::APInt(32, Length + 1),
           ArrayType::Normal, 0);
       Expr *Lit = StringLiteral::Create(
-          Context, StringRef(ThisTokBegin, Length), StringLiteral::Ascii,
+          Context, StringRef(TokSpelling.data(), Length), StringLiteral::Ascii,
           /*Pascal*/false, StrTy, &TokLoc, 1);
       return BuildLiteralOperatorCall(R, OpNameInfo,
                                       llvm::makeArrayRef(&Lit, 1), TokLoc);
@@ -2709,7 +2783,7 @@ ExprResult Sema::ActOnNumericConstant(const Token &Tok, Scope *UDLScope) {
       bool CharIsUnsigned = Context.CharTy->isUnsignedIntegerType();
       llvm::APSInt Value(CharBits, CharIsUnsigned);
       for (unsigned I = 0, N = Literal.getUDSuffixOffset(); I != N; ++I) {
-        Value = ThisTokBegin[I];
+        Value = TokSpelling[I];
         TemplateArgument Arg(Context, Value, Context.CharTy);
         TemplateArgumentLocInfo ArgInfo;
         ExplicitArgs.addArgument(TemplateArgumentLoc(Arg, ArgInfo));
@@ -2747,11 +2821,15 @@ ExprResult Sema::ActOnNumericConstant(const Token &Tok, Scope *UDLScope) {
   } else {
     QualType Ty;
 
-    // long long is a C99 feature.
-    if (!getLangOpts().C99 && Literal.isLongLong)
-      Diag(Tok.getLocation(),
-           getLangOpts().CPlusPlus0x ?
-             diag::warn_cxx98_compat_longlong : diag::ext_longlong);
+    // 'long long' is a C99 or C++11 feature.
+    if (!getLangOpts().C99 && Literal.isLongLong) {
+      if (getLangOpts().CPlusPlus)
+        Diag(Tok.getLocation(),
+             getLangOpts().CPlusPlus0x ?
+             diag::warn_cxx98_compat_longlong : diag::ext_cxx11_longlong);
+      else
+        Diag(Tok.getLocation(), diag::ext_c99_longlong);
+    }
 
     // Get the value in the widest-possible width.
     unsigned MaxWidth = Context.getTargetInfo().getIntMaxTWidth();
@@ -3140,7 +3218,7 @@ Sema::ActOnUnaryExprOrTypeTraitExpr(SourceLocation OpLoc,
 
   Expr *ArgEx = (Expr *)TyOrEx;
   ExprResult Result = CreateUnaryExprOrTypeTraitExpr(ArgEx, OpLoc, ExprKind);
-  return move(Result);
+  return Result;
 }
 
 static QualType CheckRealImagOperand(Sema &S, ExprResult &V, SourceLocation Loc,
@@ -3167,7 +3245,7 @@ static QualType CheckRealImagOperand(Sema &S, ExprResult &V, SourceLocation Loc,
   ExprResult PR = S.CheckPlaceholderExpr(V.get());
   if (PR.isInvalid()) return QualType();
   if (PR.get() != V.get()) {
-    V = move(PR);
+    V = PR;
     return CheckRealImagOperand(S, V, Loc, IsReal);
   }
 
@@ -3442,8 +3520,7 @@ ExprResult Sema::BuildCXXDefaultArgExpr(SourceLocation CallLoc,
     Expr *ResultE = Result.takeAs<Expr>();
 
     InitializationSequence InitSeq(*this, Entity, Kind, &ResultE, 1);
-    Result = InitSeq.Perform(*this, Entity, Kind,
-                             MultiExprArg(*this, &ResultE, 1));
+    Result = InitSeq.Perform(*this, Entity, Kind, ResultE);
     if (Result.isInvalid())
       return ExprError();
 
@@ -3776,28 +3853,25 @@ ExprResult
 Sema::ActOnCallExpr(Scope *S, Expr *Fn, SourceLocation LParenLoc,
                     MultiExprArg ArgExprs, SourceLocation RParenLoc,
                     Expr *ExecConfig, bool IsExecConfig) {
-  unsigned NumArgs = ArgExprs.size();
-
   // Since this might be a postfix expression, get rid of ParenListExprs.
   ExprResult Result = MaybeConvertParenListExprToParenExpr(S, Fn);
   if (Result.isInvalid()) return ExprError();
   Fn = Result.take();
 
-  Expr **Args = ArgExprs.release();
-
   if (getLangOpts().CPlusPlus) {
     // If this is a pseudo-destructor expression, build the call immediately.
     if (isa<CXXPseudoDestructorExpr>(Fn)) {
-      if (NumArgs > 0) {
+      if (!ArgExprs.empty()) {
         // Pseudo-destructor calls should not have any arguments.
         Diag(Fn->getLocStart(), diag::err_pseudo_dtor_call_with_args)
           << FixItHint::CreateRemoval(
-                                    SourceRange(Args[0]->getLocStart(),
-                                                Args[NumArgs-1]->getLocEnd()));
+                                    SourceRange(ArgExprs[0]->getLocStart(),
+                                                ArgExprs.back()->getLocEnd()));
       }
 
-      return Owned(new (Context) CallExpr(Context, Fn, 0, 0, Context.VoidTy,
-                                          VK_RValue, RParenLoc));
+      return Owned(new (Context) CallExpr(Context, Fn, MultiExprArg(),
+                                          Context.VoidTy, VK_RValue,
+                                          RParenLoc));
     }
 
     // Determine whether this is a dependent call inside a C++ template,
@@ -3807,17 +3881,16 @@ Sema::ActOnCallExpr(Scope *S, Expr *Fn, SourceLocation LParenLoc,
     bool Dependent = false;
     if (Fn->isTypeDependent())
       Dependent = true;
-    else if (Expr::hasAnyTypeDependentArguments(
-        llvm::makeArrayRef(Args, NumArgs)))
+    else if (Expr::hasAnyTypeDependentArguments(ArgExprs))
       Dependent = true;
 
     if (Dependent) {
       if (ExecConfig) {
         return Owned(new (Context) CUDAKernelCallExpr(
-            Context, Fn, cast<CallExpr>(ExecConfig), Args, NumArgs,
+            Context, Fn, cast<CallExpr>(ExecConfig), ArgExprs,
             Context.DependentTy, VK_RValue, RParenLoc));
       } else {
-        return Owned(new (Context) CallExpr(Context, Fn, Args, NumArgs,
+        return Owned(new (Context) CallExpr(Context, Fn, ArgExprs,
                                             Context.DependentTy, VK_RValue,
                                             RParenLoc));
       }
@@ -3825,8 +3898,9 @@ Sema::ActOnCallExpr(Scope *S, Expr *Fn, SourceLocation LParenLoc,
 
     // Determine whether this is a call to an object (C++ [over.call.object]).
     if (Fn->getType()->isRecordType())
-      return Owned(BuildCallToObjectOfClassType(S, Fn, LParenLoc, Args, NumArgs,
-                                                RParenLoc));
+      return Owned(BuildCallToObjectOfClassType(S, Fn, LParenLoc,
+                                                ArgExprs.data(),
+                                                ArgExprs.size(), RParenLoc));
 
     if (Fn->getType() == Context.UnknownAnyTy) {
       ExprResult result = rebuildUnknownAnyFunction(*this, Fn);
@@ -3835,8 +3909,8 @@ Sema::ActOnCallExpr(Scope *S, Expr *Fn, SourceLocation LParenLoc,
     }
 
     if (Fn->getType() == Context.BoundMemberTy) {
-      return BuildCallToMemberFunction(S, Fn, LParenLoc, Args, NumArgs,
-                                       RParenLoc);
+      return BuildCallToMemberFunction(S, Fn, LParenLoc, ArgExprs.data(),
+                                       ArgExprs.size(), RParenLoc);
     }
   }
 
@@ -3849,11 +3923,11 @@ Sema::ActOnCallExpr(Scope *S, Expr *Fn, SourceLocation LParenLoc,
       OverloadExpr *ovl = find.Expression;
       if (isa<UnresolvedLookupExpr>(ovl)) {
         UnresolvedLookupExpr *ULE = cast<UnresolvedLookupExpr>(ovl);
-        return BuildOverloadedCallExpr(S, Fn, ULE, LParenLoc, Args, NumArgs,
-                                       RParenLoc, ExecConfig);
+        return BuildOverloadedCallExpr(S, Fn, ULE, LParenLoc, ArgExprs.data(),
+                                       ArgExprs.size(), RParenLoc, ExecConfig);
       } else {
-        return BuildCallToMemberFunction(S, Fn, LParenLoc, Args, NumArgs,
-                                         RParenLoc);
+        return BuildCallToMemberFunction(S, Fn, LParenLoc, ArgExprs.data(),
+                                         ArgExprs.size(), RParenLoc);
       }
     }
   }
@@ -3877,8 +3951,9 @@ Sema::ActOnCallExpr(Scope *S, Expr *Fn, SourceLocation LParenLoc,
   else if (isa<MemberExpr>(NakedFn))
     NDecl = cast<MemberExpr>(NakedFn)->getMemberDecl();
 
-  return BuildResolvedCallExpr(Fn, NDecl, LParenLoc, Args, NumArgs, RParenLoc,
-                               ExecConfig, IsExecConfig);
+  return BuildResolvedCallExpr(Fn, NDecl, LParenLoc, ArgExprs.data(),
+                               ArgExprs.size(), RParenLoc, ExecConfig,
+                               IsExecConfig);
 }
 
 ExprResult
@@ -3932,9 +4007,19 @@ Sema::BuildResolvedCallExpr(Expr *Fn, NamedDecl *NDecl,
                             SourceLocation RParenLoc,
                             Expr *Config, bool IsExecConfig) {
   FunctionDecl *FDecl = dyn_cast_or_null<FunctionDecl>(NDecl);
+  unsigned BuiltinID = (FDecl ? FDecl->getBuiltinID() : 0);
 
   // Promote the function operand.
-  ExprResult Result = UsualUnaryConversions(Fn);
+  // We special-case function promotion here because we only allow promoting
+  // builtin functions to function pointers in the callee of a call.
+  ExprResult Result;
+  if (BuiltinID &&
+      Fn->getType()->isSpecificBuiltinType(BuiltinType::BuiltinFn)) {
+    Result = ImpCastExprToType(Fn, Context.getPointerType(FDecl->getType()),
+                               CK_BuiltinFnToFnPtr).take();
+  } else {
+    Result = UsualUnaryConversions(Fn);
+  }
   if (Result.isInvalid())
     return ExprError();
   Fn = Result.take();
@@ -3945,19 +4030,17 @@ Sema::BuildResolvedCallExpr(Expr *Fn, NamedDecl *NDecl,
   if (Config)
     TheCall = new (Context) CUDAKernelCallExpr(Context, Fn,
                                                cast<CallExpr>(Config),
-                                               Args, NumArgs,
+                                               llvm::makeArrayRef(Args,NumArgs),
                                                Context.BoolTy,
                                                VK_RValue,
                                                RParenLoc);
   else
     TheCall = new (Context) CallExpr(Context, Fn,
-                                     Args, NumArgs,
+                                     llvm::makeArrayRef(Args, NumArgs),
                                      Context.BoolTy,
                                      VK_RValue,
                                      RParenLoc);
 
-  unsigned BuiltinID = (FDecl ? FDecl->getBuiltinID() : 0);
-
   // Bail out early if calling a builtin with custom typechecking.
   if (BuiltinID && Context.BuiltinInfo.hasCustomTypechecking(BuiltinID))
     return CheckBuiltinFunctionCall(BuiltinID, TheCall);
@@ -4143,9 +4226,8 @@ Sema::BuildCompoundLiteralExpr(SourceLocation LParenLoc, TypeSourceInfo *TInfo,
                                            SourceRange(LParenLoc, RParenLoc),
                                            /*InitList=*/true);
   InitializationSequence InitSeq(*this, Entity, Kind, &LiteralExpr, 1);
-  ExprResult Result = InitSeq.Perform(*this, Entity, Kind,
-                                       MultiExprArg(*this, &LiteralExpr, 1),
-                                            &literalType);
+  ExprResult Result = InitSeq.Perform(*this, Entity, Kind, LiteralExpr,
+                                      &literalType);
   if (Result.isInvalid())
     return ExprError();
   LiteralExpr = Result.get();
@@ -4167,28 +4249,25 @@ Sema::BuildCompoundLiteralExpr(SourceLocation LParenLoc, TypeSourceInfo *TInfo,
 ExprResult
 Sema::ActOnInitList(SourceLocation LBraceLoc, MultiExprArg InitArgList,
                     SourceLocation RBraceLoc) {
-  unsigned NumInit = InitArgList.size();
-  Expr **InitList = InitArgList.release();
-
   // Immediately handle non-overload placeholders.  Overloads can be
   // resolved contextually, but everything else here can't.
-  for (unsigned I = 0; I != NumInit; ++I) {
-    if (InitList[I]->getType()->isNonOverloadPlaceholderType()) {
-      ExprResult result = CheckPlaceholderExpr(InitList[I]);
+  for (unsigned I = 0, E = InitArgList.size(); I != E; ++I) {
+    if (InitArgList[I]->getType()->isNonOverloadPlaceholderType()) {
+      ExprResult result = CheckPlaceholderExpr(InitArgList[I]);
 
       // Ignore failures; dropping the entire initializer list because
       // of one failure would be terrible for indexing/etc.
       if (result.isInvalid()) continue;
 
-      InitList[I] = result.take();
+      InitArgList[I] = result.take();
     }
   }
 
   // Semantic analysis for initializers is done by ActOnDeclarator() and
   // CheckInitializer() - it requires knowledge of the object being intialized.
 
-  InitListExpr *E = new (Context) InitListExpr(Context, LBraceLoc, InitList,
-                                               NumInit, RBraceLoc);
+  InitListExpr *E = new (Context) InitListExpr(Context, LBraceLoc, InitArgList,
+                                               RBraceLoc);
   E->setType(Context.VoidTy); // FIXME: just a place holder for now.
   return Owned(E);
 }
@@ -4575,8 +4654,7 @@ ExprResult Sema::BuildVectorLiteral(SourceLocation LParenLoc,
   // FIXME: This means that pretty-printing the final AST will produce curly
   // braces instead of the original commas.
   InitListExpr *initE = new (Context) InitListExpr(Context, LParenLoc,
-                                                   &initExprs[0],
-                                                   initExprs.size(), RParenLoc);
+                                                   initExprs, RParenLoc);
   initE->setType(Ty);
   return BuildCompoundLiteralExpr(LParenLoc, TInfo, RParenLoc, initE);
 }
@@ -4603,10 +4681,8 @@ Sema::MaybeConvertParenListExprToParenExpr(Scope *S, Expr *OrigExpr) {
 ExprResult Sema::ActOnParenListExpr(SourceLocation L,
                                     SourceLocation R,
                                     MultiExprArg Val) {
-  unsigned nexprs = Val.size();
-  Expr **exprs = reinterpret_cast<Expr**>(Val.release());
-  assert((exprs != 0) && "ActOnParenOrParenListExpr() missing expr list");
-  Expr *expr = new (Context) ParenListExpr(Context, L, exprs, nexprs, R);
+  assert(Val.data() != 0 && "ActOnParenOrParenListExpr() missing expr list");
+  Expr *expr = new (Context) ParenListExpr(Context, L, Val, R);
   return Owned(expr);
 }
 
@@ -4884,11 +4960,11 @@ QualType Sema::CheckConditionalOperands(ExprResult &Cond, ExprResult &LHS,
 
   ExprResult LHSResult = CheckPlaceholderExpr(LHS.get());
   if (!LHSResult.isUsable()) return QualType();
-  LHS = move(LHSResult);
+  LHS = LHSResult;
 
   ExprResult RHSResult = CheckPlaceholderExpr(RHS.get());
   if (!RHSResult.isUsable()) return QualType();
-  RHS = move(RHSResult);
+  RHS = RHSResult;
 
   // C++ is sufficiently different to merit its own checker.
   if (getLangOpts().CPlusPlus)
@@ -5247,7 +5323,7 @@ static void DiagnoseConditionalPrecedence(Sema &Self,
       << BinaryOperator::getOpcodeStr(CondOpcode);
 
   SuggestParentheses(Self, OpLoc,
-    Self.PDiag(diag::note_precedence_conditional_silence)
+    Self.PDiag(diag::note_precedence_silence)
       << BinaryOperator::getOpcodeStr(CondOpcode),
     SourceRange(Condition->getLocStart(), Condition->getLocEnd()));
 
@@ -5811,8 +5887,7 @@ static void ConstructTransparentUnion(Sema &S, ASTContext &C,
   // of the transparent union.
   Expr *E = EResult.take();
   InitListExpr *Initializer = new (C) InitListExpr(C, SourceLocation(),
-                                                   &E, 1,
-                                                   SourceLocation());
+                                                   E, SourceLocation());
   Initializer->setType(UnionType);
   Initializer->setInitializedFieldInUnion(Field);
 
@@ -5910,7 +5985,7 @@ Sema::CheckSingleAssignmentConstraints(QualType LHSType, ExprResult &RHS,
           !CheckObjCARCUnavailableWeakConversion(LHSType,
                                                  RHS.get()->getType()))
         result = IncompatibleObjCWeakRef;
-      RHS = move(Res);
+      RHS = Res;
       return result;
     }
 
@@ -6708,7 +6783,7 @@ static void diagnoseFunctionPointerToVoidComparison(Sema &S, SourceLocation Loc,
 }
 
 static bool isObjCObjectLiteral(ExprResult &E) {
-  switch (E.get()->getStmtClass()) {
+  switch (E.get()->IgnoreParenImpCasts()->getStmtClass()) {
   case Stmt::ObjCArrayLiteralClass:
   case Stmt::ObjCDictionaryLiteralClass:
   case Stmt::ObjCStringLiteralClass:
@@ -6800,6 +6875,7 @@ static void diagnoseObjCLiteralComparison(Sema &S, SourceLocation Loc,
     LK_String
   } LiteralKind;
 
+  Literal = Literal->IgnoreParenImpCasts();
   switch (Literal->getStmtClass()) {
   case Stmt::ObjCStringLiteralClass:
     // "string literal"
@@ -7202,7 +7278,10 @@ QualType Sema::CheckCompareOperands(ExprResult &LHS, ExprResult &RHS,
       (LHSType->isIntegerType() && RHSType->isAnyPointerType())) {
     unsigned DiagID = 0;
     bool isError = false;
-    if ((LHSIsNull && LHSType->isIntegerType()) ||
+    if (LangOpts.DebuggerSupport) {
+      // Under a debugger, allow the comparison of pointers to integers,
+      // since users tend to want to compare addresses.
+    } else if ((LHSIsNull && LHSType->isIntegerType()) ||
         (RHSIsNull && RHSType->isIntegerType())) {
       if (IsRelational && !getLangOpts().CPlusPlus)
         DiagID = diag::ext_typecheck_ordered_comparison_of_pointer_and_zero;
@@ -7419,12 +7498,12 @@ inline QualType Sema::CheckLogicalOperands( // C99 6.5.[13,14]
   ExprResult LHSRes = PerformContextuallyConvertToBool(LHS.get());
   if (LHSRes.isInvalid())
     return InvalidOperands(Loc, LHS, RHS);
-  LHS = move(LHSRes);
+  LHS = LHSRes;
 
   ExprResult RHSRes = PerformContextuallyConvertToBool(RHS.get());
   if (RHSRes.isInvalid())
     return InvalidOperands(Loc, LHS, RHS);
-  RHS = move(RHSRes);
+  RHS = RHSRes;
 
   // C++ [expr.log.and]p2
   // C++ [expr.log.or]p2
@@ -7683,10 +7762,31 @@ QualType Sema::CheckAssignmentOperands(Expr *LHSExpr, ExprResult &RHS,
     }
 
     if (ConvTy == Compatible) {
-      if (LHSType.getObjCLifetime() == Qualifiers::OCL_Strong)
-        checkRetainCycles(LHSExpr, RHS.get());
-      else if (getLangOpts().ObjCAutoRefCount)
+      if (LHSType.getObjCLifetime() == Qualifiers::OCL_Strong) {
+        // Warn about retain cycles where a block captures the LHS, but
+        // not if the LHS is a simple variable into which the block is
+        // being stored...unless that variable can be captured by reference!
+        const Expr *InnerLHS = LHSExpr->IgnoreParenCasts();
+        const DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(InnerLHS);
+        if (!DRE || DRE->getDecl()->hasAttr<BlocksAttr>())
+          checkRetainCycles(LHSExpr, RHS.get());
+
+        // It is safe to assign a weak reference into a strong variable.
+        // Although this code can still have problems:
+        //   id x = self.weakProp;
+        //   id y = self.weakProp;
+        // we do not warn to warn spuriously when 'x' and 'y' are on separate
+        // paths through the function. This should be revisited if
+        // -Wrepeated-use-of-weak is made flow-sensitive.
+        DiagnosticsEngine::Level Level =
+          Diags.getDiagnosticLevel(diag::warn_arc_repeated_use_of_weak,
+                                   RHS.get()->getLocStart());
+        if (Level != DiagnosticsEngine::Ignored)
+          getCurFunction()->markSafeWeakUse(RHS.get());
+
+      } else if (getLangOpts().ObjCAutoRefCount) {
         checkUnsafeExprAssigns(Loc, LHSExpr, RHS.get());
+      }
     }
   } else {
     // Compound assignment "x += y"
@@ -7972,8 +8072,16 @@ static QualType CheckAddressOfOperand(Sema &S, ExprResult &OrigOp,
 
     // The method was named without a qualifier.
     } else if (!DRE->getQualifier()) {
-      S.Diag(OpLoc, diag::err_unqualified_pointer_member_function)
-        << op->getSourceRange();
+      if (MD->getParent()->getName().empty())
+        S.Diag(OpLoc, diag::err_unqualified_pointer_member_function)
+          << op->getSourceRange();
+      else {
+        SmallString<32> Str;
+        StringRef Qual = (MD->getParent()->getName() + "::").toStringRef(Str);
+        S.Diag(OpLoc, diag::err_unqualified_pointer_member_function)
+          << op->getSourceRange()
+          << FixItHint::CreateInsertion(op->getSourceRange().getBegin(), Qual);
+      }
     }
 
     return S.Context.getMemberPointerType(op->getType(),
@@ -8216,8 +8324,7 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
     InitializedEntity Entity =
         InitializedEntity::InitializeTemporary(LHSExpr->getType());
     InitializationSequence InitSeq(*this, Entity, Kind, &RHSExpr, 1);
-    ExprResult Init = InitSeq.Perform(*this, Entity, Kind,
-                                      MultiExprArg(&RHSExpr, 1));
+    ExprResult Init = InitSeq.Perform(*this, Entity, Kind, RHSExpr);
     if (Init.isInvalid())
       return Init;
     RHSExpr = Init.take();
@@ -8340,7 +8447,8 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
 
   if (CompResultTy.isNull())
     return Owned(new (Context) BinaryOperator(LHS.take(), RHS.take(), Opc,
-                                              ResultTy, VK, OK, OpLoc));
+                                              ResultTy, VK, OK, OpLoc,
+                                              FPFeatures.fp_contract));
   if (getLangOpts().CPlusPlus && LHS.get()->getObjectKind() !=
       OK_ObjCProperty) {
     VK = VK_LValue;
@@ -8348,7 +8456,8 @@ ExprResult Sema::CreateBuiltinBinOp(SourceLocation OpLoc,
   }
   return Owned(new (Context) CompoundAssignOperator(LHS.take(), RHS.take(), Opc,
                                                     ResultTy, VK, OK, CompLHSTy,
-                                                    CompResultTy, OpLoc));
+                                                    CompResultTy, OpLoc,
+                                                    FPFeatures.fp_contract));
 }
 
 /// DiagnoseBitwisePrecedence - Emit a warning when bitwise and comparison
@@ -8383,8 +8492,8 @@ static void DiagnoseBitwisePrecedence(Sema &Self, BinaryOperatorKind Opc,
   SourceRange DiagRange = isLeftComp ? SourceRange(LHSExpr->getLocStart(),
                                                    OpLoc)
                                      : SourceRange(OpLoc, RHSExpr->getLocEnd());
-  std::string OpStr = isLeftComp ? BinOp::getOpcodeStr(LHSopc)
-                                 : BinOp::getOpcodeStr(RHSopc);
+  StringRef OpStr = isLeftComp ? BinOp::getOpcodeStr(LHSopc)
+                               : BinOp::getOpcodeStr(RHSopc);
   SourceRange ParensRange = isLeftComp ?
       SourceRange(cast<BinOp>(LHSExpr)->getRHS()->getLocStart(),
                   RHSExpr->getLocEnd())
@@ -8394,7 +8503,7 @@ static void DiagnoseBitwisePrecedence(Sema &Self, BinaryOperatorKind Opc,
   Self.Diag(OpLoc, diag::warn_precedence_bitwise_rel)
     << DiagRange << BinOp::getOpcodeStr(Opc) << OpStr;
   SuggestParentheses(Self, OpLoc,
-    Self.PDiag(diag::note_precedence_bitwise_silence) << OpStr,
+    Self.PDiag(diag::note_precedence_silence) << OpStr,
     (isLeftComp ? LHSExpr : RHSExpr)->getSourceRange());
   SuggestParentheses(Self, OpLoc,
     Self.PDiag(diag::note_precedence_bitwise_first) << BinOp::getOpcodeStr(Opc),
@@ -8411,7 +8520,8 @@ EmitDiagnosticForBitwiseAndInBitwiseOr(Sema &Self, SourceLocation OpLoc,
   Self.Diag(Bop->getOperatorLoc(), diag::warn_bitwise_and_in_bitwise_or)
       << Bop->getSourceRange() << OpLoc;
   SuggestParentheses(Self, Bop->getOperatorLoc(),
-    Self.PDiag(diag::note_bitwise_and_in_bitwise_or_silence),
+    Self.PDiag(diag::note_precedence_silence)
+      << Bop->getOpcodeStr(),
     Bop->getSourceRange());
 }
 
@@ -8425,7 +8535,8 @@ EmitDiagnosticForLogicalAndInLogicalOr(Sema &Self, SourceLocation OpLoc,
   Self.Diag(Bop->getOperatorLoc(), diag::warn_logical_and_in_logical_or)
       << Bop->getSourceRange() << OpLoc;
   SuggestParentheses(Self, Bop->getOperatorLoc(),
-    Self.PDiag(diag::note_logical_and_in_logical_or_silence),
+    Self.PDiag(diag::note_precedence_silence)
+      << Bop->getOpcodeStr(),
     Bop->getSourceRange());
 }
 
@@ -8489,6 +8600,20 @@ static void DiagnoseBitwiseAndInBitwiseOr(Sema &S, SourceLocation OpLoc,
   }
 }
 
+static void DiagnoseAdditionInShift(Sema &S, SourceLocation OpLoc,
+                                    Expr *SubExpr, StringRef Shift) {
+  if (BinaryOperator *Bop = dyn_cast<BinaryOperator>(SubExpr)) {
+    if (Bop->getOpcode() == BO_Add || Bop->getOpcode() == BO_Sub) {
+      StringRef Op = Bop->getOpcodeStr();
+      S.Diag(Bop->getOperatorLoc(), diag::warn_addition_in_bitshift)
+          << Bop->getSourceRange() << OpLoc << Shift << Op;
+      SuggestParentheses(S, Bop->getOperatorLoc(),
+          S.PDiag(diag::note_precedence_silence) << Op,
+          Bop->getSourceRange());
+    }
+  }
+}
+
 /// DiagnoseBinOpPrecedence - Emit warnings for expressions with tricky
 /// precedence.
 static void DiagnoseBinOpPrecedence(Sema &Self, BinaryOperatorKind Opc,
@@ -8510,6 +8635,13 @@ static void DiagnoseBinOpPrecedence(Sema &Self, BinaryOperatorKind Opc,
     DiagnoseLogicalAndInLogicalOrLHS(Self, OpLoc, LHSExpr, RHSExpr);
     DiagnoseLogicalAndInLogicalOrRHS(Self, OpLoc, LHSExpr, RHSExpr);
   }
+
+  if ((Opc == BO_Shl && LHSExpr->getType()->isIntegralType(Self.getASTContext()))
+      || Opc == BO_Shr) {
+    StringRef Shift = BinaryOperator::getOpcodeStr(Opc);
+    DiagnoseAdditionInShift(Self, OpLoc, LHSExpr, Shift);
+    DiagnoseAdditionInShift(Self, OpLoc, RHSExpr, Shift);
+  }
 }
 
 // Binary Operators.  'Tok' is the token for the operator.
@@ -8647,6 +8779,7 @@ ExprResult Sema::CreateBuiltinUnaryOp(SourceLocation OpLoc,
     break;
   case UO_Deref: {
     Input = DefaultFunctionArrayLvalueConversion(Input.take());
+    if (Input.isInvalid()) return ExprError();
     resultType = CheckIndirectionOperand(*this, Input.get(), VK, OpLoc);
     break;
   }
@@ -9147,8 +9280,7 @@ ExprResult Sema::BuildBuiltinOffsetOf(SourceLocation BuiltinLoc,
   }
   
   return Owned(OffsetOfExpr::Create(Context, Context.getSizeType(), BuiltinLoc, 
-                                    TInfo, Comps.data(), Comps.size(),
-                                    Exprs.data(), Exprs.size(), RParenLoc));  
+                                    TInfo, Comps, Exprs, RParenLoc));
 }
 
 ExprResult Sema::ActOnBuiltinOffsetOf(Scope *S,
@@ -9526,6 +9658,16 @@ ExprResult Sema::BuildVAArgExpr(SourceLocation BuiltinLoc,
     if (Result.isInvalid())
       return ExprError();
     E = Result.take();
+  } else if (VaListType->isRecordType() && getLangOpts().CPlusPlus) {
+    // If va_list is a record type and we are compiling in C++ mode,
+    // check the argument using reference binding.
+    InitializedEntity Entity
+      = InitializedEntity::InitializeParameter(Context,
+          Context.getLValueReferenceType(VaListType), false);
+    ExprResult Init = PerformCopyInitialization(Entity, SourceLocation(), E);
+    if (Init.isInvalid())
+      return ExprError();
+    E = Init.takeAs<Expr>();
   } else {
     // Otherwise, the va_list argument must be an l-value because
     // it is modified by va_arg.
@@ -10094,6 +10236,14 @@ Sema::PushExpressionEvaluationContext(ExpressionEvaluationContext NewContext,
     std::swap(MaybeODRUseExprs, ExprEvalContexts.back().SavedMaybeODRUseExprs);
 }
 
+void
+Sema::PushExpressionEvaluationContext(ExpressionEvaluationContext NewContext,
+                                      ReuseLambdaContextDecl_t,
+                                      bool IsDecltype) {
+  Decl *LambdaContextDecl = ExprEvalContexts.back().LambdaContextDecl;
+  PushExpressionEvaluationContext(NewContext, LambdaContextDecl, IsDecltype);
+}
+
 void Sema::PopExpressionEvaluationContext() {
   ExpressionEvaluationContextRecord& Rec = ExprEvalContexts.back();
 
@@ -10191,15 +10341,44 @@ void Sema::MarkFunctionReferenced(SourceLocation Loc, FunctionDecl *Func) {
 
   Func->setReferenced();
 
-  // Don't mark this function as used multiple times, unless it's a constexpr
-  // function which we need to instantiate.
-  if (Func->isUsed(false) &&
-      !(Func->isConstexpr() && !Func->getBody() &&
-        Func->isImplicitlyInstantiable()))
-    return;
-
-  if (!IsPotentiallyEvaluatedContext(*this))
-    return;
+  // C++11 [basic.def.odr]p3:
+  //   A function whose name appears as a potentially-evaluated expression is
+  //   odr-used if it is the unique lookup result or the selected member of a
+  //   set of overloaded functions [...].
+  //
+  // We (incorrectly) mark overload resolution as an unevaluated context, so we
+  // can just check that here. Skip the rest of this function if we've already
+  // marked the function as used.
+  if (Func->isUsed(false) || !IsPotentiallyEvaluatedContext(*this)) {
+    // C++11 [temp.inst]p3:
+    //   Unless a function template specialization has been explicitly
+    //   instantiated or explicitly specialized, the function template
+    //   specialization is implicitly instantiated when the specialization is
+    //   referenced in a context that requires a function definition to exist.
+    //
+    // We consider constexpr function templates to be referenced in a context
+    // that requires a definition to exist whenever they are referenced.
+    //
+    // FIXME: This instantiates constexpr functions too frequently. If this is
+    // really an unevaluated context (and we're not just in the definition of a
+    // function template or overload resolution or other cases which we
+    // incorrectly consider to be unevaluated contexts), and we're not in a
+    // subexpression which we actually need to evaluate (for instance, a
+    // template argument, array bound or an expression in a braced-init-list),
+    // we are not permitted to instantiate this constexpr function definition.
+    //
+    // FIXME: This also implicitly defines special members too frequently. They
+    // are only supposed to be implicitly defined if they are odr-used, but they
+    // are not odr-used from constant expressions in unevaluated contexts.
+    // However, they cannot be referenced if they are deleted, and they are
+    // deleted whenever the implicit definition of the special member would
+    // fail.
+    if (!Func->isConstexpr() || Func->getBody())
+      return;
+    CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Func);
+    if (!Func->isImplicitlyInstantiable() && (!MD || MD->isUserProvided()))
+      return;
+  }
 
   // Note that this declaration has been used.
   if (CXXConstructorDecl *Constructor = dyn_cast<CXXConstructorDecl>(Func)) {
@@ -10469,8 +10648,7 @@ static ExprResult captureInLambda(Sema &S, LambdaScopeInfo *LSI,
   InitializationSequence Init(S, Entities.back(), InitKind, &Ref, 1);
   ExprResult Result(true);
   if (!Init.Diagnose(S, Entities.back(), InitKind, &Ref, 1))
-    Result = Init.Perform(S, Entities.back(), InitKind, 
-                          MultiExprArg(S, &Ref, 1));
+    Result = Init.Perform(S, Entities.back(), InitKind, Ref);
 
   // If this initialization requires any cleanups (e.g., due to a
   // default argument to a copy constructor), note that for the
@@ -10886,20 +11064,21 @@ static void DoMarkVarDeclReferenced(Sema &SemaRef, SourceLocation Loc,
     }
   }
 
-  // Per C++11 [basic.def.odr], a variable is odr-used "unless it is 
-  // an object that satisfies the requirements for appearing in a
-  // constant expression (5.19) and the lvalue-to-rvalue conversion (4.1)
+  // Per C++11 [basic.def.odr], a variable is odr-used "unless it satisfies
+  // the requirements for appearing in a constant expression (5.19) and, if
+  // it is an object, the lvalue-to-rvalue conversion (4.1)
   // is immediately applied."  We check the first part here, and
   // Sema::UpdateMarkingForLValueToRValue deals with the second part.
   // Note that we use the C++11 definition everywhere because nothing in
-  // C++03 depends on whether we get the C++03 version correct. This does not
-  // apply to references, since they are not objects.
+  // C++03 depends on whether we get the C++03 version correct. The second
+  // part does not apply to references, since they are not objects.
   const VarDecl *DefVD;
-  if (E && !isa<ParmVarDecl>(Var) && !Var->getType()->isReferenceType() &&
+  if (E && !isa<ParmVarDecl>(Var) &&
       Var->isUsableInConstantExpressions(SemaRef.Context) &&
-      Var->getAnyInitializer(DefVD) && DefVD->checkInitIsICE())
-    SemaRef.MaybeODRUseExprs.insert(E);
-  else
+      Var->getAnyInitializer(DefVD) && DefVD->checkInitIsICE()) {
+    if (!Var->getType()->isReferenceType())
+      SemaRef.MaybeODRUseExprs.insert(E);
+  } else
     MarkVarDeclODRUsed(SemaRef, Var, Loc);
 }
 
@@ -11205,7 +11384,9 @@ void Sema::DiagnoseAssignmentAsCondition(Expr *E) {
 
     IsOrAssign = Op->getOperator() == OO_PipeEqual;
     Loc = Op->getOperatorLoc();
-  } else {
+  } else if (PseudoObjectExpr *POE = dyn_cast<PseudoObjectExpr>(E))
+    return DiagnoseAssignmentAsCondition(POE->getSyntacticForm());
+  else {
     // Not an assignment.
     return;
   }
@@ -11759,6 +11940,10 @@ ExprResult Sema::CheckPlaceholderExpr(Expr *E) {
   case BuiltinType::PseudoObject:
     return checkPseudoObjectRValue(E);
 
+  case BuiltinType::BuiltinFn:
+    Diag(E->getLocStart(), diag::err_builtin_fn_use);
+    return ExprError();
+
   // Everything else should be impossible.
 #define BUILTIN_TYPE(Id, SingletonId) \
   case BuiltinType::Id:
@@ -11783,6 +11968,18 @@ ExprResult
 Sema::ActOnObjCBoolLiteral(SourceLocation OpLoc, tok::TokenKind Kind) {
   assert((Kind == tok::kw___objc_yes || Kind == tok::kw___objc_no) &&
          "Unknown Objective-C Boolean value!");
+  QualType BoolT = Context.ObjCBuiltinBoolTy;
+  if (!Context.getBOOLDecl()) {
+    LookupResult Result(*this, &Context.Idents.get("BOOL"), OpLoc,
+                        Sema::LookupOrdinaryName);
+    if (LookupName(Result, getCurScope()) && Result.isSingleResult()) {
+      NamedDecl *ND = Result.getFoundDecl();
+      if (TypedefDecl *TD = dyn_cast<TypedefDecl>(ND)) 
+        Context.setBOOLDecl(TD);
+    }
+  }
+  if (Context.getBOOLDecl())
+    BoolT = Context.getBOOLType();
   return Owned(new (Context) ObjCBoolLiteralExpr(Kind == tok::kw___objc_yes,
-                                        Context.ObjCBuiltinBoolTy, OpLoc));
+                                        BoolT, OpLoc));
 }
diff --git a/lib/Sema/SemaExprCXX.cpp b/lib/Sema/SemaExprCXX.cpp
index 2740259..0919bc5 100644
--- a/lib/Sema/SemaExprCXX.cpp
+++ b/lib/Sema/SemaExprCXX.cpp
@@ -410,33 +410,13 @@ Sema::ActOnCXXTypeid(SourceLocation OpLoc, SourceLocation LParenLoc,
   return BuildCXXTypeId(TypeInfoType, OpLoc, (Expr*)TyOrExpr, RParenLoc);
 }
 
-/// Retrieve the UuidAttr associated with QT.
-static UuidAttr *GetUuidAttrOfType(QualType QT) {
-  // Optionally remove one level of pointer, reference or array indirection.
-  const Type *Ty = QT.getTypePtr();;
-  if (QT->isPointerType() || QT->isReferenceType())
-    Ty = QT->getPointeeType().getTypePtr();
-  else if (QT->isArrayType())
-    Ty = cast<ArrayType>(QT)->getElementType().getTypePtr();
-
-  // Loop all record redeclaration looking for an uuid attribute.
-  CXXRecordDecl *RD = Ty->getAsCXXRecordDecl();
-  for (CXXRecordDecl::redecl_iterator I = RD->redecls_begin(),
-       E = RD->redecls_end(); I != E; ++I) {
-    if (UuidAttr *Uuid = I->getAttr<UuidAttr>())
-      return Uuid;
-  }
-
-  return 0;
-}
-
 /// \brief Build a Microsoft __uuidof expression with a type operand.
 ExprResult Sema::BuildCXXUuidof(QualType TypeInfoType,
                                 SourceLocation TypeidLoc,
                                 TypeSourceInfo *Operand,
                                 SourceLocation RParenLoc) {
   if (!Operand->getType()->isDependentType()) {
-    if (!GetUuidAttrOfType(Operand->getType()))
+    if (!CXXUuidofExpr::GetUuidAttrOfType(Operand->getType()))
       return ExprError(Diag(TypeidLoc, diag::err_uuidof_without_guid));
   }
 
@@ -452,7 +432,7 @@ ExprResult Sema::BuildCXXUuidof(QualType TypeInfoType,
                                 Expr *E,
                                 SourceLocation RParenLoc) {
   if (!E->getType()->isDependentType()) {
-    if (!GetUuidAttrOfType(E->getType()) &&
+    if (!CXXUuidofExpr::GetUuidAttrOfType(E->getType()) &&
         !E->isNullPointerConstant(Context, Expr::NPC_ValueDependentIsNull))
       return ExprError(Diag(TypeidLoc, diag::err_uuidof_without_guid));
   }
@@ -808,21 +788,18 @@ Sema::BuildCXXTypeConstructExpr(TypeSourceInfo *TInfo,
                                 MultiExprArg exprs,
                                 SourceLocation RParenLoc) {
   QualType Ty = TInfo->getType();
-  unsigned NumExprs = exprs.size();
-  Expr **Exprs = (Expr**)exprs.get();
   SourceLocation TyBeginLoc = TInfo->getTypeLoc().getBeginLoc();
 
-  if (Ty->isDependentType() ||
-      CallExpr::hasAnyTypeDependentArguments(
-        llvm::makeArrayRef(Exprs, NumExprs))) {
-    exprs.release();
-
+  if (Ty->isDependentType() || CallExpr::hasAnyTypeDependentArguments(exprs)) {
     return Owned(CXXUnresolvedConstructExpr::Create(Context, TInfo,
                                                     LParenLoc,
-                                                    Exprs, NumExprs,
+                                                    exprs,
                                                     RParenLoc));
   }
 
+  unsigned NumExprs = exprs.size();
+  Expr **Exprs = exprs.data();
+
   bool ListInitialization = LParenLoc.isInvalid();
   assert((!ListInitialization || (NumExprs == 1 && isa<InitListExpr>(Exprs[0])))
          && "List initialization must have initializer list as expression.");
@@ -835,7 +812,6 @@ Sema::BuildCXXTypeConstructExpr(TypeSourceInfo *TInfo,
   // corresponding cast expression.
   if (NumExprs == 1 && !ListInitialization) {
     Expr *Arg = Exprs[0];
-    exprs.release();
     return BuildCXXFunctionalCastExpr(TInfo, LParenLoc, Arg, RParenLoc);
   }
 
@@ -865,7 +841,7 @@ Sema::BuildCXXTypeConstructExpr(TypeSourceInfo *TInfo,
                : InitializationKind::CreateValue(TyBeginLoc,
                                                  LParenLoc, RParenLoc);
   InitializationSequence InitSeq(*this, Entity, Kind, Exprs, NumExprs);
-  ExprResult Result = InitSeq.Perform(*this, Entity, Kind, move(exprs));
+  ExprResult Result = InitSeq.Perform(*this, Entity, Kind, exprs);
 
   if (!Result.isInvalid() && ListInitialization &&
       isa<InitListExpr>(Result.get())) {
@@ -881,7 +857,7 @@ Sema::BuildCXXTypeConstructExpr(TypeSourceInfo *TInfo,
   }
 
   // FIXME: Improve AST representation?
-  return move(Result);
+  return Result;
 }
 
 /// doesUsualArrayDeleteWantSize - Answers whether the usual
@@ -1011,9 +987,9 @@ Sema::ActOnCXXNew(SourceLocation StartLoc, bool UseGlobal,
   if (ParenListExpr *List = dyn_cast_or_null<ParenListExpr>(Initializer))
     DirectInitRange = List->getSourceRange();
 
-  return BuildCXXNew(StartLoc, UseGlobal,
+  return BuildCXXNew(SourceRange(StartLoc, D.getLocEnd()), UseGlobal,
                      PlacementLParen,
-                     move(PlacementArgs),
+                     PlacementArgs,
                      PlacementRParen,
                      TypeIdParens,
                      AllocType,
@@ -1044,7 +1020,7 @@ static bool isLegalArrayNewInitializer(CXXNewExpr::InitializationStyle Style,
 }
 
 ExprResult
-Sema::BuildCXXNew(SourceLocation StartLoc, bool UseGlobal,
+Sema::BuildCXXNew(SourceRange Range, bool UseGlobal,
                   SourceLocation PlacementLParen,
                   MultiExprArg PlacementArgs,
                   SourceLocation PlacementRParen,
@@ -1056,6 +1032,7 @@ Sema::BuildCXXNew(SourceLocation StartLoc, bool UseGlobal,
                   Expr *Initializer,
                   bool TypeMayContainAuto) {
   SourceRange TypeRange = AllocTypeInfo->getTypeLoc().getSourceRange();
+  SourceLocation StartLoc = Range.getBegin();
 
   CXXNewExpr::InitializationStyle initStyle;
   if (DirectInitRange.isValid()) {
@@ -1279,21 +1256,13 @@ Sema::BuildCXXNew(SourceLocation StartLoc, bool UseGlobal,
       }
     }
 
-    // ARC: warn about ABI issues.
-    if (getLangOpts().ObjCAutoRefCount) {
-      QualType BaseAllocType = Context.getBaseElementType(AllocType);
-      if (BaseAllocType.hasStrongOrWeakObjCLifetime())
-        Diag(StartLoc, diag::warn_err_new_delete_object_array)
-          << 0 << BaseAllocType;
-    }
-
     // Note that we do *not* convert the argument in any way.  It can
     // be signed, larger than size_t, whatever.
   }
 
   FunctionDecl *OperatorNew = 0;
   FunctionDecl *OperatorDelete = 0;
-  Expr **PlaceArgs = (Expr**)PlacementArgs.get();
+  Expr **PlaceArgs = PlacementArgs.data();
   unsigned NumPlaceArgs = PlacementArgs.size();
 
   if (!AllocType->isDependentType() &&
@@ -1432,15 +1401,14 @@ Sema::BuildCXXNew(SourceLocation StartLoc, bool UseGlobal,
     }
   }
 
-  PlacementArgs.release();
-
   return Owned(new (Context) CXXNewExpr(Context, UseGlobal, OperatorNew,
                                         OperatorDelete,
                                         UsualArrayDeleteWantsSize,
-                                        PlaceArgs, NumPlaceArgs, TypeIdParens,
+                                   llvm::makeArrayRef(PlaceArgs, NumPlaceArgs),
+                                        TypeIdParens,
                                         ArraySize, initStyle, Initializer,
                                         ResultType, AllocTypeInfo,
-                                        StartLoc, DirectInitRange));
+                                        Range, DirectInitRange));
 }
 
 /// \brief Checks that a type is suitable as the allocated type
@@ -1638,7 +1606,7 @@ bool Sema::FindAllocationFunctions(SourceLocation StartLoc, SourceRange Range,
             = dyn_cast<FunctionTemplateDecl>((*D)->getUnderlyingDecl())) {
         // Perform template argument deduction to try to match the
         // expected function type.
-        TemplateDeductionInfo Info(Context, StartLoc);
+        TemplateDeductionInfo Info(StartLoc);
         if (DeduceTemplateArguments(FnTmpl, 0, ExpectedFunctionType, Fn, Info))
           continue;
       } else
@@ -2100,7 +2068,7 @@ Sema::ActOnCXXDelete(SourceLocation StartLoc, bool UseGlobal,
                             ObjectPtrConversions.front()->getConversionType(),
                                     AA_Converting);
         if (Res.isUsable()) {
-          Ex = move(Res);
+          Ex = Res;
           Type = Ex.get()->getType();
         }
       }
@@ -2211,13 +2179,6 @@ Sema::ActOnCXXDelete(SourceLocation StartLoc, bool UseGlobal,
         }
       }
 
-    } else if (getLangOpts().ObjCAutoRefCount &&
-               PointeeElem->isObjCLifetimeType() &&
-               (PointeeElem.getObjCLifetime() == Qualifiers::OCL_Strong ||
-                PointeeElem.getObjCLifetime() == Qualifiers::OCL_Weak) &&
-               ArrayForm) {
-      Diag(StartLoc, diag::warn_err_new_delete_object_array)
-        << 1 << PointeeElem;
     }
 
     if (!OperatorDelete) {
@@ -2287,7 +2248,7 @@ ExprResult Sema::CheckConditionVariable(VarDecl *ConditionVar,
       return ExprError();
   }
 
-  return move(Condition);
+  return Condition;
 }
 
 /// CheckCXXBooleanCondition - Returns true if a conversion to bool is invalid.
@@ -2354,11 +2315,9 @@ static ExprResult BuildCXXCastArgument(Sema &S,
   default: llvm_unreachable("Unhandled cast kind!");
   case CK_ConstructorConversion: {
     CXXConstructorDecl *Constructor = cast<CXXConstructorDecl>(Method);
-    ASTOwningVector<Expr*> ConstructorArgs(S);
+    SmallVector<Expr*, 8> ConstructorArgs;
 
-    if (S.CompleteConstructorCall(Constructor,
-                                  MultiExprArg(&From, 1),
-                                  CastLoc, ConstructorArgs))
+    if (S.CompleteConstructorCall(Constructor, From, CastLoc, ConstructorArgs))
       return ExprError();
 
     S.CheckConstructorAccess(CastLoc, Constructor,
@@ -2367,7 +2326,7 @@ static ExprResult BuildCXXCastArgument(Sema &S,
     
     ExprResult Result
       = S.BuildCXXConstructExpr(CastLoc, Ty, cast<CXXConstructorDecl>(Method),
-                                move_arg(ConstructorArgs), 
+                                ConstructorArgs, 
                                 HadMultipleCandidates, /*ZeroInit*/ false, 
                                 CXXConstructExpr::CK_Complete, SourceRange());
     if (Result.isInvalid())
@@ -2511,15 +2470,14 @@ Sema::PerformImplicitConversion(Expr *From, QualType ToType,
     // FIXME: When can ToType be a reference type?
     assert(!ToType->isReferenceType());
     if (SCS.Second == ICK_Derived_To_Base) {
-      ASTOwningVector<Expr*> ConstructorArgs(*this);
+      SmallVector<Expr*, 8> ConstructorArgs;
       if (CompleteConstructorCall(cast<CXXConstructorDecl>(SCS.CopyConstructor),
-                                  MultiExprArg(*this, &From, 1),
-                                  /*FIXME:ConstructLoc*/SourceLocation(),
+                                  From, /*FIXME:ConstructLoc*/SourceLocation(),
                                   ConstructorArgs))
         return ExprError();
       return BuildCXXConstructExpr(/*FIXME:ConstructLoc*/SourceLocation(),
                                    ToType, SCS.CopyConstructor,
-                                   move_arg(ConstructorArgs),
+                                   ConstructorArgs,
                                    /*HadMultipleCandidates*/ false,
                                    /*ZeroInit*/ false,
                                    CXXConstructExpr::CK_Complete,
@@ -2527,8 +2485,7 @@ Sema::PerformImplicitConversion(Expr *From, QualType ToType,
     }
     return BuildCXXConstructExpr(/*FIXME:ConstructLoc*/SourceLocation(),
                                  ToType, SCS.CopyConstructor,
-                                 MultiExprArg(*this, &From, 1),
-                                 /*HadMultipleCandidates*/ false,
+                                 From, /*HadMultipleCandidates*/ false,
                                  /*ZeroInit*/ false,
                                  CXXConstructExpr::CK_Complete,
                                  SourceRange());
@@ -2602,8 +2559,16 @@ Sema::PerformImplicitConversion(Expr *From, QualType ToType,
 
   case ICK_Integral_Promotion:
   case ICK_Integral_Conversion:
-    From = ImpCastExprToType(From, ToType, CK_IntegralCast, 
-                             VK_RValue, /*BasePath=*/0, CCK).take();
+    if (ToType->isBooleanType()) {
+      assert(FromType->castAs<EnumType>()->getDecl()->isFixed() &&
+             SCS.Second == ICK_Integral_Promotion &&
+             "only enums with fixed underlying type can promote to bool");
+      From = ImpCastExprToType(From, ToType, CK_IntegralToBoolean,
+                               VK_RValue, /*BasePath=*/0, CCK).take();
+    } else {
+      From = ImpCastExprToType(From, ToType, CK_IntegralCast,
+                               VK_RValue, /*BasePath=*/0, CCK).take();
+    }
     break;
 
   case ICK_Floating_Promotion:
@@ -2943,6 +2908,7 @@ static bool CheckUnaryTypeTraitTypeCompleteness(Sema &S,
   case UTT_IsEmpty:
   case UTT_IsPolymorphic:
   case UTT_IsAbstract:
+  case UTT_IsInterfaceClass:
     // Fall-through
 
   // These traits require a complete type.
@@ -3007,7 +2973,7 @@ static bool EvaluateUnaryTypeTrait(Sema &Self, UnaryTypeTrait UTT,
   case UTT_IsUnion:
     return T->isUnionType();
   case UTT_IsClass:
-    return T->isClassType() || T->isStructureType();
+    return T->isClassType() || T->isStructureType() || T->isInterfaceType();
   case UTT_IsFunction:
     return T->isFunctionType();
 
@@ -3073,6 +3039,10 @@ static bool EvaluateUnaryTypeTrait(Sema &Self, UnaryTypeTrait UTT,
     if (const CXXRecordDecl *RD = T->getAsCXXRecordDecl())
       return RD->isAbstract();
     return false;
+  case UTT_IsInterfaceClass:
+    if (const CXXRecordDecl *RD = T->getAsCXXRecordDecl())
+      return RD->isInterface();
+    return false;
   case UTT_IsFinal:
     if (const CXXRecordDecl *RD = T->getAsCXXRecordDecl())
       return RD->hasAttr<FinalAttr>();
@@ -3417,9 +3387,7 @@ static bool evaluateTypeTrait(Sema &S, TypeTrait Kind, SourceLocation KWLoc,
     if (Init.Failed())
       return false;
     
-    ExprResult Result = Init.Perform(S, To, InitKind, 
-                                     MultiExprArg(ArgExprs.data(), 
-                                                  ArgExprs.size()));
+    ExprResult Result = Init.Perform(S, To, InitKind, ArgExprs);
     if (Result.isInvalid() || SFINAE.hasErrorOccurred())
       return false;
 
@@ -3577,7 +3545,7 @@ static bool EvaluateBinaryTypeTrait(Sema &Self, BinaryTypeTrait BTT,
     if (Init.Failed())
       return false;
 
-    ExprResult Result = Init.Perform(Self, To, Kind, MultiExprArg(&FromPtr, 1));
+    ExprResult Result = Init.Perform(Self, To, Kind, FromPtr);
     return !Result.isInvalid() && !SFINAE.hasErrorOccurred();
   }
       
@@ -3774,7 +3742,7 @@ ExprResult Sema::ActOnExpressionTrait(ExpressionTrait ET,
 
   ExprResult Result = BuildExpressionTrait(ET, KWLoc, Queried, RParen);
 
-  return move(Result);
+  return Result;
 }
 
 static bool EvaluateExpressionTrait(ExpressionTrait ET, Expr *E) {
@@ -4056,14 +4024,14 @@ static bool FindConditionalOverload(Sema &Self, ExprResult &LHS, ExprResult &RHS
                                        Best->Conversions[0], Sema::AA_Converting);
       if (LHSRes.isInvalid())
         break;
-      LHS = move(LHSRes);
+      LHS = LHSRes;
 
       ExprResult RHSRes =
         Self.PerformImplicitConversion(RHS.get(), Best->BuiltinTypes.ParamTypes[1],
                                        Best->Conversions[1], Sema::AA_Converting);
       if (RHSRes.isInvalid())
         break;
-      RHS = move(RHSRes);
+      RHS = RHSRes;
       if (Best->Function)
         Self.MarkFunctionReferenced(QuestionLoc, Best->Function);
       return false;
@@ -4104,7 +4072,7 @@ static bool ConvertForConditional(Sema &Self, ExprResult &E, QualType T) {
                                                            SourceLocation());
   Expr *Arg = E.take();
   InitializationSequence InitSeq(Self, Entity, Kind, &Arg, 1);
-  ExprResult Result = InitSeq.Perform(Self, Entity, Kind, MultiExprArg(&Arg, 1));
+  ExprResult Result = InitSeq.Perform(Self, Entity, Kind, Arg);
   if (Result.isInvalid())
     return true;
 
@@ -4129,7 +4097,7 @@ QualType Sema::CXXCheckConditionalOperands(ExprResult &Cond, ExprResult &LHS,
     ExprResult CondRes = CheckCXXBooleanCondition(Cond.take());
     if (CondRes.isInvalid())
       return QualType();
-    Cond = move(CondRes);
+    Cond = CondRes;
   }
 
   // Assume r-value.
@@ -4160,6 +4128,9 @@ QualType Sema::CXXCheckConditionalOperands(ExprResult &Cond, ExprResult &LHS,
     ExprResult &NonVoid = LVoid ? RHS : LHS;
     if (NonVoid.get()->getType()->isRecordType() &&
         NonVoid.get()->isGLValue()) {
+      if (RequireNonAbstractType(QuestionLoc, NonVoid.get()->getType(),
+                             diag::err_allocation_of_abstract_type))
+        return QualType();
       InitializedEntity Entity =
           InitializedEntity::InitializeTemporary(NonVoid.get()->getType());
       NonVoid = PerformCopyInitialization(Entity, SourceLocation(), NonVoid);
@@ -4302,7 +4273,11 @@ QualType Sema::CXXCheckConditionalOperands(ExprResult &Cond, ExprResult &LHS,
   if (Context.getCanonicalType(LTy) == Context.getCanonicalType(RTy)) {
     if (LTy->isRecordType()) {
       // The operands have class type. Make a temporary copy.
+      if (RequireNonAbstractType(QuestionLoc, LTy,
+                                 diag::err_allocation_of_abstract_type))
+        return QualType();
       InitializedEntity Entity = InitializedEntity::InitializeTemporary(LTy);
+
       ExprResult LHSCopy = PerformCopyInitialization(Entity,
                                                      SourceLocation(),
                                                      LHS);
@@ -4566,14 +4541,14 @@ QualType Sema::FindCompositePointerType(SourceLocation Loc,
 
     // Convert E1 to Composite1
     ExprResult E1Result
-      = E1ToC1.Perform(*this, Entity1, Kind, MultiExprArg(*this,&E1,1));
+      = E1ToC1.Perform(*this, Entity1, Kind, E1);
     if (E1Result.isInvalid())
       return QualType();
     E1 = E1Result.takeAs<Expr>();
 
     // Convert E2 to Composite1
     ExprResult E2Result
-      = E2ToC1.Perform(*this, Entity1, Kind, MultiExprArg(*this,&E2,1));
+      = E2ToC1.Perform(*this, Entity1, Kind, E2);
     if (E2Result.isInvalid())
       return QualType();
     E2 = E2Result.takeAs<Expr>();
@@ -4591,14 +4566,14 @@ QualType Sema::FindCompositePointerType(SourceLocation Loc,
 
   // Convert E1 to Composite2
   ExprResult E1Result
-    = E1ToC2.Perform(*this, Entity2, Kind, MultiExprArg(*this, &E1, 1));
+    = E1ToC2.Perform(*this, Entity2, Kind, E1);
   if (E1Result.isInvalid())
     return QualType();
   E1 = E1Result.takeAs<Expr>();
 
   // Convert E2 to Composite2
   ExprResult E2Result
-    = E2ToC2.Perform(*this, Entity2, Kind, MultiExprArg(*this, &E2, 1));
+    = E2ToC2.Perform(*this, Entity2, Kind, E2);
   if (E2Result.isInvalid())
     return QualType();
   E2 = E2Result.takeAs<Expr>();
@@ -4839,7 +4814,8 @@ ExprResult Sema::ActOnDecltypeExpression(Expr *E) {
                                                 BO_Comma, BO->getType(),
                                                 BO->getValueKind(),
                                                 BO->getObjectKind(),
-                                                BO->getOperatorLoc()));
+                                                BO->getOperatorLoc(),
+                                                BO->isFPContractable()));
     }
   }
 
@@ -4991,7 +4967,7 @@ Sema::ActOnStartCXXMemberReference(Scope *S, Expr *Base, SourceLocation OpLoc,
   //   type C (or of pointer to a class type C), the unqualified-id is looked
   //   up in the scope of class C. [...]
   ObjectType = ParsedType::make(BaseType);
-  return move(Base);
+  return Base;
 }
 
 ExprResult Sema::DiagnoseDtorReference(SourceLocation NameLoc,
@@ -5056,7 +5032,8 @@ ExprResult Sema::BuildPseudoDestructorExpr(Expr *Base,
   if (CheckArrow(*this, ObjectType, Base, OpKind, OpLoc))
     return ExprError();
 
-  if (!ObjectType->isDependentType() && !ObjectType->isScalarType()) {
+  if (!ObjectType->isDependentType() && !ObjectType->isScalarType() &&
+      !ObjectType->isVectorType()) {
     if (getLangOpts().MicrosoftMode && ObjectType->isVoidType())
       Diag(OpLoc, diag::ext_pseudo_dtor_on_void) << Base->getSourceRange();
     else
@@ -5203,8 +5180,7 @@ ExprResult Sema::ActOnPseudoDestructorExpr(Scope *S, Expr *Base,
   } else {
     // Resolve the template-id to a type.
     TemplateIdAnnotation *TemplateId = SecondTypeName.TemplateId;
-    ASTTemplateArgsPtr TemplateArgsPtr(*this,
-                                       TemplateId->getTemplateArgs(),
+    ASTTemplateArgsPtr TemplateArgsPtr(TemplateId->getTemplateArgs(),
                                        TemplateId->NumArgs);
     TypeResult T = ActOnTemplateIdType(TemplateId->SS,
                                        TemplateId->TemplateKWLoc,
@@ -5253,8 +5229,7 @@ ExprResult Sema::ActOnPseudoDestructorExpr(Scope *S, Expr *Base,
     } else {
       // Resolve the template-id to a type.
       TemplateIdAnnotation *TemplateId = FirstTypeName.TemplateId;
-      ASTTemplateArgsPtr TemplateArgsPtr(*this,
-                                         TemplateId->getTemplateArgs(),
+      ASTTemplateArgsPtr TemplateArgsPtr(TemplateId->getTemplateArgs(),
                                          TemplateId->NumArgs);
       TypeResult T = ActOnTemplateIdType(TemplateId->SS,
                                          TemplateId->TemplateKWLoc,
@@ -5353,7 +5328,7 @@ ExprResult Sema::BuildCXXMemberCallExpr(Expr *E, NamedDecl *FoundDecl,
 
   MarkFunctionReferenced(Exp.get()->getLocStart(), Method);
   CXXMemberCallExpr *CE =
-    new (Context) CXXMemberCallExpr(Context, ME, 0, 0, ResultType, VK,
+    new (Context) CXXMemberCallExpr(Context, ME, MultiExprArg(), ResultType, VK,
                                     Exp.get()->getLocEnd());
   return CE;
 }
diff --git a/lib/Sema/SemaExprMember.cpp b/lib/Sema/SemaExprMember.cpp
index 8f445e2..a7fd471 100644
--- a/lib/Sema/SemaExprMember.cpp
+++ b/lib/Sema/SemaExprMember.cpp
@@ -13,6 +13,7 @@
 #include "clang/Sema/SemaInternal.h"
 #include "clang/Sema/Lookup.h"
 #include "clang/Sema/Scope.h"
+#include "clang/Sema/ScopeInfo.h"
 #include "clang/AST/DeclCXX.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/DeclTemplate.h"
@@ -353,7 +354,7 @@ CheckExtVectorComponent(Sema &S, QualType baseType, ExprValueKind &VK,
   // Now look up the TypeDefDecl from the vector type. Without this,
   // diagostics look bad. We want extended vector types to appear built-in.
   for (Sema::ExtVectorDeclsType::iterator 
-         I = S.ExtVectorDecls.begin(S.ExternalSource),
+         I = S.ExtVectorDecls.begin(S.getExternalSource()),
          E = S.ExtVectorDecls.end(); 
        I != E; ++I) {
     if ((*I)->getUnderlyingType() == VT)
@@ -605,7 +606,8 @@ LookupMemberExprInRecord(Sema &SemaRef, LookupResult &R,
     R.addDecl(ND);
     SemaRef.Diag(R.getNameLoc(), diag::err_no_member_suggest)
       << Name << DC << CorrectedQuotedStr << SS.getRange()
-      << FixItHint::CreateReplacement(R.getNameLoc(), CorrectedStr);
+      << FixItHint::CreateReplacement(Corrected.getCorrectionRange(),
+                                      CorrectedStr);
     SemaRef.Diag(ND->getLocation(), diag::note_previous_decl)
       << ND->getDeclName();
   }
@@ -656,7 +658,7 @@ Sema::BuildMemberReferenceExpr(Expr *Base, QualType BaseType,
     }
 
     if (Result.get())
-      return move(Result);
+      return Result;
 
     // LookupMemberExpr can modify Base, and thus change BaseType
     BaseType = Base->getType();
@@ -1021,7 +1023,7 @@ static bool ShouldTryAgainWithRedefinitionType(Sema &S, ExprResult &base) {
   // Do the substitution as long as the redefinition type isn't just a
   // possibly-qualified pointer to builtin-id or builtin-Class again.
   opty = redef->getAs<ObjCObjectPointerType>();
-  if (opty && !opty->getObjectType()->getInterface() != 0)
+  if (opty && !opty->getObjectType()->getInterface())
     return false;
 
   base = S.ImpCastExprToType(base.take(), redef, CK_BitCast);
@@ -1272,9 +1274,23 @@ Sema::LookupMemberExpr(LookupResult &R, ExprResult &BaseExpr,
       if (warn)
         Diag(MemberLoc, diag::warn_direct_ivar_access) << IV->getDeclName();
     }
-    return Owned(new (Context) ObjCIvarRefExpr(IV, IV->getType(),
-                                               MemberLoc, BaseExpr.take(),
-                                               IsArrow));
+
+    ObjCIvarRefExpr *Result = new (Context) ObjCIvarRefExpr(IV, IV->getType(),
+                                                            MemberLoc,
+                                                            BaseExpr.take(),
+                                                            IsArrow);
+
+    if (getLangOpts().ObjCAutoRefCount) {
+      if (IV->getType().getObjCLifetime() == Qualifiers::OCL_Weak) {
+        DiagnosticsEngine::Level Level =
+          Diags.getDiagnosticLevel(diag::warn_arc_repeated_use_of_weak,
+                                   MemberLoc);
+        if (Level != DiagnosticsEngine::Ignored)
+          getCurFunction()->recordUseOfWeak(Result);
+      }
+    }
+
+    return Owned(Result);
   }
 
   // Objective-C property access.
@@ -1550,7 +1566,7 @@ ExprResult Sema::ActOnMemberAccessExpr(Scope *S, Expr *Base,
           Id.getKind() == UnqualifiedId::IK_DestructorName)
         return DiagnoseDtorReference(NameInfo.getLoc(), Result.get());
 
-      return move(Result);
+      return Result;
     }
 
     ActOnMemberAccessExtraArgs ExtraArgs = {S, Id, ObjCImpDecl, HasTrailingLParen};
@@ -1560,7 +1576,7 @@ ExprResult Sema::ActOnMemberAccessExpr(Scope *S, Expr *Base,
                                       false, &ExtraArgs);
   }
 
-  return move(Result);
+  return Result;
 }
 
 static ExprResult
diff --git a/lib/Sema/SemaExprObjC.cpp b/lib/Sema/SemaExprObjC.cpp
index 0aabf8b..e43b6bf 100644
--- a/lib/Sema/SemaExprObjC.cpp
+++ b/lib/Sema/SemaExprObjC.cpp
@@ -229,7 +229,7 @@ static ObjCMethodDecl *getNSNumberFactoryMethod(Sema &S, SourceLocation Loc,
                                     S.NSNumberPointer, ResultTInfo,
                                     S.NSNumberDecl,
                                     /*isInstance=*/false, /*isVariadic=*/false,
-                                    /*isSynthesized=*/false,
+                                    /*isPropertyAccessor=*/false,
                                     /*isImplicitlyDeclared=*/true,
                                     /*isDefined=*/false,
                                     ObjCMethodDecl::Required,
@@ -345,7 +345,7 @@ static ExprResult CheckObjCCollectionLiteralElement(Sema &S, Expr *Element,
                                        SourceLocation());
     InitializationSequence Seq(S, Entity, Kind, &Element, 1);
     if (!Seq.Failed())
-      return Seq.Perform(S, Entity, Kind, MultiExprArg(S, &Element, 1));
+      return Seq.Perform(S, Entity, Kind, Element);
   }
 
   Expr *OrigElement = Element;
@@ -477,7 +477,7 @@ ExprResult Sema::BuildObjCBoxedExpr(SourceRange SR, Expr *ValueExpr) {
                                    stringWithUTF8String, NSStringPointer,
                                    ResultTInfo, NSStringDecl,
                                    /*isInstance=*/false, /*isVariadic=*/false,
-                                   /*isSynthesized=*/false,
+                                   /*isPropertyAccessor=*/false,
                                    /*isImplicitlyDeclared=*/true,
                                    /*isDefined=*/false,
                                    ObjCMethodDecl::Required,
@@ -646,7 +646,7 @@ ExprResult Sema::BuildObjCArrayLiteral(SourceRange SR, MultiExprArg Elements) {
                            ResultTInfo,
                            Context.getTranslationUnitDecl(),
                            false /*Instance*/, false/*isVariadic*/,
-                           /*isSynthesized=*/false,
+                           /*isPropertyAccessor=*/false,
                            /*isImplicitlyDeclared=*/true, /*isDefined=*/false,
                            ObjCMethodDecl::Required,
                            false);
@@ -708,7 +708,7 @@ ExprResult Sema::BuildObjCArrayLiteral(SourceRange SR, MultiExprArg Elements) {
 
   // Check that each of the elements provided is valid in a collection literal,
   // performing conversions as necessary.
-  Expr **ElementsBuffer = Elements.get();
+  Expr **ElementsBuffer = Elements.data();
   for (unsigned I = 0, N = Elements.size(); I != N; ++I) {
     ExprResult Converted = CheckObjCCollectionLiteralElement(*this,
                                                              ElementsBuffer[I],
@@ -724,10 +724,8 @@ ExprResult Sema::BuildObjCArrayLiteral(SourceRange SR, MultiExprArg Elements) {
                                     Context.getObjCInterfaceType(NSArrayDecl));
 
   return MaybeBindToTemporary(
-           ObjCArrayLiteral::Create(Context, 
-                                    llvm::makeArrayRef(Elements.get(), 
-                                                       Elements.size()), 
-                                    Ty, ArrayWithObjectsMethod, SR));
+           ObjCArrayLiteral::Create(Context, Elements, Ty,
+                                    ArrayWithObjectsMethod, SR));
 }
 
 ExprResult Sema::BuildObjCDictionaryLiteral(SourceRange SR, 
@@ -766,7 +764,7 @@ ExprResult Sema::BuildObjCDictionaryLiteral(SourceRange SR,
                            0 /*TypeSourceInfo */,
                            Context.getTranslationUnitDecl(),
                            false /*Instance*/, false/*isVariadic*/,
-                           /*isSynthesized=*/false,
+                           /*isPropertyAccessor=*/false,
                            /*isImplicitlyDeclared=*/true, /*isDefined=*/false,
                            ObjCMethodDecl::Required,
                            false);
@@ -1125,7 +1123,9 @@ void Sema::EmitRelatedResultTypeNote(const Expr *E) {
 
 bool Sema::CheckMessageArgumentTypes(QualType ReceiverType,
                                      Expr **Args, unsigned NumArgs,
-                                     Selector Sel, ObjCMethodDecl *Method,
+                                     Selector Sel, 
+                                     ArrayRef<SourceLocation> SelectorLocs,
+                                     ObjCMethodDecl *Method,
                                      bool isClassMessage, bool isSuperMessage,
                                      SourceLocation lbrac, SourceLocation rbrac,
                                      QualType &ReturnType, ExprValueKind &VK) {
@@ -1149,7 +1149,8 @@ bool Sema::CheckMessageArgumentTypes(QualType ReceiverType,
                               : diag::warn_inst_method_not_found;
     if (!getLangOpts().DebuggerSupport)
       Diag(lbrac, DiagID)
-        << Sel << isClassMessage << SourceRange(lbrac, rbrac);
+        << Sel << isClassMessage << SourceRange(SelectorLocs.front(), 
+                                                SelectorLocs.back());
 
     // In debuggers, we want to use __unknown_anytype for these
     // results so that clients can cast them.
@@ -1304,8 +1305,8 @@ static void DiagnoseARCUseOfWeakReceiver(Sema &S, Expr *Receiver) {
   Expr *RExpr = Receiver->IgnoreParenImpCasts();
   SourceLocation Loc = RExpr->getLocStart();
   QualType T = RExpr->getType();
-  ObjCPropertyDecl *PDecl = 0;
-  ObjCMethodDecl *GDecl = 0;
+  const ObjCPropertyDecl *PDecl = 0;
+  const ObjCMethodDecl *GDecl = 0;
   if (PseudoObjectExpr *POE = dyn_cast<PseudoObjectExpr>(RExpr)) {
     RExpr = POE->getSyntacticForm();
     if (ObjCPropertyRefExpr *PRE = dyn_cast<ObjCPropertyRefExpr>(RExpr)) {
@@ -1327,34 +1328,29 @@ static void DiagnoseARCUseOfWeakReceiver(Sema &S, Expr *Receiver) {
     // See if receiver is a method which envokes a synthesized getter
     // backing a 'weak' property.
     ObjCMethodDecl *Method = ME->getMethodDecl();
-    if (Method && Method->isSynthesized()) {
-      Selector Sel = Method->getSelector();
-      if (Sel.getNumArgs() == 0) {
-        const DeclContext *Container = Method->getDeclContext();
-        PDecl = 
-          S.LookupPropertyDecl(cast<ObjCContainerDecl>(Container),
-                               Sel.getIdentifierInfoForSlot(0));
-      }
+    if (Method && Method->getSelector().getNumArgs() == 0) {
+      PDecl = Method->findPropertyDecl();
       if (PDecl)
         T = PDecl->getType();
     }
   }
   
-  if (T.getObjCLifetime() == Qualifiers::OCL_Weak) {
-    S.Diag(Loc, diag::warn_receiver_is_weak) 
-      << ((!PDecl && !GDecl) ? 0 : (PDecl ? 1 : 2));
-    if (PDecl)
-      S.Diag(PDecl->getLocation(), diag::note_property_declare);
-    else if (GDecl)
-      S.Diag(GDecl->getLocation(), diag::note_method_declared_at) << GDecl;
-    return;
+  if (T.getObjCLifetime() != Qualifiers::OCL_Weak) {
+    if (!PDecl)
+      return;
+    if (!(PDecl->getPropertyAttributes() & ObjCPropertyDecl::OBJC_PR_weak))
+      return;
   }
-  
-  if (PDecl && 
-      (PDecl->getPropertyAttributes() & ObjCPropertyDecl::OBJC_PR_weak)) {
-    S.Diag(Loc, diag::warn_receiver_is_weak) << 1;
+
+  S.Diag(Loc, diag::warn_receiver_is_weak)
+    << ((!PDecl && !GDecl) ? 0 : (PDecl ? 1 : 2));
+
+  if (PDecl)
     S.Diag(PDecl->getLocation(), diag::note_property_declare);
-  }
+  else if (GDecl)
+    S.Diag(GDecl->getLocation(), diag::note_method_declared_at) << GDecl;
+
+  S.Diag(Loc, diag::note_arc_assign_to_strong);
 }
 
 /// HandleExprPropertyRefExpr - Handle foo.bar where foo is a pointer to an
@@ -1776,19 +1772,17 @@ ExprResult Sema::ActOnSuperMessage(Scope *S,
 
   // We are in a method whose class has a superclass, so 'super'
   // is acting as a keyword.
-  if (Method->isInstanceMethod()) {
-    if (Sel.getMethodFamily() == OMF_dealloc)
-      getCurFunction()->ObjCShouldCallSuperDealloc = false;
-    if (Sel.getMethodFamily() == OMF_finalize)
-      getCurFunction()->ObjCShouldCallSuperFinalize = false;
+  if (Method->getSelector() == Sel)
+    getCurFunction()->ObjCShouldCallSuper = false;
 
+  if (Method->isInstanceMethod()) {
     // Since we are in an instance method, this is an instance
     // message to the superclass instance.
     QualType SuperTy = Context.getObjCInterfaceType(Super);
     SuperTy = Context.getObjCObjectPointerType(SuperTy);
     return BuildInstanceMessage(0, SuperTy, SuperLoc,
                                 Sel, /*Method=*/0,
-                                LBracLoc, SelectorLocs, RBracLoc, move(Args));
+                                LBracLoc, SelectorLocs, RBracLoc, Args);
   }
   
   // Since we are in a class method, this is a class message to
@@ -1796,7 +1790,7 @@ ExprResult Sema::ActOnSuperMessage(Scope *S,
   return BuildClassMessage(/*ReceiverTypeInfo=*/0,
                            Context.getObjCInterfaceType(Super),
                            SuperLoc, Sel, /*Method=*/0,
-                           LBracLoc, SelectorLocs, RBracLoc, move(Args));
+                           LBracLoc, SelectorLocs, RBracLoc, Args);
 }
 
 
@@ -1911,7 +1905,7 @@ ExprResult Sema::BuildClassMessage(TypeSourceInfo *ReceiverTypeInfo,
     // If the receiver type is dependent, we can't type-check anything
     // at this point. Build a dependent expression.
     unsigned NumArgs = ArgsIn.size();
-    Expr **Args = reinterpret_cast<Expr **>(ArgsIn.release());
+    Expr **Args = ArgsIn.data();
     assert(SuperLoc.isInvalid() && "Message to super with dependent type");
     return Owned(ObjCMessageExpr::Create(Context, ReceiverType,
                                          VK_RValue, LBracLoc, ReceiverTypeInfo,
@@ -1965,8 +1959,9 @@ ExprResult Sema::BuildClassMessage(TypeSourceInfo *ReceiverTypeInfo,
   ExprValueKind VK = VK_RValue;
 
   unsigned NumArgs = ArgsIn.size();
-  Expr **Args = reinterpret_cast<Expr **>(ArgsIn.release());
-  if (CheckMessageArgumentTypes(ReceiverType, Args, NumArgs, Sel, Method, true,
+  Expr **Args = ArgsIn.data();
+  if (CheckMessageArgumentTypes(ReceiverType, Args, NumArgs, Sel, SelectorLocs,
+                                Method, true,
                                 SuperLoc.isValid(), LBracLoc, RBracLoc, 
                                 ReturnType, VK))
     return ExprError();
@@ -2016,7 +2011,7 @@ ExprResult Sema::ActOnClassMessage(Scope *S,
 
   return BuildClassMessage(ReceiverTypeInfo, ReceiverType, 
                            /*SuperLoc=*/SourceLocation(), Sel, /*Method=*/0,
-                           LBracLoc, SelectorLocs, RBracLoc, move(Args));
+                           LBracLoc, SelectorLocs, RBracLoc, Args);
 }
 
 ExprResult Sema::BuildInstanceMessageImplicit(Expr *Receiver,
@@ -2095,7 +2090,7 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
       // If the receiver is type-dependent, we can't type-check anything
       // at this point. Build a dependent expression.
       unsigned NumArgs = ArgsIn.size();
-      Expr **Args = reinterpret_cast<Expr **>(ArgsIn.release());
+      Expr **Args = ArgsIn.data();
       assert(SuperLoc.isInvalid() && "Message to super with dependent type");
       return Owned(ObjCMessageExpr::Create(Context, Context.DependentTy,
                                            VK_RValue, LBracLoc, Receiver, Sel, 
@@ -2282,7 +2277,7 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
                                       LBracLoc,
                                       SelectorLocs,
                                       RBracLoc,
-                                      move(ArgsIn));
+                                      ArgsIn);
         } else {
           // Reject other random receiver types (e.g. structs).
           Diag(Loc, diag::err_bad_receiver_type)
@@ -2295,12 +2290,13 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
 
   // Check the message arguments.
   unsigned NumArgs = ArgsIn.size();
-  Expr **Args = reinterpret_cast<Expr **>(ArgsIn.release());
+  Expr **Args = ArgsIn.data();
   QualType ReturnType;
   ExprValueKind VK = VK_RValue;
   bool ClassMessage = (ReceiverType->isObjCClassType() ||
                        ReceiverType->isObjCQualifiedClassType());
-  if (CheckMessageArgumentTypes(ReceiverType, Args, NumArgs, Sel, Method, 
+  if (CheckMessageArgumentTypes(ReceiverType, Args, NumArgs, Sel,
+                                SelectorLocs, Method, 
                                 ClassMessage, SuperLoc.isValid(), 
                                 LBracLoc, RBracLoc, ReturnType, VK))
     return ExprError();
@@ -2428,6 +2424,24 @@ ExprResult Sema::BuildInstanceMessage(Expr *Receiver,
     // In ARC, check for message sends which are likely to introduce
     // retain cycles.
     checkRetainCycles(Result);
+
+    if (!isImplicit && Method) {
+      if (const ObjCPropertyDecl *Prop = Method->findPropertyDecl()) {
+        bool IsWeak =
+          Prop->getPropertyAttributes() & ObjCPropertyDecl::OBJC_PR_weak;
+        if (!IsWeak && Sel.isUnarySelector())
+          IsWeak = ReturnType.getObjCLifetime() & Qualifiers::OCL_Weak;
+
+        if (IsWeak) {
+          DiagnosticsEngine::Level Level =
+            Diags.getDiagnosticLevel(diag::warn_arc_repeated_use_of_weak,
+                                     LBracLoc);
+          if (Level != DiagnosticsEngine::Ignored)
+            getCurFunction()->recordUseOfWeak(Result, Prop);
+
+        }
+      }
+    }
   }
       
   return MaybeBindToTemporary(Result);
@@ -2448,7 +2462,7 @@ ExprResult Sema::ActOnInstanceMessage(Scope *S,
 
   return BuildInstanceMessage(Receiver, Receiver->getType(),
                               /*SuperLoc=*/SourceLocation(), Sel, /*Method=*/0, 
-                              LBracLoc, SelectorLocs, RBracLoc, move(Args));
+                              LBracLoc, SelectorLocs, RBracLoc, Args);
 }
 
 enum ARCConversionTypeClass {
@@ -3079,8 +3093,8 @@ Expr *Sema::stripARCUnbridgedCast(Expr *e) {
 
     return new (Context) GenericSelectionExpr(Context, gse->getGenericLoc(),
                                               gse->getControllingExpr(),
-                                              subTypes.data(), subExprs.data(),
-                                              n, gse->getDefaultLoc(),
+                                              subTypes, subExprs,
+                                              gse->getDefaultLoc(),
                                               gse->getRParenLoc(),
                                        gse->containsUnexpandedParameterPack(),
                                               gse->getResultIndex());
@@ -3101,8 +3115,8 @@ bool Sema::CheckObjCARCUnavailableWeakConversion(QualType castType,
       canExprType->isObjCObjectPointerType()) {
     if (const ObjCObjectPointerType *ObjT =
         canExprType->getAs<ObjCObjectPointerType>())
-      if (ObjT->getInterfaceDecl()->isArcWeakrefUnavailable())
-        return false;
+      if (const ObjCInterfaceDecl *ObjI = ObjT->getInterfaceDecl())
+        return !ObjI->isArcWeakrefUnavailable();
   }
   return true;
 }
diff --git a/lib/Sema/SemaInit.cpp b/lib/Sema/SemaInit.cpp
index 62ab1e6..3596bbf 100644
--- a/lib/Sema/SemaInit.cpp
+++ b/lib/Sema/SemaInit.cpp
@@ -1316,6 +1316,8 @@ void InitListChecker::CheckStructUnionTypes(const InitializedEntity &Entity,
   // If the record is invalid, some of it's members are invalid. To avoid
   // confusion, we forgo checking the intializer for the entire record.
   if (structDecl->isInvalidDecl()) {
+    // Assume it was supposed to consume a single initializer.
+    ++Index;
     hadError = true;
     return;
   }
@@ -1503,11 +1505,14 @@ static void ExpandAnonymousFieldDesignator(Sema &SemaRef,
 ///  corresponds to FieldName.
 static IndirectFieldDecl *FindIndirectFieldDesignator(FieldDecl *AnonField,
                                                  IdentifierInfo *FieldName) {
+  if (!FieldName)
+    return 0;
+
   assert(AnonField->isAnonymousStructOrUnion());
   Decl *NextDecl = AnonField->getNextDeclInContext();
   while (IndirectFieldDecl *IF = 
           dyn_cast_or_null<IndirectFieldDecl>(NextDecl)) {
-    if (FieldName && FieldName == IF->getAnonField()->getIdentifier())
+    if (FieldName == IF->getAnonField()->getIdentifier())
       return IF;
     NextDecl = NextDecl->getNextDeclInContext();
   }
@@ -1521,8 +1526,8 @@ static DesignatedInitExpr *CloneDesignatedInitExpr(Sema &SemaRef,
   for (unsigned I = 0; I < NumIndexExprs; ++I)
     IndexExprs[I] = DIE->getSubExpr(I + 1);
   return DesignatedInitExpr::Create(SemaRef.Context, DIE->designators_begin(),
-                                    DIE->size(), IndexExprs.data(),
-                                    NumIndexExprs, DIE->getEqualOrColonLoc(),
+                                    DIE->size(), IndexExprs,
+                                    DIE->getEqualOrColonLoc(),
                                     DIE->usesGNUSyntax(), DIE->getInit());
 }
 
@@ -1562,7 +1567,7 @@ class FieldInitializerValidatorCCC : public CorrectionCandidateCallback {
 ///
 /// @param DesigIdx  The index of the current designator.
 ///
-/// @param DeclType  The type of the "current object" (C99 6.7.8p17),
+/// @param CurrentObjectType The type of the "current object" (C99 6.7.8p17),
 /// into which the designation in @p DIE should refer.
 ///
 /// @param NextField  If non-NULL and the first designator in @p DIE is
@@ -2068,7 +2073,7 @@ InitListChecker::getStructuredSubobjectInit(InitListExpr *IList, unsigned Index,
 
   InitListExpr *Result
     = new (SemaRef.Context) InitListExpr(SemaRef.Context,
-                                         InitRange.getBegin(), 0, 0,
+                                         InitRange.getBegin(), MultiExprArg(),
                                          InitRange.getEnd());
 
   QualType ResultType = CurrentObjectType;
@@ -2261,8 +2266,8 @@ ExprResult Sema::ActOnDesignatedInitializer(Designation &Desig,
   DesignatedInitExpr *DIE
     = DesignatedInitExpr::Create(Context,
                                  Designators.data(), Designators.size(),
-                                 InitExpressions.data(), InitExpressions.size(),
-                                 Loc, GNUSyntax, Init.takeAs<Expr>());
+                                 InitExpressions, Loc, GNUSyntax,
+                                 Init.takeAs<Expr>());
 
   if (!getLangOpts().C99)
     Diag(DIE->getLocStart(), diag::ext_designated_init)
@@ -2814,14 +2819,6 @@ static void TryConstructorInitialization(Sema &S,
   assert((!InitListSyntax || (NumArgs == 1 && isa<InitListExpr>(Args[0]))) &&
          "InitListSyntax must come with a single initializer list argument.");
 
-  // Check constructor arguments for self reference.
-  if (DeclaratorDecl *DD = Entity.getDecl())
-    // Parameters arguments are occassionially constructed with itself,
-    // for instance, in recursive functions.  Skip them.
-    if (!isa<ParmVarDecl>(DD))
-      for (unsigned i = 0; i < NumArgs; ++i)
-        S.CheckSelfReference(DD, Args[i]);
-
   // The type we're constructing needs to be complete.
   if (S.RequireCompleteType(Kind.getLocation(), DestType, 0)) {
     Sequence.setIncompleteTypeFailure(DestType);
@@ -3614,8 +3611,8 @@ static void TryValueInitialization(Sema &S,
       //    user-provided or deleted default constructor, then the object is
       //    zero-initialized and, if T has a non-trivial default constructor,
       //    default-initialized;
-      // FIXME: The 'non-union' here is a defect (not yet assigned an issue
-      // number). Update the quotation when the defect is resolved.
+      // The 'non-union' here was removed by DR1502. The 'non-trivial default
+      // constructor' part was removed by DR1507.
       if (NeedZeroInitialization)
         Sequence.AddZeroInitializationStep(Entity.getType());
 
@@ -3703,8 +3700,14 @@ static void TryUserDefinedConversion(Sema &S,
 
     // Try to complete the type we're converting to.
     if (!S.RequireCompleteType(Kind.getLocation(), DestType, 0)) {
-      DeclContext::lookup_iterator Con, ConEnd;
-      for (llvm::tie(Con, ConEnd) = S.LookupConstructors(DestRecordDecl);
+      DeclContext::lookup_iterator ConOrig, ConEndOrig;
+      llvm::tie(ConOrig, ConEndOrig) = S.LookupConstructors(DestRecordDecl);
+      // The container holding the constructors can under certain conditions
+      // be changed while iterating. To be safe we copy the lookup results
+      // to a new container.
+      SmallVector<NamedDecl*, 8> CopyOfCon(ConOrig, ConEndOrig);
+      for (SmallVector<NamedDecl*, 8>::iterator
+             Con = CopyOfCon.begin(), ConEnd = CopyOfCon.end();
            Con != ConEnd; ++Con) {
         NamedDecl *D = *Con;
         DeclAccessPair FoundDecl = DeclAccessPair::make(D, D->getAccess());
@@ -4413,7 +4416,7 @@ static ExprResult CopyObject(Sema &S,
   if (const RecordType *Record = T->getAs<RecordType>())
     Class = cast<CXXRecordDecl>(Record->getDecl());
   if (!Class)
-    return move(CurInit);
+    return CurInit;
 
   // C++0x [class.copy]p32:
   //   When certain criteria are met, an implementation is allowed to
@@ -4435,7 +4438,7 @@ static ExprResult CopyObject(Sema &S,
 
   // Make sure that the type we are copying is complete.
   if (S.RequireCompleteType(Loc, T, diag::err_temp_copy_incomplete))
-    return move(CurInit);
+    return CurInit;
 
   // Perform overload resolution using the class's copy/move constructors.
   // Only consider constructors and constructor templates. Per
@@ -4460,7 +4463,7 @@ static ExprResult CopyObject(Sema &S,
     CandidateSet.NoteCandidates(S, OCD_AllCandidates, CurInitExpr);
     if (!IsExtraneousCopy || S.isSFINAEContext())
       return ExprError();
-    return move(CurInit);
+    return CurInit;
 
   case OR_Ambiguous:
     S.Diag(Loc, diag::err_temp_copy_ambiguous)
@@ -4478,7 +4481,7 @@ static ExprResult CopyObject(Sema &S,
   }
 
   CXXConstructorDecl *Constructor = cast<CXXConstructorDecl>(Best->Function);
-  ASTOwningVector<Expr*> ConstructorArgs(S);
+  SmallVector<Expr*, 8> ConstructorArgs;
   CurInit.release(); // Ownership transferred into MultiExprArg, below.
 
   S.CheckConstructorAccess(Loc, Constructor, Entity,
@@ -4521,7 +4524,7 @@ static ExprResult CopyObject(Sema &S,
 
   // Actually perform the constructor call.
   CurInit = S.BuildCXXConstructExpr(Loc, T, Constructor, Elidable,
-                                    move_arg(ConstructorArgs),
+                                    ConstructorArgs,
                                     HadMultipleCandidates,
                                     /*ZeroInit*/ false,
                                     CXXConstructExpr::CK_Complete,
@@ -4530,7 +4533,7 @@ static ExprResult CopyObject(Sema &S,
   // If we're supposed to bind temporaries, do so.
   if (!CurInit.isInvalid() && shouldBindAsTemporary(Entity))
     CurInit = S.MaybeBindToTemporary(CurInit.takeAs<Expr>());
-  return move(CurInit);
+  return CurInit;
 }
 
 /// \brief Check whether elidable copy construction for binding a reference to
@@ -4619,7 +4622,7 @@ PerformConstructorInitialization(Sema &S,
   bool HadMultipleCandidates = Step.Function.HadMultipleCandidates;
 
   // Build a call to the selected constructor.
-  ASTOwningVector<Expr*> ConstructorArgs(S);
+  SmallVector<Expr*, 8> ConstructorArgs;
   SourceLocation Loc = (Kind.isCopyInit() && Kind.getEqualLoc().isValid())
                          ? Kind.getEqualLoc()
                          : Kind.getLocation();
@@ -4648,7 +4651,7 @@ PerformConstructorInitialization(Sema &S,
 
   // Determine the arguments required to actually perform the constructor
   // call.
-  if (S.CompleteConstructorCall(Constructor, move(Args),
+  if (S.CompleteConstructorCall(Constructor, Args,
                                 Loc, ConstructorArgs,
                                 AllowExplicitConv))
     return ExprError();
@@ -4660,8 +4663,6 @@ PerformConstructorInitialization(Sema &S,
         (Kind.getKind() == InitializationKind::IK_Direct ||
          Kind.getKind() == InitializationKind::IK_Value)))) {
     // An explicitly-constructed temporary, e.g., X(1, 2).
-    unsigned NumExprs = ConstructorArgs.size();
-    Expr **Exprs = (Expr **)ConstructorArgs.take();
     S.MarkFunctionReferenced(Loc, Constructor);
     S.DiagnoseUseOfDecl(Constructor, Loc);
 
@@ -4675,8 +4676,7 @@ PerformConstructorInitialization(Sema &S,
     CurInit = S.Owned(new (S.Context) CXXTemporaryObjectExpr(S.Context,
                                                              Constructor,
                                                              TSInfo,
-                                                             Exprs,
-                                                             NumExprs,
+                                                             ConstructorArgs,
                                                              ParenRange,
                                                      HadMultipleCandidates,
                                          ConstructorInitRequiresZeroInit));
@@ -4702,7 +4702,7 @@ PerformConstructorInitialization(Sema &S,
     if (Entity.allowsNRVO())
       CurInit = S.BuildCXXConstructExpr(Loc, Entity.getType(),
                                         Constructor, /*Elidable=*/true,
-                                        move_arg(ConstructorArgs),
+                                        ConstructorArgs,
                                         HadMultipleCandidates,
                                         ConstructorInitRequiresZeroInit,
                                         ConstructKind,
@@ -4710,7 +4710,7 @@ PerformConstructorInitialization(Sema &S,
     else
       CurInit = S.BuildCXXConstructExpr(Loc, Entity.getType(),
                                         Constructor,
-                                        move_arg(ConstructorArgs),
+                                        ConstructorArgs,
                                         HadMultipleCandidates,
                                         ConstructorInitRequiresZeroInit,
                                         ConstructKind,
@@ -4727,7 +4727,7 @@ PerformConstructorInitialization(Sema &S,
   if (shouldBindAsTemporary(Entity))
     CurInit = S.MaybeBindToTemporary(CurInit.takeAs<Expr>());
 
-  return move(CurInit);
+  return CurInit;
 }
 
 /// Determine whether the specified InitializedEntity definitely has a lifetime
@@ -4775,7 +4775,7 @@ InitializationSequence::Perform(Sema &S,
                                 QualType *ResultType) {
   if (Failed()) {
     unsigned NumArgs = Args.size();
-    Diagnose(S, Entity, Kind, (Expr **)Args.release(), NumArgs);
+    Diagnose(S, Entity, Kind, Args.data(), NumArgs);
     return ExprError();
   }
 
@@ -4795,7 +4795,7 @@ InitializationSequence::Perform(Sema &S,
         // introduced and such).  So, we fall back to making the array
         // type a dependently-sized array type with no specified
         // bound.
-        if (isa<InitListExpr>((Expr *)Args.get()[0])) {
+        if (isa<InitListExpr>((Expr *)Args[0])) {
           SourceRange Brackets;
 
           // Scavange the location of the brackets from the entity, if we can.
@@ -4823,12 +4823,12 @@ InitializationSequence::Perform(Sema &S,
       // Rebuild the ParenListExpr.
       SourceRange ParenRange = Kind.getParenRange();
       return S.ActOnParenListExpr(ParenRange.getBegin(), ParenRange.getEnd(),
-                                  move(Args));
+                                  Args);
     }
     assert(Kind.getKind() == InitializationKind::IK_Copy ||
            Kind.isExplicitCast() || 
            Kind.getKind() == InitializationKind::IK_DirectList);
-    return ExprResult(Args.release()[0]);
+    return ExprResult(Args[0]);
   }
 
   // No steps means no initialization.
@@ -4836,22 +4836,22 @@ InitializationSequence::Perform(Sema &S,
     return S.Owned((Expr *)0);
 
   if (S.getLangOpts().CPlusPlus0x && Entity.getType()->isReferenceType() &&
-      Args.size() == 1 && isa<InitListExpr>(Args.get()[0]) &&
+      Args.size() == 1 && isa<InitListExpr>(Args[0]) &&
       Entity.getKind() != InitializedEntity::EK_Parameter) {
     // Produce a C++98 compatibility warning if we are initializing a reference
     // from an initializer list. For parameters, we produce a better warning
     // elsewhere.
-    Expr *Init = Args.get()[0];
+    Expr *Init = Args[0];
     S.Diag(Init->getLocStart(), diag::warn_cxx98_compat_reference_list_init)
       << Init->getSourceRange();
   }
 
   // Diagnose cases where we initialize a pointer to an array temporary, and the
   // pointer obviously outlives the temporary.
-  if (Args.size() == 1 && Args.get()[0]->getType()->isArrayType() &&
+  if (Args.size() == 1 && Args[0]->getType()->isArrayType() &&
       Entity.getType()->isPointerType() &&
       InitializedEntityOutlivesFullExpression(Entity)) {
-    Expr *Init = Args.get()[0];
+    Expr *Init = Args[0];
     Expr::LValueClassification Kind = Init->ClassifyLValue(S.Context);
     if (Kind == Expr::LV_ClassTemporary || Kind == Expr::LV_ArrayTemporary)
       S.Diag(Init->getLocStart(), diag::warn_temporary_array_to_pointer_decay)
@@ -4897,7 +4897,7 @@ InitializationSequence::Perform(Sema &S,
   case SK_ProduceObjCObject:
   case SK_StdInitializerList: {
     assert(Args.size() == 1);
-    CurInit = Args.get()[0];
+    CurInit = Args[0];
     if (!CurInit.get()) return ExprError();
     break;
   }
@@ -4924,7 +4924,7 @@ InitializationSequence::Perform(Sema &S,
       // initializer to reflect that choice.
       S.CheckAddressOfMemberAccess(CurInit.get(), Step->Function.FoundDecl);
       S.DiagnoseUseOfDecl(Step->Function.FoundDecl, Kind.getLocation());
-      CurInit = S.FixOverloadedFunctionReference(move(CurInit),
+      CurInit = S.FixOverloadedFunctionReference(CurInit,
                                                  Step->Function.FoundDecl,
                                                  Step->Function.Function);
       break;
@@ -5016,7 +5016,7 @@ InitializationSequence::Perform(Sema &S,
       break;
 
     case SK_ExtraneousCopyToTemporary:
-      CurInit = CopyObject(S, Step->Type, Entity, move(CurInit),
+      CurInit = CopyObject(S, Step->Type, Entity, CurInit,
                            /*IsExtraneousCopy=*/true);
       break;
 
@@ -5031,7 +5031,7 @@ InitializationSequence::Perform(Sema &S,
       bool CreatedObject = false;
       if (CXXConstructorDecl *Constructor = dyn_cast<CXXConstructorDecl>(Fn)) {
         // Build a call to the selected constructor.
-        ASTOwningVector<Expr*> ConstructorArgs(S);
+        SmallVector<Expr*, 8> ConstructorArgs;
         SourceLocation Loc = CurInit.get()->getLocStart();
         CurInit.release(); // Ownership transferred into MultiExprArg, below.
 
@@ -5045,7 +5045,7 @@ InitializationSequence::Perform(Sema &S,
 
         // Build an expression that constructs a temporary.
         CurInit = S.BuildCXXConstructExpr(Loc, Step->Type, Constructor,
-                                          move_arg(ConstructorArgs),
+                                          ConstructorArgs,
                                           HadMultipleCandidates,
                                           /*ZeroInit*/ false,
                                           CXXConstructExpr::CK_Complete,
@@ -5079,7 +5079,7 @@ InitializationSequence::Perform(Sema &S,
                                                 FoundFn, Conversion);
         if(CurInitExprRes.isInvalid())
           return ExprError();
-        CurInit = move(CurInitExprRes);
+        CurInit = CurInitExprRes;
 
         // Build the actual call to the conversion function.
         CurInit = S.BuildCXXMemberCallExpr(CurInit.get(), FoundFn, Conversion,
@@ -5115,7 +5115,7 @@ InitializationSequence::Perform(Sema &S,
         CurInit = S.MaybeBindToTemporary(CurInit.takeAs<Expr>());
       if (RequiresCopy)
         CurInit = CopyObject(S, Entity.getType().getNonReferenceType(), Entity,
-                             move(CurInit), /*IsExtraneousCopy=*/false);
+                             CurInit, /*IsExtraneousCopy=*/false);
       break;
     }
 
@@ -5144,7 +5144,7 @@ InitializationSequence::Perform(Sema &S,
                                     getAssignmentAction(Entity), CCK);
       if (CurInitExprRes.isInvalid())
         return ExprError();
-      CurInit = move(CurInitExprRes);
+      CurInit = CurInitExprRes;
       break;
     }
 
@@ -5195,13 +5195,13 @@ InitializationSequence::Perform(Sema &S,
                                         Entity.getType().getNonReferenceType());
       bool UseTemporary = Entity.getType()->isReferenceType();
       assert(Args.size() == 1 && "expected a single argument for list init");
-      InitListExpr *InitList = cast<InitListExpr>(Args.get()[0]);
+      InitListExpr *InitList = cast<InitListExpr>(Args[0]);
       S.Diag(InitList->getExprLoc(), diag::warn_cxx98_compat_ctor_list_init)
         << InitList->getSourceRange();
       MultiExprArg Arg(InitList->getInits(), InitList->getNumInits());
       CurInit = PerformConstructorInitialization(S, UseTemporary ? TempEntity :
                                                                    Entity,
-                                                 Kind, move(Arg), *Step,
+                                                 Kind, Arg, *Step,
                                                ConstructorInitRequiresZeroInit);
       break;
     }
@@ -5214,7 +5214,7 @@ InitializationSequence::Perform(Sema &S,
       Expr *E = CurInit.take();
       InitListExpr *Syntactic = Step->WrappingSyntacticList;
       InitListExpr *ILE = new (S.Context) InitListExpr(S.Context,
-          Syntactic->getLBraceLoc(), &E, 1, Syntactic->getRBraceLoc());
+          Syntactic->getLBraceLoc(), E, Syntactic->getRBraceLoc());
       ILE->setSyntacticForm(Syntactic);
       ILE->setType(E->getType());
       ILE->setValueKind(E->getValueKind());
@@ -5234,7 +5234,7 @@ InitializationSequence::Perform(Sema &S,
       bool UseTemporary = Entity.getType()->isReferenceType();
       CurInit = PerformConstructorInitialization(S, UseTemporary ? TempEntity
                                                                  : Entity,
-                                                 Kind, move(Args), *Step,
+                                                 Kind, Args, *Step,
                                                ConstructorInitRequiresZeroInit);
       break;
     }
@@ -5268,15 +5268,15 @@ InitializationSequence::Perform(Sema &S,
 
     case SK_CAssignment: {
       QualType SourceType = CurInit.get()->getType();
-      ExprResult Result = move(CurInit);
+      ExprResult Result = CurInit;
       Sema::AssignConvertType ConvTy =
         S.CheckSingleAssignmentConstraints(Step->Type, Result);
       if (Result.isInvalid())
         return ExprError();
-      CurInit = move(Result);
+      CurInit = Result;
 
       // If this is a call, allow conversion to a transparent union.
-      ExprResult CurInitExprRes = move(CurInit);
+      ExprResult CurInitExprRes = CurInit;
       if (ConvTy != Sema::Compatible &&
           Entity.getKind() == InitializedEntity::EK_Parameter &&
           S.CheckTransparentUnionArgumentConstraints(Step->Type, CurInitExprRes)
@@ -5284,7 +5284,7 @@ InitializationSequence::Perform(Sema &S,
         ConvTy = Sema::Compatible;
       if (CurInitExprRes.isInvalid())
         return ExprError();
-      CurInit = move(CurInitExprRes);
+      CurInit = CurInitExprRes;
 
       bool Complained;
       if (S.DiagnoseAssignmentResult(ConvTy, Kind.getLocation(),
@@ -5398,7 +5398,7 @@ InitializationSequence::Perform(Sema &S,
       }
       InitListExpr *Semantic = new (S.Context)
           InitListExpr(S.Context, ILE->getLBraceLoc(),
-                       Converted.data(), NumInits, ILE->getRBraceLoc());
+                       Converted, ILE->getRBraceLoc());
       Semantic->setSyntacticForm(ILE);
       Semantic->setType(Dest);
       Semantic->setInitializesStdInitializerList();
@@ -5415,7 +5415,7 @@ InitializationSequence::Perform(Sema &S,
                                   cast<FieldDecl>(Entity.getDecl()),
                                   CurInit.get());
 
-  return move(CurInit);
+  return CurInit;
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/lib/Sema/SemaLambda.cpp b/lib/Sema/SemaLambda.cpp
index 6414c6f..15cd2a7 100644
--- a/lib/Sema/SemaLambda.cpp
+++ b/lib/Sema/SemaLambda.cpp
@@ -22,13 +22,14 @@ using namespace clang;
 using namespace sema;
 
 CXXRecordDecl *Sema::createLambdaClosureType(SourceRange IntroducerRange,
+                                             TypeSourceInfo *Info,
                                              bool KnownDependent) {
   DeclContext *DC = CurContext;
   while (!(DC->isFunctionOrMethod() || DC->isRecord() || DC->isFileContext()))
     DC = DC->getParent();
   
   // Start constructing the lambda class.
-  CXXRecordDecl *Class = CXXRecordDecl::CreateLambda(Context, DC, 
+  CXXRecordDecl *Class = CXXRecordDecl::CreateLambda(Context, DC, Info,
                                                      IntroducerRange.getBegin(),
                                                      KnownDependent);
   DC->addDecl(Class);
@@ -369,15 +370,13 @@ void Sema::ActOnStartOfLambdaDefinition(LambdaIntroducer &Intro,
     if (!TmplScope->decl_empty())
       KnownDependent = true;
   
-  CXXRecordDecl *Class = createLambdaClosureType(Intro.Range, KnownDependent);
-  
   // Determine the signature of the call operator.
   TypeSourceInfo *MethodTyInfo;
   bool ExplicitParams = true;
   bool ExplicitResultType = true;
   bool ContainsUnexpandedParameterPack = false;
   SourceLocation EndLoc;
-  llvm::ArrayRef<ParmVarDecl *> Params;
+  llvm::SmallVector<ParmVarDecl *, 8> Params;
   if (ParamInfo.getNumTypeObjects() == 0) {
     // C++11 [expr.prim.lambda]p4:
     //   If a lambda-expression does not include a lambda-declarator, it is as 
@@ -410,17 +409,25 @@ void Sema::ActOnStartOfLambdaDefinition(LambdaIntroducer &Intro,
     ExplicitResultType
       = MethodTyInfo->getType()->getAs<FunctionType>()->getResultType() 
                                                         != Context.DependentTy;
-    
-    TypeLoc TL = MethodTyInfo->getTypeLoc();
-    FunctionProtoTypeLoc Proto = cast<FunctionProtoTypeLoc>(TL);
-    Params = llvm::ArrayRef<ParmVarDecl *>(Proto.getParmArray(), 
-                                           Proto.getNumArgs());
+
+    if (FTI.NumArgs == 1 && !FTI.isVariadic && FTI.ArgInfo[0].Ident == 0 &&
+        cast<ParmVarDecl>(FTI.ArgInfo[0].Param)->getType()->isVoidType()) {
+      // Empty arg list, don't push any params.
+      checkVoidParamDecl(cast<ParmVarDecl>(FTI.ArgInfo[0].Param));
+    } else {
+      Params.reserve(FTI.NumArgs);
+      for (unsigned i = 0, e = FTI.NumArgs; i != e; ++i)
+        Params.push_back(cast<ParmVarDecl>(FTI.ArgInfo[i].Param));
+    }
 
     // Check for unexpanded parameter packs in the method type.
     if (MethodTyInfo->getType()->containsUnexpandedParameterPack())
       ContainsUnexpandedParameterPack = true;
   }
-  
+
+  CXXRecordDecl *Class = createLambdaClosureType(Intro.Range, MethodTyInfo,
+                                                 KnownDependent);
+
   CXXMethodDecl *Method = startLambdaDefinition(Class, Intro.Range,
                                                 MethodTyInfo, EndLoc, Params);
   
@@ -528,6 +535,10 @@ void Sema::ActOnStartOfLambdaDefinition(LambdaIntroducer &Intro,
       continue;
     }
 
+    // Ignore invalid decls; they'll just confuse the code later.
+    if (Var->isInvalidDecl())
+      continue;
+
     if (!Var->hasLocalStorage()) {
       Diag(C->Loc, diag::err_capture_non_automatic_variable) << C->Id;
       Diag(Var->getLocation(), diag::note_previous_decl) << C->Id;
diff --git a/lib/Sema/SemaLookup.cpp b/lib/Sema/SemaLookup.cpp
index dad196b..f6987e7 100644
--- a/lib/Sema/SemaLookup.cpp
+++ b/lib/Sema/SemaLookup.cpp
@@ -707,7 +707,7 @@ static bool LookupDirect(Sema &S, LookupResult &R, const DeclContext *DC) {
     // result), perform template argument deduction and place the
     // specialization into the result set. We do this to avoid forcing all
     // callers to perform special deduction for conversion functions.
-    TemplateDeductionInfo Info(R.getSema().Context, R.getNameLoc());
+    TemplateDeductionInfo Info(R.getNameLoc());
     FunctionDecl *Specialization = 0;
 
     const FunctionProtoType *ConvProto
@@ -1725,15 +1725,17 @@ bool Sema::DiagnoseAmbiguousLookup(LookupResult &Result) {
 
 namespace {
   struct AssociatedLookup {
-    AssociatedLookup(Sema &S,
+    AssociatedLookup(Sema &S, SourceLocation InstantiationLoc,
                      Sema::AssociatedNamespaceSet &Namespaces,
                      Sema::AssociatedClassSet &Classes)
-      : S(S), Namespaces(Namespaces), Classes(Classes) {
+      : S(S), Namespaces(Namespaces), Classes(Classes),
+        InstantiationLoc(InstantiationLoc) {
     }
 
     Sema &S;
     Sema::AssociatedNamespaceSet &Namespaces;
     Sema::AssociatedClassSet &Classes;
+    SourceLocation InstantiationLoc;
   };
 }
 
@@ -1796,6 +1798,7 @@ addAssociatedClassesAndNamespaces(AssociatedLookup &Result,
     case TemplateArgument::Declaration:
     case TemplateArgument::Integral:
     case TemplateArgument::Expression:
+    case TemplateArgument::NullPtr:
       // [Note: non-type template arguments do not contribute to the set of
       //  associated namespaces. ]
       break;
@@ -1864,8 +1867,10 @@ addAssociatedClassesAndNamespaces(AssociatedLookup &Result,
 
   // Only recurse into base classes for complete types.
   if (!Class->hasDefinition()) {
-    // FIXME: we might need to instantiate templates here
-    return;
+    QualType type = Result.S.Context.getTypeDeclType(Class);
+    if (Result.S.RequireCompleteType(Result.InstantiationLoc, type,
+                                     /*no diagnostic*/ 0))
+      return;
   }
 
   // Add direct and indirect base classes along with their associated
@@ -2069,13 +2074,15 @@ addAssociatedClassesAndNamespaces(AssociatedLookup &Result, QualType Ty) {
 /// namespaces searched by argument-dependent lookup
 /// (C++ [basic.lookup.argdep]) for a given set of arguments.
 void
-Sema::FindAssociatedClassesAndNamespaces(llvm::ArrayRef<Expr *> Args,
+Sema::FindAssociatedClassesAndNamespaces(SourceLocation InstantiationLoc,
+                                         llvm::ArrayRef<Expr *> Args,
                                  AssociatedNamespaceSet &AssociatedNamespaces,
                                  AssociatedClassSet &AssociatedClasses) {
   AssociatedNamespaces.clear();
   AssociatedClasses.clear();
 
-  AssociatedLookup Result(*this, AssociatedNamespaces, AssociatedClasses);
+  AssociatedLookup Result(*this, InstantiationLoc,
+                          AssociatedNamespaces, AssociatedClasses);
 
   // C++ [basic.lookup.koenig]p2:
   //   For each argument type T in the function call, there is a set
@@ -2642,17 +2649,14 @@ void ADLResult::insert(NamedDecl *New) {
 void Sema::ArgumentDependentLookup(DeclarationName Name, bool Operator,
                                    SourceLocation Loc,
                                    llvm::ArrayRef<Expr *> Args,
-                                   ADLResult &Result,
-                                   bool StdNamespaceIsAssociated) {
+                                   ADLResult &Result) {
   // Find all of the associated namespaces and classes based on the
   // arguments we have.
   AssociatedNamespaceSet AssociatedNamespaces;
   AssociatedClassSet AssociatedClasses;
-  FindAssociatedClassesAndNamespaces(Args,
+  FindAssociatedClassesAndNamespaces(Loc, Args,
                                      AssociatedNamespaces,
                                      AssociatedClasses);
-  if (StdNamespaceIsAssociated && StdNamespace)
-    AssociatedNamespaces.insert(getStdNamespace());
 
   QualType T1, T2;
   if (Operator) {
@@ -2661,13 +2665,6 @@ void Sema::ArgumentDependentLookup(DeclarationName Name, bool Operator,
       T2 = Args[1]->getType();
   }
 
-  // Try to complete all associated classes, in case they contain a
-  // declaration of a friend function.
-  for (AssociatedClassSet::iterator C = AssociatedClasses.begin(),
-                                    CEnd = AssociatedClasses.end();
-       C != CEnd; ++C)
-    RequireCompleteType(Loc, Context.getRecordType(*C), 0);
-
   // C++ [basic.lookup.argdep]p3:
   //   Let X be the lookup set produced by unqualified lookup (3.4.1)
   //   and let Y be the lookup set produced by argument dependent
@@ -4056,7 +4053,9 @@ TypoCorrection Sema::CorrectTypo(const DeclarationNameInfo &TypoName,
     if (IsUnqualifiedLookup)
       UnqualifiedTyposCorrected[Typo] = Result;
 
-    return Result;
+    TypoCorrection TC = Result;
+    TC.setCorrectionRange(SS, TypoName);
+    return TC;
   }
   else if (BestResults.size() > 1
            // Ugly hack equivalent to CTC == CTC_ObjCMessageReceiver;
@@ -4076,7 +4075,9 @@ TypoCorrection Sema::CorrectTypo(const DeclarationNameInfo &TypoName,
     if (IsUnqualifiedLookup)
       UnqualifiedTyposCorrected[Typo] = BestResults["super"].front();
 
-    return BestResults["super"].front();
+    TypoCorrection TC = BestResults["super"].front();
+    TC.setCorrectionRange(SS, TypoName);
+    return TC;
   }
 
   // If this was an unqualified lookup and we believe the callback object did
diff --git a/lib/Sema/SemaObjCProperty.cpp b/lib/Sema/SemaObjCProperty.cpp
index 27deab2..8d70860 100644
--- a/lib/Sema/SemaObjCProperty.cpp
+++ b/lib/Sema/SemaObjCProperty.cpp
@@ -22,6 +22,7 @@
 #include "clang/Basic/SourceManager.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/SmallString.h"
+#include "clang/Lex/Preprocessor.h"
 
 using namespace clang;
 
@@ -43,7 +44,7 @@ static Qualifiers::ObjCLifetime getImpliedARCOwnership(
   if (attrs & (ObjCPropertyDecl::OBJC_PR_retain |
                ObjCPropertyDecl::OBJC_PR_strong |
                ObjCPropertyDecl::OBJC_PR_copy)) {
-    return type->getObjCARCImplicitLifetime();
+    return Qualifiers::OCL_Strong;
   } else if (attrs & ObjCPropertyDecl::OBJC_PR_weak) {
     return Qualifiers::OCL_Weak;
   } else if (attrs & ObjCPropertyDecl::OBJC_PR_unsafe_unretained) {
@@ -102,6 +103,15 @@ static void checkARCPropertyDecl(Sema &S, ObjCPropertyDecl *property) {
     << propertyLifetime;
 }
 
+static unsigned deduceWeakPropertyFromType(Sema &S, QualType T) {
+  if ((S.getLangOpts().getGC() != LangOptions::NonGC && 
+       T.isObjCGCWeak()) ||
+      (S.getLangOpts().ObjCAutoRefCount &&
+       T.getObjCLifetime() == Qualifiers::OCL_Weak))
+    return ObjCDeclSpec::DQ_PR_weak;
+  return 0;
+}
+
 Decl *Sema::ActOnProperty(Scope *S, SourceLocation AtLoc,
                           SourceLocation LParenLoc,
                           FieldDeclarator &FD,
@@ -114,12 +124,8 @@ Decl *Sema::ActOnProperty(Scope *S, SourceLocation AtLoc,
   unsigned Attributes = ODS.getPropertyAttributes();
   TypeSourceInfo *TSI = GetTypeForDeclarator(FD.D, S);
   QualType T = TSI->getType();
-  if ((getLangOpts().getGC() != LangOptions::NonGC && 
-       T.isObjCGCWeak()) ||
-      (getLangOpts().ObjCAutoRefCount &&
-       T.getObjCLifetime() == Qualifiers::OCL_Weak))
-    Attributes |= ObjCDeclSpec::DQ_PR_weak;
-
+  Attributes |= deduceWeakPropertyFromType(*this, T);
+  
   bool isReadWrite = ((Attributes & ObjCDeclSpec::DQ_PR_readwrite) ||
                       // default is readwrite!
                       !(Attributes & ObjCDeclSpec::DQ_PR_readonly));
@@ -236,6 +242,15 @@ static bool LocPropertyAttribute( ASTContext &Context, const char *attrName,
   
 }
 
+static unsigned getOwnershipRule(unsigned attr) {
+  return attr & (ObjCPropertyDecl::OBJC_PR_assign |
+                 ObjCPropertyDecl::OBJC_PR_retain |
+                 ObjCPropertyDecl::OBJC_PR_copy   |
+                 ObjCPropertyDecl::OBJC_PR_weak   |
+                 ObjCPropertyDecl::OBJC_PR_strong |
+                 ObjCPropertyDecl::OBJC_PR_unsafe_unretained);
+}
+
 Decl *
 Sema::HandlePropertyInClassExtension(Scope *S,
                                      SourceLocation AtLoc,
@@ -335,6 +350,7 @@ Sema::HandlePropertyInClassExtension(Scope *S,
       Diag(AtLoc, 
           diag::err_type_mismatch_continuation_class) << PDecl->getType();
       Diag(PIDecl->getLocation(), diag::note_property_declare);
+      return 0;
     }
   }
     
@@ -342,13 +358,11 @@ Sema::HandlePropertyInClassExtension(Scope *S,
   // with continuation class's readwrite property attribute!
   unsigned PIkind = PIDecl->getPropertyAttributesAsWritten();
   if (isReadWrite && (PIkind & ObjCPropertyDecl::OBJC_PR_readonly)) {
-    unsigned retainCopyNonatomic =
-    (ObjCPropertyDecl::OBJC_PR_retain |
-     ObjCPropertyDecl::OBJC_PR_strong |
-     ObjCPropertyDecl::OBJC_PR_copy |
-     ObjCPropertyDecl::OBJC_PR_nonatomic);
-    if ((Attributes & retainCopyNonatomic) !=
-        (PIkind & retainCopyNonatomic)) {
+    PIkind |= deduceWeakPropertyFromType(*this, PIDecl->getType());
+    unsigned ClassExtensionMemoryModel = getOwnershipRule(Attributes);
+    unsigned PrimaryClassMemoryModel = getOwnershipRule(PIkind);
+    if (PrimaryClassMemoryModel && ClassExtensionMemoryModel &&
+        (PrimaryClassMemoryModel != ClassExtensionMemoryModel)) {
       Diag(AtLoc, diag::warn_property_attr_mismatch);
       Diag(PIDecl->getLocation(), diag::note_property_declare);
     }
@@ -397,6 +411,7 @@ Sema::HandlePropertyInClassExtension(Scope *S,
     Diag(AtLoc, diag)
       << CCPrimary->getDeclName();
     Diag(PIDecl->getLocation(), diag::note_property_declare);
+    return 0;
   }
   *isOverridingProperty = true;
   // Make sure setter decl is synthesized, and added to primary class's list.
@@ -405,7 +420,7 @@ Sema::HandlePropertyInClassExtension(Scope *S,
   PDecl->setSetterMethodDecl(PIDecl->getSetterMethodDecl());
   if (ASTMutationListener *L = Context.getASTMutationListener())
     L->AddedObjCPropertyInClassExtension(PDecl, PIDecl, CDecl);
-  return 0;
+  return PDecl;
 }
 
 ObjCPropertyDecl *Sema::CreatePropertyDecl(Scope *S,
@@ -543,6 +558,23 @@ static void checkARCPropertyImpl(Sema &S, SourceLocation propertyImplLoc,
       ivarLifetime == Qualifiers::OCL_Autoreleasing)
     return;
 
+  // If the ivar is private, and it's implicitly __unsafe_unretained
+  // becaues of its type, then pretend it was actually implicitly
+  // __strong.  This is only sound because we're processing the
+  // property implementation before parsing any method bodies.
+  if (ivarLifetime == Qualifiers::OCL_ExplicitNone &&
+      propertyLifetime == Qualifiers::OCL_Strong &&
+      ivar->getAccessControl() == ObjCIvarDecl::Private) {
+    SplitQualType split = ivarType.split();
+    if (split.Quals.hasObjCLifetime()) {
+      assert(ivarType->isObjCARCImplicitlyUnretainedType());
+      split.Quals.setObjCLifetime(Qualifiers::OCL_Strong);
+      ivarType = S.Context.getQualifiedType(split);
+      ivar->setType(ivarType);
+      return;
+    }
+  }
+
   switch (propertyLifetime) {
   case Qualifiers::OCL_Strong:
     S.Diag(propertyImplLoc, diag::err_arc_strong_property_ownership)
@@ -632,7 +664,13 @@ DiagnoseClassAndClassExtPropertyMismatch(Sema &S, ObjCInterfaceDecl *ClassDecl,
       // property.
       if (Attributes & ObjCDeclSpec::DQ_PR_readonly) {
         if (!classExtPropertyAttr ||
-            (classExtPropertyAttr & ObjCDeclSpec::DQ_PR_readwrite))
+            (classExtPropertyAttr & 
+              (ObjCDeclSpec::DQ_PR_readwrite|
+               ObjCDeclSpec::DQ_PR_assign |
+               ObjCDeclSpec::DQ_PR_unsafe_unretained |
+               ObjCDeclSpec::DQ_PR_copy |
+               ObjCDeclSpec::DQ_PR_retain |
+               ObjCDeclSpec::DQ_PR_strong)))
           continue;
         warn = true;
         break;
@@ -857,13 +895,15 @@ Decl *Sema::ActOnPropertyImplDecl(Scope *S,
           if (lifetime == Qualifiers::OCL_Weak) {
             bool err = false;
             if (const ObjCObjectPointerType *ObjT =
-                PropertyIvarType->getAs<ObjCObjectPointerType>())
-              if (ObjT->getInterfaceDecl()->isArcWeakrefUnavailable()) {
+                PropertyIvarType->getAs<ObjCObjectPointerType>()) {
+              const ObjCInterfaceDecl *ObjI = ObjT->getInterfaceDecl();
+              if (ObjI && ObjI->isArcWeakrefUnavailable()) {
                 Diag(PropertyDiagLoc, diag::err_arc_weak_unavailable_property);
                 Diag(property->getLocation(), diag::note_property_declare);
                 err = true;
               }
-            if (!err && !getLangOpts().ObjCRuntimeHasWeak) {
+            }
+            if (!err && !getLangOpts().ObjCARCWeak) {
               Diag(PropertyDiagLoc, diag::err_arc_weak_no_runtime);
               Diag(property->getLocation(), diag::note_property_declare);
             }
@@ -891,7 +931,6 @@ Decl *Sema::ActOnPropertyImplDecl(Scope *S,
         Ivar->setInvalidDecl();
       ClassImpDecl->addDecl(Ivar);
       IDecl->makeDeclVisibleInContext(Ivar);
-      property->setPropertyIvarDecl(Ivar);
 
       if (getLangOpts().ObjCRuntime.isFragile())
         Diag(PropertyDiagLoc, diag::error_missing_property_ivar_decl)
@@ -907,14 +946,15 @@ Decl *Sema::ActOnPropertyImplDecl(Scope *S,
       << Ivar << Ivar->getName();
       // Note! I deliberately want it to fall thru so more errors are caught.
     }
+    property->setPropertyIvarDecl(Ivar);
+
     QualType IvarType = Context.getCanonicalType(Ivar->getType());
 
     // Check that type of property and its ivar are type compatible.
     if (!Context.hasSameType(PropertyIvarType, IvarType)) {
-      compat = false;
       if (isa<ObjCObjectPointerType>(PropertyIvarType) 
           && isa<ObjCObjectPointerType>(IvarType))
-        compat = 
+        compat =
           Context.canAssignObjCInterfaces(
                                   PropertyIvarType->getAs<ObjCObjectPointerType>(),
                                   IvarType->getAs<ObjCObjectPointerType>());
@@ -988,19 +1028,21 @@ Decl *Sema::ActOnPropertyImplDecl(Scope *S,
       // For Objective-C++, need to synthesize the AST for the IVAR object to be
       // returned by the getter as it must conform to C++'s copy-return rules.
       // FIXME. Eventually we want to do this for Objective-C as well.
+      SynthesizedFunctionScope Scope(*this, getterMethod);
       ImplicitParamDecl *SelfDecl = getterMethod->getSelfDecl();
       DeclRefExpr *SelfExpr = 
         new (Context) DeclRefExpr(SelfDecl, false, SelfDecl->getType(),
-                                  VK_RValue, SourceLocation());
+                                  VK_RValue, PropertyDiagLoc);
+      MarkDeclRefReferenced(SelfExpr);
       Expr *IvarRefExpr =
-        new (Context) ObjCIvarRefExpr(Ivar, Ivar->getType(), AtLoc,
+        new (Context) ObjCIvarRefExpr(Ivar, Ivar->getType(), PropertyDiagLoc,
                                       SelfExpr, true, true);
       ExprResult Res = 
         PerformCopyInitialization(InitializedEntity::InitializeResult(
-                                    SourceLocation(),
+                                    PropertyDiagLoc,
                                     getterMethod->getResultType(),
                                     /*NRVO=*/false),
-                                  SourceLocation(),
+                                  PropertyDiagLoc,
                                   Owned(IvarRefExpr));
       if (!Res.isInvalid()) {
         Expr *ResExpr = Res.takeAs<Expr>();
@@ -1021,19 +1063,22 @@ Decl *Sema::ActOnPropertyImplDecl(Scope *S,
     if (getLangOpts().CPlusPlus && Synthesize && !CompleteTypeErr &&
         Ivar->getType()->isRecordType()) {
       // FIXME. Eventually we want to do this for Objective-C as well.
+      SynthesizedFunctionScope Scope(*this, setterMethod);
       ImplicitParamDecl *SelfDecl = setterMethod->getSelfDecl();
       DeclRefExpr *SelfExpr = 
         new (Context) DeclRefExpr(SelfDecl, false, SelfDecl->getType(),
-                                  VK_RValue, SourceLocation());
+                                  VK_RValue, PropertyDiagLoc);
+      MarkDeclRefReferenced(SelfExpr);
       Expr *lhs =
-        new (Context) ObjCIvarRefExpr(Ivar, Ivar->getType(), AtLoc,
+        new (Context) ObjCIvarRefExpr(Ivar, Ivar->getType(), PropertyDiagLoc,
                                       SelfExpr, true, true);
       ObjCMethodDecl::param_iterator P = setterMethod->param_begin();
       ParmVarDecl *Param = (*P);
       QualType T = Param->getType().getNonReferenceType();
-      Expr *rhs = new (Context) DeclRefExpr(Param, false, T,
-                                            VK_LValue, SourceLocation());
-      ExprResult Res = BuildBinOp(S, lhs->getLocEnd(), 
+      DeclRefExpr *rhs = new (Context) DeclRefExpr(Param, false, T,
+                                                   VK_LValue, PropertyDiagLoc);
+      MarkDeclRefReferenced(rhs);
+      ExprResult Res = BuildBinOp(S, PropertyDiagLoc, 
                                   BO_Assign, lhs, rhs);
       if (property->getPropertyAttributes() & 
           ObjCPropertyDecl::OBJC_PR_atomic) {
@@ -1043,7 +1088,7 @@ Decl *Sema::ActOnPropertyImplDecl(Scope *S,
           if (const FunctionDecl *FuncDecl = CXXCE->getDirectCallee())
             if (!FuncDecl->isTrivial())
               if (property->getType()->isReferenceType()) {
-                Diag(PropertyLoc, 
+                Diag(PropertyDiagLoc, 
                      diag::err_atomic_property_nontrivial_assign_op)
                     << property->getType();
                 Diag(FuncDecl->getLocStart(), 
@@ -1395,8 +1440,8 @@ bool Sema::isPropertyReadonly(ObjCPropertyDecl *PDecl,
 /// CollectImmediateProperties - This routine collects all properties in
 /// the class and its conforming protocols; but not those it its super class.
 void Sema::CollectImmediateProperties(ObjCContainerDecl *CDecl,
-            llvm::DenseMap<IdentifierInfo *, ObjCPropertyDecl*>& PropMap,
-            llvm::DenseMap<IdentifierInfo *, ObjCPropertyDecl*>& SuperPropMap) {
+            ObjCContainerDecl::PropertyMap &PropMap,
+            ObjCContainerDecl::PropertyMap &SuperPropMap) {
   if (ObjCInterfaceDecl *IDecl = dyn_cast<ObjCInterfaceDecl>(CDecl)) {
     for (ObjCContainerDecl::prop_iterator P = IDecl->prop_begin(),
          E = IDecl->prop_end(); P != E; ++P) {
@@ -1442,118 +1487,38 @@ void Sema::CollectImmediateProperties(ObjCContainerDecl *CDecl,
   }
 }
 
-/// CollectClassPropertyImplementations - This routine collects list of
-/// properties to be implemented in the class. This includes, class's
-/// and its conforming protocols' properties.
-static void CollectClassPropertyImplementations(ObjCContainerDecl *CDecl,
-                llvm::DenseMap<IdentifierInfo *, ObjCPropertyDecl*>& PropMap) {
-  if (ObjCInterfaceDecl *IDecl = dyn_cast<ObjCInterfaceDecl>(CDecl)) {
-    for (ObjCContainerDecl::prop_iterator P = IDecl->prop_begin(),
-         E = IDecl->prop_end(); P != E; ++P) {
-      ObjCPropertyDecl *Prop = *P;
-      PropMap[Prop->getIdentifier()] = Prop;
-    }
-    for (ObjCInterfaceDecl::all_protocol_iterator
-         PI = IDecl->all_referenced_protocol_begin(),
-         E = IDecl->all_referenced_protocol_end(); PI != E; ++PI)
-      CollectClassPropertyImplementations((*PI), PropMap);
-  }
-  else if (ObjCProtocolDecl *PDecl = dyn_cast<ObjCProtocolDecl>(CDecl)) {
-    for (ObjCProtocolDecl::prop_iterator P = PDecl->prop_begin(),
-         E = PDecl->prop_end(); P != E; ++P) {
-      ObjCPropertyDecl *Prop = *P;
-      if (!PropMap.count(Prop->getIdentifier()))
-        PropMap[Prop->getIdentifier()] = Prop;
-    }
-    // scan through protocol's protocols.
-    for (ObjCProtocolDecl::protocol_iterator PI = PDecl->protocol_begin(),
-         E = PDecl->protocol_end(); PI != E; ++PI)
-      CollectClassPropertyImplementations((*PI), PropMap);
-  }
-}
-
 /// CollectSuperClassPropertyImplementations - This routine collects list of
 /// properties to be implemented in super class(s) and also coming from their
 /// conforming protocols.
 static void CollectSuperClassPropertyImplementations(ObjCInterfaceDecl *CDecl,
-                llvm::DenseMap<IdentifierInfo *, ObjCPropertyDecl*>& PropMap) {
+                                    ObjCInterfaceDecl::PropertyMap &PropMap) {
   if (ObjCInterfaceDecl *SDecl = CDecl->getSuperClass()) {
     while (SDecl) {
-      CollectClassPropertyImplementations(SDecl, PropMap);
+      SDecl->collectPropertiesToImplement(PropMap);
       SDecl = SDecl->getSuperClass();
     }
   }
 }
 
-/// LookupPropertyDecl - Looks up a property in the current class and all
-/// its protocols.
-ObjCPropertyDecl *Sema::LookupPropertyDecl(const ObjCContainerDecl *CDecl,
-                                     IdentifierInfo *II) {
-  if (const ObjCInterfaceDecl *IDecl =
-        dyn_cast<ObjCInterfaceDecl>(CDecl)) {
-    for (ObjCContainerDecl::prop_iterator P = IDecl->prop_begin(),
-         E = IDecl->prop_end(); P != E; ++P) {
-      ObjCPropertyDecl *Prop = *P;
-      if (Prop->getIdentifier() == II)
-        return Prop;
-    }
-    // scan through class's protocols.
-    for (ObjCInterfaceDecl::all_protocol_iterator
-         PI = IDecl->all_referenced_protocol_begin(),
-         E = IDecl->all_referenced_protocol_end(); PI != E; ++PI) {
-      ObjCPropertyDecl *Prop = LookupPropertyDecl((*PI), II);
-      if (Prop)
-        return Prop;
-    }
-  }
-  else if (const ObjCProtocolDecl *PDecl =
-            dyn_cast<ObjCProtocolDecl>(CDecl)) {
-    for (ObjCProtocolDecl::prop_iterator P = PDecl->prop_begin(),
-         E = PDecl->prop_end(); P != E; ++P) {
-      ObjCPropertyDecl *Prop = *P;
-      if (Prop->getIdentifier() == II)
-        return Prop;
-    }
-    // scan through protocol's protocols.
-    for (ObjCProtocolDecl::protocol_iterator PI = PDecl->protocol_begin(),
-         E = PDecl->protocol_end(); PI != E; ++PI) {
-      ObjCPropertyDecl *Prop = LookupPropertyDecl((*PI), II);
-      if (Prop)
-        return Prop;
-    }
-  }
-  return 0;
-}
-
-static IdentifierInfo * getDefaultSynthIvarName(ObjCPropertyDecl *Prop,
-                                                ASTContext &Ctx) {
-  SmallString<128> ivarName;
-  {
-    llvm::raw_svector_ostream os(ivarName);
-    os << '_' << Prop->getIdentifier()->getName();
-  }
-  return &Ctx.Idents.get(ivarName.str());
-}
-
 /// \brief Default synthesizes all properties which must be synthesized
 /// in class's \@implementation.
 void Sema::DefaultSynthesizeProperties(Scope *S, ObjCImplDecl* IMPDecl,
                                        ObjCInterfaceDecl *IDecl) {
   
-  llvm::DenseMap<IdentifierInfo *, ObjCPropertyDecl*> PropMap;
-  CollectClassPropertyImplementations(IDecl, PropMap);
+  ObjCInterfaceDecl::PropertyMap PropMap;
+  IDecl->collectPropertiesToImplement(PropMap);
   if (PropMap.empty())
     return;
-  llvm::DenseMap<IdentifierInfo *, ObjCPropertyDecl*> SuperPropMap;
+  ObjCInterfaceDecl::PropertyMap SuperPropMap;
   CollectSuperClassPropertyImplementations(IDecl, SuperPropMap);
   
-  for (llvm::DenseMap<IdentifierInfo *, ObjCPropertyDecl*>::iterator
+  for (ObjCInterfaceDecl::PropertyMap::iterator
        P = PropMap.begin(), E = PropMap.end(); P != E; ++P) {
     ObjCPropertyDecl *Prop = P->second;
     // If property to be implemented in the super class, ignore.
     if (SuperPropMap[Prop->getIdentifier()])
       continue;
-    // Is there a matching propery synthesize/dynamic?
+    // Is there a matching property synthesize/dynamic?
     if (Prop->isInvalidDecl() ||
         Prop->getPropertyImplementation() == ObjCPropertyDecl::Optional ||
         IMPDecl->FindPropertyImplIvarDecl(Prop->getIdentifier()))
@@ -1583,7 +1548,7 @@ void Sema::DefaultSynthesizeProperties(Scope *S, ObjCImplDecl* IMPDecl,
       ActOnPropertyImplDecl(S, SourceLocation(), SourceLocation(),
                             true,
                             /* property = */ Prop->getIdentifier(),
-                            /* ivar = */ getDefaultSynthIvarName(Prop, Context),
+                            /* ivar = */ Prop->getDefaultSynthIvarName(Context),
                             Prop->getLocation()));
     if (PIDecl) {
       Diag(Prop->getLocation(), diag::warn_missing_explicit_synthesis);
@@ -1606,11 +1571,11 @@ void Sema::DefaultSynthesizeProperties(Scope *S, Decl *D) {
 void Sema::DiagnoseUnimplementedProperties(Scope *S, ObjCImplDecl* IMPDecl,
                                       ObjCContainerDecl *CDecl,
                                       const SelectorSet &InsMap) {
-  llvm::DenseMap<IdentifierInfo *, ObjCPropertyDecl*> SuperPropMap;
+  ObjCContainerDecl::PropertyMap SuperPropMap;
   if (ObjCInterfaceDecl *IDecl = dyn_cast<ObjCInterfaceDecl>(CDecl))
     CollectSuperClassPropertyImplementations(IDecl, SuperPropMap);
   
-  llvm::DenseMap<IdentifierInfo *, ObjCPropertyDecl*> PropMap;
+  ObjCContainerDecl::PropertyMap PropMap;
   CollectImmediateProperties(CDecl, PropMap, SuperPropMap);
   if (PropMap.empty())
     return;
@@ -1621,7 +1586,7 @@ void Sema::DiagnoseUnimplementedProperties(Scope *S, ObjCImplDecl* IMPDecl,
        EI = IMPDecl->propimpl_end(); I != EI; ++I)
     PropImplMap.insert(I->getPropertyDecl());
 
-  for (llvm::DenseMap<IdentifierInfo *, ObjCPropertyDecl*>::iterator
+  for (ObjCContainerDecl::PropertyMap::iterator
        P = PropMap.begin(), E = PropMap.end(); P != E; ++P) {
     ObjCPropertyDecl *Prop = P->second;
     // Is there a matching propery synthesize/dynamic?
@@ -1847,7 +1812,7 @@ void Sema::ProcessPropertyDecl(ObjCPropertyDecl *property,
     GetterMethod = ObjCMethodDecl::Create(Context, Loc, Loc,
                              property->getGetterName(),
                              property->getType(), 0, CD, /*isInstance=*/true,
-                             /*isVariadic=*/false, /*isSynthesized=*/true,
+                             /*isVariadic=*/false, /*isPropertyAccessor=*/true,
                              /*isImplicitlyDeclared=*/true, /*isDefined=*/false,
                              (property->getPropertyImplementation() ==
                               ObjCPropertyDecl::Optional) ?
@@ -1867,7 +1832,7 @@ void Sema::ProcessPropertyDecl(ObjCPropertyDecl *property,
   } else
     // A user declared getter will be synthesize when @synthesize of
     // the property with the same name is seen in the @implementation
-    GetterMethod->setSynthesized(true);
+    GetterMethod->setPropertyAccessor(true);
   property->setGetterMethodDecl(GetterMethod);
 
   // Skip setter if property is read-only.
@@ -1885,7 +1850,7 @@ void Sema::ProcessPropertyDecl(ObjCPropertyDecl *property,
         ObjCMethodDecl::Create(Context, Loc, Loc,
                                property->getSetterName(), Context.VoidTy, 0,
                                CD, /*isInstance=*/true, /*isVariadic=*/false,
-                               /*isSynthesized=*/true,
+                               /*isPropertyAccessor=*/true,
                                /*isImplicitlyDeclared=*/true,
                                /*isDefined=*/false,
                                (property->getPropertyImplementation() ==
@@ -1916,7 +1881,7 @@ void Sema::ProcessPropertyDecl(ObjCPropertyDecl *property,
     } else
       // A user declared setter will be synthesize when @synthesize of
       // the property with the same name is seen in the @implementation
-      SetterMethod->setSynthesized(true);
+      SetterMethod->setPropertyAccessor(true);
     property->setSetterMethodDecl(SetterMethod);
   }
   // Add any synthesized methods to the global pool. This allows us to
@@ -2121,7 +2086,9 @@ void Sema::CheckObjCPropertyAttributes(Decl *PDecl,
         // issue any warning.
         if (isAnyClassTy && getLangOpts().getGC() == LangOptions::NonGC)
           ;
-        else {
+        else if (propertyInPrimaryClass) {
+          // Don't issue warning on property with no life time in class 
+          // extension as it is inherited from property in primary class.
           // Skip this warning in gc-only mode.
           if (getLangOpts().getGC() != LangOptions::GCOnly)
             Diag(Loc, diag::warn_objc_property_no_assignment_attribute);
diff --git a/lib/Sema/SemaOverload.cpp b/lib/Sema/SemaOverload.cpp
index 9382f7d..9111878 100644
--- a/lib/Sema/SemaOverload.cpp
+++ b/lib/Sema/SemaOverload.cpp
@@ -49,7 +49,7 @@ CreateFunctionRefExpr(Sema &S, FunctionDecl *Fn, bool HadMultipleCandidates,
   E = S.DefaultFunctionArrayConversion(E.take());
   if (E.isInvalid())
     return ExprError();
-  return move(E);
+  return E;
 }
 
 static bool IsStandardConversion(Sema &S, Expr* From, QualType ToType,
@@ -555,6 +555,7 @@ static MakeDeductionFailureInfo(ASTContext &Context,
   Result.Data = 0;
   switch (TDK) {
   case Sema::TDK_Success:
+  case Sema::TDK_Invalid:
   case Sema::TDK_InstantiationDepth:
   case Sema::TDK_TooManyArguments:
   case Sema::TDK_TooFewArguments:
@@ -597,6 +598,7 @@ static MakeDeductionFailureInfo(ASTContext &Context,
 void OverloadCandidate::DeductionFailureInfo::Destroy() {
   switch (static_cast<Sema::TemplateDeductionResult>(Result)) {
   case Sema::TDK_Success:
+  case Sema::TDK_Invalid:
   case Sema::TDK_InstantiationDepth:
   case Sema::TDK_Incomplete:
   case Sema::TDK_TooManyArguments:
@@ -637,6 +639,7 @@ TemplateParameter
 OverloadCandidate::DeductionFailureInfo::getTemplateParameter() {
   switch (static_cast<Sema::TemplateDeductionResult>(Result)) {
   case Sema::TDK_Success:
+  case Sema::TDK_Invalid:
   case Sema::TDK_InstantiationDepth:
   case Sema::TDK_TooManyArguments:
   case Sema::TDK_TooFewArguments:
@@ -664,6 +667,7 @@ TemplateArgumentList *
 OverloadCandidate::DeductionFailureInfo::getTemplateArgumentList() {
   switch (static_cast<Sema::TemplateDeductionResult>(Result)) {
     case Sema::TDK_Success:
+    case Sema::TDK_Invalid:
     case Sema::TDK_InstantiationDepth:
     case Sema::TDK_TooManyArguments:
     case Sema::TDK_TooFewArguments:
@@ -688,6 +692,7 @@ OverloadCandidate::DeductionFailureInfo::getTemplateArgumentList() {
 const TemplateArgument *OverloadCandidate::DeductionFailureInfo::getFirstArg() {
   switch (static_cast<Sema::TemplateDeductionResult>(Result)) {
   case Sema::TDK_Success:
+  case Sema::TDK_Invalid:
   case Sema::TDK_InstantiationDepth:
   case Sema::TDK_Incomplete:
   case Sema::TDK_TooManyArguments:
@@ -713,6 +718,7 @@ const TemplateArgument *
 OverloadCandidate::DeductionFailureInfo::getSecondArg() {
   switch (static_cast<Sema::TemplateDeductionResult>(Result)) {
   case Sema::TDK_Success:
+  case Sema::TDK_Invalid:
   case Sema::TDK_InstantiationDepth:
   case Sema::TDK_Incomplete:
   case Sema::TDK_TooManyArguments:
@@ -734,13 +740,17 @@ OverloadCandidate::DeductionFailureInfo::getSecondArg() {
   return 0;
 }
 
-void OverloadCandidateSet::clear() {
+void OverloadCandidateSet::destroyCandidates() {
   for (iterator i = begin(), e = end(); i != e; ++i) {
     for (unsigned ii = 0, ie = i->NumConversions; ii != ie; ++ii)
       i->Conversions[ii].~ImplicitConversionSequence();
     if (!i->Viable && i->FailureKind == ovl_fail_bad_deduction)
       i->DeductionFailure.Destroy();
   }
+}
+
+void OverloadCandidateSet::clear() {
+  destroyCandidates();
   NumInlineSequences = 0;
   Candidates.clear();
   Functions.clear();
@@ -1668,7 +1678,7 @@ bool Sema::IsIntegralPromotion(Expr *From, QualType FromType, QualType ToType) {
     return To->getKind() == BuiltinType::UInt;
   }
 
-  // C++0x [conv.prom]p3:
+  // C++11 [conv.prom]p3:
   //   A prvalue of an unscoped enumeration type whose underlying type is not
   //   fixed (7.2) can be converted to an rvalue a prvalue of the first of the
   //   following types that can represent all the values of the enumeration
@@ -1680,12 +1690,26 @@ bool Sema::IsIntegralPromotion(Expr *From, QualType FromType, QualType ToType) {
   //   with lowest integer conversion rank (4.13) greater than the rank of long
   //   long in which all the values of the enumeration can be represented. If
   //   there are two such extended types, the signed one is chosen.
+  // C++11 [conv.prom]p4:
+  //   A prvalue of an unscoped enumeration type whose underlying type is fixed
+  //   can be converted to a prvalue of its underlying type. Moreover, if
+  //   integral promotion can be applied to its underlying type, a prvalue of an
+  //   unscoped enumeration type whose underlying type is fixed can also be
+  //   converted to a prvalue of the promoted underlying type.
   if (const EnumType *FromEnumType = FromType->getAs<EnumType>()) {
     // C++0x 7.2p9: Note that this implicit enum to int conversion is not
     // provided for a scoped enumeration.
     if (FromEnumType->getDecl()->isScoped())
       return false;
 
+    // We can perform an integral promotion to the underlying type of the enum,
+    // even if that's not the promoted type.
+    if (FromEnumType->getDecl()->isFixed()) {
+      QualType Underlying = FromEnumType->getDecl()->getIntegerType();
+      return Context.hasSameUnqualifiedType(Underlying, ToType) ||
+             IsIntegralPromotion(From, Underlying, ToType);
+    }
+
     // We have already pre-calculated the promotion type, so this is trivial.
     if (ToType->isIntegerType() &&
         !RequireCompleteType(From->getLocStart(), FromType, 0))
@@ -2899,8 +2923,6 @@ IsInitializerListConstructorConversion(Sema &S, Expr *From, QualType ToType,
   case OR_Success: {
     // Record the standard conversion we used and the conversion function.
     CXXConstructorDecl *Constructor = cast<CXXConstructorDecl>(Best->Function);
-    S.MarkFunctionReferenced(From->getLocStart(), Constructor);
-
     QualType ThisType = Constructor->getThisType(S.Context);
     // Initializer lists don't have conversions as such.
     User.Before.setAsIdentityConversion();
@@ -3081,8 +3103,6 @@ IsUserDefinedConversion(Sema &S, Expr *From, QualType ToType,
     // Record the standard conversion we used and the conversion function.
     if (CXXConstructorDecl *Constructor
           = dyn_cast<CXXConstructorDecl>(Best->Function)) {
-      S.MarkFunctionReferenced(From->getLocStart(), Constructor);
-
       // C++ [over.ics.user]p1:
       //   If the user-defined conversion is specified by a
       //   constructor (12.3.1), the initial standard conversion
@@ -3111,8 +3131,6 @@ IsUserDefinedConversion(Sema &S, Expr *From, QualType ToType,
     }
     if (CXXConversionDecl *Conversion
                  = dyn_cast<CXXConversionDecl>(Best->Function)) {
-      S.MarkFunctionReferenced(From->getLocStart(), Conversion);
-
       // C++ [over.ics.user]p1:
       //
       //   [...] If the user-defined conversion is specified by a
@@ -4025,8 +4043,6 @@ FindConversionForRefInit(Sema &S, ImplicitConversionSequence &ICS,
     if (!Best->FinalConversion.DirectBinding)
       return false;
 
-    if (Best->Function)
-      S.MarkFunctionReferenced(DeclLoc, Best->Function);
     ICS.setUserDefined();
     ICS.UserDefined.Before = Best->Conversions[0].Standard;
     ICS.UserDefined.After = Best->FinalConversion;
@@ -5531,7 +5547,7 @@ Sema::AddMethodTemplateCandidate(FunctionTemplateDecl *MethodTmpl,
   //   functions. In such a case, the candidate functions generated from each
   //   function template are combined with the set of non-template candidate
   //   functions.
-  TemplateDeductionInfo Info(Context, CandidateSet.getLocation());
+  TemplateDeductionInfo Info(CandidateSet.getLocation());
   FunctionDecl *Specialization = 0;
   if (TemplateDeductionResult Result
       = DeduceTemplateArguments(MethodTmpl, ExplicitTemplateArgs, Args,
@@ -5581,7 +5597,7 @@ Sema::AddTemplateOverloadCandidate(FunctionTemplateDecl *FunctionTemplate,
   //   functions. In such a case, the candidate functions generated from each
   //   function template are combined with the set of non-template candidate
   //   functions.
-  TemplateDeductionInfo Info(Context, CandidateSet.getLocation());
+  TemplateDeductionInfo Info(CandidateSet.getLocation());
   FunctionDecl *Specialization = 0;
   if (TemplateDeductionResult Result
         = DeduceTemplateArguments(FunctionTemplate, ExplicitTemplateArgs, Args,
@@ -5703,7 +5719,7 @@ Sema::AddConversionCandidate(CXXConversionDecl *Conversion,
   // there are 0 arguments (i.e., nothing is allocated using ASTContext's
   // allocator).
   QualType CallResultType = ConversionType.getNonLValueExprType(Context);
-  CallExpr Call(Context, &ConversionFn, 0, 0, CallResultType, VK,
+  CallExpr Call(Context, &ConversionFn, MultiExprArg(), CallResultType, VK,
                 From->getLocStart());
   ImplicitConversionSequence ICS =
     TryCopyInitialization(*this, &Call, ToType,
@@ -5765,7 +5781,7 @@ Sema::AddTemplateConversionCandidate(FunctionTemplateDecl *FunctionTemplate,
   if (!CandidateSet.isNewCandidate(FunctionTemplate))
     return;
 
-  TemplateDeductionInfo Info(Context, CandidateSet.getLocation());
+  TemplateDeductionInfo Info(CandidateSet.getLocation());
   CXXConversionDecl *Specialization = 0;
   if (TemplateDeductionResult Result
         = DeduceTemplateArguments(FunctionTemplate, ToType,
@@ -6770,17 +6786,16 @@ public:
   //        bool       operator==(T, T);
   //        bool       operator!=(T, T);
   void addRelationalPointerOrEnumeralOverloads() {
-    // C++ [over.built]p1:
-    //   If there is a user-written candidate with the same name and parameter
-    //   types as a built-in candidate operator function, the built-in operator
-    //   function is hidden and is not included in the set of candidate
-    //   functions.
+    // C++ [over.match.oper]p3:
+    //   [...]the built-in candidates include all of the candidate operator
+    //   functions defined in 13.6 that, compared to the given operator, [...]
+    //   do not have the same parameter-type-list as any non-template non-member
+    //   candidate.
     //
-    // The text is actually in a note, but if we don't implement it then we end
-    // up with ambiguities when the user provides an overloaded operator for
-    // an enumeration type. Note that only enumeration types have this problem,
-    // so we track which enumeration types we've seen operators for. Also, the
-    // only other overloaded operator with enumeration argumenst, operator=,
+    // Note that in practice, this only affects enumeration types because there
+    // aren't any built-in candidates of record type, and a user-defined operator
+    // must have an operand of record or enumeration type. Also, the only other
+    // overloaded operator with enumeration arguments, operator=,
     // cannot be overloaded for enumeration types, so this is the only place
     // where we must suppress candidates like this.
     llvm::DenseSet<std::pair<CanQualType, CanQualType> >
@@ -6795,6 +6810,9 @@ public:
           if (!C->Viable || !C->Function || C->Function->getNumParams() != 2)
             continue;
 
+          if (C->Function->isFunctionTemplateSpecialization())
+            continue;
+
           QualType FirstParamType =
             C->Function->getParamDecl(0)->getType().getUnqualifiedType();
           QualType SecondParamType =
@@ -7652,8 +7670,7 @@ Sema::AddArgumentDependentLookupCandidates(DeclarationName Name,
                                            llvm::ArrayRef<Expr *> Args,
                                  TemplateArgumentListInfo *ExplicitTemplateArgs,
                                            OverloadCandidateSet& CandidateSet,
-                                           bool PartialOverloading,
-                                           bool StdNamespaceIsAssociated) {
+                                           bool PartialOverloading) {
   ADLResult Fns;
 
   // FIXME: This approach for uniquing ADL results (and removing
@@ -7664,8 +7681,7 @@ Sema::AddArgumentDependentLookupCandidates(DeclarationName Name,
   // we supposed to consider on ADL candidates, anyway?
 
   // FIXME: Pass in the explicit template arguments?
-  ArgumentDependentLookup(Name, Operator, Loc, Args, Fns,
-                          StdNamespaceIsAssociated);
+  ArgumentDependentLookup(Name, Operator, Loc, Args, Fns);
 
   // Erase all of the candidates we already knew about.
   for (OverloadCandidateSet::iterator Cand = CandidateSet.begin(),
@@ -7976,10 +7992,20 @@ void ImplicitConversionSequence::DiagnoseAmbiguousConversion(
                                  const PartialDiagnostic &PDiag) const {
   S.Diag(CaretLoc, PDiag)
     << Ambiguous.getFromType() << Ambiguous.getToType();
-  for (AmbiguousConversionSequence::const_iterator
-         I = Ambiguous.begin(), E = Ambiguous.end(); I != E; ++I) {
+  // FIXME: The note limiting machinery is borrowed from
+  // OverloadCandidateSet::NoteCandidates; there's an opportunity for
+  // refactoring here.
+  const OverloadsShown ShowOverloads = S.Diags.getShowOverloads();
+  unsigned CandsShown = 0;
+  AmbiguousConversionSequence::const_iterator I, E;
+  for (I = Ambiguous.begin(), E = Ambiguous.end(); I != E; ++I) {
+    if (CandsShown >= 4 && ShowOverloads == Ovl_Best)
+      break;
+    ++CandsShown;
     S.NoteOverloadCandidate(*I);
   }
+  if (I != E)
+    S.Diag(SourceLocation(), diag::note_ovl_too_many_candidates) << int(E - I);
 }
 
 namespace {
@@ -8515,7 +8541,7 @@ void NoteSurrogateCandidate(Sema &S, OverloadCandidate *Cand) {
 }
 
 void NoteBuiltinOperatorCandidate(Sema &S,
-                                  const char *Opc,
+                                  StringRef Opc,
                                   SourceLocation OpLoc,
                                   OverloadCandidate *Cand) {
   assert(Cand->NumConversions <= 2 && "builtin operator is not binary");
@@ -8561,6 +8587,7 @@ RankDeductionFailure(const OverloadCandidate::DeductionFailureInfo &DFI) {
   case Sema::TDK_Success:
     llvm_unreachable("TDK_success while diagnosing bad deduction");
 
+  case Sema::TDK_Invalid:
   case Sema::TDK_Incomplete:
     return 1;
 
@@ -8783,7 +8810,7 @@ void CompleteNonViableCandidate(Sema &S, OverloadCandidate *Cand,
 void OverloadCandidateSet::NoteCandidates(Sema &S,
                                           OverloadCandidateDisplayKind OCD,
                                           llvm::ArrayRef<Expr *> Args,
-                                          const char *Opc,
+                                          StringRef Opc,
                                           SourceLocation OpLoc) {
   // Sort the candidates by viability and position.  Sorting directly would
   // be prohibitive, so we make a set of pointers and sort those.
@@ -8807,8 +8834,7 @@ void OverloadCandidateSet::NoteCandidates(Sema &S,
   bool ReportedAmbiguousConversions = false;
 
   SmallVectorImpl<OverloadCandidate*>::iterator I, E;
-  const DiagnosticsEngine::OverloadsShown ShowOverloads = 
-      S.Diags.getShowOverloads();
+  const OverloadsShown ShowOverloads = S.Diags.getShowOverloads();
   unsigned CandsShown = 0;
   for (I = Cands.begin(), E = Cands.end(); I != E; ++I) {
     OverloadCandidate *Cand = *I;
@@ -8816,7 +8842,7 @@ void OverloadCandidateSet::NoteCandidates(Sema &S,
     // Set an arbitrary limit on the number of candidate functions we'll spam
     // the user with.  FIXME: This limit should depend on details of the
     // candidate list.
-    if (CandsShown >= 4 && ShowOverloads == DiagnosticsEngine::Ovl_Best) {
+    if (CandsShown >= 4 && ShowOverloads == Ovl_Best) {
       break;
     }
     ++CandsShown;
@@ -8979,7 +9005,7 @@ private:
     //   function template specialization, which is added to the set of
     //   overloaded functions considered.
     FunctionDecl *Specialization = 0;
-    TemplateDeductionInfo Info(Context, OvlExpr->getNameLoc());
+    TemplateDeductionInfo Info(OvlExpr->getNameLoc());
     if (Sema::TemplateDeductionResult Result
           = S.DeduceTemplateArguments(FunctionTemplate, 
                                       &OvlExplicitTemplateArgs,
@@ -9201,7 +9227,6 @@ Sema::ResolveAddressOfOverloadedFunction(Expr *AddressOfExpr,
     Fn = Resolver.getMatchingFunctionDecl();
     assert(Fn);
     FoundResult = *Resolver.getMatchingFunctionAccessPair();
-    MarkFunctionReferenced(AddressOfExpr->getLocStart(), Fn);
     if (Complain)
       CheckAddressOfMemberAccess(AddressOfExpr, FoundResult);
   }
@@ -9257,7 +9282,7 @@ Sema::ResolveSingleFunctionTemplateSpecialization(OverloadExpr *ovl,
     //   function template specialization, which is added to the set of
     //   overloaded functions considered.
     FunctionDecl *Specialization = 0;
-    TemplateDeductionInfo Info(Context, ovl->getNameLoc());
+    TemplateDeductionInfo Info(ovl->getNameLoc());
     if (TemplateDeductionResult Result
           = DeduceTemplateArguments(FunctionTemplate, &ExplicitTemplateArgs,
                                     Specialization, Info)) {
@@ -9457,8 +9482,7 @@ void Sema::AddOverloadedCallCandidates(UnresolvedLookupExpr *ULE,
     AddArgumentDependentLookupCandidates(ULE->getName(), /*Operator*/ false,
                                          ULE->getExprLoc(),
                                          Args, ExplicitTemplateArgs,
-                                         CandidateSet, PartialOverloading,
-                                         ULE->isStdAssociatedNamespace());
+                                         CandidateSet, PartialOverloading);
 }
 
 /// Attempt to recover from an ill-formed use of a non-dependent name in a
@@ -9509,7 +9533,7 @@ DiagnoseTwoPhaseLookup(Sema &SemaRef, SourceLocation FnLoc,
       // declaring the function there instead.
       Sema::AssociatedNamespaceSet AssociatedNamespaces;
       Sema::AssociatedClassSet AssociatedClasses;
-      SemaRef.FindAssociatedClassesAndNamespaces(Args,
+      SemaRef.FindAssociatedClassesAndNamespaces(FnLoc, Args,
                                                  AssociatedNamespaces,
                                                  AssociatedClasses);
       // Never suggest declaring a function within namespace 'std'. 
@@ -9632,6 +9656,20 @@ class NoTypoCorrectionCCC : public CorrectionCandidateCallback {
     return false;
   }
 };
+
+class BuildRecoveryCallExprRAII {
+  Sema &SemaRef;
+public:
+  BuildRecoveryCallExprRAII(Sema &S) : SemaRef(S) {
+    assert(SemaRef.IsBuildingRecoveryCallExpr == false);
+    SemaRef.IsBuildingRecoveryCallExpr = true;
+  }
+
+  ~BuildRecoveryCallExprRAII() {
+    SemaRef.IsBuildingRecoveryCallExpr = false;
+  }
+};
+
 }
 
 /// Attempts to recover from a call where no functions were found.
@@ -9644,6 +9682,15 @@ BuildRecoveryCallExpr(Sema &SemaRef, Scope *S, Expr *Fn,
                       llvm::MutableArrayRef<Expr *> Args,
                       SourceLocation RParenLoc,
                       bool EmptyLookup, bool AllowTypoCorrection) {
+  // Do not try to recover if it is already building a recovery call.
+  // This stops infinite loops for template instantiations like
+  //
+  // template <typename T> auto foo(T t) -> decltype(foo(t)) {}
+  // template <typename T> auto foo(T t) -> decltype(foo(&t)) {}
+  //
+  if (SemaRef.IsBuildingRecoveryCallExpr)
+    return ExprError();
+  BuildRecoveryCallExprRAII RCE(SemaRef);
 
   CXXScopeSpec SS;
   SS.Adopt(ULE->getQualifierLoc());
@@ -9695,20 +9742,15 @@ BuildRecoveryCallExpr(Sema &SemaRef, Scope *S, Expr *Fn,
                                RParenLoc);
 }
 
-/// ResolveOverloadedCallFn - Given the call expression that calls Fn
-/// (which eventually refers to the declaration Func) and the call
-/// arguments Args/NumArgs, attempt to resolve the function call down
-/// to a specific function. If overload resolution succeeds, returns
-/// the function declaration produced by overload
-/// resolution. Otherwise, emits diagnostics, deletes all of the
-/// arguments and Fn, and returns NULL.
-ExprResult
-Sema::BuildOverloadedCallExpr(Scope *S, Expr *Fn, UnresolvedLookupExpr *ULE,
-                              SourceLocation LParenLoc,
-                              Expr **Args, unsigned NumArgs,
-                              SourceLocation RParenLoc,
-                              Expr *ExecConfig,
-                              bool AllowTypoCorrection) {
+/// \brief Constructs and populates an OverloadedCandidateSet from
+/// the given function.
+/// \returns true when an the ExprResult output parameter has been set.
+bool Sema::buildOverloadedCallSet(Scope *S, Expr *Fn,
+                                  UnresolvedLookupExpr *ULE,
+                                  Expr **Args, unsigned NumArgs,
+                                  SourceLocation RParenLoc,
+                                  OverloadCandidateSet *CandidateSet,
+                                  ExprResult *Result) {
 #ifndef NDEBUG
   if (ULE->requiresADL()) {
     // To do ADL, we must have found an unqualified name.
@@ -9724,62 +9766,79 @@ Sema::BuildOverloadedCallExpr(Scope *S, Expr *Fn, UnresolvedLookupExpr *ULE,
 
     // We don't perform ADL in C.
     assert(getLangOpts().CPlusPlus && "ADL enabled in C");
-  } else
-    assert(!ULE->isStdAssociatedNamespace() &&
-           "std is associated namespace but not doing ADL");
+  }
 #endif
 
   UnbridgedCastsSet UnbridgedCasts;
-  if (checkArgPlaceholdersForOverload(*this, Args, NumArgs, UnbridgedCasts))
-    return ExprError();
-
-  OverloadCandidateSet CandidateSet(Fn->getExprLoc());
+  if (checkArgPlaceholdersForOverload(*this, Args, NumArgs, UnbridgedCasts)) {
+    *Result = ExprError();
+    return true;
+  }
 
   // Add the functions denoted by the callee to the set of candidate
   // functions, including those from argument-dependent lookup.
   AddOverloadedCallCandidates(ULE, llvm::makeArrayRef(Args, NumArgs),
-                              CandidateSet);
+                              *CandidateSet);
 
   // If we found nothing, try to recover.
   // BuildRecoveryCallExpr diagnoses the error itself, so we just bail
   // out if it fails.
-  if (CandidateSet.empty()) {
+  if (CandidateSet->empty()) {
     // In Microsoft mode, if we are inside a template class member function then
     // create a type dependent CallExpr. The goal is to postpone name lookup
     // to instantiation time to be able to search into type dependent base
     // classes.
     if (getLangOpts().MicrosoftMode && CurContext->isDependentContext() && 
         (isa<FunctionDecl>(CurContext) || isa<CXXRecordDecl>(CurContext))) {
-      CallExpr *CE = new (Context) CallExpr(Context, Fn, Args, NumArgs,
-                                          Context.DependentTy, VK_RValue,
-                                          RParenLoc);
+      CallExpr *CE = new (Context) CallExpr(Context, Fn,
+                                            llvm::makeArrayRef(Args, NumArgs),
+                                            Context.DependentTy, VK_RValue,
+                                            RParenLoc);
       CE->setTypeDependent(true);
-      return Owned(CE);
+      *Result = Owned(CE);
+      return true;
     }
-    return BuildRecoveryCallExpr(*this, S, Fn, ULE, LParenLoc,
-                                 llvm::MutableArrayRef<Expr *>(Args, NumArgs),
-                                 RParenLoc, /*EmptyLookup=*/true,
-                                 AllowTypoCorrection);
+    return false;
   }
 
   UnbridgedCasts.restore();
+  return false;
+}
 
-  OverloadCandidateSet::iterator Best;
-  switch (CandidateSet.BestViableFunction(*this, Fn->getLocStart(), Best)) {
+/// FinishOverloadedCallExpr - given an OverloadCandidateSet, builds and returns
+/// the completed call expression. If overload resolution fails, emits
+/// diagnostics and returns ExprError()
+static ExprResult FinishOverloadedCallExpr(Sema &SemaRef, Scope *S, Expr *Fn,
+                                           UnresolvedLookupExpr *ULE,
+                                           SourceLocation LParenLoc,
+                                           Expr **Args, unsigned NumArgs,
+                                           SourceLocation RParenLoc,
+                                           Expr *ExecConfig,
+                                           OverloadCandidateSet *CandidateSet,
+                                           OverloadCandidateSet::iterator *Best,
+                                           OverloadingResult OverloadResult,
+                                           bool AllowTypoCorrection) {
+  if (CandidateSet->empty())
+    return BuildRecoveryCallExpr(SemaRef, S, Fn, ULE, LParenLoc,
+                                 llvm::MutableArrayRef<Expr *>(Args, NumArgs),
+                                 RParenLoc, /*EmptyLookup=*/true,
+                                 AllowTypoCorrection);
+
+  switch (OverloadResult) {
   case OR_Success: {
-    FunctionDecl *FDecl = Best->Function;
-    MarkFunctionReferenced(Fn->getExprLoc(), FDecl);
-    CheckUnresolvedLookupAccess(ULE, Best->FoundDecl);
-    DiagnoseUseOfDecl(FDecl, ULE->getNameLoc());
-    Fn = FixOverloadedFunctionReference(Fn, Best->FoundDecl, FDecl);
-    return BuildResolvedCallExpr(Fn, FDecl, LParenLoc, Args, NumArgs, RParenLoc,
-                                 ExecConfig);
+    FunctionDecl *FDecl = (*Best)->Function;
+    SemaRef.MarkFunctionReferenced(Fn->getExprLoc(), FDecl);
+    SemaRef.CheckUnresolvedLookupAccess(ULE, (*Best)->FoundDecl);
+    SemaRef.DiagnoseUseOfDecl(FDecl, ULE->getNameLoc());
+    Fn = SemaRef.FixOverloadedFunctionReference(Fn, (*Best)->FoundDecl, FDecl);
+    return SemaRef.BuildResolvedCallExpr(Fn, FDecl, LParenLoc, Args, NumArgs,
+                                         RParenLoc, ExecConfig);
   }
 
   case OR_No_Viable_Function: {
     // Try to recover by looking for viable functions which the user might
     // have meant to call.
-    ExprResult Recovery = BuildRecoveryCallExpr(*this, S, Fn, ULE, LParenLoc,
+    ExprResult Recovery = BuildRecoveryCallExpr(SemaRef, S, Fn, ULE, LParenLoc,
                                   llvm::MutableArrayRef<Expr *>(Args, NumArgs),
                                                 RParenLoc,
                                                 /*EmptyLookup=*/false,
@@ -9787,44 +9846,73 @@ Sema::BuildOverloadedCallExpr(Scope *S, Expr *Fn, UnresolvedLookupExpr *ULE,
     if (!Recovery.isInvalid())
       return Recovery;
 
-    Diag(Fn->getLocStart(),
+    SemaRef.Diag(Fn->getLocStart(),
          diag::err_ovl_no_viable_function_in_call)
       << ULE->getName() << Fn->getSourceRange();
-    CandidateSet.NoteCandidates(*this, OCD_AllCandidates,
-                                llvm::makeArrayRef(Args, NumArgs));
+    CandidateSet->NoteCandidates(SemaRef, OCD_AllCandidates,
+                                 llvm::makeArrayRef(Args, NumArgs));
     break;
   }
 
   case OR_Ambiguous:
-    Diag(Fn->getLocStart(), diag::err_ovl_ambiguous_call)
+    SemaRef.Diag(Fn->getLocStart(), diag::err_ovl_ambiguous_call)
       << ULE->getName() << Fn->getSourceRange();
-    CandidateSet.NoteCandidates(*this, OCD_ViableCandidates,
-                                llvm::makeArrayRef(Args, NumArgs));
+    CandidateSet->NoteCandidates(SemaRef, OCD_ViableCandidates,
+                                 llvm::makeArrayRef(Args, NumArgs));
     break;
 
-  case OR_Deleted:
-    {
-      Diag(Fn->getLocStart(), diag::err_ovl_deleted_call)
-        << Best->Function->isDeleted()
-        << ULE->getName()
-        << getDeletedOrUnavailableSuffix(Best->Function)
-        << Fn->getSourceRange();
-      CandidateSet.NoteCandidates(*this, OCD_AllCandidates,
-                                  llvm::makeArrayRef(Args, NumArgs));
+  case OR_Deleted: {
+    SemaRef.Diag(Fn->getLocStart(), diag::err_ovl_deleted_call)
+      << (*Best)->Function->isDeleted()
+      << ULE->getName()
+      << SemaRef.getDeletedOrUnavailableSuffix((*Best)->Function)
+      << Fn->getSourceRange();
+    CandidateSet->NoteCandidates(SemaRef, OCD_AllCandidates,
+                                 llvm::makeArrayRef(Args, NumArgs));
 
-      // We emitted an error for the unvailable/deleted function call but keep
-      // the call in the AST.
-      FunctionDecl *FDecl = Best->Function;
-      Fn = FixOverloadedFunctionReference(Fn, Best->FoundDecl, FDecl);
-      return BuildResolvedCallExpr(Fn, FDecl, LParenLoc, Args, NumArgs,
-                                   RParenLoc, ExecConfig);
-    }
+    // We emitted an error for the unvailable/deleted function call but keep
+    // the call in the AST.
+    FunctionDecl *FDecl = (*Best)->Function;
+    Fn = SemaRef.FixOverloadedFunctionReference(Fn, (*Best)->FoundDecl, FDecl);
+    return SemaRef.BuildResolvedCallExpr(Fn, FDecl, LParenLoc, Args, NumArgs,
+                                 RParenLoc, ExecConfig);
+  }
   }
 
   // Overload resolution failed.
   return ExprError();
 }
 
+/// BuildOverloadedCallExpr - Given the call expression that calls Fn
+/// (which eventually refers to the declaration Func) and the call
+/// arguments Args/NumArgs, attempt to resolve the function call down
+/// to a specific function. If overload resolution succeeds, returns
+/// the call expression produced by overload resolution.
+/// Otherwise, emits diagnostics and returns ExprError.
+ExprResult Sema::BuildOverloadedCallExpr(Scope *S, Expr *Fn,
+                                         UnresolvedLookupExpr *ULE,
+                                         SourceLocation LParenLoc,
+                                         Expr **Args, unsigned NumArgs,
+                                         SourceLocation RParenLoc,
+                                         Expr *ExecConfig,
+                                         bool AllowTypoCorrection) {
+  OverloadCandidateSet CandidateSet(Fn->getExprLoc());
+  ExprResult result;
+
+  if (buildOverloadedCallSet(S, Fn, ULE, Args, NumArgs, LParenLoc,
+                             &CandidateSet, &result))
+    return result;
+
+  OverloadCandidateSet::iterator Best;
+  OverloadingResult OverloadResult =
+      CandidateSet.BestViableFunction(*this, Fn->getLocStart(), Best);
+
+  return FinishOverloadedCallExpr(*this, S, Fn, ULE, LParenLoc, Args, NumArgs,
+                                  RParenLoc, ExecConfig, &CandidateSet,
+                                  &Best, OverloadResult,
+                                  AllowTypoCorrection);
+}
+
 static bool IsOverloaded(const UnresolvedSetImpl &Functions) {
   return Functions.size() > 1 ||
     (Functions.size() == 1 && isa<FunctionTemplateDecl>(*Functions.begin()));
@@ -9889,10 +9977,10 @@ Sema::CreateOverloadedUnaryOp(SourceLocation OpLoc, unsigned OpcIn,
                                      /*ADL*/ true, IsOverloaded(Fns),
                                      Fns.begin(), Fns.end());
     return Owned(new (Context) CXXOperatorCallExpr(Context, Op, Fn,
-                                                  &Args[0], NumArgs,
+                                              llvm::makeArrayRef(Args, NumArgs),
                                                    Context.DependentTy,
                                                    VK_RValue,
-                                                   OpLoc));
+                                                   OpLoc, false));
   }
 
   // Build an empty overload set.
@@ -9968,7 +10056,8 @@ Sema::CreateOverloadedUnaryOp(SourceLocation OpLoc, unsigned OpcIn,
       Args[0] = Input;
       CallExpr *TheCall =
         new (Context) CXXOperatorCallExpr(Context, Op, FnExpr.take(),
-                                          Args, NumArgs, ResultTy, VK, OpLoc);
+                                          llvm::makeArrayRef(Args, NumArgs),
+                                          ResultTy, VK, OpLoc, false);
 
       if (CheckCallReturnType(FnDecl->getResultType(), OpLoc, TheCall,
                               FnDecl))
@@ -10069,7 +10158,8 @@ Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
         return Owned(new (Context) BinaryOperator(Args[0], Args[1], Opc,
                                                   Context.DependentTy,
                                                   VK_RValue, OK_Ordinary,
-                                                  OpLoc));
+                                                  OpLoc,
+                                                  FPFeatures.fp_contract));
 
       return Owned(new (Context) CompoundAssignOperator(Args[0], Args[1], Opc,
                                                         Context.DependentTy,
@@ -10077,7 +10167,8 @@ Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
                                                         OK_Ordinary,
                                                         Context.DependentTy,
                                                         Context.DependentTy,
-                                                        OpLoc));
+                                                        OpLoc,
+                                                        FPFeatures.fp_contract));
     }
 
     // FIXME: save results of ADL from here?
@@ -10089,11 +10180,9 @@ Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
                                      NestedNameSpecifierLoc(), OpNameInfo, 
                                      /*ADL*/ true, IsOverloaded(Fns),
                                      Fns.begin(), Fns.end());
-    return Owned(new (Context) CXXOperatorCallExpr(Context, Op, Fn,
-                                                   Args, 2,
-                                                   Context.DependentTy,
-                                                   VK_RValue,
-                                                   OpLoc));
+    return Owned(new (Context) CXXOperatorCallExpr(Context, Op, Fn, Args,
+                                                Context.DependentTy, VK_RValue,
+                                                OpLoc, FPFeatures.fp_contract));
   }
 
   // Always do placeholder-like conversions on the RHS.
@@ -10208,7 +10297,8 @@ Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
 
         CXXOperatorCallExpr *TheCall =
           new (Context) CXXOperatorCallExpr(Context, Op, FnExpr.take(),
-                                            Args, 2, ResultTy, VK, OpLoc);
+                                            Args, ResultTy, VK, OpLoc,
+                                            FPFeatures.fp_contract);
 
         if (CheckCallReturnType(FnDecl->getResultType(), OpLoc, TheCall,
                                 FnDecl))
@@ -10270,7 +10360,7 @@ Sema::CreateOverloadedBinOp(SourceLocation OpLoc,
       if (Result.isInvalid())
         CandidateSet.NoteCandidates(*this, OCD_AllCandidates, Args,
                                     BinaryOperator::getOpcodeStr(Opc), OpLoc);
-      return move(Result);
+      return Result;
     }
 
     case OR_Ambiguous:
@@ -10337,10 +10427,10 @@ Sema::CreateOverloadedArraySubscriptExpr(SourceLocation LLoc,
     // Can't add any actual overloads yet
 
     return Owned(new (Context) CXXOperatorCallExpr(Context, OO_Subscript, Fn,
-                                                   Args, 2,
+                                                   Args,
                                                    Context.DependentTy,
                                                    VK_RValue,
-                                                   RLoc));
+                                                   RLoc, false));
   }
 
   // Handle placeholders on both operands.
@@ -10416,8 +10506,9 @@ Sema::CreateOverloadedArraySubscriptExpr(SourceLocation LLoc,
 
         CXXOperatorCallExpr *TheCall =
           new (Context) CXXOperatorCallExpr(Context, OO_Subscript,
-                                            FnExpr.take(), Args, 2,
-                                            ResultTy, VK, RLoc);
+                                            FnExpr.take(), Args,
+                                            ResultTy, VK, RLoc,
+                                            false);
 
         if (CheckCallReturnType(FnDecl->getResultType(), LLoc, TheCall,
                                 FnDecl))
@@ -10534,7 +10625,8 @@ Sema::BuildCallToMemberFunction(Scope *S, Expr *MemExprE,
     }
               
     CXXMemberCallExpr *call
-      = new (Context) CXXMemberCallExpr(Context, MemExprE, Args, NumArgs,
+      = new (Context) CXXMemberCallExpr(Context, MemExprE,
+                                        llvm::makeArrayRef(Args, NumArgs),
                                         resultType, valueKind, RParenLoc);
 
     if (CheckCallReturnType(proto->getResultType(),
@@ -10676,7 +10768,8 @@ Sema::BuildCallToMemberFunction(Scope *S, Expr *MemExprE,
 
   assert(Method && "Member call to something that isn't a method?");
   CXXMemberCallExpr *TheCall =
-    new (Context) CXXMemberCallExpr(Context, MemExprE, Args, NumArgs,
+    new (Context) CXXMemberCallExpr(Context, MemExprE,
+                                    llvm::makeArrayRef(Args, NumArgs),
                                     ResultType, VK, RParenLoc);
 
   // Check for a valid return type.
@@ -10904,6 +10997,11 @@ Sema::BuildCallToObjectOfClassType(Scope *S, Expr *Obj,
   // that calls this method, using Object for the implicit object
   // parameter and passing along the remaining arguments.
   CXXMethodDecl *Method = cast<CXXMethodDecl>(Best->Function);
+
+  // An error diagnostic has already been printed when parsing the declaration.
+  if (Method->isInvalidDecl())
+    return ExprError();
+
   const FunctionProtoType *Proto =
     Method->getType()->getAs<FunctionProtoType>();
 
@@ -10942,8 +11040,8 @@ Sema::BuildCallToObjectOfClassType(Scope *S, Expr *Obj,
 
   CXXOperatorCallExpr *TheCall =
     new (Context) CXXOperatorCallExpr(Context, OO_Call, NewFn.take(),
-                                      MethodArgs, NumArgs + 1,
-                                      ResultTy, VK, RParenLoc);
+                                      llvm::makeArrayRef(MethodArgs, NumArgs+1),
+                                      ResultTy, VK, RParenLoc, false);
   delete [] MethodArgs;
 
   if (CheckCallReturnType(Method->getResultType(), LParenLoc, TheCall,
@@ -10966,7 +11064,7 @@ Sema::BuildCallToObjectOfClassType(Scope *S, Expr *Obj,
   if (ObjRes.isInvalid())
     IsError = true;
   else
-    Object = move(ObjRes);
+    Object = ObjRes;
   TheCall->setArg(0, Object.take());
 
   // Check the argument types.
@@ -11116,7 +11214,7 @@ Sema::BuildOverloadedArrowExpr(Scope *S, Expr *Base, SourceLocation OpLoc) {
   ResultTy = ResultTy.getNonLValueExprType(Context);
   CXXOperatorCallExpr *TheCall =
     new (Context) CXXOperatorCallExpr(Context, OO_Arrow, FnExpr.take(),
-                                      &Base, 1, ResultTy, VK, OpLoc);
+                                      Base, ResultTy, VK, OpLoc, false);
 
   if (CheckCallReturnType(Method->getResultType(), OpLoc, TheCall,
                           Method))
@@ -11187,7 +11285,8 @@ ExprResult Sema::BuildLiteralOperatorCall(LookupResult &R,
   ResultTy = ResultTy.getNonLValueExprType(Context);
 
   UserDefinedLiteral *UDL =
-    new (Context) UserDefinedLiteral(Context, Fn.take(), ConvArgs, Args.size(),
+    new (Context) UserDefinedLiteral(Context, Fn.take(),
+                                     llvm::makeArrayRef(ConvArgs, Args.size()),
                                      ResultTy, VK, LitEndLoc, UDSuffixLoc);
 
   if (CheckCallReturnType(FD->getResultType(), UDSuffixLoc, UDL, FD))
@@ -11199,6 +11298,80 @@ ExprResult Sema::BuildLiteralOperatorCall(LookupResult &R,
   return MaybeBindToTemporary(UDL);
 }
 
+/// Build a call to 'begin' or 'end' for a C++11 for-range statement. If the
+/// given LookupResult is non-empty, it is assumed to describe a member which
+/// will be invoked. Otherwise, the function will be found via argument
+/// dependent lookup.
+/// CallExpr is set to a valid expression and FRS_Success returned on success,
+/// otherwise CallExpr is set to ExprError() and some non-success value
+/// is returned.
+Sema::ForRangeStatus
+Sema::BuildForRangeBeginEndCall(Scope *S, SourceLocation Loc,
+                                SourceLocation RangeLoc, VarDecl *Decl,
+                                BeginEndFunction BEF,
+                                const DeclarationNameInfo &NameInfo,
+                                LookupResult &MemberLookup,
+                                OverloadCandidateSet *CandidateSet,
+                                Expr *Range, ExprResult *CallExpr) {
+  CandidateSet->clear();
+  if (!MemberLookup.empty()) {
+    ExprResult MemberRef =
+        BuildMemberReferenceExpr(Range, Range->getType(), Loc,
+                                 /*IsPtr=*/false, CXXScopeSpec(),
+                                 /*TemplateKWLoc=*/SourceLocation(),
+                                 /*FirstQualifierInScope=*/0,
+                                 MemberLookup,
+                                 /*TemplateArgs=*/0);
+    if (MemberRef.isInvalid()) {
+      *CallExpr = ExprError();
+      Diag(Range->getLocStart(), diag::note_in_for_range)
+          << RangeLoc << BEF << Range->getType();
+      return FRS_DiagnosticIssued;
+    }
+    *CallExpr = ActOnCallExpr(S, MemberRef.get(), Loc, MultiExprArg(), Loc, 0);
+    if (CallExpr->isInvalid()) {
+      *CallExpr = ExprError();
+      Diag(Range->getLocStart(), diag::note_in_for_range)
+          << RangeLoc << BEF << Range->getType();
+      return FRS_DiagnosticIssued;
+    }
+  } else {
+    UnresolvedSet<0> FoundNames;
+    UnresolvedLookupExpr *Fn =
+      UnresolvedLookupExpr::Create(Context, /*NamingClass=*/0,
+                                   NestedNameSpecifierLoc(), NameInfo,
+                                   /*NeedsADL=*/true, /*Overloaded=*/false,
+                                   FoundNames.begin(), FoundNames.end());
+
+    bool CandidateSetError = buildOverloadedCallSet(S, Fn, Fn, &Range, 1, Loc,
+                                                    CandidateSet, CallExpr);
+    if (CandidateSet->empty() || CandidateSetError) {
+      *CallExpr = ExprError();
+      return FRS_NoViableFunction;
+    }
+    OverloadCandidateSet::iterator Best;
+    OverloadingResult OverloadResult =
+        CandidateSet->BestViableFunction(*this, Fn->getLocStart(), Best);
+
+    if (OverloadResult == OR_No_Viable_Function) {
+      *CallExpr = ExprError();
+      return FRS_NoViableFunction;
+    }
+    *CallExpr = FinishOverloadedCallExpr(*this, S, Fn, Fn, Loc, &Range, 1,
+                                         Loc, 0, CandidateSet, &Best,
+                                         OverloadResult,
+                                         /*AllowTypoCorrection=*/false);
+    if (CallExpr->isInvalid() || OverloadResult != OR_Success) {
+      *CallExpr = ExprError();
+      Diag(Range->getLocStart(), diag::note_in_for_range)
+          << RangeLoc << BEF << Range->getType();
+      return FRS_DiagnosticIssued;
+    }
+  }
+  return FRS_Success;
+}
+
+
 /// FixOverloadedFunctionReference - E is an expression that refers to
 /// a C++ overloaded function (possibly with some parentheses and
 /// perhaps a '&' around it). We have resolved the overloaded function
@@ -11358,6 +11531,7 @@ Expr *Sema::FixOverloadedFunctionReference(Expr *E, DeclAccessPair Found,
                                         TemplateArgs,
                                         type, valueKind, OK_Ordinary);
     ME->setHadMultipleCandidates(true);
+    MarkMemberReferenced(ME);
     return ME;
   }
 
diff --git a/lib/Sema/SemaPseudoObject.cpp b/lib/Sema/SemaPseudoObject.cpp
index 722ac19..a8d75b2 100644
--- a/lib/Sema/SemaPseudoObject.cpp
+++ b/lib/Sema/SemaPseudoObject.cpp
@@ -31,6 +31,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/ScopeInfo.h"
 #include "clang/Sema/Initialization.h"
 #include "clang/AST/ExprObjC.h"
 #include "clang/Lex/Preprocessor.h"
@@ -91,9 +92,8 @@ namespace {
         return new (S.Context) GenericSelectionExpr(S.Context,
                                                     gse->getGenericLoc(),
                                                     gse->getControllingExpr(),
-                                                    assocTypes.data(),
-                                                    assocs.data(),
-                                                    numAssocs,
+                                                    assocTypes,
+                                                    assocs,
                                                     gse->getDefaultLoc(),
                                                     gse->getRParenLoc(),
                                       gse->containsUnexpandedParameterPack(),
@@ -187,7 +187,7 @@ namespace {
                                     UnaryOperatorKind opcode,
                                     Expr *op);
 
-    ExprResult complete(Expr *syntacticForm);
+    virtual ExprResult complete(Expr *syntacticForm);
 
     OpaqueValueExpr *capture(Expr *op);
     OpaqueValueExpr *captureValueAsResult(Expr *op);
@@ -198,7 +198,14 @@ namespace {
     }
 
     /// Return true if assignments have a non-void result.
-    virtual bool assignmentsHaveResult() { return true; }
+    bool CanCaptureValueOfType(QualType ty) {
+      assert(!ty->isIncompleteType());
+      assert(!ty->isDependentType());
+
+      if (const CXXRecordDecl *ClassDecl = ty->getAsCXXRecordDecl())
+        return ClassDecl->isTriviallyCopyable();
+      return true;
+    }
 
     virtual Expr *rebuildAndCaptureObject(Expr *) = 0;
     virtual ExprResult buildGet() = 0;
@@ -206,7 +213,7 @@ namespace {
                                 bool captureSetValueAsResult) = 0;
   };
 
-  /// A PseudoOpBuilder for Objective-C @properties.
+  /// A PseudoOpBuilder for Objective-C \@properties.
   class ObjCPropertyOpBuilder : public PseudoOpBuilder {
     ObjCPropertyRefExpr *RefExpr;
     ObjCPropertyRefExpr *SyntacticRefExpr;
@@ -239,6 +246,9 @@ namespace {
     Expr *rebuildAndCaptureObject(Expr *syntacticBase);
     ExprResult buildGet();
     ExprResult buildSet(Expr *op, SourceLocation, bool);
+    ExprResult complete(Expr *SyntacticForm);
+
+    bool isWeakProperty() const;
   };
 
  /// A PseudoOpBuilder for Objective-C array/dictionary indexing.
@@ -292,7 +302,7 @@ OpaqueValueExpr *PseudoOpBuilder::capture(Expr *e) {
 /// operation.  This routine is safe against expressions which may
 /// already be captured.
 ///
-/// \param Returns the captured expression, which will be the
+/// \returns the captured expression, which will be the
 ///   same as the input if the input was already captured
 OpaqueValueExpr *PseudoOpBuilder::captureValueAsResult(Expr *e) {
   assert(ResultIndex == PseudoObjectExpr::NoResult);
@@ -353,7 +363,7 @@ PseudoOpBuilder::buildAssignmentOperation(Scope *Sc, SourceLocation opcLoc,
     syntactic = new (S.Context) BinaryOperator(syntacticLHS, capturedRHS,
                                                opcode, capturedRHS->getType(),
                                                capturedRHS->getValueKind(),
-                                               OK_Ordinary, opcLoc);
+                                               OK_Ordinary, opcLoc, false);
   } else {
     ExprResult opLHS = buildGet();
     if (opLHS.isInvalid()) return ExprError();
@@ -372,12 +382,12 @@ PseudoOpBuilder::buildAssignmentOperation(Scope *Sc, SourceLocation opcLoc,
                                              OK_Ordinary,
                                              opLHS.get()->getType(),
                                              result.get()->getType(),
-                                             opcLoc);
+                                             opcLoc, false);
   }
 
   // The result of the assignment, if not void, is the value set into
   // the l-value.
-  result = buildSet(result.take(), opcLoc, assignmentsHaveResult());
+  result = buildSet(result.take(), opcLoc, /*captureSetValueAsResult*/ true);
   if (result.isInvalid()) return ExprError();
   addSemanticExpr(result.take());
 
@@ -401,7 +411,7 @@ PseudoOpBuilder::buildIncDecOperation(Scope *Sc, SourceLocation opcLoc,
   QualType resultType = result.get()->getType();
 
   // That's the postfix result.
-  if (UnaryOperator::isPostfix(opcode) && assignmentsHaveResult()) {
+  if (UnaryOperator::isPostfix(opcode) && CanCaptureValueOfType(resultType)) {
     result = capture(result.take());
     setResultToLastSemantic();
   }
@@ -420,8 +430,7 @@ PseudoOpBuilder::buildIncDecOperation(Scope *Sc, SourceLocation opcLoc,
 
   // Store that back into the result.  The value stored is the result
   // of a prefix operation.
-  result = buildSet(result.take(), opcLoc,
-             UnaryOperator::isPrefix(opcode) && assignmentsHaveResult());
+  result = buildSet(result.take(), opcLoc, UnaryOperator::isPrefix(opcode));
   if (result.isInvalid()) return ExprError();
   addSemanticExpr(result.take());
 
@@ -472,6 +481,23 @@ static ObjCMethodDecl *LookupMethodInReceiverType(Sema &S, Selector sel,
   return S.LookupMethodInObjectType(sel, IT, false);
 }
 
+bool ObjCPropertyOpBuilder::isWeakProperty() const {
+  QualType T;
+  if (RefExpr->isExplicitProperty()) {
+    const ObjCPropertyDecl *Prop = RefExpr->getExplicitProperty();
+    if (Prop->getPropertyAttributes() & ObjCPropertyDecl::OBJC_PR_weak)
+      return true;
+
+    T = Prop->getType();
+  } else if (Getter) {
+    T = Getter->getResultType();
+  } else {
+    return false;
+  }
+
+  return T.getObjCLifetime() == Qualifiers::OCL_Weak;
+}
+
 bool ObjCPropertyOpBuilder::findGetter() {
   if (Getter) return true;
 
@@ -532,7 +558,7 @@ bool ObjCPropertyOpBuilder::findSetter(bool warn) {
   // Do a normal method lookup first.
   if (ObjCMethodDecl *setter =
         LookupMethodInReceiverType(S, SetterSelector, RefExpr)) {
-    if (setter->isSynthesized() && warn)
+    if (setter->isPropertyAccessor() && warn)
       if (const ObjCInterfaceDecl *IFace =
           dyn_cast<ObjCInterfaceDecl>(setter->getDeclContext())) {
         const StringRef thisPropertyName(prop->getName());
@@ -617,7 +643,7 @@ ExprResult ObjCPropertyOpBuilder::buildGet() {
 
 /// Store to an Objective-C property reference.
 ///
-/// \param bindSetValueAsResult - If true, capture the actual
+/// \param captureSetValueAsResult If true, capture the actual
 ///   value being set as the value of the property operation.
 ExprResult ObjCPropertyOpBuilder::buildSet(Expr *op, SourceLocation opcLoc,
                                            bool captureSetValueAsResult) {
@@ -676,7 +702,8 @@ ExprResult ObjCPropertyOpBuilder::buildSet(Expr *op, SourceLocation opcLoc,
     ObjCMessageExpr *msgExpr =
       cast<ObjCMessageExpr>(msg.get()->IgnoreImplicit());
     Expr *arg = msgExpr->getArg(0);
-    msgExpr->setArg(0, captureValueAsResult(arg));
+    if (CanCaptureValueOfType(arg->getType()))
+      msgExpr->setArg(0, captureValueAsResult(arg));
   }
 
   return msg;
@@ -819,6 +846,19 @@ ObjCPropertyOpBuilder::buildIncDecOperation(Scope *Sc, SourceLocation opcLoc,
   return PseudoOpBuilder::buildIncDecOperation(Sc, opcLoc, opcode, op);
 }
 
+ExprResult ObjCPropertyOpBuilder::complete(Expr *SyntacticForm) {
+  if (S.getLangOpts().ObjCAutoRefCount && isWeakProperty()) {
+    DiagnosticsEngine::Level Level =
+      S.Diags.getDiagnosticLevel(diag::warn_arc_repeated_use_of_weak,
+                                 SyntacticForm->getLocStart());
+    if (Level != DiagnosticsEngine::Ignored)
+      S.getCurFunction()->recordUseOfWeak(SyntacticRefExpr,
+                                         SyntacticRefExpr->isMessagingGetter());
+  }
+
+  return PseudoOpBuilder::complete(SyntacticForm);
+}
+
 // ObjCSubscript build stuff.
 //
 
@@ -1035,7 +1075,7 @@ bool ObjCSubscriptOpBuilder::findAtIndexGetter() {
                            0 /*TypeSourceInfo */,
                            S.Context.getTranslationUnitDecl(),
                            true /*Instance*/, false/*isVariadic*/,
-                           /*isSynthesized=*/false,
+                           /*isPropertyAccessor=*/false,
                            /*isImplicitlyDeclared=*/true, /*isDefined=*/false,
                            ObjCMethodDecl::Required,
                            false);
@@ -1151,7 +1191,7 @@ bool ObjCSubscriptOpBuilder::findAtIndexSetter() {
                            ResultTInfo,
                            S.Context.getTranslationUnitDecl(),
                            true /*Instance*/, false/*isVariadic*/,
-                           /*isSynthesized=*/false,
+                           /*isPropertyAccessor=*/false,
                            /*isImplicitlyDeclared=*/true, /*isDefined=*/false,
                            ObjCMethodDecl::Required,
                            false); 
@@ -1255,7 +1295,7 @@ ExprResult ObjCSubscriptOpBuilder::buildGet() {
 /// Store into the container the "op" object at "Index"'ed location
 /// by building this messaging expression:
 /// - (void)setObject:(id)object atIndexedSubscript:(NSInteger)index;
-/// \param bindSetValueAsResult - If true, capture the actual
+/// \param captureSetValueAsResult If true, capture the actual
 ///   value being set as the value of the property operation.
 ExprResult ObjCSubscriptOpBuilder::buildSet(Expr *op, SourceLocation opcLoc,
                                            bool captureSetValueAsResult) {
@@ -1279,7 +1319,8 @@ ExprResult ObjCSubscriptOpBuilder::buildSet(Expr *op, SourceLocation opcLoc,
     ObjCMessageExpr *msgExpr =
       cast<ObjCMessageExpr>(msg.get()->IgnoreImplicit());
     Expr *arg = msgExpr->getArg(0);
-    msgExpr->setArg(0, captureValueAsResult(arg));
+    if (CanCaptureValueOfType(arg->getType()))
+      msgExpr->setArg(0, captureValueAsResult(arg));
   }
   
   return msg;
@@ -1333,7 +1374,7 @@ ExprResult Sema::checkPseudoObjectAssignment(Scope *S, SourceLocation opcLoc,
   // Do nothing if either argument is dependent.
   if (LHS->isTypeDependent() || RHS->isTypeDependent())
     return new (Context) BinaryOperator(LHS, RHS, opcode, Context.DependentTy,
-                                        VK_RValue, OK_Ordinary, opcLoc);
+                                        VK_RValue, OK_Ordinary, opcLoc, false);
 
   // Filter out non-overload placeholder types in the RHS.
   if (RHS->getType()->isNonOverloadPlaceholderType()) {
@@ -1404,14 +1445,14 @@ Expr *Sema::recreateSyntacticForm(PseudoObjectExpr *E) {
                                                 cop->getObjectKind(),
                                                 cop->getComputationLHSType(),
                                                 cop->getComputationResultType(),
-                                                cop->getOperatorLoc());
+                                                cop->getOperatorLoc(), false);
   } else if (BinaryOperator *bop = dyn_cast<BinaryOperator>(syntax)) {
     Expr *lhs = stripOpaqueValuesFromPseudoObjectRef(*this, bop->getLHS());
     Expr *rhs = cast<OpaqueValueExpr>(bop->getRHS())->getSourceExpr();
     return new (Context) BinaryOperator(lhs, rhs, bop->getOpcode(),
                                         bop->getType(), bop->getValueKind(),
                                         bop->getObjectKind(),
-                                        bop->getOperatorLoc());
+                                        bop->getOperatorLoc(), false);
   } else {
     assert(syntax->hasPlaceholderType(BuiltinType::PseudoObject));
     return stripOpaqueValuesFromPseudoObjectRef(*this, syntax);
diff --git a/lib/Sema/SemaStmt.cpp b/lib/Sema/SemaStmt.cpp
index 86884b7..f55174e 100644
--- a/lib/Sema/SemaStmt.cpp
+++ b/lib/Sema/SemaStmt.cpp
@@ -28,27 +28,10 @@
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Basic/TargetInfo.h"
 #include "llvm/ADT/ArrayRef.h"
-#include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/SmallPtrSet.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/SmallVector.h"
-#include "llvm/ADT/Triple.h"
-#include "llvm/MC/MCAsmInfo.h"
-#include "llvm/MC/MCContext.h"
-#include "llvm/MC/MCInst.h"
-#include "llvm/MC/MCInstPrinter.h"
-#include "llvm/MC/MCInstrInfo.h"
-#include "llvm/MC/MCObjectFileInfo.h"
-#include "llvm/MC/MCRegisterInfo.h"
-#include "llvm/MC/MCStreamer.h"
-#include "llvm/MC/MCSubtargetInfo.h"
-#include "llvm/MC/MCTargetAsmParser.h"
-#include "llvm/MC/MCParser/MCAsmLexer.h"
-#include "llvm/MC/MCParser/MCAsmParser.h"
-#include "llvm/Support/SourceMgr.h"
-#include "llvm/Support/TargetRegistry.h"
-#include "llvm/Support/TargetSelect.h"
 using namespace clang;
 using namespace sema;
 
@@ -177,6 +160,13 @@ void Sema::DiagnoseUnusedExprResult(const Stmt *S) {
       !E->isUnusedResultAWarning(WarnExpr, Loc, R1, R2, Context))
     return;
 
+  // If this is a GNU statement expression expanded from a macro, it is probably
+  // unused because it is a function-like macro that can be used as either an
+  // expression or statement.  Don't warn, because it is almost certainly a
+  // false positive.
+  if (isa<StmtExpr>(E) && Loc.isMacroID())
+    return;
+
   // Okay, we have an unused result.  Depending on what the base expression is,
   // we might want to make a more specific diagnostic.  Check for one of these
   // cases now.
@@ -271,7 +261,7 @@ StmtResult
 Sema::ActOnCompoundStmt(SourceLocation L, SourceLocation R,
                         MultiStmtArg elts, bool isStmtExpr) {
   unsigned NumElts = elts.size();
-  Stmt **Elts = reinterpret_cast<Stmt**>(elts.release());
+  Stmt **Elts = elts.data();
   // If we're in C89 mode, check that we don't have any decls after stmts.  If
   // so, emit an extension diagnostic.
   if (!getLangOpts().C99 && !getLangOpts().CPlusPlus) {
@@ -381,8 +371,10 @@ Sema::ActOnLabelStmt(SourceLocation IdentLoc, LabelDecl *TheDecl,
   // Otherwise, things are good.  Fill in the declaration and return it.
   LabelStmt *LS = new (Context) LabelStmt(IdentLoc, TheDecl, SubStmt);
   TheDecl->setStmt(LS);
-  if (!TheDecl->isGnuLocal())
+  if (!TheDecl->isGnuLocal()) {
+    TheDecl->setLocStart(IdentLoc);
     TheDecl->setLocation(IdentLoc);
+  }
   return Owned(LS);
 }
 
@@ -1566,25 +1558,6 @@ Sema::ActOnObjCForCollectionStmt(SourceLocation ForLoc,
                                                    ForLoc, RParenLoc));
 }
 
-namespace {
-
-enum BeginEndFunction {
-  BEF_begin,
-  BEF_end
-};
-
-/// Build a variable declaration for a for-range statement.
-static VarDecl *BuildForRangeVarDecl(Sema &SemaRef, SourceLocation Loc,
-                                     QualType Type, const char *Name) {
-  DeclContext *DC = SemaRef.CurContext;
-  IdentifierInfo *II = &SemaRef.PP.getIdentifierTable().get(Name);
-  TypeSourceInfo *TInfo = SemaRef.Context.getTrivialTypeSourceInfo(Type, Loc);
-  VarDecl *Decl = VarDecl::Create(SemaRef.Context, DC, Loc, Loc, II, Type,
-                                  TInfo, SC_Auto, SC_None);
-  Decl->setImplicit();
-  return Decl;
-}
-
 /// Finish building a variable declaration for a for-range statement.
 /// \return true if an error occurs.
 static bool FinishForRangeVarDecl(Sema &SemaRef, VarDecl *Decl, Expr *Init,
@@ -1617,12 +1590,14 @@ static bool FinishForRangeVarDecl(Sema &SemaRef, VarDecl *Decl, Expr *Init,
   return false;
 }
 
+namespace {
+
 /// Produce a note indicating which begin/end function was implicitly called
-/// by a C++0x for-range statement. This is often not obvious from the code,
+/// by a C++11 for-range statement. This is often not obvious from the code,
 /// nor from the diagnostics produced when analysing the implicit expressions
 /// required in a for-range statement.
 void NoteForRangeBeginEndFunction(Sema &SemaRef, Expr *E,
-                                  BeginEndFunction BEF) {
+                                  Sema::BeginEndFunction BEF) {
   CallExpr *CE = dyn_cast<CallExpr>(E);
   if (!CE)
     return;
@@ -1643,56 +1618,16 @@ void NoteForRangeBeginEndFunction(Sema &SemaRef, Expr *E,
     << BEF << IsTemplate << Description << E->getType();
 }
 
-/// Build a call to 'begin' or 'end' for a C++0x for-range statement. If the
-/// given LookupResult is non-empty, it is assumed to describe a member which
-/// will be invoked. Otherwise, the function will be found via argument
-/// dependent lookup.
-static ExprResult BuildForRangeBeginEndCall(Sema &SemaRef, Scope *S,
-                                            SourceLocation Loc,
-                                            VarDecl *Decl,
-                                            BeginEndFunction BEF,
-                                            const DeclarationNameInfo &NameInfo,
-                                            LookupResult &MemberLookup,
-                                            Expr *Range) {
-  ExprResult CallExpr;
-  if (!MemberLookup.empty()) {
-    ExprResult MemberRef =
-      SemaRef.BuildMemberReferenceExpr(Range, Range->getType(), Loc,
-                                       /*IsPtr=*/false, CXXScopeSpec(),
-                                       /*TemplateKWLoc=*/SourceLocation(),
-                                       /*FirstQualifierInScope=*/0,
-                                       MemberLookup,
-                                       /*TemplateArgs=*/0);
-    if (MemberRef.isInvalid())
-      return ExprError();
-    CallExpr = SemaRef.ActOnCallExpr(S, MemberRef.get(), Loc, MultiExprArg(),
-                                     Loc, 0);
-    if (CallExpr.isInvalid())
-      return ExprError();
-  } else {
-    UnresolvedSet<0> FoundNames;
-    // C++0x [stmt.ranged]p1: For the purposes of this name lookup, namespace
-    // std is an associated namespace.
-    UnresolvedLookupExpr *Fn =
-      UnresolvedLookupExpr::Create(SemaRef.Context, /*NamingClass=*/0,
-                                   NestedNameSpecifierLoc(), NameInfo,
-                                   /*NeedsADL=*/true, /*Overloaded=*/false,
-                                   FoundNames.begin(), FoundNames.end(),
-                                   /*LookInStdNamespace=*/true);
-    CallExpr = SemaRef.BuildOverloadedCallExpr(S, Fn, Fn, Loc, &Range, 1, Loc,
-                                               0, /*AllowTypoCorrection=*/false);
-    if (CallExpr.isInvalid()) {
-      SemaRef.Diag(Range->getLocStart(), diag::note_for_range_type)
-        << Range->getType();
-      return ExprError();
-    }
-  }
-  if (FinishForRangeVarDecl(SemaRef, Decl, CallExpr.get(), Loc,
-                            diag::err_for_range_iter_deduction_failure)) {
-    NoteForRangeBeginEndFunction(SemaRef, CallExpr.get(), BEF);
-    return ExprError();
-  }
-  return CallExpr;
+/// Build a variable declaration for a for-range statement.
+VarDecl *BuildForRangeVarDecl(Sema &SemaRef, SourceLocation Loc,
+                              QualType Type, const char *Name) {
+  DeclContext *DC = SemaRef.CurContext;
+  IdentifierInfo *II = &SemaRef.PP.getIdentifierTable().get(Name);
+  TypeSourceInfo *TInfo = SemaRef.Context.getTrivialTypeSourceInfo(Type, Loc);
+  VarDecl *Decl = VarDecl::Create(SemaRef.Context, DC, Loc, Loc, II, Type,
+                                  TInfo, SC_Auto, SC_None);
+  Decl->setImplicit();
+  return Decl;
 }
 
 }
@@ -1723,7 +1658,7 @@ static bool ObjCEnumerationCollection(Expr *Collection) {
 StmtResult
 Sema::ActOnCXXForRangeStmt(SourceLocation ForLoc,
                            Stmt *First, SourceLocation ColonLoc, Expr *Range,
-                           SourceLocation RParenLoc) {
+                           SourceLocation RParenLoc, BuildForRangeKind Kind) {
   if (!First || !Range)
     return StmtError();
 
@@ -1761,15 +1696,137 @@ Sema::ActOnCXXForRangeStmt(SourceLocation ForLoc,
 
   return BuildCXXForRangeStmt(ForLoc, ColonLoc, RangeDecl.get(),
                               /*BeginEndDecl=*/0, /*Cond=*/0, /*Inc=*/0, DS,
-                              RParenLoc);
+                              RParenLoc, Kind);
 }
 
-/// BuildCXXForRangeStmt - Build or instantiate a C++0x for-range statement.
+/// \brief Create the initialization, compare, and increment steps for
+/// the range-based for loop expression.
+/// This function does not handle array-based for loops,
+/// which are created in Sema::BuildCXXForRangeStmt.
+///
+/// \returns a ForRangeStatus indicating success or what kind of error occurred.
+/// BeginExpr and EndExpr are set and FRS_Success is returned on success;
+/// CandidateSet and BEF are set and some non-success value is returned on
+/// failure.
+static Sema::ForRangeStatus BuildNonArrayForRange(Sema &SemaRef, Scope *S,
+                                            Expr *BeginRange, Expr *EndRange,
+                                            QualType RangeType,
+                                            VarDecl *BeginVar,
+                                            VarDecl *EndVar,
+                                            SourceLocation ColonLoc,
+                                            OverloadCandidateSet *CandidateSet,
+                                            ExprResult *BeginExpr,
+                                            ExprResult *EndExpr,
+                                            Sema::BeginEndFunction *BEF) {
+  DeclarationNameInfo BeginNameInfo(
+      &SemaRef.PP.getIdentifierTable().get("begin"), ColonLoc);
+  DeclarationNameInfo EndNameInfo(&SemaRef.PP.getIdentifierTable().get("end"),
+                                  ColonLoc);
+
+  LookupResult BeginMemberLookup(SemaRef, BeginNameInfo,
+                                 Sema::LookupMemberName);
+  LookupResult EndMemberLookup(SemaRef, EndNameInfo, Sema::LookupMemberName);
+
+  if (CXXRecordDecl *D = RangeType->getAsCXXRecordDecl()) {
+    // - if _RangeT is a class type, the unqualified-ids begin and end are
+    //   looked up in the scope of class _RangeT as if by class member access
+    //   lookup (3.4.5), and if either (or both) finds at least one
+    //   declaration, begin-expr and end-expr are __range.begin() and
+    //   __range.end(), respectively;
+    SemaRef.LookupQualifiedName(BeginMemberLookup, D);
+    SemaRef.LookupQualifiedName(EndMemberLookup, D);
+
+    if (BeginMemberLookup.empty() != EndMemberLookup.empty()) {
+      SourceLocation RangeLoc = BeginVar->getLocation();
+      *BEF = BeginMemberLookup.empty() ? Sema::BEF_end : Sema::BEF_begin;
+
+      SemaRef.Diag(RangeLoc, diag::err_for_range_member_begin_end_mismatch)
+          << RangeLoc << BeginRange->getType() << *BEF;
+      return Sema::FRS_DiagnosticIssued;
+    }
+  } else {
+    // - otherwise, begin-expr and end-expr are begin(__range) and
+    //   end(__range), respectively, where begin and end are looked up with
+    //   argument-dependent lookup (3.4.2). For the purposes of this name
+    //   lookup, namespace std is an associated namespace.
+
+  }
+
+  *BEF = Sema::BEF_begin;
+  Sema::ForRangeStatus RangeStatus =
+      SemaRef.BuildForRangeBeginEndCall(S, ColonLoc, ColonLoc, BeginVar,
+                                        Sema::BEF_begin, BeginNameInfo,
+                                        BeginMemberLookup, CandidateSet,
+                                        BeginRange, BeginExpr);
+
+  if (RangeStatus != Sema::FRS_Success)
+    return RangeStatus;
+  if (FinishForRangeVarDecl(SemaRef, BeginVar, BeginExpr->get(), ColonLoc,
+                            diag::err_for_range_iter_deduction_failure)) {
+    NoteForRangeBeginEndFunction(SemaRef, BeginExpr->get(), *BEF);
+    return Sema::FRS_DiagnosticIssued;
+  }
+
+  *BEF = Sema::BEF_end;
+  RangeStatus =
+      SemaRef.BuildForRangeBeginEndCall(S, ColonLoc, ColonLoc, EndVar,
+                                        Sema::BEF_end, EndNameInfo,
+                                        EndMemberLookup, CandidateSet,
+                                        EndRange, EndExpr);
+  if (RangeStatus != Sema::FRS_Success)
+    return RangeStatus;
+  if (FinishForRangeVarDecl(SemaRef, EndVar, EndExpr->get(), ColonLoc,
+                            diag::err_for_range_iter_deduction_failure)) {
+    NoteForRangeBeginEndFunction(SemaRef, EndExpr->get(), *BEF);
+    return Sema::FRS_DiagnosticIssued;
+  }
+  return Sema::FRS_Success;
+}
+
+/// Speculatively attempt to dereference an invalid range expression.
+/// If the attempt fails, this function will return a valid, null StmtResult
+/// and emit no diagnostics.
+static StmtResult RebuildForRangeWithDereference(Sema &SemaRef, Scope *S,
+                                                 SourceLocation ForLoc,
+                                                 Stmt *LoopVarDecl,
+                                                 SourceLocation ColonLoc,
+                                                 Expr *Range,
+                                                 SourceLocation RangeLoc,
+                                                 SourceLocation RParenLoc) {
+  // Determine whether we can rebuild the for-range statement with a
+  // dereferenced range expression.
+  ExprResult AdjustedRange;
+  {
+    Sema::SFINAETrap Trap(SemaRef);
+
+    AdjustedRange = SemaRef.BuildUnaryOp(S, RangeLoc, UO_Deref, Range);
+    if (AdjustedRange.isInvalid())
+      return StmtResult();
+
+    StmtResult SR =
+      SemaRef.ActOnCXXForRangeStmt(ForLoc, LoopVarDecl, ColonLoc,
+                                   AdjustedRange.get(), RParenLoc,
+                                   Sema::BFRK_Check);
+    if (SR.isInvalid())
+      return StmtResult();
+  }
+
+  // The attempt to dereference worked well enough that it could produce a valid
+  // loop. Produce a fixit, and rebuild the loop with diagnostics enabled, in
+  // case there are any other (non-fatal) problems with it.
+  SemaRef.Diag(RangeLoc, diag::err_for_range_dereference)
+    << Range->getType() << FixItHint::CreateInsertion(RangeLoc, "*");
+  return SemaRef.ActOnCXXForRangeStmt(ForLoc, LoopVarDecl, ColonLoc,
+                                      AdjustedRange.get(), RParenLoc,
+                                      Sema::BFRK_Rebuild);
+}
+
+/// BuildCXXForRangeStmt - Build or instantiate a C++11 for-range statement.
 StmtResult
 Sema::BuildCXXForRangeStmt(SourceLocation ForLoc, SourceLocation ColonLoc,
                            Stmt *RangeDecl, Stmt *BeginEnd, Expr *Cond,
                            Expr *Inc, Stmt *LoopVarDecl,
-                           SourceLocation RParenLoc) {
+                           SourceLocation RParenLoc, BuildForRangeKind Kind) {
   Scope *S = getCurScope();
 
   DeclStmt *RangeDS = cast<DeclStmt>(RangeDecl);
@@ -1855,50 +1912,43 @@ Sema::BuildCXXForRangeStmt(SourceLocation ForLoc, SourceLocation ColonLoc,
         return StmtError();
       }
     } else {
-      DeclarationNameInfo BeginNameInfo(&PP.getIdentifierTable().get("begin"),
-                                        ColonLoc);
-      DeclarationNameInfo EndNameInfo(&PP.getIdentifierTable().get("end"),
-                                      ColonLoc);
-
-      LookupResult BeginMemberLookup(*this, BeginNameInfo, LookupMemberName);
-      LookupResult EndMemberLookup(*this, EndNameInfo, LookupMemberName);
-
-      if (CXXRecordDecl *D = RangeType->getAsCXXRecordDecl()) {
-        // - if _RangeT is a class type, the unqualified-ids begin and end are
-        //   looked up in the scope of class _RangeT as if by class member access
-        //   lookup (3.4.5), and if either (or both) finds at least one
-        //   declaration, begin-expr and end-expr are __range.begin() and
-        //   __range.end(), respectively;
-        LookupQualifiedName(BeginMemberLookup, D);
-        LookupQualifiedName(EndMemberLookup, D);
-
-        if (BeginMemberLookup.empty() != EndMemberLookup.empty()) {
-          Diag(ColonLoc, diag::err_for_range_member_begin_end_mismatch)
-            << RangeType << BeginMemberLookup.empty();
-          return StmtError();
-        }
-      } else {
-        // - otherwise, begin-expr and end-expr are begin(__range) and
-        //   end(__range), respectively, where begin and end are looked up with
-        //   argument-dependent lookup (3.4.2). For the purposes of this name
-        //   lookup, namespace std is an associated namespace.
+      OverloadCandidateSet CandidateSet(RangeLoc);
+      Sema::BeginEndFunction BEFFailure;
+      ForRangeStatus RangeStatus =
+          BuildNonArrayForRange(*this, S, BeginRangeRef.get(),
+                                EndRangeRef.get(), RangeType,
+                                BeginVar, EndVar, ColonLoc, &CandidateSet,
+                                &BeginExpr, &EndExpr, &BEFFailure);
+
+      // If building the range failed, try dereferencing the range expression
+      // unless a diagnostic was issued or the end function is problematic.
+      if (Kind == BFRK_Build && RangeStatus == FRS_NoViableFunction &&
+          BEFFailure == BEF_begin) {
+        StmtResult SR = RebuildForRangeWithDereference(*this, S, ForLoc,
+                                                       LoopVarDecl, ColonLoc,
+                                                       Range, RangeLoc,
+                                                       RParenLoc);
+        if (SR.isInvalid() || SR.isUsable())
+          return SR;
       }
 
-      BeginExpr = BuildForRangeBeginEndCall(*this, S, ColonLoc, BeginVar,
-                                            BEF_begin, BeginNameInfo,
-                                            BeginMemberLookup,
-                                            BeginRangeRef.get());
-      if (BeginExpr.isInvalid())
-        return StmtError();
-
-      EndExpr = BuildForRangeBeginEndCall(*this, S, ColonLoc, EndVar,
-                                          BEF_end, EndNameInfo,
-                                          EndMemberLookup, EndRangeRef.get());
-      if (EndExpr.isInvalid())
+      // Otherwise, emit diagnostics if we haven't already.
+      if (RangeStatus == FRS_NoViableFunction) {
+        Expr *Range = BEFFailure ? EndRangeRef.get() : BeginRangeRef.get();
+        Diag(Range->getLocStart(), diag::err_for_range_invalid)
+            << RangeLoc << Range->getType() << BEFFailure;
+        CandidateSet.NoteCandidates(*this, OCD_AllCandidates,
+                                    llvm::makeArrayRef(&Range, /*NumArgs=*/1));
+      }
+      // Return an error if no fix was discovered.
+      if (RangeStatus != FRS_Success)
         return StmtError();
     }
 
-    // C++0x [decl.spec.auto]p6: BeginType and EndType must be the same.
+    assert(!BeginExpr.isInvalid() && !EndExpr.isInvalid() &&
+           "invalid range expression in for loop");
+
+    // C++11 [dcl.spec.auto]p7: BeginType and EndType must be the same.
     QualType BeginType = BeginVar->getType(), EndType = EndVar->getType();
     if (!Context.hasSameType(BeginType, EndType)) {
       Diag(RangeLoc, diag::err_for_range_begin_end_types_differ)
@@ -1930,6 +1980,8 @@ Sema::BuildCXXForRangeStmt(SourceLocation ForLoc, SourceLocation ColonLoc,
     NotEqExpr = ActOnBooleanCondition(S, ColonLoc, NotEqExpr.get());
     NotEqExpr = ActOnFinishFullExpr(NotEqExpr.get());
     if (NotEqExpr.isInvalid()) {
+      Diag(RangeLoc, diag::note_for_range_invalid_iterator)
+        << RangeLoc << 0 << BeginRangeRef.get()->getType();
       NoteForRangeBeginEndFunction(*this, BeginExpr.get(), BEF_begin);
       if (!Context.hasSameType(BeginType, EndType))
         NoteForRangeBeginEndFunction(*this, EndExpr.get(), BEF_end);
@@ -1945,6 +1997,8 @@ Sema::BuildCXXForRangeStmt(SourceLocation ForLoc, SourceLocation ColonLoc,
     IncrExpr = ActOnUnaryOp(S, ColonLoc, tok::plusplus, BeginRef.get());
     IncrExpr = ActOnFinishFullExpr(IncrExpr.get());
     if (IncrExpr.isInvalid()) {
+      Diag(RangeLoc, diag::note_for_range_invalid_iterator)
+        << RangeLoc << 2 << BeginRangeRef.get()->getType() ;
       NoteForRangeBeginEndFunction(*this, BeginExpr.get(), BEF_begin);
       return StmtError();
     }
@@ -1957,12 +2011,15 @@ Sema::BuildCXXForRangeStmt(SourceLocation ForLoc, SourceLocation ColonLoc,
 
     ExprResult DerefExpr = ActOnUnaryOp(S, ColonLoc, tok::star, BeginRef.get());
     if (DerefExpr.isInvalid()) {
+      Diag(RangeLoc, diag::note_for_range_invalid_iterator)
+        << RangeLoc << 1 << BeginRangeRef.get()->getType();
       NoteForRangeBeginEndFunction(*this, BeginExpr.get(), BEF_begin);
       return StmtError();
     }
 
-    // Attach  *__begin  as initializer for VD.
-    if (!LoopVar->isInvalidDecl()) {
+    // Attach  *__begin  as initializer for VD. Don't touch it if we're just
+    // trying to determine whether this would be a valid range.
+    if (!LoopVar->isInvalidDecl() && Kind != BFRK_Check) {
       AddInitializerToDecl(LoopVar, DerefExpr.get(), /*DirectInit=*/false,
                            /*TypeMayContainAuto=*/true);
       if (LoopVar->isInvalidDecl())
@@ -1973,6 +2030,11 @@ Sema::BuildCXXForRangeStmt(SourceLocation ForLoc, SourceLocation ColonLoc,
     RangeVar->setUsed();
   }
 
+  // Don't bother to actually allocate the result if we're just trying to
+  // determine whether it would be valid.
+  if (Kind == BFRK_Check)
+    return StmtResult();
+
   return Owned(new (Context) CXXForRangeStmt(RangeDS,
                                      cast_or_null<DeclStmt>(BeginEndDecl.get()),
                                              NotEqExpr.take(), IncrExpr.take(),
@@ -2485,600 +2547,6 @@ Sema::ActOnReturnStmt(SourceLocation ReturnLoc, Expr *RetValExp) {
   return Owned(Result);
 }
 
-/// CheckAsmLValue - GNU C has an extremely ugly extension whereby they silently
-/// ignore "noop" casts in places where an lvalue is required by an inline asm.
-/// We emulate this behavior when -fheinous-gnu-extensions is specified, but
-/// provide a strong guidance to not use it.
-///
-/// This method checks to see if the argument is an acceptable l-value and
-/// returns false if it is a case we can handle.
-static bool CheckAsmLValue(const Expr *E, Sema &S) {
-  // Type dependent expressions will be checked during instantiation.
-  if (E->isTypeDependent())
-    return false;
-
-  if (E->isLValue())
-    return false;  // Cool, this is an lvalue.
-
-  // Okay, this is not an lvalue, but perhaps it is the result of a cast that we
-  // are supposed to allow.
-  const Expr *E2 = E->IgnoreParenNoopCasts(S.Context);
-  if (E != E2 && E2->isLValue()) {
-    if (!S.getLangOpts().HeinousExtensions)
-      S.Diag(E2->getLocStart(), diag::err_invalid_asm_cast_lvalue)
-        << E->getSourceRange();
-    else
-      S.Diag(E2->getLocStart(), diag::warn_invalid_asm_cast_lvalue)
-        << E->getSourceRange();
-    // Accept, even if we emitted an error diagnostic.
-    return false;
-  }
-
-  // None of the above, just randomly invalid non-lvalue.
-  return true;
-}
-
-/// isOperandMentioned - Return true if the specified operand # is mentioned
-/// anywhere in the decomposed asm string.
-static bool isOperandMentioned(unsigned OpNo,
-                         ArrayRef<AsmStmt::AsmStringPiece> AsmStrPieces) {
-  for (unsigned p = 0, e = AsmStrPieces.size(); p != e; ++p) {
-    const AsmStmt::AsmStringPiece &Piece = AsmStrPieces[p];
-    if (!Piece.isOperand()) continue;
-
-    // If this is a reference to the input and if the input was the smaller
-    // one, then we have to reject this asm.
-    if (Piece.getOperandNo() == OpNo)
-      return true;
-  }
-  return false;
-}
-
-StmtResult Sema::ActOnAsmStmt(SourceLocation AsmLoc, bool IsSimple,
-                              bool IsVolatile, unsigned NumOutputs,
-                              unsigned NumInputs, IdentifierInfo **Names,
-                              MultiExprArg constraints, MultiExprArg exprs,
-                              Expr *asmString, MultiExprArg clobbers,
-                              SourceLocation RParenLoc, bool MSAsm) {
-  unsigned NumClobbers = clobbers.size();
-  StringLiteral **Constraints =
-    reinterpret_cast<StringLiteral**>(constraints.get());
-  Expr **Exprs = exprs.get();
-  StringLiteral *AsmString = cast<StringLiteral>(asmString);
-  StringLiteral **Clobbers = reinterpret_cast<StringLiteral**>(clobbers.get());
-
-  SmallVector<TargetInfo::ConstraintInfo, 4> OutputConstraintInfos;
-
-  // The parser verifies that there is a string literal here.
-  if (!AsmString->isAscii())
-    return StmtError(Diag(AsmString->getLocStart(),diag::err_asm_wide_character)
-      << AsmString->getSourceRange());
-
-  for (unsigned i = 0; i != NumOutputs; i++) {
-    StringLiteral *Literal = Constraints[i];
-    if (!Literal->isAscii())
-      return StmtError(Diag(Literal->getLocStart(),diag::err_asm_wide_character)
-        << Literal->getSourceRange());
-
-    StringRef OutputName;
-    if (Names[i])
-      OutputName = Names[i]->getName();
-
-    TargetInfo::ConstraintInfo Info(Literal->getString(), OutputName);
-    if (!Context.getTargetInfo().validateOutputConstraint(Info))
-      return StmtError(Diag(Literal->getLocStart(),
-                            diag::err_asm_invalid_output_constraint)
-                       << Info.getConstraintStr());
-
-    // Check that the output exprs are valid lvalues.
-    Expr *OutputExpr = Exprs[i];
-    if (CheckAsmLValue(OutputExpr, *this)) {
-      return StmtError(Diag(OutputExpr->getLocStart(),
-                  diag::err_asm_invalid_lvalue_in_output)
-        << OutputExpr->getSourceRange());
-    }
-
-    OutputConstraintInfos.push_back(Info);
-  }
-
-  SmallVector<TargetInfo::ConstraintInfo, 4> InputConstraintInfos;
-
-  for (unsigned i = NumOutputs, e = NumOutputs + NumInputs; i != e; i++) {
-    StringLiteral *Literal = Constraints[i];
-    if (!Literal->isAscii())
-      return StmtError(Diag(Literal->getLocStart(),diag::err_asm_wide_character)
-        << Literal->getSourceRange());
-
-    StringRef InputName;
-    if (Names[i])
-      InputName = Names[i]->getName();
-
-    TargetInfo::ConstraintInfo Info(Literal->getString(), InputName);
-    if (!Context.getTargetInfo().validateInputConstraint(OutputConstraintInfos.data(),
-                                                NumOutputs, Info)) {
-      return StmtError(Diag(Literal->getLocStart(),
-                            diag::err_asm_invalid_input_constraint)
-                       << Info.getConstraintStr());
-    }
-
-    Expr *InputExpr = Exprs[i];
-
-    // Only allow void types for memory constraints.
-    if (Info.allowsMemory() && !Info.allowsRegister()) {
-      if (CheckAsmLValue(InputExpr, *this))
-        return StmtError(Diag(InputExpr->getLocStart(),
-                              diag::err_asm_invalid_lvalue_in_input)
-                         << Info.getConstraintStr()
-                         << InputExpr->getSourceRange());
-    }
-
-    if (Info.allowsRegister()) {
-      if (InputExpr->getType()->isVoidType()) {
-        return StmtError(Diag(InputExpr->getLocStart(),
-                              diag::err_asm_invalid_type_in_input)
-          << InputExpr->getType() << Info.getConstraintStr()
-          << InputExpr->getSourceRange());
-      }
-    }
-
-    ExprResult Result = DefaultFunctionArrayLvalueConversion(Exprs[i]);
-    if (Result.isInvalid())
-      return StmtError();
-
-    Exprs[i] = Result.take();
-    InputConstraintInfos.push_back(Info);
-  }
-
-  // Check that the clobbers are valid.
-  for (unsigned i = 0; i != NumClobbers; i++) {
-    StringLiteral *Literal = Clobbers[i];
-    if (!Literal->isAscii())
-      return StmtError(Diag(Literal->getLocStart(),diag::err_asm_wide_character)
-        << Literal->getSourceRange());
-
-    StringRef Clobber = Literal->getString();
-
-    if (!Context.getTargetInfo().isValidClobber(Clobber))
-      return StmtError(Diag(Literal->getLocStart(),
-                  diag::err_asm_unknown_register_name) << Clobber);
-  }
-
-  AsmStmt *NS =
-    new (Context) AsmStmt(Context, AsmLoc, IsSimple, IsVolatile, MSAsm,
-                          NumOutputs, NumInputs, Names, Constraints, Exprs,
-                          AsmString, NumClobbers, Clobbers, RParenLoc);
-  // Validate the asm string, ensuring it makes sense given the operands we
-  // have.
-  SmallVector<AsmStmt::AsmStringPiece, 8> Pieces;
-  unsigned DiagOffs;
-  if (unsigned DiagID = NS->AnalyzeAsmString(Pieces, Context, DiagOffs)) {
-    Diag(getLocationOfStringLiteralByte(AsmString, DiagOffs), DiagID)
-           << AsmString->getSourceRange();
-    return StmtError();
-  }
-
-  // Validate tied input operands for type mismatches.
-  for (unsigned i = 0, e = InputConstraintInfos.size(); i != e; ++i) {
-    TargetInfo::ConstraintInfo &Info = InputConstraintInfos[i];
-
-    // If this is a tied constraint, verify that the output and input have
-    // either exactly the same type, or that they are int/ptr operands with the
-    // same size (int/long, int*/long, are ok etc).
-    if (!Info.hasTiedOperand()) continue;
-
-    unsigned TiedTo = Info.getTiedOperand();
-    unsigned InputOpNo = i+NumOutputs;
-    Expr *OutputExpr = Exprs[TiedTo];
-    Expr *InputExpr = Exprs[InputOpNo];
-
-    if (OutputExpr->isTypeDependent() || InputExpr->isTypeDependent())
-      continue;
-
-    QualType InTy = InputExpr->getType();
-    QualType OutTy = OutputExpr->getType();
-    if (Context.hasSameType(InTy, OutTy))
-      continue;  // All types can be tied to themselves.
-
-    // Decide if the input and output are in the same domain (integer/ptr or
-    // floating point.
-    enum AsmDomain {
-      AD_Int, AD_FP, AD_Other
-    } InputDomain, OutputDomain;
-
-    if (InTy->isIntegerType() || InTy->isPointerType())
-      InputDomain = AD_Int;
-    else if (InTy->isRealFloatingType())
-      InputDomain = AD_FP;
-    else
-      InputDomain = AD_Other;
-
-    if (OutTy->isIntegerType() || OutTy->isPointerType())
-      OutputDomain = AD_Int;
-    else if (OutTy->isRealFloatingType())
-      OutputDomain = AD_FP;
-    else
-      OutputDomain = AD_Other;
-
-    // They are ok if they are the same size and in the same domain.  This
-    // allows tying things like:
-    //   void* to int*
-    //   void* to int            if they are the same size.
-    //   double to long double   if they are the same size.
-    //
-    uint64_t OutSize = Context.getTypeSize(OutTy);
-    uint64_t InSize = Context.getTypeSize(InTy);
-    if (OutSize == InSize && InputDomain == OutputDomain &&
-        InputDomain != AD_Other)
-      continue;
-
-    // If the smaller input/output operand is not mentioned in the asm string,
-    // then we can promote the smaller one to a larger input and the asm string
-    // won't notice.
-    bool SmallerValueMentioned = false;
-
-    // If this is a reference to the input and if the input was the smaller
-    // one, then we have to reject this asm.
-    if (isOperandMentioned(InputOpNo, Pieces)) {
-      // This is a use in the asm string of the smaller operand.  Since we
-      // codegen this by promoting to a wider value, the asm will get printed
-      // "wrong".
-      SmallerValueMentioned |= InSize < OutSize;
-    }
-    if (isOperandMentioned(TiedTo, Pieces)) {
-      // If this is a reference to the output, and if the output is the larger
-      // value, then it's ok because we'll promote the input to the larger type.
-      SmallerValueMentioned |= OutSize < InSize;
-    }
-
-    // If the smaller value wasn't mentioned in the asm string, and if the
-    // output was a register, just extend the shorter one to the size of the
-    // larger one.
-    if (!SmallerValueMentioned && InputDomain != AD_Other &&
-        OutputConstraintInfos[TiedTo].allowsRegister())
-      continue;
-
-    // Either both of the operands were mentioned or the smaller one was
-    // mentioned.  One more special case that we'll allow: if the tied input is
-    // integer, unmentioned, and is a constant, then we'll allow truncating it
-    // down to the size of the destination.
-    if (InputDomain == AD_Int && OutputDomain == AD_Int &&
-        !isOperandMentioned(InputOpNo, Pieces) &&
-        InputExpr->isEvaluatable(Context)) {
-      CastKind castKind =
-        (OutTy->isBooleanType() ? CK_IntegralToBoolean : CK_IntegralCast);
-      InputExpr = ImpCastExprToType(InputExpr, OutTy, castKind).take();
-      Exprs[InputOpNo] = InputExpr;
-      NS->setInputExpr(i, InputExpr);
-      continue;
-    }
-
-    Diag(InputExpr->getLocStart(),
-         diag::err_asm_tying_incompatible_types)
-      << InTy << OutTy << OutputExpr->getSourceRange()
-      << InputExpr->getSourceRange();
-    return StmtError();
-  }
-
-  return Owned(NS);
-}
-
-// isMSAsmKeyword - Return true if this is an MS-style inline asm keyword. These
-// require special handling.
-static bool isMSAsmKeyword(StringRef Name) {
-  bool Ret = llvm::StringSwitch<bool>(Name)
-    .Cases("EVEN", "ALIGN", true) // Alignment directives.
-    .Cases("LENGTH", "SIZE", "TYPE", true) // Type and variable sizes.
-    .Case("_emit", true) // _emit Pseudoinstruction.
-    .Default(false);
-  return Ret;
-}
-
-static StringRef getSpelling(Sema &SemaRef, Token AsmTok) {
-  StringRef Asm;
-  SmallString<512> TokenBuf;
-  TokenBuf.resize(512);
-  bool StringInvalid = false;
-  Asm = SemaRef.PP.getSpelling(AsmTok, TokenBuf, &StringInvalid);
-  assert (!StringInvalid && "Expected valid string!");
-  return Asm;
-}
-
-static void patchMSAsmStrings(Sema &SemaRef, bool &IsSimple,
-                              SourceLocation AsmLoc,
-                              ArrayRef<Token> AsmToks,
-                              const TargetInfo &TI,
-                              std::vector<llvm::BitVector> &AsmRegs,
-                              std::vector<llvm::BitVector> &AsmNames,
-                              std::vector<std::string> &AsmStrings) {
-  assert (!AsmToks.empty() && "Didn't expect an empty AsmToks!");
-
-  // Assume simple asm stmt until we parse a non-register identifer (or we just
-  // need to bail gracefully).
-  IsSimple = true;
-
-  SmallString<512> Asm;
-  unsigned NumAsmStrings = 0;
-  for (unsigned i = 0, e = AsmToks.size(); i != e; ++i) {
-
-    // Determine if this should be considered a new asm.
-    bool isNewAsm = i == 0 || AsmToks[i].isAtStartOfLine() ||
-      AsmToks[i].is(tok::kw_asm);
-
-    // Emit the previous asm string.
-    if (i && isNewAsm) {
-      AsmStrings[NumAsmStrings++] = Asm.c_str();
-      if (AsmToks[i].is(tok::kw_asm)) {
-        ++i; // Skip __asm
-        assert (i != e && "Expected another token.");
-      }
-    }
-
-    // Start a new asm string with the opcode.
-    if (isNewAsm) {
-      AsmRegs[NumAsmStrings].resize(AsmToks.size());
-      AsmNames[NumAsmStrings].resize(AsmToks.size());
-
-      StringRef Piece = AsmToks[i].getIdentifierInfo()->getName();
-      // MS-style inline asm keywords require special handling.
-      if (isMSAsmKeyword(Piece))
-        IsSimple = false;
-
-      // TODO: Verify this is a valid opcode.
-      Asm = Piece;
-      continue;
-    }
-
-    if (i && AsmToks[i].hasLeadingSpace())
-      Asm += ' ';
-
-    // Check the operand(s).
-    switch (AsmToks[i].getKind()) {
-    default:
-      IsSimple = false;
-      Asm += getSpelling(SemaRef, AsmToks[i]);
-      break;
-    case tok::comma: Asm += ","; break;
-    case tok::colon: Asm += ":"; break;
-    case tok::l_square: Asm += "["; break;
-    case tok::r_square: Asm += "]"; break;
-    case tok::l_brace: Asm += "{"; break;
-    case tok::r_brace: Asm += "}"; break;
-    case tok::numeric_constant:
-      Asm += getSpelling(SemaRef, AsmToks[i]);
-      break;
-    case tok::identifier: {
-      IdentifierInfo *II = AsmToks[i].getIdentifierInfo();
-      StringRef Name = II->getName();
-
-      // Valid register?
-      if (TI.isValidGCCRegisterName(Name)) {
-        AsmRegs[NumAsmStrings].set(i);
-        Asm += Name;
-        break;
-      }
-
-      IsSimple = false;
-
-      // MS-style inline asm keywords require special handling.
-      if (isMSAsmKeyword(Name)) {
-        IsSimple = false;
-        Asm += Name;
-        break;
-      }
-
-      // FIXME: Why are we missing this segment register?
-      if (Name == "fs") {
-        Asm += Name;
-        break;
-      }
-
-      // Lookup the identifier.
-      // TODO: Someone with more experience with clang should verify this the
-      // proper way of doing a symbol lookup.
-      DeclarationName DeclName(II);
-      Scope *CurScope = SemaRef.getCurScope();
-      LookupResult R(SemaRef, DeclName, AsmLoc, Sema::LookupOrdinaryName);
-      if (!SemaRef.LookupName(R, CurScope, false/*AllowBuiltinCreation*/))
-        break;
-
-      assert (R.isSingleResult() && "Expected a single result?!");
-      NamedDecl *Decl = R.getFoundDecl();
-      switch (Decl->getKind()) {
-      default:
-        assert(0 && "Unknown decl kind.");
-        break;
-      case Decl::Var: {
-      case Decl::ParmVar:
-        AsmNames[NumAsmStrings].set(i);
-
-        VarDecl *Var = cast<VarDecl>(Decl);
-        QualType Ty = Var->getType();
-        (void)Ty; // Avoid warning.
-        // TODO: Patch identifier with valid operand.  One potential idea is to
-        // probe the backend with type information to guess the possible
-        // operand.
-        break;
-      }
-      }
-      break;
-    }
-    }
-  }
-
-  // Emit the final (and possibly only) asm string.
-  AsmStrings[NumAsmStrings] = Asm.c_str();
-}
-
-// Build the unmodified MSAsmString.
-static std::string buildMSAsmString(Sema &SemaRef,
-                                    ArrayRef<Token> AsmToks,
-                                    unsigned &NumAsmStrings) {
-  assert (!AsmToks.empty() && "Didn't expect an empty AsmToks!");
-  NumAsmStrings = 0;
-
-  SmallString<512> Asm;
-  for (unsigned i = 0, e = AsmToks.size(); i < e; ++i) {
-    bool isNewAsm = i == 0 || AsmToks[i].isAtStartOfLine() ||
-      AsmToks[i].is(tok::kw_asm);
-
-    if (isNewAsm) {
-      ++NumAsmStrings;
-      if (i)
-        Asm += '\n';
-      if (AsmToks[i].is(tok::kw_asm)) {
-        i++; // Skip __asm
-        assert (i != e && "Expected another token");
-      }
-    }
-
-    if (i && AsmToks[i].hasLeadingSpace() && !isNewAsm)
-      Asm += ' ';
-
-    Asm += getSpelling(SemaRef, AsmToks[i]);
-  }
-  return Asm.c_str();
-}
-
-StmtResult Sema::ActOnMSAsmStmt(SourceLocation AsmLoc,
-                                SourceLocation LBraceLoc,
-                                ArrayRef<Token> AsmToks,
-                                SourceLocation EndLoc) {
-  // MS-style inline assembly is not fully supported, so emit a warning.
-  Diag(AsmLoc, diag::warn_unsupported_msasm);
-  SmallVector<StringRef,4> Clobbers;
-  std::set<std::string> ClobberRegs;
-  SmallVector<IdentifierInfo*, 4> Inputs;
-  SmallVector<IdentifierInfo*, 4> Outputs;
-
-  // Empty asm statements don't need to instantiate the AsmParser, etc.
-  if (AsmToks.empty()) {
-    StringRef AsmString;
-    MSAsmStmt *NS =
-      new (Context) MSAsmStmt(Context, AsmLoc, LBraceLoc, /*IsSimple*/ true,
-                              /*IsVolatile*/ true, AsmToks, Inputs, Outputs,
-                              AsmString, Clobbers, EndLoc);
-    return Owned(NS);
-  }
-
-  unsigned NumAsmStrings;
-  std::string AsmString = buildMSAsmString(*this, AsmToks, NumAsmStrings);
-
-  bool IsSimple;
-  std::vector<llvm::BitVector> Regs;
-  std::vector<llvm::BitVector> Names;
-  std::vector<std::string> PatchedAsmStrings;
-
-  Regs.resize(NumAsmStrings);
-  Names.resize(NumAsmStrings);
-  PatchedAsmStrings.resize(NumAsmStrings);
-
-  // Rewrite operands to appease the AsmParser.
-  patchMSAsmStrings(*this, IsSimple, AsmLoc, AsmToks,
-                    Context.getTargetInfo(), Regs, Names, PatchedAsmStrings);
-
-  // patchMSAsmStrings doesn't correctly patch non-simple asm statements.
-  if (!IsSimple) {
-    MSAsmStmt *NS =
-      new (Context) MSAsmStmt(Context, AsmLoc, LBraceLoc, /*IsSimple*/ true,
-                              /*IsVolatile*/ true, AsmToks, Inputs, Outputs,
-                              AsmString, Clobbers, EndLoc);
-    return Owned(NS);
-  }
-
-  // Initialize targets and assembly printers/parsers.
-  llvm::InitializeAllTargetInfos();
-  llvm::InitializeAllTargetMCs();
-  llvm::InitializeAllAsmParsers();
-
-  // Get the target specific parser.
-  std::string Error;
-  const std::string &TT = Context.getTargetInfo().getTriple().getTriple();
-  const llvm::Target *TheTarget(llvm::TargetRegistry::lookupTarget(TT, Error));
-
-  OwningPtr<llvm::MCAsmInfo> MAI(TheTarget->createMCAsmInfo(TT));
-  OwningPtr<llvm::MCRegisterInfo> MRI(TheTarget->createMCRegInfo(TT));
-  OwningPtr<llvm::MCObjectFileInfo> MOFI(new llvm::MCObjectFileInfo());
-  OwningPtr<llvm::MCSubtargetInfo>
-    STI(TheTarget->createMCSubtargetInfo(TT, "", ""));
-
-  for (unsigned i = 0, e = PatchedAsmStrings.size(); i != e; ++i) {
-    llvm::SourceMgr SrcMgr;
-    llvm::MCContext Ctx(*MAI, *MRI, MOFI.get(), &SrcMgr);
-    llvm::MemoryBuffer *Buffer =
-      llvm::MemoryBuffer::getMemBuffer(PatchedAsmStrings[i], "<inline asm>");
-
-    // Tell SrcMgr about this buffer, which is what the parser will pick up.
-    SrcMgr.AddNewSourceBuffer(Buffer, llvm::SMLoc());
-
-    OwningPtr<llvm::MCStreamer> Str(createNullStreamer(Ctx));
-    OwningPtr<llvm::MCAsmParser>
-      Parser(createMCAsmParser(SrcMgr, Ctx, *Str.get(), *MAI));
-    OwningPtr<llvm::MCTargetAsmParser>
-      TargetParser(TheTarget->createMCAsmParser(*STI, *Parser));
-    // Change to the Intel dialect.
-    Parser->setAssemblerDialect(1);
-    Parser->setTargetParser(*TargetParser.get());
-
-    // Prime the lexer.
-    Parser->Lex();
-
-    // Parse the opcode.
-    StringRef IDVal;
-    Parser->ParseIdentifier(IDVal);
-
-    // Canonicalize the opcode to lower case.
-    SmallString<128> Opcode;
-    for (unsigned i = 0, e = IDVal.size(); i != e; ++i)
-      Opcode.push_back(tolower(IDVal[i]));
-
-    // Parse the operands.
-    llvm::SMLoc IDLoc;
-    SmallVector<llvm::MCParsedAsmOperand*, 8> Operands;
-    bool HadError = TargetParser->ParseInstruction(Opcode.str(), IDLoc,
-                                                   Operands);
-    assert (!HadError && "Unexpected error parsing instruction");
-
-    // Match the MCInstr.
-    SmallVector<llvm::MCInst, 2> Instrs;
-    HadError = TargetParser->MatchInstruction(IDLoc, Operands, Instrs);
-    assert (!HadError && "Unexpected error matching instruction");
-    assert ((Instrs.size() == 1) && "Expected only a single instruction.");
-
-    // Get the instruction descriptor.
-    llvm::MCInst Inst = Instrs[0];
-    const llvm::MCInstrInfo *MII = TheTarget->createMCInstrInfo();
-    const llvm::MCInstrDesc &Desc = MII->get(Inst.getOpcode());
-    llvm::MCInstPrinter *IP =
-      TheTarget->createMCInstPrinter(1, *MAI, *MII, *MRI, *STI);
-
-    // Build the list of clobbers.
-    for (unsigned i = 0, e = Desc.getNumDefs(); i != e; ++i) {
-      const llvm::MCOperand &Op = Inst.getOperand(i);
-      if (!Op.isReg())
-        continue;
-
-      std::string Reg;
-      llvm::raw_string_ostream OS(Reg);
-      IP->printRegName(OS, Op.getReg());
-
-      StringRef Clobber(OS.str());
-      if (!Context.getTargetInfo().isValidClobber(Clobber))
-        return StmtError(Diag(AsmLoc, diag::err_asm_unknown_register_name) <<
-                         Clobber);
-      ClobberRegs.insert(Reg);
-    }
-  }
-  for (std::set<std::string>::iterator I = ClobberRegs.begin(),
-         E = ClobberRegs.end(); I != E; ++I)
-    Clobbers.push_back(*I);
-
-  MSAsmStmt *NS =
-    new (Context) MSAsmStmt(Context, AsmLoc, LBraceLoc, IsSimple,
-                            /*IsVolatile*/ true, AsmToks, Inputs, Outputs,
-                            AsmString, Clobbers, EndLoc);
-  return Owned(NS);
-}
-
 StmtResult
 Sema::ActOnObjCAtCatchStmt(SourceLocation AtLoc,
                            SourceLocation RParen, Decl *Parm,
@@ -3104,7 +2572,7 @@ Sema::ActOnObjCAtTryStmt(SourceLocation AtLoc, Stmt *Try,
   getCurFunction()->setHasBranchProtectedScope();
   unsigned NumCatchStmts = CatchStmts.size();
   return Owned(ObjCAtTryStmt::Create(Context, AtLoc, Try,
-                                     CatchStmts.release(),
+                                     CatchStmts.data(),
                                      NumCatchStmts,
                                      Finally));
 }
@@ -3239,7 +2707,7 @@ Sema::ActOnCXXTryBlock(SourceLocation TryLoc, Stmt *TryBlock,
   unsigned NumHandlers = RawHandlers.size();
   assert(NumHandlers > 0 &&
          "The parser shouldn't call this if there are no handlers.");
-  Stmt **Handlers = RawHandlers.get();
+  Stmt **Handlers = RawHandlers.data();
 
   SmallVector<TypeWithHandler, 8> TypesWithHandlers;
 
diff --git a/lib/Sema/SemaStmtAsm.cpp b/lib/Sema/SemaStmtAsm.cpp
new file mode 100644
index 0000000..7c2c766
--- /dev/null
+++ b/lib/Sema/SemaStmtAsm.cpp
@@ -0,0 +1,661 @@
+//===--- SemaStmtAsm.cpp - Semantic Analysis for Asm Statements -----------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file implements semantic analysis for inline asm statements.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Sema/SemaInternal.h"
+#include "clang/Sema/Scope.h"
+#include "clang/Sema/ScopeInfo.h"
+#include "clang/Sema/Initialization.h"
+#include "clang/Sema/Lookup.h"
+#include "clang/AST/RecordLayout.h"
+#include "clang/AST/TypeLoc.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Basic/TargetInfo.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/BitVector.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/MC/MCAsmInfo.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCObjectFileInfo.h"
+#include "llvm/MC/MCRegisterInfo.h"
+#include "llvm/MC/MCStreamer.h"
+#include "llvm/MC/MCSubtargetInfo.h"
+#include "llvm/MC/MCTargetAsmParser.h"
+#include "llvm/MC/MCParser/MCAsmParser.h"
+#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/TargetRegistry.h"
+#include "llvm/Support/TargetSelect.h"
+using namespace clang;
+using namespace sema;
+
+/// CheckAsmLValue - GNU C has an extremely ugly extension whereby they silently
+/// ignore "noop" casts in places where an lvalue is required by an inline asm.
+/// We emulate this behavior when -fheinous-gnu-extensions is specified, but
+/// provide a strong guidance to not use it.
+///
+/// This method checks to see if the argument is an acceptable l-value and
+/// returns false if it is a case we can handle.
+static bool CheckAsmLValue(const Expr *E, Sema &S) {
+  // Type dependent expressions will be checked during instantiation.
+  if (E->isTypeDependent())
+    return false;
+
+  if (E->isLValue())
+    return false;  // Cool, this is an lvalue.
+
+  // Okay, this is not an lvalue, but perhaps it is the result of a cast that we
+  // are supposed to allow.
+  const Expr *E2 = E->IgnoreParenNoopCasts(S.Context);
+  if (E != E2 && E2->isLValue()) {
+    if (!S.getLangOpts().HeinousExtensions)
+      S.Diag(E2->getLocStart(), diag::err_invalid_asm_cast_lvalue)
+        << E->getSourceRange();
+    else
+      S.Diag(E2->getLocStart(), diag::warn_invalid_asm_cast_lvalue)
+        << E->getSourceRange();
+    // Accept, even if we emitted an error diagnostic.
+    return false;
+  }
+
+  // None of the above, just randomly invalid non-lvalue.
+  return true;
+}
+
+/// isOperandMentioned - Return true if the specified operand # is mentioned
+/// anywhere in the decomposed asm string.
+static bool isOperandMentioned(unsigned OpNo,
+                         ArrayRef<GCCAsmStmt::AsmStringPiece> AsmStrPieces) {
+  for (unsigned p = 0, e = AsmStrPieces.size(); p != e; ++p) {
+    const GCCAsmStmt::AsmStringPiece &Piece = AsmStrPieces[p];
+    if (!Piece.isOperand()) continue;
+
+    // If this is a reference to the input and if the input was the smaller
+    // one, then we have to reject this asm.
+    if (Piece.getOperandNo() == OpNo)
+      return true;
+  }
+  return false;
+}
+
+StmtResult Sema::ActOnGCCAsmStmt(SourceLocation AsmLoc, bool IsSimple,
+                                 bool IsVolatile, unsigned NumOutputs,
+                                 unsigned NumInputs, IdentifierInfo **Names,
+                                 MultiExprArg constraints, MultiExprArg exprs,
+                                 Expr *asmString, MultiExprArg clobbers,
+                                 SourceLocation RParenLoc) {
+  unsigned NumClobbers = clobbers.size();
+  StringLiteral **Constraints =
+    reinterpret_cast<StringLiteral**>(constraints.data());
+  Expr **Exprs = exprs.data();
+  StringLiteral *AsmString = cast<StringLiteral>(asmString);
+  StringLiteral **Clobbers = reinterpret_cast<StringLiteral**>(clobbers.data());
+
+  SmallVector<TargetInfo::ConstraintInfo, 4> OutputConstraintInfos;
+
+  // The parser verifies that there is a string literal here.
+  if (!AsmString->isAscii())
+    return StmtError(Diag(AsmString->getLocStart(),diag::err_asm_wide_character)
+      << AsmString->getSourceRange());
+
+  for (unsigned i = 0; i != NumOutputs; i++) {
+    StringLiteral *Literal = Constraints[i];
+    if (!Literal->isAscii())
+      return StmtError(Diag(Literal->getLocStart(),diag::err_asm_wide_character)
+        << Literal->getSourceRange());
+
+    StringRef OutputName;
+    if (Names[i])
+      OutputName = Names[i]->getName();
+
+    TargetInfo::ConstraintInfo Info(Literal->getString(), OutputName);
+    if (!Context.getTargetInfo().validateOutputConstraint(Info))
+      return StmtError(Diag(Literal->getLocStart(),
+                            diag::err_asm_invalid_output_constraint)
+                       << Info.getConstraintStr());
+
+    // Check that the output exprs are valid lvalues.
+    Expr *OutputExpr = Exprs[i];
+    if (CheckAsmLValue(OutputExpr, *this)) {
+      return StmtError(Diag(OutputExpr->getLocStart(),
+                  diag::err_asm_invalid_lvalue_in_output)
+        << OutputExpr->getSourceRange());
+    }
+
+    OutputConstraintInfos.push_back(Info);
+  }
+
+  SmallVector<TargetInfo::ConstraintInfo, 4> InputConstraintInfos;
+
+  for (unsigned i = NumOutputs, e = NumOutputs + NumInputs; i != e; i++) {
+    StringLiteral *Literal = Constraints[i];
+    if (!Literal->isAscii())
+      return StmtError(Diag(Literal->getLocStart(),diag::err_asm_wide_character)
+        << Literal->getSourceRange());
+
+    StringRef InputName;
+    if (Names[i])
+      InputName = Names[i]->getName();
+
+    TargetInfo::ConstraintInfo Info(Literal->getString(), InputName);
+    if (!Context.getTargetInfo().validateInputConstraint(OutputConstraintInfos.data(),
+                                                NumOutputs, Info)) {
+      return StmtError(Diag(Literal->getLocStart(),
+                            diag::err_asm_invalid_input_constraint)
+                       << Info.getConstraintStr());
+    }
+
+    Expr *InputExpr = Exprs[i];
+
+    // Only allow void types for memory constraints.
+    if (Info.allowsMemory() && !Info.allowsRegister()) {
+      if (CheckAsmLValue(InputExpr, *this))
+        return StmtError(Diag(InputExpr->getLocStart(),
+                              diag::err_asm_invalid_lvalue_in_input)
+                         << Info.getConstraintStr()
+                         << InputExpr->getSourceRange());
+    }
+
+    if (Info.allowsRegister()) {
+      if (InputExpr->getType()->isVoidType()) {
+        return StmtError(Diag(InputExpr->getLocStart(),
+                              diag::err_asm_invalid_type_in_input)
+          << InputExpr->getType() << Info.getConstraintStr()
+          << InputExpr->getSourceRange());
+      }
+    }
+
+    ExprResult Result = DefaultFunctionArrayLvalueConversion(Exprs[i]);
+    if (Result.isInvalid())
+      return StmtError();
+
+    Exprs[i] = Result.take();
+    InputConstraintInfos.push_back(Info);
+  }
+
+  // Check that the clobbers are valid.
+  for (unsigned i = 0; i != NumClobbers; i++) {
+    StringLiteral *Literal = Clobbers[i];
+    if (!Literal->isAscii())
+      return StmtError(Diag(Literal->getLocStart(),diag::err_asm_wide_character)
+        << Literal->getSourceRange());
+
+    StringRef Clobber = Literal->getString();
+
+    if (!Context.getTargetInfo().isValidClobber(Clobber))
+      return StmtError(Diag(Literal->getLocStart(),
+                  diag::err_asm_unknown_register_name) << Clobber);
+  }
+
+  GCCAsmStmt *NS =
+    new (Context) GCCAsmStmt(Context, AsmLoc, IsSimple, IsVolatile, NumOutputs,
+                             NumInputs, Names, Constraints, Exprs, AsmString,
+                             NumClobbers, Clobbers, RParenLoc);
+  // Validate the asm string, ensuring it makes sense given the operands we
+  // have.
+  SmallVector<GCCAsmStmt::AsmStringPiece, 8> Pieces;
+  unsigned DiagOffs;
+  if (unsigned DiagID = NS->AnalyzeAsmString(Pieces, Context, DiagOffs)) {
+    Diag(getLocationOfStringLiteralByte(AsmString, DiagOffs), DiagID)
+           << AsmString->getSourceRange();
+    return StmtError();
+  }
+
+  // Validate constraints and modifiers.
+  for (unsigned i = 0, e = Pieces.size(); i != e; ++i) {
+    GCCAsmStmt::AsmStringPiece &Piece = Pieces[i];
+    if (!Piece.isOperand()) continue;
+
+    // Look for the correct constraint index.
+    unsigned Idx = 0;
+    unsigned ConstraintIdx = 0;
+    for (unsigned i = 0, e = NS->getNumOutputs(); i != e; ++i, ++ConstraintIdx) {
+      TargetInfo::ConstraintInfo &Info = OutputConstraintInfos[i];
+      if (Idx == Piece.getOperandNo())
+        break;
+      ++Idx;
+
+      if (Info.isReadWrite()) {
+        if (Idx == Piece.getOperandNo())
+          break;
+        ++Idx;
+      }
+    }
+
+    for (unsigned i = 0, e = NS->getNumInputs(); i != e; ++i, ++ConstraintIdx) {
+      TargetInfo::ConstraintInfo &Info = InputConstraintInfos[i];
+      if (Idx == Piece.getOperandNo())
+        break;
+      ++Idx;
+
+      if (Info.isReadWrite()) {
+        if (Idx == Piece.getOperandNo())
+          break;
+        ++Idx;
+      }
+    }
+
+    // Now that we have the right indexes go ahead and check.
+    StringLiteral *Literal = Constraints[ConstraintIdx];
+    const Type *Ty = Exprs[ConstraintIdx]->getType().getTypePtr();
+    if (Ty->isDependentType() || Ty->isIncompleteType())
+      continue;
+
+    unsigned Size = Context.getTypeSize(Ty);
+    if (!Context.getTargetInfo()
+          .validateConstraintModifier(Literal->getString(), Piece.getModifier(),
+                                      Size))
+      Diag(Exprs[ConstraintIdx]->getLocStart(),
+           diag::warn_asm_mismatched_size_modifier);
+  }
+
+  // Validate tied input operands for type mismatches.
+  for (unsigned i = 0, e = InputConstraintInfos.size(); i != e; ++i) {
+    TargetInfo::ConstraintInfo &Info = InputConstraintInfos[i];
+
+    // If this is a tied constraint, verify that the output and input have
+    // either exactly the same type, or that they are int/ptr operands with the
+    // same size (int/long, int*/long, are ok etc).
+    if (!Info.hasTiedOperand()) continue;
+
+    unsigned TiedTo = Info.getTiedOperand();
+    unsigned InputOpNo = i+NumOutputs;
+    Expr *OutputExpr = Exprs[TiedTo];
+    Expr *InputExpr = Exprs[InputOpNo];
+
+    if (OutputExpr->isTypeDependent() || InputExpr->isTypeDependent())
+      continue;
+
+    QualType InTy = InputExpr->getType();
+    QualType OutTy = OutputExpr->getType();
+    if (Context.hasSameType(InTy, OutTy))
+      continue;  // All types can be tied to themselves.
+
+    // Decide if the input and output are in the same domain (integer/ptr or
+    // floating point.
+    enum AsmDomain {
+      AD_Int, AD_FP, AD_Other
+    } InputDomain, OutputDomain;
+
+    if (InTy->isIntegerType() || InTy->isPointerType())
+      InputDomain = AD_Int;
+    else if (InTy->isRealFloatingType())
+      InputDomain = AD_FP;
+    else
+      InputDomain = AD_Other;
+
+    if (OutTy->isIntegerType() || OutTy->isPointerType())
+      OutputDomain = AD_Int;
+    else if (OutTy->isRealFloatingType())
+      OutputDomain = AD_FP;
+    else
+      OutputDomain = AD_Other;
+
+    // They are ok if they are the same size and in the same domain.  This
+    // allows tying things like:
+    //   void* to int*
+    //   void* to int            if they are the same size.
+    //   double to long double   if they are the same size.
+    //
+    uint64_t OutSize = Context.getTypeSize(OutTy);
+    uint64_t InSize = Context.getTypeSize(InTy);
+    if (OutSize == InSize && InputDomain == OutputDomain &&
+        InputDomain != AD_Other)
+      continue;
+
+    // If the smaller input/output operand is not mentioned in the asm string,
+    // then we can promote the smaller one to a larger input and the asm string
+    // won't notice.
+    bool SmallerValueMentioned = false;
+
+    // If this is a reference to the input and if the input was the smaller
+    // one, then we have to reject this asm.
+    if (isOperandMentioned(InputOpNo, Pieces)) {
+      // This is a use in the asm string of the smaller operand.  Since we
+      // codegen this by promoting to a wider value, the asm will get printed
+      // "wrong".
+      SmallerValueMentioned |= InSize < OutSize;
+    }
+    if (isOperandMentioned(TiedTo, Pieces)) {
+      // If this is a reference to the output, and if the output is the larger
+      // value, then it's ok because we'll promote the input to the larger type.
+      SmallerValueMentioned |= OutSize < InSize;
+    }
+
+    // If the smaller value wasn't mentioned in the asm string, and if the
+    // output was a register, just extend the shorter one to the size of the
+    // larger one.
+    if (!SmallerValueMentioned && InputDomain != AD_Other &&
+        OutputConstraintInfos[TiedTo].allowsRegister())
+      continue;
+
+    // Either both of the operands were mentioned or the smaller one was
+    // mentioned.  One more special case that we'll allow: if the tied input is
+    // integer, unmentioned, and is a constant, then we'll allow truncating it
+    // down to the size of the destination.
+    if (InputDomain == AD_Int && OutputDomain == AD_Int &&
+        !isOperandMentioned(InputOpNo, Pieces) &&
+        InputExpr->isEvaluatable(Context)) {
+      CastKind castKind =
+        (OutTy->isBooleanType() ? CK_IntegralToBoolean : CK_IntegralCast);
+      InputExpr = ImpCastExprToType(InputExpr, OutTy, castKind).take();
+      Exprs[InputOpNo] = InputExpr;
+      NS->setInputExpr(i, InputExpr);
+      continue;
+    }
+
+    Diag(InputExpr->getLocStart(),
+         diag::err_asm_tying_incompatible_types)
+      << InTy << OutTy << OutputExpr->getSourceRange()
+      << InputExpr->getSourceRange();
+    return StmtError();
+  }
+
+  return Owned(NS);
+}
+
+// getSpelling - Get the spelling of the AsmTok token.
+static StringRef getSpelling(Sema &SemaRef, Token AsmTok) {
+  StringRef Asm;
+  SmallString<512> TokenBuf;
+  TokenBuf.resize(512);
+  bool StringInvalid = false;
+  Asm = SemaRef.PP.getSpelling(AsmTok, TokenBuf, &StringInvalid);
+  assert (!StringInvalid && "Expected valid string!");
+  return Asm;
+}
+
+// Build the inline assembly string.  Returns true on error.
+static bool buildMSAsmString(Sema &SemaRef,
+                             SourceLocation AsmLoc,
+                             ArrayRef<Token> AsmToks,
+                             llvm::SmallVectorImpl<unsigned> &TokOffsets,
+                             std::string &AsmString) {
+  assert (!AsmToks.empty() && "Didn't expect an empty AsmToks!");
+
+  SmallString<512> Asm;
+  for (unsigned i = 0, e = AsmToks.size(); i < e; ++i) {
+    bool isNewAsm = ((i == 0) ||
+                     AsmToks[i].isAtStartOfLine() ||
+                     AsmToks[i].is(tok::kw_asm));
+    if (isNewAsm) {
+      if (i != 0)
+        Asm += "\n\t";
+
+      if (AsmToks[i].is(tok::kw_asm)) {
+        i++; // Skip __asm
+        if (i == e) {
+          SemaRef.Diag(AsmLoc, diag::err_asm_empty);
+          return true;
+        }
+
+      }
+    }
+
+    if (i && AsmToks[i].hasLeadingSpace() && !isNewAsm)
+      Asm += ' ';
+
+    StringRef Spelling = getSpelling(SemaRef, AsmToks[i]);
+    Asm += Spelling;
+    TokOffsets.push_back(Asm.size());
+  }
+  AsmString = Asm.str();
+  return false;
+}
+
+namespace {
+
+class MCAsmParserSemaCallbackImpl : public llvm::MCAsmParserSemaCallback {
+  Sema &SemaRef;
+  SourceLocation AsmLoc;
+  ArrayRef<Token> AsmToks;
+  ArrayRef<unsigned> TokOffsets;
+
+public:
+  MCAsmParserSemaCallbackImpl(Sema &Ref, SourceLocation Loc,
+                              ArrayRef<Token> Toks,
+                              ArrayRef<unsigned> Offsets)
+    : SemaRef(Ref), AsmLoc(Loc), AsmToks(Toks), TokOffsets(Offsets) { }
+  ~MCAsmParserSemaCallbackImpl() {}
+
+  void *LookupInlineAsmIdentifier(StringRef Name, void *SrcLoc, unsigned &Size){
+    SourceLocation Loc = SourceLocation::getFromPtrEncoding(SrcLoc);
+    NamedDecl *OpDecl = SemaRef.LookupInlineAsmIdentifier(Name, Loc, Size);
+    return static_cast<void *>(OpDecl);
+  }
+
+  bool LookupInlineAsmField(StringRef Base, StringRef Member,
+                            unsigned &Offset) {
+    return SemaRef.LookupInlineAsmField(Base, Member, Offset, AsmLoc);
+  }
+
+  static void MSAsmDiagHandlerCallback(const llvm::SMDiagnostic &D,
+                                       void *Context) {
+    ((MCAsmParserSemaCallbackImpl*)Context)->MSAsmDiagHandler(D);
+  }
+  void MSAsmDiagHandler(const llvm::SMDiagnostic &D) {
+    // Compute an offset into the inline asm buffer.
+    // FIXME: This isn't right if .macro is involved (but hopefully, no
+    // real-world code does that).
+    const llvm::SourceMgr &LSM = *D.getSourceMgr();
+    const llvm::MemoryBuffer *LBuf =
+    LSM.getMemoryBuffer(LSM.FindBufferContainingLoc(D.getLoc()));
+    unsigned Offset = D.getLoc().getPointer()  - LBuf->getBufferStart();
+
+    // Figure out which token that offset points into.
+    const unsigned *OffsetPtr =
+        std::lower_bound(TokOffsets.begin(), TokOffsets.end(), Offset);
+    unsigned TokIndex = OffsetPtr - TokOffsets.begin();
+
+    // If we come up with an answer which seems sane, use it; otherwise,
+    // just point at the __asm keyword.
+    // FIXME: Assert the answer is sane once we handle .macro correctly.
+    SourceLocation Loc = AsmLoc;
+    if (TokIndex < AsmToks.size()) {
+      const Token *Tok = &AsmToks[TokIndex];
+      Loc = Tok->getLocation();
+      Loc = Loc.getLocWithOffset(Offset - (*OffsetPtr - Tok->getLength()));
+    }
+    SemaRef.Diag(Loc, diag::err_inline_ms_asm_parsing) << D.getMessage();
+  }
+};
+
+}
+
+NamedDecl *Sema::LookupInlineAsmIdentifier(StringRef Name, SourceLocation Loc,
+                                           unsigned &Size) {
+  Size = 0;
+  LookupResult Result(*this, &Context.Idents.get(Name), Loc,
+                      Sema::LookupOrdinaryName);
+
+  if (!LookupName(Result, getCurScope())) {
+    // If we don't find anything, return null; the AsmParser will assume
+    // it is a label of some sort.
+    return 0;
+  }
+
+  if (!Result.isSingleResult()) {
+    // FIXME: Diagnose result.
+    return 0;
+  }
+
+  NamedDecl *ND = Result.getFoundDecl();
+  if (isa<VarDecl>(ND) || isa<FunctionDecl>(ND)) {
+    if (VarDecl *Var = dyn_cast<VarDecl>(ND))
+      Size = Context.getTypeInfo(Var->getType()).first;
+
+    return ND;
+  }
+
+  // FIXME: Handle other kinds of results? (FieldDecl, etc.)
+  // FIXME: Diagnose if we find something we can't handle, like a typedef.
+  return 0;
+}
+
+bool Sema::LookupInlineAsmField(StringRef Base, StringRef Member,
+                                unsigned &Offset, SourceLocation AsmLoc) {
+  Offset = 0;
+  LookupResult BaseResult(*this, &Context.Idents.get(Base), SourceLocation(),
+                          LookupOrdinaryName);
+
+  if (!LookupName(BaseResult, getCurScope()))
+    return true;
+
+  if (!BaseResult.isSingleResult())
+    return true;
+
+  NamedDecl *FoundDecl = BaseResult.getFoundDecl();
+  const RecordType *RT = 0;
+  if (VarDecl *VD = dyn_cast<VarDecl>(FoundDecl)) {
+    RT = VD->getType()->getAs<RecordType>();
+  } else if (TypedefDecl *TD = dyn_cast<TypedefDecl>(FoundDecl)) {
+    RT = TD->getUnderlyingType()->getAs<RecordType>();
+  }
+  if (!RT)
+    return true;
+
+  if (RequireCompleteType(AsmLoc, QualType(RT, 0), 0))
+    return true;
+
+  LookupResult FieldResult(*this, &Context.Idents.get(Member), SourceLocation(),
+                           LookupMemberName);
+
+  if (!LookupQualifiedName(FieldResult, RT->getDecl()))
+    return true;
+
+  // FIXME: Handle IndirectFieldDecl?
+  FieldDecl *FD = dyn_cast<FieldDecl>(FieldResult.getFoundDecl());
+  if (!FD)
+    return true;
+
+  const ASTRecordLayout &RL = Context.getASTRecordLayout(RT->getDecl());
+  unsigned i = FD->getFieldIndex();
+  CharUnits Result = Context.toCharUnitsFromBits(RL.getFieldOffset(i));
+  Offset = (unsigned)Result.getQuantity();
+
+  return false;
+}
+
+StmtResult Sema::ActOnMSAsmStmt(SourceLocation AsmLoc, SourceLocation LBraceLoc,
+                                ArrayRef<Token> AsmToks,SourceLocation EndLoc) {
+  SmallVector<IdentifierInfo*, 4> Names;
+  SmallVector<StringRef, 4> ConstraintRefs;
+  SmallVector<Expr*, 4> Exprs;
+  SmallVector<StringRef, 4> ClobberRefs;
+
+  // Empty asm statements don't need to instantiate the AsmParser, etc.
+  if (AsmToks.empty()) {
+    StringRef EmptyAsmStr;
+    MSAsmStmt *NS =
+      new (Context) MSAsmStmt(Context, AsmLoc, LBraceLoc, /*IsSimple*/ true,
+                              /*IsVolatile*/ true, AsmToks, /*NumOutputs*/ 0,
+                              /*NumInputs*/ 0, Names, ConstraintRefs, Exprs,
+                              EmptyAsmStr, ClobberRefs, EndLoc);
+    return Owned(NS);
+  }
+
+  std::string AsmString;
+  llvm::SmallVector<unsigned, 8> TokOffsets;
+  if (buildMSAsmString(*this, AsmLoc, AsmToks, TokOffsets, AsmString))
+    return StmtError();
+
+  // Get the target specific parser.
+  std::string Error;
+  const std::string &TT = Context.getTargetInfo().getTriple().getTriple();
+  const llvm::Target *TheTarget(llvm::TargetRegistry::lookupTarget(TT, Error));
+
+  OwningPtr<llvm::MCAsmInfo> MAI(TheTarget->createMCAsmInfo(TT));
+  OwningPtr<llvm::MCRegisterInfo> MRI(TheTarget->createMCRegInfo(TT));
+  OwningPtr<llvm::MCObjectFileInfo> MOFI(new llvm::MCObjectFileInfo());
+  OwningPtr<llvm::MCSubtargetInfo>
+    STI(TheTarget->createMCSubtargetInfo(TT, "", ""));
+
+  llvm::SourceMgr SrcMgr;
+  llvm::MCContext Ctx(*MAI, *MRI, MOFI.get(), &SrcMgr);
+  llvm::MemoryBuffer *Buffer =
+    llvm::MemoryBuffer::getMemBuffer(AsmString, "<inline asm>");
+
+  // Tell SrcMgr about this buffer, which is what the parser will pick up.
+  SrcMgr.AddNewSourceBuffer(Buffer, llvm::SMLoc());
+
+  OwningPtr<llvm::MCStreamer> Str(createNullStreamer(Ctx));
+  OwningPtr<llvm::MCAsmParser>
+    Parser(createMCAsmParser(SrcMgr, Ctx, *Str.get(), *MAI));
+  OwningPtr<llvm::MCTargetAsmParser>
+    TargetParser(TheTarget->createMCAsmParser(*STI, *Parser));
+
+  // Get the instruction descriptor.
+  const llvm::MCInstrInfo *MII = TheTarget->createMCInstrInfo(); 
+  llvm::MCInstPrinter *IP =
+    TheTarget->createMCInstPrinter(1, *MAI, *MII, *MRI, *STI);
+
+  // Change to the Intel dialect.
+  Parser->setAssemblerDialect(1);
+  Parser->setTargetParser(*TargetParser.get());
+  Parser->setParsingInlineAsm(true);
+  TargetParser->setParsingInlineAsm(true);
+
+  MCAsmParserSemaCallbackImpl MCAPSI(*this, AsmLoc, AsmToks, TokOffsets);
+  TargetParser->setSemaCallback(&MCAPSI);
+  SrcMgr.setDiagHandler(MCAsmParserSemaCallbackImpl::MSAsmDiagHandlerCallback,
+                        &MCAPSI);
+
+  unsigned NumOutputs;
+  unsigned NumInputs;
+  std::string AsmStringIR;
+  SmallVector<std::pair<void *, bool>, 4> OpDecls;
+  SmallVector<std::string, 4> Constraints;
+  SmallVector<std::string, 4> Clobbers;
+  if (Parser->ParseMSInlineAsm(AsmLoc.getPtrEncoding(), AsmStringIR,
+                               NumOutputs, NumInputs, OpDecls, Constraints,
+                               Clobbers, MII, IP, MCAPSI))
+    return StmtError();
+
+  // Build the vector of clobber StringRefs.
+  unsigned NumClobbers = Clobbers.size();
+  ClobberRefs.resize(NumClobbers);
+  for (unsigned i = 0; i != NumClobbers; ++i)
+    ClobberRefs[i] = StringRef(Clobbers[i]);
+
+  // Recast the void pointers and build the vector of constraint StringRefs.
+  unsigned NumExprs = NumOutputs + NumInputs;
+  Names.resize(NumExprs);
+  ConstraintRefs.resize(NumExprs);
+  Exprs.resize(NumExprs);
+  for (unsigned i = 0, e = NumExprs; i != e; ++i) {
+    NamedDecl *OpDecl = static_cast<NamedDecl *>(OpDecls[i].first);
+    if (!OpDecl)
+      return StmtError();
+
+    DeclarationNameInfo NameInfo(OpDecl->getDeclName(), AsmLoc);
+    ExprResult OpExpr = BuildDeclarationNameExpr(CXXScopeSpec(), NameInfo,
+                                                 OpDecl);
+    if (OpExpr.isInvalid())
+      return StmtError();
+
+    // Need offset of variable.
+    if (OpDecls[i].second)
+      OpExpr = BuildUnaryOp(getCurScope(), AsmLoc, clang::UO_AddrOf,
+                            OpExpr.take());
+
+    Names[i] = OpDecl->getIdentifier();
+    ConstraintRefs[i] = StringRef(Constraints[i]);
+    Exprs[i] = OpExpr.take();
+  }
+
+  bool IsSimple = NumExprs > 0;
+  MSAsmStmt *NS =
+    new (Context) MSAsmStmt(Context, AsmLoc, LBraceLoc, IsSimple,
+                            /*IsVolatile*/ true, AsmToks, NumOutputs, NumInputs,
+                            Names, ConstraintRefs, Exprs, AsmStringIR,
+                            ClobberRefs, EndLoc);
+  return Owned(NS);
+}
diff --git a/lib/Sema/SemaStmtAttr.cpp b/lib/Sema/SemaStmtAttr.cpp
index 3c15b7a..b268b45 100644
--- a/lib/Sema/SemaStmtAttr.cpp
+++ b/lib/Sema/SemaStmtAttr.cpp
@@ -48,11 +48,16 @@ static Attr *handleFallThroughAttr(Sema &S, Stmt *St, const AttributeList &A,
 static Attr *ProcessStmtAttribute(Sema &S, Stmt *St, const AttributeList &A,
                                   SourceRange Range) {
   switch (A.getKind()) {
+  case AttributeList::UnknownAttribute:
+    S.Diag(A.getLoc(), A.isDeclspecAttribute() ?
+           diag::warn_unhandled_ms_attribute_ignored :
+           diag::warn_unknown_attribute_ignored) << A.getName();
+    return 0;
   case AttributeList::AT_FallThrough:
     return handleFallThroughAttr(S, St, A, Range);
   default:
-    // if we're here, then we parsed an attribute, but didn't recognize it as a
-    // statement attribute => it is declaration attribute
+    // if we're here, then we parsed a known attribute, but didn't recognize
+    // it as a statement attribute => it is declaration attribute
     S.Diag(A.getRange().getBegin(), diag::warn_attribute_invalid_on_stmt)
         << A.getName()->getName() << St->getLocStart();
     return 0;
diff --git a/lib/Sema/SemaTemplate.cpp b/lib/Sema/SemaTemplate.cpp
index 4dbf3e4..f56b054 100644
--- a/lib/Sema/SemaTemplate.cpp
+++ b/lib/Sema/SemaTemplate.cpp
@@ -333,7 +333,8 @@ void Sema::LookupTemplateName(LookupResult &Found,
         if (LookupCtx)
           Diag(Found.getNameLoc(), diag::err_no_member_template_suggest)
             << Name << LookupCtx << CorrectedQuotedStr << SS.getRange()
-            << FixItHint::CreateReplacement(Found.getNameLoc(), CorrectedStr);
+            << FixItHint::CreateReplacement(Corrected.getCorrectionRange(),
+                                            CorrectedStr);
         else
           Diag(Found.getNameLoc(), diag::err_no_template_suggest)
             << Name << CorrectedQuotedStr
@@ -1104,6 +1105,9 @@ Sema::CheckClassTemplate(Scope *S, unsigned TagSpec, TagUseKind TUK,
   if (Attr)
     ProcessDeclAttributeList(S, NewClass, Attr);
 
+  if (PrevClassTemplate)
+    mergeDeclAttributes(NewClass, PrevClassTemplate->getTemplatedDecl());
+
   AddPushedVisibilityAttribute(NewClass);
 
   if (TUK != TUK_Friend)
@@ -1138,8 +1142,6 @@ Sema::CheckClassTemplate(Scope *S, unsigned TagSpec, TagUseKind TUK,
     NewTemplate->setInvalidDecl();
     NewClass->setInvalidDecl();
   }
-  if (PrevClassTemplate)
-    mergeDeclAttributes(NewClass, PrevClassTemplate->getTemplatedDecl());
 
   ActOnDocumentableDecl(NewTemplate);
 
@@ -1204,11 +1206,17 @@ static bool DiagnoseDefaultTemplateArgument(Sema &S,
 /// of a template template parameter, recursively.
 static bool DiagnoseUnexpandedParameterPacks(Sema &S,
                                              TemplateTemplateParmDecl *TTP) {
+  // A template template parameter which is a parameter pack is also a pack
+  // expansion.
+  if (TTP->isParameterPack())
+    return false;
+
   TemplateParameterList *Params = TTP->getTemplateParameters();
   for (unsigned I = 0, N = Params->size(); I != N; ++I) {
     NamedDecl *P = Params->getParam(I);
     if (NonTypeTemplateParmDecl *NTTP = dyn_cast<NonTypeTemplateParmDecl>(P)) {
-      if (S.DiagnoseUnexpandedParameterPack(NTTP->getLocation(),
+      if (!NTTP->isParameterPack() &&
+          S.DiagnoseUnexpandedParameterPack(NTTP->getLocation(),
                                             NTTP->getTypeSourceInfo(),
                                       Sema::UPPC_NonTypeTemplateParameterType))
         return true;
@@ -1321,7 +1329,8 @@ bool Sema::CheckTemplateParameterList(TemplateParameterList *NewParams,
     } else if (NonTypeTemplateParmDecl *NewNonTypeParm
                = dyn_cast<NonTypeTemplateParmDecl>(*NewParam)) {
       // Check for unexpanded parameter packs.
-      if (DiagnoseUnexpandedParameterPack(NewNonTypeParm->getLocation(),
+      if (!NewNonTypeParm->isParameterPack() &&
+          DiagnoseUnexpandedParameterPack(NewNonTypeParm->getLocation(),
                                           NewNonTypeParm->getTypeSourceInfo(),
                                           UPPC_NonTypeTemplateParameterType)) {
         Invalid = true;
@@ -1342,7 +1351,8 @@ bool Sema::CheckTemplateParameterList(TemplateParameterList *NewParams,
       if (NewNonTypeParm->isParameterPack()) {
         assert(!NewNonTypeParm->hasDefaultArgument() &&
                "Parameter packs can't have a default argument!");
-        SawParameterPack = true;
+        if (!NewNonTypeParm->isPackExpansion())
+          SawParameterPack = true;
       } else if (OldNonTypeParm && OldNonTypeParm->hasDefaultArgument() &&
           NewNonTypeParm->hasDefaultArgument()) {
         OldDefaultLoc = OldNonTypeParm->getDefaultArgumentLoc();
@@ -1389,7 +1399,8 @@ bool Sema::CheckTemplateParameterList(TemplateParameterList *NewParams,
       if (NewTemplateParm->isParameterPack()) {
         assert(!NewTemplateParm->hasDefaultArgument() &&
                "Parameter packs can't have a default argument!");
-        SawParameterPack = true;
+        if (!NewTemplateParm->isPackExpansion())
+          SawParameterPack = true;
       } else if (OldTemplateParm && OldTemplateParm->hasDefaultArgument() &&
           NewTemplateParm->hasDefaultArgument()) {
         OldDefaultLoc = OldTemplateParm->getDefaultArgument().getLocation();
@@ -1416,10 +1427,10 @@ bool Sema::CheckTemplateParameterList(TemplateParameterList *NewParams,
         MissingDefaultArg = true;
     }
 
-    // C++0x [temp.param]p11:
+    // C++11 [temp.param]p11:
     //   If a template parameter of a primary class template or alias template
     //   is a template parameter pack, it shall be the last template parameter.
-    if (SawParameterPack && (NewParam + 1) != NewParamEnd && 
+    if (SawParameterPack && (NewParam + 1) != NewParamEnd &&
         (TPC == TPC_ClassTemplate || TPC == TPC_TypeAliasTemplate)) {
       Diag((*NewParam)->getLocation(),
            diag::err_template_param_pack_must_be_last_template_parameter);
@@ -1998,9 +2009,11 @@ QualType Sema::CheckTemplateIdType(TemplateName Name,
     for (unsigned I = 0; I < Depth; ++I)
       TemplateArgLists.addOuterTemplateArguments(0, 0);
 
+    LocalInstantiationScope Scope(*this);
     InstantiatingTemplate Inst(*this, TemplateLoc, Template);
     if (Inst)
       return QualType();
+
     CanonType = SubstType(Pattern->getUnderlyingType(),
                           TemplateArgLists, AliasTemplate->getLocation(),
                           AliasTemplate->getDeclName());
@@ -2085,7 +2098,8 @@ QualType Sema::CheckTemplateIdType(TemplateName Name,
                                                      Converted.data(),
                                                      Converted.size(), 0);
       ClassTemplate->AddSpecialization(Decl, InsertPos);
-      Decl->setLexicalDeclContext(CurContext);
+      if (ClassTemplate->isOutOfLine())
+        Decl->setLexicalDeclContext(ClassTemplate->getLexicalDeclContext());
     }
 
     CanonType = Context.getTypeDeclType(Decl);
@@ -2137,7 +2151,6 @@ Sema::ActOnTemplateIdType(CXXScopeSpec &SS, SourceLocation TemplateKWLoc,
   }
   
   QualType Result = CheckTemplateIdType(Template, TemplateLoc, TemplateArgs);
-  TemplateArgsIn.release();
 
   if (Result.isNull())
     return true;
@@ -2831,6 +2844,7 @@ bool Sema::CheckTemplateArgument(NamedDecl *Param,
 
     case TemplateArgument::Declaration:
     case TemplateArgument::Integral:
+    case TemplateArgument::NullPtr:
       // We've already checked this template argument, so just copy
       // it to the list of converted arguments.
       Converted.push_back(Arg.getArgument());
@@ -2947,7 +2961,7 @@ bool Sema::CheckTemplateArgument(NamedDecl *Param,
 
   case TemplateArgument::Template:
   case TemplateArgument::TemplateExpansion:
-    if (CheckTemplateArgument(TempParm, Arg))
+    if (CheckTemplateArgument(TempParm, Arg, ArgumentPackIndex))
       return true;
 
     Converted.push_back(Arg.getArgument());
@@ -2965,6 +2979,8 @@ bool Sema::CheckTemplateArgument(NamedDecl *Param,
     llvm_unreachable("Declaration argument with template template parameter");
   case TemplateArgument::Integral:
     llvm_unreachable("Integral argument with template template parameter");
+  case TemplateArgument::NullPtr:
+    llvm_unreachable("Null pointer argument with template template parameter");
 
   case TemplateArgument::Pack:
     llvm_unreachable("Caller must expand template argument packs");
@@ -2996,6 +3012,33 @@ static bool diagnoseArityMismatch(Sema &S, TemplateDecl *Template,
   return true;
 }
 
+/// \brief Check whether the template parameter is a pack expansion, and if so,
+/// determine the number of parameters produced by that expansion. For instance:
+///
+/// \code
+/// template<typename ...Ts> struct A {
+///   template<Ts ...NTs, template<Ts> class ...TTs, typename ...Us> struct B;
+/// };
+/// \endcode
+///
+/// In \c A<int,int>::B, \c NTs and \c TTs have expanded pack size 2, and \c Us
+/// is not a pack expansion, so returns an empty Optional.
+static llvm::Optional<unsigned> getExpandedPackSize(NamedDecl *Param) {
+  if (NonTypeTemplateParmDecl *NTTP
+        = dyn_cast<NonTypeTemplateParmDecl>(Param)) {
+    if (NTTP->isExpandedParameterPack())
+      return NTTP->getNumExpansionTypes();
+  }
+
+  if (TemplateTemplateParmDecl *TTP
+        = dyn_cast<TemplateTemplateParmDecl>(Param)) {
+    if (TTP->isExpandedParameterPack())
+      return TTP->getNumExpansionTemplateParameters();
+  }
+
+  return llvm::Optional<unsigned>();
+}
+
 /// \brief Check that the given template argument list is well-formed
 /// for specializing the given template.
 bool Sema::CheckTemplateArgumentList(TemplateDecl *Template,
@@ -3008,15 +3051,9 @@ bool Sema::CheckTemplateArgumentList(TemplateDecl *Template,
     *ExpansionIntoFixedList = false;
 
   TemplateParameterList *Params = Template->getTemplateParameters();
-  unsigned NumParams = Params->size();
-  unsigned NumArgs = TemplateArgs.size();
-  bool Invalid = false;
 
   SourceLocation RAngleLoc = TemplateArgs.getRAngleLoc();
 
-  bool HasParameterPack =
-    NumParams > 0 && Params->getParam(NumParams - 1)->isTemplateParameterPack();
-  
   // C++ [temp.arg]p1:
   //   [...] The type and form of each template-argument specified in
   //   a template-id shall match the type and form specified for the
@@ -3024,38 +3061,50 @@ bool Sema::CheckTemplateArgumentList(TemplateDecl *Template,
   //   template-parameter-list.
   bool isTemplateTemplateParameter = isa<TemplateTemplateParmDecl>(Template);
   SmallVector<TemplateArgument, 2> ArgumentPack;
-  TemplateParameterList::iterator Param = Params->begin(),
-                               ParamEnd = Params->end();
-  unsigned ArgIdx = 0;
+  unsigned ArgIdx = 0, NumArgs = TemplateArgs.size();
   LocalInstantiationScope InstScope(*this, true);
-  bool SawPackExpansion = false;
-  while (Param != ParamEnd) {
-    if (ArgIdx < NumArgs) {
-      // If we have an expanded parameter pack, make sure we don't have too
-      // many arguments.
-      // FIXME: This really should fall out from the normal arity checking.
-      if (NonTypeTemplateParmDecl *NTTP
-                                = dyn_cast<NonTypeTemplateParmDecl>(*Param)) {
-        if (NTTP->isExpandedParameterPack() &&
-            ArgumentPack.size() >= NTTP->getNumExpansionTypes()) {
-          Diag(TemplateLoc, diag::err_template_arg_list_different_arity)
-            << true
-            << (isa<ClassTemplateDecl>(Template)? 0 :
-                isa<FunctionTemplateDecl>(Template)? 1 :
-                isa<TemplateTemplateParmDecl>(Template)? 2 : 3)
-            << Template;
-          Diag(Template->getLocation(), diag::note_template_decl_here)
-            << Params->getSourceRange();
-          return true;
-        }
+  for (TemplateParameterList::iterator Param = Params->begin(),
+                                       ParamEnd = Params->end();
+       Param != ParamEnd; /* increment in loop */) {
+    // If we have an expanded parameter pack, make sure we don't have too
+    // many arguments.
+    if (llvm::Optional<unsigned> Expansions = getExpandedPackSize(*Param)) {
+      if (*Expansions == ArgumentPack.size()) {
+        // We're done with this parameter pack. Pack up its arguments and add
+        // them to the list.
+        Converted.push_back(
+          TemplateArgument::CreatePackCopy(Context,
+                                           ArgumentPack.data(),
+                                           ArgumentPack.size()));
+        ArgumentPack.clear();
+
+        // This argument is assigned to the next parameter.
+        ++Param;
+        continue;
+      } else if (ArgIdx == NumArgs && !PartialTemplateArgs) {
+        // Not enough arguments for this parameter pack.
+        Diag(TemplateLoc, diag::err_template_arg_list_different_arity)
+          << false
+          << (isa<ClassTemplateDecl>(Template)? 0 :
+              isa<FunctionTemplateDecl>(Template)? 1 :
+              isa<TemplateTemplateParmDecl>(Template)? 2 : 3)
+          << Template;
+        Diag(Template->getLocation(), diag::note_template_decl_here)
+          << Params->getSourceRange();
+        return true;
       }
+    }
 
+    if (ArgIdx < NumArgs) {
       // Check the template argument we were given.
       if (CheckTemplateArgument(*Param, TemplateArgs[ArgIdx], Template,
                                 TemplateLoc, RAngleLoc,
                                 ArgumentPack.size(), Converted))
         return true;
 
+      // We're now done with this argument.
+      ++ArgIdx;
+
       if ((*Param)->isTemplateParameterPack()) {
         // The template parameter was a template parameter pack, so take the
         // deduced argument and place it on the argument pack. Note that we
@@ -3067,16 +3116,47 @@ bool Sema::CheckTemplateArgumentList(TemplateDecl *Template,
         // Move to the next template parameter.
         ++Param;
       }
-      
-      // If this template argument is a pack expansion, record that fact
-      // and break out; we can't actually check any more.
-      if (TemplateArgs[ArgIdx].getArgument().isPackExpansion()) {
-        SawPackExpansion = true;
-        ++ArgIdx;
-        break;
+
+      // If we just saw a pack expansion, then directly convert the remaining
+      // arguments, because we don't know what parameters they'll match up
+      // with.
+      if (TemplateArgs[ArgIdx-1].getArgument().isPackExpansion()) {
+        bool InFinalParameterPack = Param != ParamEnd &&
+                                    Param + 1 == ParamEnd &&
+                                    (*Param)->isTemplateParameterPack() &&
+                                    !getExpandedPackSize(*Param);
+
+        if (!InFinalParameterPack && !ArgumentPack.empty()) {
+          // If we were part way through filling in an expanded parameter pack,
+          // fall back to just producing individual arguments.
+          Converted.insert(Converted.end(),
+                           ArgumentPack.begin(), ArgumentPack.end());
+          ArgumentPack.clear();
+        }
+
+        while (ArgIdx < NumArgs) {
+          if (InFinalParameterPack)
+            ArgumentPack.push_back(TemplateArgs[ArgIdx].getArgument());
+          else
+            Converted.push_back(TemplateArgs[ArgIdx].getArgument());
+          ++ArgIdx;
+        }
+
+        // Push the argument pack onto the list of converted arguments.
+        if (InFinalParameterPack) {
+          Converted.push_back(
+            TemplateArgument::CreatePackCopy(Context,
+                                             ArgumentPack.data(),
+                                             ArgumentPack.size()));
+          ArgumentPack.clear();
+        } else if (ExpansionIntoFixedList) {
+          // We have expanded a pack into a fixed list.
+          *ExpansionIntoFixedList = true;
+        }
+
+        return false;
       }
-      
-      ++ArgIdx;
+
       continue;
     }
 
@@ -3087,14 +3167,30 @@ bool Sema::CheckTemplateArgumentList(TemplateDecl *Template,
                                                          ArgumentPack.data(),
                                                          ArgumentPack.size()));
         
-      return Invalid;
+      return false;
     }
 
     // If we have a template parameter pack with no more corresponding
     // arguments, just break out now and we'll fill in the argument pack below.
-    if ((*Param)->isTemplateParameterPack())
-      break;
-    
+    if ((*Param)->isTemplateParameterPack()) {
+      assert(!getExpandedPackSize(*Param) &&
+             "Should have dealt with this already");
+
+      // A non-expanded parameter pack before the end of the parameter list
+      // only occurs for an ill-formed template parameter list, unless we've
+      // got a partial argument list for a function template, so just bail out.
+      if (Param + 1 != ParamEnd)
+        return true;
+
+      Converted.push_back(TemplateArgument::CreatePackCopy(Context,
+                                                       ArgumentPack.data(),
+                                                       ArgumentPack.size()));
+      ArgumentPack.clear();
+
+      ++Param;
+      continue;
+    }
+
     // Check whether we have a default argument.
     TemplateArgumentLoc Arg;
 
@@ -3181,86 +3277,12 @@ bool Sema::CheckTemplateArgumentList(TemplateDecl *Template,
     ++ArgIdx;
   }
 
-  // If we saw a pack expansion, then directly convert the remaining arguments,
-  // because we don't know what parameters they'll match up with.
-  if (SawPackExpansion) {
-    bool AddToArgumentPack
-      = Param != ParamEnd && (*Param)->isTemplateParameterPack();
-    while (ArgIdx < NumArgs) {
-      if (AddToArgumentPack)
-        ArgumentPack.push_back(TemplateArgs[ArgIdx].getArgument());
-      else
-        Converted.push_back(TemplateArgs[ArgIdx].getArgument());
-      ++ArgIdx;
-    }
-
-    // Push the argument pack onto the list of converted arguments.
-    if (AddToArgumentPack) {
-      if (ArgumentPack.empty())
-        Converted.push_back(TemplateArgument(0, 0));
-      else {
-        Converted.push_back(
-          TemplateArgument::CreatePackCopy(Context,
-                                           ArgumentPack.data(),
-                                           ArgumentPack.size()));
-        ArgumentPack.clear();
-      }      
-    } else if (ExpansionIntoFixedList) {
-      // We have expanded a pack into a fixed list.
-      *ExpansionIntoFixedList = true;
-    }
-
-    return Invalid;
-  }
-
   // If we have any leftover arguments, then there were too many arguments.
   // Complain and fail.
   if (ArgIdx < NumArgs)
     return diagnoseArityMismatch(*this, Template, TemplateLoc, TemplateArgs);
-  
-  // If we have an expanded parameter pack, make sure we don't have too
-  // many arguments.
-  // FIXME: This really should fall out from the normal arity checking.
-  if (Param != ParamEnd) {
-    if (NonTypeTemplateParmDecl *NTTP
-          = dyn_cast<NonTypeTemplateParmDecl>(*Param)) {
-      if (NTTP->isExpandedParameterPack() &&
-          ArgumentPack.size() < NTTP->getNumExpansionTypes()) {
-        Diag(TemplateLoc, diag::err_template_arg_list_different_arity)
-          << false
-          << (isa<ClassTemplateDecl>(Template)? 0 :
-              isa<FunctionTemplateDecl>(Template)? 1 :
-              isa<TemplateTemplateParmDecl>(Template)? 2 : 3)
-          << Template;
-        Diag(Template->getLocation(), diag::note_template_decl_here)
-          << Params->getSourceRange();
-        return true;
-      }
-    }
-  }
-  
-  // Form argument packs for each of the parameter packs remaining.
-  while (Param != ParamEnd) {
-    // If we're checking a partial list of template arguments, don't fill
-    // in arguments for non-template parameter packs.
-    if ((*Param)->isTemplateParameterPack()) {
-      if (!HasParameterPack)
-        return true;
-      if (ArgumentPack.empty())
-        Converted.push_back(TemplateArgument(0, 0));
-      else {
-        Converted.push_back(TemplateArgument::CreatePackCopy(Context,
-                                                          ArgumentPack.data(),
-                                                         ArgumentPack.size()));
-        ArgumentPack.clear();
-      }
-    } else if (!PartialTemplateArgs)
-      return diagnoseArityMismatch(*this, Template, TemplateLoc, TemplateArgs);
-
-    ++Param;
-  }
 
-  return Invalid;
+  return false;
 }
 
 namespace {
@@ -3650,7 +3672,7 @@ CheckTemplateArgumentAddressOfObjectOrFunction(Sema &S,
     switch (isNullPointerValueTemplateArgument(S, Param, ParamType, Arg)) {
     case NPV_NullPointer:
       S.Diag(Arg->getExprLoc(), diag::warn_cxx98_compat_template_arg_null);
-      Converted = TemplateArgument((Decl *)0);
+      Converted = TemplateArgument(ParamType, /*isNullPtr*/true);
       return false;
 
     case NPV_Error:
@@ -3738,7 +3760,7 @@ CheckTemplateArgumentAddressOfObjectOrFunction(Sema &S,
     return true;
   }
 
-  NamedDecl *Entity = DRE->getDecl();
+  ValueDecl *Entity = DRE->getDecl();
 
   // Cannot refer to non-static data members
   if (FieldDecl *Field = dyn_cast<FieldDecl>(Entity)) {
@@ -3926,7 +3948,8 @@ CheckTemplateArgumentAddressOfObjectOrFunction(Sema &S,
   }
 
   // Create the template argument.
-  Converted = TemplateArgument(Entity->getCanonicalDecl());
+  Converted = TemplateArgument(cast<ValueDecl>(Entity->getCanonicalDecl()),
+                               ParamType->isReferenceType());
   S.MarkAnyDeclReferenced(Arg->getLocStart(), Entity);
   return false;
 }
@@ -3947,7 +3970,7 @@ static bool CheckTemplateArgumentPointerToMember(Sema &S,
     return true;
   case NPV_NullPointer:
     S.Diag(Arg->getExprLoc(), diag::warn_cxx98_compat_template_arg_null);
-    Converted = TemplateArgument((Decl *)0);
+    Converted = TemplateArgument(ParamType, /*isNullPtr*/true);
     return false;
   case NPV_NotNullPointer:
     break;
@@ -4016,10 +4039,12 @@ static bool CheckTemplateArgumentPointerToMember(Sema &S,
         if (isa<NonTypeTemplateParmDecl>(VD) ||
             (isa<VarDecl>(VD) &&
              S.Context.getCanonicalType(VD->getType()).isConstQualified())) {
-          if (Arg->isTypeDependent() || Arg->isValueDependent())
+          if (Arg->isTypeDependent() || Arg->isValueDependent()) {
             Converted = TemplateArgument(Arg);
-          else
-            Converted = TemplateArgument(VD->getCanonicalDecl());
+          } else {
+            VD = cast<ValueDecl>(VD->getCanonicalDecl());
+            Converted = TemplateArgument(VD, /*isReferenceParam*/false);
+          }
           return Invalid;
         }
       }
@@ -4040,10 +4065,12 @@ static bool CheckTemplateArgumentPointerToMember(Sema &S,
 
     // Okay: this is the address of a non-static member, and therefore
     // a member pointer constant.
-    if (Arg->isTypeDependent() || Arg->isValueDependent())
+    if (Arg->isTypeDependent() || Arg->isValueDependent()) {
       Converted = TemplateArgument(Arg);
-    else
-      Converted = TemplateArgument(DRE->getDecl()->getCanonicalDecl());
+    } else {
+      ValueDecl *D = cast<ValueDecl>(DRE->getDecl()->getCanonicalDecl());
+      Converted = TemplateArgument(D, /*isReferenceParam*/false);
+    }
     return Invalid;
   }
 
@@ -4396,8 +4423,8 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param,
       
     case NPV_NullPointer:
       Diag(Arg->getExprLoc(), diag::warn_cxx98_compat_template_arg_null);
-      Converted = TemplateArgument((Decl *)0);
-      return Owned(Arg);;
+      Converted = TemplateArgument(ParamType, /*isNullPtr*/true);
+      return Owned(Arg);
     }
   }
 
@@ -4417,8 +4444,9 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param,
 /// This routine implements the semantics of C++ [temp.arg.template].
 /// It returns true if an error occurred, and false otherwise.
 bool Sema::CheckTemplateArgument(TemplateTemplateParmDecl *Param,
-                                 const TemplateArgumentLoc &Arg) {
-  TemplateName Name = Arg.getArgument().getAsTemplate();
+                                 const TemplateArgumentLoc &Arg,
+                                 unsigned ArgumentPackIndex) {
+  TemplateName Name = Arg.getArgument().getAsTemplateOrTemplatePattern();
   TemplateDecl *Template = Name.getAsTemplateDecl();
   if (!Template) {
     // Any dependent template name is fine.
@@ -4448,8 +4476,12 @@ bool Sema::CheckTemplateArgument(TemplateTemplateParmDecl *Param,
       << Template;
   }
 
+  TemplateParameterList *Params = Param->getTemplateParameters();
+  if (Param->isExpandedParameterPack())
+    Params = Param->getExpansionTemplateParameters(ArgumentPackIndex);
+
   return !TemplateParameterListsAreEqual(Template->getTemplateParameters(),
-                                         Param->getTemplateParameters(),
+                                         Params,
                                          true,
                                          TPL_TemplateTemplateArgumentMatch,
                                          Arg.getLocation());
@@ -4463,12 +4495,9 @@ ExprResult
 Sema::BuildExpressionFromDeclTemplateArgument(const TemplateArgument &Arg,
                                               QualType ParamType,
                                               SourceLocation Loc) {
-  assert(Arg.getKind() == TemplateArgument::Declaration &&
-         "Only declaration template arguments permitted here");
-  
   // For a NULL non-type template argument, return nullptr casted to the
   // parameter's type.
-  if (!Arg.getAsDecl()) {
+  if (Arg.getKind() == TemplateArgument::NullPtr) {
     return ImpCastExprToType(
              new (Context) CXXNullPtrLiteralExpr(Context.NullPtrTy, Loc),
                              ParamType,
@@ -4476,7 +4505,9 @@ Sema::BuildExpressionFromDeclTemplateArgument(const TemplateArgument &Arg,
                                ? CK_NullToMemberPointer
                                : CK_NullToPointer);
   }
-  
+  assert(Arg.getKind() == TemplateArgument::Declaration &&
+         "Only declaration template arguments permitted here");
+
   ValueDecl *VD = cast<ValueDecl>(Arg.getAsDecl());
 
   if (VD->getDeclContext()->isRecord() &&
@@ -4524,7 +4555,7 @@ Sema::BuildExpressionFromDeclTemplateArgument(const TemplateArgument &Arg,
       assert(!RefExpr.isInvalid() &&
              Context.hasSameType(((Expr*) RefExpr.get())->getType(),
                                  ParamType.getUnqualifiedType()));
-      return move(RefExpr);
+      return RefExpr;
     }
   }
 
@@ -4542,7 +4573,7 @@ Sema::BuildExpressionFromDeclTemplateArgument(const TemplateArgument &Arg,
       if (RefExpr.isInvalid())
         return ExprError();
 
-      return move(RefExpr);
+      return RefExpr;
     }
 
     // Take the address of everything else
@@ -5071,10 +5102,10 @@ static bool CheckNonTypeClassTemplatePartialSpecializationArgs(Sema &S,
       continue;
     }
 
-    Expr *ArgExpr = Args[I].getAsExpr();
-    if (!ArgExpr) {
+    if (Args[I].getKind() != TemplateArgument::Expression)
       continue;
-    }
+
+    Expr *ArgExpr = Args[I].getAsExpr();
 
     // We can have a pack expansion of any of the bullets below.
     if (PackExpansionExpr *Expansion = dyn_cast<PackExpansionExpr>(ArgExpr))
@@ -5173,7 +5204,7 @@ Sema::ActOnClassTemplateSpecialization(Scope *S, unsigned TagSpec,
   // NOTE: KWLoc is the location of the tag keyword. This will instead
   // store the location of the outermost template keyword in the declaration.
   SourceLocation TemplateKWLoc = TemplateParameterLists.size() > 0
-    ? TemplateParameterLists.get()[0]->getTemplateLoc() : SourceLocation();
+    ? TemplateParameterLists[0]->getTemplateLoc() : SourceLocation();
 
   // Find the class template we're specializing
   TemplateName Name = TemplateD.getAsVal<TemplateName>();
@@ -5199,7 +5230,7 @@ Sema::ActOnClassTemplateSpecialization(Scope *S, unsigned TagSpec,
     = MatchTemplateParametersToScopeSpecifier(TemplateNameLoc, 
                                               TemplateNameLoc,
                                               SS,
-                        (TemplateParameterList**)TemplateParameterLists.get(),
+                                              TemplateParameterLists.data(),
                                               TemplateParameterLists.size(),
                                               TUK == TUK_Friend,
                                               isExplicitSpecialization,
@@ -5354,7 +5385,7 @@ Sema::ActOnClassTemplateSpecialization(Scope *S, unsigned TagSpec,
     if (TemplateParameterLists.size() > 0) {
       Specialization->setTemplateParameterListsInfo(Context,
                                               TemplateParameterLists.size(),
-                    (TemplateParameterList**) TemplateParameterLists.release());
+                                              TemplateParameterLists.data());
     }
     PrevDecl = 0;
     CanonType = Context.getTypeDeclType(Specialization);
@@ -5382,7 +5413,7 @@ Sema::ActOnClassTemplateSpecialization(Scope *S, unsigned TagSpec,
                                 TemplateParams,
                                 AS_none, /*ModulePrivateLoc=*/SourceLocation(),
                                 TemplateParameterLists.size() - 1,
-                  (TemplateParameterList**) TemplateParameterLists.release());
+                                TemplateParameterLists.data());
     }
 
     // Create a new class template partial specialization declaration node.
@@ -5406,7 +5437,7 @@ Sema::ActOnClassTemplateSpecialization(Scope *S, unsigned TagSpec,
     if (TemplateParameterLists.size() > 1 && SS.isSet()) {
       Partial->setTemplateParameterListsInfo(Context,
                                              TemplateParameterLists.size() - 1,
-                    (TemplateParameterList**) TemplateParameterLists.release());
+                                             TemplateParameterLists.data());
     }
 
     if (!PrevPartial)
@@ -5461,7 +5492,7 @@ Sema::ActOnClassTemplateSpecialization(Scope *S, unsigned TagSpec,
     if (TemplateParameterLists.size() > 0) {
       Specialization->setTemplateParameterListsInfo(Context,
                                               TemplateParameterLists.size(),
-                    (TemplateParameterList**) TemplateParameterLists.release());
+                                              TemplateParameterLists.data());
     }
 
     if (!PrevDecl)
@@ -5544,7 +5575,6 @@ Sema::ActOnClassTemplateSpecialization(Scope *S, unsigned TagSpec,
     Specialization->setTypeAsWritten(WrittenTy);
     Specialization->setTemplateKeywordLoc(TemplateKWLoc);
   }
-  TemplateArgsIn.release();
 
   // C++ [temp.expl.spec]p9:
   //   A template explicit specialization is in the scope of the
@@ -5579,7 +5609,7 @@ Sema::ActOnClassTemplateSpecialization(Scope *S, unsigned TagSpec,
 Decl *Sema::ActOnTemplateDeclarator(Scope *S,
                               MultiTemplateParamsArg TemplateParameterLists,
                                     Declarator &D) {
-  Decl *NewDecl = HandleDeclarator(S, D, move(TemplateParameterLists));
+  Decl *NewDecl = HandleDeclarator(S, D, TemplateParameterLists);
   ActOnDocumentableDecl(NewDecl);
   return NewDecl;
 }
@@ -5598,7 +5628,7 @@ Decl *Sema::ActOnStartOfFunctionTemplateDef(Scope *FnBodyScope,
 
   D.setFunctionDefinitionKind(FDK_Definition);
   Decl *DP = HandleDeclarator(ParentScope, D,
-                              move(TemplateParameterLists));
+                              TemplateParameterLists);
   if (FunctionTemplateDecl *FunctionTemplate
         = dyn_cast_or_null<FunctionTemplateDecl>(DP))
     return ActOnStartOfFunctionDef(FnBodyScope,
@@ -5902,7 +5932,7 @@ Sema::CheckFunctionTemplateSpecialization(FunctionDecl *FD,
       // Perform template argument deduction to determine whether we may be
       // specializing this template.
       // FIXME: It is somewhat wasteful to build
-      TemplateDeductionInfo Info(Context, FD->getLocation());
+      TemplateDeductionInfo Info(FD->getLocation());
       FunctionDecl *Specialization = 0;
       if (TemplateDeductionResult TDK
             = DeduceTemplateArguments(FunTmpl, ExplicitTemplateArgs,
@@ -6399,7 +6429,6 @@ Sema::ActOnExplicitInstantiation(Scope *S,
                                                 TemplateArgs,
                                   Context.getTypeDeclType(Specialization));
   Specialization->setTypeAsWritten(WrittenTy);
-  TemplateArgsIn.release();
 
   // Set source locations for keywords.
   Specialization->setExternLoc(ExternLoc);
@@ -6475,9 +6504,8 @@ Sema::ActOnExplicitInstantiation(Scope *S,
   Decl *TagD = ActOnTag(S, TagSpec, Sema::TUK_Reference,
                         KWLoc, SS, Name, NameLoc, Attr, AS_none,
                         /*ModulePrivateLoc=*/SourceLocation(),
-                        MultiTemplateParamsArg(*this, 0, 0),
-                        Owned, IsDependent, SourceLocation(), false,
-                        TypeResult());
+                        MultiTemplateParamsArg(), Owned, IsDependent,
+                        SourceLocation(), false, TypeResult());
   assert(!IsDependent && "explicit instantiation of dependent name not yet handled");
 
   if (!TagD)
@@ -6729,12 +6757,10 @@ DeclResult Sema::ActOnExplicitInstantiation(Scope *S,
     TemplateIdAnnotation *TemplateId = D.getName().TemplateId;
     TemplateArgs.setLAngleLoc(TemplateId->LAngleLoc);
     TemplateArgs.setRAngleLoc(TemplateId->RAngleLoc);
-    ASTTemplateArgsPtr TemplateArgsPtr(*this,
-                                       TemplateId->getTemplateArgs(),
+    ASTTemplateArgsPtr TemplateArgsPtr(TemplateId->getTemplateArgs(),
                                        TemplateId->NumArgs);
     translateTemplateArguments(TemplateArgsPtr, TemplateArgs);
     HasExplicitTemplateArgs = true;
-    TemplateArgsPtr.release();
   }
 
   // C++ [temp.explicit]p1:
@@ -6762,7 +6788,7 @@ DeclResult Sema::ActOnExplicitInstantiation(Scope *S,
     if (!FunTmpl)
       continue;
 
-    TemplateDeductionInfo Info(Context, D.getIdentifierLoc());
+    TemplateDeductionInfo Info(D.getIdentifierLoc());
     FunctionDecl *Specialization = 0;
     if (TemplateDeductionResult TDK
           = DeduceTemplateArguments(FunTmpl,
diff --git a/lib/Sema/SemaTemplateDeduction.cpp b/lib/Sema/SemaTemplateDeduction.cpp
index 9500ec3..bf4533d 100644
--- a/lib/Sema/SemaTemplateDeduction.cpp
+++ b/lib/Sema/SemaTemplateDeduction.cpp
@@ -158,9 +158,6 @@ static NonTypeTemplateParmDecl *getDeducedParameterFromExpr(Expr *E) {
 /// \brief Determine whether two declaration pointers refer to the same
 /// declaration.
 static bool isSameDeclaration(Decl *X, Decl *Y) {
-  if (!X || !Y)
-    return !X && !Y;
-
   if (NamedDecl *NX = dyn_cast<NamedDecl>(X))
     X = NX->getUnderlyingDecl();
   if (NamedDecl *NY = dyn_cast<NamedDecl>(Y))
@@ -262,7 +259,27 @@ checkDeducedTemplateArguments(ASTContext &Context,
     // If we deduced two declarations, make sure they they refer to the
     // same declaration.
     if (Y.getKind() == TemplateArgument::Declaration &&
-        isSameDeclaration(X.getAsDecl(), Y.getAsDecl()))
+        isSameDeclaration(X.getAsDecl(), Y.getAsDecl()) &&
+        X.isDeclForReferenceParam() == Y.isDeclForReferenceParam())
+      return X;
+
+    // All other combinations are incompatible.
+    return DeducedTemplateArgument();
+
+  case TemplateArgument::NullPtr:
+    // If we deduced a null pointer and a dependent expression, keep the
+    // null pointer.
+    if (Y.getKind() == TemplateArgument::Expression)
+      return X;
+
+    // If we deduced a null pointer and an integral constant, keep the
+    // integral constant.
+    if (Y.getKind() == TemplateArgument::Integral)
+      return Y;
+
+    // If we deduced two null pointers, make sure they have the same type.
+    if (Y.getKind() == TemplateArgument::NullPtr &&
+        Context.hasSameType(X.getNullPtrType(), Y.getNullPtrType()))
       return X;
 
     // All other combinations are incompatible.
@@ -356,13 +373,15 @@ DeduceNonTypeTemplateArgument(Sema &S,
 static Sema::TemplateDeductionResult
 DeduceNonTypeTemplateArgument(Sema &S,
                               NonTypeTemplateParmDecl *NTTP,
-                              Decl *D,
+                              ValueDecl *D,
                               TemplateDeductionInfo &Info,
                     SmallVectorImpl<DeducedTemplateArgument> &Deduced) {
   assert(NTTP->getDepth() == 0 &&
          "Cannot deduce non-type template argument with depth > 0");
 
-  DeducedTemplateArgument NewDeduced(D? D->getCanonicalDecl() : 0);
+  D = D ? cast<ValueDecl>(D->getCanonicalDecl()) : 0;
+  TemplateArgument New(D, NTTP->getType()->isReferenceType());
+  DeducedTemplateArgument NewDeduced(New);
   DeducedTemplateArgument Result = checkDeducedTemplateArguments(S.Context,
                                                      Deduced[NTTP->getIndex()],
                                                                  NewDeduced);
@@ -570,7 +589,10 @@ static void PrepareArgumentPackDeduction(Sema &S,
     SavedPacks[I] = Deduced[PackIndices[I]];
     Deduced[PackIndices[I]] = TemplateArgument();
 
-    // If the template arugment pack was explicitly specified, add that to
+    if (!S.CurrentInstantiationScope)
+      continue;
+
+    // If the template argument pack was explicitly specified, add that to
     // the set of deduced arguments.
     const TemplateArgument *ExplicitArgs;
     unsigned NumExplicitArgs;
@@ -612,7 +634,7 @@ FinishArgumentPackDeduction(Sema &S,
 
     if (NewlyDeducedPacks[I].empty()) {
       // If we deduced an empty argument pack, create it now.
-      NewPack = DeducedTemplateArgument(TemplateArgument(0, 0));
+      NewPack = DeducedTemplateArgument(TemplateArgument::getEmptyPack());
     } else {
       TemplateArgument *ArgumentPack
         = new (S.Context) TemplateArgument [NewlyDeducedPacks[I].size()];
@@ -1371,9 +1393,11 @@ DeduceTemplateArgumentsByTypeMatch(Sema &S,
             // If this is a base class, try to perform template argument
             // deduction from it.
             if (NextT != RecordT) {
+              TemplateDeductionInfo BaseInfo(Info.getLocation());
               Sema::TemplateDeductionResult BaseResult
                 = DeduceTemplateArguments(S, TemplateParams, SpecParam,
-                                          QualType(NextT, 0), Info, Deduced);
+                                          QualType(NextT, 0), BaseInfo,
+                                          Deduced);
 
               // If template argument deduction for this base was successful,
               // note that we had some success. Otherwise, ignore any deductions
@@ -1382,6 +1406,9 @@ DeduceTemplateArgumentsByTypeMatch(Sema &S,
                 Successful = true;
                 DeducedOrig.clear();
                 DeducedOrig.append(Deduced.begin(), Deduced.end());
+                Info.Param = BaseInfo.Param;
+                Info.FirstArg = BaseInfo.FirstArg;
+                Info.SecondArg = BaseInfo.SecondArg;
               }
               else
                 Deduced = DeducedOrig;
@@ -1596,7 +1623,17 @@ DeduceTemplateArguments(Sema &S,
 
   case TemplateArgument::Declaration:
     if (Arg.getKind() == TemplateArgument::Declaration &&
-        isSameDeclaration(Param.getAsDecl(), Arg.getAsDecl()))
+        isSameDeclaration(Param.getAsDecl(), Arg.getAsDecl()) &&
+        Param.isDeclForReferenceParam() == Arg.isDeclForReferenceParam())
+      return Sema::TDK_Success;
+
+    Info.FirstArg = Param;
+    Info.SecondArg = Arg;
+    return Sema::TDK_NonDeducedMismatch;
+
+  case TemplateArgument::NullPtr:
+    if (Arg.getKind() == TemplateArgument::NullPtr &&
+        S.Context.hasSameType(Param.getNullPtrType(), Arg.getNullPtrType()))
       return Sema::TDK_Success;
 
     Info.FirstArg = Param;
@@ -1867,7 +1904,11 @@ static bool isSameTemplateArg(ASTContext &Context,
              Context.getCanonicalType(Y.getAsType());
 
     case TemplateArgument::Declaration:
-      return isSameDeclaration(X.getAsDecl(), Y.getAsDecl());
+      return isSameDeclaration(X.getAsDecl(), Y.getAsDecl()) &&
+             X.isDeclForReferenceParam() == Y.isDeclForReferenceParam();
+
+    case TemplateArgument::NullPtr:
+      return Context.hasSameType(X.getNullPtrType(), Y.getNullPtrType());
 
     case TemplateArgument::Template:
     case TemplateArgument::TemplateExpansion:
@@ -1937,6 +1978,14 @@ getTrivialTemplateArgumentLoc(Sema &S,
     return TemplateArgumentLoc(TemplateArgument(E), E);
   }
 
+  case TemplateArgument::NullPtr: {
+    Expr *E
+      = S.BuildExpressionFromDeclTemplateArgument(Arg, NTTPType, Loc)
+          .takeAs<Expr>();
+    return TemplateArgumentLoc(TemplateArgument(NTTPType, /*isNullPtr*/true),
+                               E);
+  }
+
   case TemplateArgument::Integral: {
     Expr *E
       = S.BuildExpressionFromIntegralTemplateArgument(Arg, Loc).takeAs<Expr>();
@@ -2162,6 +2211,9 @@ Sema::TemplateDeductionResult
 Sema::DeduceTemplateArguments(ClassTemplatePartialSpecializationDecl *Partial,
                               const TemplateArgumentList &TemplateArgs,
                               TemplateDeductionInfo &Info) {
+  if (Partial->isInvalidDecl())
+    return TDK_Invalid;
+
   // C++ [temp.class.spec.match]p2:
   //   A partial specialization matches a given actual template
   //   argument list if the template arguments of the partial
@@ -2601,12 +2653,13 @@ Sema::FinishTemplateArgumentDeduction(FunctionTemplateDecl *FunctionTemplate,
       // explicitly-specified set (C++0x [temp.arg.explicit]p9).
       const TemplateArgument *ExplicitArgs;
       unsigned NumExplicitArgs;
-      if (CurrentInstantiationScope->getPartiallySubstitutedPack(&ExplicitArgs,
+      if (CurrentInstantiationScope &&
+          CurrentInstantiationScope->getPartiallySubstitutedPack(&ExplicitArgs,
                                                              &NumExplicitArgs)
           == Param)
         Builder.push_back(TemplateArgument(ExplicitArgs, NumExplicitArgs));
       else
-        Builder.push_back(TemplateArgument(0, 0));
+        Builder.push_back(TemplateArgument::getEmptyPack());
 
       continue;
     }
@@ -2784,7 +2837,7 @@ ResolveOverloadForDeduction(Sema &S, TemplateParameterList *TemplateParams,
       
       // Otherwise, see if we can resolve a function type 
       FunctionDecl *Specialization = 0;
-      TemplateDeductionInfo Info(S.Context, Ovl->getNameLoc());
+      TemplateDeductionInfo Info(Ovl->getNameLoc());
       if (S.DeduceTemplateArguments(FunTmpl, &ExplicitTemplateArgs,
                                     Specialization, Info))
         continue;
@@ -2815,7 +2868,7 @@ ResolveOverloadForDeduction(Sema &S, TemplateParameterList *TemplateParams,
     // So we do not reject deductions which were made elsewhere.
     SmallVector<DeducedTemplateArgument, 8>
       Deduced(TemplateParams->size());
-    TemplateDeductionInfo Info(S.Context, Ovl->getNameLoc());
+    TemplateDeductionInfo Info(Ovl->getNameLoc());
     Sema::TemplateDeductionResult Result
       = DeduceTemplateArgumentsByTypeMatch(S, TemplateParams, ParamType,
                                            ArgType, Info, Deduced, TDF);
@@ -2992,8 +3045,6 @@ DeduceTemplateArgumentByListElement(Sema &S,
 ///
 /// \param Args the function call arguments
 ///
-/// \param NumArgs the number of arguments in Args
-///
 /// \param Name the name of the function being called. This is only significant
 /// when the function template is a conversion function template, in which
 /// case this routine will also perform template argument deduction based on
@@ -3013,6 +3064,9 @@ Sema::DeduceTemplateArguments(FunctionTemplateDecl *FunctionTemplate,
                               llvm::ArrayRef<Expr *> Args,
                               FunctionDecl *&Specialization,
                               TemplateDeductionInfo &Info) {
+  if (FunctionTemplate->isInvalidDecl())
+    return TDK_Invalid;
+
   FunctionDecl *Function = FunctionTemplate->getTemplatedDecl();
 
   // C++ [temp.deduct.call]p1:
@@ -3269,6 +3323,9 @@ Sema::DeduceTemplateArguments(FunctionTemplateDecl *FunctionTemplate,
                               QualType ArgFunctionType,
                               FunctionDecl *&Specialization,
                               TemplateDeductionInfo &Info) {
+  if (FunctionTemplate->isInvalidDecl())
+    return TDK_Invalid;
+
   FunctionDecl *Function = FunctionTemplate->getTemplatedDecl();
   TemplateParameterList *TemplateParams
     = FunctionTemplate->getTemplateParameters();
@@ -3328,6 +3385,9 @@ Sema::DeduceTemplateArguments(FunctionTemplateDecl *FunctionTemplate,
                               QualType ToType,
                               CXXConversionDecl *&Specialization,
                               TemplateDeductionInfo &Info) {
+  if (FunctionTemplate->isInvalidDecl())
+    return TDK_Invalid;
+
   CXXConversionDecl *Conv
     = cast<CXXConversionDecl>(FunctionTemplate->getTemplatedDecl());
   QualType FromType = Conv->getConversionType();
@@ -3572,7 +3632,7 @@ Sema::DeduceAutoType(TypeSourceInfo *Type, Expr *&Init,
   QualType InitType = Init->getType();
   unsigned TDF = 0;
 
-  TemplateDeductionInfo Info(Context, Loc);
+  TemplateDeductionInfo Info(Loc);
 
   InitListExpr *InitList = dyn_cast<InitListExpr>(Init);
   if (InitList) {
@@ -3594,10 +3654,11 @@ Sema::DeduceAutoType(TypeSourceInfo *Type, Expr *&Init,
       return DAR_Failed;
   }
 
-  QualType DeducedType = Deduced[0].getAsType();
-  if (DeducedType.isNull())
+  if (Deduced[0].getKind() != TemplateArgument::Type)
     return DAR_Failed;
 
+  QualType DeducedType = Deduced[0].getAsType();
+
   if (InitList) {
     DeducedType = BuildStdInitializerList(DeducedType, Loc);
     if (DeducedType.isNull())
@@ -3637,26 +3698,23 @@ MarkUsedTemplateParameters(ASTContext &Ctx, QualType T,
                            llvm::SmallBitVector &Deduced);
 
 /// \brief If this is a non-static member function,
-static void MaybeAddImplicitObjectParameterType(ASTContext &Context,
+static void AddImplicitObjectParameterType(ASTContext &Context,
                                                 CXXMethodDecl *Method,
                                  SmallVectorImpl<QualType> &ArgTypes) {
-  if (Method->isStatic())
-    return;
-
-  // C++ [over.match.funcs]p4:
-  //
-  //   For non-static member functions, the type of the implicit
-  //   object parameter is
-  //     - "lvalue reference to cv X" for functions declared without a
-  //       ref-qualifier or with the & ref-qualifier
-  //     - "rvalue reference to cv X" for functions declared with the
-  //       && ref-qualifier
+  // C++11 [temp.func.order]p3:
+  //   [...] The new parameter is of type "reference to cv A," where cv are
+  //   the cv-qualifiers of the function template (if any) and A is
+  //   the class of which the function template is a member.
   //
-  // FIXME: We don't have ref-qualifiers yet, so we don't do that part.
+  // The standard doesn't say explicitly, but we pick the appropriate kind of
+  // reference type based on [over.match.funcs]p4.
   QualType ArgTy = Context.getTypeDeclType(Method->getParent());
   ArgTy = Context.getQualifiedType(ArgTy,
                         Qualifiers::fromCVRMask(Method->getTypeQualifiers()));
-  ArgTy = Context.getLValueReferenceType(ArgTy);
+  if (Method->getRefQualifier() == RQ_RValue)
+    ArgTy = Context.getRValueReferenceType(ArgTy);
+  else
+    ArgTy = Context.getLValueReferenceType(ArgTy);
   ArgTypes.push_back(ArgTy);
 }
 
@@ -3682,7 +3740,7 @@ static bool isAtLeastAsSpecializedAs(Sema &S,
   // C++0x [temp.deduct.partial]p3:
   //   The types used to determine the ordering depend on the context in which
   //   the partial ordering is done:
-  TemplateDeductionInfo Info(S.Context, Loc);
+  TemplateDeductionInfo Info(Loc);
   CXXMethodDecl *Method1 = 0;
   CXXMethodDecl *Method2 = 0;
   bool IsNonStatic2 = false;
@@ -3697,7 +3755,7 @@ static bool isAtLeastAsSpecializedAs(Sema &S,
     IsNonStatic1 = Method1 && !Method1->isStatic();
     IsNonStatic2 = Method2 && !Method2->isStatic();
 
-    // C++0x [temp.func.order]p3:
+    // C++11 [temp.func.order]p3:
     //   [...] If only one of the function templates is a non-static
     //   member, that function template is considered to have a new
     //   first parameter inserted in its function parameter list. The
@@ -3705,22 +3763,25 @@ static bool isAtLeastAsSpecializedAs(Sema &S,
     //   the cv-qualifiers of the function template (if any) and A is
     //   the class of which the function template is a member.
     //
+    // Note that we interpret this to mean "if one of the function
+    // templates is a non-static member and the other is a non-member";
+    // otherwise, the ordering rules for static functions against non-static
+    // functions don't make any sense.
+    //
     // C++98/03 doesn't have this provision, so instead we drop the
-    // first argument of the free function or static member, which
-    // seems to match existing practice.
+    // first argument of the free function, which seems to match
+    // existing practice.
     SmallVector<QualType, 4> Args1;
-    unsigned Skip1 = !S.getLangOpts().CPlusPlus0x &&
-      IsNonStatic2 && !IsNonStatic1;
-    if (S.getLangOpts().CPlusPlus0x && IsNonStatic1 && !IsNonStatic2)
-      MaybeAddImplicitObjectParameterType(S.Context, Method1, Args1);
+    unsigned Skip1 = !S.getLangOpts().CPlusPlus0x && IsNonStatic2 && !Method1;
+    if (S.getLangOpts().CPlusPlus0x && IsNonStatic1 && !Method2)
+      AddImplicitObjectParameterType(S.Context, Method1, Args1);
     Args1.insert(Args1.end(),
                  Proto1->arg_type_begin() + Skip1, Proto1->arg_type_end());
 
     SmallVector<QualType, 4> Args2;
-    Skip2 = !S.getLangOpts().CPlusPlus0x &&
-      IsNonStatic1 && !IsNonStatic2;
-    if (S.getLangOpts().CPlusPlus0x && IsNonStatic2 && !IsNonStatic1)
-      MaybeAddImplicitObjectParameterType(S.Context, Method2, Args2);
+    Skip2 = !S.getLangOpts().CPlusPlus0x && IsNonStatic1 && !Method2;
+    if (S.getLangOpts().CPlusPlus0x && IsNonStatic2 && !Method1)
+      AddImplicitObjectParameterType(S.Context, Method2, Args2);
     Args2.insert(Args2.end(),
                  Proto2->arg_type_begin() + Skip2, Proto2->arg_type_end());
 
@@ -4118,7 +4179,7 @@ Sema::getMoreSpecializedPartialSpecialization(
   // template partial specialization's template arguments, for
   // example.
   SmallVector<DeducedTemplateArgument, 4> Deduced;
-  TemplateDeductionInfo Info(Context, Loc);
+  TemplateDeductionInfo Info(Loc);
 
   QualType PT1 = PS1->getInjectedSpecializationType();
   QualType PT2 = PS2->getInjectedSpecializationType();
@@ -4496,6 +4557,11 @@ MarkUsedTemplateParameters(ASTContext &Ctx,
   case TemplateArgument::Declaration:
     break;
 
+  case TemplateArgument::NullPtr:
+    MarkUsedTemplateParameters(Ctx, TemplateArg.getNullPtrType(), OnlyDeduced,
+                               Depth, Used);
+    break;
+
   case TemplateArgument::Type:
     MarkUsedTemplateParameters(Ctx, TemplateArg.getAsType(), OnlyDeduced,
                                Depth, Used);
diff --git a/lib/Sema/SemaTemplateInstantiate.cpp b/lib/Sema/SemaTemplateInstantiate.cpp
index 20e755f..665dd07 100644
--- a/lib/Sema/SemaTemplateInstantiate.cpp
+++ b/lib/Sema/SemaTemplateInstantiate.cpp
@@ -769,7 +769,7 @@ namespace {
     /// instantiating it.
     Decl *TransformDefinition(SourceLocation Loc, Decl *D);
 
-    /// \bried Transform the first qualifier within a scope by instantiating the
+    /// \brief Transform the first qualifier within a scope by instantiating the
     /// declaration.
     NamedDecl *TransformFirstQualifierInScope(NamedDecl *D, SourceLocation Loc);
       
@@ -802,11 +802,24 @@ namespace {
     ExprResult TransformPredefinedExpr(PredefinedExpr *E);
     ExprResult TransformDeclRefExpr(DeclRefExpr *E);
     ExprResult TransformCXXDefaultArgExpr(CXXDefaultArgExpr *E);
+
     ExprResult TransformTemplateParmRefExpr(DeclRefExpr *E,
                                             NonTypeTemplateParmDecl *D);
     ExprResult TransformSubstNonTypeTemplateParmPackExpr(
                                            SubstNonTypeTemplateParmPackExpr *E);
-    
+
+    /// \brief Rebuild a DeclRefExpr for a ParmVarDecl reference.
+    ExprResult RebuildParmVarDeclRefExpr(ParmVarDecl *PD, SourceLocation Loc);
+
+    /// \brief Transform a reference to a function parameter pack.
+    ExprResult TransformFunctionParmPackRefExpr(DeclRefExpr *E,
+                                                ParmVarDecl *PD);
+
+    /// \brief Transform a FunctionParmPackExpr which was built when we couldn't
+    /// expand a function parameter pack reference which refers to an expanded
+    /// pack.
+    ExprResult TransformFunctionParmPackExpr(FunctionParmPackExpr *E);
+
     QualType TransformFunctionProtoType(TypeLocBuilder &TLB,
                                         FunctionProtoTypeLoc TL);
     QualType TransformFunctionProtoType(TypeLocBuilder &TLB,
@@ -835,7 +848,7 @@ namespace {
       ExprResult Result =
           TreeTransform<TemplateInstantiator>::TransformCallExpr(CE);
       getSema().CallsUndergoingInstantiation.pop_back();
-      return move(Result);
+      return Result;
     }
 
     ExprResult TransformLambdaExpr(LambdaExpr *E) {
@@ -1161,10 +1174,11 @@ ExprResult TemplateInstantiator::transformNonTypeTemplateParmRef(
     result = SemaRef.Owned(argExpr);
     type = argExpr->getType();
 
-  } else if (arg.getKind() == TemplateArgument::Declaration) {
+  } else if (arg.getKind() == TemplateArgument::Declaration ||
+             arg.getKind() == TemplateArgument::NullPtr) {
     ValueDecl *VD;
-    if (Decl *D = arg.getAsDecl()) {
-      VD = cast<ValueDecl>(D);
+    if (arg.getKind() == TemplateArgument::Declaration) {
+      VD = cast<ValueDecl>(arg.getAsDecl());
 
       // Find the instantiation of the template argument.  This is
       // required for nested templates.
@@ -1230,8 +1244,81 @@ TemplateInstantiator::TransformSubstNonTypeTemplateParmPackExpr(
 }
 
 ExprResult
+TemplateInstantiator::RebuildParmVarDeclRefExpr(ParmVarDecl *PD,
+                                                SourceLocation Loc) {
+  DeclarationNameInfo NameInfo(PD->getDeclName(), Loc);
+  return getSema().BuildDeclarationNameExpr(CXXScopeSpec(), NameInfo, PD);
+}
+
+ExprResult
+TemplateInstantiator::TransformFunctionParmPackExpr(FunctionParmPackExpr *E) {
+  if (getSema().ArgumentPackSubstitutionIndex != -1) {
+    // We can expand this parameter pack now.
+    ParmVarDecl *D = E->getExpansion(getSema().ArgumentPackSubstitutionIndex);
+    ValueDecl *VD = cast_or_null<ValueDecl>(TransformDecl(E->getExprLoc(), D));
+    if (!VD)
+      return ExprError();
+    return RebuildParmVarDeclRefExpr(cast<ParmVarDecl>(VD), E->getExprLoc());
+  }
+
+  QualType T = TransformType(E->getType());
+  if (T.isNull())
+    return ExprError();
+
+  // Transform each of the parameter expansions into the corresponding
+  // parameters in the instantiation of the function decl.
+  llvm::SmallVector<Decl*, 8> Parms;
+  Parms.reserve(E->getNumExpansions());
+  for (FunctionParmPackExpr::iterator I = E->begin(), End = E->end();
+       I != End; ++I) {
+    ParmVarDecl *D =
+        cast_or_null<ParmVarDecl>(TransformDecl(E->getExprLoc(), *I));
+    if (!D)
+      return ExprError();
+    Parms.push_back(D);
+  }
+
+  return FunctionParmPackExpr::Create(getSema().Context, T,
+                                      E->getParameterPack(),
+                                      E->getParameterPackLocation(), Parms);
+}
+
+ExprResult
+TemplateInstantiator::TransformFunctionParmPackRefExpr(DeclRefExpr *E,
+                                                       ParmVarDecl *PD) {
+  typedef LocalInstantiationScope::DeclArgumentPack DeclArgumentPack;
+  llvm::PointerUnion<Decl *, DeclArgumentPack *> *Found
+    = getSema().CurrentInstantiationScope->findInstantiationOf(PD);
+  assert(Found && "no instantiation for parameter pack");
+
+  Decl *TransformedDecl;
+  if (DeclArgumentPack *Pack = Found->dyn_cast<DeclArgumentPack *>()) {
+    // If this is a reference to a function parameter pack which we can substitute
+    // but can't yet expand, build a FunctionParmPackExpr for it.
+    if (getSema().ArgumentPackSubstitutionIndex == -1) {
+      QualType T = TransformType(E->getType());
+      if (T.isNull())
+        return ExprError();
+      return FunctionParmPackExpr::Create(getSema().Context, T, PD,
+                                          E->getExprLoc(), *Pack);
+    }
+
+    TransformedDecl = (*Pack)[getSema().ArgumentPackSubstitutionIndex];
+  } else {
+    TransformedDecl = Found->get<Decl*>();
+  }
+
+  // We have either an unexpanded pack or a specific expansion.
+  return RebuildParmVarDeclRefExpr(cast<ParmVarDecl>(TransformedDecl),
+                                   E->getExprLoc());
+}
+
+ExprResult
 TemplateInstantiator::TransformDeclRefExpr(DeclRefExpr *E) {
   NamedDecl *D = E->getDecl();
+
+  // Handle references to non-type template parameters and non-type template
+  // parameter packs.
   if (NonTypeTemplateParmDecl *NTTP = dyn_cast<NonTypeTemplateParmDecl>(D)) {
     if (NTTP->getDepth() < TemplateArgs.getNumLevels())
       return TransformTemplateParmRefExpr(E, NTTP);
@@ -1240,6 +1327,11 @@ TemplateInstantiator::TransformDeclRefExpr(DeclRefExpr *E) {
     // FindInstantiatedDecl will find it in the local instantiation scope.
   }
 
+  // Handle references to function parameter packs.
+  if (ParmVarDecl *PD = dyn_cast<ParmVarDecl>(D))
+    if (PD->isParameterPack())
+      return TransformFunctionParmPackRefExpr(E, PD);
+
   return TreeTransform<TemplateInstantiator>::TransformDeclRefExpr(E);
 }
 
@@ -2159,7 +2251,7 @@ Sema::InstantiateClassTemplateSpecialization(
   Template->getPartialSpecializations(PartialSpecs);
   for (unsigned I = 0, N = PartialSpecs.size(); I != N; ++I) {
     ClassTemplatePartialSpecializationDecl *Partial = PartialSpecs[I];
-    TemplateDeductionInfo Info(Context, PointOfInstantiation);
+    TemplateDeductionInfo Info(PointOfInstantiation);
     if (TemplateDeductionResult Result
           = DeduceTemplateArguments(Partial,
                                     ClassTemplateSpec->getTemplateArgs(),
@@ -2543,8 +2635,25 @@ bool Sema::Subst(const TemplateArgumentLoc *Args, unsigned NumArgs,
   return Instantiator.TransformTemplateArguments(Args, NumArgs, Result);
 }
 
+
+static const Decl* getCanonicalParmVarDecl(const Decl *D) {
+  // When storing ParmVarDecls in the local instantiation scope, we always
+  // want to use the ParmVarDecl from the canonical function declaration,
+  // since the map is then valid for any redeclaration or definition of that
+  // function.
+  if (const ParmVarDecl *PV = dyn_cast<ParmVarDecl>(D)) {
+    if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(PV->getDeclContext())) {
+      unsigned i = PV->getFunctionScopeIndex();
+      return FD->getCanonicalDecl()->getParamDecl(i);
+    }
+  }
+  return D;
+}
+
+
 llvm::PointerUnion<Decl *, LocalInstantiationScope::DeclArgumentPack *> *
 LocalInstantiationScope::findInstantiationOf(const Decl *D) {
+  D = getCanonicalParmVarDecl(D);
   for (LocalInstantiationScope *Current = this; Current;
        Current = Current->Outer) {
 
@@ -2576,6 +2685,7 @@ LocalInstantiationScope::findInstantiationOf(const Decl *D) {
 }
 
 void LocalInstantiationScope::InstantiatedLocal(const Decl *D, Decl *Inst) {
+  D = getCanonicalParmVarDecl(D);
   llvm::PointerUnion<Decl *, DeclArgumentPack *> &Stored = LocalDecls[D];
   if (Stored.isNull())
     Stored = Inst;
@@ -2588,11 +2698,13 @@ void LocalInstantiationScope::InstantiatedLocal(const Decl *D, Decl *Inst) {
 
 void LocalInstantiationScope::InstantiatedLocalPackArg(const Decl *D, 
                                                        Decl *Inst) {
+  D = getCanonicalParmVarDecl(D);
   DeclArgumentPack *Pack = LocalDecls[D].get<DeclArgumentPack *>();
   Pack->push_back(Inst);
 }
 
 void LocalInstantiationScope::MakeInstantiatedLocalArgPack(const Decl *D) {
+  D = getCanonicalParmVarDecl(D);
   llvm::PointerUnion<Decl *, DeclArgumentPack *> &Stored = LocalDecls[D];
   assert(Stored.isNull() && "Already instantiated this local");
   DeclArgumentPack *Pack = new DeclArgumentPack;
diff --git a/lib/Sema/SemaTemplateInstantiateDecl.cpp b/lib/Sema/SemaTemplateInstantiateDecl.cpp
index bdbe71d..19c46ab 100644
--- a/lib/Sema/SemaTemplateInstantiateDecl.cpp
+++ b/lib/Sema/SemaTemplateInstantiateDecl.cpp
@@ -158,6 +158,22 @@ Decl *TemplateDeclInstantiator::InstantiateTypedefNameDecl(TypedefNameDecl *D,
     SemaRef.MarkDeclarationsReferencedInType(D->getLocation(), DI->getType());
   }
 
+  // HACK: g++ has a bug where it gets the value kind of ?: wrong.
+  // libstdc++ relies upon this bug in its implementation of common_type.
+  // If we happen to be processing that implementation, fake up the g++ ?:
+  // semantics. See LWG issue 2141 for more information on the bug.
+  const DecltypeType *DT = DI->getType()->getAs<DecltypeType>();
+  CXXRecordDecl *RD = dyn_cast<CXXRecordDecl>(D->getDeclContext());
+  if (DT && RD && isa<ConditionalOperator>(DT->getUnderlyingExpr()) &&
+      DT->isReferenceType() &&
+      RD->getEnclosingNamespaceContext() == SemaRef.getStdNamespace() &&
+      RD->getIdentifier() && RD->getIdentifier()->isStr("common_type") &&
+      D->getIdentifier() && D->getIdentifier()->isStr("type") &&
+      SemaRef.getSourceManager().isInSystemHeader(D->getLocStart()))
+    // Fold it to the (non-reference) type which g++ would have produced.
+    DI = SemaRef.Context.getTrivialTypeSourceInfo(
+      DI->getType().getNonReferenceType());
+
   // Create the new typedef
   TypedefNameDecl *Typedef;
   if (IsTypeAlias)
@@ -510,7 +526,7 @@ Decl *TemplateDeclInstantiator::VisitFriendDecl(FriendDecl *D) {
     if (!InstTy)
       return 0;
 
-    FriendDecl *FD = SemaRef.CheckFriendTypeDecl(D->getLocation(),
+    FriendDecl *FD = SemaRef.CheckFriendTypeDecl(D->getLocStart(),
                                                  D->getFriendLoc(), InstTy);
     if (!FD)
       return 0;
@@ -1008,6 +1024,30 @@ Decl *TemplateDeclInstantiator::VisitCXXRecordDecl(CXXRecordDecl *D) {
   return Record;
 }
 
+/// \brief Adjust the given function type for an instantiation of the
+/// given declaration, to cope with modifications to the function's type that
+/// aren't reflected in the type-source information.
+///
+/// \param D The declaration we're instantiating.
+/// \param TInfo The already-instantiated type.
+static QualType adjustFunctionTypeForInstantiation(ASTContext &Context,
+                                                   FunctionDecl *D,
+                                                   TypeSourceInfo *TInfo) {
+  const FunctionProtoType *OrigFunc
+    = D->getType()->castAs<FunctionProtoType>();
+  const FunctionProtoType *NewFunc
+    = TInfo->getType()->castAs<FunctionProtoType>();
+  if (OrigFunc->getExtInfo() == NewFunc->getExtInfo())
+    return TInfo->getType();
+
+  FunctionProtoType::ExtProtoInfo NewEPI = NewFunc->getExtProtoInfo();
+  NewEPI.ExtInfo = OrigFunc->getExtInfo();
+  return Context.getFunctionType(NewFunc->getResultType(),
+                                 NewFunc->arg_type_begin(),
+                                 NewFunc->getNumArgs(),
+                                 NewEPI);
+}
+
 /// Normal class members are of more specific types and therefore
 /// don't make it here.  This function serves two purposes:
 ///   1) instantiating function templates
@@ -1048,7 +1088,7 @@ Decl *TemplateDeclInstantiator::VisitFunctionDecl(FunctionDecl *D,
   TypeSourceInfo *TInfo = SubstFunctionType(D, Params);
   if (!TInfo)
     return 0;
-  QualType T = TInfo->getType();
+  QualType T = adjustFunctionTypeForInstantiation(SemaRef.Context, D, TInfo);
 
   NestedNameSpecifierLoc QualifierLoc = D->getQualifierLoc();
   if (QualifierLoc) {
@@ -1075,7 +1115,7 @@ Decl *TemplateDeclInstantiator::VisitFunctionDecl(FunctionDecl *D,
 
   FunctionDecl *Function =
       FunctionDecl::Create(SemaRef.Context, DC, D->getInnerLocStart(),
-                           D->getLocation(), D->getDeclName(), T, TInfo,
+                           D->getNameInfo(), T, TInfo,
                            D->getStorageClass(), D->getStorageClassAsWritten(),
                            D->isInlineSpecified(), D->hasWrittenPrototype(),
                            D->isConstexpr());
@@ -1366,7 +1406,7 @@ TemplateDeclInstantiator::VisitCXXMethodDecl(CXXMethodDecl *D,
   TypeSourceInfo *TInfo = SubstFunctionType(D, Params);
   if (!TInfo)
     return 0;
-  QualType T = TInfo->getType();
+  QualType T = adjustFunctionTypeForInstantiation(SemaRef.Context, D, TInfo);
 
   // \brief If the type of this function, after ignoring parentheses,
   // is not *directly* a function type, then we're instantiating a function
@@ -1657,7 +1697,7 @@ Decl *TemplateDeclInstantiator::VisitNonTypeTemplateParmDecl(
     IsExpandedParameterPack = true;
     DI = D->getTypeSourceInfo();
     T = DI->getType();
-  } else if (isa<PackExpansionTypeLoc>(TL)) {
+  } else if (D->isPackExpansion()) {
     // The non-type template parameter pack's type is a pack expansion of types.
     // Determine whether we need to expand this parameter pack into separate
     // types.
@@ -1771,27 +1811,121 @@ Decl *TemplateDeclInstantiator::VisitNonTypeTemplateParmDecl(
   return Param;
 }
 
+static void collectUnexpandedParameterPacks(
+    Sema &S,
+    TemplateParameterList *Params,
+    SmallVectorImpl<UnexpandedParameterPack> &Unexpanded) {
+  for (TemplateParameterList::const_iterator I = Params->begin(),
+                                             E = Params->end(); I != E; ++I) {
+    if ((*I)->isTemplateParameterPack())
+      continue;
+    if (NonTypeTemplateParmDecl *NTTP = dyn_cast<NonTypeTemplateParmDecl>(*I))
+      S.collectUnexpandedParameterPacks(NTTP->getTypeSourceInfo()->getTypeLoc(),
+                                        Unexpanded);
+    if (TemplateTemplateParmDecl *TTP = dyn_cast<TemplateTemplateParmDecl>(*I))
+      collectUnexpandedParameterPacks(S, TTP->getTemplateParameters(),
+                                      Unexpanded);
+  }
+}
+
 Decl *
 TemplateDeclInstantiator::VisitTemplateTemplateParmDecl(
                                                   TemplateTemplateParmDecl *D) {
   // Instantiate the template parameter list of the template template parameter.
   TemplateParameterList *TempParams = D->getTemplateParameters();
   TemplateParameterList *InstParams;
-  {
+  SmallVector<TemplateParameterList*, 8> ExpandedParams;
+
+  bool IsExpandedParameterPack = false;
+
+  if (D->isExpandedParameterPack()) {
+    // The template template parameter pack is an already-expanded pack
+    // expansion of template parameters. Substitute into each of the expanded
+    // parameters.
+    ExpandedParams.reserve(D->getNumExpansionTemplateParameters());
+    for (unsigned I = 0, N = D->getNumExpansionTemplateParameters();
+         I != N; ++I) {
+      LocalInstantiationScope Scope(SemaRef);
+      TemplateParameterList *Expansion =
+        SubstTemplateParams(D->getExpansionTemplateParameters(I));
+      if (!Expansion)
+        return 0;
+      ExpandedParams.push_back(Expansion);
+    }
+
+    IsExpandedParameterPack = true;
+    InstParams = TempParams;
+  } else if (D->isPackExpansion()) {
+    // The template template parameter pack expands to a pack of template
+    // template parameters. Determine whether we need to expand this parameter
+    // pack into separate parameters.
+    SmallVector<UnexpandedParameterPack, 2> Unexpanded;
+    collectUnexpandedParameterPacks(SemaRef, D->getTemplateParameters(),
+                                    Unexpanded);
+
+    // Determine whether the set of unexpanded parameter packs can and should
+    // be expanded.
+    bool Expand = true;
+    bool RetainExpansion = false;
+    llvm::Optional<unsigned> NumExpansions;
+    if (SemaRef.CheckParameterPacksForExpansion(D->getLocation(),
+                                                TempParams->getSourceRange(),
+                                                Unexpanded,
+                                                TemplateArgs,
+                                                Expand, RetainExpansion,
+                                                NumExpansions))
+      return 0;
+
+    if (Expand) {
+      for (unsigned I = 0; I != *NumExpansions; ++I) {
+        Sema::ArgumentPackSubstitutionIndexRAII SubstIndex(SemaRef, I);
+        LocalInstantiationScope Scope(SemaRef);
+        TemplateParameterList *Expansion = SubstTemplateParams(TempParams);
+        if (!Expansion)
+          return 0;
+        ExpandedParams.push_back(Expansion);
+      }
+
+      // Note that we have an expanded parameter pack. The "type" of this
+      // expanded parameter pack is the original expansion type, but callers
+      // will end up using the expanded parameter pack types for type-checking.
+      IsExpandedParameterPack = true;
+      InstParams = TempParams;
+    } else {
+      // We cannot fully expand the pack expansion now, so just substitute
+      // into the pattern.
+      Sema::ArgumentPackSubstitutionIndexRAII SubstIndex(SemaRef, -1);
+
+      LocalInstantiationScope Scope(SemaRef);
+      InstParams = SubstTemplateParams(TempParams);
+      if (!InstParams)
+        return 0;
+    }
+  } else {
     // Perform the actual substitution of template parameters within a new,
     // local instantiation scope.
     LocalInstantiationScope Scope(SemaRef);
     InstParams = SubstTemplateParams(TempParams);
     if (!InstParams)
-      return NULL;
+      return 0;
   }
 
   // Build the template template parameter.
-  TemplateTemplateParmDecl *Param
-    = TemplateTemplateParmDecl::Create(SemaRef.Context, Owner, D->getLocation(),
+  TemplateTemplateParmDecl *Param;
+  if (IsExpandedParameterPack)
+    Param = TemplateTemplateParmDecl::Create(SemaRef.Context, Owner,
+                                             D->getLocation(),
+                                   D->getDepth() - TemplateArgs.getNumLevels(),
+                                             D->getPosition(),
+                                             D->getIdentifier(), InstParams,
+                                             ExpandedParams);
+  else
+    Param = TemplateTemplateParmDecl::Create(SemaRef.Context, Owner,
+                                             D->getLocation(),
                                    D->getDepth() - TemplateArgs.getNumLevels(),
-                                       D->getPosition(), D->isParameterPack(),
-                                       D->getIdentifier(), InstParams);
+                                             D->getPosition(),
+                                             D->isParameterPack(),
+                                             D->getIdentifier(), InstParams);
   Param->setDefaultArgument(D->getDefaultArgument(), false);
   Param->setAccess(AS_public);
 
@@ -1813,7 +1947,12 @@ Decl *TemplateDeclInstantiator::VisitUsingDirectiveDecl(UsingDirectiveDecl *D) {
                                  D->getIdentLocation(),
                                  D->getNominatedNamespace(),
                                  D->getCommonAncestor());
-  Owner->addDecl(Inst);
+
+  // Add the using directive to its declaration context
+  // only if this is not a function or method.
+  if (!Owner->isFunctionOrMethod())
+    Owner->addDecl(Inst);
+
   return Inst;
 }
 
@@ -2863,7 +3002,7 @@ Sema::InstantiateMemInitializers(CXXConstructorDecl *New,
                            const MultiLevelTemplateArgumentList &TemplateArgs) {
 
   SmallVector<CXXCtorInitializer*, 4> NewInits;
-  bool AnyErrors = false;
+  bool AnyErrors = Tmpl->isInvalidDecl();
 
   // Instantiate all the initializers.
   for (CXXConstructorDecl::init_const_iterator Inits = Tmpl->init_begin(),
@@ -3031,7 +3170,7 @@ ExprResult Sema::SubstInitializer(Expr *Init,
       isa<CXXTemporaryObjectExpr>(Construct))
     return SubstExpr(Init, TemplateArgs);
 
-  ASTOwningVector<Expr*> NewArgs(*this);
+  SmallVector<Expr*, 8> NewArgs;
   if (SubstExprs(Construct->getArgs(), Construct->getNumArgs(), true,
                  TemplateArgs, NewArgs))
     return ExprError();
@@ -3043,7 +3182,7 @@ ExprResult Sema::SubstInitializer(Expr *Init,
   // Build a ParenListExpr to represent anything else.
   // FIXME: Fake locations!
   SourceLocation Loc = PP.getLocForEndOfToken(Init->getLocStart());
-  return ActOnParenListExpr(Loc, Loc, move_arg(NewArgs));
+  return ActOnParenListExpr(Loc, Loc, NewArgs);
 }
 
 // TODO: this could be templated if the various decl types used the
@@ -3298,7 +3437,8 @@ NamedDecl *Sema::FindInstantiatedDecl(SourceLocation Loc, NamedDecl *D,
       if (Decl *FD = Found->dyn_cast<Decl *>())
         return cast<NamedDecl>(FD);
 
-      unsigned PackIdx = ArgumentPackSubstitutionIndex;
+      int PackIdx = ArgumentPackSubstitutionIndex;
+      assert(PackIdx != -1 && "found declaration pack but not pack expanding");
       return cast<NamedDecl>((*Found->get<DeclArgumentPack *>())[PackIdx]);
     }
 
diff --git a/lib/Sema/SemaTemplateVariadic.cpp b/lib/Sema/SemaTemplateVariadic.cpp
index aece90b..6147d63 100644
--- a/lib/Sema/SemaTemplateVariadic.cpp
+++ b/lib/Sema/SemaTemplateVariadic.cpp
@@ -727,6 +727,7 @@ bool Sema::containsUnexpandedParameterPacks(Declarator &D) {
   case TST_enum:
   case TST_union:
   case TST_struct:
+  case TST_interface:
   case TST_class:
   case TST_auto:
   case TST_unknown_anytype:
diff --git a/lib/Sema/SemaType.cpp b/lib/Sema/SemaType.cpp
index 54f8dba..4b23167 100644
--- a/lib/Sema/SemaType.cpp
+++ b/lib/Sema/SemaType.cpp
@@ -105,7 +105,8 @@ static void diagnoseBadTypeAttribute(Sema &S, const AttributeList &attr,
     case AttributeList::AT_ThisCall: \
     case AttributeList::AT_Pascal: \
     case AttributeList::AT_Regparm: \
-    case AttributeList::AT_Pcs \
+    case AttributeList::AT_Pcs: \
+    case AttributeList::AT_PnaclCall \
 
 namespace {
   /// An object which stores processing state for the entire
@@ -552,19 +553,28 @@ static void maybeSynthesizeBlockSignature(TypeProcessingState &state,
   SourceLocation loc = declarator.getLocStart();
 
   // ...and *prepend* it to the declarator.
+  SourceLocation NoLoc;
   declarator.AddInnermostTypeInfo(DeclaratorChunk::getFunction(
-                             /*proto*/ true,
-                             /*variadic*/ false,
-                             /*ambiguous*/ false, SourceLocation(),
-                             /*args*/ 0, 0,
-                             /*type quals*/ 0,
-                             /*ref-qualifier*/true, SourceLocation(),
-                             /*const qualifier*/SourceLocation(),
-                             /*volatile qualifier*/SourceLocation(),
-                             /*mutable qualifier*/SourceLocation(),
-                             /*EH*/ EST_None, SourceLocation(), 0, 0, 0, 0,
-                             /*parens*/ loc, loc,
-                             declarator));
+                             /*HasProto=*/true,
+                             /*IsAmbiguous=*/false,
+                             /*LParenLoc=*/NoLoc,
+                             /*ArgInfo=*/0,
+                             /*NumArgs=*/0,
+                             /*EllipsisLoc=*/NoLoc,
+                             /*RParenLoc=*/NoLoc,
+                             /*TypeQuals=*/0,
+                             /*RefQualifierIsLvalueRef=*/true,
+                             /*RefQualifierLoc=*/NoLoc,
+                             /*ConstQualifierLoc=*/NoLoc,
+                             /*VolatileQualifierLoc=*/NoLoc,
+                             /*MutableLoc=*/NoLoc,
+                             EST_None,
+                             /*ESpecLoc=*/NoLoc,
+                             /*Exceptions=*/0,
+                             /*ExceptionRanges=*/0,
+                             /*NumExceptions=*/0,
+                             /*NoexceptExpr=*/0,
+                             loc, loc, declarator));
 
   // For consistency, make sure the state still has us as processing
   // the decl spec.
@@ -636,7 +646,7 @@ static QualType ConvertDeclSpecToType(TypeProcessingState &state) {
     // "<proto1,proto2>" is an objc qualified ID with a missing id.
     if (DeclSpec::ProtocolQualifierListTy PQ = DS.getProtocolQualifiers()) {
       Result = Context.getObjCObjectType(Context.ObjCBuiltinIdTy,
-                                         (ObjCProtocolDecl**)PQ,
+                                         (ObjCProtocolDecl*const*)PQ,
                                          DS.getNumProtocolQualifiers());
       Result = Context.getObjCObjectPointerType(Result);
       break;
@@ -698,11 +708,15 @@ static QualType ConvertDeclSpecToType(TypeProcessingState &state) {
       case DeclSpec::TSW_longlong:
         Result = Context.LongLongTy;
 
-        // long long is a C99 feature.
-        if (!S.getLangOpts().C99)
-          S.Diag(DS.getTypeSpecWidthLoc(),
-                 S.getLangOpts().CPlusPlus0x ?
-                   diag::warn_cxx98_compat_longlong : diag::ext_longlong);
+        // 'long long' is a C99 or C++11 feature.
+        if (!S.getLangOpts().C99) {
+          if (S.getLangOpts().CPlusPlus)
+            S.Diag(DS.getTypeSpecWidthLoc(),
+                   S.getLangOpts().CPlusPlus0x ?
+                   diag::warn_cxx98_compat_longlong : diag::ext_cxx11_longlong);
+          else
+            S.Diag(DS.getTypeSpecWidthLoc(), diag::ext_c99_longlong);
+        }
         break;
       }
     } else {
@@ -713,11 +727,15 @@ static QualType ConvertDeclSpecToType(TypeProcessingState &state) {
       case DeclSpec::TSW_longlong:
         Result = Context.UnsignedLongLongTy;
 
-        // long long is a C99 feature.
-        if (!S.getLangOpts().C99)
-          S.Diag(DS.getTypeSpecWidthLoc(),
-                 S.getLangOpts().CPlusPlus0x ?
-                   diag::warn_cxx98_compat_longlong : diag::ext_longlong);
+        // 'long long' is a C99 or C++11 feature.
+        if (!S.getLangOpts().C99) {
+          if (S.getLangOpts().CPlusPlus)
+            S.Diag(DS.getTypeSpecWidthLoc(),
+                   S.getLangOpts().CPlusPlus0x ?
+                   diag::warn_cxx98_compat_longlong : diag::ext_cxx11_longlong);
+          else
+            S.Diag(DS.getTypeSpecWidthLoc(), diag::ext_c99_longlong);
+        }
         break;
       }
     }
@@ -753,7 +771,8 @@ static QualType ConvertDeclSpecToType(TypeProcessingState &state) {
   case DeclSpec::TST_class:
   case DeclSpec::TST_enum:
   case DeclSpec::TST_union:
-  case DeclSpec::TST_struct: {
+  case DeclSpec::TST_struct:
+  case DeclSpec::TST_interface: {
     TypeDecl *D = dyn_cast_or_null<TypeDecl>(DS.getRepAsDecl());
     if (!D) {
       // This can happen in C++ with ambiguous lookups.
@@ -794,18 +813,18 @@ static QualType ConvertDeclSpecToType(TypeProcessingState &state) {
 
         if (DS.getNumProtocolQualifiers())
           Result = Context.getObjCObjectType(Result,
-                                             (ObjCProtocolDecl**) PQ,
+                                             (ObjCProtocolDecl*const*) PQ,
                                              DS.getNumProtocolQualifiers());
       } else if (Result->isObjCIdType()) {
         // id<protocol-list>
         Result = Context.getObjCObjectType(Context.ObjCBuiltinIdTy,
-                                           (ObjCProtocolDecl**) PQ,
+                                           (ObjCProtocolDecl*const*) PQ,
                                            DS.getNumProtocolQualifiers());
         Result = Context.getObjCObjectPointerType(Result);
       } else if (Result->isObjCClassType()) {
         // Class<protocol-list>
         Result = Context.getObjCObjectType(Context.ObjCBuiltinClassTy,
-                                           (ObjCProtocolDecl**) PQ,
+                                           (ObjCProtocolDecl*const*) PQ,
                                            DS.getNumProtocolQualifiers());
         Result = Context.getObjCObjectPointerType(Result);
       } else {
@@ -1853,30 +1872,31 @@ static QualType GetDeclSpecTypeForDeclarator(TypeProcessingState &state,
       case TTK_Struct: Error = 1; /* Struct member */ break;
       case TTK_Union:  Error = 2; /* Union member */ break;
       case TTK_Class:  Error = 3; /* Class member */ break;
+      case TTK_Interface: Error = 4; /* Interface member */ break;
       }
       break;
     case Declarator::CXXCatchContext:
     case Declarator::ObjCCatchContext:
-      Error = 4; // Exception declaration
+      Error = 5; // Exception declaration
       break;
     case Declarator::TemplateParamContext:
-      Error = 5; // Template parameter
+      Error = 6; // Template parameter
       break;
     case Declarator::BlockLiteralContext:
-      Error = 6; // Block literal
+      Error = 7; // Block literal
       break;
     case Declarator::TemplateTypeArgContext:
-      Error = 7; // Template type argument
+      Error = 8; // Template type argument
       break;
     case Declarator::AliasDeclContext:
     case Declarator::AliasTemplateContext:
-      Error = 9; // Type alias
+      Error = 10; // Type alias
       break;
     case Declarator::TrailingReturnContext:
-      Error = 10; // Function return type
+      Error = 11; // Function return type
       break;
     case Declarator::TypeNameContext:
-      Error = 11; // Generic
+      Error = 12; // Generic
       break;
     case Declarator::FileContext:
     case Declarator::BlockContext:
@@ -1887,11 +1907,11 @@ static QualType GetDeclSpecTypeForDeclarator(TypeProcessingState &state,
     }
 
     if (D.getDeclSpec().getStorageClassSpec() == DeclSpec::SCS_typedef)
-      Error = 8;
+      Error = 9;
 
     // In Objective-C it is an error to use 'auto' on a function declarator.
     if (D.isFunctionDeclarator())
-      Error = 10;
+      Error = 11;
 
     // C++11 [dcl.spec.auto]p2: 'auto' is always fine if the declarator
     // contains a trailing return type. That is only legal at the outermost
@@ -2149,16 +2169,6 @@ static TypeSourceInfo *GetFullTypeForDeclarator(TypeProcessingState &state,
   ASTContext &Context = S.Context;
   const LangOptions &LangOpts = S.getLangOpts();
 
-  bool ImplicitlyNoexcept = false;
-  if (D.getName().getKind() == UnqualifiedId::IK_OperatorFunctionId &&
-      LangOpts.CPlusPlus0x) {
-    OverloadedOperatorKind OO = D.getName().OperatorFunctionId.Operator;
-    /// In C++0x, deallocation functions (normal and array operator delete)
-    /// are implicitly noexcept.
-    if (OO == OO_Delete || OO == OO_Array_Delete)
-      ImplicitlyNoexcept = true;
-  }
-
   // The name we're declaring, if any.
   DeclarationName Name;
   if (D.getIdentifier())
@@ -2558,12 +2568,6 @@ static TypeSourceInfo *GetFullTypeForDeclarator(TypeProcessingState &state,
                                       Exceptions,
                                       EPI);
 
-        if (FTI.getExceptionSpecType() == EST_None &&
-            ImplicitlyNoexcept && chunkIndex == 0) {
-          // Only the outermost chunk is marked noexcept, of course.
-          EPI.ExceptionSpecType = EST_BasicNoexcept;
-        }
-
         T = Context.getFunctionType(T, ArgTys.data(), ArgTys.size(), EPI);
       }
 
@@ -2657,6 +2661,9 @@ static TypeSourceInfo *GetFullTypeForDeclarator(TypeProcessingState &state,
 
     // C++0x [dcl.constexpr]p8: A constexpr specifier for a non-static member
     // function that is not a constructor declares that function to be const.
+    // FIXME: This should be deferred until we know whether this is a static
+    //        member function (for an out-of-class definition, we don't know
+    //        this until we perform redeclaration lookup).
     if (D.getDeclSpec().isConstexprSpecified() && !FreeFunction &&
         D.getDeclSpec().getStorageClassSpec() != DeclSpec::SCS_static &&
         D.getName().getKind() != UnqualifiedId::IK_ConstructorName &&
@@ -2668,6 +2675,12 @@ static TypeSourceInfo *GetFullTypeForDeclarator(TypeProcessingState &state,
       T = Context.getFunctionType(FnTy->getResultType(),
                                   FnTy->arg_type_begin(),
                                   FnTy->getNumArgs(), EPI);
+      // Rebuild any parens around the identifier in the function type.
+      for (unsigned i = 0, e = D.getNumTypeObjects(); i != e; ++i) {
+        if (D.getTypeObject(i).Kind != DeclaratorChunk::Paren)
+          break;
+        T = S.BuildParenType(T);
+      }
     }
 
     // C++11 [dcl.fct]p6 (w/DR1417):
@@ -2721,6 +2734,12 @@ static TypeSourceInfo *GetFullTypeForDeclarator(TypeProcessingState &state,
       T = Context.getFunctionType(FnTy->getResultType(),
                                   FnTy->arg_type_begin(),
                                   FnTy->getNumArgs(), EPI);
+      // Rebuild any parens around the identifier in the function type.
+      for (unsigned i = 0, e = D.getNumTypeObjects(); i != e; ++i) {
+        if (D.getTypeObject(i).Kind != DeclaratorChunk::Paren)
+          break;
+        T = S.BuildParenType(T);
+      }
     }
   }
 
@@ -2985,6 +3004,8 @@ static AttributeList::Kind getAttrListKind(AttributedType::Kind kind) {
     return AttributeList::AT_Pascal;
   case AttributedType::attr_pcs:
     return AttributeList::AT_Pcs;
+  case AttributedType::attr_pnaclcall:
+    return AttributeList::AT_PnaclCall;
   }
   llvm_unreachable("unexpected attribute kind!");
 }
@@ -3271,6 +3292,8 @@ namespace {
       TL.setLocalRangeEnd(Chunk.EndLoc);
 
       const DeclaratorChunk::FunctionTypeInfo &FTI = Chunk.Fun;
+      TL.setLParenLoc(FTI.getLParenLoc());
+      TL.setRParenLoc(FTI.getRParenLoc());
       for (unsigned i = 0, e = TL.getNumArgs(), tpi = 0; i != e; ++i) {
         ParmVarDecl *Param = cast<ParmVarDecl>(FTI.ArgInfo[i].Param);
         TL.setArg(tpi++, Param);
@@ -3588,7 +3611,7 @@ static bool handleObjCOwnershipTypeAttr(TypeProcessingState &state,
 
   // Forbid __weak if the runtime doesn't support it.
   if (lifetime == Qualifiers::OCL_Weak &&
-      !S.getLangOpts().ObjCRuntimeHasWeak && !NonObjCPointer) {
+      !S.getLangOpts().ObjCARCWeak && !NonObjCPointer) {
 
     // Actually, delay this until we know what we're parsing.
     if (S.DelayedDiagnostics.shouldDelayDiagnostics()) {
@@ -3611,11 +3634,12 @@ static bool handleObjCOwnershipTypeAttr(TypeProcessingState &state,
     while (const PointerType *ptr = T->getAs<PointerType>())
       T = ptr->getPointeeType();
     if (const ObjCObjectPointerType *ObjT = T->getAs<ObjCObjectPointerType>()) {
-      ObjCInterfaceDecl *Class = ObjT->getInterfaceDecl();
-      if (Class->isArcWeakrefUnavailable()) {
-          S.Diag(AttrLoc, diag::err_arc_unsupported_weak_class);
-          S.Diag(ObjT->getInterfaceDecl()->getLocation(),
-                 diag::note_class_declared);
+      if (ObjCInterfaceDecl *Class = ObjT->getInterfaceDecl()) {
+        if (Class->isArcWeakrefUnavailable()) {
+            S.Diag(AttrLoc, diag::err_arc_unsupported_weak_class);
+            S.Diag(ObjT->getInterfaceDecl()->getLocation(),
+                   diag::note_class_declared);
+        }
       }
     }
   }
@@ -3875,14 +3899,14 @@ static bool handleFunctionTypeAttr(TypeProcessingState &state,
     return true;
   }
 
+  // Delay if the type didn't work out to a function.
+  if (!unwrapped.isFunctionType()) return false;
+
   // Otherwise, a calling convention.
   CallingConv CC;
   if (S.CheckCallingConvAttr(attr, CC))
     return true;
 
-  // Delay if the type didn't work out to a function.
-  if (!unwrapped.isFunctionType()) return false;
-
   const FunctionType *fn = unwrapped.get();
   CallingConv CCOld = fn->getCallConv();
   if (S.Context.getCanonicalCallConv(CC) ==
@@ -4429,6 +4453,20 @@ bool Sema::RequireCompleteType(SourceLocation Loc, QualType T,
   return RequireCompleteType(Loc, T, Diagnoser);
 }
 
+/// \brief Get diagnostic %select index for tag kind for
+/// literal type diagnostic message.
+/// WARNING: Indexes apply to particular diagnostics only!
+///
+/// \returns diagnostic %select index.
+static unsigned getLiteralDiagFromTagKind(TagTypeKind Tag) {
+  switch (Tag) {
+  case TTK_Struct: return 0;
+  case TTK_Interface: return 1;
+  case TTK_Class:  return 2;
+  default: llvm_unreachable("Invalid tag kind for literal type diagnostic!");
+  }
+}
+
 /// @brief Ensure that the type T is a literal type.
 ///
 /// This routine checks whether the type @p T is a literal type. If @p T is an
@@ -4485,7 +4523,7 @@ bool Sema::RequireLiteralType(SourceLocation Loc, QualType T,
   // of constexpr constructors.
   if (RD->getNumVBases()) {
     Diag(RD->getLocation(), diag::note_non_literal_virtual_base)
-      << RD->isStruct() << RD->getNumVBases();
+      << getLiteralDiagFromTagKind(RD->getTagKind()) << RD->getNumVBases();
     for (CXXRecordDecl::base_class_const_iterator I = RD->vbases_begin(),
            E = RD->vbases_end(); I != E; ++I)
       Diag(I->getLocStart(),
@@ -4578,15 +4616,21 @@ static QualType getDecltypeForExpr(Sema &S, Expr *E) {
   //       member access (5.2.5), decltype(e) is the type of the entity named
   //       by e. If there is no such entity, or if e names a set of overloaded
   //       functions, the program is ill-formed;
+  //
+  // We apply the same rules for Objective-C ivar and property references.
   if (const DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(E)) {
     if (const ValueDecl *VD = dyn_cast<ValueDecl>(DRE->getDecl()))
       return VD->getType();
-  }
-  if (const MemberExpr *ME = dyn_cast<MemberExpr>(E)) {
+  } else if (const MemberExpr *ME = dyn_cast<MemberExpr>(E)) {
     if (const FieldDecl *FD = dyn_cast<FieldDecl>(ME->getMemberDecl()))
       return FD->getType();
+  } else if (const ObjCIvarRefExpr *IR = dyn_cast<ObjCIvarRefExpr>(E)) {
+    return IR->getDecl()->getType();
+  } else if (const ObjCPropertyRefExpr *PR = dyn_cast<ObjCPropertyRefExpr>(E)) {
+    if (PR->isExplicitProperty())
+      return PR->getExplicitProperty()->getType();
   }
-
+  
   // C++11 [expr.lambda.prim]p18:
   //   Every occurrence of decltype((x)) where x is a possibly
   //   parenthesized id-expression that names an entity of automatic
diff --git a/lib/Sema/TreeTransform.h b/lib/Sema/TreeTransform.h
index 619ad33..294d742 100644
--- a/lib/Sema/TreeTransform.h
+++ b/lib/Sema/TreeTransform.h
@@ -574,6 +574,10 @@ public:
   /// \brief Transform the captures and body of a lambda expression.
   ExprResult TransformLambdaScope(LambdaExpr *E, CXXMethodDecl *CallOperator);
 
+  ExprResult TransformAddressOfOperand(Expr *E);
+  ExprResult TransformDependentScopeDeclRefExpr(DependentScopeDeclRefExpr *E,
+                                                bool IsAddressOfOperand);
+
 #define STMT(Node, Parent)                        \
   StmtResult Transform##Node(Node *S);
 #define EXPR(Node, Parent)                        \
@@ -1162,32 +1166,23 @@ public:
   ///
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
-  StmtResult RebuildAsmStmt(SourceLocation AsmLoc,
-                                  bool IsSimple,
-                                  bool IsVolatile,
-                                  unsigned NumOutputs,
-                                  unsigned NumInputs,
-                                  IdentifierInfo **Names,
-                                  MultiExprArg Constraints,
-                                  MultiExprArg Exprs,
-                                  Expr *AsmString,
-                                  MultiExprArg Clobbers,
-                                  SourceLocation RParenLoc,
-                                  bool MSAsm) {
-    return getSema().ActOnAsmStmt(AsmLoc, IsSimple, IsVolatile, NumOutputs,
-                                  NumInputs, Names, move(Constraints),
-                                  Exprs, AsmString, Clobbers,
-                                  RParenLoc, MSAsm);
+  StmtResult RebuildGCCAsmStmt(SourceLocation AsmLoc, bool IsSimple,
+                               bool IsVolatile, unsigned NumOutputs,
+                               unsigned NumInputs, IdentifierInfo **Names,
+                               MultiExprArg Constraints, MultiExprArg Exprs,
+                               Expr *AsmString, MultiExprArg Clobbers,
+                               SourceLocation RParenLoc) {
+    return getSema().ActOnGCCAsmStmt(AsmLoc, IsSimple, IsVolatile, NumOutputs,
+                                     NumInputs, Names, Constraints, Exprs,
+                                     AsmString, Clobbers, RParenLoc);
   }
 
   /// \brief Build a new MS style inline asm statement.
   ///
   /// By default, performs semantic analysis to build the new statement.
   /// Subclasses may override this routine to provide different behavior.
-  StmtResult RebuildMSAsmStmt(SourceLocation AsmLoc,
-                              SourceLocation LBraceLoc,
-                              ArrayRef<Token> AsmToks,
-                              SourceLocation EndLoc) {
+  StmtResult RebuildMSAsmStmt(SourceLocation AsmLoc, SourceLocation LBraceLoc,
+                              ArrayRef<Token> AsmToks, SourceLocation EndLoc) {
     return getSema().ActOnMSAsmStmt(AsmLoc, LBraceLoc, AsmToks, EndLoc);
   }
 
@@ -1199,7 +1194,7 @@ public:
                                         Stmt *TryBody,
                                         MultiStmtArg CatchStmts,
                                         Stmt *Finally) {
-    return getSema().ActOnObjCAtTryStmt(AtLoc, TryBody, move(CatchStmts),
+    return getSema().ActOnObjCAtTryStmt(AtLoc, TryBody, CatchStmts,
                                         Finally);
   }
 
@@ -1325,7 +1320,7 @@ public:
   StmtResult RebuildCXXTryStmt(SourceLocation TryLoc,
                                Stmt *TryBlock,
                                MultiStmtArg Handlers) {
-    return getSema().ActOnCXXTryBlock(TryLoc, TryBlock, move(Handlers));
+    return getSema().ActOnCXXTryBlock(TryLoc, TryBlock, Handlers);
   }
 
   /// \brief Build a new C++0x range-based for statement.
@@ -1339,7 +1334,8 @@ public:
                                     Stmt *LoopVar,
                                     SourceLocation RParenLoc) {
     return getSema().BuildCXXForRangeStmt(ForLoc, ColonLoc, Range, BeginEnd,
-                                          Cond, Inc, LoopVar, RParenLoc);
+                                          Cond, Inc, LoopVar, RParenLoc,
+                                          Sema::BFRK_Rebuild);
   }
 
   /// \brief Build a new C++0x range-based for statement.
@@ -1478,7 +1474,7 @@ public:
     if (Result.isInvalid())
       return ExprError();
 
-    return move(Result);
+    return Result;
   }
 
   /// \brief Build a new array subscript expression.
@@ -1503,7 +1499,7 @@ public:
                                    SourceLocation RParenLoc,
                                    Expr *ExecConfig = 0) {
     return getSema().ActOnCallExpr(/*Scope=*/0, Callee, LParenLoc,
-                                   move(Args), RParenLoc, ExecConfig);
+                                   Args, RParenLoc, ExecConfig);
   }
 
   /// \brief Build a new member access expression.
@@ -1638,15 +1634,15 @@ public:
                              SourceLocation RBraceLoc,
                              QualType ResultTy) {
     ExprResult Result
-      = SemaRef.ActOnInitList(LBraceLoc, move(Inits), RBraceLoc);
+      = SemaRef.ActOnInitList(LBraceLoc, Inits, RBraceLoc);
     if (Result.isInvalid() || ResultTy->isDependentType())
-      return move(Result);
+      return Result;
 
     // Patch in the result type we were given, which may have been computed
     // when the initial InitListExpr was built.
     InitListExpr *ILE = cast<InitListExpr>((Expr *)Result.get());
     ILE->setType(ResultTy);
-    return move(Result);
+    return Result;
   }
 
   /// \brief Build a new designated initializer expression.
@@ -1664,8 +1660,7 @@ public:
     if (Result.isInvalid())
       return ExprError();
 
-    ArrayExprs.release();
-    return move(Result);
+    return Result;
   }
 
   /// \brief Build a new value-initialized expression.
@@ -1696,7 +1691,7 @@ public:
   ExprResult RebuildParenListExpr(SourceLocation LParenLoc,
                                   MultiExprArg SubExprs,
                                   SourceLocation RParenLoc) {
-    return getSema().ActOnParenListExpr(LParenLoc, RParenLoc, move(SubExprs));
+    return getSema().ActOnParenListExpr(LParenLoc, RParenLoc, SubExprs);
   }
 
   /// \brief Build a new address-of-label expression.
@@ -1974,8 +1969,7 @@ public:
                                            SourceLocation LParenLoc,
                                            SourceLocation RParenLoc) {
     return getSema().BuildCXXTypeConstructExpr(TSInfo, LParenLoc,
-                                               MultiExprArg(getSema(), 0, 0),
-                                               RParenLoc);
+                                               MultiExprArg(), RParenLoc);
   }
 
   /// \brief Build a new C++ "new" expression.
@@ -1995,7 +1989,7 @@ public:
                                Expr *Initializer) {
     return getSema().BuildCXXNew(StartLoc, UseGlobal,
                                  PlacementLParen,
-                                 move(PlacementArgs),
+                                 PlacementArgs,
                                  PlacementRParen,
                                  TypeIdParens,
                                  AllocatedType,
@@ -2083,7 +2077,8 @@ public:
                                           NestedNameSpecifierLoc QualifierLoc,
                                           SourceLocation TemplateKWLoc,
                                        const DeclarationNameInfo &NameInfo,
-                              const TemplateArgumentListInfo *TemplateArgs) {
+                              const TemplateArgumentListInfo *TemplateArgs,
+                                          bool IsAddressOfOperand) {
     CXXScopeSpec SS;
     SS.Adopt(QualifierLoc);
 
@@ -2091,7 +2086,8 @@ public:
       return getSema().BuildQualifiedTemplateIdExpr(SS, TemplateKWLoc,
                                                     NameInfo, TemplateArgs);
 
-    return getSema().BuildQualifiedDeclarationNameExpr(SS, NameInfo);
+    return getSema().BuildQualifiedDeclarationNameExpr(SS, NameInfo,
+                                                       IsAddressOfOperand);
   }
 
   /// \brief Build a new template-id expression.
@@ -2120,13 +2116,13 @@ public:
                                      bool RequiresZeroInit,
                              CXXConstructExpr::ConstructionKind ConstructKind,
                                      SourceRange ParenRange) {
-    ASTOwningVector<Expr*> ConvertedArgs(SemaRef);
-    if (getSema().CompleteConstructorCall(Constructor, move(Args), Loc,
+    SmallVector<Expr*, 8> ConvertedArgs;
+    if (getSema().CompleteConstructorCall(Constructor, Args, Loc,
                                           ConvertedArgs))
       return ExprError();
 
     return getSema().BuildCXXConstructExpr(Loc, T, Constructor, IsElidable,
-                                           move_arg(ConvertedArgs),
+                                           ConvertedArgs,
                                            HadMultipleCandidates,
                                            RequiresZeroInit, ConstructKind,
                                            ParenRange);
@@ -2142,7 +2138,7 @@ public:
                                            SourceLocation RParenLoc) {
     return getSema().BuildCXXTypeConstructExpr(TSInfo,
                                                LParenLoc,
-                                               move(Args),
+                                               Args,
                                                RParenLoc);
   }
 
@@ -2156,7 +2152,7 @@ public:
                                                SourceLocation RParenLoc) {
     return getSema().BuildCXXTypeConstructExpr(TSInfo,
                                                LParenLoc,
-                                               move(Args),
+                                               Args,
                                                RParenLoc);
   }
 
@@ -2288,7 +2284,7 @@ public:
                                      ReceiverTypeInfo->getType(),
                                      /*SuperLoc=*/SourceLocation(),
                                      Sel, Method, LBracLoc, SelectorLocs,
-                                     RBracLoc, move(Args));
+                                     RBracLoc, Args);
   }
 
   /// \brief Build a new Objective-C instance message.
@@ -2303,7 +2299,7 @@ public:
                                         Receiver->getType(),
                                         /*SuperLoc=*/SourceLocation(),
                                         Sel, Method, LBracLoc, SelectorLocs,
-                                        RBracLoc, move(Args));
+                                        RBracLoc, Args);
   }
 
   /// \brief Build a new Objective-C ivar reference expression.
@@ -2326,7 +2322,7 @@ public:
       return ExprError();
 
     if (Result.get())
-      return move(Result);
+      return Result;
 
     return getSema().BuildMemberReferenceExpr(Base.get(), Base.get()->getType(),
                                               /*FIXME:*/IvarLoc, IsArrow,
@@ -2355,7 +2351,7 @@ public:
       return ExprError();
 
     if (Result.get())
-      return move(Result);
+      return Result;
 
     return getSema().BuildMemberReferenceExpr(Base.get(), Base.get()->getType(),
                                               /*FIXME:*/PropertyLoc, IsArrow,
@@ -2398,7 +2394,7 @@ public:
       return ExprError();
 
     if (Result.get())
-      return move(Result);
+      return Result;
 
     return getSema().BuildMemberReferenceExpr(Base.get(), Base.get()->getType(),
                                               /*FIXME:*/IsaLoc, IsArrow,
@@ -2424,21 +2420,17 @@ public:
 
     // Build a reference to the __builtin_shufflevector builtin
     FunctionDecl *Builtin = cast<FunctionDecl>(*Lookup.first);
-    ExprResult Callee
-      = SemaRef.Owned(new (SemaRef.Context) DeclRefExpr(Builtin, false,
-                                                        Builtin->getType(),
-                                                        VK_LValue, BuiltinLoc));
-    Callee = SemaRef.UsualUnaryConversions(Callee.take());
-    if (Callee.isInvalid())
-      return ExprError();
+    Expr *Callee = new (SemaRef.Context) DeclRefExpr(Builtin, false,
+                                                  SemaRef.Context.BuiltinFnTy,
+                                                  VK_RValue, BuiltinLoc);
+    QualType CalleePtrTy = SemaRef.Context.getPointerType(Builtin->getType());
+    Callee = SemaRef.ImpCastExprToType(Callee, CalleePtrTy,
+                                       CK_BuiltinFnToFnPtr).take();
 
     // Build the CallExpr
-    unsigned NumSubExprs = SubExprs.size();
-    Expr **Subs = (Expr **)SubExprs.release();
     ExprResult TheCall = SemaRef.Owned(
-      new (SemaRef.Context) CallExpr(SemaRef.Context, Callee.take(),
-                                                       Subs, NumSubExprs,
-                                                   Builtin->getCallResultType(),
+      new (SemaRef.Context) CallExpr(SemaRef.Context, Callee, SubExprs,
+                                     Builtin->getCallResultType(),
                             Expr::getValueKindForType(Builtin->getResultType()),
                                      RParenLoc));
 
@@ -2478,6 +2470,7 @@ public:
     case TemplateArgument::Declaration:
     case TemplateArgument::Pack:
     case TemplateArgument::TemplateExpansion:
+    case TemplateArgument::NullPtr:
       llvm_unreachable("Pack expansion pattern has no parameter packs");
 
     case TemplateArgument::Type:
@@ -2515,10 +2508,7 @@ public:
     // Just create the expression; there is not any interesting semantic
     // analysis here because we can't actually build an AtomicExpr until
     // we are sure it is semantically sound.
-    unsigned NumSubExprs = SubExprs.size();
-    Expr **Subs = (Expr **)SubExprs.release();
-    return new (SemaRef.Context) AtomicExpr(BuiltinLoc, Subs,
-                                            NumSubExprs, RetTy, Op,
+    return new (SemaRef.Context) AtomicExpr(BuiltinLoc, SubExprs, RetTy, Op,
                                             RParenLoc);
   }
 
@@ -2963,6 +2953,7 @@ void TreeTransform<Derived>::InventTemplateArgumentLoc(
   case TemplateArgument::Declaration:
   case TemplateArgument::Integral:
   case TemplateArgument::Pack:
+  case TemplateArgument::NullPtr:
     Output = TemplateArgumentLoc(Arg, TemplateArgumentLocInfo());
     break;
   }
@@ -2976,8 +2967,10 @@ bool TreeTransform<Derived>::TransformTemplateArgument(
   switch (Arg.getKind()) {
   case TemplateArgument::Null:
   case TemplateArgument::Integral:
-    Output = Input;
-    return false;
+  case TemplateArgument::Pack:
+  case TemplateArgument::Declaration:
+  case TemplateArgument::NullPtr:
+    llvm_unreachable("Unexpected TemplateArgument");
 
   case TemplateArgument::Type: {
     TypeSourceInfo *DI = Input.getTypeSourceInfo();
@@ -2991,28 +2984,6 @@ bool TreeTransform<Derived>::TransformTemplateArgument(
     return false;
   }
 
-  case TemplateArgument::Declaration: {
-    // FIXME: we should never have to transform one of these.
-    DeclarationName Name;
-    if (NamedDecl *ND = dyn_cast<NamedDecl>(Arg.getAsDecl()))
-      Name = ND->getDeclName();
-    TemporaryBase Rebase(*this, Input.getLocation(), Name);
-    Decl *D = getDerived().TransformDecl(Input.getLocation(), Arg.getAsDecl());
-    if (!D) return true;
-
-    Expr *SourceExpr = Input.getSourceDeclExpression();
-    if (SourceExpr) {
-      EnterExpressionEvaluationContext Unevaluated(getSema(),
-                                                   Sema::ConstantEvaluated);
-      ExprResult E = getDerived().TransformExpr(SourceExpr);
-      E = SemaRef.ActOnConstantExpression(E);
-      SourceExpr = (E.isInvalid() ? 0 : E.take());
-    }
-
-    Output = TemplateArgumentLoc(TemplateArgument(D), SourceExpr);
-    return false;
-  }
-
   case TemplateArgument::Template: {
     NestedNameSpecifierLoc QualifierLoc = Input.getTemplateQualifierLoc();
     if (QualifierLoc) {
@@ -3051,35 +3022,6 @@ bool TreeTransform<Derived>::TransformTemplateArgument(
     Output = TemplateArgumentLoc(TemplateArgument(E.take()), E.take());
     return false;
   }
-
-  case TemplateArgument::Pack: {
-    SmallVector<TemplateArgument, 4> TransformedArgs;
-    TransformedArgs.reserve(Arg.pack_size());
-    for (TemplateArgument::pack_iterator A = Arg.pack_begin(),
-                                      AEnd = Arg.pack_end();
-         A != AEnd; ++A) {
-
-      // FIXME: preserve source information here when we start
-      // caring about parameter packs.
-
-      TemplateArgumentLoc InputArg;
-      TemplateArgumentLoc OutputArg;
-      getDerived().InventTemplateArgumentLoc(*A, InputArg);
-      if (getDerived().TransformTemplateArgument(InputArg, OutputArg))
-        return true;
-
-      TransformedArgs.push_back(OutputArg.getArgument());
-    }
-
-    TemplateArgument *TransformedArgsPtr
-      = new (getSema().Context) TemplateArgument[TransformedArgs.size()];
-    std::copy(TransformedArgs.begin(), TransformedArgs.end(),
-              TransformedArgsPtr);
-    Output = TemplateArgumentLoc(TemplateArgument(TransformedArgsPtr,
-                                                  TransformedArgs.size()),
-                                 Input.getLocInfo());
-    return false;
-  }
   }
 
   // Work around bogus GCC warning
@@ -4260,6 +4202,8 @@ TreeTransform<Derived>::TransformFunctionProtoType(TypeLocBuilder &TLB,
 
   FunctionProtoTypeLoc NewTL = TLB.push<FunctionProtoTypeLoc>(Result);
   NewTL.setLocalRangeBegin(TL.getLocalRangeBegin());
+  NewTL.setLParenLoc(TL.getLParenLoc());
+  NewTL.setRParenLoc(TL.getRParenLoc());
   NewTL.setLocalRangeEnd(TL.getLocalRangeEnd());
   for (unsigned i = 0, e = NewTL.getNumArgs(); i != e; ++i)
     NewTL.setArg(i, ParamDecls[i]);
@@ -4283,6 +4227,8 @@ QualType TreeTransform<Derived>::TransformFunctionNoProtoType(
 
   FunctionNoProtoTypeLoc NewTL = TLB.push<FunctionNoProtoTypeLoc>(Result);
   NewTL.setLocalRangeBegin(TL.getLocalRangeBegin());
+  NewTL.setLParenLoc(TL.getLParenLoc());
+  NewTL.setRParenLoc(TL.getRParenLoc());
   NewTL.setLocalRangeEnd(TL.getLocalRangeEnd());
 
   return Result;
@@ -4339,7 +4285,8 @@ template<typename Derived>
 QualType TreeTransform<Derived>::TransformTypeOfExprType(TypeLocBuilder &TLB,
                                                       TypeOfExprTypeLoc TL) {
   // typeof expressions are not potentially evaluated contexts
-  EnterExpressionEvaluationContext Unevaluated(SemaRef, Sema::Unevaluated);
+  EnterExpressionEvaluationContext Unevaluated(SemaRef, Sema::Unevaluated,
+                                               Sema::ReuseLambdaContextDecl);
 
   ExprResult E = getDerived().TransformExpr(TL.getUnderlyingExpr());
   if (E.isInvalid())
@@ -5099,7 +5046,7 @@ TreeTransform<Derived>::TransformCompoundStmt(CompoundStmt *S,
 
   bool SubStmtInvalid = false;
   bool SubStmtChanged = false;
-  ASTOwningVector<Stmt*> Statements(getSema());
+  SmallVector<Stmt*, 8> Statements;
   for (CompoundStmt::body_iterator B = S->body_begin(), BEnd = S->body_end();
        B != BEnd; ++B) {
     StmtResult Result = getDerived().TransformStmt(*B);
@@ -5126,7 +5073,7 @@ TreeTransform<Derived>::TransformCompoundStmt(CompoundStmt *S,
     return SemaRef.Owned(S);
 
   return getDerived().RebuildCompoundStmt(S->getLBracLoc(),
-                                          move_arg(Statements),
+                                          Statements,
                                           S->getRBracLoc(),
                                           IsStmtExpr);
 }
@@ -5533,14 +5480,14 @@ TreeTransform<Derived>::TransformDeclStmt(DeclStmt *S) {
 
 template<typename Derived>
 StmtResult
-TreeTransform<Derived>::TransformAsmStmt(AsmStmt *S) {
+TreeTransform<Derived>::TransformGCCAsmStmt(GCCAsmStmt *S) {
 
-  ASTOwningVector<Expr*> Constraints(getSema());
-  ASTOwningVector<Expr*> Exprs(getSema());
+  SmallVector<Expr*, 8> Constraints;
+  SmallVector<Expr*, 8> Exprs;
   SmallVector<IdentifierInfo *, 4> Names;
 
   ExprResult AsmString;
-  ASTOwningVector<Expr*> Clobbers(getSema());
+  SmallVector<Expr*, 8> Clobbers;
 
   bool ExprsChanged = false;
 
@@ -5585,23 +5532,15 @@ TreeTransform<Derived>::TransformAsmStmt(AsmStmt *S) {
 
   // Go through the clobbers.
   for (unsigned I = 0, E = S->getNumClobbers(); I != E; ++I)
-    Clobbers.push_back(S->getClobber(I));
+    Clobbers.push_back(S->getClobberStringLiteral(I));
 
   // No need to transform the asm string literal.
   AsmString = SemaRef.Owned(S->getAsmString());
-
-  return getDerived().RebuildAsmStmt(S->getAsmLoc(),
-                                     S->isSimple(),
-                                     S->isVolatile(),
-                                     S->getNumOutputs(),
-                                     S->getNumInputs(),
-                                     Names.data(),
-                                     move_arg(Constraints),
-                                     move_arg(Exprs),
-                                     AsmString.get(),
-                                     move_arg(Clobbers),
-                                     S->getRParenLoc(),
-                                     S->isMSAsm());
+  return getDerived().RebuildGCCAsmStmt(S->getAsmLoc(), S->isSimple(),
+                                        S->isVolatile(), S->getNumOutputs(),
+                                        S->getNumInputs(), Names.data(),
+                                        Constraints, Exprs, AsmString.get(),
+                                        Clobbers, S->getRParenLoc());
 }
 
 template<typename Derived>
@@ -5624,7 +5563,7 @@ TreeTransform<Derived>::TransformObjCAtTryStmt(ObjCAtTryStmt *S) {
 
   // Transform the @catch statements (if present).
   bool AnyCatchChanged = false;
-  ASTOwningVector<Stmt*> CatchStmts(SemaRef);
+  SmallVector<Stmt*, 8> CatchStmts;
   for (unsigned I = 0, N = S->getNumCatchStmts(); I != N; ++I) {
     StmtResult Catch = getDerived().TransformStmt(S->getCatchStmt(I));
     if (Catch.isInvalid())
@@ -5651,7 +5590,7 @@ TreeTransform<Derived>::TransformObjCAtTryStmt(ObjCAtTryStmt *S) {
 
   // Build a new statement.
   return getDerived().RebuildObjCAtTryStmt(S->getAtTryLoc(), TryBody.get(),
-                                           move_arg(CatchStmts), Finally.get());
+                                           CatchStmts, Finally.get());
 }
 
 template<typename Derived>
@@ -5855,7 +5794,7 @@ TreeTransform<Derived>::TransformCXXTryStmt(CXXTryStmt *S) {
 
   // Transform the handlers.
   bool HandlerChanged = false;
-  ASTOwningVector<Stmt*> Handlers(SemaRef);
+  SmallVector<Stmt*, 8> Handlers;
   for (unsigned I = 0, N = S->getNumHandlers(); I != N; ++I) {
     StmtResult Handler
       = getDerived().TransformCXXCatchStmt(S->getHandler(I));
@@ -5872,7 +5811,7 @@ TreeTransform<Derived>::TransformCXXTryStmt(CXXTryStmt *S) {
     return SemaRef.Owned(S);
 
   return getDerived().RebuildCXXTryStmt(S->getTryLoc(), TryBlock.get(),
-                                        move_arg(Handlers));
+                                        Handlers);
 }
 
 template<typename Derived>
@@ -6205,10 +6144,22 @@ TreeTransform<Derived>::TransformParenExpr(ParenExpr *E) {
                                        E->getRParen());
 }
 
+/// \brief The operand of a unary address-of operator has special rules: it's
+/// allowed to refer to a non-static member of a class even if there's no 'this'
+/// object available.
+template<typename Derived>
+ExprResult
+TreeTransform<Derived>::TransformAddressOfOperand(Expr *E) {
+  if (DependentScopeDeclRefExpr *DRE = dyn_cast<DependentScopeDeclRefExpr>(E))
+    return getDerived().TransformDependentScopeDeclRefExpr(DRE, true);
+  else
+    return getDerived().TransformExpr(E);
+}
+
 template<typename Derived>
 ExprResult
 TreeTransform<Derived>::TransformUnaryOperator(UnaryOperator *E) {
-  ExprResult SubExpr = getDerived().TransformExpr(E->getSubExpr());
+  ExprResult SubExpr = TransformAddressOfOperand(E->getSubExpr());
   if (SubExpr.isInvalid())
     return ExprError();
 
@@ -6338,7 +6289,8 @@ TreeTransform<Derived>::TransformUnaryExprOrTypeTraitExpr(
   // C++0x [expr.sizeof]p1:
   //   The operand is either an expression, which is an unevaluated operand
   //   [...]
-  EnterExpressionEvaluationContext Unevaluated(SemaRef, Sema::Unevaluated);
+  EnterExpressionEvaluationContext Unevaluated(SemaRef, Sema::Unevaluated,
+                                               Sema::ReuseLambdaContextDecl);
 
   ExprResult SubExpr = getDerived().TransformExpr(E->getArgumentExpr());
   if (SubExpr.isInvalid())
@@ -6386,7 +6338,7 @@ TreeTransform<Derived>::TransformCallExpr(CallExpr *E) {
 
   // Transform arguments.
   bool ArgChanged = false;
-  ASTOwningVector<Expr*> Args(SemaRef);
+  SmallVector<Expr*, 8> Args;
   if (getDerived().TransformExprs(E->getArgs(), E->getNumArgs(), true, Args,
                                   &ArgChanged))
     return ExprError();
@@ -6394,13 +6346,13 @@ TreeTransform<Derived>::TransformCallExpr(CallExpr *E) {
   if (!getDerived().AlwaysRebuild() &&
       Callee.get() == E->getCallee() &&
       !ArgChanged)
-    return SemaRef.MaybeBindToTemporary(E);;
+    return SemaRef.MaybeBindToTemporary(E);
 
   // FIXME: Wrong source location information for the '('.
   SourceLocation FakeLParenLoc
     = ((Expr *)Callee.get())->getSourceRange().getBegin();
   return getDerived().RebuildCallExpr(Callee.get(), FakeLParenLoc,
-                                      move_arg(Args),
+                                      Args,
                                       E->getRParenLoc());
 }
 
@@ -6499,6 +6451,9 @@ TreeTransform<Derived>::TransformBinaryOperator(BinaryOperator *E) {
       RHS.get() == E->getRHS())
     return SemaRef.Owned(E);
 
+  Sema::FPContractStateRAII FPContractState(getSema());
+  getSema().FPFeatures.fp_contract = E->isFPContractable();
+
   return getDerived().RebuildBinaryOperator(E->getOperatorLoc(), E->getOpcode(),
                                             LHS.get(), RHS.get());
 }
@@ -6645,7 +6600,7 @@ ExprResult
 TreeTransform<Derived>::TransformInitListExpr(InitListExpr *E) {
   bool InitChanged = false;
 
-  ASTOwningVector<Expr*, 4> Inits(SemaRef);
+  SmallVector<Expr*, 4> Inits;
   if (getDerived().TransformExprs(E->getInits(), E->getNumInits(), false,
                                   Inits, &InitChanged))
     return ExprError();
@@ -6653,7 +6608,7 @@ TreeTransform<Derived>::TransformInitListExpr(InitListExpr *E) {
   if (!getDerived().AlwaysRebuild() && !InitChanged)
     return SemaRef.Owned(E);
 
-  return getDerived().RebuildInitList(E->getLBraceLoc(), move_arg(Inits),
+  return getDerived().RebuildInitList(E->getLBraceLoc(), Inits,
                                       E->getRBraceLoc(), E->getType());
 }
 
@@ -6668,7 +6623,7 @@ TreeTransform<Derived>::TransformDesignatedInitExpr(DesignatedInitExpr *E) {
     return ExprError();
 
   // transform the designators.
-  ASTOwningVector<Expr*, 4> ArrayExprs(SemaRef);
+  SmallVector<Expr*, 4> ArrayExprs;
   bool ExprChanged = false;
   for (DesignatedInitExpr::designators_iterator D = E->designators_begin(),
                                              DEnd = E->designators_end();
@@ -6720,7 +6675,7 @@ TreeTransform<Derived>::TransformDesignatedInitExpr(DesignatedInitExpr *E) {
       !ExprChanged)
     return SemaRef.Owned(E);
 
-  return getDerived().RebuildDesignatedInitExpr(Desig, move_arg(ArrayExprs),
+  return getDerived().RebuildDesignatedInitExpr(Desig, ArrayExprs,
                                                 E->getEqualOrColonLoc(),
                                                 E->usesGNUSyntax(), Init.get());
 }
@@ -6768,13 +6723,13 @@ template<typename Derived>
 ExprResult
 TreeTransform<Derived>::TransformParenListExpr(ParenListExpr *E) {
   bool ArgumentChanged = false;
-  ASTOwningVector<Expr*, 4> Inits(SemaRef);
+  SmallVector<Expr*, 4> Inits;
   if (TransformExprs(E->getExprs(), E->getNumExprs(), true, Inits,
                      &ArgumentChanged))
     return ExprError();
 
   return getDerived().RebuildParenListExpr(E->getLParenLoc(),
-                                           move_arg(Inits),
+                                           Inits,
                                            E->getRParenLoc());
 }
 
@@ -6875,13 +6830,13 @@ TreeTransform<Derived>::TransformCXXOperatorCallExpr(CXXOperatorCallExpr *E) {
                               static_cast<Expr *>(Object.get())->getLocEnd());
 
     // Transform the call arguments.
-    ASTOwningVector<Expr*> Args(SemaRef);
+    SmallVector<Expr*, 8> Args;
     if (getDerived().TransformExprs(E->getArgs() + 1, E->getNumArgs() - 1, true,
                                     Args))
       return ExprError();
 
     return getDerived().RebuildCallExpr(Object.get(), FakeLParenLoc,
-                                        move_arg(Args),
+                                        Args,
                                         E->getLocEnd());
   }
 
@@ -6905,7 +6860,11 @@ TreeTransform<Derived>::TransformCXXOperatorCallExpr(CXXOperatorCallExpr *E) {
   if (Callee.isInvalid())
     return ExprError();
 
-  ExprResult First = getDerived().TransformExpr(E->getArg(0));
+  ExprResult First;
+  if (E->getOperator() == OO_Amp)
+    First = getDerived().TransformAddressOfOperand(E->getArg(0));
+  else
+    First = getDerived().TransformExpr(E->getArg(0));
   if (First.isInvalid())
     return ExprError();
 
@@ -6922,6 +6881,9 @@ TreeTransform<Derived>::TransformCXXOperatorCallExpr(CXXOperatorCallExpr *E) {
       (E->getNumArgs() != 2 || Second.get() == E->getArg(1)))
     return SemaRef.MaybeBindToTemporary(E);
 
+  Sema::FPContractStateRAII FPContractState(getSema());
+  getSema().FPFeatures.fp_contract = E->isFPContractable();
+
   return getDerived().RebuildCXXOperatorCallExpr(E->getOperator(),
                                                  E->getOperatorLoc(),
                                                  Callee.get(),
@@ -6950,7 +6912,7 @@ TreeTransform<Derived>::TransformCUDAKernelCallExpr(CUDAKernelCallExpr *E) {
 
   // Transform arguments.
   bool ArgChanged = false;
-  ASTOwningVector<Expr*> Args(SemaRef);
+  SmallVector<Expr*, 8> Args;
   if (getDerived().TransformExprs(E->getArgs(), E->getNumArgs(), true, Args,
                                   &ArgChanged))
     return ExprError();
@@ -6964,7 +6926,7 @@ TreeTransform<Derived>::TransformCUDAKernelCallExpr(CUDAKernelCallExpr *E) {
   SourceLocation FakeLParenLoc
     = ((Expr *)Callee.get())->getSourceRange().getBegin();
   return getDerived().RebuildCallExpr(Callee.get(), FakeLParenLoc,
-                                      move_arg(Args),
+                                      Args,
                                       E->getRParenLoc(), EC.get());
 }
 
@@ -6989,9 +6951,6 @@ TreeTransform<Derived>::TransformCXXNamedCastExpr(CXXNamedCastExpr *E) {
   SourceLocation FakeLAngleLoc
     = SemaRef.PP.getLocForEndOfToken(E->getOperatorLoc());
   SourceLocation FakeRAngleLoc = E->getSubExpr()->getSourceRange().getBegin();
-  SourceLocation FakeRParenLoc
-    = SemaRef.PP.getLocForEndOfToken(
-                                  E->getSubExpr()->getSourceRange().getEnd());
   return getDerived().RebuildCXXNamedCastExpr(E->getOperatorLoc(),
                                               E->getStmtClass(),
                                               FakeLAngleLoc,
@@ -6999,7 +6958,7 @@ TreeTransform<Derived>::TransformCXXNamedCastExpr(CXXNamedCastExpr *E) {
                                               FakeRAngleLoc,
                                               FakeRAngleLoc,
                                               SubExpr.get(),
-                                              FakeRParenLoc);
+                                              E->getRParenLoc());
 }
 
 template<typename Derived>
@@ -7074,7 +7033,8 @@ TreeTransform<Derived>::TransformCXXTypeidExpr(CXXTypeidExpr *E) {
   // after we perform semantic analysis.  We speculatively assume it is
   // unevaluated; it will get fixed later if the subexpression is in fact
   // potentially evaluated.
-  EnterExpressionEvaluationContext Unevaluated(SemaRef, Sema::Unevaluated);
+  EnterExpressionEvaluationContext Unevaluated(SemaRef, Sema::Unevaluated,
+                                               Sema::ReuseLambdaContextDecl);
 
   ExprResult SubExpr = getDerived().TransformExpr(E->getExprOperand());
   if (SubExpr.isInvalid())
@@ -7222,7 +7182,7 @@ TreeTransform<Derived>::TransformCXXNewExpr(CXXNewExpr *E) {
 
   // Transform the placement arguments (if any).
   bool ArgumentChanged = false;
-  ASTOwningVector<Expr*> PlacementArgs(SemaRef);
+  SmallVector<Expr*, 8> PlacementArgs;
   if (getDerived().TransformExprs(E->getPlacementArgs(),
                                   E->getNumPlacementArgs(), true,
                                   PlacementArgs, &ArgumentChanged))
@@ -7313,7 +7273,7 @@ TreeTransform<Derived>::TransformCXXNewExpr(CXXNewExpr *E) {
   return getDerived().RebuildCXXNewExpr(E->getLocStart(),
                                         E->isGlobalNew(),
                                         /*FIXME:*/E->getLocStart(),
-                                        move_arg(PlacementArgs),
+                                        PlacementArgs,
                                         /*FIXME:*/E->getLocStart(),
                                         E->getTypeIdParens(),
                                         AllocType,
@@ -7512,7 +7472,8 @@ TreeTransform<Derived>::TransformUnresolvedLookupExpr(
   // If we have template arguments, rebuild them, then rebuild the
   // templateid expression.
   TemplateArgumentListInfo TransArgs(Old->getLAngleLoc(), Old->getRAngleLoc());
-  if (getDerived().TransformTemplateArguments(Old->getTemplateArgs(),
+  if (Old->hasExplicitTemplateArgs() &&
+      getDerived().TransformTemplateArguments(Old->getTemplateArgs(),
                                               Old->getNumTemplateArgs(),
                                               TransArgs))
     return ExprError();
@@ -7732,6 +7693,14 @@ template<typename Derived>
 ExprResult
 TreeTransform<Derived>::TransformDependentScopeDeclRefExpr(
                                                DependentScopeDeclRefExpr *E) {
+  return TransformDependentScopeDeclRefExpr(E, /*IsAddressOfOperand*/false);
+}
+
+template<typename Derived>
+ExprResult
+TreeTransform<Derived>::TransformDependentScopeDeclRefExpr(
+                                               DependentScopeDeclRefExpr *E,
+                                               bool IsAddressOfOperand) {
   NestedNameSpecifierLoc QualifierLoc
   = getDerived().TransformNestedNameSpecifierLoc(E->getQualifierLoc());
   if (!QualifierLoc)
@@ -7758,7 +7727,8 @@ TreeTransform<Derived>::TransformDependentScopeDeclRefExpr(
     return getDerived().RebuildDependentScopeDeclRefExpr(QualifierLoc,
                                                          TemplateKWLoc,
                                                          NameInfo,
-                                                         /*TemplateArgs*/ 0);
+                                                         /*TemplateArgs*/ 0,
+                                                         IsAddressOfOperand);
   }
 
   TemplateArgumentListInfo TransArgs(E->getLAngleLoc(), E->getRAngleLoc());
@@ -7770,7 +7740,8 @@ TreeTransform<Derived>::TransformDependentScopeDeclRefExpr(
   return getDerived().RebuildDependentScopeDeclRefExpr(QualifierLoc,
                                                        TemplateKWLoc,
                                                        NameInfo,
-                                                       &TransArgs);
+                                                       &TransArgs,
+                                                       IsAddressOfOperand);
 }
 
 template<typename Derived>
@@ -7796,7 +7767,7 @@ TreeTransform<Derived>::TransformCXXConstructExpr(CXXConstructExpr *E) {
     return ExprError();
 
   bool ArgumentChanged = false;
-  ASTOwningVector<Expr*> Args(SemaRef);
+  SmallVector<Expr*, 8> Args;
   if (getDerived().TransformExprs(E->getArgs(), E->getNumArgs(), true, Args,
                                   &ArgumentChanged))
     return ExprError();
@@ -7813,7 +7784,7 @@ TreeTransform<Derived>::TransformCXXConstructExpr(CXXConstructExpr *E) {
 
   return getDerived().RebuildCXXConstructExpr(T, /*FIXME:*/E->getLocStart(),
                                               Constructor, E->isElidable(),
-                                              move_arg(Args),
+                                              Args,
                                               E->hadMultipleCandidates(),
                                               E->requiresZeroInitialization(),
                                               E->getConstructionKind(),
@@ -7857,7 +7828,7 @@ TreeTransform<Derived>::TransformCXXTemporaryObjectExpr(
     return ExprError();
 
   bool ArgumentChanged = false;
-  ASTOwningVector<Expr*> Args(SemaRef);
+  SmallVector<Expr*, 8> Args;
   Args.reserve(E->getNumArgs());
   if (TransformExprs(E->getArgs(), E->getNumArgs(), true, Args,
                      &ArgumentChanged))
@@ -7874,19 +7845,13 @@ TreeTransform<Derived>::TransformCXXTemporaryObjectExpr(
 
   return getDerived().RebuildCXXTemporaryObjectExpr(T,
                                           /*FIXME:*/T->getTypeLoc().getEndLoc(),
-                                                    move_arg(Args),
+                                                    Args,
                                                     E->getLocEnd());
 }
 
 template<typename Derived>
 ExprResult
 TreeTransform<Derived>::TransformLambdaExpr(LambdaExpr *E) {
-  // Create the local class that will describe the lambda.
-  CXXRecordDecl *Class
-    = getSema().createLambdaClosureType(E->getIntroducerRange(),
-                                        /*KnownDependent=*/false);
-  getDerived().transformedLocalDecl(E->getLambdaClass(), Class);
-
   // Transform the type of the lambda parameters and start the definition of
   // the lambda itself.
   TypeSourceInfo *MethodTy
@@ -7894,6 +7859,13 @@ TreeTransform<Derived>::TransformLambdaExpr(LambdaExpr *E) {
   if (!MethodTy)
     return ExprError();
 
+  // Create the local class that will describe the lambda.
+  CXXRecordDecl *Class
+    = getSema().createLambdaClosureType(E->getIntroducerRange(),
+                                        MethodTy,
+                                        /*KnownDependent=*/false);
+  getDerived().transformedLocalDecl(E->getLambdaClass(), Class);
+
   // Transform lambda parameters.
   llvm::SmallVector<QualType, 4> ParamTypes;
   llvm::SmallVector<ParmVarDecl *, 4> Params;
@@ -8038,7 +8010,7 @@ TreeTransform<Derived>::TransformCXXUnresolvedConstructExpr(
     return ExprError();
 
   bool ArgumentChanged = false;
-  ASTOwningVector<Expr*> Args(SemaRef);
+  SmallVector<Expr*, 8> Args;
   Args.reserve(E->arg_size());
   if (getDerived().TransformExprs(E->arg_begin(), E->arg_size(), true, Args,
                                   &ArgumentChanged))
@@ -8052,7 +8024,7 @@ TreeTransform<Derived>::TransformCXXUnresolvedConstructExpr(
   // FIXME: we're faking the locations of the commas
   return getDerived().RebuildCXXUnresolvedConstructExpr(T,
                                                         E->getLParenLoc(),
-                                                        move_arg(Args),
+                                                        Args,
                                                         E->getRParenLoc());
 }
 
@@ -8346,6 +8318,13 @@ TreeTransform<Derived>::TransformSubstNonTypeTemplateParmExpr(
 
 template<typename Derived>
 ExprResult
+TreeTransform<Derived>::TransformFunctionParmPackExpr(FunctionParmPackExpr *E) {
+  // Default behavior is to do nothing with this transformation.
+  return SemaRef.Owned(E);
+}
+
+template<typename Derived>
+ExprResult
 TreeTransform<Derived>::TransformMaterializeTemporaryExpr(
                                                   MaterializeTemporaryExpr *E) {
   return getDerived().TransformExpr(E->GetTemporaryExpr());
@@ -8576,7 +8555,7 @@ ExprResult
 TreeTransform<Derived>::TransformObjCMessageExpr(ObjCMessageExpr *E) {
   // Transform arguments.
   bool ArgChanged = false;
-  ASTOwningVector<Expr*> Args(SemaRef);
+  SmallVector<Expr*, 8> Args;
   Args.reserve(E->getNumArgs());
   if (getDerived().TransformExprs(E->getArgs(), E->getNumArgs(), false, Args,
                                   &ArgChanged))
@@ -8602,7 +8581,7 @@ TreeTransform<Derived>::TransformObjCMessageExpr(ObjCMessageExpr *E) {
                                                SelLocs,
                                                E->getMethodDecl(),
                                                E->getLeftLoc(),
-                                               move_arg(Args),
+                                               Args,
                                                E->getRightLoc());
   }
 
@@ -8627,7 +8606,7 @@ TreeTransform<Derived>::TransformObjCMessageExpr(ObjCMessageExpr *E) {
                                              SelLocs,
                                              E->getMethodDecl(),
                                              E->getLeftLoc(),
-                                             move_arg(Args),
+                                             Args,
                                              E->getRightLoc());
 }
 
@@ -8740,7 +8719,7 @@ template<typename Derived>
 ExprResult
 TreeTransform<Derived>::TransformShuffleVectorExpr(ShuffleVectorExpr *E) {
   bool ArgumentChanged = false;
-  ASTOwningVector<Expr*> SubExprs(SemaRef);
+  SmallVector<Expr*, 8> SubExprs;
   SubExprs.reserve(E->getNumSubExprs());
   if (getDerived().TransformExprs(E->getSubExprs(), E->getNumSubExprs(), false,
                                   SubExprs, &ArgumentChanged))
@@ -8751,7 +8730,7 @@ TreeTransform<Derived>::TransformShuffleVectorExpr(ShuffleVectorExpr *E) {
     return SemaRef.Owned(E);
 
   return getDerived().RebuildShuffleVectorExpr(E->getBuiltinLoc(),
-                                               move_arg(SubExprs),
+                                               SubExprs,
                                                E->getRParenLoc());
 }
 
@@ -8854,7 +8833,7 @@ ExprResult
 TreeTransform<Derived>::TransformAtomicExpr(AtomicExpr *E) {
   QualType RetTy = getDerived().TransformType(E->getType());
   bool ArgumentChanged = false;
-  ASTOwningVector<Expr*> SubExprs(SemaRef);
+  SmallVector<Expr*, 8> SubExprs;
   SubExprs.reserve(E->getNumSubExprs());
   if (getDerived().TransformExprs(E->getSubExprs(), E->getNumSubExprs(), false,
                                   SubExprs, &ArgumentChanged))
@@ -8864,7 +8843,7 @@ TreeTransform<Derived>::TransformAtomicExpr(AtomicExpr *E) {
       !ArgumentChanged)
     return SemaRef.Owned(E);
 
-  return getDerived().RebuildAtomicExpr(E->getBuiltinLoc(), move_arg(SubExprs),
+  return getDerived().RebuildAtomicExpr(E->getBuiltinLoc(), SubExprs,
                                         RetTy, E->getOp(), E->getRParenLoc());
 }
 
@@ -9185,7 +9164,7 @@ TreeTransform<Derived>::RebuildCXXOperatorCallExpr(OverloadedOperatorKind Op,
       if (Result.isInvalid())
         return ExprError();
 
-      return move(Result);
+      return Result;
     }
   }
 
@@ -9200,7 +9179,12 @@ TreeTransform<Derived>::RebuildCXXOperatorCallExpr(OverloadedOperatorKind Op,
     // IsAcceptableNonMemberOperatorCandidate for each of these?
     Functions.append(ULE->decls_begin(), ULE->decls_end());
   } else {
-    Functions.addDecl(cast<DeclRefExpr>(Callee)->getDecl());
+    // If we've resolved this to a particular non-member function, just call
+    // that function. If we resolved it to a member function,
+    // CreateOverloaded* will find that function for us.
+    NamedDecl *ND = cast<DeclRefExpr>(Callee)->getDecl();
+    if (!isa<CXXMethodDecl>(ND))
+      Functions.addDecl(ND);
   }
 
   // Add any functions found via argument-dependent lookup.
@@ -9240,7 +9224,7 @@ TreeTransform<Derived>::RebuildCXXOperatorCallExpr(OverloadedOperatorKind Op,
   if (Result.isInvalid())
     return ExprError();
 
-  return move(Result);
+  return Result;
 }
 
 template<typename Derived>
diff --git a/lib/Serialization/ASTCommon.cpp b/lib/Serialization/ASTCommon.cpp
index 67f74f7..0ec03cf 100644
--- a/lib/Serialization/ASTCommon.cpp
+++ b/lib/Serialization/ASTCommon.cpp
@@ -60,6 +60,9 @@ serialization::TypeIdxFromBuiltin(const BuiltinType *BT) {
   case BuiltinType::ObjCId:     ID = PREDEF_TYPE_OBJC_ID;       break;
   case BuiltinType::ObjCClass:  ID = PREDEF_TYPE_OBJC_CLASS;    break;
   case BuiltinType::ObjCSel:    ID = PREDEF_TYPE_OBJC_SEL;      break;
+  case BuiltinType::BuiltinFn:
+                                ID = PREDEF_TYPE_BUILTIN_FN; break;
+
   }
 
   return TypeIdx(ID);
diff --git a/lib/Serialization/ASTReader.cpp b/lib/Serialization/ASTReader.cpp
index 3adbc57..deba302 100644
--- a/lib/Serialization/ASTReader.cpp
+++ b/lib/Serialization/ASTReader.cpp
@@ -30,13 +30,16 @@
 #include "clang/Lex/MacroInfo.h"
 #include "clang/Lex/PreprocessingRecord.h"
 #include "clang/Lex/Preprocessor.h"
+#include "clang/Lex/PreprocessorOptions.h"
 #include "clang/Lex/HeaderSearch.h"
+#include "clang/Lex/HeaderSearchOptions.h"
 #include "clang/Basic/OnDiskHashTable.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/SourceManagerInternals.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/FileSystemStatCache.h"
 #include "clang/Basic/TargetInfo.h"
+#include "clang/Basic/TargetOptions.h"
 #include "clang/Basic/Version.h"
 #include "clang/Basic/VersionTuple.h"
 #include "llvm/ADT/StringExtras.h"
@@ -62,353 +65,306 @@ using namespace clang::serialization::reader;
 
 ASTReaderListener::~ASTReaderListener() {}
 
-bool
-PCHValidator::ReadLanguageOptions(const LangOptions &LangOpts) {
-  const LangOptions &PPLangOpts = PP.getLangOpts();
-  
-#define LANGOPT(Name, Bits, Default, Description)         \
-  if (PPLangOpts.Name != LangOpts.Name) {                 \
-    Reader.Diag(diag::err_pch_langopt_mismatch)           \
-      << Description << LangOpts.Name << PPLangOpts.Name; \
+/// \brief Compare the given set of language options against an existing set of
+/// language options.
+///
+/// \param Diags If non-NULL, diagnostics will be emitted via this engine.
+///
+/// \returns true if the languagae options mis-match, false otherwise.
+static bool checkLanguageOptions(const LangOptions &LangOpts,
+                                 const LangOptions &ExistingLangOpts,
+                                 DiagnosticsEngine *Diags) {
+#define LANGOPT(Name, Bits, Default, Description)                 \
+  if (ExistingLangOpts.Name != LangOpts.Name) {                   \
+    if (Diags)                                                    \
+      Diags->Report(diag::err_pch_langopt_mismatch)               \
+        << Description << LangOpts.Name << ExistingLangOpts.Name; \
+    return true;                                                  \
+  }
+
+#define VALUE_LANGOPT(Name, Bits, Default, Description)   \
+  if (ExistingLangOpts.Name != LangOpts.Name) {           \
+    if (Diags)                                            \
+      Diags->Report(diag::err_pch_langopt_value_mismatch) \
+        << Description;                                   \
     return true;                                          \
   }
 
-#define VALUE_LANGOPT(Name, Bits, Default, Description) \
-  if (PPLangOpts.Name != LangOpts.Name) {               \
-    Reader.Diag(diag::err_pch_langopt_value_mismatch)   \
-      << Description;                                   \
-  return true;                                          \
-}
-
-#define ENUM_LANGOPT(Name, Type, Bits, Default, Description) \
-  if (PPLangOpts.get##Name() != LangOpts.get##Name()) {      \
-    Reader.Diag(diag::err_pch_langopt_value_mismatch)        \
-      << Description;                                        \
-    return true;                                             \
+#define ENUM_LANGOPT(Name, Type, Bits, Default, Description)   \
+  if (ExistingLangOpts.get##Name() != LangOpts.get##Name()) {  \
+    if (Diags)                                                 \
+      Diags->Report(diag::err_pch_langopt_value_mismatch)      \
+        << Description;                                        \
+    return true;                                               \
   }
 
 #define BENIGN_LANGOPT(Name, Bits, Default, Description)
 #define BENIGN_ENUM_LANGOPT(Name, Type, Bits, Default, Description)
 #include "clang/Basic/LangOptions.def"
 
-  if (PPLangOpts.ObjCRuntime != LangOpts.ObjCRuntime) {
-    Reader.Diag(diag::err_pch_langopt_value_mismatch)
+  if (ExistingLangOpts.ObjCRuntime != LangOpts.ObjCRuntime) {
+    if (Diags)
+      Diags->Report(diag::err_pch_langopt_value_mismatch)
       << "target Objective-C runtime";
     return true;
   }
-  
+
   return false;
 }
 
-bool PCHValidator::ReadTargetTriple(StringRef Triple) {
-  if (Triple == PP.getTargetInfo().getTriple().str())
-    return false;
-
-  Reader.Diag(diag::warn_pch_target_triple)
-    << Triple << PP.getTargetInfo().getTriple().str();
-  return true;
-}
+/// \brief Compare the given set of target options against an existing set of
+/// target options.
+///
+/// \param Diags If non-NULL, diagnostics will be emitted via this engine.
+///
+/// \returns true if the target options mis-match, false otherwise.
+static bool checkTargetOptions(const TargetOptions &TargetOpts,
+                               const TargetOptions &ExistingTargetOpts,
+                               DiagnosticsEngine *Diags) {
+#define CHECK_TARGET_OPT(Field, Name)                             \
+  if (TargetOpts.Field != ExistingTargetOpts.Field) {             \
+    if (Diags)                                                    \
+      Diags->Report(diag::err_pch_targetopt_mismatch)             \
+        << Name << TargetOpts.Field << ExistingTargetOpts.Field;  \
+    return true;                                                  \
+  }
+
+  CHECK_TARGET_OPT(Triple, "target");
+  CHECK_TARGET_OPT(CPU, "target CPU");
+  CHECK_TARGET_OPT(ABI, "target ABI");
+  CHECK_TARGET_OPT(CXXABI, "target C++ ABI");
+  CHECK_TARGET_OPT(LinkerVersion, "target linker version");
+#undef CHECK_TARGET_OPT
+
+  // Compare feature sets.
+  SmallVector<StringRef, 4> ExistingFeatures(
+                                             ExistingTargetOpts.FeaturesAsWritten.begin(),
+                                             ExistingTargetOpts.FeaturesAsWritten.end());
+  SmallVector<StringRef, 4> ReadFeatures(TargetOpts.FeaturesAsWritten.begin(),
+                                         TargetOpts.FeaturesAsWritten.end());
+  std::sort(ExistingFeatures.begin(), ExistingFeatures.end());
+  std::sort(ReadFeatures.begin(), ReadFeatures.end());
+
+  unsigned ExistingIdx = 0, ExistingN = ExistingFeatures.size();
+  unsigned ReadIdx = 0, ReadN = ReadFeatures.size();
+  while (ExistingIdx < ExistingN && ReadIdx < ReadN) {
+    if (ExistingFeatures[ExistingIdx] == ReadFeatures[ReadIdx]) {
+      ++ExistingIdx;
+      ++ReadIdx;
+      continue;
+    }
 
-namespace {
-  struct EmptyStringRef {
-    bool operator ()(StringRef r) const { return r.empty(); }
-  };
-  struct EmptyBlock {
-    bool operator ()(const PCHPredefinesBlock &r) const {return r.Data.empty();}
-  };
-}
+    if (ReadFeatures[ReadIdx] < ExistingFeatures[ExistingIdx]) {
+      if (Diags)
+        Diags->Report(diag::err_pch_targetopt_feature_mismatch)
+          << false << ReadFeatures[ReadIdx];
+      return true;
+    }
 
-static bool EqualConcatenations(SmallVector<StringRef, 2> L,
-                                PCHPredefinesBlocks R) {
-  // First, sum up the lengths.
-  unsigned LL = 0, RL = 0;
-  for (unsigned I = 0, N = L.size(); I != N; ++I) {
-    LL += L[I].size();
-  }
-  for (unsigned I = 0, N = R.size(); I != N; ++I) {
-    RL += R[I].Data.size();
-  }
-  if (LL != RL)
-    return false;
-  if (LL == 0 && RL == 0)
+    if (Diags)
+      Diags->Report(diag::err_pch_targetopt_feature_mismatch)
+        << true << ExistingFeatures[ExistingIdx];
     return true;
-
-  // Kick out empty parts, they confuse the algorithm below.
-  L.erase(std::remove_if(L.begin(), L.end(), EmptyStringRef()), L.end());
-  R.erase(std::remove_if(R.begin(), R.end(), EmptyBlock()), R.end());
-
-  // Do it the hard way. At this point, both vectors must be non-empty.
-  StringRef LR = L[0], RR = R[0].Data;
-  unsigned LI = 0, RI = 0, LN = L.size(), RN = R.size();
-  (void) RN;
-  for (;;) {
-    // Compare the current pieces.
-    if (LR.size() == RR.size()) {
-      // If they're the same length, it's pretty easy.
-      if (LR != RR)
-        return false;
-      // Both pieces are done, advance.
-      ++LI;
-      ++RI;
-      // If either string is done, they're both done, since they're the same
-      // length.
-      if (LI == LN) {
-        assert(RI == RN && "Strings not the same length after all?");
-        return true;
-      }
-      LR = L[LI];
-      RR = R[RI].Data;
-    } else if (LR.size() < RR.size()) {
-      // Right piece is longer.
-      if (!RR.startswith(LR))
-        return false;
-      ++LI;
-      assert(LI != LN && "Strings not the same length after all?");
-      RR = RR.substr(LR.size());
-      LR = L[LI];
-    } else {
-      // Left piece is longer.
-      if (!LR.startswith(RR))
-        return false;
-      ++RI;
-      assert(RI != RN && "Strings not the same length after all?");
-      LR = LR.substr(RR.size());
-      RR = R[RI].Data;
-    }
   }
-}
 
-static std::pair<FileID, StringRef::size_type>
-FindMacro(const PCHPredefinesBlocks &Buffers, StringRef MacroDef) {
-  std::pair<FileID, StringRef::size_type> Res;
-  for (unsigned I = 0, N = Buffers.size(); I != N; ++I) {
-    Res.second = Buffers[I].Data.find(MacroDef);
-    if (Res.second != StringRef::npos) {
-      Res.first = Buffers[I].BufferID;
-      break;
-    }
+  if (ExistingIdx < ExistingN) {
+    if (Diags)
+      Diags->Report(diag::err_pch_targetopt_feature_mismatch)
+        << true << ExistingFeatures[ExistingIdx];
+    return true;
   }
-  return Res;
-}
-
-bool PCHValidator::ReadPredefinesBuffer(const PCHPredefinesBlocks &Buffers,
-                                        StringRef OriginalFileName,
-                                        std::string &SuggestedPredefines,
-                                        FileManager &FileMgr) {
-  // We are in the context of an implicit include, so the predefines buffer will
-  // have a #include entry for the PCH file itself (as normalized by the
-  // preprocessor initialization). Find it and skip over it in the checking
-  // below.
-  SmallString<256> PCHInclude;
-  PCHInclude += "#include \"";
-  PCHInclude += HeaderSearch::NormalizeDashIncludePath(OriginalFileName,
-                                                       FileMgr);
-  PCHInclude += "\"\n";
-  std::pair<StringRef,StringRef> Split =
-    StringRef(PP.getPredefines()).split(PCHInclude.str());
-  StringRef Left =  Split.first, Right = Split.second;
-  if (Left == PP.getPredefines()) {
-    Error("Missing PCH include entry!");
+
+  if (ReadIdx < ReadN) {
+    if (Diags)
+      Diags->Report(diag::err_pch_targetopt_feature_mismatch)
+        << false << ReadFeatures[ReadIdx];
     return true;
   }
 
-  // If the concatenation of all the PCH buffers is equal to the adjusted
-  // command line, we're done.
-  SmallVector<StringRef, 2> CommandLine;
-  CommandLine.push_back(Left);
-  CommandLine.push_back(Right);
-  if (EqualConcatenations(CommandLine, Buffers))
-    return false;
+  return false;
+}
 
-  SourceManager &SourceMgr = PP.getSourceManager();
-
-  // The predefines buffers are different. Determine what the differences are,
-  // and whether they require us to reject the PCH file.
-  SmallVector<StringRef, 8> PCHLines;
-  for (unsigned I = 0, N = Buffers.size(); I != N; ++I)
-    Buffers[I].Data.split(PCHLines, "\n", /*MaxSplit=*/-1, /*KeepEmpty=*/false);
-
-  SmallVector<StringRef, 8> CmdLineLines;
-  Left.split(CmdLineLines, "\n", /*MaxSplit=*/-1, /*KeepEmpty=*/false);
-
-  // Pick out implicit #includes after the PCH and don't consider them for
-  // validation; we will insert them into SuggestedPredefines so that the
-  // preprocessor includes them.
-  std::string IncludesAfterPCH;
-  SmallVector<StringRef, 8> AfterPCHLines;
-  Right.split(AfterPCHLines, "\n", /*MaxSplit=*/-1, /*KeepEmpty=*/false);
-  for (unsigned i = 0, e = AfterPCHLines.size(); i != e; ++i) {
-    if (AfterPCHLines[i].startswith("#include ")) {
-      IncludesAfterPCH += AfterPCHLines[i];
-      IncludesAfterPCH += '\n';
-    } else {
-      CmdLineLines.push_back(AfterPCHLines[i]);
-    }
-  }
-
-  // Make sure we add the includes last into SuggestedPredefines before we
-  // exit this function.
-  struct AddIncludesRAII {
-    std::string &SuggestedPredefines;
-    std::string &IncludesAfterPCH;
-
-    AddIncludesRAII(std::string &SuggestedPredefines,
-                    std::string &IncludesAfterPCH)
-      : SuggestedPredefines(SuggestedPredefines),
-        IncludesAfterPCH(IncludesAfterPCH) { }
-    ~AddIncludesRAII() {
-      SuggestedPredefines += IncludesAfterPCH;
-    }
-  } AddIncludes(SuggestedPredefines, IncludesAfterPCH);
-
-  // Sort both sets of predefined buffer lines, since we allow some extra
-  // definitions and they may appear at any point in the output.
-  std::sort(CmdLineLines.begin(), CmdLineLines.end());
-  std::sort(PCHLines.begin(), PCHLines.end());
-
-  // Determine which predefines that were used to build the PCH file are missing
-  // from the command line.
-  std::vector<StringRef> MissingPredefines;
-  std::set_difference(PCHLines.begin(), PCHLines.end(),
-                      CmdLineLines.begin(), CmdLineLines.end(),
-                      std::back_inserter(MissingPredefines));
-
-  bool MissingDefines = false;
-  bool ConflictingDefines = false;
-  for (unsigned I = 0, N = MissingPredefines.size(); I != N; ++I) {
-    StringRef Missing = MissingPredefines[I];
-    if (Missing.startswith("#include ")) {
-      // An -include was specified when generating the PCH; it is included in
-      // the PCH, just ignore it.
-      continue;
-    }
-    if (!Missing.startswith("#define ")) {
-      Reader.Diag(diag::warn_pch_compiler_options_mismatch);
-      return true;
-    }
+bool
+PCHValidator::ReadLanguageOptions(const LangOptions &LangOpts,
+                                  bool Complain) {
+  const LangOptions &ExistingLangOpts = PP.getLangOpts();
+  return checkLanguageOptions(LangOpts, ExistingLangOpts,
+                              Complain? &Reader.Diags : 0);
+}
 
-    // This is a macro definition. Determine the name of the macro we're
-    // defining.
-    std::string::size_type StartOfMacroName = strlen("#define ");
-    std::string::size_type EndOfMacroName
-      = Missing.find_first_of("( \n\r", StartOfMacroName);
-    assert(EndOfMacroName != std::string::npos &&
-           "Couldn't find the end of the macro name");
-    StringRef MacroName = Missing.slice(StartOfMacroName, EndOfMacroName);
-
-    // Determine whether this macro was given a different definition on the
-    // command line.
-    std::string MacroDefStart = "#define " + MacroName.str();
-    std::string::size_type MacroDefLen = MacroDefStart.size();
-    SmallVector<StringRef, 8>::iterator ConflictPos
-      = std::lower_bound(CmdLineLines.begin(), CmdLineLines.end(),
-                         MacroDefStart);
-    for (; ConflictPos != CmdLineLines.end(); ++ConflictPos) {
-      if (!ConflictPos->startswith(MacroDefStart)) {
-        // Different macro; we're done.
-        ConflictPos = CmdLineLines.end();
-        break;
-      }
+bool PCHValidator::ReadTargetOptions(const TargetOptions &TargetOpts,
+                                     bool Complain) {
+  const TargetOptions &ExistingTargetOpts = PP.getTargetInfo().getTargetOpts();
+  return checkTargetOptions(TargetOpts, ExistingTargetOpts,
+                            Complain? &Reader.Diags : 0);
+}
 
-      assert(ConflictPos->size() > MacroDefLen &&
-             "Invalid #define in predefines buffer?");
-      if ((*ConflictPos)[MacroDefLen] != ' ' &&
-          (*ConflictPos)[MacroDefLen] != '(')
-        continue; // Longer macro name; keep trying.
+namespace {
+  typedef llvm::StringMap<std::pair<StringRef, bool /*IsUndef*/> >
+    MacroDefinitionsMap;
+}
 
-      // We found a conflicting macro definition.
-      break;
-    }
+/// \brief Collect the macro definitions provided by the given preprocessor
+/// options.
+static void collectMacroDefinitions(const PreprocessorOptions &PPOpts,
+                                    MacroDefinitionsMap &Macros,
+                                    SmallVectorImpl<StringRef> *MacroNames = 0){
+  for (unsigned I = 0, N = PPOpts.Macros.size(); I != N; ++I) {
+    StringRef Macro = PPOpts.Macros[I].first;
+    bool IsUndef = PPOpts.Macros[I].second;
 
-    if (ConflictPos != CmdLineLines.end()) {
-      Reader.Diag(diag::warn_cmdline_conflicting_macro_def)
-          << MacroName;
+    std::pair<StringRef, StringRef> MacroPair = Macro.split('=');
+    StringRef MacroName = MacroPair.first;
+    StringRef MacroBody = MacroPair.second;
 
-      // Show the definition of this macro within the PCH file.
-      std::pair<FileID, StringRef::size_type> MacroLoc =
-          FindMacro(Buffers, Missing);
-      assert(MacroLoc.second!=StringRef::npos && "Unable to find macro!");
-      SourceLocation PCHMissingLoc =
-          SourceMgr.getLocForStartOfFile(MacroLoc.first)
-            .getLocWithOffset(MacroLoc.second);
-      Reader.Diag(PCHMissingLoc, diag::note_pch_macro_defined_as) << MacroName;
+    // For an #undef'd macro, we only care about the name.
+    if (IsUndef) {
+      if (MacroNames && !Macros.count(MacroName))
+        MacroNames->push_back(MacroName);
 
-      ConflictingDefines = true;
+      Macros[MacroName] = std::make_pair("", true);
       continue;
     }
 
-    // If the macro doesn't conflict, then we'll just pick up the macro
-    // definition from the PCH file. Warn the user that they made a mistake.
-    if (ConflictingDefines)
-      continue; // Don't complain if there are already conflicting defs
-
-    if (!MissingDefines) {
-      Reader.Diag(diag::warn_cmdline_missing_macro_defs);
-      MissingDefines = true;
+    // For a #define'd macro, figure out the actual definition.
+    if (MacroName.size() == Macro.size())
+      MacroBody = "1";
+    else {
+      // Note: GCC drops anything following an end-of-line character.
+      StringRef::size_type End = MacroBody.find_first_of("\n\r");
+      MacroBody = MacroBody.substr(0, End);
     }
 
-    // Show the definition of this macro within the PCH file.
-    std::pair<FileID, StringRef::size_type> MacroLoc =
-        FindMacro(Buffers, Missing);
-    assert(MacroLoc.second!=StringRef::npos && "Unable to find macro!");
-    SourceLocation PCHMissingLoc =
-        SourceMgr.getLocForStartOfFile(MacroLoc.first)
-          .getLocWithOffset(MacroLoc.second);
-    Reader.Diag(PCHMissingLoc, diag::note_using_macro_def_from_pch);
+    if (MacroNames && !Macros.count(MacroName))
+      MacroNames->push_back(MacroName);
+    Macros[MacroName] = std::make_pair(MacroBody, false);
   }
+}
+         
+/// \brief Check the preprocessor options deserialized from the control block
+/// against the preprocessor options in an existing preprocessor.
+///
+/// \param Diags If non-null, produce diagnostics for any mismatches incurred.
+static bool checkPreprocessorOptions(const PreprocessorOptions &PPOpts,
+                                     const PreprocessorOptions &ExistingPPOpts,
+                                     DiagnosticsEngine *Diags,
+                                     FileManager &FileMgr,
+                                     std::string &SuggestedPredefines) {
+  // Check macro definitions.
+  MacroDefinitionsMap ASTFileMacros;
+  collectMacroDefinitions(PPOpts, ASTFileMacros);
+  MacroDefinitionsMap ExistingMacros;
+  SmallVector<StringRef, 4> ExistingMacroNames;
+  collectMacroDefinitions(ExistingPPOpts, ExistingMacros, &ExistingMacroNames);
+
+  for (unsigned I = 0, N = ExistingMacroNames.size(); I != N; ++I) {
+    // Dig out the macro definition in the existing preprocessor options.
+    StringRef MacroName = ExistingMacroNames[I];
+    std::pair<StringRef, bool> Existing = ExistingMacros[MacroName];
+
+    // Check whether we know anything about this macro name or not.
+    llvm::StringMap<std::pair<StringRef, bool /*IsUndef*/> >::iterator Known
+      = ASTFileMacros.find(MacroName);
+    if (Known == ASTFileMacros.end()) {
+      // FIXME: Check whether this identifier was referenced anywhere in the
+      // AST file. If so, we should reject the AST file. Unfortunately, this
+      // information isn't in the control block. What shall we do about it?
+
+      if (Existing.second) {
+        SuggestedPredefines += "#undef ";
+        SuggestedPredefines += MacroName.str();
+        SuggestedPredefines += '\n';
+      } else {
+        SuggestedPredefines += "#define ";
+        SuggestedPredefines += MacroName.str();
+        SuggestedPredefines += ' ';
+        SuggestedPredefines += Existing.first.str();
+        SuggestedPredefines += '\n';
+      }
+      continue;
+    }
 
-  if (ConflictingDefines)
-    return true;
-
-  // Determine what predefines were introduced based on command-line
-  // parameters that were not present when building the PCH
-  // file. Extra #defines are okay, so long as the identifiers being
-  // defined were not used within the precompiled header.
-  std::vector<StringRef> ExtraPredefines;
-  std::set_difference(CmdLineLines.begin(), CmdLineLines.end(),
-                      PCHLines.begin(), PCHLines.end(),
-                      std::back_inserter(ExtraPredefines));
-  for (unsigned I = 0, N = ExtraPredefines.size(); I != N; ++I) {
-    StringRef &Extra = ExtraPredefines[I];
-    if (!Extra.startswith("#define ")) {
-      Reader.Diag(diag::warn_pch_compiler_options_mismatch);
+    // If the macro was defined in one but undef'd in the other, we have a
+    // conflict.
+    if (Existing.second != Known->second.second) {
+      if (Diags) {
+        Diags->Report(diag::err_pch_macro_def_undef)
+          << MacroName << Known->second.second;
+      }
       return true;
     }
 
-    // This is an extra macro definition. Determine the name of the
-    // macro we're defining.
-    std::string::size_type StartOfMacroName = strlen("#define ");
-    std::string::size_type EndOfMacroName
-      = Extra.find_first_of("( \n\r", StartOfMacroName);
-    assert(EndOfMacroName != std::string::npos &&
-           "Couldn't find the end of the macro name");
-    StringRef MacroName = Extra.slice(StartOfMacroName, EndOfMacroName);
-
-    // Check whether this name was used somewhere in the PCH file. If
-    // so, defining it as a macro could change behavior, so we reject
-    // the PCH file.
-    if (IdentifierInfo *II = Reader.get(MacroName)) {
-      Reader.Diag(diag::warn_macro_name_used_in_pch) << II;
-      return true;
+    // If the macro was #undef'd in both, or if the macro bodies are identical,
+    // it's fine.
+    if (Existing.second || Existing.first == Known->second.first)
+      continue;
+
+    // The macro bodies differ; complain.
+    if (Diags) {
+      Diags->Report(diag::err_pch_macro_def_conflict)
+        << MacroName << Known->second.first << Existing.first;
+    }
+    return true;
+  }
+
+  // Check whether we're using predefines.
+  if (PPOpts.UsePredefines != ExistingPPOpts.UsePredefines) {
+    if (Diags) {
+      Diags->Report(diag::err_pch_undef) << ExistingPPOpts.UsePredefines;
     }
+    return true;
+  }
+
+  // Compute the #include and #include_macros lines we need.
+  for (unsigned I = 0, N = ExistingPPOpts.Includes.size(); I != N; ++I) {
+    StringRef File = ExistingPPOpts.Includes[I];
+    if (File == ExistingPPOpts.ImplicitPCHInclude)
+      continue;
+
+    if (std::find(PPOpts.Includes.begin(), PPOpts.Includes.end(), File)
+          != PPOpts.Includes.end())
+      continue;
 
-    // Add this definition to the suggested predefines buffer.
-    SuggestedPredefines += Extra;
-    SuggestedPredefines += '\n';
+    SuggestedPredefines += "#include \"";
+    SuggestedPredefines +=
+      HeaderSearch::NormalizeDashIncludePath(File, FileMgr);
+    SuggestedPredefines += "\"\n";
+  }
+
+  for (unsigned I = 0, N = ExistingPPOpts.MacroIncludes.size(); I != N; ++I) {
+    StringRef File = ExistingPPOpts.MacroIncludes[I];
+    if (std::find(PPOpts.MacroIncludes.begin(), PPOpts.MacroIncludes.end(),
+                  File)
+        != PPOpts.MacroIncludes.end())
+      continue;
+
+    SuggestedPredefines += "#__include_macros \"";
+    SuggestedPredefines +=
+      HeaderSearch::NormalizeDashIncludePath(File, FileMgr);
+    SuggestedPredefines += "\"\n##\n";
   }
 
-  // If we get here, it's because the predefines buffer had compatible
-  // contents. Accept the PCH file.
   return false;
 }
 
+bool PCHValidator::ReadPreprocessorOptions(const PreprocessorOptions &PPOpts,
+                                           bool Complain,
+                                           std::string &SuggestedPredefines) {
+  const PreprocessorOptions &ExistingPPOpts = PP.getPreprocessorOpts();
+
+  return checkPreprocessorOptions(PPOpts, ExistingPPOpts,
+                                  Complain? &Reader.Diags : 0,
+                                  PP.getFileManager(),
+                                  SuggestedPredefines);
+}
+
 void PCHValidator::ReadHeaderFileInfo(const HeaderFileInfo &HFI,
                                       unsigned ID) {
   PP.getHeaderSearchInfo().setHeaderFileInfoForUID(HFI, ID);
   ++NumHeaderInfos;
 }
 
-void PCHValidator::ReadCounter(unsigned Value) {
+void PCHValidator::ReadCounter(const ModuleFile &M, unsigned Value) {
   PP.setCounterValue(Value);
 }
 
@@ -527,6 +483,7 @@ IdentifierInfo *ASTIdentifierLookupTrait::ReadData(const internal_key_type& k,
     return II;
   }
 
+  unsigned ObjCOrBuiltinID = ReadUnalignedLE16(d);
   unsigned Bits = ReadUnalignedLE16(d);
   bool CPlusPlusOperatorKeyword = Bits & 0x01;
   Bits >>= 1;
@@ -536,13 +493,11 @@ IdentifierInfo *ASTIdentifierLookupTrait::ReadData(const internal_key_type& k,
   Bits >>= 1;
   bool ExtensionToken = Bits & 0x01;
   Bits >>= 1;
-  bool hasMacroDefinition = Bits & 0x01;
+  bool hadMacroDefinition = Bits & 0x01;
   Bits >>= 1;
-  unsigned ObjCOrBuiltinID = Bits & 0x7FF;
-  Bits >>= 11;
 
   assert(Bits == 0 && "Extra bits in the identifier?");
-  DataLen -= 6;
+  DataLen -= 8;
 
   // Build the IdentifierInfo itself and link the identifier ID with
   // the new IdentifierInfo.
@@ -570,31 +525,14 @@ IdentifierInfo *ASTIdentifierLookupTrait::ReadData(const internal_key_type& k,
 
   // If this identifier is a macro, deserialize the macro
   // definition.
-  if (hasMacroDefinition) {
-    // FIXME: Check for conflicts?
-    uint32_t Offset = ReadUnalignedLE32(d);
-    unsigned LocalSubmoduleID = ReadUnalignedLE32(d);
-    
-    // Determine whether this macro definition should be visible now, or
-    // whether it is in a hidden submodule.
-    bool Visible = true;
-    if (SubmoduleID GlobalSubmoduleID
-          = Reader.getGlobalSubmoduleID(F, LocalSubmoduleID)) {
-      if (Module *Owner = Reader.getSubmodule(GlobalSubmoduleID)) {
-        if (Owner->NameVisibility == Module::Hidden) {
-          // The owning module is not visible, and this macro definition should
-          // not be, either.
-          Visible = false;
-          
-          // Note that this macro definition was hidden because its owning 
-          // module is not yet visible.
-          Reader.HiddenNamesMap[Owner].push_back(II);
-        }
-      } 
+  if (hadMacroDefinition) {
+    SmallVector<MacroID, 4> MacroIDs;
+    while (uint32_t LocalID = ReadUnalignedLE32(d)) {
+      MacroIDs.push_back(Reader.getGlobalMacroID(F, LocalID));
+      DataLen -= 4;
     }
-    
-    Reader.setIdentifierIsMacro(II, F, Offset, Visible);
-    DataLen -= 8;
+    DataLen -= 4;
+    Reader.setIdentifierIsMacro(II, MacroIDs);
   }
 
   Reader.SetIdentifierInfo(ID, II);
@@ -780,16 +718,6 @@ void ASTReader::Error(unsigned DiagID,
     Diag(DiagID) << Arg1 << Arg2;
 }
 
-/// \brief Tell the AST listener about the predefines buffers in the chain.
-bool ASTReader::CheckPredefinesBuffers() {
-  if (Listener)
-    return Listener->ReadPredefinesBuffer(PCHPredefinesBuffers,
-                                          ActualOriginalFileName,
-                                          SuggestedPredefines,
-                                          FileMgr);
-  return false;
-}
-
 //===----------------------------------------------------------------------===//
 // Source Manager Deserialization
 //===----------------------------------------------------------------------===//
@@ -808,7 +736,7 @@ bool ASTReader::ParseLineTable(ModuleFile &F,
     unsigned FilenameLen = Record[Idx++];
     std::string Filename(&Record[Idx], &Record[Idx] + FilenameLen);
     Idx += FilenameLen;
-    MaybeAddSystemRootToFilename(Filename);
+    MaybeAddSystemRootToFilename(F, Filename);
     FileIDs[I] = LineTable.getLineTableFilenameID(Filename);
   }
 
@@ -841,106 +769,8 @@ bool ASTReader::ParseLineTable(ModuleFile &F,
   return false;
 }
 
-namespace {
-
-class ASTStatData {
-public:
-  const ino_t ino;
-  const dev_t dev;
-  const mode_t mode;
-  const time_t mtime;
-  const off_t size;
-
-  ASTStatData(ino_t i, dev_t d, mode_t mo, time_t m, off_t s)
-    : ino(i), dev(d), mode(mo), mtime(m), size(s) {}
-};
-
-class ASTStatLookupTrait {
- public:
-  typedef const char *external_key_type;
-  typedef const char *internal_key_type;
-
-  typedef ASTStatData data_type;
-
-  static unsigned ComputeHash(const char *path) {
-    return llvm::HashString(path);
-  }
-
-  static internal_key_type GetInternalKey(const char *path) { return path; }
-
-  static bool EqualKey(internal_key_type a, internal_key_type b) {
-    return strcmp(a, b) == 0;
-  }
-
-  static std::pair<unsigned, unsigned>
-  ReadKeyDataLength(const unsigned char*& d) {
-    unsigned KeyLen = (unsigned) clang::io::ReadUnalignedLE16(d);
-    unsigned DataLen = (unsigned) *d++;
-    return std::make_pair(KeyLen + 1, DataLen);
-  }
-
-  static internal_key_type ReadKey(const unsigned char *d, unsigned) {
-    return (const char *)d;
-  }
-
-  static data_type ReadData(const internal_key_type, const unsigned char *d,
-                            unsigned /*DataLen*/) {
-    using namespace clang::io;
-
-    ino_t ino = (ino_t) ReadUnalignedLE32(d);
-    dev_t dev = (dev_t) ReadUnalignedLE32(d);
-    mode_t mode = (mode_t) ReadUnalignedLE16(d);
-    time_t mtime = (time_t) ReadUnalignedLE64(d);
-    off_t size = (off_t) ReadUnalignedLE64(d);
-    return data_type(ino, dev, mode, mtime, size);
-  }
-};
-
-/// \brief stat() cache for precompiled headers.
-///
-/// This cache is very similar to the stat cache used by pretokenized
-/// headers.
-class ASTStatCache : public FileSystemStatCache {
-  typedef OnDiskChainedHashTable<ASTStatLookupTrait> CacheTy;
-  CacheTy *Cache;
-
-  unsigned &NumStatHits, &NumStatMisses;
-public:
-  ASTStatCache(const unsigned char *Buckets, const unsigned char *Base,
-               unsigned &NumStatHits, unsigned &NumStatMisses)
-    : Cache(0), NumStatHits(NumStatHits), NumStatMisses(NumStatMisses) {
-    Cache = CacheTy::Create(Buckets, Base);
-  }
-
-  ~ASTStatCache() { delete Cache; }
-
-  LookupResult getStat(const char *Path, struct stat &StatBuf,
-                       int *FileDescriptor) {
-    // Do the lookup for the file's data in the AST file.
-    CacheTy::iterator I = Cache->find(Path);
-
-    // If we don't get a hit in the AST file just forward to 'stat'.
-    if (I == Cache->end()) {
-      ++NumStatMisses;
-      return statChained(Path, StatBuf, FileDescriptor);
-    }
-
-    ++NumStatHits;
-    ASTStatData Data = *I;
-
-    StatBuf.st_ino = Data.ino;
-    StatBuf.st_dev = Data.dev;
-    StatBuf.st_mtime = Data.mtime;
-    StatBuf.st_mode = Data.mode;
-    StatBuf.st_size = Data.size;
-    return CacheExists;
-  }
-};
-} // end anonymous namespace
-
-
 /// \brief Read a source manager block
-ASTReader::ASTReadResult ASTReader::ReadSourceManagerBlock(ModuleFile &F) {
+bool ASTReader::ReadSourceManagerBlock(ModuleFile &F) {
   using namespace SrcMgr;
 
   llvm::BitstreamCursor &SLocEntryCursor = F.SLocEntryCursor;
@@ -954,13 +784,13 @@ ASTReader::ASTReadResult ASTReader::ReadSourceManagerBlock(ModuleFile &F) {
   // The stream itself is going to skip over the source manager block.
   if (F.Stream.SkipBlock()) {
     Error("malformed block record in AST file");
-    return Failure;
+    return true;
   }
 
   // Enter the source manager block.
   if (SLocEntryCursor.EnterSubBlock(SOURCE_MANAGER_BLOCK_ID)) {
     Error("malformed source manager block record in AST file");
-    return Failure;
+    return true;
   }
 
   RecordData Record;
@@ -969,9 +799,9 @@ ASTReader::ASTReadResult ASTReader::ReadSourceManagerBlock(ModuleFile &F) {
     if (Code == llvm::bitc::END_BLOCK) {
       if (SLocEntryCursor.ReadBlockEnd()) {
         Error("error at end of Source Manager block in AST file");
-        return Failure;
+        return true;
       }
-      return Success;
+      return false;
     }
 
     if (Code == llvm::bitc::ENTER_SUBBLOCK) {
@@ -979,7 +809,7 @@ ASTReader::ASTReadResult ASTReader::ReadSourceManagerBlock(ModuleFile &F) {
       SLocEntryCursor.ReadSubBlockID();
       if (SLocEntryCursor.SkipBlock()) {
         Error("malformed block record in AST file");
-        return Failure;
+        return true;
       }
       continue;
     }
@@ -1001,7 +831,7 @@ ASTReader::ASTReadResult ASTReader::ReadSourceManagerBlock(ModuleFile &F) {
     case SM_SLOC_BUFFER_ENTRY:
     case SM_SLOC_EXPANSION_ENTRY:
       // Once we hit one of the source location entries, we're done.
-      return Success;
+      return false;
     }
   }
 }
@@ -1039,14 +869,13 @@ resolveFileRelativeToOriginalDir(const std::string &Filename,
   return currPCHPath.str();
 }
 
-/// \brief Read in the source location entry with the given ID.
-ASTReader::ASTReadResult ASTReader::ReadSLocEntryRecord(int ID) {
+bool ASTReader::ReadSLocEntry(int ID) {
   if (ID == 0)
-    return Success;
+    return false;
 
   if (unsigned(-ID) - 2 >= getTotalNumSLocs() || ID > 0) {
     Error("source location entry ID out-of-range for AST file");
-    return Failure;
+    return true;
   }
 
   ModuleFile *F = GlobalSLocEntryMap.find(-ID)->second;
@@ -1060,7 +889,7 @@ ASTReader::ASTReadResult ASTReader::ReadSLocEntryRecord(int ID) {
       Code == llvm::bitc::ENTER_SUBBLOCK ||
       Code == llvm::bitc::DEFINE_ABBREV) {
     Error("incorrectly-formatted source location entry in AST file");
-    return Failure;
+    return true;
   }
 
   RecordData Record;
@@ -1069,58 +898,18 @@ ASTReader::ASTReadResult ASTReader::ReadSLocEntryRecord(int ID) {
   switch (SLocEntryCursor.ReadRecord(Code, Record, &BlobStart, &BlobLen)) {
   default:
     Error("incorrectly-formatted source location entry in AST file");
-    return Failure;
+    return true;
 
   case SM_SLOC_FILE_ENTRY: {
-    if (Record.size() < 7) {
-      Error("source location entry is incorrect");
-      return Failure;
-    }
-
     // We will detect whether a file changed and return 'Failure' for it, but
     // we will also try to fail gracefully by setting up the SLocEntry.
-    ASTReader::ASTReadResult Result = Success;
-
-    bool OverriddenBuffer = Record[6];
-    
-    std::string OrigFilename(BlobStart, BlobStart + BlobLen);
-    std::string Filename = OrigFilename;
-    MaybeAddSystemRootToFilename(Filename);
-    const FileEntry *File = 
-      OverriddenBuffer? FileMgr.getVirtualFile(Filename, (off_t)Record[4],
-                                               (time_t)Record[5])
-                      : FileMgr.getFile(Filename, /*OpenFile=*/false);
-    if (File == 0 && !OriginalDir.empty() && !CurrentDir.empty() &&
-        OriginalDir != CurrentDir) {
-      std::string resolved = resolveFileRelativeToOriginalDir(Filename,
-                                                              OriginalDir,
-                                                              CurrentDir);
-      if (!resolved.empty())
-        File = FileMgr.getFile(resolved);
-    }
-    if (File == 0)
-      File = FileMgr.getVirtualFile(Filename, (off_t)Record[4],
-                                    (time_t)Record[5]);
-    if (File == 0) {
-      std::string ErrorStr = "could not find file '";
-      ErrorStr += Filename;
-      ErrorStr += "' referenced by AST file";
-      Error(ErrorStr.c_str());
-      return Failure;
-    }
+    unsigned InputID = Record[4];
+    InputFile IF = getInputFile(*F, InputID);
+    const FileEntry *File = IF.getPointer();
+    bool OverriddenBuffer = IF.getInt();
 
-    if (!DisableValidation &&
-        ((off_t)Record[4] != File->getSize()
-#if !defined(LLVM_ON_WIN32)
-        // In our regression testing, the Windows file system seems to
-        // have inconsistent modification times that sometimes
-        // erroneously trigger this error-handling path.
-         || (time_t)Record[5] != File->getModificationTime()
-#endif
-        )) {
-      Error(diag::err_fe_pch_file_modified, Filename);
-      Result = Failure;
-    }
+    if (!IF.getPointer())
+      return true;
 
     SourceLocation IncludeLoc = ReadSourceLocation(*F, Record[1]);
     if (IncludeLoc.isInvalid() && F->Kind != MK_MainFile) {
@@ -1133,12 +922,12 @@ ASTReader::ASTReadResult ASTReader::ReadSLocEntryRecord(int ID) {
                                         ID, BaseOffset + Record[0]);
     SrcMgr::FileInfo &FileInfo =
           const_cast<SrcMgr::FileInfo&>(SourceMgr.getSLocEntry(FID).getFile());
-    FileInfo.NumCreatedFIDs = Record[7];
+    FileInfo.NumCreatedFIDs = Record[5];
     if (Record[3])
       FileInfo.setHasLineDirectives();
 
-    const DeclID *FirstDecl = F->FileSortedDecls + Record[8];
-    unsigned NumFileDecls = Record[9];
+    const DeclID *FirstDecl = F->FileSortedDecls + Record[6];
+    unsigned NumFileDecls = Record[7];
     if (NumFileDecls) {
       assert(F->FileSortedDecls && "FILE_SORTED_DECLS not encountered yet ?");
       FileDeclIDs[FID] = FileDeclsInfo(F, llvm::makeArrayRef(FirstDecl,
@@ -1157,23 +946,24 @@ ASTReader::ASTReadResult ASTReader::ReadSLocEntryRecord(int ID) {
       
       if (RecCode != SM_SLOC_BUFFER_BLOB) {
         Error("AST record has invalid code");
-        return Failure;
+        return true;
       }
       
       llvm::MemoryBuffer *Buffer
         = llvm::MemoryBuffer::getMemBuffer(StringRef(BlobStart, BlobLen - 1),
-                                           Filename);
+                                           File->getName());
       SourceMgr.overrideFileContents(File, Buffer);
     }
 
-    if (Result == Failure)
-      return Failure;
     break;
   }
 
   case SM_SLOC_BUFFER_ENTRY: {
     const char *Name = BlobStart;
     unsigned Offset = Record[0];
+    SrcMgr::CharacteristicKind
+      FileCharacter = (SrcMgr::CharacteristicKind)Record[2];
+    SourceLocation IncludeLoc = ReadSourceLocation(*F, Record[1]);
     unsigned Code = SLocEntryCursor.ReadCode();
     Record.clear();
     unsigned RecCode
@@ -1181,23 +971,14 @@ ASTReader::ASTReadResult ASTReader::ReadSLocEntryRecord(int ID) {
 
     if (RecCode != SM_SLOC_BUFFER_BLOB) {
       Error("AST record has invalid code");
-      return Failure;
+      return true;
     }
 
     llvm::MemoryBuffer *Buffer
       = llvm::MemoryBuffer::getMemBuffer(StringRef(BlobStart, BlobLen - 1),
                                          Name);
-    FileID BufferID = SourceMgr.createFileIDForMemBuffer(Buffer, ID,
-                                                         BaseOffset + Offset);
-
-    if (strcmp(Name, "<built-in>") == 0 && F->Kind == MK_PCH) {
-      PCHPredefinesBlock Block = {
-        BufferID,
-        StringRef(BlobStart, BlobLen - 1)
-      };
-      PCHPredefinesBuffers.push_back(Block);
-    }
-
+    SourceMgr.createFileIDForMemBuffer(Buffer, FileCharacter, ID,
+                                       BaseOffset + Offset, IncludeLoc);
     break;
   }
 
@@ -1213,7 +994,7 @@ ASTReader::ASTReadResult ASTReader::ReadSLocEntryRecord(int ID) {
   }
   }
 
-  return Success;
+  return false;
 }
 
 /// \brief Find the location where the module F is imported.
@@ -1258,7 +1039,8 @@ bool ASTReader::ReadBlockAbbrevs(llvm::BitstreamCursor &Cursor,
   }
 }
 
-void ASTReader::ReadMacroRecord(ModuleFile &F, uint64_t Offset) {
+void ASTReader::ReadMacroRecord(ModuleFile &F, uint64_t Offset,
+                                MacroInfo *Hint) {
   llvm::BitstreamCursor &Stream = F.MacroCursor;
 
   // Keep track of where we are in the stream, then jump back there
@@ -1270,6 +1052,24 @@ void ASTReader::ReadMacroRecord(ModuleFile &F, uint64_t Offset) {
   SmallVector<IdentifierInfo*, 16> MacroArgs;
   MacroInfo *Macro = 0;
 
+  // RAII object to add the loaded macro information once we're done
+  // adding tokens.
+  struct AddLoadedMacroInfoRAII {
+    Preprocessor &PP;
+    MacroInfo *Hint;
+    MacroInfo *MI;
+    IdentifierInfo *II;
+
+    AddLoadedMacroInfoRAII(Preprocessor &PP, MacroInfo *Hint)
+      : PP(PP), Hint(Hint), MI(), II() { }
+    ~AddLoadedMacroInfoRAII( ) {
+      if (MI) {
+        // Finally, install the macro.
+        PP.addLoadedMacroInfo(II, MI, Hint);
+      }
+    }
+  } AddLoadedMacroInfo(PP, Hint);
+
   while (true) {
     unsigned Code = Stream.ReadCode();
     switch (Code) {
@@ -1312,18 +1112,31 @@ void ASTReader::ReadMacroRecord(ModuleFile &F, uint64_t Offset) {
         Error("macro must have a name in AST file");
         return;
       }
-      
-      SourceLocation Loc = ReadSourceLocation(F, Record[1]);
-      bool isUsed = Record[2];
 
+      unsigned GlobalID = getGlobalMacroID(F, Record[1]);
+
+      // If this macro has already been loaded, don't do so again.
+      if (MacrosLoaded[GlobalID - NUM_PREDEF_MACRO_IDS])
+        return;
+
+      SubmoduleID GlobalSubmoduleID = getGlobalSubmoduleID(F, Record[2]);
+      unsigned NextIndex = 3;
+      SourceLocation Loc = ReadSourceLocation(F, Record, NextIndex);
       MacroInfo *MI = PP.AllocateMacroInfo(Loc);
-      MI->setIsUsed(isUsed);
+
+      // Record this macro.
+      MacrosLoaded[GlobalID - NUM_PREDEF_MACRO_IDS] = MI;
+
+      SourceLocation UndefLoc = ReadSourceLocation(F, Record, NextIndex);
+      if (UndefLoc.isValid())
+        MI->setUndefLoc(UndefLoc);
+
+      MI->setIsUsed(Record[NextIndex++]);
       MI->setIsFromAST();
 
-      bool IsPublic = Record[3];
-      unsigned NextIndex = 4;
+      bool IsPublic = Record[NextIndex++];
       MI->setVisibility(IsPublic, ReadSourceLocation(F, Record, NextIndex));
-      
+
       if (RecType == PP_MACRO_FUNCTION_LIKE) {
         // Decode function-like macro info.
         bool isC99VarArgs = Record[NextIndex++];
@@ -1341,8 +1154,60 @@ void ASTReader::ReadMacroRecord(ModuleFile &F, uint64_t Offset) {
                             PP.getPreprocessorAllocator());
       }
 
-      // Finally, install the macro.
-      PP.setMacroInfo(II, MI, /*LoadedFromAST=*/true);
+      if (DeserializationListener)
+        DeserializationListener->MacroRead(GlobalID, MI);
+
+      // If an update record marked this as undefined, do so now.
+      // FIXME: Only if the submodule this update came from is visible?
+      MacroUpdatesMap::iterator Update = MacroUpdates.find(GlobalID);
+      if (Update != MacroUpdates.end()) {
+        if (MI->getUndefLoc().isInvalid()) {
+          for (unsigned I = 0, N = Update->second.size(); I != N; ++I) {
+            bool Hidden = false;
+            if (unsigned SubmoduleID = Update->second[I].first) {
+              if (Module *Owner = getSubmodule(SubmoduleID)) {
+                if (Owner->NameVisibility == Module::Hidden) {
+                  // Note that this #undef is hidden.
+                  Hidden = true;
+
+                  // Record this hiding for later.
+                  HiddenNamesMap[Owner].push_back(
+                    HiddenName(II, MI, Update->second[I].second.UndefLoc));
+                }
+              }
+            }
+
+            if (!Hidden) {
+              MI->setUndefLoc(Update->second[I].second.UndefLoc);
+              if (PPMutationListener *Listener = PP.getPPMutationListener())
+                Listener->UndefinedMacro(MI);
+              break;
+            }
+          }
+        }
+        MacroUpdates.erase(Update);
+      }
+
+      // Determine whether this macro definition is visible.
+      bool Hidden = !MI->isPublic();
+      if (!Hidden && GlobalSubmoduleID) {
+        if (Module *Owner = getSubmodule(GlobalSubmoduleID)) {
+          if (Owner->NameVisibility == Module::Hidden) {
+            // The owning module is not visible, and this macro definition
+            // should not be, either.
+            Hidden = true;
+
+            // Note that this macro definition was hidden because its owning
+            // module is not yet visible.
+            HiddenNamesMap[Owner].push_back(HiddenName(II, MI));
+          }
+        }
+      }
+      MI->setHidden(Hidden);
+
+      // Make sure we install the macro once we're done.
+      AddLoadedMacroInfo.MI = MI;
+      AddLoadedMacroInfo.II = II;
 
       // Remember that we saw this macro last so that we add the tokens that
       // form its body to it.
@@ -1451,18 +1316,16 @@ HeaderFileInfoTrait::ReadData(const internal_key_type, const unsigned char *d,
   return HFI;
 }
 
-void ASTReader::setIdentifierIsMacro(IdentifierInfo *II, ModuleFile &F,
-                                     uint64_t LocalOffset, bool Visible) {
-  if (Visible) {
-    // Note that this identifier has a macro definition.
-    II->setHasMacroDefinition(true);
-  }
-  
-  // Adjust the offset to a global offset.
-  UnreadMacroRecordOffsets[II] = F.GlobalBitOffset + LocalOffset;
+void ASTReader::setIdentifierIsMacro(IdentifierInfo *II, ArrayRef<MacroID> IDs){
+  II->setHadMacroDefinition(true);
+  assert(NumCurrentElementsDeserializing > 0 &&"Missing deserialization guard");
+  PendingMacroIDs[II].append(IDs.begin(), IDs.end());
 }
 
 void ASTReader::ReadDefinedMacros() {
+  // Note that we are loading defined macros.
+  Deserializing Macros(this);
+
   for (ModuleReverseIterator I = ModuleMgr.rbegin(),
       E = ModuleMgr.rend(); I != E; ++I) {
     llvm::BitstreamCursor &MacroCursor = (*I)->MacroCursor;
@@ -1514,26 +1377,6 @@ void ASTReader::ReadDefinedMacros() {
       }
     }
   }
-  
-  // Drain the unread macro-record offsets map.
-  while (!UnreadMacroRecordOffsets.empty())
-    LoadMacroDefinition(UnreadMacroRecordOffsets.begin());
-}
-
-void ASTReader::LoadMacroDefinition(
-                    llvm::DenseMap<IdentifierInfo *, uint64_t>::iterator Pos) {
-  assert(Pos != UnreadMacroRecordOffsets.end() && "Unknown macro definition");
-  uint64_t Offset = Pos->second;
-  UnreadMacroRecordOffsets.erase(Pos);
-  
-  RecordLocation Loc = getLocalBitOffset(Offset);
-  ReadMacroRecord(*Loc.F, Loc.Offset);
-}
-
-void ASTReader::LoadMacroDefinition(IdentifierInfo *II) {
-  llvm::DenseMap<IdentifierInfo *, uint64_t>::iterator Pos
-    = UnreadMacroRecordOffsets.find(II);
-  LoadMacroDefinition(Pos);
 }
 
 namespace {
@@ -1582,6 +1425,9 @@ namespace {
 }
 
 void ASTReader::updateOutOfDateIdentifier(IdentifierInfo &II) {
+  // Note that we are loading an identifier.
+  Deserializing AnIdentifier(this);
+
   unsigned PriorGeneration = 0;
   if (getContext().getLangOpts().Modules)
     PriorGeneration = IdentifierGeneration[&II];
@@ -1602,14 +1448,132 @@ void ASTReader::markIdentifierUpToDate(IdentifierInfo *II) {
     IdentifierGeneration[II] = CurrentGeneration;
 }
 
+llvm::PointerIntPair<const FileEntry *, 1, bool> 
+ASTReader::getInputFile(ModuleFile &F, unsigned ID, bool Complain) {
+  // If this ID is bogus, just return an empty input file.
+  if (ID == 0 || ID > F.InputFilesLoaded.size())
+    return InputFile();
+
+  // If we've already loaded this input file, return it.
+  if (F.InputFilesLoaded[ID-1].getPointer())
+    return F.InputFilesLoaded[ID-1];
+
+  // Go find this input file.
+  llvm::BitstreamCursor &Cursor = F.InputFilesCursor;
+  SavedStreamPosition SavedPosition(Cursor);
+  Cursor.JumpToBit(F.InputFileOffsets[ID-1]);
+  
+  unsigned Code = Cursor.ReadCode();
+  RecordData Record;
+  const char *BlobStart = 0;
+  unsigned BlobLen = 0;
+  switch ((InputFileRecordTypes)Cursor.ReadRecord(Code, Record,
+                                                  &BlobStart, &BlobLen)) {
+  case INPUT_FILE: {
+    unsigned StoredID = Record[0];
+    assert(ID == StoredID && "Bogus stored ID or offset");
+    (void)StoredID;
+    off_t StoredSize = (off_t)Record[1];
+    time_t StoredTime = (time_t)Record[2];
+    bool Overridden = (bool)Record[3];
+    
+    // Get the file entry for this input file.
+    StringRef OrigFilename(BlobStart, BlobLen);
+    std::string Filename = OrigFilename;
+    MaybeAddSystemRootToFilename(F, Filename);
+    const FileEntry *File 
+      = Overridden? FileMgr.getVirtualFile(Filename, StoredSize, StoredTime)
+                  : FileMgr.getFile(Filename, /*OpenFile=*/false);
+    
+    // If we didn't find the file, resolve it relative to the
+    // original directory from which this AST file was created.
+    if (File == 0 && !F.OriginalDir.empty() && !CurrentDir.empty() &&
+        F.OriginalDir != CurrentDir) {
+      std::string Resolved = resolveFileRelativeToOriginalDir(Filename,
+                                                              F.OriginalDir,
+                                                              CurrentDir);
+      if (!Resolved.empty())
+        File = FileMgr.getFile(Resolved);
+    }
+    
+    // For an overridden file, create a virtual file with the stored
+    // size/timestamp.
+    if (Overridden && File == 0) {
+      File = FileMgr.getVirtualFile(Filename, StoredSize, StoredTime);
+    }
+    
+    if (File == 0) {
+      if (Complain) {
+        std::string ErrorStr = "could not find file '";
+        ErrorStr += Filename;
+        ErrorStr += "' referenced by AST file";
+        Error(ErrorStr.c_str());
+      }
+      return InputFile();
+    }
+    
+    // Note that we've loaded this input file.
+    F.InputFilesLoaded[ID-1] = InputFile(File, Overridden);
+    
+    // Check if there was a request to override the contents of the file
+    // that was part of the precompiled header. Overridding such a file
+    // can lead to problems when lexing using the source locations from the
+    // PCH.
+    SourceManager &SM = getSourceManager();
+    if (!Overridden && SM.isFileOverridden(File)) {
+      Error(diag::err_fe_pch_file_overridden, Filename);
+      // After emitting the diagnostic, recover by disabling the override so
+      // that the original file will be used.
+      SM.disableFileContentsOverride(File);
+      // The FileEntry is a virtual file entry with the size of the contents
+      // that would override the original contents. Set it to the original's
+      // size/time.
+      FileMgr.modifyFileEntry(const_cast<FileEntry*>(File),
+                              StoredSize, StoredTime);
+    }
+
+    // For an overridden file, there is nothing to validate.
+    if (Overridden)
+      return InputFile(File, Overridden);
+
+    // The stat info from the FileEntry came from the cached stat
+    // info of the PCH, so we cannot trust it.
+    struct stat StatBuf;
+    if (::stat(File->getName(), &StatBuf) != 0) {
+      StatBuf.st_size = File->getSize();
+      StatBuf.st_mtime = File->getModificationTime();
+    }
+
+    if ((StoredSize != StatBuf.st_size
+#if !defined(LLVM_ON_WIN32)
+         // In our regression testing, the Windows file system seems to
+         // have inconsistent modification times that sometimes
+         // erroneously trigger this error-handling path.
+         || StoredTime != StatBuf.st_mtime
+#endif
+         )) {
+      if (Complain)
+        Error(diag::err_fe_pch_file_modified, Filename);
+      
+      return InputFile();
+    }
+
+    return InputFile(File, Overridden);
+  }
+  }
+
+  return InputFile();
+}
+
 const FileEntry *ASTReader::getFileEntry(StringRef filenameStrRef) {
+  ModuleFile &M = ModuleMgr.getPrimaryModule();
   std::string Filename = filenameStrRef;
-  MaybeAddSystemRootToFilename(Filename);
+  MaybeAddSystemRootToFilename(M, Filename);
   const FileEntry *File = FileMgr.getFile(Filename);
-  if (File == 0 && !OriginalDir.empty() && !CurrentDir.empty() &&
-      OriginalDir != CurrentDir) {
+  if (File == 0 && !M.OriginalDir.empty() && !CurrentDir.empty() &&
+      M.OriginalDir != CurrentDir) {
     std::string resolved = resolveFileRelativeToOriginalDir(Filename,
-                                                            OriginalDir,
+                                                            M.OriginalDir,
                                                             CurrentDir);
     if (!resolved.empty())
       File = FileMgr.getFile(resolved);
@@ -1621,9 +1585,10 @@ const FileEntry *ASTReader::getFileEntry(StringRef filenameStrRef) {
 /// \brief If we are loading a relocatable PCH file, and the filename is
 /// not an absolute path, add the system root to the beginning of the file
 /// name.
-void ASTReader::MaybeAddSystemRootToFilename(std::string &Filename) {
+void ASTReader::MaybeAddSystemRootToFilename(ModuleFile &M,
+                                             std::string &Filename) {
   // If this is not a relocatable PCH file, there's nothing to do.
-  if (!RelocatablePCH)
+  if (!M.RelocatablePCH)
     return;
 
   if (Filename.empty() || llvm::sys::path::is_absolute(Filename))
@@ -1643,47 +1608,244 @@ void ASTReader::MaybeAddSystemRootToFilename(std::string &Filename) {
 }
 
 ASTReader::ASTReadResult
-ASTReader::ReadASTBlock(ModuleFile &F) {
+ASTReader::ReadControlBlock(ModuleFile &F,
+                            llvm::SmallVectorImpl<ModuleFile *> &Loaded,
+                            unsigned ClientLoadCapabilities) {
   llvm::BitstreamCursor &Stream = F.Stream;
 
-  if (Stream.EnterSubBlock(AST_BLOCK_ID)) {
+  if (Stream.EnterSubBlock(CONTROL_BLOCK_ID)) {
     Error("malformed block record in AST file");
     return Failure;
   }
 
-  // Read all of the records and blocks for the ASt file.
+  // Read all of the records and blocks in the control block.
   RecordData Record;
   while (!Stream.AtEndOfStream()) {
     unsigned Code = Stream.ReadCode();
     if (Code == llvm::bitc::END_BLOCK) {
       if (Stream.ReadBlockEnd()) {
-        Error("error at end of module block in AST file");
+        Error("error at end of control block in AST file");
         return Failure;
       }
 
+      // Validate all of the input files.
+      if (!DisableValidation) {
+        bool Complain = (ClientLoadCapabilities & ARR_OutOfDate) == 0;
+        for (unsigned I = 0, N = Record[0]; I < N; ++I)
+          if (!getInputFile(F, I+1, Complain).getPointer())
+            return OutOfDate;
+      }
+
       return Success;
     }
 
     if (Code == llvm::bitc::ENTER_SUBBLOCK) {
       switch (Stream.ReadSubBlockID()) {
-      case DECLTYPES_BLOCK_ID:
-        // We lazily load the decls block, but we want to set up the
-        // DeclsCursor cursor to point into it.  Clone our current bitcode
-        // cursor to it, enter the block and read the abbrevs in that block.
-        // With the main cursor, we just skip over it.
-        F.DeclsCursor = Stream;
-        if (Stream.SkipBlock() ||  // Skip with the main cursor.
+      case INPUT_FILES_BLOCK_ID:
+        F.InputFilesCursor = Stream;
+        if (Stream.SkipBlock() || // Skip with the main cursor
+            // Read the abbreviations
+            ReadBlockAbbrevs(F.InputFilesCursor, INPUT_FILES_BLOCK_ID)) {
+          Error("malformed block record in AST file");
+          return Failure;
+        }
+        continue;
+        
+      default:
+        if (!Stream.SkipBlock())
+          continue;
+        break;
+      }
+
+      Error("malformed block record in AST file");
+      return Failure;
+    }
+
+    if (Code == llvm::bitc::DEFINE_ABBREV) {
+      Stream.ReadAbbrevRecord();
+      continue;
+    }
+
+    // Read and process a record.
+    Record.clear();
+    const char *BlobStart = 0;
+    unsigned BlobLen = 0;
+    switch ((ControlRecordTypes)Stream.ReadRecord(Code, Record,
+                                                  &BlobStart, &BlobLen)) {
+    case METADATA: {
+      if (Record[0] != VERSION_MAJOR && !DisableValidation) {
+        if ((ClientLoadCapabilities & ARR_VersionMismatch) == 0)
+          Diag(Record[0] < VERSION_MAJOR? diag::warn_pch_version_too_old
+                                        : diag::warn_pch_version_too_new);
+        return VersionMismatch;
+      }
+
+      bool hasErrors = Record[5];
+      if (hasErrors && !DisableValidation && !AllowASTWithCompilerErrors) {
+        Diag(diag::err_pch_with_compiler_errors);
+        return HadErrors;
+      }
+
+      F.RelocatablePCH = Record[4];
+
+      const std::string &CurBranch = getClangFullRepositoryVersion();
+      StringRef ASTBranch(BlobStart, BlobLen);
+      if (StringRef(CurBranch) != ASTBranch && !DisableValidation) {
+        if ((ClientLoadCapabilities & ARR_VersionMismatch) == 0)
+          Diag(diag::warn_pch_different_branch) << ASTBranch << CurBranch;
+        return VersionMismatch;
+      }
+      break;
+    }
+
+    case IMPORTS: {
+      // Load each of the imported PCH files. 
+      unsigned Idx = 0, N = Record.size();
+      while (Idx < N) {
+        // Read information about the AST file.
+        ModuleKind ImportedKind = (ModuleKind)Record[Idx++];
+        unsigned Length = Record[Idx++];
+        SmallString<128> ImportedFile(Record.begin() + Idx,
+                                      Record.begin() + Idx + Length);
+        Idx += Length;
+
+        // Load the AST file.
+        switch(ReadASTCore(ImportedFile, ImportedKind, &F, Loaded,
+                           ClientLoadCapabilities)) {
+        case Failure: return Failure;
+          // If we have to ignore the dependency, we'll have to ignore this too.
+        case OutOfDate: return OutOfDate;
+        case VersionMismatch: return VersionMismatch;
+        case ConfigurationMismatch: return ConfigurationMismatch;
+        case HadErrors: return HadErrors;
+        case Success: break;
+        }
+      }
+      break;
+    }
+
+    case LANGUAGE_OPTIONS: {
+      bool Complain = (ClientLoadCapabilities & ARR_ConfigurationMismatch) == 0;
+      if (Listener && &F == *ModuleMgr.begin() &&
+          ParseLanguageOptions(Record, Complain, *Listener) &&
+          !DisableValidation)
+        return ConfigurationMismatch;
+      break;
+    }
+
+    case TARGET_OPTIONS: {
+      bool Complain = (ClientLoadCapabilities & ARR_ConfigurationMismatch)==0;
+      if (Listener && &F == *ModuleMgr.begin() &&
+          ParseTargetOptions(Record, Complain, *Listener) &&
+          !DisableValidation)
+        return ConfigurationMismatch;
+      break;
+    }
+
+    case DIAGNOSTIC_OPTIONS: {
+      bool Complain = (ClientLoadCapabilities & ARR_ConfigurationMismatch)==0;
+      if (Listener && &F == *ModuleMgr.begin() &&
+          ParseDiagnosticOptions(Record, Complain, *Listener) &&
+          !DisableValidation)
+        return ConfigurationMismatch;
+      break;
+    }
+
+    case FILE_SYSTEM_OPTIONS: {
+      bool Complain = (ClientLoadCapabilities & ARR_ConfigurationMismatch)==0;
+      if (Listener && &F == *ModuleMgr.begin() &&
+          ParseFileSystemOptions(Record, Complain, *Listener) &&
+          !DisableValidation)
+        return ConfigurationMismatch;
+      break;
+    }
+
+    case HEADER_SEARCH_OPTIONS: {
+      bool Complain = (ClientLoadCapabilities & ARR_ConfigurationMismatch)==0;
+      if (Listener && &F == *ModuleMgr.begin() &&
+          ParseHeaderSearchOptions(Record, Complain, *Listener) &&
+          !DisableValidation)
+        return ConfigurationMismatch;
+      break;
+    }
+
+    case PREPROCESSOR_OPTIONS: {
+      bool Complain = (ClientLoadCapabilities & ARR_ConfigurationMismatch)==0;
+      if (Listener && &F == *ModuleMgr.begin() &&
+          ParsePreprocessorOptions(Record, Complain, *Listener,
+                                   SuggestedPredefines) &&
+          !DisableValidation)
+        return ConfigurationMismatch;
+      break;
+    }
+
+    case ORIGINAL_FILE:
+      F.OriginalSourceFileID = FileID::get(Record[0]);
+      F.ActualOriginalSourceFileName.assign(BlobStart, BlobLen);
+      F.OriginalSourceFileName = F.ActualOriginalSourceFileName;
+      MaybeAddSystemRootToFilename(F, F.OriginalSourceFileName);
+      break;
+
+    case ORIGINAL_PCH_DIR:
+      F.OriginalDir.assign(BlobStart, BlobLen);
+      break;
+
+    case INPUT_FILE_OFFSETS:
+      F.InputFileOffsets = (const uint32_t *)BlobStart;
+      F.InputFilesLoaded.resize(Record[0]);
+      break;
+    }
+  }
+
+  Error("premature end of bitstream in AST file");
+  return Failure;
+}
+
+bool ASTReader::ReadASTBlock(ModuleFile &F) {
+  llvm::BitstreamCursor &Stream = F.Stream;
+
+  if (Stream.EnterSubBlock(AST_BLOCK_ID)) {
+    Error("malformed block record in AST file");
+    return true;
+  }
+
+  // Read all of the records and blocks for the AST file.
+  RecordData Record;
+  while (!Stream.AtEndOfStream()) {
+    unsigned Code = Stream.ReadCode();
+    if (Code == llvm::bitc::END_BLOCK) {
+      if (Stream.ReadBlockEnd()) {
+        Error("error at end of module block in AST file");
+        return true;
+      }
+
+      DeclContext *DC = Context.getTranslationUnitDecl();
+      if (!DC->hasExternalVisibleStorage() && DC->hasExternalLexicalStorage())
+        DC->setMustBuildLookupTable();
+
+      return false;
+    }
+
+    if (Code == llvm::bitc::ENTER_SUBBLOCK) {
+      switch (Stream.ReadSubBlockID()) {
+      case DECLTYPES_BLOCK_ID:
+        // We lazily load the decls block, but we want to set up the
+        // DeclsCursor cursor to point into it.  Clone our current bitcode
+        // cursor to it, enter the block and read the abbrevs in that block.
+        // With the main cursor, we just skip over it.
+        F.DeclsCursor = Stream;
+        if (Stream.SkipBlock() ||  // Skip with the main cursor.
             // Read the abbrevs.
             ReadBlockAbbrevs(F.DeclsCursor, DECLTYPES_BLOCK_ID)) {
           Error("malformed block record in AST file");
-          return Failure;
+          return true;
         }
         break;
 
       case DECL_UPDATES_BLOCK_ID:
         if (Stream.SkipBlock()) {
           Error("malformed block record in AST file");
-          return Failure;
+          return true;
         }
         break;
 
@@ -1695,7 +1857,7 @@ ASTReader::ReadASTBlock(ModuleFile &F) {
         if (Stream.SkipBlock() ||
             ReadBlockAbbrevs(F.MacroCursor, PREPROCESSOR_BLOCK_ID)) {
           Error("malformed block record in AST file");
-          return Failure;
+          return true;
         }
         F.MacroStartOffset = F.MacroCursor.GetCurrentBitNo();
         break;
@@ -1706,7 +1868,7 @@ ASTReader::ReadASTBlock(ModuleFile &F) {
             ReadBlockAbbrevs(F.PreprocessorDetailCursor, 
                              PREPROCESSOR_DETAIL_BLOCK_ID)) {
           Error("malformed preprocessor detail record in AST file");
-          return Failure;
+          return true;
         }
         F.PreprocessorDetailStartOffset
           = F.PreprocessorDetailCursor.GetCurrentBitNo();
@@ -1718,31 +1880,13 @@ ASTReader::ReadASTBlock(ModuleFile &F) {
         break;
         
       case SOURCE_MANAGER_BLOCK_ID:
-        switch (ReadSourceManagerBlock(F)) {
-        case Success:
-          break;
-
-        case Failure:
-          Error("malformed source manager block in AST file");
-          return Failure;
-
-        case IgnorePCH:
-          return IgnorePCH;
-        }
+        if (ReadSourceManagerBlock(F))
+          return true;
         break;
 
       case SUBMODULE_BLOCK_ID:
-        switch (ReadSubmoduleBlock(F)) {
-        case Success:
-          break;
-          
-        case Failure:
-          Error("malformed submodule block in AST file");
-          return Failure;
-          
-        case IgnorePCH:
-          return IgnorePCH;            
-        }
+        if (ReadSubmoduleBlock(F))
+          return true;
         break;
 
       case COMMENTS_BLOCK_ID: {
@@ -1750,7 +1894,7 @@ ASTReader::ReadASTBlock(ModuleFile &F) {
         if (Stream.SkipBlock() ||
             ReadBlockAbbrevs(C, COMMENTS_BLOCK_ID)) {
           Error("malformed comments block in AST file");
-          return Failure;
+          return true;
         }
         CommentsCursors.push_back(std::make_pair(C, &F));
         break;
@@ -1760,7 +1904,7 @@ ASTReader::ReadASTBlock(ModuleFile &F) {
         if (!Stream.SkipBlock())
           break;
         Error("malformed block record in AST file");
-        return Failure;
+        return true;
       }
       continue;
     }
@@ -1779,54 +1923,10 @@ ASTReader::ReadASTBlock(ModuleFile &F) {
     default:  // Default behavior: ignore.
       break;
 
-    case METADATA: {
-      if (Record[0] != VERSION_MAJOR && !DisableValidation) {
-        Diag(Record[0] < VERSION_MAJOR? diag::warn_pch_version_too_old
-                                           : diag::warn_pch_version_too_new);
-        return IgnorePCH;
-      }
-
-      bool hasErrors = Record[5];
-      if (hasErrors && !DisableValidation && !AllowASTWithCompilerErrors) {
-        Diag(diag::err_pch_with_compiler_errors);
-        return IgnorePCH;
-      }
-
-      RelocatablePCH = Record[4];
-      if (Listener) {
-        std::string TargetTriple(BlobStart, BlobLen);
-        if (Listener->ReadTargetTriple(TargetTriple))
-          return IgnorePCH;
-      }
-      break;
-    }
-
-    case IMPORTS: {
-      // Load each of the imported PCH files. 
-      unsigned Idx = 0, N = Record.size();
-      while (Idx < N) {
-        // Read information about the AST file.
-        ModuleKind ImportedKind = (ModuleKind)Record[Idx++];
-        unsigned Length = Record[Idx++];
-        SmallString<128> ImportedFile(Record.begin() + Idx,
-                                            Record.begin() + Idx + Length);
-        Idx += Length;
-
-        // Load the AST file.
-        switch(ReadASTCore(ImportedFile, ImportedKind, &F)) {
-        case Failure: return Failure;
-          // If we have to ignore the dependency, we'll have to ignore this too.
-        case IgnorePCH: return IgnorePCH;
-        case Success: break;
-        }
-      }
-      break;
-    }
-
     case TYPE_OFFSET: {
       if (F.LocalNumTypes != 0) {
         Error("duplicate TYPE_OFFSET record in AST file");
-        return Failure;
+        return true;
       }
       F.TypeOffsets = (const uint32_t *)BlobStart;
       F.LocalNumTypes = Record[0];
@@ -1850,7 +1950,7 @@ ASTReader::ReadASTBlock(ModuleFile &F) {
     case DECL_OFFSET: {
       if (F.LocalNumDecls != 0) {
         Error("duplicate DECL_OFFSET record in AST file");
-        return Failure;
+        return true;
       }
       F.DeclOffsets = (const DeclOffset *)BlobStart;
       F.LocalNumDecls = Record[0];
@@ -1904,11 +2004,6 @@ ASTReader::ReadASTBlock(ModuleFile &F) {
       break;
     }
 
-    case LANGUAGE_OPTIONS:
-      if (ParseLanguageOptions(Record) && !DisableValidation)
-        return IgnorePCH;
-      break;
-
     case IDENTIFIER_TABLE:
       F.IdentifierTableData = BlobStart;
       if (Record[0]) {
@@ -1925,7 +2020,7 @@ ASTReader::ReadASTBlock(ModuleFile &F) {
     case IDENTIFIER_OFFSET: {
       if (F.LocalNumIdentifiers != 0) {
         Error("duplicate IDENTIFIER_OFFSET record in AST file");
-        return Failure;
+        return true;
       }
       F.IdentifierOffsets = (const uint32_t *)BlobStart;
       F.LocalNumIdentifiers = Record[0];
@@ -1949,7 +2044,7 @@ ASTReader::ReadASTBlock(ModuleFile &F) {
       }
       break;
     }
-        
+
     case EXTERNAL_DEFINITIONS:
       for (unsigned I = 0, N = Record.size(); I != N; ++I)
         ExternalDefinitions.push_back(getGlobalDeclID(F, Record[I]));
@@ -1980,7 +2075,7 @@ ASTReader::ReadASTBlock(ModuleFile &F) {
     case WEAK_UNDECLARED_IDENTIFIERS:
       if (Record.size() % 4 != 0) {
         Error("invalid weak identifiers record");
-        return Failure;
+        return true;
       }
         
       // FIXME: Ignore weak undeclared identifiers from non-original PCH 
@@ -2050,11 +2145,12 @@ ASTReader::ReadASTBlock(ModuleFile &F) {
 
     case PP_COUNTER_VALUE:
       if (!Record.empty() && Listener)
-        Listener->ReadCounter(Record[0]);
+        Listener->ReadCounter(F, Record[0]);
       break;
       
     case FILE_SORTED_DECLS:
       F.FileSortedDecls = (const DeclID *)BlobStart;
+      F.NumFileSortedDecls = Record[0];
       break;
 
     case SOURCE_LOCATION_OFFSETS: {
@@ -2098,7 +2194,9 @@ ASTReader::ReadASTBlock(ModuleFile &F) {
       ContinuousRangeMap<uint32_t, int, 2>::Builder SLocRemap(F.SLocRemap);
       ContinuousRangeMap<uint32_t, int, 2>::Builder 
         IdentifierRemap(F.IdentifierRemap);
-      ContinuousRangeMap<uint32_t, int, 2>::Builder 
+      ContinuousRangeMap<uint32_t, int, 2>::Builder
+        MacroRemap(F.MacroRemap);
+      ContinuousRangeMap<uint32_t, int, 2>::Builder
         PreprocessedEntityRemap(F.PreprocessedEntityRemap);
       ContinuousRangeMap<uint32_t, int, 2>::Builder 
         SubmoduleRemap(F.SubmoduleRemap);
@@ -2114,11 +2212,12 @@ ASTReader::ReadASTBlock(ModuleFile &F) {
         ModuleFile *OM = ModuleMgr.lookup(Name);
         if (!OM) {
           Error("SourceLocation remap refers to unknown module");
-          return Failure;
+          return true;
         }
 
         uint32_t SLocOffset = io::ReadUnalignedLE32(Data);
         uint32_t IdentifierIDOffset = io::ReadUnalignedLE32(Data);
+        uint32_t MacroIDOffset = io::ReadUnalignedLE32(Data);
         uint32_t PreprocessedEntityIDOffset = io::ReadUnalignedLE32(Data);
         uint32_t SubmoduleIDOffset = io::ReadUnalignedLE32(Data);
         uint32_t SelectorIDOffset = io::ReadUnalignedLE32(Data);
@@ -2131,6 +2230,8 @@ ASTReader::ReadASTBlock(ModuleFile &F) {
         IdentifierRemap.insert(
           std::make_pair(IdentifierIDOffset, 
                          OM->BaseIdentifierID - IdentifierIDOffset));
+        MacroRemap.insert(std::make_pair(MacroIDOffset,
+                                         OM->BaseMacroID - MacroIDOffset));
         PreprocessedEntityRemap.insert(
           std::make_pair(PreprocessedEntityIDOffset, 
             OM->BasePreprocessedEntityID - PreprocessedEntityIDOffset));
@@ -2152,12 +2253,7 @@ ASTReader::ReadASTBlock(ModuleFile &F) {
 
     case SOURCE_MANAGER_LINE_TABLE:
       if (ParseLineTable(F, Record))
-        return Failure;
-      break;
-
-    case FILE_SOURCE_LOCATION_OFFSETS:
-      F.SLocFileOffsets = (const uint32_t *)BlobStart;
-      F.LocalNumSLocFileEntries = Record[0];
+        return true;
       break;
 
     case SOURCE_LOCATION_PRELOADS: {
@@ -2165,25 +2261,13 @@ ASTReader::ReadASTBlock(ModuleFile &F) {
       // which is based off F.SLocEntryBaseID.
       if (!F.PreloadSLocEntries.empty()) {
         Error("Multiple SOURCE_LOCATION_PRELOADS records in AST file");
-        return Failure;
+        return true;
       }
       
       F.PreloadSLocEntries.swap(Record);
       break;
     }
 
-    case STAT_CACHE: {
-      if (!DisableStatCache) {
-        ASTStatCache *MyStatCache =
-          new ASTStatCache((const unsigned char *)BlobStart + Record[0],
-                           (const unsigned char *)BlobStart,
-                           NumStatHits, NumStatMisses);
-        FileMgr.addStatCache(MyStatCache);
-        F.StatCache = MyStatCache;
-      }
-      break;
-    }
-
     case EXT_VECTOR_DECLS:
       for (unsigned I = 0, N = Record.size(); I != N; ++I)
         ExtVectorDecls.push_back(getGlobalDeclID(F, Record[I]));
@@ -2192,7 +2276,7 @@ ASTReader::ReadASTBlock(ModuleFile &F) {
     case VTABLE_USES:
       if (Record.size() % 3 != 0) {
         Error("Invalid VTABLE_USES record");
-        return Failure;
+        return true;
       }
         
       // Later tables overwrite earlier ones.
@@ -2215,13 +2299,15 @@ ASTReader::ReadASTBlock(ModuleFile &F) {
 
     case PENDING_IMPLICIT_INSTANTIATIONS:
       if (PendingInstantiations.size() % 2 != 0) {
+        Error("Invalid existing PendingInstantiations");
+        return true;
+      }
+
+      if (Record.size() % 2 != 0) {
         Error("Invalid PENDING_IMPLICIT_INSTANTIATIONS block");
-        return Failure;
+        return true;
       }
-        
-      // Later lists of pending instantiations overwrite earlier ones.
-      // FIXME: This is most certainly wrong for modules.
-      PendingInstantiations.clear();
+
       for (unsigned I = 0, N = Record.size(); I != N; /* in loop */) {
         PendingInstantiations.push_back(getGlobalDeclID(F, Record[I++]));
         PendingInstantiations.push_back(
@@ -2237,34 +2323,6 @@ ASTReader::ReadASTBlock(ModuleFile &F) {
         SemaDeclRefs.push_back(getGlobalDeclID(F, Record[I]));
       break;
 
-    case ORIGINAL_FILE_NAME:
-      // The primary AST will be the last to get here, so it will be the one
-      // that's used.
-      ActualOriginalFileName.assign(BlobStart, BlobLen);
-      OriginalFileName = ActualOriginalFileName;
-      MaybeAddSystemRootToFilename(OriginalFileName);
-      break;
-
-    case ORIGINAL_FILE_ID:
-      OriginalFileID = FileID::get(Record[0]);
-      break;
-        
-    case ORIGINAL_PCH_DIR:
-      // The primary AST will be the last to get here, so it will be the one
-      // that's used.
-      OriginalDir.assign(BlobStart, BlobLen);
-      break;
-
-    case VERSION_CONTROL_BRANCH_REVISION: {
-      const std::string &CurBranch = getClangFullRepositoryVersion();
-      StringRef ASTBranch(BlobStart, BlobLen);
-      if (StringRef(CurBranch) != ASTBranch && !DisableValidation) {
-        Diag(diag::warn_pch_different_branch) << ASTBranch << CurBranch;
-        return IgnorePCH;
-      }
-      break;
-    }
-
     case PPD_ENTITIES_OFFSETS: {
       F.PreprocessedEntityOffsets = (const PPEntityOffset *)BlobStart;
       assert(BlobLen % sizeof(PPEntityOffset) == 0);
@@ -2300,7 +2358,7 @@ ASTReader::ReadASTBlock(ModuleFile &F) {
     case DECL_UPDATE_OFFSETS: {
       if (Record.size() % 2 != 0) {
         Error("invalid DECL_UPDATE_OFFSETS block in AST file");
-        return Failure;
+        return true;
       }
       for (unsigned I = 0, N = Record.size(); I != N; I += 2)
         DeclUpdateOffsets[getGlobalDeclID(F, Record[I])]
@@ -2311,7 +2369,7 @@ ASTReader::ReadASTBlock(ModuleFile &F) {
     case DECL_REPLACEMENTS: {
       if (Record.size() % 3 != 0) {
         Error("invalid DECL_REPLACEMENTS block in AST file");
-        return Failure;
+        return true;
       }
       for (unsigned I = 0, N = Record.size(); I != N; I += 3)
         ReplacedDecls[getGlobalDeclID(F, Record[I])]
@@ -2322,7 +2380,7 @@ ASTReader::ReadASTBlock(ModuleFile &F) {
     case OBJC_CATEGORIES_MAP: {
       if (F.LocalNumObjCCategoriesInMap != 0) {
         Error("duplicate OBJC_CATEGORIES_MAP record in AST file");
-        return Failure;
+        return true;
       }
       
       F.LocalNumObjCCategoriesInMap = Record[0];
@@ -2337,7 +2395,7 @@ ASTReader::ReadASTBlock(ModuleFile &F) {
     case CXX_BASE_SPECIFIER_OFFSETS: {
       if (F.LocalNumCXXBaseSpecifiers != 0) {
         Error("duplicate CXX_BASE_SPECIFIER_OFFSETS record in AST file");
-        return Failure;
+        return true;
       }
       
       F.LocalNumCXXBaseSpecifiers = Record[0];
@@ -2347,11 +2405,6 @@ ASTReader::ReadASTBlock(ModuleFile &F) {
     }
 
     case DIAG_PRAGMA_MAPPINGS:
-      if (Record.size() % 2 != 0) {
-        Error("invalid DIAG_USER_MAPPINGS block in AST file");
-        return Failure;
-      }
-        
       if (F.PragmaDiagMappings.empty())
         F.PragmaDiagMappings.swap(Record);
       else
@@ -2428,7 +2481,7 @@ ASTReader::ReadASTBlock(ModuleFile &F) {
     case LOCAL_REDECLARATIONS_MAP: {
       if (F.LocalNumRedeclarationsInMap != 0) {
         Error("duplicate LOCAL_REDECLARATIONS_MAP record in AST file");
-        return Failure;
+        return true;
       }
       
       F.LocalNumRedeclarationsInMap = Record[0];
@@ -2445,113 +2498,77 @@ ASTReader::ReadASTBlock(ModuleFile &F) {
       }
       break;
     }
-    }
-  }
-  Error("premature end of bitstream in AST file");
-  return Failure;
-}
-
-ASTReader::ASTReadResult ASTReader::validateFileEntries(ModuleFile &M) {
-  llvm::BitstreamCursor &SLocEntryCursor = M.SLocEntryCursor;
 
-  for (unsigned i = 0, e = M.LocalNumSLocFileEntries; i != e; ++i) {
-    SLocEntryCursor.JumpToBit(M.SLocFileOffsets[i]);
-    unsigned Code = SLocEntryCursor.ReadCode();
-    if (Code == llvm::bitc::END_BLOCK ||
-        Code == llvm::bitc::ENTER_SUBBLOCK ||
-        Code == llvm::bitc::DEFINE_ABBREV) {
-      Error("incorrectly-formatted source location entry in AST file");
-      return Failure;
-    }
+    case MACRO_OFFSET: {
+      if (F.LocalNumMacros != 0) {
+        Error("duplicate MACRO_OFFSET record in AST file");
+        return true;
+      }
+      F.MacroOffsets = (const uint32_t *)BlobStart;
+      F.LocalNumMacros = Record[0];
+      unsigned LocalBaseMacroID = Record[1];
+      F.BaseMacroID = getTotalNumMacros();
 
-    RecordData Record;
-    const char *BlobStart;
-    unsigned BlobLen;
-    switch (SLocEntryCursor.ReadRecord(Code, Record, &BlobStart, &BlobLen)) {
-    default:
-      Error("incorrectly-formatted source location entry in AST file");
-      return Failure;
+      if (F.LocalNumMacros > 0) {
+        // Introduce the global -> local mapping for macros within this module.
+        GlobalMacroMap.insert(std::make_pair(getTotalNumMacros() + 1, &F));
 
-    case SM_SLOC_FILE_ENTRY: {
-      // If the buffer was overridden, the file need not exist.
-      if (Record[6])
-        break;
-      
-      StringRef Filename(BlobStart, BlobLen);
-      const FileEntry *File = getFileEntry(Filename);
+        // Introduce the local -> global mapping for macros within this module.
+        F.MacroRemap.insertOrReplace(
+          std::make_pair(LocalBaseMacroID,
+                         F.BaseMacroID - LocalBaseMacroID));
 
-      if (File == 0) {
-        std::string ErrorStr = "could not find file '";
-        ErrorStr += Filename;
-        ErrorStr += "' referenced by AST file";
-        Error(ErrorStr.c_str());
-        return IgnorePCH;
-      }
-
-      if (Record.size() < 7) {
-        Error("source location entry is incorrect");
-        return Failure;
-      }
-      
-      off_t StoredSize = (off_t)Record[4];
-      time_t StoredTime = (time_t)Record[5];
-
-      // Check if there was a request to override the contents of the file
-      // that was part of the precompiled header. Overridding such a file
-      // can lead to problems when lexing using the source locations from the
-      // PCH.
-      SourceManager &SM = getSourceManager();
-      if (SM.isFileOverridden(File)) {
-        Error(diag::err_fe_pch_file_overridden, Filename);
-        // After emitting the diagnostic, recover by disabling the override so
-        // that the original file will be used.
-        SM.disableFileContentsOverride(File);
-        // The FileEntry is a virtual file entry with the size of the contents
-        // that would override the original contents. Set it to the original's
-        // size/time.
-        FileMgr.modifyFileEntry(const_cast<FileEntry*>(File),
-                                StoredSize, StoredTime);
+        MacrosLoaded.resize(MacrosLoaded.size() + F.LocalNumMacros);
       }
+      break;
+    }
 
-      // The stat info from the FileEntry came from the cached stat
-      // info of the PCH, so we cannot trust it.
-      struct stat StatBuf;
-      if (::stat(File->getName(), &StatBuf) != 0) {
-        StatBuf.st_size = File->getSize();
-        StatBuf.st_mtime = File->getModificationTime();
-      }
+    case MACRO_UPDATES: {
+      for (unsigned I = 0, N = Record.size(); I != N; /* in loop */) {
+        MacroID ID = getGlobalMacroID(F, Record[I++]);
+        if (I == N)
+          break;
 
-      if ((StoredSize != StatBuf.st_size
-#if !defined(LLVM_ON_WIN32)
-          // In our regression testing, the Windows file system seems to
-          // have inconsistent modification times that sometimes
-          // erroneously trigger this error-handling path.
-           || StoredTime != StatBuf.st_mtime
-#endif
-          )) {
-        Error(diag::err_fe_pch_file_modified, Filename);
-        return IgnorePCH;
+        SourceLocation UndefLoc = ReadSourceLocation(F, Record, I);
+        SubmoduleID SubmoduleID = getGlobalSubmoduleID(F, Record[I++]);;
+        MacroUpdate Update;
+        Update.UndefLoc = UndefLoc;
+        MacroUpdates[ID].push_back(std::make_pair(SubmoduleID, Update));
       }
-
       break;
     }
     }
   }
-
-  return Success;
+  Error("premature end of bitstream in AST file");
+  return true;
 }
 
 void ASTReader::makeNamesVisible(const HiddenNames &Names) {
   for (unsigned I = 0, N = Names.size(); I != N; ++I) {
-    if (Decl *D = Names[I].dyn_cast<Decl *>())
-      D->Hidden = false;
-    else {
-      IdentifierInfo *II = Names[I].get<IdentifierInfo *>();
-      if (!II->hasMacroDefinition()) {
-        II->setHasMacroDefinition(true);
-        if (DeserializationListener)
-          DeserializationListener->MacroVisible(II);
+    switch (Names[I].getKind()) {
+    case HiddenName::Declaration:
+      Names[I].getDecl()->Hidden = false;
+      break;
+
+    case HiddenName::MacroVisibility: {
+      std::pair<IdentifierInfo *, MacroInfo *> Macro = Names[I].getMacro();
+      Macro.second->setHidden(!Macro.second->isPublic());
+      if (Macro.second->isDefined()) {
+        PP.makeLoadedMacroInfoVisible(Macro.first, Macro.second);
       }
+      break;
+    }
+
+    case HiddenName::MacroUndef: {
+      std::pair<IdentifierInfo *, MacroInfo *> Macro = Names[I].getMacro();
+      if (Macro.second->isDefined()) {
+        Macro.second->setUndefLoc(Names[I].getMacroUndefLoc());
+        if (PPMutationListener *Listener = PP.getPPMutationListener())
+          Listener->UndefinedMacro(Macro.second);
+        PP.makeLoadedMacroInfoVisible(Macro.first, Macro.second);
+      }
+      break;
+    }
     }
   }
 }
@@ -2631,7 +2648,7 @@ void ASTReader::makeModuleVisible(Module *Mod,
     
     for (unsigned I = 0, N = Mod->Imports.size(); I != N; ++I) {
       Module *Imported = Mod->Imports[I];
-      if (Visited.count(Imported))
+      if (!Visited.insert(Imported))
         continue;
       
       bool Acceptable = UnrestrictedWildcard;
@@ -2649,32 +2666,62 @@ void ASTReader::makeModuleVisible(Module *Mod,
       if (!Acceptable)
         continue;
       
-      Visited.insert(Imported);
       Stack.push_back(Imported);
     }
   }
 }
 
 ASTReader::ASTReadResult ASTReader::ReadAST(const std::string &FileName,
-                                            ModuleKind Type) {
+                                            ModuleKind Type,
+                                            unsigned ClientLoadCapabilities) {
   // Bump the generation number.
   unsigned PreviousGeneration = CurrentGeneration++;
-  
-  switch(ReadASTCore(FileName, Type, /*ImportedBy=*/0)) {
-  case Failure: return Failure;
-  case IgnorePCH: return IgnorePCH;
-  case Success: break;
+
+  unsigned NumModules = ModuleMgr.size();
+  llvm::SmallVector<ModuleFile *, 4> Loaded;
+  switch(ASTReadResult ReadResult = ReadASTCore(FileName, Type,
+                                                /*ImportedBy=*/0, Loaded,
+                                                ClientLoadCapabilities)) {
+  case Failure:
+  case OutOfDate:
+  case VersionMismatch:
+  case ConfigurationMismatch:
+  case HadErrors:
+    ModuleMgr.removeModules(ModuleMgr.begin() + NumModules, ModuleMgr.end());
+    return ReadResult;
+
+  case Success:
+    break;
   }
 
   // Here comes stuff that we only do once the entire chain is loaded.
 
-  // Check the predefines buffers.
-  if (!DisableValidation && Type == MK_PCH &&
-      // FIXME: CheckPredefinesBuffers also sets the SuggestedPredefines;
-      // if DisableValidation is true, defines that were set on command-line
-      // but not in the PCH file will not be added to SuggestedPredefines.
-      CheckPredefinesBuffers())
-    return IgnorePCH;
+  // Load the AST blocks of all of the modules that we loaded.
+  for (llvm::SmallVectorImpl<ModuleFile *>::iterator M = Loaded.begin(),
+                                                  MEnd = Loaded.end();
+       M != MEnd; ++M) {
+    ModuleFile &F = **M;
+
+    // Read the AST block.
+    if (ReadASTBlock(F))
+      return Failure;
+
+    // Once read, set the ModuleFile bit base offset and update the size in 
+    // bits of all files we've seen.
+    F.GlobalBitOffset = TotalModulesSizeInBits;
+    TotalModulesSizeInBits += F.SizeInBits;
+    GlobalBitOffsetsMap.insert(std::make_pair(F.GlobalBitOffset, &F));
+    
+    // Preload SLocEntries.
+    for (unsigned I = 0, N = F.PreloadSLocEntries.size(); I != N; ++I) {
+      int Index = int(F.PreloadSLocEntries[I] - 1) + F.SLocEntryBaseID;
+      // Load it through the SourceManager and don't call ReadSLocEntry()
+      // directly because the entry may have already been loaded in which case
+      // calling ReadSLocEntry() directly would trigger an assertion in
+      // SourceManager.
+      SourceMgr.getLoadedSLocEntryByID(Index);
+    }
+  }
 
   // Mark all of the identifiers in the identifier table as being out of date,
   // so that various accessors know to check the loaded modules when the
@@ -2707,17 +2754,19 @@ ASTReader::ASTReadResult ASTReader::ReadAST(const std::string &FileName,
   if (DeserializationListener)
     DeserializationListener->ReaderInitialized(this);
 
-  if (!OriginalFileID.isInvalid()) {
-    OriginalFileID = FileID::get(ModuleMgr.getPrimaryModule().SLocEntryBaseID
-                                      + OriginalFileID.getOpaqueValue() - 1);
+  ModuleFile &PrimaryModule = ModuleMgr.getPrimaryModule();
+  if (!PrimaryModule.OriginalSourceFileID.isInvalid()) {
+    PrimaryModule.OriginalSourceFileID 
+      = FileID::get(PrimaryModule.SLocEntryBaseID
+                    + PrimaryModule.OriginalSourceFileID.getOpaqueValue() - 1);
 
-    // If this AST file is a precompiled preamble, then set the preamble file ID
-    // of the source manager to the file source file from which the preamble was
-    // built.
+    // If this AST file is a precompiled preamble, then set the
+    // preamble file ID of the source manager to the file source file
+    // from which the preamble was built.
     if (Type == MK_Preamble) {
-      SourceMgr.setPreambleFileID(OriginalFileID);
+      SourceMgr.setPreambleFileID(PrimaryModule.OriginalSourceFileID);
     } else if (Type == MK_MainFile) {
-      SourceMgr.setMainFileID(OriginalFileID);
+      SourceMgr.setMainFileID(PrimaryModule.OriginalSourceFileID);
     }
   }
   
@@ -2732,9 +2781,12 @@ ASTReader::ASTReadResult ASTReader::ReadAST(const std::string &FileName,
   return Success;
 }
 
-ASTReader::ASTReadResult ASTReader::ReadASTCore(StringRef FileName,
-                                                ModuleKind Type,
-                                                ModuleFile *ImportedBy) {
+ASTReader::ASTReadResult
+ASTReader::ReadASTCore(StringRef FileName,
+                       ModuleKind Type,
+                       ModuleFile *ImportedBy,
+                       llvm::SmallVectorImpl<ModuleFile *> &Loaded,
+                       unsigned ClientLoadCapabilities) {
   ModuleFile *M;
   bool NewModule;
   std::string ErrorStr;
@@ -2785,7 +2837,7 @@ ASTReader::ASTReadResult ASTReader::ReadASTCore(StringRef FileName,
 
     unsigned BlockID = Stream.ReadSubBlockID();
 
-    // We only know the AST subblock ID.
+    // We only know the control subblock ID.
     switch (BlockID) {
     case llvm::bitc::BLOCKINFO_BLOCK_ID:
       if (Stream.ReadBlockInfoBlock()) {
@@ -2793,29 +2845,23 @@ ASTReader::ASTReadResult ASTReader::ReadASTCore(StringRef FileName,
         return Failure;
       }
       break;
-    case AST_BLOCK_ID:
-      switch (ReadASTBlock(F)) {
+    case CONTROL_BLOCK_ID:
+      switch (ReadControlBlock(F, Loaded, ClientLoadCapabilities)) {
       case Success:
         break;
 
-      case Failure:
-        return Failure;
-
-      case IgnorePCH:
-        // FIXME: We could consider reading through to the end of this
-        // AST block, skipping subblocks, to see if there are other
-        // AST blocks elsewhere.
-
-        // FIXME: We can't clear loaded slocentries anymore.
-        //SourceMgr.ClearPreallocatedSLocEntries();
-
-        // Remove the stat cache.
-        if (F.StatCache)
-          FileMgr.removeStatCache((ASTStatCache*)F.StatCache);
-
-        return IgnorePCH;
+      case Failure: return Failure;
+      case OutOfDate: return OutOfDate;
+      case VersionMismatch: return VersionMismatch;
+      case ConfigurationMismatch: return ConfigurationMismatch;
+      case HadErrors: return HadErrors;
       }
       break;
+    case AST_BLOCK_ID:
+      // Record that we've loaded this module.
+      Loaded.push_back(M);
+      return Success;
+
     default:
       if (Stream.SkipBlock()) {
         Error("malformed block record in AST file");
@@ -2825,32 +2871,6 @@ ASTReader::ASTReadResult ASTReader::ReadASTCore(StringRef FileName,
     }
   }
   
-  // Once read, set the ModuleFile bit base offset and update the size in 
-  // bits of all files we've seen.
-  F.GlobalBitOffset = TotalModulesSizeInBits;
-  TotalModulesSizeInBits += F.SizeInBits;
-  GlobalBitOffsetsMap.insert(std::make_pair(F.GlobalBitOffset, &F));
-
-  // Make sure that the files this module was built against are still available.
-  if (!DisableValidation) {
-    switch(validateFileEntries(*M)) {
-    case Failure: return Failure;
-    case IgnorePCH: return IgnorePCH;
-    case Success: break;
-    }
-  }
-  
-  // Preload SLocEntries.
-  for (unsigned I = 0, N = M->PreloadSLocEntries.size(); I != N; ++I) {
-    int Index = int(M->PreloadSLocEntries[I] - 1) + F.SLocEntryBaseID;
-    // Load it through the SourceManager and don't call ReadSLocEntryRecord()
-    // directly because the entry may have already been loaded in which case
-    // calling ReadSLocEntryRecord() directly would trigger an assertion in
-    // SourceManager.
-    SourceMgr.getLoadedSLocEntryByID(Index);
-  }
-
-
   return Success;
 }
 
@@ -3038,8 +3058,8 @@ std::string ASTReader::getOriginalSourceFile(const std::string &ASTFileName,
 
       // We only know the AST subblock ID.
       switch (BlockID) {
-      case AST_BLOCK_ID:
-        if (Stream.EnterSubBlock(AST_BLOCK_ID)) {
+      case CONTROL_BLOCK_ID:
+        if (Stream.EnterSubBlock(CONTROL_BLOCK_ID)) {
           Diags.Report(diag::err_fe_pch_malformed_block) << ASTFileName;
           return std::string();
         }
@@ -3071,19 +3091,191 @@ std::string ASTReader::getOriginalSourceFile(const std::string &ASTFileName,
     Record.clear();
     const char *BlobStart = 0;
     unsigned BlobLen = 0;
-    if (Stream.ReadRecord(Code, Record, &BlobStart, &BlobLen)
-          == ORIGINAL_FILE_NAME)
+    if (Stream.ReadRecord(Code, Record, &BlobStart, &BlobLen) == ORIGINAL_FILE)
       return std::string(BlobStart, BlobLen);
   }
 
   return std::string();
 }
 
-ASTReader::ASTReadResult ASTReader::ReadSubmoduleBlock(ModuleFile &F) {
+namespace {
+  class SimplePCHValidator : public ASTReaderListener {
+    const LangOptions &ExistingLangOpts;
+    const TargetOptions &ExistingTargetOpts;
+    const PreprocessorOptions &ExistingPPOpts;
+    FileManager &FileMgr;
+    
+  public:
+    SimplePCHValidator(const LangOptions &ExistingLangOpts,
+                       const TargetOptions &ExistingTargetOpts,
+                       const PreprocessorOptions &ExistingPPOpts,
+                       FileManager &FileMgr)
+      : ExistingLangOpts(ExistingLangOpts),
+        ExistingTargetOpts(ExistingTargetOpts),
+        ExistingPPOpts(ExistingPPOpts),
+        FileMgr(FileMgr)
+    {
+    }
+
+    virtual bool ReadLanguageOptions(const LangOptions &LangOpts,
+                                     bool Complain) {
+      return checkLanguageOptions(ExistingLangOpts, LangOpts, 0);
+    }
+    virtual bool ReadTargetOptions(const TargetOptions &TargetOpts,
+                                   bool Complain) {
+      return checkTargetOptions(ExistingTargetOpts, TargetOpts, 0);
+    }
+    virtual bool ReadPreprocessorOptions(const PreprocessorOptions &PPOpts,
+                                         bool Complain,
+                                         std::string &SuggestedPredefines) {
+      return checkPreprocessorOptions(ExistingPPOpts, PPOpts, 0, FileMgr,
+                                      SuggestedPredefines);
+    }
+  };
+}
+
+bool ASTReader::readASTFileControlBlock(StringRef Filename,
+                                        FileManager &FileMgr,
+                                        ASTReaderListener &Listener) {
+  // Open the AST file.
+  std::string ErrStr;
+  OwningPtr<llvm::MemoryBuffer> Buffer;
+  Buffer.reset(FileMgr.getBufferForFile(Filename, &ErrStr));
+  if (!Buffer) {
+    return true;
+  }
+
+  // Initialize the stream
+  llvm::BitstreamReader StreamFile;
+  llvm::BitstreamCursor Stream;
+  StreamFile.init((const unsigned char *)Buffer->getBufferStart(),
+                  (const unsigned char *)Buffer->getBufferEnd());
+  Stream.init(StreamFile);
+
+  // Sniff for the signature.
+  if (Stream.Read(8) != 'C' ||
+      Stream.Read(8) != 'P' ||
+      Stream.Read(8) != 'C' ||
+      Stream.Read(8) != 'H') {
+    return true;
+  }
+
+  RecordData Record;
+  bool InControlBlock = false;
+  while (!Stream.AtEndOfStream()) {
+    unsigned Code = Stream.ReadCode();
+
+    if (Code == llvm::bitc::ENTER_SUBBLOCK) {
+      unsigned BlockID = Stream.ReadSubBlockID();
+
+      // We only know the control subblock ID.
+      switch (BlockID) {
+      case CONTROL_BLOCK_ID:
+        if (Stream.EnterSubBlock(CONTROL_BLOCK_ID)) {
+          return true;
+        } else {
+          InControlBlock = true;
+        }
+        break;
+
+      default:
+        if (Stream.SkipBlock())
+          return true;
+        break;
+      }
+      continue;
+    }
+
+    if (Code == llvm::bitc::END_BLOCK) {
+      if (Stream.ReadBlockEnd()) {
+        return true;
+      }
+
+      InControlBlock = false;
+      continue;
+    }
+
+    if (Code == llvm::bitc::DEFINE_ABBREV) {
+      Stream.ReadAbbrevRecord();
+      continue;
+    }
+
+    Record.clear();
+    const char *BlobStart = 0;
+    unsigned BlobLen = 0;
+    unsigned RecCode = Stream.ReadRecord(Code, Record, &BlobStart, &BlobLen);
+    if (InControlBlock) {
+      switch ((ControlRecordTypes)RecCode) {
+      case METADATA: {
+        if (Record[0] != VERSION_MAJOR) {
+          return true;
+        }
+
+        const std::string &CurBranch = getClangFullRepositoryVersion();
+        StringRef ASTBranch(BlobStart, BlobLen);
+        if (StringRef(CurBranch) != ASTBranch)
+          return true;
+
+        break;
+      }
+      case LANGUAGE_OPTIONS:
+        if (ParseLanguageOptions(Record, false, Listener))
+          return true;
+        break;
+
+      case TARGET_OPTIONS:
+        if (ParseTargetOptions(Record, false, Listener))
+          return true;
+        break;
+
+      case DIAGNOSTIC_OPTIONS:
+        if (ParseDiagnosticOptions(Record, false, Listener))
+          return true;
+        break;
+
+      case FILE_SYSTEM_OPTIONS:
+        if (ParseFileSystemOptions(Record, false, Listener))
+          return true;
+        break;
+
+      case HEADER_SEARCH_OPTIONS:
+        if (ParseHeaderSearchOptions(Record, false, Listener))
+          return true;
+        break;
+
+      case PREPROCESSOR_OPTIONS: {
+        std::string IgnoredSuggestedPredefines;
+        if (ParsePreprocessorOptions(Record, false, Listener,
+                                     IgnoredSuggestedPredefines))
+          return true;
+        break;
+      }
+
+      default:
+        // No other validation to perform.
+        break;
+      }
+    }
+  }
+  
+  return false;
+}
+
+
+bool ASTReader::isAcceptableASTFile(StringRef Filename,
+                                    FileManager &FileMgr,
+                                    const LangOptions &LangOpts,
+                                    const TargetOptions &TargetOpts,
+                                    const PreprocessorOptions &PPOpts) {
+  SimplePCHValidator validator(LangOpts, TargetOpts, PPOpts, FileMgr);
+  return !readASTFileControlBlock(Filename, FileMgr, validator);
+}
+
+bool ASTReader::ReadSubmoduleBlock(ModuleFile &F) {
   // Enter the submodule block.
   if (F.Stream.EnterSubBlock(SUBMODULE_BLOCK_ID)) {
     Error("malformed submodule block record in AST file");
-    return Failure;
+    return true;
   }
 
   ModuleMap &ModMap = PP.getHeaderSearchInfo().getModuleMap();
@@ -3095,9 +3287,9 @@ ASTReader::ASTReadResult ASTReader::ReadSubmoduleBlock(ModuleFile &F) {
     if (Code == llvm::bitc::END_BLOCK) {
       if (F.Stream.ReadBlockEnd()) {
         Error("error at end of submodule block in AST file");
-        return Failure;
+        return true;
       }
-      return Success;
+      return false;
     }
     
     if (Code == llvm::bitc::ENTER_SUBBLOCK) {
@@ -3105,7 +3297,7 @@ ASTReader::ASTReadResult ASTReader::ReadSubmoduleBlock(ModuleFile &F) {
       F.Stream.ReadSubBlockID();
       if (F.Stream.SkipBlock()) {
         Error("malformed block record in AST file");
-        return Failure;
+        return true;
       }
       continue;
     }
@@ -3126,12 +3318,12 @@ ASTReader::ASTReadResult ASTReader::ReadSubmoduleBlock(ModuleFile &F) {
     case SUBMODULE_DEFINITION: {
       if (First) {
         Error("missing submodule metadata record at beginning of block");
-        return Failure;
+        return true;
       }
 
       if (Record.size() < 7) {
         Error("malformed module definition");
-        return Failure;
+        return true;
       }
       
       StringRef Name(BlobStart, BlobLen);
@@ -3157,9 +3349,10 @@ ASTReader::ASTReadResult ASTReader::ReadSubmoduleBlock(ModuleFile &F) {
       if (GlobalIndex >= SubmodulesLoaded.size() ||
           SubmodulesLoaded[GlobalIndex]) {
         Error("too many submodules");
-        return Failure;
+        return true;
       }
       
+      CurrentModule->setASTFile(F.File);
       CurrentModule->IsFromModuleFile = true;
       CurrentModule->IsSystem = IsSystem || CurrentModule->IsSystem;
       CurrentModule->InferSubmodules = InferSubmodules;
@@ -3175,7 +3368,7 @@ ASTReader::ASTReadResult ASTReader::ReadSubmoduleBlock(ModuleFile &F) {
     case SUBMODULE_UMBRELLA_HEADER: {
       if (First) {
         Error("missing submodule metadata record at beginning of block");
-        return Failure;
+        return true;
       }
 
       if (!CurrentModule)
@@ -3187,7 +3380,7 @@ ASTReader::ASTReadResult ASTReader::ReadSubmoduleBlock(ModuleFile &F) {
           ModMap.setUmbrellaHeader(CurrentModule, Umbrella);
         else if (CurrentModule->getUmbrellaHeader() != Umbrella) {
           Error("mismatched umbrella headers in submodule");
-          return Failure;
+          return true;
         }
       }
       break;
@@ -3196,7 +3389,7 @@ ASTReader::ASTReadResult ASTReader::ReadSubmoduleBlock(ModuleFile &F) {
     case SUBMODULE_HEADER: {
       if (First) {
         Error("missing submodule metadata record at beginning of block");
-        return Failure;
+        return true;
       }
 
       if (!CurrentModule)
@@ -3208,15 +3401,51 @@ ASTReader::ASTReadResult ASTReader::ReadSubmoduleBlock(ModuleFile &F) {
         if (std::find(CurrentModule->Headers.begin(), 
                       CurrentModule->Headers.end(), 
                       File) == CurrentModule->Headers.end())
-          ModMap.addHeader(CurrentModule, File);
+          ModMap.addHeader(CurrentModule, File, false);
       }
       break;      
     }
-        
+
+    case SUBMODULE_EXCLUDED_HEADER: {
+      if (First) {
+        Error("missing submodule metadata record at beginning of block");
+        return true;
+      }
+
+      if (!CurrentModule)
+        break;
+      
+      // FIXME: Be more lazy about this!
+      StringRef FileName(BlobStart, BlobLen);
+      if (const FileEntry *File = PP.getFileManager().getFile(FileName)) {
+        if (std::find(CurrentModule->Headers.begin(), 
+                      CurrentModule->Headers.end(), 
+                      File) == CurrentModule->Headers.end())
+          ModMap.addHeader(CurrentModule, File, true);
+      }
+      break;      
+    }
+
+    case SUBMODULE_TOPHEADER: {
+      if (First) {
+        Error("missing submodule metadata record at beginning of block");
+        return true;
+      }
+
+      if (!CurrentModule)
+        break;
+
+      // FIXME: Be more lazy about this!
+      StringRef FileName(BlobStart, BlobLen);
+      if (const FileEntry *File = PP.getFileManager().getFile(FileName))
+        CurrentModule->TopHeaders.insert(File);
+      break;
+    }
+
     case SUBMODULE_UMBRELLA_DIR: {
       if (First) {
         Error("missing submodule metadata record at beginning of block");
-        return Failure;
+        return true;
       }
       
       if (!CurrentModule)
@@ -3229,7 +3458,7 @@ ASTReader::ASTReadResult ASTReader::ReadSubmoduleBlock(ModuleFile &F) {
           ModMap.setUmbrellaDir(CurrentModule, Umbrella);
         else if (CurrentModule->getUmbrellaDir() != Umbrella) {
           Error("mismatched umbrella directories in submodule");
-          return Failure;
+          return true;
         }
       }
       break;
@@ -3238,7 +3467,7 @@ ASTReader::ASTReadResult ASTReader::ReadSubmoduleBlock(ModuleFile &F) {
     case SUBMODULE_METADATA: {
       if (!First) {
         Error("submodule metadata record not at beginning of block");
-        return Failure;
+        return true;
       }
       First = false;
       
@@ -3264,7 +3493,7 @@ ASTReader::ASTReadResult ASTReader::ReadSubmoduleBlock(ModuleFile &F) {
     case SUBMODULE_IMPORTS: {
       if (First) {
         Error("missing submodule metadata record at beginning of block");
-        return Failure;
+        return true;
       }
       
       if (!CurrentModule)
@@ -3285,7 +3514,7 @@ ASTReader::ASTReadResult ASTReader::ReadSubmoduleBlock(ModuleFile &F) {
     case SUBMODULE_EXPORTS: {
       if (First) {
         Error("missing submodule metadata record at beginning of block");
-        return Failure;
+        return true;
       }
       
       if (!CurrentModule)
@@ -3309,7 +3538,7 @@ ASTReader::ASTReadResult ASTReader::ReadSubmoduleBlock(ModuleFile &F) {
     case SUBMODULE_REQUIRES: {
       if (First) {
         Error("missing submodule metadata record at beginning of block");
-        return Failure;
+        return true;
       }
 
       if (!CurrentModule)
@@ -3331,27 +3560,144 @@ ASTReader::ASTReadResult ASTReader::ReadSubmoduleBlock(ModuleFile &F) {
 /// them to the AST listener if one is set.
 ///
 /// \returns true if the listener deems the file unacceptable, false otherwise.
-bool ASTReader::ParseLanguageOptions(const RecordData &Record) {
-  if (Listener) {
-    LangOptions LangOpts;
-    unsigned Idx = 0;
+bool ASTReader::ParseLanguageOptions(const RecordData &Record,
+                                     bool Complain,
+                                     ASTReaderListener &Listener) {
+  LangOptions LangOpts;
+  unsigned Idx = 0;
 #define LANGOPT(Name, Bits, Default, Description) \
   LangOpts.Name = Record[Idx++];
 #define ENUM_LANGOPT(Name, Type, Bits, Default, Description) \
   LangOpts.set##Name(static_cast<LangOptions::Type>(Record[Idx++]));
 #include "clang/Basic/LangOptions.def"
 
-    ObjCRuntime::Kind runtimeKind = (ObjCRuntime::Kind) Record[Idx++];
-    VersionTuple runtimeVersion = ReadVersionTuple(Record, Idx);
-    LangOpts.ObjCRuntime = ObjCRuntime(runtimeKind, runtimeVersion);
-    
-    unsigned Length = Record[Idx++];
-    LangOpts.CurrentModule.assign(Record.begin() + Idx, 
-                                  Record.begin() + Idx + Length);
-    return Listener->ReadLanguageOptions(LangOpts);
+  ObjCRuntime::Kind runtimeKind = (ObjCRuntime::Kind) Record[Idx++];
+  VersionTuple runtimeVersion = ReadVersionTuple(Record, Idx);
+  LangOpts.ObjCRuntime = ObjCRuntime(runtimeKind, runtimeVersion);
+  
+  unsigned Length = Record[Idx++];
+  LangOpts.CurrentModule.assign(Record.begin() + Idx, 
+                                Record.begin() + Idx + Length);
+  return Listener.ReadLanguageOptions(LangOpts, Complain);
+}
+
+bool ASTReader::ParseTargetOptions(const RecordData &Record,
+                                   bool Complain,
+                                   ASTReaderListener &Listener) {
+  unsigned Idx = 0;
+  TargetOptions TargetOpts;
+  TargetOpts.Triple = ReadString(Record, Idx);
+  TargetOpts.CPU = ReadString(Record, Idx);
+  TargetOpts.ABI = ReadString(Record, Idx);
+  TargetOpts.CXXABI = ReadString(Record, Idx);
+  TargetOpts.LinkerVersion = ReadString(Record, Idx);
+  for (unsigned N = Record[Idx++]; N; --N) {
+    TargetOpts.FeaturesAsWritten.push_back(ReadString(Record, Idx));
+  }
+  for (unsigned N = Record[Idx++]; N; --N) {
+    TargetOpts.Features.push_back(ReadString(Record, Idx));
   }
 
-  return false;
+  return Listener.ReadTargetOptions(TargetOpts, Complain);
+}
+
+bool ASTReader::ParseDiagnosticOptions(const RecordData &Record, bool Complain,
+                                       ASTReaderListener &Listener) {
+  DiagnosticOptions DiagOpts;
+  unsigned Idx = 0;
+#define DIAGOPT(Name, Bits, Default) DiagOpts.Name = Record[Idx++];
+#define ENUM_DIAGOPT(Name, Type, Bits, Default) \
+  DiagOpts.set##Name(static_cast<Type>(Record[Idx++]));
+#include "clang/Basic/DiagnosticOptions.def"
+
+  for (unsigned N = Record[Idx++]; N; --N) {
+    DiagOpts.Warnings.push_back(ReadString(Record, Idx));
+  }
+
+  return Listener.ReadDiagnosticOptions(DiagOpts, Complain);
+}
+
+bool ASTReader::ParseFileSystemOptions(const RecordData &Record, bool Complain,
+                                       ASTReaderListener &Listener) {
+  FileSystemOptions FSOpts;
+  unsigned Idx = 0;
+  FSOpts.WorkingDir = ReadString(Record, Idx);
+  return Listener.ReadFileSystemOptions(FSOpts, Complain);
+}
+
+bool ASTReader::ParseHeaderSearchOptions(const RecordData &Record,
+                                         bool Complain,
+                                         ASTReaderListener &Listener) {
+  HeaderSearchOptions HSOpts;
+  unsigned Idx = 0;
+  HSOpts.Sysroot = ReadString(Record, Idx);
+
+  // Include entries.
+  for (unsigned N = Record[Idx++]; N; --N) {
+    std::string Path = ReadString(Record, Idx);
+    frontend::IncludeDirGroup Group
+      = static_cast<frontend::IncludeDirGroup>(Record[Idx++]);
+    bool IsUserSupplied = Record[Idx++];
+    bool IsFramework = Record[Idx++];
+    bool IgnoreSysRoot = Record[Idx++];
+    bool IsInternal = Record[Idx++];
+    bool ImplicitExternC = Record[Idx++];
+    HSOpts.UserEntries.push_back(
+      HeaderSearchOptions::Entry(Path, Group, IsUserSupplied, IsFramework,
+                                 IgnoreSysRoot, IsInternal, ImplicitExternC));
+  }
+
+  // System header prefixes.
+  for (unsigned N = Record[Idx++]; N; --N) {
+    std::string Prefix = ReadString(Record, Idx);
+    bool IsSystemHeader = Record[Idx++];
+    HSOpts.SystemHeaderPrefixes.push_back(
+      HeaderSearchOptions::SystemHeaderPrefix(Prefix, IsSystemHeader));
+  }
+
+  HSOpts.ResourceDir = ReadString(Record, Idx);
+  HSOpts.ModuleCachePath = ReadString(Record, Idx);
+  HSOpts.DisableModuleHash = Record[Idx++];
+  HSOpts.UseBuiltinIncludes = Record[Idx++];
+  HSOpts.UseStandardSystemIncludes = Record[Idx++];
+  HSOpts.UseStandardCXXIncludes = Record[Idx++];
+  HSOpts.UseLibcxx = Record[Idx++];
+
+  return Listener.ReadHeaderSearchOptions(HSOpts, Complain);
+}
+
+bool ASTReader::ParsePreprocessorOptions(const RecordData &Record,
+                                         bool Complain,
+                                         ASTReaderListener &Listener,
+                                         std::string &SuggestedPredefines) {
+  PreprocessorOptions PPOpts;
+  unsigned Idx = 0;
+
+  // Macro definitions/undefs
+  for (unsigned N = Record[Idx++]; N; --N) {
+    std::string Macro = ReadString(Record, Idx);
+    bool IsUndef = Record[Idx++];
+    PPOpts.Macros.push_back(std::make_pair(Macro, IsUndef));
+  }
+
+  // Includes
+  for (unsigned N = Record[Idx++]; N; --N) {
+    PPOpts.Includes.push_back(ReadString(Record, Idx));
+  }
+
+  // Macro Includes
+  for (unsigned N = Record[Idx++]; N; --N) {
+    PPOpts.MacroIncludes.push_back(ReadString(Record, Idx));
+  }
+
+  PPOpts.UsePredefines = Record[Idx++];
+  PPOpts.ImplicitPCHInclude = ReadString(Record, Idx);
+  PPOpts.ImplicitPTHInclude = ReadString(Record, Idx);
+  PPOpts.ObjCXXARCStandardLibrary =
+    static_cast<ObjCXXARCStandardLibraryKind>(Record[Idx++]);
+  SuggestedPredefines.clear();
+  return Listener.ReadPreprocessorOptions(PPOpts, Complain,
+                                          SuggestedPredefines);
 }
 
 std::pair<ModuleFile *, unsigned>
@@ -3365,6 +3711,23 @@ ASTReader::getModulePreprocessedEntity(unsigned GlobalIndex) {
   return std::make_pair(M, LocalIndex);
 }
 
+std::pair<PreprocessingRecord::iterator, PreprocessingRecord::iterator>
+ASTReader::getModulePreprocessedEntities(ModuleFile &Mod) const {
+  if (PreprocessingRecord *PPRec = PP.getPreprocessingRecord())
+    return PPRec->getIteratorsForLoadedRange(Mod.BasePreprocessedEntityID,
+                                             Mod.NumPreprocessedEntities);
+
+  return std::make_pair(PreprocessingRecord::iterator(),
+                        PreprocessingRecord::iterator());
+}
+
+std::pair<ASTReader::ModuleDeclIterator, ASTReader::ModuleDeclIterator>
+ASTReader::getModuleFileLevelDecls(ModuleFile &Mod) {
+  return std::make_pair(ModuleDeclIterator(this, &Mod, Mod.FileSortedDecls),
+                        ModuleDeclIterator(this, &Mod,
+                                 Mod.FileSortedDecls + Mod.NumFileSortedDecls));
+}
+
 PreprocessedEntity *ASTReader::ReadPreprocessedEntity(unsigned Index) {
   PreprocessedEntityID PPID = Index+1;
   std::pair<ModuleFile *, unsigned> PPInfo = getModulePreprocessedEntity(Index);
@@ -3455,7 +3818,7 @@ PreprocessedEntity *ASTReader::ReadPreprocessedEntity(unsigned Index) {
     InclusionDirective *ID
       = new (PPRec) InclusionDirective(PPRec, Kind,
                                        StringRef(BlobStart, Record[0]),
-                                       Record[1],
+                                       Record[1], Record[3],
                                        File,
                                        Range);
     return ID;
@@ -3476,7 +3839,7 @@ PreprocessedEntityID ASTReader::findNextPreprocessedEntity(
          EndI = GlobalSLocOffsetMap.end(); SLocMapI != EndI; ++SLocMapI) {
     ModuleFile &M = *SLocMapI->second;
     if (M.NumPreprocessedEntities)
-      return getGlobalPreprocessedEntityID(M, M.BasePreprocessedEntityID);
+      return M.BasePreprocessedEntityID;
   }
 
   return getTotalNumPreprocessedEntities();
@@ -3559,8 +3922,7 @@ ASTReader::findBeginPreprocessedEntity(SourceLocation BLoc) const {
   if (PPI == pp_end)
     return findNextPreprocessedEntity(SLocMapI);
 
-  return getGlobalPreprocessedEntityID(M,
-                                 M.BasePreprocessedEntityID + (PPI - pp_begin));
+  return M.BasePreprocessedEntityID + (PPI - pp_begin);
 }
 
 /// \brief Returns the first preprocessed entity ID that begins after \arg ELoc.
@@ -3589,8 +3951,7 @@ ASTReader::findEndPreprocessedEntity(SourceLocation ELoc) const {
   if (PPI == pp_end)
     return findNextPreprocessedEntity(SLocMapI);
 
-  return getGlobalPreprocessedEntityID(M,
-                                 M.BasePreprocessedEntityID + (PPI - pp_begin));
+  return M.BasePreprocessedEntityID + (PPI - pp_begin);
 }
 
 /// \brief Returns a pair of [Begin, End) indices of preallocated
@@ -3681,14 +4042,31 @@ HeaderFileInfo ASTReader::GetHeaderFileInfo(const FileEntry *FE) {
 }
 
 void ASTReader::ReadPragmaDiagnosticMappings(DiagnosticsEngine &Diag) {
+  // FIXME: Make it work properly with modules.
+  llvm::SmallVector<DiagnosticsEngine::DiagState *, 32> DiagStates;
   for (ModuleIterator I = ModuleMgr.begin(), E = ModuleMgr.end(); I != E; ++I) {
     ModuleFile &F = *(*I);
     unsigned Idx = 0;
+    DiagStates.clear();
+    assert(!Diag.DiagStates.empty());
+    DiagStates.push_back(&Diag.DiagStates.front()); // the command-line one.
     while (Idx < F.PragmaDiagMappings.size()) {
       SourceLocation Loc = ReadSourceLocation(F, F.PragmaDiagMappings[Idx++]);
+      unsigned DiagStateID = F.PragmaDiagMappings[Idx++];
+      if (DiagStateID != 0) {
+        Diag.DiagStatePoints.push_back(
+                    DiagnosticsEngine::DiagStatePoint(DiagStates[DiagStateID-1],
+                    FullSourceLoc(Loc, SourceMgr)));
+        continue;
+      }
+      
+      assert(DiagStateID == 0);
+      // A new DiagState was created here.
       Diag.DiagStates.push_back(*Diag.GetCurDiagState());
+      DiagnosticsEngine::DiagState *NewState = &Diag.DiagStates.back();
+      DiagStates.push_back(NewState);
       Diag.DiagStatePoints.push_back(
-          DiagnosticsEngine::DiagStatePoint(&Diag.DiagStates.back(),
+          DiagnosticsEngine::DiagStatePoint(NewState,
                                             FullSourceLoc(Loc, SourceMgr)));
       while (1) {
         assert(Idx < F.PragmaDiagMappings.size() &&
@@ -4258,6 +4636,8 @@ void TypeLocReader::VisitExtVectorTypeLoc(ExtVectorTypeLoc TL) {
 }
 void TypeLocReader::VisitFunctionTypeLoc(FunctionTypeLoc TL) {
   TL.setLocalRangeBegin(ReadSourceLocation(Record, Idx));
+  TL.setLParenLoc(ReadSourceLocation(Record, Idx));
+  TL.setRParenLoc(ReadSourceLocation(Record, Idx));
   TL.setLocalRangeEnd(ReadSourceLocation(Record, Idx));
   for (unsigned i = 0, e = TL.getNumArgs(); i != e; ++i) {
     TL.setArg(i, ReadDeclAs<ParmVarDecl>(Record, Idx));
@@ -4467,6 +4847,10 @@ QualType ASTReader::GetType(TypeID ID) {
     case PREDEF_TYPE_VA_LIST_TAG:
       T = Context.getVaListTagType();
       break;
+
+    case PREDEF_TYPE_BUILTIN_FN:
+      T = Context.BuiltinFnTy;
+      break;
     }
 
     assert(!T.isNull() && "Unknown predefined type");
@@ -4537,7 +4921,9 @@ ASTReader::GetTemplateArgumentLocInfo(ModuleFile &F,
   case TemplateArgument::Null:
   case TemplateArgument::Integral:
   case TemplateArgument::Declaration:
+  case TemplateArgument::NullPtr:
   case TemplateArgument::Pack:
+    // FIXME: Is this right?
     return TemplateArgumentLocInfo();
   }
   llvm_unreachable("unexpected template argument loc");
@@ -4593,7 +4979,7 @@ CXXBaseSpecifier *ASTReader::GetExternalCXXBaseSpecifiers(uint64_t Offset) {
 }
 
 serialization::DeclID 
-ASTReader::getGlobalDeclID(ModuleFile &F, unsigned LocalID) const {
+ASTReader::getGlobalDeclID(ModuleFile &F, LocalDeclID LocalID) const {
   if (LocalID < NUM_PREDEF_DECL_IDS)
     return LocalID;
 
@@ -4930,8 +5316,10 @@ namespace {
           continue;
 
         if (ND->getDeclName() != This->Name) {
-          assert(!This->Name.getCXXNameType().isNull() && 
-                 "Name mismatch without a type");
+          // A name might be null because the decl's redeclarable part is
+          // currently read before reading its name. The lookup is triggered by
+          // building that decl (likely indirectly), and so it is later in the
+          // sense of "already existing" and can be ignored here.
           continue;
         }
       
@@ -5132,13 +5520,15 @@ void ASTReader::PrintStats() {
     = IdentifiersLoaded.size() - std::count(IdentifiersLoaded.begin(),
                                             IdentifiersLoaded.end(),
                                             (IdentifierInfo *)0);
+  unsigned NumMacrosLoaded
+    = MacrosLoaded.size() - std::count(MacrosLoaded.begin(),
+                                       MacrosLoaded.end(),
+                                       (MacroInfo *)0);
   unsigned NumSelectorsLoaded
     = SelectorsLoaded.size() - std::count(SelectorsLoaded.begin(),
                                           SelectorsLoaded.end(),
                                           Selector());
 
-  std::fprintf(stderr, "  %u stat cache hits\n", NumStatHits);
-  std::fprintf(stderr, "  %u stat cache misses\n", NumStatMisses);
   if (unsigned TotalNumSLocEntries = getTotalNumSLocs())
     std::fprintf(stderr, "  %u/%u source location entries read (%f%%)\n",
                  NumSLocEntriesRead, TotalNumSLocEntries,
@@ -5155,6 +5545,10 @@ void ASTReader::PrintStats() {
     std::fprintf(stderr, "  %u/%u identifiers read (%f%%)\n",
                  NumIdentifiersLoaded, (unsigned)IdentifiersLoaded.size(),
                  ((float)NumIdentifiersLoaded/IdentifiersLoaded.size() * 100));
+  if (!MacrosLoaded.empty())
+    std::fprintf(stderr, "  %u/%u macros read (%f%%)\n",
+                 NumMacrosLoaded, (unsigned)MacrosLoaded.size(),
+                 ((float)NumMacrosLoaded/MacrosLoaded.size() * 100));
   if (!SelectorsLoaded.empty())
     std::fprintf(stderr, "  %u/%u selectors read (%f%%)\n",
                  NumSelectorsLoaded, (unsigned)SelectorsLoaded.size(),
@@ -5213,6 +5607,7 @@ void ASTReader::dump() {
   dumpModuleIDMap("Global type map", GlobalTypeMap);
   dumpModuleIDMap("Global declaration map", GlobalDeclMap);
   dumpModuleIDMap("Global identifier map", GlobalIdentifierMap);
+  dumpModuleIDMap("Global macro map", GlobalMacroMap);
   dumpModuleIDMap("Global submodule map", GlobalSubmoduleMap);
   dumpModuleIDMap("Global selector map", GlobalSelectorMap);
   dumpModuleIDMap("Global preprocessed entity map", 
@@ -5246,7 +5641,7 @@ void ASTReader::getMemoryBufferSizes(MemoryBufferSizes &sizes) const {
 
 void ASTReader::InitializeSema(Sema &S) {
   SemaObj = &S;
-  S.ExternalSource = this;
+  S.addExternalSource(this);
 
   // Makes sure any declarations that were deserialized "too early"
   // still get added to the identifier's declaration chains.
@@ -5281,6 +5676,9 @@ void ASTReader::InitializeSema(Sema &S) {
 }
 
 IdentifierInfo* ASTReader::get(const char *NameStart, const char *NameEnd) {
+  // Note that we are loading an identifier.
+  Deserializing AnIdentifier(this);
+  
   IdentifierLookupVisitor Visitor(StringRef(NameStart, NameEnd - NameStart),
                                   /*PriorGeneration=*/0);
   ModuleMgr.visit(IdentifierLookupVisitor::visit, &Visitor);
@@ -5570,6 +5968,7 @@ void ASTReader::ReadPendingInstantiations(
     ValueDecl *D = cast<ValueDecl>(GetDecl(PendingInstantiations[Idx++]));
     SourceLocation Loc
       = SourceLocation::getFromRawEncoding(PendingInstantiations[Idx++]);
+
     Pending.push_back(std::make_pair(D, Loc));
   }  
   PendingInstantiations.clear();
@@ -5682,8 +6081,37 @@ IdentifierID ASTReader::getGlobalIdentifierID(ModuleFile &M, unsigned LocalID) {
   return LocalID + I->second;
 }
 
-bool ASTReader::ReadSLocEntry(int ID) {
-  return ReadSLocEntryRecord(ID) != Success;
+MacroInfo *ASTReader::getMacro(MacroID ID, MacroInfo *Hint) {
+  if (ID == 0)
+    return 0;
+
+  if (MacrosLoaded.empty()) {
+    Error("no macro table in AST file");
+    return 0;
+  }
+
+  ID -= NUM_PREDEF_MACRO_IDS;
+  if (!MacrosLoaded[ID]) {
+    GlobalMacroMapType::iterator I
+      = GlobalMacroMap.find(ID + NUM_PREDEF_MACRO_IDS);
+    assert(I != GlobalMacroMap.end() && "Corrupted global macro map");
+    ModuleFile *M = I->second;
+    unsigned Index = ID - M->BaseMacroID;
+    ReadMacroRecord(*M, M->MacroOffsets[Index], Hint);
+  }
+
+  return MacrosLoaded[ID];
+}
+
+MacroID ASTReader::getGlobalMacroID(ModuleFile &M, unsigned LocalID) {
+  if (LocalID < NUM_PREDEF_MACRO_IDS)
+    return LocalID;
+
+  ContinuousRangeMap<uint32_t, int, 2>::iterator I
+    = M.MacroRemap.find(LocalID - NUM_PREDEF_MACRO_IDS);
+  assert(I != M.MacroRemap.end() && "Invalid index into macro index remap");
+
+  return LocalID + I->second;
 }
 
 serialization::SubmoduleID
@@ -5694,7 +6122,7 @@ ASTReader::getGlobalSubmoduleID(ModuleFile &M, unsigned LocalID) {
   ContinuousRangeMap<uint32_t, int, 2>::iterator I
     = M.SubmoduleRemap.find(LocalID - NUM_PREDEF_SUBMODULE_IDS);
   assert(I != M.SubmoduleRemap.end() 
-         && "Invalid index into identifier index remap");
+         && "Invalid index into submodule index remap");
   
   return LocalID + I->second;
 }
@@ -5759,7 +6187,7 @@ ASTReader::getGlobalSelectorID(ModuleFile &M, unsigned LocalID) const {
   ContinuousRangeMap<uint32_t, int, 2>::iterator I
     = M.SelectorRemap.find(LocalID - NUM_PREDEF_SELECTOR_IDS);
   assert(I != M.SelectorRemap.end() 
-         && "Invalid index into identifier index remap");
+         && "Invalid index into selector index remap");
   
   return LocalID + I->second;
 }
@@ -5926,8 +6354,13 @@ ASTReader::ReadTemplateArgument(ModuleFile &F,
     return TemplateArgument();
   case TemplateArgument::Type:
     return TemplateArgument(readType(F, Record, Idx));
-  case TemplateArgument::Declaration:
-    return TemplateArgument(ReadDecl(F, Record, Idx));
+  case TemplateArgument::Declaration: {
+    ValueDecl *D = ReadDeclAs<ValueDecl>(F, Record, Idx);
+    bool ForReferenceParam = Record[Idx++];
+    return TemplateArgument(D, ForReferenceParam);
+  }
+  case TemplateArgument::NullPtr:
+    return TemplateArgument(readType(F, Record, Idx), /*isNullPtr*/true);
   case TemplateArgument::Integral: {
     llvm::APSInt Value = ReadAPSInt(Record, Idx);
     QualType T = readType(F, Record, Idx);
@@ -6340,7 +6773,8 @@ void ASTReader::ReadComments() {
 }
 
 void ASTReader::finishPendingActions() {
-  while (!PendingIdentifierInfos.empty() || !PendingDeclChains.empty()) {
+  while (!PendingIdentifierInfos.empty() || !PendingDeclChains.empty() ||
+         !PendingMacroIDs.empty()) {
     // If any identifiers with corresponding top-level declarations have
     // been loaded, load those declarations now.
     while (!PendingIdentifierInfos.empty()) {
@@ -6355,6 +6789,18 @@ void ASTReader::finishPendingActions() {
       PendingDeclChainsKnown.erase(PendingDeclChains[I]);
     }
     PendingDeclChains.clear();
+
+    // Load any pending macro definitions.
+    for (unsigned I = 0; I != PendingMacroIDs.size(); ++I) {
+      // FIXME: std::move here
+      SmallVector<MacroID, 2> GlobalIDs = PendingMacroIDs.begin()[I].second;
+      MacroInfo *Hint = 0;
+      for (unsigned IDIdx = 0, NumIDs = GlobalIDs.size(); IDIdx !=  NumIDs;
+           ++IDIdx) {
+        Hint = getMacro(GlobalIDs[IDIdx], Hint);
+      }
+    }
+    PendingMacroIDs.clear();
   }
   
   // If we deserialized any C++ or Objective-C class definitions, any
@@ -6408,9 +6854,29 @@ void ASTReader::finishPendingActions() {
     for (RedeclarableTemplateDecl::redecl_iterator R = RTD->redecls_begin(),
                                                 REnd = RTD->redecls_end();
          R != REnd; ++R)
-      R->Common = RTD->Common;    
+      R->Common = RTD->Common;
   }
   PendingDefinitions.clear();
+
+  // Load the bodies of any functions or methods we've encountered. We do
+  // this now (delayed) so that we can be sure that the declaration chains
+  // have been fully wired up.
+  for (PendingBodiesMap::iterator PB = PendingBodies.begin(),
+                               PBEnd = PendingBodies.end();
+       PB != PBEnd; ++PB) {
+    if (FunctionDecl *FD = dyn_cast<FunctionDecl>(PB->first)) {
+      // FIXME: Check for =delete/=default?
+      // FIXME: Complain about ODR violations here?
+      if (!getContext().getLangOpts().Modules || !FD->hasBody())
+        FD->setLazyBody(PB->second);
+      continue;
+    }
+
+    ObjCMethodDecl *MD = cast<ObjCMethodDecl>(PB->first);
+    if (!getContext().getLangOpts().Modules || !MD->hasBody())
+      MD->setLazyBody(PB->second);
+  }
+  PendingBodies.clear();
 }
 
 void ASTReader::FinishedDeserializing() {
@@ -6442,17 +6908,14 @@ void ASTReader::FinishedDeserializing() {
 
 ASTReader::ASTReader(Preprocessor &PP, ASTContext &Context,
                      StringRef isysroot, bool DisableValidation,
-                     bool DisableStatCache, bool AllowASTWithCompilerErrors)
+                     bool AllowASTWithCompilerErrors)
   : Listener(new PCHValidator(PP, *this)), DeserializationListener(0),
     SourceMgr(PP.getSourceManager()), FileMgr(PP.getFileManager()),
     Diags(PP.getDiagnostics()), SemaObj(0), PP(PP), Context(Context),
-    Consumer(0), ModuleMgr(FileMgr.getFileSystemOptions()),
-    RelocatablePCH(false), isysroot(isysroot),
-    DisableValidation(DisableValidation),
-    DisableStatCache(DisableStatCache),
+    Consumer(0), ModuleMgr(PP.getFileManager()),
+    isysroot(isysroot), DisableValidation(DisableValidation),
     AllowASTWithCompilerErrors(AllowASTWithCompilerErrors), 
     CurrentGeneration(0), CurrSwitchCaseStmts(&SwitchCaseStmts),
-    NumStatHits(0), NumStatMisses(0), 
     NumSLocEntriesRead(0), TotalNumSLocEntries(0), 
     NumStatementsRead(0), TotalNumStatements(0), NumMacrosRead(0), 
     TotalNumMacros(0), NumSelectorsRead(0), NumMethodPoolEntriesRead(0), 
diff --git a/lib/Serialization/ASTReaderDecl.cpp b/lib/Serialization/ASTReaderDecl.cpp
index cb21f82..c42944d 100644
--- a/lib/Serialization/ASTReaderDecl.cpp
+++ b/lib/Serialization/ASTReaderDecl.cpp
@@ -37,7 +37,6 @@ namespace clang {
   class ASTDeclReader : public DeclVisitor<ASTDeclReader, void> {
     ASTReader &Reader;
     ModuleFile &F;
-    llvm::BitstreamCursor &Cursor;
     const DeclID ThisDeclID;
     const unsigned RawLocation;
     typedef ASTReader::RecordData RecordData;
@@ -48,6 +47,8 @@ namespace clang {
     DeclID DeclContextIDForTemplateParmDecl;
     DeclID LexicalDeclContextIDForTemplateParmDecl;
 
+    bool HasPendingBody;
+
     uint64_t GetCurrentCursorOffset();
     
     SourceLocation ReadSourceLocation(const RecordData &R, unsigned &I) {
@@ -116,7 +117,7 @@ namespace clang {
       GlobalDeclID FirstID;
       mutable bool Owning;
       
-      RedeclarableResult &operator=(RedeclarableResult&); // DO NOT IMPLEMENT
+      void operator=(RedeclarableResult &) LLVM_DELETED_FUNCTION;
       
     public:
       RedeclarableResult(ASTReader &Reader, GlobalDeclID FirstID)
@@ -162,7 +163,7 @@ namespace clang {
       NamedDecl *Existing;
       mutable bool AddResult;
       
-      FindExistingResult &operator=(FindExistingResult&); // DO NOT IMPLEMENT
+      void operator=(FindExistingResult&) LLVM_DELETED_FUNCTION;
       
     public:
       FindExistingResult(ASTReader &Reader)
@@ -194,16 +195,19 @@ namespace clang {
     
   public:
     ASTDeclReader(ASTReader &Reader, ModuleFile &F,
-                  llvm::BitstreamCursor &Cursor, DeclID thisDeclID,
+                  DeclID thisDeclID,
                   unsigned RawLocation,
                   const RecordData &Record, unsigned &Idx)
-      : Reader(Reader), F(F), Cursor(Cursor), ThisDeclID(thisDeclID),
+      : Reader(Reader), F(F), ThisDeclID(thisDeclID),
         RawLocation(RawLocation), Record(Record), Idx(Idx),
-        TypeIDForTypeDecl(0) { }
+        TypeIDForTypeDecl(0), HasPendingBody(false) { }
 
     static void attachPreviousDecl(Decl *D, Decl *previous);
     static void attachLatestDecl(Decl *D, Decl *latest);
 
+    /// \brief Determine whether this declaration has a pending body.
+    bool hasPendingBody() const { return HasPendingBody; }
+
     void Visit(Decl *D);
 
     void UpdateDecl(Decl *D, ModuleFile &ModuleFile,
@@ -321,8 +325,14 @@ void ASTDeclReader::Visit(Decl *D) {
     ID->TypeForDecl = Reader.GetType(TypeIDForTypeDecl).getTypePtrOrNull();
   } else if (FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
     // FunctionDecl's body was written last after all other Stmts/Exprs.
-    if (Record[Idx++])
-      FD->setLazyBody(GetCurrentCursorOffset());
+    // We only read it if FD doesn't already have a body (e.g., from another
+    // module).
+    // FIXME: Also consider = default and = delete.
+    // FIXME: Can we diagnose ODR violations somehow?
+    if (Record[Idx++]) {
+      Reader.PendingBodies[FD] = GetCurrentCursorOffset();
+      HasPendingBody = true;
+    }
   } else if (D->isTemplateParameter()) {
     // If we have a fully initialized template parameter, we can now
     // set its DeclContext.
@@ -590,8 +600,10 @@ void ASTDeclReader::VisitFunctionDecl(FunctionDecl *FD) {
                                                   TemplArgs.size(), C);
       void *InsertPos = 0;
       CanonTemplate->getSpecializations().FindNodeOrInsertPos(ID, InsertPos);
-      assert(InsertPos && "Another specialization already inserted!");
-      CanonTemplate->getSpecializations().InsertNode(FTInfo, InsertPos);
+      if (InsertPos)
+        CanonTemplate->getSpecializations().InsertNode(FTInfo, InsertPos);
+      else
+        assert(0 && "Another specialization already inserted!");
     }
     break;
   }
@@ -628,19 +640,16 @@ void ASTDeclReader::VisitFunctionDecl(FunctionDecl *FD) {
 void ASTDeclReader::VisitObjCMethodDecl(ObjCMethodDecl *MD) {
   VisitNamedDecl(MD);
   if (Record[Idx++]) {
-    // In practice, this won't be executed (since method definitions
-    // don't occur in header files).
-    // Switch case IDs for this method body.
-    ASTReader::SwitchCaseMapTy SwitchCaseStmtsForObjCMethod;
-    SaveAndRestore<ASTReader::SwitchCaseMapTy *>
-      SCFOM(Reader.CurrSwitchCaseStmts, &SwitchCaseStmtsForObjCMethod);
-    MD->setBody(Reader.ReadStmt(F));
+    // Load the body on-demand. Most clients won't care, because method
+    // definitions rarely show up in headers.
+    Reader.PendingBodies[MD] = GetCurrentCursorOffset();
+    HasPendingBody = true;
     MD->setSelfDecl(ReadDeclAs<ImplicitParamDecl>(Record, Idx));
     MD->setCmdDecl(ReadDeclAs<ImplicitParamDecl>(Record, Idx));
   }
   MD->setInstanceMethod(Record[Idx++]);
   MD->setVariadic(Record[Idx++]);
-  MD->setSynthesized(Record[Idx++]);
+  MD->setPropertyAccessor(Record[Idx++]);
   MD->setDefined(Record[Idx++]);
   MD->IsOverriding = Record[Idx++];
 
@@ -846,6 +855,8 @@ void ASTDeclReader::VisitObjCImplementationDecl(ObjCImplementationDecl *D) {
   D->setSuperClass(ReadDeclAs<ObjCInterfaceDecl>(Record, Idx));
   D->setIvarLBraceLoc(ReadSourceLocation(Record, Idx));
   D->setIvarRBraceLoc(ReadSourceLocation(Record, Idx));
+  D->setHasNonZeroConstructors(Record[Idx++]);
+  D->setHasDestructors(Record[Idx++]);
   llvm::tie(D->IvarInitializers, D->NumIvarInitializers)
       = Reader.ReadCXXCtorInitializers(F, Record, Idx);
 }
@@ -897,7 +908,8 @@ void ASTDeclReader::VisitVarDecl(VarDecl *VD) {
   VD->VarDeclBits.NRVOVariable = Record[Idx++];
   VD->VarDeclBits.CXXForRangeDecl = Record[Idx++];
   VD->VarDeclBits.ARCPseudoStrong = Record[Idx++];
-
+  VD->VarDeclBits.IsConstexpr = Record[Idx++];
+  
   // Only true variables (not parameters or implicit parameters) can be merged.
   if (VD->getKind() == Decl::Var)
     mergeRedeclarable(VD, Redecl);
@@ -1135,6 +1147,7 @@ void ASTDeclReader::ReadCXXDefinitionData(
     Lambda.Captures 
       = (Capture*)Reader.Context.Allocate(sizeof(Capture)*Lambda.NumCaptures);
     Capture *ToCapture = Lambda.Captures;
+    Lambda.MethodTyInfo = GetTypeSourceInfo(Record, Idx);
     for (unsigned I = 0, N = Lambda.NumCaptures; I != N; ++I) {
       SourceLocation Loc = ReadSourceLocation(Record, Idx);
       bool IsImplicit = Record[Idx++];
@@ -1155,7 +1168,8 @@ void ASTDeclReader::VisitCXXRecordDecl(CXXRecordDecl *D) {
     // allocate the appropriate DefinitionData structure.
     bool IsLambda = Record[Idx++];
     if (IsLambda)
-      D->DefinitionData = new (C) CXXRecordDecl::LambdaDefinitionData(D, false);
+      D->DefinitionData = new (C) CXXRecordDecl::LambdaDefinitionData(D, 0,
+                                                                      false);
     else
       D->DefinitionData = new (C) struct CXXRecordDecl::DefinitionData(D);
     
@@ -1242,6 +1256,7 @@ void ASTDeclReader::VisitImportDecl(ImportDecl *D) {
   SourceLocation *StoredLocs = reinterpret_cast<SourceLocation *>(D + 1);
   for (unsigned I = 0, N = Record.back(); I != N; ++I)
     StoredLocs[I] = ReadSourceLocation(Record, Idx);
+  ++Idx; // The number of stored source locations.
 }
 
 void ASTDeclReader::VisitAccessSpecDecl(AccessSpecDecl *D) {
@@ -1308,10 +1323,12 @@ ASTDeclReader::VisitRedeclarableTemplateDecl(RedeclarableTemplateDecl *D) {
         D->setMemberSpecialization();
     }
   }
-     
+
   VisitTemplateDecl(D);
   D->IdentifierNamespace = Record[Idx++];
-  
+
+  mergeRedeclarable(D, Redecl);
+
   return Redecl;
 }
 
@@ -1336,10 +1353,10 @@ void ASTDeclReader::VisitClassTemplateDecl(ClassTemplateDecl *D) {
     for (unsigned I = 0; I != Size; ++I)
       SpecIDs.push_back(ReadDeclID(Record, Idx));
 
+    ClassTemplateDecl::Common *CommonPtr = D->getCommonPtr();
     if (SpecIDs[0]) {
       typedef serialization::DeclID DeclID;
       
-      ClassTemplateDecl::Common *CommonPtr = D->getCommonPtr();
       // FIXME: Append specializations!
       CommonPtr->LazySpecializations
         = new (Reader.getContext()) DeclID [SpecIDs.size()];
@@ -1347,7 +1364,7 @@ void ASTDeclReader::VisitClassTemplateDecl(ClassTemplateDecl *D) {
              SpecIDs.size() * sizeof(DeclID));
     }
     
-    // InjectedClassNameType is computed.
+    CommonPtr->InjectedClassNameType = Reader.readType(F, Record, Idx);
   }
 }
 
@@ -1391,14 +1408,17 @@ void ASTDeclReader::VisitClassTemplateSpecializationDecl(
                                                      TemplArgs.size());
   D->PointOfInstantiation = ReadSourceLocation(Record, Idx);
   D->SpecializationKind = (TemplateSpecializationKind)Record[Idx++];
-  
-  if (D->isCanonicalDecl()) { // It's kept in the folding set.
+
+  bool writtenAsCanonicalDecl = Record[Idx++];
+  if (writtenAsCanonicalDecl) {
     ClassTemplateDecl *CanonPattern = ReadDeclAs<ClassTemplateDecl>(Record,Idx);
-    if (ClassTemplatePartialSpecializationDecl *Partial
-                       = dyn_cast<ClassTemplatePartialSpecializationDecl>(D)) {
-      CanonPattern->getCommonPtr()->PartialSpecializations.InsertNode(Partial);
-    } else {
-      CanonPattern->getCommonPtr()->Specializations.InsertNode(D);
+    if (D->isCanonicalDecl()) { // It's kept in the folding set.
+      if (ClassTemplatePartialSpecializationDecl *Partial
+                        = dyn_cast<ClassTemplatePartialSpecializationDecl>(D)) {
+  CanonPattern->getCommonPtr()->PartialSpecializations.GetOrInsertNode(Partial);
+      } else {
+        CanonPattern->getCommonPtr()->Specializations.GetOrInsertNode(D);
+      }
     }
   }
 }
@@ -1486,11 +1506,18 @@ void ASTDeclReader::VisitTemplateTemplateParmDecl(TemplateTemplateParmDecl *D) {
   // TemplateParmPosition.
   D->setDepth(Record[Idx++]);
   D->setPosition(Record[Idx++]);
-  // Rest of TemplateTemplateParmDecl.
-  TemplateArgumentLoc Arg = Reader.ReadTemplateArgumentLoc(F, Record, Idx);
-  bool IsInherited = Record[Idx++];
-  D->setDefaultArgument(Arg, IsInherited);
-  D->ParameterPack = Record[Idx++];
+  if (D->isExpandedParameterPack()) {
+    void **Data = reinterpret_cast<void **>(D + 1);
+    for (unsigned I = 0, N = D->getNumExpansionTemplateParameters();
+         I != N; ++I)
+      Data[I] = Reader.ReadTemplateParameterList(F, Record, Idx);
+  } else {
+    // Rest of TemplateTemplateParmDecl.
+    TemplateArgumentLoc Arg = Reader.ReadTemplateArgumentLoc(F, Record, Idx);
+    bool IsInherited = Record[Idx++];
+    D->setDefaultArgument(Arg, IsInherited);
+    D->ParameterPack = Record[Idx++];
+  }
 }
 
 void ASTDeclReader::VisitTypeAliasTemplateDecl(TypeAliasTemplateDecl *D) {
@@ -1640,7 +1667,7 @@ inline void ASTReader::LoadedDecl(unsigned Index, Decl *D) {
 /// This routine should return true for anything that might affect
 /// code generation, e.g., inline function definitions, Objective-C
 /// declarations with metadata, etc.
-static bool isConsumerInterestedIn(Decl *D) {
+static bool isConsumerInterestedIn(Decl *D, bool HasBody) {
   // An ObjCMethodDecl is never considered as "interesting" because its
   // implementation container always is.
 
@@ -1652,7 +1679,7 @@ static bool isConsumerInterestedIn(Decl *D) {
     return Var->isFileVarDecl() &&
            Var->isThisDeclarationADefinition() == VarDecl::Definition;
   if (FunctionDecl *Func = dyn_cast<FunctionDecl>(D))
-    return Func->doesThisDeclarationHaveABody();
+    return Func->doesThisDeclarationHaveABody() || HasBody;
   
   return false;
 }
@@ -1719,8 +1746,10 @@ static bool isSameEntity(NamedDecl *X, NamedDecl *Y) {
   if (TagDecl *TagX = dyn_cast<TagDecl>(X)) {
     TagDecl *TagY = cast<TagDecl>(Y);
     return (TagX->getTagKind() == TagY->getTagKind()) ||
-      ((TagX->getTagKind() == TTK_Struct || TagX->getTagKind() == TTK_Class) &&
-       (TagY->getTagKind() == TTK_Struct || TagY->getTagKind() == TTK_Class));
+      ((TagX->getTagKind() == TTK_Struct || TagX->getTagKind() == TTK_Class ||
+        TagX->getTagKind() == TTK_Interface) &&
+       (TagY->getTagKind() == TTK_Struct || TagY->getTagKind() == TTK_Class ||
+        TagY->getTagKind() == TTK_Interface));
   }
   
   // Functions with the same type and linkage match.
@@ -1731,7 +1760,7 @@ static bool isSameEntity(NamedDecl *X, NamedDecl *Y) {
     return (FuncX->getLinkage() == FuncY->getLinkage()) &&
       FuncX->getASTContext().hasSameType(FuncX->getType(), FuncY->getType());
   }
-  
+
   // Variables with the same type and linkage match.
   if (VarDecl *VarX = dyn_cast<VarDecl>(X)) {
     VarDecl *VarY = cast<VarDecl>(Y);
@@ -1744,7 +1773,11 @@ static bool isSameEntity(NamedDecl *X, NamedDecl *Y) {
     NamespaceDecl *NamespaceY = cast<NamespaceDecl>(Y);
     return NamespaceX->isInline() == NamespaceY->isInline();
   }
-      
+
+  // Identical template names and kinds match.
+  if (isa<TemplateDecl>(X))
+    return true;
+
   // FIXME: Many other cases to implement.
   return false;
 }
@@ -1753,11 +1786,13 @@ ASTDeclReader::FindExistingResult::~FindExistingResult() {
   if (!AddResult || Existing)
     return;
   
-  DeclContext *DC = New->getDeclContext()->getRedeclContext();
-  if (DC->isTranslationUnit() && Reader.SemaObj) {
+  if (New->getDeclContext()->getRedeclContext()->isTranslationUnit()
+      && Reader.SemaObj) {
     Reader.SemaObj->IdResolver.tryAddTopLevelDecl(New, New->getDeclName());
-  } else if (DC->isNamespace()) {
-    DC->addDecl(New);
+  } else {
+    DeclContext *DC = New->getLexicalDeclContext();
+    if (DC->isNamespace())
+      DC->addDecl(New);
   }
 }
 
@@ -1899,7 +1934,7 @@ Decl *ASTReader::ReadDeclRecord(DeclID ID) {
   RecordData Record;
   unsigned Code = DeclsCursor.ReadCode();
   unsigned Idx = 0;
-  ASTDeclReader Reader(*this, *Loc.F, DeclsCursor, ID, RawLocation, Record,Idx);
+  ASTDeclReader Reader(*this, *Loc.F, ID, RawLocation, Record,Idx);
 
   Decl *D = 0;
   switch ((DeclCode)DeclsCursor.ReadRecord(Code, Record)) {
@@ -2002,6 +2037,10 @@ Decl *ASTReader::ReadDeclRecord(DeclID ID) {
   case DECL_TEMPLATE_TEMPLATE_PARM:
     D = TemplateTemplateParmDecl::CreateDeserialized(Context, ID);
     break;
+  case DECL_EXPANDED_TEMPLATE_TEMPLATE_PARM_PACK:
+    D = TemplateTemplateParmDecl::CreateDeserialized(Context, ID,
+                                                     Record[Idx++]);
+    break;
   case DECL_TYPE_ALIAS_TEMPLATE:
     D = TypeAliasTemplateDecl::CreateDeserialized(Context, ID);
     break;
@@ -2110,6 +2149,9 @@ Decl *ASTReader::ReadDeclRecord(DeclID ID) {
       }
       PendingVisibleUpdates.erase(I);
     }
+
+    if (!DC->hasExternalVisibleStorage() && DC->hasExternalLexicalStorage())
+      DC->setMustBuildLookupTable();
   }
   assert(Idx == Record.size());
 
@@ -2125,9 +2167,9 @@ Decl *ASTReader::ReadDeclRecord(DeclID ID) {
   // AST consumer might need to know about, queue it.
   // We don't pass it to the consumer immediately because we may be in recursive
   // loading, and some declarations may still be initializing.
-  if (isConsumerInterestedIn(D))
+  if (isConsumerInterestedIn(D, Reader.hasPendingBody()))
     InterestingDecls.push_back(D);
-  
+
   return D;
 }
 
@@ -2152,7 +2194,7 @@ void ASTReader::loadDeclUpdateRecords(serialization::DeclID ID, Decl *D) {
       assert(RecCode == DECL_UPDATES && "Expected DECL_UPDATES record!");
       
       unsigned Idx = 0;
-      ASTDeclReader Reader(*this, *F, Cursor, ID, 0, Record, Idx);
+      ASTDeclReader Reader(*this, *F, ID, 0, Record, Idx);
       Reader.UpdateDecl(D, *F, Record);
     }
   }
@@ -2207,10 +2249,8 @@ namespace {
       if (!D)
         return;
       
-      if (Deserialized.count(D)) {
-        Deserialized.erase(D);
+      if (Deserialized.erase(D))
         Chain.push_back(D);
-      }
     }
     
     void searchForID(ModuleFile &M, GlobalDeclID GlobalID) {
@@ -2275,7 +2315,7 @@ void ASTReader::loadPendingDeclChain(serialization::GlobalDeclID ID) {
   }
   MergedDeclsMap::iterator MergedPos = combineStoredMergedDecls(CanonDecl, ID);
   if (MergedPos != MergedDecls.end())
-    SearchDecls.append(MergedPos->second.begin(), MergedPos->second.end());  
+    SearchDecls.append(MergedPos->second.begin(), MergedPos->second.end());
   
   // Build up the list of redeclarations.
   RedeclChainVisitor Visitor(*this, SearchDecls, RedeclsDeserialized, CanonID);
@@ -2331,9 +2371,8 @@ namespace {
     
     void add(ObjCCategoryDecl *Cat) {
       // Only process each category once.
-      if (!Deserialized.count(Cat))
+      if (!Deserialized.erase(Cat))
         return;
-      Deserialized.erase(Cat);
       
       // Check for duplicate categories.
       if (Cat->getDeclName()) {
diff --git a/lib/Serialization/ASTReaderStmt.cpp b/lib/Serialization/ASTReaderStmt.cpp
index c5325b5..367f75f 100644
--- a/lib/Serialization/ASTReaderStmt.cpp
+++ b/lib/Serialization/ASTReaderStmt.cpp
@@ -288,7 +288,7 @@ void ASTStmtReader::VisitDeclStmt(DeclStmt *S) {
   }
 }
 
-void ASTStmtReader::VisitAsmStmt(AsmStmt *S) {
+void ASTStmtReader::VisitGCCAsmStmt(GCCAsmStmt *S) {
   VisitStmt(S);
   unsigned NumOutputs = Record[Idx++];
   unsigned NumInputs = Record[Idx++];
@@ -297,7 +297,6 @@ void ASTStmtReader::VisitAsmStmt(AsmStmt *S) {
   S->setRParenLoc(ReadSourceLocation(Record, Idx));
   S->setVolatile(Record[Idx++]);
   S->setSimple(Record[Idx++]);
-  S->setMSAsm(Record[Idx++]);
 
   S->setAsmString(cast_or_null<StringLiteral>(Reader.ReadSubStmt()));
 
@@ -566,6 +565,7 @@ void ASTStmtReader::VisitBinaryOperator(BinaryOperator *E) {
   E->setRHS(Reader.ReadSubExpr());
   E->setOpcode((BinaryOperator::Opcode)Record[Idx++]);
   E->setOperatorLoc(ReadSourceLocation(Record, Idx));
+  E->setFPContractable((bool)Record[Idx++]);
 }
 
 void ASTStmtReader::VisitCompoundAssignOperator(CompoundAssignOperator *E) {
@@ -627,7 +627,8 @@ void ASTStmtReader::VisitExtVectorElementExpr(ExtVectorElementExpr *E) {
 
 void ASTStmtReader::VisitInitListExpr(InitListExpr *E) {
   VisitExpr(E);
-  E->setSyntacticForm(cast_or_null<InitListExpr>(Reader.ReadSubStmt()));
+  if (InitListExpr *SyntForm = cast_or_null<InitListExpr>(Reader.ReadSubStmt()))
+    E->setSyntacticForm(SyntForm);
   E->setLBraceLoc(ReadSourceLocation(Record, Idx));
   E->setRBraceLoc(ReadSourceLocation(Record, Idx));
   bool isArrayFiller = Record[Idx++];
@@ -1087,6 +1088,7 @@ void ASTStmtReader::VisitCXXOperatorCallExpr(CXXOperatorCallExpr *E) {
   VisitCallExpr(E);
   E->Operator = (OverloadedOperatorKind)Record[Idx++];
   E->Range = Reader.ReadSourceRange(F, Record, Idx);
+  E->setFPContractable((bool)Record[Idx++]);
 }
 
 void ASTStmtReader::VisitCXXConstructExpr(CXXConstructExpr *E) {
@@ -1242,7 +1244,7 @@ void ASTStmtReader::VisitCXXNewExpr(CXXNewExpr *E) {
   E->setOperatorDelete(ReadDeclAs<FunctionDecl>(Record, Idx));
   E->AllocatedTypeInfo = GetTypeSourceInfo(Record, Idx);
   E->TypeIdParens = ReadSourceRange(Record, Idx);
-  E->StartLoc = ReadSourceLocation(Record, Idx);
+  E->Range = ReadSourceRange(Record, Idx);
   E->DirectInitRange = ReadSourceRange(Record, Idx);
 
   E->AllocateArgsArray(Reader.getContext(), isArray, NumPlacementArgs,
@@ -1367,8 +1369,6 @@ void ASTStmtReader::VisitUnresolvedMemberExpr(UnresolvedMemberExpr *E) {
 void ASTStmtReader::VisitUnresolvedLookupExpr(UnresolvedLookupExpr *E) {
   VisitOverloadExpr(E);
   E->RequiresADL = Record[Idx++];
-  if (E->RequiresADL)
-    E->StdIsAssociatedNamespace = Record[Idx++];
   E->Overloaded = Record[Idx++];
   E->NamingClass = ReadDeclAs<CXXRecordDecl>(Record, Idx);
 }
@@ -1469,6 +1469,16 @@ void ASTStmtReader::VisitSubstNonTypeTemplateParmPackExpr(
   E->NameLoc = ReadSourceLocation(Record, Idx);
 }
 
+void ASTStmtReader::VisitFunctionParmPackExpr(FunctionParmPackExpr *E) {
+  VisitExpr(E);
+  E->NumParameters = Record[Idx++];
+  E->ParamPack = ReadDeclAs<ParmVarDecl>(Record, Idx);
+  E->NameLoc = ReadSourceLocation(Record, Idx);
+  ParmVarDecl **Parms = reinterpret_cast<ParmVarDecl**>(E+1);
+  for (unsigned i = 0, n = E->NumParameters; i != n; ++i)
+    Parms[i] = ReadDeclAs<ParmVarDecl>(Record, Idx);
+}
+
 void ASTStmtReader::VisitMaterializeTemporaryExpr(MaterializeTemporaryExpr *E) {
   VisitExpr(E);
   E->Temporary = Reader.ReadSubExpr();
@@ -1701,8 +1711,12 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) {
       S = new (Context) DeclStmt(Empty);
       break;
 
-    case STMT_ASM:
-      S = new (Context) AsmStmt(Empty);
+    case STMT_GCCASM:
+      S = new (Context) GCCAsmStmt(Empty);
+      break;
+
+    case STMT_MSASM:
+      S = new (Context) MSAsmStmt(Empty);
       break;
 
     case EXPR_PREDEFINED:
@@ -2180,6 +2194,11 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) {
     case EXPR_SUBST_NON_TYPE_TEMPLATE_PARM_PACK:
       S = new (Context) SubstNonTypeTemplateParmPackExpr(Empty);
       break;
+
+    case EXPR_FUNCTION_PARM_PACK:
+      S = FunctionParmPackExpr::CreateEmpty(Context,
+                                          Record[ASTStmtReader::NumExprFields]);
+      break;
         
     case EXPR_MATERIALIZE_TEMPORARY:
       S = new (Context) MaterializeTemporaryExpr(Empty);
diff --git a/lib/Serialization/ASTWriter.cpp b/lib/Serialization/ASTWriter.cpp
index 425d2e3..a2e8b71 100644
--- a/lib/Serialization/ASTWriter.cpp
+++ b/lib/Serialization/ASTWriter.cpp
@@ -25,9 +25,11 @@
 #include "clang/AST/Type.h"
 #include "clang/AST/TypeLocVisitor.h"
 #include "clang/Serialization/ASTReader.h"
+#include "clang/Lex/HeaderSearchOptions.h"
 #include "clang/Lex/MacroInfo.h"
 #include "clang/Lex/PreprocessingRecord.h"
 #include "clang/Lex/Preprocessor.h"
+#include "clang/Lex/PreprocessorOptions.h"
 #include "clang/Lex/HeaderSearch.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/FileSystemStatCache.h"
@@ -35,6 +37,7 @@
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/SourceManagerInternals.h"
 #include "clang/Basic/TargetInfo.h"
+#include "clang/Basic/TargetOptions.h"
 #include "clang/Basic/Version.h"
 #include "clang/Basic/VersionTuple.h"
 #include "llvm/ADT/APFloat.h"
@@ -485,6 +488,8 @@ void TypeLocWriter::VisitExtVectorTypeLoc(ExtVectorTypeLoc TL) {
 }
 void TypeLocWriter::VisitFunctionTypeLoc(FunctionTypeLoc TL) {
   Writer.AddSourceLocation(TL.getLocalRangeBegin(), Record);
+  Writer.AddSourceLocation(TL.getLParenLoc(), Record);
+  Writer.AddSourceLocation(TL.getRParenLoc(), Record);
   Writer.AddSourceLocation(TL.getLocalRangeEnd(), Record);
   for (unsigned i = 0, e = TL.getNumArgs(); i != e; ++i)
     Writer.AddDeclRef(TL.getArg(i), Record);
@@ -667,7 +672,8 @@ static void AddStmtsExprs(llvm::BitstreamWriter &Stream,
   RECORD(STMT_BREAK);
   RECORD(STMT_RETURN);
   RECORD(STMT_DECL);
-  RECORD(STMT_ASM);
+  RECORD(STMT_GCCASM);
+  RECORD(STMT_MSASM);
   RECORD(EXPR_PREDEFINED);
   RECORD(EXPR_DECL_REF);
   RECORD(EXPR_INTEGER_LITERAL);
@@ -763,14 +769,27 @@ void ASTWriter::WriteBlockInfoBlock() {
 #define BLOCK(X) EmitBlockID(X ## _ID, #X, Stream, Record)
 #define RECORD(X) EmitRecordID(X, #X, Stream, Record)
 
+  // Control Block.
+  BLOCK(CONTROL_BLOCK);
+  RECORD(METADATA);
+  RECORD(IMPORTS);
+  RECORD(LANGUAGE_OPTIONS);
+  RECORD(TARGET_OPTIONS);
+  RECORD(ORIGINAL_FILE);
+  RECORD(ORIGINAL_PCH_DIR);
+  RECORD(INPUT_FILE_OFFSETS);
+  RECORD(DIAGNOSTIC_OPTIONS);
+  RECORD(FILE_SYSTEM_OPTIONS);
+  RECORD(HEADER_SEARCH_OPTIONS);
+  RECORD(PREPROCESSOR_OPTIONS);
+
+  BLOCK(INPUT_FILES_BLOCK);
+  RECORD(INPUT_FILE);
+
   // AST Top-Level Block.
   BLOCK(AST_BLOCK);
-  RECORD(ORIGINAL_FILE_NAME);
-  RECORD(ORIGINAL_FILE_ID);
   RECORD(TYPE_OFFSET);
   RECORD(DECL_OFFSET);
-  RECORD(LANGUAGE_OPTIONS);
-  RECORD(METADATA);
   RECORD(IDENTIFIER_OFFSET);
   RECORD(IDENTIFIER_TABLE);
   RECORD(EXTERNAL_DEFINITIONS);
@@ -784,11 +803,8 @@ void ASTWriter::WriteBlockInfoBlock() {
   RECORD(PP_COUNTER_VALUE);
   RECORD(SOURCE_LOCATION_OFFSETS);
   RECORD(SOURCE_LOCATION_PRELOADS);
-  RECORD(STAT_CACHE);
   RECORD(EXT_VECTOR_DECLS);
-  RECORD(VERSION_CONTROL_BRANCH_REVISION);
   RECORD(PPD_ENTITIES_OFFSETS);
-  RECORD(IMPORTS);
   RECORD(REFERENCED_SELECTOR_POOL);
   RECORD(TU_UPDATE_LEXICAL);
   RECORD(LOCAL_REDECLARATIONS_MAP);
@@ -803,11 +819,9 @@ void ASTWriter::WriteBlockInfoBlock() {
   RECORD(DIAG_PRAGMA_MAPPINGS);
   RECORD(CUDA_SPECIAL_DECL_REFS);
   RECORD(HEADER_SEARCH_TABLE);
-  RECORD(ORIGINAL_PCH_DIR);
   RECORD(FP_PRAGMA_OPTIONS);
   RECORD(OPENCL_EXTENSIONS);
   RECORD(DELEGATING_CTORS);
-  RECORD(FILE_SOURCE_LOCATION_OFFSETS);
   RECORD(KNOWN_NAMESPACES);
   RECORD(MODULE_OFFSET_MAP);
   RECORD(SOURCE_MANAGER_LINE_TABLE);
@@ -817,6 +831,8 @@ void ASTWriter::WriteBlockInfoBlock() {
   RECORD(MERGED_DECLARATIONS);
   RECORD(LOCAL_REDECLARATIONS);
   RECORD(OBJC_CATEGORIES);
+  RECORD(MACRO_OFFSET);
+  RECORD(MACRO_UPDATES);
 
   // SourceManager Block.
   BLOCK(SOURCE_MANAGER_BLOCK);
@@ -972,25 +988,25 @@ adjustFilenameForRelocatablePCH(const char *Filename, StringRef isysroot) {
   return Filename + Pos;
 }
 
-/// \brief Write the AST metadata (e.g., i686-apple-darwin9).
-void ASTWriter::WriteMetadata(ASTContext &Context, StringRef isysroot,
-                              const std::string &OutputFile) {
+/// \brief Write the control block.
+void ASTWriter::WriteControlBlock(Preprocessor &PP, ASTContext &Context,
+                                  StringRef isysroot,
+                                  const std::string &OutputFile) {
   using namespace llvm;
-
-  // Metadata
-  const TargetInfo &Target = Context.getTargetInfo();
-  BitCodeAbbrev *MetaAbbrev = new BitCodeAbbrev();
-  MetaAbbrev->Add(BitCodeAbbrevOp(METADATA));
-  MetaAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 16)); // AST major
-  MetaAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 16)); // AST minor
-  MetaAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 16)); // Clang major
-  MetaAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 16)); // Clang minor
-  MetaAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // Relocatable
-  MetaAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // Has errors
-  MetaAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // Target triple
-  unsigned MetaAbbrevCode = Stream.EmitAbbrev(MetaAbbrev);
-
+  Stream.EnterSubblock(CONTROL_BLOCK_ID, 5);
   RecordData Record;
+  
+  // Metadata
+  BitCodeAbbrev *MetadataAbbrev = new BitCodeAbbrev();
+  MetadataAbbrev->Add(BitCodeAbbrevOp(METADATA));
+  MetadataAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 16)); // Major
+  MetadataAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 16)); // Minor
+  MetadataAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 16)); // Clang maj.
+  MetadataAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 16)); // Clang min.
+  MetadataAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // Relocatable
+  MetadataAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // Errors
+  MetadataAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // SVN branch/tag
+  unsigned MetadataAbbrevCode = Stream.EmitAbbrev(MetadataAbbrev);
   Record.push_back(METADATA);
   Record.push_back(VERSION_MAJOR);
   Record.push_back(VERSION_MINOR);
@@ -998,9 +1014,10 @@ void ASTWriter::WriteMetadata(ASTContext &Context, StringRef isysroot,
   Record.push_back(CLANG_VERSION_MINOR);
   Record.push_back(!isysroot.empty());
   Record.push_back(ASTHasCompilerErrors);
-  const std::string &Triple = Target.getTriple().getTriple();
-  Stream.EmitRecordWithBlob(MetaAbbrevCode, Record, Triple);
+  Stream.EmitRecordWithBlob(MetadataAbbrevCode, Record,
+                            getClangFullRepositoryVersion());
 
+  // Imports
   if (Chain) {
     serialization::ModuleManager &Mgr = Chain->getModuleManager();
     llvm::SmallVector<char, 128> ModulePaths;
@@ -1022,11 +1039,131 @@ void ASTWriter::WriteMetadata(ASTContext &Context, StringRef isysroot,
     Stream.EmitRecord(IMPORTS, Record);
   }
 
+  // Language options.
+  Record.clear();
+  const LangOptions &LangOpts = Context.getLangOpts();
+#define LANGOPT(Name, Bits, Default, Description) \
+  Record.push_back(LangOpts.Name);
+#define ENUM_LANGOPT(Name, Type, Bits, Default, Description) \
+  Record.push_back(static_cast<unsigned>(LangOpts.get##Name()));
+#include "clang/Basic/LangOptions.def"  
+
+  Record.push_back((unsigned) LangOpts.ObjCRuntime.getKind());
+  AddVersionTuple(LangOpts.ObjCRuntime.getVersion(), Record);
+  
+  Record.push_back(LangOpts.CurrentModule.size());
+  Record.append(LangOpts.CurrentModule.begin(), LangOpts.CurrentModule.end());
+  Stream.EmitRecord(LANGUAGE_OPTIONS, Record);
+
+  // Target options.
+  Record.clear();
+  const TargetInfo &Target = Context.getTargetInfo();
+  const TargetOptions &TargetOpts = Target.getTargetOpts();
+  AddString(TargetOpts.Triple, Record);
+  AddString(TargetOpts.CPU, Record);
+  AddString(TargetOpts.ABI, Record);
+  AddString(TargetOpts.CXXABI, Record);
+  AddString(TargetOpts.LinkerVersion, Record);
+  Record.push_back(TargetOpts.FeaturesAsWritten.size());
+  for (unsigned I = 0, N = TargetOpts.FeaturesAsWritten.size(); I != N; ++I) {
+    AddString(TargetOpts.FeaturesAsWritten[I], Record);
+  }
+  Record.push_back(TargetOpts.Features.size());
+  for (unsigned I = 0, N = TargetOpts.Features.size(); I != N; ++I) {
+    AddString(TargetOpts.Features[I], Record);
+  }
+  Stream.EmitRecord(TARGET_OPTIONS, Record);
+
+  // Diagnostic options.
+  Record.clear();
+  const DiagnosticOptions &DiagOpts
+    = Context.getDiagnostics().getDiagnosticOptions();
+#define DIAGOPT(Name, Bits, Default) Record.push_back(DiagOpts.Name);
+#define ENUM_DIAGOPT(Name, Type, Bits, Default) \
+  Record.push_back(static_cast<unsigned>(DiagOpts.get##Name()));
+#include "clang/Basic/DiagnosticOptions.def"
+  Record.push_back(DiagOpts.Warnings.size());
+  for (unsigned I = 0, N = DiagOpts.Warnings.size(); I != N; ++I)
+    AddString(DiagOpts.Warnings[I], Record);
+  // Note: we don't serialize the log or serialization file names, because they
+  // are generally transient files and will almost always be overridden.
+  Stream.EmitRecord(DIAGNOSTIC_OPTIONS, Record);
+
+  // File system options.
+  Record.clear();
+  const FileSystemOptions &FSOpts
+    = Context.getSourceManager().getFileManager().getFileSystemOptions();
+  AddString(FSOpts.WorkingDir, Record);
+  Stream.EmitRecord(FILE_SYSTEM_OPTIONS, Record);
+
+  // Header search options.
+  Record.clear();
+  const HeaderSearchOptions &HSOpts
+    = PP.getHeaderSearchInfo().getHeaderSearchOpts();
+  AddString(HSOpts.Sysroot, Record);
+
+  // Include entries.
+  Record.push_back(HSOpts.UserEntries.size());
+  for (unsigned I = 0, N = HSOpts.UserEntries.size(); I != N; ++I) {
+    const HeaderSearchOptions::Entry &Entry = HSOpts.UserEntries[I];
+    AddString(Entry.Path, Record);
+    Record.push_back(static_cast<unsigned>(Entry.Group));
+    Record.push_back(Entry.IsUserSupplied);
+    Record.push_back(Entry.IsFramework);
+    Record.push_back(Entry.IgnoreSysRoot);
+    Record.push_back(Entry.IsInternal);
+    Record.push_back(Entry.ImplicitExternC);
+  }
+
+  // System header prefixes.
+  Record.push_back(HSOpts.SystemHeaderPrefixes.size());
+  for (unsigned I = 0, N = HSOpts.SystemHeaderPrefixes.size(); I != N; ++I) {
+    AddString(HSOpts.SystemHeaderPrefixes[I].Prefix, Record);
+    Record.push_back(HSOpts.SystemHeaderPrefixes[I].IsSystemHeader);
+  }
+
+  AddString(HSOpts.ResourceDir, Record);
+  AddString(HSOpts.ModuleCachePath, Record);
+  Record.push_back(HSOpts.DisableModuleHash);
+  Record.push_back(HSOpts.UseBuiltinIncludes);
+  Record.push_back(HSOpts.UseStandardSystemIncludes);
+  Record.push_back(HSOpts.UseStandardCXXIncludes);
+  Record.push_back(HSOpts.UseLibcxx);
+  Stream.EmitRecord(HEADER_SEARCH_OPTIONS, Record);
+
+  // Preprocessor options.
+  Record.clear();
+  const PreprocessorOptions &PPOpts = PP.getPreprocessorOpts();
+
+  // Macro definitions.
+  Record.push_back(PPOpts.Macros.size());
+  for (unsigned I = 0, N = PPOpts.Macros.size(); I != N; ++I) {
+    AddString(PPOpts.Macros[I].first, Record);
+    Record.push_back(PPOpts.Macros[I].second);
+  }
+
+  // Includes
+  Record.push_back(PPOpts.Includes.size());
+  for (unsigned I = 0, N = PPOpts.Includes.size(); I != N; ++I)
+    AddString(PPOpts.Includes[I], Record);
+
+  // Macro includes
+  Record.push_back(PPOpts.MacroIncludes.size());
+  for (unsigned I = 0, N = PPOpts.MacroIncludes.size(); I != N; ++I)
+    AddString(PPOpts.MacroIncludes[I], Record);
+
+  Record.push_back(PPOpts.UsePredefines);
+  AddString(PPOpts.ImplicitPCHInclude, Record);
+  AddString(PPOpts.ImplicitPTHInclude, Record);
+  Record.push_back(static_cast<unsigned>(PPOpts.ObjCXXARCStandardLibrary));
+  Stream.EmitRecord(PREPROCESSOR_OPTIONS, Record);
+
   // Original file name and file ID
   SourceManager &SM = Context.getSourceManager();
   if (const FileEntry *MainFile = SM.getFileEntryForID(SM.getMainFileID())) {
     BitCodeAbbrev *FileAbbrev = new BitCodeAbbrev();
-    FileAbbrev->Add(BitCodeAbbrevOp(ORIGINAL_FILE_NAME));
+    FileAbbrev->Add(BitCodeAbbrevOp(ORIGINAL_FILE));
+    FileAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // File ID
     FileAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // File name
     unsigned FileAbbrevCode = Stream.EmitAbbrev(FileAbbrev);
 
@@ -1037,13 +1174,10 @@ void ASTWriter::WriteMetadata(ASTContext &Context, StringRef isysroot,
     const char *MainFileNameStr = MainFilePath.c_str();
     MainFileNameStr = adjustFilenameForRelocatablePCH(MainFileNameStr,
                                                       isysroot);
-    RecordData Record;
-    Record.push_back(ORIGINAL_FILE_NAME);
-    Stream.EmitRecordWithBlob(FileAbbrevCode, Record, MainFileNameStr);
-    
     Record.clear();
+    Record.push_back(ORIGINAL_FILE);
     Record.push_back(SM.getMainFileID().getOpaqueValue());
-    Stream.EmitRecord(ORIGINAL_FILE_ID, Record);
+    Stream.EmitRecordWithBlob(FileAbbrevCode, Record, MainFileNameStr);
   }
 
   // Original PCH directory
@@ -1063,32 +1197,86 @@ void ASTWriter::WriteMetadata(ASTContext &Context, StringRef isysroot,
     Stream.EmitRecordWithBlob(AbbrevCode, Record, origDir);
   }
 
-  // Repository branch/version information.
-  BitCodeAbbrev *RepoAbbrev = new BitCodeAbbrev();
-  RepoAbbrev->Add(BitCodeAbbrevOp(VERSION_CONTROL_BRANCH_REVISION));
-  RepoAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // SVN branch/tag
-  unsigned RepoAbbrevCode = Stream.EmitAbbrev(RepoAbbrev);
-  Record.clear();
-  Record.push_back(VERSION_CONTROL_BRANCH_REVISION);
-  Stream.EmitRecordWithBlob(RepoAbbrevCode, Record,
-                            getClangFullRepositoryVersion());
+  WriteInputFiles(Context.SourceMgr, isysroot);
+  Stream.ExitBlock();
 }
 
-/// \brief Write the LangOptions structure.
-void ASTWriter::WriteLanguageOptions(const LangOptions &LangOpts) {
+void ASTWriter::WriteInputFiles(SourceManager &SourceMgr, StringRef isysroot) {
+  using namespace llvm;
+  Stream.EnterSubblock(INPUT_FILES_BLOCK_ID, 4);
   RecordData Record;
-#define LANGOPT(Name, Bits, Default, Description) \
-  Record.push_back(LangOpts.Name);
-#define ENUM_LANGOPT(Name, Type, Bits, Default, Description) \
-  Record.push_back(static_cast<unsigned>(LangOpts.get##Name()));
-#include "clang/Basic/LangOptions.def"  
+  
+  // Create input-file abbreviation.
+  BitCodeAbbrev *IFAbbrev = new BitCodeAbbrev();
+  IFAbbrev->Add(BitCodeAbbrevOp(INPUT_FILE));
+  IFAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // ID
+  IFAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 12)); // Size
+  IFAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 32)); // Modification time
+  IFAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // Overridden
+  IFAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // File name
+  unsigned IFAbbrevCode = Stream.EmitAbbrev(IFAbbrev);
+
+  // Write out all of the input files.
+  std::vector<uint32_t> InputFileOffsets;
+  for (unsigned I = 1, N = SourceMgr.local_sloc_entry_size(); I != N; ++I) {
+    // Get this source location entry.
+    const SrcMgr::SLocEntry *SLoc = &SourceMgr.getLocalSLocEntry(I);
+    assert(&SourceMgr.getSLocEntry(FileID::get(I)) == SLoc);
 
-  Record.push_back((unsigned) LangOpts.ObjCRuntime.getKind());
-  AddVersionTuple(LangOpts.ObjCRuntime.getVersion(), Record);
+    // We only care about file entries that were not overridden.
+    if (!SLoc->isFile())
+      continue;
+    const SrcMgr::ContentCache *Cache = SLoc->getFile().getContentCache();
+    if (!Cache->OrigEntry)
+      continue;
+
+    // Record this entry's offset.
+    InputFileOffsets.push_back(Stream.GetCurrentBitNo());
+    InputFileIDs[Cache->OrigEntry] = InputFileOffsets.size();
+
+    Record.clear();
+    Record.push_back(INPUT_FILE);
+    Record.push_back(InputFileOffsets.size());
+
+    // Emit size/modification time for this file.
+    Record.push_back(Cache->OrigEntry->getSize());
+    Record.push_back(Cache->OrigEntry->getModificationTime());
+
+    // Whether this file was overridden.
+    Record.push_back(Cache->BufferOverridden);
+
+    // Turn the file name into an absolute path, if it isn't already.
+    const char *Filename = Cache->OrigEntry->getName();
+    SmallString<128> FilePath(Filename);
+    
+    // Ask the file manager to fixup the relative path for us. This will 
+    // honor the working directory.
+    SourceMgr.getFileManager().FixupRelativePath(FilePath);
+    
+    // FIXME: This call to make_absolute shouldn't be necessary, the
+    // call to FixupRelativePath should always return an absolute path.
+    llvm::sys::fs::make_absolute(FilePath);
+    Filename = FilePath.c_str();
+    
+    Filename = adjustFilenameForRelocatablePCH(Filename, isysroot);
+
+    Stream.EmitRecordWithBlob(IFAbbrevCode, Record, Filename);
+  }  
   
-  Record.push_back(LangOpts.CurrentModule.size());
-  Record.append(LangOpts.CurrentModule.begin(), LangOpts.CurrentModule.end());
-  Stream.EmitRecord(LANGUAGE_OPTIONS, Record);
+  Stream.ExitBlock();
+
+  // Create input file offsets abbreviation.
+  BitCodeAbbrev *OffsetsAbbrev = new BitCodeAbbrev();
+  OffsetsAbbrev->Add(BitCodeAbbrevOp(INPUT_FILE_OFFSETS));
+  OffsetsAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // # input files
+  OffsetsAbbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));   // Array
+  unsigned OffsetsAbbrevCode = Stream.EmitAbbrev(OffsetsAbbrev);
+
+  // Write input file offsets.
+  Record.clear();
+  Record.push_back(INPUT_FILE_OFFSETS);
+  Record.push_back(InputFileOffsets.size());
+  Stream.EmitRecordWithBlob(OffsetsAbbrevCode, Record, data(InputFileOffsets));
 }
 
 //===----------------------------------------------------------------------===//
@@ -1139,46 +1327,6 @@ public:
 };
 } // end anonymous namespace
 
-/// \brief Write the stat() system call cache to the AST file.
-void ASTWriter::WriteStatCache(MemorizeStatCalls &StatCalls) {
-  // Build the on-disk hash table containing information about every
-  // stat() call.
-  OnDiskChainedHashTableGenerator<ASTStatCacheTrait> Generator;
-  unsigned NumStatEntries = 0;
-  for (MemorizeStatCalls::iterator Stat = StatCalls.begin(),
-                                StatEnd = StatCalls.end();
-       Stat != StatEnd; ++Stat, ++NumStatEntries) {
-    StringRef Filename = Stat->first();
-    Generator.insert(Filename.data(), Stat->second);
-  }
-
-  // Create the on-disk hash table in a buffer.
-  SmallString<4096> StatCacheData;
-  uint32_t BucketOffset;
-  {
-    llvm::raw_svector_ostream Out(StatCacheData);
-    // Make sure that no bucket is at offset 0
-    clang::io::Emit32(Out, 0);
-    BucketOffset = Generator.Emit(Out);
-  }
-
-  // Create a blob abbreviation
-  using namespace llvm;
-  BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
-  Abbrev->Add(BitCodeAbbrevOp(STAT_CACHE));
-  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
-  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
-  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
-  unsigned StatCacheAbbrev = Stream.EmitAbbrev(Abbrev);
-
-  // Write the stat cache
-  RecordData Record;
-  Record.push_back(STAT_CACHE);
-  Record.push_back(BucketOffset);
-  Record.push_back(NumStatEntries);
-  Stream.EmitRecordWithBlob(StatCacheAbbrev, Record, StatCacheData.str());
-}
-
 //===----------------------------------------------------------------------===//
 // Source Manager Serialization
 //===----------------------------------------------------------------------===//
@@ -1194,13 +1342,10 @@ static unsigned CreateSLocFileAbbrev(llvm::BitstreamWriter &Stream) {
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 2)); // Characteristic
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // Line directives
   // FileEntry fields.
-  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 12)); // Size
-  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 32)); // Modification time
-  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // BufferOverridden
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Input File ID
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // NumCreatedFIDs
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 24)); // FirstDeclIndex
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // NumDecls
-  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // File name
   return Stream.EmitAbbrev(Abbrev);
 }
 
@@ -1329,8 +1474,6 @@ namespace {
 /// \brief Write the header search block for the list of files that 
 ///
 /// \param HS The header search structure to save.
-///
-/// \param Chain Whether we're creating a chained AST file.
 void ASTWriter::WriteHeaderSearch(const HeaderSearch &HS, StringRef isysroot) {
   SmallVector<const FileEntry *, 16> FilesByUID;
   HS.getFileMgr().GetUniqueIDMapping(FilesByUID);
@@ -1427,15 +1570,14 @@ void ASTWriter::WriteSourceManagerBlock(SourceManager &SourceMgr,
   // Write out the source location entry table. We skip the first
   // entry, which is always the same dummy entry.
   std::vector<uint32_t> SLocEntryOffsets;
-  // Write out the offsets of only source location file entries.
-  // We will go through them in ASTReader::validateFileEntries().
-  std::vector<uint32_t> SLocFileEntryOffsets;
   RecordData PreloadSLocs;
   SLocEntryOffsets.reserve(SourceMgr.local_sloc_entry_size() - 1);
   for (unsigned I = 1, N = SourceMgr.local_sloc_entry_size();
        I != N; ++I) {
     // Get this source location entry.
     const SrcMgr::SLocEntry *SLoc = &SourceMgr.getLocalSLocEntry(I);
+    FileID FID = FileID::get(I);
+    assert(&SourceMgr.getSLocEntry(FID) == SLoc);
 
     // Record the offset of this source-location entry.
     SLocEntryOffsets.push_back(Stream.GetCurrentBitNo());
@@ -1446,7 +1588,6 @@ void ASTWriter::WriteSourceManagerBlock(SourceManager &SourceMgr,
       const SrcMgr::ContentCache *Cache = SLoc->getFile().getContentCache();
       if (Cache->OrigEntry) {
         Code = SM_SLOC_FILE_ENTRY;
-        SLocFileEntryOffsets.push_back(Stream.GetCurrentBitNo());
       } else
         Code = SM_SLOC_BUFFER_ENTRY;
     } else
@@ -1467,16 +1608,13 @@ void ASTWriter::WriteSourceManagerBlock(SourceManager &SourceMgr,
         assert(Content->OrigEntry == Content->ContentsEntry &&
                "Writing to AST an overridden file is not supported");
 
-        // The source location entry is a file. The blob associated
-        // with this entry is the file name.
+        // The source location entry is a file. Emit input file ID.
+        assert(InputFileIDs[Content->OrigEntry] != 0 && "Missed file entry");
+        Record.push_back(InputFileIDs[Content->OrigEntry]);
 
-        // Emit size/modification time for this file.
-        Record.push_back(Content->OrigEntry->getSize());
-        Record.push_back(Content->OrigEntry->getModificationTime());
-        Record.push_back(Content->BufferOverridden);
         Record.push_back(File.NumCreatedFIDs);
         
-        FileDeclIDsTy::iterator FDI = FileDeclIDs.find(SLoc);
+        FileDeclIDsTy::iterator FDI = FileDeclIDs.find(FID);
         if (FDI != FileDeclIDs.end()) {
           Record.push_back(FDI->second->FirstDeclIndex);
           Record.push_back(FDI->second->DeclIDs.size());
@@ -1485,21 +1623,7 @@ void ASTWriter::WriteSourceManagerBlock(SourceManager &SourceMgr,
           Record.push_back(0);
         }
         
-        // Turn the file name into an absolute path, if it isn't already.
-        const char *Filename = Content->OrigEntry->getName();
-        SmallString<128> FilePath(Filename);
-
-        // Ask the file manager to fixup the relative path for us. This will 
-        // honor the working directory.
-        SourceMgr.getFileManager().FixupRelativePath(FilePath);
-
-        // FIXME: This call to make_absolute shouldn't be necessary, the
-        // call to FixupRelativePath should always return an absolute path.
-        llvm::sys::fs::make_absolute(FilePath);
-        Filename = FilePath.c_str();
-
-        Filename = adjustFilenameForRelocatablePCH(Filename, isysroot);
-        Stream.EmitRecordWithBlob(SLocFileAbbrv, Record, Filename);
+        Stream.EmitRecordWithAbbrev(SLocFileAbbrv, Record);
         
         if (Content->BufferOverridden) {
           Record.clear();
@@ -1570,18 +1694,6 @@ void ASTWriter::WriteSourceManagerBlock(SourceManager &SourceMgr,
   Record.push_back(SourceMgr.getNextLocalOffset() - 1); // skip dummy
   Stream.EmitRecordWithBlob(SLocOffsetsAbbrev, Record, data(SLocEntryOffsets));
 
-  Abbrev = new BitCodeAbbrev();
-  Abbrev->Add(BitCodeAbbrevOp(FILE_SOURCE_LOCATION_OFFSETS));
-  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 16)); // # of slocs
-  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // offsets
-  unsigned SLocFileOffsetsAbbrev = Stream.EmitAbbrev(Abbrev);
-
-  Record.clear();
-  Record.push_back(FILE_SOURCE_LOCATION_OFFSETS);
-  Record.push_back(SLocFileEntryOffsets.size());
-  Stream.EmitRecordWithBlob(SLocFileOffsetsAbbrev, Record,
-                            data(SLocFileEntryOffsets));
-
   // Write the source location entry preloads array, telling the AST
   // reader which source locations entries it should load eagerly.
   Stream.EmitRecord(SOURCE_LOCATION_PRELOADS, PreloadSLocs);
@@ -1675,102 +1787,129 @@ void ASTWriter::WritePreprocessor(const Preprocessor &PP, bool IsModule) {
   SmallVector<std::pair<const IdentifierInfo *, MacroInfo *>, 2> 
     MacrosToEmit;
   llvm::SmallPtrSet<const IdentifierInfo*, 4> MacroDefinitionsSeen;
-  for (Preprocessor::macro_iterator I = PP.macro_begin(Chain == 0), 
+  for (Preprocessor::macro_iterator I = PP.macro_begin(Chain == 0),
                                     E = PP.macro_end(Chain == 0);
        I != E; ++I) {
-    const IdentifierInfo *Name = I->first;
     if (!IsModule || I->second->isPublic()) {
-      MacroDefinitionsSeen.insert(Name);
+      MacroDefinitionsSeen.insert(I->first);
       MacrosToEmit.push_back(std::make_pair(I->first, I->second));
     }
   }
-  
+
   // Sort the set of macro definitions that need to be serialized by the
   // name of the macro, to provide a stable ordering.
-  llvm::array_pod_sort(MacrosToEmit.begin(), MacrosToEmit.end(), 
+  llvm::array_pod_sort(MacrosToEmit.begin(), MacrosToEmit.end(),
                        &compareMacroDefinitions);
-  
-  // Resolve any identifiers that defined macros at the time they were
-  // deserialized, adding them to the list of macros to emit (if appropriate).
-  for (unsigned I = 0, N = DeserializedMacroNames.size(); I != N; ++I) {
-    IdentifierInfo *Name
-      = const_cast<IdentifierInfo *>(DeserializedMacroNames[I]);
-    if (Name->hasMacroDefinition() && MacroDefinitionsSeen.insert(Name))
-      MacrosToEmit.push_back(std::make_pair(Name, PP.getMacroInfo(Name)));
-  }
-  
+
+  /// \brief Offsets of each of the macros into the bitstream, indexed by
+  /// the local macro ID
+  ///
+  /// For each identifier that is associated with a macro, this map
+  /// provides the offset into the bitstream where that macro is
+  /// defined.
+  std::vector<uint32_t> MacroOffsets;
+
   for (unsigned I = 0, N = MacrosToEmit.size(); I != N; ++I) {
     const IdentifierInfo *Name = MacrosToEmit[I].first;
-    MacroInfo *MI = MacrosToEmit[I].second;
-    if (!MI)
-      continue;
-    
-    // Don't emit builtin macros like __LINE__ to the AST file unless they have
-    // been redefined by the header (in which case they are not isBuiltinMacro).
-    // Also skip macros from a AST file if we're chaining.
-
-    // FIXME: There is a (probably minor) optimization we could do here, if
-    // the macro comes from the original PCH but the identifier comes from a
-    // chained PCH, by storing the offset into the original PCH rather than
-    // writing the macro definition a second time.
-    if (MI->isBuiltinMacro() ||
-        (Chain && 
-         Name->isFromAST() && !Name->hasChangedSinceDeserialization() && 
-         MI->isFromAST() && !MI->hasChangedAfterLoad()))
-      continue;
 
-    AddIdentifierRef(Name, Record);
-    MacroOffsets[Name] = Stream.GetCurrentBitNo();
-    Record.push_back(MI->getDefinitionLoc().getRawEncoding());
-    Record.push_back(MI->isUsed());
-    Record.push_back(MI->isPublic());
-    AddSourceLocation(MI->getVisibilityLocation(), Record);
-    unsigned Code;
-    if (MI->isObjectLike()) {
-      Code = PP_MACRO_OBJECT_LIKE;
-    } else {
-      Code = PP_MACRO_FUNCTION_LIKE;
+    for (MacroInfo *MI = MacrosToEmit[I].second; MI;
+         MI = MI->getPreviousDefinition()) {
+      MacroID ID = getMacroRef(MI);
+      if (!ID)
+        continue;
 
-      Record.push_back(MI->isC99Varargs());
-      Record.push_back(MI->isGNUVarargs());
-      Record.push_back(MI->getNumArgs());
-      for (MacroInfo::arg_iterator I = MI->arg_begin(), E = MI->arg_end();
-           I != E; ++I)
-        AddIdentifierRef(*I, Record);
-    }
+      // Skip macros from a AST file if we're chaining.
+      if (Chain && MI->isFromAST() && !MI->hasChangedAfterLoad())
+        continue;
 
-    // If we have a detailed preprocessing record, record the macro definition
-    // ID that corresponds to this macro.
-    if (PPRec)
-      Record.push_back(MacroDefinitions[PPRec->findMacroDefinition(MI)]);
+      if (ID < FirstMacroID) {
+        // This will have been dealt with via an update record.
+        assert(MacroUpdates.count(MI) > 0 && "Missing macro update");
+        continue;
+      }
 
-    Stream.EmitRecord(Code, Record);
-    Record.clear();
+      // Record the local offset of this macro.
+      unsigned Index = ID - FirstMacroID;
+      if (Index == MacroOffsets.size())
+        MacroOffsets.push_back(Stream.GetCurrentBitNo());
+      else {
+        if (Index > MacroOffsets.size())
+          MacroOffsets.resize(Index + 1);
 
-    // Emit the tokens array.
-    for (unsigned TokNo = 0, e = MI->getNumTokens(); TokNo != e; ++TokNo) {
-      // Note that we know that the preprocessor does not have any annotation
-      // tokens in it because they are created by the parser, and thus can't be
-      // in a macro definition.
-      const Token &Tok = MI->getReplacementToken(TokNo);
-
-      Record.push_back(Tok.getLocation().getRawEncoding());
-      Record.push_back(Tok.getLength());
-
-      // FIXME: When reading literal tokens, reconstruct the literal pointer if
-      // it is needed.
-      AddIdentifierRef(Tok.getIdentifierInfo(), Record);
-      // FIXME: Should translate token kind to a stable encoding.
-      Record.push_back(Tok.getKind());
-      // FIXME: Should translate token flags to a stable encoding.
-      Record.push_back(Tok.getFlags());
-
-      Stream.EmitRecord(PP_TOKEN, Record);
+        MacroOffsets[Index] = Stream.GetCurrentBitNo();
+      }
+
+      AddIdentifierRef(Name, Record);
+      addMacroRef(MI, Record);
+      Record.push_back(inferSubmoduleIDFromLocation(MI->getDefinitionLoc()));
+      AddSourceLocation(MI->getDefinitionLoc(), Record);
+      AddSourceLocation(MI->getUndefLoc(), Record);
+      Record.push_back(MI->isUsed());
+      Record.push_back(MI->isPublic());
+      AddSourceLocation(MI->getVisibilityLocation(), Record);
+      unsigned Code;
+      if (MI->isObjectLike()) {
+        Code = PP_MACRO_OBJECT_LIKE;
+      } else {
+        Code = PP_MACRO_FUNCTION_LIKE;
+
+        Record.push_back(MI->isC99Varargs());
+        Record.push_back(MI->isGNUVarargs());
+        Record.push_back(MI->getNumArgs());
+        for (MacroInfo::arg_iterator I = MI->arg_begin(), E = MI->arg_end();
+             I != E; ++I)
+          AddIdentifierRef(*I, Record);
+      }
+
+      // If we have a detailed preprocessing record, record the macro definition
+      // ID that corresponds to this macro.
+      if (PPRec)
+        Record.push_back(MacroDefinitions[PPRec->findMacroDefinition(MI)]);
+
+      Stream.EmitRecord(Code, Record);
       Record.clear();
+
+      // Emit the tokens array.
+      for (unsigned TokNo = 0, e = MI->getNumTokens(); TokNo != e; ++TokNo) {
+        // Note that we know that the preprocessor does not have any annotation
+        // tokens in it because they are created by the parser, and thus can't
+        // be in a macro definition.
+        const Token &Tok = MI->getReplacementToken(TokNo);
+
+        Record.push_back(Tok.getLocation().getRawEncoding());
+        Record.push_back(Tok.getLength());
+
+        // FIXME: When reading literal tokens, reconstruct the literal pointer
+        // if it is needed.
+        AddIdentifierRef(Tok.getIdentifierInfo(), Record);
+        // FIXME: Should translate token kind to a stable encoding.
+        Record.push_back(Tok.getKind());
+        // FIXME: Should translate token flags to a stable encoding.
+        Record.push_back(Tok.getFlags());
+
+        Stream.EmitRecord(PP_TOKEN, Record);
+        Record.clear();
+      }
+      ++NumMacros;
     }
-    ++NumMacros;
   }
   Stream.ExitBlock();
+
+  // Write the offsets table for macro IDs.
+  using namespace llvm;
+  BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
+  Abbrev->Add(BitCodeAbbrevOp(MACRO_OFFSET));
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // # of macros
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // first ID
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
+
+  unsigned MacroOffsetAbbrev = Stream.EmitAbbrev(Abbrev);
+  Record.clear();
+  Record.push_back(MACRO_OFFSET);
+  Record.push_back(MacroOffsets.size());
+  Record.push_back(FirstMacroID - NUM_PREDEF_MACRO_IDS);
+  Stream.EmitRecordWithBlob(MacroOffsetAbbrev, Record,
+                            data(MacroOffsets));
 }
 
 void ASTWriter::WritePreprocessorDetail(PreprocessingRecord &PPRec) {
@@ -1794,6 +1933,7 @@ void ASTWriter::WritePreprocessorDetail(PreprocessingRecord &PPRec) {
     Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32)); // filename length
     Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // in quotes
     Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 2)); // kind
+    Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // imported module
     Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
     InclusionAbbrev = Stream.EmitAbbrev(Abbrev);
   }
@@ -1836,6 +1976,7 @@ void ASTWriter::WritePreprocessorDetail(PreprocessingRecord &PPRec) {
       Record.push_back(ID->getFileName().size());
       Record.push_back(ID->wasInQuotes());
       Record.push_back(static_cast<unsigned>(ID->getKind()));
+      Record.push_back(ID->importedModule());
       SmallString<64> Buffer;
       Buffer += ID->getFileName();
       // Check that the FileEntry is not null because it was not resolved and
@@ -1935,6 +2076,11 @@ void ASTWriter::WriteSubmodules(Module *WritingModule) {
   unsigned HeaderAbbrev = Stream.EmitAbbrev(Abbrev);
 
   Abbrev = new BitCodeAbbrev();
+  Abbrev->Add(BitCodeAbbrevOp(SUBMODULE_TOPHEADER));
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // Name
+  unsigned TopHeaderAbbrev = Stream.EmitAbbrev(Abbrev);
+
+  Abbrev = new BitCodeAbbrev();
   Abbrev->Add(BitCodeAbbrevOp(SUBMODULE_UMBRELLA_DIR));
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // Name
   unsigned UmbrellaDirAbbrev = Stream.EmitAbbrev(Abbrev);
@@ -1944,6 +2090,11 @@ void ASTWriter::WriteSubmodules(Module *WritingModule) {
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // Feature
   unsigned RequiresAbbrev = Stream.EmitAbbrev(Abbrev);
 
+  Abbrev = new BitCodeAbbrev();
+  Abbrev->Add(BitCodeAbbrevOp(SUBMODULE_EXCLUDED_HEADER));
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // Name
+  unsigned ExcludedHeaderAbbrev = Stream.EmitAbbrev(Abbrev);
+
   // Write the submodule metadata block.
   RecordData Record;
   Record.push_back(getNumberOfModules(WritingModule));
@@ -2005,6 +2156,19 @@ void ASTWriter::WriteSubmodules(Module *WritingModule) {
       Stream.EmitRecordWithBlob(HeaderAbbrev, Record, 
                                 Mod->Headers[I]->getName());
     }
+    // Emit the excluded headers.
+    for (unsigned I = 0, N = Mod->ExcludedHeaders.size(); I != N; ++I) {
+      Record.clear();
+      Record.push_back(SUBMODULE_EXCLUDED_HEADER);
+      Stream.EmitRecordWithBlob(ExcludedHeaderAbbrev, Record, 
+                                Mod->ExcludedHeaders[I]->getName());
+    }
+    for (unsigned I = 0, N = Mod->TopHeaders.size(); I != N; ++I) {
+      Record.clear();
+      Record.push_back(SUBMODULE_TOPHEADER);
+      Stream.EmitRecordWithBlob(TopHeaderAbbrev, Record,
+                                Mod->TopHeaders[I]->getName());
+    }
 
     // Emit the imports. 
     if (!Mod->Imports.empty()) {
@@ -2067,24 +2231,35 @@ ASTWriter::inferSubmoduleIDFromLocation(SourceLocation Loc) {
 }
 
 void ASTWriter::WritePragmaDiagnosticMappings(const DiagnosticsEngine &Diag) {
+  // FIXME: Make it work properly with modules.
+  llvm::SmallDenseMap<const DiagnosticsEngine::DiagState *, unsigned, 64>
+      DiagStateIDMap;
+  unsigned CurrID = 0;
+  DiagStateIDMap[&Diag.DiagStates.front()] = ++CurrID; // the command-line one.
   RecordData Record;
   for (DiagnosticsEngine::DiagStatePointsTy::const_iterator
          I = Diag.DiagStatePoints.begin(), E = Diag.DiagStatePoints.end();
          I != E; ++I) {
-    const DiagnosticsEngine::DiagStatePoint &point = *I; 
+    const DiagnosticsEngine::DiagStatePoint &point = *I;
     if (point.Loc.isInvalid())
       continue;
 
     Record.push_back(point.Loc.getRawEncoding());
-    for (DiagnosticsEngine::DiagState::const_iterator
-           I = point.State->begin(), E = point.State->end(); I != E; ++I) {
-      if (I->second.isPragma()) {
-        Record.push_back(I->first);
-        Record.push_back(I->second.getMapping());
+    unsigned &DiagStateID = DiagStateIDMap[point.State];
+    Record.push_back(DiagStateID);
+    
+    if (DiagStateID == 0) {
+      DiagStateID = ++CurrID;
+      for (DiagnosticsEngine::DiagState::const_iterator
+             I = point.State->begin(), E = point.State->end(); I != E; ++I) {
+        if (I->second.isPragma()) {
+          Record.push_back(I->first);
+          Record.push_back(I->second.getMapping());
+        }
       }
+      Record.push_back(-1); // mark the end of the diag/map pairs for this
+                            // location.
     }
-    Record.push_back(-1); // mark the end of the diag/map pairs for this
-                          // location.
   }
 
   if (!Record.empty())
@@ -2238,9 +2413,11 @@ void ASTWriter::WriteFileDeclIDsMap() {
 
   BitCodeAbbrev *Abbrev = new BitCodeAbbrev();
   Abbrev->Add(BitCodeAbbrevOp(FILE_SORTED_DECLS));
+  Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 32));
   Abbrev->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob));
   unsigned AbbrevCode = Stream.EmitAbbrev(Abbrev);
   Record.push_back(FILE_SORTED_DECLS);
+  Record.push_back(FileSortedIDs.size());
   Stream.EmitRecordWithBlob(AbbrevCode, Record, data(FileSortedIDs));
 }
 
@@ -2495,17 +2672,17 @@ class ASTIdentifierTableTrait {
         II->getFETokenInfo<void>())
       return true;
 
-    return hasMacroDefinition(II, Macro);
+    return hadMacroDefinition(II, Macro);
   }
-  
-  bool hasMacroDefinition(IdentifierInfo *II, MacroInfo *&Macro) {
-    if (!II->hasMacroDefinition())
+
+  bool hadMacroDefinition(IdentifierInfo *II, MacroInfo *&Macro) {
+    if (!II->hadMacroDefinition())
       return false;
-    
-    if (Macro || (Macro = PP.getMacroInfo(II)))
+
+    if (Macro || (Macro = PP.getMacroInfoHistory(II)))
       return !Macro->isBuiltinMacro() && (!IsModule || Macro->isPublic());
-    
-    return false;    
+
+    return false;
   }
 
 public:
@@ -2529,10 +2706,17 @@ public:
     unsigned DataLen = 4; // 4 bytes for the persistent ID << 1
     MacroInfo *Macro = 0;
     if (isInterestingIdentifier(II, Macro)) {
-      DataLen += 2; // 2 bytes for builtin ID, flags
-      if (hasMacroDefinition(II, Macro))
-        DataLen += 8;
-      
+      DataLen += 2; // 2 bytes for builtin ID
+      DataLen += 2; // 2 bytes for flags
+      if (hadMacroDefinition(II, Macro)) {
+        for (MacroInfo *M = Macro; M; M = M->getPreviousDefinition()) {
+          if (Writer.getMacroRef(M) != 0)
+            DataLen += 4;
+        }
+
+        DataLen += 4;
+      }
+
       for (IdentifierResolver::iterator D = IdResolver.begin(II),
                                      DEnd = IdResolver.end();
            D != DEnd; ++D)
@@ -2563,23 +2747,28 @@ public:
     }
 
     clang::io::Emit32(Out, (ID << 1) | 0x01);
-    uint32_t Bits = 0;
-    bool HasMacroDefinition = hasMacroDefinition(II, Macro);
-    Bits = (uint32_t)II->getObjCOrBuiltinID();
-    assert((Bits & 0x7ff) == Bits && "ObjCOrBuiltinID too big for ASTReader.");
-    Bits = (Bits << 1) | unsigned(HasMacroDefinition);
+    uint32_t Bits = (uint32_t)II->getObjCOrBuiltinID();
+    assert((Bits & 0xffff) == Bits && "ObjCOrBuiltinID too big for ASTReader.");
+    clang::io::Emit16(Out, Bits);
+    Bits = 0;
+    bool HadMacroDefinition = hadMacroDefinition(II, Macro);
+    Bits = (Bits << 1) | unsigned(HadMacroDefinition);
     Bits = (Bits << 1) | unsigned(II->isExtensionToken());
     Bits = (Bits << 1) | unsigned(II->isPoisoned());
     Bits = (Bits << 1) | unsigned(II->hasRevertedTokenIDToIdentifier());
     Bits = (Bits << 1) | unsigned(II->isCPlusPlusOperatorKeyword());
     clang::io::Emit16(Out, Bits);
 
-    if (HasMacroDefinition) {
-      clang::io::Emit32(Out, Writer.getMacroOffset(II));
-      clang::io::Emit32(Out, 
-        Writer.inferSubmoduleIDFromLocation(Macro->getDefinitionLoc()));
+    if (HadMacroDefinition) {
+      // Write all of the macro IDs associated with this identifier.
+      for (MacroInfo *M = Macro; M; M = M->getPreviousDefinition()) {
+        if (MacroID ID = Writer.getMacroRef(M))
+          clang::io::Emit32(Out, ID);
+      }
+
+      clang::io::Emit32(Out, 0);
     }
-    
+
     // Emit the declaration IDs in reverse order, because the
     // IdentifierResolver provides the declarations as they would be
     // visible (e.g., the function "stat" would come before the struct
@@ -2756,30 +2945,32 @@ public:
   void EmitKey(raw_ostream& Out, DeclarationName Name, unsigned) {
     using namespace clang::io;
 
-    assert(Name.getNameKind() < 0x100 && "Invalid name kind ?");
     Emit8(Out, Name.getNameKind());
     switch (Name.getNameKind()) {
     case DeclarationName::Identifier:
       Emit32(Out, Writer.getIdentifierRef(Name.getAsIdentifierInfo()));
-      break;
+      return;
     case DeclarationName::ObjCZeroArgSelector:
     case DeclarationName::ObjCOneArgSelector:
     case DeclarationName::ObjCMultiArgSelector:
       Emit32(Out, Writer.getSelectorRef(Name.getObjCSelector()));
-      break;
+      return;
     case DeclarationName::CXXOperatorName:
-      assert(Name.getCXXOverloadedOperator() < 0x100 && "Invalid operator ?");
+      assert(Name.getCXXOverloadedOperator() < NUM_OVERLOADED_OPERATORS &&
+             "Invalid operator?");
       Emit8(Out, Name.getCXXOverloadedOperator());
-      break;
+      return;
     case DeclarationName::CXXLiteralOperatorName:
       Emit32(Out, Writer.getIdentifierRef(Name.getCXXLiteralIdentifier()));
-      break;
+      return;
     case DeclarationName::CXXConstructorName:
     case DeclarationName::CXXDestructorName:
     case DeclarationName::CXXConversionFunctionName:
     case DeclarationName::CXXUsingDirective:
-      break;
+      return;
     }
+
+    llvm_unreachable("Invalid name kind?");
   }
 
   void EmitData(raw_ostream& Out, key_type_ref,
@@ -3147,7 +3338,8 @@ ASTWriter::ASTWriter(llvm::BitstreamWriter &Stream)
     ASTHasCompilerErrors(false),
     FirstDeclID(NUM_PREDEF_DECL_IDS), NextDeclID(FirstDeclID),
     FirstTypeID(NUM_PREDEF_TYPE_IDS), NextTypeID(FirstTypeID),
-    FirstIdentID(NUM_PREDEF_IDENT_IDS), NextIdentID(FirstIdentID), 
+    FirstIdentID(NUM_PREDEF_IDENT_IDS), NextIdentID(FirstIdentID),
+    FirstMacroID(NUM_PREDEF_MACRO_IDS), NextMacroID(FirstMacroID),
     FirstSubmoduleID(NUM_PREDEF_SUBMODULE_IDS), 
     NextSubmoduleID(FirstSubmoduleID),
     FirstSelectorID(NUM_PREDEF_SELECTOR_IDS), NextSelectorID(FirstSelectorID),
@@ -3171,7 +3363,7 @@ ASTWriter::~ASTWriter() {
     delete I->second;
 }
 
-void ASTWriter::WriteAST(Sema &SemaRef, MemorizeStatCalls *StatCalls,
+void ASTWriter::WriteAST(Sema &SemaRef,
                          const std::string &OutputFile,
                          Module *WritingModule, StringRef isysroot,
                          bool hasErrors) {
@@ -3190,7 +3382,7 @@ void ASTWriter::WriteAST(Sema &SemaRef, MemorizeStatCalls *StatCalls,
   Context = &SemaRef.Context;
   PP = &SemaRef.PP;
   this->WritingModule = WritingModule;
-  WriteASTCore(SemaRef, StatCalls, isysroot, OutputFile, WritingModule);
+  WriteASTCore(SemaRef, isysroot, OutputFile, WritingModule);
   Context = 0;
   PP = 0;
   this->WritingModule = 0;
@@ -3207,7 +3399,7 @@ static void AddLazyVectorDecls(ASTWriter &Writer, Vector &Vec,
   }
 }
 
-void ASTWriter::WriteASTCore(Sema &SemaRef, MemorizeStatCalls *StatCalls,
+void ASTWriter::WriteASTCore(Sema &SemaRef,
                              StringRef isysroot,
                              const std::string &OutputFile, 
                              Module *WritingModule) {
@@ -3357,14 +3549,13 @@ void ASTWriter::WriteASTCore(Sema &SemaRef, MemorizeStatCalls *StatCalls,
     if (!I->second)
       AddDeclRef(I->first, KnownNamespaces);
   }
-  
+
+  // Write the control block
+  WriteControlBlock(PP, Context, isysroot, OutputFile);
+
   // Write the remaining AST contents.
   RecordData Record;
   Stream.EnterSubblock(AST_BLOCK_ID, 5);
-  WriteMetadata(Context, isysroot, OutputFile);
-  WriteLanguageOptions(Context.getLangOpts());
-  if (StatCalls && isysroot.empty())
-    WriteStatCache(*StatCalls);
 
   // Create a lexical update block containing all of the declarations in the
   // translation unit that do not come from other AST files.
@@ -3482,6 +3673,7 @@ void ASTWriter::WriteASTCore(Sema &SemaRef, MemorizeStatCalls *StatCalls,
         Out.write(FileName.data(), FileName.size());
         io::Emit32(Out, (*M)->SLocEntryBaseOffset);
         io::Emit32(Out, (*M)->BaseIdentifierID);
+        io::Emit32(Out, (*M)->BaseMacroID);
         io::Emit32(Out, (*M)->BasePreprocessedEntityID);
         io::Emit32(Out, (*M)->BaseSubmoduleID);
         io::Emit32(Out, (*M)->BaseSelectorID);
@@ -3595,7 +3787,8 @@ void ASTWriter::WriteASTCore(Sema &SemaRef, MemorizeStatCalls *StatCalls,
       Stream.EmitRecord(IMPORTED_MODULES, ImportedModules);
     }
   }
-  
+
+  WriteMacroUpdates();
   WriteDeclUpdatesBlocks();
   WriteDeclReplacementsBlock();
   WriteMergedDecls();
@@ -3612,6 +3805,21 @@ void ASTWriter::WriteASTCore(Sema &SemaRef, MemorizeStatCalls *StatCalls,
   Stream.ExitBlock();
 }
 
+void ASTWriter::WriteMacroUpdates() {
+  if (MacroUpdates.empty())
+    return;
+
+  RecordData Record;
+  for (MacroUpdatesMap::iterator I = MacroUpdates.begin(),
+                                 E = MacroUpdates.end();
+       I != E; ++I) {
+    addMacroRef(I->first, Record);
+    AddSourceLocation(I->second.UndefLoc, Record);
+    Record.push_back(inferSubmoduleIDFromLocation(I->second.UndefLoc));
+  }
+  Stream.EmitRecord(MACRO_UPDATES, Record);
+}
+
 /// \brief Go through the declaration update blocks and resolve declaration
 /// pointers into declaration IDs.
 void ASTWriter::ResolveDeclUpdatesBlocks() {
@@ -3707,6 +3915,10 @@ void ASTWriter::AddIdentifierRef(const IdentifierInfo *II, RecordDataImpl &Recor
   Record.push_back(getIdentifierRef(II));
 }
 
+void ASTWriter::addMacroRef(MacroInfo *MI, RecordDataImpl &Record) {
+  Record.push_back(getMacroRef(MI));
+}
+
 IdentID ASTWriter::getIdentifierRef(const IdentifierInfo *II) {
   if (II == 0)
     return 0;
@@ -3717,6 +3929,19 @@ IdentID ASTWriter::getIdentifierRef(const IdentifierInfo *II) {
   return ID;
 }
 
+MacroID ASTWriter::getMacroRef(MacroInfo *MI) {
+  // Don't emit builtin macros like __LINE__ to the AST file unless they
+  // have been redefined by the header (in which case they are not
+  // isBuiltinMacro).
+  if (MI == 0 || MI->isBuiltinMacro())
+    return 0;
+
+  MacroID &ID = MacroIDs[MI];
+  if (ID == 0)
+    ID = NextMacroID++;
+  return ID;
+}
+
 void ASTWriter::AddSelectorRef(const Selector SelRef, RecordDataImpl &Record) {
   Record.push_back(getSelectorRef(SelRef));
 }
@@ -3774,7 +3999,9 @@ void ASTWriter::AddTemplateArgumentLocInfo(TemplateArgument::ArgKind Kind,
   case TemplateArgument::Null:
   case TemplateArgument::Integral:
   case TemplateArgument::Declaration:
+  case TemplateArgument::NullPtr:
   case TemplateArgument::Pack:
+    // FIXME: Is this right?
     break;
   }
 }
@@ -3931,10 +4158,9 @@ void ASTWriter::associateDeclWithFile(const Decl *D, DeclID ID) {
   llvm::tie(FID, Offset) = SM.getDecomposedLoc(FileLoc);
   if (FID.isInvalid())
     return;
-  const SrcMgr::SLocEntry *Entry = &SM.getSLocEntry(FID);
-  assert(Entry->isFile());
+  assert(SM.getSLocEntry(FID).isFile());
 
-  DeclIDInFileInfo *&Info = FileDeclIDs[Entry];
+  DeclIDInFileInfo *&Info = FileDeclIDs[FID];
   if (!Info)
     Info = new DeclIDInFileInfo();
 
@@ -4191,6 +4417,10 @@ void ASTWriter::AddTemplateArgument(const TemplateArgument &Arg,
     break;
   case TemplateArgument::Declaration:
     AddDeclRef(Arg.getAsDecl(), Record);
+    Record.push_back(Arg.isDeclForReferenceParam());
+    break;
+  case TemplateArgument::NullPtr:
+    AddTypeRef(Arg.getNullPtrType(), Record);
     break;
   case TemplateArgument::Integral:
     AddAPSInt(Arg.getAsIntegral(), Record);
@@ -4403,6 +4633,7 @@ void ASTWriter::AddCXXDefinitionData(const CXXRecordDecl *D, RecordDataImpl &Rec
     Record.push_back(Lambda.NumExplicitCaptures);
     Record.push_back(Lambda.ManglingNumber);
     AddDeclRef(Lambda.ContextDecl, Record);
+    AddTypeSourceInfo(Lambda.MethodTyInfo, Record);
     for (unsigned I = 0, N = Lambda.NumCaptures; I != N; ++I) {
       LambdaExpr::Capture &Capture = Lambda.Captures[I];
       AddSourceLocation(Capture.getLocation(), Record);
@@ -4423,6 +4654,7 @@ void ASTWriter::ReaderInitialized(ASTReader *Reader) {
   assert(FirstDeclID == NextDeclID &&
          FirstTypeID == NextTypeID &&
          FirstIdentID == NextIdentID &&
+         FirstMacroID == NextMacroID &&
          FirstSubmoduleID == NextSubmoduleID &&
          FirstSelectorID == NextSelectorID &&
          "Setting chain after writing has started.");
@@ -4432,19 +4664,23 @@ void ASTWriter::ReaderInitialized(ASTReader *Reader) {
   FirstDeclID = NUM_PREDEF_DECL_IDS + Chain->getTotalNumDecls();
   FirstTypeID = NUM_PREDEF_TYPE_IDS + Chain->getTotalNumTypes();
   FirstIdentID = NUM_PREDEF_IDENT_IDS + Chain->getTotalNumIdentifiers();
+  FirstMacroID = NUM_PREDEF_MACRO_IDS + Chain->getTotalNumMacros();
   FirstSubmoduleID = NUM_PREDEF_SUBMODULE_IDS + Chain->getTotalNumSubmodules();
   FirstSelectorID = NUM_PREDEF_SELECTOR_IDS + Chain->getTotalNumSelectors();
   NextDeclID = FirstDeclID;
   NextTypeID = FirstTypeID;
   NextIdentID = FirstIdentID;
+  NextMacroID = FirstMacroID;
   NextSelectorID = FirstSelectorID;
   NextSubmoduleID = FirstSubmoduleID;
 }
 
 void ASTWriter::IdentifierRead(IdentID ID, IdentifierInfo *II) {
   IdentifierIDs[II] = ID;
-  if (II->hasMacroDefinition())
-    DeserializedMacroNames.push_back(II);
+}
+
+void ASTWriter::MacroRead(serialization::MacroID ID, MacroInfo *MI) {
+  MacroIDs[MI] = ID;
 }
 
 void ASTWriter::TypeRead(TypeIdx Idx, QualType T) {
@@ -4468,15 +4704,15 @@ void ASTWriter::MacroDefinitionRead(serialization::PreprocessedEntityID ID,
   MacroDefinitions[MD] = ID;
 }
 
-void ASTWriter::MacroVisible(IdentifierInfo *II) {
-  DeserializedMacroNames.push_back(II);
-}
-
 void ASTWriter::ModuleRead(serialization::SubmoduleID ID, Module *Mod) {
   assert(SubmoduleIDs.find(Mod) == SubmoduleIDs.end());
   SubmoduleIDs[Mod] = ID;
 }
 
+void ASTWriter::UndefinedMacro(MacroInfo *MI) {
+  MacroUpdates[MI].UndefLoc = MI->getUndefLoc();
+}
+
 void ASTWriter::CompletedTagDefinition(const TagDecl *D) {
   assert(D->isCompleteDefinition());
   assert(!WritingAST && "Already writing the AST!");
@@ -4490,6 +4726,7 @@ void ASTWriter::CompletedTagDefinition(const TagDecl *D) {
     }
   }
 }
+
 void ASTWriter::AddedVisibleDecl(const DeclContext *DC, const Decl *D) {
   assert(!WritingAST && "Already writing the AST!");
 
diff --git a/lib/Serialization/ASTWriterDecl.cpp b/lib/Serialization/ASTWriterDecl.cpp
index 602943b..7486565 100644
--- a/lib/Serialization/ASTWriterDecl.cpp
+++ b/lib/Serialization/ASTWriterDecl.cpp
@@ -416,7 +416,7 @@ void ASTDeclWriter::VisitObjCMethodDecl(ObjCMethodDecl *D) {
   }
   Record.push_back(D->isInstanceMethod());
   Record.push_back(D->isVariadic());
-  Record.push_back(D->isSynthesized());
+  Record.push_back(D->isPropertyAccessor());
   Record.push_back(D->isDefined());
   Record.push_back(D->IsOverriding);
 
@@ -606,6 +606,8 @@ void ASTDeclWriter::VisitObjCImplementationDecl(ObjCImplementationDecl *D) {
   Writer.AddDeclRef(D->getSuperClass(), Record);
   Writer.AddSourceLocation(D->getIvarLBraceLoc(), Record);
   Writer.AddSourceLocation(D->getIvarRBraceLoc(), Record);
+  Record.push_back(D->hasNonZeroConstructors());
+  Record.push_back(D->hasDestructors());
   Writer.AddCXXCtorInitializers(D->IvarInitializers, D->NumIvarInitializers,
                                 Record);
   Code = serialization::DECL_OBJC_IMPLEMENTATION;
@@ -675,6 +677,8 @@ void ASTDeclWriter::VisitVarDecl(VarDecl *D) {
   Record.push_back(D->isNRVOVariable());
   Record.push_back(D->isCXXForRangeDecl());
   Record.push_back(D->isARCPseudoStrong());
+  Record.push_back(D->isConstexpr());
+
   if (D->getInit()) {
     Record.push_back(!D->isInitKnownICE() ? 1 : (D->isInitICE() ? 3 : 2));
     Writer.AddStmt(D->getInit());
@@ -705,6 +709,7 @@ void ASTDeclWriter::VisitVarDecl(VarDecl *D) {
       D->getInitStyle() == VarDecl::CInit &&
       D->getInit() == 0 &&
       !isa<ParmVarDecl>(D) &&
+      !D->isConstexpr() &&
       !SpecInfo)
     AbbrevToUse = Writer.getDeclVarAbbrev();
 
@@ -945,11 +950,16 @@ void ASTDeclWriter::VisitCXXRecordDecl(CXXRecordDecl *D) {
 
 void ASTDeclWriter::VisitCXXMethodDecl(CXXMethodDecl *D) {
   VisitFunctionDecl(D);
-  Record.push_back(D->size_overridden_methods());
-  for (CXXMethodDecl::method_iterator
-         I = D->begin_overridden_methods(), E = D->end_overridden_methods();
-         I != E; ++I)
-    Writer.AddDeclRef(*I, Record);
+  if (D->isCanonicalDecl()) {
+    Record.push_back(D->size_overridden_methods());
+    for (CXXMethodDecl::method_iterator
+           I = D->begin_overridden_methods(), E = D->end_overridden_methods();
+           I != E; ++I)
+      Writer.AddDeclRef(*I, Record);
+  } else {
+    // We only need to record overridden methods once for the canonical decl.
+    Record.push_back(0);
+  }
   Code = serialization::DECL_CXX_METHOD;
 }
 
@@ -981,6 +991,7 @@ void ASTDeclWriter::VisitCXXConversionDecl(CXXConversionDecl *D) {
 
 void ASTDeclWriter::VisitImportDecl(ImportDecl *D) {
   VisitDecl(D);
+  Record.push_back(Writer.getSubmoduleID(D->getImportedModule()));
   ArrayRef<SourceLocation> IdentifierLocs = D->getIdentifierLocs();
   Record.push_back(!IdentifierLocs.empty());
   if (IdentifierLocs.empty()) {
@@ -1073,7 +1084,7 @@ void ASTDeclWriter::VisitClassTemplateDecl(ClassTemplateDecl *D) {
       Writer.AddDeclRef(&*I, Record); 
     }
 
-    // InjectedClassNameType is computed, no need to write it.
+    Writer.AddTypeRef(D->getCommonPtr()->InjectedClassNameType, Record);
   }
   Code = serialization::DECL_CLASS_TEMPLATE;
 }
@@ -1103,6 +1114,7 @@ void ASTDeclWriter::VisitClassTemplateSpecializationDecl(
   Writer.AddTemplateArgumentList(&D->getTemplateArgs(), Record);
   Writer.AddSourceLocation(D->getPointOfInstantiation(), Record);
   Record.push_back(D->getSpecializationKind());
+  Record.push_back(D->isCanonicalDecl());
 
   if (D->isCanonicalDecl()) {
     // When reading, we'll add it to the folding set of the following template. 
@@ -1172,7 +1184,8 @@ void ASTDeclWriter::VisitTemplateTypeParmDecl(TemplateTypeParmDecl *D) {
 
 void ASTDeclWriter::VisitNonTypeTemplateParmDecl(NonTypeTemplateParmDecl *D) {
   // For an expanded parameter pack, record the number of expansion types here
-  // so that it's easier for 
+  // so that it's easier for deserialization to allocate the right amount of
+  // memory.
   if (D->isExpandedParameterPack())
     Record.push_back(D->getNumExpansionTypes());
   
@@ -1201,15 +1214,30 @@ void ASTDeclWriter::VisitNonTypeTemplateParmDecl(NonTypeTemplateParmDecl *D) {
 }
 
 void ASTDeclWriter::VisitTemplateTemplateParmDecl(TemplateTemplateParmDecl *D) {
+  // For an expanded parameter pack, record the number of expansion types here
+  // so that it's easier for deserialization to allocate the right amount of
+  // memory.
+  if (D->isExpandedParameterPack())
+    Record.push_back(D->getNumExpansionTemplateParameters());
+
   VisitTemplateDecl(D);
   // TemplateParmPosition.
   Record.push_back(D->getDepth());
   Record.push_back(D->getPosition());
-  // Rest of TemplateTemplateParmDecl.
-  Writer.AddTemplateArgumentLoc(D->getDefaultArgument(), Record);
-  Record.push_back(D->defaultArgumentWasInherited());
-  Record.push_back(D->isParameterPack());
-  Code = serialization::DECL_TEMPLATE_TEMPLATE_PARM;
+
+  if (D->isExpandedParameterPack()) {
+    for (unsigned I = 0, N = D->getNumExpansionTemplateParameters();
+         I != N; ++I)
+      Writer.AddTemplateParameterList(D->getExpansionTemplateParameters(I),
+                                      Record);
+    Code = serialization::DECL_EXPANDED_TEMPLATE_TEMPLATE_PARM_PACK;
+  } else {
+    // Rest of TemplateTemplateParmDecl.
+    Writer.AddTemplateArgumentLoc(D->getDefaultArgument(), Record);
+    Record.push_back(D->defaultArgumentWasInherited());
+    Record.push_back(D->isParameterPack());
+    Code = serialization::DECL_TEMPLATE_TEMPLATE_PARM;
+  }
 }
 
 void ASTDeclWriter::VisitTypeAliasTemplateDecl(TypeAliasTemplateDecl *D) {
@@ -1462,6 +1490,7 @@ void ASTWriter::WriteDeclsBlockAbbrevs() {
   Abv->Add(BitCodeAbbrevOp(0));                       // isNRVOVariable
   Abv->Add(BitCodeAbbrevOp(0));                       // isCXXForRangeDecl
   Abv->Add(BitCodeAbbrevOp(0));                       // isARCPseudoStrong
+  Abv->Add(BitCodeAbbrevOp(0));                       // isConstexpr
   Abv->Add(BitCodeAbbrevOp(0));                       // HasInit
   Abv->Add(BitCodeAbbrevOp(0));                   // HasMemberSpecializationInfo
   // ParmVarDecl
@@ -1540,6 +1569,7 @@ void ASTWriter::WriteDeclsBlockAbbrevs() {
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isNRVOVariable
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isCXXForRangeDecl
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // isARCPseudoStrong
+  Abv->Add(BitCodeAbbrevOp(0));                         // isConstexpr
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // HasInit
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // HasMemberSpecInfo
   // Type Source Info
@@ -1603,7 +1633,7 @@ void ASTWriter::WriteDeclsBlockAbbrevs() {
   //Character Literal
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // getValue
   Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Location
-  Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); //IsWide
+  Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 2)); // getKind
   CharacterLiteralAbbrev = Stream.EmitAbbrev(Abv);
 
   Abv = new BitCodeAbbrev();
diff --git a/lib/Serialization/ASTWriterStmt.cpp b/lib/Serialization/ASTWriterStmt.cpp
index f63388f..7e8ce42 100644
--- a/lib/Serialization/ASTWriterStmt.cpp
+++ b/lib/Serialization/ASTWriterStmt.cpp
@@ -218,7 +218,7 @@ void ASTStmtWriter::VisitDeclStmt(DeclStmt *S) {
   Code = serialization::STMT_DECL;
 }
 
-void ASTStmtWriter::VisitAsmStmt(AsmStmt *S) {
+void ASTStmtWriter::VisitGCCAsmStmt(GCCAsmStmt *S) {
   VisitStmt(S);
   Record.push_back(S->getNumOutputs());
   Record.push_back(S->getNumInputs());
@@ -227,7 +227,6 @@ void ASTStmtWriter::VisitAsmStmt(AsmStmt *S) {
   Writer.AddSourceLocation(S->getRParenLoc(), Record);
   Record.push_back(S->isVolatile());
   Record.push_back(S->isSimple());
-  Record.push_back(S->isMSAsm());
   Writer.AddStmt(S->getAsmString());
 
   // Outputs
@@ -246,14 +245,16 @@ void ASTStmtWriter::VisitAsmStmt(AsmStmt *S) {
 
   // Clobbers
   for (unsigned I = 0, N = S->getNumClobbers(); I != N; ++I)
-    Writer.AddStmt(S->getClobber(I));
+    Writer.AddStmt(S->getClobberStringLiteral(I));
 
-  Code = serialization::STMT_ASM;
+  Code = serialization::STMT_GCCASM;
 }
 
 void ASTStmtWriter::VisitMSAsmStmt(MSAsmStmt *S) {
   // FIXME: Statement writer not yet implemented for MS style inline asm.
   VisitStmt(S);
+
+  Code = serialization::STMT_MSASM;
 }
 
 void ASTStmtWriter::VisitExpr(Expr *E) {
@@ -535,6 +536,7 @@ void ASTStmtWriter::VisitBinaryOperator(BinaryOperator *E) {
   Writer.AddStmt(E->getRHS());
   Record.push_back(E->getOpcode()); // FIXME: stable encoding
   Writer.AddSourceLocation(E->getOperatorLoc(), Record);
+  Record.push_back(E->isFPContractable());
   Code = serialization::EXPR_BINARY_OPERATOR;
 }
 
@@ -604,6 +606,8 @@ void ASTStmtWriter::VisitExtVectorElementExpr(ExtVectorElementExpr *E) {
 
 void ASTStmtWriter::VisitInitListExpr(InitListExpr *E) {
   VisitExpr(E);
+  // NOTE: only add the (possibly null) syntactic form.
+  // No need to serialize the isSemanticForm flag and the semantic form.
   Writer.AddStmt(E->getSyntacticForm());
   Writer.AddSourceLocation(E->getLBraceLoc(), Record);
   Writer.AddSourceLocation(E->getRBraceLoc(), Record);
@@ -1054,6 +1058,7 @@ void ASTStmtWriter::VisitCXXOperatorCallExpr(CXXOperatorCallExpr *E) {
   VisitCallExpr(E);
   Record.push_back(E->getOperator());
   Writer.AddSourceRange(E->Range, Record);
+  Record.push_back(E->isFPContractable());
   Code = serialization::EXPR_CXX_OPERATOR_CALL;
 }
 
@@ -1233,7 +1238,7 @@ void ASTStmtWriter::VisitCXXNewExpr(CXXNewExpr *E) {
   Writer.AddDeclRef(E->getOperatorDelete(), Record);
   Writer.AddTypeSourceInfo(E->getAllocatedTypeSourceInfo(), Record);
   Writer.AddSourceRange(E->getTypeIdParens(), Record);
-  Writer.AddSourceLocation(E->getStartLoc(), Record);
+  Writer.AddSourceRange(E->getSourceRange(), Record);
   Writer.AddSourceRange(E->getDirectInitRange(), Record);
   for (CXXNewExpr::arg_iterator I = E->raw_arg_begin(), e = E->raw_arg_end();
        I != e; ++I)
@@ -1382,8 +1387,6 @@ void ASTStmtWriter::VisitUnresolvedMemberExpr(UnresolvedMemberExpr *E) {
 void ASTStmtWriter::VisitUnresolvedLookupExpr(UnresolvedLookupExpr *E) {
   VisitOverloadExpr(E);
   Record.push_back(E->requiresADL());
-  if (E->requiresADL())
-    Record.push_back(E->isStdAssociatedNamespace());
   Record.push_back(E->isOverloaded());
   Writer.AddDeclRef(E->getNamingClass(), Record);
   Code = serialization::EXPR_CXX_UNRESOLVED_LOOKUP;
@@ -1480,6 +1483,17 @@ void ASTStmtWriter::VisitSubstNonTypeTemplateParmPackExpr(
   Code = serialization::EXPR_SUBST_NON_TYPE_TEMPLATE_PARM_PACK;
 }
 
+void ASTStmtWriter::VisitFunctionParmPackExpr(FunctionParmPackExpr *E) {
+  VisitExpr(E);
+  Record.push_back(E->getNumExpansions());
+  Writer.AddDeclRef(E->getParameterPack(), Record);
+  Writer.AddSourceLocation(E->getParameterPackLocation(), Record);
+  for (FunctionParmPackExpr::iterator I = E->begin(), End = E->end();
+       I != End; ++I)
+    Writer.AddDeclRef(*I, Record);
+  Code = serialization::EXPR_FUNCTION_PARM_PACK;
+}
+
 void ASTStmtWriter::VisitMaterializeTemporaryExpr(MaterializeTemporaryExpr *E) {
   VisitExpr(E);
   Writer.AddStmt(E->Temporary);
diff --git a/lib/Serialization/GeneratePCH.cpp b/lib/Serialization/GeneratePCH.cpp
index 02aed10..870d654 100644
--- a/lib/Serialization/GeneratePCH.cpp
+++ b/lib/Serialization/GeneratePCH.cpp
@@ -18,7 +18,6 @@
 #include "clang/AST/ASTConsumer.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Basic/FileManager.h"
-#include "clang/Basic/FileSystemStatCache.h"
 #include "llvm/Bitcode/BitstreamWriter.h"
 #include "llvm/Support/raw_ostream.h"
 #include <string>
@@ -32,11 +31,7 @@ PCHGenerator::PCHGenerator(const Preprocessor &PP,
                            raw_ostream *OS)
   : PP(PP), OutputFile(OutputFile), Module(Module), 
     isysroot(isysroot.str()), Out(OS), 
-    SemaPtr(0), StatCalls(0), Stream(Buffer), Writer(Stream) {
-  // Install a stat() listener to keep track of all of the stat()
-  // calls.
-  StatCalls = new MemorizeStatCalls();
-  PP.getFileManager().addStatCache(StatCalls, /*AtBeginning=*/false);
+    SemaPtr(0), Stream(Buffer), Writer(Stream) {
 }
 
 PCHGenerator::~PCHGenerator() {
@@ -48,7 +43,7 @@ void PCHGenerator::HandleTranslationUnit(ASTContext &Ctx) {
   
   // Emit the PCH file
   assert(SemaPtr && "No Sema?");
-  Writer.WriteAST(*SemaPtr, StatCalls, OutputFile, Module, isysroot);
+  Writer.WriteAST(*SemaPtr, OutputFile, Module, isysroot);
 
   // Write the generated bitstream to "Out".
   Out->write((char *)&Buffer.front(), Buffer.size());
@@ -60,6 +55,10 @@ void PCHGenerator::HandleTranslationUnit(ASTContext &Ctx) {
   Buffer.clear();
 }
 
+PPMutationListener *PCHGenerator::GetPPMutationListener() {
+  return &Writer;
+}
+
 ASTMutationListener *PCHGenerator::GetASTMutationListener() {
   return &Writer;
 }
diff --git a/lib/Serialization/Module.cpp b/lib/Serialization/Module.cpp
index ff241d3..5e42ab4 100644
--- a/lib/Serialization/Module.cpp
+++ b/lib/Serialization/Module.cpp
@@ -21,12 +21,15 @@ using namespace serialization;
 using namespace reader;
 
 ModuleFile::ModuleFile(ModuleKind Kind, unsigned Generation)
-  : Kind(Kind), DirectlyImported(false), Generation(Generation), SizeInBits(0), 
+  : Kind(Kind), File(0), DirectlyImported(false),
+    Generation(Generation), SizeInBits(0),
     LocalNumSLocEntries(0), SLocEntryBaseID(0),
     SLocEntryBaseOffset(0), SLocEntryOffsets(0),
-    SLocFileOffsets(0), LocalNumIdentifiers(0), 
+    LocalNumIdentifiers(0),
     IdentifierOffsets(0), BaseIdentifierID(0), IdentifierTableData(0),
-    IdentifierLookupTable(0), BasePreprocessedEntityID(0),
+    IdentifierLookupTable(0),
+    LocalNumMacros(0), MacroOffsets(0),
+    BasePreprocessedEntityID(0),
     PreprocessedEntityOffsets(0), NumPreprocessedEntities(0),
     LocalNumHeaderFileInfos(0), 
     HeaderFileInfoTableData(0), HeaderFileInfoTable(0),
@@ -35,9 +38,10 @@ ModuleFile::ModuleFile(ModuleKind Kind, unsigned Generation)
     SelectorLookupTableData(0), SelectorLookupTable(0), LocalNumDecls(0),
     DeclOffsets(0), BaseDeclID(0),
     LocalNumCXXBaseSpecifiers(0), CXXBaseSpecifiersOffsets(0),
-    FileSortedDecls(0), RedeclarationsMap(0), LocalNumRedeclarationsInMap(0),
+    FileSortedDecls(0), NumFileSortedDecls(0),
+    RedeclarationsMap(0), LocalNumRedeclarationsInMap(0),
     ObjCCategoriesMap(0), LocalNumObjCCategoriesInMap(0),
-    LocalNumTypes(0), TypeOffsets(0), BaseTypeIndex(0), StatCache(0)
+    LocalNumTypes(0), TypeOffsets(0), BaseTypeIndex(0)
 {}
 
 ModuleFile::~ModuleFile() {
@@ -89,6 +93,10 @@ void ModuleFile::dump() {
                << "  Number of identifiers: " << LocalNumIdentifiers << '\n';
   dumpLocalRemap("Identifier ID local -> global map", IdentifierRemap);
 
+  llvm::errs() << "  Base macro ID: " << BaseMacroID << '\n'
+               << "  Number of macros: " << LocalNumMacros << '\n';
+  dumpLocalRemap("Macro ID local -> global map", MacroRemap);
+
   llvm::errs() << "  Base submodule ID: " << BaseSubmoduleID << '\n'
                << "  Number of submodules: " << LocalNumSubmodules << '\n';
   dumpLocalRemap("Submodule ID local -> global map", SubmoduleRemap);
diff --git a/lib/Serialization/ModuleManager.cpp b/lib/Serialization/ModuleManager.cpp
index ab364b7..efe4421 100644
--- a/lib/Serialization/ModuleManager.cpp
+++ b/lib/Serialization/ModuleManager.cpp
@@ -50,6 +50,7 @@ ModuleManager::addModule(StringRef FileName, ModuleKind Type,
     // Allocate a new module.
     ModuleFile *New = new ModuleFile(Type, Generation);
     New->FileName = FileName.str();
+    New->File = Entry;
     Chain.push_back(New);
     NewModule = true;
     ModuleEntry = New;
@@ -87,6 +88,45 @@ ModuleManager::addModule(StringRef FileName, ModuleKind Type,
   return std::make_pair(ModuleEntry, NewModule);
 }
 
+namespace {
+  /// \brief Predicate that checks whether a module file occurs within
+  /// the given set.
+  class IsInModuleFileSet : public std::unary_function<ModuleFile *, bool> {
+    llvm::SmallPtrSet<ModuleFile *, 4> &Removed;
+
+  public:
+    IsInModuleFileSet(llvm::SmallPtrSet<ModuleFile *, 4> &Removed)
+    : Removed(Removed) { }
+
+    bool operator()(ModuleFile *MF) const {
+      return Removed.count(MF);
+    }
+  };
+}
+
+void ModuleManager::removeModules(ModuleIterator first, ModuleIterator last) {
+  if (first == last)
+    return;
+
+  // Collect the set of module file pointers that we'll be removing.
+  llvm::SmallPtrSet<ModuleFile *, 4> victimSet(first, last);
+
+  // Remove any references to the now-destroyed modules.
+  IsInModuleFileSet checkInSet(victimSet);
+  for (unsigned i = 0, n = Chain.size(); i != n; ++i) {
+    Chain[i]->ImportedBy.remove_if(checkInSet);
+  }
+
+  // Delete the modules and erase them from the various structures.
+  for (ModuleIterator victim = first; victim != last; ++victim) {
+    Modules.erase((*victim)->File);
+    delete *victim;
+  }
+
+  // Remove the modules from the chain.
+  Chain.erase(first, last);
+}
+
 void ModuleManager::addInMemoryBuffer(StringRef FileName, 
                                       llvm::MemoryBuffer *Buffer) {
   
@@ -95,7 +135,7 @@ void ModuleManager::addInMemoryBuffer(StringRef FileName,
   InMemoryBuffers[Entry] = Buffer;
 }
 
-ModuleManager::ModuleManager(const FileSystemOptions &FSO) : FileMgr(FSO) { }
+ModuleManager::ModuleManager(FileManager &FileMgr) : FileMgr(FileMgr) { }
 
 ModuleManager::~ModuleManager() {
   for (unsigned i = 0, e = Chain.size(); i != e; ++i)
diff --git a/lib/StaticAnalyzer/Checkers/AdjustedReturnValueChecker.cpp b/lib/StaticAnalyzer/Checkers/AdjustedReturnValueChecker.cpp
deleted file mode 100644
index 84ea8c7..0000000
--- a/lib/StaticAnalyzer/Checkers/AdjustedReturnValueChecker.cpp
+++ /dev/null
@@ -1,92 +0,0 @@
-//== AdjustedReturnValueChecker.cpp -----------------------------*- C++ -*--==//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This file defines AdjustedReturnValueChecker, a simple check to see if the
-// return value of a function call is different than the one the caller thinks
-// it is.
-//
-//===----------------------------------------------------------------------===//
-
-#include "ClangSACheckers.h"
-#include "clang/StaticAnalyzer/Core/Checker.h"
-#include "clang/StaticAnalyzer/Core/CheckerManager.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
-#include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
-
-using namespace clang;
-using namespace ento;
-
-namespace {
-class AdjustedReturnValueChecker : 
-    public Checker< check::PostStmt<CallExpr> > {
-public:
-  void checkPostStmt(const CallExpr *CE, CheckerContext &C) const;
-};
-}
-
-void AdjustedReturnValueChecker::checkPostStmt(const CallExpr *CE,
-                                               CheckerContext &C) const {
-  
-  // Get the result type of the call.
-  QualType expectedResultTy = CE->getType();
-
-  // Fetch the signature of the called function.
-  ProgramStateRef state = C.getState();
-  const LocationContext *LCtx = C.getLocationContext();
-
-  SVal V = state->getSVal(CE, LCtx);
-  
-  if (V.isUnknown())
-    return;
-  
-  // Casting to void?  Discard the value.
-  if (expectedResultTy->isVoidType()) {
-    C.addTransition(state->BindExpr(CE, LCtx, UnknownVal()));
-    return;
-  }                   
-
-  const MemRegion *callee = state->getSVal(CE->getCallee(), LCtx).getAsRegion();
-  if (!callee)
-    return;
-
-  QualType actualResultTy;
-  
-  if (const FunctionTextRegion *FT = dyn_cast<FunctionTextRegion>(callee)) {
-    const FunctionDecl *FD = FT->getDecl();
-    actualResultTy = FD->getResultType();
-  }
-  else if (const BlockDataRegion *BD = dyn_cast<BlockDataRegion>(callee)) {
-    const BlockTextRegion *BR = BD->getCodeRegion();
-    const BlockPointerType *BT=BR->getLocationType()->getAs<BlockPointerType>();
-    const FunctionType *FT = BT->getPointeeType()->getAs<FunctionType>();
-    actualResultTy = FT->getResultType();
-  }
-
-  // Can this happen?
-  if (actualResultTy.isNull())
-    return;
-
-  // For now, ignore references.
-  if (actualResultTy->getAs<ReferenceType>())
-    return;
-  
-
-  // Are they the same?
-  if (expectedResultTy != actualResultTy) {
-    // FIXME: Do more checking and actual emit an error. At least performing
-    // the cast avoids some assertion failures elsewhere.
-    SValBuilder &svalBuilder = C.getSValBuilder();
-    V = svalBuilder.evalCast(V, expectedResultTy, actualResultTy);
-    C.addTransition(state->BindExpr(CE, LCtx, V));
-  }
-}
-
-void ento::registerAdjustedReturnValueChecker(CheckerManager &mgr) {
-  mgr.registerChecker<AdjustedReturnValueChecker>();
-}
diff --git a/lib/StaticAnalyzer/Checkers/ArrayBoundChecker.cpp b/lib/StaticAnalyzer/Checkers/ArrayBoundChecker.cpp
index b2ad184..535d8ee 100644
--- a/lib/StaticAnalyzer/Checkers/ArrayBoundChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/ArrayBoundChecker.cpp
@@ -78,7 +78,7 @@ void ArrayBoundChecker::checkLocation(SVal l, bool isLoad, const Stmt* LoadS,
       new BugReport(*BT, BT->getDescription(), N);
 
     report->addRange(LoadS->getSourceRange());
-    C.EmitReport(report);
+    C.emitReport(report);
     return;
   }
   
diff --git a/lib/StaticAnalyzer/Checkers/ArrayBoundCheckerV2.cpp b/lib/StaticAnalyzer/Checkers/ArrayBoundCheckerV2.cpp
index c6efe94..457c870 100644
--- a/lib/StaticAnalyzer/Checkers/ArrayBoundCheckerV2.cpp
+++ b/lib/StaticAnalyzer/Checkers/ArrayBoundCheckerV2.cpp
@@ -208,7 +208,7 @@ void ArrayBoundCheckerV2::reportOOB(CheckerContext &checkerContext,
     break;
   }
 
-  checkerContext.EmitReport(new BugReport(*BT, os.str(), errorNode));
+  checkerContext.emitReport(new BugReport(*BT, os.str(), errorNode));
 }
 
 void RegionRawOffsetV2::dump() const {
diff --git a/lib/StaticAnalyzer/Checkers/AttrNonNullChecker.cpp b/lib/StaticAnalyzer/Checkers/AttrNonNullChecker.cpp
index c582cfc..81e8dd8 100644
--- a/lib/StaticAnalyzer/Checkers/AttrNonNullChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/AttrNonNullChecker.cpp
@@ -105,9 +105,9 @@ void AttrNonNullChecker::checkPreCall(const CallEvent &Call,
         // Highlight the range of the argument that was null.
         R->addRange(Call.getArgSourceRange(idx));
         if (const Expr *ArgE = Call.getArgExpr(idx))
-          bugreporter::addTrackNullOrUndefValueVisitor(errorNode, ArgE, R);
+          bugreporter::trackNullOrUndefValue(errorNode, ArgE, *R);
         // Emit the bug report.
-        C.EmitReport(R);
+        C.emitReport(R);
       }
 
       // Always return.  Either we cached out or we just emitted an error.
diff --git a/lib/StaticAnalyzer/Checkers/BasicObjCFoundationChecks.cpp b/lib/StaticAnalyzer/Checkers/BasicObjCFoundationChecks.cpp
index 955e79a..eba534e 100644
--- a/lib/StaticAnalyzer/Checkers/BasicObjCFoundationChecks.cpp
+++ b/lib/StaticAnalyzer/Checkers/BasicObjCFoundationChecks.cpp
@@ -117,7 +117,7 @@ void NilArgChecker::WarnNilArg(CheckerContext &C,
 
     BugReport *R = new BugReport(*BT, os.str(), N);
     R->addRange(msg.getArgSourceRange(Arg));
-    C.EmitReport(R);
+    C.emitReport(R);
   }
 }
 
@@ -358,20 +358,20 @@ void CFNumberCreateChecker::checkPreStmt(const CallExpr *CE,
     
     BugReport *report = new BugReport(*BT, os.str(), N);
     report->addRange(CE->getArg(2)->getSourceRange());
-    C.EmitReport(report);
+    C.emitReport(report);
   }
 }
 
 //===----------------------------------------------------------------------===//
-// CFRetain/CFRelease checking for null arguments.
+// CFRetain/CFRelease/CFMakeCollectable checking for null arguments.
 //===----------------------------------------------------------------------===//
 
 namespace {
 class CFRetainReleaseChecker : public Checker< check::PreStmt<CallExpr> > {
   mutable OwningPtr<APIMisuse> BT;
-  mutable IdentifierInfo *Retain, *Release;
+  mutable IdentifierInfo *Retain, *Release, *MakeCollectable;
 public:
-  CFRetainReleaseChecker(): Retain(0), Release(0) {}
+  CFRetainReleaseChecker(): Retain(0), Release(0), MakeCollectable(0) {}
   void checkPreStmt(const CallExpr *CE, CheckerContext &C) const;
 };
 } // end anonymous namespace
@@ -392,12 +392,14 @@ void CFRetainReleaseChecker::checkPreStmt(const CallExpr *CE,
     ASTContext &Ctx = C.getASTContext();
     Retain = &Ctx.Idents.get("CFRetain");
     Release = &Ctx.Idents.get("CFRelease");
-    BT.reset(new APIMisuse("null passed to CFRetain/CFRelease"));
+    MakeCollectable = &Ctx.Idents.get("CFMakeCollectable");
+    BT.reset(
+      new APIMisuse("null passed to CFRetain/CFRelease/CFMakeCollectable"));
   }
 
-  // Check if we called CFRetain/CFRelease.
+  // Check if we called CFRetain/CFRelease/CFMakeCollectable.
   const IdentifierInfo *FuncII = FD->getIdentifier();
-  if (!(FuncII == Retain || FuncII == Release))
+  if (!(FuncII == Retain || FuncII == Release || FuncII == MakeCollectable))
     return;
 
   // FIXME: The rest of this just checks that the argument is non-null.
@@ -426,14 +428,20 @@ void CFRetainReleaseChecker::checkPreStmt(const CallExpr *CE,
     if (!N)
       return;
 
-    const char *description = (FuncII == Retain)
-                            ? "Null pointer argument in call to CFRetain"
-                            : "Null pointer argument in call to CFRelease";
+    const char *description;
+    if (FuncII == Retain)
+      description = "Null pointer argument in call to CFRetain";
+    else if (FuncII == Release)
+      description = "Null pointer argument in call to CFRelease";
+    else if (FuncII == MakeCollectable)
+      description = "Null pointer argument in call to CFMakeCollectable";
+    else
+      llvm_unreachable("impossible case");
 
     BugReport *report = new BugReport(*BT, description, N);
     report->addRange(Arg->getSourceRange());
-    bugreporter::addTrackNullOrUndefValueVisitor(N, Arg, report);
-    C.EmitReport(report);
+    bugreporter::trackNullOrUndefValue(N, Arg, *report);
+    C.emitReport(report);
     return;
   }
 
@@ -491,7 +499,7 @@ void ClassReleaseChecker::checkPreObjCMessage(const ObjCMethodCall &msg,
   
     BugReport *report = new BugReport(*BT, os.str(), N);
     report->addRange(msg.getSourceRange());
-    C.EmitReport(report);
+    C.emitReport(report);
   }
 }
 
@@ -644,7 +652,7 @@ void VariadicMethodTypeChecker::checkPreObjCMessage(const ObjCMethodCall &msg,
 
     BugReport *R = new BugReport(*BT, os.str(), errorNode.getValue());
     R->addRange(msg.getArgSourceRange(I));
-    C.EmitReport(R);
+    C.emitReport(R);
   }
 }
 
@@ -716,6 +724,73 @@ void ObjCLoopChecker::checkPostStmt(const ObjCForCollectionStmt *FCS,
   C.addTransition(State);
 }
 
+namespace {
+/// \class ObjCNonNilReturnValueChecker
+/// \brief The checker restricts the return values of APIs known to
+/// never (or almost never) return 'nil'.
+class ObjCNonNilReturnValueChecker
+  : public Checker<check::PostObjCMessage> {
+    mutable bool Initialized;
+    mutable Selector ObjectAtIndex;
+    mutable Selector ObjectAtIndexedSubscript;
+
+public:
+  ObjCNonNilReturnValueChecker() : Initialized(false) {}
+  void checkPostObjCMessage(const ObjCMethodCall &M, CheckerContext &C) const;
+};
+}
+
+static ProgramStateRef assumeExprIsNonNull(const Expr *NonNullExpr,
+                                           ProgramStateRef State,
+                                           CheckerContext &C) {
+  SVal Val = State->getSVal(NonNullExpr, C.getLocationContext());
+  if (DefinedOrUnknownSVal *DV = dyn_cast<DefinedOrUnknownSVal>(&Val))
+    return State->assume(*DV, true);
+  return State;
+}
+
+void ObjCNonNilReturnValueChecker::checkPostObjCMessage(const ObjCMethodCall &M,
+                                                        CheckerContext &C)
+                                                        const {
+  ProgramStateRef State = C.getState();
+
+  if (!Initialized) {
+    ASTContext &Ctx = C.getASTContext();
+    ObjectAtIndex = GetUnarySelector("objectAtIndex", Ctx);
+    ObjectAtIndexedSubscript = GetUnarySelector("objectAtIndexedSubscript", Ctx);
+  }
+
+  // Check the receiver type.
+  if (const ObjCInterfaceDecl *Interface = M.getReceiverInterface()) {
+
+    // Assume that object returned from '[self init]' or '[super init]' is not
+    // 'nil' if we are processing an inlined function/method.
+    //
+    // A defensive callee will (and should) check if the object returned by
+    // '[super init]' is 'nil' before doing it's own initialization. However,
+    // since 'nil' is rarely returned in practice, we should not warn when the
+    // caller to the defensive constructor uses the object in contexts where
+    // 'nil' is not accepted.
+    if (!C.inTopFrame() && M.getDecl() &&
+        M.getDecl()->getMethodFamily() == OMF_init &&
+        M.isReceiverSelfOrSuper()) {
+      State = assumeExprIsNonNull(M.getOriginExpr(), State, C);
+    }
+
+    // Objects returned from
+    // [NSArray|NSOrderedSet]::[ObjectAtIndex|ObjectAtIndexedSubscript]
+    // are never 'nil'.
+    FoundationClass Cl = findKnownClass(Interface);
+    if (Cl == FC_NSArray || Cl == FC_NSOrderedSet) {
+      Selector Sel = M.getSelector();
+      if (Sel == ObjectAtIndex || Sel == ObjectAtIndexedSubscript) {
+        // Go ahead and assume the value is non-nil.
+        State = assumeExprIsNonNull(M.getOriginExpr(), State, C);
+      }
+    }
+  }
+  C.addTransition(State);
+}
 
 //===----------------------------------------------------------------------===//
 // Check registration.
@@ -744,3 +819,7 @@ void ento::registerVariadicMethodTypeChecker(CheckerManager &mgr) {
 void ento::registerObjCLoopChecker(CheckerManager &mgr) {
   mgr.registerChecker<ObjCLoopChecker>();
 }
+
+void ento::registerObjCNonNilReturnValueChecker(CheckerManager &mgr) {
+  mgr.registerChecker<ObjCNonNilReturnValueChecker>();
+}
diff --git a/lib/StaticAnalyzer/Checkers/BoolAssignmentChecker.cpp b/lib/StaticAnalyzer/Checkers/BoolAssignmentChecker.cpp
index a4fc396..92edefe 100644
--- a/lib/StaticAnalyzer/Checkers/BoolAssignmentChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/BoolAssignmentChecker.cpp
@@ -35,7 +35,7 @@ void BoolAssignmentChecker::emitReport(ProgramStateRef state,
   if (ExplodedNode *N = C.addTransition(state)) {
     if (!BT)
       BT.reset(new BuiltinBug("Assignment of a non-Boolean value"));    
-    C.EmitReport(new BugReport(*BT, BT->getDescription(), N));
+    C.emitReport(new BugReport(*BT, BT->getDescription(), N));
   }
 }
 
diff --git a/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp b/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp
index 509bc79..6ef022b 100644
--- a/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/BuiltinFunctionChecker.cpp
@@ -55,7 +55,7 @@ bool BuiltinFunctionChecker::evalCall(const CallExpr *CE,
     // FIXME: Refactor into StoreManager itself?
     MemRegionManager& RM = C.getStoreManager().getRegionManager();
     const AllocaRegion* R =
-      RM.getAllocaRegion(CE, C.getCurrentBlockCount(), C.getLocationContext());
+      RM.getAllocaRegion(CE, C.blockCount(), C.getLocationContext());
 
     // Set the extent of the region in bytes. This enables us to use the
     // SVal of the argument directly. If we save the extent in bits, we
diff --git a/lib/StaticAnalyzer/Checkers/CMakeLists.txt b/lib/StaticAnalyzer/Checkers/CMakeLists.txt
index 7fe51d3..8e455de 100644
--- a/lib/StaticAnalyzer/Checkers/CMakeLists.txt
+++ b/lib/StaticAnalyzer/Checkers/CMakeLists.txt
@@ -4,7 +4,6 @@ clang_tablegen(Checkers.inc -gen-clang-sa-checkers
   TARGET ClangSACheckers)
 
 add_clang_library(clangStaticAnalyzerCheckers
-  AdjustedReturnValueChecker.cpp
   AnalyzerStatsChecker.cpp
   ArrayBoundChecker.cpp
   ArrayBoundCheckerV2.cpp
@@ -28,12 +27,15 @@ add_clang_library(clangStaticAnalyzerCheckers
   DeadStoresChecker.cpp
   DebugCheckers.cpp
   DereferenceChecker.cpp
+  DirectIvarAssignment.cpp
   DivZeroChecker.cpp
   DynamicTypePropagation.cpp
   ExprInspectionChecker.cpp
+  SimpleStreamChecker.cpp
   FixedAddressChecker.cpp
   GenericTaintChecker.cpp
   IdempotentOperationChecker.cpp
+  IvarInvalidationChecker.cpp
   LLVMConventionsChecker.cpp
   MacOSKeychainAPIChecker.cpp
   MacOSXAPIChecker.cpp
@@ -43,10 +45,10 @@ add_clang_library(clangStaticAnalyzerCheckers
   NSAutoreleasePoolChecker.cpp
   NSErrorChecker.cpp
   NoReturnFunctionChecker.cpp
-  OSAtomicChecker.cpp
   ObjCAtSyncChecker.cpp
   ObjCContainersASTChecker.cpp
   ObjCContainersChecker.cpp
+  ObjCMissingSuperCallChecker.cpp
   ObjCSelfInitChecker.cpp
   ObjCUnusedIVarsChecker.cpp
   PointerArithChecker.cpp
diff --git a/lib/StaticAnalyzer/Checkers/CStringChecker.cpp b/lib/StaticAnalyzer/Checkers/CStringChecker.cpp
index 483082a..eae9ddf 100644
--- a/lib/StaticAnalyzer/Checkers/CStringChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/CStringChecker.cpp
@@ -188,21 +188,9 @@ public:
                                             NonLoc right) const;
 };
 
-class CStringLength {
-public:
-  typedef llvm::ImmutableMap<const MemRegion *, SVal> EntryMap;
-};
 } //end anonymous namespace
 
-namespace clang {
-namespace ento {
-  template <>
-  struct ProgramStateTrait<CStringLength> 
-    : public ProgramStatePartialTrait<CStringLength::EntryMap> {
-    static void *GDMIndex() { return CStringChecker::getTag(); }
-  };
-}
-}
+REGISTER_MAP_WITH_PROGRAMSTATE(CStringLength, const MemRegion *, SVal)
 
 //===----------------------------------------------------------------------===//
 // Individual checks and utility methods.
@@ -252,8 +240,8 @@ ProgramStateRef CStringChecker::checkNonNull(CheckerContext &C,
     BugReport *report = new BugReport(*BT, os.str(), N);
 
     report->addRange(S->getSourceRange());
-    bugreporter::addTrackNullOrUndefValueVisitor(N, S, report);
-    C.EmitReport(report);
+    bugreporter::trackNullOrUndefValue(N, S, *report);
+    C.emitReport(report);
     return NULL;
   }
 
@@ -327,7 +315,7 @@ ProgramStateRef CStringChecker::CheckLocation(CheckerContext &C,
     // reference is outside the range.
 
     report->addRange(S->getSourceRange());
-    C.EmitReport(report);
+    C.emitReport(report);
     return NULL;
   }
   
@@ -544,7 +532,7 @@ void CStringChecker::emitOverlapBug(CheckerContext &C, ProgramStateRef state,
   report->addRange(First->getSourceRange());
   report->addRange(Second->getSourceRange());
 
-  C.EmitReport(report);
+  C.emitReport(report);
 }
 
 ProgramStateRef CStringChecker::checkAdditionOverflow(CheckerContext &C,
@@ -607,7 +595,7 @@ ProgramStateRef CStringChecker::checkAdditionOverflow(CheckerContext &C,
 
       // Generate a report for this bug.
       BugReport *report = new BugReport(*BT_AdditionOverflow, warning, N);
-      C.EmitReport(report);        
+      C.emitReport(report);        
 
       return NULL;
     }
@@ -673,11 +661,11 @@ SVal CStringChecker::getCStringLengthForRegion(CheckerContext &C,
   }
   
   // Otherwise, get a new symbol and update the state.
-  unsigned Count = C.getCurrentBlockCount();
   SValBuilder &svalBuilder = C.getSValBuilder();
   QualType sizeTy = svalBuilder.getContext().getSizeType();
   SVal strLength = svalBuilder.getMetadataSymbolVal(CStringChecker::getTag(),
-                                                    MR, Ex, sizeTy, Count);
+                                                    MR, Ex, sizeTy,
+                                                    C.blockCount());
 
   if (!hypothetical)
     state = state->set<CStringLength>(MR, strLength);
@@ -714,7 +702,7 @@ SVal CStringChecker::getCStringLength(CheckerContext &C, ProgramStateRef &state,
                                                           os.str(), N);
 
         report->addRange(Ex->getSourceRange());
-        C.EmitReport(report);        
+        C.emitReport(report);        
       }
       return UndefinedVal();
 
@@ -778,7 +766,7 @@ SVal CStringChecker::getCStringLength(CheckerContext &C, ProgramStateRef &state,
                                                         os.str(), N);
 
       report->addRange(Ex->getSourceRange());
-      C.EmitReport(report);        
+      C.emitReport(report);        
     }
 
     return UndefinedVal();
@@ -826,15 +814,14 @@ ProgramStateRef CStringChecker::InvalidateBuffer(CheckerContext &C,
     }
 
     // Invalidate this region.
-    unsigned Count = C.getCurrentBlockCount();
     const LocationContext *LCtx = C.getPredecessor()->getLocationContext();
-    return state->invalidateRegions(R, E, Count, LCtx);
+    return state->invalidateRegions(R, E, C.blockCount(), LCtx);
   }
 
   // If we have a non-region value by chance, just remove the binding.
   // FIXME: is this necessary or correct? This handles the non-Region
   //  cases.  Is it ever valid to store to these?
-  return state->unbindLoc(*L);
+  return state->killBinding(*L);
 }
 
 bool CStringChecker::SummarizeRegion(raw_ostream &os, ASTContext &Ctx,
@@ -843,7 +830,7 @@ bool CStringChecker::SummarizeRegion(raw_ostream &os, ASTContext &Ctx,
 
   switch (MR->getKind()) {
   case MemRegion::FunctionTextRegionKind: {
-    const FunctionDecl *FD = cast<FunctionTextRegion>(MR)->getDecl();
+    const NamedDecl *FD = cast<FunctionTextRegion>(MR)->getDecl();
     if (FD)
       os << "the address of the function '" << *FD << '\'';
     else
@@ -957,9 +944,8 @@ void CStringChecker::evalCopyCommon(CheckerContext &C,
       } else {
         // If we don't know how much we copied, we can at least
         // conjure a return value for later.
-        unsigned Count = C.getCurrentBlockCount();
-        SVal result =
-          C.getSValBuilder().getConjuredSymbolVal(NULL, CE, LCtx, Count);
+        SVal result = C.getSValBuilder().conjureSymbolVal(0, CE, LCtx,
+                                                          C.blockCount());
         state = state->BindExpr(CE, LCtx, result);
       }
 
@@ -1093,8 +1079,7 @@ void CStringChecker::evalMemcmp(CheckerContext &C, const CallExpr *CE) const {
       state = CheckBufferAccess(C, state, Size, Left, Right);
       if (state) {
         // The return value is the comparison result, which we don't know.
-        unsigned Count = C.getCurrentBlockCount();
-        SVal CmpV = svalBuilder.getConjuredSymbolVal(NULL, CE, LCtx, Count);
+        SVal CmpV = svalBuilder.conjureSymbolVal(0, CE, LCtx, C.blockCount());
         state = state->BindExpr(CE, LCtx, CmpV);
         C.addTransition(state);
       }
@@ -1206,8 +1191,7 @@ void CStringChecker::evalstrLengthCommon(CheckerContext &C, const CallExpr *CE,
       // no guarantee the full string length will actually be returned.
       // All we know is the return value is the min of the string length
       // and the limit. This is better than nothing.
-      unsigned Count = C.getCurrentBlockCount();
-      result = C.getSValBuilder().getConjuredSymbolVal(NULL, CE, LCtx, Count);
+      result = C.getSValBuilder().conjureSymbolVal(0, CE, LCtx, C.blockCount());
       NonLoc *resultNL = cast<NonLoc>(&result);
 
       if (strLengthNL) {
@@ -1234,8 +1218,7 @@ void CStringChecker::evalstrLengthCommon(CheckerContext &C, const CallExpr *CE,
     // If we don't know the length of the string, conjure a return
     // value, so it can be used in constraints, at least.
     if (result.isUnknown()) {
-      unsigned Count = C.getCurrentBlockCount();
-      result = C.getSValBuilder().getConjuredSymbolVal(NULL, CE, LCtx, Count);
+      result = C.getSValBuilder().conjureSymbolVal(0, CE, LCtx, C.blockCount());
     }
   }
 
@@ -1612,8 +1595,7 @@ void CStringChecker::evalStrcpyCommon(CheckerContext &C, const CallExpr *CE,
   // If this is a stpcpy-style copy, but we were unable to check for a buffer
   // overflow, we still need a result. Conjure a return value.
   if (returnEnd && Result.isUnknown()) {
-    unsigned Count = C.getCurrentBlockCount();
-    Result = svalBuilder.getConjuredSymbolVal(NULL, CE, LCtx, Count);
+    Result = svalBuilder.conjureSymbolVal(0, CE, LCtx, C.blockCount());
   }
 
   // Set the return value.
@@ -1770,8 +1752,7 @@ void CStringChecker::evalStrcmpCommon(CheckerContext &C, const CallExpr *CE,
 
   if (!canComputeResult) {
     // Conjure a symbolic value. It's the best we can do.
-    unsigned Count = C.getCurrentBlockCount();
-    SVal resultVal = svalBuilder.getConjuredSymbolVal(NULL, CE, LCtx, Count);
+    SVal resultVal = svalBuilder.conjureSymbolVal(0, CE, LCtx, C.blockCount());
     state = state->BindExpr(CE, LCtx, resultVal);
   }
 
@@ -1885,7 +1866,7 @@ void CStringChecker::checkPreStmt(const DeclStmt *DS, CheckerContext &C) const {
 }
 
 bool CStringChecker::wantsRegionChangeUpdate(ProgramStateRef state) const {
-  CStringLength::EntryMap Entries = state->get<CStringLength>();
+  CStringLengthTy Entries = state->get<CStringLength>();
   return !Entries.isEmpty();
 }
 
@@ -1895,7 +1876,7 @@ CStringChecker::checkRegionChanges(ProgramStateRef state,
                                    ArrayRef<const MemRegion *> ExplicitRegions,
                                    ArrayRef<const MemRegion *> Regions,
                                    const CallEvent *Call) const {
-  CStringLength::EntryMap Entries = state->get<CStringLength>();
+  CStringLengthTy Entries = state->get<CStringLength>();
   if (Entries.isEmpty())
     return state;
 
@@ -1915,10 +1896,10 @@ CStringChecker::checkRegionChanges(ProgramStateRef state,
     }
   }
 
-  CStringLength::EntryMap::Factory &F = state->get_context<CStringLength>();
+  CStringLengthTy::Factory &F = state->get_context<CStringLength>();
 
   // Then loop over the entries in the current state.
-  for (CStringLength::EntryMap::iterator I = Entries.begin(),
+  for (CStringLengthTy::iterator I = Entries.begin(),
        E = Entries.end(); I != E; ++I) {
     const MemRegion *MR = I.getKey();
 
@@ -1945,9 +1926,9 @@ CStringChecker::checkRegionChanges(ProgramStateRef state,
 void CStringChecker::checkLiveSymbols(ProgramStateRef state,
                                       SymbolReaper &SR) const {
   // Mark all symbols in our string length map as valid.
-  CStringLength::EntryMap Entries = state->get<CStringLength>();
+  CStringLengthTy Entries = state->get<CStringLength>();
 
-  for (CStringLength::EntryMap::iterator I = Entries.begin(), E = Entries.end();
+  for (CStringLengthTy::iterator I = Entries.begin(), E = Entries.end();
        I != E; ++I) {
     SVal Len = I.getData();
 
@@ -1963,12 +1944,12 @@ void CStringChecker::checkDeadSymbols(SymbolReaper &SR,
     return;
 
   ProgramStateRef state = C.getState();
-  CStringLength::EntryMap Entries = state->get<CStringLength>();
+  CStringLengthTy Entries = state->get<CStringLength>();
   if (Entries.isEmpty())
     return;
 
-  CStringLength::EntryMap::Factory &F = state->get_context<CStringLength>();
-  for (CStringLength::EntryMap::iterator I = Entries.begin(), E = Entries.end();
+  CStringLengthTy::Factory &F = state->get_context<CStringLength>();
+  for (CStringLengthTy::iterator I = Entries.begin(), E = Entries.end();
        I != E; ++I) {
     SVal Len = I.getData();
     if (SymbolRef Sym = Len.getAsSymbol()) {
diff --git a/lib/StaticAnalyzer/Checkers/CStringSyntaxChecker.cpp b/lib/StaticAnalyzer/Checkers/CStringSyntaxChecker.cpp
index befc935..f1a3aac 100644
--- a/lib/StaticAnalyzer/Checkers/CStringSyntaxChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/CStringSyntaxChecker.cpp
@@ -33,7 +33,6 @@ namespace {
 class WalkAST: public StmtVisitor<WalkAST> {
   BugReporter &BR;
   AnalysisDeclContext* AC;
-  ASTContext &ASTC;
 
   /// Check if two expressions refer to the same declaration.
   inline bool sameDecl(const Expr *A1, const Expr *A2) {
@@ -58,8 +57,8 @@ class WalkAST: public StmtVisitor<WalkAST> {
       const FunctionDecl *FD = CE->getDirectCallee();
       if (!FD)
         return false;
-      return (CheckerContext::isCLibraryFunction(FD, "strlen", ASTC)
-          && sameDecl(CE->getArg(0), WithArg));
+      return (CheckerContext::isCLibraryFunction(FD, "strlen") &&
+              sameDecl(CE->getArg(0), WithArg));
     }
     return false;
   }
@@ -83,7 +82,7 @@ class WalkAST: public StmtVisitor<WalkAST> {
 
 public:
   WalkAST(BugReporter &br, AnalysisDeclContext* ac) :
-      BR(br), AC(ac), ASTC(AC->getASTContext()) {
+      BR(br), AC(ac) {
   }
 
   // Statement visitor methods.
@@ -136,7 +135,7 @@ void WalkAST::VisitCallExpr(CallExpr *CE) {
   if (!FD)
     return;
 
-  if (CheckerContext::isCLibraryFunction(FD, "strncat", ASTC)) {
+  if (CheckerContext::isCLibraryFunction(FD, "strncat")) {
     if (containsBadStrncatPattern(CE)) {
       const Expr *DstArg = CE->getArg(0);
       const Expr *LenArg = CE->getArg(2);
diff --git a/lib/StaticAnalyzer/Checkers/CallAndMessageChecker.cpp b/lib/StaticAnalyzer/Checkers/CallAndMessageChecker.cpp
index 5edcf09..82bc136 100644
--- a/lib/StaticAnalyzer/Checkers/CallAndMessageChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/CallAndMessageChecker.cpp
@@ -75,13 +75,13 @@ void CallAndMessageChecker::emitBadCall(BugType *BT, CheckerContext &C,
   BugReport *R = new BugReport(*BT, BT->getName(), N);
   if (BadE) {
     R->addRange(BadE->getSourceRange());
-    bugreporter::addTrackNullOrUndefValueVisitor(N, BadE, R);
+    bugreporter::trackNullOrUndefValue(N, BadE, *R);
   }
-  C.EmitReport(R);
+  C.emitReport(R);
 }
 
-StringRef describeUninitializedArgumentInCall(const CallEvent &Call,
-                                              bool IsFirstArgument) {
+static StringRef describeUninitializedArgumentInCall(const CallEvent &Call,
+                                                     bool IsFirstArgument) {
   switch (Call.getKind()) {
   case CE_ObjCMessage: {
     const ObjCMethodCall &Msg = cast<ObjCMethodCall>(Call);
@@ -122,8 +122,8 @@ bool CallAndMessageChecker::PreVisitProcessArg(CheckerContext &C,
       BugReport *R = new BugReport(*BT, Desc, N);
       R->addRange(argRange);
       if (argEx)
-        bugreporter::addTrackNullOrUndefValueVisitor(N, argEx, R);
-      C.EmitReport(R);
+        bugreporter::trackNullOrUndefValue(N, argEx, *R);
+      C.emitReport(R);
     }
     return true;
   }
@@ -207,7 +207,7 @@ bool CallAndMessageChecker::PreVisitProcessArg(CheckerContext &C,
 
         // FIXME: enhance track back for uninitialized value for arbitrary
         // memregions
-        C.EmitReport(R);
+        C.emitReport(R);
       }
       return true;
     }
@@ -335,8 +335,8 @@ void CallAndMessageChecker::checkPreObjCMessage(const ObjCMethodCall &msg,
 
       // FIXME: getTrackNullOrUndefValueVisitor can't handle "super" yet.
       if (const Expr *ReceiverE = ME->getInstanceReceiver())
-        bugreporter::addTrackNullOrUndefValueVisitor(N, ReceiverE, R);
-      C.EmitReport(R);
+        bugreporter::trackNullOrUndefValue(N, ReceiverE, *R);
+      C.emitReport(R);
     }
     return;
   } else {
@@ -377,9 +377,9 @@ void CallAndMessageChecker::emitNilReceiverBug(CheckerContext &C,
   report->addRange(ME->getReceiverRange());
   // FIXME: This won't track "self" in messages to super.
   if (const Expr *receiver = ME->getInstanceReceiver()) {
-    bugreporter::addTrackNullOrUndefValueVisitor(N, receiver, report);
+    bugreporter::trackNullOrUndefValue(N, receiver, *report);
   }
-  C.EmitReport(report);
+  C.emitReport(report);
 }
 
 static bool supportsNilWithFloatRet(const llvm::Triple &triple) {
diff --git a/lib/StaticAnalyzer/Checkers/CastSizeChecker.cpp b/lib/StaticAnalyzer/Checkers/CastSizeChecker.cpp
index 2e184fb..1cb8a8d 100644
--- a/lib/StaticAnalyzer/Checkers/CastSizeChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/CastSizeChecker.cpp
@@ -75,7 +75,7 @@ void CastSizeChecker::checkPreStmt(const CastExpr *CE,CheckerContext &C) const {
       BugReport *R = new BugReport(*BT, BT->getDescription(),
                                                errorNode);
       R->addRange(CE->getSourceRange());
-      C.EmitReport(R);
+      C.emitReport(R);
     }
   }
 }
diff --git a/lib/StaticAnalyzer/Checkers/CastToStructChecker.cpp b/lib/StaticAnalyzer/Checkers/CastToStructChecker.cpp
index 1407638..d6d0e3c 100644
--- a/lib/StaticAnalyzer/Checkers/CastToStructChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/CastToStructChecker.cpp
@@ -64,7 +64,7 @@ void CastToStructChecker::checkPreStmt(const CastExpr *CE,
                             "errors or data corruption."));
       BugReport *R = new BugReport(*BT,BT->getDescription(), N);
       R->addRange(CE->getSourceRange());
-      C.EmitReport(R);
+      C.emitReport(R);
     }
   }
 }
diff --git a/lib/StaticAnalyzer/Checkers/CheckObjCDealloc.cpp b/lib/StaticAnalyzer/Checkers/CheckObjCDealloc.cpp
index 7a25865..9087205 100644
--- a/lib/StaticAnalyzer/Checkers/CheckObjCDealloc.cpp
+++ b/lib/StaticAnalyzer/Checkers/CheckObjCDealloc.cpp
@@ -85,7 +85,7 @@ static bool scan_ivar_release(Stmt *S, ObjCIvarDecl *ID,
                                             Expr::NPC_ValueDependentIsNull)) {
               // This is only a 'release' if the property kind is not
               // 'assign'.
-              return PD->getSetterKind() != ObjCPropertyDecl::Assign;;
+              return PD->getSetterKind() != ObjCPropertyDecl::Assign;
             }
 
   // Recurse to children.
diff --git a/lib/StaticAnalyzer/Checkers/CheckSecuritySyntaxOnly.cpp b/lib/StaticAnalyzer/Checkers/CheckSecuritySyntaxOnly.cpp
index b8b7c36..5cd6194 100644
--- a/lib/StaticAnalyzer/Checkers/CheckSecuritySyntaxOnly.cpp
+++ b/lib/StaticAnalyzer/Checkers/CheckSecuritySyntaxOnly.cpp
@@ -270,6 +270,7 @@ void WalkAST::checkLoopConditionForFloat(const ForStmt *FS) {
 
   // Emit the error.  First figure out which DeclRefExpr in the condition
   // referenced the compared variable.
+  assert(drInc->getDecl());
   const DeclRefExpr *drCond = vdLHS == drInc->getDecl() ? drLHS : drRHS;
 
   SmallVector<SourceRange, 2> ranges;
diff --git a/lib/StaticAnalyzer/Checkers/CheckerDocumentation.cpp b/lib/StaticAnalyzer/Checkers/CheckerDocumentation.cpp
index 0e9efaa..efaec2b 100644
--- a/lib/StaticAnalyzer/Checkers/CheckerDocumentation.cpp
+++ b/lib/StaticAnalyzer/Checkers/CheckerDocumentation.cpp
@@ -25,6 +25,7 @@ using namespace ento;
 // All checkers should be placed into anonymous namespace.
 // We place the CheckerDocumentation inside ento namespace to make the
 // it visible in doxygen.
+namespace clang {
 namespace ento {
 
 /// This checker documents the callback functions checkers can use to implement
@@ -33,8 +34,8 @@ namespace ento {
 /// checking.
 ///
 /// \sa CheckerContext
-class CheckerDocumentation : public Checker< check::PreStmt<DeclStmt>,
-                                       check::PostStmt<CallExpr>,
+class CheckerDocumentation : public Checker< check::PreStmt<ReturnStmt>,
+                                       check::PostStmt<DeclStmt>,
                                        check::PreObjCMessage,
                                        check::PostObjCMessage,
                                        check::PreCall,
@@ -64,8 +65,8 @@ public:
   /// See checkBranchCondition() callback for performing custom processing of
   /// the branching statements.
   ///
-  /// check::PreStmt<DeclStmt>
-  void checkPreStmt(const DeclStmt *DS, CheckerContext &C) const {}
+  /// check::PreStmt<ReturnStmt>
+  void checkPreStmt(const ReturnStmt *DS, CheckerContext &C) const {}
 
   /// \brief Post-visit the Statement.
   ///
@@ -74,8 +75,8 @@ public:
   /// which does not include the control flow statements such as IfStmt. The
   /// callback can be specialized to be called with any subclass of Stmt.
   ///
-  /// check::PostStmt<CallExpr>
-  void checkPostStmt(const CallExpr *DS, CheckerContext &C) const;
+  /// check::PostStmt<DeclStmt>
+  void checkPostStmt(const DeclStmt *DS, CheckerContext &C) const;
 
   /// \brief Pre-visit the Objective C message.
   ///
@@ -98,8 +99,8 @@ public:
   /// behavior for functions and methods no matter how they are being invoked.
   ///
   /// Note that this includes ALL cross-body invocations, so if you want to
-  /// limit your checks to, say, function calls, you can either test for that
-  /// or fall back to the explicit callback (i.e. check::PreStmt).
+  /// limit your checks to, say, function calls, you should test for that at the
+  /// beginning of your callback function.
   ///
   /// check::PreCall
   void checkPreCall(const CallEvent &Call, CheckerContext &C) const {}
@@ -151,9 +152,8 @@ public:
   /// check::DeadSymbols
   void checkDeadSymbols(SymbolReaper &SR, CheckerContext &C) const {}
 
-  /// \brief Called when an end of path is reached in the ExplodedGraph.
-  ///
-  /// This callback should be used to check if the allocated resources are freed.
+  /// \brief Called when the analyzer core reaches the end of the top-level
+  /// function being analyzed.
   ///
   /// check::EndPath
   void checkEndPath(CheckerContext &Ctx) const {}
@@ -213,21 +213,35 @@ public:
   /// check::LiveSymbols
   void checkLiveSymbols(ProgramStateRef State, SymbolReaper &SR) const {}
 
-
+  /// \brief Called to determine if the checker currently needs to know if when
+  /// contents of any regions change.
+  ///
+  /// Since it is not necessarily cheap to compute which regions are being
+  /// changed, this allows the analyzer core to skip the more expensive
+  /// #checkRegionChanges when no checkers are tracking any state.
   bool wantsRegionChangeUpdate(ProgramStateRef St) const { return true; }
   
-  /// \brief Allows tracking regions which get invalidated.
+  /// \brief Called when the contents of one or more regions change.
+  ///
+  /// This can occur in many different ways: an explicit bind, a blanket
+  /// invalidation of the region contents, or by passing a region to a function
+  /// call whose behavior the analyzer cannot model perfectly.
   ///
   /// \param State The current program state.
   /// \param Invalidated A set of all symbols potentially touched by the change.
   /// \param ExplicitRegions The regions explicitly requested for invalidation.
-  ///   For example, in the case of a function call, these would be arguments.
-  /// \param Regions The transitive closure of accessible regions,
-  ///   i.e. all regions that may have been touched by this change.
-  /// \param Call The call expression wrapper if the regions are invalidated
-  ///   by a call, 0 otherwise.
-  /// Note, in order to be notified, the checker should also implement the
-  /// wantsRegionChangeUpdate callback.
+  ///        For a function call, this would be the arguments. For a bind, this
+  ///        would be the region being bound to.
+  /// \param Regions The transitive closure of regions accessible from,
+  ///        \p ExplicitRegions, i.e. all regions that may have been touched
+  ///        by this change. For a simple bind, this list will be the same as
+  ///        \p ExplicitRegions, since a bind does not affect the contents of
+  ///        anything accessible through the base region.
+  /// \param Call The opaque call triggering this invalidation. Will be 0 if the
+  ///        change was not triggered by a call.
+  ///
+  /// Note that this callback will not be invoked unless
+  /// #wantsRegionChangeUpdate returns \c true.
   ///
   /// check::RegionChanges
   ProgramStateRef 
@@ -256,9 +270,10 @@ public:
 
 };
 
-void CheckerDocumentation::checkPostStmt(const CallExpr *DS,
+void CheckerDocumentation::checkPostStmt(const DeclStmt *DS,
                                          CheckerContext &C) const {
   return;
 }
 
-} // end namespace
+} // end namespace ento
+} // end namespace clang
diff --git a/lib/StaticAnalyzer/Checkers/Checkers.td b/lib/StaticAnalyzer/Checkers/Checkers.td
index 8110bd0..235e633 100644
--- a/lib/StaticAnalyzer/Checkers/Checkers.td
+++ b/lib/StaticAnalyzer/Checkers/Checkers.td
@@ -13,33 +13,33 @@ include "clang/StaticAnalyzer/Checkers/CheckerBase.td"
 // Packages.
 //===----------------------------------------------------------------------===//
 
-def Experimental : Package<"experimental">;
+def Alpha : Package<"alpha">;
 
 def Core : Package<"core">;
 def CoreBuiltin : Package<"builtin">, InPackage<Core>;
 def CoreUninitialized  : Package<"uninitialized">, InPackage<Core>;
-def CoreExperimental : Package<"core">, InPackage<Experimental>, Hidden;
+def CoreAlpha : Package<"core">, InPackage<Alpha>, Hidden;
 
 def Cplusplus : Package<"cplusplus">;
-def CplusplusExperimental : Package<"cplusplus">, InPackage<Experimental>, Hidden;
+def CplusplusAlpha : Package<"cplusplus">, InPackage<Alpha>, Hidden;
 
 def DeadCode : Package<"deadcode">;
-def DeadCodeExperimental : Package<"deadcode">, InPackage<Experimental>, Hidden;
+def DeadCodeAlpha : Package<"deadcode">, InPackage<Alpha>, Hidden;
 
 def Security : Package <"security">;
 def InsecureAPI : Package<"insecureAPI">, InPackage<Security>;
-def SecurityExperimental : Package<"security">, InPackage<Experimental>, Hidden;
-def Taint : Package<"taint">, InPackage<SecurityExperimental>, Hidden;  
+def SecurityAlpha : Package<"security">, InPackage<Alpha>, Hidden;
+def Taint : Package<"taint">, InPackage<SecurityAlpha>, Hidden;  
 
 def Unix : Package<"unix">;
-def UnixExperimental : Package<"unix">, InPackage<Experimental>, Hidden;
+def UnixAlpha : Package<"unix">, InPackage<Alpha>, Hidden;
 def CString : Package<"cstring">, InPackage<Unix>, Hidden;
-def CStringExperimental : Package<"cstring">, InPackage<UnixExperimental>, Hidden;
+def CStringAlpha : Package<"cstring">, InPackage<UnixAlpha>, Hidden;
 
 def OSX : Package<"osx">;
-def OSXExperimental : Package<"osx">, InPackage<Experimental>, Hidden;
+def OSXAlpha : Package<"osx">, InPackage<Alpha>, Hidden;
 def Cocoa : Package<"cocoa">, InPackage<OSX>;
-def CocoaExperimental : Package<"cocoa">, InPackage<OSXExperimental>, Hidden;
+def CocoaAlpha : Package<"cocoa">, InPackage<OSXAlpha>, Hidden;
 def CoreFoundation : Package<"coreFoundation">, InPackage<OSX>;
 def Containers : Package<"containers">, InPackage<CoreFoundation>;
 
@@ -60,10 +60,6 @@ def CallAndMessageChecker : Checker<"CallAndMessage">,
   HelpText<"Check for logical errors for function calls and Objective-C message expressions (e.g., uninitialized arguments, null function pointers)">,
   DescFile<"CallAndMessageChecker.cpp">;
 
-def AdjustedReturnValueChecker : Checker<"AdjustedReturnValue">,
-  HelpText<"Check to see if the return value of a function call is different than the caller expects (e.g., from calls through function pointers)">,
-  DescFile<"AdjustedReturnValueChecker.cpp">;
-
 def AttrNonNullChecker : Checker<"AttributeNonNull">,
   HelpText<"Check for null pointers passed as arguments to a function whose arguments are marked with the 'nonnull' attribute">,
   DescFile<"AttrNonNullChecker.cpp">;
@@ -90,7 +86,7 @@ def DynamicTypePropagation : Checker<"DynamicTypePropagation">,
 
 } // end "core"
 
-let ParentPackage = CoreExperimental in {
+let ParentPackage = CoreAlpha in {
 
 def BoolAssignmentChecker : Checker<"BoolAssignment">,
   HelpText<"Warn about assigning non-{0,1} values to Boolean variables">,
@@ -120,7 +116,7 @@ def SizeofPointerChecker : Checker<"SizeofPtr">,
   HelpText<"Warn about unintended use of sizeof() on pointer expressions">,
   DescFile<"CheckSizeofPointer.cpp">;
 
-} // end "core.experimental"
+} // end "alpha.core"
 
 //===----------------------------------------------------------------------===//
 // Evaluate "builtin" functions.
@@ -170,13 +166,13 @@ def ReturnUndefChecker : Checker<"UndefReturn">,
 // C++ checkers.
 //===----------------------------------------------------------------------===//
 
-let ParentPackage = CplusplusExperimental in {
+let ParentPackage = CplusplusAlpha in {
 
 def VirtualCallChecker : Checker<"VirtualCall">,
   HelpText<"Check virtual function calls during construction or destruction">, 
   DescFile<"VirtualCallChecker.cpp">;
 
-} // end: "cplusplus.experimental"
+} // end: "alpha.cplusplus"
 
 //===----------------------------------------------------------------------===//
 // Deadcode checkers.
@@ -189,7 +185,7 @@ def DeadStoresChecker : Checker<"DeadStores">,
   DescFile<"DeadStoresChecker.cpp">;
 } // end DeadCode
 
-let ParentPackage = DeadCodeExperimental in {
+let ParentPackage = DeadCodeAlpha in {
 
 def IdempotentOperationChecker : Checker<"IdempotentOperations">,
   HelpText<"Warn about idempotent operations">,
@@ -199,7 +195,7 @@ def UnreachableCodeChecker : Checker<"UnreachableCode">,
   HelpText<"Check unreachable code">,
   DescFile<"UnreachableCodeChecker.cpp">;
 
-} // end "deadcode.experimental"
+} // end "alpha.deadcode"
 
 //===----------------------------------------------------------------------===//
 // Security checkers.
@@ -237,7 +233,7 @@ let ParentPackage = Security in {
     DescFile<"CheckSecuritySyntaxOnly.cpp">;
 }
 
-let ParentPackage = SecurityExperimental in {
+let ParentPackage = SecurityAlpha in {
 
 def ArrayBoundChecker : Checker<"ArrayBound">,
   HelpText<"Warn about buffer overflows (older checker)">,
@@ -255,7 +251,7 @@ def MallocOverflowSecurityChecker : Checker<"MallocOverflow">,
   HelpText<"Check for overflows in the arguments to malloc()">,
   DescFile<"MallocOverflowSecurityChecker.cpp">;
 
-} // end "security.experimental"
+} // end "alpha.security"
 
 //===----------------------------------------------------------------------===//
 // Taint checkers.
@@ -267,7 +263,7 @@ def GenericTaintChecker : Checker<"TaintPropagation">,
   HelpText<"Generate taint information used by other checkers">,
   DescFile<"GenericTaintChecker.cpp">;
 
-} // end "experimental.security.taint"
+} // end "alpha.security.taint"
 
 //===----------------------------------------------------------------------===//
 // Unix API checkers.
@@ -289,7 +285,7 @@ def MallocSizeofChecker : Checker<"MallocSizeof">,
   
 } // end "unix"
 
-let ParentPackage = UnixExperimental in {
+let ParentPackage = UnixAlpha in {
 
 def ChrootChecker : Checker<"Chroot">,
   HelpText<"Check improper use of chroot">,
@@ -307,7 +303,11 @@ def StreamChecker : Checker<"Stream">,
   HelpText<"Check stream handling functions">,
   DescFile<"StreamChecker.cpp">;
 
-} // end "unix.experimental"
+def SimpleStreamChecker : Checker<"SimpleStream">,
+  HelpText<"Check for misuses of stream APIs">,
+  DescFile<"SimpleStreamChecker.cpp">;
+
+} // end "alpha.unix"
 
 let ParentPackage = CString in {
 
@@ -320,7 +320,7 @@ def CStringSyntaxChecker : Checker<"BadSizeArg">,
   DescFile<"CStringSyntaxChecker.cpp">;  
 }
 
-let ParentPackage = CStringExperimental in {
+let ParentPackage = CStringAlpha in {
 
 def CStringOutOfBounds : Checker<"OutOfBounds">,
   HelpText<"Check for out-of-bounds access in string functions">,
@@ -346,11 +346,6 @@ def MacOSXAPIChecker : Checker<"API">,
   HelpText<"Check for proper uses of various Mac OS X APIs">,
   DescFile<"MacOSXAPIChecker.cpp">;
 
-def OSAtomicChecker : Checker<"AtomicCAS">,
-  InPackage<OSX>,
-  HelpText<"Evaluate calls to OSAtomic functions">,
-  DescFile<"OSAtomicChecker.cpp">;
-
 def MacOSKeychainAPIChecker : Checker<"SecKeychainAPI">,
   InPackage<OSX>,
   HelpText<"Check for proper uses of Secure Keychain APIs">,
@@ -397,6 +392,10 @@ def ObjCLoopChecker : Checker<"Loops">,
   HelpText<"Improved modeling of loops using Cocoa collection types">,
   DescFile<"BasicObjCFoundationChecks.cpp">;
 
+def ObjCNonNilReturnValueChecker : Checker<"NonNilReturnValue">,
+  HelpText<"Model the APIs that are guaranteed to return a non-nil value">,
+  DescFile<"BasicObjCFoundationChecks.cpp">;
+
 def NSErrorChecker : Checker<"NSError">,
   HelpText<"Check usage of NSError** parameters">,
   DescFile<"NSErrorChecker.cpp">;
@@ -407,13 +406,25 @@ def RetainCountChecker : Checker<"RetainCount">,
 
 } // end "osx.cocoa"
 
-let ParentPackage = CocoaExperimental in {
+let ParentPackage = CocoaAlpha in {
 
 def ObjCDeallocChecker : Checker<"Dealloc">,
   HelpText<"Warn about Objective-C classes that lack a correct implementation of -dealloc">,
   DescFile<"CheckObjCDealloc.cpp">;
 
-} // end "cocoa.experimental"
+def IvarInvalidationChecker : Checker<"InstanceVariableInvalidation">,
+  HelpText<"Check that the invalidatable instance variables are invalidated in the methods annotated with objc_instance_variable_invalidator">,
+  DescFile<"IvarInvalidationChecker.cpp">;
+
+def DirectIvarAssignment : Checker<"DirectIvarAssignment">,
+  HelpText<"Check that the invalidatable instance variables are invalidated in the methods annotated with objc_instance_variable_invalidator">,
+  DescFile<"DirectIvarAssignment.cpp">;
+
+def ObjCSuperCallChecker : Checker<"MissingSuperCall">,
+  HelpText<"Warn about Objective-C methods that lack a necessary call to super">,
+  DescFile<"ObjCMissingSuperCallChecker.cpp">;
+
+} // end "alpha.osx.cocoa"
 
 let ParentPackage = CoreFoundation in {
 
@@ -422,7 +433,7 @@ def CFNumberCreateChecker : Checker<"CFNumber">,
   DescFile<"BasicObjCFoundationChecks.cpp">;
 
 def CFRetainReleaseChecker : Checker<"CFRetainRelease">,
-  HelpText<"Check for null arguments to CFRetain/CFRelease">,
+  HelpText<"Check for null arguments to CFRetain/CFRelease/CFMakeCollectable">,
   DescFile<"BasicObjCFoundationChecks.cpp">;
 
 def CFErrorChecker : Checker<"CFError">,
@@ -479,6 +490,10 @@ def CallGraphDumper : Checker<"DumpCallGraph">,
   HelpText<"Display Call Graph">,
   DescFile<"DebugCheckers.cpp">;
 
+def ConfigDumper : Checker<"ConfigDumper">,
+  HelpText<"Dump config table">,
+  DescFile<"DebugCheckers.cpp">;
+
 def TraversalDumper : Checker<"DumpTraversal">,
   HelpText<"Print branch conditions as they are traversed by the engine">,
   DescFile<"TraversalChecker.cpp">;
diff --git a/lib/StaticAnalyzer/Checkers/ChrootChecker.cpp b/lib/StaticAnalyzer/Checkers/ChrootChecker.cpp
index 30d0609..c885616 100644
--- a/lib/StaticAnalyzer/Checkers/ChrootChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/ChrootChecker.cpp
@@ -147,7 +147,7 @@ void ChrootChecker::checkPreStmt(const CallExpr *CE, CheckerContext &C) const {
                                         "after chroot"));
         BugReport *R = new BugReport(*BT_BreakJail, 
                                      BT_BreakJail->getDescription(), N);
-        C.EmitReport(R);
+        C.emitReport(R);
       }
   
   return;
diff --git a/lib/StaticAnalyzer/Checkers/DeadStoresChecker.cpp b/lib/StaticAnalyzer/Checkers/DeadStoresChecker.cpp
index 510e8cd..59e03ec 100644
--- a/lib/StaticAnalyzer/Checkers/DeadStoresChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/DeadStoresChecker.cpp
@@ -13,22 +13,54 @@
 //===----------------------------------------------------------------------===//
 
 #include "ClangSACheckers.h"
-#include "clang/StaticAnalyzer/Core/Checker.h"
-#include "clang/Analysis/Analyses/LiveVariables.h"
-#include "clang/Analysis/Visitors/CFGRecStmtVisitor.h"
-#include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
-#include "clang/Analysis/Visitors/CFGRecStmtDeclVisitor.h"
-#include "clang/Basic/Diagnostic.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/ParentMap.h"
-#include "llvm/ADT/SmallPtrSet.h"
+#include "clang/AST/RecursiveASTVisitor.h"
+#include "clang/Analysis/Analyses/LiveVariables.h"
+#include "clang/Analysis/Visitors/CFGRecStmtDeclVisitor.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
+#include "clang/StaticAnalyzer/Core/Checker.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/AnalysisManager.h"
+#include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/SmallString.h"
+#include "llvm/Support/SaveAndRestore.h"
 
 using namespace clang;
 using namespace ento;
 
-namespace {
+namespace {  
+  
+/// A simple visitor to record what VarDecls occur in EH-handling code.
+class EHCodeVisitor : public RecursiveASTVisitor<EHCodeVisitor> {
+public:
+  bool inEH;
+  llvm::DenseSet<const VarDecl *> &S;
+  
+  bool TraverseObjCAtFinallyStmt(ObjCAtFinallyStmt *S) {
+    SaveAndRestore<bool> inFinally(inEH, true);
+    return ::RecursiveASTVisitor<EHCodeVisitor>::TraverseObjCAtFinallyStmt(S);
+  }
+  
+  bool TraverseObjCAtCatchStmt(ObjCAtCatchStmt *S) {
+    SaveAndRestore<bool> inCatch(inEH, true);
+    return ::RecursiveASTVisitor<EHCodeVisitor>::TraverseObjCAtCatchStmt(S);
+  }
+  
+  bool TraverseCXXCatchStmt(CXXCatchStmt *S) {
+    SaveAndRestore<bool> inCatch(inEH, true);
+    return TraverseStmt(S->getHandlerBlock());
+  }
+  
+  bool VisitDeclRefExpr(DeclRefExpr *DR) {
+    if (inEH)
+      if (const VarDecl *D = dyn_cast<VarDecl>(DR->getDecl()))
+        S.insert(D);
+    return true;
+  }
+  
+  EHCodeVisitor(llvm::DenseSet<const VarDecl *> &S) :
+  inEH(false), S(S) {}
+};
 
 // FIXME: Eventually migrate into its own file, and have it managed by
 // AnalysisManager.
@@ -93,6 +125,7 @@ class DeadStoreObs : public LiveVariables::Observer {
   llvm::SmallPtrSet<const VarDecl*, 20> Escaped;
   OwningPtr<ReachableCode> reachableCode;
   const CFGBlock *currentBlock;
+  llvm::OwningPtr<llvm::DenseSet<const VarDecl *> > InEH;
 
   enum DeadStoreKind { Standard, Enclosing, DeadIncrement, DeadInit };
 
@@ -105,6 +138,23 @@ public:
 
   virtual ~DeadStoreObs() {}
 
+  bool isLive(const LiveVariables::LivenessValues &Live, const VarDecl *D) {
+    if (Live.isLive(D))
+      return true;
+    // Lazily construct the set that records which VarDecls are in
+    // EH code.
+    if (!InEH.get()) {
+      InEH.reset(new llvm::DenseSet<const VarDecl *>());
+      EHCodeVisitor V(*InEH.get());
+      V.TraverseStmt(AC->getBody());
+    }
+    // Treat all VarDecls that occur in EH code as being "always live"
+    // when considering to suppress dead stores.  Frequently stores
+    // are followed by reads in EH code, but we don't have the ability
+    // to analyze that yet.
+    return InEH->count(D);
+  }
+  
   void Report(const VarDecl *V, DeadStoreKind dsk,
               PathDiagnosticLocation L, SourceRange R) {
     if (Escaped.count(V))
@@ -159,7 +209,7 @@ public:
     if (VD->getType()->getAs<ReferenceType>())
       return;
 
-    if (!Live.isLive(VD) && 
+    if (!isLive(Live, VD) &&
         !(VD->getAttr<UnusedAttr>() || VD->getAttr<BlocksAttr>())) {
 
       PathDiagnosticLocation ExLoc =
@@ -285,7 +335,7 @@ public:
             // A dead initialization is a variable that is dead after it
             // is initialized.  We don't flag warnings for those variables
             // marked 'unused'.
-            if (!Live.isLive(V) && V->getAttr<UnusedAttr>() == 0) {
+            if (!isLive(Live, V) && V->getAttr<UnusedAttr>() == 0) {
               // Special case: check for initializations with constants.
               //
               //  e.g. : int x = 0;
diff --git a/lib/StaticAnalyzer/Checkers/DebugCheckers.cpp b/lib/StaticAnalyzer/Checkers/DebugCheckers.cpp
index 34053cd..7ad9c59 100644
--- a/lib/StaticAnalyzer/Checkers/DebugCheckers.cpp
+++ b/lib/StaticAnalyzer/Checkers/DebugCheckers.cpp
@@ -144,3 +144,38 @@ public:
 void ento::registerCallGraphDumper(CheckerManager &mgr) {
   mgr.registerChecker<CallGraphDumper>();
 }
+
+
+//===----------------------------------------------------------------------===//
+// ConfigDumper
+//===----------------------------------------------------------------------===//
+
+namespace {
+class ConfigDumper : public Checker< check::EndOfTranslationUnit > {
+public:
+  void checkEndOfTranslationUnit(const TranslationUnitDecl *TU,
+                                 AnalysisManager& mgr,
+                                 BugReporter &BR) const {
+
+    const AnalyzerOptions::ConfigTable &Config = mgr.options.Config;
+    AnalyzerOptions::ConfigTable::const_iterator I =
+      Config.begin(), E = Config.end();
+
+    std::vector<StringRef> Keys;
+    for (; I != E ; ++I) { Keys.push_back(I->getKey()); }
+    sort(Keys.begin(), Keys.end());
+    
+    llvm::errs() << "[config]\n";
+    for (unsigned i = 0, n = Keys.size(); i < n ; ++i) {
+      StringRef Key = Keys[i];
+      I = Config.find(Key);
+      llvm::errs() << Key << " = " << I->second << '\n';
+    }
+    llvm::errs() << "[stats]\n" << "num-entries = " << Keys.size() << '\n';
+  }
+};
+}
+
+void ento::registerConfigDumper(CheckerManager &mgr) {
+  mgr.registerChecker<ConfigDumper>();
+}
diff --git a/lib/StaticAnalyzer/Checkers/DereferenceChecker.cpp b/lib/StaticAnalyzer/Checkers/DereferenceChecker.cpp
index e98c131..3ace4be 100644
--- a/lib/StaticAnalyzer/Checkers/DereferenceChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/DereferenceChecker.cpp
@@ -39,7 +39,7 @@ public:
                      CheckerContext &C) const;
   void checkBind(SVal L, SVal V, const Stmt *S, CheckerContext &C) const;
 
-  static const MemRegion *AddDerefSource(raw_ostream &os,
+  static void AddDerefSource(raw_ostream &os,
                              SmallVectorImpl<SourceRange> &Ranges,
                              const Expr *Ex, const ProgramState *state,
                              const LocationContext *LCtx,
@@ -47,7 +47,7 @@ public:
 };
 } // end anonymous namespace
 
-const MemRegion *
+void
 DereferenceChecker::AddDerefSource(raw_ostream &os,
                                    SmallVectorImpl<SourceRange> &Ranges,
                                    const Expr *Ex,
@@ -55,7 +55,6 @@ DereferenceChecker::AddDerefSource(raw_ostream &os,
                                    const LocationContext *LCtx,
                                    bool loadedFrom) {
   Ex = Ex->IgnoreParenLValueCasts();
-  const MemRegion *sourceR = 0;
   switch (Ex->getStmtClass()) {
     default:
       break;
@@ -65,7 +64,6 @@ DereferenceChecker::AddDerefSource(raw_ostream &os,
         os << " (" << (loadedFrom ? "loaded from" : "from")
            << " variable '" <<  VD->getName() << "')";
         Ranges.push_back(DR->getSourceRange());
-        sourceR = state->getLValue(VD, LCtx).getAsRegion();
       }
       break;
     }
@@ -78,7 +76,6 @@ DereferenceChecker::AddDerefSource(raw_ostream &os,
       break;
     }
   }
-  return sourceR;
 }
 
 void DereferenceChecker::reportBug(ProgramStateRef State, const Stmt *S,
@@ -94,6 +91,8 @@ void DereferenceChecker::reportBug(ProgramStateRef State, const Stmt *S,
     BT_null.reset(new BuiltinBug("Dereference of null pointer"));
 
   SmallString<100> buf;
+  llvm::raw_svector_ostream os(buf);
+
   SmallVector<SourceRange, 2> Ranges;
 
   // Walk through lvalue casts to get the original expression
@@ -101,8 +100,6 @@ void DereferenceChecker::reportBug(ProgramStateRef State, const Stmt *S,
   if (const Expr *expr = dyn_cast<Expr>(S))
     S = expr->IgnoreParenLValueCasts();
 
-  const MemRegion *sourceR = 0;
-
   if (IsBind) {
     if (const BinaryOperator *BO = dyn_cast<BinaryOperator>(S)) {
       if (BO->isAssignmentOp())
@@ -117,68 +114,55 @@ void DereferenceChecker::reportBug(ProgramStateRef State, const Stmt *S,
 
   switch (S->getStmtClass()) {
   case Stmt::ArraySubscriptExprClass: {
-    llvm::raw_svector_ostream os(buf);
     os << "Array access";
     const ArraySubscriptExpr *AE = cast<ArraySubscriptExpr>(S);
-    sourceR = AddDerefSource(os, Ranges, AE->getBase()->IgnoreParenCasts(),
-                             State.getPtr(), N->getLocationContext());
+    AddDerefSource(os, Ranges, AE->getBase()->IgnoreParenCasts(),
+                   State.getPtr(), N->getLocationContext());
     os << " results in a null pointer dereference";
     break;
   }
   case Stmt::UnaryOperatorClass: {
-    llvm::raw_svector_ostream os(buf);
     os << "Dereference of null pointer";
     const UnaryOperator *U = cast<UnaryOperator>(S);
-    sourceR = AddDerefSource(os, Ranges, U->getSubExpr()->IgnoreParens(),
-                             State.getPtr(), N->getLocationContext(), true);
+    AddDerefSource(os, Ranges, U->getSubExpr()->IgnoreParens(),
+                   State.getPtr(), N->getLocationContext(), true);
     break;
   }
   case Stmt::MemberExprClass: {
     const MemberExpr *M = cast<MemberExpr>(S);
-    if (M->isArrow()) {
-      llvm::raw_svector_ostream os(buf);
+    if (M->isArrow() || bugreporter::isDeclRefExprToReference(M->getBase())) {
       os << "Access to field '" << M->getMemberNameInfo()
          << "' results in a dereference of a null pointer";
-      sourceR = AddDerefSource(os, Ranges, M->getBase()->IgnoreParenCasts(),
-                               State.getPtr(), N->getLocationContext(), true);
+      AddDerefSource(os, Ranges, M->getBase()->IgnoreParenCasts(),
+                     State.getPtr(), N->getLocationContext(), true);
     }
     break;
   }
   case Stmt::ObjCIvarRefExprClass: {
     const ObjCIvarRefExpr *IV = cast<ObjCIvarRefExpr>(S);
-    if (const DeclRefExpr *DR =
-        dyn_cast<DeclRefExpr>(IV->getBase()->IgnoreParenCasts())) {
-      if (const VarDecl *VD = dyn_cast<VarDecl>(DR->getDecl())) {
-        llvm::raw_svector_ostream os(buf);
-        os << "Instance variable access (via '" << VD->getName()
-           << "') results in a null pointer dereference";
-      }
-    }
-    Ranges.push_back(IV->getSourceRange());
+    os << "Access to instance variable '" << *IV->getDecl()
+       << "' results in a dereference of a null pointer";
+    AddDerefSource(os, Ranges, IV->getBase()->IgnoreParenCasts(),
+                   State.getPtr(), N->getLocationContext(), true);
     break;
   }
   default:
     break;
   }
 
+  os.flush();
   BugReport *report =
     new BugReport(*BT_null,
                   buf.empty() ? BT_null->getDescription() : buf.str(),
                   N);
 
-  bugreporter::addTrackNullOrUndefValueVisitor(N, bugreporter::GetDerefExpr(N),
-                                               report);
+  bugreporter::trackNullOrUndefValue(N, bugreporter::GetDerefExpr(N), *report);
 
   for (SmallVectorImpl<SourceRange>::iterator
        I = Ranges.begin(), E = Ranges.end(); I!=E; ++I)
     report->addRange(*I);
 
-  if (sourceR) {
-    report->markInteresting(sourceR);
-    report->markInteresting(State->getRawSVal(loc::MemRegionVal(sourceR)));
-  }
-
-  C.EmitReport(report);
+  C.emitReport(report);
 }
 
 void DereferenceChecker::checkLocation(SVal l, bool isLoad, const Stmt* S,
@@ -191,11 +175,9 @@ void DereferenceChecker::checkLocation(SVal l, bool isLoad, const Stmt* S,
 
       BugReport *report =
         new BugReport(*BT_undef, BT_undef->getDescription(), N);
-      bugreporter::addTrackNullOrUndefValueVisitor(N,
-                                                   bugreporter::GetDerefExpr(N),
-                                                   report);
-      report->disablePathPruning();
-      C.EmitReport(report);
+      bugreporter::trackNullOrUndefValue(N, bugreporter::GetDerefExpr(N),
+                                         *report);
+      C.emitReport(report);
     }
     return;
   }
diff --git a/lib/StaticAnalyzer/Checkers/DirectIvarAssignment.cpp b/lib/StaticAnalyzer/Checkers/DirectIvarAssignment.cpp
new file mode 100644
index 0000000..dc90b67
--- /dev/null
+++ b/lib/StaticAnalyzer/Checkers/DirectIvarAssignment.cpp
@@ -0,0 +1,180 @@
+//=- DirectIvarAssignment.cpp - Check rules on ObjC properties -*- C++ ----*-==//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  Check that Objective C properties follow the following rules:
+//    - The property should be set with the setter, not though a direct
+//      assignment.
+//
+//===----------------------------------------------------------------------===//
+
+#include "ClangSACheckers.h"
+#include "clang/StaticAnalyzer/Core/Checker.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/AnalysisManager.h"
+#include "clang/AST/DeclObjC.h"
+#include "clang/AST/StmtVisitor.h"
+#include "llvm/ADT/DenseMap.h"
+
+using namespace clang;
+using namespace ento;
+
+namespace {
+
+class DirectIvarAssignment :
+  public Checker<check::ASTDecl<ObjCImplementationDecl> > {
+
+  typedef llvm::DenseMap<const ObjCIvarDecl*,
+                         const ObjCPropertyDecl*> IvarToPropertyMapTy;
+
+  /// A helper class, which walks the AST and locates all assignments to ivars
+  /// in the given function.
+  class MethodCrawler : public ConstStmtVisitor<MethodCrawler> {
+    const IvarToPropertyMapTy &IvarToPropMap;
+    const ObjCMethodDecl *MD;
+    const ObjCInterfaceDecl *InterfD;
+    BugReporter &BR;
+    LocationOrAnalysisDeclContext DCtx;
+
+  public:
+    MethodCrawler(const IvarToPropertyMapTy &InMap, const ObjCMethodDecl *InMD,
+        const ObjCInterfaceDecl *InID,
+        BugReporter &InBR, AnalysisDeclContext *InDCtx)
+    : IvarToPropMap(InMap), MD(InMD), InterfD(InID), BR(InBR), DCtx(InDCtx) {}
+
+    void VisitStmt(const Stmt *S) { VisitChildren(S); }
+
+    void VisitBinaryOperator(const BinaryOperator *BO);
+
+    void VisitChildren(const Stmt *S) {
+      for (Stmt::const_child_range I = S->children(); I; ++I)
+        if (*I)
+         this->Visit(*I);
+    }
+  };
+
+public:
+  void checkASTDecl(const ObjCImplementationDecl *D, AnalysisManager& Mgr,
+                    BugReporter &BR) const;
+};
+
+static const ObjCIvarDecl *findPropertyBackingIvar(const ObjCPropertyDecl *PD,
+                                               const ObjCInterfaceDecl *InterD,
+                                               ASTContext &Ctx) {
+  // Check for synthesized ivars.
+  ObjCIvarDecl *ID = PD->getPropertyIvarDecl();
+  if (ID)
+    return ID;
+
+  ObjCInterfaceDecl *NonConstInterD = const_cast<ObjCInterfaceDecl*>(InterD);
+
+  // Check for existing "_PropName".
+  ID = NonConstInterD->lookupInstanceVariable(PD->getDefaultSynthIvarName(Ctx));
+  if (ID)
+    return ID;
+
+  // Check for existing "PropName".
+  IdentifierInfo *PropIdent = PD->getIdentifier();
+  ID = NonConstInterD->lookupInstanceVariable(PropIdent);
+
+  return ID;
+}
+
+void DirectIvarAssignment::checkASTDecl(const ObjCImplementationDecl *D,
+                                       AnalysisManager& Mgr,
+                                       BugReporter &BR) const {
+  const ObjCInterfaceDecl *InterD = D->getClassInterface();
+
+
+  IvarToPropertyMapTy IvarToPropMap;
+
+  // Find all properties for this class.
+  for (ObjCInterfaceDecl::prop_iterator I = InterD->prop_begin(),
+      E = InterD->prop_end(); I != E; ++I) {
+    ObjCPropertyDecl *PD = *I;
+
+    // Find the corresponding IVar.
+    const ObjCIvarDecl *ID = findPropertyBackingIvar(PD, InterD,
+                                                     Mgr.getASTContext());
+
+    if (!ID)
+      continue;
+
+    // Store the IVar to property mapping.
+    IvarToPropMap[ID] = PD;
+  }
+
+  if (IvarToPropMap.empty())
+    return;
+
+  for (ObjCImplementationDecl::instmeth_iterator I = D->instmeth_begin(),
+      E = D->instmeth_end(); I != E; ++I) {
+
+    ObjCMethodDecl *M = *I;
+    AnalysisDeclContext *DCtx = Mgr.getAnalysisDeclContext(M);
+
+    // Skip the init, dealloc functions and any functions that might be doing
+    // initialization based on their name.
+    if (M->getMethodFamily() == OMF_init ||
+        M->getMethodFamily() == OMF_dealloc ||
+        M->getMethodFamily() == OMF_copy ||
+        M->getMethodFamily() == OMF_mutableCopy ||
+        M->getSelector().getNameForSlot(0).find("init") != StringRef::npos ||
+        M->getSelector().getNameForSlot(0).find("Init") != StringRef::npos)
+      continue;
+
+    const Stmt *Body = M->getBody();
+    assert(Body);
+
+    MethodCrawler MC(IvarToPropMap, M->getCanonicalDecl(), InterD, BR, DCtx);
+    MC.VisitStmt(Body);
+  }
+}
+
+void DirectIvarAssignment::MethodCrawler::VisitBinaryOperator(
+                                                    const BinaryOperator *BO) {
+  if (!BO->isAssignmentOp())
+    return;
+
+  const ObjCIvarRefExpr *IvarRef =
+          dyn_cast<ObjCIvarRefExpr>(BO->getLHS()->IgnoreParenCasts());
+
+  if (!IvarRef)
+    return;
+
+  if (const ObjCIvarDecl *D = IvarRef->getDecl()) {
+    IvarToPropertyMapTy::const_iterator I = IvarToPropMap.find(D);
+    if (I != IvarToPropMap.end()) {
+      const ObjCPropertyDecl *PD = I->second;
+
+      ObjCMethodDecl *GetterMethod =
+          InterfD->getInstanceMethod(PD->getGetterName());
+      ObjCMethodDecl *SetterMethod =
+          InterfD->getInstanceMethod(PD->getSetterName());
+
+      if (SetterMethod && SetterMethod->getCanonicalDecl() == MD)
+        return;
+
+      if (GetterMethod && GetterMethod->getCanonicalDecl() == MD)
+        return;
+
+      BR.EmitBasicReport(MD,
+          "Property access",
+          categories::CoreFoundationObjectiveC,
+          "Direct assignment to an instance variable backing a property; "
+          "use the setter instead", PathDiagnosticLocation(IvarRef,
+                                                          BR.getSourceManager(),
+                                                          DCtx));
+    }
+  }
+}
+}
+
+void ento::registerDirectIvarAssignment(CheckerManager &mgr) {
+  mgr.registerChecker<DirectIvarAssignment>();
+}
diff --git a/lib/StaticAnalyzer/Checkers/DivZeroChecker.cpp b/lib/StaticAnalyzer/Checkers/DivZeroChecker.cpp
index dcf6a86..76fb3f2 100644
--- a/lib/StaticAnalyzer/Checkers/DivZeroChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/DivZeroChecker.cpp
@@ -39,13 +39,9 @@ void DivZeroChecker::reportBug(const char *Msg,
     if (!BT)
       BT.reset(new BuiltinBug("Division by zero"));
 
-    BugReport *R =
-      new BugReport(*BT, Msg, N);
-
-    bugreporter::addTrackNullOrUndefValueVisitor(N,
-                                                 bugreporter::GetDenomExpr(N),
-                                                 R);
-    C.EmitReport(R);
+    BugReport *R = new BugReport(*BT, Msg, N);
+    bugreporter::trackNullOrUndefValue(N, bugreporter::GetDenomExpr(N), *R);
+    C.emitReport(R);
   }
 }
 
diff --git a/lib/StaticAnalyzer/Checkers/DynamicTypePropagation.cpp b/lib/StaticAnalyzer/Checkers/DynamicTypePropagation.cpp
index b636efb..b0a4bc6 100644
--- a/lib/StaticAnalyzer/Checkers/DynamicTypePropagation.cpp
+++ b/lib/StaticAnalyzer/Checkers/DynamicTypePropagation.cpp
@@ -83,14 +83,14 @@ void DynamicTypePropagation::checkPreCall(const CallEvent &Call,
 
   if (const CXXDestructorCall *Dtor = dyn_cast<CXXDestructorCall>(&Call)) {
     // C++11 [class.cdtor]p4 (see above)
+    if (!Dtor->isBaseDestructor())
+      return;
 
     const MemRegion *Target = Dtor->getCXXThisVal().getAsRegion();
     if (!Target)
       return;
 
-    // FIXME: getRuntimeDefinition() can be expensive. It would be better to do
-    // this when we are entering the stack frame for the destructor.
-    const Decl *D = Dtor->getRuntimeDefinition().getDecl();
+    const Decl *D = Dtor->getDecl();
     if (!D)
       return;
 
@@ -105,8 +105,7 @@ void DynamicTypePropagation::checkPostCall(const CallEvent &Call,
   if (const ObjCMethodCall *Msg = dyn_cast<ObjCMethodCall>(&Call)) {
 
     // Get the returned value if it's a region.
-    SVal Result = C.getSVal(Call.getOriginExpr());
-    const MemRegion *RetReg = Result.getAsRegion();
+    const MemRegion *RetReg = Call.getReturnValue().getAsRegion();
     if (!RetReg)
       return;
 
diff --git a/lib/StaticAnalyzer/Checkers/ExprInspectionChecker.cpp b/lib/StaticAnalyzer/Checkers/ExprInspectionChecker.cpp
index 7acf223..e7e3162 100644
--- a/lib/StaticAnalyzer/Checkers/ExprInspectionChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/ExprInspectionChecker.cpp
@@ -93,7 +93,7 @@ void ExprInspectionChecker::analyzerEval(const CallExpr *CE,
     BT.reset(new BugType("Checking analyzer assumptions", "debug"));
 
   BugReport *R = new BugReport(*BT, getArgumentValueString(CE, C), N);
-  C.EmitReport(R);
+  C.emitReport(R);
 }
 
 void ExprInspectionChecker::analyzerCheckInlined(const CallExpr *CE,
@@ -113,7 +113,7 @@ void ExprInspectionChecker::analyzerCheckInlined(const CallExpr *CE,
     BT.reset(new BugType("Checking analyzer assumptions", "debug"));
 
   BugReport *R = new BugReport(*BT, getArgumentValueString(CE, C), N);
-  C.EmitReport(R);
+  C.emitReport(R);
 }
 
 void ento::registerExprInspectionChecker(CheckerManager &Mgr) {
diff --git a/lib/StaticAnalyzer/Checkers/FixedAddressChecker.cpp b/lib/StaticAnalyzer/Checkers/FixedAddressChecker.cpp
index a1f2f3b..7fde689 100644
--- a/lib/StaticAnalyzer/Checkers/FixedAddressChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/FixedAddressChecker.cpp
@@ -58,7 +58,7 @@ void FixedAddressChecker::checkPreStmt(const BinaryOperator *B,
                           "environments or platforms."));
     BugReport *R = new BugReport(*BT, BT->getDescription(), N);
     R->addRange(B->getRHS()->getSourceRange());
-    C.EmitReport(R);
+    C.emitReport(R);
   }
 }
 
diff --git a/lib/StaticAnalyzer/Checkers/GenericTaintChecker.cpp b/lib/StaticAnalyzer/Checkers/GenericTaintChecker.cpp
index afb862c..a9e0217 100644
--- a/lib/StaticAnalyzer/Checkers/GenericTaintChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/GenericTaintChecker.cpp
@@ -192,13 +192,7 @@ const char GenericTaintChecker::MsgTaintedBufferSize[] =
 /// to the call post-visit. The values are unsigned integers, which are either
 /// ReturnValueIndex, or indexes of the pointer/reference argument, which
 /// points to data, which should be tainted on return.
-namespace { struct TaintArgsOnPostVisit{}; }
-namespace clang { namespace ento {
-template<> struct ProgramStateTrait<TaintArgsOnPostVisit>
-    :  public ProgramStatePartialTrait<llvm::ImmutableSet<unsigned> > {
-  static void *GDMIndex() { return GenericTaintChecker::getTag(); }
-};
-}}
+REGISTER_SET_WITH_PROGRAMSTATE(TaintArgsOnPostVisit, unsigned)
 
 GenericTaintChecker::TaintPropagationRule
 GenericTaintChecker::TaintPropagationRule::getTaintPropagationRule(
@@ -337,7 +331,7 @@ bool GenericTaintChecker::propagateFromPre(const CallExpr *CE,
   // Depending on what was tainted at pre-visit, we determined a set of
   // arguments which should be tainted after the function returns. These are
   // stored in the state as TaintArgsOnPostVisit set.
-  llvm::ImmutableSet<unsigned> TaintArgs = State->get<TaintArgsOnPostVisit>();
+  TaintArgsOnPostVisitTy TaintArgs = State->get<TaintArgsOnPostVisit>();
   if (TaintArgs.isEmpty())
     return false;
 
@@ -653,7 +647,7 @@ bool GenericTaintChecker::generateReportIfTainted(const Expr *E,
     initBugType();
     BugReport *report = new BugReport(*BT, Msg, N);
     report->addRange(E->getSourceRange());
-    C.EmitReport(report);
+    C.emitReport(report);
     return true;
   }
   return false;
diff --git a/lib/StaticAnalyzer/Checkers/IdempotentOperationChecker.cpp b/lib/StaticAnalyzer/Checkers/IdempotentOperationChecker.cpp
index 9d0b83f..ffbbb8b 100644
--- a/lib/StaticAnalyzer/Checkers/IdempotentOperationChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/IdempotentOperationChecker.cpp
@@ -430,7 +430,7 @@ void IdempotentOperationChecker::checkEndAnalysis(ExplodedGraph &G,
         FindLastStoreBRVisitor::registerStatementVarDecls(*report, RHS);
       }
 
-      BR.EmitReport(report);
+      BR.emitReport(report);
     }
   }
 
diff --git a/lib/StaticAnalyzer/Checkers/IvarInvalidationChecker.cpp b/lib/StaticAnalyzer/Checkers/IvarInvalidationChecker.cpp
new file mode 100644
index 0000000..bf256cd
--- /dev/null
+++ b/lib/StaticAnalyzer/Checkers/IvarInvalidationChecker.cpp
@@ -0,0 +1,550 @@
+//=- IvarInvalidationChecker.cpp - -*- C++ -------------------------------*-==//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This checker implements annotation driven invalidation checking. If a class
+//  contains a method annotated with 'objc_instance_variable_invalidator',
+//  - (void) foo
+//           __attribute__((annotate("objc_instance_variable_invalidator")));
+//  all the "ivalidatable" instance variables of this class should be
+//  invalidated. We call an instance variable ivalidatable if it is an object of
+//  a class which contains an invalidation method. There could be multiple
+//  methods annotated with such annotations per class, either one can be used
+//  to invalidate the ivar. An ivar or property are considered to be
+//  invalidated if they are being assigned 'nil' or an invalidation method has
+//  been called on them. An invalidation method should either invalidate all
+//  the ivars or call another invalidation method (on self).
+//
+//===----------------------------------------------------------------------===//
+
+#include "ClangSACheckers.h"
+#include "clang/StaticAnalyzer/Core/Checker.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/AnalysisManager.h"
+#include "clang/AST/DeclObjC.h"
+#include "clang/AST/StmtVisitor.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallString.h"
+
+using namespace clang;
+using namespace ento;
+
+namespace {
+class IvarInvalidationChecker :
+  public Checker<check::ASTDecl<ObjCMethodDecl> > {
+
+  typedef llvm::DenseSet<const ObjCMethodDecl*> MethodSet;
+  typedef llvm::DenseMap<const ObjCMethodDecl*,
+                         const ObjCIvarDecl*> MethToIvarMapTy;
+  typedef llvm::DenseMap<const ObjCPropertyDecl*,
+                         const ObjCIvarDecl*> PropToIvarMapTy;
+  typedef llvm::DenseMap<const ObjCIvarDecl*,
+                         const ObjCPropertyDecl*> IvarToPropMapTy;
+
+
+  struct IvarInfo {
+    /// Has the ivar been invalidated?
+    bool IsInvalidated;
+
+    /// The methods which can be used to invalidate the ivar.
+    MethodSet InvalidationMethods;
+
+    IvarInfo() : IsInvalidated(false) {}
+    void addInvalidationMethod(const ObjCMethodDecl *MD) {
+      InvalidationMethods.insert(MD);
+    }
+
+    bool needsInvalidation() const {
+      return !InvalidationMethods.empty();
+    }
+
+    void markInvalidated() {
+      IsInvalidated = true;
+    }
+
+    bool markInvalidated(const ObjCMethodDecl *MD) {
+      if (IsInvalidated)
+        return true;
+      for (MethodSet::iterator I = InvalidationMethods.begin(),
+          E = InvalidationMethods.end(); I != E; ++I) {
+        if (*I == MD) {
+          IsInvalidated = true;
+          return true;
+        }
+      }
+      return false;
+    }
+
+    bool isInvalidated() const {
+      return IsInvalidated;
+    }
+  };
+
+  typedef llvm::DenseMap<const ObjCIvarDecl*, IvarInfo> IvarSet;
+
+  /// Statement visitor, which walks the method body and flags the ivars
+  /// referenced in it (either directly or via property).
+  class MethodCrawler : public ConstStmtVisitor<MethodCrawler> {
+    /// The set of Ivars which need to be invalidated.
+    IvarSet &IVars;
+
+    /// Flag is set as the result of a message send to another
+    /// invalidation method.
+    bool &CalledAnotherInvalidationMethod;
+
+    /// Property setter to ivar mapping.
+    const MethToIvarMapTy &PropertySetterToIvarMap;
+
+    /// Property getter to ivar mapping.
+    const MethToIvarMapTy &PropertyGetterToIvarMap;
+
+    /// Property to ivar mapping.
+    const PropToIvarMapTy &PropertyToIvarMap;
+
+    /// The invalidation method being currently processed.
+    const ObjCMethodDecl *InvalidationMethod;
+
+    ASTContext &Ctx;
+
+    /// Peel off parens, casts, OpaqueValueExpr, and PseudoObjectExpr.
+    const Expr *peel(const Expr *E) const;
+
+    /// Does this expression represent zero: '0'?
+    bool isZero(const Expr *E) const;
+
+    /// Mark the given ivar as invalidated.
+    void markInvalidated(const ObjCIvarDecl *Iv);
+
+    /// Checks if IvarRef refers to the tracked IVar, if yes, marks it as
+    /// invalidated.
+    void checkObjCIvarRefExpr(const ObjCIvarRefExpr *IvarRef);
+
+    /// Checks if ObjCPropertyRefExpr refers to the tracked IVar, if yes, marks
+    /// it as invalidated.
+    void checkObjCPropertyRefExpr(const ObjCPropertyRefExpr *PA);
+
+    /// Checks if ObjCMessageExpr refers to (is a getter for) the tracked IVar,
+    /// if yes, marks it as invalidated.
+    void checkObjCMessageExpr(const ObjCMessageExpr *ME);
+
+    /// Checks if the Expr refers to an ivar, if yes, marks it as invalidated.
+    void check(const Expr *E);
+
+  public:
+    MethodCrawler(IvarSet &InIVars,
+                  bool &InCalledAnotherInvalidationMethod,
+                  const MethToIvarMapTy &InPropertySetterToIvarMap,
+                  const MethToIvarMapTy &InPropertyGetterToIvarMap,
+                  const PropToIvarMapTy &InPropertyToIvarMap,
+                  ASTContext &InCtx)
+    : IVars(InIVars),
+      CalledAnotherInvalidationMethod(InCalledAnotherInvalidationMethod),
+      PropertySetterToIvarMap(InPropertySetterToIvarMap),
+      PropertyGetterToIvarMap(InPropertyGetterToIvarMap),
+      PropertyToIvarMap(InPropertyToIvarMap),
+      InvalidationMethod(0),
+      Ctx(InCtx) {}
+
+    void VisitStmt(const Stmt *S) { VisitChildren(S); }
+
+    void VisitBinaryOperator(const BinaryOperator *BO);
+
+    void VisitObjCMessageExpr(const ObjCMessageExpr *ME);
+
+    void VisitChildren(const Stmt *S) {
+      for (Stmt::const_child_range I = S->children(); I; ++I) {
+        if (*I)
+          this->Visit(*I);
+        if (CalledAnotherInvalidationMethod)
+          return;
+      }
+    }
+  };
+
+  /// Check if the any of the methods inside the interface are annotated with
+  /// the invalidation annotation, update the IvarInfo accordingly.
+  static void containsInvalidationMethod(const ObjCContainerDecl *D,
+                                         IvarInfo &Out);
+
+  /// Check if ivar should be tracked and add to TrackedIvars if positive.
+  /// Returns true if ivar should be tracked.
+  static bool trackIvar(const ObjCIvarDecl *Iv, IvarSet &TrackedIvars);
+
+  /// Given the property declaration, and the list of tracked ivars, finds
+  /// the ivar backing the property when possible. Returns '0' when no such
+  /// ivar could be found.
+  static const ObjCIvarDecl *findPropertyBackingIvar(
+      const ObjCPropertyDecl *Prop,
+      const ObjCInterfaceDecl *InterfaceD,
+      IvarSet &TrackedIvars);
+
+public:
+  void checkASTDecl(const ObjCMethodDecl *D, AnalysisManager& Mgr,
+                    BugReporter &BR) const;
+
+  // TODO: We are currently ignoring the ivars coming from class extensions.
+};
+
+static bool isInvalidationMethod(const ObjCMethodDecl *M) {
+  for (specific_attr_iterator<AnnotateAttr>
+       AI = M->specific_attr_begin<AnnotateAttr>(),
+       AE = M->specific_attr_end<AnnotateAttr>(); AI != AE; ++AI) {
+    const AnnotateAttr *Ann = *AI;
+    if (Ann->getAnnotation() == "objc_instance_variable_invalidator")
+      return true;
+  }
+  return false;
+}
+
+void IvarInvalidationChecker::containsInvalidationMethod(
+    const ObjCContainerDecl *D, IvarInfo &OutInfo) {
+
+  // TODO: Cache the results.
+
+  if (!D)
+    return;
+
+  // Check all methods.
+  for (ObjCContainerDecl::method_iterator
+      I = D->meth_begin(),
+      E = D->meth_end(); I != E; ++I) {
+      const ObjCMethodDecl *MDI = *I;
+      if (isInvalidationMethod(MDI))
+        OutInfo.addInvalidationMethod(
+                               cast<ObjCMethodDecl>(MDI->getCanonicalDecl()));
+  }
+
+  // If interface, check all parent protocols and super.
+  // TODO: Visit all categories in case the invalidation method is declared in
+  // a category.
+  if (const ObjCInterfaceDecl *InterfaceD = dyn_cast<ObjCInterfaceDecl>(D)) {
+    for (ObjCInterfaceDecl::protocol_iterator
+        I = InterfaceD->protocol_begin(),
+        E = InterfaceD->protocol_end(); I != E; ++I) {
+      containsInvalidationMethod(*I, OutInfo);
+    }
+    containsInvalidationMethod(InterfaceD->getSuperClass(), OutInfo);
+    return;
+  }
+
+  // If protocol, check all parent protocols.
+  if (const ObjCProtocolDecl *ProtD = dyn_cast<ObjCProtocolDecl>(D)) {
+    for (ObjCInterfaceDecl::protocol_iterator
+        I = ProtD->protocol_begin(),
+        E = ProtD->protocol_end(); I != E; ++I) {
+      containsInvalidationMethod(*I, OutInfo);
+    }
+    return;
+  }
+
+  llvm_unreachable("One of the casts above should have succeeded.");
+}
+
+bool IvarInvalidationChecker::trackIvar(const ObjCIvarDecl *Iv,
+                                        IvarSet &TrackedIvars) {
+  QualType IvQTy = Iv->getType();
+  const ObjCObjectPointerType *IvTy = IvQTy->getAs<ObjCObjectPointerType>();
+  if (!IvTy)
+    return false;
+  const ObjCInterfaceDecl *IvInterf = IvTy->getInterfaceDecl();
+
+  IvarInfo Info;
+  containsInvalidationMethod(IvInterf, Info);
+  if (Info.needsInvalidation()) {
+    TrackedIvars[cast<ObjCIvarDecl>(Iv->getCanonicalDecl())] = Info;
+    return true;
+  }
+  return false;
+}
+
+const ObjCIvarDecl *IvarInvalidationChecker::findPropertyBackingIvar(
+                        const ObjCPropertyDecl *Prop,
+                        const ObjCInterfaceDecl *InterfaceD,
+                        IvarSet &TrackedIvars) {
+  const ObjCIvarDecl *IvarD = 0;
+
+  // Lookup for the synthesized case.
+  IvarD = Prop->getPropertyIvarDecl();
+  if (IvarD) {
+    if (TrackedIvars.count(IvarD)) {
+      return IvarD;
+    }
+    // If the ivar is synthesized we still want to track it.
+    if (trackIvar(IvarD, TrackedIvars))
+      return IvarD;
+  }
+
+  // Lookup IVars named "_PropName"or "PropName" among the tracked Ivars.
+  StringRef PropName = Prop->getIdentifier()->getName();
+  for (IvarSet::const_iterator I = TrackedIvars.begin(),
+                               E = TrackedIvars.end(); I != E; ++I) {
+    const ObjCIvarDecl *Iv = I->first;
+    StringRef IvarName = Iv->getName();
+
+    if (IvarName == PropName)
+      return Iv;
+
+    SmallString<128> PropNameWithUnderscore;
+    {
+      llvm::raw_svector_ostream os(PropNameWithUnderscore);
+      os << '_' << PropName;
+    }
+    if (IvarName == PropNameWithUnderscore.str())
+      return Iv;
+  }
+
+  // Note, this is a possible source of false positives. We could look at the
+  // getter implementation to find the ivar when its name is not derived from
+  // the property name.
+  return 0;
+}
+
+void IvarInvalidationChecker::checkASTDecl(const ObjCMethodDecl *D,
+                                          AnalysisManager& Mgr,
+                                          BugReporter &BR) const {
+  // We are only interested in checking the cleanup methods.
+  if (!D->hasBody() || !isInvalidationMethod(D))
+    return;
+
+  // Collect all ivars that need cleanup.
+  IvarSet Ivars;
+  const ObjCInterfaceDecl *InterfaceD = D->getClassInterface();
+
+  // Collect ivars declared in this class, its extensions and its implementation
+  ObjCInterfaceDecl *IDecl = const_cast<ObjCInterfaceDecl *>(InterfaceD);
+  for (const ObjCIvarDecl *Iv = IDecl->all_declared_ivar_begin(); Iv;
+       Iv= Iv->getNextIvar())
+    trackIvar(Iv, Ivars);
+
+  // Construct Property/Property Accessor to Ivar maps to assist checking if an
+  // ivar which is backing a property has been reset.
+  MethToIvarMapTy PropSetterToIvarMap;
+  MethToIvarMapTy PropGetterToIvarMap;
+  PropToIvarMapTy PropertyToIvarMap;
+  IvarToPropMapTy IvarToPopertyMap;
+
+  ObjCInterfaceDecl::PropertyMap PropMap;
+  InterfaceD->collectPropertiesToImplement(PropMap);
+
+  for (ObjCInterfaceDecl::PropertyMap::iterator
+      I = PropMap.begin(), E = PropMap.end(); I != E; ++I) {
+    const ObjCPropertyDecl *PD = I->second;
+
+    const ObjCIvarDecl *ID = findPropertyBackingIvar(PD, InterfaceD, Ivars);
+    if (!ID) {
+      continue;
+    }
+
+    // Store the mappings.
+    PD = cast<ObjCPropertyDecl>(PD->getCanonicalDecl());
+    PropertyToIvarMap[PD] = ID;
+    IvarToPopertyMap[ID] = PD;
+
+    // Find the setter and the getter.
+    const ObjCMethodDecl *SetterD = PD->getSetterMethodDecl();
+    if (SetterD) {
+      SetterD = cast<ObjCMethodDecl>(SetterD->getCanonicalDecl());
+      PropSetterToIvarMap[SetterD] = ID;
+    }
+
+    const ObjCMethodDecl *GetterD = PD->getGetterMethodDecl();
+    if (GetterD) {
+      GetterD = cast<ObjCMethodDecl>(GetterD->getCanonicalDecl());
+      PropGetterToIvarMap[GetterD] = ID;
+    }
+  }
+
+
+  // Check which ivars have been invalidated in the method body.
+  bool CalledAnotherInvalidationMethod = false;
+  MethodCrawler(Ivars,
+                CalledAnotherInvalidationMethod,
+                PropSetterToIvarMap,
+                PropGetterToIvarMap,
+                PropertyToIvarMap,
+                BR.getContext()).VisitStmt(D->getBody());
+
+  if (CalledAnotherInvalidationMethod)
+    return;
+
+  // Warn on the ivars that were not accessed by the method.
+  for (IvarSet::const_iterator I = Ivars.begin(), E = Ivars.end(); I != E; ++I){
+    if (!I->second.isInvalidated()) {
+      const ObjCIvarDecl *IvarDecl = I->first;
+
+      PathDiagnosticLocation IvarDecLocation =
+          PathDiagnosticLocation::createEnd(D->getBody(), BR.getSourceManager(),
+                                            Mgr.getAnalysisDeclContext(D));
+
+      SmallString<128> sbuf;
+      llvm::raw_svector_ostream os(sbuf);
+
+      // Construct the warning message.
+      if (IvarDecl->getSynthesize()) {
+        const ObjCPropertyDecl *PD = IvarToPopertyMap[IvarDecl];
+        assert(PD &&
+               "Do we synthesize ivars for something other than properties?");
+        os << "Property "<< PD->getName() <<
+              " needs to be invalidated or set to nil";
+      } else {
+        os << "Instance variable "<< IvarDecl->getName()
+             << " needs to be invalidated or set to nil";
+      }
+
+      BR.EmitBasicReport(D,
+          "Incomplete invalidation",
+          categories::CoreFoundationObjectiveC, os.str(),
+          IvarDecLocation);
+    }
+  }
+}
+
+void IvarInvalidationChecker::MethodCrawler::markInvalidated(
+    const ObjCIvarDecl *Iv) {
+  IvarSet::iterator I = IVars.find(Iv);
+  if (I != IVars.end()) {
+    // If InvalidationMethod is present, we are processing the message send and
+    // should ensure we are invalidating with the appropriate method,
+    // otherwise, we are processing setting to 'nil'.
+    if (InvalidationMethod)
+      I->second.markInvalidated(InvalidationMethod);
+    else
+      I->second.markInvalidated();
+  }
+}
+
+const Expr *IvarInvalidationChecker::MethodCrawler::peel(const Expr *E) const {
+  E = E->IgnoreParenCasts();
+  if (const PseudoObjectExpr *POE = dyn_cast<PseudoObjectExpr>(E))
+    E = POE->getSyntacticForm()->IgnoreParenCasts();
+  if (const OpaqueValueExpr *OVE = dyn_cast<OpaqueValueExpr>(E))
+    E = OVE->getSourceExpr()->IgnoreParenCasts();
+  return E;
+}
+
+void IvarInvalidationChecker::MethodCrawler::checkObjCIvarRefExpr(
+    const ObjCIvarRefExpr *IvarRef) {
+  if (const Decl *D = IvarRef->getDecl())
+    markInvalidated(cast<ObjCIvarDecl>(D->getCanonicalDecl()));
+}
+
+void IvarInvalidationChecker::MethodCrawler::checkObjCMessageExpr(
+    const ObjCMessageExpr *ME) {
+  const ObjCMethodDecl *MD = ME->getMethodDecl();
+  if (MD) {
+    MD = cast<ObjCMethodDecl>(MD->getCanonicalDecl());
+    MethToIvarMapTy::const_iterator IvI = PropertyGetterToIvarMap.find(MD);
+    if (IvI != PropertyGetterToIvarMap.end())
+      markInvalidated(IvI->second);
+  }
+}
+
+void IvarInvalidationChecker::MethodCrawler::checkObjCPropertyRefExpr(
+    const ObjCPropertyRefExpr *PA) {
+
+  if (PA->isExplicitProperty()) {
+    const ObjCPropertyDecl *PD = PA->getExplicitProperty();
+    if (PD) {
+      PD = cast<ObjCPropertyDecl>(PD->getCanonicalDecl());
+      PropToIvarMapTy::const_iterator IvI = PropertyToIvarMap.find(PD);
+      if (IvI != PropertyToIvarMap.end())
+        markInvalidated(IvI->second);
+      return;
+    }
+  }
+
+  if (PA->isImplicitProperty()) {
+    const ObjCMethodDecl *MD = PA->getImplicitPropertySetter();
+    if (MD) {
+      MD = cast<ObjCMethodDecl>(MD->getCanonicalDecl());
+      MethToIvarMapTy::const_iterator IvI =PropertyGetterToIvarMap.find(MD);
+      if (IvI != PropertyGetterToIvarMap.end())
+        markInvalidated(IvI->second);
+      return;
+    }
+  }
+}
+
+bool IvarInvalidationChecker::MethodCrawler::isZero(const Expr *E) const {
+  E = peel(E);
+
+  return (E->isNullPointerConstant(Ctx, Expr::NPC_ValueDependentIsNotNull)
+           != Expr::NPCK_NotNull);
+}
+
+void IvarInvalidationChecker::MethodCrawler::check(const Expr *E) {
+  E = peel(E);
+
+  if (const ObjCIvarRefExpr *IvarRef = dyn_cast<ObjCIvarRefExpr>(E)) {
+    checkObjCIvarRefExpr(IvarRef);
+    return;
+  }
+
+  if (const ObjCPropertyRefExpr *PropRef = dyn_cast<ObjCPropertyRefExpr>(E)) {
+    checkObjCPropertyRefExpr(PropRef);
+    return;
+  }
+
+  if (const ObjCMessageExpr *MsgExpr = dyn_cast<ObjCMessageExpr>(E)) {
+    checkObjCMessageExpr(MsgExpr);
+    return;
+  }
+}
+
+void IvarInvalidationChecker::MethodCrawler::VisitBinaryOperator(
+    const BinaryOperator *BO) {
+  VisitStmt(BO);
+
+  if (BO->getOpcode() != BO_Assign)
+    return;
+
+  // Do we assign zero?
+  if (!isZero(BO->getRHS()))
+    return;
+
+  // Check the variable we are assigning to.
+  check(BO->getLHS());
+}
+
+void IvarInvalidationChecker::MethodCrawler::VisitObjCMessageExpr(
+    const ObjCMessageExpr *ME) {
+  const ObjCMethodDecl *MD = ME->getMethodDecl();
+  const Expr *Receiver = ME->getInstanceReceiver();
+
+  // Stop if we are calling '[self invalidate]'.
+  if (Receiver && isInvalidationMethod(MD))
+    if (Receiver->isObjCSelfExpr()) {
+      CalledAnotherInvalidationMethod = true;
+      return;
+    }
+
+  // Check if we call a setter and set the property to 'nil'.
+  if (MD && (ME->getNumArgs() == 1) && isZero(ME->getArg(0))) {
+    MD = cast<ObjCMethodDecl>(MD->getCanonicalDecl());
+    MethToIvarMapTy::const_iterator IvI = PropertySetterToIvarMap.find(MD);
+    if (IvI != PropertySetterToIvarMap.end()) {
+      markInvalidated(IvI->second);
+      return;
+    }
+  }
+
+  // Check if we call the 'invalidation' routine on the ivar.
+  if (Receiver) {
+    InvalidationMethod = MD;
+    check(Receiver->IgnoreParenCasts());
+    InvalidationMethod = 0;
+  }
+
+  VisitStmt(ME);
+}
+}
+
+// Register the checker.
+void ento::registerIvarInvalidationChecker(CheckerManager &mgr) {
+  mgr.registerChecker<IvarInvalidationChecker>();
+}
diff --git a/lib/StaticAnalyzer/Checkers/MacOSKeychainAPIChecker.cpp b/lib/StaticAnalyzer/Checkers/MacOSKeychainAPIChecker.cpp
index 969f2dd..76f20b6 100644
--- a/lib/StaticAnalyzer/Checkers/MacOSKeychainAPIChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/MacOSKeychainAPIChecker.cpp
@@ -158,16 +158,9 @@ private:
 /// ProgramState traits to store the currently allocated (and not yet freed)
 /// symbols. This is a map from the allocated content symbol to the
 /// corresponding AllocationState.
-typedef llvm::ImmutableMap<SymbolRef,
-                       MacOSKeychainAPIChecker::AllocationState> AllocatedSetTy;
-
-namespace { struct AllocatedData {}; }
-namespace clang { namespace ento {
-template<> struct ProgramStateTrait<AllocatedData>
-    :  public ProgramStatePartialTrait<AllocatedSetTy > {
-  static void *GDMIndex() { static int index = 0; return &index; }
-};
-}}
+REGISTER_MAP_WITH_PROGRAMSTATE(AllocatedData,
+                               SymbolRef,
+                               MacOSKeychainAPIChecker::AllocationState)
 
 static bool isEnclosingFunctionParam(const Expr *E) {
   E = E->IgnoreParenCasts();
@@ -282,7 +275,7 @@ void MacOSKeychainAPIChecker::
   Report->addVisitor(new SecKeychainBugVisitor(AP.first));
   Report->addRange(ArgExpr->getSourceRange());
   markInteresting(Report, AP);
-  C.EmitReport(Report);
+  C.emitReport(Report);
 }
 
 void MacOSKeychainAPIChecker::checkPreStmt(const CallExpr *CE,
@@ -323,7 +316,7 @@ void MacOSKeychainAPIChecker::checkPreStmt(const CallExpr *CE,
           Report->addVisitor(new SecKeychainBugVisitor(V));
           Report->addRange(ArgExpr->getSourceRange());
           Report->markInteresting(AS->Region);
-          C.EmitReport(Report);
+          C.emitReport(Report);
         }
       }
     return;
@@ -376,7 +369,7 @@ void MacOSKeychainAPIChecker::checkPreStmt(const CallExpr *CE,
     Report->addRange(ArgExpr->getSourceRange());
     if (AS)
       Report->markInteresting(AS->Region);
-    C.EmitReport(Report);
+    C.emitReport(Report);
     return;
   }
 
@@ -440,7 +433,7 @@ void MacOSKeychainAPIChecker::checkPreStmt(const CallExpr *CE,
     Report->addVisitor(new SecKeychainBugVisitor(ArgSM));
     Report->addRange(ArgExpr->getSourceRange());
     Report->markInteresting(AS->Region);
-    C.EmitReport(Report);
+    C.emitReport(Report);
     return;
   }
 
@@ -571,13 +564,13 @@ BugReport *MacOSKeychainAPIChecker::
 void MacOSKeychainAPIChecker::checkDeadSymbols(SymbolReaper &SR,
                                                CheckerContext &C) const {
   ProgramStateRef State = C.getState();
-  AllocatedSetTy ASet = State->get<AllocatedData>();
+  AllocatedDataTy ASet = State->get<AllocatedData>();
   if (ASet.isEmpty())
     return;
 
   bool Changed = false;
   AllocationPairVec Errors;
-  for (AllocatedSetTy::iterator I = ASet.begin(), E = ASet.end(); I != E; ++I) {
+  for (AllocatedDataTy::iterator I = ASet.begin(), E = ASet.end(); I != E; ++I) {
     if (SR.isLive(I->first))
       continue;
 
@@ -585,7 +578,9 @@ void MacOSKeychainAPIChecker::checkDeadSymbols(SymbolReaper &SR,
     State = State->remove<AllocatedData>(I->first);
     // If the allocated symbol is null or if the allocation call might have
     // returned an error, do not report.
-    if (State->getSymVal(I->first) ||
+    ConstraintManager &CMgr = State->getConstraintManager();
+    ConditionTruthVal AllocFailed = CMgr.isNull(State, I.getKey());
+    if (AllocFailed.isConstrainedTrue() ||
         definitelyReturnedError(I->second.Region, State, C.getSValBuilder()))
       continue;
     Errors.push_back(std::make_pair(I->first, &I->second));
@@ -602,7 +597,7 @@ void MacOSKeychainAPIChecker::checkDeadSymbols(SymbolReaper &SR,
   // Generate the error reports.
   for (AllocationPairVec::iterator I = Errors.begin(), E = Errors.end();
                                                        I != E; ++I) {
-    C.EmitReport(generateAllocatedDataNotReleasedReport(*I, N, C));
+    C.emitReport(generateAllocatedDataNotReleasedReport(*I, N, C));
   }
 
   // Generate the new, cleaned up state.
@@ -617,7 +612,7 @@ void MacOSKeychainAPIChecker::checkEndPath(CheckerContext &C) const {
   if (C.getLocationContext()->getParent() != 0)
     return;
 
-  AllocatedSetTy AS = state->get<AllocatedData>();
+  AllocatedDataTy AS = state->get<AllocatedData>();
   if (AS.isEmpty())
     return;
 
@@ -625,12 +620,14 @@ void MacOSKeychainAPIChecker::checkEndPath(CheckerContext &C) const {
   // found here, so report it.
   bool Changed = false;
   AllocationPairVec Errors;
-  for (AllocatedSetTy::iterator I = AS.begin(), E = AS.end(); I != E; ++I ) {
+  for (AllocatedDataTy::iterator I = AS.begin(), E = AS.end(); I != E; ++I ) {
     Changed = true;
     state = state->remove<AllocatedData>(I->first);
     // If the allocated symbol is null or if error code was returned at
     // allocation, do not report.
-    if (state->getSymVal(I.getKey()) ||
+    ConstraintManager &CMgr = state->getConstraintManager();
+    ConditionTruthVal AllocFailed = CMgr.isNull(state, I.getKey());
+    if (AllocFailed.isConstrainedTrue() ||
         definitelyReturnedError(I->second.Region, state,
                                 C.getSValBuilder())) {
       continue;
@@ -650,7 +647,7 @@ void MacOSKeychainAPIChecker::checkEndPath(CheckerContext &C) const {
   // Generate the error reports.
   for (AllocationPairVec::iterator I = Errors.begin(), E = Errors.end();
                                                        I != E; ++I) {
-    C.EmitReport(generateAllocatedDataNotReleasedReport(*I, N, C));
+    C.emitReport(generateAllocatedDataNotReleasedReport(*I, N, C));
   }
 
   C.addTransition(state, N);
diff --git a/lib/StaticAnalyzer/Checkers/MacOSXAPIChecker.cpp b/lib/StaticAnalyzer/Checkers/MacOSXAPIChecker.cpp
index cfdb55d..467b8b1 100644
--- a/lib/StaticAnalyzer/Checkers/MacOSXAPIChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/MacOSXAPIChecker.cpp
@@ -70,6 +70,16 @@ void MacOSXAPIChecker::CheckDispatchOnce(CheckerContext &C, const CallExpr *CE,
     BT_dispatchOnce.reset(new BugType("Improper use of 'dispatch_once'",
                                       "Mac OS X API"));
 
+  // Handle _dispatch_once.  In some versions of the OS X SDK we have the case
+  // that dispatch_once is a macro that wraps a call to _dispatch_once.
+  // _dispatch_once is then a function which then calls the real dispatch_once.
+  // Users do not care; they just want the warning at the top-level call.
+  if (CE->getLocStart().isMacroID()) {
+    StringRef TrimmedFName = FName.ltrim("_");
+    if (TrimmedFName != FName)
+      FName = TrimmedFName;
+  }
+  
   SmallString<256> S;
   llvm::raw_svector_ostream os(S);
   os << "Call to '" << FName << "' uses";
@@ -84,7 +94,7 @@ void MacOSXAPIChecker::CheckDispatchOnce(CheckerContext &C, const CallExpr *CE,
 
   BugReport *report = new BugReport(*BT_dispatchOnce, os.str(), N);
   report->addRange(CE->getArg(0)->getSourceRange());
-  C.EmitReport(report);
+  C.emitReport(report);
 }
 
 //===----------------------------------------------------------------------===//
@@ -99,7 +109,9 @@ void MacOSXAPIChecker::checkPreStmt(const CallExpr *CE,
 
   SubChecker SC =
     llvm::StringSwitch<SubChecker>(Name)
-      .Cases("dispatch_once", "dispatch_once_f",
+      .Cases("dispatch_once",
+             "_dispatch_once",
+             "dispatch_once_f",
              &MacOSXAPIChecker::CheckDispatchOnce)
       .Default(NULL);
 
diff --git a/lib/StaticAnalyzer/Checkers/MallocChecker.cpp b/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
index dfcedf6..caf70ca 100644
--- a/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
@@ -26,6 +26,7 @@
 #include "llvm/ADT/ImmutableMap.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/StringExtras.h"
 #include <climits>
 
 using namespace clang;
@@ -70,17 +71,31 @@ public:
   }
 };
 
+enum ReallocPairKind {
+  RPToBeFreedAfterFailure,
+  // The symbol has been freed when reallocation failed.
+  RPIsFreeOnFailure,
+  // The symbol does not need to be freed after reallocation fails.
+  RPDoNotTrackAfterFailure
+};
+
+/// \class ReallocPair
+/// \brief Stores information about the symbol being reallocated by a call to
+/// 'realloc' to allow modeling failed reallocation later in the path.
 struct ReallocPair {
+  // \brief The symbol which realloc reallocated.
   SymbolRef ReallocatedSym;
-  bool IsFreeOnFailure;
-  ReallocPair(SymbolRef S, bool F) : ReallocatedSym(S), IsFreeOnFailure(F) {}
+  ReallocPairKind Kind;
+
+  ReallocPair(SymbolRef S, ReallocPairKind K) :
+    ReallocatedSym(S), Kind(K) {}
   void Profile(llvm::FoldingSetNodeID &ID) const {
-    ID.AddInteger(IsFreeOnFailure);
+    ID.AddInteger(Kind);
     ID.AddPointer(ReallocatedSym);
   }
   bool operator==(const ReallocPair &X) const {
     return ReallocatedSym == X.ReallocatedSym &&
-           IsFreeOnFailure == X.IsFreeOnFailure;
+           Kind == X.Kind;
   }
 };
 
@@ -92,7 +107,7 @@ class MallocChecker : public Checker<check::DeadSymbols,
                                      check::PreStmt<CallExpr>,
                                      check::PostStmt<CallExpr>,
                                      check::PostStmt<BlockExpr>,
-                                     check::PreObjCMessage,
+                                     check::PostObjCMessage,
                                      check::Location,
                                      check::Bind,
                                      eval::Assume,
@@ -120,7 +135,7 @@ public:
 
   void checkPreStmt(const CallExpr *S, CheckerContext &C) const;
   void checkPostStmt(const CallExpr *CE, CheckerContext &C) const;
-  void checkPreObjCMessage(const ObjCMethodCall &Call, CheckerContext &C) const;
+  void checkPostObjCMessage(const ObjCMethodCall &Call, CheckerContext &C) const;
   void checkPostStmt(const BlockExpr *BE, CheckerContext &C) const;
   void checkDeadSymbols(SymbolReaper &SymReaper, CheckerContext &C) const;
   void checkEndPath(CheckerContext &C) const;
@@ -177,11 +192,15 @@ private:
                               const OwnershipAttr* Att) const;
   ProgramStateRef FreeMemAux(CheckerContext &C, const CallExpr *CE,
                              ProgramStateRef state, unsigned Num,
-                             bool Hold) const;
+                             bool Hold,
+                             bool &ReleasedAllocated,
+                             bool ReturnsNullOnFailure = false) const;
   ProgramStateRef FreeMemAux(CheckerContext &C, const Expr *Arg,
                              const Expr *ParentExpr,
-                             ProgramStateRef state,
-                             bool Hold) const;
+                             ProgramStateRef State,
+                             bool Hold,
+                             bool &ReleasedAllocated,
+                             bool ReturnsNullOnFailure = false) const;
 
   ProgramStateRef ReallocMem(CheckerContext &C, const CallExpr *CE,
                              bool FreesMemOnFailure) const;
@@ -301,13 +320,14 @@ private:
         : StackHintGeneratorForSymbol(S, M) {}
 
       virtual std::string getMessageForArg(const Expr *ArgE, unsigned ArgIndex) {
+        // Printed parameters start at 1, not 0.
+        ++ArgIndex;
+
         SmallString<200> buf;
         llvm::raw_svector_ostream os(buf);
 
-        os << "Reallocation of ";
-        // Printed parameters start at 1, not 0.
-        printOrdinal(++ArgIndex, os);
-        os << " parameter failed";
+        os << "Reallocation of " << ArgIndex << llvm::getOrdinalSuffix(ArgIndex)
+           << " parameter failed";
 
         return os.str();
       }
@@ -320,25 +340,12 @@ private:
 };
 } // end anonymous namespace
 
-typedef llvm::ImmutableMap<SymbolRef, RefState> RegionStateTy;
-typedef llvm::ImmutableMap<SymbolRef, ReallocPair > ReallocMap;
-class RegionState {};
-class ReallocPairs {};
-namespace clang {
-namespace ento {
-  template <>
-  struct ProgramStateTrait<RegionState> 
-    : public ProgramStatePartialTrait<RegionStateTy> {
-    static void *GDMIndex() { static int x; return &x; }
-  };
+REGISTER_MAP_WITH_PROGRAMSTATE(RegionState, SymbolRef, RefState)
+REGISTER_MAP_WITH_PROGRAMSTATE(ReallocPairs, SymbolRef, ReallocPair)
 
-  template <>
-  struct ProgramStateTrait<ReallocPairs>
-    : public ProgramStatePartialTrait<ReallocMap> {
-    static void *GDMIndex() { static int x; return &x; }
-  };
-}
-}
+// A map from the freed symbol to the symbol representing the return value of 
+// the free function.
+REGISTER_MAP_WITH_PROGRAMSTATE(FreeReturnValue, SymbolRef, SymbolRef)
 
 namespace {
 class StopTrackingCallback : public SymbolVisitor {
@@ -426,11 +433,15 @@ bool MallocChecker::isFreeFunction(const FunctionDecl *FD, ASTContext &C) const
 }
 
 void MallocChecker::checkPostStmt(const CallExpr *CE, CheckerContext &C) const {
+  if (C.wasInlined)
+    return;
+  
   const FunctionDecl *FD = C.getCalleeDecl(CE);
   if (!FD)
     return;
 
   ProgramStateRef State = C.getState();
+  bool ReleasedAllocatedMemory = false;
 
   if (FD->getKind() == Decl::Function) {
     initIdentifierInfo(C.getASTContext());
@@ -447,7 +458,7 @@ void MallocChecker::checkPostStmt(const CallExpr *CE, CheckerContext &C) const {
     } else if (FunI == II_calloc) {
       State = CallocMem(C, CE);
     } else if (FunI == II_free) {
-      State = FreeMemAux(C, CE, State, 0, false);
+      State = FreeMemAux(C, CE, State, 0, false, ReleasedAllocatedMemory);
     } else if (FunI == II_strdup) {
       State = MallocUpdateRefState(C, CE, State);
     } else if (FunI == II_strndup) {
@@ -487,21 +498,26 @@ static bool isFreeWhenDoneSetToZero(const ObjCMethodCall &Call) {
   return false;
 }
 
-void MallocChecker::checkPreObjCMessage(const ObjCMethodCall &Call,
-                                        CheckerContext &C) const {
+void MallocChecker::checkPostObjCMessage(const ObjCMethodCall &Call,
+                                         CheckerContext &C) const {
   // If the first selector is dataWithBytesNoCopy, assume that the memory will
   // be released with 'free' by the new object.
   // Ex:  [NSData dataWithBytesNoCopy:bytes length:10];
   // Unless 'freeWhenDone' param set to 0.
   // TODO: Check that the memory was allocated with malloc.
+  bool ReleasedAllocatedMemory = false;
   Selector S = Call.getSelector();
   if ((S.getNameForSlot(0) == "dataWithBytesNoCopy" ||
        S.getNameForSlot(0) == "initWithBytesNoCopy" ||
        S.getNameForSlot(0) == "initWithCharactersNoCopy") &&
       !isFreeWhenDoneSetToZero(Call)){
     unsigned int argIdx  = 0;
-    C.addTransition(FreeMemAux(C, Call.getArgExpr(argIdx),
-                    Call.getOriginExpr(), C.getState(), true));
+    ProgramStateRef State = FreeMemAux(C, Call.getArgExpr(argIdx),
+                                       Call.getOriginExpr(), C.getState(), true,
+                                       ReleasedAllocatedMemory,
+                                       /* RetNullOnFailure*/ true);
+
+    C.addTransition(State);
   }
 }
 
@@ -526,7 +542,7 @@ ProgramStateRef MallocChecker::MallocMemAux(CheckerContext &C,
   // Bind the return value to the symbolic value from the heap region.
   // TODO: We could rewrite post visit to eval call; 'malloc' does not have
   // side effects other than what we model here.
-  unsigned Count = C.getCurrentBlockCount();
+  unsigned Count = C.blockCount();
   SValBuilder &svalBuilder = C.getSValBuilder();
   const LocationContext *LCtx = C.getPredecessor()->getLocationContext();
   DefinedSVal RetVal =
@@ -584,11 +600,13 @@ ProgramStateRef MallocChecker::FreeMemAttr(CheckerContext &C,
     return 0;
 
   ProgramStateRef State = C.getState();
+  bool ReleasedAllocated = false;
 
   for (OwnershipAttr::args_iterator I = Att->args_begin(), E = Att->args_end();
        I != E; ++I) {
     ProgramStateRef StateI = FreeMemAux(C, CE, State, *I,
-                               Att->getOwnKind() == OwnershipAttr::Holds);
+                               Att->getOwnKind() == OwnershipAttr::Holds,
+                               ReleasedAllocated);
     if (StateI)
       State = StateI;
   }
@@ -599,20 +617,40 @@ ProgramStateRef MallocChecker::FreeMemAux(CheckerContext &C,
                                           const CallExpr *CE,
                                           ProgramStateRef state,
                                           unsigned Num,
-                                          bool Hold) const {
+                                          bool Hold,
+                                          bool &ReleasedAllocated,
+                                          bool ReturnsNullOnFailure) const {
   if (CE->getNumArgs() < (Num + 1))
     return 0;
 
-  return FreeMemAux(C, CE->getArg(Num), CE, state, Hold);
+  return FreeMemAux(C, CE->getArg(Num), CE, state, Hold,
+                    ReleasedAllocated, ReturnsNullOnFailure);
+}
+
+/// Checks if the previous call to free on the given symbol failed - if free
+/// failed, returns true. Also, returns the corresponding return value symbol.
+bool didPreviousFreeFail(ProgramStateRef State,
+                         SymbolRef Sym, SymbolRef &RetStatusSymbol) {
+  const SymbolRef *Ret = State->get<FreeReturnValue>(Sym);
+  if (Ret) {
+    assert(*Ret && "We should not store the null return symbol");
+    ConstraintManager &CMgr = State->getConstraintManager();
+    ConditionTruthVal FreeFailed = CMgr.isNull(State, *Ret);
+    RetStatusSymbol = *Ret;
+    return FreeFailed.isConstrainedTrue();
+  }
+  return false;
 }
 
 ProgramStateRef MallocChecker::FreeMemAux(CheckerContext &C,
                                           const Expr *ArgExpr,
                                           const Expr *ParentExpr,
-                                          ProgramStateRef state,
-                                          bool Hold) const {
+                                          ProgramStateRef State,
+                                          bool Hold,
+                                          bool &ReleasedAllocated,
+                                          bool ReturnsNullOnFailure) const {
 
-  SVal ArgVal = state->getSVal(ArgExpr, C.getLocationContext());
+  SVal ArgVal = State->getSVal(ArgExpr, C.getLocationContext());
   if (!isa<DefinedOrUnknownSVal>(ArgVal))
     return 0;
   DefinedOrUnknownSVal location = cast<DefinedOrUnknownSVal>(ArgVal);
@@ -623,7 +661,7 @@ ProgramStateRef MallocChecker::FreeMemAux(CheckerContext &C,
 
   // The explicit NULL case, no operation is performed.
   ProgramStateRef notNullState, nullState;
-  llvm::tie(notNullState, nullState) = state->assume(location);
+  llvm::tie(notNullState, nullState) = State->assume(location);
   if (nullState && !notNullState)
     return 0;
 
@@ -672,10 +710,14 @@ ProgramStateRef MallocChecker::FreeMemAux(CheckerContext &C,
     return 0;
 
   SymbolRef Sym = SR->getSymbol();
-  const RefState *RS = state->get<RegionState>(Sym);
+  const RefState *RS = State->get<RegionState>(Sym);
+  SymbolRef PreviousRetStatusSymbol = 0;
 
   // Check double free.
-  if (RS && (RS->isReleased() || RS->isRelinquished())) {
+  if (RS &&
+      (RS->isReleased() || RS->isRelinquished()) &&
+      !didPreviousFreeFail(State, Sym, PreviousRetStatusSymbol)) {
+
     if (ExplodedNode *N = C.generateSink()) {
       if (!BT_DoubleFree)
         BT_DoubleFree.reset(
@@ -685,16 +727,34 @@ ProgramStateRef MallocChecker::FreeMemAux(CheckerContext &C,
                             "Attempt to free non-owned memory"), N);
       R->addRange(ArgExpr->getSourceRange());
       R->markInteresting(Sym);
+      if (PreviousRetStatusSymbol)
+        R->markInteresting(PreviousRetStatusSymbol);
       R->addVisitor(new MallocBugVisitor(Sym));
-      C.EmitReport(R);
+      C.emitReport(R);
     }
     return 0;
   }
 
+  ReleasedAllocated = (RS != 0);
+
+  // Clean out the info on previous call to free return info.
+  State = State->remove<FreeReturnValue>(Sym);
+
+  // Keep track of the return value. If it is NULL, we will know that free 
+  // failed.
+  if (ReturnsNullOnFailure) {
+    SVal RetVal = C.getSVal(ParentExpr);
+    SymbolRef RetStatusSymbol = RetVal.getAsSymbol();
+    if (RetStatusSymbol) {
+      C.getSymbolManager().addSymbolDependency(Sym, RetStatusSymbol);
+      State = State->set<FreeReturnValue>(Sym, RetStatusSymbol);
+    }
+  }
+
   // Normal free.
   if (Hold)
-    return state->set<RegionState>(Sym, RefState::getRelinquished(ParentExpr));
-  return state->set<RegionState>(Sym, RefState::getReleased(ParentExpr));
+    return State->set<RegionState>(Sym, RefState::getRelinquished(ParentExpr));
+  return State->set<RegionState>(Sym, RefState::getReleased(ParentExpr));
 }
 
 bool MallocChecker::SummarizeValue(raw_ostream &os, SVal V) {
@@ -714,7 +774,7 @@ bool MallocChecker::SummarizeRegion(raw_ostream &os,
                                     const MemRegion *MR) {
   switch (MR->getKind()) {
   case MemRegion::FunctionTextRegionKind: {
-    const FunctionDecl *FD = cast<FunctionTextRegion>(MR)->getDecl();
+    const NamedDecl *FD = cast<FunctionTextRegion>(MR)->getDecl();
     if (FD)
       os << "the address of the function '" << *FD << '\'';
     else
@@ -819,7 +879,7 @@ void MallocChecker::ReportBadFree(CheckerContext &C, SVal ArgVal,
     BugReport *R = new BugReport(*BT_BadFree, os.str(), N);
     R->markInteresting(MR);
     R->addRange(range);
-    C.EmitReport(R);
+    C.emitReport(R);
   }
 }
 
@@ -886,9 +946,12 @@ ProgramStateRef MallocChecker::ReallocMem(CheckerContext &C,
   if (!FromPtr || !ToPtr)
     return 0;
 
+  bool ReleasedAllocated = false;
+
   // If the size is 0, free the memory.
   if (SizeIsZero)
-    if (ProgramStateRef stateFree = FreeMemAux(C, CE, StateSizeIsZero,0,false)){
+    if (ProgramStateRef stateFree = FreeMemAux(C, CE, StateSizeIsZero, 0,
+                                               false, ReleasedAllocated)){
       // The semantics of the return value are:
       // If size was equal to 0, either NULL or a pointer suitable to be passed
       // to free() is returned. We just free the input pointer and do not add
@@ -897,14 +960,25 @@ ProgramStateRef MallocChecker::ReallocMem(CheckerContext &C,
     }
 
   // Default behavior.
-  if (ProgramStateRef stateFree = FreeMemAux(C, CE, state, 0, false)) {
-    // FIXME: We should copy the content of the original buffer.
+  if (ProgramStateRef stateFree =
+        FreeMemAux(C, CE, state, 0, false, ReleasedAllocated)) {
+
     ProgramStateRef stateRealloc = MallocMemAux(C, CE, CE->getArg(1),
                                                 UnknownVal(), stateFree);
     if (!stateRealloc)
       return 0;
+
+    ReallocPairKind Kind = RPToBeFreedAfterFailure;
+    if (FreesOnFail)
+      Kind = RPIsFreeOnFailure;
+    else if (!ReleasedAllocated)
+      Kind = RPDoNotTrackAfterFailure;
+
+    // Record the info about the reallocated symbol so that we could properly
+    // process failed reallocation.
     stateRealloc = stateRealloc->set<ReallocPairs>(ToPtr,
-                                            ReallocPair(FromPtr, FreesOnFail));
+                                                   ReallocPair(FromPtr, Kind));
+    // The reallocated symbol should stay alive for as long as the new symbol.
     C.getSymbolManager().addSymbolDependency(ToPtr, FromPtr);
     return stateRealloc;
   }
@@ -1004,7 +1078,7 @@ void MallocChecker::reportLeak(SymbolRef Sym, ExplodedNode *N,
   BugReport *R = new BugReport(*BT_Leak, os.str(), N, LocUsedForUniqueing);
   R->markInteresting(Sym);
   R->addVisitor(new MallocBugVisitor(Sym, true));
-  C.EmitReport(R);
+  C.emitReport(R);
 }
 
 void MallocChecker::checkDeadSymbols(SymbolReaper &SymReaper,
@@ -1017,14 +1091,11 @@ void MallocChecker::checkDeadSymbols(SymbolReaper &SymReaper,
   RegionStateTy RS = state->get<RegionState>();
   RegionStateTy::Factory &F = state->get_context<RegionState>();
 
-  bool generateReport = false;
   llvm::SmallVector<SymbolRef, 2> Errors;
   for (RegionStateTy::iterator I = RS.begin(), E = RS.end(); I != E; ++I) {
     if (SymReaper.isDead(I->first)) {
-      if (I->second.isAllocated()) {
-        generateReport = true;
+      if (I->second.isAllocated())
         Errors.push_back(I->first);
-      }
       // Remove the dead symbol from the map.
       RS = F.remove(RS, I->first);
 
@@ -1032,24 +1103,34 @@ void MallocChecker::checkDeadSymbols(SymbolReaper &SymReaper,
   }
   
   // Cleanup the Realloc Pairs Map.
-  ReallocMap RP = state->get<ReallocPairs>();
-  for (ReallocMap::iterator I = RP.begin(), E = RP.end(); I != E; ++I) {
+  ReallocPairsTy RP = state->get<ReallocPairs>();
+  for (ReallocPairsTy::iterator I = RP.begin(), E = RP.end(); I != E; ++I) {
     if (SymReaper.isDead(I->first) ||
         SymReaper.isDead(I->second.ReallocatedSym)) {
       state = state->remove<ReallocPairs>(I->first);
     }
   }
 
-  // Generate leak node.
-  static SimpleProgramPointTag Tag("MallocChecker : DeadSymbolsLeak");
-  ExplodedNode *N = C.addTransition(C.getState(), C.getPredecessor(), &Tag);
+  // Cleanup the FreeReturnValue Map.
+  FreeReturnValueTy FR = state->get<FreeReturnValue>();
+  for (FreeReturnValueTy::iterator I = FR.begin(), E = FR.end(); I != E; ++I) {
+    if (SymReaper.isDead(I->first) ||
+        SymReaper.isDead(I->second)) {
+      state = state->remove<FreeReturnValue>(I->first);
+    }
+  }
 
-  if (generateReport) {
+  // Generate leak node.
+  ExplodedNode *N = C.getPredecessor();
+  if (!Errors.empty()) {
+    static SimpleProgramPointTag Tag("MallocChecker : DeadSymbolsLeak");
+    N = C.addTransition(C.getState(), C.getPredecessor(), &Tag);
     for (llvm::SmallVector<SymbolRef, 2>::iterator
-         I = Errors.begin(), E = Errors.end(); I != E; ++I) {
+        I = Errors.begin(), E = Errors.end(); I != E; ++I) {
       reportLeak(*I, N, C);
     }
   }
+
   C.addTransition(state->set<RegionState>(RS), N);
 }
 
@@ -1182,7 +1263,7 @@ bool MallocChecker::checkUseAfterFree(SymbolRef Sym, CheckerContext &C,
         R->addRange(S->getSourceRange());
       R->markInteresting(Sym);
       R->addVisitor(new MallocBugVisitor(Sym));
-      C.EmitReport(R);
+      C.emitReport(R);
       return true;
     }
   }
@@ -1249,28 +1330,36 @@ ProgramStateRef MallocChecker::evalAssume(ProgramStateRef state,
                                               bool Assumption) const {
   RegionStateTy RS = state->get<RegionState>();
   for (RegionStateTy::iterator I = RS.begin(), E = RS.end(); I != E; ++I) {
-    // If the symbol is assumed to NULL or another constant, this will
-    // return an APSInt*.
-    if (state->getSymVal(I.getKey()))
+    // If the symbol is assumed to be NULL, remove it from consideration.
+    ConstraintManager &CMgr = state->getConstraintManager();
+    ConditionTruthVal AllocFailed = CMgr.isNull(state, I.getKey());
+    if (AllocFailed.isConstrainedTrue())
       state = state->remove<RegionState>(I.getKey());
   }
 
   // Realloc returns 0 when reallocation fails, which means that we should
   // restore the state of the pointer being reallocated.
-  ReallocMap RP = state->get<ReallocPairs>();
-  for (ReallocMap::iterator I = RP.begin(), E = RP.end(); I != E; ++I) {
-    // If the symbol is assumed to NULL or another constant, this will
-    // return an APSInt*.
-    if (state->getSymVal(I.getKey())) {
-      SymbolRef ReallocSym = I.getData().ReallocatedSym;
-      const RefState *RS = state->get<RegionState>(ReallocSym);
-      if (RS) {
-        if (RS->isReleased() && ! I.getData().IsFreeOnFailure)
+  ReallocPairsTy RP = state->get<ReallocPairs>();
+  for (ReallocPairsTy::iterator I = RP.begin(), E = RP.end(); I != E; ++I) {
+    // If the symbol is assumed to be NULL, remove it from consideration.
+    ConstraintManager &CMgr = state->getConstraintManager();
+    ConditionTruthVal AllocFailed = CMgr.isNull(state, I.getKey());
+    if (!AllocFailed.isConstrainedTrue())
+      continue;
+
+    SymbolRef ReallocSym = I.getData().ReallocatedSym;
+    if (const RefState *RS = state->get<RegionState>(ReallocSym)) {
+      if (RS->isReleased()) {
+        if (I.getData().Kind == RPToBeFreedAfterFailure)
           state = state->set<RegionState>(ReallocSym,
-                             RefState::getAllocated(RS->getStmt()));
+              RefState::getAllocated(RS->getStmt()));
+        else if (I.getData().Kind == RPDoNotTrackAfterFailure)
+          state = state->remove<RegionState>(ReallocSym);
+        else
+          assert(I.getData().Kind == RPIsFreeOnFailure);
       }
-      state = state->remove<ReallocPairs>(I.getKey());
     }
+    state = state->remove<ReallocPairs>(I.getKey());
   }
 
   return state;
@@ -1463,10 +1552,10 @@ MallocChecker::checkRegionChanges(ProgramStateRef State,
 
 static SymbolRef findFailedReallocSymbol(ProgramStateRef currState,
                                          ProgramStateRef prevState) {
-  ReallocMap currMap = currState->get<ReallocPairs>();
-  ReallocMap prevMap = prevState->get<ReallocPairs>();
+  ReallocPairsTy currMap = currState->get<ReallocPairs>();
+  ReallocPairsTy prevMap = prevState->get<ReallocPairs>();
 
-  for (ReallocMap::iterator I = prevMap.begin(), E = prevMap.end();
+  for (ReallocPairsTy::iterator I = prevMap.begin(), E = prevMap.end();
        I != E; ++I) {
     SymbolRef sym = I.getKey();
     if (!currMap.lookup(sym))
diff --git a/lib/StaticAnalyzer/Checkers/MallocSizeofChecker.cpp b/lib/StaticAnalyzer/Checkers/MallocSizeofChecker.cpp
index 6292a47..fb40f22 100644
--- a/lib/StaticAnalyzer/Checkers/MallocSizeofChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/MallocSizeofChecker.cpp
@@ -146,9 +146,9 @@ static bool typesCompatible(ASTContext &C, QualType A, QualType B) {
     
     if (const PointerType *ptrA = A->getAs<PointerType>())
       if (const PointerType *ptrB = B->getAs<PointerType>()) {
-	A = ptrA->getPointeeType();
-	B = ptrB->getPointeeType();
-	continue;
+        A = ptrA->getPointeeType();
+        B = ptrB->getPointeeType();
+        continue;
       }
       
     break;
@@ -157,6 +157,18 @@ static bool typesCompatible(ASTContext &C, QualType A, QualType B) {
   return false;
 }
 
+static bool compatibleWithArrayType(ASTContext &C, QualType PT, QualType T) {
+  // Ex: 'int a[10][2]' is compatible with 'int', 'int[2]', 'int[10][2]'.
+  while (const ArrayType *AT = T->getAsArrayTypeUnsafe()) {
+    QualType ElemType = AT->getElementType();
+    if (typesCompatible(C, PT, AT->getElementType()))
+      return true;
+    T = ElemType;
+  }
+
+  return false;
+}
+
 class MallocSizeofChecker : public Checker<check::ASTCodeBody> {
 public:
   void checkASTCodeBody(const Decl *D, AnalysisManager& mgr,
@@ -184,38 +196,49 @@ public:
           continue;
 
         QualType SizeofType = SFinder.Sizeofs[0]->getTypeOfArgument();
-        if (!typesCompatible(BR.getContext(), PointeeType, SizeofType)) {
-          const TypeSourceInfo *TSI = 0;
-          if (i->CastedExprParent.is<const VarDecl *>()) {
-            TSI =
+
+        if (typesCompatible(BR.getContext(), PointeeType, SizeofType))
+          continue;
+
+        // If the argument to sizeof is an array, the result could be a
+        // pointer to any array element.
+        if (compatibleWithArrayType(BR.getContext(), PointeeType, SizeofType))
+          continue;
+
+        const TypeSourceInfo *TSI = 0;
+        if (i->CastedExprParent.is<const VarDecl *>()) {
+          TSI =
               i->CastedExprParent.get<const VarDecl *>()->getTypeSourceInfo();
-          } else {
-            TSI = i->ExplicitCastType;
-          }
-
-          SmallString<64> buf;
-          llvm::raw_svector_ostream OS(buf);
-
-          OS << "Result of '"
-             << i->AllocCall->getDirectCallee()->getIdentifier()->getName()
-             << "' is converted to a pointer of type '"
-             << PointeeType.getAsString() << "', which is incompatible with "
-             << "sizeof operand type '" << SizeofType.getAsString() << "'";
-          llvm::SmallVector<SourceRange, 4> Ranges;
-          Ranges.push_back(i->AllocCall->getCallee()->getSourceRange());
-          Ranges.push_back(SFinder.Sizeofs[0]->getSourceRange());
-          if (TSI)
-            Ranges.push_back(TSI->getTypeLoc().getSourceRange());
-
-          PathDiagnosticLocation L =
+        } else {
+          TSI = i->ExplicitCastType;
+        }
+
+        SmallString<64> buf;
+        llvm::raw_svector_ostream OS(buf);
+
+        OS << "Result of ";
+        const FunctionDecl *Callee = i->AllocCall->getDirectCallee();
+        if (Callee && Callee->getIdentifier())
+          OS << '\'' << Callee->getIdentifier()->getName() << '\'';
+        else
+          OS << "call";
+        OS << " is converted to a pointer of type '"
+            << PointeeType.getAsString() << "', which is incompatible with "
+            << "sizeof operand type '" << SizeofType.getAsString() << "'";
+        llvm::SmallVector<SourceRange, 4> Ranges;
+        Ranges.push_back(i->AllocCall->getCallee()->getSourceRange());
+        Ranges.push_back(SFinder.Sizeofs[0]->getSourceRange());
+        if (TSI)
+          Ranges.push_back(TSI->getTypeLoc().getSourceRange());
+
+        PathDiagnosticLocation L =
             PathDiagnosticLocation::createBegin(i->AllocCall->getCallee(),
-                                                BR.getSourceManager(), ADC);
+                BR.getSourceManager(), ADC);
 
-          BR.EmitBasicReport(D, "Allocator sizeof operand mismatch",
-                             categories::UnixAPI,
-                             OS.str(),
-                             L, Ranges.data(), Ranges.size());
-        }
+        BR.EmitBasicReport(D, "Allocator sizeof operand mismatch",
+            categories::UnixAPI,
+            OS.str(),
+            L, Ranges.data(), Ranges.size());
       }
     }
   }
diff --git a/lib/StaticAnalyzer/Checkers/NSAutoreleasePoolChecker.cpp b/lib/StaticAnalyzer/Checkers/NSAutoreleasePoolChecker.cpp
index aad3b0f..3331bc8 100644
--- a/lib/StaticAnalyzer/Checkers/NSAutoreleasePoolChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/NSAutoreleasePoolChecker.cpp
@@ -71,7 +71,7 @@ void NSAutoreleasePoolChecker::checkPreObjCMessage(const ObjCMethodCall &msg,
   BugReport *Report = new BugReport(*BT, "Use -drain instead of -release when "
     "using NSAutoreleasePool and garbage collection", N);
   Report->addRange(msg.getSourceRange());
-  C.EmitReport(Report);
+  C.emitReport(Report);
 }
 
 void ento::registerNSAutoreleasePoolChecker(CheckerManager &mgr) {
diff --git a/lib/StaticAnalyzer/Checkers/NSErrorChecker.cpp b/lib/StaticAnalyzer/Checkers/NSErrorChecker.cpp
index f826573..7a66ec3 100644
--- a/lib/StaticAnalyzer/Checkers/NSErrorChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/NSErrorChecker.cpp
@@ -163,23 +163,9 @@ public:
 };
 }
 
-namespace { struct NSErrorOut {}; }
-namespace { struct CFErrorOut {}; }
-
 typedef llvm::ImmutableMap<SymbolRef, unsigned> ErrorOutFlag;
-
-namespace clang {
-namespace ento {
-  template <>
-  struct ProgramStateTrait<NSErrorOut> : public ProgramStatePartialTrait<ErrorOutFlag> {  
-    static void *GDMIndex() { static int index = 0; return &index; }
-  };
-  template <>
-  struct ProgramStateTrait<CFErrorOut> : public ProgramStatePartialTrait<ErrorOutFlag> {  
-    static void *GDMIndex() { static int index = 0; return &index; }
-  };
-}
-}
+REGISTER_TRAIT_WITH_PROGRAMSTATE(NSErrorOut, ErrorOutFlag)
+REGISTER_TRAIT_WITH_PROGRAMSTATE(CFErrorOut, ErrorOutFlag)
 
 template <typename T>
 static bool hasFlag(SVal val, ProgramStateRef state) {
@@ -285,7 +271,7 @@ void NSOrCFErrorDerefChecker::checkEvent(ImplicitNullDerefEvent event) const {
     bug = new CFErrorDerefBug();
   BugReport *report = new BugReport(*bug, os.str(),
                                                     event.SinkNode);
-  BR.EmitReport(report);
+  BR.emitReport(report);
 }
 
 static bool IsNSError(QualType T, IdentifierInfo *II) {
diff --git a/lib/StaticAnalyzer/Checkers/OSAtomicChecker.cpp b/lib/StaticAnalyzer/Checkers/OSAtomicChecker.cpp
deleted file mode 100644
index 7b724d2..0000000
--- a/lib/StaticAnalyzer/Checkers/OSAtomicChecker.cpp
+++ /dev/null
@@ -1,218 +0,0 @@
-//=== OSAtomicChecker.cpp - OSAtomic functions evaluator --------*- C++ -*-===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This checker evaluates OSAtomic functions.
-//
-//===----------------------------------------------------------------------===//
-
-#include "ClangSACheckers.h"
-#include "clang/StaticAnalyzer/Core/Checker.h"
-#include "clang/StaticAnalyzer/Core/CheckerManager.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
-#include "clang/Basic/Builtins.h"
-
-using namespace clang;
-using namespace ento;
-
-namespace {
-
-class OSAtomicChecker : public Checker<eval::InlineCall> {
-public:
-  bool inlineCall(const CallExpr *CE, ExprEngine &Eng,
-                  ExplodedNode *Pred, ExplodedNodeSet &Dst) const;
-
-private:
-  bool evalOSAtomicCompareAndSwap(const CallExpr *CE,
-                                  ExprEngine &Eng,
-                                  ExplodedNode *Pred,
-                                  ExplodedNodeSet &Dst) const;
-};
-}
-
-static StringRef getCalleeName(ProgramStateRef State,
-                               const CallExpr *CE,
-                               const LocationContext *LCtx) {
-  const Expr *Callee = CE->getCallee();
-  SVal L = State->getSVal(Callee, LCtx);
-  const FunctionDecl *funDecl =  L.getAsFunctionDecl();
-  if (!funDecl)
-    return StringRef();
-  IdentifierInfo *funI = funDecl->getIdentifier();
-  if (!funI)
-    return StringRef();
-  return funI->getName();
-}
-
-bool OSAtomicChecker::inlineCall(const CallExpr *CE,
-                                 ExprEngine &Eng,
-                                 ExplodedNode *Pred,
-                                 ExplodedNodeSet &Dst) const {
-  StringRef FName = getCalleeName(Pred->getState(),
-                                  CE, Pred->getLocationContext());
-  if (FName.empty())
-    return false;
-
-  // Check for compare and swap.
-  if (FName.startswith("OSAtomicCompareAndSwap") ||
-      FName.startswith("objc_atomicCompareAndSwap"))
-    return evalOSAtomicCompareAndSwap(CE, Eng, Pred, Dst);
-
-  // FIXME: Other atomics.
-  return false;
-}
-
-bool OSAtomicChecker::evalOSAtomicCompareAndSwap(const CallExpr *CE,
-                                                 ExprEngine &Eng,
-                                                 ExplodedNode *Pred,
-                                                 ExplodedNodeSet &Dst) const {
-  // Not enough arguments to match OSAtomicCompareAndSwap?
-  if (CE->getNumArgs() != 3)
-    return false;
-
-  ASTContext &Ctx = Eng.getContext();
-  const Expr *oldValueExpr = CE->getArg(0);
-  QualType oldValueType = Ctx.getCanonicalType(oldValueExpr->getType());
-
-  const Expr *newValueExpr = CE->getArg(1);
-  QualType newValueType = Ctx.getCanonicalType(newValueExpr->getType());
-
-  // Do the types of 'oldValue' and 'newValue' match?
-  if (oldValueType != newValueType)
-    return false;
-
-  const Expr *theValueExpr = CE->getArg(2);
-  const PointerType *theValueType=theValueExpr->getType()->getAs<PointerType>();
-
-  // theValueType not a pointer?
-  if (!theValueType)
-    return false;
-
-  QualType theValueTypePointee =
-    Ctx.getCanonicalType(theValueType->getPointeeType()).getUnqualifiedType();
-
-  // The pointee must match newValueType and oldValueType.
-  if (theValueTypePointee != newValueType)
-    return false;
-
-  static SimpleProgramPointTag OSAtomicLoadTag("OSAtomicChecker : Load");
-  static SimpleProgramPointTag OSAtomicStoreTag("OSAtomicChecker : Store");
-  
-  // Load 'theValue'.
-  ProgramStateRef state = Pred->getState();
-  const LocationContext *LCtx = Pred->getLocationContext();
-  ExplodedNodeSet Tmp;
-  SVal location = state->getSVal(theValueExpr, LCtx);
-  // Here we should use the value type of the region as the load type, because
-  // we are simulating the semantics of the function, not the semantics of 
-  // passing argument. So the type of theValue expr is not we are loading.
-  // But usually the type of the varregion is not the type we want either,
-  // we still need to do a CastRetrievedVal in store manager. So actually this
-  // LoadTy specifying can be omitted. But we put it here to emphasize the 
-  // semantics.
-  QualType LoadTy;
-  if (const TypedValueRegion *TR =
-      dyn_cast_or_null<TypedValueRegion>(location.getAsRegion())) {
-    LoadTy = TR->getValueType();
-  }
-  Eng.evalLoad(Tmp, CE, theValueExpr, Pred,
-               state, location, &OSAtomicLoadTag, LoadTy);
-
-  if (Tmp.empty()) {
-    // If no nodes were generated, other checkers must have generated sinks. 
-    // We return an empty Dst.
-    return true;
-  }
- 
-  for (ExplodedNodeSet::iterator I = Tmp.begin(), E = Tmp.end();
-       I != E; ++I) {
-
-    ExplodedNode *N = *I;
-    ProgramStateRef stateLoad = N->getState();
-
-    // Use direct bindings from the environment since we are forcing a load
-    // from a location that the Environment would typically not be used
-    // to bind a value.
-    SVal theValueVal_untested = stateLoad->getSVal(theValueExpr, LCtx, true);
-
-    SVal oldValueVal_untested = stateLoad->getSVal(oldValueExpr, LCtx);
-
-    // FIXME: Issue an error.
-    if (theValueVal_untested.isUndef() || oldValueVal_untested.isUndef()) {
-      return false;
-    }
-    
-    DefinedOrUnknownSVal theValueVal =
-      cast<DefinedOrUnknownSVal>(theValueVal_untested);
-    DefinedOrUnknownSVal oldValueVal =
-      cast<DefinedOrUnknownSVal>(oldValueVal_untested);
-
-    SValBuilder &svalBuilder = Eng.getSValBuilder();
-
-    // Perform the comparison.
-    DefinedOrUnknownSVal Cmp =
-      svalBuilder.evalEQ(stateLoad,theValueVal,oldValueVal);
-
-    ProgramStateRef stateEqual = stateLoad->assume(Cmp, true);
-
-    // Were they equal?
-    if (stateEqual) {
-      // Perform the store.
-      ExplodedNodeSet TmpStore;
-      SVal val = stateEqual->getSVal(newValueExpr, LCtx);
-
-      // Handle implicit value casts.
-      if (const TypedValueRegion *R =
-          dyn_cast_or_null<TypedValueRegion>(location.getAsRegion())) {
-        val = svalBuilder.evalCast(val,R->getValueType(), newValueExpr->getType());
-      }
-
-      Eng.evalStore(TmpStore, CE, theValueExpr, N,
-                    stateEqual, location, val, &OSAtomicStoreTag);
-
-      if (TmpStore.empty()) {
-        // If no nodes were generated, other checkers must have generated sinks. 
-        // We return an empty Dst.
-        return true;
-      }
-      
-      StmtNodeBuilder B(TmpStore, Dst, Eng.getBuilderContext());
-      // Now bind the result of the comparison.
-      for (ExplodedNodeSet::iterator I2 = TmpStore.begin(),
-           E2 = TmpStore.end(); I2 != E2; ++I2) {
-        ExplodedNode *predNew = *I2;
-        ProgramStateRef stateNew = predNew->getState();
-        // Check for 'void' return type if we have a bogus function prototype.
-        SVal Res = UnknownVal();
-        QualType T = CE->getType();
-        if (!T->isVoidType())
-          Res = Eng.getSValBuilder().makeTruthVal(true, T);
-        B.generateNode(CE, predNew, stateNew->BindExpr(CE, LCtx, Res),
-                       false, this);
-      }
-    }
-
-    // Were they not equal?
-    if (ProgramStateRef stateNotEqual = stateLoad->assume(Cmp, false)) {
-      // Check for 'void' return type if we have a bogus function prototype.
-      SVal Res = UnknownVal();
-      QualType T = CE->getType();
-      if (!T->isVoidType())
-        Res = Eng.getSValBuilder().makeTruthVal(false, CE->getType());
-      StmtNodeBuilder B(N, Dst, Eng.getBuilderContext());    
-      B.generateNode(CE, N, stateNotEqual->BindExpr(CE, LCtx, Res),
-                     false, this);
-    }
-  }
-
-  return true;
-}
-
-void ento::registerOSAtomicChecker(CheckerManager &mgr) {
-  mgr.registerChecker<OSAtomicChecker>();
-}
diff --git a/lib/StaticAnalyzer/Checkers/ObjCAtSyncChecker.cpp b/lib/StaticAnalyzer/Checkers/ObjCAtSyncChecker.cpp
index 4cc92ce..9d84f52 100644
--- a/lib/StaticAnalyzer/Checkers/ObjCAtSyncChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/ObjCAtSyncChecker.cpp
@@ -18,7 +18,6 @@
 #include "clang/StaticAnalyzer/Core/CheckerManager.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
-#include "clang/StaticAnalyzer/Checkers/DereferenceChecker.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
 
 using namespace clang;
@@ -50,8 +49,8 @@ void ObjCAtSyncChecker::checkPreStmt(const ObjCAtSynchronizedStmt *S,
                                   "for @synchronized"));
       BugReport *report =
         new BugReport(*BT_undef, BT_undef->getDescription(), N);
-      bugreporter::addTrackNullOrUndefValueVisitor(N, Ex, report);
-      C.EmitReport(report);
+      bugreporter::trackNullOrUndefValue(N, Ex, *report);
+      C.emitReport(report);
     }
     return;
   }
@@ -73,9 +72,9 @@ void ObjCAtSyncChecker::checkPreStmt(const ObjCAtSynchronizedStmt *S,
                                    "(no synchronization will occur)"));
         BugReport *report =
           new BugReport(*BT_null, BT_null->getDescription(), N);
-        bugreporter::addTrackNullOrUndefValueVisitor(N, Ex, report);
+        bugreporter::trackNullOrUndefValue(N, Ex, *report);
 
-        C.EmitReport(report);
+        C.emitReport(report);
         return;
       }
     }
diff --git a/lib/StaticAnalyzer/Checkers/ObjCContainersASTChecker.cpp b/lib/StaticAnalyzer/Checkers/ObjCContainersASTChecker.cpp
index f2929c0..63a8480 100644
--- a/lib/StaticAnalyzer/Checkers/ObjCContainersASTChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/ObjCContainersASTChecker.cpp
@@ -31,8 +31,6 @@ class WalkAST : public StmtVisitor<WalkAST> {
   ASTContext &ASTC;
   uint64_t PtrWidth;
 
-  static const unsigned InvalidArgIndex = UINT_MAX;
-
   /// Check if the type has pointer size (very conservative).
   inline bool isPointerSize(const Type *T) {
     if (!T)
@@ -102,16 +100,18 @@ void WalkAST::VisitCallExpr(CallExpr *CE) {
     return;
 
   const Expr *Arg = 0;
-  unsigned ArgNum = InvalidArgIndex;
+  unsigned ArgNum;
 
   if (Name.equals("CFArrayCreate") || Name.equals("CFSetCreate")) {
+    if (CE->getNumArgs() != 4)
+      return;
     ArgNum = 1;
     Arg = CE->getArg(ArgNum)->IgnoreParenCasts();
     if (hasPointerToPointerSizedType(Arg))
         return;
-  }
-
-  if (Arg == 0 && Name.equals("CFDictionaryCreate")) {
+  } else if (Name.equals("CFDictionaryCreate")) {
+    if (CE->getNumArgs() != 6)
+      return;
     // Check first argument.
     ArgNum = 1;
     Arg = CE->getArg(ArgNum)->IgnoreParenCasts();
@@ -125,17 +125,18 @@ void WalkAST::VisitCallExpr(CallExpr *CE) {
     }
   }
 
-  if (ArgNum != InvalidArgIndex) {
+  if (Arg) {
     assert(ArgNum == 1 || ArgNum == 2);
 
-    SmallString<256> BufName;
+    SmallString<64> BufName;
     llvm::raw_svector_ostream OsName(BufName);
-    assert(ArgNum == 1 || ArgNum == 2);
     OsName << " Invalid use of '" << Name << "'" ;
 
     SmallString<256> Buf;
     llvm::raw_svector_ostream Os(Buf);
-    Os << " The "<< ((ArgNum == 1) ? "first" : "second") << " argument to '"
+    // Use "second" and "third" since users will expect 1-based indexing
+    // for parameter names when mentioned in prose.
+    Os << " The "<< ((ArgNum == 1) ? "second" : "third") << " argument to '"
         << Name << "' must be a C array of pointer-sized values, not '"
         << Arg->getType().getAsString() << "'";
 
diff --git a/lib/StaticAnalyzer/Checkers/ObjCContainersChecker.cpp b/lib/StaticAnalyzer/Checkers/ObjCContainersChecker.cpp
index 2ab49ed..999c994 100644
--- a/lib/StaticAnalyzer/Checkers/ObjCContainersChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/ObjCContainersChecker.cpp
@@ -55,16 +55,8 @@ public:
 };
 } // end anonymous namespace
 
-// ProgramState trait - a map from array symbol to it's state.
-typedef llvm::ImmutableMap<SymbolRef, DefinedSVal> ArraySizeM;
-
-namespace { struct ArraySizeMap {}; }
-namespace clang { namespace ento {
-template<> struct ProgramStateTrait<ArraySizeMap>
-    :  public ProgramStatePartialTrait<ArraySizeM > {
-  static void *GDMIndex() { return ObjCContainersChecker::getTag(); }
-};
-}}
+// ProgramState trait - a map from array symbol to its state.
+REGISTER_MAP_WITH_PROGRAMSTATE(ArraySizeMap, SymbolRef, DefinedSVal)
 
 void ObjCContainersChecker::addSizeInfo(const Expr *Array, const Expr *Size,
                                         CheckerContext &C) const {
@@ -146,7 +138,7 @@ void ObjCContainersChecker::checkPreStmt(const CallExpr *CE,
       initBugType();
       BugReport *R = new BugReport(*BT, "Index is out of bounds", N);
       R->addRange(IdxExpr->getSourceRange());
-      C.EmitReport(R);
+      C.emitReport(R);
       return;
     }
   }
diff --git a/lib/StaticAnalyzer/Checkers/ObjCMissingSuperCallChecker.cpp b/lib/StaticAnalyzer/Checkers/ObjCMissingSuperCallChecker.cpp
new file mode 100644
index 0000000..e906e8a
--- /dev/null
+++ b/lib/StaticAnalyzer/Checkers/ObjCMissingSuperCallChecker.cpp
@@ -0,0 +1,203 @@
+//==- ObjCMissingSuperCallChecker.cpp - Check missing super-calls in ObjC --==//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines a ObjCMissingSuperCallChecker, a checker that
+//  analyzes a UIViewController implementation to determine if it
+//  correctly calls super in the methods where this is mandatory.
+//
+//===----------------------------------------------------------------------===//
+
+#include "ClangSACheckers.h"
+#include "clang/StaticAnalyzer/Core/Checker.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/AnalysisManager.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/PathDiagnostic.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
+#include "clang/AST/ExprObjC.h"
+#include "clang/AST/Expr.h"
+#include "clang/AST/DeclObjC.h"
+#include "clang/AST/RecursiveASTVisitor.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/SmallSet.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace clang;
+using namespace ento;
+
+static bool isUIViewControllerSubclass(ASTContext &Ctx, 
+                                       const ObjCImplementationDecl *D) {
+  IdentifierInfo *ViewControllerII = &Ctx.Idents.get("UIViewController");
+  const ObjCInterfaceDecl *ID = D->getClassInterface();
+
+  for ( ; ID; ID = ID->getSuperClass())
+    if (ID->getIdentifier() == ViewControllerII)
+      return true;
+  return false;  
+}
+
+//===----------------------------------------------------------------------===//
+// FindSuperCallVisitor - Identify specific calls to the superclass.
+//===----------------------------------------------------------------------===//
+
+class FindSuperCallVisitor : public RecursiveASTVisitor<FindSuperCallVisitor> {
+public:
+  explicit FindSuperCallVisitor(Selector S) : DoesCallSuper(false), Sel(S) {}
+
+  bool VisitObjCMessageExpr(ObjCMessageExpr *E) {
+    if (E->getSelector() == Sel)
+      if (E->getReceiverKind() == ObjCMessageExpr::SuperInstance)
+        DoesCallSuper = true;
+
+    // Recurse if we didn't find the super call yet.
+    return !DoesCallSuper; 
+  }
+
+  bool DoesCallSuper;
+
+private:
+  Selector Sel;
+};
+
+//===----------------------------------------------------------------------===//
+// ObjCSuperCallChecker 
+//===----------------------------------------------------------------------===//
+
+namespace {
+class ObjCSuperCallChecker : public Checker<
+                                      check::ASTDecl<ObjCImplementationDecl> > {
+public:
+  void checkASTDecl(const ObjCImplementationDecl *D, AnalysisManager &Mgr,
+                    BugReporter &BR) const;
+};
+}
+
+void ObjCSuperCallChecker::checkASTDecl(const ObjCImplementationDecl *D,
+                                        AnalysisManager &Mgr,
+                                        BugReporter &BR) const {
+  ASTContext &Ctx = BR.getContext();
+
+  if (!isUIViewControllerSubclass(Ctx, D))
+    return;
+
+  const char *SelectorNames[] = 
+    {"addChildViewController", "viewDidAppear", "viewDidDisappear", 
+     "viewWillAppear", "viewWillDisappear", "removeFromParentViewController",
+     "didReceiveMemoryWarning", "viewDidUnload", "viewWillUnload",
+     "viewDidLoad"};
+  const unsigned SelectorArgumentCounts[] =
+   {1, 1, 1, 1, 1, 0, 0, 0, 0, 0};
+  const size_t SelectorCount = llvm::array_lengthof(SelectorNames);
+  assert(llvm::array_lengthof(SelectorArgumentCounts) == SelectorCount);
+
+  // Fill the Selectors SmallSet with all selectors we want to check.
+  llvm::SmallSet<Selector, 16> Selectors;
+  for (size_t i = 0; i < SelectorCount; i++) { 
+    unsigned ArgumentCount = SelectorArgumentCounts[i];
+    const char *SelectorCString = SelectorNames[i];
+
+    // Get the selector.
+    IdentifierInfo *II = &Ctx.Idents.get(SelectorCString);
+    Selectors.insert(Ctx.Selectors.getSelector(ArgumentCount, &II));
+  }
+
+  // Iterate over all instance methods.
+  for (ObjCImplementationDecl::instmeth_iterator I = D->instmeth_begin(),
+                                                 E = D->instmeth_end();
+       I != E; ++I) {
+    Selector S = (*I)->getSelector();
+    // Find out whether this is a selector that we want to check.
+    if (!Selectors.count(S))
+      continue;
+
+    ObjCMethodDecl *MD = *I;
+
+    // Check if the method calls its superclass implementation.
+    if (MD->getBody())
+    {
+      FindSuperCallVisitor Visitor(S);
+      Visitor.TraverseDecl(MD);
+
+      // It doesn't call super, emit a diagnostic.
+      if (!Visitor.DoesCallSuper) {
+        PathDiagnosticLocation DLoc =
+          PathDiagnosticLocation::createEnd(MD->getBody(),
+                                            BR.getSourceManager(),
+                                            Mgr.getAnalysisDeclContext(D));
+
+        const char *Name = "Missing call to superclass";
+        SmallString<256> Buf;
+        llvm::raw_svector_ostream os(Buf);
+
+        os << "The '" << S.getAsString() 
+           << "' instance method in UIViewController subclass '" << *D
+           << "' is missing a [super " << S.getAsString() << "] call";
+
+        BR.EmitBasicReport(MD, Name, categories::CoreFoundationObjectiveC,
+                           os.str(), DLoc);
+      }
+    }
+  }
+}
+
+
+//===----------------------------------------------------------------------===//
+// Check registration.
+//===----------------------------------------------------------------------===//
+
+void ento::registerObjCSuperCallChecker(CheckerManager &Mgr) {
+  Mgr.registerChecker<ObjCSuperCallChecker>();
+}
+
+
+/*
+ ToDo list for expanding this check in the future, the list is not exhaustive.
+ There are also cases where calling super is suggested but not "mandatory".
+ In addition to be able to check the classes and methods below, architectural
+ improvements like being able to allow for the super-call to be done in a called
+ method would be good too.
+
+*** trivial cases:
+UIResponder subclasses
+- resignFirstResponder
+
+NSResponder subclasses
+- cursorUpdate
+
+*** more difficult cases:
+
+UIDocument subclasses
+- finishedHandlingError:recovered: (is multi-arg)
+- finishedHandlingError:recovered: (is multi-arg)
+
+UIViewController subclasses
+- loadView (should *never* call super)
+- transitionFromViewController:toViewController:
+         duration:options:animations:completion: (is multi-arg)
+
+UICollectionViewController subclasses
+- loadView (take care because UIViewController subclasses should NOT call super
+            in loadView, but UICollectionViewController subclasses should)
+
+NSObject subclasses
+- doesNotRecognizeSelector (it only has to call super if it doesn't throw)
+
+UIPopoverBackgroundView subclasses (some of those are class methods)
+- arrowDirection (should *never* call super)
+- arrowOffset (should *never* call super)
+- arrowBase (should *never* call super)
+- arrowHeight (should *never* call super)
+- contentViewInsets (should *never* call super)
+
+UITextSelectionRect subclasses (some of those are properties)
+- rect (should *never* call super)
+- range (should *never* call super)
+- writingDirection (should *never* call super)
+- isVertical (should *never* call super)
+- containsStart (should *never* call super)
+- containsEnd (should *never* call super)
+*/
diff --git a/lib/StaticAnalyzer/Checkers/ObjCSelfInitChecker.cpp b/lib/StaticAnalyzer/Checkers/ObjCSelfInitChecker.cpp
index be45da1..98d2a85a 100644
--- a/lib/StaticAnalyzer/Checkers/ObjCSelfInitChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/ObjCSelfInitChecker.cpp
@@ -72,6 +72,8 @@ public:
   void checkPreCall(const CallEvent &CE, CheckerContext &C) const;
   void checkPostCall(const CallEvent &CE, CheckerContext &C) const;
 
+  void printState(raw_ostream &Out, ProgramStateRef State,
+                  const char *NL, const char *Sep) const;
 };
 } // end anonymous namespace
 
@@ -97,31 +99,14 @@ enum SelfFlagEnum {
 };
 }
 
-typedef llvm::ImmutableMap<SymbolRef, unsigned> SelfFlag;
-namespace { struct CalledInit {}; }
-namespace { struct PreCallSelfFlags {}; }
-
-namespace clang {
-namespace ento {
-  template<>
-  struct ProgramStateTrait<SelfFlag> : public ProgramStatePartialTrait<SelfFlag> {
-    static void *GDMIndex() { static int index = 0; return &index; }
-  };
-  template <>
-  struct ProgramStateTrait<CalledInit> : public ProgramStatePartialTrait<bool> {
-    static void *GDMIndex() { static int index = 0; return &index; }
-  };
-
-  /// \brief A call receiving a reference to 'self' invalidates the object that
-  /// 'self' contains. This keeps the "self flags" assigned to the 'self'
-  /// object before the call so we can assign them to the new object that 'self'
-  /// points to after the call.
-  template <>
-  struct ProgramStateTrait<PreCallSelfFlags> : public ProgramStatePartialTrait<unsigned> {
-    static void *GDMIndex() { static int index = 0; return &index; }
-  };
-}
-}
+REGISTER_MAP_WITH_PROGRAMSTATE(SelfFlag, SymbolRef, unsigned)
+REGISTER_TRAIT_WITH_PROGRAMSTATE(CalledInit, bool)
+
+/// \brief A call receiving a reference to 'self' invalidates the object that
+/// 'self' contains. This keeps the "self flags" assigned to the 'self'
+/// object before the call so we can assign them to the new object that 'self'
+/// points to after the call.
+REGISTER_TRAIT_WITH_PROGRAMSTATE(PreCallSelfFlags, unsigned)
 
 static SelfFlagEnum getSelfFlags(SVal val, ProgramStateRef state) {
   if (SymbolRef sym = val.getAsSymbol())
@@ -138,7 +123,8 @@ static void addSelfFlag(ProgramStateRef state, SVal val,
                         SelfFlagEnum flag, CheckerContext &C) {
   // We tag the symbol that the SVal wraps.
   if (SymbolRef sym = val.getAsSymbol())
-    C.addTransition(state->set<SelfFlag>(sym, getSelfFlags(val, C) | flag));
+    state = state->set<SelfFlag>(sym, getSelfFlags(val, state) | flag);
+  C.addTransition(state);
 }
 
 static bool hasSelfFlag(SVal val, SelfFlagEnum flag, CheckerContext &C) {
@@ -176,7 +162,7 @@ static void checkForInvalidSelf(const Expr *E, CheckerContext &C,
 
   BugReport *report =
     new BugReport(*new InitSelfBug(), errorStr, N);
-  C.EmitReport(report);
+  C.emitReport(report);
 }
 
 void ObjCSelfInitChecker::checkPostObjCMessage(const ObjCMethodCall &Msg,
@@ -305,13 +291,12 @@ void ObjCSelfInitChecker::checkPostCall(const CallEvent &CE,
       // returns 'self'. So assign the flags, which were set on 'self' to the
       // return value.
       // EX: self = performMoreInitialization(self)
-      const Expr *CallExpr = CE.getOriginExpr();
-      if (CallExpr)
-        addSelfFlag(state, state->getSVal(CallExpr, C.getLocationContext()),
-                                          prevFlags, C);
+      addSelfFlag(state, CE.getReturnValue(), prevFlags, C);
       return;
     }
   }
+
+  C.addTransition(state);
 }
 
 void ObjCSelfInitChecker::checkLocation(SVal location, bool isLoad,
@@ -346,6 +331,53 @@ void ObjCSelfInitChecker::checkBind(SVal loc, SVal val, const Stmt *S,
   }
 }
 
+void ObjCSelfInitChecker::printState(raw_ostream &Out, ProgramStateRef State,
+                                     const char *NL, const char *Sep) const {
+  SelfFlagTy FlagMap = State->get<SelfFlag>();
+  bool DidCallInit = State->get<CalledInit>();
+  SelfFlagEnum PreCallFlags = (SelfFlagEnum)State->get<PreCallSelfFlags>();
+
+  if (FlagMap.isEmpty() && !DidCallInit && !PreCallFlags)
+    return;
+
+  Out << Sep << NL << "ObjCSelfInitChecker:" << NL;
+
+  if (DidCallInit)
+    Out << "  An init method has been called." << NL;
+
+  if (PreCallFlags != SelfFlag_None) {
+    if (PreCallFlags & SelfFlag_Self) {
+      Out << "  An argument of the current call came from the 'self' variable."
+          << NL;
+    }
+    if (PreCallFlags & SelfFlag_InitRes) {
+      Out << "  An argument of the current call came from an init method."
+          << NL;
+    }
+  }
+
+  Out << NL;
+  for (SelfFlagTy::iterator I = FlagMap.begin(), E = FlagMap.end();
+       I != E; ++I) {
+    Out << I->first << " : ";
+
+    if (I->second == SelfFlag_None)
+      Out << "none";
+
+    if (I->second & SelfFlag_Self)
+      Out << "self variable";
+
+    if (I->second & SelfFlag_InitRes) {
+      if (I->second != SelfFlag_InitRes)
+        Out << " | ";
+      Out << "result of init method";
+    }
+
+    Out << NL;
+  }
+}
+
+
 // FIXME: A callback should disable checkers at the start of functions.
 static bool shouldRunOnFunctionOrMethod(const NamedDecl *ND) {
   if (!ND)
diff --git a/lib/StaticAnalyzer/Checkers/PointerArithChecker.cpp b/lib/StaticAnalyzer/Checkers/PointerArithChecker.cpp
index fe4845b..b5d9959 100644
--- a/lib/StaticAnalyzer/Checkers/PointerArithChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/PointerArithChecker.cpp
@@ -59,7 +59,7 @@ void PointerArithChecker::checkPreStmt(const BinaryOperator *B,
                             "dangerous."));
       BugReport *R = new BugReport(*BT, BT->getDescription(), N);
       R->addRange(B->getSourceRange());
-      C.EmitReport(R);
+      C.emitReport(R);
     }
   }
 }
diff --git a/lib/StaticAnalyzer/Checkers/PointerSubChecker.cpp b/lib/StaticAnalyzer/Checkers/PointerSubChecker.cpp
index fa5c6a3..47da87f 100644
--- a/lib/StaticAnalyzer/Checkers/PointerSubChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/PointerSubChecker.cpp
@@ -67,7 +67,7 @@ void PointerSubChecker::checkPreStmt(const BinaryOperator *B,
                           "the same memory chunk may cause incorrect result."));
     BugReport *R = new BugReport(*BT, BT->getDescription(), N);
     R->addRange(B->getSourceRange());
-    C.EmitReport(R);
+    C.emitReport(R);
   }
 }
 
diff --git a/lib/StaticAnalyzer/Checkers/PthreadLockChecker.cpp b/lib/StaticAnalyzer/Checkers/PthreadLockChecker.cpp
index 2d018ef..d9b6384 100644
--- a/lib/StaticAnalyzer/Checkers/PthreadLockChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/PthreadLockChecker.cpp
@@ -43,15 +43,7 @@ public:
 } // end anonymous namespace
 
 // GDM Entry for tracking lock state.
-namespace { class LockSet {}; }
-namespace clang {
-namespace ento {
-template <> struct ProgramStateTrait<LockSet> :
-  public ProgramStatePartialTrait<llvm::ImmutableList<const MemRegion*> > {
-    static void *GDMIndex() { static int x = 0; return &x; }
-};
-} // end of ento (ProgramState) namespace
-} // end clang namespace
+REGISTER_LIST_WITH_PROGRAMSTATE(LockSet, const MemRegion *)
 
 
 void PthreadLockChecker::checkPostStmt(const CallExpr *CE,
@@ -118,7 +110,7 @@ void PthreadLockChecker::AcquireLock(CheckerContext &C, const CallExpr *CE,
                                                       "This lock has already "
                                                       "been acquired", N);
     report->addRange(CE->getArg(0)->getSourceRange());
-    C.EmitReport(report);
+    C.emitReport(report);
     return;
   }
 
@@ -163,7 +155,7 @@ void PthreadLockChecker::ReleaseLock(CheckerContext &C, const CallExpr *CE,
     return;
   
   ProgramStateRef state = C.getState();
-  llvm::ImmutableList<const MemRegion*> LS = state->get<LockSet>();
+  LockSetTy LS = state->get<LockSet>();
 
   // FIXME: Better analysis requires IPA for wrappers.
   // FIXME: check for double unlocks
@@ -183,7 +175,7 @@ void PthreadLockChecker::ReleaseLock(CheckerContext &C, const CallExpr *CE,
                                                       "Possible lock order "
                                                       "reversal", N);
     report->addRange(CE->getArg(0)->getSourceRange());
-    C.EmitReport(report);
+    C.emitReport(report);
     return;
   }
 
diff --git a/lib/StaticAnalyzer/Checkers/RetainCountChecker.cpp b/lib/StaticAnalyzer/Checkers/RetainCountChecker.cpp
index 3c00d99..304051c 100644
--- a/lib/StaticAnalyzer/Checkers/RetainCountChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/RetainCountChecker.cpp
@@ -40,24 +40,6 @@ using namespace clang;
 using namespace ento;
 using llvm::StrInStrNoCase;
 
-namespace {
-/// Wrapper around different kinds of node builder, so that helper functions
-/// can have a common interface.
-class GenericNodeBuilderRefCount {
-  CheckerContext *C;
-  const ProgramPointTag *tag;
-public:
-  GenericNodeBuilderRefCount(CheckerContext &c,
-                             const ProgramPointTag *t = 0)
-  : C(&c), tag(t){}
-
-  ExplodedNode *MakeNode(ProgramStateRef state, ExplodedNode *Pred,
-                         bool MarkAsSink = false) {
-    return C->addTransition(state, Pred, tag, MarkAsSink);
-  }
-};
-} // end anonymous namespace
-
 //===----------------------------------------------------------------------===//
 // Primitives used for constructing summaries for function/method calls.
 //===----------------------------------------------------------------------===//
@@ -66,9 +48,23 @@ public:
 /// particular argument.
 enum ArgEffect { DoNothing, Autorelease, Dealloc, DecRef, DecRefMsg,
                  DecRefBridgedTransfered,
-                 DecRefAndStopTracking, DecRefMsgAndStopTracking,
                  IncRefMsg, IncRef, MakeCollectable, MayEscape,
-                 NewAutoreleasePool, StopTracking };
+                 NewAutoreleasePool,
+
+                 // Stop tracking the argument - the effect of the call is
+                 // unknown.
+                 StopTracking,
+
+                 // In some cases, we obtain a better summary for this checker
+                 // by looking at the call site than by inlining the function.
+                 // Signifies that we should stop tracking the symbol even if
+                 // the function is inlined.
+                 StopTrackingHard,
+
+                 // The function decrements the reference count and the checker 
+                 // should stop tracking the argument.
+                 DecRefAndStopTrackingHard, DecRefMsgAndStopTrackingHard
+               };
 
 namespace llvm {
 template <> struct FoldingSetTrait<ArgEffect> {
@@ -90,7 +86,13 @@ class RetEffect {
 public:
   enum Kind { NoRet, OwnedSymbol, OwnedAllocatedSymbol,
               NotOwnedSymbol, GCNotOwnedSymbol, ARCNotOwnedSymbol,
-              OwnedWhenTrackedReceiver };
+              OwnedWhenTrackedReceiver,
+              // Treat this function as returning a non-tracked symbol even if
+              // the function has been inlined. This is used where the call
+              // site summary is more presise than the summary indirectly produced 
+              // by inlining the function
+              NoRetHard
+            };
 
   enum ObjKind { CF, ObjC, AnyObj };
 
@@ -133,6 +135,9 @@ public:
   static RetEffect MakeNoRet() {
     return RetEffect(NoRet);
   }
+  static RetEffect MakeNoRetHard() {
+    return RetEffect(NoRetHard);
+  }
 
   void Profile(llvm::FoldingSetNodeID& ID) const {
     ID.AddInteger((unsigned) K);
@@ -337,20 +342,7 @@ void RefVal::print(raw_ostream &Out) const {
 // RefBindings - State used to track object reference counts.
 //===----------------------------------------------------------------------===//
 
-typedef llvm::ImmutableMap<SymbolRef, RefVal> RefBindings;
-
-namespace clang {
-namespace ento {
-template<>
-struct ProgramStateTrait<RefBindings>
-  : public ProgramStatePartialTrait<RefBindings> {
-  static void *GDMIndex() {
-    static int RefBIndex = 0;
-    return &RefBIndex;
-  }
-};
-}
-}
+REGISTER_MAP_WITH_PROGRAMSTATE(RefBindings, SymbolRef, RefVal)
 
 static inline const RefVal *getRefBinding(ProgramStateRef State,
                                           SymbolRef Sym) {
@@ -893,7 +885,7 @@ static bool isMakeCollectable(const FunctionDecl *FD, StringRef FName) {
   return FName.find("MakeCollectable") != StringRef::npos;
 }
 
-static ArgEffect getStopTrackingEquivalent(ArgEffect E) {
+static ArgEffect getStopTrackingHardEquivalent(ArgEffect E) {
   switch (E) {
   case DoNothing:
   case Autorelease:
@@ -904,13 +896,14 @@ static ArgEffect getStopTrackingEquivalent(ArgEffect E) {
   case MayEscape:
   case NewAutoreleasePool:
   case StopTracking:
-    return StopTracking;
+  case StopTrackingHard:
+    return StopTrackingHard;
   case DecRef:
-  case DecRefAndStopTracking:
-    return DecRefAndStopTracking;
+  case DecRefAndStopTrackingHard:
+    return DecRefAndStopTrackingHard;
   case DecRefMsg:
-  case DecRefMsgAndStopTracking:
-    return DecRefMsgAndStopTracking;
+  case DecRefMsgAndStopTrackingHard:
+    return DecRefMsgAndStopTrackingHard;
   case Dealloc:
     return Dealloc;
   }
@@ -921,33 +914,65 @@ static ArgEffect getStopTrackingEquivalent(ArgEffect E) {
 void RetainSummaryManager::updateSummaryForCall(const RetainSummary *&S,
                                                 const CallEvent &Call) {
   if (Call.hasNonZeroCallbackArg()) {
-    ArgEffect RecEffect = getStopTrackingEquivalent(S->getReceiverEffect());
-    ArgEffect DefEffect = getStopTrackingEquivalent(S->getDefaultArgEffect());
+    ArgEffect RecEffect =
+      getStopTrackingHardEquivalent(S->getReceiverEffect());
+    ArgEffect DefEffect =
+      getStopTrackingHardEquivalent(S->getDefaultArgEffect());
 
     ArgEffects CustomArgEffects = S->getArgEffects();
     for (ArgEffects::iterator I = CustomArgEffects.begin(),
                               E = CustomArgEffects.end();
          I != E; ++I) {
-      ArgEffect Translated = getStopTrackingEquivalent(I->second);
+      ArgEffect Translated = getStopTrackingHardEquivalent(I->second);
       if (Translated != DefEffect)
         ScratchArgs = AF.add(ScratchArgs, I->first, Translated);
     }
 
-    RetEffect RE = RetEffect::MakeNoRet();
+    RetEffect RE = RetEffect::MakeNoRetHard();
 
     // Special cases where the callback argument CANNOT free the return value.
     // This can generally only happen if we know that the callback will only be
     // called when the return value is already being deallocated.
     if (const FunctionCall *FC = dyn_cast<FunctionCall>(&Call)) {
-      IdentifierInfo *Name = FC->getDecl()->getIdentifier();
-
-      // This callback frees the associated buffer.
-      if (Name->isStr("CGBitmapContextCreateWithData"))
-        RE = S->getRetEffect();
+      if (IdentifierInfo *Name = FC->getDecl()->getIdentifier()) {
+        // When the CGBitmapContext is deallocated, the callback here will free
+        // the associated data buffer.
+        if (Name->isStr("CGBitmapContextCreateWithData"))
+          RE = S->getRetEffect();
+      }
     }
 
     S = getPersistentSummary(RE, RecEffect, DefEffect);
   }
+
+  // Special case '[super init];' and '[self init];'
+  //
+  // Even though calling '[super init]' without assigning the result to self
+  // and checking if the parent returns 'nil' is a bad pattern, it is common.
+  // Additionally, our Self Init checker already warns about it. To avoid
+  // overwhelming the user with messages from both checkers, we model the case
+  // of '[super init]' in cases when it is not consumed by another expression
+  // as if the call preserves the value of 'self'; essentially, assuming it can 
+  // never fail and return 'nil'.
+  // Note, we don't want to just stop tracking the value since we want the
+  // RetainCount checker to report leaks and use-after-free if SelfInit checker
+  // is turned off.
+  if (const ObjCMethodCall *MC = dyn_cast<ObjCMethodCall>(&Call)) {
+    if (MC->getMethodFamily() == OMF_init && MC->isReceiverSelfOrSuper()) {
+
+      // Check if the message is not consumed, we know it will not be used in
+      // an assignment, ex: "self = [super init]".
+      const Expr *ME = MC->getOriginExpr();
+      const LocationContext *LCtx = MC->getLocationContext();
+      ParentMap &PM = LCtx->getAnalysisDeclContext()->getParentMap();
+      if (!PM.isConsumedExpr(ME)) {
+        RetainSummaryTemplate ModifiableSummaryTemplate(S, *this);
+        ModifiableSummaryTemplate->setReceiverEffect(DoNothing);
+        ModifiableSummaryTemplate->setRetEffect(RetEffect::MakeNoRet());
+      }
+    }
+
+  }
 }
 
 const RetainSummary *
@@ -1036,6 +1061,8 @@ RetainSummaryManager::getFunctionSummary(const FunctionDecl *FD) {
       // The headers on OS X 10.8 use cf_consumed/ns_returns_retained,
       // but we can fully model NSMakeCollectable ourselves.
       AllowAnnotations = false;
+    } else if (FName == "CFPlugInInstanceCreate") {
+      S = getPersistentSummary(RetEffect::MakeNoRet());
     } else if (FName == "IOBSDNameMatching" ||
                FName == "IOServiceMatching" ||
                FName == "IOServiceNameMatching" ||
@@ -1108,6 +1135,11 @@ RetainSummaryManager::getFunctionSummary(const FunctionDecl *FD) {
       break;
 
     if (RetTy->isPointerType()) {
+      if (FD->getAttr<CFAuditedTransferAttr>()) {
+        S = getCFCreateGetRuleSummary(FD);
+        break;
+      }
+      
       // For CoreFoundation ('CF') types.
       if (cocoa::isRefType(RetTy, "CF", FName)) {
         if (isRetain(FD, FName))
@@ -1347,22 +1379,6 @@ RetainSummaryManager::updateSummaryFromAnnotations(const RetainSummary *&Summ,
 const RetainSummary *
 RetainSummaryManager::getStandardMethodSummary(const ObjCMethodDecl *MD,
                                                Selector S, QualType RetTy) {
-
-  if (MD) {
-    // Scan the method decl for 'void*' arguments.  These should be treated
-    // as 'StopTracking' because they are often used with delegates.
-    // Delegates are a frequent form of false positives with the retain
-    // count checker.
-    unsigned i = 0;
-    for (ObjCMethodDecl::param_const_iterator I = MD->param_begin(),
-         E = MD->param_end(); I != E; ++I, ++i)
-      if (const ParmVarDecl *PD = *I) {
-        QualType Ty = Ctx.getCanonicalType(PD->getType());
-        if (Ty.getLocalUnqualifiedType() == Ctx.VoidPtrTy)
-          ScratchArgs = AF.add(ScratchArgs, i, StopTracking);
-      }
-  }
-
   // Any special effects?
   ArgEffect ReceiverEff = DoNothing;
   RetEffect ResultEff = RetEffect::MakeNoRet();
@@ -1441,9 +1457,9 @@ RetainSummaryManager::getStandardMethodSummary(const ObjCMethodDecl *MD,
       StringRef Slot = S.getNameForSlot(i);
       if (Slot.substr(Slot.size() - 8).equals_lower("delegate")) {
         if (ResultEff == ObjCInitRetE)
-          ResultEff = RetEffect::MakeNoRet();
+          ResultEff = RetEffect::MakeNoRetHard();
         else
-          ReceiverEff = StopTracking;
+          ReceiverEff = StopTrackingHard;
       }
     }
   }
@@ -2174,6 +2190,7 @@ GetAllocationSite(ProgramStateManager& StateMgr, const ExplodedNode *N,
 
   // If allocation happened in a function different from the leak node context,
   // do not report the binding.
+  assert(N && "Could not find allocation node");
   if (N->getLocationContext() != LeakContext) {
     FirstBinding = 0;
   }
@@ -2229,27 +2246,36 @@ CFRefLeakReportVisitor::getEndPath(BugReporterContext &BRC,
 
   // Get the retain count.
   const RefVal* RV = getRefBinding(EndN->getState(), Sym);
+  assert(RV);
 
   if (RV->getKind() == RefVal::ErrorLeakReturned) {
     // FIXME: Per comments in rdar://6320065, "create" only applies to CF
     // objects.  Only "copy", "alloc", "retain" and "new" transfer ownership
     // to the caller for NS objects.
     const Decl *D = &EndN->getCodeDecl();
-    if (const ObjCMethodDecl *MD = dyn_cast<ObjCMethodDecl>(D)) {
-      os << " is returned from a method whose name ('"
-         << MD->getSelector().getAsString()
-         << "') does not start with 'copy', 'mutableCopy', 'alloc' or 'new'."
-            "  This violates the naming convention rules"
-            " given in the Memory Management Guide for Cocoa";
-    }
+    
+    os << (isa<ObjCMethodDecl>(D) ? " is returned from a method "
+                                  : " is returned from a function ");
+    
+    if (D->getAttr<CFReturnsNotRetainedAttr>())
+      os << "that is annotated as CF_RETURNS_NOT_RETAINED";
+    else if (D->getAttr<NSReturnsNotRetainedAttr>())
+      os << "that is annotated as NS_RETURNS_NOT_RETAINED";
     else {
-      const FunctionDecl *FD = cast<FunctionDecl>(D);
-      os << " is returned from a function whose name ('"
-         << *FD
-         << "') does not contain 'Copy' or 'Create'.  This violates the naming"
-            " convention rules given in the Memory Management Guide for Core"
-            " Foundation";
-    }    
+      if (const ObjCMethodDecl *MD = dyn_cast<ObjCMethodDecl>(D)) {
+        os << "whose name ('" << MD->getSelector().getAsString()
+           << "') does not start with 'copy', 'mutableCopy', 'alloc' or 'new'."
+              "  This violates the naming convention rules"
+              " given in the Memory Management Guide for Cocoa";
+      }
+      else {
+        const FunctionDecl *FD = cast<FunctionDecl>(D);
+        os << "whose name ('" << *FD
+           << "') does not contain 'Copy' or 'Create'.  This violates the naming"
+              " convention rules given in the Memory Management Guide for Core"
+              " Foundation";
+      }
+    }
   }
   else if (RV->getKind() == RefVal::ErrorGCLeakReturned) {
     ObjCMethodDecl &MD = cast<ObjCMethodDecl>(EndN->getCodeDecl());
@@ -2474,6 +2500,10 @@ public:
   void checkSummary(const RetainSummary &Summ, const CallEvent &Call,
                     CheckerContext &C) const;
 
+  void processSummaryOfInlined(const RetainSummary &Summ,
+                               const CallEvent &Call,
+                               CheckerContext &C) const;
+
   bool evalCall(const CallExpr *CE, CheckerContext &C) const;
 
   ProgramStateRef evalAssume(ProgramStateRef state, SVal Cond,
@@ -2499,8 +2529,8 @@ public:
   void checkEndPath(CheckerContext &C) const;
 
   ProgramStateRef updateSymbol(ProgramStateRef state, SymbolRef sym,
-                                   RefVal V, ArgEffect E, RefVal::Kind &hasErr,
-                                   CheckerContext &C) const;
+                               RefVal V, ArgEffect E, RefVal::Kind &hasErr,
+                               CheckerContext &C) const;
 
   void processNonLeakError(ProgramStateRef St, SourceRange ErrorRange,
                            RefVal::Kind ErrorKind, SymbolRef Sym,
@@ -2515,13 +2545,12 @@ public:
                                     SmallVectorImpl<SymbolRef> &Leaked) const;
 
   std::pair<ExplodedNode *, ProgramStateRef >
-  handleAutoreleaseCounts(ProgramStateRef state, 
-                          GenericNodeBuilderRefCount Bd, ExplodedNode *Pred,
-                          CheckerContext &Ctx, SymbolRef Sym, RefVal V) const;
+  handleAutoreleaseCounts(ProgramStateRef state, ExplodedNode *Pred,
+                          const ProgramPointTag *Tag, CheckerContext &Ctx,
+                          SymbolRef Sym, RefVal V) const;
 
   ExplodedNode *processLeaks(ProgramStateRef state,
                              SmallVectorImpl<SymbolRef> &Leaked,
-                             GenericNodeBuilderRefCount &Builder,
                              CheckerContext &Ctx,
                              ExplodedNode *Pred = 0) const;
 };
@@ -2685,11 +2714,13 @@ void RetainCountChecker::checkPostStmt(const ObjCBoxedExpr *Ex,
 
 void RetainCountChecker::checkPostCall(const CallEvent &Call,
                                        CheckerContext &C) const {
-  if (C.wasInlined)
-    return;
-
   RetainSummaryManager &Summaries = getSummaryManager(C);
   const RetainSummary *Summ = Summaries.getSummary(Call, C.getState());
+
+  if (C.wasInlined) {
+    processSummaryOfInlined(*Summ, Call, C);
+    return;
+  }
   checkSummary(*Summ, Call, C);
 }
 
@@ -2721,6 +2752,45 @@ static QualType GetReturnType(const Expr *RetE, ASTContext &Ctx) {
   return RetTy;
 }
 
+// We don't always get the exact modeling of the function with regards to the
+// retain count checker even when the function is inlined. For example, we need
+// to stop tracking the symbols which were marked with StopTrackingHard.
+void RetainCountChecker::processSummaryOfInlined(const RetainSummary &Summ,
+                                                 const CallEvent &CallOrMsg,
+                                                 CheckerContext &C) const {
+  ProgramStateRef state = C.getState();
+
+  // Evaluate the effect of the arguments.
+  for (unsigned idx = 0, e = CallOrMsg.getNumArgs(); idx != e; ++idx) {
+    if (Summ.getArg(idx) == StopTrackingHard) {
+      SVal V = CallOrMsg.getArgSVal(idx);
+      if (SymbolRef Sym = V.getAsLocSymbol()) {
+        state = removeRefBinding(state, Sym);
+      }
+    }
+  }
+
+  // Evaluate the effect on the message receiver.
+  const ObjCMethodCall *MsgInvocation = dyn_cast<ObjCMethodCall>(&CallOrMsg);
+  if (MsgInvocation) {
+    if (SymbolRef Sym = MsgInvocation->getReceiverSVal().getAsLocSymbol()) {
+      if (Summ.getReceiverEffect() == StopTrackingHard) {
+        state = removeRefBinding(state, Sym);
+      }
+    }
+  }
+
+  // Consult the summary for the return value.
+  RetEffect RE = Summ.getRetEffect();
+  if (RE.getKind() == RetEffect::NoRetHard) {
+    SymbolRef Sym = CallOrMsg.getReturnValue().getAsSymbol();
+    if (Sym)
+      state = removeRefBinding(state, Sym);
+  }
+  
+  C.addTransition(state);
+}
+
 void RetainCountChecker::checkSummary(const RetainSummary &Summ,
                                       const CallEvent &CallOrMsg,
                                       CheckerContext &C) const {
@@ -2755,7 +2825,7 @@ void RetainCountChecker::checkSummary(const RetainSummary &Summ,
         if (const RefVal *T = getRefBinding(state, Sym)) {
           ReceiverIsTracked = true;
           state = updateSymbol(state, Sym, *T, Summ.getReceiverEffect(),
-                               hasErr, C);
+                                 hasErr, C);
           if (hasErr) {
             ErrorRange = MsgInvocation->getOriginExpr()->getReceiverRange();
             ErrorSym = Sym;
@@ -2786,13 +2856,13 @@ void RetainCountChecker::checkSummary(const RetainSummary &Summ,
       llvm_unreachable("Unhandled RetEffect.");
 
     case RetEffect::NoRet:
+    case RetEffect::NoRetHard:
       // No work necessary.
       break;
 
     case RetEffect::OwnedAllocatedSymbol:
     case RetEffect::OwnedSymbol: {
-      SymbolRef Sym = state->getSVal(CallOrMsg.getOriginExpr(),
-                                     C.getLocationContext()).getAsSymbol();
+      SymbolRef Sym = CallOrMsg.getReturnValue().getAsSymbol();
       if (!Sym)
         break;
 
@@ -2811,10 +2881,10 @@ void RetainCountChecker::checkSummary(const RetainSummary &Summ,
     case RetEffect::ARCNotOwnedSymbol:
     case RetEffect::NotOwnedSymbol: {
       const Expr *Ex = CallOrMsg.getOriginExpr();
-      SymbolRef Sym = state->getSVal(Ex, C.getLocationContext()).getAsSymbol();
+      SymbolRef Sym = CallOrMsg.getReturnValue().getAsSymbol();
       if (!Sym)
         break;
-
+      assert(Ex);
       // Use GetReturnType in order to give [NSFoo alloc] the type NSFoo *.
       QualType ResultTy = GetReturnType(Ex, C.getASTContext());
       state = setRefBinding(state, Sym, RefVal::makeNotOwned(RE.getObjKind(),
@@ -2864,8 +2934,8 @@ RetainCountChecker::updateSymbol(ProgramStateRef state, SymbolRef sym,
   case DecRefMsg:
     E = IgnoreRetainMsg ? DoNothing : DecRef;
     break;
-  case DecRefMsgAndStopTracking:
-    E = IgnoreRetainMsg ? StopTracking : DecRefAndStopTracking;
+  case DecRefMsgAndStopTrackingHard:
+    E = IgnoreRetainMsg ? StopTracking : DecRefAndStopTrackingHard;
     break;
   case MakeCollectable:
     E = C.isObjCGCEnabled() ? DecRef : DoNothing;
@@ -2886,7 +2956,7 @@ RetainCountChecker::updateSymbol(ProgramStateRef state, SymbolRef sym,
     case DecRefMsg:
     case IncRefMsg:
     case MakeCollectable:
-    case DecRefMsgAndStopTracking:
+    case DecRefMsgAndStopTrackingHard:
       llvm_unreachable("DecRefMsg/IncRefMsg/MakeCollectable already converted");
 
     case Dealloc:
@@ -2935,6 +3005,7 @@ RetainCountChecker::updateSymbol(ProgramStateRef state, SymbolRef sym,
       break;
 
     case StopTracking:
+    case StopTrackingHard:
       return removeRefBinding(state, sym);
 
     case IncRef:
@@ -2955,7 +3026,7 @@ RetainCountChecker::updateSymbol(ProgramStateRef state, SymbolRef sym,
 
     case DecRef:
     case DecRefBridgedTransfered:
-    case DecRefAndStopTracking:
+    case DecRefAndStopTrackingHard:
       switch (V.getKind()) {
         default:
           // case 'RefVal::Released' handled above.
@@ -2966,7 +3037,7 @@ RetainCountChecker::updateSymbol(ProgramStateRef state, SymbolRef sym,
           if (V.getCount() == 1)
             V = V ^ (E == DecRefBridgedTransfered ? 
                       RefVal::NotOwned : RefVal::Released);
-          else if (E == DecRefAndStopTracking)
+          else if (E == DecRefAndStopTrackingHard)
             return removeRefBinding(state, sym);
 
           V = V - 1;
@@ -2974,7 +3045,7 @@ RetainCountChecker::updateSymbol(ProgramStateRef state, SymbolRef sym,
 
         case RefVal::NotOwned:
           if (V.getCount() > 0) {
-            if (E == DecRefAndStopTracking)
+            if (E == DecRefAndStopTrackingHard)
               return removeRefBinding(state, sym);
             V = V - 1;
           } else {
@@ -3035,7 +3106,7 @@ void RetainCountChecker::processNonLeakError(ProgramStateRef St,
                                         C.isObjCGCEnabled(), SummaryLog,
                                         N, Sym);
   report->addRange(ErrorRange);
-  C.EmitReport(report);
+  C.emitReport(report);
 }
 
 //===----------------------------------------------------------------------===//
@@ -3090,8 +3161,7 @@ bool RetainCountChecker::evalCall(const CallExpr *CE, CheckerContext &C) const {
   if (RetVal.isUnknown()) {
     // If the receiver is unknown, conjure a return value.
     SValBuilder &SVB = C.getSValBuilder();
-    unsigned Count = C.getCurrentBlockCount();
-    RetVal = SVB.getConjuredSymbolVal(0, CE, LCtx, ResultTy, Count);
+    RetVal = SVB.conjureSymbolVal(0, CE, LCtx, ResultTy, C.blockCount());
   }
   state = state->BindExpr(CE, LCtx, RetVal, false);
 
@@ -3105,8 +3175,7 @@ bool RetainCountChecker::evalCall(const CallExpr *CE, CheckerContext &C) const {
       Binding = getRefBinding(state, Sym);
 
     // Invalidate the argument region.
-    unsigned Count = C.getCurrentBlockCount();
-    state = state->invalidateRegions(ArgRegion, CE, Count, LCtx);
+    state = state->invalidateRegions(ArgRegion, CE, C.blockCount(), LCtx);
 
     // Restore the refcount status of the argument.
     if (Binding)
@@ -3121,12 +3190,6 @@ bool RetainCountChecker::evalCall(const CallExpr *CE, CheckerContext &C) const {
 // Handle return statements.
 //===----------------------------------------------------------------------===//
 
-// Return true if the current LocationContext has no caller context.
-static bool inTopFrame(CheckerContext &C) {
-  const LocationContext *LC = C.getLocationContext();
-  return LC->getParent() == 0;  
-}
-
 void RetainCountChecker::checkPreStmt(const ReturnStmt *S,
                                       CheckerContext &C) const {
 
@@ -3135,7 +3198,7 @@ void RetainCountChecker::checkPreStmt(const ReturnStmt *S,
   // better checking even for inlined calls, and see if they match
   // with their expected semantics (e.g., the method should return a retained
   // object, etc.).
-  if (!inTopFrame(C))
+  if (!C.inTopFrame())
     return;
 
   const Expr *RetE = S->getRetValue();
@@ -3196,8 +3259,8 @@ void RetainCountChecker::checkPreStmt(const ReturnStmt *S,
   // Update the autorelease counts.
   static SimpleProgramPointTag
          AutoreleaseTag("RetainCountChecker : Autorelease");
-  GenericNodeBuilderRefCount Bd(C, &AutoreleaseTag);
-  llvm::tie(Pred, state) = handleAutoreleaseCounts(state, Bd, Pred, C, Sym, X);
+  llvm::tie(Pred, state) = handleAutoreleaseCounts(state, Pred, &AutoreleaseTag,
+                                                   C, Sym, X);
 
   // Did we cache out?
   if (!Pred)
@@ -3267,7 +3330,7 @@ void RetainCountChecker::checkReturnWithRetEffect(const ReturnStmt *S,
             new CFRefLeakReport(*getLeakAtReturnBug(LOpts, GCEnabled),
                                 LOpts, GCEnabled, SummaryLog,
                                 N, Sym, C);
-          C.EmitReport(report);
+          C.emitReport(report);
         }
       }
     }
@@ -3288,7 +3351,7 @@ void RetainCountChecker::checkReturnWithRetEffect(const ReturnStmt *S,
             new CFRefReport(*returnNotOwnedForOwned,
                             C.getASTContext().getLangOpts(), 
                             C.isObjCGCEnabled(), SummaryLog, N, Sym);
-        C.EmitReport(report);
+        C.emitReport(report);
       }
     }
   }
@@ -3354,18 +3417,19 @@ ProgramStateRef RetainCountChecker::evalAssume(ProgramStateRef state,
   //  too bad since the number of symbols we will track in practice are
   //  probably small and evalAssume is only called at branches and a few
   //  other places.
-  RefBindings B = state->get<RefBindings>();
+  RefBindingsTy B = state->get<RefBindings>();
 
   if (B.isEmpty())
     return state;
 
   bool changed = false;
-  RefBindings::Factory &RefBFactory = state->get_context<RefBindings>();
+  RefBindingsTy::Factory &RefBFactory = state->get_context<RefBindings>();
 
-  for (RefBindings::iterator I = B.begin(), E = B.end(); I != E; ++I) {
-    // Check if the symbol is null (or equal to any constant).
-    // If this is the case, stop tracking the symbol.
-    if (state->getSymVal(I.getKey())) {
+  for (RefBindingsTy::iterator I = B.begin(), E = B.end(); I != E; ++I) {
+    // Check if the symbol is null stop tracking the symbol.
+    ConstraintManager &CMgr = state->getConstraintManager();
+    ConditionTruthVal AllocFailed = CMgr.isNull(state, I.getKey());
+    if (AllocFailed.isConstrainedTrue()) {
       changed = true;
       B = RefBFactory.remove(B, I.getKey());
     }
@@ -3410,8 +3474,8 @@ RetainCountChecker::checkRegionChanges(ProgramStateRef state,
 
 std::pair<ExplodedNode *, ProgramStateRef >
 RetainCountChecker::handleAutoreleaseCounts(ProgramStateRef state, 
-                                            GenericNodeBuilderRefCount Bd,
                                             ExplodedNode *Pred,
+                                            const ProgramPointTag *Tag,
                                             CheckerContext &Ctx,
                                             SymbolRef Sym, RefVal V) const {
   unsigned ACnt = V.getAutoreleaseCount();
@@ -3440,7 +3504,7 @@ RetainCountChecker::handleAutoreleaseCounts(ProgramStateRef state,
       V.setAutoreleaseCount(0);
     }
     state = setRefBinding(state, Sym, V);
-    ExplodedNode *N = Bd.MakeNode(state, Pred);
+    ExplodedNode *N = Ctx.addTransition(state, Pred, Tag);
     if (N == 0)
       state = 0;
     return std::make_pair(N, state);
@@ -3451,7 +3515,8 @@ RetainCountChecker::handleAutoreleaseCounts(ProgramStateRef state,
   V = V ^ RefVal::ErrorOverAutorelease;
   state = setRefBinding(state, Sym, V);
 
-  if (ExplodedNode *N = Bd.MakeNode(state, Pred, true)) {
+  ExplodedNode *N = Ctx.generateSink(state, Pred, Tag);
+  if (N) {
     SmallString<128> sbuf;
     llvm::raw_svector_ostream os(sbuf);
     os << "Object over-autoreleased: object was sent -autorelease ";
@@ -3466,7 +3531,7 @@ RetainCountChecker::handleAutoreleaseCounts(ProgramStateRef state,
     CFRefReport *report =
       new CFRefReport(*overAutorelease, LOpts, /* GCEnabled = */ false,
                       SummaryLog, N, Sym, os.str());
-    Ctx.EmitReport(report);
+    Ctx.emitReport(report);
   }
 
   return std::make_pair((ExplodedNode *)0, (ProgramStateRef )0);
@@ -3492,14 +3557,13 @@ RetainCountChecker::handleSymbolDeath(ProgramStateRef state,
 ExplodedNode *
 RetainCountChecker::processLeaks(ProgramStateRef state,
                                  SmallVectorImpl<SymbolRef> &Leaked,
-                                 GenericNodeBuilderRefCount &Builder,
                                  CheckerContext &Ctx,
                                  ExplodedNode *Pred) const {
   if (Leaked.empty())
     return Pred;
 
   // Generate an intermediate node representing the leak point.
-  ExplodedNode *N = Builder.MakeNode(state, Pred);
+  ExplodedNode *N = Ctx.addTransition(state, Pred);
 
   if (N) {
     for (SmallVectorImpl<SymbolRef>::iterator
@@ -3513,7 +3577,7 @@ RetainCountChecker::processLeaks(ProgramStateRef state,
 
       CFRefLeakReport *report = new CFRefLeakReport(*BT, LOpts, GCEnabled, 
                                                     SummaryLog, N, *I, Ctx);
-      Ctx.EmitReport(report);
+      Ctx.emitReport(report);
     }
   }
 
@@ -3522,13 +3586,12 @@ RetainCountChecker::processLeaks(ProgramStateRef state,
 
 void RetainCountChecker::checkEndPath(CheckerContext &Ctx) const {
   ProgramStateRef state = Ctx.getState();
-  GenericNodeBuilderRefCount Bd(Ctx);
-  RefBindings B = state->get<RefBindings>();
+  RefBindingsTy B = state->get<RefBindings>();
   ExplodedNode *Pred = Ctx.getPredecessor();
 
-  for (RefBindings::iterator I = B.begin(), E = B.end(); I != E; ++I) {
-    llvm::tie(Pred, state) = handleAutoreleaseCounts(state, Bd, Pred, Ctx,
-                                                     I->first, I->second);
+  for (RefBindingsTy::iterator I = B.begin(), E = B.end(); I != E; ++I) {
+    llvm::tie(Pred, state) = handleAutoreleaseCounts(state, Pred, /*Tag=*/0,
+                                                     Ctx, I->first, I->second);
     if (!state)
       return;
   }
@@ -3543,10 +3606,10 @@ void RetainCountChecker::checkEndPath(CheckerContext &Ctx) const {
   B = state->get<RefBindings>();
   SmallVector<SymbolRef, 10> Leaked;
 
-  for (RefBindings::iterator I = B.begin(), E = B.end(); I != E; ++I)
+  for (RefBindingsTy::iterator I = B.begin(), E = B.end(); I != E; ++I)
     state = handleSymbolDeath(state, I->first, I->second, Leaked);
 
-  processLeaks(state, Leaked, Bd, Ctx, Pred);
+  processLeaks(state, Leaked, Ctx, Pred);
 }
 
 const ProgramPointTag *
@@ -3567,7 +3630,7 @@ void RetainCountChecker::checkDeadSymbols(SymbolReaper &SymReaper,
   ExplodedNode *Pred = C.getPredecessor();
 
   ProgramStateRef state = C.getState();
-  RefBindings B = state->get<RefBindings>();
+  RefBindingsTy B = state->get<RefBindings>();
 
   // Update counts from autorelease pools
   for (SymbolReaper::dead_iterator I = SymReaper.dead_begin(),
@@ -3576,8 +3639,8 @@ void RetainCountChecker::checkDeadSymbols(SymbolReaper &SymReaper,
     if (const RefVal *T = B.lookup(Sym)){
       // Use the symbol as the tag.
       // FIXME: This might not be as unique as we would like.
-      GenericNodeBuilderRefCount Bd(C, getDeadSymbolTag(Sym));
-      llvm::tie(Pred, state) = handleAutoreleaseCounts(state, Bd, Pred, C,
+      const ProgramPointTag *Tag = getDeadSymbolTag(Sym);
+      llvm::tie(Pred, state) = handleAutoreleaseCounts(state, Pred, Tag, C,
                                                        Sym, *T);
       if (!state)
         return;
@@ -3593,17 +3656,14 @@ void RetainCountChecker::checkDeadSymbols(SymbolReaper &SymReaper,
       state = handleSymbolDeath(state, *I, *T, Leaked);
   }
 
-  {
-    GenericNodeBuilderRefCount Bd(C, this);
-    Pred = processLeaks(state, Leaked, Bd, C, Pred);
-  }
+  Pred = processLeaks(state, Leaked, C, Pred);
 
   // Did we cache out?
   if (!Pred)
     return;
 
   // Now generate a new node that nukes the old bindings.
-  RefBindings::Factory &F = state->get_context<RefBindings>();
+  RefBindingsTy::Factory &F = state->get_context<RefBindings>();
 
   for (SymbolReaper::dead_iterator I = SymReaper.dead_begin(),
        E = SymReaper.dead_end(); I != E; ++I)
@@ -3616,12 +3676,12 @@ void RetainCountChecker::checkDeadSymbols(SymbolReaper &SymReaper,
 void RetainCountChecker::printState(raw_ostream &Out, ProgramStateRef State,
                                     const char *NL, const char *Sep) const {
 
-  RefBindings B = State->get<RefBindings>();
+  RefBindingsTy B = State->get<RefBindings>();
 
   if (!B.isEmpty())
     Out << Sep << NL;
 
-  for (RefBindings::iterator I = B.begin(), E = B.end(); I != E; ++I) {
+  for (RefBindingsTy::iterator I = B.begin(), E = B.end(); I != E; ++I) {
     Out << I->first << " : ";
     I->second.print(Out);
     Out << NL;
diff --git a/lib/StaticAnalyzer/Checkers/ReturnPointerRangeChecker.cpp b/lib/StaticAnalyzer/Checkers/ReturnPointerRangeChecker.cpp
index 6e56593..f3560aa 100644
--- a/lib/StaticAnalyzer/Checkers/ReturnPointerRangeChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/ReturnPointerRangeChecker.cpp
@@ -82,7 +82,7 @@ void ReturnPointerRangeChecker::checkPreStmt(const ReturnStmt *RS,
       new BugReport(*BT, BT->getDescription(), N);
 
     report->addRange(RetE->getSourceRange());
-    C.EmitReport(report);
+    C.emitReport(report);
   }
 }
 
diff --git a/lib/StaticAnalyzer/Checkers/ReturnUndefChecker.cpp b/lib/StaticAnalyzer/Checkers/ReturnUndefChecker.cpp
index ca2a55d..37ec1aa 100644
--- a/lib/StaticAnalyzer/Checkers/ReturnUndefChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/ReturnUndefChecker.cpp
@@ -16,6 +16,7 @@
 #include "ClangSACheckers.h"
 #include "clang/StaticAnalyzer/Core/Checker.h"
 #include "clang/StaticAnalyzer/Core/CheckerManager.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
 
@@ -41,6 +42,19 @@ void ReturnUndefChecker::checkPreStmt(const ReturnStmt *RS,
   if (!C.getState()->getSVal(RetE, C.getLocationContext()).isUndef())
     return;
   
+  // "return;" is modeled to evaluate to an UndefinedValue. Allow UndefinedValue
+  // to be returned in functions returning void to support the following pattern:
+  // void foo() {
+  //  return;
+  // }
+  // void test() {
+  //   return foo();
+  // }
+  const StackFrameContext *SFC = C.getStackFrame();
+  QualType RT = CallEvent::getDeclaredResultType(SFC->getDecl());
+  if (!RT.isNull() && RT->isSpecificBuiltinType(BuiltinType::Void))
+    return;
+
   ExplodedNode *N = C.generateSink();
 
   if (!N)
@@ -53,11 +67,10 @@ void ReturnUndefChecker::checkPreStmt(const ReturnStmt *RS,
   BugReport *report = 
     new BugReport(*BT, BT->getDescription(), N);
 
-  report->disablePathPruning();
   report->addRange(RetE->getSourceRange());
-  bugreporter::addTrackNullOrUndefValueVisitor(N, RetE, report);
+  bugreporter::trackNullOrUndefValue(N, RetE, *report);
 
-  C.EmitReport(report);
+  C.emitReport(report);
 }
 
 void ento::registerReturnUndefChecker(CheckerManager &mgr) {
diff --git a/lib/StaticAnalyzer/Checkers/SimpleStreamChecker.cpp b/lib/StaticAnalyzer/Checkers/SimpleStreamChecker.cpp
new file mode 100644
index 0000000..ee055ad
--- /dev/null
+++ b/lib/StaticAnalyzer/Checkers/SimpleStreamChecker.cpp
@@ -0,0 +1,348 @@
+//===-- SimpleStreamChecker.cpp -----------------------------------------*- C++ -*--//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// Defines a checker for proper use of fopen/fclose APIs.
+//   - If a file has been closed with fclose, it should not be accessed again.
+//   Accessing a closed file results in undefined behavior.
+//   - If a file was opened with fopen, it must be closed with fclose before
+//   the execution ends. Failing to do so results in a resource leak.
+//
+//===----------------------------------------------------------------------===//
+
+#include "ClangSACheckers.h"
+#include "clang/StaticAnalyzer/Core/Checker.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
+
+using namespace clang;
+using namespace ento;
+
+namespace {
+typedef llvm::SmallVector<SymbolRef, 2> SymbolVector;
+
+struct StreamState {
+private:
+  enum Kind { Opened, Closed } K;
+  StreamState(Kind InK) : K(InK) { }
+
+public:
+  bool isOpened() const { return K == Opened; }
+  bool isClosed() const { return K == Closed; }
+
+  static StreamState getOpened() { return StreamState(Opened); }
+  static StreamState getClosed() { return StreamState(Closed); }
+
+  bool operator==(const StreamState &X) const {
+    return K == X.K;
+  }
+  void Profile(llvm::FoldingSetNodeID &ID) const {
+    ID.AddInteger(K);
+  }
+};
+
+class SimpleStreamChecker : public Checker<check::PostCall,
+                                           check::PreCall,
+                                           check::DeadSymbols,
+                                           check::Bind,
+                                           check::RegionChanges> {
+
+  mutable IdentifierInfo *IIfopen, *IIfclose;
+
+  OwningPtr<BugType> DoubleCloseBugType;
+  OwningPtr<BugType> LeakBugType;
+
+  void initIdentifierInfo(ASTContext &Ctx) const;
+
+  void reportDoubleClose(SymbolRef FileDescSym,
+                         const CallEvent &Call,
+                         CheckerContext &C) const;
+
+  void reportLeaks(SymbolVector LeakedStreams,
+                   CheckerContext &C,
+                   ExplodedNode *ErrNode) const;
+
+  bool guaranteedNotToCloseFile(const CallEvent &Call) const;
+
+public:
+  SimpleStreamChecker();
+
+  /// Process fopen.
+  void checkPostCall(const CallEvent &Call, CheckerContext &C) const;
+  /// Process fclose.
+  void checkPreCall(const CallEvent &Call, CheckerContext &C) const;
+
+  void checkDeadSymbols(SymbolReaper &SymReaper, CheckerContext &C) const;
+
+  /// Deal with symbol escape as a byproduct of a bind.
+  void checkBind(SVal location, SVal val, const Stmt*S,
+                 CheckerContext &C) const;
+
+  /// Deal with symbol escape as a byproduct of a region change.
+  ProgramStateRef
+  checkRegionChanges(ProgramStateRef state,
+                     const StoreManager::InvalidatedSymbols *invalidated,
+                     ArrayRef<const MemRegion *> ExplicitRegions,
+                     ArrayRef<const MemRegion *> Regions,
+                     const CallEvent *Call) const;
+  bool wantsRegionChangeUpdate(ProgramStateRef state) const {
+    return true;
+  }
+};
+
+} // end anonymous namespace
+
+/// The state of the checker is a map from tracked stream symbols to their
+/// state. Let's store it in the ProgramState.
+REGISTER_MAP_WITH_PROGRAMSTATE(StreamMap, SymbolRef, StreamState)
+
+namespace {
+class StopTrackingCallback : public SymbolVisitor {
+  ProgramStateRef state;
+public:
+  StopTrackingCallback(ProgramStateRef st) : state(st) {}
+  ProgramStateRef getState() const { return state; }
+
+  bool VisitSymbol(SymbolRef sym) {
+    state = state->remove<StreamMap>(sym);
+    return true;
+  }
+};
+} // end anonymous namespace
+
+
+SimpleStreamChecker::SimpleStreamChecker() : IIfopen(0), IIfclose(0) {
+  // Initialize the bug types.
+  DoubleCloseBugType.reset(new BugType("Double fclose",
+                                       "Unix Stream API Error"));
+
+  LeakBugType.reset(new BugType("Resource Leak",
+                                "Unix Stream API Error"));
+  // Sinks are higher importance bugs as well as calls to assert() or exit(0).
+  LeakBugType->setSuppressOnSink(true);
+}
+
+void SimpleStreamChecker::checkPostCall(const CallEvent &Call,
+                                        CheckerContext &C) const {
+  initIdentifierInfo(C.getASTContext());
+
+  if (!Call.isGlobalCFunction())
+    return;
+
+  if (Call.getCalleeIdentifier() != IIfopen)
+    return;
+
+  // Get the symbolic value corresponding to the file handle.
+  SymbolRef FileDesc = Call.getReturnValue().getAsSymbol();
+  if (!FileDesc)
+    return;
+
+  // Generate the next transition (an edge in the exploded graph).
+  ProgramStateRef State = C.getState();
+  State = State->set<StreamMap>(FileDesc, StreamState::getOpened());
+  C.addTransition(State);
+}
+
+void SimpleStreamChecker::checkPreCall(const CallEvent &Call,
+                                       CheckerContext &C) const {
+  initIdentifierInfo(C.getASTContext());
+
+  if (!Call.isGlobalCFunction())
+    return;
+
+  if (Call.getCalleeIdentifier() != IIfclose)
+    return;
+
+  if (Call.getNumArgs() != 1)
+    return;
+
+  // Get the symbolic value corresponding to the file handle.
+  SymbolRef FileDesc = Call.getArgSVal(0).getAsSymbol();
+  if (!FileDesc)
+    return;
+
+  // Check if the stream has already been closed.
+  ProgramStateRef State = C.getState();
+  const StreamState *SS = State->get<StreamMap>(FileDesc);
+  if (SS && SS->isClosed()) {
+    reportDoubleClose(FileDesc, Call, C);
+    return;
+  }
+
+  // Generate the next transition, in which the stream is closed.
+  State = State->set<StreamMap>(FileDesc, StreamState::getClosed());
+  C.addTransition(State);
+}
+
+static bool isLeaked(SymbolRef Sym, const StreamState &SS,
+                     bool IsSymDead, ProgramStateRef State) {
+  if (IsSymDead && SS.isOpened()) {
+    // If a symbol is NULL, assume that fopen failed on this path.
+    // A symbol should only be considered leaked if it is non-null.
+    ConstraintManager &CMgr = State->getConstraintManager();
+    ConditionTruthVal OpenFailed = CMgr.isNull(State, Sym);
+    return !OpenFailed.isConstrainedTrue();
+  }
+  return false;
+}
+
+void SimpleStreamChecker::checkDeadSymbols(SymbolReaper &SymReaper,
+                                           CheckerContext &C) const {
+  ProgramStateRef State = C.getState();
+  SymbolVector LeakedStreams;
+  StreamMapTy TrackedStreams = State->get<StreamMap>();
+  for (StreamMapTy::iterator I = TrackedStreams.begin(),
+                             E = TrackedStreams.end(); I != E; ++I) {
+    SymbolRef Sym = I->first;
+    bool IsSymDead = SymReaper.isDead(Sym);
+
+    // Collect leaked symbols.
+    if (isLeaked(Sym, I->second, IsSymDead, State))
+      LeakedStreams.push_back(Sym);
+
+    // Remove the dead symbol from the streams map.
+    if (IsSymDead)
+      State = State->remove<StreamMap>(Sym);
+  }
+
+  ExplodedNode *N = C.addTransition(State);
+  reportLeaks(LeakedStreams, C, N);
+}
+
+void SimpleStreamChecker::reportDoubleClose(SymbolRef FileDescSym,
+                                            const CallEvent &Call,
+                                            CheckerContext &C) const {
+  // We reached a bug, stop exploring the path here by generating a sink.
+  ExplodedNode *ErrNode = C.generateSink();
+  // If we've already reached this node on another path, return.
+  if (!ErrNode)
+    return;
+
+  // Generate the report.
+  BugReport *R = new BugReport(*DoubleCloseBugType,
+      "Closing a previously closed file stream", ErrNode);
+  R->addRange(Call.getSourceRange());
+  R->markInteresting(FileDescSym);
+  C.emitReport(R);
+}
+
+void SimpleStreamChecker::reportLeaks(SymbolVector LeakedStreams,
+                                               CheckerContext &C,
+                                               ExplodedNode *ErrNode) const {
+  // Attach bug reports to the leak node.
+  // TODO: Identify the leaked file descriptor.
+  for (llvm::SmallVector<SymbolRef, 2>::iterator
+      I = LeakedStreams.begin(), E = LeakedStreams.end(); I != E; ++I) {
+    BugReport *R = new BugReport(*LeakBugType,
+        "Opened file is never closed; potential resource leak", ErrNode);
+    R->markInteresting(*I);
+    C.emitReport(R);
+  }
+}
+
+// Check various ways a symbol can be invalidated.
+// Stop tracking symbols when a value escapes as a result of checkBind.
+// A value escapes in three possible cases:
+// (1) We are binding to something that is not a memory region.
+// (2) We are binding to a MemRegion that does not have stack storage
+// (3) We are binding to a MemRegion with stack storage that the store
+//     does not understand.
+void SimpleStreamChecker::checkBind(SVal loc, SVal val, const Stmt *S,
+                                    CheckerContext &C) const {
+  // Are we storing to something that causes the value to "escape"?
+  bool escapes = true;
+  ProgramStateRef state = C.getState();
+
+  if (loc::MemRegionVal *regionLoc = dyn_cast<loc::MemRegionVal>(&loc)) {
+    escapes = !regionLoc->getRegion()->hasStackStorage();
+
+    if (!escapes) {
+      // To test (3), generate a new state with the binding added.  If it is
+      // the same state, then it escapes (since the store cannot represent
+      // the binding). Do this only if we know that the store is not supposed
+      // to generate the same state.
+      SVal StoredVal = state->getSVal(regionLoc->getRegion());
+      if (StoredVal != val)
+        escapes = (state == (state->bindLoc(*regionLoc, val)));
+    }
+  }
+
+  // If our store can represent the binding and we aren't storing to something
+  // that doesn't have local storage then just return the state and
+  // continue as is.
+  if (!escapes)
+    return;
+
+  // Otherwise, find all symbols referenced by 'val' that we are tracking
+  // and stop tracking them.
+  state = state->scanReachableSymbols<StopTrackingCallback>(val).getState();
+  C.addTransition(state);
+}
+
+bool SimpleStreamChecker::guaranteedNotToCloseFile(const CallEvent &Call) const{
+  // If it's not in a system header, assume it might close a file.
+  if (!Call.isInSystemHeader())
+    return false;
+
+  // Handle cases where we know a buffer's /address/ can escape.
+  if (Call.argumentsMayEscape())
+    return false;
+
+  // Note, even though fclose closes the file, we do not list it here
+  // since the checker is modeling the call.
+
+  return true;
+}
+
+// If the symbol we are tracking is invalidated, do not track the symbol as
+// we cannot reason about it anymore.
+ProgramStateRef
+SimpleStreamChecker::checkRegionChanges(ProgramStateRef State,
+    const StoreManager::InvalidatedSymbols *invalidated,
+    ArrayRef<const MemRegion *> ExplicitRegions,
+    ArrayRef<const MemRegion *> Regions,
+    const CallEvent *Call) const {
+
+  if (!invalidated || invalidated->empty())
+    return State;
+
+  // If it's a call which might close the file, we assume that all regions
+  // (explicit and implicit) escaped. Otherwise, whitelist explicit pointers
+  // (the parameters to the call); we still can track them.
+  llvm::SmallPtrSet<SymbolRef, 8> WhitelistedSymbols;
+  if (!Call || guaranteedNotToCloseFile(*Call)) {
+    for (ArrayRef<const MemRegion *>::iterator I = ExplicitRegions.begin(),
+        E = ExplicitRegions.end(); I != E; ++I) {
+      if (const SymbolicRegion *R = (*I)->StripCasts()->getAs<SymbolicRegion>())
+        WhitelistedSymbols.insert(R->getSymbol());
+    }
+  }
+
+  for (StoreManager::InvalidatedSymbols::const_iterator I=invalidated->begin(),
+       E = invalidated->end(); I!=E; ++I) {
+    SymbolRef sym = *I;
+    if (WhitelistedSymbols.count(sym))
+      continue;
+    // The symbol escaped. Optimistically, assume that the corresponding file
+    // handle will be closed somewhere else.
+    State = State->remove<StreamMap>(sym);
+  }
+  return State;
+}
+
+void SimpleStreamChecker::initIdentifierInfo(ASTContext &Ctx) const {
+  if (IIfopen)
+    return;
+  IIfopen = &Ctx.Idents.get("fopen");
+  IIfclose = &Ctx.Idents.get("fclose");
+}
+
+void ento::registerSimpleStreamChecker(CheckerManager &mgr) {
+  mgr.registerChecker<SimpleStreamChecker>();
+}
diff --git a/lib/StaticAnalyzer/Checkers/StackAddrEscapeChecker.cpp b/lib/StaticAnalyzer/Checkers/StackAddrEscapeChecker.cpp
index 54cf569..0c2f266 100644
--- a/lib/StaticAnalyzer/Checkers/StackAddrEscapeChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/StackAddrEscapeChecker.cpp
@@ -109,7 +109,7 @@ void StackAddrEscapeChecker::EmitStackError(CheckerContext &C, const MemRegion *
   if (range.isValid())
     report->addRange(range);
 
-  C.EmitReport(report);
+  C.emitReport(report);
 }
 
 void StackAddrEscapeChecker::checkPreStmt(const ReturnStmt *RS,
@@ -118,8 +118,10 @@ void StackAddrEscapeChecker::checkPreStmt(const ReturnStmt *RS,
   const Expr *RetE = RS->getRetValue();
   if (!RetE)
     return;
- 
-  SVal V = C.getState()->getSVal(RetE, C.getLocationContext());
+  RetE = RetE->IgnoreParens();
+
+  const LocationContext *LCtx = C.getLocationContext();
+  SVal V = C.getState()->getSVal(RetE, LCtx);
   const MemRegion *R = V.getAsRegion();
 
   if (!R)
@@ -132,8 +134,9 @@ void StackAddrEscapeChecker::checkPreStmt(const ReturnStmt *RS,
     return;
 
   // Return stack memory in an ancestor stack frame is fine.
-  const StackFrameContext *SFC = SS->getStackFrame();
-  if (SFC != C.getLocationContext()->getCurrentStackFrame())
+  const StackFrameContext *CurFrame = LCtx->getCurrentStackFrame();
+  const StackFrameContext *MemFrame = SS->getStackFrame();
+  if (MemFrame != CurFrame)
     return;
 
   // Automatic reference counting automatically copies blocks.
@@ -141,6 +144,14 @@ void StackAddrEscapeChecker::checkPreStmt(const ReturnStmt *RS,
       isa<BlockDataRegion>(R))
     return;
 
+  // Returning a record by value is fine. (In this case, the returned
+  // expression will be a copy-constructor, possibly wrapped in an
+  // ExprWithCleanups node.)
+  if (const ExprWithCleanups *Cleanup = dyn_cast<ExprWithCleanups>(RetE))
+    RetE = Cleanup->getSubExpr();
+  if (isa<CXXConstructExpr>(RetE) && RetE->getType()->isRecordType())
+    return;
+
   EmitStackError(C, R, RetE);
 }
 
@@ -221,7 +232,7 @@ void StackAddrEscapeChecker::checkEndPath(CheckerContext &Ctx) const {
     if (range.isValid())
       report->addRange(range);
 
-    Ctx.EmitReport(report);
+    Ctx.emitReport(report);
   }
 }
 
diff --git a/lib/StaticAnalyzer/Checkers/StreamChecker.cpp b/lib/StaticAnalyzer/Checkers/StreamChecker.cpp
index 731dd66..c06ba7c 100644
--- a/lib/StaticAnalyzer/Checkers/StreamChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/StreamChecker.cpp
@@ -104,15 +104,8 @@ private:
 
 } // end anonymous namespace
 
-namespace clang {
-namespace ento {
-  template <>
-  struct ProgramStateTrait<StreamState> 
-    : public ProgramStatePartialTrait<llvm::ImmutableMap<SymbolRef, StreamState> > {
-    static void *GDMIndex() { static int x; return &x; }
-  };
-}
-}
+REGISTER_MAP_WITH_PROGRAMSTATE(StreamMap, SymbolRef, StreamState)
+
 
 bool StreamChecker::evalCall(const CallExpr *CE, CheckerContext &C) const {
   const FunctionDecl *FD = C.getCalleeDecl(CE);
@@ -219,11 +212,11 @@ void StreamChecker::Tmpfile(CheckerContext &C, const CallExpr *CE) const {
 
 void StreamChecker::OpenFileAux(CheckerContext &C, const CallExpr *CE) const {
   ProgramStateRef state = C.getState();
-  unsigned Count = C.getCurrentBlockCount();
   SValBuilder &svalBuilder = C.getSValBuilder();
   const LocationContext *LCtx = C.getPredecessor()->getLocationContext();
   DefinedSVal RetVal =
-    cast<DefinedSVal>(svalBuilder.getConjuredSymbolVal(0, CE, LCtx, Count));
+    cast<DefinedSVal>(svalBuilder.conjureSymbolVal(0, CE, LCtx,
+                                                   C.blockCount()));
   state = state->BindExpr(CE, C.getLocationContext(), RetVal);
   
   ConstraintManager &CM = C.getConstraintManager();
@@ -235,9 +228,9 @@ void StreamChecker::OpenFileAux(CheckerContext &C, const CallExpr *CE) const {
   if (SymbolRef Sym = RetVal.getAsSymbol()) {
     // if RetVal is not NULL, set the symbol's state to Opened.
     stateNotNull =
-      stateNotNull->set<StreamState>(Sym,StreamState::getOpened(CE));
+      stateNotNull->set<StreamMap>(Sym,StreamState::getOpened(CE));
     stateNull =
-      stateNull->set<StreamState>(Sym, StreamState::getOpenFailed(CE));
+      stateNull->set<StreamMap>(Sym, StreamState::getOpenFailed(CE));
 
     C.addTransition(stateNotNull);
     C.addTransition(stateNull);
@@ -287,7 +280,7 @@ void StreamChecker::Fseek(CheckerContext &C, const CallExpr *CE) const {
 					"SEEK_SET, SEEK_END, or SEEK_CUR."));
     BugReport *R = new BugReport(*BT_illegalwhence, 
 				 BT_illegalwhence->getDescription(), N);
-    C.EmitReport(R);
+    C.emitReport(R);
   }
 }
 
@@ -363,7 +356,7 @@ ProgramStateRef StreamChecker::CheckNullStream(SVal SV, ProgramStateRef state,
         BT_nullfp.reset(new BuiltinBug("NULL stream pointer",
                                      "Stream pointer might be NULL."));
       BugReport *R =new BugReport(*BT_nullfp, BT_nullfp->getDescription(), N);
-      C.EmitReport(R);
+      C.emitReport(R);
     }
     return 0;
   }
@@ -378,7 +371,7 @@ ProgramStateRef StreamChecker::CheckDoubleClose(const CallExpr *CE,
   if (!Sym)
     return state;
   
-  const StreamState *SS = state->get<StreamState>(Sym);
+  const StreamState *SS = state->get<StreamMap>(Sym);
 
   // If the file stream is not tracked, return.
   if (!SS)
@@ -395,22 +388,24 @@ ProgramStateRef StreamChecker::CheckDoubleClose(const CallExpr *CE,
                                         " closed. Cause undefined behaviour."));
       BugReport *R = new BugReport(*BT_doubleclose,
                                    BT_doubleclose->getDescription(), N);
-      C.EmitReport(R);
+      C.emitReport(R);
     }
     return NULL;
   }
   
   // Close the File Descriptor.
-  return state->set<StreamState>(Sym, StreamState::getClosed(CE));
+  return state->set<StreamMap>(Sym, StreamState::getClosed(CE));
 }
 
 void StreamChecker::checkDeadSymbols(SymbolReaper &SymReaper,
                                      CheckerContext &C) const {
+  // TODO: Clean up the state.
   for (SymbolReaper::dead_iterator I = SymReaper.dead_begin(),
          E = SymReaper.dead_end(); I != E; ++I) {
     SymbolRef Sym = *I;
     ProgramStateRef state = C.getState();
-    const StreamState *SS = state->get<StreamState>(Sym);
+    const StreamState *SS = state->get<StreamMap>(Sym);
+    // TODO: Shouldn't we have a continue here?
     if (!SS)
       return;
 
@@ -422,7 +417,7 @@ void StreamChecker::checkDeadSymbols(SymbolReaper &SymReaper,
                          "Opened File never closed. Potential Resource leak."));
         BugReport *R = new BugReport(*BT_ResourceLeak, 
                                      BT_ResourceLeak->getDescription(), N);
-        C.EmitReport(R);
+        C.emitReport(R);
       }
     }
   }
@@ -430,10 +425,9 @@ void StreamChecker::checkDeadSymbols(SymbolReaper &SymReaper,
 
 void StreamChecker::checkEndPath(CheckerContext &Ctx) const {
   ProgramStateRef state = Ctx.getState();
-  typedef llvm::ImmutableMap<SymbolRef, StreamState> SymMap;
-  SymMap M = state->get<StreamState>();
+  StreamMapTy M = state->get<StreamMap>();
   
-  for (SymMap::iterator I = M.begin(), E = M.end(); I != E; ++I) {
+  for (StreamMapTy::iterator I = M.begin(), E = M.end(); I != E; ++I) {
     StreamState SS = I->second;
     if (SS.isOpened()) {
       ExplodedNode *N = Ctx.addTransition(state);
@@ -443,7 +437,7 @@ void StreamChecker::checkEndPath(CheckerContext &Ctx) const {
                          "Opened File never closed. Potential Resource leak."));
         BugReport *R = new BugReport(*BT_ResourceLeak, 
                                      BT_ResourceLeak->getDescription(), N);
-        Ctx.EmitReport(R);
+        Ctx.emitReport(R);
       }
     }
   }
@@ -460,12 +454,12 @@ void StreamChecker::checkPreStmt(const ReturnStmt *S, CheckerContext &C) const {
   if (!Sym)
     return;
   
-  const StreamState *SS = state->get<StreamState>(Sym);
+  const StreamState *SS = state->get<StreamMap>(Sym);
   if(!SS)
     return;
 
   if (SS->isOpened())
-    state = state->set<StreamState>(Sym, StreamState::getEscaped(S));
+    state = state->set<StreamMap>(Sym, StreamState::getEscaped(S));
 
   C.addTransition(state);
 }
diff --git a/lib/StaticAnalyzer/Checkers/TaintTesterChecker.cpp b/lib/StaticAnalyzer/Checkers/TaintTesterChecker.cpp
index 1133682..382be84 100644
--- a/lib/StaticAnalyzer/Checkers/TaintTesterChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/TaintTesterChecker.cpp
@@ -52,7 +52,7 @@ void TaintTesterChecker::checkPostStmt(const Expr *E,
       initBugType();
       BugReport *report = new BugReport(*BT, "tainted",N);
       report->addRange(E->getSourceRange());
-      C.EmitReport(report);
+      C.emitReport(report);
     }
   }
 }
diff --git a/lib/StaticAnalyzer/Checkers/UndefBranchChecker.cpp b/lib/StaticAnalyzer/Checkers/UndefBranchChecker.cpp
index 70a33c7..70e141e 100644
--- a/lib/StaticAnalyzer/Checkers/UndefBranchChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/UndefBranchChecker.cpp
@@ -99,11 +99,10 @@ void UndefBranchChecker::checkBranchCondition(const Stmt *Condition,
 
       // Emit the bug report.
       BugReport *R = new BugReport(*BT, BT->getDescription(), N);
-      bugreporter::addTrackNullOrUndefValueVisitor(N, Ex, R);
+      bugreporter::trackNullOrUndefValue(N, Ex, *R);
       R->addRange(Ex->getSourceRange());
-      R->disablePathPruning();
 
-      Ctx.EmitReport(R);
+      Ctx.emitReport(R);
     }
   }
 }
diff --git a/lib/StaticAnalyzer/Checkers/UndefCapturedBlockVarChecker.cpp b/lib/StaticAnalyzer/Checkers/UndefCapturedBlockVarChecker.cpp
index 675b38a..30ccffa 100644
--- a/lib/StaticAnalyzer/Checkers/UndefCapturedBlockVarChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/UndefCapturedBlockVarChecker.cpp
@@ -96,7 +96,7 @@ UndefCapturedBlockVarChecker::checkPostStmt(const BlockExpr *BE,
         R->addVisitor(new FindLastStoreBRVisitor(VRVal, VR));
         R->disablePathPruning();
         // need location of block
-        C.EmitReport(R);
+        C.emitReport(R);
       }
   }
 }
diff --git a/lib/StaticAnalyzer/Checkers/UndefResultChecker.cpp b/lib/StaticAnalyzer/Checkers/UndefResultChecker.cpp
index e220499..415bab5 100644
--- a/lib/StaticAnalyzer/Checkers/UndefResultChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/UndefResultChecker.cpp
@@ -76,13 +76,12 @@ void UndefResultChecker::checkPostStmt(const BinaryOperator *B,
     BugReport *report = new BugReport(*BT, OS.str(), N);
     if (Ex) {
       report->addRange(Ex->getSourceRange());
-      bugreporter::addTrackNullOrUndefValueVisitor(N, Ex, report);
+      bugreporter::trackNullOrUndefValue(N, Ex, *report);
     }
     else
-      bugreporter::addTrackNullOrUndefValueVisitor(N, B, report);
+      bugreporter::trackNullOrUndefValue(N, B, *report);
     
-    report->disablePathPruning();
-    C.EmitReport(report);
+    C.emitReport(report);
   }
 }
 
diff --git a/lib/StaticAnalyzer/Checkers/UndefinedArraySubscriptChecker.cpp b/lib/StaticAnalyzer/Checkers/UndefinedArraySubscriptChecker.cpp
index 6ae3c18..b3a83e8 100644
--- a/lib/StaticAnalyzer/Checkers/UndefinedArraySubscriptChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/UndefinedArraySubscriptChecker.cpp
@@ -42,8 +42,8 @@ UndefinedArraySubscriptChecker::checkPreStmt(const ArraySubscriptExpr *A,
       // Generate a report for this bug.
       BugReport *R = new BugReport(*BT, BT->getName(), N);
       R->addRange(A->getIdx()->getSourceRange());
-      bugreporter::addTrackNullOrUndefValueVisitor(N, A->getIdx(), R);
-      C.EmitReport(R);
+      bugreporter::trackNullOrUndefValue(N, A->getIdx(), *R);
+      C.emitReport(R);
     }
   }
 }
diff --git a/lib/StaticAnalyzer/Checkers/UndefinedAssignmentChecker.cpp b/lib/StaticAnalyzer/Checkers/UndefinedAssignmentChecker.cpp
index 14a884e..410010a 100644
--- a/lib/StaticAnalyzer/Checkers/UndefinedAssignmentChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/UndefinedAssignmentChecker.cpp
@@ -78,10 +78,9 @@ void UndefinedAssignmentChecker::checkBind(SVal location, SVal val,
   BugReport *R = new BugReport(*BT, str, N);
   if (ex) {
     R->addRange(ex->getSourceRange());
-    bugreporter::addTrackNullOrUndefValueVisitor(N, ex, R);
+    bugreporter::trackNullOrUndefValue(N, ex, *R);
   }
-  R->disablePathPruning();
-  C.EmitReport(R);
+  C.emitReport(R);
 }
 
 void ento::registerUndefinedAssignmentChecker(CheckerManager &mgr) {
diff --git a/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp b/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp
index d35455c..171e15b 100644
--- a/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/UnixAPIChecker.cpp
@@ -41,6 +41,7 @@ public:
   void CheckCallocZero(CheckerContext &C, const CallExpr *CE) const;
   void CheckMallocZero(CheckerContext &C, const CallExpr *CE) const;
   void CheckReallocZero(CheckerContext &C, const CallExpr *CE) const;
+  void CheckReallocfZero(CheckerContext &C, const CallExpr *CE) const;
   void CheckAllocaZero(CheckerContext &C, const CallExpr *CE) const;
   void CheckVallocZero(CheckerContext &C, const CallExpr *CE) const;
 
@@ -138,7 +139,7 @@ void UnixAPIChecker::CheckOpen(CheckerContext &C, const CallExpr *CE) const {
                             "Call to 'open' requires a third argument when "
                             "the 'O_CREAT' flag is set", N);
     report->addRange(oflagsEx->getSourceRange());
-    C.EmitReport(report);
+    C.emitReport(report);
   }
 }
 
@@ -183,11 +184,12 @@ void UnixAPIChecker::CheckPthreadOnce(CheckerContext &C,
 
   BugReport *report = new BugReport(*BT_pthreadOnce, os.str(), N);
   report->addRange(CE->getArg(0)->getSourceRange());
-  C.EmitReport(report);
+  C.emitReport(report);
 }
 
 //===----------------------------------------------------------------------===//
-// "calloc", "malloc", "realloc", "alloca" and "valloc" with allocation size 0
+// "calloc", "malloc", "realloc", "reallocf", "alloca" and "valloc"
+// with allocation size 0
 //===----------------------------------------------------------------------===//
 // FIXME: Eventually these should be rolled into the MallocChecker, but right now
 // they're more basic and valuable for widespread use.
@@ -224,8 +226,8 @@ bool UnixAPIChecker::ReportZeroByteAllocation(CheckerContext &C,
   BugReport *report = new BugReport(*BT_mallocZero, os.str(), N);
 
   report->addRange(arg->getSourceRange());
-  bugreporter::addTrackNullOrUndefValueVisitor(N, arg, report);
-  C.EmitReport(report);
+  bugreporter::trackNullOrUndefValue(N, arg, *report);
+  C.emitReport(report);
 
   return true;
 }
@@ -307,6 +309,11 @@ void UnixAPIChecker::CheckReallocZero(CheckerContext &C,
   BasicAllocationCheck(C, CE, 2, 1, "realloc");
 }
 
+void UnixAPIChecker::CheckReallocfZero(CheckerContext &C,
+                                       const CallExpr *CE) const {
+  BasicAllocationCheck(C, CE, 2, 1, "reallocf");
+}
+
 void UnixAPIChecker::CheckAllocaZero(CheckerContext &C,
                                      const CallExpr *CE) const {
   BasicAllocationCheck(C, CE, 1, 0, "alloca");
@@ -339,6 +346,7 @@ void UnixAPIChecker::checkPreStmt(const CallExpr *CE,
       .Case("calloc", &UnixAPIChecker::CheckCallocZero)
       .Case("malloc", &UnixAPIChecker::CheckMallocZero)
       .Case("realloc", &UnixAPIChecker::CheckReallocZero)
+      .Case("reallocf", &UnixAPIChecker::CheckReallocfZero)
       .Cases("alloca", "__builtin_alloca", &UnixAPIChecker::CheckAllocaZero)
       .Case("valloc", &UnixAPIChecker::CheckVallocZero)
       .Default(NULL);
diff --git a/lib/StaticAnalyzer/Checkers/VLASizeChecker.cpp b/lib/StaticAnalyzer/Checkers/VLASizeChecker.cpp
index fab4adf..58f9ec0 100644
--- a/lib/StaticAnalyzer/Checkers/VLASizeChecker.cpp
+++ b/lib/StaticAnalyzer/Checkers/VLASizeChecker.cpp
@@ -69,8 +69,8 @@ void VLASizeChecker::reportBug(VLASize_Kind Kind,
 
   BugReport *report = new BugReport(*BT, os.str(), N);
   report->addRange(SizeE->getSourceRange());
-  bugreporter::addTrackNullOrUndefValueVisitor(N, SizeE, report);
-  C.EmitReport(report);
+  bugreporter::trackNullOrUndefValue(N, SizeE, *report);
+  C.emitReport(report);
   return;
 }
 
diff --git a/lib/StaticAnalyzer/Core/AnalysisManager.cpp b/lib/StaticAnalyzer/Core/AnalysisManager.cpp
index efeba17..011d4c09 100644
--- a/lib/StaticAnalyzer/Core/AnalysisManager.cpp
+++ b/lib/StaticAnalyzer/Core/AnalysisManager.cpp
@@ -20,33 +20,19 @@ AnalysisManager::AnalysisManager(ASTContext &ctx, DiagnosticsEngine &diags,
                                  StoreManagerCreator storemgr,
                                  ConstraintManagerCreator constraintmgr, 
                                  CheckerManager *checkerMgr,
-                                 unsigned maxnodes, unsigned maxvisit,
-                                 bool vizdot, bool vizubi,
-                                 AnalysisPurgeMode purge,
-                                 bool eager, bool trim,
-                                 bool useUnoptimizedCFG,
-                                 bool addImplicitDtors,
-                                 bool eagerlyTrimEGraph,
-                                 AnalysisIPAMode ipa,
-                                 unsigned inlineMaxStack,
-                                 unsigned inlineMaxFunctionSize,
-                                 AnalysisInliningMode IMode,
-                                 bool NoRetry)
-  : AnaCtxMgr(useUnoptimizedCFG, addImplicitDtors, /*addInitializers=*/true),
-    Ctx(ctx), Diags(diags), LangOpts(lang),
+                                 AnalyzerOptions &Options)
+  : AnaCtxMgr(Options.UnoptimizedCFG,
+              /*AddImplicitDtors=*/true,
+              /*AddInitializers=*/true,
+              Options.includeTemporaryDtorsInCFG(),
+              Options.shouldSynthesizeBodies()),
+    Ctx(ctx),
+    Diags(diags),
+    LangOpts(lang),
     PathConsumers(PDC),
     CreateStoreMgr(storemgr), CreateConstraintMgr(constraintmgr),
-    CheckerMgr(checkerMgr), 
-    MaxNodes(maxnodes), MaxVisit(maxvisit),
-    VisualizeEGDot(vizdot), VisualizeEGUbi(vizubi), PurgeDead(purge),
-    EagerlyAssume(eager), TrimGraph(trim),
-    EagerlyTrimEGraph(eagerlyTrimEGraph),
-    IPAMode(ipa),
-    InlineMaxStackDepth(inlineMaxStack),
-    InlineMaxFunctionSize(inlineMaxFunctionSize),
-    InliningMode(IMode),
-    NoRetryExhausted(NoRetry)
-{
+    CheckerMgr(checkerMgr),
+    options(Options) {
   AnaCtxMgr.getCFGBuildOptions().setAllAlwaysAdd();
 }
 
diff --git a/lib/StaticAnalyzer/Core/AnalyzerOptions.cpp b/lib/StaticAnalyzer/Core/AnalyzerOptions.cpp
new file mode 100644
index 0000000..da88589
--- /dev/null
+++ b/lib/StaticAnalyzer/Core/AnalyzerOptions.cpp
@@ -0,0 +1,138 @@
+//===-- AnalyzerOptions.cpp - Analysis Engine Options -----------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains special accessors for analyzer configuration options
+// with string representations.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/StaticAnalyzer/Core/AnalyzerOptions.h"
+#include "llvm/ADT/StringSwitch.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace clang;
+using namespace llvm;
+
+bool
+AnalyzerOptions::mayInlineCXXMemberFunction(CXXInlineableMemberKind K) {
+  if (IPAMode < Inlining)
+    return false;
+
+  if (!CXXMemberInliningMode) {
+    static const char *ModeKey = "c++-inlining";
+    
+    StringRef ModeStr(Config.GetOrCreateValue(ModeKey,
+                                              "methods").getValue());
+
+    CXXInlineableMemberKind &MutableMode =
+      const_cast<CXXInlineableMemberKind &>(CXXMemberInliningMode);
+
+    MutableMode = llvm::StringSwitch<CXXInlineableMemberKind>(ModeStr)
+      .Case("constructors", CIMK_Constructors)
+      .Case("destructors", CIMK_Destructors)
+      .Case("none", CIMK_None)
+      .Case("methods", CIMK_MemberFunctions)
+      .Default(CXXInlineableMemberKind());
+
+    if (!MutableMode) {
+      // FIXME: We should emit a warning here about an unknown inlining kind,
+      // but the AnalyzerOptions doesn't have access to a diagnostic engine.
+      MutableMode = CIMK_None;
+    }
+  }
+
+  return CXXMemberInliningMode >= K;
+}
+
+static StringRef toString(bool b) { return b ? "true" : "false"; }
+
+bool AnalyzerOptions::getBooleanOption(StringRef Name, bool DefaultVal) {
+  // FIXME: We should emit a warning here if the value is something other than
+  // "true", "false", or the empty string (meaning the default value),
+  // but the AnalyzerOptions doesn't have access to a diagnostic engine.
+  StringRef V(Config.GetOrCreateValue(Name, toString(DefaultVal)).getValue());
+  return llvm::StringSwitch<bool>(V)
+      .Case("true", true)
+      .Case("false", false)
+      .Default(DefaultVal);
+}
+
+bool AnalyzerOptions::getBooleanOption(llvm::Optional<bool> &V,
+                                       StringRef Name,
+                                       bool DefaultVal) {
+  if (!V.hasValue())
+    V = getBooleanOption(Name, DefaultVal);
+  return V.getValue();
+}
+
+bool AnalyzerOptions::includeTemporaryDtorsInCFG() {
+  return getBooleanOption(IncludeTemporaryDtorsInCFG,
+                          "cfg-temporary-dtors",
+                          /* Default = */ false);
+}
+
+bool AnalyzerOptions::mayInlineCXXStandardLibrary() {
+  return getBooleanOption(InlineCXXStandardLibrary,
+                          "c++-stdlib-inlining",
+                          /*Default=*/true);
+}
+
+bool AnalyzerOptions::mayInlineTemplateFunctions() {
+  return getBooleanOption(InlineTemplateFunctions,
+                          "c++-template-inlining",
+                          /*Default=*/true);
+}
+
+bool AnalyzerOptions::mayInlineObjCMethod() {
+  return getBooleanOption(ObjCInliningMode,
+                          "objc-inlining",
+                          /* Default = */ true);
+}
+
+bool AnalyzerOptions::shouldPruneNullReturnPaths() {
+  return getBooleanOption(PruneNullReturnPaths,
+                          "suppress-null-return-paths",
+                          /* Default = */ true);
+}
+
+bool AnalyzerOptions::shouldAvoidSuppressingNullArgumentPaths() {
+  return getBooleanOption(AvoidSuppressingNullArgumentPaths,
+                          "avoid-suppressing-null-argument-paths",
+                          /* Default = */ false);
+}
+
+int AnalyzerOptions::getOptionAsInteger(StringRef Name, int DefaultVal) {
+  llvm::SmallString<10> StrBuf;
+  llvm::raw_svector_ostream OS(StrBuf);
+  OS << DefaultVal;
+  
+  StringRef V(Config.GetOrCreateValue(Name, OS.str()).getValue());
+  int Res = DefaultVal;
+  bool b = V.getAsInteger(10, Res);
+  assert(!b && "analyzer-config option should be numeric");
+  (void) b;
+  return Res;
+}
+
+unsigned AnalyzerOptions::getAlwaysInlineSize() {
+  if (!AlwaysInlineSize.hasValue())
+    AlwaysInlineSize = getOptionAsInteger("ipa-always-inline-size", 3);
+  return AlwaysInlineSize.getValue();
+}
+
+unsigned AnalyzerOptions::getGraphTrimInterval() {
+  if (!GraphTrimInterval.hasValue())
+    GraphTrimInterval = getOptionAsInteger("graph-trim-interval", 1000);
+  return GraphTrimInterval.getValue();
+}
+
+bool AnalyzerOptions::shouldSynthesizeBodies() {
+  return getBooleanOption("faux-bodies", true);
+}
diff --git a/lib/StaticAnalyzer/Core/BasicConstraintManager.cpp b/lib/StaticAnalyzer/Core/BasicConstraintManager.cpp
deleted file mode 100644
index 8897756..0000000
--- a/lib/StaticAnalyzer/Core/BasicConstraintManager.cpp
+++ /dev/null
@@ -1,446 +0,0 @@
-//== BasicConstraintManager.cpp - Manage basic constraints.------*- C++ -*--==//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//  This file defines BasicConstraintManager, a class that tracks simple
-//  equality and inequality constraints on symbolic values of ProgramState.
-//
-//===----------------------------------------------------------------------===//
-
-#include "SimpleConstraintManager.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/APSIntType.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h"
-#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramStateTrait.h"
-#include "llvm/Support/raw_ostream.h"
-
-using namespace clang;
-using namespace ento;
-
-
-namespace { class ConstNotEq {}; }
-namespace { class ConstEq {}; }
-
-typedef llvm::ImmutableMap<SymbolRef,ProgramState::IntSetTy> ConstNotEqTy;
-typedef llvm::ImmutableMap<SymbolRef,const llvm::APSInt*> ConstEqTy;
-
-static int ConstEqIndex = 0;
-static int ConstNotEqIndex = 0;
-
-namespace clang {
-namespace ento {
-template<>
-struct ProgramStateTrait<ConstNotEq> :
-  public ProgramStatePartialTrait<ConstNotEqTy> {
-  static inline void *GDMIndex() { return &ConstNotEqIndex; }
-};
-
-template<>
-struct ProgramStateTrait<ConstEq> : public ProgramStatePartialTrait<ConstEqTy> {
-  static inline void *GDMIndex() { return &ConstEqIndex; }
-};
-}
-}
-
-namespace {
-// BasicConstraintManager only tracks equality and inequality constraints of
-// constants and integer variables.
-class BasicConstraintManager
-  : public SimpleConstraintManager {
-  ProgramState::IntSetTy::Factory ISetFactory;
-public:
-  BasicConstraintManager(ProgramStateManager &statemgr, SubEngine &subengine)
-    : SimpleConstraintManager(subengine, statemgr.getBasicVals()),
-      ISetFactory(statemgr.getAllocator()) {}
-
-  ProgramStateRef assumeSymEquality(ProgramStateRef State, SymbolRef Sym,
-                                    const llvm::APSInt &V,
-                                    const llvm::APSInt &Adjustment,
-                                    bool Assumption);
-
-  ProgramStateRef assumeSymNE(ProgramStateRef State, SymbolRef Sym,
-                              const llvm::APSInt &V,
-                              const llvm::APSInt &Adjustment) {
-    return assumeSymEquality(State, Sym, V, Adjustment, false);
-  }
-
-  ProgramStateRef assumeSymEQ(ProgramStateRef State, SymbolRef Sym,
-                              const llvm::APSInt &V,
-                              const llvm::APSInt &Adjustment) {
-    return assumeSymEquality(State, Sym, V, Adjustment, true);
-  }
-
-  ProgramStateRef assumeSymLT(ProgramStateRef state,
-                                  SymbolRef sym,
-                                  const llvm::APSInt& V,
-                                  const llvm::APSInt& Adjustment);
-
-  ProgramStateRef assumeSymGT(ProgramStateRef state,
-                                  SymbolRef sym,
-                                  const llvm::APSInt& V,
-                                  const llvm::APSInt& Adjustment);
-
-  ProgramStateRef assumeSymGE(ProgramStateRef state,
-                                  SymbolRef sym,
-                                  const llvm::APSInt& V,
-                                  const llvm::APSInt& Adjustment);
-
-  ProgramStateRef assumeSymLE(ProgramStateRef state,
-                                  SymbolRef sym,
-                                  const llvm::APSInt& V,
-                                  const llvm::APSInt& Adjustment);
-
-  ProgramStateRef AddEQ(ProgramStateRef state,
-                            SymbolRef sym,
-                            const llvm::APSInt& V);
-
-  ProgramStateRef AddNE(ProgramStateRef state,
-                            SymbolRef sym,
-                            const llvm::APSInt& V);
-
-  const llvm::APSInt* getSymVal(ProgramStateRef state,
-                                SymbolRef sym) const;
-
-  bool isNotEqual(ProgramStateRef state,
-                  SymbolRef sym,
-                  const llvm::APSInt& V) const;
-
-  bool isEqual(ProgramStateRef state,
-               SymbolRef sym,
-               const llvm::APSInt& V) const;
-
-  ProgramStateRef removeDeadBindings(ProgramStateRef state,
-                                         SymbolReaper& SymReaper);
-
-  bool performTest(llvm::APSInt SymVal, llvm::APSInt Adjustment,
-                   BinaryOperator::Opcode Op, llvm::APSInt ComparisonVal);
-
-  void print(ProgramStateRef state,
-             raw_ostream &Out,
-             const char* nl,
-             const char *sep);
-};
-
-} // end anonymous namespace
-
-ConstraintManager*
-ento::CreateBasicConstraintManager(ProgramStateManager& statemgr,
-                                   SubEngine &subengine) {
-  return new BasicConstraintManager(statemgr, subengine);
-}
-
-// FIXME: This is a more general utility and should live somewhere else.
-bool BasicConstraintManager::performTest(llvm::APSInt SymVal,
-                                         llvm::APSInt Adjustment,
-                                         BinaryOperator::Opcode Op,
-                                         llvm::APSInt ComparisonVal) {
-  APSIntType Type(Adjustment);
-  Type.apply(SymVal);
-  Type.apply(ComparisonVal);
-  SymVal += Adjustment;
-
-  assert(BinaryOperator::isComparisonOp(Op));
-  BasicValueFactory &BVF = getBasicVals();
-  const llvm::APSInt *Result = BVF.evalAPSInt(Op, SymVal, ComparisonVal);
-  assert(Result && "Comparisons should always have valid results.");
-
-  return Result->getBoolValue();
-}
-
-ProgramStateRef
-BasicConstraintManager::assumeSymEquality(ProgramStateRef State, SymbolRef Sym,
-                                          const llvm::APSInt &V,
-                                          const llvm::APSInt &Adjustment,
-                                          bool Assumption) {
-  // Before we do any real work, see if the value can even show up.
-  APSIntType AdjustmentType(Adjustment);
-  if (AdjustmentType.testInRange(V) != APSIntType::RTR_Within)
-    return Assumption ? NULL : State;
-
-  // Get the symbol type.
-  BasicValueFactory &BVF = getBasicVals();
-  ASTContext &Ctx = BVF.getContext();
-  APSIntType SymbolType = BVF.getAPSIntType(Sym->getType(Ctx));
-
-  // First, see if the adjusted value is within range for the symbol.
-  llvm::APSInt Adjusted = AdjustmentType.convert(V) - Adjustment;
-  if (SymbolType.testInRange(Adjusted) != APSIntType::RTR_Within)
-    return Assumption ? NULL : State;
-
-  // Now we can do things properly in the symbol space.
-  SymbolType.apply(Adjusted);
-
-  // Second, determine if sym == X, where X+Adjustment != V.
-  if (const llvm::APSInt *X = getSymVal(State, Sym)) {
-    bool IsFeasible = (*X == Adjusted);
-    return (IsFeasible == Assumption) ? State : NULL;
-  }
-
-  // Third, determine if we already know sym+Adjustment != V.
-  if (isNotEqual(State, Sym, Adjusted))
-    return Assumption ? NULL : State;
-
-  // If we reach here, sym is not a constant and we don't know if it is != V.
-  // Make the correct assumption.
-  if (Assumption)
-    return AddEQ(State, Sym, Adjusted);
-  else
-    return AddNE(State, Sym, Adjusted);
-}
-
-// The logic for these will be handled in another ConstraintManager.
-// Approximate it here anyway by handling some edge cases.
-ProgramStateRef 
-BasicConstraintManager::assumeSymLT(ProgramStateRef state,
-                                    SymbolRef sym,
-                                    const llvm::APSInt &V,
-                                    const llvm::APSInt &Adjustment) {
-  APSIntType ComparisonType(V), AdjustmentType(Adjustment);
-
-  // Is 'V' out of range above the type?
-  llvm::APSInt Max = AdjustmentType.getMaxValue();
-  if (V > ComparisonType.convert(Max)) {
-    // This path is trivially feasible.
-    return state;
-  }
-
-  // Is 'V' the smallest possible value, or out of range below the type?
-  llvm::APSInt Min = AdjustmentType.getMinValue();
-  if (V <= ComparisonType.convert(Min)) {
-    // sym cannot be any value less than 'V'.  This path is infeasible.
-    return NULL;
-  }
-
-  // Reject a path if the value of sym is a constant X and !(X+Adj < V).
-  if (const llvm::APSInt *X = getSymVal(state, sym)) {
-    bool isFeasible = performTest(*X, Adjustment, BO_LT, V);
-    return isFeasible ? state : NULL;
-  }
-
-  // FIXME: For now have assuming x < y be the same as assuming sym != V;
-  return assumeSymNE(state, sym, V, Adjustment);
-}
-
-ProgramStateRef 
-BasicConstraintManager::assumeSymGT(ProgramStateRef state,
-                                    SymbolRef sym,
-                                    const llvm::APSInt &V,
-                                    const llvm::APSInt &Adjustment) {
-  APSIntType ComparisonType(V), AdjustmentType(Adjustment);
-
-  // Is 'V' the largest possible value, or out of range above the type?
-  llvm::APSInt Max = AdjustmentType.getMaxValue();
-  if (V >= ComparisonType.convert(Max)) {
-    // sym cannot be any value greater than 'V'.  This path is infeasible.
-    return NULL;
-  }
-
-  // Is 'V' out of range below the type?
-  llvm::APSInt Min = AdjustmentType.getMinValue();
-  if (V < ComparisonType.convert(Min)) {
-    // This path is trivially feasible.
-    return state;
-  }
-
-  // Reject a path if the value of sym is a constant X and !(X+Adj > V).
-  if (const llvm::APSInt *X = getSymVal(state, sym)) {
-    bool isFeasible = performTest(*X, Adjustment, BO_GT, V);
-    return isFeasible ? state : NULL;
-  }
-
-  // FIXME: For now have assuming x > y be the same as assuming sym != V;
-  return assumeSymNE(state, sym, V, Adjustment);
-}
-
-ProgramStateRef 
-BasicConstraintManager::assumeSymGE(ProgramStateRef state,
-                                    SymbolRef sym,
-                                    const llvm::APSInt &V,
-                                    const llvm::APSInt &Adjustment) {
-  APSIntType ComparisonType(V), AdjustmentType(Adjustment);
-
-  // Is 'V' the largest possible value, or out of range above the type?
-  llvm::APSInt Max = AdjustmentType.getMaxValue();
-  ComparisonType.apply(Max);
-
-  if (V > Max) {
-    // sym cannot be any value greater than 'V'.  This path is infeasible.
-    return NULL;
-  } else if (V == Max) {
-    // If the path is feasible then as a consequence we know that
-    // 'sym+Adjustment == V' because there are no larger values.
-    // Add this constraint.
-    return assumeSymEQ(state, sym, V, Adjustment);
-  }
-
-  // Is 'V' out of range below the type?
-  llvm::APSInt Min = AdjustmentType.getMinValue();
-  if (V < ComparisonType.convert(Min)) {
-    // This path is trivially feasible.
-    return state;
-  }
-
-  // Reject a path if the value of sym is a constant X and !(X+Adj >= V).
-  if (const llvm::APSInt *X = getSymVal(state, sym)) {
-    bool isFeasible = performTest(*X, Adjustment, BO_GE, V);
-    return isFeasible ? state : NULL;
-  }
-
-  return state;
-}
-
-ProgramStateRef 
-BasicConstraintManager::assumeSymLE(ProgramStateRef state,
-                                    SymbolRef sym,
-                                    const llvm::APSInt &V,
-                                    const llvm::APSInt &Adjustment) {
-  APSIntType ComparisonType(V), AdjustmentType(Adjustment);
-
-  // Is 'V' out of range above the type?
-  llvm::APSInt Max = AdjustmentType.getMaxValue();
-  if (V > ComparisonType.convert(Max)) {
-    // This path is trivially feasible.
-    return state;
-  }
-
-  // Is 'V' the smallest possible value, or out of range below the type?
-  llvm::APSInt Min = AdjustmentType.getMinValue();
-  ComparisonType.apply(Min);
-
-  if (V < Min) {
-    // sym cannot be any value less than 'V'.  This path is infeasible.
-    return NULL;
-  } else if (V == Min) {
-    // If the path is feasible then as a consequence we know that
-    // 'sym+Adjustment == V' because there are no smaller values.
-    // Add this constraint.
-    return assumeSymEQ(state, sym, V, Adjustment);
-  }
-
-  // Reject a path if the value of sym is a constant X and !(X+Adj >= V).
-  if (const llvm::APSInt *X = getSymVal(state, sym)) {
-    bool isFeasible = performTest(*X, Adjustment, BO_LE, V);
-    return isFeasible ? state : NULL;
-  }
-
-  return state;
-}
-
-ProgramStateRef BasicConstraintManager::AddEQ(ProgramStateRef state,
-                                                  SymbolRef sym,
-                                             const llvm::APSInt& V) {
-  // Now that we have an actual value, we can throw out the NE-set.
-  // Create a new state with the old bindings replaced.
-  state = state->remove<ConstNotEq>(sym);
-  return state->set<ConstEq>(sym, &getBasicVals().getValue(V));
-}
-
-ProgramStateRef BasicConstraintManager::AddNE(ProgramStateRef state,
-                                                  SymbolRef sym,
-                                                  const llvm::APSInt& V) {
-
-  // First, retrieve the NE-set associated with the given symbol.
-  ConstNotEqTy::data_type* T = state->get<ConstNotEq>(sym);
-  ProgramState::IntSetTy S = T ? *T : ISetFactory.getEmptySet();
-
-  // Now add V to the NE set.
-  S = ISetFactory.add(S, &getBasicVals().getValue(V));
-
-  // Create a new state with the old binding replaced.
-  return state->set<ConstNotEq>(sym, S);
-}
-
-const llvm::APSInt* BasicConstraintManager::getSymVal(ProgramStateRef state,
-                                                      SymbolRef sym) const {
-  const ConstEqTy::data_type* T = state->get<ConstEq>(sym);
-  return T ? *T : NULL;
-}
-
-bool BasicConstraintManager::isNotEqual(ProgramStateRef state,
-                                        SymbolRef sym,
-                                        const llvm::APSInt& V) const {
-
-  // Retrieve the NE-set associated with the given symbol.
-  const ConstNotEqTy::data_type* T = state->get<ConstNotEq>(sym);
-
-  // See if V is present in the NE-set.
-  return T ? T->contains(&getBasicVals().getValue(V)) : false;
-}
-
-bool BasicConstraintManager::isEqual(ProgramStateRef state,
-                                     SymbolRef sym,
-                                     const llvm::APSInt& V) const {
-  // Retrieve the EQ-set associated with the given symbol.
-  const ConstEqTy::data_type* T = state->get<ConstEq>(sym);
-  // See if V is present in the EQ-set.
-  return T ? **T == V : false;
-}
-
-/// Scan all symbols referenced by the constraints. If the symbol is not alive
-/// as marked in LSymbols, mark it as dead in DSymbols.
-ProgramStateRef 
-BasicConstraintManager::removeDeadBindings(ProgramStateRef state,
-                                           SymbolReaper& SymReaper) {
-
-  ConstEqTy CE = state->get<ConstEq>();
-  ConstEqTy::Factory& CEFactory = state->get_context<ConstEq>();
-
-  for (ConstEqTy::iterator I = CE.begin(), E = CE.end(); I!=E; ++I) {
-    SymbolRef sym = I.getKey();
-    if (SymReaper.maybeDead(sym))
-      CE = CEFactory.remove(CE, sym);
-  }
-  state = state->set<ConstEq>(CE);
-
-  ConstNotEqTy CNE = state->get<ConstNotEq>();
-  ConstNotEqTy::Factory& CNEFactory = state->get_context<ConstNotEq>();
-
-  for (ConstNotEqTy::iterator I = CNE.begin(), E = CNE.end(); I != E; ++I) {
-    SymbolRef sym = I.getKey();
-    if (SymReaper.maybeDead(sym))
-      CNE = CNEFactory.remove(CNE, sym);
-  }
-
-  return state->set<ConstNotEq>(CNE);
-}
-
-void BasicConstraintManager::print(ProgramStateRef state,
-                                   raw_ostream &Out,
-                                   const char* nl, const char *sep) {
-  // Print equality constraints.
-
-  ConstEqTy CE = state->get<ConstEq>();
-
-  if (!CE.isEmpty()) {
-    Out << nl << sep << "'==' constraints:";
-    for (ConstEqTy::iterator I = CE.begin(), E = CE.end(); I!=E; ++I)
-      Out << nl << " $" << I.getKey() << " : " << *I.getData();
-  }
-
-  // Print != constraints.
-
-  ConstNotEqTy CNE = state->get<ConstNotEq>();
-
-  if (!CNE.isEmpty()) {
-    Out << nl << sep << "'!=' constraints:";
-
-    for (ConstNotEqTy::iterator I = CNE.begin(), EI = CNE.end(); I!=EI; ++I) {
-      Out << nl << " $" << I.getKey() << " : ";
-      bool isFirst = true;
-
-      ProgramState::IntSetTy::iterator J = I.getData().begin(),
-                                  EJ = I.getData().end();
-
-      for ( ; J != EJ; ++J) {
-        if (isFirst) isFirst = false;
-        else Out << ", ";
-
-        Out << (*J)->getSExtValue(); // Hack: should print to raw_ostream.
-      }
-    }
-  }
-}
diff --git a/lib/StaticAnalyzer/Core/BasicValueFactory.cpp b/lib/StaticAnalyzer/Core/BasicValueFactory.cpp
index 20c7361..a6c400f 100644
--- a/lib/StaticAnalyzer/Core/BasicValueFactory.cpp
+++ b/lib/StaticAnalyzer/Core/BasicValueFactory.cpp
@@ -101,11 +101,7 @@ const llvm::APSInt& BasicValueFactory::getValue(uint64_t X, unsigned BitWidth,
 
 const llvm::APSInt& BasicValueFactory::getValue(uint64_t X, QualType T) {
 
-  unsigned bits = Ctx.getTypeSize(T);
-  llvm::APSInt V(bits, 
-                 T->isUnsignedIntegerOrEnumerationType() || Loc::isLocType(T));
-  V = X;
-  return getValue(V);
+  return getValue(getAPSIntType(T).getValue(X));
 }
 
 const CompoundValData*
diff --git a/lib/StaticAnalyzer/Core/BugReporter.cpp b/lib/StaticAnalyzer/Core/BugReporter.cpp
index 571baec..c898d65 100644
--- a/lib/StaticAnalyzer/Core/BugReporter.cpp
+++ b/lib/StaticAnalyzer/Core/BugReporter.cpp
@@ -118,10 +118,82 @@ GetCurrentOrNextStmt(const ExplodedNode *N) {
 // Diagnostic cleanup.
 //===----------------------------------------------------------------------===//
 
+static PathDiagnosticEventPiece *
+eventsDescribeSameCondition(PathDiagnosticEventPiece *X,
+                            PathDiagnosticEventPiece *Y) {
+  // Prefer diagnostics that come from ConditionBRVisitor over
+  // those that came from TrackConstraintBRVisitor.
+  const void *tagPreferred = ConditionBRVisitor::getTag();
+  const void *tagLesser = TrackConstraintBRVisitor::getTag();
+  
+  if (X->getLocation() != Y->getLocation())
+    return 0;
+  
+  if (X->getTag() == tagPreferred && Y->getTag() == tagLesser)
+    return X;
+  
+  if (Y->getTag() == tagPreferred && X->getTag() == tagLesser)
+    return Y;
+  
+  return 0;
+}
+
+/// An optimization pass over PathPieces that removes redundant diagnostics
+/// generated by both ConditionBRVisitor and TrackConstraintBRVisitor.  Both
+/// BugReporterVisitors use different methods to generate diagnostics, with
+/// one capable of emitting diagnostics in some cases but not in others.  This
+/// can lead to redundant diagnostic pieces at the same point in a path.
+static void removeRedundantMsgs(PathPieces &path) {
+  unsigned N = path.size();
+  if (N < 2)
+    return;
+  // NOTE: this loop intentionally is not using an iterator.  Instead, we
+  // are streaming the path and modifying it in place.  This is done by
+  // grabbing the front, processing it, and if we decide to keep it append
+  // it to the end of the path.  The entire path is processed in this way.
+  for (unsigned i = 0; i < N; ++i) {
+    IntrusiveRefCntPtr<PathDiagnosticPiece> piece(path.front());
+    path.pop_front();
+    
+    switch (piece->getKind()) {
+      case clang::ento::PathDiagnosticPiece::Call:
+        removeRedundantMsgs(cast<PathDiagnosticCallPiece>(piece)->path);
+        break;
+      case clang::ento::PathDiagnosticPiece::Macro:
+        removeRedundantMsgs(cast<PathDiagnosticMacroPiece>(piece)->subPieces);
+        break;
+      case clang::ento::PathDiagnosticPiece::ControlFlow:
+        break;
+      case clang::ento::PathDiagnosticPiece::Event: {
+        if (i == N-1)
+          break;
+        
+        if (PathDiagnosticEventPiece *nextEvent =
+            dyn_cast<PathDiagnosticEventPiece>(path.front().getPtr())) {
+          PathDiagnosticEventPiece *event =
+            cast<PathDiagnosticEventPiece>(piece);
+          // Check to see if we should keep one of the two pieces.  If we
+          // come up with a preference, record which piece to keep, and consume
+          // another piece from the path.
+          if (PathDiagnosticEventPiece *pieceToKeep =
+              eventsDescribeSameCondition(event, nextEvent)) {
+            piece = pieceToKeep;
+            path.pop_front();
+            ++i;
+          }
+        }
+        break;
+      }
+    }
+    path.push_back(piece);
+  }
+}
+
 /// Recursively scan through a path and prune out calls and macros pieces
 /// that aren't needed.  Return true if afterwards the path contains
 /// "interesting stuff" which means it should be pruned from the parent path.
-static bool RemoveUneededCalls(PathPieces &pieces) {
+bool BugReporter::RemoveUneededCalls(PathPieces &pieces, BugReport *R,
+                                     PathDiagnosticCallPiece *CallWithLoc) {
   bool containsSomethingInteresting = false;
   const unsigned N = pieces.size();
   
@@ -131,30 +203,49 @@ static bool RemoveUneededCalls(PathPieces &pieces) {
     IntrusiveRefCntPtr<PathDiagnosticPiece> piece(pieces.front());
     pieces.pop_front();
     
+    // Throw away pieces with invalid locations.
+    if (piece->getKind() != PathDiagnosticPiece::Call &&
+        piece->getLocation().asLocation().isInvalid())
+      continue;
+
     switch (piece->getKind()) {
       case PathDiagnosticPiece::Call: {
         PathDiagnosticCallPiece *call = cast<PathDiagnosticCallPiece>(piece);
+        // Check if the location context is interesting.
+        assert(LocationContextMap.count(call));
+        if (R->isInteresting(LocationContextMap[call])) {
+          containsSomethingInteresting = true;
+          break;
+        }
         // Recursively clean out the subclass.  Keep this call around if
         // it contains any informative diagnostics.
-        if (!RemoveUneededCalls(call->path))
+        PathDiagnosticCallPiece *NewCallWithLoc =
+          call->getLocation().asLocation().isValid()
+            ? call : CallWithLoc;
+        
+        if (!RemoveUneededCalls(call->path, R, NewCallWithLoc))
           continue;
+
+        if (NewCallWithLoc == CallWithLoc && CallWithLoc) {
+          call->callEnter = CallWithLoc->callEnter;
+        }
+        
         containsSomethingInteresting = true;
         break;
       }
       case PathDiagnosticPiece::Macro: {
         PathDiagnosticMacroPiece *macro = cast<PathDiagnosticMacroPiece>(piece);
-        if (!RemoveUneededCalls(macro->subPieces))
+        if (!RemoveUneededCalls(macro->subPieces, R))
           continue;
         containsSomethingInteresting = true;
         break;
       }
       case PathDiagnosticPiece::Event: {
         PathDiagnosticEventPiece *event = cast<PathDiagnosticEventPiece>(piece);
+        
         // We never throw away an event, but we do throw it away wholesale
         // as part of a path if we throw the entire path away.
-        if (event->isPrunable())
-          continue;
-        containsSomethingInteresting = true;
+        containsSomethingInteresting |= !event->isPrunable();
         break;
       }
       case PathDiagnosticPiece::ControlFlow:
@@ -382,6 +473,35 @@ PathDiagnosticBuilder::getEnclosingStmtLocation(const Stmt *S) {
 }
 
 //===----------------------------------------------------------------------===//
+// "Visitors only" path diagnostic generation algorithm.
+//===----------------------------------------------------------------------===//
+static bool GenerateVisitorsOnlyPathDiagnostic(PathDiagnostic &PD,
+                                               PathDiagnosticBuilder &PDB,
+                                               const ExplodedNode *N,
+                                      ArrayRef<BugReporterVisitor *> visitors) {
+  // All path generation skips the very first node (the error node).
+  // This is because there is special handling for the end-of-path note.
+  N = N->getFirstPred();
+  if (!N)
+    return true;
+
+  BugReport *R = PDB.getBugReport();
+  while (const ExplodedNode *Pred = N->getFirstPred()) {
+    for (ArrayRef<BugReporterVisitor *>::iterator I = visitors.begin(),
+                                                  E = visitors.end();
+         I != E; ++I) {
+      // Visit all the node pairs, but throw the path pieces away.
+      PathDiagnosticPiece *Piece = (*I)->VisitNode(N, Pred, PDB, *R);
+      delete Piece;
+    }
+
+    N = Pred;
+  }
+
+  return R->isValid();
+}
+
+//===----------------------------------------------------------------------===//
 // "Minimal" path diagnostic generation algorithm.
 //===----------------------------------------------------------------------===//
 typedef std::pair<PathDiagnosticCallPiece*, const ExplodedNode*> StackDiagPair;
@@ -412,7 +532,7 @@ static void updateStackPiecesWithMessage(PathDiagnosticPiece *P,
 
 static void CompactPathDiagnostic(PathPieces &path, const SourceManager& SM);
 
-static void GenerateMinimalPathDiagnostic(PathDiagnostic& PD,
+static bool GenerateMinimalPathDiagnostic(PathDiagnostic& PD,
                                           PathDiagnosticBuilder &PDB,
                                           const ExplodedNode *N,
                                       ArrayRef<BugReporterVisitor *> visitors) {
@@ -430,55 +550,60 @@ static void GenerateMinimalPathDiagnostic(PathDiagnostic& PD,
     NextNode = GetPredecessorNode(N);
 
     ProgramPoint P = N->getLocation();
-    
-    if (const CallExitEnd *CE = dyn_cast<CallExitEnd>(&P)) {
-      PathDiagnosticCallPiece *C =
-        PathDiagnosticCallPiece::construct(N, *CE, SMgr);
-      PD.getActivePath().push_front(C);
-      PD.pushActivePath(&C->path);
-      CallStack.push_back(StackDiagPair(C, N));
-      continue;      
-    }
-    
-    if (const CallEnter *CE = dyn_cast<CallEnter>(&P)) {
-      // Flush all locations, and pop the active path.
-      bool VisitedEntireCall = PD.isWithinCall();
-      PD.popActivePath();
-
-      // Either we just added a bunch of stuff to the top-level path, or
-      // we have a previous CallExitEnd.  If the former, it means that the
-      // path terminated within a function call.  We must then take the
-      // current contents of the active path and place it within
-      // a new PathDiagnosticCallPiece.
-      PathDiagnosticCallPiece *C;
-      if (VisitedEntireCall) {
-        C = cast<PathDiagnosticCallPiece>(PD.getActivePath().front());
-      } else {
-        const Decl *Caller = CE->getLocationContext()->getDecl();
-        C = PathDiagnosticCallPiece::construct(PD.getActivePath(), Caller);
+
+    do {
+      if (const CallExitEnd *CE = dyn_cast<CallExitEnd>(&P)) {
+        PathDiagnosticCallPiece *C =
+            PathDiagnosticCallPiece::construct(N, *CE, SMgr);
+        GRBugReporter& BR = PDB.getBugReporter();
+        BR.addCallPieceLocationContextPair(C, CE->getCalleeContext());
+        PD.getActivePath().push_front(C);
+        PD.pushActivePath(&C->path);
+        CallStack.push_back(StackDiagPair(C, N));
+        break;
       }
 
-      C->setCallee(*CE, SMgr);
-      if (!CallStack.empty()) {
-        assert(CallStack.back().first == C);
-        CallStack.pop_back();
+      if (const CallEnter *CE = dyn_cast<CallEnter>(&P)) {
+        // Flush all locations, and pop the active path.
+        bool VisitedEntireCall = PD.isWithinCall();
+        PD.popActivePath();
+
+        // Either we just added a bunch of stuff to the top-level path, or
+        // we have a previous CallExitEnd.  If the former, it means that the
+        // path terminated within a function call.  We must then take the
+        // current contents of the active path and place it within
+        // a new PathDiagnosticCallPiece.
+        PathDiagnosticCallPiece *C;
+        if (VisitedEntireCall) {
+          C = cast<PathDiagnosticCallPiece>(PD.getActivePath().front());
+        } else {
+          const Decl *Caller = CE->getLocationContext()->getDecl();
+          C = PathDiagnosticCallPiece::construct(PD.getActivePath(), Caller);
+          GRBugReporter& BR = PDB.getBugReporter();
+          BR.addCallPieceLocationContextPair(C, CE->getCalleeContext());
+        }
+
+        C->setCallee(*CE, SMgr);
+        if (!CallStack.empty()) {
+          assert(CallStack.back().first == C);
+          CallStack.pop_back();
+        }
+        break;
       }
-      continue;
-    }
 
-    if (const BlockEdge *BE = dyn_cast<BlockEdge>(&P)) {
-      const CFGBlock *Src = BE->getSrc();
-      const CFGBlock *Dst = BE->getDst();
-      const Stmt *T = Src->getTerminator();
+      if (const BlockEdge *BE = dyn_cast<BlockEdge>(&P)) {
+        const CFGBlock *Src = BE->getSrc();
+        const CFGBlock *Dst = BE->getDst();
+        const Stmt *T = Src->getTerminator();
 
-      if (!T)
-        continue;
+        if (!T)
+          break;
 
-      PathDiagnosticLocation Start =
-        PathDiagnosticLocation::createBegin(T, SMgr,
-                                                N->getLocationContext());
+        PathDiagnosticLocation Start =
+            PathDiagnosticLocation::createBegin(T, SMgr,
+                N->getLocationContext());
 
-      switch (T->getStmtClass()) {
+        switch (T->getStmtClass()) {
         default:
           break;
 
@@ -487,16 +612,16 @@ static void GenerateMinimalPathDiagnostic(PathDiagnostic& PD,
           const Stmt *S = GetNextStmt(N);
 
           if (!S)
-            continue;
+            break;
 
           std::string sbuf;
           llvm::raw_string_ostream os(sbuf);
           const PathDiagnosticLocation &End = PDB.getEnclosingStmtLocation(S);
 
           os << "Control jumps to line "
-          << End.asLocation().getExpansionLineNumber();
-          PD.getActivePath().push_front(new PathDiagnosticControlFlowPiece(Start, End,
-                                                                os.str()));
+              << End.asLocation().getExpansionLineNumber();
+          PD.getActivePath().push_front(new PathDiagnosticControlFlowPiece(
+              Start, End, os.str()));
           break;
         }
 
@@ -509,52 +634,52 @@ static void GenerateMinimalPathDiagnostic(PathDiagnostic& PD,
             PathDiagnosticLocation End(S, SMgr, LC);
 
             switch (S->getStmtClass()) {
-              default:
-                os << "No cases match in the switch statement. "
-                "Control jumps to line "
-                << End.asLocation().getExpansionLineNumber();
-                break;
-              case Stmt::DefaultStmtClass:
-                os << "Control jumps to the 'default' case at line "
-                << End.asLocation().getExpansionLineNumber();
-                break;
-
-              case Stmt::CaseStmtClass: {
-                os << "Control jumps to 'case ";
-                const CaseStmt *Case = cast<CaseStmt>(S);
-                const Expr *LHS = Case->getLHS()->IgnoreParenCasts();
-
-                // Determine if it is an enum.
-                bool GetRawInt = true;
-
-                if (const DeclRefExpr *DR = dyn_cast<DeclRefExpr>(LHS)) {
-                  // FIXME: Maybe this should be an assertion.  Are there cases
-                  // were it is not an EnumConstantDecl?
-                  const EnumConstantDecl *D =
+            default:
+              os << "No cases match in the switch statement. "
+              "Control jumps to line "
+              << End.asLocation().getExpansionLineNumber();
+              break;
+            case Stmt::DefaultStmtClass:
+              os << "Control jumps to the 'default' case at line "
+              << End.asLocation().getExpansionLineNumber();
+              break;
+
+            case Stmt::CaseStmtClass: {
+              os << "Control jumps to 'case ";
+              const CaseStmt *Case = cast<CaseStmt>(S);
+              const Expr *LHS = Case->getLHS()->IgnoreParenCasts();
+
+              // Determine if it is an enum.
+              bool GetRawInt = true;
+
+              if (const DeclRefExpr *DR = dyn_cast<DeclRefExpr>(LHS)) {
+                // FIXME: Maybe this should be an assertion.  Are there cases
+                // were it is not an EnumConstantDecl?
+                const EnumConstantDecl *D =
                     dyn_cast<EnumConstantDecl>(DR->getDecl());
 
-                  if (D) {
-                    GetRawInt = false;
-                    os << *D;
-                  }
+                if (D) {
+                  GetRawInt = false;
+                  os << *D;
                 }
+              }
 
-                if (GetRawInt)
-                  os << LHS->EvaluateKnownConstInt(PDB.getASTContext());
+              if (GetRawInt)
+                os << LHS->EvaluateKnownConstInt(PDB.getASTContext());
 
-                os << ":'  at line "
-                << End.asLocation().getExpansionLineNumber();
-                break;
-              }
+              os << ":'  at line "
+                  << End.asLocation().getExpansionLineNumber();
+              break;
             }
-            PD.getActivePath().push_front(new PathDiagnosticControlFlowPiece(Start, End,
-                                                             os.str()));
+            }
+            PD.getActivePath().push_front(new PathDiagnosticControlFlowPiece(
+                Start, End, os.str()));
           }
           else {
             os << "'Default' branch taken. ";
             const PathDiagnosticLocation &End = PDB.ExecutionContinues(os, N);
-            PD.getActivePath().push_front(new PathDiagnosticControlFlowPiece(Start, End,
-                                                             os.str()));
+            PD.getActivePath().push_front(new PathDiagnosticControlFlowPiece(
+                Start, End, os.str()));
           }
 
           break;
@@ -565,12 +690,12 @@ static void GenerateMinimalPathDiagnostic(PathDiagnostic& PD,
           std::string sbuf;
           llvm::raw_string_ostream os(sbuf);
           PathDiagnosticLocation End = PDB.ExecutionContinues(os, N);
-          PD.getActivePath().push_front(new PathDiagnosticControlFlowPiece(Start, End,
-                                                           os.str()));
+          PD.getActivePath().push_front(new PathDiagnosticControlFlowPiece(
+              Start, End, os.str()));
           break;
         }
 
-          // Determine control-flow for ternary '?'.
+        // Determine control-flow for ternary '?'.
         case Stmt::BinaryConditionalOperatorClass:
         case Stmt::ConditionalOperatorClass: {
           std::string sbuf;
@@ -587,12 +712,12 @@ static void GenerateMinimalPathDiagnostic(PathDiagnostic& PD,
           if (const Stmt *S = End.asStmt())
             End = PDB.getEnclosingStmtLocation(S);
 
-          PD.getActivePath().push_front(new PathDiagnosticControlFlowPiece(Start, End,
-                                                           os.str()));
+          PD.getActivePath().push_front(new PathDiagnosticControlFlowPiece(
+              Start, End, os.str()));
           break;
         }
 
-          // Determine control-flow for short-circuited '&&' and '||'.
+        // Determine control-flow for short-circuited '&&' and '||'.
         case Stmt::BinaryOperatorClass: {
           if (!PDB.supportsLogicalOpControlFlow())
             break;
@@ -609,16 +734,16 @@ static void GenerateMinimalPathDiagnostic(PathDiagnostic& PD,
               os << "false";
               PathDiagnosticLocation End(B->getLHS(), SMgr, LC);
               PathDiagnosticLocation Start =
-                PathDiagnosticLocation::createOperatorLoc(B, SMgr);
-              PD.getActivePath().push_front(new PathDiagnosticControlFlowPiece(Start, End,
-                                                               os.str()));
+                  PathDiagnosticLocation::createOperatorLoc(B, SMgr);
+              PD.getActivePath().push_front(new PathDiagnosticControlFlowPiece(
+                  Start, End, os.str()));
             }
             else {
               os << "true";
               PathDiagnosticLocation Start(B->getLHS(), SMgr, LC);
               PathDiagnosticLocation End = PDB.ExecutionContinues(N);
-              PD.getActivePath().push_front(new PathDiagnosticControlFlowPiece(Start, End,
-                                                               os.str()));
+              PD.getActivePath().push_front(new PathDiagnosticControlFlowPiece(
+                  Start, End, os.str()));
             }
           }
           else {
@@ -629,16 +754,16 @@ static void GenerateMinimalPathDiagnostic(PathDiagnostic& PD,
               os << "false";
               PathDiagnosticLocation Start(B->getLHS(), SMgr, LC);
               PathDiagnosticLocation End = PDB.ExecutionContinues(N);
-              PD.getActivePath().push_front(new PathDiagnosticControlFlowPiece(Start, End,
-                                                               os.str()));
+              PD.getActivePath().push_front(new PathDiagnosticControlFlowPiece(
+                  Start, End, os.str()));
             }
             else {
               os << "true";
               PathDiagnosticLocation End(B->getLHS(), SMgr, LC);
               PathDiagnosticLocation Start =
-                PathDiagnosticLocation::createOperatorLoc(B, SMgr);
-              PD.getActivePath().push_front(new PathDiagnosticControlFlowPiece(Start, End,
-                                                               os.str()));
+                  PathDiagnosticLocation::createOperatorLoc(B, SMgr);
+              PD.getActivePath().push_front(new PathDiagnosticControlFlowPiece(
+                  Start, End, os.str()));
             }
           }
 
@@ -656,8 +781,8 @@ static void GenerateMinimalPathDiagnostic(PathDiagnostic& PD,
             if (const Stmt *S = End.asStmt())
               End = PDB.getEnclosingStmtLocation(S);
 
-            PD.getActivePath().push_front(new PathDiagnosticControlFlowPiece(Start, End,
-                                                             os.str()));
+            PD.getActivePath().push_front(new PathDiagnosticControlFlowPiece(
+                Start, End, os.str()));
           }
           else {
             PathDiagnosticLocation End = PDB.ExecutionContinues(N);
@@ -665,8 +790,8 @@ static void GenerateMinimalPathDiagnostic(PathDiagnostic& PD,
             if (const Stmt *S = End.asStmt())
               End = PDB.getEnclosingStmtLocation(S);
 
-            PD.getActivePath().push_front(new PathDiagnosticControlFlowPiece(Start, End,
-                              "Loop condition is false.  Exiting loop"));
+            PD.getActivePath().push_front(new PathDiagnosticControlFlowPiece(
+                Start, End, "Loop condition is false.  Exiting loop"));
           }
 
           break;
@@ -683,16 +808,16 @@ static void GenerateMinimalPathDiagnostic(PathDiagnostic& PD,
             if (const Stmt *S = End.asStmt())
               End = PDB.getEnclosingStmtLocation(S);
 
-            PD.getActivePath().push_front(new PathDiagnosticControlFlowPiece(Start, End,
-                                                             os.str()));
+            PD.getActivePath().push_front(new PathDiagnosticControlFlowPiece(
+                Start, End, os.str()));
           }
           else {
             PathDiagnosticLocation End = PDB.ExecutionContinues(N);
             if (const Stmt *S = End.asStmt())
               End = PDB.getEnclosingStmtLocation(S);
 
-            PD.getActivePath().push_front(new PathDiagnosticControlFlowPiece(Start, End,
-                            "Loop condition is true.  Entering loop body"));
+            PD.getActivePath().push_front(new PathDiagnosticControlFlowPiece(
+                Start, End, "Loop condition is true.  Entering loop body"));
           }
 
           break;
@@ -705,16 +830,17 @@ static void GenerateMinimalPathDiagnostic(PathDiagnostic& PD,
             End = PDB.getEnclosingStmtLocation(S);
 
           if (*(Src->succ_begin()+1) == Dst)
-            PD.getActivePath().push_front(new PathDiagnosticControlFlowPiece(Start, End,
-                                                        "Taking false branch"));
+            PD.getActivePath().push_front(new PathDiagnosticControlFlowPiece(
+                Start, End, "Taking false branch"));
           else
-            PD.getActivePath().push_front(new PathDiagnosticControlFlowPiece(Start, End,
-                                                         "Taking true branch"));
+            PD.getActivePath().push_front(new PathDiagnosticControlFlowPiece(
+                Start, End, "Taking true branch"));
 
           break;
         }
+        }
       }
-    }
+    } while(0);
 
     if (NextNode) {
       // Add diagnostic pieces from custom visitors.
@@ -730,9 +856,13 @@ static void GenerateMinimalPathDiagnostic(PathDiagnostic& PD,
     }
   }
 
+  if (!PDB.getBugReport()->isValid())
+    return false;
+
   // After constructing the full PathDiagnostic, do a pass over it to compact
   // PathDiagnosticPieces that occur within a macro.
   CompactPathDiagnostic(PD.getMutablePieces(), PDB.getSourceManager());
+  return true;
 }
 
 //===----------------------------------------------------------------------===//
@@ -944,6 +1074,11 @@ void EdgeBuilder::rawAddEdge(PathDiagnosticLocation NewLoc) {
   const PathDiagnosticLocation &NewLocClean = cleanUpLocation(NewLoc);
   const PathDiagnosticLocation &PrevLocClean = cleanUpLocation(PrevLoc);
 
+  if (PrevLocClean.asLocation().isInvalid()) {
+    PrevLoc = NewLoc;
+    return;
+  }
+  
   if (NewLocClean.asLocation() == PrevLocClean.asLocation())
     return;
 
@@ -1133,7 +1268,7 @@ static void reversePropagateInterestingSymbols(BugReport &R,
   }
 }
                                                
-static void GenerateExtensivePathDiagnostic(PathDiagnostic& PD,
+static bool GenerateExtensivePathDiagnostic(PathDiagnostic& PD,
                                             PathDiagnosticBuilder &PDB,
                                             const ExplodedNode *N,
                                       ArrayRef<BugReporterVisitor *> visitors) {
@@ -1166,6 +1301,8 @@ static void GenerateExtensivePathDiagnostic(PathDiagnostic& PD,
         
         PathDiagnosticCallPiece *C =
           PathDiagnosticCallPiece::construct(N, *CE, SM);
+        GRBugReporter& BR = PDB.getBugReporter();
+        BR.addCallPieceLocationContextPair(C, CE->getCalleeContext());
 
         EB.addEdge(C->callReturn, true);
         EB.flushLocations();
@@ -1202,6 +1339,8 @@ static void GenerateExtensivePathDiagnostic(PathDiagnostic& PD,
         } else {
           const Decl *Caller = CE->getLocationContext()->getDecl();
           C = PathDiagnosticCallPiece::construct(PD.getActivePath(), Caller);
+          GRBugReporter& BR = PDB.getBugReporter();
+          BR.addCallPieceLocationContextPair(C, CE->getCalleeContext());
         }
 
         C->setCallee(*CE, SM);
@@ -1234,20 +1373,15 @@ static void GenerateExtensivePathDiagnostic(PathDiagnostic& PD,
           }
         }
        
-        const CFGBlock &Blk = *BE->getSrc();
-        const Stmt *Term = Blk.getTerminator();
-
         // Are we jumping to the head of a loop?  Add a special diagnostic.
-        if (const Stmt *Loop = BE->getDst()->getLoopTarget()) {
+        if (const Stmt *Loop = BE->getSrc()->getLoopTarget()) {
           PathDiagnosticLocation L(Loop, SM, PDB.LC);
           const CompoundStmt *CS = NULL;
 
-          if (!Term) {
-            if (const ForStmt *FS = dyn_cast<ForStmt>(Loop))
-              CS = dyn_cast<CompoundStmt>(FS->getBody());
-            else if (const WhileStmt *WS = dyn_cast<WhileStmt>(Loop))
-              CS = dyn_cast<CompoundStmt>(WS->getBody());
-          }
+          if (const ForStmt *FS = dyn_cast<ForStmt>(Loop))
+            CS = dyn_cast<CompoundStmt>(FS->getBody());
+          else if (const WhileStmt *WS = dyn_cast<WhileStmt>(Loop))
+            CS = dyn_cast<CompoundStmt>(WS->getBody());
 
           PathDiagnosticEventPiece *p =
             new PathDiagnosticEventPiece(L,
@@ -1263,15 +1397,16 @@ static void GenerateExtensivePathDiagnostic(PathDiagnostic& PD,
             EB.addEdge(BL);
           }
         }
-
-        if (Term)
+        
+        if (const Stmt *Term = BE->getSrc()->getTerminator())
           EB.addContext(Term);
 
         break;
       }
 
       if (const BlockEntrance *BE = dyn_cast<BlockEntrance>(&P)) {
-        if (const CFGStmt *S = BE->getFirstElement().getAs<CFGStmt>()) {
+        CFGElement First = BE->getFirstElement();
+        if (const CFGStmt *S = First.getAs<CFGStmt>()) {
           const Stmt *stmt = S->getStmt();
           if (IsControlFlowExpr(stmt)) {
             // Add the proper context for '&&', '||', and '?'.
@@ -1306,6 +1441,8 @@ static void GenerateExtensivePathDiagnostic(PathDiagnostic& PD,
       }
     }
   }
+
+  return PDB.getBugReport()->isValid();
 }
 
 //===----------------------------------------------------------------------===//
@@ -1414,6 +1551,12 @@ void BugReport::markInteresting(SVal V) {
   markInteresting(V.getAsSymbol());
 }
 
+void BugReport::markInteresting(const LocationContext *LC) {
+  if (!LC)
+    return;
+  InterestingLocationContexts.insert(LC);
+}
+
 bool BugReport::isInteresting(SVal V) {
   return isInteresting(V.getAsRegion()) || isInteresting(V.getAsSymbol());
 }
@@ -1438,6 +1581,12 @@ bool BugReport::isInteresting(const MemRegion *R) {
   return false;
 }
 
+bool BugReport::isInteresting(const LocationContext *LC) {
+  if (!LC)
+    return false;
+  return InterestingLocationContexts.count(LC);
+}
+
 void BugReport::lazyInitializeInterestingSets() {
   if (interestingSymbols.empty()) {
     interestingSymbols.push_back(new Symbols());
@@ -1823,17 +1972,27 @@ static void CompactPathDiagnostic(PathPieces &path, const SourceManager& SM) {
     path.push_back(*I);
 }
 
-void GRBugReporter::GeneratePathDiagnostic(PathDiagnostic& PD,
+bool GRBugReporter::generatePathDiagnostic(PathDiagnostic& PD,
                                            PathDiagnosticConsumer &PC,
                                            ArrayRef<BugReport *> &bugReports) {
-
   assert(!bugReports.empty());
+
+  bool HasValid = false;
   SmallVector<const ExplodedNode *, 10> errorNodes;
   for (ArrayRef<BugReport*>::iterator I = bugReports.begin(),
                                       E = bugReports.end(); I != E; ++I) {
+    if ((*I)->isValid()) {
+      HasValid = true;
       errorNodes.push_back((*I)->getErrorNode());
+    } else {
+      errorNodes.push_back(0);
+    }
   }
 
+  // If all the reports have been marked invalid, we're done.
+  if (!HasValid)
+    return false;
+
   // Construct a new graph that contains only a single path from the error
   // node to a root.
   const std::pair<std::pair<ExplodedGraph*, NodeBackMap*>,
@@ -1844,6 +2003,7 @@ void GRBugReporter::GeneratePathDiagnostic(PathDiagnostic& PD,
   assert(GPair.second.second < bugReports.size());
   BugReport *R = bugReports[GPair.second.second];
   assert(R && "No original report found for sliced graph.");
+  assert(R->isValid() && "Report selected from trimmed graph marked invalid.");
 
   OwningPtr<ExplodedGraph> ReportGraph(GPair.first.first);
   OwningPtr<NodeBackMap> BackMap(GPair.first.second);
@@ -1870,36 +2030,53 @@ void GRBugReporter::GeneratePathDiagnostic(PathDiagnostic& PD,
        visitors.push_back((*I)->clone());
 
     // Clear out the active path from any previous work.
-    PD.getActivePath().clear();
+    PD.resetPath();
     originalReportConfigToken = R->getConfigurationChangeToken();
 
     // Generate the very last diagnostic piece - the piece is visible before 
     // the trace is expanded.
-    PathDiagnosticPiece *LastPiece = 0;
-    for (BugReport::visitor_iterator I = visitors.begin(), E = visitors.end();
-         I != E; ++I) {
-      if (PathDiagnosticPiece *Piece = (*I)->getEndPath(PDB, N, *R)) {
-        assert (!LastPiece &&
-                "There can only be one final piece in a diagnostic.");
-        LastPiece = Piece;
+    if (PDB.getGenerationScheme() != PathDiagnosticConsumer::None) {
+      PathDiagnosticPiece *LastPiece = 0;
+      for (BugReport::visitor_iterator I = visitors.begin(), E = visitors.end();
+           I != E; ++I) {
+        if (PathDiagnosticPiece *Piece = (*I)->getEndPath(PDB, N, *R)) {
+          assert (!LastPiece &&
+                  "There can only be one final piece in a diagnostic.");
+          LastPiece = Piece;
+        }
       }
+      if (!LastPiece)
+        LastPiece = BugReporterVisitor::getDefaultEndPath(PDB, N, *R);
+      if (LastPiece)
+        PD.setEndOfPath(LastPiece);
+      else
+        return false;
     }
-    if (!LastPiece)
-      LastPiece = BugReporterVisitor::getDefaultEndPath(PDB, N, *R);
-    if (LastPiece)
-      PD.getActivePath().push_back(LastPiece);
-    else
-      return;
 
     switch (PDB.getGenerationScheme()) {
     case PathDiagnosticConsumer::Extensive:
-      GenerateExtensivePathDiagnostic(PD, PDB, N, visitors);
+      if (!GenerateExtensivePathDiagnostic(PD, PDB, N, visitors)) {
+        assert(!R->isValid() && "Failed on valid report");
+        // Try again. We'll filter out the bad report when we trim the graph.
+        // FIXME: It would be more efficient to use the same intermediate
+        // trimmed graph, and just repeat the shortest-path search.
+        return generatePathDiagnostic(PD, PC, bugReports);
+      }
       break;
     case PathDiagnosticConsumer::Minimal:
-      GenerateMinimalPathDiagnostic(PD, PDB, N, visitors);
+      if (!GenerateMinimalPathDiagnostic(PD, PDB, N, visitors)) {
+        assert(!R->isValid() && "Failed on valid report");
+        // Try again. We'll filter out the bad report when we trim the graph.
+        return generatePathDiagnostic(PD, PC, bugReports);
+      }
       break;
     case PathDiagnosticConsumer::None:
-      llvm_unreachable("PathDiagnosticConsumer::None should never appear here");
+      if (!GenerateVisitorsOnlyPathDiagnostic(PD, PDB, N, visitors)) {
+        assert(!R->isValid() && "Failed on valid report");
+        // Try again. We'll filter out the bad report when we trim the graph.
+        return generatePathDiagnostic(PD, PC, bugReports);
+      }
+      break;
     }
 
     // Clean up the visitors we used.
@@ -1910,18 +2087,26 @@ void GRBugReporter::GeneratePathDiagnostic(PathDiagnostic& PD,
   } while(finalReportConfigToken != originalReportConfigToken);
 
   // Finally, prune the diagnostic path of uninteresting stuff.
-  if (R->shouldPrunePath()) {
-    bool hasSomethingInteresting = RemoveUneededCalls(PD.getMutablePieces());
-    assert(hasSomethingInteresting);
-    (void) hasSomethingInteresting;
+  if (!PD.path.empty()) {
+    // Remove messages that are basically the same.
+    removeRedundantMsgs(PD.getMutablePieces());
+
+    if (R->shouldPrunePath()) {
+      bool hasSomethingInteresting = RemoveUneededCalls(PD.getMutablePieces(),
+                                                        R);
+      assert(hasSomethingInteresting);
+      (void) hasSomethingInteresting;
+    }
   }
+
+  return true;
 }
 
 void BugReporter::Register(BugType *BT) {
   BugTypes = F.add(BugTypes, BT);
 }
 
-void BugReporter::EmitReport(BugReport* R) {
+void BugReporter::emitReport(BugReport* R) {
   // Compute the bug report's hash to determine its equivalence class.
   llvm::FoldingSetNodeID ID;
   R->Profile(ID);
@@ -2078,17 +2263,17 @@ void BugReporter::FlushReport(BugReport *exampleReport,
   OwningPtr<PathDiagnostic>
     D(new PathDiagnostic(exampleReport->getDeclWithIssue(),
                          exampleReport->getBugType().getName(),
-                         PD.useVerboseDescription()
-                         ? exampleReport->getDescription() 
-                         : exampleReport->getShortDescription(),
+                         exampleReport->getDescription(),
+                         exampleReport->getShortDescription(/*Fallback=*/false),
                          BT.getCategory()));
 
   // Generate the full path diagnostic, using the generation scheme
-  // specified by the PathDiagnosticConsumer.
-  if (PD.getGenerationScheme() != PathDiagnosticConsumer::None) {
-    if (!bugReports.empty())
-      GeneratePathDiagnostic(*D.get(), PD, bugReports);
-  }
+  // specified by the PathDiagnosticConsumer. Note that we have to generate
+  // path diagnostics even for consumers which do not support paths, because
+  // the BugReporterVisitors may mark this bug as a false positive.
+  if (!bugReports.empty())
+    if (!generatePathDiagnostic(*D.get(), PD, bugReports))
+      return;
 
   // If the path is empty, generate a single step path with the location
   // of the issue.
@@ -2100,7 +2285,7 @@ void BugReporter::FlushReport(BugReport *exampleReport,
     llvm::tie(Beg, End) = exampleReport->getRanges();
     for ( ; Beg != End; ++Beg)
       piece->addRange(*Beg);
-    D->getActivePath().push_back(piece);
+    D->setEndOfPath(piece);
   }
 
   // Get the meta data.
@@ -2124,7 +2309,7 @@ void BugReporter::EmitBasicReport(const Decl *DeclWithIssue,
   BugReport *R = new BugReport(*BT, str, Loc);
   R->setDeclWithIssue(DeclWithIssue);
   for ( ; NumRanges > 0 ; --NumRanges, ++RBeg) R->addRange(*RBeg);
-  EmitReport(R);
+  emitReport(R);
 }
 
 BugType *BugReporter::getBugTypeForName(StringRef name,
diff --git a/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp b/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
index e729587..328e8a6 100644
--- a/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
+++ b/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
@@ -17,10 +17,13 @@
 #include "clang/AST/ExprObjC.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/PathDiagnostic.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/ExplodedGraph.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h"
 #include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringExtras.h"
 
 using namespace clang;
 using namespace ento;
@@ -29,10 +32,24 @@ using namespace ento;
 // Utility functions.
 //===----------------------------------------------------------------------===//
 
+bool bugreporter::isDeclRefExprToReference(const Expr *E) {
+  if (const DeclRefExpr *DRE = dyn_cast<DeclRefExpr>(E)) {
+    return DRE->getDecl()->getType()->isReferenceType();
+  }
+  return false;
+}
+
 const Stmt *bugreporter::GetDerefExpr(const ExplodedNode *N) {
   // Pattern match for a few useful cases (do something smarter later):
   //   a[0], p->f, *p
-  const Stmt *S = N->getLocationAs<PostStmt>()->getStmt();
+  const PostStmt *Loc = N->getLocationAs<PostStmt>();
+  if (!Loc)
+    return 0;
+
+  const Expr *S = dyn_cast<Expr>(Loc->getStmt());
+  if (!S)
+    return 0;
+  S = S->IgnoreParenCasts();
 
   while (true) {
     if (const BinaryOperator *B = dyn_cast<BinaryOperator>(S)) {
@@ -45,7 +62,12 @@ const Stmt *bugreporter::GetDerefExpr(const ExplodedNode *N) {
         return U->getSubExpr()->IgnoreParenCasts();
     }
     else if (const MemberExpr *ME = dyn_cast<MemberExpr>(S)) {
-      return ME->getBase()->IgnoreParenCasts();
+      if (ME->isArrow() || isDeclRefExprToReference(ME->getBase())) {
+        return ME->getBase()->IgnoreParenCasts();
+      }
+    }
+    else if (const ObjCIvarRefExpr *IvarRef = dyn_cast<ObjCIvarRefExpr>(S)) {
+      return IvarRef->getBase()->IgnoreParenCasts();
     }
     else if (const ArraySubscriptExpr *AE = dyn_cast<ArraySubscriptExpr>(S)) {
       return AE->getBase();
@@ -103,6 +125,228 @@ BugReporterVisitor::getDefaultEndPath(BugReporterContext &BRC,
 }
 
 
+namespace {
+/// Emits an extra note at the return statement of an interesting stack frame.
+///
+/// The returned value is marked as an interesting value, and if it's null,
+/// adds a visitor to track where it became null.
+///
+/// This visitor is intended to be used when another visitor discovers that an
+/// interesting value comes from an inlined function call.
+class ReturnVisitor : public BugReporterVisitorImpl<ReturnVisitor> {
+  const StackFrameContext *StackFrame;
+  enum {
+    Initial,
+    MaybeSuppress,
+    Satisfied
+  } Mode;
+
+public:
+  ReturnVisitor(const StackFrameContext *Frame)
+    : StackFrame(Frame), Mode(Initial) {}
+
+  static void *getTag() {
+    static int Tag = 0;
+    return static_cast<void *>(&Tag);
+  }
+
+  virtual void Profile(llvm::FoldingSetNodeID &ID) const {
+    ID.AddPointer(ReturnVisitor::getTag());
+    ID.AddPointer(StackFrame);
+  }
+
+  /// Adds a ReturnVisitor if the given statement represents a call that was
+  /// inlined.
+  ///
+  /// This will search back through the ExplodedGraph, starting from the given
+  /// node, looking for when the given statement was processed. If it turns out
+  /// the statement is a call that was inlined, we add the visitor to the
+  /// bug report, so it can print a note later.
+  static void addVisitorIfNecessary(const ExplodedNode *Node, const Stmt *S,
+                                    BugReport &BR) {
+    if (!CallEvent::isCallStmt(S))
+      return;
+    
+    // First, find when we processed the statement.
+    do {
+      if (const CallExitEnd *CEE = Node->getLocationAs<CallExitEnd>())
+        if (CEE->getCalleeContext()->getCallSite() == S)
+          break;
+      if (const StmtPoint *SP = Node->getLocationAs<StmtPoint>())
+        if (SP->getStmt() == S)
+          break;
+
+      Node = Node->getFirstPred();
+    } while (Node);
+
+    // Next, step over any post-statement checks.
+    while (Node && isa<PostStmt>(Node->getLocation()))
+      Node = Node->getFirstPred();
+
+    // Finally, see if we inlined the call.
+    if (Node) {
+      if (const CallExitEnd *CEE = Node->getLocationAs<CallExitEnd>()) {
+        const StackFrameContext *CalleeContext = CEE->getCalleeContext();
+        if (CalleeContext->getCallSite() == S) {
+          BR.markInteresting(CalleeContext);
+          BR.addVisitor(new ReturnVisitor(CalleeContext));
+        }
+      }
+    }
+  }
+
+  /// Returns true if any counter-suppression heuristics are enabled for
+  /// ReturnVisitor.
+  static bool hasCounterSuppression(AnalyzerOptions &Options) {
+    return Options.shouldAvoidSuppressingNullArgumentPaths();
+  }
+
+  PathDiagnosticPiece *visitNodeInitial(const ExplodedNode *N,
+                                        const ExplodedNode *PrevN,
+                                        BugReporterContext &BRC,
+                                        BugReport &BR) {
+    // Only print a message at the interesting return statement.
+    if (N->getLocationContext() != StackFrame)
+      return 0;
+
+    const StmtPoint *SP = N->getLocationAs<StmtPoint>();
+    if (!SP)
+      return 0;
+
+    const ReturnStmt *Ret = dyn_cast<ReturnStmt>(SP->getStmt());
+    if (!Ret)
+      return 0;
+
+    // Okay, we're at the right return statement, but do we have the return
+    // value available?
+    ProgramStateRef State = N->getState();
+    SVal V = State->getSVal(Ret, StackFrame);
+    if (V.isUnknownOrUndef())
+      return 0;
+
+    // Don't print any more notes after this one.
+    Mode = Satisfied;
+
+    const Expr *RetE = Ret->getRetValue();
+    assert(RetE && "Tracking a return value for a void function");
+    RetE = RetE->IgnoreParenCasts();
+
+    // If we can't prove the return value is 0, just mark it interesting, and
+    // make sure to track it into any further inner functions.
+    if (State->assume(cast<DefinedSVal>(V), true)) {
+      BR.markInteresting(V);
+      ReturnVisitor::addVisitorIfNecessary(N, RetE, BR);
+      return 0;
+    }
+      
+    // If we're returning 0, we should track where that 0 came from.
+    bugreporter::trackNullOrUndefValue(N, RetE, BR);
+
+    // Build an appropriate message based on the return value.
+    SmallString<64> Msg;
+    llvm::raw_svector_ostream Out(Msg);
+
+    if (isa<Loc>(V)) {
+      // If we are pruning null-return paths as unlikely error paths, mark the
+      // report invalid. We still want to emit a path note, however, in case
+      // the report is resurrected as valid later on.
+      ExprEngine &Eng = BRC.getBugReporter().getEngine();
+      AnalyzerOptions &Options = Eng.getAnalysisManager().options;
+      if (Options.shouldPruneNullReturnPaths()) {
+        if (hasCounterSuppression(Options))
+          Mode = MaybeSuppress;
+        else
+          BR.markInvalid(ReturnVisitor::getTag(), StackFrame);
+      }
+
+      if (RetE->getType()->isObjCObjectPointerType())
+        Out << "Returning nil";
+      else
+        Out << "Returning null pointer";
+    } else {
+      Out << "Returning zero";
+    }
+
+    // FIXME: We should have a more generalized location printing mechanism.
+    if (const DeclRefExpr *DR = dyn_cast<DeclRefExpr>(RetE))
+      if (const DeclaratorDecl *DD = dyn_cast<DeclaratorDecl>(DR->getDecl()))
+        Out << " (loaded from '" << *DD << "')";
+
+    PathDiagnosticLocation L(Ret, BRC.getSourceManager(), StackFrame);
+    return new PathDiagnosticEventPiece(L, Out.str());
+  }
+
+  PathDiagnosticPiece *visitNodeMaybeSuppress(const ExplodedNode *N,
+                                              const ExplodedNode *PrevN,
+                                              BugReporterContext &BRC,
+                                              BugReport &BR) {
+    // Are we at the entry node for this call?
+    const CallEnter *CE = N->getLocationAs<CallEnter>();
+    if (!CE)
+      return 0;
+
+    if (CE->getCalleeContext() != StackFrame)
+      return 0;
+
+    Mode = Satisfied;
+
+    ExprEngine &Eng = BRC.getBugReporter().getEngine();
+    AnalyzerOptions &Options = Eng.getAnalysisManager().options;
+    if (Options.shouldAvoidSuppressingNullArgumentPaths()) {
+      // Don't automatically suppress a report if one of the arguments is
+      // known to be a null pointer. Instead, start tracking /that/ null
+      // value back to its origin.
+      ProgramStateManager &StateMgr = BRC.getStateManager();
+      CallEventManager &CallMgr = StateMgr.getCallEventManager();
+
+      ProgramStateRef State = N->getState();
+      CallEventRef<> Call = CallMgr.getCaller(StackFrame, State);
+      for (unsigned I = 0, E = Call->getNumArgs(); I != E; ++I) {
+        SVal ArgV = Call->getArgSVal(I);
+        if (!isa<Loc>(ArgV))
+          continue;
+
+        const Expr *ArgE = Call->getArgExpr(I);
+        if (!ArgE)
+          continue;
+
+        // Is it possible for this argument to be non-null?
+        if (State->assume(cast<Loc>(ArgV), true))
+          continue;
+
+        if (bugreporter::trackNullOrUndefValue(N, ArgE, BR, /*IsArg=*/true))
+          return 0;
+
+        // If we /can't/ track the null pointer, we should err on the side of
+        // false negatives, and continue towards marking this report invalid.
+        // (We will still look at the other arguments, though.)
+      }
+    }
+
+    // There is no reason not to suppress this report; go ahead and do it.
+    BR.markInvalid(ReturnVisitor::getTag(), StackFrame);
+    return 0;
+  }
+
+  PathDiagnosticPiece *VisitNode(const ExplodedNode *N,
+                                 const ExplodedNode *PrevN,
+                                 BugReporterContext &BRC,
+                                 BugReport &BR) {
+    switch (Mode) {
+    case Initial:
+      return visitNodeInitial(N, PrevN, BRC, BR);
+    case MaybeSuppress:
+      return visitNodeMaybeSuppress(N, PrevN, BRC, BR);
+    case Satisfied:
+      return 0;
+    }
+
+    llvm_unreachable("Invalid visit mode!");
+  }
+};
+} // end anonymous namespace
+
+
 void FindLastStoreBRVisitor ::Profile(llvm::FoldingSetNodeID &ID) const {
   static int tag = 0;
   ID.AddPointer(&tag);
@@ -110,59 +354,90 @@ void FindLastStoreBRVisitor ::Profile(llvm::FoldingSetNodeID &ID) const {
   ID.Add(V);
 }
 
-PathDiagnosticPiece *FindLastStoreBRVisitor::VisitNode(const ExplodedNode *N,
-                                                     const ExplodedNode *PrevN,
-                                                     BugReporterContext &BRC,
-                                                     BugReport &BR) {
+PathDiagnosticPiece *FindLastStoreBRVisitor::VisitNode(const ExplodedNode *Succ,
+                                                       const ExplodedNode *Pred,
+                                                       BugReporterContext &BRC,
+                                                       BugReport &BR) {
 
   if (satisfied)
     return NULL;
 
-  if (!StoreSite) {
-    // Make sure the region is actually bound to value V here.
-    // This is necessary because the region may not actually be live at the
-    // report's error node.
-    if (N->getState()->getSVal(R) != V)
-      return NULL;
-
-    const ExplodedNode *Node = N, *Last = N;
-
-    // Now look for the store of V.
-    for ( ; Node ; Node = Node->getFirstPred()) {
-      if (const VarRegion *VR = dyn_cast<VarRegion>(R)) {
-        if (const PostStmt *P = Node->getLocationAs<PostStmt>())
-          if (const DeclStmt *DS = P->getStmtAs<DeclStmt>())
-            if (DS->getSingleDecl() == VR->getDecl()) {
-              // Record the last seen initialization point.
-              Last = Node;
-              break;
-            }
+  const ExplodedNode *StoreSite = 0;
+  const Expr *InitE = 0;
+  bool IsParam = false;
+
+  // First see if we reached the declaration of the region.
+  if (const VarRegion *VR = dyn_cast<VarRegion>(R)) {
+    if (const PostStmt *P = Pred->getLocationAs<PostStmt>()) {
+      if (const DeclStmt *DS = P->getStmtAs<DeclStmt>()) {
+        if (DS->getSingleDecl() == VR->getDecl()) {
+          StoreSite = Pred;
+          InitE = VR->getDecl()->getInit();
+        }
       }
-
-      // Does the region still bind to value V?  If not, we are done
-      // looking for store sites.
-      if (Node->getState()->getSVal(R) != V)
-        break;
-
-      Last = Node;
     }
+  }
 
-    if (!Node) {
-      satisfied = true;
+  // Otherwise, check that Succ has this binding and Pred does not, i.e. this is
+  // where the binding first occurred.
+  if (!StoreSite) {
+    if (Succ->getState()->getSVal(R) != V)
+      return NULL;
+    if (Pred->getState()->getSVal(R) == V)
       return NULL;
-    }
 
-    StoreSite = Last;
+    StoreSite = Succ;
+
+    // If this is an assignment expression, we can track the value
+    // being assigned.
+    if (const PostStmt *P = Succ->getLocationAs<PostStmt>())
+      if (const BinaryOperator *BO = P->getStmtAs<BinaryOperator>())
+        if (BO->isAssignmentOp())
+          InitE = BO->getRHS();
+
+    // If this is a call entry, the variable should be a parameter.
+    // FIXME: Handle CXXThisRegion as well. (This is not a priority because
+    // 'this' should never be NULL, but this visitor isn't just for NULL and
+    // UndefinedVal.)
+    if (const CallEnter *CE = Succ->getLocationAs<CallEnter>()) {
+      const VarRegion *VR = cast<VarRegion>(R);
+      const ParmVarDecl *Param = cast<ParmVarDecl>(VR->getDecl());
+      
+      ProgramStateManager &StateMgr = BRC.getStateManager();
+      CallEventManager &CallMgr = StateMgr.getCallEventManager();
+
+      CallEventRef<> Call = CallMgr.getCaller(CE->getCalleeContext(),
+                                              Succ->getState());
+      InitE = Call->getArgExpr(Param->getFunctionScopeIndex());
+      IsParam = true;
+    }
   }
 
-  if (StoreSite != N)
+  if (!StoreSite)
     return NULL;
-
   satisfied = true;
+
+  // If we have an expression that provided the value, try to track where it
+  // came from.
+  if (InitE) {
+    if (V.isUndef() || isa<loc::ConcreteInt>(V)) {
+      if (!IsParam)
+        InitE = InitE->IgnoreParenCasts();
+      bugreporter::trackNullOrUndefValue(StoreSite, InitE, BR, IsParam);
+    } else {
+      ReturnVisitor::addVisitorIfNecessary(StoreSite, InitE->IgnoreParenCasts(),
+                                           BR);
+    }
+  }
+
+  if (!R->canPrintPretty())
+    return 0;
+
+  // Okay, we've found the binding. Emit an appropriate message.
   SmallString<256> sbuf;
   llvm::raw_svector_ostream os(sbuf);
 
-  if (const PostStmt *PS = N->getLocationAs<PostStmt>()) {
+  if (const PostStmt *PS = StoreSite->getLocationAs<PostStmt>()) {
     if (const DeclStmt *DS = PS->getStmtAs<DeclStmt>()) {
 
       if (const VarRegion *VR = dyn_cast<VarRegion>(R)) {
@@ -201,6 +476,30 @@ PathDiagnosticPiece *FindLastStoreBRVisitor::VisitNode(const ExplodedNode *N,
         os << "initialized here";
       }
     }
+  } else if (isa<CallEnter>(StoreSite->getLocation())) {
+    const ParmVarDecl *Param = cast<ParmVarDecl>(cast<VarRegion>(R)->getDecl());
+
+    os << "Passing ";
+
+    if (isa<loc::ConcreteInt>(V)) {
+      if (Param->getType()->isObjCObjectPointerType())
+        os << "nil object reference";
+      else
+        os << "null pointer value";
+    } else if (V.isUndef()) {
+      os << "uninitialized value";
+    } else if (isa<nonloc::ConcreteInt>(V)) {
+      os << "the value " << cast<nonloc::ConcreteInt>(V).getValue();
+    } else {
+      os << "value";
+    }
+
+    // Printed parameter indexes are 1-based, not 0-based.
+    unsigned Idx = Param->getFunctionScopeIndex() + 1;
+    os << " via " << Idx << llvm::getOrdinalSuffix(Idx) << " parameter '";
+
+    R->printPretty(os);
+    os << '\'';
   }
 
   if (os.str().empty()) {
@@ -228,17 +527,19 @@ PathDiagnosticPiece *FindLastStoreBRVisitor::VisitNode(const ExplodedNode *N,
     else
       os << "Value assigned to ";
 
-    if (const VarRegion *VR = dyn_cast<VarRegion>(R)) {
-      os << '\'' << *VR->getDecl() << '\'';
-    }
-    else
-      return NULL;
+    os << '\'';
+    R->printPretty(os);
+    os << '\'';
   }
 
   // Construct a new PathDiagnosticPiece.
-  ProgramPoint P = N->getLocation();
-  PathDiagnosticLocation L =
-    PathDiagnosticLocation::create(P, BRC.getSourceManager());
+  ProgramPoint P = StoreSite->getLocation();
+  PathDiagnosticLocation L;
+  if (isa<CallEnter>(P))
+    L = PathDiagnosticLocation(InitE, BRC.getSourceManager(),
+                               P.getLocationContext());
+  else
+    L = PathDiagnosticLocation::create(P, BRC.getSourceManager());
   if (!L.isValid())
     return NULL;
   return new PathDiagnosticEventPiece(L, os.str());
@@ -251,6 +552,12 @@ void TrackConstraintBRVisitor::Profile(llvm::FoldingSetNodeID &ID) const {
   ID.Add(Constraint);
 }
 
+/// Return the tag associated with this visitor.  This tag will be used
+/// to make all PathDiagnosticPieces created by this visitor.
+const char *TrackConstraintBRVisitor::getTag() {
+  return "TrackConstraintBRVisitor";
+}
+
 PathDiagnosticPiece *
 TrackConstraintBRVisitor::VisitNode(const ExplodedNode *N,
                                     const ExplodedNode *PrevN,
@@ -290,62 +597,97 @@ TrackConstraintBRVisitor::VisitNode(const ExplodedNode *N,
       PathDiagnosticLocation::create(P, BRC.getSourceManager());
     if (!L.isValid())
       return NULL;
-    return new PathDiagnosticEventPiece(L, os.str());
+    
+    PathDiagnosticEventPiece *X = new PathDiagnosticEventPiece(L, os.str());
+    X->setTag(getTag());
+    return X;
   }
 
   return NULL;
 }
 
-void bugreporter::addTrackNullOrUndefValueVisitor(const ExplodedNode *N,
-                                                  const Stmt *S,
-                                                  BugReport *report) {
+bool bugreporter::trackNullOrUndefValue(const ExplodedNode *N, const Stmt *S,
+                                        BugReport &report, bool IsArg) {
   if (!S || !N)
-    return;
+    return false;
 
-  ProgramStateManager &StateMgr = N->getState()->getStateManager();
+  if (const OpaqueValueExpr *OVE = dyn_cast<OpaqueValueExpr>(S))
+    S = OVE->getSourceExpr();
+
+  if (IsArg) {
+    assert(isa<CallEnter>(N->getLocation()) && "Tracking arg but not at call");
+  } else {
+    // Walk through nodes until we get one that matches the statement exactly.
+    do {
+      const ProgramPoint &pp = N->getLocation();
+      if (const PostStmt *ps = dyn_cast<PostStmt>(&pp)) {
+        if (ps->getStmt() == S)
+          break;
+      } else if (const CallExitEnd *CEE = dyn_cast<CallExitEnd>(&pp)) {
+        if (CEE->getCalleeContext()->getCallSite() == S)
+          break;
+      }
+      N = N->getFirstPred();
+    } while (N);
 
-  // Walk through nodes until we get one that matches the statement
-  // exactly.
-  while (N) {
-    const ProgramPoint &pp = N->getLocation();
-    if (const PostStmt *ps = dyn_cast<PostStmt>(&pp)) {
-      if (ps->getStmt() == S)
-        break;
-    }
-    N = N->getFirstPred();
+    if (!N)
+      return false;
   }
-
-  if (!N)
-    return;
   
   ProgramStateRef state = N->getState();
 
-  // Walk through lvalue-to-rvalue conversions.
-  const Expr *Ex = dyn_cast<Expr>(S);
-  if (Ex) {
+  // See if the expression we're interested refers to a variable. 
+  // If so, we can track both its contents and constraints on its value.
+  if (const Expr *Ex = dyn_cast<Expr>(S)) {
+    // Strip off parens and casts. Note that this will never have issues with
+    // C++ user-defined implicit conversions, because those have a constructor
+    // or function call inside.
     Ex = Ex->IgnoreParenCasts();
     if (const DeclRefExpr *DR = dyn_cast<DeclRefExpr>(Ex)) {
+      // FIXME: Right now we only track VarDecls because it's non-trivial to
+      // get a MemRegion for any other DeclRefExprs. <rdar://problem/12114812>
       if (const VarDecl *VD = dyn_cast<VarDecl>(DR->getDecl())) {
-        const VarRegion *R =
-          StateMgr.getRegionManager().getVarRegion(VD, N->getLocationContext());
+        ProgramStateManager &StateMgr = state->getStateManager();
+        MemRegionManager &MRMgr = StateMgr.getRegionManager();
+        const VarRegion *R = MRMgr.getVarRegion(VD, N->getLocationContext());
 
-        // What did we load?
+        // Mark both the variable region and its contents as interesting.
         SVal V = state->getRawSVal(loc::MemRegionVal(R));
-        report->markInteresting(R);
-        report->markInteresting(V);
 
+        // If the value matches the default for the variable region, that
+        // might mean that it's been cleared out of the state. Fall back to
+        // the full argument expression (with casts and such intact).
+        if (IsArg) {
+          bool UseArgValue = V.isUnknownOrUndef() || V.isZeroConstant();
+          if (!UseArgValue) {
+            const SymbolRegionValue *SRV =
+              dyn_cast_or_null<SymbolRegionValue>(V.getAsLocSymbol());
+            if (SRV)
+              UseArgValue = (SRV->getRegion() == R);
+          }
+          if (UseArgValue)
+            V = state->getSValAsScalarOrLoc(S, N->getLocationContext());
+        }
+
+        report.markInteresting(R);
+        report.markInteresting(V);
+        report.addVisitor(new UndefOrNullArgVisitor(R));
+
+        // If the contents are symbolic, find out when they became null.
         if (V.getAsLocSymbol()) {
           BugReporterVisitor *ConstraintTracker
-            = new TrackConstraintBRVisitor(cast<loc::MemRegionVal>(V), false);
-          report->addVisitor(ConstraintTracker);
+            = new TrackConstraintBRVisitor(cast<DefinedSVal>(V), false);
+          report.addVisitor(ConstraintTracker);
         }
 
-        report->addVisitor(new FindLastStoreBRVisitor(V, R));
-        return;
+        report.addVisitor(new FindLastStoreBRVisitor(V, R));
+        return true;
       }
     }
   }
 
+  // If the expression does NOT refer to a variable, we can still track
+  // constraints on its contents.
   SVal V = state->getSValAsScalarOrLoc(S, N->getLocationContext());
 
   // Uncomment this to find cases where we aren't properly getting the
@@ -354,17 +696,27 @@ void bugreporter::addTrackNullOrUndefValueVisitor(const ExplodedNode *N,
 
   // Is it a symbolic value?
   if (loc::MemRegionVal *L = dyn_cast<loc::MemRegionVal>(&V)) {
-    const SubRegion *R = cast<SubRegion>(L->getRegion());
-    while (R && !isa<SymbolicRegion>(R)) {
-      R = dyn_cast<SubRegion>(R->getSuperRegion());
-    }
+    // At this point we are dealing with the region's LValue.
+    // However, if the rvalue is a symbolic region, we should track it as well.
+    SVal RVal = state->getSVal(L->getRegion());
+    const MemRegion *RegionRVal = RVal.getAsRegion();
+    report.addVisitor(new UndefOrNullArgVisitor(L->getRegion()));
+
 
-    if (R) {
-      report->markInteresting(R);
-      report->addVisitor(new TrackConstraintBRVisitor(loc::MemRegionVal(R),
-                                                      false));
+    if (RegionRVal && isa<SymbolicRegion>(RegionRVal)) {
+      report.markInteresting(RegionRVal);
+      report.addVisitor(new TrackConstraintBRVisitor(
+        loc::MemRegionVal(RegionRVal), false));
     }
+  } else {
+    // Otherwise, if the value came from an inlined function call,
+    // we should at least make sure that function isn't pruned in our output.
+    if (const Expr *E = dyn_cast<Expr>(S))
+      S = E->IgnoreParenCasts();
+    ReturnVisitor::addVisitorIfNecessary(N, S, report);
   }
+
+  return true;
 }
 
 BugReporterVisitor *
@@ -406,7 +758,7 @@ PathDiagnosticPiece *NilReceiverBRVisitor::VisitNode(const ExplodedNode *N,
   // The receiver was nil, and hence the method was skipped.
   // Register a BugReporterVisitor to issue a message telling us how
   // the receiver was null.
-  bugreporter::addTrackNullOrUndefValueVisitor(N, Receiver, &BR);
+  bugreporter::trackNullOrUndefValue(N, Receiver, BR);
   // Issue a message saying that the method was skipped.
   PathDiagnosticLocation L(Receiver, BRC.getSourceManager(),
                                      N->getLocationContext());
@@ -452,14 +804,23 @@ void FindLastStoreBRVisitor::registerStatementVarDecls(BugReport &BR,
 //===----------------------------------------------------------------------===//
 // Visitor that tries to report interesting diagnostics from conditions.
 //===----------------------------------------------------------------------===//
+
+/// Return the tag associated with this visitor.  This tag will be used
+/// to make all PathDiagnosticPieces created by this visitor.
+const char *ConditionBRVisitor::getTag() {
+  return "ConditionBRVisitor";
+}
+
 PathDiagnosticPiece *ConditionBRVisitor::VisitNode(const ExplodedNode *N,
                                                    const ExplodedNode *Prev,
                                                    BugReporterContext &BRC,
                                                    BugReport &BR) {
   PathDiagnosticPiece *piece = VisitNodeImpl(N, Prev, BRC, BR);
-  if (PathDiagnosticEventPiece *ev =
-      dyn_cast_or_null<PathDiagnosticEventPiece>(piece))
-    ev->setPrunable(true, /* override */ false);
+  if (piece) {
+    piece->setTag(getTag());
+    if (PathDiagnosticEventPiece *ev=dyn_cast<PathDiagnosticEventPiece>(piece))
+      ev->setPrunable(true, /* override */ false);
+  }
   return piece;
 }
 
@@ -468,8 +829,7 @@ PathDiagnosticPiece *ConditionBRVisitor::VisitNodeImpl(const ExplodedNode *N,
                                                        BugReporterContext &BRC,
                                                        BugReport &BR) {
   
-  const ProgramPoint &progPoint = N->getLocation();
-
+  ProgramPoint progPoint = N->getLocation();
   ProgramStateRef CurrentState = N->getState();
   ProgramStateRef PrevState = Prev->getState();
   
@@ -494,7 +854,7 @@ PathDiagnosticPiece *ConditionBRVisitor::VisitNodeImpl(const ExplodedNode *N,
     // violation.
     const std::pair<const ProgramPointTag *, const ProgramPointTag *> &tags =      
       cast<GRBugReporter>(BRC.getBugReporter()).
-        getEngine().getEagerlyAssumeTags();
+        getEngine().geteagerlyAssumeBinOpBifurcationTags();
 
     const ProgramPointTag *tag = PS->getTag();
     if (tag == tags.first)
@@ -533,8 +893,7 @@ ConditionBRVisitor::VisitTerminator(const Stmt *Term,
   assert(Cond);
   assert(srcBlk->succ_size() == 2);
   const bool tookTrue = *(srcBlk->succ_begin()) == dstBlk;
-  return VisitTrueTest(Cond->IgnoreParenNoopCasts(BRC.getASTContext()),
-                       tookTrue, BRC, R, N);
+  return VisitTrueTest(Cond, tookTrue, BRC, R, N);
 }
 
 PathDiagnosticPiece *
@@ -547,7 +906,7 @@ ConditionBRVisitor::VisitTrueTest(const Expr *Cond,
   const Expr *Ex = Cond;
   
   while (true) {
-    Ex = Ex->IgnoreParens();
+    Ex = Ex->IgnoreParenCasts();
     switch (Ex->getStmtClass()) {
       default:
         return 0;
@@ -561,7 +920,7 @@ ConditionBRVisitor::VisitTrueTest(const Expr *Cond,
         const UnaryOperator *UO = cast<UnaryOperator>(Ex);
         if (UO->getOpcode() == UO_LNot) {
           tookTrue = !tookTrue;
-          Ex = UO->getSubExpr()->IgnoreParenNoopCasts(BRC.getASTContext());
+          Ex = UO->getSubExpr();
           continue;
         }
         return 0;
@@ -802,3 +1161,54 @@ ConditionBRVisitor::VisitTrueTest(const Expr *Cond,
   return event;
 }
 
+PathDiagnosticPiece *
+UndefOrNullArgVisitor::VisitNode(const ExplodedNode *N,
+                                  const ExplodedNode *PrevN,
+                                  BugReporterContext &BRC,
+                                  BugReport &BR) {
+
+  ProgramStateRef State = N->getState();
+  ProgramPoint ProgLoc = N->getLocation();
+
+  // We are only interested in visiting CallEnter nodes.
+  CallEnter *CEnter = dyn_cast<CallEnter>(&ProgLoc);
+  if (!CEnter)
+    return 0;
+
+  // Check if one of the arguments is the region the visitor is tracking.
+  CallEventManager &CEMgr = BRC.getStateManager().getCallEventManager();
+  CallEventRef<> Call = CEMgr.getCaller(CEnter->getCalleeContext(), State);
+  unsigned Idx = 0;
+  for (CallEvent::param_iterator I = Call->param_begin(),
+                                 E = Call->param_end(); I != E; ++I, ++Idx) {
+    const MemRegion *ArgReg = Call->getArgSVal(Idx).getAsRegion();
+
+    // Are we tracking the argument or its subregion?
+    if ( !ArgReg || (ArgReg != R && !R->isSubRegionOf(ArgReg->StripCasts())))
+      continue;
+
+    // Check the function parameter type.
+    const ParmVarDecl *ParamDecl = *I;
+    assert(ParamDecl && "Formal parameter has no decl?");
+    QualType T = ParamDecl->getType();
+
+    if (!(T->isAnyPointerType() || T->isReferenceType())) {
+      // Function can only change the value passed in by address.
+      continue;
+    }
+    
+    // If it is a const pointer value, the function does not intend to
+    // change the value.
+    if (T->getPointeeType().isConstQualified())
+      continue;
+
+    // Mark the call site (LocationContext) as interesting if the value of the 
+    // argument is undefined or '0'/'NULL'.
+    SVal BoundVal = State->getSVal(R);
+    if (BoundVal.isUndef() || BoundVal.isZeroConstant()) {
+      BR.markInteresting(CEnter->getCalleeContext());
+      return 0;
+    }
+  }
+  return 0;
+}
diff --git a/lib/StaticAnalyzer/Core/CMakeLists.txt b/lib/StaticAnalyzer/Core/CMakeLists.txt
index b16b233d..91f15b3 100644
--- a/lib/StaticAnalyzer/Core/CMakeLists.txt
+++ b/lib/StaticAnalyzer/Core/CMakeLists.txt
@@ -1,9 +1,9 @@
 set(LLVM_LINK_COMPONENTS support)
 
 add_clang_library(clangStaticAnalyzerCore
-  AnalysisManager.cpp
   APSIntType.cpp
-  BasicConstraintManager.cpp
+  AnalysisManager.cpp
+  AnalyzerOptions.cpp
   BasicValueFactory.cpp
   BlockCounter.cpp
   BugReporter.cpp
@@ -14,6 +14,7 @@ add_clang_library(clangStaticAnalyzerCore
   CheckerHelpers.cpp
   CheckerManager.cpp
   CheckerRegistry.cpp
+  ConstraintManager.cpp
   CoreEngine.cpp
   Environment.cpp
   ExplodedGraph.cpp
@@ -54,5 +55,5 @@ target_link_libraries(clangStaticAnalyzerCore
   clangLex
   clangAST
   clangFrontend
-  clangRewrite
+  clangRewriteCore
   )
diff --git a/lib/StaticAnalyzer/Core/CallEvent.cpp b/lib/StaticAnalyzer/Core/CallEvent.cpp
index 5345bd5..c5cb317 100644
--- a/lib/StaticAnalyzer/Core/CallEvent.cpp
+++ b/lib/StaticAnalyzer/Core/CallEvent.cpp
@@ -14,6 +14,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h"
+#include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h"
 #include "clang/Analysis/ProgramPoint.h"
 #include "clang/AST/ParentMap.h"
 #include "llvm/ADT/SmallSet.h"
@@ -23,10 +24,26 @@ using namespace clang;
 using namespace ento;
 
 QualType CallEvent::getResultType() const {
-  QualType ResultTy = getDeclaredResultType();
+  const Expr *E = getOriginExpr();
+  assert(E && "Calls without origin expressions do not have results");
+  QualType ResultTy = E->getType();
 
-  if (ResultTy.isNull())
-    ResultTy = getOriginExpr()->getType();
+  ASTContext &Ctx = getState()->getStateManager().getContext();
+
+  // A function that returns a reference to 'int' will have a result type
+  // of simply 'int'. Check the origin expr's value kind to recover the
+  // proper type.
+  switch (E->getValueKind()) {
+  case VK_LValue:
+    ResultTy = Ctx.getLValueReferenceType(ResultTy);
+    break;
+  case VK_XValue:
+    ResultTy = Ctx.getRValueReferenceType(ResultTy);
+    break;
+  case VK_RValue:
+    // No adjustment is necessary.
+    break;
+  }
 
   return ResultTy;
 }
@@ -45,7 +62,7 @@ static bool isCallbackArg(SVal V, QualType T) {
   // Check if a callback is passed inside a struct (for both, struct passed by
   // reference and by value). Dig just one level into the struct for now.
 
-  if (isa<PointerType>(T) || isa<ReferenceType>(T))
+  if (T->isAnyPointerType() || T->isReferenceType())
     T = T->getPointeeType();
 
   if (const RecordType *RT = T->getAsStructureType()) {
@@ -83,6 +100,14 @@ bool CallEvent::hasNonZeroCallbackArg() const {
   return false;
 }
 
+bool CallEvent::isGlobalCFunction(StringRef FunctionName) const {
+  const FunctionDecl *FD = dyn_cast_or_null<FunctionDecl>(getDecl());
+  if (!FD)
+    return false;
+
+  return CheckerContext::isCLibraryFunction(FD, FunctionName);
+}
+
 /// \brief Returns true if a type is a pointer-to-const or reference-to-const
 /// with no further indirection.
 static bool isPointerToConst(QualType Ty) {
@@ -207,6 +232,13 @@ SourceRange CallEvent::getArgSourceRange(unsigned Index) const {
   return ArgE->getSourceRange();
 }
 
+SVal CallEvent::getReturnValue() const {
+  const Expr *E = getOriginExpr();
+  if (!E)
+    return UndefinedVal();
+  return getSVal(E);
+}
+
 void CallEvent::dump() const {
   dump(llvm::errs());
 }
@@ -230,10 +262,20 @@ void CallEvent::dump(raw_ostream &Out) const {
 }
 
 
-bool CallEvent::mayBeInlined(const Stmt *S) {
-  // FIXME: Kill this.
+bool CallEvent::isCallStmt(const Stmt *S) {
   return isa<CallExpr>(S) || isa<ObjCMessageExpr>(S)
-                          || isa<CXXConstructExpr>(S);
+                          || isa<CXXConstructExpr>(S)
+                          || isa<CXXNewExpr>(S);
+}
+
+/// \brief Returns the result type, adjusted for references.
+QualType CallEvent::getDeclaredResultType(const Decl *D) {
+  assert(D);
+  if (const FunctionDecl* FD = dyn_cast<FunctionDecl>(D))
+    return FD->getResultType();
+  else if (const ObjCMethodDecl* MD = dyn_cast<ObjCMethodDecl>(D))
+    return MD->getResultType();
+  return QualType();
 }
 
 static void addParameterValuesToBindings(const StackFrameContext *CalleeCtx,
@@ -285,14 +327,6 @@ void AnyFunctionCall::getInitialStackFrameContents(
                                D->param_begin(), D->param_end());
 }
 
-QualType AnyFunctionCall::getDeclaredResultType() const {
-  const FunctionDecl *D = getDecl();
-  if (!D)
-    return QualType();
-
-  return D->getResultType();
-}
-
 bool AnyFunctionCall::argumentsMayEscape() const {
   if (hasNonZeroCallbackArg())
     return true;
@@ -303,7 +337,7 @@ bool AnyFunctionCall::argumentsMayEscape() const {
 
   const IdentifierInfo *II = D->getIdentifier();
   if (!II)
-    return true;
+    return false;
 
   // This set of "escaping" APIs is 
 
@@ -376,6 +410,17 @@ void CXXInstanceCall::getExtraInvalidatedRegions(RegionList &Regions) const {
     Regions.push_back(R);
 }
 
+SVal CXXInstanceCall::getCXXThisVal() const {
+  const Expr *Base = getCXXThisExpr();
+  // FIXME: This doesn't handle an overloaded ->* operator.
+  if (!Base)
+    return UnknownVal();
+
+  SVal ThisVal = getSVal(Base);
+  assert(ThisVal.isUnknownOrUndef() || isa<Loc>(ThisVal));
+  return ThisVal;
+}
+
 
 RuntimeDefinition CXXInstanceCall::getRuntimeDefinition() const {
   // Do we have a decl at all?
@@ -400,13 +445,30 @@ RuntimeDefinition CXXInstanceCall::getRuntimeDefinition() const {
 
   // Is the type a C++ class? (This is mostly a defensive check.)
   QualType RegionType = DynType.getType()->getPointeeType();
+  assert(!RegionType.isNull() && "DynamicTypeInfo should always be a pointer.");
+
   const CXXRecordDecl *RD = RegionType->getAsCXXRecordDecl();
   if (!RD || !RD->hasDefinition())
     return RuntimeDefinition();
 
   // Find the decl for this method in that class.
   const CXXMethodDecl *Result = MD->getCorrespondingMethodInClass(RD, true);
-  assert(Result && "At the very least the static decl should show up.");
+  if (!Result) {
+    // We might not even get the original statically-resolved method due to
+    // some particularly nasty casting (e.g. casts to sister classes).
+    // However, we should at least be able to search up and down our own class
+    // hierarchy, and some real bugs have been caught by checking this.
+    assert(!RD->isDerivedFrom(MD->getParent()) && "Couldn't find known method");
+    
+    // FIXME: This is checking that our DynamicTypeInfo is at least as good as
+    // the static type. However, because we currently don't update
+    // DynamicTypeInfo when an object is cast, we can't actually be sure the
+    // DynamicTypeInfo is up to date. This assert should be re-enabled once
+    // this is fixed. <rdar://problem/12287087>
+    //assert(!MD->getParent()->isDerivedFrom(RD) && "Bad DynamicTypeInfo");
+
+    return RuntimeDefinition();
+  }
 
   // Does the decl that we found have an implementation?
   const FunctionDecl *Definition;
@@ -459,6 +521,18 @@ const Expr *CXXMemberCall::getCXXThisExpr() const {
   return getOriginExpr()->getImplicitObjectArgument();
 }
 
+RuntimeDefinition CXXMemberCall::getRuntimeDefinition() const {
+  // C++11 [expr.call]p1: ...If the selected function is non-virtual, or if the
+  // id-expression in the class member access expression is a qualified-id,
+  // that function is called. Otherwise, its final overrider in the dynamic type
+  // of the object expression is called.
+  if (const MemberExpr *ME = dyn_cast<MemberExpr>(getOriginExpr()->getCallee()))
+    if (ME->hasQualifier())
+      return AnyFunctionCall::getRuntimeDefinition();
+  
+  return CXXInstanceCall::getRuntimeDefinition();
+}
+
 
 const Expr *CXXMemberOperatorCall::getCXXThisExpr() const {
   return getOriginExpr()->getArg(0);
@@ -501,15 +575,6 @@ void BlockCall::getInitialStackFrameContents(const StackFrameContext *CalleeCtx,
 }
 
 
-QualType BlockCall::getDeclaredResultType() const {
-  const BlockDataRegion *BR = getBlockRegion();
-  if (!BR)
-    return QualType();
-  QualType BlockTy = BR->getCodeRegion()->getLocationType();
-  return cast<FunctionType>(BlockTy->getPointeeType())->getResultType();
-}
-
-
 SVal CXXConstructorCall::getCXXThisVal() const {
   if (Data)
     return loc::MemRegionVal(static_cast<const MemRegion *>(Data));
@@ -539,10 +604,19 @@ void CXXConstructorCall::getInitialStackFrameContents(
 
 SVal CXXDestructorCall::getCXXThisVal() const {
   if (Data)
-    return loc::MemRegionVal(static_cast<const MemRegion *>(Data));
+    return loc::MemRegionVal(DtorDataTy::getFromOpaqueValue(Data).getPointer());
   return UnknownVal();
 }
 
+RuntimeDefinition CXXDestructorCall::getRuntimeDefinition() const {
+  // Base destructors are always called non-virtually.
+  // Skip CXXInstanceCall's devirtualization logic in this case.
+  if (isBaseDestructor())
+    return AnyFunctionCall::getRuntimeDefinition();
+
+  return CXXInstanceCall::getRuntimeDefinition();
+}
+
 
 CallEvent::param_iterator ObjCMethodCall::param_begin() const {
   const ObjCMethodDecl *D = getDecl();
@@ -566,12 +640,12 @@ ObjCMethodCall::getExtraInvalidatedRegions(RegionList &Regions) const {
     Regions.push_back(R);
 }
 
-QualType ObjCMethodCall::getDeclaredResultType() const {
-  const ObjCMethodDecl *D = getDecl();
-  if (!D)
-    return QualType();
-
-  return D->getResultType();
+SVal ObjCMethodCall::getSelfSVal() const {
+  const LocationContext *LCtx = getLocationContext();
+  const ImplicitParamDecl *SelfDecl = LCtx->getSelfDecl();
+  if (!SelfDecl)
+    return SVal();
+  return getState()->getSVal(getState()->getRegion(SelfDecl, LCtx));
 }
 
 SVal ObjCMethodCall::getReceiverSVal() const {
@@ -584,10 +658,23 @@ SVal ObjCMethodCall::getReceiverSVal() const {
 
   // An instance message with no expression means we are sending to super.
   // In this case the object reference is the same as 'self'.
-  const LocationContext *LCtx = getLocationContext();
-  const ImplicitParamDecl *SelfDecl = LCtx->getSelfDecl();
-  assert(SelfDecl && "No message receiver Expr, but not in an ObjC method");
-  return getState()->getSVal(getState()->getRegion(SelfDecl, LCtx));
+  assert(getOriginExpr()->getReceiverKind() == ObjCMessageExpr::SuperInstance);
+  SVal SelfVal = getSelfSVal();
+  assert(SelfVal.isValid() && "Calling super but not in ObjC method");
+  return SelfVal;
+}
+
+bool ObjCMethodCall::isReceiverSelfOrSuper() const {
+  if (getOriginExpr()->getReceiverKind() == ObjCMessageExpr::SuperInstance ||
+      getOriginExpr()->getReceiverKind() == ObjCMessageExpr::SuperClass)
+      return true;
+
+  if (!isInstanceMessage())
+    return false;
+
+  SVal RecVal = getSVal(getOriginExpr()->getInstanceReceiver());
+
+  return (RecVal == getSelfSVal());
 }
 
 SourceRange ObjCMethodCall::getSourceRange() const {
@@ -820,7 +907,8 @@ CallEventManager::getCaller(const StackFrameContext *CalleeCtx,
       return getSimpleCall(CE, State, CallerCtx);
 
     switch (CallSite->getStmtClass()) {
-    case Stmt::CXXConstructExprClass: {
+    case Stmt::CXXConstructExprClass:
+    case Stmt::CXXTemporaryObjectExprClass: {
       SValBuilder &SVB = State->getStateManager().getSValBuilder();
       const CXXMethodDecl *Ctor = cast<CXXMethodDecl>(CalleeCtx->getDecl());
       Loc ThisPtr = SVB.getCXXThis(Ctor, CalleeCtx);
@@ -858,5 +946,5 @@ CallEventManager::getCaller(const StackFrameContext *CalleeCtx,
     Trigger = Dtor->getBody();
 
   return getCXXDestructorCall(Dtor, Trigger, ThisVal.getAsRegion(),
-                              State, CallerCtx);
+                              isa<CFGBaseDtor>(E), State, CallerCtx);
 }
diff --git a/lib/StaticAnalyzer/Core/CheckerContext.cpp b/lib/StaticAnalyzer/Core/CheckerContext.cpp
index 0a047d9..74eeef1 100644
--- a/lib/StaticAnalyzer/Core/CheckerContext.cpp
+++ b/lib/StaticAnalyzer/Core/CheckerContext.cpp
@@ -38,17 +38,14 @@ StringRef CheckerContext::getCalleeName(const FunctionDecl *FunDecl) const {
 
 bool CheckerContext::isCLibraryFunction(const FunctionDecl *FD,
                                         StringRef Name) {
-  return isCLibraryFunction(FD, Name, getASTContext());
-}
-
-bool CheckerContext::isCLibraryFunction(const FunctionDecl *FD,
-                                        StringRef Name, ASTContext &Context) {
   // To avoid false positives (Ex: finding user defined functions with
   // similar names), only perform fuzzy name matching when it's a builtin.
   // Using a string compare is slow, we might want to switch on BuiltinID here.
   unsigned BId = FD->getBuiltinID();
   if (BId != 0) {
-    StringRef BName = Context.BuiltinInfo.GetName(BId);
+    if (Name.empty())
+      return true;
+    StringRef BName = FD->getASTContext().BuiltinInfo.GetName(BId);
     if (BName.find(Name) != StringRef::npos)
       return true;
   }
@@ -59,6 +56,24 @@ bool CheckerContext::isCLibraryFunction(const FunctionDecl *FD,
   if (!II)
     return false;
 
+  // Look through 'extern "C"' and anything similar invented in the future.
+  const DeclContext *DC = FD->getDeclContext();
+  while (DC->isTransparentContext())
+    DC = DC->getParent();
+
+  // If this function is in a namespace, it is not a C library function.
+  if (!DC->isTranslationUnit())
+    return false;
+
+  // If this function is not externally visible, it is not a C library function.
+  // Note that we make an exception for inline functions, which may be
+  // declared in header files without external linkage.
+  if (!FD->isInlined() && FD->getLinkage() != ExternalLinkage)
+    return false;
+
+  if (Name.empty())
+    return true;
+
   StringRef FName = II->getName();
   if (FName.equals(Name))
     return true;
diff --git a/lib/StaticAnalyzer/Core/CheckerManager.cpp b/lib/StaticAnalyzer/Core/CheckerManager.cpp
index c786655..3672952 100644
--- a/lib/StaticAnalyzer/Core/CheckerManager.cpp
+++ b/lib/StaticAnalyzer/Core/CheckerManager.cpp
@@ -36,8 +36,7 @@ bool CheckerManager::hasPathSensitiveCheckers() const {
          !DeadSymbolsCheckers.empty()       ||
          !RegionChangesCheckers.empty()     ||
          !EvalAssumeCheckers.empty()        ||
-         !EvalCallCheckers.empty()          ||
-         !InlineCallCheckers.empty();
+         !EvalCallCheckers.empty();
 }
 
 void CheckerManager::finishedCheckerRegistration() {
@@ -314,20 +313,19 @@ namespace {
     SVal Val;
     const Stmt *S;
     ExprEngine &Eng;
-    ProgramPoint::Kind PointKind;
+    const ProgramPoint &PP;
 
     CheckersTy::const_iterator checkers_begin() { return Checkers.begin(); }
     CheckersTy::const_iterator checkers_end() { return Checkers.end(); }
 
     CheckBindContext(const CheckersTy &checkers,
                      SVal loc, SVal val, const Stmt *s, ExprEngine &eng,
-                     ProgramPoint::Kind PK)
-      : Checkers(checkers), Loc(loc), Val(val), S(s), Eng(eng), PointKind(PK) {}
+                     const ProgramPoint &pp)
+      : Checkers(checkers), Loc(loc), Val(val), S(s), Eng(eng), PP(pp) {}
 
     void runChecker(CheckerManager::CheckBindFunc checkFn,
                     NodeBuilder &Bldr, ExplodedNode *Pred) {
-      const ProgramPoint &L = ProgramPoint::getProgramPoint(S, PointKind,
-                                Pred->getLocationContext(), checkFn.Checker);
+      const ProgramPoint &L = PP.withTag(checkFn.Checker);
       CheckerContext C(Bldr, Eng, Pred, L);
 
       checkFn(Loc, Val, S, C);
@@ -340,8 +338,8 @@ void CheckerManager::runCheckersForBind(ExplodedNodeSet &Dst,
                                         const ExplodedNodeSet &Src,
                                         SVal location, SVal val,
                                         const Stmt *S, ExprEngine &Eng,
-                                        ProgramPoint::Kind PointKind) {
-  CheckBindContext C(BindCheckers, location, val, S, Eng, PointKind);
+                                        const ProgramPoint &PP) {
+  CheckBindContext C(BindCheckers, location, val, S, Eng, PP);
   expandGraphWithCheckers(C, Dst, Src);
 }
 
@@ -357,8 +355,8 @@ void CheckerManager::runCheckersForEndAnalysis(ExplodedGraph &G,
 // for this callback since end of path nodes are expected to be final.
 void CheckerManager::runCheckersForEndPath(NodeBuilderContext &BC,
                                            ExplodedNodeSet &Dst,
+                                           ExplodedNode *Pred,
                                            ExprEngine &Eng) {
-  ExplodedNode *Pred = BC.Pred;
   
   // We define the builder outside of the loop bacause if at least one checkers
   // creates a sucsessor for Pred, we do not need to generate an 
@@ -509,41 +507,13 @@ void CheckerManager::runCheckersForEvalCall(ExplodedNodeSet &Dst,
   const CallExpr *CE = cast<CallExpr>(Call.getOriginExpr());
   for (ExplodedNodeSet::iterator
          NI = Src.begin(), NE = Src.end(); NI != NE; ++NI) {
-
     ExplodedNode *Pred = *NI;
     bool anyEvaluated = false;
 
-    // First, check if any of the InlineCall callbacks can evaluate the call.
-    assert(InlineCallCheckers.size() <= 1 &&
-           "InlineCall is a special hacky callback to allow intrusive"
-           "evaluation of the call (which simulates inlining). It is "
-           "currently only used by OSAtomicChecker and should go away "
-           "at some point.");
-    for (std::vector<InlineCallFunc>::iterator
-           EI = InlineCallCheckers.begin(), EE = InlineCallCheckers.end();
-         EI != EE; ++EI) {
-      ExplodedNodeSet checkDst;
-      bool evaluated = (*EI)(CE, Eng, Pred, checkDst);
-      assert(!(evaluated && anyEvaluated)
-             && "There are more than one checkers evaluating the call");
-      if (evaluated) {
-        anyEvaluated = true;
-        Dst.insert(checkDst);
-#ifdef NDEBUG
-        break; // on release don't check that no other checker also evals.
-#endif
-      }
-    }
-
-#ifdef NDEBUG // on release don't check that no other checker also evals.
-    if (anyEvaluated) {
-      break;
-    }
-#endif
-
     ExplodedNodeSet checkDst;
     NodeBuilder B(Pred, checkDst, Eng.getBuilderContext());
-    // Next, check if any of the EvalCall callbacks can evaluate the call.
+
+    // Check if any of the EvalCall callbacks can evaluate the call.
     for (std::vector<EvalCallFunc>::iterator
            EI = EvalCallCheckers.begin(), EE = EvalCallCheckers.end();
          EI != EE; ++EI) {
@@ -679,10 +649,6 @@ void CheckerManager::_registerForEvalCall(EvalCallFunc checkfn) {
   EvalCallCheckers.push_back(checkfn);
 }
 
-void CheckerManager::_registerForInlineCall(InlineCallFunc checkfn) {
-  InlineCallCheckers.push_back(checkfn);
-}
-
 void CheckerManager::_registerForEndOfTranslationUnit(
                                             CheckEndOfTranslationUnit checkfn) {
   EndOfTranslationUnitCheckers.push_back(checkfn);
diff --git a/lib/StaticAnalyzer/Core/ConstraintManager.cpp b/lib/StaticAnalyzer/Core/ConstraintManager.cpp
new file mode 100644
index 0000000..4dec526
--- /dev/null
+++ b/lib/StaticAnalyzer/Core/ConstraintManager.cpp
@@ -0,0 +1,39 @@
+//== ConstraintManager.cpp - Constraints on symbolic values -----*- C++ -*--==//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defined the interface to manage constraints on symbolic values.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h"
+
+using namespace clang;
+using namespace ento;
+
+ConstraintManager::~ConstraintManager() {}
+
+static DefinedSVal getLocFromSymbol(const ProgramStateRef &State,
+                                    SymbolRef Sym) {
+  const MemRegion *R = State->getStateManager().getRegionManager()
+                                               .getSymbolicRegion(Sym);
+  return loc::MemRegionVal(R);
+}
+
+ConditionTruthVal ConstraintManager::checkNull(ProgramStateRef State,
+                                               SymbolRef Sym) {  
+  QualType Ty = Sym->getType();
+  DefinedSVal V = Loc::isLocType(Ty) ? getLocFromSymbol(State, Sym)
+                                     : nonloc::SymbolVal(Sym);
+  const ProgramStatePair &P = assumeDual(State, V);
+  if (P.first && !P.second)
+    return ConditionTruthVal(false);
+  if (!P.first && P.second)
+    return ConditionTruthVal(true);
+  return ConditionTruthVal();
+}
diff --git a/lib/StaticAnalyzer/Core/CoreEngine.cpp b/lib/StaticAnalyzer/Core/CoreEngine.cpp
index 1f13742..ec23792 100644
--- a/lib/StaticAnalyzer/Core/CoreEngine.cpp
+++ b/lib/StaticAnalyzer/Core/CoreEngine.cpp
@@ -243,11 +243,6 @@ void CoreEngine::dispatchWorkItem(ExplodedNode* Pred, ProgramPoint Loc,
 
     case ProgramPoint::CallEnterKind: {
       CallEnter CEnter = cast<CallEnter>(Loc);
-      if (AnalyzedCallees)
-        if (const CallExpr* CE =
-            dyn_cast_or_null<CallExpr>(CEnter.getCallExpr()))
-          if (const Decl *CD = CE->getCalleeDecl())
-            AnalyzedCallees->insert(CD);
       SubEng.processCallEnter(CEnter, Pred);
       break;
     }
@@ -303,7 +298,7 @@ void CoreEngine::HandleBlockEdge(const BlockEdge &L, ExplodedNode *Pred) {
             && "EXIT block cannot contain Stmts.");
 
     // Process the final state transition.
-    SubEng.processEndOfFunction(BuilderCtx);
+    SubEng.processEndOfFunction(BuilderCtx, Pred);
 
     // This path is done. Don't enqueue any more nodes.
     return;
@@ -313,7 +308,7 @@ void CoreEngine::HandleBlockEdge(const BlockEdge &L, ExplodedNode *Pred) {
   ExplodedNodeSet dstNodes;
   BlockEntrance BE(Blk, Pred->getLocationContext());
   NodeBuilderWithSinks nodeBuilder(Pred, dstNodes, BuilderCtx, BE);
-  SubEng.processCFGBlockEntrance(L, nodeBuilder);
+  SubEng.processCFGBlockEntrance(L, nodeBuilder, Pred);
 
   // Auto-generate a node.
   if (!nodeBuilder.hasGeneratedNodes()) {
@@ -519,9 +514,9 @@ void CoreEngine::enqueueStmtNode(ExplodedNode *N,
     return;
   }
 
-  const CFGStmt *CS = (*Block)[Idx].getAs<CFGStmt>();
-  const Stmt *St = CS ? CS->getStmt() : 0;
-  PostStmt Loc(St, N->getLocationContext());
+  // At this point, we know we're processing a normal statement.
+  CFGStmt CS = cast<CFGStmt>((*Block)[Idx]);
+  PostStmt Loc(CS.getStmt(), N->getLocationContext());
 
   if (Loc == N->getLocation()) {
     // Note: 'N' should be a fresh node because otherwise it shouldn't be
diff --git a/lib/StaticAnalyzer/Core/Environment.cpp b/lib/StaticAnalyzer/Core/Environment.cpp
index 52644f7..bab89c5 100644
--- a/lib/StaticAnalyzer/Core/Environment.cpp
+++ b/lib/StaticAnalyzer/Core/Environment.cpp
@@ -20,6 +20,44 @@
 using namespace clang;
 using namespace ento;
 
+static const Expr *ignoreTransparentExprs(const Expr *E) {
+  E = E->IgnoreParens();
+
+  switch (E->getStmtClass()) {
+  case Stmt::OpaqueValueExprClass:
+    E = cast<OpaqueValueExpr>(E)->getSourceExpr();
+    break;
+  case Stmt::ExprWithCleanupsClass:
+    E = cast<ExprWithCleanups>(E)->getSubExpr();
+    break;
+  case Stmt::CXXBindTemporaryExprClass:
+    E = cast<CXXBindTemporaryExpr>(E)->getSubExpr();
+    break;
+  case Stmt::SubstNonTypeTemplateParmExprClass:
+    E = cast<SubstNonTypeTemplateParmExpr>(E)->getReplacement();
+    break;
+  case Stmt::CXXDefaultArgExprClass:
+    E = cast<CXXDefaultArgExpr>(E)->getExpr();
+    break;
+  default:
+    // This is the base case: we can't look through more than we already have.
+    return E;
+  }
+
+  return ignoreTransparentExprs(E);
+}
+
+static const Stmt *ignoreTransparentExprs(const Stmt *S) {
+  if (const Expr *E = dyn_cast<Expr>(S))
+    return ignoreTransparentExprs(E);
+  return S;
+}
+
+EnvironmentEntry::EnvironmentEntry(const Stmt *S, const LocationContext *L)
+  : std::pair<const Stmt *,
+              const StackFrameContext *>(ignoreTransparentExprs(S),
+                                         L ? L->getCurrentStackFrame() : 0) {}
+
 SVal Environment::lookupExpr(const EnvironmentEntry &E) const {
   const SVal* X = ExprBindings.lookup(E);
   if (X) {
@@ -30,101 +68,72 @@ SVal Environment::lookupExpr(const EnvironmentEntry &E) const {
 }
 
 SVal Environment::getSVal(const EnvironmentEntry &Entry,
-                          SValBuilder& svalBuilder,
-                          bool useOnlyDirectBindings) const {
-
-  if (useOnlyDirectBindings) {
-    // This branch is rarely taken, but can be exercised by
-    // checkers that explicitly bind values to arbitrary
-    // expressions.  It is crucial that we do not ignore any
-    // expression here, and do a direct lookup.
-    return lookupExpr(Entry);
+                          SValBuilder& svalBuilder) const {
+  const Stmt *S = Entry.getStmt();
+  const LocationContext *LCtx = Entry.getLocationContext();
+
+  switch (S->getStmtClass()) {
+  case Stmt::CXXBindTemporaryExprClass:
+  case Stmt::CXXDefaultArgExprClass:
+  case Stmt::ExprWithCleanupsClass:
+  case Stmt::GenericSelectionExprClass:
+  case Stmt::OpaqueValueExprClass:
+  case Stmt::ParenExprClass:
+  case Stmt::SubstNonTypeTemplateParmExprClass:
+    llvm_unreachable("Should have been handled by ignoreTransparentExprs");
+
+  case Stmt::AddrLabelExprClass:
+    return svalBuilder.makeLoc(cast<AddrLabelExpr>(S));
+
+  case Stmt::CharacterLiteralClass: {
+    const CharacterLiteral *C = cast<CharacterLiteral>(S);
+    return svalBuilder.makeIntVal(C->getValue(), C->getType());
   }
 
-  const Stmt *E = Entry.getStmt();
-  const LocationContext *LCtx = Entry.getLocationContext();
-  
-  for (;;) {
-    if (const Expr *Ex = dyn_cast<Expr>(E))
-      E = Ex->IgnoreParens();
-
-    switch (E->getStmtClass()) {
-      case Stmt::AddrLabelExprClass:
-        return svalBuilder.makeLoc(cast<AddrLabelExpr>(E));
-      case Stmt::OpaqueValueExprClass: {
-        const OpaqueValueExpr *ope = cast<OpaqueValueExpr>(E);
-        E = ope->getSourceExpr();
-        continue;        
-      }        
-      case Stmt::ParenExprClass:
-      case Stmt::GenericSelectionExprClass:
-        llvm_unreachable("ParenExprs and GenericSelectionExprs should "
-                         "have been handled by IgnoreParens()");
-      case Stmt::CharacterLiteralClass: {
-        const CharacterLiteral* C = cast<CharacterLiteral>(E);
-        return svalBuilder.makeIntVal(C->getValue(), C->getType());
-      }
-      case Stmt::CXXBoolLiteralExprClass: {
-        const SVal *X = ExprBindings.lookup(EnvironmentEntry(E, LCtx));
-        if (X) 
-          return *X;
-        else 
-          return svalBuilder.makeBoolVal(cast<CXXBoolLiteralExpr>(E));
-      }
-      case Stmt::CXXScalarValueInitExprClass:
-      case Stmt::ImplicitValueInitExprClass: {
-        QualType Ty = cast<Expr>(E)->getType();
-        return svalBuilder.makeZeroVal(Ty);
-      }
-      case Stmt::IntegerLiteralClass: {
-        // In C++, this expression may have been bound to a temporary object.
-        SVal const *X = ExprBindings.lookup(EnvironmentEntry(E, LCtx));
-        if (X)
-          return *X;
-        else
-          return svalBuilder.makeIntVal(cast<IntegerLiteral>(E));
-      }
-      case Stmt::ObjCBoolLiteralExprClass:
-        return svalBuilder.makeBoolVal(cast<ObjCBoolLiteralExpr>(E));
-
-      // For special C0xx nullptr case, make a null pointer SVal.
-      case Stmt::CXXNullPtrLiteralExprClass:
-        return svalBuilder.makeNull();
-      case Stmt::ExprWithCleanupsClass:
-        E = cast<ExprWithCleanups>(E)->getSubExpr();
-        continue;
-      case Stmt::CXXBindTemporaryExprClass:
-        E = cast<CXXBindTemporaryExpr>(E)->getSubExpr();
-        continue;
-      case Stmt::SubstNonTypeTemplateParmExprClass:
-        E = cast<SubstNonTypeTemplateParmExpr>(E)->getReplacement();
-        continue;
-      case Stmt::ObjCStringLiteralClass: {
-        MemRegionManager &MRMgr = svalBuilder.getRegionManager();
-        const ObjCStringLiteral *SL = cast<ObjCStringLiteral>(E);
-        return svalBuilder.makeLoc(MRMgr.getObjCStringRegion(SL));
-      }
-      case Stmt::StringLiteralClass: {
-        MemRegionManager &MRMgr = svalBuilder.getRegionManager();
-        const StringLiteral *SL = cast<StringLiteral>(E);
-        return svalBuilder.makeLoc(MRMgr.getStringRegion(SL));
-      }
-      case Stmt::ReturnStmtClass: {
-        const ReturnStmt *RS = cast<ReturnStmt>(E);
-        if (const Expr *RE = RS->getRetValue()) {
-          E = RE;
-          continue;
-        }
-        return UndefinedVal();        
-      }
-        
-      // Handle all other Stmt* using a lookup.
-      default:
-        break;
-    };
+  case Stmt::CXXBoolLiteralExprClass:
+    return svalBuilder.makeBoolVal(cast<CXXBoolLiteralExpr>(S));
+
+  case Stmt::CXXScalarValueInitExprClass:
+  case Stmt::ImplicitValueInitExprClass: {
+    QualType Ty = cast<Expr>(S)->getType();
+    return svalBuilder.makeZeroVal(Ty);
+  }
+
+  case Stmt::IntegerLiteralClass:
+    return svalBuilder.makeIntVal(cast<IntegerLiteral>(S));
+
+  case Stmt::ObjCBoolLiteralExprClass:
+    return svalBuilder.makeBoolVal(cast<ObjCBoolLiteralExpr>(S));
+
+  // For special C0xx nullptr case, make a null pointer SVal.
+  case Stmt::CXXNullPtrLiteralExprClass:
+    return svalBuilder.makeNull();
+
+  case Stmt::ObjCStringLiteralClass: {
+    MemRegionManager &MRMgr = svalBuilder.getRegionManager();
+    const ObjCStringLiteral *SL = cast<ObjCStringLiteral>(S);
+    return svalBuilder.makeLoc(MRMgr.getObjCStringRegion(SL));
+  }
+
+  case Stmt::StringLiteralClass: {
+    MemRegionManager &MRMgr = svalBuilder.getRegionManager();
+    const StringLiteral *SL = cast<StringLiteral>(S);
+    return svalBuilder.makeLoc(MRMgr.getStringRegion(SL));
+  }
+
+  case Stmt::ReturnStmtClass: {
+    const ReturnStmt *RS = cast<ReturnStmt>(S);
+    if (const Expr *RE = RS->getRetValue())
+      return getSVal(EnvironmentEntry(RE, LCtx), svalBuilder);
+    return UndefinedVal();        
+  }
+    
+  // Handle all other Stmt* using a lookup.
+  default:
     break;
   }
-  return lookupExpr(EnvironmentEntry(E, LCtx));
+  
+  return lookupExpr(EnvironmentEntry(S, LCtx));
 }
 
 Environment EnvironmentManager::bindExpr(Environment Env,
@@ -140,16 +149,16 @@ Environment EnvironmentManager::bindExpr(Environment Env,
   return Environment(F.add(Env.ExprBindings, E, V));
 }
 
-static inline EnvironmentEntry MakeLocation(const EnvironmentEntry &E) {
-  const Stmt *S = E.getStmt();
-  S = (const Stmt*) (((uintptr_t) S) | 0x1);
-  return EnvironmentEntry(S, E.getLocationContext());
+EnvironmentEntry EnvironmentEntry::makeLocation() const {
+  EnvironmentEntry Result = *this;
+  reinterpret_cast<uintptr_t &>(Result.first) |= 0x1;
+  return Result;
 }
 
 Environment EnvironmentManager::bindExprAndLocation(Environment Env,
                                                     const EnvironmentEntry &E,
                                                     SVal location, SVal V) {
-  return Environment(F.add(F.add(Env.ExprBindings, MakeLocation(E), location),
+  return Environment(F.add(F.add(Env.ExprBindings, E.makeLocation(), location),
                            E, V));
 }
 
@@ -286,7 +295,8 @@ void Environment::printAux(raw_ostream &Out, bool printLocations,
       S = (Stmt*) (((uintptr_t) S) & ((uintptr_t) ~0x1));
     }
     
-    Out << " (" << (void*) En.getLocationContext() << ',' << (void*) S << ") ";
+    Out << " (" << (const void*) En.getLocationContext() << ','
+      << (const void*) S << ") ";
     LangOptions LO; // FIXME.
     S->printPretty(Out, 0, PrintingPolicy(LO));
     Out << " : " << I.getData();
diff --git a/lib/StaticAnalyzer/Core/ExplodedGraph.cpp b/lib/StaticAnalyzer/Core/ExplodedGraph.cpp
index b79f3f5..c284bd7 100644
--- a/lib/StaticAnalyzer/Core/ExplodedGraph.cpp
+++ b/lib/StaticAnalyzer/Core/ExplodedGraph.cpp
@@ -47,10 +47,8 @@ void ExplodedNode::SetAuditor(ExplodedNode::Auditor* A) {
 // Cleanup.
 //===----------------------------------------------------------------------===//
 
-static const unsigned CounterTop = 1000;
-
 ExplodedGraph::ExplodedGraph()
-  : NumNodes(0), reclaimNodes(false), reclaimCounter(CounterTop) {}
+  : NumNodes(0), ReclaimNodeInterval(0) {}
 
 ExplodedGraph::~ExplodedGraph() {}
 
@@ -63,12 +61,12 @@ bool ExplodedGraph::shouldCollect(const ExplodedNode *node) {
   //
   // (1) 1 predecessor (that has one successor)
   // (2) 1 successor (that has one predecessor)
-  // (3) The ProgramPoint is for a PostStmt.
+  // (3) The ProgramPoint is for a PostStmt, but not a PostStore.
   // (4) There is no 'tag' for the ProgramPoint.
   // (5) The 'store' is the same as the predecessor.
   // (6) The 'GDM' is the same as the predecessor.
   // (7) The LocationContext is the same as the predecessor.
-  // (8) The PostStmt is for a non-consumed Stmt or Expr.
+  // (8) The PostStmt isn't for a non-consumed Stmt or Expr.
   // (9) The successor is not a CallExpr StmtPoint (so that we would be able to
   //     find it when retrying a call with no inlining).
   // FIXME: It may be safe to reclaim PreCall and PostCall nodes as well.
@@ -87,16 +85,13 @@ bool ExplodedGraph::shouldCollect(const ExplodedNode *node) {
 
   // Condition 3.
   ProgramPoint progPoint = node->getLocation();
-  if (!isa<PostStmt>(progPoint))
+  if (!isa<PostStmt>(progPoint) || isa<PostStore>(progPoint))
     return false;
 
   // Condition 4.
   PostStmt ps = cast<PostStmt>(progPoint);
   if (ps.getTag())
     return false;
-  
-  if (isa<BinaryOperator>(ps.getStmt()))
-    return false;
 
   // Conditions 5, 6, and 7.
   ProgramStateRef state = node->getState();
@@ -106,6 +101,12 @@ bool ExplodedGraph::shouldCollect(const ExplodedNode *node) {
     return false;
   
   // Condition 8.
+  // Do not collect nodes for non-consumed Stmt or Expr to ensure precise
+  // diagnostic generation; specifically, so that we could anchor arrows
+  // pointing to the beginning of statements (as written in code).
+  if (!isa<Expr>(ps.getStmt()))
+    return false;
+  
   if (const Expr *Ex = dyn_cast<Expr>(ps.getStmt())) {
     ParentMap &PM = progPoint.getLocationContext()->getParentMap();
     if (!PM.isConsumedExpr(Ex))
@@ -115,7 +116,7 @@ bool ExplodedGraph::shouldCollect(const ExplodedNode *node) {
   // Condition 9.
   const ProgramPoint SuccLoc = succ->getLocation();
   if (const StmtPoint *SP = dyn_cast<StmtPoint>(&SuccLoc))
-    if (CallEvent::mayBeInlined(SP->getStmt()))
+    if (CallEvent::isCallStmt(SP->getStmt()))
       return false;
 
   return true;
@@ -141,13 +142,13 @@ void ExplodedGraph::reclaimRecentlyAllocatedNodes() {
   if (ChangedNodes.empty())
     return;
 
-  // Only periodically relcaim nodes so that we can build up a set of
+  // Only periodically reclaim nodes so that we can build up a set of
   // nodes that meet the reclamation criteria.  Freshly created nodes
   // by definition have no successor, and thus cannot be reclaimed (see below).
-  assert(reclaimCounter > 0);
-  if (--reclaimCounter != 0)
+  assert(ReclaimCounter > 0);
+  if (--ReclaimCounter != 0)
     return;
-  reclaimCounter = CounterTop;
+  ReclaimCounter = ReclaimNodeInterval;
 
   for (NodeVector::iterator it = ChangedNodes.begin(), et = ChangedNodes.end();
        it != et; ++it) {
@@ -162,9 +163,18 @@ void ExplodedGraph::reclaimRecentlyAllocatedNodes() {
 // ExplodedNode.
 //===----------------------------------------------------------------------===//
 
-static inline BumpVector<ExplodedNode*>& getVector(void *P) {
-  return *reinterpret_cast<BumpVector<ExplodedNode*>*>(P);
-}
+// An NodeGroup's storage type is actually very much like a TinyPtrVector:
+// it can be either a pointer to a single ExplodedNode, or a pointer to a
+// BumpVector allocated with the ExplodedGraph's allocator. This allows the
+// common case of single-node NodeGroups to be implemented with no extra memory.
+//
+// Consequently, each of the NodeGroup methods have up to four cases to handle:
+// 1. The flag is set and this group does not actually contain any nodes.
+// 2. The group is empty, in which case the storage value is null.
+// 3. The group contains a single node.
+// 4. The group contains more than one node.
+typedef BumpVector<ExplodedNode *> ExplodedNodeVector;
+typedef llvm::PointerUnion<ExplodedNode *, ExplodedNodeVector *> GroupStorage;
 
 void ExplodedNode::addPredecessor(ExplodedNode *V, ExplodedGraph &G) {
   assert (!V->isSink());
@@ -176,71 +186,77 @@ void ExplodedNode::addPredecessor(ExplodedNode *V, ExplodedGraph &G) {
 }
 
 void ExplodedNode::NodeGroup::replaceNode(ExplodedNode *node) {
-  assert(getKind() == Size1);
-  P = reinterpret_cast<uintptr_t>(node);
-  assert(getKind() == Size1);
+  assert(!getFlag());
+
+  GroupStorage &Storage = reinterpret_cast<GroupStorage&>(P);
+  assert(Storage.is<ExplodedNode *>());
+  Storage = node;
+  assert(Storage.is<ExplodedNode *>());
 }
 
 void ExplodedNode::NodeGroup::addNode(ExplodedNode *N, ExplodedGraph &G) {
-  assert((reinterpret_cast<uintptr_t>(N) & Mask) == 0x0);
   assert(!getFlag());
 
-  if (getKind() == Size1) {
-    if (ExplodedNode *NOld = getNode()) {
-      BumpVectorContext &Ctx = G.getNodeAllocator();
-      BumpVector<ExplodedNode*> *V = 
-        G.getAllocator().Allocate<BumpVector<ExplodedNode*> >();
-      new (V) BumpVector<ExplodedNode*>(Ctx, 4);
-      
-      assert((reinterpret_cast<uintptr_t>(V) & Mask) == 0x0);
-      V->push_back(NOld, Ctx);
-      V->push_back(N, Ctx);
-      P = reinterpret_cast<uintptr_t>(V) | SizeOther;
-      assert(getPtr() == (void*) V);
-      assert(getKind() == SizeOther);
-    }
-    else {
-      P = reinterpret_cast<uintptr_t>(N);
-      assert(getKind() == Size1);
-    }
+  GroupStorage &Storage = reinterpret_cast<GroupStorage&>(P);
+  if (Storage.isNull()) {
+    Storage = N;
+    assert(Storage.is<ExplodedNode *>());
+    return;
   }
-  else {
-    assert(getKind() == SizeOther);
-    getVector(getPtr()).push_back(N, G.getNodeAllocator());
+
+  ExplodedNodeVector *V = Storage.dyn_cast<ExplodedNodeVector *>();
+
+  if (!V) {
+    // Switch from single-node to multi-node representation.
+    ExplodedNode *Old = Storage.get<ExplodedNode *>();
+
+    BumpVectorContext &Ctx = G.getNodeAllocator();
+    V = G.getAllocator().Allocate<ExplodedNodeVector>();
+    new (V) ExplodedNodeVector(Ctx, 4);
+    V->push_back(Old, Ctx);
+
+    Storage = V;
+    assert(!getFlag());
+    assert(Storage.is<ExplodedNodeVector *>());
   }
+
+  V->push_back(N, G.getNodeAllocator());
 }
 
 unsigned ExplodedNode::NodeGroup::size() const {
   if (getFlag())
     return 0;
 
-  if (getKind() == Size1)
-    return getNode() ? 1 : 0;
-  else
-    return getVector(getPtr()).size();
+  const GroupStorage &Storage = reinterpret_cast<const GroupStorage &>(P);
+  if (Storage.isNull())
+    return 0;
+  if (ExplodedNodeVector *V = Storage.dyn_cast<ExplodedNodeVector *>())
+    return V->size();
+  return 1;
 }
 
-ExplodedNode **ExplodedNode::NodeGroup::begin() const {
+ExplodedNode * const *ExplodedNode::NodeGroup::begin() const {
   if (getFlag())
-    return NULL;
+    return 0;
 
-  if (getKind() == Size1)
-    return (ExplodedNode**) (getPtr() ? &P : NULL);
-  else
-    return const_cast<ExplodedNode**>(&*(getVector(getPtr()).begin()));
+  const GroupStorage &Storage = reinterpret_cast<const GroupStorage &>(P);
+  if (Storage.isNull())
+    return 0;
+  if (ExplodedNodeVector *V = Storage.dyn_cast<ExplodedNodeVector *>())
+    return V->begin();
+  return Storage.getAddrOfPtr1();
 }
 
-ExplodedNode** ExplodedNode::NodeGroup::end() const {
+ExplodedNode * const *ExplodedNode::NodeGroup::end() const {
   if (getFlag())
-    return NULL;
-
-  if (getKind() == Size1)
-    return (ExplodedNode**) (getPtr() ? &P+1 : NULL);
-  else {
-    // Dereferencing end() is undefined behaviour. The vector is not empty, so
-    // we can dereference the last elem and then add 1 to the result.
-    return const_cast<ExplodedNode**>(getVector(getPtr()).end());
-  }
+    return 0;
+
+  const GroupStorage &Storage = reinterpret_cast<const GroupStorage &>(P);
+  if (Storage.isNull())
+    return 0;
+  if (ExplodedNodeVector *V = Storage.dyn_cast<ExplodedNodeVector *>())
+    return V->end();
+  return Storage.getAddrOfPtr1() + 1;
 }
 
 ExplodedNode *ExplodedGraph::getNode(const ProgramPoint &L,
@@ -266,7 +282,7 @@ ExplodedNode *ExplodedGraph::getNode(const ProgramPoint &L,
 
     new (V) NodeTy(L, State, IsSink);
 
-    if (reclaimNodes)
+    if (ReclaimNodeInterval)
       ChangedNodes.push_back(V);
 
     // Insert the node into the node set and return it.
@@ -314,8 +330,8 @@ ExplodedGraph::TrimInternal(const ExplodedNode* const* BeginSources,
 
   // ===- Pass 1 (reverse DFS) -===
   for (const ExplodedNode* const* I = BeginSources; I != EndSources; ++I) {
-    assert(*I);
-    WL1.push_back(*I);
+    if (*I)
+      WL1.push_back(*I);
   }
 
   // Process the first worklist until it is empty.  Because it is a std::list
@@ -338,7 +354,8 @@ ExplodedGraph::TrimInternal(const ExplodedNode* const* BeginSources,
     }
 
     // Visit our predecessors and enqueue them.
-    for (ExplodedNode** I=N->Preds.begin(), **E=N->Preds.end(); I!=E; ++I)
+    for (ExplodedNode::pred_iterator I = N->Preds.begin(), E = N->Preds.end();
+         I != E; ++I)
       WL1.push_back(*I);
   }
 
@@ -375,7 +392,8 @@ ExplodedGraph::TrimInternal(const ExplodedNode* const* BeginSources,
 
     // Walk through the predecessors of 'N' and hook up their corresponding
     // nodes in the new graph (if any) to the freshly created node.
-    for (ExplodedNode **I=N->Preds.begin(), **E=N->Preds.end(); I!=E; ++I) {
+    for (ExplodedNode::pred_iterator I = N->Preds.begin(), E = N->Preds.end();
+         I != E; ++I) {
       Pass2Ty::iterator PI = Pass2.find(*I);
       if (PI == Pass2.end())
         continue;
@@ -387,7 +405,8 @@ ExplodedGraph::TrimInternal(const ExplodedNode* const* BeginSources,
     // been created, we should hook them up as successors.  Otherwise, enqueue
     // the new nodes from the original graph that should have nodes created
     // in the new graph.
-    for (ExplodedNode **I=N->Succs.begin(), **E=N->Succs.end(); I!=E; ++I) {
+    for (ExplodedNode::succ_iterator I = N->Succs.begin(), E = N->Succs.end();
+         I != E; ++I) {
       Pass2Ty::iterator PI = Pass2.find(*I);
       if (PI != Pass2.end()) {
         PI->second->addPredecessor(NewN, *G);
diff --git a/lib/StaticAnalyzer/Core/ExprEngine.cpp b/lib/StaticAnalyzer/Core/ExprEngine.cpp
index b0435fb..045591c 100644
--- a/lib/StaticAnalyzer/Core/ExprEngine.cpp
+++ b/lib/StaticAnalyzer/Core/ExprEngine.cpp
@@ -55,32 +55,32 @@ STATISTIC(NumTimesRetriedWithoutInlining,
 //===----------------------------------------------------------------------===//
 
 ExprEngine::ExprEngine(AnalysisManager &mgr, bool gcEnabled,
-                       SetOfConstDecls *VisitedCallees,
+                       SetOfConstDecls *VisitedCalleesIn,
                        FunctionSummariesTy *FS)
   : AMgr(mgr),
     AnalysisDeclContexts(mgr.getAnalysisDeclContextManager()),
-    Engine(*this, VisitedCallees, FS),
+    Engine(*this, FS),
     G(Engine.getGraph()),
     StateMgr(getContext(), mgr.getStoreManagerCreator(),
              mgr.getConstraintManagerCreator(), G.getAllocator(),
-             *this),
+             this),
     SymMgr(StateMgr.getSymbolManager()),
     svalBuilder(StateMgr.getSValBuilder()),
     EntryNode(NULL),
-    currentStmt(NULL), currentStmtIdx(0), currentBuilderContext(0),
-    NSExceptionII(NULL), NSExceptionInstanceRaiseSelectors(NULL),
-    RaiseSel(GetNullarySelector("raise", getContext())),
-    ObjCGCEnabled(gcEnabled), BR(mgr, *this) {
-  
-  if (mgr.shouldEagerlyTrimExplodedGraph()) {
-    // Enable eager node reclaimation when constructing the ExplodedGraph.  
-    G.enableNodeReclamation();
+    currStmt(NULL), currStmtIdx(0), currBldrCtx(0),
+    ObjCNoRet(mgr.getASTContext()),
+    ObjCGCEnabled(gcEnabled), BR(mgr, *this),
+    VisitedCallees(VisitedCalleesIn)
+{
+  unsigned TrimInterval = mgr.options.getGraphTrimInterval();
+  if (TrimInterval != 0) {
+    // Enable eager node reclaimation when constructing the ExplodedGraph.
+    G.enableNodeReclamation(TrimInterval);
   }
 }
 
 ExprEngine::~ExprEngine() {
   BR.FlushReports();
-  delete [] NSExceptionInstanceRaiseSelectors;
 }
 
 //===----------------------------------------------------------------------===//
@@ -164,6 +164,23 @@ ProgramStateRef ExprEngine::getInitialState(const LocationContext *InitLoc) {
   return state;
 }
 
+/// If the value of the given expression is a NonLoc, copy it into a new
+/// temporary region, and replace the value of the expression with that.
+static ProgramStateRef createTemporaryRegionIfNeeded(ProgramStateRef State,
+                                                     const LocationContext *LC,
+                                                     const Expr *E) {
+  SVal V = State->getSVal(E, LC);
+
+  if (isa<NonLoc>(V)) {
+    MemRegionManager &MRMgr = State->getStateManager().getRegionManager();
+    const MemRegion *R  = MRMgr.getCXXTempObjectRegion(E, LC);
+    State = State->bindLoc(loc::MemRegionVal(R), V);
+    State = State->BindExpr(E, LC, loc::MemRegionVal(R));
+  }
+
+  return State;
+}
+
 //===----------------------------------------------------------------------===//
 // Top-level transfer function logic (Dispatcher).
 //===----------------------------------------------------------------------===//
@@ -200,8 +217,8 @@ void ExprEngine::processEndWorklist(bool hasWorkRemaining) {
 
 void ExprEngine::processCFGElement(const CFGElement E, ExplodedNode *Pred,
                                    unsigned StmtIdx, NodeBuilderContext *Ctx) {
-  currentStmtIdx = StmtIdx;
-  currentBuilderContext = Ctx;
+  currStmtIdx = StmtIdx;
+  currBldrCtx = Ctx;
 
   switch (E.getKind()) {
     case CFGElement::Invalid:
@@ -219,7 +236,7 @@ void ExprEngine::processCFGElement(const CFGElement E, ExplodedNode *Pred,
       ProcessImplicitDtor(*E.getAs<CFGImplicitDtor>(), Pred);
       return;
   }
-  currentBuilderContext = 0;
+  currBldrCtx = 0;
 }
 
 static bool shouldRemoveDeadBindings(AnalysisManager &AMgr,
@@ -228,7 +245,7 @@ static bool shouldRemoveDeadBindings(AnalysisManager &AMgr,
                                      const LocationContext *LC) {
   
   // Are we never purging state values?
-  if (AMgr.getPurgeMode() == PurgeNone)
+  if (AMgr.options.AnalysisPurgeOpt == PurgeNone)
     return false;
 
   // Is this the beginning of a basic block?
@@ -240,7 +257,7 @@ static bool shouldRemoveDeadBindings(AnalysisManager &AMgr,
     return true;
     
   // Run before processing a call.
-  if (CallEvent::mayBeInlined(S.getStmt()))
+  if (CallEvent::isCallStmt(S.getStmt()))
     return true;
 
   // Is this an expression that is consumed by another expression?  If so,
@@ -251,12 +268,12 @@ static bool shouldRemoveDeadBindings(AnalysisManager &AMgr,
 
 void ExprEngine::removeDead(ExplodedNode *Pred, ExplodedNodeSet &Out,
                             const Stmt *ReferenceStmt,
-                            const LocationContext *LC,
+                            const StackFrameContext *LC,
                             const Stmt *DiagnosticStmt,
                             ProgramPoint::Kind K) {
   assert((K == ProgramPoint::PreStmtPurgeDeadSymbolsKind ||
-          ReferenceStmt == 0) && "PreStmt is not generally supported by "
-                                 "the SymbolReaper yet");
+          ReferenceStmt == 0)
+          && "PostStmt is not generally supported by the SymbolReaper yet");
   NumRemoveDeadBindings++;
   CleanedState = Pred->getState();
   SymbolReaper SymReaper(LC, ReferenceStmt, SymMgr, getStoreManager());
@@ -276,8 +293,8 @@ void ExprEngine::removeDead(ExplodedNode *Pred, ExplodedNodeSet &Out,
     // Generate a CleanedNode that has the environment and store cleaned
     // up. Since no symbols are dead, we can optimize and not clean out
     // the constraint manager.
-    StmtNodeBuilder Bldr(Pred, Out, *currentBuilderContext);
-    Bldr.generateNode(DiagnosticStmt, Pred, CleanedState, false, &cleanupTag,K);
+    StmtNodeBuilder Bldr(Pred, Out, *currBldrCtx);
+    Bldr.generateNode(DiagnosticStmt, Pred, CleanedState, &cleanupTag, K);
 
   } else {
     // Call checkers with the non-cleaned state so that they could query the
@@ -289,7 +306,7 @@ void ExprEngine::removeDead(ExplodedNode *Pred, ExplodedNodeSet &Out,
     // For each node in CheckedSet, generate CleanedNodes that have the
     // environment, the store, and the constraints cleaned up but have the
     // user-supplied states as the predecessors.
-    StmtNodeBuilder Bldr(CheckedSet, Out, *currentBuilderContext);
+    StmtNodeBuilder Bldr(CheckedSet, Out, *currBldrCtx);
     for (ExplodedNodeSet::const_iterator
           I = CheckedSet.begin(), E = CheckedSet.end(); I != E; ++I) {
       ProgramStateRef CheckerState = (*I)->getState();
@@ -309,8 +326,7 @@ void ExprEngine::removeDead(ExplodedNode *Pred, ExplodedNodeSet &Out,
       // generate a transition to that state.
       ProgramStateRef CleanedCheckerSt =
         StateMgr.getPersistentStateWithGDM(CleanedState, CheckerState);
-      Bldr.generateNode(DiagnosticStmt, *I, CleanedCheckerSt, false,
-                        &cleanupTag, K);
+      Bldr.generateNode(DiagnosticStmt, *I, CleanedCheckerSt, &cleanupTag, K);
     }
   }
 }
@@ -320,17 +336,17 @@ void ExprEngine::ProcessStmt(const CFGStmt S,
   // Reclaim any unnecessary nodes in the ExplodedGraph.
   G.reclaimRecentlyAllocatedNodes();
 
-  currentStmt = S.getStmt();
+  currStmt = S.getStmt();
   PrettyStackTraceLoc CrashInfo(getContext().getSourceManager(),
-                                currentStmt->getLocStart(),
+                                currStmt->getLocStart(),
                                 "Error evaluating statement");
 
   // Remove dead bindings and symbols.
   EntryNode = Pred;
   ExplodedNodeSet CleanedStates;
   if (shouldRemoveDeadBindings(AMgr, S, Pred, EntryNode->getLocationContext())){
-    removeDead(EntryNode, CleanedStates, currentStmt,
-               Pred->getLocationContext(), currentStmt);
+    removeDead(EntryNode, CleanedStates, currStmt,
+               Pred->getStackFrame(), currStmt);
   } else
     CleanedStates.Add(EntryNode);
 
@@ -340,44 +356,45 @@ void ExprEngine::ProcessStmt(const CFGStmt S,
                                  E = CleanedStates.end(); I != E; ++I) {
     ExplodedNodeSet DstI;
     // Visit the statement.
-    Visit(currentStmt, *I, DstI);
+    Visit(currStmt, *I, DstI);
     Dst.insert(DstI);
   }
 
   // Enqueue the new nodes onto the work list.
-  Engine.enqueue(Dst, currentBuilderContext->getBlock(), currentStmtIdx);
+  Engine.enqueue(Dst, currBldrCtx->getBlock(), currStmtIdx);
 
   // NULL out these variables to cleanup.
   CleanedState = NULL;
   EntryNode = NULL;
-  currentStmt = 0;
+  currStmt = 0;
 }
 
 void ExprEngine::ProcessInitializer(const CFGInitializer Init,
                                     ExplodedNode *Pred) {
-  ExplodedNodeSet Dst;
-  NodeBuilder Bldr(Pred, Dst, *currentBuilderContext);
-
-  ProgramStateRef State = Pred->getState();
-
   const CXXCtorInitializer *BMI = Init.getInitializer();
 
   PrettyStackTraceLoc CrashInfo(getContext().getSourceManager(),
                                 BMI->getSourceLocation(),
                                 "Error evaluating initializer");
 
-  // We don't set EntryNode and currentStmt. And we don't clean up state.
+  // We don't set EntryNode and currStmt. And we don't clean up state.
   const StackFrameContext *stackFrame =
                            cast<StackFrameContext>(Pred->getLocationContext());
   const CXXConstructorDecl *decl =
                            cast<CXXConstructorDecl>(stackFrame->getDecl());
+
+  ProgramStateRef State = Pred->getState();
   SVal thisVal = State->getSVal(svalBuilder.getCXXThis(decl, stackFrame));
 
+  PostInitializer PP(BMI, stackFrame);
+  ExplodedNodeSet Tmp(Pred);
+
   // Evaluate the initializer, if necessary
   if (BMI->isAnyMemberInitializer()) {
     // Constructors build the object directly in the field,
     // but non-objects must be copied in from the initializer.
-    if (!isa<CXXConstructExpr>(BMI->getInit())) {
+    const Expr *Init = BMI->getInit();
+    if (!isa<CXXConstructExpr>(Init)) {
       SVal FieldLoc;
       if (BMI->isIndirectMemberInitializer())
         FieldLoc = State->getLValue(BMI->getIndirectMember(), thisVal);
@@ -385,22 +402,26 @@ void ExprEngine::ProcessInitializer(const CFGInitializer Init,
         FieldLoc = State->getLValue(BMI->getMember(), thisVal);
 
       SVal InitVal = State->getSVal(BMI->getInit(), stackFrame);
-      State = State->bindLoc(FieldLoc, InitVal);
+
+      Tmp.clear();
+      evalBind(Tmp, Init, Pred, FieldLoc, InitVal, /*isInit=*/true, &PP);
     }
   } else {
     assert(BMI->isBaseInitializer() || BMI->isDelegatingInitializer());
     // We already did all the work when visiting the CXXConstructExpr.
   }
 
-  // Construct a PostInitializer node whether the state changed or not,
+  // Construct PostInitializer nodes whether the state changed or not,
   // so that the diagnostics don't get confused.
-  PostInitializer PP(BMI, stackFrame);
-  // Builder automatically add the generated node to the deferred set,
-  // which are processed in the builder's dtor.
-  Bldr.generateNode(PP, State, Pred);
+  ExplodedNodeSet Dst;
+  NodeBuilder Bldr(Tmp, Dst, *currBldrCtx);
+  for (ExplodedNodeSet::iterator I = Tmp.begin(), E = Tmp.end(); I != E; ++I) {
+    ExplodedNode *N = *I;
+    Bldr.generateNode(PP, N->getState(), N);
+  }
 
   // Enqueue the new nodes onto the work list.
-  Engine.enqueue(Dst, currentBuilderContext->getBlock(), currentStmtIdx);
+  Engine.enqueue(Dst, currBldrCtx->getBlock(), currStmtIdx);
 }
 
 void ExprEngine::ProcessImplicitDtor(const CFGImplicitDtor D,
@@ -424,7 +445,7 @@ void ExprEngine::ProcessImplicitDtor(const CFGImplicitDtor D,
   }
 
   // Enqueue the new nodes onto the work list.
-  Engine.enqueue(Dst, currentBuilderContext->getBlock(), currentStmtIdx);
+  Engine.enqueue(Dst, currBldrCtx->getBlock(), currStmtIdx);
 }
 
 void ExprEngine::ProcessAutomaticObjDtor(const CFGAutomaticObjDtor Dtor,
@@ -441,7 +462,7 @@ void ExprEngine::ProcessAutomaticObjDtor(const CFGAutomaticObjDtor Dtor,
   Loc dest = state->getLValue(varDecl, Pred->getLocationContext());
 
   VisitCXXDestructor(varType, cast<loc::MemRegionVal>(dest).getRegion(),
-                     Dtor.getTriggerStmt(), Pred, Dst);
+                     Dtor.getTriggerStmt(), /*IsBase=*/false, Pred, Dst);
 }
 
 void ExprEngine::ProcessBaseDtor(const CFGBaseDtor D,
@@ -459,7 +480,7 @@ void ExprEngine::ProcessBaseDtor(const CFGBaseDtor D,
   SVal BaseVal = getStoreManager().evalDerivedToBase(ThisVal, BaseTy);
 
   VisitCXXDestructor(BaseTy, cast<loc::MemRegionVal>(BaseVal).getRegion(),
-                     CurDtor->getBody(), Pred, Dst);
+                     CurDtor->getBody(), /*IsBase=*/true, Pred, Dst);
 }
 
 void ExprEngine::ProcessMemberDtor(const CFGMemberDtor D,
@@ -475,7 +496,7 @@ void ExprEngine::ProcessMemberDtor(const CFGMemberDtor D,
 
   VisitCXXDestructor(Member->getType(),
                      cast<loc::MemRegionVal>(FieldVal).getRegion(),
-                     CurDtor->getBody(), Pred, Dst);
+                     CurDtor->getBody(), /*IsBase=*/false, Pred, Dst);
 }
 
 void ExprEngine::ProcessTemporaryDtor(const CFGTemporaryDtor D,
@@ -488,7 +509,7 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
                                 S->getLocStart(),
                                 "Error evaluating statement");
   ExplodedNodeSet Dst;
-  StmtNodeBuilder Bldr(Pred, DstTop, *currentBuilderContext);
+  StmtNodeBuilder Bldr(Pred, DstTop, *currBldrCtx);
 
   // Expressions to ignore.
   if (const Expr *Ex = dyn_cast<Expr>(S))
@@ -498,7 +519,7 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
   //  this check when we KNOW that there is no block-level subexpression.
   //  The motivation is that this check requires a hashtable lookup.
 
-  if (S != currentStmt && Pred->getLocationContext()->getCFG()->isBlkExpr(S))
+  if (S != currStmt && Pred->getLocationContext()->getCFG()->isBlkExpr(S))
     return;
 
   switch (S->getStmtClass()) {
@@ -521,21 +542,16 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
     case Stmt::CXXNoexceptExprClass:
     case Stmt::PackExpansionExprClass:
     case Stmt::SubstNonTypeTemplateParmPackExprClass:
+    case Stmt::FunctionParmPackExprClass:
     case Stmt::SEHTryStmtClass:
     case Stmt::SEHExceptStmtClass:
     case Stmt::LambdaExprClass:
     case Stmt::SEHFinallyStmtClass: {
-      const ExplodedNode *node = Bldr.generateNode(S, Pred, Pred->getState(),
-                                                   /* sink */ true);
-      Engine.addAbortedBlock(node, currentBuilderContext->getBlock());
+      const ExplodedNode *node = Bldr.generateSink(S, Pred, Pred->getState());
+      Engine.addAbortedBlock(node, currBldrCtx->getBlock());
       break;
     }
     
-    // We don't handle default arguments either yet, but we can fake it
-    // for now by just skipping them.
-    case Stmt::CXXDefaultArgExprClass:
-      break;
-
     case Stmt::ParenExprClass:
       llvm_unreachable("ParenExprs already handled.");
     case Stmt::GenericSelectionExprClass:
@@ -607,11 +623,6 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
     case Stmt::AtomicExprClass:
       // Fall through.
 
-    // Currently all handling of 'throw' just falls to the CFG.  We
-    // can consider doing more if necessary.
-    case Stmt::CXXThrowExprClass:
-      // Fall through.
-      
     // Cases we intentionally don't evaluate, since they don't need
     // to be explicitly evaluated.
     case Stmt::AddrLabelExprClass:
@@ -626,6 +637,7 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
     case Stmt::StringLiteralClass:
     case Stmt::ObjCStringLiteralClass:
     case Stmt::CXXBindTemporaryExprClass:
+    case Stmt::CXXDefaultArgExprClass:
     case Stmt::SubstNonTypeTemplateParmExprClass:
     case Stmt::CXXNullPtrLiteralExprClass: {
       Bldr.takeNodes(Pred);
@@ -647,7 +659,7 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
       getCheckerManager().runCheckersForPreStmt(preVisit, Pred, S, *this);
       
       ExplodedNodeSet Tmp;
-      StmtNodeBuilder Bldr2(preVisit, Tmp, *currentBuilderContext);
+      StmtNodeBuilder Bldr2(preVisit, Tmp, *currBldrCtx);
 
       const Expr *Ex = cast<Expr>(S);
       QualType resultType = Ex->getType();
@@ -656,9 +668,8 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
            it != et; ++it) {      
         ExplodedNode *N = *it;
         const LocationContext *LCtx = N->getLocationContext();
-        SVal result =
-          svalBuilder.getConjuredSymbolVal(0, Ex, LCtx, resultType, 
-                                 currentBuilderContext->getCurrentBlockCount());
+        SVal result = svalBuilder.conjureSymbolVal(0, Ex, LCtx, resultType,
+                                                   currBldrCtx->blockCount());
         ProgramStateRef state = N->getState()->BindExpr(Ex, LCtx, result);
         Bldr2.generateNode(S, N, state);
       }
@@ -674,9 +685,9 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
       Bldr.addNodes(Dst);
       break;
 
-    case Stmt::AsmStmtClass:
+    case Stmt::GCCAsmStmtClass:
       Bldr.takeNodes(Pred);
-      VisitAsmStmt(cast<AsmStmt>(S), Pred, Dst);
+      VisitGCCAsmStmt(cast<GCCAsmStmt>(S), Pred, Dst);
       Bldr.addNodes(Dst);
       break;
 
@@ -711,11 +722,11 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
 
       Bldr.takeNodes(Pred);
       
-      if (AMgr.shouldEagerlyAssume() &&
+      if (AMgr.options.eagerlyAssumeBinOpBifurcation &&
           (B->isRelationalOp() || B->isEqualityOp())) {
         ExplodedNodeSet Tmp;
         VisitBinaryOperator(cast<BinaryOperator>(S), Pred, Tmp);
-        evalEagerlyAssume(Dst, Tmp, cast<Expr>(S));
+        evalEagerlyAssumeBinOpBifurcation(Dst, Tmp, cast<Expr>(S));
       }
       else
         VisitBinaryOperator(cast<BinaryOperator>(S), Pred, Dst);
@@ -724,8 +735,26 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
       break;
     }
 
+    case Stmt::CXXOperatorCallExprClass: {
+      const CXXOperatorCallExpr *OCE = cast<CXXOperatorCallExpr>(S);
+
+      // For instance method operators, make sure the 'this' argument has a
+      // valid region.
+      const Decl *Callee = OCE->getCalleeDecl();
+      if (const CXXMethodDecl *MD = dyn_cast_or_null<CXXMethodDecl>(Callee)) {
+        if (MD->isInstance()) {
+          ProgramStateRef State = Pred->getState();
+          const LocationContext *LCtx = Pred->getLocationContext();
+          ProgramStateRef NewState =
+            createTemporaryRegionIfNeeded(State, LCtx, OCE->getArg(0));
+          if (NewState != State)
+            Pred = Bldr.generateNode(OCE, Pred, NewState, /*Tag=*/0,
+                                     ProgramPoint::PreStmtKind);
+        }
+      }
+      // FALLTHROUGH
+    }
     case Stmt::CallExprClass:
-    case Stmt::CXXOperatorCallExprClass:
     case Stmt::CXXMemberCallExprClass:
     case Stmt::UserDefinedLiteralClass: {
       Bldr.takeNodes(Pred);
@@ -846,12 +875,8 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
 
     case Expr::MaterializeTemporaryExprClass: {
       Bldr.takeNodes(Pred);
-      const MaterializeTemporaryExpr *Materialize
-                                            = cast<MaterializeTemporaryExpr>(S);
-      if (Materialize->getType()->isRecordType())
-        Dst.Add(Pred);
-      else
-        CreateCXXTemporaryObject(Materialize, Pred, Dst);
+      const MaterializeTemporaryExpr *MTE = cast<MaterializeTemporaryExpr>(S);
+      CreateCXXTemporaryObject(MTE, Pred, Dst);
       Bldr.addNodes(Dst);
       break;
     }
@@ -886,12 +911,12 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
       Bldr.addNodes(Dst);
       break;
 
-    case Stmt::ObjCAtThrowStmtClass: {
+    case Stmt::ObjCAtThrowStmtClass:
+    case Stmt::CXXThrowExprClass:
       // FIXME: This is not complete.  We basically treat @throw as
       // an abort.
-      Bldr.generateNode(S, Pred, Pred->getState());
+      Bldr.generateSink(S, Pred, Pred->getState());
       break;
-    }
 
     case Stmt::ReturnStmtClass:
       Bldr.takeNodes(Pred);
@@ -935,10 +960,10 @@ void ExprEngine::Visit(const Stmt *S, ExplodedNode *Pred,
     case Stmt::UnaryOperatorClass: {
       Bldr.takeNodes(Pred);
       const UnaryOperator *U = cast<UnaryOperator>(S);
-      if (AMgr.shouldEagerlyAssume() && (U->getOpcode() == UO_LNot)) {
+      if (AMgr.options.eagerlyAssumeBinOpBifurcation && (U->getOpcode() == UO_LNot)) {
         ExplodedNodeSet Tmp;
         VisitUnaryOperator(U, Pred, Tmp);
-        evalEagerlyAssume(Dst, Tmp, U);
+        evalEagerlyAssumeBinOpBifurcation(Dst, Tmp, U);
       }
       else
         VisitUnaryOperator(U, Pred, Dst);
@@ -1030,19 +1055,18 @@ bool ExprEngine::replayWithoutInlining(ExplodedNode *N,
 
 /// Block entrance.  (Update counters).
 void ExprEngine::processCFGBlockEntrance(const BlockEdge &L,
-                                         NodeBuilderWithSinks &nodeBuilder) {
+                                         NodeBuilderWithSinks &nodeBuilder, 
+                                         ExplodedNode *Pred) {
   
   // FIXME: Refactor this into a checker.
-  ExplodedNode *pred = nodeBuilder.getContext().getPred();
-  
-  if (nodeBuilder.getContext().getCurrentBlockCount() >= AMgr.getMaxVisit()) {
+  if (nodeBuilder.getContext().blockCount() >= AMgr.options.maxBlockVisitOnPath) {
     static SimpleProgramPointTag tag("ExprEngine : Block count exceeded");
     const ExplodedNode *Sink =
-                   nodeBuilder.generateNode(pred->getState(), pred, &tag, true);
+                   nodeBuilder.generateSink(Pred->getState(), Pred, &tag);
 
     // Check if we stopped at the top level function or not.
     // Root node should have the location context of the top most function.
-    const LocationContext *CalleeLC = pred->getLocation().getLocationContext();
+    const LocationContext *CalleeLC = Pred->getLocation().getLocationContext();
     const LocationContext *CalleeSF = CalleeLC->getCurrentStackFrame();
     const LocationContext *RootLC =
                         (*G.roots_begin())->getLocation().getLocationContext();
@@ -1053,7 +1077,8 @@ void ExprEngine::processCFGBlockEntrance(const BlockEdge &L,
       // no-inlining policy in the state and enqueuing the new work item on
       // the list. Replay should almost never fail. Use the stats to catch it
       // if it does.
-      if ((!AMgr.NoRetryExhausted && replayWithoutInlining(pred, CalleeLC)))
+      if ((!AMgr.options.NoRetryExhausted &&
+           replayWithoutInlining(Pred, CalleeLC)))
         return;
       NumMaxBlockCountReachedInInlined++;
     } else
@@ -1155,7 +1180,7 @@ void ExprEngine::processBranch(const Stmt *Condition, const Stmt *Term,
                                ExplodedNodeSet &Dst,
                                const CFGBlock *DstT,
                                const CFGBlock *DstF) {
-  currentBuilderContext = &BldCtx;
+  currBldrCtx = &BldCtx;
 
   // Check for NULL conditions; e.g. "for(;;)"
   if (!Condition) {
@@ -1238,7 +1263,7 @@ void ExprEngine::processBranch(const Stmt *Condition, const Stmt *Term,
         builder.markInfeasible(false);
     }
   }
-  currentBuilderContext = 0;
+  currBldrCtx = 0;
 }
 
 /// processIndirectGoto - Called by CoreEngine.  Used to generate successor
@@ -1287,10 +1312,25 @@ void ExprEngine::processIndirectGoto(IndirectGotoNodeBuilder &builder) {
 
 /// ProcessEndPath - Called by CoreEngine.  Used to generate end-of-path
 ///  nodes when the control reaches the end of a function.
-void ExprEngine::processEndOfFunction(NodeBuilderContext& BC) {
-  StateMgr.EndPath(BC.Pred->getState());
+void ExprEngine::processEndOfFunction(NodeBuilderContext& BC,
+                                      ExplodedNode *Pred) {
+  StateMgr.EndPath(Pred->getState());
+
   ExplodedNodeSet Dst;
-  getCheckerManager().runCheckersForEndPath(BC, Dst, *this);
+  if (Pred->getLocationContext()->inTopFrame()) {
+    // Remove dead symbols.
+    ExplodedNodeSet AfterRemovedDead;
+    removeDeadOnEndOfFunction(BC, Pred, AfterRemovedDead);
+
+    // Notify checkers.
+    for (ExplodedNodeSet::iterator I = AfterRemovedDead.begin(),
+        E = AfterRemovedDead.end(); I != E; ++I) {
+      getCheckerManager().runCheckersForEndPath(BC, Dst, *I, *this);
+    }
+  } else {
+    getCheckerManager().runCheckersForEndPath(BC, Dst, Pred, *this);
+  }
+
   Engine.enqueueEndOfFunction(Dst);
 }
 
@@ -1404,7 +1444,7 @@ void ExprEngine::processSwitch(SwitchNodeBuilder& builder) {
 void ExprEngine::VisitCommonDeclRefExpr(const Expr *Ex, const NamedDecl *D,
                                         ExplodedNode *Pred,
                                         ExplodedNodeSet &Dst) {
-  StmtNodeBuilder Bldr(Pred, Dst, *currentBuilderContext);
+  StmtNodeBuilder Bldr(Pred, Dst, *currBldrCtx);
 
   ProgramStateRef state = Pred->getState();
   const LocationContext *LCtx = Pred->getLocationContext();
@@ -1422,7 +1462,7 @@ void ExprEngine::VisitCommonDeclRefExpr(const Expr *Ex, const NamedDecl *D,
         V = UnknownVal();
     }
 
-    Bldr.generateNode(Ex, Pred, state->BindExpr(Ex, LCtx, V), false, 0,
+    Bldr.generateNode(Ex, Pred, state->BindExpr(Ex, LCtx, V), 0,
                       ProgramPoint::PostLValueKind);
     return;
   }
@@ -1434,19 +1474,23 @@ void ExprEngine::VisitCommonDeclRefExpr(const Expr *Ex, const NamedDecl *D,
   }
   if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
     SVal V = svalBuilder.getFunctionPointer(FD);
-    Bldr.generateNode(Ex, Pred, state->BindExpr(Ex, LCtx, V), false, 0,
+    Bldr.generateNode(Ex, Pred, state->BindExpr(Ex, LCtx, V), 0,
                       ProgramPoint::PostLValueKind);
     return;
   }
   if (isa<FieldDecl>(D)) {
-    // FIXME: Compute lvalue of fields.
-    Bldr.generateNode(Ex, Pred, state->BindExpr(Ex, LCtx, UnknownVal()),
-		      false, 0, ProgramPoint::PostLValueKind);
+    // FIXME: Compute lvalue of field pointers-to-member.
+    // Right now we just use a non-null void pointer, so that it gives proper
+    // results in boolean contexts.
+    SVal V = svalBuilder.conjureSymbolVal(Ex, LCtx, getContext().VoidPtrTy,
+                                          currBldrCtx->blockCount());
+    state = state->assume(cast<DefinedOrUnknownSVal>(V), true);
+    Bldr.generateNode(Ex, Pred, state->BindExpr(Ex, LCtx, V), 0,
+		      ProgramPoint::PostLValueKind);
     return;
   }
 
-  assert (false &&
-          "ValueDecl support for this ValueDecl not implemented.");
+  llvm_unreachable("Support for this Decl not implemented.");
 }
 
 /// VisitArraySubscriptExpr - Transfer function for array accesses
@@ -1461,7 +1505,7 @@ void ExprEngine::VisitLvalArraySubscriptExpr(const ArraySubscriptExpr *A,
   ExplodedNodeSet checkerPreStmt;
   getCheckerManager().runCheckersForPreStmt(checkerPreStmt, Pred, A, *this);
 
-  StmtNodeBuilder Bldr(checkerPreStmt, Dst, *currentBuilderContext);
+  StmtNodeBuilder Bldr(checkerPreStmt, Dst, *currBldrCtx);
 
   for (ExplodedNodeSet::iterator it = checkerPreStmt.begin(),
                                  ei = checkerPreStmt.end(); it != ei; ++it) {
@@ -1471,8 +1515,8 @@ void ExprEngine::VisitLvalArraySubscriptExpr(const ArraySubscriptExpr *A,
                               state->getSVal(Idx, LCtx),
                               state->getSVal(Base, LCtx));
     assert(A->isGLValue());
-    Bldr.generateNode(A, *it, state->BindExpr(A, LCtx, V),
-                      false, 0, ProgramPoint::PostLValueKind);
+    Bldr.generateNode(A, *it, state->BindExpr(A, LCtx, V), 0, 
+                      ProgramPoint::PostLValueKind);
   }
 }
 
@@ -1480,52 +1524,40 @@ void ExprEngine::VisitLvalArraySubscriptExpr(const ArraySubscriptExpr *A,
 void ExprEngine::VisitMemberExpr(const MemberExpr *M, ExplodedNode *Pred,
                                  ExplodedNodeSet &TopDst) {
 
-  StmtNodeBuilder Bldr(Pred, TopDst, *currentBuilderContext);
+  StmtNodeBuilder Bldr(Pred, TopDst, *currBldrCtx);
   ExplodedNodeSet Dst;
-  Decl *member = M->getMemberDecl();
+  ValueDecl *Member = M->getMemberDecl();
 
-  if (VarDecl *VD = dyn_cast<VarDecl>(member)) {
-    assert(M->isGLValue());
+  // Handle static member variables and enum constants accessed via
+  // member syntax.
+  if (isa<VarDecl>(Member) || isa<EnumConstantDecl>(Member)) {
     Bldr.takeNodes(Pred);
-    VisitCommonDeclRefExpr(M, VD, Pred, Dst);
+    VisitCommonDeclRefExpr(M, Member, Pred, Dst);
     Bldr.addNodes(Dst);
     return;
   }
 
-  // Handle C++ method calls.
-  if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(member)) {
-    Bldr.takeNodes(Pred);
-    SVal MDVal = svalBuilder.getFunctionPointer(MD);
-    ProgramStateRef state =
-      Pred->getState()->BindExpr(M, Pred->getLocationContext(), MDVal);
-    Bldr.generateNode(M, Pred, state);
-    return;
-  }
+  ProgramStateRef state = Pred->getState();
+  const LocationContext *LCtx = Pred->getLocationContext();
+  Expr *BaseExpr = M->getBase();
 
+  // Handle C++ method calls.
+  if (const CXXMethodDecl *MD = dyn_cast<CXXMethodDecl>(Member)) {
+    if (MD->isInstance())
+      state = createTemporaryRegionIfNeeded(state, LCtx, BaseExpr);
 
-  FieldDecl *field = dyn_cast<FieldDecl>(member);
-  if (!field) // FIXME: skipping member expressions for non-fields
-    return;
+    SVal MDVal = svalBuilder.getFunctionPointer(MD);
+    state = state->BindExpr(M, LCtx, MDVal);
 
-  Expr *baseExpr = M->getBase()->IgnoreParens();
-  ProgramStateRef state = Pred->getState();
-  const LocationContext *LCtx = Pred->getLocationContext();
-  SVal baseExprVal = state->getSVal(baseExpr, Pred->getLocationContext());
-  if (isa<nonloc::LazyCompoundVal>(baseExprVal) ||
-      isa<nonloc::CompoundVal>(baseExprVal) ||
-      // FIXME: This can originate by conjuring a symbol for an unknown
-      // temporary struct object, see test/Analysis/fields.c:
-      // (p = getit()).x
-      isa<nonloc::SymbolVal>(baseExprVal)) {
-    Bldr.generateNode(M, Pred, state->BindExpr(M, LCtx, UnknownVal()));
+    Bldr.generateNode(M, Pred, state);
     return;
   }
 
-  // FIXME: Should we insert some assumption logic in here to determine
-  // if "Base" is a valid piece of memory?  Before we put this assumption
-  // later when using FieldOffset lvals (which we no longer have).
+  // Handle regular struct fields / member variables.
+  state = createTemporaryRegionIfNeeded(state, LCtx, BaseExpr);
+  SVal baseExprVal = state->getSVal(BaseExpr, LCtx);
 
-  // For all other cases, compute an lvalue.    
+  FieldDecl *field = cast<FieldDecl>(Member);
   SVal L = state->getLValue(field, baseExprVal);
   if (M->isGLValue()) {
     if (field->getType()->isReferenceType()) {
@@ -1535,7 +1567,7 @@ void ExprEngine::VisitMemberExpr(const MemberExpr *M, ExplodedNode *Pred,
         L = UnknownVal();
     }
 
-    Bldr.generateNode(M, Pred, state->BindExpr(M, LCtx, L), false, 0,
+    Bldr.generateNode(M, Pred, state->BindExpr(M, LCtx, L), 0,
                       ProgramPoint::PostLValueKind);
   } else {
     Bldr.takeNodes(Pred);
@@ -1548,40 +1580,48 @@ void ExprEngine::VisitMemberExpr(const MemberExpr *M, ExplodedNode *Pred,
 ///  This method is used by evalStore and (soon) VisitDeclStmt, and others.
 void ExprEngine::evalBind(ExplodedNodeSet &Dst, const Stmt *StoreE,
                           ExplodedNode *Pred,
-                          SVal location, SVal Val, bool atDeclInit) {
+                          SVal location, SVal Val,
+                          bool atDeclInit, const ProgramPoint *PP) {
+
+  const LocationContext *LC = Pred->getLocationContext();
+  PostStmt PS(StoreE, LC);
+  if (!PP)
+    PP = &PS;
 
   // Do a previsit of the bind.
   ExplodedNodeSet CheckedSet;
   getCheckerManager().runCheckersForBind(CheckedSet, Pred, location, Val,
-                                         StoreE, *this,
-                                         ProgramPoint::PostStmtKind);
+                                         StoreE, *this, *PP);
 
+  // If the location is not a 'Loc', it will already be handled by
+  // the checkers.  There is nothing left to do.
+  if (!isa<Loc>(location)) {
+    Dst = CheckedSet;
+    return;
+  }
+  
   ExplodedNodeSet TmpDst;
-  StmtNodeBuilder Bldr(CheckedSet, TmpDst, *currentBuilderContext);
+  StmtNodeBuilder Bldr(CheckedSet, TmpDst, *currBldrCtx);
 
-  const LocationContext *LC = Pred->getLocationContext();
   for (ExplodedNodeSet::iterator I = CheckedSet.begin(), E = CheckedSet.end();
        I!=E; ++I) {
     ExplodedNode *PredI = *I;
     ProgramStateRef state = PredI->getState();
-
-    if (atDeclInit) {
-      const VarRegion *VR =
-        cast<VarRegion>(cast<loc::MemRegionVal>(location).getRegion());
-
-      state = state->bindDecl(VR, Val);
-    } else {
-      state = state->bindLoc(location, Val);
-    }
-
+    
+    // When binding the value, pass on the hint that this is a initialization.
+    // For initializations, we do not need to inform clients of region
+    // changes.
+    state = state->bindLoc(cast<Loc>(location),
+                           Val, /* notifyChanges = */ !atDeclInit);
+    
     const MemRegion *LocReg = 0;
-    if (loc::MemRegionVal *LocRegVal = dyn_cast<loc::MemRegionVal>(&location))
+    if (loc::MemRegionVal *LocRegVal = dyn_cast<loc::MemRegionVal>(&location)) {
       LocReg = LocRegVal->getRegion();
-
+    }
+    
     const ProgramPoint L = PostStore(StoreE, LC, LocReg, 0);
-    Bldr.generateNode(L, PredI, state, false);
+    Bldr.generateNode(L, state, PredI);
   }
-
   Dst.insert(TmpDst);
 }
 
@@ -1671,7 +1711,7 @@ void ExprEngine::evalLoadCommon(ExplodedNodeSet &Dst,
   if (Tmp.empty())
     return;
 
-  StmtNodeBuilder Bldr(Tmp, Dst, *currentBuilderContext);
+  StmtNodeBuilder Bldr(Tmp, Dst, *currBldrCtx);
   if (location.isUndef())
     return;
 
@@ -1684,8 +1724,7 @@ void ExprEngine::evalLoadCommon(ExplodedNodeSet &Dst,
       // This is important.  We must nuke the old binding.
       Bldr.generateNode(NodeEx, *NI,
                         state->BindExpr(BoundEx, LCtx, UnknownVal()),
-                        false, tag,
-                        ProgramPoint::PostLoadKind);
+                        tag, ProgramPoint::PostLoadKind);
     }
     else {
       if (LoadTy.isNull())
@@ -1693,7 +1732,7 @@ void ExprEngine::evalLoadCommon(ExplodedNodeSet &Dst,
       SVal V = state->getSVal(cast<Loc>(location), LoadTy);
       Bldr.generateNode(NodeEx, *NI,
                         state->bindExprAndLocation(BoundEx, LCtx, location, V),
-                        false, tag, ProgramPoint::PostLoadKind);
+                        tag, ProgramPoint::PostLoadKind);
     }
   }
 }
@@ -1706,7 +1745,7 @@ void ExprEngine::evalLocation(ExplodedNodeSet &Dst,
                               SVal location,
                               const ProgramPointTag *tag,
                               bool isLoad) {
-  StmtNodeBuilder BldrTop(Pred, Dst, *currentBuilderContext);
+  StmtNodeBuilder BldrTop(Pred, Dst, *currBldrCtx);
   // Early checks for performance reason.
   if (location.isUnknown()) {
     return;
@@ -1714,7 +1753,7 @@ void ExprEngine::evalLocation(ExplodedNodeSet &Dst,
 
   ExplodedNodeSet Src;
   BldrTop.takeNodes(Pred);
-  StmtNodeBuilder Bldr(Pred, Src, *currentBuilderContext);
+  StmtNodeBuilder Bldr(Pred, Src, *currBldrCtx);
   if (Pred->getState() != state) {
     // Associate this new state with an ExplodedNode.
     // FIXME: If I pass null tag, the graph is incorrect, e.g for
@@ -1725,9 +1764,8 @@ void ExprEngine::evalLocation(ExplodedNodeSet &Dst,
     // instead "int *p" is noted as
     // "Variable 'p' initialized to a null pointer value"
     
-    // FIXME: why is 'tag' not used instead of etag?
-    static SimpleProgramPointTag etag("ExprEngine: Location");
-    Bldr.generateNode(NodeEx, Pred, state, false, &etag);
+    static SimpleProgramPointTag tag("ExprEngine: Location");
+    Bldr.generateNode(NodeEx, Pred, state, &tag);
   }
   ExplodedNodeSet Tmp;
   getCheckerManager().runCheckersForLocation(Tmp, Src, location, isLoad,
@@ -1736,16 +1774,18 @@ void ExprEngine::evalLocation(ExplodedNodeSet &Dst,
 }
 
 std::pair<const ProgramPointTag *, const ProgramPointTag*>
-ExprEngine::getEagerlyAssumeTags() {
+ExprEngine::geteagerlyAssumeBinOpBifurcationTags() {
   static SimpleProgramPointTag
-         EagerlyAssumeTrue("ExprEngine : Eagerly Assume True"),
-         EagerlyAssumeFalse("ExprEngine : Eagerly Assume False");
-  return std::make_pair(&EagerlyAssumeTrue, &EagerlyAssumeFalse);
+         eagerlyAssumeBinOpBifurcationTrue("ExprEngine : Eagerly Assume True"),
+         eagerlyAssumeBinOpBifurcationFalse("ExprEngine : Eagerly Assume False");
+  return std::make_pair(&eagerlyAssumeBinOpBifurcationTrue,
+                        &eagerlyAssumeBinOpBifurcationFalse);
 }
 
-void ExprEngine::evalEagerlyAssume(ExplodedNodeSet &Dst, ExplodedNodeSet &Src,
-                                   const Expr *Ex) {
-  StmtNodeBuilder Bldr(Src, Dst, *currentBuilderContext);
+void ExprEngine::evalEagerlyAssumeBinOpBifurcation(ExplodedNodeSet &Dst,
+                                                   ExplodedNodeSet &Src,
+                                                   const Expr *Ex) {
+  StmtNodeBuilder Bldr(Src, Dst, *currBldrCtx);
   
   for (ExplodedNodeSet::iterator I=Src.begin(), E=Src.end(); I!=E; ++I) {
     ExplodedNode *Pred = *I;
@@ -1762,28 +1802,28 @@ void ExprEngine::evalEagerlyAssume(ExplodedNodeSet &Dst, ExplodedNodeSet &Src,
     nonloc::SymbolVal *SEV = dyn_cast<nonloc::SymbolVal>(&V);
     if (SEV && SEV->isExpression()) {
       const std::pair<const ProgramPointTag *, const ProgramPointTag*> &tags =
-        getEagerlyAssumeTags();
+        geteagerlyAssumeBinOpBifurcationTags();
 
       // First assume that the condition is true.
       if (ProgramStateRef StateTrue = state->assume(*SEV, true)) {
         SVal Val = svalBuilder.makeIntVal(1U, Ex->getType());        
         StateTrue = StateTrue->BindExpr(Ex, Pred->getLocationContext(), Val);
-        Bldr.generateNode(Ex, Pred, StateTrue, false, tags.first);
+        Bldr.generateNode(Ex, Pred, StateTrue, tags.first);
       }
 
       // Next, assume that the condition is false.
       if (ProgramStateRef StateFalse = state->assume(*SEV, false)) {
         SVal Val = svalBuilder.makeIntVal(0U, Ex->getType());
         StateFalse = StateFalse->BindExpr(Ex, Pred->getLocationContext(), Val);
-        Bldr.generateNode(Ex, Pred, StateFalse, false, tags.second);
+        Bldr.generateNode(Ex, Pred, StateFalse, tags.second);
       }
     }
   }
 }
 
-void ExprEngine::VisitAsmStmt(const AsmStmt *A, ExplodedNode *Pred,
-                              ExplodedNodeSet &Dst) {
-  StmtNodeBuilder Bldr(Pred, Dst, *currentBuilderContext);
+void ExprEngine::VisitGCCAsmStmt(const GCCAsmStmt *A, ExplodedNode *Pred,
+                                 ExplodedNodeSet &Dst) {
+  StmtNodeBuilder Bldr(Pred, Dst, *currBldrCtx);
   // We have processed both the inputs and the outputs.  All of the outputs
   // should evaluate to Locs.  Nuke all of their values.
 
@@ -1793,7 +1833,7 @@ void ExprEngine::VisitAsmStmt(const AsmStmt *A, ExplodedNode *Pred,
 
   ProgramStateRef state = Pred->getState();
 
-  for (AsmStmt::const_outputs_iterator OI = A->begin_outputs(),
+  for (GCCAsmStmt::const_outputs_iterator OI = A->begin_outputs(),
        OE = A->end_outputs(); OI != OE; ++OI) {
     SVal X = state->getSVal(*OI, Pred->getLocationContext());
     assert (!isa<NonLoc>(X));  // Should be an Lval, or unknown, undef.
@@ -1807,7 +1847,7 @@ void ExprEngine::VisitAsmStmt(const AsmStmt *A, ExplodedNode *Pred,
 
 void ExprEngine::VisitMSAsmStmt(const MSAsmStmt *A, ExplodedNode *Pred,
                                 ExplodedNodeSet &Dst) {
-  StmtNodeBuilder Bldr(Pred, Dst, *currentBuilderContext);
+  StmtNodeBuilder Bldr(Pred, Dst, *currBldrCtx);
   Bldr.generateNode(A, Pred, Pred->getState());
 }
 
@@ -1932,7 +1972,7 @@ struct DOTGraphTraits<ExplodedNode*> :
         if (StmtPoint *L = dyn_cast<StmtPoint>(&Loc)) {
           const Stmt *S = L->getStmt();
 
-          Out << S->getStmtClassName() << ' ' << (void*) S << ' ';
+          Out << S->getStmtClassName() << ' ' << (const void*) S << ' ';
           LangOptions LO; // FIXME.
           S->printPretty(Out, 0, PrintingPolicy(LO));
           printLocation(Out, S->getLocStart());
@@ -2038,8 +2078,8 @@ struct DOTGraphTraits<ExplodedNode*> :
     }
 
     ProgramStateRef state = N->getState();
-    Out << "\\|StateID: " << (void*) state.getPtr()
-        << " NodeID: " << (void*) N << "\\|";
+    Out << "\\|StateID: " << (const void*) state.getPtr()
+        << " NodeID: " << (const void*) N << "\\|";
     state->printDOT(Out);
 
     Out << "\\l";    
diff --git a/lib/StaticAnalyzer/Core/ExprEngineC.cpp b/lib/StaticAnalyzer/Core/ExprEngineC.cpp
index 46cba81..00b2f4a 100644
--- a/lib/StaticAnalyzer/Core/ExprEngineC.cpp
+++ b/lib/StaticAnalyzer/Core/ExprEngineC.cpp
@@ -45,8 +45,8 @@ void ExprEngine::VisitBinaryOperator(const BinaryOperator* B,
       // EXPERIMENTAL: "Conjured" symbols.
       // FIXME: Handle structs.
       if (RightV.isUnknown()) {
-        unsigned Count = currentBuilderContext->getCurrentBlockCount();
-        RightV = svalBuilder.getConjuredSymbolVal(NULL, B->getRHS(), LCtx, Count);
+        unsigned Count = currBldrCtx->blockCount();
+        RightV = svalBuilder.conjureSymbolVal(0, B->getRHS(), LCtx, Count);
       }
       // Simulate the effects of a "store":  bind the value of the RHS
       // to the L-Value represented by the LHS.
@@ -57,7 +57,7 @@ void ExprEngine::VisitBinaryOperator(const BinaryOperator* B,
     }
       
     if (!B->isAssignmentOp()) {
-      StmtNodeBuilder Bldr(*it, Tmp2, *currentBuilderContext);
+      StmtNodeBuilder Bldr(*it, Tmp2, *currBldrCtx);
 
       if (B->isAdditiveOp()) {
         // If one of the operands is a location, conjure a symbol for the other
@@ -65,16 +65,16 @@ void ExprEngine::VisitBinaryOperator(const BinaryOperator* B,
         // results in an ElementRegion.
         // TODO: This can be removed after we enable history tracking with
         // SymSymExpr.
-        unsigned Count = currentBuilderContext->getCurrentBlockCount();
+        unsigned Count = currBldrCtx->blockCount();
         if (isa<Loc>(LeftV) &&
             RHS->getType()->isIntegerType() && RightV.isUnknown()) {
-          RightV = svalBuilder.getConjuredSymbolVal(RHS, LCtx,
-                                                    RHS->getType(), Count);
+          RightV = svalBuilder.conjureSymbolVal(RHS, LCtx, RHS->getType(),
+                                                Count);
         }
         if (isa<Loc>(RightV) &&
             LHS->getType()->isIntegerType() && LeftV.isUnknown()) {
-          LeftV = svalBuilder.getConjuredSymbolVal(LHS, LCtx,
-                                                   LHS->getType(), Count);
+          LeftV = svalBuilder.conjureSymbolVal(LHS, LCtx, LHS->getType(),
+                                               Count);
         }
       }
 
@@ -145,15 +145,11 @@ void ExprEngine::VisitBinaryOperator(const BinaryOperator* B,
       SVal LHSVal;
       
       if (Result.isUnknown()) {
-        
-        unsigned Count = currentBuilderContext->getCurrentBlockCount();
-        
         // The symbolic value is actually for the type of the left-hand side
         // expression, not the computation type, as this is the value the
         // LValue on the LHS will bind to.
-        LHSVal = svalBuilder.getConjuredSymbolVal(NULL, B->getRHS(), LCtx,
-						  LTy, Count);
-        
+        LHSVal = svalBuilder.conjureSymbolVal(0, B->getRHS(), LCtx, LTy,
+                                              currBldrCtx->blockCount());
         // However, we need to convert the symbol to the computation type.
         Result = svalBuilder.evalCast(LHSVal, CTy, LTy);
       }
@@ -208,11 +204,10 @@ void ExprEngine::VisitBlockExpr(const BlockExpr *BE, ExplodedNode *Pred,
   }
   
   ExplodedNodeSet Tmp;
-  StmtNodeBuilder Bldr(Pred, Tmp, *currentBuilderContext);
+  StmtNodeBuilder Bldr(Pred, Tmp, *currBldrCtx);
   Bldr.generateNode(BE, Pred,
                     State->BindExpr(BE, Pred->getLocationContext(), V),
-                    false, 0,
-                    ProgramPoint::PostLValueKind);
+                    0, ProgramPoint::PostLValueKind);
   
   // FIXME: Move all post/pre visits to ::Visit().
   getCheckerManager().runCheckersForPostStmt(Dst, Tmp, BE, *this);
@@ -242,12 +237,14 @@ void ExprEngine::VisitCast(const CastExpr *CastE, const Expr *Ex,
   if (const ExplicitCastExpr *ExCast=dyn_cast_or_null<ExplicitCastExpr>(CastE))
     T = ExCast->getTypeAsWritten();
   
-  StmtNodeBuilder Bldr(dstPreStmt, Dst, *currentBuilderContext);
+  StmtNodeBuilder Bldr(dstPreStmt, Dst, *currBldrCtx);
   for (ExplodedNodeSet::iterator I = dstPreStmt.begin(), E = dstPreStmt.end();
        I != E; ++I) {
     
     Pred = *I;
-    
+    ProgramStateRef state = Pred->getState();
+    const LocationContext *LCtx = Pred->getLocationContext();
+
     switch (CastE->getCastKind()) {
       case CK_LValueToRValue:
         llvm_unreachable("LValueToRValue casts handled earlier.");
@@ -267,7 +264,10 @@ void ExprEngine::VisitCast(const CastExpr *CastE, const Expr *Ex,
       case CK_NonAtomicToAtomic:
         // True no-ops.
       case CK_NoOp:
-      case CK_FunctionToPointerDecay: {
+      case CK_ConstructorConversion:
+      case CK_UserDefinedConversion:
+      case CK_FunctionToPointerDecay:
+      case CK_BuiltinFnToFnPtr: {
         // Copy the SVal of Ex to CastE.
         ProgramStateRef state = Pred->getState();
         const LocationContext *LCtx = Pred->getLocationContext();
@@ -276,6 +276,9 @@ void ExprEngine::VisitCast(const CastExpr *CastE, const Expr *Ex,
         Bldr.generateNode(CastE, Pred, state);
         continue;
       }
+      case CK_MemberPointerToBoolean:
+        // FIXME: For now, member pointers are represented by void *.
+        // FALLTHROUGH
       case CK_Dependent:
       case CK_ArrayToPointerDecay:
       case CK_BitCast:
@@ -304,8 +307,6 @@ void ExprEngine::VisitCast(const CastExpr *CastE, const Expr *Ex,
       case CK_AnyPointerToBlockPointerCast:  
       case CK_ObjCObjectLValueCast: {
         // Delegate to SValBuilder to process.
-        ProgramStateRef state = Pred->getState();
-        const LocationContext *LCtx = Pred->getLocationContext();
         SVal V = state->getSVal(Ex, LCtx);
         V = svalBuilder.evalCast(V, T, ExTy);
         state = state->BindExpr(CastE, LCtx, V);
@@ -315,8 +316,6 @@ void ExprEngine::VisitCast(const CastExpr *CastE, const Expr *Ex,
       case CK_DerivedToBase:
       case CK_UncheckedDerivedToBase: {
         // For DerivedToBase cast, delegate to the store manager.
-        ProgramStateRef state = Pred->getState();
-        const LocationContext *LCtx = Pred->getLocationContext();
         SVal val = state->getSVal(Ex, LCtx);
         val = getStoreManager().evalDerivedToBase(val, CastE);
         state = state->BindExpr(CastE, LCtx, val);
@@ -325,8 +324,6 @@ void ExprEngine::VisitCast(const CastExpr *CastE, const Expr *Ex,
       }
       // Handle C++ dyn_cast.
       case CK_Dynamic: {
-        ProgramStateRef state = Pred->getState();
-        const LocationContext *LCtx = Pred->getLocationContext();
         SVal val = state->getSVal(Ex, LCtx);
 
         // Compute the type of the result.
@@ -347,7 +344,7 @@ void ExprEngine::VisitCast(const CastExpr *CastE, const Expr *Ex,
           if (T->isReferenceType()) {
             // A bad_cast exception is thrown if input value is a reference.
             // Currently, we model this, by generating a sink.
-            Bldr.generateNode(CastE, Pred, state, true);
+            Bldr.generateSink(CastE, Pred, state);
             continue;
           } else {
             // If the cast fails on a pointer, bind to 0.
@@ -356,9 +353,9 @@ void ExprEngine::VisitCast(const CastExpr *CastE, const Expr *Ex,
         } else {
           // If we don't know if the cast succeeded, conjure a new symbol.
           if (val.isUnknown()) {
-            DefinedOrUnknownSVal NewSym = svalBuilder.getConjuredSymbolVal(NULL,
-                                 CastE, LCtx, resultType,
-                                 currentBuilderContext->getCurrentBlockCount());
+            DefinedOrUnknownSVal NewSym =
+              svalBuilder.conjureSymbolVal(0, CastE, LCtx, resultType,
+                                           currBldrCtx->blockCount());
             state = state->BindExpr(CastE, LCtx, NewSym);
           } else 
             // Else, bind to the derived region value.
@@ -367,27 +364,29 @@ void ExprEngine::VisitCast(const CastExpr *CastE, const Expr *Ex,
         Bldr.generateNode(CastE, Pred, state);
         continue;
       }
+      case CK_NullToMemberPointer: {
+        // FIXME: For now, member pointers are represented by void *.
+        SVal V = svalBuilder.makeIntValWithPtrWidth(0, true);
+        state = state->BindExpr(CastE, LCtx, V);
+        Bldr.generateNode(CastE, Pred, state);
+        continue;
+      }
       // Various C++ casts that are not handled yet.
       case CK_ToUnion:
       case CK_BaseToDerived:
-      case CK_NullToMemberPointer:
       case CK_BaseToDerivedMemberPointer:
       case CK_DerivedToBaseMemberPointer:
       case CK_ReinterpretMemberPointer:
-      case CK_UserDefinedConversion:
-      case CK_ConstructorConversion:
       case CK_VectorSplat:
-      case CK_MemberPointerToBoolean:
       case CK_LValueBitCast: {
         // Recover some path-sensitivty by conjuring a new value.
         QualType resultType = CastE->getType();
         if (CastE->isGLValue())
           resultType = getContext().getPointerType(resultType);
-        const LocationContext *LCtx = Pred->getLocationContext();
-        SVal result = svalBuilder.getConjuredSymbolVal(NULL, CastE, LCtx,
-                    resultType, currentBuilderContext->getCurrentBlockCount());
-        ProgramStateRef state = Pred->getState()->BindExpr(CastE, LCtx,
-                                                               result);
+        SVal result = svalBuilder.conjureSymbolVal(0, CastE, LCtx,
+                                                   resultType,
+                                                   currBldrCtx->blockCount());
+        state = state->BindExpr(CastE, LCtx, result);
         Bldr.generateNode(CastE, Pred, state);
         continue;
       }
@@ -398,7 +397,7 @@ void ExprEngine::VisitCast(const CastExpr *CastE, const Expr *Ex,
 void ExprEngine::VisitCompoundLiteralExpr(const CompoundLiteralExpr *CL,
                                           ExplodedNode *Pred,
                                           ExplodedNodeSet &Dst) {
-  StmtNodeBuilder B(Pred, Dst, *currentBuilderContext);
+  StmtNodeBuilder B(Pred, Dst, *currBldrCtx);
 
   const InitListExpr *ILE 
     = cast<InitListExpr>(CL->getInitializer()->IgnoreParens());
@@ -442,7 +441,7 @@ void ExprEngine::VisitDeclStmt(const DeclStmt *DS, ExplodedNode *Pred,
   ExplodedNodeSet dstPreVisit;
   getCheckerManager().runCheckersForPreStmt(dstPreVisit, Pred, DS, *this);
   
-  StmtNodeBuilder B(dstPreVisit, Dst, *currentBuilderContext);
+  StmtNodeBuilder B(dstPreVisit, Dst, *currBldrCtx);
   const VarDecl *VD = dyn_cast<VarDecl>(D);
   for (ExplodedNodeSet::iterator I = dstPreVisit.begin(), E = dstPreVisit.end();
        I!=E; ++I) {
@@ -478,8 +477,8 @@ void ExprEngine::VisitDeclStmt(const DeclStmt *DS, ExplodedNode *Pred,
             Ty = getContext().getPointerType(Ty);
           }
 
-          InitVal = svalBuilder.getConjuredSymbolVal(NULL, InitEx, LC, Ty,
-                                   currentBuilderContext->getCurrentBlockCount());
+          InitVal = svalBuilder.conjureSymbolVal(0, InitEx, LC, Ty,
+                                                 currBldrCtx->blockCount());
         }
         B.takeNodes(N);
         ExplodedNodeSet Dst2;
@@ -488,7 +487,7 @@ void ExprEngine::VisitDeclStmt(const DeclStmt *DS, ExplodedNode *Pred,
       }
     }
     else {
-      B.generateNode(DS, N,state->bindDeclWithNoInit(state->getRegion(VD, LC)));
+      B.generateNode(DS, N, state);
     }
   }
 }
@@ -498,7 +497,7 @@ void ExprEngine::VisitLogicalExpr(const BinaryOperator* B, ExplodedNode *Pred,
   assert(B->getOpcode() == BO_LAnd ||
          B->getOpcode() == BO_LOr);
 
-  StmtNodeBuilder Bldr(Pred, Dst, *currentBuilderContext);
+  StmtNodeBuilder Bldr(Pred, Dst, *currBldrCtx);
   ProgramStateRef state = Pred->getState();
 
   ExplodedNode *N = Pred;
@@ -531,10 +530,28 @@ void ExprEngine::VisitLogicalExpr(const BinaryOperator* B, ExplodedNode *Pred,
   else {
     // If there is no terminator, by construction the last statement
     // in SrcBlock is the value of the enclosing expression.
+    // However, we still need to constrain that value to be 0 or 1.
     assert(!SrcBlock->empty());
     CFGStmt Elem = cast<CFGStmt>(*SrcBlock->rbegin());
-    const Stmt *S = Elem.getStmt();
-    X = N->getState()->getSVal(S, Pred->getLocationContext());
+    const Expr *RHS = cast<Expr>(Elem.getStmt());
+    SVal RHSVal = N->getState()->getSVal(RHS, Pred->getLocationContext());
+
+    DefinedOrUnknownSVal DefinedRHS = cast<DefinedOrUnknownSVal>(RHSVal);
+    ProgramStateRef StTrue, StFalse;
+    llvm::tie(StTrue, StFalse) = N->getState()->assume(DefinedRHS);
+    if (StTrue) {
+      if (StFalse) {
+        // We can't constrain the value to 0 or 1; the best we can do is a cast.
+        X = getSValBuilder().evalCast(RHSVal, B->getType(), RHS->getType());
+      } else {
+        // The value is known to be true.
+        X = getSValBuilder().makeIntVal(1, B->getType());
+      }
+    } else {
+      // The value is known to be false.
+      assert(StFalse && "Infeasible path!");
+      X = getSValBuilder().makeIntVal(0, B->getType());
+    }
   }
 
   Bldr.generateNode(B, Pred, state->BindExpr(B, Pred->getLocationContext(), X));
@@ -543,14 +560,15 @@ void ExprEngine::VisitLogicalExpr(const BinaryOperator* B, ExplodedNode *Pred,
 void ExprEngine::VisitInitListExpr(const InitListExpr *IE,
                                    ExplodedNode *Pred,
                                    ExplodedNodeSet &Dst) {
-  StmtNodeBuilder B(Pred, Dst, *currentBuilderContext);
+  StmtNodeBuilder B(Pred, Dst, *currBldrCtx);
 
   ProgramStateRef state = Pred->getState();
   const LocationContext *LCtx = Pred->getLocationContext();
   QualType T = getContext().getCanonicalType(IE->getType());
   unsigned NumInitElements = IE->getNumInits();
   
-  if (T->isArrayType() || T->isRecordType() || T->isVectorType()) {
+  if (T->isArrayType() || T->isRecordType() || T->isVectorType() ||
+      T->isAnyComplexType()) {
     llvm::ImmutableList<SVal> vals = getBasicVals().getEmptySValList();
     
     // Handle base case where the initializer has no elements.
@@ -590,7 +608,7 @@ void ExprEngine::VisitGuardedExpr(const Expr *Ex,
                                   const Expr *R,
                                   ExplodedNode *Pred,
                                   ExplodedNodeSet &Dst) {
-  StmtNodeBuilder B(Pred, Dst, *currentBuilderContext);
+  StmtNodeBuilder B(Pred, Dst, *currBldrCtx);
   ProgramStateRef state = Pred->getState();
   const LocationContext *LCtx = Pred->getLocationContext();
   const CFGBlock *SrcBlock = 0;
@@ -631,7 +649,7 @@ void ExprEngine::VisitGuardedExpr(const Expr *Ex,
 void ExprEngine::
 VisitOffsetOfExpr(const OffsetOfExpr *OOE, 
                   ExplodedNode *Pred, ExplodedNodeSet &Dst) {
-  StmtNodeBuilder B(Pred, Dst, *currentBuilderContext);
+  StmtNodeBuilder B(Pred, Dst, *currBldrCtx);
   APSInt IV;
   if (OOE->EvaluateAsInt(IV, getContext())) {
     assert(IV.getBitWidth() == getContext().getTypeSize(OOE->getType()));
@@ -650,7 +668,7 @@ void ExprEngine::
 VisitUnaryExprOrTypeTraitExpr(const UnaryExprOrTypeTraitExpr *Ex,
                               ExplodedNode *Pred,
                               ExplodedNodeSet &Dst) {
-  StmtNodeBuilder Bldr(Pred, Dst, *currentBuilderContext);
+  StmtNodeBuilder Bldr(Pred, Dst, *currBldrCtx);
 
   QualType T = Ex->getTypeOfArgument();
   
@@ -683,7 +701,7 @@ VisitUnaryExprOrTypeTraitExpr(const UnaryExprOrTypeTraitExpr *Ex,
 void ExprEngine::VisitUnaryOperator(const UnaryOperator* U, 
                                     ExplodedNode *Pred,
                                     ExplodedNodeSet &Dst) {
-  StmtNodeBuilder Bldr(Pred, Dst, *currentBuilderContext);
+  StmtNodeBuilder Bldr(Pred, Dst, *currBldrCtx);
   switch (U->getOpcode()) {
     default: {
       Bldr.takeNodes(Pred);
@@ -816,7 +834,7 @@ void ExprEngine::VisitIncrementDecrementOperator(const UnaryOperator* U,
   evalLoad(Tmp, U, Ex, Pred, state, loc);
   
   ExplodedNodeSet Dst2;
-  StmtNodeBuilder Bldr(Tmp, Dst2, *currentBuilderContext);
+  StmtNodeBuilder Bldr(Tmp, Dst2, *currBldrCtx);
   for (ExplodedNodeSet::iterator I=Tmp.begin(), E=Tmp.end();I!=E;++I) {
     
     state = (*I)->getState();
@@ -840,16 +858,17 @@ void ExprEngine::VisitIncrementDecrementOperator(const UnaryOperator* U,
     
     if (U->getType()->isAnyPointerType())
       RHS = svalBuilder.makeArrayIndex(1);
-    else
+    else if (U->getType()->isIntegralOrEnumerationType())
       RHS = svalBuilder.makeIntVal(1, U->getType());
+    else
+      RHS = UnknownVal();
     
     SVal Result = evalBinOp(state, Op, V2, RHS, U->getType());
     
     // Conjure a new symbol if necessary to recover precision.
     if (Result.isUnknown()){
       DefinedOrUnknownSVal SymVal =
-	svalBuilder.getConjuredSymbolVal(NULL, Ex, LCtx,
-                               currentBuilderContext->getCurrentBlockCount());
+        svalBuilder.conjureSymbolVal(0, Ex, LCtx, currBldrCtx->blockCount());
       Result = SymVal;
       
       // If the value is a location, ++/-- should always preserve
diff --git a/lib/StaticAnalyzer/Core/ExprEngineCXX.cpp b/lib/StaticAnalyzer/Core/ExprEngineCXX.cpp
index 44a860f..b3baa79 100644
--- a/lib/StaticAnalyzer/Core/ExprEngineCXX.cpp
+++ b/lib/StaticAnalyzer/Core/ExprEngineCXX.cpp
@@ -25,20 +25,28 @@ using namespace ento;
 void ExprEngine::CreateCXXTemporaryObject(const MaterializeTemporaryExpr *ME,
                                           ExplodedNode *Pred,
                                           ExplodedNodeSet &Dst) {
-  StmtNodeBuilder Bldr(Pred, Dst, *currentBuilderContext);
+  StmtNodeBuilder Bldr(Pred, Dst, *currBldrCtx);
   const Expr *tempExpr = ME->GetTemporaryExpr()->IgnoreParens();
   ProgramStateRef state = Pred->getState();
   const LocationContext *LCtx = Pred->getLocationContext();
 
   // Bind the temporary object to the value of the expression. Then bind
   // the expression to the location of the object.
-  SVal V = state->getSVal(tempExpr, Pred->getLocationContext());
-
-  const MemRegion *R =
-    svalBuilder.getRegionManager().getCXXTempObjectRegion(ME, LCtx);
+  SVal V = state->getSVal(tempExpr, LCtx);
+
+  // If the value is already a CXXTempObjectRegion, it is fine as it is.
+  // Otherwise, create a new CXXTempObjectRegion, and copy the value into it.
+  const MemRegion *MR = V.getAsRegion();
+  if (!MR || !isa<CXXTempObjectRegion>(MR)) {
+    const MemRegion *R =
+      svalBuilder.getRegionManager().getCXXTempObjectRegion(ME, LCtx);
+
+    SVal L = loc::MemRegionVal(R);
+    state = state->bindLoc(L, V);
+    V = L;
+  }
 
-  state = state->bindLoc(loc::MemRegionVal(R), V);
-  Bldr.generateNode(ME, Pred, state->BindExpr(ME, LCtx, loc::MemRegionVal(R)));
+  Bldr.generateNode(ME, Pred, state->BindExpr(ME, LCtx, V));
 }
 
 void ExprEngine::VisitCXXConstructExpr(const CXXConstructExpr *CE,
@@ -53,9 +61,9 @@ void ExprEngine::VisitCXXConstructExpr(const CXXConstructExpr *CE,
   case CXXConstructExpr::CK_Complete: {
     // See if we're constructing an existing region by looking at the next
     // element in the CFG.
-    const CFGBlock *B = currentBuilderContext->getBlock();
-    if (currentStmtIdx + 1 < B->size()) {
-      CFGElement Next = (*B)[currentStmtIdx+1];
+    const CFGBlock *B = currBldrCtx->getBlock();
+    if (currStmtIdx + 1 < B->size()) {
+      CFGElement Next = (*B)[currStmtIdx+1];
 
       // Is this a constructor for a local variable?
       if (const CFGStmt *StmtElem = dyn_cast<CFGStmt>(&Next)) {
@@ -101,8 +109,12 @@ void ExprEngine::VisitCXXConstructExpr(const CXXConstructExpr *CE,
       // FIXME: This will eventually need to handle new-expressions as well.
     }
 
-    // If we couldn't find an existing region to construct into, we'll just
-    // generate a symbolic region, which is fine.
+    // If we couldn't find an existing region to construct into, assume we're
+    // constructing a temporary.
+    if (!Target) {
+      MemRegionManager &MRMgr = getSValBuilder().getRegionManager();
+      Target = MRMgr.getCXXTempObjectRegion(CE, LCtx);
+    }
 
     break;
   }
@@ -137,7 +149,7 @@ void ExprEngine::VisitCXXConstructExpr(const CXXConstructExpr *CE,
                                             *Call, *this);
 
   ExplodedNodeSet DstInvalidated;
-  StmtNodeBuilder Bldr(DstPreCall, DstInvalidated, *currentBuilderContext);
+  StmtNodeBuilder Bldr(DstPreCall, DstInvalidated, *currBldrCtx);
   for (ExplodedNodeSet::iterator I = DstPreCall.begin(), E = DstPreCall.end();
        I != E; ++I)
     defaultEvalCall(Bldr, *I, *Call);
@@ -151,6 +163,7 @@ void ExprEngine::VisitCXXConstructExpr(const CXXConstructExpr *CE,
 void ExprEngine::VisitCXXDestructor(QualType ObjectType,
                                     const MemRegion *Dest,
                                     const Stmt *S,
+                                    bool IsBaseDtor,
                                     ExplodedNode *Pred, 
                                     ExplodedNodeSet &Dst) {
   const LocationContext *LCtx = Pred->getLocationContext();
@@ -171,7 +184,7 @@ void ExprEngine::VisitCXXDestructor(QualType ObjectType,
 
   CallEventManager &CEMgr = getStateManager().getCallEventManager();
   CallEventRef<CXXDestructorCall> Call =
-    CEMgr.getCXXDestructorCall(DtorDecl, S, Dest, State, LCtx);
+    CEMgr.getCXXDestructorCall(DtorDecl, S, Dest, IsBaseDtor, State, LCtx);
 
   PrettyStackTraceLoc CrashInfo(getContext().getSourceManager(),
                                 Call->getSourceRange().getBegin(),
@@ -182,7 +195,7 @@ void ExprEngine::VisitCXXDestructor(QualType ObjectType,
                                             *Call, *this);
 
   ExplodedNodeSet DstInvalidated;
-  StmtNodeBuilder Bldr(DstPreCall, DstInvalidated, *currentBuilderContext);
+  StmtNodeBuilder Bldr(DstPreCall, DstInvalidated, *currBldrCtx);
   for (ExplodedNodeSet::iterator I = DstPreCall.begin(), E = DstPreCall.end();
        I != E; ++I)
     defaultEvalCall(Bldr, *I, *Call);
@@ -198,12 +211,13 @@ void ExprEngine::VisitCXXNewExpr(const CXXNewExpr *CNE, ExplodedNode *Pred,
   // Also, we need to decide how allocators actually work -- they're not
   // really part of the CXXNewExpr because they happen BEFORE the
   // CXXConstructExpr subexpression. See PR12014 for some discussion.
-  StmtNodeBuilder Bldr(Pred, Dst, *currentBuilderContext);
+  StmtNodeBuilder Bldr(Pred, Dst, *currBldrCtx);
   
-  unsigned blockCount = currentBuilderContext->getCurrentBlockCount();
+  unsigned blockCount = currBldrCtx->blockCount();
   const LocationContext *LCtx = Pred->getLocationContext();
-  DefinedOrUnknownSVal symVal =
-    svalBuilder.getConjuredSymbolVal(0, CNE, LCtx, CNE->getType(), blockCount);
+  DefinedOrUnknownSVal symVal = svalBuilder.conjureSymbolVal(0, CNE, LCtx,
+                                                             CNE->getType(),
+                                                             blockCount);
   ProgramStateRef State = Pred->getState();
 
   CallEventManager &CEMgr = getStateManager().getCallEventManager();
@@ -215,6 +229,18 @@ void ExprEngine::VisitCXXNewExpr(const CXXNewExpr *CNE, ExplodedNode *Pred,
   // we should be using the usual pre-/(default-)eval-/post-call checks here.
   State = Call->invalidateRegions(blockCount);
 
+  // If we're compiling with exceptions enabled, and this allocation function
+  // is not declared as non-throwing, failures /must/ be signalled by
+  // exceptions, and thus the return value will never be NULL.
+  // C++11 [basic.stc.dynamic.allocation]p3.
+  FunctionDecl *FD = CNE->getOperatorNew();
+  if (FD && getContext().getLangOpts().CXXExceptions) {
+    QualType Ty = FD->getType();
+    if (const FunctionProtoType *ProtoType = Ty->getAs<FunctionProtoType>())
+      if (!ProtoType->isNothrow(getContext()))
+        State = State->assume(symVal, true);
+  }
+
   if (CNE->isArray()) {
     // FIXME: allocating an array requires simulating the constructors.
     // For now, just return a symbolicated region.
@@ -232,11 +258,12 @@ void ExprEngine::VisitCXXNewExpr(const CXXNewExpr *CNE, ExplodedNode *Pred,
   // CXXNewExpr, we need to make sure that the constructed object is not
   // immediately invalidated here. (The placement call should happen before
   // the constructor call anyway.)
-  FunctionDecl *FD = CNE->getOperatorNew();
   if (FD && FD->isReservedGlobalPlacementOperator()) {
     // Non-array placement new should always return the placement location.
     SVal PlacementLoc = State->getSVal(CNE->getPlacementArg(0), LCtx);
-    State = State->BindExpr(CNE, LCtx, PlacementLoc);
+    SVal Result = svalBuilder.evalCast(PlacementLoc, CNE->getType(),
+                                       CNE->getPlacementArg(0)->getType());
+    State = State->BindExpr(CNE, LCtx, Result);
   } else {
     State = State->BindExpr(CNE, LCtx, symVal);
   }
@@ -259,7 +286,7 @@ void ExprEngine::VisitCXXNewExpr(const CXXNewExpr *CNE, ExplodedNode *Pred,
 
 void ExprEngine::VisitCXXDeleteExpr(const CXXDeleteExpr *CDE, 
                                     ExplodedNode *Pred, ExplodedNodeSet &Dst) {
-  StmtNodeBuilder Bldr(Pred, Dst, *currentBuilderContext);
+  StmtNodeBuilder Bldr(Pred, Dst, *currBldrCtx);
   ProgramStateRef state = Pred->getState();
   Bldr.generateNode(CDE, Pred, state);
 }
@@ -274,18 +301,18 @@ void ExprEngine::VisitCXXCatchStmt(const CXXCatchStmt *CS,
   }
 
   const LocationContext *LCtx = Pred->getLocationContext();
-  SVal V = svalBuilder.getConjuredSymbolVal(CS, LCtx, VD->getType(),
-                                 currentBuilderContext->getCurrentBlockCount());
+  SVal V = svalBuilder.conjureSymbolVal(CS, LCtx, VD->getType(),
+                                        currBldrCtx->blockCount());
   ProgramStateRef state = Pred->getState();
   state = state->bindLoc(state->getLValue(VD, LCtx), V);
 
-  StmtNodeBuilder Bldr(Pred, Dst, *currentBuilderContext);
+  StmtNodeBuilder Bldr(Pred, Dst, *currBldrCtx);
   Bldr.generateNode(CS, Pred, state);
 }
 
 void ExprEngine::VisitCXXThisExpr(const CXXThisExpr *TE, ExplodedNode *Pred,
                                     ExplodedNodeSet &Dst) {
-  StmtNodeBuilder Bldr(Pred, Dst, *currentBuilderContext);
+  StmtNodeBuilder Bldr(Pred, Dst, *currBldrCtx);
 
   // Get the this object region from StoreManager.
   const LocationContext *LCtx = Pred->getLocationContext();
diff --git a/lib/StaticAnalyzer/Core/ExprEngineCallAndReturn.cpp b/lib/StaticAnalyzer/Core/ExprEngineCallAndReturn.cpp
index 3b2e4ec..3ead081 100644
--- a/lib/StaticAnalyzer/Core/ExprEngineCallAndReturn.cpp
+++ b/lib/StaticAnalyzer/Core/ExprEngineCallAndReturn.cpp
@@ -17,19 +17,22 @@
 #include "clang/StaticAnalyzer/Core/CheckerManager.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/ExprEngine.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h"
+#include "clang/AST/CXXInheritance.h"
 #include "clang/AST/DeclCXX.h"
+#include "clang/AST/ParentMap.h"
 #include "llvm/ADT/SmallSet.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Support/SaveAndRestore.h"
 
-#define CXX_INLINING_ENABLED 1
-
 using namespace clang;
 using namespace ento;
 
 STATISTIC(NumOfDynamicDispatchPathSplits,
   "The # of times we split the path due to imprecise dynamic dispatch info");
 
+STATISTIC(NumInlinedCalls,
+  "The # of times we inlined a call");
+
 void ExprEngine::processCallEnter(CallEnter CE, ExplodedNode *Pred) {
   // Get the entry block in the CFG of the callee.
   const StackFrameContext *calleeCtx = CE.getCalleeContext();
@@ -64,35 +67,47 @@ static std::pair<const Stmt*,
   const StackFrameContext *SF =
           Node->getLocation().getLocationContext()->getCurrentStackFrame();
 
-  // Back up through the ExplodedGraph until we reach a statement node.
+  // Back up through the ExplodedGraph until we reach a statement node in this
+  // stack frame.
   while (Node) {
     const ProgramPoint &PP = Node->getLocation();
 
-    if (const StmtPoint *SP = dyn_cast<StmtPoint>(&PP)) {
-      S = SP->getStmt();
-      break;
-    } else if (const CallExitEnd *CEE = dyn_cast<CallExitEnd>(&PP)) {
-      S = CEE->getCalleeContext()->getCallSite();
-      if (S)
+    if (PP.getLocationContext()->getCurrentStackFrame() == SF) {
+      if (const StmtPoint *SP = dyn_cast<StmtPoint>(&PP)) {
+        S = SP->getStmt();
         break;
-      // If we have an implicit call, we'll probably end up with a
-      // StmtPoint inside the callee, which is acceptable.
-      // (It's possible a function ONLY contains implicit calls -- such as an
-      // implicitly-generated destructor -- so we shouldn't just skip back to
-      // the CallEnter node and keep going.)
+      } else if (const CallExitEnd *CEE = dyn_cast<CallExitEnd>(&PP)) {
+        S = CEE->getCalleeContext()->getCallSite();
+        if (S)
+          break;
+
+        // If there is no statement, this is an implicitly-generated call.
+        // We'll walk backwards over it and then continue the loop to find
+        // an actual statement.
+        const CallEnter *CE;
+        do {
+          Node = Node->getFirstPred();
+          CE = Node->getLocationAs<CallEnter>();
+        } while (!CE || CE->getCalleeContext() != CEE->getCalleeContext());
+
+        // Continue searching the graph.
+      }
     } else if (const CallEnter *CE = dyn_cast<CallEnter>(&PP)) {
       // If we reached the CallEnter for this function, it has no statements.
       if (CE->getCalleeContext() == SF)
         break;
     }
 
+    if (Node->pred_empty())
+      return std::pair<const Stmt*, const CFGBlock*>((Stmt*)0, (CFGBlock*)0);
+
     Node = *Node->pred_begin();
   }
 
   const CFGBlock *Blk = 0;
   if (S) {
     // Now, get the enclosing basic block.
-    while (Node && Node->pred_size() >=1 ) {
+    while (Node) {
       const ProgramPoint &PP = Node->getLocation();
       if (isa<BlockEdge>(PP) &&
           (PP.getLocationContext()->getCurrentStackFrame() == SF)) {
@@ -100,6 +115,9 @@ static std::pair<const Stmt*,
         Blk = EPP.getDst();
         break;
       }
+      if (Node->pred_empty())
+        return std::pair<const Stmt*, const CFGBlock*>(S, (CFGBlock*)0);
+
       Node = *Node->pred_begin();
     }
   }
@@ -107,6 +125,82 @@ static std::pair<const Stmt*,
   return std::pair<const Stmt*, const CFGBlock*>(S, Blk);
 }
 
+/// Adjusts a return value when the called function's return type does not
+/// match the caller's expression type. This can happen when a dynamic call
+/// is devirtualized, and the overridding method has a covariant (more specific)
+/// return type than the parent's method. For C++ objects, this means we need
+/// to add base casts.
+static SVal adjustReturnValue(SVal V, QualType ExpectedTy, QualType ActualTy,
+                              StoreManager &StoreMgr) {
+  // For now, the only adjustments we handle apply only to locations.
+  if (!isa<Loc>(V))
+    return V;
+
+  // If the types already match, don't do any unnecessary work.
+  ExpectedTy = ExpectedTy.getCanonicalType();
+  ActualTy = ActualTy.getCanonicalType();
+  if (ExpectedTy == ActualTy)
+    return V;
+
+  // No adjustment is needed between Objective-C pointer types.
+  if (ExpectedTy->isObjCObjectPointerType() &&
+      ActualTy->isObjCObjectPointerType())
+    return V;
+
+  // C++ object pointers may need "derived-to-base" casts.
+  const CXXRecordDecl *ExpectedClass = ExpectedTy->getPointeeCXXRecordDecl();
+  const CXXRecordDecl *ActualClass = ActualTy->getPointeeCXXRecordDecl();
+  if (ExpectedClass && ActualClass) {
+    CXXBasePaths Paths(/*FindAmbiguities=*/true, /*RecordPaths=*/true,
+                       /*DetectVirtual=*/false);
+    if (ActualClass->isDerivedFrom(ExpectedClass, Paths) &&
+        !Paths.isAmbiguous(ActualTy->getCanonicalTypeUnqualified())) {
+      return StoreMgr.evalDerivedToBase(V, Paths.front());
+    }
+  }
+
+  // Unfortunately, Objective-C does not enforce that overridden methods have
+  // covariant return types, so we can't assert that that never happens.
+  // Be safe and return UnknownVal().
+  return UnknownVal();
+}
+
+void ExprEngine::removeDeadOnEndOfFunction(NodeBuilderContext& BC,
+                                           ExplodedNode *Pred,
+                                           ExplodedNodeSet &Dst) {
+  NodeBuilder Bldr(Pred, Dst, BC);
+
+  // Find the last statement in the function and the corresponding basic block.
+  const Stmt *LastSt = 0;
+  const CFGBlock *Blk = 0;
+  llvm::tie(LastSt, Blk) = getLastStmt(Pred);
+  if (!Blk || !LastSt) {
+    return;
+  }
+  
+  // If the last statement is return, everything it references should stay live.
+  if (isa<ReturnStmt>(LastSt))
+    return;
+
+  // Here, we call the Symbol Reaper with 0 stack context telling it to clean up
+  // everything on the stack. We use LastStmt as a diagnostic statement, with 
+  // which the PreStmtPurgeDead point will be associated.
+  currBldrCtx = &BC;
+  removeDead(Pred, Dst, 0, 0, LastSt,
+             ProgramPoint::PostStmtPurgeDeadSymbolsKind);
+  currBldrCtx = 0;
+}
+
+static bool wasDifferentDeclUsedForInlining(CallEventRef<> Call,
+    const StackFrameContext *calleeCtx) {
+  const Decl *RuntimeCallee = calleeCtx->getDecl();
+  const Decl *StaticDecl = Call->getDecl();
+  assert(RuntimeCallee);
+  if (!StaticDecl)
+    return true;
+  return RuntimeCallee->getCanonicalDecl() != StaticDecl->getCanonicalDecl();
+}
+
 /// The call exit is simulated with a sequence of nodes, which occur between 
 /// CallExitBegin and CallExitEnd. The following operations occur between the 
 /// two program points:
@@ -133,6 +227,11 @@ void ExprEngine::processCallExit(ExplodedNode *CEBNode) {
   const CFGBlock *Blk = 0;
   llvm::tie(LastSt, Blk) = getLastStmt(CEBNode);
 
+  // Generate a CallEvent /before/ cleaning the state, so that we can get the
+  // correct value for 'this' (if necessary).
+  CallEventManager &CEMgr = getStateManager().getCallEventManager();
+  CallEventRef<> Call = CEMgr.getCaller(calleeCtx, state);
+
   // Step 2: generate node with bound return value: CEBNode -> BindedRetNode.
 
   // If the callee returns an expression, bind its value to CallExpr.
@@ -140,6 +239,19 @@ void ExprEngine::processCallExit(ExplodedNode *CEBNode) {
     if (const ReturnStmt *RS = dyn_cast_or_null<ReturnStmt>(LastSt)) {
       const LocationContext *LCtx = CEBNode->getLocationContext();
       SVal V = state->getSVal(RS, LCtx);
+
+      // Ensure that the return type matches the type of the returned Expr.
+      if (wasDifferentDeclUsedForInlining(Call, calleeCtx)) {
+        QualType ReturnedTy =
+          CallEvent::getDeclaredResultType(calleeCtx->getDecl());
+        if (!ReturnedTy.isNull()) {
+          if (const Expr *Ex = dyn_cast<Expr>(CE)) {
+            V = adjustReturnValue(V, Ex->getType(), ReturnedTy,
+                                  getStoreManager());
+          }
+        }
+      }
+
       state = state->BindExpr(CE, callerCtx, V);
     }
 
@@ -149,23 +261,25 @@ void ExprEngine::processCallExit(ExplodedNode *CEBNode) {
         svalBuilder.getCXXThis(CCE->getConstructor()->getParent(), calleeCtx);
       SVal ThisV = state->getSVal(This);
 
-      // Always bind the region to the CXXConstructExpr.
+      // If the constructed object is a prvalue, get its bindings.
+      // Note that we have to be careful here because constructors embedded
+      // in DeclStmts are not marked as lvalues.
+      if (!CCE->isGLValue())
+        if (const MemRegion *MR = ThisV.getAsRegion())
+          if (isa<CXXTempObjectRegion>(MR))
+            ThisV = state->getSVal(cast<Loc>(ThisV));
+
       state = state->BindExpr(CCE, callerCtx, ThisV);
     }
   }
 
-  // Generate a CallEvent /before/ cleaning the state, so that we can get the
-  // correct value for 'this' (if necessary).
-  CallEventManager &CEMgr = getStateManager().getCallEventManager();
-  CallEventRef<> Call = CEMgr.getCaller(calleeCtx, state);
-
   // Step 3: BindedRetNode -> CleanedNodes
   // If we can find a statement and a block in the inlined function, run remove
   // dead bindings before returning from the call. This is important to ensure
   // that we report the issues such as leaks in the stack contexts in which
   // they occurred.
   ExplodedNodeSet CleanedNodes;
-  if (LastSt && Blk) {
+  if (LastSt && Blk && AMgr.options.AnalysisPurgeOpt != PurgeNone) {
     static SimpleProgramPointTag retValBind("ExprEngine : Bind Return Value");
     PostStmt Loc(LastSt, calleeCtx, &retValBind);
     bool isNew;
@@ -175,14 +289,14 @@ void ExprEngine::processCallExit(ExplodedNode *CEBNode) {
       return;
 
     NodeBuilderContext Ctx(getCoreEngine(), Blk, BindedRetNode);
-    currentBuilderContext = &Ctx;
+    currBldrCtx = &Ctx;
     // Here, we call the Symbol Reaper with 0 statement and caller location
     // context, telling it to clean up everything in the callee's context
     // (and it's children). We use LastStmt as a diagnostic statement, which
     // which the PreStmtPurge Dead point will be associated.
     removeDead(BindedRetNode, CleanedNodes, 0, callerCtx, LastSt,
                ProgramPoint::PostStmtPurgeDeadSymbolsKind);
-    currentBuilderContext = 0;
+    currBldrCtx = 0;
   } else {
     CleanedNodes.Add(CEBNode);
   }
@@ -204,9 +318,9 @@ void ExprEngine::processCallExit(ExplodedNode *CEBNode) {
     // result onto the work list.
     // CEENode -> Dst -> WorkList
     NodeBuilderContext Ctx(Engine, calleeCtx->getCallSiteBlock(), CEENode);
-    SaveAndRestore<const NodeBuilderContext*> NBCSave(currentBuilderContext,
+    SaveAndRestore<const NodeBuilderContext*> NBCSave(currBldrCtx,
         &Ctx);
-    SaveAndRestore<unsigned> CBISave(currentStmtIdx, calleeCtx->getIndex());
+    SaveAndRestore<unsigned> CBISave(currStmtIdx, calleeCtx->getIndex());
 
     CallEventRef<> UpdatedCall = Call.cloneWithState(CEEState);
 
@@ -236,14 +350,48 @@ void ExprEngine::processCallExit(ExplodedNode *CEBNode) {
   }
 }
 
-static unsigned getNumberStackFrames(const LocationContext *LCtx) {
-  unsigned count = 0;
+void ExprEngine::examineStackFrames(const Decl *D, const LocationContext *LCtx,
+                               bool &IsRecursive, unsigned &StackDepth) {
+  IsRecursive = false;
+  StackDepth = 0;
+
   while (LCtx) {
-    if (isa<StackFrameContext>(LCtx))
-      ++count;
+    if (const StackFrameContext *SFC = dyn_cast<StackFrameContext>(LCtx)) {
+      const Decl *DI = SFC->getDecl();
+
+      // Mark recursive (and mutually recursive) functions and always count
+      // them when measuring the stack depth.
+      if (DI == D) {
+        IsRecursive = true;
+        ++StackDepth;
+        LCtx = LCtx->getParent();
+        continue;
+      }
+
+      // Do not count the small functions when determining the stack depth.
+      AnalysisDeclContext *CalleeADC = AMgr.getAnalysisDeclContext(DI);
+      const CFG *CalleeCFG = CalleeADC->getCFG();
+      if (CalleeCFG->getNumBlockIDs() > AMgr.options.getAlwaysInlineSize())
+        ++StackDepth;
+    }
     LCtx = LCtx->getParent();
   }
-  return count;  
+
+}
+
+static bool IsInStdNamespace(const FunctionDecl *FD) {
+  const DeclContext *DC = FD->getEnclosingNamespaceContext();
+  const NamespaceDecl *ND = dyn_cast<NamespaceDecl>(DC);
+  if (!ND)
+    return false;
+  
+  while (const DeclContext *Parent = ND->getParent()) {
+    if (!isa<NamespaceDecl>(Parent))
+      break;
+    ND = cast<NamespaceDecl>(Parent);
+  }
+
+  return ND->getName() == "std";
 }
 
 // Determine if we should inline the call.
@@ -256,14 +404,18 @@ bool ExprEngine::shouldInlineDecl(const Decl *D, ExplodedNode *Pred) {
   if (!CalleeCFG)
     return false;
 
-  if (getNumberStackFrames(Pred->getLocationContext())
-        == AMgr.InlineMaxStackDepth)
+  bool IsRecursive = false;
+  unsigned StackDepth = 0;
+  examineStackFrames(D, Pred->getLocationContext(), IsRecursive, StackDepth);
+  if ((StackDepth >= AMgr.options.InlineMaxStackDepth) &&
+       ((CalleeCFG->getNumBlockIDs() > AMgr.options.getAlwaysInlineSize())
+         || IsRecursive))
     return false;
 
   if (Engine.FunctionSummaries->hasReachedMaxBlockCount(D))
     return false;
 
-  if (CalleeCFG->getNumBlockIDs() > AMgr.InlineMaxFunctionSize)
+  if (CalleeCFG->getNumBlockIDs() > AMgr.options.InlineMaxFunctionSize)
     return false;
 
   // Do not inline variadic calls (for now).
@@ -276,6 +428,21 @@ bool ExprEngine::shouldInlineDecl(const Decl *D, ExplodedNode *Pred) {
       return false;
   }
 
+  if (getContext().getLangOpts().CPlusPlus) {
+    if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D)) {
+      // Conditionally allow the inlining of template functions.
+      if (!getAnalysisManager().options.mayInlineTemplateFunctions())
+        if (FD->getTemplatedKind() != FunctionDecl::TK_NonTemplate)
+          return false;
+
+      // Conditionally allow the inlining of C++ standard library functions.
+      if (!getAnalysisManager().options.mayInlineCXXStandardLibrary())
+        if (getContext().getSourceManager().isInSystemHeader(FD->getLocation()))
+          if (IsInStdNamespace(FD))
+            return false;
+    }
+  }
+
   // It is possible that the live variables analysis cannot be
   // run.  If so, bail out.
   if (!CalleeADC->getAnalysis<RelaxedLiveVariables>())
@@ -284,26 +451,21 @@ bool ExprEngine::shouldInlineDecl(const Decl *D, ExplodedNode *Pred) {
   return true;
 }
 
-/// The GDM component containing the dynamic dispatch bifurcation info. When
-/// the exact type of the receiver is not known, we want to explore both paths -
-/// one on which we do inline it and the other one on which we don't. This is
-/// done to ensure we do not drop coverage.
-/// This is the map from the receiver region to a bool, specifying either we
-/// consider this region's information precise or not along the given path.
-namespace clang {
-namespace ento {
-enum DynamicDispatchMode { DynamicDispatchModeInlined = 1,
-                           DynamicDispatchModeConservative };
-
-struct DynamicDispatchBifurcationMap {};
-typedef llvm::ImmutableMap<const MemRegion*,
-                           unsigned int> DynamicDispatchBifur;
-template<> struct ProgramStateTrait<DynamicDispatchBifurcationMap>
-    :  public ProgramStatePartialTrait<DynamicDispatchBifur> {
-  static void *GDMIndex() { static int index; return &index; }
-};
-
-}}
+// The GDM component containing the dynamic dispatch bifurcation info. When
+// the exact type of the receiver is not known, we want to explore both paths -
+// one on which we do inline it and the other one on which we don't. This is
+// done to ensure we do not drop coverage.
+// This is the map from the receiver region to a bool, specifying either we
+// consider this region's information precise or not along the given path.
+namespace {
+  enum DynamicDispatchMode {
+    DynamicDispatchModeInlined = 1,
+    DynamicDispatchModeConservative
+  };
+}
+REGISTER_TRAIT_WITH_PROGRAMSTATE(DynamicDispatchBifurcationMap,
+                                 CLANG_ENTO_PROGRAMSTATE_MAP(const MemRegion *,
+                                                             unsigned))
 
 bool ExprEngine::inlineCall(const CallEvent &Call, const Decl *D,
                             NodeBuilder &Bldr, ExplodedNode *Pred,
@@ -314,24 +476,19 @@ bool ExprEngine::inlineCall(const CallEvent &Call, const Decl *D,
   const StackFrameContext *CallerSFC = CurLC->getCurrentStackFrame();
   const LocationContext *ParentOfCallee = 0;
 
+  AnalyzerOptions &Opts = getAnalysisManager().options;
+
   // FIXME: Refactor this check into a hypothetical CallEvent::canInline.
   switch (Call.getKind()) {
   case CE_Function:
     break;
   case CE_CXXMember:
   case CE_CXXMemberOperator:
-    if (!CXX_INLINING_ENABLED)
+    if (!Opts.mayInlineCXXMemberFunction(CIMK_MemberFunctions))
       return false;
     break;
   case CE_CXXConstructor: {
-    if (!CXX_INLINING_ENABLED)
-      return false;
-
-    // Only inline constructors and destructors if we built the CFGs for them
-    // properly.
-    const AnalysisDeclContext *ADC = CallerSFC->getAnalysisDeclContext();
-    if (!ADC->getCFGBuildOptions().AddImplicitDtors ||
-        !ADC->getCFGBuildOptions().AddInitializers)
+    if (!Opts.mayInlineCXXMemberFunction(CIMK_Constructors))
       return false;
 
     const CXXConstructorCall &Ctor = cast<CXXConstructorCall>(Call);
@@ -341,9 +498,31 @@ bool ExprEngine::inlineCall(const CallEvent &Call, const Decl *D,
     if (Target && isa<ElementRegion>(Target))
       return false;
 
+    // FIXME: This is a hack. We don't use the correct region for a new
+    // expression, so if we inline the constructor its result will just be
+    // thrown away. This short-term hack is tracked in <rdar://problem/12180598>
+    // and the longer-term possible fix is discussed in PR12014.
+    const CXXConstructExpr *CtorExpr = Ctor.getOriginExpr();
+    if (const Stmt *Parent = CurLC->getParentMap().getParent(CtorExpr))
+      if (isa<CXXNewExpr>(Parent))
+        return false;
+
+    // Inlining constructors requires including initializers in the CFG.
+    const AnalysisDeclContext *ADC = CallerSFC->getAnalysisDeclContext();
+    assert(ADC->getCFGBuildOptions().AddInitializers && "No CFG initializers");
+    (void)ADC;
+
+    // If the destructor is trivial, it's always safe to inline the constructor.
+    if (Ctor.getDecl()->getParent()->hasTrivialDestructor())
+      break;
+    
+    // For other types, only inline constructors if destructor inlining is
+    // also enabled.
+    if (!Opts.mayInlineCXXMemberFunction(CIMK_Destructors))
+      return false;
+
     // FIXME: This is a hack. We don't handle temporary destructors
     // right now, so we shouldn't inline their constructors.
-    const CXXConstructExpr *CtorExpr = Ctor.getOriginExpr();
     if (CtorExpr->getConstructionKind() == CXXConstructExpr::CK_Complete)
       if (!Target || !isa<DeclRegion>(Target))
         return false;
@@ -351,15 +530,13 @@ bool ExprEngine::inlineCall(const CallEvent &Call, const Decl *D,
     break;
   }
   case CE_CXXDestructor: {
-    if (!CXX_INLINING_ENABLED)
+    if (!Opts.mayInlineCXXMemberFunction(CIMK_Destructors))
       return false;
 
-    // Only inline constructors and destructors if we built the CFGs for them
-    // properly.
+    // Inlining destructors requires building the CFG correctly.
     const AnalysisDeclContext *ADC = CallerSFC->getAnalysisDeclContext();
-    if (!ADC->getCFGBuildOptions().AddImplicitDtors ||
-        !ADC->getCFGBuildOptions().AddInitializers)
-      return false;
+    assert(ADC->getCFGBuildOptions().AddImplicitDtors && "No CFG destructors");
+    (void)ADC;
 
     const CXXDestructorCall &Dtor = cast<CXXDestructorCall>(Call);
 
@@ -371,9 +548,6 @@ bool ExprEngine::inlineCall(const CallEvent &Call, const Decl *D,
     break;
   }
   case CE_CXXAllocator:
-    if (!CXX_INLINING_ENABLED)
-      return false;
-
     // Do not inline allocators until we model deallocators.
     // This is unfortunate, but basically necessary for smart pointers and such.
     return false;
@@ -387,8 +561,10 @@ bool ExprEngine::inlineCall(const CallEvent &Call, const Decl *D,
     break;
   }
   case CE_ObjCMessage:
-    if (!(getAnalysisManager().IPAMode == DynamicDispatch ||
-          getAnalysisManager().IPAMode == DynamicDispatchBifurcate))
+    if (!Opts.mayInlineObjCMethod())
+      return false;
+    if (!(getAnalysisManager().options.IPAMode == DynamicDispatch ||
+          getAnalysisManager().options.IPAMode == DynamicDispatchBifurcate))
       return false;
     break;
   }
@@ -406,8 +582,8 @@ bool ExprEngine::inlineCall(const CallEvent &Call, const Decl *D,
   AnalysisDeclContext *CalleeADC = AMgr.getAnalysisDeclContext(D);
   const StackFrameContext *CalleeSFC =
     CalleeADC->getStackFrame(ParentOfCallee, CallE,
-                             currentBuilderContext->getBlock(),
-                             currentStmtIdx);
+                             currBldrCtx->getBlock(),
+                             currStmtIdx);
   
   CallEnter Loc(CallE, CalleeSFC, CurLC);
 
@@ -426,6 +602,12 @@ bool ExprEngine::inlineCall(const CallEvent &Call, const Decl *D,
   // added onto the work list so remove it from the node builder.
   Bldr.takeNodes(Pred);
 
+  NumInlinedCalls++;
+
+  // Mark the decl as visited.
+  if (VisitedCallees)
+    VisitedCallees->insert(D);
+
   return true;
 }
 
@@ -520,8 +702,8 @@ ProgramStateRef ExprEngine::bindReturnValue(const CallEvent &Call,
   // Conjure a symbol if the return value is unknown.
   QualType ResultTy = Call.getResultType();
   SValBuilder &SVB = getSValBuilder();
-  unsigned Count = currentBuilderContext->getCurrentBlockCount();
-  SVal R = SVB.getConjuredSymbolVal(0, E, LCtx, ResultTy, Count);
+  unsigned Count = currBldrCtx->blockCount();
+  SVal R = SVB.conjureSymbolVal(0, E, LCtx, ResultTy, Count);
   return State->BindExpr(E, LCtx, R);
 }
 
@@ -529,8 +711,7 @@ ProgramStateRef ExprEngine::bindReturnValue(const CallEvent &Call,
 // a conjured return value.
 void ExprEngine::conservativeEvalCall(const CallEvent &Call, NodeBuilder &Bldr,
                                       ExplodedNode *Pred, ProgramStateRef State) {
-  unsigned Count = currentBuilderContext->getCurrentBlockCount();
-  State = Call.invalidateRegions(Count, State);
+  State = Call.invalidateRegions(currBldrCtx->blockCount(), State);
   State = bindReturnValue(Call, Pred->getLocationContext(), State);
 
   // And make the result node.
@@ -562,13 +743,13 @@ void ExprEngine::defaultEvalCall(NodeBuilder &Bldr, ExplodedNode *Pred,
     if (D) {
       if (RD.mayHaveOtherDefinitions()) {
         // Explore with and without inlining the call.
-        if (getAnalysisManager().IPAMode == DynamicDispatchBifurcate) {
+        if (getAnalysisManager().options.IPAMode == DynamicDispatchBifurcate) {
           BifurcateCall(RD.getDispatchRegion(), *Call, D, Bldr, Pred);
           return;
         }
 
         // Don't inline if we're not in any dynamic dispatch mode.
-        if (getAnalysisManager().IPAMode != DynamicDispatch) {
+        if (getAnalysisManager().options.IPAMode != DynamicDispatch) {
           conservativeEvalCall(*Call, Bldr, Pred, State);
           return;
         }
@@ -593,7 +774,7 @@ void ExprEngine::BifurcateCall(const MemRegion *BifurReg,
   // Check if we've performed the split already - note, we only want
   // to split the path once per memory region.
   ProgramStateRef State = Pred->getState();
-  const unsigned int *BState =
+  const unsigned *BState =
                         State->get<DynamicDispatchBifurcationMap>(BifurReg);
   if (BState) {
     // If we are on "inline path", keep inlining if possible.
@@ -630,7 +811,7 @@ void ExprEngine::VisitReturnStmt(const ReturnStmt *RS, ExplodedNode *Pred,
   ExplodedNodeSet dstPreVisit;
   getCheckerManager().runCheckersForPreStmt(dstPreVisit, Pred, RS, *this);
 
-  StmtNodeBuilder B(dstPreVisit, Dst, *currentBuilderContext);
+  StmtNodeBuilder B(dstPreVisit, Dst, *currBldrCtx);
   
   if (RS->getRetValue()) {
     for (ExplodedNodeSet::iterator it = dstPreVisit.begin(),
diff --git a/lib/StaticAnalyzer/Core/ExprEngineObjC.cpp b/lib/StaticAnalyzer/Core/ExprEngineObjC.cpp
index e3bc498..51dda19b 100644
--- a/lib/StaticAnalyzer/Core/ExprEngineObjC.cpp
+++ b/lib/StaticAnalyzer/Core/ExprEngineObjC.cpp
@@ -28,7 +28,7 @@ void ExprEngine::VisitLvalObjCIvarRefExpr(const ObjCIvarRefExpr *Ex,
   SVal location = state->getLValue(Ex->getDecl(), baseVal);
   
   ExplodedNodeSet dstIvar;
-  StmtNodeBuilder Bldr(Pred, dstIvar, *currentBuilderContext);
+  StmtNodeBuilder Bldr(Pred, dstIvar, *currBldrCtx);
   Bldr.generateNode(Ex, Pred, state->BindExpr(Ex, LCtx, location));
   
   // Perform the post-condition check of the ObjCIvarRefExpr and store
@@ -88,7 +88,7 @@ void ExprEngine::VisitObjCForCollectionStmt(const ObjCForCollectionStmt *S,
   evalLocation(dstLocation, S, elem, Pred, state, elementV, NULL, false);
 
   ExplodedNodeSet Tmp;
-  StmtNodeBuilder Bldr(Pred, Tmp, *currentBuilderContext);
+  StmtNodeBuilder Bldr(Pred, Tmp, *currBldrCtx);
 
   for (ExplodedNodeSet::iterator NI = dstLocation.begin(),
        NE = dstLocation.end(); NI!=NE; ++NI) {
@@ -112,8 +112,8 @@ void ExprEngine::VisitObjCForCollectionStmt(const ObjCForCollectionStmt *S,
         //  For now, just 'conjure' up a symbolic value.
         QualType T = R->getValueType();
         assert(Loc::isLocType(T));
-        unsigned Count = currentBuilderContext->getCurrentBlockCount();
-        SymbolRef Sym = SymMgr.getConjuredSymbol(elem, LCtx, T, Count);
+        SymbolRef Sym = SymMgr.conjureSymbol(elem, LCtx, T,
+                                             currBldrCtx->blockCount());
         SVal V = svalBuilder.makeLoc(Sym);
         hasElems = hasElems->bindLoc(elementV, V);
         
@@ -132,14 +132,6 @@ void ExprEngine::VisitObjCForCollectionStmt(const ObjCForCollectionStmt *S,
   getCheckerManager().runCheckersForPostStmt(Dst, Tmp, S, *this);
 }
 
-static bool isSubclass(const ObjCInterfaceDecl *Class, IdentifierInfo *II) {
-  if (!Class)
-    return false;
-  if (Class->getIdentifier() == II)
-    return true;
-  return isSubclass(Class->getSuperClass(), II);
-}
-
 void ExprEngine::VisitObjCMessage(const ObjCMessageExpr *ME,
                                   ExplodedNode *Pred,
                                   ExplodedNodeSet &Dst) {
@@ -157,7 +149,7 @@ void ExprEngine::VisitObjCMessage(const ObjCMessageExpr *ME,
 
   // Proceed with evaluate the message expression.
   ExplodedNodeSet dstEval;
-  StmtNodeBuilder Bldr(dstGenericPrevisit, dstEval, *currentBuilderContext);
+  StmtNodeBuilder Bldr(dstGenericPrevisit, dstEval, *currBldrCtx);
 
   for (ExplodedNodeSet::iterator DI = dstGenericPrevisit.begin(),
        DE = dstGenericPrevisit.end(); DI != DE; ++DI) {
@@ -184,68 +176,30 @@ void ExprEngine::VisitObjCMessage(const ObjCMessageExpr *ME,
         
         // Check if the "raise" message was sent.
         assert(notNilState);
-        if (Msg->getSelector() == RaiseSel) {
+        if (ObjCNoRet.isImplicitNoReturn(ME)) {
           // If we raise an exception, for now treat it as a sink.
           // Eventually we will want to handle exceptions properly.
-          Bldr.generateNode(currentStmt, Pred, State, true);
+          Bldr.generateSink(currStmt, Pred, State);
           continue;
         }
         
         // Generate a transition to non-Nil state.
-        if (notNilState != State)
-          Pred = Bldr.generateNode(currentStmt, Pred, notNilState);
+        if (notNilState != State) {
+          Pred = Bldr.generateNode(currStmt, Pred, notNilState);
+          assert(Pred && "Should have cached out already!");
+        }
       }
     } else {
-      // Check for special class methods.
-      if (const ObjCInterfaceDecl *Iface = Msg->getReceiverInterface()) {
-        if (!NSExceptionII) {
-          ASTContext &Ctx = getContext();
-          NSExceptionII = &Ctx.Idents.get("NSException");
-        }
-        
-        if (isSubclass(Iface, NSExceptionII)) {
-          enum { NUM_RAISE_SELECTORS = 2 };
-          
-          // Lazily create a cache of the selectors.
-          if (!NSExceptionInstanceRaiseSelectors) {
-            ASTContext &Ctx = getContext();
-            NSExceptionInstanceRaiseSelectors =
-              new Selector[NUM_RAISE_SELECTORS];
-            SmallVector<IdentifierInfo*, NUM_RAISE_SELECTORS> II;
-            unsigned idx = 0;
-            
-            // raise:format:
-            II.push_back(&Ctx.Idents.get("raise"));
-            II.push_back(&Ctx.Idents.get("format"));
-            NSExceptionInstanceRaiseSelectors[idx++] =
-              Ctx.Selectors.getSelector(II.size(), &II[0]);
-            
-            // raise:format:arguments:
-            II.push_back(&Ctx.Idents.get("arguments"));
-            NSExceptionInstanceRaiseSelectors[idx++] =
-              Ctx.Selectors.getSelector(II.size(), &II[0]);
-          }
-          
-          Selector S = Msg->getSelector();
-          bool RaisesException = false;
-          for (unsigned i = 0; i < NUM_RAISE_SELECTORS; ++i) {
-            if (S == NSExceptionInstanceRaiseSelectors[i]) {
-              RaisesException = true;
-              break;
-            }
-          }
-          if (RaisesException) {
-            // If we raise an exception, for now treat it as a sink.
-            // Eventually we will want to handle exceptions properly.
-            Bldr.generateNode(currentStmt, Pred, Pred->getState(), true);
-            continue;
-          }
-
-        }
+      // Check for special class methods that are known to not return
+      // and that we should treat as a sink.
+      if (ObjCNoRet.isImplicitNoReturn(ME)) {
+        // If we raise an exception, for now treat it as a sink.
+        // Eventually we will want to handle exceptions properly.
+        Bldr.generateSink(currStmt, Pred, Pred->getState());
+        continue;
       }
     }
 
-    // Evaluate the call.
     defaultEvalCall(Bldr, Pred, *UpdatedMsg);
   }
   
diff --git a/lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp b/lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp
index 982bcbf..fd875f6 100644
--- a/lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp
+++ b/lib/StaticAnalyzer/Core/HTMLDiagnostics.cpp
@@ -17,8 +17,8 @@
 #include "clang/AST/Decl.h"
 #include "clang/Basic/SourceManager.h"
 #include "clang/Basic/FileManager.h"
-#include "clang/Rewrite/Rewriter.h"
-#include "clang/Rewrite/HTMLRewrite.h"
+#include "clang/Rewrite/Core/Rewriter.h"
+#include "clang/Rewrite/Core/HTMLRewrite.h"
 #include "clang/Lex/Lexer.h"
 #include "clang/Lex/Preprocessor.h"
 #include "llvm/Support/FileSystem.h"
@@ -189,7 +189,7 @@ void HTMLDiagnostics::ReportDiag(const PathDiagnostic& D,
       << (*path.rbegin())->getLocation().asLocation().getExpansionColumnNumber()
       << "</a></td></tr>\n"
          "<tr><td class=\"rowname\">Description:</td><td>"
-      << D.getDescription() << "</td></tr>\n";
+      << D.getVerboseDescription() << "</td></tr>\n";
 
     // Output any other meta data.
 
@@ -209,15 +209,15 @@ void HTMLDiagnostics::ReportDiag(const PathDiagnostic& D,
     std::string s;
     llvm::raw_string_ostream os(s);
 
-    const std::string& BugDesc = D.getDescription();
+    StringRef BugDesc = D.getVerboseDescription();
     if (!BugDesc.empty())
       os << "\n<!-- BUGDESC " << BugDesc << " -->\n";
 
-    const std::string& BugType = D.getBugType();
+    StringRef BugType = D.getBugType();
     if (!BugType.empty())
       os << "\n<!-- BUGTYPE " << BugType << " -->\n";
 
-    const std::string& BugCategory = D.getCategory();
+    StringRef BugCategory = D.getCategory();
     if (!BugCategory.empty())
       os << "\n<!-- BUGCATEGORY " << BugCategory << " -->\n";
 
@@ -267,8 +267,7 @@ void HTMLDiagnostics::ReportDiag(const PathDiagnostic& D,
   }
 
   if (filesMade) {
-    filesMade->push_back(std::make_pair(StringRef(getName()),
-                                        llvm::sys::path::filename(H.str())));
+    filesMade->addDiagnostic(D, getName(), llvm::sys::path::filename(H.str()));
   }
 
   // Emit the HTML to disk.
diff --git a/lib/StaticAnalyzer/Core/MemRegion.cpp b/lib/StaticAnalyzer/Core/MemRegion.cpp
index 62e602a..fab10cf 100644
--- a/lib/StaticAnalyzer/Core/MemRegion.cpp
+++ b/lib/StaticAnalyzer/Core/MemRegion.cpp
@@ -352,7 +352,7 @@ void ElementRegion::Profile(llvm::FoldingSetNodeID& ID) const {
 }
 
 void FunctionTextRegion::ProfileRegion(llvm::FoldingSetNodeID& ID,
-                                       const FunctionDecl *FD,
+                                       const NamedDecl *FD,
                                        const MemRegion*) {
   ID.AddInteger(MemRegion::FunctionTextRegionKind);
   ID.AddPointer(FD);
@@ -444,7 +444,7 @@ void MemRegion::dumpToStream(raw_ostream &os) const {
 }
 
 void AllocaRegion::dumpToStream(raw_ostream &os) const {
-  os << "alloca{" << (void*) Ex << ',' << Cnt << '}';
+  os << "alloca{" << (const void*) Ex << ',' << Cnt << '}';
 }
 
 void FunctionTextRegion::dumpToStream(raw_ostream &os) const {
@@ -452,7 +452,7 @@ void FunctionTextRegion::dumpToStream(raw_ostream &os) const {
 }
 
 void BlockTextRegion::dumpToStream(raw_ostream &os) const {
-  os << "block_code{" << (void*) this << '}';
+  os << "block_code{" << (const void*) this << '}';
 }
 
 void BlockDataRegion::dumpToStream(raw_ostream &os) const {
@@ -461,12 +461,12 @@ void BlockDataRegion::dumpToStream(raw_ostream &os) const {
 
 void CompoundLiteralRegion::dumpToStream(raw_ostream &os) const {
   // FIXME: More elaborate pretty-printing.
-  os << "{ " << (void*) CL <<  " }";
+  os << "{ " << (const void*) CL <<  " }";
 }
 
 void CXXTempObjectRegion::dumpToStream(raw_ostream &os) const {
   os << "temp_object{" << getValueType().getAsString() << ','
-     << (void*) Ex << '}';
+     << (const void*) Ex << '}';
 }
 
 void CXXBaseObjectRegion::dumpToStream(raw_ostream &os) const {
@@ -748,11 +748,11 @@ const VarRegion* MemRegionManager::getVarRegion(const VarDecl *D,
       }
       else {
         assert(D->isStaticLocal());
-        const Decl *D = STC->getDecl();
-        if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(D))
+        const Decl *STCD = STC->getDecl();
+        if (isa<FunctionDecl>(STCD) || isa<ObjCMethodDecl>(STCD))
           sReg = getGlobalsRegion(MemRegion::StaticGlobalSpaceRegionKind,
-                                  getFunctionTextRegion(FD));
-        else if (const BlockDecl *BD = dyn_cast<BlockDecl>(D)) {
+                                  getFunctionTextRegion(cast<NamedDecl>(STCD)));
+        else if (const BlockDecl *BD = dyn_cast<BlockDecl>(STCD)) {
           const BlockTextRegion *BTR =
             getBlockTextRegion(BD,
                      C.getCanonicalType(BD->getSignatureAsWritten()->getType()),
@@ -761,8 +761,6 @@ const VarRegion* MemRegionManager::getVarRegion(const VarDecl *D,
                                   BTR);
         }
         else {
-          // FIXME: For ObjC-methods, we need a new CodeTextRegion.  For now
-          // just use the main global memspace.
           sReg = getGlobalsRegion();
         }
       }
@@ -845,7 +843,7 @@ MemRegionManager::getElementRegion(QualType elementType, NonLoc Idx,
 }
 
 const FunctionTextRegion *
-MemRegionManager::getFunctionTextRegion(const FunctionDecl *FD) {
+MemRegionManager::getFunctionTextRegion(const NamedDecl *FD) {
   return getSubRegion<FunctionTextRegion>(FD, getCodeRegion());
 }
 
@@ -990,6 +988,10 @@ const MemRegion *MemRegion::getBaseRegion() const {
   return R;
 }
 
+bool MemRegion::isSubRegionOf(const MemRegion *R) const {
+  return false;
+}
+
 //===----------------------------------------------------------------------===//
 // View handling.
 //===----------------------------------------------------------------------===//
@@ -1107,7 +1109,7 @@ RegionOffset MemRegion::getAsOffset() const {
         // If our base region is symbolic, we don't know what type it really is.
         // Pretend the type of the symbol is the true dynamic type.
         // (This will at least be self-consistent for the life of the symbol.)
-        Ty = SR->getSymbol()->getType(getContext())->getPointeeType();
+        Ty = SR->getSymbol()->getType()->getPointeeType();
       }
       
       const CXXRecordDecl *Child = Ty->getAsCXXRecordDecl();
@@ -1166,8 +1168,12 @@ RegionOffset MemRegion::getAsOffset() const {
       R = FR->getSuperRegion();
 
       const RecordDecl *RD = FR->getDecl()->getParent();
-      if (!RD->isCompleteDefinition()) {
+      if (RD->isUnion() || !RD->isCompleteDefinition()) {
         // We cannot compute offset for incomplete type.
+        // For unions, we could treat everything as offset 0, but we'd rather
+        // treat each field as a symbolic offset so they aren't stored on top
+        // of each other, since we depend on things in typed regions actually
+        // matching their types.
         SymbolicOffsetBase = R;
       }
 
diff --git a/lib/StaticAnalyzer/Core/PathDiagnostic.cpp b/lib/StaticAnalyzer/Core/PathDiagnostic.cpp
index c849778..0f48d1e 100644
--- a/lib/StaticAnalyzer/Core/PathDiagnostic.cpp
+++ b/lib/StaticAnalyzer/Core/PathDiagnostic.cpp
@@ -21,6 +21,7 @@
 #include "clang/AST/ParentMap.h"
 #include "clang/AST/StmtCXX.h"
 #include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringExtras.h"
 
 using namespace clang;
 using namespace ento;
@@ -104,11 +105,12 @@ void PathPieces::flattenTo(PathPieces &Primary, PathPieces &Current,
 PathDiagnostic::~PathDiagnostic() {}
 
 PathDiagnostic::PathDiagnostic(const Decl *declWithIssue,
-                               StringRef bugtype, StringRef desc,
-                               StringRef category)
+                               StringRef bugtype, StringRef verboseDesc,
+                               StringRef shortDesc, StringRef category)
   : DeclWithIssue(declWithIssue),
     BugType(StripTrailingDots(bugtype)),
-    Desc(StripTrailingDots(desc)),
+    VerboseDesc(StripTrailingDots(verboseDesc)),
+    ShortDesc(StripTrailingDots(shortDesc)),
     Category(StripTrailingDots(category)),
     path(pathImpl) {}
 
@@ -198,6 +200,7 @@ void PathDiagnosticConsumer::HandlePathDiagnostic(PathDiagnostic *D) {
     if (orig_size <= new_size)
       return;
 
+    assert(orig != D);
     Diags.RemoveNode(orig);
     delete orig;
   }
@@ -205,39 +208,151 @@ void PathDiagnosticConsumer::HandlePathDiagnostic(PathDiagnostic *D) {
   Diags.InsertNode(OwningD.take());
 }
 
+static llvm::Optional<bool> comparePath(const PathPieces &X,
+                                        const PathPieces &Y);
+static llvm::Optional<bool>
+compareControlFlow(const PathDiagnosticControlFlowPiece &X,
+                   const PathDiagnosticControlFlowPiece &Y) {
+  FullSourceLoc XSL = X.getStartLocation().asLocation();
+  FullSourceLoc YSL = Y.getStartLocation().asLocation();
+  if (XSL != YSL)
+    return XSL.isBeforeInTranslationUnitThan(YSL);
+  FullSourceLoc XEL = X.getEndLocation().asLocation();
+  FullSourceLoc YEL = Y.getEndLocation().asLocation();
+  if (XEL != YEL)
+    return XEL.isBeforeInTranslationUnitThan(YEL);
+  return llvm::Optional<bool>();
+}
+
+static llvm::Optional<bool>
+compareMacro(const PathDiagnosticMacroPiece &X,
+             const PathDiagnosticMacroPiece &Y) {
+  return comparePath(X.subPieces, Y.subPieces);
+}
+
+static llvm::Optional<bool>
+compareCall(const PathDiagnosticCallPiece &X,
+            const PathDiagnosticCallPiece &Y) {
+  FullSourceLoc X_CEL = X.callEnter.asLocation();
+  FullSourceLoc Y_CEL = Y.callEnter.asLocation();
+  if (X_CEL != Y_CEL)
+    return X_CEL.isBeforeInTranslationUnitThan(Y_CEL);
+  FullSourceLoc X_CEWL = X.callEnterWithin.asLocation();
+  FullSourceLoc Y_CEWL = Y.callEnterWithin.asLocation();
+  if (X_CEWL != Y_CEWL)
+    return X_CEWL.isBeforeInTranslationUnitThan(Y_CEWL);
+  FullSourceLoc X_CRL = X.callReturn.asLocation();
+  FullSourceLoc Y_CRL = Y.callReturn.asLocation();
+  if (X_CRL != Y_CRL)
+    return X_CRL.isBeforeInTranslationUnitThan(Y_CRL);
+  return comparePath(X.path, Y.path);
+}
+
+static llvm::Optional<bool> comparePiece(const PathDiagnosticPiece &X,
+                                         const PathDiagnosticPiece &Y) {
+  if (X.getKind() != Y.getKind())
+    return X.getKind() < Y.getKind();
+  
+  FullSourceLoc XL = X.getLocation().asLocation();
+  FullSourceLoc YL = Y.getLocation().asLocation();
+  if (XL != YL)
+    return XL.isBeforeInTranslationUnitThan(YL);
+
+  if (X.getString() != Y.getString())
+    return X.getString() < Y.getString();
+
+  if (X.getRanges().size() != Y.getRanges().size())
+    return X.getRanges().size() < Y.getRanges().size();
+
+  const SourceManager &SM = XL.getManager();
+  
+  for (unsigned i = 0, n = X.getRanges().size(); i < n; ++i) {
+    SourceRange XR = X.getRanges()[i];
+    SourceRange YR = Y.getRanges()[i];
+    if (XR != YR) {
+      if (XR.getBegin() != YR.getBegin())
+        return SM.isBeforeInTranslationUnit(XR.getBegin(), YR.getBegin());
+      return SM.isBeforeInTranslationUnit(XR.getEnd(), YR.getEnd());
+    }
+  }
+  
+  switch (X.getKind()) {
+    case clang::ento::PathDiagnosticPiece::ControlFlow:
+      return compareControlFlow(cast<PathDiagnosticControlFlowPiece>(X),
+                                cast<PathDiagnosticControlFlowPiece>(Y));
+    case clang::ento::PathDiagnosticPiece::Event:
+      return llvm::Optional<bool>();
+    case clang::ento::PathDiagnosticPiece::Macro:
+      return compareMacro(cast<PathDiagnosticMacroPiece>(X),
+                          cast<PathDiagnosticMacroPiece>(Y));
+    case clang::ento::PathDiagnosticPiece::Call:
+      return compareCall(cast<PathDiagnosticCallPiece>(X),
+                         cast<PathDiagnosticCallPiece>(Y));
+  }
+  llvm_unreachable("all cases handled");
+}
+
+static llvm::Optional<bool> comparePath(const PathPieces &X,
+                                        const PathPieces &Y) {
+  if (X.size() != Y.size())
+    return X.size() < Y.size();
+  for (unsigned i = 0, n = X.size(); i != n; ++i) {
+    llvm::Optional<bool> b = comparePiece(*X[i], *Y[i]);
+    if (b.hasValue())
+      return b.getValue();
+  }
+  return llvm::Optional<bool>();
+}
+
+static bool compare(const PathDiagnostic &X, const PathDiagnostic &Y) {
+  FullSourceLoc XL = X.getLocation().asLocation();
+  FullSourceLoc YL = Y.getLocation().asLocation();
+  if (XL != YL)
+    return XL.isBeforeInTranslationUnitThan(YL);
+  if (X.getBugType() != Y.getBugType())
+    return X.getBugType() < Y.getBugType();
+  if (X.getCategory() != Y.getCategory())
+    return X.getCategory() < Y.getCategory();
+  if (X.getVerboseDescription() != Y.getVerboseDescription())
+    return X.getVerboseDescription() < Y.getVerboseDescription();
+  if (X.getShortDescription() != Y.getShortDescription())
+    return X.getShortDescription() < Y.getShortDescription();
+  if (X.getDeclWithIssue() != Y.getDeclWithIssue()) {
+    const Decl *XD = X.getDeclWithIssue();
+    if (!XD)
+      return true;
+    const Decl *YD = Y.getDeclWithIssue();
+    if (!YD)
+      return false;
+    SourceLocation XDL = XD->getLocation();
+    SourceLocation YDL = YD->getLocation();
+    if (XDL != YDL) {
+      const SourceManager &SM = XL.getManager();
+      return SM.isBeforeInTranslationUnit(XDL, YDL);
+    }
+  }
+  PathDiagnostic::meta_iterator XI = X.meta_begin(), XE = X.meta_end();
+  PathDiagnostic::meta_iterator YI = Y.meta_begin(), YE = Y.meta_end();
+  if (XE - XI != YE - YI)
+    return (XE - XI) < (YE - YI);
+  for ( ; XI != XE ; ++XI, ++YI) {
+    if (*XI != *YI)
+      return (*XI) < (*YI);
+  }
+  llvm::Optional<bool> b = comparePath(X.path, Y.path);
+  assert(b.hasValue());
+  return b.getValue();
+}
 
 namespace {
 struct CompareDiagnostics {
   // Compare if 'X' is "<" than 'Y'.
   bool operator()(const PathDiagnostic *X, const PathDiagnostic *Y) const {
-    // First compare by location
-    const FullSourceLoc &XLoc = X->getLocation().asLocation();
-    const FullSourceLoc &YLoc = Y->getLocation().asLocation();
-    if (XLoc < YLoc)
-      return true;
-    if (XLoc != YLoc)
+    if (X == Y)
       return false;
-    
-    // Next, compare by bug type.
-    StringRef XBugType = X->getBugType();
-    StringRef YBugType = Y->getBugType();
-    if (XBugType < YBugType)
-      return true;
-    if (XBugType != YBugType)
-      return false;
-    
-    // Next, compare by bug description.
-    StringRef XDesc = X->getDescription();
-    StringRef YDesc = Y->getDescription();
-    if (XDesc < YDesc)
-      return true;
-    if (XDesc != YDesc)
-      return false;
-    
-    // FIXME: Further refine by comparing PathDiagnosticPieces?
-    return false;    
-  }  
-};  
+    return compare(*X, *Y);
+  }
+};
 }
 
 void PathDiagnosticConsumer::FlushDiagnostics(
@@ -250,11 +365,9 @@ void PathDiagnosticConsumer::FlushDiagnostics(
   std::vector<const PathDiagnostic *> BatchDiags;
   for (llvm::FoldingSet<PathDiagnostic>::iterator it = Diags.begin(),
        et = Diags.end(); it != et; ++it) {
-    BatchDiags.push_back(&*it);
+    const PathDiagnostic *D = &*it;
+    BatchDiags.push_back(D);
   }
-  
-  // Clear out the FoldingSet.
-  Diags.clear();
 
   // Sort the diagnostics so that they are always emitted in a deterministic
   // order.
@@ -269,6 +382,42 @@ void PathDiagnosticConsumer::FlushDiagnostics(
     const PathDiagnostic *D = *it;
     delete D;
   }
+  
+  // Clear out the FoldingSet.
+  Diags.clear();
+}
+
+void PathDiagnosticConsumer::FilesMade::addDiagnostic(const PathDiagnostic &PD,
+                                                      StringRef ConsumerName,
+                                                      StringRef FileName) {
+  llvm::FoldingSetNodeID NodeID;
+  NodeID.Add(PD);
+  void *InsertPos;
+  PDFileEntry *Entry = FindNodeOrInsertPos(NodeID, InsertPos);
+  if (!Entry) {
+    Entry = Alloc.Allocate<PDFileEntry>();
+    Entry = new (Entry) PDFileEntry(NodeID);
+    InsertNode(Entry, InsertPos);
+  }
+  
+  // Allocate persistent storage for the file name.
+  char *FileName_cstr = (char*) Alloc.Allocate(FileName.size(), 1);
+  memcpy(FileName_cstr, FileName.data(), FileName.size());
+
+  Entry->files.push_back(std::make_pair(ConsumerName,
+                                        StringRef(FileName_cstr,
+                                                  FileName.size())));
+}
+
+PathDiagnosticConsumer::PDFileEntry::ConsumerFiles *
+PathDiagnosticConsumer::FilesMade::getFiles(const PathDiagnostic &PD) {
+  llvm::FoldingSetNodeID NodeID;
+  NodeID.Add(PD);
+  void *InsertPos;
+  PDFileEntry *Entry = FindNodeOrInsertPos(NodeID, InsertPos);
+  if (!Entry)
+    return 0;
+  return &Entry->files;
 }
 
 //===----------------------------------------------------------------------===//
@@ -437,8 +586,8 @@ PathDiagnosticLocation
     const CFGBlock *BSrc = BE->getSrc();
     S = BSrc->getTerminatorCondition();
   }
-  else if (const PostStmt *PS = dyn_cast<PostStmt>(&P)) {
-    S = PS->getStmt();
+  else if (const StmtPoint *SP = dyn_cast<StmtPoint>(&P)) {
+    S = SP->getStmt();
   }
   else if (const PostImplicitCall *PIE = dyn_cast<PostImplicitCall>(&P)) {
     return PathDiagnosticLocation(PIE->getLocation(), SMng);
@@ -453,6 +602,9 @@ PathDiagnosticLocation
                                 CEE->getLocationContext(),
                                 SMng);
   }
+  else {
+    llvm_unreachable("Unexpected ProgramPoint");
+  }
 
   return PathDiagnosticLocation(S, SMng, P.getLocationContext());
 }
@@ -463,21 +615,26 @@ PathDiagnosticLocation
   assert(N && "Cannot create a location with a null node.");
 
   const ExplodedNode *NI = N;
+  const Stmt *S = 0;
 
   while (NI) {
     ProgramPoint P = NI->getLocation();
-    const LocationContext *LC = P.getLocationContext();
     if (const StmtPoint *PS = dyn_cast<StmtPoint>(&P))
-      return PathDiagnosticLocation(PS->getStmt(), SM, LC);
-    else if (const BlockEdge *BE = dyn_cast<BlockEdge>(&P)) {
-      const Stmt *Term = BE->getSrc()->getTerminator();
-      if (Term) {
-        return PathDiagnosticLocation(Term, SM, LC);
-      }
-    }
+      S = PS->getStmt();
+    else if (const BlockEdge *BE = dyn_cast<BlockEdge>(&P))
+      S = BE->getSrc()->getTerminator();
+    if (S)
+      break;
     NI = NI->succ_empty() ? 0 : *(NI->succ_begin());
   }
 
+  if (S) {
+    const LocationContext *LC = NI->getLocationContext();
+    if (S->getLocStart().isValid())
+      return PathDiagnosticLocation(S, SM, LC);
+    return PathDiagnosticLocation(getValidSourceLocation(S, LC), SM);
+  }
+
   return createDeclEnd(N->getLocationContext(), SM);
 }
 
@@ -587,24 +744,6 @@ void PathDiagnosticLocation::flatten() {
   }
 }
 
-PathDiagnosticLocation PathDiagnostic::getLocation() const {
-  assert(path.size() > 0 &&
-         "getLocation() requires a non-empty PathDiagnostic.");
-  
-  PathDiagnosticPiece *p = path.rbegin()->getPtr();
-  
-  while (true) {
-    if (PathDiagnosticCallPiece *cp = dyn_cast<PathDiagnosticCallPiece>(p)) {
-      assert(!cp->path.empty());
-      p = cp->path.rbegin()->getPtr();
-      continue;
-    }
-    break;
-  }
-  
-  return p->getLocation();
-}
-
 //===----------------------------------------------------------------------===//
 // Manipulation of PathDiagnosticCallPieces.
 //===----------------------------------------------------------------------===//
@@ -753,10 +892,9 @@ void PathDiagnosticMacroPiece::Profile(llvm::FoldingSetNodeID &ID) const {
 }
 
 void PathDiagnostic::Profile(llvm::FoldingSetNodeID &ID) const {
-  if (!path.empty())
-    getLocation().Profile(ID);
+  ID.Add(getLocation());
   ID.AddString(BugType);
-  ID.AddString(Desc);
+  ID.AddString(VerboseDesc);
   ID.AddString(Category);
 }
 
@@ -818,42 +956,16 @@ std::string StackHintGeneratorForSymbol::getMessage(const ExplodedNode *N){
   return getMessageForSymbolNotFound();
 }
 
-/// TODO: This is copied from clang diagnostics. Maybe we could just move it to
-/// some common place. (Same as HandleOrdinalModifier.)
-void StackHintGeneratorForSymbol::printOrdinal(unsigned ValNo,
-                                               llvm::raw_svector_ostream &Out) {
-  assert(ValNo != 0 && "ValNo must be strictly positive!");
-
-  // We could use text forms for the first N ordinals, but the numeric
-  // forms are actually nicer in diagnostics because they stand out.
-  Out << ValNo;
-
-  // It is critically important that we do this perfectly for
-  // user-written sequences with over 100 elements.
-  switch (ValNo % 100) {
-  case 11:
-  case 12:
-  case 13:
-    Out << "th"; return;
-  default:
-    switch (ValNo % 10) {
-    case 1: Out << "st"; return;
-    case 2: Out << "nd"; return;
-    case 3: Out << "rd"; return;
-    default: Out << "th"; return;
-    }
-  }
-}
-
 std::string StackHintGeneratorForSymbol::getMessageForArg(const Expr *ArgE,
-                                                        unsigned ArgIndex) {
+                                                          unsigned ArgIndex) {
+  // Printed parameters start at 1, not 0.
+  ++ArgIndex;
+
   SmallString<200> buf;
   llvm::raw_svector_ostream os(buf);
 
-  os << Msg << " via ";
-  // Printed parameters start at 1, not 0.
-  printOrdinal(++ArgIndex, os);
-  os << " parameter";
+  os << Msg << " via " << ArgIndex << llvm::getOrdinalSuffix(ArgIndex)
+     << " parameter";
 
   return os.str();
 }
diff --git a/lib/StaticAnalyzer/Core/PlistDiagnostics.cpp b/lib/StaticAnalyzer/Core/PlistDiagnostics.cpp
index d5fdd9d..17ef4cf 100644
--- a/lib/StaticAnalyzer/Core/PlistDiagnostics.cpp
+++ b/lib/StaticAnalyzer/Core/PlistDiagnostics.cpp
@@ -13,8 +13,9 @@
 
 #include "clang/StaticAnalyzer/Core/PathDiagnosticConsumers.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/PathDiagnostic.h"
-#include "clang/Basic/SourceManager.h"
 #include "clang/Basic/FileManager.h"
+#include "clang/Basic/SourceManager.h"
+#include "clang/Basic/Version.h"
 #include "clang/Lex/Preprocessor.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/Casting.h"
@@ -47,7 +48,6 @@ namespace {
     PathGenerationScheme getGenerationScheme() const { return Extensive; }
     bool supportsLogicalOpControlFlow() const { return true; }
     bool supportsAllBlockEdges() const { return true; }
-    virtual bool useVerboseDescription() const { return false; }
     virtual bool supportsCrossFileDiagnostics() const {
       return SupportsCrossFileDiagnostics;
     }
@@ -247,6 +247,7 @@ static void ReportEvent(raw_ostream &o, const PathDiagnosticPiece& P,
   // Output the short text.
   // FIXME: Really use a short string.
   Indent(o, indent) << "<key>message</key>\n";
+  Indent(o, indent);
   EmitString(o, P.getString()) << '\n';
   
   // Finish up.
@@ -409,10 +410,13 @@ void PlistDiagnostics::FlushDiagnosticsImpl(
   "<plist version=\"1.0\">\n";
 
   // Write the root object: a <dict> containing...
+  //  - "clang_version", the string representation of clang version
   //  - "files", an <array> mapping from FIDs to file names
   //  - "diagnostics", an <array> containing the path diagnostics
-  o << "<dict>\n"
-       " <key>files</key>\n"
+  o << "<dict>\n" <<
+       " <key>clang_version</key>\n";
+  EmitString(o, getClangFullVersion()) << '\n';
+  o << " <key>files</key>\n"
        " <array>\n";
 
   for (SmallVectorImpl<FileID>::iterator I=Fids.begin(), E=Fids.end();
@@ -443,7 +447,7 @@ void PlistDiagnostics::FlushDiagnosticsImpl(
 
     // Output the bug type and bug category.
     o << "   <key>description</key>";
-    EmitString(o, D->getDescription()) << '\n';
+    EmitString(o, D->getShortDescription()) << '\n';
     o << "   <key>category</key>";
     EmitString(o, D->getCategory()) << '\n';
     o << "   <key>type</key>";
@@ -499,19 +503,23 @@ void PlistDiagnostics::FlushDiagnosticsImpl(
     // Output the diagnostic to the sub-diagnostic client, if any.
     if (!filesMade->empty()) {
       StringRef lastName;
-      for (FilesMade::iterator I = filesMade->begin(), E = filesMade->end();
-           I != E; ++I) {
-        StringRef newName = I->first;
-        if (newName != lastName) {
-          if (!lastName.empty())
-            o << "  </array>\n";
-          lastName = newName;
-          o <<  "  <key>" << lastName << "_files</key>\n";
-          o << "  <array>\n";
+      PDFileEntry::ConsumerFiles *files = filesMade->getFiles(*D);
+      if (files) {
+        for (PDFileEntry::ConsumerFiles::const_iterator CI = files->begin(),
+                CE = files->end(); CI != CE; ++CI) {
+          StringRef newName = CI->first;
+          if (newName != lastName) {
+            if (!lastName.empty()) {
+              o << "  </array>\n";
+            }
+            lastName = newName;
+            o <<  "  <key>" << lastName << "_files</key>\n";
+            o << "  <array>\n";
+          }
+          o << "   <string>" << CI->second << "</string>\n";
         }
-        o << "   <string>" << I->second << "</string>\n";
+        o << "  </array>\n";
       }
-      o << "  </array>\n";
     }
 
     // Close up the entry.
@@ -521,10 +529,5 @@ void PlistDiagnostics::FlushDiagnosticsImpl(
   o << " </array>\n";
 
   // Finish.
-  o << "</dict>\n</plist>";
-  
-  if (filesMade) {
-    StringRef Name(getName());
-    filesMade->push_back(std::make_pair(Name, OutputFile));
-  }
+  o << "</dict>\n</plist>";  
 }
diff --git a/lib/StaticAnalyzer/Core/ProgramState.cpp b/lib/StaticAnalyzer/Core/ProgramState.cpp
index 2000338..b49a11e 100644
--- a/lib/StaticAnalyzer/Core/ProgramState.cpp
+++ b/lib/StaticAnalyzer/Core/ProgramState.cpp
@@ -22,10 +22,6 @@
 using namespace clang;
 using namespace ento;
 
-// Give the vtable for ConstraintManager somewhere to live.
-// FIXME: Move this elsewhere.
-ConstraintManager::~ConstraintManager() {}
-
 namespace clang { namespace  ento {
 /// Increments the number of times this state is referenced.
 
@@ -75,8 +71,8 @@ ProgramStateManager::ProgramStateManager(ASTContext &Ctx,
                                          StoreManagerCreator CreateSMgr,
                                          ConstraintManagerCreator CreateCMgr,
                                          llvm::BumpPtrAllocator &alloc,
-                                         SubEngine &SubEng)
-  : Eng(&SubEng), EnvMgr(alloc), GDMFactory(alloc),
+                                         SubEngine *SubEng)
+  : Eng(SubEng), EnvMgr(alloc), GDMFactory(alloc),
     svalBuilder(createSimpleSValBuilder(alloc, Ctx, *this)),
     CallEventMgr(new CallEventManager(alloc)), Alloc(alloc) {
   StoreMgr.reset((*CreateSMgr)(*this));
@@ -110,47 +106,25 @@ ProgramStateManager::removeDeadBindings(ProgramStateRef state,
                                                    SymReaper);
   NewState.setStore(newStore);
   SymReaper.setReapedStore(newStore);
-  
-  return getPersistentState(NewState);
-}
-
-ProgramStateRef ProgramStateManager::MarshalState(ProgramStateRef state,
-                                            const StackFrameContext *InitLoc) {
-  // make up an empty state for now.
-  ProgramState State(this,
-                EnvMgr.getInitialEnvironment(),
-                StoreMgr->getInitialStore(InitLoc),
-                GDMFactory.getEmptyMap());
 
-  return getPersistentState(State);
+  ProgramStateRef Result = getPersistentState(NewState);
+  return ConstraintMgr->removeDeadBindings(Result, SymReaper);
 }
 
 ProgramStateRef ProgramState::bindCompoundLiteral(const CompoundLiteralExpr *CL,
                                             const LocationContext *LC,
                                             SVal V) const {
   const StoreRef &newStore = 
-    getStateManager().StoreMgr->BindCompoundLiteral(getStore(), CL, LC, V);
+    getStateManager().StoreMgr->bindCompoundLiteral(getStore(), CL, LC, V);
   return makeWithStore(newStore);
 }
 
-ProgramStateRef ProgramState::bindDecl(const VarRegion* VR, SVal IVal) const {
-  const StoreRef &newStore =
-    getStateManager().StoreMgr->BindDecl(getStore(), VR, IVal);
-  return makeWithStore(newStore);
-}
-
-ProgramStateRef ProgramState::bindDeclWithNoInit(const VarRegion* VR) const {
-  const StoreRef &newStore =
-    getStateManager().StoreMgr->BindDeclWithNoInit(getStore(), VR);
-  return makeWithStore(newStore);
-}
-
-ProgramStateRef ProgramState::bindLoc(Loc LV, SVal V) const {
+ProgramStateRef ProgramState::bindLoc(Loc LV, SVal V, bool notifyChanges) const {
   ProgramStateManager &Mgr = getStateManager();
   ProgramStateRef newState = makeWithStore(Mgr.StoreMgr->Bind(getStore(), 
                                                              LV, V));
   const MemRegion *MR = LV.getAsRegion();
-  if (MR && Mgr.getOwningEngine())
+  if (MR && Mgr.getOwningEngine() && notifyChanges)
     return Mgr.getOwningEngine()->processRegionChange(newState, MR);
 
   return newState;
@@ -204,11 +178,12 @@ ProgramState::invalidateRegionsImpl(ArrayRef<const MemRegion *> Regions,
   return makeWithStore(newStore);
 }
 
-ProgramStateRef ProgramState::unbindLoc(Loc LV) const {
+ProgramStateRef ProgramState::killBinding(Loc LV) const {
   assert(!isa<loc::MemRegionVal>(LV) && "Use invalidateRegion instead.");
 
   Store OldStore = getStore();
-  const StoreRef &newStore = getStateManager().StoreMgr->Remove(OldStore, LV);
+  const StoreRef &newStore =
+    getStateManager().StoreMgr->killBinding(OldStore, LV);
 
   if (newStore.getStore() == OldStore)
     return this;
@@ -249,7 +224,9 @@ SVal ProgramState::getSVal(Loc location, QualType T) const {
   // about).
   if (!T.isNull()) {
     if (SymbolRef sym = V.getAsSymbol()) {
-      if (const llvm::APSInt *Int = getSymVal(sym)) {
+      if (const llvm::APSInt *Int = getStateManager()
+                                    .getConstraintManager()
+                                    .getSymVal(this, sym)) {
         // FIXME: Because we don't correctly model (yet) sign-extension
         // and truncation of symbolic values, we need to convert
         // the integer value to the correct signedness and bitwidth.
@@ -710,7 +687,9 @@ bool ProgramState::isTainted(SymbolRef Sym, TaintTagType Kind) const {
   bool Tainted = false;
   for (SymExpr::symbol_iterator SI = Sym->symbol_begin(), SE =Sym->symbol_end();
        SI != SE; ++SI) {
-    assert(isa<SymbolData>(*SI));
+    if (!isa<SymbolData>(*SI))
+      continue;
+    
     const TaintTagType *Tag = get<TaintMap>(*SI);
     Tainted = (Tag && *Tag == Kind);
 
@@ -734,15 +713,10 @@ bool ProgramState::isTainted(SymbolRef Sym, TaintTagType Kind) const {
 }
 
 /// The GDM component containing the dynamic type info. This is a map from a
-/// symbol to it's most likely type.
-namespace clang {
-namespace ento {
-typedef llvm::ImmutableMap<const MemRegion *, DynamicTypeInfo> DynamicTypeMap;
-template<> struct ProgramStateTrait<DynamicTypeMap>
-    : public ProgramStatePartialTrait<DynamicTypeMap> {
-  static void *GDMIndex() { static int index; return &index; }
-};
-}}
+/// symbol to its most likely type.
+REGISTER_TRAIT_WITH_PROGRAMSTATE(DynamicTypeMap,
+                                 CLANG_ENTO_PROGRAMSTATE_MAP(const MemRegion *,
+                                                             DynamicTypeInfo))
 
 DynamicTypeInfo ProgramState::getDynamicTypeInfo(const MemRegion *Reg) const {
   Reg = Reg->StripCasts();
@@ -758,7 +732,7 @@ DynamicTypeInfo ProgramState::getDynamicTypeInfo(const MemRegion *Reg) const {
 
   if (const SymbolicRegion *SR = dyn_cast<SymbolicRegion>(Reg)) {
     SymbolRef Sym = SR->getSymbol();
-    return DynamicTypeInfo(Sym->getType(getStateManager().getContext()));
+    return DynamicTypeInfo(Sym->getType());
   }
 
   return DynamicTypeInfo();
diff --git a/lib/StaticAnalyzer/Core/RangeConstraintManager.cpp b/lib/StaticAnalyzer/Core/RangeConstraintManager.cpp
index 550404a..411094b 100644
--- a/lib/StaticAnalyzer/Core/RangeConstraintManager.cpp
+++ b/lib/StaticAnalyzer/Core/RangeConstraintManager.cpp
@@ -24,9 +24,6 @@
 using namespace clang;
 using namespace ento;
 
-namespace { class ConstraintRange {}; }
-static int ConstraintRangeIndex = 0;
-
 /// A Range represents the closed range [from, to].  The caller must
 /// guarantee that from <= to.  Note that Range is immutable, so as not
 /// to subvert RangeSet's immutability.
@@ -280,23 +277,15 @@ public:
 };
 } // end anonymous namespace
 
-typedef llvm::ImmutableMap<SymbolRef,RangeSet> ConstraintRangeTy;
-
-namespace clang {
-namespace ento {
-template<>
-struct ProgramStateTrait<ConstraintRange>
-  : public ProgramStatePartialTrait<ConstraintRangeTy> {
-  static inline void *GDMIndex() { return &ConstraintRangeIndex; }
-};
-}
-}
+REGISTER_TRAIT_WITH_PROGRAMSTATE(ConstraintRange,
+                                 CLANG_ENTO_PROGRAMSTATE_MAP(SymbolRef,
+                                                             RangeSet))
 
 namespace {
 class RangeConstraintManager : public SimpleConstraintManager{
   RangeSet GetRange(ProgramStateRef state, SymbolRef sym);
 public:
-  RangeConstraintManager(SubEngine &subengine, BasicValueFactory &BVF)
+  RangeConstraintManager(SubEngine *subengine, BasicValueFactory &BVF)
     : SimpleConstraintManager(subengine, BVF) {}
 
   ProgramStateRef assumeSymNE(ProgramStateRef state, SymbolRef sym,
@@ -324,12 +313,7 @@ public:
                              const llvm::APSInt& Adjustment);
 
   const llvm::APSInt* getSymVal(ProgramStateRef St, SymbolRef sym) const;
-
-  // FIXME: Refactor into SimpleConstraintManager?
-  bool isEqual(ProgramStateRef St, SymbolRef sym, const llvm::APSInt& V) const {
-    const llvm::APSInt *i = getSymVal(St, sym);
-    return i ? *i == V : false;
-  }
+  ConditionTruthVal checkNull(ProgramStateRef State, SymbolRef Sym);
 
   ProgramStateRef removeDeadBindings(ProgramStateRef St, SymbolReaper& SymReaper);
 
@@ -343,7 +327,7 @@ private:
 } // end anonymous namespace
 
 ConstraintManager *
-ento::CreateRangeConstraintManager(ProgramStateManager &StMgr, SubEngine &Eng) {
+ento::CreateRangeConstraintManager(ProgramStateManager &StMgr, SubEngine *Eng) {
   return new RangeConstraintManager(Eng, StMgr.getBasicVals());
 }
 
@@ -353,6 +337,30 @@ const llvm::APSInt* RangeConstraintManager::getSymVal(ProgramStateRef St,
   return T ? T->getConcreteValue() : NULL;
 }
 
+ConditionTruthVal RangeConstraintManager::checkNull(ProgramStateRef State,
+                                                    SymbolRef Sym) {
+  const RangeSet *Ranges = State->get<ConstraintRange>(Sym);
+
+  // If we don't have any information about this symbol, it's underconstrained.
+  if (!Ranges)
+    return ConditionTruthVal();
+
+  // If we have a concrete value, see if it's zero.
+  if (const llvm::APSInt *Value = Ranges->getConcreteValue())
+    return *Value == 0;
+
+  BasicValueFactory &BV = getBasicVals();
+  APSIntType IntType = BV.getAPSIntType(Sym->getType());
+  llvm::APSInt Zero = IntType.getZeroValue();
+
+  // Check if zero is in the set of possible values.
+  if (Ranges->Intersect(BV, F, Zero, Zero).isEmpty())
+    return false;
+
+  // Zero is a possible value, but it is not the /only/ possible value.
+  return ConditionTruthVal();
+}
+
 /// Scan all symbols referenced by the constraints. If the symbol is not alive
 /// as marked in LSymbols, mark it as dead in DSymbols.
 ProgramStateRef 
@@ -379,8 +387,18 @@ RangeConstraintManager::GetRange(ProgramStateRef state, SymbolRef sym) {
   // Lazily generate a new RangeSet representing all possible values for the
   // given symbol type.
   BasicValueFactory &BV = getBasicVals();
-  QualType T = sym->getType(BV.getContext());
-  return RangeSet(F, BV.getMinValue(T), BV.getMaxValue(T));
+  QualType T = sym->getType();
+
+  RangeSet Result(F, BV.getMinValue(T), BV.getMaxValue(T));
+
+  // Special case: references are known to be non-zero.
+  if (T->isReferenceType()) {
+    APSIntType IntType = BV.getAPSIntType(T);
+    Result = Result.Intersect(BV, F, ++IntType.getZeroValue(),
+                                     --IntType.getZeroValue());
+  }
+
+  return Result;
 }
 
 //===------------------------------------------------------------------------===
diff --git a/lib/StaticAnalyzer/Core/RegionStore.cpp b/lib/StaticAnalyzer/Core/RegionStore.cpp
index bc4e4bb..aed994d 100644
--- a/lib/StaticAnalyzer/Core/RegionStore.cpp
+++ b/lib/StaticAnalyzer/Core/RegionStore.cpp
@@ -15,9 +15,6 @@
 //
 //===----------------------------------------------------------------------===//
 #include "clang/AST/CharUnits.h"
-#include "clang/AST/DeclCXX.h"
-#include "clang/AST/ExprCXX.h"
-#include "clang/AST/CXXInheritance.h"
 #include "clang/Analysis/Analyses/LiveVariables.h"
 #include "clang/Analysis/AnalysisContext.h"
 #include "clang/Basic/TargetInfo.h"
@@ -193,19 +190,6 @@ public:
   ///  casts from arrays to pointers.
   SVal ArrayToPointer(Loc Array);
 
-  /// For DerivedToBase casts, create a CXXBaseObjectRegion and return it.
-  virtual SVal evalDerivedToBase(SVal derived, QualType basePtrType);
-
-  /// \brief Evaluates C++ dynamic_cast cast.
-  /// The callback may result in the following 3 scenarios:
-  ///  - Successful cast (ex: derived is subclass of base).
-  ///  - Failed cast (ex: derived is definitely not a subclass of base).
-  ///  - We don't know (base is a symbolic region and we don't have 
-  ///    enough info to determine if the cast will succeed at run time).
-  /// The function returns an SVal representing the derived class; it's
-  /// valid only if Failed flag is set to false.
-  virtual SVal evalDynamicCast(SVal base, QualType derivedPtrType,bool &Failed);
-
   StoreRef getInitialStore(const LocationContext *InitLoc) {
     return StoreRef(RBFactory.getEmptyMap().getRootWithoutRetain(), *this);
   }
@@ -244,7 +228,7 @@ public:   // Made public for helper classes.
 
   RegionBindings removeBinding(RegionBindings B, BindingKey K);
   RegionBindings removeBinding(RegionBindings B, const MemRegion *R,
-                        BindingKey::Kind k);
+                               BindingKey::Kind k);
 
   RegionBindings removeBinding(RegionBindings B, const MemRegion *R) {
     return removeBinding(removeBinding(B, R, BindingKey::Direct), R,
@@ -266,15 +250,20 @@ public: // Part of public interface to class.
                       .getRootWithoutRetain(), *this);
   }
 
-  StoreRef BindCompoundLiteral(Store store, const CompoundLiteralExpr *CL,
+  /// \brief Create a new store that binds a value to a compound literal.
+  ///
+  /// \param ST The original store whose bindings are the basis for the new
+  ///        store.
+  ///
+  /// \param CL The compound literal to bind (the binding key).
+  ///
+  /// \param LC The LocationContext for the binding.
+  ///
+  /// \param V The value to bind to the compound literal.
+  StoreRef bindCompoundLiteral(Store ST,
+                               const CompoundLiteralExpr *CL,
                                const LocationContext *LC, SVal V);
 
-  StoreRef BindDecl(Store store, const VarRegion *VR, SVal InitVal);
-
-  StoreRef BindDeclWithNoInit(Store store, const VarRegion *) {
-    return StoreRef(store, *this);
-  }
-
   /// BindStruct - Bind a compound value to a structure.
   StoreRef BindStruct(Store store, const TypedValueRegion* R, SVal V);
 
@@ -287,7 +276,10 @@ public: // Part of public interface to class.
   /// as a Default binding.
   StoreRef BindAggregate(Store store, const TypedRegion *R, SVal DefaultVal);
 
-  StoreRef Remove(Store store, Loc LV);
+  /// \brief Create a new store with the specified binding removed.
+  /// \param ST the original store, that is the basis for the new store.
+  /// \param L the location whose binding should be removed.
+  StoreRef killBinding(Store ST, Loc L);
 
   void incrementReferenceCount(Store store) {
     GetRegionBindings(store).manualRetain();    
@@ -477,12 +469,8 @@ public:
   }
 
   bool AddToWorkList(const MemRegion *R, const ClusterBindings *C) {
-    if (C) {
-      if (Visited.count(C))
-        return false;
-      Visited.insert(C);
-    }
-
+    if (C && !Visited.insert(C))
+      return false;
     WL.push_back(R);
     return true;
   }
@@ -534,6 +522,46 @@ bool RegionStoreManager::scanReachableSymbols(Store S, const MemRegion *R,
   return true;
 }
 
+static inline bool isUnionField(const FieldRegion *FR) {
+  return FR->getDecl()->getParent()->isUnion();
+}
+
+typedef SmallVector<const FieldDecl *, 8> FieldVector;
+
+void getSymbolicOffsetFields(BindingKey K, FieldVector &Fields) {
+  assert(K.hasSymbolicOffset() && "Not implemented for concrete offset keys");
+
+  const MemRegion *Base = K.getConcreteOffsetRegion();
+  const MemRegion *R = K.getRegion();
+
+  while (R != Base) {
+    if (const FieldRegion *FR = dyn_cast<FieldRegion>(R))
+      if (!isUnionField(FR))
+        Fields.push_back(FR->getDecl());
+
+    R = cast<SubRegion>(R)->getSuperRegion();
+  }
+}
+
+static bool isCompatibleWithFields(BindingKey K, const FieldVector &Fields) {
+  assert(K.hasSymbolicOffset() && "Not implemented for concrete offset keys");
+
+  if (Fields.empty())
+    return true;
+
+  FieldVector FieldsInBindingKey;
+  getSymbolicOffsetFields(K, FieldsInBindingKey);
+
+  ptrdiff_t Delta = FieldsInBindingKey.size() - Fields.size();
+  if (Delta >= 0)
+    return std::equal(FieldsInBindingKey.begin() + Delta,
+                      FieldsInBindingKey.end(),
+                      Fields.begin());
+  else
+    return std::equal(FieldsInBindingKey.begin(), FieldsInBindingKey.end(),
+                      Fields.begin() - Delta);
+}
+
 RegionBindings RegionStoreManager::removeSubRegionBindings(RegionBindings B,
                                                            const SubRegion *R) {
   BindingKey SRKey = BindingKey::Make(R, BindingKey::Default);
@@ -543,10 +571,12 @@ RegionBindings RegionStoreManager::removeSubRegionBindings(RegionBindings B,
     return RBFactory.remove(B, R);
   }
 
-  if (SRKey.hasSymbolicOffset()) {
-    const SubRegion *Base = cast<SubRegion>(SRKey.getConcreteOffsetRegion());
-    B = removeSubRegionBindings(B, Base);
-    return addBinding(B, Base, BindingKey::Default, UnknownVal());
+  FieldVector FieldsInSymbolicSubregions;
+  bool HasSymbolicOffset = SRKey.hasSymbolicOffset();
+  if (HasSymbolicOffset) {
+    getSymbolicOffsetFields(SRKey, FieldsInSymbolicSubregions);
+    R = cast<SubRegion>(SRKey.getConcreteOffsetRegion());
+    SRKey = BindingKey::Make(R, BindingKey::Default);
   }
 
   // This assumes the region being invalidated is char-aligned. This isn't
@@ -574,11 +604,17 @@ RegionBindings RegionStoreManager::removeSubRegionBindings(RegionBindings B,
        I != E; ++I) {
     BindingKey NextKey = I.getKey();
     if (NextKey.getRegion() == SRKey.getRegion()) {
+      // FIXME: This doesn't catch the case where we're really invalidating a
+      // region with a symbolic offset. Example:
+      //      R: points[i].y
+      //   Next: points[0].x
+
       if (NextKey.getOffset() > SRKey.getOffset() &&
           NextKey.getOffset() - SRKey.getOffset() < Length) {
         // Case 1: The next binding is inside the region we're invalidating.
         // Remove it.
         Result = CBFactory.remove(Result, NextKey);
+
       } else if (NextKey.getOffset() == SRKey.getOffset()) {
         // Case 2: The next binding is at the same offset as the region we're
         // invalidating. In this case, we need to leave default bindings alone,
@@ -589,6 +625,7 @@ RegionBindings RegionStoreManager::removeSubRegionBindings(RegionBindings B,
         if (NextKey.isDirect())
           Result = CBFactory.remove(Result, NextKey);
       }
+      
     } else if (NextKey.hasSymbolicOffset()) {
       const MemRegion *Base = NextKey.getConcreteOffsetRegion();
       if (R->isSubRegionOf(Base)) {
@@ -596,16 +633,24 @@ RegionBindings RegionStoreManager::removeSubRegionBindings(RegionBindings B,
         // its concrete region. We don't know if the binding is still valid, so
         // we'll be conservative and remove it.
         if (NextKey.isDirect())
-          Result = CBFactory.remove(Result, NextKey);
+          if (isCompatibleWithFields(NextKey, FieldsInSymbolicSubregions))
+            Result = CBFactory.remove(Result, NextKey);
       } else if (const SubRegion *BaseSR = dyn_cast<SubRegion>(Base)) {
         // Case 4: The next key is symbolic, but we changed a known
         // super-region. In this case the binding is certainly no longer valid.
         if (R == Base || BaseSR->isSubRegionOf(R))
-          Result = CBFactory.remove(Result, NextKey);
+          if (isCompatibleWithFields(NextKey, FieldsInSymbolicSubregions))
+            Result = CBFactory.remove(Result, NextKey);
       }
     }
   }
 
+  // If we're invalidating a region with a symbolic offset, we need to make sure
+  // we don't treat the base region as uninitialized anymore.
+  // FIXME: This isn't very precise; see the example in the loop.
+  if (HasSymbolicOffset)
+    Result = CBFactory.add(Result, SRKey, UnknownVal());
+
   if (Result.isEmpty())
     return RBFactory.remove(B, ClusterHead);
   return RBFactory.add(B, ClusterHead, Result);
@@ -724,7 +769,7 @@ void invalidateRegionsWorker::VisitBaseRegion(const MemRegion *baseR) {
     // Invalidate the region by setting its default value to
     // conjured symbol. The type of the symbol is irrelavant.
     DefinedOrUnknownSVal V =
-      svalBuilder.getConjuredSymbolVal(baseR, Ex, LCtx, Ctx.IntTy, Count);
+      svalBuilder.conjureSymbolVal(baseR, Ex, LCtx, Ctx.IntTy, Count);
     B = RM.addBinding(B, baseR, BindingKey::Default, V);
     return;
   }
@@ -739,8 +784,8 @@ void invalidateRegionsWorker::VisitBaseRegion(const MemRegion *baseR) {
   if (T->isStructureOrClassType()) {
     // Invalidate the region by setting its default value to
     // conjured symbol. The type of the symbol is irrelavant.
-    DefinedOrUnknownSVal V =
-      svalBuilder.getConjuredSymbolVal(baseR, Ex, LCtx, Ctx.IntTy, Count);
+    DefinedOrUnknownSVal V = svalBuilder.conjureSymbolVal(baseR, Ex, LCtx,
+                                                          Ctx.IntTy, Count);
     B = RM.addBinding(B, baseR, BindingKey::Default, V);
     return;
   }
@@ -748,7 +793,7 @@ void invalidateRegionsWorker::VisitBaseRegion(const MemRegion *baseR) {
   if (const ArrayType *AT = Ctx.getAsArrayType(T)) {
       // Set the default value of the array to conjured symbol.
     DefinedOrUnknownSVal V =
-    svalBuilder.getConjuredSymbolVal(baseR, Ex, LCtx,
+    svalBuilder.conjureSymbolVal(baseR, Ex, LCtx,
                                      AT->getElementType(), Count);
     B = RM.addBinding(B, baseR, BindingKey::Default, V);
     return;
@@ -764,8 +809,8 @@ void invalidateRegionsWorker::VisitBaseRegion(const MemRegion *baseR) {
   }
   
 
-  DefinedOrUnknownSVal V = svalBuilder.getConjuredSymbolVal(baseR, Ex, LCtx,
-                                                            T,Count);
+  DefinedOrUnknownSVal V = svalBuilder.conjureSymbolVal(baseR, Ex, LCtx,
+                                                        T,Count);
   assert(SymbolManager::canSymbolicate(T) || V.isUnknown());
   B = RM.addBinding(B, baseR, BindingKey::Direct, V);
 }
@@ -779,10 +824,9 @@ RegionBindings RegionStoreManager::invalidateGlobalRegion(MemRegion::Kind K,
   // Bind the globals memory space to a new symbol that we will use to derive
   // the bindings for all globals.
   const GlobalsSpaceRegion *GS = MRMgr.getGlobalsRegion(K);
-  SVal V =
-      svalBuilder.getConjuredSymbolVal(/* SymbolTag = */ (void*) GS, Ex, LCtx,
-          /* symbol type, doesn't matter */ Ctx.IntTy,
-          Count);
+  SVal V = svalBuilder.conjureSymbolVal(/* SymbolTag = */ (const void*) GS, Ex, LCtx,
+                                        /* type does not matter */ Ctx.IntTy,
+                                        Count);
 
   B = removeBinding(B, GS);
   B = addBinding(B, BindingKey::Make(GS, BindingKey::Default), V);
@@ -897,103 +941,6 @@ SVal RegionStoreManager::ArrayToPointer(Loc Array) {
   return loc::MemRegionVal(MRMgr.getElementRegion(T, ZeroIdx, ArrayR, Ctx));
 }
 
-// This mirrors Type::getCXXRecordDeclForPointerType(), but there doesn't
-// appear to be another need for this in the rest of the codebase.
-static const CXXRecordDecl *GetCXXRecordDeclForReferenceType(QualType Ty) {
-  if (const ReferenceType *RT = Ty->getAs<ReferenceType>())
-    if (const RecordType *RCT = RT->getPointeeType()->getAs<RecordType>())
-      return dyn_cast<CXXRecordDecl>(RCT->getDecl());
-  return 0;
-}
-
-SVal RegionStoreManager::evalDerivedToBase(SVal derived, QualType baseType) {
-  const CXXRecordDecl *baseDecl;
-  
-  if (baseType->isPointerType())
-    baseDecl = baseType->getCXXRecordDeclForPointerType();
-  else if (baseType->isReferenceType())
-    baseDecl = GetCXXRecordDeclForReferenceType(baseType);
-  else
-    baseDecl = baseType->getAsCXXRecordDecl();
-
-  assert(baseDecl && "not a CXXRecordDecl?");
-
-  loc::MemRegionVal *derivedRegVal = dyn_cast<loc::MemRegionVal>(&derived);
-  if (!derivedRegVal)
-    return derived;
-
-  const MemRegion *baseReg = 
-    MRMgr.getCXXBaseObjectRegion(baseDecl, derivedRegVal->getRegion()); 
-
-  return loc::MemRegionVal(baseReg);
-}
-
-SVal RegionStoreManager::evalDynamicCast(SVal base, QualType derivedType,
-                                         bool &Failed) {
-  Failed = false;
-
-  loc::MemRegionVal *baseRegVal = dyn_cast<loc::MemRegionVal>(&base);
-  if (!baseRegVal)
-    return UnknownVal();
-  const MemRegion *BaseRegion = baseRegVal->stripCasts(/*StripBases=*/false);
-
-  // Assume the derived class is a pointer or a reference to a CXX record.
-  derivedType = derivedType->getPointeeType();
-  assert(!derivedType.isNull());
-  const CXXRecordDecl *DerivedDecl = derivedType->getAsCXXRecordDecl();
-  if (!DerivedDecl && !derivedType->isVoidType())
-    return UnknownVal();
-
-  // Drill down the CXXBaseObject chains, which represent upcasts (casts from
-  // derived to base).
-  const MemRegion *SR = BaseRegion;
-  while (const TypedRegion *TSR = dyn_cast_or_null<TypedRegion>(SR)) {
-    QualType BaseType = TSR->getLocationType()->getPointeeType();
-    assert(!BaseType.isNull());
-    const CXXRecordDecl *SRDecl = BaseType->getAsCXXRecordDecl();
-    if (!SRDecl)
-      return UnknownVal();
-
-    // If found the derived class, the cast succeeds.
-    if (SRDecl == DerivedDecl)
-      return loc::MemRegionVal(TSR);
-
-    if (!derivedType->isVoidType()) {
-      // Static upcasts are marked as DerivedToBase casts by Sema, so this will
-      // only happen when multiple or virtual inheritance is involved.
-      CXXBasePaths Paths(/*FindAmbiguities=*/false, /*RecordPaths=*/true,
-                         /*DetectVirtual=*/false);
-      if (SRDecl->isDerivedFrom(DerivedDecl, Paths)) {
-        SVal Result = loc::MemRegionVal(TSR);
-        const CXXBasePath &Path = *Paths.begin();
-        for (CXXBasePath::const_iterator I = Path.begin(), E = Path.end();
-             I != E; ++I) {
-          Result = evalDerivedToBase(Result, I->Base->getType());
-        }
-        return Result;
-      }
-    }
-
-    if (const CXXBaseObjectRegion *R = dyn_cast<CXXBaseObjectRegion>(TSR))
-      // Drill down the chain to get the derived classes.
-      SR = R->getSuperRegion();
-    else {
-      // We reached the bottom of the hierarchy.
-
-      // If this is a cast to void*, return the region.
-      if (derivedType->isVoidType())
-        return loc::MemRegionVal(TSR);
-
-      // We did not find the derived class. We we must be casting the base to
-      // derived, so the cast should fail.
-      Failed = true;
-      return UnknownVal();
-    }
-  }
-
-  return UnknownVal();
-}
-
 //===----------------------------------------------------------------------===//
 // Loading values from regions.
 //===----------------------------------------------------------------------===//
@@ -1047,7 +994,7 @@ SVal RegionStoreManager::getBinding(Store store, Loc L, QualType T) {
         T = TR->getLocationType();
       else {
         const SymbolicRegion *SR = cast<SymbolicRegion>(MR);
-        T = SR->getSymbol()->getType(Ctx);
+        T = SR->getSymbol()->getType();
       }
     }
     MR = GetElementZeroRegion(MR, T);
@@ -1540,14 +1487,14 @@ bool RegionStoreManager::includedInBindings(Store store,
 // Binding values to regions.
 //===----------------------------------------------------------------------===//
 
-StoreRef RegionStoreManager::Remove(Store store, Loc L) {
+StoreRef RegionStoreManager::killBinding(Store ST, Loc L) {
   if (isa<loc::MemRegionVal>(L))
     if (const MemRegion* R = cast<loc::MemRegionVal>(L).getRegion())
-      return StoreRef(removeBinding(GetRegionBindings(store),
+      return StoreRef(removeBinding(GetRegionBindings(ST),
                                     R).getRootWithoutRetain(),
                       *this);
 
-  return StoreRef(store, *this);
+  return StoreRef(ST, *this);
 }
 
 StoreRef RegionStoreManager::Bind(Store store, Loc L, SVal V) {
@@ -1560,6 +1507,8 @@ StoreRef RegionStoreManager::Bind(Store store, Loc L, SVal V) {
   // Check if the region is a struct region.
   if (const TypedValueRegion* TR = dyn_cast<TypedValueRegion>(R)) {
     QualType Ty = TR->getValueType();
+    if (Ty->isArrayType())
+      return BindArray(store, TR, V);
     if (Ty->isStructureOrClassType())
       return BindStruct(store, TR, V);
     if (Ty->isVectorType())
@@ -1569,13 +1518,9 @@ StoreRef RegionStoreManager::Bind(Store store, Loc L, SVal V) {
   if (const SymbolicRegion *SR = dyn_cast<SymbolicRegion>(R)) {
     // Binding directly to a symbolic region should be treated as binding
     // to element 0.
-    QualType T = SR->getSymbol()->getType(Ctx);
-
-    // FIXME: Is this the right way to handle symbols that are references?
-    if (const PointerType *PT = T->getAs<PointerType>())
-      T = PT->getPointeeType();
-    else
-      T = T->getAs<ReferenceType>()->getPointeeType();
+    QualType T = SR->getSymbol()->getType();
+    if (T->isAnyPointerType() || T->isReferenceType())
+      T = T->getPointeeType();
 
     R = GetElementZeroRegion(SR, T);
   }
@@ -1589,26 +1534,12 @@ StoreRef RegionStoreManager::Bind(Store store, Loc L, SVal V) {
   return StoreRef(addBinding(B, Key, V).getRootWithoutRetain(), *this);
 }
 
-StoreRef RegionStoreManager::BindDecl(Store store, const VarRegion *VR,
-                                      SVal InitVal) {
-
-  QualType T = VR->getDecl()->getType();
-
-  if (T->isArrayType())
-    return BindArray(store, VR, InitVal);
-  if (T->isStructureOrClassType())
-    return BindStruct(store, VR, InitVal);
-
-  return Bind(store, svalBuilder.makeLoc(VR), InitVal);
-}
-
 // FIXME: this method should be merged into Bind().
-StoreRef RegionStoreManager::BindCompoundLiteral(Store store,
+StoreRef RegionStoreManager::bindCompoundLiteral(Store ST,
                                                  const CompoundLiteralExpr *CL,
                                                  const LocationContext *LC,
                                                  SVal V) {
-  return Bind(store, loc::MemRegionVal(MRMgr.getCompoundLiteralRegion(CL, LC)),
-              V);
+  return Bind(ST, loc::MemRegionVal(MRMgr.getCompoundLiteralRegion(CL, LC)), V);
 }
 
 StoreRef RegionStoreManager::setImplicitDefaultValue(Store store,
@@ -1864,7 +1795,7 @@ RegionBindings RegionStoreManager::removeBinding(RegionBindings B,
 
 RegionBindings RegionStoreManager::removeBinding(RegionBindings B,
                                                  const MemRegion *R,
-                                                BindingKey::Kind k){
+                                                 BindingKey::Kind k){
   return removeBinding(B, BindingKey::Make(R, k));
 }
 
@@ -1897,7 +1828,6 @@ public:
   void VisitAddedToCluster(const MemRegion *baseR, const ClusterBindings &C);
   void VisitCluster(const MemRegion *baseR, const ClusterBindings &C);
 
-  void VisitBindingKey(BindingKey K);
   bool UpdatePostponed();
   void VisitBinding(SVal V);
 };
@@ -1932,17 +1862,21 @@ void removeDeadBindingsWorker::VisitAddedToCluster(const MemRegion *baseR,
     const StackArgumentsSpaceRegion *StackReg =
       cast<StackArgumentsSpaceRegion>(TR->getSuperRegion());
     const StackFrameContext *RegCtx = StackReg->getStackFrame();
-    if (RegCtx == CurrentLCtx || RegCtx->isParentOf(CurrentLCtx))
+    if (CurrentLCtx &&
+        (RegCtx == CurrentLCtx || RegCtx->isParentOf(CurrentLCtx)))
       AddToWorkList(TR, &C);
   }
 }
 
 void removeDeadBindingsWorker::VisitCluster(const MemRegion *baseR,
                                             const ClusterBindings &C) {
-  for (ClusterBindings::iterator I = C.begin(), E = C.end(); I != E; ++I) {
-    VisitBindingKey(I.getKey());
+  // Mark the symbol for any SymbolicRegion with live bindings as live itself.
+  // This means we should continue to track that symbol.
+  if (const SymbolicRegion *SymR = dyn_cast<SymbolicRegion>(baseR))
+    SymReaper.markLive(SymR->getSymbol());
+
+  for (ClusterBindings::iterator I = C.begin(), E = C.end(); I != E; ++I)
     VisitBinding(I.getData());
-  }
 }
 
 void removeDeadBindingsWorker::VisitBinding(SVal V) {
@@ -1979,8 +1913,8 @@ void removeDeadBindingsWorker::VisitBinding(SVal V) {
     if (const BlockDataRegion *BR = dyn_cast<BlockDataRegion>(R)) {
       BlockDataRegion::referenced_vars_iterator I = BR->referenced_vars_begin(),
                                                 E = BR->referenced_vars_end();
-        for ( ; I != E; ++I)
-          AddToWorkList(I.getCapturedRegion());
+      for ( ; I != E; ++I)
+        AddToWorkList(I.getCapturedRegion());
     }
   }
     
@@ -1991,20 +1925,6 @@ void removeDeadBindingsWorker::VisitBinding(SVal V) {
     SymReaper.markLive(*SI);
 }
 
-void removeDeadBindingsWorker::VisitBindingKey(BindingKey K) {
-  const MemRegion *R = K.getRegion();
-
-  // Mark this region "live" by adding it to the worklist.  This will cause
-  // use to visit all regions in the cluster (if we haven't visited them
-  // already).
-  if (AddToWorkList(R)) {
-    // Mark the symbol for any live SymbolicRegion as "live".  This means we
-    // should continue to track that symbol.
-    if (const SymbolicRegion *SymR = dyn_cast<SymbolicRegion>(R))
-      SymReaper.markLive(SymR->getSymbol());
-  }
-}
-
 bool removeDeadBindingsWorker::UpdatePostponed() {
   // See if any postponed SymbolicRegions are actually live now, after
   // having done a scan.
@@ -2012,7 +1932,7 @@ bool removeDeadBindingsWorker::UpdatePostponed() {
 
   for (SmallVectorImpl<const SymbolicRegion*>::iterator
         I = Postponed.begin(), E = Postponed.end() ; I != E ; ++I) {
-    if (const SymbolicRegion *SR = cast_or_null<SymbolicRegion>(*I)) {
+    if (const SymbolicRegion *SR = *I) {
       if (SymReaper.isLive(SR->getSymbol())) {
         changed |= AddToWorkList(SR);
         *I = NULL;
diff --git a/lib/StaticAnalyzer/Core/SValBuilder.cpp b/lib/StaticAnalyzer/Core/SValBuilder.cpp
index d1936cd..b87169a 100644
--- a/lib/StaticAnalyzer/Core/SValBuilder.cpp
+++ b/lib/StaticAnalyzer/Core/SValBuilder.cpp
@@ -106,25 +106,23 @@ SValBuilder::getRegionValueSymbolVal(const TypedValueRegion* region) {
   return nonloc::SymbolVal(sym);
 }
 
-DefinedOrUnknownSVal
-SValBuilder::getConjuredSymbolVal(const void *symbolTag,
-                                  const Expr *expr,
-                                  const LocationContext *LCtx,
-                                  unsigned count) {
+DefinedOrUnknownSVal SValBuilder::conjureSymbolVal(const void *symbolTag,
+                                                   const Expr *expr,
+                                                   const LocationContext *LCtx,
+                                                   unsigned count) {
   QualType T = expr->getType();
-  return getConjuredSymbolVal(symbolTag, expr, LCtx, T, count);
+  return conjureSymbolVal(symbolTag, expr, LCtx, T, count);
 }
 
-DefinedOrUnknownSVal
-SValBuilder::getConjuredSymbolVal(const void *symbolTag,
-                                  const Expr *expr,
-                                  const LocationContext *LCtx,
-                                  QualType type,
-                                  unsigned count) {
+DefinedOrUnknownSVal SValBuilder::conjureSymbolVal(const void *symbolTag,
+                                                   const Expr *expr,
+                                                   const LocationContext *LCtx,
+                                                   QualType type,
+                                                   unsigned count) {
   if (!SymbolManager::canSymbolicate(type))
     return UnknownVal();
 
-  SymbolRef sym = SymMgr.getConjuredSymbol(expr, LCtx, type, count, symbolTag);
+  SymbolRef sym = SymMgr.conjureSymbol(expr, LCtx, type, count, symbolTag);
 
   if (Loc::isLocType(type))
     return loc::MemRegionVal(MemMgr.getSymbolicRegion(sym));
@@ -133,15 +131,14 @@ SValBuilder::getConjuredSymbolVal(const void *symbolTag,
 }
 
 
-DefinedOrUnknownSVal
-SValBuilder::getConjuredSymbolVal(const Stmt *stmt,
-                                  const LocationContext *LCtx,
-                                  QualType type,
-                                  unsigned visitCount) {
+DefinedOrUnknownSVal SValBuilder::conjureSymbolVal(const Stmt *stmt,
+                                                   const LocationContext *LCtx,
+                                                   QualType type,
+                                                   unsigned visitCount) {
   if (!SymbolManager::canSymbolicate(type))
     return UnknownVal();
 
-  SymbolRef sym = SymMgr.getConjuredSymbol(stmt, LCtx, type, visitCount);
+  SymbolRef sym = SymMgr.conjureSymbol(stmt, LCtx, type, visitCount);
   
   if (Loc::isLocType(type))
     return loc::MemRegionVal(MemMgr.getSymbolicRegion(sym));
@@ -157,7 +154,7 @@ SValBuilder::getConjuredHeapSymbolVal(const Expr *E,
   assert(Loc::isLocType(T));
   assert(SymbolManager::canSymbolicate(T));
 
-  SymbolRef sym = SymMgr.getConjuredSymbol(E, LCtx, T, VisitCount);
+  SymbolRef sym = SymMgr.conjureSymbol(E, LCtx, T, VisitCount);
   return loc::MemRegionVal(MemMgr.getSymbolicHeapRegion(sym));
 }
 
diff --git a/lib/StaticAnalyzer/Core/SVals.cpp b/lib/StaticAnalyzer/Core/SVals.cpp
index 8437f50..e34ab6a 100644
--- a/lib/StaticAnalyzer/Core/SVals.cpp
+++ b/lib/StaticAnalyzer/Core/SVals.cpp
@@ -51,7 +51,8 @@ const FunctionDecl *SVal::getAsFunctionDecl() const {
   if (const loc::MemRegionVal* X = dyn_cast<loc::MemRegionVal>(this)) {
     const MemRegion* R = X->getRegion();
     if (const FunctionTextRegion *CTR = R->getAs<FunctionTextRegion>())
-      return CTR->getDecl();
+      if (const FunctionDecl *FD = dyn_cast<FunctionDecl>(CTR->getDecl()))
+        return FD;
   }
 
   return 0;
diff --git a/lib/StaticAnalyzer/Core/SimpleConstraintManager.cpp b/lib/StaticAnalyzer/Core/SimpleConstraintManager.cpp
index 5568f1c..4236ee4 100644
--- a/lib/StaticAnalyzer/Core/SimpleConstraintManager.cpp
+++ b/lib/StaticAnalyzer/Core/SimpleConstraintManager.cpp
@@ -67,7 +67,9 @@ ProgramStateRef SimpleConstraintManager::assume(ProgramStateRef state,
 ProgramStateRef SimpleConstraintManager::assume(ProgramStateRef state, Loc cond,
                                                bool assumption) {
   state = assumeAux(state, cond, assumption);
-  return SU.processAssume(state, cond, assumption);
+  if (NotifyAssumeClients && SU)
+    return SU->processAssume(state, cond, assumption);
+  return state;
 }
 
 ProgramStateRef SimpleConstraintManager::assumeAux(ProgramStateRef state,
@@ -113,7 +115,9 @@ ProgramStateRef SimpleConstraintManager::assume(ProgramStateRef state,
                                                NonLoc cond,
                                                bool assumption) {
   state = assumeAux(state, cond, assumption);
-  return SU.processAssume(state, cond, assumption);
+  if (NotifyAssumeClients && SU)
+    return SU->processAssume(state, cond, assumption);
+  return state;
 }
 
 static BinaryOperator::Opcode NegateComparison(BinaryOperator::Opcode op) {
@@ -136,7 +140,7 @@ ProgramStateRef
 SimpleConstraintManager::assumeAuxForSymbol(ProgramStateRef State,
                                             SymbolRef Sym, bool Assumption) {
   BasicValueFactory &BVF = getBasicVals();
-  QualType T = Sym->getType(BVF.getContext());
+  QualType T = Sym->getType();
 
   // None of the constraint solvers currently support non-integer types.
   if (!T->isIntegerType())
@@ -186,7 +190,7 @@ ProgramStateRef SimpleConstraintManager::assumeAux(ProgramStateRef state,
       BinaryOperator::Opcode op = SE->getOpcode();
       // Implicitly compare non-comparison expressions to 0.
       if (!BinaryOperator::isComparisonOp(op)) {
-        QualType T = SE->getType(BasicVals.getContext());
+        QualType T = SE->getType();
         const llvm::APSInt &zero = BasicVals.getValue(0, T);
         op = (Assumption ? BO_NE : BO_EQ);
         return assumeSymRel(state, SE, op, zero);
@@ -235,11 +239,9 @@ ProgramStateRef SimpleConstraintManager::assumeSymRel(ProgramStateRef state,
   assert(BinaryOperator::isComparisonOp(op) &&
          "Non-comparison ops should be rewritten as comparisons to zero.");
 
-  BasicValueFactory &BVF = getBasicVals();
-  ASTContext &Ctx = BVF.getContext();
-
   // Get the type used for calculating wraparound.
-  APSIntType WraparoundType = BVF.getAPSIntType(LHS->getType(Ctx));
+  BasicValueFactory &BVF = getBasicVals();
+  APSIntType WraparoundType = BVF.getAPSIntType(LHS->getType());
 
   // We only handle simple comparisons of the form "$sym == constant"
   // or "($sym+constant1) == constant2".
diff --git a/lib/StaticAnalyzer/Core/SimpleConstraintManager.h b/lib/StaticAnalyzer/Core/SimpleConstraintManager.h
index 088d70c..01f0b4e 100644
--- a/lib/StaticAnalyzer/Core/SimpleConstraintManager.h
+++ b/lib/StaticAnalyzer/Core/SimpleConstraintManager.h
@@ -22,10 +22,10 @@ namespace clang {
 namespace ento {
 
 class SimpleConstraintManager : public ConstraintManager {
-  SubEngine &SU;
+  SubEngine *SU;
   BasicValueFactory &BVF;
 public:
-  SimpleConstraintManager(SubEngine &subengine, BasicValueFactory &BV)
+  SimpleConstraintManager(SubEngine *subengine, BasicValueFactory &BV)
     : SU(subengine), BVF(BV) {}
   virtual ~SimpleConstraintManager();
 
diff --git a/lib/StaticAnalyzer/Core/SimpleSValBuilder.cpp b/lib/StaticAnalyzer/Core/SimpleSValBuilder.cpp
index ad58a07..fbc6ba0 100644
--- a/lib/StaticAnalyzer/Core/SimpleSValBuilder.cpp
+++ b/lib/StaticAnalyzer/Core/SimpleSValBuilder.cpp
@@ -81,7 +81,7 @@ SVal SimpleSValBuilder::evalCastFromNonLoc(NonLoc val, QualType castTy) {
   }
 
   if (const SymExpr *se = val.getAsSymbolicExpression()) {
-    QualType T = Context.getCanonicalType(se->getType(Context));
+    QualType T = Context.getCanonicalType(se->getType());
     // If types are the same or both are integers, ignore the cast.
     // FIXME: Remove this hack when we support symbolic truncation/extension.
     // HACK: If both castTy and T are integers, ignore the cast.  This is
@@ -276,7 +276,7 @@ SVal SimpleSValBuilder::MakeSymIntVal(const SymExpr *LHS,
     // with the given constant.
     // FIXME: This is an approximation of Sema::UsualArithmeticConversions.
     ASTContext &Ctx = getContext();
-    QualType SymbolType = LHS->getType(Ctx);
+    QualType SymbolType = LHS->getType();
     uint64_t ValWidth = RHS.getBitWidth();
     uint64_t TypeWidth = Ctx.getTypeSize(SymbolType);
 
@@ -318,7 +318,9 @@ SVal SimpleSValBuilder::evalBinOpNN(ProgramStateRef state,
         return makeTruthVal(false, resultTy);
       case BO_Xor:
       case BO_Sub:
-        return makeIntVal(0, resultTy);
+        if (resultTy->isIntegralOrEnumerationType())
+          return makeIntVal(0, resultTy);
+        return evalCastFromNonLoc(makeIntVal(0, /*Unsigned=*/false), resultTy);
       case BO_Or:
       case BO_And:
         return evalCastFromNonLoc(lhs, resultTy);
@@ -459,7 +461,7 @@ SVal SimpleSValBuilder::evalBinOpNN(ProgramStateRef state,
           case BO_NE:
             // Negate the comparison and make a value.
             opc = NegateComparison(opc);
-            assert(symIntExpr->getType(Context) == resultTy);
+            assert(symIntExpr->getType() == resultTy);
             return makeNonLoc(symIntExpr->getLHS(), opc,
                 symIntExpr->getRHS(), resultTy);
           }
@@ -505,7 +507,8 @@ SVal SimpleSValBuilder::evalBinOpNN(ProgramStateRef state,
       } else if (isa<SymbolData>(Sym)) {
         // Does the symbol simplify to a constant?  If so, "fold" the constant
         // by setting 'lhs' to a ConcreteInt and try again.
-        if (const llvm::APSInt *Constant = state->getSymVal(Sym)) {
+        if (const llvm::APSInt *Constant = state->getConstraintManager()
+                                                  .getSymVal(state, Sym)) {
           lhs = nonloc::ConcreteInt(*Constant);
           continue;
         }
@@ -916,14 +919,8 @@ SVal SimpleSValBuilder::evalBinOpLN(ProgramStateRef state,
     else if (isa<SubRegion>(region)) {
       superR = region;
       index = rhs;
-      if (const PointerType *PT = resultTy->getAs<PointerType>()) {
-        elementType = PT->getPointeeType();
-      }
-      else {
-        const ObjCObjectPointerType *OT =
-          resultTy->getAs<ObjCObjectPointerType>();
-        elementType = OT->getPointeeType();
-      }
+      if (resultTy->isAnyPointerType())
+        elementType = resultTy->getPointeeType();
     }
 
     if (NonLoc *indexV = dyn_cast<NonLoc>(&index)) {
@@ -946,7 +943,7 @@ const llvm::APSInt *SimpleSValBuilder::getKnownValue(ProgramStateRef state,
     return &X->getValue();
 
   if (SymbolRef Sym = V.getAsSymbol())
-    return state->getSymVal(Sym);
+    return state->getConstraintManager().getSymVal(state, Sym);
 
   // FIXME: Add support for SymExprs.
   return NULL;
diff --git a/lib/StaticAnalyzer/Core/Store.cpp b/lib/StaticAnalyzer/Core/Store.cpp
index 3af60a1..939ae54 100644
--- a/lib/StaticAnalyzer/Core/Store.cpp
+++ b/lib/StaticAnalyzer/Core/Store.cpp
@@ -15,6 +15,7 @@
 #include "clang/StaticAnalyzer/Core/PathSensitive/CallEvent.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/ProgramState.h"
 #include "clang/AST/CharUnits.h"
+#include "clang/AST/CXXInheritance.h"
 #include "clang/AST/DeclObjC.h"
 
 using namespace clang;
@@ -233,6 +234,91 @@ SVal StoreManager::evalDerivedToBase(SVal Derived, const CastExpr *Cast) {
   return Result;
 }
 
+SVal StoreManager::evalDerivedToBase(SVal Derived, const CXXBasePath &Path) {
+  // Walk through the path to create nested CXXBaseRegions.
+  SVal Result = Derived;
+  for (CXXBasePath::const_iterator I = Path.begin(), E = Path.end();
+       I != E; ++I) {
+    Result = evalDerivedToBase(Result, I->Base->getType());
+  }
+  return Result;
+}
+
+SVal StoreManager::evalDerivedToBase(SVal Derived, QualType BaseType) {
+  loc::MemRegionVal *DerivedRegVal = dyn_cast<loc::MemRegionVal>(&Derived);
+  if (!DerivedRegVal)
+    return Derived;
+
+  const CXXRecordDecl *BaseDecl = BaseType->getPointeeCXXRecordDecl();
+  if (!BaseDecl)
+    BaseDecl = BaseType->getAsCXXRecordDecl();
+  assert(BaseDecl && "not a C++ object?");
+
+  const MemRegion *BaseReg =
+    MRMgr.getCXXBaseObjectRegion(BaseDecl, DerivedRegVal->getRegion());
+
+  return loc::MemRegionVal(BaseReg);
+}
+
+SVal StoreManager::evalDynamicCast(SVal Base, QualType DerivedType,
+                                   bool &Failed) {
+  Failed = false;
+
+  loc::MemRegionVal *BaseRegVal = dyn_cast<loc::MemRegionVal>(&Base);
+  if (!BaseRegVal)
+    return UnknownVal();
+  const MemRegion *BaseRegion = BaseRegVal->stripCasts(/*StripBases=*/false);
+
+  // Assume the derived class is a pointer or a reference to a CXX record.
+  DerivedType = DerivedType->getPointeeType();
+  assert(!DerivedType.isNull());
+  const CXXRecordDecl *DerivedDecl = DerivedType->getAsCXXRecordDecl();
+  if (!DerivedDecl && !DerivedType->isVoidType())
+    return UnknownVal();
+
+  // Drill down the CXXBaseObject chains, which represent upcasts (casts from
+  // derived to base).
+  const MemRegion *SR = BaseRegion;
+  while (const TypedRegion *TSR = dyn_cast_or_null<TypedRegion>(SR)) {
+    QualType BaseType = TSR->getLocationType()->getPointeeType();
+    assert(!BaseType.isNull());
+    const CXXRecordDecl *SRDecl = BaseType->getAsCXXRecordDecl();
+    if (!SRDecl)
+      return UnknownVal();
+
+    // If found the derived class, the cast succeeds.
+    if (SRDecl == DerivedDecl)
+      return loc::MemRegionVal(TSR);
+
+    if (!DerivedType->isVoidType()) {
+      // Static upcasts are marked as DerivedToBase casts by Sema, so this will
+      // only happen when multiple or virtual inheritance is involved.
+      CXXBasePaths Paths(/*FindAmbiguities=*/false, /*RecordPaths=*/true,
+                         /*DetectVirtual=*/false);
+      if (SRDecl->isDerivedFrom(DerivedDecl, Paths))
+        return evalDerivedToBase(loc::MemRegionVal(TSR), Paths.front());
+    }
+
+    if (const CXXBaseObjectRegion *R = dyn_cast<CXXBaseObjectRegion>(TSR))
+      // Drill down the chain to get the derived classes.
+      SR = R->getSuperRegion();
+    else {
+      // We reached the bottom of the hierarchy.
+
+      // If this is a cast to void*, return the region.
+      if (DerivedType->isVoidType())
+        return loc::MemRegionVal(TSR);
+
+      // We did not find the derived class. We we must be casting the base to
+      // derived, so the cast should fail.
+      Failed = true;
+      return UnknownVal();
+    }
+  }
+  
+  return UnknownVal();
+}
+
 
 /// CastRetrievedVal - Used by subclasses of StoreManager to implement
 ///  implicit casts that arise from loads from regions that are reinterpreted
diff --git a/lib/StaticAnalyzer/Core/SymbolManager.cpp b/lib/StaticAnalyzer/Core/SymbolManager.cpp
index 0bc192d..0c5098b 100644
--- a/lib/StaticAnalyzer/Core/SymbolManager.cpp
+++ b/lib/StaticAnalyzer/Core/SymbolManager.cpp
@@ -117,21 +117,17 @@ bool SymExpr::symbol_iterator::operator!=(const symbol_iterator &X) const {
 
 SymExpr::symbol_iterator::symbol_iterator(const SymExpr *SE) {
   itr.push_back(SE);
-  while (!isa<SymbolData>(itr.back())) expand();
 }
 
 SymExpr::symbol_iterator &SymExpr::symbol_iterator::operator++() {
   assert(!itr.empty() && "attempting to iterate on an 'end' iterator");
-  assert(isa<SymbolData>(itr.back()));
-  itr.pop_back();
-  if (!itr.empty())
-    while (!isa<SymbolData>(itr.back())) expand();
+  expand();
   return *this;
 }
 
 SymbolRef SymExpr::symbol_iterator::operator*() {
   assert(!itr.empty() && "attempting to dereference an 'end' iterator");
-  return cast<SymbolData>(itr.back());
+  return itr.back();
 }
 
 void SymExpr::symbol_iterator::expand() {
@@ -187,11 +183,11 @@ SymbolManager::getRegionValueSymbol(const TypedValueRegion* R) {
   return cast<SymbolRegionValue>(SD);
 }
 
-const SymbolConjured*
-SymbolManager::getConjuredSymbol(const Stmt *E, const LocationContext *LCtx,
-                                 QualType T, unsigned Count,
-                                 const void *SymbolTag) {
-
+const SymbolConjured* SymbolManager::conjureSymbol(const Stmt *E,
+                                                   const LocationContext *LCtx,
+                                                   QualType T,
+                                                   unsigned Count,
+                                                   const void *SymbolTag) {
   llvm::FoldingSetNodeID profile;
   SymbolConjured::Profile(profile, E, T, Count, LCtx, SymbolTag);
   void *InsertPos;
@@ -328,23 +324,24 @@ const SymSymExpr *SymbolManager::getSymSymExpr(const SymExpr *lhs,
   return cast<SymSymExpr>(data);
 }
 
-QualType SymbolConjured::getType(ASTContext&) const {
+QualType SymbolConjured::getType() const {
   return T;
 }
 
-QualType SymbolDerived::getType(ASTContext &Ctx) const {
+QualType SymbolDerived::getType() const {
   return R->getValueType();
 }
 
-QualType SymbolExtent::getType(ASTContext &Ctx) const {
+QualType SymbolExtent::getType() const {
+  ASTContext &Ctx = R->getMemRegionManager()->getContext();
   return Ctx.getSizeType();
 }
 
-QualType SymbolMetadata::getType(ASTContext&) const {
+QualType SymbolMetadata::getType() const {
   return T;
 }
 
-QualType SymbolRegionValue::getType(ASTContext &C) const {
+QualType SymbolRegionValue::getType() const {
   return R->getValueType();
 }
 
@@ -466,41 +463,56 @@ bool SymbolReaper::isLive(SymbolRef sym) {
     markDependentsLive(sym);
     return true;
   }
-
-  if (const SymbolDerived *derived = dyn_cast<SymbolDerived>(sym)) {
-    if (isLive(derived->getParentSymbol())) {
-      markLive(sym);
-      return true;
-    }
-    return false;
-  }
-
-  if (const SymbolExtent *extent = dyn_cast<SymbolExtent>(sym)) {
-    if (isLiveRegion(extent->getRegion())) {
-      markLive(sym);
-      return true;
-    }
-    return false;
+  
+  bool KnownLive;
+  
+  switch (sym->getKind()) {
+  case SymExpr::RegionValueKind:
+    // FIXME: We should be able to use isLiveRegion here (this behavior
+    // predates isLiveRegion), but doing so causes test failures. Investigate.
+    KnownLive = true;
+    break;
+  case SymExpr::ConjuredKind:
+    KnownLive = false;
+    break;
+  case SymExpr::DerivedKind:
+    KnownLive = isLive(cast<SymbolDerived>(sym)->getParentSymbol());
+    break;
+  case SymExpr::ExtentKind:
+    KnownLive = isLiveRegion(cast<SymbolExtent>(sym)->getRegion());
+    break;
+  case SymExpr::MetadataKind:
+    KnownLive = MetadataInUse.count(sym) &&
+                isLiveRegion(cast<SymbolMetadata>(sym)->getRegion());
+    if (KnownLive)
+      MetadataInUse.erase(sym);
+    break;
+  case SymExpr::SymIntKind:
+    KnownLive = isLive(cast<SymIntExpr>(sym)->getLHS());
+    break;
+  case SymExpr::IntSymKind:
+    KnownLive = isLive(cast<IntSymExpr>(sym)->getRHS());
+    break;
+  case SymExpr::SymSymKind:
+    KnownLive = isLive(cast<SymSymExpr>(sym)->getLHS()) &&
+                isLive(cast<SymSymExpr>(sym)->getRHS());
+    break;
+  case SymExpr::CastSymbolKind:
+    KnownLive = isLive(cast<SymbolCast>(sym)->getOperand());
+    break;
   }
 
-  if (const SymbolMetadata *metadata = dyn_cast<SymbolMetadata>(sym)) {
-    if (MetadataInUse.count(sym)) {
-      if (isLiveRegion(metadata->getRegion())) {
-        markLive(sym);
-        MetadataInUse.erase(sym);
-        return true;
-      }
-    }
-    return false;
-  }
+  if (KnownLive)
+    markLive(sym);
 
-  // Interogate the symbol.  It may derive from an input value to
-  // the analyzed function/method.
-  return isa<SymbolRegionValue>(sym);
+  return KnownLive;
 }
 
 bool
 SymbolReaper::isLive(const Stmt *ExprVal, const LocationContext *ELCtx) const {
+  if (LCtx == 0)
+    return false;
+
   if (LCtx != ELCtx) {
     // If the reaper's location context is a parent of the expression's
     // location context, then the expression value is now "out of scope".
@@ -508,6 +520,7 @@ SymbolReaper::isLive(const Stmt *ExprVal, const LocationContext *ELCtx) const {
       return false;
     return true;
   }
+
   // If no statement is provided, everything is this and parent contexts is live.
   if (!Loc)
     return true;
@@ -517,10 +530,16 @@ SymbolReaper::isLive(const Stmt *ExprVal, const LocationContext *ELCtx) const {
 
 bool SymbolReaper::isLive(const VarRegion *VR, bool includeStoreBindings) const{
   const StackFrameContext *VarContext = VR->getStackFrame();
+
+  if (!VarContext)
+    return true;
+
+  if (!LCtx)
+    return false;
   const StackFrameContext *CurrentContext = LCtx->getCurrentStackFrame();
 
   if (VarContext == CurrentContext) {
-    // If no statemetnt is provided, everything is live.
+    // If no statement is provided, everything is live.
     if (!Loc)
       return true;
 
diff --git a/lib/StaticAnalyzer/Core/TextPathDiagnostics.cpp b/lib/StaticAnalyzer/Core/TextPathDiagnostics.cpp
index 66bf4bb..e09f4e3 100644
--- a/lib/StaticAnalyzer/Core/TextPathDiagnostics.cpp
+++ b/lib/StaticAnalyzer/Core/TextPathDiagnostics.cpp
@@ -41,7 +41,6 @@ public:
   PathGenerationScheme getGenerationScheme() const { return Minimal; }
   bool supportsLogicalOpControlFlow() const { return true; }
   bool supportsAllBlockEdges() const { return true; }
-  virtual bool useVerboseDescription() const { return true; }
   virtual bool supportsCrossFileDiagnostics() const { return true; }
 };
 
diff --git a/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp b/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp
index 34b5266..7dbac3c 100644
--- a/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp
+++ b/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp
@@ -34,7 +34,7 @@
 
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/SourceManager.h"
-#include "clang/Frontend/AnalyzerOptions.h"
+#include "clang/StaticAnalyzer/Core/AnalyzerOptions.h"
 #include "clang/Lex/Preprocessor.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/Path.h"
@@ -78,7 +78,6 @@ public:
   ClangDiagPathDiagConsumer(DiagnosticsEngine &Diag) : Diag(Diag) {}
   virtual ~ClangDiagPathDiagConsumer() {}
   virtual StringRef getName() const { return "ClangDiags"; }
-  virtual bool useVerboseDescription() const { return false; }
   virtual PathGenerationScheme getGenerationScheme() const { return None; }
 
   void FlushDiagnosticsImpl(std::vector<const PathDiagnostic *> &Diags,
@@ -86,7 +85,7 @@ public:
     for (std::vector<const PathDiagnostic*>::iterator I = Diags.begin(),
          E = Diags.end(); I != E; ++I) {
       const PathDiagnostic *PD = *I;
-      StringRef desc = PD->getDescription();
+      StringRef desc = PD->getShortDescription();
       SmallString<512> TmpStr;
       llvm::raw_svector_ostream Out(TmpStr);
       for (StringRef::iterator I=desc.begin(), E=desc.end(); I!=E; ++I) {
@@ -121,11 +120,12 @@ namespace {
 
 class AnalysisConsumer : public ASTConsumer,
                          public RecursiveASTVisitor<AnalysisConsumer> {
-  enum AnalysisMode {
-    ANALYSIS_SYNTAX,
-    ANALYSIS_PATH,
-    ANALYSIS_ALL
+  enum {
+    AM_None = 0,
+    AM_Syntax = 0x1,
+    AM_Path = 0x2
   };
+  typedef unsigned AnalysisMode;
 
   /// Mode of the analyzes while recursively visiting Decls.
   AnalysisMode RecVisitorMode;
@@ -136,7 +136,7 @@ public:
   ASTContext *Ctx;
   const Preprocessor &PP;
   const std::string OutDir;
-  AnalyzerOptions Opts;
+  AnalyzerOptionsRef Opts;
   ArrayRef<std::string> Plugins;
 
   /// \brief Stores the declarations from the local translation unit.
@@ -164,19 +164,19 @@ public:
 
   AnalysisConsumer(const Preprocessor& pp,
                    const std::string& outdir,
-                   const AnalyzerOptions& opts,
+                   AnalyzerOptionsRef opts,
                    ArrayRef<std::string> plugins)
-    : RecVisitorMode(ANALYSIS_ALL), RecVisitorBR(0),
+    : RecVisitorMode(0), RecVisitorBR(0),
       Ctx(0), PP(pp), OutDir(outdir), Opts(opts), Plugins(plugins) {
     DigestAnalyzerOptions();
-    if (Opts.PrintStats) {
+    if (Opts->PrintStats) {
       llvm::EnableStatistics();
       TUTotalTimer = new llvm::Timer("Analyzer Total Time");
     }
   }
 
   ~AnalysisConsumer() {
-    if (Opts.PrintStats)
+    if (Opts->PrintStats)
       delete TUTotalTimer;
   }
 
@@ -185,49 +185,52 @@ public:
     PathConsumers.push_back(new ClangDiagPathDiagConsumer(PP.getDiagnostics()));
 
     if (!OutDir.empty()) {
-      switch (Opts.AnalysisDiagOpt) {
+      switch (Opts->AnalysisDiagOpt) {
       default:
 #define ANALYSIS_DIAGNOSTICS(NAME, CMDFLAG, DESC, CREATEFN, AUTOCREATE) \
         case PD_##NAME: CREATEFN(PathConsumers, OutDir, PP); break;
-#include "clang/Frontend/Analyses.def"
+#include "clang/StaticAnalyzer/Core/Analyses.def"
       }
-    } else if (Opts.AnalysisDiagOpt == PD_TEXT) {
+    } else if (Opts->AnalysisDiagOpt == PD_TEXT) {
       // Create the text client even without a specified output file since
       // it just uses diagnostic notes.
       createTextPathDiagnosticConsumer(PathConsumers, "", PP);
     }
 
     // Create the analyzer component creators.
-    switch (Opts.AnalysisStoreOpt) {
+    switch (Opts->AnalysisStoreOpt) {
     default:
       llvm_unreachable("Unknown store manager.");
 #define ANALYSIS_STORE(NAME, CMDFLAG, DESC, CREATEFN)           \
       case NAME##Model: CreateStoreMgr = CREATEFN; break;
-#include "clang/Frontend/Analyses.def"
+#include "clang/StaticAnalyzer/Core/Analyses.def"
     }
 
-    switch (Opts.AnalysisConstraintsOpt) {
+    switch (Opts->AnalysisConstraintsOpt) {
     default:
       llvm_unreachable("Unknown store manager.");
 #define ANALYSIS_CONSTRAINTS(NAME, CMDFLAG, DESC, CREATEFN)     \
       case NAME##Model: CreateConstraintMgr = CREATEFN; break;
-#include "clang/Frontend/Analyses.def"
+#include "clang/StaticAnalyzer/Core/Analyses.def"
     }
   }
 
   void DisplayFunction(const Decl *D, AnalysisMode Mode) {
-    if (!Opts.AnalyzerDisplayProgress)
+    if (!Opts->AnalyzerDisplayProgress)
       return;
 
     SourceManager &SM = Mgr->getASTContext().getSourceManager();
     PresumedLoc Loc = SM.getPresumedLoc(D->getLocation());
     if (Loc.isValid()) {
       llvm::errs() << "ANALYZE";
-      switch (Mode) {
-        case ANALYSIS_SYNTAX: llvm::errs() << "(Syntax)"; break;
-        case ANALYSIS_PATH: llvm::errs() << "(Path Sensitive)"; break;
-        case ANALYSIS_ALL: break;
-      };
+
+      if (Mode == AM_Syntax)
+        llvm::errs() << " (Syntax)";
+      else if (Mode == AM_Path)
+        llvm::errs() << " (Path)";
+      else
+        assert(Mode == (AM_Syntax | AM_Path) && "Unexpected mode!");
+
       llvm::errs() << ": " << Loc.getFilename();
       if (isa<FunctionDecl>(D) || isa<ObjCMethodDecl>(D)) {
         const NamedDecl *ND = cast<NamedDecl>(D);
@@ -246,7 +249,7 @@ public:
 
   virtual void Initialize(ASTContext &Context) {
     Ctx = &Context;
-    checkerMgr.reset(createCheckerManager(Opts, PP.getLangOpts(), Plugins,
+    checkerMgr.reset(createCheckerManager(*Opts, PP.getLangOpts(), Plugins,
                                           PP.getDiagnostics()));
     Mgr.reset(new AnalysisManager(*Ctx,
                                   PP.getDiagnostics(),
@@ -255,17 +258,7 @@ public:
                                   CreateStoreMgr,
                                   CreateConstraintMgr,
                                   checkerMgr.get(),
-                                  Opts.MaxNodes, Opts.MaxLoop,
-                                  Opts.VisualizeEGDot, Opts.VisualizeEGUbi,
-                                  Opts.AnalysisPurgeOpt, Opts.EagerlyAssume,
-                                  Opts.TrimGraph,
-                                  Opts.UnoptimizedCFG, Opts.CFGAddImplicitDtors,
-                                  Opts.EagerlyTrimEGraph,
-                                  Opts.IPAMode,
-                                  Opts.InlineMaxStackDepth,
-                                  Opts.InlineMaxFunctionSize,
-                                  Opts.InliningMode,
-                                  Opts.NoRetryExhausted));
+                                  *Opts));
   }
 
   /// \brief Store the top level decls in the set to be processed later on.
@@ -277,7 +270,7 @@ public:
 
   /// \brief Build the call graph for all the top level decls of this TU and
   /// use it to define the order in which the functions should be visited.
-  void HandleDeclsGallGraph(const unsigned LocalTUDeclsSize);
+  void HandleDeclsCallGraph(const unsigned LocalTUDeclsSize);
 
   /// \brief Run analyzes(syntax or path sensitive) on the given function.
   /// \param Mode - determines if we are requesting syntax only or path
@@ -297,7 +290,9 @@ public:
 
   /// Handle callbacks for arbitrary Decls.
   bool VisitDecl(Decl *D) {
-    checkerMgr->runCheckersOnASTDecl(D, *Mgr, *RecVisitorBR);
+    AnalysisMode Mode = getModeForDecl(D, RecVisitorMode);
+    if (Mode & AM_Syntax)
+      checkerMgr->runCheckersOnASTDecl(D, *Mgr, *RecVisitorBR);
     return true;
   }
 
@@ -316,7 +311,6 @@ public:
   }
 
   bool VisitObjCMethodDecl(ObjCMethodDecl *MD) {
-    checkerMgr->runCheckersOnASTDecl(MD, *Mgr, *RecVisitorBR);
     if (MD->isThisDeclarationADefinition())
       HandleCode(MD, RecVisitorMode);
     return true;
@@ -326,7 +320,7 @@ private:
   void storeTopLevelDecls(DeclGroupRef DG);
 
   /// \brief Check if we should skip (not analyze) the given function.
-  bool skipFunction(Decl *D);
+  AnalysisMode getModeForDecl(Decl *D, AnalysisMode Mode);
 
 };
 } // end anonymous namespace
@@ -358,7 +352,23 @@ void AnalysisConsumer::storeTopLevelDecls(DeclGroupRef DG) {
   }
 }
 
-void AnalysisConsumer::HandleDeclsGallGraph(const unsigned LocalTUDeclsSize) {
+static bool shouldSkipFunction(CallGraphNode *N,
+                               SmallPtrSet<CallGraphNode*,24> Visited) {
+  // We want to re-analyse the functions as top level in several cases:
+  // - The 'init' methods should be reanalyzed because
+  //   ObjCNonNilReturnValueChecker assumes that '[super init]' never returns
+  //   'nil' and unless we analyze the 'init' functions as top level, we will not
+  //   catch errors within defensive code.
+  // - We want to reanalyze all ObjC methods as top level to report Retain
+  //   Count naming convention errors more aggressively.
+  if (isa<ObjCMethodDecl>(N->getDecl()))
+    return false;
+
+  // Otherwise, if we visited the function before, do not reanalyze it.
+  return Visited.count(N);
+}
+
+void AnalysisConsumer::HandleDeclsCallGraph(const unsigned LocalTUDeclsSize) {
   // Otherwise, use the Callgraph to derive the order.
   // Build the Call Graph.
   CallGraph CG;
@@ -407,21 +417,21 @@ void AnalysisConsumer::HandleDeclsGallGraph(const unsigned LocalTUDeclsSize) {
     // Push the children into the queue.
     for (CallGraphNode::const_iterator CI = N->begin(),
          CE = N->end(); CI != CE; ++CI) {
-      if (!Visited.count(*CI))
+      if (!shouldSkipFunction(*CI, Visited))
         BFSQueue.push_back(*CI);
     }
 
     // Skip the functions which have been processed already or previously
     // inlined.
-    if (Visited.count(N))
+    if (shouldSkipFunction(N, Visited))
       continue;
 
     // Analyze the function.
     SetOfConstDecls VisitedCallees;
     Decl *D = N->getDecl();
     assert(D);
-    HandleCode(D, ANALYSIS_PATH,
-               (Mgr->InliningMode == All ? 0 : &VisitedCallees));
+    HandleCode(D, AM_Path,
+               (Mgr->options.InliningMode == All ? 0 : &VisitedCallees));
 
     // Add the visited callees to the global visited set.
     for (SetOfConstDecls::iterator I = VisitedCallees.begin(),
@@ -451,7 +461,9 @@ void AnalysisConsumer::HandleTranslationUnit(ASTContext &C) {
     // Run the AST-only checks using the order in which functions are defined.
     // If inlining is not turned on, use the simplest function order for path
     // sensitive analyzes as well.
-    RecVisitorMode = (Mgr->shouldInlineCall() ? ANALYSIS_SYNTAX : ANALYSIS_ALL);
+    RecVisitorMode = AM_Syntax;
+    if (!Mgr->shouldInlineCall())
+      RecVisitorMode |= AM_Path;
     RecVisitorBR = &BR;
 
     // Process all the top level declarations.
@@ -466,7 +478,7 @@ void AnalysisConsumer::HandleTranslationUnit(ASTContext &C) {
     }
 
     if (Mgr->shouldInlineCall())
-      HandleDeclsGallGraph(LocalTUDeclsSize);
+      HandleDeclsCallGraph(LocalTUDeclsSize);
 
     // After all decls handled, run checkers on the entire TranslationUnit.
     checkerMgr->runCheckersOnEndOfTranslationUnit(TU, *Mgr, BR);
@@ -513,24 +525,32 @@ static std::string getFunctionName(const Decl *D) {
   return "";
 }
 
-bool AnalysisConsumer::skipFunction(Decl *D) {
-  if (!Opts.AnalyzeSpecificFunction.empty() &&
-      getFunctionName(D) != Opts.AnalyzeSpecificFunction)
-    return true;
-
-  // Don't run the actions on declarations in header files unless
-  // otherwise specified.
+AnalysisConsumer::AnalysisMode
+AnalysisConsumer::getModeForDecl(Decl *D, AnalysisMode Mode) {
+  if (!Opts->AnalyzeSpecificFunction.empty() &&
+      getFunctionName(D) != Opts->AnalyzeSpecificFunction)
+    return AM_None;
+
+  // Unless -analyze-all is specified, treat decls differently depending on
+  // where they came from:
+  // - Main source file: run both path-sensitive and non-path-sensitive checks.
+  // - Header files: run non-path-sensitive checks only.
+  // - System headers: don't run any checks.
   SourceManager &SM = Ctx->getSourceManager();
   SourceLocation SL = SM.getExpansionLoc(D->getLocation());
-  if (!Opts.AnalyzeAll && !SM.isFromMainFile(SL))
-    return true;
+  if (!Opts->AnalyzeAll && !SM.isFromMainFile(SL)) {
+    if (SL.isInvalid() || SM.isInSystemHeader(SL))
+      return AM_None;
+    return Mode & ~AM_Path;
+  }
 
-  return false;
+  return Mode;
 }
 
 void AnalysisConsumer::HandleCode(Decl *D, AnalysisMode Mode,
                                   SetOfConstDecls *VisitedCallees) {
-  if (skipFunction(D))
+  Mode = getModeForDecl(D, Mode);
+  if (Mode == AM_None)
     return;
 
   DisplayFunction(D, Mode);
@@ -548,16 +568,16 @@ void AnalysisConsumer::HandleCode(Decl *D, AnalysisMode Mode,
   SmallVector<Decl*, 10> WL;
   WL.push_back(D);
 
-  if (D->hasBody() && Opts.AnalyzeNestedBlocks)
+  if (D->hasBody() && Opts->AnalyzeNestedBlocks)
     FindBlocks(cast<DeclContext>(D), WL);
 
   BugReporter BR(*Mgr);
   for (SmallVectorImpl<Decl*>::iterator WI=WL.begin(), WE=WL.end();
        WI != WE; ++WI)
     if ((*WI)->hasBody()) {
-      if (Mode != ANALYSIS_PATH)
+      if (Mode & AM_Syntax)
         checkerMgr->runCheckersOnASTBody(*WI, *Mgr, BR);
-      if (Mode != ANALYSIS_SYNTAX && checkerMgr->hasPathSensitiveCheckers()) {
+      if ((Mode & AM_Path) && checkerMgr->hasPathSensitiveCheckers()) {
         RunPathSensitiveChecks(*WI, VisitedCallees);
         NumFunctionsAnalyzed++;
       }
@@ -583,22 +603,22 @@ void AnalysisConsumer::ActionExprEngine(Decl *D, bool ObjCGCEnabled,
 
   // Set the graph auditor.
   OwningPtr<ExplodedNode::Auditor> Auditor;
-  if (Mgr->shouldVisualizeUbigraph()) {
+  if (Mgr->options.visualizeExplodedGraphWithUbiGraph) {
     Auditor.reset(CreateUbiViz());
     ExplodedNode::SetAuditor(Auditor.get());
   }
 
   // Execute the worklist algorithm.
   Eng.ExecuteWorkList(Mgr->getAnalysisDeclContextManager().getStackFrame(D),
-                      Mgr->getMaxNodes());
+                      Mgr->options.MaxNodes);
 
   // Release the auditor (if any) so that it doesn't monitor the graph
   // created BugReporter.
   ExplodedNode::SetAuditor(0);
 
   // Visualize the exploded graph.
-  if (Mgr->shouldVisualizeGraphviz())
-    Eng.ViewGraph(Mgr->shouldTrimGraph());
+  if (Mgr->options.visualizeExplodedGraphWithGraphViz)
+    Eng.ViewGraph(Mgr->options.TrimGraph);
 
   // Display warnings.
   Eng.getBugReporter().FlushReports();
@@ -629,7 +649,7 @@ void AnalysisConsumer::RunPathSensitiveChecks(Decl *D,
 
 ASTConsumer* ento::CreateAnalysisConsumer(const Preprocessor& pp,
                                           const std::string& outDir,
-                                          const AnalyzerOptions& opts,
+                                          AnalyzerOptionsRef opts,
                                           ArrayRef<std::string> plugins) {
   // Disable the effects of '-Werror' when using the AnalysisConsumer.
   pp.getDiagnostics().setWarningsAsErrors(false);
diff --git a/lib/StaticAnalyzer/Frontend/AnalysisConsumer.h b/lib/StaticAnalyzer/Frontend/AnalysisConsumer.h
index 5a16bff..b75220b 100644
--- a/lib/StaticAnalyzer/Frontend/AnalysisConsumer.h
+++ b/lib/StaticAnalyzer/Frontend/AnalysisConsumer.h
@@ -16,11 +16,11 @@
 #define LLVM_CLANG_GR_ANALYSISCONSUMER_H
 
 #include "clang/Basic/LLVM.h"
+#include "clang/StaticAnalyzer/Core/AnalyzerOptions.h"
 #include <string>
 
 namespace clang {
 
-class AnalyzerOptions;
 class ASTConsumer;
 class Preprocessor;
 class DiagnosticsEngine;
@@ -33,7 +33,7 @@ class CheckerManager;
 /// options.)
 ASTConsumer* CreateAnalysisConsumer(const Preprocessor &pp,
                                     const std::string &output,
-                                    const AnalyzerOptions& opts,
+                                    AnalyzerOptionsRef opts,
                                     ArrayRef<std::string> plugins);
 
 } // end GR namespace
diff --git a/lib/StaticAnalyzer/Frontend/CMakeLists.txt b/lib/StaticAnalyzer/Frontend/CMakeLists.txt
index 06d1485..aafb249 100644
--- a/lib/StaticAnalyzer/Frontend/CMakeLists.txt
+++ b/lib/StaticAnalyzer/Frontend/CMakeLists.txt
@@ -25,6 +25,7 @@ target_link_libraries(clangStaticAnalyzerFrontend
   clangLex
   clangAST
   clangFrontend
-  clangRewrite
+  clangRewriteCore
+  clangRewriteFrontend
   clangStaticAnalyzerCheckers
   )
diff --git a/lib/StaticAnalyzer/Frontend/CheckerRegistration.cpp b/lib/StaticAnalyzer/Frontend/CheckerRegistration.cpp
index 0229aed..e8daa65 100644
--- a/lib/StaticAnalyzer/Frontend/CheckerRegistration.cpp
+++ b/lib/StaticAnalyzer/Frontend/CheckerRegistration.cpp
@@ -17,7 +17,7 @@
 #include "clang/StaticAnalyzer/Core/CheckerManager.h"
 #include "clang/StaticAnalyzer/Core/CheckerOptInfo.h"
 #include "clang/StaticAnalyzer/Core/CheckerRegistry.h"
-#include "clang/Frontend/AnalyzerOptions.h"
+#include "clang/StaticAnalyzer/Core/AnalyzerOptions.h"
 #include "clang/Frontend/FrontendDiagnostic.h"
 #include "clang/Basic/Diagnostic.h"
 #include "llvm/Support/DynamicLibrary.h"
diff --git a/lib/Tooling/CMakeLists.txt b/lib/Tooling/CMakeLists.txt
index 49d3101..d29e564 100644
--- a/lib/Tooling/CMakeLists.txt
+++ b/lib/Tooling/CMakeLists.txt
@@ -2,8 +2,10 @@ set(LLVM_LINK_COMPONENTS support)
 
 add_clang_library(clangTooling
   ArgumentsAdjusters.cpp
-  CommandLineClangTool.cpp
+  CommonOptionsParser.cpp
   CompilationDatabase.cpp
+  FileMatchTrie.cpp
+  JSONCompilationDatabase.cpp
   Refactoring.cpp
   RefactoringCallbacks.cpp
   Tooling.cpp
@@ -23,5 +25,6 @@ target_link_libraries(clangTooling
   clangFrontend
   clangAST
   clangASTMatchers
-  clangRewrite
+  clangRewriteCore
+  clangRewriteFrontend
   )
diff --git a/lib/Tooling/CommandLineClangTool.cpp b/lib/Tooling/CommandLineClangTool.cpp
deleted file mode 100644
index 8da2a33..0000000
--- a/lib/Tooling/CommandLineClangTool.cpp
+++ /dev/null
@@ -1,80 +0,0 @@
-//===--- CommandLineClangTool.cpp - command-line clang tools driver -------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//  This file implements the CommandLineClangTool class used to run clang
-//  tools as separate command-line applications with a consistent common
-//  interface for handling compilation database and input files.
-//
-//  It provides a common subset of command-line options, common algorithm
-//  for locating a compilation database and source files, and help messages
-//  for the basic command-line interface.
-//
-//  It creates a CompilationDatabase, initializes a ClangTool and runs a
-//  user-specified FrontendAction over all TUs in which the given files are
-//  compiled.
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/Frontend/FrontendActions.h"
-#include "clang/Tooling/CommandLineClangTool.h"
-#include "clang/Tooling/Tooling.h"
-
-using namespace clang::tooling;
-using namespace llvm;
-
-static const char *MoreHelpText =
-    "\n"
-    "-p <build-path> is used to read a compile command database.\n"
-    "\n"
-    "\tFor example, it can be a CMake build directory in which a file named\n"
-    "\tcompile_commands.json exists (use -DCMAKE_EXPORT_COMPILE_COMMANDS=ON\n"
-    "\tCMake option to get this output). When no build path is specified,\n"
-    "\tclang-check will attempt to locate it automatically using all parent\n"
-    "\tpaths of the first input file. See:\n"
-    "\thttp://clang.llvm.org/docs/HowToSetupToolingForLLVM.html for an\n"
-    "\texample of setting up Clang Tooling on a source tree.\n"
-    "\n"
-    "<source0> ... specify the paths of source files. These paths are looked\n"
-    "\tup in the compile command database. If the path of a file is absolute,\n"
-    "\tit needs to point into CMake's source tree. If the path is relative,\n"
-    "\tthe current working directory needs to be in the CMake source tree and\n"
-    "\tthe file must be in a subdirectory of the current working directory.\n"
-    "\t\"./\" prefixes in the relative files will be automatically removed,\n"
-    "\tbut the rest of a relative path must be a suffix of a path in the\n"
-    "\tcompile command database.\n"
-    "\n";
-
-CommandLineClangTool::CommandLineClangTool() :
-    BuildPath("p", cl::desc("Build path"), cl::Optional),
-    SourcePaths(cl::Positional, cl::desc("<source0> [... <sourceN>]"),
-                cl::OneOrMore),
-    MoreHelp(MoreHelpText) {
-}
-
-void CommandLineClangTool::initialize(int argc, const char **argv) {
-  Compilations.reset(FixedCompilationDatabase::loadFromCommandLine(argc, argv));
-  cl::ParseCommandLineOptions(argc, argv);
-  if (!Compilations) {
-    std::string ErrorMessage;
-    if (!BuildPath.empty()) {
-      Compilations.reset(CompilationDatabase::autoDetectFromDirectory(
-                              BuildPath, ErrorMessage));
-    } else {
-      Compilations.reset(CompilationDatabase::autoDetectFromSource(
-                              SourcePaths[0], ErrorMessage));
-    }
-    if (!Compilations)
-      llvm::report_fatal_error(ErrorMessage);
-  }
-}
-
-int CommandLineClangTool::run(FrontendActionFactory *ActionFactory) {
-  ClangTool Tool(*Compilations, SourcePaths);
-  return Tool.run(ActionFactory);
-}
diff --git a/lib/Tooling/CommonOptionsParser.cpp b/lib/Tooling/CommonOptionsParser.cpp
new file mode 100644
index 0000000..15091c7
--- /dev/null
+++ b/lib/Tooling/CommonOptionsParser.cpp
@@ -0,0 +1,79 @@
+//===--- CommonOptionsParser.cpp - common options for clang tools ---------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file implements the CommonOptionsParser class used to parse common
+//  command-line options for clang tools, so that they can be run as separate
+//  command-line applications with a consistent common interface for handling
+//  compilation database and input files.
+//
+//  It provides a common subset of command-line options, common algorithm
+//  for locating a compilation database and source files, and help messages
+//  for the basic command-line interface.
+//
+//  It creates a CompilationDatabase and reads common command-line options.
+//
+//  This class uses the Clang Tooling infrastructure, see
+//    http://clang.llvm.org/docs/HowToSetupToolingForLLVM.html
+//  for details on setting it up with LLVM source tree.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Support/CommandLine.h"
+#include "clang/Tooling/CommonOptionsParser.h"
+#include "clang/Tooling/Tooling.h"
+
+using namespace clang::tooling;
+using namespace llvm;
+
+const char *const CommonOptionsParser::HelpMessage =
+    "\n"
+    "-p <build-path> is used to read a compile command database.\n"
+    "\n"
+    "\tFor example, it can be a CMake build directory in which a file named\n"
+    "\tcompile_commands.json exists (use -DCMAKE_EXPORT_COMPILE_COMMANDS=ON\n"
+    "\tCMake option to get this output). When no build path is specified,\n"
+    "\tclang-check will attempt to locate it automatically using all parent\n"
+    "\tpaths of the first input file. See:\n"
+    "\thttp://clang.llvm.org/docs/HowToSetupToolingForLLVM.html for an\n"
+    "\texample of setting up Clang Tooling on a source tree.\n"
+    "\n"
+    "<source0> ... specify the paths of source files. These paths are\n"
+    "\tlooked up in the compile command database. If the path of a file is\n"
+    "\tabsolute, it needs to point into CMake's source tree. If the path is\n"
+    "\trelative, the current working directory needs to be in the CMake\n"
+    "\tsource tree and the file must be in a subdirectory of the current\n"
+    "\tworking directory. \"./\" prefixes in the relative files will be\n"
+    "\tautomatically removed, but the rest of a relative path must be a\n"
+    "\tsuffix of a path in the compile command database.\n"
+    "\n";
+
+CommonOptionsParser::CommonOptionsParser(int &argc, const char **argv) {
+  static cl::opt<std::string> BuildPath(
+      "p", cl::desc("Build path"), cl::Optional);
+
+  static cl::list<std::string> SourcePaths(
+      cl::Positional, cl::desc("<source0> [... <sourceN>]"), cl::OneOrMore);
+
+  Compilations.reset(FixedCompilationDatabase::loadFromCommandLine(argc,
+                                                                   argv));
+  cl::ParseCommandLineOptions(argc, argv);
+  SourcePathList = SourcePaths;
+  if (!Compilations) {
+    std::string ErrorMessage;
+    if (!BuildPath.empty()) {
+      Compilations.reset(CompilationDatabase::autoDetectFromDirectory(
+                              BuildPath, ErrorMessage));
+    } else {
+      Compilations.reset(CompilationDatabase::autoDetectFromSource(
+                              SourcePaths[0], ErrorMessage));
+    }
+    if (!Compilations)
+      llvm::report_fatal_error(ErrorMessage);
+  }
+}
diff --git a/lib/Tooling/CompilationDatabase.cpp b/lib/Tooling/CompilationDatabase.cpp
index 3139cc2..4149cda 100644
--- a/lib/Tooling/CompilationDatabase.cpp
+++ b/lib/Tooling/CompilationDatabase.cpp
@@ -7,132 +7,49 @@
 //
 //===----------------------------------------------------------------------===//
 //
-//  This file contains multiple implementations for CompilationDatabases.
+//  This file contains implementations of the CompilationDatabase base class
+//  and the FixedCompilationDatabase.
 //
 //===----------------------------------------------------------------------===//
 
+#include <sstream>
 #include "clang/Tooling/CompilationDatabase.h"
+#include "clang/Tooling/CompilationDatabasePluginRegistry.h"
 #include "clang/Tooling/Tooling.h"
 #include "llvm/ADT/SmallString.h"
-#include "llvm/Support/YAMLParser.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/system_error.h"
 
-#ifdef USE_CUSTOM_COMPILATION_DATABASE
-#include "CustomCompilationDatabase.h"
-#endif
-
 namespace clang {
 namespace tooling {
 
-namespace {
-
-/// \brief A parser for escaped strings of command line arguments.
-///
-/// Assumes \-escaping for quoted arguments (see the documentation of
-/// unescapeCommandLine(...)).
-class CommandLineArgumentParser {
- public:
-  CommandLineArgumentParser(StringRef CommandLine)
-      : Input(CommandLine), Position(Input.begin()-1) {}
-
-  std::vector<std::string> parse() {
-    bool HasMoreInput = true;
-    while (HasMoreInput && nextNonWhitespace()) {
-      std::string Argument;
-      HasMoreInput = parseStringInto(Argument);
-      CommandLine.push_back(Argument);
-    }
-    return CommandLine;
-  }
-
- private:
-  // All private methods return true if there is more input available.
-
-  bool parseStringInto(std::string &String) {
-    do {
-      if (*Position == '"') {
-        if (!parseQuotedStringInto(String)) return false;
-      } else {
-        if (!parseFreeStringInto(String)) return false;
-      }
-    } while (*Position != ' ');
-    return true;
-  }
-
-  bool parseQuotedStringInto(std::string &String) {
-    if (!next()) return false;
-    while (*Position != '"') {
-      if (!skipEscapeCharacter()) return false;
-      String.push_back(*Position);
-      if (!next()) return false;
-    }
-    return next();
-  }
-
-  bool parseFreeStringInto(std::string &String) {
-    do {
-      if (!skipEscapeCharacter()) return false;
-      String.push_back(*Position);
-      if (!next()) return false;
-    } while (*Position != ' ' && *Position != '"');
-    return true;
-  }
-
-  bool skipEscapeCharacter() {
-    if (*Position == '\\') {
-      return next();
-    }
-    return true;
-  }
-
-  bool nextNonWhitespace() {
-    do {
-      if (!next()) return false;
-    } while (*Position == ' ');
-    return true;
-  }
-
-  bool next() {
-    ++Position;
-    return Position != Input.end();
-  }
-
-  const StringRef Input;
-  StringRef::iterator Position;
-  std::vector<std::string> CommandLine;
-};
-
-std::vector<std::string> unescapeCommandLine(
-    StringRef EscapedCommandLine) {
-  CommandLineArgumentParser parser(EscapedCommandLine);
-  return parser.parse();
-}
-
-} // end namespace
-
 CompilationDatabase::~CompilationDatabase() {}
 
 CompilationDatabase *
 CompilationDatabase::loadFromDirectory(StringRef BuildDirectory,
                                        std::string &ErrorMessage) {
-  llvm::SmallString<1024> JSONDatabasePath(BuildDirectory);
-  llvm::sys::path::append(JSONDatabasePath, "compile_commands.json");
-  llvm::OwningPtr<CompilationDatabase> Database(
-    JSONCompilationDatabase::loadFromFile(JSONDatabasePath, ErrorMessage));
-  if (!Database) {
-    return NULL;
+  std::stringstream ErrorStream;
+  for (CompilationDatabasePluginRegistry::iterator
+       It = CompilationDatabasePluginRegistry::begin(),
+       Ie = CompilationDatabasePluginRegistry::end();
+       It != Ie; ++It) {
+    std::string DatabaseErrorMessage;
+    OwningPtr<CompilationDatabasePlugin> Plugin(It->instantiate());
+    if (CompilationDatabase *DB =
+        Plugin->loadFromDirectory(BuildDirectory, DatabaseErrorMessage))
+      return DB;
+    else
+      ErrorStream << It->getName() << ": " << DatabaseErrorMessage << "\n";
   }
-  return Database.take();
+  ErrorMessage = ErrorStream.str();
+  return NULL;
 }
 
 static CompilationDatabase *
-findCompilationDatabaseFromDirectory(StringRef Directory) {
-#ifdef USE_CUSTOM_COMPILATION_DATABASE
-  if (CompilationDatabase *DB =
-      ::clang::tooling::findCompilationDatabaseForDirectory(Directory))
-    return DB;
-#endif
+findCompilationDatabaseFromDirectory(StringRef Directory,
+                                     std::string &ErrorMessage) {
+  std::stringstream ErrorStream;
+  bool HasErrorMessage = false;
   while (!Directory.empty()) {
     std::string LoadErrorMessage;
 
@@ -140,8 +57,15 @@ findCompilationDatabaseFromDirectory(StringRef Directory) {
            CompilationDatabase::loadFromDirectory(Directory, LoadErrorMessage))
       return DB;
 
+    if (!HasErrorMessage) {
+      ErrorStream << "No compilation database found in " << Directory.str()
+                  << " or any parent directory\n" << LoadErrorMessage;
+      HasErrorMessage = true;
+    }
+
     Directory = llvm::sys::path::parent_path(Directory);
   }
+  ErrorMessage = ErrorStream.str();
   return NULL;
 }
 
@@ -151,11 +75,12 @@ CompilationDatabase::autoDetectFromSource(StringRef SourceFile,
   llvm::SmallString<1024> AbsolutePath(getAbsolutePath(SourceFile));
   StringRef Directory = llvm::sys::path::parent_path(AbsolutePath);
 
-  CompilationDatabase *DB = findCompilationDatabaseFromDirectory(Directory);
+  CompilationDatabase *DB = findCompilationDatabaseFromDirectory(Directory,
+                                                                 ErrorMessage);
 
   if (!DB)
     ErrorMessage = ("Could not auto-detect compilation database for file \"" +
-                   SourceFile + "\"").str();
+                   SourceFile + "\"\n" + ErrorMessage).str();
   return DB;
 }
 
@@ -164,14 +89,17 @@ CompilationDatabase::autoDetectFromDirectory(StringRef SourceDir,
                                              std::string &ErrorMessage) {
   llvm::SmallString<1024> AbsolutePath(getAbsolutePath(SourceDir));
 
-  CompilationDatabase *DB = findCompilationDatabaseFromDirectory(AbsolutePath);
+  CompilationDatabase *DB = findCompilationDatabaseFromDirectory(AbsolutePath,
+                                                                 ErrorMessage);
 
   if (!DB)
     ErrorMessage = ("Could not auto-detect compilation database from directory \"" +
-                   SourceDir + "\"").str();
+                   SourceDir + "\"\n" + ErrorMessage).str();
   return DB;
 }
 
+CompilationDatabasePlugin::~CompilationDatabasePlugin() {}
+
 FixedCompilationDatabase *
 FixedCompilationDatabase::loadFromCommandLine(int &Argc,
                                               const char **Argv,
@@ -204,153 +132,10 @@ FixedCompilationDatabase::getAllFiles() const {
   return std::vector<std::string>();
 }
 
-JSONCompilationDatabase *
-JSONCompilationDatabase::loadFromFile(StringRef FilePath,
-                                      std::string &ErrorMessage) {
-  llvm::OwningPtr<llvm::MemoryBuffer> DatabaseBuffer;
-  llvm::error_code Result =
-    llvm::MemoryBuffer::getFile(FilePath, DatabaseBuffer);
-  if (Result != 0) {
-    ErrorMessage = "Error while opening JSON database: " + Result.message();
-    return NULL;
-  }
-  llvm::OwningPtr<JSONCompilationDatabase> Database(
-    new JSONCompilationDatabase(DatabaseBuffer.take()));
-  if (!Database->parse(ErrorMessage))
-    return NULL;
-  return Database.take();
-}
-
-JSONCompilationDatabase *
-JSONCompilationDatabase::loadFromBuffer(StringRef DatabaseString,
-                                        std::string &ErrorMessage) {
-  llvm::OwningPtr<llvm::MemoryBuffer> DatabaseBuffer(
-      llvm::MemoryBuffer::getMemBuffer(DatabaseString));
-  llvm::OwningPtr<JSONCompilationDatabase> Database(
-    new JSONCompilationDatabase(DatabaseBuffer.take()));
-  if (!Database->parse(ErrorMessage))
-    return NULL;
-  return Database.take();
-}
-
-std::vector<CompileCommand>
-JSONCompilationDatabase::getCompileCommands(StringRef FilePath) const {
-  llvm::SmallString<128> NativeFilePath;
-  llvm::sys::path::native(FilePath, NativeFilePath);
-  llvm::StringMap< std::vector<CompileCommandRef> >::const_iterator
-    CommandsRefI = IndexByFile.find(NativeFilePath);
-  if (CommandsRefI == IndexByFile.end())
-    return std::vector<CompileCommand>();
-  const std::vector<CompileCommandRef> &CommandsRef = CommandsRefI->getValue();
-  std::vector<CompileCommand> Commands;
-  for (int I = 0, E = CommandsRef.size(); I != E; ++I) {
-    llvm::SmallString<8> DirectoryStorage;
-    llvm::SmallString<1024> CommandStorage;
-    Commands.push_back(CompileCommand(
-      // FIXME: Escape correctly:
-      CommandsRef[I].first->getValue(DirectoryStorage),
-      unescapeCommandLine(CommandsRef[I].second->getValue(CommandStorage))));
-  }
-  return Commands;
-}
-
-std::vector<std::string>
-JSONCompilationDatabase::getAllFiles() const {
-  std::vector<std::string> Result;
-
-  llvm::StringMap< std::vector<CompileCommandRef> >::const_iterator
-    CommandsRefI = IndexByFile.begin();
-  const llvm::StringMap< std::vector<CompileCommandRef> >::const_iterator
-    CommandsRefEnd = IndexByFile.end();
-  for (; CommandsRefI != CommandsRefEnd; ++CommandsRefI) {
-    Result.push_back(CommandsRefI->first().str());
-  }
-
-  return Result;
-}
-
-bool JSONCompilationDatabase::parse(std::string &ErrorMessage) {
-  llvm::yaml::document_iterator I = YAMLStream.begin();
-  if (I == YAMLStream.end()) {
-    ErrorMessage = "Error while parsing YAML.";
-    return false;
-  }
-  llvm::yaml::Node *Root = I->getRoot();
-  if (Root == NULL) {
-    ErrorMessage = "Error while parsing YAML.";
-    return false;
-  }
-  llvm::yaml::SequenceNode *Array =
-    llvm::dyn_cast<llvm::yaml::SequenceNode>(Root);
-  if (Array == NULL) {
-    ErrorMessage = "Expected array.";
-    return false;
-  }
-  for (llvm::yaml::SequenceNode::iterator AI = Array->begin(),
-                                          AE = Array->end();
-       AI != AE; ++AI) {
-    llvm::yaml::MappingNode *Object =
-      llvm::dyn_cast<llvm::yaml::MappingNode>(&*AI);
-    if (Object == NULL) {
-      ErrorMessage = "Expected object.";
-      return false;
-    }
-    llvm::yaml::ScalarNode *Directory = NULL;
-    llvm::yaml::ScalarNode *Command = NULL;
-    llvm::yaml::ScalarNode *File = NULL;
-    for (llvm::yaml::MappingNode::iterator KVI = Object->begin(),
-                                           KVE = Object->end();
-         KVI != KVE; ++KVI) {
-      llvm::yaml::Node *Value = (*KVI).getValue();
-      if (Value == NULL) {
-        ErrorMessage = "Expected value.";
-        return false;
-      }
-      llvm::yaml::ScalarNode *ValueString =
-        llvm::dyn_cast<llvm::yaml::ScalarNode>(Value);
-      if (ValueString == NULL) {
-        ErrorMessage = "Expected string as value.";
-        return false;
-      }
-      llvm::yaml::ScalarNode *KeyString =
-        llvm::dyn_cast<llvm::yaml::ScalarNode>((*KVI).getKey());
-      if (KeyString == NULL) {
-        ErrorMessage = "Expected strings as key.";
-        return false;
-      }
-      llvm::SmallString<8> KeyStorage;
-      if (KeyString->getValue(KeyStorage) == "directory") {
-        Directory = ValueString;
-      } else if (KeyString->getValue(KeyStorage) == "command") {
-        Command = ValueString;
-      } else if (KeyString->getValue(KeyStorage) == "file") {
-        File = ValueString;
-      } else {
-        ErrorMessage = ("Unknown key: \"" +
-                        KeyString->getRawValue() + "\"").str();
-        return false;
-      }
-    }
-    if (!File) {
-      ErrorMessage = "Missing key: \"file\".";
-      return false;
-    }
-    if (!Command) {
-      ErrorMessage = "Missing key: \"command\".";
-      return false;
-    }
-    if (!Directory) {
-      ErrorMessage = "Missing key: \"directory\".";
-      return false;
-    }
-    llvm::SmallString<8> FileStorage;
-    llvm::SmallString<128> NativeFilePath;
-    llvm::sys::path::native(File->getValue(FileStorage), NativeFilePath);
-    IndexByFile[NativeFilePath].push_back(
-      CompileCommandRef(Directory, Command));
-  }
-  return true;
-}
+// This anchor is used to force the linker to link in the generated object file
+// and thus register the JSONCompilationDatabasePlugin.
+extern volatile int JSONAnchorSource;
+static int JSONAnchorDest = JSONAnchorSource;
 
 } // end namespace tooling
 } // end namespace clang
diff --git a/lib/Tooling/CustomCompilationDatabase.h b/lib/Tooling/CustomCompilationDatabase.h
deleted file mode 100644
index b375f8d..0000000
--- a/lib/Tooling/CustomCompilationDatabase.h
+++ /dev/null
@@ -1,42 +0,0 @@
-//===--- CustomCompilationDatabase.h --------------------------------------===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-//  This file contains a hook to supply a custom \c CompilationDatabase
-//  implementation.
-//
-//  The mechanism can be used by IDEs or non-public code bases to integrate with
-//  their build system. Currently we support statically linking in an
-//  implementation of \c findCompilationDatabaseForDirectory and enabling it
-//  with -DUSE_CUSTOM_COMPILATION_DATABASE when compiling the Tooling library.
-//
-//  FIXME: The strategy forward is to provide a plugin system that can load
-//  custom compilation databases and make enabling that a build option.
-//
-//===----------------------------------------------------------------------===//
-#ifndef LLVM_CLANG_TOOLING_CUSTOM_COMPILATION_DATABASE_H
-#define LLVM_CLANG_TOOLING_CUSTOM_COMPILATION_DATABASE_H
-
-#include "llvm/ADT/StringRef.h"
-
-namespace clang {
-namespace tooling {
-class CompilationDatabase;
-
-/// \brief Returns a CompilationDatabase for the given \c Directory.
-///
-/// \c Directory can be any directory within a project. This methods will
-/// then try to find compilation database files in \c Directory or any of its
-/// parents. If a compilation database cannot be found or loaded, returns NULL.
-clang::tooling::CompilationDatabase *findCompilationDatabaseForDirectory(
-  llvm::StringRef Directory);
-
-} // namespace tooling
-} // namespace clang
-
-#endif // LLVM_CLANG_TOOLING_CUSTOM_COMPILATION_DATABASE_H
diff --git a/lib/Tooling/FileMatchTrie.cpp b/lib/Tooling/FileMatchTrie.cpp
new file mode 100644
index 0000000..8f25a8c
--- /dev/null
+++ b/lib/Tooling/FileMatchTrie.cpp
@@ -0,0 +1,188 @@
+//===--- FileMatchTrie.cpp - ----------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file contains the implementation of a FileMatchTrie.
+//
+//===----------------------------------------------------------------------===//
+
+#include <sstream>
+#include "clang/Tooling/FileMatchTrie.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/PathV2.h"
+#include "llvm/Support/raw_ostream.h"
+
+namespace clang {
+namespace tooling {
+
+/// \brief Default \c PathComparator using \c llvm::sys::fs::equivalent().
+struct DefaultPathComparator : public PathComparator {
+  virtual ~DefaultPathComparator() {}
+  virtual bool equivalent(StringRef FileA, StringRef FileB) const {
+    return FileA == FileB || llvm::sys::fs::equivalent(FileA, FileB);
+  }
+};
+
+/// \brief A node of the \c FileMatchTrie.
+///
+/// Each node has storage for up to one path and a map mapping a path segment to
+/// child nodes. The trie starts with an empty root node.
+class FileMatchTrieNode {
+public:
+  /// \brief Inserts 'NewPath' into this trie. \c ConsumedLength denotes
+  /// the number of \c NewPath's trailing characters already consumed during
+  /// recursion.
+  ///
+  /// An insert of a path
+  /// 'p'starts at the root node and does the following:
+  /// - If the node is empty, insert 'p' into its storage and abort.
+  /// - If the node has a path 'p2' but no children, take the last path segment
+  ///   's' of 'p2', put a new child into the map at 's' an insert the rest of
+  ///   'p2' there.
+  /// - Insert a new child for the last segment of 'p' and insert the rest of
+  ///   'p' there.
+  ///
+  /// An insert operation is linear in the number of a path's segments.
+  void insert(StringRef NewPath, unsigned ConsumedLength = 0) {
+    // We cannot put relative paths into the FileMatchTrie as then a path can be
+    // a postfix of another path, violating a core assumption of the trie.
+    if (llvm::sys::path::is_relative(NewPath))
+      return;
+    if (Path.empty()) {
+      // This is an empty leaf. Store NewPath and return.
+      Path = NewPath;
+      return;
+    }
+    if (Children.empty()) {
+      // This is a leaf, ignore duplicate entry if 'Path' equals 'NewPath'.
+      if (NewPath == Path)
+          return;
+      // Make this a node and create a child-leaf with 'Path'.
+      StringRef Element(llvm::sys::path::filename(
+          StringRef(Path).drop_back(ConsumedLength)));
+      Children[Element].Path = Path;
+    }
+    StringRef Element(llvm::sys::path::filename(
+          StringRef(NewPath).drop_back(ConsumedLength)));
+    Children[Element].insert(NewPath, ConsumedLength + Element.size() + 1);
+  }
+
+  /// \brief Tries to find the node under this \c FileMatchTrieNode that best
+  /// matches 'FileName'.
+  ///
+  /// If multiple paths fit 'FileName' equally well, \c IsAmbiguous is set to
+  /// \c true and an empty string is returned. If no path fits 'FileName', an
+  /// empty string is returned. \c ConsumedLength denotes the number of
+  /// \c Filename's trailing characters already consumed during recursion.
+  ///
+  /// To find the best matching node for a given path 'p', the
+  /// \c findEquivalent() function is called recursively for each path segment
+  /// (back to fron) of 'p' until a node 'n' is reached that does not ..
+  /// - .. have children. In this case it is checked
+  ///   whether the stored path is equivalent to 'p'. If yes, the best match is
+  ///   found. Otherwise continue with the parent node as if this node did not
+  ///   exist.
+  /// - .. a child matching the next path segment. In this case, all children of
+  ///   'n' are an equally good match for 'p'. All children are of 'n' are found
+  ///   recursively and their equivalence to 'p' is determined. If none are
+  ///   equivalent, continue with the parent node as if 'n' didn't exist. If one
+  ///   is equivalent, the best match is found. Otherwise, report and ambigiuity
+  ///   error.
+  StringRef findEquivalent(const PathComparator& Comparator,
+                           StringRef FileName,
+                           bool &IsAmbiguous,
+                           unsigned ConsumedLength = 0) const {
+    if (Children.empty()) {
+      if (Comparator.equivalent(StringRef(Path), FileName))
+        return StringRef(Path);
+      return StringRef();
+    }
+    StringRef Element(llvm::sys::path::filename(FileName.drop_back(
+        ConsumedLength)));
+    llvm::StringMap<FileMatchTrieNode>::const_iterator MatchingChild =
+        Children.find(Element);
+    if (MatchingChild != Children.end()) {
+      StringRef Result = MatchingChild->getValue().findEquivalent(
+          Comparator, FileName, IsAmbiguous,
+          ConsumedLength + Element.size() + 1);
+      if (!Result.empty() || IsAmbiguous)
+        return Result;
+    }
+    std::vector<StringRef> AllChildren;
+    getAll(AllChildren, MatchingChild);
+    StringRef Result;
+    for (unsigned i = 0; i < AllChildren.size(); i++) {
+      if (Comparator.equivalent(AllChildren[i], FileName)) {
+        if (Result.empty()) {
+          Result = AllChildren[i];
+        } else {
+          IsAmbiguous = true;
+          return StringRef();
+        }
+      }
+    }
+    return Result;
+  }
+
+private:
+  /// \brief Gets all paths under this FileMatchTrieNode.
+  void getAll(std::vector<StringRef> &Results,
+              llvm::StringMap<FileMatchTrieNode>::const_iterator Except) const {
+    if (Path.empty())
+      return;
+    if (Children.empty()) {
+      Results.push_back(StringRef(Path));
+      return;
+    }
+    for (llvm::StringMap<FileMatchTrieNode>::const_iterator
+         It = Children.begin(), E = Children.end();
+         It != E; ++It) {
+      if (It == Except)
+        continue;
+      It->getValue().getAll(Results, Children.end());
+    }
+  }
+
+  // The stored absolute path in this node. Only valid for leaf nodes, i.e.
+  // nodes where Children.empty().
+  std::string Path;
+
+  // The children of this node stored in a map based on the next path segment.
+  llvm::StringMap<FileMatchTrieNode> Children;
+};
+
+FileMatchTrie::FileMatchTrie()
+  : Root(new FileMatchTrieNode), Comparator(new DefaultPathComparator()) {}
+
+FileMatchTrie::FileMatchTrie(PathComparator *Comparator)
+  : Root(new FileMatchTrieNode), Comparator(Comparator) {}
+
+FileMatchTrie::~FileMatchTrie() {
+  delete Root;
+}
+
+void FileMatchTrie::insert(StringRef NewPath) {
+  Root->insert(NewPath);
+}
+
+StringRef FileMatchTrie::findEquivalent(StringRef FileName,
+                                        llvm::raw_ostream &Error) const {
+  if (llvm::sys::path::is_relative(FileName)) {
+    Error << "Cannot resolve relative paths";
+    return StringRef();
+  }
+  bool IsAmbiguous = false;
+  StringRef Result = Root->findEquivalent(*Comparator, FileName, IsAmbiguous);
+  if (IsAmbiguous)
+    Error << "Path is ambiguous";
+  return Result;
+}
+
+} // end namespace tooling
+} // end namespace clang
diff --git a/lib/Tooling/JSONCompilationDatabase.cpp b/lib/Tooling/JSONCompilationDatabase.cpp
new file mode 100644
index 0000000..cf35a25
--- /dev/null
+++ b/lib/Tooling/JSONCompilationDatabase.cpp
@@ -0,0 +1,303 @@
+//===--- JSONCompilationDatabase.cpp - ------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file contains the implementation of the JSONCompilationDatabase.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Tooling/JSONCompilationDatabase.h"
+
+#include "clang/Tooling/CompilationDatabase.h"
+#include "clang/Tooling/CompilationDatabasePluginRegistry.h"
+#include "clang/Tooling/Tooling.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/system_error.h"
+
+namespace clang {
+namespace tooling {
+
+namespace {
+
+/// \brief A parser for escaped strings of command line arguments.
+///
+/// Assumes \-escaping for quoted arguments (see the documentation of
+/// unescapeCommandLine(...)).
+class CommandLineArgumentParser {
+ public:
+  CommandLineArgumentParser(StringRef CommandLine)
+      : Input(CommandLine), Position(Input.begin()-1) {}
+
+  std::vector<std::string> parse() {
+    bool HasMoreInput = true;
+    while (HasMoreInput && nextNonWhitespace()) {
+      std::string Argument;
+      HasMoreInput = parseStringInto(Argument);
+      CommandLine.push_back(Argument);
+    }
+    return CommandLine;
+  }
+
+ private:
+  // All private methods return true if there is more input available.
+
+  bool parseStringInto(std::string &String) {
+    do {
+      if (*Position == '"') {
+        if (!parseQuotedStringInto(String)) return false;
+      } else {
+        if (!parseFreeStringInto(String)) return false;
+      }
+    } while (*Position != ' ');
+    return true;
+  }
+
+  bool parseQuotedStringInto(std::string &String) {
+    if (!next()) return false;
+    while (*Position != '"') {
+      if (!skipEscapeCharacter()) return false;
+      String.push_back(*Position);
+      if (!next()) return false;
+    }
+    return next();
+  }
+
+  bool parseFreeStringInto(std::string &String) {
+    do {
+      if (!skipEscapeCharacter()) return false;
+      String.push_back(*Position);
+      if (!next()) return false;
+    } while (*Position != ' ' && *Position != '"');
+    return true;
+  }
+
+  bool skipEscapeCharacter() {
+    if (*Position == '\\') {
+      return next();
+    }
+    return true;
+  }
+
+  bool nextNonWhitespace() {
+    do {
+      if (!next()) return false;
+    } while (*Position == ' ');
+    return true;
+  }
+
+  bool next() {
+    ++Position;
+    return Position != Input.end();
+  }
+
+  const StringRef Input;
+  StringRef::iterator Position;
+  std::vector<std::string> CommandLine;
+};
+
+std::vector<std::string> unescapeCommandLine(
+    StringRef EscapedCommandLine) {
+  CommandLineArgumentParser parser(EscapedCommandLine);
+  return parser.parse();
+}
+
+} // end namespace
+
+class JSONCompilationDatabasePlugin : public CompilationDatabasePlugin {
+  virtual CompilationDatabase *loadFromDirectory(
+      StringRef Directory, std::string &ErrorMessage) {
+    llvm::SmallString<1024> JSONDatabasePath(Directory);
+    llvm::sys::path::append(JSONDatabasePath, "compile_commands.json");
+    llvm::OwningPtr<CompilationDatabase> Database(
+        JSONCompilationDatabase::loadFromFile(JSONDatabasePath, ErrorMessage));
+    if (!Database)
+      return NULL;
+    return Database.take();
+  }
+};
+
+// Register the JSONCompilationDatabasePlugin with the
+// CompilationDatabasePluginRegistry using this statically initialized variable.
+static CompilationDatabasePluginRegistry::Add<JSONCompilationDatabasePlugin>
+X("json-compilation-database", "Reads JSON formatted compilation databases");
+
+// This anchor is used to force the linker to link in the generated object file
+// and thus register the JSONCompilationDatabasePlugin.
+volatile int JSONAnchorSource = 0;
+
+JSONCompilationDatabase *
+JSONCompilationDatabase::loadFromFile(StringRef FilePath,
+                                      std::string &ErrorMessage) {
+  llvm::OwningPtr<llvm::MemoryBuffer> DatabaseBuffer;
+  llvm::error_code Result =
+    llvm::MemoryBuffer::getFile(FilePath, DatabaseBuffer);
+  if (Result != 0) {
+    ErrorMessage = "Error while opening JSON database: " + Result.message();
+    return NULL;
+  }
+  llvm::OwningPtr<JSONCompilationDatabase> Database(
+    new JSONCompilationDatabase(DatabaseBuffer.take()));
+  if (!Database->parse(ErrorMessage))
+    return NULL;
+  return Database.take();
+}
+
+JSONCompilationDatabase *
+JSONCompilationDatabase::loadFromBuffer(StringRef DatabaseString,
+                                        std::string &ErrorMessage) {
+  llvm::OwningPtr<llvm::MemoryBuffer> DatabaseBuffer(
+      llvm::MemoryBuffer::getMemBuffer(DatabaseString));
+  llvm::OwningPtr<JSONCompilationDatabase> Database(
+    new JSONCompilationDatabase(DatabaseBuffer.take()));
+  if (!Database->parse(ErrorMessage))
+    return NULL;
+  return Database.take();
+}
+
+std::vector<CompileCommand>
+JSONCompilationDatabase::getCompileCommands(StringRef FilePath) const {
+  llvm::SmallString<128> NativeFilePath;
+  llvm::sys::path::native(FilePath, NativeFilePath);
+  std::vector<StringRef> PossibleMatches;
+  std::string Error;
+  llvm::raw_string_ostream ES(Error);
+  StringRef Match = MatchTrie.findEquivalent(NativeFilePath.str(), ES);
+  if (Match.empty()) {
+    if (Error.empty())
+      Error = "No match found.";
+    llvm::outs() << Error << "\n";
+    return std::vector<CompileCommand>();
+  }
+  llvm::StringMap< std::vector<CompileCommandRef> >::const_iterator
+    CommandsRefI = IndexByFile.find(Match);
+  if (CommandsRefI == IndexByFile.end())
+    return std::vector<CompileCommand>();
+  const std::vector<CompileCommandRef> &CommandsRef = CommandsRefI->getValue();
+  std::vector<CompileCommand> Commands;
+  for (int I = 0, E = CommandsRef.size(); I != E; ++I) {
+    llvm::SmallString<8> DirectoryStorage;
+    llvm::SmallString<1024> CommandStorage;
+    Commands.push_back(CompileCommand(
+      // FIXME: Escape correctly:
+      CommandsRef[I].first->getValue(DirectoryStorage),
+      unescapeCommandLine(CommandsRef[I].second->getValue(CommandStorage))));
+  }
+  return Commands;
+}
+
+std::vector<std::string>
+JSONCompilationDatabase::getAllFiles() const {
+  std::vector<std::string> Result;
+
+  llvm::StringMap< std::vector<CompileCommandRef> >::const_iterator
+    CommandsRefI = IndexByFile.begin();
+  const llvm::StringMap< std::vector<CompileCommandRef> >::const_iterator
+    CommandsRefEnd = IndexByFile.end();
+  for (; CommandsRefI != CommandsRefEnd; ++CommandsRefI) {
+    Result.push_back(CommandsRefI->first().str());
+  }
+
+  return Result;
+}
+
+bool JSONCompilationDatabase::parse(std::string &ErrorMessage) {
+  llvm::yaml::document_iterator I = YAMLStream.begin();
+  if (I == YAMLStream.end()) {
+    ErrorMessage = "Error while parsing YAML.";
+    return false;
+  }
+  llvm::yaml::Node *Root = I->getRoot();
+  if (Root == NULL) {
+    ErrorMessage = "Error while parsing YAML.";
+    return false;
+  }
+  llvm::yaml::SequenceNode *Array =
+    llvm::dyn_cast<llvm::yaml::SequenceNode>(Root);
+  if (Array == NULL) {
+    ErrorMessage = "Expected array.";
+    return false;
+  }
+  for (llvm::yaml::SequenceNode::iterator AI = Array->begin(),
+                                          AE = Array->end();
+       AI != AE; ++AI) {
+    llvm::yaml::MappingNode *Object =
+      llvm::dyn_cast<llvm::yaml::MappingNode>(&*AI);
+    if (Object == NULL) {
+      ErrorMessage = "Expected object.";
+      return false;
+    }
+    llvm::yaml::ScalarNode *Directory = NULL;
+    llvm::yaml::ScalarNode *Command = NULL;
+    llvm::yaml::ScalarNode *File = NULL;
+    for (llvm::yaml::MappingNode::iterator KVI = Object->begin(),
+                                           KVE = Object->end();
+         KVI != KVE; ++KVI) {
+      llvm::yaml::Node *Value = (*KVI).getValue();
+      if (Value == NULL) {
+        ErrorMessage = "Expected value.";
+        return false;
+      }
+      llvm::yaml::ScalarNode *ValueString =
+        llvm::dyn_cast<llvm::yaml::ScalarNode>(Value);
+      if (ValueString == NULL) {
+        ErrorMessage = "Expected string as value.";
+        return false;
+      }
+      llvm::yaml::ScalarNode *KeyString =
+        llvm::dyn_cast<llvm::yaml::ScalarNode>((*KVI).getKey());
+      if (KeyString == NULL) {
+        ErrorMessage = "Expected strings as key.";
+        return false;
+      }
+      llvm::SmallString<8> KeyStorage;
+      if (KeyString->getValue(KeyStorage) == "directory") {
+        Directory = ValueString;
+      } else if (KeyString->getValue(KeyStorage) == "command") {
+        Command = ValueString;
+      } else if (KeyString->getValue(KeyStorage) == "file") {
+        File = ValueString;
+      } else {
+        ErrorMessage = ("Unknown key: \"" +
+                        KeyString->getRawValue() + "\"").str();
+        return false;
+      }
+    }
+    if (!File) {
+      ErrorMessage = "Missing key: \"file\".";
+      return false;
+    }
+    if (!Command) {
+      ErrorMessage = "Missing key: \"command\".";
+      return false;
+    }
+    if (!Directory) {
+      ErrorMessage = "Missing key: \"directory\".";
+      return false;
+    }
+    llvm::SmallString<8> FileStorage;
+    StringRef FileName = File->getValue(FileStorage);
+    llvm::SmallString<128> NativeFilePath;
+    if (llvm::sys::path::is_relative(FileName)) {
+      llvm::SmallString<8> DirectoryStorage;
+      llvm::SmallString<128> AbsolutePath(
+          Directory->getValue(DirectoryStorage));
+      llvm::sys::path::append(AbsolutePath, FileName);
+      llvm::sys::path::native(AbsolutePath.str(), NativeFilePath);
+    } else {
+      llvm::sys::path::native(FileName, NativeFilePath);
+    }
+    IndexByFile[NativeFilePath].push_back(
+        CompileCommandRef(Directory, Command));
+    MatchTrie.insert(NativeFilePath.str());
+  }
+  return true;
+}
+
+} // end namespace tooling
+} // end namespace clang
diff --git a/lib/Tooling/Refactoring.cpp b/lib/Tooling/Refactoring.cpp
index 6284353..c5002ef 100644
--- a/lib/Tooling/Refactoring.cpp
+++ b/lib/Tooling/Refactoring.cpp
@@ -11,12 +11,12 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "clang/Basic/DiagnosticOptions.h"
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/SourceManager.h"
-#include "clang/Frontend/DiagnosticOptions.h"
 #include "clang/Frontend/TextDiagnosticPrinter.h"
 #include "clang/Lex/Lexer.h"
-#include "clang/Rewrite/Rewriter.h"
+#include "clang/Rewrite/Core/Rewriter.h"
 #include "clang/Tooling/Refactoring.h"
 #include "llvm/Support/raw_os_ostream.h"
 
@@ -164,12 +164,11 @@ Replacements &RefactoringTool::getReplacements() { return Replace; }
 int RefactoringTool::run(FrontendActionFactory *ActionFactory) {
   int Result = Tool.run(ActionFactory);
   LangOptions DefaultLangOptions;
-  DiagnosticOptions DefaultDiagnosticOptions;
-  TextDiagnosticPrinter DiagnosticPrinter(llvm::errs(),
-                                          DefaultDiagnosticOptions);
+  IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
+  TextDiagnosticPrinter DiagnosticPrinter(llvm::errs(), &*DiagOpts);
   DiagnosticsEngine Diagnostics(
       llvm::IntrusiveRefCntPtr<DiagnosticIDs>(new DiagnosticIDs()),
-      &DiagnosticPrinter, false);
+      &*DiagOpts, &DiagnosticPrinter, false);
   SourceManager Sources(Diagnostics, Tool.getFiles());
   Rewriter Rewrite(Sources, DefaultLangOptions);
   if (!applyAllReplacements(Replace, Rewrite)) {
diff --git a/lib/Tooling/Tooling.cpp b/lib/Tooling/Tooling.cpp
index e93e0c9..af20254 100644
--- a/lib/Tooling/Tooling.cpp
+++ b/lib/Tooling/Tooling.cpp
@@ -97,17 +97,22 @@ static clang::CompilerInvocation *newInvocation(
 
 bool runToolOnCode(clang::FrontendAction *ToolAction, const Twine &Code,
                    const Twine &FileName) {
+  return runToolOnCodeWithArgs(
+      ToolAction, Code, std::vector<std::string>(), FileName);
+}
+
+bool runToolOnCodeWithArgs(clang::FrontendAction *ToolAction, const Twine &Code,
+                           const std::vector<std::string> &Args,
+                           const Twine &FileName) {
   SmallString<16> FileNameStorage;
   StringRef FileNameRef = FileName.toNullTerminatedStringRef(FileNameStorage);
-  const char *const CommandLine[] = {
-      "clang-tool", "-fsyntax-only", FileNameRef.data()
-  };
+  std::vector<std::string> Commands;
+  Commands.push_back("clang-tool");
+  Commands.push_back("-fsyntax-only");
+  Commands.insert(Commands.end(), Args.begin(), Args.end());
+  Commands.push_back(FileNameRef.data());
   FileManager Files((FileSystemOptions()));
-  ToolInvocation Invocation(
-      std::vector<std::string>(
-          CommandLine,
-          CommandLine + llvm::array_lengthof(CommandLine)),
-      ToolAction, &Files);
+  ToolInvocation Invocation(Commands, ToolAction, &Files);
 
   SmallString<1024> CodeStorage;
   Invocation.mapVirtualFile(FileNameRef,
@@ -154,11 +159,12 @@ bool ToolInvocation::run() {
   for (int I = 0, E = CommandLine.size(); I != E; ++I)
     Argv.push_back(CommandLine[I].c_str());
   const char *const BinaryName = Argv[0];
-  DiagnosticOptions DefaultDiagnosticOptions;
+  IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts = new DiagnosticOptions();
   TextDiagnosticPrinter DiagnosticPrinter(
-      llvm::errs(), DefaultDiagnosticOptions);
-  DiagnosticsEngine Diagnostics(llvm::IntrusiveRefCntPtr<clang::DiagnosticIDs>(
-      new DiagnosticIDs()), &DiagnosticPrinter, false);
+      llvm::errs(), &*DiagOpts);
+  DiagnosticsEngine Diagnostics(
+    llvm::IntrusiveRefCntPtr<clang::DiagnosticIDs>(new DiagnosticIDs()),
+    &*DiagOpts, &DiagnosticPrinter, false);
 
   const llvm::OwningPtr<clang::driver::Driver> Driver(
       newDriver(&Diagnostics, BinaryName));
-- 
cgit v1.1


From a8471d78beb78d784d4237ece85c2df6dfaaf138 Mon Sep 17 00:00:00 2001
From: kevlo <kevlo@FreeBSD.org>
Date: Tue, 4 Dec 2012 09:53:09 +0000
Subject: Document that bind(2) can fail with EAFNOSUPPORT.

Reviewed by:	glebius
---
 lib/libc/sys/bind.2 | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'lib')

diff --git a/lib/libc/sys/bind.2 b/lib/libc/sys/bind.2
index fecc5fa..896b005 100644
--- a/lib/libc/sys/bind.2
+++ b/lib/libc/sys/bind.2
@@ -94,6 +94,8 @@ is not a socket.
 The specified address is not available from the local machine.
 .It Bq Er EADDRINUSE
 The specified address is already in use.
+.It Bq Er EAFNOSUPPORT
+Addresses in the specified address family cannot be used with this socket.
 .It Bq Er EACCES
 The requested address is protected, and the current user
 has inadequate permission to access it.
-- 
cgit v1.1


From 74acd0aa7d7396577a2cb30cb186ea86e8b8e03d Mon Sep 17 00:00:00 2001
From: jilles <jilles@FreeBSD.org>
Date: Tue, 4 Dec 2012 16:54:43 +0000
Subject: libc: Use the new 'e' fopen() mode option to simplify fstab.c.

No functional change is intended.
---
 lib/libc/gen/fstab.c | 12 +-----------
 1 file changed, 1 insertion(+), 11 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/gen/fstab.c b/lib/libc/gen/fstab.c
index b68db2e..6a77abd 100644
--- a/lib/libc/gen/fstab.c
+++ b/lib/libc/gen/fstab.c
@@ -39,7 +39,6 @@ __FBSDID("$FreeBSD$");
 #include <sys/stat.h>
 
 #include <errno.h>
-#include <fcntl.h>
 #include <fstab.h>
 #include <paths.h>
 #include <stdio.h>
@@ -255,8 +254,6 @@ getfsfile(const char *name)
 int
 setfsent(void)
 {
-	int fd;
-
 	if (_fs_fp) {
 		rewind(_fs_fp);
 		LineNo = 0;
@@ -268,18 +265,11 @@ setfsent(void)
 		else
 			setfstab(getenv("PATH_FSTAB"));
 	}
-	fd = _open(path_fstab, O_RDONLY | O_CLOEXEC);
-	if (fd == -1) {
-		error(errno);
-		return (0);
-	}
-	_fs_fp = fdopen(fd, "r");
-	if (_fs_fp  != NULL) {
+	if ((_fs_fp = fopen(path_fstab, "re")) != NULL) {
 		LineNo = 0;
 		return (1);
 	}
 	error(errno);
-	_close(fd);
 	return (0);
 }
 
-- 
cgit v1.1


From fd5e2d2ee32092bd75f13be0af0b4080836f3238 Mon Sep 17 00:00:00 2001
From: erwin <erwin@FreeBSD.org>
Date: Wed, 5 Dec 2012 12:53:50 +0000
Subject: Vendor import of Bind 9.8.4

Approved by:	delphij (mentor)
Sponsored by:	DK Hostmaster A/S
---
 lib/Makefile.in                                  |   2 +-
 lib/bind9/Makefile.in                            |   2 +-
 lib/bind9/api                                    |   2 +-
 lib/bind9/check.c                                |  69 ++-
 lib/bind9/include/Makefile.in                    |   2 +-
 lib/bind9/include/bind9/Makefile.in              |   2 +-
 lib/dns/Makefile.in                              |   5 +-
 lib/dns/adb.c                                    |  44 +-
 lib/dns/api                                      |   4 +-
 lib/dns/db.c                                     |   9 +-
 lib/dns/dnssec.c                                 | 115 ++++-
 lib/dns/ds.c                                     |  35 +-
 lib/dns/dst_api.c                                |  16 +-
 lib/dns/dst_internal.h                           |   3 +
 lib/dns/dst_openssl.h                            |   3 +
 lib/dns/dst_parse.c                              |  61 ++-
 lib/dns/dst_parse.h                              |   5 +-
 lib/dns/dst_result.c                             |   4 +-
 lib/dns/include/Makefile.in                      |   2 +-
 lib/dns/include/dns/db.h                         |   8 +-
 lib/dns/include/dns/dnssec.h                     |  13 +-
 lib/dns/include/dns/ds.h                         |   7 +-
 lib/dns/include/dns/iptable.h                    |   4 +-
 lib/dns/include/dns/keyvalues.h                  |  10 +-
 lib/dns/include/dns/log.h                        |   1 +
 lib/dns/include/dns/rdataset.h                   |  20 +
 lib/dns/include/dns/rpz.h                        |  13 +-
 lib/dns/include/dns/stats.h                      |  10 +
 lib/dns/include/dns/view.h                       |   2 +
 lib/dns/include/dns/zone.h                       |   2 +-
 lib/dns/include/dst/Makefile.in                  |   2 +-
 lib/dns/include/dst/dst.h                        |   2 +
 lib/dns/include/dst/result.h                     |   6 +-
 lib/dns/log.c                                    |   1 +
 lib/dns/master.c                                 |  47 +-
 lib/dns/masterdump.c                             |  12 +-
 lib/dns/openssl_link.c                           |  40 +-
 lib/dns/openssldh_link.c                         |  18 +-
 lib/dns/openssldsa_link.c                        |  34 +-
 lib/dns/opensslecdsa_link.c                      | 596 +++++++++++++++++++++++
 lib/dns/opensslgost_link.c                       |  75 ++-
 lib/dns/opensslrsa_link.c                        |  52 +-
 lib/dns/rbtdb.c                                  |  39 +-
 lib/dns/rcode.c                                  |   2 +
 lib/dns/rdata.c                                  |  12 +-
 lib/dns/rdata/generic/dlv_32769.c                |  12 +-
 lib/dns/rdata/generic/ds_43.c                    |  12 +-
 lib/dns/rdataset.c                               |  28 ++
 lib/dns/resolver.c                               |  50 +-
 lib/dns/rpz.c                                    | 133 +++--
 lib/dns/spnego_asn1.pl                           |   2 +-
 lib/dns/validator.c                              |  10 +-
 lib/dns/view.c                                   |   2 +
 lib/dns/zone.c                                   | 101 ++--
 lib/export/Makefile.in                           |   2 +-
 lib/export/dns/Makefile.in                       |   4 +-
 lib/export/dns/include/Makefile.in               |   2 +-
 lib/export/dns/include/dns/Makefile.in           |   2 +-
 lib/export/dns/include/dst/Makefile.in           |   2 +-
 lib/export/irs/include/irs/Makefile.in           |   2 +-
 lib/export/isc/Makefile.in                       |   2 +-
 lib/export/isc/include/isc/Makefile.in           |   2 +-
 lib/export/isc/nls/Makefile.in                   |   2 +-
 lib/export/isc/nothreads/Makefile.in             |   2 +-
 lib/export/isc/nothreads/include/isc/Makefile.in |   2 +-
 lib/export/isc/pthreads/Makefile.in              |   2 +-
 lib/export/isc/pthreads/include/isc/Makefile.in  |   2 +-
 lib/export/isc/unix/Makefile.in                  |   2 +-
 lib/export/isc/unix/include/isc/Makefile.in      |   2 +-
 lib/export/isccfg/include/isccfg/Makefile.in     |   2 +-
 lib/export/samples/Makefile-postinstall.in       |   2 +-
 lib/export/samples/Makefile.in                   |   2 +-
 lib/irs/Makefile.in                              |   2 +-
 lib/irs/include/Makefile.in                      |   2 +-
 lib/irs/include/irs/Makefile.in                  |   2 +-
 lib/isc/alpha/Makefile.in                        |   2 +-
 lib/isc/alpha/include/Makefile.in                |   2 +-
 lib/isc/alpha/include/isc/Makefile.in            |   2 +-
 lib/isc/api                                      |   6 +-
 lib/isc/ia64/Makefile.in                         |   2 +-
 lib/isc/ia64/include/Makefile.in                 |   2 +-
 lib/isc/ia64/include/isc/Makefile.in             |   2 +-
 lib/isc/ia64/include/isc/atomic.h                |   6 +-
 lib/isc/include/Makefile.in                      |   2 +-
 lib/isc/include/isc/file.h                       |  13 +-
 lib/isc/include/isc/namespace.h                  |   2 +
 lib/isc/include/isc/task.h                       |  27 +
 lib/isc/mem.c                                    |  10 +-
 lib/isc/mips/Makefile.in                         |   2 +-
 lib/isc/mips/include/Makefile.in                 |   2 +-
 lib/isc/mips/include/isc/Makefile.in             |   2 +-
 lib/isc/nls/Makefile.in                          |   2 +-
 lib/isc/noatomic/Makefile.in                     |   2 +-
 lib/isc/noatomic/include/Makefile.in             |   2 +-
 lib/isc/noatomic/include/isc/Makefile.in         |   2 +-
 lib/isc/nothreads/Makefile.in                    |   2 +-
 lib/isc/nothreads/include/Makefile.in            |   2 +-
 lib/isc/nothreads/include/isc/Makefile.in        |   2 +-
 lib/isc/powerpc/Makefile.in                      |   2 +-
 lib/isc/powerpc/include/Makefile.in              |   2 +-
 lib/isc/powerpc/include/isc/Makefile.in          |   2 +-
 lib/isc/pthreads/Makefile.in                     |   2 +-
 lib/isc/pthreads/condition.c                     |  11 +-
 lib/isc/pthreads/include/Makefile.in             |   2 +-
 lib/isc/pthreads/include/isc/Makefile.in         |   2 +-
 lib/isc/sparc64/Makefile.in                      |   2 +-
 lib/isc/sparc64/include/Makefile.in              |   2 +-
 lib/isc/sparc64/include/isc/Makefile.in          |   2 +-
 lib/isc/task.c                                   |  45 +-
 lib/isc/task_api.c                               |  11 +
 lib/isc/unix/Makefile.in                         |   2 +-
 lib/isc/unix/file.c                              |  31 ++
 lib/isc/unix/include/Makefile.in                 |   2 +-
 lib/isc/unix/include/isc/Makefile.in             |   2 +-
 lib/isc/x86_32/Makefile.in                       |   2 +-
 lib/isc/x86_32/include/Makefile.in               |   2 +-
 lib/isc/x86_32/include/isc/Makefile.in           |   2 +-
 lib/isc/x86_64/Makefile.in                       |   2 +-
 lib/isc/x86_64/include/Makefile.in               |   2 +-
 lib/isc/x86_64/include/isc/Makefile.in           |   2 +-
 lib/isccc/api                                    |   2 +-
 lib/isccc/cc.c                                   |   6 +-
 lib/isccc/include/Makefile.in                    |   2 +-
 lib/isccc/include/isccc/Makefile.in              |   2 +-
 lib/isccfg/api                                   |   2 +-
 lib/isccfg/include/Makefile.in                   |   2 +-
 lib/isccfg/include/isccfg/Makefile.in            |   2 +-
 lib/isccfg/namedconf.c                           | 224 ++++++---
 lib/lwres/Makefile.in                            |   2 +-
 lib/lwres/api                                    |   2 +-
 lib/lwres/getaddrinfo.c                          |   8 +-
 lib/lwres/include/Makefile.in                    |   2 +-
 lib/lwres/include/lwres/Makefile.in              |   2 +-
 lib/lwres/man/Makefile.in                        |   2 +-
 134 files changed, 1813 insertions(+), 486 deletions(-)
 create mode 100644 lib/dns/opensslecdsa_link.c

(limited to 'lib')

diff --git a/lib/Makefile.in b/lib/Makefile.in
index e46aef2..e3f0bdb 100644
--- a/lib/Makefile.in
+++ b/lib/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2004, 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2004, 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 # Copyright (C) 1998-2001, 2003  Internet Software Consortium.
 #
 # Permission to use, copy, modify, and/or distribute this software for any
diff --git a/lib/bind9/Makefile.in b/lib/bind9/Makefile.in
index 35c4022..73285e1 100644
--- a/lib/bind9/Makefile.in
+++ b/lib/bind9/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2004, 2007, 2009  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2004, 2007, 2009, 2012  Internet Systems Consortium, Inc. ("ISC")
 # Copyright (C) 2001  Internet Software Consortium.
 #
 # Permission to use, copy, modify, and/or distribute this software for any
diff --git a/lib/bind9/api b/lib/bind9/api
index 089e782..7e9b115 100644
--- a/lib/bind9/api
+++ b/lib/bind9/api
@@ -4,5 +4,5 @@
 # 9.8: 80-89
 # 9.9: 90-109
 LIBINTERFACE = 80
-LIBREVISION = 5
+LIBREVISION = 7
 LIBAGE = 0
diff --git a/lib/bind9/check.c b/lib/bind9/check.c
index 26eaa1a..f765604 100644
--- a/lib/bind9/check.c
+++ b/lib/bind9/check.c
@@ -287,10 +287,6 @@ disabled_algorithms(const cfg_obj_t *disabled, isc_log_t *logctx) {
 
 		tresult = dns_secalg_fromtext(&alg, &r);
 		if (tresult != ISC_R_SUCCESS) {
-			isc_uint8_t ui;
-			result = isc_parse_uint8(&ui, r.base, 10);
-		}
-		if (tresult != ISC_R_SUCCESS) {
 			cfg_obj_log(cfg_listelt_value(element), logctx,
 				    ISC_LOG_ERROR, "invalid algorithm '%s'",
 				    r.base);
@@ -1259,6 +1255,29 @@ typedef struct {
 } optionstable;
 
 static isc_result_t
+check_nonzero(const cfg_obj_t *options, isc_log_t *logctx) {
+	isc_result_t result = ISC_R_SUCCESS;
+	const cfg_obj_t *obj = NULL;
+	unsigned int i;
+
+	static const char *nonzero[] = { "max-retry-time", "min-retry-time",
+				 "max-refresh-time", "min-refresh-time" };
+	/*
+	 * Check if value is zero.
+	 */
+	for (i = 0; i < sizeof(nonzero) / sizeof(nonzero[0]); i++) {
+		obj = NULL;
+		if (cfg_map_get(options, nonzero[i], &obj) == ISC_R_SUCCESS &&
+		    cfg_obj_asuint32(obj) == 0) {
+			cfg_obj_log(obj, logctx, ISC_LOG_ERROR,
+				    "'%s' must not be zero", nonzero[i]);
+			result = ISC_R_FAILURE;
+		}
+	}
+	return (result);
+}
+
+static isc_result_t
 check_zoneconf(const cfg_obj_t *zconfig, const cfg_obj_t *voptions,
 	       const cfg_obj_t *config, isc_symtab_t *symtab,
 	       dns_rdataclass_t defclass, cfg_aclconfctx_t *actx,
@@ -1267,7 +1286,7 @@ check_zoneconf(const cfg_obj_t *zconfig, const cfg_obj_t *voptions,
 	const char *znamestr;
 	const char *typestr;
 	unsigned int ztype;
-	const cfg_obj_t *zoptions;
+	const cfg_obj_t *zoptions, *goptions = NULL;
 	const cfg_obj_t *obj = NULL;
 	isc_result_t result = ISC_R_SUCCESS;
 	isc_result_t tresult;
@@ -1288,8 +1307,10 @@ check_zoneconf(const cfg_obj_t *zconfig, const cfg_obj_t *voptions,
 	{ "also-notify", MASTERZONE | SLAVEZONE },
 	{ "dialup", MASTERZONE | SLAVEZONE | STUBZONE },
 	{ "delegation-only", HINTZONE | STUBZONE | DELEGATIONZONE },
-	{ "forward", MASTERZONE | SLAVEZONE | STUBZONE | FORWARDZONE },
-	{ "forwarders", MASTERZONE | SLAVEZONE | STUBZONE | FORWARDZONE },
+	{ "forward", MASTERZONE | SLAVEZONE | STUBZONE |
+	  STATICSTUBZONE | FORWARDZONE },
+	{ "forwarders", MASTERZONE | SLAVEZONE | STUBZONE |
+	  STATICSTUBZONE | FORWARDZONE },
 	{ "maintain-ixfr-base", MASTERZONE | SLAVEZONE },
 	{ "max-ixfr-log-size", MASTERZONE | SLAVEZONE },
 	{ "notify-source", MASTERZONE | SLAVEZONE },
@@ -1345,10 +1366,14 @@ check_zoneconf(const cfg_obj_t *zconfig, const cfg_obj_t *voptions,
 	{ "passive", SLAVEZONE | STUBZONE },
 	};
 
+
 	znamestr = cfg_obj_asstring(cfg_tuple_get(zconfig, "name"));
 
 	zoptions = cfg_tuple_get(zconfig, "options");
 
+	if (config != NULL)
+		cfg_map_get(config, "options", &goptions);
+
 	obj = NULL;
 	(void)cfg_map_get(zoptions, "type", &obj);
 	if (obj == NULL) {
@@ -1430,6 +1455,12 @@ check_zoneconf(const cfg_obj_t *zconfig, const cfg_obj_t *voptions,
 	}
 
 	/*
+	 * Check if value is zero.
+	 */
+	if (check_nonzero(zoptions, logctx) != ISC_R_SUCCESS)
+		result = ISC_R_FAILURE;
+
+	/*
 	 * Look for inappropriate options for the given zone type.
 	 * Check that ACLs expand correctly.
 	 */
@@ -2170,6 +2201,14 @@ check_viewconf(const cfg_obj_t *config, const cfg_obj_t *voptions,
 	}
 
 	/*
+	 * Check non-zero options at the global and view levels.
+	 */
+	if (options != NULL && check_nonzero(options, logctx) != ISC_R_SUCCESS)
+		result = ISC_R_FAILURE;
+	if (voptions != NULL &&check_nonzero(voptions, logctx) != ISC_R_SUCCESS)
+		result = ISC_R_FAILURE;
+
+	/*
 	 * Check that dual-stack-servers is reasonable.
 	 */
 	if (voptions == NULL) {
@@ -2196,15 +2235,15 @@ check_viewconf(const cfg_obj_t *config, const cfg_obj_t *voptions,
 	tresult = isc_symtab_create(mctx, 1000, freekey, mctx,
 				    ISC_FALSE, &symtab);
 	if (tresult != ISC_R_SUCCESS)
-		return (ISC_R_NOMEMORY);
+		goto cleanup;
 
 	(void)cfg_map_get(config, "key", &keys);
 	tresult = check_keylist(keys, symtab, mctx, logctx);
 	if (tresult == ISC_R_EXISTS)
 		result = ISC_R_FAILURE;
 	else if (tresult != ISC_R_SUCCESS) {
-		isc_symtab_destroy(&symtab);
-		return (tresult);
+		result = tresult;
+		goto cleanup;
 	}
 
 	if (voptions != NULL) {
@@ -2214,8 +2253,8 @@ check_viewconf(const cfg_obj_t *config, const cfg_obj_t *voptions,
 		if (tresult == ISC_R_EXISTS)
 			result = ISC_R_FAILURE;
 		else if (tresult != ISC_R_SUCCESS) {
-			isc_symtab_destroy(&symtab);
-			return (tresult);
+			result = tresult;
+			goto cleanup;
 		}
 	}
 
@@ -2336,7 +2375,11 @@ check_viewconf(const cfg_obj_t *config, const cfg_obj_t *voptions,
 	if (tresult != ISC_R_SUCCESS)
 		result = tresult;
 
-	cfg_aclconfctx_detach(&actx);
+ cleanup:
+	if (symtab != NULL)
+		isc_symtab_destroy(&symtab);
+	if (actx != NULL)
+		cfg_aclconfctx_detach(&actx);
 
 	return (result);
 }
diff --git a/lib/bind9/include/Makefile.in b/lib/bind9/include/Makefile.in
index 65eecb0..0a7436c 100644
--- a/lib/bind9/include/Makefile.in
+++ b/lib/bind9/include/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2004, 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2004, 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 # Copyright (C) 2001  Internet Software Consortium.
 #
 # Permission to use, copy, modify, and/or distribute this software for any
diff --git a/lib/bind9/include/bind9/Makefile.in b/lib/bind9/include/bind9/Makefile.in
index 8abfaf6..11ae586 100644
--- a/lib/bind9/include/bind9/Makefile.in
+++ b/lib/bind9/include/bind9/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2004, 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2004, 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 # Copyright (C) 2001  Internet Software Consortium.
 #
 # Permission to use, copy, modify, and/or distribute this software for any
diff --git a/lib/dns/Makefile.in b/lib/dns/Makefile.in
index cfaf775..a01bb41 100644
--- a/lib/dns/Makefile.in
+++ b/lib/dns/Makefile.in
@@ -47,7 +47,8 @@ LIBS =		@LIBS@
 # Alphabetically
 
 OPENSSLLINKOBJS = openssl_link.@O@ openssldh_link.@O@ openssldsa_link.@O@ \
-		  opensslgost_link.@O@ opensslrsa_link.@O@
+		  opensslecdsa_link.@O@ opensslgost_link.@O@ \
+		  opensslrsa_link.@O@
 
 DSTOBJS =	@DST_EXTRA_OBJS@ @OPENSSLLINKOBJS@ \
 		dst_api.@O@ dst_lib.@O@ dst_parse.@O@ dst_result.@O@ \
@@ -76,7 +77,7 @@ OBJS=		${DNSOBJS} ${OTHEROBJS} ${DSTOBJS}
 
 # Alphabetically
 OPENSSLLINKSRCS = openssl_link.c openssldh_link.c openssldsa_link.c \
-		  opensslgost_link.c opensslrsa_link.c
+		  opensslecdsa_link.c opensslgost_link.c opensslrsa_link.c
 
 DSTSRCS =	@DST_EXTRA_SRCS@ @OPENSSLLINKSRCS@ \
 		dst_api.c dst_lib.c dst_parse.c \
diff --git a/lib/dns/adb.c b/lib/dns/adb.c
index 1dba252..531d112 100644
--- a/lib/dns/adb.c
+++ b/lib/dns/adb.c
@@ -111,6 +111,7 @@ struct dns_adb {
 
 	isc_taskmgr_t                  *taskmgr;
 	isc_task_t                     *task;
+	isc_task_t                     *excl;
 
 	isc_interval_t                  tick_interval;
 	int                             next_cleanbucket;
@@ -1627,10 +1628,12 @@ new_adbname(dns_adb_t *adb, dns_name_t *dnsname) {
 
 	LOCK(&adb->namescntlock);
 	adb->namescnt++;
-	if (!adb->grownames_sent && adb->namescnt > (adb->nnames * 8)) {
+	if (!adb->grownames_sent && adb->excl != NULL &&
+	    adb->namescnt > (adb->nnames * 8))
+	{
 		isc_event_t *event = &adb->grownames;
 		inc_adb_irefcnt(adb);
-		isc_task_send(adb->task, &event);
+		isc_task_send(adb->excl, &event);
 		adb->grownames_sent = ISC_TRUE;
 	}
 	UNLOCK(&adb->namescntlock);
@@ -1751,8 +1754,9 @@ new_adbentry(dns_adb_t *adb) {
 	ISC_LINK_INIT(e, plink);
 	LOCK(&adb->entriescntlock);
 	adb->entriescnt++;
-	if (!adb->growentries_sent &&
-	    adb->entriescnt > (adb->nentries * 8)) {
+	if (!adb->growentries_sent && adb->growentries_sent &&
+	    adb->entriescnt > (adb->nentries * 8))
+	{
 		isc_event_t *event = &adb->growentries;
 		inc_adb_irefcnt(adb);
 		isc_task_send(adb->task, &event);
@@ -2327,6 +2331,7 @@ destroy(dns_adb_t *adb) {
 	adb->magic = 0;
 
 	isc_task_detach(&adb->task);
+	isc_task_detach(&adb->excl);
 
 	isc_mempool_destroy(&adb->nmp);
 	isc_mempool_destroy(&adb->nhmp);
@@ -2410,6 +2415,7 @@ dns_adb_create(isc_mem_t *mem, dns_view_t *view, isc_timermgr_t *timermgr,
 	adb->aimp = NULL;
 	adb->afmp = NULL;
 	adb->task = NULL;
+	adb->excl = NULL;
 	adb->mctx = NULL;
 	adb->view = view;
 	adb->taskmgr = taskmgr;
@@ -2445,6 +2451,16 @@ dns_adb_create(isc_mem_t *mem, dns_view_t *view, isc_timermgr_t *timermgr,
 		       adb, NULL, NULL);
 	adb->grownames_sent = ISC_FALSE;
 
+	result = isc_taskmgr_excltask(adb->taskmgr, &adb->excl);
+	if (result != ISC_R_SUCCESS) {
+		DP(ISC_LOG_INFO, "adb: task-exclusive mode unavailable, "
+				 "intializing table sizes to %u\n",
+				 nbuckets[11]);
+		adb->nentries = nbuckets[11];
+		adb->nnames= nbuckets[11];
+
+	}
+
 	isc_mem_attach(mem, &adb->mctx);
 
 	result = isc_mutex_init(&adb->lock);
@@ -2557,6 +2573,7 @@ dns_adb_create(isc_mem_t *mem, dns_view_t *view, isc_timermgr_t *timermgr,
 	result = isc_task_create(adb->taskmgr, 0, &adb->task);
 	if (result != ISC_R_SUCCESS)
 		goto fail3;
+
 	isc_task_setname(adb->task, "ADB", adb);
 
 	/*
@@ -3904,8 +3921,10 @@ dns_adb_adjustsrtt(dns_adb_t *adb, dns_adbaddrinfo_t *addr,
 	addr->entry->srtt = new_srtt;
 	addr->srtt = new_srtt;
 
-	isc_stdtime_get(&now);
-	addr->entry->expires = now + ADB_ENTRY_WINDOW;
+	if (addr->entry->expires == 0) {
+		isc_stdtime_get(&now);
+		addr->entry->expires = now + ADB_ENTRY_WINDOW;
+	}
 
 	UNLOCK(&adb->entrylocks[bucket]);
 }
@@ -3915,6 +3934,7 @@ dns_adb_changeflags(dns_adb_t *adb, dns_adbaddrinfo_t *addr,
 		    unsigned int bits, unsigned int mask)
 {
 	int bucket;
+	isc_stdtime_t now;
 
 	REQUIRE(DNS_ADB_VALID(adb));
 	REQUIRE(DNS_ADBADDRINFO_VALID(addr));
@@ -3923,6 +3943,11 @@ dns_adb_changeflags(dns_adb_t *adb, dns_adbaddrinfo_t *addr,
 	LOCK(&adb->entrylocks[bucket]);
 
 	addr->entry->flags = (addr->entry->flags & ~mask) | (bits & mask);
+	if (addr->entry->expires == 0) {
+		isc_stdtime_get(&now);
+		addr->entry->expires = now + ADB_ENTRY_WINDOW;
+	}
+
 	/*
 	 * Note that we do not update the other bits in addr->flags with
 	 * the most recent values from addr->entry->flags.
@@ -4001,15 +4026,16 @@ dns_adb_freeaddrinfo(dns_adb_t *adb, dns_adbaddrinfo_t **addrp) {
 	entry = addr->entry;
 	REQUIRE(DNS_ADBENTRY_VALID(entry));
 
-	isc_stdtime_get(&now);
-
 	*addrp = NULL;
 	overmem = isc_mem_isovermem(adb->mctx);
 
 	bucket = addr->entry->lock_bucket;
 	LOCK(&adb->entrylocks[bucket]);
 
-	entry->expires = now + ADB_ENTRY_WINDOW;
+	if (entry->expires == 0) {
+		isc_stdtime_get(&now);
+		entry->expires = now + ADB_ENTRY_WINDOW;
+	}
 
 	want_check_exit = dec_entry_refcnt(adb, overmem, entry, ISC_FALSE);
 
diff --git a/lib/dns/api b/lib/dns/api
index 9e783a5..325781a 100644
--- a/lib/dns/api
+++ b/lib/dns/api
@@ -3,6 +3,6 @@
 # 9.7: 60-79
 # 9.8: 80-89
 # 9.9: 90-109
-LIBINTERFACE = 87
+LIBINTERFACE = 89
 LIBREVISION = 1
-LIBAGE = 6
+LIBAGE = 1
diff --git a/lib/dns/db.c b/lib/dns/db.c
index 1252c81..0cf2c27 100644
--- a/lib/dns/db.c
+++ b/lib/dns/db.c
@@ -952,14 +952,13 @@ dns_db_rpz_enabled(dns_db_t *db, dns_rpz_st_t *st)
 		(db->methods->rpz_enabled)(db, st);
 }
 
-isc_result_t
+void
 dns_db_rpz_findips(dns_rpz_zone_t *rpz, dns_rpz_type_t rpz_type,
 		   dns_zone_t *zone, dns_db_t *db, dns_dbversion_t *version,
 		   dns_rdataset_t *ardataset, dns_rpz_st_t *st,
 		   dns_name_t *query_qname)
 {
-	if (db->methods->rpz_findips == NULL)
-		return (ISC_R_NOTIMPLEMENTED);
-	return ((db->methods->rpz_findips)(rpz, rpz_type, zone, db, version,
-					   ardataset, st, query_qname));
+	if (db->methods->rpz_findips != NULL)
+		(db->methods->rpz_findips)(rpz, rpz_type, zone, db, version,
+					   ardataset, st, query_qname);
 }
diff --git a/lib/dns/dnssec.c b/lib/dns/dnssec.c
index b72e82d..3569ad7 100644
--- a/lib/dns/dnssec.c
+++ b/lib/dns/dnssec.c
@@ -44,10 +44,13 @@
 #include <dns/rdataset.h>
 #include <dns/rdatastruct.h>
 #include <dns/result.h>
+#include <dns/stats.h>
 #include <dns/tsig.h>		/* for DNS_TSIG_FUDGE */
 
 #include <dst/result.h>
 
+LIBDNS_EXTERNAL_DATA isc_stats_t *dns_dnssec_stats;
+
 #define is_response(msg) (msg->flags & DNS_MESSAGEFLAG_QR)
 
 #define RETERR(x) do { \
@@ -77,6 +80,12 @@ digest_callback(void *arg, isc_region_t *data) {
 	return (dst_context_adddata(ctx, data));
 }
 
+static inline void
+inc_stat(isc_statscounter_t counter) {
+	if (dns_dnssec_stats != NULL)
+		isc_stats_increment(dns_dnssec_stats, counter);
+}
+
 /*
  * Make qsort happy.
  */
@@ -153,7 +162,9 @@ dns_dnssec_keyfromrdata(dns_name_t *name, dns_rdata_t *rdata, isc_mem_t *mctx,
 }
 
 static isc_result_t
-digest_sig(dst_context_t *ctx, dns_rdata_t *sigrdata, dns_rdata_rrsig_t *sig) {
+digest_sig(dst_context_t *ctx, isc_boolean_t downcase, dns_rdata_t *sigrdata,
+	   dns_rdata_rrsig_t *rrsig)
+{
 	isc_region_t r;
 	isc_result_t ret;
 	dns_fixedname_t fname;
@@ -165,11 +176,16 @@ digest_sig(dst_context_t *ctx, dns_rdata_t *sigrdata, dns_rdata_rrsig_t *sig) {
 	ret = dst_context_adddata(ctx, &r);
 	if (ret != ISC_R_SUCCESS)
 		return (ret);
-	dns_fixedname_init(&fname);
-	RUNTIME_CHECK(dns_name_downcase(&sig->signer,
-					dns_fixedname_name(&fname), NULL)
-		      == ISC_R_SUCCESS);
-	dns_name_toregion(dns_fixedname_name(&fname), &r);
+	if (downcase) {
+		dns_fixedname_init(&fname);
+
+		RUNTIME_CHECK(dns_name_downcase(&rrsig->signer,
+						dns_fixedname_name(&fname),
+						NULL) == ISC_R_SUCCESS);
+		dns_name_toregion(dns_fixedname_name(&fname), &r);
+	} else
+		dns_name_toregion(&rrsig->signer, &r);
+
 	return (dst_context_adddata(ctx, &r));
 }
 
@@ -191,6 +207,7 @@ dns_dnssec_sign(dns_name_t *name, dns_rdataset_t *set, dst_key_t *key,
 	isc_uint32_t flags;
 	unsigned int sigsize;
 	dns_fixedname_t fnewname;
+	dns_fixedname_t fsigner;
 
 	REQUIRE(name != NULL);
 	REQUIRE(dns_name_countlabels(name) <= 255);
@@ -218,8 +235,14 @@ dns_dnssec_sign(dns_name_t *name, dns_rdataset_t *set, dst_key_t *key,
 	sig.common.rdtype = dns_rdatatype_rrsig;
 	ISC_LINK_INIT(&sig.common, link);
 
+	/*
+	 * Downcase signer.
+	 */
 	dns_name_init(&sig.signer, NULL);
-	dns_name_clone(dst_key_name(key), &sig.signer);
+	dns_fixedname_init(&fsigner);
+	RUNTIME_CHECK(dns_name_downcase(dst_key_name(key),
+		      dns_fixedname_name(&fsigner), NULL) == ISC_R_SUCCESS);
+	dns_name_clone(dns_fixedname_name(&fsigner), &sig.signer);
 
 	sig.covered = set->type;
 	sig.algorithm = dst_key_alg(key);
@@ -259,7 +282,7 @@ dns_dnssec_sign(dns_name_t *name, dns_rdataset_t *set, dst_key_t *key,
 	/*
 	 * Digest the SIG rdata.
 	 */
-	ret = digest_sig(ctx, &tmpsigrdata, &sig);
+	ret = digest_sig(ctx, ISC_FALSE, &tmpsigrdata, &sig);
 	if (ret != ISC_R_SUCCESS)
 		goto cleanup_context;
 
@@ -332,7 +355,7 @@ dns_dnssec_sign(dns_name_t *name, dns_rdataset_t *set, dst_key_t *key,
 	memcpy(sig.signature, r.base, sig.siglen);
 
 	ret = dns_rdata_fromstruct(sigrdata, sig.common.rdclass,
-				  sig.common.rdtype, &sig, buffer);
+				   sig.common.rdtype, &sig, buffer);
 
 cleanup_array:
 	isc_mem_put(mctx, rdatas, nrdatas * sizeof(dns_rdata_t));
@@ -363,6 +386,7 @@ dns_dnssec_verify2(dns_name_t *name, dns_rdataset_t *set, dst_key_t *key,
 	dst_context_t *ctx = NULL;
 	int labels = 0;
 	isc_uint32_t flags;
+	isc_boolean_t downcase = ISC_FALSE;
 
 	REQUIRE(name != NULL);
 	REQUIRE(set != NULL);
@@ -377,8 +401,10 @@ dns_dnssec_verify2(dns_name_t *name, dns_rdataset_t *set, dst_key_t *key,
 	if (set->type != sig.covered)
 		return (DNS_R_SIGINVALID);
 
-	if (isc_serial_lt(sig.timeexpire, sig.timesigned))
+	if (isc_serial_lt(sig.timeexpire, sig.timesigned)) {
+		inc_stat(dns_dnssecstats_fail);
 		return (DNS_R_SIGINVALID);
+	}
 
 	if (!ignoretime) {
 		isc_stdtime_get(&now);
@@ -386,10 +412,13 @@ dns_dnssec_verify2(dns_name_t *name, dns_rdataset_t *set, dst_key_t *key,
 		/*
 		 * Is SIG temporally valid?
 		 */
-		if (isc_serial_lt((isc_uint32_t)now, sig.timesigned))
+		if (isc_serial_lt((isc_uint32_t)now, sig.timesigned)) {
+			inc_stat(dns_dnssecstats_fail);
 			return (DNS_R_SIGFUTURE);
-		else if (isc_serial_lt(sig.timeexpire, (isc_uint32_t)now))
+		} else if (isc_serial_lt(sig.timeexpire, (isc_uint32_t)now)) {
+			inc_stat(dns_dnssecstats_fail);
 			return (DNS_R_SIGEXPIRED);
+		}
 	}
 
 	/*
@@ -400,16 +429,22 @@ dns_dnssec_verify2(dns_name_t *name, dns_rdataset_t *set, dst_key_t *key,
 	case dns_rdatatype_ns:
 	case dns_rdatatype_soa:
 	case dns_rdatatype_dnskey:
-		if (!dns_name_equal(name, &sig.signer))
+		if (!dns_name_equal(name, &sig.signer)) {
+			inc_stat(dns_dnssecstats_fail);
 			return (DNS_R_SIGINVALID);
+		}
 		break;
 	case dns_rdatatype_ds:
-		if (dns_name_equal(name, &sig.signer))
+		if (dns_name_equal(name, &sig.signer)) {
+			inc_stat(dns_dnssecstats_fail);
 			return (DNS_R_SIGINVALID);
+		}
 		/* FALLTHROUGH */
 	default:
-		if (!dns_name_issubdomain(name, &sig.signer))
+		if (!dns_name_issubdomain(name, &sig.signer)) {
+			inc_stat(dns_dnssecstats_fail);
 			return (DNS_R_SIGINVALID);
+		}
 		break;
 	}
 
@@ -417,11 +452,16 @@ dns_dnssec_verify2(dns_name_t *name, dns_rdataset_t *set, dst_key_t *key,
 	 * Is the key allowed to sign data?
 	 */
 	flags = dst_key_flags(key);
-	if (flags & DNS_KEYTYPE_NOAUTH)
+	if (flags & DNS_KEYTYPE_NOAUTH) {
+		inc_stat(dns_dnssecstats_fail);
 		return (DNS_R_KEYUNAUTHORIZED);
-	if ((flags & DNS_KEYFLAG_OWNERMASK) != DNS_KEYOWNER_ZONE)
+	}
+	if ((flags & DNS_KEYFLAG_OWNERMASK) != DNS_KEYOWNER_ZONE) {
+		inc_stat(dns_dnssecstats_fail);
 		return (DNS_R_KEYUNAUTHORIZED);
+	}
 
+ again:
 	ret = dst_context_create(key, mctx, &ctx);
 	if (ret != ISC_R_SUCCESS)
 		goto cleanup_struct;
@@ -429,7 +469,7 @@ dns_dnssec_verify2(dns_name_t *name, dns_rdataset_t *set, dst_key_t *key,
 	/*
 	 * Digest the SIG rdata (not including the signature).
 	 */
-	ret = digest_sig(ctx, sigrdata, &sig);
+	ret = digest_sig(ctx, downcase, sigrdata, &sig);
 	if (ret != ISC_R_SUCCESS)
 		goto cleanup_context;
 
@@ -508,21 +548,40 @@ dns_dnssec_verify2(dns_name_t *name, dns_rdataset_t *set, dst_key_t *key,
 	r.base = sig.signature;
 	r.length = sig.siglen;
 	ret = dst_context_verify(ctx, &r);
-	if (ret == DST_R_VERIFYFAILURE)
-		ret = DNS_R_SIGINVALID;
+	if (ret == ISC_R_SUCCESS && downcase) {
+		char namebuf[DNS_NAME_FORMATSIZE];
+		dns_name_format(&sig.signer, namebuf, sizeof(namebuf));
+		isc_log_write(dns_lctx, DNS_LOGCATEGORY_GENERAL,
+			      DNS_LOGMODULE_DNSSEC, ISC_LOG_INFO,
+			      "sucessfully validated after lower casing "
+			      "signer '%s'", namebuf);
+		inc_stat(dns_dnssecstats_downcase);
+	} else if (ret == ISC_R_SUCCESS)
+		inc_stat(dns_dnssecstats_asis);
 
 cleanup_array:
 	isc_mem_put(mctx, rdatas, nrdatas * sizeof(dns_rdata_t));
 cleanup_context:
 	dst_context_destroy(&ctx);
+	if (ret == DST_R_VERIFYFAILURE && !downcase) {
+		downcase = ISC_TRUE;
+		goto again;
+	}
 cleanup_struct:
 	dns_rdata_freestruct(&sig);
 
+	if (ret == DST_R_VERIFYFAILURE)
+		ret = DNS_R_SIGINVALID;
+
+	if (ret != ISC_R_SUCCESS)
+		inc_stat(dns_dnssecstats_fail);
+
 	if (ret == ISC_R_SUCCESS && labels - sig.labels > 0) {
 		if (wild != NULL)
 			RUNTIME_CHECK(dns_name_concatenate(dns_wildcardname,
 						 dns_fixedname_name(&fnewname),
 						 wild, NULL) == ISC_R_SUCCESS);
+		inc_stat(dns_dnssecstats_wildcard);
 		ret = DNS_R_FROMWILDCARD;
 	}
 	return (ret);
@@ -1325,11 +1384,12 @@ dns_dnssec_findmatchingkeys(dns_name_t *origin, const char *directory,
  * the keys in the keyset, regardless of whether they have
  * metadata indicating they should be deactivated or removed.
  */
-static void
+static isc_result_t
 addkey(dns_dnsseckeylist_t *keylist, dst_key_t **newkey,
        isc_boolean_t savekeys, isc_mem_t *mctx)
 {
 	dns_dnsseckey_t *key;
+	isc_result_t result;
 
 	/* Skip duplicates */
 	for (key = ISC_LIST_HEAD(*keylist);
@@ -1357,10 +1417,12 @@ addkey(dns_dnsseckeylist_t *keylist, dst_key_t **newkey,
 		}
 
 		key->source = dns_keysource_zoneapex;
-		return;
+		return (ISC_R_SUCCESS);
 	}
 
-	dns_dnsseckey_create(mctx, newkey, &key);
+	result = dns_dnsseckey_create(mctx, newkey, &key);
+	if (result != ISC_R_SUCCESS)
+		return (result);
 	if (key->legacy || savekeys) {
 		key->force_publish = ISC_TRUE;
 		key->force_sign = dst_key_isprivate(key->key);
@@ -1368,6 +1430,7 @@ addkey(dns_dnsseckeylist_t *keylist, dst_key_t **newkey,
 	key->source = dns_keysource_zoneapex;
 	ISC_LIST_APPEND(*keylist, key, link);
 	*newkey = NULL;
+	return (ISC_R_SUCCESS);
 }
 
 
@@ -1457,7 +1520,7 @@ dns_dnssec_keylistfromrdataset(dns_name_t *origin,
 			goto skip;
 
 		if (public) {
-			addkey(keylist, &pubkey, savekeys, mctx);
+			RETERR(addkey(keylist, &pubkey, savekeys, mctx));
 			goto skip;
 		}
 
@@ -1510,7 +1573,7 @@ dns_dnssec_keylistfromrdataset(dns_name_t *origin,
 		}
 
 		if (result == ISC_R_FILENOTFOUND || result == ISC_R_NOPERM) {
-			addkey(keylist, &pubkey, savekeys, mctx);
+			RETERR(addkey(keylist, &pubkey, savekeys, mctx));
 			goto skip;
 		}
 		RETERR(result);
@@ -1519,7 +1582,7 @@ dns_dnssec_keylistfromrdataset(dns_name_t *origin,
 		if ((dst_key_flags(privkey) & DNS_KEYTYPE_NOAUTH) != 0)
 			goto skip;
 
-		addkey(keylist, &privkey, savekeys, mctx);
+		RETERR(addkey(keylist, &privkey, savekeys, mctx));
  skip:
 		if (pubkey != NULL)
 			dst_key_free(&pubkey);
diff --git a/lib/dns/ds.c b/lib/dns/ds.c
index 3c74d1d..e72ecbb 100644
--- a/lib/dns/ds.c
+++ b/lib/dns/ds.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2004-2007, 2010  Internet Systems Consortium, Inc. ("ISC")
+ * Copyright (C) 2004-2007, 2010, 2012  Internet Systems Consortium, Inc. ("ISC")
  * Copyright (C) 2002, 2003  Internet Software Consortium.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
@@ -52,12 +52,13 @@ dns_ds_buildrdata(dns_name_t *owner, dns_rdata_t *key,
 {
 	dns_fixedname_t fname;
 	dns_name_t *name;
-	unsigned char digest[ISC_SHA256_DIGESTLENGTH];
+	unsigned char digest[ISC_SHA384_DIGESTLENGTH];
 	isc_region_t r;
 	isc_buffer_t b;
 	dns_rdata_ds_t ds;
 	isc_sha1_t sha1;
 	isc_sha256_t sha256;
+	isc_sha384_t sha384;
 #ifdef HAVE_OPENSSL_GOST
 	EVP_MD_CTX ctx;
 	const EVP_MD *md;
@@ -86,17 +87,18 @@ dns_ds_buildrdata(dns_name_t *owner, dns_rdata_t *key,
 		isc_sha1_update(&sha1, r.base, r.length);
 		isc_sha1_final(&sha1, digest);
 		break;
+
 #ifdef HAVE_OPENSSL_GOST
 #define CHECK(x)					\
 	if ((x) != 1) {					\
 		EVP_MD_CTX_cleanup(&ctx);		\
-		return (DST_R_OPENSSLFAILURE);		\
+		return (DST_R_CRYPTOFAILURE);		\
 	}
 
 	case DNS_DSDIGEST_GOST:
 		md = EVP_gost();
 		if (md == NULL)
-			return (DST_R_OPENSSLFAILURE);
+			return (DST_R_CRYPTOFAILURE);
 		EVP_MD_CTX_init(&ctx);
 		CHECK(EVP_DigestInit(&ctx, md));
 		dns_name_toregion(name, &r);
@@ -111,6 +113,18 @@ dns_ds_buildrdata(dns_name_t *owner, dns_rdata_t *key,
 		CHECK(EVP_DigestFinal(&ctx, digest, NULL));
 		break;
 #endif
+
+	case DNS_DSDIGEST_SHA384:
+		isc_sha384_init(&sha384);
+		dns_name_toregion(name, &r);
+		isc_sha384_update(&sha384, r.base, r.length);
+		dns_rdata_toregion(key, &r);
+		INSIST(r.length >= 4);
+		isc_sha384_update(&sha384, r.base, r.length);
+		isc_sha384_final(digest, &sha384);
+		break;
+
+	case DNS_DSDIGEST_SHA256:
 	default:
 		isc_sha256_init(&sha256);
 		dns_name_toregion(name, &r);
@@ -132,11 +146,18 @@ dns_ds_buildrdata(dns_name_t *owner, dns_rdata_t *key,
 	case DNS_DSDIGEST_SHA1:
 		ds.length = ISC_SHA1_DIGESTLENGTH;
 		break;
+
 #ifdef HAVE_OPENSSL_GOST
 	case DNS_DSDIGEST_GOST:
 		ds.length = ISC_GOST_DIGESTLENGTH;
 		break;
 #endif
+
+	case DNS_DSDIGEST_SHA384:
+		ds.length = ISC_SHA384_DIGESTLENGTH;
+		break;
+
+	case DNS_DSDIGEST_SHA256:
 	default:
 		ds.length = ISC_SHA256_DIGESTLENGTH;
 		break;
@@ -152,9 +173,11 @@ dns_ds_digest_supported(unsigned int digest_type) {
 #ifdef HAVE_OPENSSL_GOST
 	return  (ISC_TF(digest_type == DNS_DSDIGEST_SHA1 ||
 			digest_type == DNS_DSDIGEST_SHA256 ||
-			digest_type == DNS_DSDIGEST_GOST));
+			digest_type == DNS_DSDIGEST_GOST ||
+			digest_type == DNS_DSDIGEST_SHA384));
 #else
 	return  (ISC_TF(digest_type == DNS_DSDIGEST_SHA1 ||
-			digest_type == DNS_DSDIGEST_SHA256));
+			digest_type == DNS_DSDIGEST_SHA256 ||
+			digest_type == DNS_DSDIGEST_SHA384));
 #endif
 }
diff --git a/lib/dns/dst_api.c b/lib/dns/dst_api.c
index a14b268..f5dd89a 100644
--- a/lib/dns/dst_api.c
+++ b/lib/dns/dst_api.c
@@ -56,6 +56,7 @@
 #include <isc/string.h>
 #include <isc/time.h>
 #include <isc/util.h>
+#include <isc/file.h>
 
 #include <dns/fixedname.h>
 #include <dns/keyvalues.h>
@@ -226,6 +227,10 @@ dst_lib_init2(isc_mem_t *mctx, isc_entropy_t *ectx,
 #ifdef HAVE_OPENSSL_GOST
 	RETERR(dst__opensslgost_init(&dst_t_func[DST_ALG_ECCGOST]));
 #endif
+#ifdef HAVE_OPENSSL_ECDSA
+	RETERR(dst__opensslecdsa_init(&dst_t_func[DST_ALG_ECDSA256]));
+	RETERR(dst__opensslecdsa_init(&dst_t_func[DST_ALG_ECDSA384]));
+#endif
 #endif /* OPENSSL */
 #ifdef GSSAPI
 	RETERR(dst__gssapi_init(&dst_t_func[DST_ALG_GSSAPI]));
@@ -1110,6 +1115,12 @@ dst_key_sigsize(const dst_key_t *key, unsigned int *n) {
 	case DST_ALG_ECCGOST:
 		*n = DNS_SIG_GOSTSIGSIZE;
 		break;
+	case DST_ALG_ECDSA256:
+		*n = DNS_SIG_ECDSA256SIZE;
+		break;
+	case DST_ALG_ECDSA384:
+		*n = DNS_SIG_ECDSA384SIZE;
+		break;
 	case DST_ALG_HMACMD5:
 		*n = 16;
 		break;
@@ -1415,6 +1426,8 @@ issymmetric(const dst_key_t *key) {
 	case DST_ALG_NSEC3DSA:
 	case DST_ALG_DH:
 	case DST_ALG_ECCGOST:
+	case DST_ALG_ECDSA256:
+	case DST_ALG_ECDSA384:
 		return (ISC_FALSE);
 	case DST_ALG_HMACMD5:
 	case DST_ALG_GSSAPI:
@@ -1691,7 +1704,8 @@ algorithm_status(unsigned int alg) {
 	    alg == DST_ALG_HMACMD5 || alg == DST_ALG_NSEC3DSA ||
 	    alg == DST_ALG_NSEC3RSASHA1 ||
 	    alg == DST_ALG_RSASHA256 || alg == DST_ALG_RSASHA512 ||
-	    alg == DST_ALG_ECCGOST)
+	    alg == DST_ALG_ECCGOST ||
+	    alg == DST_ALG_ECDSA256 || alg == DST_ALG_ECDSA384)
 		return (DST_R_NOCRYPTO);
 #endif
 	return (DST_R_UNSUPPORTEDALG);
diff --git a/lib/dns/dst_internal.h b/lib/dns/dst_internal.h
index d9f6ac8..2f4f946 100644
--- a/lib/dns/dst_internal.h
+++ b/lib/dns/dst_internal.h
@@ -217,6 +217,9 @@ isc_result_t dst__gssapi_init(struct dst_func **funcp);
 #ifdef HAVE_OPENSSL_GOST
 isc_result_t dst__opensslgost_init(struct dst_func **funcp);
 #endif
+#ifdef HAVE_OPENSSL_ECDSA
+isc_result_t dst__opensslecdsa_init(struct dst_func **funcp);
+#endif
 
 /*%
  * Destructors
diff --git a/lib/dns/dst_openssl.h b/lib/dns/dst_openssl.h
index 7237a5e..a30fd6a 100644
--- a/lib/dns/dst_openssl.h
+++ b/lib/dns/dst_openssl.h
@@ -39,6 +39,9 @@ ISC_LANG_BEGINDECLS
 isc_result_t
 dst__openssl_toresult(isc_result_t fallback);
 
+isc_result_t
+dst__openssl_toresult2(const char *funcname, isc_result_t fallback);
+
 #ifdef USE_ENGINE
 ENGINE *
 dst__openssl_getengine(const char *engine);
diff --git a/lib/dns/dst_parse.c b/lib/dns/dst_parse.c
index 8e9efbd..95896c4 100644
--- a/lib/dns/dst_parse.c
+++ b/lib/dns/dst_parse.c
@@ -44,8 +44,10 @@
 #include <isc/stdtime.h>
 #include <isc/string.h>
 #include <isc/util.h>
+#include <isc/file.h>
 
 #include <dns/time.h>
+#include <dns/log.h>
 
 #include "dst_internal.h"
 #include "dst_parse.h"
@@ -106,6 +108,8 @@ static struct parse_map map[] = {
 
 	{TAG_GOST_PRIVASN1, "GostAsn1:"},
 
+	{TAG_ECDSA_PRIVATEKEY, "PrivateKey:"},
+
 	{TAG_HMACMD5_KEY, "Key:"},
 	{TAG_HMACMD5_BITS, "Bits:"},
 
@@ -251,6 +255,15 @@ check_gost(const dst_private_t *priv) {
 }
 
 static int
+check_ecdsa(const dst_private_t *priv) {
+	if (priv->nelements != ECDSA_NTAGS)
+		return (-1);
+	if (priv->elements[0].tag != TAG(DST_ALG_ECDSA256, 0))
+		return (-1);
+	return (0);
+}
+
+static int
 check_hmac_md5(const dst_private_t *priv, isc_boolean_t old) {
 	int i, j;
 
@@ -302,13 +315,20 @@ check_data(const dst_private_t *priv, const unsigned int alg,
 	switch (alg) {
 	case DST_ALG_RSAMD5:
 	case DST_ALG_RSASHA1:
+	case DST_ALG_NSEC3RSASHA1:
+	case DST_ALG_RSASHA256:
+	case DST_ALG_RSASHA512:
 		return (check_rsa(priv));
 	case DST_ALG_DH:
 		return (check_dh(priv));
 	case DST_ALG_DSA:
+	case DST_ALG_NSEC3DSA:
 		return (check_dsa(priv));
 	case DST_ALG_ECCGOST:
 		return (check_gost(priv));
+	case DST_ALG_ECDSA256:
+	case DST_ALG_ECDSA384:
+		return (check_ecdsa(priv));
 	case DST_ALG_HMACMD5:
 		return (check_hmac_md5(priv, old));
 	case DST_ALG_HMACSHA1:
@@ -345,7 +365,7 @@ isc_result_t
 dst__privstruct_parse(dst_key_t *key, unsigned int alg, isc_lex_t *lex,
 		      isc_mem_t *mctx, dst_private_t *priv)
 {
-	int n = 0, major, minor;
+	int n = 0, major, minor, check;
 	isc_buffer_t b;
 	isc_token_t token;
 	unsigned char *data = NULL;
@@ -515,8 +535,14 @@ dst__privstruct_parse(dst_key_t *key, unsigned int alg, isc_lex_t *lex,
 		data = NULL;
 	}
  done:
-	if (check_data(priv, alg, ISC_TRUE) < 0)
+	check = check_data(priv, alg, ISC_TRUE);
+	if (check < 0) {
+		ret = DST_R_INVALIDPRIVATEKEY;
+		goto fail;
+	} else if (check != ISC_R_SUCCESS) {
+		ret = check;
 		goto fail;
+	}
 
 	return (ISC_R_SUCCESS);
 
@@ -533,7 +559,6 @@ dst__privstruct_writefile(const dst_key_t *key, const dst_private_t *priv,
 			  const char *directory)
 {
 	FILE *fp;
-	int ret, i;
 	isc_result_t result;
 	char filename[ISC_DIR_NAMEMAX];
 	char buffer[MAXFIELDSIZE * 2];
@@ -543,16 +568,32 @@ dst__privstruct_writefile(const dst_key_t *key, const dst_private_t *priv,
 	isc_buffer_t b;
 	isc_region_t r;
 	int major, minor;
+	mode_t mode;
+	int i, ret;
 
 	REQUIRE(priv != NULL);
 
-	if (check_data(priv, dst_key_alg(key), ISC_FALSE) < 0)
+	ret = check_data(priv, dst_key_alg(key), ISC_FALSE);
+	if (ret < 0)
 		return (DST_R_INVALIDPRIVATEKEY);
+	else if (ret != ISC_R_SUCCESS)
+		return (ret);
 
 	isc_buffer_init(&b, filename, sizeof(filename));
-	ret = dst_key_buildfilename(key, DST_TYPE_PRIVATE, directory, &b);
-	if (ret != ISC_R_SUCCESS)
-		return (ret);
+	result = dst_key_buildfilename(key, DST_TYPE_PRIVATE, directory, &b);
+	if (result != ISC_R_SUCCESS)
+		return (result);
+
+	result = isc_file_mode(filename, &mode);
+	if (result == ISC_R_SUCCESS && mode != 0600) {
+		/* File exists; warn that we are changing its permissions */
+		isc_log_write(dns_lctx, DNS_LOGCATEGORY_GENERAL,
+			      DNS_LOGMODULE_DNSSEC, ISC_LOG_WARNING,
+			      "Permissions on the file %s "
+			      "have changed from 0%o to 0600 as "
+			      "a result of this operation.",
+			      filename, (unsigned int)mode);
+	}
 
 	if ((fp = fopen(filename, "w")) == NULL)
 		return (DST_R_WRITEERROR);
@@ -603,6 +644,12 @@ dst__privstruct_writefile(const dst_key_t *key, const dst_private_t *priv,
 	case DST_ALG_ECCGOST:
 		fprintf(fp, "(ECC-GOST)\n");
 		break;
+	case DST_ALG_ECDSA256:
+		fprintf(fp, "(ECDSAP256SHA256)\n");
+		break;
+	case DST_ALG_ECDSA384:
+		fprintf(fp, "(ECDSAP384SHA384)\n");
+		break;
 	case DST_ALG_HMACMD5:
 		fprintf(fp, "(HMAC_MD5)\n");
 		break;
diff --git a/lib/dns/dst_parse.h b/lib/dns/dst_parse.h
index 91b072f..f048bf0 100644
--- a/lib/dns/dst_parse.h
+++ b/lib/dns/dst_parse.h
@@ -1,5 +1,5 @@
 /*
- * Portions Copyright (C) 2004-2010  Internet Systems Consortium, Inc. ("ISC")
+ * Portions Copyright (C) 2004-2010, 2012  Internet Systems Consortium, Inc. ("ISC")
  * Portions Copyright (C) 2000-2002  Internet Software Consortium.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
@@ -81,6 +81,9 @@
 #define GOST_NTAGS		1
 #define TAG_GOST_PRIVASN1	((DST_ALG_ECCGOST << TAG_SHIFT) + 0)
 
+#define ECDSA_NTAGS		1
+#define TAG_ECDSA_PRIVATEKEY	((DST_ALG_ECDSA256 << TAG_SHIFT) + 0)
+
 #define OLD_HMACMD5_NTAGS	1
 #define HMACMD5_NTAGS		2
 #define TAG_HMACMD5_KEY		((DST_ALG_HMACMD5 << TAG_SHIFT) + 0)
diff --git a/lib/dns/dst_result.c b/lib/dns/dst_result.c
index 429dbb2..297e809 100644
--- a/lib/dns/dst_result.c
+++ b/lib/dns/dst_result.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2004, 2005, 2007, 2008  Internet Systems Consortium, Inc. ("ISC")
+ * Copyright (C) 2004, 2005, 2007, 2008, 2012  Internet Systems Consortium, Inc. ("ISC")
  * Copyright (C) 1999-2001  Internet Software Consortium.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
@@ -30,7 +30,7 @@
 
 static const char *text[DST_R_NRESULTS] = {
 	"algorithm is unsupported",		/*%< 0 */
-	"openssl failure",			/*%< 1 */
+	"crypto failure",			/*%< 1 */
 	"built with no crypto support",		/*%< 2 */
 	"illegal operation for a null key",	/*%< 3 */
 	"public key is invalid",		/*%< 4 */
diff --git a/lib/dns/include/Makefile.in b/lib/dns/include/Makefile.in
index b52cb98..10d798d 100644
--- a/lib/dns/include/Makefile.in
+++ b/lib/dns/include/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2004, 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2004, 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 # Copyright (C) 1998-2001  Internet Software Consortium.
 #
 # Permission to use, copy, modify, and/or distribute this software for any
diff --git a/lib/dns/include/dns/db.h b/lib/dns/include/dns/db.h
index 6aa02d2..fe268f4 100644
--- a/lib/dns/include/dns/db.h
+++ b/lib/dns/include/dns/db.h
@@ -172,7 +172,7 @@ typedef struct dns_dbmethods {
 	isc_boolean_t	(*isdnssec)(dns_db_t *db);
 	dns_stats_t	*(*getrrsetstats)(dns_db_t *db);
 	void		(*rpz_enabled)(dns_db_t *db, dns_rpz_st_t *st);
-	isc_result_t	(*rpz_findips)(dns_rpz_zone_t *rpz,
+	void		(*rpz_findips)(dns_rpz_zone_t *rpz,
 				       dns_rpz_type_t rpz_type,
 				       dns_zone_t *zone, dns_db_t *db,
 				       dns_dbversion_t *version,
@@ -1507,7 +1507,7 @@ dns_db_rpz_enabled(dns_db_t *db, dns_rpz_st_t *st);
  * DNS_RPZ_TYPE_NSDNAME records.
  */
 
-isc_result_t
+void
 dns_db_rpz_findips(dns_rpz_zone_t *rpz, dns_rpz_type_t rpz_type,
 		   dns_zone_t *zone, dns_db_t *db, dns_dbversion_t *version,
 		   dns_rdataset_t *ardataset, dns_rpz_st_t *st,
@@ -1524,10 +1524,6 @@ dns_db_rpz_findips(dns_rpz_zone_t *rpz, dns_rpz_type_t rpz_type,
  * \li	'ardataset' is an A or AAAA rdataset of addresses to check
  * \li	'found' specifies the previous best match if any or
  *	    or NULL, an empty name, 0, DNS_RPZ_POLICY_MISS, and 0
- *
- * Returns:
- * \li	#ISC_R_SUCCESS
- * \li	#ISC_R_UNEXPECTED
  */
 
 ISC_LANG_ENDDECLS
diff --git a/lib/dns/include/dns/dnssec.h b/lib/dns/include/dns/dnssec.h
index b87fd5c..e986d40 100644
--- a/lib/dns/include/dns/dnssec.h
+++ b/lib/dns/include/dns/dnssec.h
@@ -24,6 +24,7 @@
 
 #include <isc/lang.h>
 #include <isc/stdtime.h>
+#include <isc/stats.h>
 
 #include <dns/diff.h>
 #include <dns/types.h>
@@ -32,6 +33,8 @@
 
 ISC_LANG_BEGINDECLS
 
+LIBDNS_EXTERNAL_DATA extern isc_stats_t *dns_dnssec_stats;
+
 /*%< Maximum number of keys supported in a zone. */
 #define DNS_MAXZONEKEYS 32
 
@@ -96,8 +99,8 @@ dns_dnssec_sign(dns_name_t *name, dns_rdataset_t *set, dst_key_t *key,
 		isc_stdtime_t *inception, isc_stdtime_t *expire,
 		isc_mem_t *mctx, isc_buffer_t *buffer, dns_rdata_t *sigrdata);
 /*%<
- *	Generates a SIG record covering this rdataset.  This has no effect
- *	on existing SIG records.
+ *	Generates a RRSIG record covering this rdataset.  This has no effect
+ *	on existing RRSIG records.
  *
  *	Requires:
  *\li		'name' (the owner name of the record) is a valid name
@@ -130,9 +133,9 @@ dns_dnssec_verify2(dns_name_t *name, dns_rdataset_t *set, dst_key_t *key,
 		   isc_boolean_t ignoretime, isc_mem_t *mctx,
 		   dns_rdata_t *sigrdata, dns_name_t *wild);
 /*%<
- *	Verifies the SIG record covering this rdataset signed by a specific
- *	key.  This does not determine if the key's owner is authorized to
- *	sign this record, as this requires a resolver or database.
+ *	Verifies the RRSIG record covering this rdataset signed by a specific
+ *	key.  This does not determine if the key's owner is authorized to sign
+ *	this record, as this requires a resolver or database.
  *	If 'ignoretime' is ISC_TRUE, temporal validity will not be checked.
  *
  *	Requires:
diff --git a/lib/dns/include/dns/ds.h b/lib/dns/include/dns/ds.h
index b20b408..03ab0ed 100644
--- a/lib/dns/include/dns/ds.h
+++ b/lib/dns/include/dns/ds.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2004-2007, 2010  Internet Systems Consortium, Inc. ("ISC")
+ * Copyright (C) 2004-2007, 2010, 2012  Internet Systems Consortium, Inc. ("ISC")
  * Copyright (C) 2002  Internet Software Consortium.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
@@ -27,15 +27,16 @@
 #define DNS_DSDIGEST_SHA1 (1)
 #define DNS_DSDIGEST_SHA256 (2)
 #define DNS_DSDIGEST_GOST (3)
+#define DNS_DSDIGEST_SHA384 (4)
 
 /* should not be here... */
 
 #define ISC_GOST_DIGESTLENGTH 32U
 
 /*
- * Assuming SHA-256 digest type.
+ * Assuming SHA-384 digest type.
  */
-#define DNS_DS_BUFFERSIZE (36)
+#define DNS_DS_BUFFERSIZE (52)
 
 ISC_LANG_BEGINDECLS
 
diff --git a/lib/dns/include/dns/iptable.h b/lib/dns/include/dns/iptable.h
index d7eb140..2ce8e18 100644
--- a/lib/dns/include/dns/iptable.h
+++ b/lib/dns/include/dns/iptable.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2007  Internet Systems Consortium, Inc. ("ISC")
+ * Copyright (C) 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -23,6 +23,8 @@
 #include <isc/magic.h>
 #include <isc/radix.h>
 
+#include <dns/types.h>
+
 struct dns_iptable {
 	unsigned int		magic;
 	isc_mem_t		*mctx;
diff --git a/lib/dns/include/dns/keyvalues.h b/lib/dns/include/dns/keyvalues.h
index 55c0b8f..0c392ca 100644
--- a/lib/dns/include/dns/keyvalues.h
+++ b/lib/dns/include/dns/keyvalues.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2004-2010  Internet Systems Consortium, Inc. ("ISC")
+ * Copyright (C) 2004-2010, 2012  Internet Systems Consortium, Inc. ("ISC")
  * Copyright (C) 1999-2001, 2003  Internet Software Consortium.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
@@ -71,6 +71,8 @@
 #define DNS_KEYALG_RSASHA256	8
 #define DNS_KEYALG_RSASHA512	10
 #define DNS_KEYALG_ECCGOST	12
+#define DNS_KEYALG_ECDSA256	13
+#define DNS_KEYALG_ECDSA384	14
 #define DNS_KEYALG_INDIRECT	252
 #define DNS_KEYALG_PRIVATEDNS	253
 #define DNS_KEYALG_PRIVATEOID	254     /*%< Key begins with OID giving alg */
@@ -101,4 +103,10 @@
 
 #define DNS_SIG_GOSTSIGSIZE	64
 
+#define DNS_SIG_ECDSA256SIZE	64
+#define DNS_SIG_ECDSA384SIZE	96
+
+#define DNS_KEY_ECDSA256SIZE	64
+#define DNS_KEY_ECDSA384SIZE	96
+
 #endif /* DNS_KEYVALUES_H */
diff --git a/lib/dns/include/dns/log.h b/lib/dns/include/dns/log.h
index b73b17f..689b148 100644
--- a/lib/dns/include/dns/log.h
+++ b/lib/dns/include/dns/log.h
@@ -75,6 +75,7 @@ LIBDNS_EXTERNAL_DATA extern isc_logmodule_t dns_modules[];
 #define DNS_LOGMODULE_ACACHE		(&dns_modules[25])
 #define DNS_LOGMODULE_DLZ		(&dns_modules[26])
 #define DNS_LOGMODULE_DNSSEC		(&dns_modules[27])
+#define DNS_LOGMODULE_CRYPTO		(&dns_modules[28])
 
 ISC_LANG_BEGINDECLS
 
diff --git a/lib/dns/include/dns/rdataset.h b/lib/dns/include/dns/rdataset.h
index 7918551..b2b8370 100644
--- a/lib/dns/include/dns/rdataset.h
+++ b/lib/dns/include/dns/rdataset.h
@@ -56,6 +56,7 @@
 #include <isc/stdtime.h>
 
 #include <dns/types.h>
+#include <dns/rdatastruct.h>
 
 ISC_LANG_BEGINDECLS
 
@@ -651,6 +652,25 @@ dns_rdataset_expire(dns_rdataset_t *rdataset);
  * Mark the rdataset to be expired in the backing database.
  */
 
+void
+dns_rdataset_trimttl(dns_rdataset_t *rdataset, dns_rdataset_t *sigrdataset,
+		     dns_rdata_rrsig_t *rrsig, isc_stdtime_t now,
+		     isc_boolean_t acceptexpired);
+/*%<
+ * Trim the ttl of 'rdataset' and 'sigrdataset' so that they will expire
+ * at or before 'rrsig->expiretime'.  If 'acceptexpired' is true and the
+ * signature has expired or will expire in the next 120 seconds, limit
+ * the ttl to be no more than 120 seconds.
+ *
+ * The ttl is further limited by the original ttl as stored in 'rrsig'
+ * and the original ttl values of 'rdataset' and 'sigrdataset'.
+ *
+ * Requires:
+ * \li	'rdataset' is a valid rdataset.
+ * \li	'sigrdataset' is a valid rdataset.
+ * \li	'rrsig' is non NULL.
+ */
+
 const char *
 dns_trust_totext(dns_trust_t trust);
 /*
diff --git a/lib/dns/include/dns/rpz.h b/lib/dns/include/dns/rpz.h
index 59d4f87..4227dd4 100644
--- a/lib/dns/include/dns/rpz.h
+++ b/lib/dns/include/dns/rpz.h
@@ -30,6 +30,7 @@ ISC_LANG_BEGINDECLS
 #define DNS_RPZ_IP_ZONE		"rpz-ip"
 #define DNS_RPZ_NSIP_ZONE	"rpz-nsip"
 #define DNS_RPZ_NSDNAME_ZONE	"rpz-nsdname"
+#define DNS_RPZ_PASSTHRU_ZONE	"rpz-passthru"
 
 typedef isc_uint8_t		dns_rpz_cidr_bits_t;
 
@@ -66,11 +67,14 @@ typedef struct dns_rpz_zone dns_rpz_zone_t;
 
 struct dns_rpz_zone {
 	ISC_LINK(dns_rpz_zone_t) link;
-	int			 num;
+	int			 num;	  /* ordinal in list of policy zones */
 	dns_name_t		 origin;  /* Policy zone name */
 	dns_name_t		 nsdname; /* DNS_RPZ_NSDNAME_ZONE.origin */
-	dns_rpz_policy_t	 policy;  /* DNS_RPZ_POLICY_GIVEN or override */
+	dns_name_t		 passthru;/* DNS_RPZ_PASSTHRU_ZONE. */
 	dns_name_t		 cname;	  /* override value for ..._CNAME */
+	dns_ttl_t		 max_policy_ttl;
+	dns_rpz_policy_t	 policy;  /* DNS_RPZ_POLICY_GIVEN or override */
+	isc_boolean_t		 recursive_only;
 };
 
 /*
@@ -143,6 +147,7 @@ typedef struct {
 } dns_rpz_st_t;
 
 #define DNS_RPZ_TTL_DEFAULT		5
+#define DNS_RPZ_MAX_TTL_DEFAULT		DNS_RPZ_TTL_DEFAULT
 
 /*
  * So various response policy zone messages can be turned up or down.
@@ -152,6 +157,7 @@ typedef struct {
 #define DNS_RPZ_DEBUG_LEVEL1	ISC_LOG_DEBUG(1)
 #define DNS_RPZ_DEBUG_LEVEL2	ISC_LOG_DEBUG(2)
 #define DNS_RPZ_DEBUG_LEVEL3	ISC_LOG_DEBUG(3)
+#define DNS_RPZ_DEBUG_QUIET	(DNS_RPZ_DEBUG_LEVEL3+1)
 
 const char *
 dns_rpz_type2str(dns_rpz_type_t type);
@@ -192,7 +198,8 @@ dns_rpz_cidr_find(dns_rpz_cidr_t *cidr, const isc_netaddr_t *netaddr,
 		  dns_name_t *search_name, dns_rpz_cidr_bits_t *prefix);
 
 dns_rpz_policy_t
-dns_rpz_decode_cname(dns_rdataset_t *, dns_name_t *selfname);
+dns_rpz_decode_cname(dns_rpz_zone_t *rpz, dns_rdataset_t *rdataset,
+		     dns_name_t *selfname);
 
 ISC_LANG_ENDDECLS
 
diff --git a/lib/dns/include/dns/stats.h b/lib/dns/include/dns/stats.h
index bc77d1e..5364267 100644
--- a/lib/dns/include/dns/stats.h
+++ b/lib/dns/include/dns/stats.h
@@ -64,6 +64,16 @@ enum {
 
 	dns_resstatscounter_max = 30,
 
+	/*
+	 * DNSSEC stats.
+	 */
+	dns_dnssecstats_asis = 0,
+	dns_dnssecstats_downcase = 1,
+	dns_dnssecstats_wildcard = 2,
+	dns_dnssecstats_fail = 3,
+
+	dns_dnssecstats_max = 4,
+
 	/*%
 	 * Zone statistics counters.
 	 */
diff --git a/lib/dns/include/dns/view.h b/lib/dns/include/dns/view.h
index 7db600b..4a04867 100644
--- a/lib/dns/include/dns/view.h
+++ b/lib/dns/include/dns/view.h
@@ -162,6 +162,8 @@ struct dns_view {
 	dns_dns64list_t 		dns64;
 	unsigned int 			dns64cnt;
 	ISC_LIST(dns_rpz_zone_t)	rpz_zones;
+	isc_boolean_t			rpz_recursive_only;
+	isc_boolean_t			rpz_break_dnssec;
 
 	/*
 	 * Configurable data for server use only,
diff --git a/lib/dns/include/dns/zone.h b/lib/dns/include/dns/zone.h
index 9eada59..9db825c 100644
--- a/lib/dns/include/dns/zone.h
+++ b/lib/dns/include/dns/zone.h
@@ -1815,7 +1815,7 @@ dns_zone_setsignatures(dns_zone_t *zone, isc_uint32_t signatures);
 
 isc_result_t
 dns_zone_signwithkey(dns_zone_t *zone, dns_secalg_t algorithm,
-		     isc_uint16_t keyid, isc_boolean_t delete);
+		     isc_uint16_t keyid, isc_boolean_t deleteit);
 /*%<
  * Initiate/resume signing of the entire zone with the zone DNSKEY(s)
  * that match the given algorithm and keyid.
diff --git a/lib/dns/include/dst/Makefile.in b/lib/dns/include/dst/Makefile.in
index 4ed4ec0..cece67d 100644
--- a/lib/dns/include/dst/Makefile.in
+++ b/lib/dns/include/dst/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2004, 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2004, 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 # Copyright (C) 1998-2001  Internet Software Consortium.
 #
 # Permission to use, copy, modify, and/or distribute this software for any
diff --git a/lib/dns/include/dst/dst.h b/lib/dns/include/dst/dst.h
index bf314f3..b0fa690 100644
--- a/lib/dns/include/dst/dst.h
+++ b/lib/dns/include/dst/dst.h
@@ -59,6 +59,8 @@ typedef struct dst_context 	dst_context_t;
 #define DST_ALG_RSASHA256	8
 #define DST_ALG_RSASHA512	10
 #define DST_ALG_ECCGOST		12
+#define DST_ALG_ECDSA256	13
+#define DST_ALG_ECDSA384	14
 #define DST_ALG_HMACMD5		157
 #define DST_ALG_GSSAPI		160
 #define DST_ALG_HMACSHA1	161	/* XXXMPA */
diff --git a/lib/dns/include/dst/result.h b/lib/dns/include/dst/result.h
index d77b72e..00640a1 100644
--- a/lib/dns/include/dst/result.h
+++ b/lib/dns/include/dst/result.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2004-2008  Internet Systems Consortium, Inc. ("ISC")
+ * Copyright (C) 2004-2008, 2012  Internet Systems Consortium, Inc. ("ISC")
  * Copyright (C) 1999-2001  Internet Software Consortium.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
@@ -34,7 +34,9 @@
 #include <isc/result.h>		/* Contractual promise. */
 
 #define DST_R_UNSUPPORTEDALG		(ISC_RESULTCLASS_DST + 0)
-#define DST_R_OPENSSLFAILURE		(ISC_RESULTCLASS_DST + 1)
+#define DST_R_CRYPTOFAILURE		(ISC_RESULTCLASS_DST + 1)
+/* compat */
+#define DST_R_OPENSSLFAILURE		DST_R_CRYPTOFAILURE
 #define DST_R_NOCRYPTO			(ISC_RESULTCLASS_DST + 2)
 #define DST_R_NULLKEY			(ISC_RESULTCLASS_DST + 3)
 #define DST_R_INVALIDPUBLICKEY		(ISC_RESULTCLASS_DST + 4)
diff --git a/lib/dns/log.c b/lib/dns/log.c
index 8f3774f..d286d10 100644
--- a/lib/dns/log.c
+++ b/lib/dns/log.c
@@ -81,6 +81,7 @@ LIBDNS_EXTERNAL_DATA isc_logmodule_t dns_modules[] = {
 	{ "dns/acache",		0 },
 	{ "dns/dlz",		0 },
 	{ "dns/dnssec",		0 },
+	{ "dns/crypto",		0 },
 	{ NULL, 		0 }
 };
 
diff --git a/lib/dns/master.c b/lib/dns/master.c
index 7f6cf58..8304507 100644
--- a/lib/dns/master.c
+++ b/lib/dns/master.c
@@ -2076,19 +2076,21 @@ load_raw(dns_loadctx_t *lctx) {
 	unsigned int loop_cnt = 0;
 	dns_rdatacallbacks_t *callbacks;
 	unsigned char namebuf[DNS_NAME_MAXWIRE];
-	isc_region_t r;
-	dns_name_t name;
+	dns_fixedname_t fixed;
+	dns_name_t *name;
 	rdatalist_head_t head, dummy;
 	dns_rdatalist_t rdatalist;
 	isc_mem_t *mctx = lctx->mctx;
 	dns_rdata_t *rdata = NULL;
 	unsigned int rdata_size = 0;
 	int target_size = TSIZ;
-	isc_buffer_t target;
+	isc_buffer_t target, buf;
 	unsigned char *target_mem = NULL;
+	dns_decompress_t dctx;
 
 	REQUIRE(DNS_LCTX_VALID(lctx));
 	callbacks = lctx->callbacks;
+	dns_decompress_init(&dctx, -1, DNS_DECOMPRESS_NONE);
 
 	if (lctx->first) {
 		dns_masterrawheader_t header;
@@ -2145,6 +2147,9 @@ load_raw(dns_loadctx_t *lctx) {
 	}
 	isc_buffer_init(&target, target_mem, target_size);
 
+	dns_fixedname_init(&fixed);
+	name = dns_fixedname_name(&fixed);
+
 	/*
 	 * In the following loop, we regard any error fatal regardless of
 	 * whether "MANYERRORS" is set in the context option.  This is because
@@ -2156,7 +2161,7 @@ load_raw(dns_loadctx_t *lctx) {
 	for (loop_cnt = 0;
 	     (lctx->loop_cnt == 0 || loop_cnt < lctx->loop_cnt);
 	     loop_cnt++) {
-		unsigned int i, rdcount, consumed_name;
+		unsigned int i, rdcount;
 		isc_uint16_t namelen;
 		isc_uint32_t totallen;
 		size_t minlen, readlen;
@@ -2246,12 +2251,11 @@ load_raw(dns_loadctx_t *lctx) {
 					lctx->f);
 		if (result != ISC_R_SUCCESS)
 			goto cleanup;
+
 		isc_buffer_setactive(&target, (unsigned int)namelen);
-		isc_buffer_activeregion(&target, &r);
-		dns_name_init(&name, NULL);
-		dns_name_fromregion(&name, &r);
-		isc_buffer_forward(&target, (unsigned int)namelen);
-		consumed_name = isc_buffer_consumedlength(&target);
+		result = dns_name_fromwire(name, &target, &dctx, 0, NULL);
+		if (result != ISC_R_SUCCESS)
+			goto cleanup;
 
 		/* Rdata contents. */
 		if (rdcount > rdata_size) {
@@ -2282,7 +2286,7 @@ load_raw(dns_loadctx_t *lctx) {
 
 				/* Partial Commit. */
 				ISC_LIST_APPEND(head, &rdatalist, link);
-				result = commit(callbacks, lctx, &head, &name,
+				result = commit(callbacks, lctx, &head, name,
 						NULL, 0);
 				for (j = 0; j < i; j++) {
 					ISC_LIST_UNLINK(rdatalist.rdata,
@@ -2294,8 +2298,6 @@ load_raw(dns_loadctx_t *lctx) {
 
 				/* Rewind the buffer and continue */
 				isc_buffer_clear(&target);
-				isc_buffer_add(&target, consumed_name);
-				isc_buffer_forward(&target, consumed_name);
 
 				rdcount -= i;
 
@@ -2315,11 +2317,20 @@ load_raw(dns_loadctx_t *lctx) {
 			if (result != ISC_R_SUCCESS)
 				goto cleanup;
 			isc_buffer_setactive(&target, (unsigned int)rdlen);
-			isc_buffer_activeregion(&target, &r);
-			isc_buffer_forward(&target, (unsigned int)rdlen);
-			dns_rdata_fromregion(&rdata[i], rdatalist.rdclass,
-					     rdatalist.type, &r);
-
+			/*
+			 * It is safe to have the source active region and
+			 * the target available region be the same if
+			 * decompression is disabled (see dctx above) and we
+			 * are not downcasing names (options == 0).
+			 */
+			isc_buffer_init(&buf, isc_buffer_current(&target),
+					(unsigned int)rdlen);
+			result = dns_rdata_fromwire(&rdata[i],
+						    rdatalist.rdclass,
+						    rdatalist.type, &target,
+						    &dctx, 0, &buf);
+			if (result != ISC_R_SUCCESS)
+				goto cleanup;
 			ISC_LIST_APPEND(rdatalist.rdata, &rdata[i], link);
 		}
 
@@ -2336,7 +2347,7 @@ load_raw(dns_loadctx_t *lctx) {
 		ISC_LIST_APPEND(head, &rdatalist, link);
 
 		/* Commit this RRset.  rdatalist will be unlinked. */
-		result = commit(callbacks, lctx, &head, &name, NULL, 0);
+		result = commit(callbacks, lctx, &head, name, NULL, 0);
 
 		for (i = 0; i < rdcount; i++) {
 			ISC_LIST_UNLINK(rdatalist.rdata, &rdata[i], link);
diff --git a/lib/dns/masterdump.c b/lib/dns/masterdump.c
index 0c17c1a..a10e6f2 100644
--- a/lib/dns/masterdump.c
+++ b/lib/dns/masterdump.c
@@ -1545,7 +1545,8 @@ dns_master_dumptostream2(isc_mem_t *mctx, dns_db_t *db,
 }
 
 static isc_result_t
-opentmp(isc_mem_t *mctx, const char *file, char **tempp, FILE **fp) {
+opentmp(isc_mem_t *mctx, dns_masterformat_t format, const char *file,
+	char **tempp, FILE **fp) {
 	FILE *f = NULL;
 	isc_result_t result;
 	char *tempname = NULL;
@@ -1560,7 +1561,10 @@ opentmp(isc_mem_t *mctx, const char *file, char **tempp, FILE **fp) {
 	if (result != ISC_R_SUCCESS)
 		goto cleanup;
 
-	result = isc_file_openunique(tempname, &f);
+	if (format == dns_masterformat_text)
+		result = isc_file_openunique(tempname, &f);
+	else
+		result = isc_file_bopenunique(tempname, &f);
 	if (result != ISC_R_SUCCESS) {
 		isc_log_write(dns_lctx, ISC_LOGCATEGORY_GENERAL,
 			      DNS_LOGMODULE_MASTERDUMP, ISC_LOG_ERROR,
@@ -1604,7 +1608,7 @@ dns_master_dumpinc2(isc_mem_t *mctx, dns_db_t *db, dns_dbversion_t *version,
 	if (file == NULL)
 		return (ISC_R_NOMEMORY);
 
-	result = opentmp(mctx, filename, &tempname, &f);
+	result = opentmp(mctx, format, filename, &tempname, &f);
 	if (result != ISC_R_SUCCESS)
 		goto cleanup;
 
@@ -1658,7 +1662,7 @@ dns_master_dump2(isc_mem_t *mctx, dns_db_t *db, dns_dbversion_t *version,
 	char *tempname;
 	dns_dumpctx_t *dctx = NULL;
 
-	result = opentmp(mctx, filename, &tempname, &f);
+	result = opentmp(mctx, format, filename, &tempname, &f);
 	if (result != ISC_R_SUCCESS)
 		return (result);
 
diff --git a/lib/dns/openssl_link.c b/lib/dns/openssl_link.c
index 59626f2..d186761 100644
--- a/lib/dns/openssl_link.c
+++ b/lib/dns/openssl_link.c
@@ -45,6 +45,8 @@
 #include <isc/thread.h>
 #include <isc/util.h>
 
+#include <dns/log.h>
+
 #include <dst/result.h>
 
 #include "dst_internal.h"
@@ -172,6 +174,8 @@ dst__openssl_init(const char *engine) {
 	CRYPTO_set_locking_callback(lock_callback);
 	CRYPTO_set_id_callback(id_callback);
 
+	ERR_load_crypto_strings();
+
 	rm = mem_alloc(sizeof(RAND_METHOD));
 	if (rm == NULL) {
 		result = ISC_R_NOMEMORY;
@@ -285,7 +289,7 @@ dst__openssl_destroy() {
 isc_result_t
 dst__openssl_toresult(isc_result_t fallback) {
 	isc_result_t result = fallback;
-	int err = ERR_get_error();
+	unsigned long err = ERR_get_error();
 
 	switch (ERR_GET_REASON(err)) {
 	case ERR_R_MALLOC_FAILURE:
@@ -298,6 +302,40 @@ dst__openssl_toresult(isc_result_t fallback) {
 	return (result);
 }
 
+isc_result_t
+dst__openssl_toresult2(const char *funcname, isc_result_t fallback) {
+	isc_result_t result = fallback;
+	unsigned long err = ERR_peek_error();
+	const char *file, *data;
+	int line, flags;
+	char buf[256];
+
+	switch (ERR_GET_REASON(err)) {
+	case ERR_R_MALLOC_FAILURE:
+		result = ISC_R_NOMEMORY;
+		goto done;
+	default:
+		break;
+	}
+	isc_log_write(dns_lctx, DNS_LOGCATEGORY_GENERAL,
+		      DNS_LOGMODULE_CRYPTO, ISC_LOG_WARNING,
+		      "%s failed", funcname);
+	for (;;) {
+		err = ERR_get_error_line_data(&file, &line, &data, &flags);
+		if (err == 0U)
+			goto done;
+		ERR_error_string_n(err, buf, sizeof(buf));
+		isc_log_write(dns_lctx, DNS_LOGCATEGORY_GENERAL,
+			      DNS_LOGMODULE_CRYPTO, ISC_LOG_INFO,
+			      "%s:%s:%d:%s", buf, file, line,
+			      (flags & ERR_TXT_STRING) ? data : "");
+	}
+
+    done:
+	ERR_clear_error();
+	return (result);
+}
+
 #if defined(USE_ENGINE)
 ENGINE *
 dst__openssl_getengine(const char *engine) {
diff --git a/lib/dns/openssldh_link.c b/lib/dns/openssldh_link.c
index 71b4b12..9fe9bb5 100644
--- a/lib/dns/openssldh_link.c
+++ b/lib/dns/openssldh_link.c
@@ -1,5 +1,5 @@
 /*
- * Portions Copyright (C) 2004-2009, 2011  Internet Systems Consortium, Inc. ("ISC")
+ * Portions Copyright (C) 2004-2009, 2011, 2012  Internet Systems Consortium, Inc. ("ISC")
  * Portions Copyright (C) 1999-2002  Internet Software Consortium.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
@@ -94,7 +94,8 @@ openssldh_computesecret(const dst_key_t *pub, const dst_key_t *priv,
 		return (ISC_R_NOSPACE);
 	ret = DH_compute_key(r.base, dhpub->pub_key, dhpriv);
 	if (ret == 0)
-		return (dst__openssl_toresult(DST_R_COMPUTESECRETFAILURE));
+		return (dst__openssl_toresult2("DH_compute_key",
+					       DST_R_COMPUTESECRETFAILURE));
 	isc_buffer_add(secret, len);
 	return (ISC_R_SUCCESS);
 }
@@ -204,7 +205,7 @@ openssldh_generate(dst_key_t *key, int generator, void (*callback)(int)) {
 #if OPENSSL_VERSION_NUMBER > 0x00908000L
 		dh = DH_new();
 		if (dh == NULL)
-			return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
+			return (dst__openssl_toresult(ISC_R_NOMEMORY));
 
 		if (callback == NULL) {
 			BN_GENCB_set_old(&cb, NULL, NULL);
@@ -216,7 +217,9 @@ openssldh_generate(dst_key_t *key, int generator, void (*callback)(int)) {
 		if (!DH_generate_parameters_ex(dh, key->key_size, generator,
 					       &cb)) {
 			DH_free(dh);
-			return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
+			return (dst__openssl_toresult2(
+					"DH_generate_parameters_ex",
+					DST_R_OPENSSLFAILURE));
 		}
 #else
 		dh = DH_generate_parameters(key->key_size, generator,
@@ -225,11 +228,13 @@ openssldh_generate(dst_key_t *key, int generator, void (*callback)(int)) {
 	}
 
 	if (dh == NULL)
-		return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
+		return (dst__openssl_toresult2("DH_generate_parameters",
+					       DST_R_OPENSSLFAILURE));
 
 	if (DH_generate_key(dh) == 0) {
 		DH_free(dh);
-		return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
+		return (dst__openssl_toresult2("DH_generate_key",
+					       DST_R_OPENSSLFAILURE));
 	}
 	dh->flags &= ~DH_FLAG_CACHE_MONT_P;
 
@@ -460,6 +465,7 @@ openssldh_tofile(const dst_key_t *key, const char *directory) {
 
 	dh = key->keydata.dh;
 
+	memset(bufs, 0, sizeof(bufs));
 	for (i = 0; i < 4; i++) {
 		bufs[i] = isc_mem_get(key->mctx, BN_num_bytes(dh->p));
 		if (bufs[i] == NULL) {
diff --git a/lib/dns/openssldsa_link.c b/lib/dns/openssldsa_link.c
index 39c0615..e2cf8cd 100644
--- a/lib/dns/openssldsa_link.c
+++ b/lib/dns/openssldsa_link.c
@@ -168,7 +168,8 @@ openssldsa_sign(dst_context_t *dctx, isc_buffer_t *sig) {
 	if (!EVP_SignFinal(evp_md_ctx, sigbuf, &siglen, pkey)) {
 		EVP_PKEY_free(pkey);
 		free(sigbuf);
-		return (ISC_R_FAILURE);
+		return (dst__openssl_toresult2("EVP_SignFinal",
+					       ISC_R_FAILURE));
 	}
 	INSIST(EVP_PKEY_size(pkey) >= (int) siglen);
 	EVP_PKEY_free(pkey);
@@ -181,23 +182,26 @@ openssldsa_sign(dst_context_t *dctx, isc_buffer_t *sig) {
 	sb = sigbuf;
 	if (d2i_DSA_SIG(&dsasig, &sb, (long) siglen) == NULL) {
 		free(sigbuf);
-		return (ISC_R_FAILURE);
+		return (dst__openssl_toresult2("d2i_DSA_SIG", ISC_R_FAILURE));
 	}
 	free(sigbuf);
 #elif 0
 	/* Only use EVP for the Digest */
 	if (!EVP_DigestFinal_ex(evp_md_ctx, digest, &siglen)) {
-		return (ISC_R_FAILURE);
+		return (dst__openssl_toresult2("EVP_DigestFinal_ex",
+					       ISC_R_FAILURE));
 	}
 	dsasig = DSA_do_sign(digest, ISC_SHA1_DIGESTLENGTH, dsa);
 	if (dsasig == NULL)
-		return (dst__openssl_toresult(DST_R_SIGNFAILURE));
+		return (dst__openssl_toresult2("DSA_do_sign",
+					       DST_R_SIGNFAILURE));
 #else
 	isc_sha1_final(sha1ctx, digest);
 
 	dsasig = DSA_do_sign(digest, ISC_SHA1_DIGESTLENGTH, dsa);
 	if (dsasig == NULL)
-		return (dst__openssl_toresult(DST_R_SIGNFAILURE));
+		return (dst__openssl_toresult2("DSA_do_sign",
+					       DST_R_SIGNFAILURE));
 #endif
 	*r.base++ = (key->key_size - 512)/64;
 	BN_bn2bin_fixed(dsasig->r, r.base, ISC_SHA1_DIGESTLENGTH);
@@ -276,10 +280,15 @@ openssldsa_verify(dst_context_t *dctx, const isc_region_t *sig) {
 	status = DSA_do_verify(digest, ISC_SHA1_DIGESTLENGTH, dsasig, dsa);
 #endif
 	DSA_SIG_free(dsasig);
-	if (status != 1)
+	switch (status) {
+	case 1:
+		return (ISC_R_SUCCESS);
+	case 0:
 		return (dst__openssl_toresult(DST_R_VERIFYFAILURE));
-
-	return (ISC_R_SUCCESS);
+	default:
+		return (dst__openssl_toresult2("DSA_do_verify",
+					       DST_R_VERIFYFAILURE));
+	}
 }
 
 static isc_boolean_t
@@ -370,19 +379,22 @@ openssldsa_generate(dst_key_t *key, int unused, void (*callback)(int)) {
 					&cb))
 	{
 		DSA_free(dsa);
-		return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
+		return (dst__openssl_toresult2("DSA_generate_parameters_ex",
+					       DST_R_OPENSSLFAILURE));
 	}
 #else
 	dsa = DSA_generate_parameters(key->key_size, rand_array,
 				      ISC_SHA1_DIGESTLENGTH, NULL, NULL,
 				      NULL, NULL);
 	if (dsa == NULL)
-		return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
+		return (dst__openssl_toresult2("DSA_generate_parameters",
+					       DST_R_OPENSSLFAILURE));
 #endif
 
 	if (DSA_generate_key(dsa) == 0) {
 		DSA_free(dsa);
-		return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
+		return (dst__openssl_toresult2("DSA_generate_key",
+					       DST_R_OPENSSLFAILURE));
 	}
 	dsa->flags &= ~DSA_FLAG_CACHE_MONT_P;
 
diff --git a/lib/dns/opensslecdsa_link.c b/lib/dns/opensslecdsa_link.c
new file mode 100644
index 0000000..e6c9b67
--- /dev/null
+++ b/lib/dns/opensslecdsa_link.c
@@ -0,0 +1,596 @@
+/*
+ * Copyright (C) 2012  Internet Systems Consortium, Inc. ("ISC")
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
+ * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+ * AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
+ * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+ * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
+ * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+ * PERFORMANCE OF THIS SOFTWARE.
+ */
+
+/* $Id$ */
+
+#include <config.h>
+
+#ifdef HAVE_OPENSSL_ECDSA
+
+#if !defined(HAVE_EVP_SHA256) || !defined(HAVE_EVP_SHA384)
+#error "ECDSA without EVP for SHA2?"
+#endif
+
+#include <isc/entropy.h>
+#include <isc/mem.h>
+#include <isc/sha2.h>
+#include <isc/string.h>
+#include <isc/util.h>
+
+#include <dns/keyvalues.h>
+#include <dst/result.h>
+
+#include "dst_internal.h"
+#include "dst_openssl.h"
+#include "dst_parse.h"
+
+#include <openssl/err.h>
+#include <openssl/objects.h>
+#include <openssl/ecdsa.h>
+#include <openssl/bn.h>
+
+#ifndef NID_X9_62_prime256v1
+#error "P-256 group is not known (NID_X9_62_prime256v1)"
+#endif
+#ifndef NID_secp384r1
+#error "P-384 group is not known (NID_secp384r1)"
+#endif
+
+#define DST_RET(a) {ret = a; goto err;}
+
+static isc_result_t opensslecdsa_todns(const dst_key_t *key,
+				       isc_buffer_t *data);
+
+static isc_result_t
+opensslecdsa_createctx(dst_key_t *key, dst_context_t *dctx) {
+	EVP_MD_CTX *evp_md_ctx;
+	const EVP_MD *type = NULL;
+
+	UNUSED(key);
+	REQUIRE(dctx->key->key_alg == DST_ALG_ECDSA256 ||
+		dctx->key->key_alg == DST_ALG_ECDSA384);
+
+	evp_md_ctx = EVP_MD_CTX_create();
+	if (evp_md_ctx == NULL)
+		return (ISC_R_NOMEMORY);
+	if (dctx->key->key_alg == DST_ALG_ECDSA256)
+		type = EVP_sha256();
+	else
+		type = EVP_sha384();
+
+	if (!EVP_DigestInit_ex(evp_md_ctx, type, NULL)) {
+		EVP_MD_CTX_destroy(evp_md_ctx);
+		return (dst__openssl_toresult2("EVP_DigestInit_ex",
+					       ISC_R_FAILURE));
+	}
+
+	dctx->ctxdata.evp_md_ctx = evp_md_ctx;
+
+	return (ISC_R_SUCCESS);
+}
+
+static void
+opensslecdsa_destroyctx(dst_context_t *dctx) {
+	EVP_MD_CTX *evp_md_ctx = dctx->ctxdata.evp_md_ctx;
+
+	REQUIRE(dctx->key->key_alg == DST_ALG_ECDSA256 ||
+		dctx->key->key_alg == DST_ALG_ECDSA384);
+
+	if (evp_md_ctx != NULL) {
+		EVP_MD_CTX_destroy(evp_md_ctx);
+		dctx->ctxdata.evp_md_ctx = NULL;
+	}
+}
+
+static isc_result_t
+opensslecdsa_adddata(dst_context_t *dctx, const isc_region_t *data) {
+	EVP_MD_CTX *evp_md_ctx = dctx->ctxdata.evp_md_ctx;
+
+	REQUIRE(dctx->key->key_alg == DST_ALG_ECDSA256 ||
+		dctx->key->key_alg == DST_ALG_ECDSA384);
+
+	if (!EVP_DigestUpdate(evp_md_ctx, data->base, data->length))
+		return (dst__openssl_toresult2("EVP_DigestUpdate",
+					       ISC_R_FAILURE));
+
+	return (ISC_R_SUCCESS);
+}
+
+static int
+BN_bn2bin_fixed(BIGNUM *bn, unsigned char *buf, int size) {
+	int bytes = size - BN_num_bytes(bn);
+
+	while (bytes-- > 0)
+		*buf++ = 0;
+	BN_bn2bin(bn, buf);
+	return (size);
+}
+
+static isc_result_t
+opensslecdsa_sign(dst_context_t *dctx, isc_buffer_t *sig) {
+	isc_result_t ret;
+	dst_key_t *key = dctx->key;
+	isc_region_t r;
+	ECDSA_SIG *ecdsasig;
+	EVP_MD_CTX *evp_md_ctx = dctx->ctxdata.evp_md_ctx;
+	EVP_PKEY *pkey = key->keydata.pkey;
+	EC_KEY *eckey = EVP_PKEY_get1_EC_KEY(pkey);
+	unsigned int dgstlen, siglen;
+	unsigned char digest[EVP_MAX_MD_SIZE];
+
+	REQUIRE(key->key_alg == DST_ALG_ECDSA256 ||
+		key->key_alg == DST_ALG_ECDSA384);
+
+	if (eckey == NULL)
+		return (ISC_R_FAILURE);
+
+	if (key->key_alg == DST_ALG_ECDSA256)
+		siglen = DNS_SIG_ECDSA256SIZE;
+	else
+		siglen = DNS_SIG_ECDSA384SIZE;
+
+	isc_buffer_availableregion(sig, &r);
+	if (r.length < siglen)
+		DST_RET(ISC_R_NOSPACE);
+
+	if (!EVP_DigestFinal(evp_md_ctx, digest, &dgstlen))
+		DST_RET(dst__openssl_toresult2("EVP_DigestFinal",
+					       ISC_R_FAILURE));
+
+	ecdsasig = ECDSA_do_sign(digest, dgstlen, eckey);
+	if (ecdsasig == NULL)
+		DST_RET(dst__openssl_toresult2("ECDSA_do_sign",
+					       DST_R_SIGNFAILURE));
+	BN_bn2bin_fixed(ecdsasig->r, r.base, siglen / 2);
+	r.base += siglen / 2;
+	BN_bn2bin_fixed(ecdsasig->s, r.base, siglen / 2);
+	r.base += siglen / 2;
+	ECDSA_SIG_free(ecdsasig);
+	isc_buffer_add(sig, siglen);
+	ret = ISC_R_SUCCESS;
+
+ err:
+	if (eckey != NULL)
+		EC_KEY_free(eckey);
+	return (ret);
+}
+
+static isc_result_t
+opensslecdsa_verify(dst_context_t *dctx, const isc_region_t *sig) {
+	isc_result_t ret;
+	dst_key_t *key = dctx->key;
+	int status;
+	unsigned char *cp = sig->base;
+	ECDSA_SIG *ecdsasig = NULL;
+	EVP_MD_CTX *evp_md_ctx = dctx->ctxdata.evp_md_ctx;
+	EVP_PKEY *pkey = key->keydata.pkey;
+	EC_KEY *eckey = EVP_PKEY_get1_EC_KEY(pkey);
+	unsigned int dgstlen, siglen;
+	unsigned char digest[EVP_MAX_MD_SIZE];
+
+	REQUIRE(key->key_alg == DST_ALG_ECDSA256 ||
+		key->key_alg == DST_ALG_ECDSA384);
+
+	if (eckey == NULL)
+		return (ISC_R_FAILURE);
+
+	if (key->key_alg == DST_ALG_ECDSA256)
+		siglen = DNS_SIG_ECDSA256SIZE;
+	else
+		siglen = DNS_SIG_ECDSA384SIZE;
+
+	if (sig->length != siglen)
+		return (DST_R_VERIFYFAILURE);
+
+	if (!EVP_DigestFinal_ex(evp_md_ctx, digest, &dgstlen))
+		DST_RET (dst__openssl_toresult2("EVP_DigestFinal_ex",
+						ISC_R_FAILURE));
+
+	ecdsasig = ECDSA_SIG_new();
+	if (ecdsasig == NULL)
+		DST_RET (ISC_R_NOMEMORY);
+	ecdsasig->r = BN_bin2bn(cp, siglen / 2, NULL);
+	cp += siglen / 2;
+	ecdsasig->s = BN_bin2bn(cp, siglen / 2, NULL);
+	/* cp += siglen / 2; */
+
+	status = ECDSA_do_verify(digest, dgstlen, ecdsasig, eckey);
+	switch (status) {
+	case 1:
+		ret = ISC_R_SUCCESS;
+		break;
+	case 0:
+		ret = dst__openssl_toresult(DST_R_VERIFYFAILURE);
+		break;
+	default:
+		ret = dst__openssl_toresult2("ECDSA_do_verify",
+					     DST_R_VERIFYFAILURE);
+		break;
+	}
+
+ err:
+	if (ecdsasig != NULL)
+		ECDSA_SIG_free(ecdsasig);
+	if (eckey != NULL)
+		EC_KEY_free(eckey);
+	return (ret);
+}
+
+static isc_boolean_t
+opensslecdsa_compare(const dst_key_t *key1, const dst_key_t *key2) {
+	isc_boolean_t ret;
+	int status;
+	EVP_PKEY *pkey1 = key1->keydata.pkey;
+	EVP_PKEY *pkey2 = key2->keydata.pkey;
+	EC_KEY *eckey1 = NULL;
+	EC_KEY *eckey2 = NULL;
+	const BIGNUM *priv1, *priv2;
+
+	if (pkey1 == NULL && pkey2 == NULL)
+		return (ISC_TRUE);
+	else if (pkey1 == NULL || pkey2 == NULL)
+		return (ISC_FALSE);
+
+	eckey1 = EVP_PKEY_get1_EC_KEY(pkey1);
+	eckey2 = EVP_PKEY_get1_EC_KEY(pkey2);
+	if (eckey1 == NULL && eckey2 == NULL) {
+		DST_RET (ISC_TRUE);
+	} else if (eckey1 == NULL || eckey2 == NULL)
+		DST_RET (ISC_FALSE);
+
+	status = EVP_PKEY_cmp(pkey1, pkey2);
+	if (status != 1)
+		DST_RET (ISC_FALSE);
+
+	priv1 = EC_KEY_get0_private_key(eckey1);
+	priv2 = EC_KEY_get0_private_key(eckey2);
+	if (priv1 != NULL || priv2 != NULL) {
+		if (priv1 == NULL || priv2 == NULL)
+			DST_RET (ISC_FALSE);
+		if (BN_cmp(priv1, priv2) != 0)
+			DST_RET (ISC_FALSE);
+	}
+	ret = ISC_TRUE;
+
+ err:
+	if (eckey1 != NULL)
+		EC_KEY_free(eckey1);
+	if (eckey2 != NULL)
+		EC_KEY_free(eckey2);
+	return (ret);
+}
+
+static isc_result_t
+opensslecdsa_generate(dst_key_t *key, int unused, void (*callback)(int)) {
+	isc_result_t ret;
+	EVP_PKEY *pkey;
+	EC_KEY *eckey = NULL;
+	int group_nid;
+
+	REQUIRE(key->key_alg == DST_ALG_ECDSA256 ||
+		key->key_alg == DST_ALG_ECDSA384);
+	UNUSED(unused);
+	UNUSED(callback);
+
+	if (key->key_alg == DST_ALG_ECDSA256)
+		group_nid = NID_X9_62_prime256v1;
+	else
+		group_nid = NID_secp384r1;
+
+	eckey = EC_KEY_new_by_curve_name(group_nid);
+	if (eckey == NULL)
+		return (dst__openssl_toresult2("EC_KEY_new_by_curve_name",
+					       DST_R_OPENSSLFAILURE));
+
+	if (EC_KEY_generate_key(eckey) != 1)
+		DST_RET (dst__openssl_toresult2("EC_KEY_generate_key",
+						DST_R_OPENSSLFAILURE));
+
+	pkey = EVP_PKEY_new();
+	if (pkey == NULL)
+		DST_RET (ISC_R_NOMEMORY);
+	if (!EVP_PKEY_set1_EC_KEY(pkey, eckey)) {
+		EVP_PKEY_free(pkey);
+		DST_RET (ISC_R_FAILURE);
+	}
+	key->keydata.pkey = pkey;
+	ret = ISC_R_SUCCESS;
+
+ err:
+	if (eckey != NULL)
+		EC_KEY_free(eckey);
+	return (ret);
+}
+
+static isc_boolean_t
+opensslecdsa_isprivate(const dst_key_t *key) {
+	isc_boolean_t ret;
+	EVP_PKEY *pkey = key->keydata.pkey;
+	EC_KEY *eckey = EVP_PKEY_get1_EC_KEY(pkey);
+
+	ret = ISC_TF(eckey != NULL && EC_KEY_get0_private_key(eckey) != NULL);
+	if (eckey != NULL)
+		EC_KEY_free(eckey);
+	return (ret);
+}
+
+static void
+opensslecdsa_destroy(dst_key_t *key) {
+	EVP_PKEY *pkey = key->keydata.pkey;
+
+	EVP_PKEY_free(pkey);
+	key->keydata.pkey = NULL;
+}
+
+static isc_result_t
+opensslecdsa_todns(const dst_key_t *key, isc_buffer_t *data) {
+	isc_result_t ret;
+	EVP_PKEY *pkey;
+	EC_KEY *eckey = NULL;
+	isc_region_t r;
+	int len;
+	unsigned char *cp;
+	unsigned char buf[DNS_KEY_ECDSA384SIZE + 1];
+
+	REQUIRE(key->keydata.pkey != NULL);
+
+	pkey = key->keydata.pkey;
+	eckey = EVP_PKEY_get1_EC_KEY(pkey);
+	if (eckey == NULL)
+		return (dst__openssl_toresult(ISC_R_FAILURE));
+	len = i2o_ECPublicKey(eckey, NULL);
+	/* skip form */
+	len--;
+
+	isc_buffer_availableregion(data, &r);
+	if (r.length < (unsigned int) len)
+		DST_RET (ISC_R_NOSPACE);
+	cp = buf;
+	if (!i2o_ECPublicKey(eckey, &cp))
+		DST_RET (dst__openssl_toresult(ISC_R_FAILURE));
+	memcpy(r.base, buf + 1, len);
+	isc_buffer_add(data, len);
+	ret = ISC_R_SUCCESS;
+
+ err:
+	if (eckey != NULL)
+		EC_KEY_free(eckey);
+	return (ret);
+}
+
+static isc_result_t
+opensslecdsa_fromdns(dst_key_t *key, isc_buffer_t *data) {
+	isc_result_t ret;
+	EVP_PKEY *pkey;
+	EC_KEY *eckey = NULL;
+	isc_region_t r;
+	int group_nid;
+	unsigned int len;
+	const unsigned char *cp;
+	unsigned char buf[DNS_KEY_ECDSA384SIZE + 1];
+
+	REQUIRE(key->key_alg == DST_ALG_ECDSA256 ||
+		key->key_alg == DST_ALG_ECDSA384);
+
+	if (key->key_alg == DST_ALG_ECDSA256) {
+		len = DNS_KEY_ECDSA256SIZE;
+		group_nid = NID_X9_62_prime256v1;
+	} else {
+		len = DNS_KEY_ECDSA384SIZE;
+		group_nid = NID_secp384r1;
+	}
+
+	isc_buffer_remainingregion(data, &r);
+	if (r.length == 0)
+		return (ISC_R_SUCCESS);
+	if (r.length < len)
+		return (DST_R_INVALIDPUBLICKEY);
+
+	eckey = EC_KEY_new_by_curve_name(group_nid);
+	if (eckey == NULL)
+		return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
+
+	buf[0] = POINT_CONVERSION_UNCOMPRESSED;
+	memcpy(buf + 1, r.base, len);
+	cp = buf;
+	if (o2i_ECPublicKey(&eckey,
+			    (const unsigned char **) &cp,
+			    (long) len + 1) == NULL)
+		DST_RET (dst__openssl_toresult(DST_R_INVALIDPUBLICKEY));
+	if (EC_KEY_check_key(eckey) != 1)
+		DST_RET (dst__openssl_toresult(DST_R_INVALIDPUBLICKEY));
+
+	pkey = EVP_PKEY_new();
+	if (pkey == NULL)
+		DST_RET (ISC_R_NOMEMORY);
+	if (!EVP_PKEY_set1_EC_KEY(pkey, eckey)) {
+		EVP_PKEY_free(pkey);
+		DST_RET (dst__openssl_toresult(ISC_R_FAILURE));
+	}
+
+	isc_buffer_forward(data, len);
+	key->keydata.pkey = pkey;
+	ret = ISC_R_SUCCESS;
+
+ err:
+	if (eckey != NULL)
+		EC_KEY_free(eckey);
+	return (ret);
+}
+
+static isc_result_t
+opensslecdsa_tofile(const dst_key_t *key, const char *directory) {
+	isc_result_t ret;
+	EVP_PKEY *pkey;
+	EC_KEY *eckey = NULL;
+	const BIGNUM *privkey;
+	dst_private_t priv;
+	unsigned char *buf = NULL;
+
+	if (key->keydata.pkey == NULL)
+		return (DST_R_NULLKEY);
+
+	pkey = key->keydata.pkey;
+	eckey = EVP_PKEY_get1_EC_KEY(pkey);
+	if (eckey == NULL)
+		return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
+	privkey = EC_KEY_get0_private_key(eckey);
+	if (privkey == NULL)
+		DST_RET (ISC_R_FAILURE);
+
+	buf = isc_mem_get(key->mctx, BN_num_bytes(privkey));
+	if (buf == NULL)
+		DST_RET (ISC_R_NOMEMORY);
+
+	priv.elements[0].tag = TAG_ECDSA_PRIVATEKEY;
+	priv.elements[0].length = BN_num_bytes(privkey);
+	BN_bn2bin(privkey, buf);
+	priv.elements[0].data = buf;
+	priv.nelements = ECDSA_NTAGS;
+	ret = dst__privstruct_writefile(key, &priv, directory);
+
+ err:
+	if (eckey != NULL)
+		EC_KEY_free(eckey);
+	if (buf != NULL)
+		isc_mem_put(key->mctx, buf, BN_num_bytes(privkey));
+	return (ret);
+}
+
+static isc_result_t
+ecdsa_check(EC_KEY *eckey, dst_key_t *pub)
+{
+	isc_result_t ret = ISC_R_FAILURE;
+	EVP_PKEY *pkey;
+	EC_KEY *pubeckey = NULL;
+	const EC_POINT *pubkey;
+
+	if (pub == NULL)
+		return (ISC_R_SUCCESS);
+	pkey = pub->keydata.pkey;
+	if (pkey == NULL)
+		return (ISC_R_SUCCESS);
+	pubeckey = EVP_PKEY_get1_EC_KEY(pkey);
+	if (pubeckey == NULL)
+		return (ISC_R_SUCCESS);
+	pubkey = EC_KEY_get0_public_key(pubeckey);
+	if (pubkey == NULL)
+		DST_RET (ISC_R_SUCCESS);
+	if (EC_KEY_set_public_key(eckey, pubkey) != 1)
+		DST_RET (ISC_R_SUCCESS);
+	if (EC_KEY_check_key(eckey) == 1)
+		DST_RET (ISC_R_SUCCESS);
+
+ err:
+	if (pubeckey != NULL)
+		EC_KEY_free(pubeckey);
+	return (ret);
+}
+
+static isc_result_t
+opensslecdsa_parse(dst_key_t *key, isc_lex_t *lexer, dst_key_t *pub) {
+	dst_private_t priv;
+	isc_result_t ret;
+	EVP_PKEY *pkey;
+	EC_KEY *eckey = NULL;
+	BIGNUM *privkey;
+	int group_nid;
+	isc_mem_t *mctx = key->mctx;
+
+	REQUIRE(key->key_alg == DST_ALG_ECDSA256 ||
+		key->key_alg == DST_ALG_ECDSA384);
+
+	if (key->key_alg == DST_ALG_ECDSA256)
+		group_nid = NID_X9_62_prime256v1;
+	else
+		group_nid = NID_secp384r1;
+
+	eckey = EC_KEY_new_by_curve_name(group_nid);
+	if (eckey == NULL)
+		return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
+
+	/* read private key file */
+	ret = dst__privstruct_parse(key, DST_ALG_ECDSA256, lexer, mctx, &priv);
+	if (ret != ISC_R_SUCCESS)
+		goto err;
+
+	privkey = BN_bin2bn(priv.elements[0].data,
+			    priv.elements[0].length, NULL);
+	if (privkey == NULL)
+		DST_RET(ISC_R_NOMEMORY);
+	if (!EC_KEY_set_private_key(eckey, privkey))
+		DST_RET(ISC_R_NOMEMORY);
+	if (ecdsa_check(eckey, pub) != ISC_R_SUCCESS)
+		DST_RET(DST_R_INVALIDPRIVATEKEY);
+	dst__privstruct_free(&priv, mctx);
+	memset(&priv, 0, sizeof(priv));
+
+	pkey = EVP_PKEY_new();
+	if (pkey == NULL)
+		DST_RET (ISC_R_NOMEMORY);
+	if (!EVP_PKEY_set1_EC_KEY(pkey, eckey)) {
+		EVP_PKEY_free(pkey);
+		DST_RET (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
+	}
+	key->keydata.pkey = pkey;
+	ret = ISC_R_SUCCESS;
+
+ err:
+	if (eckey != NULL)
+		EC_KEY_free(eckey);
+	dst__privstruct_free(&priv, mctx);
+	memset(&priv, 0, sizeof(priv));
+	return (ret);
+}
+
+static dst_func_t opensslecdsa_functions = {
+	opensslecdsa_createctx,
+	opensslecdsa_destroyctx,
+	opensslecdsa_adddata,
+	opensslecdsa_sign,
+	opensslecdsa_verify,
+	NULL, /*%< computesecret */
+	opensslecdsa_compare,
+	NULL, /*%< paramcompare */
+	opensslecdsa_generate,
+	opensslecdsa_isprivate,
+	opensslecdsa_destroy,
+	opensslecdsa_todns,
+	opensslecdsa_fromdns,
+	opensslecdsa_tofile,
+	opensslecdsa_parse,
+	NULL, /*%< cleanup */
+	NULL, /*%< fromlabel */
+	NULL, /*%< dump */
+	NULL, /*%< restore */
+};
+
+isc_result_t
+dst__opensslecdsa_init(dst_func_t **funcp) {
+	REQUIRE(funcp != NULL);
+	if (*funcp == NULL)
+		*funcp = &opensslecdsa_functions;
+	return (ISC_R_SUCCESS);
+}
+
+#else /* HAVE_OPENSSL_ECDSA */
+
+#include <isc/util.h>
+
+EMPTY_TRANSLATION_UNIT
+
+#endif /* HAVE_OPENSSL_ECDSA */
+/*! \file */
diff --git a/lib/dns/opensslgost_link.c b/lib/dns/opensslgost_link.c
index e92d50f..8a55a6b 100644
--- a/lib/dns/opensslgost_link.c
+++ b/lib/dns/opensslgost_link.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2010, 2011  Internet Systems Consortium, Inc. ("ISC")
+ * Copyright (C) 2010-2012  Internet Systems Consortium, Inc. ("ISC")
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -121,10 +121,15 @@ opensslgost_verify(dst_context_t *dctx, const isc_region_t *sig) {
 	EVP_PKEY *pkey = key->keydata.pkey;
 
 	status = EVP_VerifyFinal(evp_md_ctx, sig->base, sig->length, pkey);
-	if (status != 1)
+	switch (status) {
+	case 1:
+		return (ISC_R_SUCCESS);
+	case 0:
 		return (dst__openssl_toresult(DST_R_VERIFYFAILURE));
-
-	return (ISC_R_SUCCESS);
+	default:
+		return (dst__openssl_toresult2("EVP_VerifyFinal",
+					       DST_R_VERIFYFAILURE));
+	}
 }
 
 static isc_boolean_t
@@ -168,22 +173,27 @@ opensslgost_generate(dst_key_t *key, int unused, void (*callback)(int)) {
 		void (*fptr)(int);
 	} u;
 	EVP_PKEY *pkey = NULL;
+	isc_result_t ret;
 
 	UNUSED(unused);
 	ctx = EVP_PKEY_CTX_new_id(NID_id_GostR3410_2001, NULL);
 	if (ctx == NULL)
-		goto err;
+		DST_RET(dst__openssl_toresult2("EVP_PKEY_CTX_new_id",
+					       DST_R_OPENSSLFAILURE));
 	if (callback != NULL) {
 		u.fptr = callback;
 		EVP_PKEY_CTX_set_app_data(ctx, u.dptr);
 		EVP_PKEY_CTX_set_cb(ctx, &progress_cb);
 	}
 	if (EVP_PKEY_keygen_init(ctx) <= 0)
-		goto err;
+		DST_RET(dst__openssl_toresult2("EVP_PKEY_keygen_init",
+					       DST_R_OPENSSLFAILURE));
 	if (EVP_PKEY_CTX_ctrl_str(ctx, "paramset", "A") <= 0)
-		goto err;
+		DST_RET(dst__openssl_toresult2("EVP_PKEY_CTX_ctrl_str",
+					       DST_R_OPENSSLFAILURE));
 	if (EVP_PKEY_keygen(ctx, &pkey) <= 0)
-		goto err;
+		DST_RET(dst__openssl_toresult2("EVP_PKEY_keygen",
+					       DST_R_OPENSSLFAILURE));
 	key->keydata.pkey = pkey;
 	EVP_PKEY_CTX_free(ctx);
 	return (ISC_R_SUCCESS);
@@ -193,7 +203,7 @@ err:
 		EVP_PKEY_free(pkey);
 	if (ctx != NULL)
 		EVP_PKEY_CTX_free(ctx);
-	return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
+	return (ret);
 }
 
 static isc_boolean_t
@@ -267,7 +277,8 @@ opensslgost_fromdns(dst_key_t *key, isc_buffer_t *data) {
 
 	p = der;
 	if (d2i_PUBKEY(&pkey, &p, (long) sizeof(der)) == NULL)
-		return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
+		return (dst__openssl_toresult2("d2i_PUBKEY",
+					       DST_R_OPENSSLFAILURE));
 	key->keydata.pkey = pkey;
 
 	return (ISC_R_SUCCESS);
@@ -293,7 +304,8 @@ opensslgost_tofile(const dst_key_t *key, const char *directory) {
 
 	p = der;
 	if (i2d_PrivateKey(pkey, &p) != len) {
-		result = dst__openssl_toresult(DST_R_OPENSSLFAILURE);
+		result = dst__openssl_toresult2("i2d_PrivateKey",
+						DST_R_OPENSSLFAILURE);
 		goto fail;
 	}
 
@@ -328,7 +340,8 @@ opensslgost_parse(dst_key_t *key, isc_lex_t *lexer, dst_key_t *pub) {
 	p = priv.elements[0].data;
 	if (d2i_PrivateKey(NID_id_GostR3410_2001, &pkey, &p,
 			   (long) priv.elements[0].length) == NULL)
-		DST_RET(DST_R_INVALIDPRIVATEKEY);
+		DST_RET(dst__openssl_toresult2("d2i_PrivateKey",
+					       DST_R_INVALIDPRIVATEKEY));
 	key->keydata.pkey = pkey;
 	key->key_size = EVP_PKEY_bits(pkey);
 	dst__privstruct_free(&priv, mctx);
@@ -377,35 +390,47 @@ static dst_func_t opensslgost_functions = {
 
 isc_result_t
 dst__opensslgost_init(dst_func_t **funcp) {
+	isc_result_t ret;
+
 	REQUIRE(funcp != NULL);
 
 	/* check if the gost engine works properly */
 	e = ENGINE_by_id("gost");
 	if (e == NULL)
-		return (DST_R_OPENSSLFAILURE);
+		return (dst__openssl_toresult2("ENGINE_by_id",
+					       DST_R_OPENSSLFAILURE));
 	if (ENGINE_init(e) <= 0) {
 		ENGINE_free(e);
 		e = NULL;
-		return (DST_R_OPENSSLFAILURE);
+		return (dst__openssl_toresult2("ENGINE_init",
+					       DST_R_OPENSSLFAILURE));
 	}
 	/* better than to rely on digest_gost symbol */
 	opensslgost_digest = ENGINE_get_digest(e, NID_id_GostR3411_94);
+	if (opensslgost_digest == NULL)
+		DST_RET(dst__openssl_toresult2("ENGINE_get_digest",
+					       DST_R_OPENSSLFAILURE));
 	/* from openssl.cnf */
-	if ((opensslgost_digest == NULL) ||
-	    (ENGINE_register_pkey_asn1_meths(e) <= 0) ||
-	    (ENGINE_ctrl_cmd_string(e,
-				    "CRYPT_PARAMS",
-				    "id-Gost28147-89-CryptoPro-A-ParamSet",
-				    0) <= 0)) {
-		ENGINE_finish(e);
-		ENGINE_free(e);
-		e = NULL;
-		return (DST_R_OPENSSLFAILURE);
-	}
+	if (ENGINE_register_pkey_asn1_meths(e) <= 0)
+		DST_RET(dst__openssl_toresult2(
+				"ENGINE_register_pkey_asn1_meths",
+				DST_R_OPENSSLFAILURE));
+	if (ENGINE_ctrl_cmd_string(e,
+				   "CRYPT_PARAMS",
+				   "id-Gost28147-89-CryptoPro-A-ParamSet",
+				   0) <= 0)
+		DST_RET(dst__openssl_toresult2("ENGINE_ctrl_cmd_string",
+					       DST_R_OPENSSLFAILURE));
 
 	if (*funcp == NULL)
 		*funcp = &opensslgost_functions;
 	return (ISC_R_SUCCESS);
+
+ err:
+	ENGINE_finish(e);
+	ENGINE_free(e);
+	e = NULL;
+	return (ret);
 }
 
 #else /* HAVE_OPENSSL_GOST */
diff --git a/lib/dns/opensslrsa_link.c b/lib/dns/opensslrsa_link.c
index a245618..80c3f57 100644
--- a/lib/dns/opensslrsa_link.c
+++ b/lib/dns/opensslrsa_link.c
@@ -156,7 +156,8 @@ opensslrsa_createctx(dst_key_t *key, dst_context_t *dctx) {
 
 	if (!EVP_DigestInit_ex(evp_md_ctx, type, NULL)) {
 		EVP_MD_CTX_destroy(evp_md_ctx);
-		return (ISC_R_FAILURE);
+		return (dst__openssl_toresult2("EVP_DigestInit_ex",
+					       ISC_R_FAILURE));
 	}
 	dctx->ctxdata.evp_md_ctx = evp_md_ctx;
 #else
@@ -304,7 +305,8 @@ opensslrsa_adddata(dst_context_t *dctx, const isc_region_t *data) {
 
 #if USE_EVP
 	if (!EVP_DigestUpdate(evp_md_ctx, data->base, data->length)) {
-		return (ISC_R_FAILURE);
+		return (dst__openssl_toresult2("EVP_DigestUpdate",
+					       ISC_R_FAILURE));
 	}
 #else
 	switch (dctx->key->key_alg) {
@@ -374,10 +376,6 @@ opensslrsa_sign(dst_context_t *dctx, isc_buffer_t *sig) {
 	int status;
 	int type = 0;
 	unsigned int digestlen = 0;
-	char *message;
-	unsigned long err;
-	const char* file;
-	int line;
 #if OPENSSL_VERSION_NUMBER < 0x00908000L
 	unsigned int prefixlen = 0;
 	const unsigned char *prefix = NULL;
@@ -397,7 +395,8 @@ opensslrsa_sign(dst_context_t *dctx, isc_buffer_t *sig) {
 		return (ISC_R_NOSPACE);
 
 	if (!EVP_SignFinal(evp_md_ctx, r.base, &siglen, pkey)) {
-		return (ISC_R_FAILURE);
+		return (dst__openssl_toresult2("EVP_SignFinal",
+					       ISC_R_FAILURE));
 	}
 #else
 	if (r.length < (unsigned int) RSA_size(rsa))
@@ -489,13 +488,9 @@ opensslrsa_sign(dst_context_t *dctx, isc_buffer_t *sig) {
 	INSIST(type != 0);
 	status = RSA_sign(type, digest, digestlen, r.base, &siglen, rsa);
 #endif
-	if (status == 0) {
-		err = ERR_peek_error_line(&file, &line);
-		if (err != 0U) {
-			message = ERR_error_string(err, NULL);
-		}
-		return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
-	}
+	if (status == 0)
+		return (dst__openssl_toresult2("RSA_sign",
+					       DST_R_OPENSSLFAILURE));
 #endif
 
 	isc_buffer_add(sig, siglen);
@@ -615,7 +610,9 @@ opensslrsa_verify(dst_context_t *dctx, const isc_region_t *sig) {
 						    original, rsa,
 						    RSA_PKCS1_PADDING);
 			if (status <= 0)
-				return (DST_R_VERIFYFAILURE);
+				return (dst__openssl_toresult2(
+						"RSA_public_decrypt",
+						DST_R_VERIFYFAILURE));
 			if (status != (int)(prefixlen + digestlen))
 				return (DST_R_VERIFYFAILURE);
 			if (memcmp(original, prefix, prefixlen))
@@ -636,7 +633,8 @@ opensslrsa_verify(dst_context_t *dctx, const isc_region_t *sig) {
 #endif
 #endif
 	if (status != 1)
-		return (dst__openssl_toresult(DST_R_VERIFYFAILURE));
+		return (dst__openssl_toresult2("RSA_verify",
+					       DST_R_VERIFYFAILURE));
 
 	return (ISC_R_SUCCESS);
 }
@@ -727,6 +725,7 @@ progress_cb(int p, int n, BN_GENCB *cb)
 static isc_result_t
 opensslrsa_generate(dst_key_t *key, int exp, void (*callback)(int)) {
 #if OPENSSL_VERSION_NUMBER > 0x00908000L
+	isc_result_t ret = DST_R_OPENSSLFAILURE;
 	BN_GENCB cb;
 	union {
 		void *dptr;
@@ -776,6 +775,8 @@ opensslrsa_generate(dst_key_t *key, int exp, void (*callback)(int)) {
 #endif
 		return (ISC_R_SUCCESS);
 	}
+	ret = dst__openssl_toresult2("RSA_generate_key_ex",
+				     DST_R_OPENSSLFAILURE);
 
 err:
 #if USE_EVP
@@ -786,7 +787,7 @@ err:
 		BN_free(e);
 	if (rsa != NULL)
 		RSA_free(rsa);
-	return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
+	return (dst__openssl_toresult(ret));
 #else
 	RSA *rsa;
 	unsigned long e;
@@ -810,7 +811,8 @@ err:
 #if USE_EVP
 		EVP_PKEY_free(pkey);
 #endif
-		return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
+		return (dst__openssl_toresult2("RSA_generate_key",
+					       DST_R_OPENSSLFAILURE));
 	}
 	SET_FLAGS(rsa);
 #if USE_EVP
@@ -1009,6 +1011,7 @@ opensslrsa_tofile(const dst_key_t *key, const char *directory) {
 	rsa = key->keydata.rsa;
 #endif
 
+	memset(bufs, 0, sizeof(bufs));
 	for (i = 0; i < 8; i++) {
 		bufs[i] = isc_mem_get(key->mctx, BN_num_bytes(rsa->n));
 		if (bufs[i] == NULL) {
@@ -1162,7 +1165,7 @@ opensslrsa_parse(dst_key_t *key, isc_lex_t *lexer, dst_key_t *pub) {
 	/* read private key file */
 	ret = dst__privstruct_parse(key, DST_ALG_RSA, lexer, mctx, &priv);
 	if (ret != ISC_R_SUCCESS)
-		return (ret);
+		goto err;
 
 	for (i = 0; i < priv.nelements; i++) {
 		switch (priv.elements[i].tag) {
@@ -1188,10 +1191,10 @@ opensslrsa_parse(dst_key_t *key, isc_lex_t *lexer, dst_key_t *pub) {
 		if (e == NULL)
 			DST_RET(DST_R_NOENGINE);
 		pkey = ENGINE_load_private_key(e, label, NULL, NULL);
-		if (pkey == NULL) {
-			/* ERR_print_errors_fp(stderr); */
-			DST_RET(ISC_R_NOTFOUND);
-		}
+		if (pkey == NULL)
+			DST_RET(dst__openssl_toresult2(
+					"ENGINE_load_private_key",
+					ISC_R_NOTFOUND));
 		key->engine = isc_mem_strdup(key->mctx, engine);
 		if (key->engine == NULL)
 			DST_RET(ISC_R_NOMEMORY);
@@ -1336,7 +1339,8 @@ opensslrsa_fromlabel(dst_key_t *key, const char *engine, const char *label,
 	}
 	pkey = ENGINE_load_private_key(e, label, NULL, NULL);
 	if (pkey == NULL)
-		DST_RET(ISC_R_NOTFOUND);
+		DST_RET(dst__openssl_toresult2("ENGINE_load_private_key",
+					       ISC_R_NOTFOUND));
 	if (engine != NULL) {
 		key->engine = isc_mem_strdup(key->mctx, engine);
 		if (key->engine == NULL)
diff --git a/lib/dns/rbtdb.c b/lib/dns/rbtdb.c
index a2523ba..ef721b8 100644
--- a/lib/dns/rbtdb.c
+++ b/lib/dns/rbtdb.c
@@ -113,6 +113,8 @@ typedef isc_uint32_t                    rbtdb_rdatatype_t;
 		RBTDB_RDATATYPE_VALUE(dns_rdatatype_rrsig, dns_rdatatype_cname)
 #define RBTDB_RDATATYPE_SIGDNAME \
 		RBTDB_RDATATYPE_VALUE(dns_rdatatype_rrsig, dns_rdatatype_dname)
+#define RBTDB_RDATATYPE_SIGDDS \
+		RBTDB_RDATATYPE_VALUE(dns_rdatatype_rrsig, dns_rdatatype_ds)
 #define RBTDB_RDATATYPE_NCACHEANY \
 		RBTDB_RDATATYPE_VALUE(0, dns_rdatatype_any)
 
@@ -4572,7 +4574,7 @@ get_rpz_enabled(dns_db_t *db, dns_rpz_st_t *st)
  * configured earlier than this policy zone and does not have a higher
  * precedence type.
  */
-static isc_result_t
+static void
 rpz_findips(dns_rpz_zone_t *rpz, dns_rpz_type_t rpz_type,
 	    dns_zone_t *zone, dns_db_t *db, dns_dbversion_t *version,
 	    dns_rdataset_t *ardataset, dns_rpz_st_t *st,
@@ -4597,7 +4599,7 @@ rpz_findips(dns_rpz_zone_t *rpz, dns_rpz_type_t rpz_type,
 
 	if (rbtdb->rpz_cidr == NULL) {
 		RWUNLOCK(&rbtdb->tree_lock, isc_rwlocktype_read);
-		return (ISC_R_UNEXPECTED);
+		return;
 	}
 
 	dns_fixedname_init(&selfnamef);
@@ -4659,7 +4661,7 @@ rpz_findips(dns_rpz_zone_t *rpz, dns_rpz_type_t rpz_type,
 			dns_name_format(qname, namebuf, sizeof(namebuf));
 			isc_log_write(dns_lctx, DNS_LOGCATEGORY_RPZ,
 				      DNS_LOGMODULE_RBTDB, DNS_RPZ_ERROR_LEVEL,
-				      "rpz_findips findnode(%s): %s",
+				      "rpz_findips findnode(%s) failed: %s",
 				      namebuf, isc_result_totext(result));
 			continue;
 		}
@@ -4680,7 +4682,8 @@ rpz_findips(dns_rpz_zone_t *rpz, dns_rpz_type_t rpz_type,
 			if (zrdataset.type != dns_rdatatype_cname) {
 				rpz_policy = DNS_RPZ_POLICY_RECORD;
 			} else {
-				rpz_policy = dns_rpz_decode_cname(&zrdataset,
+				rpz_policy = dns_rpz_decode_cname(rpz,
+								  &zrdataset,
 								  selfname);
 				if (rpz_policy == DNS_RPZ_POLICY_RECORD ||
 				    rpz_policy == DNS_RPZ_POLICY_WILDCNAME)
@@ -4738,7 +4741,7 @@ rpz_findips(dns_rpz_zone_t *rpz, dns_rpz_type_t rpz_type,
 		st->m.type = rpz_type;
 		st->m.prefix = prefix;
 		st->m.policy = rpz_policy;
-		st->m.ttl = ttl;
+		st->m.ttl = ISC_MIN(ttl, rpz->max_policy_ttl);
 		st->m.result = result;
 		dns_name_copy(qname, st->qname, NULL);
 		if ((rpz_policy == DNS_RPZ_POLICY_RECORD ||
@@ -4755,7 +4758,6 @@ rpz_findips(dns_rpz_zone_t *rpz, dns_rpz_type_t rpz_type,
 	}
 
 	RWUNLOCK(&rbtdb->tree_lock, isc_rwlocktype_read);
-	return (ISC_R_SUCCESS);
 }
 #endif
 
@@ -5914,13 +5916,12 @@ add(dns_rbtdb_t *rbtdb, dns_rbtnode_t *rbtnode, rbtdb_version_t *rbtversion,
 	negtype = 0;
 	if (rbtversion == NULL && !newheader_nx) {
 		rdtype = RBTDB_RDATATYPE_BASE(newheader->type);
+		covers = RBTDB_RDATATYPE_EXT(newheader->type);
+		sigtype = RBTDB_RDATATYPE_VALUE(dns_rdatatype_rrsig, covers);
 		if (NEGATIVE(newheader)) {
 			/*
 			 * We're adding a negative cache entry.
 			 */
-			covers = RBTDB_RDATATYPE_EXT(newheader->type);
-			sigtype = RBTDB_RDATATYPE_VALUE(dns_rdatatype_rrsig,
-							covers);
 			for (topheader = rbtnode->data;
 			     topheader != NULL;
 			     topheader = topheader->next) {
@@ -5953,14 +5954,20 @@ add(dns_rbtdb_t *rbtdb, dns_rbtnode_t *rbtnode, rbtdb_version_t *rbtversion,
 			 * We're adding something that isn't a
 			 * negative cache entry.  Look for an extant
 			 * non-stale NXDOMAIN/NODATA(QTYPE=ANY) negative
-			 * cache entry.
+			 * cache entry.  If we're adding an RRSIG, also
+			 * check for an extant non-stale NODATA ncache
+			 * entry which covers the same type as the RRSIG.
 			 */
 			for (topheader = rbtnode->data;
 			     topheader != NULL;
 			     topheader = topheader->next) {
-				if (topheader->type ==
-				    RBTDB_RDATATYPE_NCACHEANY)
-					break;
+				if ((topheader->type ==
+					RBTDB_RDATATYPE_NCACHEANY) ||
+					(newheader->type == sigtype &&
+					topheader->type ==
+					RBTDB_RDATATYPE_VALUE(0, covers))) {
+						break;
+					}
 			}
 			if (topheader != NULL && EXISTS(topheader) &&
 			    topheader->rdh_ttl > now) {
@@ -5983,7 +5990,7 @@ add(dns_rbtdb_t *rbtdb, dns_rbtnode_t *rbtnode, rbtdb_version_t *rbtversion,
 				}
 				/*
 				 * The new rdataset is better.  Expire the
-				 * NXDOMAIN/NODATA(QTYPE=ANY).
+				 * ncache entry.
 				 */
 				set_ttl(rbtdb, topheader, 0);
 				topheader->attributes |= RDATASET_ATTR_STALE;
@@ -6145,7 +6152,9 @@ add(dns_rbtdb_t *rbtdb, dns_rbtnode_t *rbtnode, rbtdb_version_t *rbtversion,
 		}
 		if (IS_CACHE(rbtdb) && header->rdh_ttl > now &&
 		    (header->type == dns_rdatatype_a ||
-		     header->type == dns_rdatatype_aaaa) &&
+		     header->type == dns_rdatatype_aaaa ||
+		     header->type == dns_rdatatype_ds ||
+		     header->type == RBTDB_RDATATYPE_SIGDDS) &&
 		    !header_nx && !newheader_nx &&
 		    header->trust >= newheader->trust &&
 		    dns_rdataslab_equal((unsigned char *)header,
diff --git a/lib/dns/rcode.c b/lib/dns/rcode.c
index 09f6d83..0b7fe8c 100644
--- a/lib/dns/rcode.c
+++ b/lib/dns/rcode.c
@@ -108,6 +108,8 @@
 	{ DNS_KEYALG_RSASHA256, "RSASHA256", 0 }, \
 	{ DNS_KEYALG_RSASHA512, "RSASHA512", 0 }, \
 	{ DNS_KEYALG_ECCGOST, "ECCGOST", 0 }, \
+	{ DNS_KEYALG_ECDSA256, "ECDSAP256SHA256", 0 }, \
+	{ DNS_KEYALG_ECDSA384, "ECDSAP384SHA384", 0 }, \
 	{ DNS_KEYALG_INDIRECT, "INDIRECT", 0 }, \
 	{ DNS_KEYALG_PRIVATEDNS, "PRIVATEDNS", 0 }, \
 	{ DNS_KEYALG_PRIVATEOID, "PRIVATEOID", 0 }, \
diff --git a/lib/dns/rdata.c b/lib/dns/rdata.c
index d200f1b..60890e0 100644
--- a/lib/dns/rdata.c
+++ b/lib/dns/rdata.c
@@ -436,6 +436,8 @@ dns_rdata_fromwire(dns_rdata_t *rdata, dns_rdataclass_t rdclass,
 		REQUIRE(DNS_RDATA_INITIALIZED(rdata));
 		REQUIRE(DNS_RDATA_VALIDFLAGS(rdata));
 	}
+	REQUIRE(source != NULL);
+	REQUIRE(target != NULL);
 
 	if (type == 0)
 		return (DNS_R_FORMERR);
@@ -536,13 +538,11 @@ rdata_validate(isc_buffer_t *src, isc_buffer_t *dest, dns_rdataclass_t rdclass,
 	    dns_rdatatype_t type)
 {
 	dns_decompress_t dctx;
-	dns_rdata_t rdata = DNS_RDATA_INIT;
 	isc_result_t result;
 
 	dns_decompress_init(&dctx, -1, DNS_DECOMPRESS_NONE);
 	isc_buffer_setactive(src, isc_buffer_usedlength(src));
-	result = dns_rdata_fromwire(&rdata, rdclass, type, src,
-				    &dctx, 0, dest);
+	result = dns_rdata_fromwire(NULL, rdclass, type, src, &dctx, 0, dest);
 	dns_decompress_invalidate(&dctx);
 
 	return (result);
@@ -1167,7 +1167,8 @@ txt_fromwire(isc_buffer_t *source, isc_buffer_t *target) {
 	if (n > tregion.length)
 		return (ISC_R_NOSPACE);
 
-	memcpy(tregion.base, sregion.base, n);
+	if (tregion.base != sregion.base)
+		memcpy(tregion.base, sregion.base, n);
 	isc_buffer_forward(source, n);
 	isc_buffer_add(target, n);
 	return (ISC_R_SUCCESS);
@@ -1341,7 +1342,8 @@ mem_tobuffer(isc_buffer_t *target, void *base, unsigned int length) {
 	isc_buffer_availableregion(target, &tr);
 	if (length > tr.length)
 		return (ISC_R_NOSPACE);
-	memcpy(tr.base, base, length);
+	if (tr.base != base)
+		memcpy(tr.base, base, length);
 	isc_buffer_add(target, length);
 	return (ISC_R_SUCCESS);
 }
diff --git a/lib/dns/rdata/generic/dlv_32769.c b/lib/dns/rdata/generic/dlv_32769.c
index f447e7c..4dbcb1e 100644
--- a/lib/dns/rdata/generic/dlv_32769.c
+++ b/lib/dns/rdata/generic/dlv_32769.c
@@ -84,6 +84,9 @@ fromtext_dlv(ARGS_FROMTEXT) {
 	case DNS_DSDIGEST_GOST:
 		length = ISC_GOST_DIGESTLENGTH;
 		break;
+	case DNS_DSDIGEST_SHA384:
+		length = ISC_SHA384_DIGESTLENGTH;
+		break;
 	default:
 		length = -1;
 		break;
@@ -162,7 +165,9 @@ fromwire_dlv(ARGS_FROMWIRE) {
 	    (sr.base[3] == DNS_DSDIGEST_SHA256 &&
 	     sr.length < 4 + ISC_SHA256_DIGESTLENGTH) ||
 	    (sr.base[3] == DNS_DSDIGEST_GOST &&
-	     sr.length < 4 + ISC_GOST_DIGESTLENGTH))
+	     sr.length < 4 + ISC_GOST_DIGESTLENGTH) ||
+	    (sr.base[3] == DNS_DSDIGEST_SHA384 &&
+	     sr.length < 4 + ISC_SHA384_DIGESTLENGTH))
 		return (ISC_R_UNEXPECTEDEND);
 
 	/*
@@ -176,6 +181,8 @@ fromwire_dlv(ARGS_FROMWIRE) {
 		sr.length = 4 + ISC_SHA256_DIGESTLENGTH;
 	else if (sr.base[3] == DNS_DSDIGEST_GOST)
 		sr.length = 4 + ISC_GOST_DIGESTLENGTH;
+	else if (sr.base[3] == DNS_DSDIGEST_SHA384)
+		sr.length = 4 + ISC_SHA384_DIGESTLENGTH;
 
 	isc_buffer_forward(source, sr.length);
 	return (mem_tobuffer(target, sr.base, sr.length));
@@ -228,6 +235,9 @@ fromstruct_dlv(ARGS_FROMSTRUCT) {
 	case DNS_DSDIGEST_GOST:
 		REQUIRE(dlv->length == ISC_GOST_DIGESTLENGTH);
 		break;
+	case DNS_DSDIGEST_SHA384:
+		REQUIRE(dlv->length == ISC_SHA384_DIGESTLENGTH);
+		break;
 	}
 
 	UNUSED(type);
diff --git a/lib/dns/rdata/generic/ds_43.c b/lib/dns/rdata/generic/ds_43.c
index cece442..20bac85 100644
--- a/lib/dns/rdata/generic/ds_43.c
+++ b/lib/dns/rdata/generic/ds_43.c
@@ -84,6 +84,9 @@ fromtext_ds(ARGS_FROMTEXT) {
 	case DNS_DSDIGEST_GOST:
 		length = ISC_GOST_DIGESTLENGTH;
 		break;
+	case DNS_DSDIGEST_SHA384:
+		length = ISC_SHA384_DIGESTLENGTH;
+		break;
 	default:
 		length = -1;
 		break;
@@ -162,7 +165,9 @@ fromwire_ds(ARGS_FROMWIRE) {
 	    (sr.base[3] == DNS_DSDIGEST_SHA256 &&
 	     sr.length < 4 + ISC_SHA256_DIGESTLENGTH) ||
 	    (sr.base[3] == DNS_DSDIGEST_GOST &&
-	     sr.length < 4 + ISC_GOST_DIGESTLENGTH))
+	     sr.length < 4 + ISC_GOST_DIGESTLENGTH) ||
+	    (sr.base[3] == DNS_DSDIGEST_SHA384 &&
+	     sr.length < 4 + ISC_SHA384_DIGESTLENGTH))
 		return (ISC_R_UNEXPECTEDEND);
 
 	/*
@@ -176,6 +181,8 @@ fromwire_ds(ARGS_FROMWIRE) {
 		sr.length = 4 + ISC_SHA256_DIGESTLENGTH;
 	else if (sr.base[3] == DNS_DSDIGEST_GOST)
 		sr.length = 4 + ISC_GOST_DIGESTLENGTH;
+	else if (sr.base[3] == DNS_DSDIGEST_SHA384)
+		sr.length = 4 + ISC_SHA384_DIGESTLENGTH;
 
 	isc_buffer_forward(source, sr.length);
 	return (mem_tobuffer(target, sr.base, sr.length));
@@ -228,6 +235,9 @@ fromstruct_ds(ARGS_FROMSTRUCT) {
 	case DNS_DSDIGEST_GOST:
 		REQUIRE(ds->length == ISC_GOST_DIGESTLENGTH);
 		break;
+	case DNS_DSDIGEST_SHA384:
+		REQUIRE(ds->length == ISC_SHA384_DIGESTLENGTH);
+		break;
 	}
 
 	UNUSED(type);
diff --git a/lib/dns/rdataset.c b/lib/dns/rdataset.c
index 8c86549..026d771 100644
--- a/lib/dns/rdataset.c
+++ b/lib/dns/rdataset.c
@@ -26,6 +26,7 @@
 #include <isc/buffer.h>
 #include <isc/mem.h>
 #include <isc/random.h>
+#include <isc/serial.h>
 #include <isc/util.h>
 
 #include <dns/name.h>
@@ -772,3 +773,30 @@ dns_rdataset_expire(dns_rdataset_t *rdataset) {
 	if (rdataset->methods->expire != NULL)
 		(rdataset->methods->expire)(rdataset);
 }
+
+void
+dns_rdataset_trimttl(dns_rdataset_t *rdataset, dns_rdataset_t *sigrdataset,
+		     dns_rdata_rrsig_t *rrsig, isc_stdtime_t now,
+		     isc_boolean_t acceptexpired)
+{
+	isc_uint32_t ttl = 0;
+
+	REQUIRE(DNS_RDATASET_VALID(rdataset));
+	REQUIRE(DNS_RDATASET_VALID(sigrdataset));
+	REQUIRE(rrsig != NULL);
+
+	/*
+	 * If we accept expired RRsets keep them for no more than 120 seconds.
+	 */
+	if (acceptexpired &&
+	    (isc_serial_le(rrsig->timeexpire, ((now + 120) & 0xffffffff)) ||
+	     isc_serial_le(rrsig->timeexpire, now)))
+		ttl = 120;
+	else if (isc_serial_ge(rrsig->timeexpire, now))
+		ttl = rrsig->timeexpire - now;
+
+	ttl = ISC_MIN(ISC_MIN(rdataset->ttl, sigrdataset->ttl),
+		      ISC_MIN(rrsig->originalttl, ttl));
+	rdataset->ttl = ttl;
+	sigrdataset->ttl = ttl;
+}
diff --git a/lib/dns/resolver.c b/lib/dns/resolver.c
index 4c8b144..503f1d2 100644
--- a/lib/dns/resolver.c
+++ b/lib/dns/resolver.c
@@ -105,8 +105,21 @@
 #define QTRACE(m)
 #endif
 
+#define US_PER_SEC 1000000U
+/*
+ * The maximum time we will wait for a single query.
+ */
+#define MAX_SINGLE_QUERY_TIMEOUT 9U
+#define MAX_SINGLE_QUERY_TIMEOUT_US (MAX_SINGLE_QUERY_TIMEOUT*US_PER_SEC)
+
+/*
+ * We need to allow a individual query time to complete / timeout.
+ */
+#define MINIMUM_QUERY_TIMEOUT (MAX_SINGLE_QUERY_TIMEOUT + 1U)
+
+/* The default time in seconds for the whole query to live. */
 #ifndef DEFAULT_QUERY_TIMEOUT
-#define DEFAULT_QUERY_TIMEOUT 30  /* The default time in seconds for the whole query to live. */
+#define DEFAULT_QUERY_TIMEOUT MINIMUM_QUERY_TIMEOUT
 #endif
 
 #ifndef MAXIMUM_QUERY_TIMEOUT
@@ -821,8 +834,8 @@ fctx_cancelquery(resquery_t **queryp, dns_dispatchevent_t **deventp,
 			 */
 			INSIST(no_response);
 			rtt = query->addrinfo->srtt + 200000;
-			if (rtt > 10000000)
-				rtt = 10000000;
+			if (rtt > MAX_SINGLE_QUERY_TIMEOUT_US)
+				rtt = MAX_SINGLE_QUERY_TIMEOUT_US;
 			/*
 			 * Replace the current RTT with our value.
 			 */
@@ -1336,12 +1349,18 @@ fctx_setretryinterval(fetchctx_t *fctx, unsigned int rtt) {
 		us = (800000 << (fctx->restarts - 2));
 
 	/*
-	 * Double the round-trip time.
+	 * Add a fudge factor to the expected rtt based on the current
+	 * estimate.
 	 */
-	rtt *= 2;
+	if (rtt < 50000)
+		rtt += 50000;
+	else if (rtt < 100000)
+		rtt += 100000;
+	else
+		rtt += 200000;
 
 	/*
-	 * Always wait for at least the doubled round-trip time.
+	 * Always wait for at least the expected rtt.
 	 */
 	if (us < rtt)
 		us = rtt;
@@ -1349,11 +1368,11 @@ fctx_setretryinterval(fetchctx_t *fctx, unsigned int rtt) {
 	/*
 	 * But don't ever wait for more than 10 seconds.
 	 */
-	if (us > 10000000)
-		us = 10000000;
+	if (us > MAX_SINGLE_QUERY_TIMEOUT_US)
+		us = MAX_SINGLE_QUERY_TIMEOUT_US;
 
-	seconds = us / 1000000;
-	us -= seconds * 1000000;
+	seconds = us / US_PER_SEC;
+	us -= seconds * US_PER_SEC;
 	isc_interval_set(&fctx->interval, seconds, us * 1000);
 }
 
@@ -1375,6 +1394,11 @@ fctx_query(fetchctx_t *fctx, dns_adbaddrinfo_t *addrinfo,
 	task = res->buckets[fctx->bucketnum].task;
 
 	srtt = addrinfo->srtt;
+
+	/*
+	 * A forwarder needs to make multiple queries. Give it at least
+	 * a second to do these in.
+	 */
 	if (ISFORWARDER(addrinfo) && srtt < 1000000)
 		srtt = 1000000;
 
@@ -8211,8 +8235,8 @@ dns_resolver_logfetch(dns_fetch_t *fetch, isc_log_t *lctx,
 			      "timeout:%u,lame:%u,neterr:%u,badresp:%u,"
 			      "adberr:%u,findfail:%u,valfail:%u]",
 			      __FILE__, fctx->exitline, fctx->info,
-			      fctx->duration / 1000000,
-			      fctx->duration % 1000000,
+			      fctx->duration / US_PER_SEC,
+			      fctx->duration % US_PER_SEC,
 			      isc_result_totext(fctx->result),
 			      isc_result_totext(fctx->vresult), domainbuf,
 			      fctx->referrals, fctx->restarts,
@@ -8818,6 +8842,8 @@ dns_resolver_settimeout(dns_resolver_t *resolver, unsigned int seconds) {
 		seconds = DEFAULT_QUERY_TIMEOUT;
 	if (seconds > MAXIMUM_QUERY_TIMEOUT)
 		seconds = MAXIMUM_QUERY_TIMEOUT;
+	if (seconds < MINIMUM_QUERY_TIMEOUT)
+		seconds =  MINIMUM_QUERY_TIMEOUT;
 
 	resolver->query_timeout = seconds;
 }
diff --git a/lib/dns/rpz.c b/lib/dns/rpz.c
index e3f4989..7865859 100644
--- a/lib/dns/rpz.c
+++ b/lib/dns/rpz.c
@@ -118,9 +118,9 @@ struct dns_rpz_cidr {
 	isc_mem_t		*mctx;
 	isc_boolean_t		have_nsdname;	/* zone has NSDNAME record */
 	dns_rpz_cidr_node_t	*root;
-	dns_name_t		ip_name;	/* RPZ_IP_ZONE.LOCALHOST. */
-	dns_name_t		nsip_name;      /* RPZ_NSIP_ZONE.LOCALHOST. */
-	dns_name_t		nsdname_name;	/* RPZ_NSDNAME_ZONE.LOCALHOST */
+	dns_name_t		ip_name;	/* RPZ_IP_ZONE.origin. */
+	dns_name_t		nsip_name;      /* RPZ_NSIP_ZONE.origin. */
+	dns_name_t		nsdname_name;	/* RPZ_NSDNAME_ZONE.origin */
 };
 
 static isc_boolean_t		have_rpz_zones = ISC_FALSE;
@@ -183,7 +183,7 @@ dns_rpz_policy2str(dns_rpz_policy_t policy) {
 		str = "NODATA";
 		break;
 	case DNS_RPZ_POLICY_RECORD:
-		str = "records";
+		str = "Local-Data";
 		break;
 	case DNS_RPZ_POLICY_CNAME:
 	case DNS_RPZ_POLICY_WILDCNAME:
@@ -255,6 +255,8 @@ dns_rpz_view_destroy(dns_view_t *view) {
 		ISC_LIST_UNLINK(view->rpz_zones, zone, link);
 		if (dns_name_dynamic(&zone->origin))
 			dns_name_free(&zone->origin, view->mctx);
+		if (dns_name_dynamic(&zone->passthru))
+			dns_name_free(&zone->passthru, view->mctx);
 		if (dns_name_dynamic(&zone->nsdname))
 			dns_name_free(&zone->nsdname, view->mctx);
 		if (dns_name_dynamic(&zone->cname))
@@ -427,14 +429,16 @@ new_node(dns_rpz_cidr_t *cidr, const dns_rpz_cidr_key_t *ip,
 }
 
 static void
-badname(int level, dns_name_t *name, const char *comment) {
+badname(int level, dns_name_t *name, const char *str1, const char *str2) {
 	char printname[DNS_NAME_FORMATSIZE];
 
-	if (isc_log_wouldlog(dns_lctx, level)) {
+	if (level < DNS_RPZ_DEBUG_QUIET
+	    && isc_log_wouldlog(dns_lctx, level)) {
 		dns_name_format(name, printname, sizeof(printname));
 		isc_log_write(dns_lctx, DNS_LOGCATEGORY_RPZ,
 			      DNS_LOGMODULE_RBTDB, level,
-			      "invalid rpz \"%s\"%s", printname, comment);
+			      "invalid rpz IP address \"%s\"%s%s",
+			      printname, str1, str2);
 	}
 }
 
@@ -566,11 +570,11 @@ name2ipkey(dns_rpz_cidr_t *cidr, int level, dns_name_t *src_name,
 	   dns_rpz_type_t type, dns_rpz_cidr_key_t *tgt_ip,
 	   dns_rpz_cidr_bits_t *tgt_prefix)
 {
-	isc_buffer_t buffer;
-	unsigned char data[DNS_NAME_MAXWIRE+1];
+	isc_result_t result;
 	dns_fixedname_t fname;
-	dns_name_t *name;
-	const char *cp, *end;
+	dns_name_t *ipname;
+	char ipstr[DNS_NAME_FORMATSIZE];
+	const char *prefix_str, *cp, *end;
 	char *cp2;
 	int ip_labels;
 	dns_rpz_cidr_bits_t bits;
@@ -585,37 +589,43 @@ name2ipkey(dns_rpz_cidr_t *cidr, int level, dns_name_t *src_name,
 	ip_labels -= dns_name_countlabels(&cidr->ip_name);
 	ip_labels--;
 	if (ip_labels < 1) {
-		badname(level, src_name, ", too short");
+		badname(level, src_name, "; too short", "");
 		return (ISC_R_FAILURE);
 	}
 
 	/*
-	 * Get text for the IP address without RPZ_x_ZONE.rpz.LOCALHOST.
+	 * Get text for the IP address
 	 */
 	dns_fixedname_init(&fname);
-	name = dns_fixedname_name(&fname);
+	ipname = dns_fixedname_name(&fname);
 	dns_name_split(src_name, dns_name_countlabels(&cidr->ip_name),
-		       name, NULL);
-	isc_buffer_init(&buffer, data, sizeof(data));
-	dns_name_totext(name, ISC_TRUE, &buffer);
-	isc_buffer_putuint8(&buffer, '\0');
-	cp = isc_buffer_base(&buffer);
-
-	prefix = strtoul(cp, &cp2, 10);
-	if (prefix < 1U || prefix > 128U || *cp2 != '.') {
-		badname(level, src_name, ", bad prefix length");
+		       ipname, NULL);
+	dns_name_format(ipname, ipstr, sizeof(ipstr));
+	end = &ipstr[strlen(ipstr)+1];
+	prefix_str = ipstr;
+
+	prefix = strtoul(prefix_str, &cp2, 10);
+	if (*cp2 != '.') {
+		badname(level, src_name,
+			"; invalid leading prefix length", "");
+		return (ISC_R_FAILURE);
+	}
+	*cp2 = '\0';
+	if (prefix < 1U || prefix > 128U) {
+		badname(level, src_name,
+			"; invalid prefix length of ", prefix_str);
 		return (ISC_R_FAILURE);
 	}
 	cp = cp2+1;
 
-	end = isc_buffer_used(&buffer);
 	if (ip_labels == 4 && !strchr(cp, 'z')) {
 		/*
 		 * Convert an IPv4 address
 		 * from the form "prefix.w.z.y.x"
 		 */
 		if (prefix > 32U) {
-			badname(level, src_name, "; bad IPv4 prefix length");
+			badname(level, src_name,
+				"; invalid IPv4 prefix length of ", prefix_str);
 			return (ISC_R_FAILURE);
 		}
 		prefix += 96;
@@ -627,7 +637,10 @@ name2ipkey(dns_rpz_cidr_t *cidr, int level, dns_name_t *src_name,
 		for (i = 0; i < 32; i += 8) {
 			l = strtoul(cp, &cp2, 10);
 			if (l > 255U || (*cp2 != '.' && *cp2 != '\0')) {
-				badname(level, src_name, "; bad IPv4 address");
+				if (*cp2 == '.')
+					*cp2 = '\0';
+				badname(level, src_name,
+					"; invalid IPv4 octet ", cp);
 				return (ISC_R_FAILURE);
 			}
 			tgt_ip->w[3] |= l << i;
@@ -654,7 +667,10 @@ name2ipkey(dns_rpz_cidr_t *cidr, int level, dns_name_t *src_name,
 				l = strtoul(cp, &cp2, 16);
 				if (l > 0xffffu ||
 				    (*cp2 != '.' && *cp2 != '\0')) {
-					badname(level, src_name, "");
+					if (*cp2 == '.')
+					    *cp2 = '\0';
+					badname(level, src_name,
+						"; invalid IPv6 word ", cp);
 					return (ISC_R_FAILURE);
 				}
 				if ((i & 1) == 0)
@@ -667,7 +683,7 @@ name2ipkey(dns_rpz_cidr_t *cidr, int level, dns_name_t *src_name,
 		}
 	}
 	if (cp != end) {
-		badname(level, src_name, "");
+		badname(level, src_name, "", "");
 		return (ISC_R_FAILURE);
 	}
 
@@ -681,7 +697,8 @@ name2ipkey(dns_rpz_cidr_t *cidr, int level, dns_name_t *src_name,
 		i = bits % DNS_RPZ_CIDR_WORD_BITS;
 		aword = tgt_ip->w[bits / DNS_RPZ_CIDR_WORD_BITS];
 		if ((aword & ~DNS_RPZ_WORD_MASK(i)) != 0) {
-			badname(level, src_name, "; wrong prefix length");
+			badname(level, src_name,
+				"; too small prefix length of ", prefix_str);
 			return (ISC_R_FAILURE);
 		}
 		bits -= i;
@@ -689,13 +706,13 @@ name2ipkey(dns_rpz_cidr_t *cidr, int level, dns_name_t *src_name,
 	}
 
 	/*
-	 * Convert the IPv6 address back to a canonical policy domain name
+	 * Convert the address back to a canonical policy domain name
 	 * to ensure that it is in canonical form.
 	 */
-	if (ISC_R_SUCCESS != ip2name(cidr, tgt_ip, (dns_rpz_cidr_bits_t)prefix,
-				     type, NULL, name) ||
-	    !dns_name_equal(src_name, name)) {
-		badname(level, src_name, "; not canonical");
+	result = ip2name(cidr, tgt_ip, (dns_rpz_cidr_bits_t) prefix,
+			 type, NULL, ipname);
+	if (result != ISC_R_SUCCESS || !dns_name_equal(src_name, ipname)) {
+		badname(level, src_name, "; not canonical", "");
 		return (ISC_R_FAILURE);
 	}
 
@@ -934,6 +951,7 @@ search(dns_rpz_cidr_t *cidr, const dns_rpz_cidr_key_t *tgt_ip,
  */
 void
 dns_rpz_cidr_addip(dns_rpz_cidr_t *cidr, dns_name_t *name) {
+	isc_result_t result;
 	dns_rpz_cidr_key_t tgt_ip;
 	dns_rpz_cidr_bits_t tgt_prefix;
 	dns_rpz_type_t type;
@@ -956,19 +974,22 @@ dns_rpz_cidr_addip(dns_rpz_cidr_t *cidr, dns_name_t *name) {
 	case DNS_RPZ_TYPE_BAD:
 		return;
 	}
-	if (ISC_R_SUCCESS != name2ipkey(cidr, DNS_RPZ_ERROR_LEVEL, name,
-					type, &tgt_ip, &tgt_prefix))
+	result = name2ipkey(cidr, DNS_RPZ_ERROR_LEVEL, name,
+			    type, &tgt_ip, &tgt_prefix);
+	if (result != ISC_R_SUCCESS)
 		return;
 
-	if (ISC_R_EXISTS == search(cidr, &tgt_ip, tgt_prefix, type,
-				   ISC_TRUE, NULL) &&
-	    isc_log_wouldlog(dns_lctx, DNS_RPZ_ERROR_LEVEL)) {
+	result = search(cidr, &tgt_ip, tgt_prefix, type, ISC_TRUE, NULL);
+	if (result == ISC_R_EXISTS &&
+	    isc_log_wouldlog(dns_lctx, DNS_RPZ_ERROR_LEVEL))
+	{
 		char printname[DNS_NAME_FORMATSIZE];
 
 		dns_name_format(name, printname, sizeof(printname));
 		isc_log_write(dns_lctx, DNS_LOGCATEGORY_RPZ,
 			      DNS_LOGMODULE_RBTDB, DNS_RPZ_ERROR_LEVEL,
-			      "duplicate rpz name \"%s\"", printname);
+			      "rpz add failed; \"%s\" is a duplicate name",
+			      printname);
 	}
 }
 
@@ -978,6 +999,7 @@ dns_rpz_cidr_addip(dns_rpz_cidr_t *cidr, dns_name_t *name) {
  */
 void
 dns_rpz_cidr_deleteip(dns_rpz_cidr_t *cidr, dns_name_t *name) {
+	isc_result_t result;
 	dns_rpz_cidr_key_t tgt_ip;
 	dns_rpz_cidr_bits_t tgt_prefix;
 	dns_rpz_type_t type;
@@ -1010,19 +1032,14 @@ dns_rpz_cidr_deleteip(dns_rpz_cidr_t *cidr, dns_name_t *name) {
 	/*
 	 * Do not get excited about the deletion of interior rbt nodes.
 	 */
-	if (ISC_R_SUCCESS != name2ipkey(cidr, DNS_RPZ_DEBUG_LEVEL3, name,
-					type, &tgt_ip, &tgt_prefix))
+	result = name2ipkey(cidr, DNS_RPZ_DEBUG_QUIET, name,
+			    type, &tgt_ip, &tgt_prefix);
+	if (result != ISC_R_SUCCESS)
 		return;
-	if (ISC_R_SUCCESS != search(cidr, &tgt_ip, tgt_prefix, type,
-				    ISC_FALSE, &tgt)) {
-		if (isc_log_wouldlog(dns_lctx, DNS_RPZ_ERROR_LEVEL)) {
-			char printname[DNS_NAME_FORMATSIZE];
-
-			dns_name_format(name, printname, sizeof(printname));
-			isc_log_write(dns_lctx, DNS_LOGCATEGORY_RPZ,
-				      DNS_LOGMODULE_RBTDB, DNS_RPZ_ERROR_LEVEL,
-				      "missing rpz node \"%s\"", printname);
-		}
+
+	result = search(cidr, &tgt_ip, tgt_prefix, type, ISC_FALSE, &tgt);
+	if (result != ISC_R_SUCCESS) {
+		badname(DNS_RPZ_ERROR_LEVEL, name, "; missing rpz node", "");
 		return;
 	}
 
@@ -1135,7 +1152,9 @@ dns_rpz_cidr_find(dns_rpz_cidr_t *cidr, const isc_netaddr_t *netaddr,
  * Translate CNAME rdata to a QNAME response policy action.
  */
 dns_rpz_policy_t
-dns_rpz_decode_cname(dns_rdataset_t *rdataset, dns_name_t *selfname) {
+dns_rpz_decode_cname(dns_rpz_zone_t *rpz, dns_rdataset_t *rdataset,
+		     dns_name_t *selfname)
+{
 	dns_rdata_t rdata = DNS_RDATA_INIT;
 	dns_rdata_cname_t cname;
 	isc_result_t result;
@@ -1171,7 +1190,13 @@ dns_rpz_decode_cname(dns_rdataset_t *rdataset, dns_name_t *selfname) {
 	}
 
 	/*
-	 * 128.1.0.127.rpz-ip CNAME  128.1.0.0.127. means "do not rewrite"
+	 * CNAME PASSTHRU.origin means "do not rewrite.
+	 */
+	if (dns_name_equal(&cname.cname, &rpz->passthru))
+		return (DNS_RPZ_POLICY_PASSTHRU);
+
+	/*
+	 * 128.1.0.127.rpz-ip CNAME  128.1.0.0.127. is obsolete PASSTHRU
 	 */
 	if (selfname != NULL && dns_name_equal(&cname.cname, selfname))
 		return (DNS_RPZ_POLICY_PASSTHRU);
diff --git a/lib/dns/spnego_asn1.pl b/lib/dns/spnego_asn1.pl
index 93dd676..0aaa57f 100755
--- a/lib/dns/spnego_asn1.pl
+++ b/lib/dns/spnego_asn1.pl
@@ -1,6 +1,6 @@
 #!/bin/bin/perl -w
 #
-# Copyright (C) 2006, 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2006, 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/dns/validator.c b/lib/dns/validator.c
index 3f9aae7..674675f 100644
--- a/lib/dns/validator.c
+++ b/lib/dns/validator.c
@@ -2075,15 +2075,13 @@ validate(dns_validator_t *val, isc_boolean_t resume) {
 			validator_log(val, ISC_LOG_DEBUG(3),
 				      "failed to verify rdataset");
 		else {
-			isc_uint32_t ttl;
 			isc_stdtime_t now;
 
 			isc_stdtime_get(&now);
-			ttl = ISC_MIN(event->rdataset->ttl,
-				      ISC_MIN(val->siginfo->originalttl,
-					      val->siginfo->timeexpire - now));
-			event->rdataset->ttl = ttl;
-			event->sigrdataset->ttl = ttl;
+			dns_rdataset_trimttl(event->rdataset,
+					     event->sigrdataset,
+					     val->siginfo, now,
+					     val->view->acceptexpired);
 		}
 
 		if (val->keynode != NULL)
diff --git a/lib/dns/view.c b/lib/dns/view.c
index 4590693..6750058 100644
--- a/lib/dns/view.c
+++ b/lib/dns/view.c
@@ -193,6 +193,8 @@ dns_view_create(isc_mem_t *mctx, dns_rdataclass_t rdclass,
 	view->v4_aaaa = dns_v4_aaaa_ok;
 	view->v4_aaaa_acl = NULL;
 	ISC_LIST_INIT(view->rpz_zones);
+	view->rpz_recursive_only = ISC_TRUE;
+	view->rpz_break_dnssec = ISC_FALSE;
 	dns_fixedname_init(&view->dlv_fixed);
 	view->managed_keys = NULL;
 #ifdef BIND9
diff --git a/lib/dns/zone.c b/lib/dns/zone.c
index efe31eb..22db239 100644
--- a/lib/dns/zone.c
+++ b/lib/dns/zone.c
@@ -3082,7 +3082,7 @@ zone_journal(dns_zone_t *zone, dns_diff_t *diff, const char *caller) {
 					  ISC_TRUE, &journal);
 		if (result != ISC_R_SUCCESS) {
 			dns_zone_log(zone, ISC_LOG_ERROR,
-				     "%s:dns_journal_open -> %s\n",
+				     "%s:dns_journal_open -> %s",
 				     caller, dns_result_totext(result));
 			return (result);
 		}
@@ -3091,7 +3091,7 @@ zone_journal(dns_zone_t *zone, dns_diff_t *diff, const char *caller) {
 		dns_journal_destroy(&journal);
 		if (result != ISC_R_SUCCESS) {
 			dns_zone_log(zone, ISC_LOG_ERROR,
-				     "%s:dns_journal_write_transaction -> %s\n",
+				     "%s:dns_journal_write_transaction -> %s",
 				     caller, dns_result_totext(result));
 			return (result);
 		}
@@ -3116,7 +3116,7 @@ add_soa(dns_zone_t *zone, dns_db_t *db) {
 	result = dns_db_newversion(db, &ver);
 	if (result != ISC_R_SUCCESS) {
 		dns_zone_log(zone, ISC_LOG_ERROR,
-			     "add_soa:dns_db_newversion -> %s\n",
+			     "add_soa:dns_db_newversion -> %s",
 			     dns_result_totext(result));
 		goto failure;
 	}
@@ -3126,7 +3126,7 @@ add_soa(dns_zone_t *zone, dns_db_t *db) {
 				    0, 0, 0, 0, 0, buf, &rdata);
 	if (result != ISC_R_SUCCESS) {
 		dns_zone_log(zone, ISC_LOG_ERROR,
-			     "add_soa:dns_soa_buildrdata -> %s\n",
+			     "add_soa:dns_soa_buildrdata -> %s",
 			     dns_result_totext(result));
 		goto failure;
 	}
@@ -3178,7 +3178,7 @@ sync_keyzone(dns_zone_t *zone, dns_db_t *db) {
 	result = dns_db_newversion(db, &ver);
 	if (result != ISC_R_SUCCESS) {
 		dns_zone_log(zone, ISC_LOG_ERROR,
-			     "sync_keyzone:dns_db_newversion -> %s\n",
+			     "sync_keyzone:dns_db_newversion -> %s",
 			     dns_result_totext(result));
 		goto failure;
 	}
@@ -4939,7 +4939,7 @@ zone_resigninc(dns_zone_t *zone) {
 	result = dns_db_newversion(db, &version);
 	if (result != ISC_R_SUCCESS) {
 		dns_zone_log(zone, ISC_LOG_ERROR,
-			     "zone_resigninc:dns_db_newversion -> %s\n",
+			     "zone_resigninc:dns_db_newversion -> %s",
 			     dns_result_totext(result));
 		goto failure;
 	}
@@ -4948,7 +4948,7 @@ zone_resigninc(dns_zone_t *zone) {
 				zone_keys, &nkeys);
 	if (result != ISC_R_SUCCESS) {
 		dns_zone_log(zone, ISC_LOG_ERROR,
-			     "zone_resigninc:find_zone_keys -> %s\n",
+			     "zone_resigninc:find_zone_keys -> %s",
 			     dns_result_totext(result));
 		goto failure;
 	}
@@ -4972,7 +4972,7 @@ zone_resigninc(dns_zone_t *zone) {
 	result = dns_db_getsigningtime(db, &rdataset, name);
 	if (result != ISC_R_SUCCESS && result != ISC_R_NOTFOUND) {
 		dns_zone_log(zone, ISC_LOG_ERROR,
-			     "zone_resigninc:dns_db_getsigningtime -> %s\n",
+			     "zone_resigninc:dns_db_getsigningtime -> %s",
 			     dns_result_totext(result));
 	}
 
@@ -4996,7 +4996,7 @@ zone_resigninc(dns_zone_t *zone) {
 				  zone_keys, nkeys, now, ISC_TRUE);
 		if (result != ISC_R_SUCCESS) {
 			dns_zone_log(zone, ISC_LOG_ERROR,
-				     "zone_resigninc:del_sigs -> %s\n",
+				     "zone_resigninc:del_sigs -> %s",
 				     dns_result_totext(result));
 			break;
 		}
@@ -5006,7 +5006,7 @@ zone_resigninc(dns_zone_t *zone) {
 				  expire, check_ksk, keyset_kskonly);
 		if (result != ISC_R_SUCCESS) {
 			dns_zone_log(zone, ISC_LOG_ERROR,
-				     "zone_resigninc:add_sigs -> %s\n",
+				     "zone_resigninc:add_sigs -> %s",
 				     dns_result_totext(result));
 			break;
 		}
@@ -5018,7 +5018,7 @@ zone_resigninc(dns_zone_t *zone) {
 		}
 		if (result != ISC_R_SUCCESS)
 			dns_zone_log(zone, ISC_LOG_ERROR,
-			     "zone_resigninc:dns_db_getsigningtime -> %s\n",
+			     "zone_resigninc:dns_db_getsigningtime -> %s",
 				     dns_result_totext(result));
 	}
 
@@ -5029,7 +5029,7 @@ zone_resigninc(dns_zone_t *zone) {
 			  &sig_diff, zone_keys, nkeys, now, ISC_TRUE);
 	if (result != ISC_R_SUCCESS) {
 		dns_zone_log(zone, ISC_LOG_ERROR,
-			     "zone_resigninc:del_sigs -> %s\n",
+			     "zone_resigninc:del_sigs -> %s",
 			     dns_result_totext(result));
 		goto failure;
 	}
@@ -5044,7 +5044,7 @@ zone_resigninc(dns_zone_t *zone) {
 	result = increment_soa_serial(db, version, &sig_diff, zone->mctx);
 	if (result != ISC_R_SUCCESS) {
 		dns_zone_log(zone, ISC_LOG_ERROR,
-			     "zone_resigninc:increment_soa_serial -> %s\n",
+			     "zone_resigninc:increment_soa_serial -> %s",
 			     dns_result_totext(result));
 		goto failure;
 	}
@@ -5058,7 +5058,7 @@ zone_resigninc(dns_zone_t *zone) {
 			  soaexpire, check_ksk, keyset_kskonly);
 	if (result != ISC_R_SUCCESS) {
 		dns_zone_log(zone, ISC_LOG_ERROR,
-			     "zone_resigninc:add_sigs -> %s\n",
+			     "zone_resigninc:add_sigs -> %s",
 			     dns_result_totext(result));
 		goto failure;
 	}
@@ -5753,7 +5753,7 @@ update_sigs(dns_diff_t *diff, dns_db_t *db, dns_dbversion_t *version,
 				  zone_keys, nkeys, now, ISC_FALSE);
 		if (result != ISC_R_SUCCESS) {
 			dns_zone_log(zone, ISC_LOG_ERROR,
-				     "update_sigs:del_sigs -> %s\n",
+				     "update_sigs:del_sigs -> %s",
 				     dns_result_totext(result));
 			return (result);
 		}
@@ -5763,7 +5763,7 @@ update_sigs(dns_diff_t *diff, dns_db_t *db, dns_dbversion_t *version,
 				  expire, check_ksk, keyset_kskonly);
 		if (result != ISC_R_SUCCESS) {
 			dns_zone_log(zone, ISC_LOG_ERROR,
-				     "update_sigs:add_sigs -> %s\n",
+				     "update_sigs:add_sigs -> %s",
 				     dns_result_totext(result));
 			return (result);
 		}
@@ -5848,7 +5848,7 @@ zone_nsec3chain(dns_zone_t *zone) {
 	result = dns_db_newversion(db, &version);
 	if (result != ISC_R_SUCCESS) {
 		dns_zone_log(zone, ISC_LOG_ERROR,
-			     "zone_nsec3chain:dns_db_newversion -> %s\n",
+			     "zone_nsec3chain:dns_db_newversion -> %s",
 			     dns_result_totext(result));
 		goto failure;
 	}
@@ -5857,7 +5857,7 @@ zone_nsec3chain(dns_zone_t *zone) {
 				DNS_MAXZONEKEYS, zone_keys, &nkeys);
 	if (result != ISC_R_SUCCESS) {
 		dns_zone_log(zone, ISC_LOG_ERROR,
-			     "zone_nsec3chain:find_zone_keys -> %s\n",
+			     "zone_nsec3chain:find_zone_keys -> %s",
 			     dns_result_totext(result));
 		goto failure;
 	}
@@ -6003,7 +6003,7 @@ zone_nsec3chain(dns_zone_t *zone) {
 					    &nsec3_diff);
 		if (result != ISC_R_SUCCESS) {
 			dns_zone_log(zone, ISC_LOG_ERROR, "zone_nsec3chain:"
-				     "dns_nsec3_addnsec3 -> %s\n",
+				     "dns_nsec3_addnsec3 -> %s",
 				     dns_result_totext(result));
 			goto failure;
 		}
@@ -6061,7 +6061,7 @@ zone_nsec3chain(dns_zone_t *zone) {
 			} else if (result != ISC_R_SUCCESS) {
 				dns_zone_log(zone, ISC_LOG_ERROR,
 					     "zone_nsec3chain:"
-					     "dns_dbiterator_next -> %s\n",
+					     "dns_dbiterator_next -> %s",
 					     dns_result_totext(result));
 				goto failure;
 			} else if (delegation) {
@@ -6119,7 +6119,7 @@ zone_nsec3chain(dns_zone_t *zone) {
 			if (result != ISC_R_SUCCESS) {
 				dns_zone_log(zone, ISC_LOG_ERROR,
 					     "zone_nsec3chain:"
-					     "need_nsec_chain -> %s\n",
+					     "need_nsec_chain -> %s",
 					     dns_result_totext(result));
 				goto failure;
 			}
@@ -6144,7 +6144,7 @@ zone_nsec3chain(dns_zone_t *zone) {
 				if (result != ISC_R_SUCCESS) {
 					dns_zone_log(zone, ISC_LOG_ERROR,
 						     "zone_nsec3chain:"
-						     "fixup_nsec3param -> %s\n",
+						     "fixup_nsec3param -> %s",
 						     dns_result_totext(result));
 					goto failure;
 				}
@@ -6159,7 +6159,7 @@ zone_nsec3chain(dns_zone_t *zone) {
 			if (result != ISC_R_SUCCESS) {
 				dns_zone_log(zone, ISC_LOG_ERROR,
 					     "zone_nsec3chain:"
-					     "deletematchingnsec3 -> %s\n",
+					     "deletematchingnsec3 -> %s",
 					     dns_result_totext(result));
 				goto failure;
 			}
@@ -6260,7 +6260,7 @@ zone_nsec3chain(dns_zone_t *zone) {
 				if (result != ISC_R_SUCCESS) {
 					dns_zone_log(zone, ISC_LOG_ERROR,
 						     "zone_nsec3chain:"
-						     "fixup_nsec3param -> %s\n",
+						     "fixup_nsec3param -> %s",
 						     dns_result_totext(result));
 					goto failure;
 				}
@@ -6268,7 +6268,7 @@ zone_nsec3chain(dns_zone_t *zone) {
 			} else if (result != ISC_R_SUCCESS) {
 				dns_zone_log(zone, ISC_LOG_ERROR,
 					     "zone_nsec3chain:"
-					     "dns_dbiterator_next -> %s\n",
+					     "dns_dbiterator_next -> %s",
 					     dns_result_totext(result));
 				goto failure;
 			} else if (delegation) {
@@ -6305,7 +6305,7 @@ zone_nsec3chain(dns_zone_t *zone) {
 		result = dns_db_allrdatasets(db, node, version, 0, &iterator);
 		if (result != ISC_R_SUCCESS) {
 			dns_zone_log(zone, ISC_LOG_ERROR, "zone_nsec3chain:"
-				     "dns_db_allrdatasets -> %s\n",
+				     "dns_db_allrdatasets -> %s",
 				     dns_result_totext(result));
 			goto failure;
 		}
@@ -6331,7 +6331,7 @@ zone_nsec3chain(dns_zone_t *zone) {
 			if (result != ISC_R_SUCCESS) {
 				dns_zone_log(zone, ISC_LOG_ERROR,
 					     "zone_nsec3chain:"
-					     "updatesecure -> %s\n",
+					     "updatesecure -> %s",
 					     dns_result_totext(result));
 				goto failure;
 			}
@@ -6344,7 +6344,7 @@ zone_nsec3chain(dns_zone_t *zone) {
 			if (result != ISC_R_SUCCESS) {
 				dns_zone_log(zone, ISC_LOG_ERROR,
 					     "zone_nsec3chain:"
-					     "dns_nsec3_addnsec3s -> %s\n",
+					     "dns_nsec3_addnsec3s -> %s",
 					     dns_result_totext(result));
 				goto failure;
 			}
@@ -6359,7 +6359,7 @@ zone_nsec3chain(dns_zone_t *zone) {
 			     check_ksk, keyset_kskonly, &sig_diff);
 	if (result != ISC_R_SUCCESS) {
 		dns_zone_log(zone, ISC_LOG_ERROR, "zone_nsec3chain:"
-			     "update_sigs -> %s\n", dns_result_totext(result));
+			     "update_sigs -> %s", dns_result_totext(result));
 		goto failure;
 	}
 
@@ -6372,7 +6372,7 @@ zone_nsec3chain(dns_zone_t *zone) {
 			     check_ksk, keyset_kskonly, &sig_diff);
 	if (result != ISC_R_SUCCESS) {
 		dns_zone_log(zone, ISC_LOG_ERROR, "zone_nsec3chain:"
-			     "update_sigs -> %s\n", dns_result_totext(result));
+			     "update_sigs -> %s", dns_result_totext(result));
 		goto failure;
 	}
 
@@ -6383,7 +6383,7 @@ zone_nsec3chain(dns_zone_t *zone) {
 				      zone->minimum, ISC_FALSE, &nsec_diff);
 		if (result != ISC_R_SUCCESS) {
 			dns_zone_log(zone, ISC_LOG_ERROR, "zone_nsec3chain:"
-				     "updatesecure -> %s\n",
+				     "updatesecure -> %s",
 				     dns_result_totext(result));
 			goto failure;
 		}
@@ -6394,7 +6394,7 @@ zone_nsec3chain(dns_zone_t *zone) {
 			     check_ksk, keyset_kskonly, &sig_diff);
 	if (result != ISC_R_SUCCESS) {
 		dns_zone_log(zone, ISC_LOG_ERROR, "zone_nsec3chain:"
-			     "update_sigs -> %s\n", dns_result_totext(result));
+			     "update_sigs -> %s", dns_result_totext(result));
 		goto failure;
 	}
 
@@ -6409,14 +6409,14 @@ zone_nsec3chain(dns_zone_t *zone) {
 			  &sig_diff, zone_keys, nkeys, now, ISC_FALSE);
 	if (result != ISC_R_SUCCESS) {
 		dns_zone_log(zone, ISC_LOG_ERROR, "zone_nsec3chain:"
-			     "del_sigs -> %s\n", dns_result_totext(result));
+			     "del_sigs -> %s", dns_result_totext(result));
 		goto failure;
 	}
 
 	result = increment_soa_serial(db, version, &sig_diff, zone->mctx);
 	if (result != ISC_R_SUCCESS) {
 		dns_zone_log(zone, ISC_LOG_ERROR, "zone_nsec3chain:"
-			     "increment_soa_serial -> %s\n",
+			     "increment_soa_serial -> %s",
 			     dns_result_totext(result));
 		goto failure;
 	}
@@ -6426,7 +6426,7 @@ zone_nsec3chain(dns_zone_t *zone) {
 			  soaexpire, check_ksk, keyset_kskonly);
 	if (result != ISC_R_SUCCESS) {
 		dns_zone_log(zone, ISC_LOG_ERROR, "zone_nsec3chain:"
-			     "add_sigs -> %s\n", dns_result_totext(result));
+			     "add_sigs -> %s", dns_result_totext(result));
 		goto failure;
 	}
 
@@ -6470,7 +6470,7 @@ zone_nsec3chain(dns_zone_t *zone) {
 
  failure:
 	if (result != ISC_R_SUCCESS)
-		dns_zone_log(zone, ISC_LOG_ERROR, "zone_nsec3chain: %s\n",
+		dns_zone_log(zone, ISC_LOG_ERROR, "zone_nsec3chain: %s",
 			     dns_result_totext(result));
 	/*
 	 * On error roll back the current nsec3chain.
@@ -6673,7 +6673,7 @@ zone_sign(dns_zone_t *zone) {
 	result = dns_db_newversion(db, &version);
 	if (result != ISC_R_SUCCESS) {
 		dns_zone_log(zone, ISC_LOG_ERROR,
-			     "zone_sign:dns_db_newversion -> %s\n",
+			     "zone_sign:dns_db_newversion -> %s",
 			     dns_result_totext(result));
 		goto failure;
 	}
@@ -6682,7 +6682,7 @@ zone_sign(dns_zone_t *zone) {
 				DNS_MAXZONEKEYS, zone_keys, &nkeys);
 	if (result != ISC_R_SUCCESS) {
 		dns_zone_log(zone, ISC_LOG_ERROR,
-			     "zone_sign:find_zone_keys -> %s\n",
+			     "zone_sign:find_zone_keys -> %s",
 			     dns_result_totext(result));
 		goto failure;
 	}
@@ -6903,7 +6903,7 @@ zone_sign(dns_zone_t *zone) {
 					if (result != ISC_R_SUCCESS) {
 						dns_zone_log(zone,
 							     ISC_LOG_ERROR,
-						    "updatesecure -> %s\n",
+						    "updatesecure -> %s",
 						    dns_result_totext(result));
 						goto failure;
 					}
@@ -6915,8 +6915,7 @@ zone_sign(dns_zone_t *zone) {
 							   &post_diff);
 				if (result != ISC_R_SUCCESS) {
 					dns_zone_log(zone, ISC_LOG_ERROR,
-						     "updatesignwithkey "
-						     "-> %s\n",
+						     "updatesignwithkey -> %s",
 						     dns_result_totext(result));
 					goto failure;
 				}
@@ -6924,7 +6923,7 @@ zone_sign(dns_zone_t *zone) {
 				goto next_signing;
 			} else if (result != ISC_R_SUCCESS) {
 				dns_zone_log(zone, ISC_LOG_ERROR,
-					"zone_sign:dns_dbiterator_next -> %s\n",
+					"zone_sign:dns_dbiterator_next -> %s",
 					     dns_result_totext(result));
 				goto failure;
 			} else if (delegation) {
@@ -6950,7 +6949,7 @@ zone_sign(dns_zone_t *zone) {
 				     check_ksk, keyset_kskonly, &sig_diff);
 		if (result != ISC_R_SUCCESS) {
 			dns_zone_log(zone, ISC_LOG_ERROR, "zone_sign:"
-				     "update_sigs -> %s\n",
+				     "update_sigs -> %s",
 				     dns_result_totext(result));
 			goto failure;
 		}
@@ -6970,7 +6969,7 @@ zone_sign(dns_zone_t *zone) {
 			  &sig_diff, zone_keys, nkeys, now, ISC_FALSE);
 	if (result != ISC_R_SUCCESS) {
 		dns_zone_log(zone, ISC_LOG_ERROR,
-			     "zone_sign:del_sigs -> %s\n",
+			     "zone_sign:del_sigs -> %s",
 			     dns_result_totext(result));
 		goto failure;
 	}
@@ -6978,7 +6977,7 @@ zone_sign(dns_zone_t *zone) {
 	result = increment_soa_serial(db, version, &sig_diff, zone->mctx);
 	if (result != ISC_R_SUCCESS) {
 		dns_zone_log(zone, ISC_LOG_ERROR,
-			     "zone_sign:increment_soa_serial -> %s\n",
+			     "zone_sign:increment_soa_serial -> %s",
 			     dns_result_totext(result));
 		goto failure;
 	}
@@ -6992,7 +6991,7 @@ zone_sign(dns_zone_t *zone) {
 			  soaexpire, check_ksk, keyset_kskonly);
 	if (result != ISC_R_SUCCESS) {
 		dns_zone_log(zone, ISC_LOG_ERROR,
-			     "zone_sign:add_sigs -> %s\n",
+			     "zone_sign:add_sigs -> %s",
 			     dns_result_totext(result));
 		goto failure;
 	}
@@ -8034,7 +8033,7 @@ zone_maintenance(dns_zone_t *zone) {
 		    DNS_ZONE_FLAG(zone, DNS_ZONEFLG_LOADED) &&
 		    DNS_ZONE_FLAG(zone, DNS_ZONEFLG_NEEDDUMP)) {
 			dumping = was_dumping(zone);
-		} else 
+		} else
 			dumping = ISC_TRUE;
 		UNLOCK_ZONE(zone);
 		if (!dumping) {
@@ -14008,7 +14007,7 @@ sign_apex(dns_zone_t *zone, dns_db_t *db, dns_dbversion_t *ver,
 				zone_keys, &nkeys);
 	if (result != ISC_R_SUCCESS) {
 		dns_zone_log(zone, ISC_LOG_ERROR,
-			     "sign_apex:find_zone_keys -> %s\n",
+			     "sign_apex:find_zone_keys -> %s",
 			     dns_result_totext(result));
 		return (result);
 	}
@@ -14039,7 +14038,7 @@ sign_apex(dns_zone_t *zone, dns_db_t *db, dns_dbversion_t *ver,
 				  zone_keys, nkeys, now, ISC_FALSE);
 		if (result != ISC_R_SUCCESS) {
 			dns_zone_log(zone, ISC_LOG_ERROR,
-				     "sign_apex:del_sigs -> %s\n",
+				     "sign_apex:del_sigs -> %s",
 				     dns_result_totext(result));
 			goto failure;
 		}
@@ -14049,7 +14048,7 @@ sign_apex(dns_zone_t *zone, dns_db_t *db, dns_dbversion_t *ver,
 				  keyset_kskonly);
 		if (result != ISC_R_SUCCESS) {
 			dns_zone_log(zone, ISC_LOG_ERROR,
-				     "sign_apex:add_sigs -> %s\n",
+				     "sign_apex:add_sigs -> %s",
 				     dns_result_totext(result));
 			goto failure;
 		}
@@ -14061,7 +14060,7 @@ sign_apex(dns_zone_t *zone, dns_db_t *db, dns_dbversion_t *ver,
 
 	if (result != ISC_R_SUCCESS) {
 		dns_zone_log(zone, ISC_LOG_ERROR,
-			     "sign_apex:update_sigs -> %s\n",
+			     "sign_apex:update_sigs -> %s",
 			     dns_result_totext(result));
 		goto failure;
 	}
diff --git a/lib/export/Makefile.in b/lib/export/Makefile.in
index fc9d4ad..1fd7216 100644
--- a/lib/export/Makefile.in
+++ b/lib/export/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2009  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2009, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/export/dns/Makefile.in b/lib/export/dns/Makefile.in
index cd72988..6df36fe 100644
--- a/lib/export/dns/Makefile.in
+++ b/lib/export/dns/Makefile.in
@@ -44,7 +44,7 @@ LIBS =		@LIBS@
 # Alphabetically
 
 OPENSSLLINKOBJS = openssl_link.@O@ openssldh_link.@O@ openssldsa_link.@O@ \
-		opensslgost_link.@O@ opensslrsa_link.@O@
+		opensslecdsa_link.@O@ opensslgost_link.@O@ opensslrsa_link.@O@
 
 DSTOBJS =	@OPENSSLLINKOBJS@ \
 		dst_api.@O@ dst_lib.@O@ dst_parse.@O@ dst_result.@O@ \
@@ -72,7 +72,7 @@ OBJS=		${DNSOBJS} ${OTHEROBJS} ${DSTOBJS} ${PORTDNSOBJS}
 # Alphabetically
 
 OPENSSLLINKSRCS = openssl_link.c openssldh_link.c openssldsa_link.c \
-		opensslgost_link.c opensslrsa_link.c
+		opensslecdsa_link.c opensslgost_link.c opensslrsa_link.c
 
 DSTSRCS =	@OPENSSLLINKSRCS@ \
 		dst_api.c dst_lib.c dst_parse.c \
diff --git a/lib/export/dns/include/Makefile.in b/lib/export/dns/include/Makefile.in
index 9fc0b66..6bf1205 100644
--- a/lib/export/dns/include/Makefile.in
+++ b/lib/export/dns/include/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2009  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2009, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/export/dns/include/dns/Makefile.in b/lib/export/dns/include/dns/Makefile.in
index 2d7f2c7..b7f51b4 100644
--- a/lib/export/dns/include/dns/Makefile.in
+++ b/lib/export/dns/include/dns/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2009  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2009, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/export/dns/include/dst/Makefile.in b/lib/export/dns/include/dst/Makefile.in
index 259e62e..f6f540a 100644
--- a/lib/export/dns/include/dst/Makefile.in
+++ b/lib/export/dns/include/dst/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2009  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2009, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/export/irs/include/irs/Makefile.in b/lib/export/irs/include/irs/Makefile.in
index c850757..530e67c 100644
--- a/lib/export/irs/include/irs/Makefile.in
+++ b/lib/export/irs/include/irs/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2009  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2009, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/export/isc/Makefile.in b/lib/export/isc/Makefile.in
index a55a1f4..86726ab 100644
--- a/lib/export/isc/Makefile.in
+++ b/lib/export/isc/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2009, 2010  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2009, 2010, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/export/isc/include/isc/Makefile.in b/lib/export/isc/include/isc/Makefile.in
index 2499185..91f538c 100644
--- a/lib/export/isc/include/isc/Makefile.in
+++ b/lib/export/isc/include/isc/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2009  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2009, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/export/isc/nls/Makefile.in b/lib/export/isc/nls/Makefile.in
index a9e779f..2515685 100644
--- a/lib/export/isc/nls/Makefile.in
+++ b/lib/export/isc/nls/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2009  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2009, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/export/isc/nothreads/Makefile.in b/lib/export/isc/nothreads/Makefile.in
index 93b2109..994da63 100644
--- a/lib/export/isc/nothreads/Makefile.in
+++ b/lib/export/isc/nothreads/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2009, 2010  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2009, 2010, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/export/isc/nothreads/include/isc/Makefile.in b/lib/export/isc/nothreads/include/isc/Makefile.in
index eb25c88..9bda987 100644
--- a/lib/export/isc/nothreads/include/isc/Makefile.in
+++ b/lib/export/isc/nothreads/include/isc/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2009  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2009, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/export/isc/pthreads/Makefile.in b/lib/export/isc/pthreads/Makefile.in
index 92788ec..f08e5c6 100644
--- a/lib/export/isc/pthreads/Makefile.in
+++ b/lib/export/isc/pthreads/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2009  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2009, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/export/isc/pthreads/include/isc/Makefile.in b/lib/export/isc/pthreads/include/isc/Makefile.in
index 77d5c07..4319768 100644
--- a/lib/export/isc/pthreads/include/isc/Makefile.in
+++ b/lib/export/isc/pthreads/include/isc/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2009  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2009, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/export/isc/unix/Makefile.in b/lib/export/isc/unix/Makefile.in
index 5a8eed8..f5cf7e8 100644
--- a/lib/export/isc/unix/Makefile.in
+++ b/lib/export/isc/unix/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2009  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2009, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/export/isc/unix/include/isc/Makefile.in b/lib/export/isc/unix/include/isc/Makefile.in
index f19b8c6..7159c76 100644
--- a/lib/export/isc/unix/include/isc/Makefile.in
+++ b/lib/export/isc/unix/include/isc/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2009  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2009, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/export/isccfg/include/isccfg/Makefile.in b/lib/export/isccfg/include/isccfg/Makefile.in
index 5e9ea78..57a344c 100644
--- a/lib/export/isccfg/include/isccfg/Makefile.in
+++ b/lib/export/isccfg/include/isccfg/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2009  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2009, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/export/samples/Makefile-postinstall.in b/lib/export/samples/Makefile-postinstall.in
index 174aed6..5b1aafb 100644
--- a/lib/export/samples/Makefile-postinstall.in
+++ b/lib/export/samples/Makefile-postinstall.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2009  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2009, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/export/samples/Makefile.in b/lib/export/samples/Makefile.in
index c60baac..cdc66b1 100644
--- a/lib/export/samples/Makefile.in
+++ b/lib/export/samples/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2009  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2009, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/irs/Makefile.in b/lib/irs/Makefile.in
index ed86967..d3c47b0 100644
--- a/lib/irs/Makefile.in
+++ b/lib/irs/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2009  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2009, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/irs/include/Makefile.in b/lib/irs/include/Makefile.in
index eca1945..91099f1 100644
--- a/lib/irs/include/Makefile.in
+++ b/lib/irs/include/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2009  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2009, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/irs/include/irs/Makefile.in b/lib/irs/include/irs/Makefile.in
index 3c3b612..63e7fd6 100644
--- a/lib/irs/include/irs/Makefile.in
+++ b/lib/irs/include/irs/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2009  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2009, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/isc/alpha/Makefile.in b/lib/isc/alpha/Makefile.in
index 324db07..9c24cdf 100644
--- a/lib/isc/alpha/Makefile.in
+++ b/lib/isc/alpha/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/isc/alpha/include/Makefile.in b/lib/isc/alpha/include/Makefile.in
index f1d8bdd..e399559 100644
--- a/lib/isc/alpha/include/Makefile.in
+++ b/lib/isc/alpha/include/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/isc/alpha/include/isc/Makefile.in b/lib/isc/alpha/include/isc/Makefile.in
index 5f116ca..4927e21 100644
--- a/lib/isc/alpha/include/isc/Makefile.in
+++ b/lib/isc/alpha/include/isc/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/isc/api b/lib/isc/api
index 3b91551..18de29c 100644
--- a/lib/isc/api
+++ b/lib/isc/api
@@ -3,6 +3,6 @@
 # 9.7: 60-79
 # 9.8: 80-89
 # 9.9: 90-109
-LIBINTERFACE = 83
-LIBREVISION = 5
-LIBAGE = 0
+LIBINTERFACE = 85
+LIBREVISION = 0
+LIBAGE = 1
diff --git a/lib/isc/ia64/Makefile.in b/lib/isc/ia64/Makefile.in
index 324db07..9c24cdf 100644
--- a/lib/isc/ia64/Makefile.in
+++ b/lib/isc/ia64/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/isc/ia64/include/Makefile.in b/lib/isc/ia64/include/Makefile.in
index f1d8bdd..e399559 100644
--- a/lib/isc/ia64/include/Makefile.in
+++ b/lib/isc/ia64/include/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/isc/ia64/include/isc/Makefile.in b/lib/isc/ia64/include/isc/Makefile.in
index 5f116ca..4927e21 100644
--- a/lib/isc/ia64/include/isc/Makefile.in
+++ b/lib/isc/ia64/include/isc/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/isc/ia64/include/isc/atomic.h b/lib/isc/ia64/include/isc/atomic.h
index 6c22f2a..557941d 100644
--- a/lib/isc/ia64/include/isc/atomic.h
+++ b/lib/isc/ia64/include/isc/atomic.h
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2006, 2007, 2009  Internet Systems Consortium, Inc. ("ISC")
+ * Copyright (C) 2006, 2007, 2009, 2012  Internet Systems Consortium, Inc. ("ISC")
  *
  * Permission to use, copy, modify, and/or distribute this software for any
  * purpose with or without fee is hereby granted, provided that the above
@@ -41,7 +41,7 @@ isc_atomic_xadd(isc_int32_t *p, isc_int32_t val)
 	for (prev = *(volatile isc_int32_t *)p; ; prev = swapped) {
 		swapped = prev + val;
 		__asm__ volatile(
-			"mov ar.ccv=%2;"
+			"mov ar.ccv=%2;;"
 			"cmpxchg4.acq %0=%4,%3,ar.ccv"
 			: "=r" (swapped), "=m" (*p)
 			: "r" (prev), "r" (swapped), "m" (*p)
@@ -84,7 +84,7 @@ isc_atomic_cmpxchg(isc_int32_t *p, isc_int32_t cmpval, isc_int32_t val)
 	isc_int32_t ret;
 
 	__asm__ volatile(
-		"mov ar.ccv=%2;"
+		"mov ar.ccv=%2;;"
 		"cmpxchg4.acq %0=%4,%3,ar.ccv"
 		: "=r" (ret), "=m" (*p)
 		: "r" (cmpval), "r" (val), "m" (*p)
diff --git a/lib/isc/include/Makefile.in b/lib/isc/include/Makefile.in
index 04778d7..70c165e 100644
--- a/lib/isc/include/Makefile.in
+++ b/lib/isc/include/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2004, 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2004, 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 # Copyright (C) 1998-2001  Internet Software Consortium.
 #
 # Permission to use, copy, modify, and/or distribute this software for any
diff --git a/lib/isc/include/isc/file.h b/lib/isc/include/isc/file.h
index 8794065..38f78b7 100644
--- a/lib/isc/include/isc/file.h
+++ b/lib/isc/include/isc/file.h
@@ -25,6 +25,7 @@
 #include <stdio.h>
 
 #include <isc/lang.h>
+#include <isc/stat.h>
 #include <isc/types.h>
 
 ISC_LANG_BEGINDECLS
@@ -33,6 +34,9 @@ isc_result_t
 isc_file_settime(const char *file, isc_time_t *time);
 
 isc_result_t
+isc_file_mode(const char *file, mode_t *modep);
+
+isc_result_t
 isc_file_getmodtime(const char *file, isc_time_t *time);
 /*!<
  * \brief Get the time of last modification of a file.
@@ -97,15 +101,22 @@ isc_file_mktemplate(const char *path, char *buf, size_t buflen);
  *				of the path with the internal template string.
  */
 
-
 isc_result_t
 isc_file_openunique(char *templet, FILE **fp);
 isc_result_t
 isc_file_openuniqueprivate(char *templet, FILE **fp);
 isc_result_t
 isc_file_openuniquemode(char *templet, int mode, FILE **fp);
+isc_result_t
+isc_file_bopenunique(char *templet, FILE **fp);
+isc_result_t
+isc_file_bopenuniqueprivate(char *templet, FILE **fp);
+isc_result_t
+isc_file_bopenuniquemode(char *templet, int mode, FILE **fp);
 /*!<
  * \brief Create and open a file with a unique name based on 'templet'.
+ *	isc_file_bopen*() open the file in binary mode in Windows. 
+ *	isc_file_open*() open the file in text mode in Windows. 
  *
  * Notes:
  *\li	'template' is a reserved work in C++.  If you want to complain
diff --git a/lib/isc/include/isc/namespace.h b/lib/isc/include/isc/namespace.h
index 158cfe5..ae1801d 100644
--- a/lib/isc/include/isc/namespace.h
+++ b/lib/isc/include/isc/namespace.h
@@ -146,6 +146,8 @@
 #define isc_task_getcurrenttime isc__task_getcurrenttime
 #define isc_taskmgr_create isc__taskmgr_create
 #define isc_taskmgr_destroy isc__taskmgr_destroy
+#define isc_taskmgr_setexcltask isc__taskmgr_setexcltask
+#define isc_taskmgr_excltask isc__taskmgr_excltask
 #define isc_task_beginexclusive isc__task_beginexclusive
 #define isc_task_endexclusive isc__task_endexclusive
 
diff --git a/lib/isc/include/isc/task.h b/lib/isc/include/isc/task.h
index 36fca36..19d4783 100644
--- a/lib/isc/include/isc/task.h
+++ b/lib/isc/include/isc/task.h
@@ -106,6 +106,8 @@ typedef struct isc_taskmgrmethods {
 	isc_result_t	(*taskcreate)(isc_taskmgr_t *manager,
 				      unsigned int quantum,
 				      isc_task_t **taskp);
+	void (*setexcltask)(isc_taskmgr_t *mgr, isc_task_t *task);
+	isc_result_t (*excltask)(isc_taskmgr_t *mgr, isc_task_t **taskp);
 } isc_taskmgrmethods_t;
 
 typedef struct isc_taskmethods {
@@ -697,6 +699,31 @@ isc_taskmgr_destroy(isc_taskmgr_t **managerp);
  *	have been freed.
  */
 
+void
+isc_taskmgr_setexcltask(isc_taskmgr_t *mgr, isc_task_t *task);
+/*%<
+ * Set a task which will be used for all task-exclusive operations.
+ *
+ * Requires:
+ *\li	'manager' is a valid task manager.
+ *
+ *\li	'task' is a valid task.
+ */
+
+isc_result_t
+isc_taskmgr_excltask(isc_taskmgr_t *mgr, isc_task_t **taskp);
+/*%<
+ * Attach '*taskp' to the task set by isc_taskmgr_getexcltask().
+ * This task should be used whenever running in task-exclusive mode,
+ * so as to prevent deadlock between two exclusive tasks.
+ *
+ * Requires:
+ *\li	'manager' is a valid task manager.
+
+ *\li	taskp != NULL && *taskp == NULL
+ */
+
+
 #ifdef HAVE_LIBXML2
 
 void
diff --git a/lib/isc/mem.c b/lib/isc/mem.c
index 5b4b16c..1964b7a 100644
--- a/lib/isc/mem.c
+++ b/lib/isc/mem.c
@@ -1191,7 +1191,7 @@ isc___mem_putanddetach(isc_mem_t **ctxp, void *ptr, size_t size FLARG) {
 				oldsize -= ALIGNMENT_SIZE;
 			INSIST(oldsize == size);
 		}
-		isc_mem_free((isc_mem_t *)ctx, ptr);
+		isc__mem_free((isc_mem_t *)ctx, ptr FLARG_PASS);
 
 		MCTXLOCK(ctx, &ctx->lock);
 		ctx->references--;
@@ -1327,7 +1327,7 @@ isc___mem_put(isc_mem_t *ctx0, void *ptr, size_t size FLARG) {
 				oldsize -= ALIGNMENT_SIZE;
 			INSIST(oldsize == size);
 		}
-		isc_mem_free((isc_mem_t *)ctx, ptr);
+		isc__mem_free((isc_mem_t *)ctx, ptr FLARG_PASS);
 		return;
 	}
 
@@ -1592,7 +1592,11 @@ isc___mem_reallocate(isc_mem_t *ctx0, void *ptr, size_t size FLARG) {
 			oldsize = (((size_info *)ptr)[-1]).u.size;
 			INSIST(oldsize >= ALIGNMENT_SIZE);
 			oldsize -= ALIGNMENT_SIZE;
-			copysize = oldsize > size ? size : oldsize;
+			if ((isc_mem_debugging & ISC_MEM_DEBUGCTX) != 0) {
+				INSIST(oldsize >= ALIGNMENT_SIZE);
+				oldsize -= ALIGNMENT_SIZE;
+			}
+			copysize = (oldsize > size) ? size : oldsize;
 			memcpy(new_ptr, ptr, copysize);
 			isc__mem_free(ctx0, ptr FLARG_PASS);
 		}
diff --git a/lib/isc/mips/Makefile.in b/lib/isc/mips/Makefile.in
index 324db07..9c24cdf 100644
--- a/lib/isc/mips/Makefile.in
+++ b/lib/isc/mips/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/isc/mips/include/Makefile.in b/lib/isc/mips/include/Makefile.in
index f1d8bdd..e399559 100644
--- a/lib/isc/mips/include/Makefile.in
+++ b/lib/isc/mips/include/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/isc/mips/include/isc/Makefile.in b/lib/isc/mips/include/isc/Makefile.in
index 5f116ca..4927e21 100644
--- a/lib/isc/mips/include/isc/Makefile.in
+++ b/lib/isc/mips/include/isc/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/isc/nls/Makefile.in b/lib/isc/nls/Makefile.in
index aca4a27..7bacf1c 100644
--- a/lib/isc/nls/Makefile.in
+++ b/lib/isc/nls/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2004, 2007, 2009  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2004, 2007, 2009, 2012  Internet Systems Consortium, Inc. ("ISC")
 # Copyright (C) 1999-2001  Internet Software Consortium.
 #
 # Permission to use, copy, modify, and/or distribute this software for any
diff --git a/lib/isc/noatomic/Makefile.in b/lib/isc/noatomic/Makefile.in
index 324db07..9c24cdf 100644
--- a/lib/isc/noatomic/Makefile.in
+++ b/lib/isc/noatomic/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/isc/noatomic/include/Makefile.in b/lib/isc/noatomic/include/Makefile.in
index f1d8bdd..e399559 100644
--- a/lib/isc/noatomic/include/Makefile.in
+++ b/lib/isc/noatomic/include/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/isc/noatomic/include/isc/Makefile.in b/lib/isc/noatomic/include/isc/Makefile.in
index 5f116ca..4927e21 100644
--- a/lib/isc/noatomic/include/isc/Makefile.in
+++ b/lib/isc/noatomic/include/isc/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/isc/nothreads/Makefile.in b/lib/isc/nothreads/Makefile.in
index 7e7abd6..540b243 100644
--- a/lib/isc/nothreads/Makefile.in
+++ b/lib/isc/nothreads/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2004, 2007, 2009, 2010  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2004, 2007, 2009, 2010, 2012  Internet Systems Consortium, Inc. ("ISC")
 # Copyright (C) 2000, 2001  Internet Software Consortium.
 #
 # Permission to use, copy, modify, and/or distribute this software for any
diff --git a/lib/isc/nothreads/include/Makefile.in b/lib/isc/nothreads/include/Makefile.in
index a52310a..662a72d 100644
--- a/lib/isc/nothreads/include/Makefile.in
+++ b/lib/isc/nothreads/include/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2004, 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2004, 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 # Copyright (C) 2000, 2001  Internet Software Consortium.
 #
 # Permission to use, copy, modify, and/or distribute this software for any
diff --git a/lib/isc/nothreads/include/isc/Makefile.in b/lib/isc/nothreads/include/isc/Makefile.in
index 3c9eab0..a2c347e 100644
--- a/lib/isc/nothreads/include/isc/Makefile.in
+++ b/lib/isc/nothreads/include/isc/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2004, 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2004, 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 # Copyright (C) 2000, 2001  Internet Software Consortium.
 #
 # Permission to use, copy, modify, and/or distribute this software for any
diff --git a/lib/isc/powerpc/Makefile.in b/lib/isc/powerpc/Makefile.in
index 324db07..9c24cdf 100644
--- a/lib/isc/powerpc/Makefile.in
+++ b/lib/isc/powerpc/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/isc/powerpc/include/Makefile.in b/lib/isc/powerpc/include/Makefile.in
index f1d8bdd..e399559 100644
--- a/lib/isc/powerpc/include/Makefile.in
+++ b/lib/isc/powerpc/include/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/isc/powerpc/include/isc/Makefile.in b/lib/isc/powerpc/include/isc/Makefile.in
index 5f116ca..4927e21 100644
--- a/lib/isc/powerpc/include/isc/Makefile.in
+++ b/lib/isc/powerpc/include/isc/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/isc/pthreads/Makefile.in b/lib/isc/pthreads/Makefile.in
index d6e7c76..9f66ef3 100644
--- a/lib/isc/pthreads/Makefile.in
+++ b/lib/isc/pthreads/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2004, 2007, 2009  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2004, 2007, 2009, 2012  Internet Systems Consortium, Inc. ("ISC")
 # Copyright (C) 1998-2001  Internet Software Consortium.
 #
 # Permission to use, copy, modify, and/or distribute this software for any
diff --git a/lib/isc/pthreads/condition.c b/lib/isc/pthreads/condition.c
index 50281a2..9053cf0 100644
--- a/lib/isc/pthreads/condition.c
+++ b/lib/isc/pthreads/condition.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2004, 2005, 2007  Internet Systems Consortium, Inc. ("ISC")
+ * Copyright (C) 2004, 2005, 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
  * Copyright (C) 1998-2001  Internet Software Consortium.
  *
  * Permission to use, copy, modify, and/or distribute this software for any
@@ -43,7 +43,14 @@ isc_condition_waituntil(isc_condition_t *c, isc_mutex_t *m, isc_time_t *t) {
 	 * POSIX defines a timespec's tv_sec as time_t.
 	 */
 	result = isc_time_secondsastimet(t, &ts.tv_sec);
-	if (result != ISC_R_SUCCESS)
+
+	/*
+	 * If we have a range error ts.tv_sec is most probably a signed
+	 * 32 bit value.  Set ts.tv_sec to INT_MAX.  This is a kludge.
+	 */
+	if (result == ISC_R_RANGE)
+		ts.tv_sec = INT_MAX;
+	else if (result != ISC_R_SUCCESS)
 		return (result);
 
 	/*!
diff --git a/lib/isc/pthreads/include/Makefile.in b/lib/isc/pthreads/include/Makefile.in
index 0303ab1..46c243e 100644
--- a/lib/isc/pthreads/include/Makefile.in
+++ b/lib/isc/pthreads/include/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2004, 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2004, 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 # Copyright (C) 1998-2001  Internet Software Consortium.
 #
 # Permission to use, copy, modify, and/or distribute this software for any
diff --git a/lib/isc/pthreads/include/isc/Makefile.in b/lib/isc/pthreads/include/isc/Makefile.in
index 11675ec..7cadcf4 100644
--- a/lib/isc/pthreads/include/isc/Makefile.in
+++ b/lib/isc/pthreads/include/isc/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2004, 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2004, 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 # Copyright (C) 1998-2001  Internet Software Consortium.
 #
 # Permission to use, copy, modify, and/or distribute this software for any
diff --git a/lib/isc/sparc64/Makefile.in b/lib/isc/sparc64/Makefile.in
index 324db07..9c24cdf 100644
--- a/lib/isc/sparc64/Makefile.in
+++ b/lib/isc/sparc64/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/isc/sparc64/include/Makefile.in b/lib/isc/sparc64/include/Makefile.in
index f1d8bdd..e399559 100644
--- a/lib/isc/sparc64/include/Makefile.in
+++ b/lib/isc/sparc64/include/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/isc/sparc64/include/isc/Makefile.in b/lib/isc/sparc64/include/isc/Makefile.in
index 5f116ca..4927e21 100644
--- a/lib/isc/sparc64/include/isc/Makefile.in
+++ b/lib/isc/sparc64/include/isc/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/isc/task.c b/lib/isc/task.c
index f4e1265..a5f6ef9 100644
--- a/lib/isc/task.c
+++ b/lib/isc/task.c
@@ -152,6 +152,7 @@ struct isc__taskmgr {
 	unsigned int			tasks_running;
 	isc_boolean_t			exclusive_requested;
 	isc_boolean_t			exiting;
+	isc__task_t			*excl;
 #ifdef USE_SHARED_MANAGER
 	unsigned int			refs;
 #endif /* ISC_PLATFORM_USETHREADS */
@@ -221,6 +222,10 @@ isc__taskmgr_create(isc_mem_t *mctx, unsigned int workers,
 		    unsigned int default_quantum, isc_taskmgr_t **managerp);
 ISC_TASKFUNC_SCOPE void
 isc__taskmgr_destroy(isc_taskmgr_t **managerp);
+ISC_TASKFUNC_SCOPE void
+isc__taskmgr_setexcltask(isc_taskmgr_t *mgr0, isc_task_t *task0);
+ISC_TASKFUNC_SCOPE isc_result_t
+isc__taskmgr_excltask(isc_taskmgr_t *mgr0, isc_task_t **taskp);
 ISC_TASKFUNC_SCOPE isc_result_t
 isc__task_beginexclusive(isc_task_t *task);
 ISC_TASKFUNC_SCOPE void
@@ -261,7 +266,9 @@ static struct isc__taskmethods {
 
 static isc_taskmgrmethods_t taskmgrmethods = {
 	isc__taskmgr_destroy,
-	isc__task_create
+	isc__task_create,
+	isc__taskmgr_setexcltask,
+	isc__taskmgr_excltask
 };
 
 /***
@@ -1262,6 +1269,7 @@ isc__taskmgr_create(isc_mem_t *mctx, unsigned int workers,
 	manager->tasks_running = 0;
 	manager->exclusive_requested = ISC_FALSE;
 	manager->exiting = ISC_FALSE;
+	manager->excl = NULL;
 
 	isc_mem_attach(mctx, &manager->mctx);
 
@@ -1344,6 +1352,12 @@ isc__taskmgr_destroy(isc_taskmgr_t **managerp) {
 	 */
 
 	/*
+	 * Detach the exclusive task before acquiring the manager lock
+	 */
+	if (manager->excl != NULL)
+		isc__task_detach((isc_task_t **) &manager->excl);
+
+	/*
 	 * Unlike elsewhere, we're going to hold this lock a long time.
 	 * We need to do so, because otherwise the list of tasks could
 	 * change while we were traversing it.
@@ -1440,12 +1454,41 @@ isc__taskmgr_dispatch(isc_taskmgr_t *manager0) {
 
 #endif /* USE_WORKER_THREADS */
 
+ISC_TASKFUNC_SCOPE void
+isc__taskmgr_setexcltask(isc_taskmgr_t *mgr0, isc_task_t *task0) {
+	isc__taskmgr_t *mgr = (isc__taskmgr_t *) mgr0;
+	isc__task_t *task = (isc__task_t *) task0;
+
+	REQUIRE(VALID_MANAGER(mgr));
+	REQUIRE(VALID_TASK(task));
+	if (mgr->excl != NULL)
+		isc__task_detach((isc_task_t **) &mgr->excl);
+	isc__task_attach(task0, (isc_task_t **) &mgr->excl);
+}
+
+ISC_TASKFUNC_SCOPE isc_result_t
+isc__taskmgr_excltask(isc_taskmgr_t *mgr0, isc_task_t **taskp) {
+	isc__taskmgr_t *mgr = (isc__taskmgr_t *) mgr0;
+
+	REQUIRE(VALID_MANAGER(mgr));
+	REQUIRE(taskp != NULL && *taskp == NULL);
+
+	if (mgr->excl == NULL)
+		return (ISC_R_NOTFOUND);
+
+	isc__task_attach((isc_task_t *) mgr->excl, taskp);
+	return (ISC_R_SUCCESS);
+}
+
 ISC_TASKFUNC_SCOPE isc_result_t
 isc__task_beginexclusive(isc_task_t *task0) {
 #ifdef USE_WORKER_THREADS
 	isc__task_t *task = (isc__task_t *)task0;
 	isc__taskmgr_t *manager = task->manager;
+
 	REQUIRE(task->state == task_state_running);
+	/* XXX: Require task == manager->excl? */
+
 	LOCK(&manager->lock);
 	if (manager->exclusive_requested) {
 		UNLOCK(&manager->lock);
diff --git a/lib/isc/task_api.c b/lib/isc/task_api.c
index 551d0d37..06a8d24 100644
--- a/lib/isc/task_api.c
+++ b/lib/isc/task_api.c
@@ -187,6 +187,17 @@ isc_task_purge(isc_task_t *task, void *sender, isc_eventtype_t type, void *tag)
 	return (task->methods->purgeevents(task, sender, type, tag));
 }
 
+void
+isc_taskmgr_setexcltask(isc_taskmgr_t *mgr, isc_task_t *task) {
+	REQUIRE(ISCAPI_TASK_VALID(task));
+	return (mgr->methods->setexcltask(mgr, task));
+}
+
+isc_result_t
+isc_taskmgr_excltask(isc_taskmgr_t *mgr, isc_task_t **taskp) {
+	return (mgr->methods->excltask(mgr, taskp));
+}
+
 isc_result_t
 isc_task_beginexclusive(isc_task_t *task) {
 	REQUIRE(ISCAPI_TASK_VALID(task));
diff --git a/lib/isc/unix/Makefile.in b/lib/isc/unix/Makefile.in
index 9884ca9..c1411cb 100644
--- a/lib/isc/unix/Makefile.in
+++ b/lib/isc/unix/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2004, 2007, 2009  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2004, 2007, 2009, 2012  Internet Systems Consortium, Inc. ("ISC")
 # Copyright (C) 1998-2001  Internet Software Consortium.
 #
 # Permission to use, copy, modify, and/or distribute this software for any
diff --git a/lib/isc/unix/file.c b/lib/isc/unix/file.c
index 0538761..99c02ec 100644
--- a/lib/isc/unix/file.c
+++ b/lib/isc/unix/file.c
@@ -98,6 +98,20 @@ file_stats(const char *file, struct stat *stats) {
 }
 
 isc_result_t
+isc_file_mode(const char *file, mode_t *modep) {
+	isc_result_t result;
+	struct stat stats;
+
+	REQUIRE(modep != NULL);
+
+	result = file_stats(file, &stats);
+	if (result == ISC_R_SUCCESS)
+		*modep = (stats.st_mode & 07777);
+
+	return (result);
+}
+
+isc_result_t
 isc_file_getmodtime(const char *file, isc_time_t *time) {
 	isc_result_t result;
 	struct stat stats;
@@ -313,6 +327,23 @@ isc_file_openuniquemode(char *templet, int mode, FILE **fp) {
 }
 
 isc_result_t
+isc_file_bopenunique(char *templet, FILE **fp) {
+	int mode = S_IWUSR|S_IRUSR|S_IRGRP|S_IWGRP|S_IROTH|S_IWOTH;
+	return (isc_file_openuniquemode(templet, mode, fp));
+}
+
+isc_result_t
+isc_file_bopenuniqueprivate(char *templet, FILE **fp) {
+	int mode = S_IWUSR|S_IRUSR;
+	return (isc_file_openuniquemode(templet, mode, fp));
+}
+
+isc_result_t
+isc_file_bopenuniquemode(char *templet, int mode, FILE **fp) {
+	return (isc_file_openuniquemode(templet, mode, fp));
+}
+
+isc_result_t
 isc_file_remove(const char *filename) {
 	int r;
 
diff --git a/lib/isc/unix/include/Makefile.in b/lib/isc/unix/include/Makefile.in
index 0303ab1..46c243e 100644
--- a/lib/isc/unix/include/Makefile.in
+++ b/lib/isc/unix/include/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2004, 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2004, 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 # Copyright (C) 1998-2001  Internet Software Consortium.
 #
 # Permission to use, copy, modify, and/or distribute this software for any
diff --git a/lib/isc/unix/include/isc/Makefile.in b/lib/isc/unix/include/isc/Makefile.in
index 2f4d216..d3b5084 100644
--- a/lib/isc/unix/include/isc/Makefile.in
+++ b/lib/isc/unix/include/isc/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2004, 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2004, 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 # Copyright (C) 1998-2001  Internet Software Consortium.
 #
 # Permission to use, copy, modify, and/or distribute this software for any
diff --git a/lib/isc/x86_32/Makefile.in b/lib/isc/x86_32/Makefile.in
index 324db07..9c24cdf 100644
--- a/lib/isc/x86_32/Makefile.in
+++ b/lib/isc/x86_32/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/isc/x86_32/include/Makefile.in b/lib/isc/x86_32/include/Makefile.in
index f1d8bdd..e399559 100644
--- a/lib/isc/x86_32/include/Makefile.in
+++ b/lib/isc/x86_32/include/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/isc/x86_32/include/isc/Makefile.in b/lib/isc/x86_32/include/isc/Makefile.in
index 5f116ca..4927e21 100644
--- a/lib/isc/x86_32/include/isc/Makefile.in
+++ b/lib/isc/x86_32/include/isc/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/isc/x86_64/Makefile.in b/lib/isc/x86_64/Makefile.in
index 324db07..9c24cdf 100644
--- a/lib/isc/x86_64/Makefile.in
+++ b/lib/isc/x86_64/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/isc/x86_64/include/Makefile.in b/lib/isc/x86_64/include/Makefile.in
index f1d8bdd..e399559 100644
--- a/lib/isc/x86_64/include/Makefile.in
+++ b/lib/isc/x86_64/include/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/isc/x86_64/include/isc/Makefile.in b/lib/isc/x86_64/include/isc/Makefile.in
index f33ae99..9a988bb 100644
--- a/lib/isc/x86_64/include/isc/Makefile.in
+++ b/lib/isc/x86_64/include/isc/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 #
 # Permission to use, copy, modify, and/or distribute this software for any
 # purpose with or without fee is hereby granted, provided that the above
diff --git a/lib/isccc/api b/lib/isccc/api
index a629bf7..ba19dd9 100644
--- a/lib/isccc/api
+++ b/lib/isccc/api
@@ -4,5 +4,5 @@
 # 9.8: 80-89
 # 9.9: 90-109
 LIBINTERFACE = 80
-LIBREVISION = 1
+LIBREVISION = 2
 LIBAGE = 0
diff --git a/lib/isccc/cc.c b/lib/isccc/cc.c
index b549d6cb..1ab9479 100644
--- a/lib/isccc/cc.c
+++ b/lib/isccc/cc.c
@@ -399,8 +399,6 @@ table_fromwire(isccc_region_t *source, isccc_region_t *secret,
 		first_tag = ISC_FALSE;
 	}
 
-	*alistp = alist;
-
 	if (secret != NULL) {
 		if (checksum_rstart != NULL)
 			result = verify(alist, checksum_rstart,
@@ -412,7 +410,9 @@ table_fromwire(isccc_region_t *source, isccc_region_t *secret,
 		result = ISC_R_SUCCESS;
 
  bad:
-	if (result != ISC_R_SUCCESS)
+	if (result == ISC_R_SUCCESS)
+		*alistp = alist;
+	else
 		isccc_sexpr_free(&alist);
 
 	return (result);
diff --git a/lib/isccc/include/Makefile.in b/lib/isccc/include/Makefile.in
index 9f727c3..6b222a5 100644
--- a/lib/isccc/include/Makefile.in
+++ b/lib/isccc/include/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2004, 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2004, 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 # Copyright (C) 2001  Internet Software Consortium.
 #
 # Permission to use, copy, modify, and/or distribute this software for any
diff --git a/lib/isccc/include/isccc/Makefile.in b/lib/isccc/include/isccc/Makefile.in
index ae5bec7..c4af19a 100644
--- a/lib/isccc/include/isccc/Makefile.in
+++ b/lib/isccc/include/isccc/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2004, 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2004, 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 # Copyright (C) 2001  Internet Software Consortium.
 #
 # Permission to use, copy, modify, and/or distribute this software for any
diff --git a/lib/isccfg/api b/lib/isccfg/api
index 9f3d38c..cde1e2f 100644
--- a/lib/isccfg/api
+++ b/lib/isccfg/api
@@ -4,5 +4,5 @@
 # 9.8: 80-89
 # 9.9: 90-109
 LIBINTERFACE = 82
-LIBREVISION = 2
+LIBREVISION = 3
 LIBAGE = 0
diff --git a/lib/isccfg/include/Makefile.in b/lib/isccfg/include/Makefile.in
index 1f24003..5c6976a 100644
--- a/lib/isccfg/include/Makefile.in
+++ b/lib/isccfg/include/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2004, 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2004, 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 # Copyright (C) 2001  Internet Software Consortium.
 #
 # Permission to use, copy, modify, and/or distribute this software for any
diff --git a/lib/isccfg/include/isccfg/Makefile.in b/lib/isccfg/include/isccfg/Makefile.in
index a6fd412..211583a 100644
--- a/lib/isccfg/include/isccfg/Makefile.in
+++ b/lib/isccfg/include/isccfg/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2004, 2005, 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2004, 2005, 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 # Copyright (C) 2001, 2002  Internet Software Consortium.
 #
 # Permission to use, copy, modify, and/or distribute this software for any
diff --git a/lib/isccfg/namedconf.c b/lib/isccfg/namedconf.c
index 3f9454d..4d09f11 100644
--- a/lib/isccfg/namedconf.c
+++ b/lib/isccfg/namedconf.c
@@ -1014,66 +1014,60 @@ static cfg_type_t cfg_type_masterformat = {
 
 
 
-/*
+/*%
  *  response-policy {
  *	zone <string> [ policy (given|disabled|passthru|
- *					nxdomain|nodata|cname <domain> ) ];
- *  };
- *
- * this is a chimera of doc_optional_keyvalue() and cfg_doc_enum()
+ *					nxdomain|nodata|cname <domain> ) ]
+ *		      [ recursive-only yes|no ]
+ *		      [ max-policy-ttl number ] ;
+ *  } [ recursive-only yes|no ] [ break-dnssec yes|no ]
+ *	[ max-policy-ttl number ] ;
  */
+
 static void
-doc_rpz_policies(cfg_printer_t *pctx, const cfg_type_t *type) {
-	const keyword_type_t *kw;
+doc_rpz_policy(cfg_printer_t *pctx, const cfg_type_t *type) {
 	const char * const *p;
-
-	kw = type->of;
-	cfg_print_chars(pctx, "[ ", 2);
-	cfg_print_cstr(pctx, kw->name);
-	cfg_print_chars(pctx, " ", 1);
-
+	/*
+	 * This is cfg_doc_enum() without the trailing " )".
+	 */
 	cfg_print_chars(pctx, "( ", 2);
-	for (p = kw->type->of; *p != NULL; p++) {
+	for (p = type->of; *p != NULL; p++) {
 		cfg_print_cstr(pctx, *p);
 		if (p[1] != NULL)
 			cfg_print_chars(pctx, " | ", 3);
 	}
 }
 
-/*
- * print_qstring() from parser.c
- */
-static void
-print_rpz_cname(cfg_printer_t *pctx, const cfg_obj_t *obj)
-{
-	cfg_print_chars(pctx, "\"", 1);
-	cfg_print_ustring(pctx, obj);
-	cfg_print_chars(pctx, "\"", 1);
-}
-
 static void
 doc_rpz_cname(cfg_printer_t *pctx, const cfg_type_t *type) {
 	cfg_doc_terminal(pctx, type);
-	cfg_print_chars(pctx, " ) ]", 4);
+	cfg_print_chars(pctx, " )", 2);
 }
 
+/*
+ * Parse
+ *	given|disabled|passthru|nxdomain|nodata|cname <domain>
+ */
 static isc_result_t
-parse_rpz(cfg_parser_t *pctx, const cfg_type_t *type, cfg_obj_t **ret) {
+cfg_parse_rpz_policy(cfg_parser_t *pctx, const cfg_type_t *type,
+		     cfg_obj_t **ret)
+{
 	isc_result_t result;
-	cfg_obj_t *obj = NULL;
-	const cfg_tuplefielddef_t *fields = type->of;
+	cfg_obj_t *obj;
+	const cfg_tuplefielddef_t *fields;
 
 	CHECK(cfg_create_tuple(pctx, type, &obj));
+
+	fields = type->of;
 	CHECK(cfg_parse_obj(pctx, fields[0].type, &obj->value.tuple[0]));
-	CHECK(cfg_parse_obj(pctx, fields[1].type, &obj->value.tuple[1]));
 	/*
 	 * parse cname domain only after "policy cname"
 	 */
-	if (cfg_obj_isvoid(obj->value.tuple[1]) ||
-	    strcasecmp("cname", cfg_obj_asstring(obj->value.tuple[1]))) {
-		CHECK(cfg_parse_void(pctx, NULL, &obj->value.tuple[2]));
+	if (strcasecmp("cname", cfg_obj_asstring(obj->value.tuple[0])) != 0) {
+		CHECK(cfg_parse_void(pctx, NULL, &obj->value.tuple[1]));
 	} else {
-		CHECK(cfg_parse_obj(pctx, fields[2].type, &obj->value.tuple[2]));
+		CHECK(cfg_parse_obj(pctx, fields[1].type,
+				    &obj->value.tuple[1]));
 	}
 
 	*ret = obj;
@@ -1084,50 +1078,160 @@ cleanup:
 	return (result);
 }
 
+/*
+ * Parse a tuple consisting of any kind of  required field followed
+ * by 2 or more optional keyvalues that can be in any order.
+ */
+static isc_result_t
+cfg_parse_kv_tuple(cfg_parser_t *pctx, const cfg_type_t *type, cfg_obj_t **ret) {
+	const cfg_tuplefielddef_t *fields, *f;
+	cfg_obj_t *obj;
+	int fn;
+	isc_result_t result;
+
+	obj = NULL;
+	CHECK(cfg_create_tuple(pctx, type, &obj));
+
+	/*
+	 * The zone first field is required and always first.
+	 */
+	fields = type->of;
+	CHECK(cfg_parse_obj(pctx, fields[0].type, &obj->value.tuple[0]));
+
+	for (;;) {
+		CHECK(cfg_peektoken(pctx, CFG_LEXOPT_QSTRING));
+		if (pctx->token.type != isc_tokentype_string)
+			break;
+
+		for (fn = 1, f = &fields[1]; ; ++fn, ++f) {
+			if (f->name == NULL) {
+				cfg_parser_error(pctx, 0, "unexpected '%s'",
+						 TOKEN_STRING(pctx));
+				result = ISC_R_UNEXPECTEDTOKEN;
+				goto cleanup;
+			}
+			if (obj->value.tuple[fn] == NULL &&
+			    strcasecmp(f->name, TOKEN_STRING(pctx)) == 0)
+				break;
+		}
+
+		CHECK(cfg_gettoken(pctx, 0));
+		CHECK(cfg_parse_obj(pctx, f->type, &obj->value.tuple[fn]));
+	}
+
+	for (fn = 1, f = &fields[1]; f->name != NULL; ++fn, ++f) {
+		if (obj->value.tuple[fn] == NULL)
+			CHECK(cfg_parse_void(pctx, NULL,
+					     &obj->value.tuple[fn]));
+	}
+
+	*ret = obj;
+	return (ISC_R_SUCCESS);
+
+cleanup:
+	CLEANUP_OBJ(obj);
+	return (result);
+}
+
+static void
+cfg_print_kv_tuple(cfg_printer_t *pctx, const cfg_obj_t *obj) {
+	unsigned int i;
+	const cfg_tuplefielddef_t *fields, *f;
+	const cfg_obj_t *fieldobj;
+
+	fields = obj->type->of;
+	for (f = fields, i = 0; f->name != NULL; f++, i++) {
+		fieldobj = obj->value.tuple[i];
+		if (fieldobj->type->print == cfg_print_void)
+			continue;
+		if (i != 0) {
+			cfg_print_chars(pctx, " ", 1);
+			cfg_print_cstr(pctx, f->name);
+			cfg_print_chars(pctx, " ", 1);
+		}
+		cfg_print_obj(pctx, fieldobj);
+	}
+}
+
+static void
+cfg_doc_kv_tuple(cfg_printer_t *pctx, const cfg_type_t *type) {
+	const cfg_tuplefielddef_t *fields, *f;
+
+	fields = type->of;
+	for (f = fields; f->name != NULL; f++) {
+		if (f != fields) {
+			cfg_print_chars(pctx, " [ ", 3);
+			cfg_print_cstr(pctx, f->name);
+			if (f->type->doc != cfg_doc_void)
+				cfg_print_chars(pctx, " ", 1);
+		}
+		cfg_doc_obj(pctx, f->type);
+		if (f != fields)
+			cfg_print_chars(pctx, " ]", 2);
+	}
+}
+
+static keyword_type_t zone_kw = {"zone", &cfg_type_qstring};
+static cfg_type_t cfg_type_rpz_zone = {
+	"zone", parse_keyvalue, print_keyvalue,
+	doc_keyvalue, &cfg_rep_string,
+	&zone_kw
+};
 static const char *rpz_policies[] = {
 	"given", "disabled", "passthru", "no-op", "nxdomain", "nodata",
 	"cname", NULL
 };
-static cfg_type_t cfg_type_rpz_policylist = {
-	"policies", cfg_parse_enum, cfg_print_ustring, cfg_doc_enum,
-	&cfg_rep_string, &rpz_policies
+static cfg_type_t cfg_type_rpz_policy_name = {
+	"policy name", cfg_parse_enum, cfg_print_ustring,
+	doc_rpz_policy, &cfg_rep_string,
+	&rpz_policies
 };
-static keyword_type_t rpz_policies_kw = {
-	"policy", &cfg_type_rpz_policylist
+static cfg_type_t cfg_type_rpz_cname = {
+	"quoted_string", cfg_parse_astring, NULL,
+	doc_rpz_cname, &cfg_rep_string,
+	NULL
 };
-static cfg_type_t cfg_type_rpz_policy = {
-	"optional_policy", parse_optional_keyvalue, print_keyvalue,
-	doc_rpz_policies, &cfg_rep_string, &rpz_policies_kw
+static cfg_tuplefielddef_t rpz_policy_fields[] = {
+	{ "policy name", &cfg_type_rpz_policy_name, 0 },
+	{ "cname", &cfg_type_rpz_cname, 0 },
+	{ NULL, NULL, 0 }
 };
-static cfg_type_t cfg_type_cname = {
-	"domain", cfg_parse_astring, print_rpz_cname, doc_rpz_cname,
-	&cfg_rep_string, NULL
+static cfg_type_t cfg_type_rpz_policy = {
+	"policy tuple", cfg_parse_rpz_policy,
+	cfg_print_tuple, cfg_doc_tuple, &cfg_rep_tuple,
+	rpz_policy_fields
 };
-static cfg_tuplefielddef_t rpzone_fields[] = {
-	{ "name", &cfg_type_astring, 0 },
+static cfg_tuplefielddef_t rpz_zone_fields[] = {
+	{ "zone name", &cfg_type_rpz_zone, 0 },
 	{ "policy", &cfg_type_rpz_policy, 0 },
-	{ "cname", &cfg_type_cname, 0 },
+	{ "recursive-only", &cfg_type_boolean, 0 },
+	{ "max-policy-ttl", &cfg_type_uint32, 0 },
 	{ NULL, NULL, 0 }
 };
-static cfg_type_t cfg_type_rpzone = {
-	"rpzone", parse_rpz, cfg_print_tuple, cfg_doc_tuple,
-	&cfg_rep_tuple, rpzone_fields
+static cfg_type_t cfg_type_rpz_tuple = {
+	"rpz tuple", cfg_parse_kv_tuple,
+	cfg_print_kv_tuple, cfg_doc_kv_tuple, &cfg_rep_tuple,
+	rpz_zone_fields
 };
-static cfg_clausedef_t rpz_clauses[] = {
-	{ "zone", &cfg_type_rpzone, CFG_CLAUSEFLAG_MULTI },
-	{ NULL, NULL, 0 }
+static cfg_type_t cfg_type_rpz_list = {
+	"zone list", cfg_parse_bracketed_list, cfg_print_bracketed_list,
+	cfg_doc_bracketed_list, &cfg_rep_list,
+	&cfg_type_rpz_tuple
 };
-static cfg_clausedef_t *rpz_clausesets[] = {
-	rpz_clauses,
-	NULL
+static cfg_tuplefielddef_t rpz_fields[] = {
+	{ "zone list", &cfg_type_rpz_list, 0 },
+	{ "recursive-only", &cfg_type_boolean, 0 },
+	{ "break-dnssec", &cfg_type_boolean, 0 },
+	{ "max-policy-ttl", &cfg_type_uint32, 0 },
+	{ NULL, NULL, 0 }
 };
 static cfg_type_t cfg_type_rpz = {
-	"rpz", cfg_parse_map, cfg_print_map, cfg_doc_map,
-	&cfg_rep_map, rpz_clausesets
+	"rpz", cfg_parse_kv_tuple,
+	cfg_print_kv_tuple, cfg_doc_kv_tuple, &cfg_rep_tuple,
+	rpz_fields
 };
 
 
-
 /*%
  * dnssec-lookaside
  */
diff --git a/lib/lwres/Makefile.in b/lib/lwres/Makefile.in
index 858b325..0cf873b 100644
--- a/lib/lwres/Makefile.in
+++ b/lib/lwres/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2004, 2005, 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2004, 2005, 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 # Copyright (C) 2000, 2001  Internet Software Consortium.
 #
 # Permission to use, copy, modify, and/or distribute this software for any
diff --git a/lib/lwres/api b/lib/lwres/api
index ba19dd9..1e51baa 100644
--- a/lib/lwres/api
+++ b/lib/lwres/api
@@ -4,5 +4,5 @@
 # 9.8: 80-89
 # 9.9: 90-109
 LIBINTERFACE = 80
-LIBREVISION = 2
+LIBREVISION = 3
 LIBAGE = 0
diff --git a/lib/lwres/getaddrinfo.c b/lib/lwres/getaddrinfo.c
index 8e916f3..811a2fe 100644
--- a/lib/lwres/getaddrinfo.c
+++ b/lib/lwres/getaddrinfo.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (C) 2004-2008  Internet Systems Consortium, Inc. ("ISC")
+ * Copyright (C) 2004-2008, 2012  Internet Systems Consortium, Inc. ("ISC")
  * Copyright (C) 1999-2001  Internet Software Consortium.
  *
  * This code is derived from software contributed to ISC by
@@ -398,7 +398,7 @@ lwres_getaddrinfo(const char *hostname, const char *servname,
 				goto inet6_addr;
 			}
 			addrsize = sizeof(struct in_addr);
-			addroff = (char *)(&SIN(0)->sin_addr) - (char *)0;
+			addroff = offsetof(struct sockaddr_in, sin_addr);
 			family = AF_INET;
 			goto common;
 #ifdef LWRES_HAVE_SIN6_SCOPE_ID
@@ -408,7 +408,7 @@ lwres_getaddrinfo(const char *hostname, const char *servname,
 			if (family && family != AF_INET6)
 				return (EAI_NONAME);
 			addrsize = sizeof(struct in6_addr);
-			addroff = (char *)(&SIN6(0)->sin6_addr) - (char *)0;
+			addroff = offsetof(struct sockaddr_in6, sin6_addr);
 			family = AF_INET6;
 			goto common;
 #endif
@@ -417,7 +417,7 @@ lwres_getaddrinfo(const char *hostname, const char *servname,
 				return (EAI_NONAME);
 		inet6_addr:
 			addrsize = sizeof(struct in6_addr);
-			addroff = (char *)(&SIN6(0)->sin6_addr) - (char *)0;
+			addroff = offsetof(struct sockaddr_in6, sin6_addr);
 			family = AF_INET6;
 
 		common:
diff --git a/lib/lwres/include/Makefile.in b/lib/lwres/include/Makefile.in
index 4750a5e..6c3d07f 100644
--- a/lib/lwres/include/Makefile.in
+++ b/lib/lwres/include/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2004, 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2004, 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 # Copyright (C) 2000, 2001  Internet Software Consortium.
 #
 # Permission to use, copy, modify, and/or distribute this software for any
diff --git a/lib/lwres/include/lwres/Makefile.in b/lib/lwres/include/lwres/Makefile.in
index fc3126f..36b8b03 100644
--- a/lib/lwres/include/lwres/Makefile.in
+++ b/lib/lwres/include/lwres/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2004, 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2004, 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 # Copyright (C) 2000, 2001  Internet Software Consortium.
 #
 # Permission to use, copy, modify, and/or distribute this software for any
diff --git a/lib/lwres/man/Makefile.in b/lib/lwres/man/Makefile.in
index cb723c2..80db9f2 100644
--- a/lib/lwres/man/Makefile.in
+++ b/lib/lwres/man/Makefile.in
@@ -1,4 +1,4 @@
-# Copyright (C) 2004, 2007  Internet Systems Consortium, Inc. ("ISC")
+# Copyright (C) 2004, 2007, 2012  Internet Systems Consortium, Inc. ("ISC")
 # Copyright (C) 2001  Internet Software Consortium.
 #
 # Permission to use, copy, modify, and/or distribute this software for any
-- 
cgit v1.1


From 3b0b9fa5f9dce3324c28e6b0b1afb52c1702a562 Mon Sep 17 00:00:00 2001
From: bapt <bapt@FreeBSD.org>
Date: Wed, 5 Dec 2012 20:28:44 +0000
Subject: remove redundant yyparse declarations

PR:		conf/174079
Submitted by:	Garrett Cooper <yanegomi@gmail.com>
---
 lib/libipsec/policy_parse.y | 1 -
 1 file changed, 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libipsec/policy_parse.y b/lib/libipsec/policy_parse.y
index 9e2f979..46e54e5 100644
--- a/lib/libipsec/policy_parse.y
+++ b/lib/libipsec/policy_parse.y
@@ -88,7 +88,6 @@ static caddr_t policy_parse(char *msg, int msglen);
 
 extern void __policy__strbuffer__init__(char *msg);
 extern void __policy__strbuffer__free__(void);
-extern int yyparse(void);
 extern int yylex(void);
 
 extern char *__libipsecyytext;	/*XXX*/
-- 
cgit v1.1


From 0bf300ea4be2e21bf74b325c7549c9812ac56ddb Mon Sep 17 00:00:00 2001
From: sem <sem@FreeBSD.org>
Date: Thu, 6 Dec 2012 19:00:37 +0000
Subject: - Rewrite radius servers traversal algorithm. - Add functions for
 working with IPv6 attributes.

Approved by:	ae
---
 lib/libradius/Makefile         |   1 +
 lib/libradius/libradius.3      |  20 +++-
 lib/libradius/radius.conf.5    |  16 +++-
 lib/libradius/radlib.c         | 202 +++++++++++++++++++++++++++++++++--------
 lib/libradius/radlib.h         |   5 +
 lib/libradius/radlib_private.h |   9 +-
 lib/libradius/radlib_vs.h      |   1 +
 7 files changed, 212 insertions(+), 42 deletions(-)

(limited to 'lib')

diff --git a/lib/libradius/Makefile b/lib/libradius/Makefile
index d71d8f2..cb68eaa 100644
--- a/lib/libradius/Makefile
+++ b/lib/libradius/Makefile
@@ -35,6 +35,7 @@ MAN=		libradius.3 radius.conf.5
 
 MLINKS+=libradius.3 rad_acct_open.3 \
 	libradius.3 rad_add_server.3 \
+	libradius.3 rad_add_server_ex.3 \
 	libradius.3 rad_auth_open.3 \
 	libradius.3 rad_bind_to.3 \
 	libradius.3 rad_close.3 \
diff --git a/lib/libradius/libradius.3 b/lib/libradius/libradius.3
index ae0ee24..253659c 100644
--- a/lib/libradius/libradius.3
+++ b/lib/libradius/libradius.3
@@ -37,6 +37,8 @@
 .Fn rad_acct_open "void"
 .Ft int
 .Fn rad_add_server "struct rad_handle *h" "const char *host" "int port" "const char *secret" "int timeout" "int max_tries"
+.Ft int
+.Fn rad_add_server_ex "struct rad_handle *h" "const char *host" "int port" "const char *secret" "int timeout" "int max_tries" "int dead_time" "struct in_addr bindto"
 .Ft "struct rad_handle *"
 .Fn rad_auth_open "void"
 .Ft void
@@ -153,7 +155,12 @@ is used.
 returns 0 on success, or \-1 if an error occurs.
 .Pp
 The library can also be configured programmatically by calls to
-.Fn rad_add_server .
+.Fn rad_add_server
+or
+.Fn rad_add_server_ex .
+.Fn rad_add_server
+is a backward compatible function, implemented via
+.Fn rad_add_server_ex .
 The
 .Fa host
 parameter specifies the server host, either as a fully qualified
@@ -188,11 +195,20 @@ The maximum number of repeated
 requests to make before giving up is passed into the
 .Fa max_tries
 parameter.
+Time interval in seconds when the server will not be requested
+if it is marked as dead (did not answer on the last try) set with
+.Fa dead_time
+parameter.
+.Fa bindto
+parameter is an IP address on the multihomed host that is used as
+a source address for all requests.
 .Fn rad_add_server
 returns 0 on success, or \-1 if an error occurs.
 .Pp
 .Fn rad_add_server
-may be called multiple times, and it may be used together with
+or
+.Fn rad_add_server_ex
+may be called multiple times, and they may be used together with
 .Fn rad_config .
 At most 10 servers may be specified.
 When multiple servers are given, they are tried in round-robin
diff --git a/lib/libradius/radius.conf.5 b/lib/libradius/radius.conf.5
index 6ac84e0..6f89c3a 100644
--- a/lib/libradius/radius.conf.5
+++ b/lib/libradius/radius.conf.5
@@ -44,7 +44,7 @@ Leading
 white space is ignored, as are empty lines and lines containing
 only comments.
 .Pp
-A RADIUS server is described by three to five fields on a line:
+A RADIUS server is described by three to seven fields on a line:
 .Pp
 .Bl -item -offset indent -compact
 .It
@@ -57,6 +57,10 @@ Shared secret
 Timeout
 .It
 Retries
+.It
+Dead time
+.It
+Bind address
 .El
 .Pp
 The fields are separated by white space.
@@ -139,6 +143,13 @@ If omitted, it defaults to 3 attempts.
 Note,
 this is the total number of attempts and not the number of retries.
 .Pp
+The sixth field contains a decimal integer specifying a time interval
+in seconds when the server will not requested if it was inaccessible
+on the last try. 0 means ask always.
+.Pp
+The seventh field contains an IP address on multihomed host. All
+requests will be binded to this IP.
+.Pp
 Up to 10 RADIUS servers may be specified for each service type.
 The servers are tried in
 round-robin fashion, until a valid response is received or the
@@ -161,6 +172,9 @@ acct  radius1.domain.com  OurLittleSecret
 # timeout and maximum tries:
 auth  auth.domain.com:1645  "I can't see you"  5  4
 
+# As above but set dead time and bind address
+auth  auth.domain.com:1645  "I can't see you"  5  4  60  192.168.1.8
+
 # A server specified by its IP address:
 auth  192.168.27.81  $X*#..38947ax-+=
 .Ed
diff --git a/lib/libradius/radlib.c b/lib/libradius/radlib.c
index 46a9b58..0b67b0f 100644
--- a/lib/libradius/radlib.c
+++ b/lib/libradius/radlib.c
@@ -43,6 +43,8 @@ __FBSDID("$FreeBSD$");
 #include <md5.h>
 #endif
 
+#define	MAX_FIELDS	7
+
 /* We need the MPPE_KEY_LEN define */
 #include <netgraph/ng_mppc.h>
 
@@ -379,6 +381,18 @@ int
 rad_add_server(struct rad_handle *h, const char *host, int port,
     const char *secret, int timeout, int tries)
 {
+    	struct in_addr bindto;
+	bindto.s_addr = INADDR_ANY;
+
+	return rad_add_server_ex(h, host, port, secret, timeout, tries,
+		DEAD_TIME, &bindto);
+}
+
+int
+rad_add_server_ex(struct rad_handle *h, const char *host, int port,
+    const char *secret, int timeout, int tries, int dead_time,
+    struct in_addr *bindto)
+{
 	struct rad_server *srvp;
 
 	if (h->num_servers >= MAXSERVERS) {
@@ -421,6 +435,10 @@ rad_add_server(struct rad_handle *h, const char *host, int port,
 	srvp->timeout = timeout;
 	srvp->max_tries = tries;
 	srvp->num_tries = 0;
+	srvp->is_dead = 0;
+	srvp->dead_time = dead_time;
+	srvp->next_probe = 0;
+	srvp->bindto = bindto->s_addr;
 	h->num_servers++;
 	return 0;
 }
@@ -441,6 +459,13 @@ rad_close(struct rad_handle *h)
 	free(h);
 }
 
+void
+rad_bind_to(struct rad_handle *h, in_addr_t addr)
+{
+
+	h->bindto = addr;
+}
+
 int
 rad_config(struct rad_handle *h, const char *path)
 {
@@ -468,11 +493,15 @@ rad_config(struct rad_handle *h, const char *path)
 		char *secret;
 		char *timeout_str;
 		char *maxtries_str;
+		char *dead_time_str;
+		char *bindto_str;
 		char *end;
 		char *wanttype;
 		unsigned long timeout;
 		unsigned long maxtries;
+		unsigned long dead_time;
 		int port;
+		struct in_addr bindto;
 		int i;
 
 		linenum++;
@@ -491,7 +520,7 @@ rad_config(struct rad_handle *h, const char *path)
 		buf[len - 1] = '\0';
 
 		/* Extract the fields from the line. */
-		nfields = split(buf, fields, 5, msg, sizeof msg);
+		nfields = split(buf, fields, MAX_FIELDS, msg, sizeof msg);
 		if (nfields == -1) {
 			generr(h, "%s:%d: %s", path, linenum, msg);
 			retval = -1;
@@ -507,7 +536,7 @@ rad_config(struct rad_handle *h, const char *path)
 		 */
 		if (strcmp(fields[0], "auth") != 0 &&
 		    strcmp(fields[0], "acct") != 0) {
-			if (nfields >= 5) {
+			if (nfields >= MAX_FIELDS) {
 				generr(h, "%s:%d: invalid service type", path,
 				    linenum);
 				retval = -1;
@@ -529,6 +558,8 @@ rad_config(struct rad_handle *h, const char *path)
 		secret = fields[2];
 		timeout_str = fields[3];
 		maxtries_str = fields[4];
+		dead_time_str = fields[5];
+		bindto_str = fields[6];
 
 		/* Ignore the line if it is for the wrong service type. */
 		wanttype = h->type == RADIUS_AUTH ? "auth" : "acct";
@@ -570,8 +601,30 @@ rad_config(struct rad_handle *h, const char *path)
 		} else
 			maxtries = MAXTRIES;
 
-		if (rad_add_server(h, host, port, secret, timeout, maxtries) ==
-		    -1) {
+		if (dead_time_str != NULL) {
+			dead_time = strtoul(dead_time_str, &end, 10);
+			if (*end != '\0') {
+				generr(h, "%s:%d: invalid dead_time", path,
+				    linenum);
+				retval = -1;
+				break;
+			}
+		} else
+		    	dead_time = DEAD_TIME;
+
+		if (bindto_str != NULL) {
+		    	bindto.s_addr = inet_addr(bindto_str);
+			if (bindto.s_addr == INADDR_NONE) {
+				generr(h, "%s:%d: invalid bindto", path,
+				    linenum);
+				retval = -1;
+				break;
+			}
+		} else
+		    	bindto.s_addr = INADDR_ANY;
+
+		if (rad_add_server_ex(h, host, port, secret, timeout, maxtries,
+			    dead_time, &bindto) == -1) {
 			strcpy(msg, h->errmsg);
 			generr(h, "%s:%d: %s", path, linenum, msg);
 			retval = -1;
@@ -596,7 +649,9 @@ int
 rad_continue_send_request(struct rad_handle *h, int selected, int *fd,
                           struct timeval *tv)
 {
-	int n;
+	int n, cur_srv;
+	time_t now;
+	struct sockaddr_in sin;
 
 	if (h->type == RADIUS_SERVER) {
 		generr(h, "denied function call");
@@ -621,19 +676,61 @@ rad_continue_send_request(struct rad_handle *h, int selected, int *fd,
 		}
 	}
 
-	if (h->try == h->total_tries) {
-		generr(h, "No valid RADIUS responses received");
-		return -1;
-	}
-
 	/*
          * Scan round-robin to the next server that has some
          * tries left.  There is guaranteed to be one, or we
          * would have exited this loop by now.
 	 */
-	while (h->servers[h->srv].num_tries >= h->servers[h->srv].max_tries)
-		if (++h->srv >= h->num_servers)
-			h->srv = 0;
+	cur_srv = h->srv;
+	now = time(NULL);
+	if (h->servers[h->srv].num_tries >= h->servers[h->srv].max_tries) {
+		/* Set next probe time for this server */
+		if (h->servers[h->srv].dead_time) {
+			h->servers[h->srv].is_dead = 1;
+			h->servers[h->srv].next_probe = now +
+			    h->servers[h->srv].dead_time;
+		}
+		do {
+		    	h->srv++;
+			if (h->srv >= h->num_servers)
+				h->srv = 0;
+			if (h->servers[h->srv].is_dead == 0)
+			    	break;
+			if (h->servers[h->srv].dead_time &&
+			    h->servers[h->srv].next_probe <= now) {
+			    	h->servers[h->srv].is_dead = 0;
+				h->servers[h->srv].num_tries = 0;
+				break;
+			}
+		} while (h->srv != cur_srv);
+
+		if (h->srv == cur_srv) {
+			generr(h, "No valid RADIUS responses received");
+			return (-1);
+		}
+	}
+
+	/* Rebind */
+	if (h->bindto != h->servers[h->srv].bindto) {
+	    	h->bindto = h->servers[h->srv].bindto;
+		close(h->fd);
+		if ((h->fd = socket(PF_INET, SOCK_DGRAM, IPPROTO_UDP)) == -1) {
+			generr(h, "Cannot create socket: %s", strerror(errno));
+			return -1;
+		}
+		memset(&sin, 0, sizeof sin);
+		sin.sin_len = sizeof sin;
+		sin.sin_family = AF_INET;
+		sin.sin_addr.s_addr = h->bindto;
+		sin.sin_port = 0;
+		if (bind(h->fd, (const struct sockaddr *)&sin,
+		    sizeof sin) == -1) {
+			generr(h, "bind: %s", strerror(errno));
+			close(h->fd);
+			h->fd = -1;
+			return (-1);
+		}
+	}
 
 	if (h->out[POS_CODE] == RAD_ACCESS_REQUEST) {
 		/* Insert the scrambled password into the request */
@@ -641,9 +738,11 @@ rad_continue_send_request(struct rad_handle *h, int selected, int *fd,
 			insert_scrambled_password(h, h->srv);
 	}
 	insert_message_authenticator(h, 0);
+
 	if (h->out[POS_CODE] != RAD_ACCESS_REQUEST) {
 		/* Insert the request authenticator into the request */
-		insert_request_authenticator(h, h->srv);
+		memset(&h->out[POS_AUTH], 0, LEN_AUTH);
+		insert_request_authenticator(h, 0);
 	}
 
 	/* Send the request */
@@ -654,7 +753,6 @@ rad_continue_send_request(struct rad_handle *h, int selected, int *fd,
 		tv->tv_sec = 1; /* Do not wait full timeout if send failed. */
 	else
 		tv->tv_sec = h->servers[h->srv].timeout;
-	h->try++;
 	h->servers[h->srv].num_tries++;
 	tv->tv_usec = 0;
 	*fd = h->fd;
@@ -740,6 +838,10 @@ rad_create_request(struct rad_handle *h, int code)
 		generr(h, "denied function call");
 		return (-1);
 	}
+	if (h->num_servers == 0) {
+	    	generr(h, "No RADIUS servers specified");
+		return (-1);
+	}
 	h->out[POS_CODE] = code;
 	h->out[POS_IDENT] = ++h->ident;
 	if (code == RAD_ACCESS_REQUEST) {
@@ -756,16 +858,9 @@ rad_create_request(struct rad_handle *h, int code)
 	clear_password(h);
 	h->authentic_pos = 0;
 	h->out_created = 1;
-	h->bindto = INADDR_ANY;
 	return 0;
 }
 
-void
-rad_bind_to(struct rad_handle *h, in_addr_t addr)
-{
-    h->bindto = addr;
-}
-
 int
 rad_create_response(struct rad_handle *h, int code)
 {
@@ -793,6 +888,15 @@ rad_cvt_addr(const void *data)
 	return value;
 }
 
+struct in6_addr
+rad_cvt_addr6(const void *data)
+{
+	struct in6_addr value;
+
+	memcpy(&value.s6_addr, data, sizeof value.s6_addr);
+	return value;
+}
+
 u_int32_t
 rad_cvt_int(const void *data)
 {
@@ -848,6 +952,8 @@ int
 rad_init_send_request(struct rad_handle *h, int *fd, struct timeval *tv)
 {
 	int srv;
+	time_t now;
+	struct sockaddr_in sin;
 
 	if (h->type == RADIUS_SERVER) {
 		generr(h, "denied function call");
@@ -855,8 +961,6 @@ rad_init_send_request(struct rad_handle *h, int *fd, struct timeval *tv)
 	}
 	/* Make sure we have a socket to use */
 	if (h->fd == -1) {
-		struct sockaddr_in sin;
-
 		if ((h->fd = socket(PF_INET, SOCK_DGRAM, IPPROTO_UDP)) == -1) {
 			generr(h, "Cannot create socket: %s", strerror(errno));
 			return -1;
@@ -902,21 +1006,30 @@ rad_init_send_request(struct rad_handle *h, int *fd, struct timeval *tv)
 	h->out[POS_LENGTH] = h->out_len >> 8;
 	h->out[POS_LENGTH+1] = h->out_len;
 
-	/*
-	 * Count the total number of tries we will make, and zero the
-	 * counter for each server.
-	 */
-	h->total_tries = 0;
-	for (srv = 0;  srv < h->num_servers;  srv++) {
-		h->total_tries += h->servers[srv].max_tries;
+	h->srv = 0;
+	now = time(NULL);
+	for (srv = 0;  srv < h->num_servers;  srv++)
 		h->servers[srv].num_tries = 0;
-	}
-	if (h->total_tries == 0) {
-		generr(h, "No RADIUS servers specified");
-		return -1;
+	/* Find a first good server. */
+	for (srv = 0;  srv < h->num_servers;  srv++) {
+		if (h->servers[srv].is_dead == 0)
+			break;
+		if (h->servers[srv].dead_time &&
+		    h->servers[srv].next_probe <= now) {
+		    	h->servers[srv].is_dead = 0;
+			break;
+		}
+		h->srv++;
 	}
 
-	h->try = h->srv = 0;
+	/* If all servers was dead on the last probe, try from beginning */
+	if (h->srv == h->num_servers) {
+		for (srv = 0;  srv < h->num_servers;  srv++) {
+		    	h->servers[srv].is_dead = 0;
+			h->servers[srv].next_probe = 0;
+		}
+		h->srv = 0;
+	}
 
 	return rad_continue_send_request(h, 0, fd, tv);
 }
@@ -946,6 +1059,7 @@ rad_auth_open(void)
 		h->type = RADIUS_AUTH;
 		h->out_created = 0;
 		h->eap_msg = 0;
+		h->bindto = INADDR_ANY;
 	}
 	return h;
 }
@@ -987,6 +1101,13 @@ rad_put_addr(struct rad_handle *h, int type, struct in_addr addr)
 }
 
 int
+rad_put_addr6(struct rad_handle *h, int type, struct in6_addr addr)
+{
+
+	return rad_put_attr(h, type, &addr.s6_addr, sizeof addr.s6_addr);
+}
+
+int
 rad_put_attr(struct rad_handle *h, int type, const void *value, size_t len)
 {
 	int result;
@@ -1229,6 +1350,15 @@ rad_put_vendor_addr(struct rad_handle *h, int vendor, int type,
 }
 
 int
+rad_put_vendor_addr6(struct rad_handle *h, int vendor, int type,
+    struct in6_addr addr)
+{
+
+	return (rad_put_vendor_attr(h, vendor, type, &addr.s6_addr,
+	    sizeof addr.s6_addr));
+}
+
+int
 rad_put_vendor_attr(struct rad_handle *h, int vendor, int type,
     const void *value, size_t len)
 {
diff --git a/lib/libradius/radlib.h b/lib/libradius/radlib.h
index f9e451b..a479318 100644
--- a/lib/libradius/radlib.h
+++ b/lib/libradius/radlib.h
@@ -194,6 +194,9 @@ __BEGIN_DECLS
 struct rad_handle	*rad_acct_open(void);
 int			 rad_add_server(struct rad_handle *,
 			    const char *, int, const char *, int, int);
+int			 rad_add_server_ex(struct rad_handle *,
+			    const char *, int, const char *, int, int,
+			    int, struct in_addr *);
 struct rad_handle	*rad_auth_open(void);
 void			 rad_bind_to(struct rad_handle *, in_addr_t);
 void			 rad_close(struct rad_handle *);
@@ -203,6 +206,7 @@ int			 rad_continue_send_request(struct rad_handle *, int,
 int			 rad_create_request(struct rad_handle *, int);
 int			 rad_create_response(struct rad_handle *, int);
 struct in_addr		 rad_cvt_addr(const void *);
+struct in6_addr		 rad_cvt_addr6(const void *);
 u_int32_t		 rad_cvt_int(const void *);
 char			*rad_cvt_string(const void *, size_t);
 int			 rad_get_attr(struct rad_handle *, const void **,
@@ -211,6 +215,7 @@ int			 rad_init_send_request(struct rad_handle *, int *,
 			    struct timeval *);
 struct rad_handle	*rad_open(void);  /* Deprecated, == rad_auth_open */
 int			 rad_put_addr(struct rad_handle *, int, struct in_addr);
+int			 rad_put_addr6(struct rad_handle *, int, struct in6_addr);
 int			 rad_put_attr(struct rad_handle *, int,
 			    const void *, size_t);
 int			 rad_put_int(struct rad_handle *, int, u_int32_t);
diff --git a/lib/libradius/radlib_private.h b/lib/libradius/radlib_private.h
index a76e594aa..bfbbbd1 100644
--- a/lib/libradius/radlib_private.h
+++ b/lib/libradius/radlib_private.h
@@ -46,6 +46,7 @@
 #define RADIUS_PORT		1812
 #define RADACCT_PORT		1813
 #define TIMEOUT			3	/* In seconds */
+#define	DEAD_TIME		0
 
 /* Limits */
 #define ERRSIZE		128		/* Maximum error message length */
@@ -68,6 +69,10 @@ struct rad_server {
 	int		 timeout;	/* Timeout in seconds */
 	int		 max_tries;	/* Number of tries before giving up */
 	int		 num_tries;	/* Number of tries so far */
+	int		 is_dead;	/* The server did not answer last time */
+	time_t		 dead_time;	/* Don't try this server for the time period if it is dead */
+	time_t		 next_probe;	/* Time of a next probe after failure */
+	in_addr_t	 bindto;	/* Bind to address */
 };
 
 struct rad_handle {
@@ -88,11 +93,9 @@ struct rad_handle {
 	unsigned char	 in[MSGSIZE];	/* Response received */
 	int		 in_len;	/* Length of response */
 	int		 in_pos;	/* Current position scanning attrs */
-	int		 total_tries;	/* How many requests we'll send */
-	int		 try;		/* How many requests we've sent */
 	int		 srv;		/* Server number we did last */
 	int		 type;		/* Handle type */
-	in_addr_t	 bindto;	/* Bind to address */
+	in_addr_t	 bindto;	/* Current bind address */
 };
 
 struct vendor_attribute {
diff --git a/lib/libradius/radlib_vs.h b/lib/libradius/radlib_vs.h
index 8b3a75e..0acfb0e 100644
--- a/lib/libradius/radlib_vs.h
+++ b/lib/libradius/radlib_vs.h
@@ -73,6 +73,7 @@ struct rad_handle;
 __BEGIN_DECLS
 int	 rad_get_vendor_attr(u_int32_t *, const void **, size_t *);
 int	 rad_put_vendor_addr(struct rad_handle *, int, int, struct in_addr);
+int	 rad_put_vendor_addr6(struct rad_handle *, int, int, struct in6_addr);
 int	 rad_put_vendor_attr(struct rad_handle *, int, int, const void *,
 	    size_t);
 int	 rad_put_vendor_int(struct rad_handle *, int, int, u_int32_t);
-- 
cgit v1.1


From 71dd34a67c96f1199178861ca3bf33b48f8fbaf5 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Fri, 7 Dec 2012 01:36:53 +0000
Subject: Fix build: reflect the increased field number.

---
 lib/libradius/radlib.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libradius/radlib.c b/lib/libradius/radlib.c
index 0b67b0f..55c6f5c 100644
--- a/lib/libradius/radlib.c
+++ b/lib/libradius/radlib.c
@@ -484,7 +484,7 @@ rad_config(struct rad_handle *h, const char *path)
 	linenum = 0;
 	while (fgets(buf, sizeof buf, fp) != NULL) {
 		int len;
-		char *fields[5];
+		char *fields[MAX_FIELDS];
 		int nfields;
 		char msg[ERRSIZE];
 		char *type;
-- 
cgit v1.1


From 6170f80efba53e2d170e91a15eba38ea8f6dda89 Mon Sep 17 00:00:00 2001
From: kevlo <kevlo@FreeBSD.org>
Date: Fri, 7 Dec 2012 02:26:08 +0000
Subject: Document that socket(2) may fail with EAFNOSUPPORT if the family
 cannot be found.

Reviewed by:	glebius
Obtained from:	NetBSD
---
 lib/libc/sys/socket.2 | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/sys/socket.2 b/lib/libc/sys/socket.2
index dae33d0..24e4293 100644
--- a/lib/libc/sys/socket.2
+++ b/lib/libc/sys/socket.2
@@ -28,7 +28,7 @@
 .\"     From: @(#)socket.2	8.1 (Berkeley) 6/4/93
 .\" $FreeBSD$
 .\"
-.Dd January 5, 2009
+.Dd December 7, 2012
 .Dt SOCKET 2
 .Os
 .Sh NAME
@@ -248,21 +248,26 @@ The
 .Fn socket
 system call fails if:
 .Bl -tag -width Er
-.It Bq Er EPROTONOSUPPORT
-The protocol type or the specified protocol is not supported
-within this domain.
+.It Bq Er EACCES
+Permission to create a socket of the specified type and/or protocol
+is denied.
+.It Bq Er EAFNOSUPPORT
+The address family (domain) is not supported or the
+specified domain is not supported by this protocol family.
 .It Bq Er EMFILE
 The per-process descriptor table is full.
 .It Bq Er ENFILE
 The system file table is full.
-.It Bq Er EACCES
-Permission to create a socket of the specified type and/or protocol
-is denied.
 .It Bq Er ENOBUFS
 Insufficient buffer space is available.
 The socket cannot be created until sufficient resources are freed.
 .It Bq Er EPERM
 User has insufficient privileges to carry out the requested operation.
+.It Bq Er EPROTONOSUPPORT
+The protocol type or the specified protocol is not supported
+within this domain.
+.It Bq Er EPROTOTYPE
+The socket type is not supported by the protocol.
 .El
 .Sh SEE ALSO
 .Xr accept 2 ,
-- 
cgit v1.1


From 450d7e4b366f20224b54e3f1ee82583f0d9b40a0 Mon Sep 17 00:00:00 2001
From: erwin <erwin@FreeBSD.org>
Date: Fri, 7 Dec 2012 16:05:04 +0000
Subject: Re-disable GSSAPI, which does not build on several archs.

Approved by:	delphij (mentor)
---
 lib/bind/config.mk | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/bind/config.mk b/lib/bind/config.mk
index 88ec258..ebac59a 100644
--- a/lib/bind/config.mk
+++ b/lib/bind/config.mk
@@ -27,10 +27,10 @@ INTERNALLIB=
 .endif
 .endif
 
-# Enable GSSAPI support if available
-.if ${MK_KERBEROS} != "no"
-CFLAGS+=	-DGSSAPI
-.endif
+# GSSAPI support is incomplete in 9.3.0
+#.if ${MK_KERBEROS} != "no"
+#CFLAGS+=	-DGSSAPI
+#.endif
 
 # Enable IPv6 support if available
 .if ${MK_INET6_SUPPORT} != "no"
-- 
cgit v1.1


From 8f8a6363fa6769d8a9a530131101dfc5972b9cc9 Mon Sep 17 00:00:00 2001
From: brooks <brooks@FreeBSD.org>
Date: Mon, 10 Dec 2012 17:34:33 +0000
Subject: Improve style(9) compliance of function declarations.

---
 lib/libc/locale/setrunelocale.c | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/locale/setrunelocale.c b/lib/libc/locale/setrunelocale.c
index 3ffb649..927b200 100644
--- a/lib/libc/locale/setrunelocale.c
+++ b/lib/libc/locale/setrunelocale.c
@@ -73,9 +73,11 @@ static int		__setrunelocale(struct xlocale_ctype *l, const char *);
 #define __collate_chain_pri_table (table->__collate_chain_pri_table)
 
 
-static void destruct_ctype(void *v)
+static void
+destruct_ctype(void *v)
 {
 	struct xlocale_ctype *l = v;
+
 	if (strcmp(l->runes->__encoding, "EUC") == 0)
 		free(l->runes->__variable);
 	if (&_DefaultRuneLocale != l->runes) 
@@ -83,13 +85,17 @@ static void destruct_ctype(void *v)
 	free(l);
 }
 
-const _RuneLocale *__getCurrentRuneLocale(void)
+const _RuneLocale *
+__getCurrentRuneLocale(void)
 {
+
 	return XLOCALE_CTYPE(__get_locale())->runes;
 }
 
-static void free_runes(_RuneLocale *rl)
+static void
+free_runes(_RuneLocale *rl)
 {
+
 	/* FIXME: The "EUC" check here is a hideous abstraction violation. */
 	if ((rl != &_DefaultRuneLocale) && (rl)) {
 		if (strcmp(rl->__encoding, "EUC") == 0) {
@@ -191,7 +197,8 @@ __wrap_setrunelocale(const char *locale)
 
 #ifndef __NO_TLS
 void
-__set_thread_rune_locale(locale_t loc) {
+__set_thread_rune_locale(locale_t loc)
+{
 
 	if (loc == NULL) {
 		_ThreadRuneLocale = &_DefaultRuneLocale;
@@ -205,6 +212,7 @@ void *
 __ctype_load(const char *locale, locale_t unused)
 {
 	struct xlocale_ctype *l = calloc(sizeof(struct xlocale_ctype), 1);
+
 	l->header.header.destructor = destruct_ctype;
 	if (__setrunelocale(l, locale))
 	{
-- 
cgit v1.1


From c8062fa2a473cce32f1049fa8353dea6b9cdb3f2 Mon Sep 17 00:00:00 2001
From: jilles <jilles@FreeBSD.org>
Date: Mon, 10 Dec 2012 17:56:51 +0000
Subject: libc: Make various internal file descriptors close-on-exec.

These are obtained via fopen().
---
 lib/libc/gen/fmtmsg.c       | 2 +-
 lib/libc/gen/getcap.c       | 4 ++--
 lib/libc/gen/getgrent.c     | 8 ++++----
 lib/libc/gen/getnetgrent.c  | 4 ++--
 lib/libc/gen/getttyent.c    | 2 +-
 lib/libc/gen/getusershell.c | 2 +-
 lib/libc/gen/getutxent.c    | 2 +-
 7 files changed, 12 insertions(+), 12 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/gen/fmtmsg.c b/lib/libc/gen/fmtmsg.c
index c6ecbd5..d2c67a6 100644
--- a/lib/libc/gen/fmtmsg.c
+++ b/lib/libc/gen/fmtmsg.c
@@ -83,7 +83,7 @@ def:
 		if (output == NULL)
 			return (MM_NOCON);
 		if (*output != '\0') {
-			if ((fp = fopen("/dev/console", "a")) == NULL) {
+			if ((fp = fopen("/dev/console", "ae")) == NULL) {
 				free(output);
 				return (MM_NOCON);
 			}
diff --git a/lib/libc/gen/getcap.c b/lib/libc/gen/getcap.c
index c321d12..f2f3777 100644
--- a/lib/libc/gen/getcap.c
+++ b/lib/libc/gen/getcap.c
@@ -654,7 +654,7 @@ cgetnext(char **bp, char **db_array)
 	if (dbp == NULL)
 		dbp = db_array;
 
-	if (pfp == NULL && (pfp = fopen(*dbp, "r")) == NULL) {
+	if (pfp == NULL && (pfp = fopen(*dbp, "re")) == NULL) {
 		(void)cgetclose();
 		return (-1);
 	}
@@ -679,7 +679,7 @@ cgetnext(char **bp, char **db_array)
 						(void)cgetclose();
 						return (0);
 					} else if ((pfp =
-					    fopen(*dbp, "r")) == NULL) {
+					    fopen(*dbp, "re")) == NULL) {
 						(void)cgetclose();
 						return (-1);
 					} else
diff --git a/lib/libc/gen/getgrent.c b/lib/libc/gen/getgrent.c
index 18f64a8..f9480c3 100644
--- a/lib/libc/gen/getgrent.c
+++ b/lib/libc/gen/getgrent.c
@@ -810,7 +810,7 @@ files_setgrent(void *retval, void *mdata, va_list ap)
 		if (st->fp != NULL)
 			rewind(st->fp);
 		else if (stayopen)
-			st->fp = fopen(_PATH_GROUP, "r");
+			st->fp = fopen(_PATH_GROUP, "re");
 		break;
 	case ENDGRENT:
 		if (st->fp != NULL) {
@@ -861,7 +861,7 @@ files_group(void *retval, void *mdata, va_list ap)
 	if (*errnop != 0)
 		return (NS_UNAVAIL);
 	if (st->fp == NULL &&
-	    ((st->fp = fopen(_PATH_GROUP, "r")) == NULL)) {
+	    ((st->fp = fopen(_PATH_GROUP, "re")) == NULL)) {
 		*errnop = errno;
 		return (NS_UNAVAIL);
 	}
@@ -1251,7 +1251,7 @@ compat_setgrent(void *retval, void *mdata, va_list ap)
 		if (st->fp != NULL)
 			rewind(st->fp);
 		else if (stayopen)
-			st->fp = fopen(_PATH_GROUP, "r");
+			st->fp = fopen(_PATH_GROUP, "re");
 		set_setent(dtab, mdata);
 		(void)_nsdispatch(NULL, dtab, NSDB_GROUP_COMPAT, "setgrent",
 		    compatsrc, 0);
@@ -1335,7 +1335,7 @@ compat_group(void *retval, void *mdata, va_list ap)
 	if (*errnop != 0)
 		return (NS_UNAVAIL);
 	if (st->fp == NULL &&
-	    ((st->fp = fopen(_PATH_GROUP, "r")) == NULL)) {
+	    ((st->fp = fopen(_PATH_GROUP, "re")) == NULL)) {
 		*errnop = errno;
 		rv = NS_UNAVAIL;
 		goto fin;
diff --git a/lib/libc/gen/getnetgrent.c b/lib/libc/gen/getnetgrent.c
index 51b3e37..4c56461 100644
--- a/lib/libc/gen/getnetgrent.c
+++ b/lib/libc/gen/getnetgrent.c
@@ -173,7 +173,7 @@ setnetgrent(const char *group)
 		if (((stat(_PATH_NETGROUP, &_yp_statp) < 0) &&
 		    errno == ENOENT) || _yp_statp.st_size == 0)
 			_use_only_yp = _netgr_yp_enabled = 1;
-		if ((netf = fopen(_PATH_NETGROUP,"r")) != NULL ||_use_only_yp){
+		if ((netf = fopen(_PATH_NETGROUP,"re")) != NULL ||_use_only_yp){
 		/*
 		 * Icky: grab the first character of the netgroup file
 		 * and turn on NIS if it's a '+'. rewind the stream
@@ -197,7 +197,7 @@ setnetgrent(const char *group)
 				return;
 			}
 #else
-		if ((netf = fopen(_PATH_NETGROUP, "r"))) {
+		if ((netf = fopen(_PATH_NETGROUP, "re"))) {
 #endif
 			if (parse_netgrp(group))
 				endnetgrent();
diff --git a/lib/libc/gen/getttyent.c b/lib/libc/gen/getttyent.c
index f2fc298..b82c30a 100644
--- a/lib/libc/gen/getttyent.c
+++ b/lib/libc/gen/getttyent.c
@@ -211,7 +211,7 @@ setttyent(void)
 	if (tf) {
 		rewind(tf);
 		return (1);
-	} else if ( (tf = fopen(_PATH_TTYS, "r")) )
+	} else if ( (tf = fopen(_PATH_TTYS, "re")) )
 		return (1);
 	return (0);
 }
diff --git a/lib/libc/gen/getusershell.c b/lib/libc/gen/getusershell.c
index ce50069..53536e1 100644
--- a/lib/libc/gen/getusershell.c
+++ b/lib/libc/gen/getusershell.c
@@ -115,7 +115,7 @@ _local_initshells(rv, cb_data, ap)
 		sl_free(sl, 1);
 	sl = sl_init();
 
-	if ((fp = fopen(_PATH_SHELLS, "r")) == NULL)
+	if ((fp = fopen(_PATH_SHELLS, "re")) == NULL)
 		return NS_UNAVAIL;
 
 	cp = line;
diff --git a/lib/libc/gen/getutxent.c b/lib/libc/gen/getutxent.c
index a2e938a..20e8859 100644
--- a/lib/libc/gen/getutxent.c
+++ b/lib/libc/gen/getutxent.c
@@ -71,7 +71,7 @@ setutxdb(int db, const char *file)
 
 	if (uf != NULL)
 		fclose(uf);
-	uf = fopen(file, "r");
+	uf = fopen(file, "re");
 	if (uf == NULL)
 		return (-1);
 
-- 
cgit v1.1


From e8cbfbc59ff52ee33615892e1a404252ad9625c5 Mon Sep 17 00:00:00 2001
From: dim <dim@FreeBSD.org>
Date: Tue, 11 Dec 2012 20:08:57 +0000
Subject: Cleanup a few duplicate llvm libs that snuck in with the recent
 update.

Noticed by:	rdivacky
---
 lib/clang/Makefile | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/clang/Makefile b/lib/clang/Makefile
index 1d993cd..cfc1554 100644
--- a/lib/clang/Makefile
+++ b/lib/clang/Makefile
@@ -71,15 +71,12 @@ SUBDIR=	libclanganalysis \
 	libllvmx86utils
 
 .if ${MK_CLANG_EXTRAS} != "no"
-SUBDIR+=libllvmarchive \
-	libllvmdebuginfo \
+SUBDIR+=libllvmdebuginfo \
 	libllvmexecutionengine \
 	libllvminterpreter \
 	libllvmjit \
-	libllvmlinker \
 	libllvmmcdisassembler \
 	libllvmmcjit \
-	libllvmobject \
 	libllvmruntimedyld
 .endif
 .endif
-- 
cgit v1.1


From 4e143eee29affcd6805799afb9cd273451342ad5 Mon Sep 17 00:00:00 2001
From: jilles <jilles@FreeBSD.org>
Date: Tue, 11 Dec 2012 22:52:56 +0000
Subject: libc: Make various internal file descriptors close-on-exec.

These are obtained via fopen().
---
 lib/libc/locale/collate.c       | 2 +-
 lib/libc/locale/setrunelocale.c | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/locale/collate.c b/lib/libc/locale/collate.c
index 676d41c..56513f4 100644
--- a/lib/libc/locale/collate.c
+++ b/lib/libc/locale/collate.c
@@ -135,7 +135,7 @@ __collate_load_tables_l(const char *encoding, struct xlocale_collate *table)
 	(void)strcat(buf, "/");
 	(void)strcat(buf, encoding);
 	(void)strcat(buf, "/LC_COLLATE");
-	if ((fp = fopen(buf, "r")) == NULL)
+	if ((fp = fopen(buf, "re")) == NULL)
 		return (_LDP_ERROR);
 
 	if (fread(strbuf, sizeof(strbuf), 1, fp) != 1) {
diff --git a/lib/libc/locale/setrunelocale.c b/lib/libc/locale/setrunelocale.c
index 927b200..49e6f6e 100644
--- a/lib/libc/locale/setrunelocale.c
+++ b/lib/libc/locale/setrunelocale.c
@@ -129,7 +129,7 @@ __setrunelocale(struct xlocale_ctype *l, const char *encoding)
 	(void) strcat(name, encoding);
 	(void) strcat(name, "/LC_CTYPE");
 
-	if ((fp = fopen(name, "r")) == NULL)
+	if ((fp = fopen(name, "re")) == NULL)
 		return (errno == 0 ? ENOENT : errno);
 
 	if ((rl = _Read_RuneMagi(fp)) == NULL) {
-- 
cgit v1.1


From dddecfb2fbb9da35ba472f32182b6707381d55c7 Mon Sep 17 00:00:00 2001
From: pjd <pjd@FreeBSD.org>
Date: Wed, 12 Dec 2012 15:27:33 +0000
Subject: Eliminate redundant variable.

---
 lib/libc/gen/sysctlbyname.c | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/gen/sysctlbyname.c b/lib/libc/gen/sysctlbyname.c
index a2e0d5f..88b860d 100644
--- a/lib/libc/gen/sysctlbyname.c
+++ b/lib/libc/gen/sysctlbyname.c
@@ -19,13 +19,10 @@ sysctlbyname(const char *name, void *oldp, size_t *oldlenp,
     const void *newp, size_t newlen)
 {
 	int real_oid[CTL_MAXNAME+2];
-	int error;
 	size_t oidlen;
 
 	oidlen = sizeof(real_oid) / sizeof(int);
-	error = sysctlnametomib(name, real_oid, &oidlen);
-	if (error < 0) 
-		return (error);
-	error = sysctl(real_oid, oidlen, oldp, oldlenp, newp, newlen);
-	return (error);
+	if (sysctlnametomib(name, real_oid, &oidlen) < 0)
+		return (-1);
+	return (sysctl(real_oid, oidlen, oldp, oldlenp, newp, newlen));
 }
-- 
cgit v1.1


From 7c744361c22d7640b2f1c05924fd05090c957fd5 Mon Sep 17 00:00:00 2001
From: trociny <trociny@FreeBSD.org>
Date: Sat, 15 Dec 2012 18:19:48 +0000
Subject: New devstat metrics for devstat_compute_statistics():

  DSM_TOTAL_DURATION
  DSM_TOTAL_DURATION_READ
  DSM_TOTAL_DURATION_WRITE
  DSM_TOTAL_DURATION_FREE
  DSM_TOTAL_DURATION_OTHER
  DSM_TOTAL_BUSY_TIME
---
 lib/libdevstat/devstat.3 | 31 ++++++++++++++++++++++++++++++-
 lib/libdevstat/devstat.c | 47 +++++++++++++++++++++++++++++++++++++++--------
 lib/libdevstat/devstat.h |  6 ++++++
 3 files changed, 75 insertions(+), 9 deletions(-)

(limited to 'lib')

diff --git a/lib/libdevstat/devstat.3 b/lib/libdevstat/devstat.3
index 8d0ce4d..39be6ac 100644
--- a/lib/libdevstat/devstat.3
+++ b/lib/libdevstat/devstat.3
@@ -27,7 +27,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd March 18, 2003
+.Dd December 15, 2012
 .Dt DEVSTAT 3
 .Os
 .Sh NAME
@@ -526,6 +526,35 @@ the acquisition of
 .Fa previous
 and
 .Fa current .
+.It Dv DSM_TOTAL_DURATION
+type:
+.Vt "long double *"
+.Pp
+The total duration of transactions, in seconds, between the acquisition of
+.Fa previous
+and
+.Fa current .
+.It Dv DSM_TOTAL_DURATION_OTHER
+.It Dv DSM_TOTAL_DURATION_READ
+.It Dv DSM_TOTAL_DURATION_WRITE
+.It Dv DSM_TOTAL_DURATION_FREE
+type:
+.Vt "long double *"
+.Pp
+The total duration of transactions of the specified type between
+the acquisition of
+.Fa previous
+and
+.Fa current .
+.It Dv DSM_TOTAL_BUSY_TIME
+type:
+.Vt "long double *"
+.Pp
+Total time the device had one or more transactions outstanding
+between the acquisition of
+.Fa previous
+and
+.Fa current .
 .It Dv DSM_TOTAL_BLOCKS
 type:
 .Vt "uint64_t *"
diff --git a/lib/libdevstat/devstat.c b/lib/libdevstat/devstat.c
index ff0767a..0f313fb 100644
--- a/lib/libdevstat/devstat.c
+++ b/lib/libdevstat/devstat.c
@@ -133,6 +133,12 @@ struct devstat_args {
 	{ DSM_MS_PER_TRANSACTION_FREE, DEVSTAT_ARG_LD },
 	{ DSM_BUSY_PCT, DEVSTAT_ARG_LD },
 	{ DSM_QUEUE_LENGTH, DEVSTAT_ARG_UINT64 },
+	{ DSM_TOTAL_DURATION, DEVSTAT_ARG_LD },
+	{ DSM_TOTAL_DURATION_READ, DEVSTAT_ARG_LD },
+	{ DSM_TOTAL_DURATION_WRITE, DEVSTAT_ARG_LD },
+	{ DSM_TOTAL_DURATION_FREE, DEVSTAT_ARG_LD },
+	{ DSM_TOTAL_DURATION_OTHER, DEVSTAT_ARG_LD },
+	{ DSM_TOTAL_BUSY_TIME, DEVSTAT_ARG_LD },
 };
 
 static const char *namelist[] = {
@@ -1217,11 +1223,13 @@ devstat_compute_statistics(struct devstat *current, struct devstat *previous,
 	u_int64_t totaltransfers, totaltransfersread, totaltransferswrite;
 	u_int64_t totaltransfersother, totalblocks, totalblocksread;
 	u_int64_t totalblockswrite, totaltransfersfree, totalblocksfree;
+	long double totalduration, totaldurationread, totaldurationwrite;
+	long double totaldurationfree, totaldurationother;
 	va_list ap;
 	devstat_metric metric;
 	u_int64_t *destu64;
 	long double *destld;
-	int retval, i;
+	int retval;
 
 	retval = 0;
 
@@ -1263,6 +1271,13 @@ devstat_compute_statistics(struct devstat *current, struct devstat *previous,
 		totalblocksfree /= 512;
 	}
 
+	totaldurationread = DELTA_T(duration[DEVSTAT_READ]);
+	totaldurationwrite = DELTA_T(duration[DEVSTAT_WRITE]);
+	totaldurationfree = DELTA_T(duration[DEVSTAT_FREE]);
+	totaldurationother = DELTA_T(duration[DEVSTAT_NO_DATA]);
+	totalduration = totaldurationread + totaldurationwrite +
+	    totaldurationfree + totaldurationother;
+
 	va_start(ap, etime);
 
 	while ((metric = (devstat_metric)va_arg(ap, devstat_metric)) != 0) {
@@ -1484,9 +1499,7 @@ devstat_compute_statistics(struct devstat *current, struct devstat *previous,
 		 */
 		case DSM_MS_PER_TRANSACTION:
 			if (totaltransfers > 0) {
-				*destld = 0;
-				for (i = 0; i < DEVSTAT_N_TRANS_FLAGS; i++)
-					*destld += DELTA_T(duration[i]);
+				*destld = totalduration;
 				*destld /= totaltransfers;
 				*destld *= 1000;
 			} else
@@ -1499,7 +1512,7 @@ devstat_compute_statistics(struct devstat *current, struct devstat *previous,
 		 */
 		case DSM_MS_PER_TRANSACTION_READ:
 			if (totaltransfersread > 0) {
-				*destld = DELTA_T(duration[DEVSTAT_READ]);
+				*destld = totaldurationread;
 				*destld /= totaltransfersread;
 				*destld *= 1000;
 			} else
@@ -1507,7 +1520,7 @@ devstat_compute_statistics(struct devstat *current, struct devstat *previous,
 			break;
 		case DSM_MS_PER_TRANSACTION_WRITE:
 			if (totaltransferswrite > 0) {
-				*destld = DELTA_T(duration[DEVSTAT_WRITE]);
+				*destld = totaldurationwrite;
 				*destld /= totaltransferswrite;
 				*destld *= 1000;
 			} else
@@ -1515,7 +1528,7 @@ devstat_compute_statistics(struct devstat *current, struct devstat *previous,
 			break;
 		case DSM_MS_PER_TRANSACTION_FREE:
 			if (totaltransfersfree > 0) {
-				*destld = DELTA_T(duration[DEVSTAT_FREE]);
+				*destld = totaldurationfree;
 				*destld /= totaltransfersfree;
 				*destld *= 1000;
 			} else
@@ -1523,7 +1536,7 @@ devstat_compute_statistics(struct devstat *current, struct devstat *previous,
 			break;
 		case DSM_MS_PER_TRANSACTION_OTHER:
 			if (totaltransfersother > 0) {
-				*destld = DELTA_T(duration[DEVSTAT_NO_DATA]);
+				*destld = totaldurationother;
 				*destld /= totaltransfersother;
 				*destld *= 1000;
 			} else
@@ -1541,6 +1554,24 @@ devstat_compute_statistics(struct devstat *current, struct devstat *previous,
 		case DSM_QUEUE_LENGTH:
 			*destu64 = current->start_count - current->end_count;
 			break;
+		case DSM_TOTAL_DURATION:
+			*destld = totalduration;
+			break;
+		case DSM_TOTAL_DURATION_READ:
+			*destld = totaldurationread;
+			break;
+		case DSM_TOTAL_DURATION_WRITE:
+			*destld = totaldurationwrite;
+			break;
+		case DSM_TOTAL_DURATION_FREE:
+			*destld = totaldurationfree;
+			break;
+		case DSM_TOTAL_DURATION_OTHER:
+			*destld = totaldurationother;
+			break;
+		case DSM_TOTAL_BUSY_TIME:
+			*destld = DELTA_T(busy_time);
+			break;
 /*
  * XXX: comment out the default block to see if any case's are missing.
  */
diff --git a/lib/libdevstat/devstat.h b/lib/libdevstat/devstat.h
index 7717cb1..9471f93 100644
--- a/lib/libdevstat/devstat.h
+++ b/lib/libdevstat/devstat.h
@@ -97,6 +97,12 @@ typedef enum {
 	DSM_MS_PER_TRANSACTION_FREE,
 	DSM_BUSY_PCT,
 	DSM_QUEUE_LENGTH,
+	DSM_TOTAL_DURATION,
+	DSM_TOTAL_DURATION_READ,
+	DSM_TOTAL_DURATION_WRITE,
+	DSM_TOTAL_DURATION_FREE,
+	DSM_TOTAL_DURATION_OTHER,
+	DSM_TOTAL_BUSY_TIME,
 	DSM_MAX
 } devstat_metric;
 
-- 
cgit v1.1


From 3268de3a0df48cce2d34cda24995077176c1e0be Mon Sep 17 00:00:00 2001
From: jilles <jilles@FreeBSD.org>
Date: Mon, 17 Dec 2012 10:38:51 +0000
Subject: libc/iconv: Fix race condition with setting FD_CLOEXEC.

A fork/exec could happen between open and fcntl, leaking a file descriptor.
Using O_CLOEXEC fixes this and as a side effect simplifies the code.

NetBSD already had this (I checked this after making the change myself).

Reviewed by:	gabor
---
 lib/libc/iconv/citrus_mmap.c | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/iconv/citrus_mmap.c b/lib/libc/iconv/citrus_mmap.c
index dd5d059..2aaf73c 100644
--- a/lib/libc/iconv/citrus_mmap.c
+++ b/lib/libc/iconv/citrus_mmap.c
@@ -57,12 +57,8 @@ _citrus_map_file(struct _citrus_region * __restrict r,
 
 	_region_init(r, NULL, 0);
 
-	if ((fd = open(path, O_RDONLY)) == -1)
+	if ((fd = open(path, O_RDONLY | O_CLOEXEC)) == -1)
 		return (errno);
-	if (fcntl(fd, F_SETFD, FD_CLOEXEC) == -1) {
-		ret = errno;
-		goto error;
-	}
 
 	if (fstat(fd, &st)  == -1) {
 		ret = errno;
-- 
cgit v1.1


From 1c3512cb7668497436ae7b91ce7523c4eee7eaf8 Mon Sep 17 00:00:00 2001
From: eadler <eadler@FreeBSD.org>
Date: Mon, 17 Dec 2012 12:57:36 +0000
Subject: Fix warning from valgrind when a failed entry is tested.

PR:		kern/173008
Submitted by:	Zhihao Yuan <lichray@gmail.com>
Reviewed by:	gabor
Approved by:	cperciva (implicit)
MFC after:	1 week
---
 lib/libc/nls/msgcat.c | 1 +
 1 file changed, 1 insertion(+)

(limited to 'lib')

diff --git a/lib/libc/nls/msgcat.c b/lib/libc/nls/msgcat.c
index 44b1440..2859916 100644
--- a/lib/libc/nls/msgcat.c
+++ b/lib/libc/nls/msgcat.c
@@ -82,6 +82,7 @@ __FBSDID("$FreeBSD$");
 				  if (np != NULL) {				\
 				  	np->name = strdup(n);			\
 					np->path = NULL;			\
+					np->catd = NLERR;			\
 					np->lang = (l == NULL) ? NULL :		\
 					    strdup(l);				\
 					np->caterrno = e;			\
-- 
cgit v1.1


From 3a4e3b814af8211f139991b6ab47d3c08a587f80 Mon Sep 17 00:00:00 2001
From: andrew <andrew@FreeBSD.org>
Date: Tue, 18 Dec 2012 07:26:55 +0000
Subject: Get libcompiler-rt and libgcc building on ARM with clang.

* Don't provide clear_cache or the __sync_* functions on ARM with clang as
  they are provided by clang as builtin functions.
* Tell clang it is aloud to compile some libgcc code using heinous GCC
  extensions.
---
 lib/libcompiler_rt/Makefile | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/libcompiler_rt/Makefile b/lib/libcompiler_rt/Makefile
index 84f9b36..af1fcb6 100644
--- a/lib/libcompiler_rt/Makefile
+++ b/lib/libcompiler_rt/Makefile
@@ -28,7 +28,6 @@ SRCF=	absvdi2 \
 	ashlti3 \
 	ashrdi3 \
 	ashrti3 \
-	clear_cache \
 	clzdi2 \
 	clzsi2 \
 	clzti2 \
@@ -126,6 +125,11 @@ SRCF=	absvdi2 \
 	umoddi3 \
 	umodti3
 
+# Don't build clear_cache on ARM with clang as it is a builtin there.
+.if ${MACHINE_CPUARCH} != "arm" || ${COMPILER_TYPE} != "clang"
+SRCF+=	clear_cache
+.endif
+
 # These are already shipped by libc.a on arm and mips
 .if ${MACHINE_CPUARCH} != "arm" && ${MACHINE_CPUARCH} != "mips"
 SRCF+=	adddf3 \
@@ -148,8 +152,9 @@ SRCF+=	adddf3 \
 	umodsi3
 .endif
 
-# FreeBSD-specific atomic intrinsics.
-.if ${MACHINE_CPUARCH} == "arm" || ${MACHINE_CPUARCH} == "mips"
+# FreeBSD-specific atomic intrinsics. Clang provides them as a builtin.
+.if (${MACHINE_CPUARCH} == "arm" && ${COMPILER_TYPE} != "clang") || \
+    ${MACHINE_CPUARCH} == "mips"
 SRCF+=	__sync_fetch_and_add_4 \
 	__sync_fetch_and_and_4 \
 	__sync_fetch_and_or_4 \
-- 
cgit v1.1


From 6547dbdbaff0d10e29457c19876b28de348a0643 Mon Sep 17 00:00:00 2001
From: rpaulo <rpaulo@FreeBSD.org>
Date: Fri, 21 Dec 2012 15:15:35 +0000
Subject: Move the 64-bit _setjmp to lib/libstand.

---
 lib/libstand/amd64/_setjmp.S | 93 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 93 insertions(+)
 create mode 100644 lib/libstand/amd64/_setjmp.S

(limited to 'lib')

diff --git a/lib/libstand/amd64/_setjmp.S b/lib/libstand/amd64/_setjmp.S
new file mode 100644
index 0000000..e841f49
--- /dev/null
+++ b/lib/libstand/amd64/_setjmp.S
@@ -0,0 +1,93 @@
+/*-
+ * Copyright (c) 1990 The Regents of the University of California.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to Berkeley by
+ * William Jolitz.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#if defined(LIBC_SCCS) && !defined(lint)
+	.asciz "@(#)_setjmp.s	5.1 (Berkeley) 4/23/90"
+#endif /* LIBC_SCCS and not lint */
+#include <machine/asm.h>
+__FBSDID("$FreeBSD$");
+
+/*
+ * C library -- _setjmp, _longjmp
+ *
+ *	_longjmp(a,v)
+ * will generate a "return(v)" from the last call to
+ *	_setjmp(a)
+ * by restoring registers from the environment 'a'.
+ * The previous signal state is NOT restored.
+ */
+
+ENTRY(_setjmp)
+	movq	%rdi,%rax
+	movq	0(%rsp),%rdx		/* retval */
+	movq	%rdx, 0(%rax)		/* 0; retval */
+	movq	%rbx, 8(%rax)		/* 1; rbx */
+	movq	%rsp,16(%rax)		/* 2; rsp */
+	movq	%rbp,24(%rax)		/* 3; rbp */
+	movq	%r12,32(%rax)		/* 4; r12 */
+	movq	%r13,40(%rax)		/* 5; r13 */
+	movq	%r14,48(%rax)		/* 6; r14 */
+	movq	%r15,56(%rax)		/* 7; r15 */
+	fnstcw	64(%rax)		/* 8; fpu cw */
+	stmxcsr	68(%rax)		/*    and mxcsr */
+	xorq	%rax,%rax
+	ret
+END(_setjmp)
+
+	.weak	CNAME(_longjmp)
+ENTRY(_longjmp)
+	movq	%rdi,%rdx
+	/* Restore the mxcsr, but leave exception flags intact. */
+	stmxcsr	-4(%rsp)
+	movl	68(%rdx),%eax
+	andl	$0xffffffc0,%eax
+	movl	-4(%rsp),%edi
+	andl	$0x3f,%edi
+	xorl	%eax,%edi
+	movl	%edi,-4(%rsp)
+	ldmxcsr -4(%rsp)
+	movq	%rsi,%rax		/* retval */
+	movq	0(%rdx),%rcx
+	movq	8(%rdx),%rbx
+	movq	16(%rdx),%rsp
+	movq	24(%rdx),%rbp
+	movq	32(%rdx),%r12
+	movq	40(%rdx),%r13
+	movq	48(%rdx),%r14
+	movq	56(%rdx),%r15
+	fldcw	64(%rdx)
+	testq	%rax,%rax
+	jnz	1f
+	incq	%rax
+1:	movq	%rcx,0(%rsp)
+	ret
+END(_longjmp)
-- 
cgit v1.1


From 804c67a4867afb8b7ef41a3ade748cab870b788c Mon Sep 17 00:00:00 2001
From: kevlo <kevlo@FreeBSD.org>
Date: Fri, 21 Dec 2012 15:54:13 +0000
Subject: Fix socket calls on error post-r243965.

Submitted by:	Garrett Cooper
---
 lib/libnetgraph/sock.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/libnetgraph/sock.c b/lib/libnetgraph/sock.c
index fca3900..5f9f563 100644
--- a/lib/libnetgraph/sock.c
+++ b/lib/libnetgraph/sock.c
@@ -71,10 +71,10 @@ NgMkSockNode(const char *name, int *csp, int *dsp)
 		name = NULL;
 
 	/* Create control socket; this also creates the netgraph node.
-	   If we get an EPROTONOSUPPORT then the socket node type is
+	   If we get an EAFNOSUPPORT then the socket node type is
 	   not loaded, so load it and try again. */
 	if ((cs = socket(AF_NETGRAPH, SOCK_DGRAM, NG_CONTROL)) < 0) {
-		if (errno == EPROTONOSUPPORT) {
+		if (errno == EAFNOSUPPORT) {
 			if (kldload(NG_SOCKET_KLD) < 0) {
 				errnosv = errno;
 				if (_gNgDebugLevel >= 1)
-- 
cgit v1.1


From 3523462cfba20ab0ca86dbb59d8eff46c0338b6e Mon Sep 17 00:00:00 2001
From: brooks <brooks@FreeBSD.org>
Date: Fri, 21 Dec 2012 20:37:38 +0000
Subject: Add libnetbsd, a thin compatibility layer intended to allow a limited
 set of NetBSD software to compile as part of the FreeBSD build with little or
 no modifiction.  It is built as a static library and not installed for
 general use.  Likewise, its header files are not installed.

Sponsored by:	DARPA, AFRL
---
 lib/Makefile               |   1 +
 lib/libnetbsd/Makefile     |  13 +++
 lib/libnetbsd/README       |   7 ++
 lib/libnetbsd/rmd160.h     |  44 +++++++++
 lib/libnetbsd/sha1.h       |  43 +++++++++
 lib/libnetbsd/sha2.h       |  39 ++++++++
 lib/libnetbsd/stdlib.h     |  70 ++++++++++++++
 lib/libnetbsd/strsuftoll.c | 222 +++++++++++++++++++++++++++++++++++++++++++++
 lib/libnetbsd/sys/cdefs.h  |  50 ++++++++++
 lib/libnetbsd/util.c       |  59 ++++++++++++
 lib/libnetbsd/util.h       |  41 +++++++++
 11 files changed, 589 insertions(+)
 create mode 100644 lib/libnetbsd/Makefile
 create mode 100644 lib/libnetbsd/README
 create mode 100644 lib/libnetbsd/rmd160.h
 create mode 100644 lib/libnetbsd/sha1.h
 create mode 100644 lib/libnetbsd/sha2.h
 create mode 100644 lib/libnetbsd/stdlib.h
 create mode 100644 lib/libnetbsd/strsuftoll.c
 create mode 100644 lib/libnetbsd/sys/cdefs.h
 create mode 100644 lib/libnetbsd/util.c
 create mode 100644 lib/libnetbsd/util.h

(limited to 'lib')

diff --git a/lib/Makefile b/lib/Makefile
index 7c3ccbd..6e1a402 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -90,6 +90,7 @@ SUBDIR=	${SUBDIR_ORDERED} \
 	${_libmilter} \
 	${_libmp} \
 	${_libnandfs} \
+	libnetbsd \
 	${_libngatm} \
 	libopie \
 	libpam \
diff --git a/lib/libnetbsd/Makefile b/lib/libnetbsd/Makefile
new file mode 100644
index 0000000..09c6985
--- /dev/null
+++ b/lib/libnetbsd/Makefile
@@ -0,0 +1,13 @@
+# $FreeBSD$
+
+.include <bsd.own.mk>
+
+LIB=	netbsd
+
+CFLAGS+=	-I${.CURDIR}
+
+SRCS+=	strsuftoll.c util.c util.h
+
+INTERNALLIB=
+
+.include <bsd.lib.mk>
diff --git a/lib/libnetbsd/README b/lib/libnetbsd/README
new file mode 100644
index 0000000..ab30300
--- /dev/null
+++ b/lib/libnetbsd/README
@@ -0,0 +1,7 @@
+$FreeBSD$
+
+libnetbsd is a thin compatibility layer intended to allow a limited
+set of NetBSD software to compile as part of the FreeBSD build with
+little or no modifiction.  It is built as a static library and not
+installed for general use.  Likewise, its header files are not
+installed.
diff --git a/lib/libnetbsd/rmd160.h b/lib/libnetbsd/rmd160.h
new file mode 100644
index 0000000..7b81199
--- /dev/null
+++ b/lib/libnetbsd/rmd160.h
@@ -0,0 +1,44 @@
+/*	$FreeBSD$	 */
+
+/*-
+ * Copyright (c) 2012 SRI International
+ * All rights reserved.
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237)
+ * ("CTSRD"), as part of the DARPA CRASH research programme.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _RMD160_H_
+#define _RMD160_H_
+
+#include <ripemd.h>
+
+#define RMD160_CTX	RIPEMD160_CTX
+#define	RMD160End	RIPEMD160_End
+#define	RMD160File	RIPEMD160_File
+#define	RMD160Init	RIPEMD160_Init
+#define	RMD160Update	RIPEMD160_Update
+
+#endif /* _RMD160_H_ */
diff --git a/lib/libnetbsd/sha1.h b/lib/libnetbsd/sha1.h
new file mode 100644
index 0000000..41280c5
--- /dev/null
+++ b/lib/libnetbsd/sha1.h
@@ -0,0 +1,43 @@
+/*	$FreeBSD$	*/
+
+/*-
+ * Copyright (c) 2012 SRI International
+ * All rights reserved.
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237)
+ * ("CTSRD"), as part of the DARPA CRASH research programme.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _SHA1_H_
+#define _SHA1_H_
+
+#include <sha.h>
+
+#define	SHA1End		SHA1_End
+#define	SHA1File	SHA1_File
+#define	SHA1Init	SHA1_Init
+#define	SHA1Update	SHA1_Update
+
+#endif /* _SHA1_H_ */
diff --git a/lib/libnetbsd/sha2.h b/lib/libnetbsd/sha2.h
new file mode 100644
index 0000000..b67691d
--- /dev/null
+++ b/lib/libnetbsd/sha2.h
@@ -0,0 +1,39 @@
+/*	$FreeBSD$	*/
+
+/*-
+ * Copyright (c) 2012 SRI International
+ * All rights reserved.
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237)
+ * ("CTSRD"), as part of the DARPA CRASH research programme.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _SHA2_H_
+#define _SHA2_H_
+
+#include <sha256.h>
+#include <sha512.h>
+
+#endif /* _SHA2_H_ */
diff --git a/lib/libnetbsd/stdlib.h b/lib/libnetbsd/stdlib.h
new file mode 100644
index 0000000..d481913
--- /dev/null
+++ b/lib/libnetbsd/stdlib.h
@@ -0,0 +1,70 @@
+/* $FreeBSD$ */
+
+/*-
+ * Copyright (c) 2012 SRI International
+ * Copyright (c) 2001-2002,2004 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Luke Mewburn.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+/*-
+ * Copyright (c) 1992, 1993
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _LIBNETBSD_STDLIB_H_
+#define _LIBNETBSD_STDLIB_H_
+
+#include_next <stdlib.h>
+
+long long strsuftoll(const char *, const char *, long long, long long);
+long long strsuftollx(const char *, const char *,
+    long long, long long, char *, size_t);
+
+#endif /* _LIBNETBSD_STDLIB_H_ */
diff --git a/lib/libnetbsd/strsuftoll.c b/lib/libnetbsd/strsuftoll.c
new file mode 100644
index 0000000..657e525
--- /dev/null
+++ b/lib/libnetbsd/strsuftoll.c
@@ -0,0 +1,222 @@
+/*	$NetBSD: strsuftoll.c,v 1.6 2004/03/05 05:58:29 lukem Exp $	*/
+/*-
+ * Copyright (c) 2001-2002,2004 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Luke Mewburn.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+/*-
+ * Copyright (c) 1991, 1993, 1994
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * This code is derived from software contributed to Berkeley by
+ * Keith Muller of the University of California, San Diego and Lance
+ * Visser of Convex Computer Corporation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/types.h>
+#include <sys/time.h>
+
+#include <assert.h>
+#include <ctype.h>
+#include <err.h>
+#include <errno.h>
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#ifdef _LIBC
+# ifdef __weak_alias
+__weak_alias(strsuftoll, _strsuftoll)
+__weak_alias(strsuftollx, _strsuftollx)
+# endif
+#endif /* LIBC */
+
+/*
+ * Convert an expression of the following forms to a (u)int64_t.
+ * 	1) A positive decimal number.
+ *	2) A positive decimal number followed by a b (mult by 512).
+ *	3) A positive decimal number followed by a k (mult by 1024).
+ *	4) A positive decimal number followed by a m (mult by 1048576).
+ *	5) A positive decimal number followed by a g (mult by 1073741824).
+ *	6) A positive decimal number followed by a t (mult by 1099511627776).
+ *	7) A positive decimal number followed by a w (mult by sizeof int)
+ *	8) Two or more positive decimal numbers (with/without k,b or w).
+ *	   separated by x (also * for backwards compatibility), specifying
+ *	   the product of the indicated values.
+ * Returns the result upon successful conversion, or exits with an
+ * appropriate error.
+ * 
+ */
+
+/*
+ * As strsuftoll(), but returns the error message into the provided buffer
+ * rather than exiting with it.
+ */
+/* LONGLONG */
+long long
+strsuftollx(const char *desc, const char *val,
+    long long min, long long max, char *ebuf, size_t ebuflen)
+{
+	long long num, t;
+	char	*expr;
+
+	errno = 0;
+	ebuf[0] = '\0';
+
+	while (isspace((unsigned char)*val))	/* Skip leading space */
+		val++;
+
+	num = strtoll(val, &expr, 10);
+	if (errno == ERANGE)
+		goto erange;			/* Overflow */
+
+	if (expr == val)			/* No digits */
+		goto badnum;
+
+	switch (*expr) {
+	case 'b':
+		t = num;
+		num *= 512;			/* 1 block */
+		if (t > num)
+			goto erange;
+		++expr;
+		break;
+	case 'k':
+		t = num;
+		num *= 1024;			/* 1 kilobyte */
+		if (t > num)
+			goto erange;
+		++expr;
+		break;
+	case 'm':
+		t = num;
+		num *= 1048576;			/* 1 megabyte */
+		if (t > num)
+			goto erange;
+		++expr;
+		break;
+	case 'g':
+		t = num;
+		num *= 1073741824;		/* 1 gigabyte */
+		if (t > num)
+			goto erange;
+		++expr;
+		break;
+	case 't':
+		t = num;
+		num *= 1099511627776LL;		/* 1 terabyte */
+		if (t > num)
+			goto erange;
+		++expr;
+		break;
+	case 'w':
+		t = num;
+		num *= sizeof(int);		/* 1 word */
+		if (t > num)
+			goto erange;
+		++expr;
+		break;
+	}
+
+	switch (*expr) {
+	case '\0':
+		break;
+	case '*':				/* Backward compatible */
+	case 'x':
+		t = num;
+		num *= strsuftollx(desc, expr + 1, min, max, ebuf, ebuflen);
+		if (*ebuf != '\0')
+			return (0);
+		if (t > num) {
+ erange:	 	
+			snprintf(ebuf, ebuflen,
+			    "%s: %s", desc, strerror(ERANGE));
+			return (0);
+		}
+		break;
+	default:
+ badnum:	snprintf(ebuf, ebuflen,
+		    "%s `%s': illegal number", desc, val);
+		return (0);
+	}
+	if (num < min) {
+			/* LONGLONG */
+		snprintf(ebuf, ebuflen, "%s %lld is less than %lld.",
+		    desc, (long long)num, (long long)min);
+		return (0);
+	}
+	if (num > max) {
+			/* LONGLONG */
+		snprintf(ebuf, ebuflen,
+		    "%s %lld is greater than %lld.",
+		    desc, (long long)num, (long long)max);
+		return (0);
+	}
+	*ebuf = '\0';
+	return (num);
+}
+
+/* LONGLONG */
+long long
+strsuftoll(const char *desc, const char *val,
+    long long min, long long max)
+{
+	long long result;
+	char	errbuf[100];
+
+	result = strsuftollx(desc, val, min, max, errbuf, sizeof(errbuf));
+	if (*errbuf != '\0')
+		errx(1, "%s", errbuf);
+	return (result);
+}
diff --git a/lib/libnetbsd/sys/cdefs.h b/lib/libnetbsd/sys/cdefs.h
new file mode 100644
index 0000000..09a7ca4
--- /dev/null
+++ b/lib/libnetbsd/sys/cdefs.h
@@ -0,0 +1,50 @@
+/* $FreeBSD$ */
+
+/*-
+ * Copyright (c) 2012 SRI International
+ * Copyright (c) 1992, 1993
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _LIBNETBSD_SYS_CDEFS_H_
+#define _LIBNETBSD_SYS_CDEFS_H_
+
+#include_next <sys/cdefs.h>
+
+#ifdef __dead2
+#define __dead __dead2
+#else
+#define __dead
+#endif
+
+/*
+ * Return the number of elements in a statically-allocated array,
+ * __x.
+ */
+#define	__arraycount(__x)	(sizeof(__x) / sizeof(__x[0]))
+
+#endif /* _LIBNETBSD_SYS_CDEFS_H_ */
diff --git a/lib/libnetbsd/util.c b/lib/libnetbsd/util.c
new file mode 100644
index 0000000..36b9421
--- /dev/null
+++ b/lib/libnetbsd/util.c
@@ -0,0 +1,59 @@
+/*-
+ * Copyright (c) 2012 SRI International
+ * All rights reserved.
+ *
+ * This software was developed by SRI International and the University of
+ * Cambridge Computer Laboratory under DARPA/AFRL contract (FA8750-10-C-0237)
+ * ("CTSRD"), as part of the DARPA CRASH research programme.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/cdefs.h>
+#include <sys/types.h>
+
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <util.h>
+
+char *
+flags_to_string(u_long flags, const char *def)
+{
+	char *str;
+
+	str = fflagstostr(flags);
+	if (*str == '\0') {
+		free(str);
+		str = strdup(def);
+	}
+	return (str);
+}
+
+int
+string_to_flags(char **stringp, u_long *setp, u_long *clrp)
+{
+
+	return strtofflags(stringp, setp, clrp);
+}
diff --git a/lib/libnetbsd/util.h b/lib/libnetbsd/util.h
new file mode 100644
index 0000000..c7d32e1
--- /dev/null
+++ b/lib/libnetbsd/util.h
@@ -0,0 +1,41 @@
+/*	$FreeBSD$	*/
+
+/*-
+ * Copyright (c) 2012 SRI International
+ * Copyright (c) 1995
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _UTIL_H_
+#define _UTIL_H_
+
+#include <libutil.h>
+
+char	*flags_to_string(u_long flags, const char *def);
+int	 string_to_flags(char **stringp, u_long *setp, u_long *clrp);
+
+#endif	/* _UTIL_H_ */
-- 
cgit v1.1


From 23ae20eae272e8a2cfae89d5760be977f0fd4be5 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Fri, 21 Dec 2012 22:20:59 +0000
Subject:  - Reduce buffer size from LINE_MAX to PATH_MAX, there is no point to
 store    path longer than this.  - Fix an unreached case of check against
 sizeof buf, which in turn leads    to an off-by-one nul byte write on the
 stack.  The original condition    can never be satisfied because the passed
 boundary is the maximum value    that can be returned, so code was harmless.

MFC after:	1 month
---
 lib/libc/gen/check_utility_compat.c | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/gen/check_utility_compat.c b/lib/libc/gen/check_utility_compat.c
index 0ccdec1..04c594b 100644
--- a/lib/libc/gen/check_utility_compat.c
+++ b/lib/libc/gen/check_utility_compat.c
@@ -35,32 +35,28 @@ __FBSDID("$FreeBSD$");
  * are threaded, so I'm not concerned about cancellation points or other
  * niceties.
  */
+#include <sys/limits.h>
+
 #include <limits.h>
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>
 
-#ifndef LINE_MAX
-#define	LINE_MAX _POSIX2_LINE_MAX
-#endif
-
 #define	_PATH_UTIL_COMPAT	"/etc/compat-FreeBSD-4-util"
 #define	_ENV_UTIL_COMPAT	"_COMPAT_FreeBSD_4"
 
 int
 check_utility_compat(const char *utility)
 {
-	char buf[LINE_MAX];
+	char buf[PATH_MAX];
 	char *p, *bp;
 	int len;
 
 	if ((p = getenv(_ENV_UTIL_COMPAT)) != NULL) {
 		strlcpy(buf, p, sizeof buf);
 	} else {
-		if ((len = readlink(_PATH_UTIL_COMPAT, buf, sizeof buf)) < 0)
+		if ((len = readlink(_PATH_UTIL_COMPAT, buf, sizeof(buf) - 1)) < 0)
 			return 0;
-		if (len > sizeof buf)
-			len = sizeof buf;
 		buf[len] = '\0';
 	}
 	if (buf[0] == '\0')
-- 
cgit v1.1


From a931043751d62613cff21d7eea760e17669bef4f Mon Sep 17 00:00:00 2001
From: dim <dim@FreeBSD.org>
Date: Sun, 23 Dec 2012 13:04:00 +0000
Subject: Upgrade our copy of llvm/clang to 3.2 release.

Release notes for llvm:
http://llvm.org/releases/3.2/docs/ReleaseNotes.html

Release notes for clang:
http://llvm.org/releases/3.2/tools/clang/docs/ReleaseNotes.html

MFC after:	2 weeks
---
 lib/clang/include/clang/Basic/Version.inc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/clang/include/clang/Basic/Version.inc b/lib/clang/include/clang/Basic/Version.inc
index d912900..84a96a6 100644
--- a/lib/clang/include/clang/Basic/Version.inc
+++ b/lib/clang/include/clang/Basic/Version.inc
@@ -5,6 +5,6 @@
 #define	CLANG_VERSION_MINOR	2
 
 #define	CLANG_VENDOR		"FreeBSD "
-#define	CLANG_VENDOR_SUFFIX	" 20121130"
+#define	CLANG_VENDOR_SUFFIX	" 20121221"
 
-#define	SVN_REVISION		"168974"
+#define	SVN_REVISION		"170710"
-- 
cgit v1.1


From 1c06919638a5a652db6f6496cacf300b62880652 Mon Sep 17 00:00:00 2001
From: zont <zont@FreeBSD.org>
Date: Tue, 25 Dec 2012 13:43:01 +0000
Subject: - Update manual pages accordingly to r244384 and r244385.

Approved by:	kib (mentor)
MFC after:	1 week
---
 lib/libc/sys/mlock.2    | 17 +++++++++++------
 lib/libc/sys/mlockall.2 | 17 ++++++++++++++---
 2 files changed, 25 insertions(+), 9 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/sys/mlock.2 b/lib/libc/sys/mlock.2
index ed9c15a..0e5fef8 100644
--- a/lib/libc/sys/mlock.2
+++ b/lib/libc/sys/mlock.2
@@ -28,7 +28,7 @@
 .\"	@(#)mlock.2	8.2 (Berkeley) 12/11/93
 .\" $FreeBSD$
 .\"
-.Dd July 27, 2010
+.Dd December 25, 2012
 .Dt MLOCK 2
 .Os
 .Sh NAME
@@ -94,12 +94,15 @@ limited in how much they can lock down.
 A single process can
 .Fn mlock
 the minimum of
-a system-wide ``wired pages'' limit and
-the per-process
+a system-wide ``wired pages'' limit
+.Va vm.max_wired
+and the per-process
 .Li RLIMIT_MEMLOCK
 resource limit.
 .Pp
-These calls are only available to the super-user.
+If
+.Va security.bsd.unprivileged_mlock
+is set to 0 these calls are only available to the super-user.
 .Sh RETURN VALUES
 .Rv -std
 .Pp
@@ -112,7 +115,8 @@ system call
 will fail if:
 .Bl -tag -width Er
 .It Bq Er EPERM
-The caller is not the super-user.
+.Va security.bsd.unprivileged_mlock
+is set to 0 and the caller is not the super-user.
 .It Bq Er EINVAL
 The address given is not page aligned or the length is negative.
 .It Bq Er EAGAIN
@@ -129,7 +133,8 @@ system call
 will fail if:
 .Bl -tag -width Er
 .It Bq Er EPERM
-The caller is not the super-user.
+.Va security.bsd.unprivileged_mlock
+is set to 0 and the caller is not the super-user.
 .It Bq Er EINVAL
 The address given is not page aligned or the length is negative.
 .It Bq Er ENOMEM
diff --git a/lib/libc/sys/mlockall.2 b/lib/libc/sys/mlockall.2
index 54ae23e..23c644a 100644
--- a/lib/libc/sys/mlockall.2
+++ b/lib/libc/sys/mlockall.2
@@ -30,7 +30,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd July 27, 2010
+.Dd December 25, 2012
 .Dt MLOCKALL 2
 .Os
 .Sh NAME
@@ -68,11 +68,22 @@ Since physical memory is a potentially scarce resource, processes are
 limited in how much they can lock down.
 A single process can lock the minimum of a system-wide
 .Dq wired pages
-limit and the per-process
+limit
+.Va vm.max_wired
+and the per-process
 .Dv RLIMIT_MEMLOCK
 resource limit.
 .Pp
-These calls are only available to the super-user.
+If
+.Va security.bsd.unprivileged_mlock
+is set to 0 these calls are only available to the super-user.
+If
+.Va vm.old_mlock
+is set to 1 the per-process
+.Dv RLIMIT_MEMLOCK
+resource limit will not be applied for
+.Fn mlockall
+calls.
 .Pp
 The
 .Fn munlockall
-- 
cgit v1.1


From 55e18ebdf3e07929f1a0c7d68309e1edb2c655be Mon Sep 17 00:00:00 2001
From: pluknet <pluknet@FreeBSD.org>
Date: Wed, 26 Dec 2012 05:11:48 +0000
Subject: Fix libproc test case to work with clang premature optimization
 observed with -O2 (used by default). Avoid function inlining for t1_bkpt_t on
 which we set a breakpoint. Otherwise the address of the function is never
 called thus the breakpoint never triggers.

Reported by:	zont
Reviewed by:	rpaulo
---
 lib/libproc/test/t1-bkpt/t1-bkpt.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libproc/test/t1-bkpt/t1-bkpt.c b/lib/libproc/test/t1-bkpt/t1-bkpt.c
index 37a9fcf..1cd4d17 100644
--- a/lib/libproc/test/t1-bkpt/t1-bkpt.c
+++ b/lib/libproc/test/t1-bkpt/t1-bkpt.c
@@ -36,7 +36,7 @@
 #include <string.h>
 #include <libproc.h>
 
-int
+int __noinline
 t1_bkpt_t()
 {
 	printf("TEST OK\n");
-- 
cgit v1.1


From 925e83c0bd84aa809deda8a20cab358ea1c0780b Mon Sep 17 00:00:00 2001
From: bapt <bapt@FreeBSD.org>
Date: Thu, 27 Dec 2012 14:09:50 +0000
Subject: Use flopen(3) instead of open(2) + flock(2)

---
 lib/libutil/gr_util.c | 6 ++----
 lib/libutil/pw_util.c | 7 ++-----
 2 files changed, 4 insertions(+), 9 deletions(-)

(limited to 'lib')

diff --git a/lib/libutil/gr_util.c b/lib/libutil/gr_util.c
index be34395..6bf102f 100644
--- a/lib/libutil/gr_util.c
+++ b/lib/libutil/gr_util.c
@@ -106,10 +106,8 @@ gr_lock(void)
 	for (;;) {
 		struct stat st;
 
-		lockfd = open(group_file, O_RDONLY, 0);
-		if (lockfd < 0 || fcntl(lockfd, F_SETFD, 1) == -1)
-			err(1, "%s", group_file);
-		if (flock(lockfd, LOCK_EX|LOCK_NB) == -1) {
+		lockfd = flopen(group_file, O_RDONLY|O_NONBLOCK, 0);
+		if (lockfd == -1) {
 			if (errno == EWOULDBLOCK) {
 				errx(1, "the group file is busy");
 			} else {
diff --git a/lib/libutil/pw_util.c b/lib/libutil/pw_util.c
index 4bf3001..24c0263 100644
--- a/lib/libutil/pw_util.c
+++ b/lib/libutil/pw_util.c
@@ -179,11 +179,8 @@ pw_lock(void)
 	for (;;) {
 		struct stat st;
 
-		lockfd = open(masterpasswd, O_RDONLY, 0);
-		if (lockfd < 0 || fcntl(lockfd, F_SETFD, 1) == -1)
-			err(1, "%s", masterpasswd);
-		/* XXX vulnerable to race conditions */
-		if (flock(lockfd, LOCK_EX|LOCK_NB) == -1) {
+		lockfd = flopen(masterpasswd, O_RDONLY|O_NONBLOCK, 0);
+		if (lockfd == -1) {
 			if (errno == EWOULDBLOCK) {
 				errx(1, "the password db file is busy");
 			} else {
-- 
cgit v1.1


From b63557c001769d5ed1338eaa39730ba3ae77245b Mon Sep 17 00:00:00 2001
From: bapt <bapt@FreeBSD.org>
Date: Thu, 27 Dec 2012 14:30:19 +0000
Subject: New gr_add function to provide a clean and safe method to append a
 new member into an existing group.

Submitted by:	db
---
 lib/libutil/gr_util.c | 40 ++++++++++++++++++++++++++++++++++++++++
 lib/libutil/libutil.h |  2 ++
 2 files changed, 42 insertions(+)

(limited to 'lib')

diff --git a/lib/libutil/gr_util.c b/lib/libutil/gr_util.c
index 6bf102f..90062eb 100644
--- a/lib/libutil/gr_util.c
+++ b/lib/libutil/gr_util.c
@@ -479,6 +479,46 @@ gr_dup(const struct group *gr)
 }
 
 /*
+ * Add a new member name to a struct group.
+ */
+struct group *
+gr_add(struct group *gr, const char *newmember)
+{
+	size_t mlen;
+	int num_mem=0;
+	char **members;
+	struct group *newgr;
+
+	if (newmember == NULL)
+		return(gr_dup(gr));
+
+	if (gr->gr_mem != NULL) {
+		for (num_mem = 0; gr->gr_mem[num_mem] != NULL; num_mem++) {
+			if (strcmp(gr->gr_mem[num_mem], newmember) == 0) {
+				errno = EEXIST;
+				return (NULL);
+			}
+		}
+	}
+	/* Allocate enough for current pointers + 1 more and NULL marker */
+	mlen = (num_mem + 2) * sizeof(*gr->gr_mem);
+	if ((members = calloc(1, mlen )) == NULL) {
+		errno = ENOMEM;
+		return (NULL);
+	}
+	memcpy(members, gr->gr_mem, num_mem * sizeof(*gr->gr_mem));
+	members[num_mem++] = (char *)newmember;
+	members[num_mem] = NULL;
+	gr->gr_mem = members;
+	newgr = gr_dup(gr);
+	if (newgr == NULL)
+		errno = ENOMEM;
+
+	free(members);
+	return (newgr);
+}
+
+/*
  * Scan a line and place it into a group structure.
  */
 static bool
diff --git a/lib/libutil/libutil.h b/lib/libutil/libutil.h
index bf42766..fcd74e1 100644
--- a/lib/libutil/libutil.h
+++ b/lib/libutil/libutil.h
@@ -166,6 +166,8 @@ int 	gr_copy(int __ffd, int _tfd, const struct group *_gr,
 	    struct group *_old_gr);
 struct group *
 	gr_dup(const struct group *_gr);
+struct group *
+	gr_add(struct group *_gr, const char *_newmember);
 int	gr_equal(const struct group *_gr1, const struct group *_gr2);
 void	gr_fini(void);
 int	gr_init(const char *_dir, const char *_master);
-- 
cgit v1.1


From 247c2a40864f8d06b2dcc9ee18b183e22b3b29f7 Mon Sep 17 00:00:00 2001
From: bapt <bapt@FreeBSD.org>
Date: Thu, 27 Dec 2012 16:51:29 +0000
Subject: - Clean up previous gr_add use malloc instead of calloc - Fix
 tinderbox error

Submitted by:	db
---
 lib/libutil/gr_util.c | 7 +++----
 lib/libutil/libutil.h | 2 +-
 2 files changed, 4 insertions(+), 5 deletions(-)

(limited to 'lib')

diff --git a/lib/libutil/gr_util.c b/lib/libutil/gr_util.c
index 90062eb..0cd8055 100644
--- a/lib/libutil/gr_util.c
+++ b/lib/libutil/gr_util.c
@@ -482,7 +482,7 @@ gr_dup(const struct group *gr)
  * Add a new member name to a struct group.
  */
 struct group *
-gr_add(struct group *gr, const char *newmember)
+gr_add(struct group *gr, char *newmember)
 {
 	size_t mlen;
 	int num_mem=0;
@@ -502,18 +502,17 @@ gr_add(struct group *gr, const char *newmember)
 	}
 	/* Allocate enough for current pointers + 1 more and NULL marker */
 	mlen = (num_mem + 2) * sizeof(*gr->gr_mem);
-	if ((members = calloc(1, mlen )) == NULL) {
+	if ((members = malloc(mlen)) == NULL) {
 		errno = ENOMEM;
 		return (NULL);
 	}
 	memcpy(members, gr->gr_mem, num_mem * sizeof(*gr->gr_mem));
-	members[num_mem++] = (char *)newmember;
+	members[num_mem++] = newmember;
 	members[num_mem] = NULL;
 	gr->gr_mem = members;
 	newgr = gr_dup(gr);
 	if (newgr == NULL)
 		errno = ENOMEM;
-
 	free(members);
 	return (newgr);
 }
diff --git a/lib/libutil/libutil.h b/lib/libutil/libutil.h
index fcd74e1..b1b2405 100644
--- a/lib/libutil/libutil.h
+++ b/lib/libutil/libutil.h
@@ -167,7 +167,7 @@ int 	gr_copy(int __ffd, int _tfd, const struct group *_gr,
 struct group *
 	gr_dup(const struct group *_gr);
 struct group *
-	gr_add(struct group *_gr, const char *_newmember);
+	gr_add(struct group *_gr, char *_newmember);
 int	gr_equal(const struct group *_gr1, const struct group *_gr2);
 void	gr_fini(void);
 int	gr_init(const char *_dir, const char *_master);
-- 
cgit v1.1


From 2f0af9e7e55ba465a435c3161ac2b17c6c2f18f0 Mon Sep 17 00:00:00 2001
From: bapt <bapt@FreeBSD.org>
Date: Thu, 27 Dec 2012 19:33:43 +0000
Subject: gr_dup: simplify duplication of group

Submitted by:	db
---
 lib/libutil/gr_util.c | 41 ++++++++++++++++++++---------------------
 1 file changed, 20 insertions(+), 21 deletions(-)

(limited to 'lib')

diff --git a/lib/libutil/gr_util.c b/lib/libutil/gr_util.c
index 0cd8055..2f87bd1 100644
--- a/lib/libutil/gr_util.c
+++ b/lib/libutil/gr_util.c
@@ -44,11 +44,6 @@ __FBSDID("$FreeBSD$");
 #include <string.h>
 #include <unistd.h>
 
-struct group_storage {
-	struct group	 gr;
-	char		*members[];
-};
-
 static int lockfd = -1;
 static char group_dir[PATH_MAX];
 static char group_file[PATH_MAX];
@@ -434,14 +429,14 @@ gr_make(const struct group *gr)
 struct group *
 gr_dup(const struct group *gr)
 {
+	struct group *newgr;
 	char *dst;
 	size_t len;
-	struct group_storage *gs;
 	int ndx;
 	int num_mem;
 
 	/* Calculate size of the group. */
-	len = sizeof(*gs);
+	len = sizeof(*newgr);
 	if (gr->gr_name != NULL)
 		len += strlen(gr->gr_name) + 1;
 	if (gr->gr_passwd != NULL)
@@ -452,30 +447,34 @@ gr_dup(const struct group *gr)
 		len += (num_mem + 1) * sizeof(*gr->gr_mem);
 	} else
 		num_mem = -1;
-
 	/* Create new group and copy old group into it. */
-	if ((gs = calloc(1, len)) == NULL)
+	if ((newgr = malloc(len)) == NULL)
 		return (NULL);
-	dst = (char *)&gs->members[num_mem + 1];
+	/* point new gr_mem to end of struct + 1 */
+	if (gr->gr_mem != NULL)
+		newgr->gr_mem = (char **)newgr + sizeof(struct group);
+	else
+		newgr->gr_mem = NULL;
+	/* point dst after the end of all the gr_mem pointers in newgr */
+	dst = (char *)newgr + sizeof(struct group) +
+	    (num_mem + 1) * sizeof(*gr->gr_mem);
 	if (gr->gr_name != NULL) {
-		gs->gr.gr_name = dst;
-		dst = stpcpy(gs->gr.gr_name, gr->gr_name) + 1;
+		newgr->gr_name = dst;
+		dst = stpcpy(dst, gr->gr_name) + 1;
 	}
 	if (gr->gr_passwd != NULL) {
-		gs->gr.gr_passwd = dst;
-		dst = stpcpy(gs->gr.gr_passwd, gr->gr_passwd) + 1;
+		newgr->gr_passwd = dst;
+		dst = stpcpy(dst, gr->gr_passwd) + 1;
 	}
-	gs->gr.gr_gid = gr->gr_gid;
+	newgr->gr_gid = gr->gr_gid;
 	if (gr->gr_mem != NULL) {
-		gs->gr.gr_mem = gs->members;
 		for (ndx = 0; ndx < num_mem; ndx++) {
-			gs->gr.gr_mem[ndx] = dst;
-			dst = stpcpy(gs->gr.gr_mem[ndx], gr->gr_mem[ndx]) + 1;
+			newgr->gr_mem[ndx] = dst;
+			dst = stpcpy(dst, gr->gr_mem[ndx]) + 1;
 		}
-		gs->gr.gr_mem[ndx] = NULL;
+		newgr->gr_mem[ndx] = NULL;
 	}
-
-	return (&gs->gr);
+	return (newgr);
 }
 
 /*
-- 
cgit v1.1


From 85a7bc88636c4a46a80bb6c42be258f2efc6b20b Mon Sep 17 00:00:00 2001
From: bapt <bapt@FreeBSD.org>
Date: Thu, 27 Dec 2012 20:24:44 +0000
Subject: Add O_CLOEXEC to flopen

Requested by:	jilles
---
 lib/libutil/gr_util.c | 2 +-
 lib/libutil/pw_util.c | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/libutil/gr_util.c b/lib/libutil/gr_util.c
index 2f87bd1..4583257 100644
--- a/lib/libutil/gr_util.c
+++ b/lib/libutil/gr_util.c
@@ -101,7 +101,7 @@ gr_lock(void)
 	for (;;) {
 		struct stat st;
 
-		lockfd = flopen(group_file, O_RDONLY|O_NONBLOCK, 0);
+		lockfd = flopen(group_file, O_RDONLY|O_NONBLOCK|O_CLOEXEC, 0);
 		if (lockfd == -1) {
 			if (errno == EWOULDBLOCK) {
 				errx(1, "the group file is busy");
diff --git a/lib/libutil/pw_util.c b/lib/libutil/pw_util.c
index 24c0263..befd1fb 100644
--- a/lib/libutil/pw_util.c
+++ b/lib/libutil/pw_util.c
@@ -179,7 +179,7 @@ pw_lock(void)
 	for (;;) {
 		struct stat st;
 
-		lockfd = flopen(masterpasswd, O_RDONLY|O_NONBLOCK, 0);
+		lockfd = flopen(masterpasswd, O_RDONLY|O_NONBLOCK|O_CLOEXEC, 0);
 		if (lockfd == -1) {
 			if (errno == EWOULDBLOCK) {
 				errx(1, "the password db file is busy");
-- 
cgit v1.1


From 85f7f7664e7b3873fec0954f8300bb43549e331c Mon Sep 17 00:00:00 2001
From: bapt <bapt@FreeBSD.org>
Date: Thu, 27 Dec 2012 20:31:12 +0000
Subject: cast to uintptr_t to properly calculate offset

Reported by:	mdf
Submitted by:	db
---
 lib/libutil/gr_util.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libutil/gr_util.c b/lib/libutil/gr_util.c
index 4583257..f4a08c4 100644
--- a/lib/libutil/gr_util.c
+++ b/lib/libutil/gr_util.c
@@ -452,7 +452,7 @@ gr_dup(const struct group *gr)
 		return (NULL);
 	/* point new gr_mem to end of struct + 1 */
 	if (gr->gr_mem != NULL)
-		newgr->gr_mem = (char **)newgr + sizeof(struct group);
+		newgr->gr_mem = (char **)((uintptr_t)newgr + sizeof(struct group));
 	else
 		newgr->gr_mem = NULL;
 	/* point dst after the end of all the gr_mem pointers in newgr */
-- 
cgit v1.1


From 9708a96be58af22cd8b296b5394cd23b5c7764a7 Mon Sep 17 00:00:00 2001
From: bapt <bapt@FreeBSD.org>
Date: Thu, 27 Dec 2012 20:47:34 +0000
Subject: avoid arithmetic on uintptr_t

Submitted by:	pjd
Reviewed by:	jilles
---
 lib/libutil/gr_util.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libutil/gr_util.c b/lib/libutil/gr_util.c
index f4a08c4..437fd78 100644
--- a/lib/libutil/gr_util.c
+++ b/lib/libutil/gr_util.c
@@ -452,7 +452,7 @@ gr_dup(const struct group *gr)
 		return (NULL);
 	/* point new gr_mem to end of struct + 1 */
 	if (gr->gr_mem != NULL)
-		newgr->gr_mem = (char **)((uintptr_t)newgr + sizeof(struct group));
+		newgr->gr_mem = (char **)(newgr + 1);
 	else
 		newgr->gr_mem = NULL;
 	/* point dst after the end of all the gr_mem pointers in newgr */
-- 
cgit v1.1


From b179cbf13b45dd7aa21cb35b74d5896f88a87746 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Fri, 28 Dec 2012 01:09:30 +0000
Subject: Add zh_CN.UTF-8 catalog.

Reviewed by:	alphachi <alphachi mediaspirit.org>
MFC after:	2 weeks
---
 lib/libc/nls/Makefile.inc    |   1 +
 lib/libc/nls/zh_CN.UTF-8.msg | 295 +++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 296 insertions(+)
 create mode 100644 lib/libc/nls/zh_CN.UTF-8.msg

(limited to 'lib')

diff --git a/lib/libc/nls/Makefile.inc b/lib/libc/nls/Makefile.inc
index f5c6885..d0d3392 100644
--- a/lib/libc/nls/Makefile.inc
+++ b/lib/libc/nls/Makefile.inc
@@ -36,3 +36,4 @@ NLS+=	ru_RU.KOI8-R
 NLS+=	sk_SK.ISO8859-2
 NLS+=	sv_SE.ISO8859-1
 NLS+=	uk_UA.UTF-8
+NLS+=	zh_CN.UTF-8
diff --git a/lib/libc/nls/zh_CN.UTF-8.msg b/lib/libc/nls/zh_CN.UTF-8.msg
new file mode 100644
index 0000000..9475ea8
--- /dev/null
+++ b/lib/libc/nls/zh_CN.UTF-8.msg
@@ -0,0 +1,295 @@
+$ $FreeBSD$
+$
+$ Message catalog for zh_CN.UTF-8 locale
+$
+$ strerror() support catalog
+$
+$set 1
+$ EPERM
+1 ��许的�作
+$ ENOENT
+2 文件或目录�存在
+$ ESRCH
+3 进程�存在
+$ EINTR
+4 系统调用中止
+$ EIO
+5 输入/输出错误
+$ ENXIO
+6 未�置的设备
+$ E2BIG
+7 �数表过长
+$ ENOEXEC
+8 �执行文件格�错误
+$ EBADF
+9 文件�述符无效
+$ ECHILD
+10 指定�进程�存在
+$ EDEADLK
+11 此�作会导致死�
+$ ENOMEM
+12 无法分�内存
+$ EACCES
+13 拒�访问
+$ EFAULT
+14 无效地�
+$ ENOTBLK
+15 该�作需��设备
+$ EBUSY
+16 设备使用中
+$ EEXIST
+17 文件已存在
+$ EXDEV
+18 链接跨设备
+$ ENODEV
+19 设备�支�此�作
+$ ENOTDIR
+20 对象�目录
+$ EISDIR
+21 对象为目录
+$ EINVAL
+22 �数无效
+$ ENFILE
+23 系统打开文件过多
+$ EMFILE
+24 进程打开文件过多
+$ ENOTTY
+25 无效设备 ioctl
+$ ETXTBSY
+26 �执行文件忙
+$ EFBIG
+27 文件过大
+$ ENOSPC
+28 设备无�用空间
+$ ESPIPE
+29 ��许执行 seek �作
+$ EROFS
+30 文件系统�读
+$ EMLINK
+31 文件链接过多
+$ EPIPE
+32 管�已中止
+$ EDOM
+33 数值�数越界
+$ ERANGE
+34 结果过大
+$ EAGAIN, EWOULDBLOCK
+35 资�暂时��用
+$ EINPROGRESS
+36 �作进行中
+$ EALREADY
+37 �作已开始
+$ ENOTSOCK
+38 �试在� socket 上执行 socket �作
+$ EDESTADDRREQ
+39 需�目的地�
+$ EMSGSIZE
+40 消�过长
+$ EPROTOTYPE
+41 socket�议类型错误
+$ ENOPROTOOPT
+42 �议��用
+$ EPROTONOSUPPORT
+43 �支�的�议
+$ ESOCKTNOSUPPORT
+44 �支�的 socket 类型
+$ EOPNOTSUPP
+45 �支�的�作
+$ EPFNOSUPPORT
+46 �支�的�议�
+$ EAFNOSUPPORT
+47 �议��支�的地��
+$ EADDRINUSE
+48 地�已被�用
+$ EADDRNOTAVAIL
+49 无法指定请求的地�
+$ ENETDOWN
+50 网络已关闭
+$ ENETUNREACH
+51 网络��达
+$ ENETRESET
+52 ��导致网络连接丢失
+$ ECONNABORTED
+53 软件导致的连接中止
+$ ECONNRESET
+54 对方��了连接
+$ ENOBUFS
+55 缓冲区空间�足
+$ EISCONN
+56 socket 已连接
+$ ENOTCONN
+57 socket 未连接
+$ ESHUTDOWN
+58 socket shutdown 之�无法��数�
+$ ETOOMANYREFS
+59 引用数过多：无法拼接
+$ ETIMEDOUT
+60 �作超时
+$ ECONNREFUSED
+61 拒�连接
+$ ELOOP
+62 符�链接层数过多
+$ ENAMETOOLONG
+63 文件�过长
+$ EHOSTDOWN
+64 主机已关闭
+$ EHOSTUNREACH
+65 没有到主机的路由
+$ ENOTEMPTY
+66 目录�空
+$ EPROCLIM
+67 进程数超�
+$ EUSERS
+68 用户数超�
+$ EDQUOT
+69 �盘空间��超�
+$ ESTALE
+70 NFS 文件�柄已失效
+$ EREMOTE
+71 远程目录层数过多
+$ EBADRPC
+72 RPC 结构无效
+$ ERPCMISMATCH
+73 RPC 版本错误
+$ EPROGUNAVAIL
+74 RPC 程���用
+$ EPROGMISMATCH
+75 程�版本错误
+$ EPROCUNAVAIL
+76 未�供的远程函数
+$ ENOLCK
+77 �支��
+$ ENOSYS
+78 功能未实现
+$ EFTYPE
+79 文件类型或格�无效
+$ EAUTH
+80 身份无效
+$ ENEEDAUTH
+81 无信任凭�
+$ EIDRM
+82 无标识符
+$ ENOMSG
+83 无消�类型
+$ EOVERFLOW
+84 数值溢出
+$ ECANCELED
+85 �作已�消
+$ EILSEQ
+86 无效字符�列
+$ ENOATTR
+87 无扩展属性
+$ EDOOFUS
+88 程�设计错误
+$ EBADMSG
+89 无效消�
+$ EMULTIHOP
+90 Multihop attempted
+$ ENOLINK
+91 Link has been severed
+$ EPROTO
+92 �议错误
+$ ENOTCAPABLE
+93 �能�足
+$
+$ strsignal() support catalog
+$
+$set 2
+$ SIGHUP
+1 终端线路挂断
+$ SIGINT
+2 中断
+$ SIGQUIT
+3 退出
+$ SIGILL
+4 无效指令
+$ SIGTRAP
+5 跟踪/BPT 陷阱
+$ SIGABRT
+6 终止陷阱
+$ SIGEMT
+7 EMT 陷阱
+$ SIGFPE
+8 浮点异常
+$ SIGKILL
+9 强制终止进程
+$ SIGBUS
+10 系统内存访问越界
+$ SIGSEGV
+11 内存访问越界
+$ SIGSYS
+12 无效系统调用
+$ SIGPIPE
+13 写管�时读�者�存在
+$ SIGALRM
+14 实时倒计时到期
+$ SIGTERM
+15 终止
+$ SIGURG
+16 需立�处�的 I/O 紧急状况
+$ SIGSTOP
+17 挂起 (信�)
+$ SIGTSTP
+18 挂起
+$ SIGCONT
+19 继续�行
+$ SIGCHLD
+20 �进程终止
+$ SIGTTIN
+21 TTY 输入阻塞
+$ SIGTTOU
+22 TTY 输出阻塞
+$ SIGIO
+23 I/O 就绪
+$ SIGXCPU
+24 CPU 使用时间过长
+$ SIGXFSZ
+25 文件尺寸过大
+$ SIGVTALRM
+26 虚拟倒计时到期
+$ SIGPROF
+27 剖�倒计时到期
+$ SIGWINCH
+28 窗�尺寸�化
+$ SIGINFO
+29 请求信�
+$ SIGUSR1
+30 用户自定义信� 1
+$ SIGUSR2
+31 用户自定义信� 2
+$
+$ gai_strerror() support catalog
+$
+$set 3
+$ 1 (obsolete)
+1 主机�使用了�支�的地�类型
+$ EAI_AGAIN
+2 暂时无法解��称
+$ EAI_BADFLAGS
+3 无效的 ai_flags 值
+$ EAI_FAIL
+4 解��称时出现无法��的错误
+$ EAI_FAMILY
+5 �支�的地�信�类型
+$ EAI_MEMORY
+6 内存分�失败
+$ 7 (obsolete)
+7 主机�无相关�的地�
+$ EAI_NONAME
+8 未知的主机�或�务�
+$ EAI_SERVICE
+9 套接字类型�支�此�务�
+$ EAI_SOCKTYPE
+10 �支�的套接字类型
+$ EAI_SYSTEM
+11 系统调用返回了 errno 错误
+$ EAI_BADHINTS
+12 无效�示�数
+$ EAI_PROTOCOL
+13 未知�议
+$ EAI_OVERFLOW
+14 �数缓冲溢出
+$ 0
+32766 �功
+$ NL_MSGMAX
+32767 未知错误
-- 
cgit v1.1


From ab5f5cd77fff00bbd21ff6dd06a063aa727f5279 Mon Sep 17 00:00:00 2001
From: delphij <delphij@FreeBSD.org>
Date: Fri, 28 Dec 2012 01:23:12 +0000
Subject: Add derived versions for common Simplified Chinese encodings.

MFC after:	2 weeks
---
 lib/libc/nls/Makefile.inc      |   2 +
 lib/libc/nls/zh_CN.GB18030.msg | 297 +++++++++++++++++++++++++++++++++++++++++
 lib/libc/nls/zh_CN.GB2312.msg  | 297 +++++++++++++++++++++++++++++++++++++++++
 3 files changed, 596 insertions(+)
 create mode 100644 lib/libc/nls/zh_CN.GB18030.msg
 create mode 100644 lib/libc/nls/zh_CN.GB2312.msg

(limited to 'lib')

diff --git a/lib/libc/nls/Makefile.inc b/lib/libc/nls/Makefile.inc
index d0d3392..962d16b 100644
--- a/lib/libc/nls/Makefile.inc
+++ b/lib/libc/nls/Makefile.inc
@@ -36,4 +36,6 @@ NLS+=	ru_RU.KOI8-R
 NLS+=	sk_SK.ISO8859-2
 NLS+=	sv_SE.ISO8859-1
 NLS+=	uk_UA.UTF-8
+NLS+=	zh_CN.GB18030
+NLS+=	zh_CN.GB2312
 NLS+=	zh_CN.UTF-8
diff --git a/lib/libc/nls/zh_CN.GB18030.msg b/lib/libc/nls/zh_CN.GB18030.msg
new file mode 100644
index 0000000..0e35d7e
--- /dev/null
+++ b/lib/libc/nls/zh_CN.GB18030.msg
@@ -0,0 +1,297 @@
+$ $FreeBSD$
+$
+$ Message catalog for zh_CN.GB18030 locale
+$
+$ Derived from FreeBSD: head/lib/libc/nls/zh_CN.UTF-8.msg 244756 2012-12-28 01:09:30Z delphij
+$
+$ strerror() support catalog
+$
+$set 1
+$ EPERM
+1 ²»ÔÊÐíµÄ²Ù×÷
+$ ENOENT
+2 Îļþ»òĿ¼²»´æÔÚ
+$ ESRCH
+3 ½ø³Ì²»´æÔÚ
+$ EINTR
+4 ϵͳµ÷ÓÃÖÐÖ¹
+$ EIO
+5 ÊäÈë/Êä³ö´íÎó
+$ ENXIO
+6 δÅäÖõÄÉ豸
+$ E2BIG
+7 ²ÎÊý±í¹ý³¤
+$ ENOEXEC
+8 ¿ÉÖ´ÐÐÎļþ¸ñʽ´íÎó
+$ EBADF
+9 ÎļþÃèÊö·ûÎÞЧ
+$ ECHILD
+10 Ö¸¶¨×Ó½ø³Ì²»´æÔÚ
+$ EDEADLK
+11 ´Ë²Ù×÷»áµ¼ÖÂËÀËø
+$ ENOMEM
+12 ÎÞ·¨·ÖÅäÄÚ´æ
+$ EACCES
+13 ¾Ü¾ø·ÃÎÊ
+$ EFAULT
+14 ÎÞЧµØÖ·
+$ ENOTBLK
+15 ¸Ã²Ù×÷ÐèÒª¿éÉ豸
+$ EBUSY
+16 É豸ʹÓÃÖÐ
+$ EEXIST
+17 ÎļþÒÑ´æÔÚ
+$ EXDEV
+18 Á´½Ó¿çÉ豸
+$ ENODEV
+19 É豸²»Ö§³Ö´Ë²Ù×÷
+$ ENOTDIR
+20 ¶ÔÏó·ÇĿ¼
+$ EISDIR
+21 ¶ÔÏóΪĿ¼
+$ EINVAL
+22 ²ÎÊýÎÞЧ
+$ ENFILE
+23 ϵͳ´ò¿ªÎļþ¹ý¶à
+$ EMFILE
+24 ½ø³Ì´ò¿ªÎļþ¹ý¶à
+$ ENOTTY
+25 ÎÞЧÉ豸 ioctl
+$ ETXTBSY
+26 ¿ÉÖ´ÐÐÎļþæ
+$ EFBIG
+27 Îļþ¹ý´ó
+$ ENOSPC
+28 É豸ÎÞ¿ÉÓÿռä
+$ ESPIPE
+29 ²»ÔÊÐíÖ´ÐÐ seek ²Ù×÷
+$ EROFS
+30 Îļþϵͳֻ¶Á
+$ EMLINK
+31 ÎļþÁ´½Ó¹ý¶à
+$ EPIPE
+32 ¹ÜµÀÒÑÖÐÖ¹
+$ EDOM
+33 ÊýÖµ²ÎÊýÔ½½ç
+$ ERANGE
+34 ½á¹û¹ý´ó
+$ EAGAIN, EWOULDBLOCK
+35 ×ÊÔ´ÔÝʱ²»¿ÉÓÃ
+$ EINPROGRESS
+36 ²Ù×÷½øÐÐÖÐ
+$ EALREADY
+37 ²Ù×÷ÒÑ¿ªÊ¼
+$ ENOTSOCK
+38 ³¢ÊÔÔÚ·Ç socket ÉÏÖ´ÐÐ socket ²Ù×÷
+$ EDESTADDRREQ
+39 ÐèҪĿµÄµØÖ·
+$ EMSGSIZE
+40 ÏûÏ¢¹ý³¤
+$ EPROTOTYPE
+41 socketЭÒéÀàÐÍ´íÎó
+$ ENOPROTOOPT
+42 ЭÒé²»¿ÉÓÃ
+$ EPROTONOSUPPORT
+43 ²»Ö§³ÖµÄЭÒé
+$ ESOCKTNOSUPPORT
+44 ²»Ö§³ÖµÄ socket ÀàÐÍ
+$ EOPNOTSUPP
+45 ²»Ö§³ÖµÄ²Ù×÷
+$ EPFNOSUPPORT
+46 ²»Ö§³ÖµÄЭÒé×å
+$ EAFNOSUPPORT
+47 ЭÒé×å²»Ö§³ÖµÄµØÖ·×å
+$ EADDRINUSE
+48 µØÖ·Òѱ»Õ¼ÓÃ
+$ EADDRNOTAVAIL
+49 ÎÞ·¨Ö¸¶¨ÇëÇóµÄµØÖ·
+$ ENETDOWN
+50 ÍøÂçÒѹرÕ
+$ ENETUNREACH
+51 ÍøÂç²»¿É´ï
+$ ENETRESET
+52 ¸´Î»µ¼ÖÂÍøÂçÁ¬½Ó¶ªÊ§
+$ ECONNABORTED
+53 Èí¼þµ¼ÖµÄÁ¬½ÓÖÐÖ¹
+$ ECONNRESET
+54 ¶Ô·½¸´Î»ÁËÁ¬½Ó
+$ ENOBUFS
+55 »º³åÇø¿Õ¼ä²»×ã
+$ EISCONN
+56 socket ÒÑÁ¬½Ó
+$ ENOTCONN
+57 socket δÁ¬½Ó
+$ ESHUTDOWN
+58 socket shutdown Ö®ºóÎÞ·¨·¢ËÍÊý¾Ý
+$ ETOOMANYREFS
+59 ÒýÓÃÊý¹ý¶à£ºÎÞ·¨Æ´½Ó
+$ ETIMEDOUT
+60 ²Ù×÷³¬Ê±
+$ ECONNREFUSED
+61 ¾Ü¾øÁ¬½Ó
+$ ELOOP
+62 ·ûºÅÁ´½Ó²ãÊý¹ý¶à
+$ ENAMETOOLONG
+63 ÎļþÃû¹ý³¤
+$ EHOSTDOWN
+64 Ö÷»úÒѹرÕ
+$ EHOSTUNREACH
+65 ûÓе½Ö÷»úµÄ·ÓÉ
+$ ENOTEMPTY
+66 Ŀ¼·Ç¿Õ
+$ EPROCLIM
+67 ½ø³ÌÊý³¬ÏÞ
+$ EUSERS
+68 Óû§Êý³¬ÏÞ
+$ EDQUOT
+69 ´ÅÅÌ¿Õ¼äÅä¶î³¬ÏÞ
+$ ESTALE
+70 NFS Îļþ¾ä±úÒÑʧЧ
+$ EREMOTE
+71 Ô¶³ÌĿ¼²ãÊý¹ý¶à
+$ EBADRPC
+72 RPC ½á¹¹ÎÞЧ
+$ ERPCMISMATCH
+73 RPC °æ±¾´íÎó
+$ EPROGUNAVAIL
+74 RPC ³ÌÐò²»¿ÉÓÃ
+$ EPROGMISMATCH
+75 ³ÌÐò°æ±¾´íÎó
+$ EPROCUNAVAIL
+76 δÌṩµÄÔ¶³Ìº¯Êý
+$ ENOLCK
+77 ²»Ö§³ÖËø
+$ ENOSYS
+78 ¹¦ÄÜδʵÏÖ
+$ EFTYPE
+79 ÎļþÀàÐÍ»ò¸ñʽÎÞЧ
+$ EAUTH
+80 Éí·ÝÎÞЧ
+$ ENEEDAUTH
+81 ÎÞÐÅÈÎƾ¾Ý
+$ EIDRM
+82 ÎÞ±êʶ·û
+$ ENOMSG
+83 ÎÞÏûÏ¢ÀàÐÍ
+$ EOVERFLOW
+84 ÊýÖµÒç³ö
+$ ECANCELED
+85 ²Ù×÷ÒÑÈ¡Ïû
+$ EILSEQ
+86 ÎÞЧ×Ö·ûÐòÁÐ
+$ ENOATTR
+87 ÎÞÀ©Õ¹ÊôÐÔ
+$ EDOOFUS
+88 ³ÌÐòÉè¼Æ´íÎó
+$ EBADMSG
+89 ÎÞЧÏûÏ¢
+$ EMULTIHOP
+90 Multihop attempted
+$ ENOLINK
+91 Link has been severed
+$ EPROTO
+92 ЭÒé´íÎó
+$ ENOTCAPABLE
+93 ȨÄܲ»×ã
+$
+$ strsignal() support catalog
+$
+$set 2
+$ SIGHUP
+1 ÖÕ¶ËÏß·¹Ò¶Ï
+$ SIGINT
+2 ÖжÏ
+$ SIGQUIT
+3 Í˳ö
+$ SIGILL
+4 ÎÞЧָÁî
+$ SIGTRAP
+5 ¸ú×Ù/BPT ÏÝÚå
+$ SIGABRT
+6 ÖÕÖ¹ÏÝÚå
+$ SIGEMT
+7 EMT ÏÝÚå
+$ SIGFPE
+8 ¸¡µãÒì³£
+$ SIGKILL
+9 Ç¿ÖÆÖÕÖ¹½ø³Ì
+$ SIGBUS
+10 ϵͳÄÚ´æ·ÃÎÊÔ½½ç
+$ SIGSEGV
+11 ÄÚ´æ·ÃÎÊÔ½½ç
+$ SIGSYS
+12 ÎÞЧϵͳµ÷ÓÃ
+$ SIGPIPE
+13 д¹ÜµÀʱ¶ÁÈ¡Õß²»´æÔÚ
+$ SIGALRM
+14 ʵʱµ¹¼Æʱµ½ÆÚ
+$ SIGTERM
+15 ÖÕÖ¹
+$ SIGURG
+16 ÐèÁ¢¼´´¦ÀíµÄ I/O ½ô¼±×´¿ö
+$ SIGSTOP
+17 ¹ÒÆð (ÐźÅ)
+$ SIGTSTP
+18 ¹ÒÆð
+$ SIGCONT
+19 ¼ÌÐøÔËÐÐ
+$ SIGCHLD
+20 ×Ó½ø³ÌÖÕÖ¹
+$ SIGTTIN
+21 TTY ÊäÈë×èÈû
+$ SIGTTOU
+22 TTY Êä³ö×èÈû
+$ SIGIO
+23 I/O ¾ÍÐ÷
+$ SIGXCPU
+24 CPU ʹÓÃʱ¼ä¹ý³¤
+$ SIGXFSZ
+25 Îļþ³ß´ç¹ý´ó
+$ SIGVTALRM
+26 ÐéÄâµ¹¼Æʱµ½ÆÚ
+$ SIGPROF
+27 ÆÊÎöµ¹¼Æʱµ½ÆÚ
+$ SIGWINCH
+28 ´°¿Ú³ß´ç±ä»¯
+$ SIGINFO
+29 ÇëÇóÐÅÏ¢
+$ SIGUSR1
+30 Óû§×Ô¶¨ÒåÐźŠ1
+$ SIGUSR2
+31 Óû§×Ô¶¨ÒåÐźŠ2
+$
+$ gai_strerror() support catalog
+$
+$set 3
+$ 1 (obsolete)
+1 Ö÷»úÃûʹÓÃÁ˲»Ö§³ÖµÄµØÖ·ÀàÐÍ
+$ EAI_AGAIN
+2 ÔÝʱÎÞ·¨½âÎöÃû³Æ
+$ EAI_BADFLAGS
+3 ÎÞЧµÄ ai_flags Öµ
+$ EAI_FAIL
+4 ½âÎöÃû³Æʱ³öÏÖÎÞ·¨»Ö¸´µÄ´íÎó
+$ EAI_FAMILY
+5 ²»Ö§³ÖµÄµØÖ·ÐÅÏ¢ÀàÐÍ
+$ EAI_MEMORY
+6 ÄÚ´æ·ÖÅäʧ°Ü
+$ 7 (obsolete)
+7 Ö÷»úÃûÎÞÏà¹ØÁªµÄµØÖ·
+$ EAI_NONAME
+8 δ֪µÄÖ÷»úÃû»ò·þÎñÃû
+$ EAI_SERVICE
+9 Ì×½Ó×ÖÀàÐͲ»Ö§³Ö´Ë·þÎñÃû
+$ EAI_SOCKTYPE
+10 ²»Ö§³ÖµÄÌ×½Ó×ÖÀàÐÍ
+$ EAI_SYSTEM
+11 ϵͳµ÷Ó÷µ»ØÁË errno ´íÎó
+$ EAI_BADHINTS
+12 ÎÞЧÌáʾ²ÎÊý
+$ EAI_PROTOCOL
+13 δ֪ЭÒé
+$ EAI_OVERFLOW
+14 ²ÎÊý»º³åÒç³ö
+$ 0
+32766 ³É¹¦
+$ NL_MSGMAX
+32767 δ֪´íÎó
diff --git a/lib/libc/nls/zh_CN.GB2312.msg b/lib/libc/nls/zh_CN.GB2312.msg
new file mode 100644
index 0000000..46ebb42
--- /dev/null
+++ b/lib/libc/nls/zh_CN.GB2312.msg
@@ -0,0 +1,297 @@
+$ $FreeBSD$
+$
+$ Message catalog for zh_CN.GB2312 locale
+$
+$ Derived from FreeBSD: head/lib/libc/nls/zh_CN.UTF-8.msg 244756 2012-12-28 01:09:30Z delphij
+$
+$ strerror() support catalog
+$
+$set 1
+$ EPERM
+1 ²»ÔÊÐíµÄ²Ù×÷
+$ ENOENT
+2 Îļþ»òĿ¼²»´æÔÚ
+$ ESRCH
+3 ½ø³Ì²»´æÔÚ
+$ EINTR
+4 ϵͳµ÷ÓÃÖÐÖ¹
+$ EIO
+5 ÊäÈë/Êä³ö´íÎó
+$ ENXIO
+6 δÅäÖõÄÉ豸
+$ E2BIG
+7 ²ÎÊý±í¹ý³¤
+$ ENOEXEC
+8 ¿ÉÖ´ÐÐÎļþ¸ñʽ´íÎó
+$ EBADF
+9 ÎļþÃèÊö·ûÎÞЧ
+$ ECHILD
+10 Ö¸¶¨×Ó½ø³Ì²»´æÔÚ
+$ EDEADLK
+11 ´Ë²Ù×÷»áµ¼ÖÂËÀËø
+$ ENOMEM
+12 ÎÞ·¨·ÖÅäÄÚ´æ
+$ EACCES
+13 ¾Ü¾ø·ÃÎÊ
+$ EFAULT
+14 ÎÞЧµØÖ·
+$ ENOTBLK
+15 ¸Ã²Ù×÷ÐèÒª¿éÉ豸
+$ EBUSY
+16 É豸ʹÓÃÖÐ
+$ EEXIST
+17 ÎļþÒÑ´æÔÚ
+$ EXDEV
+18 Á´½Ó¿çÉ豸
+$ ENODEV
+19 É豸²»Ö§³Ö´Ë²Ù×÷
+$ ENOTDIR
+20 ¶ÔÏó·ÇĿ¼
+$ EISDIR
+21 ¶ÔÏóΪĿ¼
+$ EINVAL
+22 ²ÎÊýÎÞЧ
+$ ENFILE
+23 ϵͳ´ò¿ªÎļþ¹ý¶à
+$ EMFILE
+24 ½ø³Ì´ò¿ªÎļþ¹ý¶à
+$ ENOTTY
+25 ÎÞЧÉ豸 ioctl
+$ ETXTBSY
+26 ¿ÉÖ´ÐÐÎļþæ
+$ EFBIG
+27 Îļþ¹ý´ó
+$ ENOSPC
+28 É豸ÎÞ¿ÉÓÿռä
+$ ESPIPE
+29 ²»ÔÊÐíÖ´ÐÐ seek ²Ù×÷
+$ EROFS
+30 Îļþϵͳֻ¶Á
+$ EMLINK
+31 ÎļþÁ´½Ó¹ý¶à
+$ EPIPE
+32 ¹ÜµÀÒÑÖÐÖ¹
+$ EDOM
+33 ÊýÖµ²ÎÊýÔ½½ç
+$ ERANGE
+34 ½á¹û¹ý´ó
+$ EAGAIN, EWOULDBLOCK
+35 ×ÊÔ´ÔÝʱ²»¿ÉÓÃ
+$ EINPROGRESS
+36 ²Ù×÷½øÐÐÖÐ
+$ EALREADY
+37 ²Ù×÷ÒÑ¿ªÊ¼
+$ ENOTSOCK
+38 ³¢ÊÔÔÚ·Ç socket ÉÏÖ´ÐÐ socket ²Ù×÷
+$ EDESTADDRREQ
+39 ÐèҪĿµÄµØÖ·
+$ EMSGSIZE
+40 ÏûÏ¢¹ý³¤
+$ EPROTOTYPE
+41 socketЭÒéÀàÐÍ´íÎó
+$ ENOPROTOOPT
+42 ЭÒé²»¿ÉÓÃ
+$ EPROTONOSUPPORT
+43 ²»Ö§³ÖµÄЭÒé
+$ ESOCKTNOSUPPORT
+44 ²»Ö§³ÖµÄ socket ÀàÐÍ
+$ EOPNOTSUPP
+45 ²»Ö§³ÖµÄ²Ù×÷
+$ EPFNOSUPPORT
+46 ²»Ö§³ÖµÄЭÒé×å
+$ EAFNOSUPPORT
+47 ЭÒé×å²»Ö§³ÖµÄµØÖ·×å
+$ EADDRINUSE
+48 µØÖ·Òѱ»Õ¼ÓÃ
+$ EADDRNOTAVAIL
+49 ÎÞ·¨Ö¸¶¨ÇëÇóµÄµØÖ·
+$ ENETDOWN
+50 ÍøÂçÒѹرÕ
+$ ENETUNREACH
+51 ÍøÂç²»¿É´ï
+$ ENETRESET
+52 ¸´Î»µ¼ÖÂÍøÂçÁ¬½Ó¶ªÊ§
+$ ECONNABORTED
+53 Èí¼þµ¼ÖµÄÁ¬½ÓÖÐÖ¹
+$ ECONNRESET
+54 ¶Ô·½¸´Î»ÁËÁ¬½Ó
+$ ENOBUFS
+55 »º³åÇø¿Õ¼ä²»×ã
+$ EISCONN
+56 socket ÒÑÁ¬½Ó
+$ ENOTCONN
+57 socket δÁ¬½Ó
+$ ESHUTDOWN
+58 socket shutdown Ö®ºóÎÞ·¨·¢ËÍÊý¾Ý
+$ ETOOMANYREFS
+59 ÒýÓÃÊý¹ý¶à£ºÎÞ·¨Æ´½Ó
+$ ETIMEDOUT
+60 ²Ù×÷³¬Ê±
+$ ECONNREFUSED
+61 ¾Ü¾øÁ¬½Ó
+$ ELOOP
+62 ·ûºÅÁ´½Ó²ãÊý¹ý¶à
+$ ENAMETOOLONG
+63 ÎļþÃû¹ý³¤
+$ EHOSTDOWN
+64 Ö÷»úÒѹرÕ
+$ EHOSTUNREACH
+65 ûÓе½Ö÷»úµÄ·ÓÉ
+$ ENOTEMPTY
+66 Ŀ¼·Ç¿Õ
+$ EPROCLIM
+67 ½ø³ÌÊý³¬ÏÞ
+$ EUSERS
+68 Óû§Êý³¬ÏÞ
+$ EDQUOT
+69 ´ÅÅÌ¿Õ¼äÅä¶î³¬ÏÞ
+$ ESTALE
+70 NFS Îļþ¾ä±úÒÑʧЧ
+$ EREMOTE
+71 Ô¶³ÌĿ¼²ãÊý¹ý¶à
+$ EBADRPC
+72 RPC ½á¹¹ÎÞЧ
+$ ERPCMISMATCH
+73 RPC °æ±¾´íÎó
+$ EPROGUNAVAIL
+74 RPC ³ÌÐò²»¿ÉÓÃ
+$ EPROGMISMATCH
+75 ³ÌÐò°æ±¾´íÎó
+$ EPROCUNAVAIL
+76 δÌṩµÄÔ¶³Ìº¯Êý
+$ ENOLCK
+77 ²»Ö§³ÖËø
+$ ENOSYS
+78 ¹¦ÄÜδʵÏÖ
+$ EFTYPE
+79 ÎļþÀàÐÍ»ò¸ñʽÎÞЧ
+$ EAUTH
+80 Éí·ÝÎÞЧ
+$ ENEEDAUTH
+81 ÎÞÐÅÈÎƾ¾Ý
+$ EIDRM
+82 ÎÞ±êʶ·û
+$ ENOMSG
+83 ÎÞÏûÏ¢ÀàÐÍ
+$ EOVERFLOW
+84 ÊýÖµÒç³ö
+$ ECANCELED
+85 ²Ù×÷ÒÑÈ¡Ïû
+$ EILSEQ
+86 ÎÞЧ×Ö·ûÐòÁÐ
+$ ENOATTR
+87 ÎÞÀ©Õ¹ÊôÐÔ
+$ EDOOFUS
+88 ³ÌÐòÉè¼Æ´íÎó
+$ EBADMSG
+89 ÎÞЧÏûÏ¢
+$ EMULTIHOP
+90 Multihop attempted
+$ ENOLINK
+91 Link has been severed
+$ EPROTO
+92 ЭÒé´íÎó
+$ ENOTCAPABLE
+93 ȨÄܲ»×ã
+$
+$ strsignal() support catalog
+$
+$set 2
+$ SIGHUP
+1 ÖÕ¶ËÏß·¹Ò¶Ï
+$ SIGINT
+2 ÖжÏ
+$ SIGQUIT
+3 Í˳ö
+$ SIGILL
+4 ÎÞЧָÁî
+$ SIGTRAP
+5 ¸ú×Ù/BPT ÏÝÚå
+$ SIGABRT
+6 ÖÕÖ¹ÏÝÚå
+$ SIGEMT
+7 EMT ÏÝÚå
+$ SIGFPE
+8 ¸¡µãÒì³£
+$ SIGKILL
+9 Ç¿ÖÆÖÕÖ¹½ø³Ì
+$ SIGBUS
+10 ϵͳÄÚ´æ·ÃÎÊÔ½½ç
+$ SIGSEGV
+11 ÄÚ´æ·ÃÎÊÔ½½ç
+$ SIGSYS
+12 ÎÞЧϵͳµ÷ÓÃ
+$ SIGPIPE
+13 д¹ÜµÀʱ¶ÁÈ¡Õß²»´æÔÚ
+$ SIGALRM
+14 ʵʱµ¹¼Æʱµ½ÆÚ
+$ SIGTERM
+15 ÖÕÖ¹
+$ SIGURG
+16 ÐèÁ¢¼´´¦ÀíµÄ I/O ½ô¼±×´¿ö
+$ SIGSTOP
+17 ¹ÒÆð (ÐźÅ)
+$ SIGTSTP
+18 ¹ÒÆð
+$ SIGCONT
+19 ¼ÌÐøÔËÐÐ
+$ SIGCHLD
+20 ×Ó½ø³ÌÖÕÖ¹
+$ SIGTTIN
+21 TTY ÊäÈë×èÈû
+$ SIGTTOU
+22 TTY Êä³ö×èÈû
+$ SIGIO
+23 I/O ¾ÍÐ÷
+$ SIGXCPU
+24 CPU ʹÓÃʱ¼ä¹ý³¤
+$ SIGXFSZ
+25 Îļþ³ß´ç¹ý´ó
+$ SIGVTALRM
+26 ÐéÄâµ¹¼Æʱµ½ÆÚ
+$ SIGPROF
+27 ÆÊÎöµ¹¼Æʱµ½ÆÚ
+$ SIGWINCH
+28 ´°¿Ú³ß´ç±ä»¯
+$ SIGINFO
+29 ÇëÇóÐÅÏ¢
+$ SIGUSR1
+30 Óû§×Ô¶¨ÒåÐźŠ1
+$ SIGUSR2
+31 Óû§×Ô¶¨ÒåÐźŠ2
+$
+$ gai_strerror() support catalog
+$
+$set 3
+$ 1 (obsolete)
+1 Ö÷»úÃûʹÓÃÁ˲»Ö§³ÖµÄµØÖ·ÀàÐÍ
+$ EAI_AGAIN
+2 ÔÝʱÎÞ·¨½âÎöÃû³Æ
+$ EAI_BADFLAGS
+3 ÎÞЧµÄ ai_flags Öµ
+$ EAI_FAIL
+4 ½âÎöÃû³Æʱ³öÏÖÎÞ·¨»Ö¸´µÄ´íÎó
+$ EAI_FAMILY
+5 ²»Ö§³ÖµÄµØÖ·ÐÅÏ¢ÀàÐÍ
+$ EAI_MEMORY
+6 ÄÚ´æ·ÖÅäʧ°Ü
+$ 7 (obsolete)
+7 Ö÷»úÃûÎÞÏà¹ØÁªµÄµØÖ·
+$ EAI_NONAME
+8 δ֪µÄÖ÷»úÃû»ò·þÎñÃû
+$ EAI_SERVICE
+9 Ì×½Ó×ÖÀàÐͲ»Ö§³Ö´Ë·þÎñÃû
+$ EAI_SOCKTYPE
+10 ²»Ö§³ÖµÄÌ×½Ó×ÖÀàÐÍ
+$ EAI_SYSTEM
+11 ϵͳµ÷Ó÷µ»ØÁË errno ´íÎó
+$ EAI_BADHINTS
+12 ÎÞЧÌáʾ²ÎÊý
+$ EAI_PROTOCOL
+13 δ֪ЭÒé
+$ EAI_OVERFLOW
+14 ²ÎÊý»º³åÒç³ö
+$ 0
+32766 ³É¹¦
+$ NL_MSGMAX
+32767 δ֪´íÎó
-- 
cgit v1.1


From b69ff2af2f03611d0e09375381f91413c7d309a2 Mon Sep 17 00:00:00 2001
From: bapt <bapt@FreeBSD.org>
Date: Fri, 28 Dec 2012 20:19:54 +0000
Subject: Do not leave parts of the new group uninitialized in gr_dup().

Submitted by:	Christoph Mallon <christoph.mallon@gmx.de>
Reported by:	pjd
---
 lib/libutil/gr_util.c | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'lib')

diff --git a/lib/libutil/gr_util.c b/lib/libutil/gr_util.c
index 437fd78..759e6e8 100644
--- a/lib/libutil/gr_util.c
+++ b/lib/libutil/gr_util.c
@@ -461,10 +461,14 @@ gr_dup(const struct group *gr)
 	if (gr->gr_name != NULL) {
 		newgr->gr_name = dst;
 		dst = stpcpy(dst, gr->gr_name) + 1;
+	} else {
+		newgr->gr_name = NULL;
 	}
 	if (gr->gr_passwd != NULL) {
 		newgr->gr_passwd = dst;
 		dst = stpcpy(dst, gr->gr_passwd) + 1;
+	} else {
+		newgr->gr_passwd = NULL;
 	}
 	newgr->gr_gid = gr->gr_gid;
 	if (gr->gr_mem != NULL) {
-- 
cgit v1.1


From 83035e89840797684386d0bd3a46f2c24030771b Mon Sep 17 00:00:00 2001
From: bapt <bapt@FreeBSD.org>
Date: Fri, 28 Dec 2012 20:21:14 +0000
Subject: malloc() sets errno to ENOMEM already.

Submitted by:	Christoph Mallon <christoph.mallon@gmx.de>
---
 lib/libutil/gr_util.c | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

(limited to 'lib')

diff --git a/lib/libutil/gr_util.c b/lib/libutil/gr_util.c
index 759e6e8..6676720 100644
--- a/lib/libutil/gr_util.c
+++ b/lib/libutil/gr_util.c
@@ -505,17 +505,13 @@ gr_add(struct group *gr, char *newmember)
 	}
 	/* Allocate enough for current pointers + 1 more and NULL marker */
 	mlen = (num_mem + 2) * sizeof(*gr->gr_mem);
-	if ((members = malloc(mlen)) == NULL) {
+	if ((members = malloc(mlen)) == NULL)
 		errno = ENOMEM;
-		return (NULL);
-	}
 	memcpy(members, gr->gr_mem, num_mem * sizeof(*gr->gr_mem));
 	members[num_mem++] = newmember;
 	members[num_mem] = NULL;
 	gr->gr_mem = members;
 	newgr = gr_dup(gr);
-	if (newgr == NULL)
-		errno = ENOMEM;
 	free(members);
 	return (newgr);
 }
-- 
cgit v1.1


From 3c6bd22efebf26dd6f310e2b25fbd4a1f544aee1 Mon Sep 17 00:00:00 2001
From: bapt <bapt@FreeBSD.org>
Date: Fri, 28 Dec 2012 20:30:04 +0000
Subject: errno = ENOMEM was supposed to be removed not return (NULL);

Submitted by:	gcooper
---
 lib/libutil/gr_util.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libutil/gr_util.c b/lib/libutil/gr_util.c
index 6676720..5f803af 100644
--- a/lib/libutil/gr_util.c
+++ b/lib/libutil/gr_util.c
@@ -506,7 +506,7 @@ gr_add(struct group *gr, char *newmember)
 	/* Allocate enough for current pointers + 1 more and NULL marker */
 	mlen = (num_mem + 2) * sizeof(*gr->gr_mem);
 	if ((members = malloc(mlen)) == NULL)
-		errno = ENOMEM;
+		return (NULL);
 	memcpy(members, gr->gr_mem, num_mem * sizeof(*gr->gr_mem));
 	members[num_mem++] = newmember;
 	members[num_mem] = NULL;
-- 
cgit v1.1


From 9db4d3200a9d4f5851e975b66f6e53e8d66b55ae Mon Sep 17 00:00:00 2001
From: bapt <bapt@FreeBSD.org>
Date: Fri, 28 Dec 2012 20:44:10 +0000
Subject: Simplify pointing dst after the end of all the gr_mem pointers in
 newgr

Submitted by:	pjd
Reviewed by:	db
---
 lib/libutil/gr_util.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/libutil/gr_util.c b/lib/libutil/gr_util.c
index 5f803af..acb9767 100644
--- a/lib/libutil/gr_util.c
+++ b/lib/libutil/gr_util.c
@@ -456,8 +456,7 @@ gr_dup(const struct group *gr)
 	else
 		newgr->gr_mem = NULL;
 	/* point dst after the end of all the gr_mem pointers in newgr */
-	dst = (char *)newgr + sizeof(struct group) +
-	    (num_mem + 1) * sizeof(*gr->gr_mem);
+	dst = (char *)&newgr->gr_mem[num_mem + 1];
 	if (gr->gr_name != NULL) {
 		newgr->gr_name = dst;
 		dst = stpcpy(dst, gr->gr_name) + 1;
-- 
cgit v1.1


From 1e6b48e616e0254370d121cb6a306ec4ceaa9c96 Mon Sep 17 00:00:00 2001
From: nwhitehorn <nwhitehorn@FreeBSD.org>
Date: Sun, 30 Dec 2012 16:28:06 +0000
Subject: With the old sade removed, libdisk is no longer used by anything in
 HEAD and uses a number of problematic pre-gpart interfaces. Since it has been
 entirely obsoleted by interfaces in geom, remove it.

---
 lib/Makefile                     |   1 -
 lib/libdisk/Makefile             |  43 ---
 lib/libdisk/blocks.c             |  50 ----
 lib/libdisk/change.c             | 129 ---------
 lib/libdisk/chunk.c              | 595 ---------------------------------------
 lib/libdisk/create_chunk.c       | 296 -------------------
 lib/libdisk/disk.c               | 411 ---------------------------
 lib/libdisk/libdisk.3            | 341 ----------------------
 lib/libdisk/libdisk.h            | 368 ------------------------
 lib/libdisk/open_disk.c          | 311 --------------------
 lib/libdisk/open_ia64_disk.c     | 239 ----------------
 lib/libdisk/rules.c              | 296 -------------------
 lib/libdisk/tst01.c              | 323 ---------------------
 lib/libdisk/write_amd64_disk.c   | 204 --------------
 lib/libdisk/write_arm_disk.c     |  47 ----
 lib/libdisk/write_disk.c         |  76 -----
 lib/libdisk/write_i386_disk.c    | 204 --------------
 lib/libdisk/write_ia64_disk.c    | 423 ----------------------------
 lib/libdisk/write_mips_disk.c    |  48 ----
 lib/libdisk/write_pc98_disk.c    | 179 ------------
 lib/libdisk/write_powerpc_disk.c |  64 -----
 lib/libdisk/write_sparc64_disk.c | 106 -------
 22 files changed, 4754 deletions(-)
 delete mode 100644 lib/libdisk/Makefile
 delete mode 100644 lib/libdisk/blocks.c
 delete mode 100644 lib/libdisk/change.c
 delete mode 100644 lib/libdisk/chunk.c
 delete mode 100644 lib/libdisk/create_chunk.c
 delete mode 100644 lib/libdisk/disk.c
 delete mode 100644 lib/libdisk/libdisk.3
 delete mode 100644 lib/libdisk/libdisk.h
 delete mode 100644 lib/libdisk/open_disk.c
 delete mode 100644 lib/libdisk/open_ia64_disk.c
 delete mode 100644 lib/libdisk/rules.c
 delete mode 100644 lib/libdisk/tst01.c
 delete mode 100644 lib/libdisk/write_amd64_disk.c
 delete mode 100644 lib/libdisk/write_arm_disk.c
 delete mode 100644 lib/libdisk/write_disk.c
 delete mode 100644 lib/libdisk/write_i386_disk.c
 delete mode 100644 lib/libdisk/write_ia64_disk.c
 delete mode 100644 lib/libdisk/write_mips_disk.c
 delete mode 100644 lib/libdisk/write_pc98_disk.c
 delete mode 100644 lib/libdisk/write_powerpc_disk.c
 delete mode 100644 lib/libdisk/write_sparc64_disk.c

(limited to 'lib')

diff --git a/lib/Makefile b/lib/Makefile
index 6e1a402..3dd274e 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -69,7 +69,6 @@ SUBDIR=	${SUBDIR_ORDERED} \
 	libcompat \
 	libdevinfo \
 	libdevstat \
-	libdisk \
 	libdwarf \
 	libedit \
 	${_libefi} \
diff --git a/lib/libdisk/Makefile b/lib/libdisk/Makefile
deleted file mode 100644
index 9f3e093..0000000
--- a/lib/libdisk/Makefile
+++ /dev/null
@@ -1,43 +0,0 @@
-# $FreeBSD$
-
-.if ${MACHINE_CPUARCH} == "ia64"
-_open_disk=	open_ia64_disk.c
-.else
-_change	=	change.c
-_open_disk=	open_disk.c
-.endif
-
-LIB=	disk
-SRCS=	blocks.c ${_change} chunk.c create_chunk.c disk.c ${_open_disk} \
-	rules.c write_disk.c 
-SRCS+=  write_${MACHINE}_disk.c
-
-INCS=	libdisk.h
-
-WARNS?=	2
-
-CFLAGS+=	-I${.CURDIR}/../../sys/geom
-
-.if ${MACHINE} == "pc98"
-CFLAGS+=	-DPC98
-.endif
-
-CLEANFILES+=	tmp.c tst01 tst01.o
-NO_PROFILE=
-NO_PIC=
-
-MAN=		libdisk.3
-
-.include <bsd.lib.mk>
-
-tst01: tst01.o libdisk.a
-	cc ${CFLAGS} -static tst01.o -o tst01 libdisk.a
-
-ad0:	all install tst01
-	./tst01 ad0
-
-da0:	all install tst01
-	./tst01 da0
-
-da1:	all install tst01
-	./tst01 da1
diff --git a/lib/libdisk/blocks.c b/lib/libdisk/blocks.c
deleted file mode 100644
index 313c5cc..0000000
--- a/lib/libdisk/blocks.c
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * ----------------------------------------------------------------------------
- * "THE BEER-WARE LICENSE" (Revision 42):
- * <phk@FreeBSD.org> wrote this file.  As long as you retain this notice you
- * can do whatever you want with this stuff. If we meet some day, and you think
- * this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
- * ----------------------------------------------------------------------------
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include "libdisk.h"
-
-void *
-read_block(int fd, daddr_t block, u_long sector_size)
-{
-	void *foo;
-	int i;
-
-	foo = malloc(sector_size);
-	if (foo == NULL)
-		return (NULL);
-	if (-1 == lseek(fd, (off_t)block * sector_size, SEEK_SET)) {
-		free (foo);
-		return (NULL);
-	}
-	i = read(fd, foo, sector_size);
-	if ((int)sector_size != i) {
-		free (foo);
-		return (NULL);
-	}
-	return foo;
-}
-
-int
-write_block(int fd, daddr_t block, const void *foo, u_long sector_size)
-{
-	int i;
-
-	if (-1 == lseek(fd, (off_t)block * sector_size, SEEK_SET))
-		return (-1);
-	i = write(fd, foo, sector_size);
-	if ((int)sector_size != i)
-		return (-1);
-	return 0;
-}
diff --git a/lib/libdisk/change.c b/lib/libdisk/change.c
deleted file mode 100644
index 5f0c9c6..0000000
--- a/lib/libdisk/change.c
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * ----------------------------------------------------------------------------
- * "THE BEER-WARE LICENSE" (Revision 42):
- * <phk@FreeBSD.org> wrote this file.  As long as you retain this notice you
- * can do whatever you want with this stuff. If we meet some day, and you think
- * this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
- * ----------------------------------------------------------------------------
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <string.h>
-#include <sys/types.h>
-#include "libdisk.h"
-
-void
-Set_Bios_Geom(struct disk *disk, u_long cyl, u_long hd, u_long sect)
-{
-
-	disk->bios_cyl = cyl;
-	disk->bios_hd = hd;
-	disk->bios_sect = sect;
-}
-
-void
-Sanitize_Bios_Geom(struct disk *disk)
-{
-	int sane;
-
-	sane = 1;
-
-	if (disk->bios_cyl >= 65536)
-		sane = 0;
-#ifdef PC98
-	if (disk->bios_hd >= 256)
-		sane = 0;
-	if (disk->bios_sect >= 256)
-		sane = 0;
-#else
-	if (disk->bios_hd > 256)
-		sane = 0;
-	if (disk->bios_sect > 63)
-		sane = 0;
-#endif
-#if 0	/* Disable a check on a disk size.  It's too strict. */
-	if (disk->bios_cyl * disk->bios_hd * disk->bios_sect !=
-	    disk->chunks->size)
-		sane = 0;
-#endif
-	if (sane)
-		return;
-
-	/* First try something that IDE can handle */
-	disk->bios_sect = 63;
-	disk->bios_hd = 16;
-	disk->bios_cyl = disk->chunks->size /
-		(disk->bios_sect * disk->bios_hd);
-
-#ifdef PC98
-	if (disk->bios_cyl < 65536)
-#else
-	if (disk->bios_cyl < 1024)
-#endif
-		return;
-
-	/* Hmm, try harder... */
-	/* Assume standard SCSI parameter */
-#ifdef PC98
-	disk->bios_sect = 128;
-	disk->bios_hd = 8;
-#else
-	disk->bios_hd = 255;
-#endif
-	disk->bios_cyl = disk->chunks->size /
-		(disk->bios_sect * disk->bios_hd);
-
-#ifdef PC98
-	if (disk->bios_cyl < 65536)
-		return;
-
-	/* Assume UIDE-133/98-A Challenger BIOS 0.9821C parameter */
-	disk->bios_sect = 255;
-	disk->bios_hd = 16;
-	disk->bios_cyl = disk->chunks->size /
-		(disk->bios_sect * disk->bios_hd);
-
-	if (disk->bios_cyl < 65536)
-		return;
-
-	/* BIG-na-Drive? */
-	disk->bios_hd = 255;
-	disk->bios_cyl = disk->chunks->size /
-		(disk->bios_sect * disk->bios_hd);
-#endif
-}
-
-void
-All_FreeBSD(struct disk *d, int force_all)
-{
-	struct chunk *c;
-	int type;
-
-#ifdef PC98
-	type = 0xc494;
-#else
-	type = 0xa5;
-#endif
-
-again:
-	for (c = d->chunks->part; c; c = c->next)
-		if (c->type != unused) {
-			Delete_Chunk(d, c);
-			goto again;
-		}
-	c = d->chunks;
-	if (force_all) {
-		Sanitize_Bios_Geom(d);
-		Create_Chunk(d, c->offset, c->size, freebsd, type,
-		    CHUNK_FORCE_ALL, "FreeBSD");
-	} else {
-		Create_Chunk(d, c->offset, c->size, freebsd, type, 0,
-		    "FreeBSD");
-	}
-}
diff --git a/lib/libdisk/chunk.c b/lib/libdisk/chunk.c
deleted file mode 100644
index fb43ef5..0000000
--- a/lib/libdisk/chunk.c
+++ /dev/null
@@ -1,595 +0,0 @@
-/*
- * ----------------------------------------------------------------------------
- * "THE BEER-WARE LICENSE" (Revision 42):
- * <phk@FreeBSD.org> wrote this file.  As long as you retain this notice you
- * can do whatever you want with this stuff. If we meet some day, and you think
- * this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
- * ----------------------------------------------------------------------------
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/types.h>
-#include <sys/stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <string.h>
-#include <err.h>
-#include "libdisk.h"
-
-struct chunk *
-New_Chunk(void)
-{
-	struct chunk *c;
-
-	c = malloc(sizeof *c);
-	if (c != NULL)
-		memset(c, 0, sizeof *c);
-	return (c);
-}
-
-/* Is c2 completely inside c1 ? */
-
-static int
-Chunk_Inside(const struct chunk *c1, const struct chunk *c2)
-{
-	/* if c1 ends before c2 do */
-	if (c1->end < c2->end)
-		return 0;
-	/* if c1 starts after c2 do */
-	if (c1->offset > c2->offset)
-		return 0;
-	return 1;
-}
-
-static struct chunk *
-Find_Mother_Chunk(struct chunk *chunks, daddr_t offset, daddr_t end,
-		  chunk_e type)
-{
-	struct chunk *c1, *c2, ct;
-
-	ct.offset = offset;
-	ct.end = end;
-	switch (type) {
-	case whole:
-		if (Chunk_Inside(chunks, &ct))
-			return chunks;
-	case extended:
-		for (c1 = chunks->part; c1; c1 = c1->next) {
-			if (c1->type != type)
-				continue;
-			if (Chunk_Inside(c1, &ct))
-				return c1;
-		}
-		return 0;
-	case freebsd:
-		for (c1 = chunks->part; c1; c1 = c1->next) {
-			if (c1->type == type)
-				if (Chunk_Inside(c1, &ct))
-					return c1;
-			if (c1->type != extended)
-				continue;
-			for (c2 = c1->part; c2; c2 = c2->next)
-				if (c2->type == type && Chunk_Inside(c2, &ct))
-					return c2;
-		}
-		return 0;
-#ifdef __powerpc__
-	case apple:
-		for (c1 = chunks->part; c1; c1 = c1->next) {
-			if (c1->type == type)
-				if (Chunk_Inside(c1, &ct))
-					return c1;
-		}
-		return 0;
-#endif
-	default:
-		warn("Unsupported mother type in Find_Mother_Chunk");
-		return 0;
-	}
-}
-
-void
-Free_Chunk(struct chunk *c1)
-{
-	if(c1 == NULL)
-		return;
-	if(c1->private_data && c1->private_free)
-		(*c1->private_free)(c1->private_data);
-	if(c1->part != NULL)
-		Free_Chunk(c1->part);
-	if(c1->next != NULL)
-		Free_Chunk(c1->next);
-	if (c1->name != NULL)
-		free(c1->name);
-	if (c1->sname != NULL)
-		free(c1->sname);
-	free(c1);
-}
-
-struct chunk *
-Clone_Chunk(const struct chunk *c1)
-{
-	struct chunk *c2;
-
-	if(!c1)
-		return NULL;
-	c2 = New_Chunk();
-	if (c2 == NULL)
-		return NULL;
-	*c2 = *c1;
-	if (c1->private_data && c1->private_clone)
-		c2->private_data = c2->private_clone(c2->private_data);
-	c2->name = strdup(c2->name);
-	if (c2->sname != NULL)
-		c2->sname = strdup(c2->sname);
-	c2->next = Clone_Chunk(c2->next);
-	c2->part = Clone_Chunk(c2->part);
-	return c2;
-}
-
-int
-Insert_Chunk(struct chunk *c2, daddr_t offset, daddr_t size, const char *name,
-	chunk_e type, int subtype, u_long flags, const char *sname)
-{
-	struct chunk *ct,*cs;
-
-	/* We will only insert into empty spaces */
-	if (c2->type != unused)
-		return __LINE__;
-
-	ct = New_Chunk();
-	if (ct == NULL)
-		return __LINE__;
-	ct->disk = c2->disk;
-	ct->offset = offset;
-	ct->size = size;
-	ct->end = offset + size - 1;
-	ct->type = type;
-	if (sname != NULL)
-		ct->sname = strdup(sname);
-	ct->name = strdup(name);
-	ct->subtype = subtype;
-	ct->flags = flags;
-
-	if (!Chunk_Inside(c2, ct)) {
-		Free_Chunk(ct);
-		return __LINE__;
-	}
-
-	if ((type == freebsd || type == extended || type == apple)) {
-		cs = New_Chunk();
-		if (cs == NULL)
-			return __LINE__;
-		cs->disk = c2->disk;
-		cs->offset = offset;
-		cs->size = size;
-		cs->end = offset + size - 1;
-		cs->type = unused;
-		if (sname != NULL)
-			cs->sname = strdup(sname);
-		cs->name = strdup("-");
-		ct->part = cs;
-	}
-
-	/* Make a new chunk for any trailing unused space */
-	if (c2->end > ct->end) {
-		cs = New_Chunk();
-		if (cs == NULL)
-			 return __LINE__;
-		*cs = *c2;
-		cs->disk = c2->disk;
-		cs->offset = ct->end + 1;
-		cs->size = c2->end - ct->end;
-		if (c2->sname != NULL)
-			cs->sname = strdup(c2->sname);
-		if (c2->name)
-			cs->name = strdup(c2->name);
-		c2->next = cs;
-		c2->size -= c2->end - ct->end;
-		c2->end = ct->end;
-	}
-	/* If no leading unused space just occupy the old chunk */
-	if (c2->offset == ct->offset) {
-		c2->sname = ct->sname;
-		c2->name = ct->name;
-		c2->type = ct->type;
-		c2->part = ct->part;
-		c2->subtype = ct->subtype;
-		c2->flags = ct->flags;
-		ct->sname = NULL;
-		ct->name = NULL;
-		ct->part = 0;
-		Free_Chunk(ct);
-		return 0;
-	}
-	/* else insert new chunk and adjust old one */
-	c2->end = ct->offset - 1;
-	c2->size -= ct->size;
-	ct->next = c2->next;
-	c2->next = ct;
-	return 0;
-}
-
-int
-Add_Chunk(struct disk *d, daddr_t offset, daddr_t size, const char *name,
-	  chunk_e type, int subtype, u_long flags, const char *sname)
-{
-	struct chunk *c1, *c2, ct;
-	daddr_t end = offset + size - 1;
-	ct.offset = offset;
-	ct.end = end;
-	ct.size = size;
-
-	if (type == whole) {
-		d->chunks = c1 = New_Chunk();
-		if (c1 == NULL)
-			return __LINE__;
-		c2 = c1->part = New_Chunk();
-		if (c2 == NULL)
-			 return __LINE__;
-		c2->disk = c1->disk = d;
-		c2->offset = c1->offset = offset;
-		c2->size = c1->size = size;
-		c2->end = c1->end = end;
-		c1->sname = strdup(sname);
-		c2->sname = strdup("-");
-		c1->name = strdup(name);
-		c2->name = strdup("-");
-		c1->type = type;
-		c2->type = unused;
-		c1->flags = flags;
-		c1->subtype = subtype;
-		return 0;
-	}
-
-	c1 = 0;
-	/* PLATFORM POLICY BEGIN ------------------------------------- */
-	switch(platform) {
-	case p_i386:
-	case p_amd64:
-		switch (type) {
-		case fat:
-		case gpt:
-		case mbr:
-		case extended:
-		case freebsd:
-			c1 = Find_Mother_Chunk(d->chunks, offset, end, whole);
-			break;
-		case part:
-			c1 = Find_Mother_Chunk(d->chunks, offset, end, freebsd);
-			break;
-		default:
-			return(-1);
-		}
-		break;
-	case p_ia64:
-		switch (type) {
-		case freebsd:
-			subtype = 0xa5;
-			/* FALL THROUGH */
-		case fat:
-		case efi:
-		case mbr:
-			c1 = Find_Mother_Chunk(d->chunks, offset, end, whole);
-			break;
-		case part:
-			c1 = Find_Mother_Chunk(d->chunks, offset, end,
-			    freebsd);
-			if (!c1)
-				c1 = Find_Mother_Chunk(d->chunks, offset, end,
-				    whole);
-			break;
-		default:
-			return (-1);
-		}
-		break;
-	case p_pc98:
-		switch (type) {
-		case fat:
-		case pc98:
-		case freebsd:
-			c1 = Find_Mother_Chunk(d->chunks, offset, end, whole);
-			break;
-		case part:
-			c1 = Find_Mother_Chunk(d->chunks, offset, end, freebsd);
-			break;
-		default:
-			return(-1);
-		}
-		break;
-	case p_sparc64:
-	case p_alpha:
-		switch (type) {
-		case freebsd:
-			c1 = Find_Mother_Chunk(d->chunks, offset, end, whole);
-			break;
-		case part:
-			c1 = Find_Mother_Chunk(d->chunks, offset, end, freebsd);
-			break;
-		default:
-			return(-1);
-		}
-		break;
-	case p_ppc:
-		switch (type) {
-		case apple:
-			c1 = Find_Mother_Chunk(d->chunks, offset, end, whole);
-			break;
-		case part:
-			c1 = Find_Mother_Chunk(d->chunks, offset, end, apple);
-			break;
-		default:
-			return (-1);
-		}
-		break;
-	default:
-		return (-1);
-	}
-	/* PLATFORM POLICY END ---------------------------------------- */
-
-	if(!c1)
-		return __LINE__;
-	for(c2 = c1->part; c2; c2 = c2->next) {
-		if (c2->type != unused)
-			continue;
-		if(!Chunk_Inside(c2, &ct))
-			continue;
-/* PLATFORM POLICY BEGIN ------------------------------------- */
-		if (platform == p_sparc64) {
-			offset = Prev_Cyl_Aligned(d, offset);
-			size = Next_Cyl_Aligned(d, size);
-		} else if (platform == p_i386 || platform == p_pc98 ||
-		    platform == p_amd64) {
-			if (type != freebsd)
-				break;
-			if (!(flags & CHUNK_ALIGN))
-				break;
-			if (offset == d->chunks->offset &&
-			    end == d->chunks->end)
-				break;
-
-			/* Round down to prev cylinder */
-			offset = Prev_Cyl_Aligned(d,offset);
-			/* Stay inside the parent */
-			if (offset < c2->offset)
-				offset = c2->offset;
-			/* Round up to next cylinder */
-			offset = Next_Cyl_Aligned(d, offset);
-			/* Keep one track clear in front of parent */
-			if (offset == c1->offset)
-				offset = Next_Track_Aligned(d, offset + 1);
-			/* Work on the (end+1) */
-			size += offset;
-			/* Round up to cylinder */
-			size = Next_Cyl_Aligned(d, size);
-			/* Stay inside parent */
-			if ((size-1) > c2->end)
-				size = c2->end + 1;
-			/* Round down to cylinder */
-			size = Prev_Cyl_Aligned(d, size);
-
-			/* Convert back to size */
-			size -= offset;
-		}
-		break;
-
-/* PLATFORM POLICY END ------------------------------------- */
-	}
-	if (c2 == NULL)
-		return (__LINE__);
-	return Insert_Chunk(c2, offset, size, name, type, subtype, flags,
-			    sname);
-}
-
-char *
-ShowChunkFlags(struct chunk *c)
-{
-	static char ret[10];
-	int i = 0;
-
-	if (c->flags & CHUNK_ACTIVE)
-		ret[i++] = 'A';
-	if (c->flags & CHUNK_ALIGN)
-		ret[i++] = '=';
-	if (c->flags & CHUNK_IS_ROOT)
-		ret[i++] = 'R';
-	ret[i++] = '\0';
-
-	return ret;
-}
-
-static void
-Print_Chunk(struct chunk *c1, int offset)
-{
-	int i;
-
-	if (!c1)
-		return;
-	for (i = 0; i < offset - 2; i++)
-		putchar(' ');
-	for (; i < offset; i++)
-		putchar('-');
-	putchar('>');
-	for (; i < 10; i++)
-		putchar(' ');
-#ifndef __ia64__
-	printf("%p ", c1);
-#endif
-	printf("%8jd %8jd %8jd %-8s %-16s %-8s 0x%02x %s",
-	    (intmax_t)c1->offset, (intmax_t)c1->size, (intmax_t)c1->end,
-	    c1->name, c1->sname, chunk_name(c1->type), c1->subtype,
-	    ShowChunkFlags(c1));
-	putchar('\n');
-	Print_Chunk(c1->part, offset + 2);
-	Print_Chunk(c1->next, offset);
-}
-
-void
-Debug_Chunk(struct chunk *c1)
-{
-
-	Print_Chunk(c1, 2);
-}
-
-int
-Delete_Chunk(struct disk *d, struct chunk *c)
-{
-
-	return (Delete_Chunk2(d, c, 0));
-}
-
-int
-Delete_Chunk2(struct disk *d, struct chunk *c, int rflags)
-{
-	struct chunk *c1, *c2, *c3;
-	daddr_t offset = c->offset;
-
-	switch (c->type) {
-	case whole:
-	case unused:
-		return 1;
-	case extended:
-		c1 = Find_Mother_Chunk(d->chunks, c->offset, c->end, whole);
-		break;
-	case part:
-		c1 = Find_Mother_Chunk(d->chunks, c->offset, c->end, freebsd);
-#ifdef __ia64__
-		if (c1 == NULL)
-			c1 = Find_Mother_Chunk(d->chunks, c->offset, c->end,
-			    whole);
-#endif
-#ifdef __powerpc__
-		if (c1 == NULL)
-			c1 = Find_Mother_Chunk(d->chunks, c->offset, c->end,
-			    apple);
-#endif
-		break;
-	default:
-		c1 = Find_Mother_Chunk(d->chunks, c->offset, c->end, extended);
-		if (c1 == NULL)
-			c1 = Find_Mother_Chunk(d->chunks, c->offset, c->end,
-			    whole);
-		break;
-	}
-	if (c1 == NULL)
-		return 1;
-	for (c2 = c1->part; c2; c2 = c2->next) {
-		if (c2 == c) {
-			c2->type = unused;
-			c2->subtype = 0;
-			c2->flags = 0;
-			if (c2->sname != NULL)
-				free(c2->sname);
-			c2->sname = strdup("-");
-			free(c2->name);
-			c2->name = strdup("-");
-			Free_Chunk(c2->part);
-			c2->part =0;
-			goto scan;
-		}
-	}
-	return 1;
-scan:
-	/*
-	 * Collapse multiple unused elements together, and attempt
-	 * to extend the previous chunk into the freed chunk.
-	 *
-	 * We only extend non-unused elements which are marked
-	 * for newfs (we can't extend working filesystems), and
-	 * only if we are called with DELCHUNK_RECOVER.
-	 */
-	for (c2 = c1->part; c2; c2 = c2->next) {
-		if (c2->type != unused) {
-			if (c2->offset + c2->size != offset ||
-			    (rflags & DELCHUNK_RECOVER) == 0 ||
-			    (c2->flags & CHUNK_NEWFS) == 0) {
-				continue;
-			}
-			/* else extend into free area */
-		}
-		if (!c2->next)
-			continue;
-		if (c2->next->type != unused)
-			continue;
-		c3 = c2->next;
-		c2->size += c3->size;
-		c2->end = c3->end;
-		c2->next = c3->next;
-		c3->next = 0;
-		Free_Chunk(c3);
-		goto scan;
-	}
-	Fixup_Names(d);
-	return 0;
-}
-
-#if 0
-int
-Collapse_Chunk(struct disk *d, struct chunk *c1)
-{
-	struct chunk *c2, *c3;
-
-	if (c1->next && Collapse_Chunk(d, c1->next))
-		return 1;
-
-	if (c1->type == unused && c1->next && c1->next->type == unused) {
-		c3 = c1->next;
-		c1->size += c3->size;
-		c1->end = c3->end;
-		c1->next = c3->next;
-		c3->next = 0;
-		Free_Chunk(c3);
-		return 1;
-	}
-	c3 = c1->part;
-	if (!c3)
-		return 0;
-	if (Collapse_Chunk(d, c1->part))
-		return 1;
-
-	if (c1->type == whole)
-		return 0;
-
-	if (c3->type == unused && c3->size == c1->size) {
-		Delete_Chunk(d, c1);
-		return 1;
-	}
-	if (c3->type == unused) {
-		c2 = New_Chunk();
-		if (c2 == NULL)
-			barfout(1, "malloc failed");
-		*c2 = *c1;
-		c1->next = c2;
-		c1->disk = d;
-		c1->sname = strdup("-");
-		c1->name = strdup("-");
-		c1->part = 0;
-		c1->type = unused;
-		c1->flags = 0;
-		c1->subtype = 0;
-		c1->size = c3->size;
-		c1->end = c3->end;
-		c2->offset += c1->size;
-		c2->size -= c1->size;
-		c2->part = c3->next;
-		c3->next = 0;
-		Free_Chunk(c3);
-		return 1;
-	}
-	for (c2 = c3; c2->next; c2 = c2->next)
-		c3 = c2;
-	if (c2 && c2->type == unused) {
-		c3->next = 0;
-		c2->next = c1->next;
-		c1->next = c2;
-		c1->size -= c2->size;
-		c1->end -= c2->size;
-		return 1;
-	}
-
-	return 0;
-}
-#endif
diff --git a/lib/libdisk/create_chunk.c b/lib/libdisk/create_chunk.c
deleted file mode 100644
index 7f1453e..0000000
--- a/lib/libdisk/create_chunk.c
+++ /dev/null
@@ -1,296 +0,0 @@
-/*
- * ----------------------------------------------------------------------------
- * "THE BEER-WARE LICENSE" (Revision 42):
- * <phk@FreeBSD.org> wrote this file.  As long as you retain this notice you
- * can do whatever you want with this stuff. If we meet some day, and you think
- * this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
- * ----------------------------------------------------------------------------
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <string.h>
-#include <ctype.h>
-#include <fcntl.h>
-#include <stdarg.h>
-#include <sys/param.h>
-#include <sys/disklabel.h>
-#ifdef PC98
-#include <sys/diskpc98.h>
-#else
-#include <sys/diskmbr.h>
-#endif
-#include <sys/mount.h>
-#include <sys/stat.h>
-#include <sys/sysctl.h>
-#include <grp.h>
-#include <paths.h>
-#include <pwd.h>
-#include "libdisk.h"
-
-static int
-Fixup_FreeBSD_Names(struct chunk *c)
-{
-	struct chunk *c1, *c3;
-	uint j;
-    
-	if (!strcmp(c->name, "X"))
-		return 0;
-    
-	/* reset all names to "X" */
-	for (c1 = c->part; c1; c1 = c1->next) {
-		c1->oname = c1->name;
-		c1->name = malloc(12);
-		if (!c1->name)
-			return -1;
-		strcpy(c1->name,"X");
-	}
-    
-	/* Allocate the first swap-partition we find */
-	for (c1 = c->part; c1; c1 = c1->next) {
-		if (c1->type == unused)
-			continue;
-		if (c1->subtype != FS_SWAP)
-			continue;
-		sprintf(c1->name, "%s%c", c->name, SWAP_PART + 'a');
-		break;
-	}
-    
-	/* Allocate the first root-partition we find */
-	for (c1 = c->part; c1; c1 = c1->next) {
-		if (c1->type == unused)
-			continue;
-		if (!(c1->flags & CHUNK_IS_ROOT))
-			continue;
-		sprintf(c1->name, "%s%c", c->name, 0 + 'a');
-		break;
-	}
-    
-	/* Try to give them the same as they had before */
-	for (c1 = c->part; c1; c1 = c1->next) {
-		if (strcmp(c1->name, "X"))
-			continue;
-		for (c3 = c->part; c3 ; c3 = c3->next)
-			if (c1 != c3 && !strcmp(c3->name, c1->oname))
-				goto newname;
-		strcpy(c1->name, c1->oname);
-	newname:
-		;
-	}
-
-	/* Allocate the rest sequentially */
-	for (c1 = c->part; c1; c1 = c1->next) {
-		const char order[] = "defghab";
-
-		if (c1->type == unused)
-			continue;
-		if (strcmp("X", c1->name))
-			continue;
-	
-		for (j = 0; j < strlen(order); j++) {
-			sprintf(c1->name, "%s%c", c->name, order[j]);
-			for (c3 = c->part; c3 ; c3 = c3->next)
-				if (c1 != c3 && !strcmp(c3->name, c1->name))
-					goto match;
-			break;
-		match:
-			strcpy(c1->name, "X");
-			continue;
-		}
-	}
-	for (c1 = c->part; c1; c1 = c1->next) {
-		free(c1->oname);
-		c1->oname = 0;
-	}
-	return 0;
-}
-
-#ifndef PC98
-static int
-Fixup_Extended_Names(struct chunk *c)
-{
-	struct chunk *c1;
-	int j = 5;
-    
-	for (c1 = c->part; c1; c1 = c1->next) {
-		if (c1->type == unused)
-			continue;
-		free(c1->name);
-		c1->name = malloc(12);
-		if (!c1->name)
-			return -1;
-		sprintf(c1->name, "%ss%d", c->disk->chunks->name, j++);
-		if (c1->type == freebsd)
-			if (Fixup_FreeBSD_Names(c1) != 0)
-				return -1;
-	}
-	return 0;
-}
-#endif
-
-#ifdef __powerpc__
-static int
-Fixup_Apple_Names(struct chunk *c)
-{
-	struct chunk *c1;
-
-	for (c1 = c->part; c1; c1 = c1->next) {
-		if (c1->type == unused)
-			continue;
-		free(c1->name);
-		c1->name = strdup(c->name);
-		if (!c1->name)
-			return (-1);
-	}
-	return 0;
-}
-#endif
-
-int
-Fixup_Names(struct disk *d)
-{
-	struct chunk *c1, *c2;
-#if defined(__i386__) || defined(__ia64__) || defined(__amd64__)
-	struct chunk *c3;
-	int j, max;
-#endif
-
-	c1 = d->chunks;
-	for (c2 = c1->part; c2 ; c2 = c2->next) {
-		if (c2->type == unused)
-			continue;
-		if (strcmp(c2->name, "X"))
-			continue;
-#if defined(__i386__) || defined(__ia64__) || defined(__amd64__)
-		c2->oname = malloc(12);
-		if (!c2->oname)
-			return -1;
-#ifdef __ia64__
-		max = d->gpt_size;
-#else
-		max = NDOSPART;
-#endif
-		for (j = 1; j <= max; j++) {
-#ifdef __ia64__
-			sprintf(c2->oname, "%s%c%d", c1->name,
-			    (c1->type == whole) ? 'p' : 's', j);
-#else
-			sprintf(c2->oname, "%ss%d", c1->name, j);
-#endif
-			for (c3 = c1->part; c3; c3 = c3->next)
-				if (c3 != c2 && !strcmp(c3->name, c2->oname))
-					goto match;
-			free(c2->name);
-			c2->name = c2->oname;
-			c2->oname = 0;
-			break;
-		match:
-			continue;
-		}
-		if (c2->oname)
-			free(c2->oname);
-#else
-		free(c2->name);
-		c2->name = strdup(c1->name);
-#endif /*__i386__*/
-	}
-	for (c2 = c1->part; c2; c2 = c2->next) {
-		if (c2->type == freebsd)
-			Fixup_FreeBSD_Names(c2);
-#ifdef __powerpc__
-		else if (c2->type == apple)
-			Fixup_Apple_Names(c2);
-#endif
-#ifndef PC98
-		else if (c2->type == extended)
-			Fixup_Extended_Names(c2);
-#endif
-	}
-	return 0;
-}
-
-int
-Create_Chunk(struct disk *d, daddr_t offset, daddr_t size, chunk_e type,
-	     int subtype, u_long flags, const char *sname)
-{
-	int i;
-
-#ifndef __ia64__
-	if (!(flags & CHUNK_FORCE_ALL)) {
-		daddr_t l;
-#ifdef PC98
-		/* Never use the first cylinder */
-		if (!offset) {
-			offset += (d->bios_sect * d->bios_hd);
-			size -= (d->bios_sect * d->bios_hd);
-		}
-#else
-		/* Never use the first track */
-		if (!offset) {
-			offset += d->bios_sect;
-			size -= d->bios_sect;
-		}
-#endif /* PC98 */
-		
-		/* Always end on cylinder boundary */
-		l = (offset + size) % (d->bios_sect * d->bios_hd);
-		size -= l;
-	}
-#endif
-	
-	i = Add_Chunk(d, offset, size, "X", type, subtype, flags, sname);
-	Fixup_Names(d);
-	return i;
-}
-
-struct chunk *
-Create_Chunk_DWIM(struct disk *d, struct chunk *parent, daddr_t size,
-		  chunk_e type, int subtype, u_long flags)
-{
-	int i;
-	struct chunk *c1;
-	daddr_t offset;
-	
-	if (!parent)
-		parent = d->chunks;
-
-	if ((parent->type == freebsd  && type == part && parent->part == NULL) 
-	    || (parent->type == apple && type == part && parent->part == NULL)) {
-		c1 = New_Chunk();
-		if (c1 == NULL)
-			return (NULL);
-		c1->disk = parent->disk;
-		c1->offset = parent->offset;
-		c1->size = parent->size;
-		c1->end = parent->offset + parent->size - 1;
-		c1->type = unused;
-		if (parent->sname != NULL)
-			c1->sname = strdup(parent->sname);
-		c1->name = strdup("-");
-		parent->part = c1;
-	}
-		
-	for (c1 = parent->part; c1; c1 = c1->next) {
-		if (c1->type != unused)
-			continue;
-		if (c1->size < size)
-			continue;
-		offset = c1->offset;
-		goto found;
-	}
-	return 0;
-found:
-	i = Add_Chunk(d, offset, size, "X", type, subtype, flags, "-");
-	if (i)
-		return 0;
-	Fixup_Names(d);
-	for (c1 = parent->part; c1; c1 = c1->next)
-		if (c1->offset == offset)
-			return c1;
-	/* barfout(1, "Serious internal trouble"); */
-	return 0;
-}
diff --git a/lib/libdisk/disk.c b/lib/libdisk/disk.c
deleted file mode 100644
index c00e0e7..0000000
--- a/lib/libdisk/disk.c
+++ /dev/null
@@ -1,411 +0,0 @@
-/*
- * ----------------------------------------------------------------------------
- * "THE BEER-WARE LICENSE" (Revision 42):
- * <phk@FreeBSD.org> wrote this file.  As long as you retain this notice you
- * can do whatever you want with this stuff. If we meet some day, and you think
- * this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
- * ----------------------------------------------------------------------------
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <string.h>
-#include <inttypes.h>
-#include <err.h>
-#include <sys/sysctl.h>
-#include <sys/stdint.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/ioctl.h>
-#include <sys/disklabel.h>
-#include <sys/uuid.h>
-#include <sys/gpt.h>
-#include <paths.h>
-#include "libdisk.h"
-
-#include <ctype.h>
-#include <errno.h>
-#include <assert.h>
-#include <uuid.h>
-
-const enum platform platform =
-#if defined (P_DEBUG)
-	P_DEBUG
-#elif defined (PC98)
-	p_pc98
-#elif defined(__i386__)
-	p_i386
-#elif defined(__alpha__)
-	p_alpha
-#elif defined(__sparc64__)
-	p_sparc64
-#elif defined(__ia64__)
-	p_ia64
-#elif defined(__ppc__)
-	p_ppc
-#elif defined(__amd64__)
-	p_amd64
-#elif defined(__arm__)
-	p_arm
-#elif defined(__mips__)
-	p_mips
-#else
-	IHAVENOIDEA
-#endif
-	;
-
-const char *
-chunk_name(chunk_e type)
-{
-	switch(type) {
-	case unused:	return ("unused");
-	case mbr:	return ("mbr");
-	case part:	return ("part");
-	case gpt:	return ("gpt");
-	case pc98:	return ("pc98");
-	case sun:	return ("sun");
-	case freebsd:	return ("freebsd");
-	case fat:	return ("fat");
-	case spare:	return ("spare");
-	case efi:	return ("efi");
-	case apple:     return ("apple");
-	default:	return ("??");
-	}
-}
-
-struct disk *
-Open_Disk(const char *name)
-{
-	struct disk *d;
-	char *conftxt;
-	size_t txtsize;
-	int error;
-
-	error = sysctlbyname("kern.geom.conftxt", NULL, &txtsize, NULL, 0);
-	if (error) {
-		warn("kern.geom.conftxt sysctl not available, giving up!");
-		return (NULL);
-	}
-	conftxt = malloc(txtsize+1);
-	if (conftxt == NULL) {
-		warn("cannot malloc memory for conftxt");
-		return (NULL);
-	}
-	error = sysctlbyname("kern.geom.conftxt", conftxt, &txtsize, NULL, 0);
-	if (error) {
-		warn("error reading kern.geom.conftxt from the system");
-		free(conftxt);
-		return (NULL);
-	}
-	conftxt[txtsize] = '\0';	/* in case kernel bug is still there */
-	d = Int_Open_Disk(name, conftxt);
-	free(conftxt);
-
-	return (d);
-}
-
-void
-Debug_Disk(struct disk *d)
-{
-
-	printf("Debug_Disk(%s)", d->name);
-
-#ifndef __ia64__
-	printf("  bios_geom=%lu/%lu/%lu = %lu\n",
-		d->bios_cyl, d->bios_hd, d->bios_sect,
-		d->bios_cyl * d->bios_hd * d->bios_sect);
-#if defined(PC98)
-	printf("  boot1=%p, boot2=%p, bootipl=%p, bootmenu=%p\n",
-		d->boot1, d->boot2, d->bootipl, d->bootmenu);
-#elif defined(__i386__) || defined(__amd64__)
-	printf("  boot1=%p, boot2=%p, bootmgr=%p\n",
-		d->boot1, d->boot2, d->bootmgr);
-#elif defined(__alpha__)
-	printf("  boot1=%p, bootmgr=%p\n",
-		d->boot1, d->bootmgr);
-#else
-/* Should be: error "Debug_Disk: unknown arch"; */
-#endif
-#else	/* __ia64__ */
-	printf("  media size=%lu, sector size=%lu\n", d->media_size,
-	    d->sector_size);
-#endif
-
-	Debug_Chunk(d->chunks);
-}
-
-void
-Free_Disk(struct disk *d)
-{
-	if (d->chunks)
-		Free_Chunk(d->chunks);
-	if (d->name)
-		free(d->name);
-#ifdef PC98
-	if (d->bootipl)
-		free(d->bootipl);
-	if (d->bootmenu)
-		free(d->bootmenu);
-#else
-#if !defined(__ia64__)
-	if (d->bootmgr)
-		free(d->bootmgr);
-#endif
-#endif
-#if !defined(__ia64__)
-	if (d->boot1)
-		free(d->boot1);
-#endif
-#if defined(__i386__) || defined(__amd64__)
-	if (d->boot2)
-		free(d->boot2);
-#endif
-	free(d);
-}
-
-#if 0
-void
-Collapse_Disk(struct disk *d)
-{
-
-	while (Collapse_Chunk(d, d->chunks))
-		;
-}
-#endif
-
-static int
-qstrcmp(const void* a, const void* b)
-{
-	const char *str1 = *(char* const*)a;
-	const char *str2 = *(char* const*)b;
-
-	return strcmp(str1, str2);
-}
-
-char **
-Disk_Names()
-{
-	int disk_cnt;
-	char **disks;
-	int error;
-	size_t listsize;
-	char *disklist, *disk1, *disk2;
-
-	error = sysctlbyname("kern.disks", NULL, &listsize, NULL, 0);
-	if (error) {
-		warn("kern.disks sysctl not available");
-		return NULL;
-	}
-
-	if (listsize == 0)
-		return (NULL);
-
-	disks = malloc(sizeof *disks * (1 + MAX_NO_DISKS));
-	if (disks == NULL)
-		return NULL;
-	disk1 = disklist = (char *)malloc(listsize + 1);
-	if (disklist == NULL) {
-		free(disks);
-		return NULL;
-	}
-	memset(disks,0,sizeof *disks * (1 + MAX_NO_DISKS));
-	memset(disklist, 0, listsize + 1);
-	error = sysctlbyname("kern.disks", disklist, &listsize, NULL, 0);
-	if (error || disklist[0] == 0) {
-		free(disklist);
-		free(disks);
-		return NULL;
-	}
-	for (disk_cnt = 0; disk_cnt < MAX_NO_DISKS; disk_cnt++) {
-		disk2 = strsep(&disk1, " ");
-		if (disk2 == NULL)
-			break;
-		disks[disk_cnt] = strdup(disk2);
-		if (disks[disk_cnt] == NULL) {
-			for (disk_cnt--; disk_cnt >= 0; disk_cnt--)
-				free(disks[disk_cnt]);
-			free(disklist);
-			free(disks);
-			return (NULL);
-		}
-	}
-	qsort(disks, disk_cnt, sizeof(char*), qstrcmp);
-	free(disklist);
-	return disks;
-}
-
-#ifdef PC98
-void
-Set_Boot_Mgr(struct disk *d, const u_char *bootipl, const size_t bootipl_size,
-	const u_char *bootmenu, const size_t bootmenu_size)
-#else
-void
-Set_Boot_Mgr(struct disk *d, const u_char *b, const size_t s)
-#endif
-{
-#if !defined(__ia64__)
-#ifdef PC98
-	if (d->sector_size == 0)
-		return;
-	if (bootipl_size % d->sector_size != 0)
-		return;
-	if (d->bootipl)
-		free(d->bootipl);
-	if (!bootipl) {
-		d->bootipl = NULL;
-	} else {
-		d->bootipl_size = bootipl_size;
-		d->bootipl = malloc(bootipl_size);
-		if (!d->bootipl)
-			return;
-		memcpy(d->bootipl, bootipl, bootipl_size);
-	}
-
-	if (bootmenu_size % d->sector_size != 0)
-		return;
-	if (d->bootmenu)
-		free(d->bootmenu);
-	if (!bootmenu) {
-		d->bootmenu = NULL;
-	} else {
-		d->bootmenu_size = bootmenu_size;
-		d->bootmenu = malloc(bootmenu_size);
-		if (!d->bootmenu)
-			return;
-		memcpy(d->bootmenu, bootmenu, bootmenu_size);
-	}
-#else
-	if (d->sector_size == 0)
-		return;
-	if (s % d->sector_size != 0)
-		return;
-	if (d->bootmgr)
-		free(d->bootmgr);
-	if (!b) {
-		d->bootmgr = NULL;
-	} else {
-		d->bootmgr_size = s;
-		d->bootmgr = malloc(s);
-		if (!d->bootmgr)
-			return;
-		memcpy(d->bootmgr, b, s);
-	}
-#endif
-#endif
-}
-
-int
-Set_Boot_Blocks(struct disk *d, const u_char *b1, const u_char *b2)
-{
-#if defined(__i386__) || defined(__amd64__)
-	if (d->boot1)
-		free(d->boot1);
-	d->boot1 = malloc(512);
-	if (!d->boot1)
-		return -1;
-	memcpy(d->boot1, b1, 512);
-	if (d->boot2)
-		free(d->boot2);
-	d->boot2 = malloc(15 * 512);
-	if (!d->boot2)
-		return -1;
-	memcpy(d->boot2, b2, 15 * 512);
-#elif defined(__alpha__)
-	if (d->boot1)
-		free(d->boot1);
-	d->boot1 = malloc(15 * 512);
-	if (!d->boot1)
-		return -1;
-	memcpy(d->boot1, b1, 15 * 512);
-#elif defined(__sparc64__)
-	if (d->boot1 != NULL)
-		free(d->boot1);
-	d->boot1 = malloc(16 * 512);
-	if (d->boot1 == NULL)
-		return (-1);
-	memcpy(d->boot1, b1, 16 * 512);
-#elif defined(__ia64__)
-	/* nothing */
-#else
-/* Should be: #error "Set_Boot_Blocks: unknown arch"; */
-#endif
-	return 0;
-}
-
-const char *
-slice_type_name( int type, int subtype )
-{
-
-	switch (type) {
-	case whole:
-		return "whole";
-	case mbr:
-		switch (subtype) {
-		case 1:		return "fat (12-bit)";
-		case 2:		return "XENIX /";
-		case 3:		return "XENIX /usr";
-		case 4:         return "fat (16-bit,<=32Mb)";
-		case 5:		return "extended DOS";
-		case 6:         return "fat (16-bit,>32Mb)";
-		case 7:         return "NTFS/HPFS/QNX";
-		case 8:         return "AIX bootable";
-		case 9:         return "AIX data";
-		case 10:	return "OS/2 bootmgr";
-		case 11:        return "fat (32-bit)";
-		case 12:        return "fat (32-bit,LBA)";
-		case 14:        return "fat (16-bit,>32Mb,LBA)";
-		case 15:        return "extended DOS, LBA";
-		case 18:        return "Compaq Diagnostic";
-		case 57:	return "Plan 9";
-		case 77:	return "QNX 4.X";
-		case 78:	return "QNX 4.X 2nd part";
-		case 79:	return "QNX 4.X 3rd part";
-		case 84:	return "OnTrack diskmgr";
-		case 100:	return "Netware 2.x";
-		case 101:	return "Netware 3.x";
-		case 115:	return "SCO UnixWare";
-		case 128:	return "Minix 1.1";
-		case 129:	return "Minix 1.5";
-		case 130:	return "linux_swap";
-		case 131:	return "ext2fs";
-		case 133:	return "linux extended";
-		case 166:	return "OpenBSD FFS";	/* 0xA6 */
-		case 168:	return "Mac OS-X";
-		case 169:	return "NetBSD FFS";	/* 0xA9 */
-		case 171:	return "Mac OS-X Boot";
-		case 182:	return "OpenBSD";	/* dedicated */
-		case 183:	return "bsd/os";
-		case 184:	return "bsd/os swap";
-		case 191:	return "Solaris (new)";
-		case 238:	return "EFI GPT";
-		case 239:	return "EFI Sys. Part.";
-		default:	return "unknown";
-		}
-	case fat:
-		return "fat";
-	case freebsd:
-		switch (subtype) {
-#ifdef PC98
-		case 0xc494:	return "freebsd";
-#else
-		case 165:	return "freebsd";
-#endif
-		default:	return "unknown";
-		}
-	case extended:
-		return "extended";
-	case part:
-		return "part";
-	case efi:
-		return "efi";
-	case unused:
-		return "unused";
-	default:
-		return "unknown";
-	}
-}
diff --git a/lib/libdisk/libdisk.3 b/lib/libdisk/libdisk.3
deleted file mode 100644
index 1b436e2..0000000
--- a/lib/libdisk/libdisk.3
+++ /dev/null
@@ -1,341 +0,0 @@
-.\"
-.\" Copyright (c) 1996 Joerg Wunsch
-.\"
-.\" All rights reserved.
-.\"
-.\" This program is free software.
-.\"
-.\" Redistribution and use in source and binary forms, with or without
-.\" modification, are permitted provided that the following conditions
-.\" are met:
-.\" 1. Redistributions of source code must retain the above copyright
-.\"    notice, this list of conditions and the following disclaimer.
-.\" 2. Redistributions in binary form must reproduce the above copyright
-.\"    notice, this list of conditions and the following disclaimer in the
-.\"    documentation and/or other materials provided with the distribution.
-.\"
-.\" THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY EXPRESS OR
-.\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
-.\" OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
-.\" IN NO EVENT SHALL THE DEVELOPERS BE LIABLE FOR ANY DIRECT, INDIRECT,
-.\" INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
-.\" NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-.\" DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-.\" THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-.\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
-.\" THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-.\"
-.\" $FreeBSD$
-.\" "
-.Dd January 30, 2006
-.Dt LIBDISK 3
-.Os
-.Sh NAME
-.Nm Open_Disk ,
-.Nm Free_Disk ,
-.Nm Debug_Disk ,
-.Nm Set_Bios_Geom ,
-.Nm Delete_Chunk ,
-.Nm Collapse_Disk ,
-.Nm Collapse_Chunk ,
-.Nm Create_Chunk ,
-.Nm All_FreeBSD ,
-.Nm CheckRules ,
-.Nm Disk_Names ,
-.Nm Set_Boot_Mgr ,
-.Nm Set_Boot_Blocks ,
-.Nm Write_Disk ,
-.Nm Cyl_Aligned ,
-.Nm Next_Cyl_Aligned ,
-.Nm Prev_Cyl_Aligned ,
-.Nm Track_Aligned ,
-.Nm Next_Track_Aligned ,
-.Nm Prev_Track_Aligned ,
-.Nm Create_Chunk_DWIM ,
-.Nm MakeDev ,
-.Nm MakeDevDisk ,
-.Nm ShowChunkFlags ,
-.Nm chunk_name ,
-.Nm slice_type_name
-.Nd library interface to slice and partition labels
-.Sh LIBRARY
-.Lb libdisk
-.Sh SYNOPSIS
-.In sys/types.h
-.In libdisk.h
-.Pp
-.Ft struct disk *
-.Fn Open_Disk "const char *devname"
-.Ft void
-.Fn Free_Disk "struct disk *disk"
-.Ft void
-.Fn Debug_Disk "struct disk *disk"
-.Ft void
-.Fn Set_Bios_Geom "struct disk *disk" "u_long cyl" "u_long heads" "u_long sects"
-.Ft int
-.Fn Delete_Chunk "struct disk *disk" "struct chunk *"
-.Ft void
-.Fn Collapse_Disk "struct disk *disk"
-.Ft int
-.Fn Collapse_Chunk "struct disk *disk" "struct chunk *chunk"
-.Ft int
-.Fn Create_Chunk "struct disk *disk" "daddr_t offset" "daddr_t size" "chunk_e type" "int subtype" "u_long flags" "const char *sname"
-.Ft void
-.Fn All_FreeBSD "struct disk *d" "int force_all"
-.Ft char *
-.Fn CheckRules "const struct disk *"
-.Ft char **
-.Fn Disk_Names "void"
-.Ft void
-.Fn Set_Boot_Mgr "struct disk *d" "const u_char *bootmgr" "const size_t bootmgr_size"
-.Ft int
-.Fn Set_Boot_Blocks "struct disk *d" "const u_char *boot1" "const u_char *boot2"
-.Ft int
-.Fn Write_Disk "const struct disk *d"
-.Ft int
-.Fn Cyl_Aligned "struct disk *d" "daddr_t offset"
-.Ft daddr_t
-.Fn Next_Cyl_Aligned "const struct disk *d" "daddr_t offset"
-.Ft daddr_t
-.Fn Prev_Cyl_Aligned "const struct disk *d" "daddr_t offset"
-.Ft int
-.Fn Track_Aligned "const struct disk *d" "daddr_t offset"
-.Ft daddr_t
-.Fn Next_Track_Aligned "const struct disk *d" "daddr_t offset"
-.Ft daddr_t
-.Fn Prev_Track_Aligned "const struct disk *d" "daddr_t offset"
-.Ft struct chunk *
-.Fn Create_Chunk_DWIM "struct disk *d" "struct chunk *parent" "daddr_t size" "chunk_e type" "int subtype" "u_long flags"
-.Ft int
-.Fn MakeDev "struct chunk *c" "const char *path"
-.Ft int
-.Fn MakeDevDisk "struct disk *d" "const char *path"
-.Ft char *
-.Fn ShowChunkFlags "struct chunk *c"
-.Ft const char *
-.Fn chunk_name "chunk_e type"
-.Ft const char *
-.Fn slice_type_name "int type" "int subtype"
-.Sh DESCRIPTION
-.Bf Sy
-.Nm libdisk
-is just for the use of
-.Xr sysinstall 8 .
-You should consider using
-.Xr libgeom 3
-instead.
-.Ef
-.Pp
-The
-.Nm libdisk
-library provides an interface to the low-level disk slice and partition labels.
-Most functions operate with arguments of the types
-.Ql struct disk ,
-or
-.Ql struct chunk .
-.Pp
-While both types are mostly opaque to the programmer, the internal
-structure is mentioned below for the sake of completeness.
-.Bd -literal -offset indent
-struct disk {
-	char		*name;
-	u_long		flags;
-	u_long		bios_cyl;
-	u_long		bios_hd;
-	u_long		bios_sect;
-	u_char		*bootmgr;
-	u_char		*boot1;
-	u_char		*boot2;
-	struct chunk	*chunks;
-	u_long		sector_size;
-};
-.Ed
-The only flag value by now is
-.Ql DISK_ON_TRACK ,
-meaning that this disk is handled by the On-Track Disk Manager.
-.Bd -literal -offset indent
-struct chunk {
-	struct chunk	*next;
-	struct chunk	*part;
-	struct disk	*disk;
-	daddr_t		offset;
-	daddr_t		size;
-	daddr_t		end;
-	char		*name;
-	char		*oname;
-	chunk_e		type;
-	int		subtype;
-	u_long		flags;
-	void		(*private_free)(void*);
-	void		*(*private_clone)(void*);
-	void		*private_data;
-};
-.Ed
-The
-.Ql type
-field can be one of the following values:
-.Ql whole, unknown, fat, freebsd, extended, part, unused .
-.Pp
-These are the valid
-.Ql flags
-values for a
-.Ql struct chunk .
-.Bl -tag -offset indent -width CHUNK_AUTO_SIZEXX
-.It CHUNK_ALIGN
-This chunk should be aligned.
-.It CHUNK_IS_ROOT
-This
-.Ql part
-is a rootfs, allocate partition
-.Sq a .
-.It CHUNK_ACTIVE
-This is the active slice in the MBR.
-.It CHUNK_FORCE_ALL
-Force a dedicated disk for
-.Fx ,
-bypassing all BIOS geometry considerations.
-.It CHUNK_AUTO_SIZE
-This chunk was auto-sized and can fill out any deleted following chunks.
-.It CHUNK_NEWFS
-newfs pending, used to enable auto-resizing on delete (along with AUTO_SIZE).
-.El
-.Pp
-The
-.Ql private_data ,
-.Ql private_free ,
-and
-.Ql private_clone
-fields are for data private to the application, and the management
-thereof.
-If the functions are not provided, no storage management is
-done, cloning will just copy the pointer and freeing will just forget
-it.
-.Pp
-.Fn Open_Disk
-will open the named disk, and return populated tree.
-.Pp
-.Fn Free_Disk
-frees a tree made with
-.Fn Open_Disk
-or
-.Fn Clone_Disk .
-.Pp
-.Fn Debug_Disk
-prints the content of the tree to
-.Dv stdout .
-.Pp
-.Fn Set_Bios_Geom
-sets the geometry the bios uses.
-.Pp
-.Fn Delete_Chunk
-frees a chunk of disk_space.
-.Pp
-.Fn Collapse_Disk
-and
-.Fn Collapse_Chunk
-are experimental, do not use.
-.Pp
-.Fn Create_Chunk
-creates a chunk with the specified parameters.
-.Pp
-.Fn All_FreeBSD
-makes one
-.Fx
-chunk covering the entire disk; if
-.Ql force_all
-is set, bypass all BIOS geometry considerations.
-.Pp
-.Fn CheckRules
-returns
-.Ql char*
-to warnings about broken design rules in this disklayout.
-.Pp
-.Fn Disk_Names
-returns
-.Ql char**
-with all disk's names (wd0, wd1 ...).
-You must free each pointer, as
-well as the array by hand.
-.Pp
-.Fn Set_Boot_Mgr
-sets this boot-manager for use on this disk.
-Gets written when
-.Fn Write_Disk
-is called.
-.Pp
-.Fn Set_Boot_Blocks
-sets the boot-blocks for use on this disk.
-Gets written when
-.Fn Write_Disk
-is called.
-.Pp
-.Fn Write_Disk
-writes all the MBRs, disklabels, bootblocks and boot managers.
-.Pp
-.Fn Cyl_Aligned
-checks if
-.Ql offset
-is aligned on a cylinder according to the BIOS geometry.
-.Pp
-.Fn Next_Cyl_Aligned
-rounds
-.Ql offset
-up to next cylinder according to the BIOS geometry.
-.Pp
-.Fn Prev_Cyl_Aligned
-rounds
-.Ql offset
-down to previous cylinder according to the BIOS geometry.
-.Pp
-.Fn Track_Aligned
-checks if
-.Ql offset
-is aligned on a track according to the BIOS geometry.
-.Pp
-.Fn Next_Track_Aligned
-rounds
-.Ql offset
-up to next track according to the BIOS geometry.
-.Pp
-.Fn Prev_Track_Aligned
-rounds
-.Ql offset
-up to previous track according to the BIOS geometry.
-.Pp
-.Fn Create_Chunk_DWIM
-creates a partition inside the given parent of the given size, and
-returns a pointer to it.
-The first unused chunk big enough is used.
-.Pp
-.Fn MakeDev
-makes the device nodes for this chunk.
-.Pp
-.Fn MakeDevDisk
-makes the device nodes for all chunks on this disk.
-.Pp
-.Fn ShowChunkFlags
-returns a string to show flags.
-.Pp
-The
-.Fn chunk_name
-function takes the enumerated chunk type and returns its name.
-.Fn chunk_name
-replaces the old external array
-.Va chunk_n .
-.Pp
-.Fn slice_type_name
-returns the name strings associated with the specified
-.Ql type .
-.Ql subtype .
-If
-.Fn slice_type_name
-returns "unknown" for slices it is not familiar with.
-.Sh AUTHORS
-.An -nosplit
-The
-.Nm libdisk
-library was written by
-.An Poul-Henning Kamp .
-.Pp
-This manual page was written by
-.An J\(:org Wunsch .
diff --git a/lib/libdisk/libdisk.h b/lib/libdisk/libdisk.h
deleted file mode 100644
index a302c64..0000000
--- a/lib/libdisk/libdisk.h
+++ /dev/null
@@ -1,368 +0,0 @@
-/*
-* ----------------------------------------------------------------------------
-* "THE BEER-WARE LICENSE" (Revision 42):
-* <phk@FreeBSD.org> wrote this file.  As long as you retain this notice you
-* can do whatever you want with this stuff. If we meet some day, and you think
-* this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
-* ----------------------------------------------------------------------------
-*
-* $FreeBSD$
-*
-*/
-
-#ifndef _LIBDISK_H_
-#define _LIBDISK_H_
-
-/* #define DEBUG 1 */
-/* You can define a particular architecture here if you are debugging. */
-/* #define P_DEBUG p_sparc64 */
-
-#define MAX_NO_DISKS	32
-/* Max # of disks Disk_Names() will return */
-
-#define MAX_SEC_SIZE    2048  /* maximum sector size that is supported */
-#define MIN_SEC_SIZE	512   /* the sector size to end sensing at */
-
-enum platform {
-	p_any,			/* for debugging ! */
-	p_alpha,
-	p_i386,
-	p_pc98,
-	p_sparc64,
-	p_ia64,
-	p_ppc,
-	p_amd64,
-	p_arm,
-	p_mips
-};
-extern const enum platform platform;
-
-typedef enum {
-	whole,
-	unknown,
-
-	sun,
-	pc98,
-	mbr,
-	gpt,
-
-	efi,
-	fat,
-	freebsd,
-	extended,
-	part,
-	spare,
-	unused,
-
-	apple
-} chunk_e;
-
-__BEGIN_DECLS
-#ifndef __ia64__
-struct disk {
-	char		*name;
-	u_long		bios_cyl;
-	u_long		bios_hd;
-	u_long		bios_sect;
-#ifdef PC98
-	u_char		*bootipl;
-	size_t		bootipl_size;
-	u_char		*bootmenu;
-	size_t		bootmenu_size;
-#else
-	u_char		*bootmgr;
-	size_t		bootmgr_size;
-#endif
-	u_char		*boot1;
-#if defined(__i386__) || defined(__amd64__) /* the i386 needs extra help... */
-	u_char		*boot2;
-#endif
-	struct chunk	*chunks;
-	u_long		sector_size; /* media sector size, a power of 2 */
-};
-#else	/* !__ia64__ */
-struct disk {
-	char		*name;
-	struct chunk	*chunks;
-	u_long		media_size;
-	u_long		sector_size;
-	u_long		lba_start;
-	u_long		lba_end;
-	u_int		gpt_size;	/* Number of entries */
-};
-#endif
-
-struct chunk {
-	struct chunk	*next;
-	struct chunk	*part;
-	struct disk	*disk;
-	daddr_t		offset;
-	daddr_t		size;
-	daddr_t		end;
-	char		*sname;		/* PC98 field */
-	char		*name;
-	char		*oname;
-	/* Used during Fixup_Names() to avoid renaming more than
-	 * absolutely needed.
-	 */
-	chunk_e		type;
-	int		subtype;
-	u_long		flags;
-	void		(*private_free)(void*);
-	void		*(*private_clone)(void*);
-	void		*private_data;
-	/* For data private to the application, and the management
-	 * thereof.  If the functions are not provided, no storage
-	 * management is done, Cloning will just copy the pointer
-	 * and freeing will just forget it.
-	 */
-};
-
-/*
- * flags:
- *
- * ALIGN	-	This chunk should be aligned
- * IS_ROOT	-	This 'part' is a rootfs, allocate 'a'
- * ACTIVE	-	This is the active slice in the MBR
- * FORCE_ALL	-	Force a dedicated disk for FreeBSD, bypassing
- *			all BIOS geometry considerations
- * AUTO_SIZE	-	This chunk was auto-sized and can fill-out a
- *			following chunk if the following chunk is deleted.
- * NEWFS	-	newfs pending, used to enable auto-resizing on
- *			delete (along with AUTO_SIZE).
- */
-
-#define CHUNK_ALIGN		0x0008
-#define CHUNK_IS_ROOT		0x0010
-#define CHUNK_ACTIVE		0x0020
-#define CHUNK_FORCE_ALL		0x0040	
-#define CHUNK_AUTO_SIZE		0x0080
-#define CHUNK_NEWFS		0x0100
-#define	CHUNK_HAS_INDEX		0x0200
-#define	CHUNK_ITOF(i)		((i & 0xFFFF) << 16)
-#define	CHUNK_FTOI(f)		((f >> 16) & 0xFFFF)
-
-#define DELCHUNK_NORMAL		0x0000
-#define DELCHUNK_RECOVER	0x0001
-
-const char *chunk_name(chunk_e);
-
-const char *
-slice_type_name(int, int);
-/* "chunk_n" for subtypes too */
-
-struct disk *
-Open_Disk(const char *);
-/* Will open the named disk, and return populated tree.  */
-
-void
-Free_Disk(struct disk *);
-/* Free a tree made with Open_Disk() or Clone_Disk() */
-
-void
-Debug_Disk(struct disk *);
-/* Print the content of the tree to stdout */
-
-void
-Set_Bios_Geom(struct disk *, u_long, u_long, u_long);
-/* Set the geometry the bios uses. */
-
-void
-Sanitize_Bios_Geom(struct disk *);
-/* Set the bios geometry to something sane */
-
-int
-Insert_Chunk(struct chunk *, daddr_t, daddr_t, const char *, chunk_e, int,
-	u_long, const char *);
-
-int
-Delete_Chunk2(struct disk *, struct chunk *, int);
-/* Free a chunk of disk_space modified by the passed flags. */
-
-int
-Delete_Chunk(struct disk *, struct chunk *);
-/* Free a chunk of disk_space */
-
-void
-Collapse_Disk(struct disk *);
-/* Experimental, do not use. */
-
-int
-Collapse_Chunk(struct disk *, struct chunk *);
-/* Experimental, do not use. */
-
-int
-Create_Chunk(struct disk *, daddr_t, daddr_t, chunk_e, int, u_long, const char *);
-/* Create a chunk with the specified paramters */
-
-void
-All_FreeBSD(struct disk *, int);
-/*
- * Make one FreeBSD chunk covering the entire disk;
- * if force_all is set, bypass all BIOS geometry
- * considerations.
- */
-
-char *
-CheckRules(const struct disk *);
-/* Return char* to warnings about broken design rules in this disklayout */
-
-char **
-Disk_Names(void);
-/*
- * Return char** with all disk's names (wd0, wd1 ...).  You must free
- * each pointer, as well as the array by hand
- */
-
-#ifdef PC98
-void
-Set_Boot_Mgr(struct disk *, const u_char *, const size_t, const u_char *,
-	const size_t);
-#else
-void
-Set_Boot_Mgr(struct disk *, const u_char *, const size_t);
-#endif
-/*
- * Use this boot-manager on this disk.  Gets written when Write_Disk()
- * is called
- */
-
-int
-Set_Boot_Blocks(struct disk *, const u_char *, const u_char *);
-/*
- * Use these boot-blocks on this disk.  Gets written when Write_Disk()
- * is called. Returns nonzero upon failure.
- */
-
-int
-Write_Disk(const struct disk *);
-/* Write all the MBRs, disklabels, bootblocks and boot managers */
-
-daddr_t
-Next_Cyl_Aligned(const struct disk *, daddr_t);
-/* Round offset up to next cylinder according to the bios-geometry */
-
-daddr_t
-Prev_Cyl_Aligned(const struct disk *, daddr_t);
-/* Round offset down to previous cylinder according to the bios-geometry */
-
-int
-Track_Aligned(const struct disk *, daddr_t);
-/* Check if offset is aligned on a track according to the bios geometry */
-
-daddr_t
-Next_Track_Aligned(const struct disk *, daddr_t);
-/* Round offset up to next track according to the bios-geometry */
-
-daddr_t
-Prev_Track_Aligned(const struct disk *, daddr_t);
-/* Check if offset is aligned on a track according to the bios geometry */
-
-struct chunk *
-Create_Chunk_DWIM(struct disk *, struct chunk *, daddr_t, chunk_e, int,
-	u_long);
-/*
- * This one creates a partition inside the given parent of the given
- * size, and returns a pointer to it.  The first unused chunk big
- * enough is used.
- */
-
-char *
-ShowChunkFlags(struct chunk *);
-/* Return string to show flags. */
-
-/*
- * Implementation details  >>> DO NOT USE <<<
- */
-
-struct disklabel;
-
-void Fill_Disklabel(struct disklabel *, const struct disk *,
-	const struct chunk *);
-void Debug_Chunk(struct chunk *);
-struct chunk *New_Chunk(void);
-void Free_Chunk(struct chunk *);
-struct chunk *Clone_Chunk(const struct chunk *);
-int Add_Chunk(struct disk *, daddr_t, daddr_t, const char *, chunk_e, int,
-	u_long, const char *);
-void *read_block(int, daddr_t, u_long);
-int write_block(int, daddr_t, const void *, u_long);
-struct disklabel *read_disklabel(int, daddr_t, u_long);
-struct disk *Int_Open_Disk(const char *, char *);
-int Fixup_Names(struct disk *);
-int MakeDevChunk(const struct chunk *, const char *);
-__END_DECLS
-
-/* TODO
- *
- * Need an error string mechanism from the functions instead of warn()
- *
- * Make sure only FreeBSD start at offset==0
- *
- * Collapse must align.
- *
- * Make Write_Disk(struct disk*)
- *
- * Consider booting from OnTrack'ed disks.
- *
- * Get Bios-geom, ST506 & OnTrack from driver (or otherwise)
- *
- * Make Create_DWIM().
- *
- * Make Is_Unchanged(struct disk *d1, struct chunk *c1)
- *
- * don't rename slices unless we have to
- *
- *Sample output from tst01:
- *
- * Debug_Disk(wd0)  flags=0  bios_geom=0/0/0
- * >>        0x3d040          0    1411200    1411199 wd0      0 whole    0 0
- * >>>>      0x3d080          0     960120     960119 wd0s1    3 freebsd  0 8
- * >>>>>>    0x3d100          0      40960      40959 wd0s1a   5 part     0 0
- * >>>>>>    0x3d180      40960     131072     172031 wd0s1b   5 part     0 0
- * >>>>>>    0x3d1c0     172032     409600     581631 wd0s1e   5 part     0 0
- * >>>>>>    0x3d200     581632     378488     960119 wd0s1f   5 part     0 0
- * >>>>      0x3d140     960120       5670     965789 wd0s2    4 extended 0 8
- * >>>>>>    0x3d2c0     960120         63     960182 -        6 unused   0 0
- * >>>>>>    0x3d0c0     960183       5607     965789 wd0s5    2 fat      0 8
- * >>>>      0x3d280     965790       1890     967679 wd0s3    1 foo      -2 8
- * >>>>      0x3d300     967680     443520    1411199 wd0s4    3 freebsd  0 8
- * >>>>>>    0x3d340     967680     443520    1411199 wd0s4a   5 part     0 0
- *
- * ^            ^           ^          ^          ^     ^      ^ ^        ^ ^
- * level    chunkptr      start      size        end  name    type  subtype flags
- *
- * Underlying data structure:
- *
- *	Legend:
- *		<struct chunk> --> part
- *			|
- *			v next
- *
- *	<wd0> --> <wd0s1> --> <wd0s1a>
- *		     |           |
- *		     |           v
- *		     |        <wd0s1b>
- *		     |           |
- *		     |           v
- *		     |        <wd0s1e>
- *		     |           |
- *		     |           v
- *		     |        <wd0s1f>
- *		     |
- *		     v
- *		  <wd0s2> --> <unused>
- *		     |           |
- *		     |           v
- *		     |        <wd0s5>
- *		     |
- *		     v
- *		  <wd0s3>
- *		     |
- *		     v
- *		  <wd0s4> --> <wd0s4a>
- *
- *
- */
-
-#endif /* _LIBDISK_H_ */
diff --git a/lib/libdisk/open_disk.c b/lib/libdisk/open_disk.c
deleted file mode 100644
index 6cc9acb..0000000
--- a/lib/libdisk/open_disk.c
+++ /dev/null
@@ -1,311 +0,0 @@
-/*
- * ----------------------------------------------------------------------------
- * "THE BEER-WARE LICENSE" (Revision 42):
- * <phk@FreeBSD.org> wrote this file.  As long as you retain this notice you
- * can do whatever you want with this stuff. If we meet some day, and you think
- * this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
- * ----------------------------------------------------------------------------
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <string.h>
-#include <inttypes.h>
-#include <err.h>
-#include <sys/sysctl.h>
-#include <sys/stdint.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/ioctl.h>
-#include <sys/disklabel.h>
-#include <sys/gpt.h>
-#include <paths.h>
-#include "libdisk.h"
-
-#include <ctype.h>
-#include <errno.h>
-#include <assert.h>
-
-#ifdef DEBUG
-#define	DPRINT(x)	warn x
-#define	DPRINTX(x)	warnx x
-#else
-#define	DPRINT(x)
-#define	DPRINTX(x)
-#endif
-
-struct disk *
-Int_Open_Disk(const char *name, char *conftxt)
-{
-	struct disk *d;
-	int i, line = 1;
-	char *p, *q, *r, *a, *b, *n, *t, *sn;
-	daddr_t o, len, off;
-	u_int l, s, ty, sc, hd, alt;
-	daddr_t lo[10];
-
-	/*
-	 * Locate the disk (by name) in our sysctl output
-	 */
-	for (p = conftxt; p != NULL && *p; p = strchr(p, '\n'), line++) {
-		if (*p == '\n')
-			p++;
-		a = strsep(&p, " ");
-		/* Skip anything not with index 0 */
-		if (strcmp(a, "0"))
-			continue;
-
-		/* Skip anything not a disk */
-		a = strsep(&p, " ");
-		if (strcmp(a, "DISK"))
-			continue;
-
-		a = strsep(&p, " ");
-		if (strcmp(a, name))
-			continue;
-		break;
-	}
-
-	q = strchr(p, '\n');
-	if (q != NULL)
-		*q++ = '\0';
-
-	d = (struct disk *)calloc(sizeof *d, 1);
-	if(d == NULL)
-		return NULL;
-
-	d->name = strdup(name);
-
-	a = strsep(&p, " ");	/* length in bytes */
-	len = strtoimax(a, &r, 0);
-	if (*r) {
-		printf("libdisk: Int_Open_Disk(%s): can't parse length in line %d (r='%s')\n",
-			name, line, r);
-		return NULL;
-	}
-
-	a = strsep(&p, " ");	/* sectorsize */
-	s = strtoul(a, &r, 0);
-	if (*r) {
-		printf("libdisk: Int_Open_Disk(%s): can't parse sector size in line %d (r='%s')\n",
-			name, line, r);
-		return NULL;
-	}
-
-	if (s == 0)
-		return (NULL);
-	d->sector_size = s;
-	len /= s;	/* media size in number of sectors. */
-
-	if (Add_Chunk(d, 0, len, name, whole, 0, 0, "-")) {
-		DPRINT(("Failed to add 'whole' chunk"));
-	}
-
-	/* Try to parse any fields after the sector size in the DISK entry line */
-	for (;;) {
-		a = strsep(&p, " ");
-		if (a == NULL)
-			break;
-		b = strsep(&p, " ");
-		o = strtoimax(b, &r, 0);
-		if (*r) {
-			printf("libdisk: Int_Open_Disk(%s): can't parse parameter '%s' in line %d (r='%s')\n",
-				name, a, line, r);
-			return NULL;
-		}
-		if (!strcmp(a, "hd"))
-			d->bios_hd = o;
-		else if (!strcmp(a, "sc"))
-			d->bios_sect = o;
-		else
-			printf("libdisk: Int_Open_Disk(%s): unknown parameter '%s' with value '%s' in line %d, ignored\n",
-				name, a, b, line);
-	}
-
-	/* Sanitize the parameters. */
-	Sanitize_Bios_Geom(d);
-
-	/*
-	 * Calculate the number of cylinders this disk must have. If we have
-	 * an obvious insanity, we set the number of cylinders to zero.
-	 */
-	o = d->bios_hd * d->bios_sect;
-	d->bios_cyl = (o != 0) ? len / o : 0;
-
-	p = q; line++; /* p is now the start of the line _after_ the DISK entry */
-	lo[0] = 0;
-
-	for (; p != NULL && *p; p = q, line++) {
-		sn = NULL;
-		q = strchr(p, '\n');
-		if (q != NULL)
-			*q++ = '\0';
-		a = strsep(&p, " ");	/* Index */
-		/*
-		 * If we find index 0 again, this means we've encountered another disk, so it's safe to assume this disk
-		 * has been processed.
-		 */
-		if (!strcmp(a, "0"))
-			break;
-		l = strtoimax(a, &r, 0);
-		if (*r) {
-			printf("libdisk: Int_Open_Disk(%s): can't parse depth '%s' in line %d (r='%s')\n",
-				name, a, line, r);
-			return NULL;
-
-		}
-		t = strsep(&p, " ");	/* Type {SUN, BSD, MBR, PC98, GPT} */
-		n = strsep(&p, " ");	/* name */
-		a = strsep(&p, " ");	/* len */
-		len = strtoimax(a, &r, 0);
-		if (*r) {
-			printf("libdisk: Int_Open_Disk(%s): can't parse length '%s' in line %d (r='%s')\n",
-				name, a, line, r);
-			continue;
-		}
-		a = strsep(&p, " ");	/* secsize */
-		s = strtoimax(a, &r, 0);
-		if (*r) {
-			printf("libdisk: Int_Open_Disk(%s): can't parse sector size '%s' in line %d (r='%s')\n",
-				name, a, line, r);
-			continue;
-		}
-		for (;;) {
-			a = strsep(&p, " ");
-			if (a == NULL)
-				break;
-			/* XXX: Slice name may include a space. */
-			if (!strcmp(a, "sn")) {
-				sn = p;
-				break;
-			}
-			b = strsep(&p, " ");
-			o = strtoimax(b, &r, 0);
-			/* APPLE have ty as a string */
-			if ((*r) && strcmp(t, "APPLE") &&
-			    strcmp(t, "GPT") && strcmp(t, "PART")) {
-				printf("libdisk: Int_Open_Disk(%s): can't parse parameter '%s' in line %d (r='%s')\n",
-					name, a, line, r);
-				break;
-			}
-			if (!strcmp(a, "o"))
-				off = o;
-			else if (!strcmp(a, "i"))
-				i = (!strcmp(t, "PART")) ? o - 1 : o;
-			else if (!strcmp(a, "ty"))
-				ty = o;
-			else if (!strcmp(a, "sc"))
-				sc = o;
-			else if (!strcmp(a, "hd"))
-				hd = o;
-			else if (!strcmp(a, "alt"))
-				alt = o;
-			else if (!strcmp(a, "xs"))
-				t = b;
-			else if (!strcmp(a, "xt")) {
-				if (*r)
-					sn = b;
-				else
-					ty = o;
-			}
-		}
-
-		/* PLATFORM POLICY BEGIN ----------------------------------- */
-		if (platform == p_sparc64 && !strcmp(t, "SUN") && i == 2)
-			continue;
-		if (platform == p_sparc64 && !strcmp(t, "SUN") &&
-		    d->chunks->part->part == NULL) {
-			d->bios_hd = hd;
-			d->bios_sect = sc;
-			o = d->chunks->size / (hd * sc);
-			o *= (hd * sc);
-			o -= alt * hd * sc;
-			if (Add_Chunk(d, 0, o, name, freebsd, 0, 0, "-")) {
-				DPRINT(("Failed to add 'freebsd' chunk"));
-			}
-		}
-		if (platform == p_alpha && !strcmp(t, "BSD") &&
-		    d->chunks->part->part == NULL) {
-			if (Add_Chunk(d, 0, d->chunks->size, name, freebsd,
-				      0, 0, "-")) {
-				DPRINT(("Failed to add 'freebsd' chunk"));
-			}
-		}
-		if (!strcmp(t, "BSD") && i == RAW_PART)
-			continue;
-		/* PLATFORM POLICY END ------------------------------------- */
-
-		off /= s;
-		len /= s;
-		off += lo[l - 1];
-		lo[l] = off;
-		if (!strcmp(t, "SUN"))
-			i = Add_Chunk(d, off, len, n, part, 0, 0, 0);
-		else if (!strncmp(t, "MBR", 3)) {
-			switch (ty) {
-			case 0xa5:
-				i = Add_Chunk(d, off, len, n, freebsd, ty, 0, 0);
-				break;
-			case 0x01:
-			case 0x04:
-			case 0x06:
-			case 0x0b:
-			case 0x0c:
-			case 0x0e:
-				i = Add_Chunk(d, off, len, n, fat, ty, 0, 0);
-				break;
-			case 0xef:	/* EFI */
-				i = Add_Chunk(d, off, len, n, efi, ty, 0, 0);
-				break;
-			default:
-				i = Add_Chunk(d, off, len, n, mbr, ty, 0, 0);
-				break;
-			}
-		} else if (!strcmp(t, "BSD"))
-			i = Add_Chunk(d, off, len, n, part, ty, 0, 0);
-		else if (!strcmp(t, "PC98")) {
-			switch (ty & 0x7f) {
-			case 0x14:
-				i = Add_Chunk(d, off, len, n, freebsd, ty, 0,
-					      sn);
-				break;
-			case 0x20:
-			case 0x21:
-			case 0x22:
-			case 0x23:
-			case 0x24:
-				i = Add_Chunk(d, off, len, n, fat, ty, 0, sn);
-				break;
-			default:
-				i = Add_Chunk(d, off, len, n, pc98, ty, 0, sn);
-				break;
-			}
-		} else if (!strcmp(t, "GPT"))
-			i = Add_Chunk(d, off, len, n, gpt, 0, 0, b);
-		else if (!strcmp(t, "APPLE"))
-			i = Add_Chunk(d, off, len, n, apple, 0, 0, sn);
-		else
-			; /* Ignore unknown classes. */
-	}
-	/* PLATFORM POLICY BEGIN ------------------------------------- */
-	/* We have a chance to do things on a blank disk here */
-	if (platform == p_sparc64 && d->chunks->part->part == NULL) {
-		hd = d->bios_hd;
-		sc = d->bios_sect;
-		o = d->chunks->size / (hd * sc);
-		o *= (hd * sc);
-		o -= 2 * hd * sc;
-		if (Add_Chunk(d, 0, o, name, freebsd, 0, 0, "-")) {
-			DPRINT(("Failed to add 'freebsd' chunk"));
-		}
-	}
-	/* PLATFORM POLICY END --------------------------------------- */
-
-	return (d);
-	i = 0;
-}
diff --git a/lib/libdisk/open_ia64_disk.c b/lib/libdisk/open_ia64_disk.c
deleted file mode 100644
index 287757d..0000000
--- a/lib/libdisk/open_ia64_disk.c
+++ /dev/null
@@ -1,239 +0,0 @@
-/*
- * Copyright (c) 2003 Marcel Moolenaar
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
- * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
- * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/types.h>
-#include <sys/disklabel.h>
-#include <sys/diskmbr.h>
-#include <sys/gpt.h>
-#include <sys/uuid.h>
-
-#include <fcntl.h>
-#include <inttypes.h>
-#include <paths.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-
-#include "libdisk.h"
-
-static struct disk *
-parse_disk(char *conftxt, const char *name)
-{
-	char devname[64];
-	struct disk *disk;
-	struct dos_partition *part;
-	struct gpt_hdr *gpt;
-	char *buffer, *p, *q;
-	int fd, i;
-
-	disk = (struct disk *)calloc(sizeof *disk, 1);
-	if (disk == NULL)
-		return (NULL);
-
-	disk->name = strdup(name);
-	p = strsep(&conftxt, " ");			/* media size */
-	disk->media_size = strtoimax(p, &q, 0);
-	if (*q)
-		goto fail;
-
-	p = strsep(&conftxt, " ");			/* sector size */
-	disk->sector_size = strtoul(p, &q, 0);
-	if (*q)
-		goto fail;
-
-	if (disk->sector_size == 0)
-		disk->sector_size = 512;
-
-	if (disk->media_size % disk->sector_size)
-		goto fail;
-
-	/*
-	 * We need to read the disk to get GPT specific information.
-	 */
-
-	snprintf(devname, sizeof(devname), "%s%s", _PATH_DEV, name);
-	fd = open(devname, O_RDONLY);
-	if (fd == -1)
-		goto fail;
-	buffer = malloc(2 * disk->sector_size);
-	if (buffer == NULL) {
-		close (fd);
-		goto fail;
-	}
-	if (read(fd, buffer, 2 * disk->sector_size) == -1) {
-		free(buffer);
-		close(fd);
-		goto fail;
-	}
-	close(fd);
-
-	gpt = (struct gpt_hdr *)(buffer + disk->sector_size);
-	if (memcmp(gpt->hdr_sig, GPT_HDR_SIG, sizeof(gpt->hdr_sig))) {
-		/*
-		 * No GPT present. Check if the MBR is empty (if present)
-		 * or is a PMBR before declaring this disk as empty. If
-		 * the MBR isn't empty, bail out. Let's not risk nuking a
-		 * disk.
-		 */
-		if (*(u_short *)(buffer + DOSMAGICOFFSET) == DOSMAGIC) {
-			for (i = 0; i < 4; i++) {
-				part = (struct dos_partition *)
-				    (buffer + DOSPARTOFF + i * DOSPARTSIZE);
-				if (part->dp_typ != 0 &&
-				    part->dp_typ != DOSPTYP_PMBR)
-					break;
-			}
-			if (i < 4) {
-				free(buffer);
-				goto fail;
-			}
-		}
-		disk->gpt_size = 128;
-		disk->lba_start = (disk->gpt_size * sizeof(struct gpt_ent)) /
-		    disk->sector_size + 2;
-		disk->lba_end = (disk->media_size / disk->sector_size) -
-		    disk->lba_start;
-	} else {
-		disk->lba_start = gpt->hdr_lba_start;
-		disk->lba_end = gpt->hdr_lba_end;
-		disk->gpt_size = gpt->hdr_entries;
-	}
-	free(buffer);
-	Add_Chunk(disk, disk->lba_start, disk->lba_end - disk->lba_start + 1,
-	    name, whole, 0, 0, "-");
-	return (disk);
-
-fail:
-	free(disk->name);
-	free(disk);
-	return (NULL);
-}
-
-struct disk *
-Int_Open_Disk(const char *name, char *conftxt)
-{
-	struct chunk chunk;
-	struct disk *disk;
-	char *p, *q, *r, *s, *sd;
-	u_long i;
-
-	p = conftxt;
-	while (p != NULL && *p != 0) {
-		q = strsep(&p, " ");
-		if (strcmp(q, "0") == 0) {
-			q = strsep(&p, " ");
-			if (strcmp(q, "DISK") == 0) {
-				q = strsep(&p, " ");
-				if (strcmp(q, name) == 0)
-					break;
-			}
-		}
-		p = strchr(p, '\n');
-		if (p != NULL && *p == '\n')
-			p++;
-		conftxt = p;
-	}
-	if (p == NULL || *p == 0)
-		return (NULL);
-
-	conftxt = strchr(p, '\n');
-	if (conftxt != NULL)
-		*conftxt++ = '\0';
-
-	disk = parse_disk(p, name);
-	if (disk == NULL)
-		return (NULL);
-
-	while (conftxt != NULL && *conftxt != 0) {
-		p = conftxt;
-		conftxt = strchr(p, '\n');
-		if (conftxt != NULL)
-			*conftxt++ = '\0';
-
-		/*
-		 * 1 PART da0p4 34359738368 512
-		 *	i 4 o 52063912960 ty freebsd-ufs
-		 *	xs GPT xt 516e7cb6-6ecf-11d6-8ff8-00022d09712b
-		 */
-		sd = strsep(&p, " ");			/* depth */
-		if (strcmp(sd, "0") == 0)
-			break;
-
-		q = strsep(&p, " ");			/* type */
-		if (strcmp(q, "PART") != 0)
-			continue;
-
-		chunk.name = strsep(&p, " ");		/* name */
-
-		q = strsep(&p, " ");			/* length */
-		i = strtoimax(q, &r, 0);
-		if (*r)
-			abort();
-		chunk.end = i / disk->sector_size;
-
-		q = strsep(&p, " ");			/* sector size */
-
-		for (;;) {
-			q = strsep(&p, " ");
-			if (q == NULL)
-				break;
-			r = strsep(&p, " ");
-			i = strtoimax(r, &s, 0);
-			if (strcmp(q, "ty") == 0 && *s != '\0') {
-				if (!strcmp(r, "efi"))
-					chunk.type = efi;
-				else if (!strcmp(r, "freebsd")) {
-					chunk.type = freebsd;
-					chunk.subtype = 0xa5;
-				} else if (!strcmp(r, "freebsd-swap")) {
-					chunk.type = part;
-					chunk.subtype = FS_SWAP;
-				} else if (!strcmp(r, "freebsd-ufs")) {
-					chunk.type = part;
-					chunk.subtype = FS_BSDFFS;
-				} else {
-					chunk.type = part;
-					chunk.subtype = FS_OTHER;
-				}
-			} else {
-				if (!strcmp(q, "o"))
-					chunk.offset = i / disk->sector_size;
-				else if (!strcmp(q, "i"))
-					chunk.flags = CHUNK_ITOF(i) |
-					    CHUNK_HAS_INDEX;
-			}
-		}
-
-		Add_Chunk(disk, chunk.offset, chunk.end, chunk.name,
-		    chunk.type, chunk.subtype, chunk.flags, 0);
-	}
-
-	return (disk);
-}
diff --git a/lib/libdisk/rules.c b/lib/libdisk/rules.c
deleted file mode 100644
index 1f12f4c..0000000
--- a/lib/libdisk/rules.c
+++ /dev/null
@@ -1,296 +0,0 @@
-/*
- * ----------------------------------------------------------------------------
- * "THE BEER-WARE LICENSE" (Revision 42):
- * <phk@FreeBSD.org> wrote this file.  As long as you retain this notice you
- * can do whatever you want with this stuff. If we meet some day, and you think
- * this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
- * ----------------------------------------------------------------------------
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/types.h>
-#include <sys/stdint.h>
-#include <sys/disklabel.h>
-#ifdef PC98
-#include <sys/diskpc98.h>
-#else
-#include <sys/diskmbr.h>
-#endif
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <string.h>
-#include "libdisk.h"
-
-int
-Track_Aligned(const struct disk *d, daddr_t offset)
-{
-#ifndef __ia64__
-	if (!d->bios_sect)
-		return 1;
-	if (offset % d->bios_sect)
-		return 0;
-#endif /* __ia64__ */
-	return 1;
-}
-
-daddr_t
-Prev_Track_Aligned(const struct disk *d, daddr_t offset)
-{
-#ifndef __ia64__
-	if (!d->bios_sect)
-		return offset;
-	return (offset / d->bios_sect) * d->bios_sect;
-#else
-	return 1;
-#endif
-}
-
-daddr_t
-Next_Track_Aligned(const struct disk *d, daddr_t offset)
-{
-#ifndef __ia64__
-	if (!d->bios_sect)
-		return offset;
-	return Prev_Track_Aligned(d, offset + d->bios_sect-1);
-#else
-	return 1;
-#endif
-}
-
-static int
-Cyl_Aligned(const struct disk *d, daddr_t offset)
-{
-#ifndef __ia64__
-	if (!d->bios_sect || !d->bios_hd)
-		return 1;
-	if (offset % (d->bios_sect * d->bios_hd))
-		return 0;
-#endif
-	return 1;
-}
-
-daddr_t
-Prev_Cyl_Aligned(const struct disk *d, daddr_t offset)
-{
-#ifndef __ia64__
-	if (!d->bios_sect || !d->bios_hd)
-		return offset;
-	return (offset / (d->bios_sect * d->bios_hd)) * d->bios_sect *
-	    d->bios_hd;
-#else
-	return 1;
-#endif
-}
-
-daddr_t
-Next_Cyl_Aligned(const struct disk *d, daddr_t offset)
-{
-#ifndef __ia64__
-	if (!d->bios_sect || !d->bios_hd)
-		return offset;
-	return Prev_Cyl_Aligned(d,offset + (d->bios_sect * d->bios_hd) - 1);
-#else
-	return 1;
-#endif
-}
-
-/*
- *  Rule#0:
- *	Chunks of type 'whole' can have max NDOSPART children.
- *	Only one of them can have the "active" flag
- */
-static void
-Rule_000(__unused const struct disk *d, const struct chunk *c, char *msg)
-{
-#ifdef PC98
-	int i = 0;
-#else
-	int i = 0, j = 0;
-#endif
-	struct chunk *c1;
-
-	if (c->type != whole)
-		return;
-	for (c1 = c->part; c1; c1 = c1->next) {
-		if (c1->type != unused)
-			continue;
-#ifndef PC98
-		if (c1->flags & CHUNK_ACTIVE)
-			j++;
-#endif
-		i++;
-	}
-	if (i > NDOSPART)
-		sprintf(msg + strlen(msg),
-			"%d is too many children of the 'whole' chunk."
-			"  Max is %d\n", i, NDOSPART);
-#ifndef PC98
-	if (j > 1)
-		sprintf(msg + strlen(msg),
-			"Too many active children of 'whole'");
-#endif
-}
-
-/*
- * Rule#1:
- *	All children of 'whole' and 'extended'  must be track-aligned.
- *	Exception: the end can be unaligned if it matches the end of 'whole'
- */
-static void
-Rule_001(const struct disk *d, const struct chunk *c, char *msg)
-{
-	struct chunk *c1;
-
-	if (c->type != whole && c->type != extended)
-		return;
-	for (c1 = c->part; c1; c1 = c1->next) {
-		if (c1->type == unused)
-			continue;
-		c1->flags |= CHUNK_ALIGN;
-#ifdef PC98
-		if (!Cyl_Aligned(d, c1->offset))
-#else
-		if (!Track_Aligned(d, c1->offset))
-#endif
-			sprintf(msg + strlen(msg),
-#ifdef PC98
-				"chunk '%s' [%jd..%jd] does not start"
-				" on a cylinder boundary\n",
-#else
-				"chunk '%s' [%jd..%jd] does not start"
-				" on a track boundary\n",
-#endif
-			    c1->name, (intmax_t)c1->offset, (intmax_t)c1->end);
-		if ((c->type == whole || c->end == c1->end)
-		    || Cyl_Aligned(d, c1->end + 1))
-			;
-		else
-			sprintf(msg + strlen(msg),
-				"chunk '%s' [%jd..%jd] does not end"
-				" on a cylinder boundary\n",
-			    c1->name, (intmax_t)c1->offset, (intmax_t)c1->end);
-	}
-}
-
-/*
- * Rule#2:
- *	Max one 'fat' as child of 'whole'
- */
-static void
-Rule_002(__unused const struct disk *d, const struct chunk *c, char *msg)
-{
-	int i;
-	struct chunk *c1;
-
-	if (c->type != whole)
-		return;
-	for (i = 0, c1 = c->part; c1; c1 = c1->next) {
-		if (c1->type != fat)
-			continue;
-		i++;
-	}
-	if (i > 1) {
-		sprintf(msg + strlen(msg),
-			"Max one 'fat' allowed as child of 'whole'\n");
-	}
-}
-
-/*
- * Rule#3:
- *	Max one extended as child of 'whole'
- */
-static void
-Rule_003(__unused const struct disk *d, const struct chunk *c, char *msg)
-{
-	int i;
-	struct chunk *c1;
-
-	if (c->type != whole)
-		return;
-	for (i = 0, c1 = c->part; c1; c1 = c1->next) {
-		if (c1->type != extended)
-			continue;
-		i++;
-	}
-	if (i > 1) {
-		sprintf(msg + strlen(msg),
-			"Max one 'extended' allowed as child of 'whole'\n");
-	}
-}
-
-/*
- * Rule#4:
- *	Max seven 'part' as children of 'freebsd'
- *	Max one CHUNK_IS_ROOT child per 'freebsd'
- */
-static void
-Rule_004(__unused const struct disk *d, const struct chunk *c, char *msg)
-{
-	int i = 0, k = 0;
-	struct chunk *c1;
-
-	if (c->type != freebsd)
-		return;
-
-	for (c1 = c->part; c1; c1 = c1->next) {
-		if (c1->type != part)
-			continue;
-		if (c1->flags & CHUNK_IS_ROOT)
-			k++;
-		i++;
-	}
-	if (i > 7) {
-		sprintf(msg + strlen(msg),
-			"Max seven partitions per freebsd slice\n");
-	}
-	if (k > 1) {
-		sprintf(msg + strlen(msg),
-			"Max one root partition child per freebsd slice\n");
-	}
-}
-
-static void
-Check_Chunk(const struct disk *d, const struct chunk *c, char *msg)
-{
-
-	switch (platform) {
-	case p_i386:
-	case p_amd64:
-		Rule_000(d, c, msg);
-		Rule_001(d, c, msg);
-		Rule_002(d, c, msg);
-		Rule_003(d, c, msg);
-		Rule_004(d, c, msg);
-		if (c->part)
-			Check_Chunk(d, c->part, msg);
-		if (c->next)
-			Check_Chunk(d, c->next, msg);
-		break;
-	case p_pc98:
-		Rule_000(d, c, msg);
-		Rule_001(d, c, msg);
-		Rule_004(d, c, msg);
-		if (c->part)
-			Check_Chunk(d, c->part, msg);
-		if (c->next)
-			Check_Chunk(d, c->next, msg);
-		break;
-	default:
-		break;
-	}
-}
-
-char *
-CheckRules(const struct disk *d)
-{
-	char msg[BUFSIZ];
-
-	*msg = '\0';
-	Check_Chunk(d, d->chunks, msg);
-	if (*msg)
-		return strdup(msg);
-	return 0;
-}
diff --git a/lib/libdisk/tst01.c b/lib/libdisk/tst01.c
deleted file mode 100644
index 7bb7579..0000000
--- a/lib/libdisk/tst01.c
+++ /dev/null
@@ -1,323 +0,0 @@
-/*
- * ----------------------------------------------------------------------------
- * "THE BEER-WARE LICENSE" (Revision 42):
- * <phk@FreeBSD.org> wrote this file.  As long as you retain this notice you
- * can do whatever you want with this stuff. If we meet some day, and you think
- * this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
- * ----------------------------------------------------------------------------
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <string.h>
-#include <err.h>
-#ifdef READLINE
-#include <readline/readline.h>
-#include <readline/history.h>
-#endif
-#include <sys/types.h>
-#include <paths.h>
-#include "libdisk.h"
-
-#ifndef PC98
-u_char mbrboot[] = {
-250,51,192,142,208,188,0,124,139,244,80,7,80,31,251,252,191,0,6,185,0,1,
-242,165,234,29,6,0,0,190,190,7,179,4,128,60,128,116,14,128,60,0,117,28,
-131,198,16,254,203,117,239,205,24,139,20,139,76,2,139,238,131,198,16,254,
-203,116,26,128,60,0,116,244,190,139,6,172,60,0,116,11,86,187,7,0,180,14,
-205,16,94,235,240,235,254,191,5,0,187,0,124,184,1,2,87,205,19,95,115,12,
-51,192,205,19,79,117,237,190,163,6,235,211,190,194,6,191,254,125,129,61,
-85,170,117,199,139,245,234,0,124,0,0,73,110,118,97,108,105,100,32,112,97,
-114,116,105,116,105,111,110,32,116,97,98,108,101,0,69,114,114,111,114,32,
-108,111,97,100,105,110,103,32,111,112,101,114,97,116,105,110,103,32,115,
-121,115,116,101,109,0,77,105,115,115,105,110,103,32,111,112,101,114,97,
-116,105,110,103,32,115,121,115,116,101,109,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,128,
-1,1,0,4,15,63,60,63,0,0,0,241,239,0,0,0,0,1,61,5,15,63,243,48,240,0,0,144,
-208,2,0,0,0,1,244,165,15,63,170,192,192,3,0,144,208,2,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,85,170
-};
-
-u_char bteasy17[] = {
-51,192,142,192,142,216,142,208,188,0,124,252,139,244,191,0,6,185,0,1,242,
-165,234,96,6,0,0,139,213,88,162,72,7,60,53,116,28,180,16,246,228,5,174,
-4,150,246,68,4,255,116,62,198,4,128,232,218,0,138,116,1,139,76,2,235,8,
-232,207,0,185,1,0,50,209,187,0,124,184,1,2,205,19,114,30,129,191,254,1,
-85,170,117,22,234,0,124,0,0,128,250,129,116,2,178,128,139,234,66,128,242,
-179,136,22,58,7,191,190,7,185,4,0,198,6,45,7,49,50,246,136,45,138,69,4,
-60,0,116,35,60,5,116,31,254,198,190,42,7,232,113,0,190,72,7,70,70,139,28,
-10,255,116,5,50,125,4,117,243,141,183,114,7,232,90,0,131,199,16,254,6,45,
-7,226,203,128,62,117,4,2,116,11,190,59,7,10,246,117,10,205,24,235,172,190,
-42,7,232,57,0,232,54,0,50,228,205,26,139,218,131,195,96,180,1,205,22,180,
-0,117,11,205,26,59,211,114,242,160,72,7,235,10,205,22,138,196,60,28,116,
-243,4,246,60,49,114,214,60,53,119,210,80,190,40,7,187,27,6,83,252,172,80,
-36,127,180,14,205,16,88,168,128,116,242,195,86,184,1,3,187,0,6,185,1,0,
-50,246,205,19,94,198,6,72,7,63,195,13,138,13,10,70,48,32,46,32,46,32,46,
-160,100,105,115,107,32,49,13,10,10,68,101,102,97,117,108,116,58,32,70,63,
-160,0,1,0,4,0,6,3,7,7,10,10,99,14,100,14,101,20,128,20,129,25,130,30,147,
-36,165,39,159,43,117,47,82,47,219,50,64,55,242,61,0,100,111,243,72,80,70,
-211,79,115,178,85,110,105,248,78,111,118,101,108,236,77,105,110,105,248,
-76,105,110,117,248,65,109,111,101,98,225,66,83,196,66,83,68,233,80,67,73,
-216,67,80,205,86,101,110,105,248,68,111,115,115,101,227,63,191,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
-0,85,170
-};
-#endif
-
-int
-scan_block(int fd, daddr_t block)
-{
-	u_char foo[512];
-
-	if (-1 == lseek(fd, (off_t)block * 512, SEEK_SET))
-		err(1, "lseek");
-	if (512 != read(fd, foo, 512))
-		return 1;
-	return 0;
-}
-
-void
-Scan_Disk(struct disk *d)
-{
-	char device[64];
-	u_long l;
-	int i, j, fd;
-
-        strcpy(device, _PATH_DEV);
-        strcat(device, d->name);
-
-        fd = open(device, O_RDWR);
-        if (fd < 0) {
-                warn("open(%s) failed", device);
-                return;
-        }
-	for (i = -1, l = 0; ; l++) {
-		j = scan_block(fd, l);
-		if (j != i) {
-			if (i == -1) {
-				printf("%c: %lu.",j ? 'B' : 'G', l);
-				fflush(stdout);
-			} else if (i == 0) {
-				printf(".%lu\nB: %lu.", l - 1, l);
-				fflush(stdout);
-			} else {
-				printf(".%lu\nG: %lu.", l - 1, l);
-				fflush(stdout);
-			}
-			i = j;
-		}
-	}
-	close(fd);
-}
-
-int
-main(int argc, char **argv)
-{
-	struct disk *d,*db;
-	char myprompt[BUFSIZ];
-#ifndef READLINE
-	char input[BUFSIZ];
-#endif
-	char *p,*q = 0;
-	char **cp, *cmds[200];
-	int ncmd, i;
-
-	if (argc < 2) {
-		fprintf(stderr, "Usage:\n\t%s diskname\n", argv[0]);
-		exit(1);
-	}
-	d = Open_Disk(argv[1]);
-	if (!d)
-		err(1, "Couldn't open disk %s", argv[1]);
-
-	sprintf(myprompt, "%s %s> ", argv[0], argv[1]);
-	while (1) {
-		printf("--==##==--\n");
-		p = CheckRules(d);
-		Debug_Disk(d);
-		if (p) {
-			printf("%s",p);
-			free(p);
-		}
-#ifdef READLINE
-		if (q)
-			free(q);
-		q = p = readline(myprompt);
-#else
-		printf("%s", myprompt);
-		fflush(stdout);
-		q = p = fgets(input, sizeof(input), stdin);
-#endif
-		if(!p)
-			break;
-		for (cp = cmds; (*cp = strsep(&p, " \t\n")) != NULL;)
-			if (**cp != '\0')
-				cp++;
-		ncmd = cp - cmds;
-		if (!ncmd)
-			continue;
-		if (!strcasecmp(*cmds, "quit"))
-			break;
-		if (!strcasecmp(*cmds, "exit"))
-			break;
-		if (!strcasecmp(*cmds, "q"))
-			break;
-		if (!strcasecmp(*cmds, "x"))
-			break;
-		if (!strcasecmp(*cmds, "dwim") && ncmd == 6) {
-			printf("dwim = %p\n",
-			       Create_Chunk_DWIM(d,
-					 (struct chunk *)strtol(cmds[1], 0, 0),
-					 strtol(cmds[2], 0, 0),
-					 strtol(cmds[3], 0, 0),
-					 strtol(cmds[4], 0, 0),
-					 strtol(cmds[5], 0, 0)));
-			continue;
-		}
-		if (!strcasecmp(*cmds, "delete") && ncmd == 2) {
-			printf("delete = %d\n",
-			       Delete_Chunk(d,
-				    (struct chunk *)strtol(cmds[1], 0, 0)));
-			continue;
-		}
-#ifndef __ia64__
-		if (!strcasecmp(*cmds, "allfreebsd")) {
-			All_FreeBSD(d, 0);
-			continue;
-		}
-		if (!strcasecmp(*cmds, "dedicate")) {
-			All_FreeBSD(d, 1);
-			continue;
-		}
-		if (!strcasecmp(*cmds, "sanitize")) {
-			Sanitize_Bios_Geom(d);
-			continue;
-		}
-		if (!strcasecmp(*cmds, "bios") && ncmd == 4) {
-			Set_Bios_Geom(d, strtol(cmds[1], 0, 0),
-				      strtol(cmds[2], 0, 0),
-				      strtol(cmds[3], 0, 0));
-			continue;
-		}
-#endif
-		if (!strcasecmp(*cmds, "list")) {
-			cp = Disk_Names();
-			printf("Disks:");
-			for (i = 0; cp[i]; i++) {
-				printf(" %s", cp[i]);
-				free(cp[i]);
-			}
-			free(cp);
-			continue;
-		}
-#ifdef PC98
-		if (!strcasecmp(*cmds, "create") && ncmd == 7) {
-#else
-		if (!strcasecmp(*cmds,"create") && ncmd == 6) {
-#endif
-
-			printf("Create=%d\n",
-			       Create_Chunk(d,
-					    strtol(cmds[1], 0, 0),
-					    strtol(cmds[2], 0, 0),
-					    strtol(cmds[3], 0, 0),
-					    strtol(cmds[4], 0, 0),
-#ifdef PC98
-					    strtol(cmds[5], 0, 0), cmds[6]));
-#else
-					    strtol(cmds[5], 0, 0), NULL));
-#endif
-			continue;
-		}
-		if (!strcasecmp(*cmds,"read")) {
-			db = d;
-			if (ncmd > 1)
-				d = Open_Disk(cmds[1]);
-			else
-				d = Open_Disk(argv[1]);
-			if (d)
-				Free_Disk(db);
-			else
-				d = db;
-			continue;
-		}
-		if (!strcasecmp(*cmds,"scan")) {
-			Scan_Disk(d);
-			continue;
-		}
-#ifndef PC98
-		if (!strcasecmp(*cmds,"bteasy")) {
-			Set_Boot_Mgr(d, bteasy17, sizeof (bteasy17));
-			continue;
-		}
-		if (!strcasecmp(*cmds, "mbr")) {
-			Set_Boot_Mgr(d, mbrboot, sizeof (mbrboot));
-			continue;
-		}
-#endif
-#if 0		/* XXX boot1 undefined, fix me */
-		if (!strcasecmp(*cmds, "boot")) {
-			Set_Boot_Blocks(d, boot1, boot2);
-			continue;
-		}
-#endif
-		if (!strcasecmp(*cmds, "write")) {
-			printf("Write=%d\n",
-				Write_Disk(d));
-			Free_Disk(d);
-			d = Open_Disk(argv[1]);
-			continue;
-		}
-		if (strcasecmp(*cmds, "help"))
-			printf("\007ERROR\n");
-		printf("CMDS:\n");
-		printf("\tallfreebsd\n");
-		printf("\tdedicate\n");
-		printf("\tbios cyl hd sect\n");
-		printf("\tboot\n");
-#ifndef PC98
-		printf("\tbteasy17\n");
-#endif
-#if 0
-		printf("\tcollapse [pointer]\n");
-#endif
-#ifdef PC98
-		printf("\tcreate offset size enum subtype flags name\n");
-#else
-		printf("\tcreate offset size enum subtype flags\n");
-#endif
-		printf("\t\tsubtype(part): swap=1, ffs=7\n");
-		printf("\tdelete pointer\n");
-		printf("\tlist\n");
-#ifndef PC98
-		printf("\tmbr\n");
-#endif
-#if 0
-		printf("\tphys cyl hd sect\n");
-#endif
-		printf("\tquit\n");
-		printf("\tread [disk]\n");
-		printf("\tscan\n");
-		printf("\twrite\n");
-		printf("\nENUM:\n\t");
-#if 0
-		for (i = 0; chunk_n[i]; i++)
-			printf("%d = %s%s", i, chunk_n[i],
-			       i == 4 ? "\n\t" : "  ");
-#endif
-		printf("\n");
-
-	}
-	exit (0);
-}
diff --git a/lib/libdisk/write_amd64_disk.c b/lib/libdisk/write_amd64_disk.c
deleted file mode 100644
index 9899258..0000000
--- a/lib/libdisk/write_amd64_disk.c
+++ /dev/null
@@ -1,204 +0,0 @@
-/*
- * ----------------------------------------------------------------------------
- * "THE BEER-WARE LICENSE" (Revision 42):
- * <phk@FreeBSD.org> wrote this file.  As long as you retain this notice you
- * can do whatever you want with this stuff. If we meet some day, and you think
- * this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
- * ----------------------------------------------------------------------------
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <string.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/disklabel.h>
-#include <sys/diskmbr.h>
-#include <paths.h>
-#include "libdisk.h"
-
-/*
- * XXX: A lot of hardcoded 512s probably should be foo->sector_size;
- *	I'm not sure which, so I leave it like it worked before. --schweikh
- */
-static int
-Write_FreeBSD(int fd, const struct disk *new, const struct chunk *c1)
-{
-	struct disklabel *dl;
-	int i;
-	void *p;
-	u_char buf[BBSIZE];
-
-	for (i = 0; i < BBSIZE/512; i++) {
-		if (!(p = read_block(fd, i + c1->offset, 512)))
-			return (1);
-		memcpy(buf + 512 * i, p, 512);
-		free(p);
-	}
-	if (new->boot1)
-		memcpy(buf, new->boot1, 512);
-
-	if (new->boot2)
-		memcpy(buf + 512, new->boot2, BBSIZE - 512);
-
-	dl = (struct disklabel *)(buf + 512 * LABELSECTOR + LABELOFFSET);
-	Fill_Disklabel(dl, new, c1);
-
-	for (i = 0; i < BBSIZE / 512; i++)
-		write_block(fd, i + c1->offset, buf + 512 * i, 512);
-
-	return 0;
-}
-
-static void
-Write_Int32(u_int32_t *p, u_int32_t v)
-{
-	u_int8_t *bp = (u_int8_t *)p;
-
-	bp[0] = (v >> 0) & 0xff;
-	bp[1] = (v >> 8) & 0xff;
-	bp[2] = (v >> 16) & 0xff;
-	bp[3] = (v >> 24) & 0xff;
-}
-
-/*
- * Special install-time configuration for the i386 boot0 boot manager.
- */
-static void
-Cfg_Boot_Mgr(u_char *mbr, int edd)
-{
-
-	if (mbr[0x1b0] == 0x66 && mbr[0x1b1] == 0xbb) {
-		if (edd)
-			mbr[0x1bb] |= 0x80;	/* Packet mode on */
-		else
-			mbr[0x1bb] &= 0x7f;	/* Packet mode off */
-	}
-}
-
-int
-Write_Disk(const struct disk *d1)
-{
-	int fd, j;
-	uint i;
-	struct chunk *c1;
-	int ret = 0;
-	char device[64];
-	u_char *mbr;
-	struct dos_partition *dp,work[NDOSPART];
-	int s[4];
-	int need_edd = 0;	/* Need EDD (packet interface) */
-
-	strcpy(device, _PATH_DEV);
-        strcat(device, d1->name);
-
-        fd = open(device, O_RDWR);
-        if (fd < 0)
-                return 1;
-
-	memset(s, 0, sizeof s);
-	if (!(mbr = read_block(fd, 0, d1->sector_size))) {
-		close (fd);
-		return (1);
-	}
-	dp = (struct dos_partition *)(mbr + DOSPARTOFF);
-	memcpy(work, dp, sizeof work);
-	dp = work;
-	free(mbr);
-	for (c1 = d1->chunks->part; c1; c1 = c1->next) {
-		if (c1->type == unused)
-			continue;
-		if (!strcmp(c1->name, "X"))
-			continue;
-		j = c1->name[strlen(d1->name) + 1] - '1';
-		if (j < 0 || j > 3)
-			continue;
-		s[j]++;
-		if (c1->type == freebsd)
-			ret += Write_FreeBSD(fd, d1, c1);
-
-		Write_Int32(&dp[j].dp_start, c1->offset);
-		Write_Int32(&dp[j].dp_size, c1->size);
-
-		i = c1->offset;
-		if (i >= 1024 * d1->bios_sect * d1->bios_hd) {
-			dp[j].dp_ssect = 0xff;
-			dp[j].dp_shd = 0xff;
-			dp[j].dp_scyl = 0xff;
-			need_edd++;
-		} else {
-			dp[j].dp_ssect = i % d1->bios_sect;
-			i -= dp[j].dp_ssect++;
-			i /= d1->bios_sect;
-			dp[j].dp_shd =  i % d1->bios_hd;
-			i -= dp[j].dp_shd;
-			i /= d1->bios_hd;
-			dp[j].dp_scyl = i;
-			i -= dp[j].dp_scyl;
-			dp[j].dp_ssect |= i >> 2;
-		}
-#ifdef DEBUG
-		printf("S:%lu = (%x/%x/%x)", c1->offset,
-		       dp[j].dp_scyl, dp[j].dp_shd, dp[j].dp_ssect);
-#endif
-
-		i = c1->end;
-		dp[j].dp_esect = i % d1->bios_sect;
-		i -= dp[j].dp_esect++;
-		i /= d1->bios_sect;
-		dp[j].dp_ehd =  i % d1->bios_hd;
-		i -= dp[j].dp_ehd;
-		i /= d1->bios_hd;
-		if (i > 1023)
-			i = 1023;
-		dp[j].dp_ecyl = i;
-		i -= dp[j].dp_ecyl;
-		dp[j].dp_esect |= i >> 2;
-#ifdef DEBUG
-		printf("  E:%lu = (%x/%x/%x)\n", c1->end,
-		       dp[j].dp_ecyl, dp[j].dp_ehd, dp[j].dp_esect);
-#endif
-
-		dp[j].dp_typ = c1->subtype;
-		if (c1->flags & CHUNK_ACTIVE)
-			dp[j].dp_flag = 0x80;
-		else
-			dp[j].dp_flag = 0;
-	}
-	j = 0;
-	for (i = 0; i < NDOSPART; i++) {
-		if (!s[i])
-			memset(dp + i, 0, sizeof *dp);
-		if (dp[i].dp_flag)
-			j++;
-	}
-	if (!j)
-		for(i = 0; i < NDOSPART; i++)
-			if (dp[i].dp_typ == 0xa5)
-				dp[i].dp_flag = 0x80;
-
-	if (!(mbr = read_block(fd, 0, d1->sector_size))) {
-		close (fd);
-		return (1);
-	}
-	if (d1->bootmgr) {
-		memcpy(mbr, d1->bootmgr, DOSPARTOFF);
-		Cfg_Boot_Mgr(mbr, need_edd);
-        }
-	memcpy(mbr + DOSPARTOFF, dp, sizeof *dp * NDOSPART);
-	mbr[512-2] = 0x55;
-	mbr[512-1] = 0xaa;
-	write_block(fd, 0, mbr, d1->sector_size);
-	if (d1->bootmgr && d1->bootmgr_size > d1->sector_size)
-		for (i = 1; i * d1->sector_size <= d1->bootmgr_size; i++)
-			write_block(fd, i, &d1->bootmgr[i * d1->sector_size],
-				    d1->sector_size);
-
-	close(fd);
-	return 0;
-}
diff --git a/lib/libdisk/write_arm_disk.c b/lib/libdisk/write_arm_disk.c
deleted file mode 100644
index 5dbb0ad..0000000
--- a/lib/libdisk/write_arm_disk.c
+++ /dev/null
@@ -1,47 +0,0 @@
-/*-
- * Copyright (c) 2006 Olivier Houchard
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <string.h>
-#include <err.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/ioctl.h>
-#include <sys/disklabel.h>
-#include <paths.h>
-#include "libdisk.h"
-
-int
-Write_Disk(const struct disk *d1)
-{
-	return (0);
-}
diff --git a/lib/libdisk/write_disk.c b/lib/libdisk/write_disk.c
deleted file mode 100644
index 024981b..0000000
--- a/lib/libdisk/write_disk.c
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * ----------------------------------------------------------------------------
- * "THE BEER-WARE LICENSE" (Revision 42):
- * <phk@FreeBSD.org> wrote this file.  As long as you retain this notice you
- * can do whatever you want with this stuff. If we meet some day, and you think
- * this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
- * ----------------------------------------------------------------------------
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <string.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/ioctl.h>
-#include <sys/disklabel.h>
-#include <paths.h>
-#include "libdisk.h"
-
-void
-Fill_Disklabel(struct disklabel *dl, const struct disk *new,
-    const struct chunk *c1)
-{
-	struct chunk *c2;
-	int j;
-
-	memset(dl, 0, sizeof *dl);
-
-	for (c2 = c1->part; c2; c2 = c2->next) {
-		if (c2->type == unused)
-			continue;
-		if (!strcmp(c2->name, "X"))
-			continue;
-		j = c2->name[strlen(c2->name) - 1] - 'a';
-		if (j < 0 || j >= MAXPARTITIONS || j == RAW_PART)
-			continue;
-		dl->d_partitions[j].p_size = c2->size;
-		dl->d_partitions[j].p_offset = c2->offset;
-		dl->d_partitions[j].p_fstype = c2->subtype;
-	}
-
-	dl->d_bbsize = BBSIZE;
-	/*
-	 * Add in defaults for superblock size, interleave, and rpms
-	 */
-	dl->d_sbsize = 0;
-
-	strcpy(dl->d_typename, c1->name);
-
-	dl->d_secsize = 512;
-	dl->d_secperunit = new->chunks->size;
-#ifndef __ia64__
-	dl->d_ncylinders = new->bios_cyl;
-	dl->d_ntracks = new->bios_hd;
-	dl->d_nsectors = new->bios_sect;
-#endif
-	dl->d_secpercyl = dl->d_ntracks * dl->d_nsectors;
-
-	dl->d_npartitions = MAXPARTITIONS;
-
-	dl->d_type = new->name[0] == 's' || new->name[0] == 'd' ||
-	    new->name[0] == 'o' ? DTYPE_SCSI : DTYPE_ESDI;
-	dl->d_partitions[RAW_PART].p_size = c1->size;
-	dl->d_partitions[RAW_PART].p_offset = c1->offset;
-	dl->d_rpm = 3600;
-	dl->d_interleave = 1;
-
-	dl->d_magic = DISKMAGIC;
-	dl->d_magic2 = DISKMAGIC;
-	dl->d_checksum = dkcksum(dl);
-}
diff --git a/lib/libdisk/write_i386_disk.c b/lib/libdisk/write_i386_disk.c
deleted file mode 100644
index 15317d3..0000000
--- a/lib/libdisk/write_i386_disk.c
+++ /dev/null
@@ -1,204 +0,0 @@
-/*
- * ----------------------------------------------------------------------------
- * "THE BEER-WARE LICENSE" (Revision 42):
- * <phk@FreeBSD.org> wrote this file.  As long as you retain this notice you
- * can do whatever you want with this stuff. If we meet some day, and you think
- * this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
- * ----------------------------------------------------------------------------
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <string.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/disklabel.h>
-#include <sys/diskmbr.h>
-#include <paths.h>
-#include "libdisk.h"
-
-/*
- * XXX: A lot of hardcoded 512s probably should be foo->sector_size;
- *	I'm not sure which, so I leave it like it worked before. --schweikh
- */
-static int
-Write_FreeBSD(int fd, const struct disk *new, const struct chunk *c1)
-{
-	struct disklabel *dl;
-	int i;
-	void *p;
-	u_char buf[BBSIZE];
-
-	for (i = 0; i < BBSIZE/512; i++) {
-		if (!(p = read_block(fd, i + c1->offset, 512)))
-			return (1);
-		memcpy(buf + 512 * i, p, 512);
-		free(p);
-	}
-	if (new->boot1)
-		memcpy(buf, new->boot1, 512);
-
-	if (new->boot2)
-		memcpy(buf + 512, new->boot2, BBSIZE - 512);
-
-	dl = (struct disklabel *)(buf + 512 * LABELSECTOR + LABELOFFSET);
-	Fill_Disklabel(dl, new, c1);
-
-	for (i = 0; i < BBSIZE / 512; i++)
-		write_block(fd, i + c1->offset, buf + 512 * i, 512);
-
-	return 0;
-}
-
-static void
-Write_Int32(u_int32_t *p, u_int32_t v)
-{
-	u_int8_t *bp = (u_int8_t *)p;
-
-	bp[0] = (v >> 0) & 0xff;
-	bp[1] = (v >> 8) & 0xff;
-	bp[2] = (v >> 16) & 0xff;
-	bp[3] = (v >> 24) & 0xff;
-}
-
-/*
- * Special install-time configuration for the i386 boot0 boot manager.
- */
-static void
-Cfg_Boot_Mgr(u_char *mbrblk, int edd)
-{
-
-	if (mbrblk[0x1b0] == 0x66 && mbrblk[0x1b1] == 0xbb) {
-		if (edd)
-			mbrblk[0x1bb] |= 0x80;	/* Packet mode on */
-		else
-			mbrblk[0x1bb] &= 0x7f;	/* Packet mode off */
-	}
-}
-
-int
-Write_Disk(const struct disk *d1)
-{
-	int fd, j;
-	uint i;
-	struct chunk *c1;
-	int ret = 0;
-	char device[64];
-	u_char *mbrblk;
-	struct dos_partition *dp,work[NDOSPART];
-	int s[4];
-	int need_edd = 0;	/* Need EDD (packet interface) */
-
-	strcpy(device, _PATH_DEV);
-        strcat(device, d1->name);
-
-        fd = open(device, O_RDWR);
-        if (fd < 0)
-                return 1;
-
-	memset(s, 0, sizeof s);
-	if (!(mbrblk = read_block(fd, 0, d1->sector_size))) {
-		close (fd);
-		return (1);
-	}
-	dp = (struct dos_partition *)(mbrblk + DOSPARTOFF);
-	memcpy(work, dp, sizeof work);
-	dp = work;
-	free(mbrblk);
-	for (c1 = d1->chunks->part; c1; c1 = c1->next) {
-		if (c1->type == unused)
-			continue;
-		if (!strcmp(c1->name, "X"))
-			continue;
-		j = c1->name[strlen(d1->name) + 1] - '1';
-		if (j < 0 || j > 3)
-			continue;
-		s[j]++;
-		if (c1->type == freebsd)
-			ret += Write_FreeBSD(fd, d1, c1);
-
-		Write_Int32(&dp[j].dp_start, c1->offset);
-		Write_Int32(&dp[j].dp_size, c1->size);
-
-		i = c1->offset;
-		if (i >= 1024 * d1->bios_sect * d1->bios_hd) {
-			dp[j].dp_ssect = 0xff;
-			dp[j].dp_shd = 0xff;
-			dp[j].dp_scyl = 0xff;
-			need_edd++;
-		} else {
-			dp[j].dp_ssect = i % d1->bios_sect;
-			i -= dp[j].dp_ssect++;
-			i /= d1->bios_sect;
-			dp[j].dp_shd =  i % d1->bios_hd;
-			i -= dp[j].dp_shd;
-			i /= d1->bios_hd;
-			dp[j].dp_scyl = i;
-			i -= dp[j].dp_scyl;
-			dp[j].dp_ssect |= i >> 2;
-		}
-#ifdef DEBUG
-		printf("S:%lu = (%x/%x/%x)", c1->offset,
-		       dp[j].dp_scyl, dp[j].dp_shd, dp[j].dp_ssect);
-#endif
-
-		i = c1->end;
-		dp[j].dp_esect = i % d1->bios_sect;
-		i -= dp[j].dp_esect++;
-		i /= d1->bios_sect;
-		dp[j].dp_ehd =  i % d1->bios_hd;
-		i -= dp[j].dp_ehd;
-		i /= d1->bios_hd;
-		if (i > 1023)
-			i = 1023;
-		dp[j].dp_ecyl = i;
-		i -= dp[j].dp_ecyl;
-		dp[j].dp_esect |= i >> 2;
-#ifdef DEBUG
-		printf("  E:%lu = (%x/%x/%x)\n", c1->end,
-		       dp[j].dp_ecyl, dp[j].dp_ehd, dp[j].dp_esect);
-#endif
-
-		dp[j].dp_typ = c1->subtype;
-		if (c1->flags & CHUNK_ACTIVE)
-			dp[j].dp_flag = 0x80;
-		else
-			dp[j].dp_flag = 0;
-	}
-	j = 0;
-	for (i = 0; i < NDOSPART; i++) {
-		if (!s[i])
-			memset(dp + i, 0, sizeof *dp);
-		if (dp[i].dp_flag)
-			j++;
-	}
-	if (!j)
-		for(i = 0; i < NDOSPART; i++)
-			if (dp[i].dp_typ == 0xa5)
-				dp[i].dp_flag = 0x80;
-
-	if (!(mbrblk = read_block(fd, 0, d1->sector_size))) {
-		close (fd);
-		return (1);
-	}
-	if (d1->bootmgr) {
-		memcpy(mbrblk, d1->bootmgr, DOSPARTOFF);
-		Cfg_Boot_Mgr(mbrblk, need_edd);
-        }
-	memcpy(mbrblk + DOSPARTOFF, dp, sizeof *dp * NDOSPART);
-	mbrblk[512-2] = 0x55;
-	mbrblk[512-1] = 0xaa;
-	write_block(fd, 0, mbrblk, d1->sector_size);
-	if (d1->bootmgr && d1->bootmgr_size > d1->sector_size)
-		for (i = 1; i * d1->sector_size <= d1->bootmgr_size; i++)
-			write_block(fd, i, &d1->bootmgr[i * d1->sector_size],
-				    d1->sector_size);
-
-	close(fd);
-	return 0;
-}
diff --git a/lib/libdisk/write_ia64_disk.c b/lib/libdisk/write_ia64_disk.c
deleted file mode 100644
index 6a57322..0000000
--- a/lib/libdisk/write_ia64_disk.c
+++ /dev/null
@@ -1,423 +0,0 @@
-/*
- * Copyright (c) 2003 Marcel Moolenaar
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
- * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
- * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * CRC32 code derived from work by Gary S. Brown.
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/types.h>
-#include <sys/disklabel.h>
-#include <sys/diskmbr.h>
-#include <sys/gpt.h>
-#include <sys/stat.h>
-
-#include <errno.h>
-#include <fcntl.h>
-#include <paths.h>
-#include <stddef.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-#include <uuid.h>
-
-#include "libdisk.h"
-
-static uuid_t _efi = GPT_ENT_TYPE_EFI;
-static uuid_t _fbsd = GPT_ENT_TYPE_FREEBSD;
-static uuid_t _swap = GPT_ENT_TYPE_FREEBSD_SWAP;
-static uuid_t _ufs = GPT_ENT_TYPE_FREEBSD_UFS;
-
-static uint32_t crc32_tab[] = {
-	0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f,
-	0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988,
-	0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2,
-	0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7,
-	0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9,
-	0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172,
-	0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c,
-	0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59,
-	0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423,
-	0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924,
-	0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, 0x01db7106,
-	0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433,
-	0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d,
-	0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e,
-	0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950,
-	0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65,
-	0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7,
-	0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0,
-	0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa,
-	0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f,
-	0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81,
-	0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a,
-	0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84,
-	0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1,
-	0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb,
-	0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc,
-	0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e,
-	0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b,
-	0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55,
-	0x316e8eef, 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236,
-	0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28,
-	0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d,
-	0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f,
-	0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38,
-	0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242,
-	0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777,
-	0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69,
-	0x616bffd3, 0x166ccf45, 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2,
-	0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc,
-	0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9,
-	0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693,
-	0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94,
-	0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d
-};
-
-static uint32_t
-crc32(const void *buf, size_t size)
-{
-	const uint8_t *p;
-	uint32_t crc;
-
-	p = buf;
-	crc = ~0U;
-
-	while (size--)
-		crc = crc32_tab[(crc ^ *p++) & 0xFF] ^ (crc >> 8);
-
-	return (crc ^ ~0U);
-}
-
-static int
-write_pmbr(int fd, const struct disk *disk)
-{
-	struct dos_partition dp;
-	char *buffer;
-	u_long nsects;
-	int error;
-
-	error = 0;
-	nsects = disk->media_size / disk->sector_size;
-	nsects--;	/* The GPT starts at LBA 1 */
-
-	buffer = calloc(disk->sector_size, 1);
-	if (buffer == NULL)
-		return (ENOMEM);
-	buffer[DOSMAGICOFFSET] = DOSMAGIC & 0xff;
-	buffer[DOSMAGICOFFSET + 1] = DOSMAGIC >> 8;
-
-	dp.dp_flag = 0;
-	dp.dp_shd = dp.dp_ssect = dp.dp_scyl = 0xff;
-	dp.dp_typ = DOSPTYP_PMBR;
-	dp.dp_ehd = dp.dp_esect = dp.dp_ecyl = 0xff;
-	dp.dp_start = 1;
-	dp.dp_size = (nsects > 0xffffffffu) ? ~0u : nsects;
-	memcpy(buffer + DOSPARTOFF, &dp, DOSPARTSIZE);
-
-	if (lseek(fd, 0L, SEEK_SET) != 0L ||
-	    write(fd, buffer, disk->sector_size) != disk->sector_size)
-		error = (errno) ? errno : EAGAIN;
-
-	free(buffer);
-	return (error);
-}
-
-static int
-read_gpt(int fd, const struct disk *disk, struct gpt_hdr *hdr,
-    struct gpt_ent *tbl)
-{
-	char *buffer;
-	off_t off;
-	size_t nsects, sz;
-	int error, i;
-
-	error = 0;
-	nsects = disk->gpt_size * sizeof(struct gpt_ent) / disk->sector_size;
-	nsects++;
-	sz = nsects * disk->sector_size;
-	buffer = malloc(sz);
-	if (buffer == NULL)
-		return (ENOMEM);
-
-	if (lseek(fd, disk->sector_size, SEEK_SET) != disk->sector_size ||
-	    read(fd, buffer, disk->sector_size) != disk->sector_size) {
-		error = (errno) ? errno : EAGAIN;
-		goto bail;
-	}
-	if (memcmp(buffer, GPT_HDR_SIG, sizeof(hdr->hdr_sig)) != 0) {
-		/*
-		 * No GPT on disk. Create one out of thin air.
-		 */
-		bzero(&hdr[0], sizeof(struct gpt_hdr));
-		memcpy(hdr[0].hdr_sig, GPT_HDR_SIG, sizeof(hdr[0].hdr_sig));
-		hdr[0].hdr_revision = GPT_HDR_REVISION;
-		hdr[0].hdr_size = offsetof(struct gpt_hdr, padding);
-		hdr[0].hdr_lba_self = 1;
-		hdr[0].hdr_lba_alt = disk->media_size / disk->sector_size - 1L;
-		hdr[0].hdr_lba_start = disk->lba_start;
-		hdr[0].hdr_lba_end = disk->lba_end;
-		uuid_create(&hdr[0].hdr_uuid, NULL);
-		hdr[0].hdr_lba_table = 2;
-		hdr[0].hdr_entries = disk->gpt_size;
-		hdr[0].hdr_entsz = sizeof(struct gpt_ent);
-		hdr[1] = hdr[0];
-		hdr[1].hdr_lba_self = hdr[0].hdr_lba_alt;
-		hdr[1].hdr_lba_alt = hdr[0].hdr_lba_self;
-		hdr[1].hdr_lba_table = disk->lba_end + 1;
-
-		for (i = 0; i < disk->gpt_size; i++) {
-			bzero(&tbl[i], sizeof(struct gpt_ent));
-			uuid_create(&tbl[i].ent_uuid, NULL);
-		}
-
-		goto bail;
-	}
-
-	/*
-	 * We have a GPT on disk. Read it.
-	 */
-	memcpy(&hdr[0], buffer, sizeof(struct gpt_hdr));
-	off = hdr->hdr_lba_table * disk->sector_size;
-	if (lseek(fd, off, SEEK_SET) != off ||
-	    read(fd, buffer, sz) != sz) {
-		error = (errno) ? errno : EAGAIN;
-		goto bail;
-	}
-	memcpy(tbl, buffer, sizeof(struct gpt_ent) * disk->gpt_size);
-	off = hdr->hdr_lba_alt * disk->sector_size;
-	if (lseek(fd, off, SEEK_SET) != off ||
-	    read(fd, buffer, disk->sector_size) != disk->sector_size) {
-		error = (errno) ? errno : EAGAIN;
-		goto bail;
-	}
-	memcpy(&hdr[1], buffer, sizeof(struct gpt_hdr));
-
-bail:
-	free(buffer);
-	return (error);
-}
-
-static int
-update_gpt(int fd, const struct disk *disk, struct gpt_hdr *hdr,
-    struct gpt_ent *tbl)
-{
-	struct gpt_ent *save;
-	char *buffer;
-	struct chunk *c;
-	off_t off;
-	size_t bufsz;
-	int error, idx, sav;
-
-	error = 0;
-
-	/*
-	 * Save the entries of those chunks that have an index. They are
-	 * the ones that exist on disk already.
-	 */
-	sav = 0;
-	for (c = disk->chunks->part; c != NULL; c = c->next) {
-		if ((c->flags & CHUNK_HAS_INDEX))
-			sav++;
-	}
-	if (sav > 0) {
-		save = malloc(sav * sizeof(struct gpt_ent));
-		if (save == NULL)
-			abort();
-		sav = 0;
-		for (c = disk->chunks->part; c != NULL; c = c->next) {
-			if ((c->flags & CHUNK_HAS_INDEX)) {
-				idx = CHUNK_FTOI(c->flags);
-				save[sav] = tbl[idx];
-				c->flags ^= CHUNK_ITOF(idx);
-				c->flags |= CHUNK_ITOF(sav);
-				sav++;
-			}
-		}
-	} else
-		save = NULL;
-
-	/*
-	 * Clear the table entries.
-	 */
-	for (idx = 0; idx < disk->gpt_size; idx++) {
-		uuid_create_nil(&tbl[idx].ent_type, NULL);
-		tbl[idx].ent_lba_start = 0;
-		tbl[idx].ent_lba_end = 0;
-		tbl[idx].ent_attr = 0;
-		bzero(tbl[idx].ent_name, sizeof(tbl[idx].ent_name));
-	}
-
-	/*
-	 * Repopulate the table from the chunks, possibly using saved
-	 * information.
-	 */
-	idx = 0;
-	for (c = disk->chunks->part; c != NULL; c = c->next) {
-		if (!(c->flags & CHUNK_HAS_INDEX)) {
-			switch (c->type) {
-			case freebsd:
-				tbl[idx].ent_type = _fbsd;
-				break;
-			case efi:
-				tbl[idx].ent_type = _efi;
-				break;
-			case part:
-				switch (c->subtype) {
-				case FS_SWAP:
-					tbl[idx].ent_type = _swap;
-					break;
-				case FS_BSDFFS:
-					tbl[idx].ent_type = _ufs;
-					break;
-				default:
-					return (EINVAL);
-				}
-				break;
-			default:
-				return (EINVAL);
-			}
-		} else {
-			sav = CHUNK_FTOI(c->flags);
-			tbl[idx].ent_type = save[sav].ent_type;
-			memcpy(tbl[idx].ent_name, save[sav].ent_name,
-			    sizeof(tbl[idx].ent_name));
-		}
-		tbl[idx].ent_lba_start = c->offset;
-		tbl[idx].ent_lba_end = c->end;
-
-		idx++;
-		if (idx == disk->gpt_size)
-			return (ENOSPC);
- 	}
-	if (save != NULL)
-		free(save);
-
-	hdr[0].hdr_crc_table = crc32(tbl,
-	    disk->gpt_size * sizeof(struct gpt_ent));
-	hdr[0].hdr_crc_self = 0;
-	hdr[0].hdr_crc_self = crc32(&hdr[0], hdr[0].hdr_size);
-
-	hdr[1].hdr_crc_table = hdr[0].hdr_crc_table;
-	hdr[1].hdr_crc_self = 0;
-	hdr[1].hdr_crc_self = crc32(&hdr[1], hdr[1].hdr_size);
-
-	/*
-	 * Write the new GPT back to the disk.
-	 */
-	bufsz = disk->gpt_size * sizeof(struct gpt_ent);
-	if (bufsz == 0 || bufsz % disk->sector_size)
-		bufsz += disk->sector_size;
-	bufsz = (bufsz / disk->sector_size) * disk->sector_size;
-	buffer = calloc(1, bufsz);
-
-	memcpy(buffer, &hdr[0], sizeof(struct gpt_hdr));
-	off = hdr[0].hdr_lba_self * disk->sector_size;
-	if (lseek(fd, off, SEEK_SET) != off ||
-	    write(fd, buffer, disk->sector_size) != disk->sector_size) {
-		error = (errno) ? errno : EAGAIN;
-		goto bail;
-	}
-	memcpy(buffer, &hdr[1], sizeof(struct gpt_hdr));
-	off = hdr[1].hdr_lba_self * disk->sector_size;
-	if (lseek(fd, off, SEEK_SET) != off ||
-	    write(fd, buffer, disk->sector_size) != disk->sector_size) {
-		error = (errno) ? errno : EAGAIN;
-		goto bail;
-	}
-	memcpy(buffer, tbl, disk->gpt_size * sizeof(struct gpt_ent));
-	off = hdr[0].hdr_lba_table * disk->sector_size;
-	if (lseek(fd, off, SEEK_SET) != off ||
-	    write(fd, buffer, bufsz) != bufsz) {
-		error = (errno) ? errno : EAGAIN;
-		goto bail;
-	}
-	off = hdr[1].hdr_lba_table * disk->sector_size;
-	if (lseek(fd, off, SEEK_SET) != off ||
-	    write(fd, buffer, bufsz) != bufsz) {
-		error = (errno) ? errno : EAGAIN;
-		goto bail;
-	}
-
-bail:
-	free(buffer);
-	return (error);
-}
-
-int
-Write_Disk(const struct disk *disk)
-{
-	char devname[64];
-	struct gpt_hdr *hdr;
-	struct gpt_ent *tbl;
-	int error, fd;
-
-	hdr = malloc(sizeof(struct gpt_hdr) * 2);
-	if (hdr == NULL)
-		return (ENOMEM);
-	tbl = malloc(sizeof(struct gpt_ent) * disk->gpt_size);
-	if (tbl == NULL) {
-		free(hdr);
-		return (ENOMEM);
-	}
-
-	snprintf(devname, sizeof(devname), "%s%s", _PATH_DEV, disk->name);
-	fd = open(devname, O_RDWR);
-	if (fd == -1) {
-		free(tbl);
-		free(hdr);
-		return (errno);
-	}
-
-	/*
-	 * We can always write the PMBR, because we reject disks that do not
-	 * have a PMBR and are not virgin.
-	 */
-	error = write_pmbr(fd, disk);
-	if (error)
-		goto bail;
-
-	/*
-	 * Read the existing GPT from disk or otherwise create one out of
-	 * thin air. This way we can preserve the UUIDs and the entry names
-	 * when updating it.
-	 */
-	error = read_gpt(fd, disk, hdr, tbl);
-	if (error)
-		goto bail;
-
-	/*
-	 * Update and write the in-memory copy of the GPT.
-	 */
-	error = update_gpt(fd, disk, hdr, tbl);
-
-bail:
-	close(fd);
-	free(tbl);
-	free(hdr);
-	return (error);
-}
diff --git a/lib/libdisk/write_mips_disk.c b/lib/libdisk/write_mips_disk.c
deleted file mode 100644
index bd0cb87..0000000
--- a/lib/libdisk/write_mips_disk.c
+++ /dev/null
@@ -1,48 +0,0 @@
-/*-
- * Copyright (c) 2006 Olivier Houchard
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <string.h>
-#include <err.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/ioctl.h>
-#include <sys/disklabel.h>
-#include <paths.h>
-#include "libdisk.h"
-
-int
-Write_Disk(const struct disk *d1)
-{
-
-	return (0);
-}
diff --git a/lib/libdisk/write_pc98_disk.c b/lib/libdisk/write_pc98_disk.c
deleted file mode 100644
index 8f2a45d..0000000
--- a/lib/libdisk/write_pc98_disk.c
+++ /dev/null
@@ -1,179 +0,0 @@
-/*
- * ----------------------------------------------------------------------------
- * "THE BEER-WARE LICENSE" (Revision 42):
- * <phk@FreeBSD.org> wrote this file.  As long as you retain this notice you
- * can do whatever you want with this stuff. If we meet some day, and you think
- * this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
- * ----------------------------------------------------------------------------
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <string.h>
-#include <err.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/ioctl.h>
-#include <sys/disklabel.h>
-#include <sys/diskpc98.h>
-#include <paths.h>
-#include "libdisk.h"
-
-/*
- * XXX: A lot of hardcoded 512s probably should be foo->sector_size;
- *	I'm not sure which, so I leave it like it worked before. --schweikh
- */
-static int
-Write_FreeBSD(int fd, const struct disk *new, const struct chunk *c1)
-{
-	struct disklabel *dl;
-	int i;
-	void *p;
-	u_char buf[BBSIZE];
-
-	for (i = 0; i < BBSIZE / 512; i++) {
-		if (!(p = read_block(fd, i + c1->offset, 512)))
-			return (1);
-		memcpy(buf + 512 * i, p, 512);
-		free(p);
-	}
-	if (new->boot1)
-		memcpy(buf, new->boot1, 512);
-
-	if (new->boot2)
-		memcpy(buf + 512, new->boot2, BBSIZE - 512);
-
-	dl = (struct disklabel *)(buf + 512 * LABELSECTOR + LABELOFFSET);
-	Fill_Disklabel(dl, new, c1);
-
-	for (i = 0; i < BBSIZE / 512; i++)
-		write_block(fd, i + c1->offset, buf + 512 * i, 512);
-
-	return 0;
-}
-
-
-int
-Write_Disk(const struct disk *d1)
-{
-	int fd, i, j;
-	struct chunk *c1;
-	int ret = 0;
-	char device[64];
-	u_char *mbrblk;
-	struct pc98_partition *dp, work[NDOSPART];
-	int s[7];
-	int PC98_EntireDisk = 0;
-
-	strcpy(device, _PATH_DEV);
-        strcat(device, d1->name);
-
-	/* XXX - for entire FreeBSD(98) */
-	for (c1 = d1->chunks->part; c1; c1 = c1->next) {
-	    if ((c1->type == freebsd) || (c1->offset == 0))
-		device[9] = 0;
-	}
-
-	fd = open(device, O_RDWR);
-	if (fd < 0) {
-#ifdef DEBUG
-		warn("open(%s) failed", device);
-#endif
-		return 1;
-	}
-
-	memset(s, 0, sizeof s);
-	if (!(mbrblk = read_block(fd, 1, d1->sector_size))) {
-		close (fd);
-		return (1);
-	}
-	dp = (struct pc98_partition *)(mbrblk + DOSPARTOFF);
-	memcpy(work, dp, sizeof work);
-	dp = work;
-	free(mbrblk);
-	for (c1 = d1->chunks->part; c1; c1 = c1->next) {
-		if (c1->type == unused)
-			continue;
-		if (!strcmp(c1->name, "X"))
-			continue;
-		j = c1->name[strlen(d1->name) + 1] - '1';
-		if (j < 0 || j > 7)
-			continue;
-		s[j]++;
-		if (c1->type == freebsd)
-			ret += Write_FreeBSD(fd, d1, c1);
-
-		i = c1->offset;
-		dp[j].dp_ssect = dp[j].dp_ipl_sct = i % d1->bios_sect;
-		i -= dp[j].dp_ssect;
-		i /= d1->bios_sect;
-		dp[j].dp_shd = dp[j].dp_ipl_head = i % d1->bios_hd;
-		i -= dp[j].dp_shd;
-		i /= d1->bios_hd;
-		dp[j].dp_scyl = dp[j].dp_ipl_cyl = i;
-#ifdef DEBUG
-		printf("S:%lu = (%x/%x/%x)", c1->offset,
-		       dp[j].dp_scyl, dp[j].dp_shd, dp[j].dp_ssect);
-#endif
-
-		i = c1->end;
-#if 1
-		dp[j].dp_esect = dp[j].dp_ehd = 0;
-		dp[j].dp_ecyl = i / (d1->bios_sect * d1->bios_hd);
-#else
-		dp[j].dp_esect = i % d1->bios_sect;
-		i -= dp[j].dp_esect;
-		i /= d1->bios_sect;
-		dp[j].dp_ehd = i % d1->bios_hd;
-		i -= dp[j].dp_ehd;
-		i /= d1->bios_hd;
-		dp[j].dp_ecyl = i;
-#endif
-#ifdef DEBUG
-		printf("  E:%lu = (%x/%x/%x)\n", c1->end,
-		       dp[j].dp_ecyl, dp[j].dp_ehd, dp[j].dp_esect);
-#endif
-
-		dp[j].dp_mid = c1->subtype & 0xff;
-		dp[j].dp_sid = c1->subtype >> 8;
-		if (c1->flags & CHUNK_ACTIVE)
-			dp[j].dp_mid |= 0x80;
-
-		strncpy(dp[j].dp_name, c1->sname, 16);
-	}
-	j = 0;
-	for (i = 0; i < NDOSPART; i++) {
-		if (!s[i])
-			memset(dp + i, 0, sizeof *dp);
-	}
-
-	if (d1->bootipl)
-		write_block(fd, 0, d1->bootipl, d1->sector_size);
-
-	if (!(mbrblk = read_block(fd, 1, d1->sector_size))) {
-		close (fd);
-		return (1);
-	}
-	memcpy(mbrblk + DOSPARTOFF, dp, sizeof *dp * NDOSPART);
-	/* XXX - for entire FreeBSD(98) */
-	for (c1 = d1->chunks->part; c1; c1 = c1->next)
-		if (((c1->type == freebsd) || (c1->type == fat))
-			 && (c1->offset == 0))
-			PC98_EntireDisk = 1;
-	if (PC98_EntireDisk == 0)
-		write_block(fd, 1, mbrblk, d1->sector_size);
-
-	if (d1->bootmenu)
-		for (i = 0; i * d1->sector_size < d1->bootmenu_size; i++)
-			write_block(fd, 2 + i,
-				    &d1->bootmenu[i * d1->sector_size],
-				    d1->sector_size);
-
-	close(fd);
-	return 0;
-}
diff --git a/lib/libdisk/write_powerpc_disk.c b/lib/libdisk/write_powerpc_disk.c
deleted file mode 100644
index ec955e2..0000000
--- a/lib/libdisk/write_powerpc_disk.c
+++ /dev/null
@@ -1,64 +0,0 @@
-/*-
- * Copyright (c) 2004 Suleiman Souhlal.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <string.h>
-#include <err.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/ioctl.h>
-#include <sys/disklabel.h>
-#include <paths.h>
-#include "libdisk.h"
-
-int
-Write_Disk(const struct disk *d1)
-{
-	/* 
-	 * We don't have to write any label, so we only check that we can
-	 * open the disk.
-	 */
-	int fd;
-	char device[64];
-
-	strcpy(device, _PATH_DEV);
-	strcat(device, d1->name);
-
-	fd = open(device, O_RDWR);
-	if (fd < 0) {
-		close(fd);
-		return 1;
-	}
-
-	close(fd);
-	return 0;
-}
diff --git a/lib/libdisk/write_sparc64_disk.c b/lib/libdisk/write_sparc64_disk.c
deleted file mode 100644
index 9832b0d..0000000
--- a/lib/libdisk/write_sparc64_disk.c
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * ----------------------------------------------------------------------------
- * "THE BEER-WARE LICENSE" (Revision 42):
- * <phk@FreeBSD.org> wrote this file.  As long as you retain this notice you
- * can do whatever you want with this stuff. If we meet some day, and you think
- * this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
- * ----------------------------------------------------------------------------
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <err.h>
-#include <string.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <sys/ioctl.h>
-#include <sys/sun_disklabel.h>
-#include <paths.h>
-#include <errno.h>
-#include "libdisk.h"
-
-#include "geom_sunlabel_enc.c"
-
-int
-Write_Disk(const struct disk *d1)
-{
-	struct sun_disklabel *sl;
-	struct chunk *c, *c1, *c2;
-	int i;
-	char *p;
-	u_long secpercyl;
-	char device[64];
-	u_char buf[SUN_SIZE];
-	int fd;
-
-	strcpy(device, _PATH_DEV);
-	strcat(device, d1->name);
-
-	fd = open(device, O_RDWR);
-	if (fd < 0) {
-		warn("open(%s) failed", device);
-		return (1);
-	}
-
-	sl = calloc(sizeof *sl, 1);
-	c = d1->chunks;
-	c2 = c->part;
-	secpercyl = d1->bios_sect * d1->bios_hd;
-	sl->sl_pcylinders = c->size / secpercyl;
-	sl->sl_ncylinders = c2->size / secpercyl;
-	sl->sl_acylinders = sl->sl_pcylinders - sl->sl_ncylinders;
-	sl->sl_magic = SUN_DKMAGIC;
-	sl->sl_nsectors = d1->bios_sect;
-	sl->sl_ntracks = d1->bios_hd;
-	if (c->size > 4999 * 1024 * 2) {
-		sprintf(sl->sl_text, "FreeBSD%luG cyl %u alt %u hd %u sec %u",
-			(c->size + 1024 * 1024) / (2 * 1024 * 1024),
-			sl->sl_ncylinders, sl->sl_acylinders,
-			sl->sl_ntracks, sl->sl_nsectors);
-	} else {
-		sprintf(sl->sl_text, "FreeBSD%luM cyl %u alt %u hd %u sec %u",
-			(c->size + 1024) / (2 * 1024),
-			sl->sl_ncylinders, sl->sl_acylinders,
-			sl->sl_ntracks, sl->sl_nsectors);
-	}
-	sl->sl_interleave = 1;
-	sl->sl_sparespercyl = 0;
-	sl->sl_rpm = 3600;
-
-	for (c1 = c2->part; c1 != NULL; c1 = c1->next) {
-		p = c1->name;
-		p += strlen(p);
-		p--;
-		if (*p < 'a')
-			continue;
-		i = *p - 'a';
-		if (i >= SUN_NPART)
-			continue;
-		sl->sl_part[i].sdkp_cyloffset = c1->offset / secpercyl;
-		sl->sl_part[i].sdkp_nsectors = c1->size;
-		for (i = 1; i < 16; i++) {
-			write_block(fd, c1->offset + i, d1->boot1 + (i * 512),
-			    512);
-		}
-	}
-
-	/*
-	 * We need to fill in the "RAW" partition as well.  Emperical data
-	 * seems to indicate that this covers the "obviously" visible part
-	 * of the disk, ie: sl->sl_ncylinders.
-	 */
-	sl->sl_part[SUN_RAWPART].sdkp_cyloffset = 0;
-	sl->sl_part[SUN_RAWPART].sdkp_nsectors = sl->sl_ncylinders * secpercyl;
-
-	memset(buf, 0, sizeof buf);
-	sunlabel_enc(buf, sl);
-	write_block(fd, 0, buf, sizeof buf);
-
-	close(fd);
-	return 0;
-}
-- 
cgit v1.1


From 388760425c9362d3b543a8f67b15e92345891eda Mon Sep 17 00:00:00 2001
From: antoine <antoine@FreeBSD.org>
Date: Tue, 1 Jan 2013 18:29:25 +0000
Subject: Add missing DPADD

---
 lib/libbsnmp/libbsnmp/Makefile | 1 +
 1 file changed, 1 insertion(+)

(limited to 'lib')

diff --git a/lib/libbsnmp/libbsnmp/Makefile b/lib/libbsnmp/libbsnmp/Makefile
index f21f1d3..20015b9 100644
--- a/lib/libbsnmp/libbsnmp/Makefile
+++ b/lib/libbsnmp/libbsnmp/Makefile
@@ -15,6 +15,7 @@ CFLAGS+= -DHAVE_STDINT_H -DHAVE_INTTYPES_H -DQUADFMT='"llu"' -DQUADXFMT='"llx"'
 
 .if ${MK_OPENSSL} != "no"
 CFLAGS+= -DHAVE_LIBCRYPTO
+DPADD+= ${LIBCRYPTO}
 LDADD+= -lcrypto
 .endif
 
-- 
cgit v1.1


From ea3d4153432447110dcfd6563fd07cc9b7c0e9c2 Mon Sep 17 00:00:00 2001
From: pfg <pfg@FreeBSD.org>
Date: Tue, 1 Jan 2013 19:20:51 +0000
Subject: libedit: bind the correct command when using "bind -k".

"ed-argument-digit" (i. e. command 0) was incorrectly used
instead.
This bug comes from the original sources imported in 1994
and has been confirmed in upstream NetBSD.

Reported by:	Yamagi Burmeister
Submitted by:	Christoph Mallon
MFC after:	3 days
---
 lib/libedit/map.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/libedit/map.c b/lib/libedit/map.c
index 591118f..c70904a 100644
--- a/lib/libedit/map.c
+++ b/lib/libedit/map.c
@@ -1250,7 +1250,7 @@ map_bind(EditLine *el, int argc, const char **argv)
 	char inbuf[EL_BUFSIZ];
 	char outbuf[EL_BUFSIZ];
 	const char *in = NULL;
-	char *out = NULL;
+	char *out;
 	el_bindings_t *bp, *ep;
 	int cmd;
 	int key;
@@ -1368,7 +1368,7 @@ map_bind(EditLine *el, int argc, const char **argv)
 			return (-1);
 		}
 		if (key)
-			term_set_arrow(el, in, key_map_str(el, out), ntype);
+			term_set_arrow(el, in, key_map_cmd(el, cmd), ntype);
 		else {
 			if (in[1]) {
 				key_add(el, in, key_map_cmd(el, cmd), ntype);
-- 
cgit v1.1


From 79c215314de47d4f325b2533d27525197fd3d0a8 Mon Sep 17 00:00:00 2001
From: andrew <andrew@FreeBSD.org>
Date: Sun, 6 Jan 2013 00:49:06 +0000
Subject: Silence a clang warning by telling it we are only interested in left
 shifting the lower 32bits of the floating point value when we demangle it.

---
 lib/libc/arm/softfloat/arm-gcc.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libc/arm/softfloat/arm-gcc.h b/lib/libc/arm/softfloat/arm-gcc.h
index 1204108..15bc509 100644
--- a/lib/libc/arm/softfloat/arm-gcc.h
+++ b/lib/libc/arm/softfloat/arm-gcc.h
@@ -95,7 +95,7 @@ what the endianness of the CPU.  VFP is sane.
 #define FLOAT64_DEMANGLE(a)	(a)
 #define FLOAT64_MANGLE(a)	(a)
 #else
-#define FLOAT64_DEMANGLE(a)	(((a) << 32) | ((a) >> 32))
+#define FLOAT64_DEMANGLE(a)	((((a) & 0xfffffffful) << 32) | ((a) >> 32))
 #define FLOAT64_MANGLE(a)	FLOAT64_DEMANGLE(a)
 #endif
 #endif
-- 
cgit v1.1


From 6d5842f6eea3f16811bd199c8dd2945772ddb351 Mon Sep 17 00:00:00 2001
From: kib <kib@FreeBSD.org>
Date: Mon, 7 Jan 2013 17:58:27 +0000
Subject: Only assign the environ in the startup code when environ is NULL.
 Preloaded library could have changed the environment, and unconditional
 assingment to the environ undoes the customization. The binaries needs to be
 recompiled to get the fix.

Move the common code to set up environ and __progname into the helper.
Note that ia64 possibly not fixed, due to it still using old csu.

Reported and tested by:	John Hein <jhein@symmetricom.com>
Reviewed by:	kan, scf
Approved by:	secteam (simon)
MFC after:	2 weeks
---
 lib/csu/amd64/crt1.c         |  4 +---
 lib/csu/arm/crt1.c           |  5 +----
 lib/csu/common/ignore_init.c | 14 +++++++++-----
 lib/csu/i386-elf/crt1_c.c    |  5 +----
 lib/csu/mips/crt1.c          |  4 +---
 lib/csu/powerpc/crt1.c       |  4 +---
 lib/csu/powerpc64/crt1.c     |  5 +----
 lib/csu/sparc64/crt1.c       |  4 +---
 8 files changed, 16 insertions(+), 29 deletions(-)

(limited to 'lib')

diff --git a/lib/csu/amd64/crt1.c b/lib/csu/amd64/crt1.c
index f33aad6..3740e73 100644
--- a/lib/csu/amd64/crt1.c
+++ b/lib/csu/amd64/crt1.c
@@ -61,9 +61,7 @@ _start(char **ap, void (*cleanup)(void))
 	argc = *(long *)(void *)ap;
 	argv = ap + 1;
 	env = ap + 2 + argc;
-	environ = env;
-	if (argc > 0 && argv[0] != NULL)
-		handle_progname(argv[0]);
+	handle_argv(argc, argv, env);
 
 	if (&_DYNAMIC != NULL)
 		atexit(cleanup);
diff --git a/lib/csu/arm/crt1.c b/lib/csu/arm/crt1.c
index 127c28d..e3529b8 100644
--- a/lib/csu/arm/crt1.c
+++ b/lib/csu/arm/crt1.c
@@ -98,10 +98,7 @@ __start(int argc, char **argv, char **env, struct ps_strings *ps_strings,
     const struct Struct_Obj_Entry *obj __unused, void (*cleanup)(void))
 {
 
-	environ = env;
-
-	if (argc > 0 && argv[0] != NULL)
-		handle_progname(argv[0]);
+	handle_argv(argc, argv, env);
 
 	if (ps_strings != (struct ps_strings *)0)
 		__ps_strings = ps_strings;
diff --git a/lib/csu/common/ignore_init.c b/lib/csu/common/ignore_init.c
index e3d2441..89b3734 100644
--- a/lib/csu/common/ignore_init.c
+++ b/lib/csu/common/ignore_init.c
@@ -87,14 +87,18 @@ handle_static_init(int argc, char **argv, char **env)
 }
 
 static inline void
-handle_progname(const char *v)
+handle_argv(int argc, char *argv[], char **env)
 {
 	const char *s;
 
-	__progname = v;
-	for (s = __progname; *s != '\0'; s++) {
-		if (*s == '/')
-			__progname = s + 1;
+	if (environ == NULL)
+		environ = env;
+	if (argc > 0 && argv[0] != NULL) {
+		__progname = argv[0];
+		for (s = __progname; *s != '\0'; s++) {
+			if (*s == '/')
+				__progname = s + 1;
+		}
 	}
 }
 
diff --git a/lib/csu/i386-elf/crt1_c.c b/lib/csu/i386-elf/crt1_c.c
index 3249069..65de04c 100644
--- a/lib/csu/i386-elf/crt1_c.c
+++ b/lib/csu/i386-elf/crt1_c.c
@@ -61,10 +61,7 @@ _start1(fptr cleanup, int argc, char *argv[])
 	char **env;
 
 	env = argv + argc + 1;
-	environ = env;
-	if (argc > 0 && argv[0] != NULL)
-		handle_progname(argv[0]);
-
+	handle_argv(argc, argv, env);
 	if (&_DYNAMIC != NULL)
 		atexit(cleanup);
 	else
diff --git a/lib/csu/mips/crt1.c b/lib/csu/mips/crt1.c
index 1968f06..95348b7 100644
--- a/lib/csu/mips/crt1.c
+++ b/lib/csu/mips/crt1.c
@@ -71,9 +71,7 @@ __start(char **ap,
 	argc = * (long *) ap;
 	argv = ap + 1;
 	env  = ap + 2 + argc;
-	environ = env;
-	if (argc > 0 && argv[0] != NULL)
-		handle_progname(argv[0]);
+	handle_argv(argc, argv, env);
 
 	if (&_DYNAMIC != NULL)
 		atexit(cleanup);
diff --git a/lib/csu/powerpc/crt1.c b/lib/csu/powerpc/crt1.c
index c3be90d..d1a3ea0 100644
--- a/lib/csu/powerpc/crt1.c
+++ b/lib/csu/powerpc/crt1.c
@@ -81,10 +81,8 @@ _start(int argc, char **argv, char **env,
     struct ps_strings *ps_strings)
 {
 
-	environ = env;
 
-	if (argc > 0 && argv[0] != NULL)
-		handle_progname(argv[0]);
+	handle_argv(argc, argv, env);
 
 	if (ps_strings != (struct ps_strings *)0)
 		__ps_strings = ps_strings;
diff --git a/lib/csu/powerpc64/crt1.c b/lib/csu/powerpc64/crt1.c
index a7c3581..35c5a6e 100644
--- a/lib/csu/powerpc64/crt1.c
+++ b/lib/csu/powerpc64/crt1.c
@@ -81,10 +81,7 @@ _start(int argc, char **argv, char **env,
     struct ps_strings *ps_strings)
 {
 
-	environ = env;
-
-	if (argc > 0 && argv[0] != NULL)
-		handle_progname(argv[0]);
+	handle_argv(argc, argv, env);
 
 	if (ps_strings != (struct ps_strings *)0)
 		__ps_strings = ps_strings;
diff --git a/lib/csu/sparc64/crt1.c b/lib/csu/sparc64/crt1.c
index 3b3ecc2..e11ae39 100644
--- a/lib/csu/sparc64/crt1.c
+++ b/lib/csu/sparc64/crt1.c
@@ -85,9 +85,7 @@ _start(char **ap, void (*cleanup)(void), struct Struct_Obj_Entry *obj __unused,
 	argc = *(long *)(void *)ap;
 	argv = ap + 1;
 	env  = ap + 2 + argc;
-	environ = env;
-	if (argc > 0 && argv[0] != NULL)
-		handle_progname(argv[0]);
+	handle_argv(argc, argv, env);
 
 	if (&_DYNAMIC != NULL)
 		atexit(cleanup);
-- 
cgit v1.1


From 3d32a931a1a2fa6cbcda4ecc497192795994945f Mon Sep 17 00:00:00 2001
From: ume <ume@FreeBSD.org>
Date: Wed, 9 Jan 2013 15:22:37 +0000
Subject: Disable destination address selection support of getipnodebyname(1). 
 RFC 2553 mentions IPv6 addresses are returned 1st.

Spotted by:	uqs
MFC after:	1 week
---
 lib/libc/net/name6.c | 8 ++++++++
 1 file changed, 8 insertions(+)

(limited to 'lib')

diff --git a/lib/libc/net/name6.c b/lib/libc/net/name6.c
index 0031f46..4d7cf79 100644
--- a/lib/libc/net/name6.c
+++ b/lib/libc/net/name6.c
@@ -200,6 +200,7 @@ static struct	 hostent *_hpmapv6(struct hostent *, int *);
 #endif
 static struct	 hostent *_hpsort(struct hostent *, res_state);
 
+#ifdef ENABLE_IP6ADDRCTL
 static struct	 hostent *_hpreorder(struct hostent *);
 static int	 get_addrselectpolicy(struct policyhead *);
 static void	 free_addrselectpolicy(struct policyhead *);
@@ -209,6 +210,7 @@ static void	 set_source(struct hp_order *, struct policyhead *);
 static int	 matchlen(struct sockaddr *, struct sockaddr *);
 static int	 comp_dst(const void *, const void *);
 static int	 gai_addr2scopetype(struct sockaddr *);
+#endif
 
 /*
  * Functions defined in RFC2553
@@ -309,7 +311,11 @@ getipnodebyname(const char *name, int af, int flags, int *errp)
 		*errp = statp->res_h_errno;
 	
 	statp->options = options;
+#ifdef ENABLE_IP6ADDRCTL
 	return _hpreorder(_hpsort(hp, statp));
+#else
+	return _hpsort(hp, statp);
+#endif
 }
 
 struct hostent *
@@ -632,6 +638,7 @@ _hpsort(struct hostent *hp, res_state statp)
 	return hp;
 }
 
+#ifdef ENABLE_IP6ADDRCTL
 /*
  * _hpreorder: sort address by default address selection
  */
@@ -1109,3 +1116,4 @@ gai_addr2scopetype(struct sockaddr *sa)
 		return(-1);
 	}
 }
+#endif
-- 
cgit v1.1


From 4873e359cf040abbf1858a6450b4f3e5a6f61431 Mon Sep 17 00:00:00 2001
From: ae <ae@FreeBSD.org>
Date: Wed, 9 Jan 2013 19:49:35 +0000
Subject: Fix the bindto parameter declaration.

Submitted by:	sem
---
 lib/libradius/libradius.3 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libradius/libradius.3 b/lib/libradius/libradius.3
index 253659c..f3ff700 100644
--- a/lib/libradius/libradius.3
+++ b/lib/libradius/libradius.3
@@ -38,7 +38,7 @@
 .Ft int
 .Fn rad_add_server "struct rad_handle *h" "const char *host" "int port" "const char *secret" "int timeout" "int max_tries"
 .Ft int
-.Fn rad_add_server_ex "struct rad_handle *h" "const char *host" "int port" "const char *secret" "int timeout" "int max_tries" "int dead_time" "struct in_addr bindto"
+.Fn rad_add_server_ex "struct rad_handle *h" "const char *host" "int port" "const char *secret" "int timeout" "int max_tries" "int dead_time" "struct in_addr *bindto"
 .Ft "struct rad_handle *"
 .Fn rad_auth_open "void"
 .Ft void
-- 
cgit v1.1


From 48d4ca6f43e841b9dc56825f68e423088aaa4dfe Mon Sep 17 00:00:00 2001
From: ume <ume@FreeBSD.org>
Date: Thu, 10 Jan 2013 14:08:19 +0000
Subject: Re-enable ip6addrctl support but only for IPv6 address.

Requested by:	Ben Morrow <ben__at__morrow.me.uk>
MFC after:	1 week
---
 lib/libc/net/name6.c | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/net/name6.c b/lib/libc/net/name6.c
index 4d7cf79..f360162 100644
--- a/lib/libc/net/name6.c
+++ b/lib/libc/net/name6.c
@@ -200,7 +200,7 @@ static struct	 hostent *_hpmapv6(struct hostent *, int *);
 #endif
 static struct	 hostent *_hpsort(struct hostent *, res_state);
 
-#ifdef ENABLE_IP6ADDRCTL
+#ifdef INET6
 static struct	 hostent *_hpreorder(struct hostent *);
 static int	 get_addrselectpolicy(struct policyhead *);
 static void	 free_addrselectpolicy(struct policyhead *);
@@ -287,8 +287,10 @@ getipnodebyname(const char *name, int af, int flags, int *errp)
 	
 	hp = gethostbyname2(name, af);
 	hp = _hpcopy(hp, errp);
-
 #ifdef INET6
+	if (af == AF_INET6)
+		hp = _hpreorder(hp);
+
 	if (af == AF_INET6 && ((flags & AI_ALL) || hp == NULL) &&
 	    MAPADDRENABLED(flags)) {
 		struct hostent *hp2 = gethostbyname2(name, AF_INET);
@@ -311,11 +313,7 @@ getipnodebyname(const char *name, int af, int flags, int *errp)
 		*errp = statp->res_h_errno;
 	
 	statp->options = options;
-#ifdef ENABLE_IP6ADDRCTL
-	return _hpreorder(_hpsort(hp, statp));
-#else
 	return _hpsort(hp, statp);
-#endif
 }
 
 struct hostent *
@@ -638,7 +636,7 @@ _hpsort(struct hostent *hp, res_state statp)
 	return hp;
 }
 
-#ifdef ENABLE_IP6ADDRCTL
+#ifdef INET6
 /*
  * _hpreorder: sort address by default address selection
  */
-- 
cgit v1.1


From 4059b9293fea590e8986a3c9ae8be3cea3f02a11 Mon Sep 17 00:00:00 2001
From: brooks <brooks@FreeBSD.org>
Date: Fri, 11 Jan 2013 15:50:01 +0000
Subject: In r244401 I accidently moved strunvis and strunvisx from version 1.0
 to 1.3 breaking the libc ABI. Revert that change (breaking the ABI again for
 users who updated after December 18th).

---
 lib/libc/gen/Symbol.map | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/gen/Symbol.map b/lib/libc/gen/Symbol.map
index 0c6f48e..4cbf07c 100644
--- a/lib/libc/gen/Symbol.map
+++ b/lib/libc/gen/Symbol.map
@@ -298,6 +298,8 @@ FBSD_1.0 {
 	ualarm;
 	ulimit;
 	uname;
+	strunvis;
+	strunvisx;
 	usleep;
 	utime;
 	valloc;
@@ -391,8 +393,6 @@ FBSD_1.3 {
 	snvis;
 	strnunvis;
 	strnunvisx;
-	strunvis;
-	strunvisx;
 	strnvis;
 	strnvisx;
 	strsnvis;
-- 
cgit v1.1


From 05884d1e6d79cdf2481b31f3475ff4fc34369cb5 Mon Sep 17 00:00:00 2001
From: brooks <brooks@FreeBSD.org>
Date: Fri, 11 Jan 2013 17:51:03 +0000
Subject: Add contrib/libc-vis to the include path so we reliably pick up the
 right version of vis.h.

Reported by:	dim
---
 lib/libc/gen/Makefile.inc | 1 +
 1 file changed, 1 insertion(+)

(limited to 'lib')

diff --git a/lib/libc/gen/Makefile.inc b/lib/libc/gen/Makefile.inc
index abface8..0bbbb33 100644
--- a/lib/libc/gen/Makefile.inc
+++ b/lib/libc/gen/Makefile.inc
@@ -40,6 +40,7 @@ SRCS+=  __getosreldate.c __xuname.c \
 SRCS+=	pwcache.c pwcache.h
 
 .PATH: ${.CURDIR}/../../contrib/libc-vis
+CFLAGS+=	-I${.CURDIR}/../../contrib/libc-vis
 SRCS+=	unvis.c vis.c
 
 MISRCS+=modf.c
-- 
cgit v1.1


From 63a1e394aa3156ea61f726ad1fccbf6a8627e84b Mon Sep 17 00:00:00 2001
From: joel <joel@FreeBSD.org>
Date: Sat, 12 Jan 2013 09:07:19 +0000
Subject: editrc only read from $HOME.

Submitted by:	LEVAI Daniel (via jmc@OpenBSD)
---
 lib/libedit/editline.3 | 2 --
 1 file changed, 2 deletions(-)

(limited to 'lib')

diff --git a/lib/libedit/editline.3 b/lib/libedit/editline.3
index fe58321..33f7666 100644
--- a/lib/libedit/editline.3
+++ b/lib/libedit/editline.3
@@ -526,8 +526,6 @@ If
 is
 .Dv NULL ,
 try
-.Pa $PWD/.editrc
-then
 .Pa $HOME/.editrc .
 Refer to
 .Xr editrc 5
-- 
cgit v1.1


From 874cf76c76a7aaae4b28a1efd69de0089b19c920 Mon Sep 17 00:00:00 2001
From: joel <joel@FreeBSD.org>
Date: Sat, 12 Jan 2013 09:08:37 +0000
Subject: Add FILES section.

Submitted by:	jmc@OpenBSD
---
 lib/libedit/editrc.5 | 7 +++++++
 1 file changed, 7 insertions(+)

(limited to 'lib')

diff --git a/lib/libedit/editrc.5 b/lib/libedit/editrc.5
index f46b874..6e04ea1 100644
--- a/lib/libedit/editrc.5
+++ b/lib/libedit/editrc.5
@@ -473,6 +473,13 @@ Move down one line.
 Editline extended command.
 .El
 .\" End of section automatically generated with makelist
+.Sh FILES
+.Bl -tag -width "~/.editrcXXX"
+.It Pa ~/.editrc
+User configuration file for the
+.Xr editline 3
+library.
+.El
 .Sh SEE ALSO
 .Xr editline 3 ,
 .Xr regex 3 ,
-- 
cgit v1.1


From 15b9700547d6667a4e855e30cc7aaf11a2a9908a Mon Sep 17 00:00:00 2001
From: mjg <mjg@FreeBSD.org>
Date: Sun, 13 Jan 2013 21:25:43 +0000
Subject: libutil: eliminate 'found' variable in gr_equal

Submitted by:	Christoph Mallon <christoph.mallon gmx.de>
---
 lib/libutil/gr_util.c | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

(limited to 'lib')

diff --git a/lib/libutil/gr_util.c b/lib/libutil/gr_util.c
index acb9767..5e35ebe 100644
--- a/lib/libutil/gr_util.c
+++ b/lib/libutil/gr_util.c
@@ -346,7 +346,6 @@ gr_equal(const struct group *gr1, const struct group *gr2)
 {
 	int gr1_ndx;
 	int gr2_ndx;
-	bool found;
 
 	/* Check that the non-member information is the same. */
 	if (gr1->gr_name == NULL || gr2->gr_name == NULL) {
@@ -367,17 +366,15 @@ gr_equal(const struct group *gr1, const struct group *gr2)
 		if (gr1->gr_mem != gr2->gr_mem)
 			return (false);
 	} else {
-		for (found = false, gr1_ndx = 0; gr1->gr_mem[gr1_ndx] != NULL;
-		    gr1_ndx++) {
-			for (gr2_ndx = 0; gr2->gr_mem[gr2_ndx] != NULL;
-			    gr2_ndx++)
+		for (gr1_ndx = 0; gr1->gr_mem[gr1_ndx] != NULL; gr1_ndx++) {
+			for (gr2_ndx = 0;; gr2_ndx++) {
+				if (gr2->gr_mem[gr2_ndx] == NULL)
+					return (false);
 				if (strcmp(gr1->gr_mem[gr1_ndx],
 				    gr2->gr_mem[gr2_ndx]) == 0) {
-					found = true;
 					break;
 				}
-			if (!found)
-				return (false);
+			}
 		}
 
 		/* Check that group2 does not have more members than group1. */
-- 
cgit v1.1


From c942285112a9fbe906033157e30ef8176d864337 Mon Sep 17 00:00:00 2001
From: mjg <mjg@FreeBSD.org>
Date: Sun, 13 Jan 2013 21:26:57 +0000
Subject: libutil: move group_line_format into the scop of its only user.

Submitted by:	Christoph Mallon <christoph.mallon gmx.de>
---
 lib/libutil/gr_util.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/libutil/gr_util.c b/lib/libutil/gr_util.c
index 5e35ebe..70a4c03 100644
--- a/lib/libutil/gr_util.c
+++ b/lib/libutil/gr_util.c
@@ -50,8 +50,6 @@ static char group_file[PATH_MAX];
 static char tempname[PATH_MAX];
 static int initialized;
 
-static const char group_line_format[] = "%s:%s:%ju:";
-
 /*
  * Initialize statics
  */
@@ -391,6 +389,7 @@ gr_equal(const struct group *gr1, const struct group *gr2)
 char *
 gr_make(const struct group *gr)
 {
+	const char *group_line_format = "%s:%s:%ju:";
 	char *line;
 	size_t line_size;
 	int ndx;
-- 
cgit v1.1


From fecf48554c4a48311d707e584ba242e1a1e8243f Mon Sep 17 00:00:00 2001
From: mjg <mjg@FreeBSD.org>
Date: Sun, 13 Jan 2013 21:28:47 +0000
Subject: libutil: utilize strsep instead of strcat in a loop in gr_make

Submitted by:	Christoph Mallon <christoph.mallon gmx.de>
---
 lib/libutil/gr_util.c | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

(limited to 'lib')

diff --git a/lib/libutil/gr_util.c b/lib/libutil/gr_util.c
index 70a4c03..451a4a4 100644
--- a/lib/libutil/gr_util.c
+++ b/lib/libutil/gr_util.c
@@ -390,7 +390,9 @@ char *
 gr_make(const struct group *gr)
 {
 	const char *group_line_format = "%s:%s:%ju:";
+	const char *sep;
 	char *line;
+	char *p;
 	size_t line_size;
 	int ndx;
 
@@ -405,16 +407,18 @@ gr_make(const struct group *gr)
 	}
 
 	/* Create the group line and fill it. */
-	if ((line = malloc(line_size)) == NULL)
+	if ((line = p = malloc(line_size)) == NULL)
 		return (NULL);
-	snprintf(line, line_size, group_line_format, gr->gr_name, gr->gr_passwd,
+	p += sprintf(p, group_line_format, gr->gr_name, gr->gr_passwd,
 	    (uintmax_t)gr->gr_gid);
-	if (gr->gr_mem != NULL)
+	if (gr->gr_mem != NULL) {
+		sep = "";
 		for (ndx = 0; gr->gr_mem[ndx] != NULL; ndx++) {
-			strcat(line, gr->gr_mem[ndx]);
-			if (gr->gr_mem[ndx + 1] != NULL)
-				strcat(line, ",");
+			p = stpcpy(p, sep);
+			p = stpcpy(p, gr->gr_mem[ndx]);
+			sep = ",";
 		}
+	}
 
 	return (line);
 }
-- 
cgit v1.1


From 89fb0f254358fa3cbe1f95560d01fbec42006840 Mon Sep 17 00:00:00 2001
From: mjg <mjg@FreeBSD.org>
Date: Sun, 13 Jan 2013 22:08:18 +0000
Subject: libutil: fix typo in comment for gr_fini.

Submitted by:	Christoph Mallon <christoph.mallon gmx.de>
---
 lib/libutil/gr_util.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libutil/gr_util.c b/lib/libutil/gr_util.c
index 451a4a4..3f7e199 100644
--- a/lib/libutil/gr_util.c
+++ b/lib/libutil/gr_util.c
@@ -316,7 +316,7 @@ gr_mkdb(void)
 }
 
 /*
- * Clean up. Preserver errno for the caller's convenience.
+ * Clean up. Preserves errno for the caller's convenience.
  */
 void
 gr_fini(void)
-- 
cgit v1.1


From 01fbf755f7ffdfa6fff78e7de2f20c60f11b2605 Mon Sep 17 00:00:00 2001
From: zont <zont@FreeBSD.org>
Date: Tue, 15 Jan 2013 14:09:08 +0000
Subject: - Use standard RETURN VALUES section.

Approved by:	kib (mentor)
MFC after:	1 week
---
 lib/libc/sys/chroot.2 | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/sys/chroot.2 b/lib/libc/sys/chroot.2
index 060d064..36d0364 100644
--- a/lib/libc/sys/chroot.2
+++ b/lib/libc/sys/chroot.2
@@ -92,12 +92,8 @@ system call.
 Any other value for
 .Ql kern.chroot_allow_open_directories
 will bypass the check for open directories
-.Pp
-Upon successful completion, a value of 0 is returned.
-Otherwise,
-a value of -1 is returned and
-.Va errno
-is set to indicate an error.
+.Sh RETURN VALUES
+.Rv -std
 .Sh ERRORS
 The
 .Fn chroot
-- 
cgit v1.1


From 3ade1f6acb35053936d191f505bbbf66b6761392 Mon Sep 17 00:00:00 2001
From: jilles <jilles@FreeBSD.org>
Date: Fri, 18 Jan 2013 23:08:40 +0000
Subject: libthr: Always use the threaded rtld lock implementation.

The threaded rtld lock implementation is faster even in the single-threaded
case because it postpones signal handlers via THR_CRITICAL_ENTER and
THR_CRITICAL_LEAVE instead of calling sigprocmask(2).

As a result, exception handling becomes faster in single-threaded
applications linked with libthr.

Reviewed by:	kib
---
 lib/libthr/thread/thr_init.c | 6 ++++++
 lib/libthr/thread/thr_kern.c | 5 -----
 2 files changed, 6 insertions(+), 5 deletions(-)

(limited to 'lib')

diff --git a/lib/libthr/thread/thr_init.c b/lib/libthr/thread/thr_init.c
index c29e8c3..937d83f 100644
--- a/lib/libthr/thread/thr_init.c
+++ b/lib/libthr/thread/thr_init.c
@@ -363,6 +363,12 @@ _libpthread_init(struct pthread *curthread)
 		_thr_signal_init();
 		if (_thread_event_mask & TD_CREATE)
 			_thr_report_creation(curthread, curthread);
+		/*
+		 * Always use our rtld lock implementation.
+		 * It is faster because it postpones signal handlers
+		 * instead of calling sigprocmask(2).
+		 */
+		_thr_rtld_init();
 	}
 }
 
diff --git a/lib/libthr/thread/thr_kern.c b/lib/libthr/thread/thr_kern.c
index 1e7cb51..b5a8358 100644
--- a/lib/libthr/thread/thr_kern.c
+++ b/lib/libthr/thread/thr_kern.c
@@ -57,11 +57,6 @@ _thr_setthreaded(int threaded)
 		return (0);
 
 	__isthreaded = threaded;
-	if (threaded != 0) {
-		_thr_rtld_init();
-	} else {
-		_thr_rtld_fini();
-	}
 	return (0);
 }
 
-- 
cgit v1.1


From f2ad512ac756b1b261350e839de0d10ff085688f Mon Sep 17 00:00:00 2001
From: andrew <andrew@FreeBSD.org>
Date: Sat, 19 Jan 2013 02:28:44 +0000
Subject: Add the __aeabi_*divmod functions to the compiler-rt build

---
 lib/libcompiler_rt/Makefile | 7 +++++++
 1 file changed, 7 insertions(+)

(limited to 'lib')

diff --git a/lib/libcompiler_rt/Makefile b/lib/libcompiler_rt/Makefile
index af1fcb6..72b55d8 100644
--- a/lib/libcompiler_rt/Makefile
+++ b/lib/libcompiler_rt/Makefile
@@ -181,6 +181,13 @@ SRCS+=	${file}.c
 . endif
 .endfor
 
+.if ${MACHINE_CPUARCH} == "arm" && ${MK_ARM_EABI} != "no"
+SRCS+= aeabi_idivmod.S \
+	aeabi_ldivmod.S \
+	aeabi_uidivmod.S \
+	aeabi_uldivmod.S
+.endif
+
 .if ${MACHINE_CPUARCH} != "mips"
 . if ${MK_INSTALLLIB} != "no"
 SYMLINKS+=libcompiler_rt.a ${LIBDIR}/libgcc.a
-- 
cgit v1.1


From c3e3cd888a40bf5ef4db38bead1b0b215e6483b6 Mon Sep 17 00:00:00 2001
From: andrew <andrew@FreeBSD.org>
Date: Sat, 19 Jan 2013 02:33:57 +0000
Subject: For ARM EABI we only need a subset of the quad functions, the rest
 are provided by libgcc.

---
 lib/libc/quad/Makefile.inc | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'lib')

diff --git a/lib/libc/quad/Makefile.inc b/lib/libc/quad/Makefile.inc
index 016fa34..9cef22e 100644
--- a/lib/libc/quad/Makefile.inc
+++ b/lib/libc/quad/Makefile.inc
@@ -8,6 +8,10 @@
 
 SRCS+=	cmpdi2.c divdi3.c moddi3.c qdivrem.c ucmpdi2.c udivdi3.c umoddi3.c
 
+.elif ${LIBC_ARCH} == "arm" && ${MK_ARM_EABI} != "no"
+
+SRCS+=	adddi3.c anddi3.c floatunsdidf.c iordi3.c lshldi3.c notdi2.c \
+	qdivrem.c subdi3.c xordi3.c
 .else
 
 SRCS+=	adddi3.c anddi3.c ashldi3.c ashrdi3.c cmpdi2.c divdi3.c fixdfdi.c \
-- 
cgit v1.1


From 388178c195ed8f369019ba43899e39da72e35e87 Mon Sep 17 00:00:00 2001
From: andrew <andrew@FreeBSD.org>
Date: Sat, 19 Jan 2013 03:47:18 +0000
Subject: There should have been a tab after SRCS+=

---
 lib/libcompiler_rt/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libcompiler_rt/Makefile b/lib/libcompiler_rt/Makefile
index 72b55d8..44df5e0 100644
--- a/lib/libcompiler_rt/Makefile
+++ b/lib/libcompiler_rt/Makefile
@@ -182,7 +182,7 @@ SRCS+=	${file}.c
 .endfor
 
 .if ${MACHINE_CPUARCH} == "arm" && ${MK_ARM_EABI} != "no"
-SRCS+= aeabi_idivmod.S \
+SRCS+=	aeabi_idivmod.S \
 	aeabi_ldivmod.S \
 	aeabi_uidivmod.S \
 	aeabi_uldivmod.S
-- 
cgit v1.1


From 109e426da134da0aa0a02bfa0f7bc2c02de562df Mon Sep 17 00:00:00 2001
From: andrew <andrew@FreeBSD.org>
Date: Sat, 19 Jan 2013 04:03:18 +0000
Subject: Update the syscall calling convention for ARM EABI. We store the
 syscall in r7 and use ip to store the old version of r7 as it is not
 guaranteed to be kept when calling a subroutine. The kernel will preserve the
 register across system calls.

---
 lib/libc/arm/SYS.h | 8 ++++++++
 1 file changed, 8 insertions(+)

(limited to 'lib')

diff --git a/lib/libc/arm/SYS.h b/lib/libc/arm/SYS.h
index 584c151..ed1a045 100644
--- a/lib/libc/arm/SYS.h
+++ b/lib/libc/arm/SYS.h
@@ -39,7 +39,15 @@
 #include <sys/syscall.h>
 #include <machine/swi.h>
 
+#ifdef __ARM_EABI__
+#define SYSTRAP(x)							\
+			mov ip, r7;					\
+			ldr r7, =SYS_ ## x;				\
+			swi 0;						\
+			mov r7, ip
+#else
 #define SYSTRAP(x)	swi 0 | SYS_ ## x
+#endif
 
 #define	CERROR		_C_LABEL(cerror)
 #define	CURBRK		_C_LABEL(curbrk)
-- 
cgit v1.1


From 3600c83b820e00959d61600e67e9dcb32ef6b518 Mon Sep 17 00:00:00 2001
From: andrew <andrew@FreeBSD.org>
Date: Sat, 19 Jan 2013 04:11:45 +0000
Subject: Use the compiler-rt version __{u,}divsi3 and __{u,}modsi3 on ARM EABI

---
 lib/libc/arm/Makefile.inc     |  6 ++++++
 lib/libc/arm/Symbol.map       |  4 ----
 lib/libc/arm/Symbol_oabi.map  | 16 ++++++++++++++++
 lib/libc/arm/gen/Makefile.inc |  6 +++++-
 lib/libcompiler_rt/Makefile   | 10 +++++++---
 5 files changed, 34 insertions(+), 8 deletions(-)
 create mode 100644 lib/libc/arm/Symbol_oabi.map

(limited to 'lib')

diff --git a/lib/libc/arm/Makefile.inc b/lib/libc/arm/Makefile.inc
index 4c72e56..dafc8d6 100644
--- a/lib/libc/arm/Makefile.inc
+++ b/lib/libc/arm/Makefile.inc
@@ -8,3 +8,9 @@ SOFTFLOAT_BITS=32
 # Long double is just double precision.
 MDSRCS+=machdep_ldisd.c
 SYM_MAPS+=${.CURDIR}/arm/Symbol.map
+
+.if ${MK_ARM_EABI} == "no"
+# This contains the symbols that were removed when moving to the ARM EABI
+SYM_MAPS+=${.CURDIR}/arm/Symbol_oabi.map
+.endif
+
diff --git a/lib/libc/arm/Symbol.map b/lib/libc/arm/Symbol.map
index 48f6747..1c83b0a 100644
--- a/lib/libc/arm/Symbol.map
+++ b/lib/libc/arm/Symbol.map
@@ -46,10 +46,6 @@ FBSDprivate_1.0 {
 	_set_tp;
 	__aeabi_read_tp;
 	___longjmp;
-	__umodsi3;
-	__modsi3;
-	__udivsi3;
-	__divsi3;
 	__makecontext;
 	__longjmp;
 	signalcontext;
diff --git a/lib/libc/arm/Symbol_oabi.map b/lib/libc/arm/Symbol_oabi.map
new file mode 100644
index 0000000..0c22e4a
--- /dev/null
+++ b/lib/libc/arm/Symbol_oabi.map
@@ -0,0 +1,16 @@
+/*
+ * $FreeBSD$
+ */
+
+/*
+ * This only needs to contain symbols that are not listed in
+ * symbol maps from other parts of libc (i.e., not found in
+ * stdlib/Symbol.map, string/Symbol.map, sys/Symbol.map, ...)
+ * and are not used in the ARM EABI.
+ */
+FBSDprivate_1.0 {
+	__umodsi3;
+	__modsi3;
+	__udivsi3;
+	__divsi3;
+};
diff --git a/lib/libc/arm/gen/Makefile.inc b/lib/libc/arm/gen/Makefile.inc
index a78cbb9..ec36d45 100644
--- a/lib/libc/arm/gen/Makefile.inc
+++ b/lib/libc/arm/gen/Makefile.inc
@@ -3,4 +3,8 @@
 
 SRCS+=	_ctx_start.S _setjmp.S _set_tp.c alloca.S fabs.c \
 	getcontextx.c infinity.c ldexp.c makecontext.c \
-	__aeabi_read_tp.S setjmp.S signalcontext.c sigsetjmp.S divsi3.S flt_rounds.c
+	__aeabi_read_tp.S setjmp.S signalcontext.c sigsetjmp.S flt_rounds.c
+
+.if ${MK_ARM_EABI} == "no"
+SRCS+=	divsi3.S
+.endif
diff --git a/lib/libcompiler_rt/Makefile b/lib/libcompiler_rt/Makefile
index 44df5e0..26ce4f7a 100644
--- a/lib/libcompiler_rt/Makefile
+++ b/lib/libcompiler_rt/Makefile
@@ -136,18 +136,22 @@ SRCF+=	adddf3 \
 	addsf3 \
 	divdf3 \
 	divsf3 \
-	divsi3 \
 	extendsfdf2 \
 	fixdfsi \
 	fixsfsi \
 	floatsidf \
 	floatsisf \
-	modsi3 \
 	muldf3 \
 	mulsf3 \
 	subdf3 \
 	subsf3 \
-	truncdfsf2 \
+	truncdfsf2
+.endif
+
+.if ${MACHINE_CPUARCH} != "mips" && \
+    (${MACHINE_CPUARCH} != "arm" || ${MK_ARM_EABI} != "no")
+SRCF+=	divsi3 \
+	modsi3 \
 	udivsi3 \
 	umodsi3
 .endif
-- 
cgit v1.1


From ee70ac6d85ad5c8eafb0a4630acfe22119788f5b Mon Sep 17 00:00:00 2001
From: andrew <andrew@FreeBSD.org>
Date: Sat, 19 Jan 2013 05:33:55 +0000
Subject: Add the required __aeabi_* functions to libc.

The floating point functions are here rather than compiler-rt because the
libc softfloat code allows us to set the rounding mode.
---
 lib/libc/arm/Makefile.inc             |   2 +
 lib/libc/arm/aeabi/Makefile.inc       |  11 ++++
 lib/libc/arm/aeabi/Symbol.map         |  47 ++++++++++++++++
 lib/libc/arm/aeabi/aeabi_atexit.c     |  38 +++++++++++++
 lib/libc/arm/aeabi/aeabi_double.c     | 101 ++++++++++++++++++++++++++++++++++
 lib/libc/arm/aeabi/aeabi_float.c      | 101 ++++++++++++++++++++++++++++++++++
 lib/libc/arm/aeabi/aeabi_unwind_cpp.c |  61 ++++++++++++++++++++
 7 files changed, 361 insertions(+)
 create mode 100644 lib/libc/arm/aeabi/Makefile.inc
 create mode 100644 lib/libc/arm/aeabi/Symbol.map
 create mode 100644 lib/libc/arm/aeabi/aeabi_atexit.c
 create mode 100644 lib/libc/arm/aeabi/aeabi_double.c
 create mode 100644 lib/libc/arm/aeabi/aeabi_float.c
 create mode 100644 lib/libc/arm/aeabi/aeabi_unwind_cpp.c

(limited to 'lib')

diff --git a/lib/libc/arm/Makefile.inc b/lib/libc/arm/Makefile.inc
index dafc8d6..1ae1298 100644
--- a/lib/libc/arm/Makefile.inc
+++ b/lib/libc/arm/Makefile.inc
@@ -12,5 +12,7 @@ SYM_MAPS+=${.CURDIR}/arm/Symbol.map
 .if ${MK_ARM_EABI} == "no"
 # This contains the symbols that were removed when moving to the ARM EABI
 SYM_MAPS+=${.CURDIR}/arm/Symbol_oabi.map
+.else
+.include "${.CURDIR}/arm/aeabi/Makefile.inc"
 .endif
 
diff --git a/lib/libc/arm/aeabi/Makefile.inc b/lib/libc/arm/aeabi/Makefile.inc
new file mode 100644
index 0000000..ac147c0
--- /dev/null
+++ b/lib/libc/arm/aeabi/Makefile.inc
@@ -0,0 +1,11 @@
+# $FreeBSD$
+
+.PATH: ${.CURDIR}/arm/aeabi
+
+SRCS+=	aeabi_atexit.c		\
+	aeabi_double.c		\
+	aeabi_float.c		\
+	aeabi_unwind_cpp.c
+
+SYM_MAPS+=${.CURDIR}/arm/aeabi/Symbol.map
+
diff --git a/lib/libc/arm/aeabi/Symbol.map b/lib/libc/arm/aeabi/Symbol.map
new file mode 100644
index 0000000..9493427
--- /dev/null
+++ b/lib/libc/arm/aeabi/Symbol.map
@@ -0,0 +1,47 @@
+/*
+ * $FreeBSD$
+ */
+
+/*
+ * This only needs to contain AEABI symbols that are not listed in
+ * symbol maps from other parts of libc (i.e., not found in
+ * stdlib/Symbol.map, string/Symbol.map, sys/Symbol.map, ...).
+ */
+FBSDprivate_1.0 {
+	__aeabi_atexit;
+
+	__aeabi_dcmpeq;
+	__aeabi_dcmplt;
+	__aeabi_dcmple;
+	__aeabi_dcmpge;
+	__aeabi_dcmpgt;
+	__aeabi_dcmpun;
+
+	__aeabi_d2iz;
+	__aeabi_d2f;
+
+	__aeabi_dadd;
+	__aeabi_ddiv;
+	__aeabi_dmul;
+	__aeabi_dsub;
+
+
+	__aeabi_fcmpeq;
+	__aeabi_fcmplt;
+	__aeabi_fcmple;
+	__aeabi_fcmpge;
+	__aeabi_fcmpgt;
+	__aeabi_fcmpun;
+
+	__aeabi_f2iz;
+	__aeabi_f2d;
+
+	__aeabi_fadd;
+	__aeabi_fdiv;
+	__aeabi_fmul;
+	__aeabi_fsub;
+
+
+	__aeabi_i2d;
+	__aeabi_i2f;
+};
diff --git a/lib/libc/arm/aeabi/aeabi_atexit.c b/lib/libc/arm/aeabi/aeabi_atexit.c
new file mode 100644
index 0000000..8502595
--- /dev/null
+++ b/lib/libc/arm/aeabi/aeabi_atexit.c
@@ -0,0 +1,38 @@
+/*
+ * Copyright (C) 2012 Andrew Turner
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+int __cxa_atexit(void (*)(void *), void *, void *);
+
+int
+__aeabi_atexit(void *object, void (*func)(void*), void *dso)
+{
+	return __cxa_atexit(func, object, dso);
+}
+
diff --git a/lib/libc/arm/aeabi/aeabi_double.c b/lib/libc/arm/aeabi/aeabi_double.c
new file mode 100644
index 0000000..5f9065c
--- /dev/null
+++ b/lib/libc/arm/aeabi/aeabi_double.c
@@ -0,0 +1,101 @@
+/*
+ * Copyright (C) 2012 Andrew Turner
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "softfloat-for-gcc.h"
+#include "milieu.h"
+#include "softfloat.h"
+
+flag __unorddf2(float64, float64);
+
+int __aeabi_dcmpeq(float64 a, float64 b)
+{
+	return float64_eq(a, b);
+}
+
+int __aeabi_dcmplt(float64 a, float64 b)
+{
+	return float64_lt(a, b);
+}
+
+int __aeabi_dcmple(float64 a, float64 b)
+{
+	return float64_le(a, b);
+}
+
+int __aeabi_dcmpge(float64 a, float64 b)
+{
+	return float64_le(b, a);
+}
+
+int __aeabi_dcmpgt(float64 a, float64 b)
+{
+	return float64_lt(b, a);
+}
+
+int __aeabi_dcmpun(float64 a, float64 b)
+{
+	return __unorddf2(a, b);
+}
+
+int __aeabi_d2iz(float64 a)
+{
+	return float64_to_int32_round_to_zero(a);
+}
+
+float32 __aeabi_d2f(float64 a)
+{
+	return float64_to_float32(a);
+}
+
+float64 __aeabi_i2d(int a)
+{
+	return int32_to_float64(a);
+}
+
+float64 __aeabi_dadd(float64 a, float64 b)
+{
+	return float64_add(a, b);
+}
+
+float64 __aeabi_ddiv(float64 a, float64 b)
+{
+	return float64_div(a, b);
+}
+
+float64 __aeabi_dmul(float64 a, float64 b)
+{
+	return float64_mul(a, b);
+}
+
+float64 __aeabi_dsub(float64 a, float64 b)
+{
+	return float64_sub(a, b);
+}
+
diff --git a/lib/libc/arm/aeabi/aeabi_float.c b/lib/libc/arm/aeabi/aeabi_float.c
new file mode 100644
index 0000000..97751ad
--- /dev/null
+++ b/lib/libc/arm/aeabi/aeabi_float.c
@@ -0,0 +1,101 @@
+/*
+ * Copyright (C) 2012 Andrew Turner
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "softfloat-for-gcc.h"
+#include "milieu.h"
+#include "softfloat.h"
+
+flag __unordsf2(float32, float32);
+
+int __aeabi_fcmpeq(float32 a, float32 b)
+{
+	return float32_eq(a, b);
+}
+
+int __aeabi_fcmplt(float32 a, float32 b)
+{
+	return float32_lt(a, b);
+}
+
+int __aeabi_fcmple(float32 a, float32 b)
+{
+	return float32_le(a, b);
+}
+
+int __aeabi_fcmpge(float32 a, float32 b)
+{
+	return float32_le(b, a);
+}
+
+int __aeabi_fcmpgt(float32 a, float32 b)
+{
+	return float32_lt(b, a);
+}
+
+int __aeabi_fcmpun(float32 a, float32 b)
+{
+	return __unordsf2(a, b);
+}
+
+int __aeabi_f2iz(float32 a)
+{
+	return float32_to_int32_round_to_zero(a);
+}
+
+float32 __aeabi_f2d(float32 a)
+{
+	return float32_to_float64(a);
+}
+
+float32 __aeabi_i2f(int a)
+{
+	return int32_to_float32(a);
+}
+
+float32 __aeabi_fadd(float32 a, float32 b)
+{
+	return float32_add(a, b);
+}
+
+float32 __aeabi_fdiv(float32 a, float32 b)
+{
+	return float32_div(a, b);
+}
+
+float32 __aeabi_fmul(float32 a, float32 b)
+{
+	return float32_mul(a, b);
+}
+
+float32 __aeabi_fsub(float32 a, float32 b)
+{
+	return float32_sub(a, b);
+}
+
diff --git a/lib/libc/arm/aeabi/aeabi_unwind_cpp.c b/lib/libc/arm/aeabi/aeabi_unwind_cpp.c
new file mode 100644
index 0000000..c07cbbd
--- /dev/null
+++ b/lib/libc/arm/aeabi/aeabi_unwind_cpp.c
@@ -0,0 +1,61 @@
+/*
+ * Copyright (C) 2011 Andrew Turner
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ */
+
+/*
+ * Provide an implementation of __aeabi_unwind_cpp_pr{0,1,2}. These are
+ * required by libc but are implemented in libgcc_eh.a which we don't link
+ * against. The libgcc_eh.a version will be called so we call abort to
+ * check this.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <stdlib.h>
+
+void __aeabi_unwind_cpp_pr0(void) __hidden;
+void __aeabi_unwind_cpp_pr1(void) __hidden;
+void __aeabi_unwind_cpp_pr2(void) __hidden;
+
+void
+__aeabi_unwind_cpp_pr0(void)
+{
+	abort();
+}
+
+void
+__aeabi_unwind_cpp_pr1(void)
+{
+	abort();
+}
+
+void
+__aeabi_unwind_cpp_pr2(void)
+{
+	abort();
+}
+
-- 
cgit v1.1


From 5ab6899eb1159c7769dcc378b262f65de85d3d07 Mon Sep 17 00:00:00 2001
From: pjd <pjd@FreeBSD.org>
Date: Mon, 21 Jan 2013 22:07:34 +0000
Subject: Note that SIGCHLD is special and if ignored, won't be recorded by the
 filter.

---
 lib/libc/sys/kqueue.2 | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/sys/kqueue.2 b/lib/libc/sys/kqueue.2
index 63a662f..6ae7914 100644
--- a/lib/libc/sys/kqueue.2
+++ b/lib/libc/sys/kqueue.2
@@ -24,7 +24,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd February 15, 2012
+.Dd January 21, 2013
 .Dt KQUEUE 2
 .Os
 .Sh NAME
@@ -417,7 +417,9 @@ and
 facilities, and has a lower precedence.
 The filter will record
 all attempts to deliver a signal to a process, even if the signal has
-been marked as SIG_IGN.
+been marked as SIG_IGN, except for the
+.Dv SIGCHLD
+signal, which, if ignored, won't be recorded by the filter.
 Event notification happens after normal
 signal delivery processing.
 .Va data
-- 
cgit v1.1


From 107e0c39a85867841fc50b7f302276228d36d632 Mon Sep 17 00:00:00 2001
From: glebius <glebius@FreeBSD.org>
Date: Wed, 23 Jan 2013 10:50:52 +0000
Subject: posix_fadvise(2) first appeared in FreeBSD 9.1

---
 lib/libc/sys/posix_fadvise.2 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libc/sys/posix_fadvise.2 b/lib/libc/sys/posix_fadvise.2
index f303f42..7a9a648 100644
--- a/lib/libc/sys/posix_fadvise.2
+++ b/lib/libc/sys/posix_fadvise.2
@@ -134,4 +134,4 @@ interface conforms to
 The
 .Fn posix_fadvise
 system call first appeared in
-.Fx 10.0 .
+.Fx 9.1 .
-- 
cgit v1.1


From fd84d8cd1a8f0c396c9543101786d0aa454d9166 Mon Sep 17 00:00:00 2001
From: brooks <brooks@FreeBSD.org>
Date: Wed, 23 Jan 2013 23:53:26 +0000
Subject: Remove several MLINKS that were listed twice.

---
 lib/libarchive/Makefile | 7 -------
 1 file changed, 7 deletions(-)

(limited to 'lib')

diff --git a/lib/libarchive/Makefile b/lib/libarchive/Makefile
index 29508ad..73b3c22 100644
--- a/lib/libarchive/Makefile
+++ b/lib/libarchive/Makefile
@@ -231,9 +231,6 @@ MLINKS+=	archive_entry_perms.3 archive_entry_set_fflags.3
 MLINKS+=	archive_entry_perms.3 archive_entry_set_gid.3
 MLINKS+=	archive_entry_perms.3 archive_entry_set_gname.3
 MLINKS+=	archive_entry_perms.3 archive_entry_perm.3
-MLINKS+=	archive_entry_perms.3 archive_entry_set_fflags.3
-MLINKS+=	archive_entry_perms.3 archive_entry_set_gid.3
-MLINKS+=	archive_entry_perms.3 archive_entry_set_gname.3
 MLINKS+=	archive_entry_perms.3 archive_entry_set_perm.3
 MLINKS+=	archive_entry_perms.3 archive_entry_set_uid.3
 MLINKS+=	archive_entry_perms.3 archive_entry_set_uname.3
@@ -264,10 +261,6 @@ MLINKS+=	archive_entry_stat.3 archive_entry_set_filetype.3
 MLINKS+=	archive_entry_stat.3 archive_entry_set_ino.3
 MLINKS+=	archive_entry_stat.3 archive_entry_set_ino64.3
 MLINKS+=	archive_entry_stat.3 archive_entry_set_mode.3
-MLINKS+=	archive_entry_stat.3 archive_entry_set_rdev.3
-MLINKS+=	archive_entry_stat.3 archive_entry_set_rdevmajor.3
-MLINKS+=	archive_entry_stat.3 archive_entry_set_rdevminor.3
-MLINKS+=	archive_entry_stat.3 archive_entry_set_size.3
 MLINKS+=	archive_entry_stat.3 archive_entry_set_nlink.3
 MLINKS+=	archive_entry_stat.3 archive_entry_set_rdev.3
 MLINKS+=	archive_entry_stat.3 archive_entry_set_rdevmajor.3
-- 
cgit v1.1


From 28f52e1e5a8f19f88ebe5d7325aa92d250abdcea Mon Sep 17 00:00:00 2001
From: brooks <brooks@FreeBSD.org>
Date: Thu, 24 Jan 2013 16:38:47 +0000
Subject: Only install manpages and html documentation in the ncurses/*w
 (wchar) builds so that it is only installed once.  This is consistent with
 the existing decision to only install headers in the that case.

---
 lib/ncurses/form/Makefile    | 4 +++-
 lib/ncurses/menu/Makefile    | 4 +++-
 lib/ncurses/ncurses/Makefile | 8 +++++++-
 lib/ncurses/panel/Makefile   | 4 +++-
 4 files changed, 16 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/ncurses/form/Makefile b/lib/ncurses/form/Makefile
index fe89d45..d05cbfe 100644
--- a/lib/ncurses/form/Makefile
+++ b/lib/ncurses/form/Makefile
@@ -59,7 +59,6 @@ LDADD=	-lncurses${LIB_SUFFIX}
 
 .if defined(ENABLE_WIDEC)
 INCS=	form.h
-.endif
 
 .PATH: ${NCURSES_DIR}/man
 MAN=	\
@@ -157,6 +156,9 @@ MLINKS=	form_cursor.3 pos_form_cursor.3 \
 	form_win.3 scale_form.3 \
 	form_win.3 set_form_sub.3 \
 	form_win.3 set_form_win.3
+.else
+NO_MAN=
+.endif
 
 .include <bsd.lib.mk>
 
diff --git a/lib/ncurses/menu/Makefile b/lib/ncurses/menu/Makefile
index a853d8c..19a1ff3 100644
--- a/lib/ncurses/menu/Makefile
+++ b/lib/ncurses/menu/Makefile
@@ -45,7 +45,6 @@ LDADD=	-lncurses${LIB_SUFFIX}
 
 .if defined(ENABLE_WIDEC)
 INCS=	menu.h eti.h
-.endif
 
 .PATH: ${NCURSES_DIR}/man
 MAN=	\
@@ -130,6 +129,9 @@ MLINKS=	menu_attributes.3 menu_back.3 \
 	mitem_value.3 item_value.3 \
 	mitem_value.3 set_item_value.3 \
 	mitem_visible.3 item_visible.3
+.else
+NO_MAN=
+.endif
 
 .include <bsd.lib.mk>
 
diff --git a/lib/ncurses/ncurses/Makefile b/lib/ncurses/ncurses/Makefile
index bdac622..869fb2e 100644
--- a/lib/ncurses/ncurses/Makefile
+++ b/lib/ncurses/ncurses/Makefile
@@ -2,6 +2,10 @@
 
 SHLIBDIR?=	/lib
 
+.if !defined(ENABLE_WIDEC)
+NO_MAN=
+.endif
+
 .include <bsd.own.mk>
 
 .include "${.CURDIR}/../config.mk"
@@ -304,8 +308,10 @@ SYMLINKS+=	libncurses${LIB_SUFFIX}_p.a ${LIBDIR}/libtermlib${LIB_SUFFIX}_p.a
 SYMLINKS+=	libncurses${LIB_SUFFIX}_p.a ${LIBDIR}/libtinfo${LIB_SUFFIX}_p.a
 .endif
 
+.if defined(ENABLE_WIDEC)
 DOCSDIR=	${SHAREDIR}/doc/ncurses
 DOCS=		ncurses-intro.html hackguide.html
+.endif
 
 .if ${MK_HTML} != "no"
 .PATH: ${NCURSES_DIR}/doc/html
@@ -457,6 +463,7 @@ terminfo.5: MKterminfo.sh terminfo.head Caps
 
 CLEANFILES+=	terminfo.5
 
+.if defined(ENABLE_WIDEC)
 .PATH: ${NCURSES_DIR}/man
 MAN=	\
 	curs_addch.3 \
@@ -517,7 +524,6 @@ MAN=	\
 	resizeterm.3 \
 	wresize.3
 
-.if defined(ENABLE_WIDEC)
 MAN+=	\
 	curs_add_wch.3 \
 	curs_add_wchstr.3 \
diff --git a/lib/ncurses/panel/Makefile b/lib/ncurses/panel/Makefile
index 12b8b06..689586d 100644
--- a/lib/ncurses/panel/Makefile
+++ b/lib/ncurses/panel/Makefile
@@ -34,7 +34,6 @@ LDADD=	-lncurses${LIB_SUFFIX}
 
 .if defined(ENABLE_WIDEC)
 INCS=	panel.h
-.endif
 
 # generate MAN
 .PATH: ${NCURSES_DIR}/man
@@ -58,6 +57,9 @@ MLINKS=	panel.3 bottom_panel.3 \
 	panel.3 show_panel.3 \
 	panel.3 top_panel.3 \
 	panel.3 update_panels.3
+.else
+NO_MAN=
+.endif
 	
 .include <bsd.lib.mk>
 
-- 
cgit v1.1


From b923a5521baf7a53ffa6239bb920ea3b7d10e780 Mon Sep 17 00:00:00 2001
From: theraven <theraven@FreeBSD.org>
Date: Mon, 28 Jan 2013 11:12:49 +0000
Subject: Fix some symbol version mismatches between libstdc++ and
 libsupc++/libcxxrt that were causing the runtime and STL libraries to see
 different versions of various classes and functions when libstdc++ is used as
 a filter.

Note: This changes the ABI for libcxxrt, but libcxxrt is currently only in
-STABLE for testing and is not used by anything unless explicitly enabled by
the end user.  No default compiler configurations use it.

libc++ will need to be recompiled after this change.  make buildworld will do
this automatically, but make in lib/libc++ will not necessarily work unless the
new libcxxrt is installed first.

PR:		kern/171610, stand/175453
Reviewed by:	kib
MFC after:	1 week
---
 lib/libcxxrt/Version.map | 102 +++++++++++++++++++++--------------------------
 1 file changed, 46 insertions(+), 56 deletions(-)

(limited to 'lib')

diff --git a/lib/libcxxrt/Version.map b/lib/libcxxrt/Version.map
index ca990a0..1fdda27 100644
--- a/lib/libcxxrt/Version.map
+++ b/lib/libcxxrt/Version.map
@@ -209,18 +209,7 @@ CXXABI_1.3 {
 
         "std::type_info::type_info(std::type_info const&)";
         "std::type_info::type_info(std::type_info const&)";
-        "std::type_info::~type_info()";
-        "std::type_info::~type_info()";
-        "std::type_info::~type_info()";
         "std::type_info::operator=(std::type_info const&)";
-        "std::unexpected()";
-        "std::get_terminate()";
-        "std::set_terminate(void (*)())";
-        "std::get_unexpected()";
-        "std::set_unexpected(void (*)())";
-        "std::set_new_handler(void (*)())";
-        "std::uncaught_exception()";
-        "std::terminate()";
 
 
         # Extensions
@@ -243,67 +232,22 @@ CXXABI_1.3.1 {
 CXXRT_1.0 {
 
     extern "C++" {
-        "std::bad_cast::what() const";
-        "std::bad_typeid::what() const";
-        "std::bad_alloc::what() const";
-        "std::exception::what() const";
         "std::type_info::name() const";
         "std::type_info::before(std::type_info const&) const";
         "std::type_info::operator==(std::type_info const&) const";
         "std::type_info::operator!=(std::type_info const&) const";
-        "std::bad_typeid::bad_typeid(std::bad_typeid const&)";
-        "std::bad_typeid::bad_typeid()";
-        "std::bad_typeid::bad_typeid(std::bad_typeid const&)";
-        "std::bad_typeid::bad_typeid()";
-        "std::bad_typeid::~bad_typeid()";
-        "std::bad_typeid::~bad_typeid()";
-        "std::bad_typeid::~bad_typeid()";
-        "std::bad_typeid::operator=(std::bad_typeid const&)";
         "std::bad_cast::bad_cast(std::bad_cast const&)";
         "std::bad_cast::bad_cast()";
         "std::bad_cast::bad_cast(std::bad_cast const&)";
         "std::bad_cast::bad_cast()";
-        "std::bad_cast::~bad_cast()";
-        "std::bad_cast::~bad_cast()";
-        "std::bad_cast::~bad_cast()";
         "std::bad_cast::operator=(std::bad_cast const&)";
-        "std::bad_alloc::bad_alloc(std::bad_alloc const&)";
-        "std::bad_alloc::bad_alloc()";
-        "std::bad_alloc::bad_alloc(std::bad_alloc const&)";
-        "std::bad_alloc::bad_alloc()";
-        "std::bad_alloc::~bad_alloc()";
-        "std::bad_alloc::~bad_alloc()";
-        "std::bad_alloc::~bad_alloc()";
-        "std::bad_alloc::operator=(std::bad_alloc const&)";
         "std::exception::exception(std::exception const&)";
         "std::exception::exception()";
         "std::exception::exception(std::exception const&)";
         "std::exception::exception()";
-        "std::exception::~exception()";
-        "std::exception::~exception()";
-        "std::exception::~exception()";
         "std::exception::operator=(std::exception const&)";
 
 
-        "vtable for std::bad_typeid";
-        "vtable for std::bad_cast";
-        "vtable for std::bad_alloc";
-        "vtable for std::exception";
-        "vtable for std::type_info";
-        "typeinfo for std::bad_typeid";
-        "typeinfo for std::bad_cast";
-        "typeinfo for std::bad_alloc";
-        "typeinfo for std::exception";
-        "typeinfo for std::type_info";
-        "typeinfo name for std::bad_typeid";
-        "typeinfo name for std::bad_cast";
-        "typeinfo name for std::bad_alloc";
-        "typeinfo name for std::exception";
-        "typeinfo name for std::type_info";
-
-        "std::type_info::__is_function_p() const";
-        "std::type_info::__do_upcast(__cxxabiv1::__class_type_info const*, void**) const";
-        "std::type_info::__is_pointer_p() const";
 
 
 
@@ -317,6 +261,7 @@ CXXRT_1.0 {
 
 } CXXABI_1.3.1;
 
+
 GLIBCXX_3.4 {
     extern "C++" {
         "operator delete[](void*)";
@@ -327,5 +272,50 @@ GLIBCXX_3.4 {
         "operator new[](unsigned long)";
         "operator new(unsigned long)";
         "operator new(unsigned long, std::nothrow_t const&)";
+
+        "std::unexpected()";
+        "std::get_terminate()";
+        "std::get_unexpected()";
+        "std::uncaught_exception()";
+        "std::terminate()";
+
+        "std::type_info::~type_info()";
+        "std::bad_cast::~bad_cast()";
+        "std::exception::~exception()";
+
+        std::set_new_handler*;
+        std::set_terminate*;
+        std::set_unexpected*;
+        std::exception*;
+        std::bad_alloc;
+        std::bad_typeid;
+        std::type_info*;
+
+        "vtable for std::bad_alloc";
+        "vtable for std::bad_cast";
+        "vtable for std::bad_typeid";
+        "vtable for std::exception";
+        "vtable for std::type_info";
+
+        "typeinfo for std::bad_alloc";
+        "typeinfo for std::bad_typeid";
+        "typeinfo for std::exception";
+        "typeinfo for std::bad_cast";
+        "typeinfo for std::exception";
+        "typeinfo for std::type_info";
+        "typeinfo name for std::bad_typeid";
+        "typeinfo name for std::bad_cast";
+        "typeinfo name for std::exception";
+        "typeinfo name for std::type_info";
+
     };
 };
+
+GLIBCXX_3.4.9 {
+    extern "C++" {
+        "std::bad_typeid::what() const";
+        "std::bad_cast::what() const";
+        "std::bad_alloc::what() const";
+    };
+} GLIBCXX_3.4;
+
-- 
cgit v1.1


From 7756bef786978bd5d07c05b55bc8428454d6837d Mon Sep 17 00:00:00 2001
From: jhb <jhb@FreeBSD.org>
Date: Mon, 28 Jan 2013 15:48:31 +0000
Subject: - Compute the correct size to reallocate when doubling the size of
 the   array of loaded objects to avoid a buffer overrun. - Use reallocf() to
 avoid leaking memory if the realloc() fails.

PR:		kern/175648
Submitted by:	yuri@rawbw.com (1)
MFC after:	1 week
---
 lib/libproc/proc_rtld.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/libproc/proc_rtld.c b/lib/libproc/proc_rtld.c
index 2a9ed39..5259049 100644
--- a/lib/libproc/proc_rtld.c
+++ b/lib/libproc/proc_rtld.c
@@ -44,7 +44,8 @@ map_iter(const rd_loadobj_t *lop, void *arg)
 
 	if (phdl->nobjs >= phdl->rdobjsz) {
 		phdl->rdobjsz *= 2;
-		phdl->rdobjs = realloc(phdl->rdobjs, phdl->rdobjsz);
+		phdl->rdobjs = reallocf(phdl->rdobjs, sizeof(*phdl->rdobjs) *
+		    phdl->rdobjsz);
 		if (phdl->rdobjs == NULL)
 			return (-1);
 	}
-- 
cgit v1.1


From 2ae2587f8343b7f0559fbe2ebfd6d62814451c37 Mon Sep 17 00:00:00 2001
From: glebius <glebius@FreeBSD.org>
Date: Tue, 29 Jan 2013 21:37:56 +0000
Subject: Remove unused file.

---
 lib/libstand/if_ether.h | 261 ------------------------------------------------
 1 file changed, 261 deletions(-)
 delete mode 100644 lib/libstand/if_ether.h

(limited to 'lib')

diff --git a/lib/libstand/if_ether.h b/lib/libstand/if_ether.h
deleted file mode 100644
index 373f95a..0000000
--- a/lib/libstand/if_ether.h
+++ /dev/null
@@ -1,261 +0,0 @@
-/*	$NetBSD: if_ether.h,v 1.25 1997/01/17 17:06:06 mikel Exp $	*/
-
-/*
- * Copyright (c) 1982, 1986, 1993
- *	The Regents of the University of California.  All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 4. Neither the name of the University nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- *	@(#)if_ether.h	8.1 (Berkeley) 6/10/93
- *
- * $FreeBSD$
- */
-
-/*
- * Ethernet address - 6 octets
- * this is only used by the ethers(3) functions.
- */
-struct ether_addr {
-	u_int8_t ether_addr_octet[6];
-};
-
-/*
- * Structure of a 10Mb/s Ethernet header.
- */
-#define	ETHER_ADDR_LEN	6
-
-struct	ether_header {
-	u_int8_t  ether_dhost[ETHER_ADDR_LEN];
-	u_int8_t  ether_shost[ETHER_ADDR_LEN];
-	u_int16_t ether_type;
-};
-
-#define	ETHERTYPE_PUP		0x0200	/* PUP protocol */
-#define	ETHERTYPE_IP		0x0800	/* IP protocol */
-#define	ETHERTYPE_ARP		0x0806	/* address resolution protocol */
-#define	ETHERTYPE_REVARP	0x8035	/* reverse addr resolution protocol */
-
-/*
- * The ETHERTYPE_NTRAILER packet types starting at ETHERTYPE_TRAIL have
- * (type-ETHERTYPE_TRAIL)*512 bytes of data followed
- * by an ETHER type (as given above) and then the (variable-length) header.
- */
-#define	ETHERTYPE_TRAIL		0x1000		/* Trailer packet */
-#define	ETHERTYPE_NTRAILER	16
-
-#define	ETHER_IS_MULTICAST(addr) (*(addr) & 0x01) /* is address mcast/bcast? */
-
-#define	ETHERMTU	1500
-#define	ETHERMIN	(60-14)
-
-#ifdef _KERNEL
-/*
- * Macro to map an IP multicast address to an Ethernet multicast address.
- * The high-order 25 bits of the Ethernet address are statically assigned,
- * and the low-order 23 bits are taken from the low end of the IP address.
- */
-#define ETHER_MAP_IP_MULTICAST(ipaddr, enaddr)				\
-	/* struct in_addr *ipaddr; */					\
-	/* u_int8_t enaddr[ETHER_ADDR_LEN]; */				\
-{									\
-	(enaddr)[0] = 0x01;						\
-	(enaddr)[1] = 0x00;						\
-	(enaddr)[2] = 0x5e;						\
-	(enaddr)[3] = ((u_int8_t *)ipaddr)[1] & 0x7f;			\
-	(enaddr)[4] = ((u_int8_t *)ipaddr)[2];				\
-	(enaddr)[5] = ((u_int8_t *)ipaddr)[3];				\
-}
-#endif
-
-/*
- * Ethernet Address Resolution Protocol.
- *
- * See RFC 826 for protocol description.  Structure below is adapted
- * to resolving internet addresses.  Field names used correspond to 
- * RFC 826.
- */
-struct	ether_arp {
-	struct	 arphdr ea_hdr;			/* fixed-size header */
-	u_int8_t arp_sha[ETHER_ADDR_LEN];	/* sender hardware address */
-	u_int8_t arp_spa[4];			/* sender protocol address */
-	u_int8_t arp_tha[ETHER_ADDR_LEN];	/* target hardware address */
-	u_int8_t arp_tpa[4];			/* target protocol address */
-};
-#define	arp_hrd	ea_hdr.ar_hrd
-#define	arp_pro	ea_hdr.ar_pro
-#define	arp_hln	ea_hdr.ar_hln
-#define	arp_pln	ea_hdr.ar_pln
-#define	arp_op	ea_hdr.ar_op
-
-/*
- * Structure shared between the ethernet driver modules and
- * the address resolution code.  For example, each ec_softc or il_softc
- * begins with this structure.
- */
-struct	arpcom {
-	struct	 ifnet ac_if;			/* network-visible interface */
-	u_int8_t ac_enaddr[ETHER_ADDR_LEN];	/* ethernet hardware address */
-	char	 ac__pad[2];			/* be nice to m68k ports */
-	LIST_HEAD(, ether_multi) ac_multiaddrs;	/* list of ether multicast addrs */
-	int	 ac_multicnt;			/* length of ac_multiaddrs list */
-};
-
-struct llinfo_arp {
-	LIST_ENTRY(llinfo_arp) la_list;
-	struct	rtentry *la_rt;
-	struct	mbuf *la_hold;		/* last packet until resolved/timeout */
-	long	la_asked;		/* last time we QUERIED for this addr */
-#define la_timer la_rt->rt_rmx.rmx_expire /* deletion time in seconds */
-};
-
-struct sockaddr_inarp {
-	u_int8_t  sin_len;
-	u_int8_t  sin_family;
-	u_int16_t sin_port;
-	struct	  in_addr sin_addr;
-	struct	  in_addr sin_srcaddr;
-	u_int16_t sin_tos;
-	u_int16_t sin_other;
-#define SIN_PROXY 1
-};
-
-/*
- * IP and ethernet specific routing flags
- */
-#define	RTF_USETRAILERS	RTF_PROTO1	/* use trailers */
-#define	RTF_ANNOUNCE	RTF_PROTO2	/* announce new arp entry */
-
-#ifdef	_KERNEL
-u_int8_t etherbroadcastaddr[ETHER_ADDR_LEN];
-u_int8_t ether_ipmulticast_min[ETHER_ADDR_LEN];
-u_int8_t ether_ipmulticast_max[ETHER_ADDR_LEN];
-struct	ifqueue arpintrq;
-
-void	arpwhohas(struct arpcom *, struct in_addr *);
-void	arpintr(void);
-int	arpresolve(struct arpcom *,
-	    struct rtentry *, struct mbuf *, struct sockaddr *, u_char *, struct llentry **);
-void	arp_ifinit(struct arpcom *, struct ifaddr *);
-void	arp_rtrequest(int, struct rtentry *, struct sockaddr *);
-
-int	ether_addmulti(struct ifreq *, struct arpcom *);
-int	ether_delmulti(struct ifreq *, struct arpcom *);
-#endif /* _KERNEL */
-
-/*
- * Ethernet multicast address structure.  There is one of these for each
- * multicast address or range of multicast addresses that we are supposed
- * to listen to on a particular interface.  They are kept in a linked list,
- * rooted in the interface's arpcom structure.  (This really has nothing to
- * do with ARP, or with the Internet address family, but this appears to be
- * the minimally-disrupting place to put it.)
- */
-struct ether_multi {
-	u_int8_t enm_addrlo[ETHER_ADDR_LEN]; /* low  or only address of range */
-	u_int8_t enm_addrhi[ETHER_ADDR_LEN]; /* high or only address of range */
-	struct	 arpcom *enm_ac;	/* back pointer to arpcom */
-	u_int	 enm_refcount;		/* no. claims to this addr/range */
-	LIST_ENTRY(ether_multi) enm_list;
-};
-
-/*
- * Structure used by macros below to remember position when stepping through
- * all of the ether_multi records.
- */
-struct ether_multistep {
-	struct ether_multi  *e_enm;
-};
-
-/*
- * Macro for looking up the ether_multi record for a given range of Ethernet
- * multicast addresses connected to a given arpcom structure.  If no matching
- * record is found, "enm" returns NULL.
- */
-#define ETHER_LOOKUP_MULTI(addrlo, addrhi, ac, enm)			\
-	/* u_int8_t addrlo[ETHER_ADDR_LEN]; */				\
-	/* u_int8_t addrhi[ETHER_ADDR_LEN]; */				\
-	/* struct arpcom *ac; */					\
-	/* struct ether_multi *enm; */					\
-{									\
-	for ((enm) = (ac)->ac_multiaddrs.lh_first;			\
-	    (enm) != NULL &&						\
-	    (bcmp((enm)->enm_addrlo, (addrlo), ETHER_ADDR_LEN) != 0 ||	\
-	     bcmp((enm)->enm_addrhi, (addrhi), ETHER_ADDR_LEN) != 0);	\
-		(enm) = (enm)->enm_list.le_next);			\
-}
-
-/*
- * Macro to step through all of the ether_multi records, one at a time.
- * The current position is remembered in "step", which the caller must
- * provide.  ETHER_FIRST_MULTI(), below, must be called to initialize "step"
- * and get the first record.  Both macros return a NULL "enm" when there
- * are no remaining records.
- */
-#define ETHER_NEXT_MULTI(step, enm) \
-	/* struct ether_multistep step; */  \
-	/* struct ether_multi *enm; */  \
-{ \
-	if (((enm) = (step).e_enm) != NULL) \
-		(step).e_enm = (enm)->enm_list.le_next; \
-}
-
-#define ETHER_FIRST_MULTI(step, ac, enm) \
-	/* struct ether_multistep step; */ \
-	/* struct arpcom *ac; */ \
-	/* struct ether_multi *enm; */ \
-{ \
-	(step).e_enm = (ac)->ac_multiaddrs.lh_first; \
-	ETHER_NEXT_MULTI((step), (enm)); \
-}
-
-#ifdef _KERNEL
-void arp_rtrequest(int, struct rtentry *, struct sockaddr *);
-int arpresolve(struct arpcom *, struct rtentry *, struct mbuf *,
-		    struct sockaddr *, u_char *, struct llentry **);
-void arpintr(void);
-int arpioctl(u_long, caddr_t);
-void arp_ifinit(struct arpcom *, struct ifaddr *);
-void revarpinput(struct mbuf *);
-void in_revarpinput(struct mbuf *);
-void revarprequest(struct ifnet *);
-int revarpwhoarewe(struct ifnet *, struct in_addr *, struct in_addr *);
-int revarpwhoami(struct in_addr *, struct ifnet *);
-int db_show_arptab(void);
-#endif
-
-/*
- * Prototype ethers(3) functions.
- */
-#ifndef _KERNEL
-#include <sys/cdefs.h>
-__BEGIN_DECLS
-char *	ether_ntoa(struct ether_addr *);
-struct ether_addr *
-	ether_aton(char *);
-int	ether_ntohost(char *, struct ether_addr *);
-int	ether_hostton(char *, struct ether_addr *);
-int	ether_line(char *, struct ether_addr *, char *);
-__END_DECLS
-#endif
-- 
cgit v1.1


From 3913be8cf6bae0deea4548df2c3655b08fc13966 Mon Sep 17 00:00:00 2001
From: kib <kib@FreeBSD.org>
Date: Wed, 30 Jan 2013 12:48:16 +0000
Subject: Rework the __vdso_* symbols attributes to only make the symbols weak,
 but use normal references instead of weak.  This makes the statically linked
 binaries to use fast gettimeofday(2) by forcing the linker to resolve
 references and providing the neccessary functions.

Reported by:	bde
Tested by:	marius (sparc64)
MFC after:	2 weeks
---
 lib/libc/amd64/sys/__vdso_gettc.c     | 11 ++++++++
 lib/libc/arm/sys/Makefile.inc         |  2 ++
 lib/libc/arm/sys/__vdso_gettc.c       | 48 +++++++++++++++++++++++++++++++++++
 lib/libc/i386/sys/__vdso_gettc.c      | 10 ++++++++
 lib/libc/ia64/sys/Makefile.inc        |  2 ++
 lib/libc/ia64/sys/__vdso_gettc.c      | 48 +++++++++++++++++++++++++++++++++++
 lib/libc/mips/sys/Makefile.inc        |  2 ++
 lib/libc/mips/sys/__vdso_gettc.c      | 48 +++++++++++++++++++++++++++++++++++
 lib/libc/powerpc/Makefile.inc         |  2 ++
 lib/libc/powerpc/sys/__vdso_gettc.c   | 48 +++++++++++++++++++++++++++++++++++
 lib/libc/powerpc64/Makefile.inc       |  2 ++
 lib/libc/powerpc64/sys/__vdso_gettc.c | 48 +++++++++++++++++++++++++++++++++++
 lib/libc/sparc64/Makefile.inc         |  2 ++
 lib/libc/sparc64/sys/__vdso_gettc.c   | 48 +++++++++++++++++++++++++++++++++++
 lib/libc/sys/__vdso_gettimeofday.c    |  4 ++-
 lib/libc/sys/gettimeofday.c           |  5 +---
 16 files changed, 325 insertions(+), 5 deletions(-)
 create mode 100644 lib/libc/arm/sys/__vdso_gettc.c
 create mode 100644 lib/libc/ia64/sys/__vdso_gettc.c
 create mode 100644 lib/libc/mips/sys/__vdso_gettc.c
 create mode 100644 lib/libc/powerpc/sys/__vdso_gettc.c
 create mode 100644 lib/libc/powerpc64/sys/__vdso_gettc.c
 create mode 100644 lib/libc/sparc64/sys/__vdso_gettc.c

(limited to 'lib')

diff --git a/lib/libc/amd64/sys/__vdso_gettc.c b/lib/libc/amd64/sys/__vdso_gettc.c
index 091fe26..c6f2dfb 100644
--- a/lib/libc/amd64/sys/__vdso_gettc.c
+++ b/lib/libc/amd64/sys/__vdso_gettc.c
@@ -27,9 +27,11 @@
 __FBSDID("$FreeBSD$");
 
 #include <sys/types.h>
+#include <sys/elf.h>
 #include <sys/time.h>
 #include <sys/vdso.h>
 #include <machine/cpufunc.h>
+#include "libc_private.h"
 
 static u_int
 __vdso_gettc_low(const struct vdso_timehands *th)
@@ -41,9 +43,18 @@ __vdso_gettc_low(const struct vdso_timehands *th)
 	return (rv);
 }
 
+#pragma weak __vdso_gettc
 u_int
 __vdso_gettc(const struct vdso_timehands *th)
 {
 
 	return (th->th_x86_shift > 0 ? __vdso_gettc_low(th) : rdtsc32());
 }
+
+#pragma weak __vdso_gettimekeep
+int
+__vdso_gettimekeep(struct vdso_timekeep **tk)
+{
+
+	return (_elf_aux_info(AT_TIMEKEEP, tk, sizeof(*tk)));
+}
diff --git a/lib/libc/arm/sys/Makefile.inc b/lib/libc/arm/sys/Makefile.inc
index 1a58eae..fd251c8 100644
--- a/lib/libc/arm/sys/Makefile.inc
+++ b/lib/libc/arm/sys/Makefile.inc
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+SRCS+=	__vdso_gettc.c
+
 MDASM= Ovfork.S brk.S cerror.S pipe.S ptrace.S sbrk.S shmat.S sigreturn.S syscall.S
 
 # Don't generate default code for these syscalls:
diff --git a/lib/libc/arm/sys/__vdso_gettc.c b/lib/libc/arm/sys/__vdso_gettc.c
new file mode 100644
index 0000000..b99bbc4
--- /dev/null
+++ b/lib/libc/arm/sys/__vdso_gettc.c
@@ -0,0 +1,48 @@
+/*-
+ * Copyright (c) 2013 Konstantin Belousov <kib@FreeBSD.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/types.h>
+#include <sys/time.h>
+#include <sys/vdso.h>
+#include <errno.h>
+
+#pragma weak __vdso_gettc
+u_int
+__vdso_gettc(const struct vdso_timehands *th)
+{
+
+	return (0);
+}
+
+#pragma weak __vdso_gettimekeep
+int
+__vdso_gettimekeep(struct vdso_timekeep **tk)
+{
+
+	return (ENOSYS);
+}
diff --git a/lib/libc/i386/sys/__vdso_gettc.c b/lib/libc/i386/sys/__vdso_gettc.c
index 4419141..c6f2dfb 100644
--- a/lib/libc/i386/sys/__vdso_gettc.c
+++ b/lib/libc/i386/sys/__vdso_gettc.c
@@ -27,9 +27,11 @@
 __FBSDID("$FreeBSD$");
 
 #include <sys/types.h>
+#include <sys/elf.h>
 #include <sys/time.h>
 #include <sys/vdso.h>
 #include <machine/cpufunc.h>
+#include "libc_private.h"
 
 static u_int
 __vdso_gettc_low(const struct vdso_timehands *th)
@@ -48,3 +50,11 @@ __vdso_gettc(const struct vdso_timehands *th)
 
 	return (th->th_x86_shift > 0 ? __vdso_gettc_low(th) : rdtsc32());
 }
+
+#pragma weak __vdso_gettimekeep
+int
+__vdso_gettimekeep(struct vdso_timekeep **tk)
+{
+
+	return (_elf_aux_info(AT_TIMEKEEP, tk, sizeof(*tk)));
+}
diff --git a/lib/libc/ia64/sys/Makefile.inc b/lib/libc/ia64/sys/Makefile.inc
index 3876d3a..2846590 100644
--- a/lib/libc/ia64/sys/Makefile.inc
+++ b/lib/libc/ia64/sys/Makefile.inc
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+SRCS+=	__vdso_gettc.c
+
 MDASM+=	Ovfork.S brk.S cerror.S exect.S fork.S getcontext.S pipe.S ptrace.S \
 	sbrk.S setlogin.S sigreturn.S swapcontext.S
 
diff --git a/lib/libc/ia64/sys/__vdso_gettc.c b/lib/libc/ia64/sys/__vdso_gettc.c
new file mode 100644
index 0000000..b99bbc4
--- /dev/null
+++ b/lib/libc/ia64/sys/__vdso_gettc.c
@@ -0,0 +1,48 @@
+/*-
+ * Copyright (c) 2013 Konstantin Belousov <kib@FreeBSD.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/types.h>
+#include <sys/time.h>
+#include <sys/vdso.h>
+#include <errno.h>
+
+#pragma weak __vdso_gettc
+u_int
+__vdso_gettc(const struct vdso_timehands *th)
+{
+
+	return (0);
+}
+
+#pragma weak __vdso_gettimekeep
+int
+__vdso_gettimekeep(struct vdso_timekeep **tk)
+{
+
+	return (ENOSYS);
+}
diff --git a/lib/libc/mips/sys/Makefile.inc b/lib/libc/mips/sys/Makefile.inc
index 3601909..fc11349 100644
--- a/lib/libc/mips/sys/Makefile.inc
+++ b/lib/libc/mips/sys/Makefile.inc
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+SRCS+=	__vdso_gettc.c
+
 MDASM=  Ovfork.S brk.S cerror.S exect.S \
 	fork.S pipe.S ptrace.S sbrk.S syscall.S
 
diff --git a/lib/libc/mips/sys/__vdso_gettc.c b/lib/libc/mips/sys/__vdso_gettc.c
new file mode 100644
index 0000000..b99bbc4
--- /dev/null
+++ b/lib/libc/mips/sys/__vdso_gettc.c
@@ -0,0 +1,48 @@
+/*-
+ * Copyright (c) 2013 Konstantin Belousov <kib@FreeBSD.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/types.h>
+#include <sys/time.h>
+#include <sys/vdso.h>
+#include <errno.h>
+
+#pragma weak __vdso_gettc
+u_int
+__vdso_gettc(const struct vdso_timehands *th)
+{
+
+	return (0);
+}
+
+#pragma weak __vdso_gettimekeep
+int
+__vdso_gettimekeep(struct vdso_timekeep **tk)
+{
+
+	return (ENOSYS);
+}
diff --git a/lib/libc/powerpc/Makefile.inc b/lib/libc/powerpc/Makefile.inc
index 453726a..104f80c 100644
--- a/lib/libc/powerpc/Makefile.inc
+++ b/lib/libc/powerpc/Makefile.inc
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+SRCS+=	__vdso_gettc.c
+
 # Long double is 64-bits
 MDSRCS+=machdep_ldisd.c
 SYM_MAPS+=${.CURDIR}/powerpc/Symbol.map
diff --git a/lib/libc/powerpc/sys/__vdso_gettc.c b/lib/libc/powerpc/sys/__vdso_gettc.c
new file mode 100644
index 0000000..b99bbc4
--- /dev/null
+++ b/lib/libc/powerpc/sys/__vdso_gettc.c
@@ -0,0 +1,48 @@
+/*-
+ * Copyright (c) 2013 Konstantin Belousov <kib@FreeBSD.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/types.h>
+#include <sys/time.h>
+#include <sys/vdso.h>
+#include <errno.h>
+
+#pragma weak __vdso_gettc
+u_int
+__vdso_gettc(const struct vdso_timehands *th)
+{
+
+	return (0);
+}
+
+#pragma weak __vdso_gettimekeep
+int
+__vdso_gettimekeep(struct vdso_timekeep **tk)
+{
+
+	return (ENOSYS);
+}
diff --git a/lib/libc/powerpc64/Makefile.inc b/lib/libc/powerpc64/Makefile.inc
index e4ccbb9..a30779d 100644
--- a/lib/libc/powerpc64/Makefile.inc
+++ b/lib/libc/powerpc64/Makefile.inc
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+SRCS+=	__vdso_gettc.c
+
 # Long double is 64-bits
 MDSRCS+=machdep_ldisd.c
 SYM_MAPS+=${.CURDIR}/powerpc64/Symbol.map
diff --git a/lib/libc/powerpc64/sys/__vdso_gettc.c b/lib/libc/powerpc64/sys/__vdso_gettc.c
new file mode 100644
index 0000000..b99bbc4
--- /dev/null
+++ b/lib/libc/powerpc64/sys/__vdso_gettc.c
@@ -0,0 +1,48 @@
+/*-
+ * Copyright (c) 2013 Konstantin Belousov <kib@FreeBSD.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/types.h>
+#include <sys/time.h>
+#include <sys/vdso.h>
+#include <errno.h>
+
+#pragma weak __vdso_gettc
+u_int
+__vdso_gettc(const struct vdso_timehands *th)
+{
+
+	return (0);
+}
+
+#pragma weak __vdso_gettimekeep
+int
+__vdso_gettimekeep(struct vdso_timekeep **tk)
+{
+
+	return (ENOSYS);
+}
diff --git a/lib/libc/sparc64/Makefile.inc b/lib/libc/sparc64/Makefile.inc
index 76cc8b9..84a0e06 100644
--- a/lib/libc/sparc64/Makefile.inc
+++ b/lib/libc/sparc64/Makefile.inc
@@ -5,6 +5,8 @@
 
 .include "fpu/Makefile.inc"
 
+SRCS+=	__vdso_gettc.c
+
 # Long double is quad precision
 GDTOASRCS+=strtorQ.c
 MDSRCS+=machdep_ldisQ.c
diff --git a/lib/libc/sparc64/sys/__vdso_gettc.c b/lib/libc/sparc64/sys/__vdso_gettc.c
new file mode 100644
index 0000000..b99bbc4
--- /dev/null
+++ b/lib/libc/sparc64/sys/__vdso_gettc.c
@@ -0,0 +1,48 @@
+/*-
+ * Copyright (c) 2013 Konstantin Belousov <kib@FreeBSD.org>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/types.h>
+#include <sys/time.h>
+#include <sys/vdso.h>
+#include <errno.h>
+
+#pragma weak __vdso_gettc
+u_int
+__vdso_gettc(const struct vdso_timehands *th)
+{
+
+	return (0);
+}
+
+#pragma weak __vdso_gettimekeep
+int
+__vdso_gettimekeep(struct vdso_timekeep **tk)
+{
+
+	return (ENOSYS);
+}
diff --git a/lib/libc/sys/__vdso_gettimeofday.c b/lib/libc/sys/__vdso_gettimeofday.c
index 32abb69..a305173 100644
--- a/lib/libc/sys/__vdso_gettimeofday.c
+++ b/lib/libc/sys/__vdso_gettimeofday.c
@@ -79,6 +79,7 @@ binuptime(struct bintime *bt, struct vdso_timekeep *tk, int abs)
 
 static struct vdso_timekeep *tk;
 
+#pragma weak __vdso_gettimeofday
 int
 __vdso_gettimeofday(struct timeval *tv, struct timezone *tz)
 {
@@ -88,7 +89,7 @@ __vdso_gettimeofday(struct timeval *tv, struct timezone *tz)
 	if (tz != NULL)
 		return (ENOSYS);
 	if (tk == NULL) {
-		error = _elf_aux_info(AT_TIMEKEEP, &tk, sizeof(tk));
+		error = __vdso_gettimekeep(&tk);
 		if (error != 0 || tk == NULL)
 			return (ENOSYS);
 	}
@@ -101,6 +102,7 @@ __vdso_gettimeofday(struct timeval *tv, struct timezone *tz)
 	return (0);
 }
 
+#pragma weak __vdso_clock_gettime
 int
 __vdso_clock_gettime(clockid_t clock_id, struct timespec *ts)
 {
diff --git a/lib/libc/sys/gettimeofday.c b/lib/libc/sys/gettimeofday.c
index 4cc87e1..5b12523 100644
--- a/lib/libc/sys/gettimeofday.c
+++ b/lib/libc/sys/gettimeofday.c
@@ -41,10 +41,7 @@ __gettimeofday(struct timeval *tv, struct timezone *tz)
 {
 	int error;
 
-	if (__vdso_gettimeofday != NULL && __vdso_gettc != NULL)
-		error = __vdso_gettimeofday(tv, tz);
-	else
-		error = ENOSYS;
+	error = __vdso_gettimeofday(tv, tz);
 	if (error == ENOSYS)
 		error = __sys_gettimeofday(tv, tz);
 	return (error);
-- 
cgit v1.1


From e7ca48d15b0d2114430b38f3d1cd793d3e50a3ae Mon Sep 17 00:00:00 2001
From: gahr <gahr@FreeBSD.org>
Date: Wed, 30 Jan 2013 14:59:26 +0000
Subject: Add fmemopen(3), an interface to get a FILE * from a buffer in
 memory, along with the respective regression test. See
 http://pubs.opengroup.org/onlinepubs/9699919799/functions/fmemopen.html

Reviewed by:	cognet
Approved by:	cognet
---
 lib/libc/stdio/Makefile.inc |   5 +-
 lib/libc/stdio/Symbol.map   |   1 +
 lib/libc/stdio/fmemopen.c   | 182 ++++++++++++++++++++++++++++++++++++++++++++
 lib/libc/stdio/fopen.3      |  43 +++++++++--
 4 files changed, 224 insertions(+), 7 deletions(-)
 create mode 100644 lib/libc/stdio/fmemopen.c

(limited to 'lib')

diff --git a/lib/libc/stdio/Makefile.inc b/lib/libc/stdio/Makefile.inc
index 6570051..bc3b9e5 100644
--- a/lib/libc/stdio/Makefile.inc
+++ b/lib/libc/stdio/Makefile.inc
@@ -8,7 +8,8 @@ SRCS+=	_flock_stub.c asprintf.c clrerr.c dprintf.c \
 	fclose.c fcloseall.c fdopen.c \
 	feof.c ferror.c fflush.c fgetc.c fgetln.c fgetpos.c fgets.c fgetwc.c \
 	fgetwln.c fgetws.c \
-	fileno.c findfp.c flags.c fopen.c fprintf.c fpurge.c fputc.c fputs.c \
+	fileno.c findfp.c flags.c fmemopen.c fopen.c fprintf.c fpurge.c \
+	fputc.c fputs.c \
 	fputwc.c fputws.c fread.c freopen.c fscanf.c fseek.c fsetpos.c \
 	ftell.c funopen.c fvwrite.c fwalk.c fwide.c fwprintf.c fwscanf.c \
 	fwrite.c getc.c getchar.c getdelim.c getline.c \
@@ -48,7 +49,7 @@ MLINKS+=ferror.3 ferror_unlocked.3 \
 MLINKS+=fflush.3 fpurge.3
 MLINKS+=fgets.3 gets.3
 MLINKS+=flockfile.3 ftrylockfile.3 flockfile.3 funlockfile.3
-MLINKS+=fopen.3 fdopen.3 fopen.3 freopen.3
+MLINKS+=fopen.3 fdopen.3 fopen.3 freopen.3 fopen.3 fmemopen.3
 MLINKS+=fputs.3 puts.3
 MLINKS+=fread.3 fwrite.3
 MLINKS+=fseek.3 fgetpos.3 fseek.3 fseeko.3 fseek.3 fsetpos.3 fseek.3 ftell.3 \
diff --git a/lib/libc/stdio/Symbol.map b/lib/libc/stdio/Symbol.map
index cafb0c6..b78de5a 100644
--- a/lib/libc/stdio/Symbol.map
+++ b/lib/libc/stdio/Symbol.map
@@ -155,6 +155,7 @@ FBSD_1.3 {
 	getwchar_l;
 	putwc_l;
 	putwchar_l;
+	fmemopen;
 };
 
 FBSDprivate_1.0 {
diff --git a/lib/libc/stdio/fmemopen.c b/lib/libc/stdio/fmemopen.c
new file mode 100644
index 0000000..2cdf324
--- /dev/null
+++ b/lib/libc/stdio/fmemopen.c
@@ -0,0 +1,182 @@
+/*-
+Copyright (C) 2013 Pietro Cerutti <gahr@FreeBSD.org>
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+1. Redistributions of source code must retain the above copyright
+   notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+   notice, this list of conditions and the following disclaimer in the
+   documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGE.
+*/
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+
+struct __fmemopen_cookie
+{
+	char *buf;	/* pointer to the memory region */
+	char  own;	/* did we allocate the buffer ourselves? */
+	long  len;	/* buffer length in bytes */
+	long  off;	/* current offset into the buffer */
+};
+
+static int	fmemopen_read  (void *cookie, char *buf, int nbytes);
+static int	fmemopen_write (void *cookie, const char *buf, int nbytes);
+static fpos_t	fmemopen_seek  (void *cookie, fpos_t offset, int whence);
+static int	fmemopen_close (void *cookie);
+
+FILE *
+fmemopen (void * __restrict buf, size_t size, const char * __restrict mode)
+{
+	/* allocate cookie */
+	struct __fmemopen_cookie *ck = malloc (sizeof (struct __fmemopen_cookie));
+	if (ck == NULL) {
+		errno = ENOMEM;
+		return (NULL);
+	}
+
+	ck->off = 0;
+	ck->len = size;
+
+	/* do we have to allocate the buffer ourselves? */
+	ck->own = ((ck->buf = buf) == NULL);
+	if (ck->own) {
+		ck->buf = malloc (size);
+		if (ck->buf == NULL) {
+			free (ck);
+			errno = ENOMEM;
+			return (NULL);
+		}
+		ck->buf[0] = '\0';
+	}
+
+	if (mode[0] == 'a')
+		ck->off = strnlen(ck->buf, ck->len);
+
+	/* actuall wrapper */
+	FILE *f = funopen ((void *)ck, fmemopen_read, fmemopen_write,
+	    fmemopen_seek, fmemopen_close);
+
+	if (f == NULL) {
+		if (ck->own)
+			free (ck->buf);
+		free (ck);
+		return (NULL);
+	}
+
+	/* turn off buffering, so a write past the end of the buffer
+	 * correctly returns a short object count */
+	setvbuf (f, (char *) NULL, _IONBF, 0);
+
+	return (f);
+}
+
+static int
+fmemopen_read (void *cookie, char *buf, int nbytes)
+{
+	struct __fmemopen_cookie *ck = cookie;
+
+	if (nbytes > ck->len - ck->off)
+		nbytes = ck->len - ck->off;
+
+	if (nbytes == 0)
+		return (0);
+
+	memcpy (buf, ck->buf + ck->off, nbytes);
+
+	ck->off += nbytes;
+
+	return (nbytes);
+}
+
+static int
+fmemopen_write (void *cookie, const char *buf, int nbytes)
+{
+	struct __fmemopen_cookie *ck = cookie;
+
+	if (nbytes > ck->len - ck->off)
+		nbytes = ck->len - ck->off;
+
+	if (nbytes == 0)
+		return (0);
+
+	memcpy (ck->buf + ck->off, buf, nbytes);
+
+	ck->off += nbytes;
+
+	if (ck->off < ck->len && ck->buf[ck->off - 1] != '\0')
+		ck->buf[ck->off] = '\0';
+
+	return (nbytes);
+}
+
+static fpos_t
+fmemopen_seek (void *cookie, fpos_t offset, int whence)
+{
+	struct __fmemopen_cookie *ck = cookie;
+
+
+	switch (whence) {
+	case SEEK_SET:
+		if (offset > ck->len) {
+			errno = EINVAL;
+			return (-1);
+		}
+		ck->off = offset;
+		break;
+
+	case SEEK_CUR:
+		if (ck->off + offset > ck->len) {
+			errno = EINVAL;
+			return (-1);
+		}
+		ck->off += offset;
+		break;
+
+	case SEEK_END:
+		if (offset > 0 || -offset > ck->len) {
+			errno = EINVAL;
+			return (-1);
+		}
+		ck->off = ck->len + offset;
+		break;
+
+	default:
+		errno = EINVAL;
+		return (-1);
+	}
+
+	return (ck->off);
+}
+
+static int
+fmemopen_close (void *cookie)
+{
+	struct __fmemopen_cookie *ck = cookie;
+
+	if (ck->own)
+		free (ck->buf);
+
+	free (ck);
+
+	return (0);
+}
diff --git a/lib/libc/stdio/fopen.3 b/lib/libc/stdio/fopen.3
index da89ed4..41d66b7 100644
--- a/lib/libc/stdio/fopen.3
+++ b/lib/libc/stdio/fopen.3
@@ -32,13 +32,14 @@
 .\"     @(#)fopen.3	8.1 (Berkeley) 6/4/93
 .\" $FreeBSD$
 .\"
-.Dd November 30, 2012
+.Dd January 30, 2013
 .Dt FOPEN 3
 .Os
 .Sh NAME
 .Nm fopen ,
 .Nm fdopen ,
-.Nm freopen
+.Nm freopen ,
+.Nm fmemopen
 .Nd stream open functions
 .Sh LIBRARY
 .Lb libc
@@ -50,6 +51,8 @@
 .Fn fdopen "int fildes" "const char *mode"
 .Ft FILE *
 .Fn freopen "const char *path" "const char *mode" "FILE *stream"
+.Ft FILE *
+.Fn fmemopen "void *restrict *buf" "size_t size" "const char * restrict mode"
 .Sh DESCRIPTION
 The
 .Fn fopen
@@ -202,6 +205,29 @@ standard text stream
 .Dv ( stderr , stdin ,
 or
 .Dv stdout ) .
+.Pp
+The
+.Fn fmemopen
+function
+associates the buffer given by the
+.Fa buf
+and
+.Fa size
+arguments with a stream.
+The
+.Fa buf
+argument shall be either a null pointer or point to a buffer that
+is at least
+.Fa size
+bytes long.
+If a null pointer is specified as the
+.Fa buf
+argument,
+.Fn fmemopen
+shall allocate
+.Fa size
+bytes of memory. This buffer shall be automatically freed when the
+stream is closed.
 .Sh RETURN VALUES
 Upon successful completion
 .Fn fopen ,
@@ -225,16 +251,18 @@ argument
 to
 .Fn fopen ,
 .Fn fdopen ,
+.Fn freopen ,
 or
-.Fn freopen
+.Fn fmemopen
 was invalid.
 .El
 .Pp
 The
 .Fn fopen ,
-.Fn fdopen
-and
+.Fn fdopen ,
 .Fn freopen
+and
+.Fn fmemopen
 functions
 may also fail and set
 .Va errno
@@ -294,3 +322,8 @@ The
 .Dq Li e
 mode option does not conform to any standard
 but is also supported by glibc.
+The
+.Fn fmemopen
+function
+conforms to 
+.St -p1003.1-2008 .
-- 
cgit v1.1


From 3706909a8374ad86e702bde80ab176b0c1fcb9b7 Mon Sep 17 00:00:00 2001
From: gahr <gahr@FreeBSD.org>
Date: Thu, 31 Jan 2013 16:39:50 +0000
Subject: - Remove underscores from the internal structure name, as it doesn't
 collide   with the user's namespace.

- Correct size and position variables type from long to size_t.

- Do not set errno to ENOMEM on malloc failure, as malloc already does so.

- Implement the concept of "buffer data length", which mandates what SEEK_END
  refers to and the allowed extent for a read.

- Use NULL as read-callback if the buffer is opened in write-only mode.
  Conversely, use NULL as write-callback when opened in read-only mode.

- Implement the handling of the ``b'' character in the mode argument. A binary
  buffer differs from a text buffer (default mode if ``b'' is omitted) in that
  NULL bytes are never appended to writes and that the "buffer data length"
  equals to the size of the buffer.

- Remove shall from the man page. Use indicative instead. Also, specify that
  the ``b'' flag does not conform with POSIX but is supported by glibc.

- Update the regression test so that the ``b'' functionality and the "buffer
  data length" concepts are tested.

- Minor style(9) corrections.

Suggested by:	jilles
Reviewed by:	cognet
Approved by:	cognet
---
 lib/libc/stdio/fmemopen.c | 131 ++++++++++++++++++++++++++++++++++++----------
 lib/libc/stdio/fopen.3    |  19 +++++--
 2 files changed, 118 insertions(+), 32 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/stdio/fmemopen.c b/lib/libc/stdio/fmemopen.c
index 2cdf324..dec2c86 100644
--- a/lib/libc/stdio/fmemopen.c
+++ b/lib/libc/stdio/fmemopen.c
@@ -26,17 +26,21 @@ SUCH DAMAGE.
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include <fcntl.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <errno.h>
+#include "local.h"
 
-struct __fmemopen_cookie
+struct fmemopen_cookie
 {
-	char *buf;	/* pointer to the memory region */
-	char  own;	/* did we allocate the buffer ourselves? */
-	long  len;	/* buffer length in bytes */
-	long  off;	/* current offset into the buffer */
+	char	*buf;	/* pointer to the memory region */
+	char	 own;	/* did we allocate the buffer ourselves? */
+	char     bin;   /* is this a binary buffer? */
+	size_t	 size;	/* buffer length in bytes */
+	size_t	 len;	/* data length in bytes */
+	size_t	 off;	/* current offset into the buffer */
 };
 
 static int	fmemopen_read  (void *cookie, char *buf, int nbytes);
@@ -47,33 +51,95 @@ static int	fmemopen_close (void *cookie);
 FILE *
 fmemopen (void * __restrict buf, size_t size, const char * __restrict mode)
 {
-	/* allocate cookie */
-	struct __fmemopen_cookie *ck = malloc (sizeof (struct __fmemopen_cookie));
+	struct fmemopen_cookie *ck;
+	FILE *f;
+	int flags, rc;
+
+	/* 
+	 * Retrieve the flags as used by open(2) from the mode argument, and
+	 * validate them.
+	 * */
+	rc = __sflags (mode, &flags);
+	if (rc == 0) {
+		errno = EINVAL;
+		return (NULL);
+	}
+
+	/* 
+	 * There's no point in requiring an automatically allocated buffer
+	 * in write-only mode.
+	 */
+	if (!(flags & O_RDWR) && buf == NULL) {
+		errno = EINVAL;
+		return (NULL);
+	}
+	
+	/* Allocate a cookie. */
+	ck = malloc (sizeof (struct fmemopen_cookie));
 	if (ck == NULL) {
-		errno = ENOMEM;
 		return (NULL);
 	}
 
-	ck->off = 0;
-	ck->len = size;
+	ck->off  = 0;
+	ck->size = size;
 
-	/* do we have to allocate the buffer ourselves? */
+	/* Check whether we have to allocate the buffer ourselves. */
 	ck->own = ((ck->buf = buf) == NULL);
 	if (ck->own) {
 		ck->buf = malloc (size);
 		if (ck->buf == NULL) {
 			free (ck);
-			errno = ENOMEM;
 			return (NULL);
 		}
+	}
+
+	/*
+	 * POSIX distinguishes between w+ and r+, in that w+ is supposed to
+	 * truncate the buffer.
+	 */
+	if (ck->own || mode[0] == 'w') {
 		ck->buf[0] = '\0';
 	}
 
-	if (mode[0] == 'a')
-		ck->off = strnlen(ck->buf, ck->len);
+	/* Check for binary mode. */
+	ck->bin = strchr(mode, 'b') != NULL;
+
+	/*
+	 * The size of the current buffer contents is set depending on the
+	 * mode:
+	 *
+	 * for append (text-mode), the position of the first NULL byte, or the
+	 * size of the buffer if none is found
+	 *
+	 * for append (binary-mode), the size of the buffer
+	 *
+	 * for read, the size of the buffer
+	 *
+	 * for write, 0
+	 */
+	switch (mode[0]) {
+	case 'a':
+		if (ck->bin) {
+			/* 
+			 * This isn't useful, since the buffer isn't
+			 * allowed to grow.
+			 */
+			ck->off = ck->len = size;
+		} else
+			ck->off = ck->len = strnlen(ck->buf, ck->size);
+		break;
+	case 'r':
+		ck->len = size;
+		break;
+	case 'w':
+		ck->len = 0;
+		break;
+	}
 
-	/* actuall wrapper */
-	FILE *f = funopen ((void *)ck, fmemopen_read, fmemopen_write,
+	/* Actuall wrapper. */
+	f = funopen ((void *)ck,
+	    flags & O_WRONLY ? NULL : fmemopen_read, 
+	    flags & O_RDONLY ? NULL : fmemopen_write,
 	    fmemopen_seek, fmemopen_close);
 
 	if (f == NULL) {
@@ -83,8 +149,10 @@ fmemopen (void * __restrict buf, size_t size, const char * __restrict mode)
 		return (NULL);
 	}
 
-	/* turn off buffering, so a write past the end of the buffer
-	 * correctly returns a short object count */
+	/*
+	 * Turn off buffering, so a write past the end of the buffer
+	 * correctly returns a short object count.
+	 */
 	setvbuf (f, (char *) NULL, _IONBF, 0);
 
 	return (f);
@@ -93,7 +161,7 @@ fmemopen (void * __restrict buf, size_t size, const char * __restrict mode)
 static int
 fmemopen_read (void *cookie, char *buf, int nbytes)
 {
-	struct __fmemopen_cookie *ck = cookie;
+	struct fmemopen_cookie *ck = cookie;
 
 	if (nbytes > ck->len - ck->off)
 		nbytes = ck->len - ck->off;
@@ -111,10 +179,10 @@ fmemopen_read (void *cookie, char *buf, int nbytes)
 static int
 fmemopen_write (void *cookie, const char *buf, int nbytes)
 {
-	struct __fmemopen_cookie *ck = cookie;
+	struct fmemopen_cookie *ck = cookie;
 
-	if (nbytes > ck->len - ck->off)
-		nbytes = ck->len - ck->off;
+	if (nbytes > ck->size - ck->off)
+		nbytes = ck->size - ck->off;
 
 	if (nbytes == 0)
 		return (0);
@@ -123,7 +191,16 @@ fmemopen_write (void *cookie, const char *buf, int nbytes)
 
 	ck->off += nbytes;
 
-	if (ck->off < ck->len && ck->buf[ck->off - 1] != '\0')
+	if (ck->off > ck->len)
+		ck->len = ck->off;
+
+	/*
+	 * We append a NULL byte if all these conditions are met:
+	 * - the buffer is not binary
+	 * - the buffer is not full
+	 * - the data just written doesn't already end with a NULL byte
+	 */
+	if (!ck->bin && ck->off < ck->size && ck->buf[ck->off - 1] != '\0')
 		ck->buf[ck->off] = '\0';
 
 	return (nbytes);
@@ -132,12 +209,12 @@ fmemopen_write (void *cookie, const char *buf, int nbytes)
 static fpos_t
 fmemopen_seek (void *cookie, fpos_t offset, int whence)
 {
-	struct __fmemopen_cookie *ck = cookie;
+	struct fmemopen_cookie *ck = cookie;
 
 
 	switch (whence) {
 	case SEEK_SET:
-		if (offset > ck->len) {
+		if (offset > ck->size) {
 			errno = EINVAL;
 			return (-1);
 		}
@@ -145,7 +222,7 @@ fmemopen_seek (void *cookie, fpos_t offset, int whence)
 		break;
 
 	case SEEK_CUR:
-		if (ck->off + offset > ck->len) {
+		if (ck->off + offset > ck->size) {
 			errno = EINVAL;
 			return (-1);
 		}
@@ -171,7 +248,7 @@ fmemopen_seek (void *cookie, fpos_t offset, int whence)
 static int
 fmemopen_close (void *cookie)
 {
-	struct __fmemopen_cookie *ck = cookie;
+	struct fmemopen_cookie *ck = cookie;
 
 	if (ck->own)
 		free (ck->buf);
diff --git a/lib/libc/stdio/fopen.3 b/lib/libc/stdio/fopen.3
index 41d66b7..a07be38 100644
--- a/lib/libc/stdio/fopen.3
+++ b/lib/libc/stdio/fopen.3
@@ -118,7 +118,9 @@ after either the
 or the first letter.
 This is strictly for compatibility with
 .St -isoC
-and has no effect; the ``b'' is ignored.
+and has effect only for
+.Fn fmemopen
+; otherwise the ``b'' is ignored.
 .Pp
 Any created files will have mode
 .Do Dv S_IRUSR
@@ -216,7 +218,7 @@ and
 arguments with a stream.
 The
 .Fa buf
-argument shall be either a null pointer or point to a buffer that
+argument is either a null pointer or point to a buffer that
 is at least
 .Fa size
 bytes long.
@@ -224,10 +226,15 @@ If a null pointer is specified as the
 .Fa buf
 argument,
 .Fn fmemopen
-shall allocate
+allocates
 .Fa size
-bytes of memory. This buffer shall be automatically freed when the
-stream is closed.
+bytes of memory. This buffer is automatically freed when the
+stream is closed. Buffers can be opened in text-mode (default) or binary-mode
+(if ``b'' is present in the second or third position of the
+.Fa mode
+argument). Buffers opened in text-mode make sure that writes are terminated with
+a NULL byte, if the last write hasn't filled up the whole buffer. Buffers
+opened in binary-mode never append a NULL byte.
 .Sh RETURN VALUES
 Upon successful completion
 .Fn fopen ,
@@ -327,3 +334,5 @@ The
 function
 conforms to 
 .St -p1003.1-2008 .
+The ``b'' mode does not conform to any standard
+but is also supported by glibc.
-- 
cgit v1.1


From 577b42d393006042797668cd419aef5a2e419bb9 Mon Sep 17 00:00:00 2001
From: sbruno <sbruno@FreeBSD.org>
Date: Thu, 31 Jan 2013 22:09:53 +0000
Subject: Update hwpmc to support the Xeon class of Ivybridge processors. case
 0x3E:      /* Per Intel document 325462-045US 01/2013. */

Add manpage to document all the goodness that is available in this
processor model.

No support for uncore events at this time.

Submitted by:	hiren panchasara <hiren.panchasara@gmail.com>
Reviewed by:	davide, jimharris, sbruno
Obtained from:	Yahoo! Inc.
MFC after:	2 weeks
---
 lib/libpmc/Makefile              |   1 +
 lib/libpmc/libpmc.c              |  23 +-
 lib/libpmc/pmc.ivybridge.3       |   1 +
 lib/libpmc/pmc.ivybridgexeon.3   | 911 +++++++++++++++++++++++++++++++++++++++
 lib/libpmc/pmc.sandybridge.3     |   2 +
 lib/libpmc/pmc.sandybridgexeon.3 |   1 +
 6 files changed, 938 insertions(+), 1 deletion(-)
 create mode 100644 lib/libpmc/pmc.ivybridgexeon.3

(limited to 'lib')

diff --git a/lib/libpmc/Makefile b/lib/libpmc/Makefile
index f59dc2b..c15d38a 100644
--- a/lib/libpmc/Makefile
+++ b/lib/libpmc/Makefile
@@ -29,6 +29,7 @@ MAN+=	pmc.core.3
 MAN+=	pmc.core2.3
 MAN+=	pmc.iaf.3
 MAN+=	pmc.ivybridge.3
+MAN+=	pmc.ivybridgexeon.3
 MAN+=	pmc.ucf.3
 MAN+=	pmc.k7.3
 MAN+=	pmc.k8.3
diff --git a/lib/libpmc/libpmc.c b/lib/libpmc/libpmc.c
index 4fb2572..1866e7f 100644
--- a/lib/libpmc/libpmc.c
+++ b/lib/libpmc/libpmc.c
@@ -188,6 +188,11 @@ static const struct pmc_event_descr ivybridge_event_table[] =
 	__PMC_EV_ALIAS_IVYBRIDGE()
 };
 
+static const struct pmc_event_descr ivybridge_xeon_event_table[] = 
+{
+	__PMC_EV_ALIAS_IVYBRIDGE_XEON()
+};
+
 static const struct pmc_event_descr sandybridge_event_table[] = 
 {
 	__PMC_EV_ALIAS_SANDYBRIDGE()
@@ -233,6 +238,7 @@ PMC_MDEP_TABLE(core, IAP, PMC_CLASS_SOFT, PMC_CLASS_TSC);
 PMC_MDEP_TABLE(core2, IAP, PMC_CLASS_SOFT, PMC_CLASS_IAF, PMC_CLASS_TSC);
 PMC_MDEP_TABLE(corei7, IAP, PMC_CLASS_SOFT, PMC_CLASS_IAF, PMC_CLASS_TSC, PMC_CLASS_UCF, PMC_CLASS_UCP);
 PMC_MDEP_TABLE(ivybridge, IAP, PMC_CLASS_SOFT, PMC_CLASS_IAF, PMC_CLASS_TSC);
+PMC_MDEP_TABLE(ivybridge_xeon, IAP, PMC_CLASS_SOFT, PMC_CLASS_IAF, PMC_CLASS_TSC);
 PMC_MDEP_TABLE(sandybridge, IAP, PMC_CLASS_SOFT, PMC_CLASS_IAF, PMC_CLASS_TSC, PMC_CLASS_UCF, PMC_CLASS_UCP);
 PMC_MDEP_TABLE(sandybridge_xeon, IAP, PMC_CLASS_SOFT, PMC_CLASS_IAF, PMC_CLASS_TSC);
 PMC_MDEP_TABLE(westmere, IAP, PMC_CLASS_SOFT, PMC_CLASS_IAF, PMC_CLASS_TSC, PMC_CLASS_UCF, PMC_CLASS_UCP);
@@ -272,6 +278,7 @@ PMC_CLASS_TABLE_DESC(core, IAP, core, iap);
 PMC_CLASS_TABLE_DESC(core2, IAP, core2, iap);
 PMC_CLASS_TABLE_DESC(corei7, IAP, corei7, iap);
 PMC_CLASS_TABLE_DESC(ivybridge, IAP, ivybridge, iap);
+PMC_CLASS_TABLE_DESC(ivybridge_xeon, IAP, ivybridge_xeon, iap);
 PMC_CLASS_TABLE_DESC(sandybridge, IAP, sandybridge, iap);
 PMC_CLASS_TABLE_DESC(sandybridge_xeon, IAP, sandybridge_xeon, iap);
 PMC_CLASS_TABLE_DESC(westmere, IAP, westmere, iap);
@@ -577,6 +584,8 @@ static struct pmc_event_alias core2_aliases_without_iaf[] = {
 #define corei7_aliases_without_iaf	core2_aliases_without_iaf
 #define ivybridge_aliases		core2_aliases
 #define ivybridge_aliases_without_iaf	core2_aliases_without_iaf
+#define ivybridge_xeon_aliases		core2_aliases
+#define ivybridge_xeon_aliases_without_iaf	core2_aliases_without_iaf
 #define sandybridge_aliases		core2_aliases
 #define sandybridge_aliases_without_iaf	core2_aliases_without_iaf
 #define sandybridge_xeon_aliases	core2_aliases
@@ -807,7 +816,8 @@ iap_allocate_pmc(enum pmc_event pe, char *ctrspec,
 				return (-1);
 		} else if (cpu_info.pm_cputype == PMC_CPU_INTEL_SANDYBRIDGE ||
 		    cpu_info.pm_cputype == PMC_CPU_INTEL_SANDYBRIDGE_XEON ||
-			cpu_info.pm_cputype == PMC_CPU_INTEL_IVYBRIDGE) {
+			cpu_info.pm_cputype == PMC_CPU_INTEL_IVYBRIDGE ||
+			cpu_info.pm_cputype == PMC_CPU_INTEL_IVYBRIDGE_XEON ) {
 			if (KWPREFIXMATCH(p, IAP_KW_RSP "=")) {
 				n = pmc_parse_mask(iap_rsp_mask_sb_sbx_ib, p, &rsp);
 			} else
@@ -2684,6 +2694,10 @@ pmc_event_names_of_class(enum pmc_class cl, const char ***eventnames,
 			ev = ivybridge_event_table;
 			count = PMC_EVENT_TABLE_SIZE(ivybridge);
 			break;
+		case PMC_CPU_INTEL_IVYBRIDGE_XEON:
+			ev = ivybridge_xeon_event_table;
+			count = PMC_EVENT_TABLE_SIZE(ivybridge_xeon);
+			break;
 		case PMC_CPU_INTEL_SANDYBRIDGE:
 			ev = sandybridge_event_table;
 			count = PMC_EVENT_TABLE_SIZE(sandybridge);
@@ -2983,6 +2997,9 @@ pmc_init(void)
 	case PMC_CPU_INTEL_IVYBRIDGE:
 		PMC_MDEP_INIT_INTEL_V2(ivybridge);
 		break;
+	case PMC_CPU_INTEL_IVYBRIDGE_XEON:
+		PMC_MDEP_INIT_INTEL_V2(ivybridge_xeon);
+		break;
 	case PMC_CPU_INTEL_SANDYBRIDGE:
 		pmc_class_table[n++] = &ucf_class_table_descr;
 		pmc_class_table[n++] = &sandybridgeuc_class_table_descr;
@@ -3125,6 +3142,10 @@ _pmc_name_of_event(enum pmc_event pe, enum pmc_cputype cpu)
 			ev = ivybridge_event_table;
 			evfence = ivybridge_event_table + PMC_EVENT_TABLE_SIZE(ivybridge);
 			break;
+		case PMC_CPU_INTEL_IVYBRIDGE_XEON:
+			ev = ivybridge_xeon_event_table;
+			evfence = ivybridge_xeon_event_table + PMC_EVENT_TABLE_SIZE(ivybridge_xeon);
+			break;
 		case PMC_CPU_INTEL_SANDYBRIDGE:
 			ev = sandybridge_event_table;
 			evfence = sandybridge_event_table + PMC_EVENT_TABLE_SIZE(sandybridge);
diff --git a/lib/libpmc/pmc.ivybridge.3 b/lib/libpmc/pmc.ivybridge.3
index 9a81da4..7d8bdca 100644
--- a/lib/libpmc/pmc.ivybridge.3
+++ b/lib/libpmc/pmc.ivybridge.3
@@ -853,6 +853,7 @@ Dirty L2 cache lines evicted by the MLC prefetcher.
 .Xr pmc.p6 3 ,
 .Xr pmc.corei7 3 ,
 .Xr pmc.corei7uc 3 ,
+.Xr pmc.ivybridgexeon 3 ,
 .Xr pmc.sandybridge 3 ,
 .Xr pmc.sandybridgeuc 3 ,
 .Xr pmc.sandybridgexeon 3 ,
diff --git a/lib/libpmc/pmc.ivybridgexeon.3 b/lib/libpmc/pmc.ivybridgexeon.3
new file mode 100644
index 0000000..76d5e60
--- /dev/null
+++ b/lib/libpmc/pmc.ivybridgexeon.3
@@ -0,0 +1,911 @@
+.\" Copyright (c) 2013 Hiren Panchasara <hiren.panchasara@gmail.com>
+.\" All rights reserved.
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\" $FreeBSD$
+.\"
+.Dd Jan 25, 2013
+.Dt PMC.IVYBRIDGEXEON 3
+.Os
+.Sh NAME
+.Nm pmc.ivybridgexeon
+.Nd measurement events for
+.Tn Intel
+.Tn Ivy Bridge Xeon
+family CPUs
+.Sh LIBRARY
+.Lb libpmc
+.Sh SYNOPSIS
+.In pmc.h
+.Sh DESCRIPTION
+.Tn Intel
+.Tn "Ivy Bridge Xeon"
+CPUs contain PMCs conforming to version 2 of the
+.Tn Intel
+performance measurement architecture.
+These CPUs may contain up to three classes of PMCs:
+.Bl -tag -width "Li PMC_CLASS_IAP"
+.It Li PMC_CLASS_IAF
+Fixed-function counters that count only one hardware event per counter.
+.It Li PMC_CLASS_IAP
+Programmable counters that may be configured to count one of a defined
+set of hardware events.
+.El
+.Pp
+The number of PMCs available in each class and their widths need to be
+determined at run time by calling
+.Xr pmc_cpuinfo 3 .
+.Pp
+Intel Ivy Bridge Xeon PMCs are documented in
+.Rs
+.%B "Intel(R) 64 and IA-32 Architectures Software Developer's Manual"
+.%N "Order Number: 325462-045US"
+.%D January 2013
+.%Q "Intel Corporation"
+.Re
+.Ss IVYBRIDGE FIXED FUNCTION PMCS
+These PMCs and their supported events are documented in
+.Xr pmc.iaf 3 .
+.Ss IVYBRIDGE PROGRAMMABLE PMCS
+The programmable PMCs support the following capabilities:
+.Bl -column "PMC_CAP_INTERRUPT" "Support"
+.It Em Capability Ta Em Support
+.It PMC_CAP_CASCADE Ta \&No
+.It PMC_CAP_EDGE Ta Yes
+.It PMC_CAP_INTERRUPT Ta Yes
+.It PMC_CAP_INVERT Ta Yes
+.It PMC_CAP_READ Ta Yes
+.It PMC_CAP_PRECISE Ta \&No
+.It PMC_CAP_SYSTEM Ta Yes
+.It PMC_CAP_TAGGING Ta \&No
+.It PMC_CAP_THRESHOLD Ta Yes
+.It PMC_CAP_USER Ta Yes
+.It PMC_CAP_WRITE Ta Yes
+.El
+.Ss Event Qualifiers
+Event specifiers for these PMCs support the following common
+qualifiers:
+.Bl -tag -width indent
+.It Li rsp= Ns Ar value
+Configure the Off-core Response bits.
+.Bl -tag -width indent
+.It Li REQ_DMND_DATA_RD
+Counts the number of demand and DCU prefetch data reads of full and partial
+cachelines as well as demand data page table entry cacheline reads. Does not
+count L2 data read prefetches or instruction fetches.
+.It Li REQ_DMND_RFO
+Counts the number of demand and DCU prefetch reads for ownership (RFO)
+requests generated by a write to data cacheline. Does not count L2 RFO
+prefetches.
+.It Li REQ_DMND_IFETCH
+Counts the number of demand and DCU prefetch instruction cacheline reads.
+Does not count L2 code read prefetches.
+.It Li REQ_WB
+Counts the number of writeback (modified to exclusive) transactions.
+.It Li REQ_PF_DATA_RD
+Counts the number of data cacheline reads generated by L2 prefetchers.
+.It Li REQ_PF_RFO
+Counts the number of RFO requests generated by L2 prefetchers.
+.It Li REQ_PF_IFETCH
+Counts the number of code reads generated by L2 prefetchers.
+.It Li REQ_PF_LLC_DATA_RD
+L2 prefetcher to L3 for loads.
+.It Li REQ_PF_LLC_RFO
+RFO requests generated by L2 prefetcher
+.It Li REQ_PF_LLC_IFETCH
+L2 prefetcher to L3 for instruction fetches.
+.It Li REQ_BUS_LOCKS
+Bus lock and split lock requests.
+.It Li REQ_STRM_ST
+Streaming store requests.
+.It Li REQ_OTHER
+Any other request that crosses IDI, including I/O.
+.It Li RES_ANY
+Catch all value for any response types.
+.It Li RES_SUPPLIER_NO_SUPP
+No Supplier Information available.
+.It Li RES_SUPPLIER_LLC_HITM
+M-state initial lookup stat in L3.
+.It Li RES_SUPPLIER_LLC_HITE
+E-state.
+.It Li RES_SUPPLIER_LLC_HITS
+S-state.
+.It Li RES_SUPPLIER_LLC_HITF
+F-state.
+.It Li RES_SUPPLIER_LOCAL
+Local DRAM Controller.
+.It Li RES_SNOOP_SNP_NONE
+No details on snoop-related information.
+.It Li RES_SNOOP_SNP_NO_NEEDED
+No snoop was needed to satisfy the request.
+.It Li RES_SNOOP_SNP_MISS
+A snoop was needed and it missed all snooped caches:
+-For LLC Hit, ReslHitl was returned by all cores
+-For LLC Miss, Rspl was returned by all sockets and data was returned from
+DRAM.
+.It Li RES_SNOOP_HIT_NO_FWD
+A snoop was needed and it hits in at least one snooped cache. Hit denotes a
+cache-line was valid before snoop effect. This includes:
+-Snoop Hit w/ Invalidation (LLC Hit, RFO)
+-Snoop Hit, Left Shared (LLC Hit/Miss, IFetch/Data_RD)
+-Snoop Hit w/ Invalidation and No Forward (LLC Miss, RFO Hit S)
+In the LLC Miss case, data is returned from DRAM.
+.It Li RES_SNOOP_HIT_FWD
+A snoop was needed and data was forwarded from a remote socket.
+This includes:
+-Snoop Forward Clean, Left Shared (LLC Hit/Miss, IFetch/Data_RD/RFT).
+.It Li RES_SNOOP_HITM
+A snoop was needed and it HitM-ed in local or remote cache. HitM denotes a
+cache-line was in modified state before effect as a results of snoop. This
+includes:
+-Snoop HitM w/ WB (LLC miss, IFetch/Data_RD)
+-Snoop Forward Modified w/ Invalidation (LLC Hit/Miss, RFO)
+-Snoop MtoS (LLC Hit, IFetch/Data_RD).
+.It Li RES_NON_DRAM
+Target was non-DRAM system address. This includes MMIO transactions.
+.El
+.It Li cmask= Ns Ar value
+Configure the PMC to increment only if the number of configured
+events measured in a cycle is greater than or equal to
+.Ar value .
+.It Li edge
+Configure the PMC to count the number of de-asserted to asserted
+transitions of the conditions expressed by the other qualifiers.
+If specified, the counter will increment only once whenever a
+condition becomes true, irrespective of the number of clocks during
+which the condition remains true.
+.It Li inv
+Invert the sense of comparison when the
+.Dq Li cmask
+qualifier is present, making the counter increment when the number of
+events per cycle is less than the value specified by the
+.Dq Li cmask
+qualifier.
+.It Li os
+Configure the PMC to count events happening at processor privilege
+level 0.
+.It Li usr
+Configure the PMC to count events occurring at privilege levels 1, 2
+or 3.
+.El
+.Pp
+If neither of the
+.Dq Li os
+or
+.Dq Li usr
+qualifiers are specified, the default is to enable both.
+.Ss Event Specifiers (Programmable PMCs)
+Ivy Bridge programmable PMCs support the following events:
+.Bl -tag -width indent
+.It Li LD_BLOCKS.STORE_FORWARD
+.Pq Event 03H , Umask 02H
+loads blocked by overlapping with store buffer that cannot be forwarded .
+.It Li MISALIGN_MEM_REF.LOADS
+.Pq Event 05H , Umask 01H
+Speculative cache-line split load uops dispatched to L1D.
+.It Li MISALIGN_MEM_REF.STORES
+.Pq Event 05H , Umask 02H
+Speculative cache-line split Store- address uops dispatched to L1D.
+.It Li LD_BLOCKS_PARTIAL.ADDRESS_ALIAS
+.Pq Event 07H , Umask 01H
+False dependencies in MOB due to partial compare on address.
+.It Li DTLB_LOAD_MISSES.DEMAND_LD_MISS_CAUSES_A_WALK
+.Pq Event 08H , Umask 81H
+Misses in all TLB levels that cause a page walk of any page size from demand loads.
+.It Li DTLB_LOAD_MISSES.DEMAND_LD_WALK_COMPLETED
+.Pq Event 08H , Umask 82H
+Misses in all TLB levels that caused page walk completed of any size by demand loads.
+.It Li DTLB_LOAD_MISSES.DEMAND_LD_WALK_DURATION
+.Pq Event 08H , Umask 84H
+Cycle PMH is busy with a walk due to demand loads.
+.It Li UOPS_ISSUED.ANY
+.Pq Event 0EH , Umask 01H
+Increments each cycle the # of Uops issued by the RAT to RS.
+Set Cmask = 1, Inv = 1to count stalled cycles.
+Set Cmask = 1, Inv = 1, Any= 1to count stalled cycles of this core.
+.It Li UOPS_ISSUED.FLAGS_MERGE
+.Pq Event 0EH , Umask 10H
+Number of flags-merge uops allocated. Such uops adds delay.
+.It Li UOPS_ISSUED.SLOW_LEA
+.Pq Event 0EH , Umask 20H
+Number of slow LEA or similar uops allocated. Such uop has 3 sources (e.g. 2
+sources + immediate) regardless if as a result of LEA instruction or not.
+.It Li UOPS_ISSUED.SINGLE_MUL
+.Pq Event 0EH , Umask 40H
+Number of multiply packed/scalar single precision uops allocated.
+.It Li ARITH.FPU_DIV_ACTIVE
+.Pq Event 14H , Umask 01H
+Cycles that the divider is active, includes INT and FP. Set 'edge =1,
+cmask=1' to count the number of divides.
+.It Li L2_RQSTS.DEMAND_DATA_RD_HIT
+.Pq Event 24H , Umask 01H
+Demand Data Read requests that hit L2 cache.
+.It Li L2_RQSTS.ALL_DEMAND_DATA_RD
+.Pq Event 24H , Umask 03H
+Counts any demand and L1 HW prefetch data load requests to L2.
+.It Li L2_RQSTS.RFO_HITS
+.Pq Event 24H , Umask 04H
+Counts the number of store RFO requests that hit the L2 cache.
+.It Li L2_RQSTS.RFO_MISS
+.Pq Event 24H , Umask 08H
+Counts the number of store RFO requests that miss the L2 cache.
+.It Li L2_RQSTS.ALL_RFO
+.Pq Event 24H , Umask 0CH
+Counts all L2 store RFO requests.
+.It Li L2_RQSTS.CODE_RD_HIT
+.Pq Event 24H , Umask 10H
+Number of instruction fetches that hit the L2 cache.
+.It Li L2_RQSTS.CODE_RD_MISS
+.Pq Event 24H , Umask 20H
+Number of instruction fetches that missed the L2 cache.
+.It Li L2_RQSTS.ALL_CODE_RD
+.Pq Event 24H , Umask 30H
+Counts all L2 code requests.
+.It Li L2_RQSTS.PF_HIT
+.Pq Event 24H , Umask 40H
+Counts all L2 HW prefetcher requests that hit L2.
+.It Li L2_RQSTS.PF_MISS
+.Pq Event 24H , Umask 80H
+Counts all L2 HW prefetcher requests that missed L2.
+.It Li L2_RQSTS.ALL_PF
+.Pq Event 24H , Umask C0H
+Counts all L2 HW prefetcher requests.
+.It Li L2_STORE_LOCK_RQSTS.MISS
+.Pq Event 27H , Umask 01H
+RFOs that miss cache lines.
+.It Li L2_STORE_LOCK_RQSTS.HIT_M
+.Pq Event 27H , Umask 08H
+RFOs that hit cache lines in M state.
+.It Li L2_STORE_LOCK_RQSTS.ALL
+.Pq Event 27H , Umask 0FH
+RFOs that access cache lines in any state.
+.It Li L2_L1D_WB_RQSTS.MISS
+.Pq Event 28H , Umask 01H
+Not rejected writebacks that missed LLC.
+.It Li L2_L1D_WB_RQSTS.HIT_E
+.Pq Event 28H , Umask 04H
+Not rejected writebacks from L1D to L2 cache lines in E state.
+.It Li L2_L1D_WB_RQSTS.HIT_M
+.Pq Event 28H , Umask 08H
+Not rejected writebacks from L1D to L2 cache lines in M state.
+.It Li L2_L1D_WB_RQSTS.ALL
+.Pq Event 28H , Umask 0FH
+Not rejected writebacks from L1D to L2 cache lines in any state.
+.It Li LONGEST_LAT_CACHE.REFERENCE
+.Pq Event 2EH , Umask 4FH
+This event counts requests originating from the core that reference a cache
+line in the last level cache.
+.It Li LONGEST_LAT_CACHE.MISS
+.Pq Event 2EH , Umask 41H
+This event counts each cache miss condition for references to the last level
+cache.
+.It Li CPU_CLK_UNHALTED.THREAD_P
+.Pq Event 3CH , Umask 00H
+Counts the number of thread cycles while the thread is not in a halt state.
+The thread enters the halt state when it is running the HLT instruction. The
+core frequency may change from time to time due to power or thermal
+throttling.
+.It Li CPU_CLK_THREAD_UNHALTED.REF_XCLK
+.Pq Event 3CH , Umask 01H
+Increments at the frequency of XCLK (100 MHz) when not halted.
+.It Li L1D_PEND_MISS.PENDING
+.Pq Event 48H , Umask 01H
+Increments the number of outstanding L1D misses every cycle. Set Cmaks = 1
+and Edge =1 to count occurrences.
+Counter 2 only.
+Set Cmask = 1 to count cycles.
+.It Li DTLB_STORE_MISSES.MISS_CAUSES_A_WALK
+.Pq Event 49H , Umask 01H
+Miss in all TLB levels causes an page walk of any page size (4K/2M/4M/1G).
+.It Li DTLB_STORE_MISSES.WALK_COMPLETED
+.Pq Event 49H , Umask 02H
+Miss in all TLB levels causes a page walk that completes of any page size
+(4K/2M/4M/1G).
+.It Li DTLB_STORE_MISSES.WALK_DURATION
+.Pq Event 49H , Umask 04H
+Cycles PMH is busy with this walk.
+.It Li DTLB_STORE_MISSES.STLB_HIT
+.Pq Event 49H , Umask 10H
+Store operations that miss the first TLB level but hit the second and do not
+cause page walks.
+.It Li LOAD_HIT_PRE.SW_PF
+.Pq Event 4CH , Umask 01H
+Non-SW-prefetch load dispatches that hit fill buffer allocated for S/W prefetch.
+.It Li LOAD_HIT_PRE.HW_PF
+.Pq Event 4CH , Umask 02H
+Non-SW-prefetch load dispatches that hit fill buffer allocated for H/W prefetch.
+.It Li L1D.REPLACEMENT
+.Pq Event 51H , Umask 01H
+Counts the number of lines brought into the L1 data cache.
+.It Li MOVE_ELIMINATION.INT_NOT_ELIMINATED
+.Pq Event 58H , Umask 01H
+Number of integer Move Elimination candidate uops that were not eliminated.
+.It Li MOVE_ELIMINATION.SIMD_NOT_ELIMINATED
+.Pq Event 58H , Umask 02H
+Number of SIMD Move Elimination candidate uops that were not eliminated.
+.It Li MOVE_ELIMINATION.INT_ELIMINATED
+.Pq Event 58H , Umask 04H
+Number of integer Move Elimination candidate uops that were eliminated.
+.It Li MOVE_ELIMINATION.SIMD_ELIMINATED
+.Pq Event 58H , Umask 08H
+Number of SIMD Move Elimination candidate uops that were eliminated.
+.It Li CPL_CYCLES.RING0
+.Pq Event 5CH , Umask 01H
+Unhalted core cycles when the thread is in ring 0.
+Use Edge to count transition.
+.It Li CPL_CYCLES.RING123
+.Pq Event 5CH , Umask 02H
+Unhalted core cycles when the thread is not in ring 0.
+.It Li RS_EVENTS.EMPTY_CYCLES
+.Pq Event 5EH , Umask 01H
+Cycles the RS is empty for the thread.
+.It Li DTLB_LOAD_MISSES.STLB_HIT
+.Pq Event 5FH , Umask 04H
+Counts load operations that missed 1st level DTLB but hit the 2nd level.
+.It Li OFFCORE_REQUESTS_OUTSTANDING.DEMAND_DATA_RD
+.Pq Event 60H , Umask 01H
+Offcore outstanding Demand Data Read transactions in SQ to uncore. Set
+Cmask=1 to count cycles.
+.It Li OFFCORE_REQUESTS_OUTSTANDING.DEMAND_CODE_RD
+.Pq Event 60H , Umask 02H
+Offcore outstanding Demand Code Read transactions in SQ to uncore. Set
+Cmask=1 to count cycles.
+.It Li OFFCORE_REQUESTS_OUTSTANDING.DEMAND_RFO
+.Pq Event 60H , Umask 04H
+Offcore outstanding RFO store transactions in SQ to uncore. Set Cmask=1 to
+count cycles.
+.It Li OFFCORE_REQUESTS_OUTSTANDING.ALL_DATA_RD
+.Pq Event 60H , Umask 08H
+Offcore outstanding cacheable data read transactions in SQ to uncore. Set
+Cmask=1 to count cycles.
+.It Li LOCK_CYCLES.SPLIT_LOCK_UC_LOCK_DURATION
+.Pq Event 63H , Umask 01H
+Cycles in which the L1D and L2 are locked, due to a UC lock or split lock.
+.It Li LOCK_CYCLES.CACHE_LOCK_DURATION
+.Pq Event 63H , Umask 02H
+Cycles in which the L1D is locked.
+.It Li IDQ.EMPTY
+.Pq Event 79H , Umask 02H
+Counts cycles the IDQ is empty.
+.It Li IDQ.MITE_UOPS
+.Pq Event 79H , Umask 04H
+Increment each cycle # of uops delivered to IDQ from MITE path.
+Can combine Umask 04H and 20H.
+Set Cmask = 1 to count cycles.
+.It Li IDQ.DSB_UOPS
+.Pq Event 79H , Umask 08H
+Increment each cycle. # of uops delivered to IDQ from DSB path.
+Can combine Umask 08H and 10H
+Set Cmask = 1 to count cycles.
+.It Li IDQ.MS_DSB_UOPS
+.Pq Event 79H , Umask 10H
+Increment each cycle # of uops delivered to IDQ when MS_busy by DSB. Set
+Cmask = 1 to count cycles. Add Edge=1 to count # of delivery.
+Can combine Umask 04H, 08H.
+.It Li IDQ.MS_MITE_UOPS
+.Pq Event 79H , Umask 20H
+Increment each cycle # of uops delivered to IDQ when MS_busy by MITE. Set
+Cmask = 1 to count cycles.
+Can combine Umask 04H, 08H.
+.It Li IDQ.MS_UOPS
+.Pq Event 79H , Umask 30H
+Increment each cycle # of uops delivered to IDQ from MS by either DSB or
+MITE. Set Cmask = 1 to count cycles.
+Can combine Umask 04H, 08H.
+.It Li IDQ.ALL_DSB_CYCLES_ANY_UOPS
+.Pq Event 79H , Umask 18H
+Counts cycles DSB is delivered at least one uops. Set Cmask = 1.
+.It Li IDQ.ALL_DSB_CYCLES_4_UOPS
+.Pq Event 79H , Umask 18H
+Counts cycles DSB is delivered four uops. Set Cmask = 4.
+.It Li IDQ.ALL_MITE_CYCLES_ANY_UOPS
+.Pq Event 79H , Umask 24H
+Counts cycles MITE is delivered at least one uops. Set Cmask = 1.
+.It Li IDQ.ALL_MITE_CYCLES_4_UOPS
+.Pq Event 79H , Umask 24H
+Counts cycles MITE is delivered four uops. Set Cmask = 4.
+.It Li IDQ.MITE_ALL_UOPS
+.Pq Event 79H , Umask 3CH
+# of uops delivered to IDQ from any path.
+.It Li ICACHE.MISSES
+.Pq Event 80H , Umask 02H
+Number of Instruction Cache, Streaming Buffer and Victim Cache Misses.
+Includes UC accesses.
+.It Li ITLB_MISSES.MISS_CAUSES_A_WALK
+.Pq Event 85H , Umask 01H
+Misses in all ITLB levels that cause page walks.
+.It Li ITLB_MISSES.WALK_COMPLETED
+.Pq Event 85H , Umask 02H
+Misses in all ITLB levels that cause completed page walks.
+.It Li ITLB_MISSES.WALK_DURATION
+.Pq Event 85H , Umask 04H
+Cycle PMH is busy with a walk.
+.It Li ITLB_MISSES.STLB_HIT
+.Pq Event 85H , Umask 10H
+Number of cache load STLB hits. No page walk.
+.It Li ILD_STALL.LCP
+.Pq Event 87H , Umask 01H
+Stalls caused by changing prefix length of the instruction.
+.It Li ILD_STALL.IQ_FULL
+.Pq Event 87H , Umask 04H
+Stall cycles due to IQ is full.
+.It Li BR_INST_EXEC.COND
+.Pq Event 88H , Umask 01H
+Qualify conditional near branch instructions executed, but not necessarily
+retired.
+Must combine with umask 40H, 80H.
+.It Li BR_INST_EXEC.DIRECT_JMP
+.Pq Event 88H , Umask 02H
+Qualify all unconditional near branch instructions excluding calls and
+indirect branches.
+Must combine with umask 80H.
+.It Li BR_INST_EXEC.INDIRECT_JMP_NON_CALL_RET
+.Pq Event 88H , Umask 04H
+Qualify executed indirect near branch instructions that are not calls nor
+returns.
+Must combine with umask 80H.
+.It Li BR_INST_EXEC.RETURN_NEAR
+.Pq Event 88H , Umask 08H
+Qualify indirect near branches that have a return mnemonic.
+Must combine with umask 80H.
+.It Li BR_INST_EXEC.DIRECT_NEAR_CALL
+.Pq Event 88H , Umask 10H
+Qualify unconditional near call branch instructions, excluding non call
+branch, executed.
+Must combine with umask 80H.
+.It Li BR_INST_EXEC.INDIRECT_NEAR_CALL
+.Pq Event 88H , Umask 20H
+Qualify indirect near calls, including both register and memory indirect,
+executed.
+Must combine with umask 80H.
+.It Li BR_INST_EXEC.NONTAKEN
+.Pq Event 88H , Umask 40H
+Qualify non-taken near branches executed.
+Applicable to umask 01H only.
+.It Li BR_INST_EXEC.TAKEN
+.Pq Event 88H , Umask 80H
+Qualify taken near branches executed. Must combine with 01H,02H, 04H, 08H,
+10H, 20H.
+.It Li BR_INST_EXEC.ALL_BRANCHES
+.Pq Event 88H , Umask FFH
+Counts all near executed branches (not necessarily retired).
+.It Li BR_MISP_EXEC.COND
+.Pq Event 89H , Umask 01H
+Qualify conditional near branch instructions mispredicted.
+Must combine with umask 40H, 80H.
+.It Li BR_MISP_EXEC.INDIRECT_JMP_NON_CALL_RET
+.Pq Event 89H , Umask 04H
+Qualify mispredicted indirect near branch instructions that are not calls
+nor returns.
+Must combine with umask 80H.
+.It Li BR_MISP_EXEC.RETURN_NEAR
+.Pq Event 89H , Umask 08H
+Qualify mispredicted indirect near branches that have a return mnemonic.
+Must combine with umask 80H.
+.It Li BR_MISP_EXEC.DIRECT_NEAR_CALL
+.Pq Event 89H , Umask 10H
+Qualify mispredicted unconditional near call branch instructions, excluding
+non call branch, executed.
+Must combine with umask 80H.
+.It Li BR_MISP_EXEC.INDIRECT_NEAR_CALL
+.Pq Event 89H , Umask 20H
+Qualify mispredicted indirect near calls, including both register and memory
+indirect, executed.
+Must combine with umask 80H.
+.It Li BR_MISP_EXEC.NONTAKEN
+.Pq Event 89H , Umask 40H
+Qualify mispredicted non-taken near branches executed.
+Applicable to umask 01H only.
+.It Li BR_MISP_EXEC.TAKEN
+.Pq Event 89H , Umask 80H
+Qualify mispredicted taken near branches executed. Must combine with
+01H,02H, 04H, 08H, 10H, 20H.
+.It Li BR_MISP_EXEC.ALL_BRANCHES
+.Pq Event 89H , Umask FFH
+Counts all near executed branches (not necessarily retired).
+.It Li IDQ_UOPS_NOT_DELIVERED.CORE
+.Pq Event 9CH , Umask 01H
+Count number of non-delivered uops to RAT per thread.
+Use Cmask to qualify uop b/w.
+.It Li UOPS_DISPATCHED_PORT.PORT_0
+.Pq Event A1H , Umask 01H
+Cycles which a Uop is dispatched on port 0.
+.It Li UOPS_DISPATCHED_PORT.PORT_1
+.Pq Event A1H , Umask 02H
+Cycles which a Uop is dispatched on port 1.
+.It Li UOPS_DISPATCHED_PORT.PORT_2_LD
+.Pq Event A1H , Umask 04H
+Cycles which a load uop is dispatched on port 2.
+.It Li UOPS_DISPATCHED_PORT.PORT_2_STA
+.Pq Event A1H , Umask 08H
+Cycles which a store address uop is dispatched on port 2.
+.It Li UOPS_DISPATCHED_PORT.PORT_2
+.Pq Event A1H , Umask 0CH
+Cycles which a Uop is dispatched on port 2.
+.It Li UOPS_DISPATCHED_PORT.PORT_3_LD
+.Pq Event A1H , Umask 10H
+Cycles which a load uop is dispatched on port 3.
+.It Li UOPS_DISPATCHED_PORT.PORT_3_STA
+.Pq Event A1H , Umask 20H
+Cycles which a store address uop is dispatched on port 3.
+.It Li UOPS_DISPATCHED_PORT.PORT_3
+.Pq Event A1H , Umask 30H
+Cycles which a Uop is dispatched on port 3.
+.It Li UOPS_DISPATCHED_PORT.PORT_4
+.Pq Event A1H , Umask 40H
+Cycles which a Uop is dispatched on port 4.
+.It Li UOPS_DISPATCHED_PORT.PORT_5
+.Pq Event A1H , Umask 80H
+Cycles which a Uop is dispatched on port 5.
+.It Li RESOURCE_STALLS.ANY
+.Pq Event A2H , Umask 01H
+Cycles Allocation is stalled due to Resource Related reason.
+.It Li RESOURCE_STALLS.RS
+.Pq Event A2H , Umask 04H
+Cycles stalled due to no eligible RS entry available.
+.It Li RESOURCE_STALLS.SB
+.Pq Event A2H , Umask 08H
+Cycles stalled due to no store buffers available. (not including draining
+form sync).
+.It Li RESOURCE_STALLS.ROB
+.Pq Event A2H , Umask 10H
+Cycles stalled due to re-order buffer full.
+.It Li CYCLE_ACTIVITY.CYCLES_L2_PENDING
+.Pq Event A3H , Umask 01H
+Cycles with pending L2 miss loads. Set AnyThread to count per core.
+.It Li CYCLE_ACTIVITY.CYCLES_LDM_PENDING
+.Pq Event A3H , Umask 02H
+Cycles with pending memory loads. Set AnyThread to count per core.
+.It Li CYCLE_ACTIVITY.CYCLES_NO_EXECUTE
+.Pq Event A3H , Umask 04H
+Cycles of dispatch stalls. Set AnyThread to count per core.
+.It Li CYCLE_ACTIVITY.CYCLES_L1D_PENDING
+.Pq Event A3H , Umask 08H
+Cycles with pending L1 cache miss loads. Set AnyThread to count per core.
+.It Li DSB2MITE_SWITCHES.COUNT
+.Pq Event ABH , Umask 01H
+Number of DSB to MITE switches.
+.It Li DSB2MITE_SWITCHES.PENALTY_CYCLES
+.Pq Event ABH , Umask 02H
+Cycles DSB to MITE switches caused delay.
+.It Li DSB_FILL.EXCEED_DSB_LINES
+.Pq Event ACH , Umask 08H
+DSB Fill encountered > 3 DSB lines.
+.It Li ITLB.ITLB_FLUSH
+.Pq Event AEH , Umask 01H
+Counts the number of ITLB flushes, includes 4k/2M/4M pages.
+.It Li OFFCORE_REQUESTS.DEMAND_DATA_RD
+.Pq Event B0H , Umask 01H
+Demand data read requests sent to uncore.
+.It Li OFFCORE_REQUESTS.DEMAND_CODE_RD
+.Pq Event B0H , Umask 02H
+Demand code read requests sent to uncore.
+.It Li OFFCORE_REQUESTS.DEMAND_RFO
+.Pq Event B0H , Umask 04H
+Demand RFO read requests sent to uncore, including regular RFOs, locks,
+ItoM.
+.It Li OFFCORE_REQUESTS.ALL_DATA_RD
+.Pq Event B0H , Umask 08H
+Data read requests sent to uncore (demand and prefetch).
+.It Li UOPS_EXECUTED.THREAD
+.Pq Event B1H , Umask 01H
+Counts total number of uops to be executed per-thread each cycle. Set Cmask
+= 1, INV =1 to count stall cycles.
+.It Li UOPS_EXECUTED.CORE
+.Pq Event B1H , Umask 02H
+Counts total number of uops to be executed per-core each cycle.
+Do not need to set ANY.
+.It Li OFF_CORE_RESPONSE_0
+.Pq Event B7H , Umask 01H
+Off-core Response Performance Monitoring.
+PMC0 only.
+Requires programming MSR 01A6H.
+.It Li OFF_CORE_RESPONSE_1
+.Pq Event BBH , Umask 01H
+Off-core Response Performance Monitoring.
+PMC3 only.
+Requires programming MSR 01A7H.
+.It Li TLB_FLUSH.DTLB_THREAD
+.Pq Event BDH , Umask 01H
+DTLB flush attempts of the thread- specific entries.
+.It Li TLB_FLUSH.STLB_ANY
+.Pq Event BDH , Umask 20H
+Count number of STLB flush attempts.
+.It Li INST_RETIRED.ANY_P
+.Pq Event C0H , Umask 00H
+Number of instructions at retirement.
+.It Li INST_RETIRED.ALL
+.Pq Event C0H , Umask 01H
+Precise instruction retired event with HW to reduce effect of PEBS shadow in
+IP distribution.
+PMC1 only.
+Must quiesce other PMCs.
+.It Li OTHER_ASSISTS.AVX_STORE
+.Pq Event C1H , Umask 08H
+Number of assists associated with 256-bit AVX store operations.
+.It Li OTHER_ASSISTS.AVX_TO_SSE
+.Pq Event C1H , Umask 10H
+Number of transitions from AVX- 256 to legacy SSE when penalty applicable.
+.It Li OTHER_ASSISTS.SSE_TO_AVX
+.Pq Event C1H , Umask 20H
+Number of transitions from SSE to AVX-256 when penalty applicable.
+.It Li UOPS_RETIRED.ALL
+.Pq Event C2H , Umask 01H
+Counts the number of micro-ops retired, Use cmask=1 and invert to count
+active cycles or stalled cycles.
+Supports PEBS, use Any=1 for core granular.
+.It Li UOPS_RETIRED.RETIRE_SLOTS
+.Pq Event C2H , Umask 02H
+Counts the number of retirement slots used each cycle.
+.It Li MACHINE_CLEARS.MEMORY_ORDERING
+.Pq Event C3H , Umask 02H
+Counts the number of machine clears due to memory order conflicts.
+.It Li MACHINE_CLEARS.SMC
+.Pq Event C3H , Umask 04H
+Number of self-modifying-code machine clears detected.
+.It Li MACHINE_CLEARS.MASKMOV
+.Pq Event C3H , Umask 20H
+Counts the number of executed AVX masked load operations that refer to an
+illegal address range with the mask bits set to 0.
+.It Li BR_INST_RETIRED.ALL_BRANCHES
+.Pq Event C4H , Umask 00H
+Branch instructions at retirement.
+.It Li BR_INST_RETIRED.CONDITIONAL
+.Pq Event C4H , Umask 01H
+Counts the number of conditional branch instructions retired.
+Supports PEBS.
+.It Li BR_INST_RETIRED.NEAR_CALL
+.Pq Event C4H , Umask 02H
+Direct and indirect near call instructions retired.
+.It Li BR_INST_RETIRED.ALL_BRANCHES
+.Pq Event C4H , Umask 04H
+Counts the number of branch instructions retired.
+.It Li BR_INST_RETIRED.NEAR_RETURN
+.Pq Event C4H , Umask 08H
+Counts the number of near return instructions retired.
+.It Li BR_INST_RETIRED.NOT_TAKEN
+.Pq Event C4H , Umask 10H
+Counts the number of not taken branch instructions retired.
+.It Li BR_INST_RETIRED.NEAR_TAKEN
+.Pq Event C4H , Umask 20H
+Number of near taken branches retired.
+.It Li BR_INST_RETIRED.FAR_BRANCH
+.Pq Event C4H , Umask 40H
+Number of far branches retired.
+.It Li BR_MISP_RETIRED.ALL_BRANCHES
+.Pq Event C5H , Umask 00H
+Mispredicted branch instructions at retirement.
+.It Li BR_MISP_RETIRED.CONDITIONAL
+.Pq Event C5H , Umask 01H
+Mispredicted conditional branch instructions retired.
+Supports PEBS.
+.It Li BR_MISP_RETIRED.NEAR_CALL
+.Pq Event C5H , Umask 02H
+Direct and indirect mispredicted near call instructions retired.
+.It Li BR_MISP_RETIRED.ALL_BRANCHES
+.Pq Event C5H , Umask 04H
+Mispredicted macro branch instructions retired.
+.It Li BR_MISP_RETIRED.NOT_TAKEN
+.Pq Event C5H , Umask 10H
+Mispredicted not taken branch instructions retired.
+.It Li BR_MISP_RETIRED.TAKEN
+.Pq Event C5H , Umask 20H
+Mispredicted taken branch instructions retired.
+.It Li FP_ASSIST.X87_OUTPUT
+.Pq Event CAH , Umask 02H
+Number of X87 FP assists due to Output values.
+.It Li FP_ASSIST.X87_INPUT
+.Pq Event CAH , Umask 04H
+Number of X87 FP assists due to input values.
+.It Li FP_ASSIST.SIMD_OUTPUT
+.Pq Event CAH , Umask 08H
+Number of SIMD FP assists due to Output values.
+.It Li FP_ASSIST.SIMD_INPUT
+.Pq Event CAH , Umask 10H
+Number of SIMD FP assists due to input values.
+.It Li FP_ASSIST.ANY
+.Pq Event CAH , Umask 1EH
+Cycles with any input/output SSE* or FP assists.
+.It Li ROB_MISC_EVENTS.LBR_INSERTS
+.Pq Event CCH , Umask 20H
+Count cases of saving new LBR records by hardware.
+.It Li MEM_TRANS_RETIRED.LOAD_LATENCY
+.Pq Event CDH , Umask 01H
+Sample loads with specified latency threshold.
+PMC3 only.
+Specify threshold in MSR 0x3F6.
+.It Li MEM_TRANS_RETIRED.PRECISE_STORE
+.Pq Event CDH , Umask 02H
+Sample stores and collect precise store operation via PEBS record.
+PMC3 only.
+.It Li MEM_UOP_RETIRED.LOADS
+.Pq Event D0H , Umask 01H
+Qualify retired memory uops that are loads. Combine with umask 10H, 20H,
+40H, 80H.
+Supports PEBS.
+.It Li MEM_UOP_RETIRED.STORES
+.Pq Event D0H , Umask 02H
+Qualify retired memory uops that are stores. Combine with umask 10H, 20H,
+40H, 80H.
+.It Li MEM_UOP_RETIRED.STLB_MISS
+.Pq Event D0H , Umask 10H
+Qualify retired memory uops with STLB miss. Must combine with umask 01H,
+02H, to produce counts.
+.It Li MEM_UOP_RETIRED.LOCK
+.Pq Event D0H , Umask 20H
+Qualify retired memory uops with lock. Must combine with umask 01H, 02H, to
+produce counts.
+.It Li MEM_UOP_RETIRED.SPLIT
+.Pq Event D0H , Umask 40H
+Qualify retired memory uops with line split. Must combine with umask 01H,
+02H, to produce counts.
+.It Li MEM_UOP_RETIRED.ALL
+.Pq Event D0H , Umask 80H
+Qualify any retired memory uops. Must combine with umask 01H, 02H, to
+produce counts.
+.It Li MEM_LOAD_UOPS_RETIRED.L1_HIT
+.Pq Event D1H , Umask 01H
+Retired load uops with L1 cache hits as data sources.
+Supports PEBS.
+.It Li MEM_LOAD_UOPS_RETIRED.L2_HIT
+.Pq Event D1H , Umask 02H
+Retired load uops with L2 cache hits as data sources.
+.It Li MEM_LOAD_UOPS_RETIRED.LLC_HIT
+.Pq Event D1H , Umask 04H
+Retired load uops whose data source was LLC hit with no snoop required.
+.It Li MEM_LOAD_UOPS_RETIRED.LLC_MISS
+.Pq Event D1H , Umask 20H
+Retired load uops whose data source is LLC miss.
+.It Li MEM_LOAD_UOPS_RETIRED.HIT_LFB
+.Pq Event D1H , Umask 40H
+Retired load uops which data sources were load uops missed L1 but hit FB due
+to preceding miss to the same cache line with data not ready.
+.It Li MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_MISS
+.Pq Event D2H , Umask 01H
+Retired load uops which data sources were LLC hit and cross-core snoop
+missed in on-pkg core cache.
+Supports PEBS.
+.It Li MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HIT
+.Pq Event D2H , Umask 02H
+Retired load uops which data sources were LLC and cross-core snoop hits in
+on-pkg core cache.
+Supports PEBS.
+.It Li MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_HITM
+.Pq Event D2H , Umask 04H
+Retired load uops which data sources were HitM responses from shared LLC.
+.It Li MEM_LOAD_UOPS_LLC_HIT_RETIRED.XSNP_NONE
+.Pq Event D2H , Umask 08H
+Retired load uops which data sources were hits in LLC without snoops
+required.
+.It Li MEM_LOAD_UOPS_LLC_MISS_RETIRED.LOCAL_DRAM
+.Pq Event D3H , Umask 01H
+Retired load uops which data sources missed LLC but serviced from local
+dram.
+Supports PEBS.
+.It Li MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_DRAM
+.Pq Event D3H , Umask 04H
+Retired load uops whose data source was remote DRAM.
+.It Li MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_HITM
+.Pq Event D3H , Umask 10H
+Retired load uops whose data source was remote HITM.
+.It Li MEM_LOAD_UOPS_LLC_MISS_RETIRED.REMOTE_FWD
+.Pq Event D3H , Umask 20H
+Retired load uops whose data source was forwards from a remote cache.
+.It Li BACLEARS.ANY
+.Pq Event E6H , Umask 1FH
+Number of front end re-steers due to BPU misprediction.
+.It Li L2_TRANS.DEMAND_DATA_RD
+.Pq Event F0H , Umask 01H
+Demand Data Read requests that access L2 cache.
+.It Li L2_TRANS.RFO
+.Pq Event F0H , Umask 02H
+RFO requests that access L2 cache.
+.It Li L2_TRANS.CODE_RD
+.Pq Event F0H , Umask 04H
+L2 cache accesses when fetching instructions.
+.It Li L2_TRANS.ALL_PF
+.Pq Event F0H , Umask 08H
+Any MLC or LLC HW prefetch accessing L2, including rejects.
+.It Li L2_TRANS.L1D_WB
+.Pq Event F0H , Umask 10H
+L1D writebacks that access L2 cache.
+.It Li L2_TRANS.L2_FILL
+.Pq Event F0H , Umask 20H
+L2 fill requests that access L2 cache.
+.It Li L2_TRANS.L2_WB
+.Pq Event F0H , Umask 40H
+L2 writebacks that access L2 cache.
+.It Li L2_TRANS.ALL_REQUESTS
+.Pq Event F0H , Umask 80H
+Transactions accessing L2 pipe.
+.It Li L2_LINES_IN.I
+.Pq Event F1H , Umask 01H
+L2 cache lines in I state filling L2.
+Counting does not cover rejects.
+.It Li L2_LINES_IN.S
+.Pq Event F1H , Umask 02H
+L2 cache lines in S state filling L2.
+Counting does not cover rejects.
+.It Li L2_LINES_IN.E
+.Pq Event F1H , Umask 04H
+L2 cache lines in E state filling L2.
+Counting does not cover rejects.
+.It Li L2_LINES_IN.ALL
+.Pq Event F1H , Umask 07H
+L2 cache lines filling L2.
+Counting does not cover rejects.
+.It Li L2_LINES_OUT.DEMAND_CLEAN
+.Pq Event F2H , Umask 01H
+Clean L2 cache lines evicted by demand.
+.It Li L2_LINES_OUT.DEMAND_DIRTY
+.Pq Event F2H , Umask 02H
+Dirty L2 cache lines evicted by demand.
+.It Li L2_LINES_OUT.PF_CLEAN
+.Pq Event F2H , Umask 04H
+Clean L2 cache lines evicted by the MLC prefetcher.
+.It Li L2_LINES_OUT.PF_DIRTY
+.Pq Event F2H , Umask 08H
+Dirty L2 cache lines evicted by the MLC prefetcher.
+.It Li L2_LINES_OUT.DIRTY_ALL
+.Pq Event F2H , Umask 0AH
+Dirty L2 cache lines filling the L2.
+.El
+.Sh SEE ALSO
+.Xr pmc 3 ,
+.Xr pmc.atom 3 ,
+.Xr pmc.core 3 ,
+.Xr pmc.iaf 3 ,
+.Xr pmc.ucf 3 ,
+.Xr pmc.k7 3 ,
+.Xr pmc.k8 3 ,
+.Xr pmc.p4 3 ,
+.Xr pmc.p5 3 ,
+.Xr pmc.p6 3 ,
+.Xr pmc.corei7 3 ,
+.Xr pmc.corei7uc 3 ,
+.Xr pmc.ivybridge 3 ,
+.Xr pmc.sandybridge 3 ,
+.Xr pmc.sandybridgeuc 3 ,
+.Xr pmc.sandybridgexeon 3 ,
+.Xr pmc.westmere 3 ,
+.Xr pmc.westmereuc 3 ,
+.Xr pmc.soft 3 ,
+.Xr pmc.tsc 3 ,
+.Xr pmc_cpuinfo 3 ,
+.Xr pmclog 3 ,
+.Xr hwpmc 4
+.Sh HISTORY
+The
+.Nm pmc
+library first appeared in
+.Fx 6.0 .
+.Sh AUTHORS
+The
+.Lb libpmc
+library was written by
+.An "Joseph Koshy"
+.Aq jkoshy@FreeBSD.org .
+The support for the Ivy Bridge Xeon
+microarchitecture was written by
+.An "Hiren Panchasara"
+.Aq hiren.panchasara@gmail.com .
diff --git a/lib/libpmc/pmc.sandybridge.3 b/lib/libpmc/pmc.sandybridge.3
index 62f8f28..931bd9d 100644
--- a/lib/libpmc/pmc.sandybridge.3
+++ b/lib/libpmc/pmc.sandybridge.3
@@ -926,6 +926,8 @@ Split locks in SQ.
 .Xr pmc.corei7 3 ,
 .Xr pmc.corei7uc 3 ,
 .Xr pmc.iaf 3 ,
+.Xr pmc.ivybridge 3 ,
+.Xr pmc.ivybridgexeon 3 ,
 .Xr pmc.k7 3 ,
 .Xr pmc.k8 3 ,
 .Xr pmc.p4 3 ,
diff --git a/lib/libpmc/pmc.sandybridgexeon.3 b/lib/libpmc/pmc.sandybridgexeon.3
index c62a82b..0ddaeda 100644
--- a/lib/libpmc/pmc.sandybridgexeon.3
+++ b/lib/libpmc/pmc.sandybridgexeon.3
@@ -997,6 +997,7 @@ Split locks in SQ.
 .Xr pmc.corei7 3 ,
 .Xr pmc.corei7uc 3 ,
 .Xr pmc.ivybridge 3 ,
+.Xr pmc.ivybridgexeon 3 ,
 .Xr pmc.sandybridge 3 ,
 .Xr pmc.sandybridgeuc 3 ,
 .Xr pmc.westmere 3 ,
-- 
cgit v1.1


From a3f8831b82dd5a08661d27eca5752267505d1ab3 Mon Sep 17 00:00:00 2001
From: gahr <gahr@FreeBSD.org>
Date: Fri, 1 Feb 2013 13:04:06 +0000
Subject: - Fix more style(9)-related issues (copyright header, spaces after
 function   names, unnecessary casts) - Change type of boolean variable from
 char to bool

Suggested by:	jhb, zont, jmallett
Reviewed by:	cognet
Approved by:	cognet
---
 lib/libc/stdio/fmemopen.c | 111 +++++++++++++++++++++++-----------------------
 1 file changed, 55 insertions(+), 56 deletions(-)

(limited to 'lib')

diff --git a/lib/libc/stdio/fmemopen.c b/lib/libc/stdio/fmemopen.c
index dec2c86..581a91e 100644
--- a/lib/libc/stdio/fmemopen.c
+++ b/lib/libc/stdio/fmemopen.c
@@ -1,32 +1,33 @@
 /*-
-Copyright (C) 2013 Pietro Cerutti <gahr@FreeBSD.org>
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions
-are met:
-1. Redistributions of source code must retain the above copyright
-   notice, this list of conditions and the following disclaimer.
-2. Redistributions in binary form must reproduce the above copyright
-   notice, this list of conditions and the following disclaimer in the
-   documentation and/or other materials provided with the distribution.
-
-THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
-OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
-OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-SUCH DAMAGE.
-*/
+ * Copyright (C) 2013 Pietro Cerutti <gahr@FreeBSD.org>
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
 
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
 #include <fcntl.h>
+#include <stdbool.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -36,36 +37,36 @@ __FBSDID("$FreeBSD$");
 struct fmemopen_cookie
 {
 	char	*buf;	/* pointer to the memory region */
-	char	 own;	/* did we allocate the buffer ourselves? */
+	bool	 own;	/* did we allocate the buffer ourselves? */
 	char     bin;   /* is this a binary buffer? */
 	size_t	 size;	/* buffer length in bytes */
 	size_t	 len;	/* data length in bytes */
 	size_t	 off;	/* current offset into the buffer */
 };
 
-static int	fmemopen_read  (void *cookie, char *buf, int nbytes);
-static int	fmemopen_write (void *cookie, const char *buf, int nbytes);
-static fpos_t	fmemopen_seek  (void *cookie, fpos_t offset, int whence);
-static int	fmemopen_close (void *cookie);
+static int	fmemopen_read(void *cookie, char *buf, int nbytes);
+static int	fmemopen_write(void *cookie, const char *buf, int nbytes);
+static fpos_t	fmemopen_seek(void *cookie, fpos_t offset, int whence);
+static int	fmemopen_close(void *cookie);
 
 FILE *
-fmemopen (void * __restrict buf, size_t size, const char * __restrict mode)
+fmemopen(void * __restrict buf, size_t size, const char * __restrict mode)
 {
 	struct fmemopen_cookie *ck;
 	FILE *f;
 	int flags, rc;
 
-	/* 
+	/*
 	 * Retrieve the flags as used by open(2) from the mode argument, and
 	 * validate them.
-	 * */
-	rc = __sflags (mode, &flags);
+	 */
+	rc = __sflags(mode, &flags);
 	if (rc == 0) {
 		errno = EINVAL;
 		return (NULL);
 	}
 
-	/* 
+	/*
 	 * There's no point in requiring an automatically allocated buffer
 	 * in write-only mode.
 	 */
@@ -74,8 +75,7 @@ fmemopen (void * __restrict buf, size_t size, const char * __restrict mode)
 		return (NULL);
 	}
 	
-	/* Allocate a cookie. */
-	ck = malloc (sizeof (struct fmemopen_cookie));
+	ck = malloc(sizeof(struct fmemopen_cookie));
 	if (ck == NULL) {
 		return (NULL);
 	}
@@ -86,9 +86,9 @@ fmemopen (void * __restrict buf, size_t size, const char * __restrict mode)
 	/* Check whether we have to allocate the buffer ourselves. */
 	ck->own = ((ck->buf = buf) == NULL);
 	if (ck->own) {
-		ck->buf = malloc (size);
+		ck->buf = malloc(size);
 		if (ck->buf == NULL) {
-			free (ck);
+			free(ck);
 			return (NULL);
 		}
 	}
@@ -107,22 +107,22 @@ fmemopen (void * __restrict buf, size_t size, const char * __restrict mode)
 	/*
 	 * The size of the current buffer contents is set depending on the
 	 * mode:
-	 *
+	 * 
 	 * for append (text-mode), the position of the first NULL byte, or the
 	 * size of the buffer if none is found
 	 *
 	 * for append (binary-mode), the size of the buffer
-	 *
+	 * 
 	 * for read, the size of the buffer
-	 *
+	 * 
 	 * for write, 0
 	 */
 	switch (mode[0]) {
 	case 'a':
 		if (ck->bin) {
-			/* 
-			 * This isn't useful, since the buffer isn't
-			 * allowed to grow.
+			/*
+			 * This isn't useful, since the buffer isn't allowed
+			 * to grow.
 			 */
 			ck->off = ck->len = size;
 		} else
@@ -136,16 +136,15 @@ fmemopen (void * __restrict buf, size_t size, const char * __restrict mode)
 		break;
 	}
 
-	/* Actuall wrapper. */
-	f = funopen ((void *)ck,
+	f = funopen(ck,
 	    flags & O_WRONLY ? NULL : fmemopen_read, 
 	    flags & O_RDONLY ? NULL : fmemopen_write,
 	    fmemopen_seek, fmemopen_close);
 
 	if (f == NULL) {
 		if (ck->own)
-			free (ck->buf);
-		free (ck);
+			free(ck->buf);
+		free(ck);
 		return (NULL);
 	}
 
@@ -153,13 +152,13 @@ fmemopen (void * __restrict buf, size_t size, const char * __restrict mode)
 	 * Turn off buffering, so a write past the end of the buffer
 	 * correctly returns a short object count.
 	 */
-	setvbuf (f, (char *) NULL, _IONBF, 0);
+	setvbuf(f, NULL, _IONBF, 0);
 
 	return (f);
 }
 
 static int
-fmemopen_read (void *cookie, char *buf, int nbytes)
+fmemopen_read(void *cookie, char *buf, int nbytes)
 {
 	struct fmemopen_cookie *ck = cookie;
 
@@ -169,7 +168,7 @@ fmemopen_read (void *cookie, char *buf, int nbytes)
 	if (nbytes == 0)
 		return (0);
 
-	memcpy (buf, ck->buf + ck->off, nbytes);
+	memcpy(buf, ck->buf + ck->off, nbytes);
 
 	ck->off += nbytes;
 
@@ -177,7 +176,7 @@ fmemopen_read (void *cookie, char *buf, int nbytes)
 }
 
 static int
-fmemopen_write (void *cookie, const char *buf, int nbytes)
+fmemopen_write(void *cookie, const char *buf, int nbytes)
 {
 	struct fmemopen_cookie *ck = cookie;
 
@@ -187,7 +186,7 @@ fmemopen_write (void *cookie, const char *buf, int nbytes)
 	if (nbytes == 0)
 		return (0);
 
-	memcpy (ck->buf + ck->off, buf, nbytes);
+	memcpy(ck->buf + ck->off, buf, nbytes);
 
 	ck->off += nbytes;
 
@@ -207,7 +206,7 @@ fmemopen_write (void *cookie, const char *buf, int nbytes)
 }
 
 static fpos_t
-fmemopen_seek (void *cookie, fpos_t offset, int whence)
+fmemopen_seek(void *cookie, fpos_t offset, int whence)
 {
 	struct fmemopen_cookie *ck = cookie;
 
@@ -246,14 +245,14 @@ fmemopen_seek (void *cookie, fpos_t offset, int whence)
 }
 
 static int
-fmemopen_close (void *cookie)
+fmemopen_close(void *cookie)
 {
 	struct fmemopen_cookie *ck = cookie;
 
 	if (ck->own)
-		free (ck->buf);
+		free(ck->buf);
 
-	free (ck);
+	free(ck);
 
 	return (0);
 }
-- 
cgit v1.1


From 70538df621421ce04b5c5d7e1f5b62d5a7f9fc7a Mon Sep 17 00:00:00 2001
From: dim <dim@FreeBSD.org>
Date: Sat, 2 Feb 2013 22:28:29 +0000
Subject: Pull in r170135 from upstream clang trunk:

  Dont use/link ARCMT, StaticAnalyzer and Rewriter to clang when the user
  specifies not to. Dont build ASTMatchers with Rewriter disabled and
  StaticAnalyzer when it's disabled.

  Without all those three, the clang binary shrinks (x86_64) from ~36MB
  to ~32MB (unstripped).

To disable these clang components, and get a smaller clang binary built
and installed, set WITHOUT_CLANG_FULL in src.conf(5).  During the
initial stages of buildworld, those extra components are already
disabled automatically, to save some build time.

MFC after:	1 week
---
 lib/clang/Makefile                                | 25 +++++++++++++++--------
 lib/clang/clang.build.mk                          |  6 ++++++
 lib/clang/libclanganalysis/Makefile               |  2 ++
 lib/clang/libclangarcmigrate/Makefile             |  2 ++
 lib/clang/libclangast/Makefile                    |  2 ++
 lib/clang/libclangbasic/Makefile                  |  2 ++
 lib/clang/libclangcodegen/Makefile                |  2 ++
 lib/clang/libclangdriver/Makefile                 |  2 ++
 lib/clang/libclangedit/Makefile                   |  2 ++
 lib/clang/libclangfrontend/Makefile               |  2 ++
 lib/clang/libclangfrontendtool/Makefile           |  2 ++
 lib/clang/libclanglex/Makefile                    |  2 ++
 lib/clang/libclangparse/Makefile                  |  2 ++
 lib/clang/libclangrewritecore/Makefile            |  2 ++
 lib/clang/libclangrewritefrontend/Makefile        |  2 ++
 lib/clang/libclangsema/Makefile                   |  2 ++
 lib/clang/libclangserialization/Makefile          |  2 ++
 lib/clang/libclangstaticanalyzercheckers/Makefile |  2 ++
 lib/clang/libclangstaticanalyzercore/Makefile     |  2 ++
 lib/clang/libclangstaticanalyzerfrontend/Makefile |  2 ++
 lib/clang/libllvmarchive/Makefile                 |  2 ++
 lib/clang/libllvmarmasmparser/Makefile            |  2 ++
 lib/clang/libllvmarmcodegen/Makefile              |  2 ++
 lib/clang/libllvmarmdesc/Makefile                 |  2 ++
 lib/clang/libllvmarmdisassembler/Makefile         |  2 ++
 lib/clang/libllvmarminfo/Makefile                 |  2 ++
 lib/clang/libllvmarminstprinter/Makefile          |  2 ++
 lib/clang/libllvmasmparser/Makefile               |  2 ++
 lib/clang/libllvmasmprinter/Makefile              |  2 ++
 lib/clang/libllvmbitreader/Makefile               |  2 ++
 lib/clang/libllvmbitwriter/Makefile               |  2 ++
 lib/clang/libllvmcodegen/Makefile                 |  2 ++
 lib/clang/libllvmcore/Makefile                    |  2 ++
 lib/clang/libllvmdebuginfo/Makefile               |  2 ++
 lib/clang/libllvmexecutionengine/Makefile         |  2 ++
 lib/clang/libllvminstcombine/Makefile             |  2 ++
 lib/clang/libllvminstrumentation/Makefile         |  2 ++
 lib/clang/libllvminterpreter/Makefile             |  2 ++
 lib/clang/libllvmjit/Makefile                     |  2 ++
 lib/clang/libllvmlinker/Makefile                  |  2 ++
 lib/clang/libllvmmcdisassembler/Makefile          |  2 ++
 lib/clang/libllvmmcparser/Makefile                |  2 ++
 lib/clang/libllvmmipsasmparser/Makefile           |  2 ++
 lib/clang/libllvmmipscodegen/Makefile             |  2 ++
 lib/clang/libllvmmipsdesc/Makefile                |  2 ++
 lib/clang/libllvmmipsdisassembler/Makefile        |  2 ++
 lib/clang/libllvmmipsinfo/Makefile                |  2 ++
 lib/clang/libllvmmipsinstprinter/Makefile         |  2 ++
 lib/clang/libllvmobject/Makefile                  |  2 ++
 lib/clang/libllvmpowerpccodegen/Makefile          |  2 ++
 lib/clang/libllvmpowerpcdesc/Makefile             |  2 ++
 lib/clang/libllvmpowerpcinfo/Makefile             |  2 ++
 lib/clang/libllvmpowerpcinstprinter/Makefile      |  2 ++
 lib/clang/libllvmruntimedyld/Makefile             |  2 ++
 lib/clang/libllvmselectiondag/Makefile            |  2 ++
 lib/clang/libllvmtablegen/Makefile                |  2 ++
 lib/clang/libllvmtarget/Makefile                  |  2 ++
 lib/clang/libllvmvectorize/Makefile               |  2 ++
 lib/clang/libllvmx86asmparser/Makefile            |  2 ++
 lib/clang/libllvmx86codegen/Makefile              |  2 ++
 lib/clang/libllvmx86desc/Makefile                 |  2 ++
 lib/clang/libllvmx86info/Makefile                 |  2 ++
 lib/clang/libllvmx86instprinter/Makefile          |  2 ++
 lib/clang/libllvmx86utils/Makefile                |  2 ++
 64 files changed, 147 insertions(+), 8 deletions(-)

(limited to 'lib')

diff --git a/lib/clang/Makefile b/lib/clang/Makefile
index cfc1554..a77d241 100644
--- a/lib/clang/Makefile
+++ b/lib/clang/Makefile
@@ -3,8 +3,20 @@
 .include <bsd.own.mk>
 
 .if !make(install)
+.if !defined(EARLY_BUILD) && defined(MK_CLANG_FULL) && ${MK_CLANG_FULL} != "no"
+_libclangstaticanalyzer= \
+	libclangstaticanalyzercheckers \
+	libclangstaticanalyzercore \
+	libclangstaticanalyzerfrontend
+_libclangarcmigrate= \
+	libclangarcmigrate
+_libclangrewriter= \
+	libclangrewritecore \
+	libclangrewritefrontend
+.endif # !EARLY_BUILD && MK_CLANG_FULL
+
 SUBDIR=	libclanganalysis \
-	libclangarcmigrate \
+	${_libclangarcmigrate} \
 	libclangast \
 	libclangbasic \
 	libclangcodegen \
@@ -14,13 +26,10 @@ SUBDIR=	libclanganalysis \
 	libclangfrontendtool \
 	libclanglex \
 	libclangparse \
-	libclangrewritecore \
-	libclangrewritefrontend \
+	${_libclangrewriter} \
 	libclangsema \
 	libclangserialization \
-	libclangstaticanalyzercheckers \
-	libclangstaticanalyzercore \
-	libclangstaticanalyzerfrontend \
+	${_libclangstaticanalyzer} \
 	\
 	libllvmanalysis \
 	libllvmarchive \
@@ -78,8 +87,8 @@ SUBDIR+=libllvmdebuginfo \
 	libllvmmcdisassembler \
 	libllvmmcjit \
 	libllvmruntimedyld
-.endif
-.endif
+.endif # MK_CLANG_EXTRAS
+.endif # !make(install)
 
 SUBDIR+= include
 
diff --git a/lib/clang/clang.build.mk b/lib/clang/clang.build.mk
index 298a2eb..0015e3d 100644
--- a/lib/clang/clang.build.mk
+++ b/lib/clang/clang.build.mk
@@ -8,6 +8,12 @@ CFLAGS+=	-I${LLVM_SRCS}/include -I${CLANG_SRCS}/include \
 		-DLLVM_ON_UNIX -DLLVM_ON_FREEBSD \
 		-D__STDC_LIMIT_MACROS -D__STDC_CONSTANT_MACROS #-DNDEBUG
 
+.if !defined(EARLY_BUILD) && defined(MK_CLANG_FULL) && ${MK_CLANG_FULL} != "no"
+CFLAGS+=	-DCLANG_ENABLE_ARCMT \
+		-DCLANG_ENABLE_REWRITER \
+		-DCLANG_ENABLE_STATIC_ANALYZER
+.endif # !EARLY_BUILD && MK_CLANG_FULL
+
 # LLVM is not strict aliasing safe as of 12/31/2011
 CFLAGS+=	-fno-strict-aliasing
 
diff --git a/lib/clang/libclanganalysis/Makefile b/lib/clang/libclanganalysis/Makefile
index 61c45be..76bf1e5 100644
--- a/lib/clang/libclanganalysis/Makefile
+++ b/lib/clang/libclanganalysis/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	clanganalysis
 
 SRCDIR=	tools/clang/lib/Analysis
diff --git a/lib/clang/libclangarcmigrate/Makefile b/lib/clang/libclangarcmigrate/Makefile
index 6544805..7597e9c 100644
--- a/lib/clang/libclangarcmigrate/Makefile
+++ b/lib/clang/libclangarcmigrate/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	clangarcmigrate
 
 SRCDIR=	tools/clang/lib/ARCMigrate
diff --git a/lib/clang/libclangast/Makefile b/lib/clang/libclangast/Makefile
index 0b851e6..0ddeb53 100644
--- a/lib/clang/libclangast/Makefile
+++ b/lib/clang/libclangast/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	clangast
 
 SRCDIR=	tools/clang/lib/AST
diff --git a/lib/clang/libclangbasic/Makefile b/lib/clang/libclangbasic/Makefile
index f0efd74..eec941a 100644
--- a/lib/clang/libclangbasic/Makefile
+++ b/lib/clang/libclangbasic/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	clangbasic
 
 SRCDIR=	tools/clang/lib/Basic
diff --git a/lib/clang/libclangcodegen/Makefile b/lib/clang/libclangcodegen/Makefile
index 5252e7c..7aefa7c 100644
--- a/lib/clang/libclangcodegen/Makefile
+++ b/lib/clang/libclangcodegen/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	clangcodegen
 
 SRCDIR=	tools/clang/lib/CodeGen
diff --git a/lib/clang/libclangdriver/Makefile b/lib/clang/libclangdriver/Makefile
index 4950cfa..aae2a05 100644
--- a/lib/clang/libclangdriver/Makefile
+++ b/lib/clang/libclangdriver/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	clangdriver
 
 SRCDIR=	tools/clang/lib/Driver
diff --git a/lib/clang/libclangedit/Makefile b/lib/clang/libclangedit/Makefile
index a0e45a9..59b0b54 100644
--- a/lib/clang/libclangedit/Makefile
+++ b/lib/clang/libclangedit/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	clangedit
 
 SRCDIR=	tools/clang/lib/Edit
diff --git a/lib/clang/libclangfrontend/Makefile b/lib/clang/libclangfrontend/Makefile
index c2813b4..44ba2fb 100644
--- a/lib/clang/libclangfrontend/Makefile
+++ b/lib/clang/libclangfrontend/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	clangfrontend
 
 SRCDIR=	tools/clang/lib/Frontend
diff --git a/lib/clang/libclangfrontendtool/Makefile b/lib/clang/libclangfrontendtool/Makefile
index f37979f..f20aa35 100644
--- a/lib/clang/libclangfrontendtool/Makefile
+++ b/lib/clang/libclangfrontendtool/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	clangfrontendtool
 
 SRCDIR=	tools/clang/lib/FrontendTool
diff --git a/lib/clang/libclanglex/Makefile b/lib/clang/libclanglex/Makefile
index 3ad0e3f..768d43c 100644
--- a/lib/clang/libclanglex/Makefile
+++ b/lib/clang/libclanglex/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	clanglex
 
 SRCDIR=	tools/clang/lib/Lex
diff --git a/lib/clang/libclangparse/Makefile b/lib/clang/libclangparse/Makefile
index 599a034..dbe9adb 100644
--- a/lib/clang/libclangparse/Makefile
+++ b/lib/clang/libclangparse/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	clangparse
 
 SRCDIR=	tools/clang/lib/Parse
diff --git a/lib/clang/libclangrewritecore/Makefile b/lib/clang/libclangrewritecore/Makefile
index 3a628d0..80115ee 100644
--- a/lib/clang/libclangrewritecore/Makefile
+++ b/lib/clang/libclangrewritecore/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	clangrewritecore
 
 SRCDIR=	tools/clang/lib/Rewrite/Core
diff --git a/lib/clang/libclangrewritefrontend/Makefile b/lib/clang/libclangrewritefrontend/Makefile
index 35de008..ffa11b4 100644
--- a/lib/clang/libclangrewritefrontend/Makefile
+++ b/lib/clang/libclangrewritefrontend/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	clangrewritefrontend
 
 SRCDIR=	tools/clang/lib/Rewrite/Frontend
diff --git a/lib/clang/libclangsema/Makefile b/lib/clang/libclangsema/Makefile
index 235fa13..9ced8b4 100644
--- a/lib/clang/libclangsema/Makefile
+++ b/lib/clang/libclangsema/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	clangsema
 
 SRCDIR=	tools/clang/lib/Sema
diff --git a/lib/clang/libclangserialization/Makefile b/lib/clang/libclangserialization/Makefile
index 75f68bf..962bbf0 100644
--- a/lib/clang/libclangserialization/Makefile
+++ b/lib/clang/libclangserialization/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	clangserialization
 
 SRCDIR=	tools/clang/lib/Serialization
diff --git a/lib/clang/libclangstaticanalyzercheckers/Makefile b/lib/clang/libclangstaticanalyzercheckers/Makefile
index 96274d9..9536151 100644
--- a/lib/clang/libclangstaticanalyzercheckers/Makefile
+++ b/lib/clang/libclangstaticanalyzercheckers/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	clangstaticanalyzercheckers
 
 SRCDIR=	tools/clang/lib/StaticAnalyzer/Checkers
diff --git a/lib/clang/libclangstaticanalyzercore/Makefile b/lib/clang/libclangstaticanalyzercore/Makefile
index 82ff5a3..9c263f0 100644
--- a/lib/clang/libclangstaticanalyzercore/Makefile
+++ b/lib/clang/libclangstaticanalyzercore/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	clangstaticanalyzercore
 
 SRCDIR=	tools/clang/lib/StaticAnalyzer/Core
diff --git a/lib/clang/libclangstaticanalyzerfrontend/Makefile b/lib/clang/libclangstaticanalyzerfrontend/Makefile
index 5c7cd41..9e19f92 100644
--- a/lib/clang/libclangstaticanalyzerfrontend/Makefile
+++ b/lib/clang/libclangstaticanalyzerfrontend/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	clangstaticanalyzerfrontend
 
 SRCDIR=	tools/clang/lib/StaticAnalyzer/Frontend
diff --git a/lib/clang/libllvmarchive/Makefile b/lib/clang/libllvmarchive/Makefile
index a28f2f5..d553405 100644
--- a/lib/clang/libllvmarchive/Makefile
+++ b/lib/clang/libllvmarchive/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmarchive
 
 SRCDIR=	lib/Archive
diff --git a/lib/clang/libllvmarmasmparser/Makefile b/lib/clang/libllvmarmasmparser/Makefile
index c3d0d89..05e6263 100644
--- a/lib/clang/libllvmarmasmparser/Makefile
+++ b/lib/clang/libllvmarmasmparser/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmarmasmparser
 
 SRCDIR=	lib/Target/ARM/AsmParser
diff --git a/lib/clang/libllvmarmcodegen/Makefile b/lib/clang/libllvmarmcodegen/Makefile
index 6ae9251..6a10eaf 100644
--- a/lib/clang/libllvmarmcodegen/Makefile
+++ b/lib/clang/libllvmarmcodegen/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmarmcodegen
 
 SRCDIR=	lib/Target/ARM
diff --git a/lib/clang/libllvmarmdesc/Makefile b/lib/clang/libllvmarmdesc/Makefile
index bd2e0cb..61679a6 100644
--- a/lib/clang/libllvmarmdesc/Makefile
+++ b/lib/clang/libllvmarmdesc/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmarmdesc
 
 SRCDIR=	lib/Target/ARM/MCTargetDesc
diff --git a/lib/clang/libllvmarmdisassembler/Makefile b/lib/clang/libllvmarmdisassembler/Makefile
index a3d16f7..eb46736 100644
--- a/lib/clang/libllvmarmdisassembler/Makefile
+++ b/lib/clang/libllvmarmdisassembler/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmarmdisassembler
 
 SRCDIR=	lib/Target/ARM/Disassembler
diff --git a/lib/clang/libllvmarminfo/Makefile b/lib/clang/libllvmarminfo/Makefile
index ec88367..1da434c 100644
--- a/lib/clang/libllvmarminfo/Makefile
+++ b/lib/clang/libllvmarminfo/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmarminfo
 
 SRCDIR=	lib/Target/ARM/TargetInfo
diff --git a/lib/clang/libllvmarminstprinter/Makefile b/lib/clang/libllvmarminstprinter/Makefile
index ca7e7d4..67b8fee 100644
--- a/lib/clang/libllvmarminstprinter/Makefile
+++ b/lib/clang/libllvmarminstprinter/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmarminstprinter
 
 SRCDIR=	lib/Target/ARM/InstPrinter
diff --git a/lib/clang/libllvmasmparser/Makefile b/lib/clang/libllvmasmparser/Makefile
index 8ceba69..bb8f0df 100644
--- a/lib/clang/libllvmasmparser/Makefile
+++ b/lib/clang/libllvmasmparser/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmasmparser
 
 SRCDIR=	lib/AsmParser
diff --git a/lib/clang/libllvmasmprinter/Makefile b/lib/clang/libllvmasmprinter/Makefile
index 36e626f..97bc73d3 100644
--- a/lib/clang/libllvmasmprinter/Makefile
+++ b/lib/clang/libllvmasmprinter/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmasmprinter
 
 SRCDIR=	lib/CodeGen/AsmPrinter
diff --git a/lib/clang/libllvmbitreader/Makefile b/lib/clang/libllvmbitreader/Makefile
index c426680..094e0e7 100644
--- a/lib/clang/libllvmbitreader/Makefile
+++ b/lib/clang/libllvmbitreader/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmbitreader
 
 SRCDIR=	lib/Bitcode/Reader
diff --git a/lib/clang/libllvmbitwriter/Makefile b/lib/clang/libllvmbitwriter/Makefile
index 3431b21..1cb9d41 100644
--- a/lib/clang/libllvmbitwriter/Makefile
+++ b/lib/clang/libllvmbitwriter/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmbitwriter
 
 SRCDIR=	lib/Bitcode/Writer
diff --git a/lib/clang/libllvmcodegen/Makefile b/lib/clang/libllvmcodegen/Makefile
index 8beb583..e65d19c 100644
--- a/lib/clang/libllvmcodegen/Makefile
+++ b/lib/clang/libllvmcodegen/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmcodegen
 
 SRCDIR=	lib/CodeGen
diff --git a/lib/clang/libllvmcore/Makefile b/lib/clang/libllvmcore/Makefile
index 5b597fe..d2010c0 100644
--- a/lib/clang/libllvmcore/Makefile
+++ b/lib/clang/libllvmcore/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmcore
 
 SRCDIR=	lib/VMCore
diff --git a/lib/clang/libllvmdebuginfo/Makefile b/lib/clang/libllvmdebuginfo/Makefile
index e12289b..757b909 100644
--- a/lib/clang/libllvmdebuginfo/Makefile
+++ b/lib/clang/libllvmdebuginfo/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmdebuginfo
 
 SRCDIR=	lib/DebugInfo
diff --git a/lib/clang/libllvmexecutionengine/Makefile b/lib/clang/libllvmexecutionengine/Makefile
index c0dde71..3ec39c7 100644
--- a/lib/clang/libllvmexecutionengine/Makefile
+++ b/lib/clang/libllvmexecutionengine/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmexecutionengine
 
 SRCDIR=	lib/ExecutionEngine
diff --git a/lib/clang/libllvminstcombine/Makefile b/lib/clang/libllvminstcombine/Makefile
index e989fa5..3752aef 100644
--- a/lib/clang/libllvminstcombine/Makefile
+++ b/lib/clang/libllvminstcombine/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvminstcombine
 
 SRCDIR=	lib/Transforms/InstCombine
diff --git a/lib/clang/libllvminstrumentation/Makefile b/lib/clang/libllvminstrumentation/Makefile
index ef09370..6d666e5 100644
--- a/lib/clang/libllvminstrumentation/Makefile
+++ b/lib/clang/libllvminstrumentation/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvminstrumentation
 
 SRCDIR=	lib/Transforms/Instrumentation
diff --git a/lib/clang/libllvminterpreter/Makefile b/lib/clang/libllvminterpreter/Makefile
index 6a1b453..06fcca4 100644
--- a/lib/clang/libllvminterpreter/Makefile
+++ b/lib/clang/libllvminterpreter/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvminterpreter
 
 SRCDIR=	lib/ExecutionEngine/Interpreter
diff --git a/lib/clang/libllvmjit/Makefile b/lib/clang/libllvmjit/Makefile
index ff852d5..682f704 100644
--- a/lib/clang/libllvmjit/Makefile
+++ b/lib/clang/libllvmjit/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmjit
 
 SRCDIR=	lib/ExecutionEngine/JIT
diff --git a/lib/clang/libllvmlinker/Makefile b/lib/clang/libllvmlinker/Makefile
index 73f153b..7275131 100644
--- a/lib/clang/libllvmlinker/Makefile
+++ b/lib/clang/libllvmlinker/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmlinker
 
 SRCDIR=	lib/Linker
diff --git a/lib/clang/libllvmmcdisassembler/Makefile b/lib/clang/libllvmmcdisassembler/Makefile
index dc5f294..94aff3e 100644
--- a/lib/clang/libllvmmcdisassembler/Makefile
+++ b/lib/clang/libllvmmcdisassembler/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmmcdisassembler
 
 SRCDIR=	lib/MC/MCDisassembler
diff --git a/lib/clang/libllvmmcparser/Makefile b/lib/clang/libllvmmcparser/Makefile
index 48deadf..b39926a 100644
--- a/lib/clang/libllvmmcparser/Makefile
+++ b/lib/clang/libllvmmcparser/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmmcparser
 
 SRCDIR=	lib/MC/MCParser
diff --git a/lib/clang/libllvmmipsasmparser/Makefile b/lib/clang/libllvmmipsasmparser/Makefile
index f5fb3c3..0572f8d 100644
--- a/lib/clang/libllvmmipsasmparser/Makefile
+++ b/lib/clang/libllvmmipsasmparser/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmmipsasmparser
 
 SRCDIR=	lib/Target/Mips/AsmParser
diff --git a/lib/clang/libllvmmipscodegen/Makefile b/lib/clang/libllvmmipscodegen/Makefile
index c33d954..7675a01 100644
--- a/lib/clang/libllvmmipscodegen/Makefile
+++ b/lib/clang/libllvmmipscodegen/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmmipscodegen
 
 SRCDIR=	lib/Target/Mips
diff --git a/lib/clang/libllvmmipsdesc/Makefile b/lib/clang/libllvmmipsdesc/Makefile
index 120b866..4ddb646 100644
--- a/lib/clang/libllvmmipsdesc/Makefile
+++ b/lib/clang/libllvmmipsdesc/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmmipsdesc
 
 SRCDIR=	lib/Target/Mips/MCTargetDesc
diff --git a/lib/clang/libllvmmipsdisassembler/Makefile b/lib/clang/libllvmmipsdisassembler/Makefile
index 7cdd982..1c0a4b6 100644
--- a/lib/clang/libllvmmipsdisassembler/Makefile
+++ b/lib/clang/libllvmmipsdisassembler/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmmipsdisassembler
 
 SRCDIR=	lib/Target/Mips/Disassembler
diff --git a/lib/clang/libllvmmipsinfo/Makefile b/lib/clang/libllvmmipsinfo/Makefile
index c5b96cd..c65c8c8 100644
--- a/lib/clang/libllvmmipsinfo/Makefile
+++ b/lib/clang/libllvmmipsinfo/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmmipsinfo
 
 SRCDIR=	lib/Target/Mips/TargetInfo
diff --git a/lib/clang/libllvmmipsinstprinter/Makefile b/lib/clang/libllvmmipsinstprinter/Makefile
index 36d3b04..39df7cc 100644
--- a/lib/clang/libllvmmipsinstprinter/Makefile
+++ b/lib/clang/libllvmmipsinstprinter/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmmipsinstprinter
 
 SRCDIR=	lib/Target/Mips/InstPrinter
diff --git a/lib/clang/libllvmobject/Makefile b/lib/clang/libllvmobject/Makefile
index d056dc7..c8c8068 100644
--- a/lib/clang/libllvmobject/Makefile
+++ b/lib/clang/libllvmobject/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmobject
 
 SRCDIR=	lib/Object
diff --git a/lib/clang/libllvmpowerpccodegen/Makefile b/lib/clang/libllvmpowerpccodegen/Makefile
index a97c352..a09da59 100644
--- a/lib/clang/libllvmpowerpccodegen/Makefile
+++ b/lib/clang/libllvmpowerpccodegen/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmpowerpccodegen
 
 SRCDIR=	lib/Target/PowerPC
diff --git a/lib/clang/libllvmpowerpcdesc/Makefile b/lib/clang/libllvmpowerpcdesc/Makefile
index 2a5ef6e..e48b484 100644
--- a/lib/clang/libllvmpowerpcdesc/Makefile
+++ b/lib/clang/libllvmpowerpcdesc/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmpowerpcdesc
 
 SRCDIR=	lib/Target/PowerPC/MCTargetDesc
diff --git a/lib/clang/libllvmpowerpcinfo/Makefile b/lib/clang/libllvmpowerpcinfo/Makefile
index 8310378..03bba0e 100644
--- a/lib/clang/libllvmpowerpcinfo/Makefile
+++ b/lib/clang/libllvmpowerpcinfo/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmpowerpcinfo
 
 SRCDIR=	lib/Target/PowerPC/TargetInfo
diff --git a/lib/clang/libllvmpowerpcinstprinter/Makefile b/lib/clang/libllvmpowerpcinstprinter/Makefile
index c1528ae..ac98321 100644
--- a/lib/clang/libllvmpowerpcinstprinter/Makefile
+++ b/lib/clang/libllvmpowerpcinstprinter/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmpowerpcinstprinter
 
 SRCDIR=	lib/Target/PowerPC/InstPrinter
diff --git a/lib/clang/libllvmruntimedyld/Makefile b/lib/clang/libllvmruntimedyld/Makefile
index d5d5447..02ca4bd 100644
--- a/lib/clang/libllvmruntimedyld/Makefile
+++ b/lib/clang/libllvmruntimedyld/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmruntimedyld
 
 SRCDIR=	lib/ExecutionEngine/RuntimeDyld
diff --git a/lib/clang/libllvmselectiondag/Makefile b/lib/clang/libllvmselectiondag/Makefile
index e6950127..7543102 100644
--- a/lib/clang/libllvmselectiondag/Makefile
+++ b/lib/clang/libllvmselectiondag/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmselectiondag
 
 SRCDIR=	lib/CodeGen/SelectionDAG
diff --git a/lib/clang/libllvmtablegen/Makefile b/lib/clang/libllvmtablegen/Makefile
index 8dffad0..069cec0 100644
--- a/lib/clang/libllvmtablegen/Makefile
+++ b/lib/clang/libllvmtablegen/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmtablegen
 
 SRCDIR=	lib/TableGen
diff --git a/lib/clang/libllvmtarget/Makefile b/lib/clang/libllvmtarget/Makefile
index b82377e..595d2cc 100644
--- a/lib/clang/libllvmtarget/Makefile
+++ b/lib/clang/libllvmtarget/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmtarget
 
 SRCDIR=	lib/Target
diff --git a/lib/clang/libllvmvectorize/Makefile b/lib/clang/libllvmvectorize/Makefile
index 444d672..d0f5b73 100644
--- a/lib/clang/libllvmvectorize/Makefile
+++ b/lib/clang/libllvmvectorize/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmvectorize
 
 SRCDIR=	lib/Transforms/Vectorize
diff --git a/lib/clang/libllvmx86asmparser/Makefile b/lib/clang/libllvmx86asmparser/Makefile
index c140268..252b2d0 100644
--- a/lib/clang/libllvmx86asmparser/Makefile
+++ b/lib/clang/libllvmx86asmparser/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmx86asmparser
 
 SRCDIR=	lib/Target/X86/AsmParser
diff --git a/lib/clang/libllvmx86codegen/Makefile b/lib/clang/libllvmx86codegen/Makefile
index bcc7aa0..939b257 100644
--- a/lib/clang/libllvmx86codegen/Makefile
+++ b/lib/clang/libllvmx86codegen/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmx86codegen
 
 SRCDIR=	lib/Target/X86
diff --git a/lib/clang/libllvmx86desc/Makefile b/lib/clang/libllvmx86desc/Makefile
index 3d2a9dd..a177129 100644
--- a/lib/clang/libllvmx86desc/Makefile
+++ b/lib/clang/libllvmx86desc/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmx86desc
 
 SRCDIR=	lib/Target/X86/MCTargetDesc
diff --git a/lib/clang/libllvmx86info/Makefile b/lib/clang/libllvmx86info/Makefile
index 85155a8..bc1a7d7 100644
--- a/lib/clang/libllvmx86info/Makefile
+++ b/lib/clang/libllvmx86info/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmx86info
 
 SRCDIR=	lib/Target/X86/TargetInfo
diff --git a/lib/clang/libllvmx86instprinter/Makefile b/lib/clang/libllvmx86instprinter/Makefile
index a44006c..308f91c 100644
--- a/lib/clang/libllvmx86instprinter/Makefile
+++ b/lib/clang/libllvmx86instprinter/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmx86instprinter
 
 SRCDIR=	lib/Target/X86/InstPrinter
diff --git a/lib/clang/libllvmx86utils/Makefile b/lib/clang/libllvmx86utils/Makefile
index bbed6b0..8355e7e 100644
--- a/lib/clang/libllvmx86utils/Makefile
+++ b/lib/clang/libllvmx86utils/Makefile
@@ -1,5 +1,7 @@
 # $FreeBSD$
 
+.include <bsd.own.mk>
+
 LIB=	llvmx86utils
 
 SRCDIR=	lib/Target/X86/Utils
-- 
cgit v1.1


From 69b6feceb8009d6022dae0a5b0b37328974bb185 Mon Sep 17 00:00:00 2001
From: dim <dim@FreeBSD.org>
Date: Sun, 3 Feb 2013 20:40:41 +0000
Subject: Add several missing symbols to libcxxrt's symbol version map, and
 remove a few duplicates.  This should fix building world with -stdlib=libc++
 after r246028.

Submitted by:	Yamaya Takashi <yamayan@kbh.biglobe.ne.jp>
MFC after:	1 week
X-MFC-With:	r246028
---
 lib/libcxxrt/Version.map | 26 ++++++++++++--------------
 1 file changed, 12 insertions(+), 14 deletions(-)

(limited to 'lib')

diff --git a/lib/libcxxrt/Version.map b/lib/libcxxrt/Version.map
index 1fdda27..f7e12e1 100644
--- a/lib/libcxxrt/Version.map
+++ b/lib/libcxxrt/Version.map
@@ -208,7 +208,6 @@ CXXABI_1.3 {
         "typeinfo name for __cxxabiv1::__vmi_class_type_info";
 
         "std::type_info::type_info(std::type_info const&)";
-        "std::type_info::type_info(std::type_info const&)";
         "std::type_info::operator=(std::type_info const&)";
 
 
@@ -238,18 +237,16 @@ CXXRT_1.0 {
         "std::type_info::operator!=(std::type_info const&) const";
         "std::bad_cast::bad_cast(std::bad_cast const&)";
         "std::bad_cast::bad_cast()";
-        "std::bad_cast::bad_cast(std::bad_cast const&)";
-        "std::bad_cast::bad_cast()";
         "std::bad_cast::operator=(std::bad_cast const&)";
-        "std::exception::exception(std::exception const&)";
-        "std::exception::exception()";
+        "std::bad_typeid::bad_typeid(std::bad_typeid const&)";
+        "std::bad_typeid::bad_typeid()";
+        "std::bad_typeid::operator=(std::bad_typeid const&)";
         "std::exception::exception(std::exception const&)";
         "std::exception::exception()";
         "std::exception::operator=(std::exception const&)";
-
-
-
-
+        "std::bad_alloc::bad_alloc(std::bad_alloc const&)";
+        "std::bad_alloc::bad_alloc()";
+        "std::bad_alloc::operator=(std::bad_alloc const&)";
 
     };
     __cxa_allocate_dependent_exception;
@@ -281,15 +278,16 @@ GLIBCXX_3.4 {
 
         "std::type_info::~type_info()";
         "std::bad_cast::~bad_cast()";
+        "std::bad_typeid::~bad_typeid()";
         "std::exception::~exception()";
+        "std::bad_alloc::~bad_alloc()";
+
+        "std::exception::what() const";
 
         std::set_new_handler*;
         std::set_terminate*;
         std::set_unexpected*;
-        std::exception*;
-        std::bad_alloc;
-        std::bad_typeid;
-        std::type_info*;
+        std::type_info::__*;
 
         "vtable for std::bad_alloc";
         "vtable for std::bad_cast";
@@ -299,10 +297,10 @@ GLIBCXX_3.4 {
 
         "typeinfo for std::bad_alloc";
         "typeinfo for std::bad_typeid";
-        "typeinfo for std::exception";
         "typeinfo for std::bad_cast";
         "typeinfo for std::exception";
         "typeinfo for std::type_info";
+        "typeinfo name for std::bad_alloc";
         "typeinfo name for std::bad_typeid";
         "typeinfo name for std::bad_cast";
         "typeinfo name for std::exception";
-- 
cgit v1.1


From 4276b11f079dc95b96111a7a20af3feeb9716e62 Mon Sep 17 00:00:00 2001
From: des <des@FreeBSD.org>
Date: Mon, 4 Feb 2013 19:05:53 +0000
Subject: Sort by MK_* knob like the comment says

MFC after:	1 week
---
 lib/Makefile | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

(limited to 'lib')

diff --git a/lib/Makefile b/lib/Makefile
index 132302e..1447002 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -177,6 +177,11 @@ _libiconv_modules=	libiconv_modules
 _libipx=	libipx
 .endif
 
+.if ${MK_LIBCPLUSPLUS} != "no"
+_libcxxrt=	libcxxrt
+_libcplusplus=	libc++
+.endif
+
 .if ${MK_LIBTHR} != "no"
 _libthr=	libthr
 .endif
@@ -221,11 +226,6 @@ _librtld_db=	librtld_db
 _libmp=		libmp
 .endif
 
-.if ${MK_LIBCPLUSPLUS} != "no"
-_libcxxrt=	libcxxrt
-_libcplusplus=	libc++
-.endif
-
 .if ${MK_PMC} != "no"
 _libpmc=	libpmc
 .endif
-- 
cgit v1.1


From 18658720df177c8047dfc7d4f19be4eb9553b0c1 Mon Sep 17 00:00:00 2001
From: andrew <andrew@FreeBSD.org>
Date: Tue, 5 Feb 2013 20:03:58 +0000
Subject: * Add the integer div & mod functions and ARM EABI support functions
 to   libstand. * Stop linking the ARM U-Boot loader against libgcc now
 libstand has the   required symbols.

---
 lib/libstand/Makefile | 13 +++++++++++++
 1 file changed, 13 insertions(+)

(limited to 'lib')

diff --git a/lib/libstand/Makefile b/lib/libstand/Makefile
index fd720a9..807136d 100644
--- a/lib/libstand/Makefile
+++ b/lib/libstand/Makefile
@@ -61,7 +61,20 @@ SRCS+=	bcmp.c bcopy.c bzero.c ffs.c memccpy.c memchr.c memcmp.c memcpy.c \
 .endif
 .if ${MACHINE_CPUARCH} == "arm"
 .PATH: ${.CURDIR}/../libc/arm/gen
+
+.if ${MK_ARM_EABI} == "no"
 SRCS+= divsi3.S
+.else
+# Compiler support functions
+.PATH: ${.CURDIR}/../../contrib/compiler-rt/lib/
+SRCS+=	 divmoddi4.c  divmodsi4.c  divdi3.c  divsi3.c  moddi3.c  modsi3.c
+SRCS+=	udivmoddi4.c udivmodsi4.c udivdi3.c udivsi3.c umoddi3.c umodsi3.c
+
+.PATH: ${.CURDIR}/../../contrib/compiler-rt/lib/arm/
+SRCS+=	aeabi_idivmod.S aeabi_ldivmod.S aeabi_uidivmod.S aeabi_uldivmod.S
+SRCS+=	aeabi_memcmp.S aeabi_memcpy.S aeabi_memmove.S aeabi_memset.S
+.endif
+
 .endif
 .if ${MACHINE_CPUARCH} == "ia64"
 .PATH: ${.CURDIR}/../libc/ia64/string
-- 
cgit v1.1


From 34f6d259a30c45a17229a6742c75eaf1a57452f6 Mon Sep 17 00:00:00 2001
From: andrew <andrew@FreeBSD.org>
Date: Wed, 6 Feb 2013 00:01:28 +0000
Subject: Add the __aeabi_mem* functions to compiler-rt as clang uses them.

---
 lib/libcompiler_rt/Makefile | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'lib')

diff --git a/lib/libcompiler_rt/Makefile b/lib/libcompiler_rt/Makefile
index 26ce4f7a..a9daada 100644
--- a/lib/libcompiler_rt/Makefile
+++ b/lib/libcompiler_rt/Makefile
@@ -188,6 +188,10 @@ SRCS+=	${file}.c
 .if ${MACHINE_CPUARCH} == "arm" && ${MK_ARM_EABI} != "no"
 SRCS+=	aeabi_idivmod.S \
 	aeabi_ldivmod.S \
+	aeabi_memcmp.S \
+	aeabi_memcpy.S \
+	aeabi_memmove.S \
+	aeabi_memset.S \
 	aeabi_uidivmod.S \
 	aeabi_uldivmod.S
 .endif
-- 
cgit v1.1